fluent-plugin-mysql-replicator 0.0.2 → 0.1.0

data/README.md

@@ -14,7 +14,15 @@ gem install fluent-plugin-mysql-replicator
 /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-mysql-replicator
 `````
 
-## Tutorial
+## Tutorial for Quickstart
+
+It is useful for these purpose.
+
+* try it on this plugin.
+* replicate small record under a millons table.
+
+**Note:**  
+On syncing 300 million rows table, it will consume around 800MB of memory with ruby 1.9.3 environment.
 
 #### configuration
 
@@ -57,9 +65,72 @@ $ tail -f /var/log/td-agent/td-agent.log
 2013-11-25 18:22:45 +0900 replicator.delete: {"id":"1"}
 `````
 
-## Performance
+## Tutorial for Production
+
+It is very useful to replicate a millions of records and/or multiple tables with multiple threads.  
+This architecture is storing hash table in mysql management table instead of ruby internal memory.  
+
+**Note:**  
+On syncing 300 million rows table, it will consume around 20MB of memory with ruby 1.9.3 environment.
+
+#### prepare
+
+* create database and tables.
+* add replicator configuration.
+
+```
+$ cat setup_mysql_replicator_multi.sql
+CREATE DATABASE replicator_manager;
+USE replicator_manager;
+
+CREATE TABLE `hash_tables` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `setting_name` varchar(255) NOT NULL,
+  `setting_query_pk` int(11) NOT NULL,
+  `setting_query_hash` varchar(255) NOT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `setting_query_pk` (`setting_query_pk`,`setting_name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+CREATE TABLE `settings` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `name` varchar(255) NOT NULL,
+  `host` varchar(255) NOT NULL DEFAULT 'localhost',
+  `port` int(11) NOT NULL DEFAULT '3306',
+  `username` varchar(255) NOT NULL,
+  `password` varchar(255) NOT NULL,
+  `database` varchar(255) NOT NULL,
+  `query` TEXT NOT NULL,
+  `interval` int(11) NOT NULL,
+  `tag` varchar(255) NOT NULL,
+  `primary_key` varchar(11) DEFAULT 'id',
+  `enable_delete` int(11) DEFAULT '1',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+```
+
+```
+$ mysql
+mysql> source /path/to/setup_mysql_replicator_multi.sql
+mysql> insert into source ...snip...;
+```
 
-On syncing 300 million rows table, it will consume around 800MB of memory with ruby 1.9.3 environment.
+#### configuration
+
+`````
+<source>
+  type mysql_replicator_multi
+  manager_host localhost
+  manager_username your_mysql_user
+  manager_password your_mysql_password
+  manager_database replicator_manager
+</source>
+
+<match replicator.*>
+  type stdout
+</match>
+`````
 
 ## TODO
 

data/fluent-plugin-mysql-replicator.gemspec

@@ -1,11 +1,11 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-mysql-replicator"
-  s.version     = "0.0.2"
+  s.version     = "0.1.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-mysql-replicator"
-  s.summary     = %q{Fluentd input plugin to track insert/update/delete event from MySQL database server.}
+  s.summary     = %q{Fluentd input plugin to track insert/update/delete event from MySQL databases. It will support multiple table replication to another RDB/noSQL like Elasticsearch.}
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/fluent/plugin/in_mysql_replicator.rb

@@ -18,6 +18,7 @@ module Fluent
     config_param :tag, :string
     config_param :query, :string
     config_param :primary_key, :string, :default => 'id'
+    config_param :enable_delete, :bool, :default => 'yes'
 
     def configure(conf)
       super
@@ -59,10 +60,12 @@ module Fluent
           table_hash[row[@primary_key]] = current_hash
         end
         ids = current_ids
-        deleted_ids = previous_ids - current_ids
-        if deleted_ids.count > 0
-          hash_delete_by_list(table_hash, deleted_ids)
-          deleted_ids.each {|id| emit_record(:delete, {@primary_key => id})}
+        unless @enable_delete
+          deleted_ids = previous_ids - current_ids
+          if deleted_ids.count > 0
+            hash_delete_by_list(table_hash, deleted_ids)
+            deleted_ids.each {|id| emit_record(:delete, {@primary_key => id})}
+          end
         end
         sleep @interval
       end

data/lib/fluent/plugin/in_mysql_replicator_multi.rb

@@ -0,0 +1,201 @@
+module Fluent
+  class MysqlReplicatorMultiInput < Fluent::Input
+    Plugin.register_input('mysql_replicator_multi', self)
+
+    def initialize
+      require 'mysql2'
+      require 'digest/sha1'
+      super
+    end
+
+    config_param :manager_host, :string, :default => 'localhost'
+    config_param :manager_port, :integer, :default => 3306
+    config_param :manager_username, :string, :default => nil
+    config_param :manager_password, :string, :default => ''
+    config_param :manager_database, :string, :default => 'replicator_manager'
+
+    def configure(conf)
+      super
+      @reconnect_interval = Config.time_value('10sec')
+    end
+
+    def start
+      begin
+        @threads = []
+        @mutex = Mutex.new
+        get_settings.each do |config|
+          @threads << Thread.new {
+            poll(config)
+          }
+        end
+        $log.error "mysql_replicator_multi: stop working due to empty configuration" if @threads.empty?
+      rescue StandardError => e
+        $log.error "error: #{e.message}"
+        $log.error e.backtrace.join("\n")
+      end
+    end
+
+    def shutdown
+      @threads.each do |thread|
+        Thread.kill(thread)
+      end
+    end
+
+    def get_settings
+      manager_db = get_manager_connection
+      settings = []
+      query = "SELECT * FROM settings"
+      manager_db.query(query).each do |row|
+        settings << row
+      end
+      return settings
+    end
+
+    def poll(config)
+      begin
+        @manager_db = get_manager_connection
+        masked_config = config.map {|k,v| (k == 'password') ? v.to_s.gsub(/./, '*') : v}
+        @mutex.synchronize {
+          $log.info "mysql_replicator_multi: polling start. :config=>#{masked_config}"
+        }
+        primary_key = config['primary_key']
+        previous_id = current_id = 0
+        loop do
+          db = get_origin_connection(config)
+          db.query(config['query']).each do |row|
+            @mutex.lock
+            current_id = row[primary_key]
+            detect_insert_update(config, row)
+            detect_delete(config, current_id, previous_id)
+            previous_id = current_id
+            @mutex.unlock
+          end
+          db.close
+          sleep config['interval']
+        end
+      rescue StandardError => e
+        $log.error "error: #{e.message}"
+        $log.error e.backtrace.join("\n")
+      end
+    end
+
+    def detect_insert_update(config, row)
+      primary_key = config['primary_key']
+      current_id = row[primary_key]
+      stored_hash = get_stored_hash(config['name'], current_id)
+      current_hash = Digest::SHA1.hexdigest(row.flatten.join)
+
+      event = nil
+      if stored_hash.empty?
+        event = :insert
+      elsif stored_hash != current_hash
+        event = :update
+      end
+      unless event.nil?
+        emit_record("#{config['tag']}.#{event.to_s}", row)
+        update_hashtable({:event => event, :ids => current_id, :setting_name => config['name'], :hash => current_hash})
+      end
+    end
+
+    def get_stored_hash(setting_name, id)
+      query = "SELECT setting_query_hash FROM hash_tables WHERE setting_query_pk = #{id.to_i} AND setting_name = '#{setting_name}'"
+      @manager_db.query(query).each do |row|
+        return row['setting_query_hash']
+      end
+    end
+
+    def detect_delete(config, current_id, previous_id)
+      return unless config['enable_delete'] == 1
+      deleted_ids = collect_gap_ids(config['name'], current_id, previous_id)
+      unless deleted_ids.empty?
+        event = :delete
+        deleted_ids.each do |id|
+          emit_record("#{config['tag']}.#{event.to_s}", {config['primary_key'] => id})
+        end
+        update_hashtable({:event =>  event, :ids => deleted_ids, :setting_name => config['name']})
+      end
+    end
+
+    def collect_gap_ids(setting_name, current_id, previous_id)
+      if (current_id - previous_id) > 1
+        query = "SELECT setting_query_pk FROM hash_tables
+          WHERE setting_name = '#{setting_name}' 
+          AND setting_query_pk > #{previous_id.to_i} AND setting_query_pk < #{current_id.to_i}"
+      elsif previous_id > current_id
+        query = "SELECT setting_query_pk FROM hash_tables
+          WHERE setting_name = '#{setting_name}' 
+          AND setting_query_pk > #{previous_id.to_i}"
+      elsif previous_id == current_id
+        query = "SELECT setting_query_pk FROM hash_tables
+          WHERE setting_name = '#{setting_name}' 
+          AND (setting_query_pk > #{current_id.to_i} OR setting_query_pk < #{current_id.to_i})"
+      end
+      ids = Array.new
+      unless query.nil?
+        @manager_db.query(query).each do |row|
+          ids << row['setting_query_pk']
+        end
+      end
+      return ids
+    end
+
+    def update_hashtable(opts)
+      ids = opts[:ids].is_a?(Integer) ? [opts[:ids]] : opts[:ids]
+      ids.each do |id|
+        case opts[:event]
+        when :insert
+          query = "insert into hash_tables (setting_name,setting_query_pk,setting_query_hash) values('#{opts[:setting_name]}','#{id}','#{opts[:hash]}')"
+        when :update
+          query = "update hash_tables set setting_query_hash = '#{opts[:hash]}' WHERE setting_name = '#{opts[:setting_name]}' AND setting_query_pk = '#{id}'"
+        when :delete
+          query = "delete from hash_tables WHERE setting_name = '#{opts[:setting_name]}' AND setting_query_pk = '#{id}'"
+        end
+        @manager_db.query(query) unless query.nil?
+      end
+    end
+
+    def emit_record(tag, record)
+      Engine.emit(tag, Engine.now, record)
+    end
+
+    def get_manager_connection
+      begin
+        return Mysql2::Client.new(
+          :host => @manager_host,
+          :port => @manager_port,
+          :username => @manager_username,
+          :password => @manager_password,
+          :database => @manager_database,
+          :encoding => 'utf8',
+          :reconnect => true,
+          :stream => false,
+          :cache_rows => false
+        )
+      rescue Exception => e
+        $log.warn "mysql_replicator_multi: #{e}"
+        sleep @reconnect_interval
+        retry
+      end
+    end
+
+    def get_origin_connection(config)
+      begin
+        return Mysql2::Client.new(
+          :host => config['host'],
+          :port => config['manager_port'],
+          :username => config['username'],
+          :password => config['password'],
+          :database => config['database'],
+          :encoding => 'utf8',
+          :reconnect => true,
+          :stream => true,
+          :cache_rows => false
+        )
+      rescue Exception => e
+        $log.warn "mysql_replicator_multi: #{e}"
+        sleep @reconnect_interval
+        retry
+      end
+    end
+  end
+end

data/setup_mysql_replicator_multi.sql

@@ -0,0 +1,28 @@
+CREATE DATABASE replicator_manager;
+USE replicator_manager;
+
+CREATE TABLE `hash_tables` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `setting_name` varchar(255) NOT NULL,
+  `setting_query_pk` int(11) NOT NULL,
+  `setting_query_hash` varchar(255) NOT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `setting_query_pk` (`setting_query_pk`,`setting_name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+CREATE TABLE `settings` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `name` varchar(255) NOT NULL,
+  `host` varchar(255) NOT NULL DEFAULT 'localhost',
+  `port` int(11) NOT NULL DEFAULT '3306',
+  `username` varchar(255) NOT NULL,
+  `password` varchar(255) NOT NULL,
+  `database` varchar(255) NOT NULL,
+  `query` TEXT NOT NULL,
+  `interval` int(11) NOT NULL,
+  `tag` varchar(255) NOT NULL,
+  `primary_key` varchar(11) DEFAULT 'id',
+  `enable_delete` int(11) DEFAULT '1',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;

data/test/helper.rb

@@ -23,6 +23,7 @@ unless ENV.has_key?('VERBOSE')
 end
 
 require 'fluent/plugin/in_mysql_replicator'
+require 'fluent/plugin/in_mysql_replicator_multi'
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_in_mysql_replicator.rb

@@ -28,11 +28,13 @@ class MysqlReplicatorInputTest < Test::Unit::TestCase
       interval        30
       tag             input.mysql
       query           SELECT id, text from search_text
+      enable_delete   no
     ]
     d.instance.inspect
     assert_equal 'localhost', d.instance.host
     assert_equal 3306, d.instance.port
     assert_equal 30, d.instance.interval
     assert_equal 'input.mysql', d.instance.tag
+    assert_equal false, d.instance.enable_delete
   end
 end

data/test/plugin/test_in_mysql_replicator_multi.rb

@@ -0,0 +1,26 @@
+require 'helper'
+
+class MysqlReplicatorMultiInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    manager_host      localhost
+    manager_port      3306
+    manager_username  foo
+    manager_password  bar
+  ]
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::MysqlReplicatorMultiInput, tag).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver(CONFIG)
+    d.instance.inspect
+    assert_equal 'localhost', d.instance.manager_host
+    assert_equal 3306, d.instance.manager_port
+    assert_equal 'replicator_manager', d.instance.manager_database
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-replicator
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-06 00:00:00.000000000 Z
+date: 2013-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -74,8 +74,11 @@ files:
 - Rakefile
 - fluent-plugin-mysql-replicator.gemspec
 - lib/fluent/plugin/in_mysql_replicator.rb
+- lib/fluent/plugin/in_mysql_replicator_multi.rb
+- setup_mysql_replicator_multi.sql
 - test/helper.rb
 - test/plugin/test_in_mysql_replicator.rb
+- test/plugin/test_in_mysql_replicator_multi.rb
 homepage: https://github.com/y-ken/fluent-plugin-mysql-replicator
 licenses: []
 post_install_message: 
@@ -99,8 +102,9 @@ rubyforge_project:
 rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
-summary: Fluentd input plugin to track insert/update/delete event from MySQL database
-  server.
+summary: Fluentd input plugin to track insert/update/delete event from MySQL databases.
+  It will support multiple table replication to another RDB/noSQL like Elasticsearch.
 test_files:
 - test/helper.rb
 - test/plugin/test_in_mysql_replicator.rb
+- test/plugin/test_in_mysql_replicator_multi.rb