fluent-plugin-mysql-replicator 0.2.3 → 0.3.0

data/README.md

@@ -3,7 +3,7 @@
 ## Overview
 
 Fluentd input plugin to track insert/update/delete event from MySQL database server.  
-Not only that, it could multiple table replication into Elasticsearch nodes.  
+Not only that, it could multiple table replication into Elasticsearch/Solr.  
 It's comming support replicate to another RDB/noSQL.
 
 ## Installation
@@ -162,6 +162,8 @@ CREATE TABLE `settings` (
   `interval` int(11) NOT NULL,
   `primary_key` varchar(255) DEFAULT 'id',
   `enable_delete` int(11) DEFAULT '1',
+  `enable_loose_insert` int(11) DEFAULT '0',
+  `enable_loose_delete` int(11) DEFAULT '0',
   PRIMARY KEY (`id`),
   UNIQUE KEY `name` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
@@ -184,9 +186,9 @@ VALUES
 
 it is a sample which you have inserted row.
 
-| id | is_active |     name     |      host       | port | username  |   password    | database |            query             | interval | primary_key | enable_delete |
+| id | is_active |     name     |      host       | port | username  |   password    | database |            query             | interval | primary_key | enable_delete |enable_loose_insert |enable_loose_delete |
 |----|-----------|--------------|-----------------|------|-----------|---------------|----------|------------------------------|----------|-------------|---------------|
-|  1 |         1 | mydb.mytable | 192.168.100.221 | 3306 | mysqluser | mysqlpassword | mydb     | SELECT id, text from mytable; |       5 | id          |             1 |
+|  1 |         1 | mydb.mytable | 192.168.100.221 | 3306 | mysqluser | mysqlpassword | mydb     | SELECT id, text from mytable; |       5 | id          |             1 |             0 |             0 |
 
 ### configuration
 

data/fluent-plugin-mysql-replicator.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-mysql-replicator"
-  s.version     = "0.2.3"
+  s.version     = "0.3.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-mysql-replicator"

data/lib/fluent/plugin/in_mysql_replicator.rb

@@ -53,6 +53,7 @@ module Fluent
       table_hash = Hash.new
       ids = Array.new
       loop do
+        rows_count = 0
         start_time = Time.now
         previous_ids = ids
         current_ids = Array.new
@@ -72,6 +73,7 @@ module Fluent
             emit_record(tag, row)
           end
           table_hash[row[@primary_key]] = current_hash
+          rows_count += 1
         end
         ids = current_ids
         if @enable_delete
@@ -89,7 +91,7 @@ module Fluent
           end
         end
         elapsed_time = sprintf("%0.02f", Time.now - start_time)
-        $log.info "mysql_replicator: finished execution :tag=>#{tag} :elapsed_time=>#{elapsed_time} sec"
+        $log.info "mysql_replicator: finished execution :tag=>#{tag} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
         sleep @interval
       end
     end
@@ -100,7 +102,7 @@ module Fluent
 
     def format_tag(tag, param)
       pattern = {'${event}' => param[:event].to_s, '${primary_key}' => @primary_key}
-      tag.gsub(/\${[a-z_]+(\[[0-9]+\])?}/, pattern) do
+      tag.gsub(/(\${[a-z_]+})/) do
         $log.warn "mysql_replicator: missing placeholder. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
         pattern[$1]
       end

data/lib/fluent/plugin/in_mysql_replicator_multi.rb

@@ -15,6 +15,9 @@ module Fluent
     config_param :manager_database, :string, :default => 'replicator_manager'
     config_param :tag, :string, :default => nil
 
+    config_param :bulk_insert_count, :integer, :default => 50
+    config_param :bulk_insert_timeout, :integer, :default => 10
+
     def configure(conf)
       super
       @reconnect_interval = Config.time_value('10sec')
@@ -27,6 +30,13 @@ module Fluent
       begin
         @threads = []
         @mutex = Mutex.new
+        @manager_db = get_manager_connection
+        @manager_db.query("SET SESSION wait_timeout=1800;")
+        @threads << Thread.new {
+          @hash_table_bulk_insert = []
+          @hash_table_bulk_insert_last_time = Time.now
+          hash_table_flusher
+        }
         get_settings.each do |config|
           @threads << Thread.new {
             poll(config)
@@ -57,33 +67,35 @@ module Fluent
 
     def poll(config)
       begin
-        @manager_db = get_manager_connection
         masked_config = Hash[config.map {|k,v| (k == 'password') ? [k, v.to_s.gsub(/./, '*')] : [k,v]}]
         @mutex.synchronize {
           $log.info "mysql_replicator_multi: polling start. :config=>#{masked_config}"
         }
         primary_key = config['primary_key']
-        previous_id = current_id = 0
+        previous_id = current_id = nil
         loop do
+          rows_count = 0
           start_time = Time.now
           db = get_origin_connection(config)
           db.query(config['query']).each do |row|
-            @mutex.lock
             row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date)}
             current_id = row[primary_key]
-            if row[primary_key].nil?
-              $log.error "mysql_replicator_multi: missing primary_key. :setting_name=>#{config['name']} :primary_key=>#{primary_key}"
-              @mutex.unlock
-              break
-            end
-            detect_insert_update(config, row)
-            detect_delete(config, current_id, previous_id)
+            @mutex.synchronize {
+              if row[primary_key].nil?
+                $log.error "mysql_replicator_multi: missing primary_key. :setting_name=>#{config['name']} :primary_key=>#{primary_key}"
+                break
+              end
+              detect_insert_update(config, row)
+              detect_delete(config, current_id, previous_id)
+            }
             previous_id = current_id
-            @mutex.unlock
+            rows_count += 1
           end
           db.close
           elapsed_time = sprintf("%0.02f", Time.now - start_time)
-          $log.info "mysql_replicator_multi: finished execution :setting_name=>#{config['name']} :elapsed_time=>#{elapsed_time} sec"
+          @mutex.synchronize {
+            $log.info "mysql_replicator_multi: execution finished. :setting_name=>#{config['name']} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
+          }
           sleep config['interval']
         end
       rescue StandardError => e
@@ -98,7 +110,7 @@ module Fluent
     def detect_insert_update(config, row)
       primary_key = config['primary_key']
       current_id = row[primary_key]
-      stored_hash = get_stored_hash(config['name'], current_id)
+      stored_hash = config['enable_loose_insert'] == 1 ? "" : get_stored_hash(config['name'], current_id)
       current_hash = Digest::SHA1.hexdigest(row.flatten.join)
 
       event = nil
@@ -110,20 +122,19 @@ module Fluent
       unless event.nil?
         tag = format_tag(@tag, {:name => config['name'], :event => event, :primary_key => config['primary_key']})
         emit_record(tag, row)
-        update_hashtable({:event => event, :ids => current_id, :setting_name => config['name'], :hash => current_hash})
+        update_hashtable({:event => event, :id => current_id, :setting_name => config['name'], :hash => current_hash})
       end
     end
 
     def get_stored_hash(setting_name, id)
-      query = "SELECT setting_query_hash FROM hash_tables WHERE setting_query_pk = #{id.to_i} AND setting_name = '#{setting_name}'"
-      @manager_db.query(query).each do |row|
-        return row['setting_query_hash']
-      end
+      query = "SELECT SQL_NO_CACHE setting_query_hash FROM hash_tables WHERE setting_query_pk = #{id.to_i} AND setting_name = '#{setting_name}'"
+      row = @manager_db.query(query).first
+      return row.nil? ? "" : row['setting_query_hash']
     end
 
     def detect_delete(config, current_id, previous_id)
-      return unless config['enable_delete'] == 1
-      deleted_ids = collect_gap_ids(config['name'], current_id, previous_id)
+      return if config['enable_delete'] != 1 || previous_id.nil?
+      deleted_ids = collect_gap_ids(config, current_id, previous_id)
       unless deleted_ids.empty?
         event = :delete
         deleted_ids.each do |id|
@@ -134,17 +145,20 @@ module Fluent
       end
     end
 
-    def collect_gap_ids(setting_name, current_id, previous_id)
-      if (current_id - previous_id) > 1
-        query = "SELECT setting_query_pk FROM hash_tables
+    def collect_gap_ids(config, current_id, previous_id)
+      setting_name = config['name']
+      if (current_id - previous_id) > 1 && config['enable_loose_delete'] == 0
+        query = "SELECT SQL_NO_CACHE setting_query_pk FROM hash_tables
           WHERE setting_name = '#{setting_name}' 
           AND setting_query_pk > #{previous_id.to_i} AND setting_query_pk < #{current_id.to_i}"
+      elsif (current_id - previous_id) > 1 && config['enable_loose_delete'] == 1
+        return [*previous_id...current_id] - [current_id,previous_id]
       elsif previous_id > current_id
-        query = "SELECT setting_query_pk FROM hash_tables
+        query = "SELECT SQL_NO_CACHE setting_query_pk FROM hash_tables
           WHERE setting_name = '#{setting_name}' 
           AND setting_query_pk > #{previous_id.to_i}"
       elsif previous_id == current_id
-        query = "SELECT setting_query_pk FROM hash_tables
+        query = "SELECT SQL_NO_CACHE setting_query_pk FROM hash_tables
           WHERE setting_name = '#{setting_name}' 
           AND (setting_query_pk > #{current_id.to_i} OR setting_query_pk < #{current_id.to_i})"
       end
@@ -158,28 +172,60 @@ module Fluent
     end
 
     def update_hashtable(opts)
-      ids = opts[:ids].is_a?(Integer) ? [opts[:ids]] : opts[:ids]
-      ids.each do |id|
-        case opts[:event]
-        when :insert
-          query = "insert into hash_tables (setting_name,setting_query_pk,setting_query_hash) values('#{opts[:setting_name]}','#{id}','#{opts[:hash]}')"
-        when :update
-          query = "update hash_tables set setting_query_hash = '#{opts[:hash]}' WHERE setting_name = '#{opts[:setting_name]}' AND setting_query_pk = '#{id}'"
-        when :delete
-          query = "delete from hash_tables WHERE setting_name = '#{opts[:setting_name]}' AND setting_query_pk = '#{id}'"
-        end
-        @manager_db.query(query) unless query.nil?
+      case opts[:event]
+      when :insert
+        add_hash_table_buffer(opts[:setting_name], opts[:id], opts[:hash])
+      when :update
+        query = "UPDATE hash_tables SET setting_query_hash = '#{opts[:hash]}' WHERE setting_name = '#{opts[:setting_name]}' AND setting_query_pk = '#{opts[:id]}'"
+      when :delete
+        query = "DELETE FROM hash_tables WHERE setting_name = '#{opts[:setting_name]}' AND setting_query_pk IN(#{opts[:ids].join(',')})"
       end
+      @manager_db.query(query) unless query.nil?
     end
 
     def format_tag(tag, param)
       pattern = {'${name}' => param[:name], '${event}' => param[:event].to_s, '${primary_key}' => param[:primary_key]}
-      tag.gsub(/\${[a-z_]+(\[[0-9]+\])?}/, pattern) do
-        $log.warn "mysql_replicator_multi: missing placeholder. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
+      tag.gsub(/(\${[a-z_]+})/) do
+        $log.warn "mysql_replicator_multi: unknown placeholder found. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
         pattern[$1]
       end
     end
 
+    def add_hash_table_buffer(setting_name, id, hash)
+      @hash_table_bulk_insert << "('#{setting_name}','#{id}','#{hash}')"
+      flush_hash_table if @hash_table_bulk_insert.size >= @bulk_insert_count
+    end
+
+    def hash_table_flusher
+      begin
+        loop do
+          if @hash_table_bulk_insert.empty? || @bulk_insert_timeout > (Time.now - @hash_table_bulk_insert_last_time)
+            sleep @bulk_insert_timeout
+            next
+          end
+          @mutex.synchronize {
+            flush_hash_table
+          }
+        end
+      rescue StandardError => e
+        @mutex.synchronize {
+          $log.error "mysql_replicator_multi: failed to flush buffered query. :config=>#{masked_config}"
+          $log.error "error: #{e.message}"
+          $log.error e.backtrace.join("\n")
+        }
+      end
+    end
+
+    def flush_hash_table
+      return if @hash_table_bulk_insert.empty?
+      query = "INSERT INTO hash_tables (setting_name,setting_query_pk,setting_query_hash) 
+        VALUES #{@hash_table_bulk_insert.join(',')}
+        ON DUPLICATE KEY UPDATE setting_query_hash = VALUES(setting_query_hash)"
+      @manager_db.query(query)
+      @hash_table_bulk_insert.clear
+      @hash_table_bulk_insert_last_time = Time.now
+    end
+
     def emit_record(tag, record)
       Engine.emit(tag, Engine.now, record)
     end

data/setup_mysql_replicator_multi.sql

@@ -23,6 +23,8 @@ CREATE TABLE `settings` (
   `interval` int(11) NOT NULL,
   `primary_key` varchar(255) DEFAULT 'id',
   `enable_delete` int(11) DEFAULT '1',
+  `enable_loose_insert` int(11) DEFAULT '0',
+  `enable_loose_delete` int(11) DEFAULT '0',
   PRIMARY KEY (`id`),
   UNIQUE KEY `name` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-replicator
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-16 00:00:00.000000000 Z
+date: 2013-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake