fluent-plugin-mysql-replicator 0.2.0 → 0.2.1

data/README.md

@@ -20,7 +20,7 @@ gem install fluent-plugin-mysql-replicator
 
 It is useful for these purpose.
 
-* try it on this plugin.
+* try it on this plugin quickly.
 * replicate small record under a millons table.
 
 **Note:**  
@@ -71,6 +71,9 @@ On syncing 300 million rows table, it will consume around 800MB of memory with r
 
     # Set frequency of sending bulk request to Elasticsearch node.
     flush_interval 5s
+    
+    # Queued chunks are flushed at shutdown process.
+    flush_at_shutdown yes
   </store>
 </match>
 `````
@@ -139,7 +142,7 @@ CREATE TABLE `settings` (
   `database` varchar(255) NOT NULL,
   `query` TEXT NOT NULL,
   `interval` int(11) NOT NULL,
-  `primary_key` varchar(11) DEFAULT 'id',
+  `primary_key` varchar(255) DEFAULT 'id',
   `enable_delete` int(11) DEFAULT '1',
   PRIMARY KEY (`id`),
   UNIQUE KEY `name` (`name`)
@@ -198,6 +201,9 @@ it is a sample which you have inserted row.
 
   # Set frequency of sending bulk request to Elasticsearch node.
   flush_interval 5s
+  
+  # Queued chunks are flushed at shutdown process.
+  flush_at_shutdown yes
 </match>
 `````
 

data/fluent-plugin-mysql-replicator.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-mysql-replicator"
-  s.version     = "0.2.0"
+  s.version     = "0.2.1"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-mysql-replicator"

data/lib/fluent/plugin/in_mysql_replicator.rb

@@ -43,6 +43,7 @@ module Fluent
       begin
         poll
       rescue StandardError => e
+        $log.error "mysql_replicator: failed to execute query."
         $log.error "error: #{e.message}"
         $log.error e.backtrace.join("\n")
       end
@@ -52,6 +53,7 @@ module Fluent
       table_hash = Hash.new
       ids = Array.new
       loop do
+        start_time = Time.now
         previous_ids = ids
         current_ids = Array.new
         query(@query).each do |row|
@@ -69,7 +71,11 @@ module Fluent
         end
         ids = current_ids
         if @enable_delete
-          deleted_ids = previous_ids - current_ids
+          if previous_ids.empty?
+            deleted_ids = [*1...current_ids.max] - current_ids
+          else
+            deleted_ids = previous_ids - current_ids
+          end
           if deleted_ids.count > 0
             hash_delete_by_list(table_hash, deleted_ids)
             deleted_ids.each do |id| 
@@ -78,6 +84,8 @@ module Fluent
             end
           end
         end
+        elapsed_time = sprintf("%0.02f", (Time.now - start_time) % 60)
+        $log.info "mysql_replicator: finished execution :tag=>#{tag} :elapsed_time=>#{elapsed_time} seconds"
         sleep @interval
       end
     end

data/lib/fluent/plugin/in_mysql_replicator_multi.rb

@@ -65,6 +65,7 @@ module Fluent
         primary_key = config['primary_key']
         previous_id = current_id = 0
         loop do
+          start_time = Time.now
           db = get_origin_connection(config)
           db.query(config['query']).each do |row|
             @mutex.lock
@@ -76,11 +77,16 @@ module Fluent
             @mutex.unlock
           end
           db.close
+          elapsed_time = sprintf("%0.02f", (Time.now - start_time) % 60)
+          $log.info "mysql_replicator_multi: finished execution :setting_name=>#{config['name']} :elapsed_time=>#{elapsed_time} seconds"
           sleep config['interval']
         end
       rescue StandardError => e
-        $log.error "error: #{e.message}"
-        $log.error e.backtrace.join("\n")
+        @mutex.synchronize {
+          $log.error "mysql_replicator_multi: failed to execute query. :config=>#{masked_config}"
+          $log.error "error: #{e.message}"
+          $log.error e.backtrace.join("\n")
+        }
       end
     end
 

data/setup_mysql_replicator_multi.sql

@@ -21,7 +21,7 @@ CREATE TABLE `settings` (
   `database` varchar(255) NOT NULL,
   `query` TEXT NOT NULL,
   `interval` int(11) NOT NULL,
-  `primary_key` varchar(11) DEFAULT 'id',
+  `primary_key` varchar(255) DEFAULT 'id',
   `enable_delete` int(11) DEFAULT '1',
   PRIMARY KEY (`id`),
   UNIQUE KEY `name` (`name`)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-replicator
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-11 00:00:00.000000000 Z
+date: 2013-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake