fluent-plugin-mysql-replicator 0.5.2 → 0.6.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ffa1646346a6d10f15cc8837a0551ecdcddfd0c0
+  data.tar.gz: 5a3f350ffdeda01067e3b7ab90b35078333f9ee6
+SHA512:
+  metadata.gz: d96f0e8226cb824bf44a14d961528cef03a3831df71a0a5a6810d68935733fbe1f940d44875ef7871f2b757845174e168bd69fd77239fad96f773ef967b99af2
+  data.tar.gz: 2c5886bdaa8050b70024cfeab747bdadaa6c0eb22bbb4b431aad818502969f58a3fbd23315565d41d74c4b9ee7cd9c9fca0f22681aa93c30dececa78647d5a22

data/.travis.yml

@@ -1,15 +1,17 @@
 language: ruby
 
 rvm:
+  - 2.4.0
+  - 2.3.3
   - 2.2
   - 2.1
-  - 2.0.0
-  - 1.9.3
 
 services:
   - elasticsearch
 
+before_install:
+  - gem update bundler
+
 before_script:
   - mysql < setup_mysql_replicator_multi.sql
   - curl https://raw.github.com/moliware/travis-solr/master/travis-solr.sh | SOLR_VERSION=4.6.1 bash
-

data/README.md

@@ -6,22 +6,25 @@ Fluentd input plugin to track insert/update/delete event from MySQL database ser
 Not only that, it could multiple table replication into single or multi Elasticsearch/Solr.  
 It's comming support replicate to another RDB/noSQL.
 
+## Requirements
+
+| fluent-plugin-mysql-replicator | fluentd    | ruby   |
+|--------------------|------------|--------|
+|  0.6.1            | v0.14.x | >= 2.1 |
+|  0.6.1            | v0.12.x | >= 1.9 |
+
 ## Installation
 
 install with gem or fluent-gem command as:
 
 `````
 # for system installed fluentd
-$ gem install fluent-plugin-mysql-replicator
+$ gem install fluent-plugin-mysql-replicator -v 0.6.1
 
-# for td-agent
-$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-mysql-replicator
+# for td-agent2
+$ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 0.6.1
 `````
 
-**Note:** [recommend] RPM package available which does not conflict system installed Ruby or td-agent.  
-https://github.com/y-ken/yamabiko/releases
-
-
 ## Included plugins
 
 * Input Plugin: mysql_replicator

data/example/mysql_multi_table_to_elasticsearch.md

@@ -6,7 +6,7 @@ It is a guide to replicate multiple mysql table to elasticsearch.
 
 ```
 <source>
-  type mysql_replicator_multi
+  @type mysql_replicator_multi
 
   # Database connection setting for manager table.
   manager_host localhost
@@ -22,7 +22,7 @@ It is a guide to replicate multiple mysql table to elasticsearch.
 </source>
 
 <match replicator.**>
-  type mysql_replicator_elasticsearch
+  @type mysql_replicator_elasticsearch
 
   # Set Elasticsearch connection.
   host localhost

data/example/mysql_multi_table_to_solr.md

@@ -6,7 +6,7 @@ It is a guide to replicate multiple mysql table to solr.
 
 ```
 <source>
-  type mysql_replicator_multi
+  @type mysql_replicator_multi
 
   # Database connection setting for manager table.
   manager_host localhost
@@ -22,7 +22,7 @@ It is a guide to replicate multiple mysql table to solr.
 </source>
 
 <match replicator.**>
-  type mysql_replicator_solr
+  @type mysql_replicator_solr
 
   # Set Solr connection.
   host localhost
@@ -50,7 +50,7 @@ When you use default core (won't specify), change the value of `tag_format` like
 
 ```
 <match replicator.**>
-  type mysql_replicator_solr
+  @type mysql_replicator_solr
 
   # Set Solr connection.
   host localhost

data/example/mysql_single_table_to_elasticsearch.md

@@ -6,7 +6,7 @@ It is a guide to replicate single mysql table to elasticsearch.
 
 ```
 <source>
-  type mysql_replicator
+  @type mysql_replicator
 
   # Set connection settings for replicate source.
   host localhost
@@ -31,7 +31,7 @@ It is a guide to replicate single mysql table to elasticsearch.
 </source>
 
 <match replicator.**>
-  type mysql_replicator_elasticsearch
+  @type mysql_replicator_elasticsearch
 
   # Set Elasticsearch connection.
   host localhost

data/example/mysql_single_table_to_solr.md

@@ -6,7 +6,7 @@ It is a guide to replicate single mysql table to solr.
 
 ```
 <source>
-  type mysql_replicator
+  @type mysql_replicator
 
   # Set connection settings for replicate source.
   host localhost
@@ -31,7 +31,7 @@ It is a guide to replicate single mysql table to solr.
 </source>
 
 <match replicator.**>
-  type mysql_replicator_solr
+  @type mysql_replicator_solr
 
   # Set Solr connection.
   host localhost
@@ -60,7 +60,7 @@ On this case, the solr url will be set `http://localhost:8983/solr`
 
 ```
 <match replicator.**>
-  type mysql_replicator_solr
+  @type mysql_replicator_solr
 
   # Set Solr connection.
   host localhost

data/fluent-plugin-mysql-replicator.gemspec

@@ -1,23 +1,25 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-mysql-replicator"
-  s.version     = "0.5.2"
+  s.version     = "0.6.1"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-mysql-replicator"
   s.summary     = %q{Fluentd input plugin to track insert/update/delete event from MySQL database server. Not only that, it could multiple table replication and generate nested document for Elasticsearch/Solr. It's comming support replicate to another RDB/noSQL.}
+  s.license     = "Apache-2.0"
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
-  s.license = "Apache-2.0"
+
+  s.required_ruby_version = "> 2.1"
 
   s.add_development_dependency "rake"
-  s.add_development_dependency "webmock"
+  s.add_development_dependency "webmock", "~> 1.24.0"
   s.add_development_dependency "test-unit", ">= 3.1.0"
 
-  s.add_runtime_dependency "fluentd"
+  s.add_runtime_dependency "fluentd", [">= 0.10.58", "< 2"]
   s.add_runtime_dependency "mysql2"
   s.add_runtime_dependency "rsolr"
 end

data/lib/fluent/plugin/in_mysql_replicator.rb

@@ -1,12 +1,9 @@
+require 'fluent/input'
+
 module Fluent
   class MysqlReplicatorInput < Fluent::Input
     Plugin.register_input('mysql_replicator', self)
 
-    # Define `router` method to support v0.10.57 or earlier
-    unless method_defined?(:router)
-      define_method("router") { Engine }
-    end
-
     def initialize
       require 'mysql2'
       require 'digest/sha1'
@@ -34,7 +31,7 @@ module Fluent
         raise Fluent::ConfigError, "mysql_replicator: missing 'tag' parameter. Please add following line into config like 'tag replicator.mydatabase.mytable.${event}.${primary_key}'"
       end
 
-      $log.info "adding mysql_replicator worker. :tag=>#{tag} :query=>#{@query} :prepared_query=>#{@prepared_query} :interval=>#{@interval}sec :enable_delete=>#{enable_delete}"
+      log.info "adding mysql_replicator worker. :tag=>#{tag} :query=>#{@query} :prepared_query=>#{@prepared_query} :interval=>#{@interval}sec :enable_delete=>#{enable_delete}"
     end
 
     def start
@@ -49,9 +46,9 @@ module Fluent
       begin
         poll
       rescue StandardError => e
-        $log.error "mysql_replicator: failed to execute query."
-        $log.error "error: #{e.message}"
-        $log.error e.backtrace.join("\n")
+        log.error "mysql_replicator: failed to execute query."
+        log.error "error: #{e.message}"
+        log.error e.backtrace.join("\n")
       end
     end
 
@@ -85,7 +82,7 @@ module Fluent
             prepared_con.close
           end
           if row[@primary_key].nil?
-            $log.error "mysql_replicator: missing primary_key. :tag=>#{tag} :primary_key=>#{primary_key}"
+            log.error "mysql_replicator: missing primary_key. :tag=>#{tag} :primary_key=>#{primary_key}"
             break
           end
           if !table_hash.include?(row[@primary_key])
@@ -101,7 +98,9 @@ module Fluent
         con.close
         ids = current_ids
         if @enable_delete
-          if previous_ids.empty?
+          if current_ids.empty?
+            deleted_ids = Array.new
+          elsif previous_ids.empty?
             deleted_ids = [*1...current_ids.max] - current_ids
           else
             deleted_ids = previous_ids - current_ids
@@ -115,7 +114,7 @@ module Fluent
           end
         end
         elapsed_time = sprintf("%0.02f", Time.now - start_time)
-        $log.info "mysql_replicator: finished execution :tag=>#{tag} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
+        log.info "mysql_replicator: finished execution :tag=>#{tag} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
         sleep @interval
       end
     end
@@ -127,13 +126,13 @@ module Fluent
     def format_tag(tag, param)
       pattern = {'${event}' => param[:event].to_s, '${primary_key}' => @primary_key}
       tag.gsub(/(\${[a-z_]+})/) do
-        $log.warn "mysql_replicator: missing placeholder. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
+        log.warn "mysql_replicator: missing placeholder. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
         pattern[$1]
       end
     end
 
     def emit_record(tag, record)
-      router.emit(tag, Engine.now, record)
+      router.emit(tag, Fluent::Engine.now, record)
     end
 
     def query(query, con = nil)
@@ -142,7 +141,7 @@ module Fluent
         con = con.ping ? con : get_connection
         return con.query(query), con
       rescue Exception => e
-        $log.warn "mysql_replicator: #{e}"
+        log.warn "mysql_replicator: #{e}"
         sleep @interval
         retry
       end
@@ -162,7 +161,7 @@ module Fluent
           :cache_rows => false
         })
       rescue Exception => e
-        $log.warn "mysql_replicator: #{e}"
+        log.warn "mysql_replicator: #{e}"
         sleep @interval
         retry
       end

data/lib/fluent/plugin/in_mysql_replicator_multi.rb

@@ -1,12 +1,9 @@
+require 'fluent/input'
+
 module Fluent
   class MysqlReplicatorMultiInput < Fluent::Input
     Plugin.register_input('mysql_replicator_multi', self)
 
-    # Define `router` method to support v0.10.57 or earlier
-    unless method_defined?(:router)
-      define_method("router") { Engine }
-    end
-
     def initialize
       require 'mysql2'
       require 'digest/sha1'
@@ -46,10 +43,10 @@ module Fluent
             poll(config)
           }
         end
-        $log.error "mysql_replicator_multi: stop working due to empty configuration" if @threads.empty?
+        log.error "mysql_replicator_multi: stop working due to empty configuration" if @threads.empty?
       rescue StandardError => e
-        $log.error "error: #{e.message}"
-        $log.error e.backtrace.join("\n")
+        log.error "error: #{e.message}"
+        log.error e.backtrace.join("\n")
       end
     end
 
@@ -73,7 +70,7 @@ module Fluent
       begin
         masked_config = Hash[config.map {|k,v| (k == 'password') ? [k, v.to_s.gsub(/./, '*')] : [k,v]}]
         @mutex.synchronize {
-          $log.info "mysql_replicator_multi: polling start. :config=>#{masked_config}"
+          log.info "mysql_replicator_multi: polling start. :config=>#{masked_config}"
         }
         primary_key = config['primary_key']
         previous_id = current_id = nil
@@ -99,7 +96,7 @@ module Fluent
             current_id = row[primary_key]
             @mutex.synchronize {
               if row[primary_key].nil?
-                $log.error "mysql_replicator_multi: missing primary_key. :setting_name=>#{config['name']} :primary_key=>#{primary_key}"
+                log.error "mysql_replicator_multi: missing primary_key. :setting_name=>#{config['name']} :primary_key=>#{primary_key}"
                 break
               end
               detect_insert_update(config, row)
@@ -111,15 +108,15 @@ module Fluent
           db.close
           elapsed_time = sprintf("%0.02f", Time.now - start_time)
           @mutex.synchronize {
-            $log.info "mysql_replicator_multi: execution finished. :setting_name=>#{config['name']} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
+            log.info "mysql_replicator_multi: execution finished. :setting_name=>#{config['name']} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
           }
           sleep config['interval']
         end
       rescue StandardError => e
         @mutex.synchronize {
-          $log.error "mysql_replicator_multi: failed to execute query. :config=>#{masked_config}"
-          $log.error "error: #{e.message}"
-          $log.error e.backtrace.join("\n")
+          log.error "mysql_replicator_multi: failed to execute query. :config=>#{masked_config}"
+          log.error "error: #{e.message}"
+          log.error e.backtrace.join("\n")
         }
       end
     end
@@ -203,7 +200,7 @@ module Fluent
     def format_tag(tag, param)
       pattern = {'${name}' => param[:name], '${event}' => param[:event].to_s, '${primary_key}' => param[:primary_key]}
       tag.gsub(/(\${[a-z_]+})/) do
-        $log.warn "mysql_replicator_multi: unknown placeholder found. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
+        log.warn "mysql_replicator_multi: unknown placeholder found. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
         pattern[$1]
       end
     end
@@ -226,9 +223,9 @@ module Fluent
         end
       rescue StandardError => e
         @mutex.synchronize {
-          $log.error "mysql_replicator_multi: failed to flush buffered query. :config=>#{masked_config}"
-          $log.error "error: #{e.message}"
-          $log.error e.backtrace.join("\n")
+          log.error "mysql_replicator_multi: failed to flush buffered query. :config=>#{masked_config}"
+          log.error "error: #{e.message}"
+          log.error e.backtrace.join("\n")
         }
       end
     end
@@ -244,7 +241,7 @@ module Fluent
     end
 
     def emit_record(tag, record)
-      router.emit(tag, Engine.now, record)
+      router.emit(tag, Fluent::Engine.now, record)
     end
 
     def get_manager_connection

metadata

@@ -1,110 +1,103 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-replicator
 version: !ruby/object:Gem::Version
-  version: 0.5.2
-  prerelease: 
+  version: 0.6.1
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-13 00:00:00.000000000 Z
+date: 2017-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.24.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.24.0
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.0
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.58
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.58
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rsolr
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: 
@@ -114,8 +107,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE
 - README.md
@@ -140,27 +133,26 @@ files:
 homepage: https://github.com/y-ken/fluent-plugin-mysql-replicator
 licenses:
 - Apache-2.0
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.4.5
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Fluentd input plugin to track insert/update/delete event from MySQL database
   server. Not only that, it could multiple table replication and generate nested document
   for Elasticsearch/Solr. It's comming support replicate to another RDB/noSQL.