fluent-plugin-mysql-replicator 0.2.2 → 0.2.3

data/README.md

@@ -16,7 +16,47 @@ gem install fluent-plugin-mysql-replicator
 /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-mysql-replicator
 `````
 
-## Tutorial for Quickstart
+## Included plugins
+
+* Input Plugin: mysql_replicator
+* Input Plugin: mysql_replicator_multi
+* Output Plugin: mysql_replicator_elasticsearch
+* Output Plugin: mysql_replicator_solr (experimental)
+
+## Output example
+
+It is a example when detecting insert/update/delete events.
+
+### sample query
+
+`````
+$ mysql -e "create database myweb"
+$ mysql myweb -e "create table search_test(id int auto_increment, text text, PRIMARY KEY (id))"
+$ sleep 10
+$ mysql myweb -e "insert into search_test(text) values('aaa')"
+$ sleep 10
+$ mysql myweb -e "update search_test set text='bbb' where text = 'aaa'"
+$ sleep 10
+$ mysql myweb -e "delete from search_test where text='bbb'"
+`````
+
+### result
+
+`````
+$ tail -f /var/log/td-agent/td-agent.log
+2013-11-25 18:22:25 +0900 replicator.myweb.search_test.insert.id: {"id":"1","text":"aaa"}
+2013-11-25 18:22:35 +0900 replicator.myweb.search_test.update.id: {"id":"1","text":"bbb"}
+2013-11-25 18:22:45 +0900 replicator.myweb.search_test.delete.id: {"id":"1"}
+`````
+
+## Configuration Examples
+
+* [mysql_single_table_to_elasticsearch.md](https://github.com/y-ken/fluent-plugin-mysql-replicator/blob/master/example/mysql_single_table_to_elasticsearch.md)
+* [mysql_multi_table_to_elasticsearch.md](https://github.com/y-ken/fluent-plugin-mysql-replicator/blob/master/example/mysql_multi_table_to_elasticsearch.md)
+* [mysql_single_table_to_solr.md](https://github.com/y-ken/fluent-plugin-mysql-replicator/blob/master/example/mysql_single_table_to_solr.md)
+* [mysql_multi_table_to_solr.md](https://github.com/y-ken/fluent-plugin-mysql-replicator/blob/master/example/mysql_multi_table_to_solr.md)
+
+## Tutorial for Quickstart (mysql_replicator)
 
 It is useful for these purpose.
 
@@ -67,7 +107,7 @@ On syncing 300 million rows table, it will consume around 800MB of memory with r
     port 9200
 
     # Set Elasticsearch index, type, and unique id (primary_key) from tag.
-    tag_format (?<index_name>[^\.]+)\.(?<type_name>[^\.]+).(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
+    tag_format (?<index_name>[^\.]+)\.(?<type_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
 
     # Set frequency of sending bulk request to Elasticsearch node.
     flush_interval 5s
@@ -78,29 +118,7 @@ On syncing 300 million rows table, it will consume around 800MB of memory with r
 </match>
 `````
 
-### sample query
-
-`````
-$ mysql -e "create database myweb"
-$ mysql myweb -e "create table search_test(id int auto_increment, text text, PRIMARY KEY (id))"
-$ sleep 10
-$ mysql myweb -e "insert into search_test(text) values('aaa')"
-$ sleep 10
-$ mysql myweb -e "update search_test set text='bbb' where text = 'aaa'"
-$ sleep 10
-$ mysql myweb -e "delete from search_test where text='bbb'"
-`````
-
-### result
-
-`````
-$ tail -f /var/log/td-agent/td-agent.log
-2013-11-25 18:22:25 +0900 replicator.insert.id: {"id":"1","text":"aaa"}
-2013-11-25 18:22:35 +0900 replicator.update.id: {"id":"1","text":"bbb"}
-2013-11-25 18:22:45 +0900 replicator.delete.id: {"id":"1"}
-`````
-
-## Tutorial for Production
+## Tutorial for Production (mysql_replicator_multi)
 
 It is very useful to replicate a millions of records and/or multiple tables with multiple threads.  
 This architecture is storing hash table in mysql management table instead of ruby internal memory.  
@@ -197,11 +215,14 @@ it is a sample which you have inserted row.
   port 9200
 
   # Set Elasticsearch index, type, and unique id (primary_key) from tag.
-  tag_format (?<index_name>[^\.]+)\.(?<type_name>[^\.]+).(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
+  tag_format (?<index_name>[^\.]+)\.(?<type_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
 
   # Set frequency of sending bulk request to Elasticsearch node.
   flush_interval 5s
-  
+
+  # Set maximum retry interval (required fluentd >= 0.10.41)
+  max_retry_wait 1800
+
   # Queued chunks are flushed at shutdown process.
   flush_at_shutdown yes
 </match>

data/example/mysql_multi_table_to_elasticsearch.md

@@ -0,0 +1,43 @@
+## case study
+
+It is a guide to replicate multiple mysql table to elasticsearch.
+
+## configuration
+
+```
+<source>
+  type mysql_replicator_multi
+
+  # Database connection setting for manager table.
+  manager_host localhost
+  manager_username your_mysql_user
+  manager_password your_mysql_password
+  manager_database replicator_manager
+
+  # Format output tag for each events. Placeholders usage as described below.
+  tag replicator.${name}.${event}.${primary_key}
+  # ${name} : the value of `replicator_manager.settings.name` in manager table.
+  # ${event} : the variation of row event type by insert/update/delete.
+  # ${primary_key} : the value of `replicator_manager.settings.primary_key` in manager table.
+</source>
+
+<match replicator.**>
+  type mysql_replicator_elasticsearch
+
+  # Set Elasticsearch connection.
+  host localhost
+  port 9200
+
+  # Set Elasticsearch index, type, and unique id (primary_key) from tag.
+  tag_format (?<index_name>[^\.]+)\.(?<type_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
+
+  # Set frequency of sending bulk request to Elasticsearch node.
+  flush_interval 5s
+
+  # Set maximum retry interval (required fluentd >= 0.10.41)
+  max_retry_wait 1800
+
+  # Queued chunks are flushed at shutdown process.
+  flush_at_shutdown yes
+</match>
+```

data/example/mysql_multi_table_to_solr.md

@@ -0,0 +1,69 @@
+## case study
+
+It is a guide to replicate multiple mysql table to solr.
+
+## configuration
+
+```
+<source>
+  type mysql_replicator_multi
+
+  # Database connection setting for manager table.
+  manager_host localhost
+  manager_username your_mysql_user
+  manager_password your_mysql_password
+  manager_database replicator_manager
+
+  # Format output tag for each events. Placeholders usage as described below.
+  tag replicator.${name}.${event}.${primary_key}
+  # ${name} : the value of `replicator_manager.settings.name` in manager table.
+  # ${event} : the variation of row event type by insert/update/delete.
+  # ${primary_key} : the value of `replicator_manager.settings.primary_key` in manager table.
+</source>
+
+<match replicator.**>
+  type mysql_replicator_solr
+
+  # Set Solr connection.
+  host localhost
+  port 8983
+
+  # Set Solr core name and unique id (primary_key) from tag.
+  # On this case, solr url will be http://localhost:8983/solr/${core_name}
+  tag_format (?<core_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
+
+  # Set frequency of sending bulk request to Solr.
+  flush_interval 5s
+
+  # Set maximum retry interval (required fluentd >= 0.10.41)
+  max_retry_wait 1800
+
+  # Queued chunks are flushed at shutdown process.
+  flush_at_shutdown yes
+</match>
+```
+
+When you use default core (won't specify), change the value of `tag_format` like below.
+
+```
+<match replicator.**>
+  type mysql_replicator_solr
+
+  # Set Solr connection.
+  host localhost
+  port 8983
+
+  # Set Solr core name and unique id (primary_key) from tag.
+  # On this case, solr url will be http://localhost:8983/solr/
+  tag_format (?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
+
+  # Set frequency of sending bulk request to Solr.
+  flush_interval 5s
+
+  # Set maximum retry interval (required fluentd >= 0.10.41)
+  max_retry_wait 1800
+
+  # Queued chunks are flushed at shutdown process.
+  flush_at_shutdown yes
+</match>
+```

data/example/mysql_single_table_to_elasticsearch.md

@@ -0,0 +1,52 @@
+## case study
+
+It is a guide to replicate single mysql table to elasticsearch.
+
+## configuration
+
+```
+<source>
+  type mysql_replicator
+
+  # Set connection settings for replicate source.
+  host localhost
+  username your_mysql_user
+  password your_mysql_password
+  database myweb
+
+  # Set replicate query configuration.
+  query SELECT id, text, updated_at from search_test;
+  primary_key id # specify unique key (default: id)
+  interval 10s  # execute query interval (default: 1m)
+
+  # Enable detect deletion event not only insert/update events. (default: yes)
+  # It is useful to use `enable_delete no` that keep following recently updated record with this query.
+  # `SELECT * FROM search_test WHERE DATE_ADD(updated_at, INTERVAL 5 MINUTE) > NOW();`
+  enable_delete yes
+
+  # Format output tag for each events. Placeholders usage as described below.
+  tag replicator.myweb.search_test.${event}.${primary_key}
+  # ${event} : the variation of row event type by insert/update/delete.
+  # ${primary_key} : the value of `replicator_manager.settings.primary_key` in manager table.
+</source>
+
+<match replicator.**>
+  type mysql_replicator_elasticsearch
+
+  # Set Elasticsearch connection.
+  host localhost
+  port 9200
+
+  # Set Elasticsearch index, type, and unique id (primary_key) from tag.
+  tag_format (?<core_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
+
+  # Set frequency of sending bulk request to Elasticsearch node.
+  flush_interval 5s
+
+  # Set maximum retry interval (required fluentd >= 0.10.41)
+  max_retry_wait 1800
+
+  # Queued chunks are flushed at shutdown process.
+  flush_at_shutdown yes
+</match>
+```

data/example/mysql_single_table_to_solr.md

@@ -0,0 +1,79 @@
+## case study
+
+It is a guide to replicate single mysql table to solr.
+
+## configuration
+
+```
+<source>
+  type mysql_replicator
+
+  # Set connection settings for replicate source.
+  host localhost
+  username your_mysql_user
+  password your_mysql_password
+  database myweb
+
+  # Set replicate query configuration.
+  query SELECT id, text, updated_at from search_test;
+  primary_key id # specify unique key (default: id)
+  interval 10s  # execute query interval (default: 1m)
+
+  # Enable detect deletion event not only insert/update events. (default: yes)
+  # It is useful to use `enable_delete no` that keep following recently updated record with this query.
+  # `SELECT * FROM search_test WHERE DATE_ADD(updated_at, INTERVAL 5 MINUTE) > NOW();`
+  enable_delete yes
+
+  # Format output tag for each events. Placeholders usage as described below.
+  tag replicator.myweb.search_test.${event}.${primary_key}
+  # ${event} : the variation of row event type by insert/update/delete.
+  # ${primary_key} : the value of `replicator_manager.settings.primary_key` in manager table.
+</source>
+
+<match replicator.**>
+  type mysql_replicator_solr
+
+  # Set Solr connection.
+  host localhost
+  port 8983
+
+  # Set Solr core name and unique id (primary_key) from tag.
+  # On this case, solr url will be http://localhost:8983/solr/${core_name}
+  tag_format (?<core_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
+
+  # Set frequency of sending bulk request to Solr.
+  flush_interval 5s
+
+  # Set maximum retry interval (required fluentd >= 0.10.41)
+  max_retry_wait 1800
+
+  # Queued chunks are flushed at shutdown process.
+  flush_at_shutdown yes
+</match>
+```
+
+When you use default core (won't specify), change the value of `tag_format` like below.  
+On this case, the solr url will be set `http://localhost:8983/solr`
+
+```
+<match replicator.**>
+  type mysql_replicator_solr
+
+  # Set Solr connection.
+  host localhost
+  port 8983
+
+  # Set Solr core name and unique id (primary_key) from tag.
+  # On this case, solr url will be http://localhost:8983/solr/
+  tag_format (?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
+
+  # Set frequency of sending bulk request to Solr.
+  flush_interval 5s
+
+  # Set maximum retry interval (required fluentd >= 0.10.41)
+  max_retry_wait 1800
+
+  # Queued chunks are flushed at shutdown process.
+  flush_at_shutdown yes
+</match>
+```

data/fluent-plugin-mysql-replicator.gemspec

@@ -1,11 +1,11 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-mysql-replicator"
-  s.version     = "0.2.2"
+  s.version     = "0.2.3"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-mysql-replicator"
-  s.summary     = %q{Fluentd input plugin to track insert/update/delete event from MySQL database server. Not only that, it could multiple table replication into Elasticsearch nodes. It's comming support replicate to another RDB/noSQL.}
+  s.summary     = %q{Fluentd input plugin to track insert/update/delete event from MySQL database server. Not only that, it could multiple table replication into Elasticsearch/Solr. It's comming support replicate to another RDB/noSQL.}
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
@@ -17,4 +17,5 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency "fluentd"
   s.add_runtime_dependency "mysql2"
+  s.add_runtime_dependency "rsolr"
 end

data/lib/fluent/plugin/in_mysql_replicator.rb

@@ -28,7 +28,7 @@ module Fluent
         raise Fluent::ConfigError, "mysql_replicator: missing 'tag' parameter. Please add following line into config like 'tag replicator.mydatabase.mytable.${event}.${primary_key}'"
       end
 
-      $log.info "adding mysql_replicator worker. :tag=>#{tag} :query=>[#{@query}] :interval=>#{@interval}sec :enable_delete=>#{enable_delete}"
+      $log.info "adding mysql_replicator worker. :tag=>#{tag} :query=>#{@query} :interval=>#{@interval}sec :enable_delete=>#{enable_delete}"
     end
 
     def start
@@ -89,7 +89,7 @@ module Fluent
           end
         end
         elapsed_time = sprintf("%0.02f", Time.now - start_time)
-        $log.info "mysql_replicator: finished execution :tag=>#{tag} :elapsed_time=>#{elapsed_time} seconds"
+        $log.info "mysql_replicator: finished execution :tag=>#{tag} :elapsed_time=>#{elapsed_time} sec"
         sleep @interval
       end
     end

data/lib/fluent/plugin/in_mysql_replicator_multi.rb

@@ -58,7 +58,7 @@ module Fluent
     def poll(config)
       begin
         @manager_db = get_manager_connection
-        masked_config = config.map {|k,v| (k == 'password') ? v.to_s.gsub(/./, '*') : v}
+        masked_config = Hash[config.map {|k,v| (k == 'password') ? [k, v.to_s.gsub(/./, '*')] : [k,v]}]
         @mutex.synchronize {
           $log.info "mysql_replicator_multi: polling start. :config=>#{masked_config}"
         }
@@ -83,7 +83,7 @@ module Fluent
           end
           db.close
           elapsed_time = sprintf("%0.02f", Time.now - start_time)
-          $log.info "mysql_replicator_multi: finished execution :setting_name=>#{config['name']} :elapsed_time=>#{elapsed_time} seconds"
+          $log.info "mysql_replicator_multi: finished execution :setting_name=>#{config['name']} :elapsed_time=>#{elapsed_time} sec"
           sleep config['interval']
         end
       rescue StandardError => e

data/lib/fluent/plugin/out_mysql_replicator_elasticsearch.rb

@@ -8,7 +8,7 @@ class Fluent::MysqlReplicatorElasticsearchOutput < Fluent::BufferedOutput
   config_param :port, :integer, :default => 9200
   config_param :tag_format, :string, :default => nil
 
-  DEFAULT_TAG_FORMAT = /(?<index_name>[^\.]+)\.(?<type_name>[^\.]+).(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$/
+  DEFAULT_TAG_FORMAT = /(?<index_name>[^\.]+)\.(?<type_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$/
 
   def initialize
     super

data/lib/fluent/plugin/out_mysql_replicator_solr.rb

@@ -0,0 +1,57 @@
+require 'rsolr'
+
+class Fluent::MysqlReplicatorSolrOutput < Fluent::BufferedOutput
+  Fluent::Plugin.register_output('mysql_replicator_solr', self)
+
+  config_param :host, :string,  :default => 'localhost'
+  config_param :port, :integer, :default => 8983
+  config_param :tag_format, :string, :default => nil
+
+  DEFAULT_TAG_FORMAT = /(?<core_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$/
+
+  def initialize
+    super
+  end
+
+  def configure(conf)
+    super
+
+    if @tag_format.nil? || @tag_format == DEFAULT_TAG_FORMAT
+      @tag_format = DEFAULT_TAG_FORMAT
+    else
+      @tag_format = Regexp.new(conf['tag_format'])
+    end
+  end
+
+  def start
+    super
+  end
+
+  def format(tag, time, record)
+    [tag, time, record].to_msgpack
+  end
+
+  def shutdown
+    super
+  end
+
+  def write(chunk)
+    solr_connection = {}
+
+    chunk.msgpack_each do |tag, time, record|
+      tag_parts = tag.match(@tag_format)
+      id_key = tag_parts['primary_key']
+      core_name = tag_parts['core_name'].nil? ? '' : tag_parts['core_name']
+      url = "http://#{@host}:#{@port}/solr/#{core_name}"
+      solr_connection[url] = RSolr.connect(:url => url) if solr_connection[url].nil?
+      if tag_parts['event'] == 'delete'
+        solr_connection[url].delete_by_id record[id_key]
+      else
+        message = Hash[record.map{ |k, v| [k.to_sym, v] }]
+        message[:id] = record[id_key] if id_key && record[id_key]
+        solr_connection[url].add message
+      end
+    end
+    solr_connection.each {|solr| solr.commit }
+  end
+end

data/test/helper.rb

@@ -25,6 +25,7 @@ end
 require 'fluent/plugin/in_mysql_replicator'
 require 'fluent/plugin/in_mysql_replicator_multi'
 require 'fluent/plugin/out_mysql_replicator_elasticsearch'
+require 'fluent/plugin/out_mysql_replicator_solr'
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_out_mysql_replicator_solr.rb

@@ -0,0 +1,24 @@
+require 'helper'
+
+class MysqlReplicatorSolrOutput < Test::Unit::TestCase
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    host       localhost
+    port       8983
+    tag_format (?<core_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
+  ]
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::MysqlReplicatorSolrOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver(%[])
+    assert_equal 'localhost', d.instance.host
+    assert_equal 8983, d.instance.port
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-replicator
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-13 00:00:00.000000000 Z
+date: 2013-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -75,6 +75,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rsolr
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - y.ken.studio@gmail.com
@@ -88,15 +104,21 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- example/mysql_multi_table_to_elasticsearch.md
+- example/mysql_multi_table_to_solr.md
+- example/mysql_single_table_to_elasticsearch.md
+- example/mysql_single_table_to_solr.md
 - fluent-plugin-mysql-replicator.gemspec
 - lib/fluent/plugin/in_mysql_replicator.rb
 - lib/fluent/plugin/in_mysql_replicator_multi.rb
 - lib/fluent/plugin/out_mysql_replicator_elasticsearch.rb
+- lib/fluent/plugin/out_mysql_replicator_solr.rb
 - setup_mysql_replicator_multi.sql
 - test/helper.rb
 - test/plugin/test_in_mysql_replicator.rb
 - test/plugin/test_in_mysql_replicator_multi.rb
 - test/plugin/test_out_mysql_replicator_elasticsearch.rb
+- test/plugin/test_out_mysql_replicator_solr.rb
 homepage: https://github.com/y-ken/fluent-plugin-mysql-replicator
 licenses: []
 post_install_message: 
@@ -121,10 +143,11 @@ rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Fluentd input plugin to track insert/update/delete event from MySQL database
-  server. Not only that, it could multiple table replication into Elasticsearch nodes.
+  server. Not only that, it could multiple table replication into Elasticsearch/Solr.
   It's comming support replicate to another RDB/noSQL.
 test_files:
 - test/helper.rb
 - test/plugin/test_in_mysql_replicator.rb
 - test/plugin/test_in_mysql_replicator_multi.rb
 - test/plugin/test_out_mysql_replicator_elasticsearch.rb
+- test/plugin/test_out_mysql_replicator_solr.rb