fluent-plugin-mysql-replicator 0.3.1 → 0.4.0

data/.travis.yml

@@ -1,5 +1,14 @@
 language: ruby
 
 rvm:
+  - 2.1.0
   - 2.0.0
   - 1.9.3
+
+services:
+  - elasticsearch
+
+before_script:
+  - mysql < setup_mysql_replicator_multi.sql
+  - curl https://raw.github.com/moliware/travis-solr/master/travis-solr.sh | SOLR_VERSION=4.6.1 bash
+

data/README.md

@@ -8,18 +8,17 @@ It's comming support replicate to another RDB/noSQL.
 
 ## Installation
 
-`````
-### native gem
-gem install fluent-plugin-mysql-replicator
+install with gem or fluent-gem command as:
 
-### td-agent gem
-/usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-mysql-replicator
+`````
+# for system installed fluentd
+$ gem install fluent-plugin-mysql-replicator
 
-### RPM
-### package availables at https://github.com/y-ken/yamabiko/releases
+# for td-agent
+$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-mysql-replicator
 `````
 
-**Note:** RPM package available which does not conflict system installed Ruby or td-agent.  
+**Note:** [recommend] RPM package available which does not conflict system installed Ruby or td-agent.  
 https://github.com/y-ken/yamabiko/releases
 
 
@@ -56,194 +55,50 @@ $ tail -f /var/log/td-agent/td-agent.log
 2013-11-25 18:22:45 +0900 replicator.myweb.search_test.delete.id: {"id":"1"}
 `````
 
-## Configuration Examples
-
-* [mysql_single_table_to_elasticsearch.md](https://github.com/y-ken/fluent-plugin-mysql-replicator/blob/master/example/mysql_single_table_to_elasticsearch.md)
-* [mysql_multi_table_to_elasticsearch.md](https://github.com/y-ken/fluent-plugin-mysql-replicator/blob/master/example/mysql_multi_table_to_elasticsearch.md)
-* [mysql_single_table_to_solr.md](https://github.com/y-ken/fluent-plugin-mysql-replicator/blob/master/example/mysql_single_table_to_solr.md)
-* [mysql_multi_table_to_solr.md](https://github.com/y-ken/fluent-plugin-mysql-replicator/blob/master/example/mysql_multi_table_to_solr.md)
-
-## Tutorial for Quickstart (mysql_replicator)
-
-It is useful for these purpose.
-
-* try it on this plugin quickly.
-* replicate small record under a millons table.
-
-**Note:**  
-On syncing 300 million rows table, it will consume around 800MB of memory with ruby 1.9.3 environment.
-
-### configuration
-
-`````
-<source>
-  type mysql_replicator
-
-  # Set connection settings for replicate source.
-  host localhost
-  username your_mysql_user
-  password your_mysql_password
-  database myweb
-
-  # Set replicate query configuration.
-  query SELECT id, text, updated_at from search_test;
-  primary_key id # specify unique key (default: id)
-  interval 10s  # execute query interval (default: 1m)
-
-  # Enable detect deletion event not only insert/update events. (default: yes)
-  # It is useful to use `enable_delete no` that keep following recently updated record with this query.
-  # `SELECT * FROM search_test WHERE DATE_ADD(updated_at, INTERVAL 5 MINUTE) > NOW();`
-  enable_delete yes
-
-  # Format output tag for each events. Placeholders usage as described below.
-  tag replicator.myweb.search_test.${event}.${primary_key}
-  # ${event} : the variation of row event type by insert/update/delete.
-  # ${primary_key} : the value of `replicator_manager.settings.primary_key` in manager table.
-</source>
-
-<match replicator.*>
-  type copy
-  <store>
-    type stdout
-  </store>
-  <store>
-    type mysql_replicator_elasticsearch
-
-    # Set Elasticsearch connection.
-    host localhost
-    port 9200
-
-    # Set Elasticsearch index, type, and unique id (primary_key) from tag.
-    tag_format (?<index_name>[^\.]+)\.(?<type_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
-
-    # Set frequency of sending bulk request to Elasticsearch node.
-    flush_interval 5s
-    
-    # Queued chunks are flushed at shutdown process. (recommend for more stability)
-    flush_at_shutdown yes
-    buffer_type file
-    buffer_path /var/log/td-agent/buffer/mysql_replicator_elasticsearch
-  </store>
-</match>
-`````
-
-## Tutorial for Production (mysql_replicator_multi)
-
-It is very useful to replicate a millions of records and/or multiple tables with multiple threads.  
-This architecture is storing hash table in mysql management table instead of ruby internal memory.  
-
-**Note:**  
-On syncing 300 million rows table, it will consume around 20MB of memory with ruby 1.9.3 environment.
+## Tutorial
 
-### prepare
+### mysql_replicator
 
-It has done with follwing two steps.
+It is easy to try it on this plugin quickly.  
+For more detail are described at [Tutorial-mysql_replicator.md](https://github.com/y-ken/fluent-plugin-mysql-replicator/blob/master/Tutorial-mysql_replicator.md)
 
-* create database and tables.
-* add replicator configuration.
+**Features**
 
-##### create database and tables.
+* Table (or view table) synchronization supported.
+* Replicate small record under a millons table.
+* It is recommend to use insert only table.
+* Nested documents are supported with placeholder which accessing to temporary table created at the each loop.
 
-```
-$ cat setup_mysql_replicator_multi.sql
-CREATE DATABASE replicator_manager;
-USE replicator_manager;
+**Examples**
 
-CREATE TABLE `hash_tables` (
-  `id` int(11) NOT NULL AUTO_INCREMENT,
-  `setting_name` varchar(255) NOT NULL,
-  `setting_query_pk` int(11) NOT NULL,
-  `setting_query_hash` varchar(255) NOT NULL,
-  PRIMARY KEY (`id`),
-  UNIQUE KEY `setting_query_pk` (`setting_query_pk`,`setting_name`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
-CREATE TABLE `settings` (
-  `id` int(11) NOT NULL AUTO_INCREMENT,
-  `is_active` int(11) NOT NULL DEFAULT '1',
-  `name` varchar(255) NOT NULL,
-  `host` varchar(255) NOT NULL DEFAULT 'localhost',
-  `port` int(11) NOT NULL DEFAULT '3306',
-  `username` varchar(255) NOT NULL,
-  `password` varchar(255) NOT NULL,
-  `database` varchar(255) NOT NULL,
-  `query` TEXT NOT NULL,
-  `interval` int(11) NOT NULL,
-  `primary_key` varchar(255) DEFAULT 'id',
-  `enable_delete` int(11) DEFAULT '1',
-  `enable_loose_insert` int(11) DEFAULT '0',
-  `enable_loose_delete` int(11) DEFAULT '0',
-  PRIMARY KEY (`id`),
-  UNIQUE KEY `name` (`name`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-```
-
-##### add replicator configuration.
-
-```
-$ mysql -umysqluser -p
-
--- For the first time, load schema.
-mysql> source /path/to/setup_mysql_replicator_multi.sql
-
--- Add replicate source connection and query settings like below.
-mysql> INSERT INTO `settings`
-  (`id`, `is_active`, `name`, `host`, `port`, `username`, `password`, `database`, `query`, `interval`, `primary_key`, `enable_delete`)
-VALUES
-  (NULL, 1, 'mydb.mytable', '192.168.100.221', 3306, 'mysqluser', 'mysqlpassword', 'mydb', 'SELECT id, text from mytable;', 5, 'id', 1);
-```
-
-it is a sample which you have inserted row.
-
-| id | is_active |     name     |      host       | port | username  |   password    | database |            query             | interval | primary_key | enable_delete | enable_loose_insert | enable_loose_delete |
-|----|-----------|--------------|-----------------|------|-----------|---------------|----------|------------------------------|----------|-------------|---------------|----|----|
-|  1 |         1 | mydb.mytable | 192.168.100.221 | 3306 | mysqluser | mysqlpassword | mydb     | SELECT id, text from mytable; |       5 | id          |             1 |  0 | 0 |
-
-### configuration
-
-`````
-<source>
-  type mysql_replicator_multi
-
-  # Database connection setting for manager table.
-  manager_host localhost
-  manager_username your_mysql_user
-  manager_password your_mysql_password
-  manager_database replicator_manager
-
-  # Format output tag for each events. Placeholders usage as described below.
-  tag replicator.${name}.${event}.${primary_key}
-  # ${name} : the value of `replicator_manager.settings.name` in manager table.
-  # ${event} : the variation of row event type by insert/update/delete.
-  # ${primary_key} : the value of `replicator_manager.settings.primary_key` in manager table.
-</source>
+* [mysql_single_table_to_elasticsearch.md](https://github.com/y-ken/fluent-plugin-mysql-replicator/blob/master/example/mysql_single_table_to_elasticsearch.md)
+* [mysql_single_table_to_solr.md](https://github.com/y-ken/fluent-plugin-mysql-replicator/blob/master/example/mysql_single_table_to_solr.md)
 
-<match replicator.**>
-  type mysql_replicator_elasticsearch
+### mysql_replicator_multi
 
-  # Set Elasticsearch connection.
-  host localhost
-  port 9200
+It replicates a millions of records and/or multiple tables with multiple threads.  
+This architecture is storing hash table in MySQL management table instead of ruby internal memory.  
+See tutorial at [Tutorial-mysql_replicator_multi.md](https://github.com/y-ken/fluent-plugin-mysql-replicator/blob/master/Tutorial-mysql_replicator_multi.md)
 
-  # Set Elasticsearch index, type, and unique id (primary_key) from tag.
-  tag_format (?<index_name>[^\.]+)\.(?<type_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
+**Features**
 
-  # Set frequency of sending bulk request to Elasticsearch node.
-  flush_interval 5s
+* table (or view table) synchronization supported.
+* Multiple table synchronization supported and its DSN stored in MySQL management table.
+* Using MySQL database as hash table cache to support replicate over a millions table.
+* It is recommend to make whole copy of tables.
+* Nested documents are supported with placeholder which accessing to temporary table created at the each loop.
 
-  # Set maximum retry interval (required fluentd >= 0.10.41)
-  max_retry_wait 1800
+**Examples**
 
-  # Queued chunks are flushed at shutdown process.
-  flush_at_shutdown yes
-</match>
-`````
+* [mysql_multi_table_to_elasticsearch.md](https://github.com/y-ken/fluent-plugin-mysql-replicator/blob/master/example/mysql_multi_table_to_elasticsearch.md)
+* [mysql_multi_table_to_solr.md](https://github.com/y-ken/fluent-plugin-mysql-replicator/blob/master/example/mysql_multi_table_to_solr.md)
 
 ## TODO
 
 Pull requests are very welcome like below!!
 
-* more tests.
+* more documents
+* more tests with mock.
 * support string type of primary_key.
 * support reload setting on demand.
 

data/Tutorial-mysql_replicator.md

@@ -0,0 +1,64 @@
+## Tutorial for Quickstart (mysql_replicator)
+
+It is useful for these purpose.
+
+* try it on this plugin quickly.
+* replicate small record under a millons table.
+
+**Note:**  
+On syncing 300 million rows table, it will consume around 800MB of memory with ruby 1.9.3 environment.
+
+### configuration
+
+`````
+<source>
+  type mysql_replicator
+
+  # Set connection settings for replicate source.
+  host localhost
+  username your_mysql_user
+  password your_mysql_password
+  database myweb
+
+  # Set replicate query configuration.
+  query SELECT id, text, updated_at from search_test;
+  primary_key id # specify unique key (default: id)
+  interval 10s  # execute query interval (default: 1m)
+
+  # Enable detect deletion event not only insert/update events. (default: yes)
+  # It is useful to use `enable_delete no` that keep following recently updated record with this query.
+  # `SELECT * FROM search_test WHERE DATE_ADD(updated_at, INTERVAL 5 MINUTE) > NOW();`
+  enable_delete yes
+
+  # Format output tag for each events. Placeholders usage as described below.
+  tag replicator.myweb.search_test.${event}.${primary_key}
+  # ${event} : the variation of row event type by insert/update/delete.
+  # ${primary_key} : the value of `replicator_manager.settings.primary_key` in manager table.
+</source>
+
+<match replicator.*>
+  type copy
+  <store>
+    type stdout
+  </store>
+  <store>
+    type mysql_replicator_elasticsearch
+
+    # Set Elasticsearch connection.
+    host localhost
+    port 9200
+
+    # Set Elasticsearch index, type, and unique id (primary_key) from tag.
+    tag_format (?<index_name>[^\.]+)\.(?<type_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
+
+    # Set frequency of sending bulk request to Elasticsearch node.
+    flush_interval 5s
+    
+    # Queued chunks are flushed at shutdown process. (recommend for more stability)
+    # It's sample for td-agent. If you use Yamabiko, replace path from 'td-agent' to 'yamabiko'.
+    flush_at_shutdown yes
+    buffer_type file
+    buffer_path /var/log/td-agent/buffer/mysql_replicator_elasticsearch
+  </store>
+</match>
+`````

data/Tutorial-mysql_replicator_multi.md

@@ -0,0 +1,129 @@
+## Tutorial for Production (mysql_replicator_multi)
+
+It is very useful to replicate a millions of records and/or multiple tables with multiple threads.  
+This architecture is storing hash table in mysql management table instead of ruby internal memory.  
+
+**Note:**  
+On syncing 300 million rows table, it will consume around 20MB of memory with ruby 1.9.3 environment.
+
+### prepare
+
+It has done with follwing two steps.
+
+* create database and tables.
+* add replicator configuration.
+
+##### create database and tables.
+
+```
+$ mysql -umysqluser -p
+
+-- For the first time, load schema.
+mysql> source /path/to/setup_mysql_replicator_multi.sql
+```
+
+see [setup_mysql_replicator_multi.sql](https://github.com/y-ken/fluent-plugin-mysql-replicator/blob/master/setup_mysql_replicator_multi.sql)
+
+##### add replicator configuration.
+
+Let's download sql first.
+
+```
+$ wget https://raw2.github.com/y-ken/fluent-plugin-mysql-replicator/master/setup_mysql_replicator_multi.sql
+```
+
+```sql
+-- Build 
+mysql> source /path/to/setup_mysql_replicator_multi.sql
+
+-- Set working database
+mysql> use replicator_manager;
+
+-- Add replicate source connection and query settings like below.
+mysql> INSERT INTO `settings`
+  (`id`, `is_active`, `name`, `host`, `port`, `username`, `password`, `database`, `query`, `interval`, `primary_key`, `enable_delete`)
+VALUES
+  (NULL, 1, 'mydb.mytable', '192.168.100.221', 3306, 'mysqluser', 'mysqlpassword', 'mydb', 'SELECT id, text from mytable;', 5, 'id', 1);
+```
+
+it is a sample which you have inserted row.
+
+<table>
+<thead><tr>
+<th>id</th>
+<th>is_active</th>
+<th>name</th>
+<th>host</th>
+<th>port</th>
+<th>username</th>
+<th>password</th>
+<th>database</th>
+<th>query</th>
+<th>prepared_query</th>
+<th>interval</th>
+<th>primary_key</th>
+<th>enable_delete</th>
+<th>enable_loose_insert</th>
+<th>enable_loose_delete</th>
+</tr></thead>
+<tbody><tr>
+<td>1</td>
+<td>1</td>
+<td>mydb.mytable</td>
+<td>192.168.100.221</td>
+<td>3306</td>
+<td>mysqluser</td>
+<td>mysqlpassword</td>
+<td>mydb</td>
+<td>SELECT id, text from mytable;</td>
+<td>&nbsp;</td>
+<td>5</td>
+<td>id</td>
+<td>1</td>
+<td>0</td>
+<td>0</td>
+</tr></tbody>
+</table>
+
+### configuration
+
+`````
+<source>
+  type mysql_replicator_multi
+
+  # Database connection setting for manager table.
+  manager_host localhost
+  manager_username your_mysql_user
+  manager_password your_mysql_password
+  manager_database replicator_manager
+
+  # Format output tag for each events. Placeholders usage as described below.
+  tag replicator.${name}.${event}.${primary_key}
+  # ${name} : the value of `replicator_manager.settings.name` in manager table.
+  # ${event} : the variation of row event type by insert/update/delete.
+  # ${primary_key} : the value of `replicator_manager.settings.primary_key` in manager table.
+</source>
+
+<match replicator.**>
+  type mysql_replicator_elasticsearch
+
+  # Set Elasticsearch connection.
+  host localhost
+  port 9200
+
+  # Set Elasticsearch index, type, and unique id (primary_key) from tag.
+  tag_format (?<index_name>[^\.]+)\.(?<type_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
+
+  # Set frequency of sending bulk request to Elasticsearch node.
+  flush_interval 5s
+
+  # Set maximum retry interval (required fluentd >= 0.10.41)
+  max_retry_wait 1800
+
+  # Queued chunks are flushed at shutdown process.
+  # It's sample for td-agent. If you use Yamabiko, replace path from 'td-agent' to 'yamabiko'.
+  flush_at_shutdown yes
+  buffer_type file
+  buffer_path /var/log/td-agent/buffer/mysql_replicator_elasticsearch
+</match>
+`````

data/example/mysql_multi_table_to_elasticsearch.md

@@ -38,6 +38,9 @@ It is a guide to replicate multiple mysql table to elasticsearch.
   max_retry_wait 1800
 
   # Queued chunks are flushed at shutdown process.
+  # It's sample for td-agent. If you use Yamabiko, replace path from 'td-agent' to 'yamabiko'.
   flush_at_shutdown yes
+  buffer_type file
+  buffer_path /var/log/td-agent/buffer/mysql_replicator_elasticsearch
 </match>
-```
+```

data/example/mysql_multi_table_to_solr.md

@@ -39,7 +39,10 @@ It is a guide to replicate multiple mysql table to solr.
   max_retry_wait 1800
 
   # Queued chunks are flushed at shutdown process.
+  # It's sample for td-agent. If you use Yamabiko, replace path from 'td-agent' to 'yamabiko'.
   flush_at_shutdown yes
+  buffer_type file
+  buffer_path /var/log/td-agent/buffer/mysql_replicator_elasticsearch
 </match>
 ```
 
@@ -64,6 +67,9 @@ When you use default core (won't specify), change the value of `tag_format` like
   max_retry_wait 1800
 
   # Queued chunks are flushed at shutdown process.
+  # It's sample for td-agent. If you use Yamabiko, replace path from 'td-agent' to 'yamabiko'.
   flush_at_shutdown yes
+  buffer_type file
+  buffer_path /var/log/td-agent/buffer/mysql_replicator_solr
 </match>
-```
+```

data/example/mysql_single_table_to_elasticsearch.md

@@ -47,6 +47,9 @@ It is a guide to replicate single mysql table to elasticsearch.
   max_retry_wait 1800
 
   # Queued chunks are flushed at shutdown process.
+  # It's sample for td-agent. If you use Yamabiko, replace path from 'td-agent' to 'yamabiko'.
   flush_at_shutdown yes
+  buffer_type file
+  buffer_path /var/log/td-agent/buffer/mysql_replicator_elasticsearch
 </match>
-```
+```

data/example/mysql_single_table_to_solr.md

@@ -48,7 +48,10 @@ It is a guide to replicate single mysql table to solr.
   max_retry_wait 1800
 
   # Queued chunks are flushed at shutdown process.
+  # It's sample for td-agent. If you use Yamabiko, replace path from 'td-agent' to 'yamabiko'.
   flush_at_shutdown yes
+  buffer_type file
+  buffer_path /var/log/td-agent/buffer/mysql_replicator_elasticsearch
 </match>
 ```
 
@@ -74,6 +77,9 @@ On this case, the solr url will be set `http://localhost:8983/solr`
   max_retry_wait 1800
 
   # Queued chunks are flushed at shutdown process.
+  # It's sample for td-agent. If you use Yamabiko, replace path from 'td-agent' to 'yamabiko'.
   flush_at_shutdown yes
+  buffer_type file
+  buffer_path /var/log/td-agent/buffer/mysql_replicator_solr
 </match>
-```
+```

data/fluent-plugin-mysql-replicator.gemspec

@@ -1,11 +1,11 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-mysql-replicator"
-  s.version     = "0.3.1"
+  s.version     = "0.4.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-mysql-replicator"
-  s.summary     = %q{Fluentd input plugin to track insert/update/delete event from MySQL database server. Not only that, it could multiple table replication into Elasticsearch/Solr. It's comming support replicate to another RDB/noSQL.}
+  s.summary     = %q{Fluentd input plugin to track insert/update/delete event from MySQL database server. Not only that, it could multiple table replication and generate nested document for Elasticsearch/Solr. It's comming support replicate to another RDB/noSQL.}
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/fluent/plugin/in_mysql_replicator.rb

@@ -15,6 +15,7 @@ module Fluent
     config_param :database, :string, :default => nil
     config_param :encoding, :string, :default => 'utf8'
     config_param :query, :string
+    config_param :prepared_query, :string, :default => nil
     config_param :primary_key, :string, :default => 'id'
     config_param :interval, :string, :default => '1m'
     config_param :enable_delete, :bool, :default => true
@@ -28,7 +29,7 @@ module Fluent
         raise Fluent::ConfigError, "mysql_replicator: missing 'tag' parameter. Please add following line into config like 'tag replicator.mydatabase.mytable.${event}.${primary_key}'"
       end
 
-      $log.info "adding mysql_replicator worker. :tag=>#{tag} :query=>#{@query} :interval=>#{@interval}sec :enable_delete=>#{enable_delete}"
+      $log.info "adding mysql_replicator worker. :tag=>#{tag} :query=>#{@query} :prepared_query=>#{@prepared_query} :interval=>#{@interval}sec :enable_delete=>#{enable_delete}"
     end
 
     def start
@@ -57,10 +58,20 @@ module Fluent
         start_time = Time.now
         previous_ids = ids
         current_ids = Array.new
+        prepared_con = get_connection()
+        @prepared_query.split(/;/).each do |query|
+          prepared_con.query(query)
+        end
         query(@query).each do |row|
           current_ids << row[@primary_key]
           current_hash = Digest::SHA1.hexdigest(row.flatten.join)
           row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date)}
+          row.select {|k, v| v.to_s.match(/^SELECT/i) }.each do |k, v|
+            row[k] = [] unless row[k].is_a?(Array)
+            prepared_con.query(v.gsub(/\$\{([^\}]+)\}/, row[$1].to_s)).each do |nest_row|
+              row[k] << nest_row
+            end
+          end
           if row[@primary_key].nil?
             $log.error "mysql_replicator: missing primary_key. :tag=>#{tag} :primary_key=>#{primary_key}"
             break
@@ -112,10 +123,10 @@ module Fluent
       Engine.emit(tag, Engine.now, record)
     end
 
-    def query(query)
-      @mysql ||= get_connection
+    def query(query, con = nil)
       begin
-        return @mysql.query(query)
+        mysql = get_connection if con.nil?
+        return mysql.query(query)
       rescue Exception => e
         $log.warn "mysql_replicator: #{e}"
         sleep @interval

data/lib/fluent/plugin/in_mysql_replicator_multi.rb

@@ -75,9 +75,21 @@ module Fluent
         loop do
           rows_count = 0
           start_time = Time.now
+          unless config['prepared_query'].nil?
+            nest_db = get_origin_connection(config)
+            config['prepared_query'].strip.split(/;/).each do |query|
+              nest_db.query(query)
+            end
+          end
           db = get_origin_connection(config)
           db.query(config['query']).each do |row|
             row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date)}
+            row.select {|k, v| v.to_s.match(/^SELECT/i) }.each do |k, v|
+              row[k] = [] unless row[k].is_a?(Array)
+              nest_db.query(v.gsub(/\$\{([^\}]+)\}/) {|matched| row[$1].to_s}).each do |nest_row|
+                row[k] << nest_row
+              end
+            end
             current_id = row[primary_key]
             @mutex.synchronize {
               if row[primary_key].nil?
@@ -148,17 +160,17 @@ module Fluent
       setting_name = config['name']
       if (current_id - previous_id) > 1 && config['enable_loose_delete'] == 0
         query = "SELECT SQL_NO_CACHE setting_query_pk FROM hash_tables
-          WHERE setting_name = '#{setting_name}' 
+          WHERE setting_name = '#{setting_name}'
           AND setting_query_pk > #{previous_id.to_i} AND setting_query_pk < #{current_id.to_i}"
       elsif (current_id - previous_id) > 1 && config['enable_loose_delete'] == 1
         return [*previous_id...current_id] - [current_id,previous_id]
       elsif previous_id > current_id
         query = "SELECT SQL_NO_CACHE setting_query_pk FROM hash_tables
-          WHERE setting_name = '#{setting_name}' 
+          WHERE setting_name = '#{setting_name}'
           AND setting_query_pk > #{previous_id.to_i}"
       elsif previous_id == current_id
         query = "SELECT SQL_NO_CACHE setting_query_pk FROM hash_tables
-          WHERE setting_name = '#{setting_name}' 
+          WHERE setting_name = '#{setting_name}'
           AND (setting_query_pk > #{current_id.to_i} OR setting_query_pk < #{current_id.to_i})"
       end
       ids = Array.new
@@ -217,7 +229,7 @@ module Fluent
 
     def flush_hash_table
       return if @hash_table_bulk_insert.empty?
-      query = "INSERT INTO hash_tables (setting_name,setting_query_pk,setting_query_hash) 
+      query = "INSERT INTO hash_tables (setting_name,setting_query_pk,setting_query_hash)
         VALUES #{@hash_table_bulk_insert.join(',')}
         ON DUPLICATE KEY UPDATE setting_query_hash = VALUES(setting_query_hash)"
       @manager_db.query(query)

data/setup_mysql_replicator_multi.sql

@@ -1,7 +1,7 @@
-CREATE DATABASE replicator_manager;
+CREATE DATABASE IF NOT EXISTS replicator_manager;
 USE replicator_manager;
 
-CREATE TABLE `hash_tables` (
+CREATE TABLE IF NOT EXISTS `hash_tables` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
   `setting_name` varchar(255) NOT NULL,
   `setting_query_pk` int(11) NOT NULL,
@@ -10,7 +10,7 @@ CREATE TABLE `hash_tables` (
   UNIQUE KEY `setting_query_pk` (`setting_query_pk`,`setting_name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
-CREATE TABLE `settings` (
+CREATE TABLE IF NOT EXISTS `settings` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
   `is_active` int(11) NOT NULL DEFAULT '1',
   `name` varchar(255) NOT NULL,
@@ -20,10 +20,14 @@ CREATE TABLE `settings` (
   `password` varchar(255) NOT NULL,
   `database` varchar(255) NOT NULL,
   `query` TEXT NOT NULL,
+  -- Use this field to pre execute query (TEMPORARY TABLE) for improving performance of generating nestd document.
+  `prepared_query` TEXT NOT NULL,
   `interval` int(11) NOT NULL,
   `primary_key` varchar(255) DEFAULT 'id',
   `enable_delete` int(11) DEFAULT '1',
+  -- On enabling 'enable_loose_insert: 1', make it faster synchronization to skip checking hash_tables.
   `enable_loose_insert` int(11) DEFAULT '0',
+  -- On enabling 'enable_loose_delete: 1', turn on speculative delete but performance penalty on non-contiguous primary key.
   `enable_loose_delete` int(11) DEFAULT '0',
   PRIMARY KEY (`id`),
   UNIQUE KEY `name` (`name`)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-replicator
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-07 00:00:00.000000000 Z
+date: 2014-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -104,6 +104,8 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- Tutorial-mysql_replicator.md
+- Tutorial-mysql_replicator_multi.md
 - example/mysql_multi_table_to_elasticsearch.md
 - example/mysql_multi_table_to_solr.md
 - example/mysql_single_table_to_elasticsearch.md
@@ -143,8 +145,8 @@ rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Fluentd input plugin to track insert/update/delete event from MySQL database
-  server. Not only that, it could multiple table replication into Elasticsearch/Solr.
-  It's comming support replicate to another RDB/noSQL.
+  server. Not only that, it could multiple table replication and generate nested document
+  for Elasticsearch/Solr. It's comming support replicate to another RDB/noSQL.
 test_files:
 - test/helper.rb
 - test/plugin/test_in_mysql_replicator.rb