fluent-plugin-mysql-replicator 0.1.1 → 0.2.0

data/README.md

@@ -2,7 +2,9 @@
 
 ## Overview
 
-Fluentd input plugin to track insert/update/delete event from MySQL database server.
+Fluentd input plugin to track insert/update/delete event from MySQL database server.  
+Not only that, it could multiple table replication into Elasticsearch nodes.  
+It's comming support replicate to another RDB/noSQL.
 
 ## Installation
 
@@ -24,26 +26,56 @@ It is useful for these purpose.
 **Note:**  
 On syncing 300 million rows table, it will consume around 800MB of memory with ruby 1.9.3 environment.
 
-#### configuration
+### configuration
 
 `````
 <source>
   type mysql_replicator
+
+  # Set connection settings for replicate source.
   host localhost
   username your_mysql_user
   password your_mysql_password
   database myweb
-  interval 5s
-  tag replicator
-  query SELECT id, text from search_test
+
+  # Set replicate query configuration.
+  query SELECT id, text, updated_at from search_test;
+  primary_key id # specify unique key (default: id)
+  interval 10s  # execute query interval (default: 1m)
+
+  # Enable detect deletion event not only insert/update events. (default: yes)
+  # It is useful to use `enable_delete no` that keep following recently updated record with this query.
+  # `SELECT * FROM search_test WHERE DATE_ADD(updated_at, INTERVAL 5 MINUTE) > NOW();`
+  enable_delete yes
+
+  # Format output tag for each events. Placeholders usage as described below.
+  tag replicator.myweb.search_test.${event}.${primary_key}
+  # ${event} : the variation of row event type by insert/update/delete.
+  # ${primary_key} : the value of `replicator_manager.settings.primary_key` in manager table.
 </source>
 
 <match replicator.*>
-  type stdout
+  type copy
+  <store>
+    type stdout
+  </store>
+  <store>
+    type mysql_replicator_elasticsearch
+
+    # Set Elasticsearch connection.
+    host localhost
+    port 9200
+
+    # Set Elasticsearch index, type, and unique id (primary_key) from tag.
+    tag_format (?<index_name>[^\.]+)\.(?<type_name>[^\.]+).(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
+
+    # Set frequency of sending bulk request to Elasticsearch node.
+    flush_interval 5s
+  </store>
 </match>
 `````
 
-#### sample query
+### sample query
 
 `````
 $ mysql -e "create database myweb"
@@ -56,13 +88,13 @@ $ sleep 10
 $ mysql myweb -e "delete from search_test where text='bbb'"
 `````
 
-#### result
+### result
 
 `````
 $ tail -f /var/log/td-agent/td-agent.log
-2013-11-25 18:22:25 +0900 replicator.insert: {"id":"1","text":"aaa"}
-2013-11-25 18:22:35 +0900 replicator.update: {"id":"1","text":"bbb"}
-2013-11-25 18:22:45 +0900 replicator.delete: {"id":"1"}
+2013-11-25 18:22:25 +0900 replicator.insert.id: {"id":"1","text":"aaa"}
+2013-11-25 18:22:35 +0900 replicator.update.id: {"id":"1","text":"bbb"}
+2013-11-25 18:22:45 +0900 replicator.delete.id: {"id":"1"}
 `````
 
 ## Tutorial for Production
@@ -73,11 +105,15 @@ This architecture is storing hash table in mysql management table instead of rub
 **Note:**  
 On syncing 300 million rows table, it will consume around 20MB of memory with ruby 1.9.3 environment.
 
-#### prepare
+### prepare
+
+It has done with follwing two steps.
 
 * create database and tables.
 * add replicator configuration.
 
+##### create database and tables.
+
 ```
 $ cat setup_mysql_replicator_multi.sql
 CREATE DATABASE replicator_manager;
@@ -110,13 +146,28 @@ CREATE TABLE `settings` (
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 ```
 
+##### add replicator configuration.
+
 ```
-$ mysql
+$ mysql -umysqluser -p
+
+-- For the first time, load schema.
 mysql> source /path/to/setup_mysql_replicator_multi.sql
-mysql> insert into source ...snip...;
+
+-- Add replicate source connection and query settings like below.
+mysql> INSERT INTO `settings`
+  (`id`, `is_active`, `name`, `host`, `port`, `username`, `password`, `database`, `query`, `interval`, `primary_key`, `enable_delete`)
+VALUES
+  (NULL, 1, 'mydb.mytable', '192.168.100.221', 3306, 'mysqluser', 'mysqlpassword', 'mydb', 'SELECT id, text from mytable;', 5, 'id', 1);
 ```
 
-#### configuration
+it is a sample which you have inserted row.
+
+| id | is_active |     name     |      host       | port | username  |   password    | database |            query             | interval | primary_key | enable_delete |
+|----|-----------|--------------|-----------------|------|-----------|---------------|----------|------------------------------|----------|-------------|---------------|
+|  1 |         1 | mydb.mytable | 192.168.100.221 | 3306 | mysqluser | mysqlpassword | mydb     | SELECT id, text from mytable; |       5 | id          |             1 |
+
+### configuration
 
 `````
 <source>
@@ -136,16 +187,27 @@ mysql> insert into source ...snip...;
 </source>
 
 <match replicator.**>
-  type stdout
+  type mysql_replicator_elasticsearch
+
+  # Set Elasticsearch connection.
+  host localhost
+  port 9200
+
+  # Set Elasticsearch index, type, and unique id (primary_key) from tag.
+  tag_format (?<index_name>[^\.]+)\.(?<type_name>[^\.]+).(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
+
+  # Set frequency of sending bulk request to Elasticsearch node.
+  flush_interval 5s
 </match>
 `````
 
 ## TODO
 
-* support string type primary_key.
-* support reload setting on demand.
+Pull requests are very welcome like below!!
 
-Pull requests are very welcome!!
+* more tests.
+* support string type of primary_key.
+* support reload setting on demand.
 
 ## Copyright
 

data/fluent-plugin-mysql-replicator.gemspec

@@ -1,11 +1,11 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-mysql-replicator"
-  s.version     = "0.1.1"
+  s.version     = "0.2.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-mysql-replicator"
-  s.summary     = %q{Fluentd input plugin to track insert/update/delete event from MySQL databases. It's comming support multiple table replication to another RDB/noSQL like Elasticsearch.}
+  s.summary     = %q{Fluentd input plugin to track insert/update/delete event from MySQL database server. Not only that, it could multiple table replication into Elasticsearch nodes. It's comming support replicate to another RDB/noSQL.}
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
@@ -13,6 +13,8 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_development_dependency "rake"
+  s.add_development_dependency "webmock"
+
   s.add_runtime_dependency "fluentd"
   s.add_runtime_dependency "mysql2"
 end

data/lib/fluent/plugin/in_mysql_replicator.rb

@@ -14,16 +14,21 @@ module Fluent
     config_param :password, :string, :default => nil
     config_param :database, :string, :default => nil
     config_param :encoding, :string, :default => 'utf8'
-    config_param :interval, :string, :default => '1m'
-    config_param :tag, :string
     config_param :query, :string
     config_param :primary_key, :string, :default => 'id'
-    config_param :enable_delete, :bool, :default => 'yes'
+    config_param :interval, :string, :default => '1m'
+    config_param :enable_delete, :bool, :default => true
+    config_param :tag, :string, :default => nil
 
     def configure(conf)
       super
       @interval = Config.time_value(@interval)
-      $log.info "adding mysql_replicator job: [#{@query}] interval: #{@interval}sec"
+
+      if @tag.nil?
+        raise Fluent::ConfigError, "mysql_replicator: missing 'tag' parameter. Please add following line into config like 'tag replicator.mydatabase.mytable.${event}.${primary_key}'"
+      end
+
+      $log.info "adding mysql_replicator worker. :tag=>#{tag} :query=>[#{@query}] :interval=>#{@interval}sec :enable_delete=>#{enable_delete}"
     end
 
     def start
@@ -54,18 +59,23 @@ module Fluent
           current_hash = Digest::SHA1.hexdigest(row.flatten.join)
           row.each {|k, v| row[k] = v.to_s if v.is_a? Time}
           if !table_hash.include?(row[@primary_key])
-            emit_record(:insert, row)
+            tag = format_tag(@tag, {:event => :insert})
+            emit_record(tag, row)
           elsif table_hash[row[@primary_key]] != current_hash
-            emit_record(:update, row)
+            tag = format_tag(@tag, {:event => :update})
+            emit_record(tag, row)
           end
           table_hash[row[@primary_key]] = current_hash
         end
         ids = current_ids
-        unless @enable_delete
+        if @enable_delete
           deleted_ids = previous_ids - current_ids
           if deleted_ids.count > 0
             hash_delete_by_list(table_hash, deleted_ids)
-            deleted_ids.each {|id| emit_record(:delete, {@primary_key => id})}
+            deleted_ids.each do |id| 
+              tag = format_tag(@tag, {:event => :delete})
+              emit_record(tag, {@primary_key => id})
+            end
           end
         end
         sleep @interval
@@ -76,15 +86,22 @@ module Fluent
       deleted_keys.each{|k| hash.delete(k)}
     end
 
-    def emit_record(type, record)
-      tag = "#{@tag}.#{type.to_s}"
+    def format_tag(tag, param)
+      pattern = {'${event}' => param[:event].to_s, '${primary_key}' => @primary_key}
+      tag.gsub(/\${[a-z_]+(\[[0-9]+\])?}/, pattern) do
+        $log.warn "mysql_replicator: missing placeholder. tag:#{tag} placeholder:#{$1}" unless pattern.include?($1)
+        pattern[$1]
+      end
+    end
+
+    def emit_record(tag, record)
       Engine.emit(tag, Engine.now, record)
     end
 
     def query(query)
       @mysql ||= get_connection
       begin
-        return @mysql.query(query, :cast => false, :cache_rows => false)
+        return @mysql.query(query)
       rescue Exception => e
         $log.warn "mysql_replicator: #{e}"
         sleep @interval

data/lib/fluent/plugin/out_mysql_replicator_elasticsearch.rb

@@ -0,0 +1,67 @@
+require 'net/http'
+require 'date'
+
+class Fluent::MysqlReplicatorElasticsearchOutput < Fluent::BufferedOutput
+  Fluent::Plugin.register_output('mysql_replicator_elasticsearch', self)
+
+  config_param :host, :string,  :default => 'localhost'
+  config_param :port, :integer, :default => 9200
+  config_param :tag_format, :string, :default => nil
+
+  DEFAULT_TAG_FORMAT = /(?<index_name>[^\.]+)\.(?<type_name>[^\.]+).(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$/
+
+  def initialize
+    super
+  end
+
+  def configure(conf)
+    super
+
+    if @tag_format.nil? || @tag_format == DEFAULT_TAG_FORMAT
+      @tag_format = DEFAULT_TAG_FORMAT
+    else
+      @tag_format = Regexp.new(conf['tag_format'])
+    end
+  end
+
+  def start
+    super
+  end
+
+  def format(tag, time, record)
+    [tag, time, record].to_msgpack
+  end
+
+  def shutdown
+    super
+  end
+
+  def write(chunk)
+    bulk_message = []
+
+    chunk.msgpack_each do |tag, time, record|
+      tag_parts = tag.match(@tag_format)
+      target_index = tag_parts['index_name']
+      target_type = tag_parts['type_name']
+      id_key = tag_parts['primary_key']
+
+      if tag_parts['event'] == 'delete'
+        meta = { "delete" => {"_index" => target_index, "_type" => target_type, "_id" => record[id_key]} }
+        bulk_message << Yajl::Encoder.encode(meta)
+      else
+        meta = { "index" => {"_index" => target_index, "_type" => target_type} }
+        if id_key && record[id_key]
+          meta['index']['_id'] = record[id_key]
+        end
+        bulk_message << Yajl::Encoder.encode(meta)
+        bulk_message << Yajl::Encoder.encode(record)
+      end
+    end
+    bulk_message << ""
+
+    http = Net::HTTP.new(@host, @port.to_i)
+    request = Net::HTTP::Post.new('/_bulk', {'content-type' => 'application/json; charset=utf-8'})
+    request.body = bulk_message.join("\n")
+    http.request(request).value
+  end
+end

data/test/helper.rb

@@ -24,6 +24,7 @@ end
 
 require 'fluent/plugin/in_mysql_replicator'
 require 'fluent/plugin/in_mysql_replicator_multi'
+require 'fluent/plugin/out_mysql_replicator_elasticsearch'
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_out_mysql_replicator_elasticsearch.rb

@@ -0,0 +1,127 @@
+require 'helper'
+require 'webmock/test_unit'
+
+WebMock.disable_net_connect!
+
+class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
+  attr_accessor :index_cmds, :content_type
+
+  def setup
+    Fluent::Test.setup
+    @driver = nil
+  end
+
+  def driver(tag='myindex.mytype.insert.id', conf='')
+    @driver ||= Fluent::Test::BufferedOutputTestDriver.new(Fluent::MysqlReplicatorElasticsearchOutput, tag).configure(conf)
+  end
+
+  def sample_record
+    {'age' => 26, 'request_id' => '42'}
+  end
+
+  def stub_elastic(url="http://localhost:9200/_bulk")
+    stub_request(:post, url).with do |req|
+      @content_type = req.headers["Content-Type"]
+      @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
+    end
+  end
+
+  def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
+    stub_request(:post, url).to_return(:status => [503, "Service Unavailable"])
+  end
+
+  def test_wrties_with_proper_content_type
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_equal("application/json; charset=utf-8", @content_type)
+  end
+
+  def test_writes_to_speficied_index
+    driver.configure("index_name myindex\n")
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_equal('myindex', index_cmds.first['index']['_index'])
+  end
+
+  def test_writes_to_speficied_type
+    driver.configure("type_name mytype\n")
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_equal('mytype', index_cmds.first['index']['_type'])
+  end
+
+  def test_writes_to_speficied_host
+    driver.configure("host 192.168.33.50\n")
+    elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
+    driver.emit(sample_record)
+    driver.run
+    assert_requested(elastic_request)
+  end
+
+  def test_writes_to_speficied_port
+    driver.configure("port 9201\n")
+    elastic_request = stub_elastic("http://localhost:9201/_bulk")
+    driver.emit(sample_record)
+    driver.run
+    assert_requested(elastic_request)
+  end
+
+  def test_makes_bulk_request
+    stub_elastic
+    driver.emit(sample_record)
+    driver.emit(sample_record.merge('age' => 27))
+    driver.run
+    assert_equal(4, index_cmds.count)
+  end
+
+  def test_all_records_are_preserved_in_bulk
+    stub_elastic
+    driver.emit(sample_record)
+    driver.emit(sample_record.merge('age' => 27))
+    driver.run
+    assert_equal(26, index_cmds[1]['age'])
+    assert_equal(27, index_cmds[3]['age'])
+  end
+
+
+  def test_doesnt_add_logstash_timestamp_by_default
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_nil(index_cmds[1]['@timestamp'])
+  end
+
+
+  def test_doesnt_add_tag_key_by_default
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_nil(index_cmds[1]['tag'])
+  end
+
+  def test_doesnt_add_id_key_if_missing_when_configured
+    driver.configure("id_key another_request_id\n")
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(!index_cmds[0]['index'].has_key?('_id'))
+  end
+
+  def test_adds_id_key_when_not_configured
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(!index_cmds[0]['index'].has_key?('_id'))
+  end
+
+  def test_request_error
+    stub_elastic_unavailable
+    driver.emit(sample_record)
+    assert_raise(Net::HTTPFatalError) {
+      driver.run
+    }
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-replicator
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -27,6 +27,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -75,10 +91,12 @@ files:
 - fluent-plugin-mysql-replicator.gemspec
 - lib/fluent/plugin/in_mysql_replicator.rb
 - lib/fluent/plugin/in_mysql_replicator_multi.rb
+- lib/fluent/plugin/out_mysql_replicator_elasticsearch.rb
 - setup_mysql_replicator_multi.sql
 - test/helper.rb
 - test/plugin/test_in_mysql_replicator.rb
 - test/plugin/test_in_mysql_replicator_multi.rb
+- test/plugin/test_out_mysql_replicator_elasticsearch.rb
 homepage: https://github.com/y-ken/fluent-plugin-mysql-replicator
 licenses: []
 post_install_message: 
@@ -102,9 +120,11 @@ rubyforge_project:
 rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
-summary: Fluentd input plugin to track insert/update/delete event from MySQL databases.
-  It's comming support multiple table replication to another RDB/noSQL like Elasticsearch.
+summary: Fluentd input plugin to track insert/update/delete event from MySQL database
+  server. Not only that, it could multiple table replication into Elasticsearch nodes.
+  It's comming support replicate to another RDB/noSQL.
 test_files:
 - test/helper.rb
 - test/plugin/test_in_mysql_replicator.rb
 - test/plugin/test_in_mysql_replicator_multi.rb
+- test/plugin/test_out_mysql_replicator_elasticsearch.rb