fluent-plugin-mysql-replicator 0.1.0 → 0.1.1

data/README.md

@@ -94,6 +94,7 @@ CREATE TABLE `hash_tables` (
 
 CREATE TABLE `settings` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
+  `is_active` int(11) NOT NULL DEFAULT '1',
   `name` varchar(255) NOT NULL,
   `host` varchar(255) NOT NULL DEFAULT 'localhost',
   `port` int(11) NOT NULL DEFAULT '3306',
@@ -102,7 +103,6 @@ CREATE TABLE `settings` (
   `database` varchar(255) NOT NULL,
   `query` TEXT NOT NULL,
   `interval` int(11) NOT NULL,
-  `tag` varchar(255) NOT NULL,
   `primary_key` varchar(11) DEFAULT 'id',
   `enable_delete` int(11) DEFAULT '1',
   PRIMARY KEY (`id`),
@@ -121,19 +121,30 @@ mysql> insert into source ...snip...;
 `````
 <source>
   type mysql_replicator_multi
+
+  # Database connection setting for manager table.
   manager_host localhost
   manager_username your_mysql_user
   manager_password your_mysql_password
   manager_database replicator_manager
+
+  # Format output tag for each events. Placeholders usage as described below.
+  tag replicator.${name}.${event}.${primary_key}
+  # ${name} : the value of `replicator_manager.settings.name` in manager table.
+  # ${event} : the variation of row event type by insert/update/delete.
+  # ${primary_key} : the value of `replicator_manager.settings.primary_key` in manager table.
 </source>
 
-<match replicator.*>
+<match replicator.**>
   type stdout
 </match>
 `````
 
 ## TODO
 
+* support string type primary_key.
+* support reload setting on demand.
+
 Pull requests are very welcome!!
 
 ## Copyright

data/fluent-plugin-mysql-replicator.gemspec

@@ -1,11 +1,11 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-mysql-replicator"
-  s.version     = "0.1.0"
+  s.version     = "0.1.1"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-mysql-replicator"
-  s.summary     = %q{Fluentd input plugin to track insert/update/delete event from MySQL databases. It will support multiple table replication to another RDB/noSQL like Elasticsearch.}
+  s.summary     = %q{Fluentd input plugin to track insert/update/delete event from MySQL databases. It's comming support multiple table replication to another RDB/noSQL like Elasticsearch.}
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/fluent/plugin/in_mysql_replicator.rb

@@ -52,6 +52,7 @@ module Fluent
         query(@query).each do |row|
           current_ids << row[@primary_key]
           current_hash = Digest::SHA1.hexdigest(row.flatten.join)
+          row.each {|k, v| row[k] = v.to_s if v.is_a? Time}
           if !table_hash.include?(row[@primary_key])
             emit_record(:insert, row)
           elsif table_hash[row[@primary_key]] != current_hash

data/lib/fluent/plugin/in_mysql_replicator_multi.rb

@@ -13,10 +13,14 @@ module Fluent
     config_param :manager_username, :string, :default => nil
     config_param :manager_password, :string, :default => ''
     config_param :manager_database, :string, :default => 'replicator_manager'
+    config_param :tag, :string, :default => nil
 
     def configure(conf)
       super
       @reconnect_interval = Config.time_value('10sec')
+      if @tag.nil?
+        raise Fluent::ConfigError, "mysql_replicator_multi: missing 'tag' parameter. Please add following line into config like 'tag replicator.${name}.${event}.${primary_key}'"
+      end
     end
 
     def start
@@ -44,7 +48,7 @@ module Fluent
     def get_settings
       manager_db = get_manager_connection
       settings = []
-      query = "SELECT * FROM settings"
+      query = "SELECT * FROM settings WHERE is_active = 1;"
       manager_db.query(query).each do |row|
         settings << row
       end
@@ -64,6 +68,7 @@ module Fluent
           db = get_origin_connection(config)
           db.query(config['query']).each do |row|
             @mutex.lock
+            row.each {|k, v| row[k] = v.to_s if v.is_a? Time}
             current_id = row[primary_key]
             detect_insert_update(config, row)
             detect_delete(config, current_id, previous_id)
@@ -92,7 +97,8 @@ module Fluent
         event = :update
       end
       unless event.nil?
-        emit_record("#{config['tag']}.#{event.to_s}", row)
+        tag = format_tag(@tag, {:name => config['name'], :event => event, :primary_key => config['primary_key']})
+        emit_record(tag, row)
         update_hashtable({:event => event, :ids => current_id, :setting_name => config['name'], :hash => current_hash})
       end
     end
@@ -110,7 +116,8 @@ module Fluent
       unless deleted_ids.empty?
         event = :delete
         deleted_ids.each do |id|
-          emit_record("#{config['tag']}.#{event.to_s}", {config['primary_key'] => id})
+          tag = format_tag(@tag, {:name => config['name'], :event => event, :primary_key => config['primary_key']})
+          emit_record(tag, {config['primary_key'] => id})
         end
         update_hashtable({:event =>  event, :ids => deleted_ids, :setting_name => config['name']})
       end
@@ -154,6 +161,14 @@ module Fluent
       end
     end
 
+    def format_tag(tag, param)
+      pattern = {'${name}' => param[:name], '${event}' => param[:event].to_s, '${primary_key}' => param[:primary_key]}
+      tag.gsub(/\${[a-z_]+(\[[0-9]+\])?}/, pattern) do
+        $log.warn "mysql_replicator_multi: missing placeholder. tag:#{tag} placeholder:#{$1}" unless pattern.include?($1)
+        pattern[$1]
+      end
+    end
+
     def emit_record(tag, record)
       Engine.emit(tag, Engine.now, record)
     end

data/setup_mysql_replicator_multi.sql

@@ -12,6 +12,7 @@ CREATE TABLE `hash_tables` (
 
 CREATE TABLE `settings` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
+  `is_active` int(11) NOT NULL DEFAULT '1',
   `name` varchar(255) NOT NULL,
   `host` varchar(255) NOT NULL DEFAULT 'localhost',
   `port` int(11) NOT NULL DEFAULT '3306',
@@ -20,7 +21,6 @@ CREATE TABLE `settings` (
   `database` varchar(255) NOT NULL,
   `query` TEXT NOT NULL,
   `interval` int(11) NOT NULL,
-  `tag` varchar(255) NOT NULL,
   `primary_key` varchar(11) DEFAULT 'id',
   `enable_delete` int(11) DEFAULT '1',
   PRIMARY KEY (`id`),

data/test/plugin/test_in_mysql_replicator_multi.rb

@@ -10,6 +10,7 @@ class MysqlReplicatorMultiInputTest < Test::Unit::TestCase
     manager_port      3306
     manager_username  foo
     manager_password  bar
+    tag               replicator.${name}.${event}.${primary_key}
   ]
 
   def create_driver(conf=CONFIG,tag='test')
@@ -17,6 +18,9 @@ class MysqlReplicatorMultiInputTest < Test::Unit::TestCase
   end
 
   def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
     d = create_driver(CONFIG)
     d.instance.inspect
     assert_equal 'localhost', d.instance.manager_host

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-replicator
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-09 00:00:00.000000000 Z
+date: 2013-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -103,7 +103,7 @@ rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Fluentd input plugin to track insert/update/delete event from MySQL databases.
-  It will support multiple table replication to another RDB/noSQL like Elasticsearch.
+  It's comming support multiple table replication to another RDB/noSQL like Elasticsearch.
 test_files:
 - test/helper.rb
 - test/plugin/test_in_mysql_replicator.rb