fluent-plugin-mysql-appender 0.3.5 → 0.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b9f6408d1a7b224768ddef9852ce5f68344b15c7
-  data.tar.gz: 628c15bf44d0d25a1495652f4bc5aa291dad8749
+  metadata.gz: c19611c88c6fc54e11e3e6c0e8febc24b0f36050
+  data.tar.gz: 245ffee5532f8c5b5ce032801ebf77bb8b46a031
 SHA512:
-  metadata.gz: 0414ae7c3bb515fc587e1e950da5e543787c98721f33a950eb086ddfd8b6056724bdb157c9ff666dbb2632ed48ef8e8af337d9c16614415610972ecce2f92a6e
-  data.tar.gz: a4813bb12472e8c40134dc46d8ae9e9e3983dda2c410ebac99277c4f2a8c847e20e17543bf0a5f8a0178a7e4f09ac77d17c0c2cab51f19c450429801b22a443e
+  metadata.gz: 51ddd238c9f8e72ad588db213f67b8d313fad3f045b28bfcf3207623763b1f214cdb457f0236495b9ec818bfc60b0fa00aa2c704355e64ebfbba2aa79d783e5e
+  data.tar.gz: 0fcef13e7bde505fbba8203dfdeac464d643191a831325a38fe009c49b73cf45f98294c2a84ce98ce251ad7c72fa2a58efc27603722bfbcea6d6557082811011

data/example/mysql_multi_table_to_tresure_data.md

@@ -51,6 +51,8 @@ Sample "in_tables.yml" is below.
     - column2
   last_id: -1
   td_database: sample_datasets
+  entry_time: created_at  # this column compare (now - delay) time
+  delay: 10s
 
 - table_name: test_tbl2
   primary_key: id

data/fluent-plugin-mysql-appender.gemspec

@@ -1,7 +1,7 @@
 # coding: utf-8
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mysql-appender"
-  spec.version       = "0.3.5"
+  spec.version       = "0.3.6"
   spec.authors       = ["TERASAKI Tsuyoshi"]
   spec.email         = ["tsuyoshi_terasaki@realworld.jp"]
 
@@ -20,4 +20,5 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "rake"
   spec.add_development_dependency "test-unit"
+  spec.add_development_dependency "test-unit-rr"
 end

data/lib/fluent/plugin/in_mysql_appender_multi.rb

@@ -18,7 +18,7 @@ module Fluent
     config_param :port, :integer, :default => 3306
     config_param :username, :string, :default => 'root'
     config_param :password, :string, :default => nil, :secret => true
-    config_param :database, :string, :default => nil
+    config_param :database, :string, :default => 'sample_db'
     config_param :encoding, :string, :default => 'utf8'
     config_param :interval, :string, :default => '1m'
     config_param :tag, :string, :default => 'appender_multi'
@@ -29,11 +29,11 @@ module Fluent
       @interval = Config.time_value(@interval)
 
       if @yaml_path.nil?
-        raise Fluent::ConfigError, "mysql_appender_multi: missing 'yaml_path' parameter or file not found."
+        raise Fluent::ConfigError, "mysql_appender_multi: missing 'yaml_path' parameter."
       end
 
       if !File.exist?(@yaml_path)
-        raise Fluent::ConfigError, "mysql_appender_multi: 'yaml_path' No such file."
+        raise Fluent::ConfigError, "mysql_appender_multi: No such file in 'yaml_path'."
       end
 
       if @tag.nil?
@@ -66,8 +66,9 @@ module Fluent
     def poll(config)
       begin
         tag = format_tag(config)
+        delay = Config.time_value(config['delay'] || 0)
         @mutex.synchronize {
-          $log.info "mysql_replicator_multi: polling start. :tag=>#{tag}"
+          $log.info "mysql_replicator_multi: polling start. :tag=>#{tag} :delay=>#{delay}"
         }
         con = get_connection()
         last_id = config['last_id']
@@ -76,21 +77,22 @@ module Fluent
           start_time = Time.now
           rows, con = query(get_query(config, last_id), con)
           rows.each_with_index do |row, index|
+            if !config['entry_time'].nil? then
+              entry_time = get_time(row[config['entry_time']])
+              if (Time.now - delay) < entry_time then
+                last_id = row[config['primary_key']]
+                break
+              end
+            end
             if config['time_column'].nil? then
                 td_time = Engine.now
             else
-                if row[config['time_column']].kind_of?(Time) then
-                  td_time = row[config['time_column']].to_i
-                else
-                  td_time = Time.parse(row[config['time_column']].to_s).to_i
-                end
+              td_time = get_time(row[config['time_column']]).to_i
             end
             row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
             router.emit(tag, td_time, row)
             rows_count += 1
-            if index == rows.size - 1
-              last_id = row[config['primary_key']]
-            end
+            last_id = row[config['primary_key']]
           end
           con.close
           elapsed_time = sprintf("%0.02f", Time.now - start_time)
@@ -148,5 +150,13 @@ module Fluent
         retry
       end
     end
+
+    def get_time(in_time)
+      if in_time.kind_of?(Time) then
+        in_time
+      else
+        Time.parse(in_time.to_s)
+      end
+    end
   end
 end

data/setup_sample_db.sql

@@ -0,0 +1,14 @@
+create database sample_db;
+create table test_tbl1 (
+    id int auto_increment,
+    column1 varchar(10),
+    column2 varchar(10),
+    created_at timestamp default current_timestamp,
+    primary key(id)
+);
+
+insert into test_tbl1 (column1, column2) value ('1','2');
+insert into test_tbl1 (column1, column2) value ('3','4');
+insert into test_tbl1 (column1, column2) value ('5','6');
+insert into test_tbl1 (column1, column2) value ('7','8');
+

data/test/helper.rb

@@ -7,12 +7,18 @@ rescue Bundler::BundlerError => e
   $stderr.puts "Run `bundle install` to install missing gems"
   exit e.status_code
 end
+
 require 'test/unit'
+require 'test/unit/rr'
+require 'yaml'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
+
 require 'fluent/test'
 require 'fluent/input'
+require 'fluent/log'
+
 unless ENV.has_key?('VERBOSE')
   nulllogger = Object.new
   nulllogger.instance_eval {|obj|

data/test/plugin/test_in_mysql_appender_multi.rb

@@ -0,0 +1,52 @@
+require 'helper'
+
+class MysqlAppenderMultiInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    # stubs
+    stub(File).exist?{:true}
+  end
+
+  CONFIG = %[
+    host            localhost
+    interval        30
+    yaml_path       hoge
+  ]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::MysqlAppenderMultiInput).configure(conf)
+  end
+
+  def test_configure
+    # run test
+    d = create_driver
+    assert_equal 'localhost', d.instance.host
+    assert_equal 30, d.instance.interval
+    assert_equal 'appender_multi', d.instance.tag
+  end
+
+  def test_polling
+    str = <<EOS
+- table_name: test_tbl1
+  primary_key: id
+  time_column: created_at
+  limit: 1000
+  columns:
+    - id
+    - column1
+    - column2
+    - created_at
+  last_id: -1
+  entry_time: created_at
+  delay: 3h
+  td_database: sample_datasets
+EOS
+    conf = YAML.load(str)
+    stub(YAML).load_file{conf}
+
+    d = create_driver
+    d.run do
+      sleep 2
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-appender
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.6
 platform: ruby
 authors:
 - TERASAKI Tsuyoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-24 00:00:00.000000000 Z
+date: 2016-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Simple incremental id's insert.
 email:
 - tsuyoshi_terasaki@realworld.jp
@@ -84,8 +98,10 @@ files:
 - fluent-plugin-mysql-appender.gemspec
 - lib/fluent/plugin/in_mysql_appender.rb
 - lib/fluent/plugin/in_mysql_appender_multi.rb
+- setup_sample_db.sql
 - test/helper.rb
 - test/plugin/test_in_mysql_appender.rb
+- test/plugin/test_in_mysql_appender_multi.rb
 homepage: https://github.com/realworld-jp/fluent-plugin-mysql-appender
 licenses:
 - MIT
@@ -113,3 +129,4 @@ summary: Fluentd input plugin to insert from MySQL database server.
 test_files:
 - test/helper.rb
 - test/plugin/test_in_mysql_appender.rb
+- test/plugin/test_in_mysql_appender_multi.rb