fluent-plugin-mysql-appender 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f21781c970449bb44a1aab51ea2ed1078a6c7fb9
-  data.tar.gz: 8c4e17ccd6b87360897f067a506ad92d7b81ea47
+  metadata.gz: ca968bd1cfa4b493d5175521b3ed90422d2aeb33
+  data.tar.gz: 1bc085367f4a27ac9b1c5b9196f232f6be031fe8
 SHA512:
-  metadata.gz: 3925652f1b00a0958682aa71f66404560bd36af4298511a8d460eb1dd67d957fc9a4f79cba5db62658decf1d89d0092186d38ee4a1a61c1ad2e213008fc58b08
-  data.tar.gz: df0b83ad3bd612549e0b825e7c20c0fc34546a7fe0c6dae5c1a1ac1c58185f88b78d71695ee31623ea47349d1e387457b7083fa26cb89fe9791c6946d9597802
+  metadata.gz: 3da542d0fd2f0e93929cb6f9490bf438093bcfe05c3cfbb20b61c5d24ada44dccc3f8856e033de62a40e4997038f20b473511dcf3e29322ab73d05790ea7a6ad
+  data.tar.gz: 159468925304ee7b688c068c004ac105cd06e8b0bb2d9f108a0bb1c0ae0decf9d3be5535a5c55fade50d95b99e8595476281dd5b63455ecbe4f46aee74618acb

data/fluent-plugin-mysql-appender.gemspec

@@ -1,11 +1,11 @@
 # coding: utf-8
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mysql-appender"
-  spec.version       = "0.1.3"
-  spec.authors       = ["tsuyoshi_terasaki"]
+  spec.version       = "0.1.4"
+  spec.authors       = ["TERASAKI Tsuyoshi"]
   spec.email         = ["tsuyoshi_terasaki@realworld.jp"]
 
-  spec.summary       = %q{Fluentd input plugin to track insert event from MySQL database server.}
+  spec.summary       = %q{Fluentd input plugin to insert from MySQL database server.}
   spec.description   = %q{Simple incremental id's insert.}
   spec.homepage      = "https://github.com/rw-hub/fluent-plugin-mysql-appender"
   spec.license       = "MIT"

data/lib/fluent/plugin/in_mysql_appender.rb

@@ -18,7 +18,10 @@ module Fluent
     config_param :password, :string, :default => nil, :secret => true
     config_param :database, :string, :default => nil
     config_param :encoding, :string, :default => 'utf8'
+    config_param :last_id, :integer, :default => -1
+    config_param :limit, :integer, :default => 100
     config_param :query, :string
+    config_param :time_column, :string, :default => nil
     config_param :primary_key, :string, :default => 'id'
     config_param :interval, :string, :default => '1m'
     config_param :tag, :string, :default => nil
@@ -31,7 +34,7 @@ module Fluent
         raise Fluent::ConfigError, "mysql_appender: missing 'tag' parameter. Please add following line into config like 'tag replicator.mydatabase.mytable.${event}.${primary_key}'"
       end
 
-      $log.info "adding mysql_appender worker. :tag=>#{tag} :query=>#{@query} :prepared_query=>#{@prepared_query} :interval=>#{@interval}sec"
+      $log.info "adding mysql_appender worker. :tag=>#{tag} :query=>#{@query} :limit=>#{limit} :interval=>#{@interval} sec "
     end
 
     def start
@@ -54,28 +57,28 @@ module Fluent
 
     def poll
       con = get_connection()
-      max_id = -1
       loop do
         rows_count = 0
         start_time = Time.now
-        if max_id == -1
-          select_query = @query.gsub(/"/,'') + " order by #{primary_key} asc"
-        else
-          select_query = @query.gsub(/"/,'') + " where #{primary_key} > #{max_id} order by #{primary_key} asc"
-        end
+        select_query = @query.gsub(/"/,'') + " where #{primary_key} > #{last_id} order by #{primary_key} asc limit #{limit}"
         rows, con = query(select_query, con)
         rows.each_with_index do |row, index|
           tag = format_tag(@tag, {:event => :insert})
+          if @time_column.nil? then
+              td_time = Engine.now
+          else
+              td_time = Time.parse(row[@time_column]).to_i
+          end
           row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
-          router.emit(tag, Engine.now, row)
+          router.emit(tag, td_time, row)
           rows_count += 1
           if index == rows.size - 1
-            max_id = row[@primary_key]
+            @last_id = row[@primary_key]
           end
         end
         con.close
         elapsed_time = sprintf("%0.02f", Time.now - start_time)
-        $log.info "mysql_appender: finished execution :tag=>#{tag} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
+        $log.info "mysql_appender: finished execution :tag=>#{tag} :rows_count=>#{rows_count} :last_id=>#{last_id} :elapsed_time=>#{elapsed_time} sec"
         sleep @interval
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-appender
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
-- tsuyoshi_terasaki
+- TERASAKI Tsuyoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-08 00:00:00.000000000 Z
+date: 2016-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -120,7 +120,7 @@ rubyforge_project:
 rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
-summary: Fluentd input plugin to track insert event from MySQL database server.
+summary: Fluentd input plugin to insert from MySQL database server.
 test_files:
 - test/fluent/plugin/in_mysql_appender_test.rb
 - test/test_helper.rb