fluent-plugin-mysql-appender 0.3.2 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 06cdb575e833edb1b752b8b21f22305c47f6ce66
-  data.tar.gz: 480964a2ecd5ed0bed3dfc317a47ecb2ef06671a
+  metadata.gz: 9bea290735b18887dd55d09efceaa35bc43b0193
+  data.tar.gz: 18d31c5eaea15dbeea7c3e4b95f78ac1ec63fbc7
 SHA512:
-  metadata.gz: 00599555ae75841c444bde00b3272cc06ffd7c69cb7e7937e06211edefd8101c4d069ba43d575ace8e4793858faeb3497535bcd2ee44a93d1e2acf393ea3eaf1
-  data.tar.gz: 48e844a6a9078b6482a6a3859197e370b19beb1f651dfa3d69f531e9fc61dc01a7a3fae5219795cdfb44dc7a0d988edfcfeaeeebac3e18e1c9419aec32d6519e
+  metadata.gz: 4fa61e94df8ce4ba988f60d56a2284230933e49feaf8f3fe17945aca4f19d250aaf226f3f6b5870b543e923bc5d6eefaf4ea5d9080ca74aec3b6a31c3d33c899
+  data.tar.gz: 5c6ff76d81458910aac3b0fa1df2376f276008360126dcb024476a1a3fcca46615dbdfc8dc625dc616185b012261ffb3553f99fdedbd04d626201999d8bea154

data/example/mysql_multi_table_to_tresure_data.md

@@ -50,6 +50,7 @@ Sample "in_tables.yml" is below.
     - column1
     - column2
   last_id: -1
+  buffer: 10   # last 10 records don't append (default 0).
   td_database: sample_datasets
 
 - table_name: test_tbl2
@@ -61,5 +62,6 @@ Sample "in_tables.yml" is below.
     - column1
     - column2
   last_id: -1
+  buffer: 10   # last 10 records don't append (default 0).
   td_database: sample_datasets
 ```

data/example/mysql_single_table_to_treasure_data.md

@@ -25,6 +25,7 @@ It is a guide to replicate single mysql table to treasure data.
   time_column created_at # specify TIME column.
   limit   1000 # query limit
   last_id -1   # specify primary_key start
+  buffer  10   # last 10 records don't append (default 0).
 </source>
 
 <match appender.*.*>

data/fluent-plugin-mysql-appender.gemspec

@@ -1,7 +1,7 @@
 # coding: utf-8
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mysql-appender"
-  spec.version       = "0.3.2"
+  spec.version       = "0.3.3"
   spec.authors       = ["TERASAKI Tsuyoshi"]
   spec.email         = ["tsuyoshi_terasaki@realworld.jp"]
 

data/lib/fluent/plugin/in_mysql_appender.rb

@@ -26,6 +26,7 @@ module Fluent
     config_param :primary_key, :string, :default => 'id'
     config_param :interval, :string, :default => '1m'
     config_param :tag, :string, :default => nil
+    config_param :buffer, :integer, :defalut => 0
 
     def configure(conf)
       super
@@ -63,22 +64,27 @@ module Fluent
         start_time = Time.now
         select_query = @query.gsub(/"/,'') + " where #{primary_key} > #{last_id} order by #{primary_key} asc limit #{limit}"
         rows, con = query(select_query, con)
-        rows.each_with_index do |row, index|
-          tag = format_tag(@tag, {:event => :insert})
-          if @time_column.nil? then
-              td_time = Engine.now
-          else
-              if row[@time_column].kind_of?(Time) then
-                td_time = row[@time_column].to_i
-              else
-                td_time = Time.parse(row[@time_column].to_s).to_i
-              end
+        if @buffer < rows.size then
+          if index == (rows.size - @buffer) then
+            break
           end
-          row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
-          router.emit(tag, td_time, row)
-          rows_count += 1
-          if index == rows.size - 1
-            @last_id = row[@primary_key]
+          rows.each_with_index do |row, index|
+            tag = format_tag(@tag, {:event => :insert})
+            if @time_column.nil? then
+                td_time = Engine.now
+            else
+                if row[@time_column].kind_of?(Time) then
+                  td_time = row[@time_column].to_i
+                else
+                  td_time = Time.parse(row[@time_column].to_s).to_i
+                end
+            end
+            row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
+            router.emit(tag, td_time, row)
+            rows_count += 1
+            if index == rows.size - @buffer - 1
+              @last_id = row[@primary_key]
+            end
           end
         end
         con.close

data/lib/fluent/plugin/in_mysql_appender_multi.rb

@@ -29,11 +29,11 @@ module Fluent
       @interval = Config.time_value(@interval)
 
       if @yaml_path.nil?
-        raise Fluent::ConfigError, "mysql_appender_multi: missing 'yaml_path' parameter or file not found."
+        raise Fluent::ConfigError, "mysql_appender_multi: missing 'yaml_path' parameter."
       end
 
       if !File.exist?(@yaml_path)
-        raise Fluent::ConfigError, "mysql_appender_multi: 'yaml_path' No such file."
+        raise Fluent::ConfigError, "mysql_appender_multi: No such file in 'yaml_path'."
       end
 
       if @tag.nil?
@@ -66,6 +66,7 @@ module Fluent
     def poll(config)
       begin
         tag = format_tag(config)
+        buffer = config['buffer'] || 0
         @mutex.synchronize {
           $log.info "mysql_replicator_multi: polling start. :tag=>#{tag}"
         }
@@ -75,21 +76,26 @@ module Fluent
           rows_count = 0
           start_time = Time.now
           rows, con = query(get_query(config, last_id), con)
-          rows.each_with_index do |row, index|
-            if config['time_column'].nil? then
-                td_time = Engine.now
-            else
-                if row[config['time_column']].kind_of?(Time) then
-                  td_time = row[config['time_column']].to_i
-                else
-                  td_time = Time.parse(row[config['time_column']].to_s).to_i
-                end
-            end
-            row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
-            router.emit(tag, td_time, row)
-            rows_count += 1
-            if index == rows.size - 1
-              last_id = row[config['primary_key']]
+          if buffer < rows.size then
+            rows.each_with_index do |row, index|
+              if index == (rows.size - buffer) then
+                break
+              end
+              if config['time_column'].nil? then
+                  td_time = Engine.now
+              else
+                  if row[config['time_column']].kind_of?(Time) then
+                    td_time = row[config['time_column']].to_i
+                  else
+                    td_time = Time.parse(row[config['time_column']].to_s).to_i
+                  end
+              end
+              row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
+              router.emit(tag, td_time, row)
+              rows_count += 1
+              if index == (rows.size - buffer - 1)
+                last_id = row[config['primary_key']]
+              end
             end
           end
           con.close

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-appender
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors:
 - TERASAKI Tsuyoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-17 00:00:00.000000000 Z
+date: 2016-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd