fluent-plugin-mysql-appender 0.4.9 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 424553e560bb98d0602e2f00c13e6301d72967b5
-  data.tar.gz: 189980b5c302be2016f59cc4834d92ce016b31d8
+  metadata.gz: 3fa294a4d025bbf2cde13a939dff99c60c589b32
+  data.tar.gz: a59ab31f182c72ce384a8183ca83d688f76af8e0
 SHA512:
-  metadata.gz: b53d691fb0fd6c040b91d2d2f6dd49220beb381efcb4007e71507e70d40788143ffc49d7d52db06aa517d8fd0d8c597e3f4bf9d144472b23212b108a149cbde2
-  data.tar.gz: dd33875581bd3784ff933f87f9f602d4da29087411b4c164739ee768bf51f398e9c3036d41bcd6e9e89302571ea98e665106232937faa4fe7b2fcf4ec0401e51
+  metadata.gz: a7f11ae093e4761e925112da22eca72f8668f124db5cf9c386eb3dea240f09f20211fd7c828a7e2942e96c04baa4f66289012faa74ba6048fe6f59218d5b024e
+  data.tar.gz: e1dc84259238aabe3629652c2d2884c284f9d596c65d6709c9216c027c4244b7837c72bc4f1009b795bc2f727688953e3c19464ac3b95ac1ad9c47aef3d4c732

data/example/mysql_multi_table_to_tresure_data.md

@@ -2,6 +2,16 @@
 
 It is a guide to replicate multiple mysql table to treasure data.
 
+## environment variables
+
+Please set environment variables.
+
+```
+TD_APIKEY    xxxxxxxx  # Treasure data API key.
+TD_ENDPOINT  api.treasuredata.com  # Treasure data API endpoint. e.g. "api.treasuredata.com".
+TD_DATABASE  sample_db # Treasure data database name.
+```
+
 ## configuration
 
 ```
@@ -20,9 +30,6 @@ It is a guide to replicate multiple mysql table to treasure data.
 
 <match appender_multi.*.*>
   type tdlog
-  endpoint your_td_endpoint
-  apikey your_td_apikey
-
   auto_create_table
   buffer_type file
   buffer_path /var/log/td-agent/buffer/td
@@ -42,26 +49,36 @@ Sample "in_tables.yml" is below.
 
 ```
 - table_name: test_tbl1
-  primary_key: id
-  time_column: created_at
+  primary_key: id  # incremental id
+  time_column: created_at  # assigned to td's time column
   limit: 1000
   columns:
     - id
     - column1
     - column2
-  last_id: -1
-  td_database: sample_datasets
-  entry_time: created_at  # if this column is greater (now - delay), wait insert.
+    - created_at
   delay: 10s
+  entry_time: created_at  # if this column is greater (now - delay), wait insert.
 
 - table_name: test_tbl2
-  primary_key: id
-  time_column: created_at
+  primary_key: id  # incremental id
+  time_column: created_at  # assigned to td's time column
   limit: 1000
   columns:
     - id
     - column1
     - column2
-  last_id: -1
-  td_database: sample_datasets
-```
+    - created_at
+  delay: 10s
+  entry_time: created_at  # if this column is greater (now - delay), wait insert.
+```
+
+```
+select id, column1, column2, created_ad from test_tbl1 where id > {last_id} limit 1000
+```
+
+```
+select id, column1, column2, created_ad from test_tbl2 where id > {last_id} limit 1000
+```
+
+run query in each syncronize loops.

data/fluent-plugin-mysql-appender.gemspec

@@ -1,7 +1,7 @@
 # coding: utf-8
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mysql-appender"
-  spec.version       = "0.4.9"
+  spec.version       = "0.5.0"
   spec.authors       = ["TERASAKI Tsuyoshi"]
   spec.email         = ["tsuyoshi_terasaki@realworld.jp"]
 

data/lib/fluent/plugin/in_mysql_appender_multi.rb

@@ -68,9 +68,8 @@ module Fluent
     def poll(config)
       begin
         tag = format_tag(config)
-        delay = Config.time_value(config['delay'] || 0)
         @mutex.synchronize {
-          $log.info "mysql_appender_multi: polling start. :tag=>#{tag} :delay=>#{delay}"
+          $log.info "mysql_appender_multi: polling start. :tag=>#{tag}"
         }
         last_id = get_lastid(config)
         loop do
@@ -78,20 +77,18 @@ module Fluent
           start_time = Time.now
           db = get_connection
           db.query(get_query(config, last_id)).each do |row|
-            if !config['entry_time'].nil? then
-              entry_time = get_time(row[config['entry_time']])
-              if (start_time - delay) > entry_time then
-                if config['time_column'].nil? then
-                    td_time = Engine.now
-                else
-                  td_time = get_time(row[config['time_column']]).to_i
-                end
-                row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
-                router.emit(tag, td_time, row)
-                rows_count += 1
-                last_id = row[config['primary_key']]
-              end
+            if rows_count > 0 && (last_id.to_i + 1) != row[config['primary_key']] then
+              break
             end
+            if config['time_column'].nil? then
+                td_time = Engine.now
+            else
+              td_time = get_time(row[config['time_column']]).to_i
+            end
+            row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
+            router.emit(tag, td_time, row)
+            rows_count += 1
+            last_id = row[config['primary_key']]
           end
           db.close
           elapsed_time = sprintf("%0.02f", Time.now - start_time)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-appender
 version: !ruby/object:Gem::Version
-  version: 0.4.9
+  version: 0.5.0
 platform: ruby
 authors:
 - TERASAKI Tsuyoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-05 00:00:00.000000000 Z
+date: 2017-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd