fluent-plugin-mysql-appender 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b9ddfe2a87e411ef8fa452336cee72c661081f86
-  data.tar.gz: 0eec5b99563117150a8a320d551abfe65335d994
+  metadata.gz: 05ca7cbe8c0b8975c51ba14c2e9f34dcb5e8125c
+  data.tar.gz: 2bff063d49ccb630916f3d515410e93307eec2e3
 SHA512:
-  metadata.gz: 1bb26d2dca219aa6993d71810aa8efcbe1abd285c24b75a57bbff0424c33c6883b9b1822a388d6382d6cef2a2dfa4d7e6802e5b0525dd545a2df68b1d54a832f
-  data.tar.gz: 6e7f1b8b446786df1941ca8eb435c3e26ec4c23d8f3c5179416fcb7080c9327c3b2e7fb880863be34207326eeb6b4f43214b8cfdfeb41ed5c2e95e8c52098872
+  metadata.gz: 05c5048cd4aed2b09bea444c224dc92e5816ff611d4f5d335c6573e08599d4186ee762043f4a1db0d33096b2335a6f2903c53d29977933e170e264f798c538ae
+  data.tar.gz: 72f2b0388fa398595de8463626fbd67c6058e0522c1d8b437a0f92f3a27aa22e56cb9b7c684f91e3efb267a6bb5747e1f63fc131106670a6115670efea97a104

data/fluent-plugin-mysql-appender.gemspec

@@ -1,7 +1,7 @@
 # coding: utf-8
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mysql-appender"
-  spec.version       = "0.4.0"
+  spec.version       = "0.4.1"
   spec.authors       = ["TERASAKI Tsuyoshi"]
   spec.email         = ["tsuyoshi_terasaki@realworld.jp"]
 
@@ -17,6 +17,8 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency "fluentd"
   spec.add_runtime_dependency "mysql2"
+  spec.add_runtime_dependency "td"
+  spec.add_runtime_dependency "td-client"
 
   spec.add_development_dependency "rake"
   spec.add_development_dependency "test-unit"

data/lib/fluent/plugin/in_mysql_appender_multi.rb

@@ -11,6 +11,8 @@ module Fluent
       require 'mysql2'
       require 'time'
       require 'yaml'
+      require 'td'
+      require 'td-client'
       super
     end
 
@@ -71,7 +73,7 @@ module Fluent
           $log.info "mysql_replicator_multi: polling start. :tag=>#{tag} :delay=>#{delay}"
         }
         con = get_connection()
-        last_id = config['last_id']
+        last_id = get_lastid(config)
         loop do
           rows_count = 0
           start_time = Time.now
@@ -107,6 +109,44 @@ module Fluent
       end
     end
 
+    def get_lastid(config)
+      begin
+        if !ENV.key?('TD_APIKEY') || !ENV.key?('TD_ENDPOINT') then
+          return -1
+        end
+        cln = TreasureData::Client.new(ENV['TD_APIKEY'],{:endpoint => "https://" + ENV['TD_ENDPOINT']})
+        table_exists = false
+        cln.databases.each { |db|
+          db.tables.each { |tbl|
+            if tbl.db_name == config['td_database'] && tbl.table_name == config['table_name'] then
+                table_exists = true
+                break
+            end
+          }
+        }
+        if table_exists then
+          query = "SELECT MAX(#{config['primary_key']}) FROM #{config['table_name']}"
+          job = cln.query(config['td_database'], query, nil, nil, nil , {:type => :presto})
+          until job.finished?
+            sleep 2
+            job.update_progress!
+          end
+          job.update_status!  # get latest info
+          job.result_each { |row|
+            $log.info  "mysql_replicator_multi: #{config['td_database']}.#{config['table_name']}'s last_id is #{row.first} "
+            return row.first
+          }
+        else
+          $log.info "mysql_replicator_multi: #{config['td_database']}.#{config['table_name']} is not found. "
+          return -1
+        end
+      rescue => e
+        $log.warn "mysql_appender_multi: failed to get lastid. #{config}"
+        $log.error "error: #{e.message}"
+        $log.error e.backtrace.join("\n")
+      end
+    end
+
     def get_query(config, last_id)
       "SELECT #{config['columns'].join(",")} FROM #{config['table_name']} where #{config['primary_key']} > #{last_id} order by #{config['primary_key']} asc limit #{config['limit']}"
     end
@@ -141,7 +181,7 @@ module Fluent
           :stream => true,
           :cache_rows => false
         })
-      rescue Exception => e
+      rescue Mysql2::Error => e
         $log.warn "mysql_appender_multi: #{e}"
         sleep @interval
         retry

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-appender
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - TERASAKI Tsuyoshi
@@ -38,6 +38,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: td
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: td-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement