fluent-plugin-mysql-appender 0.4.8 → 0.4.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c019cd62a986d24066118816d73c07bc47d767d2
-  data.tar.gz: 2bd589f13b322685e6b4b29a1c7fcea50ad6be3a
+  metadata.gz: 424553e560bb98d0602e2f00c13e6301d72967b5
+  data.tar.gz: 189980b5c302be2016f59cc4834d92ce016b31d8
 SHA512:
-  metadata.gz: d118c38a6a772f5968af1ab3e98e122ad3997847d02762e506d864c695448569903c031ba4c200df37eb5ca75dc9cc1b40de57dd5a7098445f54d7e54ac7c974
-  data.tar.gz: 497d35ab4ebcd2d5fd866cba96a378cd1508ef15e02eaffe284e4b26945a4f7d4803efe3f943ba8444ffdf6be153f818fdec5bf4f97f0b22342bd2dad471653b
+  metadata.gz: b53d691fb0fd6c040b91d2d2f6dd49220beb381efcb4007e71507e70d40788143ffc49d7d52db06aa517d8fd0d8c597e3f4bf9d144472b23212b108a149cbde2
+  data.tar.gz: dd33875581bd3784ff933f87f9f602d4da29087411b4c164739ee768bf51f398e9c3036d41bcd6e9e89302571ea98e665106232937faa4fe7b2fcf4ec0401e51

data/fluent-plugin-mysql-appender.gemspec

@@ -1,7 +1,7 @@
 # coding: utf-8
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mysql-appender"
-  spec.version       = "0.4.8"
+  spec.version       = "0.4.9"
   spec.authors       = ["TERASAKI Tsuyoshi"]
   spec.email         = ["tsuyoshi_terasaki@realworld.jp"]
 

data/lib/fluent/plugin/in_mysql_appender_multi.rb

@@ -37,51 +37,67 @@ module Fluent
       if !File.exist?(@yaml_path)
         raise Fluent::ConfigError, "mysql_appender_multi: No such file in 'yaml_path'."
       end
+
+      if @tag.nil?
+        raise Fluent::ConfigError, "mysql_appender_multi: missing 'tag' parameter. Please add following line into config like 'tag appender.${name}.${event}.${primary_key}'"
+      end
     end
 
     def start
-      @thread = Thread.new(&method(:run))
+      begin
+        @threads = []
+        @mutex = Mutex.new
+        YAML.load_file(@yaml_path).each do |config|
+          @threads << Thread.new {
+            poll(config)
+          }
+        end
+        $log.error "mysql_appender_multi: stop working due to empty configuration" if @threads.empty?
+      rescue => e
+        $log.error "error: #{e.message}"
+        $log.error e.backtrace.join("\n")
+      end
     end
 
     def shutdown
-      Thread.kill(@thread)
+      @threads.each do |thread|
+        Thread.kill(thread)
+      end
     end
 
-    private
-
-    def run
+    def poll(config)
       begin
-        # initialize
-        configs = YAML.load_file(@yaml_path)
-        configs.each do |config|
-          config['last_id'] = get_lastid(config)
-          config['tag'] = format_tag(config)
-          config['delay'] = Config.time_value(config['delay'] || 0)
-        end
-
+        tag = format_tag(config)
+        delay = Config.time_value(config['delay'] || 0)
+        @mutex.synchronize {
+          $log.info "mysql_appender_multi: polling start. :tag=>#{tag} :delay=>#{delay}"
+        }
+        last_id = get_lastid(config)
         loop do
+          rows_count = 0
           start_time = Time.now
           db = get_connection
-          configs.each do |config|
-            rows_count = 0
-            db.query(get_query(config)).each do |row|
+          db.query(get_query(config, last_id)).each do |row|
+            if !config['entry_time'].nil? then
               entry_time = get_time(row[config['entry_time']])
-              if (start_time - config['delay']) > entry_time then
+              if (start_time - delay) > entry_time then
                 if config['time_column'].nil? then
                     td_time = Engine.now
                 else
                   td_time = get_time(row[config['time_column']]).to_i
                 end
                 row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
-                # puts row
-                router.emit(config['tag'], td_time, row)
+                router.emit(tag, td_time, row)
                 rows_count += 1
-                config['last_id'] = row[config['primary_key']]
+                last_id = row[config['primary_key']]
               end
             end
-            $log.info "mysql_appender_multi: finished execution :tag=>#{config['tag']} :rows_count=>#{rows_count} :last_id=>#{config['last_id']}"
           end
           db.close
+          elapsed_time = sprintf("%0.02f", Time.now - start_time)
+          @mutex.synchronize {
+            $log.info "mysql_appender_multi: finished execution :tag=>#{tag} :rows_count=>#{rows_count} :last_id=>#{last_id} :elapsed_time=>#{elapsed_time} sec"
+          }
           sleep @interval
         end
       rescue => e
@@ -129,8 +145,8 @@ module Fluent
       end
     end
 
-    def get_query(config)
-      "SELECT #{config['columns'].join(",")} FROM #{config['table_name']} where #{config['primary_key']} > #{config['last_id']} order by #{config['primary_key']} asc limit #{config['limit']}"
+    def get_query(config, last_id)
+      "SELECT #{config['columns'].join(",")} FROM #{config['table_name']} where #{config['primary_key']} > #{last_id} order by #{config['primary_key']} asc limit #{config['limit']}"
     end
 
     def format_tag(config)

data/setup_sample_db.sql

@@ -12,18 +12,3 @@ insert into test_tbl1 (column1, column2) value ('3','4');
 insert into test_tbl1 (column1, column2) value ('5','6');
 insert into test_tbl1 (column1, column2) value ('7','8');
 
-create table test_tbl2 (
-    id int auto_increment,
-    column1 varchar(10),
-    column2 varchar(10),
-    created_at timestamp default current_timestamp,
-    primary key(id)
-);
-
-insert into test_tbl2 (column1, column2) value ('A','E');
-insert into test_tbl2 (column1, column2) value ('B','F');
-insert into test_tbl2 (column1, column2) value ('C','G');
-insert into test_tbl2 (column1, column2) value ('D','H');
-
-
-

data/test/plugin/test_in_mysql_appender_multi.rb

@@ -38,20 +38,7 @@ class MysqlAppenderMultiInputTest < Test::Unit::TestCase
     - created_at
   last_id: -1
   entry_time: created_at
-  delay: 5m
-  td_database: sample_datasets
-- table_name: test_tbl2
-  primary_key: id
-  time_column: created_at
-  limit: 1000
-  columns:
-    - id
-    - column1
-    - column2
-    - created_at
-  last_id: -1
-  entry_time: created_at
-  delay: 5m
+  delay: 3h
   td_database: sample_datasets
 EOS
     conf = YAML.load(str)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-appender
 version: !ruby/object:Gem::Version
-  version: 0.4.8
+  version: 0.4.9
 platform: ruby
 authors:
 - TERASAKI Tsuyoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-02 00:00:00.000000000 Z
+date: 2016-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd