fluent-plugin-mysql-appender 0.1.6 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7b69ab435ca209ad259788a0e6c4110cc22ee339
-  data.tar.gz: 24b99fb9a0a9ace7240c7b92d740ae3590425e3d
+  metadata.gz: be73928d90d0747392f99de85df361a0306d534b
+  data.tar.gz: bda6f8c57cd29be8957921b6cf6ed4b5bbebd6ed
 SHA512:
-  metadata.gz: d21432fe704fbd97539f7bd9dbffeb9dba8200ab3af2b10daed1ac82925f024cad5d3a5dcc9fe4ed856148ec48d1f9d00bb3a2d34c5be00010b5cf82460e837a
-  data.tar.gz: 78d8cad493d54cd553de76621ef9cb4aa610ab77334f99eec092dd65aaf30601e64a8d1a4d5cbc472f2a91995efd1662596fa8c2d5a43553fe047af560f8cdae
+  metadata.gz: d24412a3c37128320e4cfbf93127e2e67616353f21f4bb226539daf2f7c42be6cc67c592ba53ed8b6b53890e8841a833bc1408700ddce6dfbb4121dc3b2856be
+  data.tar.gz: cf45032f22aa50410a4e5e5f51d75b2335a96453a8701696213734d94c737afb4e134a25ecc8db53787a584777f5177467494be5c69dbf77a114ad5cfa173026

data/Rakefile

@@ -2,9 +2,9 @@ require "bundler/gem_tasks"
 require "rake/testtask"
 
 Rake::TestTask.new(:test) do |t|
-  t.libs << "test"
-  t.libs << "lib"
-  t.test_files = FileList['test/**/*_test.rb']
+  t.libs << "lib" << 'test'
+  t.pattern = 'test/**/test_*.rb'
+  t.verbose = true
 end
 
 task :default => :test

data/fluent-plugin-mysql-appender.gemspec

@@ -1,7 +1,7 @@
 # coding: utf-8
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mysql-appender"
-  spec.version       = "0.1.6"
+  spec.version       = "0.2.0"
   spec.authors       = ["TERASAKI Tsuyoshi"]
   spec.email         = ["tsuyoshi_terasaki@realworld.jp"]
 
@@ -18,8 +18,6 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "fluentd"
   spec.add_runtime_dependency "mysql2"
 
-  spec.add_development_dependency "bundler", "~> 1.13"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "minitest", "~> 5.0"
-
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "test-unit"
 end

data/lib/fluent/plugin/in_mysql_appender_multi.rb

@@ -0,0 +1,141 @@
+module Fluent
+  class MysqlAppenderMultiInput < Fluent::Input
+    Plugin.register_input('mysql_appender_multi', self)
+
+    # Define `router` method to support v0.10.57 or earlier
+    unless method_defined?(:router)
+      define_method("router") { Engine }
+    end
+
+    def initialize
+      require 'mysql2'
+      require 'time'
+      require 'yaml'
+      super
+    end
+
+    config_param :host, :string, :default => 'localhost'
+    config_param :port, :integer, :default => 3306
+    config_param :username, :string, :default => 'root'
+    config_param :password, :string, :default => nil, :secret => true
+    config_param :database, :string, :default => nil
+    config_param :encoding, :string, :default => 'utf8'
+    config_param :interval, :string, :default => '1m'
+    config_param :tag, :string, :default => 'appender_multi'
+    config_param :yaml_path, :string, :default=> nil
+
+    def configure(conf)
+      super
+      if @tag.nil?
+        raise Fluent::ConfigError, "mysql_appender_multi: missing 'tag' parameter. Please add following line into config like 'tag appender.${name}.${event}.${primary_key}'"
+      end
+    end
+
+    def start
+      begin
+        @threads = []
+        @mutex = Mutex.new
+        YAML.load_file(@yaml_path).each do |config|
+          @threads << Thread.new {
+            poll(config)
+          }
+        end
+        $log.error "mysql_appender_multi: stop working due to empty configuration" if @threads.empty?
+      rescue StandardError => e
+        $log.error "error: #{e.message}"
+        $log.error e.backtrace.join("\n")
+      end
+    end
+
+    def shutdown
+      @threads.each do |thread|
+        Thread.kill(thread)
+      end
+    end
+
+    def poll(config)
+      begin
+        @mutex.synchronize {
+          $log.info "mysql_appender_multi: polling start. :config=>#{masked_config}"
+        }
+        con = get_connection()
+        last_id = config['last_id']
+        loop do
+          rows_count = 0
+          start_time = Time.now
+          rows, con = query(get_query(config, last_id), con)
+          rows.each_with_index do |row, index|
+            tag = format_tag(config)
+            if @time_column.nil? then
+                td_time = Engine.now
+            else
+                if row[@time_column].kind_of?(Time) then
+                  td_time = row[config['time_column']].to_i
+                else
+                  td_time = Time.parse(row[config['time_column']].to_s).to_i
+                end
+            end
+            row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
+            router.emit(tag, td_time, row)
+            rows_count += 1
+            if index == rows.size - 1
+              @last_id = row[config['primary_key']]
+            end
+          end
+          con.close
+          elapsed_time = sprintf("%0.02f", Time.now - start_time)
+          @mutex.synchronize {
+            $log.info "mysql_appender: finished execution :tag=>#{tag} :rows_count=>#{rows_count} :last_id=>#{last_id} :elapsed_time=>#{elapsed_time} sec"
+          }
+          sleep @interval
+        end
+      rescue StandardError => e
+        @mutex.synchronize {
+          $log.error "mysql_appender_multi: failed to execute query. :config=>#{masked_config}"
+          $log.error "error: #{e.message}"
+          $log.error e.backtrace.join("\n")
+        }
+      end
+    end
+
+    def get_query(config, last_id)
+      "SELECT #{config['columns'].join(",")} FROM #{config['table_name']} where #{config['primary_key']} > #{last_id} order by #{config['primary_key']} asc limit #{config['limit']}"
+    end
+
+    def query(query, con = nil)
+      begin
+        con = con.nil? ? get_connection : con
+        con = con.ping ? con : get_connection
+        return con.query(query), con
+      rescue Exception => e
+        $log.warn "mysql_appender_multi: #{e}"
+        sleep @interval
+        retry
+      end
+    end
+
+    def format_tag(config)
+      "#{tag}.#{config['database']}.#{config['table_name']}"
+    end
+
+    def get_connection
+      begin
+        return Mysql2::Client.new({
+          :host => @host,
+          :port => @port,
+          :username => @username,
+          :password => @password,
+          :database => @database,
+          :encoding => @encoding,
+          :reconnect => true,
+          :stream => true,
+          :cache_rows => false
+        })
+      rescue Exception => e
+        $log.warn "mysql_appender: #{e}"
+        sleep @interval
+        retry
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,30 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+require 'fluent/input'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/in_mysql_appender'
+require 'fluent/plugin/in_mysql_appender_multi'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_in_mysql_appender.rb

@@ -0,0 +1,23 @@
+require 'helper'
+
+class MysqlAppenderInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::MysqlAppenderInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver %[
+      host            localhost
+      interval        30
+      tag             input.mysql
+      query           SELECT id, text from search_text
+    ]
+    assert_equal 'localhost', d.instance.host
+    assert_equal 30, d.instance.interval
+    assert_equal 'input.mysql', d.instance.tag
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-appender
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.2.0
 platform: ruby
 authors:
 - TERASAKI Tsuyoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-10 00:00:00.000000000 Z
+date: 2016-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -38,48 +38,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.13'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.13'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '0'
 description: Simple incremental id's insert.
 email:
 - tsuyoshi_terasaki@realworld.jp
@@ -95,8 +81,9 @@ files:
 - Rakefile
 - fluent-plugin-mysql-appender.gemspec
 - lib/fluent/plugin/in_mysql_appender.rb
-- test/fluent/plugin/in_mysql_appender_test.rb
-- test/test_helper.rb
+- lib/fluent/plugin/in_mysql_appender_multi.rb
+- test/helper.rb
+- test/plugin/test_in_mysql_appender.rb
 homepage: https://github.com/rw-hub/fluent-plugin-mysql-appender
 licenses:
 - MIT
@@ -122,5 +109,5 @@ signing_key:
 specification_version: 4
 summary: Fluentd input plugin to insert from MySQL database server.
 test_files:
-- test/fluent/plugin/in_mysql_appender_test.rb
-- test/test_helper.rb
+- test/helper.rb
+- test/plugin/test_in_mysql_appender.rb

data/test/fluent/plugin/in_mysql_appender_test.rb

@@ -1,8 +0,0 @@
-require 'test_helper'
-
-class MysqlAppenderInputTest < Minitest::Test
-
-  def test_it_does_something_useful
-    assert true
-  end
-end

data/test/test_helper.rb

@@ -1,3 +0,0 @@
-require 'fluent/plugin/in_mysql_appender'
-
-require 'minitest/autorun'