fluent-plugin-mysql-appender 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a58e4a794e52d951be2a80bc19277fc1272da17b
-  data.tar.gz: b5f596a5a4b08abe49ec27bacbf6a860dd2fbfd7
+  metadata.gz: e8aa25405eceba3f36603a7f675dfb3db91f457b
+  data.tar.gz: ca191d84cf467bf6010ce68ce02ddadcee54a414
 SHA512:
-  metadata.gz: abfd5cfdc4878e92cb5529b7c1efdc3d6505d414f072577dbc008596e7a9fc8199bb9e836a66423f008950d33ac62efe16258ff0f671f523a7b4eeb31a6eaf12
-  data.tar.gz: 8376106e8272e6a382dd98b77e987bcebe55a3f362388dbf099e202318e7f55115f29a1a87d5c7cf41fbecbef119c08b3d59c837ea477ab746a708554ca1c203
+  metadata.gz: aebe8bf4e89a0bf3d9d7ee9beb15f42ab792043905a4347230ccc1df9006430b0e8b47e961aae37311c854daa66c69e1900a8d3a05a5ad64f8b4dafdf3596051
+  data.tar.gz: 2a48bd6a93ab07536a8e5bff25482f64a1955325243559cb487633d8d17b1de9709d34f27c60fe47f71c5bd8fcc916a786e8993ea643e534f968799050a1af05

data/fluent-plugin-mysql-appender.gemspec

@@ -1,11 +1,7 @@
 # coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'fluent/plugin/mysql/appender/version'
-
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mysql-appender"
-  spec.version       = Fluent::Plugin::Mysql::Appender::VERSION
+  spec.version       = "0.1.1"
   spec.authors       = ["tsuyoshi_terasaki"]
   spec.email         = ["tsuyoshi_terasaki@realworld.jp"]
 
@@ -14,17 +10,16 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://github.com/rw-hub/fluent-plugin-mysql-appender"
   spec.license       = "MIT"
 
-  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
-    f.match(%r{^(test|spec|features)/})
-  end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.files         = `git ls-files`.split($\)
+  spec.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.add_runtime_dependency "fluentd"
+  spec.add_runtime_dependency "mysql2"
+
   spec.add_development_dependency "bundler", "~> 1.13"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest", "~> 5.0"
 
-  spec.add_runtime_dependency "fluentd"
-  spec.add_runtime_dependency "mysql2"
 end

data/lib/fluent/plugin/in_mysql_appender.rb

@@ -0,0 +1,122 @@
+module Fluent
+  class MysqlAppenderInput < Fluent::Input
+    Plugin.register_input('mysql_appender', self)
+
+    # Define `router` method to support v0.10.57 or earlier
+    unless method_defined?(:router)
+      define_method("router") { Engine }
+    end
+
+    def initialize
+      require 'mysql2'
+      super
+    end
+
+    config_param :host, :string, :default => 'localhost'
+    config_param :port, :integer, :default => 3306
+    config_param :username, :string, :default => 'root'
+    config_param :password, :string, :default => nil, :secret => true
+    config_param :database, :string, :default => nil
+    config_param :encoding, :string, :default => 'utf8'
+    config_param :query, :string
+    config_param :primary_key, :string, :default => 'id'
+    config_param :interval, :string, :default => '1m'
+    config_param :tag, :string, :default => nil
+
+    def configure(conf)
+      super
+      @interval = Config.time_value(@interval)
+
+      if @tag.nil?
+        raise Fluent::ConfigError, "mysql_appender: missing 'tag' parameter. Please add following line into config like 'tag replicator.mydatabase.mytable.${event}.${primary_key}'"
+      end
+
+      $log.info "adding mysql_appender worker. :tag=>#{tag} :query=>#{@query} :prepared_query=>#{@prepared_query} :interval=>#{@interval}sec"
+    end
+
+    def start
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      Thread.kill(@thread)
+    end
+
+    def run
+      begin
+        poll
+      rescue StandardError => e
+        $log.error "mysql_appender: failed to execute query."
+        $log.error "error: #{e.message}"
+        $log.error e.backtrace.join("\n")
+      end
+    end
+
+    def poll
+      con = get_connection()
+      max_id = -1
+      loop do
+        rows_count = 0
+        start_time = Time.now
+        if max_id == -1
+          select_query = @query + " order by #{primary_key} asc"
+        else
+          select_query = @query + " where #{primary_key} > #{max_id} order by #{primary_key} asc"
+        end
+        rows, con = query(select_query, con)
+        rows.each_with_index do |row, index|
+          tag = format_tag(@tag, {:event => :insert})
+          router.emit(tag, Engine.now, row)
+          rows_count += 1
+          if index == rows.size - 1
+            max_id = row[@primary_key]
+          end
+        end
+        con.close
+        elapsed_time = sprintf("%0.02f", Time.now - start_time)
+        $log.info "mysql_appender: finished execution :tag=>#{tag} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
+        sleep @interval
+      end
+    end
+
+    def format_tag(tag, param)
+      pattern = {'${event}' => param[:event].to_s, '${primary_key}' => @primary_key}
+      tag.gsub(/(\${[a-z_]+})/) do
+        $log.warn "mysql_appender: missing placeholder. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
+        pattern[$1]
+      end
+    end
+
+    def query(query, con = nil)
+      begin
+        con = con.nil? ? get_connection : con
+        con = con.ping ? con : get_connection
+        return con.query(query), con
+      rescue Exception => e
+        $log.warn "mysql_appender: #{e}"
+        sleep @interval
+        retry
+      end
+    end
+
+    def get_connection
+      begin
+        return Mysql2::Client.new({
+          :host => @host,
+          :port => @port,
+          :username => @username,
+          :password => @password,
+          :database => @database,
+          :encoding => @encoding,
+          :reconnect => true,
+          :stream => true,
+          :cache_rows => false
+        })
+      rescue Exception => e
+        $log.warn "mysql_appender: #{e}"
+        sleep @interval
+        retry
+      end
+    end
+  end
+end

data/test/fluent/plugin/in_mysql_appender_test.rb

@@ -0,0 +1,8 @@
+require 'test_helper'
+
+class MysqlAppenderInputTest < Minitest::Test
+
+  def test_it_does_something_useful
+    assert true
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,3 @@
+require 'fluent/plugin/in_mysql_appender'
+
+require 'minitest/autorun'

metadata

@@ -1,85 +1,85 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-appender
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - tsuyoshi_terasaki
 autorequire: 
-bindir: exe
+bindir: bin
 cert_chain: []
-date: 2016-11-07 00:00:00.000000000 Z
+date: 2016-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.13'
-  type: :development
+        version: '0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.13'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: mysql2
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
-  type: :development
+        version: '0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '1.13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '1.13'
 - !ruby/object:Gem::Dependency
-  name: fluentd
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: '10.0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.0'
 - !ruby/object:Gem::Dependency
-  name: mysql2
+  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: '5.0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '5.0'
 description: Simple incremental id's insert.
 email:
 - tsuyoshi_terasaki@realworld.jp
@@ -93,11 +93,10 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/console
-- bin/setup
 - fluent-plugin-mysql-appender.gemspec
-- lib/fluent/plugin/mysql/appender.rb
-- lib/fluent/plugin/mysql/appender/version.rb
+- lib/fluent/plugin/in_mysql_appender.rb
+- test/fluent/plugin/in_mysql_appender_test.rb
+- test/test_helper.rb
 homepage: https://github.com/rw-hub/fluent-plugin-mysql-appender
 licenses:
 - MIT
@@ -122,4 +121,6 @@ rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Fluentd input plugin to track insert event from MySQL database server.
-test_files: []
+test_files:
+- test/fluent/plugin/in_mysql_appender_test.rb
+- test/test_helper.rb

data/bin/console

@@ -1,14 +0,0 @@
-#!/usr/bin/env ruby
-
-require "bundler/setup"
-require "fluent/plugin/mysql/appender"
-
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
-IRB.start

data/bin/setup

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-IFS=$'\n\t'
-set -vx
-
-bundle install
-
-# Do any other automated setup that you need to do here

data/lib/fluent/plugin/mysql/appender/version.rb

@@ -1,9 +0,0 @@
-module Fluent
-  module Plugin
-    module Mysql
-      module Appender
-        VERSION = "0.1.0"
-      end
-    end
-  end
-end

data/lib/fluent/plugin/mysql/appender.rb

@@ -1,130 +0,0 @@
-require "fluent/plugin/mysql/appender/version"
-
-module Fluent
-  module Plugin
-    module Mysql
-      module Appender
-        class MysqlAppenderInput < Fluent::Input
-          Plugin.register_input('mysql_appender', self)
-
-          # Define `router` method to support v0.10.57 or earlier
-          unless method_defined?(:router)
-            define_method("router") { Engine }
-          end
-
-          def initialize
-            require 'mysql2'
-            super
-          end
-
-          config_param :host, :string, :default => 'localhost'
-          config_param :port, :integer, :default => 3306
-          config_param :username, :string, :default => 'root'
-          config_param :password, :string, :default => nil, :secret => true
-          config_param :database, :string, :default => nil
-          config_param :encoding, :string, :default => 'utf8'
-          config_param :query, :string
-          config_param :primary_key, :string, :default => 'id'
-          config_param :interval, :string, :default => '1m'
-          config_param :tag, :string, :default => nil
-
-          def configure(conf)
-            super
-            @interval = Config.time_value(@interval)
-
-            if @tag.nil?
-              raise Fluent::ConfigError, "mysql_appender: missing 'tag' parameter. Please add following line into config like 'tag replicator.mydatabase.mytable.${event}.${primary_key}'"
-            end
-
-            $log.info "adding mysql_appender worker. :tag=>#{tag} :query=>#{@query} :prepared_query=>#{@prepared_query} :interval=>#{@interval}sec"
-          end
-
-          def start
-            @thread = Thread.new(&method(:run))
-          end
-
-          def shutdown
-            Thread.kill(@thread)
-          end
-
-          def run
-            begin
-              poll
-            rescue StandardError => e
-              $log.error "mysql_appender: failed to execute query."
-              $log.error "error: #{e.message}"
-              $log.error e.backtrace.join("\n")
-            end
-          end
-
-          def poll
-            con = get_connection()
-            max_id = -1
-            loop do
-              rows_count = 0
-              start_time = Time.now
-              if max_id == -1
-                select_query = @query + " order by #{primary_key} asc"
-              else
-                select_query = @query + " where #{primary_key} > #{max_id} order by #{primary_key} asc"
-              end
-              rows, con = query(select_query, con)
-              rows.each_with_index do |row, index|
-                tag = format_tag(@tag, {:event => :insert})
-                router.emit(tag, Engine.now, row)
-                rows_count += 1
-                if index == rows.size - 1
-                  max_id = row[@primary_key]
-                end
-              end
-              con.close
-              elapsed_time = sprintf("%0.02f", Time.now - start_time)
-              $log.info "mysql_appender: finished execution :tag=>#{tag} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
-              sleep @interval
-            end
-          end
-
-          def format_tag(tag, param)
-            pattern = {'${event}' => param[:event].to_s, '${primary_key}' => @primary_key}
-            tag.gsub(/(\${[a-z_]+})/) do
-              $log.warn "mysql_appender: missing placeholder. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
-              pattern[$1]
-            end
-          end
-
-          def query(query, con = nil)
-            begin
-              con = con.nil? ? get_connection : con
-              con = con.ping ? con : get_connection
-              return con.query(query), con
-            rescue Exception => e
-              $log.warn "mysql_appender: #{e}"
-              sleep @interval
-              retry
-            end
-          end
-
-          def get_connection
-            begin
-              return Mysql2::Client.new({
-                :host => @host,
-                :port => @port,
-                :username => @username,
-                :password => @password,
-                :database => @database,
-                :encoding => @encoding,
-                :reconnect => true,
-                :stream => true,
-                :cache_rows => false
-              })
-            rescue Exception => e
-              $log.warn "mysql_appender: #{e}"
-              sleep @interval
-              retry
-            end
-          end
-        end
-      end
-    end
-  end
-end