fluent-plugin-mysql-appender 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 835274ff289ba1176e9aaccf84278be8572bbd0a
-  data.tar.gz: 2b1eeebdff20fc94cd2a4166f70f7abe7df539bf
+  metadata.gz: a58e4a794e52d951be2a80bc19277fc1272da17b
+  data.tar.gz: b5f596a5a4b08abe49ec27bacbf6a860dd2fbfd7
 SHA512:
-  metadata.gz: 0d9e9db4e075088608f536667cc5353cb69e7a3378cac072d6a48392170c34ee7b68bd9b772b2f0f68d452c6728db844c520857cbf35f23568bc15094b20376c
-  data.tar.gz: 8f2e3eb258ae7e88eebb4cbdc90d5b0ab3cd26c9131f2ae9c4863c965f45c577c4a0441789505fa5b0d8d7885a700924ca49d9895cbd6e5da28610d9f935e94d
+  metadata.gz: abfd5cfdc4878e92cb5529b7c1efdc3d6505d414f072577dbc008596e7a9fc8199bb9e836a66423f008950d33ac62efe16258ff0f671f523a7b4eeb31a6eaf12
+  data.tar.gz: 8376106e8272e6a382dd98b77e987bcebe55a3f362388dbf099e202318e7f55115f29a1a87d5c7cf41fbecbef119c08b3d59c837ea477ab746a708554ca1c203

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "fluent/plugin/mysql/appender"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/fluent-plugin-mysql-appender.gemspec

@@ -1,7 +1,11 @@
 # coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fluent/plugin/mysql/appender/version'
+
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mysql-appender"
-  spec.version       = "0.0.1"
+  spec.version       = Fluent::Plugin::Mysql::Appender::VERSION
   spec.authors       = ["tsuyoshi_terasaki"]
   spec.email         = ["tsuyoshi_terasaki@realworld.jp"]
 
@@ -10,16 +14,17 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://github.com/rw-hub/fluent-plugin-mysql-appender"
   spec.license       = "MIT"
 
-  spec.files         = `git ls-files`.split($\)
-  spec.executables   = spec.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "fluentd"
-  spec.add_runtime_dependency "mysql2"
-
   spec.add_development_dependency "bundler", "~> 1.13"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest", "~> 5.0"
 
+  spec.add_runtime_dependency "fluentd"
+  spec.add_runtime_dependency "mysql2"
 end

data/lib/fluent/plugin/mysql/appender.rb

@@ -0,0 +1,130 @@
+require "fluent/plugin/mysql/appender/version"
+
+module Fluent
+  module Plugin
+    module Mysql
+      module Appender
+        class MysqlAppenderInput < Fluent::Input
+          Plugin.register_input('mysql_appender', self)
+
+          # Define `router` method to support v0.10.57 or earlier
+          unless method_defined?(:router)
+            define_method("router") { Engine }
+          end
+
+          def initialize
+            require 'mysql2'
+            super
+          end
+
+          config_param :host, :string, :default => 'localhost'
+          config_param :port, :integer, :default => 3306
+          config_param :username, :string, :default => 'root'
+          config_param :password, :string, :default => nil, :secret => true
+          config_param :database, :string, :default => nil
+          config_param :encoding, :string, :default => 'utf8'
+          config_param :query, :string
+          config_param :primary_key, :string, :default => 'id'
+          config_param :interval, :string, :default => '1m'
+          config_param :tag, :string, :default => nil
+
+          def configure(conf)
+            super
+            @interval = Config.time_value(@interval)
+
+            if @tag.nil?
+              raise Fluent::ConfigError, "mysql_appender: missing 'tag' parameter. Please add following line into config like 'tag replicator.mydatabase.mytable.${event}.${primary_key}'"
+            end
+
+            $log.info "adding mysql_appender worker. :tag=>#{tag} :query=>#{@query} :prepared_query=>#{@prepared_query} :interval=>#{@interval}sec"
+          end
+
+          def start
+            @thread = Thread.new(&method(:run))
+          end
+
+          def shutdown
+            Thread.kill(@thread)
+          end
+
+          def run
+            begin
+              poll
+            rescue StandardError => e
+              $log.error "mysql_appender: failed to execute query."
+              $log.error "error: #{e.message}"
+              $log.error e.backtrace.join("\n")
+            end
+          end
+
+          def poll
+            con = get_connection()
+            max_id = -1
+            loop do
+              rows_count = 0
+              start_time = Time.now
+              if max_id == -1
+                select_query = @query + " order by #{primary_key} asc"
+              else
+                select_query = @query + " where #{primary_key} > #{max_id} order by #{primary_key} asc"
+              end
+              rows, con = query(select_query, con)
+              rows.each_with_index do |row, index|
+                tag = format_tag(@tag, {:event => :insert})
+                router.emit(tag, Engine.now, row)
+                rows_count += 1
+                if index == rows.size - 1
+                  max_id = row[@primary_key]
+                end
+              end
+              con.close
+              elapsed_time = sprintf("%0.02f", Time.now - start_time)
+              $log.info "mysql_appender: finished execution :tag=>#{tag} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
+              sleep @interval
+            end
+          end
+
+          def format_tag(tag, param)
+            pattern = {'${event}' => param[:event].to_s, '${primary_key}' => @primary_key}
+            tag.gsub(/(\${[a-z_]+})/) do
+              $log.warn "mysql_appender: missing placeholder. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
+              pattern[$1]
+            end
+          end
+
+          def query(query, con = nil)
+            begin
+              con = con.nil? ? get_connection : con
+              con = con.ping ? con : get_connection
+              return con.query(query), con
+            rescue Exception => e
+              $log.warn "mysql_appender: #{e}"
+              sleep @interval
+              retry
+            end
+          end
+
+          def get_connection
+            begin
+              return Mysql2::Client.new({
+                :host => @host,
+                :port => @port,
+                :username => @username,
+                :password => @password,
+                :database => @database,
+                :encoding => @encoding,
+                :reconnect => true,
+                :stream => true,
+                :cache_rows => false
+              })
+            rescue Exception => e
+              $log.warn "mysql_appender: #{e}"
+              sleep @interval
+              retry
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/mysql/appender/version.rb

@@ -0,0 +1,9 @@
+module Fluent
+  module Plugin
+    module Mysql
+      module Appender
+        VERSION = "0.1.0"
+      end
+    end
+  end
+end

metadata

@@ -1,85 +1,85 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-appender
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - tsuyoshi_terasaki
 autorequire: 
-bindir: bin
+bindir: exe
 cert_chain: []
 date: 2016-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: fluentd
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: '1.13'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.13'
 - !ruby/object:Gem::Dependency
-  name: mysql2
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: '10.0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.0'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.13'
+        version: '5.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.13'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
-  type: :development
+        version: '0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: mysql2
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
-  type: :development
+        version: '0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '0'
 description: Simple incremental id's insert.
 email:
 - tsuyoshi_terasaki@realworld.jp
@@ -93,10 +93,11 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/console
+- bin/setup
 - fluent-plugin-mysql-appender.gemspec
-- lib/fluent/plugin/in_mysql_appender.rb
-- test/fluent/plugin/in_mysql_appender_test.rb
-- test/test_helper.rb
+- lib/fluent/plugin/mysql/appender.rb
+- lib/fluent/plugin/mysql/appender/version.rb
 homepage: https://github.com/rw-hub/fluent-plugin-mysql-appender
 licenses:
 - MIT
@@ -121,6 +122,4 @@ rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Fluentd input plugin to track insert event from MySQL database server.
-test_files:
-- test/fluent/plugin/in_mysql_appender_test.rb
-- test/test_helper.rb
+test_files: []

data/lib/fluent/plugin/in_mysql_appender.rb

@@ -1,122 +0,0 @@
-module Fluent
-  class MysqlAppenderInput < Fluent::Input
-    Plugin.register_input('mysql_appender', self)
-
-    # Define `router` method to support v0.10.57 or earlier
-    unless method_defined?(:router)
-      define_method("router") { Engine }
-    end
-
-    def initialize
-      require 'mysql2'
-      super
-    end
-
-    config_param :host, :string, :default => 'localhost'
-    config_param :port, :integer, :default => 3306
-    config_param :username, :string, :default => 'root'
-    config_param :password, :string, :default => nil, :secret => true
-    config_param :database, :string, :default => nil
-    config_param :encoding, :string, :default => 'utf8'
-    config_param :query, :string
-    config_param :primary_key, :string, :default => 'id'
-    config_param :interval, :string, :default => '1m'
-    config_param :tag, :string, :default => nil
-
-    def configure(conf)
-      super
-      @interval = Config.time_value(@interval)
-
-      if @tag.nil?
-        raise Fluent::ConfigError, "mysql_appender: missing 'tag' parameter. Please add following line into config like 'tag replicator.mydatabase.mytable.${event}.${primary_key}'"
-      end
-
-      $log.info "adding mysql_appender worker. :tag=>#{tag} :query=>#{@query} :prepared_query=>#{@prepared_query} :interval=>#{@interval}sec"
-    end
-
-    def start
-      @thread = Thread.new(&method(:run))
-    end
-
-    def shutdown
-      Thread.kill(@thread)
-    end
-
-    def run
-      begin
-        poll
-      rescue StandardError => e
-        $log.error "mysql_appender: failed to execute query."
-        $log.error "error: #{e.message}"
-        $log.error e.backtrace.join("\n")
-      end
-    end
-
-    def poll
-      con = get_connection()
-      max_id = -1
-      loop do
-        rows_count = 0
-        start_time = Time.now
-        if max_id == -1
-          select_query = @query + " order by #{primary_key} asc"
-        else
-          select_query = @query + " where #{primary_key} > #{max_id} order by #{primary_key} asc"
-        end
-        rows, con = query(select_query, con)
-        rows.each_with_index do |row, index|
-          tag = format_tag(@tag, {:event => :insert})
-          router.emit(tag, Engine.now, row)
-          rows_count += 1
-          if index == rows.size - 1
-            max_id = row[@primary_key]
-          end
-        end
-        con.close
-        elapsed_time = sprintf("%0.02f", Time.now - start_time)
-        $log.info "mysql_appender: finished execution :tag=>#{tag} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
-        sleep @interval
-      end
-    end
-
-    def format_tag(tag, param)
-      pattern = {'${event}' => param[:event].to_s, '${primary_key}' => @primary_key}
-      tag.gsub(/(\${[a-z_]+})/) do
-        $log.warn "mysql_appender: missing placeholder. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
-        pattern[$1]
-      end
-    end
-
-    def query(query, con = nil)
-      begin
-        con = con.nil? ? get_connection : con
-        con = con.ping ? con : get_connection
-        return con.query(query), con
-      rescue Exception => e
-        $log.warn "mysql_appender: #{e}"
-        sleep @interval
-        retry
-      end
-    end
-
-    def get_connection
-      begin
-        return Mysql2::Client.new({
-          :host => @host,
-          :port => @port,
-          :username => @username,
-          :password => @password,
-          :database => @database,
-          :encoding => @encoding,
-          :reconnect => true,
-          :stream => true,
-          :cache_rows => false
-        })
-      rescue Exception => e
-        $log.warn "mysql_appender: #{e}"
-        sleep @interval
-        retry
-      end
-    end
-  end
-end

data/test/fluent/plugin/in_mysql_appender_test.rb

@@ -1,8 +0,0 @@
-require 'test_helper'
-
-class MysqlAppenderInputTest < Minitest::Test
-
-  def test_it_does_something_useful
-    assert true
-  end
-end

data/test/test_helper.rb

@@ -1,3 +0,0 @@
-require 'fluent/plugin/in_mysql_appender'
-
-require 'minitest/autorun'