fluent-plugin-mysql 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b6ce2ad3922449224b551869fffcf66e65444849
-  data.tar.gz: b912540a7fe49953ac5f52ef17573c001d110fc9
+  metadata.gz: 6c8d2f2fd5a6164a01413ca4d9179d6a93d8b4f0
+  data.tar.gz: 05a99dbf24a3458cbd2031f87da2b1cd154c7d66
 SHA512:
-  metadata.gz: a8d3a56d4a190b8be00e5f610a14b33f1713880d208a94d922ae616b9621642d4e638b92819e62170dd071af4784eb5817f189dfc9287128a0243e77a1ed3dd9
-  data.tar.gz: 61934977b358d2eaab543e2905f036dd3e12e738527683c8e7a6acbbb1ff48e2e64f2168a931cde7b94470bcd55717512379f8d20eedc9cf13e850a442abb4dc
+  metadata.gz: 1aaf277b0400baf6235a2a1ebfa954e55c3f1b09f63c81f524f1d8af2751eb9e53de9f0828630de1e2bf8c36468445a51951e88f6823528b18db4716226c0a75
+  data.tar.gz: f1889ff991d71cc77e0c35201ed8bd169464e118671865ad839717d97082b3e9118c416342f568a2e2506f290760da350034c5e9835cf5120a468a64c1caf774

data/.travis.yml

@@ -1,7 +1,6 @@
 language: ruby
 
 rvm:
-  - 2.0.0
   - 2.1
   - 2.2.3
   - 2.3.0
@@ -9,11 +8,6 @@ rvm:
 gemfile:
   - Gemfile
 
-matrix:
-  exclude:
-    - rvm: 2.0.0
-      gemfile: Gemfile
-
 before_install:
   - gem update bundler
 

data/README.md

@@ -8,6 +8,8 @@ fluent-plugin-mysql-bulk merged this repository.
 
 [mysql plugin](README_mysql.md) is deprecated. You should use mysql_bulk.
 
+v0.1.5 only supports fluentd-0.12.X and v0.2.0 only supports fluentd-0.14.X.
+
 ## Parameters
 
 param|value
@@ -228,6 +230,92 @@ then `created_at` column is set from time attribute in a fluentd packet with tim
 +-----+-----------+---------------------+
 ```
 
+## Configuration Example(bulk insert with tag placeholder for table name)
+
+This description is for v0.14.X users.
+
+```
+<match mysql.input>
+  @type mysql_bulk
+  host localhost
+  database test_app_development
+  username root
+  password hogehoge
+  column_names id,user_name,created_at
+  key_names id,user,${time}
+  table users_${tag}
+  <buffer tag>
+    @type memory
+    flush_interval 60s
+  </buffer>
+</match>
+```
+
+Assume following input is coming:
+
+```js
+2016-09-26 18:42:13+09:00: mysql.input: {"user":"toyama","dummy":"hogehoge"}
+2016-09-26 18:42:16+09:00: mysql.input: {"user":"toyama2","dummy":"hogehoge"}
+2016-09-26 18:42:19+09:00: mysql.input: {"user":"toyama3","dummy":"hogehoge"}
+```
+
+then `created_at` column is set from time attribute in a fluentd packet:
+
+```sql
+mysql> select * from users_mysql_input;
++----+-----------+---------------------+
+| id | user_name | created_at          |
++----+-----------+---------------------+
+|  1 | toyama    | 2016-09-26 18:42:13 |
+|  2 | toyama2   | 2016-09-26 18:42:16 |
+|  3 | toyama3   | 2016-09-26 18:42:19 |
++----+-----------+---------------------+
+3 rows in set (0.00 sec)
+```
+
+## Configuration Example(bulk insert with time format placeholder for table name)
+
+This description is for v0.14.X users.
+
+```
+<match mysql.input>
+  @type mysql_bulk
+  host localhost
+  database test_app_development
+  username root
+  password hogehoge
+  column_names id,user_name,created_at
+  key_names id,user,${time}
+  table users_%Y%m%d
+  <buffer time>
+    @type memory
+    timekey 60s
+    timekey_wait 60s
+  </buffer>
+</match>
+```
+
+Assume following input is coming:
+
+```js
+2016-09-26 18:37:06+09:00: mysql.input: {"user":"toyama","dummy":"hogehoge"}
+2016-09-26 18:37:08+09:00: mysql.input: {"user":"toyama2","dummy":"hogehoge"}
+2016-09-26 18:37:11+09:00: mysql.input: {"user":"toyama3","dummy":"hogehoge"}
+```
+
+then `created_at` column is set from time attribute in a fluentd packet:
+
+```sql
+mysql> select * from users_20160926;
++----+-----------+---------------------+
+| id | user_name | created_at          |
++----+-----------+---------------------+
+|  1 | toyama    | 2016-09-26 18:37:06 |
+|  2 | toyama2   | 2016-09-26 18:37:08 |
+|  3 | toyama3   | 2016-09-26 18:37:11 |
++----+-----------+---------------------+
+3 rows in set (0.00 sec)
+```
 
 ## spec
 

data/fluent-plugin-mysql.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-mysql"
-  gem.version       = "0.1.5"
+  gem.version       = "0.2.0"
   gem.authors       = ["TAGOMORI Satoshi", "Toyama Hiroshi"]
   gem.email         = ["tagomoris@gmail.com", "toyama0919@gmail.com"]
   gem.description   = %q{fluent plugin to insert mysql as json(single column) or insert statement}
@@ -14,9 +14,10 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_runtime_dependency "fluentd", '~> 0.12.0'
+  gem.add_runtime_dependency "fluentd", ['>= 0.14.8', '< 2']
   gem.add_runtime_dependency "mysql2-cs-bind"
   gem.add_runtime_dependency "jsonpath"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "test-unit"
+  gem.add_development_dependency "timecop", "~> 0.8.0"
 end

data/lib/fluent/plugin/out_mysql_bulk.rb

@@ -1,36 +1,38 @@
 # -*- encoding : utf-8 -*-
-module Fluent
-  class Fluent::MysqlBulkOutput < Fluent::BufferedOutput
+require 'fluent/plugin/output'
+
+module Fluent::Plugin
+  class MysqlBulkOutput < Output
     Fluent::Plugin.register_output('mysql_bulk', self)
 
-    include Fluent::SetTimeKeyMixin
+    helpers :compat_parameters, :inject
 
     config_param :host, :string, default: '127.0.0.1',
-                 :desc => "Database host."
+                 desc: "Database host."
     config_param :port, :integer, default: 3306,
-                 :desc => "Database port."
+                 desc: "Database port."
     config_param :database, :string,
-                 :desc => "Database name."
+                 desc: "Database name."
     config_param :username, :string,
-                 :desc => "Database user."
+                 desc: "Database user."
     config_param :password, :string, default: '', secret: true,
-                 :desc => "Database password."
+                 desc: "Database password."
 
     config_param :column_names, :string,
-                 :desc => "Bulk insert column."
+                 desc: "Bulk insert column."
     config_param :key_names, :string, default: nil,
-                 :desc => <<-DESC
+                 desc: <<-DESC
 Value key names, ${time} is placeholder Time.at(time).strftime("%Y-%m-%d %H:%M:%S").
 DESC
     config_param :json_key_names, :string, default: nil,
-                  :desc => "Key names which store data as json"
+                  desc: "Key names which store data as json"
     config_param :table, :string,
-                 :desc => "Bulk insert table."
+                 desc: "Bulk insert table."
 
     config_param :on_duplicate_key_update, :bool, default: false,
-                 :desc => "On duplicate key update enable."
+                 desc: "On duplicate key update enable."
     config_param :on_duplicate_update_keys, :string, default: nil,
-                 :desc => "On duplicate key update column, comma separator."
+                 desc: "On duplicate key update column, comma separator."
 
     attr_accessor :handler
 
@@ -39,12 +41,8 @@ DESC
       require 'mysql2-cs-bind'
     end
 
-    # Define `log` method for v0.10.42 or earlier
-    unless method_defined?(:log)
-      define_method("log") { $log }
-    end
-
     def configure(conf)
+      compat_parameters_convert(conf, :buffer, :inject)
       super
 
       if @column_names.nil?
@@ -72,8 +70,11 @@ DESC
 
     def start
       super
-      result = client.xquery("SHOW COLUMNS FROM #{@table}")
-      @max_lengths = []
+    end
+
+    def check_table_schema(database: @database, table: @table)
+      result = client(database).xquery("SHOW COLUMNS FROM #{table}")
+      max_lengths = []
       @column_names.each do |column|
         info = result.select { |x| x['Field'] == column }.first
         r = /(char|varchar)\(([\d]+)\)/
@@ -82,8 +83,9 @@ DESC
         rescue
           max_length = nil
         end
-        @max_lengths << max_length
+        max_lengths << max_length
       end
+      max_lengths
     end
 
     def shutdown
@@ -91,31 +93,45 @@ DESC
     end
 
     def format(tag, time, record)
-      [tag, time, format_proc.call(tag, time, record)].to_msgpack
+      record = inject_values_to_record(tag, time, record)
+      [tag, time, record].to_msgpack
     end
 
-    def client
+    def formatted_to_msgpack_binary
+      true
+    end
+
+    def client(database)
       Mysql2::Client.new(
           host: @host,
           port: @port,
           username: @username,
           password: @password,
-          database: @database,
+          database: database,
           flags: Mysql2::Client::MULTI_STATEMENTS
         )
     end
 
+    def expand_placeholders(metadata)
+      database = extract_placeholders(@database, metadata).gsub('.', '_')
+      table = extract_placeholders(@table, metadata).gsub('.', '_')
+      return database, table
+    end
+
     def write(chunk)
-      @handler = client
+      database, table = expand_placeholders(chunk.metadata)
+      max_lengths = check_table_schema(database: database, table: table)
+      @handler = client(database)
       values = []
       values_template = "(#{ @column_names.map { |key| '?' }.join(',') })"
       chunk.msgpack_each do |tag, time, data|
+        data = format_proc.call(tag, time, data, max_lengths)
         values << Mysql2::Client.pseudo_bind(values_template, data)
       end
-      sql = "INSERT INTO #{@table} (#{@column_names.join(',')}) VALUES #{values.join(',')}"
+      sql = "INSERT INTO #{table} (#{@column_names.join(',')}) VALUES #{values.join(',')}"
       sql += @on_duplicate_key_update_sql if @on_duplicate_key_update
 
-      log.info "bulk insert values size (table: #{@table}) => #{values.size}"
+      log.info "bulk insert values size (table: #{table}) => #{values.size}"
       @handler.xquery(sql)
       @handler.close
     end
@@ -123,16 +139,16 @@ DESC
     private
 
     def format_proc
-      proc do |tag, time, record|
+      proc do |tag, time, record, max_lengths|
         values = []
         @key_names.each_with_index do |key, i|
           if key == '${time}'
             value = Time.at(time).strftime('%Y-%m-%d %H:%M:%S')
           else
-            if @max_lengths[i].nil? || record[key].nil?
+            if max_lengths[i].nil? || record[key].nil?
               value = record[key]
             else
-              value = record[key].slice(0, @max_lengths[i])
+              value = record[key].slice(0, max_lengths[i])
             end
 
             if @json_key_names && @json_key_names.include?(key)

data/test/plugin/test_out_mysql_bulk.rb

@@ -1,13 +1,32 @@
+# coding: utf-8
 require 'helper'
 require 'mysql2-cs-bind'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/buffer'
+require 'fluent/config'
+require 'time'
+require 'timecop'
 
 class MysqlBulkOutputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
   end
 
-  def create_driver(conf = CONFIG, tag = 'test')
-    d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::MysqlBulkOutput, tag).configure(conf)
+  def config_element(name = 'test', argument = '', params = {}, elements = [])
+    Fluent::Config::Element.new(name, argument, params, elements)
+  end
+
+  CONFIG = %[
+    database test_app_development
+    username root
+    password hogehoge
+    column_names id,user_name,created_at
+    key_names id,users,created_at
+    table users
+  ]
+
+  def create_driver(conf = CONFIG)
+    d = Fluent::Test::Driver::Output.new(Fluent::Plugin::MysqlBulkOutput).configure(conf)
     d.instance.instance_eval {
       def client
         obj = Object.new
@@ -21,9 +40,88 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
     d
   end
 
+  def create_metadata(timekey: nil, tag: nil, variables: nil)
+    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
+  end
+
+  class TestExpandPlaceholders < self
+    data("table" => {"database" => "test_app_development",
+                     "table" => "users_${tag}",
+                     "extracted_database" => "test_app_development",
+                     "extracted_table" => "users_input_mysql"
+                    },
+         "database" => {"database" => "test_app_development_${tag}",
+                        "table" => "users",
+                        "extracted_database" => "test_app_development_input_mysql",
+                        "extracted_table" => "users"
+                       },
+        )
+    def test_expand_tag_placeholder(data)
+      config = config_element('ROOT', '', {
+                                "@type" => "mysql_bulk",
+                                "host" => "localhost",
+                                "database" => data["database"],
+                                "username" => "root",
+                                "password" => "hogehoge",
+                                "column_names" => "id,user_name,created_at",
+                                "table" => data["table"],
+                              }, [config_element('buffer', 'tag', {
+                                                   "@type" => "memory",
+                                                   "flush_interval" => "60s",
+                                                 }, [])])
+      d = create_driver(config)
+      time = Time.now
+      metadata = create_metadata(timekey: time.to_i, tag: 'input.mysql')
+      database, table = d.instance.expand_placeholders(metadata)
+      assert_equal(data["extracted_database"], database)
+      assert_equal(data["extracted_table"], table)
+    end
+
+    def setup
+      Timecop.freeze(Time.parse("2016-09-26"))
+    end
+
+    data("table" => {"database" => "test_app_development",
+                     "table" => "users_%Y%m%d",
+                     "extracted_database" => "test_app_development",
+                     "extracted_table" => "users_20160926"
+                    },
+         "database" => {"database" => "test_app_development_%Y%m%d",
+                        "table" => "users",
+                        "extracted_database" => "test_app_development_20160926",
+                        "extracted_table" => "users"
+                       },
+        )
+    def test_expand_time_placeholder(data)
+      config = config_element('ROOT', '', {
+                                "@type" => "mysql_bulk",
+                                "host" => "localhost",
+                                "database" => data["database"],
+                                "username" => "root",
+                                "password" => "hogehoge",
+                                "column_names" => "id,user_name,created_at",
+                                "table" => data["table"],
+                              }, [config_element('buffer', 'time', {
+                                                   "@type" => "memory",
+                                                   "timekey" => "60s",
+                                                   "timekey_wait" => "60s"
+                                                 }, [])])
+      d = create_driver(config)
+      time = Time.now
+      metadata = create_metadata(timekey: time.to_i, tag: 'input.mysql')
+      database, table = d.instance.expand_placeholders(metadata)
+      assert_equal(data["extracted_database"], database)
+      assert_equal(data["extracted_table"], table)
+    end
+
+    def teardown
+      Timecop.return
+    end
+  end
+
   def test_configure_error
     assert_raise(Fluent::ConfigError) do
-      d = create_driver %[
+      create_driver %[
         host localhost
         database test_app_development
         username root
@@ -36,7 +134,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
     end
 
     assert_raise(Fluent::ConfigError) do
-      d = create_driver %[
+      create_driver %[
         host localhost
         database test_app_development
         username root
@@ -49,7 +147,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
     end
 
     assert_raise(Fluent::ConfigError) do
-      d = create_driver %[
+      create_driver %[
         host localhost
         username root
         password hogehoge
@@ -65,7 +163,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
   def test_configure
     # not define format(default csv)
     assert_nothing_raised(Fluent::ConfigError) do
-      d = create_driver %[
+      create_driver %[
         host localhost
         database test_app_development
         username root
@@ -79,7 +177,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
     end
 
     assert_nothing_raised(Fluent::ConfigError) do
-      d = create_driver %[
+      create_driver %[
         database test_app_development
         username root
         password hogehoge
@@ -89,7 +187,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
     end
 
     assert_nothing_raised(Fluent::ConfigError) do
-      d = create_driver %[
+      create_driver %[
         database test_app_development
         username root
         password hogehoge
@@ -101,7 +199,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
     end
 
     assert_nothing_raised(Fluent::ConfigError) do
-      d = create_driver %[
+      create_driver %[
         database test_app_development
         username root
         password hogehoge
@@ -114,7 +212,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
     end
 
     assert_nothing_raised(Fluent::ConfigError) do
-      d = create_driver %[
+      create_driver %[
         database test_app_development
         username root
         password hogehoge

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
@@ -15,16 +15,22 @@ dependencies:
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.8
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: 0.14.8
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: mysql2-cs-bind
   requirement: !ruby/object:Gem::Requirement
@@ -81,6 +87,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
 description: fluent plugin to insert mysql as json(single column) or insert statement
 email:
 - tagomoris@gmail.com