fluent-plugin-mysql-load 0.0.1 → 0.0.2

data/CHANGELOG.md

@@ -0,0 +1,10 @@
+# 0.0.2 (2014/05/17)
+
+Features:
+
+- Add `key_names` option parameter
+- Add `column_names`, Remove `columns` parameters
+
+# 0.0.1 (2014/05/05)
+
+First version

data/README.md

@@ -3,22 +3,119 @@
 
 # Overview
 
-"fluent-plugin-mysql-load" is a plugin fluentd.
+BufferedOutput plugin to mysql import.
+Internal processing uses the **"LOAD DATA LOCAL INFILE"**.
 
-Output filter plugin to mysql import.
-Internal processing uses the "LOAD DATA INFILE".
+[Buffer Plugin Overview is here](http://docs.fluentd.org/articles/buffer-plugin-overview#buffer-plugin-overview)
 
 [MySQL Manual of LOAD DATA is here](http://dev.mysql.com/doc/refman/5.6/en/load-data.html)
 
 # Configuration
-## Ex.
+
+## Ex1.
+```
+<match loaddata.**>
+  type mysql_load
+  host localhost
+  port 3306
+  username taro
+  password abcdefg
+  database fluentd
+  column_names id,txt,txt2,txt3,created_at
+
+  buffer_type file
+  buffer_path /var/log/fluent/test.*.buffer
+  flush_interval 60s
+</match>
+```
+
+### In.
+```
+loaddata.test: {"id":111,"txt":"hoge","txt2":"foo","txt3":"bar","created_at":"2014-01-01 00:00:00"}
+loaddata.test: {"id":112,"txt":"hoge2","txt2":"foo2","txt3":"bar2","created_at":"2014-01-01 00:00:01"}
+loaddata.test: {"id":123,"txt":"hoge3","txt2":"foo3","txt3":"bar3","created_at":"2014-01-01 00:00:02"}
+```
+
+### Out.
+```
+mysql> desc test;
++------------+----------+------+-----+---------+----------------+
+| Field      | Type     | Null | Key | Default | Extra          |
++------------+----------+------+-----+---------+----------------+
+| id         | int(11)  | NO   | PRI | NULL    | auto_increment |
+| txt        | text     | YES  |     | NULL    |                |
+| txt2       | text     | YES  |     | NULL    |                |
+| txt3       | text     | YES  |     | NULL    |                |
+| created_at | datetime | YES  |     | NULL    |                |
++------------+----------+------+-----+---------+----------------+
+mysql> select * from test;
++---------------+------------------+---------------------+---------------------+---------------------+
+| id            | txt              | txt2                | txt3                |created_at           |
++---------------+------------------+---------------------+---------------------+---------------------+
+| 111           | hoge             | foo                 | bar                 |2014-01-01 00:00:00  |
+| 112           | hoge2            | foo2                | bar2                |2014-01-01 00:00:01  |
+| 113           | hoge3            | foo3                | bar3                |2014-01-01 00:00:02  |
++---------------+------------------+---------------------+---------------------+---------------------+
+```
+
+## Ex2.
+```
+<match loaddata.**>
+  type mysql_load
+  host localhost
+  port 3306
+  username taro
+  password abcdefg
+  database fluentd
+  key_names dummy1,dummy2,dummy3,create_d
+  column_names txt,txt2,txt3,created_at
+
+  buffer_type file
+  buffer_path /var/log/fluent/test.*.buffer
+  flush_interval 60s
+</match>
+```
+
+### In.
+```
+loaddata.test: {"dummy1":"hoge","dummy2":"foo","dummy3":"bar","create_d":"2014-01-01 00:00:00"}
+loaddata.test: {"dummy1":"hoge2","dummy2":"foo2","dummy3":"bar2","create_d":"2014-01-01 00:00:01"}
+loaddata.test: {"dummy1":"hoge3","dummy2":"foo3","dummy3":"bar3","create_d":"2014-01-01 00:00:02"}
+```
+
+### Out.
+```
+mysql> desc test;
++------------+----------+------+-----+---------+----------------+
+| Field      | Type     | Null | Key | Default | Extra          |
++------------+----------+------+-----+---------+----------------+
+| id         | int(11)  | NO   | PRI | NULL    | auto_increment |
+| txt        | text     | YES  |     | NULL    |                |
+| txt2       | text     | YES  |     | NULL    |                |
+| txt3       | text     | YES  |     | NULL    |                |
+| created_at | datetime | YES  |     | NULL    |                |
++------------+----------+------+-----+---------+----------------+
+mysql> select * from test;
++---------------+------------------+---------------------+---------------------+---------------------+
+| id            | txt              | txt2                | txt3                |created_at           |
++---------------+------------------+---------------------+---------------------+---------------------+
+| 1             | hoge             | foo                 | bar                 |2014-01-01 00:00:00  |
+| 2             | hoge2            | foo2                | bar2                |2014-01-01 00:00:01  |
+| 3             | hoge3            | foo3                | bar3                |2014-01-01 00:00:02  |
++---------------+------------------+---------------------+---------------------+---------------------+
+```
+
+## Ex3.
 ```
 <match loaddata.**>
   type mysql_load
-  password password
+  host localhost
+  port 3306
+  username taro
+  password abcdefg
   database fluentd
-  tablename test
-  columns txt,txt2,txt3
+  key_names dummy1,dummy2,dummy3,${time}
+  column_names txt,txt2,txt3,created_at
 
   buffer_type file
   buffer_path /var/log/fluent/test.*.buffer
@@ -26,58 +123,91 @@ Internal processing uses the "LOAD DATA INFILE".
 </match>
 ```
 
-## In.
+### In.
 ```
-loaddata.test: {"seq":1,"txt":"hoge","txt2":"foo","txt3":"bar","createte_at":"2014-01-01 00:00:00"}
-loaddata.test: {"seq":2,"txt":"hoge2","txt2":"foo2","txt3":"bar2","createte_at":"2014-01-01 00:00:01"}
-loaddata.test: {"seq":3,"txt":"hoge3","txt2":"foo3","txt3":"bar3","createte_at":"2014-01-01 00:00:02"}
+loaddata.test: {"dummy1":"hoge","dummy2":"foo","dummy3":"bar"}
+loaddata.test: {"dummy1":"hoge2","dummy2":"foo2","dummy3":"bar2"}
+loaddata.test: {"dummy1":"hoge3","dummy2":"foo3","dummy3":"bar3"}
 ```
 
-## Out.
+### Out.
 ```
-  +------------------+---------------------+---------------------+
-  | txt              | txt2                | txt3                |
-  +------------------+---------------------+---------------------+
-  | hoge             | foo                 | bar                 |
-  | hoge2            | foo2                | bar2                |
-  | hoge3            | foo3                | bar3                |
-  +------------------+---------------------+---------------------+
+mysql> desc test;
++------------+----------+------+-----+---------+----------------+
+| Field      | Type     | Null | Key | Default | Extra          |
++------------+----------+------+-----+---------+----------------+
+| id         | int(11)  | NO   | PRI | NULL    | auto_increment |
+| txt        | text     | YES  |     | NULL    |                |
+| txt2       | text     | YES  |     | NULL    |                |
+| txt3       | text     | YES  |     | NULL    |                |
+| created_at | datetime | YES  |     | NULL    |                |
++------------+----------+------+-----+---------+----------------+
+mysql> select * from test;
++---------------+------------------+---------------------+---------------------+---------------------+
+| id            | txt              | txt2                | txt3                |created_at           |
++---------------+------------------+---------------------+---------------------+---------------------+
+| 1             | hoge             | foo                 | bar                 |2014-01-01 00:00:00  |
+| 2             | hoge2            | foo2                | bar2                |2014-01-01 00:00:01  |
+| 3             | hoge3            | foo3                | bar3                |2014-01-01 00:00:02  |
++---------------+------------------+---------------------+---------------------+---------------------+
 ```
 
 # Parameters
 
-* host(option)
+* **host** (option)
 
   Set the host name of MySQL Server.The default is "localhost".
 
-* port(option)
+* **port** (option)
 
   Set the port of MySQL Server.The default is "3306".
 
-* username(option)
+* **username** (option)
 
   Set the user of MySQL Server.The default is "root".
 
-* password(option)
+* **password** (option)
 
   Set the password of MySQL Server.The default is ""(blank).
 
-* database(required)
+* **database** (required)
 
   Set the database of MySQL Server.
 
-* tablename(required)
+* **tablename** (required)
 
   Set the table name of the import data.
 
-* columns(option)
+* **key_names** (option)
 
-  Set if you want to specify the column of the table of data import destination.Attaches to the string key of the input data.
+  It is possible to specify the key for JSON input.
 
-* encoding(option)
+  Will be converted to a format in this embedding ${time}. (Format:'%Y-%m-%d %H:%M:%S')
+
+
+* **column_names** (required)
+
+  Set if you want to specify the column of the table of data import destination.
+
+  If you do not specify a "key_names" parameters, to be used as a key input value of JSON the parameters specified in the "column_names".
+
+* **encoding** (option)
 
   Set the encode of MySQL Server.The default is "utf8".
 
+And Buffer Plugin Parameters...
+
+* [memory Buffer Plugin](http://docs.fluentd.org/articles/buf_memory)
+* [file Buffer Plugin](http://docs.fluentd.org/articles/buf_file)
+
+And Logging of Fluentd Parameters...(>=v0.10.43)
+
+ * [Logging of Fluentd](http://docs.fluentd.org/articles/logging#per-plugin-log-fluentd-v01043-and-above)
+
+
+# ChangeLog
+
+See [CHANGELOG.md](https://github.com/fukuiretu/fluent-plugin-mysql-load/blob/master/CHANGELOG.md) for details.
 
 # Copyright
 Copyright:: Copyright (c) 2014- Fukui ReTu License:: Apache License, Version 2.0

data/fluent-plugin-mysql-load.gemspec

@@ -1,11 +1,11 @@
 # coding: utf-8
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mysql-load"
-  spec.version       = "0.0.1"
+  spec.version       = "0.0.2"
   spec.authors       = ["Fukui ReTu"]
   spec.email         = ["s0232101@gmail.com"]
-  spec.description   = %q{Output filter plugin to mysql import}
-  spec.summary       = %q{Output filter plugin to mysql import}
+  spec.description   = %q{BufferedOutput plugin to mysql import}
+  spec.summary       = %q{BufferedOutput plugin to mysql import}
   spec.homepage      = "https://github.com/fukuiretu/fluent-plugin-mysql-load"
   spec.license       = "APL2.0"
 
@@ -19,4 +19,4 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake"
   spec.add_runtime_dependency "fluentd"
   spec.add_runtime_dependency "mysql2"
-end
+end

data/lib/fluent/plugin/out_mysql_load.rb

@@ -21,17 +21,19 @@ module Fluent
     config_param :password, :string, :default => nil
     config_param :database, :string, :default => nil
     config_param :tablename, :string, :default => nil
-    config_param :columns, :string, :default => nil
+    config_param :key_names, :string, :default => nil
+    config_param :column_names, :string, :default => nil
     config_param :encoding, :string, :default => 'utf8'
 
     def configure(conf)
       super
-      if @database.nil? || @tablename.nil?
-        raise Fluent::ConfigError, "database and tablename is required!"
+      if @database.nil? || @tablename.nil? || @column_names.nil?
+        raise Fluent::ConfigError, "database and tablename and column_names is required."
       end
 
-      if (!@columns.nil?)
-        @columns = @columns.split(",")
+      @key_names = @key_names.nil? ? @column_names.split(',') : @key_names.split(',')
+      unless @column_names.split(',').count == @key_names.count
+        raise Fluent::ConfigError, "It does not take the integrity of the key_names and column_names."
       end
     end
 
@@ -44,36 +46,30 @@ module Fluent
     end
 
     def format(tag, time, record)
-      record.to_msgpack
+      values = @key_names.map { |k|
+        k == '${time}' ? Time.at(time).strftime('%Y-%m-%d %H:%M:%S') : record[k]
+      }
+      values.to_msgpack
     end
 
     def write(chunk)
-      tmp = Tempfile.new("loaddata")
-      keys = nil
+      data_count = 0
+      tmp = Tempfile.new("mysql-loaddata")
       chunk.msgpack_each { |record|
-        # keyの取得は初回のみ
-        if keys.nil?
-          # columnsが指定されている場合はそっちを有効にする
-          keys = @columns.nil? ? record.keys : @columns
-        end
-
-        values = []
-        keys.each{ |key|
-          values << record[key]
-        }
-
-        tmp.write values.join("\t") + "\n"
+        tmp.write record.join("\t") + "\n"
+        data_count += 1
       }
       tmp.close
 
-      query = QUERY_TEMPLATE % ([tmp.path, @tablename, keys.join(",")])
-
       conn = get_connection
-      conn.query(query)
+      conn.query(QUERY_TEMPLATE % ([tmp.path, @tablename, @column_names]))
       conn.close
+
+      log.info "number that is registered in the \"%s:%s\" table is %d" % ([@database, @tablename, data_count])
     end
 
     private
+
     def get_connection
         Mysql2::Client.new({
             :host => @host,
@@ -81,7 +77,8 @@ module Fluent
             :username => @username,
             :password => @password,
             :database => @database,
-            :encoding => @encoding
+            :encoding => @encoding,
+            :local_infile => true
           })
     end
   end

data/test/plugin/test_out_mysql_load.rb

@@ -8,6 +8,7 @@ class MysqlLoadOutputTest < Test::Unit::TestCase
   CONFIG = %[
     database fluentd
     tablename test
+    column_names a,b,c,d,e
   ]
 
   def create_driver(conf = CONFIG, tag='test')
@@ -16,7 +17,7 @@ class MysqlLoadOutputTest < Test::Unit::TestCase
 
   def test_configure_default
     d = create_driver
-    
+
     assert_equal 'localhost', d.instance.host
     assert_equal 3306, d.instance.port
     assert_equal 'root', d.instance.username
@@ -32,9 +33,31 @@ class MysqlLoadOutputTest < Test::Unit::TestCase
       database db
       encoding sjis
       tablename test
-      columns a,b,c,d,e
+      column_names a,b,c,d,e
+    ]
+
+    assert_equal 'abcde', d.instance.host
+    assert_equal 33306, d.instance.port
+    assert_equal 'taro', d.instance.username
+    assert_equal 'pldoeudku', d.instance.password
+    assert_equal 'db', d.instance.database
+    assert_equal 'sjis', d.instance.encoding
+    assert_equal 'test', d.instance.tablename
+    assert_equal 'a,b,c,d,e', d.instance.column_names
+    assert_equal ['a', 'b', 'c', 'd', 'e'], d.instance.key_names
+
+    d = create_driver %[
+      host abcde
+      port 33306
+      username taro
+      password pldoeudku
+      database db
+      encoding sjis
+      tablename test
+      column_names a,b,c,d,e
+      key_names f,g,h,i,j
     ]
-    
+
     assert_equal 'abcde', d.instance.host
     assert_equal 33306, d.instance.port
     assert_equal 'taro', d.instance.username
@@ -42,7 +65,8 @@ class MysqlLoadOutputTest < Test::Unit::TestCase
     assert_equal 'db', d.instance.database
     assert_equal 'sjis', d.instance.encoding
     assert_equal 'test', d.instance.tablename
-    assert_equal ['a', 'b', 'c', 'd', 'e'], d.instance.columns
+    assert_equal 'a,b,c,d,e', d.instance.column_names
+    assert_equal ['f', 'g', 'h', 'i', 'j'], d.instance.key_names
   end
 
   def test_configure_error
@@ -54,7 +78,31 @@ class MysqlLoadOutputTest < Test::Unit::TestCase
         password pldoeudku
         encoding sjis
         tablename test
-        columns a,b,c,d,e
+        column_names a,b,c,d,e
+      ]
+    end
+
+    assert_raise(Fluent::ConfigError) do
+      create_driver %[
+        host abcde
+        port 33306
+        username taro
+        password pldoeudku
+        database db
+        encoding sjis
+        column_names a,b,c,d,e
+      ]
+    end
+
+    assert_raise(Fluent::ConfigError) do
+      create_driver %[
+        host abcde
+        port 33306
+        username taro
+        password pldoeudku
+        database db
+        tablename test
+        encoding sjis
       ]
     end
 
@@ -65,25 +113,34 @@ class MysqlLoadOutputTest < Test::Unit::TestCase
         username taro
         password pldoeudku
         database db
+        tablename test
+        column_names a,b,c,d,e
+        key_names a,b,c,d,e,f,g
         encoding sjis
-        columns a,b,c,d,e
       ]
     end
   end
 
   def test_format
-    d = create_driver
+    d = create_driver %[
+      host localhost
+      port 3306
+      username root
+      database fluentd
+      tablename test
+      column_names id,txt,txt2
+    ]
 
     d.emit({"id"=>1, "txt" => "hoge", "txt2" => "foo"})
     d.emit({"id"=>2, "txt" => "hoge2", "txt2" => "foo2"})
 
-    d.expect_format({"id"=>1, "txt" => "hoge", "txt2" => "foo"}.to_msgpack)
-    d.expect_format({"id"=>2, "txt" => "hoge2", "txt2" => "foo2"}.to_msgpack)
+    d.expect_format([1, "hoge", "foo"].to_msgpack)
+    d.expect_format([2, "hoge2", "foo2"].to_msgpack)
 
-    # d.run
+    d.run
   end
 
   def test_write
     # TODO
   end
-end
+end

metadata

@@ -1,18 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-load
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
+  prerelease: 
 platform: ruby
 authors:
 - Fukui ReTu
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-04 00:00:00.000000000 Z
+date: 2014-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -20,6 +22,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -27,46 +30,52 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: Output filter plugin to mysql import
+description: BufferedOutput plugin to mysql import
 email:
 - s0232101@gmail.com
 executables: []
@@ -75,6 +84,7 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - .travis.yml
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -89,27 +99,28 @@ files:
 homepage: https://github.com/fukuiretu/fluent-plugin-mysql-load
 licenses:
 - APL2.0
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: fluent-plugin-mysql-load
-rubygems_version: 2.0.14
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
-summary: Output filter plugin to mysql import
+specification_version: 3
+summary: BufferedOutput plugin to mysql import
 test_files:
 - test/helper.rb
 - test/plugin/test_out_mysql_load.rb

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 694e88114b2538b250ecbf498b4351f06fdda1d0
-  data.tar.gz: 0d9d81e11ca091d49b1fd8620902d71a8740dcae
-SHA512:
-  metadata.gz: b93240239bcaac9cb8ab02480a73cb55bd873e60f686a6cee1b6a81d571d0aa097ace435c5c2a57c74ff261168c9c52e11542dd44787bebb1906e1d6ba449aa1
-  data.tar.gz: b32a77a8148fe0b9dea6daebfbb448eeb8b63a5c233657fb670de6922230e66eba3030b6e1f23fa9c91d2d614bb04925c5a249b14136c133042f4c71bb85a88d