fluent-plugin-mysql 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9859ad599cbae8698e0054adcbeaff3205f5a537
-  data.tar.gz: 96cdfad1bf8d00270b2e6ac9d8ac92b83987b326
+  metadata.gz: 41820afbe7391da94a6778133b8bee6b2645564e
+  data.tar.gz: 7c77c42bc38bcbf7c698c19f970e6dda05f1161c
 SHA512:
-  metadata.gz: e15db1860fe2d5925917a6fe949459e0d3e75cddd7d57f8df3d83f497f6e55dafa634b3da94273bcae01f2d89c64e5862da4540ec649034ce1a311320b4e88bc
-  data.tar.gz: b1d106391be32bb63aa2dde7ade7c894fef44eab820ccf2dd7f9901160a7bfd26629469a8f089eddc80fff2d6f94d8c13a0c0343286f140eba07b987f4bd45fe
+  metadata.gz: eafdc4d3b3021330646df1520aa81904d4ae3510d0bd9052ea564142ccfe7f3e313154a5b22de8ffca281de4381037260df01c4cf41e4461bd2a7fc4c5189990
+  data.tar.gz: 2f70d79a00930d74518cce00e16df8bacea6e2c119b16344ff2f6f84a699b49377ce1451c819e27933cf81bdc917ed2f6b41b0fdb7988240fba273748663d581

data/README.md

@@ -19,6 +19,12 @@ port|database port(default: 3306)
 database|database name(require)
 username|user(require)
 password|password(default: blank)
+sslkey|path to client key(default: nil)
+sslcert|path to client cert(default: nil)
+sslca|path to ca cert(default: nil)
+sslcapath|path to ca certs(default: nil)
+sslcipher|ssl cipher(default: nil)
+sslverify|verify server certificate(default: nil)
 column_names|bulk insert column (require)
 key_names|value key names, ${time} is placeholder Time.at(time).strftime("%Y-%m-%d %H:%M:%S") (default : column_names)
 json_key_names|Key names which store data as json, comma separator.

data/fluent-plugin-mysql.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-mysql"
-  gem.version       = "0.2.1"
+  gem.version       = "0.3.0"
   gem.authors       = ["TAGOMORI Satoshi", "Toyama Hiroshi"]
   gem.email         = ["tagomoris@gmail.com", "toyama0919@gmail.com"]
   gem.description   = %q{fluent plugin to insert mysql as json(single column) or insert statement}

data/lib/fluent/plugin/out_mysql.rb

@@ -9,6 +9,12 @@ class Fluent::MysqlOutput < Fluent::BufferedOutput
   config_param :database, :string
   config_param :username, :string
   config_param :password, :string, :default => '', :secret => true
+  config_param :sslkey, :string, :default => nil
+  config_param :sslcert, :string, :default => nil
+  config_param :sslca, :string, :default => nil
+  config_param :sslcapath, :string, :default => nil
+  config_param :sslcipher, :string, :default => nil
+  config_param :sslverify, :bool, :default => nil
 
   config_param :key_names, :string, :default => nil # nil allowed for json format
   config_param :sql, :string, :default => nil
@@ -99,7 +105,14 @@ class Fluent::MysqlOutput < Fluent::BufferedOutput
     Mysql2::Client.new({
         :host => @host, :port => @port,
         :username => @username, :password => @password,
-        :database => @database, :flags => Mysql2::Client::MULTI_STATEMENTS,
+        :database => @database,
+        :sslkey => @sslkey,
+        :sslcert => @sslcert,
+        :sslca => @sslca,
+        :sslcapath => @sslcapath,
+        :sslcipher => @sslcipher,
+        :sslverify => @sslverify,
+        :flags => Mysql2::Client::MULTI_STATEMENTS,
       })
   end
 

data/lib/fluent/plugin/out_mysql_bulk.rb

@@ -1,4 +1,3 @@
-# -*- encoding : utf-8 -*-
 require 'fluent/plugin/output'
 
 module Fluent::Plugin
@@ -17,6 +16,18 @@ module Fluent::Plugin
                  desc: "Database user."
     config_param :password, :string, default: '', secret: true,
                  desc: "Database password."
+    config_param :sslkey, :string, default: nil,
+                 desc: "SSL key."
+    config_param :sslcert, :string, default: nil,
+                 desc: "SSL cert."
+    config_param :sslca, :string, default: nil,
+                 desc: "SSL CA."
+    config_param :sslcapath, :string, default: nil,
+                     desc: "SSL CA path."
+    config_param :sslcipher, :string, default: nil,
+                         desc: "SSL cipher."
+    config_param :sslverify, :bool, default: nil,
+                             desc: "SSL Verify Server Certificate."
 
     config_param :column_names, :string,
                  desc: "Bulk insert column."
@@ -29,6 +40,9 @@ DESC
     config_param :table, :string,
                  desc: "Bulk insert table."
 
+    config_param :unixtimestamp_key_names, :string, default: nil,
+                 desc: "Key names which store data as datetime from unix time stamp"
+
     config_param :on_duplicate_key_update, :bool, default: false,
                  desc: "On duplicate key update enable."
     config_param :on_duplicate_update_keys, :string, default: nil,
@@ -83,10 +97,7 @@ DESC
       @column_names = @column_names.split(',').collect(&:strip)
       @key_names = @key_names.nil? ? @column_names : @key_names.split(',').collect(&:strip)
       @json_key_names = @json_key_names.split(',') if @json_key_names
-    end
-
-    def start
-      super
+      @unixtimestamp_key_names = @unixtimestamp_key_names.split(',') if @unixtimestamp_key_names
     end
 
     def check_table_schema(database: @database, table: @table)
@@ -105,10 +116,6 @@ DESC
       max_lengths
     end
 
-    def shutdown
-      super
-    end
-
     def format(tag, time, record)
       record = inject_values_to_record(tag, time, record)
       [tag, time, record].to_msgpack
@@ -118,6 +125,10 @@ DESC
       true
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def client(database)
       Mysql2::Client.new(
           host: @host,
@@ -125,6 +136,12 @@ DESC
           username: @username,
           password: @password,
           database: database,
+          sslkey: @sslkey,
+          sslcert: @sslcert,
+          sslca: @sslca,
+          sslcapath: @sslcapath,
+          sslcipher: @sslcipher,
+          sslverify: @sslverify,
           flags: Mysql2::Client::MULTI_STATEMENTS
         )
     end
@@ -145,7 +162,7 @@ DESC
         data = format_proc.call(tag, time, data, max_lengths)
         values << Mysql2::Client.pseudo_bind(values_template, data)
       end
-      sql = "INSERT INTO #{table} (#{@column_names.join(',')}) VALUES #{values.join(',')}"
+      sql = "INSERT INTO #{table} (#{@column_names.map{|x| "`#{x.to_s.gsub('`', '``')}`"}.join(',')}) VALUES #{values.join(',')}"
       sql += @on_duplicate_key_update_sql if @on_duplicate_key_update
 
       log.info "bulk insert values size (table: #{table}) => #{values.size}"
@@ -165,12 +182,16 @@ DESC
             if max_lengths[i].nil? || record[key].nil?
               value = record[key]
             else
-              value = record[key].slice(0, max_lengths[i])
+              value = record[key].to_s.slice(0, max_lengths[i])
             end
 
             if @json_key_names && @json_key_names.include?(key)
               value = value.to_json
             end
+
+            if @unixtimestamp_key_names && @unixtimestamp_key_names.include?(key)
+              value = Time.at(value).strftime('%Y-%m-%d %H:%M:%S')
+            end
           end
           values << value
         end

data/test/plugin/test_out_mysql_bulk.rb

@@ -266,6 +266,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
     assert_equal ['id','user_name','created_at','updated_at'], d.instance.key_names
     assert_equal ['id','user_name','created_at','updated_at'], d.instance.column_names
     assert_equal nil, d.instance.json_key_names
+    assert_equal nil, d.instance.unixtimestamp_key_names
     assert_equal " ON DUPLICATE KEY UPDATE user_name = VALUES(user_name),updated_at = VALUES(updated_at)", d.instance.instance_variable_get(:@on_duplicate_key_update_sql)
 
     d = create_driver %[
@@ -279,6 +280,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
     assert_equal ['id','user_name','created_at','updated_at'], d.instance.key_names
     assert_equal ['id','user_name','created_at','updated_at'], d.instance.column_names
     assert_equal nil, d.instance.json_key_names
+    assert_equal nil, d.instance.unixtimestamp_key_names
     assert_nil d.instance.instance_variable_get(:@on_duplicate_key_update_sql)
 
     d = create_driver %[
@@ -293,6 +295,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
     assert_equal ['id','user_name','created_at','updated_at'], d.instance.key_names
     assert_equal ['id','user','created_date','updated_date'], d.instance.column_names
     assert_equal nil, d.instance.json_key_names
+    assert_equal nil, d.instance.unixtimestamp_key_names
     assert_nil d.instance.instance_variable_get(:@on_duplicate_key_update_sql)
 
     d = create_driver %[
@@ -301,6 +304,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
       password hogehoge
       key_names id,url,request_headers,params,created_at,updated_at
       column_names id,url,request_headers_json,params_json,created_date,updated_date
+      unixtimestamp_key_names created_at,updated_at
       json_key_names request_headers,params
       table access
     ]
@@ -308,6 +312,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
     assert_equal ['id','url','request_headers','params','created_at','updated_at'], d.instance.key_names
     assert_equal ['id','url','request_headers_json','params_json','created_date','updated_date'], d.instance.column_names
     assert_equal ['request_headers','params'], d.instance.json_key_names
+    assert_equal ['created_at', 'updated_at'], d.instance.unixtimestamp_key_names
     assert_nil d.instance.instance_variable_get(:@on_duplicate_key_update_sql)
 
     d = create_driver %[
@@ -324,6 +329,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
     assert_equal ['id','user_name','login_count','created_at','updated_at'], d.instance.key_names
     assert_equal ['id','user_name','login_count','created_at','updated_at'], d.instance.column_names
     assert_equal nil, d.instance.json_key_names
+    assert_equal nil, d.instance.unixtimestamp_key_names
     assert_equal " ON DUPLICATE KEY UPDATE login_count = `login_count` + 1,updated_at = VALUES(updated_at)", d.instance.instance_variable_get(:@on_duplicate_key_update_sql)
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-14 00:00:00.000000000 Z
+date: 2017-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -142,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: fluent plugin to insert mysql
@@ -150,4 +150,3 @@ test_files:
 - test/helper.rb
 - test/plugin/test_out_mysql.rb
 - test/plugin/test_out_mysql_bulk.rb
-has_rdoc: