fluent-plugin-mysql-bulk 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bba9d673e86389bd1ef3490a0d9ff7e4e735fe11
-  data.tar.gz: 702d0b71c3e6025135931df1b0433d7662e5b593
+  metadata.gz: 7bb1a0fcf1728e6d85a9592f8790d8faf14018ef
+  data.tar.gz: 47053866831072f969671259227a663f032009d1
 SHA512:
-  metadata.gz: 1e22fc43f34272b790c52eddb6a0faaff1f28d07e7573863af68f296a745501406986ca8a849628f241639dde6bf52730447cbbdb000bbd7e8810ac076b64920
-  data.tar.gz: 62ad6b49d8d9c301465c0141b50dce0e5bcb9207f77044343f2b88e256702e9f00fe9eba39d736e972f713229879f566e534922933b657d8615b16f64c1fb350
+  metadata.gz: 6398f672df551f4d2d894e8806aff7be9792a36b505c1089a42049042ae48ae8abeced526b321a64a0889d8f0c96e2d59bda817cf7300993ed9e63f8230f626e
+  data.tar.gz: f7b746d6a8c3a3924b15a0f4945216dc827331a8add729c1dc167c3565c6876f013da61fef03e6f92a0aaf73bb8828c1a0bfccadf3ef64dbe077100a3fcf4c26

data/fluent-plugin-mysql-bulk.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-mysql-bulk"
-  gem.version       = "0.0.4"
+  gem.version       = "0.0.5"
   gem.authors       = ["Hiroshi Toyama"]
   gem.email         = ["toyama0919@gmail.com"]
   gem.description   = %q{fluent plugin mysql bulk insert is high performance and on duplicate key update respond.}

data/lib/fluent/plugin/out_mysql_bulk.rb

@@ -3,18 +3,18 @@ module Fluent
   class Fluent::MysqlBulkOutput < Fluent::BufferedOutput
     Fluent::Plugin.register_output('mysql_bulk', self)
 
-    config_param :host, :string, :default => "127.0.0.1"
-    config_param :port, :integer, :default => 3306
+    config_param :host, :string, default: '127.0.0.1'
+    config_param :port, :integer, default: 3306
     config_param :database, :string
     config_param :username, :string
-    config_param :password, :string, :default => ''
+    config_param :password, :string, default: ''
 
     config_param :column_names, :string
-    config_param :key_names, :string, :default => nil
+    config_param :key_names, :string, default: nil
     config_param :table, :string
 
-    config_param :on_duplicate_key_update, :bool, :default => false
-    config_param :on_duplicate_update_keys, :string, :default => nil
+    config_param :on_duplicate_key_update, :bool, default: false
+    config_param :on_duplicate_update_keys, :string, default: nil
 
     attr_accessor :handler
 
@@ -27,31 +27,41 @@ module Fluent
       super
 
       if @column_names.nil?
-        raise Fluent::ConfigError, "column_names MUST be specified, but missing"
+        fail Fluent::ConfigError, 'column_names MUST specified, but missing'
       end
 
       if @on_duplicate_key_update
         if @on_duplicate_update_keys.nil?
-          raise Fluent::ConfigError, "on_duplicate_key_update = true , on_duplicate_update_keys nil!"
+          fail Fluent::ConfigError, 'on_duplicate_key_update = true , on_duplicate_update_keys nil!'
         end
         @on_duplicate_update_keys = @on_duplicate_update_keys.split(',')
 
-        @on_duplicate_key_update_sql = " ON DUPLICATE KEY UPDATE "
+        @on_duplicate_key_update_sql = ' ON DUPLICATE KEY UPDATE '
         updates = []
-        @on_duplicate_update_keys.each{|update_column|
+        @on_duplicate_update_keys.each do |update_column|
           updates.push(" #{update_column} = VALUES(#{update_column})")
-        }
+        end
         @on_duplicate_key_update_sql += updates.join(',')
       end
 
       @column_names = @column_names.split(',')
       @key_names = @key_names.nil? ? @column_names : @key_names.split(',')
-      @format_proc = Proc.new{|tag, time, record| @key_names.map{|k| k == '${time}' ? Time.at(time).strftime("%Y-%m-%d %H:%M:%S") : record[k]}}
-
     end
 
     def start
       super
+      result = client.xquery("SHOW COLUMNS FROM #{@table}")
+      @max_lengths = []
+      @column_names.each do |column|
+        info = result.select { |x| x['Field'] == column }.first
+        r = /(char|varchar)\(([\d]+)\)/
+        begin
+          max_length = info['Type'].scan(r)[0][1].to_i
+        rescue
+          max_length = nil
+        end
+        @max_lengths << max_length
+      end
     end
 
     def shutdown
@@ -59,36 +69,54 @@ module Fluent
     end
 
     def format(tag, time, record)
-      [tag, time, @format_proc.call(tag, time, record)].to_msgpack
+      [tag, time, format_proc.call(tag, time, record)].to_msgpack
     end
 
     def client
-      Mysql2::Client.new({
-          :host => @host,
-          :port => @port,
-          :username => @username,
-          :password => @password,
-          :database => @database,
-          :flags => Mysql2::Client::MULTI_STATEMENTS
-        })
+      Mysql2::Client.new(
+          host: @host,
+          port: @port,
+          username: @username,
+          password: @password,
+          database: @database,
+          flags: Mysql2::Client::MULTI_STATEMENTS
+        )
     end
 
     def write(chunk)
       @handler = client
       values_templates = []
-      values = Array.new
-      chunk.msgpack_each { |tag, time, data|
-        values_templates.push "(#{@column_names.map{|key| '?'}.join(',')})"
+      values = []
+      chunk.msgpack_each do |tag, time, data|
+        values_templates << "(#{ @column_names.map { |key| '?' }.join(',') })"
         values.concat(data)
-      }
-      sql = "INSERT INTO #{@table} (#{@column_names.join(',')}) VALUES #{values_templates.join(',')}"
-      if @on_duplicate_key_update
-        sql += @on_duplicate_key_update_sql
       end
+      sql = "INSERT INTO #{@table} (#{@column_names.join(',')}) VALUES #{values_templates.join(',')}"
+      sql += @on_duplicate_key_update_sql if @on_duplicate_key_update
 
-      $log.info "bulk insert sql => [#{Mysql2::Client.pseudo_bind(sql, values)}]"
+      $log.info "bulk insert values size => #{values_templates.size}"
       @handler.xquery(sql, values)
     end
 
+    private
+
+    def format_proc
+      proc do |tag, time, record|
+        values = []
+        @key_names.each_with_index do |key, i|
+          if key == '${time}'
+            value = Time.at(time).strftime('%Y-%m-%d %H:%M:%S')
+          else
+            if @max_lengths[i].nil? || record[key].nil?
+              value = record[key]
+            else
+              value = record[key].slice(0, @max_lengths[i])
+            end
+          end
+          values << value
+        end
+        values
+      end
+    end
   end
 end

data/test/helper.rb

@@ -4,7 +4,7 @@ begin
   Bundler.setup(:default, :development)
 rescue Bundler::BundlerError => e
   $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
+  $stderr.puts 'Run `bundle install` to install missing gems'
   exit e.status_code
 end
 require 'test/unit'
@@ -12,12 +12,12 @@ require 'test/unit'
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'fluent/test'
-unless ENV.has_key?('VERBOSE')
+unless ENV.key?('VERBOSE')
   nulllogger = Object.new
-  nulllogger.instance_eval {|obj|
+  nulllogger.instance_eval do|obj|
     def method_missing(method, *args)
     end
-  }
+  end
   $log = nulllogger
 end
 

data/test/plugin/test_out_mysql_bulk.rb

@@ -6,13 +6,12 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
-  def create_driver(conf = CONFIG, tag='test')
+  def create_driver(conf = CONFIG, tag = 'test')
     Fluent::Test::BufferedOutputTestDriver.new(Fluent::MysqlBulkOutput, tag).configure(conf)
   end
 
   def test_configure_error
-
-    assert_raise(Fluent::ConfigError) {
+    assert_raise(Fluent::ConfigError) do
       d = create_driver %[
         host localhost
         database test_app_development
@@ -23,9 +22,9 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
         on_duplicate_update_keys user_name,updated_at
         flush_interval 10s
       ]
-    }
+    end
 
-    assert_raise(Fluent::ConfigError) {
+    assert_raise(Fluent::ConfigError) do
       d = create_driver %[
         host localhost
         database test_app_development
@@ -36,9 +35,9 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
         on_duplicate_key_update true
         flush_interval 10s
       ]
-    }
+    end
 
-    assert_raise(Fluent::ConfigError) {
+    assert_raise(Fluent::ConfigError) do
       d = create_driver %[
         host localhost
         username root
@@ -49,12 +48,12 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
         on_duplicate_update_keys user_name,updated_at
         flush_interval 10s
       ]
-    }
+    end
   end
 
   def test_configure
     # not define format(default csv)
-    assert_nothing_raised(Fluent::ConfigError) {
+    assert_nothing_raised(Fluent::ConfigError) do
       d = create_driver %[
         host localhost
         database test_app_development
@@ -66,9 +65,9 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
         on_duplicate_update_keys user_name,updated_at
         flush_interval 10s
       ]
-    }
+    end
 
-    assert_nothing_raised(Fluent::ConfigError) {
+    assert_nothing_raised(Fluent::ConfigError) do
       d = create_driver %[
         database test_app_development
         username root
@@ -76,9 +75,9 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
         column_names id,user_name,created_at,updated_at
         table users
       ]
-    }
+    end
 
-    assert_nothing_raised(Fluent::ConfigError) {
+    assert_nothing_raised(Fluent::ConfigError) do
       d = create_driver %[
         database test_app_development
         username root
@@ -88,9 +87,9 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
         on_duplicate_key_update true
         on_duplicate_update_keys user_name,updated_at
       ]
-    }
+    end
 
-    assert_nothing_raised(Fluent::ConfigError) {
+    assert_nothing_raised(Fluent::ConfigError) do
       d = create_driver %[
         database test_app_development
         username root
@@ -101,7 +100,6 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
         on_duplicate_key_update true
         on_duplicate_update_keys user_name,updated_at
       ]
-    }
+    end
   end
-
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-bulk
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Hiroshi Toyama
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-05 00:00:00.000000000 Z
+date: 2014-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -146,7 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.0.rc.1
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: fluent plugin mysql bulk insert is high performance and on duplicate key