fluent-plugin-mysql-2 0.3.7

data/lib/fluent/plugin/out_mysql.rb

@@ -0,0 +1,126 @@
+class Fluent::MysqlOutput < Fluent::BufferedOutput
+  Fluent::Plugin.register_output('mysql', self)
+
+  include Fluent::SetTimeKeyMixin
+  include Fluent::SetTagKeyMixin
+
+  config_param :host, :string
+  config_param :port, :integer, :default => nil
+  config_param :database, :string
+  config_param :username, :string
+  config_param :password, :string, :default => '', :secret => true
+  config_param :sslkey, :string, :default => nil
+  config_param :sslcert, :string, :default => nil
+  config_param :sslca, :string, :default => nil
+  config_param :sslcapath, :string, :default => nil
+  config_param :sslcipher, :string, :default => nil
+  config_param :sslverify, :bool, :default => nil
+
+  config_param :key_names, :string, :default => nil # nil allowed for json format
+  config_param :sql, :string, :default => nil
+  config_param :table, :string, :default => nil
+  config_param :columns, :string, :default => nil
+
+  config_param :format, :string, :default => "raw" # or json
+
+  attr_accessor :handler
+
+  def initialize
+    super
+    require 'mysql2-cs-bind'
+    require 'jsonpath'
+  end
+
+  # Define `log` method for v0.10.42 or earlier
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
+  def configure(conf)
+    super
+
+    log.warn "[mysql] This plugin deprecated. You should use mysql_bulk."
+
+    # TODO tag_mapped
+
+    case @format
+    when 'json'
+      @format_proc = Proc.new{|tag, time, record| record.to_json}
+    when 'jsonpath'
+      @key_names = @key_names.split(/\s*,\s*/)
+      @format_proc = Proc.new do |tag, time, record|
+        json = record.to_json
+        @key_names.map do |k|
+          JsonPath.new(k.strip).on(json).first
+        end
+      end
+    else
+      @key_names = @key_names.split(/\s*,\s*/)
+      @format_proc = Proc.new{|tag, time, record| @key_names.map{|k| record[k]}}
+    end
+
+    if @columns.nil? and @sql.nil?
+      raise Fluent::ConfigError, "columns or sql MUST be specified, but missing"
+    end
+    if @columns and @sql
+      raise Fluent::ConfigError, "both of columns and sql are specified, but specify one of them"
+    end
+
+    if @sql
+      begin
+        if @format == 'json'
+          Mysql2::Client.pseudo_bind(@sql, [nil])
+        else
+          Mysql2::Client.pseudo_bind(@sql, @key_names.map{|n| nil})
+        end
+      rescue ArgumentError => e
+        raise Fluent::ConfigError, "mismatch between sql placeholders and key_names"
+      end
+    else # columns
+      raise Fluent::ConfigError, "table missing" unless @table
+      @columns = @columns.split(/\s*,\s*/)
+      cols = @columns.join(',')
+      placeholders = if @format == 'json'
+                       '?'
+                     else
+                       @key_names.map{|k| '?'}.join(',')
+                     end
+      @sql = "INSERT INTO #{@table} (#{cols}) VALUES (#{placeholders})"
+    end
+  end
+
+  def start
+    super
+  end
+
+  def shutdown
+    super
+  end
+
+  def format(tag, time, record)
+    [tag, time, @format_proc.call(tag, time, record)].to_msgpack
+  end
+
+  def client
+    Mysql2::Client.new({
+        :host => @host, :port => @port,
+        :username => @username, :password => @password,
+        :database => @database,
+        :sslkey => @sslkey,
+        :sslcert => @sslcert,
+        :sslca => @sslca,
+        :sslcapath => @sslcapath,
+        :sslcipher => @sslcipher,
+        :sslverify => @sslverify,
+        :flags => Mysql2::Client::MULTI_STATEMENTS,
+      })
+  end
+
+  def write(chunk)
+    handler = self.client
+    chunk.msgpack_each { |tag, time, data|
+      handler.xquery(@sql, data)
+    }
+    handler.close
+  end
+end

data/lib/fluent/plugin/out_mysql_bulk.rb

@@ -0,0 +1,234 @@
+# frozen_string_literal: true
+
+require 'fluent/plugin/output'
+require 'oj'
+
+module Fluent::Plugin
+  class MysqlBulkOutput < Output
+    Fluent::Plugin.register_output('mysql_bulk', self)
+
+    helpers :compat_parameters, :inject
+
+    config_param :host, :string, default: '127.0.0.1',
+                 desc: "Database host."
+    config_param :port, :integer, default: 3306,
+                 desc: "Database port."
+    config_param :database, :string,
+                 desc: "Database name."
+    config_param :username, :string,
+                 desc: "Database user."
+    config_param :password, :string, default: '', secret: true,
+                 desc: "Database password."
+    config_param :sslkey, :string, default: nil,
+                 desc: "SSL key."
+    config_param :sslcert, :string, default: nil,
+                 desc: "SSL cert."
+    config_param :sslca, :string, default: nil,
+                 desc: "SSL CA."
+    config_param :sslcapath, :string, default: nil,
+                     desc: "SSL CA path."
+    config_param :sslcipher, :string, default: nil,
+                         desc: "SSL cipher."
+    config_param :sslverify, :bool, default: nil,
+                             desc: "SSL Verify Server Certificate."
+
+    config_param :column_names, :string,
+                 desc: "Bulk insert column."
+    config_param :key_names, :string, default: nil,
+                 desc: <<-DESC
+Value key names, ${time} is placeholder Time.at(time).strftime("%Y-%m-%d %H:%M:%S").
+DESC
+    config_param :json_key_names, :string, default: nil,
+                  desc: "Key names which store data as json"
+    config_param :table, :string,
+                 desc: "Bulk insert table."
+
+    config_param :unixtimestamp_key_names, :string, default: nil,
+                 desc: "Key names which store data as datetime from unix time stamp"
+
+    config_param :on_duplicate_key_update, :bool, default: false,
+                 desc: "On duplicate key update enable."
+    config_param :on_duplicate_update_keys, :string, default: nil,
+                 desc: "On duplicate key update column, comma separator."
+    config_param :on_duplicate_update_custom_values, :string, default: nil,
+                 desc: "On_duplicate_update_custom_values, comma separator. specify the column name is insert value, custom value is use ${sql conditions}"
+    config_param :insert_ignore, :bool, default: false,
+                 :desc => "Use INSERT IGNORE"
+    config_param :max_rows_per_insert, :integer, default: 0,
+                 :desc => "Maximum number of rows to insert in each statement"
+
+    config_param :transaction_isolation_level, :enum, list: [:read_uncommitted, :read_committed, :repeatable_read, :serializable], default: nil,
+                 desc: "Set transaction isolation level."
+
+    attr_accessor :handler
+
+    def initialize
+      super
+      require 'mysql2-cs-bind'
+    end
+
+    def configure(conf)
+      compat_parameters_convert(conf, :buffer, :inject)
+      super
+
+      if @column_names.nil?
+        fail Fluent::ConfigError, 'column_names MUST specified, but missing'
+      end
+
+      if @on_duplicate_key_update
+        if @on_duplicate_update_keys.nil?
+          fail Fluent::ConfigError, 'on_duplicate_key_update = true , on_duplicate_update_keys nil!'
+        end
+        @on_duplicate_update_keys = @on_duplicate_update_keys.split(',')
+
+        if !@on_duplicate_update_custom_values.nil?
+          @on_duplicate_update_custom_values = @on_duplicate_update_custom_values.split(',')
+          if @on_duplicate_update_custom_values.length != @on_duplicate_update_keys.length
+            fail Fluent::ConfigError, <<-DESC
+on_duplicate_update_keys and on_duplicate_update_custom_values must be the same length
+DESC
+          end
+        end
+
+        @on_duplicate_key_update_sql = ' ON DUPLICATE KEY UPDATE '
+        updates = []
+        @on_duplicate_update_keys.each_with_index do |update_column, i|
+          if @on_duplicate_update_custom_values.nil? || @on_duplicate_update_custom_values[i] == "#{update_column}"
+            updates << "#{update_column} = VALUES(#{update_column})"
+          else
+            value = @on_duplicate_update_custom_values[i].to_s.match(/\${(.*)}/)[1]
+            escape_value = Mysql2::Client.escape(value)
+            updates << "#{update_column} = #{escape_value}"
+          end
+        end
+        @on_duplicate_key_update_sql += updates.join(',')
+      end
+
+      @column_names = @column_names.split(',').collect(&:strip)
+      @key_names = @key_names.nil? ? @column_names : @key_names.split(',').collect(&:strip)
+      @values_template = "(#{ @column_names.map { |key| '?' }.join(',') })"
+      @insert_columns = @column_names.map{|x| "`#{x.to_s.gsub('`', '``')}`"}.join(',')
+      @json_key_names = @json_key_names.split(',') if @json_key_names
+      @unixtimestamp_key_names = @unixtimestamp_key_names.split(',') if @unixtimestamp_key_names
+    end
+
+    def check_table_schema(database: @database, table: @table)
+      _client = client(database)
+      result = _client.xquery("SHOW COLUMNS FROM #{table}")
+      max_lengths = []
+      @column_names.each do |column|
+        info = result.select { |x| x['Field'] == column }.first
+        r = /(char|varchar)\(([\d]+)\)/
+        begin
+          max_length = info['Type'].scan(r)[0][1].to_i
+        rescue
+          max_length = nil
+        end
+        max_lengths << max_length
+      end
+      max_lengths
+    ensure
+      if not _client.nil? then _client.close end
+    end
+
+    def format(tag, time, record)
+      record = inject_values_to_record(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def formatted_to_msgpack_binary
+      true
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def client(database)
+      Mysql2::Client.new(
+          host: @host,
+          port: @port,
+          username: @username,
+          password: @password,
+          database: database,
+          sslkey: @sslkey,
+          sslcert: @sslcert,
+          sslca: @sslca,
+          sslcapath: @sslcapath,
+          sslcipher: @sslcipher,
+          sslverify: @sslverify,
+          flags: Mysql2::Client::MULTI_STATEMENTS
+        )
+    end
+
+    def expand_placeholders(metadata)
+      database = extract_placeholders(@database, metadata).gsub('.', '_')
+      table = extract_placeholders(@table, metadata).gsub('.', '_')
+      return database, table
+    end
+
+    def write(chunk)
+      database, table = expand_placeholders(chunk.metadata)
+      max_lengths = check_table_schema(database: database, table: table)
+      @handler = client(database)
+      values = []
+      chunk.msgpack_each do |tag, time, data|
+        data = format_proc.call(tag, time, data, max_lengths)
+        values << Mysql2::Client.pseudo_bind(@values_template, data)
+      end
+
+      @handler.query("SET SESSION TRANSACTION ISOLATION LEVEL #{transaction_isolation_level}") if @transaction_isolation_level
+      slice_size = @max_rows_per_insert > 0 ? @max_rows_per_insert : values.length
+      values.each_slice(slice_size) do |slice|
+        sql = "INSERT #{@insert_ignore ? "IGNORE" : ""} INTO #{table} (#{@insert_columns}) VALUES #{values.join(',')}"
+        sql += @on_duplicate_key_update_sql if @on_duplicate_key_update
+
+        @handler.xquery(sql)
+      end
+      log.info "bulk insert values size (table: #{@table}) => #{values.size}"
+      @handler.close
+    end
+
+    private
+
+    def format_proc
+      proc do |tag, time, record, max_lengths|
+        values = []
+        @key_names.each_with_index do |key, i|
+          if key == '${time}'
+            value = Time.at(time).strftime('%Y-%m-%d %H:%M:%S')
+          else
+            if max_lengths[i].nil? || record[key].nil?
+              value = record[key]
+            else
+              value = record[key].to_s.slice(0, max_lengths[i])
+            end
+
+            if @json_key_names && @json_key_names.include?(key)
+              value = Oj.dump(value)
+            end
+
+            if @unixtimestamp_key_names && @unixtimestamp_key_names.include?(key)
+              value = Time.at(value).strftime('%Y-%m-%d %H:%M:%S')
+            end
+          end
+          values << value
+        end
+        values
+      end
+    end
+
+    def transaction_isolation_level
+      case @transaction_isolation_level
+      when :read_uncommitted
+        "READ UNCOMMITTED"
+      when :read_committed
+        "READ COMMITTED"
+      when :repeatable_read
+        "REPEATABLE READ"
+      when :serializable
+        "SERIALIZABLE"
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_mysql'
+require 'fluent/plugin/out_mysql_bulk'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_mysql.rb

@@ -0,0 +1,199 @@
+require 'helper'
+require 'mysql2-cs-bind'
+
+class MysqlOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+host db.local
+database testing
+username testuser
+sql INSERT INTO tbl SET jsondata=?
+format json
+  ]
+
+  def create_driver(conf = CONFIG, tag='test')
+    d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::MysqlOutput, tag).configure(conf)
+    d.instance.instance_eval {
+      def client
+        obj = Object.new
+        obj.instance_eval {
+          def xquery(*args); [1]; end
+          def close; true; end
+        }
+        obj
+      end
+    }
+    d
+  end
+
+  def test_configure
+    d = create_driver %[
+host database.local
+database foo
+username bar
+sql INSERT INTO baz SET jsondata=?
+format json
+    ]
+    d = create_driver %[
+host database.local
+database foo
+username bar
+table baz
+columns jsondata
+format json
+    ]
+    d = create_driver %[
+host database.local
+database foo
+username bar
+password mogera
+key_names field1,field2,field3
+table baz
+columns col1, col2 ,col3
+    ]
+    assert_equal ['field1', 'field2', 'field3'], d.instance.key_names
+    assert_equal 'INSERT INTO baz (col1,col2,col3) VALUES (?,?,?)', d.instance.sql
+    d = create_driver %[
+host database.local
+database foo
+username bar
+password mogera
+key_names field1 ,field2, field3
+table baz
+columns col1, col2 ,col3
+    ]
+    assert_equal ['field1', 'field2', 'field3'], d.instance.key_names
+    assert_equal 'INSERT INTO baz (col1,col2,col3) VALUES (?,?,?)', d.instance.sql
+
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+host database.local
+database foo
+username bar
+password mogera
+key_names field1,field2,field3
+sql INSERT INTO baz (col1,col2,col3,col4) VALUES (?,?,?,?)
+      ]
+    }
+
+
+  end
+
+  def test_format
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    #d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
+    #d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
+    d.expect_format ['test', time, {"a" => 1}.to_json].to_msgpack
+    d.expect_format ['test', time, {"a" => 2}.to_json].to_msgpack
+
+    d.run
+  end
+
+  def test_time_and_tag_key
+    d = create_driver %[
+host database.local
+database foo
+username bar
+password mogera
+include_time_key yes
+utc
+include_tag_key yes
+table baz
+key_names time,tag,field1,field2,field3,field4
+sql INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)
+    ]
+    assert_equal 'INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)', d.instance.sql
+
+    time = Time.parse('2012-12-17 01:23:45 UTC').to_i
+    record = {'field1'=>'value1','field2'=>'value2','field3'=>'value3','field4'=>'value4'}
+    d.emit(record, time)
+    d.expect_format ['test', time, ['2012-12-17T01:23:45Z','test','value1','value2','value3','value4']].to_msgpack
+    d.run
+  end
+
+  def test_time_and_tag_key_complex
+    d = create_driver %[
+host database.local
+database foo
+username bar
+password mogera
+include_time_key yes
+utc
+time_format %Y%m%d-%H%M%S
+time_key timekey
+include_tag_key yes
+tag_key tagkey
+table baz
+key_names timekey,tagkey,field1,field2,field3,field4
+sql INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)
+    ]
+    assert_equal 'INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)', d.instance.sql
+
+    time = Time.parse('2012-12-17 09:23:45 +0900').to_i # JST(+0900)
+    record = {'field1'=>'value1','field2'=>'value2','field3'=>'value3','field4'=>'value4'}
+    d.emit(record, time)
+    d.expect_format ['test', time, ['20121217-002345','test','value1','value2','value3','value4']].to_msgpack
+    d.run
+  end
+
+  def test_time_and_tag_key_json
+    d = create_driver %[
+host database.local
+database foo
+username bar
+password mogera
+include_time_key yes
+utc
+time_format %Y%m%d-%H%M%S
+time_key timekey
+include_tag_key yes
+tag_key tagkey
+table accesslog
+columns jsondata
+format json
+    ]
+    assert_equal 'INSERT INTO accesslog (jsondata) VALUES (?)', d.instance.sql
+
+    time = Time.parse('2012-12-17 09:23:45 +0900').to_i # JST(+0900)
+    record = {'field1'=>'value1'}
+    d.emit(record, time)
+    # Ruby 1.9.3 Hash saves its key order, so this code is OK.
+    d.expect_format ['test', time, record.merge({'timekey'=>'20121217-002345','tagkey'=>'test'}).to_json].to_msgpack
+    d.run
+  end
+
+  def test_jsonpath_format
+    d = create_driver %[
+      host database.local
+      database foo
+      username bar
+      password mogera
+      include_time_key yes
+      utc
+      include_tag_key yes
+      table baz
+      format jsonpath
+      key_names time, tag, id, data.name, tags[0]
+      sql INSERT INTO baz (coltime,coltag,id,name,tag1) VALUES (?,?,?,?,?)
+    ]
+    assert_equal 'INSERT INTO baz (coltime,coltag,id,name,tag1) VALUES (?,?,?,?,?)', d.instance.sql
+
+    time = Time.parse('2012-12-17 01:23:45 UTC').to_i
+    record = { 'id' => 15, 'data'=> {'name' => 'jsonpath' }, 'tags' => ['unit', 'simple'] }
+    d.emit(record, time)
+    d.expect_format ['test', time, ['2012-12-17T01:23:45Z','test',15,'jsonpath','unit']].to_msgpack
+    d.run
+  end
+
+  def test_write
+    # hmm....
+  end
+end