fluent-plugin-sql-enchanced 0.5.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e1b24e490528d5366af46eb41ba43c5ddccb4276
+  data.tar.gz: ee88b1faf613283b94fff2ab837ad78951006981
+SHA512:
+  metadata.gz: 6ecec17c0f370c4e762be6ed3ddfe4cbd4dfadaec1716654ab8ab4cba66a765fdc0e67eed4a106e168beaec83fca6dfa5718fe553a803a285049f22fdcc82d1d
+  data.tar.gz: 355e7a2999d82de43d5d7f60f2e9b712f85b984ba90c6e318d749f9d73e4c024a97718bf7780930d6ef02738a8f0653330858688c881db0a4cca5d86f3097120

data/.travis.yml

@@ -0,0 +1,32 @@
+sudo: false
+language: ruby
+
+addons:
+  postgresql: "9.4"
+
+rvm:
+  - 2.0
+  - 2.1
+  - 2.2.4
+  - 2.3.1
+  - ruby-head
+
+gemfile:
+ - Gemfile
+ - Gemfile.v0.12
+
+before_install:
+  - gem install bundler
+
+before_script:
+  - psql -U postgres -c "CREATE ROLE fluentd WITH LOGIN ENCRYPTED PASSWORD 'fluentd';"
+  - psql -U postgres -c "CREATE DATABASE fluentd_test OWNER fluentd;"
+
+script: bundle exec rake test
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+  exclude:
+    - rvm: 2.0
+      gemfile: Gemfile

data/Gemfile

@@ -0,0 +1,2 @@
+source "http://rubygems.org"
+gemspec

data/Gemfile.v0.12

@@ -0,0 +1,6 @@
+source "http://rubygems.org"
+
+gem 'json', '= 1.8.3'
+gem 'fluentd', '~> 0.12.0'
+
+gemspec

data/README.md

@@ -0,0 +1,148 @@
+# SQL input plugin for [Fluentd](http://fluentd.org) event collector
+
+## Overview
+
+This SQL plugin has two parts:
+
+1. SQL **input** plugin reads records from RDBMSes periodically. An example use case would be getting "diffs" of a table (based on the "updated_at" field).
+2. SQL **output** plugin that writes records into RDBMes. An example use case would be aggregating server/app/sensor logs into RDBMS systems.
+
+## Installation
+
+    $ fluent-gem install fluent-plugin-sql --no-document
+    $ fluent-gem install pg --no-document # for postgresql
+
+You should install actual RDBMS driver gem together. `pg` gem for postgresql adapter or `mysql2` gem for `mysql2` adapter. Other adapters supported by [ActiveRecord](https://github.com/rails/rails/tree/master/activerecord) should work.
+
+We recommend that mysql2 gem is higher than `0.3.12` and pg gem is higher than `0.16.0`.
+
+## Input: How It Works
+
+This plugin runs following SQL periodically:
+
+SELECT * FROM *table* WHERE *update\_column* > *last\_update\_column\_value* ORDER BY *update_column* ASC LIMIT 500
+
+What you need to configure is *update\_column*. The column should be an incremental column (such as AUTO\_ INCREMENT primary key) so that this plugin reads newly INSERTed rows. Alternatively, you can use a column incremented every time when you update the row (such as `last_updated_at` column) so that this plugin reads the UPDATEd rows as well.
+If you omit to set *update\_column* parameter, it uses primary key.
+
+It stores last selected rows to a file (named *state\_file*) to not forget the last row when Fluentd restarts.
+
+## Input: Configuration
+
+    <source>
+      @type sql
+
+      host rdb_host
+      database rdb_database
+      adapter mysql2_or_postgresql_or_etc
+      username myusername
+      password mypassword
+
+      tag_prefix my.rdb  # optional, but recommended
+
+      select_interval 60s  # optional
+      select_limit 500     # optional
+
+      state_file /var/run/fluentd/sql_state
+
+      <table>
+        table table1
+        tag table1  # optional
+        update_column update_col1
+        time_column time_col2  # optional
+      </table>
+
+      <table>
+        table table2
+        tag table2  # optional
+        update_column updated_at
+        time_column updated_at  # optional
+      </table>
+
+      # detects all tables instead of <table> sections
+      #all_tables
+    </source>
+
+* **host** RDBMS host
+* **port** RDBMS port
+* **database** RDBMS database name
+* **adapter** RDBMS driver name. You should install corresponding gem before start (mysql2 gem for mysql2 adapter, pg gem for postgresql adapter, etc.)
+* **username** RDBMS login user name
+* **password** RDBMS login password
+* **tag_prefix** prefix of tags of events. actual tag will be this\_tag\_prefix.tables\_tag (optional)
+* **select_interval** interval to run SQLs (optional)
+* **select_limit** LIMIT of number of rows for each SQL (optional)
+* **state_file** path to a file to store last rows
+* **all_tables** reads all tables instead of configuring each tables in \<table\> sections
+
+\<table\> sections:
+
+* **tag** tag name of events (optional; default value is table name)
+* **table** RDBM table name
+* **update_column**: see above description
+* **time_column** (optional): if this option is set, this plugin uses this column's value as the the event's time. Otherwise it uses current time.
+* **primary_key** (optional): if you want to get data from the table which doesn't have primary key like PostgreSQL's View, set this parameter.
+
+## Input: Limitation
+
+You should make sure target tables have index (and/or partitions) on the *update\_column*. Otherwise SELECT causes full table scan and serious performance problem.
+
+You can't replicate DELETEd rows.
+
+## Output: How It Works
+
+This plugin takes advantage of ActiveRecord underneath. For `host`, `port`, `database`, `adapter`, `username`, `password`, `socket` parameters, you can think of ActiveRecord's equivalent parameters.
+
+## Output: Configuration
+
+    <match my.rdb.*>
+      @type sql
+      host rdb_host
+      port 3306
+      database rdb_database
+      adapter mysql2_or_postgresql_or_etc
+      username myusername
+      password mypassword
+      socket path_to_socket
+      remove_tag_prefix my.rdb # optional, dual of tag_prefix in in_sql
+
+      <table>
+        table table1
+        column_mapping 'timestamp:created_at,fluentdata1:dbcol1,fluentdata2:dbcol2,fluentdata3:dbcol3'
+        # This is the default table because it has no "pattern" argument in <table>
+        # The logic is such that if all non-default <table> blocks
+        # do not match, the default one is chosen.
+        # The default table is required.
+      </table>
+
+      <table hello.*> # You can pass the same pattern you use in match statements.
+        table table2
+        # This is the non-default table. It is chosen if the tag matches the pattern
+        # AFTER remove_tag_prefix is applied to the incoming event. For example, if
+        # the message comes in with the tag my.rdb.hello.world, "remove_tag_prefix my.rdb"
+        # makes it "hello.world", which gets matched here because of "pattern hello.*".
+      </table>
+      
+      <table hello.world>
+        table table3
+        # This is the second non-default table. You can have as many non-default tables
+        # as you wish. One caveat: non-default tables are matched top-to-bottom and
+        # the events go into the first table it matches to. Hence, this particular table
+        # never gets any data, since the above "hello.*" subsumes "hello.world".
+      </table>
+    </match>
+
+* **host** RDBMS host
+* **port** RDBMS port
+* **database** RDBMS database name
+* **adapter** RDBMS driver name. You should install corresponding gem before start (mysql2 gem for mysql2 adapter, pg gem for postgresql adapter, etc.)
+* **username** RDBMS login user name
+* **password** RDBMS login password
+* **socket** RDBMS socket path
+* **remove_tag_prefix** remove the given prefix from the events. See "tag_prefix" in "Input: Configuration". (optional)
+
+\<table\> sections:
+
+* **table** RDBM table name
+* **column_mapping**: [Required] Record to table schema mapping. The format is consists of `from:to` or `key` values are separated by `,`. For example, if set 'item_id:id,item_text:data,updated_at' to **column_mapping**, `item_id` field of record is stored into `id` column and `updated_at` field of record is stored into `updated_at` column.
+* **<table pattern>**: the pattern to which the incoming event's tag (after it goes through `remove_tag_prefix`, if given). The patterns should follow the same syntax as [that of <match>](http://docs.fluentd.org/articles/config-file#match-pattern-how-you-control-the-event-flow-inside-fluentd). **Exactly one <table> element must NOT have this parameter so that it becomes the default table to store data**.

data/Rakefile

@@ -0,0 +1,14 @@
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/*/*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]
+

data/VERSION

@@ -0,0 +1 @@
+0.5.2

data/fluent-plugin-sql.gemspec

@@ -0,0 +1,28 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-sql-enchanced"
+  gem.description = "SQL input/output plugin for Fluentd event collector"
+  gem.homepage    = "https://github.com/zhron4x/fluent-plugin-sql-enchanced"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Sadayuki Furuhashi"]
+  gem.email       = "frsyuki@gmail.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+  gem.license = "Apache-2.0"
+
+  gem.add_dependency "fluentd", [">= 0.12.17", "< 2"]
+  gem.add_dependency 'activerecord', "~> 4.2"
+  gem.add_dependency 'activerecord-import', "~> 0.7"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "test-unit", "~> 3.1.0"
+  gem.add_development_dependency "test-unit-rr"
+  gem.add_development_dependency "test-unit-notify"
+  gem.add_development_dependency "pg"
+end

data/lib/fluent/plugin/in_sql.rb

@@ -0,0 +1,194 @@
+class Fluentd
+  module Setting
+    class InSql
+      include ActiveModel::Model
+      include Common
+
+      KEYS = [
+          :host,
+          :port,
+          :database,
+          :adapter,
+          :username,
+          :password,
+          :tag_prefix,
+          :select_interval,
+          :select_limit,
+          :state_file,
+          :table,
+          :all_tables
+      ].freeze
+
+      attr_accessor(*KEYS)
+      attr_accessor(:fields_descriptions)
+
+      validates :host, presence: true
+      validates :database, presence: true
+      validates :adapter, presence: true
+      validates :username, presence: true
+      validates :password, presence: true
+      validates :state_file, presence: true
+
+      def self.initial_params
+        {
+            host: 'localhost',
+            database: 'rdb_database',
+            adapter: 'mysql2',
+            username: 'myusername',
+            password: 'mypassword',
+            tag_prefix: 'my.rdb',
+            select_interval: '60s',
+            select_limit: '500',
+            state_file: "/tmp/data_enchilada-sql-#{Fluentd.instance.id}-#{Time.now.to_i}.pos",
+            table: [
+                {
+                    table: 'rdb_table',
+                    tag: 'rdb_table_tag',
+                    update_column: 'updated_at',
+                    time_column: 'updated_at',
+                    primary_key: ''
+                }
+            ]
+        }
+      end
+
+      def fields_descriptions
+        {
+            host: '* RDBMS host (required)',
+            port: 'RDBMS port (optional)',
+            database: '* RDBMS database name (required)',
+            adapter: '* RDBMS driver name. You should install corresponding gem before start (mysql2 gem for mysql2 adapter, pg gem for postgresql adapter, etc. (required)',
+            username: '* RDBMS login user name (required)',
+            password: '* RDBMS login password (required)',
+            tag_prefix: 'prefix of tags of events. actual tag will be this_tag_prefix.tables_tag (optional, but recommended)',
+            select_interval: 'interval to run SQLs (optional)',
+            select_limit: 'LIMIT of number of rows for each SQL (optional)',
+            state_file: '* path to a file to store last rows (required)',
+            all_tables: 'reads all tables instead of configuring each tables in <table> sections (optional)',
+            table: {
+                tag: 'tag name of events (optional; default value is table name)',
+                table: '* RDBM table name',
+                update_column: '* see above description',
+                time_column: "(optional): if this option is set, this plugin uses this column's value as the the event's time. Otherwise it uses current time.",
+                primary_key: "(optional): if you want to get data from the table which doesn't have primary key like PostgreSQL's View, set this parameter."
+            }
+        }
+      end
+
+      def common_options
+        [
+            :host,
+            :port,
+            :database,
+            :adapter,
+            :username,
+            :password,
+            :tag_prefix
+        ]
+      end
+
+      def advanced_options
+        [
+            :select_interval,
+            :select_limit,
+            :state_file
+        ]
+      end
+
+      def table=(value)
+        @tables = value.map do |t|
+          t.map{|field, value| ([field.to_sym, value] if table_fields.include?(field.to_sym))}.to_h
+        end
+      end
+
+      def table
+        @tables
+      end
+
+      def table_fields
+        [
+            :table,
+            :tag,
+            :update_column,
+            :time_column,
+            :primary_key
+        ]
+      end
+
+      def to_config
+        indent = "  "
+        config = "<source>\n"
+        config << "#{indent}type #{plugin_type_name}\n"
+        self.class.const_get(:KEYS).each do |key|
+          next if key == :table
+          next if key == :all_tables
+          config << indent
+          config << conf(key)
+          config << "\n"
+        end
+        tables = send(:table).reject{|t| t.values.join('') == ''} rescue []
+        if tables.present? && all_tables != '1'
+          tables.each do |tab|
+            config << "\n"
+            config << indent
+            config << "<table>\n"
+            tab.each do |key, value|
+              config << indent
+              config << indent
+              config << "#{key} #{value}"
+              config << "\n"
+            end
+            config << indent
+            config << "</table>\n"
+          end
+        else
+          config << indent
+          config << 'all_tables'
+          config << "\n"
+        end
+
+
+        config << "</source>\n"
+        config.gsub(/^[ ]*\n/m, "")
+      end
+
+      def self.create_from_config config
+        @setting = {}
+        @tables = []
+        table = {}
+        to_table = false
+        str_params = config.split("\r\n").map{|str| str.squish}
+        str_params.each do |str|
+          case str
+            when '<source>', '</source>'
+              next
+            when '<table>'
+              to_table = true
+            when '</table>'
+              to_table = false
+              @tables << table
+              table = {}
+            when 'all_tables'
+              @setting[:all_tables] = true
+            else
+              param = str.split(' ')
+              if to_table
+                table[param.first] = param.second
+              else
+                @setting[param.first] = param.second unless param.first == 'type'
+              end
+          end
+          if @tables.present?
+            @setting[:table] = @tables
+          end
+        end
+        self.new @setting
+      end
+
+      def plugin_name
+        "sql"
+      end
+    end
+  end
+end
+

data/lib/fluent/plugin/out_sql.rb

@@ -0,0 +1,261 @@
+require "fluent/output"
+
+module Fluent
+  class SQLOutput < BufferedOutput
+    Plugin.register_output('sql', self)
+
+    include SetTimeKeyMixin
+    include SetTagKeyMixin
+
+    # For fluentd v0.12.16 or earlier
+    class << self
+      unless method_defined?(:desc)
+        def desc(description)
+        end
+      end
+    end
+
+    desc 'RDBMS host'
+    config_param :host, :string
+    desc 'RDBMS port'
+    config_param :port, :integer, :default => nil
+    desc 'RDBMS driver name.'
+    config_param :adapter, :string
+    desc 'RDBMS login user name'
+    config_param :username, :string, :default => nil
+    desc 'RDBMS login password'
+    config_param :password, :string, :default => nil, :secret => true
+    desc 'RDBMS database name'
+    config_param :database, :string
+    desc 'RDBMS socket path'
+    config_param :socket, :string, :default => nil
+    desc 'remove the given prefix from the events'
+    config_param :remove_tag_prefix, :string, :default => nil
+    desc 'enable fallback'
+    config_param :enable_fallback, :bool, :default => true
+
+    attr_accessor :tables
+
+    unless method_defined?(:log)
+      define_method(:log) { $log }
+    end
+
+    # TODO: Merge SQLInput's TableElement
+    class TableElement
+      include Configurable
+
+      config_param :table, :string
+      config_param :column_mapping, :string
+      config_param :num_retries, :integer, :default => 5
+
+      attr_reader :model
+      attr_reader :pattern
+
+      def initialize(pattern, log, enable_fallback)
+        super()
+        @pattern = MatchPattern.create(pattern)
+        @log = log
+        @enable_fallback = enable_fallback
+      end
+
+      def configure(conf)
+        super
+
+        @mapping = parse_column_mapping(@column_mapping)
+        @format_proc = Proc.new { |record|
+          new_record = {}
+          @mapping.each { |k, c|
+            new_record[c] = record[k]
+          }
+          new_record
+        }
+      end
+
+      def init(base_model)
+        # See SQLInput for more details of following code
+        table_name = @table
+        @model = Class.new(base_model) do
+          self.table_name = table_name
+          self.inheritance_column = '_never_use_output_'
+        end
+
+        class_name = table_name.singularize.camelize
+        base_model.const_set(class_name, @model)
+        model_name = ActiveModel::Name.new(@model, nil, class_name)
+        @model.define_singleton_method(:model_name) { model_name }
+
+        # TODO: check column_names and table schema
+        # @model.column_names
+      end
+
+      def import(chunk)
+        records = []
+        chunk.msgpack_each { |tag, time, data|
+          begin
+            # format process should be moved to emit / format after supports error stream.
+            records << @model.new(@format_proc.call(data))
+          rescue => e
+            args = {:error => e.message, :error_class => e.class, :table => @table, :record => Yajl.dump(data)}
+            @log.warn "Failed to create the model. Ignore a record:", args
+          end
+        }
+        begin
+          @model.import(records)
+        rescue ActiveRecord::StatementInvalid, ActiveRecord::Import::MissingColumnError => e
+          if @enable_fallback
+            # ignore other exceptions to use Fluentd retry mechanizm
+            @log.warn "Got deterministic error. Fallback to one-by-one import", :error => e.message, :error_class => e.class
+            one_by_one_import(records)
+          else
+            $log.warn "Got deterministic error. Fallback is disabled", :error => e.message, :error_class => e.class
+            raise e
+          end
+        end
+      end
+
+      def one_by_one_import(records)
+        records.each { |record|
+          retries = 0
+          begin
+            @model.import([record])
+          rescue ActiveRecord::StatementInvalid, ActiveRecord::Import::MissingColumnError => e
+            @log.error "Got deterministic error again. Dump a record", :error => e.message, :error_class => e.class, :record => record
+          rescue => e
+            retries += 1
+            if retries > @num_retries
+              @log.error "Can't recover undeterministic error. Dump a record", :error => e.message, :error_class => e.class, :record => record
+              next
+            end
+
+            @log.warn "Failed to import a record: retry number = #{retries}", :error  => e.message, :error_class => e.class
+            sleep 0.5
+            retry
+          end
+        }
+      end
+
+      private
+
+      def parse_column_mapping(column_mapping_conf)
+        mapping = {}
+        column_mapping_conf.split(',').each { |column_map|
+          key, column = column_map.strip.split(':', 2)
+          column = key if column.nil?
+          mapping[key] = column
+        }
+        mapping
+      end
+    end
+
+    def initialize
+      super
+      require 'active_record'
+      require 'activerecord-import'
+    end
+
+    def configure(conf)
+      super
+
+      if remove_tag_prefix = conf['remove_tag_prefix']
+        @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
+      end
+
+      @tables = []
+      @default_table = nil
+      conf.elements.select { |e|
+        e.name == 'table'
+      }.each { |e|
+        te = TableElement.new(e.arg, log, @enable_fallback)
+        te.configure(e)
+        if e.arg.empty?
+          $log.warn "Detect duplicate default table definition" if @default_table
+          @default_table = te
+        else
+          @tables << te
+        end
+      }
+      @only_default = @tables.empty?
+
+      if @default_table.nil?
+        raise ConfigError, "There is no default table. <table> is required in sql output"
+      end
+    end
+
+    def start
+      super
+
+      config = {
+        :adapter => @adapter,
+        :host => @host,
+        :port => @port,
+        :database => @database,
+        :username => @username,
+        :password => @password,
+        :socket => @socket,
+      }
+
+      @base_model = Class.new(ActiveRecord::Base) do
+        self.abstract_class = true
+      end
+
+      SQLOutput.const_set("BaseModel_#{rand(1 << 31)}", @base_model)
+      ActiveRecord::Base.establish_connection(config)
+
+      # ignore tables if TableElement#init failed
+      @tables.reject! do |te|
+        init_table(te, @base_model)
+      end
+      init_table(@default_table, @base_model)
+    end
+
+    def shutdown
+      super
+    end
+
+    def emit(tag, es, chain)
+      if @only_default
+        super(tag, es, chain)
+      else
+        super(tag, es, chain, format_tag(tag))
+      end
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      ActiveRecord::Base.connection_pool.with_connection do
+
+        @tables.each { |table|
+          if table.pattern.match(chunk.key)
+            return table.import(chunk)
+          end
+        }
+        @default_table.import(chunk)
+      end
+    end
+
+    private
+
+    def init_table(te, base_model)
+      begin
+        te.init(base_model)
+        log.info "Selecting '#{te.table}' table"
+        false
+      rescue => e
+        log.warn "Can't handle '#{te.table}' table. Ignoring.", :error => e.message, :error_class => e.class
+        log.warn_backtrace e.backtrace
+        true
+      end
+    end
+
+    def format_tag(tag)
+      if @remove_tag_prefix
+        tag.gsub(@remove_tag_prefix, '')
+      else
+        tag
+      end
+    end
+  end
+end

data/test/fixtures/schema.rb

@@ -0,0 +1,24 @@
+require "active_record"
+ActiveRecord::Base.establish_connection(host: "localhost",
+                                        port: 5432,
+                                        username: "fluentd",
+                                        password: "fluentd",
+                                        adapter: "postgresql",
+                                        database: "fluentd_test")
+ActiveRecord::Schema.define(version: 20160225030107) do
+  create_table "logs", force: :cascade do |t|
+    t.string   "host"
+    t.string   "ident"
+    t.string   "pid"
+    t.text     "message"
+    t.datetime "created_at",  null: false
+    t.datetime "updated_at",  null: false
+  end
+
+  create_table "messages", force: :cascade do |t|
+    t.string "message"
+    t.datetime "created_at",  null: false
+    t.datetime "updated_at",  null: false
+  end
+end
+

data/test/helper.rb

@@ -0,0 +1,8 @@
+require "test/unit"
+require "test/unit/rr"
+require "test/unit/notify"
+require "fluent/test"
+require "fluent/plugin/out_sql"
+require "fluent/plugin/in_sql"
+
+load "fixtures/schema.rb"

data/test/plugin/test_in_sql.rb

@@ -0,0 +1,86 @@
+require "helper"
+
+class SqlInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def teardown
+  end
+
+  CONFIG = %[
+    adapter postgresql
+    host localhost
+    port 5432
+    database fluentd_test
+
+    username fluentd
+    password fluentd
+
+    tag_prefix db
+
+    <table>
+      table messages
+      tag logs
+      update_column updated_at
+      time_column updated_at
+    </table>
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::SQLInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    expected = {
+      host: "localhost",
+      port: 5432,
+      adapter: "postgresql",
+      database: "fluentd_test",
+      username: "fluentd",
+      password: "fluentd",
+      tag_prefix: "db"
+    }
+    actual = {
+      host: d.instance.host,
+      port: d.instance.port,
+      adapter: d.instance.adapter,
+      database: d.instance.database,
+      username: d.instance.username,
+      password: d.instance.password,
+      tag_prefix: d.instance.tag_prefix
+    }
+    assert_equal(expected, actual)
+    tables = d.instance.instance_variable_get(:@tables)
+    assert_equal(1, tables.size)
+    messages = tables.first
+    assert_equal("messages", messages.table)
+    assert_equal("logs", messages.tag)
+  end
+
+  def test_message
+    d = create_driver(CONFIG + "select_interval 1")
+    Message.create!(message: "message 1")
+    Message.create!(message: "message 2")
+    Message.create!(message: "message 3")
+
+    d.run
+
+    assert_equal("db.logs", d.emits[0][0])
+    expected = [
+      [d.emits[0][1], "message 1"],
+      [d.emits[1][1], "message 2"],
+      [d.emits[2][1], "message 3"],
+    ]
+    actual = [
+      [Time.parse(d.emits[0][2]["updated_at"]).to_i, d.emits[0][2]["message"]],
+      [Time.parse(d.emits[1][2]["updated_at"]).to_i, d.emits[1][2]["message"]],
+      [Time.parse(d.emits[2][2]["updated_at"]).to_i, d.emits[2][2]["message"]],
+    ]
+    assert_equal(expected, actual)
+  end
+
+  class Message < ActiveRecord::Base
+  end
+end

data/test/plugin/test_out_sql.rb

@@ -0,0 +1,114 @@
+require "helper"
+
+class SqlOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def teardown
+  end
+
+  CONFIG = %[
+    host localhost
+    port 5432
+    adapter postgresql
+
+    database fluentd_test
+    username fluentd
+    password fluentd
+
+    remove_tag_prefix db
+
+    <table>
+      table logs
+      column_mapping timestamp:created_at,host:host,ident:ident,pid:pid,message:message
+    </table>
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::SQLOutput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    expected = {
+      host: "localhost",
+      port: 5432,
+      adapter: "postgresql",
+      database: "fluentd_test",
+      username: "fluentd",
+      password: "fluentd",
+      remove_tag_suffix: /^db/,
+      enable_fallback: true
+    }
+    actual = {
+      host: d.instance.host,
+      port: d.instance.port,
+      adapter: d.instance.adapter,
+      database: d.instance.database,
+      username: d.instance.username,
+      password: d.instance.password,
+      remove_tag_suffix: d.instance.remove_tag_prefix,
+      enable_fallback: d.instance.enable_fallback
+    }
+    assert_equal(expected, actual)
+    assert_empty(d.instance.tables)
+    default_table = d.instance.instance_variable_get(:@default_table)
+    assert_equal("logs", default_table.table)
+  end
+
+  def test_emit
+    d = create_driver
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.emit({"message" => "message1"}, time)
+    d.emit({"message" => "message2"}, time)
+
+    d.run
+
+    default_table = d.instance.instance_variable_get(:@default_table)
+    model = default_table.instance_variable_get(:@model)
+    assert_equal(2, model.all.count)
+    messages = model.pluck(:message).sort
+    assert_equal(["message1", "message2"], messages)
+  end
+
+  class Fallback < self
+    def test_simple
+      d = create_driver
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+      d.emit({"message" => "message1"}, time)
+      d.emit({"message" => "message2"}, time)
+
+      d.run do
+        default_table = d.instance.instance_variable_get(:@default_table)
+        model = default_table.instance_variable_get(:@model)
+        mock(model).import(anything).at_least(1) do
+          raise ActiveRecord::Import::MissingColumnError.new("dummy_table", "dummy_column")
+        end
+        mock(default_table).one_by_one_import(anything)
+      end
+    end
+
+    def test_limit
+      d = create_driver
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+      d.emit({"message" => "message1"}, time)
+      d.emit({"message" => "message2"}, time)
+
+      d.run do
+        default_table = d.instance.instance_variable_get(:@default_table)
+        model = default_table.instance_variable_get(:@model)
+        mock(model).import([anything, anything]).once do
+          raise ActiveRecord::Import::MissingColumnError.new("dummy_table", "dummy_column")
+        end
+        mock(model).import([anything]).times(12) do
+          raise StandardError
+        end
+        assert_equal(5, default_table.instance_variable_get(:@num_retries))
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,174 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-sql-enchanced
+version: !ruby/object:Gem::Version
+  version: 0.5.2
+platform: ruby
+authors:
+- Sadayuki Furuhashi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-03-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.17
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.17
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: activerecord-import
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit-notify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: SQL input/output plugin for Fluentd event collector
+email: frsyuki@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".travis.yml"
+- Gemfile
+- Gemfile.v0.12
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-sql.gemspec
+- lib/fluent/plugin/in_sql.rb
+- lib/fluent/plugin/out_sql.rb
+- test/fixtures/schema.rb
+- test/helper.rb
+- test/plugin/test_in_sql.rb
+- test/plugin/test_out_sql.rb
+homepage: https://github.com/zhron4x/fluent-plugin-sql-enchanced
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: SQL input/output plugin for Fluentd event collector
+test_files: []