fluent-plugin-sql 0.6.1 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 8bdf7e835bed062213f63778607f5f4e3a6f9a02
-  data.tar.gz: d9b7d721ef9ad035193fbf0eb5a5460e88caa9aa
+SHA256:
+  metadata.gz: '00987a628d7afc94546e68d5374331e82c65eca49aa8e3aad33b8f5ffb5c8f59'
+  data.tar.gz: e809d69583ab25571878d9bc4adb64a26841e14347e9deb973e1a659262bbf47
 SHA512:
-  metadata.gz: ab807cd58fa29576d9164e4ab1e307a3b8cde72dd129e608acc78da01c15499e9ba9def596b044604bd2890c877fc3d66be79ca31d441e24f0d6f5cba8a741f6
-  data.tar.gz: 23a7698d628cd9dad85b9d1ab8f50ade3d925531cb230cbc9a93ace20e03ab157bd0ed09c1a3f97a9380931135acda9837402249e6b360e1881f694b157eff3e
+  metadata.gz: b5cf74d0668a3e7b26220d2b56179c3668a99e2a51e8d99b7468d797bdc3c7e0fe81ca813ede95713fc3a2dc6484929505953c0be7239129343f138798ad7fe4
+  data.tar.gz: c45922cd85d83e3ed89629c55232cd0fe83106b4961f1d2dfaafd8564d8db900b4740f78e2f12b7596ee6ce025a43efaf4cb409ca9ed418434dcc3ed447a4456

data/.travis.yml

@@ -5,19 +5,17 @@ addons:
   postgresql: "9.4"
 
 rvm:
-  - 2.0
-  - 2.1
-  - 2.2.4
-  - 2.3.1
-  - 2.4.1
+  - 2.4.10
+  - 2.5.8
+  - 2.6.6
+  - 2.7.1
   - ruby-head
 
 gemfile:
  - Gemfile
- - Gemfile.v0.12
 
 before_install:
-  - gem install bundler
+  - gem update bundler
 
 before_script:
   - psql -U postgres -c "CREATE ROLE fluentd WITH LOGIN ENCRYPTED PASSWORD 'fluentd';"
@@ -28,6 +26,3 @@ script: bundle exec rake test
 matrix:
   allow_failures:
     - rvm: ruby-head
-  exclude:
-    - rvm: 2.0
-      gemfile: Gemfile

data/ChangeLog

@@ -0,0 +1,20 @@
+Release 2.1.0 - 2020/09/09
+
+　* in_sql: Use Fluent::EventTime instead of Integer to preserve millisecond precision
+
+Release 2.0.0 - 2020/04/22
+
+  * out_sql: Remove v0.12 API code and use full v1 API. Change buffer format.
+
+Release 1.1.1 - 2019/05/10
+
+  * out_sql: Support schema_search_path option of PostgreSQL
+
+Release 1.1.0 - 2018/10/04
+
+  * Upgrade ActiveRecord to 5.1
+
+Release 1.0.0 - 2018/04/06
+
+  * Support v1 API
+

data/README.md

@@ -5,7 +5,16 @@
 This SQL plugin has two parts:
 
 1. SQL **input** plugin reads records from RDBMSes periodically. An example use case would be getting "diffs" of a table (based on the "updated_at" field).
-2. SQL **output** plugin that writes records into RDBMes. An example use case would be aggregating server/app/sensor logs into RDBMS systems.
+2. SQL **output** plugin that writes records into RDBMSes. An example use case would be aggregating server/app/sensor logs into RDBMS systems.
+
+## Requirements
+
+| fluent-plugin-sql | fluentd    | ruby   |
+|-------------------|------------|--------|
+| >= 1.0.0          | >= v0.14.4 | >= 2.1 |
+| <  1.0.0          | <  v0.14.0 | >= 1.9 |
+
+NOTE: fluent-plugin-sql v2's buffer format is different from v1. If you update the plugin to v2, don't reuse v1's buffer.
 
 ## Installation
 
@@ -16,6 +25,13 @@ You should install actual RDBMS driver gem together. `pg` gem for postgresql ada
 
 We recommend that mysql2 gem is higher than `0.3.12` and pg gem is higher than `0.16.0`.
 
+If you use ruby 2.1, use pg gem 0.21.0 (< 1.0.0) because ActiveRecord 5.1.4 or earlier doesn't support Ruby 2.1.
+
+### Resolve tzinfo version conflict
+
+If you want to use fluent-plugin-sql with recent fluentd/td-agent, you need to downgrade tzinfo to v1.x manually.
+See also this comment: https://github.com/fluent/fluent-plugin-sql/issues/87#issuecomment-614552292
+
 ## Input: How It Works
 
 This plugin runs following SQL periodically:
@@ -139,10 +155,12 @@ This plugin takes advantage of ActiveRecord underneath. For `host`, `port`, `dat
 * **username** RDBMS login user name
 * **password** RDBMS login password
 * **socket** RDBMS socket path
+* **pool** A connection pool synchronizes thread access to a limited number of database connections
+* **timeout** RDBMS connection timeout
 * **remove_tag_prefix** remove the given prefix from the events. See "tag_prefix" in "Input: Configuration". (optional)
 
 \<table\> sections:
 
 * **table** RDBM table name
 * **column_mapping**: [Required] Record to table schema mapping. The format is consists of `from:to` or `key` values are separated by `,`. For example, if set 'item_id:id,item_text:data,updated_at' to **column_mapping**, `item_id` field of record is stored into `id` column and `updated_at` field of record is stored into `updated_at` column.
-* **<table pattern>**: the pattern to which the incoming event's tag (after it goes through `remove_tag_prefix`, if given). The patterns should follow the same syntax as [that of <match>](http://docs.fluentd.org/articles/config-file#match-pattern-how-you-control-the-event-flow-inside-fluentd). **Exactly one <table> element must NOT have this parameter so that it becomes the default table to store data**.
+* **\<table pattern\>**: the pattern to which the incoming event's tag (after it goes through `remove_tag_prefix`, if given). The patterns should follow the same syntax as [that of \<match\>](https://docs.fluentd.org/configuration/config-file#how-match-patterns-work). **Exactly one \<table\> element must NOT have this parameter so that it becomes the default table to store data**.

data/VERSION

@@ -1 +1 @@
-0.6.1
+2.1.0

data/fluent-plugin-sql.gemspec

@@ -4,12 +4,11 @@ $:.push File.expand_path('../lib', __FILE__)
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-sql"
   gem.description = "SQL input/output plugin for Fluentd event collector"
-  gem.homepage    = "https://github.com/frsyuki/fluent-plugin-sql"
+  gem.homepage    = "https://github.com/fluent/fluent-plugin-sql"
   gem.summary     = gem.description
   gem.version     = File.read("VERSION").strip
   gem.authors     = ["Sadayuki Furuhashi"]
   gem.email       = "frsyuki@gmail.com"
-  gem.has_rdoc    = false
   #gem.platform    = Gem::Platform::RUBY
   gem.files       = `git ls-files`.split("\n")
   gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
@@ -18,11 +17,11 @@ Gem::Specification.new do |gem|
   gem.license = "Apache-2.0"
 
   gem.add_dependency "fluentd", [">= 0.12.17", "< 2"]
-  gem.add_dependency 'activerecord', "~> 4.2"
+  gem.add_dependency 'activerecord', "~> 5.1"
   gem.add_dependency 'activerecord-import', "~> 0.7"
   gem.add_development_dependency "rake", ">= 0.9.2"
-  gem.add_development_dependency "test-unit", "~> 3.1.0"
+  gem.add_development_dependency "test-unit", "> 3.1.0"
   gem.add_development_dependency "test-unit-rr"
   gem.add_development_dependency "test-unit-notify"
-  gem.add_development_dependency "pg"
+  gem.add_development_dependency "pg", '~> 1.0'
 end

data/lib/fluent/plugin/in_sql.rb

@@ -16,67 +16,60 @@
 #    limitations under the License.
 #
 
-require "fluent/input"
+require "fluent/plugin/input"
 
-module Fluent
+module Fluent::Plugin
 
   require 'active_record'
 
   class SQLInput < Input
-    Plugin.register_input('sql', self)
-
-    # For fluentd v0.12.16 or earlier
-    class << self
-      unless method_defined?(:desc)
-        def desc(description)
-        end
-      end
-    end
+    Fluent::Plugin.register_input('sql', self)
 
     desc 'RDBMS host'
     config_param :host, :string
     desc 'RDBMS port'
-    config_param :port, :integer, :default => nil
+    config_param :port, :integer, default: nil
     desc 'RDBMS driver name.'
     config_param :adapter, :string
     desc 'RDBMS database name'
     config_param :database, :string
     desc 'RDBMS login user name'
-    config_param :username, :string, :default => nil
+    config_param :username, :string, default: nil
     desc 'RDBMS login password'
-    config_param :password, :string, :default => nil, :secret => true
+    config_param :password, :string, default: nil, secret: true
     desc 'RDBMS socket path'
-    config_param :socket, :string, :default => nil
+    config_param :socket, :string, default: nil
+    desc 'PostgreSQL schema search path'
+    config_param :schema_search_path, :string, default: nil
 
     desc 'path to a file to store last rows'
-    config_param :state_file, :string, :default => nil
+    config_param :state_file, :string, default: nil
     desc 'prefix of tags of events. actual tag will be this_tag_prefix.tables_tag (optional)'
-    config_param :tag_prefix, :string, :default => nil
+    config_param :tag_prefix, :string, default: nil
     desc 'interval to run SQLs (optional)'
-    config_param :select_interval, :time, :default => 60
+    config_param :select_interval, :time, default: 60
     desc 'limit of number of rows for each SQL(optional)'
-    config_param :select_limit, :time, :default => 500
-
-    unless method_defined?(:log)
-      define_method(:log) { $log }
-    end
+    config_param :select_limit, :time, default: 500
 
     class TableElement
-      include Configurable
+      include Fluent::Configurable
 
       config_param :table, :string
-      config_param :tag, :string, :default => nil
-      config_param :update_column, :string, :default => nil
-      config_param :time_column, :string, :default => nil
-      config_param :primary_key, :string, :default => nil
+      config_param :tag, :string, default: nil
+      config_param :update_column, :string, default: nil
+      config_param :time_column, :string, default: nil
+      config_param :primary_key, :string, default: nil
+
+      attr_reader :log
 
       def configure(conf)
         super
       end
 
-      def init(tag_prefix, base_model, router)
+      def init(tag_prefix, base_model, router, log)
         @router = router
         @tag = "#{tag_prefix}.#{@tag}" if tag_prefix
+        @log = log
 
         # creates a model for this table
         table_name = @table
@@ -118,6 +111,17 @@ module Fluent
         end
       end
 
+      # Make sure we always have a Fluent::EventTime object regardless of what comes in
+      def normalized_time(tv, now)
+        return Fluent::EventTime.from_time(tv) if tv.is_a?(Time)
+        begin
+          Fluent::EventTime.parse(tv.to_s)
+        rescue
+          log.warn "Message contains invalid timestamp, using current time instead (#{now.inspect})"
+          now
+        end
+      end
+
       # emits next records and returns the last record of emitted records
       def emit_next_records(last_record, limit)
         relation = @model
@@ -127,21 +131,19 @@ module Fluent
         relation = relation.order("#{@update_column} ASC")
         relation = relation.limit(limit) if limit > 0
 
-        now = Engine.now
+        now = Fluent::Engine.now
 
-        me = MultiEventStream.new
+        me = Fluent::MultiEventStream.new
         relation.each do |obj|
           record = obj.serializable_hash rescue nil
           if record
-            if @time_column && tv = obj.read_attribute(@time_column)
-              if tv.is_a?(Time)
-                time = tv.to_i
+            time =
+              if @time_column && (tv = obj.read_attribute(@time_column))
+                normalized_time(tv, now)
               else
-                time = Time.parse(tv.to_s).to_i rescue now
+                now
               end
-            else
-              time = now
-            end
+
             me.add(time, record)
             last_record = record
           end
@@ -181,13 +183,14 @@ module Fluent
       @state_store = @state_file.nil? ? MemoryStateStore.new : StateStore.new(@state_file)
 
       config = {
-        :adapter => @adapter,
-        :host => @host,
-        :port => @port,
-        :database => @database,
-        :username => @username,
-        :password => @password,
-        :socket => @socket,
+        adapter: @adapter,
+        host: @host,
+        port: @port,
+        database: @database,
+        username: @username,
+        password: @password,
+        socket: @socket,
+        schema_search_path: @schema_search_path,
       }
 
       # creates subclass of ActiveRecord::Base so that it can have different
@@ -226,11 +229,11 @@ module Fluent
       # ignore tables if TableElement#init failed
       @tables.reject! do |te|
         begin
-          te.init(@tag_prefix, @base_model, router)
+          te.init(@tag_prefix, @base_model, router, log)
           log.info "Selecting '#{te.table}' table"
           false
         rescue => e
-          log.warn "Can't handle '#{te.table}' table. Ignoring.", :error => e.message, :error_class => e.class
+          log.warn "Can't handle '#{te.table}' table. Ignoring.", error: e
           log.warn_backtrace e.backtrace
           true
         end
@@ -264,7 +267,7 @@ module Fluent
             @state_store.last_records[t.table] = t.emit_next_records(last_record, @select_limit)
             @state_store.update!
           rescue => e
-            log.error "unexpected error", :error => e.message, :error_class => e.class
+            log.error "unexpected error", error: e
             log.error_backtrace e.backtrace
           end
         end

data/lib/fluent/plugin/out_sql.rb

@@ -1,59 +1,59 @@
-require "fluent/output"
+require "fluent/plugin/output"
 
-module Fluent
-  class SQLOutput < BufferedOutput
-    Plugin.register_output('sql', self)
+require 'active_record'
+require 'activerecord-import'
 
-    include SetTimeKeyMixin
-    include SetTagKeyMixin
+module Fluent::Plugin
+  class SQLOutput < Output
+    Fluent::Plugin.register_output('sql', self)
 
-    # For fluentd v0.12.16 or earlier
-    class << self
-      unless method_defined?(:desc)
-        def desc(description)
-        end
-      end
-    end
+    helpers :inject, :compat_parameters, :event_emitter
 
     desc 'RDBMS host'
     config_param :host, :string
     desc 'RDBMS port'
-    config_param :port, :integer, :default => nil
+    config_param :port, :integer, default: nil
     desc 'RDBMS driver name.'
     config_param :adapter, :string
     desc 'RDBMS login user name'
-    config_param :username, :string, :default => nil
+    config_param :username, :string, default: nil
     desc 'RDBMS login password'
-    config_param :password, :string, :default => nil, :secret => true
+    config_param :password, :string, default: nil, secret: true
     desc 'RDBMS database name'
     config_param :database, :string
     desc 'RDBMS socket path'
-    config_param :socket, :string, :default => nil
+    config_param :socket, :string, default: nil
+    desc 'PostgreSQL schema search path'
+    config_param :schema_search_path, :string, default: nil
     desc 'remove the given prefix from the events'
-    config_param :remove_tag_prefix, :string, :default => nil
+    config_param :remove_tag_prefix, :string, default: nil
     desc 'enable fallback'
-    config_param :enable_fallback, :bool, :default => true
+    config_param :enable_fallback, :bool, default: true
+    desc "size of ActiveRecord's connection pool"
+    config_param :pool, :integer, default: 5
+    desc "specifies the timeout to establish a new connection to the database before failing"
+    config_param :timeout, :integer, default: 5000
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ["tag"]
+    end
 
     attr_accessor :tables
 
-    unless method_defined?(:log)
-      define_method(:log) { $log }
-    end
-
     # TODO: Merge SQLInput's TableElement
     class TableElement
-      include Configurable
+      include Fluent::Configurable
 
       config_param :table, :string
       config_param :column_mapping, :string
-      config_param :num_retries, :integer, :default => 5
+      config_param :num_retries, :integer, default: 5
 
       attr_reader :model
       attr_reader :pattern
 
       def initialize(pattern, log, enable_fallback)
         super()
-        @pattern = MatchPattern.create(pattern)
+        @pattern = Fluent::MatchPattern.create(pattern)
         @log = log
         @enable_fallback = enable_fallback
       end
@@ -88,14 +88,15 @@ module Fluent
         # @model.column_names
       end
 
-      def import(chunk)
+      def import(chunk, output)
+        tag = chunk.metadata.tag
         records = []
-        chunk.msgpack_each { |tag, time, data|
+        chunk.msgpack_each { |time, data|
           begin
-            # format process should be moved to emit / format after supports error stream.
+            data = output.inject_values_to_record(tag, time, data)
             records << @model.new(@format_proc.call(data))
           rescue => e
-            args = {:error => e.message, :error_class => e.class, :table => @table, :record => Yajl.dump(data)}
+            args = {error: e, table: @table, record: Yajl.dump(data)}
             @log.warn "Failed to create the model. Ignore a record:", args
           end
         }
@@ -104,10 +105,10 @@ module Fluent
         rescue ActiveRecord::StatementInvalid, ActiveRecord::Import::MissingColumnError => e
           if @enable_fallback
             # ignore other exceptions to use Fluentd retry mechanizm
-            @log.warn "Got deterministic error. Fallback to one-by-one import", :error => e.message, :error_class => e.class
+            @log.warn "Got deterministic error. Fallback to one-by-one import", error: e
             one_by_one_import(records)
           else
-            $log.warn "Got deterministic error. Fallback is disabled", :error => e.message, :error_class => e.class
+            @log.warn "Got deterministic error. Fallback is disabled", error: e
             raise e
           end
         end
@@ -119,15 +120,15 @@ module Fluent
           begin
             @model.import([record])
           rescue ActiveRecord::StatementInvalid, ActiveRecord::Import::MissingColumnError => e
-            @log.error "Got deterministic error again. Dump a record", :error => e.message, :error_class => e.class, :record => record
+            @log.error "Got deterministic error again. Dump a record", error: e, record: record
           rescue => e
             retries += 1
             if retries > @num_retries
-              @log.error "Can't recover undeterministic error. Dump a record", :error => e.message, :error_class => e.class, :record => record
+              @log.error "Can't recover undeterministic error. Dump a record", error: e, record: record
               next
             end
 
-            @log.warn "Failed to import a record: retry number = #{retries}", :error  => e.message, :error_class => e.class
+            @log.warn "Failed to import a record: retry number = #{retries}", error: e
             sleep 0.5
             retry
           end
@@ -149,11 +150,11 @@ module Fluent
 
     def initialize
       super
-      require 'active_record'
-      require 'activerecord-import'
     end
 
     def configure(conf)
+      compat_parameters_convert(conf, :inject, :buffer)
+
       super
 
       if remove_tag_prefix = conf['remove_tag_prefix']
@@ -174,10 +175,13 @@ module Fluent
           @tables << te
         end
       }
-      @only_default = @tables.empty?
+
+      if @pool < @buffer_config.flush_thread_count
+        log.warn "connection pool size is smaller than buffer's flush_thread_count. Recommend to increase pool value", :pool => @pool, :flush_thread_count => @buffer_config.flush_thread_count
+      end
 
       if @default_table.nil?
-        raise ConfigError, "There is no default table. <table> is required in sql output"
+        raise Fluent::ConfigError, "There is no default table. <table> is required in sql output"
       end
     end
 
@@ -185,13 +189,16 @@ module Fluent
       super
 
       config = {
-        :adapter => @adapter,
-        :host => @host,
-        :port => @port,
-        :database => @database,
-        :username => @username,
-        :password => @password,
-        :socket => @socket,
+        adapter: @adapter,
+        host: @host,
+        port: @port,
+        database: @database,
+        username: @username,
+        password: @password,
+        socket: @socket,
+        schema_search_path: @schema_search_path,
+        pool: @pool,
+        timeout: @timeout,
       }
 
       @base_model = Class.new(ActiveRecord::Base) do
@@ -212,27 +219,20 @@ module Fluent
       super
     end
 
-    def emit(tag, es, chain)
-      if @only_default
-        super(tag, es, chain)
-      else
-        super(tag, es, chain, format_tag(tag))
-      end
-    end
-
-    def format(tag, time, record)
-      [tag, time, record].to_msgpack
+    def formatted_to_msgpack_binary
+      true
     end
 
     def write(chunk)
       ActiveRecord::Base.connection_pool.with_connection do
 
         @tables.each { |table|
-          if table.pattern.match(chunk.key)
-            return table.import(chunk)
+          tag = format_tag(chunk.metadata.tag)
+          if table.pattern.match(tag)
+            return table.import(chunk, self)
           end
         }
-        @default_table.import(chunk)
+        @default_table.import(chunk, self)
       end
     end
 
@@ -244,14 +244,14 @@ module Fluent
         log.info "Selecting '#{te.table}' table"
         false
       rescue => e
-        log.warn "Can't handle '#{te.table}' table. Ignoring.", :error => e.message, :error_class => e.class
+        log.warn "Can't handle '#{te.table}' table. Ignoring.", error: e
         log.warn_backtrace e.backtrace
         true
       end
     end
 
     def format_tag(tag)
-      if @remove_tag_prefix
+      if tag && @remove_tag_prefix
         tag.gsub(@remove_tag_prefix, '')
       else
         tag

data/test/fixtures/schema.rb

@@ -20,5 +20,12 @@ ActiveRecord::Schema.define(version: 20160225030107) do
     t.datetime "created_at",  null: false
     t.datetime "updated_at",  null: false
   end
+
+  create_table "messages_custom_time", force: :cascade do |t|
+    t.string "message"
+    t.datetime "created_at",  null: false
+    t.datetime "updated_at",  null: false
+    t.string "custom_time"
+  end
 end
 

data/test/plugin/test_in_sql.rb

@@ -1,4 +1,5 @@
 require "helper"
+require "fluent/test/driver/input"
 
 class SqlInputTest < Test::Unit::TestCase
   def setup
@@ -17,6 +18,8 @@ class SqlInputTest < Test::Unit::TestCase
     username fluentd
     password fluentd
 
+    schema_search_path public
+
     tag_prefix db
 
     <table>
@@ -28,7 +31,7 @@ class SqlInputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::InputTestDriver.new(Fluent::SQLInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::SQLInput).configure(conf)
   end
 
   def test_configure
@@ -40,6 +43,7 @@ class SqlInputTest < Test::Unit::TestCase
       database: "fluentd_test",
       username: "fluentd",
       password: "fluentd",
+      schema_search_path: "public",
       tag_prefix: "db"
     }
     actual = {
@@ -49,6 +53,7 @@ class SqlInputTest < Test::Unit::TestCase
       database: d.instance.database,
       username: d.instance.username,
       password: d.instance.password,
+      schema_search_path: d.instance.schema_search_path,
       tag_prefix: d.instance.tag_prefix
     }
     assert_equal(expected, actual)
@@ -65,18 +70,21 @@ class SqlInputTest < Test::Unit::TestCase
     Message.create!(message: "message 2")
     Message.create!(message: "message 3")
 
+    d.end_if do
+      d.record_count >= 3
+    end
     d.run
 
-    assert_equal("db.logs", d.emits[0][0])
+    assert_equal("db.logs", d.events[0][0])
     expected = [
-      [d.emits[0][1], "message 1"],
-      [d.emits[1][1], "message 2"],
-      [d.emits[2][1], "message 3"],
+      [d.events[0][1], "message 1"],
+      [d.events[1][1], "message 2"],
+      [d.events[2][1], "message 3"],
     ]
     actual = [
-      [Time.parse(d.emits[0][2]["updated_at"]).to_i, d.emits[0][2]["message"]],
-      [Time.parse(d.emits[1][2]["updated_at"]).to_i, d.emits[1][2]["message"]],
-      [Time.parse(d.emits[2][2]["updated_at"]).to_i, d.emits[2][2]["message"]],
+      [Fluent::EventTime.parse(d.events[0][2]["updated_at"]), d.events[0][2]["message"]],
+      [Fluent::EventTime.parse(d.events[1][2]["updated_at"]), d.events[1][2]["message"]],
+      [Fluent::EventTime.parse(d.events[2][2]["updated_at"]), d.events[2][2]["message"]],
     ]
     assert_equal(expected, actual)
   end

data/test/plugin/test_in_sql_with_custom_time.rb

@@ -0,0 +1,114 @@
+require "helper"
+require "fluent/test/driver/input"
+
+class SqlInputCustomTimeTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def teardown
+  end
+
+  CONFIG = %[
+    adapter postgresql
+    host localhost
+    port 5432
+    database fluentd_test
+
+    username fluentd
+    password fluentd
+
+    schema_search_path public
+
+    tag_prefix db
+
+    <table>
+      table messages_custom_time
+      tag logs
+      update_column updated_at
+      time_column custom_time
+    </table>
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::SQLInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    expected = {
+      host: "localhost",
+      port: 5432,
+      adapter: "postgresql",
+      database: "fluentd_test",
+      username: "fluentd",
+      password: "fluentd",
+      schema_search_path: "public",
+      tag_prefix: "db"
+    }
+    actual = {
+      host: d.instance.host,
+      port: d.instance.port,
+      adapter: d.instance.adapter,
+      database: d.instance.database,
+      username: d.instance.username,
+      password: d.instance.password,
+      schema_search_path: d.instance.schema_search_path,
+      tag_prefix: d.instance.tag_prefix
+    }
+    assert_equal(expected, actual)
+    tables = d.instance.instance_variable_get(:@tables)
+    assert_equal(1, tables.size)
+    messages_custom_time = tables.first
+    assert_equal("messages_custom_time", messages_custom_time.table)
+    assert_equal("logs", messages_custom_time.tag)
+  end
+
+  def test_message
+    d = create_driver(CONFIG + "select_interval 1")
+
+    start_time = Fluent::EventTime.now
+
+    # Create one message with a valid timestamp containing milliseconds and a time zone
+    Message.create!(message: "message 1", custom_time: '2020-08-27 15:00:16.100758000 -0400')
+
+    # Create one message without a timestamp so that we can test auto-creation
+    Message.create!(message: "message 2 (no timestamp)", custom_time: nil)
+
+    # Create one message with an unparseable timestamp so that we can check that a valid
+    # one is auto-generated.
+    Message.create!(message: "message 3 (bad timestamp)", custom_time: 'foo')
+
+    d.end_if do
+      d.record_count >= 3
+    end
+    d.run(timeout: 5)
+
+    assert_equal("db.logs", d.events[0][0])
+    expected = [
+      [d.events[0][1], "message 1"],
+      [d.events[1][1], "message 2 (no timestamp)"],
+      [d.events[2][1], "message 3 (bad timestamp)"],
+    ]
+
+    actual = [
+      [Fluent::EventTime.parse(d.events[0][2]["custom_time"]), d.events[0][2]["message"]],
+      d.events[1][2]["message"],
+      d.events[2][2]["message"],
+    ]
+
+    assert_equal(expected[0], actual[0])
+
+    # Messages 2 and 3 should have the same messages but (usually) a slightly later
+    # timestamps because they are generated by the input plugin instead of the test
+    # code
+    [1,2].each do |i|
+      assert_equal(expected[i][1], actual[i])
+      assert_operator(expected[i][0], :>=, start_time)
+    end
+  end
+
+  class Message < ActiveRecord::Base
+    self.table_name = "messages_custom_time"
+  end
+end

data/test/plugin/test_out_sql.rb

@@ -1,4 +1,5 @@
 require "helper"
+require "fluent/test/driver/output"
 
 class SqlOutputTest < Test::Unit::TestCase
   def setup
@@ -17,6 +18,8 @@ class SqlOutputTest < Test::Unit::TestCase
     username fluentd
     password fluentd
 
+    schema_search_path public
+
     remove_tag_prefix db
 
     <table>
@@ -26,7 +29,7 @@ class SqlOutputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::SQLOutput).configure(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::SQLOutput).configure(conf)
   end
 
   def test_configure
@@ -38,8 +41,10 @@ class SqlOutputTest < Test::Unit::TestCase
       database: "fluentd_test",
       username: "fluentd",
       password: "fluentd",
+      schema_search_path: 'public',
       remove_tag_suffix: /^db/,
-      enable_fallback: true
+      enable_fallback: true,
+      pool: 5
     }
     actual = {
       host: d.instance.host,
@@ -48,8 +53,10 @@ class SqlOutputTest < Test::Unit::TestCase
       database: d.instance.database,
       username: d.instance.username,
       password: d.instance.password,
+      schema_search_path: d.instance.schema_search_path,
       remove_tag_suffix: d.instance.remove_tag_prefix,
-      enable_fallback: d.instance.enable_fallback
+      enable_fallback: d.instance.enable_fallback,
+      pool: d.instance.pool
     }
     assert_equal(expected, actual)
     assert_empty(d.instance.tables)
@@ -61,10 +68,10 @@ class SqlOutputTest < Test::Unit::TestCase
     d = create_driver
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
-    d.emit({"message" => "message1"}, time)
-    d.emit({"message" => "message2"}, time)
-
-    d.run
+    d.run(default_tag: 'test') do
+      d.feed(time, {"message" => "message1"})
+      d.feed(time, {"message" => "message2"})
+    end
 
     default_table = d.instance.instance_variable_get(:@default_table)
     model = default_table.instance_variable_get(:@model)
@@ -78,10 +85,10 @@ class SqlOutputTest < Test::Unit::TestCase
       d = create_driver
       time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
-      d.emit({"message" => "message1"}, time)
-      d.emit({"message" => "message2"}, time)
+      d.run(default_tag: 'test') do
+        d.feed(time, {"message" => "message1"})
+        d.feed(time, {"message" => "message2"})
 
-      d.run do
         default_table = d.instance.instance_variable_get(:@default_table)
         model = default_table.instance_variable_get(:@model)
         mock(model).import(anything).at_least(1) do
@@ -95,10 +102,10 @@ class SqlOutputTest < Test::Unit::TestCase
       d = create_driver
       time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
-      d.emit({"message" => "message1"}, time)
-      d.emit({"message" => "message2"}, time)
+      d.run(default_tag: 'test') do
+        d.feed(time, {"message" => "message1"})
+        d.feed(time, {"message" => "message2"})
 
-      d.run do
         default_table = d.instance.instance_variable_get(:@default_table)
         model = default_table.instance_variable_get(:@model)
         mock(model).import([anything, anything]).once do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sql
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 2.1.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-27 00:00:00.000000000 Z
+date: 2020-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -36,14 +36,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: '5.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: '5.1'
 - !ruby/object:Gem::Dependency
   name: activerecord-import
   requirement: !ruby/object:Gem::Requirement
@@ -76,14 +76,14 @@ dependencies:
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
         version: 3.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
         version: 3.1.0
 - !ruby/object:Gem::Dependency
@@ -118,16 +118,16 @@ dependencies:
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
 description: SQL input/output plugin for Fluentd event collector
 email: frsyuki@gmail.com
 executables: []
@@ -135,6 +135,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".travis.yml"
+- ChangeLog
 - Gemfile
 - Gemfile.v0.12
 - README.md
@@ -146,8 +147,9 @@ files:
 - test/fixtures/schema.rb
 - test/helper.rb
 - test/plugin/test_in_sql.rb
+- test/plugin/test_in_sql_with_custom_time.rb
 - test/plugin/test_out_sql.rb
-homepage: https://github.com/frsyuki/fluent-plugin-sql
+homepage: https://github.com/fluent/fluent-plugin-sql
 licenses:
 - Apache-2.0
 metadata: {}
@@ -166,8 +168,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: SQL input/output plugin for Fluentd event collector
@@ -175,4 +176,5 @@ test_files:
 - test/fixtures/schema.rb
 - test/helper.rb
 - test/plugin/test_in_sql.rb
+- test/plugin/test_in_sql_with_custom_time.rb
 - test/plugin/test_out_sql.rb