fluent-plugin-sql 0.6.1 → 1.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8bdf7e835bed062213f63778607f5f4e3a6f9a02
-  data.tar.gz: d9b7d721ef9ad035193fbf0eb5a5460e88caa9aa
+  metadata.gz: 3e683b76d37276e4bbe276dc6148216c113e5267
+  data.tar.gz: 374e957bfaf1f9eea818a6d3bc9c9b370c58604f
 SHA512:
-  metadata.gz: ab807cd58fa29576d9164e4ab1e307a3b8cde72dd129e608acc78da01c15499e9ba9def596b044604bd2890c877fc3d66be79ca31d441e24f0d6f5cba8a741f6
-  data.tar.gz: 23a7698d628cd9dad85b9d1ab8f50ade3d925531cb230cbc9a93ace20e03ab157bd0ed09c1a3f97a9380931135acda9837402249e6b360e1881f694b157eff3e
+  metadata.gz: 1a71af14e2817621d22c85ac31a49d54d6b7f224b605ce2febaf09fbe369b435b33df779b23196e37ac6fb5d3c5a26d2039cb5aadf24dc90ec5d105ba4b283c6
+  data.tar.gz: 593be077cfbc429c9502ad7b56c404cac24afbae65268d095c723339bb77a4645a5c4297d03a6cbf73dd269cdda69824d3e2e5d4d4b498406e8871c5e0fb775e

data/.travis.yml

@@ -14,7 +14,6 @@ rvm:
 
 gemfile:
  - Gemfile
- - Gemfile.v0.12
 
 before_install:
   - gem install bundler

data/README.md

@@ -7,6 +7,15 @@ This SQL plugin has two parts:
 1. SQL **input** plugin reads records from RDBMSes periodically. An example use case would be getting "diffs" of a table (based on the "updated_at" field).
 2. SQL **output** plugin that writes records into RDBMes. An example use case would be aggregating server/app/sensor logs into RDBMS systems.
 
+## Requirements
+
+| fluent-plugin-sql | fluentd    | ruby   |
+|-------------------|------------|--------|
+| >= 1.0.0          | >= v0.14.4 | >= 2.1 |
+| <  1.0.0          | <  v0.14.0 | >= 1.9 |
+
+NOTE: fluent-plugin-sql v1.0.0 is now RC. We will release stable v1.0.0 soon.
+
 ## Installation
 
     $ fluent-gem install fluent-plugin-sql --no-document

data/VERSION

@@ -1 +1 @@
-0.6.1
+1.0.0.rc1

data/lib/fluent/plugin/in_sql.rb

@@ -16,59 +16,47 @@
 #    limitations under the License.
 #
 
-require "fluent/input"
+require "fluent/plugin/input"
 
-module Fluent
+module Fluent::Plugin
 
   require 'active_record'
 
   class SQLInput < Input
-    Plugin.register_input('sql', self)
-
-    # For fluentd v0.12.16 or earlier
-    class << self
-      unless method_defined?(:desc)
-        def desc(description)
-        end
-      end
-    end
+    Fluent::Plugin.register_input('sql', self)
 
     desc 'RDBMS host'
     config_param :host, :string
     desc 'RDBMS port'
-    config_param :port, :integer, :default => nil
+    config_param :port, :integer, default: nil
     desc 'RDBMS driver name.'
     config_param :adapter, :string
     desc 'RDBMS database name'
     config_param :database, :string
     desc 'RDBMS login user name'
-    config_param :username, :string, :default => nil
+    config_param :username, :string, default: nil
     desc 'RDBMS login password'
-    config_param :password, :string, :default => nil, :secret => true
+    config_param :password, :string, default: nil, secret: true
     desc 'RDBMS socket path'
-    config_param :socket, :string, :default => nil
+    config_param :socket, :string, default: nil
 
     desc 'path to a file to store last rows'
-    config_param :state_file, :string, :default => nil
+    config_param :state_file, :string, default: nil
     desc 'prefix of tags of events. actual tag will be this_tag_prefix.tables_tag (optional)'
-    config_param :tag_prefix, :string, :default => nil
+    config_param :tag_prefix, :string, default: nil
     desc 'interval to run SQLs (optional)'
-    config_param :select_interval, :time, :default => 60
+    config_param :select_interval, :time, default: 60
     desc 'limit of number of rows for each SQL(optional)'
-    config_param :select_limit, :time, :default => 500
-
-    unless method_defined?(:log)
-      define_method(:log) { $log }
-    end
+    config_param :select_limit, :time, default: 500
 
     class TableElement
-      include Configurable
+      include Fluent::Configurable
 
       config_param :table, :string
-      config_param :tag, :string, :default => nil
-      config_param :update_column, :string, :default => nil
-      config_param :time_column, :string, :default => nil
-      config_param :primary_key, :string, :default => nil
+      config_param :tag, :string, default: nil
+      config_param :update_column, :string, default: nil
+      config_param :time_column, :string, default: nil
+      config_param :primary_key, :string, default: nil
 
       def configure(conf)
         super
@@ -127,9 +115,9 @@ module Fluent
         relation = relation.order("#{@update_column} ASC")
         relation = relation.limit(limit) if limit > 0
 
-        now = Engine.now
+        now = Fluent::Engine.now
 
-        me = MultiEventStream.new
+        me = Fluent::MultiEventStream.new
         relation.each do |obj|
           record = obj.serializable_hash rescue nil
           if record
@@ -181,13 +169,13 @@ module Fluent
       @state_store = @state_file.nil? ? MemoryStateStore.new : StateStore.new(@state_file)
 
       config = {
-        :adapter => @adapter,
-        :host => @host,
-        :port => @port,
-        :database => @database,
-        :username => @username,
-        :password => @password,
-        :socket => @socket,
+        adapter: @adapter,
+        host: @host,
+        port: @port,
+        database: @database,
+        username: @username,
+        password: @password,
+        socket: @socket,
       }
 
       # creates subclass of ActiveRecord::Base so that it can have different
@@ -230,7 +218,7 @@ module Fluent
           log.info "Selecting '#{te.table}' table"
           false
         rescue => e
-          log.warn "Can't handle '#{te.table}' table. Ignoring.", :error => e.message, :error_class => e.class
+          log.warn "Can't handle '#{te.table}' table. Ignoring.", error: e
           log.warn_backtrace e.backtrace
           true
         end
@@ -264,7 +252,7 @@ module Fluent
             @state_store.last_records[t.table] = t.emit_next_records(last_record, @select_limit)
             @state_store.update!
           rescue => e
-            log.error "unexpected error", :error => e.message, :error_class => e.class
+            log.error "unexpected error", error: e
             log.error_backtrace e.backtrace
           end
         end

data/lib/fluent/plugin/out_sql.rb

@@ -1,59 +1,52 @@
-require "fluent/output"
+require "fluent/plugin/output"
 
-module Fluent
-  class SQLOutput < BufferedOutput
-    Plugin.register_output('sql', self)
+module Fluent::Plugin
+  class SQLOutput < Output
+    Fluent::Plugin.register_output('sql', self)
 
-    include SetTimeKeyMixin
-    include SetTagKeyMixin
+    DEFAULT_BUFFER_TYPE = "memory"
 
-    # For fluentd v0.12.16 or earlier
-    class << self
-      unless method_defined?(:desc)
-        def desc(description)
-        end
-      end
-    end
+    helpers :inject, :compat_parameters, :event_emitter
 
     desc 'RDBMS host'
     config_param :host, :string
     desc 'RDBMS port'
-    config_param :port, :integer, :default => nil
+    config_param :port, :integer, default: nil
     desc 'RDBMS driver name.'
     config_param :adapter, :string
     desc 'RDBMS login user name'
-    config_param :username, :string, :default => nil
+    config_param :username, :string, default: nil
     desc 'RDBMS login password'
-    config_param :password, :string, :default => nil, :secret => true
+    config_param :password, :string, default: nil, secret: true
     desc 'RDBMS database name'
     config_param :database, :string
     desc 'RDBMS socket path'
-    config_param :socket, :string, :default => nil
+    config_param :socket, :string, default: nil
     desc 'remove the given prefix from the events'
-    config_param :remove_tag_prefix, :string, :default => nil
+    config_param :remove_tag_prefix, :string, default: nil
     desc 'enable fallback'
-    config_param :enable_fallback, :bool, :default => true
-
-    attr_accessor :tables
+    config_param :enable_fallback, :bool, default: true
 
-    unless method_defined?(:log)
-      define_method(:log) { $log }
+    config_section :buffer do
+      config_set_default :@type, DEFAULT_BUFFER_TYPE
     end
 
+    attr_accessor :tables
+
     # TODO: Merge SQLInput's TableElement
     class TableElement
-      include Configurable
+      include Fluent::Configurable
 
       config_param :table, :string
       config_param :column_mapping, :string
-      config_param :num_retries, :integer, :default => 5
+      config_param :num_retries, :integer, default: 5
 
       attr_reader :model
       attr_reader :pattern
 
       def initialize(pattern, log, enable_fallback)
         super()
-        @pattern = MatchPattern.create(pattern)
+        @pattern = Fluent::MatchPattern.create(pattern)
         @log = log
         @enable_fallback = enable_fallback
       end
@@ -95,7 +88,7 @@ module Fluent
             # format process should be moved to emit / format after supports error stream.
             records << @model.new(@format_proc.call(data))
           rescue => e
-            args = {:error => e.message, :error_class => e.class, :table => @table, :record => Yajl.dump(data)}
+            args = {error: e, table: @table, record: Yajl.dump(data)}
             @log.warn "Failed to create the model. Ignore a record:", args
           end
         }
@@ -104,10 +97,10 @@ module Fluent
         rescue ActiveRecord::StatementInvalid, ActiveRecord::Import::MissingColumnError => e
           if @enable_fallback
             # ignore other exceptions to use Fluentd retry mechanizm
-            @log.warn "Got deterministic error. Fallback to one-by-one import", :error => e.message, :error_class => e.class
+            @log.warn "Got deterministic error. Fallback to one-by-one import", error: e
             one_by_one_import(records)
           else
-            $log.warn "Got deterministic error. Fallback is disabled", :error => e.message, :error_class => e.class
+            $log.warn "Got deterministic error. Fallback is disabled", error: e
             raise e
           end
         end
@@ -119,15 +112,15 @@ module Fluent
           begin
             @model.import([record])
           rescue ActiveRecord::StatementInvalid, ActiveRecord::Import::MissingColumnError => e
-            @log.error "Got deterministic error again. Dump a record", :error => e.message, :error_class => e.class, :record => record
+            @log.error "Got deterministic error again. Dump a record", error: e, record: record
           rescue => e
             retries += 1
             if retries > @num_retries
-              @log.error "Can't recover undeterministic error. Dump a record", :error => e.message, :error_class => e.class, :record => record
+              @log.error "Can't recover undeterministic error. Dump a record", error: e, record: record
               next
             end
 
-            @log.warn "Failed to import a record: retry number = #{retries}", :error  => e.message, :error_class => e.class
+            @log.warn "Failed to import a record: retry number = #{retries}", error: e
             sleep 0.5
             retry
           end
@@ -154,6 +147,8 @@ module Fluent
     end
 
     def configure(conf)
+      compat_parameters_convert(conf, :inject, :buffer)
+
       super
 
       if remove_tag_prefix = conf['remove_tag_prefix']
@@ -177,7 +172,7 @@ module Fluent
       @only_default = @tables.empty?
 
       if @default_table.nil?
-        raise ConfigError, "There is no default table. <table> is required in sql output"
+        raise Fluent::ConfigError, "There is no default table. <table> is required in sql output"
       end
     end
 
@@ -185,13 +180,13 @@ module Fluent
       super
 
       config = {
-        :adapter => @adapter,
-        :host => @host,
-        :port => @port,
-        :database => @database,
-        :username => @username,
-        :password => @password,
-        :socket => @socket,
+        adapter: @adapter,
+        host: @host,
+        port: @port,
+        database: @database,
+        username: @username,
+        password: @password,
+        socket: @socket,
       }
 
       @base_model = Class.new(ActiveRecord::Base) do
@@ -221,9 +216,14 @@ module Fluent
     end
 
     def format(tag, time, record)
+      record = inject_values_to_record(tag, time, record)
       [tag, time, record].to_msgpack
     end
 
+    def formatted_to_msgpack_binary
+      true
+    end
+
     def write(chunk)
       ActiveRecord::Base.connection_pool.with_connection do
 
@@ -244,7 +244,7 @@ module Fluent
         log.info "Selecting '#{te.table}' table"
         false
       rescue => e
-        log.warn "Can't handle '#{te.table}' table. Ignoring.", :error => e.message, :error_class => e.class
+        log.warn "Can't handle '#{te.table}' table. Ignoring.", error: e
         log.warn_backtrace e.backtrace
         true
       end

data/test/plugin/test_in_sql.rb

@@ -1,4 +1,5 @@
 require "helper"
+require "fluent/test/driver/input"
 
 class SqlInputTest < Test::Unit::TestCase
   def setup
@@ -28,7 +29,7 @@ class SqlInputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::InputTestDriver.new(Fluent::SQLInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::SQLInput).configure(conf)
   end
 
   def test_configure
@@ -65,18 +66,21 @@ class SqlInputTest < Test::Unit::TestCase
     Message.create!(message: "message 2")
     Message.create!(message: "message 3")
 
+    d.end_if do
+      d.record_count >= 3
+    end
     d.run
 
-    assert_equal("db.logs", d.emits[0][0])
+    assert_equal("db.logs", d.events[0][0])
     expected = [
-      [d.emits[0][1], "message 1"],
-      [d.emits[1][1], "message 2"],
-      [d.emits[2][1], "message 3"],
+      [d.events[0][1], "message 1"],
+      [d.events[1][1], "message 2"],
+      [d.events[2][1], "message 3"],
     ]
     actual = [
-      [Time.parse(d.emits[0][2]["updated_at"]).to_i, d.emits[0][2]["message"]],
-      [Time.parse(d.emits[1][2]["updated_at"]).to_i, d.emits[1][2]["message"]],
-      [Time.parse(d.emits[2][2]["updated_at"]).to_i, d.emits[2][2]["message"]],
+      [Time.parse(d.events[0][2]["updated_at"]).to_i, d.events[0][2]["message"]],
+      [Time.parse(d.events[1][2]["updated_at"]).to_i, d.events[1][2]["message"]],
+      [Time.parse(d.events[2][2]["updated_at"]).to_i, d.events[2][2]["message"]],
     ]
     assert_equal(expected, actual)
   end

data/test/plugin/test_out_sql.rb

@@ -1,4 +1,5 @@
 require "helper"
+require "fluent/test/driver/output"
 
 class SqlOutputTest < Test::Unit::TestCase
   def setup
@@ -26,7 +27,7 @@ class SqlOutputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::SQLOutput).configure(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::SQLOutput).configure(conf)
   end
 
   def test_configure
@@ -61,10 +62,10 @@ class SqlOutputTest < Test::Unit::TestCase
     d = create_driver
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
-    d.emit({"message" => "message1"}, time)
-    d.emit({"message" => "message2"}, time)
-
-    d.run
+    d.run(default_tag: 'test') do
+      d.feed(time, {"message" => "message1"})
+      d.feed(time, {"message" => "message2"})
+    end
 
     default_table = d.instance.instance_variable_get(:@default_table)
     model = default_table.instance_variable_get(:@model)
@@ -78,10 +79,10 @@ class SqlOutputTest < Test::Unit::TestCase
       d = create_driver
       time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
-      d.emit({"message" => "message1"}, time)
-      d.emit({"message" => "message2"}, time)
+      d.run(default_tag: 'test') do
+        d.feed(time, {"message" => "message1"})
+        d.feed(time, {"message" => "message2"})
 
-      d.run do
         default_table = d.instance.instance_variable_get(:@default_table)
         model = default_table.instance_variable_get(:@model)
         mock(model).import(anything).at_least(1) do
@@ -95,10 +96,10 @@ class SqlOutputTest < Test::Unit::TestCase
       d = create_driver
       time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
-      d.emit({"message" => "message1"}, time)
-      d.emit({"message" => "message2"}, time)
+      d.run(default_tag: 'test') do
+        d.feed(time, {"message" => "message1"})
+        d.feed(time, {"message" => "message2"})
 
-      d.run do
         default_table = d.instance.instance_variable_get(:@default_table)
         model = default_table.instance_variable_get(:@model)
         mock(model).import([anything, anything]).once do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sql
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 1.0.0.rc1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-27 00:00:00.000000000 Z
+date: 2017-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -162,12 +162,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: SQL input/output plugin for Fluentd event collector