fluent-plugin-sql 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d602eab48f2e5a82afd2805ea57198d5517bb1b7
-  data.tar.gz: fec9b91b4f26b49f9f8262adbf86e0a29b6b9c6a
+  metadata.gz: 3e23b9247528a135033b20b8948e2c7c8553e8a7
+  data.tar.gz: 3f6b03e77450c220f17851b50c1f799e80f1e4e8
 SHA512:
-  metadata.gz: 4e3909a2229451959f1ce21d69febb6ca102dae935da5944feb9895510f655fa0349b4f9b737bfdeba02533de55f7ac4dffab32f7cfe1e8beb80bb40864aa600
-  data.tar.gz: 82ec1c90fc5b4a0436a20235f2839d892a94eef18e3154823526b322a1f0547f0fe0b3019538b9e07a9b473e0b0581af65471452e3ec905524c21c94c5cdd4fd
+  metadata.gz: e6259bf6a96579679fa63f3cd7af978a8f5c23134a41438ba81f0c7d0744b3fd883f761c326156907e85d4b8f1d7d122f8841fbdd8905999f397f0a12a9f8ab2
+  data.tar.gz: 2eefbca9a6c289f8286bfe98472c43abe6cb5343c056f59f1490e67af65a2709157c1f7007e53fba102ef17f584352e2c139983768ec2a75b4bb9b11f1362b3f

data/.travis.yml

@@ -1,4 +1,20 @@
 sudo: false
+language: ruby
+
+addons:
+  postgresql: "9.4"
+
+rvm:
+  - 2.0.0
+  - 2.1
+  - 2.2
+  - 2.3.0
 
 before_install:
   - gem install bundler
+
+before_script:
+  - psql -U postgres -c "CREATE ROLE fluentd WITH LOGIN ENCRYPTED PASSWORD 'fluentd';"
+  - psql -U postgres -c "CREATE DATABASE fluentd_test OWNER fluentd;"
+
+script: bundle exec rake test

data/Rakefile

@@ -2,13 +2,13 @@
 require 'bundler'
 Bundler::GemHelper.install_tasks
 
-#require 'rake/testtask'
-#
-#Rake::TestTask.new(:test) do |test|
-#  test.libs << 'lib' << 'test'
-#  test.test_files = FileList['test/*.rb']
-#  test.verbose = true
-#end
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/*/*.rb']
+  test.verbose = true
+end
 
 task :default => [:build]
 

data/VERSION

@@ -1 +1 @@
-0.5.0
+0.5.1

data/fluent-plugin-sql.gemspec

@@ -21,4 +21,8 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'activerecord', "~> 4.2"
   gem.add_dependency 'activerecord-import', "~> 0.7"
   gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "test-unit", "~> 3.1.0"
+  gem.add_development_dependency "test-unit-rr"
+  gem.add_development_dependency "test-unit-notify"
+  gem.add_development_dependency "pg"
 end

data/lib/fluent/plugin/in_sql.rb

@@ -237,6 +237,8 @@ module Fluent
 
     def shutdown
       @stop_flag = true
+      $log.debug "Waiting for thread to finish"
+      @thread.join
     end
 
     def thread_main

data/lib/fluent/plugin/out_sql.rb

@@ -96,7 +96,7 @@ module Fluent
         }
         begin
           @model.import(records)
-        rescue ActiveRecord::StatementInvalid, ActiveRecord::ThrowResult, ActiveRecord::Import::MissingColumnError => e
+        rescue ActiveRecord::StatementInvalid, ActiveRecord::Import::MissingColumnError => e
           # ignore other exceptions to use Fluentd retry mechanizm
           @log.warn "Got deterministic error. Fallback to one-by-one import", :error => e.message, :error_class => e.class
           one_by_one_import(records)
@@ -108,7 +108,7 @@ module Fluent
           retries = 0
           begin
             @model.import([record])
-          rescue ActiveRecord::StatementInvalid, ActiveRecord::ThrowResult, ActiveRecord::Import::MissingColumnError => e
+          rescue ActiveRecord::StatementInvalid, ActiveRecord::Import::MissingColumnError => e
             @log.error "Got deterministic error again. Dump a record", :error => e.message, :error_class => e.class, :record => record
           rescue => e
             retries += 1
@@ -189,7 +189,7 @@ module Fluent
       end
 
       SQLOutput.const_set("BaseModel_#{rand(1 << 31)}", @base_model)
-      @base_model.establish_connection(config)
+      ActiveRecord::Base.establish_connection(config)
 
       # ignore tables if TableElement#init failed
       @tables.reject! do |te|
@@ -215,15 +215,15 @@ module Fluent
     end
 
     def write(chunk)
-      conn = @base_model.connection
-      conn.active? || conn.reconnect!
+      ActiveRecord::Base.connection_pool.with_connection do
 
-      @tables.each { |table|
-        if table.pattern.match(chunk.key)
-          return table.import(chunk)
-        end
-      }
-      @default_table.import(chunk)
+        @tables.each { |table|
+          if table.pattern.match(chunk.key)
+            return table.import(chunk)
+          end
+        }
+        @default_table.import(chunk)
+      end
     end
 
     private

data/test/fixtures/schema.rb

@@ -0,0 +1,24 @@
+require "active_record"
+ActiveRecord::Base.establish_connection(host: "localhost",
+                                        port: 5432,
+                                        username: "fluentd",
+                                        password: "fluentd",
+                                        adapter: "postgresql",
+                                        database: "fluentd_test")
+ActiveRecord::Schema.define(version: 20160225030107) do
+  create_table "logs", force: :cascade do |t|
+    t.string   "host"
+    t.string   "ident"
+    t.string   "pid"
+    t.text     "message"
+    t.datetime "created_at",  null: false
+    t.datetime "updated_at",  null: false
+  end
+
+  create_table "messages", force: :cascade do |t|
+    t.string "message"
+    t.datetime "created_at",  null: false
+    t.datetime "updated_at",  null: false
+  end
+end
+

data/test/helper.rb

@@ -0,0 +1,8 @@
+require "test/unit"
+require "test/unit/rr"
+require "test/unit/notify"
+require "fluent/test"
+require "fluent/plugin/out_sql"
+require "fluent/plugin/in_sql"
+
+load "fixtures/schema.rb"

data/test/plugin/test_in_sql.rb

@@ -0,0 +1,86 @@
+require "helper"
+
+class SqlInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def teardown
+  end
+
+  CONFIG = %[
+    adapter postgresql
+    host localhost
+    port 5432
+    database fluentd_test
+
+    username fluentd
+    password fluentd
+
+    tag_prefix db
+
+    <table>
+      table messages
+      tag logs
+      update_column updated_at
+      time_column updated_at
+    </table>
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::SQLInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    expected = {
+      host: "localhost",
+      port: 5432,
+      adapter: "postgresql",
+      database: "fluentd_test",
+      username: "fluentd",
+      password: "fluentd",
+      tag_prefix: "db"
+    }
+    actual = {
+      host: d.instance.host,
+      port: d.instance.port,
+      adapter: d.instance.adapter,
+      database: d.instance.database,
+      username: d.instance.username,
+      password: d.instance.password,
+      tag_prefix: d.instance.tag_prefix
+    }
+    assert_equal(expected, actual)
+    tables = d.instance.instance_variable_get(:@tables)
+    assert_equal(1, tables.size)
+    messages = tables.first
+    assert_equal("messages", messages.table)
+    assert_equal("logs", messages.tag)
+  end
+
+  def test_message
+    d = create_driver(CONFIG + "select_interval 1")
+    Message.create!(message: "message 1")
+    Message.create!(message: "message 2")
+    Message.create!(message: "message 3")
+
+    d.run
+
+    assert_equal("db.logs", d.emits[0][0])
+    expected = [
+      [d.emits[0][1], "message 1"],
+      [d.emits[1][1], "message 2"],
+      [d.emits[2][1], "message 3"],
+    ]
+    actual = [
+      [Time.parse(d.emits[0][2]["updated_at"]).to_i, d.emits[0][2]["message"]],
+      [Time.parse(d.emits[1][2]["updated_at"]).to_i, d.emits[1][2]["message"]],
+      [Time.parse(d.emits[2][2]["updated_at"]).to_i, d.emits[2][2]["message"]],
+    ]
+    assert_equal(expected, actual)
+  end
+
+  class Message < ActiveRecord::Base
+  end
+end

data/test/plugin/test_out_sql.rb

@@ -0,0 +1,112 @@
+require "helper"
+
+class SqlOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def teardown
+  end
+
+  CONFIG = %[
+    host localhost
+    port 5432
+    adapter postgresql
+
+    database fluentd_test
+    username fluentd
+    password fluentd
+
+    remove_tag_prefix db
+
+    <table>
+      table logs
+      column_mapping timestamp:created_at,host:host,ident:ident,pid:pid,message:message
+    </table>
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::SQLOutput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    expected = {
+      host: "localhost",
+      port: 5432,
+      adapter: "postgresql",
+      database: "fluentd_test",
+      username: "fluentd",
+      password: "fluentd",
+      remove_tag_suffix: /^db/
+    }
+    actual = {
+      host: d.instance.host,
+      port: d.instance.port,
+      adapter: d.instance.adapter,
+      database: d.instance.database,
+      username: d.instance.username,
+      password: d.instance.password,
+      remove_tag_suffix: d.instance.remove_tag_prefix
+    }
+    assert_equal(expected, actual)
+    assert_empty(d.instance.tables)
+    default_table = d.instance.instance_variable_get(:@default_table)
+    assert_equal("logs", default_table.table)
+  end
+
+  def test_emit
+    d = create_driver
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.emit({"message" => "message1"}, time)
+    d.emit({"message" => "message2"}, time)
+
+    d.run
+
+    default_table = d.instance.instance_variable_get(:@default_table)
+    model = default_table.instance_variable_get(:@model)
+    assert_equal(2, model.all.count)
+    messages = model.pluck(:message).sort
+    assert_equal(["message1", "message2"], messages)
+  end
+
+  class Fallback < self
+    def test_simple
+      d = create_driver
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+      d.emit({"message" => "message1"}, time)
+      d.emit({"message" => "message2"}, time)
+
+      d.run do
+        default_table = d.instance.instance_variable_get(:@default_table)
+        model = default_table.instance_variable_get(:@model)
+        mock(model).import(anything).at_least(1) do
+          raise ActiveRecord::Import::MissingColumnError.new("dummy_table", "dummy_column")
+        end
+        mock(default_table).one_by_one_import(anything)
+      end
+    end
+
+    def test_limit
+      d = create_driver
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+      d.emit({"message" => "message1"}, time)
+      d.emit({"message" => "message2"}, time)
+
+      d.run do
+        default_table = d.instance.instance_variable_get(:@default_table)
+        model = default_table.instance_variable_get(:@model)
+        mock(model).import([anything, anything]).once do
+          raise ActiveRecord::Import::MissingColumnError.new("dummy_table", "dummy_column")
+        end
+        mock(model).import([anything]).times(12) do
+          raise StandardError
+        end
+        assert_equal(5, default_table.instance_variable_get(:@num_retries))
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sql
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-18 00:00:00.000000000 Z
+date: 2016-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -72,6 +72,62 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit-notify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: SQL input/output plugin for Fluentd event collector
 email: frsyuki@gmail.com
 executables: []
@@ -86,6 +142,10 @@ files:
 - fluent-plugin-sql.gemspec
 - lib/fluent/plugin/in_sql.rb
 - lib/fluent/plugin/out_sql.rb
+- test/fixtures/schema.rb
+- test/helper.rb
+- test/plugin/test_in_sql.rb
+- test/plugin/test_out_sql.rb
 homepage: https://github.com/frsyuki/fluent-plugin-sql
 licenses:
 - Apache-2.0
@@ -106,8 +166,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: SQL input/output plugin for Fluentd event collector
-test_files: []
+test_files:
+- test/fixtures/schema.rb
+- test/helper.rb
+- test/plugin/test_in_sql.rb
+- test/plugin/test_out_sql.rb