fluent-plugin-sql 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15313a249bf6706d547ed4af5baa6de995f256204b17a57d9569c0efbab59bd9
-  data.tar.gz: 1909f37a4b7e5f3cfa281212a6e66c462e9cbc82f79930713f79e0cb9cc1761e
+  metadata.gz: '00987a628d7afc94546e68d5374331e82c65eca49aa8e3aad33b8f5ffb5c8f59'
+  data.tar.gz: e809d69583ab25571878d9bc4adb64a26841e14347e9deb973e1a659262bbf47
 SHA512:
-  metadata.gz: 5f5664d4a54f8cd0a383baec2610e7e9d80e752873d2235a198734805fd0290b64b6588d26c9893cb9d98f99f0204f1f56bbe11fed825269e5b1dc9b6d5d0699
-  data.tar.gz: 67a5e652889d5e36dcf8e4b5b8419ef62d46b706873779822fab0c03a4bc25ad466a22817ef89e59b2056649c90a16e613cd278785dc469f6ef5c0b2d34614cd
+  metadata.gz: b5cf74d0668a3e7b26220d2b56179c3668a99e2a51e8d99b7468d797bdc3c7e0fe81ca813ede95713fc3a2dc6484929505953c0be7239129343f138798ad7fe4
+  data.tar.gz: c45922cd85d83e3ed89629c55232cd0fe83106b4961f1d2dfaafd8564d8db900b4740f78e2f12b7596ee6ce025a43efaf4cb409ca9ed418434dcc3ed447a4456

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 2.1.0 - 2020/09/09
+
+　* in_sql: Use Fluent::EventTime instead of Integer to preserve millisecond precision
+
 Release 2.0.0 - 2020/04/22
 
   * out_sql: Remove v0.12 API code and use full v1 API. Change buffer format.

data/README.md

@@ -27,6 +27,11 @@ We recommend that mysql2 gem is higher than `0.3.12` and pg gem is higher than `
 
 If you use ruby 2.1, use pg gem 0.21.0 (< 1.0.0) because ActiveRecord 5.1.4 or earlier doesn't support Ruby 2.1.
 
+### Resolve tzinfo version conflict
+
+If you want to use fluent-plugin-sql with recent fluentd/td-agent, you need to downgrade tzinfo to v1.x manually.
+See also this comment: https://github.com/fluent/fluent-plugin-sql/issues/87#issuecomment-614552292
+
 ## Input: How It Works
 
 This plugin runs following SQL periodically:

data/VERSION

@@ -1 +1 @@
-2.0.0
+2.1.0

data/lib/fluent/plugin/in_sql.rb

@@ -60,13 +60,16 @@ module Fluent::Plugin
       config_param :time_column, :string, default: nil
       config_param :primary_key, :string, default: nil
 
+      attr_reader :log
+
       def configure(conf)
         super
       end
 
-      def init(tag_prefix, base_model, router)
+      def init(tag_prefix, base_model, router, log)
         @router = router
         @tag = "#{tag_prefix}.#{@tag}" if tag_prefix
+        @log = log
 
         # creates a model for this table
         table_name = @table
@@ -108,6 +111,17 @@ module Fluent::Plugin
         end
       end
 
+      # Make sure we always have a Fluent::EventTime object regardless of what comes in
+      def normalized_time(tv, now)
+        return Fluent::EventTime.from_time(tv) if tv.is_a?(Time)
+        begin
+          Fluent::EventTime.parse(tv.to_s)
+        rescue
+          log.warn "Message contains invalid timestamp, using current time instead (#{now.inspect})"
+          now
+        end
+      end
+
       # emits next records and returns the last record of emitted records
       def emit_next_records(last_record, limit)
         relation = @model
@@ -123,15 +137,13 @@ module Fluent::Plugin
         relation.each do |obj|
           record = obj.serializable_hash rescue nil
           if record
-            if @time_column && tv = obj.read_attribute(@time_column)
-              if tv.is_a?(Time)
-                time = tv.to_i
+            time =
+              if @time_column && (tv = obj.read_attribute(@time_column))
+                normalized_time(tv, now)
               else
-                time = Time.parse(tv.to_s).to_i rescue now
+                now
               end
-            else
-              time = now
-            end
+
             me.add(time, record)
             last_record = record
           end
@@ -217,7 +229,7 @@ module Fluent::Plugin
       # ignore tables if TableElement#init failed
       @tables.reject! do |te|
         begin
-          te.init(@tag_prefix, @base_model, router)
+          te.init(@tag_prefix, @base_model, router, log)
           log.info "Selecting '#{te.table}' table"
           false
         rescue => e

data/test/fixtures/schema.rb

@@ -20,5 +20,12 @@ ActiveRecord::Schema.define(version: 20160225030107) do
     t.datetime "created_at",  null: false
     t.datetime "updated_at",  null: false
   end
+
+  create_table "messages_custom_time", force: :cascade do |t|
+    t.string "message"
+    t.datetime "created_at",  null: false
+    t.datetime "updated_at",  null: false
+    t.string "custom_time"
+  end
 end
 

data/test/plugin/test_in_sql.rb

@@ -82,9 +82,9 @@ class SqlInputTest < Test::Unit::TestCase
       [d.events[2][1], "message 3"],
     ]
     actual = [
-      [Time.parse(d.events[0][2]["updated_at"]).to_i, d.events[0][2]["message"]],
-      [Time.parse(d.events[1][2]["updated_at"]).to_i, d.events[1][2]["message"]],
-      [Time.parse(d.events[2][2]["updated_at"]).to_i, d.events[2][2]["message"]],
+      [Fluent::EventTime.parse(d.events[0][2]["updated_at"]), d.events[0][2]["message"]],
+      [Fluent::EventTime.parse(d.events[1][2]["updated_at"]), d.events[1][2]["message"]],
+      [Fluent::EventTime.parse(d.events[2][2]["updated_at"]), d.events[2][2]["message"]],
     ]
     assert_equal(expected, actual)
   end

data/test/plugin/test_in_sql_with_custom_time.rb

@@ -0,0 +1,114 @@
+require "helper"
+require "fluent/test/driver/input"
+
+class SqlInputCustomTimeTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def teardown
+  end
+
+  CONFIG = %[
+    adapter postgresql
+    host localhost
+    port 5432
+    database fluentd_test
+
+    username fluentd
+    password fluentd
+
+    schema_search_path public
+
+    tag_prefix db
+
+    <table>
+      table messages_custom_time
+      tag logs
+      update_column updated_at
+      time_column custom_time
+    </table>
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::SQLInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    expected = {
+      host: "localhost",
+      port: 5432,
+      adapter: "postgresql",
+      database: "fluentd_test",
+      username: "fluentd",
+      password: "fluentd",
+      schema_search_path: "public",
+      tag_prefix: "db"
+    }
+    actual = {
+      host: d.instance.host,
+      port: d.instance.port,
+      adapter: d.instance.adapter,
+      database: d.instance.database,
+      username: d.instance.username,
+      password: d.instance.password,
+      schema_search_path: d.instance.schema_search_path,
+      tag_prefix: d.instance.tag_prefix
+    }
+    assert_equal(expected, actual)
+    tables = d.instance.instance_variable_get(:@tables)
+    assert_equal(1, tables.size)
+    messages_custom_time = tables.first
+    assert_equal("messages_custom_time", messages_custom_time.table)
+    assert_equal("logs", messages_custom_time.tag)
+  end
+
+  def test_message
+    d = create_driver(CONFIG + "select_interval 1")
+
+    start_time = Fluent::EventTime.now
+
+    # Create one message with a valid timestamp containing milliseconds and a time zone
+    Message.create!(message: "message 1", custom_time: '2020-08-27 15:00:16.100758000 -0400')
+
+    # Create one message without a timestamp so that we can test auto-creation
+    Message.create!(message: "message 2 (no timestamp)", custom_time: nil)
+
+    # Create one message with an unparseable timestamp so that we can check that a valid
+    # one is auto-generated.
+    Message.create!(message: "message 3 (bad timestamp)", custom_time: 'foo')
+
+    d.end_if do
+      d.record_count >= 3
+    end
+    d.run(timeout: 5)
+
+    assert_equal("db.logs", d.events[0][0])
+    expected = [
+      [d.events[0][1], "message 1"],
+      [d.events[1][1], "message 2 (no timestamp)"],
+      [d.events[2][1], "message 3 (bad timestamp)"],
+    ]
+
+    actual = [
+      [Fluent::EventTime.parse(d.events[0][2]["custom_time"]), d.events[0][2]["message"]],
+      d.events[1][2]["message"],
+      d.events[2][2]["message"],
+    ]
+
+    assert_equal(expected[0], actual[0])
+
+    # Messages 2 and 3 should have the same messages but (usually) a slightly later
+    # timestamps because they are generated by the input plugin instead of the test
+    # code
+    [1,2].each do |i|
+      assert_equal(expected[i][1], actual[i])
+      assert_operator(expected[i][0], :>=, start_time)
+    end
+  end
+
+  class Message < ActiveRecord::Base
+    self.table_name = "messages_custom_time"
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sql
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-22 00:00:00.000000000 Z
+date: 2020-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -147,6 +147,7 @@ files:
 - test/fixtures/schema.rb
 - test/helper.rb
 - test/plugin/test_in_sql.rb
+- test/plugin/test_in_sql_with_custom_time.rb
 - test/plugin/test_out_sql.rb
 homepage: https://github.com/fluent/fluent-plugin-sql
 licenses:
@@ -167,7 +168,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: SQL input/output plugin for Fluentd event collector
@@ -175,4 +176,5 @@ test_files:
 - test/fixtures/schema.rb
 - test/helper.rb
 - test/plugin/test_in_sql.rb
+- test/plugin/test_in_sql_with_custom_time.rb
 - test/plugin/test_out_sql.rb