fluent-plugin-sql 2.0.0 → 2.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 15313a249bf6706d547ed4af5baa6de995f256204b17a57d9569c0efbab59bd9
4
- data.tar.gz: 1909f37a4b7e5f3cfa281212a6e66c462e9cbc82f79930713f79e0cb9cc1761e
3
+ metadata.gz: '00987a628d7afc94546e68d5374331e82c65eca49aa8e3aad33b8f5ffb5c8f59'
4
+ data.tar.gz: e809d69583ab25571878d9bc4adb64a26841e14347e9deb973e1a659262bbf47
5
5
  SHA512:
6
- metadata.gz: 5f5664d4a54f8cd0a383baec2610e7e9d80e752873d2235a198734805fd0290b64b6588d26c9893cb9d98f99f0204f1f56bbe11fed825269e5b1dc9b6d5d0699
7
- data.tar.gz: 67a5e652889d5e36dcf8e4b5b8419ef62d46b706873779822fab0c03a4bc25ad466a22817ef89e59b2056649c90a16e613cd278785dc469f6ef5c0b2d34614cd
6
+ metadata.gz: b5cf74d0668a3e7b26220d2b56179c3668a99e2a51e8d99b7468d797bdc3c7e0fe81ca813ede95713fc3a2dc6484929505953c0be7239129343f138798ad7fe4
7
+ data.tar.gz: c45922cd85d83e3ed89629c55232cd0fe83106b4961f1d2dfaafd8564d8db900b4740f78e2f12b7596ee6ce025a43efaf4cb409ca9ed418434dcc3ed447a4456
data/ChangeLog CHANGED
@@ -1,3 +1,7 @@
1
+ Release 2.1.0 - 2020/09/09
2
+
3
+  * in_sql: Use Fluent::EventTime instead of Integer to preserve millisecond precision
4
+
1
5
  Release 2.0.0 - 2020/04/22
2
6
 
3
7
  * out_sql: Remove v0.12 API code and use full v1 API. Change buffer format.
data/README.md CHANGED
@@ -27,6 +27,11 @@ We recommend that mysql2 gem is higher than `0.3.12` and pg gem is higher than `
27
27
 
28
28
  If you use ruby 2.1, use pg gem 0.21.0 (< 1.0.0) because ActiveRecord 5.1.4 or earlier doesn't support Ruby 2.1.
29
29
 
30
+ ### Resolve tzinfo version conflict
31
+
32
+ If you want to use fluent-plugin-sql with recent fluentd/td-agent, you need to downgrade tzinfo to v1.x manually.
33
+ See also this comment: https://github.com/fluent/fluent-plugin-sql/issues/87#issuecomment-614552292
34
+
30
35
  ## Input: How It Works
31
36
 
32
37
  This plugin runs following SQL periodically:
data/VERSION CHANGED
@@ -1 +1 @@
1
- 2.0.0
1
+ 2.1.0
@@ -60,13 +60,16 @@ module Fluent::Plugin
60
60
  config_param :time_column, :string, default: nil
61
61
  config_param :primary_key, :string, default: nil
62
62
 
63
+ attr_reader :log
64
+
63
65
  def configure(conf)
64
66
  super
65
67
  end
66
68
 
67
- def init(tag_prefix, base_model, router)
69
+ def init(tag_prefix, base_model, router, log)
68
70
  @router = router
69
71
  @tag = "#{tag_prefix}.#{@tag}" if tag_prefix
72
+ @log = log
70
73
 
71
74
  # creates a model for this table
72
75
  table_name = @table
@@ -108,6 +111,17 @@ module Fluent::Plugin
108
111
  end
109
112
  end
110
113
 
114
+ # Make sure we always have a Fluent::EventTime object regardless of what comes in
115
+ def normalized_time(tv, now)
116
+ return Fluent::EventTime.from_time(tv) if tv.is_a?(Time)
117
+ begin
118
+ Fluent::EventTime.parse(tv.to_s)
119
+ rescue
120
+ log.warn "Message contains invalid timestamp, using current time instead (#{now.inspect})"
121
+ now
122
+ end
123
+ end
124
+
111
125
  # emits next records and returns the last record of emitted records
112
126
  def emit_next_records(last_record, limit)
113
127
  relation = @model
@@ -123,15 +137,13 @@ module Fluent::Plugin
123
137
  relation.each do |obj|
124
138
  record = obj.serializable_hash rescue nil
125
139
  if record
126
- if @time_column && tv = obj.read_attribute(@time_column)
127
- if tv.is_a?(Time)
128
- time = tv.to_i
140
+ time =
141
+ if @time_column && (tv = obj.read_attribute(@time_column))
142
+ normalized_time(tv, now)
129
143
  else
130
- time = Time.parse(tv.to_s).to_i rescue now
144
+ now
131
145
  end
132
- else
133
- time = now
134
- end
146
+
135
147
  me.add(time, record)
136
148
  last_record = record
137
149
  end
@@ -217,7 +229,7 @@ module Fluent::Plugin
217
229
  # ignore tables if TableElement#init failed
218
230
  @tables.reject! do |te|
219
231
  begin
220
- te.init(@tag_prefix, @base_model, router)
232
+ te.init(@tag_prefix, @base_model, router, log)
221
233
  log.info "Selecting '#{te.table}' table"
222
234
  false
223
235
  rescue => e
@@ -20,5 +20,12 @@ ActiveRecord::Schema.define(version: 20160225030107) do
20
20
  t.datetime "created_at", null: false
21
21
  t.datetime "updated_at", null: false
22
22
  end
23
+
24
+ create_table "messages_custom_time", force: :cascade do |t|
25
+ t.string "message"
26
+ t.datetime "created_at", null: false
27
+ t.datetime "updated_at", null: false
28
+ t.string "custom_time"
29
+ end
23
30
  end
24
31
 
@@ -82,9 +82,9 @@ class SqlInputTest < Test::Unit::TestCase
82
82
  [d.events[2][1], "message 3"],
83
83
  ]
84
84
  actual = [
85
- [Time.parse(d.events[0][2]["updated_at"]).to_i, d.events[0][2]["message"]],
86
- [Time.parse(d.events[1][2]["updated_at"]).to_i, d.events[1][2]["message"]],
87
- [Time.parse(d.events[2][2]["updated_at"]).to_i, d.events[2][2]["message"]],
85
+ [Fluent::EventTime.parse(d.events[0][2]["updated_at"]), d.events[0][2]["message"]],
86
+ [Fluent::EventTime.parse(d.events[1][2]["updated_at"]), d.events[1][2]["message"]],
87
+ [Fluent::EventTime.parse(d.events[2][2]["updated_at"]), d.events[2][2]["message"]],
88
88
  ]
89
89
  assert_equal(expected, actual)
90
90
  end
@@ -0,0 +1,114 @@
1
+ require "helper"
2
+ require "fluent/test/driver/input"
3
+
4
+ class SqlInputCustomTimeTest < Test::Unit::TestCase
5
+ def setup
6
+ Fluent::Test.setup
7
+ end
8
+
9
+ def teardown
10
+ end
11
+
12
+ CONFIG = %[
13
+ adapter postgresql
14
+ host localhost
15
+ port 5432
16
+ database fluentd_test
17
+
18
+ username fluentd
19
+ password fluentd
20
+
21
+ schema_search_path public
22
+
23
+ tag_prefix db
24
+
25
+ <table>
26
+ table messages_custom_time
27
+ tag logs
28
+ update_column updated_at
29
+ time_column custom_time
30
+ </table>
31
+ ]
32
+
33
+ def create_driver(conf = CONFIG)
34
+ Fluent::Test::Driver::Input.new(Fluent::Plugin::SQLInput).configure(conf)
35
+ end
36
+
37
+ def test_configure
38
+ d = create_driver
39
+ expected = {
40
+ host: "localhost",
41
+ port: 5432,
42
+ adapter: "postgresql",
43
+ database: "fluentd_test",
44
+ username: "fluentd",
45
+ password: "fluentd",
46
+ schema_search_path: "public",
47
+ tag_prefix: "db"
48
+ }
49
+ actual = {
50
+ host: d.instance.host,
51
+ port: d.instance.port,
52
+ adapter: d.instance.adapter,
53
+ database: d.instance.database,
54
+ username: d.instance.username,
55
+ password: d.instance.password,
56
+ schema_search_path: d.instance.schema_search_path,
57
+ tag_prefix: d.instance.tag_prefix
58
+ }
59
+ assert_equal(expected, actual)
60
+ tables = d.instance.instance_variable_get(:@tables)
61
+ assert_equal(1, tables.size)
62
+ messages_custom_time = tables.first
63
+ assert_equal("messages_custom_time", messages_custom_time.table)
64
+ assert_equal("logs", messages_custom_time.tag)
65
+ end
66
+
67
+ def test_message
68
+ d = create_driver(CONFIG + "select_interval 1")
69
+
70
+ start_time = Fluent::EventTime.now
71
+
72
+ # Create one message with a valid timestamp containing milliseconds and a time zone
73
+ Message.create!(message: "message 1", custom_time: '2020-08-27 15:00:16.100758000 -0400')
74
+
75
+ # Create one message without a timestamp so that we can test auto-creation
76
+ Message.create!(message: "message 2 (no timestamp)", custom_time: nil)
77
+
78
+ # Create one message with an unparseable timestamp so that we can check that a valid
79
+ # one is auto-generated.
80
+ Message.create!(message: "message 3 (bad timestamp)", custom_time: 'foo')
81
+
82
+ d.end_if do
83
+ d.record_count >= 3
84
+ end
85
+ d.run(timeout: 5)
86
+
87
+ assert_equal("db.logs", d.events[0][0])
88
+ expected = [
89
+ [d.events[0][1], "message 1"],
90
+ [d.events[1][1], "message 2 (no timestamp)"],
91
+ [d.events[2][1], "message 3 (bad timestamp)"],
92
+ ]
93
+
94
+ actual = [
95
+ [Fluent::EventTime.parse(d.events[0][2]["custom_time"]), d.events[0][2]["message"]],
96
+ d.events[1][2]["message"],
97
+ d.events[2][2]["message"],
98
+ ]
99
+
100
+ assert_equal(expected[0], actual[0])
101
+
102
+ # Messages 2 and 3 should have the same messages but (usually) a slightly later
103
+ # timestamps because they are generated by the input plugin instead of the test
104
+ # code
105
+ [1,2].each do |i|
106
+ assert_equal(expected[i][1], actual[i])
107
+ assert_operator(expected[i][0], :>=, start_time)
108
+ end
109
+ end
110
+
111
+ class Message < ActiveRecord::Base
112
+ self.table_name = "messages_custom_time"
113
+ end
114
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-sql
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0
4
+ version: 2.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-04-22 00:00:00.000000000 Z
11
+ date: 2020-09-10 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -147,6 +147,7 @@ files:
147
147
  - test/fixtures/schema.rb
148
148
  - test/helper.rb
149
149
  - test/plugin/test_in_sql.rb
150
+ - test/plugin/test_in_sql_with_custom_time.rb
150
151
  - test/plugin/test_out_sql.rb
151
152
  homepage: https://github.com/fluent/fluent-plugin-sql
152
153
  licenses:
@@ -167,7 +168,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
167
168
  - !ruby/object:Gem::Version
168
169
  version: '0'
169
170
  requirements: []
170
- rubygems_version: 3.1.2
171
+ rubygems_version: 3.0.3
171
172
  signing_key:
172
173
  specification_version: 4
173
174
  summary: SQL input/output plugin for Fluentd event collector
@@ -175,4 +176,5 @@ test_files:
175
176
  - test/fixtures/schema.rb
176
177
  - test/helper.rb
177
178
  - test/plugin/test_in_sql.rb
179
+ - test/plugin/test_in_sql_with_custom_time.rb
178
180
  - test/plugin/test_out_sql.rb