fluent-plugin-sql 1.0.0.rc1 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/linux.yml +69 -0
- data/ChangeLog +24 -0
- data/README.md +7 -3
- data/VERSION +1 -1
- data/fluent-plugin-sql.gemspec +6 -7
- data/lib/fluent/plugin/in_sql.rb +24 -9
- data/lib/fluent/plugin/out_sql.rb +27 -27
- data/test/fixtures/schema.rb +7 -0
- data/test/plugin/test_in_sql.rb +7 -3
- data/test/plugin/test_in_sql_with_custom_time.rb +114 -0
- data/test/plugin/test_out_sql.rb +8 -2
- metadata +22 -25
- data/.travis.yml +0 -32
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 02d185dee05a4611437c046d0d279fb7299a874dd6c46a76d174a03c983b2f16
|
4
|
+
data.tar.gz: 71708e3128e2ec604ccccaf4ca7aeeb13c20853f2cdf0f72bbf4264125706ea2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 94aff71abbd5fea54eca7b2d9534fe2615992586add9f2ced691770a21aae95e45c4276ed6b076c5505eca634d3ff82b1eb321a13959021791704fde88691533
|
7
|
+
data.tar.gz: ad2d4fe75de3032775ef25c6d6c5062a43ec01fc715a9a4e59371022975ae421c9e957282da0c8777a159f873b04280f09bd9aa34e2dc3fd512ac54136599ed6
|
@@ -0,0 +1,69 @@
|
|
1
|
+
name: linux
|
2
|
+
on:
|
3
|
+
- push
|
4
|
+
- pull_request
|
5
|
+
|
6
|
+
jobs:
|
7
|
+
build:
|
8
|
+
runs-on: ${{ matrix.os }}
|
9
|
+
continue-on-error: ${{ matrix.experimental }}
|
10
|
+
strategy:
|
11
|
+
fail-fast: false
|
12
|
+
matrix:
|
13
|
+
ruby: [ '2.5', '2.6', '2.7', '3.0' ]
|
14
|
+
postgres: [ '9', '10', '11', '12', '13' ]
|
15
|
+
os:
|
16
|
+
- ubuntu-latest
|
17
|
+
experimental: [false]
|
18
|
+
include:
|
19
|
+
- ruby: head
|
20
|
+
postgres: '9'
|
21
|
+
os: ubuntu-latest
|
22
|
+
experimental: true
|
23
|
+
- ruby: head
|
24
|
+
postgres: '10'
|
25
|
+
os: ubuntu-latest
|
26
|
+
experimental: true
|
27
|
+
- ruby: head
|
28
|
+
postgres: '11'
|
29
|
+
os: ubuntu-latest
|
30
|
+
experimental: true
|
31
|
+
- ruby: head
|
32
|
+
postgres: '12'
|
33
|
+
os: ubuntu-latest
|
34
|
+
experimental: true
|
35
|
+
- ruby: head
|
36
|
+
postgres: '13'
|
37
|
+
os: ubuntu-latest
|
38
|
+
experimental: true
|
39
|
+
services:
|
40
|
+
postgres:
|
41
|
+
image: postgres:${{ matrix.postgres }}
|
42
|
+
ports:
|
43
|
+
- 5432:5432
|
44
|
+
env:
|
45
|
+
POSTGRES_USER: postgres
|
46
|
+
POSTGRES_PASSWORD: password
|
47
|
+
POSTGRES_DB: test
|
48
|
+
options: >-
|
49
|
+
--health-cmd pg_isready
|
50
|
+
--health-interval 10s
|
51
|
+
--health-timeout 5s
|
52
|
+
--health-retries 5
|
53
|
+
name: Ruby ${{ matrix.ruby }} with PostgreSQL ${{ matrix.postgres }} unit testing on ${{ matrix.os }}
|
54
|
+
steps:
|
55
|
+
- uses: actions/checkout@v2
|
56
|
+
- uses: ruby/setup-ruby@v1
|
57
|
+
with:
|
58
|
+
ruby-version: ${{ matrix.ruby }}
|
59
|
+
- name: prepare database
|
60
|
+
env:
|
61
|
+
PGPASSWORD: password
|
62
|
+
run: |
|
63
|
+
psql -h localhost -p 5432 -U postgres -c "CREATE ROLE fluentd WITH LOGIN ENCRYPTED PASSWORD 'fluentd';"
|
64
|
+
psql -h localhost -p 5432 -U postgres -c "CREATE DATABASE fluentd_test OWNER fluentd;"
|
65
|
+
- name: unit testing
|
66
|
+
run: |
|
67
|
+
gem install bundler rake
|
68
|
+
bundle install --jobs 4 --retry 3
|
69
|
+
bundle exec rake test
|
data/ChangeLog
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
Release 2.2.0 - 2021/04/20
|
2
|
+
|
3
|
+
* Upgrade ActiveRecord to 6.1
|
4
|
+
|
5
|
+
Release 2.1.0 - 2020/09/09
|
6
|
+
|
7
|
+
* in_sql: Use Fluent::EventTime instead of Integer to preserve millisecond precision
|
8
|
+
|
9
|
+
Release 2.0.0 - 2020/04/22
|
10
|
+
|
11
|
+
* out_sql: Remove v0.12 API code and use full v1 API. Change buffer format.
|
12
|
+
|
13
|
+
Release 1.1.1 - 2019/05/10
|
14
|
+
|
15
|
+
* out_sql: Support schema_search_path option of PostgreSQL
|
16
|
+
|
17
|
+
Release 1.1.0 - 2018/10/04
|
18
|
+
|
19
|
+
* Upgrade ActiveRecord to 5.1
|
20
|
+
|
21
|
+
Release 1.0.0 - 2018/04/06
|
22
|
+
|
23
|
+
* Support v1 API
|
24
|
+
|
data/README.md
CHANGED
@@ -5,7 +5,7 @@
|
|
5
5
|
This SQL plugin has two parts:
|
6
6
|
|
7
7
|
1. SQL **input** plugin reads records from RDBMSes periodically. An example use case would be getting "diffs" of a table (based on the "updated_at" field).
|
8
|
-
2. SQL **output** plugin that writes records into
|
8
|
+
2. SQL **output** plugin that writes records into RDBMSes. An example use case would be aggregating server/app/sensor logs into RDBMS systems.
|
9
9
|
|
10
10
|
## Requirements
|
11
11
|
|
@@ -14,7 +14,7 @@ This SQL plugin has two parts:
|
|
14
14
|
| >= 1.0.0 | >= v0.14.4 | >= 2.1 |
|
15
15
|
| < 1.0.0 | < v0.14.0 | >= 1.9 |
|
16
16
|
|
17
|
-
NOTE: fluent-plugin-sql
|
17
|
+
NOTE: fluent-plugin-sql v2's buffer format is different from v1. If you update the plugin to v2, don't reuse v1's buffer.
|
18
18
|
|
19
19
|
## Installation
|
20
20
|
|
@@ -25,6 +25,8 @@ You should install actual RDBMS driver gem together. `pg` gem for postgresql ada
|
|
25
25
|
|
26
26
|
We recommend that mysql2 gem is higher than `0.3.12` and pg gem is higher than `0.16.0`.
|
27
27
|
|
28
|
+
If you use ruby 2.1, use pg gem 0.21.0 (< 1.0.0) because ActiveRecord 5.1.4 or earlier doesn't support Ruby 2.1.
|
29
|
+
|
28
30
|
## Input: How It Works
|
29
31
|
|
30
32
|
This plugin runs following SQL periodically:
|
@@ -148,10 +150,12 @@ This plugin takes advantage of ActiveRecord underneath. For `host`, `port`, `dat
|
|
148
150
|
* **username** RDBMS login user name
|
149
151
|
* **password** RDBMS login password
|
150
152
|
* **socket** RDBMS socket path
|
153
|
+
* **pool** A connection pool synchronizes thread access to a limited number of database connections
|
154
|
+
* **timeout** RDBMS connection timeout
|
151
155
|
* **remove_tag_prefix** remove the given prefix from the events. See "tag_prefix" in "Input: Configuration". (optional)
|
152
156
|
|
153
157
|
\<table\> sections:
|
154
158
|
|
155
159
|
* **table** RDBM table name
|
156
160
|
* **column_mapping**: [Required] Record to table schema mapping. The format is consists of `from:to` or `key` values are separated by `,`. For example, if set 'item_id:id,item_text:data,updated_at' to **column_mapping**, `item_id` field of record is stored into `id` column and `updated_at` field of record is stored into `updated_at` column.
|
157
|
-
*
|
161
|
+
* **\<table pattern\>**: the pattern to which the incoming event's tag (after it goes through `remove_tag_prefix`, if given). The patterns should follow the same syntax as [that of \<match\>](https://docs.fluentd.org/configuration/config-file#how-match-patterns-work). **Exactly one \<table\> element must NOT have this parameter so that it becomes the default table to store data**.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
2.2.0
|
data/fluent-plugin-sql.gemspec
CHANGED
@@ -4,12 +4,11 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
4
4
|
Gem::Specification.new do |gem|
|
5
5
|
gem.name = "fluent-plugin-sql"
|
6
6
|
gem.description = "SQL input/output plugin for Fluentd event collector"
|
7
|
-
gem.homepage = "https://github.com/
|
7
|
+
gem.homepage = "https://github.com/fluent/fluent-plugin-sql"
|
8
8
|
gem.summary = gem.description
|
9
9
|
gem.version = File.read("VERSION").strip
|
10
10
|
gem.authors = ["Sadayuki Furuhashi"]
|
11
11
|
gem.email = "frsyuki@gmail.com"
|
12
|
-
gem.has_rdoc = false
|
13
12
|
#gem.platform = Gem::Platform::RUBY
|
14
13
|
gem.files = `git ls-files`.split("\n")
|
15
14
|
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
@@ -17,12 +16,12 @@ Gem::Specification.new do |gem|
|
|
17
16
|
gem.require_paths = ['lib']
|
18
17
|
gem.license = "Apache-2.0"
|
19
18
|
|
20
|
-
gem.add_dependency "fluentd", [">=
|
21
|
-
gem.add_dependency 'activerecord', "~>
|
22
|
-
gem.add_dependency 'activerecord-import', "~> 0
|
19
|
+
gem.add_dependency "fluentd", [">= 1.7.0", "< 2"]
|
20
|
+
gem.add_dependency 'activerecord', "~> 6.1"
|
21
|
+
gem.add_dependency 'activerecord-import', "~> 1.0"
|
23
22
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
24
|
-
gem.add_development_dependency "test-unit", "
|
23
|
+
gem.add_development_dependency "test-unit", "> 3.1.0"
|
25
24
|
gem.add_development_dependency "test-unit-rr"
|
26
25
|
gem.add_development_dependency "test-unit-notify"
|
27
|
-
gem.add_development_dependency "pg"
|
26
|
+
gem.add_development_dependency "pg", '~> 1.0'
|
28
27
|
end
|
data/lib/fluent/plugin/in_sql.rb
CHANGED
@@ -39,6 +39,8 @@ module Fluent::Plugin
|
|
39
39
|
config_param :password, :string, default: nil, secret: true
|
40
40
|
desc 'RDBMS socket path'
|
41
41
|
config_param :socket, :string, default: nil
|
42
|
+
desc 'PostgreSQL schema search path'
|
43
|
+
config_param :schema_search_path, :string, default: nil
|
42
44
|
|
43
45
|
desc 'path to a file to store last rows'
|
44
46
|
config_param :state_file, :string, default: nil
|
@@ -58,13 +60,16 @@ module Fluent::Plugin
|
|
58
60
|
config_param :time_column, :string, default: nil
|
59
61
|
config_param :primary_key, :string, default: nil
|
60
62
|
|
63
|
+
attr_reader :log
|
64
|
+
|
61
65
|
def configure(conf)
|
62
66
|
super
|
63
67
|
end
|
64
68
|
|
65
|
-
def init(tag_prefix, base_model, router)
|
69
|
+
def init(tag_prefix, base_model, router, log)
|
66
70
|
@router = router
|
67
71
|
@tag = "#{tag_prefix}.#{@tag}" if tag_prefix
|
72
|
+
@log = log
|
68
73
|
|
69
74
|
# creates a model for this table
|
70
75
|
table_name = @table
|
@@ -106,6 +111,17 @@ module Fluent::Plugin
|
|
106
111
|
end
|
107
112
|
end
|
108
113
|
|
114
|
+
# Make sure we always have a Fluent::EventTime object regardless of what comes in
|
115
|
+
def normalized_time(tv, now)
|
116
|
+
return Fluent::EventTime.from_time(tv) if tv.is_a?(Time)
|
117
|
+
begin
|
118
|
+
Fluent::EventTime.parse(tv.to_s)
|
119
|
+
rescue
|
120
|
+
log.warn "Message contains invalid timestamp, using current time instead (#{now.inspect})"
|
121
|
+
now
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
109
125
|
# emits next records and returns the last record of emitted records
|
110
126
|
def emit_next_records(last_record, limit)
|
111
127
|
relation = @model
|
@@ -121,15 +137,13 @@ module Fluent::Plugin
|
|
121
137
|
relation.each do |obj|
|
122
138
|
record = obj.serializable_hash rescue nil
|
123
139
|
if record
|
124
|
-
|
125
|
-
if tv.
|
126
|
-
|
140
|
+
time =
|
141
|
+
if @time_column && (tv = obj.read_attribute(@time_column))
|
142
|
+
normalized_time(tv, now)
|
127
143
|
else
|
128
|
-
|
144
|
+
now
|
129
145
|
end
|
130
|
-
|
131
|
-
time = now
|
132
|
-
end
|
146
|
+
|
133
147
|
me.add(time, record)
|
134
148
|
last_record = record
|
135
149
|
end
|
@@ -176,6 +190,7 @@ module Fluent::Plugin
|
|
176
190
|
username: @username,
|
177
191
|
password: @password,
|
178
192
|
socket: @socket,
|
193
|
+
schema_search_path: @schema_search_path,
|
179
194
|
}
|
180
195
|
|
181
196
|
# creates subclass of ActiveRecord::Base so that it can have different
|
@@ -214,7 +229,7 @@ module Fluent::Plugin
|
|
214
229
|
# ignore tables if TableElement#init failed
|
215
230
|
@tables.reject! do |te|
|
216
231
|
begin
|
217
|
-
te.init(@tag_prefix, @base_model, router)
|
232
|
+
te.init(@tag_prefix, @base_model, router, log)
|
218
233
|
log.info "Selecting '#{te.table}' table"
|
219
234
|
false
|
220
235
|
rescue => e
|
@@ -1,11 +1,12 @@
|
|
1
1
|
require "fluent/plugin/output"
|
2
2
|
|
3
|
+
require 'active_record'
|
4
|
+
require 'activerecord-import'
|
5
|
+
|
3
6
|
module Fluent::Plugin
|
4
7
|
class SQLOutput < Output
|
5
8
|
Fluent::Plugin.register_output('sql', self)
|
6
9
|
|
7
|
-
DEFAULT_BUFFER_TYPE = "memory"
|
8
|
-
|
9
10
|
helpers :inject, :compat_parameters, :event_emitter
|
10
11
|
|
11
12
|
desc 'RDBMS host'
|
@@ -22,13 +23,19 @@ module Fluent::Plugin
|
|
22
23
|
config_param :database, :string
|
23
24
|
desc 'RDBMS socket path'
|
24
25
|
config_param :socket, :string, default: nil
|
26
|
+
desc 'PostgreSQL schema search path'
|
27
|
+
config_param :schema_search_path, :string, default: nil
|
25
28
|
desc 'remove the given prefix from the events'
|
26
29
|
config_param :remove_tag_prefix, :string, default: nil
|
27
30
|
desc 'enable fallback'
|
28
31
|
config_param :enable_fallback, :bool, default: true
|
32
|
+
desc "size of ActiveRecord's connection pool"
|
33
|
+
config_param :pool, :integer, default: 5
|
34
|
+
desc "specifies the timeout to establish a new connection to the database before failing"
|
35
|
+
config_param :timeout, :integer, default: 5000
|
29
36
|
|
30
37
|
config_section :buffer do
|
31
|
-
config_set_default
|
38
|
+
config_set_default :chunk_keys, ["tag"]
|
32
39
|
end
|
33
40
|
|
34
41
|
attr_accessor :tables
|
@@ -81,11 +88,12 @@ module Fluent::Plugin
|
|
81
88
|
# @model.column_names
|
82
89
|
end
|
83
90
|
|
84
|
-
def import(chunk)
|
91
|
+
def import(chunk, output)
|
92
|
+
tag = chunk.metadata.tag
|
85
93
|
records = []
|
86
|
-
chunk.msgpack_each { |
|
94
|
+
chunk.msgpack_each { |time, data|
|
87
95
|
begin
|
88
|
-
|
96
|
+
data = output.inject_values_to_record(tag, time, data)
|
89
97
|
records << @model.new(@format_proc.call(data))
|
90
98
|
rescue => e
|
91
99
|
args = {error: e, table: @table, record: Yajl.dump(data)}
|
@@ -100,7 +108,7 @@ module Fluent::Plugin
|
|
100
108
|
@log.warn "Got deterministic error. Fallback to one-by-one import", error: e
|
101
109
|
one_by_one_import(records)
|
102
110
|
else
|
103
|
-
|
111
|
+
@log.warn "Got deterministic error. Fallback is disabled", error: e
|
104
112
|
raise e
|
105
113
|
end
|
106
114
|
end
|
@@ -142,8 +150,6 @@ module Fluent::Plugin
|
|
142
150
|
|
143
151
|
def initialize
|
144
152
|
super
|
145
|
-
require 'active_record'
|
146
|
-
require 'activerecord-import'
|
147
153
|
end
|
148
154
|
|
149
155
|
def configure(conf)
|
@@ -169,7 +175,10 @@ module Fluent::Plugin
|
|
169
175
|
@tables << te
|
170
176
|
end
|
171
177
|
}
|
172
|
-
|
178
|
+
|
179
|
+
if @pool < @buffer_config.flush_thread_count
|
180
|
+
log.warn "connection pool size is smaller than buffer's flush_thread_count. Recommend to increase pool value", :pool => @pool, :flush_thread_count => @buffer_config.flush_thread_count
|
181
|
+
end
|
173
182
|
|
174
183
|
if @default_table.nil?
|
175
184
|
raise Fluent::ConfigError, "There is no default table. <table> is required in sql output"
|
@@ -187,6 +196,9 @@ module Fluent::Plugin
|
|
187
196
|
username: @username,
|
188
197
|
password: @password,
|
189
198
|
socket: @socket,
|
199
|
+
schema_search_path: @schema_search_path,
|
200
|
+
pool: @pool,
|
201
|
+
timeout: @timeout,
|
190
202
|
}
|
191
203
|
|
192
204
|
@base_model = Class.new(ActiveRecord::Base) do
|
@@ -207,19 +219,6 @@ module Fluent::Plugin
|
|
207
219
|
super
|
208
220
|
end
|
209
221
|
|
210
|
-
def emit(tag, es, chain)
|
211
|
-
if @only_default
|
212
|
-
super(tag, es, chain)
|
213
|
-
else
|
214
|
-
super(tag, es, chain, format_tag(tag))
|
215
|
-
end
|
216
|
-
end
|
217
|
-
|
218
|
-
def format(tag, time, record)
|
219
|
-
record = inject_values_to_record(tag, time, record)
|
220
|
-
[tag, time, record].to_msgpack
|
221
|
-
end
|
222
|
-
|
223
222
|
def formatted_to_msgpack_binary
|
224
223
|
true
|
225
224
|
end
|
@@ -228,11 +227,12 @@ module Fluent::Plugin
|
|
228
227
|
ActiveRecord::Base.connection_pool.with_connection do
|
229
228
|
|
230
229
|
@tables.each { |table|
|
231
|
-
|
232
|
-
|
230
|
+
tag = format_tag(chunk.metadata.tag)
|
231
|
+
if table.pattern.match(tag)
|
232
|
+
return table.import(chunk, self)
|
233
233
|
end
|
234
234
|
}
|
235
|
-
@default_table.import(chunk)
|
235
|
+
@default_table.import(chunk, self)
|
236
236
|
end
|
237
237
|
end
|
238
238
|
|
@@ -251,7 +251,7 @@ module Fluent::Plugin
|
|
251
251
|
end
|
252
252
|
|
253
253
|
def format_tag(tag)
|
254
|
-
if @remove_tag_prefix
|
254
|
+
if tag && @remove_tag_prefix
|
255
255
|
tag.gsub(@remove_tag_prefix, '')
|
256
256
|
else
|
257
257
|
tag
|
data/test/fixtures/schema.rb
CHANGED
@@ -20,5 +20,12 @@ ActiveRecord::Schema.define(version: 20160225030107) do
|
|
20
20
|
t.datetime "created_at", null: false
|
21
21
|
t.datetime "updated_at", null: false
|
22
22
|
end
|
23
|
+
|
24
|
+
create_table "messages_custom_time", force: :cascade do |t|
|
25
|
+
t.string "message"
|
26
|
+
t.datetime "created_at", null: false
|
27
|
+
t.datetime "updated_at", null: false
|
28
|
+
t.string "custom_time"
|
29
|
+
end
|
23
30
|
end
|
24
31
|
|
data/test/plugin/test_in_sql.rb
CHANGED
@@ -18,6 +18,8 @@ class SqlInputTest < Test::Unit::TestCase
|
|
18
18
|
username fluentd
|
19
19
|
password fluentd
|
20
20
|
|
21
|
+
schema_search_path public
|
22
|
+
|
21
23
|
tag_prefix db
|
22
24
|
|
23
25
|
<table>
|
@@ -41,6 +43,7 @@ class SqlInputTest < Test::Unit::TestCase
|
|
41
43
|
database: "fluentd_test",
|
42
44
|
username: "fluentd",
|
43
45
|
password: "fluentd",
|
46
|
+
schema_search_path: "public",
|
44
47
|
tag_prefix: "db"
|
45
48
|
}
|
46
49
|
actual = {
|
@@ -50,6 +53,7 @@ class SqlInputTest < Test::Unit::TestCase
|
|
50
53
|
database: d.instance.database,
|
51
54
|
username: d.instance.username,
|
52
55
|
password: d.instance.password,
|
56
|
+
schema_search_path: d.instance.schema_search_path,
|
53
57
|
tag_prefix: d.instance.tag_prefix
|
54
58
|
}
|
55
59
|
assert_equal(expected, actual)
|
@@ -78,9 +82,9 @@ class SqlInputTest < Test::Unit::TestCase
|
|
78
82
|
[d.events[2][1], "message 3"],
|
79
83
|
]
|
80
84
|
actual = [
|
81
|
-
[
|
82
|
-
[
|
83
|
-
[
|
85
|
+
[Fluent::EventTime.parse(d.events[0][2]["updated_at"]), d.events[0][2]["message"]],
|
86
|
+
[Fluent::EventTime.parse(d.events[1][2]["updated_at"]), d.events[1][2]["message"]],
|
87
|
+
[Fluent::EventTime.parse(d.events[2][2]["updated_at"]), d.events[2][2]["message"]],
|
84
88
|
]
|
85
89
|
assert_equal(expected, actual)
|
86
90
|
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
require "helper"
|
2
|
+
require "fluent/test/driver/input"
|
3
|
+
|
4
|
+
class SqlInputCustomTimeTest < Test::Unit::TestCase
|
5
|
+
def setup
|
6
|
+
Fluent::Test.setup
|
7
|
+
end
|
8
|
+
|
9
|
+
def teardown
|
10
|
+
end
|
11
|
+
|
12
|
+
CONFIG = %[
|
13
|
+
adapter postgresql
|
14
|
+
host localhost
|
15
|
+
port 5432
|
16
|
+
database fluentd_test
|
17
|
+
|
18
|
+
username fluentd
|
19
|
+
password fluentd
|
20
|
+
|
21
|
+
schema_search_path public
|
22
|
+
|
23
|
+
tag_prefix db
|
24
|
+
|
25
|
+
<table>
|
26
|
+
table messages_custom_time
|
27
|
+
tag logs
|
28
|
+
update_column updated_at
|
29
|
+
time_column custom_time
|
30
|
+
</table>
|
31
|
+
]
|
32
|
+
|
33
|
+
def create_driver(conf = CONFIG)
|
34
|
+
Fluent::Test::Driver::Input.new(Fluent::Plugin::SQLInput).configure(conf)
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_configure
|
38
|
+
d = create_driver
|
39
|
+
expected = {
|
40
|
+
host: "localhost",
|
41
|
+
port: 5432,
|
42
|
+
adapter: "postgresql",
|
43
|
+
database: "fluentd_test",
|
44
|
+
username: "fluentd",
|
45
|
+
password: "fluentd",
|
46
|
+
schema_search_path: "public",
|
47
|
+
tag_prefix: "db"
|
48
|
+
}
|
49
|
+
actual = {
|
50
|
+
host: d.instance.host,
|
51
|
+
port: d.instance.port,
|
52
|
+
adapter: d.instance.adapter,
|
53
|
+
database: d.instance.database,
|
54
|
+
username: d.instance.username,
|
55
|
+
password: d.instance.password,
|
56
|
+
schema_search_path: d.instance.schema_search_path,
|
57
|
+
tag_prefix: d.instance.tag_prefix
|
58
|
+
}
|
59
|
+
assert_equal(expected, actual)
|
60
|
+
tables = d.instance.instance_variable_get(:@tables)
|
61
|
+
assert_equal(1, tables.size)
|
62
|
+
messages_custom_time = tables.first
|
63
|
+
assert_equal("messages_custom_time", messages_custom_time.table)
|
64
|
+
assert_equal("logs", messages_custom_time.tag)
|
65
|
+
end
|
66
|
+
|
67
|
+
def test_message
|
68
|
+
d = create_driver(CONFIG + "select_interval 1")
|
69
|
+
|
70
|
+
start_time = Fluent::EventTime.now
|
71
|
+
|
72
|
+
# Create one message with a valid timestamp containing milliseconds and a time zone
|
73
|
+
Message.create!(message: "message 1", custom_time: '2020-08-27 15:00:16.100758000 -0400')
|
74
|
+
|
75
|
+
# Create one message without a timestamp so that we can test auto-creation
|
76
|
+
Message.create!(message: "message 2 (no timestamp)", custom_time: nil)
|
77
|
+
|
78
|
+
# Create one message with an unparseable timestamp so that we can check that a valid
|
79
|
+
# one is auto-generated.
|
80
|
+
Message.create!(message: "message 3 (bad timestamp)", custom_time: 'foo')
|
81
|
+
|
82
|
+
d.end_if do
|
83
|
+
d.record_count >= 3
|
84
|
+
end
|
85
|
+
d.run(timeout: 5)
|
86
|
+
|
87
|
+
assert_equal("db.logs", d.events[0][0])
|
88
|
+
expected = [
|
89
|
+
[d.events[0][1], "message 1"],
|
90
|
+
[d.events[1][1], "message 2 (no timestamp)"],
|
91
|
+
[d.events[2][1], "message 3 (bad timestamp)"],
|
92
|
+
]
|
93
|
+
|
94
|
+
actual = [
|
95
|
+
[Fluent::EventTime.parse(d.events[0][2]["custom_time"]), d.events[0][2]["message"]],
|
96
|
+
d.events[1][2]["message"],
|
97
|
+
d.events[2][2]["message"],
|
98
|
+
]
|
99
|
+
|
100
|
+
assert_equal(expected[0], actual[0])
|
101
|
+
|
102
|
+
# Messages 2 and 3 should have the same messages but (usually) a slightly later
|
103
|
+
# timestamps because they are generated by the input plugin instead of the test
|
104
|
+
# code
|
105
|
+
[1,2].each do |i|
|
106
|
+
assert_equal(expected[i][1], actual[i])
|
107
|
+
assert_operator(expected[i][0], :>=, start_time)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
class Message < ActiveRecord::Base
|
112
|
+
self.table_name = "messages_custom_time"
|
113
|
+
end
|
114
|
+
end
|
data/test/plugin/test_out_sql.rb
CHANGED
@@ -18,6 +18,8 @@ class SqlOutputTest < Test::Unit::TestCase
|
|
18
18
|
username fluentd
|
19
19
|
password fluentd
|
20
20
|
|
21
|
+
schema_search_path public
|
22
|
+
|
21
23
|
remove_tag_prefix db
|
22
24
|
|
23
25
|
<table>
|
@@ -39,8 +41,10 @@ class SqlOutputTest < Test::Unit::TestCase
|
|
39
41
|
database: "fluentd_test",
|
40
42
|
username: "fluentd",
|
41
43
|
password: "fluentd",
|
44
|
+
schema_search_path: 'public',
|
42
45
|
remove_tag_suffix: /^db/,
|
43
|
-
enable_fallback: true
|
46
|
+
enable_fallback: true,
|
47
|
+
pool: 5
|
44
48
|
}
|
45
49
|
actual = {
|
46
50
|
host: d.instance.host,
|
@@ -49,8 +53,10 @@ class SqlOutputTest < Test::Unit::TestCase
|
|
49
53
|
database: d.instance.database,
|
50
54
|
username: d.instance.username,
|
51
55
|
password: d.instance.password,
|
56
|
+
schema_search_path: d.instance.schema_search_path,
|
52
57
|
remove_tag_suffix: d.instance.remove_tag_prefix,
|
53
|
-
enable_fallback: d.instance.enable_fallback
|
58
|
+
enable_fallback: d.instance.enable_fallback,
|
59
|
+
pool: d.instance.pool
|
54
60
|
}
|
55
61
|
assert_equal(expected, actual)
|
56
62
|
assert_empty(d.instance.tables)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-sql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-04-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -16,7 +16,7 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 1.7.0
|
20
20
|
- - "<"
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: '2'
|
@@ -26,7 +26,7 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version:
|
29
|
+
version: 1.7.0
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2'
|
@@ -36,28 +36,28 @@ dependencies:
|
|
36
36
|
requirements:
|
37
37
|
- - "~>"
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version: '
|
39
|
+
version: '6.1'
|
40
40
|
type: :runtime
|
41
41
|
prerelease: false
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
44
|
- - "~>"
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: '
|
46
|
+
version: '6.1'
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: activerecord-import
|
49
49
|
requirement: !ruby/object:Gem::Requirement
|
50
50
|
requirements:
|
51
51
|
- - "~>"
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: '0
|
53
|
+
version: '1.0'
|
54
54
|
type: :runtime
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
58
|
- - "~>"
|
59
59
|
- !ruby/object:Gem::Version
|
60
|
-
version: '0
|
60
|
+
version: '1.0'
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
62
|
name: rake
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
@@ -76,14 +76,14 @@ dependencies:
|
|
76
76
|
name: test-unit
|
77
77
|
requirement: !ruby/object:Gem::Requirement
|
78
78
|
requirements:
|
79
|
-
- - "
|
79
|
+
- - ">"
|
80
80
|
- !ruby/object:Gem::Version
|
81
81
|
version: 3.1.0
|
82
82
|
type: :development
|
83
83
|
prerelease: false
|
84
84
|
version_requirements: !ruby/object:Gem::Requirement
|
85
85
|
requirements:
|
86
|
-
- - "
|
86
|
+
- - ">"
|
87
87
|
- !ruby/object:Gem::Version
|
88
88
|
version: 3.1.0
|
89
89
|
- !ruby/object:Gem::Dependency
|
@@ -118,23 +118,24 @@ dependencies:
|
|
118
118
|
name: pg
|
119
119
|
requirement: !ruby/object:Gem::Requirement
|
120
120
|
requirements:
|
121
|
-
- - "
|
121
|
+
- - "~>"
|
122
122
|
- !ruby/object:Gem::Version
|
123
|
-
version: '0'
|
123
|
+
version: '1.0'
|
124
124
|
type: :development
|
125
125
|
prerelease: false
|
126
126
|
version_requirements: !ruby/object:Gem::Requirement
|
127
127
|
requirements:
|
128
|
-
- - "
|
128
|
+
- - "~>"
|
129
129
|
- !ruby/object:Gem::Version
|
130
|
-
version: '0'
|
130
|
+
version: '1.0'
|
131
131
|
description: SQL input/output plugin for Fluentd event collector
|
132
132
|
email: frsyuki@gmail.com
|
133
133
|
executables: []
|
134
134
|
extensions: []
|
135
135
|
extra_rdoc_files: []
|
136
136
|
files:
|
137
|
-
- ".
|
137
|
+
- ".github/workflows/linux.yml"
|
138
|
+
- ChangeLog
|
138
139
|
- Gemfile
|
139
140
|
- Gemfile.v0.12
|
140
141
|
- README.md
|
@@ -146,8 +147,9 @@ files:
|
|
146
147
|
- test/fixtures/schema.rb
|
147
148
|
- test/helper.rb
|
148
149
|
- test/plugin/test_in_sql.rb
|
150
|
+
- test/plugin/test_in_sql_with_custom_time.rb
|
149
151
|
- test/plugin/test_out_sql.rb
|
150
|
-
homepage: https://github.com/
|
152
|
+
homepage: https://github.com/fluent/fluent-plugin-sql
|
151
153
|
licenses:
|
152
154
|
- Apache-2.0
|
153
155
|
metadata: {}
|
@@ -162,17 +164,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
162
164
|
version: '0'
|
163
165
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
164
166
|
requirements:
|
165
|
-
- - "
|
167
|
+
- - ">="
|
166
168
|
- !ruby/object:Gem::Version
|
167
|
-
version:
|
169
|
+
version: '0'
|
168
170
|
requirements: []
|
169
|
-
|
170
|
-
rubygems_version: 2.6.13
|
171
|
+
rubygems_version: 3.2.3
|
171
172
|
signing_key:
|
172
173
|
specification_version: 4
|
173
174
|
summary: SQL input/output plugin for Fluentd event collector
|
174
|
-
test_files:
|
175
|
-
- test/fixtures/schema.rb
|
176
|
-
- test/helper.rb
|
177
|
-
- test/plugin/test_in_sql.rb
|
178
|
-
- test/plugin/test_out_sql.rb
|
175
|
+
test_files: []
|
data/.travis.yml
DELETED
@@ -1,32 +0,0 @@
|
|
1
|
-
sudo: false
|
2
|
-
language: ruby
|
3
|
-
|
4
|
-
addons:
|
5
|
-
postgresql: "9.4"
|
6
|
-
|
7
|
-
rvm:
|
8
|
-
- 2.0
|
9
|
-
- 2.1
|
10
|
-
- 2.2.4
|
11
|
-
- 2.3.1
|
12
|
-
- 2.4.1
|
13
|
-
- ruby-head
|
14
|
-
|
15
|
-
gemfile:
|
16
|
-
- Gemfile
|
17
|
-
|
18
|
-
before_install:
|
19
|
-
- gem install bundler
|
20
|
-
|
21
|
-
before_script:
|
22
|
-
- psql -U postgres -c "CREATE ROLE fluentd WITH LOGIN ENCRYPTED PASSWORD 'fluentd';"
|
23
|
-
- psql -U postgres -c "CREATE DATABASE fluentd_test OWNER fluentd;"
|
24
|
-
|
25
|
-
script: bundle exec rake test
|
26
|
-
|
27
|
-
matrix:
|
28
|
-
allow_failures:
|
29
|
-
- rvm: ruby-head
|
30
|
-
exclude:
|
31
|
-
- rvm: 2.0
|
32
|
-
gemfile: Gemfile
|