fluent-plugin-sql 1.0.0.rc1 → 2.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/linux.yml +69 -0
- data/ChangeLog +24 -0
- data/README.md +7 -3
- data/VERSION +1 -1
- data/fluent-plugin-sql.gemspec +6 -7
- data/lib/fluent/plugin/in_sql.rb +24 -9
- data/lib/fluent/plugin/out_sql.rb +27 -27
- data/test/fixtures/schema.rb +7 -0
- data/test/plugin/test_in_sql.rb +7 -3
- data/test/plugin/test_in_sql_with_custom_time.rb +114 -0
- data/test/plugin/test_out_sql.rb +8 -2
- metadata +22 -25
- data/.travis.yml +0 -32
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 02d185dee05a4611437c046d0d279fb7299a874dd6c46a76d174a03c983b2f16
|
4
|
+
data.tar.gz: 71708e3128e2ec604ccccaf4ca7aeeb13c20853f2cdf0f72bbf4264125706ea2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 94aff71abbd5fea54eca7b2d9534fe2615992586add9f2ced691770a21aae95e45c4276ed6b076c5505eca634d3ff82b1eb321a13959021791704fde88691533
|
7
|
+
data.tar.gz: ad2d4fe75de3032775ef25c6d6c5062a43ec01fc715a9a4e59371022975ae421c9e957282da0c8777a159f873b04280f09bd9aa34e2dc3fd512ac54136599ed6
|
@@ -0,0 +1,69 @@
|
|
1
|
+
name: linux
|
2
|
+
on:
|
3
|
+
- push
|
4
|
+
- pull_request
|
5
|
+
|
6
|
+
jobs:
|
7
|
+
build:
|
8
|
+
runs-on: ${{ matrix.os }}
|
9
|
+
continue-on-error: ${{ matrix.experimental }}
|
10
|
+
strategy:
|
11
|
+
fail-fast: false
|
12
|
+
matrix:
|
13
|
+
ruby: [ '2.5', '2.6', '2.7', '3.0' ]
|
14
|
+
postgres: [ '9', '10', '11', '12', '13' ]
|
15
|
+
os:
|
16
|
+
- ubuntu-latest
|
17
|
+
experimental: [false]
|
18
|
+
include:
|
19
|
+
- ruby: head
|
20
|
+
postgres: '9'
|
21
|
+
os: ubuntu-latest
|
22
|
+
experimental: true
|
23
|
+
- ruby: head
|
24
|
+
postgres: '10'
|
25
|
+
os: ubuntu-latest
|
26
|
+
experimental: true
|
27
|
+
- ruby: head
|
28
|
+
postgres: '11'
|
29
|
+
os: ubuntu-latest
|
30
|
+
experimental: true
|
31
|
+
- ruby: head
|
32
|
+
postgres: '12'
|
33
|
+
os: ubuntu-latest
|
34
|
+
experimental: true
|
35
|
+
- ruby: head
|
36
|
+
postgres: '13'
|
37
|
+
os: ubuntu-latest
|
38
|
+
experimental: true
|
39
|
+
services:
|
40
|
+
postgres:
|
41
|
+
image: postgres:${{ matrix.postgres }}
|
42
|
+
ports:
|
43
|
+
- 5432:5432
|
44
|
+
env:
|
45
|
+
POSTGRES_USER: postgres
|
46
|
+
POSTGRES_PASSWORD: password
|
47
|
+
POSTGRES_DB: test
|
48
|
+
options: >-
|
49
|
+
--health-cmd pg_isready
|
50
|
+
--health-interval 10s
|
51
|
+
--health-timeout 5s
|
52
|
+
--health-retries 5
|
53
|
+
name: Ruby ${{ matrix.ruby }} with PostgreSQL ${{ matrix.postgres }} unit testing on ${{ matrix.os }}
|
54
|
+
steps:
|
55
|
+
- uses: actions/checkout@v2
|
56
|
+
- uses: ruby/setup-ruby@v1
|
57
|
+
with:
|
58
|
+
ruby-version: ${{ matrix.ruby }}
|
59
|
+
- name: prepare database
|
60
|
+
env:
|
61
|
+
PGPASSWORD: password
|
62
|
+
run: |
|
63
|
+
psql -h localhost -p 5432 -U postgres -c "CREATE ROLE fluentd WITH LOGIN ENCRYPTED PASSWORD 'fluentd';"
|
64
|
+
psql -h localhost -p 5432 -U postgres -c "CREATE DATABASE fluentd_test OWNER fluentd;"
|
65
|
+
- name: unit testing
|
66
|
+
run: |
|
67
|
+
gem install bundler rake
|
68
|
+
bundle install --jobs 4 --retry 3
|
69
|
+
bundle exec rake test
|
data/ChangeLog
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
Release 2.2.0 - 2021/04/20
|
2
|
+
|
3
|
+
* Upgrade ActiveRecord to 6.1
|
4
|
+
|
5
|
+
Release 2.1.0 - 2020/09/09
|
6
|
+
|
7
|
+
* in_sql: Use Fluent::EventTime instead of Integer to preserve millisecond precision
|
8
|
+
|
9
|
+
Release 2.0.0 - 2020/04/22
|
10
|
+
|
11
|
+
* out_sql: Remove v0.12 API code and use full v1 API. Change buffer format.
|
12
|
+
|
13
|
+
Release 1.1.1 - 2019/05/10
|
14
|
+
|
15
|
+
* out_sql: Support schema_search_path option of PostgreSQL
|
16
|
+
|
17
|
+
Release 1.1.0 - 2018/10/04
|
18
|
+
|
19
|
+
* Upgrade ActiveRecord to 5.1
|
20
|
+
|
21
|
+
Release 1.0.0 - 2018/04/06
|
22
|
+
|
23
|
+
* Support v1 API
|
24
|
+
|
data/README.md
CHANGED
@@ -5,7 +5,7 @@
|
|
5
5
|
This SQL plugin has two parts:
|
6
6
|
|
7
7
|
1. SQL **input** plugin reads records from RDBMSes periodically. An example use case would be getting "diffs" of a table (based on the "updated_at" field).
|
8
|
-
2. SQL **output** plugin that writes records into
|
8
|
+
2. SQL **output** plugin that writes records into RDBMSes. An example use case would be aggregating server/app/sensor logs into RDBMS systems.
|
9
9
|
|
10
10
|
## Requirements
|
11
11
|
|
@@ -14,7 +14,7 @@ This SQL plugin has two parts:
|
|
14
14
|
| >= 1.0.0 | >= v0.14.4 | >= 2.1 |
|
15
15
|
| < 1.0.0 | < v0.14.0 | >= 1.9 |
|
16
16
|
|
17
|
-
NOTE: fluent-plugin-sql
|
17
|
+
NOTE: fluent-plugin-sql v2's buffer format is different from v1. If you update the plugin to v2, don't reuse v1's buffer.
|
18
18
|
|
19
19
|
## Installation
|
20
20
|
|
@@ -25,6 +25,8 @@ You should install actual RDBMS driver gem together. `pg` gem for postgresql ada
|
|
25
25
|
|
26
26
|
We recommend that mysql2 gem is higher than `0.3.12` and pg gem is higher than `0.16.0`.
|
27
27
|
|
28
|
+
If you use ruby 2.1, use pg gem 0.21.0 (< 1.0.0) because ActiveRecord 5.1.4 or earlier doesn't support Ruby 2.1.
|
29
|
+
|
28
30
|
## Input: How It Works
|
29
31
|
|
30
32
|
This plugin runs following SQL periodically:
|
@@ -148,10 +150,12 @@ This plugin takes advantage of ActiveRecord underneath. For `host`, `port`, `dat
|
|
148
150
|
* **username** RDBMS login user name
|
149
151
|
* **password** RDBMS login password
|
150
152
|
* **socket** RDBMS socket path
|
153
|
+
* **pool** A connection pool synchronizes thread access to a limited number of database connections
|
154
|
+
* **timeout** RDBMS connection timeout
|
151
155
|
* **remove_tag_prefix** remove the given prefix from the events. See "tag_prefix" in "Input: Configuration". (optional)
|
152
156
|
|
153
157
|
\<table\> sections:
|
154
158
|
|
155
159
|
* **table** RDBM table name
|
156
160
|
* **column_mapping**: [Required] Record to table schema mapping. The format is consists of `from:to` or `key` values are separated by `,`. For example, if set 'item_id:id,item_text:data,updated_at' to **column_mapping**, `item_id` field of record is stored into `id` column and `updated_at` field of record is stored into `updated_at` column.
|
157
|
-
*
|
161
|
+
* **\<table pattern\>**: the pattern to which the incoming event's tag (after it goes through `remove_tag_prefix`, if given). The patterns should follow the same syntax as [that of \<match\>](https://docs.fluentd.org/configuration/config-file#how-match-patterns-work). **Exactly one \<table\> element must NOT have this parameter so that it becomes the default table to store data**.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
2.2.0
|
data/fluent-plugin-sql.gemspec
CHANGED
@@ -4,12 +4,11 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
4
4
|
Gem::Specification.new do |gem|
|
5
5
|
gem.name = "fluent-plugin-sql"
|
6
6
|
gem.description = "SQL input/output plugin for Fluentd event collector"
|
7
|
-
gem.homepage = "https://github.com/
|
7
|
+
gem.homepage = "https://github.com/fluent/fluent-plugin-sql"
|
8
8
|
gem.summary = gem.description
|
9
9
|
gem.version = File.read("VERSION").strip
|
10
10
|
gem.authors = ["Sadayuki Furuhashi"]
|
11
11
|
gem.email = "frsyuki@gmail.com"
|
12
|
-
gem.has_rdoc = false
|
13
12
|
#gem.platform = Gem::Platform::RUBY
|
14
13
|
gem.files = `git ls-files`.split("\n")
|
15
14
|
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
@@ -17,12 +16,12 @@ Gem::Specification.new do |gem|
|
|
17
16
|
gem.require_paths = ['lib']
|
18
17
|
gem.license = "Apache-2.0"
|
19
18
|
|
20
|
-
gem.add_dependency "fluentd", [">=
|
21
|
-
gem.add_dependency 'activerecord', "~>
|
22
|
-
gem.add_dependency 'activerecord-import', "~> 0
|
19
|
+
gem.add_dependency "fluentd", [">= 1.7.0", "< 2"]
|
20
|
+
gem.add_dependency 'activerecord', "~> 6.1"
|
21
|
+
gem.add_dependency 'activerecord-import', "~> 1.0"
|
23
22
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
24
|
-
gem.add_development_dependency "test-unit", "
|
23
|
+
gem.add_development_dependency "test-unit", "> 3.1.0"
|
25
24
|
gem.add_development_dependency "test-unit-rr"
|
26
25
|
gem.add_development_dependency "test-unit-notify"
|
27
|
-
gem.add_development_dependency "pg"
|
26
|
+
gem.add_development_dependency "pg", '~> 1.0'
|
28
27
|
end
|
data/lib/fluent/plugin/in_sql.rb
CHANGED
@@ -39,6 +39,8 @@ module Fluent::Plugin
|
|
39
39
|
config_param :password, :string, default: nil, secret: true
|
40
40
|
desc 'RDBMS socket path'
|
41
41
|
config_param :socket, :string, default: nil
|
42
|
+
desc 'PostgreSQL schema search path'
|
43
|
+
config_param :schema_search_path, :string, default: nil
|
42
44
|
|
43
45
|
desc 'path to a file to store last rows'
|
44
46
|
config_param :state_file, :string, default: nil
|
@@ -58,13 +60,16 @@ module Fluent::Plugin
|
|
58
60
|
config_param :time_column, :string, default: nil
|
59
61
|
config_param :primary_key, :string, default: nil
|
60
62
|
|
63
|
+
attr_reader :log
|
64
|
+
|
61
65
|
def configure(conf)
|
62
66
|
super
|
63
67
|
end
|
64
68
|
|
65
|
-
def init(tag_prefix, base_model, router)
|
69
|
+
def init(tag_prefix, base_model, router, log)
|
66
70
|
@router = router
|
67
71
|
@tag = "#{tag_prefix}.#{@tag}" if tag_prefix
|
72
|
+
@log = log
|
68
73
|
|
69
74
|
# creates a model for this table
|
70
75
|
table_name = @table
|
@@ -106,6 +111,17 @@ module Fluent::Plugin
|
|
106
111
|
end
|
107
112
|
end
|
108
113
|
|
114
|
+
# Make sure we always have a Fluent::EventTime object regardless of what comes in
|
115
|
+
def normalized_time(tv, now)
|
116
|
+
return Fluent::EventTime.from_time(tv) if tv.is_a?(Time)
|
117
|
+
begin
|
118
|
+
Fluent::EventTime.parse(tv.to_s)
|
119
|
+
rescue
|
120
|
+
log.warn "Message contains invalid timestamp, using current time instead (#{now.inspect})"
|
121
|
+
now
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
109
125
|
# emits next records and returns the last record of emitted records
|
110
126
|
def emit_next_records(last_record, limit)
|
111
127
|
relation = @model
|
@@ -121,15 +137,13 @@ module Fluent::Plugin
|
|
121
137
|
relation.each do |obj|
|
122
138
|
record = obj.serializable_hash rescue nil
|
123
139
|
if record
|
124
|
-
|
125
|
-
if tv.
|
126
|
-
|
140
|
+
time =
|
141
|
+
if @time_column && (tv = obj.read_attribute(@time_column))
|
142
|
+
normalized_time(tv, now)
|
127
143
|
else
|
128
|
-
|
144
|
+
now
|
129
145
|
end
|
130
|
-
|
131
|
-
time = now
|
132
|
-
end
|
146
|
+
|
133
147
|
me.add(time, record)
|
134
148
|
last_record = record
|
135
149
|
end
|
@@ -176,6 +190,7 @@ module Fluent::Plugin
|
|
176
190
|
username: @username,
|
177
191
|
password: @password,
|
178
192
|
socket: @socket,
|
193
|
+
schema_search_path: @schema_search_path,
|
179
194
|
}
|
180
195
|
|
181
196
|
# creates subclass of ActiveRecord::Base so that it can have different
|
@@ -214,7 +229,7 @@ module Fluent::Plugin
|
|
214
229
|
# ignore tables if TableElement#init failed
|
215
230
|
@tables.reject! do |te|
|
216
231
|
begin
|
217
|
-
te.init(@tag_prefix, @base_model, router)
|
232
|
+
te.init(@tag_prefix, @base_model, router, log)
|
218
233
|
log.info "Selecting '#{te.table}' table"
|
219
234
|
false
|
220
235
|
rescue => e
|
@@ -1,11 +1,12 @@
|
|
1
1
|
require "fluent/plugin/output"
|
2
2
|
|
3
|
+
require 'active_record'
|
4
|
+
require 'activerecord-import'
|
5
|
+
|
3
6
|
module Fluent::Plugin
|
4
7
|
class SQLOutput < Output
|
5
8
|
Fluent::Plugin.register_output('sql', self)
|
6
9
|
|
7
|
-
DEFAULT_BUFFER_TYPE = "memory"
|
8
|
-
|
9
10
|
helpers :inject, :compat_parameters, :event_emitter
|
10
11
|
|
11
12
|
desc 'RDBMS host'
|
@@ -22,13 +23,19 @@ module Fluent::Plugin
|
|
22
23
|
config_param :database, :string
|
23
24
|
desc 'RDBMS socket path'
|
24
25
|
config_param :socket, :string, default: nil
|
26
|
+
desc 'PostgreSQL schema search path'
|
27
|
+
config_param :schema_search_path, :string, default: nil
|
25
28
|
desc 'remove the given prefix from the events'
|
26
29
|
config_param :remove_tag_prefix, :string, default: nil
|
27
30
|
desc 'enable fallback'
|
28
31
|
config_param :enable_fallback, :bool, default: true
|
32
|
+
desc "size of ActiveRecord's connection pool"
|
33
|
+
config_param :pool, :integer, default: 5
|
34
|
+
desc "specifies the timeout to establish a new connection to the database before failing"
|
35
|
+
config_param :timeout, :integer, default: 5000
|
29
36
|
|
30
37
|
config_section :buffer do
|
31
|
-
config_set_default
|
38
|
+
config_set_default :chunk_keys, ["tag"]
|
32
39
|
end
|
33
40
|
|
34
41
|
attr_accessor :tables
|
@@ -81,11 +88,12 @@ module Fluent::Plugin
|
|
81
88
|
# @model.column_names
|
82
89
|
end
|
83
90
|
|
84
|
-
def import(chunk)
|
91
|
+
def import(chunk, output)
|
92
|
+
tag = chunk.metadata.tag
|
85
93
|
records = []
|
86
|
-
chunk.msgpack_each { |
|
94
|
+
chunk.msgpack_each { |time, data|
|
87
95
|
begin
|
88
|
-
|
96
|
+
data = output.inject_values_to_record(tag, time, data)
|
89
97
|
records << @model.new(@format_proc.call(data))
|
90
98
|
rescue => e
|
91
99
|
args = {error: e, table: @table, record: Yajl.dump(data)}
|
@@ -100,7 +108,7 @@ module Fluent::Plugin
|
|
100
108
|
@log.warn "Got deterministic error. Fallback to one-by-one import", error: e
|
101
109
|
one_by_one_import(records)
|
102
110
|
else
|
103
|
-
|
111
|
+
@log.warn "Got deterministic error. Fallback is disabled", error: e
|
104
112
|
raise e
|
105
113
|
end
|
106
114
|
end
|
@@ -142,8 +150,6 @@ module Fluent::Plugin
|
|
142
150
|
|
143
151
|
def initialize
|
144
152
|
super
|
145
|
-
require 'active_record'
|
146
|
-
require 'activerecord-import'
|
147
153
|
end
|
148
154
|
|
149
155
|
def configure(conf)
|
@@ -169,7 +175,10 @@ module Fluent::Plugin
|
|
169
175
|
@tables << te
|
170
176
|
end
|
171
177
|
}
|
172
|
-
|
178
|
+
|
179
|
+
if @pool < @buffer_config.flush_thread_count
|
180
|
+
log.warn "connection pool size is smaller than buffer's flush_thread_count. Recommend to increase pool value", :pool => @pool, :flush_thread_count => @buffer_config.flush_thread_count
|
181
|
+
end
|
173
182
|
|
174
183
|
if @default_table.nil?
|
175
184
|
raise Fluent::ConfigError, "There is no default table. <table> is required in sql output"
|
@@ -187,6 +196,9 @@ module Fluent::Plugin
|
|
187
196
|
username: @username,
|
188
197
|
password: @password,
|
189
198
|
socket: @socket,
|
199
|
+
schema_search_path: @schema_search_path,
|
200
|
+
pool: @pool,
|
201
|
+
timeout: @timeout,
|
190
202
|
}
|
191
203
|
|
192
204
|
@base_model = Class.new(ActiveRecord::Base) do
|
@@ -207,19 +219,6 @@ module Fluent::Plugin
|
|
207
219
|
super
|
208
220
|
end
|
209
221
|
|
210
|
-
def emit(tag, es, chain)
|
211
|
-
if @only_default
|
212
|
-
super(tag, es, chain)
|
213
|
-
else
|
214
|
-
super(tag, es, chain, format_tag(tag))
|
215
|
-
end
|
216
|
-
end
|
217
|
-
|
218
|
-
def format(tag, time, record)
|
219
|
-
record = inject_values_to_record(tag, time, record)
|
220
|
-
[tag, time, record].to_msgpack
|
221
|
-
end
|
222
|
-
|
223
222
|
def formatted_to_msgpack_binary
|
224
223
|
true
|
225
224
|
end
|
@@ -228,11 +227,12 @@ module Fluent::Plugin
|
|
228
227
|
ActiveRecord::Base.connection_pool.with_connection do
|
229
228
|
|
230
229
|
@tables.each { |table|
|
231
|
-
|
232
|
-
|
230
|
+
tag = format_tag(chunk.metadata.tag)
|
231
|
+
if table.pattern.match(tag)
|
232
|
+
return table.import(chunk, self)
|
233
233
|
end
|
234
234
|
}
|
235
|
-
@default_table.import(chunk)
|
235
|
+
@default_table.import(chunk, self)
|
236
236
|
end
|
237
237
|
end
|
238
238
|
|
@@ -251,7 +251,7 @@ module Fluent::Plugin
|
|
251
251
|
end
|
252
252
|
|
253
253
|
def format_tag(tag)
|
254
|
-
if @remove_tag_prefix
|
254
|
+
if tag && @remove_tag_prefix
|
255
255
|
tag.gsub(@remove_tag_prefix, '')
|
256
256
|
else
|
257
257
|
tag
|
data/test/fixtures/schema.rb
CHANGED
@@ -20,5 +20,12 @@ ActiveRecord::Schema.define(version: 20160225030107) do
|
|
20
20
|
t.datetime "created_at", null: false
|
21
21
|
t.datetime "updated_at", null: false
|
22
22
|
end
|
23
|
+
|
24
|
+
create_table "messages_custom_time", force: :cascade do |t|
|
25
|
+
t.string "message"
|
26
|
+
t.datetime "created_at", null: false
|
27
|
+
t.datetime "updated_at", null: false
|
28
|
+
t.string "custom_time"
|
29
|
+
end
|
23
30
|
end
|
24
31
|
|
data/test/plugin/test_in_sql.rb
CHANGED
@@ -18,6 +18,8 @@ class SqlInputTest < Test::Unit::TestCase
|
|
18
18
|
username fluentd
|
19
19
|
password fluentd
|
20
20
|
|
21
|
+
schema_search_path public
|
22
|
+
|
21
23
|
tag_prefix db
|
22
24
|
|
23
25
|
<table>
|
@@ -41,6 +43,7 @@ class SqlInputTest < Test::Unit::TestCase
|
|
41
43
|
database: "fluentd_test",
|
42
44
|
username: "fluentd",
|
43
45
|
password: "fluentd",
|
46
|
+
schema_search_path: "public",
|
44
47
|
tag_prefix: "db"
|
45
48
|
}
|
46
49
|
actual = {
|
@@ -50,6 +53,7 @@ class SqlInputTest < Test::Unit::TestCase
|
|
50
53
|
database: d.instance.database,
|
51
54
|
username: d.instance.username,
|
52
55
|
password: d.instance.password,
|
56
|
+
schema_search_path: d.instance.schema_search_path,
|
53
57
|
tag_prefix: d.instance.tag_prefix
|
54
58
|
}
|
55
59
|
assert_equal(expected, actual)
|
@@ -78,9 +82,9 @@ class SqlInputTest < Test::Unit::TestCase
|
|
78
82
|
[d.events[2][1], "message 3"],
|
79
83
|
]
|
80
84
|
actual = [
|
81
|
-
[
|
82
|
-
[
|
83
|
-
[
|
85
|
+
[Fluent::EventTime.parse(d.events[0][2]["updated_at"]), d.events[0][2]["message"]],
|
86
|
+
[Fluent::EventTime.parse(d.events[1][2]["updated_at"]), d.events[1][2]["message"]],
|
87
|
+
[Fluent::EventTime.parse(d.events[2][2]["updated_at"]), d.events[2][2]["message"]],
|
84
88
|
]
|
85
89
|
assert_equal(expected, actual)
|
86
90
|
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
require "helper"
|
2
|
+
require "fluent/test/driver/input"
|
3
|
+
|
4
|
+
class SqlInputCustomTimeTest < Test::Unit::TestCase
|
5
|
+
def setup
|
6
|
+
Fluent::Test.setup
|
7
|
+
end
|
8
|
+
|
9
|
+
def teardown
|
10
|
+
end
|
11
|
+
|
12
|
+
CONFIG = %[
|
13
|
+
adapter postgresql
|
14
|
+
host localhost
|
15
|
+
port 5432
|
16
|
+
database fluentd_test
|
17
|
+
|
18
|
+
username fluentd
|
19
|
+
password fluentd
|
20
|
+
|
21
|
+
schema_search_path public
|
22
|
+
|
23
|
+
tag_prefix db
|
24
|
+
|
25
|
+
<table>
|
26
|
+
table messages_custom_time
|
27
|
+
tag logs
|
28
|
+
update_column updated_at
|
29
|
+
time_column custom_time
|
30
|
+
</table>
|
31
|
+
]
|
32
|
+
|
33
|
+
def create_driver(conf = CONFIG)
|
34
|
+
Fluent::Test::Driver::Input.new(Fluent::Plugin::SQLInput).configure(conf)
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_configure
|
38
|
+
d = create_driver
|
39
|
+
expected = {
|
40
|
+
host: "localhost",
|
41
|
+
port: 5432,
|
42
|
+
adapter: "postgresql",
|
43
|
+
database: "fluentd_test",
|
44
|
+
username: "fluentd",
|
45
|
+
password: "fluentd",
|
46
|
+
schema_search_path: "public",
|
47
|
+
tag_prefix: "db"
|
48
|
+
}
|
49
|
+
actual = {
|
50
|
+
host: d.instance.host,
|
51
|
+
port: d.instance.port,
|
52
|
+
adapter: d.instance.adapter,
|
53
|
+
database: d.instance.database,
|
54
|
+
username: d.instance.username,
|
55
|
+
password: d.instance.password,
|
56
|
+
schema_search_path: d.instance.schema_search_path,
|
57
|
+
tag_prefix: d.instance.tag_prefix
|
58
|
+
}
|
59
|
+
assert_equal(expected, actual)
|
60
|
+
tables = d.instance.instance_variable_get(:@tables)
|
61
|
+
assert_equal(1, tables.size)
|
62
|
+
messages_custom_time = tables.first
|
63
|
+
assert_equal("messages_custom_time", messages_custom_time.table)
|
64
|
+
assert_equal("logs", messages_custom_time.tag)
|
65
|
+
end
|
66
|
+
|
67
|
+
def test_message
|
68
|
+
d = create_driver(CONFIG + "select_interval 1")
|
69
|
+
|
70
|
+
start_time = Fluent::EventTime.now
|
71
|
+
|
72
|
+
# Create one message with a valid timestamp containing milliseconds and a time zone
|
73
|
+
Message.create!(message: "message 1", custom_time: '2020-08-27 15:00:16.100758000 -0400')
|
74
|
+
|
75
|
+
# Create one message without a timestamp so that we can test auto-creation
|
76
|
+
Message.create!(message: "message 2 (no timestamp)", custom_time: nil)
|
77
|
+
|
78
|
+
# Create one message with an unparseable timestamp so that we can check that a valid
|
79
|
+
# one is auto-generated.
|
80
|
+
Message.create!(message: "message 3 (bad timestamp)", custom_time: 'foo')
|
81
|
+
|
82
|
+
d.end_if do
|
83
|
+
d.record_count >= 3
|
84
|
+
end
|
85
|
+
d.run(timeout: 5)
|
86
|
+
|
87
|
+
assert_equal("db.logs", d.events[0][0])
|
88
|
+
expected = [
|
89
|
+
[d.events[0][1], "message 1"],
|
90
|
+
[d.events[1][1], "message 2 (no timestamp)"],
|
91
|
+
[d.events[2][1], "message 3 (bad timestamp)"],
|
92
|
+
]
|
93
|
+
|
94
|
+
actual = [
|
95
|
+
[Fluent::EventTime.parse(d.events[0][2]["custom_time"]), d.events[0][2]["message"]],
|
96
|
+
d.events[1][2]["message"],
|
97
|
+
d.events[2][2]["message"],
|
98
|
+
]
|
99
|
+
|
100
|
+
assert_equal(expected[0], actual[0])
|
101
|
+
|
102
|
+
# Messages 2 and 3 should have the same messages but (usually) a slightly later
|
103
|
+
# timestamps because they are generated by the input plugin instead of the test
|
104
|
+
# code
|
105
|
+
[1,2].each do |i|
|
106
|
+
assert_equal(expected[i][1], actual[i])
|
107
|
+
assert_operator(expected[i][0], :>=, start_time)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
class Message < ActiveRecord::Base
|
112
|
+
self.table_name = "messages_custom_time"
|
113
|
+
end
|
114
|
+
end
|
data/test/plugin/test_out_sql.rb
CHANGED
@@ -18,6 +18,8 @@ class SqlOutputTest < Test::Unit::TestCase
|
|
18
18
|
username fluentd
|
19
19
|
password fluentd
|
20
20
|
|
21
|
+
schema_search_path public
|
22
|
+
|
21
23
|
remove_tag_prefix db
|
22
24
|
|
23
25
|
<table>
|
@@ -39,8 +41,10 @@ class SqlOutputTest < Test::Unit::TestCase
|
|
39
41
|
database: "fluentd_test",
|
40
42
|
username: "fluentd",
|
41
43
|
password: "fluentd",
|
44
|
+
schema_search_path: 'public',
|
42
45
|
remove_tag_suffix: /^db/,
|
43
|
-
enable_fallback: true
|
46
|
+
enable_fallback: true,
|
47
|
+
pool: 5
|
44
48
|
}
|
45
49
|
actual = {
|
46
50
|
host: d.instance.host,
|
@@ -49,8 +53,10 @@ class SqlOutputTest < Test::Unit::TestCase
|
|
49
53
|
database: d.instance.database,
|
50
54
|
username: d.instance.username,
|
51
55
|
password: d.instance.password,
|
56
|
+
schema_search_path: d.instance.schema_search_path,
|
52
57
|
remove_tag_suffix: d.instance.remove_tag_prefix,
|
53
|
-
enable_fallback: d.instance.enable_fallback
|
58
|
+
enable_fallback: d.instance.enable_fallback,
|
59
|
+
pool: d.instance.pool
|
54
60
|
}
|
55
61
|
assert_equal(expected, actual)
|
56
62
|
assert_empty(d.instance.tables)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-sql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-04-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -16,7 +16,7 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 1.7.0
|
20
20
|
- - "<"
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: '2'
|
@@ -26,7 +26,7 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version:
|
29
|
+
version: 1.7.0
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2'
|
@@ -36,28 +36,28 @@ dependencies:
|
|
36
36
|
requirements:
|
37
37
|
- - "~>"
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version: '
|
39
|
+
version: '6.1'
|
40
40
|
type: :runtime
|
41
41
|
prerelease: false
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
44
|
- - "~>"
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: '
|
46
|
+
version: '6.1'
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: activerecord-import
|
49
49
|
requirement: !ruby/object:Gem::Requirement
|
50
50
|
requirements:
|
51
51
|
- - "~>"
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: '0
|
53
|
+
version: '1.0'
|
54
54
|
type: :runtime
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
58
|
- - "~>"
|
59
59
|
- !ruby/object:Gem::Version
|
60
|
-
version: '0
|
60
|
+
version: '1.0'
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
62
|
name: rake
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
@@ -76,14 +76,14 @@ dependencies:
|
|
76
76
|
name: test-unit
|
77
77
|
requirement: !ruby/object:Gem::Requirement
|
78
78
|
requirements:
|
79
|
-
- - "
|
79
|
+
- - ">"
|
80
80
|
- !ruby/object:Gem::Version
|
81
81
|
version: 3.1.0
|
82
82
|
type: :development
|
83
83
|
prerelease: false
|
84
84
|
version_requirements: !ruby/object:Gem::Requirement
|
85
85
|
requirements:
|
86
|
-
- - "
|
86
|
+
- - ">"
|
87
87
|
- !ruby/object:Gem::Version
|
88
88
|
version: 3.1.0
|
89
89
|
- !ruby/object:Gem::Dependency
|
@@ -118,23 +118,24 @@ dependencies:
|
|
118
118
|
name: pg
|
119
119
|
requirement: !ruby/object:Gem::Requirement
|
120
120
|
requirements:
|
121
|
-
- - "
|
121
|
+
- - "~>"
|
122
122
|
- !ruby/object:Gem::Version
|
123
|
-
version: '0'
|
123
|
+
version: '1.0'
|
124
124
|
type: :development
|
125
125
|
prerelease: false
|
126
126
|
version_requirements: !ruby/object:Gem::Requirement
|
127
127
|
requirements:
|
128
|
-
- - "
|
128
|
+
- - "~>"
|
129
129
|
- !ruby/object:Gem::Version
|
130
|
-
version: '0'
|
130
|
+
version: '1.0'
|
131
131
|
description: SQL input/output plugin for Fluentd event collector
|
132
132
|
email: frsyuki@gmail.com
|
133
133
|
executables: []
|
134
134
|
extensions: []
|
135
135
|
extra_rdoc_files: []
|
136
136
|
files:
|
137
|
-
- ".
|
137
|
+
- ".github/workflows/linux.yml"
|
138
|
+
- ChangeLog
|
138
139
|
- Gemfile
|
139
140
|
- Gemfile.v0.12
|
140
141
|
- README.md
|
@@ -146,8 +147,9 @@ files:
|
|
146
147
|
- test/fixtures/schema.rb
|
147
148
|
- test/helper.rb
|
148
149
|
- test/plugin/test_in_sql.rb
|
150
|
+
- test/plugin/test_in_sql_with_custom_time.rb
|
149
151
|
- test/plugin/test_out_sql.rb
|
150
|
-
homepage: https://github.com/
|
152
|
+
homepage: https://github.com/fluent/fluent-plugin-sql
|
151
153
|
licenses:
|
152
154
|
- Apache-2.0
|
153
155
|
metadata: {}
|
@@ -162,17 +164,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
162
164
|
version: '0'
|
163
165
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
164
166
|
requirements:
|
165
|
-
- - "
|
167
|
+
- - ">="
|
166
168
|
- !ruby/object:Gem::Version
|
167
|
-
version:
|
169
|
+
version: '0'
|
168
170
|
requirements: []
|
169
|
-
|
170
|
-
rubygems_version: 2.6.13
|
171
|
+
rubygems_version: 3.2.3
|
171
172
|
signing_key:
|
172
173
|
specification_version: 4
|
173
174
|
summary: SQL input/output plugin for Fluentd event collector
|
174
|
-
test_files:
|
175
|
-
- test/fixtures/schema.rb
|
176
|
-
- test/helper.rb
|
177
|
-
- test/plugin/test_in_sql.rb
|
178
|
-
- test/plugin/test_out_sql.rb
|
175
|
+
test_files: []
|
data/.travis.yml
DELETED
@@ -1,32 +0,0 @@
|
|
1
|
-
sudo: false
|
2
|
-
language: ruby
|
3
|
-
|
4
|
-
addons:
|
5
|
-
postgresql: "9.4"
|
6
|
-
|
7
|
-
rvm:
|
8
|
-
- 2.0
|
9
|
-
- 2.1
|
10
|
-
- 2.2.4
|
11
|
-
- 2.3.1
|
12
|
-
- 2.4.1
|
13
|
-
- ruby-head
|
14
|
-
|
15
|
-
gemfile:
|
16
|
-
- Gemfile
|
17
|
-
|
18
|
-
before_install:
|
19
|
-
- gem install bundler
|
20
|
-
|
21
|
-
before_script:
|
22
|
-
- psql -U postgres -c "CREATE ROLE fluentd WITH LOGIN ENCRYPTED PASSWORD 'fluentd';"
|
23
|
-
- psql -U postgres -c "CREATE DATABASE fluentd_test OWNER fluentd;"
|
24
|
-
|
25
|
-
script: bundle exec rake test
|
26
|
-
|
27
|
-
matrix:
|
28
|
-
allow_failures:
|
29
|
-
- rvm: ruby-head
|
30
|
-
exclude:
|
31
|
-
- rvm: 2.0
|
32
|
-
gemfile: Gemfile
|