fluent-plugin-sql 1.1.1 → 2.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/linux.yml +69 -0
- data/ChangeLog +28 -0
- data/README.md +10 -6
- data/VERSION +1 -1
- data/fluent-plugin-sql.gemspec +3 -4
- data/lib/fluent/plugin/in_sql.rb +25 -10
- data/lib/fluent/plugin/out_sql.rb +24 -27
- data/test/fixtures/schema.rb +7 -0
- data/test/plugin/test_in_sql.rb +3 -3
- data/test/plugin/test_in_sql_with_custom_time.rb +114 -0
- data/test/plugin/test_out_sql.rb +4 -2
- metadata +13 -16
- data/.travis.yml +0 -28
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 88024ff651d07318ed1d0abf03a0c43601e61288356544c53805a7678e6e5c45
|
4
|
+
data.tar.gz: 020b0a879adfa39a5130105e9c75ed1d3e65574de63624a52f698c91b3e48d48
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c258a8f2b9eb8dbc78cfe1709cfcee97b647d1ab4067bc1e47624dc296ea3196b313b0d7fd2543f37cdbd67598b846507a1ee0f705d1d9e338098bd3977e28e9
|
7
|
+
data.tar.gz: a32904e86a3d30a586e745a60fb48f4e43a78e427a32ea81beba225dba858c7aff9a3afb85e1ce5d7325daea1505a200845d38d330cadd0c47f8e85edab7a26c
|
@@ -0,0 +1,69 @@
|
|
1
|
+
name: linux
|
2
|
+
on:
|
3
|
+
- push
|
4
|
+
- pull_request
|
5
|
+
|
6
|
+
jobs:
|
7
|
+
build:
|
8
|
+
runs-on: ${{ matrix.os }}
|
9
|
+
continue-on-error: ${{ matrix.experimental }}
|
10
|
+
strategy:
|
11
|
+
fail-fast: false
|
12
|
+
matrix:
|
13
|
+
ruby: [ '2.5', '2.6', '2.7', '3.0' ]
|
14
|
+
postgres: [ '9', '10', '11', '12', '13' ]
|
15
|
+
os:
|
16
|
+
- ubuntu-latest
|
17
|
+
experimental: [false]
|
18
|
+
include:
|
19
|
+
- ruby: head
|
20
|
+
postgres: '9'
|
21
|
+
os: ubuntu-latest
|
22
|
+
experimental: true
|
23
|
+
- ruby: head
|
24
|
+
postgres: '10'
|
25
|
+
os: ubuntu-latest
|
26
|
+
experimental: true
|
27
|
+
- ruby: head
|
28
|
+
postgres: '11'
|
29
|
+
os: ubuntu-latest
|
30
|
+
experimental: true
|
31
|
+
- ruby: head
|
32
|
+
postgres: '12'
|
33
|
+
os: ubuntu-latest
|
34
|
+
experimental: true
|
35
|
+
- ruby: head
|
36
|
+
postgres: '13'
|
37
|
+
os: ubuntu-latest
|
38
|
+
experimental: true
|
39
|
+
services:
|
40
|
+
postgres:
|
41
|
+
image: postgres:${{ matrix.postgres }}
|
42
|
+
ports:
|
43
|
+
- 5432:5432
|
44
|
+
env:
|
45
|
+
POSTGRES_USER: postgres
|
46
|
+
POSTGRES_PASSWORD: password
|
47
|
+
POSTGRES_DB: test
|
48
|
+
options: >-
|
49
|
+
--health-cmd pg_isready
|
50
|
+
--health-interval 10s
|
51
|
+
--health-timeout 5s
|
52
|
+
--health-retries 5
|
53
|
+
name: Ruby ${{ matrix.ruby }} with PostgreSQL ${{ matrix.postgres }} unit testing on ${{ matrix.os }}
|
54
|
+
steps:
|
55
|
+
- uses: actions/checkout@v2
|
56
|
+
- uses: ruby/setup-ruby@v1
|
57
|
+
with:
|
58
|
+
ruby-version: ${{ matrix.ruby }}
|
59
|
+
- name: prepare database
|
60
|
+
env:
|
61
|
+
PGPASSWORD: password
|
62
|
+
run: |
|
63
|
+
psql -h localhost -p 5432 -U postgres -c "CREATE ROLE fluentd WITH LOGIN ENCRYPTED PASSWORD 'fluentd';"
|
64
|
+
psql -h localhost -p 5432 -U postgres -c "CREATE DATABASE fluentd_test OWNER fluentd;"
|
65
|
+
- name: unit testing
|
66
|
+
run: |
|
67
|
+
gem install bundler rake
|
68
|
+
bundle install --jobs 4 --retry 3
|
69
|
+
bundle exec rake test
|
data/ChangeLog
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
Release 2.3.0 - 2021/10/08
|
2
|
+
|
3
|
+
* in_sql: Make table time_format configurable
|
4
|
+
|
5
|
+
Release 2.2.0 - 2021/04/20
|
6
|
+
|
7
|
+
* Upgrade ActiveRecord to 6.1
|
8
|
+
|
9
|
+
Release 2.1.0 - 2020/09/09
|
10
|
+
|
11
|
+
* in_sql: Use Fluent::EventTime instead of Integer to preserve millisecond precision
|
12
|
+
|
13
|
+
Release 2.0.0 - 2020/04/22
|
14
|
+
|
15
|
+
* out_sql: Remove v0.12 API code and use full v1 API. Change buffer format.
|
16
|
+
|
17
|
+
Release 1.1.1 - 2019/05/10
|
18
|
+
|
19
|
+
* out_sql: Support schema_search_path option of PostgreSQL
|
20
|
+
|
21
|
+
Release 1.1.0 - 2018/10/04
|
22
|
+
|
23
|
+
* Upgrade ActiveRecord to 5.1
|
24
|
+
|
25
|
+
Release 1.0.0 - 2018/04/06
|
26
|
+
|
27
|
+
* Support v1 API
|
28
|
+
|
data/README.md
CHANGED
@@ -5,7 +5,7 @@
|
|
5
5
|
This SQL plugin has two parts:
|
6
6
|
|
7
7
|
1. SQL **input** plugin reads records from RDBMSes periodically. An example use case would be getting "diffs" of a table (based on the "updated_at" field).
|
8
|
-
2. SQL **output** plugin that writes records into
|
8
|
+
2. SQL **output** plugin that writes records into RDBMSes. An example use case would be aggregating server/app/sensor logs into RDBMS systems.
|
9
9
|
|
10
10
|
## Requirements
|
11
11
|
|
@@ -14,19 +14,18 @@ This SQL plugin has two parts:
|
|
14
14
|
| >= 1.0.0 | >= v0.14.4 | >= 2.1 |
|
15
15
|
| < 1.0.0 | < v0.14.0 | >= 1.9 |
|
16
16
|
|
17
|
-
NOTE: fluent-plugin-sql
|
17
|
+
NOTE: fluent-plugin-sql v2's buffer format is different from v1. If you update the plugin to v2, don't reuse v1's buffer.
|
18
18
|
|
19
19
|
## Installation
|
20
20
|
|
21
21
|
$ fluent-gem install fluent-plugin-sql --no-document
|
22
|
-
$ fluent-gem install pg
|
22
|
+
$ fluent-gem install pg --no-document # for postgresql
|
23
23
|
|
24
24
|
You should install actual RDBMS driver gem together. `pg` gem for postgresql adapter or `mysql2` gem for `mysql2` adapter. Other adapters supported by [ActiveRecord](https://github.com/rails/rails/tree/master/activerecord) should work.
|
25
25
|
|
26
26
|
We recommend that mysql2 gem is higher than `0.3.12` and pg gem is higher than `0.16.0`.
|
27
27
|
|
28
|
-
|
29
|
-
NOTE: ActiveRecord has supported pg 1.0.0 since ActiveRecord 5.1.5.
|
28
|
+
If you use ruby 2.1, use pg gem 0.21.0 (< 1.0.0) because ActiveRecord 5.1.4 or earlier doesn't support Ruby 2.1.
|
30
29
|
|
31
30
|
## Input: How It Works
|
32
31
|
|
@@ -45,6 +44,7 @@ It stores last selected rows to a file (named *state\_file*) to not forget the l
|
|
45
44
|
@type sql
|
46
45
|
|
47
46
|
host rdb_host
|
47
|
+
port rdb_port
|
48
48
|
database rdb_database
|
49
49
|
adapter mysql2_or_postgresql_or_etc
|
50
50
|
username myusername
|
@@ -69,6 +69,7 @@ It stores last selected rows to a file (named *state\_file*) to not forget the l
|
|
69
69
|
tag table2 # optional
|
70
70
|
update_column updated_at
|
71
71
|
time_column updated_at # optional
|
72
|
+
time_format %Y-%m-%d %H:%M:%S.%6N # optional
|
72
73
|
</table>
|
73
74
|
|
74
75
|
# detects all tables instead of <table> sections
|
@@ -94,6 +95,7 @@ It stores last selected rows to a file (named *state\_file*) to not forget the l
|
|
94
95
|
* **update_column**: see above description
|
95
96
|
* **time_column** (optional): if this option is set, this plugin uses this column's value as the the event's time. Otherwise it uses current time.
|
96
97
|
* **primary_key** (optional): if you want to get data from the table which doesn't have primary key like PostgreSQL's View, set this parameter.
|
98
|
+
* **time_format** (optional): if you want to specify the format of the date used in the query, useful when using alternative adapters which have restrictions on format
|
97
99
|
|
98
100
|
## Input: Limitation
|
99
101
|
|
@@ -151,10 +153,12 @@ This plugin takes advantage of ActiveRecord underneath. For `host`, `port`, `dat
|
|
151
153
|
* **username** RDBMS login user name
|
152
154
|
* **password** RDBMS login password
|
153
155
|
* **socket** RDBMS socket path
|
156
|
+
* **pool** A connection pool synchronizes thread access to a limited number of database connections
|
157
|
+
* **timeout** RDBMS connection timeout
|
154
158
|
* **remove_tag_prefix** remove the given prefix from the events. See "tag_prefix" in "Input: Configuration". (optional)
|
155
159
|
|
156
160
|
\<table\> sections:
|
157
161
|
|
158
162
|
* **table** RDBM table name
|
159
163
|
* **column_mapping**: [Required] Record to table schema mapping. The format is consists of `from:to` or `key` values are separated by `,`. For example, if set 'item_id:id,item_text:data,updated_at' to **column_mapping**, `item_id` field of record is stored into `id` column and `updated_at` field of record is stored into `updated_at` column.
|
160
|
-
*
|
164
|
+
* **\<table pattern\>**: the pattern to which the incoming event's tag (after it goes through `remove_tag_prefix`, if given). The patterns should follow the same syntax as [that of \<match\>](https://docs.fluentd.org/configuration/config-file#how-match-patterns-work). **Exactly one \<table\> element must NOT have this parameter so that it becomes the default table to store data**.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
2.3.0
|
data/fluent-plugin-sql.gemspec
CHANGED
@@ -9,7 +9,6 @@ Gem::Specification.new do |gem|
|
|
9
9
|
gem.version = File.read("VERSION").strip
|
10
10
|
gem.authors = ["Sadayuki Furuhashi"]
|
11
11
|
gem.email = "frsyuki@gmail.com"
|
12
|
-
gem.has_rdoc = false
|
13
12
|
#gem.platform = Gem::Platform::RUBY
|
14
13
|
gem.files = `git ls-files`.split("\n")
|
15
14
|
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
@@ -17,9 +16,9 @@ Gem::Specification.new do |gem|
|
|
17
16
|
gem.require_paths = ['lib']
|
18
17
|
gem.license = "Apache-2.0"
|
19
18
|
|
20
|
-
gem.add_dependency "fluentd", [">=
|
21
|
-
gem.add_dependency 'activerecord', "~>
|
22
|
-
gem.add_dependency 'activerecord-import', "~> 0
|
19
|
+
gem.add_dependency "fluentd", [">= 1.7.0", "< 2"]
|
20
|
+
gem.add_dependency 'activerecord', "~> 6.1"
|
21
|
+
gem.add_dependency 'activerecord-import', "~> 1.0"
|
23
22
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
24
23
|
gem.add_development_dependency "test-unit", "> 3.1.0"
|
25
24
|
gem.add_development_dependency "test-unit-rr"
|
data/lib/fluent/plugin/in_sql.rb
CHANGED
@@ -59,22 +59,28 @@ module Fluent::Plugin
|
|
59
59
|
config_param :update_column, :string, default: nil
|
60
60
|
config_param :time_column, :string, default: nil
|
61
61
|
config_param :primary_key, :string, default: nil
|
62
|
+
config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S.%6N%z'
|
63
|
+
attr_reader :log
|
62
64
|
|
63
65
|
def configure(conf)
|
64
66
|
super
|
65
67
|
end
|
66
68
|
|
67
|
-
def init(tag_prefix, base_model, router)
|
69
|
+
def init(tag_prefix, base_model, router, log)
|
68
70
|
@router = router
|
69
71
|
@tag = "#{tag_prefix}.#{@tag}" if tag_prefix
|
72
|
+
@log = log
|
70
73
|
|
71
74
|
# creates a model for this table
|
72
75
|
table_name = @table
|
73
76
|
primary_key = @primary_key
|
77
|
+
time_format = @time_format
|
78
|
+
|
74
79
|
@model = Class.new(base_model) do
|
75
80
|
self.table_name = table_name
|
76
81
|
self.inheritance_column = '_never_use_'
|
77
82
|
self.primary_key = primary_key if primary_key
|
83
|
+
self.const_set(:TIME_FORMAT, time_format)
|
78
84
|
|
79
85
|
#self.include_root_in_json = false
|
80
86
|
|
@@ -83,7 +89,7 @@ module Fluent::Plugin
|
|
83
89
|
if v.respond_to?(:to_msgpack)
|
84
90
|
v
|
85
91
|
elsif v.is_a? Time
|
86
|
-
v.strftime(
|
92
|
+
v.strftime(self.class::TIME_FORMAT)
|
87
93
|
else
|
88
94
|
v.to_s
|
89
95
|
end
|
@@ -108,6 +114,17 @@ module Fluent::Plugin
|
|
108
114
|
end
|
109
115
|
end
|
110
116
|
|
117
|
+
# Make sure we always have a Fluent::EventTime object regardless of what comes in
|
118
|
+
def normalized_time(tv, now)
|
119
|
+
return Fluent::EventTime.from_time(tv) if tv.is_a?(Time)
|
120
|
+
begin
|
121
|
+
Fluent::EventTime.parse(tv.to_s)
|
122
|
+
rescue
|
123
|
+
log.warn "Message contains invalid timestamp, using current time instead (#{now.inspect})"
|
124
|
+
now
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
111
128
|
# emits next records and returns the last record of emitted records
|
112
129
|
def emit_next_records(last_record, limit)
|
113
130
|
relation = @model
|
@@ -123,15 +140,13 @@ module Fluent::Plugin
|
|
123
140
|
relation.each do |obj|
|
124
141
|
record = obj.serializable_hash rescue nil
|
125
142
|
if record
|
126
|
-
|
127
|
-
if tv.
|
128
|
-
|
143
|
+
time =
|
144
|
+
if @time_column && (tv = obj.read_attribute(@time_column))
|
145
|
+
normalized_time(tv, now)
|
129
146
|
else
|
130
|
-
|
147
|
+
now
|
131
148
|
end
|
132
|
-
|
133
|
-
time = now
|
134
|
-
end
|
149
|
+
|
135
150
|
me.add(time, record)
|
136
151
|
last_record = record
|
137
152
|
end
|
@@ -217,7 +232,7 @@ module Fluent::Plugin
|
|
217
232
|
# ignore tables if TableElement#init failed
|
218
233
|
@tables.reject! do |te|
|
219
234
|
begin
|
220
|
-
te.init(@tag_prefix, @base_model, router)
|
235
|
+
te.init(@tag_prefix, @base_model, router, log)
|
221
236
|
log.info "Selecting '#{te.table}' table"
|
222
237
|
false
|
223
238
|
rescue => e
|
@@ -1,11 +1,12 @@
|
|
1
1
|
require "fluent/plugin/output"
|
2
2
|
|
3
|
+
require 'active_record'
|
4
|
+
require 'activerecord-import'
|
5
|
+
|
3
6
|
module Fluent::Plugin
|
4
7
|
class SQLOutput < Output
|
5
8
|
Fluent::Plugin.register_output('sql', self)
|
6
9
|
|
7
|
-
DEFAULT_BUFFER_TYPE = "memory"
|
8
|
-
|
9
10
|
helpers :inject, :compat_parameters, :event_emitter
|
10
11
|
|
11
12
|
desc 'RDBMS host'
|
@@ -28,9 +29,13 @@ module Fluent::Plugin
|
|
28
29
|
config_param :remove_tag_prefix, :string, default: nil
|
29
30
|
desc 'enable fallback'
|
30
31
|
config_param :enable_fallback, :bool, default: true
|
32
|
+
desc "size of ActiveRecord's connection pool"
|
33
|
+
config_param :pool, :integer, default: 5
|
34
|
+
desc "specifies the timeout to establish a new connection to the database before failing"
|
35
|
+
config_param :timeout, :integer, default: 5000
|
31
36
|
|
32
37
|
config_section :buffer do
|
33
|
-
config_set_default
|
38
|
+
config_set_default :chunk_keys, ["tag"]
|
34
39
|
end
|
35
40
|
|
36
41
|
attr_accessor :tables
|
@@ -83,11 +88,12 @@ module Fluent::Plugin
|
|
83
88
|
# @model.column_names
|
84
89
|
end
|
85
90
|
|
86
|
-
def import(chunk)
|
91
|
+
def import(chunk, output)
|
92
|
+
tag = chunk.metadata.tag
|
87
93
|
records = []
|
88
|
-
chunk.msgpack_each { |
|
94
|
+
chunk.msgpack_each { |time, data|
|
89
95
|
begin
|
90
|
-
|
96
|
+
data = output.inject_values_to_record(tag, time, data)
|
91
97
|
records << @model.new(@format_proc.call(data))
|
92
98
|
rescue => e
|
93
99
|
args = {error: e, table: @table, record: Yajl.dump(data)}
|
@@ -102,7 +108,7 @@ module Fluent::Plugin
|
|
102
108
|
@log.warn "Got deterministic error. Fallback to one-by-one import", error: e
|
103
109
|
one_by_one_import(records)
|
104
110
|
else
|
105
|
-
|
111
|
+
@log.warn "Got deterministic error. Fallback is disabled", error: e
|
106
112
|
raise e
|
107
113
|
end
|
108
114
|
end
|
@@ -144,8 +150,6 @@ module Fluent::Plugin
|
|
144
150
|
|
145
151
|
def initialize
|
146
152
|
super
|
147
|
-
require 'active_record'
|
148
|
-
require 'activerecord-import'
|
149
153
|
end
|
150
154
|
|
151
155
|
def configure(conf)
|
@@ -171,7 +175,10 @@ module Fluent::Plugin
|
|
171
175
|
@tables << te
|
172
176
|
end
|
173
177
|
}
|
174
|
-
|
178
|
+
|
179
|
+
if @pool < @buffer_config.flush_thread_count
|
180
|
+
log.warn "connection pool size is smaller than buffer's flush_thread_count. Recommend to increase pool value", :pool => @pool, :flush_thread_count => @buffer_config.flush_thread_count
|
181
|
+
end
|
175
182
|
|
176
183
|
if @default_table.nil?
|
177
184
|
raise Fluent::ConfigError, "There is no default table. <table> is required in sql output"
|
@@ -190,6 +197,8 @@ module Fluent::Plugin
|
|
190
197
|
password: @password,
|
191
198
|
socket: @socket,
|
192
199
|
schema_search_path: @schema_search_path,
|
200
|
+
pool: @pool,
|
201
|
+
timeout: @timeout,
|
193
202
|
}
|
194
203
|
|
195
204
|
@base_model = Class.new(ActiveRecord::Base) do
|
@@ -210,19 +219,6 @@ module Fluent::Plugin
|
|
210
219
|
super
|
211
220
|
end
|
212
221
|
|
213
|
-
def emit(tag, es, chain)
|
214
|
-
if @only_default
|
215
|
-
super(tag, es, chain)
|
216
|
-
else
|
217
|
-
super(tag, es, chain, format_tag(tag))
|
218
|
-
end
|
219
|
-
end
|
220
|
-
|
221
|
-
def format(tag, time, record)
|
222
|
-
record = inject_values_to_record(tag, time, record)
|
223
|
-
[tag, time, record].to_msgpack
|
224
|
-
end
|
225
|
-
|
226
222
|
def formatted_to_msgpack_binary
|
227
223
|
true
|
228
224
|
end
|
@@ -231,11 +227,12 @@ module Fluent::Plugin
|
|
231
227
|
ActiveRecord::Base.connection_pool.with_connection do
|
232
228
|
|
233
229
|
@tables.each { |table|
|
234
|
-
|
235
|
-
|
230
|
+
tag = format_tag(chunk.metadata.tag)
|
231
|
+
if table.pattern.match(tag)
|
232
|
+
return table.import(chunk, self)
|
236
233
|
end
|
237
234
|
}
|
238
|
-
@default_table.import(chunk)
|
235
|
+
@default_table.import(chunk, self)
|
239
236
|
end
|
240
237
|
end
|
241
238
|
|
@@ -254,7 +251,7 @@ module Fluent::Plugin
|
|
254
251
|
end
|
255
252
|
|
256
253
|
def format_tag(tag)
|
257
|
-
if @remove_tag_prefix
|
254
|
+
if tag && @remove_tag_prefix
|
258
255
|
tag.gsub(@remove_tag_prefix, '')
|
259
256
|
else
|
260
257
|
tag
|
data/test/fixtures/schema.rb
CHANGED
@@ -20,5 +20,12 @@ ActiveRecord::Schema.define(version: 20160225030107) do
|
|
20
20
|
t.datetime "created_at", null: false
|
21
21
|
t.datetime "updated_at", null: false
|
22
22
|
end
|
23
|
+
|
24
|
+
create_table "messages_custom_time", force: :cascade do |t|
|
25
|
+
t.string "message"
|
26
|
+
t.datetime "created_at", null: false
|
27
|
+
t.datetime "updated_at", null: false
|
28
|
+
t.string "custom_time"
|
29
|
+
end
|
23
30
|
end
|
24
31
|
|
data/test/plugin/test_in_sql.rb
CHANGED
@@ -82,9 +82,9 @@ class SqlInputTest < Test::Unit::TestCase
|
|
82
82
|
[d.events[2][1], "message 3"],
|
83
83
|
]
|
84
84
|
actual = [
|
85
|
-
[
|
86
|
-
[
|
87
|
-
[
|
85
|
+
[Fluent::EventTime.parse(d.events[0][2]["updated_at"]), d.events[0][2]["message"]],
|
86
|
+
[Fluent::EventTime.parse(d.events[1][2]["updated_at"]), d.events[1][2]["message"]],
|
87
|
+
[Fluent::EventTime.parse(d.events[2][2]["updated_at"]), d.events[2][2]["message"]],
|
88
88
|
]
|
89
89
|
assert_equal(expected, actual)
|
90
90
|
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
require "helper"
|
2
|
+
require "fluent/test/driver/input"
|
3
|
+
|
4
|
+
class SqlInputCustomTimeTest < Test::Unit::TestCase
|
5
|
+
def setup
|
6
|
+
Fluent::Test.setup
|
7
|
+
end
|
8
|
+
|
9
|
+
def teardown
|
10
|
+
end
|
11
|
+
|
12
|
+
CONFIG = %[
|
13
|
+
adapter postgresql
|
14
|
+
host localhost
|
15
|
+
port 5432
|
16
|
+
database fluentd_test
|
17
|
+
|
18
|
+
username fluentd
|
19
|
+
password fluentd
|
20
|
+
|
21
|
+
schema_search_path public
|
22
|
+
|
23
|
+
tag_prefix db
|
24
|
+
|
25
|
+
<table>
|
26
|
+
table messages_custom_time
|
27
|
+
tag logs
|
28
|
+
update_column updated_at
|
29
|
+
time_column custom_time
|
30
|
+
</table>
|
31
|
+
]
|
32
|
+
|
33
|
+
def create_driver(conf = CONFIG)
|
34
|
+
Fluent::Test::Driver::Input.new(Fluent::Plugin::SQLInput).configure(conf)
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_configure
|
38
|
+
d = create_driver
|
39
|
+
expected = {
|
40
|
+
host: "localhost",
|
41
|
+
port: 5432,
|
42
|
+
adapter: "postgresql",
|
43
|
+
database: "fluentd_test",
|
44
|
+
username: "fluentd",
|
45
|
+
password: "fluentd",
|
46
|
+
schema_search_path: "public",
|
47
|
+
tag_prefix: "db"
|
48
|
+
}
|
49
|
+
actual = {
|
50
|
+
host: d.instance.host,
|
51
|
+
port: d.instance.port,
|
52
|
+
adapter: d.instance.adapter,
|
53
|
+
database: d.instance.database,
|
54
|
+
username: d.instance.username,
|
55
|
+
password: d.instance.password,
|
56
|
+
schema_search_path: d.instance.schema_search_path,
|
57
|
+
tag_prefix: d.instance.tag_prefix
|
58
|
+
}
|
59
|
+
assert_equal(expected, actual)
|
60
|
+
tables = d.instance.instance_variable_get(:@tables)
|
61
|
+
assert_equal(1, tables.size)
|
62
|
+
messages_custom_time = tables.first
|
63
|
+
assert_equal("messages_custom_time", messages_custom_time.table)
|
64
|
+
assert_equal("logs", messages_custom_time.tag)
|
65
|
+
end
|
66
|
+
|
67
|
+
def test_message
|
68
|
+
d = create_driver(CONFIG + "select_interval 1")
|
69
|
+
|
70
|
+
start_time = Fluent::EventTime.now
|
71
|
+
|
72
|
+
# Create one message with a valid timestamp containing milliseconds and a time zone
|
73
|
+
Message.create!(message: "message 1", custom_time: '2020-08-27 15:00:16.100758000 -0400')
|
74
|
+
|
75
|
+
# Create one message without a timestamp so that we can test auto-creation
|
76
|
+
Message.create!(message: "message 2 (no timestamp)", custom_time: nil)
|
77
|
+
|
78
|
+
# Create one message with an unparseable timestamp so that we can check that a valid
|
79
|
+
# one is auto-generated.
|
80
|
+
Message.create!(message: "message 3 (bad timestamp)", custom_time: 'foo')
|
81
|
+
|
82
|
+
d.end_if do
|
83
|
+
d.record_count >= 3
|
84
|
+
end
|
85
|
+
d.run(timeout: 5)
|
86
|
+
|
87
|
+
assert_equal("db.logs", d.events[0][0])
|
88
|
+
expected = [
|
89
|
+
[d.events[0][1], "message 1"],
|
90
|
+
[d.events[1][1], "message 2 (no timestamp)"],
|
91
|
+
[d.events[2][1], "message 3 (bad timestamp)"],
|
92
|
+
]
|
93
|
+
|
94
|
+
actual = [
|
95
|
+
[Fluent::EventTime.parse(d.events[0][2]["custom_time"]), d.events[0][2]["message"]],
|
96
|
+
d.events[1][2]["message"],
|
97
|
+
d.events[2][2]["message"],
|
98
|
+
]
|
99
|
+
|
100
|
+
assert_equal(expected[0], actual[0])
|
101
|
+
|
102
|
+
# Messages 2 and 3 should have the same messages but (usually) a slightly later
|
103
|
+
# timestamps because they are generated by the input plugin instead of the test
|
104
|
+
# code
|
105
|
+
[1,2].each do |i|
|
106
|
+
assert_equal(expected[i][1], actual[i])
|
107
|
+
assert_operator(expected[i][0], :>=, start_time)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
class Message < ActiveRecord::Base
|
112
|
+
self.table_name = "messages_custom_time"
|
113
|
+
end
|
114
|
+
end
|
data/test/plugin/test_out_sql.rb
CHANGED
@@ -43,7 +43,8 @@ class SqlOutputTest < Test::Unit::TestCase
|
|
43
43
|
password: "fluentd",
|
44
44
|
schema_search_path: 'public',
|
45
45
|
remove_tag_suffix: /^db/,
|
46
|
-
enable_fallback: true
|
46
|
+
enable_fallback: true,
|
47
|
+
pool: 5
|
47
48
|
}
|
48
49
|
actual = {
|
49
50
|
host: d.instance.host,
|
@@ -54,7 +55,8 @@ class SqlOutputTest < Test::Unit::TestCase
|
|
54
55
|
password: d.instance.password,
|
55
56
|
schema_search_path: d.instance.schema_search_path,
|
56
57
|
remove_tag_suffix: d.instance.remove_tag_prefix,
|
57
|
-
enable_fallback: d.instance.enable_fallback
|
58
|
+
enable_fallback: d.instance.enable_fallback,
|
59
|
+
pool: d.instance.pool
|
58
60
|
}
|
59
61
|
assert_equal(expected, actual)
|
60
62
|
assert_empty(d.instance.tables)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-sql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-10-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -16,7 +16,7 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 1.7.0
|
20
20
|
- - "<"
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: '2'
|
@@ -26,7 +26,7 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version:
|
29
|
+
version: 1.7.0
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2'
|
@@ -36,28 +36,28 @@ dependencies:
|
|
36
36
|
requirements:
|
37
37
|
- - "~>"
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version: '
|
39
|
+
version: '6.1'
|
40
40
|
type: :runtime
|
41
41
|
prerelease: false
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
44
|
- - "~>"
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: '
|
46
|
+
version: '6.1'
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: activerecord-import
|
49
49
|
requirement: !ruby/object:Gem::Requirement
|
50
50
|
requirements:
|
51
51
|
- - "~>"
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: '0
|
53
|
+
version: '1.0'
|
54
54
|
type: :runtime
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
58
|
- - "~>"
|
59
59
|
- !ruby/object:Gem::Version
|
60
|
-
version: '0
|
60
|
+
version: '1.0'
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
62
|
name: rake
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
@@ -134,7 +134,8 @@ executables: []
|
|
134
134
|
extensions: []
|
135
135
|
extra_rdoc_files: []
|
136
136
|
files:
|
137
|
-
- ".
|
137
|
+
- ".github/workflows/linux.yml"
|
138
|
+
- ChangeLog
|
138
139
|
- Gemfile
|
139
140
|
- Gemfile.v0.12
|
140
141
|
- README.md
|
@@ -146,6 +147,7 @@ files:
|
|
146
147
|
- test/fixtures/schema.rb
|
147
148
|
- test/helper.rb
|
148
149
|
- test/plugin/test_in_sql.rb
|
150
|
+
- test/plugin/test_in_sql_with_custom_time.rb
|
149
151
|
- test/plugin/test_out_sql.rb
|
150
152
|
homepage: https://github.com/fluent/fluent-plugin-sql
|
151
153
|
licenses:
|
@@ -166,13 +168,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
166
168
|
- !ruby/object:Gem::Version
|
167
169
|
version: '0'
|
168
170
|
requirements: []
|
169
|
-
|
170
|
-
rubygems_version: 2.6.14.1
|
171
|
+
rubygems_version: 3.2.3
|
171
172
|
signing_key:
|
172
173
|
specification_version: 4
|
173
174
|
summary: SQL input/output plugin for Fluentd event collector
|
174
|
-
test_files:
|
175
|
-
- test/fixtures/schema.rb
|
176
|
-
- test/helper.rb
|
177
|
-
- test/plugin/test_in_sql.rb
|
178
|
-
- test/plugin/test_out_sql.rb
|
175
|
+
test_files: []
|
data/.travis.yml
DELETED
@@ -1,28 +0,0 @@
|
|
1
|
-
sudo: false
|
2
|
-
language: ruby
|
3
|
-
|
4
|
-
addons:
|
5
|
-
postgresql: "9.4"
|
6
|
-
|
7
|
-
rvm:
|
8
|
-
- 2.3.6
|
9
|
-
- 2.4.6
|
10
|
-
- 2.5.5
|
11
|
-
- 2.6.3
|
12
|
-
- ruby-head
|
13
|
-
|
14
|
-
gemfile:
|
15
|
-
- Gemfile
|
16
|
-
|
17
|
-
before_install:
|
18
|
-
- gem update --system
|
19
|
-
|
20
|
-
before_script:
|
21
|
-
- psql -U postgres -c "CREATE ROLE fluentd WITH LOGIN ENCRYPTED PASSWORD 'fluentd';"
|
22
|
-
- psql -U postgres -c "CREATE DATABASE fluentd_test OWNER fluentd;"
|
23
|
-
|
24
|
-
script: bundle exec rake test
|
25
|
-
|
26
|
-
matrix:
|
27
|
-
allow_failures:
|
28
|
-
- rvm: ruby-head
|