fluent-plugin-sql 1.1.1 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/linux.yml +69 -0
- data/ChangeLog +28 -0
- data/README.md +10 -6
- data/VERSION +1 -1
- data/fluent-plugin-sql.gemspec +3 -4
- data/lib/fluent/plugin/in_sql.rb +25 -10
- data/lib/fluent/plugin/out_sql.rb +24 -27
- data/test/fixtures/schema.rb +7 -0
- data/test/plugin/test_in_sql.rb +3 -3
- data/test/plugin/test_in_sql_with_custom_time.rb +114 -0
- data/test/plugin/test_out_sql.rb +4 -2
- metadata +13 -16
- data/.travis.yml +0 -28
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 88024ff651d07318ed1d0abf03a0c43601e61288356544c53805a7678e6e5c45
|
4
|
+
data.tar.gz: 020b0a879adfa39a5130105e9c75ed1d3e65574de63624a52f698c91b3e48d48
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c258a8f2b9eb8dbc78cfe1709cfcee97b647d1ab4067bc1e47624dc296ea3196b313b0d7fd2543f37cdbd67598b846507a1ee0f705d1d9e338098bd3977e28e9
|
7
|
+
data.tar.gz: a32904e86a3d30a586e745a60fb48f4e43a78e427a32ea81beba225dba858c7aff9a3afb85e1ce5d7325daea1505a200845d38d330cadd0c47f8e85edab7a26c
|
@@ -0,0 +1,69 @@
|
|
1
|
+
name: linux
|
2
|
+
on:
|
3
|
+
- push
|
4
|
+
- pull_request
|
5
|
+
|
6
|
+
jobs:
|
7
|
+
build:
|
8
|
+
runs-on: ${{ matrix.os }}
|
9
|
+
continue-on-error: ${{ matrix.experimental }}
|
10
|
+
strategy:
|
11
|
+
fail-fast: false
|
12
|
+
matrix:
|
13
|
+
ruby: [ '2.5', '2.6', '2.7', '3.0' ]
|
14
|
+
postgres: [ '9', '10', '11', '12', '13' ]
|
15
|
+
os:
|
16
|
+
- ubuntu-latest
|
17
|
+
experimental: [false]
|
18
|
+
include:
|
19
|
+
- ruby: head
|
20
|
+
postgres: '9'
|
21
|
+
os: ubuntu-latest
|
22
|
+
experimental: true
|
23
|
+
- ruby: head
|
24
|
+
postgres: '10'
|
25
|
+
os: ubuntu-latest
|
26
|
+
experimental: true
|
27
|
+
- ruby: head
|
28
|
+
postgres: '11'
|
29
|
+
os: ubuntu-latest
|
30
|
+
experimental: true
|
31
|
+
- ruby: head
|
32
|
+
postgres: '12'
|
33
|
+
os: ubuntu-latest
|
34
|
+
experimental: true
|
35
|
+
- ruby: head
|
36
|
+
postgres: '13'
|
37
|
+
os: ubuntu-latest
|
38
|
+
experimental: true
|
39
|
+
services:
|
40
|
+
postgres:
|
41
|
+
image: postgres:${{ matrix.postgres }}
|
42
|
+
ports:
|
43
|
+
- 5432:5432
|
44
|
+
env:
|
45
|
+
POSTGRES_USER: postgres
|
46
|
+
POSTGRES_PASSWORD: password
|
47
|
+
POSTGRES_DB: test
|
48
|
+
options: >-
|
49
|
+
--health-cmd pg_isready
|
50
|
+
--health-interval 10s
|
51
|
+
--health-timeout 5s
|
52
|
+
--health-retries 5
|
53
|
+
name: Ruby ${{ matrix.ruby }} with PostgreSQL ${{ matrix.postgres }} unit testing on ${{ matrix.os }}
|
54
|
+
steps:
|
55
|
+
- uses: actions/checkout@v2
|
56
|
+
- uses: ruby/setup-ruby@v1
|
57
|
+
with:
|
58
|
+
ruby-version: ${{ matrix.ruby }}
|
59
|
+
- name: prepare database
|
60
|
+
env:
|
61
|
+
PGPASSWORD: password
|
62
|
+
run: |
|
63
|
+
psql -h localhost -p 5432 -U postgres -c "CREATE ROLE fluentd WITH LOGIN ENCRYPTED PASSWORD 'fluentd';"
|
64
|
+
psql -h localhost -p 5432 -U postgres -c "CREATE DATABASE fluentd_test OWNER fluentd;"
|
65
|
+
- name: unit testing
|
66
|
+
run: |
|
67
|
+
gem install bundler rake
|
68
|
+
bundle install --jobs 4 --retry 3
|
69
|
+
bundle exec rake test
|
data/ChangeLog
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
Release 2.3.0 - 2021/10/08
|
2
|
+
|
3
|
+
* in_sql: Make table time_format configurable
|
4
|
+
|
5
|
+
Release 2.2.0 - 2021/04/20
|
6
|
+
|
7
|
+
* Upgrade ActiveRecord to 6.1
|
8
|
+
|
9
|
+
Release 2.1.0 - 2020/09/09
|
10
|
+
|
11
|
+
* in_sql: Use Fluent::EventTime instead of Integer to preserve millisecond precision
|
12
|
+
|
13
|
+
Release 2.0.0 - 2020/04/22
|
14
|
+
|
15
|
+
* out_sql: Remove v0.12 API code and use full v1 API. Change buffer format.
|
16
|
+
|
17
|
+
Release 1.1.1 - 2019/05/10
|
18
|
+
|
19
|
+
* out_sql: Support schema_search_path option of PostgreSQL
|
20
|
+
|
21
|
+
Release 1.1.0 - 2018/10/04
|
22
|
+
|
23
|
+
* Upgrade ActiveRecord to 5.1
|
24
|
+
|
25
|
+
Release 1.0.0 - 2018/04/06
|
26
|
+
|
27
|
+
* Support v1 API
|
28
|
+
|
data/README.md
CHANGED
@@ -5,7 +5,7 @@
|
|
5
5
|
This SQL plugin has two parts:
|
6
6
|
|
7
7
|
1. SQL **input** plugin reads records from RDBMSes periodically. An example use case would be getting "diffs" of a table (based on the "updated_at" field).
|
8
|
-
2. SQL **output** plugin that writes records into
|
8
|
+
2. SQL **output** plugin that writes records into RDBMSes. An example use case would be aggregating server/app/sensor logs into RDBMS systems.
|
9
9
|
|
10
10
|
## Requirements
|
11
11
|
|
@@ -14,19 +14,18 @@ This SQL plugin has two parts:
|
|
14
14
|
| >= 1.0.0 | >= v0.14.4 | >= 2.1 |
|
15
15
|
| < 1.0.0 | < v0.14.0 | >= 1.9 |
|
16
16
|
|
17
|
-
NOTE: fluent-plugin-sql
|
17
|
+
NOTE: fluent-plugin-sql v2's buffer format is different from v1. If you update the plugin to v2, don't reuse v1's buffer.
|
18
18
|
|
19
19
|
## Installation
|
20
20
|
|
21
21
|
$ fluent-gem install fluent-plugin-sql --no-document
|
22
|
-
$ fluent-gem install pg
|
22
|
+
$ fluent-gem install pg --no-document # for postgresql
|
23
23
|
|
24
24
|
You should install actual RDBMS driver gem together. `pg` gem for postgresql adapter or `mysql2` gem for `mysql2` adapter. Other adapters supported by [ActiveRecord](https://github.com/rails/rails/tree/master/activerecord) should work.
|
25
25
|
|
26
26
|
We recommend that mysql2 gem is higher than `0.3.12` and pg gem is higher than `0.16.0`.
|
27
27
|
|
28
|
-
|
29
|
-
NOTE: ActiveRecord has supported pg 1.0.0 since ActiveRecord 5.1.5.
|
28
|
+
If you use ruby 2.1, use pg gem 0.21.0 (< 1.0.0) because ActiveRecord 5.1.4 or earlier doesn't support Ruby 2.1.
|
30
29
|
|
31
30
|
## Input: How It Works
|
32
31
|
|
@@ -45,6 +44,7 @@ It stores last selected rows to a file (named *state\_file*) to not forget the l
|
|
45
44
|
@type sql
|
46
45
|
|
47
46
|
host rdb_host
|
47
|
+
port rdb_port
|
48
48
|
database rdb_database
|
49
49
|
adapter mysql2_or_postgresql_or_etc
|
50
50
|
username myusername
|
@@ -69,6 +69,7 @@ It stores last selected rows to a file (named *state\_file*) to not forget the l
|
|
69
69
|
tag table2 # optional
|
70
70
|
update_column updated_at
|
71
71
|
time_column updated_at # optional
|
72
|
+
time_format %Y-%m-%d %H:%M:%S.%6N # optional
|
72
73
|
</table>
|
73
74
|
|
74
75
|
# detects all tables instead of <table> sections
|
@@ -94,6 +95,7 @@ It stores last selected rows to a file (named *state\_file*) to not forget the l
|
|
94
95
|
* **update_column**: see above description
|
95
96
|
* **time_column** (optional): if this option is set, this plugin uses this column's value as the the event's time. Otherwise it uses current time.
|
96
97
|
* **primary_key** (optional): if you want to get data from the table which doesn't have primary key like PostgreSQL's View, set this parameter.
|
98
|
+
* **time_format** (optional): if you want to specify the format of the date used in the query, useful when using alternative adapters which have restrictions on format
|
97
99
|
|
98
100
|
## Input: Limitation
|
99
101
|
|
@@ -151,10 +153,12 @@ This plugin takes advantage of ActiveRecord underneath. For `host`, `port`, `dat
|
|
151
153
|
* **username** RDBMS login user name
|
152
154
|
* **password** RDBMS login password
|
153
155
|
* **socket** RDBMS socket path
|
156
|
+
* **pool** A connection pool synchronizes thread access to a limited number of database connections
|
157
|
+
* **timeout** RDBMS connection timeout
|
154
158
|
* **remove_tag_prefix** remove the given prefix from the events. See "tag_prefix" in "Input: Configuration". (optional)
|
155
159
|
|
156
160
|
\<table\> sections:
|
157
161
|
|
158
162
|
* **table** RDBM table name
|
159
163
|
* **column_mapping**: [Required] Record to table schema mapping. The format is consists of `from:to` or `key` values are separated by `,`. For example, if set 'item_id:id,item_text:data,updated_at' to **column_mapping**, `item_id` field of record is stored into `id` column and `updated_at` field of record is stored into `updated_at` column.
|
160
|
-
*
|
164
|
+
* **\<table pattern\>**: the pattern to which the incoming event's tag (after it goes through `remove_tag_prefix`, if given). The patterns should follow the same syntax as [that of \<match\>](https://docs.fluentd.org/configuration/config-file#how-match-patterns-work). **Exactly one \<table\> element must NOT have this parameter so that it becomes the default table to store data**.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
2.3.0
|
data/fluent-plugin-sql.gemspec
CHANGED
@@ -9,7 +9,6 @@ Gem::Specification.new do |gem|
|
|
9
9
|
gem.version = File.read("VERSION").strip
|
10
10
|
gem.authors = ["Sadayuki Furuhashi"]
|
11
11
|
gem.email = "frsyuki@gmail.com"
|
12
|
-
gem.has_rdoc = false
|
13
12
|
#gem.platform = Gem::Platform::RUBY
|
14
13
|
gem.files = `git ls-files`.split("\n")
|
15
14
|
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
@@ -17,9 +16,9 @@ Gem::Specification.new do |gem|
|
|
17
16
|
gem.require_paths = ['lib']
|
18
17
|
gem.license = "Apache-2.0"
|
19
18
|
|
20
|
-
gem.add_dependency "fluentd", [">=
|
21
|
-
gem.add_dependency 'activerecord', "~>
|
22
|
-
gem.add_dependency 'activerecord-import', "~> 0
|
19
|
+
gem.add_dependency "fluentd", [">= 1.7.0", "< 2"]
|
20
|
+
gem.add_dependency 'activerecord', "~> 6.1"
|
21
|
+
gem.add_dependency 'activerecord-import', "~> 1.0"
|
23
22
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
24
23
|
gem.add_development_dependency "test-unit", "> 3.1.0"
|
25
24
|
gem.add_development_dependency "test-unit-rr"
|
data/lib/fluent/plugin/in_sql.rb
CHANGED
@@ -59,22 +59,28 @@ module Fluent::Plugin
|
|
59
59
|
config_param :update_column, :string, default: nil
|
60
60
|
config_param :time_column, :string, default: nil
|
61
61
|
config_param :primary_key, :string, default: nil
|
62
|
+
config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S.%6N%z'
|
63
|
+
attr_reader :log
|
62
64
|
|
63
65
|
def configure(conf)
|
64
66
|
super
|
65
67
|
end
|
66
68
|
|
67
|
-
def init(tag_prefix, base_model, router)
|
69
|
+
def init(tag_prefix, base_model, router, log)
|
68
70
|
@router = router
|
69
71
|
@tag = "#{tag_prefix}.#{@tag}" if tag_prefix
|
72
|
+
@log = log
|
70
73
|
|
71
74
|
# creates a model for this table
|
72
75
|
table_name = @table
|
73
76
|
primary_key = @primary_key
|
77
|
+
time_format = @time_format
|
78
|
+
|
74
79
|
@model = Class.new(base_model) do
|
75
80
|
self.table_name = table_name
|
76
81
|
self.inheritance_column = '_never_use_'
|
77
82
|
self.primary_key = primary_key if primary_key
|
83
|
+
self.const_set(:TIME_FORMAT, time_format)
|
78
84
|
|
79
85
|
#self.include_root_in_json = false
|
80
86
|
|
@@ -83,7 +89,7 @@ module Fluent::Plugin
|
|
83
89
|
if v.respond_to?(:to_msgpack)
|
84
90
|
v
|
85
91
|
elsif v.is_a? Time
|
86
|
-
v.strftime(
|
92
|
+
v.strftime(self.class::TIME_FORMAT)
|
87
93
|
else
|
88
94
|
v.to_s
|
89
95
|
end
|
@@ -108,6 +114,17 @@ module Fluent::Plugin
|
|
108
114
|
end
|
109
115
|
end
|
110
116
|
|
117
|
+
# Make sure we always have a Fluent::EventTime object regardless of what comes in
|
118
|
+
def normalized_time(tv, now)
|
119
|
+
return Fluent::EventTime.from_time(tv) if tv.is_a?(Time)
|
120
|
+
begin
|
121
|
+
Fluent::EventTime.parse(tv.to_s)
|
122
|
+
rescue
|
123
|
+
log.warn "Message contains invalid timestamp, using current time instead (#{now.inspect})"
|
124
|
+
now
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
111
128
|
# emits next records and returns the last record of emitted records
|
112
129
|
def emit_next_records(last_record, limit)
|
113
130
|
relation = @model
|
@@ -123,15 +140,13 @@ module Fluent::Plugin
|
|
123
140
|
relation.each do |obj|
|
124
141
|
record = obj.serializable_hash rescue nil
|
125
142
|
if record
|
126
|
-
|
127
|
-
if tv.
|
128
|
-
|
143
|
+
time =
|
144
|
+
if @time_column && (tv = obj.read_attribute(@time_column))
|
145
|
+
normalized_time(tv, now)
|
129
146
|
else
|
130
|
-
|
147
|
+
now
|
131
148
|
end
|
132
|
-
|
133
|
-
time = now
|
134
|
-
end
|
149
|
+
|
135
150
|
me.add(time, record)
|
136
151
|
last_record = record
|
137
152
|
end
|
@@ -217,7 +232,7 @@ module Fluent::Plugin
|
|
217
232
|
# ignore tables if TableElement#init failed
|
218
233
|
@tables.reject! do |te|
|
219
234
|
begin
|
220
|
-
te.init(@tag_prefix, @base_model, router)
|
235
|
+
te.init(@tag_prefix, @base_model, router, log)
|
221
236
|
log.info "Selecting '#{te.table}' table"
|
222
237
|
false
|
223
238
|
rescue => e
|
@@ -1,11 +1,12 @@
|
|
1
1
|
require "fluent/plugin/output"
|
2
2
|
|
3
|
+
require 'active_record'
|
4
|
+
require 'activerecord-import'
|
5
|
+
|
3
6
|
module Fluent::Plugin
|
4
7
|
class SQLOutput < Output
|
5
8
|
Fluent::Plugin.register_output('sql', self)
|
6
9
|
|
7
|
-
DEFAULT_BUFFER_TYPE = "memory"
|
8
|
-
|
9
10
|
helpers :inject, :compat_parameters, :event_emitter
|
10
11
|
|
11
12
|
desc 'RDBMS host'
|
@@ -28,9 +29,13 @@ module Fluent::Plugin
|
|
28
29
|
config_param :remove_tag_prefix, :string, default: nil
|
29
30
|
desc 'enable fallback'
|
30
31
|
config_param :enable_fallback, :bool, default: true
|
32
|
+
desc "size of ActiveRecord's connection pool"
|
33
|
+
config_param :pool, :integer, default: 5
|
34
|
+
desc "specifies the timeout to establish a new connection to the database before failing"
|
35
|
+
config_param :timeout, :integer, default: 5000
|
31
36
|
|
32
37
|
config_section :buffer do
|
33
|
-
config_set_default
|
38
|
+
config_set_default :chunk_keys, ["tag"]
|
34
39
|
end
|
35
40
|
|
36
41
|
attr_accessor :tables
|
@@ -83,11 +88,12 @@ module Fluent::Plugin
|
|
83
88
|
# @model.column_names
|
84
89
|
end
|
85
90
|
|
86
|
-
def import(chunk)
|
91
|
+
def import(chunk, output)
|
92
|
+
tag = chunk.metadata.tag
|
87
93
|
records = []
|
88
|
-
chunk.msgpack_each { |
|
94
|
+
chunk.msgpack_each { |time, data|
|
89
95
|
begin
|
90
|
-
|
96
|
+
data = output.inject_values_to_record(tag, time, data)
|
91
97
|
records << @model.new(@format_proc.call(data))
|
92
98
|
rescue => e
|
93
99
|
args = {error: e, table: @table, record: Yajl.dump(data)}
|
@@ -102,7 +108,7 @@ module Fluent::Plugin
|
|
102
108
|
@log.warn "Got deterministic error. Fallback to one-by-one import", error: e
|
103
109
|
one_by_one_import(records)
|
104
110
|
else
|
105
|
-
|
111
|
+
@log.warn "Got deterministic error. Fallback is disabled", error: e
|
106
112
|
raise e
|
107
113
|
end
|
108
114
|
end
|
@@ -144,8 +150,6 @@ module Fluent::Plugin
|
|
144
150
|
|
145
151
|
def initialize
|
146
152
|
super
|
147
|
-
require 'active_record'
|
148
|
-
require 'activerecord-import'
|
149
153
|
end
|
150
154
|
|
151
155
|
def configure(conf)
|
@@ -171,7 +175,10 @@ module Fluent::Plugin
|
|
171
175
|
@tables << te
|
172
176
|
end
|
173
177
|
}
|
174
|
-
|
178
|
+
|
179
|
+
if @pool < @buffer_config.flush_thread_count
|
180
|
+
log.warn "connection pool size is smaller than buffer's flush_thread_count. Recommend to increase pool value", :pool => @pool, :flush_thread_count => @buffer_config.flush_thread_count
|
181
|
+
end
|
175
182
|
|
176
183
|
if @default_table.nil?
|
177
184
|
raise Fluent::ConfigError, "There is no default table. <table> is required in sql output"
|
@@ -190,6 +197,8 @@ module Fluent::Plugin
|
|
190
197
|
password: @password,
|
191
198
|
socket: @socket,
|
192
199
|
schema_search_path: @schema_search_path,
|
200
|
+
pool: @pool,
|
201
|
+
timeout: @timeout,
|
193
202
|
}
|
194
203
|
|
195
204
|
@base_model = Class.new(ActiveRecord::Base) do
|
@@ -210,19 +219,6 @@ module Fluent::Plugin
|
|
210
219
|
super
|
211
220
|
end
|
212
221
|
|
213
|
-
def emit(tag, es, chain)
|
214
|
-
if @only_default
|
215
|
-
super(tag, es, chain)
|
216
|
-
else
|
217
|
-
super(tag, es, chain, format_tag(tag))
|
218
|
-
end
|
219
|
-
end
|
220
|
-
|
221
|
-
def format(tag, time, record)
|
222
|
-
record = inject_values_to_record(tag, time, record)
|
223
|
-
[tag, time, record].to_msgpack
|
224
|
-
end
|
225
|
-
|
226
222
|
def formatted_to_msgpack_binary
|
227
223
|
true
|
228
224
|
end
|
@@ -231,11 +227,12 @@ module Fluent::Plugin
|
|
231
227
|
ActiveRecord::Base.connection_pool.with_connection do
|
232
228
|
|
233
229
|
@tables.each { |table|
|
234
|
-
|
235
|
-
|
230
|
+
tag = format_tag(chunk.metadata.tag)
|
231
|
+
if table.pattern.match(tag)
|
232
|
+
return table.import(chunk, self)
|
236
233
|
end
|
237
234
|
}
|
238
|
-
@default_table.import(chunk)
|
235
|
+
@default_table.import(chunk, self)
|
239
236
|
end
|
240
237
|
end
|
241
238
|
|
@@ -254,7 +251,7 @@ module Fluent::Plugin
|
|
254
251
|
end
|
255
252
|
|
256
253
|
def format_tag(tag)
|
257
|
-
if @remove_tag_prefix
|
254
|
+
if tag && @remove_tag_prefix
|
258
255
|
tag.gsub(@remove_tag_prefix, '')
|
259
256
|
else
|
260
257
|
tag
|
data/test/fixtures/schema.rb
CHANGED
@@ -20,5 +20,12 @@ ActiveRecord::Schema.define(version: 20160225030107) do
|
|
20
20
|
t.datetime "created_at", null: false
|
21
21
|
t.datetime "updated_at", null: false
|
22
22
|
end
|
23
|
+
|
24
|
+
create_table "messages_custom_time", force: :cascade do |t|
|
25
|
+
t.string "message"
|
26
|
+
t.datetime "created_at", null: false
|
27
|
+
t.datetime "updated_at", null: false
|
28
|
+
t.string "custom_time"
|
29
|
+
end
|
23
30
|
end
|
24
31
|
|
data/test/plugin/test_in_sql.rb
CHANGED
@@ -82,9 +82,9 @@ class SqlInputTest < Test::Unit::TestCase
|
|
82
82
|
[d.events[2][1], "message 3"],
|
83
83
|
]
|
84
84
|
actual = [
|
85
|
-
[
|
86
|
-
[
|
87
|
-
[
|
85
|
+
[Fluent::EventTime.parse(d.events[0][2]["updated_at"]), d.events[0][2]["message"]],
|
86
|
+
[Fluent::EventTime.parse(d.events[1][2]["updated_at"]), d.events[1][2]["message"]],
|
87
|
+
[Fluent::EventTime.parse(d.events[2][2]["updated_at"]), d.events[2][2]["message"]],
|
88
88
|
]
|
89
89
|
assert_equal(expected, actual)
|
90
90
|
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
require "helper"
|
2
|
+
require "fluent/test/driver/input"
|
3
|
+
|
4
|
+
class SqlInputCustomTimeTest < Test::Unit::TestCase
|
5
|
+
def setup
|
6
|
+
Fluent::Test.setup
|
7
|
+
end
|
8
|
+
|
9
|
+
def teardown
|
10
|
+
end
|
11
|
+
|
12
|
+
CONFIG = %[
|
13
|
+
adapter postgresql
|
14
|
+
host localhost
|
15
|
+
port 5432
|
16
|
+
database fluentd_test
|
17
|
+
|
18
|
+
username fluentd
|
19
|
+
password fluentd
|
20
|
+
|
21
|
+
schema_search_path public
|
22
|
+
|
23
|
+
tag_prefix db
|
24
|
+
|
25
|
+
<table>
|
26
|
+
table messages_custom_time
|
27
|
+
tag logs
|
28
|
+
update_column updated_at
|
29
|
+
time_column custom_time
|
30
|
+
</table>
|
31
|
+
]
|
32
|
+
|
33
|
+
def create_driver(conf = CONFIG)
|
34
|
+
Fluent::Test::Driver::Input.new(Fluent::Plugin::SQLInput).configure(conf)
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_configure
|
38
|
+
d = create_driver
|
39
|
+
expected = {
|
40
|
+
host: "localhost",
|
41
|
+
port: 5432,
|
42
|
+
adapter: "postgresql",
|
43
|
+
database: "fluentd_test",
|
44
|
+
username: "fluentd",
|
45
|
+
password: "fluentd",
|
46
|
+
schema_search_path: "public",
|
47
|
+
tag_prefix: "db"
|
48
|
+
}
|
49
|
+
actual = {
|
50
|
+
host: d.instance.host,
|
51
|
+
port: d.instance.port,
|
52
|
+
adapter: d.instance.adapter,
|
53
|
+
database: d.instance.database,
|
54
|
+
username: d.instance.username,
|
55
|
+
password: d.instance.password,
|
56
|
+
schema_search_path: d.instance.schema_search_path,
|
57
|
+
tag_prefix: d.instance.tag_prefix
|
58
|
+
}
|
59
|
+
assert_equal(expected, actual)
|
60
|
+
tables = d.instance.instance_variable_get(:@tables)
|
61
|
+
assert_equal(1, tables.size)
|
62
|
+
messages_custom_time = tables.first
|
63
|
+
assert_equal("messages_custom_time", messages_custom_time.table)
|
64
|
+
assert_equal("logs", messages_custom_time.tag)
|
65
|
+
end
|
66
|
+
|
67
|
+
def test_message
|
68
|
+
d = create_driver(CONFIG + "select_interval 1")
|
69
|
+
|
70
|
+
start_time = Fluent::EventTime.now
|
71
|
+
|
72
|
+
# Create one message with a valid timestamp containing milliseconds and a time zone
|
73
|
+
Message.create!(message: "message 1", custom_time: '2020-08-27 15:00:16.100758000 -0400')
|
74
|
+
|
75
|
+
# Create one message without a timestamp so that we can test auto-creation
|
76
|
+
Message.create!(message: "message 2 (no timestamp)", custom_time: nil)
|
77
|
+
|
78
|
+
# Create one message with an unparseable timestamp so that we can check that a valid
|
79
|
+
# one is auto-generated.
|
80
|
+
Message.create!(message: "message 3 (bad timestamp)", custom_time: 'foo')
|
81
|
+
|
82
|
+
d.end_if do
|
83
|
+
d.record_count >= 3
|
84
|
+
end
|
85
|
+
d.run(timeout: 5)
|
86
|
+
|
87
|
+
assert_equal("db.logs", d.events[0][0])
|
88
|
+
expected = [
|
89
|
+
[d.events[0][1], "message 1"],
|
90
|
+
[d.events[1][1], "message 2 (no timestamp)"],
|
91
|
+
[d.events[2][1], "message 3 (bad timestamp)"],
|
92
|
+
]
|
93
|
+
|
94
|
+
actual = [
|
95
|
+
[Fluent::EventTime.parse(d.events[0][2]["custom_time"]), d.events[0][2]["message"]],
|
96
|
+
d.events[1][2]["message"],
|
97
|
+
d.events[2][2]["message"],
|
98
|
+
]
|
99
|
+
|
100
|
+
assert_equal(expected[0], actual[0])
|
101
|
+
|
102
|
+
# Messages 2 and 3 should have the same messages but (usually) a slightly later
|
103
|
+
# timestamps because they are generated by the input plugin instead of the test
|
104
|
+
# code
|
105
|
+
[1,2].each do |i|
|
106
|
+
assert_equal(expected[i][1], actual[i])
|
107
|
+
assert_operator(expected[i][0], :>=, start_time)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
class Message < ActiveRecord::Base
|
112
|
+
self.table_name = "messages_custom_time"
|
113
|
+
end
|
114
|
+
end
|
data/test/plugin/test_out_sql.rb
CHANGED
@@ -43,7 +43,8 @@ class SqlOutputTest < Test::Unit::TestCase
|
|
43
43
|
password: "fluentd",
|
44
44
|
schema_search_path: 'public',
|
45
45
|
remove_tag_suffix: /^db/,
|
46
|
-
enable_fallback: true
|
46
|
+
enable_fallback: true,
|
47
|
+
pool: 5
|
47
48
|
}
|
48
49
|
actual = {
|
49
50
|
host: d.instance.host,
|
@@ -54,7 +55,8 @@ class SqlOutputTest < Test::Unit::TestCase
|
|
54
55
|
password: d.instance.password,
|
55
56
|
schema_search_path: d.instance.schema_search_path,
|
56
57
|
remove_tag_suffix: d.instance.remove_tag_prefix,
|
57
|
-
enable_fallback: d.instance.enable_fallback
|
58
|
+
enable_fallback: d.instance.enable_fallback,
|
59
|
+
pool: d.instance.pool
|
58
60
|
}
|
59
61
|
assert_equal(expected, actual)
|
60
62
|
assert_empty(d.instance.tables)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-sql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-10-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -16,7 +16,7 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 1.7.0
|
20
20
|
- - "<"
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: '2'
|
@@ -26,7 +26,7 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version:
|
29
|
+
version: 1.7.0
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2'
|
@@ -36,28 +36,28 @@ dependencies:
|
|
36
36
|
requirements:
|
37
37
|
- - "~>"
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version: '
|
39
|
+
version: '6.1'
|
40
40
|
type: :runtime
|
41
41
|
prerelease: false
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
44
|
- - "~>"
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: '
|
46
|
+
version: '6.1'
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: activerecord-import
|
49
49
|
requirement: !ruby/object:Gem::Requirement
|
50
50
|
requirements:
|
51
51
|
- - "~>"
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: '0
|
53
|
+
version: '1.0'
|
54
54
|
type: :runtime
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
58
|
- - "~>"
|
59
59
|
- !ruby/object:Gem::Version
|
60
|
-
version: '0
|
60
|
+
version: '1.0'
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
62
|
name: rake
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
@@ -134,7 +134,8 @@ executables: []
|
|
134
134
|
extensions: []
|
135
135
|
extra_rdoc_files: []
|
136
136
|
files:
|
137
|
-
- ".
|
137
|
+
- ".github/workflows/linux.yml"
|
138
|
+
- ChangeLog
|
138
139
|
- Gemfile
|
139
140
|
- Gemfile.v0.12
|
140
141
|
- README.md
|
@@ -146,6 +147,7 @@ files:
|
|
146
147
|
- test/fixtures/schema.rb
|
147
148
|
- test/helper.rb
|
148
149
|
- test/plugin/test_in_sql.rb
|
150
|
+
- test/plugin/test_in_sql_with_custom_time.rb
|
149
151
|
- test/plugin/test_out_sql.rb
|
150
152
|
homepage: https://github.com/fluent/fluent-plugin-sql
|
151
153
|
licenses:
|
@@ -166,13 +168,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
166
168
|
- !ruby/object:Gem::Version
|
167
169
|
version: '0'
|
168
170
|
requirements: []
|
169
|
-
|
170
|
-
rubygems_version: 2.6.14.1
|
171
|
+
rubygems_version: 3.2.3
|
171
172
|
signing_key:
|
172
173
|
specification_version: 4
|
173
174
|
summary: SQL input/output plugin for Fluentd event collector
|
174
|
-
test_files:
|
175
|
-
- test/fixtures/schema.rb
|
176
|
-
- test/helper.rb
|
177
|
-
- test/plugin/test_in_sql.rb
|
178
|
-
- test/plugin/test_out_sql.rb
|
175
|
+
test_files: []
|
data/.travis.yml
DELETED
@@ -1,28 +0,0 @@
|
|
1
|
-
sudo: false
|
2
|
-
language: ruby
|
3
|
-
|
4
|
-
addons:
|
5
|
-
postgresql: "9.4"
|
6
|
-
|
7
|
-
rvm:
|
8
|
-
- 2.3.6
|
9
|
-
- 2.4.6
|
10
|
-
- 2.5.5
|
11
|
-
- 2.6.3
|
12
|
-
- ruby-head
|
13
|
-
|
14
|
-
gemfile:
|
15
|
-
- Gemfile
|
16
|
-
|
17
|
-
before_install:
|
18
|
-
- gem update --system
|
19
|
-
|
20
|
-
before_script:
|
21
|
-
- psql -U postgres -c "CREATE ROLE fluentd WITH LOGIN ENCRYPTED PASSWORD 'fluentd';"
|
22
|
-
- psql -U postgres -c "CREATE DATABASE fluentd_test OWNER fluentd;"
|
23
|
-
|
24
|
-
script: bundle exec rake test
|
25
|
-
|
26
|
-
matrix:
|
27
|
-
allow_failures:
|
28
|
-
- rvm: ruby-head
|