fluent-plugin-mysql-select-insert 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +5 -3
- data/LICENSE +1 -1
- data/README.md +1 -0
- data/fluent-plugin-mysql-select-insert.gemspec +3 -3
- data/lib/fluent/plugin/out_mysql_select_insert.rb +23 -8
- data/test/plugin/test_out_mysql_select_insert.rb +97 -25
- metadata +13 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0f67d87a1bfad110f3c66632eacc9768b2203457209447ff081c07561c5e6356
|
4
|
+
data.tar.gz: 3634f61c95855e08e2e1742b6f1b154a916c15825ec3ea45eb4da1de324a7e09
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 66341a37c23a9ce9f05948d30b17ef90ebfcfc44f30a23d13d177b3e859b34a82f8af632766986995f45f2a6f8397c64277038e68ded1b8f903ca4832b9a9237
|
7
|
+
data.tar.gz: 2e5695b27761643f506b048e485468c5956a46b6dbd51b258e5b72fd20c1f20ad1b41b070ae314370c64746c5ce43f505f26c2b9dfbe9e7fcc69685f0696508d
|
data/.travis.yml
CHANGED
data/LICENSE
CHANGED
@@ -187,7 +187,7 @@
|
|
187
187
|
same "printed page" as the copyright notice for easier
|
188
188
|
identification within third-party archives.
|
189
189
|
|
190
|
-
Copyright
|
190
|
+
Copyright 2018-2019 Takeshi Arabiki (abicky)
|
191
191
|
|
192
192
|
Licensed under the Apache License, Version 2.0 (the "License");
|
193
193
|
you may not use this file except in compliance with the License.
|
data/README.md
CHANGED
@@ -39,6 +39,7 @@ select_query | SELECT query without WHERE clause to insert records | (required)
|
|
39
39
|
condition_column | Column used in WHERE clause to compare with values of 'condition_key' in events | (required)
|
40
40
|
condition_key | Key whose value is used in WHERE clause to compare with 'condition_column' | (required)
|
41
41
|
extra_condition | Extra condition used in WHERE clause. You can specify placeholders like 'col1 = ? AND col2 = ?, ${tag[0]}, ${tag[1]}'. The metadata is extracted in the second or following arguments. | (empty array)
|
42
|
+
inserted_columns | Columns to be inserted. You should insert all columns by the select query if you don't specify the value. | nil
|
42
43
|
ignore | Add 'IGNORE' modifier to INSERT statement | false
|
43
44
|
|
44
45
|
## Example Configuration
|
@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |spec|
|
5
5
|
spec.name = "fluent-plugin-mysql-select-insert"
|
6
|
-
spec.version = "0.1.
|
6
|
+
spec.version = "0.1.1"
|
7
7
|
spec.authors = ["abicky"]
|
8
8
|
spec.email = ["takeshi.arabiki@gmail.com"]
|
9
9
|
|
@@ -20,9 +20,9 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.test_files = test_files
|
21
21
|
spec.require_paths = ["lib"]
|
22
22
|
|
23
|
-
spec.add_development_dependency "bundler", "
|
23
|
+
spec.add_development_dependency "bundler", ">= 1.16", "< 3"
|
24
24
|
spec.add_development_dependency "rake", "~> 12.0"
|
25
25
|
spec.add_development_dependency "test-unit", "~> 3.0"
|
26
|
-
spec.add_runtime_dependency "fluentd", [">=
|
26
|
+
spec.add_runtime_dependency "fluentd", [">= 1.2.0", "< 2"]
|
27
27
|
spec.add_runtime_dependency "mysql2-cs-bind"
|
28
28
|
end
|
@@ -42,6 +42,8 @@ module Fluent
|
|
42
42
|
desc: "Key whose value is used in WHERE clause to compare with 'condition_column'"
|
43
43
|
config_param :extra_condition, :array, value_type: :string, default: [],
|
44
44
|
desc: "Extra condition used in WHERE clause. You can specify placeholders like 'col1 = ? AND col2 = ?, ${tag[0]}, ${tag[1]}'. The metadata is extracted in the second or following arguments."
|
45
|
+
config_param :inserted_columns, :array, value_type: :string, default: nil,
|
46
|
+
desc: "Columns to be inserted. You should insert all columns by the select query if you don't specify the value."
|
45
47
|
|
46
48
|
config_param :ignore, :bool, default: false,
|
47
49
|
desc: "Add 'IGNORE' modifier to INSERT statement"
|
@@ -56,6 +58,9 @@ module Fluent
|
|
56
58
|
if select_query =~ /\bwhere\b/i
|
57
59
|
fail Fluent::ConfigError, "You can't specify WHERE clause in 'select_query'"
|
58
60
|
end
|
61
|
+
if select_query !~ /\A\s*select\b/i
|
62
|
+
fail Fluent::ConfigError, "'select_query' should begin with 'SELECT'"
|
63
|
+
end
|
59
64
|
end
|
60
65
|
|
61
66
|
def write(chunk)
|
@@ -64,19 +69,29 @@ module Fluent
|
|
64
69
|
chunk.msgpack_each do |time, record|
|
65
70
|
condition_values << "'#{client.escape(record[condition_key])}'"
|
66
71
|
end
|
72
|
+
|
67
73
|
sql = <<~SQL
|
68
|
-
INSERT #{"IGNORE" if ignore} INTO `#{table}` #{
|
74
|
+
INSERT #{"IGNORE" if ignore} INTO `#{table}` #{"(#{inserted_columns.join(",")})" if inserted_columns}
|
75
|
+
#{select_query}
|
69
76
|
WHERE #{condition_column} IN (#{condition_values.join(",")})
|
70
77
|
SQL
|
71
|
-
|
72
78
|
sql << " AND (#{extra_condition[0]})" unless extra_condition.empty?
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
+
|
80
|
+
bound_params = extra_condition[1..-1]&.map { |c| extract_placeholders(c, chunk.metadata) }
|
81
|
+
begin
|
82
|
+
if bound_params.nil? || bound_params.empty?
|
83
|
+
client.query(sql)
|
84
|
+
else
|
85
|
+
require "mysql2-cs-bind"
|
86
|
+
client.xquery(sql, bound_params)
|
87
|
+
end
|
88
|
+
rescue Mysql2::Error => e
|
89
|
+
if e.message.start_with?("Column count doesn't match value count")
|
90
|
+
raise Fluent::UnrecoverableError, "#{e.class}: #{e}"
|
91
|
+
end
|
92
|
+
raise
|
79
93
|
end
|
94
|
+
|
80
95
|
client.close
|
81
96
|
end
|
82
97
|
|
@@ -4,66 +4,81 @@ require "fluent/plugin/out_mysql_select_insert"
|
|
4
4
|
class MysqlSelectInsertOutputTest < Test::Unit::TestCase
|
5
5
|
class << self
|
6
6
|
def startup
|
7
|
-
new_client
|
7
|
+
client = new_client
|
8
|
+
client.query(<<~SQL)
|
8
9
|
CREATE TABLE IF NOT EXISTS `devices` (
|
9
10
|
`id` int(10) unsigned NOT NULL,
|
10
11
|
`app_id` int(10) unsigned NOT NULL,
|
11
12
|
`uuid` char(36) NOT NULL,
|
12
13
|
PRIMARY KEY (`id`),
|
13
14
|
UNIQUE KEY (`app_id`, `uuid`)
|
14
|
-
)
|
15
|
+
)
|
16
|
+
SQL
|
17
|
+
client.query(<<~SQL)
|
15
18
|
CREATE TABLE IF NOT EXISTS `users` (
|
16
19
|
`id` int(10) unsigned NOT NULL,
|
17
20
|
`device_id` int(10) unsigned NOT NULL,
|
18
21
|
PRIMARY KEY (`id`)
|
19
|
-
)
|
22
|
+
)
|
23
|
+
SQL
|
24
|
+
client.query(<<~SQL)
|
20
25
|
CREATE TABLE IF NOT EXISTS `accessed_users` (
|
21
26
|
`user_id` int(10) unsigned NOT NULL,
|
22
27
|
PRIMARY KEY (`user_id`)
|
23
|
-
)
|
28
|
+
)
|
24
29
|
SQL
|
25
30
|
end
|
26
31
|
|
27
32
|
def shutdown
|
28
|
-
new_client
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
SQL
|
33
|
+
client = new_client
|
34
|
+
client.query("DROP TABLE IF EXISTS `devices`")
|
35
|
+
client.query("DROP TABLE IF EXISTS `users`")
|
36
|
+
client.query("DROP TABLE IF EXISTS `accessed_users`")
|
33
37
|
end
|
34
38
|
|
35
39
|
def new_client
|
36
40
|
Mysql2::Client.new(
|
37
41
|
username: "root",
|
38
42
|
database: "fluent_plugin_mysql_select_insert",
|
39
|
-
flags: Mysql2::Client::MULTI_STATEMENTS,
|
40
43
|
)
|
41
44
|
end
|
42
45
|
end
|
43
46
|
|
47
|
+
def suppress_errors
|
48
|
+
prev_value = Thread.report_on_exception
|
49
|
+
# Supporess errors from flush treads
|
50
|
+
Thread.report_on_exception = false
|
51
|
+
# Suppress errors like "unexpected error while after_shutdown"
|
52
|
+
capture_stdout { yield }
|
53
|
+
ensure
|
54
|
+
Thread.report_on_exception = prev_value
|
55
|
+
end
|
56
|
+
|
44
57
|
setup do
|
45
58
|
Fluent::Test.setup
|
46
59
|
|
47
|
-
self.class.new_client
|
60
|
+
client = self.class.new_client
|
61
|
+
client.query(<<~SQL)
|
48
62
|
INSERT INTO `devices` (`id`, `app_id`, `uuid`) VALUES
|
49
63
|
(1, 1, '03449258-29ce-403c-900a-a2c6ea1d09a2'),
|
50
64
|
(2, 1, '11aebc82-b661-44ab-951d-d1618058699a'),
|
51
65
|
(3, 2, '2c9a7bb7-aec2-441c-ade0-edf31fdacb43'),
|
52
|
-
(4, 2, '3eae0b4b-bdd1-4c82-b04a-5335535f5b7b')
|
66
|
+
(4, 2, '3eae0b4b-bdd1-4c82-b04a-5335535f5b7b')
|
67
|
+
SQL
|
68
|
+
client.query(<<~SQL)
|
53
69
|
INSERT INTO `users` (`id`, `device_id`) VALUES
|
54
70
|
(1001, 1),
|
55
71
|
(1002, 2),
|
56
72
|
(1003, 3),
|
57
|
-
(1004, 4)
|
73
|
+
(1004, 4)
|
58
74
|
SQL
|
59
75
|
end
|
60
76
|
|
61
77
|
teardown do
|
62
|
-
self.class.new_client
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
SQL
|
78
|
+
client = self.class.new_client
|
79
|
+
client.query("TRUNCATE TABLE `devices`")
|
80
|
+
client.query("TRUNCATE TABLE `users`")
|
81
|
+
client.query("TRUNCATE TABLE `accessed_users`")
|
67
82
|
end
|
68
83
|
|
69
84
|
CONFIG = <<~CONF
|
@@ -196,22 +211,22 @@ class MysqlSelectInsertOutputTest < Test::Unit::TestCase
|
|
196
211
|
condition_column devices.uuid
|
197
212
|
CONF
|
198
213
|
|
199
|
-
|
200
|
-
# XXX: "Mysql2::Error: Table 'fluent_plugin_mysql_select_insert.accessed_users' doesn't exist" occurs without sleep
|
201
|
-
sleep 1
|
214
|
+
setup do
|
202
215
|
self.class.new_client.query("INSERT INTO `accessed_users` VALUES (1001)")
|
216
|
+
end
|
203
217
|
|
218
|
+
test "write with ignore false" do
|
204
219
|
d = create_driver("#{base_config}\n ignore false")
|
205
220
|
assert_raise Mysql2::Error do
|
206
|
-
|
207
|
-
d.
|
221
|
+
suppress_errors do
|
222
|
+
d.run(default_tag: "tag") do
|
223
|
+
d.feed(event_time, { "uuid" => "03449258-29ce-403c-900a-a2c6ea1d09a2", "app_id" => 1 })
|
224
|
+
end
|
208
225
|
end
|
209
226
|
end
|
210
227
|
end
|
211
228
|
|
212
229
|
test "write with ignore true" do
|
213
|
-
self.class.new_client.query("INSERT INTO `accessed_users` VALUES (1001)")
|
214
|
-
|
215
230
|
d = create_driver("#{base_config}\n ignore true")
|
216
231
|
assert_nothing_raised do
|
217
232
|
d.run(default_tag: "tag") do
|
@@ -221,6 +236,36 @@ class MysqlSelectInsertOutputTest < Test::Unit::TestCase
|
|
221
236
|
end
|
222
237
|
end
|
223
238
|
|
239
|
+
sub_test_case "'inserted_columns' is specified" do
|
240
|
+
setup do
|
241
|
+
self.class.new_client.query("ALTER TABLE `accessed_users` ADD `extra_column` VARCHAR(63) DEFAULT 'default'")
|
242
|
+
end
|
243
|
+
|
244
|
+
teardown do
|
245
|
+
self.class.new_client.query("ALTER TABLE `accessed_users` DROP `extra_column`")
|
246
|
+
end
|
247
|
+
|
248
|
+
test "write with inserted_columns" do
|
249
|
+
d = create_driver("#{CONFIG}\n inserted_columns 'user_id'")
|
250
|
+
d.run(default_tag: "tag") do
|
251
|
+
d.feed(event_time, { "uuid" => "03449258-29ce-403c-900a-a2c6ea1d09a2", "app_id" => 1 })
|
252
|
+
end
|
253
|
+
result = self.class.new_client.query('SELECT * FROM `accessed_users`')
|
254
|
+
assert_equal [{ "user_id" => 1001, "extra_column" => "default" }], result.to_a
|
255
|
+
end
|
256
|
+
|
257
|
+
test "write with invalid inserted_columns" do
|
258
|
+
d = create_driver("#{CONFIG}\n inserted_columns 'user_id, extra_column'")
|
259
|
+
|
260
|
+
d.run(default_tag: "tag") do
|
261
|
+
d.feed(event_time, { "uuid" => "03449258-29ce-403c-900a-a2c6ea1d09a2", "app_id" => 1 })
|
262
|
+
end
|
263
|
+
assert do
|
264
|
+
d.logs.any? { |l| l.include?("got unrecoverable error") && l.include?("Column count doesn't match") }
|
265
|
+
end
|
266
|
+
end
|
267
|
+
end
|
268
|
+
|
224
269
|
sub_test_case "WHERE clause is specified in 'select_query'" do
|
225
270
|
test "configure" do
|
226
271
|
assert_raise Fluent::ConfigError do
|
@@ -248,6 +293,33 @@ class MysqlSelectInsertOutputTest < Test::Unit::TestCase
|
|
248
293
|
end
|
249
294
|
end
|
250
295
|
|
296
|
+
sub_test_case "'select_query' doesn't begin with 'SELECT'" do
|
297
|
+
test "configure" do
|
298
|
+
assert_raise Fluent::ConfigError do
|
299
|
+
create_driver(<<~CONF)
|
300
|
+
database fluent_plugin_mysql_select_insert
|
301
|
+
username root
|
302
|
+
|
303
|
+
table accessed_users
|
304
|
+
|
305
|
+
select_query "(id) SELECT
|
306
|
+
users.id
|
307
|
+
FROM
|
308
|
+
users
|
309
|
+
INNER JOIN
|
310
|
+
devices
|
311
|
+
ON
|
312
|
+
devices.id = users.device_id
|
313
|
+
WHERE
|
314
|
+
app_id = 1"
|
315
|
+
|
316
|
+
condition_key uuid
|
317
|
+
condition_column devices.uuid
|
318
|
+
CONF
|
319
|
+
end
|
320
|
+
end
|
321
|
+
end
|
322
|
+
|
251
323
|
private
|
252
324
|
|
253
325
|
def create_driver(conf = CONFIG)
|
metadata
CHANGED
@@ -1,29 +1,35 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-mysql-select-insert
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- abicky
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-09-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '1.16'
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '3'
|
20
23
|
type: :development
|
21
24
|
prerelease: false
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
23
26
|
requirements:
|
24
|
-
- - "
|
27
|
+
- - ">="
|
25
28
|
- !ruby/object:Gem::Version
|
26
29
|
version: '1.16'
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '3'
|
27
33
|
- !ruby/object:Gem::Dependency
|
28
34
|
name: rake
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
@@ -58,7 +64,7 @@ dependencies:
|
|
58
64
|
requirements:
|
59
65
|
- - ">="
|
60
66
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
67
|
+
version: 1.2.0
|
62
68
|
- - "<"
|
63
69
|
- !ruby/object:Gem::Version
|
64
70
|
version: '2'
|
@@ -68,7 +74,7 @@ dependencies:
|
|
68
74
|
requirements:
|
69
75
|
- - ">="
|
70
76
|
- !ruby/object:Gem::Version
|
71
|
-
version:
|
77
|
+
version: 1.2.0
|
72
78
|
- - "<"
|
73
79
|
- !ruby/object:Gem::Version
|
74
80
|
version: '2'
|
@@ -121,8 +127,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
121
127
|
- !ruby/object:Gem::Version
|
122
128
|
version: '0'
|
123
129
|
requirements: []
|
124
|
-
|
125
|
-
rubygems_version: 2.7.3
|
130
|
+
rubygems_version: 3.0.3
|
126
131
|
signing_key:
|
127
132
|
specification_version: 4
|
128
133
|
summary: Fluentd output plugin to insert records by SELECT query.
|