fluent-plugin-mysql 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +6 -0
- data/fluent-plugin-mysql.gemspec +1 -1
- data/lib/fluent/plugin/out_mysql.rb +14 -1
- data/lib/fluent/plugin/out_mysql_bulk.rb +32 -11
- data/test/plugin/test_out_mysql_bulk.rb +6 -0
- metadata +3 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 41820afbe7391da94a6778133b8bee6b2645564e
|
4
|
+
data.tar.gz: 7c77c42bc38bcbf7c698c19f970e6dda05f1161c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: eafdc4d3b3021330646df1520aa81904d4ae3510d0bd9052ea564142ccfe7f3e313154a5b22de8ffca281de4381037260df01c4cf41e4461bd2a7fc4c5189990
|
7
|
+
data.tar.gz: 2f70d79a00930d74518cce00e16df8bacea6e2c119b16344ff2f6f84a699b49377ce1451c819e27933cf81bdc917ed2f6b41b0fdb7988240fba273748663d581
|
data/README.md
CHANGED
@@ -19,6 +19,12 @@ port|database port(default: 3306)
|
|
19
19
|
database|database name(require)
|
20
20
|
username|user(require)
|
21
21
|
password|password(default: blank)
|
22
|
+
sslkey|path to client key(default: nil)
|
23
|
+
sslcert|path to client cert(default: nil)
|
24
|
+
sslca|path to ca cert(default: nil)
|
25
|
+
sslcapath|path to ca certs(default: nil)
|
26
|
+
sslcipher|ssl cipher(default: nil)
|
27
|
+
sslverify|verify server certificate(default: nil)
|
22
28
|
column_names|bulk insert column (require)
|
23
29
|
key_names|value key names, ${time} is placeholder Time.at(time).strftime("%Y-%m-%d %H:%M:%S") (default : column_names)
|
24
30
|
json_key_names|Key names which store data as json, comma separator.
|
data/fluent-plugin-mysql.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
Gem::Specification.new do |gem|
|
3
3
|
gem.name = "fluent-plugin-mysql"
|
4
|
-
gem.version = "0.
|
4
|
+
gem.version = "0.3.0"
|
5
5
|
gem.authors = ["TAGOMORI Satoshi", "Toyama Hiroshi"]
|
6
6
|
gem.email = ["tagomoris@gmail.com", "toyama0919@gmail.com"]
|
7
7
|
gem.description = %q{fluent plugin to insert mysql as json(single column) or insert statement}
|
@@ -9,6 +9,12 @@ class Fluent::MysqlOutput < Fluent::BufferedOutput
|
|
9
9
|
config_param :database, :string
|
10
10
|
config_param :username, :string
|
11
11
|
config_param :password, :string, :default => '', :secret => true
|
12
|
+
config_param :sslkey, :string, :default => nil
|
13
|
+
config_param :sslcert, :string, :default => nil
|
14
|
+
config_param :sslca, :string, :default => nil
|
15
|
+
config_param :sslcapath, :string, :default => nil
|
16
|
+
config_param :sslcipher, :string, :default => nil
|
17
|
+
config_param :sslverify, :bool, :default => nil
|
12
18
|
|
13
19
|
config_param :key_names, :string, :default => nil # nil allowed for json format
|
14
20
|
config_param :sql, :string, :default => nil
|
@@ -99,7 +105,14 @@ class Fluent::MysqlOutput < Fluent::BufferedOutput
|
|
99
105
|
Mysql2::Client.new({
|
100
106
|
:host => @host, :port => @port,
|
101
107
|
:username => @username, :password => @password,
|
102
|
-
:database => @database,
|
108
|
+
:database => @database,
|
109
|
+
:sslkey => @sslkey,
|
110
|
+
:sslcert => @sslcert,
|
111
|
+
:sslca => @sslca,
|
112
|
+
:sslcapath => @sslcapath,
|
113
|
+
:sslcipher => @sslcipher,
|
114
|
+
:sslverify => @sslverify,
|
115
|
+
:flags => Mysql2::Client::MULTI_STATEMENTS,
|
103
116
|
})
|
104
117
|
end
|
105
118
|
|
@@ -1,4 +1,3 @@
|
|
1
|
-
# -*- encoding : utf-8 -*-
|
2
1
|
require 'fluent/plugin/output'
|
3
2
|
|
4
3
|
module Fluent::Plugin
|
@@ -17,6 +16,18 @@ module Fluent::Plugin
|
|
17
16
|
desc: "Database user."
|
18
17
|
config_param :password, :string, default: '', secret: true,
|
19
18
|
desc: "Database password."
|
19
|
+
config_param :sslkey, :string, default: nil,
|
20
|
+
desc: "SSL key."
|
21
|
+
config_param :sslcert, :string, default: nil,
|
22
|
+
desc: "SSL cert."
|
23
|
+
config_param :sslca, :string, default: nil,
|
24
|
+
desc: "SSL CA."
|
25
|
+
config_param :sslcapath, :string, default: nil,
|
26
|
+
desc: "SSL CA path."
|
27
|
+
config_param :sslcipher, :string, default: nil,
|
28
|
+
desc: "SSL cipher."
|
29
|
+
config_param :sslverify, :bool, default: nil,
|
30
|
+
desc: "SSL Verify Server Certificate."
|
20
31
|
|
21
32
|
config_param :column_names, :string,
|
22
33
|
desc: "Bulk insert column."
|
@@ -29,6 +40,9 @@ DESC
|
|
29
40
|
config_param :table, :string,
|
30
41
|
desc: "Bulk insert table."
|
31
42
|
|
43
|
+
config_param :unixtimestamp_key_names, :string, default: nil,
|
44
|
+
desc: "Key names which store data as datetime from unix time stamp"
|
45
|
+
|
32
46
|
config_param :on_duplicate_key_update, :bool, default: false,
|
33
47
|
desc: "On duplicate key update enable."
|
34
48
|
config_param :on_duplicate_update_keys, :string, default: nil,
|
@@ -83,10 +97,7 @@ DESC
|
|
83
97
|
@column_names = @column_names.split(',').collect(&:strip)
|
84
98
|
@key_names = @key_names.nil? ? @column_names : @key_names.split(',').collect(&:strip)
|
85
99
|
@json_key_names = @json_key_names.split(',') if @json_key_names
|
86
|
-
|
87
|
-
|
88
|
-
def start
|
89
|
-
super
|
100
|
+
@unixtimestamp_key_names = @unixtimestamp_key_names.split(',') if @unixtimestamp_key_names
|
90
101
|
end
|
91
102
|
|
92
103
|
def check_table_schema(database: @database, table: @table)
|
@@ -105,10 +116,6 @@ DESC
|
|
105
116
|
max_lengths
|
106
117
|
end
|
107
118
|
|
108
|
-
def shutdown
|
109
|
-
super
|
110
|
-
end
|
111
|
-
|
112
119
|
def format(tag, time, record)
|
113
120
|
record = inject_values_to_record(tag, time, record)
|
114
121
|
[tag, time, record].to_msgpack
|
@@ -118,6 +125,10 @@ DESC
|
|
118
125
|
true
|
119
126
|
end
|
120
127
|
|
128
|
+
def multi_workers_ready?
|
129
|
+
true
|
130
|
+
end
|
131
|
+
|
121
132
|
def client(database)
|
122
133
|
Mysql2::Client.new(
|
123
134
|
host: @host,
|
@@ -125,6 +136,12 @@ DESC
|
|
125
136
|
username: @username,
|
126
137
|
password: @password,
|
127
138
|
database: database,
|
139
|
+
sslkey: @sslkey,
|
140
|
+
sslcert: @sslcert,
|
141
|
+
sslca: @sslca,
|
142
|
+
sslcapath: @sslcapath,
|
143
|
+
sslcipher: @sslcipher,
|
144
|
+
sslverify: @sslverify,
|
128
145
|
flags: Mysql2::Client::MULTI_STATEMENTS
|
129
146
|
)
|
130
147
|
end
|
@@ -145,7 +162,7 @@ DESC
|
|
145
162
|
data = format_proc.call(tag, time, data, max_lengths)
|
146
163
|
values << Mysql2::Client.pseudo_bind(values_template, data)
|
147
164
|
end
|
148
|
-
sql = "INSERT INTO #{table} (#{@column_names.join(',')}) VALUES #{values.join(',')}"
|
165
|
+
sql = "INSERT INTO #{table} (#{@column_names.map{|x| "`#{x.to_s.gsub('`', '``')}`"}.join(',')}) VALUES #{values.join(',')}"
|
149
166
|
sql += @on_duplicate_key_update_sql if @on_duplicate_key_update
|
150
167
|
|
151
168
|
log.info "bulk insert values size (table: #{table}) => #{values.size}"
|
@@ -165,12 +182,16 @@ DESC
|
|
165
182
|
if max_lengths[i].nil? || record[key].nil?
|
166
183
|
value = record[key]
|
167
184
|
else
|
168
|
-
value = record[key].slice(0, max_lengths[i])
|
185
|
+
value = record[key].to_s.slice(0, max_lengths[i])
|
169
186
|
end
|
170
187
|
|
171
188
|
if @json_key_names && @json_key_names.include?(key)
|
172
189
|
value = value.to_json
|
173
190
|
end
|
191
|
+
|
192
|
+
if @unixtimestamp_key_names && @unixtimestamp_key_names.include?(key)
|
193
|
+
value = Time.at(value).strftime('%Y-%m-%d %H:%M:%S')
|
194
|
+
end
|
174
195
|
end
|
175
196
|
values << value
|
176
197
|
end
|
@@ -266,6 +266,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
|
|
266
266
|
assert_equal ['id','user_name','created_at','updated_at'], d.instance.key_names
|
267
267
|
assert_equal ['id','user_name','created_at','updated_at'], d.instance.column_names
|
268
268
|
assert_equal nil, d.instance.json_key_names
|
269
|
+
assert_equal nil, d.instance.unixtimestamp_key_names
|
269
270
|
assert_equal " ON DUPLICATE KEY UPDATE user_name = VALUES(user_name),updated_at = VALUES(updated_at)", d.instance.instance_variable_get(:@on_duplicate_key_update_sql)
|
270
271
|
|
271
272
|
d = create_driver %[
|
@@ -279,6 +280,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
|
|
279
280
|
assert_equal ['id','user_name','created_at','updated_at'], d.instance.key_names
|
280
281
|
assert_equal ['id','user_name','created_at','updated_at'], d.instance.column_names
|
281
282
|
assert_equal nil, d.instance.json_key_names
|
283
|
+
assert_equal nil, d.instance.unixtimestamp_key_names
|
282
284
|
assert_nil d.instance.instance_variable_get(:@on_duplicate_key_update_sql)
|
283
285
|
|
284
286
|
d = create_driver %[
|
@@ -293,6 +295,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
|
|
293
295
|
assert_equal ['id','user_name','created_at','updated_at'], d.instance.key_names
|
294
296
|
assert_equal ['id','user','created_date','updated_date'], d.instance.column_names
|
295
297
|
assert_equal nil, d.instance.json_key_names
|
298
|
+
assert_equal nil, d.instance.unixtimestamp_key_names
|
296
299
|
assert_nil d.instance.instance_variable_get(:@on_duplicate_key_update_sql)
|
297
300
|
|
298
301
|
d = create_driver %[
|
@@ -301,6 +304,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
|
|
301
304
|
password hogehoge
|
302
305
|
key_names id,url,request_headers,params,created_at,updated_at
|
303
306
|
column_names id,url,request_headers_json,params_json,created_date,updated_date
|
307
|
+
unixtimestamp_key_names created_at,updated_at
|
304
308
|
json_key_names request_headers,params
|
305
309
|
table access
|
306
310
|
]
|
@@ -308,6 +312,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
|
|
308
312
|
assert_equal ['id','url','request_headers','params','created_at','updated_at'], d.instance.key_names
|
309
313
|
assert_equal ['id','url','request_headers_json','params_json','created_date','updated_date'], d.instance.column_names
|
310
314
|
assert_equal ['request_headers','params'], d.instance.json_key_names
|
315
|
+
assert_equal ['created_at', 'updated_at'], d.instance.unixtimestamp_key_names
|
311
316
|
assert_nil d.instance.instance_variable_get(:@on_duplicate_key_update_sql)
|
312
317
|
|
313
318
|
d = create_driver %[
|
@@ -324,6 +329,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
|
|
324
329
|
assert_equal ['id','user_name','login_count','created_at','updated_at'], d.instance.key_names
|
325
330
|
assert_equal ['id','user_name','login_count','created_at','updated_at'], d.instance.column_names
|
326
331
|
assert_equal nil, d.instance.json_key_names
|
332
|
+
assert_equal nil, d.instance.unixtimestamp_key_names
|
327
333
|
assert_equal " ON DUPLICATE KEY UPDATE login_count = `login_count` + 1,updated_at = VALUES(updated_at)", d.instance.instance_variable_get(:@on_duplicate_key_update_sql)
|
328
334
|
end
|
329
335
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-mysql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TAGOMORI Satoshi
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2017-10-04 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -142,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
142
142
|
version: '0'
|
143
143
|
requirements: []
|
144
144
|
rubyforge_project:
|
145
|
-
rubygems_version: 2.
|
145
|
+
rubygems_version: 2.6.13
|
146
146
|
signing_key:
|
147
147
|
specification_version: 4
|
148
148
|
summary: fluent plugin to insert mysql
|
@@ -150,4 +150,3 @@ test_files:
|
|
150
150
|
- test/helper.rb
|
151
151
|
- test/plugin/test_out_mysql.rb
|
152
152
|
- test/plugin/test_out_mysql_bulk.rb
|
153
|
-
has_rdoc:
|