fluent-plugin-mysql 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9859ad599cbae8698e0054adcbeaff3205f5a537
4
- data.tar.gz: 96cdfad1bf8d00270b2e6ac9d8ac92b83987b326
3
+ metadata.gz: 41820afbe7391da94a6778133b8bee6b2645564e
4
+ data.tar.gz: 7c77c42bc38bcbf7c698c19f970e6dda05f1161c
5
5
  SHA512:
6
- metadata.gz: e15db1860fe2d5925917a6fe949459e0d3e75cddd7d57f8df3d83f497f6e55dafa634b3da94273bcae01f2d89c64e5862da4540ec649034ce1a311320b4e88bc
7
- data.tar.gz: b1d106391be32bb63aa2dde7ade7c894fef44eab820ccf2dd7f9901160a7bfd26629469a8f089eddc80fff2d6f94d8c13a0c0343286f140eba07b987f4bd45fe
6
+ metadata.gz: eafdc4d3b3021330646df1520aa81904d4ae3510d0bd9052ea564142ccfe7f3e313154a5b22de8ffca281de4381037260df01c4cf41e4461bd2a7fc4c5189990
7
+ data.tar.gz: 2f70d79a00930d74518cce00e16df8bacea6e2c119b16344ff2f6f84a699b49377ce1451c819e27933cf81bdc917ed2f6b41b0fdb7988240fba273748663d581
data/README.md CHANGED
@@ -19,6 +19,12 @@ port|database port(default: 3306)
19
19
  database|database name(require)
20
20
  username|user(require)
21
21
  password|password(default: blank)
22
+ sslkey|path to client key(default: nil)
23
+ sslcert|path to client cert(default: nil)
24
+ sslca|path to ca cert(default: nil)
25
+ sslcapath|path to ca certs(default: nil)
26
+ sslcipher|ssl cipher(default: nil)
27
+ sslverify|verify server certificate(default: nil)
22
28
  column_names|bulk insert column (require)
23
29
  key_names|value key names, ${time} is placeholder Time.at(time).strftime("%Y-%m-%d %H:%M:%S") (default : column_names)
24
30
  json_key_names|Key names which store data as json, comma separator.
@@ -1,7 +1,7 @@
1
1
  # -*- encoding: utf-8 -*-
2
2
  Gem::Specification.new do |gem|
3
3
  gem.name = "fluent-plugin-mysql"
4
- gem.version = "0.2.1"
4
+ gem.version = "0.3.0"
5
5
  gem.authors = ["TAGOMORI Satoshi", "Toyama Hiroshi"]
6
6
  gem.email = ["tagomoris@gmail.com", "toyama0919@gmail.com"]
7
7
  gem.description = %q{fluent plugin to insert mysql as json(single column) or insert statement}
@@ -9,6 +9,12 @@ class Fluent::MysqlOutput < Fluent::BufferedOutput
9
9
  config_param :database, :string
10
10
  config_param :username, :string
11
11
  config_param :password, :string, :default => '', :secret => true
12
+ config_param :sslkey, :string, :default => nil
13
+ config_param :sslcert, :string, :default => nil
14
+ config_param :sslca, :string, :default => nil
15
+ config_param :sslcapath, :string, :default => nil
16
+ config_param :sslcipher, :string, :default => nil
17
+ config_param :sslverify, :bool, :default => nil
12
18
 
13
19
  config_param :key_names, :string, :default => nil # nil allowed for json format
14
20
  config_param :sql, :string, :default => nil
@@ -99,7 +105,14 @@ class Fluent::MysqlOutput < Fluent::BufferedOutput
99
105
  Mysql2::Client.new({
100
106
  :host => @host, :port => @port,
101
107
  :username => @username, :password => @password,
102
- :database => @database, :flags => Mysql2::Client::MULTI_STATEMENTS,
108
+ :database => @database,
109
+ :sslkey => @sslkey,
110
+ :sslcert => @sslcert,
111
+ :sslca => @sslca,
112
+ :sslcapath => @sslcapath,
113
+ :sslcipher => @sslcipher,
114
+ :sslverify => @sslverify,
115
+ :flags => Mysql2::Client::MULTI_STATEMENTS,
103
116
  })
104
117
  end
105
118
 
@@ -1,4 +1,3 @@
1
- # -*- encoding : utf-8 -*-
2
1
  require 'fluent/plugin/output'
3
2
 
4
3
  module Fluent::Plugin
@@ -17,6 +16,18 @@ module Fluent::Plugin
17
16
  desc: "Database user."
18
17
  config_param :password, :string, default: '', secret: true,
19
18
  desc: "Database password."
19
+ config_param :sslkey, :string, default: nil,
20
+ desc: "SSL key."
21
+ config_param :sslcert, :string, default: nil,
22
+ desc: "SSL cert."
23
+ config_param :sslca, :string, default: nil,
24
+ desc: "SSL CA."
25
+ config_param :sslcapath, :string, default: nil,
26
+ desc: "SSL CA path."
27
+ config_param :sslcipher, :string, default: nil,
28
+ desc: "SSL cipher."
29
+ config_param :sslverify, :bool, default: nil,
30
+ desc: "SSL Verify Server Certificate."
20
31
 
21
32
  config_param :column_names, :string,
22
33
  desc: "Bulk insert column."
@@ -29,6 +40,9 @@ DESC
29
40
  config_param :table, :string,
30
41
  desc: "Bulk insert table."
31
42
 
43
+ config_param :unixtimestamp_key_names, :string, default: nil,
44
+ desc: "Key names which store data as datetime from unix time stamp"
45
+
32
46
  config_param :on_duplicate_key_update, :bool, default: false,
33
47
  desc: "On duplicate key update enable."
34
48
  config_param :on_duplicate_update_keys, :string, default: nil,
@@ -83,10 +97,7 @@ DESC
83
97
  @column_names = @column_names.split(',').collect(&:strip)
84
98
  @key_names = @key_names.nil? ? @column_names : @key_names.split(',').collect(&:strip)
85
99
  @json_key_names = @json_key_names.split(',') if @json_key_names
86
- end
87
-
88
- def start
89
- super
100
+ @unixtimestamp_key_names = @unixtimestamp_key_names.split(',') if @unixtimestamp_key_names
90
101
  end
91
102
 
92
103
  def check_table_schema(database: @database, table: @table)
@@ -105,10 +116,6 @@ DESC
105
116
  max_lengths
106
117
  end
107
118
 
108
- def shutdown
109
- super
110
- end
111
-
112
119
  def format(tag, time, record)
113
120
  record = inject_values_to_record(tag, time, record)
114
121
  [tag, time, record].to_msgpack
@@ -118,6 +125,10 @@ DESC
118
125
  true
119
126
  end
120
127
 
128
+ def multi_workers_ready?
129
+ true
130
+ end
131
+
121
132
  def client(database)
122
133
  Mysql2::Client.new(
123
134
  host: @host,
@@ -125,6 +136,12 @@ DESC
125
136
  username: @username,
126
137
  password: @password,
127
138
  database: database,
139
+ sslkey: @sslkey,
140
+ sslcert: @sslcert,
141
+ sslca: @sslca,
142
+ sslcapath: @sslcapath,
143
+ sslcipher: @sslcipher,
144
+ sslverify: @sslverify,
128
145
  flags: Mysql2::Client::MULTI_STATEMENTS
129
146
  )
130
147
  end
@@ -145,7 +162,7 @@ DESC
145
162
  data = format_proc.call(tag, time, data, max_lengths)
146
163
  values << Mysql2::Client.pseudo_bind(values_template, data)
147
164
  end
148
- sql = "INSERT INTO #{table} (#{@column_names.join(',')}) VALUES #{values.join(',')}"
165
+ sql = "INSERT INTO #{table} (#{@column_names.map{|x| "`#{x.to_s.gsub('`', '``')}`"}.join(',')}) VALUES #{values.join(',')}"
149
166
  sql += @on_duplicate_key_update_sql if @on_duplicate_key_update
150
167
 
151
168
  log.info "bulk insert values size (table: #{table}) => #{values.size}"
@@ -165,12 +182,16 @@ DESC
165
182
  if max_lengths[i].nil? || record[key].nil?
166
183
  value = record[key]
167
184
  else
168
- value = record[key].slice(0, max_lengths[i])
185
+ value = record[key].to_s.slice(0, max_lengths[i])
169
186
  end
170
187
 
171
188
  if @json_key_names && @json_key_names.include?(key)
172
189
  value = value.to_json
173
190
  end
191
+
192
+ if @unixtimestamp_key_names && @unixtimestamp_key_names.include?(key)
193
+ value = Time.at(value).strftime('%Y-%m-%d %H:%M:%S')
194
+ end
174
195
  end
175
196
  values << value
176
197
  end
@@ -266,6 +266,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
266
266
  assert_equal ['id','user_name','created_at','updated_at'], d.instance.key_names
267
267
  assert_equal ['id','user_name','created_at','updated_at'], d.instance.column_names
268
268
  assert_equal nil, d.instance.json_key_names
269
+ assert_equal nil, d.instance.unixtimestamp_key_names
269
270
  assert_equal " ON DUPLICATE KEY UPDATE user_name = VALUES(user_name),updated_at = VALUES(updated_at)", d.instance.instance_variable_get(:@on_duplicate_key_update_sql)
270
271
 
271
272
  d = create_driver %[
@@ -279,6 +280,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
279
280
  assert_equal ['id','user_name','created_at','updated_at'], d.instance.key_names
280
281
  assert_equal ['id','user_name','created_at','updated_at'], d.instance.column_names
281
282
  assert_equal nil, d.instance.json_key_names
283
+ assert_equal nil, d.instance.unixtimestamp_key_names
282
284
  assert_nil d.instance.instance_variable_get(:@on_duplicate_key_update_sql)
283
285
 
284
286
  d = create_driver %[
@@ -293,6 +295,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
293
295
  assert_equal ['id','user_name','created_at','updated_at'], d.instance.key_names
294
296
  assert_equal ['id','user','created_date','updated_date'], d.instance.column_names
295
297
  assert_equal nil, d.instance.json_key_names
298
+ assert_equal nil, d.instance.unixtimestamp_key_names
296
299
  assert_nil d.instance.instance_variable_get(:@on_duplicate_key_update_sql)
297
300
 
298
301
  d = create_driver %[
@@ -301,6 +304,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
301
304
  password hogehoge
302
305
  key_names id,url,request_headers,params,created_at,updated_at
303
306
  column_names id,url,request_headers_json,params_json,created_date,updated_date
307
+ unixtimestamp_key_names created_at,updated_at
304
308
  json_key_names request_headers,params
305
309
  table access
306
310
  ]
@@ -308,6 +312,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
308
312
  assert_equal ['id','url','request_headers','params','created_at','updated_at'], d.instance.key_names
309
313
  assert_equal ['id','url','request_headers_json','params_json','created_date','updated_date'], d.instance.column_names
310
314
  assert_equal ['request_headers','params'], d.instance.json_key_names
315
+ assert_equal ['created_at', 'updated_at'], d.instance.unixtimestamp_key_names
311
316
  assert_nil d.instance.instance_variable_get(:@on_duplicate_key_update_sql)
312
317
 
313
318
  d = create_driver %[
@@ -324,6 +329,7 @@ class MysqlBulkOutputTest < Test::Unit::TestCase
324
329
  assert_equal ['id','user_name','login_count','created_at','updated_at'], d.instance.key_names
325
330
  assert_equal ['id','user_name','login_count','created_at','updated_at'], d.instance.column_names
326
331
  assert_equal nil, d.instance.json_key_names
332
+ assert_equal nil, d.instance.unixtimestamp_key_names
327
333
  assert_equal " ON DUPLICATE KEY UPDATE login_count = `login_count` + 1,updated_at = VALUES(updated_at)", d.instance.instance_variable_get(:@on_duplicate_key_update_sql)
328
334
  end
329
335
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-mysql
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.1
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - TAGOMORI Satoshi
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2016-11-14 00:00:00.000000000 Z
12
+ date: 2017-10-04 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
@@ -142,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
142
142
  version: '0'
143
143
  requirements: []
144
144
  rubyforge_project:
145
- rubygems_version: 2.5.1
145
+ rubygems_version: 2.6.13
146
146
  signing_key:
147
147
  specification_version: 4
148
148
  summary: fluent plugin to insert mysql
@@ -150,4 +150,3 @@ test_files:
150
150
  - test/helper.rb
151
151
  - test/plugin/test_out_mysql.rb
152
152
  - test/plugin/test_out_mysql_bulk.rb
153
- has_rdoc: