fluent-plugin-mysql 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +49 -4
- data/fluent-plugin-mysql.gemspec +1 -1
- data/lib/fluent/plugin/out_mysql.rb +12 -5
- data/test/plugin/test_out_mysql.rb +82 -16
- metadata +2 -2
data/README.md
CHANGED
@@ -50,14 +50,59 @@ Or, insert json into single column.
|
|
50
50
|
flush_intervals 5s
|
51
51
|
</match>
|
52
52
|
|
53
|
-
|
53
|
+
To include time/tag into output, use `include_time_key` and `include_tag_key`, like this:
|
54
|
+
|
55
|
+
<match output.with.tag.and.time.*>
|
56
|
+
type mysql
|
57
|
+
host my.mysql.local
|
58
|
+
database anydatabase
|
59
|
+
username yourusername
|
60
|
+
password secret
|
61
|
+
|
62
|
+
include_time_key yes
|
63
|
+
### default `time_format` is ISO-8601
|
64
|
+
# time_format %Y%m%d-%H%M%S
|
65
|
+
### default `time_key` is 'time'
|
66
|
+
# time_key timekey
|
67
|
+
|
68
|
+
include_tag_key yes
|
69
|
+
### default `tag_key` is 'tag'
|
70
|
+
# tag_key tagkey
|
71
|
+
|
72
|
+
table anydata
|
73
|
+
key_names time,tag,field1,field2,field3,field4
|
74
|
+
sql INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)
|
75
|
+
</match>
|
76
|
+
|
77
|
+
Or, for json:
|
78
|
+
|
79
|
+
<match output.with.tag.and.time.as.json.*>
|
80
|
+
type mysql
|
81
|
+
host database.local
|
82
|
+
database foo
|
83
|
+
username root
|
84
|
+
|
85
|
+
include_time_key yes
|
86
|
+
utc # with UTC timezome output (default: localtime)
|
87
|
+
time_format %Y%m%d-%H%M%S
|
88
|
+
time_key timeattr
|
89
|
+
|
90
|
+
include_tag_key yes
|
91
|
+
tag_key tagattr
|
92
|
+
table accesslog
|
93
|
+
columns jsondata
|
94
|
+
format json
|
95
|
+
</match>
|
96
|
+
#=> inserted json data into column 'jsondata' with addtional attribute 'timeattr' and 'tagattr'
|
54
97
|
|
55
98
|
## TODO
|
56
99
|
|
57
100
|
* implement 'tag_mapped'
|
58
|
-
*
|
101
|
+
* dynamic tag based table selection
|
59
102
|
|
60
103
|
## Copyright
|
61
104
|
|
62
|
-
|
63
|
-
|
105
|
+
* Copyright
|
106
|
+
* Copyright(C) 2012- TAGOMORI Satoshi (tagomoris)
|
107
|
+
* License
|
108
|
+
* Apache License, Version 2.0
|
data/fluent-plugin-mysql.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
Gem::Specification.new do |gem|
|
3
3
|
gem.name = "fluent-plugin-mysql"
|
4
|
-
gem.version = "0.0.
|
4
|
+
gem.version = "0.0.3"
|
5
5
|
gem.authors = ["TAGOMORI Satoshi"]
|
6
6
|
gem.email = ["tagomoris@gmail.com"]
|
7
7
|
gem.description = %q{fluent plugin to insert mysql as json(single column) or insert statement}
|
@@ -1,6 +1,9 @@
|
|
1
1
|
class Fluent::MysqlOutput < Fluent::BufferedOutput
|
2
2
|
Fluent::Plugin.register_output('mysql', self)
|
3
3
|
|
4
|
+
include Fluent::SetTimeKeyMixin
|
5
|
+
include Fluent::SetTagKeyMixin
|
6
|
+
|
4
7
|
config_param :host, :string
|
5
8
|
config_param :port, :integer, :default => nil
|
6
9
|
config_param :database, :string
|
@@ -27,10 +30,8 @@ class Fluent::MysqlOutput < Fluent::BufferedOutput
|
|
27
30
|
# TODO tag_mapped
|
28
31
|
|
29
32
|
if @format == 'json'
|
30
|
-
# TODO time, tag, and json values
|
31
33
|
@format_proc = Proc.new{|tag, time, record| record.to_json}
|
32
34
|
else
|
33
|
-
# TODO time,tag in key_names
|
34
35
|
@key_names = @key_names.split(',')
|
35
36
|
@format_proc = Proc.new{|tag, time, record| @key_names.map{|k| record[k]}}
|
36
37
|
end
|
@@ -77,10 +78,16 @@ class Fluent::MysqlOutput < Fluent::BufferedOutput
|
|
77
78
|
[tag, time, @format_proc.call(tag, time, record)].to_msgpack
|
78
79
|
end
|
79
80
|
|
81
|
+
def client
|
82
|
+
Mysql2::Client.new({
|
83
|
+
:host => @host, :port => @port,
|
84
|
+
:username => @username, :password => @password,
|
85
|
+
:database => @database
|
86
|
+
})
|
87
|
+
end
|
88
|
+
|
80
89
|
def write(chunk)
|
81
|
-
handler =
|
82
|
-
:username => @username, :password => @password,
|
83
|
-
:database => @database})
|
90
|
+
handler = self.client
|
84
91
|
chunk.msgpack_each { |tag, time, data|
|
85
92
|
handler.xquery(@sql, data)
|
86
93
|
}
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'helper'
|
2
|
+
require 'mysql2-cs-bind'
|
2
3
|
|
3
4
|
class MysqlOutputTest < Test::Unit::TestCase
|
4
5
|
def setup
|
@@ -15,15 +16,16 @@ format json
|
|
15
16
|
|
16
17
|
def create_driver(conf = CONFIG, tag='test')
|
17
18
|
d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::MysqlOutput, tag).configure(conf)
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
19
|
+
d.instance.instance_eval {
|
20
|
+
def client
|
21
|
+
obj = Object.new
|
22
|
+
obj.instance_eval {
|
23
|
+
def xquery(*args); [1]; end
|
24
|
+
def close; true; end
|
25
|
+
}
|
26
|
+
obj
|
22
27
|
end
|
23
|
-
def query(*args); [1]; end
|
24
|
-
def close; true; end
|
25
28
|
}
|
26
|
-
d.instance.handler = obj
|
27
29
|
d
|
28
30
|
end
|
29
31
|
|
@@ -81,16 +83,80 @@ sql INSERT INTO baz (col1,col2,col3,col4) VALUES (?,?,?,?)
|
|
81
83
|
d.run
|
82
84
|
end
|
83
85
|
|
84
|
-
def
|
85
|
-
|
86
|
+
def test_time_and_tag_key
|
87
|
+
d = create_driver %[
|
88
|
+
host database.local
|
89
|
+
database foo
|
90
|
+
username bar
|
91
|
+
password mogera
|
92
|
+
include_time_key yes
|
93
|
+
utc
|
94
|
+
include_tag_key yes
|
95
|
+
table baz
|
96
|
+
key_names time,tag,field1,field2,field3,field4
|
97
|
+
sql INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)
|
98
|
+
]
|
99
|
+
assert_equal 'INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)', d.instance.sql
|
100
|
+
|
101
|
+
time = Time.parse('2012-12-17 01:23:45 UTC').to_i
|
102
|
+
record = {'field1'=>'value1','field2'=>'value2','field3'=>'value3','field4'=>'value4'}
|
103
|
+
d.emit(record, time)
|
104
|
+
d.expect_format ['test', time, ['2012-12-17T01:23:45Z','test','value1','value2','value3','value4']].to_msgpack
|
105
|
+
d.run
|
106
|
+
end
|
107
|
+
|
108
|
+
def test_time_and_tag_key_complex
|
109
|
+
d = create_driver %[
|
110
|
+
host database.local
|
111
|
+
database foo
|
112
|
+
username bar
|
113
|
+
password mogera
|
114
|
+
include_time_key yes
|
115
|
+
utc
|
116
|
+
time_format %Y%m%d-%H%M%S
|
117
|
+
time_key timekey
|
118
|
+
include_tag_key yes
|
119
|
+
tag_key tagkey
|
120
|
+
table baz
|
121
|
+
key_names timekey,tagkey,field1,field2,field3,field4
|
122
|
+
sql INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)
|
123
|
+
]
|
124
|
+
assert_equal 'INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)', d.instance.sql
|
86
125
|
|
87
|
-
|
88
|
-
|
89
|
-
|
126
|
+
time = Time.parse('2012-12-17 09:23:45 JST').to_i # JST(+0900)
|
127
|
+
record = {'field1'=>'value1','field2'=>'value2','field3'=>'value3','field4'=>'value4'}
|
128
|
+
d.emit(record, time)
|
129
|
+
d.expect_format ['test', time, ['20121217-002345','test','value1','value2','value3','value4']].to_msgpack
|
130
|
+
d.run
|
131
|
+
end
|
90
132
|
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
133
|
+
def test_time_and_tag_key_json
|
134
|
+
d = create_driver %[
|
135
|
+
host database.local
|
136
|
+
database foo
|
137
|
+
username bar
|
138
|
+
password mogera
|
139
|
+
include_time_key yes
|
140
|
+
utc
|
141
|
+
time_format %Y%m%d-%H%M%S
|
142
|
+
time_key timekey
|
143
|
+
include_tag_key yes
|
144
|
+
tag_key tagkey
|
145
|
+
table accesslog
|
146
|
+
columns jsondata
|
147
|
+
format json
|
148
|
+
]
|
149
|
+
assert_equal 'INSERT INTO accesslog (jsondata) VALUES (?)', d.instance.sql
|
150
|
+
|
151
|
+
time = Time.parse('2012-12-17 09:23:45 JST').to_i # JST(+0900)
|
152
|
+
record = {'field1'=>'value1'}
|
153
|
+
d.emit(record, time)
|
154
|
+
# Ruby 1.9.3 Hash saves its key order, so this code is OK.
|
155
|
+
d.expect_format ['test', time, record.merge({'timekey'=>'20121217-002345','tagkey'=>'test'}).to_json].to_msgpack
|
156
|
+
d.run
|
157
|
+
end
|
158
|
+
|
159
|
+
def test_write
|
160
|
+
# hmm....
|
95
161
|
end
|
96
162
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-mysql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-12-17 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|