fluent-plugin-mysql 0.0.2 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.md CHANGED
@@ -50,14 +50,59 @@ Or, insert json into single column.
50
50
  flush_intervals 5s
51
51
  </match>
52
52
 
53
- Now, out_mysql cannnot handle tag/time as output data.
53
+ To include time/tag into output, use `include_time_key` and `include_tag_key`, like this:
54
+
55
+ <match output.with.tag.and.time.*>
56
+ type mysql
57
+ host my.mysql.local
58
+ database anydatabase
59
+ username yourusername
60
+ password secret
61
+
62
+ include_time_key yes
63
+ ### default `time_format` is ISO-8601
64
+ # time_format %Y%m%d-%H%M%S
65
+ ### default `time_key` is 'time'
66
+ # time_key timekey
67
+
68
+ include_tag_key yes
69
+ ### default `tag_key` is 'tag'
70
+ # tag_key tagkey
71
+
72
+ table anydata
73
+ key_names time,tag,field1,field2,field3,field4
74
+ sql INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)
75
+ </match>
76
+
77
+ Or, for json:
78
+
79
+ <match output.with.tag.and.time.as.json.*>
80
+ type mysql
81
+ host database.local
82
+ database foo
83
+ username root
84
+
85
+ include_time_key yes
86
+ utc # with UTC timezome output (default: localtime)
87
+ time_format %Y%m%d-%H%M%S
88
+ time_key timeattr
89
+
90
+ include_tag_key yes
91
+ tag_key tagattr
92
+ table accesslog
93
+ columns jsondata
94
+ format json
95
+ </match>
96
+ #=> inserted json data into column 'jsondata' with addtional attribute 'timeattr' and 'tagattr'
54
97
 
55
98
  ## TODO
56
99
 
57
100
  * implement 'tag_mapped'
58
- * implement 'time' and 'tag' in key_names
101
+ * dynamic tag based table selection
59
102
 
60
103
  ## Copyright
61
104
 
62
- Copyright:: Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
63
- License:: Apache License, Version 2.0
105
+ * Copyright
106
+ * Copyright(C) 2012- TAGOMORI Satoshi (tagomoris)
107
+ * License
108
+ * Apache License, Version 2.0
@@ -1,7 +1,7 @@
1
1
  # -*- encoding: utf-8 -*-
2
2
  Gem::Specification.new do |gem|
3
3
  gem.name = "fluent-plugin-mysql"
4
- gem.version = "0.0.2"
4
+ gem.version = "0.0.3"
5
5
  gem.authors = ["TAGOMORI Satoshi"]
6
6
  gem.email = ["tagomoris@gmail.com"]
7
7
  gem.description = %q{fluent plugin to insert mysql as json(single column) or insert statement}
@@ -1,6 +1,9 @@
1
1
  class Fluent::MysqlOutput < Fluent::BufferedOutput
2
2
  Fluent::Plugin.register_output('mysql', self)
3
3
 
4
+ include Fluent::SetTimeKeyMixin
5
+ include Fluent::SetTagKeyMixin
6
+
4
7
  config_param :host, :string
5
8
  config_param :port, :integer, :default => nil
6
9
  config_param :database, :string
@@ -27,10 +30,8 @@ class Fluent::MysqlOutput < Fluent::BufferedOutput
27
30
  # TODO tag_mapped
28
31
 
29
32
  if @format == 'json'
30
- # TODO time, tag, and json values
31
33
  @format_proc = Proc.new{|tag, time, record| record.to_json}
32
34
  else
33
- # TODO time,tag in key_names
34
35
  @key_names = @key_names.split(',')
35
36
  @format_proc = Proc.new{|tag, time, record| @key_names.map{|k| record[k]}}
36
37
  end
@@ -77,10 +78,16 @@ class Fluent::MysqlOutput < Fluent::BufferedOutput
77
78
  [tag, time, @format_proc.call(tag, time, record)].to_msgpack
78
79
  end
79
80
 
81
+ def client
82
+ Mysql2::Client.new({
83
+ :host => @host, :port => @port,
84
+ :username => @username, :password => @password,
85
+ :database => @database
86
+ })
87
+ end
88
+
80
89
  def write(chunk)
81
- handler = Mysql2::Client.new({:host => @host, :port => @port,
82
- :username => @username, :password => @password,
83
- :database => @database})
90
+ handler = self.client
84
91
  chunk.msgpack_each { |tag, time, data|
85
92
  handler.xquery(@sql, data)
86
93
  }
@@ -1,4 +1,5 @@
1
1
  require 'helper'
2
+ require 'mysql2-cs-bind'
2
3
 
3
4
  class MysqlOutputTest < Test::Unit::TestCase
4
5
  def setup
@@ -15,15 +16,16 @@ format json
15
16
 
16
17
  def create_driver(conf = CONFIG, tag='test')
17
18
  d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::MysqlOutput, tag).configure(conf)
18
- obj = Object.new
19
- obj.instance_eval {
20
- def escape(v)
21
- v
19
+ d.instance.instance_eval {
20
+ def client
21
+ obj = Object.new
22
+ obj.instance_eval {
23
+ def xquery(*args); [1]; end
24
+ def close; true; end
25
+ }
26
+ obj
22
27
  end
23
- def query(*args); [1]; end
24
- def close; true; end
25
28
  }
26
- d.instance.handler = obj
27
29
  d
28
30
  end
29
31
 
@@ -81,16 +83,80 @@ sql INSERT INTO baz (col1,col2,col3,col4) VALUES (?,?,?,?)
81
83
  d.run
82
84
  end
83
85
 
84
- def test_write
85
- # d = create_driver
86
+ def test_time_and_tag_key
87
+ d = create_driver %[
88
+ host database.local
89
+ database foo
90
+ username bar
91
+ password mogera
92
+ include_time_key yes
93
+ utc
94
+ include_tag_key yes
95
+ table baz
96
+ key_names time,tag,field1,field2,field3,field4
97
+ sql INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)
98
+ ]
99
+ assert_equal 'INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)', d.instance.sql
100
+
101
+ time = Time.parse('2012-12-17 01:23:45 UTC').to_i
102
+ record = {'field1'=>'value1','field2'=>'value2','field3'=>'value3','field4'=>'value4'}
103
+ d.emit(record, time)
104
+ d.expect_format ['test', time, ['2012-12-17T01:23:45Z','test','value1','value2','value3','value4']].to_msgpack
105
+ d.run
106
+ end
107
+
108
+ def test_time_and_tag_key_complex
109
+ d = create_driver %[
110
+ host database.local
111
+ database foo
112
+ username bar
113
+ password mogera
114
+ include_time_key yes
115
+ utc
116
+ time_format %Y%m%d-%H%M%S
117
+ time_key timekey
118
+ include_tag_key yes
119
+ tag_key tagkey
120
+ table baz
121
+ key_names timekey,tagkey,field1,field2,field3,field4
122
+ sql INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)
123
+ ]
124
+ assert_equal 'INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)', d.instance.sql
86
125
 
87
- # time = Time.parse("2011-01-02 13:14:15 UTC").to_i
88
- # d.emit({"a"=>1}, time)
89
- # d.emit({"a"=>2}, time)
126
+ time = Time.parse('2012-12-17 09:23:45 JST').to_i # JST(+0900)
127
+ record = {'field1'=>'value1','field2'=>'value2','field3'=>'value3','field4'=>'value4'}
128
+ d.emit(record, time)
129
+ d.expect_format ['test', time, ['20121217-002345','test','value1','value2','value3','value4']].to_msgpack
130
+ d.run
131
+ end
90
132
 
91
- # ### FileOutput#write returns path
92
- # path = d.run
93
- # expect_path = "#{TMP_DIR}/out_file_test._0.log.gz"
94
- # assert_equal expect_path, path
133
+ def test_time_and_tag_key_json
134
+ d = create_driver %[
135
+ host database.local
136
+ database foo
137
+ username bar
138
+ password mogera
139
+ include_time_key yes
140
+ utc
141
+ time_format %Y%m%d-%H%M%S
142
+ time_key timekey
143
+ include_tag_key yes
144
+ tag_key tagkey
145
+ table accesslog
146
+ columns jsondata
147
+ format json
148
+ ]
149
+ assert_equal 'INSERT INTO accesslog (jsondata) VALUES (?)', d.instance.sql
150
+
151
+ time = Time.parse('2012-12-17 09:23:45 JST').to_i # JST(+0900)
152
+ record = {'field1'=>'value1'}
153
+ d.emit(record, time)
154
+ # Ruby 1.9.3 Hash saves its key order, so this code is OK.
155
+ d.expect_format ['test', time, record.merge({'timekey'=>'20121217-002345','tagkey'=>'test'}).to_json].to_msgpack
156
+ d.run
157
+ end
158
+
159
+ def test_write
160
+ # hmm....
95
161
  end
96
162
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-mysql
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.0.3
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-04-20 00:00:00.000000000 Z
12
+ date: 2012-12-17 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd