fluent-plugin-mysql 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-mysql.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,13 @@
1
+ Copyright (c) 2012- TAGOMORI Satoshi
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
data/README.md ADDED
@@ -0,0 +1,63 @@
1
+ # fluent-plugin-mysql
2
+
3
+ ## Component
4
+
5
+ ### MysqlOutput
6
+
7
+ Plugin to store mysql tables over SQL, to each columns per values, or to single column as json.
8
+
9
+ ## Configuration
10
+
11
+ ### MysqlOutput
12
+
13
+ MysqlOutput needs MySQL server's host/port/database/username/password, and INSERT format as SQL, or as table name and columns.
14
+
15
+ <match output.by.sql.*>
16
+ type mysql
17
+ host master.db.service.local
18
+ # port 3306 # default
19
+ database application_logs
20
+ username myuser
21
+ password mypass
22
+ key_names status,bytes,vhost,path,rhost,agent,referer
23
+ sql INSERT INTO accesslog (status,bytes,vhost,path,rhost,agent,referer) VALUES (?,?,?,?,?,?,?)
24
+ flush_intervals 5s
25
+ </match>
26
+
27
+ <match output.by.names.*>
28
+ type mysql
29
+ host master.db.service.local
30
+ database application_logs
31
+ username myuser
32
+ password mypass
33
+ key_names status,bytes,vhost,path,rhost,agent,referer
34
+ table accesslog
35
+ # 'columns' names order must be same with 'key_names'
36
+ columns status,bytes,vhost,path,rhost,agent,referer
37
+ flush_intervals 5s
38
+ </match>
39
+
40
+ Or, insert json into single column.
41
+
42
+ <match output.as.json.*>
43
+ type mysql
44
+ host master.db.service.local
45
+ database application_logs
46
+ username root
47
+ table accesslog
48
+ columns jsondata
49
+ format json
50
+ flush_intervals 5s
51
+ </match>
52
+
53
+ Now, out_mysql cannnot handle tag/time as output data.
54
+
55
+ ## TODO
56
+
57
+ * implement 'tag_mapped'
58
+ * implement 'time' and 'tag' in key_names
59
+
60
+ ## Copyright
61
+
62
+ Copyright:: Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
63
+ License:: Apache License, Version 2.0
data/Rakefile ADDED
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env rake
2
+ require "bundler/gem_tasks"
3
+
4
+ require 'rake/testtask'
5
+ Rake::TestTask.new(:test) do |test|
6
+ test.libs << 'lib' << 'test'
7
+ test.pattern = 'test/**/test_*.rb'
8
+ test.verbose = true
9
+ end
10
+
11
+ task :default => :test
@@ -0,0 +1,20 @@
1
+ # -*- encoding: utf-8 -*-
2
+ Gem::Specification.new do |gem|
3
+ gem.name = "fluent-plugin-mysql"
4
+ gem.version = "0.0.1"
5
+ gem.authors = ["TAGOMORI Satoshi"]
6
+ gem.email = ["tagomoris@gmail.com"]
7
+ gem.description = %q{fluent plugin to insert mysql as json(single column) or insert statement}
8
+ gem.summary = %q{fluent plugin to insert mysql}
9
+ gem.homepage = "https://github.com/tagomoris/fluent-plugin-mysql"
10
+
11
+ gem.files = `git ls-files`.split($\)
12
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
13
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
14
+ gem.require_paths = ["lib"]
15
+
16
+ gem.add_development_dependency "fluentd"
17
+ gem.add_development_dependency "mysql2"
18
+ gem.add_runtime_dependency "fluentd"
19
+ gem.add_runtime_dependency "mysql2"
20
+ end
@@ -0,0 +1,123 @@
1
+ class Fluent::MysqlOutput < Fluent::BufferedOutput
2
+ Fluent::Plugin.register_output('mysql', self)
3
+
4
+ config_param :host, :string
5
+ config_param :port, :integer, :default => nil
6
+ config_param :database, :string
7
+ config_param :username, :string
8
+ config_param :password, :string, :default => ''
9
+
10
+ config_param :key_names, :string, :default => nil # nil allowed for json format
11
+ config_param :sql, :string, :default => nil
12
+ config_param :table, :string, :default => nil
13
+ config_param :columns, :string, :default => nil
14
+
15
+ config_param :format, :string, :default => "raw" # or json
16
+
17
+ attr_accessor :handler
18
+
19
+ def initialize
20
+ super
21
+ require 'mysql2'
22
+ end
23
+
24
+ def configure(conf)
25
+ super
26
+
27
+ # TODO tag_mapped
28
+
29
+ if @format == 'json'
30
+ # TODO time, tag, and json values
31
+ @format_proc = Proc.new{|tag, time, record| record.to_json}
32
+ else
33
+ # TODO time,tag in key_names
34
+ @key_names = @key_names.split(',')
35
+ @format_proc = Proc.new{|tag, time, record| @key_names.map{|k| record[k]}}
36
+ end
37
+
38
+ if @columns.nil? and @sql.nil?
39
+ raise Fluent::ConfigError, "columns or sql MUST be specified, but missing"
40
+ end
41
+ if @columns and @sql
42
+ raise Fluent::ConfigError, "both of columns and sql are specified, but specify one of them"
43
+ end
44
+
45
+ if @sql
46
+ begin
47
+ # using nil to pass call of @handler.escape (@handler is set in #start)
48
+ if @format == 'json'
49
+ pseudo_bind(@sql, [nil])
50
+ else
51
+ pseudo_bind(@sql, @key_names.map{|n| nil})
52
+ end
53
+ rescue ArgumentError => e
54
+ raise Fluent::ConfigError, "mismatch between sql placeholders and key_names"
55
+ end
56
+ else # columns
57
+ raise Fluent::ConfigError, "table missing" unless @table
58
+ @columns = @columns.split(',')
59
+ cols = @columns.join(',')
60
+ placeholders = if @format == 'json'
61
+ '?'
62
+ else
63
+ @key_names.map{|k| '?'}.join(',')
64
+ end
65
+ @sql = "INSERT INTO #{@table} (#{cols}) VALUES (#{placeholders})"
66
+ end
67
+ end
68
+
69
+ def start
70
+ super
71
+ @handler ||= Mysql2::Client.new({:host => @host, :port => @port,
72
+ :username => @username, :password => @password,
73
+ :database => @database})
74
+ end
75
+
76
+ def shutdown
77
+ super
78
+ @handler.close
79
+ end
80
+
81
+ def pseudo_bind(sql, values)
82
+ sql = sql.dup
83
+
84
+ placeholders = []
85
+ search_pos = 0
86
+ while pos = sql.index('?', search_pos)
87
+ placeholders.push(pos)
88
+ search_pos = pos + 1
89
+ end
90
+ raise ArgumentError, "mismatch between placeholders number and values arguments" if placeholders.length != values.length
91
+
92
+ while pos = placeholders.pop()
93
+ rawvalue = values.pop()
94
+ if rawvalue.nil?
95
+ sql[pos] = 'NULL'
96
+ elsif rawvalue.is_a?(Time)
97
+ val = rawvalue.strftime('%Y-%m-%d %H:%M:%S')
98
+ sql[pos] = "'" + val + "'"
99
+ else
100
+ val = @handler.escape(rawvalue.to_s)
101
+ sql[pos] = "'" + val + "'"
102
+ end
103
+ end
104
+ sql
105
+ end
106
+
107
+ def query(sql, *values)
108
+ values = values.flatten
109
+ # pseudo prepared statements
110
+ return @handler.query(sql) if values.length < 1
111
+ @handler.query(self.pseudo_bind(sql, values))
112
+ end
113
+
114
+ def format(tag, time, record)
115
+ [tag, time, @format_proc.call(tag, time, record)].to_msgpack
116
+ end
117
+
118
+ def write(chunk)
119
+ chunk.msgpack_each { |tag, time, data|
120
+ query(@sql, data)
121
+ }
122
+ end
123
+ end
data/test/helper.rb ADDED
@@ -0,0 +1,28 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ begin
4
+ Bundler.setup(:default, :development)
5
+ rescue Bundler::BundlerError => e
6
+ $stderr.puts e.message
7
+ $stderr.puts "Run `bundle install` to install missing gems"
8
+ exit e.status_code
9
+ end
10
+ require 'test/unit'
11
+
12
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
14
+ require 'fluent/test'
15
+ unless ENV.has_key?('VERBOSE')
16
+ nulllogger = Object.new
17
+ nulllogger.instance_eval {|obj|
18
+ def method_missing(method, *args)
19
+ # pass
20
+ end
21
+ }
22
+ $log = nulllogger
23
+ end
24
+
25
+ require 'fluent/plugin/out_mysql'
26
+
27
+ class Test::Unit::TestCase
28
+ end
@@ -0,0 +1,107 @@
1
+ require 'helper'
2
+
3
+ class MysqlOutputTest < Test::Unit::TestCase
4
+ def setup
5
+ Fluent::Test.setup
6
+ end
7
+
8
+ CONFIG = %[
9
+ host db.local
10
+ database testing
11
+ username testuser
12
+ sql INSERT INTO tbl SET jsondata=?
13
+ format json
14
+ ]
15
+
16
+ def create_driver(conf = CONFIG, tag='test')
17
+ d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::MysqlOutput, tag).configure(conf)
18
+ obj = Object.new
19
+ obj.instance_eval {
20
+ def escape(v)
21
+ v
22
+ end
23
+ def query(*args); [1]; end
24
+ def close; true; end
25
+ }
26
+ d.instance.handler = obj
27
+ d
28
+ end
29
+
30
+ def test_configure
31
+ d = create_driver %[
32
+ host database.local
33
+ database foo
34
+ username bar
35
+ sql INSERT INTO baz SET jsondata=?
36
+ format json
37
+ ]
38
+ d = create_driver %[
39
+ host database.local
40
+ database foo
41
+ username bar
42
+ table baz
43
+ columns jsondata
44
+ format json
45
+ ]
46
+ d = create_driver %[
47
+ host database.local
48
+ database foo
49
+ username bar
50
+ password mogera
51
+ key_names field1,field2,field3
52
+ table baz
53
+ columns col1,col2,col3
54
+ ]
55
+ assert_equal 'INSERT INTO baz (col1,col2,col3) VALUES (?,?,?)', d.instance.sql
56
+
57
+ assert_raise(Fluent::ConfigError) {
58
+ d = create_driver %[
59
+ host database.local
60
+ database foo
61
+ username bar
62
+ password mogera
63
+ key_names field1,field2,field3
64
+ sql INSERT INTO baz (col1,col2,col3,col4) VALUES (?,?,?,?)
65
+ ]
66
+ }
67
+ end
68
+
69
+ def test_pseudo_bind
70
+ d = create_driver
71
+ sql = 'INSERT INTO baz SET col1=?'
72
+ # assert_equal "INSERT INTO baz SET col1='HOGE'", d.instance.pseudo_bind(sql, ['HOGE'])
73
+ assert_equal "INSERT INTO baz SET col1=NULL", d.instance.pseudo_bind(sql, [nil])
74
+ assert_equal "INSERT INTO baz SET col1='2012-04-16 17:38:00'", d.instance.pseudo_bind(sql, [Time.local(2012, 4, 16, 17, 38, 0)])
75
+ end
76
+
77
+ def test_query
78
+ end
79
+
80
+ def test_format
81
+ d = create_driver
82
+
83
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
84
+ d.emit({"a"=>1}, time)
85
+ d.emit({"a"=>2}, time)
86
+
87
+ #d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
88
+ #d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
89
+ d.expect_format ['test', time, {"a" => 1}.to_json].to_msgpack
90
+ d.expect_format ['test', time, {"a" => 2}.to_json].to_msgpack
91
+
92
+ d.run
93
+ end
94
+
95
+ def test_write
96
+ # d = create_driver
97
+
98
+ # time = Time.parse("2011-01-02 13:14:15 UTC").to_i
99
+ # d.emit({"a"=>1}, time)
100
+ # d.emit({"a"=>2}, time)
101
+
102
+ # ### FileOutput#write returns path
103
+ # path = d.run
104
+ # expect_path = "#{TMP_DIR}/out_file_test._0.log.gz"
105
+ # assert_equal expect_path, path
106
+ end
107
+ end
metadata ADDED
@@ -0,0 +1,120 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-mysql
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - TAGOMORI Satoshi
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2012-04-19 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: fluentd
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :development
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: '0'
30
+ - !ruby/object:Gem::Dependency
31
+ name: mysql2
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: '0'
38
+ type: :development
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '0'
46
+ - !ruby/object:Gem::Dependency
47
+ name: fluentd
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: '0'
54
+ type: :runtime
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ - !ruby/object:Gem::Dependency
63
+ name: mysql2
64
+ requirement: !ruby/object:Gem::Requirement
65
+ none: false
66
+ requirements:
67
+ - - ! '>='
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ type: :runtime
71
+ prerelease: false
72
+ version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ! '>='
76
+ - !ruby/object:Gem::Version
77
+ version: '0'
78
+ description: fluent plugin to insert mysql as json(single column) or insert statement
79
+ email:
80
+ - tagomoris@gmail.com
81
+ executables: []
82
+ extensions: []
83
+ extra_rdoc_files: []
84
+ files:
85
+ - .gitignore
86
+ - Gemfile
87
+ - LICENSE.txt
88
+ - README.md
89
+ - Rakefile
90
+ - fluent-plugin-mysql.gemspec
91
+ - lib/fluent/plugin/out_mysql.rb
92
+ - test/helper.rb
93
+ - test/plugin/test_out_mysql.rb
94
+ homepage: https://github.com/tagomoris/fluent-plugin-mysql
95
+ licenses: []
96
+ post_install_message:
97
+ rdoc_options: []
98
+ require_paths:
99
+ - lib
100
+ required_ruby_version: !ruby/object:Gem::Requirement
101
+ none: false
102
+ requirements:
103
+ - - ! '>='
104
+ - !ruby/object:Gem::Version
105
+ version: '0'
106
+ required_rubygems_version: !ruby/object:Gem::Requirement
107
+ none: false
108
+ requirements:
109
+ - - ! '>='
110
+ - !ruby/object:Gem::Version
111
+ version: '0'
112
+ requirements: []
113
+ rubyforge_project:
114
+ rubygems_version: 1.8.21
115
+ signing_key:
116
+ specification_version: 3
117
+ summary: fluent plugin to insert mysql
118
+ test_files:
119
+ - test/helper.rb
120
+ - test/plugin/test_out_mysql.rb