fluent-plugin-mysqlslowquerylog 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +11 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +2 -0
- data/README.md +37 -0
- data/Rakefile +10 -0
- data/fluent-plugin-mysqlslowquerylog.gemspec +21 -0
- data/lib/fluent/plugin/out_mysqlslowquerylog.rb +96 -0
- data/test/helper.rb +29 -0
- data/test/plugin/test_out_mysqlslowquerylog.rb +125 -0
- metadata +88 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
data/README.md
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
# fluent-plugin-mysqlslowquerylog
|
2
|
+
|
3
|
+
## Component
|
4
|
+
|
5
|
+
## Synopsis
|
6
|
+
|
7
|
+
## Configuration
|
8
|
+
|
9
|
+
## Installation
|
10
|
+
|
11
|
+
Add this line to your application's Gemfile:
|
12
|
+
|
13
|
+
gem 'fluent-plugin-mysqlslowquerylog'
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install fluent-plugin-mysqlslowquerylog
|
22
|
+
|
23
|
+
## Contributing
|
24
|
+
|
25
|
+
1. Fork it
|
26
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
27
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
28
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
29
|
+
5. Create new Pull Request
|
30
|
+
|
31
|
+
## Copyright
|
32
|
+
|
33
|
+
### Copyright
|
34
|
+
Copyright (c) 2012- Satoshi SUZUKI (@studio3104)
|
35
|
+
|
36
|
+
### License
|
37
|
+
Apache License, Version 2.0
|
data/Rakefile
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
|
5
|
+
Gem::Specification.new do |gem|
|
6
|
+
gem.name = "fluent-plugin-mysqlslowquerylog"
|
7
|
+
gem.version = "0.0.1"
|
8
|
+
gem.authors = ["Satoshi SUZUKI"]
|
9
|
+
gem.email = ["studio3104.com@gmail.com"]
|
10
|
+
gem.description = %q{Fluentd plugin to concat MySQL slowquerylog.}
|
11
|
+
gem.summary = %q{Fluentd plugin to concat MySQL slowquerylog.}
|
12
|
+
gem.homepage = "https://github.com/studio3104/fluent-plugin-mysqlslowquerylog"
|
13
|
+
|
14
|
+
gem.files = `git ls-files`.split($/)
|
15
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
16
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
17
|
+
gem.require_paths = ["lib"]
|
18
|
+
|
19
|
+
gem.add_development_dependency "fluentd"
|
20
|
+
gem.add_runtime_dependency "fluentd"
|
21
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
class Fluent::MySQLSlowQueryLogOutput < Fluent::Output
|
2
|
+
Fluent::Plugin.register_output('mysqlslowquerylog', self)
|
3
|
+
include Fluent::HandleTagNameMixin
|
4
|
+
|
5
|
+
config_param :explain, :bool, :default => false
|
6
|
+
config_param :dbuser, :string, :default => nil
|
7
|
+
config_param :dbpassword, :string, :default => nil
|
8
|
+
|
9
|
+
def configure(conf)
|
10
|
+
super
|
11
|
+
@slowlogs = Hash.new
|
12
|
+
|
13
|
+
if !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix
|
14
|
+
raise ConfigError, "out_slowquery: At least one of option, remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix is required to be set."
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def start
|
19
|
+
super
|
20
|
+
end
|
21
|
+
|
22
|
+
def shutdown
|
23
|
+
super
|
24
|
+
end
|
25
|
+
|
26
|
+
def emit(tag, es, chain)
|
27
|
+
if !@slowlogs[:"#{tag}"]
|
28
|
+
@slowlogs[:"#{tag}"] = Array.new
|
29
|
+
end
|
30
|
+
es.each do |time, record|
|
31
|
+
concat_messages(tag, time, record)
|
32
|
+
end
|
33
|
+
|
34
|
+
chain.next
|
35
|
+
end
|
36
|
+
|
37
|
+
def concat_messages(tag, time, record)
|
38
|
+
record.each do |key, value|
|
39
|
+
@slowlogs[:"#{tag}"] << value
|
40
|
+
if value !~ /^set timestamp=\d+\;$/i && value !~ /^use /i && value.end_with?(';')
|
41
|
+
parse_message(tag, time)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def parse_message(tag, time)
|
47
|
+
record = {}
|
48
|
+
date = nil
|
49
|
+
|
50
|
+
# Skip the message that is output when after flush-logs or restart mysqld.
|
51
|
+
# e.g.) /usr/sbin/mysqld, Version: 5.5.28-0ubuntu0.12.04.2-log ((Ubuntu)). started with:
|
52
|
+
message = @slowlogs[:"#{tag}"].shift
|
53
|
+
while !message.start_with?('#')
|
54
|
+
message = @slowlogs[:"#{tag}"].shift
|
55
|
+
end
|
56
|
+
|
57
|
+
if message.start_with?('# Time: ')
|
58
|
+
date = Time.parse(message[8..-1].strip)
|
59
|
+
message = @slowlogs[:"#{tag}"].shift
|
60
|
+
end
|
61
|
+
|
62
|
+
message =~ /^#? User\@Host:\s+(\S+)\s+\@\s+(\S+).*/
|
63
|
+
record[:user] = $1
|
64
|
+
record[:host] = $2
|
65
|
+
message = @slowlogs[:"#{tag}"].shift
|
66
|
+
|
67
|
+
message =~ /^# Query_time: ([0-9.]+)\s+Lock_time: ([0-9.]+)\s+Rows_sent: ([0-9.]+)\s+Rows_examined: ([0-9.]+).*/
|
68
|
+
record[:query_time] = $1.to_f
|
69
|
+
record[:lock_time] = $2.to_f
|
70
|
+
record[:rows_sent] = $3.to_i
|
71
|
+
record[:rows_examined] = $4.to_i
|
72
|
+
|
73
|
+
query = []
|
74
|
+
@slowlogs[:"#{tag}"].each do |m|
|
75
|
+
query << m.strip
|
76
|
+
end
|
77
|
+
record[:sql] = query.join(' ')
|
78
|
+
|
79
|
+
if date
|
80
|
+
flush_emit(tag, date.to_i, record)
|
81
|
+
else
|
82
|
+
flush_emit(tag, time, record)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def flush_emit(tag, time, record)
|
87
|
+
@slowlogs[:"#{tag}"].clear
|
88
|
+
_tag = tag.clone
|
89
|
+
filter_record(_tag, time, record)
|
90
|
+
if tag != _tag
|
91
|
+
Fluent::Engine.emit(_tag, time, record)
|
92
|
+
else
|
93
|
+
$log.warn "Can not emit message because the tag has not changed. Dropped record #{record}"
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require 'rubygems'
|
3
|
+
require 'bundler'
|
4
|
+
begin
|
5
|
+
Bundler.setup(:default, :development)
|
6
|
+
rescue Bundler::BundlerError => e
|
7
|
+
$stderr.puts e.message
|
8
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
9
|
+
exit e.status_code
|
10
|
+
end
|
11
|
+
require 'test/unit'
|
12
|
+
|
13
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
14
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
15
|
+
require 'fluent/test'
|
16
|
+
unless ENV.has_key?('VERBOSE')
|
17
|
+
nulllogger = Object.new
|
18
|
+
nulllogger.instance_eval {|obj|
|
19
|
+
def method_missing(method, *args)
|
20
|
+
# pass
|
21
|
+
end
|
22
|
+
}
|
23
|
+
$log = nulllogger
|
24
|
+
end
|
25
|
+
|
26
|
+
require 'fluent/plugin/out_mysqlslowquerylog'
|
27
|
+
|
28
|
+
class Test::Unit::TestCase
|
29
|
+
end
|
@@ -0,0 +1,125 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class MySQLSlowQueryLogOutputTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
Fluent::Test.setup
|
6
|
+
end
|
7
|
+
|
8
|
+
CONFIG = %[
|
9
|
+
add_tag_prefix concated.
|
10
|
+
]
|
11
|
+
|
12
|
+
def create_driver(conf = CONFIG, tag='test')
|
13
|
+
Fluent::Test::OutputTestDriver.new(Fluent::MySQLSlowQueryLogOutput, tag).configure(conf)
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_emit
|
17
|
+
d1 = create_driver
|
18
|
+
d2 = create_driver(CONFIG,'test2')
|
19
|
+
d3 = create_driver(CONFIG,'test3')
|
20
|
+
d4 = create_driver(CONFIG,'test4')
|
21
|
+
|
22
|
+
d1.run do
|
23
|
+
d1.emit('message' => '/usr/sbin/mysqld, Version: 5.5.28-0ubuntu0.12.04.2-log ((Ubuntu)). started with:')
|
24
|
+
d1.emit('message' => 'Tcp port: 3306 Unix socket: /var/run/mysqld/mysqld.sock')
|
25
|
+
d1.emit('message' => 'Time Id Command Argument')
|
26
|
+
d1.emit('message' => '# Time: 130105 16:43:42')
|
27
|
+
d1.emit('message' => '# User@Host: debian-sys-maint[debian-sys-maint] @ localhost []')
|
28
|
+
d1.emit('message' => '# Query_time: 0.000167 Lock_time: 0.000057 Rows_sent: 1 Rows_examined: 7')
|
29
|
+
d1.emit('message' => 'SET timestamp=1357371822;')
|
30
|
+
d1.emit('message' => "SELECT count(*) FROM mysql.user WHERE user='root' and password='';")
|
31
|
+
end
|
32
|
+
|
33
|
+
d2.run do
|
34
|
+
d2.emit('message' => '# User@Host: debian-sys-maint[debian-sys-maint] @ localhost []')
|
35
|
+
d2.emit('message' => '# Query_time: 0.002998 Lock_time: 0.000078 Rows_sent: 31 Rows_examined: 81')
|
36
|
+
d2.emit('message' => 'SET timestamp=61357371822;')
|
37
|
+
d2.emit('message' => "select concat('select count(*) into @discard from `',")
|
38
|
+
d2.emit('message' => " TABLE_SCHEMA, '`.`', TABLE_NAME, '`')")
|
39
|
+
d2.emit('message' => " from information_schema.TABLES where ENGINE='MyISAM';")
|
40
|
+
|
41
|
+
d2.emit('message' => "# Time: 130105 18:04:21")
|
42
|
+
d2.emit('message' => "# User@Host: root[root] @ localhost []")
|
43
|
+
d2.emit('message' => "# Query_time: 0.000398 Lock_time: 0.000117 Rows_sent: 7 Rows_examined: 7")
|
44
|
+
d2.emit('message' => "use mysql;")
|
45
|
+
d2.emit('message' => "SET timestamp=1357376661;")
|
46
|
+
d2.emit('message' => "select * from user;")
|
47
|
+
end
|
48
|
+
|
49
|
+
d3.run do
|
50
|
+
d3.emit('message' => "# User@Host: debian-sys-maint[debian-sys-maint] @ localhost []")
|
51
|
+
d3.emit('message' => "# Query_time: 0.014260 Lock_time: 0.000182 Rows_sent: 0 Rows_examined: 808")
|
52
|
+
|
53
|
+
d4.emit('message' => "# User@Host: debian-sys-maint[debian-sys-maint] @ localhost []")
|
54
|
+
d4.emit('message' => "# Query_time: 0.000262 Lock_time: 0.000200 Rows_sent: 0 Rows_examined: 0")
|
55
|
+
d4.emit('message' => "SET timestamp=1357371822;")
|
56
|
+
d4.emit('message' => "select count(*) into @discard from `information_schema`.`EVENTS`;")
|
57
|
+
|
58
|
+
d3.emit('message' => "SET timestamp=1357371822;")
|
59
|
+
d3.emit('message' => "select count(*) into @discard from `information_schema`.`COLUMNS`;")
|
60
|
+
end
|
61
|
+
|
62
|
+
assert_equal 1, d1.emits.size
|
63
|
+
assert_equal 2, d2.emits.size
|
64
|
+
assert_equal 2, d3.emits.size
|
65
|
+
|
66
|
+
assert_equal '2013-01-05 16:43:42 +0900', Time.at(d1.emits[0][1]).to_s
|
67
|
+
assert_equal '2013-01-05 18:04:21 +0900', Time.at(d2.emits[1][1]).to_s
|
68
|
+
|
69
|
+
assert_equal 'concated.test', d1.emits[0][0]
|
70
|
+
assert_equal 'concated.test2', d2.emits[0][0]
|
71
|
+
assert_equal 'concated.test2', d2.emits[1][0]
|
72
|
+
assert_equal 'concated.test4', d3.emits[0][0]
|
73
|
+
assert_equal 'concated.test3', d3.emits[1][0]
|
74
|
+
|
75
|
+
assert_equal({
|
76
|
+
:user=>"debian-sys-maint[debian-sys-maint]",
|
77
|
+
:host=>"localhost",
|
78
|
+
:query_time=>0.000167,
|
79
|
+
:lock_time=>5.7e-05,
|
80
|
+
:rows_sent=>1,
|
81
|
+
:rows_examined=>7,
|
82
|
+
:sql=>"SET timestamp=1357371822; SELECT count(*) FROM mysql.user WHERE user='root' and password='';"
|
83
|
+
}, d1.emits[0][2])
|
84
|
+
|
85
|
+
assert_equal({
|
86
|
+
:user=>"debian-sys-maint[debian-sys-maint]",
|
87
|
+
:host=>"localhost",
|
88
|
+
:query_time=>0.002998,
|
89
|
+
:lock_time=>7.8e-05,
|
90
|
+
:rows_sent=>31,
|
91
|
+
:rows_examined=>81,
|
92
|
+
:sql=>"SET timestamp=61357371822; select concat('select count(*) into @discard from `', TABLE_SCHEMA, '`.`', TABLE_NAME, '`') from information_schema.TABLES where ENGINE='MyISAM';"
|
93
|
+
}, d2.emits[0][2])
|
94
|
+
|
95
|
+
assert_equal({
|
96
|
+
:user=>"root[root]",
|
97
|
+
:host=>"localhost",
|
98
|
+
:query_time=>0.000398,
|
99
|
+
:lock_time=>0.000117,
|
100
|
+
:rows_sent=>7,
|
101
|
+
:rows_examined=>7,
|
102
|
+
:sql=>"use mysql; SET timestamp=1357376661; select * from user;"
|
103
|
+
}, d2.emits[1][2])
|
104
|
+
|
105
|
+
assert_equal({
|
106
|
+
:user=>"debian-sys-maint[debian-sys-maint]",
|
107
|
+
:host=>"localhost",
|
108
|
+
:query_time=>0.000262,
|
109
|
+
:lock_time=>0.0002,
|
110
|
+
:rows_sent=>0,
|
111
|
+
:rows_examined=>0,
|
112
|
+
:sql=>"SET timestamp=1357371822; select count(*) into @discard from `information_schema`.`EVENTS`;"
|
113
|
+
}, d3.emits[0][2])
|
114
|
+
|
115
|
+
assert_equal({
|
116
|
+
:user=>"debian-sys-maint[debian-sys-maint]",
|
117
|
+
:host=>"localhost",
|
118
|
+
:query_time=>0.01426,
|
119
|
+
:lock_time=>0.000182,
|
120
|
+
:rows_sent=>0,
|
121
|
+
:rows_examined=>808,
|
122
|
+
:sql=>"SET timestamp=1357371822; select count(*) into @discard from `information_schema`.`COLUMNS`;"
|
123
|
+
}, d3.emits[1][2])
|
124
|
+
end
|
125
|
+
end
|
metadata
ADDED
@@ -0,0 +1,88 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-mysqlslowquerylog
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Satoshi SUZUKI
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2013-01-07 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: fluentd
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: fluentd
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
description: Fluentd plugin to concat MySQL slowquerylog.
|
47
|
+
email:
|
48
|
+
- studio3104.com@gmail.com
|
49
|
+
executables: []
|
50
|
+
extensions: []
|
51
|
+
extra_rdoc_files: []
|
52
|
+
files:
|
53
|
+
- .gitignore
|
54
|
+
- Gemfile
|
55
|
+
- LICENSE.txt
|
56
|
+
- README.md
|
57
|
+
- Rakefile
|
58
|
+
- fluent-plugin-mysqlslowquerylog.gemspec
|
59
|
+
- lib/fluent/plugin/out_mysqlslowquerylog.rb
|
60
|
+
- test/helper.rb
|
61
|
+
- test/plugin/test_out_mysqlslowquerylog.rb
|
62
|
+
homepage: https://github.com/studio3104/fluent-plugin-mysqlslowquerylog
|
63
|
+
licenses: []
|
64
|
+
post_install_message:
|
65
|
+
rdoc_options: []
|
66
|
+
require_paths:
|
67
|
+
- lib
|
68
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
69
|
+
none: false
|
70
|
+
requirements:
|
71
|
+
- - ! '>='
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: '0'
|
74
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
75
|
+
none: false
|
76
|
+
requirements:
|
77
|
+
- - ! '>='
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: '0'
|
80
|
+
requirements: []
|
81
|
+
rubyforge_project:
|
82
|
+
rubygems_version: 1.8.23
|
83
|
+
signing_key:
|
84
|
+
specification_version: 3
|
85
|
+
summary: Fluentd plugin to concat MySQL slowquerylog.
|
86
|
+
test_files:
|
87
|
+
- test/helper.rb
|
88
|
+
- test/plugin/test_out_mysqlslowquerylog.rb
|