fluent-plugin-mysqlslowquerylog 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,11 @@
1
+ *.gem
2
+ .bundle
3
+ Gemfile.lock
4
+ pkg/*
5
+ # For TextMate, emacs, vim
6
+ *.tmproj
7
+ tmtags
8
+ *~
9
+ \#*
10
+ .\#*
11
+ *.swp
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-mysqlslowquerylog.gemspec
4
+ gemspec
@@ -0,0 +1,2 @@
1
+ Copyright (c) 2012 Satoshi SUZUKI
2
+ Apache License, Version 2.0
@@ -0,0 +1,37 @@
1
+ # fluent-plugin-mysqlslowquerylog
2
+
3
+ ## Component
4
+
5
+ ## Synopsis
6
+
7
+ ## Configuration
8
+
9
+ ## Installation
10
+
11
+ Add this line to your application's Gemfile:
12
+
13
+ gem 'fluent-plugin-mysqlslowquerylog'
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install fluent-plugin-mysqlslowquerylog
22
+
23
+ ## Contributing
24
+
25
+ 1. Fork it
26
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
27
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
28
+ 4. Push to the branch (`git push origin my-new-feature`)
29
+ 5. Create new Pull Request
30
+
31
+ ## Copyright
32
+
33
+ ### Copyright
34
+ Copyright (c) 2012- Satoshi SUZUKI (@studio3104)
35
+
36
+ ### License
37
+ Apache License, Version 2.0
@@ -0,0 +1,10 @@
1
+ require "bundler/gem_tasks"
2
+ require 'rake/testtask'
3
+
4
+ Rake::TestTask.new(:test) do |test|
5
+ test.libs << 'lib' << 'test'
6
+ test.pattern = 'test/**/test_*.rb'
7
+ test.verbose = true
8
+ end
9
+
10
+ task :default => :test
@@ -0,0 +1,21 @@
1
+ # -*- encoding: utf-8 -*-
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+
5
+ Gem::Specification.new do |gem|
6
+ gem.name = "fluent-plugin-mysqlslowquerylog"
7
+ gem.version = "0.0.1"
8
+ gem.authors = ["Satoshi SUZUKI"]
9
+ gem.email = ["studio3104.com@gmail.com"]
10
+ gem.description = %q{Fluentd plugin to concat MySQL slowquerylog.}
11
+ gem.summary = %q{Fluentd plugin to concat MySQL slowquerylog.}
12
+ gem.homepage = "https://github.com/studio3104/fluent-plugin-mysqlslowquerylog"
13
+
14
+ gem.files = `git ls-files`.split($/)
15
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
16
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
17
+ gem.require_paths = ["lib"]
18
+
19
+ gem.add_development_dependency "fluentd"
20
+ gem.add_runtime_dependency "fluentd"
21
+ end
@@ -0,0 +1,96 @@
1
+ class Fluent::MySQLSlowQueryLogOutput < Fluent::Output
2
+ Fluent::Plugin.register_output('mysqlslowquerylog', self)
3
+ include Fluent::HandleTagNameMixin
4
+
5
+ config_param :explain, :bool, :default => false
6
+ config_param :dbuser, :string, :default => nil
7
+ config_param :dbpassword, :string, :default => nil
8
+
9
+ def configure(conf)
10
+ super
11
+ @slowlogs = Hash.new
12
+
13
+ if !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix
14
+ raise ConfigError, "out_slowquery: At least one of option, remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix is required to be set."
15
+ end
16
+ end
17
+
18
+ def start
19
+ super
20
+ end
21
+
22
+ def shutdown
23
+ super
24
+ end
25
+
26
+ def emit(tag, es, chain)
27
+ if !@slowlogs[:"#{tag}"]
28
+ @slowlogs[:"#{tag}"] = Array.new
29
+ end
30
+ es.each do |time, record|
31
+ concat_messages(tag, time, record)
32
+ end
33
+
34
+ chain.next
35
+ end
36
+
37
+ def concat_messages(tag, time, record)
38
+ record.each do |key, value|
39
+ @slowlogs[:"#{tag}"] << value
40
+ if value !~ /^set timestamp=\d+\;$/i && value !~ /^use /i && value.end_with?(';')
41
+ parse_message(tag, time)
42
+ end
43
+ end
44
+ end
45
+
46
+ def parse_message(tag, time)
47
+ record = {}
48
+ date = nil
49
+
50
+ # Skip the message that is output when after flush-logs or restart mysqld.
51
+ # e.g.) /usr/sbin/mysqld, Version: 5.5.28-0ubuntu0.12.04.2-log ((Ubuntu)). started with:
52
+ message = @slowlogs[:"#{tag}"].shift
53
+ while !message.start_with?('#')
54
+ message = @slowlogs[:"#{tag}"].shift
55
+ end
56
+
57
+ if message.start_with?('# Time: ')
58
+ date = Time.parse(message[8..-1].strip)
59
+ message = @slowlogs[:"#{tag}"].shift
60
+ end
61
+
62
+ message =~ /^#? User\@Host:\s+(\S+)\s+\@\s+(\S+).*/
63
+ record[:user] = $1
64
+ record[:host] = $2
65
+ message = @slowlogs[:"#{tag}"].shift
66
+
67
+ message =~ /^# Query_time: ([0-9.]+)\s+Lock_time: ([0-9.]+)\s+Rows_sent: ([0-9.]+)\s+Rows_examined: ([0-9.]+).*/
68
+ record[:query_time] = $1.to_f
69
+ record[:lock_time] = $2.to_f
70
+ record[:rows_sent] = $3.to_i
71
+ record[:rows_examined] = $4.to_i
72
+
73
+ query = []
74
+ @slowlogs[:"#{tag}"].each do |m|
75
+ query << m.strip
76
+ end
77
+ record[:sql] = query.join(' ')
78
+
79
+ if date
80
+ flush_emit(tag, date.to_i, record)
81
+ else
82
+ flush_emit(tag, time, record)
83
+ end
84
+ end
85
+
86
+ def flush_emit(tag, time, record)
87
+ @slowlogs[:"#{tag}"].clear
88
+ _tag = tag.clone
89
+ filter_record(_tag, time, record)
90
+ if tag != _tag
91
+ Fluent::Engine.emit(_tag, time, record)
92
+ else
93
+ $log.warn "Can not emit message because the tag has not changed. Dropped record #{record}"
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,29 @@
1
+ #!/usr/bin/env ruby
2
+ require 'rubygems'
3
+ require 'bundler'
4
+ begin
5
+ Bundler.setup(:default, :development)
6
+ rescue Bundler::BundlerError => e
7
+ $stderr.puts e.message
8
+ $stderr.puts "Run `bundle install` to install missing gems"
9
+ exit e.status_code
10
+ end
11
+ require 'test/unit'
12
+
13
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
14
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
15
+ require 'fluent/test'
16
+ unless ENV.has_key?('VERBOSE')
17
+ nulllogger = Object.new
18
+ nulllogger.instance_eval {|obj|
19
+ def method_missing(method, *args)
20
+ # pass
21
+ end
22
+ }
23
+ $log = nulllogger
24
+ end
25
+
26
+ require 'fluent/plugin/out_mysqlslowquerylog'
27
+
28
+ class Test::Unit::TestCase
29
+ end
@@ -0,0 +1,125 @@
1
+ require 'helper'
2
+
3
+ class MySQLSlowQueryLogOutputTest < Test::Unit::TestCase
4
+ def setup
5
+ Fluent::Test.setup
6
+ end
7
+
8
+ CONFIG = %[
9
+ add_tag_prefix concated.
10
+ ]
11
+
12
+ def create_driver(conf = CONFIG, tag='test')
13
+ Fluent::Test::OutputTestDriver.new(Fluent::MySQLSlowQueryLogOutput, tag).configure(conf)
14
+ end
15
+
16
+ def test_emit
17
+ d1 = create_driver
18
+ d2 = create_driver(CONFIG,'test2')
19
+ d3 = create_driver(CONFIG,'test3')
20
+ d4 = create_driver(CONFIG,'test4')
21
+
22
+ d1.run do
23
+ d1.emit('message' => '/usr/sbin/mysqld, Version: 5.5.28-0ubuntu0.12.04.2-log ((Ubuntu)). started with:')
24
+ d1.emit('message' => 'Tcp port: 3306 Unix socket: /var/run/mysqld/mysqld.sock')
25
+ d1.emit('message' => 'Time Id Command Argument')
26
+ d1.emit('message' => '# Time: 130105 16:43:42')
27
+ d1.emit('message' => '# User@Host: debian-sys-maint[debian-sys-maint] @ localhost []')
28
+ d1.emit('message' => '# Query_time: 0.000167 Lock_time: 0.000057 Rows_sent: 1 Rows_examined: 7')
29
+ d1.emit('message' => 'SET timestamp=1357371822;')
30
+ d1.emit('message' => "SELECT count(*) FROM mysql.user WHERE user='root' and password='';")
31
+ end
32
+
33
+ d2.run do
34
+ d2.emit('message' => '# User@Host: debian-sys-maint[debian-sys-maint] @ localhost []')
35
+ d2.emit('message' => '# Query_time: 0.002998 Lock_time: 0.000078 Rows_sent: 31 Rows_examined: 81')
36
+ d2.emit('message' => 'SET timestamp=61357371822;')
37
+ d2.emit('message' => "select concat('select count(*) into @discard from `',")
38
+ d2.emit('message' => " TABLE_SCHEMA, '`.`', TABLE_NAME, '`')")
39
+ d2.emit('message' => " from information_schema.TABLES where ENGINE='MyISAM';")
40
+
41
+ d2.emit('message' => "# Time: 130105 18:04:21")
42
+ d2.emit('message' => "# User@Host: root[root] @ localhost []")
43
+ d2.emit('message' => "# Query_time: 0.000398 Lock_time: 0.000117 Rows_sent: 7 Rows_examined: 7")
44
+ d2.emit('message' => "use mysql;")
45
+ d2.emit('message' => "SET timestamp=1357376661;")
46
+ d2.emit('message' => "select * from user;")
47
+ end
48
+
49
+ d3.run do
50
+ d3.emit('message' => "# User@Host: debian-sys-maint[debian-sys-maint] @ localhost []")
51
+ d3.emit('message' => "# Query_time: 0.014260 Lock_time: 0.000182 Rows_sent: 0 Rows_examined: 808")
52
+
53
+ d4.emit('message' => "# User@Host: debian-sys-maint[debian-sys-maint] @ localhost []")
54
+ d4.emit('message' => "# Query_time: 0.000262 Lock_time: 0.000200 Rows_sent: 0 Rows_examined: 0")
55
+ d4.emit('message' => "SET timestamp=1357371822;")
56
+ d4.emit('message' => "select count(*) into @discard from `information_schema`.`EVENTS`;")
57
+
58
+ d3.emit('message' => "SET timestamp=1357371822;")
59
+ d3.emit('message' => "select count(*) into @discard from `information_schema`.`COLUMNS`;")
60
+ end
61
+
62
+ assert_equal 1, d1.emits.size
63
+ assert_equal 2, d2.emits.size
64
+ assert_equal 2, d3.emits.size
65
+
66
+ assert_equal '2013-01-05 16:43:42 +0900', Time.at(d1.emits[0][1]).to_s
67
+ assert_equal '2013-01-05 18:04:21 +0900', Time.at(d2.emits[1][1]).to_s
68
+
69
+ assert_equal 'concated.test', d1.emits[0][0]
70
+ assert_equal 'concated.test2', d2.emits[0][0]
71
+ assert_equal 'concated.test2', d2.emits[1][0]
72
+ assert_equal 'concated.test4', d3.emits[0][0]
73
+ assert_equal 'concated.test3', d3.emits[1][0]
74
+
75
+ assert_equal({
76
+ :user=>"debian-sys-maint[debian-sys-maint]",
77
+ :host=>"localhost",
78
+ :query_time=>0.000167,
79
+ :lock_time=>5.7e-05,
80
+ :rows_sent=>1,
81
+ :rows_examined=>7,
82
+ :sql=>"SET timestamp=1357371822; SELECT count(*) FROM mysql.user WHERE user='root' and password='';"
83
+ }, d1.emits[0][2])
84
+
85
+ assert_equal({
86
+ :user=>"debian-sys-maint[debian-sys-maint]",
87
+ :host=>"localhost",
88
+ :query_time=>0.002998,
89
+ :lock_time=>7.8e-05,
90
+ :rows_sent=>31,
91
+ :rows_examined=>81,
92
+ :sql=>"SET timestamp=61357371822; select concat('select count(*) into @discard from `', TABLE_SCHEMA, '`.`', TABLE_NAME, '`') from information_schema.TABLES where ENGINE='MyISAM';"
93
+ }, d2.emits[0][2])
94
+
95
+ assert_equal({
96
+ :user=>"root[root]",
97
+ :host=>"localhost",
98
+ :query_time=>0.000398,
99
+ :lock_time=>0.000117,
100
+ :rows_sent=>7,
101
+ :rows_examined=>7,
102
+ :sql=>"use mysql; SET timestamp=1357376661; select * from user;"
103
+ }, d2.emits[1][2])
104
+
105
+ assert_equal({
106
+ :user=>"debian-sys-maint[debian-sys-maint]",
107
+ :host=>"localhost",
108
+ :query_time=>0.000262,
109
+ :lock_time=>0.0002,
110
+ :rows_sent=>0,
111
+ :rows_examined=>0,
112
+ :sql=>"SET timestamp=1357371822; select count(*) into @discard from `information_schema`.`EVENTS`;"
113
+ }, d3.emits[0][2])
114
+
115
+ assert_equal({
116
+ :user=>"debian-sys-maint[debian-sys-maint]",
117
+ :host=>"localhost",
118
+ :query_time=>0.01426,
119
+ :lock_time=>0.000182,
120
+ :rows_sent=>0,
121
+ :rows_examined=>808,
122
+ :sql=>"SET timestamp=1357371822; select count(*) into @discard from `information_schema`.`COLUMNS`;"
123
+ }, d3.emits[1][2])
124
+ end
125
+ end
metadata ADDED
@@ -0,0 +1,88 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-mysqlslowquerylog
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Satoshi SUZUKI
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2013-01-07 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: fluentd
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :development
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: '0'
30
+ - !ruby/object:Gem::Dependency
31
+ name: fluentd
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: '0'
38
+ type: :runtime
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '0'
46
+ description: Fluentd plugin to concat MySQL slowquerylog.
47
+ email:
48
+ - studio3104.com@gmail.com
49
+ executables: []
50
+ extensions: []
51
+ extra_rdoc_files: []
52
+ files:
53
+ - .gitignore
54
+ - Gemfile
55
+ - LICENSE.txt
56
+ - README.md
57
+ - Rakefile
58
+ - fluent-plugin-mysqlslowquerylog.gemspec
59
+ - lib/fluent/plugin/out_mysqlslowquerylog.rb
60
+ - test/helper.rb
61
+ - test/plugin/test_out_mysqlslowquerylog.rb
62
+ homepage: https://github.com/studio3104/fluent-plugin-mysqlslowquerylog
63
+ licenses: []
64
+ post_install_message:
65
+ rdoc_options: []
66
+ require_paths:
67
+ - lib
68
+ required_ruby_version: !ruby/object:Gem::Requirement
69
+ none: false
70
+ requirements:
71
+ - - ! '>='
72
+ - !ruby/object:Gem::Version
73
+ version: '0'
74
+ required_rubygems_version: !ruby/object:Gem::Requirement
75
+ none: false
76
+ requirements:
77
+ - - ! '>='
78
+ - !ruby/object:Gem::Version
79
+ version: '0'
80
+ requirements: []
81
+ rubyforge_project:
82
+ rubygems_version: 1.8.23
83
+ signing_key:
84
+ specification_version: 3
85
+ summary: Fluentd plugin to concat MySQL slowquerylog.
86
+ test_files:
87
+ - test/helper.rb
88
+ - test/plugin/test_out_mysqlslowquerylog.rb