fluent-plugin-mysqlslowquerylog 0.0.1 → 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (6) hide show
  1. data/README.md +64 -1
  2. data/fluent-plugin-mysqlslowquerylog.gemspec +1 -3
  3. data/lib/fluent/plugin/out_mysqlslowquerylog.rb +19 -29
  4. data/test/helper.rb +0 -1
  5. data/test/plugin/test_out_mysqlslowquerylog.rb +55 -51
  6. metadata +2 -2
data/README.md CHANGED
@@ -2,10 +2,73 @@
2
2
 
3
3
  ## Component
4
4
 
5
- ## Synopsis
5
+ ### MySQL Slow Query Log Output
6
+
7
+ Fluentd plugin to concat MySQL slowquerylog.
6
8
 
7
9
  ## Configuration
8
10
 
11
+ Input Messages (Slow Query Log)
12
+ ```
13
+ # Time: 130107 11:36:21
14
+ # User@Host: root[root] @ localhost []
15
+ # Query_time: 0.000378 Lock_time: 0.000111 Rows_sent: 7 Rows_examined: 7
16
+ SET timestamp=1357526181;
17
+ select * from user;
18
+ # Time: 130107 11:38:47
19
+ # User@Host: root[root] @ localhost []
20
+ # Query_time: 0.002142 Lock_time: 0.000166 Rows_sent: 142 Rows_examined: 142
21
+ use information_schema;
22
+ SET timestamp=1357526327;
23
+ select * from INNODB_BUFFER_PAGE_LRU;
24
+ ```
25
+
26
+ Output Messages
27
+ ```
28
+ 2013-01-07T11:36:21+09:00 cocatenated.mysql.slowlog {"user":"root[root]","host":"localhost","query_time":0.000378,"lock_time":0.000111,"rows_sent":7,"rows_examined":7,"sql":"SET timestamp=1357526181; select * from user;"}
29
+ 2013-01-07T11:38:47+09:00 cocatenated.mysql.slowlog {"user":"root[root]","host":"localhost","query_time":0.002142,"lock_time":0.000166,"rows_sent":142,"rows_examined":142,"sql":"use information_schema; SET timestamp=1357526327; select * from INNODB_BUFFER_PAGE_LRU;"}
30
+ ```
31
+
32
+ ### Example Settings
33
+ sender (fluent-agent-lite)
34
+ ```
35
+ TAG_PREFIX="mysql"
36
+ LOGS=$(cat <<"EOF"
37
+ slowlog.db01 /var/log/mysql/mysql-slow.log
38
+ EOF
39
+ )
40
+ PRIMARY_SERVER="log_server:24224"
41
+ ```
42
+
43
+ sender (td-agent)
44
+ ```
45
+ <source>
46
+ type tail
47
+ path /var/log/mysql/mysql-slow.log
48
+ format /^(?<message>.+)$/
49
+ tag mysql.slowlog.db01
50
+ </source>
51
+ <match>
52
+ type forward
53
+ host log_server
54
+ </match>
55
+ ```
56
+
57
+ reciever
58
+ ```
59
+ <source>
60
+ type forward
61
+ </source>
62
+ <match mysql.slowlog.*>
63
+ type mysqlslowquerylog
64
+ add_tag_prefix cocatenated.
65
+ </match>
66
+ <match cocatenated.mysql.slowlog.*>
67
+ type file
68
+ path /tmp/slowtest
69
+ </match>
70
+ ```
71
+
9
72
  ## Installation
10
73
 
11
74
  Add this line to your application's Gemfile:
data/fluent-plugin-mysqlslowquerylog.gemspec CHANGED
@@ -1,10 +1,8 @@
1
1
  # -*- encoding: utf-8 -*-
2
- lib = File.expand_path('../lib', __FILE__)
3
- $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
2
 
5
3
  Gem::Specification.new do |gem|
6
4
  gem.name = "fluent-plugin-mysqlslowquerylog"
7
- gem.version = "0.0.1"
5
+ gem.version = "0.0.2"
8
6
  gem.authors = ["Satoshi SUZUKI"]
9
7
  gem.email = ["studio3104.com@gmail.com"]
10
8
  gem.description = %q{Fluentd plugin to concat MySQL slowquerylog.}
data/lib/fluent/plugin/out_mysqlslowquerylog.rb CHANGED
@@ -2,16 +2,12 @@ class Fluent::MySQLSlowQueryLogOutput < Fluent::Output
2
2
  Fluent::Plugin.register_output('mysqlslowquerylog', self)
3
3
  include Fluent::HandleTagNameMixin
4
4
 
5
- config_param :explain, :bool, :default => false
6
- config_param :dbuser, :string, :default => nil
7
- config_param :dbpassword, :string, :default => nil
8
-
9
5
  def configure(conf)
10
6
  super
11
- @slowlogs = Hash.new
7
+ @slowlogs = {}
12
8
 
13
9
  if !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix
14
- raise ConfigError, "out_slowquery: At least one of option, remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix is required to be set."
10
+ raise ConfigError, "out_myslowquerylog: At least one of option, remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix is required to be set."
15
11
  end
16
12
  end
17
13
 
@@ -25,7 +21,7 @@ class Fluent::MySQLSlowQueryLogOutput < Fluent::Output
25
21
 
26
22
  def emit(tag, es, chain)
27
23
  if !@slowlogs[:"#{tag}"]
28
- @slowlogs[:"#{tag}"] = Array.new
24
+ @slowlogs[:"#{tag}"] = []
29
25
  end
30
26
  es.each do |time, record|
31
27
  concat_messages(tag, time, record)
@@ -37,50 +33,44 @@ class Fluent::MySQLSlowQueryLogOutput < Fluent::Output
37
33
  def concat_messages(tag, time, record)
38
34
  record.each do |key, value|
39
35
  @slowlogs[:"#{tag}"] << value
40
- if value !~ /^set timestamp=\d+\;$/i && value !~ /^use /i && value.end_with?(';')
36
+ if value.end_with?(';') && !value.upcase.start_with?('USE ', 'SET TIMESTAMP=')
41
37
  parse_message(tag, time)
42
38
  end
43
39
  end
44
40
  end
45
41
 
42
+ REGEX1 = /^#? User\@Host:\s+(\S+)\s+\@\s+(\S+).*/
43
+ REGEX2 = /^# Query_time: ([0-9.]+)\s+Lock_time: ([0-9.]+)\s+Rows_sent: ([0-9.]+)\s+Rows_examined: ([0-9.]+).*/
46
44
  def parse_message(tag, time)
47
45
  record = {}
48
46
  date = nil
49
47
 
50
48
  # Skip the message that is output when after flush-logs or restart mysqld.
51
49
  # e.g.) /usr/sbin/mysqld, Version: 5.5.28-0ubuntu0.12.04.2-log ((Ubuntu)). started with:
52
- message = @slowlogs[:"#{tag}"].shift
53
- while !message.start_with?('#')
50
+ begin
54
51
  message = @slowlogs[:"#{tag}"].shift
55
- end
52
+ end while !message.start_with?('#')
56
53
 
57
54
  if message.start_with?('# Time: ')
58
55
  date = Time.parse(message[8..-1].strip)
59
56
  message = @slowlogs[:"#{tag}"].shift
60
57
  end
61
58
 
62
- message =~ /^#? User\@Host:\s+(\S+)\s+\@\s+(\S+).*/
63
- record[:user] = $1
64
- record[:host] = $2
59
+ message =~ REGEX1
60
+ record['user'] = $1
61
+ record['host'] = $2
65
62
  message = @slowlogs[:"#{tag}"].shift
66
63
 
67
- message =~ /^# Query_time: ([0-9.]+)\s+Lock_time: ([0-9.]+)\s+Rows_sent: ([0-9.]+)\s+Rows_examined: ([0-9.]+).*/
68
- record[:query_time] = $1.to_f
69
- record[:lock_time] = $2.to_f
70
- record[:rows_sent] = $3.to_i
71
- record[:rows_examined] = $4.to_i
64
+ message =~ REGEX2
65
+ record['query_time'] = $1.to_f
66
+ record['lock_time'] = $2.to_f
67
+ record['rows_sent'] = $3.to_i
68
+ record['rows_examined'] = $4.to_i
72
69
 
73
- query = []
74
- @slowlogs[:"#{tag}"].each do |m|
75
- query << m.strip
76
- end
77
- record[:sql] = query.join(' ')
70
+ record['sql'] = @slowlogs[:"#{tag}"].map {|m| m.strip}.join(' ')
78
71
 
79
- if date
80
- flush_emit(tag, date.to_i, record)
81
- else
82
- flush_emit(tag, time, record)
83
- end
72
+ time = date.to_i if date
73
+ flush_emit(tag, time, record)
84
74
  end
85
75
 
86
76
  def flush_emit(tag, time, record)
data/test/helper.rb CHANGED
@@ -1,4 +1,3 @@
1
- #!/usr/bin/env ruby
2
1
  require 'rubygems'
3
2
  require 'bundler'
4
3
  begin
data/test/plugin/test_out_mysqlslowquerylog.rb CHANGED
@@ -6,7 +6,7 @@ class MySQLSlowQueryLogOutputTest < Test::Unit::TestCase
6
6
  end
7
7
 
8
8
  CONFIG = %[
9
- add_tag_prefix concated.
9
+ add_tag_prefix cocatenated.
10
10
  ]
11
11
 
12
12
  def create_driver(conf = CONFIG, tag='test')
@@ -14,26 +14,28 @@ class MySQLSlowQueryLogOutputTest < Test::Unit::TestCase
14
14
  end
15
15
 
16
16
  def test_emit
17
+ 1.times do
18
+
17
19
  d1 = create_driver
18
20
  d2 = create_driver(CONFIG,'test2')
19
21
  d3 = create_driver(CONFIG,'test3')
20
22
  d4 = create_driver(CONFIG,'test4')
21
23
 
22
24
  d1.run do
23
- d1.emit('message' => '/usr/sbin/mysqld, Version: 5.5.28-0ubuntu0.12.04.2-log ((Ubuntu)). started with:')
24
- d1.emit('message' => 'Tcp port: 3306 Unix socket: /var/run/mysqld/mysqld.sock')
25
- d1.emit('message' => 'Time Id Command Argument')
26
- d1.emit('message' => '# Time: 130105 16:43:42')
27
- d1.emit('message' => '# User@Host: debian-sys-maint[debian-sys-maint] @ localhost []')
28
- d1.emit('message' => '# Query_time: 0.000167 Lock_time: 0.000057 Rows_sent: 1 Rows_examined: 7')
29
- d1.emit('message' => 'SET timestamp=1357371822;')
25
+ d1.emit('message' => "/usr/sbin/mysqld, Version: 5.5.28-0ubuntu0.12.04.2-log ((Ubuntu)). started with:")
26
+ d1.emit('message' => "Tcp port: 3306 Unix socket: /var/run/mysqld/mysqld.sock")
27
+ d1.emit('message' => "Time Id Command Argument")
28
+ d1.emit('message' => "# Time: 130105 16:43:42")
29
+ d1.emit('message' => "# User@Host: debian-sys-maint[debian-sys-maint] @ localhost []")
30
+ d1.emit('message' => "# Query_time: 0.000167 Lock_time: 0.000057 Rows_sent: 1 Rows_examined: 7")
31
+ d1.emit('message' => "SET timestamp=1357371822;")
30
32
  d1.emit('message' => "SELECT count(*) FROM mysql.user WHERE user='root' and password='';")
31
33
  end
32
34
 
33
35
  d2.run do
34
- d2.emit('message' => '# User@Host: debian-sys-maint[debian-sys-maint] @ localhost []')
35
- d2.emit('message' => '# Query_time: 0.002998 Lock_time: 0.000078 Rows_sent: 31 Rows_examined: 81')
36
- d2.emit('message' => 'SET timestamp=61357371822;')
36
+ d2.emit('message' => "# User@Host: debian-sys-maint[debian-sys-maint] @ localhost []")
37
+ d2.emit('message' => "# Query_time: 0.002998 Lock_time: 0.000078 Rows_sent: 31 Rows_examined: 81")
38
+ d2.emit('message' => "SET timestamp=61357371822;")
37
39
  d2.emit('message' => "select concat('select count(*) into @discard from `',")
38
40
  d2.emit('message' => " TABLE_SCHEMA, '`.`', TABLE_NAME, '`')")
39
41
  d2.emit('message' => " from information_schema.TABLES where ENGINE='MyISAM';")
@@ -66,60 +68,62 @@ class MySQLSlowQueryLogOutputTest < Test::Unit::TestCase
66
68
  assert_equal '2013-01-05 16:43:42 +0900', Time.at(d1.emits[0][1]).to_s
67
69
  assert_equal '2013-01-05 18:04:21 +0900', Time.at(d2.emits[1][1]).to_s
68
70
 
69
- assert_equal 'concated.test', d1.emits[0][0]
70
- assert_equal 'concated.test2', d2.emits[0][0]
71
- assert_equal 'concated.test2', d2.emits[1][0]
72
- assert_equal 'concated.test4', d3.emits[0][0]
73
- assert_equal 'concated.test3', d3.emits[1][0]
71
+ assert_equal 'cocatenated.test', d1.emits[0][0]
72
+ assert_equal 'cocatenated.test2', d2.emits[0][0]
73
+ assert_equal 'cocatenated.test2', d2.emits[1][0]
74
+ assert_equal 'cocatenated.test4', d3.emits[0][0]
75
+ assert_equal 'cocatenated.test3', d3.emits[1][0]
74
76
 
75
77
  assert_equal({
76
- :user=>"debian-sys-maint[debian-sys-maint]",
77
- :host=>"localhost",
78
- :query_time=>0.000167,
79
- :lock_time=>5.7e-05,
80
- :rows_sent=>1,
81
- :rows_examined=>7,
82
- :sql=>"SET timestamp=1357371822; SELECT count(*) FROM mysql.user WHERE user='root' and password='';"
78
+ "user" => "debian-sys-maint[debian-sys-maint]",
79
+ "host" => "localhost",
80
+ "query_time" => 0.000167,
81
+ "lock_time" => 5.7e-05,
82
+ "rows_sent" => 1,
83
+ "rows_examined" => 7,
84
+ "sql" => "SET timestamp=1357371822; SELECT count(*) FROM mysql.user WHERE user='root' and password='';"
83
85
  }, d1.emits[0][2])
84
86
 
85
87
  assert_equal({
86
- :user=>"debian-sys-maint[debian-sys-maint]",
87
- :host=>"localhost",
88
- :query_time=>0.002998,
89
- :lock_time=>7.8e-05,
90
- :rows_sent=>31,
91
- :rows_examined=>81,
92
- :sql=>"SET timestamp=61357371822; select concat('select count(*) into @discard from `', TABLE_SCHEMA, '`.`', TABLE_NAME, '`') from information_schema.TABLES where ENGINE='MyISAM';"
88
+ "user" => "debian-sys-maint[debian-sys-maint]",
89
+ "host" => "localhost",
90
+ "query_time" => 0.002998,
91
+ "lock_time" => 7.8e-05,
92
+ "rows_sent" => 31,
93
+ "rows_examined" => 81,
94
+ "sql" => "SET timestamp=61357371822; select concat('select count(*) into @discard from `', TABLE_SCHEMA, '`.`', TABLE_NAME, '`') from information_schema.TABLES where ENGINE='MyISAM';"
93
95
  }, d2.emits[0][2])
94
96
 
95
97
  assert_equal({
96
- :user=>"root[root]",
97
- :host=>"localhost",
98
- :query_time=>0.000398,
99
- :lock_time=>0.000117,
100
- :rows_sent=>7,
101
- :rows_examined=>7,
102
- :sql=>"use mysql; SET timestamp=1357376661; select * from user;"
98
+ "user" => "root[root]",
99
+ "host" => "localhost",
100
+ "query_time" => 0.000398,
101
+ "lock_time" => 0.000117,
102
+ "rows_sent" => 7,
103
+ "rows_examined" => 7,
104
+ "sql" => "use mysql; SET timestamp=1357376661; select * from user;"
103
105
  }, d2.emits[1][2])
104
106
 
105
107
  assert_equal({
106
- :user=>"debian-sys-maint[debian-sys-maint]",
107
- :host=>"localhost",
108
- :query_time=>0.000262,
109
- :lock_time=>0.0002,
110
- :rows_sent=>0,
111
- :rows_examined=>0,
112
- :sql=>"SET timestamp=1357371822; select count(*) into @discard from `information_schema`.`EVENTS`;"
108
+ "user" => "debian-sys-maint[debian-sys-maint]",
109
+ "host" => "localhost",
110
+ "query_time" => 0.000262,
111
+ "lock_time" => 0.0002,
112
+ "rows_sent" => 0,
113
+ "rows_examined" => 0,
114
+ "sql" => "SET timestamp=1357371822; select count(*) into @discard from `information_schema`.`EVENTS`;"
113
115
  }, d3.emits[0][2])
114
116
 
115
117
  assert_equal({
116
- :user=>"debian-sys-maint[debian-sys-maint]",
117
- :host=>"localhost",
118
- :query_time=>0.01426,
119
- :lock_time=>0.000182,
120
- :rows_sent=>0,
121
- :rows_examined=>808,
122
- :sql=>"SET timestamp=1357371822; select count(*) into @discard from `information_schema`.`COLUMNS`;"
118
+ "user" => "debian-sys-maint[debian-sys-maint]",
119
+ "host" => "localhost",
120
+ "query_time" => 0.01426,
121
+ "lock_time" => 0.000182,
122
+ "rows_sent" => 0,
123
+ "rows_examined" => 808,
124
+ "sql" => "SET timestamp=1357371822; select count(*) into @discard from `information_schema`.`COLUMNS`;"
123
125
  }, d3.emits[1][2])
126
+
127
+ end
124
128
  end
125
129
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-mysqlslowquerylog
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-01-07 00:00:00.000000000 Z
12
+ date: 2013-01-11 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd