fluent-plugin-mysqlslowquerylog 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. data/README.md +64 -1
  2. data/fluent-plugin-mysqlslowquerylog.gemspec +1 -3
  3. data/lib/fluent/plugin/out_mysqlslowquerylog.rb +19 -29
  4. data/test/helper.rb +0 -1
  5. data/test/plugin/test_out_mysqlslowquerylog.rb +55 -51
  6. metadata +2 -2
data/README.md CHANGED
@@ -2,10 +2,73 @@
2
2
 
3
3
  ## Component
4
4
 
5
- ## Synopsis
5
+ ### MySQL Slow Query Log Output
6
+
7
+ Fluentd plugin to concat MySQL slowquerylog.
6
8
 
7
9
  ## Configuration
8
10
 
11
+ Input Messages (Slow Query Log)
12
+ ```
13
+ # Time: 130107 11:36:21
14
+ # User@Host: root[root] @ localhost []
15
+ # Query_time: 0.000378 Lock_time: 0.000111 Rows_sent: 7 Rows_examined: 7
16
+ SET timestamp=1357526181;
17
+ select * from user;
18
+ # Time: 130107 11:38:47
19
+ # User@Host: root[root] @ localhost []
20
+ # Query_time: 0.002142 Lock_time: 0.000166 Rows_sent: 142 Rows_examined: 142
21
+ use information_schema;
22
+ SET timestamp=1357526327;
23
+ select * from INNODB_BUFFER_PAGE_LRU;
24
+ ```
25
+
26
+ Output Messages
27
+ ```
28
+ 2013-01-07T11:36:21+09:00 cocatenated.mysql.slowlog {"user":"root[root]","host":"localhost","query_time":0.000378,"lock_time":0.000111,"rows_sent":7,"rows_examined":7,"sql":"SET timestamp=1357526181; select * from user;"}
29
+ 2013-01-07T11:38:47+09:00 cocatenated.mysql.slowlog {"user":"root[root]","host":"localhost","query_time":0.002142,"lock_time":0.000166,"rows_sent":142,"rows_examined":142,"sql":"use information_schema; SET timestamp=1357526327; select * from INNODB_BUFFER_PAGE_LRU;"}
30
+ ```
31
+
32
+ ### Example Settings
33
+ sender (fluent-agent-lite)
34
+ ```
35
+ TAG_PREFIX="mysql"
36
+ LOGS=$(cat <<"EOF"
37
+ slowlog.db01 /var/log/mysql/mysql-slow.log
38
+ EOF
39
+ )
40
+ PRIMARY_SERVER="log_server:24224"
41
+ ```
42
+
43
+ sender (td-agent)
44
+ ```
45
+ <source>
46
+ type tail
47
+ path /var/log/mysql/mysql-slow.log
48
+ format /^(?<message>.+)$/
49
+ tag mysql.slowlog.db01
50
+ </source>
51
+ <match>
52
+ type forward
53
+ host log_server
54
+ </match>
55
+ ```
56
+
57
+ reciever
58
+ ```
59
+ <source>
60
+ type forward
61
+ </source>
62
+ <match mysql.slowlog.*>
63
+ type mysqlslowquerylog
64
+ add_tag_prefix cocatenated.
65
+ </match>
66
+ <match cocatenated.mysql.slowlog.*>
67
+ type file
68
+ path /tmp/slowtest
69
+ </match>
70
+ ```
71
+
9
72
  ## Installation
10
73
 
11
74
  Add this line to your application's Gemfile:
data/fluent-plugin-mysqlslowquerylog.gemspec CHANGED
@@ -1,10 +1,8 @@
1
1
  # -*- encoding: utf-8 -*-
2
- lib = File.expand_path('../lib', __FILE__)
3
- $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
2
 
5
3
  Gem::Specification.new do |gem|
6
4
  gem.name = "fluent-plugin-mysqlslowquerylog"
7
- gem.version = "0.0.1"
5
+ gem.version = "0.0.2"
8
6
  gem.authors = ["Satoshi SUZUKI"]
9
7
  gem.email = ["studio3104.com@gmail.com"]
10
8
  gem.description = %q{Fluentd plugin to concat MySQL slowquerylog.}
data/lib/fluent/plugin/out_mysqlslowquerylog.rb CHANGED
@@ -2,16 +2,12 @@ class Fluent::MySQLSlowQueryLogOutput < Fluent::Output
2
2
  Fluent::Plugin.register_output('mysqlslowquerylog', self)
3
3
  include Fluent::HandleTagNameMixin
4
4
 
5
- config_param :explain, :bool, :default => false
6
- config_param :dbuser, :string, :default => nil
7
- config_param :dbpassword, :string, :default => nil
8
-
9
5
  def configure(conf)
10
6
  super
11
- @slowlogs = Hash.new
7
+ @slowlogs = {}
12
8
 
13
9
  if !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix
14
- raise ConfigError, "out_slowquery: At least one of option, remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix is required to be set."
10
+ raise ConfigError, "out_myslowquerylog: At least one of option, remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix is required to be set."
15
11
  end
16
12
  end
17
13
 
@@ -25,7 +21,7 @@ class Fluent::MySQLSlowQueryLogOutput < Fluent::Output
25
21
 
26
22
  def emit(tag, es, chain)
27
23
  if !@slowlogs[:"#{tag}"]
28
- @slowlogs[:"#{tag}"] = Array.new
24
+ @slowlogs[:"#{tag}"] = []
29
25
  end
30
26
  es.each do |time, record|
31
27
  concat_messages(tag, time, record)
@@ -37,50 +33,44 @@ class Fluent::MySQLSlowQueryLogOutput < Fluent::Output
37
33
  def concat_messages(tag, time, record)
38
34
  record.each do |key, value|
39
35
  @slowlogs[:"#{tag}"] << value
40
- if value !~ /^set timestamp=\d+\;$/i && value !~ /^use /i && value.end_with?(';')
36
+ if value.end_with?(';') && !value.upcase.start_with?('USE ', 'SET TIMESTAMP=')
41
37
  parse_message(tag, time)
42
38
  end
43
39
  end
44
40
  end
45
41
 
42
+ REGEX1 = /^#? User\@Host:\s+(\S+)\s+\@\s+(\S+).*/
43
+ REGEX2 = /^# Query_time: ([0-9.]+)\s+Lock_time: ([0-9.]+)\s+Rows_sent: ([0-9.]+)\s+Rows_examined: ([0-9.]+).*/
46
44
  def parse_message(tag, time)
47
45
  record = {}
48
46
  date = nil
49
47
 
50
48
  # Skip the message that is output when after flush-logs or restart mysqld.
51
49
  # e.g.) /usr/sbin/mysqld, Version: 5.5.28-0ubuntu0.12.04.2-log ((Ubuntu)). started with:
52
- message = @slowlogs[:"#{tag}"].shift
53
- while !message.start_with?('#')
50
+ begin
54
51
  message = @slowlogs[:"#{tag}"].shift
55
- end
52
+ end while !message.start_with?('#')
56
53
 
57
54
  if message.start_with?('# Time: ')
58
55
  date = Time.parse(message[8..-1].strip)
59
56
  message = @slowlogs[:"#{tag}"].shift
60
57
  end
61
58
 
62
- message =~ /^#? User\@Host:\s+(\S+)\s+\@\s+(\S+).*/
63
- record[:user] = $1
64
- record[:host] = $2
59
+ message =~ REGEX1
60
+ record['user'] = $1
61
+ record['host'] = $2
65
62
  message = @slowlogs[:"#{tag}"].shift
66
63
 
67
- message =~ /^# Query_time: ([0-9.]+)\s+Lock_time: ([0-9.]+)\s+Rows_sent: ([0-9.]+)\s+Rows_examined: ([0-9.]+).*/
68
- record[:query_time] = $1.to_f
69
- record[:lock_time] = $2.to_f
70
- record[:rows_sent] = $3.to_i
71
- record[:rows_examined] = $4.to_i
64
+ message =~ REGEX2
65
+ record['query_time'] = $1.to_f
66
+ record['lock_time'] = $2.to_f
67
+ record['rows_sent'] = $3.to_i
68
+ record['rows_examined'] = $4.to_i
72
69
 
73
- query = []
74
- @slowlogs[:"#{tag}"].each do |m|
75
- query << m.strip
76
- end
77
- record[:sql] = query.join(' ')
70
+ record['sql'] = @slowlogs[:"#{tag}"].map {|m| m.strip}.join(' ')
78
71
 
79
- if date
80
- flush_emit(tag, date.to_i, record)
81
- else
82
- flush_emit(tag, time, record)
83
- end
72
+ time = date.to_i if date
73
+ flush_emit(tag, time, record)
84
74
  end
85
75
 
86
76
  def flush_emit(tag, time, record)
data/test/helper.rb CHANGED
@@ -1,4 +1,3 @@
1
- #!/usr/bin/env ruby
2
1
  require 'rubygems'
3
2
  require 'bundler'
4
3
  begin
data/test/plugin/test_out_mysqlslowquerylog.rb CHANGED
@@ -6,7 +6,7 @@ class MySQLSlowQueryLogOutputTest < Test::Unit::TestCase
6
6
  end
7
7
 
8
8
  CONFIG = %[
9
- add_tag_prefix concated.
9
+ add_tag_prefix cocatenated.
10
10
  ]
11
11
 
12
12
  def create_driver(conf = CONFIG, tag='test')
@@ -14,26 +14,28 @@ class MySQLSlowQueryLogOutputTest < Test::Unit::TestCase
14
14
  end
15
15
 
16
16
  def test_emit
17
+ 1.times do
18
+
17
19
  d1 = create_driver
18
20
  d2 = create_driver(CONFIG,'test2')
19
21
  d3 = create_driver(CONFIG,'test3')
20
22
  d4 = create_driver(CONFIG,'test4')
21
23
 
22
24
  d1.run do
23
- d1.emit('message' => '/usr/sbin/mysqld, Version: 5.5.28-0ubuntu0.12.04.2-log ((Ubuntu)). started with:')
24
- d1.emit('message' => 'Tcp port: 3306 Unix socket: /var/run/mysqld/mysqld.sock')
25
- d1.emit('message' => 'Time Id Command Argument')
26
- d1.emit('message' => '# Time: 130105 16:43:42')
27
- d1.emit('message' => '# User@Host: debian-sys-maint[debian-sys-maint] @ localhost []')
28
- d1.emit('message' => '# Query_time: 0.000167 Lock_time: 0.000057 Rows_sent: 1 Rows_examined: 7')
29
- d1.emit('message' => 'SET timestamp=1357371822;')
25
+ d1.emit('message' => "/usr/sbin/mysqld, Version: 5.5.28-0ubuntu0.12.04.2-log ((Ubuntu)). started with:")
26
+ d1.emit('message' => "Tcp port: 3306 Unix socket: /var/run/mysqld/mysqld.sock")
27
+ d1.emit('message' => "Time Id Command Argument")
28
+ d1.emit('message' => "# Time: 130105 16:43:42")
29
+ d1.emit('message' => "# User@Host: debian-sys-maint[debian-sys-maint] @ localhost []")
30
+ d1.emit('message' => "# Query_time: 0.000167 Lock_time: 0.000057 Rows_sent: 1 Rows_examined: 7")
31
+ d1.emit('message' => "SET timestamp=1357371822;")
30
32
  d1.emit('message' => "SELECT count(*) FROM mysql.user WHERE user='root' and password='';")
31
33
  end
32
34
 
33
35
  d2.run do
34
- d2.emit('message' => '# User@Host: debian-sys-maint[debian-sys-maint] @ localhost []')
35
- d2.emit('message' => '# Query_time: 0.002998 Lock_time: 0.000078 Rows_sent: 31 Rows_examined: 81')
36
- d2.emit('message' => 'SET timestamp=61357371822;')
36
+ d2.emit('message' => "# User@Host: debian-sys-maint[debian-sys-maint] @ localhost []")
37
+ d2.emit('message' => "# Query_time: 0.002998 Lock_time: 0.000078 Rows_sent: 31 Rows_examined: 81")
38
+ d2.emit('message' => "SET timestamp=61357371822;")
37
39
  d2.emit('message' => "select concat('select count(*) into @discard from `',")
38
40
  d2.emit('message' => " TABLE_SCHEMA, '`.`', TABLE_NAME, '`')")
39
41
  d2.emit('message' => " from information_schema.TABLES where ENGINE='MyISAM';")
@@ -66,60 +68,62 @@ class MySQLSlowQueryLogOutputTest < Test::Unit::TestCase
66
68
  assert_equal '2013-01-05 16:43:42 +0900', Time.at(d1.emits[0][1]).to_s
67
69
  assert_equal '2013-01-05 18:04:21 +0900', Time.at(d2.emits[1][1]).to_s
68
70
 
69
- assert_equal 'concated.test', d1.emits[0][0]
70
- assert_equal 'concated.test2', d2.emits[0][0]
71
- assert_equal 'concated.test2', d2.emits[1][0]
72
- assert_equal 'concated.test4', d3.emits[0][0]
73
- assert_equal 'concated.test3', d3.emits[1][0]
71
+ assert_equal 'cocatenated.test', d1.emits[0][0]
72
+ assert_equal 'cocatenated.test2', d2.emits[0][0]
73
+ assert_equal 'cocatenated.test2', d2.emits[1][0]
74
+ assert_equal 'cocatenated.test4', d3.emits[0][0]
75
+ assert_equal 'cocatenated.test3', d3.emits[1][0]
74
76
 
75
77
  assert_equal({
76
- :user=>"debian-sys-maint[debian-sys-maint]",
77
- :host=>"localhost",
78
- :query_time=>0.000167,
79
- :lock_time=>5.7e-05,
80
- :rows_sent=>1,
81
- :rows_examined=>7,
82
- :sql=>"SET timestamp=1357371822; SELECT count(*) FROM mysql.user WHERE user='root' and password='';"
78
+ "user" => "debian-sys-maint[debian-sys-maint]",
79
+ "host" => "localhost",
80
+ "query_time" => 0.000167,
81
+ "lock_time" => 5.7e-05,
82
+ "rows_sent" => 1,
83
+ "rows_examined" => 7,
84
+ "sql" => "SET timestamp=1357371822; SELECT count(*) FROM mysql.user WHERE user='root' and password='';"
83
85
  }, d1.emits[0][2])
84
86
 
85
87
  assert_equal({
86
- :user=>"debian-sys-maint[debian-sys-maint]",
87
- :host=>"localhost",
88
- :query_time=>0.002998,
89
- :lock_time=>7.8e-05,
90
- :rows_sent=>31,
91
- :rows_examined=>81,
92
- :sql=>"SET timestamp=61357371822; select concat('select count(*) into @discard from `', TABLE_SCHEMA, '`.`', TABLE_NAME, '`') from information_schema.TABLES where ENGINE='MyISAM';"
88
+ "user" => "debian-sys-maint[debian-sys-maint]",
89
+ "host" => "localhost",
90
+ "query_time" => 0.002998,
91
+ "lock_time" => 7.8e-05,
92
+ "rows_sent" => 31,
93
+ "rows_examined" => 81,
94
+ "sql" => "SET timestamp=61357371822; select concat('select count(*) into @discard from `', TABLE_SCHEMA, '`.`', TABLE_NAME, '`') from information_schema.TABLES where ENGINE='MyISAM';"
93
95
  }, d2.emits[0][2])
94
96
 
95
97
  assert_equal({
96
- :user=>"root[root]",
97
- :host=>"localhost",
98
- :query_time=>0.000398,
99
- :lock_time=>0.000117,
100
- :rows_sent=>7,
101
- :rows_examined=>7,
102
- :sql=>"use mysql; SET timestamp=1357376661; select * from user;"
98
+ "user" => "root[root]",
99
+ "host" => "localhost",
100
+ "query_time" => 0.000398,
101
+ "lock_time" => 0.000117,
102
+ "rows_sent" => 7,
103
+ "rows_examined" => 7,
104
+ "sql" => "use mysql; SET timestamp=1357376661; select * from user;"
103
105
  }, d2.emits[1][2])
104
106
 
105
107
  assert_equal({
106
- :user=>"debian-sys-maint[debian-sys-maint]",
107
- :host=>"localhost",
108
- :query_time=>0.000262,
109
- :lock_time=>0.0002,
110
- :rows_sent=>0,
111
- :rows_examined=>0,
112
- :sql=>"SET timestamp=1357371822; select count(*) into @discard from `information_schema`.`EVENTS`;"
108
+ "user" => "debian-sys-maint[debian-sys-maint]",
109
+ "host" => "localhost",
110
+ "query_time" => 0.000262,
111
+ "lock_time" => 0.0002,
112
+ "rows_sent" => 0,
113
+ "rows_examined" => 0,
114
+ "sql" => "SET timestamp=1357371822; select count(*) into @discard from `information_schema`.`EVENTS`;"
113
115
  }, d3.emits[0][2])
114
116
 
115
117
  assert_equal({
116
- :user=>"debian-sys-maint[debian-sys-maint]",
117
- :host=>"localhost",
118
- :query_time=>0.01426,
119
- :lock_time=>0.000182,
120
- :rows_sent=>0,
121
- :rows_examined=>808,
122
- :sql=>"SET timestamp=1357371822; select count(*) into @discard from `information_schema`.`COLUMNS`;"
118
+ "user" => "debian-sys-maint[debian-sys-maint]",
119
+ "host" => "localhost",
120
+ "query_time" => 0.01426,
121
+ "lock_time" => 0.000182,
122
+ "rows_sent" => 0,
123
+ "rows_examined" => 808,
124
+ "sql" => "SET timestamp=1357371822; select count(*) into @discard from `information_schema`.`COLUMNS`;"
123
125
  }, d3.emits[1][2])
126
+
127
+ end
124
128
  end
125
129
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-mysqlslowquerylog
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-01-07 00:00:00.000000000 Z
12
+ date: 2013-01-11 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd