groonga-query-log 1.0.2 → 1.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/groonga-query-log-verify-server +23 -0
- data/doc/text/news.md +12 -0
- data/lib/groonga/query-log.rb +1 -0
- data/lib/groonga/query-log/analyzer.rb +23 -13
- data/lib/groonga/query-log/analyzer/reporter.rb +10 -1
- data/lib/groonga/query-log/analyzer/reporter/console.rb +3 -3
- data/lib/groonga/query-log/analyzer/reporter/html.rb +2 -2
- data/lib/groonga/query-log/analyzer/sized-statistics.rb +0 -9
- data/lib/groonga/query-log/analyzer/statistic.rb +5 -2
- data/lib/groonga/query-log/analyzer/streamer.rb +1 -1
- data/lib/groonga/query-log/{commandline-utils.rb → command-line-utils.rb} +0 -0
- data/lib/groonga/query-log/command/detect-memory-leak.rb +1 -0
- data/lib/groonga/query-log/command/replay.rb +1 -0
- data/lib/groonga/query-log/command/verify-server.rb +131 -0
- data/lib/groonga/query-log/extractor.rb +3 -2
- data/lib/groonga/query-log/parser.rb +16 -2
- data/lib/groonga/query-log/server-verifier.rb +188 -0
- data/lib/groonga/query-log/version.rb +1 -1
- metadata +8 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a90caf244ba39ca7b7ba11c85821185a693e4d3e
|
4
|
+
data.tar.gz: b51489a8a105f9c8f5ea348e7a804193b1abcd71
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1e5bf6c8af15419ea6383172cdab9b439a772106368607f219ee2653f5a1e813edb55d26d2aa3705ec5ee29e25a0b0b2274c9dc40227e742247c91f2b6e07b8d
|
7
|
+
data.tar.gz: 2e1317950786b1018c1fa4b5eb9444354ff9447d7d401a99a8a220ccf69be384db19283a6da8c7a465b4d3fcfd497d8df3d4e85a19b28852f81510180805f6fa
|
@@ -0,0 +1,23 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
#
|
4
|
+
# Copyright (C) 2013 Kouhei Sutou <kou@clear-code.com>
|
5
|
+
#
|
6
|
+
# This library is free software; you can redistribute it and/or
|
7
|
+
# modify it under the terms of the GNU Lesser General Public
|
8
|
+
# License as published by the Free Software Foundation; either
|
9
|
+
# version 2.1 of the License, or (at your option) any later version.
|
10
|
+
#
|
11
|
+
# This library is distributed in the hope that it will be useful,
|
12
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
13
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
14
|
+
# Lesser General Public License for more details.
|
15
|
+
#
|
16
|
+
# You should have received a copy of the GNU Lesser General Public
|
17
|
+
# License along with this library; if not, write to the Free Software
|
18
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
19
|
+
|
20
|
+
require "groonga/query-log/command/verify-server"
|
21
|
+
|
22
|
+
verify_server_command = Groonga::QueryLog::Command::VerifyServer.new
|
23
|
+
verify_server_command.run(*ARGV)
|
data/doc/text/news.md
CHANGED
@@ -1,5 +1,17 @@
|
|
1
1
|
# News
|
2
2
|
|
3
|
+
## 1.0.3: 2014-01-06
|
4
|
+
|
5
|
+
### Improvements
|
6
|
+
|
7
|
+
* groonga-query-log-verify-server: Added a command that verifies two
|
8
|
+
servers returns the same response for the same request.
|
9
|
+
(experimental)
|
10
|
+
|
11
|
+
### Fixes
|
12
|
+
|
13
|
+
* groonga-query-log-analyzer: Fixed a bug `--stream` doesn't work.
|
14
|
+
|
3
15
|
## 1.0.2: 2013-11-01
|
4
16
|
|
5
17
|
### Improvements
|
data/lib/groonga/query-log.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
2
|
#
|
3
|
-
# Copyright (C) 2011-
|
3
|
+
# Copyright (C) 2011-2013 Kouhei Sutou <kou@clear-code.com>
|
4
4
|
# Copyright (C) 2012 Haruka Yoshihara <yoshihara@clear-code.com>
|
5
5
|
#
|
6
6
|
# This library is free software; you can redistribute it and/or
|
@@ -19,7 +19,7 @@
|
|
19
19
|
|
20
20
|
require "optparse"
|
21
21
|
require "json"
|
22
|
-
require "groonga/query-log/
|
22
|
+
require "groonga/query-log/command-line-utils"
|
23
23
|
require "groonga/query-log/parser"
|
24
24
|
require "groonga/query-log/analyzer/streamer"
|
25
25
|
require "groonga/query-log/analyzer/sized-statistics"
|
@@ -64,7 +64,6 @@ module Groonga
|
|
64
64
|
dynamic_sort = @options[:dynamic_sort]
|
65
65
|
statistics = SizedStatistics.new
|
66
66
|
statistics.apply_options(@options)
|
67
|
-
parser = Groonga::QueryLog::Parser.new
|
68
67
|
if stream
|
69
68
|
streamer = Streamer.new(create_reporter(statistics))
|
70
69
|
streamer.start
|
@@ -82,18 +81,12 @@ module Groonga
|
|
82
81
|
end
|
83
82
|
end
|
84
83
|
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
parser.parse(ARGF, &process_statistic)
|
84
|
+
begin
|
85
|
+
parse(log_paths, &process_statistic)
|
86
|
+
rescue Interrupt
|
87
|
+
raise unless stream
|
90
88
|
end
|
91
89
|
|
92
|
-
log_paths.each do |log_path|
|
93
|
-
File.open(log_path) do |log|
|
94
|
-
parser.parse(log, &process_statistic)
|
95
|
-
end
|
96
|
-
end
|
97
90
|
if stream
|
98
91
|
streamer.finish
|
99
92
|
return
|
@@ -120,6 +113,7 @@ module Groonga
|
|
120
113
|
@options[:report_summary] = true
|
121
114
|
|
122
115
|
@option_parser = OptionParser.new do |parser|
|
116
|
+
parser.version = VERSION
|
123
117
|
parser.banner += " LOG1 ..."
|
124
118
|
|
125
119
|
parser.on("-n", "--n-entries=N",
|
@@ -232,6 +226,22 @@ module Groonga
|
|
232
226
|
Groonga::QueryLog::StreamConsoleQueryLogReporter.new
|
233
227
|
end
|
234
228
|
end
|
229
|
+
|
230
|
+
def parse(log_paths, &process_statistic)
|
231
|
+
parser = Groonga::QueryLog::Parser.new(@options)
|
232
|
+
if log_paths.empty?
|
233
|
+
unless log_via_stdin?
|
234
|
+
raise(NoInputError, "Error: Please specify input log files.")
|
235
|
+
end
|
236
|
+
parser.parse($stdin, &process_statistic)
|
237
|
+
end
|
238
|
+
|
239
|
+
log_paths.each do |log_path|
|
240
|
+
File.open(log_path) do |log|
|
241
|
+
parser.parse(log, &process_statistic)
|
242
|
+
end
|
243
|
+
end
|
244
|
+
end
|
235
245
|
end
|
236
246
|
end
|
237
247
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
2
|
#
|
3
|
-
# Copyright (C) 2011-
|
3
|
+
# Copyright (C) 2011-2013 Kouhei Sutou <kou@clear-code.com>
|
4
4
|
# Copyright (C) 2012 Haruka Yoshihara <yoshihara@clear-code.com>
|
5
5
|
#
|
6
6
|
# This library is free software; you can redistribute it and/or
|
@@ -24,10 +24,15 @@ module Groonga
|
|
24
24
|
include Enumerable
|
25
25
|
|
26
26
|
attr_reader :output
|
27
|
+
attr_accessor :slow_operation_threshold, :slow_response_threshold
|
27
28
|
def initialize(statistics)
|
28
29
|
@statistics = statistics
|
29
30
|
@report_summary = true
|
30
31
|
@output = $stdout
|
32
|
+
@slow_operation_threshold =
|
33
|
+
Statistic::DEFAULT_SLOW_OPERATION_THRESHOLD
|
34
|
+
@slow_response_threshold =
|
35
|
+
Statistic::DEFAULT_SLOW_RESPONSE_THRESHOLD
|
31
36
|
end
|
32
37
|
|
33
38
|
def apply_options(options)
|
@@ -35,6 +40,10 @@ module Groonga
|
|
35
40
|
unless options[:report_summary].nil?
|
36
41
|
@report_summary = options[:report_summary]
|
37
42
|
end
|
43
|
+
@slow_operation_threshold =
|
44
|
+
options[:slow_operation_threshold] || @slow_operation_threshold
|
45
|
+
@slow_response_threshold =
|
46
|
+
options[:slow_response_threshold] || @slow_response_threshold
|
38
47
|
end
|
39
48
|
|
40
49
|
def output=(output)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
2
|
#
|
3
|
-
# Copyright (C) 2011-
|
3
|
+
# Copyright (C) 2011-2013 Kouhei Sutou <kou@clear-code.com>
|
4
4
|
# Copyright (C) 2012 Haruka Yoshihara <yoshihara@clear-code.com>
|
5
5
|
#
|
6
6
|
# This library is free software; you can redistribute it and/or
|
@@ -169,8 +169,8 @@ module Groonga
|
|
169
169
|
def report_summary
|
170
170
|
write("Summary:\n")
|
171
171
|
write(" Threshold:\n")
|
172
|
-
write(" slow response : #{@
|
173
|
-
write(" slow operation : #{@
|
172
|
+
write(" slow response : #{@slow_response_threshold}\n")
|
173
|
+
write(" slow operation : #{@slow_operation_threshold}\n")
|
174
174
|
write(" # of responses : #{@statistics.n_responses}\n")
|
175
175
|
write(" # of slow responses : #{@statistics.n_slow_responses}\n")
|
176
176
|
write(" responses/sec : #{@statistics.responses_per_second}\n")
|
@@ -189,11 +189,11 @@ td.name
|
|
189
189
|
<tr><th>Name</th><th>Value</th></tr>
|
190
190
|
<tr>
|
191
191
|
<th>Slow response threshold</th>
|
192
|
-
<td><%= h(@
|
192
|
+
<td><%= h(@slow_response_threshold) %>sec</td>
|
193
193
|
</tr>
|
194
194
|
<tr>
|
195
195
|
<th>Slow operation threshold</th>
|
196
|
-
<td><%= h(@
|
196
|
+
<td><%= h(@slow_operation_threshold) %>sec</td>
|
197
197
|
</tr>
|
198
198
|
</table>
|
199
199
|
</div>
|
@@ -29,12 +29,9 @@ module Groonga
|
|
29
29
|
attr_reader :n_responses, :n_slow_responses, :n_slow_operations
|
30
30
|
attr_reader :slow_operations, :total_elapsed
|
31
31
|
attr_reader :start_time, :last_time
|
32
|
-
attr_accessor :slow_operation_threshold, :slow_response_threshold
|
33
32
|
def initialize
|
34
33
|
@max_size = 10
|
35
34
|
self.order = "-elapsed"
|
36
|
-
@slow_operation_threshold = 0.1
|
37
|
-
@slow_response_threshold = 0.2
|
38
35
|
@start_time = nil
|
39
36
|
@last_time = nil
|
40
37
|
@n_responses = 0
|
@@ -53,10 +50,6 @@ module Groonga
|
|
53
50
|
def apply_options(options)
|
54
51
|
@max_size = options[:n_entries] || @max_size
|
55
52
|
self.order = options[:order] || @order
|
56
|
-
@slow_operation_threshold =
|
57
|
-
options[:slow_operation_threshold] || @slow_operation_threshold
|
58
|
-
@slow_response_threshold =
|
59
|
-
options[:slow_response_threshold] || @slow_response_threshold
|
60
53
|
unless options[:report_summary].nil?
|
61
54
|
@collect_slow_statistics = options[:report_summary]
|
62
55
|
end
|
@@ -146,8 +139,6 @@ module Groonga
|
|
146
139
|
end
|
147
140
|
|
148
141
|
def update_statistic(statistic)
|
149
|
-
statistic.slow_response_threshold = @slow_response_threshold
|
150
|
-
statistic.slow_operation_threshold = @slow_operation_threshold
|
151
142
|
@start_time ||= statistic.start_time
|
152
143
|
@start_time = [@start_time, statistic.start_time].min
|
153
144
|
@last_time ||= statistic.last_time
|
@@ -22,6 +22,9 @@ module Groonga
|
|
22
22
|
module QueryLog
|
23
23
|
class Analyzer
|
24
24
|
class Statistic
|
25
|
+
DEFAULT_SLOW_OPERATION_THRESHOLD = 0.1
|
26
|
+
DEFAULT_SLOW_RESPONSE_THRESHOLD = 0.2
|
27
|
+
|
25
28
|
attr_reader :context_id, :start_time, :raw_command
|
26
29
|
attr_reader :elapsed, :return_code
|
27
30
|
attr_accessor :slow_operation_threshold, :slow_response_threshold
|
@@ -33,8 +36,8 @@ module Groonga
|
|
33
36
|
@operations = []
|
34
37
|
@elapsed = nil
|
35
38
|
@return_code = 0
|
36
|
-
@slow_operation_threshold =
|
37
|
-
@slow_response_threshold =
|
39
|
+
@slow_operation_threshold = DEFAULT_SLOW_OPERATION_THRESHOLD
|
40
|
+
@slow_response_threshold = DEFAULT_SLOW_RESPONSE_THRESHOLD
|
38
41
|
end
|
39
42
|
|
40
43
|
def start(start_time, command)
|
File without changes
|
@@ -0,0 +1,131 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
#
|
3
|
+
# Copyright (C) 2013 Kouhei Sutou <kou@clear-code.com>
|
4
|
+
#
|
5
|
+
# This library is free software; you can redistribute it and/or
|
6
|
+
# modify it under the terms of the GNU Lesser General Public
|
7
|
+
# License as published by the Free Software Foundation; either
|
8
|
+
# version 2.1 of the License, or (at your option) any later version.
|
9
|
+
#
|
10
|
+
# This library is distributed in the hope that it will be useful,
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
13
|
+
# Lesser General Public License for more details.
|
14
|
+
#
|
15
|
+
# You should have received a copy of the GNU Lesser General Public
|
16
|
+
# License along with this library; if not, write to the Free Software
|
17
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
18
|
+
|
19
|
+
require "optparse"
|
20
|
+
|
21
|
+
require "groonga/query-log"
|
22
|
+
|
23
|
+
module Groonga
|
24
|
+
module QueryLog
|
25
|
+
module Command
|
26
|
+
class VerifyServer
|
27
|
+
def initialize
|
28
|
+
@options = ServerVerifier::Options.new
|
29
|
+
end
|
30
|
+
|
31
|
+
def run(*command_line)
|
32
|
+
input_paths = create_parser.parse(*command_line)
|
33
|
+
verifier = ServerVerifier.new(@options)
|
34
|
+
input_paths.each do |input_path|
|
35
|
+
File.open(input_path) do |input|
|
36
|
+
verifier.verify(input)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
def create_parser
|
43
|
+
parser = OptionParser.new
|
44
|
+
parser.version = VERSION
|
45
|
+
parser.banner += " QUERY_LOG1 QUERY_LOG2 ..."
|
46
|
+
|
47
|
+
parser.separator("")
|
48
|
+
parser.separator("Options:")
|
49
|
+
|
50
|
+
available_protocols = [:gqtp, :http]
|
51
|
+
available_protocols_label = "[#{available_protocols.join(', ')}]"
|
52
|
+
|
53
|
+
parser.on("--groonga1-host=HOST",
|
54
|
+
"Host name or IP address of Groonga server 1",
|
55
|
+
"[#{@options.groonga1.host}]") do |host|
|
56
|
+
@options.groonga1.host = host
|
57
|
+
end
|
58
|
+
|
59
|
+
parser.on("--groonga1-port=PORT", Integer,
|
60
|
+
"Port number of Groonga server 1",
|
61
|
+
"[#{@options.groonga1.port}]") do |port|
|
62
|
+
@options.groonga1.port = port
|
63
|
+
end
|
64
|
+
|
65
|
+
parser.on("--groonga1-protocol=PROTOCOL", available_protocols,
|
66
|
+
"Protocol of Groonga server 1",
|
67
|
+
available_protocols_label) do |protocol|
|
68
|
+
@options.groonga1.protocol = protocol
|
69
|
+
end
|
70
|
+
|
71
|
+
parser.on("--groonga2-host=HOST",
|
72
|
+
"Host name or IP address of Groonga server 2",
|
73
|
+
"[#{@options.groonga2.host}]") do |host|
|
74
|
+
@options.groonga2.host = host
|
75
|
+
end
|
76
|
+
|
77
|
+
parser.on("--groonga2-port=PORT", Integer,
|
78
|
+
"Port number of Groonga server 2",
|
79
|
+
"[#{@options.groonga2.port}]") do |port|
|
80
|
+
@options.groonga2.port = port
|
81
|
+
end
|
82
|
+
|
83
|
+
parser.on("--groonga2-protocol=PROTOCOL", available_protocols,
|
84
|
+
"Protocol of Groonga server 2",
|
85
|
+
available_protocols_label) do |protocol|
|
86
|
+
@options.groonga2.protocol = protocol
|
87
|
+
end
|
88
|
+
|
89
|
+
parser.on("--n-clients=N", Integer,
|
90
|
+
"The max number of concurrency",
|
91
|
+
"[#{@options.n_clients}]") do |n_clients|
|
92
|
+
@options.n_clients = n_clients
|
93
|
+
end
|
94
|
+
|
95
|
+
parser.on("--request-queue-size=SIZE", Integer,
|
96
|
+
"The size of request queue",
|
97
|
+
"[auto]") do |size|
|
98
|
+
@options.request_queue_size = size
|
99
|
+
end
|
100
|
+
|
101
|
+
parser.on("--disable-cache",
|
102
|
+
"Add 'cache=no' parameter to request",
|
103
|
+
"[#{@options.disable_cache?}]") do
|
104
|
+
@options.disable_cache = true
|
105
|
+
end
|
106
|
+
|
107
|
+
parser.on("--target-command-name=NAME",
|
108
|
+
"Add NAME to target command names",
|
109
|
+
"You can specify this option zero or more times",
|
110
|
+
"See also --target-command-names") do |name|
|
111
|
+
@options.target_command_names << name
|
112
|
+
end
|
113
|
+
|
114
|
+
target_command_names_label = @options.target_command_names.join(", ")
|
115
|
+
parser.on("--target-command-names=NAME1,NAME2,...", Array,
|
116
|
+
"Replay only NAME1,NAME2,... commands",
|
117
|
+
"You can use glob to choose command name",
|
118
|
+
"[#{target_command_names_label}]") do |names|
|
119
|
+
@options.target_command_names = names
|
120
|
+
end
|
121
|
+
|
122
|
+
parser.on("--output=PATH",
|
123
|
+
"Output results to PATH",
|
124
|
+
"[stdout]") do |path|
|
125
|
+
@options.output_path = path
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
@@ -20,7 +20,7 @@
|
|
20
20
|
require "ostruct"
|
21
21
|
require "optparse"
|
22
22
|
require "pathname"
|
23
|
-
require "groonga/query-log/
|
23
|
+
require "groonga/query-log/command-line-utils"
|
24
24
|
require "groonga/query-log/parser"
|
25
25
|
|
26
26
|
module Groonga
|
@@ -69,7 +69,7 @@ module Groonga
|
|
69
69
|
unless log_via_stdin?
|
70
70
|
raise(NoInputError, "Error: Please specify input log files.")
|
71
71
|
end
|
72
|
-
log =
|
72
|
+
log = $stdin
|
73
73
|
else
|
74
74
|
log = log_paths
|
75
75
|
end
|
@@ -91,6 +91,7 @@ module Groonga
|
|
91
91
|
@options.exclude_commands = []
|
92
92
|
@options.output_path = nil
|
93
93
|
@option_parser = OptionParser.new do |parser|
|
94
|
+
parser.version = VERSION
|
94
95
|
parser.banner += " QUERY_LOG1 ..."
|
95
96
|
|
96
97
|
available_formats = ["uri", "command"]
|
@@ -23,7 +23,10 @@ require "groonga/query-log/analyzer/statistic"
|
|
23
23
|
module Groonga
|
24
24
|
module QueryLog
|
25
25
|
class Parser
|
26
|
-
def initialize
|
26
|
+
def initialize(options={})
|
27
|
+
@options = options
|
28
|
+
@slow_operation_threshold = options[:slow_operation_threshold]
|
29
|
+
@slow_response_threshold = options[:slow_response_threshold]
|
27
30
|
end
|
28
31
|
|
29
32
|
# Parses query-log file as stream to
|
@@ -59,7 +62,7 @@ module Groonga
|
|
59
62
|
time_stamp, context_id, type, rest, &block)
|
60
63
|
case type
|
61
64
|
when ">"
|
62
|
-
statistic =
|
65
|
+
statistic = create_statistic(context_id)
|
63
66
|
statistic.start(time_stamp, rest)
|
64
67
|
current_statistics[context_id] = statistic
|
65
68
|
when ":"
|
@@ -82,6 +85,17 @@ module Groonga
|
|
82
85
|
block.call(statistic)
|
83
86
|
end
|
84
87
|
end
|
88
|
+
|
89
|
+
def create_statistic(context_id)
|
90
|
+
statistic = Analyzer::Statistic.new(context_id)
|
91
|
+
if @slow_operation_threshold
|
92
|
+
statistic.slow_operation_threshold = @slow_operation_threshold
|
93
|
+
end
|
94
|
+
if @slow_response_threshold
|
95
|
+
statistic.slow_response_threshold = @slow_response_threshold
|
96
|
+
end
|
97
|
+
statistic
|
98
|
+
end
|
85
99
|
end
|
86
100
|
end
|
87
101
|
end
|
@@ -0,0 +1,188 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
#
|
3
|
+
# Copyright (C) 2013 Kouhei Sutou <kou@clear-code.com>
|
4
|
+
#
|
5
|
+
# This library is free software; you can redistribute it and/or
|
6
|
+
# modify it under the terms of the GNU Lesser General Public
|
7
|
+
# License as published by the Free Software Foundation; either
|
8
|
+
# version 2.1 of the License, or (at your option) any later version.
|
9
|
+
#
|
10
|
+
# This library is distributed in the hope that it will be useful,
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
13
|
+
# Lesser General Public License for more details.
|
14
|
+
#
|
15
|
+
# You should have received a copy of the GNU Lesser General Public
|
16
|
+
# License along with this library; if not, write to the Free Software
|
17
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
18
|
+
|
19
|
+
require "time"
|
20
|
+
require "thread"
|
21
|
+
|
22
|
+
require "groonga/client"
|
23
|
+
|
24
|
+
require "groonga/query-log/parser"
|
25
|
+
|
26
|
+
module Groonga
|
27
|
+
module QueryLog
|
28
|
+
class ServerVerifier
|
29
|
+
def initialize(options)
|
30
|
+
@options = options
|
31
|
+
@queue = SizedQueue.new(@options.request_queue_size)
|
32
|
+
@different_results = Queue.new
|
33
|
+
end
|
34
|
+
|
35
|
+
def verify(input)
|
36
|
+
producer = run_producer(input)
|
37
|
+
consumers = run_consumers
|
38
|
+
reporter = run_reporter
|
39
|
+
producer.join
|
40
|
+
consumers.each(&:join)
|
41
|
+
@different_results.push(nil)
|
42
|
+
reporter.join
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
def run_producer(input)
|
47
|
+
Thread.new do
|
48
|
+
parser = Parser.new
|
49
|
+
parser.parse(input) do |statistic|
|
50
|
+
next unless target_command?(statistic.command)
|
51
|
+
@queue.push(statistic)
|
52
|
+
end
|
53
|
+
@options.n_clients.times do
|
54
|
+
@queue.push(nil)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def run_consumers
|
60
|
+
@options.n_clients.times.collect do
|
61
|
+
Thread.new do
|
62
|
+
loop do
|
63
|
+
break if run_consumer
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def run_consumer
|
70
|
+
@options.groonga1.create_client do |groonga1_client|
|
71
|
+
@options.groonga2.create_client do |groonga2_client|
|
72
|
+
loop do
|
73
|
+
statistic = @queue.pop
|
74
|
+
return true if statistic.nil?
|
75
|
+
begin
|
76
|
+
verify_command(groonga1_client, groonga2_client,
|
77
|
+
statistic.command)
|
78
|
+
rescue Groonga::Client::Connection::Error
|
79
|
+
# TODO: add error log mechanism
|
80
|
+
$stderr.puts(Time.now.iso8601)
|
81
|
+
$stderr.puts(statistic.command.original_source)
|
82
|
+
$stderr.puts($!.raw_error.message)
|
83
|
+
$stderr.puts($!.raw_error.backtrace)
|
84
|
+
return false
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def run_reporter
|
92
|
+
Thread.new do
|
93
|
+
@options.create_output do |output|
|
94
|
+
loop do
|
95
|
+
result = @different_results.pop
|
96
|
+
break if result.nil?
|
97
|
+
report_result(output, result)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
def target_command?(command)
|
104
|
+
@options.target_command_name?(command.name)
|
105
|
+
end
|
106
|
+
|
107
|
+
def verify_command(groonga1_client, groonga2_client, command)
|
108
|
+
command["cache"] = "no" if @options.disable_cache?
|
109
|
+
response1 = groonga1_client.execute(command)
|
110
|
+
response2 = groonga2_client.execute(command)
|
111
|
+
unless same_response?(response1, response2)
|
112
|
+
@different_results.push([command, response1, response2])
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def same_response?(response1, response2)
|
117
|
+
response1.body == response2.body
|
118
|
+
end
|
119
|
+
|
120
|
+
def report_result(output, result)
|
121
|
+
command, response1, response2 = result
|
122
|
+
output.puts("command: #{command.original_format}")
|
123
|
+
output.puts("response1: #{response1.body}")
|
124
|
+
output.puts("response2: #{response2.body}")
|
125
|
+
end
|
126
|
+
|
127
|
+
class Options
|
128
|
+
attr_reader :groonga1
|
129
|
+
attr_reader :groonga2
|
130
|
+
attr_accessor :n_clients
|
131
|
+
attr_writer :request_queue_size
|
132
|
+
attr_accessor :target_command_names
|
133
|
+
attr_accessor :output_path
|
134
|
+
def initialize
|
135
|
+
@groonga1 = GroongaOptions.new
|
136
|
+
@groonga2 = GroongaOptions.new
|
137
|
+
@n_clients = 8
|
138
|
+
@request_queue_size = nil
|
139
|
+
@disable_cache = false
|
140
|
+
@output_path = nil
|
141
|
+
@target_command_names = ["select"]
|
142
|
+
end
|
143
|
+
|
144
|
+
def request_queue_size
|
145
|
+
@request_queue_size || @n_clients * 3
|
146
|
+
end
|
147
|
+
|
148
|
+
def disable_cache?
|
149
|
+
@disable_cache
|
150
|
+
end
|
151
|
+
|
152
|
+
def target_command_name?(name)
|
153
|
+
@target_command_names.any? do |name_pattern|
|
154
|
+
flags = 0
|
155
|
+
flags |= File::FNM_EXTGLOB if File.const_defined?(:FNM_EXTGLOB)
|
156
|
+
File.fnmatch(name_pattern, name, flags)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
def create_output(&block)
|
161
|
+
if @output_path
|
162
|
+
File.open(@output_path, "w", &block)
|
163
|
+
else
|
164
|
+
yield($stdout)
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
class GroongaOptions
|
170
|
+
attr_accessor :host
|
171
|
+
attr_accessor :port
|
172
|
+
attr_accessor :protocol
|
173
|
+
def initialize
|
174
|
+
@host = "127.0.0.1"
|
175
|
+
@port = 10041
|
176
|
+
@protocol = :gqtp
|
177
|
+
end
|
178
|
+
|
179
|
+
def create_client(&block)
|
180
|
+
Groonga::Client.open(:host => @host,
|
181
|
+
:port => @port,
|
182
|
+
:protocol => @protocol,
|
183
|
+
&block)
|
184
|
+
end
|
185
|
+
end
|
186
|
+
end
|
187
|
+
end
|
188
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: groonga-query-log
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kouhei Sutou
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2014-01-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: groonga-command-parser
|
@@ -158,6 +158,7 @@ executables:
|
|
158
158
|
- groonga-query-log-detect-memory-leak
|
159
159
|
- groonga-query-log-replay
|
160
160
|
- groonga-query-log-extract
|
161
|
+
- groonga-query-log-verify-server
|
161
162
|
extensions: []
|
162
163
|
extra_rdoc_files: []
|
163
164
|
files:
|
@@ -168,12 +169,14 @@ files:
|
|
168
169
|
- lib/groonga/query-log.rb
|
169
170
|
- lib/groonga/query-log/parser.rb
|
170
171
|
- lib/groonga/query-log/extractor.rb
|
172
|
+
- lib/groonga/query-log/command-line-utils.rb
|
171
173
|
- lib/groonga/query-log/command/detect-memory-leak.rb
|
174
|
+
- lib/groonga/query-log/command/verify-server.rb
|
172
175
|
- lib/groonga/query-log/command/replay.rb
|
173
176
|
- lib/groonga/query-log/version.rb
|
177
|
+
- lib/groonga/query-log/server-verifier.rb
|
174
178
|
- lib/groonga/query-log/memory-leak-detector.rb
|
175
179
|
- lib/groonga/query-log/analyzer.rb
|
176
|
-
- lib/groonga/query-log/commandline-utils.rb
|
177
180
|
- lib/groonga/query-log/replayer.rb
|
178
181
|
- lib/groonga/query-log/analyzer/sized-grouped-operations.rb
|
179
182
|
- lib/groonga/query-log/analyzer/streamer.rb
|
@@ -209,6 +212,7 @@ files:
|
|
209
212
|
- bin/groonga-query-log-detect-memory-leak
|
210
213
|
- bin/groonga-query-log-replay
|
211
214
|
- bin/groonga-query-log-extract
|
215
|
+
- bin/groonga-query-log-verify-server
|
212
216
|
homepage: https://github.com/groonga/groonga-query-log
|
213
217
|
licenses:
|
214
218
|
- LGPLv2.1+
|
@@ -229,7 +233,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
229
233
|
version: '0'
|
230
234
|
requirements: []
|
231
235
|
rubyforge_project:
|
232
|
-
rubygems_version: 2.0.
|
236
|
+
rubygems_version: 2.0.14
|
233
237
|
signing_key:
|
234
238
|
specification_version: 4
|
235
239
|
summary: Groonga-query-log is a collection of library and tools to process [groonga](http://groonga.org/)'s
|