groonga-query-log 1.3.0 → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- # Copyright (C) 2013-2017 Kouhei Sutou <kou@clear-code.com>
1
+ # Copyright (C) 2013-2018 Kouhei Sutou <kou@clear-code.com>
2
2
  #
3
3
  # This library is free software; you can redistribute it and/or
4
4
  # modify it under the terms of the GNU Lesser General Public
@@ -23,226 +23,240 @@ require "groonga-query-log/parser"
23
23
  require "groonga-query-log/response-comparer"
24
24
 
25
25
  module GroongaQueryLog
26
- class ServerVerifier
27
- def initialize(options)
28
- @options = options
29
- @queue = SizedQueue.new(@options.request_queue_size)
30
- @different_results = Queue.new
31
- end
26
+ class ServerVerifier
27
+ def initialize(options)
28
+ @options = options
29
+ @queue = SizedQueue.new(@options.request_queue_size)
30
+ @different_results = Queue.new
31
+ end
32
32
 
33
- def verify(input, &callback)
34
- producer = run_producer(input, &callback)
35
- reporter = run_reporter
36
- producer.join
37
- @different_results.push(nil)
38
- reporter.join
39
- end
33
+ def verify(input, &callback)
34
+ @same = true
35
+ producer = run_producer(input, &callback)
36
+ reporter = run_reporter
37
+ producer.join
38
+ @different_results.push(nil)
39
+ reporter.join
40
+ @same
41
+ end
40
42
 
41
- private
42
- def run_producer(input, &callback)
43
- Thread.new do
44
- consumers = run_consumers
45
-
46
- parser = Parser.new
47
- n_commands = 0
48
- callback_per_n_commands = 100
49
- parser.parse(input) do |statistic|
50
- command = statistic.command
51
- next if command.nil?
52
- next unless target_command?(command)
53
- n_commands += 1
54
- @queue.push(statistic)
55
-
56
- if callback and (n_commands % callback_per_n_commands).zero?
57
- @options.n_clients.times do
58
- @queue.push(nil)
59
- end
60
- consumers.each(&:join)
61
- callback.call
62
- consumers = run_consumers
43
+ private
44
+ def run_producer(input, &callback)
45
+ Thread.new do
46
+ consumers = run_consumers
47
+
48
+ parser = Parser.new
49
+ n_commands = 0
50
+ callback_per_n_commands = 100
51
+ parser.parse(input) do |statistic|
52
+ break if !@same and @options.stop_on_failure?
53
+
54
+ command = statistic.command
55
+ next if command.nil?
56
+ next unless target_command?(command)
57
+ n_commands += 1
58
+ @queue.push(statistic)
59
+
60
+ if callback and (n_commands % callback_per_n_commands).zero?
61
+ @options.n_clients.times do
62
+ @queue.push(nil)
63
63
  end
64
+ consumers.each(&:join)
65
+ callback.call
66
+ consumers = run_consumers
64
67
  end
65
- @options.n_clients.times do
66
- @queue.push(nil)
67
- end
68
- consumers.each(&:join)
69
68
  end
69
+ @options.n_clients.times do
70
+ @queue.push(nil)
71
+ end
72
+ consumers.each(&:join)
70
73
  end
74
+ end
71
75
 
72
- def run_consumers
73
- @options.n_clients.times.collect do
74
- Thread.new do
75
- loop do
76
- break if run_consumer
77
- end
76
+ def run_consumers
77
+ @options.n_clients.times.collect do
78
+ Thread.new do
79
+ loop do
80
+ break if run_consumer
78
81
  end
79
82
  end
80
83
  end
84
+ end
81
85
 
82
- def run_consumer
83
- @options.groonga1.create_client do |groonga1_client|
84
- @options.groonga2.create_client do |groonga2_client|
85
- loop do
86
- statistic = @queue.pop
87
- return true if statistic.nil?
86
+ def run_consumer
87
+ @options.groonga1.create_client do |groonga1_client|
88
+ @options.groonga2.create_client do |groonga2_client|
89
+ loop do
90
+ statistic = @queue.pop
91
+ return true if statistic.nil?
88
92
 
89
- original_source = statistic.command.original_source
93
+ original_source = statistic.command.original_source
94
+ begin
95
+ verify_command(groonga1_client, groonga2_client,
96
+ statistic.command)
97
+ rescue
98
+ log_client_error($!) do
99
+ $stderr.puts(original_source)
100
+ end
101
+ return false
102
+ end
103
+ if @options.verify_cache?
90
104
  begin
91
105
  verify_command(groonga1_client, groonga2_client,
92
- statistic.command)
106
+ Groonga::Command::Status.new)
93
107
  rescue
94
108
  log_client_error($!) do
95
- $stderr.puts(original_source)
109
+ $stderr.puts("status after #{original_source}")
96
110
  end
97
111
  return false
98
112
  end
99
- if @options.verify_cache?
100
- begin
101
- verify_command(groonga1_client, groonga2_client,
102
- Groonga::Command::Status.new)
103
- rescue
104
- log_client_error($!) do
105
- $stderr.puts("status after #{original_source}")
106
- end
107
- return false
108
- end
109
- end
110
113
  end
111
114
  end
112
115
  end
113
116
  end
117
+ end
114
118
 
115
- def run_reporter
116
- Thread.new do
117
- @options.create_output do |output|
118
- loop do
119
- result = @different_results.pop
120
- break if result.nil?
121
- report_result(output, result)
122
- end
119
+ def run_reporter
120
+ Thread.new do
121
+ @options.create_output do |output|
122
+ loop do
123
+ result = @different_results.pop
124
+ break if result.nil?
125
+ report_result(output, result)
123
126
  end
124
127
  end
125
128
  end
129
+ end
126
130
 
127
- def target_command?(command)
128
- @options.target_command_name?(command.command_name)
129
- end
131
+ def target_command?(command)
132
+ @options.target_command_name?(command.command_name)
133
+ end
130
134
 
131
- def verify_command(groonga1_client, groonga2_client, command)
132
- command["cache"] = "no" if @options.disable_cache?
133
- command["output_type"] = "json"
134
- response1 = groonga1_client.execute(command)
135
- response2 = groonga2_client.execute(command)
136
- compare_options = {
137
- :care_order => @options.care_order,
138
- }
139
- comparer = ResponseComparer.new(command, response1, response2,
140
- compare_options)
141
- unless comparer.same?
142
- @different_results.push([command, response1, response2])
143
- end
135
+ def verify_command(groonga1_client, groonga2_client, command)
136
+ command["cache"] = "no" if @options.disable_cache?
137
+ command["output_type"] = "json"
138
+ response1 = groonga1_client.execute(command)
139
+ response2 = groonga2_client.execute(command)
140
+ compare_options = {
141
+ :care_order => @options.care_order,
142
+ :ignored_drilldown_keys => @options.ignored_drilldown_keys,
143
+ }
144
+ comparer = ResponseComparer.new(command, response1, response2,
145
+ compare_options)
146
+ unless comparer.same?
147
+ @different_results.push([command, response1, response2])
144
148
  end
149
+ end
145
150
 
146
- def report_result(output, result)
147
- command, response1, response2 = result
148
- command_source = command.original_source || command.to_uri_format
149
- output.puts("command: #{command_source}")
150
- output.puts("response1: #{response1.body.to_json}")
151
- output.puts("response2: #{response2.body.to_json}")
151
+ def report_result(output, result)
152
+ @same = false
153
+ command, response1, response2 = result
154
+ command_source = command.original_source || command.to_uri_format
155
+ output.puts("command: #{command_source}")
156
+ output.puts("response1: #{response1.body.to_json}")
157
+ output.puts("response2: #{response2.body.to_json}")
158
+ end
159
+
160
+ def log_client_error(error)
161
+ $stderr.puts(Time.now.iso8601)
162
+ yield if block_given?
163
+ if error.respond_to?(:raw_error)
164
+ target_error = error.raw_error
165
+ else
166
+ target_error = error
152
167
  end
168
+ $stderr.puts("#{target_error.class}: #{target_error.message}")
169
+ $stderr.puts(target_error.backtrace)
170
+ end
153
171
 
154
- def log_client_error(error)
155
- $stderr.puts(Time.now.iso8601)
156
- yield if block_given?
157
- if error.respond_to?(:raw_error)
158
- target_error = error.raw_error
159
- else
160
- target_error = error
161
- end
162
- $stderr.puts("#{target_error.class}: #{target_error.message}")
163
- $stderr.puts(target_error.backtrace)
172
+ class Options
173
+ attr_reader :groonga1
174
+ attr_reader :groonga2
175
+ attr_accessor :n_clients
176
+ attr_writer :request_queue_size
177
+ attr_writer :disable_cache
178
+ attr_accessor :target_command_names
179
+ attr_accessor :output_path
180
+ attr_accessor :care_order
181
+ attr_writer :verify_cache
182
+ attr_accessor :ignored_drilldown_keys
183
+ attr_writer :stop_on_failure
184
+ def initialize
185
+ @groonga1 = GroongaOptions.new
186
+ @groonga2 = GroongaOptions.new
187
+ @n_clients = 8
188
+ @request_queue_size = nil
189
+ @disable_cache = false
190
+ @output_path = nil
191
+ @target_command_names = [
192
+ "io_flush",
193
+ "logical_count",
194
+ "logical_range_filter",
195
+ "logical_shard_list",
196
+ "logical_select",
197
+ "normalize",
198
+ "object_exist",
199
+ "select",
200
+ "status",
201
+ ]
202
+ @care_order = true
203
+ @verify_cache = false
204
+ @ignored_drilldown_keys = []
205
+ @stop_on_failure = false
164
206
  end
165
207
 
166
- class Options
167
- attr_reader :groonga1
168
- attr_reader :groonga2
169
- attr_accessor :n_clients
170
- attr_writer :request_queue_size
171
- attr_writer :disable_cache
172
- attr_accessor :target_command_names
173
- attr_accessor :output_path
174
- attr_accessor :care_order
175
- attr_writer :verify_cache
176
- def initialize
177
- @groonga1 = GroongaOptions.new
178
- @groonga2 = GroongaOptions.new
179
- @n_clients = 8
180
- @request_queue_size = nil
181
- @disable_cache = false
182
- @output_path = nil
183
- @target_command_names = [
184
- "io_flush",
185
- "logical_count",
186
- "logical_range_filter",
187
- "logical_shard_list",
188
- "logical_select",
189
- "normalize",
190
- "object_exist",
191
- "select",
192
- "status",
193
- ]
194
- @care_order = true
195
- @verify_cache = false
196
- end
208
+ def request_queue_size
209
+ @request_queue_size || @n_clients * 3
210
+ end
197
211
 
198
- def request_queue_size
199
- @request_queue_size || @n_clients * 3
200
- end
212
+ def disable_cache?
213
+ @disable_cache
214
+ end
201
215
 
202
- def disable_cache?
203
- @disable_cache
204
- end
216
+ def verify_cache?
217
+ @verify_cache
218
+ end
205
219
 
206
- def verify_cache?
207
- @verify_cache
208
- end
220
+ def stop_on_failure?
221
+ @stop_on_failure
222
+ end
209
223
 
210
- def target_command_name?(name)
211
- return false if name.nil?
224
+ def target_command_name?(name)
225
+ return false if name.nil?
212
226
 
213
- @target_command_names.any? do |name_pattern|
214
- flags = 0
215
- flags |= File::FNM_EXTGLOB if File.const_defined?(:FNM_EXTGLOB)
216
- File.fnmatch(name_pattern, name, flags)
217
- end
227
+ @target_command_names.any? do |name_pattern|
228
+ flags = 0
229
+ flags |= File::FNM_EXTGLOB if File.const_defined?(:FNM_EXTGLOB)
230
+ File.fnmatch(name_pattern, name, flags)
218
231
  end
232
+ end
219
233
 
220
- def create_output(&block)
221
- if @output_path
222
- FileUtils.mkdir_p(File.dirname(@output_path))
223
- File.open(@output_path, "w", &block)
224
- else
225
- yield($stdout)
226
- end
234
+ def create_output(&block)
235
+ if @output_path
236
+ FileUtils.mkdir_p(File.dirname(@output_path))
237
+ File.open(@output_path, "w", &block)
238
+ else
239
+ yield($stdout)
227
240
  end
228
241
  end
242
+ end
229
243
 
230
- class GroongaOptions
231
- attr_accessor :host
232
- attr_accessor :port
233
- attr_accessor :protocol
234
- def initialize
235
- @host = "127.0.0.1"
236
- @port = 10041
237
- @protocol = :gqtp
238
- end
244
+ class GroongaOptions
245
+ attr_accessor :host
246
+ attr_accessor :port
247
+ attr_accessor :protocol
248
+ def initialize
249
+ @host = "127.0.0.1"
250
+ @port = 10041
251
+ @protocol = :gqtp
252
+ end
239
253
 
240
- def create_client(&block)
241
- Groonga::Client.open(:host => @host,
242
- :port => @port,
243
- :protocol => @protocol,
244
- &block)
245
- end
254
+ def create_client(&block)
255
+ Groonga::Client.open(:host => @host,
256
+ :port => @port,
257
+ :protocol => @protocol,
258
+ &block)
246
259
  end
247
260
  end
261
+ end
248
262
  end
@@ -15,5 +15,5 @@
15
15
  # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
16
 
17
17
  module GroongaQueryLog
18
- VERSION = "1.3.0"
18
+ VERSION = "1.3.1"
19
19
  end
@@ -1,4 +1,4 @@
1
- # Copyright (C) 2014-2016 Kouhei Sutou <kou@clear-code.com>
1
+ # Copyright (C) 2014-2018 Kouhei Sutou <kou@clear-code.com>
2
2
  #
3
3
  # This library is free software; you can redistribute it and/or
4
4
  # modify it under the terms of the GNU Lesser General Public
@@ -115,14 +115,33 @@ class ResponseComparerTest < Test::Unit::TestCase
115
115
  end
116
116
  end
117
117
 
118
- class SortbyTest < self
118
+ class SortKeysTest < self
119
119
  class DetectScoreSortTest < self
120
120
  private
121
- def score_sort?(sortby)
122
- @command["sortby"] = sortby
121
+ def score_sort?(sort_keys)
122
+ @command[:sort_keys] = sort_keys
123
123
  comparer([[[0], []]], [[[0], []]]).send(:score_sort?)
124
124
  end
125
125
 
126
+ class ParameterNameTest < self
127
+ def score_sort?(parameter_name)
128
+ @command[parameter_name] = "_score"
129
+ comparer([[[0], []]], [[[0], []]]).send(:score_sort?)
130
+ end
131
+
132
+ def test_sortby
133
+ assert do
134
+ score_sort?(:sortby)
135
+ end
136
+ end
137
+
138
+ def test_sort_keys
139
+ assert do
140
+ score_sort?(:sort_keys)
141
+ end
142
+ end
143
+ end
144
+
126
145
  class NoScoreTest < self
127
146
  def test_nil
128
147
  assert_false(score_sort?(nil))
@@ -306,6 +325,119 @@ class ResponseComparerTest < Test::Unit::TestCase
306
325
  end
307
326
  end
308
327
 
328
+ class FloatAccurancy < self
329
+ def create_response(latitude, longitude)
330
+ [
331
+ [
332
+ [1],
333
+ [["_id", "UInt32"], ["latitude", "Float"], ["longitude", "Float"]],
334
+ [1, latitude, longitude],
335
+ ]
336
+ ]
337
+ end
338
+
339
+ def test_all_output_columns
340
+ response1 = create_response(35.6705996342355, 139.683422370535)
341
+ response2 = create_response(35.67059963423547, 139.6834223705349)
342
+ assert do
343
+ same?(response1, response2)
344
+ end
345
+ end
346
+
347
+ def test_unary_minus_output_column
348
+ @command["output_columns"] = "_id, -value, latitude, longitude"
349
+ response1 = create_response(35.6705996342355, 139.683422370535)
350
+ response2 = create_response(35.67059963423547, 139.6834223705349)
351
+ assert do
352
+ same?(response1, response2)
353
+ end
354
+ end
355
+
356
+ def test_specific_output_columns
357
+ @command["output_columns"] = "_id, latitude, longitude"
358
+ response1 = create_response(35.6705996342355, 139.683422370535)
359
+ response2 = create_response(35.67059963423547, 139.6834223705349)
360
+ assert do
361
+ same?(response1, response2)
362
+ end
363
+ end
364
+ end
365
+
366
+ class DrilldownTest < self
367
+ def create_response(drilldown)
368
+ [
369
+ [
370
+ [10],
371
+ [["_id", "UInt32"]],
372
+ ],
373
+ [
374
+ [drilldown.size * 2],
375
+ [["_key", "ShortText"], ["_nsubrecs", "Int32"]],
376
+ *drilldown,
377
+ ]
378
+ ]
379
+ end
380
+
381
+ def test_same
382
+ response1 = create_response([["A", 10], ["B", 2]])
383
+ response2 = create_response([["A", 10], ["B", 2]])
384
+ assert do
385
+ same?(response1, response2)
386
+ end
387
+ end
388
+
389
+ def test_not_same
390
+ response1 = create_response([["A", 11], ["B", 2]])
391
+ response2 = create_response([["A", 10], ["B", 2]])
392
+ assert do
393
+ not same?(response1, response2)
394
+ end
395
+ end
396
+
397
+ class IgnoreDrilldownKeysTest < self
398
+ def create_response(drilldown1, drilldown2)
399
+ [
400
+ [
401
+ [10],
402
+ [["_id", "UInt32"]],
403
+ ],
404
+ [
405
+ [drilldown1.size * 2],
406
+ [["_key", "ShortText"], ["_nsubrecs", "Int32"]],
407
+ *drilldown1,
408
+ ],
409
+ [
410
+ [drilldown2.size * 2],
411
+ [["_key", "ShortText"], ["_nsubrecs", "Int32"]],
412
+ *drilldown2,
413
+ ],
414
+ ]
415
+ end
416
+
417
+ def test_same
418
+ @command["drilldown"] = "column1, column2"
419
+ response1 = create_response([["A", 10], ["B", 2]],
420
+ [["a", 11], ["b", 10]])
421
+ response2 = create_response([["A", 10], ["B", 2]],
422
+ [["a", 99], ["b", 20]])
423
+ assert do
424
+ same?(response1, response2, ignored_drilldown_keys: ["column2"])
425
+ end
426
+ end
427
+
428
+ def test_not_same
429
+ @command["drilldown"] = "column1, column2"
430
+ response1 = create_response([["A", 10], ["B", 2]],
431
+ [["a", 11], ["b", 10]])
432
+ response2 = create_response([["A", 10], ["B", 2]],
433
+ [["a", 99], ["b", 20]])
434
+ assert do
435
+ not same?(response1, response2)
436
+ end
437
+ end
438
+ end
439
+ end
440
+
309
441
  class ErrorTest < self
310
442
  def test_with_location
311
443
  response1_header = [