groonga-query-log 1.3.0 → 1.3.1

data/lib/groonga-query-log/server-verifier.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2018  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -23,226 +23,240 @@ require "groonga-query-log/parser"
 require "groonga-query-log/response-comparer"
 
 module GroongaQueryLog
-    class ServerVerifier
-      def initialize(options)
-        @options = options
-        @queue = SizedQueue.new(@options.request_queue_size)
-        @different_results = Queue.new
-      end
+  class ServerVerifier
+    def initialize(options)
+      @options = options
+      @queue = SizedQueue.new(@options.request_queue_size)
+      @different_results = Queue.new
+    end
 
-      def verify(input, &callback)
-        producer = run_producer(input, &callback)
-        reporter = run_reporter
-        producer.join
-        @different_results.push(nil)
-        reporter.join
-      end
+    def verify(input, &callback)
+      @same = true
+      producer = run_producer(input, &callback)
+      reporter = run_reporter
+      producer.join
+      @different_results.push(nil)
+      reporter.join
+      @same
+    end
 
-      private
-      def run_producer(input, &callback)
-        Thread.new do
-          consumers = run_consumers
-
-          parser = Parser.new
-          n_commands = 0
-          callback_per_n_commands = 100
-          parser.parse(input) do |statistic|
-            command = statistic.command
-            next if command.nil?
-            next unless target_command?(command)
-            n_commands += 1
-            @queue.push(statistic)
-
-            if callback and (n_commands % callback_per_n_commands).zero?
-              @options.n_clients.times do
-                @queue.push(nil)
-              end
-              consumers.each(&:join)
-              callback.call
-              consumers = run_consumers
+    private
+    def run_producer(input, &callback)
+      Thread.new do
+        consumers = run_consumers
+
+        parser = Parser.new
+        n_commands = 0
+        callback_per_n_commands = 100
+        parser.parse(input) do |statistic|
+          break if !@same and @options.stop_on_failure?
+
+          command = statistic.command
+          next if command.nil?
+          next unless target_command?(command)
+          n_commands += 1
+          @queue.push(statistic)
+
+          if callback and (n_commands % callback_per_n_commands).zero?
+            @options.n_clients.times do
+              @queue.push(nil)
             end
+            consumers.each(&:join)
+            callback.call
+            consumers = run_consumers
           end
-          @options.n_clients.times do
-            @queue.push(nil)
-          end
-          consumers.each(&:join)
         end
+        @options.n_clients.times do
+          @queue.push(nil)
+        end
+        consumers.each(&:join)
       end
+    end
 
-      def run_consumers
-        @options.n_clients.times.collect do
-          Thread.new do
-            loop do
-              break if run_consumer
-            end
+    def run_consumers
+      @options.n_clients.times.collect do
+        Thread.new do
+          loop do
+            break if run_consumer
           end
         end
       end
+    end
 
-      def run_consumer
-        @options.groonga1.create_client do |groonga1_client|
-          @options.groonga2.create_client do |groonga2_client|
-            loop do
-              statistic = @queue.pop
-              return true if statistic.nil?
+    def run_consumer
+      @options.groonga1.create_client do |groonga1_client|
+        @options.groonga2.create_client do |groonga2_client|
+          loop do
+            statistic = @queue.pop
+            return true if statistic.nil?
 
-              original_source = statistic.command.original_source
+            original_source = statistic.command.original_source
+            begin
+              verify_command(groonga1_client, groonga2_client,
+                             statistic.command)
+            rescue
+              log_client_error($!) do
+                $stderr.puts(original_source)
+              end
+              return false
+            end
+            if @options.verify_cache?
               begin
                 verify_command(groonga1_client, groonga2_client,
-                               statistic.command)
+                               Groonga::Command::Status.new)
               rescue
                 log_client_error($!) do
-                  $stderr.puts(original_source)
+                  $stderr.puts("status after #{original_source}")
                 end
                 return false
               end
-              if @options.verify_cache?
-                begin
-                  verify_command(groonga1_client, groonga2_client,
-                                 Groonga::Command::Status.new)
-                rescue
-                  log_client_error($!) do
-                    $stderr.puts("status after #{original_source}")
-                  end
-                  return false
-                end
-              end
             end
           end
         end
       end
+    end
 
-      def run_reporter
-        Thread.new do
-          @options.create_output do |output|
-            loop do
-              result = @different_results.pop
-              break if result.nil?
-              report_result(output, result)
-            end
+    def run_reporter
+      Thread.new do
+        @options.create_output do |output|
+          loop do
+            result = @different_results.pop
+            break if result.nil?
+            report_result(output, result)
           end
         end
       end
+    end
 
-      def target_command?(command)
-        @options.target_command_name?(command.command_name)
-      end
+    def target_command?(command)
+      @options.target_command_name?(command.command_name)
+    end
 
-      def verify_command(groonga1_client, groonga2_client, command)
-        command["cache"] = "no" if @options.disable_cache?
-        command["output_type"] = "json"
-        response1 = groonga1_client.execute(command)
-        response2 = groonga2_client.execute(command)
-        compare_options = {
-          :care_order => @options.care_order,
-        }
-        comparer = ResponseComparer.new(command, response1, response2,
-                                        compare_options)
-        unless comparer.same?
-          @different_results.push([command, response1, response2])
-        end
+    def verify_command(groonga1_client, groonga2_client, command)
+      command["cache"] = "no" if @options.disable_cache?
+      command["output_type"] = "json"
+      response1 = groonga1_client.execute(command)
+      response2 = groonga2_client.execute(command)
+      compare_options = {
+        :care_order => @options.care_order,
+        :ignored_drilldown_keys => @options.ignored_drilldown_keys,
+      }
+      comparer = ResponseComparer.new(command, response1, response2,
+                                      compare_options)
+      unless comparer.same?
+        @different_results.push([command, response1, response2])
       end
+    end
 
-      def report_result(output, result)
-        command, response1, response2 = result
-        command_source = command.original_source || command.to_uri_format
-        output.puts("command: #{command_source}")
-        output.puts("response1: #{response1.body.to_json}")
-        output.puts("response2: #{response2.body.to_json}")
+    def report_result(output, result)
+      @same = false
+      command, response1, response2 = result
+      command_source = command.original_source || command.to_uri_format
+      output.puts("command: #{command_source}")
+      output.puts("response1: #{response1.body.to_json}")
+      output.puts("response2: #{response2.body.to_json}")
+    end
+
+    def log_client_error(error)
+      $stderr.puts(Time.now.iso8601)
+      yield if block_given?
+      if error.respond_to?(:raw_error)
+        target_error = error.raw_error
+      else
+        target_error = error
       end
+      $stderr.puts("#{target_error.class}: #{target_error.message}")
+      $stderr.puts(target_error.backtrace)
+    end
 
-      def log_client_error(error)
-        $stderr.puts(Time.now.iso8601)
-        yield if block_given?
-        if error.respond_to?(:raw_error)
-          target_error = error.raw_error
-        else
-          target_error = error
-        end
-        $stderr.puts("#{target_error.class}: #{target_error.message}")
-        $stderr.puts(target_error.backtrace)
+    class Options
+      attr_reader :groonga1
+      attr_reader :groonga2
+      attr_accessor :n_clients
+      attr_writer :request_queue_size
+      attr_writer :disable_cache
+      attr_accessor :target_command_names
+      attr_accessor :output_path
+      attr_accessor :care_order
+      attr_writer :verify_cache
+      attr_accessor :ignored_drilldown_keys
+      attr_writer :stop_on_failure
+      def initialize
+        @groonga1 = GroongaOptions.new
+        @groonga2 = GroongaOptions.new
+        @n_clients = 8
+        @request_queue_size = nil
+        @disable_cache = false
+        @output_path = nil
+        @target_command_names = [
+          "io_flush",
+          "logical_count",
+          "logical_range_filter",
+          "logical_shard_list",
+          "logical_select",
+          "normalize",
+          "object_exist",
+          "select",
+          "status",
+        ]
+        @care_order = true
+        @verify_cache = false
+        @ignored_drilldown_keys = []
+        @stop_on_failure = false
       end
 
-      class Options
-        attr_reader :groonga1
-        attr_reader :groonga2
-        attr_accessor :n_clients
-        attr_writer :request_queue_size
-        attr_writer :disable_cache
-        attr_accessor :target_command_names
-        attr_accessor :output_path
-        attr_accessor :care_order
-        attr_writer :verify_cache
-        def initialize
-          @groonga1 = GroongaOptions.new
-          @groonga2 = GroongaOptions.new
-          @n_clients = 8
-          @request_queue_size = nil
-          @disable_cache = false
-          @output_path = nil
-          @target_command_names = [
-            "io_flush",
-            "logical_count",
-            "logical_range_filter",
-            "logical_shard_list",
-            "logical_select",
-            "normalize",
-            "object_exist",
-            "select",
-            "status",
-          ]
-          @care_order = true
-          @verify_cache = false
-        end
+      def request_queue_size
+        @request_queue_size || @n_clients * 3
+      end
 
-        def request_queue_size
-          @request_queue_size || @n_clients * 3
-        end
+      def disable_cache?
+        @disable_cache
+      end
 
-        def disable_cache?
-          @disable_cache
-        end
+      def verify_cache?
+        @verify_cache
+      end
 
-        def verify_cache?
-          @verify_cache
-        end
+      def stop_on_failure?
+        @stop_on_failure
+      end
 
-        def target_command_name?(name)
-          return false if name.nil?
+      def target_command_name?(name)
+        return false if name.nil?
 
-          @target_command_names.any? do |name_pattern|
-            flags = 0
-            flags |= File::FNM_EXTGLOB if File.const_defined?(:FNM_EXTGLOB)
-            File.fnmatch(name_pattern, name, flags)
-          end
+        @target_command_names.any? do |name_pattern|
+          flags = 0
+          flags |= File::FNM_EXTGLOB if File.const_defined?(:FNM_EXTGLOB)
+          File.fnmatch(name_pattern, name, flags)
         end
+      end
 
-        def create_output(&block)
-          if @output_path
-            FileUtils.mkdir_p(File.dirname(@output_path))
-            File.open(@output_path, "w", &block)
-          else
-            yield($stdout)
-          end
+      def create_output(&block)
+        if @output_path
+          FileUtils.mkdir_p(File.dirname(@output_path))
+          File.open(@output_path, "w", &block)
+        else
+          yield($stdout)
         end
       end
+    end
 
-      class GroongaOptions
-        attr_accessor :host
-        attr_accessor :port
-        attr_accessor :protocol
-        def initialize
-          @host     = "127.0.0.1"
-          @port     = 10041
-          @protocol = :gqtp
-        end
+    class GroongaOptions
+      attr_accessor :host
+      attr_accessor :port
+      attr_accessor :protocol
+      def initialize
+        @host     = "127.0.0.1"
+        @port     = 10041
+        @protocol = :gqtp
+      end
 
-        def create_client(&block)
-          Groonga::Client.open(:host     => @host,
-                               :port     => @port,
-                               :protocol => @protocol,
-                               &block)
-        end
+      def create_client(&block)
+        Groonga::Client.open(:host     => @host,
+                             :port     => @port,
+                             :protocol => @protocol,
+                             &block)
       end
     end
+  end
 end

data/lib/groonga-query-log/version.rb

@@ -15,5 +15,5 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module GroongaQueryLog
-  VERSION = "1.3.0"
+  VERSION = "1.3.1"
 end

data/test/test-response-comparer.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2016  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2014-2018  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -115,14 +115,33 @@ class ResponseComparerTest < Test::Unit::TestCase
       end
     end
 
-    class SortbyTest < self
+    class SortKeysTest < self
       class DetectScoreSortTest < self
         private
-        def score_sort?(sortby)
-          @command["sortby"] = sortby
+        def score_sort?(sort_keys)
+          @command[:sort_keys] = sort_keys
           comparer([[[0], []]], [[[0], []]]).send(:score_sort?)
         end
 
+        class ParameterNameTest < self
+          def score_sort?(parameter_name)
+            @command[parameter_name] = "_score"
+            comparer([[[0], []]], [[[0], []]]).send(:score_sort?)
+          end
+
+          def test_sortby
+            assert do
+              score_sort?(:sortby)
+            end
+          end
+
+          def test_sort_keys
+            assert do
+              score_sort?(:sort_keys)
+            end
+          end
+        end
+
         class NoScoreTest < self
           def test_nil
             assert_false(score_sort?(nil))
@@ -306,6 +325,119 @@ class ResponseComparerTest < Test::Unit::TestCase
       end
     end
 
+    class FloatAccurancy < self
+      def create_response(latitude, longitude)
+        [
+          [
+            [1],
+            [["_id", "UInt32"], ["latitude", "Float"], ["longitude", "Float"]],
+            [1, latitude, longitude],
+          ]
+        ]
+      end
+
+      def test_all_output_columns
+        response1 = create_response(35.6705996342355,  139.683422370535)
+        response2 = create_response(35.67059963423547, 139.6834223705349)
+        assert do
+          same?(response1, response2)
+        end
+      end
+
+      def test_unary_minus_output_column
+        @command["output_columns"] = "_id, -value, latitude, longitude"
+        response1 = create_response(35.6705996342355,  139.683422370535)
+        response2 = create_response(35.67059963423547, 139.6834223705349)
+        assert do
+          same?(response1, response2)
+        end
+      end
+
+      def test_specific_output_columns
+        @command["output_columns"] = "_id, latitude, longitude"
+        response1 = create_response(35.6705996342355,  139.683422370535)
+        response2 = create_response(35.67059963423547, 139.6834223705349)
+        assert do
+          same?(response1, response2)
+        end
+      end
+    end
+
+    class DrilldownTest < self
+      def create_response(drilldown)
+        [
+          [
+            [10],
+            [["_id", "UInt32"]],
+          ],
+          [
+            [drilldown.size * 2],
+            [["_key", "ShortText"], ["_nsubrecs", "Int32"]],
+            *drilldown,
+          ]
+        ]
+      end
+
+      def test_same
+        response1 = create_response([["A", 10], ["B", 2]])
+        response2 = create_response([["A", 10], ["B", 2]])
+        assert do
+          same?(response1, response2)
+        end
+      end
+
+      def test_not_same
+        response1 = create_response([["A", 11], ["B", 2]])
+        response2 = create_response([["A", 10], ["B", 2]])
+        assert do
+          not same?(response1, response2)
+        end
+      end
+
+      class IgnoreDrilldownKeysTest < self
+        def create_response(drilldown1, drilldown2)
+          [
+            [
+              [10],
+              [["_id", "UInt32"]],
+            ],
+            [
+              [drilldown1.size * 2],
+              [["_key", "ShortText"], ["_nsubrecs", "Int32"]],
+              *drilldown1,
+            ],
+            [
+              [drilldown2.size * 2],
+              [["_key", "ShortText"], ["_nsubrecs", "Int32"]],
+              *drilldown2,
+            ],
+          ]
+        end
+
+        def test_same
+          @command["drilldown"] = "column1, column2"
+          response1 = create_response([["A", 10], ["B", 2]],
+                                      [["a", 11], ["b", 10]])
+          response2 = create_response([["A", 10], ["B", 2]],
+                                      [["a", 99], ["b", 20]])
+          assert do
+            same?(response1, response2, ignored_drilldown_keys: ["column2"])
+          end
+        end
+
+        def test_not_same
+          @command["drilldown"] = "column1, column2"
+          response1 = create_response([["A", 10], ["B", 2]],
+                                      [["a", 11], ["b", 10]])
+          response2 = create_response([["A", 10], ["B", 2]],
+                                      [["a", 99], ["b", 20]])
+          assert do
+            not same?(response1, response2)
+          end
+        end
+      end
+    end
+
     class ErrorTest < self
       def test_with_location
         response1_header = [