groonga-query-log 1.1.5 → 1.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ecc113262236c80addf12b23cf930c1d85cee8ae
-  data.tar.gz: b168f92861f723e4ede99f5b023a48050b52e8d3
+  metadata.gz: fd3f1d02cb11d8b4cf26fd7cbc1418a56e6ef7d9
+  data.tar.gz: f9b7f8f92de3af6e8652db79e53652ee84a4ee1d
 SHA512:
-  metadata.gz: 6bdb3eabb1ad973af6a1f820ddf8263f14e05543db0864523b41a44e25ff4e84f220ec073778575067308677b41380adc4056e7f99141e51ad6c79b79d5c2fcd
-  data.tar.gz: a4712efb262754f0563b3458d7e25788a2844b4483d0cbb3cb31ba371402ce5aece3ec5e1c2ade38a5644340e6da91e800edcb03463e62add6d411b28d1b62f2
+  metadata.gz: 3a38dfe89fc55a24c9860aa2c8cde1c2886eb3c6b11d356bf979f4074aed0ec41ac3f5cc09dfec8f9eb79b66e128b318feb08662f8a87db1a7d61951677e35fb
+  data.tar.gz: 2ef01816c6484747c88935637f870aaccd1a0cc7580a14ee639cc0f78b1b95b0cbc52bd0129be8a2da8a9af955d0a0b622a689a3cad7839de8297c3b90656fda

data/doc/text/news.md

@@ -1,5 +1,19 @@
 # News
 
+## 1.1.6: 2015-08-19
+
+### Improvements
+
+  * groonga-query-log-run-regression-test: Added `--no-care-order`
+    option that doesn't care order of records in response.
+  * groonga-query-log-verify-server: Added `--no-care-order`
+    option that doesn't care order of records in response.
+  * groonga-query-log-verify-server: Added the following commands to
+    the default target command names:
+    * `logical_count`
+    * `logical_range_filter`
+    * `logical_select`
+
 ## 1.1.5: 2015-08-12
 
 ### Improvements

data/lib/groonga/query-log/command/extract.rb

@@ -26,167 +26,167 @@ require "groonga/query-log/command-line-utils"
 module Groonga
   module QueryLog
     module Command
-    class Extract
-      include CommandLineUtils
+      class Extract
+        include CommandLineUtils
 
-      class Error < StandardError
-      end
-
-      attr_accessor :options
-      attr_reader :option_parser
+        class Error < StandardError
+        end
 
-      def initialize
-        @options = nil
-        @option_parser = nil
-        setup_options
-      end
+        attr_accessor :options
+        attr_reader :option_parser
 
-      # Executes extractor for groonga's query logs.
-      # "groonga-query-log-extract" command runs this method.
-      #
-      # @example
-      #   extractor = Groonga::QueryLog::Command::Extract.new
-      #   extractor.run("--output", "commands.output",
-      #                 "--command", "select",
-      #                 "query.log")
-      #
-      # If only paths of query log files are specified,
-      # this method prints command(s) of them to console.
-      #
-      # @param [Array<String>] arguments arguments for
-      #   groonga-query-log-extract. Please execute
-      #   "groonga-query-log-extract --help" or see #setup_options.
-      def run(arguments)
-        begin
-          log_paths = @option_parser.parse!(arguments)
-        rescue OptionParser::ParseError
-          $stderr.puts($!.message)
-          return false
+        def initialize
+          @options = nil
+          @option_parser = nil
+          setup_options
         end
 
-        if log_paths.empty?
-          unless log_via_stdin?
-            $stderr.puts("Error: Please specify input log files.")
+        # Executes extractor for groonga's query logs.
+        # "groonga-query-log-extract" command runs this method.
+        #
+        # @example
+        #   extractor = Groonga::QueryLog::Command::Extract.new
+        #   extractor.run("--output", "commands.output",
+        #                 "--command", "select",
+        #                 "query.log")
+        #
+        # If only paths of query log files are specified,
+        # this method prints command(s) of them to console.
+        #
+        # @param [Array<String>] arguments arguments for
+        #   groonga-query-log-extract. Please execute
+        #   "groonga-query-log-extract --help" or see #setup_options.
+        def run(arguments)
+          begin
+            log_paths = @option_parser.parse!(arguments)
+          rescue OptionParser::ParseError
+            $stderr.puts($!.message)
             return false
           end
-          log = $stdin
-        else
-          log = log_paths
-        end
 
-        if @options.output_path
-          File.open(@options.output_path, "w") do |output|
-            extract(log, output)
+          if log_paths.empty?
+            unless log_via_stdin?
+              $stderr.puts("Error: Please specify input log files.")
+              return false
+            end
+            log = $stdin
+          else
+            log = log_paths
           end
-        else
-          extract(log, $stdout)
-        end
 
-        true
-      end
-
-      private
-      def setup_options
-        @options = OpenStruct.new
-        @options.unify_format = nil
-        @options.commands = []
-        @options.exclude_commands = []
-        @options.output_path = nil
-        @option_parser = OptionParser.new do |parser|
-          parser.version = VERSION
-          parser.banner += " QUERY_LOG1 ..."
-
-          available_formats = ["uri", "command"]
-          parser.on("--unify-format=FORMAT",
-                    available_formats,
-                    "Unify command format to FORMAT.",
-                    "(#{available_formats.join(', ')})",
-                    "[not unify]") do |format|
-            @options.unify_format = format
+          if @options.output_path
+            File.open(@options.output_path, "w") do |output|
+              extract(log, output)
+            end
+          else
+            extract(log, $stdout)
           end
 
-          parser.on("--command=COMMAND",
-                    "Extract only COMMAND.",
-                    "To extract one or more commands,",
-                    "specify this command a number of times.",
-                    "Use /.../ as COMMAND to match command with regular expression.",
-                    "[all commands]") do |command|
-            case command
-            when /\A\/(.*)\/(i)?\z/
-              @options.commands << Regexp.new($1, $2 == "i")
-            when
-              @options.commands << command
+          true
+        end
+
+        private
+        def setup_options
+          @options = OpenStruct.new
+          @options.unify_format = nil
+          @options.commands = []
+          @options.exclude_commands = []
+          @options.output_path = nil
+          @option_parser = OptionParser.new do |parser|
+            parser.version = VERSION
+            parser.banner += " QUERY_LOG1 ..."
+
+            available_formats = ["uri", "command"]
+            parser.on("--unify-format=FORMAT",
+                      available_formats,
+                      "Unify command format to FORMAT.",
+                      "(#{available_formats.join(', ')})",
+                      "[not unify]") do |format|
+              @options.unify_format = format
             end
-          end
 
-          parser.on("--exclude-command=COMMAND",
-                    "Don't extract COMMAND.",
-                    "To ignore one or more commands,",
-                    "specify this command a number of times.",
-                    "Use /.../ as COMMAND to match command with regular expression.",
-                    "[no commands]") do |command|
-            case command
-            when /\A\/(.*)\/(i)?\z/
-              @options.exclude_commands << Regexp.new($1, $2 == "i")
-            when
-              @options.exclude_commands << command
+            parser.on("--command=COMMAND",
+                      "Extract only COMMAND.",
+                      "To extract one or more commands,",
+                      "specify this command a number of times.",
+                      "Use /.../ as COMMAND to match command with regular expression.",
+                      "[all commands]") do |command|
+              case command
+              when /\A\/(.*)\/(i)?\z/
+                @options.commands << Regexp.new($1, $2 == "i")
+              when
+                @options.commands << command
+              end
             end
-          end
 
-          parser.on("--output=PATH",
-                    "Output to PATH.",
-                    "[standard output]") do |path|
-            @options.output_path = path
+            parser.on("--exclude-command=COMMAND",
+                      "Don't extract COMMAND.",
+                      "To ignore one or more commands,",
+                      "specify this command a number of times.",
+                      "Use /.../ as COMMAND to match command with regular expression.",
+                      "[no commands]") do |command|
+              case command
+              when /\A\/(.*)\/(i)?\z/
+                @options.exclude_commands << Regexp.new($1, $2 == "i")
+              when
+                @options.exclude_commands << command
+              end
+            end
+
+            parser.on("--output=PATH",
+                      "Output to PATH.",
+                      "[standard output]") do |path|
+              @options.output_path = path
+            end
           end
         end
-      end
 
-      def extract(log, output)
-        if log.instance_of?(Array)
-          log.each do |log_path|
-            File.open(log_path) do |log_file|
-              extract_command(log_file, output)
+        def extract(log, output)
+          if log.instance_of?(Array)
+            log.each do |log_path|
+              File.open(log_path) do |log_file|
+                extract_command(log_file, output)
+              end
             end
+          else
+            extract_command(log, output)
           end
-        else
-          extract_command(log, output)
         end
-      end
 
-      def extract_command(log, output)
-        parser = Groonga::QueryLog::Parser.new
-        parser.parse(log) do |statistic|
-          command = statistic.command
-          next unless target?(command)
-          command_text = nil
-          case @options.unify_format
-          when "uri"
-            command_text = command.to_uri_format
-          when "command"
-            command_text = command.to_command_format
-          else
-            command_text = statistic.raw_command
+        def extract_command(log, output)
+          parser = Groonga::QueryLog::Parser.new
+          parser.parse(log) do |statistic|
+            command = statistic.command
+            next unless target?(command)
+            command_text = nil
+            case @options.unify_format
+            when "uri"
+              command_text = command.to_uri_format
+            when "command"
+              command_text = command.to_command_format
+            else
+              command_text = statistic.raw_command
+            end
+            output.puts(command_text)
           end
-          output.puts(command_text)
         end
-      end
 
-      def target?(command)
-        name = command.name
-        target_commands = @options.commands
-        exclude_commands = @options.exclude_commands
+        def target?(command)
+          name = command.name
+          target_commands = @options.commands
+          exclude_commands = @options.exclude_commands
 
-        unless target_commands.empty?
-          return target_commands.any? {|target_command| target_command === name}
-        end
+          unless target_commands.empty?
+            return target_commands.any? {|target_command| target_command === name}
+          end
 
-        unless exclude_commands.empty?
-          return (not exclude_commands.any? {|exclude_command| exclude_command === name})
-        end
+          unless exclude_commands.empty?
+            return (not exclude_commands.any? {|exclude_command| exclude_command === name})
+          end
 
-        true
+          true
+        end
       end
     end
-    end
   end
 end

data/lib/groonga/query-log/command/run-regression-test.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2014-2015  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -43,6 +43,7 @@ module Groonga
           @run_queries = true
           @skip_finished_queries = false
           @output_query_log = false
+          @care_order = true
         end
 
         def run(command_line)
@@ -124,6 +125,10 @@ module Groonga
                     "Output query log in verified target Groonga servers") do
             @output_query_log = true
           end
+          parser.on("--no-care-order",
+                    "Don't care order of select response records") do
+            @care_order = false
+          end
 
           parser
         end
@@ -148,7 +153,8 @@ module Groonga
 
         def tester_options
           options = {
-            :n_clients => @n_clients,
+            :n_clients  => @n_clients,
+            :care_order => @care_order,
           }
           directory_options.merge(options)
         end
@@ -372,8 +378,9 @@ module Groonga
               "--groonga2-protocol=http",
               "--target-command-name=select",
               "--output", test_log_path.to_s,
-              query_log_path.to_s,
             ]
+            command_line << "--no-care-order" if @options[:care_order] == false
+            command_line << query_log_path.to_s
             verify_serer = VerifyServer.new
             verify_serer.run(command_line)
           end

data/lib/groonga/query-log/command/verify-server.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013-2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2015  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -124,6 +124,11 @@ module Groonga
             @options.target_command_names = names
           end
 
+          parser.on("--no-care-order",
+                    "Don't care order of select response records") do
+            @options.care_order = false
+          end
+
           parser.on("--output=PATH",
                     "Output results to PATH",
                     "[stdout]") do |path|

data/lib/groonga/query-log/response-comparer.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2014-2015  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -17,10 +17,12 @@
 module Groonga
   module QueryLog
     class ResponseComparer
-      def initialize(command, response1, response2)
+      def initialize(command, response1, response2, options={})
         @command = command
         @response1 = response1
         @response2 = response2
+        @options = options
+        @options[:care_order] = true if @options[:care_order].nil?
       end
 
       def same?
@@ -32,7 +34,7 @@ module Groonga
           end
         else
           case @command.name
-          when "select"
+          when "select", "logical_select"
             same_select_response?
           else
             same_response?
@@ -56,16 +58,26 @@ module Groonga
       end
 
       def same_select_response?
-        if random_sort?
-          same_random_sort_response?
-        elsif all_output_columns?
-          same_all_output_columns?
+        if care_order?
+          if all_output_columns?
+            same_all_output_columns?
+          else
+            same_response?
+          end
         else
-          same_response?
+          same_size_response?
         end
       end
 
+      def care_order?
+        return false unless @options[:care_order]
+        return false if random_sort?
+
+        true
+      end
+
       def random_score?
+        return false unless @command.respond_to?(:scorer)
         @command.scorer == "_score=rand()"
       end
 
@@ -81,7 +93,7 @@ module Groonga
         normalized_sort_items.include?("_score")
       end
 
-      def same_random_sort_response?
+      def same_size_response?
         records_result1 = @response1.body[0] || []
         records_result2 = @response2.body[0] || []
         return false if records_result1.size != records_result2.size

data/lib/groonga/query-log/server-verifier.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013-2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2015  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -76,7 +76,7 @@ module Groonga
               begin
                 verify_command(groonga1_client, groonga2_client,
                                statistic.command)
-              rescue Groonga::Client::Error
+              rescue
                 log_client_error($!) do
                   $stderr.puts(statistic.command.original_source)
                 end
@@ -108,7 +108,11 @@ module Groonga
         command["output_type"] = :json
         response1 = groonga1_client.execute(command)
         response2 = groonga2_client.execute(command)
-        comparer = ResponseComparer.new(command, response1, response2)
+        compare_options = {
+          :care_order => @options.care_order,
+        }
+        comparer = ResponseComparer.new(command, response1, response2,
+                                        compare_options)
         unless comparer.same?
           @different_results.push([command, response1, response2])
         end
@@ -141,6 +145,7 @@ module Groonga
         attr_writer :disable_cache
         attr_accessor :target_command_names
         attr_accessor :output_path
+        attr_accessor :care_order
         def initialize
           @groonga1 = GroongaOptions.new
           @groonga2 = GroongaOptions.new
@@ -148,7 +153,13 @@ module Groonga
           @request_queue_size = nil
           @disable_cache = false
           @output_path = nil
-          @target_command_names = ["select"]
+          @target_command_names = [
+            "logical_count",
+            "logical_range_filter",
+            "logical_select",
+            "select",
+          ]
+          @care_order = true
         end
 
         def request_queue_size

data/lib/groonga/query-log/version.rb

@@ -18,6 +18,6 @@
 
 module Groonga
   module QueryLog
-    VERSION = "1.1.5"
+    VERSION = "1.1.6"
   end
 end

data/test/{test-extractor.rb → command/test-extract.rb}

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
+# Copyright (C) 2015  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -18,17 +19,17 @@
 
 require "stringio"
 require "groonga/command"
-require "groonga/query-log/extractor"
+require "groonga/query-log/command/extract"
 
-class TestExtractor < Test::Unit::TestCase
+class ExtractCommandTest < Test::Unit::TestCase
   setup
   def setup_fixtures
-    @fixtures_path = File.join(File.dirname(__FILE__), "fixtures")
+    @fixtures_path = File.join(File.dirname(__FILE__), "..", "fixtures")
     @query_log_path = File.join(@fixtures_path, "query.log")
   end
 
   def setup
-    @extractor = Groonga::QueryLog::Extractor.new
+    @extract_command = Groonga::QueryLog::Command::Extract.new
   end
 
   class TestInputFile < self
@@ -102,7 +103,7 @@ EOC
     Tempfile.open("extract.output") do |output|
       open_error_output do |error|
         arguments << "--output" << output.path
-        if @extractor.run(arguments)
+        if @extract_command.run(arguments)
           File.read(output.path)
         else
           File.read(error.path)
@@ -137,7 +138,7 @@ EOL
     end
 
     def test_command_format
-      @extractor.options.unify_format = "command"
+      @extract_command.options.unify_format = "command"
       expected_formatted_command = "select --output_columns \"_key,name\""+
                                      " --query \"follower:@groonga\"" +
                                      " --table \"Users\"\n"
@@ -145,7 +146,7 @@ EOL
     end
 
     def test_uri_format
-      @extractor.options.unify_format = "uri"
+      @extract_command.options.unify_format = "uri"
       expected_formatted_command = "/d/select?output_columns=_key%2Cname" +
                                      "&query=follower%3A%40groonga" +
                                      "&table=Users\n"
@@ -153,7 +154,7 @@ EOL
     end
 
     def test_not_unify
-      @extractor.options.unify_format = nil
+      @extract_command.options.unify_format = nil
       expected_formatted_command = "select --table Users" +
                                      " --query follower:@groonga" +
                                      " --output_columns _key,name\n"
@@ -163,20 +164,20 @@ EOL
     private
     def extract
       output = StringIO.new
-      @extractor.send(:extract, @log, output)
+      @extract_command.send(:extract, @log, output)
       output.string
     end
   end
 
   class TestTarget < self
     def test_include
-      @extractor.options.commands = ["register"]
+      @extract_command.options.commands = ["register"]
       assert_true(target?("register"))
       assert_false(target?("dump"))
     end
 
     def test_exclude
-      @extractor.options.exclude_commands = ["dump"]
+      @extract_command.options.exclude_commands = ["dump"]
       assert_true(target?("register"))
       assert_false(target?("dump"))
     end
@@ -187,13 +188,13 @@ EOL
     end
 
     def test_regular_expression_include
-      @extractor.options.commands = [/table/]
+      @extract_command.options.commands = [/table/]
       assert_true(target?("table_create"))
       assert_false(target?("dump"))
     end
 
     def test_regular_expression_exclude
-      @extractor.options.exclude_commands = [/table/]
+      @extract_command.options.exclude_commands = [/table/]
       assert_false(target?("table_create"))
       assert_true(target?("dump"))
     end
@@ -202,7 +203,7 @@ EOL
     def target?(name)
       command_class = Groonga::Command.find(name)
       command = command_class.new(name, [])
-      @extractor.send(:target?, command)
+      @extract_command.send(:target?, command)
     end
   end
 end

data/test/test-response-comparer.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2014-2015  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -16,14 +16,15 @@
 
 class ResponseComparerTest < Test::Unit::TestCase
   private
-  def comparer(response1, response2)
+  def comparer(response1, response2, options={})
     response1 = normalize_response(response1)
     response2 = normalize_response(response2)
-    Groonga::QueryLog::ResponseComparer.new(@command, response1, response2)
+    Groonga::QueryLog::ResponseComparer.new(@command, response1, response2,
+                                            options)
   end
 
-  def same?(response1, response2)
-    comparer(response1, response2).same?
+  def same?(response1, response2, options={})
+    comparer(response1, response2, options).same?
   end
 
   def response(body)
@@ -105,7 +106,7 @@ class ResponseComparerTest < Test::Unit::TestCase
         private
         def random_score?(scorer)
           @command["scorer"] = scorer
-          comparer([[[0]]], [[[0]]]).send(:random_score?)
+          comparer([[[0], []]], [[[0], []]]).send(:random_score?)
         end
       end
     end
@@ -115,7 +116,7 @@ class ResponseComparerTest < Test::Unit::TestCase
         private
         def score_sort?(sortby)
           @command["sortby"] = sortby
-          comparer([[[0]]], [[[0]]]).send(:score_sort?)
+          comparer([[[0], []]], [[[0], []]]).send(:score_sort?)
         end
 
         class NoScoreTest < self
@@ -224,11 +225,20 @@ class ResponseComparerTest < Test::Unit::TestCase
         private
         def all_output_columns?(output_columns)
           @command["output_columns"] = output_columns if output_columns
-          comparer([[[0]]], [[[0]]]).send(:all_output_columns?)
+          comparer([[[0], []]], [[[0], []]]).send(:all_output_columns?)
         end
       end
     end
 
+    class ForceNoCareOrderTest < self
+      def test_different_order
+        @command["output_columns"] = "_id"
+        assert_true(same?([[[3], [["_id", "UInt32"]], [1], [2], [3]]],
+                          [[[3], [["_id", "UInt32"]], [3], [2], [1]]],
+                          :care_order => false))
+      end
+    end
+
     class ErrorTest < self
       def test_with_location
         response1_header = [

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: groonga-query-log
 version: !ruby/object:Gem::Version
-  version: 1.1.5
+  version: 1.1.6
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-12 00:00:00.000000000 Z
+date: 2015-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: groonga-command-parser
@@ -211,6 +211,7 @@ files:
 - lib/groonga/query-log/response-comparer.rb
 - lib/groonga/query-log/server-verifier.rb
 - lib/groonga/query-log/version.rb
+- test/command/test-extract.rb
 - test/command/test-format-regression-test-logs.rb
 - test/fixtures/multi.expected
 - test/fixtures/n_entries.expected
@@ -230,7 +231,6 @@ files:
 - test/helper.rb
 - test/run-test.rb
 - test/test-analyzer.rb
-- test/test-extractor.rb
 - test/test-incompatibility-detector.rb
 - test/test-parser.rb
 - test/test-replayer.rb
@@ -266,6 +266,7 @@ test_files:
 - test/test-analyzer.rb
 - test/test-parser.rb
 - test/test-response-comparer.rb
+- test/command/test-extract.rb
 - test/command/test-format-regression-test-logs.rb
 - test/fixtures/other-query.log
 - test/fixtures/reporter/console.expected
@@ -285,6 +286,5 @@ test_files:
 - test/helper.rb
 - test/run-test.rb
 - test/test-incompatibility-detector.rb
-- test/test-extractor.rb
 - test/test-replayer.rb
 has_rdoc: