groonga-query-log 1.3.7 → 1.3.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f15d15e5f0ca25ffbd6e5dee08c732a4cebb35b4ad4eaa44fcacca85e0e6b386
-  data.tar.gz: 14b68832f85aff8c122477a5d5ed791a5545c370da51ed291588454216e3a6e9
+  metadata.gz: 661030b254c034e486074270e46e90f9ea593ceea13f1e648f9e7b76fe75bb3c
+  data.tar.gz: c8cae31736b8265e9b160019dbf0c72e0944e4e6113b94ad67a3a897e3bb240a
 SHA512:
-  metadata.gz: b0240ddbc1584dc007f55ad7a742c6b6a7f7c643a6261f08f61bd8ba5a2359e240128d2614be6f7ce1c1be8fe034f1d22906a1222c7ceb507e929c093467b1a8
-  data.tar.gz: a95c78d3864012a3711dd3ff2e0d54bab936fafdfd4a2fadafc04131e437272ee057221a16f07fa5f6e8478492ac97fa547a872fe075724800b43d0e34a2fd41
+  metadata.gz: 821547ca78bda0bc51ea23f12c0f1d72d3bbec079e96ed24884791b355279d9904f5a2a2e03b3f064bc3ca13d519df3be30153effaa530686659bfaf0235780c
+  data.tar.gz: fb253f2e4d5d50c6a4f0cc056b7b270647733d6c4b1f99e85316ac30f41f41944bcdfca74060d166f3f6da2094bf8727db6fafe91214aa4ee59abf33c22708c9

data/doc/text/news.md

@@ -1,5 +1,41 @@
 # News
 
+## 1.3.8: 2018-10-18
+
+### Improvements
+
+  * `groonga-query-log-verify-server`:
+
+    * Changed to flush logs as soon as possible.
+
+    * Added `--vector-not-equal-empty-string` option.
+
+    * Added support an unification for `"null"` and `null` column types.
+
+  * `groonga-query-log-run-regression-test`:
+
+    * Changed to flush logs as soon as possible.
+
+    * Added support for rewriting `vector == ...` with `vector @ ...`.
+
+    * Added support for logging rewriting filters.
+
+    * Added `--vector-not-equal-empty-string` option.
+
+    * Added support an unification for `"null"` and `null` column types.
+
+### Fixes
+
+  * `groonga-query-log-verify-server`:
+
+    * Fixed a bug that logged contents are removed with `.tar.gz`
+      query log.
+
+  * `groonga-query-log-run-regression-test`:
+
+    * Fixed a bug that logged contents are removed with `.tar.gz`
+      query log.
+
 ## 1.3.7: 2018-09-11
 
 ### Improvements

data/lib/groonga-query-log/command/run-regression-test.rb

@@ -45,6 +45,9 @@ module GroongaQueryLog
         @skip_finished_queries = false
         @output_query_log = false
         @stop_on_failure = false
+        @rewrite_vector_equal = false
+        @rewrite_vector_not_equal_empty_string = false
+        @vector_accessors = []
 
         @care_order = true
         @ignored_drilldown_keys = []
@@ -151,6 +154,22 @@ module GroongaQueryLog
                   "(#{@stop_on_failure})") do |boolean|
           @stop_on_failure = boolean
         end
+        parser.on("--[no-]rewrite-vector-equal",
+                  "Rewrite 'vector == ...' with 'vector @ ...'",
+                  "(#{@rewrite_vector_equal})") do |boolean|
+          @rewrite_vector_equal = boolean
+        end
+        parser.on("--[no-]rewrite-vector-not-equal-empty-string",
+                  "Rewrite 'vector != \"\"' with 'false'",
+                  "(#{@rewrite_vector_not_equal_empty_string})") do |boolean|
+          @rewrite_vector_not_equal_empty_string = boolean
+        end
+        parser.on("--vector-accessor=ACCESSOR",
+                  "Mark ACCESSOR as rewrite vector targets",
+                  "You can specify multiple vector accessors by",
+                  "specifying this option multiple times") do |accessor|
+          @vector_accessors << accessor
+        end
 
         parser.separator("")
         parser.separator("Comparisons:")
@@ -208,6 +227,10 @@ module GroongaQueryLog
           :skip_finished_queries => @skip_finished_queries,
           :ignored_drilldown_keys => @ignored_drilldown_keys,
           :stop_on_failure => @stop_on_failure,
+          :rewrite_vector_equal => @rewrite_vector_equal,
+          :rewrite_vector_not_equal_empty_string =>
+            @rewrite_vector_not_equal_empty_string,
+          :vector_accessors => @vector_accessors,
           :target_command_names => @target_command_names,
           :read_timeout => @read_timeout,
         }
@@ -228,7 +251,16 @@ module GroongaQueryLog
                           server_options)
       end
 
+      module Loggable
+        def puts(*args)
+          $stdout.puts(*args)
+          $stdout.flush
+        end
+      end
+
       class GroongaServer
+        include Loggable
+
         attr_reader :host, :port
         def initialize(groonga, groonga_options, database_path, options)
           @input_directory = options[:input_directory] || Pathname.new(".")
@@ -369,6 +401,8 @@ module GroongaQueryLog
       end
 
       class Tester
+        include Loggable
+
         def initialize(old, new, options)
           @old = old
           @new = new
@@ -473,6 +507,17 @@ module GroongaQueryLog
           if @stop_on_failure
             command_line << "--stop-on-failure"
           end
+          if @options[:rewrite_vector_equal]
+            command_line << "--rewrite-vector-equal"
+          end
+          if @options[:rewrite_vector_not_equal_empty_string]
+            command_line << "--rewrite-vector-not-equal-empty-string"
+          end
+          vector_accessors = @options[:vector_accessors] || []
+          vector_accessors.each do |vector_accessor|
+            command_line << "--vector-accessor"
+            command_line << vector_accessor
+          end
           if @options[:target_command_names]
             command_line << "--target-command-names"
             command_line << @options[:target_command_names].join(",")

data/lib/groonga-query-log/command/verify-server.rb

@@ -201,6 +201,25 @@ module GroongaQueryLog
           @options.stop_on_failure = boolean
         end
 
+        parser.on("--[no-]rewrite-vector-equal",
+                  "Rewrite 'vector == ...' with 'vector @ ...'",
+                  "(#{@options.rewrite_vector_equal?})") do |boolean|
+          @options.rewrite_vector_equal = boolean
+        end
+
+        parser.on("--[no-]rewrite-vector-not-equal-empty-string",
+                  "Rewrite 'vector != \"\"' with 'false'",
+                  "(#{@options.rewrite_vector_not_equal_empty_string?})") do |boolean|
+          @options.rewrite_vector_not_equal_empty_string = boolean
+        end
+
+        parser.on("--vector-accessor=ACCESSOR",
+                  "Mark ACCESSOR as rewrite vector targets",
+                  "You can specify multiple vector accessors by",
+                  "specifying this option multiple times") do |accessor|
+          @options.vector_accessors << accessor
+        end
+
         parser.separator("Debug options:")
         parser.separator("")
 

data/lib/groonga-query-log/filter-rewriter.rb

@@ -0,0 +1,59 @@
+# Copyright (C) 2018  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module GroongaQueryLog
+  class FilterRewriter
+    def initialize(filter, options={})
+      @filter = filter
+      @options = options
+      @vector_accessors = @options[:vector_accessors] || []
+    end
+
+    def rewrite
+      rewritten = @filter
+      if @options[:rewrite_vector_equal]
+        rewritten = rewrite_vector_equal(rewritten)
+      end
+      if @options[:rewrite_vector_not_equal_empty_string]
+        rewritten = rewrite_vector_not_equal_empty_string(rewritten)
+      end
+      rewritten
+    end
+
+    private
+    def rewrite_vector_equal(filter)
+      filter.gsub(/([a-zA-Z0-9_.]+) *==/) do |matched|
+        variable = $1
+        if @vector_accessors.include?(variable)
+          "#{variable} @"
+        else
+          matched
+        end
+      end
+    end
+
+    def rewrite_vector_not_equal_empty_string(filter)
+      filter.gsub(/([a-zA-Z0-9_.]+) *!= *(?:''|"")/) do |matched|
+        variable = $1
+        if @vector_accessors.include?(variable)
+          "false"
+        else
+          matched
+        end
+      end
+    end
+  end
+end

data/lib/groonga-query-log/response-comparer.rb

@@ -113,8 +113,8 @@ module GroongaQueryLog
       n_hits2 = records_result2[0]
       return false if n_hits1 != n_hits2
 
-      columns1 = records_result1[1]
-      columns2 = records_result2[1]
+      columns1 = normalize_columns(records_result1[1])
+      columns2 = normalize_columns(records_result2[1])
       if all_output_columns?
         columns1.sort_by(&:first) == columns2.sort_by(&:first)
       else
@@ -137,8 +137,8 @@ module GroongaQueryLog
       n_hits2 = records_result2[0]
       return false if n_hits1 != n_hits2
 
-      columns1 = records_result1[1]
-      columns2 = records_result2[1]
+      columns1 = normalize_columns(records_result1[1])
+      columns2 = normalize_columns(records_result2[1])
       records1 = records_result1[2..-1]
       records2 = records_result2[2..-1]
 
@@ -185,8 +185,8 @@ module GroongaQueryLog
       n_hits2 = records_result2[0]
       return false if n_hits1 != n_hits2
 
-      columns1 = records_result1[1]
-      columns2 = records_result2[1]
+      columns1 = normalize_columns(records_result1[1])
+      columns2 = normalize_columns(records_result2[1])
       return false if columns1.sort_by(&:first) != columns2.sort_by(&:first)
 
       column_to_index1 = make_column_to_index_map(columns1)
@@ -223,8 +223,8 @@ module GroongaQueryLog
       n_hits2 = record_set2[0]
       return false if n_hits1 != n_hits2
 
-      columns1 = record_set1[1]
-      columns2 = record_set2[1]
+      columns1 = normalize_columns(record_set1[1])
+      columns2 = normalize_columns(record_set2[1])
       return false if columns1 != columns2
 
       records1 = record_set1[2..-1]
@@ -282,6 +282,13 @@ module GroongaQueryLog
       true
     end
 
+    def normalize_columns(columns)
+      columns.collect do |name, type|
+        type = nil if type == "null"
+        [name, type]
+      end
+    end
+
     def normalize_value(value, column)
       type = column[1]
       case type

data/lib/groonga-query-log/server-verifier.rb

@@ -19,6 +19,7 @@ require "thread"
 
 require "groonga/client"
 
+require "groonga-query-log/filter-rewriter"
 require "groonga-query-log/parser"
 require "groonga-query-log/response-comparer"
 
@@ -118,7 +119,7 @@ module GroongaQueryLog
 
     def run_reporter
       Thread.new do
-        @options.create_output do |output|
+        @options.open_output do |output|
           loop do
             result = @different_results.pop
             break if result.nil?
@@ -135,6 +136,8 @@ module GroongaQueryLog
     def verify_command(groonga1_client, groonga2_client, command)
       command["cache"] = "no" if @options.disable_cache?
       command["output_type"] = "json"
+      rewrite_filter(command, "filter")
+      rewrite_filter(command, "scorer")
       response1 = groonga1_client.execute(command)
       response2 = groonga2_client.execute(command)
       compare_options = {
@@ -148,6 +151,21 @@ module GroongaQueryLog
       end
     end
 
+    def rewrite_filter(command, name)
+      target = command[name]
+      return if target.nil?
+      return unless @options.need_filter_rewrite?
+
+      rewriter = FilterRewriter.new(target, @options.to_filter_rewriter_options)
+      rewritten_target = rewriter.rewrite
+      return if target == rewritten_target
+
+      $stderr.puts("Rewritten #{name}")
+      $stderr.puts("  Before: #{target}")
+      $stderr.puts("   After: #{rewritten_target}")
+      command[name] = rewritten_target
+    end
+
     def report_result(output, result)
       @same = false
       command, response1, response2 = result
@@ -155,6 +173,7 @@ module GroongaQueryLog
       output.puts("command: #{command_source}")
       output.puts("response1: #{response1.body.to_json}")
       output.puts("response2: #{response2.body.to_json}")
+      output.flush
     end
 
     def log_client_error(error)
@@ -181,6 +200,9 @@ module GroongaQueryLog
       attr_writer :verify_cache
       attr_accessor :ignored_drilldown_keys
       attr_writer :stop_on_failure
+      attr_writer :rewrite_vector_equal
+      attr_writer :rewrite_vector_not_equal_empty_string
+      attr_accessor :vector_accessors
       def initialize
         @groonga1 = GroongaOptions.new
         @groonga2 = GroongaOptions.new
@@ -188,6 +210,7 @@ module GroongaQueryLog
         @request_queue_size = nil
         @disable_cache = false
         @output_path = nil
+        @output_opened = false
         @target_command_names = [
           "io_flush",
           "logical_count",
@@ -203,6 +226,9 @@ module GroongaQueryLog
         @verify_cache = false
         @ignored_drilldown_keys = []
         @stop_on_failure = false
+        @rewrite_vector_equal = false
+        @rewrite_vector_not_equal_empty_string = false
+        @vector_accessors = []
       end
 
       def request_queue_size
@@ -221,6 +247,14 @@ module GroongaQueryLog
         @stop_on_failure
       end
 
+      def rewrite_vector_equal?
+        @rewrite_vector_equal
+      end
+
+      def rewrite_vector_not_equal_empty_string?
+        @rewrite_vector_not_equal_empty_string
+      end
+
       def target_command_name?(name)
         return false if name.nil?
 
@@ -231,14 +265,34 @@ module GroongaQueryLog
         end
       end
 
-      def create_output(&block)
+      def open_output(&block)
         if @output_path
-          FileUtils.mkdir_p(File.dirname(@output_path))
-          File.open(@output_path, "w", &block)
+          if @output_opened
+            mode = "a"
+          else
+            FileUtils.mkdir_p(File.dirname(@output_path))
+            mode = "w"
+            @output_opened = true
+          end
+          File.open(@output_path, mode, &block)
         else
           yield($stdout)
         end
       end
+
+      def need_filter_rewrite?
+        rewrite_vector_equal? or
+          rewrite_vector_not_equal_empty_string?
+      end
+
+      def to_filter_rewriter_options
+        {
+          :rewrite_vector_equal => rewrite_vector_equal?,
+          :rewrite_vector_not_equal_empty_string =>
+            rewrite_vector_not_equal_empty_string?,
+          :vector_accessors => vector_accessors,
+        }
+      end
     end
 
     class GroongaOptions

data/lib/groonga-query-log/version.rb

@@ -15,5 +15,5 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module GroongaQueryLog
-  VERSION = "1.3.7"
+  VERSION = "1.3.8"
 end

data/test/test-filter-rewriter.rb

@@ -0,0 +1,75 @@
+# Copyright (C) 2018  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+class FilterRewriterTest < Test::Unit::TestCase
+  private
+  def rewrite(filter, options)
+    rewriter = GroongaQueryLog::FilterRewriter.new(filter, options)
+    rewriter.rewrite
+  end
+
+  class VectorEqualTest < self
+    def rewrite(filter, vector_accessors)
+      super(filter,
+            :rewrite_vector_equal => true,
+            :vector_accessors => vector_accessors)
+    end
+
+    def test_not_target_accessor
+      assert_equal("vector == \"value\"",
+                   rewrite("vector == \"value\"",
+                           ["nonexistent"]))
+    end
+
+    def test_parenthesis
+      assert_equal("((vector @ \"value\"))",
+                   rewrite("((vector == \"value\"))",
+                           ["vector"]))
+    end
+
+    def test_under_score
+      assert_equal("vector_column @ \"value\"",
+                   rewrite("vector_column == \"value\"",
+                           ["vector_column"]))
+    end
+  end
+
+  class VectorNotEqualEmptyStringTest < self
+    def rewrite(filter, vector_accessors)
+      super(filter,
+            :rewrite_vector_not_equal_empty_string => true,
+            :vector_accessors => vector_accessors)
+    end
+
+    def test_not_target_accessor
+      assert_equal("vector != \"\"",
+                   rewrite("vector != \"\"",
+                           ["nonexistent"]))
+    end
+
+    def test_parenthesis
+      assert_equal("((false))",
+                   rewrite("((vector != \"\"))",
+                           ["vector"]))
+    end
+
+    def test_under_score
+      assert_equal("false",
+                   rewrite("vector_column != \"\"",
+                           ["vector_column"]))
+    end
+  end
+end

data/test/test-response-comparer.rb

@@ -363,6 +363,54 @@ class ResponseComparerTest < Test::Unit::TestCase
       end
     end
 
+    class ColumnType < self
+      class Null < self
+        def create_response(type)
+          [
+            [
+              [1],
+              [["snippet_html", type]],
+              ["...snippet..."],
+            ]
+          ]
+        end
+
+        def test_all_output_columns
+          response1 = create_response("null")
+          response2 = create_response(nil)
+          assert do
+            same?(response1, response2)
+          end
+        end
+
+        def test_unary_minus_output_column
+          @command["output_columns"] = "-value, snippet_html(body)"
+          response1 = create_response("null")
+          response2 = create_response(nil)
+          assert do
+            same?(response1, response2)
+          end
+        end
+
+        def test_specific_output_column
+          @command["output_columns"] = "snippet_html(body)"
+          response1 = create_response("null")
+          response2 = create_response(nil)
+          assert do
+            same?(response1, response2)
+          end
+        end
+
+        def test_not_care_order
+          response1 = create_response("null")
+          response2 = create_response(nil)
+          assert do
+            same?(response1, response2, care_order: false)
+          end
+        end
+      end
+    end
+
     class DrilldownTest < self
       def create_response(drilldown)
         [

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: groonga-query-log
 version: !ruby/object:Gem::Version
-  version: 1.3.7
+  version: 1.3.8
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-11 00:00:00.000000000 Z
+date: 2018-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: groonga-command-parser
@@ -224,6 +224,7 @@ files:
 - lib/groonga-query-log/command/run-regression-test.rb
 - lib/groonga-query-log/command/show-running-queries.rb
 - lib/groonga-query-log/command/verify-server.rb
+- lib/groonga-query-log/filter-rewriter.rb
 - lib/groonga-query-log/incompatibility-detector.rb
 - lib/groonga-query-log/memory-leak-detector.rb
 - lib/groonga-query-log/parser.rb
@@ -289,6 +290,7 @@ files:
 - test/fixtures/target-tables.expected
 - test/helper.rb
 - test/run-test.rb
+- test/test-filter-rewriter.rb
 - test/test-incompatibility-detector.rb
 - test/test-parser.rb
 - test/test-replayer.rb
@@ -321,6 +323,7 @@ summary: Groonga-query-log is a collection of library and tools to process [Groo
   as a library. You can analyze your Groonga's queries and test with your Groonga's
   query log by using groonga-query-log as a tool.
 test_files:
+- test/test-filter-rewriter.rb
 - test/fixtures/multi.expected
 - test/fixtures/target-tables.expected
 - test/fixtures/target-commands.expected