groonga-query-log 1.3.9 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 747d884c77f1a62e7266740a0a5cfe796dc9afc5
-  data.tar.gz: 17a9c5e1f2463a1d5b4deee9d187619ceec6f690
+SHA256:
+  metadata.gz: 815281fff50a93e07460fc0bb35ea453cfc8b36881f685facf2dc61b6481b564
+  data.tar.gz: 6fb79fe569bb61b35dbe7f876348be12c236791cfc53a25118c134f286974ebe
 SHA512:
-  metadata.gz: f9ed114c27916395db0d9cf82d02b3c938b7aa338863da6972b156726646ee18cd7cff536c1483ae95af065c697a945d0fb618b2bf34e5c92de15af01e87ced7
-  data.tar.gz: 5cfe3856cca14f355c1f2d6f19c36b4da3ec61918938e6e3ecdc97ccae8c183726b6ebf30309f8cd6c234f405a910f6c4d1f419caa5a401cf5a233585940fcd7
+  metadata.gz: '083d46ede2bcb5ef1902210dcd2b4b67b95ec475704efd4c3c768827772e67fd667e6d8dc53b0d250ed5a5896366af5e7ff9b42d5708f5c3b0c90b6143d3bd6f'
+  data.tar.gz: 8d5f90bf96d8815f8c4dccdca755d3e49b13f5586b56e90d5ddd3f4af5561480b3394ed3a62c6c5b2ba5662e88fceab1a5d36c07280307a4e25094cc9fd7c8ae

data/doc/text/news.md

@@ -1,5 +1,24 @@
 # News
 
+## 1.4.0: 2019-03-13
+
+### Improvements
+
+  * `groonga-query-log-run-regression-test`:
+
+    * Added support for rewriting `"column1 @ \"keyword\" && column2 @~ \"^(?!.*keyword1|keyword2|...).+$\""`.
+      Specify --rewrite-not-or-regular-expression option to enable it.
+
+### Fixes
+
+  * `groonga-query-log-check-crash`:
+
+    * Fixed missing nil check when request path is "/".
+
+  * `GroongaQueryLog::Statistic`:
+
+    * Fixed missing nil check in `#to_hash` when request path is "/".
+
 ## 1.3.9: 2018-11-20
 
 ### Improvements

data/lib/groonga-query-log/command/check-crash.rb

@@ -209,7 +209,10 @@ module GroongaQueryLog
         end
 
         def check_query_log_statistic(path, statistic)
-          case statistic.command.command_name
+          command = statistic.command
+          return if command.nil?
+
+          case command.command_name
           when "load"
             @flushed = false
             @unflushed_statistics << statistic
@@ -220,10 +223,10 @@ module GroongaQueryLog
             @flushed = false
             @unflushed_statistics << statistic
           when "io_flush"
-            check_io_flush(statistic.command)
+            check_io_flush(command)
           when "database_unmap"
             @unflushed_statistics.reject! do |statistic|
-              statistic.command.name == "load"
+              command.name == "load"
             end
           when "table_list", "column_list"
             # ignore

data/lib/groonga-query-log/command/format-regression-test-logs.rb

@@ -28,7 +28,8 @@ require "groonga-query-log/version"
 module GroongaQueryLog
     module Command
       class FormatRegressionTestLogs
-        def initialize
+        def initialize(options={})
+          @output = options[:output] || $stdout
         end
 
         def run(command_line)
@@ -68,8 +69,8 @@ module GroongaQueryLog
 
           input.each_line do |line|
             unless line.valid_encoding?
-              puts("invalid encoding line")
-              puts("#{path}:#{input.lineno}:#{line}")
+              @output.puts("invalid encoding line")
+              @output.puts("#{path}:#{input.lineno}:#{line}")
               next
             end
             case line
@@ -97,18 +98,18 @@ module GroongaQueryLog
           begin
             JSON.parse(response_old)
           rescue JSON::ParserError
-            puts(command)
-            puts("failed to parse old response: #{$!.message}")
-            puts(response_old)
+            @output.puts(command)
+            @output.puts("failed to parse old response: #{$!.message}")
+            @output.puts(response_old)
             valid = false
           end
 
           begin
             JSON.parse(response_new)
           rescue JSON::ParserError
-            puts(command)
-            puts("failed to parse new response: #{$!.message}")
-            puts(response_new)
+            @output.puts(command)
+            @output.puts("failed to parse new response: #{$!.message}")
+            @output.puts(response_new)
             valid = false
           end
 
@@ -125,33 +126,39 @@ module GroongaQueryLog
               PP.pp(JSON.parse(response_new), response_new_file)
               response_new_file.flush
               report_command(command)
-              system("diff",
-                     "--label=old",
-                     "--label=new",
-                     "-u",
-                     response_old_file.path, response_new_file.path)
+              Tempfile.open("response-diff") do |response_diff_file|
+                system("diff",
+                       "--label=old",
+                       "--label=new",
+                       "-u",
+                       response_old_file.path,
+                       response_new_file.path,
+                       out: response_diff_file)
+                response_diff_file.rewind
+                @output.puts(response_diff_file.read)
+              end
             end
           end
         end
 
         def report_error(command, message, backtrace)
           report_command(command)
-          puts("Error: #{message}")
-          puts("Backtrace:")
-          puts(backtrace)
+          @output.puts("Error: #{message}")
+          @output.puts("Backtrace:")
+          @output.puts(backtrace)
         end
 
         def report_command(command)
-          puts("Command:")
-          puts(command)
+          @output.puts("Command:")
+          @output.puts(command)
           parsed_command = Groonga::Command::Parser.parse(command)
-          puts("Name: #{parsed_command.name}")
-          puts("Arguments:")
+          @output.puts("Name: #{parsed_command.name}")
+          @output.puts("Arguments:")
           sorted_arguments = parsed_command.arguments.sort_by do |key, value|
             key
           end
           sorted_arguments.each do |key, value|
-            puts("  #{key}: #{value}")
+            @output.puts("  #{key}: #{value}")
           end
         end
       end

data/lib/groonga-query-log/command/run-regression-test.rb

@@ -50,6 +50,7 @@ module GroongaQueryLog
         @vector_accessors = []
         @rewrite_nullable_reference_number = false
         @nullable_reference_number_accessors = []
+        @rewrite_not_or_regular_expression = false
 
         @care_order = true
         @ignored_drilldown_keys = []
@@ -187,6 +188,14 @@ module GroongaQueryLog
                   "specifying this option multiple times") do |accessor|
           @nullable_reference_number_accessors << accessor
         end
+        parser.on("--[no-]rewrite-not-or-regular-expression",
+                  "Rewrite 'column1 @ \"keyword1\" && column2 @~ " +
+                  "\"^(?!.*keyword2|keyword3|...).+$\"' " +
+                  "with 'column1 @ \"keyword1\" &! column2 @ \"keyword2\" " +
+                  "&! column2 @ \"keyword3\" &! ...'",
+                  "(#{@rewrite_not_or_regular_expression})") do |boolean|
+          @rewrite_not_or_regular_expression = boolean
+        end
 
         parser.separator("")
         parser.separator("Comparisons:")
@@ -252,6 +261,8 @@ module GroongaQueryLog
             @rewrite_nullable_reference_number,
           :nullable_reference_number_accessors =>
             @nullable_reference_number_accessors,
+          :rewrite_not_or_regular_expression =>
+            @rewrite_not_or_regular_expression,
           :target_command_names => @target_command_names,
           :read_timeout => @read_timeout,
         }
@@ -547,6 +558,9 @@ module GroongaQueryLog
             command_line << "--nullable-reference-number-accessor"
             command_line << accessor
           end
+          if @options[:rewrite_not_or_regular_expression]
+            command_line << "--rewrite-not-or-regular-expression"
+          end
           if @options[:target_command_names]
             command_line << "--target-command-names"
             command_line << @options[:target_command_names].join(",")

data/lib/groonga-query-log/command/verify-server.rb

@@ -230,6 +230,15 @@ module GroongaQueryLog
           @options.rewrite_nullable_reference_number = boolean
         end
 
+        parser.on("--[no-]rewrite-not-or-regular-expression",
+                  "Rewrite 'column1 @ \"keyword1\" && column2 @~ " +
+                  "\"^(?!.*keyword2|keyword3|...).+$\"' " +
+                  "with 'column1 @ \"keyword1\" &! column2 @ \"keyword2\" " +
+                  "&! column2 @ \"keyword3\" &! ...'",
+                  "(#{@options.rewrite_not_or_regular_expression?})") do |boolean|
+          @options.rewrite_not_or_regular_expression = boolean
+        end
+
         parser.on("--nullable-reference-number-accessor=ACCESSOR",
                   "Mark ACCESSOR as rewrite nullable reference number targets",
                   "You can specify multiple accessors by",

data/lib/groonga-query-log/filter-rewriter.rb

@@ -35,6 +35,9 @@ module GroongaQueryLog
       if @options[:rewrite_nullable_reference_number]
         rewritten = rewrite_nullable_reference_number(rewritten)
       end
+      if @options[:rewrite_not_or_regular_expression]
+        rewritten = rewrite_not_or_regular_expression(rewritten)
+      end
       rewritten
     end
 
@@ -77,5 +80,25 @@ module GroongaQueryLog
         end
       end
     end
+
+    def rewrite_not_or_regular_expression(filter)
+      filter.gsub(/&& *(?<target_column>[a-zA-Z0-9_.]+) *@~ *"(?<pattern>.*?)"/) do |matched|
+        target_column = $LAST_MATCH_INFO[:target_column]
+        pattern = $LAST_MATCH_INFO[:pattern]
+
+        case pattern
+        when /\A(?<header>(?:\^|\\A)\(\?\!\.\*)
+                (?<body>.+)
+                (?<footer>\)\.[+*](?:\$|\\z))\z/x
+          body = $LAST_MATCH_INFO[:body]
+          conditions = body.split("|").collect do |word|
+            "&! #{target_column} @ \"#{word.strip}\""
+          end
+          conditions.join(" ")
+        else
+          matched
+        end
+      end
+    end
   end
 end

data/lib/groonga-query-log/server-verifier.rb

@@ -240,6 +240,7 @@ module GroongaQueryLog
       attr_accessor :vector_accessors
       attr_writer :rewrite_nullable_reference_number
       attr_accessor :nullable_reference_number_accessors
+      attr_writer :rewrite_not_or_regular_expression
       def initialize
         @groonga1 = GroongaOptions.new
         @groonga2 = GroongaOptions.new
@@ -268,6 +269,7 @@ module GroongaQueryLog
         @vector_accessors = []
         @rewrite_nullable_reference_number = false
         @nullable_reference_number_accessors = []
+        @rewrite_not_or_regular_expression = false
       end
 
       def request_queue_size
@@ -298,6 +300,10 @@ module GroongaQueryLog
         @rewrite_nullable_reference_number
       end
 
+      def rewrite_not_or_regular_expression?
+        @rewrite_not_or_regular_expression
+      end
+
       def target_command_name?(name)
         return false if name.nil?
 
@@ -326,7 +332,8 @@ module GroongaQueryLog
       def need_filter_rewrite?
         rewrite_vector_equal? or
           rewrite_vector_not_equal_empty_string? or
-          rewrite_nullable_reference_number?
+          rewrite_nullable_reference_number? or
+          rewrite_not_or_regular_expression
       end
 
       def to_filter_rewriter_options
@@ -339,6 +346,8 @@ module GroongaQueryLog
             rewrite_nullable_reference_number?,
           :nullable_reference_number_accessors =>
             nullable_reference_number_accessors,
+          :rewrite_not_or_regular_expression =>
+            rewrite_not_or_regular_expression?,
         }
       end
     end

data/lib/groonga-query-log/statistic.rb

@@ -121,14 +121,17 @@ module GroongaQueryLog
         "return_code" => return_code,
         "slow" => slow?,
       }
-      arguments = command.arguments.collect do |key, value|
-        {"key" => key, "value" => value}
+      if command
+        data["command"] = {
+          "raw" => raw_command,
+          "name" => command.name,
+          "parameters" => command_arguments,
+        }
+      else
+        data["command"] = {
+          "raw" => raw_command
+        }
       end
-      data["command"] = {
-        "raw" => raw_command,
-        "name" => command.name,
-        "parameters" => arguments,
-      }
       operations = []
       each_operation do |operation|
         operation_data = {}
@@ -192,5 +195,11 @@ module GroongaQueryLog
     def slow_operation?(elapsed)
       elapsed >= @slow_operation_threshold
     end
+
+    def command_arguments
+      command.arguments.collect do |key, value|
+        {"key" => key, "value" => value}
+      end
+    end
   end
 end

data/lib/groonga-query-log/version.rb

@@ -15,5 +15,5 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module GroongaQueryLog
-  VERSION = "1.3.9"
+  VERSION = "1.4.0"
 end

data/test/command/test-format-regression-test-logs.rb

@@ -1,3 +1,4 @@
+# coding: utf-8
 # Copyright (C) 2014-2018  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
@@ -29,13 +30,13 @@ class FormatRegressionTestLogsCommandTest < Test::Unit::TestCase
     success = false
     begin
       $stdout.reopen(output)
-      succees = @command.run(command_line)
+      success = @command.run(command_line)
     ensure
       $stdout.reopen(stdout)
     end
     output.close
     output.open
-    [succees, output.read]
+    [success, output.read]
   end
 
   def fixture_path(*components)
@@ -59,10 +60,10 @@ Arguments:
 +++ new
 @@ -1,4 +1,4 @@
  [[[2],
-   [[\"_id\", \"UInt32\"], [\"message\", \"Text\"]],
-   [1, \"log message1\"],
--  [2, \"log message2\"]]]
-+  [3, \"log message3\"]]]
+   [["_id", "UInt32"], ["message", "Text"]],
+   [1, "log message1"],
+-  [2, "log message2"]]]
++  [3, "log message3"]]]
     OUTPUT
     assert_equal([true, output],
                  run_command([fixture_path("command-format.log")]))
@@ -81,10 +82,10 @@ Arguments:
 +++ new
 @@ -1,4 +1,4 @@
  [[[2],
-   [[\"_id\", \"UInt32\"], [\"message\", \"Text\"]],
-   [1, \"log message1: 焼肉\"],
--  [2, \"log message2: 焼肉\"]]]
-+  [3, \"log message3: 焼肉\"]]]
+   [["_id", "UInt32"], ["message", "Text"]],
+   [1, "log message1: 焼肉"],
+-  [2, "log message2: 焼肉"]]]
++  [3, "log message3: 焼肉"]]]
     OUTPUT
     assert_equal([true, output],
                  run_command([fixture_path("url-format.log")]))
@@ -108,4 +109,32 @@ Backtrace:
     assert_equal([true, output],
                  run_command([fixture_path("error.log")]))
   end
+
+  sub_test_case(".new") do
+    def setup
+    end
+
+    def test_output
+      output = StringIO.new
+      options = {:output => output}
+      command = GroongaQueryLog::Command::FormatRegressionTestLogs.new(options)
+      command.run([fixture_path("command-format.log")])
+      expected = <<-OUTPUT
+Command:
+select Logs
+Name: select
+Arguments:
+  table: Logs
+--- old
++++ new
+@@ -1,4 +1,4 @@
+ [[[2],
+   [["_id", "UInt32"], ["message", "Text"]],
+   [1, "log message1"],
+-  [2, "log message2"]]]
++  [3, "log message3"]]]
+      OUTPUT
+      assert_equal(expected, output.string)
+    end
+  end
 end

data/test/test-filter-rewriter.rb

@@ -110,4 +110,31 @@ class FilterRewriterTest < Test::Unit::TestCase
                            ["ref_column.number"]))
     end
   end
+
+  class RegularExpressionTest < self
+    def rewrite(filter, enabled = true)
+      super(filter,
+            :rewrite_not_or_regular_expression => enabled)
+    end
+
+    def test_rewrite_one
+      assert_equal("column1 @ \"value1\" &! column2 @ \"value2\"",
+                   rewrite("column1 @ \"value1\" && column2 @~ \"^(?!.*value2).+$\""))
+    end
+
+    def test_reference
+      assert_equal("column1 @ \"value1\" &! reference.column2 @ \"value2\"",
+                   rewrite("column1 @ \"value1\" && reference.column2 @~ \"^(?!.*value2).+$\""))
+    end
+
+    def test_under_score
+      assert_equal("column1 @ \"value1\" &! column_2 @ \"value_2\"",
+                   rewrite("column1 @ \"value1\" && column_2 @~ \"^(?!.*value_2).+$\""))
+    end
+
+    def test_rewrite_multiple
+      assert_equal("column1 @ \"value1\" &! column2 @ \"value2\" &! column2 @ \"value3\" &! column2 @ \"value4\"",
+                   rewrite("column1 @ \"value1\" && column2 @~ \"^(?!.*value2 | value3 | value4).+$\""))
+    end
+  end
 end

data/test/test-parser.rb

@@ -43,6 +43,17 @@ class ParserTest < Test::Unit::TestCase
     assert_equal([nil], statistics.collect(&:command))
   end
 
+  def test_no_command_to_hash
+    statistics = parse(<<-LOG)
+2012-12-13 11:15:20.628105|0x7fff148c8a50|>/
+2012-12-13 11:15:21.645119|0x7fff148c8a50|<000000017041150 rc=0
+    LOG
+    expected = {
+      "raw" => "/"
+    }
+    assert_equal(expected, statistics[0].to_hash["command"])
+  end
+
   private
   def parse(log)
     statistics = []

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: groonga-query-log
 version: !ruby/object:Gem::Version
-  version: 1.3.9
+  version: 1.4.0
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-20 00:00:00.000000000 Z
+date: 2019-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: groonga-command-parser
@@ -154,17 +154,17 @@ description: ''
 email:
 - kou@clear-code.com
 executables:
-- groonga-query-log-extract
+- groonga-query-log-format-regression-test-logs
 - groonga-query-log-run-regression-test
-- groonga-query-log-analyze-load
 - groonga-query-log-check-crash
-- groonga-query-log-analyze
-- groonga-query-log-verify-server
+- groonga-query-log-replay
 - groonga-query-log-check-command-version-compatibility
-- groonga-query-log-detect-memory-leak
-- groonga-query-log-format-regression-test-logs
+- groonga-query-log-verify-server
 - groonga-query-log-show-running-queries
-- groonga-query-log-replay
+- groonga-query-log-analyze
+- groonga-query-log-extract
+- groonga-query-log-detect-memory-leak
+- groonga-query-log-analyze-load
 extensions: []
 extra_rdoc_files: []
 files:
@@ -280,8 +280,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2.1
+rubygems_version: 3.0.2
 signing_key: 
 specification_version: 4
 summary: Groonga-query-log is a collection of library and tools to process [Groonga](http://groonga.org/)'s
@@ -289,35 +288,35 @@ summary: Groonga-query-log is a collection of library and tools to process [Groo
   as a library. You can analyze your Groonga's queries and test with your Groonga's
   query log by using groonga-query-log as a tool.
 test_files:
-- test/test-response-comparer.rb
-- test/run-test.rb
-- test/test-replayer.rb
-- test/test-parser.rb
-- test/helper.rb
-- test/fixtures/query.log
-- test/fixtures/n_entries.expected
-- test/fixtures/other-query.log
-- test/fixtures/no-report-summary.expected
+- test/fixtures/reporter/console.expected
+- test/fixtures/reporter/html.expected
 - test/fixtures/reporter/json-stream.expected
 - test/fixtures/reporter/json.expected
-- test/fixtures/reporter/html.expected
-- test/fixtures/reporter/console.expected
-- test/fixtures/regression-test-logs/command-format.log
-- test/fixtures/regression-test-logs/url-format.log
-- test/fixtures/regression-test-logs/error.log
-- test/fixtures/target-commands.expected
-- test/fixtures/multi.expected
-- test/fixtures/run-regression-test/query-logs/query.log
-- test/fixtures/run-regression-test/schema/schema.grn
-- test/fixtures/run-regression-test/indexes/indexes.grn
-- test/fixtures/run-regression-test/data/data.grn
-- test/fixtures/order/-elapsed.expected
 - test/fixtures/order/start-time.expected
 - test/fixtures/order/-start-time.expected
+- test/fixtures/order/-elapsed.expected
 - test/fixtures/order/elapsed.expected
+- test/fixtures/no-report-summary.expected
+- test/fixtures/target-commands.expected
+- test/fixtures/run-regression-test/data/data.grn
+- test/fixtures/run-regression-test/query-logs/query.log
+- test/fixtures/run-regression-test/indexes/indexes.grn
+- test/fixtures/run-regression-test/schema/schema.grn
+- test/fixtures/other-query.log
+- test/fixtures/query.log
 - test/fixtures/target-tables.expected
-- test/test-filter-rewriter.rb
+- test/fixtures/n_entries.expected
+- test/fixtures/multi.expected
+- test/fixtures/regression-test-logs/command-format.log
+- test/fixtures/regression-test-logs/url-format.log
+- test/fixtures/regression-test-logs/error.log
 - test/command/test-format-regression-test-logs.rb
-- test/command/test-extract.rb
 - test/command/test-analyzer.rb
+- test/command/test-extract.rb
+- test/test-response-comparer.rb
+- test/test-replayer.rb
 - test/test-incompatibility-detector.rb
+- test/test-filter-rewriter.rb
+- test/test-parser.rb
+- test/helper.rb
+- test/run-test.rb