groonga-query-log 1.7.1 → 1.7.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc465fa277253d4a2f6f4b42ddc9553393572f6d6ad03496b17d308036f90f51
-  data.tar.gz: 1d458256bf29ef9d84100e66a535563d85a240f823e5dfd524bb45122bc1fa55
+  metadata.gz: 5c1dd6a2bc5f431a4003c5468aa573a55563107a7eb875ef6af59b2150bb0ab9
+  data.tar.gz: 503abaf07344e2453e74bf34fd053560ce2bb3acce212527ea0559520167e81b
 SHA512:
-  metadata.gz: 1954617ef0c708f19af1a204bfd26a71cf7799204237f7be72a00c42c0aafacfc177f9785ed2f456108cad47dd3d2756cd9608eddb80780fa67a6e142e85e673
-  data.tar.gz: 31df2208d65ffde3cb0b25dbcac5290f080f85310d747e3e1cd89e4a11f78a50dab071ebee27048e4c6d6bf3c59da7281180657c8ecd0e2adf2495f3ac348fba
+  metadata.gz: ff08e69d1c1197d42fc763a685e2780e24c4e6432c3b04035b5076b1cd0fe126411fc36a8965da55abbde8cf94ac59a5032bc0028f52ceea6143abb5084c9f30
+  data.tar.gz: f4d689ada13d114ffce2f7d4d6d50ca6d38dbce1974a6df7fba5e9d677234d451de0da939aef0720f5876715d6d3848741cb8abbfeb12c005f6b0fa0983eb3b7

data/doc/text/news.md

@@ -1,5 +1,49 @@
 # News
 
+## 1.7.6: 2021-02-24
+
+### Improvements
+
+  * `replayer`: Ignored `load` without `values`.
+
+  * `statistics`: Added `n_records` to each operation.
+
+## 1.7.5: 2020-11-18
+
+### Improvements
+
+  * `analyze`: Ignored static file access.
+
+## 1.7.4: 2020-11-12
+
+### Fixes
+
+  * `server-verifier`: Fixed a bug that `--max-limit` doesn't work.
+
+  * `run-regression-test`: Fixed a bug that `--max-limit` doesn't work.
+
+## 1.7.3: 2020-11-12
+
+### Improvements
+
+  * `check-crash`: Added support for showing Groonga version.
+
+  * `server-verifier`: Added `--max-limit` option.
+
+  * `run-regression-test`: Added `--max-limit` option.
+
+### Fixes
+
+  * `GroongaQueryLog::Parser`: Fixed a bug that target command filter
+    may not work.
+
+## 1.7.2: 2020-06-02
+
+### Fixes
+
+  * `server-verifier`: Fixed a bug that performance verification
+    doesn't try 5 times.
+
 ## 1.7.1: 2020-06-01
 
 ### Fixes

data/lib/groonga-query-log/command/check-crash.rb

@@ -71,6 +71,7 @@ module GroongaQueryLog
       end
 
       class GroongaProcess
+        attr_reader :version
         attr_reader :pid
         attr_reader :start_time
         attr_reader :start_log_path
@@ -80,7 +81,8 @@ module GroongaQueryLog
         attr_writer :crashed
         attr_writer :finished
         attr_reader :important_entries
-        def initialize(pid, start_time, start_log_path)
+        def initialize(version, pid, start_time, start_log_path)
+          @version = version
           @pid = pid
           @start_time = start_time
           @end_time = @start_time
@@ -121,6 +123,7 @@ module GroongaQueryLog
               need_query_log_parsing = false
               p [:process,
                  :success,
+                 process.version,
                  process.start_time.iso8601,
                  process.end_time.iso8601,
                  process.pid,
@@ -129,6 +132,7 @@ module GroongaQueryLog
             elsif process.crashed?
               p [:process,
                  :crashed,
+                 process.version,
                  process.start_time.iso8601,
                  process.end_time.iso8601,
                  process.pid,
@@ -137,6 +141,7 @@ module GroongaQueryLog
             else
               p [:process,
                  :unfinished,
+                 process.version,
                  process.start_time.iso8601,
                  process.pid,
                  process.start_log_path]
@@ -144,6 +149,7 @@ module GroongaQueryLog
 
             unless process.n_leaks.zero?
               p [:leak,
+                 process.version,
                  process.n_leaks,
                  process.end_time.iso8601,
                  process.pid,
@@ -184,7 +190,8 @@ module GroongaQueryLog
               end
             end
             unless @unflushed_statistics.empty?
-              puts("Unflushed commands in #{start.iso8601}/#{last.iso8601}")
+              puts("Unflushed commands in " +
+                   "#{start_time.iso8601}/#{end_time.iso8601}")
               @unflushed_statistics.each do |statistic|
                 puts("#{statistic.start_time.iso8601}: #{statistic.raw_command}")
               end
@@ -320,7 +327,8 @@ module GroongaQueryLog
           end
 
           case entry.message
-          when /\Agrn_init:/, /\Amroonga \d+\.\d+ started\.\z/
+          when /\Agrn_init: <(.+?)>/, /\Amroonga (\d+\.\d+) started\.\z/
+            version = $1
             process = @running_processes[entry.pid]
             if process
               process.finished = true
@@ -328,12 +336,15 @@ module GroongaQueryLog
               yield(process)
               @running_processes.delete(entry.pid)
             end
-            process = GroongaProcess.new(entry.pid, entry.timestamp, path)
+            process = GroongaProcess.new(version,
+                                         entry.pid,
+                                         entry.timestamp,
+                                         path)
             @running_processes[entry.pid] = process
           when /\Agrn_fin \((\d+)\)\z/
             n_leaks = $1.to_i
             @running_processes[entry.pid] ||=
-              GroongaProcess.new(entry.pid, Time.at(0), path)
+              GroongaProcess.new(nil, entry.pid, Time.at(0), path)
             process = @running_processes[entry.pid]
             process.n_leaks = n_leaks
             process.end_time = entry.timestamp
@@ -343,7 +354,7 @@ module GroongaQueryLog
             @running_processes.delete(entry.pid)
           else
             @running_processes[entry.pid] ||=
-              GroongaProcess.new(entry.pid, Time.at(0), path)
+              GroongaProcess.new(nil, entry.pid, Time.at(0), path)
             process = @running_processes[entry.pid]
             case entry.log_level
             when :notice

data/lib/groonga-query-log/command/run-regression-test.rb

@@ -66,6 +66,7 @@ module GroongaQueryLog
         @rewrite_and_not_operator = false
         @debug_rewrite = false
         @omit_rate = 0.0
+        @max_limit = -1
 
         @care_order = true
         @ignored_drilldown_keys = []
@@ -313,6 +314,12 @@ module GroongaQueryLog
                   "(#{@omit_rate})") do |rate|
           @omit_rate = rate
         end
+        parser.on("--max-limit=LIMIT", Integer,
+                  "Use LIMIT as the max limit value",
+                  "Negative value doesn't rewrite the limit parameter",
+                  "(#{@max_limit})") do |limit|
+          @max_limit = limit
+        end
 
         parser.separator("")
         parser.separator("Comparisons:")
@@ -466,6 +473,7 @@ module GroongaQueryLog
             @rewrite_and_not_operator,
           :debug_rewrite => @debug_rewrite,
           :omit_rate => @omit_rate,
+          :max_limit => @max_limit,
           :target_command_names => @target_command_names,
           :verify_performance => @verify_performance,
           :performance_verfifier_options => @performance_verfifier_options,
@@ -919,6 +927,10 @@ module GroongaQueryLog
             command_line << "--omit-rate"
             command_line << @options[:omit_rate].to_s
           end
+          if @options[:max_limit] >= 0
+            command_line << "--max-limit"
+            command_line << @options[:max_limit].to_s
+          end
           if @options[:target_command_names]
             command_line << "--target-command-names"
             command_line << @options[:target_command_names].join(",")

data/lib/groonga-query-log/command/verify-server.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2018  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2020  Sutou Kouhei <kou@clear-code.com>
 # Copyright (C) 2020  Horimoto Yasuhiro <horimoto@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
@@ -280,6 +280,13 @@ module GroongaQueryLog
           @options.nullable_reference_number_accessors << accessor
         end
 
+        parser.on("--max-limit=LIMIT", Integer,
+                  "Use LIMIT as the max limit value",
+                  "Negative value doesn't rewrite the limit parameter",
+                  "(#{@options.max_limit})") do |limit|
+          @options.max_limit = limit
+        end
+
         create_parser_performance(parser)
 
         parser.separator("")

data/lib/groonga-query-log/parser.rb

@@ -160,7 +160,7 @@ module GroongaQueryLog
 
     def target_statistic?(statistic)
       if @target_commands
-        unless @target_commands.include?(statistic.command.name)
+        unless @target_commands.include?(statistic.command.command_name)
           return false
         end
       end

data/lib/groonga-query-log/replayer.rb

@@ -47,6 +47,7 @@ module GroongaQueryLog
           parser.parse(input) do |statistic|
             next if statistic.command.nil?
             next unless target_command?(statistic.command)
+            next unless available_command?(statistic.command)
             # TODO: validate orignal_source is one line
             output.puts(statistic.command.original_source)
             output.flush
@@ -83,19 +84,15 @@ module GroongaQueryLog
           end
           begin
             replay_command(client, id, statistic.command)
-          rescue Groonga::Client::Error
+          rescue => error
             # TODO: add error log mechanism
             $stderr.puts(Time.now.iso8601(6))
             $stderr.puts(statistic.command.original_source)
-            $stderr.puts($!.raw_error.message)
-            $stderr.puts($!.raw_error.backtrace)
-            return false
-          rescue
-            # TODO: add error log mechanism
-            $stderr.puts(Time.now.iso8601(6))
-            $stderr.puts(statistic.command.original_source)
-            $stderr.puts($!.message)
-            $stderr.puts($!.backtrace)
+            if error.respond_to?(:raw_error)
+              error = raw_error
+            end
+            $stderr.puts(error.message)
+            $stderr.puts(error.backtrace)
             return false
           end
         end
@@ -136,6 +133,15 @@ module GroongaQueryLog
       @options.target_command_name?(command.command_name)
     end
 
+    def available_command?(command)
+      case command.command_name
+      when "load"
+        not command.values.nil?
+      else
+        true
+      end
+    end
+
     class NullOutput
       class << self
         def open

data/lib/groonga-query-log/server-verifier.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2018  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2020  Sutou Kouhei <kou@clear-code.com>
 # Copyright (C) 2020  Horimoto Yasuhiro <horimoto@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
@@ -170,6 +170,14 @@ module GroongaQueryLog
     def verify_command(groonga1_client, groonga2_client, command)
       command["cache"] = "no" if @options.disable_cache?
       command["cache"] = "no" if @options.verify_performance?
+      if @options.max_limit >= 0 and command["limit"]
+        limit = command["limit"].to_i
+        if limit >= 0
+          command["limit"] = [limit, @options.max_limit].min.to_s
+        else
+          command["limit"] = @options.max_limit.to_s
+        end
+      end
       command["output_type"] = "json"
       rewrite_filter(command, "filter")
       rewrite_filter(command, "scorer")
@@ -219,17 +227,16 @@ module GroongaQueryLog
                                            responses1,
                                            responses2,
                                            @options.performance_verifier_options)
-        if verifier.slow?
-          @slow = true
-          @events.push([:slow,
-                        command,
-                        verifier.old_elapsed_time,
-                        verifier.new_elapsed_time,
-                        verifier.old_elapsed_times,
-                        verifier.new_elapsed_times])
-          return
-        end
+        return unless verifier.slow?
       end
+
+      @slow = true
+      @events.push([:slow,
+                    command,
+                    verifier.old_elapsed_time,
+                    verifier.new_elapsed_time,
+                    verifier.old_elapsed_times,
+                    verifier.new_elapsed_times])
     end
 
     def rewrite_filter(command, name)
@@ -318,6 +325,7 @@ module GroongaQueryLog
       attr_reader :performance_verifier_options
       attr_writer :debug_rewrite
       attr_writer :omit_rate
+      attr_accessor :max_limit
       def initialize
         @groonga1 = GroongaOptions.new
         @groonga2 = GroongaOptions.new
@@ -352,6 +360,7 @@ module GroongaQueryLog
         @performance_verifier_options = PerformanceVerifier::Options.new
         @debug_rewrite = false
         @omit_rate = 0.0
+        @max_limit = -1
       end
 
       def request_queue_size

data/lib/groonga-query-log/statistic.rb

@@ -115,6 +115,7 @@ module GroongaQueryLog
     end
 
     def select_family_command?
+      return false if command.nil?
       case command.command_name
       when "select", "range_filter"
         true
@@ -151,6 +152,7 @@ module GroongaQueryLog
         operation_data["relative_elapsed"] = operation[:relative_elapsed_in_seconds]
         operation_data["context"] = operation[:context]
         operation_data["slow"] = operation[:slow?]
+        operation_data["n_records"] = operation[:n_records]
         operations << operation_data
       end
       data["operations"] = operations

data/lib/groonga-query-log/version.rb

@@ -15,5 +15,5 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module GroongaQueryLog
-  VERSION = "1.7.1"
+  VERSION = "1.7.6"
 end

data/test/command/test-analyzer.rb

@@ -1,5 +1,5 @@
 # Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
-# Copyright (C) 2014-2019  Sutou Kouhei <kou@clear-code.com>
+# Copyright (C) 2014-2021  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -42,6 +42,7 @@ class AnalyzerCommandTest < Test::Unit::TestCase
     end
 
     def test_no_specified
+      require_tty
       assert_equal("Error: Please specify input log files.\n",
                    run_analyzer)
     end

data/test/command/test-extract.rb

@@ -1,5 +1,5 @@
 # Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
-# Copyright (C) 2015-2018  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2015-2021  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -45,6 +45,7 @@ column_create --flags "COLUMN_SCALAR" --name "title" --table "Comments" --type "
     end
 
     def test_no_specified
+      require_tty
       assert_equal("Error: Please specify input log files.\n",
                    run_extractor)
     end

data/test/fixtures/reporter/json-stream.expected

@@ -1,2 +1,2 @@
 {"start_time":START_TIME,"end_time":END_TIME,"elapsed":0.003128856,"return_code":0,"slow":false,"command":{"raw":"load --table Video","name":"load","parameters":[{"key":"table","value":"Video"}]},"operations":[]}
-{"start_time":START_TIME,"end_time":END_TIME,"elapsed":0.00121714,"return_code":0,"slow":false,"command":{"raw":"select --table Users --query follower:@groonga --output_columns _key,name","name":"select","parameters":[{"key":"table","value":"Users"},{"key":"query","value":"follower:@groonga"},{"key":"output_columns","value":"_key,name"}]},"operations":[{"name":"filter","relative_elapsed":0.0008429529999999999,"context":"Users.follower match \"groonga\"","slow":false},{"name":"select","relative_elapsed":2.7947e-05,"context":null,"slow":false},{"name":"output","relative_elapsed":0.000195852,"context":"_key,name","slow":false}]}
+{"start_time":START_TIME,"end_time":END_TIME,"elapsed":0.00121714,"return_code":0,"slow":false,"command":{"raw":"select --table Users --query follower:@groonga --output_columns _key,name","name":"select","parameters":[{"key":"table","value":"Users"},{"key":"query","value":"follower:@groonga"},{"key":"output_columns","value":"_key,name"}]},"operations":[{"name":"filter","relative_elapsed":0.0008429529999999999,"context":"Users.follower match \"groonga\"","slow":false,"n_records":2},{"name":"select","relative_elapsed":2.7947e-05,"context":null,"slow":false,"n_records":2},{"name":"output","relative_elapsed":0.000195852,"context":"_key,name","slow":false,"n_records":2}]}

data/test/fixtures/reporter/json.expected

@@ -1,4 +1,4 @@
 [
 {"start_time":START_TIME,"end_time":END_TIME,"elapsed":0.003128856,"return_code":0,"slow":false,"command":{"raw":"load --table Video","name":"load","parameters":[{"key":"table","value":"Video"}]},"operations":[]},
-{"start_time":START_TIME,"end_time":END_TIME,"elapsed":0.00121714,"return_code":0,"slow":false,"command":{"raw":"select --table Users --query follower:@groonga --output_columns _key,name","name":"select","parameters":[{"key":"table","value":"Users"},{"key":"query","value":"follower:@groonga"},{"key":"output_columns","value":"_key,name"}]},"operations":[{"name":"filter","relative_elapsed":0.0008429529999999999,"context":"Users.follower match \"groonga\"","slow":false},{"name":"select","relative_elapsed":2.7947e-05,"context":null,"slow":false},{"name":"output","relative_elapsed":0.000195852,"context":"_key,name","slow":false}]}
+{"start_time":START_TIME,"end_time":END_TIME,"elapsed":0.00121714,"return_code":0,"slow":false,"command":{"raw":"select --table Users --query follower:@groonga --output_columns _key,name","name":"select","parameters":[{"key":"table","value":"Users"},{"key":"query","value":"follower:@groonga"},{"key":"output_columns","value":"_key,name"}]},"operations":[{"name":"filter","relative_elapsed":0.0008429529999999999,"context":"Users.follower match \"groonga\"","slow":false,"n_records":2},{"name":"select","relative_elapsed":2.7947e-05,"context":null,"slow":false,"n_records":2},{"name":"output","relative_elapsed":0.000195852,"context":"_key,name","slow":false,"n_records":2}]}
 ]

data/test/helper.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2021  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -29,6 +29,10 @@ module Helper
   end
 
   module Command
+    def require_tty
+      omit("Require tty") unless $stdin.tty?
+    end
+
     def open_error_output
       Tempfile.open("groonga-query-log.error") do |error|
         error.sync = true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: groonga-query-log
 version: !ruby/object:Gem::Version
-  version: 1.7.1
+  version: 1.7.6
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-31 00:00:00.000000000 Z
+date: 2021-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: charty
@@ -315,7 +315,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.0.pre1
+rubygems_version: 3.3.0.dev
 signing_key: 
 specification_version: 4
 summary: Groonga-query-log is a collection of library and tools to process [Groonga](http://groonga.org/)'s