groonga-query-log 1.5.8 → 1.5.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d28dfe11ea122d12976b21d879671c2dc2246c87bff662a41212c3247865a4c1
-  data.tar.gz: dd32fada8401876fa9ca27942c0e0f8145bcbf92162c03483aea166e66351cd8
+  metadata.gz: 99da1139f30806bb7ead68b469c2122368bcad8ea91586160096751e3919b4ab
+  data.tar.gz: ab791f07d33e784531372dd3c4cad59e57e1ae38d778529a3b65dbd8ea1d4466
 SHA512:
-  metadata.gz: 9bdc0879550bce0f87030702979315ea127c70bcfcf8fa8817f84e92924d2577c8e4263f87209d756abc804a0b4d2b2edf09b75a31253854edf0e30f00c3237e
-  data.tar.gz: c3730562b6415c493631598af4e73cb1e1da6ad3aabd6ad6c19ed87200a11dd74cf14400e4c624ad99720d179b0ddc8f696e9454a63b4d15fb8edbd0987f1fdc
+  metadata.gz: 9ae76dd7338c155860f1b74dea49fc202baeaa4216018dda0e2c22144663c4ea93db08a4042f3487c429bd565281876d46d9348a4fa27e510c082f9dcb065507
+  data.tar.gz: 6d4f6912e00abd0a0ff41c18bcbbd3c251dab17a9d142974ab0719e5dbb5309e97f7fe174ec7cd239cddd0cf1b15c8940b7da2d31e4f8c7c723337d0902940a7

data/doc/text/news.md

@@ -1,5 +1,28 @@
 # News
 
+## 1.5.9: 2020-04-06
+
+### Fixes
+
+  * `replay`: Added `--output-type` option.
+
+  * `replay`: Added support for outputting logs to stdout.
+
+  * `replay`: Fixed a bug that command name isn't processed correctly.
+
+  * `replay`: Added support for inputting from stdin.
+
+  * `run-regression-test`: Added `--old-groonga-env` option.
+
+  * `run-regression-test`: Added `--new-groonga-env` option.
+
+  * `run-regression-test`: Added support for checking the number of
+    leaked objects.
+
+  * `replay`: Added `--output-error-responses` option.
+
+  * `run-regression-test`: Added `--omit-rate` option.
+
 ## 1.5.8: 2020-02-14
 
 ### Fixes

data/lib/groonga-query-log/command/replay.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2020  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -29,9 +29,13 @@ module GroongaQueryLog
       def run(command_line)
         input_paths = create_parser.parse(command_line)
         replayer = Replayer.new(@options)
-        input_paths.each do |input_path|
-          File.open(input_path) do |input|
-            replayer.replay(input)
+        if input_paths.empty?
+          replayer.replay($stdin)
+        else
+          input_paths.each do |input_path|
+            File.open(input_path) do |input|
+              replayer.replay(input)
+            end
           end
         end
         true
@@ -117,6 +121,21 @@ module GroongaQueryLog
                   "[not output]") do |path|
           @options.responses_path = path
         end
+
+        parser.on("--output-error-responses=PATH",
+                  "Output only error responses to PATH",
+                  "[not output]") do |path|
+          @options.error_responses_path = path
+        end
+
+        available_output_types = ["json", "msgpack", "apache-arrow"]
+        available_output_type_labels = available_output_types.join(", ")
+        parser.on("--output-type=TYPE", available_output_types,
+                  "Use TYPE as the output type",
+                  "(#{available_output_type_labels})",
+                  "[use the original output type]") do |type|
+          @options.output_type = type
+        end
       end
     end
   end

data/lib/groonga-query-log/command/run-regression-test.rb

@@ -1,5 +1,5 @@
-# Copyright (C) 2014-2019  Kouhei Sutou <kou@clear-code.com>
-# Copyright (C) 2019  Horimoto Yasuhiro <horimoto@clear-code.com>
+# Copyright (C) 2014-2020  Sutou Kouhei <kou@clear-code.com>
+# Copyright (C) 2019-2020  Horimoto Yasuhiro <horimoto@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -41,10 +41,12 @@ module GroongaQueryLog
         @old_groonga = "groonga"
         @old_database = "db.old/db"
         @old_groonga_options = []
+        @old_groonga_env = {}
 
         @new_groonga = "groonga"
         @new_database = "db.new/db"
         @new_groonga_options = []
+        @new_groonga_env = {}
 
         @recreate_database = false
         @load_data = true
@@ -60,6 +62,7 @@ module GroongaQueryLog
         @rewrite_not_or_regular_expression = false
         @rewrite_and_not_operator = false
         @debug_rewrite = false
+        @omit_rate = 0.0
 
         @care_order = true
         @ignored_drilldown_keys = []
@@ -102,12 +105,16 @@ module GroongaQueryLog
                             tester_options)
         success = tester.run
         elapsed_time = Time.now - start_time
+        n_leaked_objects = tester.new.n_leaked_objects
 
-        report = format_report(success, elapsed_time)
+        report = format_report(success,
+                               elapsed_time,
+                               n_leaked_objects,
+                               tester.n_executed_commands)
         notifier.notify_finished(success, report)
         puts(report)
 
-        success
+        success and n_leaked_objects.zero?
       end
 
       private
@@ -153,11 +160,21 @@ module GroongaQueryLog
 
         parser.on("--old-groonga-option=OPTION",
                   "Add an additional old groonga option",
-                  "You can specify this option multiple times to specify multiple groonga options",
+                  "You can specify this option multiple times",
+                  "to specify multiple groonga options",
                   "(no options)") do |groonga_option|
           @old_groonga_options << groonga_option
         end
 
+        parser.on("--old-groonga-env=KEY=VALUE",
+                  "Use KEY=VALUE environment variable for old groonga",
+                  "You can specify this option multiple times",
+                  "to specify multiple environment variables",
+                  "(no environment variables)") do |env|
+          key, value = env.split("=", 2)
+          @old_groonga_env[key] = value
+        end
+
         parser.separator("")
         parser.separator("New Groonga:")
         parser.on("--new-groonga=GROONGA",
@@ -168,11 +185,21 @@ module GroongaQueryLog
 
         parser.on("--new-groonga-option=OPTION",
                   "Add an additional new groonga option",
-                  "You can specify this option multiple times to specify multiple groonga options",
+                  "You can specify this option multiple times",
+                  "to specify multiple groonga options",
                   "(no options)") do |groonga_option|
           @new_groonga_options << groonga_option
         end
 
+        parser.on("--new-groonga-env=KEY=VALUE",
+                  "Use KEY=VALUE environment variable for new groonga",
+                  "You can specify this option multiple times",
+                  "to specify multiple environment variables",
+                  "(no environment variables)") do |env|
+          key, value = env.split("=", 2)
+          @new_groonga_env[key] = value
+        end
+
         parser.separator("")
         parser.separator("Operations:")
         parser.on("--recreate-database",
@@ -252,6 +279,13 @@ module GroongaQueryLog
                   "(#{@debug_rewrite})") do |boolean|
           @debug_rewrite = boolean
         end
+        parser.on("--omit-rate=RATE", Float,
+                  "You can specify rate for omitting execution queries." +
+                  "For example, if you specify 0.9 in this option, " +
+                  "execute queries with the probability of 1/10.",
+                  "(#{@omit_rate})") do |rate|
+          @omit_rate = rate
+        end
 
         parser.separator("")
         parser.separator("Comparisons:")
@@ -398,6 +432,7 @@ module GroongaQueryLog
           :rewrite_and_not_operator =>
             @rewrite_and_not_operator,
           :debug_rewrite => @debug_rewrite,
+          :omit_rate => @omit_rate,
           :target_command_names => @target_command_names,
           :verify_performance => @verify_performance,
           :performance_verfifier_options => @performance_verfifier_options,
@@ -409,6 +444,7 @@ module GroongaQueryLog
       def old_groonga_server
         GroongaServer.new(@old_groonga,
                           @old_groonga_options,
+                          @old_groonga_env,
                           @old_database,
                           server_options)
       end
@@ -416,16 +452,26 @@ module GroongaQueryLog
       def new_groonga_server
         GroongaServer.new(@new_groonga,
                           @new_groonga_options,
+                          @new_groonga_env,
                           @new_database,
                           server_options)
       end
 
-      def format_report(success, elapsed_time)
+      def format_report(success,
+                        elapsed_time,
+                        n_leaked_objects,
+                        n_executed_commands)
         formatted = format_elapsed_time(elapsed_time)
+        formatted << "Number of executed commands: #{n_executed_commands}\n"
         if success
-          formatted << "Success"
+          formatted << "Success\n"
         else
           formatted << "Failure"
+        end
+        unless n_leaked_objects.zero?
+          formatted << "\nLeaked: #{n_leaked_objects}"
+        end
+        unless success
           output = StringIO.new
           formetter = FormatRegressionTestLogs.new(output: output)
           formetter.run([results_directory])
@@ -459,11 +505,16 @@ module GroongaQueryLog
         include Loggable
 
         attr_reader :host, :port
-        def initialize(groonga, groonga_options, database_path, options)
+        def initialize(groonga,
+                       groonga_options,
+                       groonga_env,
+                       database_path,
+                       options)
           @input_directory = options[:input_directory] || Pathname.new(".")
           @working_directory = options[:working_directory] || Pathname.new(".")
           @groonga = groonga
           @groonga_options = groonga_options
+          @groonga_env = groonga_env
           @database_path = @working_directory + database_path
           @host = "127.0.0.1"
           @port = find_unused_port
@@ -473,17 +524,20 @@ module GroongaQueryLog
         def run
           return unless @options[:run_queries]
 
-          arguments = @groonga_options.dup
-          arguments.concat(["--bind-address", @host])
-          arguments.concat(["--port", @port.to_s])
-          arguments.concat(["--protocol", "http"])
-          arguments.concat(["--log-path", log_path.to_s])
+          spawn_args = []
+          spawn_args << @groonga_env if @groonga_env
+          spawn_args << @groonga
+          spawn_args.concat(@groonga_options)
+          spawn_args.concat(["--bind-address", @host])
+          spawn_args.concat(["--port", @port.to_s])
+          spawn_args.concat(["--protocol", "http"])
+          spawn_args.concat(["--log-path", log_path.to_s])
           if @options[:output_query_log]
-            arguments.concat(["--query-log-path", query_log_path.to_s])
+            spawn_args.concat(["--query-log-path", query_log_path.to_s])
           end
-          arguments << "-s"
-          arguments << @database_path.to_s
-          @pid = spawn(@groonga, *arguments)
+          spawn_args << "-s"
+          spawn_args << @database_path.to_s
+          @pid = spawn(*spawn_args)
 
           n_retries = 10
           begin
@@ -552,6 +606,19 @@ module GroongaQueryLog
           Process.waitpid(@pid)
         end
 
+        def n_leaked_objects
+          n = 0
+          File.open(log_path) do |log|
+            log.each_line do |line|
+              case line
+              when /grn_fin \((\d+)\)/
+                n += Integer($1, 10)
+              end
+            end
+          end
+          n
+        end
+
         private
         def find_unused_port
           server = TCPServer.new(@host, 0)
@@ -600,6 +667,8 @@ module GroongaQueryLog
       class Tester
         include Loggable
 
+        attr_reader :old
+        attr_reader :new
         def initialize(old, new, options)
           @old = old
           @new = new
@@ -611,6 +680,7 @@ module GroongaQueryLog
           @stop_on_failure = options[:stop_on_failure]
           @options = options
           @n_ready_waits = 2
+          @n_executed_commands = 0
         end
 
         def run
@@ -634,6 +704,10 @@ module GroongaQueryLog
           old_thread_success and new_thread_success
         end
 
+        def n_executed_commands
+          @n_executed_commands
+        end
+
         private
         def run_test
           @n_ready_waits -= 1
@@ -737,6 +811,10 @@ module GroongaQueryLog
           if @options[:debug_rewrite]
             command_line << "--debug-rewrite"
           end
+          if @options[:omit_rate] < 1.0
+            command_line << "--omit-rate"
+            command_line << @options[:omit_rate].to_s
+          end
           if @options[:target_command_names]
             command_line << "--target-command-names"
             command_line << @options[:target_command_names].join(",")
@@ -752,7 +830,9 @@ module GroongaQueryLog
             command_line << @options[:read_timeout].to_s
           end
           verify_server = VerifyServer.new
-          verify_server.run(command_line, &callback)
+          same = verify_server.run(command_line, &callback)
+          @n_executed_commands = verify_server.n_executed_commands
+          same
         end
 
         def query_log_paths

data/lib/groonga-query-log/command/verify-server.rb

@@ -1,4 +1,5 @@
 # Copyright (C) 2013-2018  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2020  Horimoto Yasuhiro <horimoto@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -22,8 +23,10 @@ require "groonga-query-log"
 module GroongaQueryLog
   module Command
     class VerifyServer
+      attr_reader :n_executed_commands
       def initialize
         @options = ServerVerifier::Options.new
+        @n_executed_commands = 0
       end
 
       def run(command_line, &callback)
@@ -50,6 +53,7 @@ module GroongaQueryLog
             end
           end
         end
+        @n_executed_commands = verifier.n_executed_commands
         same
       end
 
@@ -254,6 +258,14 @@ module GroongaQueryLog
           @options.debug_rewrite = boolean
         end
 
+        parser.on("--omit-rate=RATE", Float,
+                  "You can specify rate for omitting execution queries." +
+                  "For example, if you specify 0.9 in this option, " +
+                  "execute queries with the probability of 1/10.",
+                  "(#{@options.omit_rate})") do |rate|
+          @options.omit_rate = rate
+        end
+
         parser.on("--nullable-reference-number-accessor=ACCESSOR",
                   "Mark ACCESSOR as rewrite nullable reference number targets",
                   "You can specify multiple accessors by",

data/lib/groonga-query-log/replayer.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2020  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -22,186 +22,218 @@ require "groonga/client"
 require "groonga-query-log/parser"
 
 module GroongaQueryLog
-    class Replayer
-      def initialize(options)
-        @options = options
-        @queue = SizedQueue.new(@options.request_queue_size)
-        @responses = Queue.new
-      end
+  class Replayer
+    def initialize(options)
+      @options = options
+      @queue = SizedQueue.new(@options.request_queue_size)
+      @responses = Queue.new
+    end
 
-      def replay(input)
-        producer = run_producer(input)
-        consumers = run_consumers
-        response_logger = run_response_logger
-        producer.join
-        consumers.each(&:join)
-        response_logger.join
-      end
+    def replay(input)
+      producer = run_producer(input)
+      consumers = run_consumers
+      response_logger = run_response_logger
+      producer.join
+      consumers.each(&:join)
+      response_logger.join
+    end
 
-      private
-      def run_producer(input)
-        Thread.new do
-          parser = Parser.new
-          id = 0
-          @options.create_request_output do |output|
-            parser.parse(input) do |statistic|
-              next if statistic.command.nil?
-              next unless target_command?(statistic.command)
-              # TODO: validate orignal_source is one line
-              output.puts(statistic.command.original_source)
-              @queue.push([id, statistic])
-              id += 1
+    private
+    def run_producer(input)
+      Thread.new do
+        parser = Parser.new
+        id = 0
+        @options.create_request_output do |output|
+          parser.parse(input) do |statistic|
+            next if statistic.command.nil?
+            next unless target_command?(statistic.command)
+            # TODO: validate orignal_source is one line
+            output.puts(statistic.command.original_source)
+            output.flush
+            if @options.output_type
+              statistic.command[:output_type] ||= @options.output_type
             end
-          end
-          @options.n_clients.times do
-            @queue.push(nil)
+            @queue.push([id, statistic])
+            id += 1
           end
         end
+        @options.n_clients.times do
+          @queue.push(nil)
+        end
       end
+    end
 
-      def run_consumers
-        @options.n_clients.times.collect do
-          Thread.new do
-            loop do
-              break if run_consumer
-            end
+    def run_consumers
+      @options.n_clients.times.collect do
+        Thread.new do
+          loop do
+            break if run_consumer
           end
         end
       end
+    end
 
-      def run_consumer
-        @options.create_client do |client|
-          loop do
-            id, statistic = @queue.pop
-            if id.nil?
-              @responses.push(nil)
-              return true
-            end
-            begin
-              replay_command(client, id, statistic.command)
-            rescue Groonga::Client::Error
-              # TODO: add error log mechanism
-              $stderr.puts(Time.now.iso8601(6))
-              $stderr.puts(statistic.command.original_source)
-              $stderr.puts($!.raw_error.message)
-              $stderr.puts($!.raw_error.backtrace)
-              return false
-            rescue
-              # TODO: add error log mechanism
-              $stderr.puts(Time.now.iso8601(6))
-              $stderr.puts(statistic.command.original_source)
-              $stderr.puts($!.message)
-              $stderr.puts($!.backtrace)
-              return false
-            end
+    def run_consumer
+      @options.create_client do |client|
+        loop do
+          id, statistic = @queue.pop
+          if id.nil?
+            @responses.push(nil)
+            return true
+          end
+          begin
+            replay_command(client, id, statistic.command)
+          rescue Groonga::Client::Error
+            # TODO: add error log mechanism
+            $stderr.puts(Time.now.iso8601(6))
+            $stderr.puts(statistic.command.original_source)
+            $stderr.puts($!.raw_error.message)
+            $stderr.puts($!.raw_error.backtrace)
+            return false
+          rescue
+            # TODO: add error log mechanism
+            $stderr.puts(Time.now.iso8601(6))
+            $stderr.puts(statistic.command.original_source)
+            $stderr.puts($!.message)
+            $stderr.puts($!.backtrace)
+            return false
           end
         end
       end
+    end
 
-      def replay_command(client, id, command)
-        command["cache"] = "no" if @options.disable_cache?
-        response = client.execute(command)
-        @responses.push(response)
+    def replay_command(client, id, command)
+      command["cache"] = "no" if @options.disable_cache?
+      response = client.execute(command)
+      case command.output_type
+      when :json, :xml, :tsv
+        response.raw << "\n" unless response.raw.end_with?("\n")
       end
+      @responses.push(response)
+    end
 
-      def run_response_logger
-        Thread.new do
-          @options.create_responses_output do |output|
+    def run_response_logger
+      Thread.new do
+        @options.create_responses_output do |output|
+          @options.create_responses_output do |error_output|
             loop do
               response = @responses.pop
               break if response.nil?
-              # TODO: ensure response is one line
               # TODO: reorder by ID
-              output.puts(response.raw)
+              output.print(response.raw)
+              output.flush
+              unless response.success?
+                error_output.print(response.raw)
+                error_output.flush
+              end
             end
           end
         end
       end
+    end
 
-      def target_command?(command)
-        @options.target_command_name?(command.name)
-      end
+    def target_command?(command)
+      @options.target_command_name?(command.command_name)
+    end
 
-      class NullOutput
-        class << self
-          def open
-            output = new
-            if block_given?
-              yield(output)
-            else
-              output
-            end
+    class NullOutput
+      class << self
+        def open
+          output = new
+          if block_given?
+            yield(output)
+          else
+            output
           end
         end
+      end
 
-        def puts(string)
-        end
+      def puts(string)
       end
 
-      class Options
-        attr_accessor :host
-        attr_accessor :port
-        attr_accessor :protocol
-        attr_accessor :read_timeout
-        attr_accessor :n_clients
-        attr_writer :request_queue_size
-        attr_writer :disable_cache
-        attr_accessor :target_command_names
-        attr_accessor :requests_path
-        attr_accessor :responses_path
-        def initialize
-          @host = "127.0.0.1"
-          @port = 10041
-          @protocol = :http
-          @read_timeout = Groonga::Client::Default::READ_TIMEOUT
-          @n_clients = 8
-          @request_queue_size = nil
-          @disable_cache = false
-          @requests_path = nil
-          @responses_path = nil
-          @target_command_names = []
-        end
+      def print(string)
+      end
 
-        def create_client(&block)
-          Groonga::Client.open(:host     => @host,
-                               :port     => @port,
-                               :protocol => @protocol,
-                               :read_timeout => @read_timeout,
-                               &block)
-        end
+      def flush
+      end
+    end
 
-        def create_request_output(&block)
-          if @requests_path
-            File.open(@requests_path, "w", &block)
-          else
-            NullOutput.open(&block)
-          end
-        end
+    class Options
+      attr_accessor :host
+      attr_accessor :port
+      attr_accessor :protocol
+      attr_accessor :read_timeout
+      attr_accessor :n_clients
+      attr_writer :request_queue_size
+      attr_writer :disable_cache
+      attr_accessor :target_command_names
+      attr_accessor :requests_path
+      attr_accessor :responses_path
+      attr_accessor :error_responses_path
+      attr_accessor :output_type
+      def initialize
+        @host = "127.0.0.1"
+        @port = 10041
+        @protocol = :http
+        @read_timeout = Groonga::Client::Default::READ_TIMEOUT
+        @n_clients = 8
+        @request_queue_size = nil
+        @disable_cache = false
+        @target_command_names = []
+        @requests_path = nil
+        @responses_path = nil
+        @error_responses_path = nil
+        @output_type = nil
+      end
 
-        def create_responses_output(&block)
-          if @responses_path
-            File.open(@responses_path, "w", &block)
-          else
-            NullOutput.open(&block)
-          end
-        end
+      def create_client(&block)
+        Groonga::Client.open(:host     => @host,
+                             :port     => @port,
+                             :protocol => @protocol,
+                             :read_timeout => @read_timeout,
+                             &block)
+      end
 
-        def request_queue_size
-          @request_queue_size || @n_clients * 3
-        end
+      def create_request_output(&block)
+        create_output(@requests_path, &block)
+      end
+
+      def create_responses_output(&block)
+        create_output(@responses_path, &block)
+      end
 
-        def disable_cache?
-          @disable_cache
+      def create_error_responses_output(&block)
+        create_output(@error_responses_path, &block)
+      end
+
+      def request_queue_size
+        @request_queue_size || @n_clients * 3
+      end
+
+      def disable_cache?
+        @disable_cache
+      end
+
+      def target_command_name?(name)
+        return true if @target_command_names.empty?
+        @target_command_names.any? do |name_pattern|
+          flags = 0
+          flags |= File::FNM_EXTGLOB if File.const_defined?(:FNM_EXTGLOB)
+          File.fnmatch(name_pattern, name, flags)
         end
+      end
 
-        def target_command_name?(name)
-          return true if @target_command_names.empty?
-          @target_command_names.any? do |name_pattern|
-            flags = 0
-            flags |= File::FNM_EXTGLOB if File.const_defined?(:FNM_EXTGLOB)
-            File.fnmatch(name_pattern, name, flags)
-          end
+      private
+      def create_output(path, &block)
+        case path
+        when nil
+          NullOutput.open(&block)
+        when "-"
+          yield($stdout)
+        else
+          File.open(path, "w", &block)
         end
       end
     end
+  end
 end

data/lib/groonga-query-log/server-verifier.rb

@@ -1,4 +1,5 @@
 # Copyright (C) 2013-2018  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2020  Horimoto Yasuhiro <horimoto@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -26,10 +27,12 @@ require "groonga-query-log/response-comparer"
 
 module GroongaQueryLog
   class ServerVerifier
+    attr_reader :n_executed_commands
     def initialize(options)
       @options = options
       @queue = SizedQueue.new(@options.request_queue_size)
       @events = Queue.new
+      @n_executed_commands = 0
     end
 
     def verify(input, &callback)
@@ -50,7 +53,6 @@ module GroongaQueryLog
         consumers = run_consumers
 
         parser = Parser.new
-        n_commands = 0
         callback_per_n_commands = 100
         parser.parse(input) do |statistic|
           break if stop?
@@ -58,10 +60,11 @@ module GroongaQueryLog
           command = statistic.command
           next if command.nil?
           next unless target_command?(command)
-          n_commands += 1
+          next if rand < @options.omit_rate
+          @n_executed_commands += 1
           @queue.push(statistic)
 
-          if callback and (n_commands % callback_per_n_commands).zero?
+          if callback and (@n_executed_commands % callback_per_n_commands).zero?
             @options.n_clients.times do
               @queue.push(nil)
             end
@@ -283,6 +286,7 @@ module GroongaQueryLog
       attr_writer :verify_performance
       attr_reader :performance_verifier_options
       attr_writer :debug_rewrite
+      attr_writer :omit_rate
       def initialize
         @groonga1 = GroongaOptions.new
         @groonga2 = GroongaOptions.new
@@ -316,6 +320,7 @@ module GroongaQueryLog
         @verify_performance = false
         @performance_verifier_options = PerformanceVerifier::Options.new
         @debug_rewrite = false
+        @omit_rate = 0.0
       end
 
       def request_queue_size
@@ -411,6 +416,10 @@ module GroongaQueryLog
       def debug_rewrite?
         @debug_rewrite
       end
+
+      def omit_rate
+        @omit_rate
+      end
     end
 
     class GroongaOptions

data/lib/groonga-query-log/version.rb

@@ -15,5 +15,5 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module GroongaQueryLog
-  VERSION = "1.5.8"
+  VERSION = "1.5.9"
 end

data/test/command/test-run-regression-test.rb

@@ -1,5 +1,5 @@
 # Copyright (C) 2019 Kentaro Hayashi <hayashi@clear-code.com>
-# Copyright (C) 2019 Horimoto Yasuhiro <horimoto@clear-code.com>
+# Copyright (C) 2019-2020 Horimoto Yasuhiro <horimoto@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -26,6 +26,7 @@ class RunRegressionTestCommandTest < Test::Unit::TestCase
     FileUtils.rm_rf(fixture_path("db.old"))
     FileUtils.rm_rf(fixture_path("db.new"))
     setup_smtp_server
+    @n_commands = 238
   end
 
   def teardown
@@ -90,6 +91,10 @@ class RunRegressionTestCommandTest < Test::Unit::TestCase
       .gsub(/^Elapsed: \d+days \d{2}:\d{2}:\d{2}$/, "Elapsed: 0days 00:00:00")
   end
 
+  def n_executed_commands(output)
+    output.slice(/^Number of executed commands:\s+(\d+)/, 1).to_i
+  end
+
   def fixture_path(*components)
     super("run-regression-test", *components)
   end
@@ -105,11 +110,20 @@ class RunRegressionTestCommandTest < Test::Unit::TestCase
     assert_equal([
                    true,
                    "Elapsed: 0days 00:00:00\n" +
+                   "Number of executed commands: #{@n_commands}\n" +
                    "Success\n"
                  ],
                 [success, normalize_output(output)])
   end
 
+  def test_reduce_execution_query
+    command_line = ["--omit-rate=0.9"]
+    _success, output = run_command(command_line)
+    assert do
+      n_executed_commands(output) < (@n_commands * 0.2)
+    end
+  end
+
   def test_mail_from
     success, _output = run_command(["--smtp-server", @smtp_host,
                                     "--smtp-port", @smtp_port.to_s,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: groonga-query-log
 version: !ruby/object:Gem::Version
-  version: 1.5.8
+  version: 1.5.9
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-14 00:00:00.000000000 Z
+date: 2020-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: charty
@@ -169,17 +169,17 @@ email:
 - kou@clear-code.com
 executables:
 - groonga-query-log-analyze
-- groonga-query-log-verify-server
-- groonga-query-log-replay
-- groonga-query-log-check-crash
-- groonga-query-log-run-regression-test
 - groonga-query-log-analyze-load
-- groonga-query-log-show-running-queries
 - groonga-query-log-check-command-version-compatibility
+- groonga-query-log-check-crash
+- groonga-query-log-check-performance-regression
 - groonga-query-log-detect-memory-leak
 - groonga-query-log-extract
 - groonga-query-log-format-regression-test-logs
-- groonga-query-log-check-performance-regression
+- groonga-query-log-replay
+- groonga-query-log-run-regression-test
+- groonga-query-log-show-running-queries
+- groonga-query-log-verify-server
 extensions: []
 extra_rdoc_files: []
 files:
@@ -314,7 +314,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.0.pre1
 signing_key: 
 specification_version: 4
 summary: Groonga-query-log is a collection of library and tools to process [Groonga](http://groonga.org/)'s
@@ -322,49 +322,49 @@ summary: Groonga-query-log is a collection of library and tools to process [Groo
   as a library. You can analyze your Groonga's queries and test with your Groonga's
   query log by using groonga-query-log as a tool.
 test_files:
-- test/helper.rb
-- test/test-filter-rewriter.rb
-- test/test-parser.rb
+- test/command/test-analyzer.rb
+- test/command/test-check-performance-regression.rb
+- test/command/test-extract.rb
+- test/command/test-format-regression-test-logs.rb
+- test/command/test-run-regression-test.rb
+- test/fixtures/check-performance-regression/cache.log
+- test/fixtures/check-performance-regression/different_operations1.log
+- test/fixtures/check-performance-regression/different_operations2.log
+- test/fixtures/check-performance-regression/nquery.log
+- test/fixtures/check-performance-regression/nquery2.log
+- test/fixtures/check-performance-regression/query1.log
+- test/fixtures/check-performance-regression/query2.log
+- test/fixtures/multi.expected
 - test/fixtures/n_entries.expected
-- test/fixtures/order/elapsed.expected
+- test/fixtures/no-report-summary.expected
 - test/fixtures/order/-elapsed.expected
 - test/fixtures/order/-start-time.expected
+- test/fixtures/order/elapsed.expected
 - test/fixtures/order/start-time.expected
-- test/fixtures/target-tables.expected
 - test/fixtures/other-query.log
-- test/fixtures/check-performance-regression/different_operations2.log
-- test/fixtures/check-performance-regression/query2.log
-- test/fixtures/check-performance-regression/nquery.log
-- test/fixtures/check-performance-regression/query1.log
-- test/fixtures/check-performance-regression/different_operations1.log
-- test/fixtures/check-performance-regression/nquery2.log
-- test/fixtures/check-performance-regression/cache.log
-- test/fixtures/regression-test-logs/url-format.log
+- test/fixtures/query.log
+- test/fixtures/regression-test-logs/command-format.log
 - test/fixtures/regression-test-logs/error.log
 - test/fixtures/regression-test-logs/slow.log
-- test/fixtures/regression-test-logs/command-format.log
+- test/fixtures/regression-test-logs/url-format.log
+- test/fixtures/reporter/console.expected
+- test/fixtures/reporter/html.expected
+- test/fixtures/reporter/json-stream.expected
+- test/fixtures/reporter/json.expected
+- test/fixtures/run-regression-test/data/data.grn
 - test/fixtures/run-regression-test/indexes/indexes.grn
+- test/fixtures/run-regression-test/mail-notifier/failure.log
+- test/fixtures/run-regression-test/mail-notifier/success.log
+- test/fixtures/run-regression-test/query-logs/query.log
 - test/fixtures/run-regression-test/results/query.log.log
-- test/fixtures/run-regression-test/data/data.grn
 - test/fixtures/run-regression-test/schema/schema.grn
-- test/fixtures/run-regression-test/query-logs/query.log
-- test/fixtures/run-regression-test/mail-notifier/success.log
-- test/fixtures/run-regression-test/mail-notifier/failure.log
 - test/fixtures/target-commands.expected
-- test/fixtures/query.log
-- test/fixtures/multi.expected
-- test/fixtures/no-report-summary.expected
-- test/fixtures/reporter/html.expected
-- test/fixtures/reporter/json.expected
-- test/fixtures/reporter/console.expected
-- test/fixtures/reporter/json-stream.expected
+- test/fixtures/target-tables.expected
+- test/helper.rb
 - test/run-test.rb
-- test/test-performance-verifier.rb
+- test/test-filter-rewriter.rb
 - test/test-incompatibility-detector.rb
+- test/test-parser.rb
+- test/test-performance-verifier.rb
 - test/test-replayer.rb
-- test/command/test-run-regression-test.rb
-- test/command/test-format-regression-test-logs.rb
-- test/command/test-extract.rb
-- test/command/test-analyzer.rb
-- test/command/test-check-performance-regression.rb
 - test/test-response-comparer.rb