groonga-query-log 1.2.8 → 1.2.9

data/lib/groonga-query-log/command/extract.rb

@@ -0,0 +1,171 @@
+# Copyright (C) 2011-2018  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "ostruct"
+require "optparse"
+require "pathname"
+
+require "groonga-query-log"
+require "groonga-query-log/command-line"
+
+module GroongaQueryLog
+    module Command
+      class Extract < CommandLine
+        attr_accessor :options
+        attr_reader :option_parser
+
+        def initialize
+          @options = nil
+          @option_parser = nil
+          setup_options
+        end
+
+        # Executes extractor for Groonga's query logs.
+        # "groonga-query-log-extract" command runs this method.
+        #
+        # @example
+        #   extractor = GroongaQueryLog::Command::Extract.new
+        #   extractor.run("--output", "commands.output",
+        #                 "--command", "select",
+        #                 "query.log")
+        #
+        # If only paths of query log files are specified,
+        # this method prints command(s) of them to console.
+        #
+        # @param [Array<String>] arguments arguments for
+        #   groonga-query-log-extract. Please execute
+        #   "groonga-query-log-extract --help" or see #setup_options.
+        def run(arguments)
+          begin
+            log_paths = @option_parser.parse!(arguments)
+          rescue OptionParser::ParseError
+            $stderr.puts($!.message)
+            return false
+          end
+
+          begin
+            if @options.output_path
+              File.open(@options.output_path, "w") do |output|
+                extract(log_paths, output)
+              end
+            else
+              extract(log_paths, $stdout)
+            end
+          rescue Interrupt
+          rescue Error
+            $stderr.puts($!.message)
+            return false
+          end
+
+          true
+        end
+
+        private
+        def setup_options
+          @options = OpenStruct.new
+          @options.unify_format = nil
+          @options.commands = []
+          @options.exclude_commands = []
+          @options.output_path = nil
+          @option_parser = OptionParser.new do |parser|
+            parser.version = VERSION
+            parser.banner += " QUERY_LOG1 ..."
+
+            available_formats = ["uri", "command"]
+            parser.on("--unify-format=FORMAT",
+                      available_formats,
+                      "Unify command format to FORMAT.",
+                      "(#{available_formats.join(', ')})",
+                      "[not unify]") do |format|
+              @options.unify_format = format
+            end
+
+            parser.on("--command=COMMAND",
+                      "Extract only COMMAND.",
+                      "To extract one or more commands,",
+                      "specify this command a number of times.",
+                      "Use /.../ as COMMAND to match command with regular expression.",
+                      "[all commands]") do |command|
+              case command
+              when /\A\/(.*)\/(i)?\z/
+                @options.commands << Regexp.new($1, $2 == "i")
+              when
+                @options.commands << command
+              end
+            end
+
+            parser.on("--exclude-command=COMMAND",
+                      "Don't extract COMMAND.",
+                      "To ignore one or more commands,",
+                      "specify this command a number of times.",
+                      "Use /.../ as COMMAND to match command with regular expression.",
+                      "[no commands]") do |command|
+              case command
+              when /\A\/(.*)\/(i)?\z/
+                @options.exclude_commands << Regexp.new($1, $2 == "i")
+              when
+                @options.exclude_commands << command
+              end
+            end
+
+            parser.on("--output=PATH",
+                      "Output to PATH.",
+                      "[standard output]") do |path|
+              @options.output_path = path
+            end
+          end
+        end
+
+        def extract(log_paths, output)
+          parser = Parser.new
+          parse_log(parser, log_paths) do |statistic|
+            extract_command(statistic, output)
+          end
+        end
+
+        def extract_command(statistic, output)
+          command = statistic.command
+          return unless target?(command)
+          command_text = nil
+          case @options.unify_format
+          when "uri"
+            command_text = command.to_uri_format
+          when "command"
+            command_text = command.to_command_format
+          else
+            command_text = statistic.raw_command
+          end
+          output.puts(command_text)
+        end
+
+        def target?(command)
+          name = command.command_name
+          target_commands = @options.commands
+          exclude_commands = @options.exclude_commands
+
+          unless target_commands.empty?
+            return target_commands.any? {|target_command| target_command === name}
+          end
+
+          unless exclude_commands.empty?
+            return (not exclude_commands.any? {|exclude_command| exclude_command === name})
+          end
+
+          true
+        end
+      end
+    end
+end

data/lib/groonga-query-log/command/format-regression-test-logs.rb

@@ -0,0 +1,143 @@
+# Copyright (C) 2014-2017  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "English"
+require "find"
+require "tempfile"
+require "pp"
+require "optparse"
+require "json"
+
+require "groonga/command/parser"
+
+require "groonga-query-log/version"
+
+module GroongaQueryLog
+    module Command
+      class FormatRegressionTestLogs
+        def initialize
+        end
+
+        def run(command_line)
+          parser = OptionParser.new
+          parser.banner += " PATH1 PATH2 ..."
+          parser.version = VERSION
+          paths = parser.parse!(command_line)
+
+          if paths.empty?
+            format_log($stdin, "-")
+          else
+            paths.each do |path|
+              if File.directory?(path)
+                Find.find(path) do |sub_path|
+                  next unless File.file?(sub_path)
+                  File.open(sub_path) do |file|
+                    format_log(file, sub_path)
+                  end
+                end
+              else
+                File.open(path) do |file|
+                  format_log(file, path)
+                end
+              end
+            end
+          end
+          true
+        end
+
+        private
+        def format_log(input, path)
+          command = nil
+          response_old = nil
+          response_new = nil
+
+          input.each_line do |line|
+            unless line.valid_encoding?
+              puts("invalid encoding line")
+              puts("#{path}:#{input.lineno}:#{line}")
+              next
+            end
+            case line
+            when /\Acommand: /
+              command = $POSTMATCH.chomp
+            when /\Aresponse1: /
+              response_old = $POSTMATCH.chomp
+            when /\Aresponse2: /
+              response_new = $POSTMATCH.chomp
+              next unless valid_entry?(command, response_old, response_new)
+              report_diff(command, response_old, response_new)
+            end
+          end
+        end
+
+        def valid_entry?(command, response_old, response_new)
+          valid = true
+
+          begin
+            JSON.parse(response_old)
+          rescue JSON::ParserError
+            puts(command)
+            puts("failed to parse old response: #{$!.message}")
+            puts(response_old)
+            valid = false
+          end
+
+          begin
+            JSON.parse(response_new)
+          rescue JSON::ParserError
+            puts(command)
+            puts("failed to parse new response: #{$!.message}")
+            puts(response_new)
+            valid = false
+          end
+
+          valid
+        end
+
+        def report_diff(command, response_old, response_new)
+          return if response_old == response_new
+
+          Tempfile.open("response-old") do |response_old_file|
+            PP.pp(JSON.parse(response_old), response_old_file)
+            response_old_file.flush
+            Tempfile.open("response-new") do |response_new_file|
+              PP.pp(JSON.parse(response_new), response_new_file)
+              response_new_file.flush
+              report_command(command)
+              system("diff",
+                     "--label=old",
+                     "--label=new",
+                     "-u",
+                     response_old_file.path, response_new_file.path)
+            end
+          end
+        end
+
+        def report_command(command)
+          puts(command)
+          parsed_command = Groonga::Command::Parser.parse(command)
+          puts("Name: #{parsed_command.name}")
+          puts("Arguments:")
+          sorted_arguments = parsed_command.arguments.sort_by do |key, value|
+            key
+          end
+          sorted_arguments.each do |key, value|
+            puts("  #{key}: #{value}")
+          end
+        end
+      end
+    end
+end

data/lib/groonga-query-log/command/replay.rb

@@ -0,0 +1,117 @@
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "optparse"
+
+require "groonga-query-log/version"
+require "groonga-query-log/replayer"
+
+module GroongaQueryLog
+  module Command
+    class Replay
+      def initialize
+        @options = Replayer::Options.new
+      end
+
+      def run(command_line)
+        input_paths = create_parser.parse(command_line)
+        replayer = Replayer.new(@options)
+        input_paths.each do |input_path|
+          File.open(input_path) do |input|
+            replayer.replay(input)
+          end
+        end
+        true
+      end
+
+      private
+      def create_parser
+        parser = OptionParser.new
+        parser.version = VERSION
+        parser.banner += " QUERY_LOG"
+
+        parser.separator("")
+        parser.separator("Options:")
+
+        parser.on("--host=HOST",
+                  "Host name or IP address of Groonga server",
+                  "[#{@options.host}]") do |host|
+          @options.host = host
+        end
+
+        parser.on("--port=PORT", Integer,
+                  "Port number of Groonga server",
+                  "[#{@options.port}]") do |port|
+          @options.port = port
+        end
+
+        available_protocols = [:gqtp, :http]
+        available_protocols_label = "(#{available_protocols.join(', ')})"
+        parser.on("--protocol=PROTOCOL", available_protocols,
+                  "Protocol of Groonga server",
+                  "[#{@options.protocol}]",
+                  available_protocols_label) do |protocol|
+          @options.protocol = protocol
+        end
+
+        parser.on("--n-clients=N", Integer,
+                  "The max number of concurrency",
+                  "[#{@options.n_clients}]") do |n_clients|
+          @options.n_clients = n_clients
+        end
+
+        parser.on("--request-queue-size=SIZE", Integer,
+                  "The size of request queue",
+                  "[auto]") do |size|
+          @options.request_queue_size = size
+        end
+
+        parser.on("--disable-cache",
+                  "Add 'cache=no' parameter to request",
+                  "[#{@options.disable_cache?}]") do
+          @options.disable_cache = true
+        end
+
+        parser.on("--target-command-name=NAME",
+                  "Add NAME to target command names",
+                  "You can specify this option zero or more times",
+                  "See also --target-command-names") do |name|
+          @options.target_command_names << name
+        end
+
+        target_command_names_label = @options.target_command_names.join(", ")
+        parser.on("--target-command-names=NAME1,NAME2,...", Array,
+                  "Replay only NAME1,NAME2,... commands",
+                  "You can use glob to choose command name",
+                  "[#{target_command_names_label}]") do |names|
+          @options.target_command_names = names
+        end
+
+        parser.on("--output-requests=PATH",
+                  "Output requests to PATH",
+                  "[not output]") do |path|
+          @options.requests_path = path
+        end
+
+        parser.on("--output-responses=PATH",
+                  "Output responses to PATH",
+                  "[not output]") do |path|
+          @options.responses_path = path
+        end
+      end
+    end
+  end
+end

data/lib/groonga-query-log/command/run-regression-test.rb

@@ -0,0 +1,432 @@
+# Copyright (C) 2014-2017  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "rbconfig"
+require "optparse"
+require "socket"
+require "fileutils"
+require "pathname"
+require "net/http"
+
+require "groonga-query-log"
+require "groonga-query-log/command/verify-server"
+
+module GroongaQueryLog
+    module Command
+      class RunRegressionTest
+        def initialize
+          @input_directory = Pathname.new(".")
+          @working_directory = Pathname.new(".")
+
+          @old_groonga = "groonga"
+          @old_database = "db.old/db"
+          @old_groonga_options = []
+
+          @new_groonga = "groonga"
+          @new_database = "db.new/db"
+          @new_groonga_options = []
+
+          @recreate_database = false
+          @load_data = true
+          @run_queries = true
+          @skip_finished_queries = false
+          @output_query_log = false
+          @care_order = true
+          @verify_cachehit_mode = false
+        end
+
+        def run(command_line)
+          option_parser = create_option_parser
+          begin
+            option_parser.parse!(command_line)
+          rescue OptionParser::ParseError => error
+            $stderr.puts(error.message)
+            return false
+          end
+
+          tester = Tester.new(old_groonga_server,
+                              new_groonga_server,
+                              tester_options)
+          tester.run
+        end
+
+        private
+        def create_option_parser
+          parser = OptionParser.new
+          parser.version = VERSION
+
+          parser.separator("")
+          parser.separator("Path:")
+          parser.on("--input-directory=DIRECTORY",
+                    "Load schema and data from DIRECTORY.",
+                    "(#{@input_directory})") do |directory|
+            @input_directory = Pathname.new(directory)
+          end
+          parser.on("--working-directory=DIRECTORY",
+                    "Use DIRECTORY as working directory.",
+                    "(#{@working_directory})") do |directory|
+            @working_directory = Pathname.new(directory)
+          end
+
+          parser.separator("")
+          parser.separator("Throughput:")
+          parser.on("--n-clients=N", Integer,
+                    "Use N clients concurrently.",
+                    "(#{@n_clients})") do |n|
+            @n_clients = n
+          end
+
+          parser.separator("")
+          parser.separator("Old Groonga:")
+          parser.on("--old-groonga=GROONGA",
+                    "Old groonga command",
+                    "(#{@old_groonga})") do |groonga|
+            @old_groonga = groonga
+          end
+
+          parser.on("--old-groonga-option=OPTION",
+                    "Add an additional old groonga option",
+                    "You can specify this option multiple times to specify multiple groonga options",
+                    "(no options)") do |groonga_option|
+            @old_groonga_options << groonga_option
+          end
+
+          parser.separator("")
+          parser.separator("New Groonga:")
+          parser.on("--new-groonga=GROONGA",
+                    "New groonga command",
+                    "(#{@new_groonga})") do |groonga|
+            @new_groonga = groonga
+          end
+
+          parser.on("--new-groonga-option=OPTION",
+                    "Add an additional new groonga option",
+                    "You can specify this option multiple times to specify multiple groonga options",
+                    "(no options)") do |groonga_option|
+            @new_groonga_options << groonga_option
+          end
+
+          parser.separator("")
+          parser.separator("Operations:")
+          parser.on("--recreate-database",
+                    "Always recreate Groonga database") do
+            @recreate_database = true
+          end
+          parser.on("--no-load-data",
+                    "Don't load data. Just loads schema to Groonga database") do
+            @load_data = false
+          end
+          parser.on("--no-run-queries",
+                    "Don't run queries. Just creates Groonga database") do
+            @run_queries = false
+          end
+          parser.on("--skip-finished-queries",
+                    "Don't run finished query logs.") do
+            @skip_finished_queries = true
+          end
+          parser.on("--output-query-log",
+                    "Output query log in verified target Groonga servers") do
+            @output_query_log = true
+          end
+          parser.on("--no-care-order",
+                    "Don't care order of select response records") do
+            @care_order = false
+          end
+
+          parser
+        end
+
+        def directory_options
+          {
+            :input_directory   => @input_directory,
+            :working_directory => @working_directory,
+          }
+        end
+
+        def server_options
+          options = {
+            :load_data             => @load_data,
+            :run_queries           => @run_queries,
+            :recreate_database     => @recreate_database,
+            :output_query_log      => @output_query_log,
+          }
+          directory_options.merge(options)
+        end
+
+        def tester_options
+          options = {
+            :n_clients  => @n_clients,
+            :care_order => @care_order,
+            :skip_finished_queries => @skip_finished_queries,
+          }
+          directory_options.merge(options)
+        end
+
+        def old_groonga_server
+          GroongaServer.new(@old_groonga,
+                            @old_groonga_options,
+                            @old_database,
+                            server_options)
+        end
+
+        def new_groonga_server
+          GroongaServer.new(@new_groonga,
+                            @new_groonga_options,
+                            @new_database,
+                            server_options)
+        end
+
+        class GroongaServer
+          attr_reader :host, :port
+          def initialize(groonga, groonga_options, database_path, options)
+            @input_directory = options[:input_directory] || Pathname.new(".")
+            @working_directory = options[:working_directory] || Pathname.new(".")
+            @groonga = groonga
+            @groonga_options = groonga_options
+            @database_path = @working_directory + database_path
+            @host = "127.0.0.1"
+            @port = find_unused_port
+            @options = options
+          end
+
+          def run
+            return unless @options[:run_queries]
+
+            arguments = @groonga_options.dup
+            arguments.concat(["--bind-address", @host])
+            arguments.concat(["--port", @port.to_s])
+            arguments.concat(["--protocol", "http"])
+            arguments.concat(["--log-path", log_path.to_s])
+            if @options[:output_query_log]
+              arguments.concat(["--query-log-path", query_log_path.to_s])
+            end
+            arguments << "-s"
+            arguments << @database_path.to_s
+            @pid = spawn(@groonga, *arguments)
+
+            n_retries = 10
+            begin
+              send_command("status")
+            rescue SystemCallError
+              sleep(1)
+              n_retries -= 1
+              raise if n_retries.zero?
+              retry
+            end
+
+            yield if block_given?
+          end
+
+          def ensure_database
+            if @options[:recreate_database]
+              FileUtils.rm_rf(@database_path.dirname.to_s)
+            end
+
+            return if @database_path.exist?
+            FileUtils.mkdir_p(@database_path.dirname.to_s)
+            system(@groonga, "-n", @database_path.to_s, "quit")
+            grn_files.each do |grn_file|
+              command = [
+                @groonga,
+                "--log-path", log_path.to_s,
+                "--file", grn_file.to_s,
+                @database_path.to_s,
+              ]
+              command_line = command.join(" ")
+              puts("Running...: #{command_line}")
+              pid = spawn(*command)
+              begin
+                pid, status = Process.waitpid2(pid)
+              rescue Interrupt
+                Process.kill(:TERM, pid)
+                pid, status = Process.waitpid2(pid)
+              end
+              unless status.success?
+                raise "Failed to run: #{command_line}"
+              end
+            end
+          end
+
+          def use_persistent_cache?
+            @groonga_options.include?("--cache-base-path")
+          end
+
+          def shutdown
+            begin
+              send_command("shutdown")
+            rescue SystemCallError
+            end
+            Process.waitpid(@pid)
+          end
+
+          private
+          def find_unused_port
+            server = TCPServer.new(@host, 0)
+            begin
+              server.addr[1]
+            ensure
+              server.close
+            end
+          end
+
+          def log_path
+            @database_path.dirname + "groonga.log"
+          end
+
+          def query_log_path
+            @database_path.dirname + "query.log"
+          end
+
+          def send_command(name)
+            Net::HTTP.start(@host, @port) do |http|
+              response = http.get("/d/#{name}")
+              response.body
+            end
+          end
+
+          def grn_files
+            files = schema_files
+            files += data_files if @options[:load_data]
+            files += index_files
+            files
+          end
+
+          def schema_files
+            Pathname.glob("#{@input_directory}/schema/**/*.grn").sort
+          end
+
+          def index_files
+            Pathname.glob("#{@input_directory}/indexes/**/*.grn").sort
+          end
+
+          def data_files
+            Pathname.glob("#{@input_directory}/data/**/*.grn").sort
+          end
+        end
+
+        class Tester
+          def initialize(old, new, options)
+            @old = old
+            @new = new
+            @input_directory = options[:input_directory] || Pathname.new(".")
+            @working_directory = options[:working_directory] || Pathname.new(".")
+            @n_clients = options[:n_clients] || 1
+            @options = options
+            @n_ready_waits = 2
+          end
+
+          def run
+            @old.ensure_database
+            @new.ensure_database
+
+            old_thread = Thread.new do
+              @old.run do
+                run_test
+              end
+            end
+            new_thread = Thread.new do
+              @new.run do
+                run_test
+              end
+            end
+
+            old_thread_success = old_thread.value
+            new_thread_success = new_thread.value
+
+            old_thread_success and new_thread_success
+          end
+
+          private
+          def run_test
+            @n_ready_waits -= 1
+            return true unless @n_ready_waits.zero?
+
+            query_log_paths.each do |query_log_path|
+              log_path = test_log_path(query_log_path)
+              if @options[:skip_finished_queries] and log_path.exist?
+                puts("Skip query log: #{query_log_path}")
+                next
+              else
+                puts("Running test against query log...: #{query_log_path}")
+              end
+              begin
+                if use_persistent_cache?
+                  callback = lambda do
+                    if @old.use_persistent_cache?
+                      @old.shutdown
+                      @old.run
+                    end
+                    if @new.use_persistent_cache?
+                      @new.shutdown
+                      @new.run
+                    end
+                  end
+                else
+                  callback = nil
+                end
+                verify_server(log_path, query_log_path, &callback)
+              rescue Interrupt
+                puts("Interrupt: #{query_log_path}")
+              end
+            end
+
+            old_thread = Thread.new do
+              @old.shutdown
+            end
+            new_thread = Thread.new do
+              @new.shutdown
+            end
+            old_thread.join
+            new_thread.join
+
+            true
+          end
+
+          def verify_server(test_log_path, query_log_path, &callback)
+            command_line = [
+              "--n-clients=#{@n_clients}",
+              "--groonga1-host=#{@old.host}",
+              "--groonga1-port=#{@old.port}",
+              "--groonga1-protocol=http",
+              "--groonga2-host=#{@new.host}",
+              "--groonga2-port=#{@new.port}",
+              "--groonga2-protocol=http",
+              "--output", test_log_path.to_s,
+            ]
+            command_line << "--no-care-order" if @options[:care_order] == false
+            command_line << query_log_path.to_s
+            if use_persistent_cache?
+              command_line << "--verify-cache"
+            end
+            verify_server = VerifyServer.new
+            verify_server.run(command_line, &callback)
+          end
+
+          def query_log_paths
+            Pathname.glob("#{@input_directory}/query-logs/**/*.log").sort
+          end
+
+          def test_log_path(query_log_path)
+            @working_directory + "results" + query_log_path.basename
+          end
+
+          def use_persistent_cache?
+            @old.use_persistent_cache? or @new.use_persistent_cache?
+          end
+        end
+      end
+    end
+end