groonga-query-log 1.2.8 → 1.2.9

data/lib/groonga/query-log/command/extract.rb

@@ -1,11 +1,9 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2011  Kouhei Sutou <kou@clear-code.com>
-# Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
+# Copyright (C) 2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
-# License version 2.1 as published by the Free Software Foundation.
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
 #
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -14,179 +12,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
-require "ostruct"
-require "optparse"
-require "pathname"
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "groonga/query-log"
-require "groonga/query-log/command-line-utils"
-
-module Groonga
-  module QueryLog
-    module Command
-      class Extract
-        include CommandLineUtils
-
-        class Error < StandardError
-        end
-
-        attr_accessor :options
-        attr_reader :option_parser
-
-        def initialize
-          @options = nil
-          @option_parser = nil
-          setup_options
-        end
-
-        # Executes extractor for groonga's query logs.
-        # "groonga-query-log-extract" command runs this method.
-        #
-        # @example
-        #   extractor = Groonga::QueryLog::Command::Extract.new
-        #   extractor.run("--output", "commands.output",
-        #                 "--command", "select",
-        #                 "query.log")
-        #
-        # If only paths of query log files are specified,
-        # this method prints command(s) of them to console.
-        #
-        # @param [Array<String>] arguments arguments for
-        #   groonga-query-log-extract. Please execute
-        #   "groonga-query-log-extract --help" or see #setup_options.
-        def run(arguments)
-          begin
-            log_paths = @option_parser.parse!(arguments)
-          rescue OptionParser::ParseError
-            $stderr.puts($!.message)
-            return false
-          end
-
-          if log_paths.empty?
-            unless log_via_stdin?
-              $stderr.puts("Error: Please specify input log files.")
-              return false
-            end
-            log = $stdin
-          else
-            log = log_paths
-          end
-
-          if @options.output_path
-            File.open(@options.output_path, "w") do |output|
-              extract(log, output)
-            end
-          else
-            extract(log, $stdout)
-          end
-
-          true
-        end
-
-        private
-        def setup_options
-          @options = OpenStruct.new
-          @options.unify_format = nil
-          @options.commands = []
-          @options.exclude_commands = []
-          @options.output_path = nil
-          @option_parser = OptionParser.new do |parser|
-            parser.version = VERSION
-            parser.banner += " QUERY_LOG1 ..."
-
-            available_formats = ["uri", "command"]
-            parser.on("--unify-format=FORMAT",
-                      available_formats,
-                      "Unify command format to FORMAT.",
-                      "(#{available_formats.join(', ')})",
-                      "[not unify]") do |format|
-              @options.unify_format = format
-            end
-
-            parser.on("--command=COMMAND",
-                      "Extract only COMMAND.",
-                      "To extract one or more commands,",
-                      "specify this command a number of times.",
-                      "Use /.../ as COMMAND to match command with regular expression.",
-                      "[all commands]") do |command|
-              case command
-              when /\A\/(.*)\/(i)?\z/
-                @options.commands << Regexp.new($1, $2 == "i")
-              when
-                @options.commands << command
-              end
-            end
-
-            parser.on("--exclude-command=COMMAND",
-                      "Don't extract COMMAND.",
-                      "To ignore one or more commands,",
-                      "specify this command a number of times.",
-                      "Use /.../ as COMMAND to match command with regular expression.",
-                      "[no commands]") do |command|
-              case command
-              when /\A\/(.*)\/(i)?\z/
-                @options.exclude_commands << Regexp.new($1, $2 == "i")
-              when
-                @options.exclude_commands << command
-              end
-            end
-
-            parser.on("--output=PATH",
-                      "Output to PATH.",
-                      "[standard output]") do |path|
-              @options.output_path = path
-            end
-          end
-        end
-
-        def extract(log, output)
-          if log.instance_of?(Array)
-            log.each do |log_path|
-              File.open(log_path) do |log_file|
-                extract_command(log_file, output)
-              end
-            end
-          else
-            extract_command(log, output)
-          end
-        end
-
-        def extract_command(log, output)
-          parser = Groonga::QueryLog::Parser.new
-          parser.parse(log) do |statistic|
-            command = statistic.command
-            next unless target?(command)
-            command_text = nil
-            case @options.unify_format
-            when "uri"
-              command_text = command.to_uri_format
-            when "command"
-              command_text = command.to_command_format
-            else
-              command_text = statistic.raw_command
-            end
-            output.puts(command_text)
-          end
-        end
-
-        def target?(command)
-          name = command.command_name
-          target_commands = @options.commands
-          exclude_commands = @options.exclude_commands
-
-          unless target_commands.empty?
-            return target_commands.any? {|target_command| target_command === name}
-          end
-
-          unless exclude_commands.empty?
-            return (not exclude_commands.any? {|exclude_command| exclude_command === name})
-          end
-
-          true
-        end
-      end
-    end
-  end
-end
+require "groonga-query-log/command/extract"

data/lib/groonga/query-log/command/format-regression-test-logs.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -14,132 +14,5 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "English"
-require "find"
-require "tempfile"
-require "pp"
-require "optparse"
-require "json"
-
-require "groonga/command/parser"
-
-require "groonga/query-log/version"
-
-module Groonga
-  module QueryLog
-    module Command
-      class FormatRegressionTestLogs
-        def initialize
-        end
-
-        def run(command_line)
-          parser = OptionParser.new
-          parser.banner += " PATH1 PATH2 ..."
-          parser.version = VERSION
-          paths = parser.parse!(command_line)
-
-          if paths.empty?
-            format_log($stdin, "-")
-          else
-            paths.each do |path|
-              if File.directory?(path)
-                Find.find(path) do |sub_path|
-                  next unless File.file?(sub_path)
-                  File.open(sub_path) do |file|
-                    format_log(file, sub_path)
-                  end
-                end
-              else
-                File.open(path) do |file|
-                  format_log(file, path)
-                end
-              end
-            end
-          end
-          true
-        end
-
-        private
-        def format_log(input, path)
-          command = nil
-          response_old = nil
-          response_new = nil
-
-          input.each_line do |line|
-            unless line.valid_encoding?
-              puts("invalid encoding line")
-              puts("#{path}:#{input.lineno}:#{line}")
-              next
-            end
-            case line
-            when /\Acommand: /
-              command = $POSTMATCH.chomp
-            when /\Aresponse1: /
-              response_old = $POSTMATCH.chomp
-            when /\Aresponse2: /
-              response_new = $POSTMATCH.chomp
-              next unless valid_entry?(command, response_old, response_new)
-              report_diff(command, response_old, response_new)
-            end
-          end
-        end
-
-        def valid_entry?(command, response_old, response_new)
-          valid = true
-
-          begin
-            JSON.parse(response_old)
-          rescue JSON::ParserError
-            puts(command)
-            puts("failed to parse old response: #{$!.message}")
-            puts(response_old)
-            valid = false
-          end
-
-          begin
-            JSON.parse(response_new)
-          rescue JSON::ParserError
-            puts(command)
-            puts("failed to parse new response: #{$!.message}")
-            puts(response_new)
-            valid = false
-          end
-
-          valid
-        end
-
-        def report_diff(command, response_old, response_new)
-          return if response_old == response_new
-
-          Tempfile.open("response-old") do |response_old_file|
-            PP.pp(JSON.parse(response_old), response_old_file)
-            response_old_file.flush
-            Tempfile.open("response-new") do |response_new_file|
-              PP.pp(JSON.parse(response_new), response_new_file)
-              response_new_file.flush
-              report_command(command)
-              system("diff",
-                     "--label=old",
-                     "--label=new",
-                     "-u",
-                     response_old_file.path, response_new_file.path)
-            end
-          end
-        end
-
-        def report_command(command)
-          puts(command)
-          parsed_command = Groonga::Command::Parser.parse(command)
-          puts("Name: #{parsed_command.name}")
-          puts("Arguments:")
-          sorted_arguments = parsed_command.arguments.sort_by do |key, value|
-            key
-          end
-          sorted_arguments.each do |key, value|
-            puts("  #{key}: #{value}")
-          end
-        end
-      end
-    end
-  end
-end
+require "groonga/query-log"
+require "groonga-query-log/command/format-regression-test-logs"

data/lib/groonga/query-log/command/reply.rb

@@ -0,0 +1,18 @@
+# Copyright (C) 2017  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga/query-log"
+require "groonga-query-log/command/reply"

data/lib/groonga/query-log/command/run-regression-test.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -14,421 +14,5 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "rbconfig"
-require "optparse"
-require "socket"
-require "fileutils"
-require "pathname"
-require "net/http"
-
 require "groonga/query-log"
-require "groonga/query-log/command/verify-server"
-
-module Groonga
-  module QueryLog
-    module Command
-      class RunRegressionTest
-        def initialize
-          @input_directory = Pathname.new(".")
-          @working_directory = Pathname.new(".")
-
-          @old_groonga = "groonga"
-          @old_database = "db.old/db"
-          @old_groonga_options = []
-
-          @new_groonga = "groonga"
-          @new_database = "db.new/db"
-          @new_groonga_options = []
-
-          @recreate_database = false
-          @load_data = true
-          @run_queries = true
-          @skip_finished_queries = false
-          @output_query_log = false
-          @care_order = true
-          @verify_cachehit_mode = false
-        end
-
-        def run(command_line)
-          option_parser = create_option_parser
-          begin
-            option_parser.parse!(command_line)
-          rescue OptionParser::ParseError => error
-            $stderr.puts(error.message)
-            return false
-          end
-
-          tester = Tester.new(old_groonga_server,
-                              new_groonga_server,
-                              tester_options)
-          tester.run
-        end
-
-        private
-        def create_option_parser
-          parser = OptionParser.new
-          parser.version = VERSION
-
-          parser.separator("")
-          parser.separator("Path:")
-          parser.on("--input-directory=DIRECTORY",
-                    "Load schema and data from DIRECTORY.",
-                    "(#{@input_directory})") do |directory|
-            @input_directory = Pathname.new(directory)
-          end
-          parser.on("--working-directory=DIRECTORY",
-                    "Use DIRECTORY as working directory.",
-                    "(#{@working_directory})") do |directory|
-            @working_directory = Pathname.new(directory)
-          end
-
-          parser.separator("")
-          parser.separator("Throughput:")
-          parser.on("--n-clients=N", Integer,
-                    "Use N clients concurrently.",
-                    "(#{@n_clients})") do |n|
-            @n_clients = n
-          end
-
-          parser.separator("")
-          parser.separator("Old Groonga:")
-          parser.on("--old-groonga=GROONGA",
-                    "Old groonga command",
-                    "(#{@old_groonga})") do |groonga|
-            @old_groonga = groonga
-          end
-
-          parser.on("--old-groonga-option=OPTION",
-                    "Add an additional old groonga option",
-                    "You can specify this option multiple times to specify multiple groonga options",
-                    "(no options)") do |groonga_option|
-            @old_groonga_options << groonga_option
-          end
-
-          parser.separator("")
-          parser.separator("New Groonga:")
-          parser.on("--new-groonga=GROONGA",
-                    "New groonga command",
-                    "(#{@new_groonga})") do |groonga|
-            @new_groonga = groonga
-          end
-
-          parser.on("--new-groonga-option=OPTION",
-                    "Add an additional new groonga option",
-                    "You can specify this option multiple times to specify multiple groonga options",
-                    "(no options)") do |groonga_option|
-            @new_groonga_options << groonga_option
-          end
-
-          parser.separator("")
-          parser.separator("Operations:")
-          parser.on("--recreate-database",
-                    "Always recreate Groonga database") do
-            @recreate_database = true
-          end
-          parser.on("--no-load-data",
-                    "Don't load data. Just loads schema to Groonga database") do
-            @load_data = false
-          end
-          parser.on("--no-run-queries",
-                    "Don't run queries. Just creates Groonga database") do
-            @run_queries = false
-          end
-          parser.on("--skip-finished-queries",
-                    "Don't run finished query logs.") do
-            @skip_finished_queries = true
-          end
-          parser.on("--output-query-log",
-                    "Output query log in verified target Groonga servers") do
-            @output_query_log = true
-          end
-          parser.on("--no-care-order",
-                    "Don't care order of select response records") do
-            @care_order = false
-          end
-
-          parser
-        end
-
-        def directory_options
-          {
-            :input_directory   => @input_directory,
-            :working_directory => @working_directory,
-          }
-        end
-
-        def server_options
-          options = {
-            :load_data             => @load_data,
-            :run_queries           => @run_queries,
-            :recreate_database     => @recreate_database,
-            :output_query_log      => @output_query_log,
-          }
-          directory_options.merge(options)
-        end
-
-        def tester_options
-          options = {
-            :n_clients  => @n_clients,
-            :care_order => @care_order,
-            :skip_finished_queries => @skip_finished_queries,
-          }
-          directory_options.merge(options)
-        end
-
-        def old_groonga_server
-          GroongaServer.new(@old_groonga,
-                            @old_groonga_options,
-                            @old_database,
-                            server_options)
-        end
-
-        def new_groonga_server
-          GroongaServer.new(@new_groonga,
-                            @new_groonga_options,
-                            @new_database,
-                            server_options)
-        end
-
-        class GroongaServer
-          attr_reader :host, :port
-          def initialize(groonga, groonga_options, database_path, options)
-            @input_directory = options[:input_directory] || Pathname.new(".")
-            @working_directory = options[:working_directory] || Pathname.new(".")
-            @groonga = groonga
-            @groonga_options = groonga_options
-            @database_path = @working_directory + database_path
-            @host = "127.0.0.1"
-            @port = find_unused_port
-            @options = options
-          end
-
-          def run
-            return unless @options[:run_queries]
-
-            arguments = @groonga_options.dup
-            arguments.concat(["--bind-address", @host])
-            arguments.concat(["--port", @port.to_s])
-            arguments.concat(["--protocol", "http"])
-            arguments.concat(["--log-path", log_path.to_s])
-            if @options[:output_query_log]
-              arguments.concat(["--query-log-path", query_log_path.to_s])
-            end
-            arguments << "-s"
-            arguments << @database_path.to_s
-            @pid = spawn(@groonga, *arguments)
-
-            n_retries = 10
-            begin
-              send_command("status")
-            rescue SystemCallError
-              sleep(1)
-              n_retries -= 1
-              raise if n_retries.zero?
-              retry
-            end
-
-            yield if block_given?
-          end
-
-          def ensure_database
-            if @options[:recreate_database]
-              FileUtils.rm_rf(@database_path.dirname.to_s)
-            end
-
-            return if @database_path.exist?
-            FileUtils.mkdir_p(@database_path.dirname.to_s)
-            system(@groonga, "-n", @database_path.to_s, "quit")
-            grn_files.each do |grn_file|
-              command = [
-                @groonga,
-                "--log-path", log_path.to_s,
-                "--file", grn_file.to_s,
-                @database_path.to_s,
-              ]
-              command_line = command.join(" ")
-              puts("Running...: #{command_line}")
-              pid = spawn(*command)
-              begin
-                pid, status = Process.waitpid2(pid)
-              rescue Interrupt
-                Process.kill(:TERM, pid)
-                pid, status = Process.waitpid2(pid)
-              end
-              unless status.success?
-                raise "Failed to run: #{command_line}"
-              end
-            end
-          end
-
-          def use_persistent_cache?
-            @groonga_options.include?("--cache-base-path")
-          end
-
-          def shutdown
-            begin
-              send_command("shutdown")
-            rescue SystemCallError
-            end
-            Process.waitpid(@pid)
-          end
-
-          private
-          def find_unused_port
-            server = TCPServer.new(@host, 0)
-            begin
-              server.addr[1]
-            ensure
-              server.close
-            end
-          end
-
-          def log_path
-            @database_path.dirname + "groonga.log"
-          end
-
-          def query_log_path
-            @database_path.dirname + "query.log"
-          end
-
-          def send_command(name)
-            Net::HTTP.start(@host, @port) do |http|
-              response = http.get("/d/#{name}")
-              response.body
-            end
-          end
-
-          def grn_files
-            files = schema_files
-            files += data_files if @options[:load_data]
-            files += index_files
-            files
-          end
-
-          def schema_files
-            Pathname.glob("#{@input_directory}/schema/**/*.grn").sort
-          end
-
-          def index_files
-            Pathname.glob("#{@input_directory}/indexes/**/*.grn").sort
-          end
-
-          def data_files
-            Pathname.glob("#{@input_directory}/data/**/*.grn").sort
-          end
-        end
-
-        class Tester
-          def initialize(old, new, options)
-            @old = old
-            @new = new
-            @input_directory = options[:input_directory] || Pathname.new(".")
-            @working_directory = options[:working_directory] || Pathname.new(".")
-            @n_clients = options[:n_clients] || 1
-            @options = options
-            @n_ready_waits = 2
-          end
-
-          def run
-            @old.ensure_database
-            @new.ensure_database
-
-            old_thread = Thread.new do
-              @old.run do
-                run_test
-              end
-            end
-            new_thread = Thread.new do
-              @new.run do
-                run_test
-              end
-            end
-
-            old_thread_success = old_thread.value
-            new_thread_success = new_thread.value
-
-            old_thread_success and new_thread_success
-          end
-
-          private
-          def run_test
-            @n_ready_waits -= 1
-            return true unless @n_ready_waits.zero?
-
-            query_log_paths.each do |query_log_path|
-              log_path = test_log_path(query_log_path)
-              if @options[:skip_finished_queries] and log_path.exist?
-                puts("Skip query log: #{query_log_path}")
-                next
-              else
-                puts("Running test against query log...: #{query_log_path}")
-              end
-              begin
-                if use_persistent_cache?
-                  callback = lambda do
-                    if @old.use_persistent_cache?
-                      @old.shutdown
-                      @old.run
-                    end
-                    if @new.use_persistent_cache?
-                      @new.shutdown
-                      @new.run
-                    end
-                  end
-                else
-                  callback = nil
-                end
-                verify_server(log_path, query_log_path, &callback)
-              rescue Interrupt
-                puts("Interrupt: #{query_log_path}")
-              end
-            end
-
-            old_thread = Thread.new do
-              @old.shutdown
-            end
-            new_thread = Thread.new do
-              @new.shutdown
-            end
-            old_thread.join
-            new_thread.join
-
-            true
-          end
-
-          def verify_server(test_log_path, query_log_path, &callback)
-            command_line = [
-              "--n-clients=#{@n_clients}",
-              "--groonga1-host=#{@old.host}",
-              "--groonga1-port=#{@old.port}",
-              "--groonga1-protocol=http",
-              "--groonga2-host=#{@new.host}",
-              "--groonga2-port=#{@new.port}",
-              "--groonga2-protocol=http",
-              "--output", test_log_path.to_s,
-            ]
-            command_line << "--no-care-order" if @options[:care_order] == false
-            command_line << query_log_path.to_s
-            if use_persistent_cache?
-              command_line << "--verify-cache"
-            end
-            verify_server = VerifyServer.new
-            verify_server.run(command_line, &callback)
-          end
-
-          def query_log_paths
-            Pathname.glob("#{@input_directory}/query-logs/**/*.log").sort
-          end
-
-          def test_log_path(query_log_path)
-            @working_directory + "results" + query_log_path.basename
-          end
-
-          def use_persistent_cache?
-            @old.use_persistent_cache? or @new.use_persistent_cache?
-          end
-        end
-      end
-    end
-  end
-end
+require "groonga-query-log/command/run-regression-test"