groonga-query-log 1.2.8 → 1.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 936e5bb7d220b45ad689535247237ae74e608af5
-  data.tar.gz: 3700a2ab52b0b28ae308e3ab8edf2629143cb499
+  metadata.gz: 213e0534e4b93c131e48f76cfe2c6f8c7fea4a44
+  data.tar.gz: 326a76731faf0f948a8362800748503036dca728
 SHA512:
-  metadata.gz: 93f1cfd0210554b770fe7d230994939f7dccbbbc2d5ee5dcedaaf06ad1ada4dee88f96f4269c35d0fb63eafdafd5905e14570b15c2124433f40113dc2b621d32
-  data.tar.gz: '09d0648d52be723c0acda34cf01ae02b1dcb1105ac02e828a1cbad8a53c178b81238eb1535de5e3816c56824881396994bcdc1619466226f1701a2770af6ad58'
+  metadata.gz: 55fd3f7d8ed2fa51f94ed333e8afeaaef19ad5230b2a4ab8c0ce9622cde117891e3c26bda0f78a0560dcb0382a3b09c843c09cc474cb62faf3b3607a59d51b44
+  data.tar.gz: eff099b4e055fc9e6e332a4d8bd8ae1987596ddab11fc842e83d965603147b68270395102cc9860cc09fb13e6004ebaf2a2bf8a949552bb9712e16552ff7bc5f

data/bin/groonga-query-log-analyze

@@ -1,8 +1,7 @@
 #!/usr/bin/env ruby
-# -*- coding: utf-8 -*-
 #
 # Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
-# Copyright (C) 2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2014-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -18,7 +17,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "groonga/query-log"
+require "groonga-query-log/command/analyzer"
 
-analyzer = Groonga::QueryLog::Analyzer.new
+analyzer = GroongaQueryLog::Command::Analyzer.new
 exit(analyzer.run(ARGV))

data/bin/groonga-query-log-analyze-load

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2017-2018  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga-query-log/command/analyze-load"
+
+analyze_load = GroongaQueryLog::Command::AnalyzeLoad.new
+exit(analyze_load.run(ARGV))

data/bin/groonga-query-log-check-command-version-compatibility

@@ -16,7 +16,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "groonga/query-log/command/check-command-version-compatibility"
+require "groonga-query-log/command/check-command-version-compatibility"
 
-command = Groonga::QueryLog::Command::CheckCommandVersionCompatibility.new
+command = GroongaQueryLog::Command::CheckCommandVersionCompatibility.new
 exit(command.run(ARGV))

data/bin/groonga-query-log-check-crash

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2018  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga-query-log/command/check-crash"
+
+check_crash = GroongaQueryLog::Command::CheckCrash.new
+exit(check_crash.run(ARGV))

data/bin/groonga-query-log-detect-memory-leak

@@ -17,7 +17,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "groonga/query-log/command/detect-memory-leak"
+require "groonga-query-log/command/detect-memory-leak"
 
-detect_memory_leak_command = Groonga::QueryLog::Command::DetectMemoryLeak.new
+detect_memory_leak_command = GroongaQueryLog::Command::DetectMemoryLeak.new
 exit(detect_memory_leak_command.run(ARGV))

data/bin/groonga-query-log-extract

@@ -17,7 +17,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "groonga/query-log/command/extract"
+require "groonga-query-log/command/extract"
 
-extract = Groonga::QueryLog::Command::Extract.new
+extract = GroongaQueryLog::Command::Extract.new
 exit(extract.run(ARGV))

data/bin/groonga-query-log-format-regression-test-logs

@@ -16,7 +16,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "groonga/query-log/command/format-regression-test-logs"
+require "groonga-query-log/command/format-regression-test-logs"
 
-command = Groonga::QueryLog::Command::FormatRegressionTestLogs.new
+command = GroongaQueryLog::Command::FormatRegressionTestLogs.new
 exit(command.run(ARGV))

data/bin/groonga-query-log-replay

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013-2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -17,9 +17,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "groonga/query-log/command/replay"
+require "groonga-query-log/command/replay"
 
-Thread.abort_on_exception = true
-
-replay_command = Groonga::QueryLog::Command::Replay.new
+replay_command = GroongaQueryLog::Command::Replay.new
 exit(replay_command.run(ARGV))

data/bin/groonga-query-log-run-regression-test

@@ -17,7 +17,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "groonga/query-log/command/run-regression-test"
+require "groonga-query-log/command/run-regression-test"
 
-command = Groonga::QueryLog::Command::RunRegressionTest.new
+command = GroongaQueryLog::Command::RunRegressionTest.new
 exit(command.run(ARGV))

data/bin/groonga-query-log-show-running-queries

@@ -17,7 +17,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "groonga/query-log/command/show-running-queries"
+require "groonga-query-log/command/show-running-queries"
 
-command = Groonga::QueryLog::Command::ShowRunningQueries.new
+command = GroongaQueryLog::Command::ShowRunningQueries.new
 exit(command.run(ARGV))

data/bin/groonga-query-log-verify-server

@@ -17,7 +17,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "groonga/query-log/command/verify-server"
+require "groonga-query-log/command/verify-server"
 
-verify_server_command = Groonga::QueryLog::Command::VerifyServer.new
+verify_server_command = GroongaQueryLog::Command::VerifyServer.new
 exit(verify_server_command.run(ARGV))

data/doc/text/news.md

@@ -1,5 +1,36 @@
 # News
 
+## 1.2.9: 2018-02-04
+
+### Improvements
+
+  * `GroongaQueryLog`: Renamed from
+    `Groonga::QueryLog`. `Groonga::QueryLog` is deprecated but still
+    usable.
+
+  * `GroongaQueryLog::Analyzer::Statistic#each_operation`: Added
+    enumerator support.
+
+  * `groonga-query-log-analyze`: Added "N records" support in HTML report.
+
+  * `groonga-query-log-replay`: Changed the default protocol to HTTP
+    from GQTP.
+
+  * `groonga-query-log-analyze`: Added `--stream-all` option.
+
+  * `GroongaQueryLog::Parser`: Added extra information support in
+    `load` and `delete` commands' query log.
+
+  * `groonga-query-log-analyze-load`: Added.
+
+  * `groonga-query-log-analyze`: Added CSV reporter.
+
+  * `groonga-query-log-check-crash`: Added.
+
+  * `GroongaQueryLog::Parser#current_path`: Added.
+
+  * Required groonga-log 0.1.2 or later.
+
 ## 1.2.8: 2017-10-26
 
 ### Fixes

data/groonga-query-log.gemspec

@@ -1,6 +1,6 @@
-# -*- mode: ruby; coding: utf-8 -*-
+# -*- ruby -*-
 #
-# Copyright (C) 2012-2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2018  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -20,7 +20,7 @@ base_dir = File.dirname(__FILE__)
 lib_dir = File.join(base_dir, "lib")
 
 $LOAD_PATH.unshift(lib_dir)
-require "groonga/query-log/version"
+require "groonga-query-log/version"
 
 clean_white_space = lambda do |entry|
   entry.gsub(/(\A\n+|\n+\z)/, '') + "\n"
@@ -28,7 +28,7 @@ end
 
 Gem::Specification.new do |spec|
   spec.name = "groonga-query-log"
-  spec.version = Groonga::QueryLog::VERSION.dup
+  spec.version = GroongaQueryLog::VERSION.dup
 
   spec.authors = ["Kouhei Sutou"]
   spec.email = ["kou@clear-code.com"]
@@ -54,6 +54,7 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency("groonga-command-parser")
   spec.add_runtime_dependency("groonga-client")
+  spec.add_runtime_dependency("groonga-log", ">= 0.1.2")
 
   spec.add_development_dependency("test-unit")
   spec.add_development_dependency("test-unit-notify")

data/lib/groonga-query-log.rb

@@ -0,0 +1,42 @@
+# Copyright (C) 2012-2018  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga-log"
+
+require "groonga-query-log/version"
+require "groonga-query-log/parser"
+require "groonga-query-log/replayer"
+require "groonga-query-log/server-verifier"
+require "groonga-query-log/command-version-compatibility-checker"
+
+module GroongaQueryLog
+  module AnalyzerNamespaceBackwardCompatibility
+    def const_missing(name)
+      case name
+      when :Analyzer
+        warn("GroongaQueryLog::Analyzer is deprecated. " +
+             "Use GroongaQueryLog::Command::Analyzer instead:\n" +
+             caller.join("\n"))
+        require "groonga-query-log/command/analyzer"
+        const_set(name, Command::Analyzer)
+      else
+        super
+      end
+    end
+  end
+
+  extend AnalyzerNamespaceBackwardCompatibility
+end

data/lib/{groonga/query-log/command-line-utils.rb → groonga-query-log/command-line.rb}

@@ -1,6 +1,5 @@
-# -*- coding: utf-8 -*-
-#
 # Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
+# Copyright (C) 2017-2018  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,20 +14,33 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-module Groonga
-  module QueryLog
-    module CommandLineUtils
-      def log_via_stdin?
-        stdin_with_pipe? or stdin_with_redirect?
-      end
+module GroongaQueryLog
+  class CommandLine
+    class Error < StandardError
+    end
 
-      def stdin_with_pipe?
-        File.pipe?($stdin)
-      end
+    class NoInputError < Error
+    end
 
-      def stdin_with_redirect?
-        not File.select([$stdin], [], [], 0).nil?
+    private
+    def parse_log(parser, log_paths, &process_statistic)
+      if log_paths.empty?
+        if stdin_with_pipe? or stdin_with_redirect?
+          parser.parse($stdin, &process_statistic)
+        else
+          raise NoInputError, "Error: Please specify input log files."
+        end
+      else
+        parser.parse_paths(log_paths, &process_statistic)
       end
     end
+
+    def stdin_with_pipe?
+      File.pipe?($stdin)
+    end
+
+    def stdin_with_redirect?
+      not File.select([$stdin], [], [], 0).nil?
+    end
   end
 end

data/lib/{groonga/query-log → groonga-query-log}/command-version-compatibility-checker.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2014-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -14,10 +14,9 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "groonga/query-log/incompatibility-detector"
+require "groonga-query-log/incompatibility-detector"
 
-module Groonga
-  module QueryLog
+module GroongaQueryLog
     class CommandVersionCompatibilityChecker
       def initialize(options)
         @options = options
@@ -96,5 +95,4 @@ module Groonga
         end
       end
     end
-  end
 end

data/lib/groonga-query-log/command/analyze-load.rb

@@ -0,0 +1,188 @@
+# Copyright (C) 2017-2018  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "optparse"
+
+require "groonga-query-log"
+require "groonga-query-log/command-line"
+
+module GroongaQueryLog
+  module Command
+    class AnalyzeLoad < CommandLine
+      def initialize
+        setup_options
+        @pending_entry = nil
+      end
+
+      # Executes load command analyzer for Groonga's query logs.
+      # "groonga-query-log-analyze-load" command run this method.
+      #
+      # @example
+      #   analyze_load = GroongaQueryLog::Command::AnalyzeLoad.new
+      #   analyze_load.run("--output", "statistics.csv",
+      #                    "query.log")
+      #
+      # If only paths of query log files are specified,
+      # this method prints a result of them to console.
+      #
+      # @param [Array<String>] arguments arguments for
+      #   groonga-query-log-analyze-load. Please execute
+      #   `groonga-query-log-analyze-load --help` or see
+      #   #setup_options.
+      def run(arguments)
+        begin
+          log_paths = @option_parser.parse!(arguments)
+        rescue OptionParser::InvalidOption => error
+          $stderr.puts(error)
+          return false
+        end
+
+        begin
+          open_output do |output|
+            report_header(output)
+            parse(log_paths) do |statistic|
+              report_statistic(output, statistic)
+            end
+            if @pending_entry
+              report_entry(output, @pending_entry)
+              @pending_entry = nil
+            end
+          end
+        rescue Interrupt
+        rescue Error
+          $stderr.puts($!.message)
+          return false
+        end
+
+        true
+      end
+
+      private
+      def setup_options
+        @options = {}
+        @options[:output] = "-"
+        @options[:target_commands] = ["select", "load"]
+
+        @option_parser = OptionParser.new do |parser|
+          parser.version = VERSION
+          parser.banner += " LOG1 ..."
+
+          parser.on("--output=PATH",
+                    "Output to PATH.",
+                    "'-' PATH means standard output.",
+                    "(#{@options[:output]})") do |output|
+            @options[:output] = output
+          end
+        end
+      end
+
+      def open_output
+        if @options[:output] == "-"
+          yield($stdout)
+        else
+          File.open(@options[:output], "w") do |output|
+            yield(output)
+          end
+        end
+      end
+
+      def parse(log_paths, &process_statistic)
+        parser = Parser.new(@options)
+        parse_log(parser, log_paths, &process_statistic)
+      end
+
+      def report_statistic(output, statistic)
+        command = statistic.command
+        if command.name == "select"
+          process_select_statistic(output, statistic, command)
+        else
+          process_load_statistic(output, statistic, command)
+        end
+      end
+
+      def process_select_statistic(output, statistic, select_command)
+        return if @pending_entry.nil?
+
+        operations = statistic.operations
+        if operations.any? {|operation| operation[:name] == "filter"}
+          return
+        end
+        select_operation = operations.find do |operation|
+          operation[:name] == "select"
+        end
+        return if select_operation.nil?
+
+        return if @pending_entry[2] != select_command[:table]
+
+        @pending_entry[6] = select_operation[:n_records]
+        report_entry(output, @pending_entry)
+        @pending_entry = nil
+      end
+
+      def process_load_statistic(output, statistic, load_command)
+        operation = statistic.operations.first
+        if operation and operation[:extra]
+          extra = operation[:extra]
+          extra_counts = extra.scan(/\[(\d+)\]/).flatten.collect(&:to_i)
+          n_loaded_records = operation[:n_records]
+          n_record_errors = extra_counts[0]
+          n_column_errors = extra_counts[1]
+          total = extra_counts[2]
+        else
+          n_loaded_records = nil
+          n_record_errors = nil
+          n_column_errors = nil
+          total = nil
+        end
+        entry = [
+          statistic.start_time.iso8601,
+          statistic.elapsed_in_seconds,
+          load_command.table,
+          n_loaded_records,
+          n_record_errors,
+          n_column_errors,
+          total,
+        ]
+        if @pending_entry
+          report_entry(output, @pending_entry)
+          @pending_entry = nil
+        end
+        if total.nil?
+          @pending_entry = entry
+        else
+          report_entry(output, entry)
+        end
+      end
+
+      def report_header(output)
+        header = [
+          "start_time",
+          "elapsed",
+          "table",
+          "n_loaded_records",
+          "n_record_errors",
+          "n_column_errors",
+          "n_total_records",
+        ]
+        output.puts(header.join(","))
+      end
+
+      def report_entry(output, entry)
+        output.puts(entry.join(","))
+      end
+    end
+  end
+end