groonga-query-log 1.2.8 → 1.2.9

data/lib/{groonga/query-log → groonga-query-log/command}/analyzer.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2016  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2018  Kouhei Sutou <kou@clear-code.com>
 # Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
@@ -17,26 +17,13 @@
 
 require "optparse"
 require "json"
-require "groonga/query-log/command-line-utils"
-require "groonga/query-log/parser"
-require "groonga/query-log/analyzer/streamer"
-require "groonga/query-log/analyzer/sized-statistics"
-require "groonga/query-log/analyzer/reporter/console"
-require "groonga/query-log/analyzer/reporter/html"
-require "groonga/query-log/analyzer/reporter/json"
-require "groonga/query-log/analyzer/reporter/json-stream"
 
-module Groonga
-  module QueryLog
-    class Analyzer
-      include CommandLineUtils
-
-      class Error < StandardError
-      end
-
-      class NoInputError < Error
-      end
+require "groonga-query-log"
+require "groonga-query-log/command-line"
 
+module GroongaQueryLog
+  module Command
+    class Analyzer < CommandLine
       class UnsupportedReporter < Error
       end
 
@@ -44,11 +31,11 @@ module Groonga
         setup_options
       end
 
-      # Executes analyzer for groonga's query logs.
+      # Executes analyzer for Groonga's query logs.
       # "groonga-query-log-analyze" command run this method.
       #
       # @example
-      #   analyzer = Groonga::QueryLog::Analyzer.new
+      #   analyzer = GroongaQueryLog::Command::Analyzer.new
       #   analyzer.run("--output", "statistics.html",
       #                "--reporter", "html",
       #                "query.log")
@@ -60,7 +47,12 @@ module Groonga
       #   groonga-query-log-analyze. Please execute
       #   "groonga-query-log-analyze --help" or see #setup_options.
       def run(arguments)
-        log_paths = @option_parser.parse!(arguments)
+        begin
+          log_paths = @option_parser.parse!(arguments)
+        rescue OptionParser::InvalidOption => error
+          $stderr.puts(error)
+          return false
+        end
 
         stream = @options[:stream]
         dynamic_sort = @options[:dynamic_sort]
@@ -68,11 +60,12 @@ module Groonga
         statistics.apply_options(@options)
         if stream
           reporter = create_reporter(statistics)
-          reporter.apply_options(@options)
           streamer = Streamer.new(reporter)
           streamer.start
           process_statistic = lambda do |statistic|
-            streamer << statistic
+            if @options[:stream_all] or statistic.slow?
+              streamer << statistic
+            end
           end
         elsif dynamic_sort
           process_statistic = lambda do |statistic|
@@ -89,6 +82,9 @@ module Groonga
           parse(log_paths, &process_statistic)
         rescue Interrupt
           raise unless stream
+        rescue Error
+          $stderr.puts($!.message)
+          return false
         end
 
         if stream
@@ -97,7 +93,6 @@ module Groonga
           statistics.replace(full_statistics) unless dynamic_sort
 
           reporter = create_reporter(statistics)
-          reporter.apply_options(@options)
           reporter.report
         end
 
@@ -118,7 +113,9 @@ module Groonga
         @options[:reporter] = "console"
         @options[:dynamic_sort] = true
         @options[:stream] = false
+        @options[:stream_all] = false
         @options[:report_summary] = true
+        @options[:report_command_line] = nil
 
         @option_parser = OptionParser.new do |parser|
           parser.version = VERSION
@@ -197,7 +194,13 @@ module Groonga
             @options[:target_tables] = tables
           end
 
-          available_reporters = ["console", "json", "json-stream", "html"]
+          available_reporters = [
+            "console",
+            "csv",
+            "html",
+            "json",
+            "json-stream",
+          ]
           parser.on("--reporter=REPORTER",
                     available_reporters,
                     "Reports statistics by REPORTER.",
@@ -220,42 +223,55 @@ module Groonga
             @options[:stream] = stream
           end
 
+          parser.on("--[no-]stream-all",
+                    "Outputs all analyzed queries in stream mode.",
+                    "NOTE: This implies --stream.",
+                    "(#{@options[:stream_all]})") do |stream_all|
+            @options[:stream] = stream_all
+            @options[:stream_all] = stream_all
+          end
+
           parser.on("--[no-]report-summary",
                     "Reports summary at the end.",
                     "(#{@options[:report_summary]})") do |report_summary|
             @options[:report_summary] = report_summary
           end
+
+          parser.on("--[no-]report-command-line",
+                    "Reports command line.",
+                    "(false for CSV reporter, true otherwise)") do |report|
+            @options[:report_command_line] = report
+          end
         end
       end
 
       def create_reporter(statistics)
         case @options[:reporter]
+        when "csv"
+          CSVReporter.new(statistics, @options)
+        when "html"
+          HTMLReporter.new(statistics, @options)
         when "json"
-          JSONReporter.new(statistics)
+          JSONReporter.new(statistics, @options)
         when "json-stream"
-          JSONStreamReporter.new(statistics)
-        when "html"
-          HTMLReporter.new(statistics)
+          JSONStreamReporter.new(statistics, @options)
         else
-          ConsoleReporter.new(statistics)
+          ConsoleReporter.new(statistics, @options)
         end
       end
 
       def parse(log_paths, &process_statistic)
-        parser = Groonga::QueryLog::Parser.new(@options)
-        if log_paths.empty?
-          unless log_via_stdin?
-            raise(NoInputError, "Error: Please specify input log files.")
-          end
-          parser.parse($stdin, &process_statistic)
-        end
-
-        log_paths.each do |log_path|
-          File.open(log_path) do |log|
-            parser.parse(log, &process_statistic)
-          end
-        end
+        parser = Parser.new(@options)
+        parse_log(parser, log_paths, &process_statistic)
       end
     end
   end
 end
+
+require "groonga-query-log/command/analyzer/streamer"
+require "groonga-query-log/command/analyzer/sized-statistics"
+require "groonga-query-log/command/analyzer/reporter/console"
+require "groonga-query-log/command/analyzer/reporter/csv"
+require "groonga-query-log/command/analyzer/reporter/html"
+require "groonga-query-log/command/analyzer/reporter/json"
+require "groonga-query-log/command/analyzer/reporter/json-stream"

data/lib/{groonga/query-log → groonga-query-log/command}/analyzer/reporter.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2011-2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2018  Kouhei Sutou <kou@clear-code.com>
 # Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
@@ -17,35 +15,32 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-module Groonga
-  module QueryLog
+require "time"
+
+module GroongaQueryLog
+  module Command
     class Analyzer
       class Reporter
         include Enumerable
 
         attr_reader :output
         attr_accessor :slow_operation_threshold, :slow_response_threshold
-        def initialize(statistics)
+        def initialize(statistics, options={})
           @statistics = statistics
-          @report_summary = true
-          @output = $stdout
+          @options = options
+          self.output = @options[:output] || $stdout
+          @report_summary = @options[:report_summary]
+          @report_summary = true if @report_summary.nil?
+          @report_command_line = @options[:report_command_line]
+          @report_command_line = true if @report_command_line.nil?
           @slow_operation_threshold =
+            @options[:slow_operation_threshold] ||
             Statistic::DEFAULT_SLOW_OPERATION_THRESHOLD
           @slow_response_threshold =
+            @options[:slow_response_threshold] ||
             Statistic::DEFAULT_SLOW_RESPONSE_THRESHOLD
         end
 
-        def apply_options(options)
-          self.output = options[:output] || @output
-          unless options[:report_summary].nil?
-            @report_summary = options[:report_summary]
-          end
-          @slow_operation_threshold =
-            options[:slow_operation_threshold] || @slow_operation_threshold
-          @slow_response_threshold =
-            options[:slow_response_threshold] || @slow_response_threshold
-        end
-
         def output=(output)
           @output = output
           @output = $stdout if @output == "-"
@@ -101,7 +96,7 @@ module Groonga
           if time.nil?
             "NaN"
           else
-            time.strftime("%Y-%m-%d %H:%M:%S.%u")
+            time.iso8601(6)
           end
         end
       end

data/lib/{groonga/query-log → groonga-query-log/command}/analyzer/reporter/console.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2011-2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2017  Kouhei Sutou <kou@clear-code.com>
 # Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
@@ -17,10 +15,10 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "groonga/query-log/analyzer/reporter"
+require "groonga-query-log/command/analyzer/reporter"
 
-module Groonga
-  module QueryLog
+module GroongaQueryLog
+  module Command
     class Analyzer
       class ConsoleReporter < Reporter
         class Color
@@ -124,9 +122,10 @@ module Groonga
           end
         end
 
-        def initialize(statistics)
+        def initialize(statistics, options)
           super
-          @color = :auto
+          @color = @options[:color]
+          @color = :auto if @color.nil?
           @reset_color = Color.new("reset")
           @color_schema = {
             :elapsed => {:foreground => :white, :background => :green},
@@ -135,11 +134,6 @@ module Groonga
           }
         end
 
-        def apply_options(options)
-          super
-          @color = options[:color] unless options[:color].nil?
-        end
-
         def report_statistics
           write("\n")
           write("Slow Queries:\n")
@@ -270,11 +264,18 @@ module Groonga
         def format_heading(statistic)
           formatted_elapsed = colorize("%8.8f" % statistic.elapsed_in_seconds,
                                        :elapsed)
-          "[%s-%s (%s)](%d): %s" % [format_time(statistic.start_time),
-                                    format_time(statistic.last_time),
-                                    formatted_elapsed,
-                                    statistic.return_code,
-                                    statistic.raw_command]
+          data = [
+            format_time(statistic.start_time),
+            format_time(statistic.last_time),
+            formatted_elapsed,
+            statistic.return_code,
+          ]
+          if @report_command_line
+            data << statistic.raw_command
+          else
+            data << statistic.command.name
+          end
+          "[%s-%s (%s)](%d): %s" % data
         end
 
         def format_time(time)

data/lib/groonga-query-log/command/analyzer/reporter/csv.rb

@@ -0,0 +1,77 @@
+# Copyright (C) 2018  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "csv"
+
+require "groonga-query-log/command/analyzer/reporter"
+
+module GroongaQueryLog
+  module Command
+    class Analyzer
+      class CSVReporter < Reporter
+        def initialize(statistics, options)
+          super
+          if @options[:report_command_line].nil?
+            @report_command_line = false
+          end
+        end
+
+        def start
+          @csv = CSV.new(@output)
+          header = [
+            "start_time",
+            "last_time",
+            "elapsed",
+            "return_code",
+            "slow",
+            "command_name",
+          ]
+          header << "command_line" if @report_command_line
+          @csv << header
+        end
+
+        def report_statistic(statistic)
+          record = [
+            format_time(statistic.start_time),
+            format_time(statistic.last_time),
+            statistic.elapsed_in_seconds,
+            statistic.return_code,
+            statistic.slow?,
+            statistic.command.command_name,
+          ]
+          record << statistic.raw_command if @report_command_line
+          @csv << record
+        end
+
+        def finish
+          @csv.close
+        end
+
+        def report_summary
+        end
+
+        private
+        def format_time(time)
+          if time.nil?
+            ""
+          else
+            super
+          end
+        end
+      end
+    end
+  end
+end

data/lib/{groonga/query-log → groonga-query-log/command}/analyzer/reporter/html.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2011-2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2018  Kouhei Sutou <kou@clear-code.com>
 # Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
@@ -18,10 +16,10 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "erb"
-require "groonga/query-log/analyzer/reporter"
+require "groonga-query-log/command/analyzer/reporter"
 
-module Groonga
-  module QueryLog
+module GroongaQueryLog
+  module Command
     class Analyzer
       class HTMLReporter < Reporter
         include ERB::Util
@@ -66,7 +64,9 @@ module Groonga
                                :slow? => statistic.slow?) %>)]
           (<%= span({:class => "return-code"}, h(statistic.return_code)) %>)
         </div>
+<% if @report_command_line %>
         <%= div({:class => "raw-command"}, h(statistic.raw_command)) %>
+<% end %>
       </div>
       <div class="statistic-parameters">
         <h3>Parameters</h3>
@@ -81,16 +81,31 @@ module Groonga
       </div>
       <div class="statistic-operations">
         <h3>Operations</h3>
-        <ol>
-<% statistic.each_operation do |operation| %>
-          <li>
-            <%= format_elapsed(operation[:relative_elapsed_in_seconds],
-                               :slow? => operation[:slow?]) %>:
-            <%= span({:class => "name"}, h(operation[:name])) %>:
-            <%= span({:class => "context"}, h(operation[:context])) %>
-          </li>
+        <table>
+          <thead>
+            <tr>
+              <th>Nth</th>
+              <th>Elapsed(sec)</th>
+              <th>Name</th>
+              <th>N records</th>
+              <th>Context</th>
+            </tr>
+          </thead>
+          <tbody>
+<% statistic.each_operation.with_index do |operation, i| %>
+            <tr>
+              <td class="n"><%= h(i + 1) %></td>
+              <td class="elapsed">
+                <%= format_elapsed(operation[:relative_elapsed_in_seconds],
+                                   :slow? => operation[:slow?]) %>
+              </td>
+              <td class="name"><%= h(operation[:name]) %></td>
+              <td class="n-records"><%= h(operation[:n_records]) %></td>
+              <td class="context"><%= h(operation[:context]) %></td>
+            </tr>
 <% end %>
-        </ol>
+          </tbody>
+        </table>
       </div>
           EOH
           write(statistic_html)
@@ -145,7 +160,8 @@ div.statistics
 
 td.elapsed,
 td.ratio,
-td.n
+td.n,
+td.n-records
 {
   text-align: right;
 }