groonga-query-log 1.5.2 → 1.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7c4745d088436d712bbc9d1b010c0219008baac00d05f5c5ebb59423912353ab
-  data.tar.gz: 3d663956a09c081278a1f6a06534dfcd08e7a46c5ee270d06469be274817f8d7
+  metadata.gz: 5f9718a90b49d69a599b168a7ab18cece39ef1849355ef0c9c511a34c54ef636
+  data.tar.gz: bf2c8374333e8d70455d64aa71eb743d2e900410e38c9b04a4acf7ec96380763
 SHA512:
-  metadata.gz: 705cfcf529854898bd27884a714ac123776f8e560e088ebacc5860049061de29c3454bcfafd45ab3655d812492659ced950d9fa66727d831fb4d7b3961cf779c
-  data.tar.gz: eccf8a59cb0f6faba614bf9e4beae35228c4b3634a04e3d2a1af0fa50d7da60750e53a6ef891a3295c301bb47301c38685ece802b001b5a2e12ef4df1c384916
+  metadata.gz: e3ab37905b01d41b982c75e74a0e585b46f81d101b690c22b92ccc4fd8098deb889c45e1cc31998ece5c7a69b587f8852e226c84a8c0ffc822e5ae09c503f636
+  data.tar.gz: e85db7c29f331f0b2673beebbfda49a5aae5096630b1524140e7751256140b3293faa6cc630fb8e6c0acc2fe6c50a81eb2aea664be5985e533523692dd9f84a6

data/doc/text/news.md

@@ -1,5 +1,18 @@
 # News
 
+## 1.5.3: 2019-10-21
+
+### Improvements
+
+  * `GroongaQueryLog::Statistic#end_time`: Renamed from
+    `#last_time`. `#last_time` is deprecated.
+
+  * `analyzer`: Added support for reporting worker idle time.
+
+  * `analyzer`: Added support for reporting the number of processed
+    requests per worker.
+
+  * `run-regression-test`: Added `--verify-performance` option.
 
 ## 1.5.2: 2019-09-26
 

data/groonga-query-log.gemspec

@@ -1,6 +1,6 @@
 # -*- ruby -*-
 #
-# Copyright (C) 2012-2018  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2019  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -52,6 +52,7 @@ Gem::Specification.new do |spec|
   spec.licenses = ["LGPLv2.1+"]
   spec.require_paths = ["lib"]
 
+  spec.add_runtime_dependency("charty")
   spec.add_runtime_dependency("diff-lcs")
   spec.add_runtime_dependency("groonga-client", ">= 0.6.2")
   spec.add_runtime_dependency("groonga-log", ">= 0.1.2")

data/lib/groonga-query-log/command/analyzer/reporter/console.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2019  Sutou Kouhei <kou@clear-code.com>
 # Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
@@ -177,11 +177,21 @@ module GroongaQueryLog
           write("  # of slow responses : #{@statistics.n_slow_responses}\n")
           write("  responses/sec       : #{@statistics.responses_per_second}\n")
           write("  start time          : #{format_time(@statistics.start_time)}\n")
-          write("  last time           : #{format_time(@statistics.last_time)}\n")
-          write("  period(sec)         : #{@statistics.period}\n")
+          write("    end time          : #{format_time(@statistics.end_time)}\n")
+          write("  period(sec)         : %.3f\n" % @statistics.period)
           slow_response_ratio = @statistics.slow_response_ratio
           write("  slow response ratio : %5.3f%%\n" % slow_response_ratio)
-          write("  total response time : #{@statistics.total_elapsed}\n")
+          write("  total response time : %.3f\n" % @statistics.total_elapsed)
+          write("  Workers:\n")
+          @statistics.each_worker do |worker|
+            write("    #{worker.id}:\n")
+            write("      # of processed requests: #{worker.n_statistics}\n")
+            write("      idle time(sec):\n")
+            write("                total : %.3f\n" % worker.idle_time_total)
+            write("                mean  : %.3f\n" % worker.idle_time_mean)
+            write("                min   : %.3f\n" % worker.idle_time_min)
+            write("                max   : %.3f\n" % worker.idle_time_max)
+          end
           report_slow_operations
         end
 
@@ -266,7 +276,7 @@ module GroongaQueryLog
                                        :elapsed)
           data = [
             format_time(statistic.start_time),
-            format_time(statistic.last_time),
+            format_time(statistic.end_time),
             formatted_elapsed,
             statistic.return_code,
           ]

data/lib/groonga-query-log/command/analyzer/reporter/csv.rb

@@ -33,7 +33,7 @@ module GroongaQueryLog
           @csv = CSV.new(@output)
           header = [
             "start_time",
-            "last_time",
+            "end_time",
             "elapsed",
             "return_code",
             "slow",
@@ -46,7 +46,7 @@ module GroongaQueryLog
         def report_statistic(statistic)
           record = [
             format_time(statistic.start_time),
-            format_time(statistic.last_time),
+            format_time(statistic.end_time),
             statistic.elapsed_in_seconds,
             statistic.return_code,
             statistic.slow?,

data/lib/groonga-query-log/command/analyzer/reporter/html.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2018  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2019  Sutou Kouhei <kou@clear-code.com>
 # Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
@@ -16,6 +16,9 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "erb"
+
+require "charty"
+
 require "groonga-query-log/command/analyzer/reporter"
 
 module GroongaQueryLog
@@ -24,6 +27,34 @@ module GroongaQueryLog
       class HTMLReporter < Reporter
         include ERB::Util
 
+        private
+        def setup_output
+          @directory_output_mode = false
+          return super unless directory_output?
+
+          @directory_output_mode = true
+          original_output = @output
+          begin
+            FileUtils.mkdir_p(@output)
+            File.open(File.join(@output, "index.html"), "w") do |output|
+              @output = output
+              yield(@output)
+            end
+          ensure
+            @output = original_output
+          end
+        end
+
+        def directory_output?
+          return false unless @output.is_a?(String)
+
+          return true if File.directory?(@output)
+          return true if File.extname(@output).empty?
+          return true if @output.end_with?("/")
+
+          false
+        end
+
         def start
           write(header)
         end
@@ -38,6 +69,7 @@ module GroongaQueryLog
     <div class="summary">
 <%= analyze_parameters %>
 <%= metrics %>
+<%= workers %>
 <%= slow_operations %>
     </div>
           EOH
@@ -56,10 +88,10 @@ module GroongaQueryLog
           statistic_html = erb(<<-EOH, __LINE__ + 1, binding)
       <div class="statistic-heading">
         <h3>Command</h3>
-        <div class="metrics">
+        <div class="statistic-metrics">
           [<%= format_time(statistic.start_time) %>
            -
-           <%= format_time(statistic.last_time) %>
+           <%= format_time(statistic.end_time) %>
            (<%= format_elapsed(statistic.elapsed_in_seconds,
                                :slow? => statistic.slow?) %>)]
           (<%= span({:class => "return-code"}, h(statistic.return_code)) %>)
@@ -111,7 +143,6 @@ module GroongaQueryLog
           write(statistic_html)
         end
 
-        private
         def erb(content, line, _binding=nil)
           _erb = ERB.new(content, nil, "<>")
           eval(_erb.src, _binding || binding, __FILE__, line)
@@ -119,8 +150,10 @@ module GroongaQueryLog
 
         def header
           erb(<<-EOH, __LINE__ + 1)
+<!DOCTYPE html>
 <html>
   <head>
+    <meta charset="utf-8">
     <title>groonga query analyzer</title>
     <style>
 table,
@@ -136,18 +169,24 @@ span.slow
   color: red;
 }
 
-div.parameters
+div.parameters,
+div.metrics,
+div.workers
 {
   float: left;
   padding: 2em;
 }
 
-div.parameters h3
+div.parameters h3,
+div.metrics h3,
+div.workers h3
 {
   text-align: center;
 }
 
-div.parameters table
+div.parameters table,
+div.metrics table,
+div.workers table
 {
   margin-right: auto;
   margin-left: auto;
@@ -218,7 +257,7 @@ td.name
 
         def metrics
           erb(<<-EOH, __LINE__ + 1)
-      <div class="parameters">
+      <div class="metrics">
         <h3>Metrics</h3>
         <table>
           <tr><th>Name</th><th>Value</th></tr>
@@ -239,8 +278,8 @@ td.name
             <td><%= format_time(@statistics.start_time) %></td>
           </tr>
           <tr>
-            <th>last time</th>
-            <td><%= format_time(@statistics.last_time) %></td>
+            <th>end time</th>
+            <td><%= format_time(@statistics.end_time) %></td>
           </tr>
           <tr>
             <th>period</th>
@@ -259,6 +298,62 @@ td.name
           EOH
         end
 
+        def workers
+          html = ""
+          html << erb(<<-WORKERS, __LINE__ + 1)
+      <div class="workers">
+        <h3>Workers</h3>
+        <table>
+          <tr>
+            <th>ID</th>
+            <th># of processed requests</th>
+            <th>idle time total</th>
+            <th>idle time mean</th>
+            <th>idle time min</th>
+            <th>idle time max</th>
+          </tr>
+<% @statistics.each_worker do |worker| %>
+          <tr>
+            <td><%= h(worker.id) %></td>
+            <td><%= h(worker.n_statistics) %></td>
+            <td><%= h("%.3f" % worker.idle_time_total) %></td>
+            <td><%= h("%.3f" % worker.idle_time_mean) %></td>
+            <td><%= h("%.3f" % worker.idle_time_min) %></td>
+            <td><%= h("%.3f" % worker.idle_time_max) %></td>
+          </tr>
+<% end %>
+        </table>
+          WORKERS
+          use_charty = false # TODO
+          if use_charty
+            plotter = Charty::Plotter.new(:google_charts)
+            @statistics.each_worker do |worker|
+              figure = plotter.curve do
+                metrics = worker.metrics
+                series(metrics[:timestamp],
+                       metrics[:idle_time],
+                       label: worker.id)
+                xlabel("timestamp")
+                ylabel("idle time")
+              end
+              html << figure.render
+              figure = plotter.curve do
+                metrics = worker.metrics
+                series(metrics[:timestamp],
+                       metrics[:elapsed],
+                       label: worker.id)
+                xlabel("timestamp")
+                ylabel("elapsed")
+              end
+              html << figure.render
+            end
+          end
+          html << erb(<<-WORKERS, __LINE__ + 1)
+      </div>
+          WORKERS
+          html
+        end
+
         def slow_operations
           erb(<<-EOH, __LINE__ + 1)
       <div class="statistics">

data/lib/groonga-query-log/command/analyzer/sized-statistics.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2019  Sutou Kouhei <kou@clear-code.com>
 # Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
@@ -16,6 +16,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "groonga-query-log/command/analyzer/sized-grouped-operations"
+require "groonga-query-log/command/analyzer/worker-statistic"
 
 module GroongaQueryLog
   module Command
@@ -23,18 +24,19 @@ module GroongaQueryLog
       class SizedStatistics < Array
         attr_reader :n_responses, :n_slow_responses, :n_slow_operations
         attr_reader :slow_operations, :total_elapsed
-        attr_reader :start_time, :last_time
+        attr_reader :start_time, :end_time
         def initialize
           @max_size = 10
           self.order = "-elapsed"
           @start_time = nil
-          @last_time = nil
+          @end_time = nil
           @n_responses = 0
           @n_slow_responses = 0
           @n_slow_operations = 0
           @slow_operations = SizedGroupedOperations.new
           @total_elapsed = 0
           @collect_slow_statistics = true
+          @workers = {}
         end
 
         def order=(new_order)
@@ -92,8 +94,8 @@ module GroongaQueryLog
         end
 
         def period
-          if @start_time and @last_time
-            @last_time - @start_time
+          if @start_time and @end_time
+            @end_time - @start_time
           else
             0
           end
@@ -111,6 +113,10 @@ module GroongaQueryLog
           end
         end
 
+        def each_worker(&block)
+          @workers.each_value(&block)
+        end
+
         private
         def create_sorter
           case @order
@@ -134,10 +140,11 @@ module GroongaQueryLog
         end
 
         def update_statistic(statistic)
+          update_worker_statistic(statistic)
           @start_time ||= statistic.start_time
           @start_time = [@start_time, statistic.start_time].min
-          @last_time ||= statistic.last_time
-          @last_time = [@last_time, statistic.last_time].max
+          @end_time ||= statistic.end_time
+          @end_time = [@end_time, statistic.end_time].max
           @n_responses += 1
           @total_elapsed += statistic.elapsed_in_seconds
           return unless @collect_slow_statistics
@@ -152,6 +159,12 @@ module GroongaQueryLog
             end
           end
         end
+
+        def update_worker_statistic(statistic)
+          id = statistic.context_id
+          @workers[id] ||= WorkerStatistic.new(id)
+          @workers[id] << statistic
+        end
       end
     end
   end

data/lib/groonga-query-log/command/analyzer/worker-statistic.rb

@@ -0,0 +1,64 @@
+# Copyright (C) 2019  Sutou Kouhei <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module GroongaQueryLog
+  module Command
+    class Analyzer
+      class WorkerStatistic
+        attr_reader :id
+        attr_reader :idle_time_total
+        attr_reader :idle_time_mean
+        attr_reader :idle_time_min
+        attr_reader :idle_time_max
+        attr_reader :n_statistics
+        attr_reader :metrics
+        def initialize(id)
+          @id = id
+          @idle_time_total = 0.0
+          @idle_time_mean = 0.0
+          @idle_time_min = 0.0
+          @idle_time_max = 0.0
+          @n_statistics = 0
+          @metrics = {
+            timestamp: [],
+            idle_time: [],
+            elapsed: [],
+          }
+          @previous_statistic = nil
+        end
+
+        def <<(statistic)
+          @n_statistics += 1
+          if @previous_statistic
+            idle_time = statistic.start_time - @previous_statistic.end_time
+            @idle_time_total += idle_time
+            @idle_time_mean += ((idle_time - @idle_time_mean) / @n_statistics)
+            if @idle_time_min.zero?
+              @idle_time_min = idle_time
+            else
+              @idle_time_min = [@idle_time_min, idle_time].min
+            end
+            @idle_time_max = [@idle_time_max, idle_time].max
+            @metrics[:timestamp] << statistic.start_time
+            @metrics[:idle_time] << idle_time
+            @metrics[:elapsed] << statistic.elapsed_in_seconds
+          end
+          @previous_statistic = statistic
+        end
+      end
+    end
+  end
+end