groonga-query-log 1.0.0

data/lib/groonga/query-log/analyzer/sized-statistics.rb

@@ -0,0 +1,172 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga/query-log/analyzer/sized-grouped-operations"
+require "groonga/query-log/analyzer/reporter/console"
+require "groonga/query-log/analyzer/reporter/html"
+require "groonga/query-log/analyzer/reporter/json"
+
+module Groonga
+  module QueryLog
+    class Analyzer
+      class SizedStatistics < Array
+        attr_reader :n_responses, :n_slow_responses, :n_slow_operations
+        attr_reader :slow_operations, :total_elapsed
+        attr_reader :start_time, :last_time
+        attr_accessor :slow_operation_threshold, :slow_response_threshold
+        def initialize
+          @max_size = 10
+          self.order = "-elapsed"
+          @slow_operation_threshold = 0.1
+          @slow_response_threshold = 0.2
+          @start_time = nil
+          @last_time = nil
+          @n_responses = 0
+          @n_slow_responses = 0
+          @n_slow_operations = 0
+          @slow_operations = SizedGroupedOperations.new
+          @total_elapsed = 0
+          @collect_slow_statistics = true
+        end
+
+        def order=(new_order)
+          @order = new_order
+          @sorter = create_sorter
+        end
+
+        def apply_options(options)
+          @max_size = options[:n_entries] || @max_size
+          self.order = options[:order] || @order
+          @slow_operation_threshold =
+            options[:slow_operation_threshold] || @slow_operation_threshold
+          @slow_response_threshold =
+            options[:slow_response_threshold] || @slow_response_threshold
+          unless options[:report_summary].nil?
+            @collect_slow_statistics = options[:report_summary]
+          end
+          @slow_operations.apply_options(options)
+        end
+
+        def <<(statistic)
+          update_statistic(statistic)
+          if size < @max_size
+            super(statistic)
+            replace(self)
+          else
+            if @sorter.call(statistic) < @sorter.call(last)
+              super(statistic)
+              replace(self)
+            end
+          end
+          self
+        end
+
+        def replace(other)
+          sorted_other = other.sort_by(&@sorter)
+          if sorted_other.size > @max_size
+            super(sorted_other[0, @max_size])
+          else
+            super(sorted_other)
+          end
+        end
+
+        def responses_per_second
+          _period = period
+          if _period.zero?
+            0
+          else
+            @n_responses.to_f / _period
+          end
+        end
+
+        def slow_response_ratio
+          if @n_responses.zero?
+            0
+          else
+            (@n_slow_responses.to_f / @n_responses) * 100
+          end
+        end
+
+        def period
+          if @start_time and @last_time
+            @last_time - @start_time
+          else
+            0
+          end
+        end
+
+        def each_slow_operation
+          @slow_operations.each do |grouped_operation|
+            total_elapsed = grouped_operation[:total_elapsed]
+            n_operations = grouped_operation[:n_operations]
+            ratios = {
+              :total_elapsed_ratio => total_elapsed / @total_elapsed * 100,
+              :n_operations_ratio => n_operations / @n_slow_operations.to_f * 100,
+            }
+            yield(grouped_operation.merge(ratios))
+          end
+        end
+
+        private
+        def create_sorter
+          case @order
+          when "-elapsed"
+            lambda do |statistic|
+              -statistic.elapsed
+            end
+          when "elapsed"
+            lambda do |statistic|
+              statistic.elapsed
+            end
+          when "-start-time"
+            lambda do |statistic|
+              -statistic.start_time.to_f
+            end
+          else
+            lambda do |statistic|
+              statistic.start_time.to_f
+            end
+          end
+        end
+
+        def update_statistic(statistic)
+          statistic.slow_response_threshold = @slow_response_threshold
+          statistic.slow_operation_threshold = @slow_operation_threshold
+          @start_time ||= statistic.start_time
+          @start_time = [@start_time, statistic.start_time].min
+          @last_time ||= statistic.last_time
+          @last_time = [@last_time, statistic.last_time].max
+          @n_responses += 1
+          @total_elapsed += statistic.elapsed_in_seconds
+          return unless @collect_slow_statistics
+          if statistic.slow?
+            @n_slow_responses += 1
+            if statistic.select_command?
+              statistic.each_operation do |operation|
+                next unless operation[:slow?]
+                @n_slow_operations += 1
+                @slow_operations << operation
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/groonga/query-log/analyzer/statistic.rb

@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2012  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga/command"
+
+module Groonga
+  module QueryLog
+    class Analyzer
+      class Statistic
+        attr_reader :context_id, :start_time, :raw_command
+        attr_reader :elapsed, :return_code
+        attr_accessor :slow_operation_threshold, :slow_response_threshold
+        def initialize(context_id)
+          @context_id = context_id
+          @start_time = nil
+          @command = nil
+          @raw_command = nil
+          @operations = []
+          @elapsed = nil
+          @return_code = 0
+          @slow_operation_threshold = 0.1
+          @slow_response_threshold = 0.2
+        end
+
+        def start(start_time, command)
+          @start_time = start_time
+          @raw_command = command
+        end
+
+        def finish(elapsed, return_code)
+          @elapsed = elapsed
+          @return_code = return_code
+        end
+
+        def command
+          Groonga::Command::Parser.parse(@raw_command) do |status, command|
+            case status
+            when :on_load_start
+              @loading = false
+              @command ||= command
+            when :on_command
+              @command ||= command
+            end
+          end
+          @command
+        end
+
+        def elapsed_in_seconds
+          nano_seconds_to_seconds(@elapsed)
+        end
+
+        def last_time
+          @start_time + elapsed_in_seconds
+        end
+
+        def slow?
+          elapsed_in_seconds >= @slow_response_threshold
+        end
+
+        def each_operation
+          previous_elapsed = 0
+          ensure_parse_command
+          operation_context_context = {
+            :filter_index => 0,
+            :drilldown_index => 0,
+          }
+          @operations.each_with_index do |operation, i|
+            relative_elapsed = operation[:elapsed] - previous_elapsed
+            relative_elapsed_in_seconds = nano_seconds_to_seconds(relative_elapsed)
+            previous_elapsed = operation[:elapsed]
+            parsed_operation = {
+              :i => i,
+              :elapsed => operation[:elapsed],
+              :elapsed_in_seconds => nano_seconds_to_seconds(operation[:elapsed]),
+              :relative_elapsed => relative_elapsed,
+              :relative_elapsed_in_seconds => relative_elapsed_in_seconds,
+              :name => operation[:name],
+              :context => operation_context(operation[:name],
+                                    operation_context_context),
+              :n_records => operation[:n_records],
+              :slow? => slow_operation?(relative_elapsed_in_seconds),
+            }
+            yield parsed_operation
+          end
+        end
+
+        def add_operation(operation)
+          @operations << operation
+        end
+
+        def operations
+          _operations = []
+          each_operation do |operation|
+            _operations << operation
+          end
+          _operations
+        end
+
+        def select_command?
+          command.name == "select"
+        end
+
+        private
+        def nano_seconds_to_seconds(nano_seconds)
+          nano_seconds / 1000.0 / 1000.0 / 1000.0
+        end
+
+        def operation_context(label, context)
+          case label
+          when "filter"
+            if @select_command.query and context[:query_used].nil?
+              context[:query_used] = true
+              "query: #{@select_command.query}"
+            else
+              index = context[:filter_index]
+              context[:filter_index] += 1
+              @select_command.conditions[index]
+            end
+          when "sort"
+            @select_command.sortby
+          when "score"
+            @select_command.scorer
+          when "output"
+            @select_command.output_columns
+          when "drilldown"
+            index = context[:drilldown_index]
+            context[:drilldown_index] += 1
+            @select_command.drilldowns[index]
+          else
+            nil
+          end
+        end
+
+        def ensure_parse_command
+          return unless select_command?
+          @select_command = Groonga::Command::Parser.parse(@raw_command)
+        end
+
+        def slow_operation?(elapsed)
+          elapsed >= @slow_operation_threshold
+        end
+      end
+    end
+  end
+end

data/lib/groonga/query-log/analyzer/streamer.rb

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module Groonga
+  module QueryLog
+    class Analyzer
+      class Streamer
+        def initialize(reporter)
+          @reporter = reporter
+        end
+
+        def start
+          @reporter.start
+        end
+
+        def <<(statistic)
+          @reporter.report_statistic(statistic)
+        end
+
+        def finish
+          @reporter.finish
+        end
+      end
+    end
+  end
+end

data/lib/groonga/query-log/parser.rb

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2012  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "English"
+
+require "groonga/query-log/analyzer/statistic"
+
+module Groonga
+  module QueryLog
+    class Parser
+      def initialize
+      end
+
+      def parse(input, &block)
+        current_statistics = {}
+        input.each_line do |line|
+          case line
+          when /\A(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)\.(\d+)\|(.+?)\|([>:<])/
+            year, month, day, hour, minutes, seconds, micro_seconds =
+              $1, $2, $3, $4, $5, $6, $7
+            context_id = $8
+            type = $9
+            rest = $POSTMATCH.strip
+            time_stamp = Time.local(year, month, day, hour, minutes, seconds,
+                                    micro_seconds)
+            parse_line(current_statistics,
+                       time_stamp, context_id, type, rest, &block)
+          end
+        end
+      end
+
+      private
+      def parse_line(current_statistics,
+                     time_stamp, context_id, type, rest, &block)
+        case type
+        when ">"
+          statistic = Analyzer::Statistic.new(context_id)
+          statistic.start(time_stamp, rest)
+          current_statistics[context_id] = statistic
+        when ":"
+          return unless /\A(\d+) (.+)\((\d+)\)/ =~ rest
+          elapsed = $1
+          name = $2
+          n_records = $3.to_i
+          statistic = current_statistics[context_id]
+          return if statistic.nil?
+          statistic.add_operation(:name => name,
+                                  :elapsed => elapsed.to_i,
+                                  :n_records => n_records)
+        when "<"
+          return unless /\A(\d+) rc=(\d+)/ =~ rest
+          elapsed = $1
+          return_code = $2
+          statistic = current_statistics.delete(context_id)
+          return if statistic.nil?
+          statistic.finish(elapsed.to_i, return_code.to_i)
+          block.call(statistic)
+        end
+      end
+    end
+  end
+end