groonga-query-log 1.0.0

data/doc/text/news.md

@@ -0,0 +1,5 @@
+# News
+
+## 1.0.0: 2012-12-14
+
+The first release!!!

data/groonga-query-log.gemspec

@@ -0,0 +1,65 @@
+# -*- mode: ruby; coding: utf-8 -*-
+#
+# Copyright (C) 2012  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+base_dir = File.dirname(__FILE__)
+lib_dir = File.join(base_dir, "lib")
+
+$LOAD_PATH.unshift(lib_dir)
+require "groonga/query-log/version"
+
+clean_white_space = lambda do |entry|
+  entry.gsub(/(\A\n+|\n+\z)/, '') + "\n"
+end
+
+Gem::Specification.new do |spec|
+  spec.name = "groonga-query-log"
+  spec.version = Groonga::QueryLog::VERSION.dup
+
+  spec.authors = ["Kouhei Sutou"]
+  spec.email = ["kou@clear-code.com"]
+
+  readme = File.read("README.md")
+  readme.force_encoding("UTF-8") if readme.respond_to?(:force_encoding)
+  entries = readme.split(/^\#\#\s(.*)$/)
+  description = clean_white_space.call(entries[entries.index("Description") + 1])
+  spec.summary, spec.description, = description.split(/\n\n+/, 3)
+
+  spec.files = ["README.md", "Rakefile", "Gemfile", "#{spec.name}.gemspec"]
+  spec.files += Dir.glob("lib/**/*.rb")
+  spec.files += Dir.glob("doc/text/*")
+  spec.files += [".yardopts"]
+  spec.test_files += Dir.glob("test/**/*")
+  Dir.chdir("bin") do
+    spec.executables = Dir.glob("*")
+  end
+
+  spec.homepage = "https://github.com/groonga/groonga-query-log"
+  spec.licenses = ["LGPLv2.1+"]
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency("groonga-command")
+
+  spec.add_development_dependency("test-unit")
+  spec.add_development_dependency("test-unit-notify")
+  spec.add_development_dependency("rake")
+  spec.add_development_dependency("bundler")
+  spec.add_development_dependency("packnga")
+  spec.add_development_dependency("yard")
+  spec.add_development_dependency("redcarpet")
+end
+

data/lib/groonga/query-log.rb

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2012  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga/query-log/version"
+require "groonga/query-log/analyzer"
+require "groonga/query-log/parser"

data/lib/groonga/query-log/analyzer.rb

@@ -0,0 +1,212 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "optparse"
+require "json"
+require "groonga/query-log/parser"
+require "groonga/query-log/analyzer/streamer"
+require "groonga/query-log/analyzer/sized-statistics"
+
+module Groonga
+  module QueryLog
+    class Analyzer
+      class Error < StandardError
+      end
+
+      class UnsupportedReporter < Error
+      end
+
+      def initialize
+        setup_options
+      end
+
+      def run(*argv)
+        log_paths = @option_parser.parse!(argv)
+
+        stream = @options[:stream]
+        dynamic_sort = @options[:dynamic_sort]
+        statistics = SizedStatistics.new
+        statistics.apply_options(@options)
+        parser = Groonga::QueryLog::Parser.new
+        if stream
+          streamer = Streamer.new(create_reporter(statistics))
+          streamer.start
+          process_statistic = lambda do |statistic|
+            streamer << statistic
+          end
+        elsif dynamic_sort
+          process_statistic = lambda do |statistic|
+            statistics << statistic
+          end
+        else
+          full_statistics = []
+          process_statistic = lambda do |statistic|
+            full_statistics << statistic
+          end
+        end
+        if log_paths.empty?
+          parser.parse(ARGF, &process_statistic)
+        else
+          log_paths.each do |log_path|
+            File.open(log_path) do |log|
+              parser.parse(log, &process_statistic)
+            end
+          end
+        end
+        if stream
+          streamer.finish
+          return
+        end
+        statistics.replace(full_statistics) unless dynamic_sort
+
+        reporter = create_reporter(statistics)
+        reporter.apply_options(@options)
+        reporter.report
+      end
+
+      private
+      def setup_options
+        @options = {}
+        @options[:n_entries] = 10
+        @options[:order] = "-elapsed"
+        @options[:color] = :auto
+        @options[:output] = "-"
+        @options[:slow_operation_threshold] = 0.1
+        @options[:slow_response_threshold] = 0.2
+        @options[:reporter] = "console"
+        @options[:dynamic_sort] = true
+        @options[:stream] = false
+        @options[:report_summary] = true
+
+        @option_parser = OptionParser.new do |parser|
+          parser.banner += " LOG1 ..."
+
+          parser.on("-n", "--n-entries=N",
+                    Integer,
+                    "Show top N entries",
+                    "(#{@options[:n_entries]})") do |n|
+            @options[:n_entries] = n
+          end
+
+          available_orders = ["elapsed", "-elapsed", "start-time", "-start-time"]
+          parser.on("--order=ORDER",
+                    available_orders,
+                    "Sort by ORDER",
+                    "available values: [#{available_orders.join(', ')}]",
+                    "(#{@options[:order]})") do |order|
+            @options[:order] = order
+          end
+
+          color_options = [
+            [:auto, :auto],
+            ["-", false],
+            ["no", false],
+            ["false", false],
+            ["+", true],
+            ["yes", true],
+            ["true", true],
+          ]
+          parser.on("--[no-]color=[auto]",
+                    color_options,
+                    "Enable color output",
+                    "(#{@options[:color]})") do |color|
+            if color.nil?
+              @options[:color] = true
+            else
+              @options[:color] = color
+            end
+          end
+
+          parser.on("--output=PATH",
+                    "Output to PATH.",
+                    "'-' PATH means standard output.",
+                    "(#{@options[:output]})") do |output|
+            @options[:output] = output
+          end
+
+          parser.on("--slow-operation-threshold=THRESHOLD",
+                    Float,
+                    "Use THRESHOLD seconds to detect slow operations.",
+                    "(#{@options[:slow_operation_threshold]})") do |threshold|
+            @options[:slow_operation_threshold] = threshold
+          end
+
+          parser.on("--slow-response-threshold=THRESHOLD",
+                    Float,
+                    "Use THRESHOLD seconds to detect slow operations.",
+                    "(#{@options[:sloq_response_threshold]})") do |threshold|
+            @options[:sloq_response_threshold] = threshold
+          end
+
+          available_reporters = ["console", "json", "html"]
+          parser.on("--reporter=REPORTER",
+                    available_reporters,
+                    "Reports statistics by REPORTER.",
+                    "available values: [#{available_reporters.join(', ')}]",
+                    "(#{@options[:reporter]})") do |reporter|
+            @options[:reporter] = reporter
+          end
+
+          parser.on("--[no-]dynamic-sort",
+                    "Sorts dynamically.",
+                    "Memory and CPU usage reduced for large query log.",
+                    "(#{@options[:dynamic_sort]})") do |sort|
+            @options[:dynamic_sort] = sort
+          end
+
+          parser.on("--[no-]stream",
+                    "Outputs analyzed query on the fly.",
+                    "NOTE: --n-entries and --order are ignored.",
+                    "(#{@options[:stream]})") do |stream|
+            @options[:stream] = stream
+          end
+
+          parser.on("--[no-]report-summary",
+                    "Reports summary at the end.",
+                    "(#{@options[:report_summary]})") do |report_summary|
+            @options[:report_summary] = report_summary
+          end
+        end
+      end
+
+      def create_reporter(statistics)
+        case @options[:reporter]
+        when "json"
+          JSONReporter.new(statistics)
+        when "html"
+          HTMLReporter.new(statistics)
+        else
+          ConsoleReporter.new(statistics)
+        end
+      end
+
+      def create_stream_reporter
+        case @options[:reporter]
+        when "json"
+          require 'json'
+          Groonga::QueryLog::StreamJSONQueryLogReporter.new
+        when "html"
+          raise UnsupportedReporter, "HTML reporter doesn't support --stream."
+        else
+          Groonga::QueryLog::StreamConsoleQueryLogReporter.new
+        end
+      end
+    end
+  end
+end

data/lib/groonga/query-log/analyzer/reporter.rb

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module Groonga
+  module QueryLog
+    class Analyzer
+      class Reporter
+        include Enumerable
+
+        attr_reader :output
+        def initialize(statistics)
+          @statistics = statistics
+          @report_summary = true
+          @output = $stdout
+        end
+
+        def apply_options(options)
+          self.output = options[:output] || @output
+          unless options[:report_summary].nil?
+            @report_summary = options[:report_summary]
+          end
+        end
+
+        def output=(output)
+          @output = output
+          @output = $stdout if @output == "-"
+        end
+
+        def each
+          @statistics.each do |statistic|
+            yield statistic
+          end
+        end
+
+        def report
+          setup do
+            report_summary if @report_summary
+            report_statistics
+          end
+        end
+
+        def report_statistics
+          each do |statistic|
+            report_statistic(statistic)
+          end
+        end
+
+        private
+        def setup
+          setup_output do
+            start
+            yield
+            finish
+          end
+        end
+
+        def setup_output
+          original_output = @output
+          if @output.is_a?(String)
+            File.open(@output, "w") do |output|
+              @output = output
+              yield(@output)
+            end
+          else
+            yield(@output)
+          end
+        ensure
+          @output = original_output
+        end
+
+        def write(*args)
+          @output.write(*args)
+        end
+
+        def format_time(time)
+          if time.nil?
+            "NaN"
+          else
+            time.strftime("%Y-%m-%d %H:%M:%S.%u")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/groonga/query-log/analyzer/reporter/console.rb

@@ -0,0 +1,291 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga/query-log/analyzer/reporter"
+
+module Groonga
+  module QueryLog
+    class Analyzer
+      class ConsoleReporter < Reporter
+        class Color
+          NAMES = ["black", "red", "green", "yellow",
+                   "blue", "magenta", "cyan", "white"]
+
+          attr_reader :name
+          def initialize(name, options={})
+            @name = name
+            @foreground = options[:foreground]
+            @foreground = true if @foreground.nil?
+            @intensity = options[:intensity]
+            @bold = options[:bold]
+            @italic = options[:italic]
+            @underline = options[:underline]
+          end
+
+          def foreground?
+            @foreground
+          end
+
+          def intensity?
+            @intensity
+          end
+
+          def bold?
+            @bold
+          end
+
+          def italic?
+            @italic
+          end
+
+          def underline?
+            @underline
+          end
+
+          def ==(other)
+            self.class === other and
+              [name, foreground?, intensity?,
+               bold?, italic?, underline?] ==
+              [other.name, other.foreground?, other.intensity?,
+               other.bold?, other.italic?, other.underline?]
+          end
+
+          def sequence
+            sequence = []
+            if @name == "none"
+            elsif @name == "reset"
+              sequence << "0"
+            else
+              foreground_parameter = foreground? ? 3 : 4
+              foreground_parameter += 6 if intensity?
+              sequence << "#{foreground_parameter}#{NAMES.index(@name)}"
+            end
+            sequence << "1" if bold?
+            sequence << "3" if italic?
+            sequence << "4" if underline?
+            sequence
+          end
+
+          def escape_sequence
+            "\e[#{sequence.join(';')}m"
+          end
+
+          def +(other)
+            MixColor.new([self, other])
+          end
+        end
+
+        class MixColor
+          attr_reader :colors
+          def initialize(colors)
+            @colors = colors
+          end
+
+          def sequence
+            @colors.inject([]) do |result, color|
+              result + color.sequence
+            end
+          end
+
+          def escape_sequence
+            "\e[#{sequence.join(';')}m"
+          end
+
+          def +(other)
+            self.class.new([self, other])
+          end
+
+          def ==(other)
+            self.class === other and colors == other.colors
+          end
+        end
+
+        def initialize(statistics)
+          super
+          @color = :auto
+          @reset_color = Color.new("reset")
+          @color_schema = {
+            :elapsed => {:foreground => :white, :background => :green},
+            :time => {:foreground => :white, :background => :cyan},
+            :slow => {:foreground => :white, :background => :red},
+          }
+        end
+
+        def apply_options(options)
+          super
+          @color = options[:color] || @color
+        end
+
+        def report_statistics
+          write("\n")
+          write("Slow Queries:\n")
+          super
+        end
+
+        def report_statistic(statistic)
+          @index += 1
+          write("%*d) %s" % [@digit, @index, format_heading(statistic)])
+          report_parameters(statistic)
+          report_operations(statistic)
+        end
+
+        def start
+          @index = 0
+          if @statistics.size.zero?
+            @digit = 1
+          else
+            @digit = Math.log10(@statistics.size).truncate + 1
+          end
+        end
+
+        def finish
+        end
+
+        private
+        def setup
+          super do
+            setup_color do
+              yield
+            end
+          end
+        end
+
+        def report_summary
+          write("Summary:\n")
+          write("  Threshold:\n")
+          write("    slow response     : #{@statistics.slow_response_threshold}\n")
+          write("    slow operation    : #{@statistics.slow_operation_threshold}\n")
+          write("  # of responses      : #{@statistics.n_responses}\n")
+          write("  # of slow responses : #{@statistics.n_slow_responses}\n")
+          write("  responses/sec       : #{@statistics.responses_per_second}\n")
+          write("  start time          : #{format_time(@statistics.start_time)}\n")
+          write("  last time           : #{format_time(@statistics.last_time)}\n")
+          write("  period(sec)         : #{@statistics.period}\n")
+          slow_response_ratio = @statistics.slow_response_ratio
+          write("  slow response ratio : %5.3f%%\n" % slow_response_ratio)
+          write("  total response time : #{@statistics.total_elapsed}\n")
+          report_slow_operations
+        end
+
+        def report_slow_operations
+          write("  Slow Operations:\n")
+          total_elapsed_digit = nil
+          total_elapsed_decimal_digit = 6
+          n_operations_digit = nil
+          @statistics.each_slow_operation do |grouped_operation|
+            total_elapsed = grouped_operation[:total_elapsed]
+            total_elapsed_digit ||= Math.log10(total_elapsed).truncate + 1
+            n_operations = grouped_operation[:n_operations]
+            n_operations_digit ||= Math.log10(n_operations).truncate + 1
+            parameters = [total_elapsed_digit + 1 + total_elapsed_decimal_digit,
+                          total_elapsed_decimal_digit,
+                          total_elapsed,
+                          grouped_operation[:total_elapsed_ratio],
+                          n_operations_digit,
+                          n_operations,
+                          grouped_operation[:n_operations_ratio],
+                          grouped_operation[:name],
+                          grouped_operation[:context]]
+            write("    [%*.*f](%5.2f%%) [%*d](%5.2f%%) %9s: %s\n" % parameters)
+          end
+        end
+
+        def report_parameters(statistic)
+          command = statistic.command
+          write("  name: <#{command.name}>\n")
+          write("  parameters:\n")
+          command.arguments.each do |key, value|
+            write("    <#{key}>: <#{value}>\n")
+          end
+        end
+
+        def report_operations(statistic)
+          statistic.each_operation do |operation|
+            relative_elapsed_in_seconds = operation[:relative_elapsed_in_seconds]
+            formatted_elapsed = "%8.8f" % relative_elapsed_in_seconds
+            if operation[:slow?]
+              formatted_elapsed = colorize(formatted_elapsed, :slow)
+            end
+            operation_report = " %2d) %s: %10s" % [operation[:i] + 1,
+                                                   formatted_elapsed,
+                                                   operation[:name]]
+            if operation[:n_records]
+              operation_report << "(%6d)" % operation[:n_records]
+            else
+              operation_report << "(%6s)" % ""
+            end
+            context = operation[:context]
+            if context
+              context = colorize(context, :slow) if operation[:slow?]
+              operation_report << " " << context
+            end
+            write("#{operation_report}\n")
+          end
+          write("\n")
+        end
+
+        def guess_color_availability(output)
+          return false unless output.tty?
+          case ENV["TERM"]
+          when /term(?:-color)?\z/, "screen"
+            true
+          else
+            return true if ENV["EMACS"] == "t"
+            false
+          end
+        end
+
+        def setup_color
+          color = @color
+          @color = guess_color_availability(@output) if @color == :auto
+          yield
+        ensure
+          @color = color
+        end
+
+        def format_heading(statistic)
+          formatted_elapsed = colorize("%8.8f" % statistic.elapsed_in_seconds,
+                                       :elapsed)
+          "[%s-%s (%s)](%d): %s" % [format_time(statistic.start_time),
+                                    format_time(statistic.last_time),
+                                    formatted_elapsed,
+                                    statistic.return_code,
+                                    statistic.raw_command]
+        end
+
+        def format_time(time)
+          colorize(super, :time)
+        end
+
+        def colorize(text, schema_name)
+          return text unless @color
+          options = @color_schema[schema_name]
+          color = Color.new("none")
+          if options[:foreground]
+            color += Color.new(options[:foreground].to_s, :bold => true)
+          end
+          if options[:background]
+            color += Color.new(options[:background].to_s, :foreground => false)
+          end
+          "%s%s%s" % [color.escape_sequence, text, @reset_color.escape_sequence]
+        end
+      end
+    end
+  end
+end