groonga-query-log 1.2.8 → 1.2.9

data/lib/{groonga/query-log → groonga-query-log/command}/analyzer/reporter/json-stream.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2014-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -16,10 +14,10 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "groonga/query-log/analyzer/reporter"
+require "groonga-query-log/command/analyzer/reporter"
 
-module Groonga
-  module QueryLog
+module GroongaQueryLog
+  module Command
     class Analyzer
       class JSONStreamReporter < Reporter
         def report_statistic(statistic)

data/lib/{groonga/query-log → groonga-query-log/command}/analyzer/reporter/json.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2011-2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2018  Kouhei Sutou <kou@clear-code.com>
 # Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
@@ -17,10 +15,10 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "groonga/query-log/analyzer/reporter"
+require "groonga-query-log/command/analyzer/reporter"
 
-module Groonga
-  module QueryLog
+module GroongaQueryLog
+  module Command
     class Analyzer
       class JSONReporter < Reporter
         def report_statistic(statistic)
@@ -46,7 +44,9 @@ module Groonga
 
         private
         def format_statistic(statistic)
-          JSON.generate(statistic.to_hash)
+          hash = statistic.to_hash
+          hash.delete("command") unless @report_command_line
+          JSON.generate(hash)
         end
       end
     end

data/lib/{groonga/query-log → groonga-query-log/command}/analyzer/sized-grouped-operations.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2011-2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2017  Kouhei Sutou <kou@clear-code.com>
 # Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
@@ -17,8 +15,8 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-module Groonga
-  module QueryLog
+module GroongaQueryLog
+  module Command
     class Analyzer
       class SizedGroupedOperations < Array
         def initialize

data/lib/{groonga/query-log → groonga-query-log/command}/analyzer/sized-statistics.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2011-2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2017  Kouhei Sutou <kou@clear-code.com>
 # Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
@@ -17,10 +15,10 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "groonga/query-log/analyzer/sized-grouped-operations"
+require "groonga-query-log/command/analyzer/sized-grouped-operations"
 
-module Groonga
-  module QueryLog
+module GroongaQueryLog
+  module Command
     class Analyzer
       class SizedStatistics < Array
         attr_reader :n_responses, :n_slow_responses, :n_slow_operations

data/lib/{groonga/query-log → groonga-query-log/command}/analyzer/streamer.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2011-2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2017  Kouhei Sutou <kou@clear-code.com>
 # Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
@@ -17,8 +15,8 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-module Groonga
-  module QueryLog
+module GroongaQueryLog
+  module Command
     class Analyzer
       class Streamer
         def initialize(reporter)
@@ -30,7 +28,7 @@ module Groonga
         end
 
         def <<(statistic)
-          @reporter.report_statistic(statistic) if statistic.slow?
+          @reporter.report_statistic(statistic)
         end
 
         def finish

data/lib/groonga-query-log/command/check-command-version-compatibility.rb

@@ -0,0 +1,69 @@
+# Copyright (C) 2014-2017  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "optparse"
+
+require "groonga-query-log"
+
+module GroongaQueryLog
+    module Command
+      class CheckCommandVersionCompatibility
+        def initialize
+          @options = CommandVersionCompatibilityChecker::Options.new
+        end
+
+        def run(command_line)
+          input_paths = create_parser.parse(command_line)
+          checker = CommandVersionCompatibilityChecker.new(@options)
+          checker.start do
+            compatible = true
+            if input_paths.empty?
+              compatible = false unless checker.check($stdin)
+            else
+              input_paths.each do |input_path|
+                File.open(input_path) do |input|
+                  compatible = false unless checker.check(input)
+                end
+              end
+            end
+            compatible
+          end
+        end
+
+        private
+        def create_parser
+          parser = OptionParser.new
+          parser.version = VERSION
+          parser.banner += " QUERY_LOG1 QUERY_LOG2 ..."
+
+          parser.separator("")
+          parser.separator("Options:")
+
+          parser.on("--target-version=VERSION", Integer,
+                    "Check incompatibility against command version VERSION",
+                    "[#{@options.target_version}]") do |version|
+            @options.target_version = version
+          end
+
+          parser.on("--output=PATH",
+                    "Output results to PATH",
+                    "[stdout]") do |path|
+            @options.output_path = path
+          end
+        end
+      end
+    end
+end

data/lib/groonga-query-log/command/check-crash.rb

@@ -0,0 +1,169 @@
+# Copyright (C) 2018  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "optparse"
+
+require "groonga-query-log"
+require "groonga-query-log/command-line"
+
+module GroongaQueryLog
+  module Command
+    class CheckCrash < CommandLine
+      def initialize
+        setup_options
+      end
+
+      def run(arguments)
+        begin
+          log_paths = @option_parser.parse!(arguments)
+        rescue OptionParser::InvalidOption => error
+          $stderr.puts(error)
+          return false
+        end
+
+        begin
+          check(log_paths)
+        rescue Interrupt
+        rescue Error
+          $stderr.puts($!.message)
+          return false
+        end
+
+        true
+      end
+
+      private
+      def setup_options
+        @options = {}
+
+        @option_parser = OptionParser.new do |parser|
+          parser.version = VERSION
+          parser.banner += " LOG1 ..."
+        end
+      end
+
+      def open_output
+        if @options[:output] == "-"
+          yield($stdout)
+        else
+          File.open(@options[:output], "w") do |output|
+            yield(output)
+          end
+        end
+      end
+
+      def check(log_paths)
+        checker = Checker.new(log_paths)
+        checker.check
+      end
+
+      class Checker
+        def initialize(log_paths)
+          @general_log_parser = GroongaLog::Parser.new
+          @query_log_parser = Parser.new
+          split_log_paths(log_paths)
+
+          @running = nil
+          @crash_sessions = []
+          @session_start = nil
+        end
+
+        def check
+          @general_log_parser.parse_paths(@general_log_paths) do |entry|
+            check_general_log_entry(@general_log_parser.current_path,
+                                    entry)
+          end
+          @crash_sessions.each do |start, last|
+            @flushed = nil
+            @unflushed_statistics = []
+            @query_log_parser.parse_paths(@query_log_paths) do |statistic|
+              next if statistic.start_time < start
+              break if statistic.start_time > last
+              check_query_log_statistic(@query_log_parser.current_path,
+                                        statistic)
+            end
+            unless @unflushed_statistics.empty?
+              puts("Unflushed statistics in #{start.iso8601}/#{last.iso8601}")
+              @unflushed_statistics.each do |statistic|
+                puts("#{statistic.start_time.iso8601}: #{statistic.raw_command}")
+              end
+            end
+          end
+        end
+
+        private
+        def split_log_paths(log_paths)
+          @general_log_paths = []
+          @query_log_paths = []
+          log_paths.each do |log_path|
+            sample_lines = GroongaLog::Input.open(log_path) do |log_file|
+              log_file.each_line.take(10)
+            end
+            if sample_lines.any? {|line| Parser.target_line?(line)}
+              @query_log_paths << log_path
+            elsif sample_lines.any? {|line| GroongaLog::Parser.target_line?(line)}
+              @general_log_paths << log_path
+            end
+          end
+        end
+
+        def check_general_log_entry(path, entry)
+          # p [path, entry]
+          case entry.log_level
+          when :emergency, :alert, :critical, :error, :warning
+            # p [entry.log_level, entry.message, entry.timestamp.iso8601]
+          end
+
+          case entry.message
+          when /\Agrn_init:/
+            if @running
+              @crash_sessions << [@session_start, entry.timestamp]
+              p [:crashed, entry.timestamp.iso8601, path]
+            end
+            @running = true
+            @session_start = entry.timestamp
+          when /\Agrn_fin \(\d+\)\z/
+            n_leaks = $1.to_i
+            @running = false
+            @session_start = nil
+            p [:leak, n_leask, entry.timestamp.iso8601] unless n_leaks.zero?
+          end
+        end
+
+        def check_query_log_statistic(path, statistic)
+          case statistic.command.command_name
+          when "load"
+            @flushed = false
+            @unflushed_statistics << statistic
+          when "io_flush"
+            @flushed = true
+            @unflushed_statistics.clear
+          when "database_unmap"
+            @unflushed_statistics.reject! do |statistic|
+              statistic.command.name == "load"
+            end
+          when /\Atable_/
+            @flushed = false
+            @unflushed_statistics << statistic
+          when /\Acolumn_/
+            @flushed = false
+            @unflushed_statistics << statistic
+          end
+        end
+      end
+    end
+  end
+end

data/lib/groonga-query-log/command/detect-memory-leak.rb

@@ -0,0 +1,89 @@
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "optparse"
+
+require "groonga-query-log/version"
+require "groonga-query-log/memory-leak-detector"
+
+module GroongaQueryLog
+    module Command
+      class DetectMemoryLeak
+        def initialize
+          @options = MemoryLeakDetector::Options.new
+        end
+
+        def run(command_line)
+          input_paths = create_parser.parse(command_line)
+          detector = MemoryLeakDetector.new(@options)
+          input_paths.each do |input_path|
+            File.open(input_path) do |input|
+              detector.detect(input)
+            end
+          end
+          true
+        end
+
+        private
+        def create_parser
+          parser = OptionParser.new
+          parser.version = VERSION
+          parser.banner += " QUERY_LOG"
+
+          parser.separator("")
+          parser.separator("Options:")
+
+          parser.on("--host=HOST",
+                    "Host name or IP address of groonga server",
+                    "[#{@options.host}]") do |host|
+            @options.host = host
+          end
+
+          parser.on("--port=PORT", Integer,
+                    "Port number of groonga server",
+                    "[#{@options.port}]") do |port|
+            @options.port = port
+          end
+
+          available_protocols = [:gqtp, :http]
+          available_protocols_label = "[#{available_protocols.join(', ')}]"
+          parser.on("--protocol=PROTOCOL", available_protocols,
+                    "Protocol of groonga server",
+                    available_protocols_label) do |protocol|
+            @options.protocol = protocol
+          end
+
+          parser.on("--pid=PID",
+                    "The PID of groonga server",
+                    "[#{@options.pid}]") do |pid|
+            @options.pid = pid
+          end
+
+          parser.on("--n-tries=N", Integer,
+                    "The number of the same request tries",
+                    "[#{@options.n_tries}]") do |n|
+            @options.n_tries = n
+          end
+
+          parser.on("--[no-]force-disable-cache",
+                    "Force disable cache of select command by cache=no parameter",
+                    "[#{@options.force_disable_cache?}]") do |boolean|
+            @options.force_disable_cache = boolean
+          end
+        end
+      end
+    end
+end