groonga-query-log 1.3.3 → 1.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b787d986aa85a124beec78ff3b1da09915dacfb9012c1619120d6a1ff3526603
-  data.tar.gz: 13f736d7fd43e6fa1f87ab5de39600ed71b17baf4a53bfa21e9f4ecca74753a6
+  metadata.gz: e09cac93999ccbd46a1521afa9ccae705d47795575d275d086a1cc15e16ed8f1
+  data.tar.gz: e39350cad4042ef656afc00e133db5913664e50ff9102ac7e11afb26a1fc7143
 SHA512:
-  metadata.gz: 8e5fdcda766d04c1b57b391f0ebec4d86f986e441869aa884c6378494198dc60bdc731ef04d07199b5d0be9e41ec48b07b61b78c2657b2eac3d10ca8cecbfd5a
-  data.tar.gz: cd6c5e20f7d73f8ef02ccc75b384178dbd3614179f797a19bad266ed16c8c7a6f0c792a46d899371d1bca7b382751d5b571e0af22d3244444d43be49b44ca986
+  metadata.gz: d06425f98da9e4d9402211298bf9b7e9baceb1735b37fb37fc0e634f8a71dae7c31bbd576682177650d72affaee9e0dbac953e0e0de26a000a121b4d15791684
+  data.tar.gz: 4a88d239858bd5e61ef0261c566039207cb95f36e2fad05ef0f847a967935765ee54128afe326c6a0e2989d071f2feb5f267f99e5136e4d15cee6b70dcab02f4

data/doc/text/news.md

@@ -1,5 +1,34 @@
 # News
 
+## 1.3.4: 2018-08-22
+
+### Improvements
+
+  * `groonga-query-log-check-crash`:
+
+    * Added support for logs generated by Mroonga.
+
+    * Added support for reporting unfinished processes.
+
+    * Added support for reporting successfully finished processes.
+
+    * Added support for reporting path that includes the last entry.
+
+  * `groonga-query-log-extract`:
+
+    * Added `--inspect-query` option.
+
+  * `groonga-query-log-analyze`:
+
+    * Added support conditions report for `range_filter`,
+      `logical_select` and `logical_range_filter`.
+
+### Fixes
+
+  * `groonga-query-log-check-crash`:
+
+    * Fixed memory leak detection logic.
+
 ## 1.3.3: 2018-07-05
 
 ### Improvements

data/lib/groonga-query-log/command/analyzer/sized-statistics.rb

@@ -143,7 +143,7 @@ module GroongaQueryLog
           return unless @collect_slow_statistics
           if statistic.slow?
             @n_slow_responses += 1
-            if statistic.select_command?
+            if statistic.select_family_command?
               statistic.each_operation do |operation|
                 next unless operation[:slow?]
                 @n_slow_operations += 1

data/lib/groonga-query-log/command/check-crash.rb

@@ -73,24 +73,39 @@ module GroongaQueryLog
       class GroongaProcess
         attr_reader :pid
         attr_reader :start_time
-        attr_reader :log_path
+        attr_reader :start_log_path
         attr_accessor :last_time
+        attr_accessor :last_log_path
         attr_accessor :n_leaks
         attr_writer :crashed
+        attr_writer :finished
         attr_reader :important_entries
-        def initialize(pid, start_time, log_path)
+        def initialize(pid, start_time, start_log_path)
           @pid = pid
           @start_time = start_time
           @last_time = @start_time
-          @log_path = log_path
+          @start_log_path = start_log_path
+          @last_log_path = @start_log_path
           @n_leaks = 0
           @crashed = false
+          @finished = false
           @important_entries = []
         end
 
         def crashed?
           @crashed
         end
+
+        def finished?
+          @finished
+        end
+
+        def successfully_finished?
+          return false if crashed?
+          return false unless finished?
+
+          true
+        end
       end
 
       class Checker
@@ -101,12 +116,38 @@ module GroongaQueryLog
         def check
           processes = ProcessEnumerator.new(@general_log_paths)
           processes.each do |process|
-            if process.crashed?
-              p [:crashed,
+            need_query_log_parsing = true
+            if process.successfully_finished?
+              need_query_log_parsing = false
+              p [:process,
+                 :success,
                  process.start_time.iso8601,
                  process.last_time.iso8601,
                  process.pid,
-                 process.log_path]
+                 process.start_log_path,
+                 process.last_log_path]
+            elsif process.crashed?
+              p [:process,
+                 :crashed,
+                 process.start_time.iso8601,
+                 process.last_time.iso8601,
+                 process.pid,
+                 process.start_log_path,
+                 process.last_log_path]
+            else
+              p [:process,
+                 :unfinished,
+                 process.start_time.iso8601,
+                 process.pid,
+                 process.start_log_path]
+            end
+
+            unless process.n_leaks.zero?
+              p [:leak,
+                 process.n_leaks,
+                 process.last_time.iso8601,
+                 process.pid,
+                 process.last_log_path]
             end
 
             unless process.important_entries.empty?
@@ -118,14 +159,7 @@ module GroongaQueryLog
               end
             end
 
-            unless process.n_leaks.zero?
-              p [:leak,
-                 process.n_leaks,
-                 process.last_time.iso8601,
-                 process.log_path]
-            end
-
-            next unless process.crashed?
+            next unless need_query_log_parsing
 
             start = process.start_time
             last = process.last_time
@@ -139,9 +173,12 @@ module GroongaQueryLog
                                         statistic)
             end
             parsing_statistics = query_log_parser.parsing_statistics
-            unless parsing_statistics.empty?
+            target_parsing_statistics = parsing_statistics.reject do |statistic|
+              statistic.start_time < start
+            end
+            unless target_parsing_statistics.empty?
               puts("Running queries:")
-              parsing_statistics.each do |statistic|
+              target_parsing_statistics.each do |statistic|
                 puts("#{statistic.start_time.iso8601}:")
                 puts(statistic.command.to_command_format(pretty_print: true))
               end
@@ -226,19 +263,23 @@ module GroongaQueryLog
           end
 
           case entry.message
-          when /\Agrn_init:/
+          when /\Agrn_init:/, /\Amroonga \d+\.\d+ started\.\z/
             process = @running_processes[entry.pid]
             if process
+              process.finished = true
               process.crashed = true
               yield(process)
               @running_processes.delete(entry.pid)
             end
             process = GroongaProcess.new(entry.pid, entry.timestamp, path)
             @running_processes[entry.pid] = process
-          when /\Agrn_fin \(\d+\)\z/
+          when /\Agrn_fin \((\d+)\)\z/
             n_leaks = $1.to_i
             process = @running_processes[entry.pid]
             process.n_leaks = n_leaks
+            process.last_time = entry.timestamp
+            process.last_log_path = path
+            process.finished = true
             yield(process)
             @running_processes.delete(entry.pid)
           else
@@ -250,9 +291,11 @@ module GroongaQueryLog
               process.important_entries << entry
             end
             process.last_time = entry.timestamp
+            process.last_log_path = path
             case entry.message
             when "-- CRASHED!!! --"
               process.crashed = true
+              process.finished = true
             when "----------------"
               if process.crashed?
                 yield(process)

data/lib/groonga-query-log/command/extract.rb

@@ -14,168 +14,239 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "ostruct"
+require "json"
 require "optparse"
+require "ostruct"
 require "pathname"
 
 require "groonga-query-log"
 require "groonga-query-log/command-line"
 
 module GroongaQueryLog
-    module Command
-      class Extract < CommandLine
-        attr_accessor :options
-        attr_reader :option_parser
-
-        def initialize
-          @options = nil
-          @option_parser = nil
-          setup_options
-        end
-
-        # Executes extractor for Groonga's query logs.
-        # "groonga-query-log-extract" command runs this method.
-        #
-        # @example
-        #   extractor = GroongaQueryLog::Command::Extract.new
-        #   extractor.run("--output", "commands.output",
-        #                 "--command", "select",
-        #                 "query.log")
-        #
-        # If only paths of query log files are specified,
-        # this method prints command(s) of them to console.
-        #
-        # @param [Array<String>] arguments arguments for
-        #   groonga-query-log-extract. Please execute
-        #   "groonga-query-log-extract --help" or see #setup_options.
-        def run(arguments)
-          begin
-            log_paths = @option_parser.parse!(arguments)
-          rescue OptionParser::ParseError
-            $stderr.puts($!.message)
-            return false
-          end
+  module Command
+    class Extract < CommandLine
+      attr_accessor :options
+      attr_reader :option_parser
+
+      def initialize
+        @options = nil
+        @option_parser = nil
+        setup_options
+      end
+
+      # Executes extractor for Groonga's query logs.
+      # "groonga-query-log-extract" command runs this method.
+      #
+      # @example
+      #   extractor = GroongaQueryLog::Command::Extract.new
+      #   extractor.run("--output", "commands.output",
+      #                 "--command", "select",
+      #                 "query.log")
+      #
+      # If only paths of query log files are specified,
+      # this method prints command(s) of them to console.
+      #
+      # @param [Array<String>] arguments arguments for
+      #   groonga-query-log-extract. Please execute
+      #   "groonga-query-log-extract --help" or see #setup_options.
+      def run(arguments)
+        begin
+          log_paths = @option_parser.parse!(arguments)
+        rescue OptionParser::ParseError
+          $stderr.puts($!.message)
+          return false
+        end
 
-          begin
-            if @options.output_path
-              File.open(@options.output_path, "w") do |output|
-                extract(log_paths, output)
-              end
-            else
-              extract(log_paths, $stdout)
+        begin
+          if @options.output
+            File.open(@options.output, "w") do |output|
+              extract(log_paths, output)
             end
-          rescue Interrupt, Errno::EPIPE
-          rescue Error
-            $stderr.puts($!.message)
-            return false
+          else
+            extract(log_paths, $stdout)
           end
+        rescue Interrupt, Errno::EPIPE
+        rescue Error
+          $stderr.puts($!.message)
+          return false
+        end
 
-          true
-        end
-
-        private
-        def setup_options
-          @options = OpenStruct.new
-          @options.unify_format = nil
-          @options.commands = []
-          @options.exclude_commands = []
-          @options.include_arguments = true
-          @options.output_path = nil
-          @option_parser = OptionParser.new do |parser|
-            parser.version = VERSION
-            parser.banner += " QUERY_LOG1 ..."
-
-            available_formats = ["uri", "command"]
-            parser.on("--unify-format=FORMAT",
-                      available_formats,
-                      "Unify command format to FORMAT.",
-                      "(#{available_formats.join(', ')})",
-                      "[not unify]") do |format|
-              @options.unify_format = format
-            end
+        true
+      end
 
-            parser.on("--command=COMMAND",
-                      "Extract only COMMAND.",
-                      "To extract one or more commands,",
-                      "specify this command a number of times.",
-                      "Use /.../ as COMMAND to match command with regular expression.",
-                      "[all commands]") do |command|
-              case command
-              when /\A\/(.*)\/(i)?\z/
-                @options.commands << Regexp.new($1, $2 == "i")
-              when
-                @options.commands << command
-              end
-            end
+      private
+      def setup_options
+        @options = OpenStruct.new
+        @options.unify_format = nil
+        @options.commands = []
+        @options.exclude_commands = []
+        @options.include_arguments = true
+        @options.output = nil
+        @options.inspect_query = false
+        @option_parser = OptionParser.new do |parser|
+          parser.version = VERSION
+          parser.banner += " QUERY_LOG1 ..."
+
+          available_formats = ["uri", "command"]
+          parser.on("--unify-format=FORMAT",
+                    available_formats,
+                    "Unify command format to FORMAT.",
+                    "(#{available_formats.join(', ')})",
+                    "[not unify]") do |format|
+            @options.unify_format = format
+          end
 
-            parser.on("--exclude-command=COMMAND",
-                      "Don't extract COMMAND.",
-                      "To ignore one or more commands,",
-                      "specify this command a number of times.",
-                      "Use /.../ as COMMAND to match command with regular expression.",
-                      "[no commands]") do |command|
-              case command
-              when /\A\/(.*)\/(i)?\z/
-                @options.exclude_commands << Regexp.new($1, $2 == "i")
-              when
-                @options.exclude_commands << command
-              end
+          parser.on("--command=COMMAND",
+                    "Extract only COMMAND.",
+                    "To extract one or more commands,",
+                    "specify this command a number of times.",
+                    "Use /.../ as COMMAND to match command with regular expression.",
+                    "[all commands]") do |command|
+            case command
+            when /\A\/(.*)\/(i)?\z/
+              @options.commands << Regexp.new($1, $2 == "i")
+            when
+              @options.commands << command
             end
+          end
 
-            parser.on("--[no-]include-arguments",
-                      "Whether include command arguments",
-                      "[#{@options.include_arguments}]") do |include_arguments|
-              @options.include_arguments = include_arguments
+          parser.on("--exclude-command=COMMAND",
+                    "Don't extract COMMAND.",
+                    "To ignore one or more commands,",
+                    "specify this command a number of times.",
+                    "Use /.../ as COMMAND to match command with regular expression.",
+                    "[no commands]") do |command|
+            case command
+            when /\A\/(.*)\/(i)?\z/
+              @options.exclude_commands << Regexp.new($1, $2 == "i")
+            when
+              @options.exclude_commands << command
             end
+          end
 
-            parser.on("--output=PATH",
-                      "Output to PATH.",
-                      "[standard output]") do |path|
-              @options.output_path = path
-            end
+          parser.on("--[no-]include-arguments",
+                    "Whether include command arguments",
+                    "[#{@options.include_arguments}]") do |include_arguments|
+            @options.include_arguments = include_arguments
           end
-        end
 
-        def extract(log_paths, output)
-          parser = Parser.new
-          parse_log(parser, log_paths) do |statistic|
-            extract_command(statistic, output)
+          parser.on("--output=OUTPUT",
+                    "If you specify path as OUTPUT,",
+                    "executed commands are printed to the path.",
+                    "If you specify a URL like",
+                    "http://localhost:10041/?table=QueryLogEntries,",
+                    "each entry are stored to QueryLogEntries Groonga table",
+                    "running at localhost on port 10041.",
+                    "[standard output]") do |output|
+            @options.output = output
           end
-        end
 
-        def extract_command(statistic, output)
-          command = statistic.command
-          return unless target?(command)
-          unless @options.include_arguments
-            command.arguments.clear
+          parser.on("--[no-]inspect-query",
+                    "Inspect query.",
+                    "[#{@options.inspect_query}]") do |boolean|
+            @options.inspect_query = boolean
           end
-          command_text = nil
-          case @options.unify_format
-          when "uri"
-            command_text = command.to_uri_format
-          when "command"
-            command_text = command.to_command_format
+        end
+      end
+
+      def extract(log_paths, output)
+        if @options.inspect_query
+          formatter = InspectFormatter.new(output)
+        else
+          formatter = DumpFormatter.new(output)
+        end
+        formatter.start
+        parser = Parser.new
+        parse_log(parser, log_paths) do |statistic|
+          extract_command(statistic, formatter)
+        end
+        formatter.finish
+      end
+
+      def extract_command(statistic, formatter)
+        command = statistic.command
+        return unless target?(command)
+        unless @options.include_arguments
+          command.arguments.clear
+        end
+        command_text = nil
+        case @options.unify_format
+        when "uri"
+          command_text = command.to_uri_format
+        when "command"
+          command_text = command.to_command_format
+        else
+          command_text = command.to_s
+        end
+        formatter.command(statistic, command_text)
+      end
+
+      def target?(command)
+        name = command.command_name
+        target_commands = @options.commands
+        exclude_commands = @options.exclude_commands
+
+        unless target_commands.empty?
+          return target_commands.any? {|target_command| target_command === name}
+        end
+
+        unless exclude_commands.empty?
+          return (not exclude_commands.any? {|exclude_command| exclude_command === name})
+        end
+
+        true
+      end
+
+      class InspectFormatter
+        def initialize(output)
+          @output = output
+          @first_comand = false
+        end
+
+        def start
+          @output.puts("[")
+        end
+
+        def command(statistic, command_text)
+          if @first_command
+            @first_command = false
           else
-            command_text = command.to_s
+            @output.puts(",")
           end
-          output.puts(command_text)
+          record = {
+            "start_time" => statistic.start_time,
+            "elapsed_time" => statistic.elapsed_in_seconds,
+            "last_time" => statistic.last_time,
+            "return_code" => statistic.return_code,
+            "command" => command_text,
+          }
+          statistic.command.arguments.each do |name, value|
+            record["argument_#{name}"] = value
+          end
+          @output.print(record.to_json)
+        end
+
+        def finish
+          @output.puts("") unless @first_comand
+          @output.puts("]")
         end
+      end
 
-        def target?(command)
-          name = command.command_name
-          target_commands = @options.commands
-          exclude_commands = @options.exclude_commands
+      class DumpFormatter
+        def initialize(output)
+          @output = output
+        end
 
-          unless target_commands.empty?
-            return target_commands.any? {|target_command| target_command === name}
-          end
+        def start
+        end
 
-          unless exclude_commands.empty?
-            return (not exclude_commands.any? {|exclude_command| exclude_command === name})
-          end
+        def command(statistic, command_text)
+          @output.puts(command_text)
+        end
 
-          true
+        def finish
         end
       end
     end
+  end
 end

data/lib/groonga-query-log/statistic.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2018  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -27,8 +27,6 @@ module GroongaQueryLog
     def initialize(context_id)
       @context_id = context_id
       @start_time = nil
-      @command = nil
-      @select_command = nil
       @raw_command = nil
       @operations = []
       @elapsed = nil
@@ -48,16 +46,7 @@ module GroongaQueryLog
     end
 
     def command
-      Groonga::Command::Parser.parse(@raw_command) do |status, command|
-        case status
-        when :on_load_start
-          @loading = false
-          @command ||= command
-        when :on_command
-          @command ||= command
-        end
-      end
-      @command
+      @command ||= parse_command
     end
 
     def elapsed_in_seconds
@@ -76,7 +65,6 @@ module GroongaQueryLog
       return to_enum(__method__) unless block_given?
 
       previous_elapsed = 0
-      ensure_parse_command
       operation_context_context = {
         :filter_index => 0,
         :drilldown_index => 0,
@@ -114,8 +102,15 @@ module GroongaQueryLog
       _operations
     end
 
-    def select_command?
-      command.name == "select"
+    def select_family_command?
+      case command.command_name
+      when "select", "range_filter"
+        true
+      when "logical_select", "logical_range_filter"
+        true
+      else
+        false
+      end
     end
 
     def to_hash
@@ -148,12 +143,20 @@ module GroongaQueryLog
     end
 
     private
+    def parse_command
+      command = nil
+      Groonga::Command::Parser.parse(@raw_command) do |_status, _command|
+        command = _command
+      end
+      command
+    end
+
     def nano_seconds_to_seconds(nano_seconds)
       nano_seconds / 1000.0 / 1000.0 / 1000.0
     end
 
     def operation_context(operation, context)
-      return nil if @select_command.nil?
+      return nil unless select_family_command?
 
       extra = operation[:extra]
       return extra if extra
@@ -161,34 +164,29 @@ module GroongaQueryLog
       label = operation[:name]
       case label
       when "filter"
-        if @select_command.query and context[:query_used].nil?
+        if command.query and context[:query_used].nil?
           context[:query_used] = true
-          "query: #{@select_command.query}"
+          "query: #{command.query}"
         else
           index = context[:filter_index]
           context[:filter_index] += 1
-          @select_command.conditions[index]
+          command.conditions[index]
         end
       when "sort"
-        @select_command.sortby
+        command.sortby
       when "score"
-        @select_command.scorer
+        command.scorer
       when "output"
-        @select_command.output_columns
+        command.output_columns
       when "drilldown"
         index = context[:drilldown_index]
         context[:drilldown_index] += 1
-        @select_command.drilldowns[index]
+        command.drilldowns[index]
       else
         nil
       end
     end
 
-    def ensure_parse_command
-      return unless select_command?
-      @select_command = Groonga::Command::Parser.parse(@raw_command)
-    end
-
     def slow_operation?(elapsed)
       elapsed >= @slow_operation_threshold
     end

data/lib/groonga-query-log/version.rb

@@ -15,5 +15,5 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module GroongaQueryLog
-  VERSION = "1.3.3"
+  VERSION = "1.3.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: groonga-query-log
 version: !ruby/object:Gem::Version
-  version: 1.3.3
+  version: 1.3.4
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-06 00:00:00.000000000 Z
+date: 2018-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: groonga-command-parser