groonga-query-log 1.2.9 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 213e0534e4b93c131e48f76cfe2c6f8c7fea4a44
-  data.tar.gz: 326a76731faf0f948a8362800748503036dca728
+SHA256:
+  metadata.gz: 2bb623843ad5665d8efc011ac13bc1e3a2b5b57deeef9c2af2f8de81e4a5a06c
+  data.tar.gz: 2cd63a89d51337d68d449f77ea0dbcf8698fa395c88a37707ce0b95a65c25bab
 SHA512:
-  metadata.gz: 55fd3f7d8ed2fa51f94ed333e8afeaaef19ad5230b2a4ab8c0ce9622cde117891e3c26bda0f78a0560dcb0382a3b09c843c09cc474cb62faf3b3607a59d51b44
-  data.tar.gz: eff099b4e055fc9e6e332a4d8bd8ae1987596ddab11fc842e83d965603147b68270395102cc9860cc09fb13e6004ebaf2a2bf8a949552bb9712e16552ff7bc5f
+  metadata.gz: a965a269d6b3f4562e55af72df38b0aaf1354fcdb27c068f9ca6c867997a130811c5386e919abb9410d3ed9895329f3a10551c5ebe01bb90845c3c9eb2cc9a73
+  data.tar.gz: dc717453f8f8f7f122eb6202c4825e543102dfd544d1f06a2c722d078e81aefa9a3949f120788f2f91bc469a98637c864a344d5acb348ecb87812509658ca4a9

data/doc/text/news.md

@@ -1,5 +1,33 @@
 # News
 
+## 1.3.0: 2018-06-11
+
+### Improvements
+
+  * Added support for filter context.
+
+  * Added `xterm-256color` as a colorable terminal.
+
+  * `groonga-query-log-check-crash`:
+
+    * Added support for multiple process logs.
+
+    * Added more crash detection patterns.
+
+    * Added support for showing running queries on crash.
+
+    * Added support for showing important messages.
+
+  * `groonga-query-log-extract`:
+
+    * Added `--no-include-arguments` option.
+
+    * Improved pipe support.
+
+  * `groonga-query-log-replay`:
+
+    * Added `--read-timeout` option.
+
 ## 1.2.9: 2018-02-04
 
 ### Improvements

data/lib/groonga-query-log/command/analyzer/reporter/console.rb

@@ -245,7 +245,7 @@ module GroongaQueryLog
         def guess_color_availability(output)
           return false unless output.tty?
           case ENV["TERM"]
-          when /term(?:-color)?\z/, "screen"
+          when /term(?:-(?:256)?color)?\z/, "screen"
             true
           else
             return true if ENV["EMACS"] == "t"

data/lib/groonga-query-log/command/check-crash.rb

@@ -70,31 +70,82 @@ module GroongaQueryLog
         checker.check
       end
 
+      class GroongaProcess
+        attr_reader :pid
+        attr_reader :start_time
+        attr_reader :log_path
+        attr_accessor :last_time
+        attr_accessor :n_leaks
+        attr_writer :crashed
+        attr_reader :important_entries
+        def initialize(pid, start_time, log_path)
+          @pid = pid
+          @start_time = start_time
+          @last_time = @start_time
+          @log_path = log_path
+          @n_leaks = 0
+          @crashed = false
+          @important_entries = []
+        end
+
+        def crashed?
+          @crashed
+        end
+      end
+
       class Checker
         def initialize(log_paths)
-          @general_log_parser = GroongaLog::Parser.new
-          @query_log_parser = Parser.new
           split_log_paths(log_paths)
-
-          @running = nil
-          @crash_sessions = []
-          @session_start = nil
         end
 
         def check
-          @general_log_parser.parse_paths(@general_log_paths) do |entry|
-            check_general_log_entry(@general_log_parser.current_path,
-                                    entry)
-          end
-          @crash_sessions.each do |start, last|
+          processes = ProcessEnumerator.new(@general_log_paths)
+          processes.each do |process|
+            if process.crashed?
+              p [:crashed,
+                 process.start_time.iso8601,
+                 process.last_time.iso8601,
+                 process.pid,
+                 process.log_path]
+            end
+
+            unless process.important_entries.empty?
+              puts("Important entries:")
+              process.important_entries.each_with_index do |entry, i|
+                puts("#{entry.timestamp.iso8601}: " +
+                     "#{entry.log_level}: " +
+                     "#{entry.message}")
+              end
+            end
+
+            unless process.n_leaks.zero?
+              p [:leak,
+                 process.n_leaks,
+                 process.last_time.iso8601,
+                 process.log_path]
+            end
+
+            next unless process.crashed?
+
+            start = process.start_time
+            last = process.last_time
             @flushed = nil
             @unflushed_statistics = []
-            @query_log_parser.parse_paths(@query_log_paths) do |statistic|
+            query_log_parser = Parser.new
+            query_log_parser.parse_paths(@query_log_paths) do |statistic|
               next if statistic.start_time < start
               break if statistic.start_time > last
-              check_query_log_statistic(@query_log_parser.current_path,
+              check_query_log_statistic(query_log_parser.current_path,
                                         statistic)
             end
+            parsing_statistics = query_log_parser.parsing_statistics
+            unless parsing_statistics.empty?
+              puts("Running queries:")
+              parsing_statistics.each do |statistic|
+                puts("#{statistic.start_time.iso8601}:")
+                puts(statistic.command.to_command_format(pretty_print: true))
+              end
+            end
             unless @unflushed_statistics.empty?
               puts("Unflushed statistics in #{start.iso8601}/#{last.iso8601}")
               @unflushed_statistics.each do |statistic|
@@ -120,29 +171,6 @@ module GroongaQueryLog
           end
         end
 
-        def check_general_log_entry(path, entry)
-          # p [path, entry]
-          case entry.log_level
-          when :emergency, :alert, :critical, :error, :warning
-            # p [entry.log_level, entry.message, entry.timestamp.iso8601]
-          end
-
-          case entry.message
-          when /\Agrn_init:/
-            if @running
-              @crash_sessions << [@session_start, entry.timestamp]
-              p [:crashed, entry.timestamp.iso8601, path]
-            end
-            @running = true
-            @session_start = entry.timestamp
-          when /\Agrn_fin \(\d+\)\z/
-            n_leaks = $1.to_i
-            @running = false
-            @session_start = nil
-            p [:leak, n_leask, entry.timestamp.iso8601] unless n_leaks.zero?
-          end
-        end
-
         def check_query_log_statistic(path, statistic)
           case statistic.command.command_name
           when "load"
@@ -164,6 +192,70 @@ module GroongaQueryLog
           end
         end
       end
+
+      class ProcessEnumerator
+        def initialize(general_log_paths)
+          @general_log_paths = general_log_paths
+          @running_processes = {}
+        end
+
+        def each(&block)
+          general_log_parser = GroongaLog::Parser.new
+          general_log_parser.parse_paths(@general_log_paths) do |entry|
+            check_general_log_entry(general_log_parser.current_path,
+                                    entry,
+                                    &block)
+          end
+          @running_processes.each_value do |process|
+            yield(process)
+          end
+        end
+
+        private
+        def check_general_log_entry(path, entry, &block)
+          # p [path, entry]
+          case entry.log_level
+          when :emergency, :alert, :critical, :error, :warning
+            # p [entry.log_level, entry.message, entry.timestamp.iso8601]
+          end
+
+          case entry.message
+          when /\Agrn_init:/
+            process = @running_processes[entry.pid]
+            if process
+              process.crashed = true
+              yield(process)
+              @running_processes.delete(entry.pid)
+            end
+            process = GroongaProcess.new(entry.pid, entry.timestamp, path)
+            @running_processes[entry.pid] = process
+          when /\Agrn_fin \(\d+\)\z/
+            n_leaks = $1.to_i
+            process = @running_processes[entry.pid]
+            process.n_leaks = n_leaks
+            yield(process)
+            @running_processes.delete(entry.pid)
+          else
+            @running_processes[entry.pid] ||=
+              GroongaProcess.new(entry.pid, Time.at(0), path)
+            process = @running_processes[entry.pid]
+            case entry.log_level
+            when :emergency, :alert, :critical, :error
+              process.important_entries << entry
+            end
+            process.last_time = entry.timestamp
+            case entry.message
+            when "-- CRASHED!!! --"
+              process.crashed = true
+            when "----------------"
+              if process.crashed?
+                yield(process)
+                @running_processes.delete(entry.pid)
+              end
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/groonga-query-log/command/extract.rb

@@ -64,7 +64,7 @@ module GroongaQueryLog
             else
               extract(log_paths, $stdout)
             end
-          rescue Interrupt
+          rescue Interrupt, Errno::EPIPE
           rescue Error
             $stderr.puts($!.message)
             return false
@@ -79,6 +79,7 @@ module GroongaQueryLog
           @options.unify_format = nil
           @options.commands = []
           @options.exclude_commands = []
+          @options.include_arguments = true
           @options.output_path = nil
           @option_parser = OptionParser.new do |parser|
             parser.version = VERSION
@@ -121,6 +122,12 @@ module GroongaQueryLog
               end
             end
 
+            parser.on("--[no-]include-arguments",
+                      "Whether include command arguments",
+                      "[#{@options.include_arguments}]") do |include_arguments|
+              @options.include_arguments = include_arguments
+            end
+
             parser.on("--output=PATH",
                       "Output to PATH.",
                       "[standard output]") do |path|
@@ -139,6 +146,9 @@ module GroongaQueryLog
         def extract_command(statistic, output)
           command = statistic.command
           return unless target?(command)
+          unless @options.include_arguments
+            command.arguments.clear
+          end
           command_text = nil
           case @options.unify_format
           when "uri"
@@ -146,7 +156,7 @@ module GroongaQueryLog
           when "command"
             command_text = command.to_command_format
           else
-            command_text = statistic.raw_command
+            command_text = command.to_s
           end
           output.puts(command_text)
         end

data/lib/groonga-query-log/command/replay.rb

@@ -67,6 +67,12 @@ module GroongaQueryLog
           @options.protocol = protocol
         end
 
+        parser.on("--read-timeout=TIMEOUT", Integer,
+                  "Read timeout",
+                  "[#{@options.read_timeout}]") do |read_timeout|
+          @options.read_timeout = read_timeout
+        end
+
         parser.on("--n-clients=N", Integer,
                   "The max number of concurrency",
                   "[#{@options.n_clients}]") do |n_clients|

data/lib/groonga-query-log/parser.rb

@@ -66,13 +66,13 @@ module GroongaQueryLog
         match_data = PATTERN.match(line)
         next if match_data.nil?
 
-        year = match_data[:year].to_i
-        month = match_data[:month].to_i
-        day = match_data[:day].to_i
-        hour = match_data[:hour].to_i
-        minute = match_data[:minute].to_i
-        second = match_data[:second].to_i
-        microsecond = match_data[:microsecond].to_i
+        year = Integer(match_data[:year], 10)
+        month = Integer(match_data[:month], 10)
+        day = Integer(match_data[:day], 10)
+        hour = Integer(match_data[:hour], 10)
+        minute = Integer(match_data[:minute], 10)
+        second = Integer(match_data[:second], 10)
+        microsecond = Integer(match_data[:microsecond], 10)
         context_id = match_data[:context_id]
         type = match_data[:type]
         rest = match_data.post_match.strip

data/lib/groonga-query-log/replayer.rb

@@ -142,6 +142,7 @@ module GroongaQueryLog
         attr_accessor :host
         attr_accessor :port
         attr_accessor :protocol
+        attr_accessor :read_timeout
         attr_accessor :n_clients
         attr_writer :request_queue_size
         attr_writer :disable_cache
@@ -152,6 +153,7 @@ module GroongaQueryLog
           @host = "127.0.0.1"
           @port = 10041
           @protocol = :http
+          @read_timeout = Groonga::Client::Default::READ_TIMEOUT
           @n_clients = 8
           @request_queue_size = nil
           @disable_cache = false
@@ -164,6 +166,7 @@ module GroongaQueryLog
           Groonga::Client.open(:host     => @host,
                                :port     => @port,
                                :protocol => @protocol,
+                               :read_timeout => @read_timeout,
                                &block)
         end
 

data/lib/groonga-query-log/statistic.rb

@@ -92,7 +92,7 @@ module GroongaQueryLog
           :relative_elapsed => relative_elapsed,
           :relative_elapsed_in_seconds => relative_elapsed_in_seconds,
           :name => operation[:name],
-          :context => operation_context(operation[:name],
+          :context => operation_context(operation,
                                         operation_context_context),
           :n_records => operation[:n_records],
           :extra => operation[:extra],
@@ -152,9 +152,13 @@ module GroongaQueryLog
       nano_seconds / 1000.0 / 1000.0 / 1000.0
     end
 
-    def operation_context(label, context)
+    def operation_context(operation, context)
       return nil if @select_command.nil?
 
+      extra = operation[:extra]
+      return extra if extra
+
+      label = operation[:name]
       case label
       when "filter"
         if @select_command.query and context[:query_used].nil?

data/lib/groonga-query-log/version.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2018  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,5 +15,5 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module GroongaQueryLog
-    VERSION = "1.2.9"
+  VERSION = "1.3.0"
 end

data/test/command/test-extract.rb

@@ -1,5 +1,3 @@
-#!/usr/bin/env ruby
-#
 # Copyright (C) 2012  Haruka Yoshihara <yoshihara@clear-code.com>
 # Copyright (C) 2015-2018  Kouhei Sutou <kou@clear-code.com>
 #
@@ -37,12 +35,12 @@ class ExtractCommandTest < Test::Unit::TestCase
     def test_multi
       other_query_log_path = File.join(@fixtures_path, "other-query.log")
       actual_commands = run_extractor(@query_log_path, other_query_log_path)
-      expected_commands = <<-EOC
-load --table Video
-select --table Users --query follower:@groonga --output_columns _key,name
-table_create --name Comments --flags TABLE_HASH_KEY --key_type UInt32
-column_create --table Comments --name title --flags COLUMN_SCALAR --type ShortText
-EOC
+      expected_commands = <<-COMMAND
+load --table "Video"
+select --output_columns "_key,name" --query "follower:@groonga" --table "Users"
+table_create --flags "TABLE_HASH_KEY" --key_type "UInt32" --name "Comments"
+column_create --flags "COLUMN_SCALAR" --name "title" --table "Comments" --type "ShortText"
+      COMMAND
       assert_equal(expected_commands, actual_commands)
     end
 
@@ -57,44 +55,47 @@ EOC
       actual_commands = run_extractor(@query_log_path,
                                       "--unify-format", "command")
 
-      expected_commands = <<-EOC
+      expected_commands = <<-COMMAND
 load --table "Video"
 select --output_columns "_key,name" --query "follower:@groonga" --table "Users"
-EOC
+      COMMAND
       assert_equal(expected_commands, actual_commands)
     end
 
     def test_uri
       actual_commands = run_extractor(@query_log_path,
                                       "--unify-format", "uri")
-      expected_commands =  <<-EOC
+      expected_commands = <<-COMMAND
 /d/load?table=Video
 /d/select?output_columns=_key%2Cname&query=follower%3A%40groonga&table=Users
-EOC
+      COMMAND
       assert_equal(expected_commands, actual_commands)
     end
 
     def test_not_unify
       actual_commands = run_extractor(@query_log_path)
-      expected_commands = <<-EOC
-load --table Video
-select --table Users --query follower:@groonga --output_columns _key,name
-EOC
+      expected_commands = <<-COMMAND
+load --table "Video"
+select --output_columns "_key,name" --query "follower:@groonga" --table "Users"
+      COMMAND
       assert_equal(expected_commands, actual_commands)
     end
   end
 
   def test_command
     actual_command = run_extractor(@query_log_path, "--command", "load")
-    expected_command = "load --table Video\n"
+    expected_command = <<-COMMAND
+load --table "Video"
+    COMMAND
 
     assert_equal(expected_command, actual_command)
   end
 
   def test_exclude_command
     actual_command = run_extractor(@query_log_path, "--exclude-command", "load")
-    expected_command = "select --table Users --query follower:@groonga" +
-                         " --output_columns _key,name\n"
+    expected_command = <<-COMMAND
+select --output_columns "_key,name" --query "follower:@groonga" --table "Users"
+    COMMAND
 
     assert_equal(expected_command, actual_command)
   end
@@ -116,20 +117,21 @@ EOC
   class TestExtract < self
     def setup
       super
-      @log = <<-EOL
+      @log = <<-LOG
 2012-12-12 17:39:17.628846|0x7fff786aa2b0|>select --table Users --query follower:@groonga --output_columns _key,name
 2012-12-12 17:39:17.629676|0x7fff786aa2b0|:000000000842953 filter(2)
 2012-12-12 17:39:17.629709|0x7fff786aa2b0|:000000000870900 select(2)
 2012-12-12 17:39:17.629901|0x7fff786aa2b0|:000000001066752 output(2)
 2012-12-12 17:39:17.630052|0x7fff786aa2b0|<000000001217140 rc=0
-EOL
+     LOG
     end
 
     def test_command_format
       @extract_command.options.unify_format = "command"
-      expected_formatted_command = "select --output_columns \"_key,name\""+
-                                     " --query \"follower:@groonga\"" +
-                                     " --table \"Users\"\n"
+      expected_formatted_command = <<-COMMAND
+select --output_columns "_key,name" --query "follower:@groonga" --table "Users"
+      COMMAND
+
       assert_equal(expected_formatted_command, extract)
     end
 
@@ -143,9 +145,10 @@ EOL
 
     def test_not_unify
       @extract_command.options.unify_format = nil
-      expected_formatted_command = "select --table Users" +
-                                     " --query follower:@groonga" +
-                                     " --output_columns _key,name\n"
+      expected_formatted_command = <<-COMMAND
+select --output_columns "_key,name" --query "follower:@groonga" --table "Users"
+      COMMAND
+
       assert_equal(expected_formatted_command, extract)
     end