groonga-query-log 1.2.8 → 1.2.9

data/lib/groonga-query-log/command/show-running-queries.rb

@@ -0,0 +1,78 @@
+# Copyright (C) 2014-2017  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "optparse"
+require "time"
+
+require "groonga-query-log/version"
+require "groonga-query-log/parser"
+
+module GroongaQueryLog
+    module Command
+      class ShowRunningQueries
+        def initialize
+          @base_time = nil
+        end
+
+        def run(command_line)
+          input_paths = create_parser.parse(command_line)
+          each_parsing_statistic(input_paths) do |statistic|
+            timestamp = statistic.start_time.strftime("%Y-%m-%d %H:%M:%S.%6N")
+            puts("#{timestamp}:#{statistic.raw_command}")
+          end
+          true
+        end
+
+        private
+        def create_parser
+          parser = OptionParser.new
+          parser.version = VERSION
+          parser.banner += " QUERY_LOG"
+
+          parser.separator("")
+          parser.separator("Options:")
+
+          parser.on("--base-time=TIME",
+                    "Show running queries at TIME",
+                    "You can use popular time format for TIME such as W3C-DTF",
+                    "(now)") do |base_time|
+            @base_time = Time.parse(base_time)
+          end
+        end
+
+        def each_parsing_statistic(input_paths)
+          parser = Parser.new
+          catch do |tag|
+            input_paths.each do |input_path|
+              File.open(input_path) do |input|
+                parser.parse(input) do |statistic|
+                  next if @base_time.nil?
+                  next if statistic.start_time < @base_time
+                  if statistic.start_time == @base_time
+                    yield(statistic)
+                  end
+                  throw(tag)
+                end
+              end
+            end
+          end
+          parser.parsing_statistics.each do |statistic|
+            yield(statistic)
+          end
+        end
+      end
+    end
+end

data/lib/{groonga/query-log/command/replay.rb → groonga-query-log/command/verify-server.rb}

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2013-2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -18,23 +16,25 @@
 
 require "optparse"
 
-require "groonga/query-log/version"
-require "groonga/query-log/replayer"
+require "groonga-query-log"
 
-module Groonga
-  module QueryLog
+module GroongaQueryLog
     module Command
-      class Replay
+      class VerifyServer
         def initialize
-          @options = Replayer::Options.new
+          @options = ServerVerifier::Options.new
         end
 
-        def run(command_line)
+        def run(command_line, &callback)
           input_paths = create_parser.parse(command_line)
-          replayer = Replayer.new(@options)
-          input_paths.each do |input_path|
-            File.open(input_path) do |input|
-              replayer.replay(input)
+          verifier = ServerVerifier.new(@options)
+          if input_paths.empty?
+            verifier.verify($stdin, &callback)
+          else
+            input_paths.each do |input_path|
+              File.open(input_path) do |input|
+                verifier.verify(input, &callback)
+              end
             end
           end
           true
@@ -44,29 +44,48 @@ module Groonga
         def create_parser
           parser = OptionParser.new
           parser.version = VERSION
-          parser.banner += " QUERY_LOG"
+          parser.banner += " QUERY_LOG1 QUERY_LOG2 ..."
 
           parser.separator("")
           parser.separator("Options:")
 
-          parser.on("--host=HOST",
-                    "Host name or IP address of groonga server",
-                    "[#{@options.host}]") do |host|
-            @options.host = host
+          available_protocols = [:gqtp, :http]
+          available_protocols_label = "[#{available_protocols.join(', ')}]"
+
+          parser.on("--groonga1-host=HOST",
+                    "Host name or IP address of Groonga server 1",
+                    "[#{@options.groonga1.host}]") do |host|
+            @options.groonga1.host = host
           end
 
-          parser.on("--port=PORT", Integer,
-                    "Port number of groonga server",
-                    "[#{@options.port}]") do |port|
-            @options.port = port
+          parser.on("--groonga1-port=PORT", Integer,
+                    "Port number of Groonga server 1",
+                    "[#{@options.groonga1.port}]") do |port|
+            @options.groonga1.port = port
           end
 
-          available_protocols = [:gqtp, :http]
-          available_protocols_label = "[#{available_protocols.join(', ')}]"
-          parser.on("--protocol=PROTOCOL", available_protocols,
-                    "Protocol of groonga server",
+          parser.on("--groonga1-protocol=PROTOCOL", available_protocols,
+                    "Protocol of Groonga server 1",
                     available_protocols_label) do |protocol|
-            @options.protocol = protocol
+            @options.groonga1.protocol = protocol
+          end
+
+          parser.on("--groonga2-host=HOST",
+                    "Host name or IP address of Groonga server 2",
+                    "[#{@options.groonga2.host}]") do |host|
+            @options.groonga2.host = host
+          end
+
+          parser.on("--groonga2-port=PORT", Integer,
+                    "Port number of Groonga server 2",
+                    "[#{@options.groonga2.port}]") do |port|
+            @options.groonga2.port = port
+          end
+
+          parser.on("--groonga2-protocol=PROTOCOL", available_protocols,
+                    "Protocol of Groonga server 2",
+                    available_protocols_label) do |protocol|
+            @options.groonga2.protocol = protocol
           end
 
           parser.on("--n-clients=N", Integer,
@@ -102,19 +121,31 @@ module Groonga
             @options.target_command_names = names
           end
 
-          parser.on("--output-requests=PATH",
-                    "Output requests to PATH",
-                    "[not output]") do |path|
-            @options.requests_path = path
+          parser.on("--no-care-order",
+                    "Don't care order of select response records") do
+            @options.care_order = false
+          end
+
+          parser.on("--output=PATH",
+                    "Output results to PATH",
+                    "[stdout]") do |path|
+            @options.output_path = path
           end
 
-          parser.on("--output-responses=PATH",
-                    "Output responses to PATH",
-                    "[not output]") do |path|
-            @options.responses_path = path
+          parser.on("--[no-]verify-cache",
+                    "Verify cache for each query.",
+                    "[#{@options.verify_cache?}]") do |verify_cache|
+            @options.verify_cache = verify_cache
+          end
+
+          parser.separator("Debug options:")
+          parser.separator("")
+
+          parser.on("--abort-on-exception",
+                    "Abort on exception in threads") do
+            Thread.abort_on_exception = true
           end
         end
       end
     end
-  end
 end

data/lib/{groonga/query-log → groonga-query-log}/incompatibility-detector.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2014-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -16,10 +16,9 @@
 
 require "fileutils"
 
-require "groonga/query-log/parser"
+require "groonga-query-log/parser"
 
-module Groonga
-  module QueryLog
+module GroongaQueryLog
     class IncompatibilityDetector
       attr_reader :version
       def initialize(version)
@@ -94,5 +93,4 @@ module Groonga
         end
       end
     end
-  end
 end

data/lib/{groonga/query-log → groonga-query-log}/memory-leak-detector.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -20,10 +18,9 @@ require "time"
 
 require "groonga/client"
 
-require "groonga/query-log/parser"
+require "groonga-query-log/parser"
 
-module Groonga
-  module QueryLog
+module GroongaQueryLog
     class MemoryLeakDetector
       def initialize(options)
         @options = options
@@ -122,5 +119,4 @@ module Groonga
         end
       end
     end
-  end
 end

data/lib/groonga-query-log/parser.rb

@@ -0,0 +1,173 @@
+# Copyright (C) 2011-2018  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga-query-log/statistic"
+
+module GroongaQueryLog
+  class Parser
+    PATTERN =
+      /\A(?<year>\d{4})-(?<month>\d\d)-(?<day>\d\d)
+         \ (?<hour>\d\d):(?<minute>\d\d):(?<second>\d\d)\.(?<microsecond>\d+)
+         \|(?<context_id>.+?)
+         \|(?<type>[>:<])/x
+
+    class << self
+      def target_line?(line)
+        if line.respond_to?(:valid_encoding?)
+          return false unless line.valid_encoding?
+        end
+
+        return false unless PATTERN.match(line)
+
+        true
+      end
+    end
+
+    attr_reader :current_path
+    def initialize(options={})
+      @options = options
+      @slow_operation_threshold = options[:slow_operation_threshold]
+      @slow_response_threshold = options[:slow_response_threshold]
+      @target_commands = options[:target_commands]
+      @target_tables = options[:target_tables]
+      @parsing_statistics = {}
+
+      @current_path = nil
+    end
+
+    # Parses query-log file as stream to
+    # {GroongaQueryLog::Analyzer::Statistic}s including some
+    # informations for each query.
+    #
+    # @param [IO] input IO for input query log file.
+    # @yield [statistics] if a block is specified, it is called
+    #   every time a query is finished parsing.
+    # @yieldparam [GroongaQueryLog::Statistic] statistic
+    #   statistics of each query in log files.
+    def parse(input, &block)
+      return to_enum(__method__, input) unless block_given?
+
+      input.each_line do |line|
+        next unless line.valid_encoding?
+
+        match_data = PATTERN.match(line)
+        next if match_data.nil?
+
+        year = match_data[:year].to_i
+        month = match_data[:month].to_i
+        day = match_data[:day].to_i
+        hour = match_data[:hour].to_i
+        minute = match_data[:minute].to_i
+        second = match_data[:second].to_i
+        microsecond = match_data[:microsecond].to_i
+        context_id = match_data[:context_id]
+        type = match_data[:type]
+        rest = match_data.post_match.strip
+        time_stamp = Time.local(year,
+                                month,
+                                day,
+                                hour,
+                                minute,
+                                second,
+                                microsecond)
+        parse_line(time_stamp, context_id, type, rest, &block)
+      end
+    end
+
+    def parse_paths(paths, &block)
+      return to_enum(__method__, paths) unless block_given?
+
+      target_paths = GroongaLog::Parser.sort_paths(paths)
+      target_paths.each do |path|
+        GroongaLog::Input.open(path) do |log|
+          @current_path = path
+          begin
+            parse(log, &block)
+          ensure
+            @current_path = nil
+          end
+        end
+      end
+    end
+
+    def parsing_statistics
+      @parsing_statistics.values
+    end
+
+    private
+    def parse_line(time_stamp, context_id, type, rest, &block)
+      case type
+      when ">"
+        return if rest.empty?
+        statistic = create_statistic(context_id)
+        statistic.start(time_stamp, rest)
+        @parsing_statistics[context_id] = statistic
+      when ":"
+        return unless /\A(\d+) (.+)\((\d+)\)(\[.+\])?(?:: (.*))?/ =~ rest
+        elapsed = $1
+        name = $2
+        name += $4 if $4
+        n_records = $3.to_i
+        extra = $5
+        statistic = @parsing_statistics[context_id]
+        return if statistic.nil?
+        statistic.add_operation(:name => name,
+                                :elapsed => elapsed.to_i,
+                                :n_records => n_records,
+                                :extra => extra)
+      when "<"
+        return unless /\A(\d+) rc=(-?\d+)/ =~ rest
+        elapsed = $1
+        return_code = $2
+        statistic = @parsing_statistics.delete(context_id)
+        return if statistic.nil?
+        statistic.finish(elapsed.to_i, return_code.to_i)
+        return unless target_statistic?(statistic)
+        block.call(statistic)
+      end
+    end
+
+    def create_statistic(context_id)
+      statistic = Statistic.new(context_id)
+      if @slow_operation_threshold
+        statistic.slow_operation_threshold = @slow_operation_threshold
+      end
+      if @slow_response_threshold
+        statistic.slow_response_threshold = @slow_response_threshold
+      end
+      statistic
+    end
+
+    def target_statistic?(statistic)
+      if @target_commands
+        unless @target_commands.include?(statistic.command.name)
+          return false
+        end
+      end
+
+      if @target_tables
+        table = statistic.command["table"]
+        return false if table.nil?
+
+        unless @target_tables.include?(table)
+          return false
+        end
+      end
+
+      true
+    end
+  end
+end