groonga-query-log 1.0.1 → 1.0.2

data/lib/groonga/query-log/parser.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2011-2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2013  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -38,6 +38,7 @@ module Groonga
       def parse(input, &block)
         current_statistics = {}
         input.each_line do |line|
+          next unless line.valid_encoding?
           case line
           when /\A(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)\.(\d+)\|(.+?)\|([>:<])/
             year, month, day, hour, minutes, seconds, micro_seconds =
@@ -72,7 +73,7 @@ module Groonga
                                   :elapsed => elapsed.to_i,
                                   :n_records => n_records)
         when "<"
-          return unless /\A(\d+) rc=(\d+)/ =~ rest
+          return unless /\A(\d+) rc=(-?\d+)/ =~ rest
           elapsed = $1
           return_code = $2
           statistic = current_statistics.delete(context_id)

data/lib/groonga/query-log/replayer.rb

@@ -0,0 +1,196 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "time"
+require "thread"
+
+require "groonga/client"
+
+require "groonga/query-log/parser"
+
+module Groonga
+  module QueryLog
+    class Replayer
+      def initialize(options)
+        @options = options
+        @queue = SizedQueue.new(@options.request_queue_size)
+        @responses = Queue.new
+      end
+
+      def replay(input)
+        producer = run_producer(input)
+        consumers = run_consumers
+        response_logger = run_response_logger
+        producer.join
+        consumers.each(&:join)
+        response_logger.join
+      end
+
+      private
+      def run_producer(input)
+        Thread.new do
+          parser = Parser.new
+          id = 0
+          @options.create_request_output do |output|
+            parser.parse(input) do |statistic|
+              next unless target_command?(statistic.command)
+              # TODO: validate orignal_source is one line
+              output.puts(statistic.command.original_source)
+              @queue.push([id, statistic])
+              id += 1
+            end
+          end
+          @options.n_clients.times do
+            @queue.push(nil)
+          end
+        end
+      end
+
+      def run_consumers
+        @options.n_clients.times.collect do
+          Thread.new do
+            loop do
+              break if run_consumer
+            end
+          end
+        end
+      end
+
+      def run_consumer
+        @options.create_client do |client|
+          loop do
+            id, statistic = @queue.pop
+            if id.nil?
+              @responses.push(nil)
+              return true
+            end
+            begin
+              replay_command(client, id, statistic.command)
+            rescue Groonga::Client::Connection::Error
+              # TODO: add error log mechanism
+              $stderr.puts(Time.now.iso8601)
+              $stderr.puts(statistic.command.original_source)
+              $stderr.puts($!.raw_error.message)
+              $stderr.puts($!.raw_error.backtrace)
+              return false
+            end
+          end
+        end
+      end
+
+      def replay_command(client, id, command)
+        command["cache"] = "no" if @options.disable_cache?
+        response = client.execute(command)
+        @responses.push(response)
+      end
+
+      def run_response_logger
+        Thread.new do
+          @options.create_responses_output do |output|
+            loop do
+              response = @responses.pop
+              break if response.nil?
+              # TODO: ensure response is one line
+              # TODO: reorder by ID
+              output.puts(response.raw)
+            end
+          end
+        end
+      end
+
+      def target_command?(command)
+        @options.target_command_name?(command.name)
+      end
+
+      class NullOutput
+        class << self
+          def open
+            output = new
+            if block_given?
+              yield(output)
+            else
+              output
+            end
+          end
+        end
+
+        def puts(string)
+        end
+      end
+
+      class Options
+        attr_accessor :host
+        attr_accessor :port
+        attr_accessor :protocol
+        attr_accessor :n_clients
+        attr_writer :request_queue_size
+        attr_accessor :target_command_names
+        def initialize
+          @host = "127.0.0.1"
+          @port = 10041
+          @protocol = :gqtp
+          @n_clients = 8
+          @request_queue_size = nil
+          @disable_cache = false
+          @requests_path = nil
+          @responses_path = nil
+          @target_command_names = ["*"]
+        end
+
+        def create_client(&block)
+          Groonga::Client.open(:host     => @host,
+                               :port     => @port,
+                               :protocol => @protocol,
+                               &block)
+        end
+
+        def create_request_output(&block)
+          if @requests_path
+            File.open(@requests_path, "w", &block)
+          else
+            NullOutput.open(&block)
+          end
+        end
+
+        def create_responses_output(&block)
+          if @responses_path
+            File.open(@responses_path, "w", &block)
+          else
+            NullOutput.open(&block)
+          end
+        end
+
+        def request_queue_size
+          @request_queue_size || @n_clients * 3
+        end
+
+        def disable_cache?
+          @disable_cache
+        end
+
+        def target_command_name?(name)
+          @target_command_names.any? do |name_pattern|
+            flags = 0
+            flags |= File::FNM_EXTGLOB if File.const_defined?(:FNM_EXTGLOB)
+            File.fnmatch(name_pattern, name, flags)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/groonga/query-log/version.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2013  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -18,6 +18,6 @@
 
 module Groonga
   module QueryLog
-    VERSION = "1.0.1"
+    VERSION = "1.0.2"
   end
 end

data/test/run-test.rb

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 #
-# Copyright (C) 2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2013  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -18,20 +18,35 @@
 
 $VERBOSE = true
 
-$KCODE = "u" if RUBY_VERSION < "1.9"
+require "pathname"
 
-base_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
-lib_dir = File.join(base_dir, "lib")
-test_dir = File.join(base_dir, "test")
+base_dir = Pathname.new(__FILE__).dirname.parent.expand_path
+top_dir = base_dir.parent
+
+gqtp_base_dir = top_dir + "gqtp"
+gqtp_lib_dir = gqtp_base_dir + "lib"
+$LOAD_PATH.unshift(gqtp_lib_dir.to_s)
+
+groonga_client_base_dir = top_dir + "groonga-client"
+groonga_client_lib_dir = groonga_client_base_dir + "lib"
+$LOAD_PATH.unshift(groonga_client_lib_dir.to_s)
+
+groonga_command_base_dir = top_dir + "groonga-command"
+groonga_command_lib_dir = groonga_command_base_dir + "lib"
+$LOAD_PATH.unshift(groonga_command_lib_dir.to_s)
+
+lib_dir = base_dir + "lib"
+test_dir = base_dir + "test"
 
 require "test-unit"
 require "test/unit/notify"
+require "test/unit/rr"
 
 Test::Unit::Priority.enable
 
-$LOAD_PATH.unshift(lib_dir)
+$LOAD_PATH.unshift(lib_dir.to_s)
 
-$LOAD_PATH.unshift(test_dir)
+$LOAD_PATH.unshift(test_dir.to_s)
 require "groonga-query-log-test-utils"
 
 Dir.glob("#{base_dir}/test/**/test{_,-}*.rb") do |file|

data/test/test-parser.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2011-2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2013  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -18,16 +18,27 @@
 
 class ParserTest < Test::Unit::TestCase
   def test_load
-    @log = <<-EOL
+    statistics = parse(<<-LOG)
 2012-12-13 11:15:21.628105|0x7fff148c8a50|>load --table Video
 2012-12-13 11:15:21.645119|0x7fff148c8a50|<000000017041150 rc=0
-EOL
+    LOG
+    parsed_command = statistics.first.command
+    assert_instance_of(Groonga::Command::Load, parsed_command)
+  end
+
+  def test_ignore_invalid_line
+    garbage = "\x80"
+    statistics = parse(<<-LOG)
+2012-12-13 11:15:20.628105|0x7fff148c8a50|>#{garbage}
+2012-12-13 11:15:21.628105|0x7fff148c8a50|>load --table Video
+2012-12-13 11:15:21.645119|0x7fff148c8a50|<000000017041150 rc=0
+    LOG
     parsed_command = statistics.first.command
     assert_instance_of(Groonga::Command::Load, parsed_command)
   end
 
   private
-  def statistics
+  def parse(log)
     statistics = []
     parser = Groonga::QueryLog::Parser.new
     parser.parse(StringIO.new(log)) do |statistic|
@@ -36,13 +47,9 @@ EOL
     statistics
   end
 
-  def log
-    @log
-  end
-
   class StatisticOperationTest < self
     def setup
-      @log = <<-EOL
+      @statistics = parse(<<-LOG)
 2011-06-02 16:27:04.731685|5091e5c0|>/d/select.join?table=Entries&filter=local_name+%40+%22gsub%22+%26%26+description+%40+%22string%22&sortby=_score&output_columns=_key&drilldown=name,class
 2011-06-02 16:27:04.733539|5091e5c0|:000000001849451 filter(15)
 2011-06-02 16:27:04.734978|5091e5c0|:000000003293459 filter(13)
@@ -52,11 +59,12 @@ EOL
 2011-06-02 16:27:04.735606|5091e5c0|:000000003921419 drilldown(3)
 2011-06-02 16:27:04.735762|5091e5c0|:000000004077552 drilldown(2)
 2011-06-02 16:27:04.735808|5091e5c0|<000000004123726 rc=0
-EOL
+      LOG
+      @statistic = @statistics.first
     end
 
     def test_context
-      operations = statistics.first.operations.collect do |operation|
+      operations = @statistic.operations.collect do |operation|
         [operation[:name], operation[:context]]
       end
       expected = [
@@ -72,7 +80,7 @@ EOL
     end
 
     def test_n_records
-      operations = statistics.first.operations.collect do |operation|
+      operations = @statistic.operations.collect do |operation|
         [operation[:name], operation[:n_records]]
       end
       expected = [
@@ -87,4 +95,22 @@ EOL
       assert_equal(expected, operations)
     end
   end
+
+  class TestRC < self
+    def test_success
+      statistics = parse(<<-LOG)
+2012-12-13 11:15:21.628105|0x7fff148c8a50|>table_create --name Videos
+2012-12-13 11:15:21.645119|0x7fff148c8a50|<000000017041150 rc=0
+      LOG
+      assert_equal([0], statistics.collect(&:return_code))
+    end
+
+    def test_fialure
+      statistics = parse(<<-LOG)
+2012-12-13 11:15:21.628105|0x7fff148c8a50|>table_create --name Videos
+2012-12-13 11:15:21.645119|0x7fff148c8a50|<000000017041150 rc=-22
+      LOG
+      assert_equal([-22], statistics.collect(&:return_code))
+    end
+  end
 end

data/test/test-replayer.rb

@@ -0,0 +1,119 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+class ReplayerTest < Test::Unit::TestCase
+  class OptionTest < self
+    class ClientTest < self
+      def setup
+        @options = Groonga::QueryLog::Replayer::Options.new
+        @options.n_clients = 1
+      end
+
+      def test_host
+        host = "example.com"
+        @options.host = host
+        mock_client_open(:host => host)
+        replay
+      end
+
+      def test_port
+        port = 2929
+        @options.port = 2929
+        mock_client_open(:port => 2929)
+        replay
+      end
+
+      private
+      def replay
+        replayer = Groonga::QueryLog::Replayer.new(@options)
+        replayer.replay(StringIO.new(""))
+      end
+
+      def mock_client_open(expected_options)
+        client = Object.new
+        default_options = {
+          :host     => "127.0.0.1",
+          :port     => 10041,
+          :protocol => :gqtp,
+        }
+        expected_open_options = default_options.merge(expected_options)
+        mock(Groonga::Client).open(expected_open_options).yields(client) do
+          client
+        end
+      end
+    end
+
+    class TargetCommandNameTest < self
+      def setup
+        @options = Groonga::QueryLog::Replayer::Options.new
+      end
+
+      def test_default
+        assert_true(@options.target_command_name?("shutdown"))
+      end
+
+      class GlobTest < self
+        def setup
+          super
+          @options.target_command_names = ["se*"]
+        end
+
+        def test_match
+          assert_true(@options.target_command_name?("select"))
+        end
+
+        def test_not_match
+          assert_false(@options.target_command_name?("status"))
+        end
+      end
+
+      class ExtGlobTest < self
+        def setup
+          super
+          @options.target_command_names = ["s{elect,tatus}"]
+          unless File.const_defined?(:FNM_EXTGLOB)
+            omit("File:::FNM_EXTGLOB (Ruby 2.0.0 or later) is required.")
+          end
+        end
+
+        def test_match
+          assert_true(@options.target_command_name?("select"))
+        end
+
+        def test_not_match
+          assert_false(@options.target_command_name?("selectX"))
+        end
+      end
+
+      class ExactMatchTest < self
+        def setup
+          super
+          @options.target_command_names = ["select"]
+        end
+
+        def test_match
+          assert_true(@options.target_command_name?("select"))
+        end
+
+        def test_not_match
+          assert_false(@options.target_command_name?("selectX"))
+        end
+      end
+    end
+  end
+end