groonga-query-log 1.0.1 → 1.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0f47438cd5625ffcd0af39cc2d693f46df36fdd6
+  data.tar.gz: 039a16dd1e43c33fb2b9e6645152d20db726c0c0
+SHA512:
+  metadata.gz: 69c52b48f51b1fe85f193a3426b340ebff7adfffc8f4f15416310ebb40e8b42e1930d17caa1dd733e4f229dd74661ff445c3ff5821a3142967bdb59686a477d3
+  data.tar.gz: 433ff8fbfa3bfddaf220a6311022459925b2c92082982bc6e959618d3b57fd88f2291154ee3352900d46c1cc00d56012a5f30641d188e73821e9399113287a62

data/Gemfile

@@ -1,6 +1,6 @@
 # -*- mode: ruby; coding: utf-8 -*-
 #
-# Copyright (C) 2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2013  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -16,6 +16,6 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-source :rubygems
+source "https://rubygems.org/"
 
 gemspec

data/README.md

@@ -1,12 +1,14 @@
 # README
 
+[![Build Status](https://travis-ci.org/groonga/groonga-query-log.png?branch=master)](https://travis-ci.org/groonga/groonga-query-log)
+
 ## Name
 
 groonga-query-log
 
 ## Description
 
-Groonga-query-log is a collection of libarary and tools to process
+Groonga-query-log is a collection of library and tools to process
 [groonga](http://groonga.org/)'s query log. You can write a program to
 process query log by using groonga-query-log as a library. You can
 analyze your groonga's queries and test with your groonga's query log

data/Rakefile

@@ -34,7 +34,7 @@ spec = helper.gemspec
 
 Packnga::DocumentTask.new(spec) do |task|
   task.original_language = "en"
-  task.translate_languages = "ja"
+  task.translate_languages = ["ja"]
 end
 
 Packnga::ReleaseTask.new(spec) do

data/bin/groonga-query-log-detect-memory-leak

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga/query-log/command/detect-memory-leak"
+
+detect_memory_leak_command = Groonga::QueryLog::Command::DetectMemoryLeak.new
+detect_memory_leak_command.run(*ARGV)

data/bin/groonga-query-log-replay

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga/query-log/command/replay"
+
+replay_command = Groonga::QueryLog::Command::Replay.new
+replay_command.run(*ARGV)

data/doc/text/news.md

@@ -1,5 +1,20 @@
 # News
 
+## 1.0.2: 2013-11-01
+
+### Improvements
+
+  * [GitHub#1] Add Travis CI status image to README.
+    Patch by Kengo Suzuki. Thanks!!!
+  * Dropped Ruby 1.8 support.
+  * Added groonga-query-log-replay that replays queries in query log.
+  * Added groonga-query-log-detect-memory-leak that detects
+    a memory leak by executing each query in query log.
+
+### Thanks
+
+  * Kengo Suzuki
+
 ## 1.0.1: 2012-12-21
 
 ### Improvements

data/groonga-query-log.gemspec

@@ -1,6 +1,6 @@
 # -*- mode: ruby; coding: utf-8 -*-
 #
-# Copyright (C) 2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2013  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -52,10 +52,12 @@ Gem::Specification.new do |spec|
   spec.licenses = ["LGPLv2.1+"]
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency("groonga-command")
+  spec.add_runtime_dependency("groonga-command-parser")
+  spec.add_runtime_dependency("groonga-client")
 
   spec.add_development_dependency("test-unit")
   spec.add_development_dependency("test-unit-notify")
+  spec.add_development_dependency("test-unit-rr")
   spec.add_development_dependency("rake")
   spec.add_development_dependency("bundler")
   spec.add_development_dependency("packnga")

data/lib/groonga/query-log.rb

@@ -20,3 +20,4 @@ require "groonga/query-log/version"
 require "groonga/query-log/analyzer"
 require "groonga/query-log/extractor"
 require "groonga/query-log/parser"
+require "groonga/query-log/replayer"

data/lib/groonga/query-log/analyzer.rb

@@ -174,9 +174,9 @@ module Groonga
 
           parser.on("--slow-response-threshold=THRESHOLD",
                     Float,
-                    "Use THRESHOLD seconds to detect slow operations.",
-                    "(#{@options[:sloq_response_threshold]})") do |threshold|
-            @options[:sloq_response_threshold] = threshold
+                    "Use THRESHOLD seconds to detect slow responses.",
+                    "(#{@options[:slow_response_threshold]})") do |threshold|
+            @options[:slow_response_threshold] = threshold
           end
 
           available_reporters = ["console", "json", "html"]

data/lib/groonga/query-log/analyzer/reporter/html.rb

@@ -54,6 +54,7 @@ module Groonga
 
         def report_statistic(statistic)
           command = statistic.command
+          _ = command # XXX: suppress warning
           statistic_html = erb(<<-EOH, __LINE__ + 1, binding)
       <div class="statistic-heading">
         <h3>Command</h3>

data/lib/groonga/query-log/analyzer/statistic.rb

@@ -16,7 +16,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "groonga/command"
+require "groonga/command/parser"
 
 module Groonga
   module QueryLog

data/lib/groonga/query-log/command/detect-memory-leak.rb

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "optparse"
+
+require "groonga/query-log/memory-leak-detector"
+
+module Groonga
+  module QueryLog
+    module Command
+      class DetectMemoryLeak
+        def initialize
+          @options = MemoryLeakDetector::Options.new
+        end
+
+        def run(*command_line)
+          input_paths = create_parser.parse(*command_line)
+          detector = MemoryLeakDetector.new(@options)
+          input_paths.each do |input_path|
+            File.open(input_path) do |input|
+              detector.detect(input)
+            end
+          end
+        end
+
+        private
+        def create_parser
+          parser = OptionParser.new
+          parser.banner += " QUERY_LOG"
+
+          parser.separator("")
+          parser.separator("Options:")
+
+          parser.on("--host=HOST",
+                    "Host name or IP address of groonga server",
+                    "[#{@options.host}]") do |host|
+            @options.host = host
+          end
+
+          parser.on("--port=PORT", Integer,
+                    "Port number of groonga server",
+                    "[#{@options.port}]") do |port|
+            @options.port = port
+          end
+
+          available_protocols = [:gqtp, :http]
+          available_protocols_label = "[#{available_protocols.join(', ')}]"
+          parser.on("--protocol=PROTOCOL", available_protocols,
+                    "Protocol of groonga server",
+                    available_protocols_label) do |protocol|
+            @options.protocol = protocol
+          end
+
+          parser.on("--pid=PID",
+                    "The PID of groonga server",
+                    "[#{@options.pid}]") do |pid|
+            @options.pid = pid
+          end
+
+          parser.on("--n-tries=N", Integer,
+                    "The number of the same request tries",
+                    "[#{@options.n_tries}]") do |n|
+            @options.n_tries = n
+          end
+
+          parser.on("--[no-]force-disable-cache",
+                    "Force disable cache of select command by cache=no parameter",
+                    "[#{@options.force_disable_cache?}]") do |boolean|
+            @options.force_disable_cache = boolean
+          end
+        end
+      end
+    end
+  end
+end

data/lib/groonga/query-log/command/replay.rb

@@ -0,0 +1,117 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "optparse"
+
+require "groonga/query-log/replayer"
+
+module Groonga
+  module QueryLog
+    module Command
+      class Replay
+        def initialize
+          @options = Replayer::Options.new
+        end
+
+        def run(*command_line)
+          input_paths = create_parser.parse(*command_line)
+          replayer = Replayer.new(@options)
+          input_paths.each do |input_path|
+            File.open(input_path) do |input|
+              replayer.replay(input)
+            end
+          end
+        end
+
+        private
+        def create_parser
+          parser = OptionParser.new
+          parser.banner += " QUERY_LOG"
+
+          parser.separator("")
+          parser.separator("Options:")
+
+          parser.on("--host=HOST",
+                    "Host name or IP address of groonga server",
+                    "[#{@options.host}]") do |host|
+            @options.host = host
+          end
+
+          parser.on("--port=PORT", Integer,
+                    "Port number of groonga server",
+                    "[#{@options.port}]") do |port|
+            @options.port = port
+          end
+
+          available_protocols = [:gqtp, :http]
+          available_protocols_label = "[#{available_protocols.join(', ')}]"
+          parser.on("--protocol=PROTOCOL", available_protocols,
+                    "Protocol of groonga server",
+                    available_protocols_label) do |protocol|
+            @options.protocol = protocol
+          end
+
+          parser.on("--n-clients=N", Integer,
+                    "The max number of concurrency",
+                    "[#{@options.n_clients}]") do |n_clients|
+            @options.n_cilents = n_clients
+          end
+
+          parser.on("--request-queue-size=SIZE", Integer,
+                    "The size of request queue",
+                    "[auto]") do |size|
+            @options.request_queue_size = size
+          end
+
+          parser.on("--disable-cache",
+                    "Add 'cache=no' parameter to request",
+                    "[#{@options.disable_cache?}]") do
+            @options.disable_cache = true
+          end
+
+          parser.on("--target-command-name=NAME",
+                    "Add NAME to target command names",
+                    "You can specify this option zero or more times",
+                    "See also --target-command-names") do |name|
+            @options.target_command_names << name
+          end
+
+          target_command_names_label = @options.target_command_names.join(", ")
+          parser.on("--target-command-names=NAME1,NAME2,...", Array,
+                    "Replay only NAME1,NAME2,... commands",
+                    "You can use glob to choose command name",
+                    "[#{target_command_names_label}]") do |names|
+            @options.target_command_names = names
+          end
+
+          parser.on("--output-requests=PATH",
+                    "Output requests to PATH",
+                    "[not output]") do |path|
+            @options.requests_path = path
+          end
+
+          parser.on("--output-responses=PATH",
+                    "Output responses to PATH",
+                    "[not output]") do |path|
+            @options.responses_path = path
+          end
+        end
+      end
+    end
+  end
+end

data/lib/groonga/query-log/extractor.rb

@@ -170,15 +170,15 @@ module Groonga
 
       def target?(command)
         name = command.name
-        commands = @options.commands
+        target_commands = @options.commands
         exclude_commands = @options.exclude_commands
 
-        unless commands.empty?
-          return commands.any? {|command| command === name}
+        unless target_commands.empty?
+          return target_commands.any? {|target_command| target_command === name}
         end
 
         unless exclude_commands.empty?
-          return (not exclude_commands.any? {|command| command === name})
+          return (not exclude_commands.any? {|exclude_command| exclude_command === name})
         end
 
         true

data/lib/groonga/query-log/memory-leak-detector.rb

@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "time"
+
+require "groonga/client"
+
+require "groonga/query-log/parser"
+
+module Groonga
+  module QueryLog
+    class MemoryLeakDetector
+      def initialize(options)
+        @options = options
+      end
+
+      def detect(input)
+        each_command(input) do |command|
+          @options.create_client do |client|
+            begin
+              check_command(client, command)
+            rescue Groonga::Client::Connection::Error
+              # TODO: add error log mechanism
+              $stderr.puts(Time.now.iso8601)
+              $stderr.puts(command.original_source)
+              $stderr.puts($!.raw_error.message)
+              $stderr.puts($!.raw_error.backtrace)
+            end
+          end
+        end
+      end
+
+      private
+      def each_command(input)
+        parser = Parser.new
+        parser.parse(input) do |statistic|
+          yield(statistic.command)
+        end
+      end
+
+      def check_command(client, command)
+        command["cache"] = "no" if @options.force_disable_cache?
+        current_n_allocations = nil
+        @options.n_tries.times do |i|
+          client.execute(command)
+          previous_n_allocations = current_n_allocations
+          current_n_allocations = n_allocations(client)
+          next if previous_n_allocations.nil?
+          if current_n_allocations > previous_n_allocations
+            max_n_digits = [
+              compute_n_digits(previous_n_allocations),
+              compute_n_digits(current_n_allocations),
+            ].max
+            puts("detect a memory leak:")
+            puts("Nth try: #{i}")
+            puts("previous: %*d" % [max_n_digits, previous_n_allocations])
+            puts(" current: %*d" % [max_n_digits, current_n_allocations])
+            puts(command.original_source)
+          end
+        end
+      end
+
+      def n_allocations(client)
+        client.status.n_allocations
+      end
+
+      def compute_n_digits(n)
+        (Math.log10(n) + 1).floor
+      end
+
+      class Options
+        attr_accessor :host
+        attr_accessor :port
+        attr_accessor :protocol
+        attr_accessor :pid
+        attr_accessor :n_tries
+        attr_writer :force_disable_cache
+        def initialize
+          @host = "127.0.0.1"
+          @port = 10041
+          @protocol = :gqtp
+          @pid = guess_groonga_server_pid
+          @n_tries = 10
+          @force_disable_cache = true
+        end
+
+        def force_disable_cache?
+          @force_disable_cache
+        end
+
+        def create_client(&block)
+          Groonga::Client.open(:host     => @host,
+                               :port     => @port,
+                               :protocol => @protocol,
+                               &block)
+        end
+
+        private
+        def guess_groonga_server_pid
+          # This command line works only for ps by procps.
+          pid = `ps -o pid --no-header -C groonga`.strip
+          if pid.empty?
+            nil
+          else
+            pid.to_i
+          end
+        end
+      end
+    end
+  end
+end