grntest 1.1.1 → 1.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9681ce146649a19fe0099c41cac952e0c53c964c
-  data.tar.gz: accc76ff8605c28f26a07014082523f7d30134ec
+  metadata.gz: 7da2ff4d46c711c2bc7bf534ed0478fc4bb9a510
+  data.tar.gz: 7734142d71d8b3d0375e9275eb168fac6df23932
 SHA512:
-  metadata.gz: 210cd594bdfb36425deb5d28084acbf798bfd765021c002f7c00fb54d0ee64c7f69a335f269f9f4ec5229e8930692cccecabc74929004e24a66e9d7813f7615e
-  data.tar.gz: 51151bee659689a783ee6eed2f9c8d8fc48e1a71245cb9ec15c8cd8a9cb48a9c3bbff2b276e402a9d1abec46f740a6760d9d95daeb39b74d9152b6aae87f805f
+  metadata.gz: d17ae23c5d7342c7aae95d96bf2c8e271661860232b4766d7397250c1276b7984d1be6b2d415f30639b5a1f2a0b4be8705c6202973bd626afdc4b38bdce63994
+  data.tar.gz: e35c94ee423590de22f75f33406f6728ce6ff016d08001aac49645dd1b8d7f74c7d68d0ac14e0396209a20e4efc27284c62910b0a0516602668d0f4ee9602050

data/README.md

@@ -1,5 +1,7 @@
 # README
 
+[![Gem Version](https://badge.fury.io/rb/grntest.svg)](http://badge.fury.io/rb/grntest)
+
 ## Name
 
 grntest
@@ -215,6 +217,8 @@ Here are available `NAME` s:
 * `long-timeout`
 * `on-error`
 * `omit`
+* `add-important-log-levels`
+* `remove-important-log-levels`
 
 `ARGUMENTS...` are depends on directive. A directive doesn't require
 any arguments but a directive requires arguments.
@@ -429,6 +433,83 @@ Example:
 new_excelent_command
 ```
 
+#### `add-important-log-levels`
+
+Usage:
+
+```
+#@add-important-log-levels LEVEL_1 LEVEL_2 ...
+```
+
+Add `LEVEL_N` to important log level list. grntest outputs log
+messages when their log level is included in important log level
+list.
+
+The default important log level list has the following log levels:
+
+  * `emergency`
+  * `alert`
+  * `critical`
+  * `error`
+  * `warning`
+  * `notice`
+
+If you want to test log messages for `info`, `debug` or `dump` log
+levels, you need to add the log level to important log level list.
+
+Example:
+
+```
+# Enable logs for debug level
+log_level --level debug
+# Collect debug level log messages
+#@add-important-log-levels debug
+log_put --level debug --message "This is a message"
+# The following messages is collected by grntest
+#|d| This is a message
+```
+
+#### `remove-important-log-levels`
+
+Usage:
+
+```
+#@remove-important-log-levels LEVEL_1 LEVEL_2 ...
+```
+
+Remove `LEVEL_N` from important log level list. grntest outputs log
+messages when their log level is included in important log level
+list.
+
+The default important log level list has the following log levels:
+
+  * `emergency`
+  * `alert`
+  * `critical`
+  * `error`
+  * `warning`
+  * `notice`
+
+You can remove them from important log level list. You can only remove
+log levels added by `add-important-log-levels`.
+
+Example:
+
+```
+# Enable logs for debug level
+log_level --level debug
+# Collect debug level log messages
+#@add-important-log-levels debug
+log_put --level debug --message "This is a message"
+# The following messages is collected by grntest
+#|d| This is a message
+
+# Disable collecting debug level log messages
+#@remove-important-log-levels debug
+log_put --level debug --message "This is a message"
+# No message is collected by grntest
+```
+
 ## Options
 
 Grntest has many options. You don't need to specify many of them
@@ -554,7 +635,7 @@ has many test scripts and uses many useful features. They will help you.
 
 ## Dependencies
 
-* Ruby 1.9.3
+* Ruby
 * msgpack gem
 
 ## Mailing list

data/doc/text/news.md

@@ -1,5 +1,16 @@
 # News
 
+## 1.1.2: 2015-07-08
+
+### Improvements
+
+  * Improve HTTP server related error case handlings.
+  * Support Valgrind.
+  * Added `--stop-on-failure` that stops testing when one test is failed.
+  * Support JSONP.
+  * Add `add-important-log-levels` directive.
+  * Add `remove-important-log-levels` directive.
+
 ## 1.1.1: 2015-02-03
 
 ### Improvements

data/lib/grntest/executors/base-executor.rb

@@ -32,14 +32,15 @@ module Grntest
       end
 
       attr_reader :context
-      def initialize(context=nil)
+      def initialize(context)
         @loading = false
         @pending_command = ""
         @pending_load_command = nil
         @current_command_name = nil
         @output_type = nil
         @long_timeout = default_long_timeout
-        @context = context || ExecutionContext.new
+        @context = context
+        @custom_important_log_levels = []
       end
 
       def execute(script_path)
@@ -191,6 +192,24 @@ module Grntest
         @context.omit
       end
 
+      def execute_directive_add_important_log_levels(line, content, options)
+        log_levels = options.collect do |log_level|
+          normalize_log_level(log_level)
+        end
+        @custom_important_log_levels |= log_levels
+      end
+
+      def execute_directive_remove_important_log_levels(line, content, options)
+        log_levels = options.collect do |log_level|
+          normalize_log_level(log_level)
+        end
+        @custom_important_log_levels -= log_levels
+      end
+
+      def normalize_log_level(level)
+        level[0]
+      end
+
       def execute_directive(line, content)
         command, *options = Shellwords.split(content)
         case command
@@ -210,6 +229,10 @@ module Grntest
           execute_directive_on_error(line, content, options)
         when "omit"
           execute_directive_omit(line, content, options)
+        when "add-important-log-levels"
+          execute_directive_add_important_log_levels(line, content, options)
+        when "remove-important-log-levels"
+          execute_directive_remove_important_log_levels(line, content, options)
         else
           log_input(line)
           log_error("#|e| unknown directive: <#{command}>")
@@ -296,7 +319,8 @@ module Grntest
       end
 
       def important_log_level?(log_level)
-        ["E", "A", "C", "e", "w"].include?(log_level)
+        ["E", "A", "C", "e", "w"].include?(log_level) or
+          @custom_important_log_levels.include?(log_level)
       end
 
       def backtrace_log_message?(message)

data/lib/grntest/executors/http-executor.rb

@@ -22,11 +22,11 @@ require "grntest/executors/base-executor"
 module Grntest
   module Executors
     class HTTPExecutor < BaseExecutor
-      def initialize(host, port, context=nil)
+      def initialize(host, port, context, options={})
         super(context)
         @host = host
         @port = port
-        @read_timeout = 3
+        @read_timeout = options[:read_timeout] || 3
       end
 
       def send_command(command)
@@ -44,13 +44,16 @@ module Grntest
         rescue Error
           n_retried += 1
           sleep(0.1)
-          retry if n_retried < 50
+          retry if n_retried < 100
           raise
         end
       end
 
       def shutdown
-        send_command(command("shutdown"))
+        begin
+          send_command(command("shutdown"))
+        rescue Error
+        end
       end
 
       def create_sub_executor(context)

data/lib/grntest/executors/standard-io-executor.rb

@@ -20,7 +20,7 @@ require "grntest/executors/base-executor"
 module Grntest
   module Executors
     class StandardIOExecutor < BaseExecutor
-      def initialize(input, output, context=nil)
+      def initialize(input, output, context)
         super(context)
         @input = input
         @output = output

data/lib/grntest/test-runner.rb

@@ -228,16 +228,51 @@ module Grntest
         end
         command_line << @tester.gdb
         gdb_command_path = context.temporary_directory_path + "groonga.gdb"
-        File.open(gdb_command_path, "w") do |gdb_command|
-          gdb_command.puts(<<-EOC)
+        gdb_command_path.open("w") do |gdb_command|
+          gdb_command.puts(<<-COMMANDS)
 break main
 run
-print chdir("#{context.temporary_directory_path}")
-EOC
+call chdir("#{context.temporary_directory_path}")
+          COMMANDS
         end
         command_line << "--command=#{gdb_command_path}"
         command_line << "--quiet"
         command_line << "--args"
+      elsif @tester.valgrind
+        if libtool_wrapper?(command)
+          command_line << find_libtool(command)
+          command_line << "--mode=execute"
+        end
+        command_line << @tester.valgrind
+        command_line << "--leak-check=full"
+        command_line << "--show-reachable=yes"
+        command_line << "--track-origins=yes"
+        valgrind_suppressions_file_path =
+          context.temporary_directory_path + "groonga.supp"
+        valgrind_suppressions_file_path.open("w") do |suppressions|
+          suppressions.puts(<<-SUPPRESSIONS)
+{
+  dlopen
+  Memcheck:Leak
+  match-leak-kinds: reachable
+  ...
+  fun:dlopen*
+  ...
+}
+{
+  _dl_catch_error
+  Memcheck:Leak
+  match-leak-kinds: reachable
+  ...
+  fun:_dl_catch_error
+}
+          SUPPRESSIONS
+        end
+        command_line << "--suppressions=#{valgrind_suppressions_file_path}"
+        if @tester.valgrind_gen_suppressions?
+          command_line << "--gen-suppressions=all"
+        end
+        command_line << "--verbose"
       else
         spawn_options[:chdir] = context.temporary_directory_path.to_s
       end
@@ -282,10 +317,15 @@ EOC
       command_line = groonga_http_command(host, port, pid_file_path, context,
                                           spawn_options)
       pid = nil
+      shutdown_wait_timeout = 5
+      options = {}
+      if @tester.gdb
+        options[:read_timeout] = 60 * 10
+      end
       begin
         pid = Process.spawn(env, *command_line, spawn_options)
         begin
-          executor = Executors::HTTPExecutor.new(host, port, context)
+          executor = Executors::HTTPExecutor.new(host, port, context, options)
           begin
             executor.ensure_groonga_ready
           rescue
@@ -298,10 +338,15 @@ EOC
           end
           yield(executor)
         ensure
-          Process.kill(:TERM, pid)
-          wait_groonga_http_shutdown(pid_file_path)
+          executor.shutdown
+          if wait_groonga_http_shutdown(pid_file_path, shutdown_wait_timeout)
+            pid = nil if wait_pid(pid, shutdown_wait_timeout)
+          end
         end
       ensure
+        return if pid.nil?
+        Process.kill(:TERM, pid)
+        wait_groonga_http_shutdown(pid_file_path, shutdown_wait_timeout)
         ensure_process_finished(pid)
       end
     end
@@ -314,20 +359,34 @@ EOC
         finished_pid = Process.waitpid(pid, Process::WNOHANG)
         break if finished_pid
         n_retries += 1
-        break if n_retries > 10
+        break if n_retries > 100
         Process.kill(:TERM, pid)
         sleep(0.1)
       end
     end
 
-    def wait_groonga_http_shutdown(pid_file_path)
+    def wait_pid(pid, timeout)
+      total_sleep_time = 0
+      sleep_time = 0.1
+      loop do
+        return true if Process.waitpid(pid, Process::WNOHANG)
+        sleep(sleep_time)
+        total_sleep_time += sleep_time
+        return false if total_sleep_time > timeout
+      end
+    end
+
+    def wait_groonga_http_shutdown(pid_file_path, timeout)
+      return false unless pid_file_path.exist?
+
       total_sleep_time = 0
       sleep_time = 0.1
       while pid_file_path.exist?
         sleep(sleep_time)
         total_sleep_time += sleep_time
-        break if total_sleep_time > 1.0
+        break if total_sleep_time > timeout
       end
+      true
     end
 
     def groonga_http_command(host, port, pid_file_path, context, spawn_options)
@@ -402,6 +461,9 @@ http {
 
     def create_empty_database(db_path)
       output_fd = Tempfile.new("create-empty-database")
+      env = {
+        "GRN_FMALLOC_PROB" => nil,
+      }
       create_database_command = [
         @tester.groonga,
         "--output-fd", output_fd.to_i.to_s,
@@ -411,7 +473,7 @@ http {
       options = {
         output_fd.to_i => output_fd.to_i
       }
-      system(*create_database_command, options)
+      system(env, *create_database_command, options)
       output_fd.close(true)
     end
 
@@ -442,8 +504,17 @@ http {
       when "json", "msgpack"
         status = nil
         values = nil
+        content = content.chomp
+        if type == "json" and /\A([^(]+\()(.+)(\);)\z/ =~ content
+          jsonp = true
+          jsonp_start = $1
+          content = $2
+          jsonp_end = $3
+        else
+          jsonp = false
+        end
         begin
-          status, *values = ResponseParser.parse(content.chomp, type)
+          status, *values = ResponseParser.parse(content, type)
         rescue ParseError
           return $!.message
         end
@@ -454,7 +525,12 @@ http {
         if normalized_output.bytesize > @max_n_columns
           normalized_output = JSON.pretty_generate(normalized_output_content)
         end
-        normalize_raw_content(normalized_output)
+        normalized_raw_content = normalize_raw_content(normalized_output)
+        if jsonp
+          "#{jsonp_start}#{normalized_raw_content.chomp}#{jsonp_end}\n"
+        else
+          normalized_raw_content
+        end
       when "xml"
         normalized_xml = normalize_output_xml(content, options)
         normalize_raw_content(normalized_xml)

data/lib/grntest/test-suites-runner.rb

@@ -64,6 +64,13 @@ module Grntest
       collect_count(:n_not_checked_tests)
     end
 
+    def have_failure?
+      @workers.any? do |worker|
+        worker.result.n_failed_tests > 0 or
+          worker.result.n_leaked_tests > 0
+      end
+    end
+
     private
     def collect_count(item)
       counts = @workers.collect do |worker|

data/lib/grntest/tester.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2012-2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2015  Kouhei Sutou <kou@clear-code.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -164,12 +164,30 @@ module Grntest
           tester.gdb = command || tester.default_gdb
         end
 
+        parser.on("--valgrind[=COMMAND]",
+                  "Run groonga on valgrind and use COMMAND as valgrind",
+                  "(#{tester.default_valgrind})") do |command|
+          tester.valgrind = command || tester.default_valgrind
+        end
+
+        parser.on("--[no-]valgrind-gen-suppressions",
+                  "Generate suppressions for Valgrind",
+                  "(#{tester.valgrind_gen_suppressions?})") do |boolean|
+          tester.valgrind_gen_suppressions = boolean
+        end
+
         parser.on("--[no-]keep-database",
                   "Keep used database for debug after test is finished",
                   "(#{tester.keep_database?})") do |boolean|
           tester.keep_database = boolean
         end
 
+        parser.on("--[no-]stop-on-failure",
+                  "Stop immediately on the first non success test",
+                  "(#{tester.stop_on_failure?})") do |boolean|
+          tester.stop_on_failure = boolean
+        end
+
         parser.on("--output=OUTPUT",
                   "Output to OUTPUT",
                   "(stdout)") do |output|
@@ -206,7 +224,10 @@ module Grntest
     attr_accessor :n_workers
     attr_accessor :output
     attr_accessor :gdb, :default_gdb
+    attr_accessor :valgrind, :default_valgrind
+    attr_writer :valgrind_gen_suppressions
     attr_writer :reporter, :keep_database, :use_color
+    attr_writer :stop_on_failure
     attr_reader :test_patterns, :test_suite_patterns
     attr_reader :exclude_test_patterns, :exclude_test_suite_patterns
     def initialize
@@ -223,12 +244,14 @@ module Grntest
       @output = $stdout
       @keep_database = false
       @use_color = nil
+      @stop_on_failure = false
       @test_patterns = []
       @test_suite_patterns = []
       @exclude_test_patterns = []
       @exclude_test_suite_patterns = []
       detect_suitable_diff
       initialize_debuggers
+      initialize_memory_checkers
     end
 
     def run(*targets)
@@ -262,6 +285,14 @@ module Grntest
       @use_color
     end
 
+    def stop_on_failure?
+      @stop_on_failure
+    end
+
+    def valgrind_gen_suppressions?
+      @valgrind_gen_suppressions
+    end
+
     def target_test?(test_name)
       selected_test?(test_name) and not excluded_test?(test_name)
     end
@@ -343,6 +374,12 @@ module Grntest
       @default_gdb = "gdb"
     end
 
+    def initialize_memory_checkers
+      @vagrind = nil
+      @default_valgrind = "valgrind"
+      @vagrind_gen_suppressions = false
+    end
+
     def command_exist?(name)
       ENV["PATH"].split(File::PATH_SEPARATOR).each do |path|
         absolute_path = File.join(path, name)

data/lib/grntest/version.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2015  Kouhei Sutou <kou@clear-code.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -14,5 +14,5 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 module Grntest
-  VERSION = "1.1.1"
+  VERSION = "1.1.2"
 end

data/lib/grntest/worker.rb

@@ -63,7 +63,7 @@ module Grntest
   end
 
   class Worker
-    attr_reader :id, :tester, :test_suites_rusult, :reporter
+    attr_reader :id, :tester, :reporter
     attr_reader :suite_name, :test_script_path, :test_name, :status, :result
     def initialize(id, tester, test_suites_result, reporter)
       @id = id
@@ -107,6 +107,9 @@ module Grntest
             succeeded = false unless runner.run
 
             break if interruptted?
+            if @tester.stop_on_failure? and @test_suites_result.have_failure?
+              break
+            end
           end
           @status = "finished"
           @reporter.on_suite_finish(@suite_name) if @suite_name

data/test/executors/test-base-executor.rb

@@ -17,8 +17,8 @@ require "grntest/executors/base-executor"
 
 class TestBaseExecutor < Test::Unit::TestCase
   def setup
-    @executor = Grntest::Executors::BaseExecutor.new
-    @context = @executor.context
+    @context = Grntest::ExecutionContext.new
+    @executor = Grntest::Executors::BaseExecutor.new(@context)
   end
 
   class TestErrorLogLevel < self

data/test/executors/test-standard-io-executor.rb

@@ -21,8 +21,9 @@ class TestStandardIOExecutor < Test::Unit::TestCase
   def setup
     input = StringIO.new
     output = StringIO.new
-    @executor = Grntest::Executors::StandardIOExecutor.new(input, output)
-    @context = @executor.context
+    @context = Grntest::ExecutionContext.new
+    @executor = Grntest::Executors::StandardIOExecutor.new(input, output,
+                                                           @context)
     @script = Tempfile.new("test-executor")
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: grntest
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.1.2
 platform: ruby
 authors:
 - Kouhei Sutou
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-03 00:00:00.000000000 Z
+date: 2015-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -204,8 +204,8 @@ specification_version: 4
 summary: Grntest is a testing framework for Groonga. You can write a test for Groonga
   by writing Groonga commands and expected result.
 test_files:
+- test/test-log-parser.rb
 - test/executors/test-base-executor.rb
 - test/executors/test-standard-io-executor.rb
 - test/run-test.rb
-- test/test-log-parser.rb
 has_rdoc: