grntest 1.6.6 → 1.6.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb9c21dbf97a69dcd226668ff67f8443bf8ab3e57559137c78180ec07c85a147
-  data.tar.gz: 5aa1f037756aa689fff0e5d665fe747ed65d9b2a9f0953015feca997b460a300
+  metadata.gz: f4c390d442cedfaff26e03d35e62f35c2958146f7bea7a8a2e6257004fc38338
+  data.tar.gz: 24bbae078beddc9704797ee722e34480d1724fa0f9a494007f1967fb7e0199d3
 SHA512:
-  metadata.gz: eb052fcb4fd03ca1b757cb4fe96f341e9214efd522f8269463267f1be16a6556f1a489e4c1fca6c94934eceff00cdd03002f32e83e6595d4ca4ae941b61ca6d2
-  data.tar.gz: 838ca099c5ea3db31f6754f4b78ec6c5d76de23fe1fd184a5579b0e4eeba399d32c779e4057dc7a6571aa6e011bcbf538b30b25ba2c50191cf278c5495430d12
+  metadata.gz: ef8e9f5d58cde904e3673cbe778849f2ff902bf21739d6a361e4121a775faeec1ce4bd1bb0a8f3420f64c6ebcbd379db10253aa99e9a0cdcee33950e5d896fd7
+  data.tar.gz: 33bd84602a4c5715e8cffdbcf72ef1efb2320881fe206dba457f229a457864592b7dc8544efaeadb3bf42cd614786fbe158217cfbeedf53079618c2bffecd43a

data/doc/text/news.md

@@ -1,5 +1,19 @@
 # News
 
+## 1.6.8: 2024-02-05
+
+### Fixes
+
+  * reporter: benchmark-json: Fixed a bug that invalid JSON is
+    generated.
+
+## 1.6.7: 2024-02-05
+
+### Improvements
+
+  * Added benchmark mode. It uses Google Benchmark compatible JSON
+    output.
+
 ## 1.6.6: 2023-10-24
 
 ### Improvements

data/lib/grntest/base-result.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2012-2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2024  Sutou Kouhei <kou@clear-code.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,16 +15,26 @@
 
 module Grntest
   class BaseResult
-    attr_accessor :elapsed_time
+    attr_accessor :cpu_elapsed_time
+    attr_accessor :real_elapsed_time
     def initialize
-      @elapsed_time = 0
+      @cpu_elapsed_time = 0
+      @real_elapsed_time = 0
     end
 
     def measure
-      start_time = Time.now
+      cpu_start_time = Process.times
+      real_start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       yield
     ensure
-      @elapsed_time = Time.now - start_time
+      cpu_finish_time = Process.times
+      real_finish_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      @cpu_elapsed_time =
+        (cpu_finish_time.utime - cpu_start_time.utime) +
+        (cpu_finish_time.stime - cpu_start_time.stime) +
+        (cpu_finish_time.cutime - cpu_start_time.cutime) +
+        (cpu_finish_time.cstime - cpu_start_time.cstime)
+      @real_elapsed_time = real_finish_time - real_start_time
     end
   end
 end

data/lib/grntest/execution-context.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2021  Sutou Kouhei <kou@clear-code.com>
+# Copyright (C) 2012-2024  Sutou Kouhei <kou@clear-code.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -39,6 +39,7 @@ module Grntest
     attr_writer :collect_query_log
     attr_writer :debug
     attr_accessor :platform
+    attr_accessor :benchmarks
     def initialize
       @logging = true
       @base_directory = Pathname(".")
@@ -70,6 +71,7 @@ module Grntest
       @collect_query_log = false
       @debug = false
       @platform = guess_platform
+      @benchmarks = []
     end
 
     def logging?

data/lib/grntest/executors/base-executor.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2023  Sutou Kouhei <kou@clear-code.com>
+# Copyright (C) 2012-2024  Sutou Kouhei <kou@clear-code.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -48,6 +48,8 @@ module Grntest
         @raw_status_response = nil
         @features = nil
         @substitutions = {}
+        @noop_benchmark_result = BenchmarkResult.new("noop", 1, 1)
+        @benchmark_result = @noop_benchmark_result
       end
 
       def execute(script_path)
@@ -520,6 +522,18 @@ module Grntest
         @substitutions.delete(pattern)
       end
 
+      def execute_directive_start_benchmark(line, content, options)
+        _, n_items, n_iterations, name = content.split(" ", 4)
+        n_items = Integer(n_items, 10)
+        n_iterations = Integer(n_iterations, 10)
+        @benchmark_result = BenchmarkResult.new(name, n_items, n_iterations)
+        @context.benchmarks << @benchmark_result
+      end
+
+      def execute_directive_finish_benchmark(line, content, options)
+        @benchmark_result = @noop_benchmark_result
+      end
+
       def execute_directive(parser, line, content)
         command, *options = Shellwords.split(content)
         case command
@@ -579,6 +593,10 @@ module Grntest
           execute_directive_add_substitution(line, content, options)
         when "remove-substitution"
           execute_directive_remove_substitution(line, content, options)
+        when "start-benchmark"
+          execute_directive_start_benchmark(line, content, options)
+        when "finish-benchmark"
+          execute_directive_finish_benchmark(line, content, options)
         else
           log_input(line)
           log_error("#|e| unknown directive: <#{command}>")
@@ -628,8 +646,12 @@ module Grntest
         timeout = @context.timeout
         response = nil
         begin
-          Timeout.timeout(timeout) do
-            response = send_command(command)
+          @benchmark_result.measure do
+            @benchmark_result.n_iterations.times do
+              Timeout.timeout(timeout) do
+                response = send_command(command)
+              end
+            end
           end
         rescue Timeout::Error
           log_error("# error: timeout (#{timeout}s)")

data/lib/grntest/reporters/base-reporter.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2023  Sutou Kouhei <kou@clear-code.com>
+# Copyright (C) 2012-2024  Sutou Kouhei <kou@clear-code.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -45,7 +45,7 @@ module Grntest
         puts(statistics_header)
         puts(colorize(statistics(result), result))
         pass_ratio = result.pass_ratio
-        elapsed_time = result.elapsed_time
+        elapsed_time = result.real_elapsed_time
         summary = "%.4g%% passed in %.4fs." % [pass_ratio, elapsed_time]
         puts(colorize(summary, result))
       end
@@ -78,10 +78,10 @@ module Grntest
       end
 
       def throughput(result)
-        if result.elapsed_time.zero?
+        if result.real_elapsed_time.zero?
           tests_per_second = 0
         else
-          tests_per_second = result.n_tests / result.elapsed_time
+          tests_per_second = result.n_tests / result.real_elapsed_time
         end
         tests_per_second
       end
@@ -160,7 +160,7 @@ module Grntest
       end
 
       def test_result_message(result, label)
-        elapsed_time = result.elapsed_time
+        elapsed_time = result.real_elapsed_time
         formatted_elapsed_time = "%.4fs" % elapsed_time
         formatted_elapsed_time = colorize(formatted_elapsed_time,
                                           elapsed_time_status(elapsed_time))

data/lib/grntest/reporters/benchmark-json-reporter.rb

@@ -0,0 +1,128 @@
+# Copyright (C) 2024  Sutou Kouhei <kou@clear-code.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require "json"
+require "time"
+
+require "grntest/reporters/base-reporter"
+
+module Grntest
+  module Reporters
+    class BenchmarkJSONReporter < BaseReporter
+      def initialize(tester)
+        super
+      end
+
+      def on_start(result)
+        puts(<<-JSON)
+{
+  "context": {
+    "date": #{Time.now.iso8601.to_json},
+    "host_name": #{Socket.gethostname.to_json},
+    "executable": #{@tester.testee.to_json},
+    "num_cpus": #{Etc.nprocessors},
+        JSON
+        cpu_cycles_per_second = detect_cpu_cycles_per_second
+        if cpu_cycles_per_second
+          puts(<<-JSON)
+    "mhz_per_cpu": #{cpu_cycles_per_second / 1_000_000.0},
+          JSON
+        end
+        puts(<<-JSON)
+    "caches": []
+  },
+  "benchmarks": [
+        JSON
+      end
+
+      def on_worker_start(worker)
+      end
+
+      def on_suite_start(worker)
+      end
+
+      def on_test_start(worker)
+      end
+
+      def on_test_success(worker, result)
+      end
+
+      def on_test_failure(worker, result)
+        output, @output = @output, $stderr
+        begin
+          report_failure(result)
+        ensure
+          @output = output
+        end
+      end
+
+      def on_test_leak(worker, result)
+      end
+
+      def on_test_omission(worker, result)
+      end
+
+      def on_test_no_check(worker, result)
+      end
+
+      def on_test_finish(worker, result)
+        return if result.benchmarks.empty?
+        benchmarks = result.benchmarks.collect do |benchmark|
+          <<-JSON.chomp
+    {
+      "name": #{result.test_name.to_json},
+      "run_name": #{benchmark.name.to_json},
+      "run_type": "iteration",
+      "iterations": #{benchmark.n_iterations},
+      "real_time": #{benchmark.real_elapsed_time},
+      "cpu_time": #{benchmark.cpu_elapsed_time},
+      "time_unit": "s",
+      "items_per_second": #{benchmark.items_per_second}
+    }
+          JSON
+        end
+        puts(benchmarks.join(",\n"))
+      end
+
+      def on_suite_finish(worker)
+      end
+
+      def on_worker_finish(worker)
+      end
+
+      def on_finish(result)
+        puts(<<-JSON)
+  ]
+}
+        JSON
+      end
+
+      private
+      def detect_cpu_cycles_per_second
+        if File.exist?("/proc/cpuinfo")
+          File.open("/proc/cpuinfo") do |cpuinfo|
+            cpuinfo.each_line do |line|
+              case line
+              when /\Acpu MHz\s+: ([\d.]+)/
+                return Float($1)
+              end
+            end
+          end
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/grntest/reporters.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2020  Sutou Kouhei <kou@clear-code.com>
+# Copyright (C) 2012-2024  Sutou Kouhei <kou@clear-code.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -13,27 +13,30 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-require "grntest/reporters/mark-reporter"
+require "grntest/reporters/benchmark-json-reporter"
 require "grntest/reporters/buffered-mark-reporter"
-require "grntest/reporters/stream-reporter"
 require "grntest/reporters/inplace-reporter"
+require "grntest/reporters/mark-reporter"
 require "grntest/reporters/progress-reporter"
+require "grntest/reporters/stream-reporter"
 
 module Grntest
   module Reporters
     class << self
       def create_reporter(tester)
         case tester.reporter
-        when :mark
-          MarkReporter.new(tester)
+        when :"benchmark-json"
+          BenchmarkJSONReporter.new(tester)
         when :"buffered-mark"
           BufferedMarkReporter.new(tester)
-        when :stream
-          StreamReporter.new(tester)
         when :inplace
           InplaceReporter.new(tester)
+        when :mark
+          MarkReporter.new(tester)
         when :progress
           ProgressReporter.new(tester)
+        when :stream
+          StreamReporter.new(tester)
         end
       end
     end

data/lib/grntest/test-runner.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2022  Sutou Kouhei <kou@clear-code.com>
+# Copyright (C) 2012-2024  Sutou Kouhei <kou@clear-code.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -28,10 +28,27 @@ require "grntest/executors"
 require "grntest/base-result"
 
 module Grntest
+  class BenchmarkResult < BaseResult
+    attr_reader :name
+    attr_reader :n_items
+    attr_reader :n_iterations
+    def initialize(name, n_items, n_iterations)
+      super()
+      @name = name
+      @n_items = n_items
+      @n_iterations = n_iterations
+    end
+
+    def items_per_second
+      @n_items / @real_elapsed_time
+    end
+  end
+
   class TestResult < BaseResult
     attr_accessor :worker_id, :test_name
     attr_accessor :expected, :actual, :n_leaked_objects
     attr_writer :omitted
+    attr_accessor :benchmarks
     def initialize(worker)
       super()
       @worker_id = worker.id
@@ -40,6 +57,7 @@ module Grntest
       @expected = nil
       @n_leaked_objects = 0
       @omitted = false
+      @benchmarks = []
     end
 
     def status
@@ -159,6 +177,7 @@ module Grntest
         check_memory_leak(context)
         result.omitted = context.omitted?
         result.actual = context.result
+        result.benchmarks = context.benchmarks
         context.close_logs
       end
     end

data/lib/grntest/tester.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2023  Sutou Kouhei <kou@clear-code.com>
+# Copyright (C) 2012-2024  Sutou Kouhei <kou@clear-code.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -161,6 +161,7 @@ module Grntest
           :stream,
           :inplace,
           :progress,
+          :"benchmark-json",
         ]
         available_reporter_labels = available_reporters.join(", ")
         parser.on("--reporter=REPORTER", available_reporters,
@@ -299,6 +300,14 @@ module Grntest
           srand(seed)
         end
 
+        parser.on("--[no-]benchmark",
+                  "Set options for benchmark") do |benchmark|
+          if benchmark
+            tester.n_workers = 1
+            tester.reporter = :"benchmark-json"
+          end
+        end
+
         parser.on("--version",
                   "Show version and exit") do
           puts(VERSION)

data/lib/grntest/version.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2023  Sutou Kouhei <kou@clear-code.com>
+# Copyright (C) 2012-2024  Sutou Kouhei <kou@clear-code.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -14,5 +14,5 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 module Grntest
-  VERSION = "1.6.6"
+  VERSION = "1.6.8"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: grntest
 version: !ruby/object:Gem::Version
-  version: 1.6.6
+  version: 1.6.8
 platform: ruby
 authors:
 - Kouhei Sutou
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-10-24 00:00:00.000000000 Z
+date: 2024-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: diff-lcs
@@ -249,6 +249,7 @@ files:
 - lib/grntest/platform.rb
 - lib/grntest/reporters.rb
 - lib/grntest/reporters/base-reporter.rb
+- lib/grntest/reporters/benchmark-json-reporter.rb
 - lib/grntest/reporters/buffered-mark-reporter.rb
 - lib/grntest/reporters/inplace-reporter.rb
 - lib/grntest/reporters/mark-reporter.rb
@@ -284,7 +285,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.0.dev
+rubygems_version: 3.5.1
 signing_key:
 specification_version: 4
 summary: Grntest is a testing framework for Groonga. You can write a test for Groonga