megatest 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5ef5044b5a70005d87f32c0da4add1e5270d3a298b20a12f4da2fefcb3b64781
-  data.tar.gz: 273e19eed16d3ad7ade77b297ee8e5819629e48d4c106808d52fbf0572538643
+  metadata.gz: dce0e1b9ec47a98020fabc266db9d93f73961574ed6ef0bf29b4885064114216
+  data.tar.gz: c4dd6ed24dea6de74effca99474a850af28b77142a0c388dff2f4faf2f4b5338
 SHA512:
-  metadata.gz: 1179b7d68e1bfd7fd614404ef3ab4e9c724af6be4944a68160de44ec0857c120f08ba363acacee46efed133d532cc9aec27d8b8432831d61373a8c845df54a33
-  data.tar.gz: 81950c2704d1553a4f6239c3bf45b96213b1418c79293d8dd9070405f884c0514e2851cb8b528ce12e84525cd96b7e0eb59a76a3a24d25abe958a56ec5f142cd
+  metadata.gz: 827fa2e37b36c4e8d7fae1317be65d259bac20210276245c119cbf33d506e967514295d4d74cc9fefd499e60aefacfd4a752f17f83feb58a46387af3d516c302
+  data.tar.gz: 3bfe1897107565dd27ae0eef08857b123eea0f181009669ab3e6c676f7fcbed7266fa3708f7b88589aadd518cb43cfaff8402a9648db5828373759b8219e2f6f

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 ## [Unreleased]
 
+## [0.3.0] - 2025-06-20
+
+- Added missing MIT license.
+- Add bisection support.
+- List slowest tests on success.
+
+## [0.2.0] - 2024-08-26
+
 - Make the VerboseReporter work with concurrent executors.
 - Fix isolated tests on forkless platforms when the config contains procs.
 - Add a `job_teardown callback` to to stand off for at_exit.

data/LICENSE.md

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Jean Boussier
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -4,17 +4,55 @@ Megatest is a test-unit like framework with a focus on usability, and designed w
 
 ## Installation
 
-Install the gem and add to the application's Gemfile by executing:
+Install the gem and add it to the application's Gemfile by executing:
 
     $ bundle add megatest
 
 ## Usage
 
+### Special Files And Directories
+
+By default, tests are assumed to live in the `test` directory. If present, the
+optional files `test/test_config.rb` and `test/test_helper.rb` are loaded
+automatically, and in that order.
+
+#### test/test_config.rb
+
+The `megatest` CLI offers options to override default settings, but if you'd
+like to have some always set, please do so in the optional
+`test/test_config.rb`:
+
+```ruby
+# test/test_config.rb
+
+Megatest.config do |config|
+  # See Megatest::Config.
+end
+```
+
+#### test/test_helper.rb
+
+The optional file `test/test_helper.rb` is meant to centralize dependencies and
+define test helpers:
+
+```ruby
+# test/test_helper.rb
+
+require "some_dependency"
+
+module MyApp
+  class Test < Megatest::Test
+    def some_helper(arg)
+    end
+  end
+end
+```
+
 ### Writing Tests
 
 Test suites are Ruby classes that inherit from `Megatest::Test`.
 
-Test cases are be defined with the `test` macro, or for compatibility with existing test suites,
+Test cases are defined with the `test` macro, or for compatibility with existing test suites,
 by defining a method starting with `test_`.
 
 All the classic `test-unit` and `minitest` assertion methods are available:
@@ -37,24 +75,7 @@ class SomeTest < MyApp::Test
 end
 ```
 
-By convention, all the `test_helper.rb` files are automatically loaded,
-which allows to centralize dependencies and define some helpers.
-
-```ruby
-# test/test_helper.rb
-
-require "some_dependency"
-
-module MyApp
-  class Test < Megatest::Test
-
-    def some_helper(arg)
-    end
-  end
-end
-```
-
-It also allow to define test inside `context` blocks, to make it easier to group
+Megatest also allows to define tests inside `context` blocks, to make it easier to group
 related tests together and have them share a common name prefix.
 
 ```ruby
@@ -78,7 +99,7 @@ blocks, nor their own namespaces.
 
 ### Command Line
 
-Contrary to many alternatives, `megatest` provide a convenient CLI interface to easily run specific tests.
+Contrary to many alternatives, `megatest` provides a convenient CLI interface to easily run specific tests.
 
 Run all tests in a directory:
 
@@ -109,7 +130,7 @@ For more detailed usage, run `megatest --help`.
 
 ### CI Parallelization
 
-Megatest offer multiple feature to allow running test suites in parallel across
+Megatest offers multiple features to allow running test suites in parallel across
 many CI jobs.
 
 #### Sharding
@@ -137,8 +158,8 @@ will be automatically inferred from the environment.
 
 A more efficient way to parallelize tests on CI is to use a Redis server to act as a queue.
 
-This allow to efficiently and dynamically ensure a near perfect test case balance across all
-the workers. And if for some reason one of the worker is lost or crashes, no test is lost,
+This allows to efficiently and dynamically ensure a near perfect test case balance across all
+the workers. And if for some reason one of the workers is lost or crashes, no test is lost,
 which for builds with hundreds of parallel jobs, is essential for stability.
 
 ```yaml

data/TODO.md

@@ -1,14 +1,5 @@
 ### Wants
 
-- Improve run by line number: if no exact match fallback to the previous one.
-
-- Test leak bisect
-  - See ci-queue bisect.
-
-- List slow tests
-  - Not just X slowest test, but up to X tests that are significantly slower than average.
-  - Exclude them with `:slow` tag.
-
 - `-j` for forkless environments (Windows / JRuby / TruffleRuby)
 
 - `minitest/mocks`

data/lib/megatest/cli.rb

@@ -28,6 +28,7 @@ module Megatest
     undef_method :puts, :print # Should only use @out.puts or @err.puts
 
     RUNNERS = {
+      "bisect" => :bisect,
       "report" => :report,
       "run" => :run,
     }.freeze
@@ -51,6 +52,8 @@ module Megatest
         report
       when nil, :run
         run_tests
+      when :bisect
+        bisect_tests
       else
         raise InvalidArgument, "Parsing failure"
       end
@@ -116,6 +119,32 @@ module Megatest
       QueueReporter.new(@config, queue, @out).run(default_reporters) ? 0 : 1
     end
 
+    def bisect_tests
+      require "megatest/multi_process"
+
+      queue = @config.build_queue
+      raise InvalidArgument, "Distributed queues can't be bisected" if queue.distributed?
+
+      @config.selectors = Selector.parse(@argv)
+      Megatest.load_config(@config)
+      Megatest.init(@config)
+      test_cases = Megatest.load_tests(@config)
+      queue.populate(test_cases)
+      candidates = queue.dup
+
+      if test_cases.empty?
+        @err.puts "No tests to run"
+        return 1
+      end
+
+      unless failure = find_failing_test(queue)
+        @err.puts "No failing test"
+        return 1
+      end
+
+      bisect_queue(candidates, failure.test_id)
+    end
+
     private
 
     def default_reporters
@@ -141,6 +170,67 @@ module Megatest
       reporters
     end
 
+    def find_failing_test(queue)
+      @config.max_consecutive_failures = 1
+      @config.jobs_count = 1
+
+      executor = MultiProcess::Executor.new(@config.dup, @out)
+      executor.run(queue, default_reporters)
+      queue.summary.failures.first
+    end
+
+    def bisect_queue(queue, failing_test_id)
+      err = Output.new(@err)
+      tests = queue.to_a
+      failing_test_index = tests.index { |test| test.id == failing_test_id }
+      failing_test = tests[failing_test_index]
+      suspects = tests[0...failing_test_index]
+
+      check_passing = @config.build_queue
+      check_passing.populate([failing_test])
+      executor = MultiProcess::Executor.new(@config.dup, @out, managed: true)
+      executor.run(check_passing, [])
+      unless check_passing.success?
+        err.puts err.red("Test failed by itself, no need to bisect")
+        return 1
+      end
+
+      run_index = 0
+      while suspects.size > 1
+        run_index += 1
+        err.puts "Attempt #{run_index}, #{suspects.size} suspects left."
+
+        before, after = suspects[0...(suspects.size / 2)], suspects[(suspects.size / 2)..]
+        candidates = @config.build_queue
+        candidates.populate(before + [failing_test])
+
+        executor = MultiProcess::Executor.new(@config.dup, @out, managed: true)
+        executor.run(candidates, default_reporters)
+
+        if candidates.success?
+          suspects = after
+        else
+          suspects = before
+        end
+
+        err.puts
+      end
+      suspect = suspects.first
+
+      validation_queue = @config.build_queue
+      validation_queue.populate([suspect, failing_test])
+      executor = MultiProcess::Executor.new(@config.dup, @out, managed: true)
+      executor.run(validation_queue, [])
+      if validation_queue.success?
+        err.puts err.red("Bisect inconclusive")
+        return 1
+      end
+
+      err.print "Found test leak: "
+      err.puts err.yellow "#{@config.program_name} #{Megatest.relative_path(suspect.location_id)} #{Megatest.relative_path(failing_test.location_id)}"
+      0
+    end
+
     def open_file(path)
       File.open(path, "w+")
     rescue Errno::ENOENT
@@ -176,6 +266,8 @@ module Megatest
           opts.banner = "Usage: #{@program_name} report [options]"
         when :run
           opts.banner = "Usage: #{@program_name} run [options] [files or directories]"
+        when :bisect
+          opts.banner = "Usage: #{@program_name} bisect [options] [files or directories]"
         else
           opts.banner = "Usage: #{@program_name} command [options] [files or directories]"
           opts.separator ""
@@ -190,6 +282,11 @@ module Megatest
           opts.separator "\treport\t\tWait for the queue to be entirely processed and report the status"
           opts.separator "\t\t\t  $ #{@program_name} report --queue redis://ci-queue.example.com --build-id $CI_BUILD_ID"
           opts.separator ""
+
+          opts.separator "\tbisect\t\tRepeatedly run subsets of the given tests."
+          opts.separator "\t\t\t  $ #{@program_name} bisect --seed 12345 test/integration"
+          opts.separator "\t\t\t  $ #{@program_name} bisect --queue path/to/test_order.log"
+          opts.separator ""
         end
 
         opts.separator ""
@@ -214,11 +311,13 @@ module Megatest
           @junit = path
         end
 
-        if runner == :run
+        if %i[run bisect].include?(runner)
           opts.on("--seed SEED", Integer, "The seed used to define run order") do |seed|
             @config.seed = seed
           end
+        end
 
+        if runner == :run
           opts.on("-j", "--jobs JOBS", Integer, "Number of processes to use") do |jobs|
             @config.jobs_count = jobs
           end
@@ -245,8 +344,10 @@ module Megatest
           @config.queue_url = queue_url
         end
 
-        opts.on("--build-id ID", String, "Unique identifier for the CI build") do |build_id|
-          @config.build_id = build_id
+        if %i[run report].include?(runner)
+          opts.on("--build-id ID", String, "Unique identifier for the CI build") do |build_id|
+            @config.build_id = build_id
+          end
         end
 
         if runner == :run

data/lib/megatest/config.rb

@@ -138,7 +138,7 @@ module Megatest
   class Config
     attr_accessor :queue_url, :retry_tolerance, :max_retries, :jobs_count, :job_index, :load_paths, :deprecations,
                   :build_id, :heartbeat_frequency, :minitest_compatibility, :ci, :selectors
-    attr_reader :before_fork_callbacks, :global_setup_callbacks, :worker_setup_callbacks, :backtrace, :circuit_breaker, :seed,
+    attr_reader :before_fork_callbacks, :global_setup_callbacks, :backtrace, :circuit_breaker, :seed,
                 :worker_id, :workers_count
     attr_writer :differ, :pretty_printer, :program_name, :colors
 
@@ -171,6 +171,11 @@ module Megatest
       CIService.configure(self, env)
     end
 
+    def initialize_dup(_)
+      super
+      @circuit_breaker = @circuit_breaker.dup
+    end
+
     def program_name
       @program_name || "megatest"
     end
@@ -214,10 +219,9 @@ module Megatest
       @differ&.call(expected, actual)
     end
 
-    def pretty_print(object)
+    def render_object(object)
       @pretty_printer.pretty_print(object)
     end
-    alias_method :pp, :pretty_print
 
     # We always return a new generator with the same seed as to
     # best reproduce remote builds locally if the same seed is given.
@@ -237,7 +241,11 @@ module Megatest
         require "megatest/redis_queue"
         RedisQueue.build(self)
       else
-        raise ArgumentError, "Unsupported queue type: #{@queue_url.inspect}"
+        if @queue_url.is_a?(String) && File.exist?(@queue_url)
+          FileQueue.build(self)
+        else
+          raise ArgumentError, "Unsupported queue type: #{@queue_url.inspect}"
+        end
       end
     end
 

data/lib/megatest/differ.rb

@@ -36,7 +36,7 @@ module Megatest
     private
 
     def pp(object)
-      @config.pretty_print(object)
+      @config.render_object(object)
     end
 
     def object_diff(expected, expected_inspect, actual_inspect)

data/lib/megatest/multi_process.rb

@@ -206,9 +206,10 @@ module Megatest
     class Executor
       attr_reader :wall_time
 
-      def initialize(config, out)
+      def initialize(config, out, managed: false)
         @config = config
         @out = Output.new(out, colors: config.colors)
+        @managed = managed
       end
 
       def concurrent?
@@ -260,7 +261,7 @@ module Megatest
         @wall_time = Megatest.now - start_time
         reporters.each { |r| r.summary(self, queue, queue.summary) }
 
-        if @config.circuit_breaker.break?
+        if @config.circuit_breaker.break? && !@managed
           @out.error("Exited early because too many failures were encountered")
         end
 

data/lib/megatest/queue.rb

@@ -94,6 +94,11 @@ module Megatest
         @results = results
       end
 
+      def initialize_dup(_)
+        super
+        @results = @results.dup
+      end
+
       # When running distributed queues, it's possible
       # that a test is considered lost and end up with both
       # a successful and a failed result.
@@ -173,6 +178,18 @@ module Megatest
       @leases = {}
     end
 
+    def initialize_dup(_other)
+      super
+      @queue = @queue.dup
+      @summary = @summary.dup
+      @retries = @retries.dup
+      @leases = @leases.dup
+    end
+
+    def to_a
+      @queue.reverse
+    end
+
     def distributed?
       false
     end
@@ -236,4 +253,15 @@ module Megatest
       true
     end
   end
+
+  class FileQueue < Queue
+    def populate(test_cases)
+      super
+
+      queue = File.readlines(@config.queue_url, chomp: true)
+      queue.reverse!
+      queue.map! { |test_id| @test_cases_index.fetch(test_id) }
+      @queue = queue
+    end
+  end
 end

data/lib/megatest/reporters.rb

@@ -88,7 +88,7 @@ module Megatest
         end
       end
 
-      def summary(executor, _queue, summary)
+      def summary(executor, queue, summary)
         @out.puts
         @out.puts
 
@@ -102,14 +102,27 @@ module Megatest
           end
         end
 
-        if (wall_time = executor.wall_time.to_f) > 0.0
-          @out.puts format(
-            "Finished in %.2fs, %d cases/s, %d assertions/s, %.2fs tests runtime.",
-            wall_time,
-            (summary.runs_count / wall_time).to_i,
-            (summary.assertions_count / wall_time).to_i,
-            summary.total_time,
-          )
+        # In case of failure we'd rather not print slow tests
+        # as it would blur the output.
+        if queue.success? && !summary.results.empty?
+          sorted_results = summary.results.sort_by(&:duration)
+          size = sorted_results.size
+          average = sorted_results.sum(&:duration) / size
+          median = sorted_results[size / 2].duration
+          p90 = sorted_results[(size * 0.9).to_i].duration
+          p99 = sorted_results[(size * 0.99).to_i].duration
+
+          @out.puts "Finished in #{s(executor.wall_time.to_f)}, average: #{ms(average)}, median: #{ms(median)}, p90: #{ms(p90)}, p99: #{ms(p99)}"
+          cuttoff = p90 * 10
+          slowest_tests = sorted_results.last(5).select { |r| r.duration > cuttoff }
+          unless slowest_tests.empty?
+            @out.puts "Slowest tests:"
+            slowest_tests.reverse_each do |result|
+              duration_string = ms(result.duration).rjust(10, " ")
+              @out.puts " - #{duration_string} #{@out.yellow(result.test_id)} @ #{@out.cyan(Megatest.relative_path(result.test_location))}"
+            end
+            @out.puts ""
+          end
         end
 
         @out.puts format(
@@ -122,6 +135,14 @@ module Megatest
           summary.skips_count,
         )
       end
+
+      def s(duration)
+        format("%.2fs", duration)
+      end
+
+      def ms(duration)
+        format("%.1fms", duration * 1000.0)
+      end
     end
 
     class VerboseReporter < SimpleReporter

data/lib/megatest/runtime.rb

@@ -144,7 +144,7 @@ module Megatest
     end
 
     def pp(object)
-      @config.pretty_print(object)
+      @config.render_object(object)
     end
 
     def diff(expected, actual)

data/lib/megatest/state.rb

@@ -585,7 +585,7 @@ module Megatest
       @test_id = test_case.id
       @test_location = test_case.location_id
       @assertions_count = 0
-      @duration = nil
+      @duration = 0.0
       @retried = false
       @failures = []
     end
@@ -663,7 +663,6 @@ module Megatest
 
     def lost
       @failures << Failure.new(LostTest.new(@test_id))
-      @duration = 0.0
       self
     end
 

data/lib/megatest/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Megatest
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/megatest.rb

@@ -38,7 +38,7 @@ module Megatest
         ::Warning[:deprecated] = true
       end
 
-      # We initiale the seed in case there is some Random use
+      # We initialize the seed in case there is some Random use
       # at code loading time.
       Random.srand(config.seed)
     end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: megatest
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Jean Boussier
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-08-26 00:00:00.000000000 Z
+date: 2025-06-20 00:00:00.000000000 Z
 dependencies: []
 description: Largely API compatible with test-unit / minitest, but with lots of extra
   modern niceties like a proper CLI, test distribution, etc.
@@ -20,6 +19,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - CHANGELOG.md
+- LICENSE.md
 - README.md
 - TODO.md
 - exe/megatest
@@ -52,14 +52,14 @@ files:
 - lib/megatest/test_task.rb
 - lib/megatest/version.rb
 homepage: https://github.com/byroot/megatest
-licenses: []
+licenses:
+- MIT
 metadata:
   allowed_push_host: https://rubygems.org/
   rubygems_mfa_required: 'true'
   homepage_uri: https://github.com/byroot/megatest
   source_code_uri: https://github.com/byroot/megatest
   changelog_uri: https://github.com/byroot/megatest/blob/main/CHANGELOG.md
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -74,8 +74,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
-signing_key:
+rubygems_version: 3.6.2
 specification_version: 4
 summary: Modern test-unit style test framework
 test_files: []