megatest 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 10f2e6b7fd746d7e66164d84868f0314d06f403f649f5a985baaa8db9eb239c4
-  data.tar.gz: 9e6691bc050c0fd73647697e600f20c693d670c1f8982b779545e8d3140a9ef9
+  metadata.gz: a5522d82ae4b5780fa9fdfa8029df9cf1e1b4ec3afaeacbea2a3f4491e93dcff
+  data.tar.gz: a3f5e2553b5c4a8654d2d0d300b2ec604e982ef19a1851a67d84c851a149e2e9
 SHA512:
-  metadata.gz: e03b691bdf5d28c09575c4c3c2d0995e067f3a7e88159f640d5d1dc0c92010470ec36c20bfb52c3cbf63b4edaf3d09afd24cc5799a5d8c9bf1a5dbd206ae3b86
-  data.tar.gz: b742c4c6d8656cf04ba801112bcf913e1224eef13f8256b5ef7c950deaf9c1a24958c7c294f76c477b01a1017adbf849358e5ff0dcea7761dc8b1c18468288f6
+  metadata.gz: b603f4ea08c46dd91937b4214a3f82da29c4a1788dcfe06d53ead22aaaa7bebc3f8247bd6ca163e3f744f7215ef72eb1ed1fb389c9dd99c035c2f0af33f6a2ad
+  data.tar.gz: 0cccdb03115bb61e7ed7ab0d1ef2d0bb6dab8acdb61e40e628958ca1e5b34b16ce51549402834d780adb08f1dc2c9ad0106af14a5a6ad94e8379a8956ffc3b24

data/CHANGELOG.md

@@ -1,11 +1,18 @@
 ## [Unreleased]
 
-## [0.6.0] - 2025-01-17
+## [0.7.0] - 2026-03-21
+
+- Automatic parallelization using cgroups or nprocs.
+- Don't output escape codes when output is not a TTY.
+- Don't retry skipped tests.
+- Improve help message.
+
+## [0.6.0] - 2026-01-17
 
 - Allow defining setup and teardown with method names.
 - Allow multiple `setup`, `around` and `teardown` blocks in the same test suite.
 
-## [0.5.0] - 2025-01-17
+## [0.5.0] - 2026-01-17
 
 - Adds `megatest/autorun`
 - Adds `assert_nothing_raised`.

data/README.md

@@ -142,7 +142,7 @@ and will generally expose environment variables to help split the workload.
 
 ```yaml
 - label: "Run Unit Tests"
-  run: megatest --workers-count $CI_NODE_INDEX --worker-id $CI_NODE_TOTAL
+  run: megatest --workers-count $CI_NODE_TOTAL --worker-id $CI_NODE_INDEX
   parallel: 8
 ```
 
@@ -154,7 +154,7 @@ very large test suites containing lots of slow test cases being sharded as one u
 If you are using CircleCI, Buildkite or HerokuCI, the workers count and worker id
 will be automatically inferred from the environment.
 
-### Redis Distribution
+#### Redis Distribution
 
 A more efficient way to parallelize tests on CI is to use a Redis server to act as a queue.
 

data/lib/megatest/cli.rb

@@ -27,7 +27,7 @@ module Megatest
 
     undef_method :puts, :print # Should only use @out.puts or @err.puts
 
-    RUNNERS = {
+    COMMANDS = {
       "bisect" => :bisect,
       "report" => :report,
       "run" => :run,
@@ -40,17 +40,17 @@ module Megatest
       @processes = nil
       @config = Config.new(env)
       @program_name = @config.program_name = program_name
-      @runner = nil
+      @command = nil
       @verbose = false
       @junit = false
     end
 
     def run
       configure
-      case @runner
+      case @command
       when :report
         report
-      when nil, :run
+      when :run
         run_tests
       when :bisect
         bisect_tests
@@ -71,24 +71,24 @@ module Megatest
     def configure
       Megatest.running = true
 
-      if @runner = RUNNERS[@argv.first]
+      if @command = COMMANDS[@argv.first]
         @argv.shift
       end
 
       Megatest.config = @config
-      @parser = build_parser(@runner)
+      @parser = build_parser
       @parser.parse!(@argv)
       @argv.shift if @argv.first == "--"
+      @queue = @config.build_queue
+      @config.parallelize_maybe if @command == :run && !@queue.distributed? && !@queue.sharded?
       @config
     end
 
     def run_tests
-      queue = @config.build_queue
-
-      if queue.distributed?
+      if @queue.distributed?
         raise InvalidArgument, "Distributed queues require a build-id" unless @config.build_id
         raise InvalidArgument, "Distributed queues require a worker-id" unless @config.worker_id
-      elsif queue.sharded?
+      elsif @queue.sharded?
         unless @config.valid_worker_index?
           raise InvalidArgument, "Splitting the queue requires a worker-id lower than workers-count, got: #{@config.worker_id.inspect}"
         end
@@ -102,44 +102,43 @@ module Megatest
       if test_cases.empty?
         @err.puts "No tests to run"
         return 1
+      elsif test_cases.size == 1
+        @config.jobs_count = 1
       end
 
-      queue.populate(test_cases)
-      executor.run(queue, default_reporters)
-      queue.success? ? 0 : 1
+      @queue.populate(test_cases)
+      executor.run(@queue, default_reporters)
+      @queue.success? ? 0 : 1
     end
 
     def report
-      queue = @config.build_queue
-
-      raise InvalidArgument, "Only distributed queues can be summarized" unless queue.distributed?
+      raise InvalidArgument, "Only distributed queues can be summarized" unless @queue.distributed?
       raise InvalidArgument, "Distributed queues require a build-id" unless @config.build_id
       raise InvalidArgument, @argv.join(" ") unless @argv.empty?
 
-      Megatest.load_config(@argv)
+      @config.selectors = Selector.new(@config).parse(@argv)
+      Megatest.load_config(@config)
 
-      QueueReporter.new(@config, queue, @out).run(default_reporters) ? 0 : 1
+      QueueReporter.new(@config, @queue, @out).run(default_reporters) ? 0 : 1
     end
 
     def bisect_tests
       require "megatest/multi_process"
-
-      queue = @config.build_queue
-      raise InvalidArgument, "Distributed queues can't be bisected" if queue.distributed?
+      raise InvalidArgument, "Distributed queues can't be bisected" if @queue.distributed?
 
       @config.selectors = Selector.new(@config).parse(@argv)
       Megatest.load_config(@config)
       Megatest.init(@config)
       test_cases = Megatest.load_tests(@config)
-      queue.populate(test_cases)
-      candidates = queue.dup
+      @queue.populate(test_cases)
+      candidates = @queue.dup
 
       if test_cases.empty?
         @err.puts "No tests to run"
         return 1
       end
 
-      unless failure = find_failing_test(queue)
+      unless failure = find_failing_test
         @err.puts "No failing test"
         return 1
       end
@@ -172,13 +171,13 @@ module Megatest
       reporters
     end
 
-    def find_failing_test(queue)
+    def find_failing_test
       @config.max_consecutive_failures = 1
       @config.jobs_count = 1
 
       executor = MultiProcess::Executor.new(@config.dup, @out)
-      executor.run(queue, default_reporters)
-      queue.summary.failures.first
+      executor.run(@queue, default_reporters)
+      @queue.summary.failures.first
     end
 
     def bisect_queue(queue, failing_test_id)
@@ -260,10 +259,9 @@ module Megatest
       end
     end
 
-    def build_parser(runner)
-      runner = :run if runner.nil?
+    def build_parser
       OptionParser.new do |opts|
-        case runner
+        case @command
         when :report
           opts.banner = "Usage: #{@program_name} report [options]"
         when :run
@@ -281,7 +279,7 @@ module Megatest
           opts.separator "\t\t\t  $ #{@program_name} test/my_test.rb:42 test/another_test.rb:36"
           opts.separator ""
 
-          opts.separator "\treport\t\tWait for the queue to be entirely processed and report the status"
+          opts.separator "\treport\t\tWait for the queue to be entirely processed and report the status."
           opts.separator "\t\t\t  $ #{@program_name} report --queue redis://ci-queue.example.com --build-id $CI_BUILD_ID"
           opts.separator ""
 
@@ -290,50 +288,51 @@ module Megatest
           opts.separator "\t\t\t  $ #{@program_name} bisect --queue path/to/test_order.log"
           opts.separator ""
         end
+        @command ||= :run
 
         opts.separator ""
         opts.separator "Options:"
         opts.separator ""
 
-        opts.on("-I PATHS", "specify $LOAD_PATH directory (may be used more than once)") do |paths|
+        opts.on("-I PATHS", "Specify $LOAD_PATH directory (may be used more than once).") do |paths|
           paths.split(":").each do |path|
             $LOAD_PATH.unshift(path)
           end
         end
 
-        opts.on("-b", "--backtrace", "Print full backtraces") do
+        opts.on("-b", "--backtrace", "Print full backtraces.") do
           @config.backtrace.full!
         end
 
-        opts.on("-v", "--verbose", "Use the verbose reporter") do
+        opts.on("-v", "--verbose", "Use the verbose reporter.") do
           @verbose = true
         end
 
-        opts.on("--junit [PATH]", String, "Generate a junit.xml file") do |path|
+        opts.on("--junit [PATH]", String, "Generate a junit.xml file.") do |path|
           @junit = path
         end
 
-        if %i[run bisect].include?(runner)
-          opts.on("--seed SEED", Integer, "The seed used to define run order") do |seed|
+        if %i[run bisect].include?(@command)
+          opts.on("--seed SEED", Integer, "The seed used to define run order.") do |seed|
             @config.seed = seed
           end
         end
 
-        if runner == :run
-          opts.on("-j", "--jobs JOBS", Integer, "Number of processes to use") do |jobs|
-            @config.jobs_count = jobs
+        if @command == :run
+          opts.on("-j", "--jobs [JOBS]", Integer, "Number of processes to use. Defaults to the number of processors.") do |jobs|
+            @config.jobs_count = jobs || :number_of_processors
           end
 
-          help = "Number of consecutive failures before exiting. Default to 1"
+          help = "Number of consecutive failures before exiting. Defaults to 1."
           opts.on("-f", "--fail-fast [COUNT]", Integer, help) do |max|
             @config.max_consecutive_failures = (max || 1)
           end
 
-          opts.on("--max-retries COUNT", Integer, "How many times a given test may be retried") do |max_retries|
+          opts.on("--max-retries COUNT", Integer, "How many times a given test may be retried.") do |max_retries|
             @config.max_retries = max_retries
           end
 
-          opts.on("--retry-tolerance RATE", Float, "The proportion of tests that may be retried. e.g. 0.05 for 5% of retried tests") do |retry_tolerance|
+          opts.on("--retry-tolerance RATE", Float, "The proportion of tests that may be retried, e.g. 0.05 for 5% of retried tests.") do |retry_tolerance|
             @config.retry_tolerance = retry_tolerance
           end
         end
@@ -342,22 +341,22 @@ module Megatest
         opts.separator "Test distribution and sharding:"
         opts.separator ""
 
-        opts.on("--queue URL", String, "URL of queue server to use for test distribution. Default to $MEGATEST_QUEUE_URL") do |queue_url|
+        opts.on("--queue URL", String, "URL of queue server to use for test distribution. Default to $MEGATEST_QUEUE_URL.") do |queue_url|
           @config.queue_url = queue_url
         end
 
-        if %i[run report].include?(runner)
-          opts.on("--build-id ID", String, "Unique identifier for the CI build") do |build_id|
+        if %i[run report].include?(@command)
+          opts.on("--build-id ID", String, "Unique identifier for the CI build.") do |build_id|
             @config.build_id = build_id
           end
         end
 
-        if runner == :run
-          opts.on("--worker-id ID", String, "Unique identifier for the CI job") do |worker_id|
+        if @command == :run
+          opts.on("--worker-id ID", String, "Unique identifier for the CI job.") do |worker_id|
             @config.worker_id = worker_id
           end
 
-          opts.on("--workers-count COUNT", Integer, "Number of CI jobs") do |workers_count|
+          opts.on("--workers-count COUNT", Integer, "Number of CI jobs.") do |workers_count|
             @config.workers_count = workers_count
           end
         end

data/lib/megatest/config.rb

@@ -136,11 +136,11 @@ module Megatest
   end
 
   class Config
-    attr_accessor :queue_url, :retry_tolerance, :max_retries, :jobs_count, :job_index, :load_paths, :deprecations,
+    attr_accessor :queue_url, :retry_tolerance, :max_retries, :job_index, :load_paths, :deprecations,
                   :build_id, :heartbeat_frequency, :minitest_compatibility, :ci, :selectors
     attr_reader :before_fork_callbacks, :global_setup_callbacks, :backtrace, :circuit_breaker, :seed,
                 :worker_id, :workers_count, :test_globs
-    attr_writer :differ, :pretty_printer, :program_name, :colors
+    attr_writer :jobs_count, :differ, :pretty_printer, :program_name, :colors
 
     def initialize(env)
       @load_paths = ["test"] # For easier transition from other frameworks
@@ -153,7 +153,7 @@ module Megatest
       @build_id = nil
       @worker_id = nil
       @workers_count = 1
-      @jobs_count = 1
+      @jobs_count = nil
       @colors = nil # auto
       @before_fork_callbacks = []
       @global_setup_callbacks = []
@@ -186,6 +186,25 @@ module Megatest
       @program_name || "megatest"
     end
 
+    def jobs_count
+      if @jobs_count == :number_of_processors
+        if Megatest.fork?
+          require "etc"
+          nprocessors = Etc.nprocessors
+          jobs = [nprocessors, cgroups_cpu_quota&.to_i || nprocessors].min
+          @jobs_count = [jobs, 1].max
+        else
+          @jobs_count = 1
+        end
+      end
+      @jobs_count ||= 1
+    end
+
+    def parallelize_maybe
+      @jobs_count ||= :number_of_processors if Megatest.fork?
+      self
+    end
+
     def worker_id=(id)
       @worker_id = if id.is_a?(String) && /\A\d+\z/.match?(id)
         Integer(id)
@@ -314,6 +333,28 @@ module Megatest
 
     private
 
+    def cgroups_cpu_quota
+      if RbConfig::CONFIG["target_os"].include?("linux")
+        if File.exist?("/sys/fs/cgroup/cpu.max")
+          # cgroups v2: https://docs.kernel.org/admin-guide/cgroup-v2.html#cpu-interface-files
+          cpu_max = File.read("/sys/fs/cgroup/cpu.max")
+          return nil if cpu_max.start_with?("max ") # no limit
+
+          max, period = cpu_max.split.map(&:to_f)
+          max / period
+        elsif File.exist?("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us")
+          # cgroups v1: https://kernel.googlesource.com/pub/scm/linux/kernel/git/glommer/memcg/+/cpu_stat/Documentation/cgroups/cpu.txt
+          max = File.read("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us").to_i
+          # If the cpu.cfs_quota_us is -1, cgroup does not adhere to any CPU time restrictions
+          # https://docs.kernel.org/scheduler/sched-bwc.html#management
+          return nil if max <= 0
+
+          period = File.read("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us").to_f
+          max / period
+        end
+      end
+    end
+
     def normalize_test_glob(patterns)
       if patterns
         Array(patterns).compact.map(&:to_s)

data/lib/megatest/dsl.rb

@@ -108,11 +108,11 @@ module Megatest
     #    end
     #  end
     #
-    # Setup and teardown callbacks are not allowed withing a context blocks,
+    # Setup and teardown callbacks are not allowed within a context blocks,
     # as it too easily lead to "write only" tests. It's only meant to help
     # group test cases together.
     #
-    # If you need a common setup procedure, just define a helper method, and explictly call it.
+    # If you need a common setup procedure, just define a helper method, and explicitly call it.
     #
     # Example:
     #
@@ -147,7 +147,7 @@ module Megatest
     end
 
     # Registers a block to be invoked around every test cases.
-    # The block will recieve a Proc as first argument and MUST
+    # The block will receive a Proc as first argument and MUST
     # call it.
     #
     # Example:

data/lib/megatest/executor.rb

@@ -3,7 +3,26 @@
 # :stopdoc:
 
 module Megatest
-  class Executor
+  class AbstractExecutor
+    def initialize(config, out)
+      @config = config
+      @out = Output.new(out, colors: @config.colors(out))
+    end
+
+    def run(queue, reporters)
+      raise NotImplementedError
+    end
+
+    def concurrent?
+      raise NotImplementedError
+    end
+
+    def wall_time
+      raise NotImplementedError
+    end
+  end
+
+  class Executor < AbstractExecutor
     class ExternalMonitor
       def initialize(config)
         require "rbconfig"
@@ -48,11 +67,6 @@ module Megatest
 
     attr_reader :wall_time
 
-    def initialize(config, out)
-      @config = config
-      @out = Output.new(out, colors: @config.colors)
-    end
-
     def concurrent?
       false
     end

data/lib/megatest/multi_process.rb

@@ -35,7 +35,7 @@ module Megatest
 
       def read
         Marshal.load(@socket)
-      rescue EOFError
+      rescue EOFError, Errno::ECONNRESET
         nil # Other side was closed
       end
 
@@ -203,12 +203,11 @@ module Megatest
       end
     end
 
-    class Executor
+    class Executor < AbstractExecutor
       attr_reader :wall_time
 
-      def initialize(config, out, managed: false)
-        @config = config
-        @out = Output.new(out, colors: config.colors)
+      def initialize(*args, managed: false)
+        super(*args)
         @managed = managed
       end
 

data/lib/megatest/patience_diff.rb

@@ -122,7 +122,7 @@ module Megatest
         last_b_pos = b_lo - 1
 
         longest_unique_subsequence(a[a_lo...a_hi], b[b_lo...b_hi]).each do |(a_pos, b_pos)|
-          # recurse betwen unique lines
+          # recurse between unique lines
           a_pos += a_lo
           b_pos += b_lo
           if (last_a_pos + 1 != a_pos) || (last_b_pos + 1 != b_pos)

data/lib/megatest/pretty_print.rb

@@ -38,7 +38,7 @@ module Megatest
       # Check whether the object_id +id+ is in the current buffer of objects
       # to be pretty printed. Used to break cycles in chains of objects to be
       # pretty printed.
-      def check_inspect_key(id)
+      def check_inspect_key?(id)
         @recursive_key&.include?(id)
       end
 
@@ -63,7 +63,7 @@ module Megatest
         # detection
         obj = obj.__getobj__ if defined?(::Delegator) && ::Delegator === obj
 
-        if check_inspect_key(obj)
+        if check_inspect_key?(obj)
           group { pretty_print_cycle(obj) }
           return
         end

data/lib/megatest/queue.rb

@@ -169,7 +169,7 @@ module Megatest
     alias_method :global_summary, :summary
 
     def initialize(config)
-      super(config)
+      super
 
       @queue = nil
       @summary = Summary.new
@@ -228,8 +228,8 @@ module Megatest
 
     def record_result(result)
       @leases.delete(result.test_id)
-      if result.failed?
-        if attempt_to_retry(result)
+      if result.failed? && !result.skipped?
+        if attempt_to_retry?(result)
           result = result.retry
         else
           @success &&= result.ok?
@@ -241,7 +241,7 @@ module Megatest
 
     private
 
-    def attempt_to_retry(result)
+    def attempt_to_retry?(result)
       return false unless @config.retries?
       return false unless @summary.retries_count < @config.total_max_retries(@size)
       return false unless @retries[result.test_id] < @config.max_retries

data/lib/megatest/redis_queue.rb

@@ -13,7 +13,7 @@ module Megatest
   #
   # - "leader-status": String, either `setup` or `ready`
   #
-  # - "queue": List, contains the test ids that haven't yet been poped.
+  # - "queue": List, contains the test ids that haven't yet been popped.
   #
   # - "running": SortedSet, members are the test ids currently being processed.
   #    Scores are the lease expiration timestamp. If the score is lower than
@@ -284,8 +284,8 @@ module Megatest
 
     def record_result(original_result)
       result = original_result
-      if result.failed?
-        if attempt_to_retry(result)
+      if result.failed? && !result.skipped?
+        if attempt_to_retry?(result)
           result = result.retry
         else
           @success = false
@@ -381,8 +381,9 @@ module Megatest
 
       return true
     LUA
+    private_constant :REQUEUE
 
-    def attempt_to_retry(result)
+    def attempt_to_retry?(result)
       return false unless @config.retries?
 
       index = @config.random.rand(0..@redis.call("llen", key("queue")))
@@ -435,7 +436,7 @@ module Megatest
     end
 
     def worker_id
-      @worker_id or raise Error, "RedisQueue not configued with a worker id"
+      @worker_id or raise Error, "RedisQueue not configured with a worker id"
     end
 
     class RetryQueue < Queue

data/lib/megatest/reporters.rb

@@ -9,7 +9,7 @@ module Megatest
 
       def initialize(config, out)
         @config = config
-        @out = Output.new(out, colors: config.colors)
+        @out = Output.new(out, colors: config.colors(out))
       end
 
       def start(_executor, _queue)
@@ -32,6 +32,7 @@ module Megatest
         failure: "Failure",
         skipped: "Skipped",
       }.freeze
+      private_constant :LABELS
 
       def render_failure(result, command: true)
         str = "#{LABELS.fetch(result.status)}: #{result.test_id}\n"
@@ -70,7 +71,9 @@ module Megatest
 
     class SimpleReporter < AbstractReporter
       def start(_executor, queue)
-        @out.puts("Running #{queue.size} test cases with --seed #{@config.seed}")
+        @out.print("Running #{queue.size} test cases with --seed #{@config.seed}")
+        @out.print(" in #{@config.jobs_count} processes") if @config.jobs_count > 1
+        @out.puts
         @out.puts
       end
 
@@ -125,7 +128,7 @@ module Megatest
           end
         end
 
-        @out.puts format(
+        @out.print format(
           "Ran %d cases, %d assertions, %d failures, %d errors, %d retries, %d skips",
           summary.runs_count,
           summary.assertions_count,
@@ -134,6 +137,8 @@ module Megatest
           summary.retries_count,
           summary.skips_count,
         )
+        @out.print(" in #{@config.jobs_count} processes") if @config.jobs_count > 1
+        @out.puts
       end
 
       def s(duration)

data/lib/megatest/runner.rb

@@ -10,9 +10,7 @@ module Megatest
 
     def execute(test_case)
       if test_case.tag(:isolated)
-        isolate(test_case) do
-          run(test_case)
-        end
+        isolate(test_case)
       else
         run(test_case)
       end
@@ -23,7 +21,7 @@ module Megatest
         read, write = IO.pipe.each(&:binmode)
         pid = Process.fork do
           read.close
-          result = yield
+          result = run(test_case)
           Marshal.dump(result, write)
           write.close
           # We don't want to run at_exit hooks the app may have

data/lib/megatest/runtime.rb

@@ -98,7 +98,7 @@ module Megatest
       elsif !keyword.nil?
         raise ArgumentError, "Can't pass both a positional and keyword assertion message"
       else
-        positional # TODO: deprecation mecanism
+        positional # TODO: deprecation mechanism
       end
     end
 

data/lib/megatest/test_task.rb

@@ -105,7 +105,7 @@ module Megatest # :nodoc:
     end
 
     def define # :nodoc:
-      desc "Run the test suite. Use N, X, A, and TESTOPTS to add flags/args."
+      desc "Run the test suite."
       task name => Array(deps) do
         sh(*make_test_cmd, verbose: verbose)
       end

data/lib/megatest/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Megatest
-  VERSION = "0.6.0"
+  VERSION = "0.7.0"
 end

data/lib/megatest.rb

@@ -73,14 +73,14 @@ module Megatest
     end
 
     def load_files(paths, name)
-      scaned = {}
+      scanned = {}
       paths.each do |path|
         path = File.dirname(path) unless File.directory?(path)
 
         while path.start_with?(PWD)
-          break if scaned[path]
+          break if scanned[path]
 
-          scaned[path] = true
+          scanned[path] = true
 
           config_path = File.join(path, name)
           if File.exist?(config_path)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: megatest
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Jean Boussier