padded_cell 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71107b6522d03ca5d3c7f11cb93cd0e6374f283b843a1b456699d69f2a131e11
-  data.tar.gz: efd0654d4986a1fca13380f8a5abcea88ead77944e7e7a92b4cd6a58a69208de
+  metadata.gz: 7d2b2f1c6d06b6cf4e72a67cfb6d511cc121fad2794d5fffc84bbf323c8431f1
+  data.tar.gz: '08aacbd7f382259c1eee0b45e3508c7dba4522be0c6d57ba4ca13b496ba70a1b'
 SHA512:
-  metadata.gz: 64a6eca62591f0b9c6c887587ae7a00be9135f4b6addb9fb4e869fe9fae10b6be9145c9a3834d05e83a2e9e225319f846b91f85a92266038db1bf383b40d3c65
-  data.tar.gz: 53049afc045f49d8a3e62504780babf42a7800988055c2075a961c57fb124085c868a31fff5356f4bf8ed5fd1db3c61e2c771ae151b2d243e62d92facb304c81
+  metadata.gz: d473da59b1f2de1ae4cee4d423c8361ffcf42060bd8b4fd264894a82c6d631cdddd8e8ee376716ae0d73db4ea4e3aefbf540fdbd42b20ed996ca024740448aba
+  data.tar.gz: 4d7a2de1bad5aaf24df760011df2a9728193586640bb7f7f14a9dcd66d38750082995692736bdfc0a4ccb951fb174384d600ac723414a39e0c479d98203f2515

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 0.3.0 (2026-11-08)
+
+- `--budget SECONDS`: a deadline for the whole run, loading included. Once
+  it's spent, nothing more is dispatched and running tests are killed. Tests
+  it left out fail the run (exit 1). `--timeout` never outlives it.
+- Workers that keep dying (more than 4 per worker, at least 16, in 10s) are
+  no longer replaced forever. padded_cell gives up, reports how many tests
+  never ran, and exits 1. Uses `supervisor-machines`.
+- A replacement worker that died before its first test crashed the run with
+  `Errno::EPIPE` (exit 2). A worker reset mid-frame is now reported as dead.
+- `--asylum` retry seeds keep all 64 bits: they were folded modulo 0xffff,
+  so large `--seed`s shared the same few retry seeds.
+
 ## 0.2.0 (2026-10-11)
 
 - Built-in coverage: `--coverage[=lines,branches,methods,eval]` (or

data/Cargo.lock

@@ -73,6 +73,12 @@ dependencies = [
  "libloading",
 ]
 
+[[package]]
+name = "deadline-machines"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "805cba0e8981fd367f11a17886e0ad53b84365601f5d39d8b66819f9bd0ba463"
+
 [[package]]
 name = "either"
 version = "1.19.0"
@@ -220,17 +226,19 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "padded_cell"
-version = "0.2.0"
+version = "0.3.0"
 dependencies = [
  "ruby-prism",
+ "supervisor-machines",
  "tempfile",
  "thiserror",
 ]
 
 [[package]]
 name = "padded_cell_native"
-version = "0.2.0"
+version = "0.3.0"
 dependencies = [
+ "deadline-machines",
  "magnus",
  "padded_cell",
 ]
@@ -435,6 +443,12 @@ version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
 
+[[package]]
+name = "supervisor-machines"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89f7646079e8a1b802737bb514b9c68e8e0743c046c9fb3c570b50e38ba03121"
+
 [[package]]
 name = "syn"
 version = "2.0.119"

data/Cargo.toml

@@ -13,6 +13,8 @@ repository = "https://github.com/seuros/padded_cell"
 padded_cell = { path = "ext/padded_cell_native/core" }
 magnus = { version = "0.9.2", default-features = false }
 ruby-prism = "1.9.0"
+deadline_machines = { package = "deadline-machines", version = "0.4.0", default-features = false }
+supervisor_machines = { package = "supervisor-machines", version = "0.4.0", default-features = false }
 tempfile = "3.27.0"
 thiserror = "2.0.21"
 

data/README.md

@@ -69,11 +69,13 @@ $ padded_cell user_test.rb:12-30:41        # tests overlapping 12..30, or spanni
 $ padded_cell -n /refund/ -e test_slow     # Minitest's --include / --exclude
 $ padded_cell -j 8 --seed 1234             # 8 workers, reproducible order
 $ padded_cell --timeout 30                 # kill and fail anything running 30s
+$ padded_cell --budget 600                 # the whole run gets 10 minutes; what's left out fails it
 $ padded_cell --fail-fast
 $ padded_cell --asylum                     # retry failures; report the flaky ones (see below)
 $ padded_cell --list                       # what would run, without loading anything
 $ padded_cell --shard 2/4                  # CI: the second quarter of the classes
 $ padded_cell --junit tmp/junit.xml --json tmp/results.jsonl
+$ PADDED_CELL_JSON=tmp/results.jsonl padded_cell   # same, from a CI runner's env
 $ padded_cell --coverage                   # line and branch coverage (see Coverage)
 ```
 
@@ -110,6 +112,23 @@ cheap, and order-dependent tests stop depending on order.
 Rails' `parallelize`. A worker that crashes or times out is still reported
 and replaced.
 
+Replaced up to a point. If workers keep dying, say a boot that segfaults or an
+`after_fork` hook that calls `exit`, every remaining test would come back as a
+crash report, one fork at a time. So past 4 deaths per worker in 10 seconds
+(16 at least), padded_cell gives up: it lists how many tests never ran and
+exits 1. The decision comes from
+[supervisor_machines](https://github.com/seuros/supervisor_machines), the same
+one-for-one restart intensity OTP uses.
+
+### Budget
+
+`--budget SECONDS` caps the whole run, loading included. Once it's spent,
+nothing more is dispatched, and a test still running is killed and fails
+("run budget of 600.0s spent"). A test's `--timeout` never outlives the
+budget. If the budget left tests out, the run fails and says how many. The
+deadline math comes from
+[deadline_machines](https://github.com/seuros/deadline_machines).
+
 The default is `test`, except when Active Record fixtures are loaded: a fork
 discards Active Record's connection pool, and every test would reload its
 fixtures, so those suites default to `worker`. The `Run options:` line shows
@@ -305,7 +324,9 @@ PaddedCell::Scheduler.batches(class_names, durations: seconds, seed: 42, shard:
 ## Rust
 
 The extension lives in `ext/padded_cell_native`: `core` does discovery and
-scheduling in plain Rust (no Ruby types), `ffi` binds it to Ruby. Neither is
+scheduling in plain Rust (no Ruby types), `ffi` binds it to Ruby. Worker
+supervision and deadlines come from the published `supervisor-machines` and
+`deadline-machines` crates. Neither is
 published as a crate.
 
 ## Development

data/ext/padded_cell_native/core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "padded_cell"
-version = "0.2.0"
+version = "0.3.0"
 description = "Static Minitest discovery and test scheduling for the padded_cell gem"
 publish = false
 edition.workspace = true
@@ -14,6 +14,7 @@ workspace = true
 
 [dependencies]
 ruby-prism.workspace = true
+supervisor_machines.workspace = true
 thiserror.workspace = true
 
 [dev-dependencies]

data/ext/padded_cell_native/core/src/lib.rs

@@ -1,5 +1,6 @@
-//! Static Minitest discovery and deterministic test scheduling: the engine
-//! of the [padded_cell](https://github.com/seuros/padded_cell) test runner.
+//! Static Minitest discovery, deterministic test scheduling and worker
+//! supervision: the engine of the
+//! [padded_cell](https://github.com/seuros/padded_cell) test runner.
 //!
 //! ```
 //! let source = b"class UserTest < Minitest::Test\n  def test_valid\n  end\nend\n";
@@ -19,6 +20,7 @@ mod error;
 mod hash;
 mod parallel;
 mod plan;
+pub mod restarts;
 mod rng;
 
 pub use discover::{
@@ -27,6 +29,7 @@ pub use discover::{
 pub use error::FileError;
 pub use hash::fnv1a64;
 pub use plan::plan;
+pub use restarts::Restarts;
 pub use rng::SplitMix64;
 
 #[cfg(doctest)]

data/ext/padded_cell_native/core/src/restarts/tests.rs

@@ -0,0 +1,29 @@
+use super::*;
+
+const SECOND: u64 = 1_000_000_000;
+
+#[test]
+fn scales_with_jobs_above_a_floor() {
+    assert_eq!(Restarts::new(1).limit(), 16);
+    assert_eq!(Restarts::new(8).limit(), 32);
+    assert_eq!(Restarts::new(100_000).limit(), 1023);
+}
+
+#[test]
+fn gives_up_past_the_limit_inside_the_window_and_stays_down() {
+    let mut restarts = Restarts::new(1);
+    for death in 0..16 {
+        assert!(restarts.died(death), "death {death}");
+    }
+    assert!(!restarts.died(16));
+    // A supervisor that gave up does not change its mind.
+    assert!(!restarts.died(WINDOW_NS * 10));
+}
+
+#[test]
+fn deaths_spread_over_time_never_give_up() {
+    let mut restarts = Restarts::new(1);
+    for death in 0..1_000 {
+        assert!(restarts.died(death * SECOND), "death {death}");
+    }
+}

data/ext/padded_cell_native/core/src/restarts.rs

@@ -0,0 +1,64 @@
+//! When to stop replacing dead workers.
+//!
+//! A worker that dies takes its test with it and gets replaced; a worker
+//! that keeps dying (a boot that segfaults, an `after_fork` hook that exits)
+//! would otherwise turn every remaining test into a crash report, one fork
+//! at a time.
+//!
+//! The decision is [`supervisor_machines`]'s: one-for-one, restart
+//! intensity over a sliding window. Spawning, killing and reaping stay with
+//! the caller.
+
+use supervisor_machines::{Decision, Exit, Intensity, Policy, Strategy, Supervisor};
+
+#[cfg(test)]
+mod tests;
+
+/// Restart timestamps kept: bounds [`Restarts::limit`].
+const HISTORY: usize = 1024;
+
+/// Deaths allowed per worker inside one window, before giving up.
+const PER_WORKER: u32 = 4;
+
+/// Deaths allowed inside one window whatever the worker count: a few tests
+/// that kill their worker are what cells and workers exist to contain.
+const FLOOR: u32 = 16;
+
+/// The sliding window, in nanoseconds.
+pub const WINDOW_NS: u64 = 10_000_000_000;
+
+/// Worker deaths seen by one pool.
+#[derive(Debug, Clone)]
+pub struct Restarts {
+    supervisor: Supervisor<1, HISTORY>,
+}
+
+impl Restarts {
+    /// For a pool of `jobs` workers: `4 * jobs` deaths per 10 seconds, at
+    /// least 16.
+    #[must_use]
+    pub fn new(jobs: usize) -> Self {
+        let jobs = u32::try_from(jobs).unwrap_or(u32::MAX);
+        let limit = jobs.saturating_mul(PER_WORKER).clamp(FLOOR, HISTORY_LIMIT);
+        Self { supervisor: Supervisor::new(Strategy::OneForOne, Intensity::new(limit, WINDOW_NS)) }
+    }
+
+    /// A worker died at `now_ns` (monotonic): true to replace it, false to
+    /// give up. Once false, always false.
+    pub const fn died(&mut self, now_ns: u64) -> bool {
+        matches!(
+            self.supervisor.stopped(0, Policy::Permanent, Exit::Abnormal, now_ns),
+            Decision::Restart(_)
+        )
+    }
+
+    /// Deaths allowed inside [`WINDOW_NS`].
+    #[must_use]
+    pub const fn limit(&self) -> u32 {
+        self.supervisor.intensity().restarts
+    }
+}
+
+/// The history must hold one more death than the limit to see it exceeded.
+#[expect(clippy::cast_possible_truncation, reason = "HISTORY is 1024")]
+const HISTORY_LIMIT: u32 = HISTORY as u32 - 1;

data/ext/padded_cell_native/ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "padded_cell_native"
-version = "0.2.0"
+version = "0.3.0"
 description = "Ruby bindings for the padded_cell crate"
 publish = false
 edition.workspace = true
@@ -15,5 +15,6 @@ crate-type = ["cdylib"]
 workspace = true
 
 [dependencies]
+deadline_machines.workspace = true
 magnus.workspace = true
 padded_cell.workspace = true

data/ext/padded_cell_native/ffi/src/lib.rs

@@ -19,6 +19,7 @@ use magnus::{Error, RArray, RString, Ruby, function};
 use padded_cell::{Discovery, FileError};
 
 mod chart;
+mod limits;
 
 /// `Errno::*` for the path, as Ruby's own file reading raises.
 fn file_error(ruby: &Ruby, error: &FileError) -> Error {
@@ -119,5 +120,6 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     module.define_module_function("discover_files", function!(discover_files, 1))?;
     module.define_module_function("plan", function!(plan, 5))?;
     module.define_module_function("chart_report", function!(chart::chart_report, 3))?;
+    limits::define(ruby, module)?;
     Ok(())
 }

data/ext/padded_cell_native/ffi/src/limits.rs

@@ -0,0 +1,83 @@
+//! What bounds a run: `PaddedCellNative::Deadline` (`--timeout` and
+//! `--budget`) and `PaddedCellNative::Restarts` (dead workers). Times are
+//! monotonic nanoseconds, the same clock in the runner, its workers and their
+//! cells.
+
+use std::cell::RefCell;
+
+use deadline_machines::{Deadline, Instant};
+use magnus::{Error, Module, Object, RModule, Ruby, function, method};
+use padded_cell::Restarts;
+
+/// A point in monotonic time past which a test (or the run) is cut short, or
+/// never.
+#[magnus::wrap(class = "PaddedCellNative::Deadline", free_immediately, size, frozen_shareable)]
+struct RbDeadline(Deadline);
+
+impl RbDeadline {
+    const fn after(now_ns: u64, ns: u64) -> Self {
+        Self(Deadline::after(Instant(now_ns), ns))
+    }
+
+    const fn never() -> Self {
+        Self(Deadline::never())
+    }
+
+    /// `timeout_ns` from `now_ns`, never past this deadline; nil: this one.
+    fn derive(&self, now_ns: u64, timeout_ns: Option<u64>) -> Self {
+        Self(timeout_ns.map_or(self.0, |ns| self.0.derive(Instant(now_ns), ns)))
+    }
+
+    /// Nanoseconds left (0 once expired), nil when never.
+    const fn remaining_ns(&self, now_ns: u64) -> Option<u64> {
+        self.0.remaining_ns(Instant(now_ns))
+    }
+
+    const fn is_expired(&self, now_ns: u64) -> bool {
+        self.0.is_expired(Instant(now_ns))
+    }
+
+    /// Monotonic nanoseconds, nil when never.
+    fn expires_at(&self) -> Option<u64> {
+        self.0.expires_at().map(|at| at.0)
+    }
+}
+
+#[magnus::wrap(class = "PaddedCellNative::Restarts", free_immediately, size)]
+struct RbRestarts(RefCell<Restarts>);
+
+impl RbRestarts {
+    fn new(jobs: usize) -> Self {
+        Self(RefCell::new(Restarts::new(jobs)))
+    }
+
+    /// True to replace the dead worker, false to give up.
+    fn died(&self, now_ns: u64) -> bool {
+        self.0.borrow_mut().died(now_ns)
+    }
+
+    fn limit(&self) -> u32 {
+        self.0.borrow().limit()
+    }
+
+    const fn window_ns() -> u64 {
+        padded_cell::restarts::WINDOW_NS
+    }
+}
+
+pub fn define(ruby: &Ruby, module: RModule) -> Result<(), Error> {
+    let deadline = module.define_class("Deadline", ruby.class_object())?;
+    deadline.define_singleton_method("after", function!(RbDeadline::after, 2))?;
+    deadline.define_singleton_method("never", function!(RbDeadline::never, 0))?;
+    deadline.define_method("derive", method!(RbDeadline::derive, 2))?;
+    deadline.define_method("remaining_ns", method!(RbDeadline::remaining_ns, 1))?;
+    deadline.define_method("expired?", method!(RbDeadline::is_expired, 1))?;
+    deadline.define_method("expires_at", method!(RbDeadline::expires_at, 0))?;
+
+    let restarts = module.define_class("Restarts", ruby.class_object())?;
+    restarts.define_singleton_method("new", function!(RbRestarts::new, 1))?;
+    restarts.define_singleton_method("window_ns", function!(RbRestarts::window_ns, 0))?;
+    restarts.define_method("died", method!(RbRestarts::died, 1))?;
+    restarts.define_method("limit", method!(RbRestarts::limit, 0))?;
+    Ok(())
+}

data/lib/padded_cell/asylum.rb

@@ -13,18 +13,20 @@ module PaddedCell
   class Asylum
     ALONE = 'timing, contention or state left by other tests'
     RESEEDED = 'randomness or test order'
+    SEEDS = 2**64
 
     # The seed of retry attempt, derived from the run's seed so a report can
-    # be replayed. Attempt 1 keeps the run's seed.
+    # be replayed. Attempt 1 keeps the run's seed. Wraps within the planner's
+    # 64-bit seeds, as --seed accepts them.
     def self.retry_seed(seed, attempt)
-      attempt == 1 ? seed : (seed + (attempt * 0x9e37)) % 0xffff
+      attempt == 1 ? seed : (seed + (attempt * 0x9e37)) % SEEDS
     end
 
-    def initialize(tests, attempts:, seed:, timeout: nil)
+    def initialize(tests, attempts:, seed:, limits: Limits.new)
       @tests = tests
       @attempts = attempts
       @seed = seed
-      @timeout = timeout
+      @limits = limits
       @success = true
     end
 
@@ -58,7 +60,7 @@ module PaddedCell
     def retry_round(pending, history, attempt)
       seed = self.class.retry_seed(@seed, attempt)
       passed = []
-      pool = Pool.new(@tests, jobs: 1, isolation: :test, seed:, timeout: @timeout)
+      pool = Pool.new(@tests, jobs: 1, isolation: :test, seed:, limits: @limits)
       pool.run([pending.dup]) do |index, result|
         recovered = result.passed? && !result.skipped?
         history[index] << attempt_entry(seed, true, recovered ? nil : result)

data/lib/padded_cell/cell.rb

@@ -9,19 +9,21 @@ module PaddedCell
     module_function
 
     # Runs klass#name in a fork and returns its Minitest::Result, or an error
-    # result when the cell died or ran past timeout seconds.
+    # result when the cell died or ran past its deadline.
     #
-    #   close: descriptors the cell must not keep open (the worker's socket)
-    #   probe: the worker's Chart::Probe, which gets the cell's coverage
-    def run(klass, name, seed:, timeout: nil, close: [], probe: nil)
+    #   limits: the run's Limits (--timeout, --budget); nil: no deadline
+    #   close:  descriptors the cell must not keep open (the worker's socket)
+    #   probe:  the worker's Chart::Probe, which gets the cell's coverage
+    def run(klass, name, seed:, limits: nil, close: [], probe: nil)
       # Binary: the result is Marshal data, and with Encoding.default_internal
       # set (Rails sets it) a text-mode pipe transcodes what is written to it.
       reader, writer = IO.pipe
       [reader, writer].each(&:binmode)
       started = Minitest.clock_time
+      deadline = limits&.for_test
       pid = fork_cell(klass, name, seed, reader, writer, close, probe)
       writer.close
-      delta, result = collect(pid, reader, klass, name, started, timeout)
+      delta, result = collect(pid, reader, klass, name, started, deadline) { limits.message(deadline) }
       delta ? probe.absorb(delta) : probe&.lost
       result
     ensure
@@ -59,14 +61,15 @@ module PaddedCell
       end
     end
 
-    # [coverage delta or nil, result]
-    def collect(pid, reader, klass, name, started, timeout)
-      data = read(reader, timeout && (started + timeout))
+    # [coverage delta or nil, result]. Yields for the message of a cell
+    # killed at its deadline.
+    def collect(pid, reader, klass, name, started, deadline)
+      data = read(reader, deadline)
       elapsed = Minitest.clock_time - started
       if data.nil?
         Process.kill(:KILL, pid)
         Process.wait(pid)
-        return [nil, Results.crashed(klass, name, "timed out after #{timeout}s", time: elapsed)]
+        return [nil, Results.crashed(klass, name, yield, time: elapsed)]
       end
 
       _, status = Process.wait2(pid)
@@ -75,11 +78,11 @@ module PaddedCell
       [nil, Results.crashed(klass, name, Results.describe(status, 'cell'), time: elapsed)]
     end
 
-    # Everything until EOF, or nil when the deadline passes first.
+    # Everything until EOF, or nil when the deadline (if any) passes first.
     def read(io, deadline)
       data = String.new(encoding: Encoding::BINARY)
       loop do
-        remaining = deadline && (deadline - Minitest.clock_time)
+        remaining = deadline && Limits.remaining(deadline)
         return if remaining&.<= 0
         next unless io.wait_readable(remaining)
 

data/lib/padded_cell/cli.rb

@@ -22,7 +22,7 @@ module PaddedCell
     # Exit status: 0 passed, 1 failed, 2 usage or load error, 3 coverage below
     # a minimum, 130 interrupted.
     def start(argv, io: $stdout)
-      options = { io:, coverage: coverage_criteria(ENV.fetch('PADDED_CELL_COVERAGE', nil)) }
+      options = { io:, coverage: coverage_criteria(ENV.fetch('PADDED_CELL_COVERAGE', nil)), **report_paths }
       parser(options).parse!(argv)
       return merge(options) if options[:coverage_merge]
 
@@ -33,6 +33,13 @@ module PaddedCell
       2
     end
 
+    # Report paths a CI runner can set without touching the command line;
+    # --json and --junit still win.
+    def report_paths
+      { json: ENV.fetch('PADDED_CELL_JSON', nil), junit: ENV.fetch('PADDED_CELL_JUNIT', nil) }
+        .reject { |_, path| path.nil? || path.empty? }
+    end
+
     def parser(options)
       OptionParser.new do |opts|
         opts.banner = BANNER
@@ -70,6 +77,10 @@ module PaddedCell
       opts.on('--timeout SECONDS', Float, 'Kill and fail a test running longer') do
         options[:timeout] = positive(it)
       end
+      opts.on('--budget SECONDS', Float, 'Stop dispatching once the run has taken SECONDS (and kill and ' \
+                                         'fail what runs past it); tests left out fail the run') do
+        options[:budget] = positive(it)
+      end
       opts.on('--fail-fast', 'Stop dispatching after the first failure') { options[:fail_fast] = true }
       opts.on('--asylum[=N]', Integer, 'Retry failed tests up to N times (default 2) after the run, alone, ' \
                                        'each in a fresh cell; report the flaky ones') do

data/lib/padded_cell/limits.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module PaddedCell
+  # What cuts a test short: its --timeout, and what is left of the run's
+  # --budget. A test's deadline never outlives the run's. Deadlines are
+  # PaddedCellNative::Deadline values in monotonic nanoseconds: one clock for
+  # the runner, its workers and their cells.
+  class Limits
+    NS = 1_000_000_000
+
+    def self.now = Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond)
+
+    # Seconds left before deadline (0.0 once passed), nil when never.
+    def self.remaining(deadline, now = self.now) = deadline.remaining_ns(now)&./(NS.to_f)
+
+    attr_reader :timeout, :budget
+
+    #   timeout: seconds per test, or nil
+    #   budget:  seconds for the whole run from started, or nil
+    def initialize(timeout: nil, budget: nil, started: Limits.now)
+      @timeout = timeout
+      @budget = budget
+      @run = budget ? PaddedCellNative::Deadline.after(started, nanoseconds(budget)) : PaddedCellNative::Deadline.never
+    end
+
+    # The deadline of a test starting now.
+    def for_test(now = Limits.now) = @run.derive(now, @timeout && nanoseconds(@timeout))
+
+    # True once the run's budget is gone: nothing more gets dispatched.
+    def spent?(now = Limits.now) = @run.expired?(now)
+
+    # Why a test that reached deadline was cut short.
+    def message(deadline)
+      if @budget && deadline.expires_at == @run.expires_at
+        "run budget of #{@budget}s spent"
+      else
+        "timed out after #{@timeout}s"
+      end
+    end
+
+    private
+
+    def nanoseconds(seconds) = (seconds * NS).round
+  end
+end

data/lib/padded_cell/pool.rb

@@ -13,20 +13,34 @@ module PaddedCell
   #
   # With isolation :test a worker forks a Cell per test; with :worker it runs
   # tests itself, faster but sharing process state. A worker that dies
-  # mid-test, or runs past the timeout, gets its test reported as an error and
-  # is replaced; its replacement continues the class.
+  # mid-test, or runs past its deadline (--timeout, --budget), gets its test
+  # reported as an error and is replaced; its replacement continues the class.
+  #
+  # Replaced, up to a point: workers that keep dying (a boot that segfaults,
+  # an after_fork hook that exits) would turn every remaining test into a
+  # crash report. Past PaddedCellNative::Restarts' limit the pool gives up.
+  # Once the run's budget is spent, it dispatches nothing more.
   class Pool
-    Slot = Struct.new(:worker, :pid, :socket, :test, :started, :batch)
+    Slot = Struct.new(:worker, :pid, :socket, :test, :started, :batch, :deadline)
+
+    # Tests left undispatched by stop! (--fail-fast, the budget, giving up).
+    attr_reader :undispatched
+
+    # Why the pool gave up on its workers, or nil.
+    attr_reader :gave_up
 
     # tests: [[klass, method_name], ...], dispatched by index.
+    # limits: the run's Limits
     # coverage: a Chart::Accumulator when measuring coverage
-    def initialize(tests, jobs:, isolation:, seed:, timeout: nil, coverage: nil)
+    def initialize(tests, jobs:, isolation:, seed:, limits: Limits.new, coverage: nil)
       @tests = tests
       @jobs = jobs
       @isolation = isolation
       @seed = seed
-      @timeout = timeout
+      @limits = limits
       @coverage = coverage
+      @restarts = PaddedCellNative::Restarts.new(jobs)
+      @undispatched = 0
     end
 
     # Runs batches of test indices in order (consuming them), yielding
@@ -36,7 +50,7 @@ module PaddedCell
       @on_result = on_result
       @sockets = []
       Hooks.run_before_fork
-      @slots = Array.new([@jobs, batches.sum(&:size)].min) { spawn(Slot.new(it, nil, nil, nil, nil, [])) }
+      @slots = Array.new([@jobs, batches.sum(&:size)].min) { spawn(Slot.new(it, nil, nil, nil, nil, [], nil)) }
       @slots.each { feed(it) }
       loop do
         busy = @slots.select(&:test)
@@ -53,6 +67,7 @@ module PaddedCell
 
     # Dispatch nothing more; tests already running still report.
     def stop!
+      @undispatched += @queue.sum(&:size) + @slots.sum { it.batch.size }
       @queue.clear
       @slots.each { it.batch.clear }
     end
@@ -67,7 +82,7 @@ module PaddedCell
         # A worker holding another worker's socket would keep it from ever
         # seeing EOF: keep only its own end.
         [parent, *@sockets].each(&:close)
-        Worker.new(@tests, isolation: @isolation, seed: @seed, timeout: @timeout, probe: @coverage&.probe)
+        Worker.new(@tests, isolation: @isolation, seed: @seed, limits: @limits, probe: @coverage&.probe)
               .run(slot.worker, child)
       end
       child.close
@@ -77,18 +92,25 @@ module PaddedCell
     end
 
     def feed(slot)
+      return stop! if @limits.spent?
+
       slot.batch = @queue.shift || steal if slot.batch.empty?
       slot.test = slot.batch.shift
       return unless slot.test
 
       slot.started = Minitest.clock_time
+      slot.deadline = @limits.for_test
       begin
         slot.socket.puts(slot.test)
       rescue Errno::EPIPE, Errno::ECONNRESET
-        # The worker died between tests: its replacement gets this one.
+        # The worker died between tests: its replacement gets this one, unless
+        # it dies too. Restarts bounds the retries.
         reap(slot)
         @coverage&.lost_worker
-        spawn(slot).socket.puts(slot.test)
+        return abandon(slot) unless @restarts.died(Limits.now)
+
+        spawn(slot)
+        retry
       end
     end
 
@@ -99,17 +121,24 @@ module PaddedCell
     end
 
     def receive(slot)
-      header = slot.socket.read(4)
+      header = read_frame(slot, 4)
       return died(slot) unless header&.bytesize == 4
 
       size = header.unpack1('N')
-      payload = slot.socket.read(size)
+      payload = read_frame(slot, size)
       return died(slot) unless payload&.bytesize == size
 
       report(slot, Results.load(payload, *@tests.fetch(slot.test)))
       feed(slot)
     end
 
+    # nil when the worker died: EOF, or a reset when it left data unread.
+    def read_frame(slot, size)
+      slot.socket.read(size)
+    rescue Errno::ECONNRESET
+      nil
+    end
+
     def report(slot, result)
       index = slot.test
       slot.test = nil
@@ -118,25 +147,42 @@ module PaddedCell
 
     def died(slot)
       status = reap(slot)
+      give_up unless @restarts.died(Limits.now)
       replace(slot, Results.describe(status, 'worker'), Minitest.clock_time - slot.started)
     end
 
-    # Cells time out inside their worker; with :worker isolation the runner
-    # kills the worker itself.
+    # Stops replacing workers: whatever was not dispatched is not run.
+    def give_up
+      window = PaddedCellNative::Restarts.window_ns / Limits::NS
+      @gave_up = "workers died more than #{@restarts.limit} times in #{window}s"
+      stop!
+    end
+
+    # A test taken for a worker that died before receiving it, and is not
+    # replaced.
+    def abandon(slot)
+      slot.test = nil
+      @undispatched += 1
+      give_up
+    end
+
+    # Cells meet their deadline inside their worker; with :worker isolation
+    # the runner kills the worker itself.
     def wait_time(busy)
-      return unless @timeout && @isolation == :worker
+      return unless @isolation == :worker
 
-      deadline = busy.map { it.started + @timeout }.min
-      [deadline - Minitest.clock_time, 0].max
+      now = Limits.now
+      busy.filter_map { Limits.remaining(it.deadline, now) }.min
     end
 
     def expire(busy)
+      now = Limits.now
       busy.each do |slot|
-        next if Minitest.clock_time - slot.started < @timeout
+        next unless slot.deadline.expired?(now)
 
         Process.kill(:KILL, slot.pid)
         reap(slot)
-        replace(slot, "timed out after #{@timeout}s", @timeout)
+        replace(slot, @limits.message(slot.deadline), Minitest.clock_time - slot.started)
       end
     end
 
@@ -145,6 +191,7 @@ module PaddedCell
       @coverage&.lost_worker
       klass, name = @tests.fetch(slot.test)
       report(slot, Results.crashed(klass, name, message, time:))
+      stop! if @limits.spent?
       return if @queue.empty? && @slots.all? { it.batch.empty? }
 
       spawn(slot)

data/lib/padded_cell/runner.rb

@@ -4,6 +4,7 @@ require 'etc'
 require 'minitest'
 require_relative '../padded_cell'
 require_relative 'results'
+require_relative 'limits'
 require_relative 'cell'
 require_relative 'worker'
 require_relative 'pool'
@@ -20,7 +21,7 @@ module PaddedCell
   class Runner
     DEFAULTS = {
       paths: [], jobs: Etc.nprocessors, seed: nil, include: nil, exclude: nil, shard: [0, 1],
-      isolation: nil, timeout: nil, asylum: nil, fail_fast: false, verbose: false, show_skips: false,
+      isolation: nil, timeout: nil, budget: nil, asylum: nil, fail_fast: false, verbose: false, show_skips: false,
       list: false, junit: nil, json: nil, timings: 'tmp/padded_cell/timings.tsv', coverage: nil,
       coverage_dir: nil, io: $stdout
     }.freeze
@@ -35,6 +36,8 @@ module PaddedCell
     # Exit status: 0 when everything passed, 1 otherwise, 3 when only
     # coverage is below a minimum, 130 on Ctrl-C.
     def run
+      # The budget covers the whole run, loading included.
+      @limits = Limits.new(timeout: options[:timeout], budget: options[:budget])
       selection = Selection.parse(options[:paths])
       refs = Discovery.call(selection.files)
       filter = Filter.new(selection, refs, include: options[:include], exclude: options[:exclude])
@@ -121,7 +124,8 @@ module PaddedCell
 
     # Yields each result as it arrives. With --asylum a failure is held back:
     # only its final result, after the retries, is reported. Returns 1 when a
-    # worker failed (its cleanup hook), 130 when interrupted, 0 otherwise.
+    # worker failed (its cleanup hook) or tests were left out (the budget ran
+    # out, workers kept dying), 130 when interrupted, 0 otherwise.
     def dispatch(tests, batches, reporter, retried)
       held = {}
       record = lambda do |index, result|
@@ -130,15 +134,16 @@ module PaddedCell
         retried << result if result.metadata? && (result.metadata[:flaky] || result.metadata[:asylum])
       end
       pool = Pool.new(tests, jobs: options[:jobs], isolation: options[:isolation], seed: options[:seed],
-                             timeout: options[:timeout], coverage: @coverage)
+                             limits: @limits, coverage: @coverage)
       pool.run(batches) do |index, result|
         yield index, result
         failed = failed?(result)
         options[:asylum] && failed ? held[index] = result : record.call(index, result)
         pool.stop! if options[:fail_fast] && failed
       end
+      complete = report_left_out(pool)
       retries_ok = retry_held(tests, held, record)
-      pool.success? && retries_ok ? 0 : 1
+      pool.success? && retries_ok && complete ? 0 : 1
     rescue Interrupt
       warn 'Interrupted. Exiting...'
       # Failures still waiting for a retry are failures.
@@ -146,11 +151,31 @@ module PaddedCell
       130
     end
 
+    # False when tests never ran because the budget ran out or workers kept
+    # dying. --fail-fast leaving them out is what it asked for.
+    def report_left_out(pool)
+      left_out = pool.undispatched
+      if pool.gave_up
+        warn "padded_cell: #{pool.gave_up}; gave up, #{left_out} tests not run"
+      elsif @limits.spent? && left_out.positive?
+        warn "padded_cell: run budget of #{@limits.budget}s spent, #{left_out} tests not run"
+      else
+        return true
+      end
+      false
+    end
+
     # False when a retry's worker failed.
     def retry_held(tests, held, record)
       return true if held.empty?
 
-      asylum = Asylum.new(tests, attempts: options[:asylum], seed: options[:seed], timeout: options[:timeout])
+      # No budget left to retry in: failures stay failures.
+      if @limits.spent?
+        held.each { |index, result| record.call(index, result) }
+        return true
+      end
+
+      asylum = Asylum.new(tests, attempts: options[:asylum], seed: options[:seed], limits: @limits)
       asylum.call(held) do |index, result|
         held.delete(index)
         record.call(index, result)

data/lib/padded_cell/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PaddedCell
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

data/lib/padded_cell/worker.rb

@@ -6,12 +6,12 @@ module PaddedCell
   # Minitest::Result as a length-prefixed Marshal frame. Exits once the
   # socket closes.
   class Worker
-    # probe: a Chart::Probe when measuring coverage
-    def initialize(tests, isolation:, seed:, timeout:, probe:)
+    # limits: the run's Limits; probe: a Chart::Probe when measuring coverage
+    def initialize(tests, isolation:, seed:, limits:, probe:)
       @tests = tests
       @isolation = isolation
       @seed = seed
-      @timeout = timeout
+      @limits = limits
       @probe = probe
     end
 
@@ -54,7 +54,7 @@ module PaddedCell
       if setup_error
         Results.crashed(klass, name, "after_fork hook failed: #{setup_error.class}: #{setup_error.message}")
       elsif @isolation == :test
-        Cell.run(klass, name, seed: @seed, timeout: @timeout, close: [socket], probe: @probe)
+        Cell.run(klass, name, seed: @seed, limits: @limits, close: [socket], probe: @probe)
       else
         Cell.execute(klass, name)
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: padded_cell
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Abdelkader Boudih
@@ -87,12 +87,15 @@ files:
 - ext/padded_cell_native/core/src/parallel.rs
 - ext/padded_cell_native/core/src/plan.rs
 - ext/padded_cell_native/core/src/plan/tests.rs
+- ext/padded_cell_native/core/src/restarts.rs
+- ext/padded_cell_native/core/src/restarts/tests.rs
 - ext/padded_cell_native/core/src/rng.rs
 - ext/padded_cell_native/core/src/rng/tests.rs
 - ext/padded_cell_native/extconf.rb
 - ext/padded_cell_native/ffi/Cargo.toml
 - ext/padded_cell_native/ffi/src/chart.rs
 - ext/padded_cell_native/ffi/src/lib.rs
+- ext/padded_cell_native/ffi/src/limits.rs
 - lib/padded_cell.rb
 - lib/padded_cell/active_support_hooks.rb
 - lib/padded_cell/asylum.rb
@@ -111,6 +114,7 @@ files:
 - lib/padded_cell/cli.rb
 - lib/padded_cell/discovery.rb
 - lib/padded_cell/hooks.rb
+- lib/padded_cell/limits.rb
 - lib/padded_cell/loader.rb
 - lib/padded_cell/native.rb
 - lib/padded_cell/pool.rb