padded_cell 0.1.0

data/ext/padded_cell_native/core/src/rng/tests.rs

@@ -0,0 +1,17 @@
+use super::*;
+
+#[test]
+fn matches_the_reference_sequence() {
+    // From Vigna's reference implementation, seed 1234567.
+    let mut rng = SplitMix64::new(1_234_567);
+    let expected = [
+        6_457_827_717_110_365_317,
+        3_203_168_211_198_807_973,
+        9_817_491_932_198_370_423,
+        4_593_380_528_125_082_431,
+        16_408_922_859_458_223_821,
+    ];
+    for value in expected {
+        assert_eq!(rng.next_u64(), value);
+    }
+}

data/ext/padded_cell_native/core/src/rng.rs

@@ -0,0 +1,28 @@
+//! Vigna's `SplitMix64`: std has no seedable generator, and a fixed algorithm
+//! gives a seed the same order on every platform and in every release.
+
+#[cfg(test)]
+mod tests;
+
+/// A `SplitMix64` generator.
+#[derive(Debug, Clone)]
+pub struct SplitMix64 {
+    state: u64,
+}
+
+impl SplitMix64 {
+    /// A generator starting from `seed`.
+    #[must_use]
+    pub const fn new(seed: u64) -> Self {
+        Self { state: seed }
+    }
+
+    /// The next 64 random bits.
+    pub const fn next_u64(&mut self) -> u64 {
+        self.state = self.state.wrapping_add(0x9e37_79b9_7f4a_7c15);
+        let mut z = self.state;
+        z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
+        z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
+        z ^ (z >> 31)
+    }
+}

data/ext/padded_cell_native/extconf.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# padded_cell's discovery and scheduling are Rust: without the extension
+# the gem cannot work, so a missing toolchain stops the install with the reason.
+def fail_native(reason)
+  abort "padded_cell: #{reason}. padded_cell needs its native extension; precompiled gems exist for " \
+        'Linux (glibc 2.34+, musl), macOS 13+ and FreeBSD 15.1.'
+end
+
+def command?(command) = system(command, out: File::NULL, err: File::NULL)
+
+# rb_sys writes a GNU Makefile; RubyGems runs the system make (BSD make here).
+BSD = RUBY_PLATFORM.include?('bsd')
+
+fail_native("#{RUBY_ENGINE} is not supported (CRuby only)") unless RUBY_ENGINE == 'ruby'
+fail_native('Cargo not found (install Rust 1.99+ from https://rustup.rs)') unless command?('cargo --version')
+fail_native('GNU make not found (pkg install gmake)') if BSD && !command?('gmake --version')
+
+require 'mkmf'
+require 'rb_sys/mkmf'
+
+create_rust_makefile('padded_cell/padded_cell_native') do |r|
+  r.ext_dir = 'ffi'
+  r.profile = ENV.fetch('RB_SYS_CARGO_PROFILE', :release).to_sym
+end
+
+# BSD make reads BSDmakefile first (GNU make ignores it): delegate to gmake.
+if BSD
+  File.write('BSDmakefile', <<~MAKE)
+    GMAKE_VARS =
+    .for var in DESTDIR sitearchdir sitelibdir
+    .if defined(${var})
+    GMAKE_VARS += ${var}="${${var}}"
+    .endif
+    .endfor
+
+    # BSD make exports its own job flags (-J) in MAKEFLAGS; gmake rejects them.
+    GMAKE = env -u MAKEFLAGS -u MFLAGS gmake -f Makefile
+
+    all:
+    \t${GMAKE} ${GMAKE_VARS}
+
+    .DEFAULT:
+    \t${GMAKE} ${GMAKE_VARS} $@
+  MAKE
+end

data/ext/padded_cell_native/ffi/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "padded_cell_native"
+version = "0.1.0"
+description = "Ruby bindings for the padded_cell crate"
+publish = false
+edition.workspace = true
+rust-version.workspace = true
+authors.workspace = true
+license.workspace = true
+
+[lib]
+crate-type = ["cdylib"]
+
+[lints]
+workspace = true
+
+[dependencies]
+magnus.workspace = true
+padded_cell.workspace = true

data/ext/padded_cell_native/ffi/src/lib.rs

@@ -0,0 +1,120 @@
+//! Ruby bindings for the `padded_cell` crate. Only primitives cross: each
+//! function mirrors a `PaddedCell::Engine` method; the runner is the only
+//! caller, so arguments are trusted.
+//!
+//! Nothing is copied twice: Ruby strings are read in place (see
+//! `with_bytes`), files are read by Rust, and results are written straight
+//! into Ruby objects.
+#![expect(
+    clippy::needless_pass_by_value,
+    reason = "magnus converts Ruby arguments into owned values; every function takes them"
+)]
+
+use std::ffi::OsStr;
+use std::os::unix::ffi::OsStrExt;
+use std::path::Path;
+
+use magnus::prelude::*;
+use magnus::{Error, RArray, RString, Ruby, function};
+use padded_cell::{Discovery, FileError};
+
+/// `Errno::*` for the path, as Ruby's own file reading raises.
+fn file_error(ruby: &Ruby, error: &FileError) -> Error {
+    let path = error.path.display().to_string();
+    let Some(errno) = error.source.raw_os_error() else {
+        return Error::new(ruby.exception_io_error(), error.to_string());
+    };
+    match ruby.exception_system_call_error().new_instance((path, errno)) {
+        Ok(exception) => exception.into(),
+        Err(error) => error,
+    }
+}
+
+/// The strings of a Ruby array. They stay reachable for the GC through the
+/// array, a Ruby argument.
+fn strings(array: RArray) -> Result<Vec<RString>, Error> {
+    (0..array.len()).map(|index| array.entry(index.cast_signed())).collect()
+}
+
+/// Lends `f` the bytes of Ruby strings, without copying them.
+///
+/// Only call it with closures that run pure Rust (the `padded_cell` crate):
+/// see the SAFETY comment.
+#[expect(unsafe_code, reason = "reads Ruby strings in place instead of copying them")]
+fn with_bytes<R>(strings: &[RString], f: impl FnOnce(&[&[u8]]) -> R) -> R {
+    // SAFETY: `RString::as_slice` requires that Ruby neither frees, moves nor
+    // modifies the strings while the slices live. The slices only live
+    // during `f`, which runs Rust code that never calls into Ruby: it
+    // allocates no Ruby object (so no GC can run) and this thread keeps the
+    // GVL (so no other Ruby thread can mutate a string). The strings stay
+    // referenced by the caller's Ruby arguments.
+    let bytes: Vec<&[u8]> = strings.iter().map(|string| unsafe { string.as_slice() }).collect();
+    f(&bytes)
+}
+
+/// `[[class_name, method_name, start_line, end_line], ...]`. A class name is
+/// one frozen Ruby string, shared by the rows of its tests.
+fn rows(ruby: &Ruby, discovery: &Discovery) -> Result<RArray, Error> {
+    // A Ruby array keeps the class names reachable for the GC while the rows
+    // are built.
+    let classes = ruby.ary_new_capa(discovery.classes().len());
+    for class in discovery.classes() {
+        let class = ruby.str_new(class);
+        class.freeze();
+        classes.push(class)?;
+    }
+    let rows = ruby.ary_new_capa(discovery.len());
+    for test in discovery {
+        let class: RString = classes.entry(test.class.cast_signed())?;
+        rows.push((class, ruby.str_new(test.name), test.start_line, test.end_line))?;
+    }
+    Ok(rows)
+}
+
+/// Checked against `PaddedCell::VERSION` before the extension is used.
+const fn version() -> &'static str {
+    env!("CARGO_PKG_VERSION")
+}
+
+fn discover(ruby: &Ruby, source: RString) -> Result<RArray, Error> {
+    let discovery = with_bytes(&[source], |bytes| padded_cell::discover(bytes[0]));
+    rows(ruby, &discovery)
+}
+
+/// Reads and parses on scoped threads: none is left running when Ruby forks
+/// next.
+fn discover_files(ruby: &Ruby, paths: RArray) -> Result<RArray, Error> {
+    let found = with_bytes(&strings(paths)?, |paths| {
+        let paths: Vec<&Path> =
+            paths.iter().map(|path| Path::new(OsStr::from_bytes(path))).collect();
+        padded_cell::discover_files(&paths)
+    })
+    .map_err(|error| file_error(ruby, &error))?;
+    let files = ruby.ary_new_capa(found.len());
+    for discovery in &found {
+        files.push(rows(ruby, discovery)?)?;
+    }
+    Ok(files)
+}
+
+fn plan(
+    classes: RArray,
+    durations: Vec<f64>,
+    seed: u64,
+    shard_index: u64,
+    shard_count: u64,
+) -> Result<Vec<Vec<usize>>, Error> {
+    Ok(with_bytes(&strings(classes)?, |classes| {
+        padded_cell::plan(classes, &durations, seed, shard_index, shard_count)
+    }))
+}
+
+#[magnus::init]
+fn init(ruby: &Ruby) -> Result<(), Error> {
+    let module = ruby.define_module("PaddedCellNative")?;
+    module.define_module_function("version", function!(version, 0))?;
+    module.define_module_function("discover", function!(discover, 1))?;
+    module.define_module_function("discover_files", function!(discover_files, 1))?;
+    module.define_module_function("plan", function!(plan, 5))?;
+    Ok(())
+}

data/lib/padded_cell/active_support_hooks.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'active_support/testing/parallelization'
+
+module PaddedCell
+  # Runs ActiveSupport's parallelization hooks around padded_cell's workers,
+  # as Rails runs them around its own: parallelize_before_fork,
+  # parallelize_setup, parallelize_teardown, and Active Record's per-worker
+  # test databases (app_test-0, app_test-1...).
+  #
+  # The runner installs this after loading the tests when ActiveSupport::TestCase
+  # is defined. Any parallelize(workers:) call in test_helper.rb stays inert:
+  # padded_cell is the executor.
+  module ActiveSupportHooks
+    module_function
+
+    def install
+      return if @installed
+
+      @installed = true
+      parallelization = ActiveSupport::Testing::Parallelization
+      PaddedCell.before_fork { parallelization.before_fork_hooks.each(&:call) }
+      PaddedCell.after_fork do |worker|
+        ActiveSupport::TestCase.parallel_worker_id = worker if ActiveSupport::TestCase.respond_to?(:parallel_worker_id=)
+        parallelization.after_fork_hooks.each { it.call(worker) }
+      end
+      PaddedCell.cleanup { |worker| parallelization.run_cleanup_hooks.each { it.call(worker) } }
+    end
+  end
+end

data/lib/padded_cell/cell.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module PaddedCell
+  # One test in its own forked process. Whatever the test does to its process
+  # (crash, exit, hang, monkey patches, leaked threads) stays in the cell.
+  module Cell
+    CHUNK = 65_536
+
+    module_function
+
+    # Runs klass#name in a fork and returns its Minitest::Result, or an error
+    # result when the cell died or ran past timeout seconds.
+    #
+    #   close: descriptors the cell must not keep open (the worker's socket)
+    def run(klass, name, seed:, timeout: nil, close: [])
+      # Binary: the result is Marshal data, and with Encoding.default_internal
+      # set (Rails sets it) a text-mode pipe transcodes what is written to it.
+      reader, writer = IO.pipe
+      [reader, writer].each(&:binmode)
+      started = Minitest.clock_time
+      pid = fork_cell(klass, name, seed, reader, writer, close)
+      writer.close
+      collect(pid, reader, klass, name, started, timeout)
+    ensure
+      reader&.close
+      writer&.close unless writer&.closed?
+    end
+
+    # Runs klass#name in this process. A test calling exit is reported as an
+    # error instead of ending the process.
+    def execute(klass, name)
+      klass.new(name).run
+    rescue SystemExit => e
+      Results.crashed(klass, name, "exit(#{e.status}) called during the test")
+    end
+
+    def fork_cell(klass, name, seed, reader, writer, close)
+      $stdout.flush
+      $stderr.flush
+      Process.fork do
+        status = 1
+        begin
+          reader.close
+          close.each(&:close)
+          srand(seed)
+          writer.write(Results.dump(execute(klass, name)))
+          status = 0
+        rescue Exception => e # anything: report it, then exit!
+          warn "padded_cell: cell #{klass}##{name} failed to report: #{e.class}: #{e.message}"
+        ensure
+          # exit! skips at_exit handlers: the cell is not the test run.
+          exit!(status)
+        end
+      end
+    end
+
+    def collect(pid, reader, klass, name, started, timeout)
+      data = read(reader, timeout && (started + timeout))
+      elapsed = Minitest.clock_time - started
+      if data.nil?
+        Process.kill(:KILL, pid)
+        Process.wait(pid)
+        return Results.crashed(klass, name, "timed out after #{timeout}s", time: elapsed)
+      end
+
+      _, status = Process.wait2(pid)
+      return Results.load(data, klass, name) unless data.empty?
+
+      Results.crashed(klass, name, Results.describe(status, 'cell'), time: elapsed)
+    end
+
+    # Everything until EOF, or nil when the deadline passes first.
+    def read(io, deadline)
+      data = String.new(encoding: Encoding::BINARY)
+      loop do
+        remaining = deadline && (deadline - Minitest.clock_time)
+        return if remaining&.<= 0
+        next unless io.wait_readable(remaining)
+
+        chunk = io.read_nonblock(CHUNK, exception: false)
+        return data if chunk.nil?
+
+        data << chunk if chunk.is_a?(String)
+      end
+    end
+  end
+end

data/lib/padded_cell/cli.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require 'optparse'
+require_relative 'runner'
+
+module PaddedCell
+  # The padded_cell command.
+  module CLI
+    BANNER = <<~TEXT
+      Usage: padded_cell [options] [paths...]
+
+      Runs a Minitest (or Rails) test suite, every test in its own forked cell.
+
+      Paths are files, directories, globs or file:line[-line][:line...].
+      Without paths: test/**/*_test.rb, minus test/{system,dummy,fixtures}.
+
+    TEXT
+
+    module_function
+
+    # Exit status: 0 passed, 1 failed, 2 usage or load error, 130 interrupted.
+    def start(argv, io: $stdout)
+      options = { io: }
+      parser(options).parse!(argv)
+      options[:paths] = argv
+      Runner.new(**options).run
+    rescue OptionParser::ParseError, ArgumentError, LoadFailure, Unsupported, SystemCallError => e
+      warn "padded_cell: #{e.message}"
+      2
+    end
+
+    def parser(options)
+      OptionParser.new do |opts|
+        opts.banner = BANNER
+        opts.version = VERSION
+        runner_options(opts, options)
+        output_options(opts, options)
+        opts.on('--version', 'Print the version') do
+          options[:io].puts "padded_cell #{VERSION}"
+          exit
+        end
+        opts.on('-h', '--help', 'Show this help') do
+          options[:io].puts opts
+          exit
+        end
+      end
+    end
+
+    def runner_options(opts, options)
+      opts.on('-j', '--jobs N', Integer, "Workers (default: #{Etc.nprocessors})") do
+        options[:jobs] = positive(it)
+      end
+      opts.on('-s', '--seed SEED', Integer, 'Order seed (default: random, or $SEED)') { options[:seed] = it }
+      opts.on('-n', '-i', '--name', '--include PATTERN', 'Only tests matching (/regexp/ or exact name)') do
+        options[:include] = it
+      end
+      opts.on('-e', '-x', '--exclude PATTERN', 'Skip tests matching (/regexp/ or exact name)') do
+        options[:exclude] = it
+      end
+      opts.on('--shard I/N', 'Run shard I of N (1-based), split by class') { options[:shard] = shard(it) }
+      opts.on('--isolation MODE', %w[test worker], 'test: a fork per test; worker: tests share their worker ' \
+                                                   '(default: worker with Active Record fixtures, else test)') do
+        options[:isolation] = it.to_sym
+      end
+      opts.on('--timeout SECONDS', Float, 'Kill and fail a test running longer') do
+        options[:timeout] = positive(it)
+      end
+      opts.on('--fail-fast', 'Stop dispatching after the first failure') { options[:fail_fast] = true }
+      opts.on('--list', 'List the tests that would run (static, loads nothing)') { options[:list] = true }
+    end
+
+    def output_options(opts, options)
+      opts.on('-v', '--verbose', 'One line per test, with timings') { options[:verbose] = true }
+      opts.on('--show-skips', 'Show skipped tests at the end') { options[:show_skips] = true }
+      opts.on('--junit PATH', 'Also write JUnit XML to PATH') { options[:junit] = it }
+      opts.on('--json PATH', 'Also write JSON lines to PATH') { options[:json] = it }
+      opts.on('--timings PATH', 'Timings file (default: tmp/padded_cell/timings.tsv)') { options[:timings] = it }
+      opts.on('--no-timings', 'Neither read nor write timings') { options[:timings] = nil }
+    end
+
+    # OptionParser names the switch in the message.
+    def positive(value)
+      raise OptionParser::InvalidArgument, "#{value} (must be positive)" unless value.positive?
+
+      value
+    end
+
+    def shard(value)
+      index, count = value.split('/', 2).map { Integer(it, 10, exception: false) }
+      unless index && count&.positive? && index.between?(1, count)
+        raise OptionParser::InvalidArgument, "--shard #{value} (expected I/N, 1 <= I <= N)"
+      end
+
+      [index - 1, count]
+    end
+  end
+end

data/lib/padded_cell/discovery.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module PaddedCell
+  # A statically discovered test.
+  TestRef = Data.define(:file, :class_name, :name, :start_line, :end_line) do
+    def id = "#{class_name}##{name}"
+
+    def lines = start_line..end_line
+  end
+
+  # Static discovery, without loading anything.
+  module Discovery
+    module_function
+
+    # TestRefs for files, in file order, then source order. The extension
+    # reads and parses the files itself, on every core.
+    def call(files) = PaddedCellNative.discover_files(files).zip(files).flat_map { |rows, file| refs(rows, file) }
+
+    # TestRefs for one source string.
+    def source(source, file: '(source)') = refs(PaddedCellNative.discover(source), file)
+
+    def refs(rows, file) = rows.map { TestRef.new(file, *it) }
+  end
+end

data/lib/padded_cell/hooks.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module PaddedCell
+  # Process lifecycle hooks, for anything that must not be shared across
+  # forks (database connections, sockets, per-worker resources):
+  #
+  #   PaddedCell.before_fork { Database.disconnect }
+  #   PaddedCell.after_fork { |worker| Database.connect("app_test_#{worker}") }
+  #   PaddedCell.cleanup { |worker| Database.drop("app_test_#{worker}") }
+  #
+  # before_fork runs in the runner before workers start, after_fork in each
+  # worker (worker index 0...jobs) and cleanup when a worker finishes.
+  # Cells (one fork per test) run no hooks: they inherit their worker.
+  module Hooks
+    @before_fork = []
+    @after_fork = []
+    @cleanup = []
+
+    class << self
+      def before_fork(&block) = @before_fork << block
+
+      def after_fork(&block) = @after_fork << block
+
+      def cleanup(&block) = @cleanup << block
+
+      def run_before_fork = @before_fork.each(&:call)
+
+      def run_after_fork(worker) = @after_fork.each { it.call(worker) }
+
+      def run_cleanup(worker) = @cleanup.each { it.call(worker) }
+    end
+  end
+end

data/lib/padded_cell/loader.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module PaddedCell
+  # A test file that would not load.
+  class LoadFailure < StandardError; end
+
+  # Loads test files once, in the runner, so every worker and cell starts
+  # from the same preloaded (copy-on-write) process.
+  module Loader
+    # Test helpers require minitest/autorun (or Rails' equivalent): its
+    # at_exit would run the whole suite again, unisolated.
+    module NoAutorun
+      def autorun = nil
+    end
+
+    module_function
+
+    def call(files, seed:)
+      Minitest.singleton_class.prepend(NoAutorun)
+      Minitest.seed = seed
+      # What `rake test` and `rails test` put on the load path.
+      %w[test lib].each do |dir|
+        path = File.expand_path(dir)
+        $LOAD_PATH.unshift(path) if File.directory?(path) && !$LOAD_PATH.include?(path)
+      end
+      files.each { load_file(it) }
+    end
+
+    def load_file(file)
+      require file
+    rescue ScriptError, StandardError => e
+      raise LoadFailure, "#{file}: #{e.class}: #{e.message}"
+    end
+
+    # [klass, method_name] for every loaded test, the way Minitest finds them.
+    def tests
+      Minitest::Runnable.runnables.flat_map do |klass|
+        klass.runnable_methods.map { [klass, it] }
+      rescue NotImplementedError
+        []
+      end
+    end
+  end
+
+  # Which loaded tests run: Minitest's --include/--exclude, plus file:line.
+  class Filter
+    def initialize(selection, refs, include: nil, exclude: nil)
+      @selection = selection
+      @include = pattern(include)
+      @exclude = pattern(exclude)
+      @ends = refs.to_h { [[it.file, it.start_line], it.end_line] }
+      @selected = selection.files.to_h { [it, true] }
+    end
+
+    def call(klass, name)
+      id = "#{klass}##{name}"
+      return false if @include && !matches?(@include, name, id)
+      return false if @exclude && matches?(@exclude, name, id)
+
+      !@selection.line_filtered? || line_selected?(klass, name)
+    end
+
+    private
+
+    # "/re/" is a Regexp, anything else an exact name, as in Minitest.
+    def pattern(value)
+      match = value.is_a?(String) && %r{\A/(.*)/\z}.match(value)
+      match ? Regexp.new(match[1]) : value
+    end
+
+    def matches?(pattern, name, id) = [name, id].any? { pattern === it }
+
+    # With file:line arguments, a test runs when its file was named and its
+    # definition overlaps a requested line, or its file was named whole.
+    def line_selected?(klass, name)
+      file, line = klass.instance_method(name).source_location
+      return false unless file && @selected[file]
+
+      @selection.covers?(file, line..@ends.fetch([file, line], line))
+    end
+  end
+end

data/lib/padded_cell/native.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# Loads the Rust extension, which does discovery and scheduling. There is no
+# fallback: platform gems ship it precompiled under the Ruby ABI version, the
+# source gem compiles it at install, and a checkout builds it with
+# `rake compile`.
+
+require 'rbconfig'
+
+shipped = File.join(__dir__, RUBY_VERSION[/\A\d+\.\d+/], "padded_cell_native.#{RbConfig::CONFIG['DLEXT']}")
+begin
+  File.exist?(shipped) ? require(shipped) : require('padded_cell/padded_cell_native')
+rescue LoadError => e
+  raise LoadError, "padded_cell: native extension not loaded (#{e.message}). Precompiled gems need " \
+                   'glibc 2.34+ (Linux), macOS 13+ or FreeBSD 15.1; elsewhere the gem builds it with Rust ' \
+                   'at install. In a checkout, run `rake compile`.'
+end
+
+# A stale build may implement other semantics.
+unless PaddedCellNative.version == PaddedCell::VERSION
+  raise LoadError, "padded_cell: native extension #{PaddedCellNative.version} does not match padded_cell " \
+                   "#{PaddedCell::VERSION}: rebuild it (`rake compile` in a checkout, or reinstall the gem)"
+end