deflake 0.1.0

data/lib/deflake/leaks/current.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Deflake
+  module Leaks
+    # ActiveSupport::CurrentAttributes left set, or with a default changed in place
+    class Current
+      # Attributes left set, and a print of every default that is not a proc
+      State = Struct.new(:attributes, :defaults)
+
+      def snapshot
+        return unless defined?(::ActiveSupport) && Leaks.loaded?(::ActiveSupport, :CurrentAttributes)
+
+        classes = ::ActiveSupport::CurrentAttributes.descendants.select(&:name)
+        attributes = classes.filter_map do |klass|
+          instance = existing_instance(klass)
+          [klass.name, changed_attributes(klass, instance)] if instance
+        end
+        State.new(attributes.to_h, classes.to_h { |klass| [klass.name, defaults(klass)] })
+      end
+
+      def diff(before, after)
+        set(before.attributes, after.attributes) + changed_defaults(before.defaults, after.defaults)
+      end
+
+      private
+
+      def set(before, after)
+        after.flat_map do |klass, attributes|
+          attributes.filter_map do |name, value|
+            next if NilClass === value # rubocop:disable Style/CaseEquality
+            next if before.fetch(klass, {}).key?(name) && same?(before[klass][name], value)
+
+            "#{klass}.#{name} is still set"
+          end
+        end
+      end
+
+      # ActiveSupport copies a default only one level deep, so a test that changes a list inside
+      # one changes the default of every later test
+      def changed_defaults(before, after)
+        after.flat_map do |klass, defaults|
+          defaults.filter_map do |name, print|
+            was = before.fetch(klass, {})[name]
+            "#{klass}.#{name}'s default was changed in place" if was && print && was != print
+          end
+        end
+      end
+
+      # Asking the class for its attributes would create the instance and run the
+      # proc defaults before the test, which may travel in time first. A class
+      # without an instance has nothing set
+      def existing_instance(klass)
+        return unless klass.respond_to?(:current_instances, true)
+
+        key = klass.respond_to?(:current_instances_key, true) ? klass.send(:current_instances_key) : klass.name
+        klass.send(:current_instances)[key]
+      end
+
+      # Attributes that differ from their defaults. A default given as a proc gives a new value
+      # after every reset, so such attributes are not compared. A value may be any object, even a
+      # mock without Kernel's methods
+      def changed_attributes(klass, instance)
+        defaults = klass.respond_to?(:defaults) ? klass.defaults : {}
+        instance.attributes.reject do |name, value|
+          default = defaults[name]
+          Proc === default || NilClass === value || same?(value, default) # rubocop:disable Style/CaseEquality
+        end
+      end
+
+      def defaults(klass)
+        return {} unless klass.respond_to?(:defaults)
+
+        klass.defaults.reject { |_, default| Proc === default }.transform_values do |default| # rubocop:disable Style/CaseEquality
+          Marshal.dump(default).hash
+        rescue StandardError
+          nil
+        end
+      end
+
+      def same?(one, other)
+        one.__id__ == other.__id__ || one == other
+      rescue StandardError
+        false
+      end
+    end
+  end
+end

data/lib/deflake/leaks/env.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Deflake
+  module Leaks
+    # Environment variables set, changed or removed
+    class Env
+      def snapshot
+        ENV.to_h
+      end
+
+      def diff(before, after)
+        (before.keys | after.keys).sort.filter_map do |key|
+          next if before[key] == after[key]
+
+          change = if !after.key?(key) then "was removed"
+                   elsif before.key?(key) then "was changed"
+                   else "was set"
+                   end
+          "ENV[#{key.inspect}] #{change}"
+        end
+      end
+    end
+  end
+end

data/lib/deflake/leaks/globals.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+module Deflake
+  module Leaks
+    # Global variables ($name) assigned or changed in place
+    class Globals
+      # Ruby itself changes these all the time, including the English names of $?, $., $_ and friends
+      IGNORED = %i[
+        $LOADED_FEATURES $LOAD_PATH $PROGRAM_NAME $FILENAME $DEBUG $VERBOSE $stdin $_
+        $ERROR_INFO $ERROR_POSITION $FS $FIELD_SEPARATOR $OFS $OUTPUT_FIELD_SEPARATOR
+        $RS $INPUT_RECORD_SEPARATOR $ORS $OUTPUT_RECORD_SEPARATOR $INPUT_LINE_NUMBER $NR
+        $LAST_READ_LINE $DEFAULT_OUTPUT $DEFAULT_INPUT $PID $PROCESS_ID $CHILD_STATUS
+        $LAST_MATCH_INFO $IGNORECASE $ARGV $MATCH $PREMATCH $POSTMATCH $LAST_PAREN_MATCH
+      ].freeze
+      # How deep fingerprint walks into containers
+      DEPTH = 32
+
+      # An object Marshal could dump (digest) or not (nil), known by its identity too
+      Leaf = Struct.new(:id, :digest)
+      # A container, part by part, and its digest when Marshal could dump it whole. In a Hash or a
+      # Set the order of the parts does not matter
+      Box = Struct.new(:id, :type, :unordered, :parts, :digest)
+
+      # Ruby creates a global as nil the first time code reads it, so nil counts as unset.
+      # The value may be any object, even one without Kernel's methods, such as a mock
+      def snapshot
+        names = global_variables.grep(/\A\$[A-Za-z_]\w*\z/) - IGNORED
+        names.filter_map do |name|
+          value = TOPLEVEL_BINDING.eval(name.to_s)
+          [name, safely(value)] unless NilClass === value # rubocop:disable Style/CaseEquality
+        end.to_h
+      end
+
+      def diff(before, after)
+        (before.keys | after.keys).sort.filter_map do |name|
+          next if before.key?(name) && after.key?(name) && same?(before[name], after[name])
+
+          before.key?(name) ? "#{name} was changed" : "#{name} was set"
+        end
+      end
+
+      private
+
+      def safely(value)
+        fingerprint(value)
+      rescue StandardError, ScriptError, SystemStackError
+        Leaf.new(value.__id__, nil)
+      end
+
+      # The contents, so a change inside any object counts, not only inside strings, arrays and
+      # hashes. Containers are walked part by part, down to DEPTH, so a proc added to an array
+      # counts as a change. Any other object is dumped with Marshal, and one Marshal cannot dump
+      # (a Mutex, a proc, an IO) counts by identity: what it keeps inside, such as a lazily opened
+      # connection, is its own business. So does one Marshal could dump before the test and not
+      # after, or the other way round: it may have set up something inside on first use
+      def fingerprint(value)
+        walk(value, {}.compare_by_identity, 0)
+      end
+
+      def walk(value, seen, depth)
+        # Deeper parts and containers met before count by identity: walking them again would never end
+        return Leaf.new(value.__id__, nil) if depth > DEPTH || seen.key?(value)
+
+        parts = parts(value)
+        return Leaf.new(value.__id__, dump_or_nil(value)) unless parts
+
+        seen[value] = true
+        box(value, parts.map { |part| walk(part, seen, depth + 1) }, depth.zero?)
+      end
+
+      # Only the whole value gets a digest: it makes comparing an unchanged value quick
+      def box(value, parts, whole)
+        Box.new(value.__id__, type(value), unordered?(value), parts, whole ? dump_or_nil(value) : nil)
+      end
+
+      # A Hash also by its default and by how it compares keys
+      def type(value)
+        return value.class unless Hash === value # rubocop:disable Style/CaseEquality
+
+        [value.class, value.compare_by_identity?, dump_or_nil(value.default), !value.default_proc.nil?]
+      end
+
+      def dump_or_nil(value)
+        Marshal.dump(value).hash
+      rescue StandardError, SystemStackError
+        nil
+      end
+
+      def same?(before, after)
+        return before.id == after.id unless before.instance_of?(after.class)
+        return leaf_same?(before, after) if before.is_a?(Leaf)
+        return false unless before.type == after.type && before.parts.size == after.parts.size
+
+        box_same?(before, after)
+      end
+
+      def leaf_same?(before, after)
+        before.digest && after.digest ? before.digest == after.digest : before.id == after.id
+      end
+
+      def box_same?(before, after)
+        return true if before.digest && before.digest == after.digest
+        return before.parts.zip(after.parts).all? { |one, other| same?(one, other) } unless before.unordered
+
+        same_in_any_order?(*unmatched(before.parts, after.parts))
+      end
+
+      def same_in_any_order?(left, right)
+        left.all? do |part|
+          index = right.index { |other| same?(part, other) }
+          index && right.delete_at(index)
+        end
+      end
+
+      # The parts of a Hash or a Set that have no equal part on the other side, found by their
+      # digests first: comparing every part with every other would take long in a big Hash
+      def unmatched(before, after)
+        counts = before.map { |part| signature(part) }.tally
+        right = after.reject { |part| take(counts, signature(part)) }
+        [before.select { |part| take(counts, signature(part)) }, right]
+      end
+
+      # Takes one of the parts with this signature, if any is left
+      def take(counts, key)
+        counts[key].to_i.positive? && (counts[key] -= 1)
+      end
+
+      # Equal parts have equal signatures. An object Marshal cannot dump has its identity instead
+      def signature(part)
+        return part.digest || [:id, part.id].hash if part.is_a?(Leaf)
+
+        parts = part.parts.map { |inner| signature(inner) }
+        [part.type, part.unordered ? parts.sort : parts].hash
+      end
+
+      def unordered?(value)
+        Hash === value || (defined?(::Set) && ::Set === value) # rubocop:disable Style/CaseEquality
+      end
+
+      # The parts of a container, nil for anything else. A Hash gives its entries, as pairs.
+      # Set and StringIO may not be loaded
+      def parts(value)
+        case value
+        when Array, Struct, Hash then value.to_a
+        when Thread::Queue then [value.size, value.closed?]
+        else loaded_parts(value)
+        end
+      end
+
+      def loaded_parts(value)
+        if defined?(::Set) && ::Set === value then value.to_a # rubocop:disable Style/CaseEquality
+        elsif defined?(::StringIO) && ::StringIO === value then [value.string, value.pos] # rubocop:disable Style/CaseEquality
+        end
+      end
+    end
+  end
+end

data/lib/deflake/leaks/jobs.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Deflake
+  module Leaks
+    # Jobs left in the ActiveJob test queue
+    class Jobs
+      # Ids of the enqueued jobs: ActiveJob::TestHelper empties the queue before
+      # every test, so comparing sizes would miss a job left after another one
+      def snapshot
+        return unless defined?(::ActiveJob) && Leaks.loaded?(::ActiveJob, :Base)
+
+        # Not queue_adapter: when no adapter is set, it sets :async, which would
+        # stop ActiveJob::TestHelper from installing its test adapter
+        adapter = ::ActiveJob::Base._queue_adapter
+        adapter.enqueued_jobs.map(&:object_id) if adapter.respond_to?(:enqueued_jobs)
+      end
+
+      def diff(before, after)
+        added = (after - before).size
+        return [] unless added.positive?
+
+        ["#{added} enqueued #{added == 1 ? "job" : "jobs"} left in the ActiveJob queue"]
+      end
+    end
+  end
+end

data/lib/deflake/leaks/rows.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+module Deflake
+  module Leaks
+    # Database rows created or deleted outside a rolled back transaction, in every connected database
+    class Rows
+      INTERNAL_TABLES = %w[schema_migrations ar_internal_metadata].freeze
+      MAIN = "ActiveRecord::Base"
+
+      # Row counts per model class that owns a connection, then per table
+      # ({ "ActiveRecord::Base" => { "users" => 2 } }), and how many times fixtures were loaded so far
+      State = Struct.new(:counts, :loads)
+
+      # The first test of a process loads fixtures in its setup, and those rows
+      # are not its leak, so the counts right after the load are its baseline.
+      # The tables a load wrote to are those of its fixture sets and any other
+      # whose count it changed, such as the join table of has_and_belongs_to_many.
+      # A load inside a transaction the test opened, which rolls back, is no baseline. One inside
+      # a transaction that was open before the test, as test-prof's before_all opens, is
+      module FixtureLoad
+        def create_fixtures(...)
+          before = Leaks.quietly { Rows.new.snapshot&.counts }
+          super.tap do |sets|
+            Leaks.quietly do
+              after = Rows.new.snapshot&.counts
+              tables = Rows.in_new_transaction? ? [] : Array(sets).map(&:table_name) | Rows.changed(before, after)
+              Rows.fixtures_loaded(after, tables)
+            end
+          end
+        end
+      end
+
+      class << self
+        attr_reader :loads
+        # Transactions open at the last snapshot, before a test
+        attr_accessor :depth
+
+        # tables: the tables this load wrote to, nil when not known
+        def fixtures_loaded(counts, tables = nil)
+          @loads = loads.to_i + 1
+          fixture_loads[@loads] = [counts, tables]
+        end
+
+        # Load number => [the counts right after it, the tables it wrote to]
+        def fixture_loads
+          @fixture_loads ||= {}
+        end
+
+        # The counts right after the last load
+        def after_fixtures = fixture_loads[loads]&.first
+
+        # Tables whose row count differs between two snapshots
+        def changed(before, after)
+          after.to_h.flat_map do |owner, tables|
+            tables.reject { |table, count| before&.dig(owner, table) == count }.keys
+          end
+        end
+
+        def in_new_transaction? = open_transactions > depth.to_i
+
+        def open_transactions = ::ActiveRecord::Base.connection_pool.with_connection(&:open_transactions)
+      end
+
+      def snapshot
+        return unless defined?(::ActiveRecord) && Leaks.loaded?(::ActiveRecord, :Base)
+
+        watch_fixtures
+        pools = connected_pools
+        return if pools.empty?
+
+        Rows.depth = Rows.open_transactions
+        State.new(pools.to_h { |owner, pool| [owner, count(pool)] }, Rows.loads.to_i)
+      end
+
+      # Only databases connected before the test are compared: the rows of one
+      # the test connected to first were there before it
+      def diff(before, after)
+        baselines = baselines(before, after)
+        changes = before.counts.flat_map do |owner, tables|
+          changes(owner, tables, after.counts.fetch(owner, {}), baselines[owner])
+        end
+        changes.sort.map { |table, change| message(table, change) }
+      end
+
+      private
+
+      # Owner => { table => rows right after the last load during the test that wrote to it }.
+      # A load whose tables are not known wrote to every table
+      def baselines(before, after)
+        ((before.loads + 1)..after.loads).each_with_object(Hash.new { |hash, owner| hash[owner] = {} }) do |load, all|
+          counts, tables = Rows.fixture_loads[load]
+          counts.to_h.each do |owner, rows|
+            all[owner].merge!(tables ? rows.slice(*tables) : rows)
+          end
+        end
+      end
+
+      # fixtures: rows in each table the fixtures were loaded into, right after the load
+      def changes(owner, before, after, fixtures)
+        after.filter_map do |table, now|
+          change = change(now, before.fetch(table, 0), fixtures&.fetch(table, nil))
+          [name(owner, table), change] if change
+        end
+      end
+
+      # Loading fixtures replaces whatever a fixture table held, so the change is counted from
+      # the load. A test without a transaction that empties the table in teardown, as such tests
+      # do with the fixtures they loaded, leaves nothing behind
+      def change(now, before, loaded)
+        change = now - (loaded || before)
+        change unless change.zero? || (loaded && now.zero?)
+      end
+
+      # Tables of the main database go by their names, others get the model class in front: "AnimalsRecord.dogs"
+      def name(owner, table)
+        owner == MAIN ? table : "#{owner}.#{table}"
+      end
+
+      def message(table, change)
+        rows = "#{change.abs} #{change.abs == 1 ? "row" : "rows"}"
+        change.positive? ? "#{table}: #{rows} left behind" : "#{table}: #{rows} deleted for good"
+      end
+
+      # Model class that owns the connection ("ActiveRecord::Base", "AnimalsRecord") => its pool.
+      # Shards other than the default one go by their name too: "ShardRecord[one]"
+      def connected_pools
+        handler = ::ActiveRecord::Base.connection_handler
+        writing = ::ActiveRecord.respond_to?(:writing_role) ? ::ActiveRecord.writing_role : :writing
+        shards = handler.connection_pool_list(writing).map(&:shard).uniq
+        handler.connection_pool_names.product(shards).filter_map do |owner, shard|
+          pool = handler.retrieve_connection_pool(owner, role: writing, shard: shard)
+          [shard == ::ActiveRecord::Base.default_shard ? owner : "#{owner}[#{shard}]", pool] if pool&.connected?
+        end
+      end
+
+      # Outside the query cache: a count the test makes itself, such as User.count, must still go to
+      # the database, and must not find the cache emptied by deflake either
+      def count(pool)
+        pool.with_connection do |connection|
+          uncached(connection) do
+            (connection.tables - INTERNAL_TABLES).to_h do |table|
+              [table, connection.select_value("SELECT COUNT(*) FROM #{connection.quote_table_name(table)}").to_i]
+            end
+          end
+        end
+      end
+
+      def uncached(connection, &)
+        return yield unless connection.respond_to?(:uncached)
+
+        # Rails 7.2 and newer would let the counts empty the cache of other connections
+        dirties = connection.method(:uncached).parameters.any? { |_, name| name == :dirties }
+        dirties ? connection.uncached(dirties: false, &) : connection.uncached(&)
+      end
+
+      # Before fixtures load for the first time, without loading them: Rails tells when they do
+      def watch_fixtures
+        if Leaks.loaded?(::ActiveRecord, :FixtureSet)
+          prepend_to(::ActiveRecord::FixtureSet)
+        elsif !@waiting && defined?(::ActiveSupport.on_load)
+          @waiting = true
+          ::ActiveSupport.on_load(:active_record_fixture_set) { Deflake::Leaks::Rows.new.send(:prepend_to, self) }
+        end
+      end
+
+      def prepend_to(fixture_set)
+        fixture_set.singleton_class.prepend(FixtureLoad) unless fixture_set.singleton_class.include?(FixtureLoad)
+      end
+    end
+  end
+end

data/lib/deflake/leaks.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative "leaks/env"
+require_relative "leaks/clock"
+require_relative "leaks/globals"
+require_relative "leaks/cache"
+require_relative "leaks/current"
+require_relative "leaks/rows"
+require_relative "leaks/jobs"
+
+module Deflake
+  # Global state a test changed and did not put back. Every detector has two methods:
+  # snapshot returns the current state (nil when it does not apply), and
+  # diff(before, after) returns one message per leak. Add your own in test_helper:
+  #
+  #   Deflake::Leaks.detectors << MyDetector.new
+  module Leaks
+    module_function
+
+    def detectors
+      @detectors ||= [Env, Clock, Globals, Cache, Current, Rows, Jobs].map(&:new)
+    end
+
+    def snapshot
+      detectors.to_h { |detector| [detector, quietly { detector.snapshot }] }
+    end
+
+    def since(snapshot)
+      snapshot.flat_map do |detector, before|
+        next [] if before.nil?
+
+        after = quietly { detector.snapshot }
+        after.nil? ? [] : quietly { detector.diff(before, after) } || []
+      end
+    end
+
+    # Whether a constant is there without loading it: asking for an autoloaded class such as
+    # ActiveRecord::Base would load it inside a test, and run the hooks that wait for it there
+    def loaded?(namespace, name)
+      namespace.const_defined?(name, false) && !namespace.autoload?(name)
+    end
+
+    # Leaks of a group of tests that are not just its tests' leaks added up: "2 jobs" after two
+    # tests that each left "1 job" is the same leak, and so are two new cache keys after two tests
+    # that each left one
+    def own(leaks, of_tests)
+      kinds = of_tests.map { |leak| kind(leak) }
+      leaks.reject { |leak| kinds.include?(kind(leak)) }
+    end
+
+    # What a leak is about, without its count and the list after a colon. A number inside a name,
+    # as in $retry2, stays
+    def kind(leak)
+      leak.sub(/:\s.*\z/m, "").sub(/\b\d+\b.*\z/m) { |rest| rest.gsub(/\b\d+\b/, "N").gsub(/s\b/, "") }
+    end
+
+    # A broken detector must never break the tests it watches. NotImplementedError
+    # (an adapter without some SQL) is a ScriptError, and a too deep structure raises
+    # a SystemStackError, neither of them a StandardError
+    def quietly
+      yield
+    rescue StandardError, ScriptError, SystemStackError
+      nil
+    end
+  end
+end

data/lib/deflake/minitest.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "minitest_reporter"
+
+if Gem::Version.new(Minitest::VERSION) < Gem::Version.new("5.11")
+  abort "deflake needs Minitest 5.11 or newer, this is #{Minitest::VERSION}"
+end
+
+module Minitest
+  # Minitest calls this at startup because the "deflake" plugin is registered
+  # below. Does nothing unless deflake started this process
+  def self.plugin_deflake_init(options)
+    path = ENV.delete(Deflake::ResultFile::ENV_KEY) or return
+
+    if ENV.delete(Deflake::ResultFile::LIST_KEY)
+      Deflake::MinitestTests.list(path)
+      # Only listing was asked for, so Minitest itself runs nothing.
+      # A string, because Minitest 5 cannot take a Regexp here
+      return options[:exclude] = "/./"
+    end
+
+    # Minitest still runs the tests itself, so class-level hooks and other plugins work as usual
+    ids = Deflake::MinitestTests.requested
+    Deflake::MinitestTests.run_only(ids) if ids
+    reporter << Deflake::MinitestReporter.new(path, options[:seed])
+  end
+end
+
+if ENV.key?(Deflake::ResultFile::ENV_KEY)
+  # Before the test file loads, so a test that a macro defines is found at the line that calls it
+  list = ENV.fetch(Deflake::ResultFile::LIST_KEY, nil)
+  Deflake::MinitestTests.watch_declarations(list) if list
+  Minitest::Runnable.singleton_class.prepend(Deflake::MinitestTests::ClassLeaks)
+end
+
+# Minitest 5 loads the plugins of other gems only while no plugin is registered, so load them first
+Minitest.load_plugins if Minitest::VERSION.to_i < 6 && !ENV["MT_NO_PLUGINS"] && !ARGV.include?("--no-plugins")
+Minitest.extensions << "deflake" unless Minitest.extensions.include?("deflake")

data/lib/deflake/minitest_reporter.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require "minitest"
+require_relative "result_file"
+require_relative "leaks"
+require_relative "minitest_tests"
+
+module Deflake
+  # Writes the result of every test to a file, one JSON line per test.
+  # Also checks every test for leaks: prerecord and record run outside the test's
+  # own run method, so whatever Rails does around it (the executor clearing
+  # Current, a rolled back transaction) has already happened.
+  # Tests in anonymous classes have no name to run them by, and results that are not tests (a
+  # class-level hook such as minitest-hooks' before_all reports its error as a result) cannot be
+  # run alone: their failures are written as failures outside of tests
+  class MinitestReporter < Minitest::AbstractReporter
+    class << self
+      # The reporter of this process, nil until Minitest starts it
+      attr_accessor :current
+    end
+
+    def initialize(path, seed)
+      super()
+      @path = path
+      @seed = seed
+      # Test id => state before it. parallelize_me! runs several tests at once
+      @before = {}
+      # Every leak the tests reported so far
+      @leaks = []
+      @lock = Mutex.new
+      ResultFile.start(path)
+      self.class.current = self
+    end
+
+    def prerecord(klass, name)
+      prefix = MinitestTests.prefix(klass) or return
+
+      id = "#{prefix}##{name}"
+      ResultFile.append(@path, started: id, name: id)
+      snapshot = Leaks.snapshot
+      @lock.synchronize { @before[id] = snapshot }
+    end
+
+    def record(result)
+      return record_unnamed(result, "#{result.name} (in a class without a name)") unless result.klass
+
+      id, before = @lock.synchronize { take(result) }
+      # Every test Minitest runs is prerecorded first
+      return record_unnamed(result, "#{id} (not a test)") unless before
+
+      leaks = Leaks.since(before)
+      @lock.synchronize { @leaks.concat(leaks) }
+      return ResultFile.append(@path, id: id, name: id, skipped: true, seed: @seed, leaks: leaks) if result.skipped?
+
+      ResultFile.append(@path, id: id, name: id, passed: result.passed?, seed: @seed,
+                               error: error_message(result.failure), leaks: leaks)
+    end
+
+    # Leaks of a whole class that none of its tests reported, as errors outside of tests
+    def around_class(klass)
+      before = Leaks.snapshot
+      reported = @leaks.size
+      yield
+    ensure
+      leaks = Leaks.own(Leaks.since(before), @leaks.drop(reported)) if before
+      if leaks&.any?
+        label = MinitestTests.prefix(klass) || "a class without a name"
+        ResultFile.append(@path, outside: nil, leaks: leaks.map { |leak| "#{label}, outside its tests: #{leak}" })
+      end
+    end
+
+    private
+
+    # The id of the test this result is for, and the state before it. A result names only its
+    # class, so of two classes with the same name the one whose test started is meant
+    def take(result)
+      ids = MinitestTests.named(result.klass).map { |klass| "#{MinitestTests.prefix(klass)}##{result.name}" }
+      ids = ["#{result.klass}##{result.name}"] if ids.empty?
+      id = ids.find { |candidate| @before.key?(candidate) } || ids.first
+      [id, @before.delete(id)]
+    end
+
+    # Passing ones are written too: they tell that tests ran, which the order of a bisect leaves out
+    def record_unnamed(result, label)
+      return ResultFile.append(@path, unnamed: label, passed: true) if result.passed? || result.skipped?
+
+      ResultFile.append(@path, unnamed: label, error: error_message(result.failure))
+    end
+
+    # An unexpected exception carries its backtrace in the message, which would
+    # split identical errors into separate groups
+    def error_message(failure)
+      case failure
+      when nil then nil
+      when Minitest::UnexpectedError then "#{failure.error.class}: #{failure.error.message}"
+      else failure.message
+      end
+    end
+  end
+end