hecks 5.0.0 → 7.0.0

data/lib/hecks/ci_speed/adapters/local_runner.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require "hecks/ci_speed/adapters/local_runner/refused"
+require "hecks/ci_speed/adapters/local_runner/refs"
+require "hecks/ci_speed/adapters/local_runner/catalog"
+require "hecks/ci_speed/adapters/local_runner/checkouts"
+require "hecks/ci_speed/adapters/local_runner/launcher"
+require "hecks/ci_speed/adapters/local_runner/load_average"
+require "hecks/ci_speed/adapters/local_runner/verdict"
+require "hecks/ci_speed/adapters/local_runner/answer"
+
+module Hecks
+  module Adapters
+    # The `LocalRunner` port's adapter: times a definition's steps in two temporary checkouts, one
+    # of the base ref and one of the candidate ref, and answers with the numbers.
+    #
+    # Each side first runs every step once untimed, so the build caches a fresh checkout lacks are
+    # paid before timing starts. The steps then run interleaved, base then candidate then base
+    # again, so a change in the machine's load while the experiment runs lands on both sides. A
+    # step the machine cannot stand in for is named in `unmeasured` and left out of both sides.
+    # Both checkouts are removed when the run ends, however it ends, and the checkout that asked
+    # is never written.
+    class LocalRunner
+      # The sides of an experiment, in the order a round samples them.
+      SIDES = %i[base candidate].freeze
+
+      # What a step's `needs` can be and still run on this machine.
+      LOCAL_NEEDS = %w[none].freeze
+
+      # The samples taken when an experiment names no number.
+      DEFAULT_REPEATS = 3
+
+      # One side of an experiment: which it is, its checkout, and the launcher that runs its steps.
+      Side = Struct.new(:name, :path, :launcher)
+
+      # @return [Array<Hash>] every sample in the order it was taken: its round, side, seconds and
+      #   the checkout it ran in
+      attr_reader :trail
+
+      # @return [Array<Hash>] the untimed pass in each checkout before sampling began: its side
+      #   and the seconds it took, which no verdict uses
+      attr_reader :warm_up
+
+      # @param registry [Runtime::Registry, nil] the booted registry whose CiSpeed bluebook holds
+      #   the definitions; unused when `catalog` is given
+      # @param catalog [Catalog, nil] the definitions to read instead of the bluebook's rows
+      # @param root [String, nil] the checkout that asks, whose refs are resolved and whose
+      #   generated tree is copied; this repository when nil
+      # @param scratch [String, nil] where the temporary checkouts are made; `$CLAUDE_JOB_DIR/tmp`
+      #   when that is set, else the system's temporary directory
+      def initialize(registry: nil, catalog: nil, root: nil, scratch: nil)
+        @registry = registry
+        @catalog = catalog
+        @root = root || File.expand_path("../../../..", __dir__)
+        @scratch = scratch || job_scratch
+        @trail = []
+        @warm_up = []
+      end
+
+      # Runs the experiment named by `held`.
+      #
+      # @param held [Hash] the `Experiment` record: `measures`, `cache_state`, `base_ref`,
+      #   `candidate_ref`, `repeats` and `lever` when set
+      # @return [Hash{Symbol => Hash}] the answer, each field in the shape `Experiment::Complete`
+      #   takes: the times, the spread and saving, the load, what went unmeasured, the verdict,
+      #   what it rests on, and the lever to tell with the estimate it should hold
+      # @raise [Refused] when the definition is unknown, disagrees with the experiment about the
+      #   cache state, no step can be measured, a ref does not resolve, or a step ends badly
+      def measure(**held)
+        asked = held.transform_values { |value| value.is_a?(Hash) ? value[:value] : value }
+        definition = definition_for(asked)
+        shas = Refs.resolve(@root, asked)
+        load = LoadAverage.now
+        sampled = sample_both(definition, shas, asked[:repeats] || DEFAULT_REPEATS)
+        answer(asked, definition, sampled, load)
+      end
+
+      private
+
+      def definition_for(asked)
+        definition = catalog.fetch(asked.fetch(:measures))
+        return definition if definition.cache_state == asked.fetch(:cache_state)
+
+        raise Refused, "#{definition.name} measures cache state #{definition.cache_state}, " \
+                       "but the experiment stands for #{asked.fetch(:cache_state)}"
+      end
+
+      def catalog
+        @catalog ||= Catalog.from_bluebook(@registry&.bluebook("CiSpeed") || raise(Refused, "no CiSpeed bluebook is booted"))
+      end
+
+      # Checks the two sides out, works out which steps both can run, and samples them interleaved.
+      def sample_both(definition, shas, repeats)
+        copy_from = @root if definition.generated_tree?
+        refuse_missing_tree(copy_from)
+        Checkouts.with(root: @root, shas: shas, scratch: @scratch) do |paths|
+          sides = paths.map { |name, path| Side.new(name, path, Launcher.new(path)) }
+          steps, unmeasured = measurable(definition.steps, sides)
+          warm(sides, steps, copy_from)
+          { unmeasured: unmeasured, samples: rounds(sides, steps, copy_from, repeats) }
+        end
+      end
+
+      # Runs every step once in each checkout before anything is timed. A fresh checkout holds no
+      # build caches, and the first run in it would pay for them on one side of the first pair.
+      def warm(sides, steps, copy_from)
+        sides.each do |side|
+          Checkouts.reset_generated_tree(side.path, copy_from)
+          seconds = steps.sum { |step| side.launcher.time(step) }
+          @warm_up << { side: side.name, seconds: seconds }
+        end
+      end
+
+      # Splits steps into those both sides can run here and a note for each that cannot.
+      def measurable(steps, sides)
+        notes = []
+        kept = steps.select do |step|
+          reason = unmeasurable(step, sides)
+          notes << "#{step.name}: #{reason}" if reason
+          reason.nil?
+        end
+        raise Refused, "no step could be measured here: #{notes.join("; ")}" if kept.empty?
+
+        [kept, notes]
+      end
+
+      def unmeasurable(step, sides)
+        return "needs #{step.needs}, which a local run does not stand in for" unless LOCAL_NEEDS.include?(step.needs)
+
+        sides.each do |side|
+          return "#{step.verb} or an argument it is given is unknown to the #{side.name} ref" unless side.launcher.knows?(step)
+        end
+        nil
+      end
+
+      # One round samples each side once, base first, and the rounds follow one another.
+      def rounds(sides, steps, copy_from, repeats)
+        taken = Array.new(repeats) { |round| sides.map { |side| sample(round, side, steps, copy_from) } }
+        taken.flatten.group_by { |sample| sample[:side] }.transform_values { |own| own.map { |one| one[:seconds] } }
+      end
+
+      def refuse_missing_tree(copy_from)
+        return if copy_from.nil? || File.directory?(File.join(copy_from, Checkouts::GENERATED_TREE))
+
+        raise Refused, "the definition needs a generated tree, and #{copy_from} holds none at #{Checkouts::GENERATED_TREE}"
+      end
+
+      def sample(round, side, steps, copy_from)
+        Checkouts.reset_generated_tree(side.path, copy_from)
+        seconds = steps.sum { |step| side.launcher.time(step) }
+        taken = { round: round, side: side.name, seconds: seconds, checkout: side.path }
+        @trail << taken
+        taken
+      end
+
+      def answer(asked, definition, sampled, load)
+        verdict = Verdict.new(base: sampled[:samples].fetch(:base), candidate: sampled[:samples].fetch(:candidate))
+        Answer.build(asked: asked, definition: definition.name, verdict: verdict, load: load,
+                     unmeasured: sampled[:unmeasured])
+      end
+
+      def job_scratch
+        job = ENV.fetch("CLAUDE_JOB_DIR", nil)
+        File.join(job, "tmp") if job && !job.empty?
+      end
+    end
+  end
+end

data/lib/hecks/ci_speed/adapters/replayed_runner.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "hecks/ci_speed/adapters/local_runner"
+
+module Hecks
+  module Adapters
+    # A `LocalRunner` that replays samples recorded here instead of running anything, for
+    # `ci_speed.behaviors`: a behaviors boot must be deterministic and fast, and the real adapter
+    # makes two checkouts and times real work. The verdict, the saving and the estimate are worked
+    # out by the real adapter's own `Verdict` and `Answer`, so only the timing is replaced.
+    class ReplayedRunner
+      # The samples each experiment name replays, in seconds, as base and candidate.
+      RECORDINGS = {
+        "wide_win"     => { base: [120.0, 122.0, 124.0], candidate: [90.0, 91.0, 92.0] },
+        "within_noise" => { base: [120.0, 126.0, 123.0], candidate: [121.0, 124.0, 122.0] },
+        "wide_loss"    => { base: [90.0, 91.0, 92.0], candidate: [120.0, 122.0, 124.0] }
+      }.freeze
+
+      # The load average every replay reports.
+      LOAD = 2.5
+
+      # @param aggregate [Object, nil] unused
+      def initialize(aggregate: nil); end
+
+      # @param held [Hash] the `Experiment` record
+      # @return [Hash{Symbol => Hash}] the answer, as the real adapter words it
+      # @raise [LocalRunner::Refused] when nothing is recorded for the experiment's name
+      def measure(**held)
+        asked = held.transform_values { |value| value.is_a?(Hash) ? value[:value] : value }
+        samples = RECORDINGS.fetch(asked.fetch(:name)) do
+          raise LocalRunner::Refused, "no samples are recorded for #{asked[:name]}; recorded: #{RECORDINGS.keys.join(", ")}"
+        end
+        verdict = LocalRunner::Verdict.new(**samples)
+        LocalRunner::Answer.build(asked: asked, definition: asked.fetch(:measures), verdict: verdict, load: LOAD,
+                                  unmeasured: [])
+      end
+    end
+  end
+end

data/lib/hecks/ci_speed/ci_speed.behaviors

@@ -0,0 +1,341 @@
+Hecks.behaviors "CiSpeed" do
+  vision "A run is written down as it finished, and a change is kept only when a following run of its cache state beat the baseline by the margin."
+
+  # `ci_speed.hecksagon` binds every aggregate to Memory and attaches Governance, so a boot of these
+  # files passes the memory-only guard and the role check.
+  loads "../../../spec/fixtures/ci_speed/ci_speed.hecksagon", "ci_speed.bluebook", "ci_speed.ports.hecksagon"
+
+  test "Observe writes down a run as it finished" do
+    tests "Observe", on: "Run"
+    input run_id: { value: "1001" }, commit: { value: "93f8e68c6" }, wall_seconds: { value: 240 },
+          cache_state: { value: "miss" }, conclusion: { value: "success" }, budget_seconds: { value: 180 },
+          longest_pole: { value: "rspec" }
+    expect emits: ["RunObserved"]
+  end
+
+  test "RecordJob refuses a job time once the run is analysed" do
+    tests "RecordJob", on: "Run"
+    setup "CiSpeed::Run.Observe", run_id: { value: "1401" }, commit: { value: "93f8e68c6" },
+          wall_seconds: { value: 150 }, cache_state: { value: "hit" }, conclusion: { value: "success" },
+          budget_seconds: { value: 180 }, longest_pole: { value: "rspec" }
+    setup "CiSpeed::Run.AnalyzeWarmUnderBudget", run: "1401", seconds: { value: 150 }
+    input run: "1401", job: { value: "rspec" }, seconds: { value: 90 }, critical: { value: "yes" }
+    expect refused: "a run's job times are written before it is analysed"
+  end
+
+  test "AnalyzeWarmUnderBudget refuses a cache-miss run" do
+    tests "AnalyzeWarmUnderBudget", on: "Run"
+    setup "CiSpeed::Run.Observe", run_id: { value: "1301" }, commit: { value: "93f8e68c6" },
+          wall_seconds: { value: 150 }, cache_state: { value: "miss" }, conclusion: { value: "success" },
+          budget_seconds: { value: 180 }, longest_pole: { value: "rspec" }
+    input run: "1301", seconds: { value: 150 }
+    expect refused: "a warm run is a cache hit that finished within budget"
+  end
+
+  # A cache-miss run over budget proposes a lever for its longest pole, and its over-budget
+  # verdict is witnessed by the Budget.
+  test "A cache-miss run over budget proposes a lever for its longest pole" do
+    tests "AnalyzeMissOverBudget", on: "Run"
+    setup "CiSpeed::Budget.Open", name: { value: "main" }, target_seconds: { value: 180 }
+    setup "CiSpeed::Run.Observe", run_id: { value: "1002" }, commit: { value: "93f8e68c6" },
+          wall_seconds: { value: 240 }, cache_state: { value: "miss" }, conclusion: { value: "success" },
+          budget_seconds: { value: 180 }, longest_pole: { value: "rspec" }
+    setup "CiSpeed::Run.RecordJob", run: "1002", job: { value: "rspec" }, seconds: { value: 200 },
+          critical: { value: "yes" }
+    input run: "1002", seconds: { value: 240 }, pole: { value: "rspec" }
+    expect emits: %w[MissRunOverBudget MissRunWitnessed LeverProposed]
+  end
+
+  # A lever is kept when a following run of the same cache state beats its baseline by the margin.
+  test "Keep keeps an attempted lever that a same-cache run beat by the margin" do
+    tests "Keep", on: "Lever"
+    setup "CiSpeed::Lever.Propose", name: { value: "postgres_io" },
+          change: { value: "postgres io specs on fewer shards" }, pole: { value: "postgres_io" },
+          affects: { value: "miss" }, expected_seconds: { value: 30 }, basis: { value: "unestimated" }, inputs: { value: "" }
+    setup "CiSpeed::Lever.Attempt", lever: "postgres_io", baseline_run: { value: "1100" },
+          baseline_seconds: { value: 300 }, baseline_cache: { value: "miss" }
+    input lever: "postgres_io", following_run: { value: "1101" }, following_seconds: { value: 250 },
+          following_cache: { value: "miss" }, evidence: { value: "1a2b3c4d" }, saved_seconds: { value: 50 }
+    expect emits: ["LeverKept"]
+  end
+
+  test "Keep refuses a saving that is not the baseline less the following time" do
+    tests "Keep", on: "Lever"
+    setup "CiSpeed::Lever.Propose", name: { value: "postgres_io" },
+          change: { value: "postgres io specs on fewer shards" }, pole: { value: "postgres_io" },
+          affects: { value: "miss" }, expected_seconds: { value: 30 }, basis: { value: "unestimated" }, inputs: { value: "" }
+    setup "CiSpeed::Lever.Attempt", lever: "postgres_io", baseline_run: { value: "1100" },
+          baseline_seconds: { value: 300 }, baseline_cache: { value: "miss" }
+    input lever: "postgres_io", following_run: { value: "1101" }, following_seconds: { value: 250 },
+          following_cache: { value: "miss" }, evidence: { value: "1a2b3c4d" }, saved_seconds: { value: 10 }
+    expect refused: "a kept lever says what it really saved"
+  end
+
+  # A following run that is not faster by the margin reverts the lever.
+  test "Revert reverts an attempted lever that the following run did not beat" do
+    tests "Revert", on: "Lever"
+    setup "CiSpeed::Lever.Propose", name: { value: "rspec_4_shards" },
+          change: { value: "rspec on fewer shards" }, pole: { value: "rspec" },
+          affects: { value: "miss" }, expected_seconds: { value: 30 }, basis: { value: "unestimated" }, inputs: { value: "" }
+    setup "CiSpeed::Lever.Attempt", lever: "rspec_4_shards", baseline_run: { value: "1200" },
+          baseline_seconds: { value: 300 }, baseline_cache: { value: "miss" }
+    input lever: "rspec_4_shards", following_run: { value: "1201" }, following_seconds: { value: 295 },
+          following_cache: { value: "miss" }
+    expect emits: ["LeverReverted"]
+  end
+
+  test "Keep refuses a following run of a different cache state" do
+    tests "Keep", on: "Lever"
+    setup "CiSpeed::Lever.Propose", name: { value: "postgres_io" },
+          change: { value: "postgres io specs on fewer shards" }, pole: { value: "postgres_io" },
+          affects: { value: "both" }, expected_seconds: { value: 30 }, basis: { value: "unestimated" }, inputs: { value: "" }
+    setup "CiSpeed::Lever.Attempt", lever: "postgres_io", baseline_run: { value: "1100" },
+          baseline_seconds: { value: 300 }, baseline_cache: { value: "miss" }
+    input lever: "postgres_io", following_run: { value: "1101" }, following_seconds: { value: 200 },
+          following_cache: { value: "hit" }, evidence: { value: "1a2b3c4d" }, saved_seconds: { value: 100 }
+    expect refused: "a following run is of the cache state the lever was measured on"
+  end
+
+  test "Attempt refuses a lever measured on a cache state it does not affect" do
+    tests "Attempt", on: "Lever"
+    setup "CiSpeed::Lever.Propose", name: { value: "feature_binaries" },
+          change: { value: "feature binaries built in legs" }, pole: { value: "feature_binaries" },
+          affects: { value: "hit" }, expected_seconds: { value: 40 }, basis: { value: "unestimated" }, inputs: { value: "" }
+    input lever: "feature_binaries", baseline_run: { value: "1300" }, baseline_seconds: { value: 300 },
+          baseline_cache: { value: "miss" }
+    expect refused: "a lever is measured on the cache state it affects"
+  end
+
+  test "Reconsider refuses a reverted lever whose inputs have not changed" do
+    tests "Reconsider", on: "Lever"
+    setup "CiSpeed::Lever.Propose", name: { value: "rspec_4_shards" },
+          change: { value: "rspec on fewer shards" }, pole: { value: "rspec" },
+          affects: { value: "miss" }, expected_seconds: { value: 30 }, basis: { value: "unestimated" }, inputs: { value: "abc123" }
+    setup "CiSpeed::Lever.Attempt", lever: "rspec_4_shards", baseline_run: { value: "1200" },
+          baseline_seconds: { value: 300 }, baseline_cache: { value: "miss" }
+    setup "CiSpeed::Lever.Revert", lever: "rspec_4_shards", following_run: { value: "1201" },
+          following_seconds: { value: 295 }, following_cache: { value: "miss" }
+    input lever: "rspec_4_shards", new_inputs: { value: "abc123" }
+    expect refused: "a reverted lever is retried only when its inputs have changed"
+  end
+
+  test "Recall keeps a lever that history settled, with the commit that settled it" do
+    tests "Recall", on: "Lever"
+    setup "CiSpeed::Lever.Propose", name: { value: "host_test_build" },
+          change: { value: "host test build warmed into its own cache" }, pole: { value: "host_test_build" },
+          affects: { value: "both" }, expected_seconds: { value: 0 }, basis: { value: "unestimated" }, inputs: { value: "" }
+    input lever: "host_test_build", evidence: { value: "504f0e0d8" }
+    expect emits: ["LeverLearned"]
+  end
+
+  test "Complete refuses while the latest cache-miss run is over budget" do
+    tests "Complete", on: "Budget"
+    setup "CiSpeed::Budget.Open", name: { value: "main" }, target_seconds: { value: 180 }
+    setup "CiSpeed::Budget.WitnessWarm", budget: "main", seconds: { value: 150 }
+    setup "CiSpeed::Budget.WitnessMiss", budget: "main", seconds: { value: 200 }
+    input budget: "main"
+    expect refused: "both cache states are under budget"
+  end
+
+  # The effort completes once the latest warm run and the latest cache-miss run are both under
+  # budget. The warm verdict is witnessed by its own policy in setup.
+  test "A cache-miss run under budget completes the effort when the warm run is under budget" do
+    tests "AnalyzeMissUnderBudget", on: "Run"
+    setup "CiSpeed::Budget.Open", name: { value: "main" }, target_seconds: { value: 180 }
+    setup "CiSpeed::Run.Observe", run_id: { value: "1201" }, commit: { value: "93f8e68c6" },
+          wall_seconds: { value: 150 }, cache_state: { value: "hit" }, conclusion: { value: "success" },
+          budget_seconds: { value: 180 }, longest_pole: { value: "rspec" }
+    setup "CiSpeed::Run.AnalyzeWarmUnderBudget", run: "1201", seconds: { value: 150 }
+    setup "CiSpeed::Run.Observe", run_id: { value: "1202" }, commit: { value: "93f8e68c6" },
+          wall_seconds: { value: 170 }, cache_state: { value: "miss" }, conclusion: { value: "success" },
+          budget_seconds: { value: 180 }, longest_pole: { value: "rspec" }
+    input run: "1202", seconds: { value: 170 }
+    expect emits: %w[MissRunUnderBudget MissRunWitnessed BudgetMet]
+  end
+
+  # The backlog is what drives the attempts: the biggest expected saving is tried first.
+  test "Next is the proposed lever with the biggest expected saving" do
+    tests "Next", on: "Lever", kind: :query
+    setup "CiSpeed::Lever.Propose", name: { value: "small_win" }, change: { value: "a small change" },
+          pole: { value: "rspec" }, affects: { value: "both" }, expected_seconds: { value: 20 },
+          basis: { value: "run 40438675259 job times" }, inputs: { value: "" }
+    setup "CiSpeed::Lever.Propose", name: { value: "big_win" }, change: { value: "a big change" },
+          pole: { value: "rspec" }, affects: { value: "both" }, expected_seconds: { value: 90 },
+          basis: { value: "run 40438675259 job times" }, inputs: { value: "" }
+    setup "CiSpeed::Lever.Propose", name: { value: "middling_win" }, change: { value: "a middling change" },
+          pole: { value: "rspec" }, affects: { value: "both" }, expected_seconds: { value: 50 },
+          basis: { value: "run 40438675259 job times" }, inputs: { value: "" }
+    expect count: 1, name: { value: "big_win" }
+  end
+
+  test "Reestimate revises the saving a proposed lever is expected to make" do
+    tests "Reestimate", on: "Lever"
+    setup "CiSpeed::Lever.Propose", name: { value: "fuzz_sqlite" }, change: { value: "split the fuzz sqlite job" },
+          pole: { value: "checks_fuzz_sqlite" }, affects: { value: "hit" }, expected_seconds: { value: 0 },
+          basis: { value: "unestimated" }, inputs: { value: "" }
+    input lever: "fuzz_sqlite", new_expected_seconds: { value: 60 },
+          new_basis: { value: "run 40438675259: the job ran 191s, the next slowest 183s" }
+    expect emits: ["LeverReestimated"]
+  end
+
+  test "Reestimate refuses an estimate that says nothing about what it rests on" do
+    tests "Reestimate", on: "Lever"
+    setup "CiSpeed::Lever.Propose", name: { value: "fuzz_sqlite" }, change: { value: "split the fuzz sqlite job" },
+          pole: { value: "checks_fuzz_sqlite" }, affects: { value: "hit" }, expected_seconds: { value: 0 },
+          basis: { value: "unestimated" }, inputs: { value: "" }
+    input lever: "fuzz_sqlite", new_expected_seconds: { value: 60 }, new_basis: { value: "unestimated" }
+    expect refused: "a revised estimate says what it rests on"
+  end
+
+  # Meeting 180 seconds is a milestone: the target drops and the backlog goes on.
+  test "Reopen sets a lower target once the last one is met" do
+    tests "Reopen", on: "Budget"
+    setup "CiSpeed::Budget.Open", name: { value: "main" }, target_seconds: { value: 180 }
+    setup "CiSpeed::Budget.WitnessWarm", budget: "main", seconds: { value: 150 }
+    setup "CiSpeed::Budget.WitnessMiss", budget: "main", seconds: { value: 170 }
+    input budget: "main", new_target_seconds: { value: 150 }
+    expect emits: ["BudgetReopened"]
+  end
+
+  test "Reopen refuses a target that is not lower than the one met" do
+    tests "Reopen", on: "Budget"
+    setup "CiSpeed::Budget.Open", name: { value: "main" }, target_seconds: { value: 180 }
+    setup "CiSpeed::Budget.WitnessWarm", budget: "main", seconds: { value: 150 }
+    setup "CiSpeed::Budget.WitnessMiss", budget: "main", seconds: { value: 170 }
+    input budget: "main", new_target_seconds: { value: 200 }
+    expect refused: "a new target is lower than the one met"
+  end
+
+  # The experiments below are answered by `spec/fixtures/ci_speed/replayed_runner.adapter`, which
+  # replays recorded samples under the experiment's name (`wide_win`, `within_noise`, `wide_loss`)
+  # and refuses any other name. The real `LocalRunner` makes two checkouts and times real work, so
+  # `spec/ci_speed_local_runner_spec.rb` holds it instead.
+  test "Request refuses a candidate that is the same ref as its base" do
+    tests "Request", on: "Experiment"
+    input name: { value: "same_ref" }, lever: { value: "" }, measures: { value: "regenerate_corpus_cold" },
+          cache_state: { value: "miss" }, base_ref: { value: "HEAD" }, candidate_ref: { value: "HEAD" },
+          repeats: { value: 3 }
+    expect refused: "a candidate is a different ref from its base"
+  end
+
+  test "Request refuses a single sample, which has no spread to judge it by" do
+    tests "Request", on: "Experiment"
+    input name: { value: "one_sample" }, lever: { value: "" }, measures: { value: "regenerate_corpus_cold" },
+          cache_state: { value: "miss" }, base_ref: { value: "HEAD~1" }, candidate_ref: { value: "HEAD" },
+          repeats: { value: 1 }
+    expect refused: "an experiment is repeated at least twice"
+  end
+
+  test "A measured faster result completes" do
+    tests "Request", on: "Experiment"
+    input name: { value: "wide_win" }, lever: { value: "" }, measures: { value: "regenerate_corpus_cold" },
+          cache_state: { value: "miss" }, base_ref: { value: "HEAD~1" }, candidate_ref: { value: "HEAD" },
+          repeats: { value: 3 }
+    expect emits: %w[ExperimentRequested ExperimentAnswered ExperimentMeasured]
+  end
+
+  test "A result inside the spread is inconclusive and re-estimates nothing" do
+    tests "Request", on: "Experiment"
+    setup "CiSpeed::Lever.Propose", name: { value: "parallel_rustfmt" },
+          change: { value: "rustfmt on one thread per core" }, pole: { value: "generated_rust" },
+          affects: { value: "miss" }, expected_seconds: { value: 0 }, basis: { value: "unestimated" }, inputs: { value: "" }
+    input name: { value: "within_noise" }, lever: { value: "parallel_rustfmt" },
+          measures: { value: "regenerate_corpus_cold" }, cache_state: { value: "miss" },
+          base_ref: { value: "HEAD~1" }, candidate_ref: { value: "HEAD" }, repeats: { value: 3 }
+    expect emits: %w[ExperimentRequested ExperimentAnswered ExperimentMeasured]
+  end
+
+  test "A faster conclusive result re-estimates the lever it names" do
+    tests "Request", on: "Experiment"
+    setup "CiSpeed::Lever.Propose", name: { value: "parallel_rustfmt" },
+          change: { value: "rustfmt on one thread per core" }, pole: { value: "generated_rust" },
+          affects: { value: "miss" }, expected_seconds: { value: 0 }, basis: { value: "unestimated" }, inputs: { value: "" }
+    input name: { value: "wide_win" }, lever: { value: "parallel_rustfmt" },
+          measures: { value: "regenerate_corpus_cold" }, cache_state: { value: "miss" },
+          base_ref: { value: "HEAD~1" }, candidate_ref: { value: "HEAD" }, repeats: { value: 3 }
+    expect emits: %w[ExperimentRequested ExperimentAnswered ExperimentMeasured LeverReestimated]
+  end
+
+  test "The lever carries what the experiment measured, as its estimate" do
+    tests "Next", on: "Lever", kind: :query
+    setup "CiSpeed::Lever.Propose", name: { value: "parallel_rustfmt" },
+          change: { value: "rustfmt on one thread per core" }, pole: { value: "generated_rust" },
+          affects: { value: "miss" }, expected_seconds: { value: 0 }, basis: { value: "unestimated" }, inputs: { value: "" }
+    setup "CiSpeed::Experiment.Request", name: { value: "wide_win" }, lever: { value: "parallel_rustfmt" },
+          measures: { value: "regenerate_corpus_cold" }, cache_state: { value: "miss" },
+          base_ref: { value: "HEAD~1" }, candidate_ref: { value: "HEAD" }, repeats: { value: 3 }
+    expect count: 1, name: { value: "parallel_rustfmt" }, expected_seconds: { value: 31 }
+  end
+
+  test "A slower result says nothing to the backlog" do
+    tests "Request", on: "Experiment"
+    setup "CiSpeed::Lever.Propose", name: { value: "parallel_rustfmt" },
+          change: { value: "rustfmt on one thread per core" }, pole: { value: "generated_rust" },
+          affects: { value: "miss" }, expected_seconds: { value: 0 }, basis: { value: "unestimated" }, inputs: { value: "" }
+    input name: { value: "wide_loss" }, lever: { value: "parallel_rustfmt" },
+          measures: { value: "regenerate_corpus_cold" }, cache_state: { value: "miss" },
+          base_ref: { value: "HEAD~1" }, candidate_ref: { value: "HEAD" }, repeats: { value: 3 }
+    expect emits: %w[ExperimentRequested ExperimentAnswered ExperimentMeasured]
+  end
+
+  test "Request is faulted when the runner refuses it" do
+    tests "Request", on: "Experiment"
+    input name: { value: "unrecorded" }, lever: { value: "" }, measures: { value: "regenerate_corpus_cold" },
+          cache_state: { value: "miss" }, base_ref: { value: "HEAD~1" }, candidate_ref: { value: "HEAD" },
+          repeats: { value: 3 }
+    expect emits: %w[ExperimentRequested ExperimentRefused ExperimentFaulted]
+  end
+
+  test "A fault records why" do
+    tests "Outcome", on: "Experiment", kind: :query
+    setup "CiSpeed::Experiment.Request", name: { value: "unrecorded" }, lever: { value: "" },
+          measures: { value: "regenerate_corpus_cold" }, cache_state: { value: "miss" },
+          base_ref: { value: "HEAD~1" }, candidate_ref: { value: "HEAD" }, repeats: { value: 3 }
+    input name: { value: "unrecorded" }
+    expect count: 1, status: "faulted",
+           refusal: { value: "Hecks::Adapters::LocalRunner::Refused: no samples are recorded for unrecorded; " \
+                             "recorded: wide_win, within_noise, wide_loss" }
+  end
+
+  # A verdict has to say what the numbers say. The record here is faulted, not requested, because
+  # the givens are checked before the lifecycle, and they are what is held to account.
+  test "Complete refuses a faster verdict whose saving is not larger than the spread" do
+    tests "Complete", on: "Experiment"
+    setup "CiSpeed::Experiment.Request", name: { value: "unrecorded" }, lever: { value: "" },
+          measures: { value: "regenerate_corpus_cold" }, cache_state: { value: "miss" },
+          base_ref: { value: "HEAD~1" }, candidate_ref: { value: "HEAD" }, repeats: { value: 3 }
+    input experiment: "unrecorded", base_seconds: { value: 120.0 }, candidate_seconds: { value: 118.0 },
+          spread_seconds: { value: 6.0 }, saved_seconds: { value: 2.0 }, load: { value: 2.5 },
+          unmeasured: { value: "" }, verdict: { value: "faster" }, basis: { value: "a guess" },
+          reestimates: { value: "" }, expected_seconds: { value: 2 }
+    expect refused: "a faster verdict needs a saving larger than the spread"
+  end
+
+  test "Complete refuses an estimate on a result that is not faster" do
+    tests "Complete", on: "Experiment"
+    setup "CiSpeed::Experiment.Request", name: { value: "unrecorded" }, lever: { value: "" },
+          measures: { value: "regenerate_corpus_cold" }, cache_state: { value: "miss" },
+          base_ref: { value: "HEAD~1" }, candidate_ref: { value: "HEAD" }, repeats: { value: 3 }
+    input experiment: "unrecorded", base_seconds: { value: 120.0 }, candidate_seconds: { value: 118.0 },
+          spread_seconds: { value: 6.0 }, saved_seconds: { value: 2.0 }, load: { value: 2.5 },
+          unmeasured: { value: "" }, verdict: { value: "inconclusive" }, basis: { value: "a guess" },
+          reestimates: { value: "" }, expected_seconds: { value: 2 }
+    expect refused: "only a faster verdict carries an estimate"
+  end
+
+  test "Record keeps a finding, with its cause and the remedy still to try" do
+    tests "Record", on: "Finding"
+    input name: { value: "cache_download_stall" }, status: { value: "untried" },
+          symptom: { value: "one gem cache download stopped at 91% for about 80 seconds" },
+          cause: { value: "the cache service, seen once" },
+          remedy: { value: "set SEGMENT_DOWNLOAD_TIMEOUT_MINS to about a minute and check what a timeout does" }
+    expect emits: %w[FindingRecorded]
+  end
+
+  test "Record refuses a finding with no name" do
+    tests "Record", on: "Finding"
+    input name: { value: "" }, status: { value: "open" },
+          symptom: { value: "a symptom" }, cause: { value: "a cause" }, remedy: { value: "a remedy" }
+    expect refused: "must match"
+  end
+end