hecks 6.0.0 → 7.0.0

data/lib/hecks/ci_pipeline/ci_pipeline.hecksagon

@@ -0,0 +1,12 @@
+# The CiPipeline bluebook, bound so it boots standing alone (a behaviors or model_check boot of
+# this directory). The ports it shares with every attaching hecksagon are in
+# ci_pipeline.ports.hecksagon.
+Hecks.hecksagon "CiPipeline" do
+  attaches "Governance"
+  persisted_by "Memory"
+end
+
+# The anti-corruption layer for Governance, which the bluebook's roles are checked against.
+Hecks.hecksagon "Governance" do
+  persisted_by "Memory"
+end

data/lib/hecks/ci_pipeline/ci_pipeline.ports.hecksagon

@@ -0,0 +1,16 @@
+# The ports CiPipeline declares, for every hecksagon that attaches the bluebook. The bluebook
+# names the questions it asks of its runner and its cache, and never says how they are
+# answered; an adapter binds each port. Declared only: no adapter lives in this bluebook yet.
+Hecks.hecksagon "CiPipeline" do
+  # The runner answers what happened in a build, and the critical path through it. GitHub
+  # Actions and a local run are each an adapter behind this port.
+  CiPipeline::CiRun.port "Runner" do
+    answers_query "Reading"
+    answers_query "CriticalPath"
+  end
+
+  # The cache answers whether it holds the corpus for a key.
+  CiPipeline::Corpus.port "Cache" do
+    answers_query "Lookup"
+  end
+end

data/lib/hecks/ci_speed/adapters/ci_reader.adapter

@@ -0,0 +1,7 @@
+# Binds the `CiReader` port to `CiReader`, which answers `Run`'s `Reading` query with one main CI run as GitHub reports it.
+# Required by name, not relative, so a copy of this directory shares the one implementation loaded.
+require "hecks/ci_speed/adapters/ci_reader"
+
+Hecks.adapter "CiReader" do
+  port "CiReader"
+end

data/lib/hecks/ci_speed/adapters/ci_reader.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require "json"
+require "open3"
+require "time"
+require "hecks/runtime/errors"
+
+module Hecks
+  module Adapters
+    # The `CiReader` port's adapter: answers `Run`'s `Reading` query with one CI run as GitHub
+    # reports it, read through `gh`. A run's cache state is read off the jobs: GitHub lists a
+    # composite action as one step, so the generated-rust action's regeneration cannot be seen on
+    # its own. A cache hit restores in seconds and a miss regenerates the corpus, so a run where any
+    # job spent the threshold on that one step is a miss.
+    class CiReader
+      # The step a job that materializes the generated Rust shows for its composite action.
+      GENERATED_RUST_STEP = "Run ./.github/actions/generated-rust"
+
+      # Seconds on that step beyond which the cache missed.
+      MISS_THRESHOLD_SECONDS = 60
+
+      # @param aggregate [Object, nil] unused
+      # @param settings [Hash] unused
+      # @param root [String, nil] the checkout `gh` runs in; this repository when nil
+      def initialize(aggregate: nil, settings: {}, root: nil)
+        @root = root || File.expand_path("../../../..", __dir__)
+      end
+
+      # Reads remote CI's verdict on one commit of a repository's main. Every run GitHub reports
+      # for the commit on main must have settled, and every job of each must have passed or been
+      # skipped.
+      #
+      # @param commit [Hash, String] the full commit hash
+      # @param repo [Hash, String] `owner/name`
+      # @return [Hash] `text:` a JSON document: `color` (`green` or `red`) and `failing_jobs`, the
+      #   failing job names joined by commas, empty when green
+      # @raise [Runtime::GivenNotMet] when no run exists for the commit or a run has not settled
+      def head(commit:, repo:)
+        sha = named(commit)
+        slug = named(repo)
+        runs = head_runs(sha, slug)
+        settled!(runs, sha)
+        failing = runs.flat_map { |run| failing_jobs(run, slug) }
+        { text: JSON.generate(commit: sha, color: failing.empty? ? "green" : "red", failing_jobs: failing.join(",")) }
+      end
+
+      # @param run_id [Hash, String, Integer] the GitHub run to read
+      # @return [Hash] `text:` a JSON document: the run's commit, wall seconds, conclusion, cache
+      #   state, longest pole and each job's seconds
+      # @raise [Runtime::GivenNotMet] when `gh` cannot read the run
+      def reading(run_id:)
+        id = run_id.is_a?(Hash) ? run_id.values.first : run_id
+        { text: JSON.generate(document(JSON.parse(view(id.to_s)))) }
+      end
+
+      private
+
+      # Refuses a commit with no run on main, and one with a run still going.
+      def settled!(runs, sha)
+        raise Runtime::GivenNotMet, "no CI run for #{sha[0, 7]} on main" if runs.empty?
+
+        still = runs.reject { |run| run["status"] == "completed" }
+        raise Runtime::GivenNotMet, "CI for #{sha[0, 7]} is still running" if still.any?
+      end
+
+      def named(value) = value.is_a?(Hash) ? value.values.first.to_s : value.to_s
+
+      def head_runs(sha, slug)
+        out, err, status = Open3.capture3("gh", "run", "list", "--repo", slug, "--commit", sha,
+                                          "--branch", "main", "--limit", "100",
+                                          "--json", "databaseId,status,conclusion,workflowName", chdir: @root)
+        raise Runtime::GivenNotMet, "gh could not list runs for #{sha[0, 7]}: #{first_line(err)}" unless status.success?
+
+        JSON.parse(out)
+      end
+
+      PASSING_JOB_CONCLUSIONS = %w[success skipped].freeze
+      private_constant :PASSING_JOB_CONCLUSIONS
+
+      # Everything up to the first line break.
+      FIRST_LINE = /[^\n]*/
+      private_constant :FIRST_LINE
+
+      # The failing jobs of one run; the workflow's name stands in when no job is named failed.
+      def failing_jobs(run, slug)
+        return [] if run["conclusion"] == "success"
+
+        failed = jobs_of(run, slug).reject { |job| PASSING_JOB_CONCLUSIONS.include?(job["conclusion"]) }
+        failed.empty? ? [run.fetch("workflowName")] : failed.map { |job| job.fetch("name") }
+      end
+
+      def jobs_of(run, slug)
+        id = run.fetch("databaseId").to_s
+        out, err, status = Open3.capture3("gh", "run", "view", id, "--repo", slug, "--json", "jobs", chdir: @root)
+        raise Runtime::GivenNotMet, "gh could not read run #{id}: #{first_line(err)}" unless status.success?
+
+        JSON.parse(out).fetch("jobs")
+      end
+
+      # The first line of a command's error output, or the empty string.
+      def first_line(text) = text.to_s[FIRST_LINE].to_s
+
+      def view(id)
+        out, err, status = Open3.capture3("gh", "run", "view", id, "--json",
+                                          "headSha,conclusion,createdAt,updatedAt,jobs", chdir: @root)
+        raise Runtime::GivenNotMet, "gh could not read run #{id}: #{err.lines.first&.strip}" unless status.success?
+
+        out
+      end
+
+      def document(run)
+        jobs = timed(run.fetch("jobs"))
+        pole = jobs.max_by { |job| job[:seconds] }
+        { commit: run.fetch("headSha"), conclusion: run.fetch("conclusion"),
+          wall_seconds: seconds(run.fetch("createdAt"), run.fetch("updatedAt")),
+          cache_state: cache_state(run.fetch("jobs")), longest_pole: pole&.fetch(:name), jobs: jobs }
+      end
+
+      def timed(jobs)
+        jobs.select { |job| job["startedAt"] && job["completedAt"] }
+            .map { |job| { name: job.fetch("name"), seconds: seconds(job["startedAt"], job["completedAt"]) } }
+      end
+
+      def cache_state(jobs)
+        regenerated = jobs.any? do |job|
+          job.fetch("steps", []).any? { |step| regenerating?(step) }
+        end
+        regenerated ? "miss" : "hit"
+      end
+
+      def regenerating?(step)
+        step["name"] == GENERATED_RUST_STEP && step["startedAt"] && step["completedAt"] &&
+          seconds(step["startedAt"], step["completedAt"]) > MISS_THRESHOLD_SECONDS
+      end
+
+      def seconds(from, to)
+        (Time.parse(to) - Time.parse(from)).round
+      end
+    end
+  end
+end

data/lib/hecks/ci_speed/adapters/local_runner/answer.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Hecks
+  module Adapters
+    class LocalRunner
+      # What an adapter of the `LocalRunner` port answers with: each field in the shape
+      # `Experiment::Complete` takes it, so the bluebook's policy passes it on as it stands.
+      module Answer
+        module_function
+
+        # @param asked [Hash{Symbol => Object}] the experiment's own fields, plain: `lever`,
+        #   `base_ref` and `candidate_ref`
+        # @param definition [String] the definition's name
+        # @param verdict [Verdict] what the samples say
+        # @param load [Float] the machine's one-minute load average when the experiment started
+        # @param unmeasured [Array<String>] a note for each step left out of both sides
+        # @return [Hash{Symbol => Hash}] the answer
+        def build(asked:, definition:, verdict:, load:, unmeasured:)
+          basis = verdict.basis(definition: definition, load: load,
+                                refs: { base: asked[:base_ref], candidate: asked[:candidate_ref] })
+          fields = timings(verdict).merge(load: load.round(2), unmeasured: unmeasured.join("; "),
+                                          verdict: verdict.verdict, basis: basis,
+                                          reestimates: asked[:lever].to_s,
+                                          expected_seconds: verdict.expected_seconds)
+          fields.transform_values { |value| wrap(value) }
+        end
+
+        # @param verdict [Verdict] what the samples say
+        # @return [Hash{Symbol => Float}] each side's time, the spread and the saving
+        def timings(verdict)
+          { base_seconds: verdict.base_seconds, candidate_seconds: verdict.candidate_seconds,
+            spread_seconds: verdict.spread_seconds, saved_seconds: verdict.saved_seconds }
+        end
+
+        # @param value [Object] a field's value
+        # @return [Hash{Symbol => Object}] the value in the single-field shape commands take
+        def wrap(value) = { value: value }
+      end
+    end
+  end
+end

data/lib/hecks/ci_speed/adapters/local_runner/catalog.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "hecks/ci_speed/adapters/local_runner/refused"
+
+module Hecks
+  module Adapters
+    class LocalRunner
+      # The experiments the bluebook knows how to run: its `ExperimentDefinition` rows, each with
+      # the `ExperimentStep` rows that name it, in the order they are written.
+      class Catalog
+        # One step: a hecks verb and the `name=value` words it is called with.
+        Step = Struct.new(:name, :verb, :arguments, :needs, keyword_init: true) do
+          # @return [Array<String>] the launcher words for the verb: a command ends in `!`, and a
+          #   query is asked with `query`
+          def form = verb.end_with?("!") ? [verb] : ["query", verb]
+
+          # @return [Array<String>] the verb's own arguments
+          def words = arguments.to_s.split
+
+          # @return [Array<String>] the names of the `name=value` arguments, as a usage line
+          #   lists them; a flag such as `--help` has no name to look up
+          def argument_names = words.reject { |word| word.start_with?("-") }.map { |word| word.split("=").first }
+        end
+
+        # One definition: what it times, and what each sample starts from.
+        Definition = Struct.new(:name, :measures, :cache_state, :needs_generated_tree, :steps, keyword_init: true) do
+          # @return [Boolean] whether each sample starts from a copy of the generated tree
+          def generated_tree? = needs_generated_tree == "yes"
+        end
+
+        # Reads the rows a bluebook declares.
+        #
+        # @param bluebook [Bluebook::Declaration] the CiSpeed bluebook
+        # @return [Catalog] its definitions
+        def self.from_bluebook(bluebook)
+          new(definitions_of(rows_of(bluebook, "ExperimentDefinition"), rows_of(bluebook, "ExperimentStep")))
+        end
+
+        # @param bluebook [Bluebook::Declaration] a bluebook to search
+        # @param object [String] the name of the value object whose rows are wanted
+        # @return [Array<Hash{Symbol => String}>] its member rows
+        def self.rows_of(bluebook, object)
+          value_objects_of(bluebook).select { |candidate| candidate.hecks_name == object }.flat_map(&:members)
+        end
+
+        # @param bluebook [Bluebook::Declaration] a bluebook to search
+        # @return [Array<Bluebook::ValueObject>] every value object of every aggregate
+        def self.value_objects_of(bluebook) = bluebook.aggregates.flat_map(&:value_objects)
+        private_class_method :value_objects_of
+
+        # @param definitions [Array<Definition>] the experiments, by their own names
+        def initialize(definitions)
+          @definitions = definitions.to_h { |definition| [definition.name, definition] }
+        end
+
+        # @param name [String] a definition's name
+        # @return [Definition] the definition
+        # @raise [Refused] when no definition has that name, listing those that do
+        def fetch(name)
+          @definitions.fetch(name) do
+            raise Refused, "no experiment definition is named #{name.inspect}; defined: #{names.join(", ")}"
+          end
+        end
+
+        # @return [Array<String>] every definition's name
+        def names = @definitions.keys
+
+        # @param definitions [Array<Hash>] `ExperimentDefinition` rows
+        # @param steps [Array<Hash>] `ExperimentStep` rows
+        # @return [Array<Definition>] each definition with the steps that name it
+        def self.definitions_of(definitions, steps)
+          definitions.map do |row|
+            own = steps.select { |step| step[:definition] == row[:name] }
+            Definition.new(name: row[:name], measures: row[:measures], cache_state: row[:cache_state],
+                           needs_generated_tree: row[:needs_generated_tree], steps: own.map { |step| step_of(step) })
+          end
+        end
+        private_class_method :definitions_of
+
+        # @param row [Hash] an `ExperimentStep` row
+        # @return [Step] the step, needing nothing unless the row says so
+        def self.step_of(row)
+          Step.new(name: row[:name], verb: row[:verb], arguments: row[:arguments], needs: row[:needs] || "none")
+        end
+        private_class_method :step_of
+      end
+    end
+  end
+end

data/lib/hecks/ci_speed/adapters/local_runner/checkouts.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "open3"
+require "tmpdir"
+require "hecks/ci_speed/adapters/local_runner/refused"
+
+module Hecks
+  module Adapters
+    class LocalRunner
+      # Two temporary, detached git worktrees of a repository, one for each side of an experiment.
+      # They live in a directory made for the run and are removed when the run ends, whatever
+      # ended it. The repository's own checkout is only asked about its refs and its generated
+      # tree, and is never written.
+      class Checkouts
+        # Where the generated Rust lives in a checkout, and what a cold checkout lacks.
+        GENERATED_TREE = "rust/src/generated"
+
+        # Makes both checkouts, yields their paths, and removes them afterwards.
+        #
+        # @param root [String] the repository whose worktrees these are
+        # @param shas [Hash{Symbol => String}] the commit each side is checked out at, by side
+        # @param scratch [String, nil] a directory to make the run's directory in; the system's
+        #   temporary directory when nil
+        # @yield [paths] the checkouts
+        # @yieldparam paths [Hash{Symbol => String}] each side's checkout path, by side
+        # @return [Object] what the block answers
+        def self.with(root:, shas:, scratch: nil)
+          FileUtils.mkdir_p(scratch) if scratch
+          directory = Dir.mktmpdir("ci-experiment-", scratch)
+          checkouts = new(root, directory)
+          yield shas.to_h { |side, sha| [side, checkouts.add(side, sha)] }
+        ensure
+          checkouts&.remove_all
+          FileUtils.remove_entry(directory) if directory && File.exist?(directory)
+        end
+
+        # @param root [String] the repository
+        # @param directory [String] where the checkouts are made
+        def initialize(root, directory)
+          @root = root
+          @directory = directory
+          @added = []
+        end
+
+        # @param side [Symbol] which side the checkout is for
+        # @param sha [String] the commit to check out, detached
+        # @return [String] the checkout's path
+        # @raise [Refused] when git cannot make the worktree
+        def add(side, sha)
+          path = File.join(@directory, side.to_s)
+          @added << path
+          git("worktree", "add", "--detach", path, sha)
+          path
+        end
+
+        # Removes every checkout made, and forgets any whose creation stopped half way.
+        #
+        # @return [void]
+        def remove_all
+          @added.each { |path| Open3.capture2e("git", "-C", @root, "worktree", "remove", "--force", path) }
+          Open3.capture2e("git", "-C", @root, "worktree", "prune")
+        end
+
+        # Puts a checkout's generated tree back to what a sample starts from: nothing, or a copy of
+        # the tree the asking checkout holds.
+        #
+        # @param checkout [String] a checkout's path
+        # @param copy_from [String, nil] the repository whose generated tree is copied in; nil
+        #   leaves the checkout with none
+        # @return [void]
+        def self.reset_generated_tree(checkout, copy_from)
+          target = File.join(checkout, GENERATED_TREE)
+          FileUtils.rm_rf(target)
+          FileUtils.cp_r(File.join(copy_from, GENERATED_TREE), target) if copy_from
+        end
+
+        private
+
+        def git(*words)
+          out, status = Open3.capture2e("git", "-C", @root, *words)
+          raise Refused, "git #{words.join(" ")} failed in #{@root}: #{out.strip}" unless status.success?
+
+          out
+        end
+      end
+    end
+  end
+end

data/lib/hecks/ci_speed/adapters/local_runner/launcher.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "open3"
+require "rbconfig"
+require "hecks/ci_speed/adapters/local_runner/refused"
+
+module Hecks
+  module Adapters
+    class LocalRunner
+      # Runs hecks verbs through `exe/hecks` of one checkout, as that checkout's own code would
+      # run them. A step is only ever a verb call: nothing here takes a shell line.
+      class Launcher
+        # What every verb runs with: memory persistence, so a measured run writes no journal and
+        # reaches no database, whatever this process was started with.
+        ENVIRONMENT = { "HECKS_ENVIRONMENT" => "memory", "HECKS_DATABASE" => nil }.freeze
+
+        # @param checkout [String] a checkout's path
+        def initialize(checkout)
+          @checkout = checkout
+        end
+
+        # @param step [Catalog::Step] the step to ask about
+        # @return [Boolean] whether this checkout's launcher knows the step's verb, and every
+        #   argument the step gives it
+        def knows?(step)
+          out, status = run(*step.form, "--help")
+          status.success? && step.argument_names.all? { |name| out.include?("#{name}=") }
+        end
+
+        # Runs a step and times it.
+        #
+        # @param step [Catalog::Step] the step to run
+        # @return [Float] the seconds it took, on a monotonic clock
+        # @raise [Refused] with everything the verb printed, when it ends badly
+        def time(step)
+          words = [*step.form, *step.words, *("--wait" if step.verb.end_with?("!"))]
+          started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          out, status = run(*words)
+          seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
+          return seconds if status.success?
+
+          raise Refused, "step #{step.name} (#{words.join(" ")}) ended with status #{status.exitstatus} " \
+                         "in #{@checkout}:\n#{out}"
+        end
+
+        private
+
+        # Runs the launcher as the leader of its own process group, so that when this process is
+        # interrupted the verb and every process it forked go with it, and none is left running in
+        # a checkout that is about to be removed.
+        def run(*words)
+          launch = [RbConfig.ruby, File.join(@checkout, "exe", "hecks"), *words]
+          Open3.popen2e(ENVIRONMENT, *launch, chdir: @checkout, pgroup: true) do |input, output, waiter|
+            input.close
+            [output.read, waiter.value]
+          ensure
+            stop(waiter.pid) if waiter.alive?
+          end
+        end
+
+        def stop(group)
+          Process.kill("TERM", -group)
+        rescue Errno::ESRCH, Errno::EPERM
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/hecks/ci_speed/adapters/local_runner/load_average.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "open3"
+require "hecks/ci_speed/adapters/local_runner/refused"
+
+module Hecks
+  module Adapters
+    class LocalRunner
+      # The machine's one-minute load average, which says how busy the machine was while an
+      # experiment ran. Ruby's standard library does not report it, so it is read where the
+      # system keeps it: `/proc/loadavg` where there is one, and what `uptime` prints elsewhere.
+      module LoadAverage
+        # The figure `uptime` prints after `load average:` or `load averages:`.
+        UPTIME_FIGURE = /load averages?:\s*([0-9]+(?:[.,][0-9]+)?)/
+
+        module_function
+
+        # @return [Float] the one-minute load average
+        # @raise [Refused] when the system reports none, since a result cannot be read without it
+        def now
+          return proc_figure if File.readable?("/proc/loadavg")
+
+          out, status = Open3.capture2e("uptime")
+          figure = out[UPTIME_FIGURE, 1] if status.success?
+          raise Refused, "cannot read the load average: uptime printed #{out.strip.inspect}" unless figure
+
+          figure.tr(",", ".").to_f
+        end
+
+        # @return [Float] the figure `/proc/loadavg` leads with
+        def proc_figure
+          words = File.read("/proc/loadavg").split
+          words.first.to_f
+        end
+      end
+    end
+  end
+end

data/lib/hecks/ci_speed/adapters/local_runner/refs.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "open3"
+require "hecks/ci_speed/adapters/local_runner/refused"
+
+module Hecks
+  module Adapters
+    class LocalRunner
+      # Turns the two refs an experiment names into the commits they stand for, in the repository
+      # that asked.
+      module Refs
+        module_function
+
+        # @param root [String] the repository whose refs these are
+        # @param asked [Hash{Symbol => Object}] the experiment's `base_ref` and `candidate_ref`
+        # @return [Hash{Symbol => String}] each side's commit
+        # @raise [Refused] when a ref names no commit, or both name the same one
+        def resolve(root, asked)
+          shas = { base: sha_of(root, asked.fetch(:base_ref)), candidate: sha_of(root, asked.fetch(:candidate_ref)) }
+          return shas unless shas.values.uniq.one?
+
+          raise Refused, "base ref #{asked[:base_ref]} and candidate ref #{asked[:candidate_ref]} " \
+                         "are the same commit, #{shas[:base]}"
+        end
+
+        # @param root [String] the repository
+        # @param ref [String] a ref git can resolve
+        # @return [String] the commit it names
+        # @raise [Refused] when it names none
+        def sha_of(root, ref)
+          out, status = Open3.capture2e("git", "-C", root, "rev-parse", "--verify", "--quiet", "#{ref}^{commit}")
+          raise Refused, "#{ref.inspect} names no commit in #{root}" unless status.success?
+
+          out.strip
+        end
+      end
+    end
+  end
+end

data/lib/hecks/ci_speed/adapters/local_runner/refused.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Hecks
+  module Adapters
+    class LocalRunner
+      # Why an experiment could not be run. The runtime records the message as the experiment's
+      # refusal, so it names what was wrong and what would have been right.
+      class Refused < StandardError; end
+    end
+  end
+end

data/lib/hecks/ci_speed/adapters/local_runner/verdict.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Hecks
+  module Adapters
+    class LocalRunner
+      # What the samples of the two sides say. A side's time is the median of its samples, which one
+      # slow sample from a busy machine does not move. The spread is the larger of the two sides'
+      # own range, so the saving has to stand out from how much either side varied by itself.
+      class Verdict
+        # @return [Float] the base side's median seconds
+        attr_reader :base_seconds
+
+        # @return [Float] the candidate side's median seconds
+        attr_reader :candidate_seconds
+
+        # @return [Float] the larger of the two sides' ranges, in seconds
+        attr_reader :spread_seconds
+
+        # @param base [Array<Float>] the base side's samples, in seconds
+        # @param candidate [Array<Float>] the candidate side's samples, in seconds
+        def initialize(base:, candidate:)
+          @base = base
+          @candidate = candidate
+          @base_seconds = median(base).round(3)
+          @candidate_seconds = median(candidate).round(3)
+          @spread_seconds = [range(base), range(candidate)].max.round(3)
+        end
+
+        # @return [Float] the base time less the candidate time: negative for a slower candidate
+        def saved_seconds = (base_seconds - candidate_seconds).round(3)
+
+        # @return [String] `faster` or `slower` when the saving is larger than the spread, else
+        #   `inconclusive`
+        def verdict
+          return "faster" if saved_seconds > spread_seconds
+          return "slower" if (saved_seconds + spread_seconds).negative?
+
+          "inconclusive"
+        end
+
+        # @return [Integer] what a lever is told it is worth: the saving in whole seconds when
+        #   conclusively faster, else zero
+        def expected_seconds = verdict == "faster" ? saved_seconds.round : 0
+
+        # @param definition [String] the definition's name
+        # @param refs [Hash{Symbol => String}] each side's ref as the experiment named it
+        # @param load [Float] the machine's one-minute load average at the start
+        # @return [String] one sentence saying what the numbers rest on
+        def basis(definition:, refs:, load:)
+          "#{definition} on this machine at load #{load.round(2)}, samples interleaved: " \
+            "base #{refs.fetch(:base)} took #{show(@base)}, candidate #{refs.fetch(:candidate)} took #{show(@candidate)}; " \
+            "the saving of #{saved_seconds}s against a spread of #{spread_seconds}s is #{verdict}"
+        end
+
+        private
+
+        def show(samples)
+          "#{median(samples).round(3)}s median (#{samples.map { |sample| sample.round(3) }.join(", ")})"
+        end
+
+        def median(samples)
+          sorted = samples.sort
+          middle = sorted.size / 2
+          sorted.size.odd? ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2.0
+        end
+
+        def range(samples) = samples.max - samples.min
+      end
+    end
+  end
+end

data/lib/hecks/ci_speed/adapters/local_runner.adapter

@@ -0,0 +1,7 @@
+# Binds the `LocalRunner` port to `LocalRunner`, which times an experiment's steps in two temporary checkouts of the refs it names.
+# Required by name, not relative, so a copy of this directory shares the one implementation loaded.
+require "hecks/ci_speed/adapters/local_runner"
+
+Hecks.adapter "LocalRunner" do
+  port "LocalRunner"
+end