holdfast 0.1.0

data/lib/holdfast/stages/fold.rb

@@ -0,0 +1,256 @@
+module Holdfast
+  module Stages
+    # Turns the test outcomes, the lines each test file ran, the pins and the
+    # gaps into one row per changed method, plus one row per changed value
+    # for the readers that sit outside every changed method. A row is
+    # compared only when a test or a pin ran on both sides and reached the
+    # target's changed lines in this run; a test that only names the method
+    # is listed and does not count.
+    module Fold
+      ORIGIN_ORDER = ["drafted pin", "inputs pin", "evidence", "authored", "kept"].freeze
+
+      def self.run(change, targets, diff, pins, gaps)
+        lines = changed_lines(change)
+        methods = targets.select(&:method?)
+        readers = targets.select { _1.kind == "reader" }
+        inside, outside = readers.partition { |r| methods.any? { within?(r, _1) } }
+        owner = owners(diff, methods, lines)
+        rows = methods.map do |t|
+          row = method_row(t, diff, lines, owner, pins.find { _1.target == t }, gaps.find { _1.target == t }, inside.select { within?(_1, t) }, change)
+          # An unchanged method's row says which changed value it reads; a
+          # caller's row ranks first.
+          t.unchanged? && t.why[1] ? row.with(stage: t.caller? ? "caller" : row.stage, reason: "#{t.why[1]}; #{row.reason}") : row
+        end
+        rows += outside.group_by { _1.reader ? _1.reader["value"] : _1.name }.map { |value, rs| value_row(value, rs, diff, owner, change) }
+        # A failure a value row already lists (its file ran a reader's line)
+        # is not repeated as a row of its own.
+        stray = diff.outcomes.each_with_index.select { |o, i| o.kind != "clean" && owner[i].nil? && outside.none? { reads_line?(diff, o, _1) } }
+        # A new spec with no base side states nothing to compare; a test
+        # that reached no changed line and did not break on head is not
+        # something the change can reach. Both stay in verdicts.jsonl only.
+        rows += stray.map { |o, _| stray_row(o) }
+        fold_into_failures(rows)
+      end
+
+      # An added method whose only comparison is a failing caller is part of
+      # that caller's row, not a row of its own.
+      def self.fold_into_failures(rows)
+        folded, kept = rows.partition { _1.stage.start_with?("under:") }
+        kept.map do |row|
+          mine = folded.select { _1.stage == "under:#{row.target}" }
+          next row if mine.empty?
+
+          row.with(evidence: row.evidence.first(1) + ["it runs through added #{mine.map(&:target).join(", ")}, which have no base side of their own"] + row.evidence.drop(1))
+        end
+      end
+
+      # file => { "base" => [lines], "head" => [lines] } from the diff hunks.
+      def self.changed_lines(change)
+        diff = Change.git(change.root, "diff", "--unified=0", change.base, change.head, "--", "app", "lib")
+        out = Hash.new { |h, k| h[k] = { "base" => [], "head" => [] } }
+        file = nil
+        diff.each_line do |l|
+          file = l[6..].strip if l.start_with?("+++ b/")
+          next unless (m = l.match(/^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/)) && file
+
+          out[file]["base"] += (m[1].to_i...(m[1].to_i + (m[2] || 1).to_i)).to_a
+          out[file]["head"] += (m[3].to_i...(m[3].to_i + (m[4] || 1).to_i)).to_a
+        end
+        out
+      end
+
+      # The changed lines of a method's body. Ruby runs a `def` line when
+      # the file loads, so a test that only loaded the file would count as
+      # reaching it; a change to the signature alone counts the whole body.
+      def self.method_lines(t, lines)
+        span = t.span
+        # An unchanged method that reads a changed value has no changed line
+        # of its own; running any line of its body is reaching it.
+        if t.unchanged?
+          whole = ->(from, to) { from && to ? (to > from + 1 ? ((from + 1)..(to - 1)) : (from..to)).to_a : [] }
+          return { "head" => whole.(span["new_start"], span["new_end"]), "base" => whole.(span["old_start"], span["old_end"]) }
+        end
+        all = lines[t.file]
+        body = ->(from, to, changed) do
+          return [] unless from && to
+
+          inner = (to > from + 1 ? ((from + 1)..(to - 1)) : (from..to)).to_a
+          (changed & inner).then { _1.empty? && (changed & (from..to).to_a).any? ? inner : _1 }
+        end
+        { "head" => body.(span["new_start"], span["new_end"], all["head"]), "base" => body.(span["old_start"], span["old_end"], all["base"]) }
+      end
+
+      def self.within?(reader, t) = reader.file == t.file && t.span["new_start"] && (t.span["new_start"]..t.span["new_end"]).cover?(reader.line)
+
+      def self.hits(diff, outcome, side, file) = Diff.ran(diff.reach, outcome, side == "base" ? "1" : "2").fetch(file, [])
+
+      def self.reached?(diff, outcome, t, lines)
+        want = method_lines(t, lines)
+        %w[head base].any? { |side| (hits(diff, outcome, side, t.file) & want[side]).any? }
+      end
+
+      def self.reads_line?(diff, outcome, reader) = hits(diff, outcome, "head", reader.file).include?(reader.line)
+
+      # Each outcome that is not clean belongs to one row, so a failure is
+      # never listed twice: its own target for a pin or an evidence test,
+      # else the first changed method whose lines its file ran.
+      def self.owners(diff, methods, lines)
+        diff.outcomes.each_with_index.to_h do |o, i|
+          own = methods.find { o.source.origin !~ /kept|authored/ && o.source.targets.include?(_1.name) }
+          hit = methods.select { reached?(diff, o, _1, lines) }
+          stem = File.basename(o.source.path).sub(/_(spec|test)\.rb\z/, "")
+          own ||= hit.find { |m| m.name.split(/[#.]/).first.split("::").map { Holdfast.snake(_1) }.include?(stem) } || hit.first
+          [i, own&.name]
+        end
+      end
+
+      def self.draw_step(t, undrawn, gap)
+        value = undrawn.find(&:value)
+        return "send a request that takes #{undrawn.first.label}: write it as a suspicion for holdfast check, or as a request spec; then holdfast run" if t.file.start_with?("app/controllers/") && !value
+        how = value ? "an input whose record holds #{value.label}" : "an input that takes #{undrawn.first.label}"
+        place = gap&.draft ? "fill #{rel(gap.draft.path)} and copy it to .holdfast/evidence/" : "add it to .holdfast/evidence/#{Holdfast.slug(t.name)}.inputs.json"
+        "draw #{how}: #{place}, or write it as a suspicion for holdfast check; then holdfast run"
+      end
+
+      def self.method_row(t, diff, lines, owner, pin, gap, readers, change)
+        want = method_lines(t, lines)
+        span = (want["head"].empty? ? want["base"] : want["head"]).then { _1.empty? ? "" : " #{t.file.split("/").last}:#{_1.min}#{"-#{_1.max}" if _1.max > _1.min}" }.then { _1.empty? ? " its changed lines" : _1 }
+        mine = diff.outcomes.each_with_index.select { |o, i| owner[i] == t.name || (o.kind == "clean" && (reached?(diff, o, t, lines) || o.source.targets.include?(t.name))) }.map(&:first)
+        reached, named = mine.partition { reached?(diff, _1, t, lines) }
+        details = groups(reached).map { "#{_1} reach#{"es" if _1.start_with?("1 ")} the changed lines" } +
+                  groups(named).map { "#{_1} name#{"s" if _1.start_with?("1 ")} it but did not reach the changed lines" }
+        details += pin_details(pin) if pin
+        new_specs = (reached + named).select { _1.kind == "no_base" }
+        if new_specs.any?
+          files = new_specs.map { Diff.shown_path(_1.source) }.uniq
+          details << "new spec, no base side: #{new_specs.size} example(s) in #{files.first(2).join(", ")}#{" +#{files.size - 2}" if files.size > 2} pass on head; base cannot load what they call, so they are not a comparison"
+        end
+        details += readers.map { |r| "reader #{r.file}:#{r.line} (#{r.why.last}): #{diff.outcomes.any? { reads_line?(diff, _1, r) } ? "reached by a test" : "no test reached it"}" }
+        bad = reached.select { _1.kind == "unintended_change" } + named.select { _1.kind == "unintended_change" }
+        meant = reached.select { _1.kind == "intended_change" }
+        agree = reached.select { _1.kind == "clean" }
+        base = { target: t.name, stage: "diff", scope: (reached + named).map(&:place).uniq.join(", ") }
+        if bad.any?
+          first = bad.min_by { ORIGIN_ORDER.index(_1.source.origin) || 9 }
+          Verdict.of(**base, kind: "unintended_change", reason: "#{count(bad, first)} passes on base and fails on head",
+                     evidence: ["#{first.test["name"].to_s.sub(/\A.*? input /, "input ")}: #{difference(first, 4)}"] + details, repro: first.repro, next_step: first.next_step)
+        elsif t.added? && (elsewhere = reached_failures(diff, t, lines, owner)).any?
+          Verdict.of(**base, stage: "under:#{elsewhere.first}", kind: "unchecked", reason: "added; its callers are its only comparison, and #{elsewhere.first} fails",
+                     evidence: details, next_step: "settle #{elsewhere.first} first, then holdfast run")
+        elsif meant.any?
+          first = meant.first
+          Verdict.of(**base, kind: "intended_change", reason: "#{count(meant, first)} differs as marked: #{difference(first, 1)}",
+                     evidence: details, repro: first.repro, next_step: "confirm the head value is the one the intent wants")
+        elsif agree.any? && (undrawn = Drawn.missing(Drawn.needs(change, t, lines), reached, diff.reach)).any?
+          # Its tests ran its lines, but none drew a value the change added
+          # or removed, so agreeing says nothing about that value.
+          Verdict.of(**base, kind: "unchecked", reason: "#{agree.size} test(s) ran it on both sides and agree, but no input drew #{undrawn.first(3).map(&:label).join(", ")}#{" (+#{undrawn.size - 3} more)" if undrawn.size > 3}",
+                     evidence: undrawn.map { "not drawn: #{_1.label} (#{_1.why})" } + Drawn.one_sided(undrawn, reached, diff.reach) + details,
+                     next_step: draw_step(t, undrawn, gap))
+        elsif agree.empty? && t.added? && (fresh = reached.select { _1.kind == "no_base" }).any?
+          # Base has no such method, so a spec that calls it has no base side:
+          # what it states is the new behavior, to read against the intent.
+          places = fresh.map(&:place).uniq
+          Verdict.of(**base, kind: "intended_change", reason: "added; only new spec(s) with no base side run it: #{places.first(2).join(", ")}#{" +#{places.size - 2}" if places.size > 2}",
+                     evidence: details, next_step: "confirm those specs state what the intent wants #{t.name} to do")
+        elsif agree.any? && !t.added? && (fresh = Drawn.missing(Drawn.needs(change, t, lines), agree, diff.reach)).any?
+          # Only a spec with no base side took what the change added, so the
+          # tests that agree never ran it: the head behavior is new, not unchanged.
+          by = new_specs.select { |o| Drawn.missing(fresh, [o], diff.reach).size < fresh.size }.map(&:place).uniq
+          Verdict.of(**base, kind: "intended_change", reason: "#{fresh.first(3).map(&:label).join(", ")} runs only in new spec(s) with no base side#{": #{by.first(2).join(", ")}" if by.any?}; the #{agree.size} test(s) that agree never took it",
+                     evidence: fresh.map { "only head-only specs took #{_1.label} (#{_1.why})" } + details,
+                     next_step: "confirm the new spec states what the intent wants for #{fresh.first.label}")
+        elsif agree.any?
+          Verdict.of(**base, kind: "clean", reason: t.added? ? "added; #{agree.size} test(s) of its callers ran on both sides, reached it on head, and agree" : "#{agree.size} test(s) ran on both sides, reached#{span}, and agree",
+                     evidence: details, next_step: "nothing; the claim covers only the tests in scope")
+        else
+          unchecked_method(t, base, details, span, pin, gap)
+        end
+      end
+
+      def self.reached_failures(diff, t, lines, owner)
+        diff.outcomes.each_with_index.filter_map { |o, i| owner[i] if o.kind == "unintended_change" && owner[i] != t.name && reached?(diff, o, t, lines) }.uniq
+      end
+
+      def self.unchecked_method(t, base, details, span, pin, gap, why: nil)
+        why ||= t.added? ? "added, and no test reached it on head" : "no test or pin ran on both sides and reached#{span}"
+        step = if gap&.draft
+                 "fill #{gap.draft.holes.join(", ")} in #{rel(gap.draft.path)}, copy it to .holdfast/evidence/, then holdfast run"
+               elsif pin
+                 "the #{pin.origin} inputs (#{rel(pin.inputs.file)}) did not reach#{span}; add inputs that take it, then holdfast run"
+               elsif t.added?
+                 "write a test that states what #{t.name} does in .holdfast/evidence/#{Holdfast.slug(t.name)}/, or name it from a caller's pin"
+               else
+                 "write a test in .holdfast/evidence/#{Holdfast.slug(t.name)}/ that reaches#{span}, then holdfast run"
+               end
+        details += ["draft: #{gap.reason}"] if gap && !gap.draft
+        Verdict.of(**base, kind: "unchecked", reason: why, evidence: details, next_step: step)
+      end
+
+      def self.pin_details(pin)
+        out = ["#{pin.observed.count { !_1["input_error"] }} #{pin.origin == "drafted" ? "drafted" : "given"} input(s) pinned at #{pin.rev[0, 7]} from #{rel(pin.inputs.file)}"]
+        out << "#{pin.input_errors.size} input(s) did not build: #{pin.input_errors.first}" if pin.input_errors.any?
+        frontier = Array(pin.report["frontier"]).reject { |f| Array(pin.report["covered"]).any? { _1["id"] == f["id"] } }.uniq { _1["id"] }
+        out << "arms no input took: #{frontier.first(3).map { "#{_1["loc"]} #{_1["sense"]} `#{_1["cond"]}`" }.join("; ")}" if frontier.any?
+        out
+      end
+
+      def self.value_row(value, readers, diff, owner, change)
+        ran = diff.outcomes.each_with_index.select { |o, _| readers.any? { reads_line?(diff, o, _1) } }
+        bad = ran.select { |o, i| o.kind == "unintended_change" && owner[i].nil? }.map(&:first)
+        unreached = readers.reject { |r| ran.any? { |o, _| reads_line?(diff, o, r) } }
+        details = readers.map { |r| "#{r.file}:#{r.line} #{r.why.first.sub(/ \(.*\)\z/, "")}: #{unreached.include?(r) ? "no test reached it" : "reached by #{ran.count { |o, _| reads_line?(diff, o, r) }} test(s)"}" }
+        base = { target: value, stage: "readers", evidence: details }
+        if bad.any?
+          Verdict.of(**base, evidence: ["#{bad.size} failing test(s), first #{bad.first.place}"] + details,
+                     kind: "unintended_change", reason: "a test that reads it fails on head: #{difference(bad.first)}", repro: bad.first.repro, next_step: bad.first.next_step)
+        elsif unreached.empty? && (undrawn = Drawn.missing(Drawn.lookup(Drawn.constants(change), value.to_s)&.then { Drawn.members_needs(value.to_s, _1) } || [], ran.map(&:first), diff.reach)).any?
+          Verdict.of(**base, kind: "unchecked", reason: "every reader was reached by a test, but no input drew #{undrawn.first(3).map(&:label).join(", ")}",
+                     evidence: undrawn.map { "not drawn: #{_1.label} (#{_1.why})" } + details,
+                     next_step: "write a test or a suspicion (holdfast check) whose record holds #{undrawn.first.label} and that runs #{readers.first.name}; then holdfast run")
+        elsif unreached.empty?
+          Verdict.of(**base, kind: "clean", reason: "every reader outside the changed methods was reached by a test that ran on both sides",
+                     next_step: "nothing; the claim covers only the tests that reached each line")
+        else
+          r = unreached.first
+          sentence = r.reader ? r.reader["test"] : "Write a test that runs #{r.name}."
+          Verdict.of(**base, kind: "unchecked", reason: "#{unreached.size} of #{readers.size} reader(s) no test reached",
+                     next_step: "#{sentence} Put it in .holdfast/evidence/#{Holdfast.slug(r.name)}/, then holdfast run")
+        end
+      end
+
+      def self.stray_row(o)
+        kind = o.kind == "no_base" ? "unchecked" : o.kind
+        Verdict.of(target: o.place, stage: "diff", kind:, reason: "#{o.source.origin} test, which reached no changed method: #{o.reason}",
+                   evidence: [o.evidence], repro: o.repro, shown: o.kind == "unintended_change",
+                   next_step: o.kind == "no_base" ? "nothing to compare; check what it exercises through an existing caller or a pin" : o.next_step)
+      end
+
+      def self.groups(outcomes)
+        outcomes.group_by { _1.source.origin }.sort_by { ORIGIN_ORDER.index(_1.first) || 9 }.map do |origin, os|
+          files = os.map { Diff.shown_path(_1.source) }.uniq
+          tally = os.map(&:kind).tally.map { |k, n| "#{n} #{k == "clean" ? "agree" : k == "no_base" ? "with no base side" : k.tr("_", " ")}" }.join(", ")
+          "#{files.size} #{origin} file#{"s" if files.size > 1} (#{tally}; #{files.first(2).join(", ")}#{" +#{files.size - 2}" if files.size > 2})"
+        end
+      end
+
+      def self.count(list, first)
+        same = list.count { _1.source.origin == first.source.origin }
+        same > 1 ? "#{same} #{first.source.origin} examples (first: #{first.place})" : "#{first.source.origin} #{first.place}"
+      end
+
+      # A pin failure lists the paths that differ; a test failure is its message.
+      def self.difference(o, limit = 3)
+        text = [o.test.dig("head", "message"), o.test.dig("base", "message")].compact.find { !_1.strip.empty? }.to_s
+        if text.include?("differs: ")
+          parts = text[/differs: (.*)/, 1].strip.split(" | ")
+          return parts.first(limit).join(" | ") + (parts.size > limit ? " (+#{parts.size - limit} more)" : "")
+        end
+
+        text.lines.map(&:strip).reject(&:empty?).first(2).join(" ")[0, 200].then { _1.empty? ? o.evidence : _1 }
+      end
+
+      def self.rel(path) = path.to_s.sub(%r{\A.*?/\.holdfast/}, ".holdfast/")
+    end
+  end
+end

data/lib/holdfast/stages/hunt.rb

@@ -0,0 +1,176 @@
+module Holdfast
+  module Stages
+    # The hunt stage: the sequences library drafts one definition per write
+    # area of the app (routes, Grape endpoints, jobs), then runs random
+    # orders of those actions against the real database under invariants,
+    # time zones and second doors. Its definitions live in
+    # .holdfast/sequences/, outside the app's tree, so a hunt never makes the
+    # next run see a change. Each area's verdict becomes one row:
+    #   FAILURE         -> bug_suspect, with the log that holds the shortest sequence
+    #   partly checked  -> unchecked, with what was not covered
+    #   unchecked       -> unchecked, with why it never ran
+    #   clean           -> clean, scoped to the seeds and cases that ran
+    # In a change, only the areas that touch a changed file run.
+    module Hunt
+      DEFAULT_BUDGET = 600
+
+      def self.run(change, record, budget:, files: nil)
+        budget ||= DEFAULT_BUDGET
+        dir = File.join(Holdfast.dir(change.root), "sequences")
+        work = File.join(Holdfast.dir(change.root), "runs", "sequences")
+        FileUtils.mkdir_p([dir, work])
+        env = { "RAILS_ENV" => "test", "SEQUENCES_OUT" => dir, "SEQUENCES_WORK" => work,
+                "SEQUENCES_LIB" => File.join(SEQUENCES, "sequences.rb") }
+        env["SEQUENCES_DATABASE_URL"] = record.database_url if record.database_url
+        bundle = overlay(change, env)
+        return [bundle] if bundle.is_a?(Verdict)
+
+        drafted = draft(change, record, dir, env)
+        return [drafted] if drafted.is_a?(Verdict)
+
+        areas = Dir.glob(File.join(dir, "*.rb")).reject { File.basename(_1) == "invariants.rb" }.sort
+        areas = touching(areas, files) if files
+        return [row("(sequences)", "unchecked", "no area touches the changed files", [], "nothing to hunt for this change")] if areas.empty?
+
+        per_area = [budget * 4 / [areas.size, 4].max, 60].max
+        cases = budget >= 900 ? 300 : 60
+        # Shrinking a failure takes minutes; a short hunt reports it as found.
+        env["SEQ_NO_SHRINK"] = "1" if budget < 900
+        json = File.join(work, "table.json")
+        FileUtils.rm_f(json)
+        out, = Open3.capture2e(env, RbConfig.ruby, File.join(SEQUENCES, "run.rb"), "-j", "4", "--cases", cases.to_s, "--seeds", "1",
+                               "--timeout", per_area.to_s, "--json", json, *areas, chdir: change.root)
+        File.write(File.join(work, "run.log"), out)
+        unless File.file?(json)
+          return [row("(sequences)", "unchecked", "the sequences run printed no table: #{out.lines.grep(/Error|error/).first.to_s.strip}", ["log: #{Fold.rel(File.join(work, "run.log"))}"], "read the log, fix the error it names, then holdfast run")]
+        end
+
+        table = JSON.parse(File.read(json))
+        rows = table.map { area_row(change, _1) }
+        [reach_row(change, table)].compact + rows
+      end
+
+      # The first line of a hunt says how much of it ran: an action that was
+      # refused every time, or never drawn, checked nothing, and an area that
+      # reached under half its actions is not listed as clean. An area that
+      # failed stopped at its failure, so its reach says nothing about setup.
+      def self.reach_row(change, table)
+        counts = table.filter_map { |r| r["verdict"] != "FAILURE" && r["reached"].to_s[%r{(\d+)/(\d+)}] && [r, $1.to_i, $2.to_i] }
+        total = counts.sum { _1[2] }
+        reached = counts.sum { _1[1] }
+        return nil if total.zero? || reached == total
+
+        thin = counts.select { |_, a, b| a * 2 < b }.sort_by { |_, a, b| a.fdiv(b) }
+        # Each thin area names its file and the first actions that checked
+        # nothing, with the refusal, so the fix starts at a line, not a count.
+        evidence = thin.map do |r, a, b|
+          missing = r["text"].to_s.lines.drop(1).map { _1.strip.delete_prefix("- ") }.reject(&:empty?)
+          "#{r["area"]} (#{r["file"].to_s.delete_prefix("#{change.root}/")}): #{a}/#{b} actions reached#{"; first: #{missing.first(2).join("; ")}" if missing.any?}"
+        end
+        worst = thin.first&.first
+        step = worst ? "start with #{worst["area"]}: make the records or the sign-in its unreached actions need in #{worst["file"].to_s.delete_prefix("#{change.root}/")}, then holdfast run" :
+                       "read each area's row below for the refusals, fix the setup or the sign-in in its file, then holdfast run"
+        row("(hunt)", "unchecked", "unreached: #{total - reached} of #{total} actions (refused every time or never drawn); #{thin.size} of #{counts.size} areas reached under half",
+            evidence, "#{step}; until then the hunt says little about these areas")
+      end
+
+      # The sequences library needs hegeltest, which the app's bundle may
+      # not list. A Gemfile of holdfast's own evaluates the app's and adds
+      # it, locked from the gems already installed, so the app's Gemfile and
+      # lock stay untouched.
+      def self.overlay(change, env)
+        dir = File.join(Holdfast.dir(change.root), "bundle")
+        FileUtils.mkdir_p(dir)
+        gemfile = File.join(dir, "Gemfile")
+        app_lock = File.join(change.root, "Gemfile.lock")
+        if File.read(app_lock).match?(/^    hegeltest /)
+          env["BUNDLE_GEMFILE"] = File.join(change.root, "Gemfile")
+          return nil
+        end
+        File.write(gemfile, %(eval_gemfile #{File.join(change.root, "Gemfile").inspect}\ngem "hegeltest", "#{Holdfast::HEGELTEST_VERSION}", require: false\n))
+        FileUtils.cp(app_lock, "#{gemfile}.lock") unless File.file?("#{gemfile}.lock")
+        out, status = Open3.capture2e({ "BUNDLE_GEMFILE" => gemfile }, "bundle", "lock", "--local", chdir: change.root)
+        return row("(sequences)", "unchecked", "could not add hegeltest to a bundle of holdfast's own: #{out.lines.last.to_s.strip}", [], "install hegeltest #{Holdfast::HEGELTEST_VERSION} for this Ruby, then holdfast run") unless status.success?
+
+        env["BUNDLE_GEMFILE"] = gemfile
+        nil
+      end
+
+      def self.draft(change, record, dir, env)
+        probe_env = env.merge(record.database_url ? { "DATABASE_URL" => "#{record.database_url}_seq0" } : {})
+        if record.side_setup
+          _, status = Open3.capture2e(probe_env, "/bin/sh", "-c", record.side_setup, chdir: change.root)
+          return row("(sequences)", "unchecked", "the probe database did not set up", [], "holdfast run --learn") unless status.success?
+        end
+        return nil if Dir.glob(File.join(dir, "*.rb")).size > 1
+
+        out, status = Open3.capture2e(probe_env, "bundle", "exec", "ruby", File.join(SEQUENCES, "propose.rb"), "--all", chdir: change.root)
+        return nil if status.success?
+
+        row("(sequences)", "unchecked", "drafting the areas failed: #{out.lines.grep(/Error|error/).first.to_s.strip}", [],
+            "fix the error it names in the app or its factories, then holdfast run")
+      end
+
+      # An area runs for a change when its definition names a changed file's
+      # table, constant or route resource.
+      def self.touching(areas, files)
+        words = files.flat_map do |f|
+          base = File.basename(f, ".rb").delete_suffix("_controller").delete_suffix("_api")
+          [base, base.end_with?("s") ? base : "#{base}s"]
+        end.uniq
+        areas.select { |a| text = File.read(a); words.any? { text.include?(_1) } }
+      end
+
+      def self.area_row(change, r)
+        text = r["text"].to_s
+        lines = text.lines.map(&:strip)
+        scope = "#{r["seeds"].to_s.split(",").size} seed(s) x #{r["cases"]} cases, #{r["reached"]} actions reached"
+        cut = text[/cut by the time limit after \S+ of \d+ cases/]
+        scope = "#{cut}, #{r["reached"]} actions reached" if cut
+        log = r["log"].to_s.delete_prefix("#{change.root}/")
+        area = "sequences: #{r["area"]}"
+        case r["verdict"]
+        when "FAILURE"
+          # The failure text is the first line under "Sequences::Failure:" in
+          # RSpec's report, kept whole: a cut message hid which row broke.
+          logged = File.file?(r["log"].to_s) ? File.readlines(r["log"], encoding: "UTF-8").map { _1.scrub.strip } : []
+          at = logged.index { _1.end_with?("Sequences::Failure:") }
+          found = at ? logged[at + 1] : logged.find { _1.match?(/ raised |invariant \w+ broken|differ|disagree/) && !_1.start_with?("#", "1)") }
+          reason = found ? found.sub(/\A.*?in '[^']+': /, "").sub(/ \(Sequences::Failure\)\z/, "") : lines.first.delete_prefix("FAILURE: ")
+          short = reason.size > 300 ? "#{reason[0, 300]}... (whole text in evidence)" : reason
+          # An error raised in holdfast's own code, not the app's, says nothing
+          # about the app: report it as holdfast's, with where it was raised.
+          frame = logged.find { _1.start_with?("# /") }
+          if at.nil? && frame&.include?(File.expand_path("..", __dir__))
+            return row(area, "unchecked", "holdfast failed while hunting here: #{short}", ["at: #{frame.delete_prefix("# ")}", "log: #{log}"],
+                       "report it to holdfast with the log; the hunt checked nothing in this area", scope:)
+          end
+          # Broken before any action ran, the world the setup drafted already
+          # breaks the rule: the setup or the drafted rule is wrong, not the app.
+          if (rule = reason[/invariant (\w+) broken after setup/, 1])
+            return row(area, "unchecked", "the drafted setup already breaks #{rule}, before any action; the hunt checked nothing here",
+                       ["failure: #{reason}", "log: #{log}"],
+                       "make the setup in #{r["file"].to_s.delete_prefix("#{change.root}/")} build records that keep #{rule}, or drop the rule from .holdfast/sequences/invariants.rb if the app does not promise it; then holdfast run", scope:)
+          end
+          row(area, "bug_suspect", short, ["failure: #{reason}", "log: #{log} (the shortest sequence and an `it` block that replays it)"],
+              "paste the `it` block from the log into #{r["file"].to_s.delete_prefix("#{change.root}/")}, see it fail, fix the code, run again", scope:)
+        when "partly checked"
+          missing = lines.drop(1).map { _1.delete_prefix("- ") }
+          row(area, "unchecked", "partly checked (#{scope}); #{missing.size} action(s) checked nothing", missing,
+              "make the record in setup, narrow a pool, or change the param draws in #{r["file"].to_s.delete_prefix("#{change.root}/")}, then holdfast run", scope:)
+        when "clean"
+          a, b = r["reached"].to_s.scan(/\d+/).map(&:to_i)
+          return row(area, "unchecked", "reached only #{a}/#{b} actions (#{scope})", [], "make the records the other actions need in #{r["file"].to_s.delete_prefix("#{change.root}/")}, then holdfast run", scope:) if a && b && a * 2 < b
+
+          row(area, "clean", "every action, offset replay and door pair reached its target (#{scope})", [], "nothing; the claim covers only these seeds and cases", scope:)
+        else
+          row(area, "unchecked", lines.first.to_s.delete_prefix("unchecked: "), ["log: #{log}"], "read the log, fix the setup it names, then holdfast run")
+        end
+      end
+
+      def self.row(target, kind, reason, evidence, next_step, scope: nil)
+        Verdict.of(target:, stage: "sequences", kind:, reason:, evidence:, next_step:, scope:)
+      end
+    end
+  end
+end

data/lib/holdfast/stages/observe.rb

@@ -0,0 +1,109 @@
+module Holdfast
+  module Stages
+    # A behavior diff that starts from inputs, not from changed methods: the
+    # hunt's area definitions (drafted on head) run every ordering of their
+    # actions up to three steps, with fixed draws, once in a checkout of base
+    # and once in a checkout of head, each on its own database. Every step
+    # records how it ended, the rows it wrote by attributes, and the fakes it
+    # touched. Where the two sides differ is behavior that changed, whether
+    # or not a changed method is anywhere near it; an unchanged caller of a
+    # changed value shows up because its own observation moves.
+    #
+    # Only the first differing step of a sequence counts (the steps after it
+    # start from another state). Each difference is matched against the
+    # words of --intent: one that names it is shown to confirm, any other is
+    # unintended.
+    module Observe
+      Diff = Data.define(:area, :action, :what, :seq, :step, :base, :head, :args)
+
+      def self.run(change, record, intent:, files:)
+        dir = File.join(Holdfast.dir(change.root), "sequences")
+        work = File.join(Holdfast.dir(change.root), "runs", "observe")
+        FileUtils.rm_rf(work) if work.end_with?("/.holdfast/runs/observe")
+        FileUtils.mkdir_p([dir, work])
+        env = { "RAILS_ENV" => "test", "SEQUENCES_OUT" => dir, "SEQUENCES_LIB" => File.join(SEQUENCES, "sequences.rb") }
+        env["SEQUENCES_DATABASE_URL"] = record.database_url if record.database_url
+        bundle = Hunt.overlay(change, env)
+        return [bundle] if bundle.is_a?(Verdict)
+
+        drafted = Hunt.draft(change, record, dir, env)
+        return [drafted] if drafted.is_a?(Verdict)
+
+        areas = Hunt.touching(Dir.glob(File.join(dir, "*.rb")).reject { File.basename(_1) == "invariants.rb" }.sort, files)
+        return [row("(observe)", "unchecked", "no drafted area touches the changed files", [], "nothing to observe for this change")] if areas.empty?
+
+        sides = { "base" => change.base, "head" => change.head }.to_h do |side, rev|
+          [side, Thread.new { observe_side(change, record, env, side, rev, areas, work) }]
+        end.transform_values(&:value)
+        failed = sides.select { |_, r| r[:error] }
+        return failed.map { |side, r| row("(observe)", "unchecked", "the #{side} side did not run: #{r[:error]}", [], "read #{Fold.rel(r[:log])}, fix the setup, then holdfast run --observe") } if failed.any?
+
+        diffs = compare(sides["base"][:lines], sides["head"][:lines])
+        steps = sides["head"][:lines].size
+        return [row("(observe)", "clean", "#{steps} step(s) in #{areas.size} area(s) observed the same on both sides", [],
+                    "nothing; the claim covers only these orderings and draws", scope: areas.map { File.basename(_1, ".rb") }.join(", "))] if diffs.empty?
+
+        words = intent.to_s.downcase.scan(/[a-z][a-z_]{3,}/).uniq - %w[that with from this have should when then into only also each same than they them]
+        diffs.group_by { [_1.area, _1.action, _1.what] }.map { |(area, action, what), ds| diff_row(area, action, what, ds, words) }
+      end
+
+      def self.observe_side(change, record, env, side, rev, areas, work)
+        tree = File.join(Dir.mktmpdir("holdfast-observe"), "tree")
+        Change.git(change.root, "worktree", "add", "--quiet", "--detach", tree, rev)
+        number = Record::SIDES.fetch(side)
+        side_env = env.merge(record.env_for(side)).merge("BUNDLE_GEMFILE" => env.fetch("BUNDLE_GEMFILE", File.join(tree, "Gemfile")))
+        log = File.join(work, "#{side}.log")
+        if record.side_setup
+          out, status = Open3.capture2e(side_env, "/bin/sh", "-c", record.side_setup, chdir: tree)
+          File.write(log, out)
+          return { error: "database setup failed", log: } unless status.success?
+        end
+        lines = []
+        areas.each do |area|
+          out_file = File.join(work, "#{side}-#{File.basename(area, ".rb")}.jsonl")
+          out, = Open3.capture2e(side_env.merge("SEQ_OBSERVE" => out_file, "TEST_ENV_NUMBER" => number), "bundle", "exec", "rspec", area, chdir: tree)
+          File.write(log, out, mode: "a")
+          lines += File.readlines(out_file).map { JSON.parse(_1) } if File.file?(out_file)
+        end
+        { lines:, log: }
+      ensure
+        if tree && File.directory?(tree)
+          system("git", "worktree", "remove", "--force", tree, chdir: change.root, err: File::NULL)
+          FileUtils.rm_rf(File.dirname(tree))
+        end
+      end
+
+      def self.compare(base, head)
+        by_key = base.to_h { [[_1["area"], _1["index"], _1["step"]], _1] }
+        head.group_by { [_1["area"], _1["index"]] }.filter_map do |_, steps|
+          steps.sort_by { _1["step"] }.each do |h|
+            b = by_key[[h["area"], h["index"], h["step"]]] or break nil
+            what = %w[outcome writes outside].find { b[_1] != h[_1] } or next
+            break Diff.new(area: h["area"], action: h["action"], what:, seq: h["seq"], step: h["step"], base: b[what], head: h[what], args: h["args"])
+          end.then { _1.is_a?(Diff) ? _1 : nil }
+        end
+      end
+
+      def self.diff_row(area, action, what, ds, words)
+        first = ds.min_by { [_1.seq.size, _1.step] }
+        named = words.select { |w| [action, JSON.generate(first.base), JSON.generate(first.head)].any? { _1.to_s.downcase.include?(w) } }
+        evidence = ["shortest: #{first.seq.first(first.step).join(" -> ")} (step #{first.step}: #{action} #{first.args})",
+                    "base #{what}: #{show(first.base)}", "head #{what}: #{show(first.head)}",
+                    "#{ds.size} ordering(s) differ first at #{action}"]
+        if named.any?
+          row("observe: #{area} #{action}", "intended_change", "#{what} differs at #{action}, and the intent names #{named.first(3).join(", ")}", evidence,
+              "confirm the head #{what} is what the intent wants")
+        else
+          row("observe: #{area} #{action}", "unintended_change", "#{what} differs at #{action} after #{first.step - 1} earlier step(s); the intent does not name it", evidence,
+              "fix the code, or say it in --intent; the orderings are in .holdfast/runs/observe/")
+        end
+      end
+
+      def self.show(v) = (v.is_a?(String) ? v : JSON.generate(v)).then { _1.size > 400 ? "#{_1[0, 400]}..." : _1 }
+
+      def self.row(target, kind, reason, evidence, next_step, scope: nil)
+        Verdict.of(target:, stage: "observe", kind:, reason:, evidence:, next_step:, scope:)
+      end
+    end
+  end
+end

data/lib/holdfast/stages/select.rb

@@ -0,0 +1,94 @@
+module Holdfast
+  # A thing the run checks: a changed method, a reader of a changed value
+  # that is not a plain call site, or a method the caller named before a
+  # change. +tests+ are the existing test files that exercise it, at base.
+  Target = Data.define(:name, :kind, :file, :line, :why, :tests, :source, :reader, :span, :reads) do
+    def initialize(reads: nil, **rest) = super(reads:, **rest)
+    def caller? = why.first == "caller"
+    def unchanged? = %w[caller affected].include?(why.first)
+    def added? = why.first == "added"
+    def method? = name.match?(/\A[A-Z][\w:]*[#.][a-z_]\w*[?!=]?\z/)
+  end
+
+  module Stages
+    # Which targets this run checks and which tests belong to each. The
+    # changed methods and their tests come from jitcatch's change brief
+    # (grep, or a jitcatch-coverage-v1 file when the record names one); the
+    # readers of each changed value come from the readers list.
+    module Select
+      Result = Data.define(:targets, :verdicts, :notes)
+
+      def self.run(change, record)
+        jc = jitcatch(change, record)
+        brief = jc.build
+        methods = brief["methods"]
+        methods = methods.select { |m| change.named.any? { named_match?(_1, m) } } unless change.changed?
+        methods = methods.reject { _1["file"].start_with?("db/") || accessor_only?(_1) }
+        removed, kept = methods.partition { _1["change"] == "removed" }
+        targets = kept.map do |m|
+          Target.new(name: m["name"], kind: "method", file: m["file"], line: m["start_line"],
+                     why: [m["change"], ("reads #{brief["constants"].find { |k| k["id"] == m["via_constant"] }&.dig("name")}, which the change edits" if m["via_constant"])].compact,
+                     tests: (m["existing_tests"] + m["entry_point_tests"]).map { _1["file"] }.uniq,
+                     source: [m["old_source"], m["new_source"]].compact.join("\n"), reader: nil,
+                     span: m.slice("old_start", "old_end", "new_start", "new_end"),
+                     reads: Callers.value_of(m, brief, change)&.then { |v| { "constant" => v.constant, "values" => v.values, "model" => v.model, "column" => v.column, "outside" => v.outside }.compact })
+        end
+        targets = Callers.targets(jc, brief, change) + targets if change.changed?
+        verdicts = []
+        unless change.changed?
+          missing = change.named.reject { |n| kept.any? { named_match?(n, _1) } }
+          verdicts += missing.map do |n|
+            Verdict.of(target: n, stage: "select", kind: "unchecked", reason: "no method or file named #{n} at #{change.short(change.head)}",
+                       next_step: "name it as Const#method, Const.method, or a path under app/, then run holdfast run --target again")
+          end
+        end
+        targets += readers(brief, test_files(change, record)) if change.changed?
+        notes = removed.map { "removed: #{_1["name"]}" } + (change.changed? ? brief["warnings"].map { "jitcatch warning: #{_1}" } : [])
+        notes << "tests chosen by #{brief["existing_tests_via"]}" unless change.changed?
+        notes << "tests chosen by #{brief["existing_tests_via"]}#{" (#{record.coverage})" if record.coverage}; a row counts a test only when it reached the changed lines in this run" if change.changed?
+        Result.new(targets:, verdicts:, notes:)
+      end
+
+      def self.jitcatch(change, record)
+        base = change.changed? ? change.base : empty_parent(change)
+        JitCatch::Changes.new(JitCatch::GitRepo.new(change.root), base, change.head, readers: change.changed?,
+                              coverage: record.coverage && File.join(change.root, record.coverage))
+      end
+
+      # Reading from the first commit's parent makes every method "added", so
+      # the same brief lists each method of head with its tests.
+      def self.empty_parent(change)
+        first = Change.git(change.root, "rev-list", "--max-parents=0", change.head).split.last
+        "#{first}^"
+      end
+
+      # A member a Data.define or attr_reader line adds holds no logic of its
+      # own; the methods that read it are the targets.
+      def self.accessor_only?(m) = m["change"] == "added" && !m["new_source"].to_s.lstrip.start_with?("def ")
+
+      def self.named_match?(named, method) = method["name"] == named || method["file"] == named
+
+      def self.test_files(change, record)
+        Change.git(change.root, "ls-tree", "-r", "--name-only", change.base, "--", "spec", "test")
+              .lines.map(&:strip).select { _1.end_with?(record.test_suffix) }
+      end
+
+      # The readers list names a line, not a test. The tests named after the
+      # reader's file (closures_controller.rb, views/closures/ -> closures_spec)
+      # are the nearest the harness can pick without coverage; a row says so.
+      def self.tests_named_after(file, tests)
+        stem = file.start_with?("app/views/") ? file.split("/")[2] : File.basename(file, ".*").sub(/_controller\z/, "")
+        tests.select { File.basename(_1).sub(/_(spec|test)\.rb\z/, "").sub(/_controller\z/, "") == stem }
+      end
+
+      # jitcatch's readers: code that reads a changed value without calling it
+      # by name (a scope's SQL, a validation, a default, a page).
+      def self.readers(brief, tests)
+        Array(brief.dig("readers", "rows")).reject { _1["file"].start_with?("spec/", "test/") }.map do |row|
+          Target.new(name: "#{row["file"]}:#{row["line"]}", kind: "reader", file: row["file"], line: row["line"],
+                     why: ["#{row["rank"]} (#{row["value"]})", row["why"]], tests: tests_named_after(row["file"], tests), source: nil, reader: row, span: {})
+        end
+      end
+    end
+  end
+end