omp-conductor 0.17.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/REFERENCE.md +12 -8
  2. package/package.json +1 -1
  3. package/schema/config.schema.json +40 -1
  4. package/src/admission.ts +263 -44
  5. package/src/ask.ts +39 -3
  6. package/src/availability.ts +27 -1
  7. package/src/backups.ts +2 -2
  8. package/src/briefs/orchestrator.md +1 -0
  9. package/src/briefs/worker.md +38 -19
  10. package/src/command-help.ts +8 -1
  11. package/src/command-manifest.ts +5 -2
  12. package/src/commands/arm.ts +6 -3
  13. package/src/commands/message.ts +32 -4
  14. package/src/commands/watch.ts +62 -3
  15. package/src/config-schema.ts +53 -0
  16. package/src/config.ts +97 -1
  17. package/src/daemon.ts +1479 -1483
  18. package/src/decisions.ts +51 -6
  19. package/src/depends-on.ts +261 -1
  20. package/src/diff-flags.ts +350 -0
  21. package/src/digest-schedule.ts +37 -0
  22. package/src/doctor.ts +310 -22
  23. package/src/escalate.ts +560 -57
  24. package/src/failure-class.ts +71 -15
  25. package/src/fleet.ts +189 -34
  26. package/src/gitops.ts +103 -24
  27. package/src/graph-health.ts +20 -7
  28. package/src/graph.ts +313 -68
  29. package/src/lifecycle.ts +43 -7
  30. package/src/omp.ts +42 -0
  31. package/src/orchestrator-tick.ts +430 -162
  32. package/src/release-policy.ts +177 -5
  33. package/src/routing.ts +11 -3
  34. package/src/session-host.ts +16 -0
  35. package/src/settlement.ts +1728 -0
  36. package/src/setup-host.ts +193 -4
  37. package/src/setup-install.ts +91 -30
  38. package/src/setup-wizard.ts +1257 -78
  39. package/src/setup.ts +153 -6
  40. package/src/status-render.ts +36 -4
  41. package/src/store.ts +411 -17
  42. package/src/tracker/github.ts +607 -12
  43. package/src/types.ts +331 -5
  44. package/src/upgrade.ts +50 -19
  45. package/src/verbs/actions.ts +66 -18
  46. package/src/verbs/protocol.ts +45 -0
  47. package/src/verbs/server.ts +270 -13
  48. package/src/worker.ts +239 -6
  49. package/src/worktree.ts +115 -8
  50. package/systemd/omp-conductor-recover.sh +73 -0
  51. package/systemd/recover-unit-test.sh +61 -0
package/src/types.ts CHANGED
@@ -95,7 +95,8 @@ export interface RepoTarget {
95
95
  /**
96
96
  * Absolute path of the **conductor-owned, index-only clone** of this repo
97
97
  * whose code-graph index workers query. Optional: absent means this repo has
98
- * no graph, and the worker brief says nothing about one.
98
+ * no graph, and the worker brief says so explicitly rather than staying
99
+ * silent — a worker that knows there is no graph stops looking for one.
99
100
  *
100
101
  * Two things it deliberately is not, and both were paid for:
101
102
  *
@@ -412,8 +413,11 @@ export type BehindBaseAction = (typeof BEHIND_BASE_ACTIONS)[number];
412
413
  * the entry price for being in this list rather than in POLICY.md.
413
414
  */
414
415
  export const RELEASE_REQUIREMENTS = [
415
- /** Every run this release covers reached `merged`, not merely `pushed-green`. */
416
+ /** Every active run of the released repo reached `merged`, not merely `pushed-green`. */
416
417
  "runs-settled",
418
+ /** Every active run in the project reached `merged` — the suite-wide form of
419
+ * `runs-settled`, for shapes that consume several repos at once. */
420
+ "fleet-runs-settled",
417
421
  /** No pull request is still open against the branch being released. */
418
422
  "no-open-prs",
419
423
  /** Nothing still carries the queue label: the batch is finished, not paused. */
@@ -581,6 +585,111 @@ export const LABEL_REASONS = [
581
585
 
582
586
  export type LabelReason = (typeof LABEL_REASONS)[number];
583
587
 
588
+ /**
589
+ * Why a green pull request is being returned to its worker for revision.
590
+ * Closed for the reason {@link MERGE_REASONS} is: a revision request is a
591
+ * decision the ledger has to be able to name, and the same request repeated
592
+ * must read the same both times.
593
+ */
594
+ export const REVIEW_REASONS = [
595
+ /** The orchestrator read the green PR and found issues that block merging. */
596
+ "blocking-findings",
597
+ ] as const;
598
+
599
+ export type ReviewReason = (typeof REVIEW_REASONS)[number];
600
+
601
+ /**
602
+ * How a queued review revision ended, written by the daemon side of the
603
+ * transport. `revised` is a revision worker that again reached a verified
604
+ * green head; `pending` is a still-unverified push; `failed` is a revision
605
+ * worker that settled terminally; `skipped` is a queued round the daemon could
606
+ * not dispatch (its run moved, its transcript vanished, or the store already
607
+ * claimed the slot).
608
+ */
609
+ export const REVIEW_REVISION_OUTCOMES = ["revised", "pending", "failed", "skipped"] as const;
610
+
611
+ export type ReviewRevisionOutcome = (typeof REVIEW_REVISION_OUTCOMES)[number];
612
+
613
+ /**
614
+ * How strictly this project's green PRs are reviewed before they merge
615
+ * (#678). One of three typed levels; each level's bar is documented once in
616
+ * {@link REVIEW_STRICTNESS_CHOICES} (setup.ts), which is the same single text
617
+ * the rendered orchestrator brief carries, so the level a config declares and
618
+ * the threshold a session actually enforces cannot disagree.
619
+ *
620
+ * `low` blocks only correctness, security, data-loss or explicit
621
+ * acceptance-criteria failures. `medium` (the recommended default for a new
622
+ * project) adds material maintainability or reliability defects likely to
623
+ * become incidents within six months. `high` adds concrete quality defects —
624
+ * never subjective style churn or unbounded refactoring.
625
+ */
626
+ export const REVIEW_STRICTNESS = ["low", "medium", "high"] as const;
627
+
628
+ export type ReviewStrictness = (typeof REVIEW_STRICTNESS)[number];
629
+
630
+ /** The recommended strictness for a project that has never answered. */
631
+ export const DEFAULT_REVIEW_STRICTNESS: ReviewStrictness = "medium";
632
+
633
+ /** The small validated range a project's own review-round ceiling may sit in. */
634
+ export const REVIEW_MAX_ROUNDS_MIN = 1;
635
+ export const REVIEW_MAX_ROUNDS_MAX = 6;
636
+
637
+ /** The typed default ceiling: three corrected heads per PR lifecycle — enough
638
+ * for real correction cycles, and a hard stop against endless polishing. */
639
+ export const DEFAULT_REVIEW_MAX_ROUNDS = 3;
640
+
641
+ /**
642
+ * One project's review policy (#678): the strictness its orchestrator applies
643
+ * when deciding whether a green PR is returned to its worker, and the hard
644
+ * ceiling on how many such rounds one PR lifecycle may consume. The ceiling is
645
+ * enforced by the dispatch side, never left to the session's reading of the
646
+ * brief: at the ceiling `conductor_pr_review` refuses, the orchestrator leaves
647
+ * the PR open, records the unresolved findings and escalates once.
648
+ */
649
+ export interface ReviewPolicy {
650
+ strictness: ReviewStrictness;
651
+ maxRounds: number;
652
+ }
653
+
654
+ /** The documented migration default: what a config written before the key
655
+ * existed loads as, deterministically, until the operator answers setup. */
656
+ export const DEFAULT_REVIEW_POLICY: ReviewPolicy = {
657
+ strictness: DEFAULT_REVIEW_STRICTNESS,
658
+ maxRounds: DEFAULT_REVIEW_MAX_ROUNDS,
659
+ };
660
+
661
+ /**
662
+ * One durable review-revision request (#677): the orchestrator returned a
663
+ * green, run-owned pull request to its worker with blocking findings, and
664
+ * everything the daemon needs to resume the *same* OMP session is recorded
665
+ * here before any worker is woken. The round numbers successive revisions of
666
+ * one run (1, 2, …); the findings text is delivered to the resumed session
667
+ * verbatim.
668
+ */
669
+ export interface ReviewRevisionRecord {
670
+ id: string;
671
+ project: string;
672
+ /** The run whose pushed-green row this revises. The row is reused, never
673
+ * cloned — no new attempt, and no failed-attempt or continuation charge. */
674
+ runId: string;
675
+ issue: number;
676
+ prUrl: string;
677
+ /** The exact reviewed head; re-verified against the live PR before persisting. */
678
+ headSha: string;
679
+ /** Structured blocking findings, delivered verbatim to the resumed session. */
680
+ findings: string;
681
+ round: number;
682
+ reason: ReviewReason;
683
+ /** The transcript the revision must resume, as the run recorded it — the
684
+ * durable "same OMP session" identity, recorded before any wake. */
685
+ sessionFile?: string;
686
+ requestedAt: number;
687
+ /** Set when the daemon handed the revision to a worker. */
688
+ dispatchedAt?: number;
689
+ settledAt?: number;
690
+ outcome?: ReviewRevisionOutcome;
691
+ }
692
+
584
693
  /**
585
694
  * A model-supplied justification: one value out of a closed set, plus prose that
586
695
  * is written down and read by nothing that decides.
@@ -612,6 +721,33 @@ export const ORCHESTRATOR_MODES = ["embedded", "external"] as const;
612
721
 
613
722
  export type OrchestratorMode = (typeof ORCHESTRATOR_MODES)[number];
614
723
 
724
+ /**
725
+ * What `arm` accepts as proof that a human just approved arming (conductor
726
+ * #613). Declared as data for the reason {@link BASE_FRESHNESS} is: the schema,
727
+ * the setup wizard and the arm gate must all read one vocabulary, and a new
728
+ * proof must be a compile-time change everywhere at once.
729
+ *
730
+ * `challenge` is the pre-#613 behaviour, byte for byte: a Telegram challenge
731
+ * goes out and the armed marker is written only after the reply appears in the
732
+ * orchestrator session. `claim-only` arms on the shared live-plumbing verdict
733
+ * alone — no Telegram send and no wait — which is what makes an unattended
734
+ * recovery re-arm possible, at the cost spelled out in the wizard question:
735
+ * anything that can invoke the already-privileged `omp-conductor arm` command
736
+ * can start dispatch once the live claim and poller pass.
737
+ */
738
+ export const ARM_PROOFS = ["challenge", "claim-only"] as const;
739
+
740
+ export type ArmProof = (typeof ARM_PROOFS)[number];
741
+
742
+ /**
743
+ * The proof a config that never answered the question gets: today's
744
+ * authenticated round-trip, so no existing install changes behaviour on
745
+ * upgrade. Read through `resolveArmProof` (config.ts), never off a raw
746
+ * `ProjectConfig` — the loader materialises `arm` complete, but a hand-built
747
+ * config may omit the key.
748
+ */
749
+ export const DEFAULT_ARM_PROOF: ArmProof = "challenge";
750
+
615
751
  /**
616
752
  * Everything the dispatcher needs to service one product: where work comes
617
753
  * from, where code goes, and what it may spend doing it. Config is per project
@@ -629,8 +765,11 @@ export interface ProjectConfig {
629
765
  * orchestrator-tick.ts. */
630
766
  groomBelow?: number;
631
767
  /** Labels the dispatcher writes back so the tracker alone shows live state
632
- * to a human who never opens the daemon's logs. */
633
- stateLabels: { inProgress: string; blocked: string; failed: string };
768
+ * to a human who never opens the daemon's logs. `backlog` is the
769
+ * operator's own park gesture (#507) not dispatcher-written, but read by
770
+ * eligibility so a parked issue can never be claimed while its queue label
771
+ * is still physically present. */
772
+ stateLabels: { inProgress: string; blocked: string; failed: string; backlog: string };
634
773
  /** Maps a `${labelPrefix}${name}` label on an issue to the checkout it
635
774
  * belongs in, so routing is declared by humans, not guessed. */
636
775
  routing: { labelPrefix: string; repos: Record<string, RepoTarget> };
@@ -682,6 +821,16 @@ export interface ProjectConfig {
682
821
  * the session that triages it. See {@link ORCHESTRATOR_MODES}.
683
822
  */
684
823
  escalation: { telegramChatId?: string; telegramTopicId?: number; fallbackToIssueComment: boolean; orchestrator: OrchestratorMode };
824
+ /**
825
+ * How `arm` proves a human just approved arming (conductor #613). Optional
826
+ * only for configs written before the key existed and for hand-built test
827
+ * configs: the loader always materialises a complete value, so read it
828
+ * through `resolveArmProof` rather than reaching for `.proof` on a project
829
+ * some test hand-built. Absent resolves to {@link DEFAULT_ARM_PROOF}
830
+ * (`challenge`), preserving today's authenticated round-trip — the one value
831
+ * that changes nothing on upgrade.
832
+ */
833
+ arm?: { proof: ArmProof };
685
834
  /**
686
835
  * Who lands green PRs, who cuts releases and who promotes. The daemon never
687
836
  * acts on this itself — it words the orchestrator's standing orders and the
@@ -714,6 +863,16 @@ export interface ProjectConfig {
714
863
  * reaching for `.merge` on a project some test hand-built.
715
864
  */
716
865
  policy?: ProjectPolicy;
866
+ /**
867
+ * How green PRs are reviewed and returned (#678): the strictness level and
868
+ * the hard ceiling on review rounds per PR lifecycle. Optional only for
869
+ * configs written before the key existed — the loader always materialises a
870
+ * complete value, so read it through `resolveReview` rather than reaching
871
+ * for `.review` on a project some test hand-built. A config that never
872
+ * answered migrates to {@link DEFAULT_REVIEW_POLICY} deterministically, and
873
+ * the schema's bounds agree with the runtime ones.
874
+ */
875
+ review?: ReviewPolicy;
717
876
  /**
718
877
  * Exact operator-authored exceptions for recovery PRs that no run recorded.
719
878
  * Optional and hand-edited: setup preserves entries but never invents them.
@@ -742,7 +901,11 @@ export interface ProjectConfig {
742
901
  * the production daemon. With a marker configured, the dispatcher refuses to
743
902
  * reattach any preserved branch missing it, holds the issue as `stale-base`
744
903
  * (distinct from capacity or dependency holds), and never silently merges
745
- * base into the user's work. Absent or empty, ordinary stale continuations
904
+ * base into the user's work. A branch the marker cannot be verified against
905
+ * at all — a failed mirror fetch, an unresolvable marker, or no probe wired
906
+ * — fails closed too, but as `critical-base-verify-error`: a verification
907
+ * failure, never a claim that the branch is stale, re-admitted automatically
908
+ * once a later probe succeeds. Absent or empty, ordinary stale continuations
746
909
  * keep today's behaviour. Optional and hand-edited, like
747
910
  * {@link recoveryMerges}.
748
911
  */
@@ -862,6 +1025,12 @@ export const SETTLEMENT_FLAG_KINDS = [
862
1025
  "assertions-removed",
863
1026
  /** A named timeout in a test file went up. */
864
1027
  "test-timeout-raised",
1028
+ /** A command the PR body claims as proof, with no matching command in the
1029
+ * run's transcript — the claim has no attempt behind it at all. */
1030
+ "claimed-proof-missing",
1031
+ /** A claimed proof command was attempted, but the shared-host guard refused
1032
+ * it — the honest blocked shape, told apart from a claim with no attempt. */
1033
+ "claimed-proof-blocked",
865
1034
  /** A terminal run recovered an OPEN PR by exact issue, repo and branch identity. */
866
1035
  "pr-adopted",
867
1036
  /** A workflow on the merge commit's base branch failed after settlement. */
@@ -962,6 +1131,14 @@ export interface Tracker {
962
1131
  * read exists to prevent (#517).
963
1132
  */
964
1133
  listComments(issue: number): Promise<IssueComment[]>;
1134
+ /**
1135
+ * One issue's body, or undefined when the tracker could not produce it.
1136
+ * The promotion echo (#724) renders the parsed file lane exactly where the
1137
+ * author asserts the claim; the queue label is being added to an issue the
1138
+ * `listReady` filter would have excluded by definition, so the verb needs
1139
+ * this read rather than a list the issue is not on.
1140
+ */
1141
+ getIssue(issue: number): Promise<ReadyIssue | undefined>;
965
1142
  addLabel(issue: number, label: string): Promise<void>;
966
1143
  removeLabel(issue: number, label: string): Promise<void>;
967
1144
  comment(issue: number, body: string): Promise<void>;
@@ -1008,6 +1185,12 @@ export interface Tracker {
1008
1185
  * when either fact cannot be read safely.
1009
1186
  */
1010
1187
  issueSnapshot(issue: number): Promise<IssueSnapshot | undefined>;
1188
+ /**
1189
+ * An issue's raw body, or undefined when it cannot be read. Used only by the
1190
+ * dependency-graph cycle pass (#421) to follow a reachable prerequisite's
1191
+ * own `Depends-on:` declarations; never a "no body" claim.
1192
+ */
1193
+ issueBody(issue: number): Promise<string | undefined>;
1011
1194
  /**
1012
1195
  * The state of one specific pull request, or undefined when this adapter
1013
1196
  * could not tell — a network failure, a deleted PR, a URL it cannot parse.
@@ -1028,6 +1211,20 @@ export interface Tracker {
1028
1211
  sha: string,
1029
1212
  opts?: { event?: string; branch?: string },
1030
1213
  ): Promise<WorkflowRun[] | undefined>;
1214
+ /** Full failed-job logs of one workflow run across its attempts, oldest
1215
+ * attempt first, one chunk per failed job — never one aggregate per attempt,
1216
+ * so a mixed failure (setup 429 + sibling compile error) stays visible to
1217
+ * the caller's all-infra guard. `undefined` when the evidence is undecided:
1218
+ * the run register, any attempt's job register, or any failed job's log
1219
+ * could not be read, or the run has more attempts than `maxAttempts` (a
1220
+ * prefix is never presented as the complete attempt history); a no-mutation
1221
+ * refusal the next pass asks again. `[]` when the run is determinately
1222
+ * green across its attempts. Logs are full and untruncated — a truncated
1223
+ * last-400-lines view can discard contrary failure evidence. Read through
1224
+ * the tracker's own guarded runner — call/refusal accounting and the
1225
+ * rate-limit breaker apply exactly as they do to a live tick's reads
1226
+ * (#638). */
1227
+ runFailedAttemptLogs(repo: string, runUrl: string, maxAttempts: number): Promise<string[] | undefined>;
1031
1228
  /** The live commit at one repository branch, or undefined when unreadable. */
1032
1229
  branchHead(repo: string, branch: string): Promise<string | undefined>;
1033
1230
  /** The newest run of this workflow on the base branch before `before`.
@@ -1060,6 +1257,13 @@ export interface Tracker {
1060
1257
  * better answer on a later tick.
1061
1258
  */
1062
1259
  prDiff(url: string): Promise<PrDiff | undefined>;
1260
+ /**
1261
+ * The pull request's body, or undefined when this adapter could not produce
1262
+ * one — the same advisory contract as {@link Tracker.prDiff}. Its only
1263
+ * consumer is the settlement audit's claimed-proof check (#582): a claim
1264
+ * extracted from a body this port could not read is no claim at all.
1265
+ */
1266
+ prBody(url: string): Promise<string | undefined>;
1063
1267
  /**
1064
1268
  * Every check on a PR with the state the tracker reports for it (#132).
1065
1269
  *
@@ -1282,6 +1486,12 @@ export interface RunRecord {
1282
1486
  autoCompactionCount?: number;
1283
1487
  /** omp session transcript, so a human can read what the worker actually did. */
1284
1488
  sessionFile?: string;
1489
+ /** The run id of the orphan-clean attempt whose session this row resumes
1490
+ * (#536, #567). Set only when the dispatch-time resume verdict fired;
1491
+ * absent means a fresh dispatch, never "resumed from unknown". Readable
1492
+ * without opening the transcript, so "how often did resume fire and did it
1493
+ * save turns" is a query, not a file walk. */
1494
+ resumedFromRunId?: string;
1285
1495
  prUrl?: string;
1286
1496
  /** Pull request head the worker observed after its deterministic CI watcher exited. */
1287
1497
  headSha?: string;
@@ -1312,6 +1522,14 @@ export interface RunRecord {
1312
1522
  /** When an operator accepted the loss or recovered the tree by hand
1313
1523
  * (`unblock --force`). Clears the hold without erasing what happened. */
1314
1524
  salvageAckAt?: number;
1525
+ /** Why the retained tree is under quarantine (#737): the cleanup pass could
1526
+ * not make its object store sound (`repairAlternates`), so fetching into it
1527
+ * was refused and the tree's commits cannot be verified against any remote.
1528
+ * Present means the tree is quarantined right now; the pass clears it the
1529
+ * moment it reaps the tree or re-verifies the store on a later pass.
1530
+ * Absent means never quarantined, or no longer — never "quarantine not
1531
+ * checked", because every retained-tree pass checks. */
1532
+ quarantineDetail?: string;
1315
1533
  startedAt: number;
1316
1534
  endedAt?: number;
1317
1535
  /** Last failure text, surfaced verbatim in escalations. */
@@ -1364,6 +1582,10 @@ export type AdmissionHoldReason =
1364
1582
  | "issue-closed"
1365
1583
  | "issue-state-lookup-error"
1366
1584
  | "issue-dequeued"
1585
+ /** An operator-parked issue: the park label beat the queue label at claim
1586
+ * time (#734). Unlike `issue-dequeued` it is an operator gesture, not a
1587
+ * queue edit — the queue label may still be physically present. */
1588
+ | "issue-parked"
1367
1589
  | "parent-lookup-error"
1368
1590
  | "sibling-active"
1369
1591
  | "repo-active"
@@ -1374,8 +1596,14 @@ export type AdmissionHoldReason =
1374
1596
  | "plan-usage-cap"
1375
1597
  | "shutting-down"
1376
1598
  | "stale-base"
1599
+ /** A ready issue whose lifecycle state label is residual: its newest run is
1600
+ * terminal (or absent), so nobody owns it — Duty 1 reconciliation work, not
1601
+ * occupied capacity (#611). */
1602
+ | "stale-lifecycle"
1603
+ | "critical-base-verify-error"
1377
1604
  | "file-lane"
1378
1605
  | "depends-on"
1606
+ | "dependency-cycle"
1379
1607
  | "unroutable:no-repo-label"
1380
1608
  | "unroutable:multiple-repo-labels"
1381
1609
  | "unroutable:unknown-repo";
@@ -1425,6 +1653,7 @@ export type FrictionAdmissionReason =
1425
1653
  | "issue-state-lookup-error"
1426
1654
  | "open-pr-lookup-error"
1427
1655
  | "stale-base"
1656
+ | "critical-base-verify-error"
1428
1657
  | "unroutable:no-repo-label"
1429
1658
  | "unroutable:multiple-repo-labels"
1430
1659
  | "unroutable:unknown-repo";
@@ -1744,6 +1973,14 @@ export interface LabelOp {
1744
1973
  * carry, so the operator reconciles by hand, which is the only safe move once
1745
1974
  * the two disagree.
1746
1975
  */
1976
+
1977
+ /**
1978
+ * A settled `ci-deterministic` row offered to the historical infrastructure
1979
+ * reconciliation, with the internal `rowid` that breaks start-time ties so the
1980
+ * persisted review cursor can resume exactly below the last decided row (#638).
1981
+ */
1982
+ export type HistoricalInfraCandidate = RunRecord & { rowid: number };
1983
+
1747
1984
  export interface Store {
1748
1985
  /** A claimed run atomically consumes and applies its issue's pending turn override. */
1749
1986
  createRun(r: Omit<RunRecord, "id">): RunRecord;
@@ -1764,6 +2001,35 @@ export interface Store {
1764
2001
  * per-issue view — a requeued continuation must never hide the PR its
1765
2002
  * predecessor opened. Merged rows keep the same recent-history bound. */
1766
2003
  runsForProjectPr(project: string, prUrl: string, mergedSinceEpochMs: number): RunRecord[];
2004
+ /** Persist one review-revision request (#677), durably, before anything is
2005
+ * woken. Returns `undefined` when a revision for the same run is already
2006
+ * pending undelivered — the atomic half of the duplicate in-flight guard
2007
+ * (the other half is the run row's state once the revision is dispatched). */
2008
+ createReviewRevision(draft: Omit<ReviewRevisionRecord, "id">): ReviewRevisionRecord | undefined;
2009
+ /** Review revisions not yet handed to a worker, oldest first — what the
2010
+ * daemon's dispatch pass wakes on its next tick. */
2011
+ pendingReviewRevisions(project: string): ReviewRevisionRecord[];
2012
+ /** Round number of the newest revision ever requested for a run (0 = never). */
2013
+ latestReviewRound(project: string, runId: string): number;
2014
+ /** Record that a revision was handed to a worker. */
2015
+ markReviewRevisionDispatched(id: string, at: number): void;
2016
+ /** Record how a revision ended. */
2017
+ settleReviewRevision(id: string, outcome: ReviewRevisionOutcome, at: number): void;
2018
+ /** Every revision row not yet settled — queued (`dispatchedAt` unset) and
2019
+ * dispatched-but-unfinished — oldest first. This is what a restart reads to
2020
+ * find review rounds a previous daemon left in flight (#692): the row is
2021
+ * the durable home of the findings, reviewed head, round and target
2022
+ * session, so re-queuing the row is how the exact session resumes. */
2023
+ unsettledReviewRevisions(project: string): ReviewRevisionRecord[];
2024
+ /** Return a dispatched revision to the pending set (clear its dispatched
2025
+ * marker) so the next dispatch pass wakes it again — the reopen half of
2026
+ * restart recovery, paired with restoring its run to `pushed-green`. */
2027
+ requeueReviewRevision(id: string): void;
2028
+ /** Atomically take a settled green run out of reviewable state:
2029
+ * `pushed-green` → `running`, and only from `pushed-green`. False when the
2030
+ * row already moved (a prior revision was dispatched, the PR settled), so
2031
+ * two concurrent dispatchers cannot both wake one run. */
2032
+ claimRunForReview(runId: string): boolean;
1767
2033
  /** Merged rows whose post-merge workflow verdict is still pending, oldest first. */
1768
2034
  runsNeedingBaseCheck(project: string, limit?: number): RunRecord[];
1769
2035
  /** Replace the current live-head health row for one routed repository. */
@@ -1808,6 +2074,11 @@ export interface Store {
1808
2074
  * can name every WIP tip a re-claim would build on and every tree that is
1809
2075
  * still the only copy. */
1810
2076
  salvagedRuns(project: string): RunRecord[];
2077
+ /** Newest attempt per issue whose retained tree is under quarantine: the
2078
+ * object store could not be made sound, so the tree's commits cannot be
2079
+ * verified against any remote and the daemon refused to fetch into it
2080
+ * (#737). Newest-attempt-only for the same reason the board is. */
2081
+ quarantinedRuns(project: string): RunRecord[];
1811
2082
  /** Total run segments, used only for the monotonically increasing run number. */
1812
2083
  attemptsFor(project: string, issue: number): number;
1813
2084
  /** Terminal implementation failures that consume `maxAttemptsPerIssue`. */
@@ -2014,6 +2285,43 @@ export interface Store {
2014
2285
  * than spending one tick on all of it.
2015
2286
  */
2016
2287
  runsNeedingClassification(project: string, limit?: number): RunRecord[];
2288
+ /**
2289
+ * Settled rows currently reading `ci-deterministic` — the only rows the
2290
+ * historical infrastructure reconciliation may repair (#638): the verdict
2291
+ * the forward classifier no longer gives, and nothing else. Newest first
2292
+ * and bounded, because each candidate costs tracker calls to re-fetch the
2293
+ * evidence its log carries.
2294
+ *
2295
+ * `after` is the persisted review cursor (the last row a prior pass decided):
2296
+ * candidates are then only rows strictly *older* than it, so a bounded pass
2297
+ * progresses through a long history instead of rescanning the newest
2298
+ * non-matches forever. `rowid` on each candidate breaks the start-time tie,
2299
+ * giving the cursor an exact resume boundary. */
2300
+ historicalInfraCandidates(
2301
+ project: string,
2302
+ limit?: number,
2303
+ after?: { startedAt: number; rowid: number },
2304
+ ): HistoricalInfraCandidate[];
2305
+ /** The persisted historical-review cursor for a project, or undefined before
2306
+ * any pass decided a row. The `classifierVersion` stamp tells the pass
2307
+ * whether the cursor is still current against the signature list. */
2308
+ historicalInfraCursor(
2309
+ project: string,
2310
+ ): { startedAt: number; rowid: number; classifierVersion: string } | undefined;
2311
+ /** Persist (or move) the historical-review cursor after a pass decided rows. */
2312
+ setHistoricalInfraCursor(
2313
+ project: string,
2314
+ startedAt: number,
2315
+ rowid: number,
2316
+ classifierVersion: string,
2317
+ ): void;
2318
+ /** Reclassify one settled `ci-deterministic` row to `ci-infra` (budget
2319
+ * exempt) after its re-fetched check log matched the closed infrastructure
2320
+ * signature. Changes `failureClass` only — `recoveryAction`/`recoveredAt`
2321
+ * stay byte-for-byte unchanged, so a months-old run is never re-animated
2322
+ * into recovery. False when the row was no longer `ci-deterministic`: a
2323
+ * no-op, exactly what a second repair pass must be. */
2324
+ reclassifyInfra(id: string): boolean;
2017
2325
  /** Unrecovered rows per class, for `status`. Empty when nothing is carrying one. */
2018
2326
  failureClassCounts(project: string): { cls: FailureClass; n: number }[];
2019
2327
  /** Rows whose recovery ran at or after `since`, newest first — the tick's
@@ -2275,6 +2583,11 @@ export const VERB_NAMES = [
2275
2583
  * install shape gates it like a release shape; execution detaches from this
2276
2584
  * session and the daemon, and the first tick after the restart verifies. */
2277
2585
  "conductor_install",
2586
+ /** Return one green, run-owned pull request to its worker with blocking
2587
+ * findings: durably record the revision, and the daemon resumes the SAME
2588
+ * OMP session on the existing branch and PR. No close, no reopen, no
2589
+ * redispatch. */
2590
+ "conductor_pr_review",
2278
2591
  /** The one read verb. It answers with the merge gate's own verdict, so
2279
2592
  * "pushed-green" is the dispatcher's reading of the PR rather than a claim the
2280
2593
  * worker makes about itself from whatever it happened to run. */
@@ -2357,6 +2670,19 @@ export const VERB_REFUSALS = [
2357
2670
  "checks-not-green",
2358
2671
  /** Another merge is in flight for this project. */
2359
2672
  "merge-in-flight",
2673
+ /**
2674
+ * The run backing a PR is not in a revisable settled-green state: a review
2675
+ * revision is already in flight (queued or dispatched), a worker is still
2676
+ * running on it, or the run settled otherwise. The detail names the state.
2677
+ */
2678
+ "review-in-flight",
2679
+ /**
2680
+ * The PR has already been through the configured review-round ceiling
2681
+ * (#678): the policy's hard bound against endless polishing. Refused rather
2682
+ * than recorded — the orchestrator leaves the PR open, records the
2683
+ * unresolved findings and escalates once instead of spending a worker round.
2684
+ */
2685
+ "review-round-ceiling",
2360
2686
  /**
2361
2687
  * The routed repository's base branch is frozen because a watched merge (or
2362
2688
  * live base observation) turned it red — a base-red-freeze. Further merges to
package/src/upgrade.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  import { existsSync, readFileSync } from "node:fs";
2
2
  import { backupTimestamp, copyToUniqueBackup } from "./backups.ts";
3
3
  import { inspectBriefLayout, type BriefLayout } from "./brief-upgrade.ts";
4
- import { pauseInstance, setPaused, statusSnapshot } from "./daemon.ts";
4
+ import { pauseInstance, setPaused, statusSnapshot, type AdmissionAckRecord } from "./daemon.ts";
5
5
  import {
6
6
  DEFAULT_HERDR_SESSION,
7
7
  fleetLayers,
@@ -335,6 +335,22 @@ export interface UpgradeScope {
335
335
  pauseKey: string | undefined;
336
336
  }
337
337
 
338
+ /**
339
+ * The subset of {@link UpgradeDeps} the pause/drain fence needs. Factored out
340
+ * so a caller that only wants the fence — `setup host` — can provide just
341
+ * these members instead of a full install-and-verify deps object. Every
342
+ * {@link UpgradeDeps} is a `DrainDeps`.
343
+ */
344
+ /**
345
+ * The daemon a transaction targets: running or not, its project when it
346
+ * recorded one, and a `generation` identifying the exact instance (#377).
347
+ */
348
+ export interface DaemonIdentity {
349
+ running: boolean;
350
+ project?: string;
351
+ generation?: string;
352
+ }
353
+
338
354
  /**
339
355
  * The subset of {@link UpgradeDeps} the pause/drain fence needs. Factored out
340
356
  * so a caller that only wants the fence — `setup host` — can provide just
@@ -352,11 +368,20 @@ export interface DrainDeps {
352
368
  * The daemon the transaction targets: running or not, its project when it
353
369
  * recorded one, and a `generation` identifying the exact instance (#377).
354
370
  */
355
- daemonIdentity(): { running: boolean; project?: string; generation?: string };
371
+ daemonIdentity(): DaemonIdentity;
356
372
  /** The active pause sentinel as an instance, when one is readable (#377). */
357
373
  pauseState(project?: string): { source: string; reason?: string; since: number } | undefined;
358
374
  /** Write or clear the durable pause sentinel for the scope's project. */
359
375
  setPaused(value: boolean, project?: string): void;
376
+ /**
377
+ * The daemon's durable admission acknowledgement for the current fence, when
378
+ * one is readable: the daemon attests it has observed the pause sentinel at
379
+ * an admission boundary and will claim nothing while it stands (#651 review
380
+ * #3). Absent when no daemon is running, when the file cannot be read, or
381
+ * when the daemon has not (yet) observed the fence. The setup barrier
382
+ * requires this before any mutation when a daemon is running.
383
+ */
384
+ admissionAck?(project?: string): AdmissionAckRecord | undefined;
360
385
  sleep(ms: number): Promise<void>;
361
386
  log(message: string): void;
362
387
  }
@@ -455,31 +480,37 @@ export async function waitForDrain(
455
480
  * live state against this snapshot, so the destructive restart call only ever
456
481
  * acts on the world the request began in (#377).
457
482
  */
458
- interface RestartBegun {
483
+ export interface RestartBegun {
459
484
  /** The durable pause sentinel this request relies on, proved readable as an instance. */
460
485
  pauseToken: { source: string; reason?: string; since: number };
461
486
  /** The daemon instance the restart targets, with its generation identity. */
462
- daemon: ReturnType<UpgradeDeps["daemonIdentity"]>;
487
+ daemon: DaemonIdentity;
463
488
  }
464
489
 
465
490
  /**
466
- * Why a draining restart can no longer act, or `undefined` when it still owns
467
- * the pause and the daemon generation it began with. The pause check is the
468
- * *restart-owned sentinel instance* — `drainAndRestart` proves ownership up
469
- * front, so a token that cannot be re-read (sentinel removed, unreadable, or
470
- * re-created — even under the same source and reason) means this request's own
471
- * pause no longer exists and it must cancel. The generation check is the
472
- * later-daemon fence: a daemon created after this request began — by
473
- * `restart --now`, a crash-restart, or an operator start — must never be
474
- * stopped by it.
491
+ * Why a pause-and-drain transaction can no longer act, or `undefined` when it
492
+ * still owns the pause and the daemon generation it began with. The pause
493
+ * check is the *transaction-owned sentinel instance* — the caller proves
494
+ * ownership up front, so a token that cannot be re-read (sentinel removed,
495
+ * unreadable, or re-created — even under the same source and reason) means
496
+ * this request's own pause no longer exists and it must cancel. The
497
+ * generation check is the later-daemon fence: a daemon created after the
498
+ * transaction began — by `restart --now`, a crash-restart, or an operator
499
+ * start — must never be stopped by it.
500
+ *
501
+ * Exported so the setup apply's acknowledged quiescence barrier runs the same
502
+ * fence instead of inventing a second counter-only one (#618): it freezes
503
+ * admission host-wide through {@link pauseAndDrain}'s machinery, and this
504
+ * check is what proves the freeze is still the one it acknowledged.
475
505
  */
476
- function restartFenceProblem(
506
+ export function restartFenceProblem(
477
507
  deps: DrainDeps,
478
508
  scope: UpgradeScope,
479
509
  begun: RestartBegun,
510
+ actor: string,
480
511
  ): string | undefined {
481
512
  if (!deps.layers(scope.pauseKey).paused) {
482
- return "restart cancelled: dispatch was resumed while the drain was in progress — nothing was restarted";
513
+ return `${actor} cancelled: dispatch was resumed while the drain was in progress — nothing was restarted`;
483
514
  }
484
515
  const owned = deps.pauseState(scope.pauseKey);
485
516
  if (
@@ -488,11 +519,11 @@ function restartFenceProblem(
488
519
  owned.source !== begun.pauseToken.source ||
489
520
  owned.reason !== begun.pauseToken.reason
490
521
  ) {
491
- return "restart cancelled: the restart-owned pause was lifted and replaced while the drain was in progress — nothing was restarted";
522
+ return `${actor} cancelled: the ${actor}-owned pause was lifted and replaced while the drain was in progress — nothing was restarted`;
492
523
  }
493
524
  const now = deps.daemonIdentity();
494
525
  if (now.running !== begun.daemon.running || (begun.daemon.running && now.generation !== begun.daemon.generation)) {
495
- return "restart cancelled: the daemon instance was replaced while the drain was in progress — the newer daemon was left running";
526
+ return `${actor} cancelled: the daemon instance was replaced while the drain was in progress — the newer daemon was left running`;
496
527
  }
497
528
  return undefined;
498
529
  }
@@ -504,7 +535,7 @@ function restartFenceProblem(
504
535
  * {@link pauseSourceToken}; a verb with a space) fails the suite instead of
505
536
  * shipping an unprovable pause (#552).
506
537
  */
507
- export const DRAIN_VERBS = ["upgrade", "restart", "setup host"] as const;
538
+ export const DRAIN_VERBS = ["upgrade", "restart", "setup host", "setup"] as const;
508
539
  export type DrainVerb = (typeof DRAIN_VERBS)[number];
509
540
 
510
541
  /**
@@ -556,7 +587,7 @@ export async function pauseAndDrain(
556
587
  pauseToken,
557
588
  daemon: deps.daemonIdentity(),
558
589
  };
559
- const stale = () => restartFenceProblem(deps, scope, begun);
590
+ const stale = () => restartFenceProblem(deps, scope, begun, verb);
560
591
  await waitForDrain(
561
592
  deps,
562
593
  scope,