omp-conductor 0.16.2 → 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +38 -4
  2. package/REFERENCE.md +18 -12
  3. package/package.json +2 -1
  4. package/schema/config.schema.json +16 -0
  5. package/src/admission.ts +159 -43
  6. package/src/availability.ts +27 -1
  7. package/src/briefs/worker.md +2 -0
  8. package/src/clack-ui.ts +83 -0
  9. package/src/command-manifest.ts +16 -7
  10. package/src/commands/arm.ts +11 -3
  11. package/src/commands/decision.ts +17 -7
  12. package/src/commands/doctor.ts +18 -1
  13. package/src/commands/hold.ts +9 -7
  14. package/src/commands/ledger.ts +25 -4
  15. package/src/commands/message.ts +32 -4
  16. package/src/commands/setup.ts +61 -10
  17. package/src/commands/stats.ts +9 -5
  18. package/src/commands/status.ts +32 -5
  19. package/src/commands/tail.ts +13 -1
  20. package/src/commands/watch.ts +16 -7
  21. package/src/config-schema.ts +20 -0
  22. package/src/config.ts +37 -0
  23. package/src/daemon.ts +1240 -18
  24. package/src/doctor.ts +310 -22
  25. package/src/escalate.ts +560 -57
  26. package/src/failure-class.ts +56 -13
  27. package/src/fleet.ts +224 -47
  28. package/src/gitops.ts +103 -24
  29. package/src/lifecycle.ts +7 -2
  30. package/src/orchestrator-tick.ts +372 -157
  31. package/src/privileged.ts +3 -0
  32. package/src/release-policy.ts +177 -5
  33. package/src/setup-answers.ts +135 -0
  34. package/src/setup-host.ts +193 -4
  35. package/src/setup-install.ts +2 -0
  36. package/src/setup-probe.ts +1 -0
  37. package/src/setup-wizard.ts +1296 -101
  38. package/src/setup.ts +60 -3
  39. package/src/status-render.ts +11 -1
  40. package/src/store.ts +333 -12
  41. package/src/tracker/github.ts +562 -13
  42. package/src/types.ts +204 -2
  43. package/src/ui/progress.ts +32 -0
  44. package/src/ui/style.ts +11 -0
  45. package/src/upgrade.ts +50 -19
  46. package/src/verbs/actions.ts +66 -18
  47. package/src/verbs/protocol.ts +45 -0
  48. package/src/verbs/server.ts +212 -11
  49. package/src/wizard-ui.ts +14 -5
  50. package/src/worker.ts +26 -0
  51. package/systemd/omp-conductor-recover.sh +73 -0
  52. package/systemd/recover-unit-test.sh +61 -0
package/src/types.ts CHANGED
@@ -412,8 +412,11 @@ export type BehindBaseAction = (typeof BEHIND_BASE_ACTIONS)[number];
412
412
  * the entry price for being in this list rather than in POLICY.md.
413
413
  */
414
414
  export const RELEASE_REQUIREMENTS = [
415
- /** Every run this release covers reached `merged`, not merely `pushed-green`. */
415
+ /** Every active run of the released repo reached `merged`, not merely `pushed-green`. */
416
416
  "runs-settled",
417
+ /** Every active run in the project reached `merged` — the suite-wide form of
418
+ * `runs-settled`, for shapes that consume several repos at once. */
419
+ "fleet-runs-settled",
417
420
  /** No pull request is still open against the branch being released. */
418
421
  "no-open-prs",
419
422
  /** Nothing still carries the queue label: the batch is finished, not paused. */
@@ -581,6 +584,63 @@ export const LABEL_REASONS = [
581
584
 
582
585
  export type LabelReason = (typeof LABEL_REASONS)[number];
583
586
 
587
+ /**
588
+ * Why a green pull request is being returned to its worker for revision.
589
+ * Closed for the reason {@link MERGE_REASONS} is: a revision request is a
590
+ * decision the ledger has to be able to name, and the same request repeated
591
+ * must read the same both times.
592
+ */
593
+ export const REVIEW_REASONS = [
594
+ /** The orchestrator read the green PR and found issues that block merging. */
595
+ "blocking-findings",
596
+ ] as const;
597
+
598
+ export type ReviewReason = (typeof REVIEW_REASONS)[number];
599
+
600
+ /**
601
+ * How a queued review revision ended, written by the daemon side of the
602
+ * transport. `revised` is a revision worker that again reached a verified
603
+ * green head; `pending` is a still-unverified push; `failed` is a revision
604
+ * worker that settled terminally; `skipped` is a queued round the daemon could
605
+ * not dispatch (its run moved, its transcript vanished, or the store already
606
+ * claimed the slot).
607
+ */
608
+ export const REVIEW_REVISION_OUTCOMES = ["revised", "pending", "failed", "skipped"] as const;
609
+
610
+ export type ReviewRevisionOutcome = (typeof REVIEW_REVISION_OUTCOMES)[number];
611
+
612
+ /**
613
+ * One durable review-revision request (#677): the orchestrator returned a
614
+ * green, run-owned pull request to its worker with blocking findings, and
615
+ * everything the daemon needs to resume the *same* OMP session is recorded
616
+ * here before any worker is woken. The round numbers successive revisions of
617
+ * one run (1, 2, …); the findings text is delivered to the resumed session
618
+ * verbatim.
619
+ */
620
+ export interface ReviewRevisionRecord {
621
+ id: string;
622
+ project: string;
623
+ /** The run whose pushed-green row this revises. The row is reused, never
624
+ * cloned — no new attempt, and no failed-attempt or continuation charge. */
625
+ runId: string;
626
+ issue: number;
627
+ prUrl: string;
628
+ /** The exact reviewed head; re-verified against the live PR before persisting. */
629
+ headSha: string;
630
+ /** Structured blocking findings, delivered verbatim to the resumed session. */
631
+ findings: string;
632
+ round: number;
633
+ reason: ReviewReason;
634
+ /** The transcript the revision must resume, as the run recorded it — the
635
+ * durable "same OMP session" identity, recorded before any wake. */
636
+ sessionFile?: string;
637
+ requestedAt: number;
638
+ /** Set when the daemon handed the revision to a worker. */
639
+ dispatchedAt?: number;
640
+ settledAt?: number;
641
+ outcome?: ReviewRevisionOutcome;
642
+ }
643
+
584
644
  /**
585
645
  * A model-supplied justification: one value out of a closed set, plus prose that
586
646
  * is written down and read by nothing that decides.
@@ -612,6 +672,33 @@ export const ORCHESTRATOR_MODES = ["embedded", "external"] as const;
612
672
 
613
673
  export type OrchestratorMode = (typeof ORCHESTRATOR_MODES)[number];
614
674
 
675
+ /**
676
+ * What `arm` accepts as proof that a human just approved arming (conductor
677
+ * #613). Declared as data for the reason {@link BASE_FRESHNESS} is: the schema,
678
+ * the setup wizard and the arm gate must all read one vocabulary, and a new
679
+ * proof must be a compile-time change everywhere at once.
680
+ *
681
+ * `challenge` is the pre-#613 behaviour, byte for byte: a Telegram challenge
682
+ * goes out and the armed marker is written only after the reply appears in the
683
+ * orchestrator session. `claim-only` arms on the shared live-plumbing verdict
684
+ * alone — no Telegram send and no wait — which is what makes an unattended
685
+ * recovery re-arm possible, at the cost spelled out in the wizard question:
686
+ * anything that can invoke the already-privileged `omp-conductor arm` command
687
+ * can start dispatch once the live claim and poller pass.
688
+ */
689
+ export const ARM_PROOFS = ["challenge", "claim-only"] as const;
690
+
691
+ export type ArmProof = (typeof ARM_PROOFS)[number];
692
+
693
+ /**
694
+ * The proof a config that never answered the question gets: today's
695
+ * authenticated round-trip, so no existing install changes behaviour on
696
+ * upgrade. Read through `resolveArmProof` (config.ts), never off a raw
697
+ * `ProjectConfig` — the loader materialises `arm` complete, but a hand-built
698
+ * config may omit the key.
699
+ */
700
+ export const DEFAULT_ARM_PROOF: ArmProof = "challenge";
701
+
615
702
  /**
616
703
  * Everything the dispatcher needs to service one product: where work comes
617
704
  * from, where code goes, and what it may spend doing it. Config is per project
@@ -682,6 +769,16 @@ export interface ProjectConfig {
682
769
  * the session that triages it. See {@link ORCHESTRATOR_MODES}.
683
770
  */
684
771
  escalation: { telegramChatId?: string; telegramTopicId?: number; fallbackToIssueComment: boolean; orchestrator: OrchestratorMode };
772
+ /**
773
+ * How `arm` proves a human just approved arming (conductor #613). Optional
774
+ * only for configs written before the key existed and for hand-built test
775
+ * configs: the loader always materialises a complete value, so read it
776
+ * through `resolveArmProof` rather than reaching for `.proof` on a project
777
+ * some test hand-built. Absent resolves to {@link DEFAULT_ARM_PROOF}
778
+ * (`challenge`), preserving today's authenticated round-trip — the one value
779
+ * that changes nothing on upgrade.
780
+ */
781
+ arm?: { proof: ArmProof };
685
782
  /**
686
783
  * Who lands green PRs, who cuts releases and who promotes. The daemon never
687
784
  * acts on this itself — it words the orchestrator's standing orders and the
@@ -742,7 +839,11 @@ export interface ProjectConfig {
742
839
  * the production daemon. With a marker configured, the dispatcher refuses to
743
840
  * reattach any preserved branch missing it, holds the issue as `stale-base`
744
841
  * (distinct from capacity or dependency holds), and never silently merges
745
- * base into the user's work. Absent or empty, ordinary stale continuations
842
+ * base into the user's work. A branch the marker cannot be verified against
843
+ * at all — a failed mirror fetch, an unresolvable marker, or no probe wired
844
+ * — fails closed too, but as `critical-base-verify-error`: a verification
845
+ * failure, never a claim that the branch is stale, re-admitted automatically
846
+ * once a later probe succeeds. Absent or empty, ordinary stale continuations
746
847
  * keep today's behaviour. Optional and hand-edited, like
747
848
  * {@link recoveryMerges}.
748
849
  */
@@ -1028,6 +1129,20 @@ export interface Tracker {
1028
1129
  sha: string,
1029
1130
  opts?: { event?: string; branch?: string },
1030
1131
  ): Promise<WorkflowRun[] | undefined>;
1132
+ /** Full failed-job logs of one workflow run across its attempts, oldest
1133
+ * attempt first, one chunk per failed job — never one aggregate per attempt,
1134
+ * so a mixed failure (setup 429 + sibling compile error) stays visible to
1135
+ * the caller's all-infra guard. `undefined` when the evidence is undecided:
1136
+ * the run register, any attempt's job register, or any failed job's log
1137
+ * could not be read, or the run has more attempts than `maxAttempts` (a
1138
+ * prefix is never presented as the complete attempt history); a no-mutation
1139
+ * refusal the next pass asks again. `[]` when the run is determinately
1140
+ * green across its attempts. Logs are full and untruncated — a truncated
1141
+ * last-400-lines view can discard contrary failure evidence. Read through
1142
+ * the tracker's own guarded runner — call/refusal accounting and the
1143
+ * rate-limit breaker apply exactly as they do to a live tick's reads
1144
+ * (#638). */
1145
+ runFailedAttemptLogs(repo: string, runUrl: string, maxAttempts: number): Promise<string[] | undefined>;
1031
1146
  /** The live commit at one repository branch, or undefined when unreadable. */
1032
1147
  branchHead(repo: string, branch: string): Promise<string | undefined>;
1033
1148
  /** The newest run of this workflow on the base branch before `before`.
@@ -1374,6 +1489,7 @@ export type AdmissionHoldReason =
1374
1489
  | "plan-usage-cap"
1375
1490
  | "shutting-down"
1376
1491
  | "stale-base"
1492
+ | "critical-base-verify-error"
1377
1493
  | "file-lane"
1378
1494
  | "depends-on"
1379
1495
  | "unroutable:no-repo-label"
@@ -1425,6 +1541,7 @@ export type FrictionAdmissionReason =
1425
1541
  | "issue-state-lookup-error"
1426
1542
  | "open-pr-lookup-error"
1427
1543
  | "stale-base"
1544
+ | "critical-base-verify-error"
1428
1545
  | "unroutable:no-repo-label"
1429
1546
  | "unroutable:multiple-repo-labels"
1430
1547
  | "unroutable:unknown-repo";
@@ -1744,6 +1861,14 @@ export interface LabelOp {
1744
1861
  * carry, so the operator reconciles by hand, which is the only safe move once
1745
1862
  * the two disagree.
1746
1863
  */
1864
+
1865
+ /**
1866
+ * A settled `ci-deterministic` row offered to the historical infrastructure
1867
+ * reconciliation, with the internal `rowid` that breaks start-time ties so the
1868
+ * persisted review cursor can resume exactly below the last decided row (#638).
1869
+ */
1870
+ export type HistoricalInfraCandidate = RunRecord & { rowid: number };
1871
+
1747
1872
  export interface Store {
1748
1873
  /** A claimed run atomically consumes and applies its issue's pending turn override. */
1749
1874
  createRun(r: Omit<RunRecord, "id">): RunRecord;
@@ -1764,6 +1889,35 @@ export interface Store {
1764
1889
  * per-issue view — a requeued continuation must never hide the PR its
1765
1890
  * predecessor opened. Merged rows keep the same recent-history bound. */
1766
1891
  runsForProjectPr(project: string, prUrl: string, mergedSinceEpochMs: number): RunRecord[];
1892
+ /** Persist one review-revision request (#677), durably, before anything is
1893
+ * woken. Returns `undefined` when a revision for the same run is already
1894
+ * pending undelivered — the atomic half of the duplicate in-flight guard
1895
+ * (the other half is the run row's state once the revision is dispatched). */
1896
+ createReviewRevision(draft: Omit<ReviewRevisionRecord, "id">): ReviewRevisionRecord | undefined;
1897
+ /** Review revisions not yet handed to a worker, oldest first — what the
1898
+ * daemon's dispatch pass wakes on its next tick. */
1899
+ pendingReviewRevisions(project: string): ReviewRevisionRecord[];
1900
+ /** Round number of the newest revision ever requested for a run (0 = never). */
1901
+ latestReviewRound(project: string, runId: string): number;
1902
+ /** Record that a revision was handed to a worker. */
1903
+ markReviewRevisionDispatched(id: string, at: number): void;
1904
+ /** Record how a revision ended. */
1905
+ settleReviewRevision(id: string, outcome: ReviewRevisionOutcome, at: number): void;
1906
+ /** Every revision row not yet settled — queued (`dispatchedAt` unset) and
1907
+ * dispatched-but-unfinished — oldest first. This is what a restart reads to
1908
+ * find review rounds a previous daemon left in flight (#692): the row is
1909
+ * the durable home of the findings, reviewed head, round and target
1910
+ * session, so re-queuing the row is how the exact session resumes. */
1911
+ unsettledReviewRevisions(project: string): ReviewRevisionRecord[];
1912
+ /** Return a dispatched revision to the pending set (clear its dispatched
1913
+ * marker) so the next dispatch pass wakes it again — the reopen half of
1914
+ * restart recovery, paired with restoring its run to `pushed-green`. */
1915
+ requeueReviewRevision(id: string): void;
1916
+ /** Atomically take a settled green run out of reviewable state:
1917
+ * `pushed-green` → `running`, and only from `pushed-green`. False when the
1918
+ * row already moved (a prior revision was dispatched, the PR settled), so
1919
+ * two concurrent dispatchers cannot both wake one run. */
1920
+ claimRunForReview(runId: string): boolean;
1767
1921
  /** Merged rows whose post-merge workflow verdict is still pending, oldest first. */
1768
1922
  runsNeedingBaseCheck(project: string, limit?: number): RunRecord[];
1769
1923
  /** Replace the current live-head health row for one routed repository. */
@@ -2014,6 +2168,43 @@ export interface Store {
2014
2168
  * than spending one tick on all of it.
2015
2169
  */
2016
2170
  runsNeedingClassification(project: string, limit?: number): RunRecord[];
2171
+ /**
2172
+ * Settled rows currently reading `ci-deterministic` — the only rows the
2173
+ * historical infrastructure reconciliation may repair (#638): the verdict
2174
+ * the forward classifier no longer gives, and nothing else. Newest first
2175
+ * and bounded, because each candidate costs tracker calls to re-fetch the
2176
+ * evidence its log carries.
2177
+ *
2178
+ * `after` is the persisted review cursor (the last row a prior pass decided):
2179
+ * candidates are then only rows strictly *older* than it, so a bounded pass
2180
+ * progresses through a long history instead of rescanning the newest
2181
+ * non-matches forever. `rowid` on each candidate breaks the start-time tie,
2182
+ * giving the cursor an exact resume boundary. */
2183
+ historicalInfraCandidates(
2184
+ project: string,
2185
+ limit?: number,
2186
+ after?: { startedAt: number; rowid: number },
2187
+ ): HistoricalInfraCandidate[];
2188
+ /** The persisted historical-review cursor for a project, or undefined before
2189
+ * any pass decided a row. The `classifierVersion` stamp tells the pass
2190
+ * whether the cursor is still current against the signature list. */
2191
+ historicalInfraCursor(
2192
+ project: string,
2193
+ ): { startedAt: number; rowid: number; classifierVersion: string } | undefined;
2194
+ /** Persist (or move) the historical-review cursor after a pass decided rows. */
2195
+ setHistoricalInfraCursor(
2196
+ project: string,
2197
+ startedAt: number,
2198
+ rowid: number,
2199
+ classifierVersion: string,
2200
+ ): void;
2201
+ /** Reclassify one settled `ci-deterministic` row to `ci-infra` (budget
2202
+ * exempt) after its re-fetched check log matched the closed infrastructure
2203
+ * signature. Changes `failureClass` only — `recoveryAction`/`recoveredAt`
2204
+ * stay byte-for-byte unchanged, so a months-old run is never re-animated
2205
+ * into recovery. False when the row was no longer `ci-deterministic`: a
2206
+ * no-op, exactly what a second repair pass must be. */
2207
+ reclassifyInfra(id: string): boolean;
2017
2208
  /** Unrecovered rows per class, for `status`. Empty when nothing is carrying one. */
2018
2209
  failureClassCounts(project: string): { cls: FailureClass; n: number }[];
2019
2210
  /** Rows whose recovery ran at or after `since`, newest first — the tick's
@@ -2275,6 +2466,11 @@ export const VERB_NAMES = [
2275
2466
  * install shape gates it like a release shape; execution detaches from this
2276
2467
  * session and the daemon, and the first tick after the restart verifies. */
2277
2468
  "conductor_install",
2469
+ /** Return one green, run-owned pull request to its worker with blocking
2470
+ * findings: durably record the revision, and the daemon resumes the SAME
2471
+ * OMP session on the existing branch and PR. No close, no reopen, no
2472
+ * redispatch. */
2473
+ "conductor_pr_review",
2278
2474
  /** The one read verb. It answers with the merge gate's own verdict, so
2279
2475
  * "pushed-green" is the dispatcher's reading of the PR rather than a claim the
2280
2476
  * worker makes about itself from whatever it happened to run. */
@@ -2357,6 +2553,12 @@ export const VERB_REFUSALS = [
2357
2553
  "checks-not-green",
2358
2554
  /** Another merge is in flight for this project. */
2359
2555
  "merge-in-flight",
2556
+ /**
2557
+ * The run backing a PR is not in a revisable settled-green state: a review
2558
+ * revision is already in flight (queued or dispatched), a worker is still
2559
+ * running on it, or the run settled otherwise. The detail names the state.
2560
+ */
2561
+ "review-in-flight",
2360
2562
  /**
2361
2563
  * The routed repository's base branch is frozen because a watched merge (or
2362
2564
  * live base observation) turned it red — a base-red-freeze. Further merges to
@@ -0,0 +1,32 @@
1
+ /** Whether the CLI owns an interactive terminal rather than a pipe or plain-UI session. */
2
+ export function interactiveUi(): boolean {
3
+ return Boolean(
4
+ process.stdin.isTTY &&
5
+ process.stdout.isTTY &&
6
+ !process.env.OMP_CONDUCTOR_PLAIN_UI,
7
+ );
8
+ }
9
+
10
+ /** Run one long, silent operation with safe Clack progress on an interactive TTY. */
11
+ export async function withProgress<T>(
12
+ message: string,
13
+ success: string,
14
+ operation: () => Promise<T>,
15
+ options: { plainMessage?: boolean } = {},
16
+ ): Promise<T> {
17
+ if (!interactiveUi()) {
18
+ if (options.plainMessage) process.stdout.write(`${message}\n`);
19
+ return operation();
20
+ }
21
+
22
+ const { log } = await import("@clack/prompts");
23
+ log.step(message);
24
+ try {
25
+ const result = await operation();
26
+ log.success(success);
27
+ return result;
28
+ } catch (err) {
29
+ log.error(`${message} failed`);
30
+ throw err;
31
+ }
32
+ }
@@ -0,0 +1,11 @@
1
+ import { styleText } from "node:util";
2
+
3
+ function styled(format: Parameters<typeof styleText>[0], text: string): string {
4
+ return process.stdout.isTTY ? styleText(format, text) : text;
5
+ }
6
+
7
+ export const heading = (text: string): string => styled(["bold", "cyan"], text);
8
+ export const ok = (text: string): string => styled("green", text);
9
+ export const warn = (text: string): string => styled("yellow", text);
10
+ export const fail = (text: string): string => styled("red", text);
11
+ export const dim = (text: string): string => styled("dim", text);
package/src/upgrade.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  import { existsSync, readFileSync } from "node:fs";
2
2
  import { backupTimestamp, copyToUniqueBackup } from "./backups.ts";
3
3
  import { inspectBriefLayout, type BriefLayout } from "./brief-upgrade.ts";
4
- import { pauseInstance, setPaused, statusSnapshot } from "./daemon.ts";
4
+ import { pauseInstance, setPaused, statusSnapshot, type AdmissionAckRecord } from "./daemon.ts";
5
5
  import {
6
6
  DEFAULT_HERDR_SESSION,
7
7
  fleetLayers,
@@ -335,6 +335,22 @@ export interface UpgradeScope {
335
335
  pauseKey: string | undefined;
336
336
  }
337
337
 
338
+ /**
339
+ * The subset of {@link UpgradeDeps} the pause/drain fence needs. Factored out
340
+ * so a caller that only wants the fence — `setup host` — can provide just
341
+ * these members instead of a full install-and-verify deps object. Every
342
+ * {@link UpgradeDeps} is a `DrainDeps`.
343
+ */
344
+ /**
345
+ * The daemon a transaction targets: running or not, its project when it
346
+ * recorded one, and a `generation` identifying the exact instance (#377).
347
+ */
348
+ export interface DaemonIdentity {
349
+ running: boolean;
350
+ project?: string;
351
+ generation?: string;
352
+ }
353
+
338
354
  /**
339
355
  * The subset of {@link UpgradeDeps} the pause/drain fence needs. Factored out
340
356
  * so a caller that only wants the fence — `setup host` — can provide just
@@ -352,11 +368,20 @@ export interface DrainDeps {
352
368
  * The daemon the transaction targets: running or not, its project when it
353
369
  * recorded one, and a `generation` identifying the exact instance (#377).
354
370
  */
355
- daemonIdentity(): { running: boolean; project?: string; generation?: string };
371
+ daemonIdentity(): DaemonIdentity;
356
372
  /** The active pause sentinel as an instance, when one is readable (#377). */
357
373
  pauseState(project?: string): { source: string; reason?: string; since: number } | undefined;
358
374
  /** Write or clear the durable pause sentinel for the scope's project. */
359
375
  setPaused(value: boolean, project?: string): void;
376
+ /**
377
+ * The daemon's durable admission acknowledgement for the current fence, when
378
+ * one is readable: the daemon attests it has observed the pause sentinel at
379
+ * an admission boundary and will claim nothing while it stands (#651 review
380
+ * #3). Absent when no daemon is running, when the file cannot be read, or
381
+ * when the daemon has not (yet) observed the fence. The setup barrier
382
+ * requires this before any mutation when a daemon is running.
383
+ */
384
+ admissionAck?(project?: string): AdmissionAckRecord | undefined;
360
385
  sleep(ms: number): Promise<void>;
361
386
  log(message: string): void;
362
387
  }
@@ -455,31 +480,37 @@ export async function waitForDrain(
455
480
  * live state against this snapshot, so the destructive restart call only ever
456
481
  * acts on the world the request began in (#377).
457
482
  */
458
- interface RestartBegun {
483
+ export interface RestartBegun {
459
484
  /** The durable pause sentinel this request relies on, proved readable as an instance. */
460
485
  pauseToken: { source: string; reason?: string; since: number };
461
486
  /** The daemon instance the restart targets, with its generation identity. */
462
- daemon: ReturnType<UpgradeDeps["daemonIdentity"]>;
487
+ daemon: DaemonIdentity;
463
488
  }
464
489
 
465
490
  /**
466
- * Why a draining restart can no longer act, or `undefined` when it still owns
467
- * the pause and the daemon generation it began with. The pause check is the
468
- * *restart-owned sentinel instance* — `drainAndRestart` proves ownership up
469
- * front, so a token that cannot be re-read (sentinel removed, unreadable, or
470
- * re-created — even under the same source and reason) means this request's own
471
- * pause no longer exists and it must cancel. The generation check is the
472
- * later-daemon fence: a daemon created after this request began — by
473
- * `restart --now`, a crash-restart, or an operator start — must never be
474
- * stopped by it.
491
+ * Why a pause-and-drain transaction can no longer act, or `undefined` when it
492
+ * still owns the pause and the daemon generation it began with. The pause
493
+ * check is the *transaction-owned sentinel instance* — the caller proves
494
+ * ownership up front, so a token that cannot be re-read (sentinel removed,
495
+ * unreadable, or re-created — even under the same source and reason) means
496
+ * this request's own pause no longer exists and it must cancel. The
497
+ * generation check is the later-daemon fence: a daemon created after the
498
+ * transaction began — by `restart --now`, a crash-restart, or an operator
499
+ * start — must never be stopped by it.
500
+ *
501
+ * Exported so the setup apply's acknowledged quiescence barrier runs the same
502
+ * fence instead of inventing a second counter-only one (#618): it freezes
503
+ * admission host-wide through {@link pauseAndDrain}'s machinery, and this
504
+ * check is what proves the freeze is still the one it acknowledged.
475
505
  */
476
- function restartFenceProblem(
506
+ export function restartFenceProblem(
477
507
  deps: DrainDeps,
478
508
  scope: UpgradeScope,
479
509
  begun: RestartBegun,
510
+ actor: string,
480
511
  ): string | undefined {
481
512
  if (!deps.layers(scope.pauseKey).paused) {
482
- return "restart cancelled: dispatch was resumed while the drain was in progress — nothing was restarted";
513
+ return `${actor} cancelled: dispatch was resumed while the drain was in progress — nothing was restarted`;
483
514
  }
484
515
  const owned = deps.pauseState(scope.pauseKey);
485
516
  if (
@@ -488,11 +519,11 @@ function restartFenceProblem(
488
519
  owned.source !== begun.pauseToken.source ||
489
520
  owned.reason !== begun.pauseToken.reason
490
521
  ) {
491
- return "restart cancelled: the restart-owned pause was lifted and replaced while the drain was in progress — nothing was restarted";
522
+ return `${actor} cancelled: the ${actor}-owned pause was lifted and replaced while the drain was in progress — nothing was restarted`;
492
523
  }
493
524
  const now = deps.daemonIdentity();
494
525
  if (now.running !== begun.daemon.running || (begun.daemon.running && now.generation !== begun.daemon.generation)) {
495
- return "restart cancelled: the daemon instance was replaced while the drain was in progress — the newer daemon was left running";
526
+ return `${actor} cancelled: the daemon instance was replaced while the drain was in progress — the newer daemon was left running`;
496
527
  }
497
528
  return undefined;
498
529
  }
@@ -504,7 +535,7 @@ function restartFenceProblem(
504
535
  * {@link pauseSourceToken}; a verb with a space) fails the suite instead of
505
536
  * shipping an unprovable pause (#552).
506
537
  */
507
- export const DRAIN_VERBS = ["upgrade", "restart", "setup host"] as const;
538
+ export const DRAIN_VERBS = ["upgrade", "restart", "setup host", "setup"] as const;
508
539
  export type DrainVerb = (typeof DRAIN_VERBS)[number];
509
540
 
510
541
  /**
@@ -556,7 +587,7 @@ export async function pauseAndDrain(
556
587
  pauseToken,
557
588
  daemon: deps.daemonIdentity(),
558
589
  };
559
- const stale = () => restartFenceProblem(deps, scope, begun);
590
+ const stale = () => restartFenceProblem(deps, scope, begun, verb);
560
591
  await waitForDrain(
561
592
  deps,
562
593
  scope,
@@ -20,6 +20,13 @@ import { stateDir } from "../config.ts";
20
20
  import { ensureMirror } from "../worktree.ts";
21
21
  import { launchTransientUnit, readUpgradeJournal } from "../upgrade-journal.ts";
22
22
  import { classifyUpgrade } from "../upgrade-verify.ts";
23
+ import {
24
+ GraphqlBreaker,
25
+ graphqlBreakerRefusal,
26
+ isGraphqlSurface,
27
+ isTransientServer5xx,
28
+ prUrlParts,
29
+ } from "../tracker/github.ts";
23
30
  import type { ProjectConfig, ReleaseShape, RepoTarget } from "../types.ts";
24
31
  import type { ActionOutcome, InstallExecution, ReleaseExecution, VerbActions } from "./server.ts";
25
32
 
@@ -204,15 +211,32 @@ export function githubVerbActions(
204
211
  project: ProjectConfig,
205
212
  run: CommandRunner = spawnCommand,
206
213
  prepareMirror: MirrorPreparer = ensureMirror,
214
+ graphqlBreaker?: GraphqlBreaker,
207
215
  ): VerbActions {
208
216
  const env = (): Record<string, string> => credentialedEnv();
209
217
 
210
218
  const runGh = async (argv: string[], cwd?: string): Promise<{ argv: string[]; result: CommandRun }> => {
211
219
  const full = ["gh", ...argv];
212
- return {
213
- argv: full,
214
- result: await run(full, { env: env(), ...(cwd === undefined ? {} : { cwd }) }),
215
- };
220
+ // The mutation surface rides the same GraphQL endpoint the tracker's
221
+ // parent/closer checks ride in the pinned gh (2.86): `pr merge`,
222
+ // `pr update-branch`, `pr edit`, `pr create`. When a transient 5xx has
223
+ // already proven that surface unavailable — observed by admission or by a
224
+ // sibling mutation this pass — a mutation must fast-fail WITHOUT spawning
225
+ // `gh`, exactly like the tracker's candidate checks (#642 comment 4):
226
+ // attempting a mutation into a known-broken surface is how one provider
227
+ // outage defeats queue locking. REST-backed commands (`gh api repos/...`,
228
+ // `gh release create`) keep running: the outage is GraphQL-only.
229
+ if (graphqlBreaker !== undefined && isGraphqlSurface(argv) && graphqlBreaker.refused()) {
230
+ return {
231
+ argv: full,
232
+ result: { ok: false, stdout: "", stderr: graphqlBreakerRefusal(graphqlBreaker.until) },
233
+ };
234
+ }
235
+ const result = await run(full, { env: env(), ...(cwd === undefined ? {} : { cwd }) });
236
+ if (!result.ok && graphqlBreaker !== undefined && isGraphqlSurface(argv) && isTransientServer5xx(result.stderr)) {
237
+ graphqlBreaker.open();
238
+ }
239
+ return { argv: full, result };
216
240
  };
217
241
 
218
242
  const gh = async (argv: string[], cwd?: string): Promise<ActionOutcome> => {
@@ -573,7 +597,44 @@ export function githubVerbActions(
573
597
 
574
598
  createPr: (target, opts) => openRunPr(project, target, opts),
575
599
 
576
- updatePrBranch: (prUrl) => gh(["pr", "update-branch", prUrl]),
600
+ updatePrBranch: async (prUrl) => {
601
+ // The branch refresh rides the REST update-branch endpoint, not `gh pr
602
+ // update-branch` (#655): `gh pr update-branch` is a GraphQL mutation, so
603
+ // a selective GraphQL outage made an otherwise mergeable PR unrefreshable
604
+ // even while REST answered. Both calls here are core REST, so they keep
605
+ // running while the GraphQL breaker is open.
606
+ const parts = prUrlParts(prUrl);
607
+ if (parts === undefined) {
608
+ return { ok: false, stderr: `could not parse ${prUrl} as a pull request URL` };
609
+ }
610
+ const pulls = `repos/${parts.owner}/${parts.repo}/pulls/${parts.number}`;
611
+ // The stale-head guard needs the exact head the update is bound to. It is
612
+ // the last observed head of the branch refresh, so a concurrent push is
613
+ // refused (422) rather than updating an unreviewed head.
614
+ const head = await runGh(["api", pulls, "--jq", ".head.sha"]);
615
+ if (!head.result.ok) return failed(head.result, head.argv);
616
+ const headSha = commitFrom(head.result.stdout);
617
+ if (headSha === undefined) {
618
+ return { ok: false, stderr: `${prUrl} returned no full head commit SHA` };
619
+ }
620
+ const updated = await gh([
621
+ "api",
622
+ "--method",
623
+ "PUT",
624
+ `${pulls}/update-branch`,
625
+ "-f",
626
+ `expected_head_sha=${headSha}`,
627
+ ]);
628
+ if (!updated.ok) return updated;
629
+ // The endpoint answers 202 Accepted and refreshes the branch
630
+ // asynchronously, so the head observed for `expected_head_sha` is the
631
+ // pre-update head. Producing it as `sha` would report a merge at a head
632
+ // nobody observed; the caller re-reads PR and check state instead.
633
+ return {
634
+ ok: true,
635
+ detail: `requested a base-branch update of ${prUrl} from its observed head ${headSha}; re-read PR and check state before merging`,
636
+ };
637
+ },
577
638
 
578
639
  updatePr: async (prUrl, fields) => {
579
640
  const argv = ["pr", "edit", prUrl];
@@ -589,19 +650,6 @@ export function githubVerbActions(
589
650
  // and braces on the one operation that cannot be undone.
590
651
  mergePr: (prUrl, headSha) => gh(["pr", "merge", prUrl, "--squash", "--match-head-commit", headSha]),
591
652
 
592
- setLabel: async (issue, label, action) => {
593
- const slug = project.tracker.repo;
594
- return gh([
595
- "issue",
596
- "edit",
597
- String(issue),
598
- "--repo",
599
- slug,
600
- action === "add" ? "--add-label" : "--remove-label",
601
- label,
602
- ]);
603
- },
604
-
605
653
  release: async (execution: ReleaseExecution): Promise<ActionOutcome> => {
606
654
  const slug = repoSlugFor(execution.repo);
607
655
  const tag = execution.tag;
@@ -26,6 +26,7 @@ import {
26
26
  RELEASE_REASONS,
27
27
  RELEASE_SHAPES,
28
28
  RESERVED_VERB_FIELDS,
29
+ REVIEW_REASONS,
29
30
  VERB_NAMES,
30
31
  } from "../types.ts";
31
32
  import type { ReservedVerbField, SessionRole, VerbName, VerbRefusal } from "../types.ts";
@@ -326,6 +327,50 @@ export const VERB_SPECS: Readonly<Record<VerbName, VerbSpec>> = {
326
327
  `install authority is the orchestrator's, and this is a ${role} session. ` +
327
328
  "A worker session holds no install grant whatever the config says.",
328
329
  },
330
+ conductor_pr_review: {
331
+ name: "conductor_pr_review",
332
+ mutating: true,
333
+ allowedRoles: ["orchestrator"],
334
+ description:
335
+ "Return one green, run-owned pull request to its worker with blocking findings (#677). " +
336
+ "The daemon records the revision durably — findings, exact reviewed head, round number, " +
337
+ "target run and session — then resumes the SAME OMP session on the existing branch and PR: " +
338
+ "no close, no reopen, no issue redispatch, no fresh brief. The worker edits and pushes the " +
339
+ "same branch; a new green head returns to the normal orchestrator review surface. Refused " +
340
+ "for a stale head, a PR no run of this project owns, a run that is not settled green, or a " +
341
+ "revision already in flight for the same PR.",
342
+ args: {
343
+ prUrl: {
344
+ type: "string",
345
+ required: true,
346
+ description:
347
+ "Full pull request URL — one a run of this project opened and pushed green. Must be in " +
348
+ "a routed repository.",
349
+ },
350
+ headSha: {
351
+ type: "string",
352
+ required: true,
353
+ description:
354
+ "The exact reviewed head. The live head must match it and the checks at it must be green.",
355
+ },
356
+ findings: {
357
+ type: "string",
358
+ required: true,
359
+ description:
360
+ "Structured blocking findings. Delivered verbatim to the resumed session; the worker is " +
361
+ "told to address exactly these and push the same branch.",
362
+ },
363
+ reason: {
364
+ type: "string",
365
+ required: true,
366
+ description: `Why this revision, from the closed set: ${REVIEW_REASONS.join(", ")}.`,
367
+ oneOf: REVIEW_REASONS,
368
+ },
369
+ rationale: RATIONALE_ARG,
370
+ },
371
+ roleRefusalText: (role) =>
372
+ `review authority is the orchestrator's; this is a ${role} session, and a worker never returns its own PR for revision.`,
373
+ },
329
374
  conductor_pr_status: {
330
375
  name: "conductor_pr_status",
331
376
  mutating: false,