omp-conductor 0.15.12 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/REFERENCE.md +81 -6
  2. package/package.json +2 -1
  3. package/schema/config.schema.json +6 -0
  4. package/src/admission.ts +745 -0
  5. package/src/ask.ts +47 -0
  6. package/src/backups.ts +19 -7
  7. package/src/board.ts +1 -2
  8. package/src/briefs/orchestrator.md +62 -4
  9. package/src/cli.ts +26 -0
  10. package/src/commands/context.ts +3 -0
  11. package/src/commands/decision.ts +10 -1
  12. package/src/commands/doctor.ts +2 -0
  13. package/src/commands/message.ts +8 -1
  14. package/src/commands/restart.ts +93 -54
  15. package/src/commands/restore-db.ts +146 -0
  16. package/src/commands/stop.ts +66 -34
  17. package/src/commands/unfreeze.ts +56 -0
  18. package/src/commands/watch.ts +77 -0
  19. package/src/config-schema.ts +9 -0
  20. package/src/config.ts +24 -0
  21. package/src/daemon.ts +485 -577
  22. package/src/dashboard/server.ts +2 -1
  23. package/src/decisions.ts +32 -7
  24. package/src/depends-on.ts +73 -0
  25. package/src/doctor.ts +418 -8
  26. package/src/escalate.ts +122 -15
  27. package/src/failure-class.ts +47 -0
  28. package/src/fleet.ts +55 -377
  29. package/src/gitops.ts +86 -1
  30. package/src/lifecycle.ts +113 -2
  31. package/src/log.ts +40 -0
  32. package/src/model-fallback.ts +3 -2
  33. package/src/omp-settings.ts +114 -0
  34. package/src/omp.ts +63 -0
  35. package/src/orchestrator-down.ts +231 -0
  36. package/src/orchestrator-tick.ts +14 -1
  37. package/src/orchestrator.ts +14 -0
  38. package/src/release-policy.ts +163 -18
  39. package/src/reports.ts +124 -12
  40. package/src/session-host.ts +6 -0
  41. package/src/setup-host.ts +386 -17
  42. package/src/setup-install.ts +40 -2
  43. package/src/setup-wizard.ts +314 -113
  44. package/src/setup.ts +58 -1
  45. package/src/status-render.ts +445 -0
  46. package/src/stop-provenance.ts +119 -0
  47. package/src/store.ts +533 -11
  48. package/src/types.ts +298 -4
  49. package/src/unblock.ts +1 -1
  50. package/src/upgrade-verify.ts +1 -1
  51. package/src/upgrade.ts +27 -8
  52. package/src/verbs/protocol.ts +16 -3
  53. package/src/verbs/server.ts +52 -1
  54. package/src/wizard-ui.ts +261 -46
  55. package/src/worker.ts +183 -10
package/src/types.ts CHANGED
@@ -657,6 +657,26 @@ export interface ProjectConfig {
657
657
  * when absent or unusable.
658
658
  */
659
659
  modelFallbackThreshold?: number;
660
+ /**
661
+ * An opaque omp settings map layered into every worker session this project
662
+ * dispatches (#537). At dispatch it is materialised verbatim to a fleet-owned
663
+ * YAML overlay under the run's session directory — never inside the worktree,
664
+ * whose diff is a PR — and passed to the session as an omp settings overlay
665
+ * (`Settings.init({ configFiles: [<path>] })`), so it layers on top of both
666
+ * the daemon account's global config and any project `<cwd>/.omp/config.yml`
667
+ * without erasing either.
668
+ *
669
+ * Conductor validates YAML shape only: a non-mapping is dropped at load, and
670
+ * everything inside the map is omp's schema to own — an unknown key is omp's
671
+ * to reject, never conductor's to understand. A config edit takes effect on
672
+ * the next dispatch (the overlay is rewritten on every attempt), and a
673
+ * project without the field dispatches byte-for-byte as it always has.
674
+ *
675
+ * The retry keys derived from {@link modelFallbacks} are merged into the
676
+ * effective overlay (unless the map already names a `retry` mapping), which
677
+ * is where #539's staging lives.
678
+ */
679
+ ompSettings?: Record<string, unknown>;
660
680
  /**
661
681
  * How a stuck run reaches a human, what to do when it cannot, and who runs
662
682
  * the session that triages it. See {@link ORCHESTRATOR_MODES}.
@@ -751,6 +771,10 @@ export interface ConductorConfig {
751
771
  version: typeof CONFIG_VERSION;
752
772
  defaults: Caps;
753
773
  projects: ProjectConfig[];
774
+ /** Absolute directory for restorable `conductor.db` snapshots; defaults to
775
+ * `<stateDir()>/backups/db` when omitted. A snapshot that cannot land there
776
+ * is `doctor`'s `db-backup` failure. */
777
+ dbBackupDir?: string;
754
778
  }
755
779
 
756
780
  /**
@@ -1102,6 +1126,11 @@ export const FAILURE_CLASSES = [
1102
1126
  "settlement-stuck",
1103
1127
  "provider-credit",
1104
1128
  "provider-transient",
1129
+ /** A run drowned in in-session provider rate limits (repeated HTTP 429s) and
1130
+ * never got clear of the throttle before the cap swallowed it. Distinct from
1131
+ * `provider-credit` (402) and `provider-transient` (a single stream fault)
1132
+ * because its remedy is different (#573). */
1133
+ "provider-capacity",
1105
1134
  /** A reviewer closed pushed-green or pushed-pending work without merging it:
1106
1135
  * a review decision, not a worker failure. */
1107
1136
  "returned-for-revision",
@@ -1151,6 +1180,30 @@ export interface BaseHealth {
1151
1180
  checkedAt: number;
1152
1181
  }
1153
1182
 
1183
+ /**
1184
+ * A per-repo merge freeze, set when a watched merge (or live base observation)
1185
+ * turns the base red and lifted mechanically on green or by the operator's
1186
+ * `unfreeze` verb. While `clearedAt` is absent the repo is frozen: `prMergeVerb`
1187
+ * refuses further merges to it with the `base-red-freeze` refusal naming the
1188
+ * suspected culprit. It is repo-scoped, never global, so sibling repos keep
1189
+ * merging while one base is broken.
1190
+ */
1191
+ export interface BaseFreeze {
1192
+ repo: string;
1193
+ /** The merged SHA suspected of breaking the base. */
1194
+ culpritSha: string;
1195
+ /** Failing workflow/run evidence naming the break. */
1196
+ detail?: string;
1197
+ /** When the freeze was set (or last refreshed). */
1198
+ setAt: number;
1199
+ /** When the freeze was lifted — absent means the repo is actively frozen. */
1200
+ clearedAt?: number;
1201
+ /** Who lifted it: `daemon` for mechanical recovery, else the operator. */
1202
+ clearedBy?: string;
1203
+ /** Why it was lifted: `base-green` or the operator's override reason. */
1204
+ clearedReason?: string;
1205
+ }
1206
+
1154
1207
  /**
1155
1208
  * Execution state is separate from the tracker's own labels on purpose: labels
1156
1209
  * are coarse and human-editable, while the loop needs to distinguish "pushed
@@ -1194,6 +1247,39 @@ export interface RunRecord {
1194
1247
  /** Effective turn ceiling for this run; operators may only raise it. */
1195
1248
  maxTurns: number;
1196
1249
  spendUsd: number;
1250
+ /** How many in-session HTTP 429 responses the harness retried before the run
1251
+ * ended, counted by the worker from the session's `message_end` events while
1252
+ * the run was live (#573). Absent means the column predates the count or no
1253
+ * worker reported one — never "no rate limit happened": a healthy run
1254
+ * records 0. The classifier only reads it as "sustained" above a threshold,
1255
+ * so a single retried 429 is noise. */
1256
+ provider429Count?: number;
1257
+ /**
1258
+ * The model that actually wrote this run's messages, read from the newest
1259
+ * assistant `message_end`'s `AssistantMessage.model` (#539, #535 slice 1).
1260
+ * Distinct from {@link RunRecord.model} (what the daemon dispatched on):
1261
+ * this is what the harness resolved and actually used, present even for a run
1262
+ * that never failed over, so "which model wrote this" is answerable without
1263
+ * reading a transcript. Absent means the run predates the column or recorded
1264
+ * no message carrying a model.
1265
+ */
1266
+ resolvedModel?: string;
1267
+ /** The provider that wrote them, from the same message. */
1268
+ resolvedProvider?: string;
1269
+ /**
1270
+ * Every within-run model fallback the harness applied (`retry_fallback_applied`),
1271
+ * newest first. The `to` target is what a settlement report names when a run
1272
+ * swapped providers mid-run instead of dying on a throttled primary (#539).
1273
+ */
1274
+ retryFallbacks?: { from: string; to: string }[];
1275
+ /** `retry_fallback_succeeded` events: within-run fallbacks the harness recovered on. */
1276
+ retryFallbackSucceeded?: number;
1277
+ /** Assistant messages whose `retryRecovery.recovery === "model"` (#539). */
1278
+ modelRecoveries?: number;
1279
+ /** `auto_retry_start` events: in-session provider retries the harness ran. */
1280
+ autoRetryCount?: number;
1281
+ /** `auto_compaction_start` events: in-session context compactions. */
1282
+ autoCompactionCount?: number;
1197
1283
  /** omp session transcript, so a human can read what the worker actually did. */
1198
1284
  sessionFile?: string;
1199
1285
  prUrl?: string;
@@ -1288,6 +1374,8 @@ export type AdmissionHoldReason =
1288
1374
  | "plan-usage-cap"
1289
1375
  | "shutting-down"
1290
1376
  | "stale-base"
1377
+ | "file-lane"
1378
+ | "depends-on"
1291
1379
  | "unroutable:no-repo-label"
1292
1380
  | "unroutable:multiple-repo-labels"
1293
1381
  | "unroutable:unknown-repo";
@@ -1298,6 +1386,14 @@ export interface AdmissionHoldSummary {
1298
1386
  count: number;
1299
1387
  /** Queue-order sample, capped before persistence and rendering. */
1300
1388
  issues: number[];
1389
+ /**
1390
+ * Per-issue detail, aligned by index with {@link AdmissionHoldSummary.issues}
1391
+ * and capped the same way. The file-lane interlock (#555) fills it so an
1392
+ * operator reading status or the digest can see *which file* blocked the
1393
+ * held issue and *which running issue* holds it — the same question the
1394
+ * plain `reason` group answers only with a count.
1395
+ */
1396
+ details?: string[];
1301
1397
  }
1302
1398
 
1303
1399
  /** Persisted outcome of the latest completed dispatch tick. */
@@ -1437,9 +1533,29 @@ export interface ReportRecord {
1437
1533
  updatedAt: number;
1438
1534
  /** Backoff gate: a `pending` row is due only once this has passed. */
1439
1535
  nextAttemptAt: number;
1440
- /** Telegram's own id for the delivered message, read out of the response
1441
- * body rather than inferred from the HTTP status. */
1536
+ /** Telegram's own id for the delivered message, read out of the response
1537
+ * body rather than inferred from the HTTP status. A report split into
1538
+ * several messages records the first part's id here; {@link messageIds}
1539
+ * carries every part. */
1442
1540
  messageId?: number;
1541
+ /** Every Telegram message id a split delivery accepted, in order. Absent
1542
+ * for rows delivered before multi-part sends existed (#566), and whenever
1543
+ * Telegram answered without an id — `ok: true` is the verdict, the id is a
1544
+ * courtesy. */
1545
+ messageIds?: number[];
1546
+ /**
1547
+ * Parts of the current delivery contract already accepted, persisted as the
1548
+ * retry's resume point. The split is a pure function of the message text, so
1549
+ * a count is the whole state: a retry whose message is byte-identical to the
1550
+ * attempt that wrote this watermark (see {@link sentPartsHash}) starts at
1551
+ * part `sentParts` instead of re-sending confirmed parts (#566).
1552
+ */
1553
+ sentParts?: number;
1554
+ /** SHA-256 of the message text the `sentParts` watermark was written under.
1555
+ * Progress is only valid while the text is unchanged — a report flagged
1556
+ * ambiguous re-renders with a possible-repeat banner, which reshapes the
1557
+ * split, so the watermark must not carry across it. */
1558
+ sentPartsHash?: string;
1443
1559
  deliveredAt?: number;
1444
1560
  /** Bounded text of the last known failure, surfaced verbatim by `status`. */
1445
1561
  lastError?: string;
@@ -1555,10 +1671,22 @@ export type DecisionState = (typeof DECISION_STATES)[number];
1555
1671
  */
1556
1672
  export const DECISION_TTL_MS = 7 * 24 * 60 * 60_000;
1557
1673
 
1674
+ /** What a pending decision row is owed — answered by a human, or a condition
1675
+ * the orchestrator set for itself with no human in the loop. The split is a
1676
+ * stored column, never inferred from the wording or from whether a condition
1677
+ * is attached: a real question may legitimately carry a condition ("ask me
1678
+ * once this PR merges"), and a watch is defined by having no human, not by
1679
+ * having one. */
1680
+ export type DecisionKind = "question" | "watch";
1681
+
1558
1682
  /** One question put to the operator, and its answer if it has one. */
1559
1683
  export interface DecisionRecord {
1560
1684
  id: string;
1561
1685
  project: string;
1686
+ /** Whether this row is a question a human must answer or a watch the
1687
+ * orchestrator set for itself. A watch turns up under its own heading and
1688
+ * is never offered to the operator to resolve. */
1689
+ kind: DecisionKind;
1562
1690
  /** The question verbatim, as it was sent. Re-asking must not reword it. */
1563
1691
  question: string;
1564
1692
  /** What is waiting on the answer — an issue, a release, a PR. Free text,
@@ -1583,6 +1711,11 @@ export interface DecisionRecord {
1583
1711
  export interface DecisionDraft {
1584
1712
  project: string;
1585
1713
  question: string;
1714
+ /** `"watch"` marks a row the orchestrator set for itself. Absent means a
1715
+ * question a human must answer — the default, so every existing caller
1716
+ * (`message` with a non-material category, `decision open`) stays a
1717
+ * question without remembering to say so. */
1718
+ kind?: DecisionKind;
1586
1719
  blocks?: string;
1587
1720
  condition?: string;
1588
1721
  at: number;
@@ -1642,6 +1775,35 @@ export interface Store {
1642
1775
  project: string,
1643
1776
  sinceEpochMs: number,
1644
1777
  ): { repo: string; baseRef?: string }[];
1778
+ /** The per-repo base-red freeze row, active or cleared, for one routed repo. */
1779
+ baseFreeze(project: string, repo: string): BaseFreeze | undefined;
1780
+ /** Every base-red freeze row for a project, active first, for `status`. */
1781
+ freezes(project: string): BaseFreeze[];
1782
+ /**
1783
+ * Set or refresh an *active* base-red freeze for one repo, naming the
1784
+ * suspected culprit merge and the failing evidence. Refreshing an
1785
+ * already-active freeze keeps the repo frozen while the same head is still
1786
+ * red. Returns true only when it *newly* froze the repo — a transition worth
1787
+ * surfacing as a material event — and false for a pure refresh so the queue
1788
+ * digest is not flooded by per-tick re-observation of a still-red base.
1789
+ */
1790
+ setBaseFreeze(
1791
+ project: string,
1792
+ freeze: { repo: string; culpritSha: string; detail?: string; setAt?: number },
1793
+ ): boolean;
1794
+ /**
1795
+ * Lift an active freeze — the mechanical green recovery or the operator's
1796
+ * `unfreeze`. Records who lifted it and the ledger reason. Returns true only
1797
+ * when an active freeze was actually lifted (so a recovery/override event is
1798
+ * emitted once, never on an already-clear row).
1799
+ */
1800
+ clearBaseFreeze(
1801
+ project: string,
1802
+ repo: string,
1803
+ by: string,
1804
+ reason: string,
1805
+ at?: number,
1806
+ ): boolean;
1645
1807
  /** Newest attempt per issue that preserved work or failed to, so `status`
1646
1808
  * can name every WIP tip a re-claim would build on and every tree that is
1647
1809
  * still the only copy. */
@@ -1736,6 +1898,25 @@ export interface Store {
1736
1898
  /** Start the cooldown only after a tick carrying these signals was sent. */
1737
1899
  markFrictionSurfaced(project: string, kinds: readonly FrictionKind[], at: number): void;
1738
1900
  markNotified(key: string): void;
1901
+ /** The live embedded-orchestrator-down incident for a project, if any. */
1902
+ orchestratorIncident(project: string): OrchestratorIncident | undefined;
1903
+ /** Record a new incident. `false` when one is already open for the project —
1904
+ * dedupe so a flapping orchestrator opens (and pages) once per incident. */
1905
+ openOrchestratorIncident(draft: OrchestratorIncidentDraft): boolean;
1906
+ /** Count tier-1 escalations diverted to an issue comment. No-op when no
1907
+ * incident is open, so a healthy or external orchestrator never accumulates. */
1908
+ bumpOrchestratorDiverted(project: string, by?: number): void;
1909
+ /** Close the open incident and hand back what it accumulated, for the
1910
+ * recovery page's downtime and diverted count. `undefined` when none. */
1911
+ closeOrchestratorIncident(project: string, at: number): OrchestratorIncident | undefined;
1912
+ /** Append one daemon stop/restart provenance line (#378). Host-wide — never
1913
+ * partitioned by project, so any project's status reads the same records. */
1914
+ recordDaemonStop(draft: DaemonStopDraft): DaemonStop;
1915
+ /** The newest stop/restart provenance line, or `undefined` when none was
1916
+ * ever recorded. Read by `status` while the daemon is down and after the
1917
+ * next start — the debrief line an operator gets instead of a bare
1918
+ * "daemon not running". */
1919
+ latestDaemonStop(): DaemonStop | undefined;
1739
1920
  /** Record one observed GitHub rate-limit refusal (the tracker's hook). Rows
1740
1921
  * older than 24h are pruned in the same write (#198). */
1741
1922
  recordGhRefusal?(at: number): void;
@@ -1789,8 +1970,21 @@ export interface Store {
1789
1970
  * means somebody else claimed it first — the guard against the same report
1790
1971
  * being concurrently in flight twice. */
1791
1972
  claimReport(id: string, attemptId: string, at: number): ReportRecord | undefined;
1792
- /** Record the message id Telegram returned, and the moment it did. */
1793
- markReportDelivered(id: string, attemptId: string, messageId: number | undefined, at: number): boolean;
1973
+ /** Record the message ids Telegram returned for every accepted part, and
1974
+ * the moment the last one landed. `messageId` (the first part's id) is
1975
+ * derived here, so the two can never disagree. */
1976
+ markReportDelivered(id: string, attemptId: string, messageIds: readonly number[], at: number): boolean;
1977
+ /** Record progress mid-split: the first `sentParts` parts of the delivery
1978
+ * contract fingerprinted by `sentPartsHash` are confirmed accepted. Written
1979
+ * before the next part ships, so a crash in the window can only ever
1980
+ * re-send the part that was in flight, never confirmed ones. */
1981
+ markReportPartsSent(
1982
+ id: string,
1983
+ attemptId: string,
1984
+ sentParts: number,
1985
+ sentPartsHash: string,
1986
+ at: number,
1987
+ ): boolean;
1794
1988
  /** The attempt ended without an answer: Telegram may hold the message. The
1795
1989
  * row *stays* `sending`, because that state means exactly "outcome unknown",
1796
1990
  * and is flagged {@link ReportRecord.ambiguous} so the retry that stale
@@ -1952,6 +2146,98 @@ export interface HeldNoticeDraft {
1952
2146
  urgent?: true;
1953
2147
  }
1954
2148
 
2149
+ /**
2150
+ * Why the embedded orchestrator is down. `start-failed` covers a session that
2151
+ * never came up (the daemon's `startOrchestrator` threw); `crashed` covers a
2152
+ * session that died after a healthy start (the terminal event reached the
2153
+ * handle and it reported not alive).
2154
+ */
2155
+ export type OrchestratorDownMode = "start-failed" | "crashed";
2156
+
2157
+ /**
2158
+ * The durable orchestrator-down incident, one per project, re-derived across
2159
+ * daemon restarts so a restart while still down cannot forget it.
2160
+ *
2161
+ * Rows persist while the incident is open and are removed on recovery. The
2162
+ * escalate path writes it from the store and the reconcile closes it against
2163
+ * the same store, so a daemon killed mid-incident leaves the row for the next
2164
+ * one — the records ledger advances only on an finished outcome, never on a
2165
+ * process that is gone.
2166
+ */
2167
+ export interface OrchestratorIncident {
2168
+ project: string;
2169
+ mode: OrchestratorDownMode;
2170
+ /** Bounded, human-readable cause. Normally the start error or a "session
2171
+ * child exited N" note. */
2172
+ cause?: string;
2173
+ /** Epoch ms when this incident began. Stable for the life of the row, and
2174
+ * the dedupe anchor: the down page and its recovery page both key on it. */
2175
+ since: number;
2176
+ /** Tier-1 escalations diverted to an issue comment while it was down. */
2177
+ diverted: number;
2178
+ }
2179
+
2180
+ /** What {@link Store.openOrchestratorIncident} is handed. */
2181
+ export interface OrchestratorIncidentDraft {
2182
+ project: string;
2183
+ mode: OrchestratorDownMode;
2184
+ cause?: string;
2185
+ since: number;
2186
+ }
2187
+
2188
+ /**
2189
+ * One durable stop/restart provenance line for the shared daemon (#378).
2190
+ *
2191
+ * Host-wide on purpose: the daemon serves every configured project, so this
2192
+ * table is NOT partitioned by project — a record written by one project's CLI
2193
+ * must be readable from another project's `status` the moment the daemon is
2194
+ * down, exactly the situation the incidents in #378 left unattributable.
2195
+ *
2196
+ * Explicit fields only, and never credentials or environment values: the audit
2197
+ * incident was a stop nobody could attribute, and a "just serialise the
2198
+ * command line" implementation would leak tokens. Caller pid/uid and session
2199
+ * role are captured when a conductor process carried the request; the
2200
+ * external-signal fallback records `unattributed` because Linux exposes no
2201
+ * sender identity for a signal, and saying which daemon and which live runs
2202
+ * were affected is the honest capture of what the receiving process knew.
2203
+ */
2204
+ export interface DaemonStop {
2205
+ id: string;
2206
+ /** Epoch ms the stop/restart was requested (mediated) or observed (fallback). */
2207
+ at: number;
2208
+ /**
2209
+ * The operator-visible control path: "cli stop", "cli restart",
2210
+ * "cli stop via systemctl" (the mediated request, as actually delivered),
2211
+ * or "external signal" (the unattributed fallback).
2212
+ */
2213
+ controlPath: string;
2214
+ /** Caller pid, when a conductor process carried the request. */
2215
+ callerPid?: number;
2216
+ /** Caller uid, when knowable. */
2217
+ callerUid?: number;
2218
+ /** The requesting session's role, when knowable (see {@link SESSION_ROLE_ENV}). */
2219
+ role?: SessionRole;
2220
+ /** "project" when the request originated from one project; "global" otherwise. */
2221
+ scope: "global" | "project";
2222
+ /** The originating project, when project-scoped. */
2223
+ project?: string;
2224
+ /** The daemon pid the stop/restart acted on, when one was known. */
2225
+ daemonPid?: number;
2226
+ /** The daemon runtime directory, when the record is the receiving daemon's
2227
+ * own fallback. Host paths only — the field never carries secrets. */
2228
+ runtimeDir?: string;
2229
+ /** Every configured project with its live-run count at request time. A
2230
+ * project-scoped stop of the shared daemon names the siblings here. */
2231
+ affected: { project: string; live: number }[];
2232
+ /** Non-secret reason; never credentials or environment values. */
2233
+ reason: string;
2234
+ /** True only when no mediated request existed — the external-signal fallback. */
2235
+ unattributed: boolean;
2236
+ }
2237
+
2238
+ /** What a caller hands {@link Store.recordDaemonStop}. The store owns `id` and `at`. */
2239
+ export type DaemonStopDraft = Omit<DaemonStop, "id" | "at">;
2240
+
1955
2241
  /**
1956
2242
  * Baseline limits used when a project omits `caps`. Data, not behaviour: kept
1957
2243
  * beside the type so the defaults cannot drift out of shape with it.
@@ -2071,6 +2357,14 @@ export const VERB_REFUSALS = [
2071
2357
  "checks-not-green",
2072
2358
  /** Another merge is in flight for this project. */
2073
2359
  "merge-in-flight",
2360
+ /**
2361
+ * The routed repository's base branch is frozen because a watched merge (or
2362
+ * live base observation) turned it red — a base-red-freeze. Further merges to
2363
+ * that repo stay refused until the base is observed green again or the
2364
+ * operator lifts the freeze with `omp-conductor unfreeze <repo>`; sibling
2365
+ * repos are unaffected.
2366
+ */
2367
+ "base-red-freeze",
2074
2368
  /** The label is not in this project's own vocabulary. */
2075
2369
  "label-not-in-vocabulary",
2076
2370
  /** The label is a lifecycle label; those transitions stay the daemon's (#26). */
package/src/unblock.ts CHANGED
@@ -35,7 +35,7 @@
35
35
  * failed-attempt budget.
36
36
  */
37
37
 
38
- import { hasContinuationBudget, hasFailedAttemptBudget } from "./daemon.ts";
38
+ import { hasContinuationBudget, hasFailedAttemptBudget } from "./admission.ts";
39
39
  import { projectLabels } from "./label-projection.ts";
40
40
  import { LIVE_STATES } from "./store.ts";
41
41
  import type { Caps, ProjectConfig, RunRecord, Store, Tracker } from "./types.ts";
@@ -26,7 +26,7 @@ import { DEFAULT_PORT } from "./lifecycle.ts";
26
26
  import { dbPath, openStore } from "./store.ts";
27
27
  import { appendJournal, readUpgradeJournal, upgradeJournalPath, type UpgradeCheck, type UpgradeJournalEntry } from "./upgrade-journal.ts";
28
28
  import type { DoctorReport } from "./doctor.ts";
29
- import type { FleetLayers } from "./fleet.ts";
29
+ import type { FleetLayers } from "./status-render.ts";
30
30
  import type { ReportDraft, ReportKind } from "./types.ts";
31
31
 
32
32
  /** One command run to completion with captured output. The transient unit, the
package/src/upgrade.ts CHANGED
@@ -8,9 +8,8 @@ import {
8
8
  LEGACY_HERDR_SESSION_HINT,
9
9
  resolveHerdrSession,
10
10
  telegramStateDir,
11
- type DispatchLayer,
12
- type FleetLayers,
13
11
  } from "./fleet.ts";
12
+ import type { DispatchLayer, FleetLayers } from "./status-render.ts";
14
13
  import { livingDaemon, restartDaemon } from "./lifecycle.ts";
15
14
  import { configBackupDir, configPath, findProject, loadConfig, resolveCaps, stateDir, writeConfigRaw } from "./config.ts";
16
15
  import { renderBriefForProject } from "./setup.ts";
@@ -364,7 +363,7 @@ export interface DrainDeps {
364
363
  */
365
364
  export function resolveScope(
366
365
  deps: DrainDeps,
367
- verb: "upgrade" | "restart" | "setup host",
366
+ verb: DrainVerb,
368
367
  project?: string,
369
368
  ): UpgradeScope {
370
369
  let configured: readonly string[] = [];
@@ -480,6 +479,16 @@ function restartFenceProblem(
480
479
  return undefined;
481
480
  }
482
481
 
482
+ /**
483
+ * Every verb {@link pauseAndDrain} accepts, declared once and used for both the
484
+ * `verb` parameter's type and the sentinel round-trip test — so a fourth verb
485
+ * that cannot be encoded as a single `source=` token (see
486
+ * {@link pauseSourceToken}; a verb with a space) fails the suite instead of
487
+ * shipping an unprovable pause (#552).
488
+ */
489
+ export const DRAIN_VERBS = ["upgrade", "restart", "setup host"] as const;
490
+ export type DrainVerb = (typeof DRAIN_VERBS)[number];
491
+
483
492
  /**
484
493
  * Pause claims and drain live workers to idle — the destructive half of the
485
494
  * trusted restart transaction, minus the restart. Exported so `setup host`
@@ -489,14 +498,18 @@ function restartFenceProblem(
489
498
  * Generation-scoped (#377) exactly as the full transaction is: the caller
490
499
  * captures the restart-owned pause and the daemon generation, and the drain
491
500
  * aborts if a `resume` lifts the pause, replaces the sentinel, or a newer
492
- * daemon generation appears. On ANY throw after the pause the fleet stays
493
- * paused (no resume in a catch), so a drain that outlives `timeoutMs` cannot
494
- * silently resume dispatch over a wedged daemon. `timeoutMs` absent waits
495
- * indefinitely, which is the `upgrade` posture.
501
+ * daemon generation appears. The fence failure path (the pause it just
502
+ * created cannot be read back as an instance) fails closed WITHOUT leaving
503
+ * that pause behind: a pause the transaction cannot prove it is still using
504
+ * is released, restoring the entry state, so the refusal never stops dispatch
505
+ * on its own (#552). A drain that outlives `timeoutMs` stays paused (no
506
+ * resume in a catch) — a drain cannot silently resume dispatch over a wedged
507
+ * daemon. `timeoutMs` absent waits indefinitely, which is the `upgrade`
508
+ * posture.
496
509
  */
497
510
  export async function pauseAndDrain(
498
511
  deps: DrainDeps,
499
- verb: "upgrade" | "restart" | "setup host",
512
+ verb: DrainVerb,
500
513
  o: { project?: string; timeoutMs?: number },
501
514
  ): Promise<{ scope: UpgradeScope; initialPaused: boolean }> {
502
515
  // Host-wide by default (#389): the daemon this restarts serves every
@@ -511,6 +524,12 @@ export async function pauseAndDrain(
511
524
  // on anything it cannot act upon either way.
512
525
  const pauseToken = deps.pauseState(scope.pauseKey);
513
526
  if (pauseToken === undefined) {
527
+ // Fail closed — but do not abandon our own pause. It is only correct to
528
+ // hold a pause the transaction is still using, and a sentinel we cannot
529
+ // prove is unusable: restore the entry state exactly, so a verb that
530
+ // paused the fleet releases it (a fleet already paused stays paused)
531
+ // before the refusal propagates (#552).
532
+ if (!initial.paused) deps.setPaused(false, scope.pauseKey);
514
533
  throw new Error(
515
534
  `${verb} cancelled: cannot prove the ${verb}-owned pause — the active pause sentinel is unreadable or malformed; nothing was restarted`,
516
535
  );
@@ -180,18 +180,21 @@ export const VERB_SPECS: Readonly<Record<VerbName, VerbSpec>> = {
180
180
  allowedRoles: ["orchestrator"],
181
181
  description:
182
182
  "Merge one open pull request. The daemon re-reads the live head immediately before merging " +
183
- "and refuses on any mismatch with headSha, and one merge is in flight per project at a time.",
183
+ "and refuses on any mismatch with headSha, and one merge is in flight per project at a time. " +
184
+ "A PR a run of this project opened merges on the holder's authority; a PR no run opened " +
185
+ "merges only as the orchestrator's own work (author=orchestrator), and only in a routed repo.",
184
186
  args: {
185
187
  prUrl: {
186
188
  type: "string",
187
189
  required: true,
188
- description: "Full pull request URL, belonging to a run in this project.",
190
+ description:
191
+ "Full pull request URL — a run's in this project, or the orchestrator's own (author=orchestrator).",
189
192
  },
190
193
  headSha: {
191
194
  type: "string",
192
195
  required: true,
193
196
  description:
194
- "The head you believe you are merging. Re-read live before the merge; a stale one is refused.",
197
+ "The exact head you believe you are merging. Re-read live before the merge; a stale one is refused.",
195
198
  },
196
199
  reason: {
197
200
  type: "string",
@@ -199,6 +202,16 @@ export const VERB_SPECS: Readonly<Record<VerbName, VerbSpec>> = {
199
202
  description: `Why this merge, from the closed set: ${MERGE_REASONS.join(", ")}.`,
200
203
  oneOf: MERGE_REASONS,
201
204
  },
205
+ author: {
206
+ type: "string",
207
+ required: false,
208
+ description:
209
+ "Optional authorship claim; only 'orchestrator'. States that no run opened this PR — it is " +
210
+ "the orchestrator's own work. Required for a PR no run in this project opened; it never " +
211
+ "unlocks a run's PR, and never bypasses the configured holder, the routed-repo scope, the " +
212
+ "live-head recheck, the green-checks gate, the pause or the ledger.",
213
+ oneOf: ["orchestrator"],
214
+ },
202
215
  rationale: RATIONALE_ARG,
203
216
  },
204
217
  // The exact sentence #126 asks a refused worker to be given. `authority`
@@ -812,6 +812,12 @@ async function prMergeVerb(
812
812
  const prUrl = String(args["prUrl"]);
813
813
  const headSha = String(args["headSha"]);
814
814
  const reason = String(args["reason"]);
815
+ // An explicit authorship claim: `<author> = "orchestrator"` states that no
816
+ // run opened this PR — it is the caller's own work. The claim is the only
817
+ // thing that can make a run-less PR eligible, and it unlocks nothing else:
818
+ // the holder gate above, the routed-repo scope, the pause, the live-head
819
+ // recheck, the green-checks gate and the ledger all still apply below.
820
+ const authoredByOrchestrator = args["author"] === "orchestrator";
815
821
  const target = runForPr(deps, project.name, prUrl);
816
822
  const recovery =
817
823
  target === undefined
@@ -836,8 +842,28 @@ async function prMergeVerb(
836
842
  : "Ask the operator which session is meant to hold it."),
837
843
  );
838
844
  }
845
+ if (authoredByOrchestrator && target !== undefined) {
846
+ return refuse(
847
+ "pr-not-this-run",
848
+ `refused: ${prUrl} is a run-opened PR in ${project.name}; the orchestrator-authors claim is only for ` +
849
+ "pull requests no run ever opened, and a run-owned PR is not claimable as the orchestrator's own work.",
850
+ );
851
+ }
839
852
  if (target === undefined && recovery === undefined) {
840
- return refuse("pr-not-this-run", `refused: ${prUrl} is not a pull request any run in ${project.name} opened.`);
853
+ if (!authoredByOrchestrator) {
854
+ return refuse(
855
+ "pr-not-this-run",
856
+ `refused: ${prUrl} is not a pull request any run in ${project.name} opened. ` +
857
+ "If the orchestrator authored it (no run opened it), re-call with --arg author=orchestrator; " +
858
+ "a PR a run opened stays that run's.",
859
+ );
860
+ }
861
+ if (routedRepo === undefined) {
862
+ return refuse(
863
+ "pr-not-this-run",
864
+ `refused: ${prUrl} is not in ${project.name}'s routed repositories; the orchestrator-authors path cannot widen project scope.`,
865
+ );
866
+ }
841
867
  }
842
868
  if (target === undefined && routedRepo === undefined) {
843
869
  return refuse(
@@ -858,6 +884,31 @@ async function prMergeVerb(
858
884
  if (paused !== undefined) return paused;
859
885
  }
860
886
 
887
+ // Base-red-freeze gate (#283): a repo whose base is frozen must not accept
888
+ // another merge, no matter how green this PR's own checks look. Deliberately
889
+ // placed here — after target-repo identity resolution, before the merge lock
890
+ // and every `gh` call — so the refusal is a verb-level assertion, not a
891
+ // daemon-side flag the merge path could miss. Only the frozen repo is gated:
892
+ // sibling repos keep merging. The freeze lifts mechanically on a green base
893
+ // re-observation or via the operator's sanctioned `omp-conductor unfreeze
894
+ // <repo>` — never by label surgery or a DB edit.
895
+ const freezeRepo = target === undefined ? routedRepo?.name : target.repo;
896
+ if (freezeRepo !== undefined) {
897
+ const freeze = deps.store.baseFreeze(project.name, freezeRepo);
898
+ if (freeze !== undefined && freeze.clearedAt === undefined) {
899
+ return refuse(
900
+ "base-red-freeze",
901
+ `refused: merges to ${freezeRepo} are frozen because its base is red at ` +
902
+ `${freeze.culpritSha.slice(0, 8)} (base-red-freeze). ${freeze.detail ?? ""} ` +
903
+ "This is not a merge-lock or check verdict: the base that merges land on is broken, so " +
904
+ "`conductor_pr_merge` for this repo is refused until the base is observed green again or the " +
905
+ `operator lifts the freeze with \`omp-conductor unfreeze ${freezeRepo}\`. The override is the sanctioned ` +
906
+ "path — never work around this refusal with label surgery or a DB edit.",
907
+ issue,
908
+ );
909
+ }
910
+ }
911
+
861
912
  // Taken before any network call, so two concurrent callers contend here
862
913
  // rather than both spending a `gh` round trip and racing at the merge.
863
914
  const holderId = randomUUID();