omp-conductor 0.15.11 → 0.15.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -18,7 +18,17 @@
18
18
  import { existsSync, readFileSync } from "node:fs";
19
19
  import { platform } from "node:os";
20
20
  import { join } from "node:path";
21
- import { stateDir } from "./config.ts";
21
+ import { loadConfig, stateDir } from "./config.ts";
22
+ import { pauseInstance, setPaused, statusSnapshot } from "./daemon.ts";
23
+ import { fleetLayers, DEFAULT_HERDR_UNIT } from "./fleet.ts";
24
+ import { livingDaemon } from "./lifecycle.ts";
25
+ import { dbPath, openStore } from "./store.ts";
26
+ import {
27
+ resolveScope,
28
+ pauseAndDrain,
29
+ type DrainDeps,
30
+ type UpgradeScope,
31
+ } from "./upgrade.ts";
22
32
  import {
23
33
  graphRepos,
24
34
  formatGraphSetup,
@@ -40,12 +50,12 @@ import {
40
50
  writeHostRuntime,
41
51
  STAGED_SERVICE_NAME,
42
52
  SYSTEMD_UNIT_DIR,
53
+ RECOVER_SCRIPT_INSTALL_PATH,
43
54
  type EscalationDeps,
44
55
  type ServiceRuntime,
45
56
  } from "./setup-host.ts";
46
- import { DEFAULT_HERDR_UNIT } from "./fleet.ts";
47
57
  import type { WizardUi } from "./wizard-ui.ts";
48
- import type { Caps, ProjectConfig } from "./types.ts";
58
+ import type { Caps, ProjectConfig, Store } from "./types.ts";
49
59
 
50
60
  /**
51
61
  * Every outcome a caller has to tell apart. `staged` is the non-Linux answer:
@@ -64,12 +74,40 @@ export interface InstallDeps {
64
74
  /** `"linux"` gates the systemd half. Injectable so the refusal is testable. */
65
75
  platform?: () => string;
66
76
  unitDir?: string;
77
+ /**
78
+ * Where the recovery playbook installs. Injectable so the idempotency gate
79
+ * is testable hermetically; the real install uses
80
+ * {@link RECOVER_SCRIPT_INSTALL_PATH}.
81
+ */
82
+ recoverScriptInstallPath?: string;
67
83
  /**
68
84
  * The runtime the plan derives from. Injectable so tests pin herdr
69
85
  * present/absent deterministically; the default ({@link defaultServiceRuntime})
70
86
  * detects herdr from the host's own PATH.
71
87
  */
72
88
  runtime?: ServiceRuntime;
89
+ /**
90
+ * Whether the host configures more than one project, which scopes the
91
+ * recovery unit's RECOVER_PROJECT (host-global: never one project's name on
92
+ * a multi-project host, #510/#530). Injectable so tests pin the shape
93
+ * hermetically; the default ({@link hostMultiProject}) reads the config.
94
+ */
95
+ multiProject?: boolean;
96
+ /**
97
+ * The pause/drain fence around the restart (#512): which projects to drain,
98
+ * how long to wait, and the store used to attribute a run the restart kills.
99
+ * Injectable so tests pin the drain hermetically; the defaults read the real
100
+ * daemon, sentinel and store.
101
+ */
102
+ drain?: {
103
+ deps?: DrainDeps;
104
+ /** How long the drain waits for live workers before giving up and staying
105
+ * paused; absent waits indefinitely, the `upgrade` posture. */
106
+ timeoutMs?: number;
107
+ /** The store consulted for runs still live at restart time, so they are
108
+ * attributed before the session host restarts under them. */
109
+ drainStore?: Pick<Store, "liveRuns" | "updateRun">;
110
+ };
73
111
  }
74
112
 
75
113
  /**
@@ -100,11 +138,21 @@ function sameFile(a: string, b: string): boolean {
100
138
  }
101
139
 
102
140
  /**
103
- * Install the supervised daemon unit: re-render, stage, then run the four steps
104
- * `planHostRuntime` used to only print.
141
+ * Install the supervised daemon unit: re-render, stage, then run the plan's
142
+ * steps in order. The steps are the plan's own {@link HostRuntimePlan.steps} —
143
+ * the same single list its {@link installCommands} display renders from — so
144
+ * what is printed and what is executed can never diverge.
105
145
  */
106
146
  export async function runHostInstall(
107
- project: ProjectConfig,
147
+ /**
148
+ * The project whose per-project tail to write, or `undefined` for a
149
+ * host-global install: the units are host-global (same staged paths, same
150
+ * systemd destinations, one shared daemon), so no project still installs
151
+ * and starts them. The per-project tail (tick config, brief link) is the
152
+ * only part that needs a name — with none, it is skipped with a printed
153
+ * note (#530).
154
+ */
155
+ project: ProjectConfig | undefined,
108
156
  caps: Caps,
109
157
  telegramStateDir: string,
110
158
  ui: WizardUi,
@@ -116,40 +164,105 @@ export async function runHostInstall(
116
164
  return { kind: "refused", reason: verdict.message };
117
165
  }
118
166
 
119
- const plan = planHostRuntime(project, caps, telegramStateDir, deps.runtime, hostInstallWorkers(caps));
120
- const { wrote, warnings } = writeHostRuntime(plan);
121
- for (const warning of warnings) ui.notify(warning, "warning");
122
167
  const unitDir = deps.unitDir ?? SYSTEMD_UNIT_DIR;
168
+ // The plan is computed eagerly — `planHostRuntime` renders every byte and
169
+ // compares it against disk, so the prompt can print exactly what would be
170
+ // written and the currentInstall gate below can decide a re-run restarts
171
+ // nothing. But rendering is read-only: no file is written here. Staging
172
+ // happens only after consent (#510), so the prompt's "Nothing has been run
173
+ // yet" is true of the staged tree when it is printed, and a declined run
174
+ // leaves every staged file byte-identical.
175
+ const plan = planHostRuntime(
176
+ project,
177
+ caps,
178
+ telegramStateDir,
179
+ deps.runtime,
180
+ hostInstallWorkers(caps),
181
+ unitDir,
182
+ deps.recoverScriptInstallPath ?? RECOVER_SCRIPT_INSTALL_PATH,
183
+ deps.multiProject ?? hostMultiProject(),
184
+ );
185
+ // No pane-shell key is planned (unusable login shell, unparseable config):
186
+ // the operator hears why before the consent prompt, not after an install
187
+ // that quietly had nothing to say about the shell.
188
+ if (plan.herdrConfigProblem !== undefined) ui.notify(plan.herdrConfigProblem, "warning");
189
+ // A host-global install (no --project): the per-project tail was
190
+ // deliberately not written, and the operator must be told which files and
191
+ // how to get them — never a silent skip.
192
+ if (plan.noProject !== undefined) {
193
+ ui.notify(
194
+ `No project named: ${plan.noProject.skipped.join(", ")} were not written. ${plan.noProject.how}.`,
195
+ "info",
196
+ );
197
+ }
123
198
  const installed = join(unitDir, STAGED_SERVICE_NAME);
124
199
  const installedHerdr = join(unitDir, DEFAULT_HERDR_UNIT);
125
200
 
126
201
  const blocked = linuxOnly(deps);
127
202
  if (blocked !== undefined) {
203
+ // A non-Linux host cannot run systemd, so staging IS the deliverable: the
204
+ // rendered files are written for the operator to copy to the box that will
205
+ // run them, and only the systemctl half is refused. There is no consent
206
+ // prompt on this path — nothing privileged to approve — so the write lands
207
+ // here, not in the (never reached) post-consent hook.
208
+ const { wrote, warnings } = writeHostRuntime(plan);
209
+ for (const warning of warnings) ui.notify(warning, "warning");
128
210
  ui.notify(`${blocked} ${plan.service.path}`, "warning");
129
211
  return { kind: "staged", wrote, reason: `${blocked} ${plan.service.path}` };
130
212
  }
131
213
 
214
+ // A no-op re-run: every unit the steps would install is already current on
215
+ // disk, so installing them again would only touch mtimes and restart the
216
+ // units for nothing. Report success without running a single privileged step.
217
+ // Nothing is staged here either: a re-run with nothing to install has no
218
+ // reason to touch the staged tree.
219
+ if (plan.currentInstall) {
220
+ ui.notify("The supervised daemon and recovery unit are already current — nothing to install, nothing restarted.", "info");
221
+ return { kind: "installed", wrote: [] };
222
+ }
223
+
132
224
  // argv, not shell: `installCommands` renders `sudo …` strings for humans to
133
225
  // read, and re-parsing those into an argv is how a path with a space becomes
134
- // two arguments. The steps are built from the same values instead.
135
- const steps: PrivilegedStep[] = [
136
- { title: `install ${STAGED_SERVICE_NAME}`, argv: ["install", "-m", "0644", plan.service.path, installed] },
137
- { title: "reload systemd", argv: ["systemctl", "daemon-reload"] },
138
- { title: `enable ${STAGED_SERVICE_NAME}`, argv: ["systemctl", "enable", STAGED_SERVICE_NAME] },
139
- { title: `restart ${STAGED_SERVICE_NAME}`, argv: ["systemctl", "restart", STAGED_SERVICE_NAME] },
140
- // The herdr session server, provisioned alongside the daemon (#456). Only
141
- // when herdr is installed and the plan therefore staged a unit.
142
- ...(plan.herdrUnit === undefined
143
- ? []
144
- : [
145
- { title: `install ${DEFAULT_HERDR_UNIT}`, argv: ["install", "-m", "0644", plan.herdrUnit.path, installedHerdr] },
146
- { title: "reload systemd", argv: ["systemctl", "daemon-reload"] },
147
- { title: `enable ${DEFAULT_HERDR_UNIT}`, argv: ["systemctl", "enable", DEFAULT_HERDR_UNIT] },
148
- { title: `restart ${DEFAULT_HERDR_UNIT}`, argv: ["systemctl", "restart", DEFAULT_HERDR_UNIT] },
149
- ]),
150
- ];
151
-
152
- const outcome = await runPrivileged(steps, ui, {
226
+ // two arguments. The plan carries the exact argv once, in
227
+ // {@link HostRuntimePlan.steps}; we execute that same list, so the printed
228
+ // plan and the run are the same steps (#509).
229
+ //
230
+ // The fence `upgrade` already has (#512): pause new issue claims, drain live
231
+ // workers to idle, restart the units, then restore the dispatch state this
232
+ // began in. The consent prompt below names how many workers will be drained
233
+ // so the operator can choose the moment; it authorises the whole batch,
234
+ // drain included, in one confirm (the `beforeRun` hook).
235
+ const drainDeps = hostInstallDrainDeps(ui, deps);
236
+ // Host-wide by default (#389): the daemon this restarts serves every
237
+ // configured project, so the drain covers all of them — narrowing to the
238
+ // named project would restart the shared session host under the other
239
+ // projects' live workers without ever counting them.
240
+ const fenceScope = resolveScope(drainDeps, "setup host");
241
+ const liveWorkers = fenceScope.selectors.reduce((n, s) => n + drainDeps.snapshot(s).liveWorkers, 0);
242
+ let fence: { scope: UpgradeScope; initialPaused: boolean } | undefined;
243
+ // What the post-consent staging actually wrote, surfaced by the outcomes that
244
+ // report it. Empty on a decline — the hook never ran.
245
+ let wrote: readonly string[] = [];
246
+ const beforeRun = async (): Promise<void> => {
247
+ // Consent has been given; the plan's files may now land. Staging here —
248
+ // post-confirm, pre-step, under the same confirm that authorised the batch
249
+ // — is what makes the prompt's "Nothing has been run yet" true of the
250
+ // staged tree when it is printed: a declined prompt never reaches this
251
+ // hook, so nothing is staged and a re-run still plans an install (#510).
252
+ const staged = writeHostRuntime(plan);
253
+ wrote = staged.wrote;
254
+ for (const warning of staged.warnings) ui.notify(warning, "warning");
255
+ fence = await pauseAndDrain(drainDeps, "setup host", {
256
+ ...(deps.drain?.timeoutMs === undefined ? {} : { timeoutMs: deps.drain.timeoutMs }),
257
+ });
258
+ // Any run still live when the session host is about to restart is killed by
259
+ // it; record the actor on its row so the classifier reads an administrative
260
+ // kill rather than charging a failure for an operator's restart (#512). The
261
+ // fence drained to idle, so in the healthy path this records nothing.
262
+ attributeRestartKills(fence.scope, project?.name ?? "", deps.drain?.drainStore);
263
+ };
264
+
265
+ const outcome = await runPrivileged(plan.steps, ui, {
153
266
  ...(deps.privileged === undefined ? {} : { deps: deps.privileged }),
154
267
  title: "Install and start the supervised session?",
155
268
  preamble: [
@@ -159,18 +272,98 @@ export async function runHostInstall(
159
272
  : [
160
273
  `Also installs the fleet session unit as ${installedHerdr} with the account's login shell so panes`,
161
274
  "run it instead of dash, then enables and restarts it.",
275
+ ...(plan.herdrConfigTarget === undefined
276
+ ? []
277
+ : [
278
+ `And merges [terminal] default_shell into ${plan.herdrConfigTarget} as you, before that restart,`,
279
+ "so the session server reads the pinned shell the moment it comes up.",
280
+ ]),
162
281
  ]),
163
282
  "The unit runs as the account that staged it; nothing here changes that.",
283
+ "",
284
+ liveWorkers === 0
285
+ ? "No workers are live — nothing to drain before the restart."
286
+ : `Pauses new claims and drains ${liveWorkers} live worker(s) to idle first — this can take minutes per worker — then restarts the units and resumes dispatch.`,
164
287
  ],
288
+ beforeRun,
165
289
  });
166
290
  if (outcome.kind === "declined") return { kind: "declined", wrote };
167
291
  if (outcome.kind === "failed") {
292
+ // Restore the preceding dispatch state: a restart that failed is still a
293
+ // finished operator action whose units may or may not be up, so the fleet
294
+ // must not sit paused for it. A throw from the drain itself (a timeout or
295
+ // a superseded fence) never reaches here — it propagates and leaves
296
+ // dispatch paused, fail-closed, exactly as upgrade does.
297
+ if (fence !== undefined && !fence.initialPaused) drainDeps.setPaused(false, fence.scope.pauseKey);
168
298
  return { kind: "failed", reason: `${outcome.step.title} exited ${outcome.exitCode}` };
169
299
  }
300
+ if (fence !== undefined && !fence.initialPaused) drainDeps.setPaused(false, fence.scope.pauseKey);
170
301
  ui.notify(`Installed and started ${STAGED_SERVICE_NAME}.`, "info");
171
302
  return { kind: "installed", wrote };
172
303
  }
173
304
 
305
+ /**
306
+ * The pause/drain fence's default accessors, overridable per test through
307
+ * {@link InstallDeps.drain.deps}. Mirrors `upgrade.ts`'s {@link DrainDeps}
308
+ * wiring, except the pause names `setup host` as the actor and the drain
309
+ * reports progress through the same UI surface.
310
+ */
311
+ function hostInstallDrainDeps(ui: WizardUi, deps: InstallDeps): DrainDeps {
312
+ if (deps.drain?.deps !== undefined) return deps.drain.deps;
313
+ return {
314
+ snapshot: statusSnapshot,
315
+ layers: fleetLayers,
316
+ projectNames: () => loadConfig().projects.map((p) => p.name),
317
+ daemonIdentity: () => {
318
+ const daemon = livingDaemon();
319
+ return daemon === undefined
320
+ ? { running: false }
321
+ : { running: true, project: daemon.project, generation: `${daemon.pid}@${daemon.startedAt}` };
322
+ },
323
+ pauseState: (project) => pauseInstance(project),
324
+ setPaused: (v, project) => setPaused(v, { source: "setup host", reason: "setup host, draining" }, project),
325
+ sleep: Bun.sleep,
326
+ log: (message) => ui.notify(message, "info"),
327
+ };
328
+ }
329
+
330
+ /**
331
+ * Attribute the runs the restart is about to kill: any run still live when the
332
+ * session host restarts underneath it is that restart's casualty, not a failed
333
+ * attempt, so its row records the actor. The classifier reads this as an
334
+ * administrative kill that costs no budget and requeues (#512). The fence
335
+ * drains to idle first, so in the healthy path this records nothing — it is
336
+ * the safety net for a claim that raced the drain or a worker the drain could
337
+ * not bring to idle.
338
+ */
339
+ function attributeRestartKills(
340
+ scope: UpgradeScope,
341
+ project: string,
342
+ store: Pick<Store, "liveRuns" | "updateRun"> | undefined,
343
+ ): void {
344
+ const attribute = (owner: Pick<Store, "liveRuns" | "updateRun">): void => {
345
+ for (const selector of scope.selectors) {
346
+ for (const run of owner.liveRuns(selector ?? project)) {
347
+ owner.updateRun(run.id, { lastError: "admin restart: setup host (draining)" });
348
+ }
349
+ }
350
+ };
351
+ if (store !== undefined) {
352
+ attribute(store);
353
+ return;
354
+ }
355
+ const path = dbPath();
356
+ // No store yet — a host that has never dispatched — means nothing to attribute
357
+ // and nothing a fresh open would do but create an empty database.
358
+ if (!existsSync(path)) return;
359
+ const owner = openStore(path);
360
+ try {
361
+ attribute(owner);
362
+ } finally {
363
+ owner.close();
364
+ }
365
+ }
366
+
174
367
  /** Fleet-wide worker sum when config is loadable; otherwise this project's caps. */
175
368
  function hostInstallWorkers(caps: Caps): number {
176
369
  try {
@@ -180,6 +373,22 @@ function hostInstallWorkers(caps: Caps): number {
180
373
  }
181
374
  }
182
375
 
376
+ /**
377
+ * Whether this host configures more than one project. The recovery unit is
378
+ * host-global and must not encode one project's name when several are
379
+ * configured, so the plan scopes RECOVER_PROJECT by this (#510/#530). A
380
+ * loadable config is the truth; when it cannot be read (a not-yet-configured
381
+ * box), there is no evidence of several projects and the named case stays
382
+ * unambiguous — the same fallback shape as {@link hostInstallWorkers}.
383
+ */
384
+ function hostMultiProject(): boolean {
385
+ try {
386
+ return loadConfig().projects.length > 1;
387
+ } catch {
388
+ return false;
389
+ }
390
+ }
391
+
183
392
  export interface GraphInstallOptions extends InstallDeps {
184
393
  /**
185
394
  * Skip the seeding step. The timer is still installed and enabled, and the
@@ -445,6 +445,27 @@ async function askPolicyPreconditions(
445
445
  return { merge, release };
446
446
  }
447
447
 
448
+ /**
449
+ * The release questions a delegated release authority unlocks, in the order the
450
+ * interview asks them after the grant: one tool gate per release shape, then
451
+ * the release-policy conditions.
452
+ *
453
+ * The authority preview counts and names them from this list — and from
454
+ * `RELEASE_SHAPES` directly — so the prompt stays honest with the questions
455
+ * that follow: a shape added to `RELEASE_SHAPES` or a condition added here moves
456
+ * the number the preview quotes at the same time the later questions appear. A
457
+ * hard-coded count is the drift this exists to prevent.
458
+ */
459
+ export function releaseUnlockPlan(): { gates: readonly string[]; policy: readonly string[] } {
460
+ return {
461
+ // `RELEASE_SHAPES`, not a copy that can rot: these are exactly the per-shape
462
+ // gates `askReleaseGrants` asks (seven, not five (#500)).
463
+ gates: [...RELEASE_SHAPES],
464
+ // The release half of `askPolicyPreconditions`, in the order it asks them.
465
+ policy: ["requirements", "required checks", "artefacts", "environments"],
466
+ };
467
+ }
468
+
448
469
  /**
449
470
  * Who merges, who releases and who promotes. Three confirms rather than one
450
471
  * four-way list: these are independent grants — delegating merges is routine,
@@ -467,12 +488,16 @@ async function askAuthority(
467
488
  "re-checked against the base branch first. Default: humans merge" +
468
489
  `${prior.merge === "orchestrator" ? " — currently delegated, answer no to take it back" : ""}.`,
469
490
  );
491
+ const unlock = releaseUnlockPlan();
492
+ const unlockCount = unlock.gates.length + unlock.policy.length;
470
493
  const release = await askYesNo(
471
494
  ui,
472
495
  "Release authority",
473
496
  "Delegate release cutting to the orchestrator session? It would tag, pin and publish by the " +
474
497
  "procedure you write into its brief — and its brief forbids cutting one before you have. " +
475
- "Default: humans release" +
498
+ `Delegating unlocks ${unlockCount} later decisions: the ${unlock.gates.length} release-tool ` +
499
+ `gates (${unlock.gates.join(", ")}) and the release-policy conditions (${unlock.policy.join(", ")}). ` +
500
+ "Declining means humans release and none are asked. Default: humans release" +
476
501
  `${prior.release === "orchestrator" ? " — currently delegated, answer no to take it back" : ""}.`,
477
502
  );
478
503
  const promotion = await askYesNo(
@@ -1479,6 +1504,32 @@ const AREA_ASKERS: { readonly [K in AmendAreaId]: AreaAsker } = {
1479
1504
  await askBrief(ui, { ...a, judgment: await askJudgment(ui, a.releaseGrants, a.judgment ?? {}) }, probes),
1480
1505
  };
1481
1506
 
1507
+ /**
1508
+ * The full interview's areas, in the exact order `collectAnswers` walks them.
1509
+ *
1510
+ * This array is the single source of both the questions and their progress
1511
+ * marker: the "m" in every `Area n of m` line is this list's `.length`, never a
1512
+ * separate constant, so an area a branch predicate prunes takes its row and its
1513
+ * slot in the denominator with it — the count cannot drift from the iterable.
1514
+ *
1515
+ * The order is load-bearing and each step documents its own rationale: the
1516
+ * graph sits straight after the repos because the answer is derived per repo;
1517
+ * the worker model sits after the policy block because the boundary is an
1518
+ * authority-shaped decision, not a runtime knob; the brief is last because its
1519
+ * write question needs the real path the rest of the answers derive.
1520
+ */
1521
+ const INTERVIEW_AREAS: readonly { label: string; asker: AreaAsker }[] = [
1522
+ { label: "tracker & repos", asker: askTrackerAndRepos },
1523
+ { label: "code graph", asker: askGraph },
1524
+ { label: "caps", asker: askCaps },
1525
+ { label: "authority & release grants", asker: askAuthorityArea },
1526
+ { label: "merge & release preconditions", asker: askPolicy },
1527
+ { label: "worker model", asker: askWorkerModel },
1528
+ { label: "escalation & triage", asker: askEscalation },
1529
+ { label: "reporting", asker: askReporting },
1530
+ { label: "orchestrator brief", asker: askBrief },
1531
+ ];
1532
+
1482
1533
  /** The two ways to answer the first question a configured project gets. Labels,
1483
1534
  * because the harness's select resolves to the label it displayed. */
1484
1535
  const AMEND_ONE = "Change one area";
@@ -1594,19 +1645,14 @@ async function collectAnswers(
1594
1645
  mirrorRoot: scoped.mirrorRoot,
1595
1646
  };
1596
1647
  }
1597
- a = await askTrackerAndRepos(ui, a, probes);
1598
- // Straight after the repos, because it is a fact about them: one clone per
1599
- // routed repo, under one root.
1600
- a = await askGraph(ui, a, probes);
1601
- a = await askCaps(ui, a, probes);
1602
- a = await askAuthorityArea(ui, a, probes, discovered);
1603
- a = await askPolicy(ui, a, probes, discovered);
1604
- a = await askWorkerModel(ui, a, probes);
1605
- a = await askEscalation(ui, a, probes);
1606
- a = await askReporting(ui, a, probes);
1607
- // Asked last, and asked with the real path in the question — which needs the
1608
- // rest of the answers to derive.
1609
- return await askBrief(ui, a, probes);
1648
+ // Walk the ordered area list once; the marker's denominator is the list's
1649
+ // length, so an area a branch prunes drops its row and its slot together.
1650
+ const areas = INTERVIEW_AREAS;
1651
+ for (const [index, area] of areas.entries()) {
1652
+ ui.notify(`Area ${index + 1} of ${areas.length}: ${area.label}`, "info");
1653
+ a = await area.asker(ui, a, probes, discovered);
1654
+ }
1655
+ return a;
1610
1656
  }
1611
1657
 
1612
1658
  /** The dry run, rendered. Same routing code the loop uses, so this is what the
@@ -2002,11 +2048,13 @@ export async function setup(
2002
2048
  : runtime.tick.action === "keep"
2003
2049
  ? `Keeps the external heartbeat config at ${runtime.tick.path}.`
2004
2050
  : `${runtime.tick.action === "create" ? "Creates" : "Updates"} the external heartbeat config at ${runtime.tick.path}.`,
2005
- runtime.briefLink.action === "skip"
2006
- ? `Leaves the existing file at ${runtime.briefLink.path} alone (an operator's file, not overwritten).`
2007
- : runtime.briefLink.action === "keep"
2008
- ? `Keeps the brief link at ${runtime.briefLink.path}.`
2009
- : `${runtime.briefLink.action === "create" ? "Creates" : "Replaces"} the brief link at ${runtime.briefLink.path} -> ${runtime.briefLink.target}.`,
2051
+ runtime.briefLink === undefined
2052
+ ? ""
2053
+ : runtime.briefLink.action === "skip"
2054
+ ? `Leaves the existing file at ${runtime.briefLink.path} alone (an operator's file, not overwritten).`
2055
+ : runtime.briefLink.action === "keep"
2056
+ ? `Keeps the brief link at ${runtime.briefLink.path}.`
2057
+ : `${runtime.briefLink.action === "create" ? "Creates" : "Replaces"} the brief link at ${runtime.briefLink.path} -> ${runtime.briefLink.target}.`,
2010
2058
  answers.writeOrchestratorBrief
2011
2059
  ? `Writes ${orchestratorBriefPath(answers)}, which is then yours to edit.`
2012
2060
  : "",
package/src/setup.ts CHANGED
@@ -155,6 +155,14 @@ export interface SetupAnswers {
155
155
  * survive an amend of some other area (#369).
156
156
  */
157
157
  groomBelow?: number;
158
+ /**
159
+ * Ordered fallback worker models and the provider-failure threshold that
160
+ * starts them. The wizard never asks for either — they are hand-edited —
161
+ * so they exist here only to survive an amend of some other area (#369,
162
+ * #286).
163
+ */
164
+ modelFallbacks?: string[];
165
+ modelFallbackThreshold?: number;
158
166
  telegramChatId?: string;
159
167
  /** Forum topic for tier-2 Telegram pages; absent keeps flat-chat 0.13 behaviour. */
160
168
  telegramTopicId?: number;
@@ -193,6 +201,13 @@ export interface SetupAnswers {
193
201
  * merge provenance gate.
194
202
  */
195
203
  recoveryMerges?: ProjectConfig["recoveryMerges"];
204
+ /**
205
+ * Hand-edited critical-base/safety markers (commit SHAs or refs) carried
206
+ * through setup unchanged, like {@link recoveryMerges}. The wizard never
207
+ * invents one; forgetting them during an unrelated amend would drop the
208
+ * stale-base interlock from a project that relies on it (#428).
209
+ */
210
+ criticalBase?: ProjectConfig["criticalBase"];
196
211
  /**
197
212
  * Whether to render `ORCHESTRATOR.md` into the project's workspace root. Not
198
213
  * part of the config — the brief is the operator's file, and the conductor
@@ -836,6 +851,10 @@ export function buildProject(a: SetupAnswers): ProjectConfig {
836
851
  : {}),
837
852
  // A documented hand-edited key, so an unrelated amend must not delete it.
838
853
  ...(a.groomBelow === undefined ? {} : { groomBelow: a.groomBelow }),
854
+ // The failover chain is hand-edited too: dropping it on an unrelated amend
855
+ // would pin every run of the project onto a dead provider again (#286).
856
+ ...(a.modelFallbacks === undefined ? {} : { modelFallbacks: [...a.modelFallbacks] }),
857
+ ...(a.modelFallbackThreshold === undefined ? {} : { modelFallbackThreshold: a.modelFallbackThreshold }),
839
858
  escalation,
840
859
  authority: { ...a.authority },
841
860
  // Written out in full, never as the legacy string: the file then says which
@@ -847,6 +866,9 @@ export function buildProject(a: SetupAnswers): ProjectConfig {
847
866
  ...(a.recoveryMerges === undefined
848
867
  ? {}
849
868
  : { recoveryMerges: a.recoveryMerges.map((entry) => ({ ...entry })) }),
869
+ // Hand-edited safety markers carried unchanged on an unrelated amend
870
+ // (#428): dropping them would silently disarm the stale-base interlock.
871
+ ...(a.criticalBase === undefined ? {} : { criticalBase: [...a.criticalBase] }),
850
872
  // Written out even when it is the default, so an operator amending the
851
873
  // volume has a line in the file to point at. Opting into availability makes
852
874
  // the schedule explicit and daily; omitting it preserves the preset's
@@ -1037,11 +1059,14 @@ export function answersFromProject(p: ProjectConfig): SetupAnswers {
1037
1059
  // what keeps the rewritten config identical to the one that was read.
1038
1060
  if (p.workerModel !== undefined) answers.workerModel = p.workerModel;
1039
1061
  if (p.groomBelow !== undefined) answers.groomBelow = p.groomBelow;
1062
+ if (p.modelFallbacks !== undefined) answers.modelFallbacks = [...p.modelFallbacks];
1063
+ if (p.modelFallbackThreshold !== undefined) answers.modelFallbackThreshold = p.modelFallbackThreshold;
1040
1064
  if (p.escalation.telegramChatId !== undefined) answers.telegramChatId = p.escalation.telegramChatId;
1041
1065
  if (p.escalation.telegramTopicId !== undefined) answers.telegramTopicId = p.escalation.telegramTopicId;
1042
1066
  if (p.recoveryMerges !== undefined) {
1043
1067
  answers.recoveryMerges = p.recoveryMerges.map((entry) => ({ ...entry }));
1044
1068
  }
1069
+ if (p.criticalBase !== undefined) answers.criticalBase = [...p.criticalBase];
1045
1070
  if (p.reporting?.digest.at !== undefined) answers.dailyDigestAt = p.reporting.digest.at;
1046
1071
  if (p.reporting?.digest.timezone !== undefined) {
1047
1072
  answers.reportingTimezone = p.reporting.digest.timezone;
package/src/store.ts CHANGED
@@ -82,6 +82,7 @@ const ACTIVE_PLACEHOLDERS = ACTIVE_STATES.map(() => "?").join(", ");
82
82
  const FRICTION_HOLD_REASONS: ReadonlySet<FrictionAdmissionReason> = new Set([
83
83
  "failed-attempts",
84
84
  "continuations",
85
+ "stale-base",
85
86
  "parent-lookup-error",
86
87
  "issue-state-lookup-error",
87
88
  "open-pr-lookup-error",
@@ -133,6 +134,7 @@ const UPDATABLE_COLUMNS: Record<string, true> = {
133
134
  failureClass: true,
134
135
  recoveryAction: true,
135
136
  recoveredAt: true,
137
+ model: true,
136
138
  };
137
139
 
138
140
  /** Everything SQLite will accept from us. */
@@ -169,6 +171,7 @@ interface RunRow {
169
171
  failureClass: string | null;
170
172
  recoveryAction: string | null;
171
173
  recoveredAt: number | null;
174
+ model: string | null;
172
175
  }
173
176
 
174
177
  /** The `base_health` table exactly as SQLite hands it back. */
@@ -354,7 +357,8 @@ CREATE TABLE IF NOT EXISTS runs (
354
357
  report TEXT,
355
358
  failureClass TEXT,
356
359
  recoveryAction TEXT,
357
- recoveredAt INTEGER
360
+ recoveredAt INTEGER,
361
+ model TEXT
358
362
  );
359
363
  CREATE INDEX IF NOT EXISTS runs_project_issue ON runs (project, issue);
360
364
  CREATE INDEX IF NOT EXISTS runs_project_state ON runs (project, state);
@@ -707,6 +711,7 @@ function toRecord(row: RunRow): RunRecord {
707
711
  if (row.failureClass !== null) record.failureClass = row.failureClass as FailureClass;
708
712
  if (row.recoveryAction !== null) record.recoveryAction = row.recoveryAction as RecoveryAction;
709
713
  if (row.recoveredAt !== null) record.recoveredAt = row.recoveredAt;
714
+ if (row.model !== null) record.model = row.model;
710
715
  return record;
711
716
  }
712
717
 
@@ -1006,6 +1011,13 @@ export function openStore(dbPath: string): Store {
1006
1011
  db.exec(`ALTER TABLE runs ADD COLUMN ${name} ${type}`);
1007
1012
  }
1008
1013
  }
1014
+ // The model a run dispatched on was first written by the modelFallbacks
1015
+ // failover chain (#286). Rows predating the column are NULL, which is the
1016
+ // honest reading: only a chain-configured project records a model, and the
1017
+ // harness's own downgrade stays on the worker result, never in this column.
1018
+ if (!columns.some((column) => column.name === "model")) {
1019
+ db.exec("ALTER TABLE runs ADD COLUMN model TEXT");
1020
+ }
1009
1021
  // Existing held-notice rows predate exact digest ownership (#274). A NULL
1010
1022
  // report id is truthful for already-digested history and means "still owed"
1011
1023
  // only while digestedAt is also NULL.
@@ -24,6 +24,7 @@
24
24
  import { credentialedEnv } from "../gitops.ts";
25
25
  import { parsePrDiff } from "../diff-flags.ts";
26
26
  import type {
27
+ IssueComment,
27
28
  IssueState,
28
29
  IssueSnapshot,
29
30
  MergedPrInfo,
@@ -760,6 +761,36 @@ export function readyIssuesFromRest(raw: string): ReadyIssue[] {
760
761
  return issues;
761
762
  }
762
763
 
764
+ /**
765
+ * Parse the REST `/repos/{owner}/{repo}/issues/{n}/comments` body — already
766
+ * projected by jq to `[{author, body}]` — onto {@link IssueComment}.
767
+ *
768
+ * The REST endpoint answers chronological order (oldest first), so the array
769
+ * order is the order the brief renders. Nulls and unrecognizable payloads are
770
+ * tolerated the same way {@link readyIssuesFromRest} tolerates them: an empty
771
+ * answer means "no comments", and *only* a thrown read means "could not tell" —
772
+ * the caller distinguishes the two, because eliding them is the #517 failure.
773
+ */
774
+ export function commentsFrom(raw: string): IssueComment[] {
775
+ let parsed: unknown;
776
+ try {
777
+ parsed = JSON.parse(raw) as unknown;
778
+ } catch {
779
+ return [];
780
+ }
781
+ if (!Array.isArray(parsed)) return [];
782
+ const comments: IssueComment[] = [];
783
+ for (const entry of parsed) {
784
+ if (entry === null || typeof entry !== "object") continue;
785
+ const row = entry as { readonly [key: string]: unknown };
786
+ const author = row["author"];
787
+ const body = row["body"];
788
+ if (typeof author !== "string" || typeof body !== "string") continue;
789
+ comments.push({ author, body });
790
+ }
791
+ return comments;
792
+ }
793
+
763
794
  /** One cached page per request URL: the ETag GitHub answered, the raw JSON
764
795
  * body it covered, and whether that page advertised a rel="next" successor.
765
796
  * Module-level default so every makeTracker() in the process (board probe,
@@ -1103,6 +1134,22 @@ export function makeTracker(
1103
1134
  return pages.flatMap(readyIssuesFromRest);
1104
1135
  },
1105
1136
 
1137
+ async listComments(issue: number): Promise<IssueComment[]> {
1138
+ // REST, like `issueSnapshot` — deliberately not `gh issue view
1139
+ // --comments`, whose renderer goes through the interactive pager and
1140
+ // prints nothing on a non-TTY while still exiting 0 (the #517 failure).
1141
+ // `gh api` output is never paged. One page: the brief truncates a huge
1142
+ // thread with an explicit marker rather than paginating.
1143
+ return commentsFrom(
1144
+ await runGh([
1145
+ "api",
1146
+ `repos/${repo}/issues/${issue}/comments`,
1147
+ "--jq",
1148
+ "[.[] | {author: .user.login, body: .body}]",
1149
+ ]),
1150
+ );
1151
+ },
1152
+
1106
1153
  async addLabel(issue: number, label: string): Promise<void> {
1107
1154
  try {
1108
1155
  // REST POST is natively idempotent: re-adding a label the issue already