omp-conductor 0.15.10 → 0.15.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/README.md +273 -2544
  2. package/REFERENCE.md +2680 -0
  3. package/package.json +3 -2
  4. package/schema/config.schema.json +11 -23
  5. package/src/arm-challenge.ts +112 -0
  6. package/src/ask.ts +434 -0
  7. package/src/board.ts +81 -15
  8. package/src/brief-upgrade.ts +114 -8
  9. package/src/briefs/orchestrator.md +113 -34
  10. package/src/briefs/policy.md +33 -8
  11. package/src/briefs/worker.md +18 -9
  12. package/src/chain-check.ts +1 -1
  13. package/src/check-trailing-newlines.ts +82 -0
  14. package/src/cli.ts +225 -1406
  15. package/src/commands/arm.ts +21 -0
  16. package/src/commands/board.ts +23 -0
  17. package/src/commands/brief-upgrade.ts +186 -0
  18. package/src/commands/context.ts +150 -0
  19. package/src/commands/daemon.ts +71 -0
  20. package/src/commands/dashboard.ts +74 -0
  21. package/src/commands/decision.ts +103 -0
  22. package/src/commands/disarm.ts +21 -0
  23. package/src/commands/doctor.ts +100 -0
  24. package/src/commands/event.ts +62 -0
  25. package/src/commands/extend.ts +64 -0
  26. package/src/commands/friction.ts +56 -0
  27. package/src/commands/help.ts +9 -0
  28. package/src/commands/hold.ts +26 -0
  29. package/src/commands/intake.ts +155 -0
  30. package/src/commands/ledger.ts +69 -0
  31. package/src/commands/message.ts +96 -0
  32. package/src/commands/report.ts +206 -0
  33. package/src/commands/restart.ts +76 -0
  34. package/src/commands/resume.ts +58 -0
  35. package/src/commands/setup.ts +143 -0
  36. package/src/commands/start.ts +23 -0
  37. package/src/commands/stats.ts +131 -0
  38. package/src/commands/status.ts +48 -0
  39. package/src/commands/stop.ts +51 -0
  40. package/src/commands/tail.ts +109 -0
  41. package/src/commands/unblock.ts +39 -0
  42. package/src/commands/upgrade-install.ts +31 -0
  43. package/src/commands/upgrade-rollback.ts +32 -0
  44. package/src/commands/upgrade.ts +25 -0
  45. package/src/commands/verb.ts +83 -0
  46. package/src/commands/version.ts +30 -0
  47. package/src/commands/worker.ts +100 -0
  48. package/src/config-schema.ts +47 -1
  49. package/src/config.ts +45 -4
  50. package/src/daemon.ts +972 -101
  51. package/src/dashboard/app.js +459 -0
  52. package/src/dashboard/index.html +61 -0
  53. package/src/dashboard/server.ts +481 -0
  54. package/src/dashboard/style.css +348 -0
  55. package/src/decisions.ts +39 -14
  56. package/src/diff-flags.ts +131 -241
  57. package/src/doctor.ts +932 -0
  58. package/src/escalate.ts +2 -2
  59. package/src/failure-class.ts +66 -3
  60. package/src/fleet.ts +58 -1
  61. package/src/gitops.ts +157 -0
  62. package/src/graph-health.ts +1 -1
  63. package/src/label-projection.ts +1 -1
  64. package/src/lifecycle.ts +198 -2
  65. package/src/model-fallback.ts +177 -0
  66. package/src/notices.ts +9 -0
  67. package/src/omp.ts +93 -13
  68. package/src/orchestrator-tick.ts +414 -20
  69. package/src/orchestrator.ts +4 -4
  70. package/src/privileged.ts +10 -0
  71. package/src/release-policy.ts +342 -30
  72. package/src/reports.ts +19 -5
  73. package/src/session-host.ts +11 -5
  74. package/src/setup-host.ts +663 -25
  75. package/src/setup-install.ts +292 -28
  76. package/src/setup-wizard.ts +255 -74
  77. package/src/setup.ts +156 -35
  78. package/src/stats.ts +331 -0
  79. package/src/store.ts +219 -22
  80. package/src/tracker/github.ts +74 -4
  81. package/src/types.ts +215 -31
  82. package/src/unblock.ts +55 -11
  83. package/src/upgrade-journal.ts +220 -0
  84. package/src/upgrade-verify.ts +506 -0
  85. package/src/upgrade.ts +385 -58
  86. package/src/verbs/actions.ts +73 -1
  87. package/src/verbs/protocol.ts +29 -4
  88. package/src/verbs/server.ts +183 -20
  89. package/src/worker.ts +3 -3
  90. package/systemd/omp-conductor-recover.sh +433 -0
  91. package/systemd/omp-conductor.service.example +14 -3
  92. package/systemd/recover-unit-test.sh +428 -0
package/src/escalate.ts CHANGED
@@ -20,12 +20,12 @@
20
20
  * those strings end up in daemon logs and, on the fallback path, in a public
21
21
  * issue comment.
22
22
  */
23
- import { createHash } from "node:crypto";
24
23
  import { readFileSync } from "node:fs";
25
24
  import { homedir } from "node:os";
26
25
  import { join } from "node:path";
27
26
 
28
27
  import { availabilityOpen, interruptDisposition, type InterruptDisposition } from "./availability.ts";
28
+ import { heldNoticeId } from "./notices.ts";
29
29
  import type { OrchestratorHandle } from "./orchestrator.ts";
30
30
  import type { Escalation, ProjectConfig, Store, Tracker } from "./types.ts";
31
31
 
@@ -240,7 +240,7 @@ export function createEscalator(
240
240
  }
241
241
  if (disposition !== "interrupt") {
242
242
  store.addHeldNotice({
243
- id: createHash("sha256").update(`held-notice\0${key}`).digest("hex"),
243
+ id: heldNoticeId(key),
244
244
  project: p.name,
245
245
  category,
246
246
  summary: e.summary,
@@ -15,7 +15,7 @@
15
15
  * green test per class" possible without a daemon, a tracker or a network.
16
16
  */
17
17
 
18
- import type { FailureClass, RecoveryAction, RunRecord } from "./types.ts";
18
+ import type { Caps, FailureClass, RecoveryAction, RunRecord } from "./types.ts";
19
19
 
20
20
  /** Facts the caller fetched, each only for the rows that need it. */
21
21
  export interface ClassifyFacts {
@@ -222,7 +222,32 @@ export function dispatchInfra(
222
222
  // front of the thrown message, so a bare /^git / would never match a real row.
223
223
  const DISPATCH_GIT_ERROR = /^(?:Error: )?git .+ exited \d+/s;
224
224
 
225
- export function classifyRun(run: RunRecord, facts: ClassifyFacts): Classification {
225
+ /** The prefix the draining/restarting process writes to a run it killed. */
226
+ const ADMIN_RESTART_MARKER = "admin restart:";
227
+
228
+ /**
229
+ * The actor a draining restart recorded on a run it killed, or `undefined`
230
+ * when the row carries no such attribution. Read from `lastError` for the
231
+ * same reason {@link dispatchInfra} is read from it: it is the one surface a
232
+ * run's own row already persists, and the restarting process writes it the
233
+ * way `recordOperatorStop` writes "operator stopped: …". Recognised as an
234
+ * administrative kill, never a charging failure, because nothing the worker
235
+ * did ended it — #512's gap was that a killed session left no attribution.
236
+ */
237
+ export function adminRestartAttribution(lastError: string | undefined): string | undefined {
238
+ if (lastError === undefined) return undefined;
239
+ const marker = lastError.indexOf(ADMIN_RESTART_MARKER);
240
+ if (marker === -1) return undefined;
241
+ const firstLine = lastError.slice(marker + ADMIN_RESTART_MARKER.length).trim().split("\n")[0];
242
+ if (firstLine === undefined || firstLine.trim() === "") return undefined;
243
+ return firstLine.trim();
244
+ }
245
+
246
+ export function classifyRun(
247
+ run: RunRecord,
248
+ facts: ClassifyFacts,
249
+ caps?: Pick<Caps, "workerWallClockMs">,
250
+ ): Classification {
226
251
  const hasArtifacts = run.prUrl !== undefined || run.headSha !== undefined || run.salvageSha !== undefined;
227
252
  const providerError =
228
253
  facts.sessionError ?? (run.lastError === undefined ? undefined : { message: run.lastError });
@@ -296,6 +321,22 @@ export function classifyRun(run: RunRecord, facts: ClassifyFacts): Classificatio
296
321
  }
297
322
  }
298
323
 
324
+ // A run the draining/restarting process attributed before it restarted the
325
+ // session host: killed under its own ceilings by an administrative action,
326
+ // not by anything the worker did. Costs no budget and requeues, naming the
327
+ // actor (#512). A killed session used to leave no attribution and fell
328
+ // through to `unknown`, charging a failure for an operator's restart.
329
+ if (run.state === "failed" || run.state === "killed") {
330
+ const actor = adminRestartAttribution(run.lastError);
331
+ if (actor !== undefined) {
332
+ return {
333
+ cls: "admin-kill",
334
+ recovery: "requeue",
335
+ evidence: `killed by an administrative restart (${actor}) — an operator's restart, not the run's work`,
336
+ };
337
+ }
338
+ }
339
+
299
340
  if (run.state === "blocked") {
300
341
  return {
301
342
  cls: "question",
@@ -337,7 +378,29 @@ export function classifyRun(run: RunRecord, facts: ClassifyFacts): Classificatio
337
378
  evidence: `turns ${run.turns}/${run.maxTurns}, no PR, no commits — $${run.spendUsd.toFixed(2)} spent`,
338
379
  };
339
380
  }
340
- // Below its own ceiling, so nothing this worker did ended it: a daemon
381
+ // A run killed at the wall-clock cap is a cap kill too, even though its
382
+ // turns ceiling never tripped — the harness would have kept burning a
383
+ // worker slot until the ticks ran out (#490). The same split as the turns
384
+ // path: artifacts mean the next attempt resumes from real work; nothing to
385
+ // show means a human re-scopes, naming the time and spend consumed.
386
+ const wallClockCap = caps?.workerWallClockMs;
387
+ const wallClockElapsed =
388
+ run.endedAt === undefined ? undefined : run.endedAt - run.startedAt;
389
+ if (wallClockCap !== undefined && wallClockElapsed !== undefined && wallClockElapsed >= wallClockCap) {
390
+ const clock = `${Math.round(wallClockElapsed / 60_000)}m of ${Math.round(wallClockCap / 60_000)}m`;
391
+ return hasArtifacts
392
+ ? {
393
+ cls: "wall-clock-cap-progress",
394
+ recovery: "continue",
395
+ evidence: `wall clock ${clock} with work to continue from (${run.prUrl ?? run.salvageSha ?? run.headSha})`,
396
+ }
397
+ : {
398
+ cls: "wall-clock-cap-spinning",
399
+ recovery: "escalate",
400
+ evidence: `wall clock ${clock}, no PR, no commits — $${run.spendUsd.toFixed(2)} spent`,
401
+ };
402
+ }
403
+ // Below its own ceilings, so nothing this worker did ended it: a daemon
341
404
  // restart, a drain, an operator. Costs no budget and needs no human.
342
405
  return {
343
406
  cls: "admin-kill",
package/src/fleet.ts CHANGED
@@ -29,6 +29,7 @@ import { homedir } from "node:os";
29
29
  import { dirname, join } from "node:path";
30
30
  import { formatZonedMinute } from "./availability.ts";
31
31
  import { findProject, loadConfig, stateDir } from "./config.ts";
32
+ import { clearArmChallenge, recordArmChallenge } from "./arm-challenge.ts";
32
33
  import { resolveProjectTopicId, sendTelegram } from "./escalate.ts";
33
34
  import { planUsageLine, readPlanUsage, sharedUsageSource } from "./usage.ts";
34
35
  import { readApprovalSurface, readDaemonProfile } from "./approval-surface.ts";
@@ -250,6 +251,18 @@ function armGate(projectName?: string): { path: string; named?: string } {
250
251
  return named === undefined ? { path } : { path, named };
251
252
  }
252
253
 
254
+ /**
255
+ * This project's current tick-arm verdict — the marker the heartbeat reads,
256
+ * returned with the path that carries it so a verb can report what actually
257
+ * remains. Read-only: the same decision `disarm` takes, so control and status
258
+ * about one fleet cannot disagree (a verb telling the operator to re-arm a
259
+ * fleet that is already armed is the defect #375 removes).
260
+ */
261
+ export function armState(projectName?: string): { path: string; armed: boolean } {
262
+ const { path, named } = armGate(projectName);
263
+ return { path, armed: resolveArmState(path, named).armed };
264
+ }
265
+
253
266
  /**
254
267
  * Clears the arm gate for this project — and `wasArmed` is the gate the
255
268
  * heartbeat reads, not merely one file's presence.
@@ -362,23 +375,34 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
362
375
  }
363
376
 
364
377
  const send = deps.sendChallenge ?? sendTelegramMessage;
378
+ const timeoutMs = deps.timeoutMs ?? ARM_CHALLENGE_TIMEOUT_MS;
365
379
  // Read before the send, not after: a transcript untouched since this instant
366
380
  // cannot contain the reply, and that is what the waiter filters on.
367
381
  const sentAt = (deps.now ?? Date.now)();
382
+ // The orchestrator must be able to classify the reply as an *active* arming
383
+ // proof for as long as this handshake is live, so the authenticated record
384
+ // (hash + expiry, never the code) is written before the challenge goes out
385
+ // and cleared the moment this end settles (#415).
386
+ recordArmChallenge(named, code, sentAt + timeoutMs);
368
387
  try {
369
388
  await send(token, channel.owner, text, topicId);
370
389
  } catch (err) {
390
+ // The challenge never went out, so it must not linger as a classifiable
391
+ // proof either.
392
+ clearArmChallenge(named);
371
393
  throw new Error(
372
394
  `arm: outbound sendMessage failed — NOT armed: ${err instanceof Error ? err.message : String(err)}`,
373
395
  );
374
396
  }
375
397
 
376
- const timeoutMs = deps.timeoutMs ?? ARM_CHALLENGE_TIMEOUT_MS;
377
398
  const injected = deps.waitForUserTurn;
378
399
  const scan: SessionScan = injected
379
400
  ? { seen: await injected(dir, code, sentAt, timeoutMs), scanned: [], ignored: [] }
380
401
  : await waitForChallengeInSessions(dir, code, sentAt, timeoutMs, deps);
381
402
  if (!scan.seen) {
403
+ // The proof missed its window: a lookalike reply must not stay classifiable
404
+ // after arming gave up, so the record is cleared before the failure lands.
405
+ clearArmChallenge(named);
382
406
  const listing = [
383
407
  `session dir: ${dir}`,
384
408
  ...scan.scanned.map((f) => ` watched: ${f}`),
@@ -402,6 +426,10 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
402
426
  // marker that re-arms the fleet, and turn into a meaningless legacy warning
403
427
  // the moment a second project is configured.
404
428
  if (arm.legacy === "honoured") rmSync(legacyArmedMarkerPath(), { force: true });
429
+ // The transcript proof landed and this project is armed: the record has done
430
+ // its job. A stale record would also keep a later unsolicited lookalike alive
431
+ // longer than the fresh challenge it was cut for.
432
+ clearArmChallenge(named);
405
433
  return { path, alreadyArmed, owner: channel.owner, challenge: code };
406
434
  }
407
435
 
@@ -1393,6 +1421,7 @@ export function formatFleetStatus(
1393
1421
  decisions: string | undefined = undefined,
1394
1422
  failureClasses: string | undefined = undefined,
1395
1423
  workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
1424
+ intake: string | undefined = undefined,
1396
1425
  ): string {
1397
1426
  const tickLine =
1398
1427
  layers.ticksDetail === undefined
@@ -1473,6 +1502,7 @@ export function formatFleetStatus(
1473
1502
  ...(brief === undefined ? [] : [brief]),
1474
1503
  ...(decisions === undefined ? [] : [decisions]),
1475
1504
  ...(failureClasses === undefined ? [] : [failureClasses]),
1505
+ ...(intake === undefined ? [] : [intake]),
1476
1506
  ...(graphBlock === undefined ? [] : [graphBlock]),
1477
1507
  daemonBlock,
1478
1508
  "",
@@ -1656,6 +1686,7 @@ export async function renderStatus(projectName?: string): Promise<string> {
1656
1686
  decisionStatusLine(project.name),
1657
1687
  failureClassBlock(project.name),
1658
1688
  workerPhases,
1689
+ intakeStatusLine(project.name),
1659
1690
  );
1660
1691
  }
1661
1692
 
@@ -1734,6 +1765,32 @@ function decisionStatusLine(projectName: string): string | undefined {
1734
1765
  }
1735
1766
  }
1736
1767
 
1768
+ /**
1769
+ * One line naming how many raw ideas are still pending (#299), or nothing when
1770
+ * there are none.
1771
+ *
1772
+ * Deliberately absent at zero, unlike the decision line: an empty intake is the
1773
+ * common state of a healthy fleet, and a row that prints "0 pending" every
1774
+ * status would be permanent noise. Only a nonzero backlog is news, exactly like
1775
+ * {@link failureClassBlock}. Omitted on any throw, like the other optional rows.
1776
+ * Exported so the conditional is testable against a real store.
1777
+ */
1778
+ export function intakeStatusLine(projectName: string): string | undefined {
1779
+ const path = dbPath();
1780
+ if (!existsSync(path)) return undefined;
1781
+ let store: Store | undefined;
1782
+ try {
1783
+ store = openStore(path);
1784
+ const pending = store.pendingIntake(projectName);
1785
+ if (pending.length === 0) return undefined;
1786
+ return `intake ${pending.length} pending idea(s) — omp-conductor intake list`;
1787
+ } catch {
1788
+ return undefined;
1789
+ } finally {
1790
+ store?.close();
1791
+ }
1792
+ }
1793
+
1737
1794
  // ---------------------------------------------------------------------------
1738
1795
  // arm proof helpers
1739
1796
  // ---------------------------------------------------------------------------
package/src/gitops.ts CHANGED
@@ -17,6 +17,7 @@
17
17
  * branch cannot disagree.
18
18
  */
19
19
 
20
+ import { existsSync } from "node:fs";
20
21
  import { join } from "node:path";
21
22
 
22
23
  import { parseChainSource, type ChainEntry } from "./chain-check.ts";
@@ -321,3 +322,159 @@ export async function readBaseChain(
321
322
  return { ok: false, stderr: err instanceof Error ? err.message : String(err) };
322
323
  }
323
324
  }
325
+
326
+ // ------------------------------------------- the shared-host critical-base guard
327
+
328
+ /**
329
+ * The stale-base admission verdict (#428): whether a preserved continuation
330
+ * branch may be reattached at all, once a project marks a base commit as
331
+ * critical. Fail closed — a branch that cannot be *proven* to contain every
332
+ * marker is refused.
333
+ */
334
+ export type CriticalBaseVerdict =
335
+ /** No preserved branch for this issue in the mirror — a clean first attempt. */
336
+ | { state: "no-branch" }
337
+ /** The reattach source contains every configured marker. */
338
+ | { state: "fresh" }
339
+ /** The reattach source predates a marker and cannot be safely advanced to it,
340
+ * so the dispatcher must not reattach it. `range` names the base commits
341
+ * the branch has not absorbed, for the hold's recovery wording. */
342
+ | { state: "stale"; marker: string; range: string[] }
343
+ /** The marker could not be verified at all (unresolvable, fetch failed). */
344
+ | { state: "unknown"; error: string };
345
+
346
+ /** The stubborn-session view of the probe, injectable for deterministic tests. */
347
+ export type CriticalBaseProbe = (
348
+ repo: RepoTarget,
349
+ markers: readonly string[],
350
+ branch: string,
351
+ ) => Promise<CriticalBaseVerdict>;
352
+
353
+ /** The bounded sample of base commits a held branch is missing. */
354
+ const CRITICAL_BASE_RANGE_MAX = 8;
355
+
356
+ /**
357
+ * Answer whether the preserved branch {@link addRunRepo} would reattach
358
+ * contains every configured critical-base marker (#428). A base safety fix
359
+ * protects only branches forked after it landed; a continuation forked before
360
+ * it still carries the dangerous code, and re-running the lifecycle suite it
361
+ * retains on a shared host is what SIGTERMed the production daemon.
362
+ *
363
+ * The reattach source is the mirror's `refs/heads/<branch>`. When it lacks a
364
+ * marker but the live remote branch now carries it and the reattach source is
365
+ * that live head's ancestor, the operator advanced the branch on GitHub (e.g.
366
+ * merged base into the PR branch): the fast-forward is folded into the mirror
367
+ * — safe and work-preserving, the same reconcile `pushRunBranch` performs —
368
+ * and the marker is accepted. Judged against the reattach source, never the
369
+ * live head alone, so a stale local copy cannot smuggle pre-fix code back in.
370
+ */
371
+ export async function probeCriticalBase(
372
+ project: Pick<ProjectConfig, "mirrorRoot">,
373
+ repo: RepoTarget,
374
+ branch: string,
375
+ markers: readonly string[],
376
+ exec: Exec = spawnCaptured,
377
+ ): Promise<CriticalBaseVerdict> {
378
+ const mirror = mirrorPath(project, repo);
379
+ // A mirror is where preserved branches live; without one there is nothing to
380
+ // reattach. Judged "no branch" rather than "unknown" so a fleet that has not
381
+ // yet created a mirror is not deadlocked at admission.
382
+ if (!existsSync(mirror)) return { state: "no-branch" };
383
+ const branchRef = `refs/heads/${branch}`;
384
+ const env = credentialedEnv();
385
+
386
+ const present = await exec(
387
+ ["git", "--git-dir", mirror, "show-ref", "--verify", "--quiet", branchRef],
388
+ { env },
389
+ );
390
+ if (present.code !== 0) return { state: "no-branch" };
391
+
392
+ // The configured marker is a commit that landed on base, so base has to be
393
+ // resolvable to prove anything about the branch against it. Fetch today's
394
+ // base tip into the mirror's tracking refs first, so the judgement is against
395
+ // the current base rather than whatever a run last left behind.
396
+ const baseRef = `refs/remotes/origin/${repo.defaultBranch}`;
397
+ const fetched = await exec(
398
+ ["git", "--git-dir", mirror, "fetch", "--no-tags", "origin", `+refs/heads/${repo.defaultBranch}:${baseRef}`],
399
+ { env },
400
+ );
401
+ if (fetched.code !== 0) {
402
+ return {
403
+ state: "unknown",
404
+ error: scrubUserinfo(fetched.stderr.trim() || fetched.stdout.trim() || `git fetch exited ${String(fetched.code)}`),
405
+ };
406
+ }
407
+
408
+ for (const marker of markers) {
409
+ const resolved = await exec(
410
+ ["git", "--git-dir", mirror, "rev-parse", "--verify", `${marker}^{commit}`],
411
+ { env },
412
+ );
413
+ if (resolved.code !== 0) {
414
+ return {
415
+ state: "unknown",
416
+ error: `critical-base marker "${marker}" is not resolvable in the ${repo.name} mirror`,
417
+ };
418
+ }
419
+ const markerSha = resolved.stdout.trim();
420
+ if (markerSha === "") {
421
+ return { state: "unknown", error: `critical-base marker "${marker}" resolved to no commit` };
422
+ }
423
+
424
+ const localHas = await exec(
425
+ ["git", "--git-dir", mirror, "merge-base", "--is-ancestor", markerSha, branchRef],
426
+ { env },
427
+ );
428
+ if (localHas.code === 0) continue;
429
+
430
+ // The reattach source lacks the marker. If the live branch now carries it
431
+ // and the reattach source is its ancestor, the operator advanced the branch
432
+ // on GitHub (e.g. merged base into the PR branch): fold that fast-forward
433
+ // into the reattach source — safe, work-preserving — and accept the marker.
434
+ const tracked = `refs/remotes/origin/${branch}`;
435
+ const liveFetched = await exec(
436
+ ["git", "--git-dir", mirror, "fetch", "--no-tags", "origin", `+refs/heads/${branch}:${tracked}`],
437
+ { env },
438
+ );
439
+ if (liveFetched.code !== 0) {
440
+ return {
441
+ state: "unknown",
442
+ error: scrubUserinfo(liveFetched.stderr.trim() || liveFetched.stdout.trim() || `git fetch ${branch} exited ${String(liveFetched.code)}`),
443
+ };
444
+ }
445
+ const liveHas = await exec(
446
+ ["git", "--git-dir", mirror, "merge-base", "--is-ancestor", markerSha, tracked],
447
+ { env },
448
+ );
449
+ if (liveHas.code === 0) {
450
+ const localIsAncestor = await exec(
451
+ ["git", "--git-dir", mirror, "merge-base", "--is-ancestor", branchRef, tracked],
452
+ { env },
453
+ );
454
+ if (localIsAncestor.code === 0) {
455
+ await exec(["git", "--git-dir", mirror, "fetch", "--no-tags", "origin", `+${branchRef}:${branchRef}`], { env });
456
+ continue;
457
+ }
458
+ }
459
+
460
+ // Predates the marker and cannot be safely advanced to it. Name the base
461
+ // commits the branch has not absorbed so the hold's recovery is concrete.
462
+ const range: string[] = [];
463
+ const missing = await exec(
464
+ ["git", "--git-dir", mirror, "log", "--oneline", "--format=%h", `${branchRef}..${baseRef}`],
465
+ { env },
466
+ );
467
+ if (missing.code === 0) {
468
+ range.push(
469
+ ...missing.stdout
470
+ .trim()
471
+ .split("\n")
472
+ .filter((line) => line.length > 0)
473
+ .slice(0, CRITICAL_BASE_RANGE_MAX),
474
+ );
475
+ }
476
+ return { state: "stale", marker: markerSha, range };
477
+ }
478
+
479
+ return { state: "fresh" };
480
+ }
@@ -264,7 +264,7 @@ export async function probeCodeGraph(
264
264
  const repos: CodeGraphRepoHealth[] = configured.map((repo) => {
265
265
  const clone: CheckState = deps.exists(repo.graphProject) ? "present" : "missing";
266
266
  const index: CheckState = roots === undefined ? "unknown" : roots.has(repo.graphProject) ? "present" : "missing";
267
- if (clone === "missing") reasons.push(`${repo.name}: configured clone is missing`);
267
+ if (clone === "missing") reasons.push(`${repo.name}: no code-graph clone at ${repo.graphProject}; run \`omp-conductor setup graph\``);
268
268
  if (index === "missing") reasons.push(`${repo.name}: no indexed project exactly matches ${repo.graphProject}`);
269
269
  return { name: repo.name, path: repo.graphProject, clone, index };
270
270
  });
@@ -90,4 +90,4 @@ export async function projectLabels(
90
90
  if (!progressed) break;
91
91
  }
92
92
  return { applied, deferred };
93
- }
93
+ }
package/src/lifecycle.ts CHANGED
@@ -66,6 +66,18 @@ const LOG_TAIL_BYTES = 4_096;
66
66
  export function runtimeDir(): string {
67
67
  const override = process.env["OMP_CONDUCTOR_RUNTIME_DIR"];
68
68
  if (override !== undefined && override.length > 0) return override;
69
+ if (process.env["NODE_ENV"] === "test") {
70
+ // A test process must never read the host-global runtime dir: doing so
71
+ // names the real installed daemon's pid, and a test `stop` that inherits
72
+ // that record would signal — and stop — production's own unit (#399).
73
+ // Fail closed: a lifecycle test that omitted OMP_CONDUCTOR_RUNTIME_DIR
74
+ // (the runtime half of the test lifecycle seam) gets a loud error rather
75
+ // than a silent read of the operator's pidfile.
76
+ throw new Error(
77
+ "OMP_CONDUCTOR_RUNTIME_DIR must be set under NODE_ENV=test — " +
78
+ "refusing to read the host runtime directory (~/.omp/run/daemons/omp-conductor) from a test process",
79
+ );
80
+ }
69
81
  return join(homedir(), ".omp", "run", "daemons", "omp-conductor");
70
82
  }
71
83
 
@@ -178,6 +190,152 @@ export function livingDaemon(): DaemonRecord | undefined {
178
190
  return undefined;
179
191
  }
180
192
 
193
+ /**
194
+ * The `daemon --once` lease file, one per process. Created `O_EXCL` so two
195
+ * simultaneous `--once` runs cannot both win ownership, and it names the
196
+ * holder's pid so a stale file (holder died) is removable rather than a
197
+ * permanent lock.
198
+ *
199
+ * The pidfile (`daemon.json`) is what makes this process *visible* as the owner
200
+ * for the duration of the tick (`status`, `worker stop`, a concurrent `daemon`
201
+ * start, orphan reconciliation all probe it); the lease file is what makes
202
+ * ownership *exclusive* — it is the thing that lets two simultaneous `--once`
203
+ * processes serialise, which the pidfile alone cannot (each would just
204
+ * overwrite it). They are deliberately two files: the lease is never the
205
+ * daemon record, so a short-lived drill can never impersonate or clear the
206
+ * managed daemon's record by virtue of holding a lock.
207
+ */
208
+ function onceLeasePath(): string {
209
+ return join(runtimeDir(), "once.lock");
210
+ }
211
+
212
+ interface OnceLeaseHolder {
213
+ pid: number;
214
+ startedAt: number;
215
+ }
216
+
217
+ /** Parses the lease file; garbage is the same answer as absence — "not held". */
218
+ function readOnceLeaseHolder(): OnceLeaseHolder | undefined {
219
+ let parsed: unknown;
220
+ try {
221
+ parsed = JSON.parse(readFileSync(onceLeasePath(), "utf8"));
222
+ } catch {
223
+ return undefined;
224
+ }
225
+ if (parsed === null || typeof parsed !== "object") return undefined;
226
+ const pid = Reflect.get(parsed, "pid");
227
+ const startedAt = Reflect.get(parsed, "startedAt");
228
+ if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 1) return undefined;
229
+ if (typeof startedAt !== "number" || !Number.isFinite(startedAt)) return undefined;
230
+ return { pid, startedAt };
231
+ }
232
+
233
+ /** Bounded attempts to clear a stale lease and re-acquire it. */
234
+ const ONCE_LEASE_RETRIES = 3;
235
+
236
+ /** A once tick has no log file of its own; the record's field is informational. */
237
+ const ONCE_RECORD_LOG = "<once tick — no daemon log>";
238
+
239
+ export interface OnceLease {
240
+ /** The owned record that makes this process the visible daemon for the tick. */
241
+ record: DaemonRecord;
242
+ /**
243
+ * Drops exclusivity — but only while it is still ours. A manager that
244
+ * claimed the record mid-tick must be left alone: clearing it would clear
245
+ * somebody else's daemon.
246
+ */
247
+ release(): void;
248
+ }
249
+
250
+ export type OnceLeaseAcquire = { ok: true; lease: OnceLease } | { ok: false; pid: number };
251
+
252
+ /**
253
+ * Acquires the exclusive daemon/state ownership a `--once` tick must hold for
254
+ * its whole lifecycle (#431). A drill is still a dispatcher — it settles rows,
255
+ * projects labels, admits and launches workers — so it takes the same
256
+ * ownership the long-running dispatcher holds via the pidfile, and refuses
257
+ * (having done nothing) while a live daemon owns the record or another live
258
+ * `--once` holds the lease.
259
+ *
260
+ * Returns `{ ok: false, pid }` — having performed no mutation — when another
261
+ * live process owns the project/state. A stale lease and a stale record whose
262
+ * holder died are cleared and retried. On success the process both holds the
263
+ * exclusive lease file and has published itself as the daemon owner, so every
264
+ * ownership probe (`livingDaemon`, a `daemon` start, a second `--once`) sees
265
+ * this process for the whole tick.
266
+ */
267
+ export function acquireOnceLease(
268
+ o: { port?: number; project?: string; logFile?: string } = {},
269
+ ): OnceLeaseAcquire {
270
+ // A live dispatcher already owns the shared record. Never run beside it.
271
+ const daemon = livingDaemon();
272
+ if (daemon !== undefined && daemon.pid !== process.pid) {
273
+ return { ok: false, pid: daemon.pid };
274
+ }
275
+
276
+ const dir = ensureRuntimeDir();
277
+ const path = onceLeasePath();
278
+
279
+ for (let attempt = 0; attempt < ONCE_LEASE_RETRIES; attempt++) {
280
+ let created = false;
281
+ try {
282
+ const fd = openSync(path, "wx", 0o600);
283
+ writeFileSync(fd, `${JSON.stringify({ pid: process.pid, startedAt: Date.now() })}\n`);
284
+ closeSync(fd);
285
+ created = true;
286
+ } catch (err) {
287
+ if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
288
+ const holder = readOnceLeaseHolder();
289
+ if (holder !== undefined && isAlive(holder.pid)) {
290
+ // A live holder — another `--once` (or a process still finishing one).
291
+ return { ok: false, pid: holder.pid };
292
+ }
293
+ // Stale or unreadable holder: clear it and try once more.
294
+ rmSync(path, { force: true });
295
+ continue;
296
+ }
297
+ if (!created) continue;
298
+
299
+ const record: DaemonRecord = {
300
+ pid: process.pid,
301
+ port: o.port ?? DEFAULT_PORT,
302
+ startedAt: Date.now(),
303
+ logFile: o.logFile ?? ONCE_RECORD_LOG,
304
+ ...(o.project === undefined ? {} : { project: o.project }),
305
+ };
306
+ try {
307
+ writeRecord(record);
308
+ } catch (err) {
309
+ rmSync(path, { force: true });
310
+ throw err;
311
+ }
312
+
313
+ // Confirm we still own the shared record: a manager may have claimed it in
314
+ // the instant between our lease create and our record write.
315
+ const current = readRecord();
316
+ if (current !== undefined && current.pid === process.pid) {
317
+ return {
318
+ ok: true,
319
+ lease: {
320
+ record,
321
+ release: () => {
322
+ const owned = readRecord();
323
+ if (owned !== undefined && owned.pid === process.pid) clearRecord();
324
+ const holder = readOnceLeaseHolder();
325
+ if (holder !== undefined && holder.pid === process.pid) {
326
+ rmSync(path, { force: true });
327
+ }
328
+ },
329
+ },
330
+ };
331
+ }
332
+ rmSync(path, { force: true });
333
+ return { ok: false, pid: current?.pid ?? process.pid };
334
+ }
335
+
336
+ throw new Error("could not acquire the daemon lease — a stale holder kept reappearing");
337
+ }
338
+
181
339
  /**
182
340
  * Probes the daemon's own health endpoint. Never throws: a refused connection,
183
341
  * a DNS-less host, a hung socket and a 500 are all just "not healthy", and the
@@ -892,8 +1050,17 @@ async function terminate(pid: number, timeoutMs: number): Promise<boolean> {
892
1050
  return !isAlive(pid);
893
1051
  }
894
1052
 
895
- /** Sends a signal, treating "already gone" as success. Returns false if the send failed. */
896
- function signal(pid: number, sig: NodeJS.Signals): boolean {
1053
+ /** Whether a raw signal should be delivered to a pid. */
1054
+ export type SignalFn = (pid: number, sig: NodeJS.Signals) => boolean;
1055
+
1056
+ /**
1057
+ * The default signal sender. Production reaches the real `kill`. Under
1058
+ * NODE_ENV=test it dispatches to the fail-closed refusal instead: a test
1059
+ * process must never signal a pid it did not explicitly authorise, and the
1060
+ * default authorises nothing (#399).
1061
+ */
1062
+ function defaultSignal(pid: number, sig: NodeJS.Signals): boolean {
1063
+ if (process.env["NODE_ENV"] === "test") return testSignal(pid, sig);
897
1064
  try {
898
1065
  process.kill(pid, sig);
899
1066
  return true;
@@ -902,6 +1069,35 @@ function signal(pid: number, sig: NodeJS.Signals): boolean {
902
1069
  }
903
1070
  }
904
1071
 
1072
+ /**
1073
+ * Child-process CLI tests cannot inject {@link setSignalForTest}. Under
1074
+ * NODE_ENV=test they therefore get a refusal: a test process may only signal
1075
+ * a pid its own code explicitly authorised, and a child has authorised none.
1076
+ * Without this, a child `stop` that reads the operator's real daemon.json
1077
+ * (an *inherited* runtime record) would SIGTERM the installed daemon — the
1078
+ * #399 incident.
1079
+ */
1080
+ function testSignal(_pid: number, _sig: NodeJS.Signals): boolean {
1081
+ return false;
1082
+ }
1083
+
1084
+ let signalFn: SignalFn = defaultSignal;
1085
+
1086
+ /**
1087
+ * Test-only: replace the raw signal sender. Pass `undefined` to restore.
1088
+ * A test that intentionally spawned a helper may authorise signals to exactly
1089
+ * that helper pid; every other pid is refused. Ownership is never inferred
1090
+ * from writing a record — a pidfile fixture authorises nothing (#399).
1091
+ */
1092
+ export function setSignalForTest(fn: SignalFn | undefined): void {
1093
+ signalFn = fn ?? defaultSignal;
1094
+ }
1095
+
1096
+ /** Sends a signal, treating "already gone" as success. Returns false if the send failed. */
1097
+ function signal(pid: number, sig: NodeJS.Signals): boolean {
1098
+ return signalFn(pid, sig);
1099
+ }
1100
+
905
1101
  /**
906
1102
  * The tail of the daemon log, formatted for an error message. Reads the last
907
1103
  * few KB by offset rather than the whole file, because the log is append-only