catherd-cli 1.4.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/README.md +93 -30
  2. package/catalog/ATTRIBUTION.md +12 -4
  3. package/catalog/models.json +8 -0
  4. package/package.json +1 -1
  5. package/plugin/.claude-plugin/plugin.json +1 -1
  6. package/plugin/.codex-plugin/plugin.json +1 -1
  7. package/plugin/bin/catherd-mcp +1 -1
  8. package/plugin/skills/catherd/SKILL.md +110 -45
  9. package/plugin/skills/catherd-setup/SKILL.md +2 -2
  10. package/src/adapters/antigravity/index.ts +18 -3
  11. package/src/adapters/backend.ts +18 -2
  12. package/src/adapters/claude-code/index.ts +126 -14
  13. package/src/adapters/codex/events.ts +10 -0
  14. package/src/adapters/codex/index.ts +17 -5
  15. package/src/adapters/cursor/index.ts +7 -1
  16. package/src/adapters/grok/home.ts +75 -20
  17. package/src/adapters/grok/index.ts +27 -8
  18. package/src/adapters/opencode/index.ts +38 -1
  19. package/src/cli.ts +5 -0
  20. package/src/domain/agents.ts +3 -1
  21. package/src/domain/brief.ts +91 -0
  22. package/src/domain/calibration.ts +7 -0
  23. package/src/domain/catalog.ts +12 -1
  24. package/src/domain/changes.ts +23 -1
  25. package/src/domain/cost.ts +8 -3
  26. package/src/domain/errors.ts +4 -0
  27. package/src/domain/failover.ts +17 -5
  28. package/src/domain/gate-brief.ts +52 -0
  29. package/src/domain/hints.ts +13 -2
  30. package/src/domain/lane.ts +99 -3
  31. package/src/domain/notice.ts +9 -2
  32. package/src/domain/profile-rules.ts +112 -16
  33. package/src/domain/profile.ts +24 -0
  34. package/src/domain/record.ts +16 -2
  35. package/src/domain/reply.ts +29 -0
  36. package/src/domain/role-prompts.ts +56 -16
  37. package/src/domain/role-scope.ts +106 -0
  38. package/src/domain/role-tools.ts +34 -0
  39. package/src/domain/route.ts +73 -0
  40. package/src/domain/select.ts +177 -25
  41. package/src/domain/state.ts +5 -0
  42. package/src/domain/util.ts +22 -0
  43. package/src/domain/workspace.ts +130 -0
  44. package/src/entry/catalog-command.ts +14 -6
  45. package/src/entry/cli-kit.ts +17 -1
  46. package/src/entry/deps.ts +2 -0
  47. package/src/entry/doctor-command.ts +20 -2
  48. package/src/entry/host-arg.ts +28 -0
  49. package/src/entry/init-command.ts +2 -1
  50. package/src/entry/knowledge-command.ts +78 -3
  51. package/src/entry/lock-command.ts +104 -8
  52. package/src/entry/mcp/dispatch-tools.ts +3 -3
  53. package/src/entry/mcp/handshake.ts +6 -2
  54. package/src/entry/mcp/lane-tools.ts +55 -8
  55. package/src/entry/mcp/protocol-tools.ts +10 -4
  56. package/src/entry/mcp/push-tools.ts +16 -0
  57. package/src/entry/mcp/role-bin.ts +20 -0
  58. package/src/entry/mcp/role-server.ts +116 -0
  59. package/src/entry/mcp/run-tools.ts +20 -4
  60. package/src/entry/mcp/server.ts +98 -7
  61. package/src/entry/mcp/setup-tools.ts +4 -3
  62. package/src/entry/mcp/workspace-tools.ts +142 -0
  63. package/src/entry/pause-command.ts +67 -0
  64. package/src/entry/profile-command.ts +23 -4
  65. package/src/entry/role-cli-command.ts +115 -0
  66. package/src/entry/runs-command.ts +78 -8
  67. package/src/entry/supervise-command.ts +30 -3
  68. package/src/entry/tui/effects.ts +4 -0
  69. package/src/entry/tui/state.ts +37 -10
  70. package/src/entry/tui/views/profiles.tsx +1 -1
  71. package/src/entry/tui/views/runs.tsx +66 -20
  72. package/src/entry/tui/views/save-dialog.tsx +6 -2
  73. package/src/entry/tui/views/status.tsx +6 -2
  74. package/src/entry/workspace-command.ts +44 -0
  75. package/src/infra/boot-lock.ts +25 -0
  76. package/src/infra/delivery.ts +3 -0
  77. package/src/infra/dispatch-dir.ts +2 -0
  78. package/src/infra/env.ts +11 -1
  79. package/src/infra/git.ts +6 -0
  80. package/src/infra/heavy-lock.ts +93 -4
  81. package/src/infra/host-probe.ts +16 -0
  82. package/src/infra/lock-activity.ts +88 -0
  83. package/src/infra/login-env.ts +48 -0
  84. package/src/infra/paths.ts +35 -0
  85. package/src/infra/role-mcp.ts +69 -0
  86. package/src/infra/sources/arena.ts +11 -3
  87. package/src/infra/sources/artificial-analysis.ts +8 -1
  88. package/src/infra/sources/cache.ts +2 -0
  89. package/src/infra/supervisor.ts +107 -16
  90. package/src/services/admission.ts +110 -23
  91. package/src/services/budget.ts +26 -6
  92. package/src/services/catalog-service.ts +20 -12
  93. package/src/services/credentials.ts +11 -6
  94. package/src/services/dispatch-service.ts +204 -16
  95. package/src/services/dispatches.ts +30 -2
  96. package/src/services/doctor-access.ts +8 -3
  97. package/src/services/doctor-docker.ts +221 -0
  98. package/src/services/doctor-role-mcp.ts +65 -0
  99. package/src/services/doctor.ts +68 -6
  100. package/src/services/end-push.ts +157 -0
  101. package/src/services/finalize.ts +12 -2
  102. package/src/services/gate-env.ts +218 -0
  103. package/src/services/gate-service.ts +216 -45
  104. package/src/services/global-install.ts +46 -13
  105. package/src/services/lane-edit.ts +154 -0
  106. package/src/services/lane-service.ts +257 -52
  107. package/src/services/milestones.ts +137 -26
  108. package/src/services/notifier.ts +56 -3
  109. package/src/services/orchestrator-wait.ts +82 -0
  110. package/src/services/pause.ts +161 -0
  111. package/src/services/peek.ts +79 -17
  112. package/src/services/ports.ts +31 -2
  113. package/src/services/preflight.ts +153 -19
  114. package/src/services/profile-service.ts +1 -0
  115. package/src/services/profile-store.ts +10 -3
  116. package/src/services/protocol.ts +119 -25
  117. package/src/services/provenance.ts +16 -12
  118. package/src/services/questions.ts +15 -0
  119. package/src/services/reconcile.ts +22 -1
  120. package/src/services/reentry.ts +7 -6
  121. package/src/services/role-access.ts +27 -0
  122. package/src/services/routing-service.ts +144 -36
  123. package/src/services/run-pin.ts +125 -0
  124. package/src/services/run-service.ts +151 -19
  125. package/src/services/run-store.ts +109 -15
  126. package/src/services/runs-page.ts +42 -7
  127. package/src/services/scratch.ts +60 -0
  128. package/src/services/sessions.ts +3 -1
  129. package/src/services/source-derive.ts +28 -5
  130. package/src/services/source-sync.ts +9 -2
  131. package/src/services/standins.ts +40 -6
  132. package/src/services/state.ts +21 -0
  133. package/src/services/summary.ts +49 -9
  134. package/src/services/treat-likes.ts +19 -3
  135. package/src/services/verifier-brief.ts +23 -0
  136. package/src/services/verifier-step.ts +48 -0
  137. package/src/services/workspace-admission.ts +267 -0
  138. package/src/services/workspace-service.ts +254 -0
  139. package/src/services/workspace-store.ts +95 -0
@@ -2,13 +2,14 @@ import { existsSync, readFileSync } from "node:fs";
2
2
  import { adapterFor } from "../adapters/registry.ts";
3
3
  import "../adapters/all.ts";
4
4
  import { type Budget, type BudgetStatus, budgetStatus, type Spend } from "../domain/budget.ts";
5
+ import { CatherdError } from "../domain/errors.ts";
5
6
  import type { RunRecord } from "../domain/record.ts";
6
7
  import { dispatchPaths } from "../infra/dispatch-dir.ts";
7
8
  import { type Dispatch, liveDispatches } from "./dispatches.ts";
8
9
  import { readAgentRuns, readRecords, type Run } from "./run-store.ts";
9
10
 
10
11
  /** Tokens a running dispatch has used so far, summed from the events its CLI has written. */
11
- export function liveTokens(d: Dispatch): number {
12
+ export function liveTokens(d: Dispatch, strict = false): number {
12
13
  const a = adapterFor(d.admit.backend);
13
14
  const file = dispatchPaths(d.dir).events;
14
15
  if (!a || !existsSync(file)) return 0;
@@ -17,8 +18,13 @@ export function liveTokens(d: Dispatch): number {
17
18
  if (!line.trim()) continue;
18
19
  try {
19
20
  const t = a.parse(line).tokens;
20
- if (t) n += t.input + t.output;
21
- } catch {
21
+ if (t) {
22
+ if (strict && Object.values(t).some((value) => !Number.isFinite(value) || value < 0))
23
+ throw new CatherdError("E_RUN_CORRUPT", `dispatch ${d.admit.dispatchId} has invalid token usage`);
24
+ n += t.input + t.output;
25
+ }
26
+ } catch (error) {
27
+ if (strict && error instanceof CatherdError) throw error;
22
28
  // a line the adapter cannot read adds nothing
23
29
  }
24
30
  }
@@ -29,15 +35,29 @@ export function liveTokens(d: Dispatch): number {
29
35
  * Spec §4.6: minutes are wall clock since run_start; tokens and dollars sum every record (every
30
36
  * backend), every native subagent run the orchestrator reported, and what live roles used so far.
31
37
  */
32
- export function spendOf(run: Run, records: RunRecord[], live: Dispatch[], now: number): Spend {
33
- const agents = readAgentRuns(run);
38
+ export function spendOf(
39
+ run: Run,
40
+ records: RunRecord[],
41
+ live: Dispatch[],
42
+ now: number,
43
+ strict = false,
44
+ ): Spend {
45
+ const agents = readAgentRuns(run, strict);
46
+ if (strict) {
47
+ const invalid = (value: number | null) => value !== null && (!Number.isFinite(value) || value < 0);
48
+ if (
49
+ records.some((record) => Object.values(record.tokens).some(invalid) || invalid(record.costUsd)) ||
50
+ agents.some((agent) => invalid(agent.totalTokens) || invalid(agent.costUsd))
51
+ )
52
+ throw new CatherdError("E_RUN_CORRUPT", `run ${run.id} has invalid cost or token usage`);
53
+ }
34
54
  const sum = <T>(xs: T[], f: (x: T) => number) => xs.reduce((n, x) => n + f(x), 0);
35
55
  return {
36
56
  minutes: Math.max(0, (now - Date.parse(run.meta.createdAt)) / 60_000),
37
57
  tokens:
38
58
  sum(records, (r) => r.tokens.input + r.tokens.output) +
39
59
  sum(agents, (a) => a.totalTokens) +
40
- sum(live, liveTokens),
60
+ sum(live, (dispatch) => liveTokens(dispatch, strict)),
41
61
  usd: sum(records, (r) => r.costUsd ?? 0) + sum(agents, (a) => a.costUsd ?? 0),
42
62
  };
43
63
  }
@@ -99,18 +99,23 @@ export function measuredSecs(base: Catalog): Catalog["secs"] {
99
99
  if (kind) add(`${canonical}|${kind}`, secs);
100
100
  };
101
101
  for (const run of listRuns().runs) {
102
- const routes = readRoutes(run);
103
- for (const r of readRecords(run).records)
104
- if (r.status === "ok") count(routes, r.rung, r.secs, r.lane, r.startedAt);
105
- for (const a of readAgentRuns(run)) {
106
- if (a.status !== "ok" || a.secs === null) continue;
107
- // a row is written when the subagent ends; it started `secs` earlier
108
- const lane = typeof a.lane === "string" ? a.lane : null;
109
- const end = Date.parse(a.at);
110
- // a duration past the Date range (record_agent_run takes any) keeps the end, not a RangeError
111
- const start = new Date(end - a.secs * 1000);
112
- const startedAt = Number.isNaN(start.getTime()) ? a.at : start.toISOString();
113
- count(routes, a.rung, a.secs, lane, startedAt);
102
+ // a run whose files cannot be read gives no timings; it never fails a route (1.2 minor)
103
+ try {
104
+ const routes = readRoutes(run);
105
+ for (const r of readRecords(run).records)
106
+ if (r.status === "ok") count(routes, r.rung, r.secs, r.lane, r.startedAt);
107
+ for (const a of readAgentRuns(run)) {
108
+ if (a.status !== "ok" || a.secs === null) continue;
109
+ // a row is written when the subagent ends; it started `secs` earlier
110
+ const lane = typeof a.lane === "string" ? a.lane : null;
111
+ const end = Date.parse(a.at);
112
+ // a duration past the Date range (record_agent_run takes any) keeps the end, not a RangeError
113
+ const start = new Date(end - a.secs * 1000);
114
+ const startedAt = Number.isNaN(start.getTime()) ? a.at : start.toISOString();
115
+ count(routes, a.rung, a.secs, lane, startedAt);
116
+ }
117
+ } catch (e) {
118
+ log("debug", "catalog", { run: run.id, timings: errorMessage(e) });
114
119
  }
115
120
  }
116
121
  const secs: Catalog["secs"] = {};
@@ -340,6 +345,8 @@ function rungRows(
340
345
  date: string;
341
346
  /** spec 1.2 §5.3: the rung a borrowed or inferred value belongs to */
342
347
  from?: string;
348
+ /** what lent it: a treat-like's mapping (`like X`) or an inferred stand-in's guess (1.2 minor) */
349
+ lent?: "treat-like" | "stand-in";
343
350
  }
344
351
  > = {};
345
352
  for (const v of valuesUsed(c, info.canonical))
@@ -351,6 +358,7 @@ function rungRows(
351
358
  source: v.source,
352
359
  date: v.date,
353
360
  ...(v.from ? { from: v.from } : {}),
361
+ ...(v.lent ? { lent: v.lent } : {}),
354
362
  };
355
363
  const like = c.treatLike[info.canonical] ?? null;
356
364
  return {
@@ -1,7 +1,8 @@
1
- import { existsSync } from "node:fs";
1
+ import { existsSync, mkdirSync } from "node:fs";
2
2
  import { join } from "node:path";
3
3
  import { z } from "zod";
4
4
  import { CatherdError, isCatherdError } from "../domain/errors.ts";
5
+ import { withFileLockSync } from "../infra/filelock.ts";
5
6
  import { log } from "../infra/log.ts";
6
7
  import { configDir } from "../infra/paths.ts";
7
8
  import { readVersioned, writeJsonAtomic } from "../infra/store.ts";
@@ -45,13 +46,17 @@ export function savedCredential(field: CredentialField): {
45
46
 
46
47
  /**
47
48
  * Keeps every other credential, and the file at mode 600 (spec §10.4). A missing file starts empty; an
48
- * unreadable or newer-schema one is refused (it throws) rather than overwritten.
49
+ * unreadable or newer-schema one is refused (it throws) rather than overwritten. The read and the write hold
50
+ * the file's lock (1.2 minor), so two concurrent `init`s never lose a key.
49
51
  */
50
52
  export function saveCredential(field: CredentialField, key: string): void {
51
- const cur: z.infer<typeof CredentialsSchema> = existsSync(credentialsPath())
52
- ? readCredentials()
53
- : { schema: 1 };
54
- writeJsonAtomic(credentialsPath(), { ...cur, schema: 1, [field]: key.trim() }, { mode: 0o600 });
53
+ mkdirSync(configDir(), { recursive: true });
54
+ withFileLockSync(credentialsPath(), () => {
55
+ const cur: z.infer<typeof CredentialsSchema> = existsSync(credentialsPath())
56
+ ? readCredentials()
57
+ : { schema: 1 };
58
+ writeJsonAtomic(credentialsPath(), { ...cur, schema: 1, [field]: key.trim() }, { mode: 0o600 });
59
+ });
55
60
  }
56
61
 
57
62
  /**
@@ -1,5 +1,6 @@
1
1
  import { assertNativeHost } from "./backends.ts";
2
2
  import { existsSync, readFileSync } from "node:fs";
3
+ import { join } from "node:path";
3
4
  import { CatherdError, errorMessage, isCatherdError } from "../domain/errors.ts";
4
5
  import { assertId, parseRung } from "../domain/ids.ts";
5
6
  import type { RunRecord } from "../domain/record.ts";
@@ -16,8 +17,9 @@ import {
16
17
  import { lockHeld, withFileLock } from "../infra/filelock.ts";
17
18
  import { log } from "../infra/log.ts";
18
19
  import { isAlive, isSurelyAlive, killGroup } from "../infra/proc.ts";
20
+ import { groupAlive, stopGroup } from "../infra/supervisor.ts";
19
21
  import { writeJsonAtomic } from "../infra/store.ts";
20
- import { admit, KILL_GRACE_MS, laneFile, launch } from "./admission.ts";
22
+ import { admit, KILL_GRACE_MS, laneFile, laneOwns, launch } from "./admission.ts";
21
23
  import { standInFor } from "./backends.ts";
22
24
  import {
23
25
  type Dispatch,
@@ -34,8 +36,11 @@ import {
34
36
  import { finalizeDispatch, finalizingElsewhere, waitForFinish } from "./finalize.ts";
35
37
  import type { Deps } from "./ports.ts";
36
38
  import { route } from "./lane-service.ts";
37
- import { findRun, readRecords, readRoutes, type Run } from "./run-store.ts";
38
- import { claimRun, ownsRun, runOwner } from "./sessions.ts";
39
+ import { currentRoute } from "../domain/route.ts";
40
+ import { appendRoute, findRun, readRecords, readRoleRoutes, readRoutes, type Run } from "./run-store.ts";
41
+ import { sessionKey } from "../domain/host.ts";
42
+ import { claimRun, currentSession, ownsRun, runOwner, type SessionRef } from "./sessions.ts";
43
+ import { pinChanges, runProfile } from "./run-pin.ts";
39
44
  import { type NotesPatch, refreshState } from "./state.ts";
40
45
 
41
46
  export interface DispatchInput {
@@ -43,7 +48,8 @@ export interface DispatchInput {
43
48
  role: Role;
44
49
  name: string;
45
50
  brief: string;
46
- rung: string;
51
+ /** spec 1.5 plan 24: optional with `lane` (the lane's current rung); required without one */
52
+ rung?: string;
47
53
  thread?: string;
48
54
  lane?: string;
49
55
  next?: string;
@@ -174,9 +180,19 @@ export async function watchersSettled(): Promise<void> {
174
180
  */
175
181
  export function watch(deps: Deps, run: Run, d: Dispatch): void {
176
182
  watching.add(d.admit.dispatchId);
183
+ // the session this watcher started under, kept apart from its live identity: an MCP connection that closes
184
+ // invalidates the scoped host (no session), yet the watcher still came from that session (PR #47 P1)
185
+ const origin = currentSession(deps);
177
186
  const w = (async () => {
178
187
  await waitForFinish(d, { pollMs: deps.pollMs, now: deps.now, onPoll: stallPoll(run, d) });
179
- const s = await settle(deps, run, d, await finalizeDispatch(run, d));
188
+ const record = await finalizeDispatch(run, d);
189
+ // a session that lost the run since it dispatched leaves a limit to the owner's claim (1.1 push minors):
190
+ // only the run's owner starts a stand-in. A process with no session (a terminal) settles as before.
191
+ if (record.status === "limit" && !limitIsMine(deps, run, origin)) {
192
+ log("info", "dispatch", { run: run.id, name: d.admit.name, limit: "left to the run's owner" });
193
+ return;
194
+ }
195
+ const s = await settle(deps, run, d, record);
180
196
  if (s.stateHints[0]) log("warn", "dispatch", { run: run.id, name: d.admit.name, hint: s.stateHints[0] });
181
197
  })()
182
198
  .catch((e: unknown) =>
@@ -189,6 +205,18 @@ export function watch(deps: Deps, run: Run, d: Dispatch): void {
189
205
  watchers.add(w);
190
206
  }
191
207
 
208
+ /**
209
+ * Whether a watcher settles a limit (fails it over): a watcher with no session, then or now (a terminal, a
210
+ * test), always does; one that started under a session does only while that session, or its live one, owns
211
+ * the run.
212
+ */
213
+ function limitIsMine(deps: Deps, run: Run, origin: SessionRef | null): boolean {
214
+ if (origin === null && currentSession(deps) === null) return true;
215
+ if (ownsRun(deps, run)) return true;
216
+ const owner = runOwner(run);
217
+ return origin !== null && owner !== null && sessionKey(owner) === sessionKey(origin);
218
+ }
219
+
192
220
  /**
193
221
  * Spec §3.3: a run this session has just taken over may have roles another session's server launched. This
194
222
  * process watches each live one it is not already watching, so it settles them and its notifier announces them
@@ -214,6 +242,16 @@ export function adopt(deps: Deps, run: Run): void {
214
242
  */
215
243
  export async function claim(deps: Deps, run: Run, o: { background?: boolean } = {}): Promise<void> {
216
244
  if (!(await claimRun(deps, run)) && !ownsRun(deps, run)) return;
245
+ await recoverOwned(deps, run, o);
246
+ }
247
+
248
+ /**
249
+ * What `claim` does for a run this session already owns, without taking it: watch its live roles, record and
250
+ * settle what finished unrecorded, settle its unsettled limits and unannounced records. A server that does not
251
+ * lead the boot runs it for each run its session owns (PR #47 P1: a role that finished while that server was
252
+ * away has no live process to watch and no record to scan).
253
+ */
254
+ export async function recoverOwned(deps: Deps, run: Run, o: { background?: boolean } = {}): Promise<void> {
217
255
  adopt(deps, run);
218
256
  let r = recovering.get(run.id);
219
257
  if (!r) {
@@ -281,24 +319,138 @@ function unannounced(run: Run): { d: Dispatch; record: RunRecord }[] {
281
319
  });
282
320
  }
283
321
 
322
+ /** How long a dispatch waits for another dispatch's route of the same lane (Jev may take tens of seconds). */
323
+ const ROUTE_LOCK_MS = 120_000;
324
+
325
+ /**
326
+ * What admission would refuse anyway, refused before a lane is routed, so a refused dispatch asks Jev nothing
327
+ * and writes no route row: the role off, the lane file missing or with no Owns:, the name or the lane already
328
+ * running. Admission checks them again under its lock.
329
+ */
330
+ function refuseBeforeRouting(deps: Deps, run: Run, i: DispatchInput & { lane: string }): void {
331
+ const profile = runProfile(deps, run);
332
+ if (!profile.roles[i.role]?.enabled)
333
+ throw new CatherdError("E_ADMIT_RUNG", `the ${i.role} role is off in profile ${profile.name}`, {
334
+ fix: "skip the role, or turn it on with profile_set",
335
+ });
336
+ laneOwns(run, i.lane);
337
+ // live only: a finished, unrecorded dispatch is recorded by admission (finalizeFinished), not refused here
338
+ const pending = liveDispatches(run, deps.now());
339
+ const same = pending.find((d) => d.admit.name === i.name);
340
+ if (same)
341
+ throw new CatherdError("E_ADMIT_DUPLICATE", `${i.name} is already running on ${same.admit.rung}`, {
342
+ fix: `its record is announced when it finishes; or cancel(run, "${i.name}")`,
343
+ });
344
+ const lane = pending.find((d) => d.admit.lane === i.lane);
345
+ if (lane)
346
+ throw new CatherdError("E_ADMIT_OVERLAP", `lane ${i.lane} is already running as ${lane.admit.name}`, {
347
+ fix: `dispatch it after ${lane.admit.name} finishes`,
348
+ });
349
+ }
350
+
351
+ /**
352
+ * Plan 22: the thread a dispatch resumes. `"latest"` is the name's last recorded thread; any other id must be one
353
+ * the name's records in this run hold (`runs.jsonl`), in any case, so a wrong id is refused before a CLI starts
354
+ * and fails on it. Null for a fresh thread.
355
+ */
356
+ function threadFor(run: Run, name: string, thread: string | undefined): string | null {
357
+ if (thread === undefined) return null;
358
+ assertId("role name", name);
359
+ const mine = readRecords(run)
360
+ .records.filter((r) => r.name === name && r.thread !== null)
361
+ .map((r) => r.thread as string);
362
+ const last = mine.at(-1);
363
+ if (thread === "latest") {
364
+ if (last) return last;
365
+ throw new CatherdError("E_ADMIT_THREAD", `${name} has no earlier thread in this run`, {
366
+ fix: "omit thread for a fresh thread",
367
+ });
368
+ }
369
+ const known = mine.findLast((t) => t.toLowerCase() === thread.toLowerCase());
370
+ if (known) return known;
371
+ throw new CatherdError("E_ADMIT_THREAD", `${thread} is not a thread of ${name} in this run`, {
372
+ fix: last
373
+ ? `pass thread: "latest" for ${name}'s last thread (${last}), or omit thread for a fresh one`
374
+ : "omit thread for a fresh thread",
375
+ });
376
+ }
377
+
378
+ /**
379
+ * Plan 22, resume hygiene: what an earlier turn on `thread` left running in its process group (a server, a
380
+ * watcher, a background command) is stopped before the thread is resumed, so it never ends the resumed CLI
381
+ * (exit 143). Only a dispatch whose supervisor and group leader are both gone is touched (with or without
382
+ * exit.json: a supervisor killed before it stopped the group writes none): a live group with no process of the
383
+ * leader's pid can only be that dispatch's leftovers (a pid is never reused while its group lives), and a pid
384
+ * that answers belongs to someone else by now. Returns a hint per group stopped.
385
+ */
386
+ async function stopLeftovers(deps: Deps, run: Run, thread: string): Promise<string[]> {
387
+ const records = new Map(readRecords(run).records.map((r) => [r.dispatchId, r]));
388
+ const hints: string[] = [];
389
+ for (const d of listDispatches(run)) {
390
+ const on = d.admit.thread ?? records.get(d.admit.dispatchId)?.thread ?? null;
391
+ if (on?.toLowerCase() !== thread.toLowerCase()) continue;
392
+ const proc = readProc(d.dir);
393
+ const pgid = proc?.pgid ?? proc?.pid;
394
+ if (!proc || pgid === undefined || pgid !== proc.pid) continue;
395
+ // a live supervisor stops its own group when its worker ends; exit.json is not the test, since a supervisor
396
+ // killed (SIGKILL, OOM) before it stopped the group never writes one
397
+ if (isAlive(proc.supervisorPid, proc.supervisorStartTime)) continue;
398
+ if (isAlive(proc.pid, null) || !groupAlive(pgid)) continue;
399
+ await stopGroup(pgid, orphanLimits.killGraceMs, deps.pollMs);
400
+ log("info", "dispatch", { run: run.id, name: d.admit.name, thread, stoppedGroup: pgid });
401
+ hints.push(`resume: stopped what ${d.admit.name}'s earlier turn left running on thread ${thread}`);
402
+ }
403
+ return hints;
404
+ }
405
+
284
406
  /**
285
407
  * Spec §4.4, plan 9 ruling 1: admit and launch one role, start its watcher, then refresh state.md with
286
408
  * `next` (after the launch, so nothing delays it). Returns at once; catherd announces the record (spec §3).
287
409
  */
288
410
  export async function dispatch(deps: Deps, i: DispatchInput): Promise<DispatchStarted> {
289
411
  const run = findRun(i.run);
412
+ // refused before the claim, so a wrong thread changes nothing
413
+ const thread = threadFor(run, i.name, i.thread);
290
414
  await claim(deps, run);
291
415
  const hints: string[] = [];
292
- let rung = i.rung;
293
- // spec 1.1 §6: a lane is routed before its first dispatch; a rung off the routed ladder starts at the routed one
294
- if (i.lane !== undefined && !readRoutes(run).some((r) => r.lane === i.lane)) {
295
- assertId("lane", i.lane);
296
- const routed = await route(deps, { run: i.run, laneFile: `lanes/${i.lane}.md`, role: i.role });
297
- if (!routed.ladder.includes(i.rung)) {
416
+ if (thread !== null) hints.push(...(await stopLeftovers(deps, run, thread)));
417
+ let rung: string;
418
+ if (i.lane !== undefined) {
419
+ const lane = i.lane;
420
+ assertId("lane", lane);
421
+ // spec 1.1 §6: a lane is routed before its first dispatch; a rung off the routed ladder starts at the routed one
422
+ let current = currentRoute(readRoutes(run), lane);
423
+ let routed: Awaited<ReturnType<typeof route>> | null = null;
424
+ if (!current) {
425
+ refuseBeforeRouting(deps, run, { ...i, lane });
426
+ // one route per lane: a second dispatch of the lane waits for the first's route and takes it
427
+ const got = await withFileLock(
428
+ join(run.dir, `route-${lane}`),
429
+ async () => {
430
+ const now = currentRoute(readRoutes(run), lane);
431
+ return now
432
+ ? { current: now, routed: null }
433
+ : {
434
+ current: null,
435
+ routed: await route(deps, { run: i.run, laneFile: `lanes/${lane}.md`, role: i.role }),
436
+ };
437
+ },
438
+ { timeoutMs: ROUTE_LOCK_MS },
439
+ );
440
+ current = got.current;
441
+ routed = got.routed;
442
+ }
443
+ // spec 1.5 plan 24: without a rung, a lane runs at its current rung (its route, after any climb)
444
+ rung = i.rung ?? routed?.rung ?? (current?.rung as string);
445
+ if (routed && i.rung !== undefined && !routed.ladder.includes(i.rung)) {
298
446
  rung = routed.rung;
299
- hints.push(`${i.rung} is not on ${i.lane}'s routed ladder: dispatched at ${routed.rung}`);
447
+ hints.push(`${i.rung} is not on ${lane}'s routed ladder: dispatched at ${routed.rung}`);
300
448
  }
301
- }
449
+ } else if (i.rung === undefined)
450
+ throw new CatherdError("E_INPUT_INVALID", `dispatch ${i.name}: a role outside a lane needs a rung`, {
451
+ fix: `route(run, role: "${i.role}") gives the role's rung; pass it as rung`,
452
+ });
453
+ else rung = i.rung;
302
454
  // the rung that runs, after routing: an off-ladder native rung routing replaced is never launched
303
455
  assertNativeHost(rung, deps.host.host);
304
456
  const { d, specPath } = await admit(
@@ -309,7 +461,7 @@ export async function dispatch(deps: Deps, i: DispatchInput): Promise<DispatchSt
309
461
  name: i.name,
310
462
  brief: i.brief,
311
463
  rung,
312
- thread: i.thread ?? null,
464
+ thread,
313
465
  lane: i.lane ?? null,
314
466
  failoverFrom: null,
315
467
  },
@@ -324,10 +476,46 @@ export async function dispatch(deps: Deps, i: DispatchInput): Promise<DispatchSt
324
476
  // a launch that failed is still watched: its record (lost, after the start grace) is announced
325
477
  watch(deps, run, d);
326
478
  }
327
- await refresh(run, i.next ? { next: i.next } : {}, hints);
479
+ if (i.lane === undefined) recordRoleDispatch(deps, run, i.role, i.name, rung);
480
+ // spec 1.5: a change since the run's pin is logged in state.md each time a role starts
481
+ const changes = pinChanges(deps, run);
482
+ if (changes.length)
483
+ hints.push(
484
+ `dispatched on the run's pinned values: ${changes.join("; ")}; run_pin(run) re-pins to the repo's now`,
485
+ );
486
+ await refresh(run, { ...(i.next ? { next: i.next } : {}), pinChanges: changes }, hints);
328
487
  return { dispatched: dispatchedOf(d), hints };
329
488
  }
330
489
 
490
+ /**
491
+ * Spec 1.5 plan 24: a lane-less dispatch's rung in routes.jsonl, so every role's decision can be audited:
492
+ * `route` when it is the rung the role's last `route` gave (with that ladder), else the coordinator's own pick.
493
+ * A failed write is logged, never a failed dispatch: the role is already running.
494
+ */
495
+ function recordRoleDispatch(deps: Deps, run: Run, role: Role, name: string, rung: string): void {
496
+ try {
497
+ const last = readRoleRoutes(run).findLast((r) => r.role === role && r.source === "route");
498
+ const routed = last?.rung === rung;
499
+ appendRoute(run, {
500
+ at: new Date(deps.now()).toISOString(),
501
+ lane: null,
502
+ role,
503
+ name,
504
+ rung,
505
+ ladder: last?.ladder ?? [rung],
506
+ source: "dispatch",
507
+ decidedBy: routed ? last.decidedBy : "orchestrator",
508
+ why: routed
509
+ ? `the rung route gave the ${role}`
510
+ : last
511
+ ? `the coordinator's rung; route gave the ${role} ${last.rung}`
512
+ : `the coordinator's rung; the ${role} was never routed`,
513
+ });
514
+ } catch (e) {
515
+ log("warn", "dispatch", { run: run.id, name, routes: errorMessage(e) });
516
+ }
517
+ }
518
+
331
519
  /** How long a settle waits for another process's failover of the same dispatch (a test seam). */
332
520
  export const failoverLock = { timeoutMs: 120_000 };
333
521
 
@@ -498,7 +686,7 @@ async function failover(deps: Deps, run: Run, d: Dispatch, limited: RunRecord):
498
686
  return failedOver(already.admit.rung, already);
499
687
  }
500
688
  // else that stand-in never ran and is over (recorded as lost, or past its start grace): admit a new one
501
- const standIn = standInFor(deps.profiles.forRepo(run.meta.repo).failover, limited.rung, run.meta.repo);
689
+ const standIn = standInFor(runProfile(deps, run).failover, limited.rung, run.meta.repo);
502
690
  if (!standIn) return { hints, started: null, pause: paused };
503
691
  // the stand-in answers to whoever owns the run now, which a claim from another host may have changed
504
692
  assertNativeHost(standIn, runOwner(run)?.host ?? deps.host.host);
@@ -1,6 +1,7 @@
1
1
  import { existsSync, readdirSync, readFileSync } from "node:fs";
2
2
  import { join, relative } from "node:path";
3
3
  import { z } from "zod";
4
+ import { CatherdError } from "../domain/errors.ts";
4
5
  import { dispatchHints } from "../domain/hints.ts";
5
6
  import { ACCESS, type RunRecord } from "../domain/record.ts";
6
7
  import { ROLES } from "../domain/roles.ts";
@@ -18,6 +19,11 @@ const AdmitSchema = z.looseObject({
18
19
  role: z.enum(ROLES),
19
20
  lane: z.string().nullable(),
20
21
  owns: z.array(z.string()),
22
+ /**
23
+ * spec 1.5 plan 21: a laneless writer's implicit Owns (its brief's Owns: line, else docs/** and *.md), so its
24
+ * edits are its own and never a concurrent lane's violation; admission never refuses an overlap with it
25
+ */
26
+ ownsImplicit: z.array(z.string()).optional(),
21
27
  rung: z.string(),
22
28
  backend: z.string(),
23
29
  thread: z.string().nullable(),
@@ -54,9 +60,11 @@ export const startLimits = { graceMs: STARTING_GRACE_MS };
54
60
  export const admitPath = (dir: string): string => join(dir, "admit.json");
55
61
  export const launchPath = (dir: string): string => join(dir, "launch.json");
56
62
  export const roleDir = (run: Run, name: string): string => join(runPaths(run.dir).roles, name);
63
+ /** Spec 1.5 plan 21: the role's TMPDIR, `<run>/scratch/<name>/`, removed with the run's scratch. */
64
+ export const scratchDir = (run: Run, name: string): string => join(runPaths(run.dir).scratch, name);
57
65
 
58
66
  /** Every admitted dispatch of the run, oldest first; a folder whose admit.json cannot be read is skipped. */
59
- export function listDispatches(run: Run): Dispatch[] {
67
+ export function listDispatches(run: Run, strict = false): Dispatch[] {
60
68
  const roles = runPaths(run.dir).roles;
61
69
  const out: Dispatch[] = [];
62
70
  if (!existsSync(roles)) return out;
@@ -68,6 +76,8 @@ export function listDispatches(run: Run): Dispatch[] {
68
76
  try {
69
77
  out.push({ dir, admit: decodeAdmit(readVersioned(admitPath(dir), AdmitSchema, 1)) });
70
78
  } catch {
79
+ if (strict && existsSync(admitPath(dir)))
80
+ throw new CatherdError("E_RUN_CORRUPT", `run ${run.id} has unreadable admission evidence: ${dir}`);
71
81
  // admission never finished writing this folder
72
82
  }
73
83
  }
@@ -124,9 +134,10 @@ export function pendingDispatches(
124
134
  run: Run,
125
135
  now = Date.now(),
126
136
  records: RunRecord[] = readRecords(run).records,
137
+ strict = false,
127
138
  ): LiveDispatch[] {
128
139
  const recorded = new Set(records.map((r) => r.dispatchId));
129
- return listDispatches(run)
140
+ return listDispatches(run, strict)
130
141
  .filter((d) => !recorded.has(d.admit.dispatchId))
131
142
  .map((d) => ({ ...d, state: dispatchState(d, now) }));
132
143
  }
@@ -134,6 +145,23 @@ export function pendingDispatches(
134
145
  export const liveDispatches = (run: Run, now = Date.now()): LiveDispatch[] =>
135
146
  pendingDispatches(run, now).filter((d) => d.state !== "finished");
136
147
 
148
+ /** The thread the supervisor saw the role's CLI name (thread.json), else null. */
149
+ export function readThread(dir: string): string | null {
150
+ const t = readJson<{ thread?: unknown }>(dispatchPaths(dir).thread)?.thread;
151
+ return typeof t === "string" && t ? t : null;
152
+ }
153
+
154
+ /**
155
+ * Spec 1.5 plan 21: the role whose thread `thread` is, in this run: a recorded thread, a resumed one, or the one
156
+ * a live role's supervisor wrote down. Null for any other thread, the orchestrator's among them.
157
+ */
158
+ export function roleThreadOf(run: Run, thread: string): string | null {
159
+ const recorded = readRecords(run).records.find((r) => r.thread === thread);
160
+ if (recorded) return recorded.name;
161
+ const d = listDispatches(run).find((x) => x.admit.thread === thread || readThread(x.dir) === thread);
162
+ return d ? d.admit.name : null;
163
+ }
164
+
137
165
  /** Spec §4.1: `roles/<name>/latest` names the newest dispatch of a role. */
138
166
  export function setLatest(run: Run, name: string, dispatchId: string): void {
139
167
  writeTextAtomic(join(roleDir(run, name), "latest"), dispatchId);
@@ -172,9 +172,14 @@ export function workspaceWriteNetwork(profiles: Profile[]): Map<string, boolean>
172
172
 
173
173
  /**
174
174
  * The `sandbox:codex` row (which `codex sandbox` form runs) and one `access:<backend>` row per backend; a
175
- * backend not in `installed` (its CLI is not on PATH) is skipped without running a probe.
175
+ * backend not in `installed` (its CLI is not on PATH) is skipped without running a probe. `repos`: where
176
+ * workers run (doctor's repo and every bound one), for a sandbox set per repo.
176
177
  */
177
- export async function accessChecks(profiles: Profile[], installed: ReadonlySet<string>): Promise<Check[]> {
178
+ export async function accessChecks(
179
+ profiles: Profile[],
180
+ installed: ReadonlySet<string>,
181
+ repos: string[] = [],
182
+ ): Promise<Check[]> {
178
183
  const checks: Check[] = [];
179
184
  for (const [id, network] of workspaceWriteNetwork(profiles)) {
180
185
  const a = adapterFor(id);
@@ -190,7 +195,7 @@ export async function accessChecks(profiles: Profile[], installed: ReadonlySet<s
190
195
  continue;
191
196
  }
192
197
  const shell = a.accessShell
193
- ? await a.accessShell({ network }).catch((e: unknown) => String(e))
198
+ ? await a.accessShell({ network, repos }).catch((e: unknown) => String(e))
194
199
  : `no way to run a shell in ${id}'s sandbox without a model turn; the live kit (docs/dev/live-verification.md) runs the five probes as one worker turn`;
195
200
  if (id === "codex")
196
201
  checks.push(