catherd-cli 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -30
- package/catalog/ATTRIBUTION.md +12 -4
- package/catalog/models.json +8 -0
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/.codex-plugin/plugin.json +1 -1
- package/plugin/bin/catherd-mcp +1 -1
- package/plugin/skills/catherd/SKILL.md +110 -45
- package/plugin/skills/catherd-setup/SKILL.md +2 -2
- package/src/adapters/antigravity/index.ts +18 -3
- package/src/adapters/backend.ts +18 -2
- package/src/adapters/claude-code/index.ts +126 -14
- package/src/adapters/codex/events.ts +10 -0
- package/src/adapters/codex/index.ts +17 -5
- package/src/adapters/cursor/index.ts +7 -1
- package/src/adapters/grok/home.ts +75 -20
- package/src/adapters/grok/index.ts +27 -8
- package/src/adapters/opencode/index.ts +38 -1
- package/src/cli.ts +5 -0
- package/src/domain/agents.ts +3 -1
- package/src/domain/brief.ts +91 -0
- package/src/domain/calibration.ts +7 -0
- package/src/domain/catalog.ts +12 -1
- package/src/domain/changes.ts +23 -1
- package/src/domain/cost.ts +8 -3
- package/src/domain/errors.ts +4 -0
- package/src/domain/failover.ts +17 -5
- package/src/domain/gate-brief.ts +52 -0
- package/src/domain/hints.ts +13 -2
- package/src/domain/lane.ts +99 -3
- package/src/domain/notice.ts +9 -2
- package/src/domain/profile-rules.ts +112 -16
- package/src/domain/profile.ts +24 -0
- package/src/domain/record.ts +16 -2
- package/src/domain/reply.ts +29 -0
- package/src/domain/role-prompts.ts +56 -16
- package/src/domain/role-scope.ts +106 -0
- package/src/domain/role-tools.ts +34 -0
- package/src/domain/route.ts +73 -0
- package/src/domain/select.ts +177 -25
- package/src/domain/state.ts +5 -0
- package/src/domain/util.ts +22 -0
- package/src/domain/workspace.ts +130 -0
- package/src/entry/catalog-command.ts +14 -6
- package/src/entry/cli-kit.ts +17 -1
- package/src/entry/deps.ts +2 -0
- package/src/entry/doctor-command.ts +20 -2
- package/src/entry/host-arg.ts +28 -0
- package/src/entry/init-command.ts +2 -1
- package/src/entry/knowledge-command.ts +78 -3
- package/src/entry/lock-command.ts +104 -8
- package/src/entry/mcp/dispatch-tools.ts +3 -3
- package/src/entry/mcp/handshake.ts +6 -2
- package/src/entry/mcp/lane-tools.ts +55 -8
- package/src/entry/mcp/protocol-tools.ts +10 -4
- package/src/entry/mcp/push-tools.ts +16 -0
- package/src/entry/mcp/role-bin.ts +20 -0
- package/src/entry/mcp/role-server.ts +116 -0
- package/src/entry/mcp/run-tools.ts +20 -4
- package/src/entry/mcp/server.ts +98 -7
- package/src/entry/mcp/setup-tools.ts +4 -3
- package/src/entry/mcp/workspace-tools.ts +142 -0
- package/src/entry/pause-command.ts +67 -0
- package/src/entry/profile-command.ts +23 -4
- package/src/entry/role-cli-command.ts +115 -0
- package/src/entry/runs-command.ts +78 -8
- package/src/entry/supervise-command.ts +30 -3
- package/src/entry/tui/effects.ts +4 -0
- package/src/entry/tui/state.ts +37 -10
- package/src/entry/tui/views/profiles.tsx +1 -1
- package/src/entry/tui/views/runs.tsx +66 -20
- package/src/entry/tui/views/save-dialog.tsx +6 -2
- package/src/entry/tui/views/status.tsx +6 -2
- package/src/entry/workspace-command.ts +44 -0
- package/src/infra/boot-lock.ts +25 -0
- package/src/infra/delivery.ts +3 -0
- package/src/infra/dispatch-dir.ts +2 -0
- package/src/infra/env.ts +11 -1
- package/src/infra/git.ts +6 -0
- package/src/infra/heavy-lock.ts +93 -4
- package/src/infra/host-probe.ts +16 -0
- package/src/infra/lock-activity.ts +88 -0
- package/src/infra/login-env.ts +48 -0
- package/src/infra/paths.ts +35 -0
- package/src/infra/role-mcp.ts +69 -0
- package/src/infra/sources/arena.ts +11 -3
- package/src/infra/sources/artificial-analysis.ts +8 -1
- package/src/infra/sources/cache.ts +2 -0
- package/src/infra/supervisor.ts +107 -16
- package/src/services/admission.ts +110 -23
- package/src/services/budget.ts +26 -6
- package/src/services/catalog-service.ts +20 -12
- package/src/services/credentials.ts +11 -6
- package/src/services/dispatch-service.ts +204 -16
- package/src/services/dispatches.ts +30 -2
- package/src/services/doctor-access.ts +8 -3
- package/src/services/doctor-docker.ts +221 -0
- package/src/services/doctor-role-mcp.ts +65 -0
- package/src/services/doctor.ts +68 -6
- package/src/services/end-push.ts +157 -0
- package/src/services/finalize.ts +12 -2
- package/src/services/gate-env.ts +218 -0
- package/src/services/gate-service.ts +216 -45
- package/src/services/global-install.ts +46 -13
- package/src/services/lane-edit.ts +154 -0
- package/src/services/lane-service.ts +257 -52
- package/src/services/milestones.ts +137 -26
- package/src/services/notifier.ts +56 -3
- package/src/services/orchestrator-wait.ts +82 -0
- package/src/services/pause.ts +161 -0
- package/src/services/peek.ts +79 -17
- package/src/services/ports.ts +31 -2
- package/src/services/preflight.ts +153 -19
- package/src/services/profile-service.ts +1 -0
- package/src/services/profile-store.ts +10 -3
- package/src/services/protocol.ts +119 -25
- package/src/services/provenance.ts +16 -12
- package/src/services/questions.ts +15 -0
- package/src/services/reconcile.ts +22 -1
- package/src/services/reentry.ts +7 -6
- package/src/services/role-access.ts +27 -0
- package/src/services/routing-service.ts +144 -36
- package/src/services/run-pin.ts +125 -0
- package/src/services/run-service.ts +151 -19
- package/src/services/run-store.ts +109 -15
- package/src/services/runs-page.ts +42 -7
- package/src/services/scratch.ts +60 -0
- package/src/services/sessions.ts +3 -1
- package/src/services/source-derive.ts +28 -5
- package/src/services/source-sync.ts +9 -2
- package/src/services/standins.ts +40 -6
- package/src/services/state.ts +21 -0
- package/src/services/summary.ts +49 -9
- package/src/services/treat-likes.ts +19 -3
- package/src/services/verifier-brief.ts +23 -0
- package/src/services/verifier-step.ts +48 -0
- package/src/services/workspace-admission.ts +267 -0
- package/src/services/workspace-service.ts +254 -0
- package/src/services/workspace-store.ts +95 -0
package/src/services/budget.ts
CHANGED
|
@@ -2,13 +2,14 @@ import { existsSync, readFileSync } from "node:fs";
|
|
|
2
2
|
import { adapterFor } from "../adapters/registry.ts";
|
|
3
3
|
import "../adapters/all.ts";
|
|
4
4
|
import { type Budget, type BudgetStatus, budgetStatus, type Spend } from "../domain/budget.ts";
|
|
5
|
+
import { CatherdError } from "../domain/errors.ts";
|
|
5
6
|
import type { RunRecord } from "../domain/record.ts";
|
|
6
7
|
import { dispatchPaths } from "../infra/dispatch-dir.ts";
|
|
7
8
|
import { type Dispatch, liveDispatches } from "./dispatches.ts";
|
|
8
9
|
import { readAgentRuns, readRecords, type Run } from "./run-store.ts";
|
|
9
10
|
|
|
10
11
|
/** Tokens a running dispatch has used so far, summed from the events its CLI has written. */
|
|
11
|
-
export function liveTokens(d: Dispatch): number {
|
|
12
|
+
export function liveTokens(d: Dispatch, strict = false): number {
|
|
12
13
|
const a = adapterFor(d.admit.backend);
|
|
13
14
|
const file = dispatchPaths(d.dir).events;
|
|
14
15
|
if (!a || !existsSync(file)) return 0;
|
|
@@ -17,8 +18,13 @@ export function liveTokens(d: Dispatch): number {
|
|
|
17
18
|
if (!line.trim()) continue;
|
|
18
19
|
try {
|
|
19
20
|
const t = a.parse(line).tokens;
|
|
20
|
-
if (t)
|
|
21
|
-
|
|
21
|
+
if (t) {
|
|
22
|
+
if (strict && Object.values(t).some((value) => !Number.isFinite(value) || value < 0))
|
|
23
|
+
throw new CatherdError("E_RUN_CORRUPT", `dispatch ${d.admit.dispatchId} has invalid token usage`);
|
|
24
|
+
n += t.input + t.output;
|
|
25
|
+
}
|
|
26
|
+
} catch (error) {
|
|
27
|
+
if (strict && error instanceof CatherdError) throw error;
|
|
22
28
|
// a line the adapter cannot read adds nothing
|
|
23
29
|
}
|
|
24
30
|
}
|
|
@@ -29,15 +35,29 @@ export function liveTokens(d: Dispatch): number {
|
|
|
29
35
|
* Spec §4.6: minutes are wall clock since run_start; tokens and dollars sum every record (every
|
|
30
36
|
* backend), every native subagent run the orchestrator reported, and what live roles used so far.
|
|
31
37
|
*/
|
|
32
|
-
export function spendOf(
|
|
33
|
-
|
|
38
|
+
export function spendOf(
|
|
39
|
+
run: Run,
|
|
40
|
+
records: RunRecord[],
|
|
41
|
+
live: Dispatch[],
|
|
42
|
+
now: number,
|
|
43
|
+
strict = false,
|
|
44
|
+
): Spend {
|
|
45
|
+
const agents = readAgentRuns(run, strict);
|
|
46
|
+
if (strict) {
|
|
47
|
+
const invalid = (value: number | null) => value !== null && (!Number.isFinite(value) || value < 0);
|
|
48
|
+
if (
|
|
49
|
+
records.some((record) => Object.values(record.tokens).some(invalid) || invalid(record.costUsd)) ||
|
|
50
|
+
agents.some((agent) => invalid(agent.totalTokens) || invalid(agent.costUsd))
|
|
51
|
+
)
|
|
52
|
+
throw new CatherdError("E_RUN_CORRUPT", `run ${run.id} has invalid cost or token usage`);
|
|
53
|
+
}
|
|
34
54
|
const sum = <T>(xs: T[], f: (x: T) => number) => xs.reduce((n, x) => n + f(x), 0);
|
|
35
55
|
return {
|
|
36
56
|
minutes: Math.max(0, (now - Date.parse(run.meta.createdAt)) / 60_000),
|
|
37
57
|
tokens:
|
|
38
58
|
sum(records, (r) => r.tokens.input + r.tokens.output) +
|
|
39
59
|
sum(agents, (a) => a.totalTokens) +
|
|
40
|
-
sum(live, liveTokens),
|
|
60
|
+
sum(live, (dispatch) => liveTokens(dispatch, strict)),
|
|
41
61
|
usd: sum(records, (r) => r.costUsd ?? 0) + sum(agents, (a) => a.costUsd ?? 0),
|
|
42
62
|
};
|
|
43
63
|
}
|
|
@@ -99,18 +99,23 @@ export function measuredSecs(base: Catalog): Catalog["secs"] {
|
|
|
99
99
|
if (kind) add(`${canonical}|${kind}`, secs);
|
|
100
100
|
};
|
|
101
101
|
for (const run of listRuns().runs) {
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
102
|
+
// a run whose files cannot be read gives no timings; it never fails a route (1.2 minor)
|
|
103
|
+
try {
|
|
104
|
+
const routes = readRoutes(run);
|
|
105
|
+
for (const r of readRecords(run).records)
|
|
106
|
+
if (r.status === "ok") count(routes, r.rung, r.secs, r.lane, r.startedAt);
|
|
107
|
+
for (const a of readAgentRuns(run)) {
|
|
108
|
+
if (a.status !== "ok" || a.secs === null) continue;
|
|
109
|
+
// a row is written when the subagent ends; it started `secs` earlier
|
|
110
|
+
const lane = typeof a.lane === "string" ? a.lane : null;
|
|
111
|
+
const end = Date.parse(a.at);
|
|
112
|
+
// a duration past the Date range (record_agent_run takes any) keeps the end, not a RangeError
|
|
113
|
+
const start = new Date(end - a.secs * 1000);
|
|
114
|
+
const startedAt = Number.isNaN(start.getTime()) ? a.at : start.toISOString();
|
|
115
|
+
count(routes, a.rung, a.secs, lane, startedAt);
|
|
116
|
+
}
|
|
117
|
+
} catch (e) {
|
|
118
|
+
log("debug", "catalog", { run: run.id, timings: errorMessage(e) });
|
|
114
119
|
}
|
|
115
120
|
}
|
|
116
121
|
const secs: Catalog["secs"] = {};
|
|
@@ -340,6 +345,8 @@ function rungRows(
|
|
|
340
345
|
date: string;
|
|
341
346
|
/** spec 1.2 §5.3: the rung a borrowed or inferred value belongs to */
|
|
342
347
|
from?: string;
|
|
348
|
+
/** what lent it: a treat-like's mapping (`like X`) or an inferred stand-in's guess (1.2 minor) */
|
|
349
|
+
lent?: "treat-like" | "stand-in";
|
|
343
350
|
}
|
|
344
351
|
> = {};
|
|
345
352
|
for (const v of valuesUsed(c, info.canonical))
|
|
@@ -351,6 +358,7 @@ function rungRows(
|
|
|
351
358
|
source: v.source,
|
|
352
359
|
date: v.date,
|
|
353
360
|
...(v.from ? { from: v.from } : {}),
|
|
361
|
+
...(v.lent ? { lent: v.lent } : {}),
|
|
354
362
|
};
|
|
355
363
|
const like = c.treatLike[info.canonical] ?? null;
|
|
356
364
|
return {
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { existsSync } from "node:fs";
|
|
1
|
+
import { existsSync, mkdirSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import { CatherdError, isCatherdError } from "../domain/errors.ts";
|
|
5
|
+
import { withFileLockSync } from "../infra/filelock.ts";
|
|
5
6
|
import { log } from "../infra/log.ts";
|
|
6
7
|
import { configDir } from "../infra/paths.ts";
|
|
7
8
|
import { readVersioned, writeJsonAtomic } from "../infra/store.ts";
|
|
@@ -45,13 +46,17 @@ export function savedCredential(field: CredentialField): {
|
|
|
45
46
|
|
|
46
47
|
/**
|
|
47
48
|
* Keeps every other credential, and the file at mode 600 (spec §10.4). A missing file starts empty; an
|
|
48
|
-
* unreadable or newer-schema one is refused (it throws) rather than overwritten.
|
|
49
|
+
* unreadable or newer-schema one is refused (it throws) rather than overwritten. The read and the write hold
|
|
50
|
+
* the file's lock (1.2 minor), so two concurrent `init`s never lose a key.
|
|
49
51
|
*/
|
|
50
52
|
export function saveCredential(field: CredentialField, key: string): void {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
:
|
|
54
|
-
|
|
53
|
+
mkdirSync(configDir(), { recursive: true });
|
|
54
|
+
withFileLockSync(credentialsPath(), () => {
|
|
55
|
+
const cur: z.infer<typeof CredentialsSchema> = existsSync(credentialsPath())
|
|
56
|
+
? readCredentials()
|
|
57
|
+
: { schema: 1 };
|
|
58
|
+
writeJsonAtomic(credentialsPath(), { ...cur, schema: 1, [field]: key.trim() }, { mode: 0o600 });
|
|
59
|
+
});
|
|
55
60
|
}
|
|
56
61
|
|
|
57
62
|
/**
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { assertNativeHost } from "./backends.ts";
|
|
2
2
|
import { existsSync, readFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
3
4
|
import { CatherdError, errorMessage, isCatherdError } from "../domain/errors.ts";
|
|
4
5
|
import { assertId, parseRung } from "../domain/ids.ts";
|
|
5
6
|
import type { RunRecord } from "../domain/record.ts";
|
|
@@ -16,8 +17,9 @@ import {
|
|
|
16
17
|
import { lockHeld, withFileLock } from "../infra/filelock.ts";
|
|
17
18
|
import { log } from "../infra/log.ts";
|
|
18
19
|
import { isAlive, isSurelyAlive, killGroup } from "../infra/proc.ts";
|
|
20
|
+
import { groupAlive, stopGroup } from "../infra/supervisor.ts";
|
|
19
21
|
import { writeJsonAtomic } from "../infra/store.ts";
|
|
20
|
-
import { admit, KILL_GRACE_MS, laneFile, launch } from "./admission.ts";
|
|
22
|
+
import { admit, KILL_GRACE_MS, laneFile, laneOwns, launch } from "./admission.ts";
|
|
21
23
|
import { standInFor } from "./backends.ts";
|
|
22
24
|
import {
|
|
23
25
|
type Dispatch,
|
|
@@ -34,8 +36,11 @@ import {
|
|
|
34
36
|
import { finalizeDispatch, finalizingElsewhere, waitForFinish } from "./finalize.ts";
|
|
35
37
|
import type { Deps } from "./ports.ts";
|
|
36
38
|
import { route } from "./lane-service.ts";
|
|
37
|
-
import {
|
|
38
|
-
import {
|
|
39
|
+
import { currentRoute } from "../domain/route.ts";
|
|
40
|
+
import { appendRoute, findRun, readRecords, readRoleRoutes, readRoutes, type Run } from "./run-store.ts";
|
|
41
|
+
import { sessionKey } from "../domain/host.ts";
|
|
42
|
+
import { claimRun, currentSession, ownsRun, runOwner, type SessionRef } from "./sessions.ts";
|
|
43
|
+
import { pinChanges, runProfile } from "./run-pin.ts";
|
|
39
44
|
import { type NotesPatch, refreshState } from "./state.ts";
|
|
40
45
|
|
|
41
46
|
export interface DispatchInput {
|
|
@@ -43,7 +48,8 @@ export interface DispatchInput {
|
|
|
43
48
|
role: Role;
|
|
44
49
|
name: string;
|
|
45
50
|
brief: string;
|
|
46
|
-
|
|
51
|
+
/** spec 1.5 plan 24: optional with `lane` (the lane's current rung); required without one */
|
|
52
|
+
rung?: string;
|
|
47
53
|
thread?: string;
|
|
48
54
|
lane?: string;
|
|
49
55
|
next?: string;
|
|
@@ -174,9 +180,19 @@ export async function watchersSettled(): Promise<void> {
|
|
|
174
180
|
*/
|
|
175
181
|
export function watch(deps: Deps, run: Run, d: Dispatch): void {
|
|
176
182
|
watching.add(d.admit.dispatchId);
|
|
183
|
+
// the session this watcher started under, kept apart from its live identity: an MCP connection that closes
|
|
184
|
+
// invalidates the scoped host (no session), yet the watcher still came from that session (PR #47 P1)
|
|
185
|
+
const origin = currentSession(deps);
|
|
177
186
|
const w = (async () => {
|
|
178
187
|
await waitForFinish(d, { pollMs: deps.pollMs, now: deps.now, onPoll: stallPoll(run, d) });
|
|
179
|
-
const
|
|
188
|
+
const record = await finalizeDispatch(run, d);
|
|
189
|
+
// a session that lost the run since it dispatched leaves a limit to the owner's claim (1.1 push minors):
|
|
190
|
+
// only the run's owner starts a stand-in. A process with no session (a terminal) settles as before.
|
|
191
|
+
if (record.status === "limit" && !limitIsMine(deps, run, origin)) {
|
|
192
|
+
log("info", "dispatch", { run: run.id, name: d.admit.name, limit: "left to the run's owner" });
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
const s = await settle(deps, run, d, record);
|
|
180
196
|
if (s.stateHints[0]) log("warn", "dispatch", { run: run.id, name: d.admit.name, hint: s.stateHints[0] });
|
|
181
197
|
})()
|
|
182
198
|
.catch((e: unknown) =>
|
|
@@ -189,6 +205,18 @@ export function watch(deps: Deps, run: Run, d: Dispatch): void {
|
|
|
189
205
|
watchers.add(w);
|
|
190
206
|
}
|
|
191
207
|
|
|
208
|
+
/**
|
|
209
|
+
* Whether a watcher settles a limit (fails it over): a watcher with no session, then or now (a terminal, a
|
|
210
|
+
* test), always does; one that started under a session does only while that session, or its live one, owns
|
|
211
|
+
* the run.
|
|
212
|
+
*/
|
|
213
|
+
function limitIsMine(deps: Deps, run: Run, origin: SessionRef | null): boolean {
|
|
214
|
+
if (origin === null && currentSession(deps) === null) return true;
|
|
215
|
+
if (ownsRun(deps, run)) return true;
|
|
216
|
+
const owner = runOwner(run);
|
|
217
|
+
return origin !== null && owner !== null && sessionKey(owner) === sessionKey(origin);
|
|
218
|
+
}
|
|
219
|
+
|
|
192
220
|
/**
|
|
193
221
|
* Spec §3.3: a run this session has just taken over may have roles another session's server launched. This
|
|
194
222
|
* process watches each live one it is not already watching, so it settles them and its notifier announces them
|
|
@@ -214,6 +242,16 @@ export function adopt(deps: Deps, run: Run): void {
|
|
|
214
242
|
*/
|
|
215
243
|
export async function claim(deps: Deps, run: Run, o: { background?: boolean } = {}): Promise<void> {
|
|
216
244
|
if (!(await claimRun(deps, run)) && !ownsRun(deps, run)) return;
|
|
245
|
+
await recoverOwned(deps, run, o);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* What `claim` does for a run this session already owns, without taking it: watch its live roles, record and
|
|
250
|
+
* settle what finished unrecorded, settle its unsettled limits and unannounced records. A server that does not
|
|
251
|
+
* lead the boot runs it for each run its session owns (PR #47 P1: a role that finished while that server was
|
|
252
|
+
* away has no live process to watch and no record to scan).
|
|
253
|
+
*/
|
|
254
|
+
export async function recoverOwned(deps: Deps, run: Run, o: { background?: boolean } = {}): Promise<void> {
|
|
217
255
|
adopt(deps, run);
|
|
218
256
|
let r = recovering.get(run.id);
|
|
219
257
|
if (!r) {
|
|
@@ -281,24 +319,138 @@ function unannounced(run: Run): { d: Dispatch; record: RunRecord }[] {
|
|
|
281
319
|
});
|
|
282
320
|
}
|
|
283
321
|
|
|
322
|
+
/** How long a dispatch waits for another dispatch's route of the same lane (Jev may take tens of seconds). */
|
|
323
|
+
const ROUTE_LOCK_MS = 120_000;
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* What admission would refuse anyway, refused before a lane is routed, so a refused dispatch asks Jev nothing
|
|
327
|
+
* and writes no route row: the role off, the lane file missing or with no Owns:, the name or the lane already
|
|
328
|
+
* running. Admission checks them again under its lock.
|
|
329
|
+
*/
|
|
330
|
+
function refuseBeforeRouting(deps: Deps, run: Run, i: DispatchInput & { lane: string }): void {
|
|
331
|
+
const profile = runProfile(deps, run);
|
|
332
|
+
if (!profile.roles[i.role]?.enabled)
|
|
333
|
+
throw new CatherdError("E_ADMIT_RUNG", `the ${i.role} role is off in profile ${profile.name}`, {
|
|
334
|
+
fix: "skip the role, or turn it on with profile_set",
|
|
335
|
+
});
|
|
336
|
+
laneOwns(run, i.lane);
|
|
337
|
+
// live only: a finished, unrecorded dispatch is recorded by admission (finalizeFinished), not refused here
|
|
338
|
+
const pending = liveDispatches(run, deps.now());
|
|
339
|
+
const same = pending.find((d) => d.admit.name === i.name);
|
|
340
|
+
if (same)
|
|
341
|
+
throw new CatherdError("E_ADMIT_DUPLICATE", `${i.name} is already running on ${same.admit.rung}`, {
|
|
342
|
+
fix: `its record is announced when it finishes; or cancel(run, "${i.name}")`,
|
|
343
|
+
});
|
|
344
|
+
const lane = pending.find((d) => d.admit.lane === i.lane);
|
|
345
|
+
if (lane)
|
|
346
|
+
throw new CatherdError("E_ADMIT_OVERLAP", `lane ${i.lane} is already running as ${lane.admit.name}`, {
|
|
347
|
+
fix: `dispatch it after ${lane.admit.name} finishes`,
|
|
348
|
+
});
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* Plan 22: the thread a dispatch resumes. `"latest"` is the name's last recorded thread; any other id must be one
|
|
353
|
+
* the name's records in this run hold (`runs.jsonl`), in any case, so a wrong id is refused before a CLI starts
|
|
354
|
+
* and fails on it. Null for a fresh thread.
|
|
355
|
+
*/
|
|
356
|
+
function threadFor(run: Run, name: string, thread: string | undefined): string | null {
|
|
357
|
+
if (thread === undefined) return null;
|
|
358
|
+
assertId("role name", name);
|
|
359
|
+
const mine = readRecords(run)
|
|
360
|
+
.records.filter((r) => r.name === name && r.thread !== null)
|
|
361
|
+
.map((r) => r.thread as string);
|
|
362
|
+
const last = mine.at(-1);
|
|
363
|
+
if (thread === "latest") {
|
|
364
|
+
if (last) return last;
|
|
365
|
+
throw new CatherdError("E_ADMIT_THREAD", `${name} has no earlier thread in this run`, {
|
|
366
|
+
fix: "omit thread for a fresh thread",
|
|
367
|
+
});
|
|
368
|
+
}
|
|
369
|
+
const known = mine.findLast((t) => t.toLowerCase() === thread.toLowerCase());
|
|
370
|
+
if (known) return known;
|
|
371
|
+
throw new CatherdError("E_ADMIT_THREAD", `${thread} is not a thread of ${name} in this run`, {
|
|
372
|
+
fix: last
|
|
373
|
+
? `pass thread: "latest" for ${name}'s last thread (${last}), or omit thread for a fresh one`
|
|
374
|
+
: "omit thread for a fresh thread",
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/**
|
|
379
|
+
* Plan 22, resume hygiene: what an earlier turn on `thread` left running in its process group (a server, a
|
|
380
|
+
* watcher, a background command) is stopped before the thread is resumed, so it never ends the resumed CLI
|
|
381
|
+
* (exit 143). Only a dispatch whose supervisor and group leader are both gone is touched (with or without
|
|
382
|
+
* exit.json: a supervisor killed before it stopped the group writes none): a live group with no process of the
|
|
383
|
+
* leader's pid can only be that dispatch's leftovers (a pid is never reused while its group lives), and a pid
|
|
384
|
+
* that answers belongs to someone else by now. Returns a hint per group stopped.
|
|
385
|
+
*/
|
|
386
|
+
async function stopLeftovers(deps: Deps, run: Run, thread: string): Promise<string[]> {
|
|
387
|
+
const records = new Map(readRecords(run).records.map((r) => [r.dispatchId, r]));
|
|
388
|
+
const hints: string[] = [];
|
|
389
|
+
for (const d of listDispatches(run)) {
|
|
390
|
+
const on = d.admit.thread ?? records.get(d.admit.dispatchId)?.thread ?? null;
|
|
391
|
+
if (on?.toLowerCase() !== thread.toLowerCase()) continue;
|
|
392
|
+
const proc = readProc(d.dir);
|
|
393
|
+
const pgid = proc?.pgid ?? proc?.pid;
|
|
394
|
+
if (!proc || pgid === undefined || pgid !== proc.pid) continue;
|
|
395
|
+
// a live supervisor stops its own group when its worker ends; exit.json is not the test, since a supervisor
|
|
396
|
+
// killed (SIGKILL, OOM) before it stopped the group never writes one
|
|
397
|
+
if (isAlive(proc.supervisorPid, proc.supervisorStartTime)) continue;
|
|
398
|
+
if (isAlive(proc.pid, null) || !groupAlive(pgid)) continue;
|
|
399
|
+
await stopGroup(pgid, orphanLimits.killGraceMs, deps.pollMs);
|
|
400
|
+
log("info", "dispatch", { run: run.id, name: d.admit.name, thread, stoppedGroup: pgid });
|
|
401
|
+
hints.push(`resume: stopped what ${d.admit.name}'s earlier turn left running on thread ${thread}`);
|
|
402
|
+
}
|
|
403
|
+
return hints;
|
|
404
|
+
}
|
|
405
|
+
|
|
284
406
|
/**
|
|
285
407
|
* Spec §4.4, plan 9 ruling 1: admit and launch one role, start its watcher, then refresh state.md with
|
|
286
408
|
* `next` (after the launch, so nothing delays it). Returns at once; catherd announces the record (spec §3).
|
|
287
409
|
*/
|
|
288
410
|
export async function dispatch(deps: Deps, i: DispatchInput): Promise<DispatchStarted> {
|
|
289
411
|
const run = findRun(i.run);
|
|
412
|
+
// refused before the claim, so a wrong thread changes nothing
|
|
413
|
+
const thread = threadFor(run, i.name, i.thread);
|
|
290
414
|
await claim(deps, run);
|
|
291
415
|
const hints: string[] = [];
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
if (i.lane !== undefined
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
416
|
+
if (thread !== null) hints.push(...(await stopLeftovers(deps, run, thread)));
|
|
417
|
+
let rung: string;
|
|
418
|
+
if (i.lane !== undefined) {
|
|
419
|
+
const lane = i.lane;
|
|
420
|
+
assertId("lane", lane);
|
|
421
|
+
// spec 1.1 §6: a lane is routed before its first dispatch; a rung off the routed ladder starts at the routed one
|
|
422
|
+
let current = currentRoute(readRoutes(run), lane);
|
|
423
|
+
let routed: Awaited<ReturnType<typeof route>> | null = null;
|
|
424
|
+
if (!current) {
|
|
425
|
+
refuseBeforeRouting(deps, run, { ...i, lane });
|
|
426
|
+
// one route per lane: a second dispatch of the lane waits for the first's route and takes it
|
|
427
|
+
const got = await withFileLock(
|
|
428
|
+
join(run.dir, `route-${lane}`),
|
|
429
|
+
async () => {
|
|
430
|
+
const now = currentRoute(readRoutes(run), lane);
|
|
431
|
+
return now
|
|
432
|
+
? { current: now, routed: null }
|
|
433
|
+
: {
|
|
434
|
+
current: null,
|
|
435
|
+
routed: await route(deps, { run: i.run, laneFile: `lanes/${lane}.md`, role: i.role }),
|
|
436
|
+
};
|
|
437
|
+
},
|
|
438
|
+
{ timeoutMs: ROUTE_LOCK_MS },
|
|
439
|
+
);
|
|
440
|
+
current = got.current;
|
|
441
|
+
routed = got.routed;
|
|
442
|
+
}
|
|
443
|
+
// spec 1.5 plan 24: without a rung, a lane runs at its current rung (its route, after any climb)
|
|
444
|
+
rung = i.rung ?? routed?.rung ?? (current?.rung as string);
|
|
445
|
+
if (routed && i.rung !== undefined && !routed.ladder.includes(i.rung)) {
|
|
298
446
|
rung = routed.rung;
|
|
299
|
-
hints.push(`${i.rung} is not on ${
|
|
447
|
+
hints.push(`${i.rung} is not on ${lane}'s routed ladder: dispatched at ${routed.rung}`);
|
|
300
448
|
}
|
|
301
|
-
}
|
|
449
|
+
} else if (i.rung === undefined)
|
|
450
|
+
throw new CatherdError("E_INPUT_INVALID", `dispatch ${i.name}: a role outside a lane needs a rung`, {
|
|
451
|
+
fix: `route(run, role: "${i.role}") gives the role's rung; pass it as rung`,
|
|
452
|
+
});
|
|
453
|
+
else rung = i.rung;
|
|
302
454
|
// the rung that runs, after routing: an off-ladder native rung routing replaced is never launched
|
|
303
455
|
assertNativeHost(rung, deps.host.host);
|
|
304
456
|
const { d, specPath } = await admit(
|
|
@@ -309,7 +461,7 @@ export async function dispatch(deps: Deps, i: DispatchInput): Promise<DispatchSt
|
|
|
309
461
|
name: i.name,
|
|
310
462
|
brief: i.brief,
|
|
311
463
|
rung,
|
|
312
|
-
thread
|
|
464
|
+
thread,
|
|
313
465
|
lane: i.lane ?? null,
|
|
314
466
|
failoverFrom: null,
|
|
315
467
|
},
|
|
@@ -324,10 +476,46 @@ export async function dispatch(deps: Deps, i: DispatchInput): Promise<DispatchSt
|
|
|
324
476
|
// a launch that failed is still watched: its record (lost, after the start grace) is announced
|
|
325
477
|
watch(deps, run, d);
|
|
326
478
|
}
|
|
327
|
-
|
|
479
|
+
if (i.lane === undefined) recordRoleDispatch(deps, run, i.role, i.name, rung);
|
|
480
|
+
// spec 1.5: a change since the run's pin is logged in state.md each time a role starts
|
|
481
|
+
const changes = pinChanges(deps, run);
|
|
482
|
+
if (changes.length)
|
|
483
|
+
hints.push(
|
|
484
|
+
`dispatched on the run's pinned values: ${changes.join("; ")}; run_pin(run) re-pins to the repo's now`,
|
|
485
|
+
);
|
|
486
|
+
await refresh(run, { ...(i.next ? { next: i.next } : {}), pinChanges: changes }, hints);
|
|
328
487
|
return { dispatched: dispatchedOf(d), hints };
|
|
329
488
|
}
|
|
330
489
|
|
|
490
|
+
/**
|
|
491
|
+
* Spec 1.5 plan 24: a lane-less dispatch's rung in routes.jsonl, so every role's decision can be audited:
|
|
492
|
+
* `route` when it is the rung the role's last `route` gave (with that ladder), else the coordinator's own pick.
|
|
493
|
+
* A failed write is logged, never a failed dispatch: the role is already running.
|
|
494
|
+
*/
|
|
495
|
+
function recordRoleDispatch(deps: Deps, run: Run, role: Role, name: string, rung: string): void {
|
|
496
|
+
try {
|
|
497
|
+
const last = readRoleRoutes(run).findLast((r) => r.role === role && r.source === "route");
|
|
498
|
+
const routed = last?.rung === rung;
|
|
499
|
+
appendRoute(run, {
|
|
500
|
+
at: new Date(deps.now()).toISOString(),
|
|
501
|
+
lane: null,
|
|
502
|
+
role,
|
|
503
|
+
name,
|
|
504
|
+
rung,
|
|
505
|
+
ladder: last?.ladder ?? [rung],
|
|
506
|
+
source: "dispatch",
|
|
507
|
+
decidedBy: routed ? last.decidedBy : "orchestrator",
|
|
508
|
+
why: routed
|
|
509
|
+
? `the rung route gave the ${role}`
|
|
510
|
+
: last
|
|
511
|
+
? `the coordinator's rung; route gave the ${role} ${last.rung}`
|
|
512
|
+
: `the coordinator's rung; the ${role} was never routed`,
|
|
513
|
+
});
|
|
514
|
+
} catch (e) {
|
|
515
|
+
log("warn", "dispatch", { run: run.id, name, routes: errorMessage(e) });
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
|
|
331
519
|
/** How long a settle waits for another process's failover of the same dispatch (a test seam). */
|
|
332
520
|
export const failoverLock = { timeoutMs: 120_000 };
|
|
333
521
|
|
|
@@ -498,7 +686,7 @@ async function failover(deps: Deps, run: Run, d: Dispatch, limited: RunRecord):
|
|
|
498
686
|
return failedOver(already.admit.rung, already);
|
|
499
687
|
}
|
|
500
688
|
// else that stand-in never ran and is over (recorded as lost, or past its start grace): admit a new one
|
|
501
|
-
const standIn = standInFor(deps
|
|
689
|
+
const standIn = standInFor(runProfile(deps, run).failover, limited.rung, run.meta.repo);
|
|
502
690
|
if (!standIn) return { hints, started: null, pause: paused };
|
|
503
691
|
// the stand-in answers to whoever owns the run now, which a claim from another host may have changed
|
|
504
692
|
assertNativeHost(standIn, runOwner(run)?.host ?? deps.host.host);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
|
2
2
|
import { join, relative } from "node:path";
|
|
3
3
|
import { z } from "zod";
|
|
4
|
+
import { CatherdError } from "../domain/errors.ts";
|
|
4
5
|
import { dispatchHints } from "../domain/hints.ts";
|
|
5
6
|
import { ACCESS, type RunRecord } from "../domain/record.ts";
|
|
6
7
|
import { ROLES } from "../domain/roles.ts";
|
|
@@ -18,6 +19,11 @@ const AdmitSchema = z.looseObject({
|
|
|
18
19
|
role: z.enum(ROLES),
|
|
19
20
|
lane: z.string().nullable(),
|
|
20
21
|
owns: z.array(z.string()),
|
|
22
|
+
/**
|
|
23
|
+
* spec 1.5 plan 21: a laneless writer's implicit Owns (its brief's Owns: line, else docs/** and *.md), so its
|
|
24
|
+
* edits are its own and never a concurrent lane's violation; admission never refuses an overlap with it
|
|
25
|
+
*/
|
|
26
|
+
ownsImplicit: z.array(z.string()).optional(),
|
|
21
27
|
rung: z.string(),
|
|
22
28
|
backend: z.string(),
|
|
23
29
|
thread: z.string().nullable(),
|
|
@@ -54,9 +60,11 @@ export const startLimits = { graceMs: STARTING_GRACE_MS };
|
|
|
54
60
|
export const admitPath = (dir: string): string => join(dir, "admit.json");
|
|
55
61
|
export const launchPath = (dir: string): string => join(dir, "launch.json");
|
|
56
62
|
export const roleDir = (run: Run, name: string): string => join(runPaths(run.dir).roles, name);
|
|
63
|
+
/** Spec 1.5 plan 21: the role's TMPDIR, `<run>/scratch/<name>/`, removed with the run's scratch. */
|
|
64
|
+
export const scratchDir = (run: Run, name: string): string => join(runPaths(run.dir).scratch, name);
|
|
57
65
|
|
|
58
66
|
/** Every admitted dispatch of the run, oldest first; a folder whose admit.json cannot be read is skipped. */
|
|
59
|
-
export function listDispatches(run: Run): Dispatch[] {
|
|
67
|
+
export function listDispatches(run: Run, strict = false): Dispatch[] {
|
|
60
68
|
const roles = runPaths(run.dir).roles;
|
|
61
69
|
const out: Dispatch[] = [];
|
|
62
70
|
if (!existsSync(roles)) return out;
|
|
@@ -68,6 +76,8 @@ export function listDispatches(run: Run): Dispatch[] {
|
|
|
68
76
|
try {
|
|
69
77
|
out.push({ dir, admit: decodeAdmit(readVersioned(admitPath(dir), AdmitSchema, 1)) });
|
|
70
78
|
} catch {
|
|
79
|
+
if (strict && existsSync(admitPath(dir)))
|
|
80
|
+
throw new CatherdError("E_RUN_CORRUPT", `run ${run.id} has unreadable admission evidence: ${dir}`);
|
|
71
81
|
// admission never finished writing this folder
|
|
72
82
|
}
|
|
73
83
|
}
|
|
@@ -124,9 +134,10 @@ export function pendingDispatches(
|
|
|
124
134
|
run: Run,
|
|
125
135
|
now = Date.now(),
|
|
126
136
|
records: RunRecord[] = readRecords(run).records,
|
|
137
|
+
strict = false,
|
|
127
138
|
): LiveDispatch[] {
|
|
128
139
|
const recorded = new Set(records.map((r) => r.dispatchId));
|
|
129
|
-
return listDispatches(run)
|
|
140
|
+
return listDispatches(run, strict)
|
|
130
141
|
.filter((d) => !recorded.has(d.admit.dispatchId))
|
|
131
142
|
.map((d) => ({ ...d, state: dispatchState(d, now) }));
|
|
132
143
|
}
|
|
@@ -134,6 +145,23 @@ export function pendingDispatches(
|
|
|
134
145
|
export const liveDispatches = (run: Run, now = Date.now()): LiveDispatch[] =>
|
|
135
146
|
pendingDispatches(run, now).filter((d) => d.state !== "finished");
|
|
136
147
|
|
|
148
|
+
/** The thread the supervisor saw the role's CLI name (thread.json), else null. */
|
|
149
|
+
export function readThread(dir: string): string | null {
|
|
150
|
+
const t = readJson<{ thread?: unknown }>(dispatchPaths(dir).thread)?.thread;
|
|
151
|
+
return typeof t === "string" && t ? t : null;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Spec 1.5 plan 21: the role whose thread `thread` is, in this run: a recorded thread, a resumed one, or the one
|
|
156
|
+
* a live role's supervisor wrote down. Null for any other thread, the orchestrator's among them.
|
|
157
|
+
*/
|
|
158
|
+
export function roleThreadOf(run: Run, thread: string): string | null {
|
|
159
|
+
const recorded = readRecords(run).records.find((r) => r.thread === thread);
|
|
160
|
+
if (recorded) return recorded.name;
|
|
161
|
+
const d = listDispatches(run).find((x) => x.admit.thread === thread || readThread(x.dir) === thread);
|
|
162
|
+
return d ? d.admit.name : null;
|
|
163
|
+
}
|
|
164
|
+
|
|
137
165
|
/** Spec §4.1: `roles/<name>/latest` names the newest dispatch of a role. */
|
|
138
166
|
export function setLatest(run: Run, name: string, dispatchId: string): void {
|
|
139
167
|
writeTextAtomic(join(roleDir(run, name), "latest"), dispatchId);
|
|
@@ -172,9 +172,14 @@ export function workspaceWriteNetwork(profiles: Profile[]): Map<string, boolean>
|
|
|
172
172
|
|
|
173
173
|
/**
|
|
174
174
|
* The `sandbox:codex` row (which `codex sandbox` form runs) and one `access:<backend>` row per backend; a
|
|
175
|
-
* backend not in `installed` (its CLI is not on PATH) is skipped without running a probe.
|
|
175
|
+
* backend not in `installed` (its CLI is not on PATH) is skipped without running a probe. `repos`: where
|
|
176
|
+
* workers run (doctor's repo and every bound one), for a sandbox set per repo.
|
|
176
177
|
*/
|
|
177
|
-
export async function accessChecks(
|
|
178
|
+
export async function accessChecks(
|
|
179
|
+
profiles: Profile[],
|
|
180
|
+
installed: ReadonlySet<string>,
|
|
181
|
+
repos: string[] = [],
|
|
182
|
+
): Promise<Check[]> {
|
|
178
183
|
const checks: Check[] = [];
|
|
179
184
|
for (const [id, network] of workspaceWriteNetwork(profiles)) {
|
|
180
185
|
const a = adapterFor(id);
|
|
@@ -190,7 +195,7 @@ export async function accessChecks(profiles: Profile[], installed: ReadonlySet<s
|
|
|
190
195
|
continue;
|
|
191
196
|
}
|
|
192
197
|
const shell = a.accessShell
|
|
193
|
-
? await a.accessShell({ network }).catch((e: unknown) => String(e))
|
|
198
|
+
? await a.accessShell({ network, repos }).catch((e: unknown) => String(e))
|
|
194
199
|
: `no way to run a shell in ${id}'s sandbox without a model turn; the live kit (docs/dev/live-verification.md) runs the five probes as one worker turn`;
|
|
195
200
|
if (id === "codex")
|
|
196
201
|
checks.push(
|