catherd-cli 0.2.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +141 -33
- package/THIRD_PARTY_NOTICES.md +37 -0
- package/catalog/jev.json +89 -0
- package/catalog/models.json +236 -0
- package/catalog/scores.json +366 -0
- package/package.json +24 -8
- package/plugin/.claude-plugin/plugin.json +2 -2
- package/plugin/.mcp.json +2 -2
- package/plugin/bin/catherd-mcp +30 -0
- package/plugin/commands/catherd.md +2 -2
- package/plugin/skills/catherd/SKILL.md +160 -97
- package/plugin/skills/catherd-setup/SKILL.md +34 -24
- package/src/adapters/access.ts +63 -0
- package/src/adapters/all.ts +8 -0
- package/src/adapters/backend.ts +159 -0
- package/src/adapters/claude-code/events.ts +86 -0
- package/src/adapters/claude-code/index.ts +317 -0
- package/src/adapters/claude-code/models-api.ts +65 -0
- package/src/adapters/claude-code/models.ts +26 -0
- package/src/adapters/cli.ts +74 -0
- package/src/adapters/codex/events.ts +65 -0
- package/src/adapters/codex/home.ts +47 -0
- package/src/adapters/codex/index.ts +282 -0
- package/src/adapters/discovery.ts +87 -0
- package/src/adapters/opencode/agents.ts +84 -0
- package/src/adapters/opencode/events.ts +109 -0
- package/src/adapters/opencode/index.ts +333 -0
- package/src/adapters/registry.ts +14 -0
- package/src/cli.ts +159 -19
- package/src/domain/agents.ts +60 -0
- package/src/domain/budget.ts +45 -0
- package/src/domain/catalog.ts +262 -0
- package/src/domain/changes.ts +52 -0
- package/src/domain/cost.ts +89 -0
- package/src/domain/errors.ts +60 -0
- package/src/domain/failover.ts +119 -0
- package/src/domain/hints.ts +22 -0
- package/src/domain/ids.ts +97 -0
- package/src/domain/jev.ts +283 -0
- package/src/domain/lane.ts +92 -0
- package/src/domain/notice.ts +95 -0
- package/src/domain/profile-rules.ts +239 -0
- package/src/domain/profile.ts +389 -0
- package/src/domain/record.ts +86 -0
- package/src/{profile → domain}/role-prompts.ts +75 -27
- package/src/domain/roles.ts +25 -0
- package/src/domain/route.ts +126 -0
- package/src/domain/runtime.ts +21 -0
- package/src/domain/sanitize.ts +29 -0
- package/src/domain/secrets.ts +57 -0
- package/src/domain/select.ts +136 -0
- package/src/domain/state.ts +49 -0
- package/src/domain/util.ts +18 -0
- package/src/entry/capture-fixtures-command.ts +57 -0
- package/src/entry/catalog-command.ts +106 -0
- package/src/entry/cli-kit.ts +36 -0
- package/src/entry/deps.ts +17 -0
- package/src/entry/doctor-command.ts +51 -0
- package/src/entry/glyphs.ts +34 -0
- package/src/entry/init-command.ts +182 -0
- package/src/entry/lock-command.ts +123 -0
- package/src/{mcp/commands.ts → entry/mcp/command.ts} +5 -1
- package/src/entry/mcp/dispatch-tools.ts +52 -0
- package/src/entry/mcp/handshake.ts +58 -0
- package/src/entry/mcp/lane-tools.ts +84 -0
- package/src/entry/mcp/protocol-tools.ts +57 -0
- package/src/entry/mcp/result.ts +51 -0
- package/src/entry/mcp/run-tools.ts +145 -0
- package/src/entry/mcp/server.ts +88 -0
- package/src/entry/mcp/setup-tools.ts +81 -0
- package/src/entry/profile-command.ts +318 -0
- package/src/entry/prompt.ts +103 -0
- package/src/entry/runs-command.ts +260 -0
- package/src/entry/supervise-bin.ts +18 -0
- package/src/entry/supervise-command.ts +37 -0
- package/src/entry/tui/commands.ts +545 -0
- package/src/entry/tui/effects.ts +312 -0
- package/src/entry/tui/fixtures.ts +585 -0
- package/src/entry/tui/profile-edits.ts +138 -0
- package/src/entry/tui/profile-tree.ts +420 -0
- package/src/entry/tui/providers/app.tsx +162 -0
- package/src/entry/tui/providers/boundary.tsx +57 -0
- package/src/entry/tui/providers/data.tsx +160 -0
- package/src/entry/tui/providers/keymap.tsx +166 -0
- package/src/entry/tui/providers/root.tsx +45 -0
- package/src/entry/tui/providers/theme.tsx +17 -0
- package/src/entry/tui/providers/toast.tsx +77 -0
- package/src/entry/tui/run.tsx +98 -0
- package/src/entry/tui/state.ts +356 -0
- package/src/entry/tui/stories.ts +107 -0
- package/src/entry/tui/text.ts +130 -0
- package/src/entry/tui/theme.ts +140 -0
- package/src/entry/tui/views/app.tsx +208 -0
- package/src/entry/tui/views/dialogs.tsx +41 -0
- package/src/entry/tui/views/profile-actions.ts +321 -0
- package/src/entry/tui/views/profiles.tsx +440 -0
- package/src/entry/tui/views/runs.tsx +573 -0
- package/src/entry/tui/views/save-dialog.tsx +332 -0
- package/src/entry/tui/views/status.tsx +209 -0
- package/src/entry/tui/widgets/chrome.tsx +114 -0
- package/src/entry/tui/widgets/dialog-confirm.tsx +58 -0
- package/src/entry/tui/widgets/dialog-prompt.tsx +52 -0
- package/src/entry/tui/widgets/dialog-select.tsx +212 -0
- package/src/entry/tui/widgets/dialog.tsx +68 -0
- package/src/entry/tui/widgets/line.tsx +92 -0
- package/src/entry/tui/widgets/list.tsx +197 -0
- package/src/entry/tui/widgets/spinner.tsx +49 -0
- package/src/entry/tui/widgets/toast.tsx +62 -0
- package/src/infra/assets.ts +4 -0
- package/src/infra/claude-session.ts +98 -0
- package/src/infra/dispatch-dir.ts +286 -0
- package/src/infra/env.ts +67 -0
- package/src/infra/filelock.ts +159 -0
- package/src/infra/git.ts +91 -0
- package/src/infra/heavy-lock.ts +36 -0
- package/src/infra/jev-client.ts +162 -0
- package/src/infra/launch.ts +25 -0
- package/src/infra/log.ts +84 -0
- package/src/infra/paths.ts +38 -0
- package/src/infra/peer-inbox.ts +92 -0
- package/src/infra/proc.ts +110 -0
- package/src/infra/store.ts +229 -0
- package/src/infra/supervisor.ts +287 -0
- package/src/infra/version.ts +6 -0
- package/src/services/admission.ts +303 -0
- package/src/services/agent-links.ts +139 -0
- package/src/services/backends.ts +44 -0
- package/src/services/budget.ts +47 -0
- package/src/services/capture.ts +233 -0
- package/src/services/catalog-service.ts +382 -0
- package/src/services/dispatch-service.ts +620 -0
- package/src/services/dispatches.ts +174 -0
- package/src/services/doctor-access.ts +231 -0
- package/src/services/doctor-backends.ts +170 -0
- package/src/services/doctor-checks.ts +148 -0
- package/src/services/doctor-push.ts +109 -0
- package/src/services/doctor.ts +338 -0
- package/src/services/finalize.ts +377 -0
- package/src/services/gate-service.ts +288 -0
- package/src/services/global-install.ts +99 -0
- package/src/services/jev-service.ts +183 -0
- package/src/services/lane-service.ts +390 -0
- package/src/services/milestones.ts +208 -0
- package/src/services/notifier.ts +296 -0
- package/src/services/peek.ts +103 -0
- package/src/services/ports.ts +135 -0
- package/src/services/preflight.ts +221 -0
- package/src/services/profile-service.ts +292 -0
- package/src/services/profile-store.ts +141 -0
- package/src/services/protocol.ts +212 -0
- package/src/services/questions.ts +102 -0
- package/src/services/reconcile.ts +130 -0
- package/src/services/reentry.ts +22 -0
- package/src/services/routing-service.ts +190 -0
- package/src/services/run-debug.ts +79 -0
- package/src/services/run-service.ts +194 -0
- package/src/services/run-store.ts +334 -0
- package/src/services/runs-page.ts +254 -0
- package/src/services/session-view.ts +152 -0
- package/src/services/sessions.ts +92 -0
- package/src/services/setup.ts +87 -0
- package/src/services/state.ts +111 -0
- package/src/services/summary.ts +241 -0
- package/catalog/catalog.json +0 -112
- package/catalog/models-dev.json +0 -56453
- package/src/core/codex-events.ts +0 -49
- package/src/core/codex.ts +0 -166
- package/src/core/harness.ts +0 -89
- package/src/core/lanes.ts +0 -67
- package/src/core/lock.ts +0 -111
- package/src/core/opencode.ts +0 -207
- package/src/core/reconcile.ts +0 -26
- package/src/core/reply.ts +0 -54
- package/src/core/runstore.ts +0 -163
- package/src/core/spawn.ts +0 -89
- package/src/core/status.ts +0 -193
- package/src/files.ts +0 -19
- package/src/mcp/dispatch.ts +0 -154
- package/src/mcp/lane-tools.ts +0 -200
- package/src/mcp/out.ts +0 -6
- package/src/mcp/preflight.ts +0 -84
- package/src/mcp/run-tools.ts +0 -121
- package/src/mcp/runs.ts +0 -118
- package/src/mcp/server.ts +0 -26
- package/src/mcp/setup-tools.ts +0 -175
- package/src/paths.ts +0 -20
- package/src/profile/agents.ts +0 -130
- package/src/profile/profile.ts +0 -228
- package/src/routing/catalog.ts +0 -227
- package/src/routing/commands.ts +0 -22
- package/src/routing/jev.ts +0 -185
- package/src/routing/route.ts +0 -155
- package/src/routing/select.ts +0 -90
- package/src/tui/backends.ts +0 -57
- package/src/tui/commands.ts +0 -53
- package/src/tui/dashboard.tsx +0 -245
- package/src/tui/editor.tsx +0 -214
- package/src/tui/init.tsx +0 -208
- package/src/tui/matrix-model.ts +0 -279
- package/src/tui/matrix.tsx +0 -512
- package/src/tui/profiles.ts +0 -38
- package/src/tui/theme.ts +0 -177
- package/src/tui/ui.tsx +0 -267
- package/src/tui/watch-model.ts +0 -83
- package/src/tui/watch.tsx +0 -236
- package/src/types.ts +0 -184
- package/src/version.ts +0 -7
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
import type { Subprocess } from "bun";
|
|
2
|
+
import { appendFileSync, closeSync, existsSync, openSync, readSync, statSync } from "node:fs";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
import { errorMessage } from "../domain/errors.ts";
|
|
5
|
+
import type { ExitInfo, ExitReason } from "../domain/record.ts";
|
|
6
|
+
import { dispatchPaths, supervisorLockTarget } from "./dispatch-dir.ts";
|
|
7
|
+
import { tryLock } from "./filelock.ts";
|
|
8
|
+
import { log } from "./log.ts";
|
|
9
|
+
import { killGroup, processStartTime } from "./proc.ts";
|
|
10
|
+
import { PRIVATE_FILE, writeJsonAtomic } from "./store.ts";
|
|
11
|
+
|
|
12
|
+
export const SuperviseSpecSchema = z.looseObject({
|
|
13
|
+
schema: z.literal(1),
|
|
14
|
+
backend: z.string(),
|
|
15
|
+
dispatchDir: z.string(),
|
|
16
|
+
cmd: z.string(),
|
|
17
|
+
args: z.array(z.string()),
|
|
18
|
+
env: z.record(z.string(), z.string()),
|
|
19
|
+
cwd: z.string(),
|
|
20
|
+
stdinPath: z.string().nullable(),
|
|
21
|
+
idleMs: z.number().positive(),
|
|
22
|
+
wallMs: z.number().positive(),
|
|
23
|
+
killGraceMs: z.number().nonnegative(),
|
|
24
|
+
graceAfterFinalMs: z.number().nonnegative().nullable(),
|
|
25
|
+
pollMs: z.number().positive(),
|
|
26
|
+
});
|
|
27
|
+
export type SuperviseSpec = z.infer<typeof SuperviseSpecSchema>;
|
|
28
|
+
|
|
29
|
+
/** What one stream line tells the supervisor: the terminal event, the thread, a tool call opening or closing. */
|
|
30
|
+
interface LineInfo {
|
|
31
|
+
final?: boolean;
|
|
32
|
+
thread?: string;
|
|
33
|
+
/** a tool call the CLI started (`open`) or finished: while one is open the run is busy, however quiet */
|
|
34
|
+
item?: { id: string; open: boolean };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface SuperviseHooks {
|
|
38
|
+
onLine?(line: string): LineInfo;
|
|
39
|
+
/** `sinceMs`: when this run started, so a backend can ignore what an earlier run on the thread left */
|
|
40
|
+
isBusy?(thread: string | null, sinceMs: number): Promise<boolean>;
|
|
41
|
+
interrupt?(thread: string | null): Promise<void>;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Reads whatever the child appended to `file` since `offset`, as complete lines. */
|
|
45
|
+
function readNew(file: string, state: { offset: number; rest: string; decoder: TextDecoder }): string[] {
|
|
46
|
+
if (!existsSync(file)) return [];
|
|
47
|
+
const size = statSync(file).size;
|
|
48
|
+
if (size <= state.offset) return [];
|
|
49
|
+
const fd = openSync(file, "r");
|
|
50
|
+
try {
|
|
51
|
+
const buf = Buffer.alloc(size - state.offset);
|
|
52
|
+
readSync(fd, buf, 0, buf.length, state.offset);
|
|
53
|
+
state.offset = size;
|
|
54
|
+
// stream: a multi-byte character split across two polls waits in the decoder for its tail.
|
|
55
|
+
const text = state.rest + state.decoder.decode(buf, { stream: true });
|
|
56
|
+
const lines = text.split("\n");
|
|
57
|
+
state.rest = lines.pop() ?? "";
|
|
58
|
+
return lines.filter((l) => l.trim());
|
|
59
|
+
} finally {
|
|
60
|
+
closeSync(fd);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** A hook's answer, or `fallback` when it throws, rejects or takes longer than `ms`. */
|
|
65
|
+
async function bounded<T>(call: () => Promise<T> | undefined, ms: number, fallback: T): Promise<T> {
|
|
66
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
67
|
+
try {
|
|
68
|
+
const answer = call();
|
|
69
|
+
if (!answer) return fallback;
|
|
70
|
+
const late = new Promise<T>((resolve) => {
|
|
71
|
+
timer = setTimeout(() => resolve(fallback), ms);
|
|
72
|
+
});
|
|
73
|
+
return await Promise.race([answer, late]);
|
|
74
|
+
} catch {
|
|
75
|
+
return fallback;
|
|
76
|
+
} finally {
|
|
77
|
+
clearTimeout(timer);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function groupAlive(pgid: number): boolean {
|
|
82
|
+
try {
|
|
83
|
+
process.kill(-pgid, 0);
|
|
84
|
+
return true;
|
|
85
|
+
} catch (e) {
|
|
86
|
+
return (e as NodeJS.ErrnoException).code === "EPERM";
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** SIGTERM the group, give it `graceMs` to leave, then SIGKILL whatever is left of it. */
|
|
91
|
+
async function stopGroup(pgid: number, graceMs: number, pollMs: number): Promise<void> {
|
|
92
|
+
killGroup(pgid, "SIGTERM");
|
|
93
|
+
const end = Date.now() + graceMs;
|
|
94
|
+
while (Date.now() < end && groupAlive(pgid))
|
|
95
|
+
await Bun.sleep(Math.max(1, Math.min(pollMs, end - Date.now())));
|
|
96
|
+
// Always, not only when the leader lingers: a member that ignores SIGTERM outlives its leader.
|
|
97
|
+
// The group only, never the bare pid: once the leader is gone its pid may belong to someone else.
|
|
98
|
+
try {
|
|
99
|
+
process.kill(-pgid, "SIGKILL");
|
|
100
|
+
} catch {
|
|
101
|
+
// the group is gone
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Supervises one dispatch's worker, as the dispatch's only supervisor: it first takes the dispatch's
|
|
107
|
+
* supervisor lock (a dead holder's is reclaimed) and holds it for its whole life. A supervisor that finds
|
|
108
|
+
* a live one holding it returns null at once, before opening events or stderr (opened with "w") and
|
|
109
|
+
* without exit.json, so a relaunch (recovery after a collector died between spawn and launch.json) never
|
|
110
|
+
* starts a second worker.
|
|
111
|
+
*/
|
|
112
|
+
export async function supervise(spec: SuperviseSpec, hooks: SuperviseHooks = {}): Promise<ExitInfo | null> {
|
|
113
|
+
const release = tryLock(supervisorLockTarget(spec.dispatchDir));
|
|
114
|
+
if (!release) {
|
|
115
|
+
log("info", "supervise", { dispatch: spec.dispatchDir, skipped: "another supervisor holds it" });
|
|
116
|
+
return null;
|
|
117
|
+
}
|
|
118
|
+
try {
|
|
119
|
+
return await superviseHeld(spec, hooks);
|
|
120
|
+
} finally {
|
|
121
|
+
release();
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
async function superviseHeld(spec: SuperviseSpec, hooks: SuperviseHooks): Promise<ExitInfo> {
|
|
126
|
+
const p = dispatchPaths(spec.dispatchDir);
|
|
127
|
+
const finish = (info: ExitInfo): ExitInfo => {
|
|
128
|
+
writeJsonAtomic(p.exit, { schema: 1, ...info });
|
|
129
|
+
return info;
|
|
130
|
+
};
|
|
131
|
+
const fds: number[] = [];
|
|
132
|
+
let child: Subprocess;
|
|
133
|
+
try {
|
|
134
|
+
const stdin = spec.stdinPath ? openSync(spec.stdinPath, "r") : "ignore";
|
|
135
|
+
if (typeof stdin === "number") fds.push(stdin);
|
|
136
|
+
const stdout = openSync(p.events, "w", PRIVATE_FILE);
|
|
137
|
+
fds.push(stdout);
|
|
138
|
+
const stderr = openSync(p.stderr, "w", PRIVATE_FILE);
|
|
139
|
+
fds.push(stderr);
|
|
140
|
+
child = Bun.spawn([spec.cmd, ...spec.args], {
|
|
141
|
+
cwd: spec.cwd,
|
|
142
|
+
env: spec.env,
|
|
143
|
+
stdin,
|
|
144
|
+
stdout,
|
|
145
|
+
stderr,
|
|
146
|
+
detached: true,
|
|
147
|
+
});
|
|
148
|
+
log("info", "spawn", {
|
|
149
|
+
backend: spec.backend,
|
|
150
|
+
dispatch: spec.dispatchDir,
|
|
151
|
+
argv: [spec.cmd, ...spec.args],
|
|
152
|
+
env: spec.env,
|
|
153
|
+
cwd: spec.cwd,
|
|
154
|
+
pid: child.pid,
|
|
155
|
+
});
|
|
156
|
+
} catch (e) {
|
|
157
|
+
try {
|
|
158
|
+
appendFileSync(p.stderr, `catherd: could not start ${spec.cmd}: ${errorMessage(e)}\n`, {
|
|
159
|
+
mode: PRIVATE_FILE,
|
|
160
|
+
});
|
|
161
|
+
} catch {
|
|
162
|
+
// exit.json below still records that the worker never ran
|
|
163
|
+
}
|
|
164
|
+
return finish({ code: null, signal: null, reason: "lost", endedAt: new Date().toISOString() });
|
|
165
|
+
} finally {
|
|
166
|
+
for (const fd of fds) closeSync(fd);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const hookMs = Math.min(10_000, spec.idleMs);
|
|
170
|
+
let reason: ExitReason | null = null;
|
|
171
|
+
let thread: string | null = null;
|
|
172
|
+
let failure: { error: unknown } | null = null;
|
|
173
|
+
let done = false;
|
|
174
|
+
void child.exited.then(() => {
|
|
175
|
+
done = true;
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
try {
|
|
179
|
+
writeJsonAtomic(p.proc, {
|
|
180
|
+
schema: 1,
|
|
181
|
+
pid: child.pid,
|
|
182
|
+
pgid: child.pid, // the detached child leads its own group
|
|
183
|
+
startTime: processStartTime(child.pid),
|
|
184
|
+
supervisorPid: process.pid,
|
|
185
|
+
supervisorStartTime: processStartTime(process.pid),
|
|
186
|
+
startedAt: new Date().toISOString(),
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
const started = Date.now();
|
|
190
|
+
let lastActivity = started;
|
|
191
|
+
let finalAt: number | null = null;
|
|
192
|
+
// spec §3.6: a quiet stretch of half the idle timeout, not busy, is a stall, reported once per dispatch
|
|
193
|
+
let stalled = false;
|
|
194
|
+
let stallChecked = false;
|
|
195
|
+
const open = new Set<string>();
|
|
196
|
+
const stream = { offset: 0, rest: "", decoder: new TextDecoder("utf-8") };
|
|
197
|
+
|
|
198
|
+
while (!done && reason === null) {
|
|
199
|
+
await Bun.sleep(spec.pollMs);
|
|
200
|
+
const lines = readNew(p.events, stream);
|
|
201
|
+
if (lines.length) {
|
|
202
|
+
lastActivity = Date.now();
|
|
203
|
+
stallChecked = false;
|
|
204
|
+
}
|
|
205
|
+
for (const line of lines) {
|
|
206
|
+
let d: LineInfo | undefined;
|
|
207
|
+
try {
|
|
208
|
+
d = hooks.onLine?.(line);
|
|
209
|
+
} catch {
|
|
210
|
+
continue; // one line the hook cannot read must not end supervision
|
|
211
|
+
}
|
|
212
|
+
if (d?.thread) thread = d.thread;
|
|
213
|
+
if (d?.final) finalAt ??= Date.now();
|
|
214
|
+
if (d?.item) {
|
|
215
|
+
if (d.item.open) open.add(d.item.id);
|
|
216
|
+
else open.delete(d.item.id);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
// a worker that ended on its own is recorded as it ended, even if a cancel or a limit arrived meanwhile
|
|
220
|
+
if (done) break;
|
|
221
|
+
const now = Date.now();
|
|
222
|
+
if (existsSync(p.cancel)) reason = "cancelled";
|
|
223
|
+
else if (now - started >= spec.wallMs) reason = "wall-timeout";
|
|
224
|
+
else if (finalAt !== null && spec.graceAfterFinalMs !== null && now - finalAt >= spec.graceAfterFinalMs)
|
|
225
|
+
reason = "after-final";
|
|
226
|
+
else if (now - lastActivity >= spec.idleMs) {
|
|
227
|
+
const busy = open.size > 0 || (await bounded(() => hooks.isBusy?.(thread, started), hookMs, false));
|
|
228
|
+
// the busy check can take up to hookMs: a worker that ended meanwhile is recorded as it ended, read
|
|
229
|
+
// from the child itself too, since `done` is set by a callback that may not have run yet
|
|
230
|
+
if (done || child.exitCode !== null || child.signalCode !== null) break;
|
|
231
|
+
if (busy) {
|
|
232
|
+
// a new quiet stretch starts: it gets its own stall check
|
|
233
|
+
lastActivity = Date.now();
|
|
234
|
+
stallChecked = false;
|
|
235
|
+
} else reason = "idle-timeout";
|
|
236
|
+
} else if (!stalled && !stallChecked && now - lastActivity >= spec.idleMs / 2) {
|
|
237
|
+
// asked once per quiet stretch: a busy answer starts a new quiet stretch, with its own half-way check,
|
|
238
|
+
// so a worker busy here and idle later is reported stalled before the idle check ends it
|
|
239
|
+
stallChecked = true;
|
|
240
|
+
const busy = open.size > 0 || (await bounded(() => hooks.isBusy?.(thread, started), hookMs, false));
|
|
241
|
+
if (busy) {
|
|
242
|
+
lastActivity = Date.now();
|
|
243
|
+
stallChecked = false;
|
|
244
|
+
} else if (!done) {
|
|
245
|
+
stalled = true;
|
|
246
|
+
// advisory: a write that fails is logged and never ends a healthy worker
|
|
247
|
+
try {
|
|
248
|
+
writeJsonAtomic(p.stall, {
|
|
249
|
+
schema: 1,
|
|
250
|
+
at: new Date().toISOString(),
|
|
251
|
+
quietMs: Date.now() - lastActivity,
|
|
252
|
+
});
|
|
253
|
+
} catch (e) {
|
|
254
|
+
log("warn", "stall", { dispatch: spec.dispatchDir, error: errorMessage(e) });
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
} catch (e) {
|
|
260
|
+
failure = { error: e };
|
|
261
|
+
reason = "lost";
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
let stopped = false;
|
|
265
|
+
if (reason !== null && !done) {
|
|
266
|
+
if (reason !== "after-final" && failure === null && hooks.interrupt) {
|
|
267
|
+
await bounded(() => hooks.interrupt?.(thread), hookMs, undefined);
|
|
268
|
+
// a server-side stop lets the CLI end on its own: it gets the kill grace before any signal
|
|
269
|
+
await Promise.race([child.exited, Bun.sleep(spec.killGraceMs)]);
|
|
270
|
+
}
|
|
271
|
+
if (!done) {
|
|
272
|
+
await stopGroup(child.pid, spec.killGraceMs, spec.pollMs);
|
|
273
|
+
stopped = true;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
const code = await child.exited;
|
|
277
|
+
// On every path: a worker that ends on its own may leave members of its group behind.
|
|
278
|
+
if (!stopped && groupAlive(child.pid)) await stopGroup(child.pid, spec.killGraceMs, spec.pollMs);
|
|
279
|
+
const info = finish({
|
|
280
|
+
code: child.signalCode ? null : code,
|
|
281
|
+
signal: child.signalCode ?? null,
|
|
282
|
+
reason: reason ?? "exited",
|
|
283
|
+
endedAt: new Date().toISOString(),
|
|
284
|
+
});
|
|
285
|
+
if (failure) throw failure.error;
|
|
286
|
+
return info;
|
|
287
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
|
|
3
|
+
/** The package version; the MCP server reports it and every run folder records it. */
|
|
4
|
+
export const VERSION: string = (
|
|
5
|
+
JSON.parse(readFileSync(new URL("../../package.json", import.meta.url), "utf8")) as { version: string }
|
|
6
|
+
).version;
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
import { appendFileSync, existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import type { BackendAdapter } from "../adapters/backend.ts";
|
|
4
|
+
import { budgetStatus, formatBudget } from "../domain/budget.ts";
|
|
5
|
+
import { CatherdError, errorMessage } from "../domain/errors.ts";
|
|
6
|
+
import { assertId, formatRung, newDispatchId, parseRung } from "../domain/ids.ts";
|
|
7
|
+
import { assertLaneHeader, overlaps } from "../domain/lane.ts";
|
|
8
|
+
import type { RunRecord } from "../domain/record.ts";
|
|
9
|
+
import { withReplyContract } from "../domain/role-prompts.ts";
|
|
10
|
+
import type { Role } from "../domain/roles.ts";
|
|
11
|
+
import { dispatchPaths, markForCollect } from "../infra/dispatch-dir.ts";
|
|
12
|
+
import { withFileLock } from "../infra/filelock.ts";
|
|
13
|
+
import { statusSnapshot } from "../infra/git.ts";
|
|
14
|
+
import { launchSupervisor } from "../infra/launch.ts";
|
|
15
|
+
import { log } from "../infra/log.ts";
|
|
16
|
+
import { processStartTime } from "../infra/proc.ts";
|
|
17
|
+
import { ensurePrivateDir, PRIVATE_FILE, writeJsonAtomic, writeTextAtomic } from "../infra/store.ts";
|
|
18
|
+
import { readyAdapter, standInFor } from "./backends.ts";
|
|
19
|
+
import { spendOf } from "./budget.ts";
|
|
20
|
+
import {
|
|
21
|
+
type Admit,
|
|
22
|
+
admitPath,
|
|
23
|
+
type Dispatch,
|
|
24
|
+
launchPath,
|
|
25
|
+
type LiveDispatch,
|
|
26
|
+
listDispatches,
|
|
27
|
+
pendingDispatches,
|
|
28
|
+
roleDir,
|
|
29
|
+
setLatest,
|
|
30
|
+
} from "./dispatches.ts";
|
|
31
|
+
import { finalizeDispatch } from "./finalize.ts";
|
|
32
|
+
import type { Deps } from "./ports.ts";
|
|
33
|
+
import { readRecords, recordsOnThread, type Run, runPaths } from "./run-store.ts";
|
|
34
|
+
import { currentSession } from "./sessions.ts";
|
|
35
|
+
|
|
36
|
+
export interface AdmitInput {
|
|
37
|
+
role: Role;
|
|
38
|
+
name: string;
|
|
39
|
+
brief: string;
|
|
40
|
+
rung: string;
|
|
41
|
+
thread: string | null;
|
|
42
|
+
lane: string | null;
|
|
43
|
+
failoverFrom: string | null;
|
|
44
|
+
/** the dispatch id of the limited dispatch this stand-in replaces */
|
|
45
|
+
failoverOf?: string;
|
|
46
|
+
/** the dispatching session (spec §3.3); absent means the session this process serves */
|
|
47
|
+
sessionId?: string | null;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** Spec §3.3: SIGTERM, then SIGKILL this long after. */
|
|
51
|
+
export const KILL_GRACE_MS = 10_000;
|
|
52
|
+
|
|
53
|
+
export const laneFile = (run: Run, lane: string): string => join(runPaths(run.dir).lanes, `${lane}.md`);
|
|
54
|
+
|
|
55
|
+
function laneOwns(run: Run, lane: string): string[] {
|
|
56
|
+
const file = laneFile(run, lane);
|
|
57
|
+
if (!existsSync(file))
|
|
58
|
+
throw new CatherdError("E_LANE_INVALID", `no lane file lanes/${lane}.md`, {
|
|
59
|
+
fix: "write it with write_run_file first",
|
|
60
|
+
});
|
|
61
|
+
const owns = assertLaneHeader(readFileSync(file, "utf8"), `lanes/${lane}.md`).owns;
|
|
62
|
+
if (owns.length === 0)
|
|
63
|
+
throw new CatherdError("E_LANE_INVALID", `lanes/${lane}.md has no Owns: line`, {
|
|
64
|
+
fix: "add `Owns: <paths>` below the lane's title",
|
|
65
|
+
});
|
|
66
|
+
return owns;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** What a caller of `admit` does with each record `admit` itself writes (dispatch settles it). */
|
|
70
|
+
export type OnRecorded = (d: Dispatch, record: RunRecord) => Promise<unknown>;
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Records each finished dispatch of the run that has none yet, as reconcile does, so one whose waiter died
|
|
74
|
+
* never blocks its name or lane, and hands each record to `onRecorded` (the caller settles it, so its owner is
|
|
75
|
+
* told). Before the admission lock: a finalize may wait on another's claim. One that throws is skipped and
|
|
76
|
+
* stays pending, so the refusal still names it; an `onRecorded` that throws never fails the admit.
|
|
77
|
+
*/
|
|
78
|
+
async function finalizeFinished(run: Run, now: number, onRecorded?: OnRecorded): Promise<void> {
|
|
79
|
+
for (const d of pendingDispatches(run, now)) {
|
|
80
|
+
if (d.state !== "finished") continue;
|
|
81
|
+
let record: RunRecord;
|
|
82
|
+
try {
|
|
83
|
+
record = await finalizeDispatch(run, d);
|
|
84
|
+
} catch {
|
|
85
|
+
// stays pending: admission refuses its name and lane, and names it
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
try {
|
|
89
|
+
await onRecorded?.(d, record);
|
|
90
|
+
} catch (e) {
|
|
91
|
+
log("warn", "admit", { run: run.id, name: d.admit.name, error: errorMessage(e) });
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* The cap on an adapter's prepare. Each CLI call in it is bounded already (15 s); this covers the slowest
|
|
98
|
+
* bounded path (opencode: a reload, then two listings) so only a prepare that is truly stuck is refused.
|
|
99
|
+
*/
|
|
100
|
+
export const prepareLimits = { timeoutMs: 60_000 };
|
|
101
|
+
|
|
102
|
+
/** The adapter's prepare, refused with E_IO_UNEXPECTED when it has not settled within the limit. */
|
|
103
|
+
async function prepared(adapter: BackendAdapter, req: Parameters<NonNullable<BackendAdapter["prepare"]>>[0]) {
|
|
104
|
+
if (!adapter.prepare) return;
|
|
105
|
+
const ms = prepareLimits.timeoutMs;
|
|
106
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
107
|
+
const late = new Promise<never>((_, reject) => {
|
|
108
|
+
timer = setTimeout(
|
|
109
|
+
() =>
|
|
110
|
+
reject(
|
|
111
|
+
new CatherdError("E_IO_UNEXPECTED", `${adapter.id} did not get ready within ${ms} ms`, {
|
|
112
|
+
fix: `check that the ${adapter.id} CLI starts and answers, then dispatch again`,
|
|
113
|
+
}),
|
|
114
|
+
),
|
|
115
|
+
ms,
|
|
116
|
+
);
|
|
117
|
+
});
|
|
118
|
+
try {
|
|
119
|
+
await Promise.race([adapter.prepare(req), late]);
|
|
120
|
+
} finally {
|
|
121
|
+
clearTimeout(timer);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Spec §4.4 step 1. The checks that read shared state and the write that makes the dispatch live
|
|
127
|
+
* happen under the run's admission lock, so parallel dispatches always see each other (audit C1).
|
|
128
|
+
*/
|
|
129
|
+
export async function admit(
|
|
130
|
+
deps: Deps,
|
|
131
|
+
run: Run,
|
|
132
|
+
i: AdmitInput,
|
|
133
|
+
onRecorded?: OnRecorded,
|
|
134
|
+
): Promise<{ d: Dispatch; specPath: string }> {
|
|
135
|
+
assertId("role name", i.name);
|
|
136
|
+
if (i.lane !== null) assertId("lane", i.lane);
|
|
137
|
+
const rung = parseRung(i.rung);
|
|
138
|
+
const profile = deps.profiles.forRepo(run.meta.repo);
|
|
139
|
+
const rc = profile.roles[i.role];
|
|
140
|
+
if (!rc?.enabled)
|
|
141
|
+
throw new CatherdError("E_ADMIT_RUNG", `the ${i.role} role is off in profile ${profile.name}`, {
|
|
142
|
+
fix: "skip the role, or turn it on with profile_set",
|
|
143
|
+
});
|
|
144
|
+
if (rung.backend === "claude")
|
|
145
|
+
throw new CatherdError("E_ADMIT_RUNG", `${i.rung} is a native Claude rung`, {
|
|
146
|
+
fix: `run it as Agent(subagent_type: "${deps.profiles.agentFor(run.meta.repo, i.role, i.rung)}"), then record_agent_run`,
|
|
147
|
+
});
|
|
148
|
+
const allowed = new Set([
|
|
149
|
+
...rc.rungs,
|
|
150
|
+
...rc.rungs.flatMap((r) => standInFor(profile.failover, r, run.meta.repo) ?? []),
|
|
151
|
+
]);
|
|
152
|
+
if (!allowed.has(formatRung(rung)))
|
|
153
|
+
throw new CatherdError(
|
|
154
|
+
"E_ADMIT_RUNG",
|
|
155
|
+
`${i.rung} is not on the ${i.role} ladder of profile ${profile.name}`,
|
|
156
|
+
{
|
|
157
|
+
fix: `use the rung route returned; the ladder is ${rc.rungs.join(", ") || "empty"}`,
|
|
158
|
+
},
|
|
159
|
+
);
|
|
160
|
+
const { adapter, probe } = await readyAdapter(rung.backend);
|
|
161
|
+
if (i.thread !== null && !(adapter.resume.supported && adapter.resume.threadPattern.test(i.thread)))
|
|
162
|
+
throw new CatherdError("E_ADMIT_THREAD", `"${i.thread}" is not a ${adapter.id} thread id`, {
|
|
163
|
+
fix: "pass the thread from the role's earlier record, or none for a fresh thread",
|
|
164
|
+
});
|
|
165
|
+
const owns = i.lane === null ? [] : laneOwns(run, i.lane);
|
|
166
|
+
const id = newDispatchId();
|
|
167
|
+
const dir = join(roleDir(run, i.name), id);
|
|
168
|
+
const p = dispatchPaths(dir);
|
|
169
|
+
// a resumed thread lives in the home it started in (CODEX_HOME), whatever the profile says now
|
|
170
|
+
const started = i.thread === null ? undefined : recordsOnThread(run, rung.backend, i.thread).at(-1);
|
|
171
|
+
const isolated = started?.isolated ?? profile.isolated[rung.backend] ?? false;
|
|
172
|
+
await prepared(adapter, { rung, access: rc.access, isolated, repo: run.meta.repo });
|
|
173
|
+
const plan = adapter.plan({
|
|
174
|
+
rung,
|
|
175
|
+
access: rc.access,
|
|
176
|
+
network: rc.network !== false,
|
|
177
|
+
thread: i.thread,
|
|
178
|
+
isolated,
|
|
179
|
+
repo: run.meta.repo,
|
|
180
|
+
briefPath: p.brief,
|
|
181
|
+
replyPath: p.reply,
|
|
182
|
+
dispatchDir: dir,
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
await finalizeFinished(run, deps.now(), onRecorded);
|
|
186
|
+
const sessionId = i.sessionId !== undefined ? i.sessionId : (currentSession(deps)?.sessionId ?? null);
|
|
187
|
+
return withFileLock(runPaths(run.dir).admission, async () => {
|
|
188
|
+
// A dispatch blocks until its record is written, not only while it runs: its finalizer diffs the
|
|
189
|
+
// tree after the exit, so a later dispatch's writes must not land in between. A finished one here
|
|
190
|
+
// could not be recorded just now. The records and the pending dispatches are one snapshot, taken
|
|
191
|
+
// under the lock appendRecord writes under, so no record lands between the two reads.
|
|
192
|
+
const { records, pending } = await withFileLock(runPaths(run.dir).runs, () => {
|
|
193
|
+
const records = readRecords(run).records;
|
|
194
|
+
return { records, pending: pendingDispatches(run, deps.now(), records) };
|
|
195
|
+
});
|
|
196
|
+
const doing = (d: LiveDispatch): string => (d.state === "finished" ? "finished, unrecorded," : "running");
|
|
197
|
+
const same = pending.find((d) => d.admit.name === i.name);
|
|
198
|
+
if (same)
|
|
199
|
+
throw new CatherdError(
|
|
200
|
+
"E_ADMIT_DUPLICATE",
|
|
201
|
+
`${i.name} is already ${doing(same)} on ${same.admit.rung}`,
|
|
202
|
+
{
|
|
203
|
+
fix:
|
|
204
|
+
same.state === "finished"
|
|
205
|
+
? `its record could not be written; see ${same.dir}, and retry`
|
|
206
|
+
: `its record is announced when it finishes; or cancel(run, "${i.name}")`,
|
|
207
|
+
},
|
|
208
|
+
);
|
|
209
|
+
for (const d of pending) {
|
|
210
|
+
const shared = overlaps(owns, d.admit.owns);
|
|
211
|
+
if (shared.length)
|
|
212
|
+
throw new CatherdError(
|
|
213
|
+
"E_ADMIT_OVERLAP",
|
|
214
|
+
`${i.name} owns ${shared.join(", ")}, which ${d.admit.name} (${doing(d)}) owns`,
|
|
215
|
+
{
|
|
216
|
+
fix: `dispatch it after ${d.admit.name} finishes`,
|
|
217
|
+
},
|
|
218
|
+
);
|
|
219
|
+
}
|
|
220
|
+
// a finished dispatch not yet recorded still spent its tokens: every pending one counts
|
|
221
|
+
const budget = budgetStatus(spendOf(run, records, pending, deps.now()), profile.budget);
|
|
222
|
+
if (budget && budget.fraction >= 1)
|
|
223
|
+
throw new CatherdError("E_RUN_BUDGET", `the run budget is spent: ${formatBudget(budget)}`, {
|
|
224
|
+
fix: "ask the user to raise profile.budget, or finish the run with what landed",
|
|
225
|
+
});
|
|
226
|
+
const admitted: Admit = {
|
|
227
|
+
schema: 1,
|
|
228
|
+
runId: run.id,
|
|
229
|
+
dispatchId: id,
|
|
230
|
+
name: i.name,
|
|
231
|
+
role: i.role,
|
|
232
|
+
lane: i.lane,
|
|
233
|
+
owns,
|
|
234
|
+
rung: formatRung(rung),
|
|
235
|
+
backend: rung.backend,
|
|
236
|
+
thread: i.thread,
|
|
237
|
+
attempt: listDispatches(run).filter((d) => d.admit.name === i.name).length + 1,
|
|
238
|
+
failoverFrom: i.failoverFrom,
|
|
239
|
+
...(i.failoverOf ? { failoverOf: i.failoverOf } : {}),
|
|
240
|
+
access: rc.access,
|
|
241
|
+
isolated,
|
|
242
|
+
cliVersion: probe.version,
|
|
243
|
+
admittedAt: new Date(deps.now()).toISOString(),
|
|
244
|
+
repo: run.meta.repo,
|
|
245
|
+
before: await statusSnapshot(run.meta.repo),
|
|
246
|
+
...(sessionId ? { sessionId } : {}),
|
|
247
|
+
};
|
|
248
|
+
ensurePrivateDir(dir);
|
|
249
|
+
// spec 1.1 §6: every brief ends with its role's reply contract, failover stand-ins' included
|
|
250
|
+
writeTextAtomic(p.brief, withReplyContract(i.role, i.brief));
|
|
251
|
+
// Spec §10.4: the adapter's overrides only; the supervisor adds its own inherited env at spawn
|
|
252
|
+
// time (src/entry/supervise-command.ts), so no credential is ever written to disk. 0600 all the same.
|
|
253
|
+
writeJsonAtomic(
|
|
254
|
+
p.spec,
|
|
255
|
+
{
|
|
256
|
+
schema: 1,
|
|
257
|
+
backend: rung.backend,
|
|
258
|
+
dispatchDir: dir,
|
|
259
|
+
cmd: plan.cmd,
|
|
260
|
+
args: plan.args,
|
|
261
|
+
env: { ...plan.env, PWD: plan.cwd },
|
|
262
|
+
cwd: plan.cwd,
|
|
263
|
+
stdinPath: plan.stdinPath,
|
|
264
|
+
idleMs: profile.timeouts.idleMin * 60_000,
|
|
265
|
+
wallMs: profile.timeouts.wallMin * 60_000,
|
|
266
|
+
killGraceMs: KILL_GRACE_MS,
|
|
267
|
+
graceAfterFinalMs: adapter.graceAfterFinalMs,
|
|
268
|
+
pollMs: deps.pollMs,
|
|
269
|
+
},
|
|
270
|
+
{ mode: 0o600 },
|
|
271
|
+
);
|
|
272
|
+
// the collect mark before admit.json: no admitted dispatch ever exists without it, so a process that
|
|
273
|
+
// dies before launching it still leaves a record (lost, once its start grace passes), unread
|
|
274
|
+
markForCollect(dir);
|
|
275
|
+
writeJsonAtomic(admitPath(dir), admitted);
|
|
276
|
+
setLatest(run, i.name, id);
|
|
277
|
+
return { d: { dir, admit: admitted }, specPath: p.spec };
|
|
278
|
+
});
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** Starts the dispatch's supervisor and records who it is; a supervisor that cannot start ends the dispatch as lost. */
|
|
282
|
+
export function launch(d: Dispatch, specPath: string): void {
|
|
283
|
+
try {
|
|
284
|
+
const pid = launchSupervisor(specPath);
|
|
285
|
+
writeJsonAtomic(launchPath(d.dir), {
|
|
286
|
+
schema: 1,
|
|
287
|
+
supervisorPid: pid,
|
|
288
|
+
supervisorStartTime: processStartTime(pid),
|
|
289
|
+
});
|
|
290
|
+
} catch (e) {
|
|
291
|
+
const p = dispatchPaths(d.dir);
|
|
292
|
+
appendFileSync(p.stderr, `catherd: could not start the supervisor: ${(e as Error).message}\n`, {
|
|
293
|
+
mode: PRIVATE_FILE,
|
|
294
|
+
});
|
|
295
|
+
writeJsonAtomic(p.exit, {
|
|
296
|
+
schema: 1,
|
|
297
|
+
code: null,
|
|
298
|
+
signal: null,
|
|
299
|
+
reason: "lost",
|
|
300
|
+
endedAt: new Date().toISOString(),
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
}
|