catherd-cli 1.4.0 → 2.0.0-beta.20261022113309
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +188 -169
- package/THIRD_PARTY_NOTICES.md +217 -35
- package/catalog/jev.json +27 -1
- package/catalog/models.json +56 -239
- package/package.json +6 -8
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/.codex-plugin/plugin.json +1 -1
- package/plugin/bin/catherd-mcp +1 -1
- package/plugin/commands/catherd.md +2 -2
- package/plugin/skills/catherd/SKILL.md +97 -356
- package/plugin/skills/catherd-setup/SKILL.md +21 -20
- package/src/adapters/all.ts +0 -4
- package/src/adapters/backend.ts +19 -4
- package/src/adapters/claude-code/index.ts +126 -14
- package/src/adapters/codex/events.ts +10 -0
- package/src/adapters/codex/index.ts +17 -5
- package/src/adapters/cursor/home.ts +1 -1
- package/src/adapters/cursor/index.ts +10 -4
- package/src/adapters/opencode/index.ts +38 -1
- package/src/cli.ts +27 -27
- package/src/domain/agents.ts +3 -1
- package/src/domain/bars.ts +33 -107
- package/src/domain/blocked.ts +72 -0
- package/src/domain/branch.ts +51 -0
- package/src/domain/brief.ts +106 -0
- package/src/domain/catalog.ts +69 -313
- package/src/domain/changes.ts +23 -1
- package/src/domain/cost.ts +19 -29
- package/src/domain/errors.ts +6 -2
- package/src/domain/fail-lines.ts +26 -0
- package/src/domain/failover.ts +37 -75
- package/src/domain/findings.ts +47 -0
- package/src/domain/gate-brief.ts +53 -0
- package/src/domain/hints.ts +13 -2
- package/src/domain/ids.ts +13 -2
- package/src/domain/jev.ts +47 -2
- package/src/domain/judge.ts +124 -0
- package/src/domain/lane.ts +118 -3
- package/src/domain/milestone-memory.ts +53 -0
- package/src/domain/notice.ts +32 -8
- package/src/domain/notify.ts +32 -0
- package/src/domain/plan-doc.ts +198 -0
- package/src/domain/plan-lint.ts +141 -0
- package/src/domain/profile-rules.ts +175 -138
- package/src/domain/profile.ts +140 -63
- package/src/domain/record.ts +68 -3
- package/src/domain/reply.ts +29 -0
- package/src/domain/role-prompts.ts +182 -56
- package/src/domain/role-scope.ts +76 -0
- package/src/domain/role-tools.ts +34 -0
- package/src/domain/roles.ts +2 -0
- package/src/domain/route.ts +90 -0
- package/src/domain/rules.ts +33 -0
- package/src/domain/run-events.ts +298 -0
- package/src/domain/run-machine.ts +668 -0
- package/src/domain/run-state.ts +1310 -0
- package/src/domain/scorecard.ts +197 -0
- package/src/domain/select.ts +229 -72
- package/src/domain/slopalytics.ts +72 -0
- package/src/domain/state.ts +5 -0
- package/src/domain/util.ts +22 -0
- package/src/domain/workspace.ts +130 -0
- package/src/entry/answer-command.ts +39 -0
- package/src/entry/catalog-command.ts +24 -165
- package/src/entry/check-bin.ts +17 -0
- package/src/entry/cli-kit.ts +18 -2
- package/src/entry/daemon-bin.ts +22 -0
- package/src/entry/deps.ts +2 -0
- package/src/entry/doctor-command.ts +20 -2
- package/src/entry/glyphs.ts +15 -2
- package/src/entry/host-arg.ts +28 -0
- package/src/entry/init-command.ts +20 -69
- package/src/entry/knowledge-command.ts +121 -5
- package/src/entry/lock-command.ts +103 -8
- package/src/entry/mcp/handshake.ts +6 -2
- package/src/entry/mcp/role-bin.ts +20 -0
- package/src/entry/mcp/role-server.ts +134 -0
- package/src/entry/mcp/server.ts +85 -21
- package/src/entry/mcp/session-tools.ts +149 -0
- package/src/entry/panel/openapi.ts +189 -0
- package/src/entry/panel/server.ts +410 -0
- package/src/entry/panel-command.ts +68 -0
- package/src/entry/pause-command.ts +67 -0
- package/src/entry/profile-command.ts +33 -18
- package/src/entry/role-cli-command.ts +159 -0
- package/src/entry/runs-command.ts +81 -27
- package/src/entry/scorecard-command.ts +187 -0
- package/src/entry/supervise-bin.ts +1 -1
- package/src/entry/supervise-command.ts +30 -3
- package/src/entry/workspace-command.ts +44 -0
- package/src/infra/boot-lock.ts +25 -0
- package/src/infra/delivery.ts +3 -0
- package/src/infra/dispatch-dir.ts +2 -0
- package/src/infra/env.ts +11 -3
- package/src/infra/git.ts +70 -0
- package/src/infra/heavy-lock.ts +133 -6
- package/src/infra/host-probe.ts +16 -0
- package/src/infra/launch.ts +63 -7
- package/src/infra/lock-activity.ts +88 -0
- package/src/infra/login-env.ts +48 -0
- package/src/infra/memory.ts +45 -0
- package/src/infra/ntfy.ts +40 -0
- package/src/infra/open-url.ts +42 -0
- package/src/infra/paths.ts +37 -2
- package/src/infra/role-mcp.ts +69 -0
- package/src/infra/slopalytics.ts +30 -0
- package/src/infra/store.ts +6 -1
- package/src/infra/supervisor.ts +107 -16
- package/src/services/admission.ts +133 -31
- package/src/services/backends.ts +4 -6
- package/src/services/budget.ts +26 -6
- package/src/services/capture.ts +1 -22
- package/src/services/catalog-service.ts +155 -254
- package/src/services/credentials.ts +15 -26
- package/src/services/daemon.ts +181 -0
- package/src/services/dispatch-service.ts +209 -17
- package/src/services/dispatches.ts +30 -2
- package/src/services/doctor-access.ts +8 -3
- package/src/services/doctor-backends.ts +1 -1
- package/src/services/doctor-checks.ts +47 -0
- package/src/services/doctor-docker.ts +221 -0
- package/src/services/doctor-role-mcp.ts +65 -0
- package/src/services/doctor.ts +72 -15
- package/src/services/end-push.ts +157 -0
- package/src/services/failover-rungs.ts +109 -0
- package/src/services/finalize.ts +24 -2
- package/src/services/gate-env.ts +217 -0
- package/src/services/gate-service.ts +231 -45
- package/src/services/global-install.ts +46 -13
- package/src/services/jev-service.ts +2 -4
- package/src/services/lane-edit.ts +154 -0
- package/src/services/lane-service.ts +291 -55
- package/src/services/milestone-check.ts +433 -0
- package/src/services/milestone-tree.ts +170 -0
- package/src/services/milestones.ts +167 -29
- package/src/services/notifier.ts +124 -4
- package/src/services/notify-service.ts +26 -0
- package/src/services/orchestrator-wait.ts +82 -0
- package/src/services/panel-setup.ts +116 -0
- package/src/services/pause.ts +161 -0
- package/src/services/peek.ts +79 -17
- package/src/services/plan-input.ts +151 -0
- package/src/services/ports.ts +60 -5
- package/src/services/preflight.ts +232 -24
- package/src/services/profile-service.ts +31 -3
- package/src/services/profile-store.ts +18 -6
- package/src/services/protocol.ts +123 -25
- package/src/services/provenance.ts +34 -89
- package/src/services/questions.ts +24 -0
- package/src/services/reconcile.ts +33 -1
- package/src/services/reentry.ts +7 -6
- package/src/services/role-access.ts +27 -0
- package/src/services/routing-service.ts +161 -42
- package/src/services/run-briefs.ts +477 -0
- package/src/services/run-client.ts +217 -0
- package/src/services/run-effects.ts +837 -0
- package/src/services/run-events-store.ts +174 -0
- package/src/services/run-evidence.ts +1 -1
- package/src/services/run-pin.ts +125 -0
- package/src/services/run-read.ts +221 -0
- package/src/services/run-report.ts +176 -0
- package/src/services/run-service.ts +204 -23
- package/src/services/run-store.ts +159 -18
- package/src/services/scorecard-metrics.ts +86 -0
- package/src/services/scorecard.ts +382 -0
- package/src/services/scratch.ts +60 -0
- package/src/services/sessions.ts +3 -1
- package/src/services/state.ts +21 -0
- package/src/services/summary.ts +76 -12
- package/src/services/tree-path.ts +27 -0
- package/src/services/verifier-brief.ts +23 -0
- package/src/services/verifier-step.ts +48 -0
- package/src/services/workspace-admission.ts +259 -0
- package/src/services/workspace-service.ts +259 -0
- package/src/services/workspace-store.ts +95 -0
- package/web/dist/assets/Brand-PKlUX5TE.js +1 -0
- package/web/dist/assets/Brand-PKlUX5TE.js.br +0 -0
- package/web/dist/assets/Brand-PKlUX5TE.js.gz +0 -0
- package/web/dist/assets/Modal-Dxc5so53.js +1 -0
- package/web/dist/assets/Modal-Dxc5so53.js.br +0 -0
- package/web/dist/assets/Modal-Dxc5so53.js.gz +0 -0
- package/web/dist/assets/Pressable-C8-T6Ed_.js +1 -0
- package/web/dist/assets/StatusChip-B5xfI_Mn.js +153 -0
- package/web/dist/assets/StatusChip-B5xfI_Mn.js.br +0 -0
- package/web/dist/assets/StatusChip-B5xfI_Mn.js.gz +0 -0
- package/web/dist/assets/_app-Ea5cpyKe.js +1 -0
- package/web/dist/assets/_app-Ea5cpyKe.js.br +0 -0
- package/web/dist/assets/_app-Ea5cpyKe.js.gz +0 -0
- package/web/dist/assets/_runId-pTG0fXVo.js +1 -0
- package/web/dist/assets/_runId-pTG0fXVo.js.br +0 -0
- package/web/dist/assets/_runId-pTG0fXVo.js.gz +0 -0
- package/web/dist/assets/drawer-Du3cdlok.js +1 -0
- package/web/dist/assets/drawer-Du3cdlok.js.br +0 -0
- package/web/dist/assets/drawer-Du3cdlok.js.gz +0 -0
- package/web/dist/assets/geist-cyrillic-ext-wght-normal-DjL33-gN.woff2 +0 -0
- package/web/dist/assets/geist-cyrillic-wght-normal-BEAKL7Jp.woff2 +0 -0
- package/web/dist/assets/geist-latin-ext-wght-normal-DC-KSUi6.woff2 +0 -0
- package/web/dist/assets/geist-latin-wght-normal-BgDaEnEv.woff2 +0 -0
- package/web/dist/assets/geist-mono-cyrillic-ext-wght-normal-X_5orZeX.woff2 +0 -0
- package/web/dist/assets/geist-mono-cyrillic-wght-normal-DiZS0aHC.woff2 +0 -0
- package/web/dist/assets/geist-mono-latin-ext-wght-normal-Bwz-egvJ.woff2 +0 -0
- package/web/dist/assets/geist-mono-latin-wght-normal-XN7g48iV.woff2 +0 -0
- package/web/dist/assets/geist-mono-symbols2-wght-normal-CO5SzqOn.woff2 +0 -0
- package/web/dist/assets/geist-mono-vietnamese-wght-normal-DadHysG0.woff2 +0 -0
- package/web/dist/assets/geist-vietnamese-wght-normal-6IgcOCM7.woff2 +0 -0
- package/web/dist/assets/index-B6GiwDkh.js +11 -0
- package/web/dist/assets/index-B6GiwDkh.js.br +0 -0
- package/web/dist/assets/index-B6GiwDkh.js.gz +0 -0
- package/web/dist/assets/index-C5IbylvM.css +2 -0
- package/web/dist/assets/index-C5IbylvM.css.br +0 -0
- package/web/dist/assets/index-C5IbylvM.css.gz +0 -0
- package/web/dist/assets/label-DFSYBc62.js +1 -0
- package/web/dist/assets/label-DFSYBc62.js.br +0 -0
- package/web/dist/assets/label-DFSYBc62.js.gz +0 -0
- package/web/dist/assets/link-D8EcKZUt.js +1 -0
- package/web/dist/assets/link-D8EcKZUt.js.br +0 -0
- package/web/dist/assets/link-D8EcKZUt.js.gz +0 -0
- package/web/dist/assets/login-DaN5HPPW.js +1 -0
- package/web/dist/assets/login-DaN5HPPW.js.br +0 -0
- package/web/dist/assets/login-DaN5HPPW.js.gz +0 -0
- package/web/dist/assets/query-Cu--ccT3.js +1 -0
- package/web/dist/assets/questions-shoO9OVj.js +1 -0
- package/web/dist/assets/questions-shoO9OVj.js.br +0 -0
- package/web/dist/assets/questions-shoO9OVj.js.gz +0 -0
- package/web/dist/assets/react-query.gen-Bu2yV-qJ.js +12 -0
- package/web/dist/assets/react-query.gen-Bu2yV-qJ.js.br +0 -0
- package/web/dist/assets/react-query.gen-Bu2yV-qJ.js.gz +0 -0
- package/web/dist/assets/rolldown-runtime-XKyaHtl9.js +1 -0
- package/web/dist/assets/run-status-C0DRuV7U.js +1 -0
- package/web/dist/assets/runs-C3rg2zZc.js +1 -0
- package/web/dist/assets/runs-C3rg2zZc.js.br +0 -0
- package/web/dist/assets/runs-C3rg2zZc.js.gz +0 -0
- package/web/dist/assets/setup-5I4Vl9fR.js +1 -0
- package/web/dist/assets/setup-5I4Vl9fR.js.br +0 -0
- package/web/dist/assets/setup-5I4Vl9fR.js.gz +0 -0
- package/web/dist/assets/table-BOxXhqS4.js +1 -0
- package/web/dist/assets/table-BOxXhqS4.js.br +0 -0
- package/web/dist/assets/table-BOxXhqS4.js.gz +0 -0
- package/web/dist/assets/textfield-BzRFkxBF.js +1 -0
- package/web/dist/assets/textfield-BzRFkxBF.js.br +0 -0
- package/web/dist/assets/textfield-BzRFkxBF.js.gz +0 -0
- package/web/dist/assets/time--9kSQbbp.js +1 -0
- package/web/dist/assets/use-css-variable-CpnN9WtA.js +5 -0
- package/web/dist/assets/use-css-variable-CpnN9WtA.js.br +0 -0
- package/web/dist/assets/use-css-variable-CpnN9WtA.js.gz +0 -0
- package/web/dist/assets/useMutation-CVPKlR34.js +1 -0
- package/web/dist/assets/useMutation-CVPKlR34.js.br +0 -0
- package/web/dist/assets/useMutation-CVPKlR34.js.gz +0 -0
- package/web/dist/index.html +15 -0
- package/catalog/ATTRIBUTION.md +0 -36
- package/catalog/scores.json +0 -3202
- package/catalog/sources.json +0 -97
- package/src/adapters/antigravity/events.ts +0 -131
- package/src/adapters/antigravity/home.ts +0 -46
- package/src/adapters/antigravity/index.ts +0 -320
- package/src/adapters/antigravity/models.ts +0 -37
- package/src/adapters/grok/events.ts +0 -112
- package/src/adapters/grok/home.ts +0 -132
- package/src/adapters/grok/index.ts +0 -339
- package/src/adapters/grok/models.ts +0 -48
- package/src/domain/calibration.ts +0 -125
- package/src/domain/sources.ts +0 -144
- package/src/entry/mcp/dispatch-tools.ts +0 -52
- package/src/entry/mcp/lane-tools.ts +0 -84
- package/src/entry/mcp/protocol-tools.ts +0 -57
- package/src/entry/mcp/run-tools.ts +0 -148
- package/src/entry/mcp/setup-tools.ts +0 -108
- package/src/entry/tui/commands.ts +0 -563
- package/src/entry/tui/effects.ts +0 -381
- package/src/entry/tui/fixtures.ts +0 -637
- package/src/entry/tui/profile-edits.ts +0 -146
- package/src/entry/tui/profile-tree.ts +0 -429
- package/src/entry/tui/providers/app.tsx +0 -162
- package/src/entry/tui/providers/boundary.tsx +0 -57
- package/src/entry/tui/providers/data.tsx +0 -160
- package/src/entry/tui/providers/keymap.tsx +0 -166
- package/src/entry/tui/providers/root.tsx +0 -45
- package/src/entry/tui/providers/theme.tsx +0 -17
- package/src/entry/tui/providers/toast.tsx +0 -77
- package/src/entry/tui/run.tsx +0 -114
- package/src/entry/tui/state.ts +0 -373
- package/src/entry/tui/stories.ts +0 -114
- package/src/entry/tui/text.ts +0 -130
- package/src/entry/tui/theme.ts +0 -140
- package/src/entry/tui/views/app.tsx +0 -208
- package/src/entry/tui/views/dialogs.tsx +0 -41
- package/src/entry/tui/views/profile-actions.ts +0 -331
- package/src/entry/tui/views/profiles.tsx +0 -520
- package/src/entry/tui/views/runs.tsx +0 -573
- package/src/entry/tui/views/save-dialog.tsx +0 -340
- package/src/entry/tui/views/status.tsx +0 -209
- package/src/entry/tui/widgets/chrome.tsx +0 -114
- package/src/entry/tui/widgets/dialog-confirm.tsx +0 -58
- package/src/entry/tui/widgets/dialog-prompt.tsx +0 -52
- package/src/entry/tui/widgets/dialog-select.tsx +0 -212
- package/src/entry/tui/widgets/dialog.tsx +0 -68
- package/src/entry/tui/widgets/line.tsx +0 -92
- package/src/entry/tui/widgets/list.tsx +0 -197
- package/src/entry/tui/widgets/spinner.tsx +0 -49
- package/src/entry/tui/widgets/toast.tsx +0 -62
- package/src/infra/sources/arena.ts +0 -87
- package/src/infra/sources/artificial-analysis.ts +0 -112
- package/src/infra/sources/cache.ts +0 -124
- package/src/infra/sources/csv.ts +0 -38
- package/src/infra/sources/epoch.ts +0 -82
- package/src/infra/sources/http.ts +0 -75
- package/src/infra/sources/litellm.ts +0 -58
- package/src/infra/sources/models-dev.ts +0 -59
- package/src/infra/sources/openrouter-endpoints.ts +0 -52
- package/src/infra/sources/openrouter-models.ts +0 -60
- package/src/infra/sources/rows.ts +0 -38
- package/src/infra/sources/vectara.ts +0 -69
- package/src/infra/sources/zip.ts +0 -48
- package/src/services/catalog-refresh.ts +0 -201
- package/src/services/doctor-sources.ts +0 -76
- package/src/services/runs-page.ts +0 -255
- package/src/services/source-derive.ts +0 -374
- package/src/services/source-sync.ts +0 -305
- package/src/services/standins.ts +0 -204
- package/src/services/treat-likes.ts +0 -97
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
import { errorMessage } from "../domain/errors.ts";
|
|
5
|
+
import { launchDaemon } from "../infra/launch.ts";
|
|
6
|
+
import type { SuperviseClock } from "../infra/supervisor.ts";
|
|
7
|
+
import { lockHeld, tryLock } from "../infra/filelock.ts";
|
|
8
|
+
import { log } from "../infra/log.ts";
|
|
9
|
+
import { processStartTime } from "../infra/proc.ts";
|
|
10
|
+
import { readVersioned, writeJsonAtomic } from "../infra/store.ts";
|
|
11
|
+
import type { Deps } from "./ports.ts";
|
|
12
|
+
import type { RunEvent } from "../domain/run-events.ts";
|
|
13
|
+
import { type Action, actionKey, confirms, failKey, next } from "../domain/run-machine.ts";
|
|
14
|
+
import { reduce, replay, type RunState } from "../domain/run-state.ts";
|
|
15
|
+
import { type Effects, realEffects } from "./run-effects.ts";
|
|
16
|
+
import { appendEvent, eventsFile, isDaemonRun, readEvents } from "./run-events-store.ts";
|
|
17
|
+
import { findRun, type Run } from "./run-store.ts";
|
|
18
|
+
|
|
19
|
+
/** The daemon holds this lock for its whole life: a second holder gets none, a dead one's is reclaimed. */
|
|
20
|
+
export const daemonLockTarget = (run: Run): string => join(run.dir, "daemon");
|
|
21
|
+
export const heartbeatFile = (run: Run): string => join(run.dir, "daemon.json");
|
|
22
|
+
|
|
23
|
+
/** `beatMs`: how often the heartbeat is written; `pollMs`: how often the loop reads the events. Tests shorten them. */
|
|
24
|
+
export const daemonLimits = { beatMs: 5_000, pollMs: 500 };
|
|
25
|
+
|
|
26
|
+
export interface Heartbeat {
|
|
27
|
+
schema: 1;
|
|
28
|
+
pid: number;
|
|
29
|
+
startTime: string | null;
|
|
30
|
+
beatAt: string;
|
|
31
|
+
seq: number;
|
|
32
|
+
version: string;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const HeartbeatSchema = z.looseObject({
|
|
36
|
+
schema: z.literal(1),
|
|
37
|
+
pid: z.number().int(),
|
|
38
|
+
startTime: z.string().nullable(),
|
|
39
|
+
beatAt: z.string(),
|
|
40
|
+
seq: z.number().int(),
|
|
41
|
+
version: z.string(),
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
/** The daemon's last beat, or null when it never wrote one or the file cannot be read. */
|
|
45
|
+
export function readHeartbeat(run: Run): Heartbeat | null {
|
|
46
|
+
try {
|
|
47
|
+
return readVersioned(heartbeatFile(run), HeartbeatSchema, 1);
|
|
48
|
+
} catch {
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Whether a live process holds the run's daemon lock: its pid and start time, never the heartbeat. */
|
|
54
|
+
export const daemonAlive = (run: Run): boolean => lockHeld(daemonLockTarget(run));
|
|
55
|
+
|
|
56
|
+
/** How the daemon is started; tests replace it so no process is spawned. */
|
|
57
|
+
export const daemonLauncher = { launch: launchDaemon };
|
|
58
|
+
|
|
59
|
+
/** Whether the run's events say it ended (`run.ended`). A `"type":"run.ended"` never occurs inside a JSON string. */
|
|
60
|
+
function hasEnded(run: Run): boolean {
|
|
61
|
+
try {
|
|
62
|
+
return readFileSync(eventsFile(run), "utf8").includes('"type":"run.ended"');
|
|
63
|
+
} catch {
|
|
64
|
+
return false;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Makes sure a 2.0 run that has not ended has a daemon: one that is alive is left alone, a dead one is
|
|
70
|
+
* started again (two callers at once start two processes, and the lock lets one run). A 1.5 run has none.
|
|
71
|
+
*/
|
|
72
|
+
export function ensureDaemon(run: Run): "alive" | "revived" | "ended" | "1.5" {
|
|
73
|
+
if (!isDaemonRun(run)) return "1.5";
|
|
74
|
+
if (hasEnded(run)) return "ended";
|
|
75
|
+
if (daemonAlive(run)) return "alive";
|
|
76
|
+
daemonLauncher.launch(run.id, run.dir);
|
|
77
|
+
return "revived";
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Writes the heartbeat: who holds the lock, since when, and how far it has read. */
|
|
81
|
+
export function beat(run: Run, deps: Deps, seq: number): void {
|
|
82
|
+
const pid = process.pid;
|
|
83
|
+
writeJsonAtomic(heartbeatFile(run), {
|
|
84
|
+
schema: 1,
|
|
85
|
+
pid,
|
|
86
|
+
startTime: processStartTime(pid),
|
|
87
|
+
beatAt: new Date(deps.now()).toISOString(),
|
|
88
|
+
seq,
|
|
89
|
+
version: deps.version,
|
|
90
|
+
} satisfies Heartbeat);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export interface DaemonOptions {
|
|
94
|
+
/** what carries the machine's actions out; the real services when absent */
|
|
95
|
+
effects?: Effects;
|
|
96
|
+
/** the loop's time; the real clock when absent */
|
|
97
|
+
clock?: SuperviseClock;
|
|
98
|
+
/** ends the loop early (tests) */
|
|
99
|
+
until?: () => boolean;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const REAL_CLOCK: SuperviseClock = { now: () => Date.now(), sleep: (ms) => Bun.sleep(ms) };
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Drives one run: takes its lock (`"lost-lock"` without it), returns `"ended"` at once for a run that ended,
|
|
106
|
+
* else replays the events, announces itself, and then feeds each new event to the machine: it clears the keys
|
|
107
|
+
* the event confirms, runs each action of `next` it has not issued (not awaited), appends one `tick` when a
|
|
108
|
+
* hold or wait has passed and beats. It leaves, releasing the lock, once the run has ended and every effect
|
|
109
|
+
* is done. The effects append the events, so a crash loses nothing: a new daemon replays and issues again
|
|
110
|
+
* what never got its event.
|
|
111
|
+
*/
|
|
112
|
+
export async function runDaemon(
|
|
113
|
+
deps: Deps,
|
|
114
|
+
runId: string,
|
|
115
|
+
o: DaemonOptions = {},
|
|
116
|
+
): Promise<"ended" | "lost-lock"> {
|
|
117
|
+
const run = findRun(runId);
|
|
118
|
+
const release = tryLock(daemonLockTarget(run));
|
|
119
|
+
if (!release) return "lost-lock";
|
|
120
|
+
try {
|
|
121
|
+
if (!isDaemonRun(run) || hasEnded(run)) return "ended";
|
|
122
|
+
const clock = o.clock ?? REAL_CLOCK;
|
|
123
|
+
const effects = o.effects ?? realEffects(deps, run);
|
|
124
|
+
let state: RunState = replay(readEvents(run), run.id);
|
|
125
|
+
const hello = await appendEvent(
|
|
126
|
+
run,
|
|
127
|
+
{ type: "daemon.started", pid: process.pid, version: deps.version },
|
|
128
|
+
new Date(clock.now()),
|
|
129
|
+
);
|
|
130
|
+
log("info", "daemon", { run: run.id, pid: process.pid, seq: hello.seq });
|
|
131
|
+
beat(run, deps, hello.seq);
|
|
132
|
+
/** action key → the key `effect.failed` frees it by */
|
|
133
|
+
const issued = new Map<string, string>();
|
|
134
|
+
const inFlight = new Set<Promise<void>>();
|
|
135
|
+
const issue = (a: Action): void => {
|
|
136
|
+
const key = actionKey(a);
|
|
137
|
+
if (issued.has(key)) return;
|
|
138
|
+
issued.set(key, failKey(a, state));
|
|
139
|
+
const p: Promise<void> = effects
|
|
140
|
+
.run(a, state)
|
|
141
|
+
.catch((e: unknown) =>
|
|
142
|
+
log("error", "daemon", { run: run.id, action: a.type, error: errorMessage(e) }),
|
|
143
|
+
)
|
|
144
|
+
.finally(() => {
|
|
145
|
+
inFlight.delete(p);
|
|
146
|
+
// a notice, and a merge check that found nothing, leave no event behind: nothing else would free their key
|
|
147
|
+
if (a.type === "notify" || a.type === "checkMerged") issued.delete(key);
|
|
148
|
+
});
|
|
149
|
+
inFlight.add(p);
|
|
150
|
+
};
|
|
151
|
+
const take = (e: RunEvent): void => {
|
|
152
|
+
state = reduce(state, e);
|
|
153
|
+
for (const k of confirms(e))
|
|
154
|
+
for (const [key, fk] of issued) if (key === k || fk === k) issued.delete(key);
|
|
155
|
+
for (const a of next(state, e)) issue(a);
|
|
156
|
+
};
|
|
157
|
+
let beatAt = clock.now();
|
|
158
|
+
while (!o.until?.()) {
|
|
159
|
+
const fresh = readEvents(run, state.seq + 1);
|
|
160
|
+
for (const e of fresh) take(e);
|
|
161
|
+
if (fresh.length > 0) continue;
|
|
162
|
+
if (state.wakeAt !== null && Date.parse(state.wakeAt) <= clock.now()) {
|
|
163
|
+
await appendEvent(run, { type: "tick" }, new Date(clock.now()));
|
|
164
|
+
continue;
|
|
165
|
+
}
|
|
166
|
+
if (clock.now() - beatAt >= daemonLimits.beatMs) {
|
|
167
|
+
beat(run, deps, state.seq);
|
|
168
|
+
beatAt = clock.now();
|
|
169
|
+
}
|
|
170
|
+
if (state.phase === "ended" && inFlight.size === 0) break;
|
|
171
|
+
await clock.sleep(daemonLimits.pollMs);
|
|
172
|
+
}
|
|
173
|
+
beat(run, deps, Math.max(state.seq, hello.seq));
|
|
174
|
+
return "ended";
|
|
175
|
+
} catch (e) {
|
|
176
|
+
log("error", "daemon", { run: runId, error: errorMessage(e) });
|
|
177
|
+
throw e;
|
|
178
|
+
} finally {
|
|
179
|
+
release();
|
|
180
|
+
}
|
|
181
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { assertNativeHost } from "./backends.ts";
|
|
2
2
|
import { existsSync, readFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
3
4
|
import { CatherdError, errorMessage, isCatherdError } from "../domain/errors.ts";
|
|
4
5
|
import { assertId, parseRung } from "../domain/ids.ts";
|
|
5
6
|
import type { RunRecord } from "../domain/record.ts";
|
|
@@ -16,8 +17,9 @@ import {
|
|
|
16
17
|
import { lockHeld, withFileLock } from "../infra/filelock.ts";
|
|
17
18
|
import { log } from "../infra/log.ts";
|
|
18
19
|
import { isAlive, isSurelyAlive, killGroup } from "../infra/proc.ts";
|
|
20
|
+
import { groupAlive, stopGroup } from "../infra/supervisor.ts";
|
|
19
21
|
import { writeJsonAtomic } from "../infra/store.ts";
|
|
20
|
-
import { admit, KILL_GRACE_MS, laneFile, launch } from "./admission.ts";
|
|
22
|
+
import { admit, KILL_GRACE_MS, laneFile, laneOwns, launch } from "./admission.ts";
|
|
21
23
|
import { standInFor } from "./backends.ts";
|
|
22
24
|
import {
|
|
23
25
|
type Dispatch,
|
|
@@ -33,9 +35,13 @@ import {
|
|
|
33
35
|
} from "./dispatches.ts";
|
|
34
36
|
import { finalizeDispatch, finalizingElsewhere, waitForFinish } from "./finalize.ts";
|
|
35
37
|
import type { Deps } from "./ports.ts";
|
|
38
|
+
import { isDaemonRun } from "./run-events-store.ts";
|
|
36
39
|
import { route } from "./lane-service.ts";
|
|
37
|
-
import {
|
|
38
|
-
import {
|
|
40
|
+
import { currentRoute } from "../domain/route.ts";
|
|
41
|
+
import { appendRoute, findRun, readRecords, readRoleRoutes, readRoutes, type Run } from "./run-store.ts";
|
|
42
|
+
import { sessionKey } from "../domain/host.ts";
|
|
43
|
+
import { claimRun, currentSession, ownsRun, runOwner, type SessionRef } from "./sessions.ts";
|
|
44
|
+
import { pinChanges, runProfile } from "./run-pin.ts";
|
|
39
45
|
import { type NotesPatch, refreshState } from "./state.ts";
|
|
40
46
|
|
|
41
47
|
export interface DispatchInput {
|
|
@@ -43,7 +49,8 @@ export interface DispatchInput {
|
|
|
43
49
|
role: Role;
|
|
44
50
|
name: string;
|
|
45
51
|
brief: string;
|
|
46
|
-
|
|
52
|
+
/** spec 1.5 plan 24: optional with `lane` (the lane's current rung); required without one */
|
|
53
|
+
rung?: string;
|
|
47
54
|
thread?: string;
|
|
48
55
|
lane?: string;
|
|
49
56
|
next?: string;
|
|
@@ -174,9 +181,19 @@ export async function watchersSettled(): Promise<void> {
|
|
|
174
181
|
*/
|
|
175
182
|
export function watch(deps: Deps, run: Run, d: Dispatch): void {
|
|
176
183
|
watching.add(d.admit.dispatchId);
|
|
184
|
+
// the session this watcher started under, kept apart from its live identity: an MCP connection that closes
|
|
185
|
+
// invalidates the scoped host (no session), yet the watcher still came from that session (PR #47 P1)
|
|
186
|
+
const origin = currentSession(deps);
|
|
177
187
|
const w = (async () => {
|
|
178
188
|
await waitForFinish(d, { pollMs: deps.pollMs, now: deps.now, onPoll: stallPoll(run, d) });
|
|
179
|
-
const
|
|
189
|
+
const record = await finalizeDispatch(run, d);
|
|
190
|
+
// a session that lost the run since it dispatched leaves a limit to the owner's claim (1.1 push minors):
|
|
191
|
+
// only the run's owner starts a stand-in. A process with no session (a terminal) settles as before.
|
|
192
|
+
if (record.status === "limit" && !limitIsMine(deps, run, origin)) {
|
|
193
|
+
log("info", "dispatch", { run: run.id, name: d.admit.name, limit: "left to the run's owner" });
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
const s = await settle(deps, run, d, record);
|
|
180
197
|
if (s.stateHints[0]) log("warn", "dispatch", { run: run.id, name: d.admit.name, hint: s.stateHints[0] });
|
|
181
198
|
})()
|
|
182
199
|
.catch((e: unknown) =>
|
|
@@ -189,12 +206,26 @@ export function watch(deps: Deps, run: Run, d: Dispatch): void {
|
|
|
189
206
|
watchers.add(w);
|
|
190
207
|
}
|
|
191
208
|
|
|
209
|
+
/**
|
|
210
|
+
* Whether a watcher settles a limit (fails it over): a watcher with no session, then or now (a terminal, a
|
|
211
|
+
* test), always does; one that started under a session does only while that session, or its live one, owns
|
|
212
|
+
* the run.
|
|
213
|
+
*/
|
|
214
|
+
function limitIsMine(deps: Deps, run: Run, origin: SessionRef | null): boolean {
|
|
215
|
+
if (origin === null && currentSession(deps) === null) return true;
|
|
216
|
+
if (ownsRun(deps, run)) return true;
|
|
217
|
+
const owner = runOwner(run);
|
|
218
|
+
return origin !== null && owner !== null && sessionKey(owner) === sessionKey(origin);
|
|
219
|
+
}
|
|
220
|
+
|
|
192
221
|
/**
|
|
193
222
|
* Spec §3.3: a run this session has just taken over may have roles another session's server launched. This
|
|
194
223
|
* process watches each live one it is not already watching, so it settles them and its notifier announces them
|
|
195
224
|
* to their new owner (a second settle of a dispatch is harmless: finalize and failover run once).
|
|
196
225
|
*/
|
|
197
226
|
export function adopt(deps: Deps, run: Run): void {
|
|
227
|
+
// a 2.0 run's daemon watches its roles and fails them over itself
|
|
228
|
+
if (isDaemonRun(run)) return;
|
|
198
229
|
for (const d of liveDispatches(run, deps.now())) if (!watching.has(d.admit.dispatchId)) watch(deps, run, d);
|
|
199
230
|
}
|
|
200
231
|
|
|
@@ -214,6 +245,17 @@ export function adopt(deps: Deps, run: Run): void {
|
|
|
214
245
|
*/
|
|
215
246
|
export async function claim(deps: Deps, run: Run, o: { background?: boolean } = {}): Promise<void> {
|
|
216
247
|
if (!(await claimRun(deps, run)) && !ownsRun(deps, run)) return;
|
|
248
|
+
await recoverOwned(deps, run, o);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* What `claim` does for a run this session already owns, without taking it: watch its live roles, record and
|
|
253
|
+
* settle what finished unrecorded, settle its unsettled limits and unannounced records. A server that does not
|
|
254
|
+
* lead the boot runs it for each run its session owns (PR #47 P1: a role that finished while that server was
|
|
255
|
+
* away has no live process to watch and no record to scan).
|
|
256
|
+
*/
|
|
257
|
+
export async function recoverOwned(deps: Deps, run: Run, o: { background?: boolean } = {}): Promise<void> {
|
|
258
|
+
if (isDaemonRun(run)) return;
|
|
217
259
|
adopt(deps, run);
|
|
218
260
|
let r = recovering.get(run.id);
|
|
219
261
|
if (!r) {
|
|
@@ -281,24 +323,138 @@ function unannounced(run: Run): { d: Dispatch; record: RunRecord }[] {
|
|
|
281
323
|
});
|
|
282
324
|
}
|
|
283
325
|
|
|
326
|
+
/** How long a dispatch waits for another dispatch's route of the same lane (Jev may take tens of seconds). */
|
|
327
|
+
const ROUTE_LOCK_MS = 120_000;
|
|
328
|
+
|
|
329
|
+
/**
|
|
330
|
+
* What admission would refuse anyway, refused before a lane is routed, so a refused dispatch asks Jev nothing
|
|
331
|
+
* and writes no route row: the role off, the lane file missing or with no Owns:, the name or the lane already
|
|
332
|
+
* running. Admission checks them again under its lock.
|
|
333
|
+
*/
|
|
334
|
+
function refuseBeforeRouting(deps: Deps, run: Run, i: DispatchInput & { lane: string }): void {
|
|
335
|
+
const profile = runProfile(deps, run);
|
|
336
|
+
if (!profile.roles[i.role]?.enabled)
|
|
337
|
+
throw new CatherdError("E_ADMIT_RUNG", `the ${i.role} role is off in profile ${profile.name}`, {
|
|
338
|
+
fix: "skip the role, or turn it on with profile_set",
|
|
339
|
+
});
|
|
340
|
+
laneOwns(run, i.lane);
|
|
341
|
+
// live only: a finished, unrecorded dispatch is recorded by admission (finalizeFinished), not refused here
|
|
342
|
+
const pending = liveDispatches(run, deps.now());
|
|
343
|
+
const same = pending.find((d) => d.admit.name === i.name);
|
|
344
|
+
if (same)
|
|
345
|
+
throw new CatherdError("E_ADMIT_DUPLICATE", `${i.name} is already running on ${same.admit.rung}`, {
|
|
346
|
+
fix: `its record is announced when it finishes; or cancel(run, "${i.name}")`,
|
|
347
|
+
});
|
|
348
|
+
const lane = pending.find((d) => d.admit.lane === i.lane);
|
|
349
|
+
if (lane)
|
|
350
|
+
throw new CatherdError("E_ADMIT_OVERLAP", `lane ${i.lane} is already running as ${lane.admit.name}`, {
|
|
351
|
+
fix: `dispatch it after ${lane.admit.name} finishes`,
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/**
|
|
356
|
+
* Plan 22: the thread a dispatch resumes. `"latest"` is the name's last recorded thread; any other id must be one
|
|
357
|
+
* the name's records in this run hold (`runs.jsonl`), in any case, so a wrong id is refused before a CLI starts
|
|
358
|
+
* and fails on it. Null for a fresh thread.
|
|
359
|
+
*/
|
|
360
|
+
function threadFor(run: Run, name: string, thread: string | undefined): string | null {
|
|
361
|
+
if (thread === undefined) return null;
|
|
362
|
+
assertId("role name", name);
|
|
363
|
+
const mine = readRecords(run)
|
|
364
|
+
.records.filter((r) => r.name === name && r.thread !== null)
|
|
365
|
+
.map((r) => r.thread as string);
|
|
366
|
+
const last = mine.at(-1);
|
|
367
|
+
if (thread === "latest") {
|
|
368
|
+
if (last) return last;
|
|
369
|
+
throw new CatherdError("E_ADMIT_THREAD", `${name} has no earlier thread in this run`, {
|
|
370
|
+
fix: "omit thread for a fresh thread",
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
const known = mine.findLast((t) => t.toLowerCase() === thread.toLowerCase());
|
|
374
|
+
if (known) return known;
|
|
375
|
+
throw new CatherdError("E_ADMIT_THREAD", `${thread} is not a thread of ${name} in this run`, {
|
|
376
|
+
fix: last
|
|
377
|
+
? `pass thread: "latest" for ${name}'s last thread (${last}), or omit thread for a fresh one`
|
|
378
|
+
: "omit thread for a fresh thread",
|
|
379
|
+
});
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/**
|
|
383
|
+
* Plan 22, resume hygiene: what an earlier turn on `thread` left running in its process group (a server, a
|
|
384
|
+
* watcher, a background command) is stopped before the thread is resumed, so it never ends the resumed CLI
|
|
385
|
+
* (exit 143). Only a dispatch whose supervisor and group leader are both gone is touched (with or without
|
|
386
|
+
* exit.json: a supervisor killed before it stopped the group writes none): a live group with no process of the
|
|
387
|
+
* leader's pid can only be that dispatch's leftovers (a pid is never reused while its group lives), and a pid
|
|
388
|
+
* that answers belongs to someone else by now. Returns a hint per group stopped.
|
|
389
|
+
*/
|
|
390
|
+
async function stopLeftovers(deps: Deps, run: Run, thread: string): Promise<string[]> {
|
|
391
|
+
const records = new Map(readRecords(run).records.map((r) => [r.dispatchId, r]));
|
|
392
|
+
const hints: string[] = [];
|
|
393
|
+
for (const d of listDispatches(run)) {
|
|
394
|
+
const on = d.admit.thread ?? records.get(d.admit.dispatchId)?.thread ?? null;
|
|
395
|
+
if (on?.toLowerCase() !== thread.toLowerCase()) continue;
|
|
396
|
+
const proc = readProc(d.dir);
|
|
397
|
+
const pgid = proc?.pgid ?? proc?.pid;
|
|
398
|
+
if (!proc || pgid === undefined || pgid !== proc.pid) continue;
|
|
399
|
+
// a live supervisor stops its own group when its worker ends; exit.json is not the test, since a supervisor
|
|
400
|
+
// killed (SIGKILL, OOM) before it stopped the group never writes one
|
|
401
|
+
if (isAlive(proc.supervisorPid, proc.supervisorStartTime)) continue;
|
|
402
|
+
if (isAlive(proc.pid, null) || !groupAlive(pgid)) continue;
|
|
403
|
+
await stopGroup(pgid, orphanLimits.killGraceMs, deps.pollMs);
|
|
404
|
+
log("info", "dispatch", { run: run.id, name: d.admit.name, thread, stoppedGroup: pgid });
|
|
405
|
+
hints.push(`resume: stopped what ${d.admit.name}'s earlier turn left running on thread ${thread}`);
|
|
406
|
+
}
|
|
407
|
+
return hints;
|
|
408
|
+
}
|
|
409
|
+
|
|
284
410
|
/**
|
|
285
411
|
* Spec §4.4, plan 9 ruling 1: admit and launch one role, start its watcher, then refresh state.md with
|
|
286
412
|
* `next` (after the launch, so nothing delays it). Returns at once; catherd announces the record (spec §3).
|
|
287
413
|
*/
|
|
288
414
|
export async function dispatch(deps: Deps, i: DispatchInput): Promise<DispatchStarted> {
|
|
289
415
|
const run = findRun(i.run);
|
|
416
|
+
// refused before the claim, so a wrong thread changes nothing
|
|
417
|
+
const thread = threadFor(run, i.name, i.thread);
|
|
290
418
|
await claim(deps, run);
|
|
291
419
|
const hints: string[] = [];
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
if (i.lane !== undefined
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
420
|
+
if (thread !== null) hints.push(...(await stopLeftovers(deps, run, thread)));
|
|
421
|
+
let rung: string;
|
|
422
|
+
if (i.lane !== undefined) {
|
|
423
|
+
const lane = i.lane;
|
|
424
|
+
assertId("lane", lane);
|
|
425
|
+
// spec 1.1 §6: a lane is routed before its first dispatch; a rung off the routed ladder starts at the routed one
|
|
426
|
+
let current = currentRoute(readRoutes(run), lane);
|
|
427
|
+
let routed: Awaited<ReturnType<typeof route>> | null = null;
|
|
428
|
+
if (!current) {
|
|
429
|
+
refuseBeforeRouting(deps, run, { ...i, lane });
|
|
430
|
+
// one route per lane: a second dispatch of the lane waits for the first's route and takes it
|
|
431
|
+
const got = await withFileLock(
|
|
432
|
+
join(run.dir, `route-${lane}`),
|
|
433
|
+
async () => {
|
|
434
|
+
const now = currentRoute(readRoutes(run), lane);
|
|
435
|
+
return now
|
|
436
|
+
? { current: now, routed: null }
|
|
437
|
+
: {
|
|
438
|
+
current: null,
|
|
439
|
+
routed: await route(deps, { run: i.run, laneFile: `lanes/${lane}.md`, role: i.role }),
|
|
440
|
+
};
|
|
441
|
+
},
|
|
442
|
+
{ timeoutMs: ROUTE_LOCK_MS },
|
|
443
|
+
);
|
|
444
|
+
current = got.current;
|
|
445
|
+
routed = got.routed;
|
|
446
|
+
}
|
|
447
|
+
// spec 1.5 plan 24: without a rung, a lane runs at its current rung (its route, after any climb)
|
|
448
|
+
rung = i.rung ?? routed?.rung ?? (current?.rung as string);
|
|
449
|
+
if (routed && i.rung !== undefined && !routed.ladder.includes(i.rung)) {
|
|
298
450
|
rung = routed.rung;
|
|
299
|
-
hints.push(`${i.rung} is not on ${
|
|
451
|
+
hints.push(`${i.rung} is not on ${lane}'s routed ladder: dispatched at ${routed.rung}`);
|
|
300
452
|
}
|
|
301
|
-
}
|
|
453
|
+
} else if (i.rung === undefined)
|
|
454
|
+
throw new CatherdError("E_INPUT_INVALID", `dispatch ${i.name}: a role outside a lane needs a rung`, {
|
|
455
|
+
fix: `route(run, role: "${i.role}") gives the role's rung; pass it as rung`,
|
|
456
|
+
});
|
|
457
|
+
else rung = i.rung;
|
|
302
458
|
// the rung that runs, after routing: an off-ladder native rung routing replaced is never launched
|
|
303
459
|
assertNativeHost(rung, deps.host.host);
|
|
304
460
|
const { d, specPath } = await admit(
|
|
@@ -309,7 +465,7 @@ export async function dispatch(deps: Deps, i: DispatchInput): Promise<DispatchSt
|
|
|
309
465
|
name: i.name,
|
|
310
466
|
brief: i.brief,
|
|
311
467
|
rung,
|
|
312
|
-
thread
|
|
468
|
+
thread,
|
|
313
469
|
lane: i.lane ?? null,
|
|
314
470
|
failoverFrom: null,
|
|
315
471
|
},
|
|
@@ -324,10 +480,46 @@ export async function dispatch(deps: Deps, i: DispatchInput): Promise<DispatchSt
|
|
|
324
480
|
// a launch that failed is still watched: its record (lost, after the start grace) is announced
|
|
325
481
|
watch(deps, run, d);
|
|
326
482
|
}
|
|
327
|
-
|
|
483
|
+
if (i.lane === undefined) recordRoleDispatch(deps, run, i.role, i.name, rung);
|
|
484
|
+
// spec 1.5: a change since the run's pin is logged in state.md each time a role starts
|
|
485
|
+
const changes = pinChanges(deps, run);
|
|
486
|
+
if (changes.length)
|
|
487
|
+
hints.push(
|
|
488
|
+
`dispatched on the run's pinned values: ${changes.join("; ")}; run_pin(run) re-pins to the repo's now`,
|
|
489
|
+
);
|
|
490
|
+
await refresh(run, { ...(i.next ? { next: i.next } : {}), pinChanges: changes }, hints);
|
|
328
491
|
return { dispatched: dispatchedOf(d), hints };
|
|
329
492
|
}
|
|
330
493
|
|
|
494
|
+
/**
|
|
495
|
+
* Spec 1.5 plan 24: a lane-less dispatch's rung in routes.jsonl, so every role's decision can be audited:
|
|
496
|
+
* `route` when it is the rung the role's last `route` gave (with that ladder), else the coordinator's own pick.
|
|
497
|
+
* A failed write is logged, never a failed dispatch: the role is already running.
|
|
498
|
+
*/
|
|
499
|
+
function recordRoleDispatch(deps: Deps, run: Run, role: Role, name: string, rung: string): void {
|
|
500
|
+
try {
|
|
501
|
+
const last = readRoleRoutes(run).findLast((r) => r.role === role && r.source === "route");
|
|
502
|
+
const routed = last?.rung === rung;
|
|
503
|
+
appendRoute(run, {
|
|
504
|
+
at: new Date(deps.now()).toISOString(),
|
|
505
|
+
lane: null,
|
|
506
|
+
role,
|
|
507
|
+
name,
|
|
508
|
+
rung,
|
|
509
|
+
ladder: last?.ladder ?? [rung],
|
|
510
|
+
source: "dispatch",
|
|
511
|
+
decidedBy: routed ? last.decidedBy : "orchestrator",
|
|
512
|
+
why: routed
|
|
513
|
+
? `the rung route gave the ${role}`
|
|
514
|
+
: last
|
|
515
|
+
? `the coordinator's rung; route gave the ${role} ${last.rung}`
|
|
516
|
+
: `the coordinator's rung; the ${role} was never routed`,
|
|
517
|
+
});
|
|
518
|
+
} catch (e) {
|
|
519
|
+
log("warn", "dispatch", { run: run.id, name, routes: errorMessage(e) });
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
|
|
331
523
|
/** How long a settle waits for another process's failover of the same dispatch (a test seam). */
|
|
332
524
|
export const failoverLock = { timeoutMs: 120_000 };
|
|
333
525
|
|
|
@@ -498,7 +690,7 @@ async function failover(deps: Deps, run: Run, d: Dispatch, limited: RunRecord):
|
|
|
498
690
|
return failedOver(already.admit.rung, already);
|
|
499
691
|
}
|
|
500
692
|
// else that stand-in never ran and is over (recorded as lost, or past its start grace): admit a new one
|
|
501
|
-
const standIn = standInFor(deps
|
|
693
|
+
const standIn = standInFor(runProfile(deps, run).failover, limited.rung, run.meta.repo);
|
|
502
694
|
if (!standIn) return { hints, started: null, pause: paused };
|
|
503
695
|
// the stand-in answers to whoever owns the run now, which a claim from another host may have changed
|
|
504
696
|
assertNativeHost(standIn, runOwner(run)?.host ?? deps.host.host);
|
|
@@ -598,7 +790,7 @@ async function stopOrphan(deps: Deps, d: Dispatch): Promise<void> {
|
|
|
598
790
|
/**
|
|
599
791
|
* Spec §4.7: stop a live dispatch (interrupt, SIGTERM, SIGKILL) and record it as cancelled. With `read` (the MCP
|
|
600
792
|
* `cancel` tool, whose caller holds the record it returns) the record is marked read, so `peek` no longer lists
|
|
601
|
-
* it. Without it (the
|
|
793
|
+
* it. Without it (the panel, `catherd runs cancel`) the record stays unread and catherd announces it to the
|
|
602
794
|
* run's owner at `later` (spec §3.6).
|
|
603
795
|
*/
|
|
604
796
|
export async function cancel(
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
|
2
2
|
import { join, relative } from "node:path";
|
|
3
3
|
import { z } from "zod";
|
|
4
|
+
import { CatherdError } from "../domain/errors.ts";
|
|
4
5
|
import { dispatchHints } from "../domain/hints.ts";
|
|
5
6
|
import { ACCESS, type RunRecord } from "../domain/record.ts";
|
|
6
7
|
import { ROLES } from "../domain/roles.ts";
|
|
@@ -18,6 +19,11 @@ const AdmitSchema = z.looseObject({
|
|
|
18
19
|
role: z.enum(ROLES),
|
|
19
20
|
lane: z.string().nullable(),
|
|
20
21
|
owns: z.array(z.string()),
|
|
22
|
+
/**
|
|
23
|
+
* spec 1.5 plan 21: a laneless writer's implicit Owns (its brief's Owns: line, else docs/** and *.md), so its
|
|
24
|
+
* edits are its own and never a concurrent lane's violation; admission never refuses an overlap with it
|
|
25
|
+
*/
|
|
26
|
+
ownsImplicit: z.array(z.string()).optional(),
|
|
21
27
|
rung: z.string(),
|
|
22
28
|
backend: z.string(),
|
|
23
29
|
thread: z.string().nullable(),
|
|
@@ -54,9 +60,11 @@ export const startLimits = { graceMs: STARTING_GRACE_MS };
|
|
|
54
60
|
export const admitPath = (dir: string): string => join(dir, "admit.json");
|
|
55
61
|
export const launchPath = (dir: string): string => join(dir, "launch.json");
|
|
56
62
|
export const roleDir = (run: Run, name: string): string => join(runPaths(run.dir).roles, name);
|
|
63
|
+
/** Spec 1.5 plan 21: the role's TMPDIR, `<run>/scratch/<name>/`, removed with the run's scratch. */
|
|
64
|
+
export const scratchDir = (run: Run, name: string): string => join(runPaths(run.dir).scratch, name);
|
|
57
65
|
|
|
58
66
|
/** Every admitted dispatch of the run, oldest first; a folder whose admit.json cannot be read is skipped. */
|
|
59
|
-
export function listDispatches(run: Run): Dispatch[] {
|
|
67
|
+
export function listDispatches(run: Run, strict = false): Dispatch[] {
|
|
60
68
|
const roles = runPaths(run.dir).roles;
|
|
61
69
|
const out: Dispatch[] = [];
|
|
62
70
|
if (!existsSync(roles)) return out;
|
|
@@ -68,6 +76,8 @@ export function listDispatches(run: Run): Dispatch[] {
|
|
|
68
76
|
try {
|
|
69
77
|
out.push({ dir, admit: decodeAdmit(readVersioned(admitPath(dir), AdmitSchema, 1)) });
|
|
70
78
|
} catch {
|
|
79
|
+
if (strict && existsSync(admitPath(dir)))
|
|
80
|
+
throw new CatherdError("E_RUN_CORRUPT", `run ${run.id} has unreadable admission evidence: ${dir}`);
|
|
71
81
|
// admission never finished writing this folder
|
|
72
82
|
}
|
|
73
83
|
}
|
|
@@ -124,9 +134,10 @@ export function pendingDispatches(
|
|
|
124
134
|
run: Run,
|
|
125
135
|
now = Date.now(),
|
|
126
136
|
records: RunRecord[] = readRecords(run).records,
|
|
137
|
+
strict = false,
|
|
127
138
|
): LiveDispatch[] {
|
|
128
139
|
const recorded = new Set(records.map((r) => r.dispatchId));
|
|
129
|
-
return listDispatches(run)
|
|
140
|
+
return listDispatches(run, strict)
|
|
130
141
|
.filter((d) => !recorded.has(d.admit.dispatchId))
|
|
131
142
|
.map((d) => ({ ...d, state: dispatchState(d, now) }));
|
|
132
143
|
}
|
|
@@ -134,6 +145,23 @@ export function pendingDispatches(
|
|
|
134
145
|
export const liveDispatches = (run: Run, now = Date.now()): LiveDispatch[] =>
|
|
135
146
|
pendingDispatches(run, now).filter((d) => d.state !== "finished");
|
|
136
147
|
|
|
148
|
+
/** The thread the supervisor saw the role's CLI name (thread.json), else null. */
|
|
149
|
+
export function readThread(dir: string): string | null {
|
|
150
|
+
const t = readJson<{ thread?: unknown }>(dispatchPaths(dir).thread)?.thread;
|
|
151
|
+
return typeof t === "string" && t ? t : null;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Spec 1.5 plan 21: the role whose thread `thread` is, in this run: a recorded thread, a resumed one, or the one
|
|
156
|
+
* a live role's supervisor wrote down. Null for any other thread, the orchestrator's among them.
|
|
157
|
+
*/
|
|
158
|
+
export function roleThreadOf(run: Run, thread: string): string | null {
|
|
159
|
+
const recorded = readRecords(run).records.find((r) => r.thread === thread);
|
|
160
|
+
if (recorded) return recorded.name;
|
|
161
|
+
const d = listDispatches(run).find((x) => x.admit.thread === thread || readThread(x.dir) === thread);
|
|
162
|
+
return d ? d.admit.name : null;
|
|
163
|
+
}
|
|
164
|
+
|
|
137
165
|
/** Spec §4.1: `roles/<name>/latest` names the newest dispatch of a role. */
|
|
138
166
|
export function setLatest(run: Run, name: string, dispatchId: string): void {
|
|
139
167
|
writeTextAtomic(join(roleDir(run, name), "latest"), dispatchId);
|
|
@@ -172,9 +172,14 @@ export function workspaceWriteNetwork(profiles: Profile[]): Map<string, boolean>
|
|
|
172
172
|
|
|
173
173
|
/**
|
|
174
174
|
* The `sandbox:codex` row (which `codex sandbox` form runs) and one `access:<backend>` row per backend; a
|
|
175
|
-
* backend not in `installed` (its CLI is not on PATH) is skipped without running a probe.
|
|
175
|
+
* backend not in `installed` (its CLI is not on PATH) is skipped without running a probe. `repos`: where
|
|
176
|
+
* workers run (doctor's repo and every bound one), for a sandbox set per repo.
|
|
176
177
|
*/
|
|
177
|
-
export async function accessChecks(
|
|
178
|
+
export async function accessChecks(
|
|
179
|
+
profiles: Profile[],
|
|
180
|
+
installed: ReadonlySet<string>,
|
|
181
|
+
repos: string[] = [],
|
|
182
|
+
): Promise<Check[]> {
|
|
178
183
|
const checks: Check[] = [];
|
|
179
184
|
for (const [id, network] of workspaceWriteNetwork(profiles)) {
|
|
180
185
|
const a = adapterFor(id);
|
|
@@ -190,7 +195,7 @@ export async function accessChecks(profiles: Profile[], installed: ReadonlySet<s
|
|
|
190
195
|
continue;
|
|
191
196
|
}
|
|
192
197
|
const shell = a.accessShell
|
|
193
|
-
? await a.accessShell({ network }).catch((e: unknown) => String(e))
|
|
198
|
+
? await a.accessShell({ network, repos }).catch((e: unknown) => String(e))
|
|
194
199
|
: `no way to run a shell in ${id}'s sandbox without a model turn; the live kit (docs/dev/live-verification.md) runs the five probes as one worker turn`;
|
|
195
200
|
if (id === "codex")
|
|
196
201
|
checks.push(
|
|
@@ -151,7 +151,7 @@ export async function backendChecks(
|
|
|
151
151
|
}
|
|
152
152
|
: { id: `backend:${id}`, label: id, state: "ok", word: "ready", detail },
|
|
153
153
|
);
|
|
154
|
-
// spec 1.3 §6.6: asked only of a logged-in CLI (a logged-out
|
|
154
|
+
// spec 1.3 §6.6: asked only of a logged-in CLI (a logged-out CLI may open a browser)
|
|
155
155
|
const quota = probe.loggedIn && a.quota ? await a.quota().catch(() => null) : null;
|
|
156
156
|
if (quota)
|
|
157
157
|
checks.push({ id: `quota:${id}`, label: `${id} quota`, state: "info", word: "quota", detail: quota });
|