catherd-cli 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -36
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/.codex-plugin/plugin.json +11 -0
- package/plugin/.mcp-codex.json +10 -0
- package/plugin/bin/catherd-mcp +1 -1
- package/plugin/skills/catherd/SKILL.md +44 -38
- package/plugin/skills/catherd-setup/SKILL.md +22 -6
- package/src/adapters/access.ts +7 -1
- package/src/adapters/cli.ts +1 -64
- package/src/cli.ts +48 -19
- package/src/domain/host.ts +17 -0
- package/src/domain/notice.ts +5 -2
- package/src/domain/profile-rules.ts +17 -0
- package/src/domain/profile.ts +32 -5
- package/src/domain/record.ts +1 -0
- package/src/entry/catalog-command.ts +4 -2
- package/src/entry/deps.ts +6 -3
- package/src/entry/doctor-command.ts +11 -2
- package/src/entry/host-arg.ts +28 -0
- package/src/entry/init-command.ts +18 -7
- package/src/entry/lock-command.ts +7 -2
- package/src/entry/mcp/handshake.ts +3 -1
- package/src/entry/mcp/run-tools.ts +5 -2
- package/src/entry/mcp/server.ts +129 -40
- package/src/entry/mcp/setup-tools.ts +7 -2
- package/src/entry/profile-command.ts +108 -12
- package/src/entry/runs-command.ts +81 -5
- package/src/entry/tui/effects.ts +19 -8
- package/src/entry/tui/fixtures.ts +33 -10
- package/src/entry/tui/run.tsx +10 -4
- package/src/entry/tui/state.ts +19 -6
- package/src/entry/tui/stories.ts +11 -4
- package/src/entry/tui/views/profile-actions.ts +4 -2
- package/src/entry/tui/views/profiles.tsx +2 -2
- package/src/entry/tui/views/save-dialog.tsx +2 -2
- package/src/infra/cli.ts +65 -0
- package/src/infra/codex-queue.ts +165 -0
- package/src/infra/delivery.ts +83 -0
- package/src/infra/dispatch-dir.ts +1 -0
- package/src/infra/env.ts +4 -1
- package/src/infra/host-context.ts +58 -0
- package/src/infra/peer-inbox.ts +4 -1
- package/src/infra/supervisor.ts +3 -2
- package/src/services/admission.ts +7 -3
- package/src/services/agent-links.ts +30 -5
- package/src/services/backends.ts +7 -0
- package/src/services/dispatch-service.ts +17 -7
- package/src/services/dispatches.ts +6 -1
- package/src/services/doctor-backends.ts +30 -7
- package/src/services/doctor-checks.ts +20 -5
- package/src/services/doctor-push.ts +69 -90
- package/src/services/doctor.ts +105 -33
- package/src/services/finalize.ts +1 -1
- package/src/services/lane-service.ts +1 -0
- package/src/services/notifier.ts +322 -70
- package/src/services/peek.ts +46 -9
- package/src/services/ports.ts +8 -2
- package/src/services/profile-service.ts +116 -30
- package/src/services/profile-store.ts +23 -6
- package/src/services/routing-service.ts +6 -1
- package/src/services/run-debug.ts +98 -4
- package/src/services/run-service.ts +2 -0
- package/src/services/run-store.ts +10 -4
- package/src/services/runs-page.ts +3 -2
- package/src/services/session-view.ts +18 -13
- package/src/services/sessions.ts +32 -16
- package/src/services/setup.ts +31 -10
- package/src/services/summary.ts +40 -5
- package/src/services/treat-likes.ts +22 -9
package/src/services/notifier.ts
CHANGED
|
@@ -1,34 +1,37 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
|
-
import { errorMessage } from "../domain/errors.ts";
|
|
3
|
+
import { CatherdError, errorMessage } from "../domain/errors.ts";
|
|
4
4
|
import { envelope, formatNotices, type Notice, type NoticePriority, priorityOf } from "../domain/notice.ts";
|
|
5
|
+
import { sessionKey, type HostSessionRef } from "../domain/host.ts";
|
|
6
|
+
import { sendToCodexQueue, type QueueSendResult } from "../infra/codex-queue.ts";
|
|
7
|
+
import {
|
|
8
|
+
deliveryState,
|
|
9
|
+
readDelivery,
|
|
10
|
+
writeDeliveryAttempt,
|
|
11
|
+
type DeliveryAttempt,
|
|
12
|
+
} from "../infra/delivery.ts";
|
|
5
13
|
import type { RunRecord } from "../domain/record.ts";
|
|
6
14
|
import { awaitsCollect, dispatchPaths } from "../infra/dispatch-dir.ts";
|
|
7
15
|
import { tryLock } from "../infra/filelock.ts";
|
|
8
16
|
import { log } from "../infra/log.ts";
|
|
9
17
|
import { type SendResult, sendToInbox } from "../infra/peer-inbox.ts";
|
|
10
18
|
import { writeJsonAtomic } from "../infra/store.ts";
|
|
11
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
readStallQuietMs,
|
|
21
|
+
type Settled,
|
|
22
|
+
settledHooks,
|
|
23
|
+
type Stalled,
|
|
24
|
+
stallHooks,
|
|
25
|
+
} from "./dispatch-service.ts";
|
|
12
26
|
import { type Dispatch, listDispatches, readFailover } from "./dispatches.ts";
|
|
13
27
|
import type { Deps } from "./ports.ts";
|
|
14
28
|
import { listRuns, readRecords, type Run } from "./run-store.ts";
|
|
15
|
-
import { currentSession, ownsRun } from "./sessions.ts";
|
|
16
|
-
|
|
17
|
-
/**
|
|
18
|
-
* Spec §3.4: runs only inside the MCP server, the session's child and so its only sender (§3.2). For every run this
|
|
19
|
-
* session owns, a dispatch that is settled with its record still unread is announced to the session's peer inbox;
|
|
20
|
-
* notices that arrive within the window of each other go as one message; a message that went out is written down
|
|
21
|
-
* (`notified.json`), so a restart never sends it twice, and claimed while it goes, so two servers of one session never
|
|
22
|
-
* both send it. Ownership is asked again under the claim and once more after the message went: a run another session
|
|
23
|
-
* claimed meanwhile is left unmarked, so its new owner announces it too (a copy to the session that gave the run up is
|
|
24
|
-
* acceptable, a notice lost to the owner is not); a notice whose claim another server holds is tried again after the
|
|
25
|
-
* window until it is sent, read, marked or no longer this session's. Disk stays the truth: a notice that cannot be
|
|
26
|
-
* sent is dropped (its claim freed), and the record waits, unread, for `result` or `peek`.
|
|
27
|
-
*/
|
|
29
|
+
import { currentSession, ownsRun, readSessionRows, runOwner } from "./sessions.ts";
|
|
28
30
|
|
|
29
31
|
export interface NotifierOptions {
|
|
30
32
|
/** notices this close to each other go as one message (spec: 3 s); tests shorten it */
|
|
31
33
|
coalesceMs?: number;
|
|
34
|
+
sendCodex?: (target: HostSessionRef, content: string) => Promise<QueueSendResult>;
|
|
32
35
|
/** the sender; tests may replace it */
|
|
33
36
|
send?: (
|
|
34
37
|
target: { socketPath: string | null; token: string | null },
|
|
@@ -50,7 +53,39 @@ export interface Notifier {
|
|
|
50
53
|
stop(): void;
|
|
51
54
|
}
|
|
52
55
|
|
|
53
|
-
|
|
56
|
+
function legacyNotified(q: Queued, target: HostSessionRef): boolean {
|
|
57
|
+
if (target.host !== "claude-code") return false;
|
|
58
|
+
try {
|
|
59
|
+
const marker = JSON.parse(readFileSync(q.mark, "utf8")) as {
|
|
60
|
+
schema?: unknown;
|
|
61
|
+
at?: unknown;
|
|
62
|
+
msgId?: unknown;
|
|
63
|
+
};
|
|
64
|
+
if (marker.schema !== 1 || typeof marker.at !== "string" || typeof marker.msgId !== "string")
|
|
65
|
+
return false;
|
|
66
|
+
const at = Date.parse(marker.at);
|
|
67
|
+
if (!Number.isFinite(at)) return false;
|
|
68
|
+
const historical = readSessionRows(q.run)
|
|
69
|
+
.filter((row) => Date.parse(row.at) <= at)
|
|
70
|
+
.at(-1);
|
|
71
|
+
const owner = runOwner(q.run);
|
|
72
|
+
const attributed = historical ?? (owner && Date.parse(owner.since) <= at ? owner : null);
|
|
73
|
+
return attributed !== null && sessionKey(attributed) === sessionKey(target);
|
|
74
|
+
} catch {
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function recoverSubmission(dir: string, eventId: string): void {
|
|
80
|
+
for (const a of readDelivery(dir)) {
|
|
81
|
+
if (a.status === "submitting" && a.eventIds.includes(eventId))
|
|
82
|
+
writeDeliveryAttempt(dir, {
|
|
83
|
+
...a,
|
|
84
|
+
status: "ambiguous",
|
|
85
|
+
reason: "Sender ended without a verified receipt; retry may duplicate input. Use peek/result.",
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
}
|
|
54
89
|
|
|
55
90
|
function replyOf(run: Run, r: RunRecord): string {
|
|
56
91
|
try {
|
|
@@ -77,6 +112,7 @@ export function finishedNotice(run: Run, d: Dispatch, r: RunRecord): Notice {
|
|
|
77
112
|
const urgent = r.status === "limit" || r.replyStatus === "blocked" || r.replyStatus === "refused";
|
|
78
113
|
return {
|
|
79
114
|
kind: "finished",
|
|
115
|
+
eventId: JSON.stringify([run.id, r.dispatchId, "finished"]),
|
|
80
116
|
runId: run.id,
|
|
81
117
|
runTitle: run.meta.title,
|
|
82
118
|
dispatchId: r.dispatchId,
|
|
@@ -97,6 +133,7 @@ export function finishedNotice(run: Run, d: Dispatch, r: RunRecord): Notice {
|
|
|
97
133
|
export function stalledNotice(run: Run, d: Dispatch, quietMs: number, now: number): Notice {
|
|
98
134
|
return {
|
|
99
135
|
kind: "stalled",
|
|
136
|
+
eventId: JSON.stringify([run.id, d.admit.dispatchId, "stalled"]),
|
|
100
137
|
runId: run.id,
|
|
101
138
|
runTitle: run.meta.title,
|
|
102
139
|
dispatchId: d.admit.dispatchId,
|
|
@@ -119,16 +156,32 @@ interface Queued {
|
|
|
119
156
|
/** the run it is about: its owner may change before the message goes */
|
|
120
157
|
run: Run;
|
|
121
158
|
notice: Notice;
|
|
159
|
+
dir: string;
|
|
122
160
|
/** the mark a message that went out leaves */
|
|
123
161
|
mark: string;
|
|
124
162
|
/** whether it is still news when the message goes */
|
|
125
163
|
due: () => boolean;
|
|
164
|
+
/** how many passes could not write its delivery claim */
|
|
165
|
+
claimFailures?: number;
|
|
166
|
+
/** how many sends the transport provably did not submit */
|
|
167
|
+
sendFailures?: number;
|
|
126
168
|
}
|
|
127
169
|
|
|
170
|
+
/** Passes a notice whose delivery claim cannot be written is kept for before it is given up (and logged). */
|
|
171
|
+
const MAX_CLAIM_FAILURES = 5;
|
|
172
|
+
|
|
173
|
+
/** Sends a notice the transport provably did not submit is tried, a coalescing window apart, before peek/result. */
|
|
174
|
+
const MAX_SEND_FAILURES = 5;
|
|
175
|
+
|
|
128
176
|
/** Starts the notifier for this process's session and hooks it to every settled dispatch. */
|
|
129
177
|
export function startNotifier(deps: Deps, o: NotifierOptions = {}): Notifier {
|
|
178
|
+
return notifierFor(deps, o);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function notifierFor(deps: Deps, o: NotifierOptions, retryEvent?: string): Notifier {
|
|
130
182
|
const window = o.coalesceMs ?? 3_000;
|
|
131
183
|
const send = o.send ?? sendToInbox;
|
|
184
|
+
const sendCodex = o.sendCodex ?? ((target, content) => sendToCodexQueue(target, content, process.env));
|
|
132
185
|
const queue = new Map<string, Queued>();
|
|
133
186
|
let timer: ReturnType<typeof setTimeout> | null = null;
|
|
134
187
|
let firstAt = 0;
|
|
@@ -158,89 +211,220 @@ export function startNotifier(deps: Deps, o: NotifierOptions = {}): Notifier {
|
|
|
158
211
|
timer = setTimeout(flush, Math.max(0, Math.min(window, firstAt + 5 * window - now)));
|
|
159
212
|
};
|
|
160
213
|
|
|
214
|
+
const pending = (q: Queued, target: HostSessionRef): boolean => {
|
|
215
|
+
if (!q.due() || legacyNotified(q, target)) return false;
|
|
216
|
+
const state = deliveryState(q.dir, target, q.notice.eventId);
|
|
217
|
+
return state === "pending" || (q.notice.eventId === retryEvent && state === "ambiguous");
|
|
218
|
+
};
|
|
219
|
+
|
|
161
220
|
async function deliver(batch: Queued[]): Promise<void> {
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
// of one session, a replacement starting while the old one still watches, never both send it. Whether it is
|
|
165
|
-
// still news is asked under the claim: the other may have announced it just before letting go.
|
|
221
|
+
const target = currentSession(deps);
|
|
222
|
+
if (stopped || !target || (target.host === "claude-code" && !deps.session)) return;
|
|
166
223
|
const claims: (() => void)[] = [];
|
|
167
224
|
const due: Queued[] = [];
|
|
168
225
|
const busy: Queued[] = [];
|
|
169
226
|
try {
|
|
170
227
|
for (const q of batch) {
|
|
171
|
-
//
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
228
|
+
// one notice's damaged evidence drops that notice alone, never the rest of the batch
|
|
229
|
+
try {
|
|
230
|
+
if (!q.due() || !owned(q.run)) continue;
|
|
231
|
+
const release = tryLock(q.mark);
|
|
232
|
+
if (!release) {
|
|
233
|
+
busy.push(q);
|
|
234
|
+
continue;
|
|
235
|
+
}
|
|
236
|
+
claims.push(release);
|
|
237
|
+
const live = currentSession(deps);
|
|
238
|
+
if (stopped || !live || sessionKey(live) !== sessionKey(target) || !owned(q.run)) continue;
|
|
239
|
+
recoverSubmission(q.dir, q.notice.eventId);
|
|
240
|
+
if (pending(q, target)) due.push(q);
|
|
241
|
+
} catch (e) {
|
|
242
|
+
log("warn", "notify", { error: errorMessage(e), dispatch: q.notice.dispatchId });
|
|
179
243
|
}
|
|
180
|
-
claims.push(release);
|
|
181
|
-
// asked again under the claim: the run may have moved, or the notice gone out, while it was taken
|
|
182
|
-
if (q.due() && owned(q.run)) due.push(q);
|
|
183
244
|
}
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
245
|
+
// nothing is sent unless every notice's claim is on disk. A claim that cannot be written rolls the batch's
|
|
246
|
+
// written claims back to failed (provably not submitted, so still deliverable), keeps that notice for a
|
|
247
|
+
// later pass, and claims the rest again at once: one bad file never holds back the others
|
|
248
|
+
let attempt: DeliveryAttempt;
|
|
249
|
+
for (;;) {
|
|
250
|
+
if (!due.length) return;
|
|
251
|
+
attempt = {
|
|
252
|
+
attemptId: crypto.randomUUID(),
|
|
253
|
+
target: {
|
|
254
|
+
host: target.host,
|
|
255
|
+
sessionId: target.sessionId,
|
|
256
|
+
hostSessionId: target.hostSessionId,
|
|
257
|
+
name: target.name,
|
|
258
|
+
},
|
|
259
|
+
eventIds: due.map((q) => q.notice.eventId),
|
|
260
|
+
at: new Date(deps.now()).toISOString(),
|
|
261
|
+
status: "submitting",
|
|
262
|
+
msgId: null,
|
|
263
|
+
reason: null,
|
|
264
|
+
};
|
|
265
|
+
const claimed: Queued[] = [];
|
|
266
|
+
const broken = due.find((q) => {
|
|
267
|
+
try {
|
|
268
|
+
writeDeliveryAttempt(q.dir, attempt);
|
|
269
|
+
claimed.push(q);
|
|
270
|
+
return false;
|
|
271
|
+
} catch (e) {
|
|
272
|
+
log("warn", "notify", { error: errorMessage(e), dispatch: q.notice.dispatchId });
|
|
273
|
+
return true;
|
|
274
|
+
}
|
|
196
275
|
});
|
|
197
|
-
|
|
276
|
+
if (!broken) break;
|
|
277
|
+
const undone: DeliveryAttempt = {
|
|
278
|
+
...attempt,
|
|
279
|
+
status: "failed",
|
|
280
|
+
reason: "A claim in this batch was not persisted; nothing was submitted.",
|
|
281
|
+
};
|
|
282
|
+
for (const c of claimed)
|
|
283
|
+
try {
|
|
284
|
+
writeDeliveryAttempt(c.dir, undone);
|
|
285
|
+
} catch (u) {
|
|
286
|
+
log("warn", "notify", { error: errorMessage(u), dispatch: c.notice.dispatchId });
|
|
287
|
+
}
|
|
288
|
+
// a claim that fails again and again is a damaged file, not a passing fault: it is given up then
|
|
289
|
+
broken.claimFailures = (broken.claimFailures ?? 0) + 1;
|
|
290
|
+
if (broken.claimFailures < MAX_CLAIM_FAILURES) busy.push(broken);
|
|
291
|
+
due.splice(due.indexOf(broken), 1);
|
|
198
292
|
}
|
|
199
|
-
const
|
|
200
|
-
|
|
201
|
-
const
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
293
|
+
const notices = due.map((q) => q.notice);
|
|
294
|
+
let outcome: QueueSendResult;
|
|
295
|
+
const content = envelope(formatNotices(notices));
|
|
296
|
+
try {
|
|
297
|
+
if (target.host === "codex") outcome = await sendCodex(target, content);
|
|
298
|
+
else {
|
|
299
|
+
const r = await send(
|
|
300
|
+
{ socketPath: deps.session!.socketPath, token: deps.session!.token },
|
|
301
|
+
content,
|
|
302
|
+
priorityOf(notices),
|
|
303
|
+
);
|
|
304
|
+
outcome =
|
|
305
|
+
r.outcome === "sent" && r.msgId
|
|
306
|
+
? { outcome: "accepted", msgId: r.msgId }
|
|
307
|
+
: {
|
|
308
|
+
// an error before any frame was written (no msg_id) provably submitted nothing: retried later
|
|
309
|
+
outcome:
|
|
310
|
+
r.outcome === "no-session" ||
|
|
311
|
+
r.outcome === "refused" ||
|
|
312
|
+
(r.outcome === "error" && !r.msgId)
|
|
313
|
+
? "not-submitted"
|
|
314
|
+
: "ambiguous",
|
|
315
|
+
reason: "Claude inbox has no verified receipt; use peek/result.",
|
|
316
|
+
};
|
|
317
|
+
}
|
|
318
|
+
} catch {
|
|
319
|
+
outcome = {
|
|
320
|
+
outcome: "ambiguous",
|
|
321
|
+
reason: "Sender ended without a verified receipt; retry may duplicate input. Use peek/result.",
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
const receipt: DeliveryAttempt = {
|
|
325
|
+
...attempt,
|
|
326
|
+
status:
|
|
327
|
+
outcome.outcome === "accepted"
|
|
328
|
+
? "accepted"
|
|
329
|
+
: outcome.outcome === "not-submitted"
|
|
330
|
+
? "failed"
|
|
331
|
+
: "ambiguous",
|
|
332
|
+
msgId: outcome.outcome === "accepted" ? outcome.msgId : null,
|
|
333
|
+
reason:
|
|
334
|
+
outcome.outcome === "accepted"
|
|
335
|
+
? null
|
|
336
|
+
: outcome.outcome === "not-submitted"
|
|
337
|
+
? "Transport did not submit input; check native transport and use peek/result."
|
|
338
|
+
: "No verified receipt; retry may duplicate input. Use peek/result.",
|
|
339
|
+
};
|
|
340
|
+
// the outcome is known for every notice: one damaged delivery file never discards it for the others
|
|
341
|
+
for (const q of due)
|
|
342
|
+
try {
|
|
343
|
+
writeDeliveryAttempt(q.dir, receipt);
|
|
344
|
+
} catch (e) {
|
|
345
|
+
log("warn", "notify", { error: errorMessage(e), dispatch: q.notice.dispatchId });
|
|
346
|
+
}
|
|
347
|
+
const live = currentSession(deps);
|
|
348
|
+
const stillHere = !stopped && live !== null && sessionKey(live) === sessionKey(target);
|
|
349
|
+
const marked = due.filter((q) => stillHere && owned(q.run));
|
|
350
|
+
// provably not submitted, so a retry cannot duplicate input: tried again a window later, a few times
|
|
351
|
+
if (receipt.status === "failed")
|
|
352
|
+
for (const q of marked) {
|
|
353
|
+
q.sendFailures = (q.sendFailures ?? 0) + 1;
|
|
354
|
+
if (q.sendFailures < MAX_SEND_FAILURES) busy.push(q);
|
|
355
|
+
}
|
|
356
|
+
if (receipt.status === "accepted" && target.host === "claude-code")
|
|
357
|
+
for (const q of marked)
|
|
358
|
+
try {
|
|
359
|
+
if (!existsSync(q.mark))
|
|
360
|
+
writeJsonAtomic(q.mark, { schema: 1, msgId: receipt.msgId, at: receipt.at });
|
|
361
|
+
} catch (e) {
|
|
362
|
+
log("warn", "notify", { error: errorMessage(e), dispatch: q.notice.dispatchId });
|
|
363
|
+
}
|
|
364
|
+
log(receipt.status === "accepted" ? "info" : "warn", "notify", {
|
|
365
|
+
outcome: receipt.status,
|
|
366
|
+
msgId: receipt.msgId,
|
|
367
|
+
reason: receipt.reason,
|
|
368
|
+
dispatches: notices.map((n) => n.dispatchId),
|
|
369
|
+
unmarked: due.filter((q) => !marked.includes(q)).map((q) => q.notice.dispatchId),
|
|
370
|
+
});
|
|
205
371
|
} finally {
|
|
206
372
|
for (const release of claims) release();
|
|
207
373
|
requeue(busy);
|
|
208
374
|
}
|
|
209
375
|
}
|
|
210
376
|
|
|
211
|
-
/**
|
|
377
|
+
/**
|
|
378
|
+
* Queues again the notices another server held the claim on, whose batch could not claim them all, or that
|
|
379
|
+
* the transport provably did not submit; the next pass drops those no longer due or owned.
|
|
380
|
+
*/
|
|
212
381
|
function requeue(busy: Queued[]): void {
|
|
213
382
|
if (stopped || busy.length === 0) return;
|
|
214
383
|
for (const q of busy) if (!queue.has(q.key)) queue.set(q.key, q);
|
|
215
384
|
schedule();
|
|
216
385
|
}
|
|
217
386
|
|
|
387
|
+
function queueNotice(q: Queued): void {
|
|
388
|
+
const target = currentSession(deps);
|
|
389
|
+
if (stopped || !target || queue.has(q.key) || !q.due() || !owned(q.run)) return;
|
|
390
|
+
let recovering = false;
|
|
391
|
+
try {
|
|
392
|
+
recovering = readDelivery(q.dir).some(
|
|
393
|
+
(a) => a.status === "submitting" && a.eventIds.includes(q.notice.eventId),
|
|
394
|
+
);
|
|
395
|
+
} catch (e) {
|
|
396
|
+
log("warn", "notify", { error: errorMessage(e), dispatch: q.notice.dispatchId });
|
|
397
|
+
return;
|
|
398
|
+
}
|
|
399
|
+
if (!recovering && !pending(q, target)) return;
|
|
400
|
+
queue.set(q.key, q);
|
|
401
|
+
schedule();
|
|
402
|
+
}
|
|
403
|
+
|
|
218
404
|
const enqueue = (run: Run, d: Dispatch, record: RunRecord): void => {
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
key:
|
|
405
|
+
const notice = finishedNotice(run, d, record);
|
|
406
|
+
queueNotice({
|
|
407
|
+
key: notice.eventId,
|
|
222
408
|
run,
|
|
223
|
-
|
|
409
|
+
dir: d.dir,
|
|
410
|
+
notice,
|
|
224
411
|
mark: dispatchPaths(d.dir).notified,
|
|
225
|
-
due: () => awaitsCollect(d.dir)
|
|
412
|
+
due: () => awaitsCollect(d.dir),
|
|
226
413
|
});
|
|
227
|
-
schedule();
|
|
228
414
|
};
|
|
229
415
|
|
|
230
416
|
const onStall = (s: Stalled): void => {
|
|
231
417
|
try {
|
|
232
418
|
const p = dispatchPaths(s.d.dir);
|
|
233
|
-
const
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
key,
|
|
419
|
+
const notice = stalledNotice(s.run, s.d, s.quietMs, deps.now());
|
|
420
|
+
queueNotice({
|
|
421
|
+
key: notice.eventId,
|
|
237
422
|
run: s.run,
|
|
238
|
-
|
|
423
|
+
dir: s.d.dir,
|
|
424
|
+
notice,
|
|
239
425
|
mark: p.stallNotified,
|
|
240
|
-
|
|
241
|
-
due: () => !existsSync(p.stallNotified) && !existsSync(p.exit),
|
|
426
|
+
due: () => !existsSync(p.exit),
|
|
242
427
|
});
|
|
243
|
-
schedule();
|
|
244
428
|
} catch (e) {
|
|
245
429
|
log("warn", "notify", { run: s.run.id, name: s.d.admit.name, error: errorMessage(e) });
|
|
246
430
|
}
|
|
@@ -264,9 +448,30 @@ export function startNotifier(deps: Deps, o: NotifierOptions = {}): Notifier {
|
|
|
264
448
|
if (!owned(run)) continue;
|
|
265
449
|
const records = new Map(readRecords(run).records.map((r) => [r.dispatchId, r]));
|
|
266
450
|
for (const d of listDispatches(run)) {
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
451
|
+
// one dispatch's damaged evidence stays ambiguous for it alone: the rest of the run is still scanned
|
|
452
|
+
try {
|
|
453
|
+
const p = dispatchPaths(d.dir);
|
|
454
|
+
for (const [kind, mark] of [
|
|
455
|
+
["finished", p.notified],
|
|
456
|
+
["stalled", p.stallNotified],
|
|
457
|
+
] as const) {
|
|
458
|
+
const release = tryLock(mark);
|
|
459
|
+
if (!release) continue;
|
|
460
|
+
try {
|
|
461
|
+
recoverSubmission(d.dir, JSON.stringify([run.id, d.admit.dispatchId, kind]));
|
|
462
|
+
} finally {
|
|
463
|
+
release();
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
const r = records.get(d.admit.dispatchId);
|
|
467
|
+
// a limit not yet failed over is reconcile's to settle first: its hook announces it then
|
|
468
|
+
if (r && !(r.status === "limit" && !readFailover(d.dir))) enqueue(run, d, r);
|
|
469
|
+
if (existsSync(p.stall) && !existsSync(p.exit)) {
|
|
470
|
+
onStall({ run, d, quietMs: readStallQuietMs(d.dir) });
|
|
471
|
+
}
|
|
472
|
+
} catch (e) {
|
|
473
|
+
log("warn", "notify", { run: run.id, name: d.admit.name, error: errorMessage(e) });
|
|
474
|
+
}
|
|
270
475
|
}
|
|
271
476
|
} catch (e) {
|
|
272
477
|
log("warn", "notify", { run: run.id, error: errorMessage(e) });
|
|
@@ -290,7 +495,54 @@ export function startNotifier(deps: Deps, o: NotifierOptions = {}): Notifier {
|
|
|
290
495
|
queue.clear();
|
|
291
496
|
},
|
|
292
497
|
};
|
|
293
|
-
|
|
294
|
-
|
|
498
|
+
if (!retryEvent) {
|
|
499
|
+
settledHooks.add(onSettled);
|
|
500
|
+
stallHooks.add(onStall);
|
|
501
|
+
}
|
|
295
502
|
return n;
|
|
296
503
|
}
|
|
504
|
+
|
|
505
|
+
/** Retry only a stored event for the current owner, after an explicit duplicate-risk decision. */
|
|
506
|
+
export async function retryDelivery(
|
|
507
|
+
deps: Deps,
|
|
508
|
+
dir: string,
|
|
509
|
+
eventId: string,
|
|
510
|
+
decision: { allowPossibleDuplicate: true },
|
|
511
|
+
): Promise<void> {
|
|
512
|
+
if (decision?.allowPossibleDuplicate !== true)
|
|
513
|
+
throw new CatherdError("E_INPUT_INVALID", "Retry requires acknowledging possible duplicate input");
|
|
514
|
+
const target = currentSession(deps);
|
|
515
|
+
const located = listRuns().runs.flatMap((run) =>
|
|
516
|
+
listDispatches(run)
|
|
517
|
+
.filter((d) => d.dir === dir)
|
|
518
|
+
.map((d) => ({ run, d })),
|
|
519
|
+
)[0];
|
|
520
|
+
if (!target || !located || !ownsRun(deps, located.run))
|
|
521
|
+
throw new CatherdError(
|
|
522
|
+
"E_INPUT_INVALID",
|
|
523
|
+
"Retry requires the validated current owner and stored dispatch",
|
|
524
|
+
);
|
|
525
|
+
const { run, d } = located;
|
|
526
|
+
const record = readRecords(run).records.find((r) => r.dispatchId === d.admit.dispatchId);
|
|
527
|
+
const finished = JSON.stringify([run.id, d.admit.dispatchId, "finished"]);
|
|
528
|
+
const stalled = JSON.stringify([run.id, d.admit.dispatchId, "stalled"]);
|
|
529
|
+
if (eventId !== finished && eventId !== stalled)
|
|
530
|
+
throw new CatherdError("E_INPUT_INVALID", "Event does not belong to this dispatch");
|
|
531
|
+
if (eventId === finished && (!record || !awaitsCollect(dir)))
|
|
532
|
+
throw new CatherdError("E_INPUT_INVALID", "Finished result is absent or already collected");
|
|
533
|
+
if (eventId === stalled && (!existsSync(dispatchPaths(dir).stall) || existsSync(dispatchPaths(dir).exit)))
|
|
534
|
+
throw new CatherdError("E_INPUT_INVALID", "Stalled event is absent or already finished");
|
|
535
|
+
// Corrupt evidence cannot be safely replaced by an explicit retry.
|
|
536
|
+
readDelivery(dir);
|
|
537
|
+
const n = notifierFor(deps, { coalesceMs: 0 }, eventId);
|
|
538
|
+
try {
|
|
539
|
+
if (eventId === finished)
|
|
540
|
+
n.onSettled({ run, d, record: record!, hints: [], started: null, pause: null, stateHints: [] });
|
|
541
|
+
else {
|
|
542
|
+
n.onStall({ run, d, quietMs: readStallQuietMs(dir) });
|
|
543
|
+
}
|
|
544
|
+
await n.idle();
|
|
545
|
+
} finally {
|
|
546
|
+
n.stop();
|
|
547
|
+
}
|
|
548
|
+
}
|
package/src/services/peek.ts
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
|
+
import type { HostContext, HostSessionRef, KnownHost } from "../domain/host.ts";
|
|
1
2
|
import { assertId } from "../domain/ids.ts";
|
|
2
3
|
import { noticeHeader } from "../domain/notice.ts";
|
|
4
|
+
import { knownQueueCapability, type QueueCapability } from "../infra/codex-queue.ts";
|
|
5
|
+
import type { DeliveryState } from "../infra/delivery.ts";
|
|
6
|
+
import { inspectionHost, inspectDelivery, inspectDeliveries, type DeliveryInspection } from "./run-debug.ts";
|
|
3
7
|
import { awaitsCollect } from "../infra/dispatch-dir.ts";
|
|
4
8
|
import { claim } from "./dispatch-service.ts";
|
|
5
9
|
import { type Dispatch, type DispatchState, listDispatches, liveDispatches } from "./dispatches.ts";
|
|
@@ -8,7 +12,7 @@ import { finishedNotice } from "./notifier.ts";
|
|
|
8
12
|
import type { Deps } from "./ports.ts";
|
|
9
13
|
import { type Reentry, reentry } from "./reentry.ts";
|
|
10
14
|
import { findRun, listRuns, readAgentRuns, readRecords, type Run } from "./run-store.ts";
|
|
11
|
-
import {
|
|
15
|
+
import { ownsRun, runOwner } from "./sessions.ts";
|
|
12
16
|
import { readNotes } from "./state.ts";
|
|
13
17
|
|
|
14
18
|
export interface PeekRole {
|
|
@@ -28,9 +32,19 @@ export interface PeekRun extends Reentry {
|
|
|
28
32
|
title: string;
|
|
29
33
|
/** the session that owns the run now, null when none has */
|
|
30
34
|
owner: string | null;
|
|
35
|
+
ownerHost: KnownHost | null;
|
|
31
36
|
live: PeekRole[];
|
|
37
|
+
/** Stalled notices are separate from unread finished role results. */
|
|
38
|
+
delivery: (DeliveryInspection & { name: string; dispatchId: string })[];
|
|
32
39
|
/** each finished record not yet read, as the first line of its message */
|
|
33
|
-
unread: {
|
|
40
|
+
unread: {
|
|
41
|
+
name: string;
|
|
42
|
+
dispatchId: string;
|
|
43
|
+
header: string;
|
|
44
|
+
eventId: string;
|
|
45
|
+
delivery: Exclude<DeliveryState, "collected">;
|
|
46
|
+
receipt: { msgId: string; target: HostSessionRef; at: string } | null;
|
|
47
|
+
}[];
|
|
34
48
|
/** the latest native Claude run recorded with record_agent_run */
|
|
35
49
|
native: { name: string; role: string; rung: string; status: string; at: string } | null;
|
|
36
50
|
/** the run's next step (state.md's last line) */
|
|
@@ -49,6 +63,7 @@ function peekRun(deps: Deps, run: Run, name: string | undefined): PeekRun {
|
|
|
49
63
|
run: run.id,
|
|
50
64
|
title: run.meta.title,
|
|
51
65
|
owner: runOwner(run)?.sessionId ?? null,
|
|
66
|
+
ownerHost: runOwner(run)?.host ?? null,
|
|
52
67
|
live: liveDispatches(run, now)
|
|
53
68
|
.filter(mine)
|
|
54
69
|
.map((d) => ({
|
|
@@ -59,13 +74,31 @@ function peekRun(deps: Deps, run: Run, name: string | undefined): PeekRun {
|
|
|
59
74
|
secs: Math.max(0, Math.round((now - Date.parse(d.admit.admittedAt)) / 1000)),
|
|
60
75
|
lastEvent: lastActivity(d),
|
|
61
76
|
})),
|
|
77
|
+
delivery: listDispatches(run)
|
|
78
|
+
.filter(mine)
|
|
79
|
+
.flatMap((d) =>
|
|
80
|
+
inspectDeliveries(run, d, false).map((inspection) => ({
|
|
81
|
+
name: d.admit.name,
|
|
82
|
+
dispatchId: d.admit.dispatchId,
|
|
83
|
+
...inspection,
|
|
84
|
+
})),
|
|
85
|
+
),
|
|
62
86
|
unread: listDispatches(run)
|
|
63
87
|
.filter(mine)
|
|
64
88
|
.flatMap((d) => {
|
|
65
89
|
const r = records.get(d.admit.dispatchId);
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
90
|
+
if (!r || !awaitsCollect(d.dir)) return [];
|
|
91
|
+
const delivery = inspectDelivery(run, d, true);
|
|
92
|
+
if (delivery.delivery === "collected") return [];
|
|
93
|
+
return [
|
|
94
|
+
{
|
|
95
|
+
...delivery,
|
|
96
|
+
delivery: delivery.delivery,
|
|
97
|
+
name: r.name,
|
|
98
|
+
dispatchId: r.dispatchId,
|
|
99
|
+
header: noticeHeader(finishedNotice(run, d, r)),
|
|
100
|
+
},
|
|
101
|
+
];
|
|
69
102
|
}),
|
|
70
103
|
native: native
|
|
71
104
|
? { name: native.name, role: native.role, rung: native.rung, status: native.status, at: native.at }
|
|
@@ -84,7 +117,7 @@ function peekRun(deps: Deps, run: Run, name: string | undefined): PeekRun {
|
|
|
84
117
|
export async function peek(
|
|
85
118
|
deps: Deps,
|
|
86
119
|
i: { run?: string; name?: string },
|
|
87
|
-
): Promise<{ runs: PeekRun[]; hints: string[] }> {
|
|
120
|
+
): Promise<{ runs: PeekRun[]; hints: string[]; host: HostContext; queue: QueueCapability | null }> {
|
|
88
121
|
if (i.name !== undefined) assertId("role name", i.name);
|
|
89
122
|
let runs: Run[];
|
|
90
123
|
if (i.run) {
|
|
@@ -94,10 +127,14 @@ export async function peek(
|
|
|
94
127
|
runs = [run];
|
|
95
128
|
} else {
|
|
96
129
|
const all = listRuns().runs;
|
|
97
|
-
const
|
|
98
|
-
const owned = me ? all.filter((r) => runOwner(r)?.sessionId === me.sessionId) : [];
|
|
130
|
+
const owned = all.filter((r) => ownsRun(deps, r));
|
|
99
131
|
runs = owned.length ? owned : all.slice(0, 1);
|
|
100
132
|
}
|
|
101
133
|
const hints = runs.length === 0 ? ["no runs yet: run_start(repo, title, a_lines) starts one"] : [];
|
|
102
|
-
return {
|
|
134
|
+
return {
|
|
135
|
+
runs: runs.map((r) => peekRun(deps, r, i.name)),
|
|
136
|
+
hints,
|
|
137
|
+
host: inspectionHost(deps.host),
|
|
138
|
+
queue: deps.host.host === "codex" && !deps.host.conflict ? knownQueueCapability(process.env) : null,
|
|
139
|
+
};
|
|
103
140
|
}
|
package/src/services/ports.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { HostContext } from "../domain/host.ts";
|
|
1
2
|
import type { Provenance } from "./provenance.ts";
|
|
2
3
|
import type { Budget } from "../domain/budget.ts";
|
|
3
4
|
import type { SessionEnv } from "../infra/claude-session.ts";
|
|
@@ -6,7 +7,7 @@ import type { Verdict } from "../domain/jev.ts";
|
|
|
6
7
|
import type { Difficulty, Kind } from "../domain/lane.ts";
|
|
7
8
|
import type { Access } from "../domain/record.ts";
|
|
8
9
|
import type { Role } from "../domain/roles.ts";
|
|
9
|
-
import type { Change, ProfilePatch } from "../domain/profile.ts";
|
|
10
|
+
import type { Change, ProfileDoc, ProfilePatch } from "../domain/profile.ts";
|
|
10
11
|
import type { Issue } from "../domain/profile-rules.ts";
|
|
11
12
|
import type { RouteJev, RouteSource } from "../domain/route.ts";
|
|
12
13
|
import type { SyncReport } from "./source-sync.ts";
|
|
@@ -46,6 +47,9 @@ export interface ProfileSaved {
|
|
|
46
47
|
}
|
|
47
48
|
|
|
48
49
|
export interface ProfilePort {
|
|
50
|
+
withHost?(host: () => HostContext): ProfilePort;
|
|
51
|
+
budgetFor?(repo: string | null): Budget;
|
|
52
|
+
raw?(name?: string, repo?: string | null): ProfileDoc;
|
|
49
53
|
/** The profile bound to `repo` (a git toplevel), else the active one; null asks for the active one. */
|
|
50
54
|
forRepo(repo: string | null): ProfileView;
|
|
51
55
|
/** Without a name (get, validate, set): the profile `repo`, a repository's toplevel, runs on. */
|
|
@@ -69,6 +73,7 @@ export interface ProfilePort {
|
|
|
69
73
|
}
|
|
70
74
|
|
|
71
75
|
export interface RouteRequest {
|
|
76
|
+
host?: HostContext["host"];
|
|
72
77
|
runDir: string;
|
|
73
78
|
repo: string;
|
|
74
79
|
profile: ProfileView;
|
|
@@ -128,12 +133,13 @@ export interface RoutingPort {
|
|
|
128
133
|
|
|
129
134
|
/** Everything a service needs from outside it; the entry layer builds one, tests build fakes. */
|
|
130
135
|
export interface Deps {
|
|
136
|
+
host: HostContext;
|
|
131
137
|
profiles: ProfilePort;
|
|
132
138
|
routing: RoutingPort;
|
|
133
139
|
version: string;
|
|
134
140
|
/** how often the supervisor and the dispatch watchers poll, in ms */
|
|
135
141
|
pollMs: number;
|
|
136
|
-
/**
|
|
142
|
+
/** Claude transport credentials and registry refresh; host identity lives in `host`. */
|
|
137
143
|
session: SessionEnv | null;
|
|
138
144
|
now: () => number;
|
|
139
145
|
/** spec 1.2 §3.2 `catalog_sync`; default: the real sync (tests inject one that never reaches the network) */
|