@stage-labs/metro 0.1.0-beta.261 → 0.1.0-beta.263
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/runtime/marketplace/plugin/.claude-plugin/plugin.json +1 -1
- package/runtime/node_modules/@metro-labs/core/src/events.ts +45 -1
- package/runtime/node_modules/@metro-labs/core/src/runner-activity.ts +72 -2
- package/runtime/node_modules/@metro-labs/daemon/src/agents/map.ts +2 -0
- package/runtime/node_modules/@metro-labs/daemon/src/claude/session-continuity.ts +11 -2
- package/runtime/node_modules/@metro-labs/daemon/src/monitor/run-event.ts +139 -0
- package/runtime/node_modules/@metro-labs/daemon/src/monitor/run-events.ts +101 -0
- package/runtime/node_modules/@metro-labs/daemon/src/routes/session-apis.ts +2 -0
- package/runtime/runtime.json +1 -1
- package/runtime/sdk-runner/node_modules/@metro-labs/core/src/events.ts +45 -1
- package/runtime/sdk-runner/node_modules/@metro-labs/core/src/runner-activity.ts +72 -2
- package/runtime/sdk-runner/runner.json +1 -1
- package/runtime/sdk-runner/src/activity.ts +81 -25
- package/runtime/sdk-runner/src/app.ts +4 -4
- package/runtime/sdk-runner/src/inbox.ts +32 -7
- package/runtime/sdk-runner/src/link.ts +1 -0
- package/runtime/sdk-runner/src/main.ts +1 -1
- package/runtime/sdk-runner/src/recovery.ts +46 -0
- package/runtime/sdk-runner/src/runner.ts +54 -20
- package/runtime/sdk-runner/src/session-store.ts +39 -9
- package/runtime/sdk-runner/src/session-watch.ts +12 -0
|
@@ -1,8 +1,9 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
1
2
|
import { readFileSync } from 'node:fs';
|
|
2
3
|
import type { SDKMessage } from '@anthropic-ai/claude-agent-sdk';
|
|
3
4
|
import { isRecord } from '@metro-labs/core/is-record';
|
|
4
5
|
import { log } from '@metro-labs/core/log';
|
|
5
|
-
import type
|
|
6
|
+
import { runnerFailureSummary, type RunnerActivity, type RunnerEvent, type RunnerEventKind, type RunnerFailure, type RunnerFailureCode, type RunnerPhase, type RunnerTool } from '@metro-labs/core/runner-activity';
|
|
6
7
|
import { writeAtomic } from '@metro-labs/core/secure-fs';
|
|
7
8
|
import { ActivityTasks, activityName } from './activity-tasks.js';
|
|
8
9
|
import type { ApprovalAsk } from './approvals.js';
|
|
@@ -16,14 +17,34 @@ type System = Extract<SDKMessage, { type: 'system' }>;
|
|
|
16
17
|
interface TrackedTool extends RunnerTool { parent: string | null }
|
|
17
18
|
interface Approval { tool: string | null; taskId: string | null; worker: boolean }
|
|
18
19
|
|
|
19
|
-
|
|
20
|
+
function sessionFailureCode(err: unknown): RunnerFailureCode {
|
|
20
21
|
const text = err instanceof Error ? err.message : String(err);
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
if (/
|
|
24
|
-
if (/
|
|
25
|
-
if (/
|
|
26
|
-
|
|
22
|
+
for (const code of ['status_unavailable', 'session_ended', 'interrupted', 'saved_state', 'authentication', 'executable', 'connection'] as const)
|
|
23
|
+
if (text === runnerFailureSummary(code)) return code;
|
|
24
|
+
if (/saved Agent SDK state/i.test(text)) return 'saved_state';
|
|
25
|
+
if (/unauthorized|authentication|invalid.*key|\b40[13]\b/i.test(text)) return 'authentication';
|
|
26
|
+
if (/ENOENT|not found|executable|spawn/i.test(text)) return 'executable';
|
|
27
|
+
if (/ECONN|network|fetch failed|timed? ?out/i.test(text)) return 'connection';
|
|
28
|
+
return 'session_error';
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function failureSummary(err: unknown): string { return runnerFailureSummary(sessionFailureCode(err)); }
|
|
32
|
+
|
|
33
|
+
const PROVIDER_ERRORS: Record<NonNullable<Extract<SDKMessage, { type: 'assistant' }>['error']>, RunnerFailureCode> = {
|
|
34
|
+
authentication_failed: 'authentication', oauth_org_not_allowed: 'authentication', cloud_credential_error: 'authentication', account_on_hold: 'authentication', verification_required: 'authentication',
|
|
35
|
+
billing_error: 'billing', rate_limit: 'rate_limit', overloaded: 'provider_unavailable', server_error: 'provider_unavailable',
|
|
36
|
+
max_output_tokens: 'output_limit', invalid_request: 'invalid_request', model_not_found: 'invalid_request', unknown: 'provider_error',
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
function providerFailure(status: number | null | undefined): RunnerFailureCode {
|
|
40
|
+
const codes: Record<number, RunnerFailureCode | undefined> = { 400: 'invalid_request', 401: 'authentication', 402: 'billing', 403: 'authentication', 404: 'invalid_request', 429: 'rate_limit' };
|
|
41
|
+
if (status === null || status === undefined) return 'provider_error';
|
|
42
|
+
return codes[status] ?? (status >= 500 && status <= 599 ? 'provider_unavailable' : 'provider_error');
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function readTokenLimit(content: unknown): boolean {
|
|
46
|
+
if (typeof content === 'string') return /File content \([\d,]+ tokens\) exceeds maximum allowed tokens \([\d,]+\)/.test(content.slice(0, 2_048));
|
|
47
|
+
return Array.isArray(content) && content.slice(0, 8).some((block: unknown) => isRecord(block) && block.type === 'text' && typeof block.text === 'string' && readTokenLimit(block.text));
|
|
27
48
|
}
|
|
28
49
|
|
|
29
50
|
function processIdentity(): Pick<RunnerActivity, 'procStart'> {
|
|
@@ -44,6 +65,8 @@ export class Activity {
|
|
|
44
65
|
private readonly approvals = new Map<string, Approval>();
|
|
45
66
|
private busy = false;
|
|
46
67
|
private failed = false;
|
|
68
|
+
private providerCode: RunnerFailureCode | null = null;
|
|
69
|
+
private interrupted: RunnerFailure | null = null;
|
|
47
70
|
private compacting = false;
|
|
48
71
|
private ready = false;
|
|
49
72
|
private timer: ReturnType<typeof setInterval> | null = null;
|
|
@@ -51,7 +74,7 @@ export class Activity {
|
|
|
51
74
|
private writeFailed = false;
|
|
52
75
|
private readonly state: RunnerActivity = {
|
|
53
76
|
pid: process.pid, ...processIdentity(), runner: 'sdk', phase: 'starting', mainPhase: 'starting', mainStartedAt: null, sessionId: null, updatedAt: Date.now(),
|
|
54
|
-
pending: 0, workers: 0, approvals: 0, tools: [], activeTools: [], tasks: [], events: [], lastError: null,
|
|
77
|
+
pending: 0, workers: 0, approvals: 0, tools: [], activeTools: [], tasks: [], events: [], lastError: null, activeFailure: null, lastFailure: null,
|
|
55
78
|
};
|
|
56
79
|
|
|
57
80
|
constructor(private readonly path: string) {}
|
|
@@ -68,6 +91,7 @@ export class Activity {
|
|
|
68
91
|
return {
|
|
69
92
|
...s, tools: [...s.tools], activeTools: s.activeTools.map((tool) => ({ ...tool })),
|
|
70
93
|
tasks: s.tasks.map((task) => ({ ...task })), events: s.events.map((event) => ({ ...event })),
|
|
94
|
+
activeFailure: s.activeFailure ? { ...s.activeFailure } : null, lastFailure: s.lastFailure ? { ...s.lastFailure } : null,
|
|
71
95
|
};
|
|
72
96
|
}
|
|
73
97
|
|
|
@@ -97,11 +121,11 @@ export class Activity {
|
|
|
97
121
|
|
|
98
122
|
fail(message: string): void {
|
|
99
123
|
this.failed = true;
|
|
100
|
-
this.
|
|
101
|
-
|
|
124
|
+
const failure = this.failure('session_failed', sessionFailureCode(message), true);
|
|
125
|
+
if (failure.code === 'interrupted') this.interrupted = failure;
|
|
102
126
|
this.state.phase = 'error';
|
|
103
127
|
this.state.mainPhase = 'error';
|
|
104
|
-
log.error({ reason:
|
|
128
|
+
log.error({ reason: failure.code }, 'sdk-runner: activity error');
|
|
105
129
|
this.flush();
|
|
106
130
|
}
|
|
107
131
|
|
|
@@ -155,8 +179,7 @@ export class Activity {
|
|
|
155
179
|
this.compacting = compacting;
|
|
156
180
|
if (message.compact_result === 'success') this.event('compacted');
|
|
157
181
|
else if (message.compact_result === 'failed') {
|
|
158
|
-
this.
|
|
159
|
-
this.event('compact_failed');
|
|
182
|
+
this.failure('compact_failed', 'compact_error', false, null, null, { id: message.uuid });
|
|
160
183
|
}
|
|
161
184
|
}
|
|
162
185
|
|
|
@@ -171,6 +194,8 @@ export class Activity {
|
|
|
171
194
|
private turn(): void {
|
|
172
195
|
this.busy = true;
|
|
173
196
|
if (this.state.mainStartedAt !== null) return;
|
|
197
|
+
this.clearFailure();
|
|
198
|
+
this.providerCode = null;
|
|
174
199
|
this.state.mainStartedAt = Date.now();
|
|
175
200
|
this.event('turn_started');
|
|
176
201
|
}
|
|
@@ -189,8 +214,8 @@ export class Activity {
|
|
|
189
214
|
}
|
|
190
215
|
|
|
191
216
|
private taskEvent(kind: RunnerEventKind, tool: string | null, taskId: string | null): void {
|
|
192
|
-
if (kind === 'task_failed') this.
|
|
193
|
-
this.event(kind, tool, taskId);
|
|
217
|
+
if (kind === 'task_failed') this.failure(kind, 'task_error', false, tool, taskId);
|
|
218
|
+
else this.event(kind, tool, taskId);
|
|
194
219
|
}
|
|
195
220
|
|
|
196
221
|
private result(message: Extract<SDKMessage, { type: 'result' }>): void {
|
|
@@ -200,25 +225,30 @@ export class Activity {
|
|
|
200
225
|
for (const [id, tool] of this.tools) if (tool.parent === null) this.tools.delete(id);
|
|
201
226
|
this.failed = message.is_error || message.subtype !== 'success';
|
|
202
227
|
if (!this.failed) {
|
|
228
|
+
this.clearFailure();
|
|
229
|
+
this.providerCode = null;
|
|
203
230
|
this.event('turn_finished');
|
|
204
231
|
return;
|
|
205
232
|
}
|
|
206
|
-
const code = message.subtype === 'success' ?
|
|
207
|
-
this.
|
|
208
|
-
this.
|
|
233
|
+
const code = message.subtype === 'success' ? this.providerCode ?? providerFailure(message.api_error_status) : message.subtype;
|
|
234
|
+
this.providerCode = null;
|
|
235
|
+
this.failure('turn_failed', code, true, null, null, { id: message.uuid });
|
|
209
236
|
log.error({ reason: code }, 'sdk-runner: turn failed');
|
|
210
237
|
}
|
|
211
238
|
|
|
212
239
|
private content(message: Extract<SDKMessage, { type: 'assistant' | 'user' }>): void {
|
|
213
240
|
const content = message.message.content;
|
|
214
241
|
const parent = message.parent_tool_use_id ?? null;
|
|
242
|
+
if (parent === null && message.type === 'assistant') {
|
|
243
|
+
this.turn();
|
|
244
|
+
if (message.error !== undefined) this.providerCode = message.error === 'unknown' ? null : PROVIDER_ERRORS[message.error];
|
|
245
|
+
}
|
|
215
246
|
if (Array.isArray(content)) for (const block of content) if (isRecord(block)) this.block(block, parent);
|
|
216
|
-
if (parent === null && message.type === 'assistant') this.turn();
|
|
217
247
|
}
|
|
218
248
|
|
|
219
249
|
private block(block: Record<string, unknown>, parent: string | null): void {
|
|
220
250
|
if (block.type === 'tool_use' && typeof block.id === 'string') this.tool(block.id, block.name, parent);
|
|
221
|
-
else if (block.type === 'tool_result' && typeof block.tool_use_id === 'string') this.finish(block.tool_use_id, block.is_error === true, parent);
|
|
251
|
+
else if (block.type === 'tool_result' && typeof block.tool_use_id === 'string') this.finish(block.tool_use_id, block.is_error === true, parent, block.content);
|
|
222
252
|
}
|
|
223
253
|
|
|
224
254
|
private tool(id: string, rawName: unknown, parent: string | null): void {
|
|
@@ -231,14 +261,13 @@ export class Activity {
|
|
|
231
261
|
log.info({ tool: name, worker: taskId }, 'sdk-runner: tool started');
|
|
232
262
|
}
|
|
233
263
|
|
|
234
|
-
private finish(id: string, failed: boolean, parent: string | null): void {
|
|
264
|
+
private finish(id: string, failed: boolean, parent: string | null, content: unknown): void {
|
|
235
265
|
const tool = this.tools.get(id);
|
|
236
266
|
if (tool === undefined && this.tasks.isHidden(parent)) return;
|
|
237
267
|
this.tools.delete(id);
|
|
238
268
|
const name = tool?.name ?? 'tool';
|
|
239
269
|
if (failed) {
|
|
240
|
-
this.
|
|
241
|
-
this.event('tool_failed', name, tool?.taskId ?? null);
|
|
270
|
+
this.toolFailure(id, tool, parent, content);
|
|
242
271
|
log.warn({ tool: name }, 'sdk-runner: tool failed');
|
|
243
272
|
return;
|
|
244
273
|
}
|
|
@@ -246,8 +275,35 @@ export class Activity {
|
|
|
246
275
|
log.info({ tool: name }, 'sdk-runner: tool finished');
|
|
247
276
|
}
|
|
248
277
|
|
|
278
|
+
private toolFailure(id: string, tool: TrackedTool | undefined, parent: string | null, content: unknown): void {
|
|
279
|
+
const name = tool?.name ?? 'tool';
|
|
280
|
+
const code = name === 'Read' && readTokenLimit(content) ? 'read_token_limit' : 'tool_error';
|
|
281
|
+
this.failure('tool_failed', code, false, name, tool?.taskId ?? this.tasks.taskFor(parent), { toolUseId: id });
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
private clearFailure(): void {
|
|
285
|
+
this.failed = false;
|
|
286
|
+
this.state.activeFailure = this.interrupted;
|
|
287
|
+
this.state.lastError = this.interrupted === null ? null : runnerFailureSummary(this.interrupted.code);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
private failure(kind: RunnerEventKind, code: RunnerFailureCode, active: boolean, tool: string | null = null, taskId: string | null = null, identity: Pick<RunnerEvent, 'id' | 'toolUseId'> = {}): RunnerFailure {
|
|
291
|
+
const failure: RunnerFailure = { id: activityName(identity.id) ?? randomUUID(), at: Date.now(), kind, code, tool, taskId, ...(activityName(identity.toolUseId) === null ? {} : { toolUseId: identity.toolUseId }) };
|
|
292
|
+
this.state.lastFailure = failure;
|
|
293
|
+
if (active) {
|
|
294
|
+
this.state.activeFailure = failure;
|
|
295
|
+
this.state.lastError = runnerFailureSummary(code, tool);
|
|
296
|
+
}
|
|
297
|
+
this.record(failure);
|
|
298
|
+
return failure;
|
|
299
|
+
}
|
|
300
|
+
|
|
249
301
|
private event(kind: RunnerEventKind, tool: string | null = null, taskId: string | null = null): void {
|
|
250
|
-
this.
|
|
302
|
+
this.record({ id: randomUUID(), at: Date.now(), kind, tool, taskId });
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
private record(event: RunnerEvent): void {
|
|
306
|
+
this.state.events.unshift(event);
|
|
251
307
|
if (this.state.events.length > MAX_EVENTS) this.state.events.length = MAX_EVENTS;
|
|
252
308
|
}
|
|
253
309
|
|
|
@@ -14,7 +14,7 @@ export interface RunningAgent {
|
|
|
14
14
|
runner: Runner;
|
|
15
15
|
link: MetroLink;
|
|
16
16
|
done: Promise<void>;
|
|
17
|
-
stop(): Promise<void>;
|
|
17
|
+
stop(cancelActive?: boolean): Promise<void>;
|
|
18
18
|
}
|
|
19
19
|
|
|
20
20
|
export interface AppHooks {
|
|
@@ -42,10 +42,10 @@ async function connected(cfg: RunnerConfig, hooks: AppHooks, release: () => Prom
|
|
|
42
42
|
lost: (reason) => { hooks.lost(reason); },
|
|
43
43
|
});
|
|
44
44
|
let stopped = false;
|
|
45
|
-
const stop = async (): Promise<void> => {
|
|
45
|
+
const stop = async (cancelActive = true): Promise<void> => {
|
|
46
46
|
if (stopped) return;
|
|
47
47
|
stopped = true;
|
|
48
|
-
runner.close();
|
|
48
|
+
runner.close(cancelActive);
|
|
49
49
|
await link.close().catch(() => { log.debug('sdk-runner: closing the metro link failed'); });
|
|
50
50
|
hooks.activity?.stop();
|
|
51
51
|
await release();
|
|
@@ -57,7 +57,7 @@ async function connected(cfg: RunnerConfig, hooks: AppHooks, release: () => Prom
|
|
|
57
57
|
runner.start(runnerOptions(cfg, tools, approvalsThrough(link, (id, ask) => { hooks.activity?.approval(id, ask); }), resume, hooks.env));
|
|
58
58
|
return { runner, link, done: runner.run(hooks.observe), stop };
|
|
59
59
|
} catch (err) {
|
|
60
|
-
runner.close();
|
|
60
|
+
runner.close(false);
|
|
61
61
|
await link.close().catch(() => { log.debug('sdk-runner: closing the metro link failed'); });
|
|
62
62
|
throw err;
|
|
63
63
|
}
|
|
@@ -8,6 +8,8 @@ export type Priority = 'now' | 'next' | 'later';
|
|
|
8
8
|
export interface Unanswered {
|
|
9
9
|
text: string;
|
|
10
10
|
at: number;
|
|
11
|
+
uuid?: Uuid;
|
|
12
|
+
state?: 'queued' | 'started';
|
|
11
13
|
}
|
|
12
14
|
|
|
13
15
|
const KINDS_MAX = 2_000;
|
|
@@ -20,6 +22,7 @@ export class Inbox implements AsyncIterable<SDKUserMessage> {
|
|
|
20
22
|
private readonly ledger = new Map<string, Unanswered>();
|
|
21
23
|
private wake: (() => void) | null = null;
|
|
22
24
|
private closed = false;
|
|
25
|
+
private restoring = false;
|
|
23
26
|
|
|
24
27
|
constructor(private readonly changed: (unanswered: Unanswered[]) => void = () => undefined) {}
|
|
25
28
|
|
|
@@ -49,14 +52,29 @@ export class Inbox implements AsyncIterable<SDKUserMessage> {
|
|
|
49
52
|
return this.kinds.get(uuid);
|
|
50
53
|
}
|
|
51
54
|
|
|
52
|
-
|
|
55
|
+
started(uuids: readonly string[]): void {
|
|
56
|
+
let changed = false;
|
|
57
|
+
for (const uuid of uuids) {
|
|
58
|
+
const entry = this.ledger.get(uuid);
|
|
59
|
+
if (entry === undefined || entry.state === 'started') continue;
|
|
60
|
+
this.ledger.set(uuid, { ...entry, state: 'started' });
|
|
61
|
+
changed = true;
|
|
62
|
+
}
|
|
63
|
+
if (changed) this.changed(this.unanswered());
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
finished(uuids = this.startedUuids()): void {
|
|
53
67
|
let changed = false;
|
|
54
68
|
for (const uuid of uuids) changed = this.ledger.delete(uuid) || changed;
|
|
55
69
|
if (changed) this.changed(this.unanswered());
|
|
56
70
|
}
|
|
57
71
|
|
|
72
|
+
startedUuids(): string[] {
|
|
73
|
+
return [...this.ledger].filter(([, entry]) => entry.state === 'started').map(([uuid]) => uuid);
|
|
74
|
+
}
|
|
75
|
+
|
|
58
76
|
unanswered(now = Date.now()): Unanswered[] {
|
|
59
|
-
return [...this.ledger.values()].filter((entry) => now - entry.at < LEDGER_AGE_MS);
|
|
77
|
+
return [...this.ledger.values()].filter((entry) => entry.state === 'started' || now - entry.at < LEDGER_AGE_MS);
|
|
60
78
|
}
|
|
61
79
|
|
|
62
80
|
close(): void {
|
|
@@ -79,16 +97,23 @@ export class Inbox implements AsyncIterable<SDKUserMessage> {
|
|
|
79
97
|
}
|
|
80
98
|
|
|
81
99
|
again(left: readonly Unanswered[]): void {
|
|
82
|
-
|
|
100
|
+
if (left.length === 0) return;
|
|
101
|
+
this.restoring = true;
|
|
102
|
+
try {
|
|
103
|
+
for (const entry of left) this.push('chat', entry.text, undefined, entry.uuid ?? randomUUID(), entry.at);
|
|
104
|
+
} finally {
|
|
105
|
+
this.restoring = false;
|
|
106
|
+
}
|
|
107
|
+
this.changed(this.unanswered());
|
|
83
108
|
}
|
|
84
109
|
|
|
85
|
-
private remember(uuid:
|
|
86
|
-
this.ledger.set(uuid, { text, at });
|
|
110
|
+
private remember(uuid: Uuid, text: string, at: number): void {
|
|
111
|
+
this.ledger.set(uuid, { text, at, uuid, state: 'queued' });
|
|
87
112
|
while (this.ledger.size > LEDGER_MAX) {
|
|
88
|
-
const oldest = this.ledger.
|
|
113
|
+
const oldest = [...this.ledger].find(([, entry]) => entry.state !== 'started')?.[0];
|
|
89
114
|
if (oldest === undefined) break;
|
|
90
115
|
this.ledger.delete(oldest);
|
|
91
116
|
}
|
|
92
|
-
this.changed(this.unanswered());
|
|
117
|
+
if (!this.restoring) this.changed(this.unanswered());
|
|
93
118
|
}
|
|
94
119
|
}
|
|
@@ -79,6 +79,7 @@ export class MetroLink implements Asker {
|
|
|
79
79
|
}
|
|
80
80
|
|
|
81
81
|
ask(ask: PermissionAsk, signal: AbortSignal): Promise<Behavior> {
|
|
82
|
+
if (signal.aborted) return Promise.resolve('deny');
|
|
82
83
|
return new Promise((resolve) => {
|
|
83
84
|
const settle = (behavior: Behavior): void => {
|
|
84
85
|
this.waiting.delete(ask.request_id);
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { closeSync, openSync, readSync } from 'node:fs';
|
|
3
|
+
import { isRecord } from '@metro-labs/core/is-record';
|
|
4
|
+
import type { Unanswered } from './inbox.js';
|
|
5
|
+
|
|
6
|
+
export const INTERRUPTED_NOTICE = 'The SDK stopped before finishing a chat request. Check Conversations before retrying; actions may already have run.';
|
|
7
|
+
|
|
8
|
+
function scanTranscript(fd: number, wanted: ReadonlySet<string>): Set<string> {
|
|
9
|
+
const found = new Set<string>();
|
|
10
|
+
const chunk = Buffer.alloc(64 * 1024);
|
|
11
|
+
let overlap = '';
|
|
12
|
+
for (let size = readSync(fd, chunk); size > 0; size = readSync(fd, chunk)) {
|
|
13
|
+
const text = overlap + chunk.toString('latin1', 0, size);
|
|
14
|
+
for (const uuid of wanted) if (text.includes(`"${uuid}"`)) found.add(uuid);
|
|
15
|
+
if (found.size === wanted.size) break;
|
|
16
|
+
overlap = text.slice(-37);
|
|
17
|
+
}
|
|
18
|
+
return found;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function transcriptUuids(path: string | null, inputs: readonly Unanswered[]): Set<string> {
|
|
22
|
+
const wanted = new Set(inputs.flatMap((input) => input.state !== 'started' && input.uuid !== undefined ? [input.uuid] : []));
|
|
23
|
+
if (path === null || wanted.size === 0) return new Set();
|
|
24
|
+
let fd: number | undefined;
|
|
25
|
+
try {
|
|
26
|
+
fd = openSync(path, 'r');
|
|
27
|
+
return scanTranscript(fd, wanted);
|
|
28
|
+
} catch (err) {
|
|
29
|
+
if (isRecord(err) && err.code === 'ENOENT') return new Set();
|
|
30
|
+
throw new Error('The saved Agent SDK transcript cannot be checked. Restore it before starting.');
|
|
31
|
+
} finally {
|
|
32
|
+
if (fd !== undefined) closeSync(fd);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function recoverInputs(inputs: readonly Unanswered[], transcript: string | null): { unanswered: Unanswered[]; interrupted: Unanswered[] } {
|
|
37
|
+
const seen = transcriptUuids(transcript, inputs);
|
|
38
|
+
const unanswered: Unanswered[] = [];
|
|
39
|
+
const interrupted: Unanswered[] = [];
|
|
40
|
+
for (const input of inputs) {
|
|
41
|
+
const entry = { ...input, uuid: input.uuid ?? randomUUID(), state: input.state ?? 'queued' };
|
|
42
|
+
if (entry.state === 'started' || seen.has(entry.uuid)) interrupted.push({ ...entry, state: 'started' });
|
|
43
|
+
else unanswered.push(entry);
|
|
44
|
+
}
|
|
45
|
+
return { unanswered, interrupted };
|
|
46
|
+
}
|
|
@@ -6,6 +6,7 @@ import { channelText, type ChannelEvent } from './channel-text.js';
|
|
|
6
6
|
import type { RunnerConfig } from './config.js';
|
|
7
7
|
import { Inbox, type Unanswered, type Uuid } from './inbox.js';
|
|
8
8
|
import { CALL_ENDED, callStarted, callWords, FRONT_RULES } from './rules.js';
|
|
9
|
+
import { INTERRUPTED_NOTICE } from './recovery.js';
|
|
9
10
|
import type { SessionStore } from './session-store.js';
|
|
10
11
|
import { SessionWatch, startedCommand, uuidsOf } from './session-watch.js';
|
|
11
12
|
import { SpeechRouter, type SpeechSink } from './speech.js';
|
|
@@ -79,6 +80,9 @@ export class Runner {
|
|
|
79
80
|
private stall: ReturnType<typeof setTimeout> | null = null;
|
|
80
81
|
private model: string | null = null;
|
|
81
82
|
private switching: Promise<void> = Promise.resolve();
|
|
83
|
+
private ended = false;
|
|
84
|
+
private readonly interrupted: number;
|
|
85
|
+
private interruptionReported = false;
|
|
82
86
|
|
|
83
87
|
constructor(private readonly parts: RunnerParts) {
|
|
84
88
|
this.inbox = new Inbox((unanswered) => {
|
|
@@ -94,7 +98,8 @@ export class Runner {
|
|
|
94
98
|
},
|
|
95
99
|
});
|
|
96
100
|
this.watch = new SessionWatch(parts.readOnly);
|
|
97
|
-
const again = parts.store.
|
|
101
|
+
const { unanswered: again, interrupted } = parts.store.recover();
|
|
102
|
+
this.interrupted = interrupted.length;
|
|
98
103
|
this.inbox.again(again);
|
|
99
104
|
if (again.length > 0) log.info({ count: again.length }, 'sdk-runner: chat messages the last session never read go in again');
|
|
100
105
|
}
|
|
@@ -118,20 +123,33 @@ export class Runner {
|
|
|
118
123
|
|
|
119
124
|
async run(observe?: (message: SDKMessage) => void): Promise<void> {
|
|
120
125
|
if (this.session === null) throw new Error('the runner was not started');
|
|
121
|
-
|
|
122
|
-
const
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
126
|
+
try {
|
|
127
|
+
for await (const message of this.session) {
|
|
128
|
+
const m: Record<string, unknown> = { ...message };
|
|
129
|
+
this.trackInput(m);
|
|
130
|
+
this.watch.observe(m);
|
|
131
|
+
this.speech.observe(message);
|
|
132
|
+
this.note(m);
|
|
133
|
+
this.parts.activity?.observe(message);
|
|
134
|
+
observe?.(message);
|
|
135
|
+
}
|
|
136
|
+
} finally {
|
|
137
|
+
this.ended = true;
|
|
130
138
|
}
|
|
131
139
|
}
|
|
132
140
|
|
|
133
141
|
chat(event: ChannelEvent): Uuid {
|
|
134
|
-
|
|
142
|
+
try {
|
|
143
|
+
return this.inbox.push('chat', channelText(METRO_SERVER, event.content, event.meta));
|
|
144
|
+
} catch (err) {
|
|
145
|
+
log.error('sdk-runner: could not save incoming chat; stopping before further work');
|
|
146
|
+
try {
|
|
147
|
+
this.parts.activity?.fail(INTERRUPTED_NOTICE);
|
|
148
|
+
} finally {
|
|
149
|
+
this.close(false);
|
|
150
|
+
}
|
|
151
|
+
throw err;
|
|
152
|
+
}
|
|
135
153
|
}
|
|
136
154
|
|
|
137
155
|
callStarted(where: string): Uuid {
|
|
@@ -149,10 +167,14 @@ export class Runner {
|
|
|
149
167
|
return this.inbox.push('note', CALL_ENDED);
|
|
150
168
|
}
|
|
151
169
|
|
|
152
|
-
close(): void {
|
|
170
|
+
close(cancelActive = true): void {
|
|
153
171
|
this.clearStall();
|
|
154
|
-
|
|
155
|
-
|
|
172
|
+
try {
|
|
173
|
+
if (cancelActive && !this.ended) this.inbox.finished();
|
|
174
|
+
} finally {
|
|
175
|
+
this.inbox.close();
|
|
176
|
+
this.session?.close();
|
|
177
|
+
}
|
|
156
178
|
}
|
|
157
179
|
|
|
158
180
|
private callPush(text: string): Uuid {
|
|
@@ -199,6 +221,11 @@ export class Runner {
|
|
|
199
221
|
if (session === null) return;
|
|
200
222
|
await session.applyFlagSettings(allowedOnly(model));
|
|
201
223
|
this.parts.activity?.connected();
|
|
224
|
+
if (this.interrupted > 0 && !this.interruptionReported) {
|
|
225
|
+
this.interruptionReported = true;
|
|
226
|
+
this.parts.activity?.fail(INTERRUPTED_NOTICE);
|
|
227
|
+
log.warn({ count: this.interrupted }, INTERRUPTED_NOTICE);
|
|
228
|
+
}
|
|
202
229
|
if (model === this.model) return;
|
|
203
230
|
const was = this.model;
|
|
204
231
|
await session.setModel(model ?? undefined);
|
|
@@ -206,13 +233,20 @@ export class Runner {
|
|
|
206
233
|
log.info({ was, now: model }, 'sdk-runner: the session thinks with the new model from now on, with no restart');
|
|
207
234
|
}
|
|
208
235
|
|
|
209
|
-
private
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
236
|
+
private trackInput(m: Record<string, unknown>): void {
|
|
237
|
+
if (m.parent_tool_use_id !== null && m.parent_tool_use_id !== undefined) return;
|
|
238
|
+
if (m.type === 'result') {
|
|
239
|
+
const uuids = uuidsOf(m);
|
|
240
|
+
if (uuids !== null) this.inbox.finished(uuids);
|
|
241
|
+
return;
|
|
215
242
|
}
|
|
243
|
+
const started = startedCommand(m);
|
|
244
|
+
this.inbox.started([...(uuidsOf(m) ?? []), ...(started === null ? [] : [started])]);
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
private keep(unanswered: Unanswered[]): void {
|
|
248
|
+
this.parts.store.saveUnanswered(unanswered);
|
|
249
|
+
this.parts.activity?.pending(unanswered.filter((entry) => entry.state !== 'started').length);
|
|
216
250
|
}
|
|
217
251
|
|
|
218
252
|
private note(m: Record<string, unknown>): void {
|
|
@@ -2,7 +2,8 @@ import { existsSync, readFileSync } from 'node:fs';
|
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { isRecord } from '@metro-labs/core/is-record';
|
|
4
4
|
import { writeAtomic } from '@metro-labs/core/secure-fs';
|
|
5
|
-
import type { Unanswered } from './inbox.js';
|
|
5
|
+
import type { Unanswered, Uuid } from './inbox.js';
|
|
6
|
+
import { recoverInputs } from './recovery.js';
|
|
6
7
|
|
|
7
8
|
const SESSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
8
9
|
|
|
@@ -11,10 +12,26 @@ export const projectFolder = (cwd: string): string => cwd.replace(/[^A-Za-z0-9]/
|
|
|
11
12
|
interface Stored {
|
|
12
13
|
sessionId: string | null;
|
|
13
14
|
unanswered: Unanswered[];
|
|
15
|
+
interrupted: Unanswered[];
|
|
14
16
|
}
|
|
15
17
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
+
function inputId(raw: unknown): Uuid | undefined {
|
|
19
|
+
if (raw === undefined) return undefined;
|
|
20
|
+
if (typeof raw !== 'string' || !SESSION_ID.test(raw)) throw new Error(UNREADABLE);
|
|
21
|
+
return raw as Uuid;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function inputState(raw: unknown): Unanswered['state'] {
|
|
25
|
+
if (raw === undefined || raw === 'queued' || raw === 'started') return raw;
|
|
26
|
+
throw new Error(UNREADABLE);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function entry(raw: unknown): Unanswered | null {
|
|
30
|
+
if (!isRecord(raw) || typeof raw.text !== 'string' || typeof raw.at !== 'number') return null;
|
|
31
|
+
const uuid = inputId(raw.uuid);
|
|
32
|
+
const state = inputState(raw.state);
|
|
33
|
+
return { text: raw.text, at: raw.at, ...(uuid === undefined ? {} : { uuid }), ...(state === undefined ? {} : { state }) };
|
|
34
|
+
}
|
|
18
35
|
|
|
19
36
|
const UNREADABLE = 'The saved Agent SDK state cannot be read. Restore it before starting.';
|
|
20
37
|
|
|
@@ -22,11 +39,15 @@ function stored(raw: unknown): Stored {
|
|
|
22
39
|
if (!isRecord(raw)) throw new Error(UNREADABLE);
|
|
23
40
|
const id = raw.sessionId ?? null;
|
|
24
41
|
if (id !== null && (typeof id !== 'string' || !SESSION_ID.test(id))) throw new Error(UNREADABLE);
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
42
|
+
return { sessionId: id, unanswered: entries(raw.unanswered), interrupted: entries(raw.interrupted) };
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function entries(raw: unknown): Unanswered[] {
|
|
46
|
+
if (raw === undefined) return [];
|
|
47
|
+
if (!Array.isArray(raw)) throw new Error(UNREADABLE);
|
|
48
|
+
const list = raw.map(entry);
|
|
49
|
+
if (!list.every((e): e is Unanswered => e !== null)) throw new Error(UNREADABLE);
|
|
50
|
+
return list;
|
|
30
51
|
}
|
|
31
52
|
|
|
32
53
|
export class SessionStore {
|
|
@@ -46,6 +67,15 @@ export class SessionStore {
|
|
|
46
67
|
return this.read().unanswered;
|
|
47
68
|
}
|
|
48
69
|
|
|
70
|
+
recover(): Stored {
|
|
71
|
+
const stored = this.read();
|
|
72
|
+
const transcript = stored.sessionId === null ? null : join(this.claudeDir, 'projects', projectFolder(this.cwd), `${stored.sessionId}.jsonl`);
|
|
73
|
+
const recovered = recoverInputs(stored.unanswered, transcript);
|
|
74
|
+
const next = { ...stored, unanswered: recovered.unanswered, interrupted: [...stored.interrupted, ...recovered.interrupted] };
|
|
75
|
+
if (stored.unanswered.length > 0) this.write(next);
|
|
76
|
+
return next;
|
|
77
|
+
}
|
|
78
|
+
|
|
49
79
|
save(id: string): void {
|
|
50
80
|
const stored = this.read();
|
|
51
81
|
if (!SESSION_ID.test(id) || stored.sessionId === id) return;
|
|
@@ -60,7 +90,7 @@ export class SessionStore {
|
|
|
60
90
|
try {
|
|
61
91
|
return stored(JSON.parse(readFileSync(this.path, 'utf8')));
|
|
62
92
|
} catch (err) {
|
|
63
|
-
if (isRecord(err) && err.code === 'ENOENT') return { sessionId: null, unanswered: [] };
|
|
93
|
+
if (isRecord(err) && err.code === 'ENOENT') return { sessionId: null, unanswered: [], interrupted: [] };
|
|
64
94
|
throw new Error(UNREADABLE);
|
|
65
95
|
}
|
|
66
96
|
}
|
|
@@ -40,6 +40,7 @@ export class SessionWatch {
|
|
|
40
40
|
private readonly compacted: (() => void)[] = [];
|
|
41
41
|
private compactingNow = false;
|
|
42
42
|
private lastContext = 0;
|
|
43
|
+
private streamUsage: Record<string, unknown> = {};
|
|
43
44
|
|
|
44
45
|
constructor(private readonly readOnly: (tool: string) => boolean) {}
|
|
45
46
|
|
|
@@ -60,6 +61,7 @@ export class SessionWatch {
|
|
|
60
61
|
else if (m.type === 'result') this.settle();
|
|
61
62
|
if (m.parent_tool_use_id !== null) return;
|
|
62
63
|
if (m.type === 'assistant' && isRecord(m.message)) this.assistant(m.message);
|
|
64
|
+
else if (m.type === 'stream_event' && isRecord(m.event)) this.stream(m.event);
|
|
63
65
|
else if (m.type === 'user') for (const id of toolResultIds(m.message)) this.done(id);
|
|
64
66
|
}
|
|
65
67
|
|
|
@@ -77,9 +79,19 @@ export class SessionWatch {
|
|
|
77
79
|
else if (m.subtype === 'compact_boundary') {
|
|
78
80
|
this.setCompacting(false);
|
|
79
81
|
this.lastContext = 0;
|
|
82
|
+
this.streamUsage = {};
|
|
80
83
|
}
|
|
81
84
|
}
|
|
82
85
|
|
|
86
|
+
private stream(event: Record<string, unknown>): void {
|
|
87
|
+
if (event.type === 'message_start' && isRecord(event.message)) {
|
|
88
|
+
this.streamUsage = isRecord(event.message.usage) ? event.message.usage : {};
|
|
89
|
+
} else if (event.type === 'message_delta' && isRecord(event.usage)) {
|
|
90
|
+
this.streamUsage = { ...this.streamUsage, ...event.usage };
|
|
91
|
+
} else return;
|
|
92
|
+
this.lastContext = contextOf(this.streamUsage) || this.lastContext;
|
|
93
|
+
}
|
|
94
|
+
|
|
83
95
|
private setCompacting(on: boolean): void {
|
|
84
96
|
this.compactingNow = on;
|
|
85
97
|
if (!on) for (const resume of this.compacted.splice(0)) resume();
|