@stage-labs/metro 0.1.0-beta.261 → 0.1.0-beta.263

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
1
  {
2
- "version": "0.1.0-beta.261",
2
+ "version": "0.1.0-beta.263",
3
3
  "dependencies": {
4
4
  "@anthropic-ai/claude-agent-sdk": "0.3.287",
5
5
  "@modelcontextprotocol/sdk": "^1.29.0",
@@ -1,8 +1,9 @@
1
+ import { randomUUID } from 'node:crypto';
1
2
  import { readFileSync } from 'node:fs';
2
3
  import type { SDKMessage } from '@anthropic-ai/claude-agent-sdk';
3
4
  import { isRecord } from '@metro-labs/core/is-record';
4
5
  import { log } from '@metro-labs/core/log';
5
- import type { RunnerActivity, RunnerEventKind, RunnerPhase, RunnerTool } from '@metro-labs/core/runner-activity';
6
+ import { runnerFailureSummary, type RunnerActivity, type RunnerEvent, type RunnerEventKind, type RunnerFailure, type RunnerFailureCode, type RunnerPhase, type RunnerTool } from '@metro-labs/core/runner-activity';
6
7
  import { writeAtomic } from '@metro-labs/core/secure-fs';
7
8
  import { ActivityTasks, activityName } from './activity-tasks.js';
8
9
  import type { ApprovalAsk } from './approvals.js';
@@ -16,14 +17,34 @@ type System = Extract<SDKMessage, { type: 'system' }>;
16
17
  interface TrackedTool extends RunnerTool { parent: string | null }
17
18
  interface Approval { tool: string | null; taskId: string | null; worker: boolean }
18
19
 
19
- export function failureSummary(err: unknown): string {
20
+ function sessionFailureCode(err: unknown): RunnerFailureCode {
20
21
  const text = err instanceof Error ? err.message : String(err);
21
- if (text === STATUS_UNAVAILABLE) return STATUS_UNAVAILABLE;
22
- if (/saved Agent SDK state/i.test(text)) return 'The saved Agent SDK state cannot be read. Restore it before starting.';
23
- if (/unauthorized|authentication|invalid.*key|401|403/i.test(text)) return 'The provider refused authentication. Check the Model page.';
24
- if (/ENOENT|not found|executable|spawn/i.test(text)) return 'The SDK executable could not start. Prepare the SDK runtime again.';
25
- if (/ECONN|network|fetch failed|timed? ?out/i.test(text)) return 'The SDK connection failed. Check the server and provider connection.';
26
- return 'The Agent SDK session failed. Check Terminal and the Model page.';
22
+ for (const code of ['status_unavailable', 'session_ended', 'interrupted', 'saved_state', 'authentication', 'executable', 'connection'] as const)
23
+ if (text === runnerFailureSummary(code)) return code;
24
+ if (/saved Agent SDK state/i.test(text)) return 'saved_state';
25
+ if (/unauthorized|authentication|invalid.*key|\b40[13]\b/i.test(text)) return 'authentication';
26
+ if (/ENOENT|not found|executable|spawn/i.test(text)) return 'executable';
27
+ if (/ECONN|network|fetch failed|timed? ?out/i.test(text)) return 'connection';
28
+ return 'session_error';
29
+ }
30
+
31
+ export function failureSummary(err: unknown): string { return runnerFailureSummary(sessionFailureCode(err)); }
32
+
33
+ const PROVIDER_ERRORS: Record<NonNullable<Extract<SDKMessage, { type: 'assistant' }>['error']>, RunnerFailureCode> = {
34
+ authentication_failed: 'authentication', oauth_org_not_allowed: 'authentication', cloud_credential_error: 'authentication', account_on_hold: 'authentication', verification_required: 'authentication',
35
+ billing_error: 'billing', rate_limit: 'rate_limit', overloaded: 'provider_unavailable', server_error: 'provider_unavailable',
36
+ max_output_tokens: 'output_limit', invalid_request: 'invalid_request', model_not_found: 'invalid_request', unknown: 'provider_error',
37
+ };
38
+
39
+ function providerFailure(status: number | null | undefined): RunnerFailureCode {
40
+ const codes: Record<number, RunnerFailureCode | undefined> = { 400: 'invalid_request', 401: 'authentication', 402: 'billing', 403: 'authentication', 404: 'invalid_request', 429: 'rate_limit' };
41
+ if (status === null || status === undefined) return 'provider_error';
42
+ return codes[status] ?? (status >= 500 && status <= 599 ? 'provider_unavailable' : 'provider_error');
43
+ }
44
+
45
+ function readTokenLimit(content: unknown): boolean {
46
+ if (typeof content === 'string') return /File content \([\d,]+ tokens\) exceeds maximum allowed tokens \([\d,]+\)/.test(content.slice(0, 2_048));
47
+ return Array.isArray(content) && content.slice(0, 8).some((block: unknown) => isRecord(block) && block.type === 'text' && typeof block.text === 'string' && readTokenLimit(block.text));
27
48
  }
28
49
 
29
50
  function processIdentity(): Pick<RunnerActivity, 'procStart'> {
@@ -44,6 +65,8 @@ export class Activity {
44
65
  private readonly approvals = new Map<string, Approval>();
45
66
  private busy = false;
46
67
  private failed = false;
68
+ private providerCode: RunnerFailureCode | null = null;
69
+ private interrupted: RunnerFailure | null = null;
47
70
  private compacting = false;
48
71
  private ready = false;
49
72
  private timer: ReturnType<typeof setInterval> | null = null;
@@ -51,7 +74,7 @@ export class Activity {
51
74
  private writeFailed = false;
52
75
  private readonly state: RunnerActivity = {
53
76
  pid: process.pid, ...processIdentity(), runner: 'sdk', phase: 'starting', mainPhase: 'starting', mainStartedAt: null, sessionId: null, updatedAt: Date.now(),
54
- pending: 0, workers: 0, approvals: 0, tools: [], activeTools: [], tasks: [], events: [], lastError: null,
77
+ pending: 0, workers: 0, approvals: 0, tools: [], activeTools: [], tasks: [], events: [], lastError: null, activeFailure: null, lastFailure: null,
55
78
  };
56
79
 
57
80
  constructor(private readonly path: string) {}
@@ -68,6 +91,7 @@ export class Activity {
68
91
  return {
69
92
  ...s, tools: [...s.tools], activeTools: s.activeTools.map((tool) => ({ ...tool })),
70
93
  tasks: s.tasks.map((task) => ({ ...task })), events: s.events.map((event) => ({ ...event })),
94
+ activeFailure: s.activeFailure ? { ...s.activeFailure } : null, lastFailure: s.lastFailure ? { ...s.lastFailure } : null,
71
95
  };
72
96
  }
73
97
 
@@ -97,11 +121,11 @@ export class Activity {
97
121
 
98
122
  fail(message: string): void {
99
123
  this.failed = true;
100
- this.state.lastError = message;
101
- this.event('session_failed');
124
+ const failure = this.failure('session_failed', sessionFailureCode(message), true);
125
+ if (failure.code === 'interrupted') this.interrupted = failure;
102
126
  this.state.phase = 'error';
103
127
  this.state.mainPhase = 'error';
104
- log.error({ reason: message }, 'sdk-runner: activity error');
128
+ log.error({ reason: failure.code }, 'sdk-runner: activity error');
105
129
  this.flush();
106
130
  }
107
131
 
@@ -155,8 +179,7 @@ export class Activity {
155
179
  this.compacting = compacting;
156
180
  if (message.compact_result === 'success') this.event('compacted');
157
181
  else if (message.compact_result === 'failed') {
158
- this.state.lastError = 'Conversation compaction failed.';
159
- this.event('compact_failed');
182
+ this.failure('compact_failed', 'compact_error', false, null, null, { id: message.uuid });
160
183
  }
161
184
  }
162
185
 
@@ -171,6 +194,8 @@ export class Activity {
171
194
  private turn(): void {
172
195
  this.busy = true;
173
196
  if (this.state.mainStartedAt !== null) return;
197
+ this.clearFailure();
198
+ this.providerCode = null;
174
199
  this.state.mainStartedAt = Date.now();
175
200
  this.event('turn_started');
176
201
  }
@@ -189,8 +214,8 @@ export class Activity {
189
214
  }
190
215
 
191
216
  private taskEvent(kind: RunnerEventKind, tool: string | null, taskId: string | null): void {
192
- if (kind === 'task_failed') this.state.lastError = 'A background task failed. See Conversations and Terminal.';
193
- this.event(kind, tool, taskId);
217
+ if (kind === 'task_failed') this.failure(kind, 'task_error', false, tool, taskId);
218
+ else this.event(kind, tool, taskId);
194
219
  }
195
220
 
196
221
  private result(message: Extract<SDKMessage, { type: 'result' }>): void {
@@ -200,25 +225,30 @@ export class Activity {
200
225
  for (const [id, tool] of this.tools) if (tool.parent === null) this.tools.delete(id);
201
226
  this.failed = message.is_error || message.subtype !== 'success';
202
227
  if (!this.failed) {
228
+ this.clearFailure();
229
+ this.providerCode = null;
203
230
  this.event('turn_finished');
204
231
  return;
205
232
  }
206
- const code = message.subtype === 'success' ? 'provider_error' : message.subtype;
207
- this.state.lastError = `SDK turn failed (${code}). Check Conversations and the Model page.`;
208
- this.event('turn_failed');
233
+ const code = message.subtype === 'success' ? this.providerCode ?? providerFailure(message.api_error_status) : message.subtype;
234
+ this.providerCode = null;
235
+ this.failure('turn_failed', code, true, null, null, { id: message.uuid });
209
236
  log.error({ reason: code }, 'sdk-runner: turn failed');
210
237
  }
211
238
 
212
239
  private content(message: Extract<SDKMessage, { type: 'assistant' | 'user' }>): void {
213
240
  const content = message.message.content;
214
241
  const parent = message.parent_tool_use_id ?? null;
242
+ if (parent === null && message.type === 'assistant') {
243
+ this.turn();
244
+ if (message.error !== undefined) this.providerCode = message.error === 'unknown' ? null : PROVIDER_ERRORS[message.error];
245
+ }
215
246
  if (Array.isArray(content)) for (const block of content) if (isRecord(block)) this.block(block, parent);
216
- if (parent === null && message.type === 'assistant') this.turn();
217
247
  }
218
248
 
219
249
  private block(block: Record<string, unknown>, parent: string | null): void {
220
250
  if (block.type === 'tool_use' && typeof block.id === 'string') this.tool(block.id, block.name, parent);
221
- else if (block.type === 'tool_result' && typeof block.tool_use_id === 'string') this.finish(block.tool_use_id, block.is_error === true, parent);
251
+ else if (block.type === 'tool_result' && typeof block.tool_use_id === 'string') this.finish(block.tool_use_id, block.is_error === true, parent, block.content);
222
252
  }
223
253
 
224
254
  private tool(id: string, rawName: unknown, parent: string | null): void {
@@ -231,14 +261,13 @@ export class Activity {
231
261
  log.info({ tool: name, worker: taskId }, 'sdk-runner: tool started');
232
262
  }
233
263
 
234
- private finish(id: string, failed: boolean, parent: string | null): void {
264
+ private finish(id: string, failed: boolean, parent: string | null, content: unknown): void {
235
265
  const tool = this.tools.get(id);
236
266
  if (tool === undefined && this.tasks.isHidden(parent)) return;
237
267
  this.tools.delete(id);
238
268
  const name = tool?.name ?? 'tool';
239
269
  if (failed) {
240
- this.state.lastError = `${name} failed. See its result in Conversations.`;
241
- this.event('tool_failed', name, tool?.taskId ?? null);
270
+ this.toolFailure(id, tool, parent, content);
242
271
  log.warn({ tool: name }, 'sdk-runner: tool failed');
243
272
  return;
244
273
  }
@@ -246,8 +275,35 @@ export class Activity {
246
275
  log.info({ tool: name }, 'sdk-runner: tool finished');
247
276
  }
248
277
 
278
+ private toolFailure(id: string, tool: TrackedTool | undefined, parent: string | null, content: unknown): void {
279
+ const name = tool?.name ?? 'tool';
280
+ const code = name === 'Read' && readTokenLimit(content) ? 'read_token_limit' : 'tool_error';
281
+ this.failure('tool_failed', code, false, name, tool?.taskId ?? this.tasks.taskFor(parent), { toolUseId: id });
282
+ }
283
+
284
+ private clearFailure(): void {
285
+ this.failed = false;
286
+ this.state.activeFailure = this.interrupted;
287
+ this.state.lastError = this.interrupted === null ? null : runnerFailureSummary(this.interrupted.code);
288
+ }
289
+
290
+ private failure(kind: RunnerEventKind, code: RunnerFailureCode, active: boolean, tool: string | null = null, taskId: string | null = null, identity: Pick<RunnerEvent, 'id' | 'toolUseId'> = {}): RunnerFailure {
291
+ const failure: RunnerFailure = { id: activityName(identity.id) ?? randomUUID(), at: Date.now(), kind, code, tool, taskId, ...(activityName(identity.toolUseId) === null ? {} : { toolUseId: identity.toolUseId }) };
292
+ this.state.lastFailure = failure;
293
+ if (active) {
294
+ this.state.activeFailure = failure;
295
+ this.state.lastError = runnerFailureSummary(code, tool);
296
+ }
297
+ this.record(failure);
298
+ return failure;
299
+ }
300
+
249
301
  private event(kind: RunnerEventKind, tool: string | null = null, taskId: string | null = null): void {
250
- this.state.events.unshift({ at: Date.now(), kind, tool, taskId });
302
+ this.record({ id: randomUUID(), at: Date.now(), kind, tool, taskId });
303
+ }
304
+
305
+ private record(event: RunnerEvent): void {
306
+ this.state.events.unshift(event);
251
307
  if (this.state.events.length > MAX_EVENTS) this.state.events.length = MAX_EVENTS;
252
308
  }
253
309
 
@@ -14,7 +14,7 @@ export interface RunningAgent {
14
14
  runner: Runner;
15
15
  link: MetroLink;
16
16
  done: Promise<void>;
17
- stop(): Promise<void>;
17
+ stop(cancelActive?: boolean): Promise<void>;
18
18
  }
19
19
 
20
20
  export interface AppHooks {
@@ -42,10 +42,10 @@ async function connected(cfg: RunnerConfig, hooks: AppHooks, release: () => Prom
42
42
  lost: (reason) => { hooks.lost(reason); },
43
43
  });
44
44
  let stopped = false;
45
- const stop = async (): Promise<void> => {
45
+ const stop = async (cancelActive = true): Promise<void> => {
46
46
  if (stopped) return;
47
47
  stopped = true;
48
- runner.close();
48
+ runner.close(cancelActive);
49
49
  await link.close().catch(() => { log.debug('sdk-runner: closing the metro link failed'); });
50
50
  hooks.activity?.stop();
51
51
  await release();
@@ -57,7 +57,7 @@ async function connected(cfg: RunnerConfig, hooks: AppHooks, release: () => Prom
57
57
  runner.start(runnerOptions(cfg, tools, approvalsThrough(link, (id, ask) => { hooks.activity?.approval(id, ask); }), resume, hooks.env));
58
58
  return { runner, link, done: runner.run(hooks.observe), stop };
59
59
  } catch (err) {
60
- runner.close();
60
+ runner.close(false);
61
61
  await link.close().catch(() => { log.debug('sdk-runner: closing the metro link failed'); });
62
62
  throw err;
63
63
  }
@@ -8,6 +8,8 @@ export type Priority = 'now' | 'next' | 'later';
8
8
  export interface Unanswered {
9
9
  text: string;
10
10
  at: number;
11
+ uuid?: Uuid;
12
+ state?: 'queued' | 'started';
11
13
  }
12
14
 
13
15
  const KINDS_MAX = 2_000;
@@ -20,6 +22,7 @@ export class Inbox implements AsyncIterable<SDKUserMessage> {
20
22
  private readonly ledger = new Map<string, Unanswered>();
21
23
  private wake: (() => void) | null = null;
22
24
  private closed = false;
25
+ private restoring = false;
23
26
 
24
27
  constructor(private readonly changed: (unanswered: Unanswered[]) => void = () => undefined) {}
25
28
 
@@ -49,14 +52,29 @@ export class Inbox implements AsyncIterable<SDKUserMessage> {
49
52
  return this.kinds.get(uuid);
50
53
  }
51
54
 
52
- consumed(uuids: readonly string[]): void {
55
+ started(uuids: readonly string[]): void {
56
+ let changed = false;
57
+ for (const uuid of uuids) {
58
+ const entry = this.ledger.get(uuid);
59
+ if (entry === undefined || entry.state === 'started') continue;
60
+ this.ledger.set(uuid, { ...entry, state: 'started' });
61
+ changed = true;
62
+ }
63
+ if (changed) this.changed(this.unanswered());
64
+ }
65
+
66
+ finished(uuids = this.startedUuids()): void {
53
67
  let changed = false;
54
68
  for (const uuid of uuids) changed = this.ledger.delete(uuid) || changed;
55
69
  if (changed) this.changed(this.unanswered());
56
70
  }
57
71
 
72
+ startedUuids(): string[] {
73
+ return [...this.ledger].filter(([, entry]) => entry.state === 'started').map(([uuid]) => uuid);
74
+ }
75
+
58
76
  unanswered(now = Date.now()): Unanswered[] {
59
- return [...this.ledger.values()].filter((entry) => now - entry.at < LEDGER_AGE_MS);
77
+ return [...this.ledger.values()].filter((entry) => entry.state === 'started' || now - entry.at < LEDGER_AGE_MS);
60
78
  }
61
79
 
62
80
  close(): void {
@@ -79,16 +97,23 @@ export class Inbox implements AsyncIterable<SDKUserMessage> {
79
97
  }
80
98
 
81
99
  again(left: readonly Unanswered[]): void {
82
- for (const entry of left) this.push('chat', entry.text, undefined, randomUUID(), entry.at);
100
+ if (left.length === 0) return;
101
+ this.restoring = true;
102
+ try {
103
+ for (const entry of left) this.push('chat', entry.text, undefined, entry.uuid ?? randomUUID(), entry.at);
104
+ } finally {
105
+ this.restoring = false;
106
+ }
107
+ this.changed(this.unanswered());
83
108
  }
84
109
 
85
- private remember(uuid: string, text: string, at: number): void {
86
- this.ledger.set(uuid, { text, at });
110
+ private remember(uuid: Uuid, text: string, at: number): void {
111
+ this.ledger.set(uuid, { text, at, uuid, state: 'queued' });
87
112
  while (this.ledger.size > LEDGER_MAX) {
88
- const oldest = this.ledger.keys().next().value;
113
+ const oldest = [...this.ledger].find(([, entry]) => entry.state !== 'started')?.[0];
89
114
  if (oldest === undefined) break;
90
115
  this.ledger.delete(oldest);
91
116
  }
92
- this.changed(this.unanswered());
117
+ if (!this.restoring) this.changed(this.unanswered());
93
118
  }
94
119
  }
@@ -79,6 +79,7 @@ export class MetroLink implements Asker {
79
79
  }
80
80
 
81
81
  ask(ask: PermissionAsk, signal: AbortSignal): Promise<Behavior> {
82
+ if (signal.aborted) return Promise.resolve('deny');
82
83
  return new Promise((resolve) => {
83
84
  const settle = (behavior: Behavior): void => {
84
85
  this.waiting.delete(ask.request_id);
@@ -8,7 +8,7 @@ let agent: RunningAgent | null = null;
8
8
  let activity: Activity | null = null;
9
9
 
10
10
  async function exit(code: number): Promise<never> {
11
- await agent?.stop();
11
+ await agent?.stop(code === 0);
12
12
  process.exit(code);
13
13
  }
14
14
 
@@ -0,0 +1,46 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import { closeSync, openSync, readSync } from 'node:fs';
3
+ import { isRecord } from '@metro-labs/core/is-record';
4
+ import type { Unanswered } from './inbox.js';
5
+
6
+ export const INTERRUPTED_NOTICE = 'The SDK stopped before finishing a chat request. Check Conversations before retrying; actions may already have run.';
7
+
8
+ function scanTranscript(fd: number, wanted: ReadonlySet<string>): Set<string> {
9
+ const found = new Set<string>();
10
+ const chunk = Buffer.alloc(64 * 1024);
11
+ let overlap = '';
12
+ for (let size = readSync(fd, chunk); size > 0; size = readSync(fd, chunk)) {
13
+ const text = overlap + chunk.toString('latin1', 0, size);
14
+ for (const uuid of wanted) if (text.includes(`"${uuid}"`)) found.add(uuid);
15
+ if (found.size === wanted.size) break;
16
+ overlap = text.slice(-37);
17
+ }
18
+ return found;
19
+ }
20
+
21
+ function transcriptUuids(path: string | null, inputs: readonly Unanswered[]): Set<string> {
22
+ const wanted = new Set(inputs.flatMap((input) => input.state !== 'started' && input.uuid !== undefined ? [input.uuid] : []));
23
+ if (path === null || wanted.size === 0) return new Set();
24
+ let fd: number | undefined;
25
+ try {
26
+ fd = openSync(path, 'r');
27
+ return scanTranscript(fd, wanted);
28
+ } catch (err) {
29
+ if (isRecord(err) && err.code === 'ENOENT') return new Set();
30
+ throw new Error('The saved Agent SDK transcript cannot be checked. Restore it before starting.');
31
+ } finally {
32
+ if (fd !== undefined) closeSync(fd);
33
+ }
34
+ }
35
+
36
+ export function recoverInputs(inputs: readonly Unanswered[], transcript: string | null): { unanswered: Unanswered[]; interrupted: Unanswered[] } {
37
+ const seen = transcriptUuids(transcript, inputs);
38
+ const unanswered: Unanswered[] = [];
39
+ const interrupted: Unanswered[] = [];
40
+ for (const input of inputs) {
41
+ const entry = { ...input, uuid: input.uuid ?? randomUUID(), state: input.state ?? 'queued' };
42
+ if (entry.state === 'started' || seen.has(entry.uuid)) interrupted.push({ ...entry, state: 'started' });
43
+ else unanswered.push(entry);
44
+ }
45
+ return { unanswered, interrupted };
46
+ }
@@ -6,6 +6,7 @@ import { channelText, type ChannelEvent } from './channel-text.js';
6
6
  import type { RunnerConfig } from './config.js';
7
7
  import { Inbox, type Unanswered, type Uuid } from './inbox.js';
8
8
  import { CALL_ENDED, callStarted, callWords, FRONT_RULES } from './rules.js';
9
+ import { INTERRUPTED_NOTICE } from './recovery.js';
9
10
  import type { SessionStore } from './session-store.js';
10
11
  import { SessionWatch, startedCommand, uuidsOf } from './session-watch.js';
11
12
  import { SpeechRouter, type SpeechSink } from './speech.js';
@@ -79,6 +80,9 @@ export class Runner {
79
80
  private stall: ReturnType<typeof setTimeout> | null = null;
80
81
  private model: string | null = null;
81
82
  private switching: Promise<void> = Promise.resolve();
83
+ private ended = false;
84
+ private readonly interrupted: number;
85
+ private interruptionReported = false;
82
86
 
83
87
  constructor(private readonly parts: RunnerParts) {
84
88
  this.inbox = new Inbox((unanswered) => {
@@ -94,7 +98,8 @@ export class Runner {
94
98
  },
95
99
  });
96
100
  this.watch = new SessionWatch(parts.readOnly);
97
- const again = parts.store.unanswered();
101
+ const { unanswered: again, interrupted } = parts.store.recover();
102
+ this.interrupted = interrupted.length;
98
103
  this.inbox.again(again);
99
104
  if (again.length > 0) log.info({ count: again.length }, 'sdk-runner: chat messages the last session never read go in again');
100
105
  }
@@ -118,20 +123,33 @@ export class Runner {
118
123
 
119
124
  async run(observe?: (message: SDKMessage) => void): Promise<void> {
120
125
  if (this.session === null) throw new Error('the runner was not started');
121
- for await (const message of this.session) {
122
- const m: Record<string, unknown> = { ...message };
123
- const started = startedCommand(m);
124
- this.inbox.consumed([...(uuidsOf(m) ?? []), ...(started === null ? [] : [started])]);
125
- this.watch.observe(m);
126
- this.speech.observe(message);
127
- this.note(m);
128
- this.parts.activity?.observe(message);
129
- observe?.(message);
126
+ try {
127
+ for await (const message of this.session) {
128
+ const m: Record<string, unknown> = { ...message };
129
+ this.trackInput(m);
130
+ this.watch.observe(m);
131
+ this.speech.observe(message);
132
+ this.note(m);
133
+ this.parts.activity?.observe(message);
134
+ observe?.(message);
135
+ }
136
+ } finally {
137
+ this.ended = true;
130
138
  }
131
139
  }
132
140
 
133
141
  chat(event: ChannelEvent): Uuid {
134
- return this.inbox.push('chat', channelText(METRO_SERVER, event.content, event.meta));
142
+ try {
143
+ return this.inbox.push('chat', channelText(METRO_SERVER, event.content, event.meta));
144
+ } catch (err) {
145
+ log.error('sdk-runner: could not save incoming chat; stopping before further work');
146
+ try {
147
+ this.parts.activity?.fail(INTERRUPTED_NOTICE);
148
+ } finally {
149
+ this.close(false);
150
+ }
151
+ throw err;
152
+ }
135
153
  }
136
154
 
137
155
  callStarted(where: string): Uuid {
@@ -149,10 +167,14 @@ export class Runner {
149
167
  return this.inbox.push('note', CALL_ENDED);
150
168
  }
151
169
 
152
- close(): void {
170
+ close(cancelActive = true): void {
153
171
  this.clearStall();
154
- this.inbox.close();
155
- this.session?.close();
172
+ try {
173
+ if (cancelActive && !this.ended) this.inbox.finished();
174
+ } finally {
175
+ this.inbox.close();
176
+ this.session?.close();
177
+ }
156
178
  }
157
179
 
158
180
  private callPush(text: string): Uuid {
@@ -199,6 +221,11 @@ export class Runner {
199
221
  if (session === null) return;
200
222
  await session.applyFlagSettings(allowedOnly(model));
201
223
  this.parts.activity?.connected();
224
+ if (this.interrupted > 0 && !this.interruptionReported) {
225
+ this.interruptionReported = true;
226
+ this.parts.activity?.fail(INTERRUPTED_NOTICE);
227
+ log.warn({ count: this.interrupted }, INTERRUPTED_NOTICE);
228
+ }
202
229
  if (model === this.model) return;
203
230
  const was = this.model;
204
231
  await session.setModel(model ?? undefined);
@@ -206,13 +233,20 @@ export class Runner {
206
233
  log.info({ was, now: model }, 'sdk-runner: the session thinks with the new model from now on, with no restart');
207
234
  }
208
235
 
209
- private keep(unanswered: Unanswered[]): void {
210
- this.parts.activity?.pending(unanswered.length);
211
- try {
212
- this.parts.store.saveUnanswered(unanswered);
213
- } catch (err) {
214
- log.warn({ err: errMsg(err) }, 'sdk-runner: could not keep the unanswered chat messages');
236
+ private trackInput(m: Record<string, unknown>): void {
237
+ if (m.parent_tool_use_id !== null && m.parent_tool_use_id !== undefined) return;
238
+ if (m.type === 'result') {
239
+ const uuids = uuidsOf(m);
240
+ if (uuids !== null) this.inbox.finished(uuids);
241
+ return;
215
242
  }
243
+ const started = startedCommand(m);
244
+ this.inbox.started([...(uuidsOf(m) ?? []), ...(started === null ? [] : [started])]);
245
+ }
246
+
247
+ private keep(unanswered: Unanswered[]): void {
248
+ this.parts.store.saveUnanswered(unanswered);
249
+ this.parts.activity?.pending(unanswered.filter((entry) => entry.state !== 'started').length);
216
250
  }
217
251
 
218
252
  private note(m: Record<string, unknown>): void {
@@ -2,7 +2,8 @@ import { existsSync, readFileSync } from 'node:fs';
2
2
  import { join } from 'node:path';
3
3
  import { isRecord } from '@metro-labs/core/is-record';
4
4
  import { writeAtomic } from '@metro-labs/core/secure-fs';
5
- import type { Unanswered } from './inbox.js';
5
+ import type { Unanswered, Uuid } from './inbox.js';
6
+ import { recoverInputs } from './recovery.js';
6
7
 
7
8
  const SESSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
8
9
 
@@ -11,10 +12,26 @@ export const projectFolder = (cwd: string): string => cwd.replace(/[^A-Za-z0-9]/
11
12
  interface Stored {
12
13
  sessionId: string | null;
13
14
  unanswered: Unanswered[];
15
+ interrupted: Unanswered[];
14
16
  }
15
17
 
16
- const entry = (raw: unknown): Unanswered | null =>
17
- isRecord(raw) && typeof raw.text === 'string' && typeof raw.at === 'number' ? { text: raw.text, at: raw.at } : null;
18
+ function inputId(raw: unknown): Uuid | undefined {
19
+ if (raw === undefined) return undefined;
20
+ if (typeof raw !== 'string' || !SESSION_ID.test(raw)) throw new Error(UNREADABLE);
21
+ return raw as Uuid;
22
+ }
23
+
24
+ function inputState(raw: unknown): Unanswered['state'] {
25
+ if (raw === undefined || raw === 'queued' || raw === 'started') return raw;
26
+ throw new Error(UNREADABLE);
27
+ }
28
+
29
+ function entry(raw: unknown): Unanswered | null {
30
+ if (!isRecord(raw) || typeof raw.text !== 'string' || typeof raw.at !== 'number') return null;
31
+ const uuid = inputId(raw.uuid);
32
+ const state = inputState(raw.state);
33
+ return { text: raw.text, at: raw.at, ...(uuid === undefined ? {} : { uuid }), ...(state === undefined ? {} : { state }) };
34
+ }
18
35
 
19
36
  const UNREADABLE = 'The saved Agent SDK state cannot be read. Restore it before starting.';
20
37
 
@@ -22,11 +39,15 @@ function stored(raw: unknown): Stored {
22
39
  if (!isRecord(raw)) throw new Error(UNREADABLE);
23
40
  const id = raw.sessionId ?? null;
24
41
  if (id !== null && (typeof id !== 'string' || !SESSION_ID.test(id))) throw new Error(UNREADABLE);
25
- const list = raw.unanswered === undefined ? [] : raw.unanswered;
26
- if (!Array.isArray(list)) throw new Error(UNREADABLE);
27
- const unanswered = list.map(entry);
28
- if (!unanswered.every((e): e is Unanswered => e !== null)) throw new Error(UNREADABLE);
29
- return { sessionId: id, unanswered };
42
+ return { sessionId: id, unanswered: entries(raw.unanswered), interrupted: entries(raw.interrupted) };
43
+ }
44
+
45
+ function entries(raw: unknown): Unanswered[] {
46
+ if (raw === undefined) return [];
47
+ if (!Array.isArray(raw)) throw new Error(UNREADABLE);
48
+ const list = raw.map(entry);
49
+ if (!list.every((e): e is Unanswered => e !== null)) throw new Error(UNREADABLE);
50
+ return list;
30
51
  }
31
52
 
32
53
  export class SessionStore {
@@ -46,6 +67,15 @@ export class SessionStore {
46
67
  return this.read().unanswered;
47
68
  }
48
69
 
70
+ recover(): Stored {
71
+ const stored = this.read();
72
+ const transcript = stored.sessionId === null ? null : join(this.claudeDir, 'projects', projectFolder(this.cwd), `${stored.sessionId}.jsonl`);
73
+ const recovered = recoverInputs(stored.unanswered, transcript);
74
+ const next = { ...stored, unanswered: recovered.unanswered, interrupted: [...stored.interrupted, ...recovered.interrupted] };
75
+ if (stored.unanswered.length > 0) this.write(next);
76
+ return next;
77
+ }
78
+
49
79
  save(id: string): void {
50
80
  const stored = this.read();
51
81
  if (!SESSION_ID.test(id) || stored.sessionId === id) return;
@@ -60,7 +90,7 @@ export class SessionStore {
60
90
  try {
61
91
  return stored(JSON.parse(readFileSync(this.path, 'utf8')));
62
92
  } catch (err) {
63
- if (isRecord(err) && err.code === 'ENOENT') return { sessionId: null, unanswered: [] };
93
+ if (isRecord(err) && err.code === 'ENOENT') return { sessionId: null, unanswered: [], interrupted: [] };
64
94
  throw new Error(UNREADABLE);
65
95
  }
66
96
  }
@@ -40,6 +40,7 @@ export class SessionWatch {
40
40
  private readonly compacted: (() => void)[] = [];
41
41
  private compactingNow = false;
42
42
  private lastContext = 0;
43
+ private streamUsage: Record<string, unknown> = {};
43
44
 
44
45
  constructor(private readonly readOnly: (tool: string) => boolean) {}
45
46
 
@@ -60,6 +61,7 @@ export class SessionWatch {
60
61
  else if (m.type === 'result') this.settle();
61
62
  if (m.parent_tool_use_id !== null) return;
62
63
  if (m.type === 'assistant' && isRecord(m.message)) this.assistant(m.message);
64
+ else if (m.type === 'stream_event' && isRecord(m.event)) this.stream(m.event);
63
65
  else if (m.type === 'user') for (const id of toolResultIds(m.message)) this.done(id);
64
66
  }
65
67
 
@@ -77,9 +79,19 @@ export class SessionWatch {
77
79
  else if (m.subtype === 'compact_boundary') {
78
80
  this.setCompacting(false);
79
81
  this.lastContext = 0;
82
+ this.streamUsage = {};
80
83
  }
81
84
  }
82
85
 
86
+ private stream(event: Record<string, unknown>): void {
87
+ if (event.type === 'message_start' && isRecord(event.message)) {
88
+ this.streamUsage = isRecord(event.message.usage) ? event.message.usage : {};
89
+ } else if (event.type === 'message_delta' && isRecord(event.usage)) {
90
+ this.streamUsage = { ...this.streamUsage, ...event.usage };
91
+ } else return;
92
+ this.lastContext = contextOf(this.streamUsage) || this.lastContext;
93
+ }
94
+
83
95
  private setCompacting(on: boolean): void {
84
96
  this.compactingNow = on;
85
97
  if (!on) for (const resume of this.compacted.splice(0)) resume();