@stage-labs/metro 0.1.0-beta.276 → 0.1.0-beta.277
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/claude.js +2 -1
- package/dist/native-recovery-scan.js +246 -0
- package/dist/native-recovery-state.js +154 -0
- package/dist/native-recovery.js +164 -0
- package/package.json +1 -1
- package/runtime/marketplace/plugin/.claude-plugin/plugin.json +1 -1
- package/runtime/marketplace/plugin/bin/policy.mjs +4 -0
- package/runtime/node_modules/@metro-labs/core/package.json +2 -0
- package/runtime/node_modules/@metro-labs/core/src/runner-activity.ts +2 -0
- package/runtime/node_modules/@metro-labs/core/src/stations/channel-directory.ts +159 -0
- package/runtime/node_modules/@metro-labs/core/src/stations/channel-pages.ts +129 -0
- package/runtime/node_modules/@metro-labs/core/src/stations/types.ts +1 -0
- package/runtime/node_modules/@metro-labs/daemon/src/claude/api.ts +1 -1
- package/runtime/node_modules/@metro-labs/daemon/src/claude/runner-process.ts +6 -4
- package/runtime/node_modules/@metro-labs/daemon/src/claude/session.ts +3 -3
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/channel-tool.ts +80 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/permission-prompt.ts +12 -1
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/policy-gate.ts +6 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/tool-catalog.ts +4 -1
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/tool-dispatch.ts +7 -0
- package/runtime/node_modules/@metro-labs/discord-bot/src/actions.ts +2 -0
- package/runtime/node_modules/@metro-labs/discord-bot/src/channel-source.ts +79 -0
- package/runtime/node_modules/@metro-labs/discord-bot/src/channels.ts +186 -0
- package/runtime/node_modules/@metro-labs/discord-bot/src/station.ts +1 -0
- package/runtime/node_modules/@metro-labs/gmail/src/actions.ts +2 -1
- package/runtime/node_modules/@metro-labs/gmail/src/channels.ts +66 -0
- package/runtime/node_modules/@metro-labs/gmail/src/station.ts +1 -0
- package/runtime/node_modules/@metro-labs/outlook/src/actions.ts +2 -1
- package/runtime/node_modules/@metro-labs/outlook/src/channels.ts +60 -0
- package/runtime/node_modules/@metro-labs/outlook/src/station.ts +1 -0
- package/runtime/node_modules/@metro-labs/telegram/src/actions.ts +19 -0
- package/runtime/node_modules/@metro-labs/telegram/src/channels.ts +58 -0
- package/runtime/node_modules/@metro-labs/telegram/src/station.ts +1 -0
- package/runtime/node_modules/@metro-labs/telegram-bot/src/actions.ts +2 -0
- package/runtime/node_modules/@metro-labs/telegram-bot/src/channel-files.ts +3 -0
- package/runtime/node_modules/@metro-labs/telegram-bot/src/channels.ts +43 -0
- package/runtime/node_modules/@metro-labs/telegram-bot/src/index.ts +5 -34
- package/runtime/node_modules/@metro-labs/telegram-bot/src/observed-chats.ts +172 -0
- package/runtime/node_modules/@metro-labs/telegram-bot/src/station.ts +4 -0
- package/runtime/node_modules/@metro-labs/telegram-bot/src/types.ts +30 -3
- package/runtime/node_modules/@metro-labs/telegram-bot/src/updates.ts +26 -0
- package/runtime/node_modules/@metro-labs/threema/src/accounts.ts +13 -1
- package/runtime/node_modules/@metro-labs/threema/src/actions.ts +7 -1
- package/runtime/node_modules/@metro-labs/threema/src/channels.ts +37 -0
- package/runtime/node_modules/@metro-labs/threema/src/chats.ts +88 -0
- package/runtime/node_modules/@metro-labs/threema/src/inbound.ts +5 -0
- package/runtime/node_modules/@metro-labs/threema/src/outbound.ts +1 -0
- package/runtime/node_modules/@metro-labs/threema/src/station.ts +10 -2
- package/runtime/node_modules/@metro-labs/whatsapp/src/actions.ts +11 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/channel-chats.ts +137 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/channels.ts +48 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/client-types.ts +2 -0
- package/runtime/node_modules/@metro-labs/whatsapp/src/client.ts +8 -6
- package/runtime/node_modules/@metro-labs/whatsapp/src/station.ts +4 -0
- package/runtime/node_modules/@metro-labs/xmtp/src/actions.ts +3 -1
- package/runtime/node_modules/@metro-labs/xmtp/src/channels.ts +56 -0
- package/runtime/node_modules/@metro-labs/xmtp/src/network.ts +4 -0
- package/runtime/node_modules/@metro-labs/xmtp/src/station.ts +1 -0
- package/runtime/runtime.json +1 -1
- package/runtime/sdk-runner/node_modules/@metro-labs/core/package.json +2 -0
- package/runtime/sdk-runner/node_modules/@metro-labs/core/src/runner-activity.ts +2 -0
- package/runtime/sdk-runner/node_modules/@metro-labs/core/src/stations/channel-directory.ts +159 -0
- package/runtime/sdk-runner/node_modules/@metro-labs/core/src/stations/channel-pages.ts +129 -0
- package/runtime/sdk-runner/node_modules/@metro-labs/core/src/stations/types.ts +1 -0
- package/runtime/sdk-runner/runner.json +1 -1
- package/runtime/sdk-runner/src/activity-tasks.ts +7 -1
- package/runtime/sdk-runner/src/activity.ts +33 -3
- package/runtime/sdk-runner/src/app.ts +5 -2
- package/runtime/sdk-runner/src/inbox.ts +10 -1
- package/runtime/sdk-runner/src/input-queue.ts +6 -4
- package/runtime/sdk-runner/src/main.ts +21 -8
- package/runtime/sdk-runner/src/rules.ts +1 -0
- package/runtime/sdk-runner/src/runner.ts +18 -4
- package/runtime/sdk-runner/src/session-store.ts +19 -4
- package/runtime/sdk-runner/src/session-watch.ts +5 -1
- package/runtime/sdk-runner/src/task-ledger.ts +229 -0
- package/runtime/sdk-runner/src/task-recovery.ts +203 -0
- package/runtime/sdk-runner/src/task-state.ts +71 -0
|
@@ -7,6 +7,7 @@ import { runnerFailureSummary, type RunnerActivity, type RunnerEvent, type Runne
|
|
|
7
7
|
import { writeAtomic } from '@metro-labs/core/secure-fs';
|
|
8
8
|
import { ActivityTasks, activityName } from './activity-tasks.js';
|
|
9
9
|
import type { ApprovalAsk } from './approvals.js';
|
|
10
|
+
import type { SavedTask } from './task-state.js';
|
|
10
11
|
|
|
11
12
|
const HEARTBEAT_MS = 5_000;
|
|
12
13
|
const MAX_TRACKED = 200;
|
|
@@ -66,6 +67,7 @@ export class Activity {
|
|
|
66
67
|
private busy = false;
|
|
67
68
|
private failed = false;
|
|
68
69
|
private providerCode: RunnerFailureCode | null = null;
|
|
70
|
+
private readonly workerErrors = new Map<string, RunnerFailureCode>();
|
|
69
71
|
private interrupted: RunnerFailure | null = null;
|
|
70
72
|
private compacting = false;
|
|
71
73
|
private ready = false;
|
|
@@ -77,7 +79,9 @@ export class Activity {
|
|
|
77
79
|
pending: 0, workers: 0, approvals: 0, tools: [], activeTools: [], tasks: [], events: [], lastError: null, activeFailure: null, lastFailure: null,
|
|
78
80
|
};
|
|
79
81
|
|
|
80
|
-
constructor(private readonly path: string) {
|
|
82
|
+
constructor(private readonly path: string, cancelSignal?: RunnerActivity['cancelSignal']) {
|
|
83
|
+
if (cancelSignal !== undefined) this.state.cancelSignal = cancelSignal;
|
|
84
|
+
}
|
|
81
85
|
|
|
82
86
|
start(): void {
|
|
83
87
|
this.flush(true);
|
|
@@ -96,6 +100,25 @@ export class Activity {
|
|
|
96
100
|
};
|
|
97
101
|
}
|
|
98
102
|
|
|
103
|
+
recover(tasks: SavedTask[]): void {
|
|
104
|
+
this.tasks.restore(tasks.map((task) => ({
|
|
105
|
+
id: task.id, kind: null, agent: null, status: task.state === 'interrupted' ? 'unknown' : task.state,
|
|
106
|
+
background: true, startedAt: 0, updatedAt: task.updatedAt, endedAt: null, lastTool: null, toolUses: 0, durationMs: 0,
|
|
107
|
+
})), tasks.flatMap((task) => task.toolUseId === null ? [] : [[task.toolUseId, task.id]]));
|
|
108
|
+
for (const task of tasks) {
|
|
109
|
+
if (task.notice === null) continue;
|
|
110
|
+
const code = task.notice.reason === 'worker_restart' ? 'interrupted' : task.notice.reason;
|
|
111
|
+
this.failure('task_failed', code, false, null, task.id);
|
|
112
|
+
}
|
|
113
|
+
this.update();
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
recovered(): void {
|
|
117
|
+
this.interrupted = null;
|
|
118
|
+
if (this.state.activeFailure?.code === 'interrupted') this.clearFailure();
|
|
119
|
+
this.update();
|
|
120
|
+
}
|
|
121
|
+
|
|
99
122
|
pending(count: number, oldest?: number | null): void {
|
|
100
123
|
this.state.pending = count;
|
|
101
124
|
if (oldest !== undefined) this.state.queueOldestAt = oldest;
|
|
@@ -169,6 +192,7 @@ export class Activity {
|
|
|
169
192
|
}
|
|
170
193
|
|
|
171
194
|
private system(message: System): void {
|
|
195
|
+
if (message.subtype === 'task_notification' && message.reason === 'worker_restart') this.workerErrors.set(message.task_id, 'interrupted');
|
|
172
196
|
switch (message.subtype) {
|
|
173
197
|
case 'init': this.init(message.session_id); break;
|
|
174
198
|
case 'status': this.status(message); break;
|
|
@@ -226,8 +250,9 @@ export class Activity {
|
|
|
226
250
|
}
|
|
227
251
|
|
|
228
252
|
private taskEvent(kind: RunnerEventKind, tool: string | null, taskId: string | null): void {
|
|
229
|
-
if (kind === 'task_failed') this.failure(kind, 'task_error', false, tool, taskId);
|
|
253
|
+
if (kind === 'task_failed') this.failure(kind, this.workerErrors.get(taskId ?? '') ?? 'task_error', false, tool, taskId);
|
|
230
254
|
else this.event(kind, tool, taskId);
|
|
255
|
+
if (taskId !== null && ['task_completed', 'task_failed', 'task_stopped'].includes(kind)) this.workerErrors.delete(taskId);
|
|
231
256
|
}
|
|
232
257
|
|
|
233
258
|
private result(message: Extract<SDKMessage, { type: 'result' }>): void {
|
|
@@ -256,7 +281,12 @@ export class Activity {
|
|
|
256
281
|
}
|
|
257
282
|
|
|
258
283
|
private assistant(message: Extract<SDKMessage, { type: 'assistant' }>, parent: string | null): void {
|
|
259
|
-
if (parent !== null) {
|
|
284
|
+
if (parent !== null) {
|
|
285
|
+
this.tasks.model(parent, message.message.model);
|
|
286
|
+
const id = this.tasks.taskFor(parent);
|
|
287
|
+
if (id !== null && message.error !== undefined) this.workerErrors.set(id, PROVIDER_ERRORS[message.error]);
|
|
288
|
+
return;
|
|
289
|
+
}
|
|
260
290
|
this.turn();
|
|
261
291
|
if (message.error !== undefined) this.providerCode = message.error === 'unknown' ? null : PROVIDER_ERRORS[message.error];
|
|
262
292
|
}
|
|
@@ -42,8 +42,11 @@ async function connected(cfg: RunnerConfig, hooks: AppHooks, release: () => Prom
|
|
|
42
42
|
lost: (reason) => { hooks.lost(reason); },
|
|
43
43
|
});
|
|
44
44
|
let stopped = false;
|
|
45
|
-
const stop = async (cancelActive =
|
|
46
|
-
if (stopped)
|
|
45
|
+
const stop = async (cancelActive = false): Promise<void> => {
|
|
46
|
+
if (stopped) {
|
|
47
|
+
if (cancelActive) runner.close(true);
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
47
50
|
stopped = true;
|
|
48
51
|
runner.close(cancelActive);
|
|
49
52
|
await link.close().catch(() => { log.debug('sdk-runner: closing the metro link failed'); });
|
|
@@ -30,6 +30,7 @@ export class Inbox implements AsyncIterable<SDKUserMessage> {
|
|
|
30
30
|
private readonly queue = new InputQueue();
|
|
31
31
|
private readonly kinds = new Map<string, InputKind>();
|
|
32
32
|
private readonly ledger = new Map<string, Unanswered>();
|
|
33
|
+
private readonly dispatched = new Set<string>();
|
|
33
34
|
private readonly timing: InputTiming;
|
|
34
35
|
private wake: (() => void) | null = null;
|
|
35
36
|
private closed = false;
|
|
@@ -41,6 +42,10 @@ export class Inbox implements AsyncIterable<SDKUserMessage> {
|
|
|
41
42
|
|
|
42
43
|
get pending(): number { return this.queue.size; }
|
|
43
44
|
|
|
45
|
+
accepts(kind: InputKind, text: string): boolean { return !this.closed && this.queue.accepts(kind, text); }
|
|
46
|
+
|
|
47
|
+
activeUuids(): Set<string> { return new Set([...this.dispatched, ...this.startedUuids()]); }
|
|
48
|
+
|
|
44
49
|
compact(): Uuid {
|
|
45
50
|
const uuid = this.push('note', '/compact');
|
|
46
51
|
this.queue.prefer(uuid);
|
|
@@ -87,7 +92,10 @@ export class Inbox implements AsyncIterable<SDKUserMessage> {
|
|
|
87
92
|
|
|
88
93
|
finished(uuids = this.startedUuids()): void {
|
|
89
94
|
let changed = false;
|
|
90
|
-
for (const uuid of uuids)
|
|
95
|
+
for (const uuid of uuids) {
|
|
96
|
+
this.dispatched.delete(uuid);
|
|
97
|
+
changed = this.ledger.delete(uuid) || changed;
|
|
98
|
+
}
|
|
91
99
|
if (changed) this.changed(this.unanswered());
|
|
92
100
|
this.timing.finish(uuids);
|
|
93
101
|
}
|
|
@@ -121,6 +129,7 @@ export class Inbox implements AsyncIterable<SDKUserMessage> {
|
|
|
121
129
|
while (!this.closed) {
|
|
122
130
|
const next = (this.hooks.ready?.() ?? true) ? this.queue.take() : undefined;
|
|
123
131
|
if (next !== undefined) {
|
|
132
|
+
if (next.uuid !== undefined && this.ledger.has(next.uuid)) this.dispatched.add(next.uuid);
|
|
124
133
|
this.hooks.dispatch?.(next);
|
|
125
134
|
if (next.uuid !== undefined) this.timing.dispatch(next.uuid);
|
|
126
135
|
this.notify();
|
|
@@ -26,11 +26,13 @@ export class InputQueue {
|
|
|
26
26
|
get size(): number { return this.items.length; }
|
|
27
27
|
get oldest(): number | null { return this.items.reduce<number | null>((at, item) => at === null ? item.at : Math.min(at, item.at), null); }
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
accepts(kind: InputKind, text: string): boolean {
|
|
30
30
|
const bytes = Buffer.byteLength(text);
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
31
|
+
return bytes <= INPUT_MAX && this.bytes + bytes <= BYTES_MAX && this.items.filter((item) => item.kind === kind).length < CAPS[kind];
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
check(kind: InputKind, text: string): void {
|
|
35
|
+
if (!this.accepts(kind, text)) throw new Error('The SDK input queue is full. Wait for queued work to finish before retrying.');
|
|
34
36
|
}
|
|
35
37
|
|
|
36
38
|
push(kind: InputKind, message: SDKUserMessage, text: string, at: number): void {
|
|
@@ -5,34 +5,47 @@ import { startAgent, type RunningAgent } from './app.js';
|
|
|
5
5
|
import { runnerConfig } from './config.js';
|
|
6
6
|
|
|
7
7
|
let agent: RunningAgent | null = null;
|
|
8
|
+
let starting: Promise<RunningAgent> | null = null;
|
|
8
9
|
let activity: Activity | null = null;
|
|
10
|
+
let exiting = false;
|
|
11
|
+
let cancelRequested = false;
|
|
9
12
|
|
|
10
|
-
async function exit(code: number): Promise<
|
|
11
|
-
|
|
13
|
+
async function exit(code: number, cancelActive = false): Promise<void> {
|
|
14
|
+
if (cancelActive && !cancelRequested) {
|
|
15
|
+
cancelRequested = true;
|
|
16
|
+
if (exiting && agent !== null) await agent.stop(true);
|
|
17
|
+
}
|
|
18
|
+
if (exiting) return;
|
|
19
|
+
exiting = true;
|
|
20
|
+
const running = agent ?? await starting?.catch(() => null);
|
|
21
|
+
await running?.stop(cancelRequested);
|
|
12
22
|
process.exit(code);
|
|
13
23
|
}
|
|
14
24
|
|
|
15
|
-
const exitWith = (code: number) => (): void => {
|
|
16
|
-
exit(code).catch(() => process.exit(code));
|
|
25
|
+
const exitWith = (code: number, cancelActive = false) => (): void => {
|
|
26
|
+
exit(code, cancelActive).catch(() => process.exit(code));
|
|
17
27
|
};
|
|
18
28
|
|
|
19
29
|
for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP'] as const) process.on(signal, exitWith(0));
|
|
30
|
+
process.on('SIGUSR2', exitWith(0, true));
|
|
20
31
|
|
|
21
32
|
try {
|
|
22
33
|
const cfg = runnerConfig();
|
|
23
|
-
activity = new Activity(join(dirname(cfg.statePath), 'agent-status.json'));
|
|
24
|
-
|
|
34
|
+
activity = new Activity(join(dirname(cfg.statePath), 'agent-status.json'), 'SIGUSR2');
|
|
35
|
+
starting = startAgent(cfg, {
|
|
25
36
|
activity,
|
|
26
37
|
lost: () => {
|
|
38
|
+
if (exiting) return;
|
|
27
39
|
activity?.fail('The SDK lost its Metro connection. The session will restart.');
|
|
28
40
|
exitWith(1)();
|
|
29
41
|
},
|
|
30
42
|
});
|
|
43
|
+
agent = await starting;
|
|
31
44
|
try {
|
|
32
45
|
await agent.done;
|
|
33
|
-
activity.fail('The Agent SDK session ended unexpectedly.');
|
|
46
|
+
if (!exiting) activity.fail('The Agent SDK session ended unexpectedly.');
|
|
34
47
|
} catch (err) {
|
|
35
|
-
activity.fail(failureSummary(err));
|
|
48
|
+
if (!exiting) activity.fail(failureSummary(err));
|
|
36
49
|
}
|
|
37
50
|
} catch (err) {
|
|
38
51
|
log.error({ reason: failureSummary(err) }, 'sdk-runner: the Agent SDK session failed to start');
|
|
@@ -3,6 +3,7 @@ import type { CallRoute, CallSource } from '@metro-labs/core/call';
|
|
|
3
3
|
export const FRONT_RULES = `You are the orchestrator of this agent's persistent Agent SDK session. Typed chat, live call words (<call> blocks), and background worker reports share this conversation.
|
|
4
4
|
- Be fast. When you can answer a chat message at once, make the reaction and the answer one response of parallel tool calls (react, then send), with no typing signal. Use typing only when the answer will take a while.
|
|
5
5
|
- Stay light: talk, decide and delegate. Real work goes to background workers (Agent tool, subagent_type "worker", run_in_background: true); keep answering while they run, and relay what they report.
|
|
6
|
+
- Keep one implementation owner per task. Reuse that owner through the permitted coordination path; avoid chains of relay-only workers and duplicate implementers. Use focused checks while iterating, then the required full gate on the final changes. Reuse verified results only while their inputs are unchanged. A refused lock or an interrupted check is not a pass.
|
|
6
7
|
- At the next safe tool boundary, answer pending live-call inputs before starting unrelated delegation or processing worker reports. Give a short, direct answer when possible; do not generate a long task prompt before replying to the caller. Never interrupt active writes or approvals, invent filler, or promise work is done before it is.
|
|
7
8
|
- Plain assistant text is never spoken. Reply to a live <call> with mcp__metro__send using its exact line, text, and speech:{callId,generation,sourceId}. This sends speech only, not a chat post. Keep it short. Never paste a spoken reply or transcript into chat unless the owner explicitly requests a chat post.
|
|
8
9
|
- Ordinary typed chat and worker results are never automatically spoken. For an explicit typed owner request to speak, use only that channel message's call_id, call_generation and call_source_id with its exact line as send speech:{callId:call_id,generation:call_generation,sourceId:call_source_id}. Without all three authorized live-call values, do not speak or guess a target. Do not reuse an earlier sourceId for a new reply.
|
|
@@ -10,7 +10,8 @@ import { Inbox, type Unanswered, type Uuid } from './inbox.js';
|
|
|
10
10
|
import { FRONT_RULES } from './rules.js';
|
|
11
11
|
import { INTERRUPTED_NOTICE } from './recovery.js';
|
|
12
12
|
import type { SessionStore } from './session-store.js';
|
|
13
|
-
import { SessionWatch, startedCommand, uuidsOf } from './session-watch.js';
|
|
13
|
+
import { completedCommand, SessionWatch, startedCommand, uuidsOf } from './session-watch.js';
|
|
14
|
+
import { TaskRecovery } from './task-recovery.js';
|
|
14
15
|
import { METRO_SERVER, type MetroTools } from './tool-proxy.js';
|
|
15
16
|
|
|
16
17
|
const STARTUP_WAIT = 'CLAUDE_CODE_MCP_STARTUP_WAIT_MS';
|
|
@@ -86,6 +87,7 @@ function visibleOutput(m: Record<string, unknown>): boolean {
|
|
|
86
87
|
export class Runner {
|
|
87
88
|
readonly inbox: Inbox;
|
|
88
89
|
readonly calls: RunnerCalls;
|
|
90
|
+
private readonly recovery: TaskRecovery;
|
|
89
91
|
private readonly watch: SessionWatch;
|
|
90
92
|
private readonly tasks = new ActivityTasks(() => undefined);
|
|
91
93
|
private compactTimer: ReturnType<typeof setTimeout> | null = null;
|
|
@@ -109,8 +111,14 @@ export class Runner {
|
|
|
109
111
|
queue: (count, oldest) => { parts.activity?.pending(count, oldest); },
|
|
110
112
|
});
|
|
111
113
|
this.calls = new RunnerCalls(this.inbox, parts.activity);
|
|
112
|
-
const { unanswered: again, interrupted, context } = parts.store.recover();
|
|
114
|
+
const { unanswered: again, interrupted, context, tasks } = parts.store.recover();
|
|
115
|
+
this.recovery = new TaskRecovery(parts.store, this.inbox, tasks, interrupted, (err) => {
|
|
116
|
+
log.error({ err: errMsg(err) }, 'sdk-runner: task recovery could not be saved');
|
|
117
|
+
this.parts.activity?.fail(INTERRUPTED_NOTICE);
|
|
118
|
+
this.close(false);
|
|
119
|
+
}, Date.now, () => { parts.activity?.recovered(); });
|
|
113
120
|
this.watch.restoreContext(context ?? 0);
|
|
121
|
+
parts.activity?.recover(tasks);
|
|
114
122
|
this.interrupted = interrupted.length;
|
|
115
123
|
this.inbox.again(again);
|
|
116
124
|
if (again.length > 0) log.info({ count: again.length }, 'sdk-runner: chat messages the last session never read go in again');
|
|
@@ -125,6 +133,7 @@ export class Runner {
|
|
|
125
133
|
this.session = (this.parts.open ?? query)({ prompt: this.inbox, options });
|
|
126
134
|
this.model = options.model ?? null;
|
|
127
135
|
this.queueModel(this.model);
|
|
136
|
+
this.recovery.start();
|
|
128
137
|
return this.session;
|
|
129
138
|
}
|
|
130
139
|
|
|
@@ -137,9 +146,11 @@ export class Runner {
|
|
|
137
146
|
if (this.session === null) throw new Error('the runner was not started');
|
|
138
147
|
try {
|
|
139
148
|
for await (const message of this.session) {
|
|
149
|
+
if (this.ended) break;
|
|
140
150
|
this.cancelCompact();
|
|
141
151
|
if (message.type === 'system') this.tasks.observe(message);
|
|
142
152
|
const m: Record<string, unknown> = { ...message };
|
|
153
|
+
this.recovery.observe(m);
|
|
143
154
|
this.trackInput(m);
|
|
144
155
|
this.watch.observe(m);
|
|
145
156
|
this.note(m);
|
|
@@ -150,6 +161,7 @@ export class Runner {
|
|
|
150
161
|
}
|
|
151
162
|
} finally {
|
|
152
163
|
this.cancelCompact();
|
|
164
|
+
this.recovery.close(false);
|
|
153
165
|
this.ended = true;
|
|
154
166
|
}
|
|
155
167
|
}
|
|
@@ -170,11 +182,11 @@ export class Runner {
|
|
|
170
182
|
}
|
|
171
183
|
}
|
|
172
184
|
|
|
173
|
-
close(cancelActive =
|
|
185
|
+
close(cancelActive = false): void {
|
|
174
186
|
this.cancelCompact();
|
|
175
187
|
this.calls.close();
|
|
176
188
|
try {
|
|
177
|
-
|
|
189
|
+
this.recovery.close(cancelActive);
|
|
178
190
|
} finally {
|
|
179
191
|
this.ended = true;
|
|
180
192
|
this.inbox.close();
|
|
@@ -216,6 +228,8 @@ export class Runner {
|
|
|
216
228
|
this.inbox.boundary();
|
|
217
229
|
return;
|
|
218
230
|
}
|
|
231
|
+
const completed = completedCommand(m);
|
|
232
|
+
if (completed !== null) { this.inbox.finished([completed]); return; }
|
|
219
233
|
const started = startedCommand(m);
|
|
220
234
|
this.inbox.started([...(uuidsOf(m) ?? []), ...(started === null ? [] : [started])]);
|
|
221
235
|
if (visibleOutput(m)) this.inbox.output();
|
|
@@ -4,6 +4,7 @@ import { isRecord } from '@metro-labs/core/is-record';
|
|
|
4
4
|
import { writeAtomic } from '@metro-labs/core/secure-fs';
|
|
5
5
|
import type { Unanswered, Uuid } from './inbox.js';
|
|
6
6
|
import { recoverInputs } from './recovery.js';
|
|
7
|
+
import { recoveredTasks, savedTasks, type SavedTask } from './task-state.js';
|
|
7
8
|
|
|
8
9
|
const SESSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
9
10
|
|
|
@@ -14,6 +15,7 @@ interface Stored {
|
|
|
14
15
|
unanswered: Unanswered[];
|
|
15
16
|
interrupted: Unanswered[];
|
|
16
17
|
context?: number;
|
|
18
|
+
tasks: SavedTask[];
|
|
17
19
|
}
|
|
18
20
|
|
|
19
21
|
function inputId(raw: unknown): Uuid | undefined {
|
|
@@ -47,7 +49,7 @@ function stored(raw: unknown): Stored {
|
|
|
47
49
|
const id = raw.sessionId ?? null;
|
|
48
50
|
if (id !== null && (typeof id !== 'string' || !SESSION_ID.test(id))) throw new Error(UNREADABLE);
|
|
49
51
|
const context = contextOf(raw.context);
|
|
50
|
-
return { sessionId: id, unanswered: entries(raw.unanswered), interrupted: entries(raw.interrupted), ...(context === undefined ? {} : { context }) };
|
|
52
|
+
return { sessionId: id, unanswered: entries(raw.unanswered), interrupted: entries(raw.interrupted), tasks: savedTasks(raw.tasks), ...(context === undefined ? {} : { context }) };
|
|
51
53
|
}
|
|
52
54
|
|
|
53
55
|
function entries(raw: unknown): Unanswered[] {
|
|
@@ -79,8 +81,8 @@ export class SessionStore {
|
|
|
79
81
|
const stored = this.read();
|
|
80
82
|
const transcript = stored.sessionId === null ? null : join(this.claudeDir, 'projects', projectFolder(this.cwd), `${stored.sessionId}.jsonl`);
|
|
81
83
|
const recovered = recoverInputs(stored.unanswered, transcript);
|
|
82
|
-
const next = { ...stored, unanswered: recovered.unanswered, interrupted: [...stored.interrupted, ...recovered.interrupted] };
|
|
83
|
-
if (stored.unanswered.length > 0) this.write(next);
|
|
84
|
+
const next = { ...stored, tasks: recoveredTasks(stored.tasks), unanswered: recovered.unanswered, interrupted: [...stored.interrupted, ...recovered.interrupted] };
|
|
85
|
+
if (stored.unanswered.length > 0 || stored.tasks.some((task) => task.state === 'running')) this.write(next);
|
|
84
86
|
return next;
|
|
85
87
|
}
|
|
86
88
|
|
|
@@ -99,11 +101,24 @@ export class SessionStore {
|
|
|
99
101
|
if (stored.context !== context) this.write({ ...stored, context });
|
|
100
102
|
}
|
|
101
103
|
|
|
104
|
+
interrupted(): Unanswered[] { return this.read().interrupted; }
|
|
105
|
+
|
|
106
|
+
saveTasks(tasks: SavedTask[], handled: ReadonlySet<string> = new Set()): void {
|
|
107
|
+
const stored = this.read();
|
|
108
|
+
this.write({ ...stored, tasks: savedTasks(tasks), interrupted: stored.interrupted.filter((entry) => !handled.has(`input:${entry.uuid ?? entry.at}`)) });
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
cancelTasks(tasks: SavedTask[], active: ReadonlySet<string>): void {
|
|
112
|
+
const stored = this.read();
|
|
113
|
+
const unanswered = stored.unanswered.filter((input) => input.state !== 'started' && (input.uuid === undefined || !active.has(input.uuid)));
|
|
114
|
+
this.write({ ...stored, tasks: savedTasks(tasks), unanswered, interrupted: [] });
|
|
115
|
+
}
|
|
116
|
+
|
|
102
117
|
private read(): Stored {
|
|
103
118
|
try {
|
|
104
119
|
return stored(JSON.parse(readFileSync(this.path, 'utf8')));
|
|
105
120
|
} catch (err) {
|
|
106
|
-
if (isRecord(err) && err.code === 'ENOENT') return { sessionId: null, unanswered: [], interrupted: [] };
|
|
121
|
+
if (isRecord(err) && err.code === 'ENOENT') return { sessionId: null, unanswered: [], interrupted: [], tasks: [] };
|
|
107
122
|
throw new Error(UNREADABLE);
|
|
108
123
|
}
|
|
109
124
|
}
|
|
@@ -10,6 +10,9 @@ export const uuidsOf = (m: Record<string, unknown>): string[] | null => {
|
|
|
10
10
|
export const startedCommand = (m: Record<string, unknown>): string | null =>
|
|
11
11
|
m.type === 'command_lifecycle' && m.state === 'started' && typeof m.command_uuid === 'string' ? m.command_uuid : null;
|
|
12
12
|
|
|
13
|
+
export const completedCommand = (m: Record<string, unknown>): string | null =>
|
|
14
|
+
m.type === 'command_lifecycle' && m.state === 'completed' && typeof m.command_uuid === 'string' ? m.command_uuid : null;
|
|
15
|
+
|
|
13
16
|
const num = (value: unknown): number => (typeof value === 'number' ? value : 0);
|
|
14
17
|
|
|
15
18
|
function contextOf(usage: unknown): number {
|
|
@@ -80,7 +83,8 @@ export class SessionWatch {
|
|
|
80
83
|
|
|
81
84
|
private acknowledge(m: Record<string, unknown>): void {
|
|
82
85
|
if (this.awaitingInput === null) return;
|
|
83
|
-
|
|
86
|
+
const completed = completedCommand(m) === this.awaitingInput;
|
|
87
|
+
if (completed || startedCommand(m) === this.awaitingInput || uuidsOf(m)?.includes(this.awaitingInput)) this.awaitingInput = null;
|
|
84
88
|
}
|
|
85
89
|
|
|
86
90
|
private lifecycle(m: Record<string, unknown>): void {
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
import { isRecord } from '@metro-labs/core/is-record';
|
|
2
|
+
import { activityName } from './activity-tasks.js';
|
|
3
|
+
import type { Unanswered } from './inbox.js';
|
|
4
|
+
import { pendingNotice, TASK_LIMIT, type SavedTask, type TaskReason } from './task-state.js';
|
|
5
|
+
|
|
6
|
+
interface Tool { name: string; parent: string | null }
|
|
7
|
+
const failed = (text: unknown): TaskReason => typeof text === 'string' && /\b429\b|rate.?limit/i.test(text.slice(0, 4_096)) ? 'rate_limit' : 'task_error';
|
|
8
|
+
|
|
9
|
+
export class TaskLedger {
|
|
10
|
+
private readonly rows = new Map<string, SavedTask>();
|
|
11
|
+
private readonly tools = new Map<string, Tool>();
|
|
12
|
+
private readonly errors = new Map<string, TaskReason>();
|
|
13
|
+
|
|
14
|
+
constructor(tasks: SavedTask[], private readonly save: (tasks: SavedTask[], handled?: ReadonlySet<string>) => void) {
|
|
15
|
+
for (const task of tasks) this.rows.set(task.id, task);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
snapshot(): SavedTask[] { return [...this.rows.values()]; }
|
|
19
|
+
|
|
20
|
+
interrupted(inputs: Unanswered[], now: number): void {
|
|
21
|
+
let changed = false;
|
|
22
|
+
for (const input of inputs) {
|
|
23
|
+
const id = `input:${input.uuid ?? input.at}`;
|
|
24
|
+
if (!this.rows.has(id) && !this.room()) break;
|
|
25
|
+
const task = this.get(id, now);
|
|
26
|
+
if (task.notice !== null) continue;
|
|
27
|
+
task.state = 'interrupted';
|
|
28
|
+
task.owner = 'main';
|
|
29
|
+
task.notice = pendingNotice('interrupted', task.attempt, now);
|
|
30
|
+
changed = true;
|
|
31
|
+
}
|
|
32
|
+
if (changed) this.save(this.snapshot());
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
observe(m: Record<string, unknown>, now: number): void {
|
|
36
|
+
if (m.type === 'assistant' || m.type === 'user') this.content(m, now);
|
|
37
|
+
else if (m.type === 'result') this.result(m, now);
|
|
38
|
+
else if (m.type === 'system') this.system(m, now);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
private result(m: Record<string, unknown>, now: number): void {
|
|
42
|
+
if (m.parent_tool_use_id !== null && m.parent_tool_use_id !== undefined) return;
|
|
43
|
+
if (m.is_error === true) this.turnFailed(m, now);
|
|
44
|
+
this.errors.delete('main');
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
private system(m: Record<string, unknown>, now: number): void {
|
|
48
|
+
if (m.subtype === 'background_tasks_changed') { this.background(m, now); return; }
|
|
49
|
+
const id = activityName(m.task_id);
|
|
50
|
+
if (id === null) return;
|
|
51
|
+
if (m.ambient === true || m.skip_transcript === true) { this.hide(id); return; }
|
|
52
|
+
const event = activityName(m.uuid);
|
|
53
|
+
const known = this.rows.get(id);
|
|
54
|
+
if (event !== null && known?.events.includes(event)) return;
|
|
55
|
+
this.taskEvent(id, m, event, now);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
cancel(): SavedTask[] {
|
|
59
|
+
for (const task of this.rows.values()) {
|
|
60
|
+
if (task.state === 'running') task.state = 'stopped';
|
|
61
|
+
task.handledAttempt = task.attempt;
|
|
62
|
+
task.notice = null;
|
|
63
|
+
}
|
|
64
|
+
return this.snapshot();
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
changed(handled?: ReadonlySet<string>): void { this.save(this.snapshot(), handled); }
|
|
68
|
+
|
|
69
|
+
private taskEvent(id: string, m: Record<string, unknown>, event: string | null, now: number): void {
|
|
70
|
+
if (m.subtype === 'task_started') this.start(id, m.tool_use_id, event, now);
|
|
71
|
+
else if (m.subtype === 'task_notification') this.end(id, m, event, now);
|
|
72
|
+
else if (m.subtype === 'task_updated' && isRecord(m.patch) && this.rows.has(id)) {
|
|
73
|
+
this.end(id, { status: m.patch.status === 'killed' ? 'stopped' : m.patch.status, summary: m.patch.error }, event, now);
|
|
74
|
+
} else if (m.subtype === 'task_progress') this.progress(id, m.tool_use_id, now);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
private progress(id: string, rawTool: unknown, now: number): void {
|
|
78
|
+
const task = this.rows.get(id);
|
|
79
|
+
if (task?.state !== 'running') return;
|
|
80
|
+
const tool = activityName(rawTool) ?? task.toolUseId;
|
|
81
|
+
if (tool === task.toolUseId && now - task.updatedAt < 10_000) return;
|
|
82
|
+
task.toolUseId = tool;
|
|
83
|
+
task.updatedAt = now;
|
|
84
|
+
this.resolveOwners();
|
|
85
|
+
this.save(this.snapshot());
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
private turnFailed(m: Record<string, unknown>, now: number): void {
|
|
89
|
+
if (m.parent_tool_use_id !== null && m.parent_tool_use_id !== undefined) return;
|
|
90
|
+
const id = activityName(m.uuid);
|
|
91
|
+
if (id === null) return;
|
|
92
|
+
const task = this.get(`turn:${id}`, now);
|
|
93
|
+
task.owner = 'main';
|
|
94
|
+
task.state = 'failed';
|
|
95
|
+
const reason = m.api_error_status === 429 ? 'rate_limit' : this.errors.get('main') ?? 'provider_error';
|
|
96
|
+
if (task.handledAttempt !== task.attempt) task.notice ??= pendingNotice(reason, task.attempt, now);
|
|
97
|
+
this.save(this.snapshot());
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
private room(): boolean {
|
|
101
|
+
if (this.rows.size < TASK_LIMIT) return true;
|
|
102
|
+
const old = this.snapshot().find((task) => task.notice === null && task.state !== 'running');
|
|
103
|
+
if (old === undefined) return false;
|
|
104
|
+
this.rows.delete(old.id);
|
|
105
|
+
return true;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
private get(id: string, now: number): SavedTask {
|
|
109
|
+
const known = this.rows.get(id);
|
|
110
|
+
if (known !== undefined) return known;
|
|
111
|
+
if (!this.room()) throw new Error('The saved Agent SDK task ledger is full. Handle pending recovery notices before starting more work.');
|
|
112
|
+
const task: SavedTask = { id, toolUseId: null, owner: null, attempt: 0, state: 'interrupted', events: [], updatedAt: now, notice: null };
|
|
113
|
+
this.rows.set(id, task);
|
|
114
|
+
return task;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
private start(id: string, rawTool: unknown, event: string | null, now: number): void {
|
|
118
|
+
const task = this.get(id, now);
|
|
119
|
+
if (task.state !== 'running') task.attempt += 1;
|
|
120
|
+
task.state = 'running';
|
|
121
|
+
if (event !== null) task.events = [...task.events, event].slice(-16);
|
|
122
|
+
task.updatedAt = now;
|
|
123
|
+
task.toolUseId = activityName(rawTool) ?? task.toolUseId;
|
|
124
|
+
this.resolveOwners();
|
|
125
|
+
this.save(this.snapshot());
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
private resolveOwners(): void {
|
|
129
|
+
for (const task of this.rows.values()) {
|
|
130
|
+
if (task.owner !== null) continue;
|
|
131
|
+
const tool = task.toolUseId === null ? undefined : this.tools.get(task.toolUseId);
|
|
132
|
+
if (tool !== undefined) task.parentToolUseId = tool.parent;
|
|
133
|
+
const parent = task.parentToolUseId;
|
|
134
|
+
if (parent === null) task.owner = 'main';
|
|
135
|
+
else if (parent !== undefined) task.owner = this.snapshot().find((owner) => owner.toolUseId === parent)?.id ?? null;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
private end(id: string, m: Record<string, unknown>, event: string | null, now: number): void {
|
|
140
|
+
if (m.status !== 'completed' && m.status !== 'failed' && m.status !== 'stopped') return;
|
|
141
|
+
const task = this.get(id, now);
|
|
142
|
+
task.toolUseId = activityName(m.tool_use_id) ?? task.toolUseId;
|
|
143
|
+
task.state = m.reason === 'worker_restart' ? 'interrupted' : m.status;
|
|
144
|
+
if (event !== null) task.events = [...task.events, event].slice(-16);
|
|
145
|
+
task.updatedAt = now;
|
|
146
|
+
if (task.handledAttempt !== task.attempt) this.failedTask(task, m, now);
|
|
147
|
+
this.errors.delete(task.toolUseId ?? '');
|
|
148
|
+
this.resolveOwners();
|
|
149
|
+
this.save(this.snapshot());
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
private failedTask(task: SavedTask, m: Record<string, unknown>, now: number): void {
|
|
153
|
+
if (m.status !== 'failed' && m.reason !== 'worker_restart') return;
|
|
154
|
+
const reason = m.reason === 'worker_restart' ? 'worker_restart' : this.errors.get(task.toolUseId ?? '') ?? failed(m.summary);
|
|
155
|
+
if (task.notice?.attempt !== task.attempt) task.notice = pendingNotice(reason, task.attempt, now);
|
|
156
|
+
else if (reason === 'rate_limit' && task.notice.reason !== reason) {
|
|
157
|
+
task.notice.reason = reason;
|
|
158
|
+
task.notice.nextAt = Math.max(task.notice.nextAt, now + 30_000);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
private hide(id: string): void {
|
|
163
|
+
if (this.rows.delete(id)) this.save(this.snapshot());
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
private background(m: Record<string, unknown>, now: number): void {
|
|
167
|
+
if (!Array.isArray(m.tasks)) return;
|
|
168
|
+
for (const row of m.tasks) {
|
|
169
|
+
if (!isRecord(row)) continue;
|
|
170
|
+
const id = activityName(row.task_id);
|
|
171
|
+
if (id === null) continue;
|
|
172
|
+
if (row.ambient === true) this.hide(id);
|
|
173
|
+
else if (!this.rows.has(id)) this.start(id, undefined, activityName(m.uuid), now);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
private content(m: Record<string, unknown>, now: number): void {
|
|
178
|
+
const parent = activityName(m.parent_tool_use_id);
|
|
179
|
+
this.rememberError(m, parent);
|
|
180
|
+
if (!isRecord(m.message) || !Array.isArray(m.message.content)) return;
|
|
181
|
+
for (const block of m.message.content.filter(isRecord)) {
|
|
182
|
+
if (block.type === 'tool_use' && m.type === 'assistant') this.tool(block, parent);
|
|
183
|
+
else if (block.type === 'tool_result' && m.type === 'user') this.receipt(block, m.tool_use_result, now);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
private rememberError(m: Record<string, unknown>, parent: string | null): void {
|
|
188
|
+
if (m.type === 'assistant' && typeof m.error === 'string') this.errors.set(parent ?? 'main', m.error === 'rate_limit' ? 'rate_limit' : 'provider_error');
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
private tool(block: Record<string, unknown>, parent: string | null): void {
|
|
192
|
+
const id = activityName(block.id);
|
|
193
|
+
const name = activityName(block.name);
|
|
194
|
+
if (id === null || name === null) return;
|
|
195
|
+
this.tools.set(id, { name, parent });
|
|
196
|
+
if (this.snapshot().some((task) => task.owner === null)) {
|
|
197
|
+
this.resolveOwners();
|
|
198
|
+
this.save(this.snapshot());
|
|
199
|
+
}
|
|
200
|
+
if (this.tools.size <= 400) return;
|
|
201
|
+
const oldest = this.tools.keys().next().value;
|
|
202
|
+
if (oldest !== undefined) this.tools.delete(oldest);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
private receipt(block: Record<string, unknown>, structured: unknown, now: number): void {
|
|
206
|
+
const tool = typeof block.tool_use_id === 'string' ? this.tools.get(block.tool_use_id) : undefined;
|
|
207
|
+
if (tool?.name !== 'SendMessage' || block.is_error === true) return;
|
|
208
|
+
const raw = isRecord(structured) ? structured : this.renderedReceipt(block.content);
|
|
209
|
+
if (!isRecord(raw) || raw.success !== true) return;
|
|
210
|
+
const id = activityName(raw.resumedAgentId);
|
|
211
|
+
if (id !== null) this.resumed(id, activityName(block.tool_use_id), now);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
private resumed(id: string, event: string | null, now: number): void {
|
|
215
|
+
const task = this.rows.get(id);
|
|
216
|
+
if (event !== null && task?.events.includes(event)) return;
|
|
217
|
+
if (task?.state === 'running') {
|
|
218
|
+
if (event !== null) { task.events = [...task.events, event].slice(-16); this.save(this.snapshot()); }
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
221
|
+
this.start(id, undefined, event, now);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
private renderedReceipt(content: unknown): unknown {
|
|
225
|
+
const text = typeof content === 'string' ? content : Array.isArray(content) ? content.filter(isRecord).filter((block) => block.type === 'text').map((block) => block.text).join('') : '';
|
|
226
|
+
if (text.length > 4_096) return null;
|
|
227
|
+
try { return JSON.parse(text); } catch { return null; }
|
|
228
|
+
}
|
|
229
|
+
}
|