@stage-labs/metro 0.1.0-beta.294 → 0.1.0-beta.296
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/runtime/marketplace/plugin/.claude-plugin/plugin.json +1 -1
- package/runtime/node_modules/@metro-labs/core/src/runner-activity.ts +42 -5
- package/runtime/node_modules/@metro-labs/daemon/src/channels/relay.ts +4 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/diagnostics.ts +102 -0
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/index.ts +4 -1
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/raw-get-stream.ts +54 -16
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/session-slot.ts +72 -6
- package/runtime/node_modules/@metro-labs/daemon/src/mcp/session.ts +30 -0
- package/runtime/node_modules/@metro-labs/daemon/src/routes/http.ts +2 -1
- package/runtime/runtime.json +1 -1
- package/runtime/sdk-runner/node_modules/@metro-labs/core/src/runner-activity.ts +42 -5
- package/runtime/sdk-runner/runner.json +1 -1
- package/runtime/sdk-runner/src/activity.ts +14 -20
- package/runtime/sdk-runner/src/app.ts +2 -2
- package/runtime/sdk-runner/src/errors.ts +131 -0
- package/runtime/sdk-runner/src/main.ts +4 -2
- package/runtime/sdk-runner/src/runner.ts +25 -11
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "metro",
|
|
3
|
-
"version": "0.1.0-beta.
|
|
3
|
+
"version": "0.1.0-beta.296",
|
|
4
4
|
"description": "Metro for Claude Code, installed by the metro daemon on this machine: every connector the agent holds as its own MCP server relayed through the daemon, the orchestrator-only guard on the main thread, and the standing rules loaded at each session start. No vendor credential ever sits in a config file.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Bonustrack",
|
|
@@ -26,6 +26,9 @@ const FAILURE = {
|
|
|
26
26
|
status_unavailable: 'The SDK status file cannot be written. Check its path and permissions before starting.',
|
|
27
27
|
saved_state: 'The saved Agent SDK state cannot be read. Restore it before starting.',
|
|
28
28
|
executable: 'The SDK executable could not start. Prepare the SDK runtime again.',
|
|
29
|
+
module: 'The SDK runtime could not load a required module. Prepare the SDK runtime again.',
|
|
30
|
+
configuration: 'The SDK launch configuration is missing or invalid. Check Harness and the Model page before starting.',
|
|
31
|
+
process_exit: 'The SDK process exited unexpectedly. Check its reported exit code or signal before retrying.',
|
|
29
32
|
connection: 'The SDK connection failed. Check the server and provider connection.',
|
|
30
33
|
session_ended: 'The Agent SDK session ended unexpectedly.',
|
|
31
34
|
interrupted: 'The SDK stopped before finishing a chat request. Check Conversations before retrying; actions may already have run.',
|
|
@@ -33,8 +36,40 @@ const FAILURE = {
|
|
|
33
36
|
} as const;
|
|
34
37
|
export type RunnerFailureCode = keyof typeof FAILURE;
|
|
35
38
|
|
|
36
|
-
|
|
37
|
-
|
|
39
|
+
const ERROR_CODES = ['ENOENT', 'EACCES', 'EPERM', 'ENOEXEC', 'ENOMEM', 'ERR_MODULE_NOT_FOUND', 'MODULE_NOT_FOUND', 'ERR_DLOPEN_FAILED', 'ECONNREFUSED', 'ECONNRESET', 'ENOTFOUND', 'EAI_AGAIN', 'ETIMEDOUT', 'CERT_HAS_EXPIRED', 'UNABLE_TO_VERIFY_LEAF_SIGNATURE'] as const;
|
|
40
|
+
const SIGNALS = ['SIGTERM', 'SIGKILL', 'SIGSEGV', 'SIGABRT', 'SIGBUS', 'SIGILL', 'SIGINT', 'SIGHUP', 'SIGPIPE'] as const;
|
|
41
|
+
|
|
42
|
+
export interface RunnerDiagnostic {
|
|
43
|
+
source: 'exception' | 'stderr';
|
|
44
|
+
errorCode?: (typeof ERROR_CODES)[number];
|
|
45
|
+
httpStatus?: number;
|
|
46
|
+
exitCode?: number;
|
|
47
|
+
signal?: (typeof SIGNALS)[number];
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const boundedNumber = (value: unknown, min: number, max: number): number | undefined => typeof value === 'number' && Number.isInteger(value) && value >= min && value <= max ? value : undefined;
|
|
51
|
+
|
|
52
|
+
export function runnerDiagnostic(raw: unknown): RunnerDiagnostic | undefined {
|
|
53
|
+
if (!isRecord(raw) || (raw.source !== 'exception' && raw.source !== 'stderr')) return undefined;
|
|
54
|
+
const errorCode = ERROR_CODES.find((code) => code === raw.errorCode);
|
|
55
|
+
const signal = SIGNALS.find((value) => value === raw.signal);
|
|
56
|
+
const httpStatus = boundedNumber(raw.httpStatus, 400, 599);
|
|
57
|
+
const exitCode = boundedNumber(raw.exitCode, 0, 255);
|
|
58
|
+
return {
|
|
59
|
+
source: raw.source,
|
|
60
|
+
...(errorCode === undefined ? {} : { errorCode }),
|
|
61
|
+
...(httpStatus === undefined ? {} : { httpStatus }),
|
|
62
|
+
...(exitCode === undefined ? {} : { exitCode }),
|
|
63
|
+
...(signal === undefined ? {} : { signal }),
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function runnerFailureSummary(code: RunnerFailureCode, tool: string | null = null, diagnostic?: RunnerDiagnostic): string {
|
|
68
|
+
const summary = code === 'tool_error' && name(tool) !== null ? `${tool} failed. See its result in Conversations.` : FAILURE[code];
|
|
69
|
+
const safe = runnerDiagnostic(diagnostic);
|
|
70
|
+
if (safe === undefined) return summary;
|
|
71
|
+
const details = [safe.errorCode, safe.httpStatus === undefined ? null : `HTTP ${safe.httpStatus}`, safe.exitCode === undefined ? null : `exit ${safe.exitCode}`, safe.signal].filter((value) => value !== undefined && value !== null);
|
|
72
|
+
return details.length === 0 ? summary : `${summary} Observed: ${details.join(', ')} (${safe.source === 'stderr' ? 'startup stderr' : 'exception'}).`;
|
|
38
73
|
}
|
|
39
74
|
|
|
40
75
|
export interface RunnerTask {
|
|
@@ -70,6 +105,7 @@ export interface RunnerEvent {
|
|
|
70
105
|
toolUseId?: string;
|
|
71
106
|
taskId: string | null;
|
|
72
107
|
code?: RunnerFailureCode;
|
|
108
|
+
diagnostic?: RunnerDiagnostic;
|
|
73
109
|
}
|
|
74
110
|
|
|
75
111
|
export interface RunnerFailure extends RunnerEvent {
|
|
@@ -146,16 +182,17 @@ function tool(raw: unknown): RunnerTool | null {
|
|
|
146
182
|
return { id: String(raw.id), name: String(raw.name), taskId: name(raw.taskId), startedAt: count(raw.startedAt), ...(typeof raw.worker === 'boolean' ? { worker: raw.worker } : {}) };
|
|
147
183
|
}
|
|
148
184
|
|
|
149
|
-
const
|
|
185
|
+
export const runnerFailureCode = (value: unknown): value is RunnerFailureCode => typeof value === 'string' && Object.hasOwn(FAILURE, value);
|
|
150
186
|
const failureKind = (value: RunnerEventKind): boolean => ['turn_failed', 'tool_failed', 'task_failed', 'compact_failed', 'session_failed'].includes(value);
|
|
151
187
|
|
|
152
188
|
function event(raw: unknown): RunnerEvent | null {
|
|
153
189
|
if (!isRecord(raw) || !eventKind(raw.kind) || count(raw.at) === 0) return null;
|
|
190
|
+
const diagnostic = runnerDiagnostic(raw.diagnostic);
|
|
154
191
|
return {
|
|
155
192
|
at: count(raw.at), kind: raw.kind, tool: name(raw.tool), taskId: name(raw.taskId),
|
|
156
193
|
...(name(raw.id) === null ? {} : { id: String(raw.id) }),
|
|
157
194
|
...(name(raw.toolUseId) === null ? {} : { toolUseId: String(raw.toolUseId) }),
|
|
158
|
-
...(failureKind(raw.kind) &&
|
|
195
|
+
...(failureKind(raw.kind) && runnerFailureCode(raw.code) ? { code: raw.code, ...(diagnostic === undefined ? {} : { diagnostic }) } : {}),
|
|
159
196
|
};
|
|
160
197
|
}
|
|
161
198
|
|
|
@@ -184,7 +221,7 @@ function processIdentity(value: unknown): Pick<RunnerActivity, 'procStart'> {
|
|
|
184
221
|
function failureFields(raw: Record<string, unknown>): Pick<RunnerActivity, 'lastError' | 'activeFailure' | 'lastFailure'> {
|
|
185
222
|
const activeFailure = failure(raw.activeFailure);
|
|
186
223
|
return {
|
|
187
|
-
lastError: Object.hasOwn(raw, 'activeFailure') ? activeFailure === null ? null : runnerFailureSummary(activeFailure.code, activeFailure.tool) : safeError(raw.lastError),
|
|
224
|
+
lastError: Object.hasOwn(raw, 'activeFailure') ? activeFailure === null ? null : runnerFailureSummary(activeFailure.code, activeFailure.tool, activeFailure.diagnostic) : safeError(raw.lastError),
|
|
188
225
|
...(Object.hasOwn(raw, 'activeFailure') ? { activeFailure } : {}),
|
|
189
226
|
...(Object.hasOwn(raw, 'lastFailure') ? { lastFailure: failure(raw.lastFailure) } : {}),
|
|
190
227
|
};
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import type { IncomingMessage, ServerResponse } from 'node:http';
|
|
3
|
+
import { bufferedSince } from '@metro-labs/core/events';
|
|
4
|
+
import { STATIONS } from '@metro-labs/core/station-names';
|
|
5
|
+
import { accountEnabled, accountFromLine, knownAccounts } from '../agents/map.js';
|
|
6
|
+
import { eventInScope } from '../agents/scope.js';
|
|
7
|
+
import type { GetStreamDiagnostic } from './raw-get-stream.js';
|
|
8
|
+
|
|
9
|
+
export const MCP_STATUS_PATH = '/mcp/status';
|
|
10
|
+
|
|
11
|
+
export const diagnosticTag = (value: string): string =>
|
|
12
|
+
createHash('sha256').update(value).digest('hex').slice(0, 16);
|
|
13
|
+
|
|
14
|
+
export interface SessionDiagnostic {
|
|
15
|
+
tag: string;
|
|
16
|
+
openedAt: string;
|
|
17
|
+
transportInitialized: boolean;
|
|
18
|
+
clientInitializedAt: string | null;
|
|
19
|
+
client: { kind: 'claude-code' | 'metro-sdk-runner' | 'other'; tag: string } | null;
|
|
20
|
+
subscribed: boolean;
|
|
21
|
+
streamAttached: boolean;
|
|
22
|
+
lastStreamAttachedAt: string | null;
|
|
23
|
+
lastStreamDetachedAt: string | null;
|
|
24
|
+
stream: GetStreamDiagnostic | null;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface SlotDiagnostic {
|
|
28
|
+
started: boolean;
|
|
29
|
+
liveEvents: boolean;
|
|
30
|
+
current: SessionDiagnostic | null;
|
|
31
|
+
replacements: { at: string; reason: 'initialize' | 'adopt'; tag: string; previous: SessionDiagnostic | null }[];
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
interface DiagnosticSlot {
|
|
35
|
+
diagnostics: (scope: Set<string>) => SlotDiagnostic;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface IncomingDiagnostic extends SlotDiagnostic {
|
|
39
|
+
version: 1;
|
|
40
|
+
observedAt: string;
|
|
41
|
+
accounts: {
|
|
42
|
+
station: string;
|
|
43
|
+
tag: string;
|
|
44
|
+
receiveEnabled: boolean;
|
|
45
|
+
bufferedEvents: number;
|
|
46
|
+
latestEventAt: string | null;
|
|
47
|
+
latestEventTag: string | null;
|
|
48
|
+
}[];
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function clientDiagnostic(client: { name: string; version: string } | undefined): SessionDiagnostic['client'] {
|
|
52
|
+
if (client === undefined) return null;
|
|
53
|
+
const kind = client.name === 'claude-code' || client.name === 'metro-sdk-runner' ? client.name : 'other';
|
|
54
|
+
return { kind, tag: diagnosticTag(JSON.stringify([client.name, client.version])) };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const eventDate = (raw: unknown): string | null => {
|
|
58
|
+
if (typeof raw !== 'string') return null;
|
|
59
|
+
const time = Date.parse(raw);
|
|
60
|
+
return Number.isFinite(time) ? new Date(time).toISOString() : null;
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
const eventTag = (messageId: unknown, id: unknown): string | null => {
|
|
64
|
+
const value = typeof messageId === 'string' && messageId !== '' ? messageId : id;
|
|
65
|
+
return typeof value === 'string' && value !== '' ? diagnosticTag(value) : null;
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
export function incomingDiagnostic(slot: DiagnosticSlot, scope: Set<string>): IncomingDiagnostic {
|
|
69
|
+
const events = bufferedSince(-1).filter(({ event }) => typeof event.line === 'string' && eventInScope(scope, event.line));
|
|
70
|
+
const accounts = knownAccounts().filter((account) => scope.has(account.agentId) && STATIONS.some((station) => station === account.station));
|
|
71
|
+
return {
|
|
72
|
+
version: 1,
|
|
73
|
+
observedAt: new Date().toISOString(),
|
|
74
|
+
...slot.diagnostics(scope),
|
|
75
|
+
accounts: accounts.map((account) => {
|
|
76
|
+
const buffered = events.filter(({ event }) => {
|
|
77
|
+
const target = accountFromLine(event.line);
|
|
78
|
+
return target?.station === account.station && target.accountId === account.id;
|
|
79
|
+
});
|
|
80
|
+
const latest = buffered.at(-1)?.event;
|
|
81
|
+
return {
|
|
82
|
+
station: account.station,
|
|
83
|
+
tag: diagnosticTag(JSON.stringify([account.station, account.id])),
|
|
84
|
+
receiveEnabled: accountEnabled(account.station, account.id),
|
|
85
|
+
bufferedEvents: buffered.length,
|
|
86
|
+
latestEventAt: latest === undefined ? null : eventDate(latest.ts),
|
|
87
|
+
latestEventTag: latest === undefined ? null : eventTag(latest.messageId, latest.id),
|
|
88
|
+
};
|
|
89
|
+
}),
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export function handleIncomingDiagnostic(req: IncomingMessage, res: ServerResponse, slot: DiagnosticSlot, scope: Set<string>): boolean {
|
|
94
|
+
if ((req.url ?? '').split('?')[0] !== MCP_STATUS_PATH) return false;
|
|
95
|
+
if (req.method !== 'GET') {
|
|
96
|
+
res.writeHead(405, { allow: 'GET' }).end();
|
|
97
|
+
return true;
|
|
98
|
+
}
|
|
99
|
+
const body = JSON.stringify(incomingDiagnostic(slot, scope));
|
|
100
|
+
res.writeHead(200, { 'content-type': 'application/json', 'cache-control': 'no-store' }).end(body);
|
|
101
|
+
return true;
|
|
102
|
+
}
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
authenticate,
|
|
9
9
|
runWithIdentity,
|
|
10
10
|
type RequestIdentity,
|
|
11
|
+
allowedAgents,
|
|
11
12
|
} from './request-identity.js';
|
|
12
13
|
import {
|
|
13
14
|
headerValue,
|
|
@@ -18,6 +19,7 @@ import { errMsg } from '@metro-labs/core/log';
|
|
|
18
19
|
import { channelLog, type McpSession } from './session.js';
|
|
19
20
|
import { SessionSlot } from './session-slot.js';
|
|
20
21
|
import { connectCallBridge } from './call-bridge.js';
|
|
22
|
+
import { handleIncomingDiagnostic } from './diagnostics.js';
|
|
21
23
|
|
|
22
24
|
const isInitialize = (b: unknown): boolean =>
|
|
23
25
|
!!b &&
|
|
@@ -84,7 +86,7 @@ async function serveGet(
|
|
|
84
86
|
scope: session.scope,
|
|
85
87
|
req,
|
|
86
88
|
res,
|
|
87
|
-
previous: session.currentSink,
|
|
89
|
+
previous: () => session.currentSink,
|
|
88
90
|
log: channelLog,
|
|
89
91
|
registerSink: (sink) => {
|
|
90
92
|
session.bindSink(sink);
|
|
@@ -128,6 +130,7 @@ export async function createMetroMcp(options: { liveEvents?: boolean } = {}): Pr
|
|
|
128
130
|
res.writeHead(401).end('unauthorized');
|
|
129
131
|
return;
|
|
130
132
|
}
|
|
133
|
+
if (handleIncomingDiagnostic(req, res, slot, allowedAgents(identity))) return;
|
|
131
134
|
const parsed = await readOrReject(req, res);
|
|
132
135
|
if (!parsed.ok) return;
|
|
133
136
|
await runWithIdentity(identity, async () => {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { IncomingMessage, ServerResponse } from 'node:http';
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
2
3
|
import type {
|
|
3
4
|
StreamableHTTPServerTransport,
|
|
4
5
|
EventId,
|
|
@@ -49,12 +50,24 @@ export const headerValue = (
|
|
|
49
50
|
return typeof raw === 'string' ? raw : undefined;
|
|
50
51
|
};
|
|
51
52
|
|
|
53
|
+
export interface GetStreamDiagnostic {
|
|
54
|
+
tag: string;
|
|
55
|
+
openedAt: string;
|
|
56
|
+
attachedAt: string | null;
|
|
57
|
+
closedAt: string | null;
|
|
58
|
+
ownsMapping: boolean;
|
|
59
|
+
}
|
|
60
|
+
|
|
52
61
|
export interface RawGetSink {
|
|
53
62
|
closed: boolean;
|
|
54
63
|
attach: (transport: StreamableHTTPServerTransport) => void;
|
|
55
64
|
close: () => void;
|
|
65
|
+
diagnostics: () => GetStreamDiagnostic;
|
|
56
66
|
}
|
|
57
67
|
|
|
68
|
+
const streamEnded = (sink: RawGetSink, res: ServerResponse): boolean =>
|
|
69
|
+
sink.closed || res.destroyed || res.writableEnded;
|
|
70
|
+
|
|
58
71
|
export interface ValidateResult {
|
|
59
72
|
ok: boolean;
|
|
60
73
|
status?: number;
|
|
@@ -93,12 +106,11 @@ export interface ServeOpts {
|
|
|
93
106
|
res: ServerResponse;
|
|
94
107
|
log: (...a: unknown[]) => void;
|
|
95
108
|
registerSink: (sink: RawGetSink | undefined) => void;
|
|
109
|
+
previous?: () => RawGetSink | undefined;
|
|
96
110
|
}
|
|
97
111
|
|
|
98
|
-
|
|
99
|
-
const { transport, eventStore, scope, req, res, log, registerSink } = opts;
|
|
112
|
+
function primeStream(transport: StreamableHTTPServerTransport, res: ServerResponse): void {
|
|
100
113
|
const sessionId = web(transport)?.sessionId;
|
|
101
|
-
|
|
102
114
|
res.writeHead(200, {
|
|
103
115
|
'content-type': 'text/event-stream',
|
|
104
116
|
'cache-control': 'no-cache, no-transform',
|
|
@@ -109,15 +121,29 @@ export async function serveStandaloneGet(opts: ServeOpts): Promise<void> {
|
|
|
109
121
|
res.flushHeaders?.();
|
|
110
122
|
res.write(RETRY_DIRECTIVE);
|
|
111
123
|
res.write(COMMENT);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export async function serveStandaloneGet(opts: ServeOpts): Promise<void> {
|
|
127
|
+
const { transport, eventStore, scope, req, res, log, registerSink } = opts;
|
|
128
|
+
primeStream(transport, res);
|
|
112
129
|
|
|
113
130
|
let active = transport;
|
|
131
|
+
let entry: StreamEntry | undefined;
|
|
132
|
+
const tag = randomUUID();
|
|
133
|
+
const openedAt = new Date().toISOString();
|
|
134
|
+
let attachedAt: string | null = null;
|
|
135
|
+
let closedAt: string | null = null;
|
|
114
136
|
const timers: { keepalive?: ReturnType<typeof setInterval> } = {};
|
|
115
137
|
const closeStream = (): void => {
|
|
116
138
|
if (sink.closed) return;
|
|
117
139
|
sink.closed = true;
|
|
140
|
+
closedAt = new Date().toISOString();
|
|
118
141
|
if (timers.keepalive) clearInterval(timers.keepalive);
|
|
119
|
-
web(active)?._streamMapping
|
|
120
|
-
|
|
142
|
+
const mapping = web(active)?._streamMapping;
|
|
143
|
+
if (entry !== undefined && mapping?.get(STANDALONE_STREAM_ID) === entry) {
|
|
144
|
+
mapping.delete(STANDALONE_STREAM_ID);
|
|
145
|
+
registerSink(undefined);
|
|
146
|
+
}
|
|
121
147
|
try {
|
|
122
148
|
res.end();
|
|
123
149
|
} catch {
|
|
@@ -128,11 +154,10 @@ export async function serveStandaloneGet(opts: ServeOpts): Promise<void> {
|
|
|
128
154
|
closed: false,
|
|
129
155
|
attach: (next: StreamableHTTPServerTransport): void => {
|
|
130
156
|
active = next;
|
|
131
|
-
|
|
157
|
+
entry = {
|
|
132
158
|
controller: {
|
|
133
159
|
enqueue: (chunk: Uint8Array): void => {
|
|
134
|
-
if (sink
|
|
135
|
-
if (res.destroyed || res.writableEnded) {
|
|
160
|
+
if (streamEnded(sink, res)) {
|
|
136
161
|
closeStream();
|
|
137
162
|
return;
|
|
138
163
|
}
|
|
@@ -143,10 +168,22 @@ export async function serveStandaloneGet(opts: ServeOpts): Promise<void> {
|
|
|
143
168
|
cleanup: closeStream,
|
|
144
169
|
};
|
|
145
170
|
web(next)?._streamMapping?.set(STANDALONE_STREAM_ID, entry);
|
|
171
|
+
attachedAt = new Date().toISOString();
|
|
146
172
|
},
|
|
147
173
|
close: closeStream,
|
|
174
|
+
diagnostics: () => ({
|
|
175
|
+
tag,
|
|
176
|
+
openedAt,
|
|
177
|
+
attachedAt,
|
|
178
|
+
closedAt,
|
|
179
|
+
ownsMapping: entry !== undefined && web(active)?._streamMapping?.get(STANDALONE_STREAM_ID) === entry,
|
|
180
|
+
}),
|
|
148
181
|
};
|
|
149
182
|
|
|
183
|
+
res.on('close', closeStream);
|
|
184
|
+
res.on('error', closeStream);
|
|
185
|
+
req.on('close', closeStream);
|
|
186
|
+
|
|
150
187
|
const lastEventId = headerValue(req, 'last-event-id');
|
|
151
188
|
if (lastEventId) {
|
|
152
189
|
await eventStore.replayEventsAfter(lastEventId, {
|
|
@@ -168,12 +205,17 @@ export async function serveStandaloneGet(opts: ServeOpts): Promise<void> {
|
|
|
168
205
|
});
|
|
169
206
|
}
|
|
170
207
|
|
|
208
|
+
if (streamEnded(sink, res)) {
|
|
209
|
+
closeStream();
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
opts.previous?.()?.close();
|
|
171
213
|
sink.attach(transport);
|
|
172
214
|
registerSink(sink);
|
|
215
|
+
if (sink.closed) return;
|
|
173
216
|
|
|
174
217
|
timers.keepalive = setInterval(() => {
|
|
175
|
-
if (sink
|
|
176
|
-
if (res.destroyed || res.writableEnded) {
|
|
218
|
+
if (streamEnded(sink, res)) {
|
|
177
219
|
closeStream();
|
|
178
220
|
return;
|
|
179
221
|
}
|
|
@@ -185,14 +227,10 @@ export async function serveStandaloneGet(opts: ServeOpts): Promise<void> {
|
|
|
185
227
|
}
|
|
186
228
|
}, KEEPALIVE_MS);
|
|
187
229
|
timers.keepalive.unref?.();
|
|
188
|
-
|
|
189
|
-
res.on('close', closeStream);
|
|
190
|
-
res.on('error', closeStream);
|
|
191
|
-
req.on('close', closeStream);
|
|
192
230
|
}
|
|
193
231
|
|
|
194
232
|
export interface ChannelGetOpts extends ServeOpts {
|
|
195
|
-
previous: RawGetSink | undefined;
|
|
233
|
+
previous: () => RawGetSink | undefined;
|
|
196
234
|
}
|
|
197
235
|
|
|
198
236
|
export async function serveChannelGet(opts: ChannelGetOpts): Promise<boolean> {
|
|
@@ -201,7 +239,7 @@ export async function serveChannelGet(opts: ChannelGetOpts): Promise<boolean> {
|
|
|
201
239
|
opts.res.writeHead(check.status ?? 400).end(check.message ?? 'bad request');
|
|
202
240
|
return false;
|
|
203
241
|
}
|
|
204
|
-
|
|
242
|
+
opts.previous()?.close();
|
|
205
243
|
await serveStandaloneGet(opts);
|
|
206
244
|
return true;
|
|
207
245
|
}
|
|
@@ -1,14 +1,29 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { currentBusSeq } from '@metro-labs/core/events';
|
|
3
|
+
import { errMsg } from '@metro-labs/core/log';
|
|
3
4
|
import { newReplayLedger } from '../channels/relay.js';
|
|
4
5
|
import { McpSession, channelLog, type SessionKind } from './session.js';
|
|
5
6
|
import { allowedAgents, type RequestIdentity } from './request-identity.js';
|
|
7
|
+
import { diagnosticTag, type SessionDiagnostic, type SlotDiagnostic } from './diagnostics.js';
|
|
8
|
+
|
|
9
|
+
interface SlotReplacement {
|
|
10
|
+
at: string;
|
|
11
|
+
reason: 'initialize' | 'adopt';
|
|
12
|
+
tag: string;
|
|
13
|
+
scope: Set<string>;
|
|
14
|
+
previousScope: Set<string>;
|
|
15
|
+
previous: SessionDiagnostic | null;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const sharesScope = (left: Set<string>, right: Set<string>): boolean => [...left].some((id) => right.has(id));
|
|
6
19
|
|
|
7
20
|
export class SessionSlot {
|
|
8
21
|
private session: McpSession | undefined;
|
|
9
22
|
private readonly ledger = newReplayLedger();
|
|
10
23
|
private started = false;
|
|
11
24
|
private live: boolean;
|
|
25
|
+
private changing: Promise<void> = Promise.resolve();
|
|
26
|
+
private readonly replacements: SlotReplacement[] = [];
|
|
12
27
|
|
|
13
28
|
constructor(
|
|
14
29
|
live = true,
|
|
@@ -25,6 +40,20 @@ export class SessionSlot {
|
|
|
25
40
|
return this.live;
|
|
26
41
|
}
|
|
27
42
|
|
|
43
|
+
diagnostics(scope: Set<string>): SlotDiagnostic {
|
|
44
|
+
return {
|
|
45
|
+
started: this.started,
|
|
46
|
+
liveEvents: this.live,
|
|
47
|
+
current: this.session !== undefined && sharesScope(scope, this.session.scope) ? this.session.diagnostics() : null,
|
|
48
|
+
replacements: this.replacements.filter((replacement) => sharesScope(scope, replacement.scope)).map((replacement) => ({
|
|
49
|
+
at: replacement.at,
|
|
50
|
+
reason: replacement.reason,
|
|
51
|
+
tag: replacement.tag,
|
|
52
|
+
previous: sharesScope(scope, replacement.previousScope) ? replacement.previous : null,
|
|
53
|
+
})),
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
28
57
|
startInbound(): void {
|
|
29
58
|
if (this.started) return;
|
|
30
59
|
this.started = true;
|
|
@@ -45,10 +74,39 @@ export class SessionSlot {
|
|
|
45
74
|
if (this.started) this.session?.startChannel();
|
|
46
75
|
}
|
|
47
76
|
|
|
48
|
-
|
|
49
|
-
identity
|
|
50
|
-
|
|
51
|
-
|
|
77
|
+
open(identity: RequestIdentity, adoptId?: string): Promise<McpSession> {
|
|
78
|
+
return this.change(() => this.replace(identity, adoptId));
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
private change<T>(run: () => Promise<T>): Promise<T> {
|
|
82
|
+
const changed = this.changing.then(run);
|
|
83
|
+
this.changing = changed.then(() => undefined).catch((err: unknown) => {
|
|
84
|
+
channelLog('session: change failed', errMsg(err));
|
|
85
|
+
});
|
|
86
|
+
return changed;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
private previousSlot(): Pick<SlotReplacement, 'previous' | 'previousScope'> {
|
|
90
|
+
return {
|
|
91
|
+
previous: this.session?.diagnostics() ?? null,
|
|
92
|
+
previousScope: this.session?.scope ?? new Set<string>(),
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
private recordReplacement(session: McpSession, adoptId: string | undefined, previous: Pick<SlotReplacement, 'previous' | 'previousScope'>): void {
|
|
97
|
+
this.replacements.push({
|
|
98
|
+
at: new Date().toISOString(),
|
|
99
|
+
reason: adoptId === undefined ? 'initialize' : 'adopt',
|
|
100
|
+
tag: diagnosticTag(session.id),
|
|
101
|
+
scope: session.scope,
|
|
102
|
+
...previous,
|
|
103
|
+
});
|
|
104
|
+
if (this.replacements.length > 8) this.replacements.shift();
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
private async replace(identity: RequestIdentity, adoptId?: string): Promise<McpSession> {
|
|
108
|
+
if (adoptId !== undefined && this.session?.id === adoptId) return this.session;
|
|
109
|
+
const previous = this.previousSlot();
|
|
52
110
|
await this.session?.close();
|
|
53
111
|
if (this.ledger.startAt < 0) this.ledger.startAt = currentBusSeq();
|
|
54
112
|
const id = adoptId ?? randomUUID();
|
|
@@ -61,15 +119,23 @@ export class SessionSlot {
|
|
|
61
119
|
live: () => this.live,
|
|
62
120
|
onClosed: (s) => {
|
|
63
121
|
if (this.session === s) this.session = undefined;
|
|
122
|
+
s.channel.whenIdle().then(() => {
|
|
123
|
+
if (this.session?.streamAttached) this.session.replayMissed();
|
|
124
|
+
}).catch((err: unknown) => {
|
|
125
|
+
channelLog('session: handoff replay failed', errMsg(err));
|
|
126
|
+
});
|
|
64
127
|
},
|
|
65
128
|
});
|
|
66
129
|
this.session = session;
|
|
130
|
+
this.recordReplacement(session, adoptId, previous);
|
|
67
131
|
if (this.started && this.live) session.startChannel();
|
|
68
132
|
channelLog('session: opened', 'id', id, 'kind', this.kind, 'adopted', adoptId !== undefined);
|
|
69
133
|
return session;
|
|
70
134
|
}
|
|
71
135
|
|
|
72
|
-
|
|
73
|
-
|
|
136
|
+
close(): Promise<void> {
|
|
137
|
+
return this.change(async () => {
|
|
138
|
+
await this.session?.close();
|
|
139
|
+
});
|
|
74
140
|
}
|
|
75
141
|
}
|
|
@@ -15,6 +15,7 @@ import { registerToolHandlers, toolSchemaSignature } from './tool-dispatch.js';
|
|
|
15
15
|
import { web, type RawGetSink } from './raw-get-stream.js';
|
|
16
16
|
import { sharedCalls } from '../voice/shared.js';
|
|
17
17
|
import { CALL_STATE } from '@metro-labs/core/call';
|
|
18
|
+
import { clientDiagnostic, diagnosticTag, type SessionDiagnostic } from './diagnostics.js';
|
|
18
19
|
|
|
19
20
|
export const channelLog = (...a: unknown[]): void => {
|
|
20
21
|
console.error('[metro-mcp]', ...a);
|
|
@@ -84,6 +85,10 @@ export class McpSession {
|
|
|
84
85
|
private announcing = false;
|
|
85
86
|
private readonly onClosed: (session: McpSession) => void;
|
|
86
87
|
private readonly kind: SessionKind;
|
|
88
|
+
private readonly openedAt = new Date().toISOString();
|
|
89
|
+
private clientInitializedAt: string | null = null;
|
|
90
|
+
private lastStreamAttachedAt: string | null = null;
|
|
91
|
+
private lastStreamDetachedAt: string | null = null;
|
|
87
92
|
|
|
88
93
|
private constructor(init: SessionInit) {
|
|
89
94
|
this.id = init.id;
|
|
@@ -102,6 +107,9 @@ export class McpSession {
|
|
|
102
107
|
instructions: voice ? VOICE_MCP_INSTRUCTIONS : MCP_INSTRUCTIONS,
|
|
103
108
|
},
|
|
104
109
|
);
|
|
110
|
+
this.server.oninitialized = (): void => {
|
|
111
|
+
this.clientInitializedAt = new Date().toISOString();
|
|
112
|
+
};
|
|
105
113
|
registerToolHandlers(this.server, {
|
|
106
114
|
markCurrent: (): void => {
|
|
107
115
|
this.issuedSchema = toolSchemaSignature();
|
|
@@ -159,6 +167,21 @@ export class McpSession {
|
|
|
159
167
|
return this.sink;
|
|
160
168
|
}
|
|
161
169
|
|
|
170
|
+
diagnostics(): SessionDiagnostic {
|
|
171
|
+
return {
|
|
172
|
+
tag: diagnosticTag(this.id),
|
|
173
|
+
openedAt: this.openedAt,
|
|
174
|
+
transportInitialized: web(this.transport)?._initialized === true,
|
|
175
|
+
clientInitializedAt: this.clientInitializedAt,
|
|
176
|
+
client: clientDiagnostic(this.server.getClientVersion()),
|
|
177
|
+
subscribed: this.unsubscribe !== undefined,
|
|
178
|
+
streamAttached: this.streamAttached,
|
|
179
|
+
lastStreamAttachedAt: this.lastStreamAttachedAt,
|
|
180
|
+
lastStreamDetachedAt: this.lastStreamDetachedAt,
|
|
181
|
+
stream: this.sink?.diagnostics() ?? null,
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
|
|
162
185
|
inScope(line: string): boolean {
|
|
163
186
|
return this.streamAttached && eventInScope(this.scope, line);
|
|
164
187
|
}
|
|
@@ -177,8 +200,14 @@ export class McpSession {
|
|
|
177
200
|
}
|
|
178
201
|
|
|
179
202
|
bindSink(sink: RawGetSink | undefined): void {
|
|
203
|
+
if (this.closed) {
|
|
204
|
+
sink?.close();
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
if (sink === undefined && this.sink !== undefined) this.lastStreamDetachedAt = new Date().toISOString();
|
|
180
208
|
this.sink = sink;
|
|
181
209
|
if (sink !== undefined) {
|
|
210
|
+
this.lastStreamAttachedAt = new Date().toISOString();
|
|
182
211
|
this.announceCallState();
|
|
183
212
|
this.announceToolSchema();
|
|
184
213
|
}
|
|
@@ -223,6 +252,7 @@ export class McpSession {
|
|
|
223
252
|
|
|
224
253
|
dropStream(): void {
|
|
225
254
|
const sink = this.sink;
|
|
255
|
+
if (sink !== undefined) this.lastStreamDetachedAt = new Date().toISOString();
|
|
226
256
|
this.sink = undefined;
|
|
227
257
|
sink?.close();
|
|
228
258
|
}
|
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
type ServerResponse,
|
|
13
13
|
} from 'node:http';
|
|
14
14
|
import { Line } from '@metro-labs/core/lines';
|
|
15
|
+
import { MCP_STATUS_PATH } from '../mcp/diagnostics.js';
|
|
15
16
|
import { errMsg, log } from '@metro-labs/core/log';
|
|
16
17
|
import {
|
|
17
18
|
classifyEvent,
|
|
@@ -214,7 +215,7 @@ export async function startWebhookServer(
|
|
|
214
215
|
|
|
215
216
|
function isMcpPath(req: IncomingMessage): boolean {
|
|
216
217
|
const path = (req.url ?? '').split('?')[0];
|
|
217
|
-
return path === '/' || path === '/mcp' || path === VOICE_MCP_PATH;
|
|
218
|
+
return path === '/' || path === '/mcp' || path === VOICE_MCP_PATH || path === MCP_STATUS_PATH;
|
|
218
219
|
}
|
|
219
220
|
|
|
220
221
|
function flatHeaders(req: IncomingMessage): Record<string, string> {
|
package/runtime/runtime.json
CHANGED
|
@@ -26,6 +26,9 @@ const FAILURE = {
|
|
|
26
26
|
status_unavailable: 'The SDK status file cannot be written. Check its path and permissions before starting.',
|
|
27
27
|
saved_state: 'The saved Agent SDK state cannot be read. Restore it before starting.',
|
|
28
28
|
executable: 'The SDK executable could not start. Prepare the SDK runtime again.',
|
|
29
|
+
module: 'The SDK runtime could not load a required module. Prepare the SDK runtime again.',
|
|
30
|
+
configuration: 'The SDK launch configuration is missing or invalid. Check Harness and the Model page before starting.',
|
|
31
|
+
process_exit: 'The SDK process exited unexpectedly. Check its reported exit code or signal before retrying.',
|
|
29
32
|
connection: 'The SDK connection failed. Check the server and provider connection.',
|
|
30
33
|
session_ended: 'The Agent SDK session ended unexpectedly.',
|
|
31
34
|
interrupted: 'The SDK stopped before finishing a chat request. Check Conversations before retrying; actions may already have run.',
|
|
@@ -33,8 +36,40 @@ const FAILURE = {
|
|
|
33
36
|
} as const;
|
|
34
37
|
export type RunnerFailureCode = keyof typeof FAILURE;
|
|
35
38
|
|
|
36
|
-
|
|
37
|
-
|
|
39
|
+
const ERROR_CODES = ['ENOENT', 'EACCES', 'EPERM', 'ENOEXEC', 'ENOMEM', 'ERR_MODULE_NOT_FOUND', 'MODULE_NOT_FOUND', 'ERR_DLOPEN_FAILED', 'ECONNREFUSED', 'ECONNRESET', 'ENOTFOUND', 'EAI_AGAIN', 'ETIMEDOUT', 'CERT_HAS_EXPIRED', 'UNABLE_TO_VERIFY_LEAF_SIGNATURE'] as const;
|
|
40
|
+
const SIGNALS = ['SIGTERM', 'SIGKILL', 'SIGSEGV', 'SIGABRT', 'SIGBUS', 'SIGILL', 'SIGINT', 'SIGHUP', 'SIGPIPE'] as const;
|
|
41
|
+
|
|
42
|
+
export interface RunnerDiagnostic {
|
|
43
|
+
source: 'exception' | 'stderr';
|
|
44
|
+
errorCode?: (typeof ERROR_CODES)[number];
|
|
45
|
+
httpStatus?: number;
|
|
46
|
+
exitCode?: number;
|
|
47
|
+
signal?: (typeof SIGNALS)[number];
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const boundedNumber = (value: unknown, min: number, max: number): number | undefined => typeof value === 'number' && Number.isInteger(value) && value >= min && value <= max ? value : undefined;
|
|
51
|
+
|
|
52
|
+
export function runnerDiagnostic(raw: unknown): RunnerDiagnostic | undefined {
|
|
53
|
+
if (!isRecord(raw) || (raw.source !== 'exception' && raw.source !== 'stderr')) return undefined;
|
|
54
|
+
const errorCode = ERROR_CODES.find((code) => code === raw.errorCode);
|
|
55
|
+
const signal = SIGNALS.find((value) => value === raw.signal);
|
|
56
|
+
const httpStatus = boundedNumber(raw.httpStatus, 400, 599);
|
|
57
|
+
const exitCode = boundedNumber(raw.exitCode, 0, 255);
|
|
58
|
+
return {
|
|
59
|
+
source: raw.source,
|
|
60
|
+
...(errorCode === undefined ? {} : { errorCode }),
|
|
61
|
+
...(httpStatus === undefined ? {} : { httpStatus }),
|
|
62
|
+
...(exitCode === undefined ? {} : { exitCode }),
|
|
63
|
+
...(signal === undefined ? {} : { signal }),
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function runnerFailureSummary(code: RunnerFailureCode, tool: string | null = null, diagnostic?: RunnerDiagnostic): string {
|
|
68
|
+
const summary = code === 'tool_error' && name(tool) !== null ? `${tool} failed. See its result in Conversations.` : FAILURE[code];
|
|
69
|
+
const safe = runnerDiagnostic(diagnostic);
|
|
70
|
+
if (safe === undefined) return summary;
|
|
71
|
+
const details = [safe.errorCode, safe.httpStatus === undefined ? null : `HTTP ${safe.httpStatus}`, safe.exitCode === undefined ? null : `exit ${safe.exitCode}`, safe.signal].filter((value) => value !== undefined && value !== null);
|
|
72
|
+
return details.length === 0 ? summary : `${summary} Observed: ${details.join(', ')} (${safe.source === 'stderr' ? 'startup stderr' : 'exception'}).`;
|
|
38
73
|
}
|
|
39
74
|
|
|
40
75
|
export interface RunnerTask {
|
|
@@ -70,6 +105,7 @@ export interface RunnerEvent {
|
|
|
70
105
|
toolUseId?: string;
|
|
71
106
|
taskId: string | null;
|
|
72
107
|
code?: RunnerFailureCode;
|
|
108
|
+
diagnostic?: RunnerDiagnostic;
|
|
73
109
|
}
|
|
74
110
|
|
|
75
111
|
export interface RunnerFailure extends RunnerEvent {
|
|
@@ -146,16 +182,17 @@ function tool(raw: unknown): RunnerTool | null {
|
|
|
146
182
|
return { id: String(raw.id), name: String(raw.name), taskId: name(raw.taskId), startedAt: count(raw.startedAt), ...(typeof raw.worker === 'boolean' ? { worker: raw.worker } : {}) };
|
|
147
183
|
}
|
|
148
184
|
|
|
149
|
-
const
|
|
185
|
+
export const runnerFailureCode = (value: unknown): value is RunnerFailureCode => typeof value === 'string' && Object.hasOwn(FAILURE, value);
|
|
150
186
|
const failureKind = (value: RunnerEventKind): boolean => ['turn_failed', 'tool_failed', 'task_failed', 'compact_failed', 'session_failed'].includes(value);
|
|
151
187
|
|
|
152
188
|
function event(raw: unknown): RunnerEvent | null {
|
|
153
189
|
if (!isRecord(raw) || !eventKind(raw.kind) || count(raw.at) === 0) return null;
|
|
190
|
+
const diagnostic = runnerDiagnostic(raw.diagnostic);
|
|
154
191
|
return {
|
|
155
192
|
at: count(raw.at), kind: raw.kind, tool: name(raw.tool), taskId: name(raw.taskId),
|
|
156
193
|
...(name(raw.id) === null ? {} : { id: String(raw.id) }),
|
|
157
194
|
...(name(raw.toolUseId) === null ? {} : { toolUseId: String(raw.toolUseId) }),
|
|
158
|
-
...(failureKind(raw.kind) &&
|
|
195
|
+
...(failureKind(raw.kind) && runnerFailureCode(raw.code) ? { code: raw.code, ...(diagnostic === undefined ? {} : { diagnostic }) } : {}),
|
|
159
196
|
};
|
|
160
197
|
}
|
|
161
198
|
|
|
@@ -184,7 +221,7 @@ function processIdentity(value: unknown): Pick<RunnerActivity, 'procStart'> {
|
|
|
184
221
|
function failureFields(raw: Record<string, unknown>): Pick<RunnerActivity, 'lastError' | 'activeFailure' | 'lastFailure'> {
|
|
185
222
|
const activeFailure = failure(raw.activeFailure);
|
|
186
223
|
return {
|
|
187
|
-
lastError: Object.hasOwn(raw, 'activeFailure') ? activeFailure === null ? null : runnerFailureSummary(activeFailure.code, activeFailure.tool) : safeError(raw.lastError),
|
|
224
|
+
lastError: Object.hasOwn(raw, 'activeFailure') ? activeFailure === null ? null : runnerFailureSummary(activeFailure.code, activeFailure.tool, activeFailure.diagnostic) : safeError(raw.lastError),
|
|
188
225
|
...(Object.hasOwn(raw, 'activeFailure') ? { activeFailure } : {}),
|
|
189
226
|
...(Object.hasOwn(raw, 'lastFailure') ? { lastFailure: failure(raw.lastFailure) } : {}),
|
|
190
227
|
};
|
|
@@ -3,11 +3,13 @@ import { readFileSync } from 'node:fs';
|
|
|
3
3
|
import type { SDKMessage } from '@anthropic-ai/claude-agent-sdk';
|
|
4
4
|
import { isRecord } from '@metro-labs/core/is-record';
|
|
5
5
|
import { log } from '@metro-labs/core/log';
|
|
6
|
-
import { runnerFailureSummary, type RunnerActivity, type RunnerEvent, type RunnerEventKind, type RunnerFailure, type RunnerFailureCode, type RunnerInput, type RunnerPhase, type RunnerTool } from '@metro-labs/core/runner-activity';
|
|
6
|
+
import { runnerFailureSummary, type RunnerActivity, type RunnerDiagnostic, type RunnerEvent, type RunnerEventKind, type RunnerFailure, type RunnerFailureCode, type RunnerInput, type RunnerPhase, type RunnerTool } from '@metro-labs/core/runner-activity';
|
|
7
7
|
import { writeAtomic } from '@metro-labs/core/secure-fs';
|
|
8
8
|
import { ActivityTasks, activityName } from './activity-tasks.js';
|
|
9
9
|
import type { ApprovalAsk } from './approvals.js';
|
|
10
10
|
import type { SavedTask } from './task-state.js';
|
|
11
|
+
import { sessionFailure } from './errors.js';
|
|
12
|
+
export { failureSummary } from './errors.js';
|
|
11
13
|
|
|
12
14
|
const HEARTBEAT_MS = 5_000;
|
|
13
15
|
const MAX_TRACKED = 200;
|
|
@@ -18,19 +20,10 @@ type System = Extract<SDKMessage, { type: 'system' }>;
|
|
|
18
20
|
interface TrackedTool extends RunnerTool { parent: string | null }
|
|
19
21
|
interface Approval { tool: string | null; taskId: string | null; worker: boolean }
|
|
20
22
|
|
|
21
|
-
function
|
|
22
|
-
|
|
23
|
-
for (const code of ['status_unavailable', 'session_ended', 'interrupted', 'saved_state', 'authentication', 'executable', 'connection'] as const)
|
|
24
|
-
if (text === runnerFailureSummary(code)) return code;
|
|
25
|
-
if (/saved Agent SDK state/i.test(text)) return 'saved_state';
|
|
26
|
-
if (/unauthorized|authentication|invalid.*key|\b40[13]\b/i.test(text)) return 'authentication';
|
|
27
|
-
if (/ENOENT|not found|executable|spawn/i.test(text)) return 'executable';
|
|
28
|
-
if (/ECONN|network|fetch failed|timed? ?out/i.test(text)) return 'connection';
|
|
29
|
-
return 'session_error';
|
|
23
|
+
function copyEvent<T extends RunnerEvent>(event: T): T {
|
|
24
|
+
return { ...event, ...(event.diagnostic === undefined ? {} : { diagnostic: { ...event.diagnostic } }) };
|
|
30
25
|
}
|
|
31
26
|
|
|
32
|
-
export function failureSummary(err: unknown): string { return runnerFailureSummary(sessionFailureCode(err)); }
|
|
33
|
-
|
|
34
27
|
const PROVIDER_ERRORS: Record<NonNullable<Extract<SDKMessage, { type: 'assistant' }>['error']>, RunnerFailureCode> = {
|
|
35
28
|
authentication_failed: 'authentication', oauth_org_not_allowed: 'authentication', cloud_credential_error: 'authentication', account_on_hold: 'authentication', verification_required: 'authentication',
|
|
36
29
|
billing_error: 'billing', rate_limit: 'rate_limit', overloaded: 'provider_unavailable', server_error: 'provider_unavailable',
|
|
@@ -94,9 +87,9 @@ export class Activity {
|
|
|
94
87
|
const s = this.state;
|
|
95
88
|
return {
|
|
96
89
|
...s, tools: [...s.tools], activeTools: s.activeTools.map((tool) => ({ ...tool })),
|
|
97
|
-
tasks: s.tasks.map((task) => ({ ...task })), events: s.events.map(
|
|
90
|
+
tasks: s.tasks.map((task) => ({ ...task })), events: s.events.map(copyEvent),
|
|
98
91
|
...(s.inputs === undefined ? {} : { inputs: s.inputs.map((input) => ({ ...input })) }),
|
|
99
|
-
activeFailure: s.activeFailure ?
|
|
92
|
+
activeFailure: s.activeFailure ? copyEvent(s.activeFailure) : null, lastFailure: s.lastFailure ? copyEvent(s.lastFailure) : null,
|
|
100
93
|
};
|
|
101
94
|
}
|
|
102
95
|
|
|
@@ -154,13 +147,14 @@ export class Activity {
|
|
|
154
147
|
this.update();
|
|
155
148
|
}
|
|
156
149
|
|
|
157
|
-
fail(
|
|
150
|
+
fail(err: unknown): void {
|
|
158
151
|
this.failed = true;
|
|
159
|
-
const
|
|
152
|
+
const safe = sessionFailure(err);
|
|
153
|
+
const failure = this.failure('session_failed', safe.code, true, null, null, {}, safe.diagnostic);
|
|
160
154
|
if (failure.code === 'interrupted') this.interrupted = failure;
|
|
161
155
|
this.state.phase = 'error';
|
|
162
156
|
this.state.mainPhase = 'error';
|
|
163
|
-
log.error({ reason: failure.code }, 'sdk-runner: activity error');
|
|
157
|
+
log.error({ reason: failure.code, diagnostic: failure.diagnostic, summary: this.state.lastError }, 'sdk-runner: activity error');
|
|
164
158
|
this.flush();
|
|
165
159
|
}
|
|
166
160
|
|
|
@@ -335,12 +329,12 @@ export class Activity {
|
|
|
335
329
|
this.state.lastError = this.interrupted === null ? null : runnerFailureSummary(this.interrupted.code);
|
|
336
330
|
}
|
|
337
331
|
|
|
338
|
-
private failure(kind: RunnerEventKind, code: RunnerFailureCode, active: boolean, tool: string | null = null, taskId: string | null = null, identity: Pick<RunnerEvent, 'id' | 'toolUseId'> = {}): RunnerFailure {
|
|
339
|
-
const failure: RunnerFailure = { id: activityName(identity.id) ?? randomUUID(), at: Date.now(), kind, code, tool, taskId, ...(activityName(identity.toolUseId) === null ? {} : { toolUseId: identity.toolUseId }) };
|
|
332
|
+
private failure(kind: RunnerEventKind, code: RunnerFailureCode, active: boolean, tool: string | null = null, taskId: string | null = null, identity: Pick<RunnerEvent, 'id' | 'toolUseId'> = {}, diagnostic?: RunnerDiagnostic): RunnerFailure {
|
|
333
|
+
const failure: RunnerFailure = { id: activityName(identity.id) ?? randomUUID(), at: Date.now(), kind, code, tool, taskId, ...(activityName(identity.toolUseId) === null ? {} : { toolUseId: identity.toolUseId }), ...(diagnostic === undefined ? {} : { diagnostic }) };
|
|
340
334
|
this.state.lastFailure = failure;
|
|
341
335
|
if (active) {
|
|
342
336
|
this.state.activeFailure = failure;
|
|
343
|
-
this.state.lastError = runnerFailureSummary(code, tool);
|
|
337
|
+
this.state.lastError = runnerFailureSummary(code, tool, diagnostic);
|
|
344
338
|
}
|
|
345
339
|
this.record(failure);
|
|
346
340
|
return failure;
|
|
@@ -4,7 +4,7 @@ import { AutomationStore } from '@metro-labs/core/automation-store';
|
|
|
4
4
|
import { errMsg, log } from '@metro-labs/core/log';
|
|
5
5
|
import { skillSourceRoot } from '@metro-labs/core/skill-source';
|
|
6
6
|
import { SkillsRefresh } from './skills.js';
|
|
7
|
-
import {
|
|
7
|
+
import type { Activity } from './activity.js';
|
|
8
8
|
import { approvalsThrough } from './approvals.js';
|
|
9
9
|
import type { RunnerConfig } from './config.js';
|
|
10
10
|
import { MetroLink } from './link.js';
|
|
@@ -79,7 +79,7 @@ export async function startAgent(cfg: RunnerConfig, hooks: AppHooks): Promise<Ru
|
|
|
79
79
|
hooks.activity?.start();
|
|
80
80
|
return await connected(cfg, hooks, release);
|
|
81
81
|
} catch (err) {
|
|
82
|
-
hooks.activity?.fail(
|
|
82
|
+
hooks.activity?.fail(err);
|
|
83
83
|
hooks.activity?.stop();
|
|
84
84
|
await release().catch(() => { log.warn('sdk-runner: releasing process ownership failed'); });
|
|
85
85
|
throw err;
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { isRecord } from '@metro-labs/core/is-record';
|
|
2
|
+
import { runnerDiagnostic, runnerFailureCode, runnerFailureSummary, type RunnerDiagnostic, type RunnerFailureCode } from '@metro-labs/core/runner-activity';
|
|
3
|
+
|
|
4
|
+
const SCAN_MAX = 2_048;
|
|
5
|
+
const ERROR_FAILURES: Record<NonNullable<RunnerDiagnostic['errorCode']>, RunnerFailureCode> = {
|
|
6
|
+
ENOENT: 'executable', EACCES: 'executable', EPERM: 'executable', ENOEXEC: 'executable', ENOMEM: 'executable',
|
|
7
|
+
ERR_MODULE_NOT_FOUND: 'module', MODULE_NOT_FOUND: 'module', ERR_DLOPEN_FAILED: 'module',
|
|
8
|
+
ECONNREFUSED: 'connection', ECONNRESET: 'connection', ENOTFOUND: 'connection', EAI_AGAIN: 'connection', ETIMEDOUT: 'connection',
|
|
9
|
+
CERT_HAS_EXPIRED: 'connection', UNABLE_TO_VERIFY_LEAF_SIGNATURE: 'connection',
|
|
10
|
+
};
|
|
11
|
+
const KNOWN = ['status_unavailable', 'session_ended', 'interrupted', 'saved_state', 'authentication', 'executable', 'connection', 'module', 'configuration', 'process_exit', 'session_error'] as const;
|
|
12
|
+
|
|
13
|
+
export interface SessionFailure {
|
|
14
|
+
code: RunnerFailureCode;
|
|
15
|
+
diagnostic?: RunnerDiagnostic;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function normalizeFailure(raw: unknown): SessionFailure {
|
|
19
|
+
if (!isRecord(raw) || !runnerFailureCode(raw.code)) return { code: 'session_error' };
|
|
20
|
+
const diagnostic = runnerDiagnostic(raw.diagnostic);
|
|
21
|
+
return { code: raw.code, ...(diagnostic === undefined ? {} : { diagnostic }) };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export class SessionError extends Error {
|
|
25
|
+
readonly failure: SessionFailure;
|
|
26
|
+
|
|
27
|
+
constructor(failure: SessionFailure) {
|
|
28
|
+
const safe = normalizeFailure(failure);
|
|
29
|
+
super(runnerFailureSummary(safe.code, null, safe.diagnostic));
|
|
30
|
+
this.failure = safe;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function statusFailure(status: number | undefined): RunnerFailureCode | null {
|
|
35
|
+
if (status === undefined) return null;
|
|
36
|
+
if (status === 401 || status === 403) return 'authentication';
|
|
37
|
+
if (status === 402) return 'billing';
|
|
38
|
+
if (status === 429) return 'rate_limit';
|
|
39
|
+
return status >= 500 ? 'provider_unavailable' : 'invalid_request';
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const TEXT_FAILURES: [RunnerFailureCode, RegExp][] = [
|
|
43
|
+
['authentication', /\bunauthorized\b|\bauthentication (?:failed|error)\b|\binvalid(?:[ -]api)?[ -]key\b/i],
|
|
44
|
+
['module', /cannot find (?:module|package)|error (?:while )?loading shared libraries|module not found/i],
|
|
45
|
+
['configuration', /METRO_(?:RUNNER_MCP_URL|AGENT_KEY) is not set|invalid (?:configuration|settings)|failed to parse (?:configuration|settings)|unknown (?:option|argument)/i],
|
|
46
|
+
['executable', /executable|spawn|command not found/i],
|
|
47
|
+
['connection', /network|fetch failed|timed? ?out/i],
|
|
48
|
+
];
|
|
49
|
+
|
|
50
|
+
function errorText(value: unknown): string {
|
|
51
|
+
if (typeof value === 'string') return value.slice(0, SCAN_MAX);
|
|
52
|
+
return isRecord(value) && typeof value.message === 'string' ? value.message.slice(0, SCAN_MAX) : '';
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function evidence(raw: Record<string, unknown>, message: string, source: RunnerDiagnostic['source']): RunnerDiagnostic | undefined {
|
|
56
|
+
const errorCodes = message.match(/\b[A-Z][A-Z_]{2,40}\b/g) ?? [];
|
|
57
|
+
return runnerDiagnostic({
|
|
58
|
+
source, errorCode: raw.code ?? errorCodes.find((code) => Object.hasOwn(ERROR_FAILURES, code)),
|
|
59
|
+
httpStatus: raw.status ?? raw.statusCode ?? Number(/(?:^|\bHTTP(?:\/[12](?:\.\d)?)?\s+|\bstatus(?: code)?[:= ]+)([45]\d\d)\b/i.exec(message)?.[1]),
|
|
60
|
+
exitCode: raw.exitCode ?? Number(/^Claude Code process exited with code (\d{1,3})\b/.exec(message)?.[1]),
|
|
61
|
+
signal: raw.signal ?? /^Claude Code process terminated by signal (SIG[A-Z]{1,8})\b/.exec(message)?.[1],
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function diagnosticCode(message: string, diagnostic: RunnerDiagnostic | undefined): RunnerFailureCode | null {
|
|
66
|
+
if (/saved Agent SDK state/i.test(message)) return 'saved_state';
|
|
67
|
+
if (diagnostic?.errorCode !== undefined) return ERROR_FAILURES[diagnostic.errorCode];
|
|
68
|
+
return statusFailure(diagnostic?.httpStatus);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function processFailure(diagnostic: RunnerDiagnostic | undefined): RunnerFailureCode | null {
|
|
72
|
+
return diagnostic?.exitCode !== undefined || diagnostic?.signal !== undefined ? 'process_exit' : null;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function classify(value: unknown, source: RunnerDiagnostic['source']): SessionFailure {
|
|
76
|
+
const message = errorText(value);
|
|
77
|
+
const known = KNOWN.find((code) => message === runnerFailureSummary(code));
|
|
78
|
+
if (known !== undefined) return { code: known };
|
|
79
|
+
const diagnostic = evidence(isRecord(value) ? value : {}, message, source);
|
|
80
|
+
const code = diagnosticCode(message, diagnostic) ?? TEXT_FAILURES.find(([, pattern]) => pattern.test(message))?.[0] ?? processFailure(diagnostic) ?? 'session_error';
|
|
81
|
+
return code === 'session_error' ? { code } : { code, ...(diagnostic === undefined ? {} : { diagnostic }) };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export function sessionFailure(err: unknown): SessionFailure {
|
|
85
|
+
try {
|
|
86
|
+
if (err instanceof SessionError) return normalizeFailure(err.failure);
|
|
87
|
+
let current = err;
|
|
88
|
+
let result: SessionFailure = { code: 'session_error' };
|
|
89
|
+
for (let depth = 0; depth < 4; depth++) {
|
|
90
|
+
const next = classify(current, 'exception');
|
|
91
|
+
if (next.code !== 'session_error') result = next;
|
|
92
|
+
if (!isRecord(current) || current.cause === undefined) break;
|
|
93
|
+
current = current.cause;
|
|
94
|
+
}
|
|
95
|
+
return result;
|
|
96
|
+
} catch {
|
|
97
|
+
return { code: 'session_error' };
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export const failureSummary = (err: unknown): string => {
|
|
102
|
+
const failure = sessionFailure(err);
|
|
103
|
+
return runnerFailureSummary(failure.code, null, failure.diagnostic);
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
export class StartupDiagnostics {
|
|
107
|
+
private tail = '';
|
|
108
|
+
private observed: SessionFailure | null = null;
|
|
109
|
+
|
|
110
|
+
stderr(text: string): void {
|
|
111
|
+
const head = text.slice(0, SCAN_MAX);
|
|
112
|
+
const candidates = [head, (this.tail + head).slice(-SCAN_MAX), text.length > SCAN_MAX ? text.slice(-SCAN_MAX) : ''];
|
|
113
|
+
for (const candidate of candidates) {
|
|
114
|
+
const failure = classify(candidate, 'stderr');
|
|
115
|
+
if (failure.code !== 'session_error' && (failure.code !== 'process_exit' || this.observed === null)) this.observed = failure;
|
|
116
|
+
}
|
|
117
|
+
this.tail = (this.tail + (text.length > SCAN_MAX ? text.slice(-SCAN_MAX) : text)).slice(-SCAN_MAX);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
failure(err: unknown): SessionFailure {
|
|
121
|
+
const failure = sessionFailure(err);
|
|
122
|
+
const observed = this.observed;
|
|
123
|
+
if (observed === null || !['session_error', 'process_exit'].includes(failure.code)) return failure;
|
|
124
|
+
return { ...observed, diagnostic: runnerDiagnostic({ ...failure.diagnostic, ...observed.diagnostic }) };
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
clear(): void {
|
|
128
|
+
this.tail = '';
|
|
129
|
+
this.observed = null;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
@@ -3,6 +3,7 @@ import { log } from '@metro-labs/core/log';
|
|
|
3
3
|
import { Activity, failureSummary } from './activity.js';
|
|
4
4
|
import { startAgent, type RunningAgent } from './app.js';
|
|
5
5
|
import { runnerConfig } from './config.js';
|
|
6
|
+
import { sessionFailure } from './errors.js';
|
|
6
7
|
|
|
7
8
|
let agent: RunningAgent | null = null;
|
|
8
9
|
let starting: Promise<RunningAgent> | null = null;
|
|
@@ -45,9 +46,10 @@ try {
|
|
|
45
46
|
await agent.done;
|
|
46
47
|
if (!exiting) activity.fail('The Agent SDK session ended unexpectedly.');
|
|
47
48
|
} catch (err) {
|
|
48
|
-
if (!exiting) activity.fail(
|
|
49
|
+
if (!exiting) activity.fail(err);
|
|
49
50
|
}
|
|
50
51
|
} catch (err) {
|
|
51
|
-
|
|
52
|
+
const failure = sessionFailure(err);
|
|
53
|
+
log.error({ reason: failure.code, diagnostic: failure.diagnostic, summary: failureSummary(err) }, 'sdk-runner: the Agent SDK session failed to start');
|
|
52
54
|
}
|
|
53
55
|
await exit(1);
|
|
@@ -18,6 +18,7 @@ import type { SessionStore } from './session-store.js';
|
|
|
18
18
|
import { completedCommand, SessionWatch, startedCommand, uuidsOf } from './session-watch.js';
|
|
19
19
|
import { TaskRecovery } from './task-recovery.js';
|
|
20
20
|
import { METRO_SERVER, type MetroTools } from './tool-proxy.js';
|
|
21
|
+
import { failureSummary, SessionError, StartupDiagnostics } from './errors.js';
|
|
21
22
|
|
|
22
23
|
const STARTUP_WAIT = 'CLAUDE_CODE_MCP_STARTUP_WAIT_MS';
|
|
23
24
|
export const COMPACT_AT = 120_000;
|
|
@@ -53,9 +54,6 @@ export function runnerOptions(cfg: RunnerConfig, tools: MetroTools, canUseTool:
|
|
|
53
54
|
includePartialMessages: true,
|
|
54
55
|
extraArgs: { 'system-prompt-snapshot': 'off' },
|
|
55
56
|
...(resume === null ? {} : { resume }),
|
|
56
|
-
stderr: (text: string) => {
|
|
57
|
-
log.debug({ text: text.slice(0, 500) }, 'sdk-runner: claude stderr');
|
|
58
|
-
},
|
|
59
57
|
};
|
|
60
58
|
}
|
|
61
59
|
|
|
@@ -98,6 +96,7 @@ export class Runner {
|
|
|
98
96
|
readonly automation: Automation | null;
|
|
99
97
|
private readonly recovery: TaskRecovery;
|
|
100
98
|
private readonly watch: SessionWatch;
|
|
99
|
+
private readonly diagnostics = new StartupDiagnostics();
|
|
101
100
|
private readonly tasks = new ActivityTasks(() => undefined);
|
|
102
101
|
private readonly skillWork = new SkillWork();
|
|
103
102
|
private compactTimer: ReturnType<typeof setTimeout> | null = null;
|
|
@@ -149,7 +148,13 @@ export class Runner {
|
|
|
149
148
|
if (this.session !== null) throw new Error('the runner was already started');
|
|
150
149
|
if (options.resume === undefined) this.watch.restoreContext(0);
|
|
151
150
|
else this.maybeCompact(true);
|
|
152
|
-
|
|
151
|
+
try {
|
|
152
|
+
this.session = (this.parts.open ?? query)({ prompt: this.inbox, options: { ...options, stderr: (text) => { if (this.sessionId === null) this.diagnostics.stderr(text); } } });
|
|
153
|
+
} catch (err) {
|
|
154
|
+
const failure = this.diagnostics.failure(err);
|
|
155
|
+
this.diagnostics.clear();
|
|
156
|
+
throw new SessionError(failure);
|
|
157
|
+
}
|
|
153
158
|
this.model = options.model ?? null;
|
|
154
159
|
this.queueModel(this.model);
|
|
155
160
|
this.parts.skills?.start({
|
|
@@ -185,15 +190,22 @@ export class Runner {
|
|
|
185
190
|
this.maybeCompact();
|
|
186
191
|
observe?.(message);
|
|
187
192
|
}
|
|
193
|
+
} catch (err) {
|
|
194
|
+
throw new SessionError(this.diagnostics.failure(err));
|
|
188
195
|
} finally {
|
|
189
|
-
this.
|
|
190
|
-
this.parts.skills?.close();
|
|
191
|
-
this.automation?.close(false);
|
|
192
|
-
this.recovery.close(false);
|
|
193
|
-
this.ended = true;
|
|
196
|
+
this.finishRun();
|
|
194
197
|
}
|
|
195
198
|
}
|
|
196
199
|
|
|
200
|
+
private finishRun(): void {
|
|
201
|
+
this.diagnostics.clear();
|
|
202
|
+
this.cancelCompact();
|
|
203
|
+
this.parts.skills?.close();
|
|
204
|
+
this.automation?.close(false);
|
|
205
|
+
this.recovery.close(false);
|
|
206
|
+
this.ended = true;
|
|
207
|
+
}
|
|
208
|
+
|
|
197
209
|
chat(event: ChannelEvent): Uuid {
|
|
198
210
|
try {
|
|
199
211
|
const uuid = this.inbox.push('chat', channelText(METRO_SERVER, event.content, event.meta));
|
|
@@ -228,8 +240,9 @@ export class Runner {
|
|
|
228
240
|
this.switching = this.switching
|
|
229
241
|
.then(() => this.applyModel(model))
|
|
230
242
|
.catch((err: unknown) => {
|
|
231
|
-
|
|
232
|
-
|
|
243
|
+
const failure = new SessionError(this.diagnostics.failure(err));
|
|
244
|
+
this.parts.activity?.fail(failure);
|
|
245
|
+
log.warn({ reason: failure.failure.code, diagnostic: failure.failure.diagnostic, summary: failureSummary(failure) }, 'sdk-runner: the session could not take the model the Model page picked');
|
|
233
246
|
});
|
|
234
247
|
}
|
|
235
248
|
|
|
@@ -333,6 +346,7 @@ export class Runner {
|
|
|
333
346
|
const id = typeof m.session_id === 'string' ? m.session_id : null;
|
|
334
347
|
if (id === null || id === this.sessionId) return;
|
|
335
348
|
this.sessionId = id;
|
|
349
|
+
this.diagnostics.clear();
|
|
336
350
|
log.info({ session: id, model: m.model, claude: m.claude_code_version }, 'sdk-runner: session up');
|
|
337
351
|
try {
|
|
338
352
|
this.parts.store.save(id);
|