@commonlyai/cli 0.1.173 → 0.1.175
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/node_modules/@commonlyai/seats/README.md +11 -0
- package/node_modules/@commonlyai/seats/package.json +22 -0
- package/node_modules/@commonlyai/seats/src/adapters/hermes.js +206 -0
- package/node_modules/@commonlyai/seats/src/adapters/stub.js +24 -0
- package/node_modules/@commonlyai/seats/src/contract.js +48 -0
- package/node_modules/@commonlyai/seats/src/failures.js +38 -0
- package/node_modules/@commonlyai/seats/src/index.js +22 -0
- package/node_modules/@commonlyai/seats/src/process-group.js +261 -0
- package/node_modules/@commonlyai/seats/src/record/format.js +635 -0
- package/node_modules/@commonlyai/seats/src/record/otel.js +238 -0
- package/node_modules/@commonlyai/seats/src/runtime-observation.js +29 -0
- package/node_modules/@commonlyai/seats/src/seat-model.js +17 -0
- package/node_modules/@commonlyai/seats/src/supervisor.js +308 -0
- package/node_modules/@commonlyai/seats/src/trace/events.js +45 -0
- package/node_modules/@commonlyai/seats/src/trace/store.js +276 -0
- package/node_modules/@commonlyai/seats/src/upstream-refusal.js +362 -0
- package/node_modules/@commonlyai/seats/src/valet/env.js +164 -0
- package/node_modules/@commonlyai/seats/src/valet/grant.js +118 -0
- package/node_modules/@commonlyai/seats/src/valet/permissions.js +213 -0
- package/package.json +6 -1
- package/src/commands/agent.js +21 -3
- package/src/lib/adapters/claude.js +8 -3
- package/src/lib/adapters/codex.js +5 -2
- package/src/lib/adapters/hermes.js +13 -194
- package/src/lib/adapters/seat-model.js +5 -12
- package/src/lib/adapters/stub.js +1 -24
- package/src/lib/runtime-observation.js +3 -27
- package/src/lib/seat-stop.js +96 -0
- package/src/lib/upstream-refusal.js +1 -354
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Append-only trace store on the local filesystem.
|
|
3
|
+
*
|
|
4
|
+
* <root>/runs/<runId>.jsonl the run record (docs/record-v1.md): one
|
|
5
|
+
* canonical, hash-chained entry per line
|
|
6
|
+
* <root>/index.jsonl one line per started run, one per settled run;
|
|
7
|
+
* a store convenience, not part of the format
|
|
8
|
+
*
|
|
9
|
+
* Files are 0600 and directories 0700: a trace holds agent output.
|
|
10
|
+
*/
|
|
11
|
+
import { randomUUID } from 'node:crypto';
|
|
12
|
+
import { appendFileSync, mkdirSync, readFileSync, existsSync, writeFileSync } from 'node:fs';
|
|
13
|
+
import { join } from 'node:path';
|
|
14
|
+
import {
|
|
15
|
+
appendEntry, hashLine, kindOfType, readRecord, traceTypeOf, validateActor, verifyRecord, BOARD_ACTOR,
|
|
16
|
+
} from '../record/format.js';
|
|
17
|
+
|
|
18
|
+
const RUN_ID = /^[A-Za-z0-9_-]{6,64}$/;
|
|
19
|
+
const readJsonl = (path) => (existsSync(path)
|
|
20
|
+
? readFileSync(path, 'utf8').split('\n').filter(Boolean).flatMap((line) => {
|
|
21
|
+
try { return [JSON.parse(line)]; } catch { return []; }
|
|
22
|
+
})
|
|
23
|
+
: []);
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Known credential values never reach a line, even inside agent text. Every
|
|
27
|
+
* registered value is replaced, whatever its length: the eight-character
|
|
28
|
+
* floor in redactKnownCredentials belongs to the refusal-text reader, not to
|
|
29
|
+
* the record. `secrets` is kept longest first, so a value that contains a
|
|
30
|
+
* shorter one is replaced whole rather than in pieces.
|
|
31
|
+
*/
|
|
32
|
+
const scrubText = (text, secrets) => {
|
|
33
|
+
let out = text;
|
|
34
|
+
for (const value of secrets) out = out.split(value).join('[redacted]');
|
|
35
|
+
return out;
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
const scrub = (value, secrets) => {
|
|
39
|
+
if (secrets.length === 0) return value;
|
|
40
|
+
if (typeof value === 'string') return scrubText(value, secrets);
|
|
41
|
+
if (Array.isArray(value)) return value.map((item) => scrub(item, secrets));
|
|
42
|
+
if (value && typeof value === 'object') {
|
|
43
|
+
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, scrub(item, secrets)]));
|
|
44
|
+
}
|
|
45
|
+
return value;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Streamed agent text arrives in deltas, so a value can be split across two or
|
|
50
|
+
* more of them and match none on its own. Each stream (kind + actor) is
|
|
51
|
+
* scrubbed over a rolling window instead: the held tail is joined to the next
|
|
52
|
+
* delta, the join is scrubbed, and the longest suffix of the result that
|
|
53
|
+
* could still begin a known value is held back again. What is written is the
|
|
54
|
+
* rest.
|
|
55
|
+
*
|
|
56
|
+
* Why that is enough: an occurrence that started inside written text and ran
|
|
57
|
+
* past its end would make the written part plus the held tail a proper prefix
|
|
58
|
+
* of that value, a longer such suffix than the one held. So no value spans a
|
|
59
|
+
* written boundary, and each written piece is itself scrubbed. The tail is
|
|
60
|
+
* shorter than the longest value, and only as long as an actual prefix match.
|
|
61
|
+
*/
|
|
62
|
+
const STREAMED_KINDS = new Set(['agent.message_chunk', 'agent.thought_chunk']);
|
|
63
|
+
// The held text is written out before these, so it stays in its own attempt
|
|
64
|
+
// and is never lost at settle, whichever path (ok, failure, cancel, kill) got there.
|
|
65
|
+
const FLUSH_BEFORE = new Set(['attempt.started', 'attempt.settled', 'run.settled']);
|
|
66
|
+
|
|
67
|
+
const heldLength = (text, secrets) => {
|
|
68
|
+
const longest = secrets.length ? secrets[0].length : 0;
|
|
69
|
+
for (let start = Math.max(0, text.length - longest + 1); start < text.length; start += 1) {
|
|
70
|
+
const tail = text.slice(start);
|
|
71
|
+
if (secrets.some((value) => value.length > tail.length && value.startsWith(tail))) return text.length - start;
|
|
72
|
+
}
|
|
73
|
+
return 0;
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
/** What readRun has always returned, plus the envelope: `type` is the trace name. */
|
|
77
|
+
const asEvent = (entry) => (entry.kind ? { ...entry, type: traceTypeOf(entry.kind) } : entry);
|
|
78
|
+
|
|
79
|
+
export const createTraceStore = ({ root, now = () => new Date() } = {}) => {
|
|
80
|
+
if (!root) throw new Error('createTraceStore needs a root directory');
|
|
81
|
+
const runsDir = join(root, 'runs');
|
|
82
|
+
mkdirSync(runsDir, { recursive: true, mode: 0o700 });
|
|
83
|
+
const append = (path, value) => appendFileSync(path, `${JSON.stringify(value)}\n`, { mode: 0o600 });
|
|
84
|
+
const indexPath = join(root, 'index.jsonl');
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* startRun({ runId?, adapter?, label?, taskId?, podId?, actor? })
|
|
88
|
+
* → { runId, emit, redact, revokeGrant, expireGrants, openGrants, settled }
|
|
89
|
+
*
|
|
90
|
+
* `taskId` is the board's task id verbatim and needs `podId`, since task
|
|
91
|
+
* ids repeat across pods. `actor` is the run's own party, normally
|
|
92
|
+
* {type: 'agent', id: <user id>, owner: <user id>}.
|
|
93
|
+
*/
|
|
94
|
+
const startRun = (meta = {}) => {
|
|
95
|
+
const runId = meta.runId || randomUUID();
|
|
96
|
+
if (!RUN_ID.test(runId)) throw new Error('runId must be 6-64 characters of [A-Za-z0-9_-]');
|
|
97
|
+
const taskId = meta.taskId || undefined;
|
|
98
|
+
const podId = meta.podId || undefined;
|
|
99
|
+
if (taskId && !podId) throw new Error('a taskId needs a podId: task ids repeat across pods');
|
|
100
|
+
const runActor = meta.actor === undefined ? null : validateActor(meta.actor);
|
|
101
|
+
const path = join(runsDir, `${runId}.jsonl`);
|
|
102
|
+
try {
|
|
103
|
+
writeFileSync(path, '', { mode: 0o600, flag: 'wx' });
|
|
104
|
+
} catch (error) {
|
|
105
|
+
if (error.code === 'EEXIST') throw new Error(`run ${runId} already exists`);
|
|
106
|
+
throw error;
|
|
107
|
+
}
|
|
108
|
+
let last = null; // the previous line, which the next one hashes
|
|
109
|
+
let seq = 0;
|
|
110
|
+
let settled = false;
|
|
111
|
+
let broken = false;
|
|
112
|
+
const secrets = []; // longest first; see scrubText
|
|
113
|
+
const streams = new Map(); // stream key → { kind, actor, held }
|
|
114
|
+
const grants = new Map(); // open grants: grantId → { parentGrantId, binding, expiresAt }
|
|
115
|
+
const startedAt = now();
|
|
116
|
+
|
|
117
|
+
const writeLine = (kind, data, actor) => {
|
|
118
|
+
const line = appendEntry(last, {
|
|
119
|
+
runId, taskId, podId, ts: now().toISOString(), kind, data: scrub(data, secrets), actor,
|
|
120
|
+
}, { runActor });
|
|
121
|
+
try {
|
|
122
|
+
appendFileSync(path, `${line}\n`, { mode: 0o600 });
|
|
123
|
+
} catch (error) {
|
|
124
|
+
// Some bytes may have landed; a chain cannot continue past an unknown tail.
|
|
125
|
+
broken = true;
|
|
126
|
+
throw error;
|
|
127
|
+
}
|
|
128
|
+
last = line;
|
|
129
|
+
seq += 1;
|
|
130
|
+
const entry = JSON.parse(line);
|
|
131
|
+
if (kind === 'grant.issued') {
|
|
132
|
+
grants.set(entry.data.grantId, {
|
|
133
|
+
parentGrantId: entry.data.parentGrantId ?? null, binding: entry.data.binding || null, expiresAt: entry.data.expiresAt || null,
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
if (kind === 'grant.revoked') for (const id of entry.data.grantIds) grants.delete(id);
|
|
137
|
+
return asEvent(entry);
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
/** A streamed delta's text, less a tail that could begin a known value. */
|
|
141
|
+
const windowed = (kind, data, actor) => {
|
|
142
|
+
if (!STREAMED_KINDS.has(kind) || typeof data?.text !== 'string') return data;
|
|
143
|
+
const key = JSON.stringify([kind, actor ?? null]);
|
|
144
|
+
const joined = scrubText(`${streams.get(key)?.held ?? ''}${data.text}`, secrets);
|
|
145
|
+
const cut = joined.length - heldLength(joined, secrets);
|
|
146
|
+
if (cut < joined.length) streams.set(key, { kind, actor, held: joined.slice(cut) });
|
|
147
|
+
else streams.delete(key);
|
|
148
|
+
return { ...data, text: joined.slice(0, cut) };
|
|
149
|
+
};
|
|
150
|
+
|
|
151
|
+
const write = (kind, data, actor) => writeLine(kind, windowed(kind, data, actor), actor);
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* The stream has ended, so each held tail can no longer complete a value.
|
|
155
|
+
* A tail is held only while it is a prefix of a known value, so it is part
|
|
156
|
+
* of a credential: write the marker in its place, never the prefix.
|
|
157
|
+
*/
|
|
158
|
+
const flushStreams = () => {
|
|
159
|
+
for (const [key, { kind, actor }] of streams) {
|
|
160
|
+
streams.delete(key);
|
|
161
|
+
writeLine(kind, { text: '[redacted]' }, actor);
|
|
162
|
+
}
|
|
163
|
+
};
|
|
164
|
+
|
|
165
|
+
const revoke = (grantIds, reason, actor) => (grantIds.length ? write('grant.revoked', { grantIds, reason }, actor) : null);
|
|
166
|
+
|
|
167
|
+
const emit = (type, data = {}, options = {}) => {
|
|
168
|
+
const kind = kindOfType(type);
|
|
169
|
+
if (!kind) throw new Error(`unknown trace event type: ${type}`);
|
|
170
|
+
if (settled) return null;
|
|
171
|
+
if (broken) throw new Error(`run ${runId}: the record is broken after a failed write`);
|
|
172
|
+
if (FLUSH_BEFORE.has(kind)) flushStreams();
|
|
173
|
+
const event = write(kind, data, options?.actor);
|
|
174
|
+
if (kind === 'task.done') {
|
|
175
|
+
// The board closed the task: every grant bound to it ends, in one write.
|
|
176
|
+
const bound = [...grants].filter(([, g]) => g.binding?.endsOn === 'task.done').map(([id]) => id);
|
|
177
|
+
revoke(bound, 'task.done', BOARD_ACTOR);
|
|
178
|
+
}
|
|
179
|
+
if (kind === 'run.settled') {
|
|
180
|
+
settled = true;
|
|
181
|
+
append(indexPath, {
|
|
182
|
+
runId,
|
|
183
|
+
settledAt: event.ts,
|
|
184
|
+
outcome: event.data.outcome,
|
|
185
|
+
failure: event.data.failure || null,
|
|
186
|
+
durationMs: event.data.durationMs ?? null,
|
|
187
|
+
entries: seq,
|
|
188
|
+
head: hashLine(last),
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
return event;
|
|
192
|
+
};
|
|
193
|
+
|
|
194
|
+
/** Revoke a grant and every descendant this run knows of, in one entry. */
|
|
195
|
+
const revokeGrant = (grantId, { actor } = {}) => {
|
|
196
|
+
if (settled || !grants.has(grantId)) return null;
|
|
197
|
+
const lineage = [grantId];
|
|
198
|
+
for (let i = 0; i < lineage.length; i += 1) {
|
|
199
|
+
for (const [id, g] of grants) if (g.parentGrantId === lineage[i] && !lineage.includes(id)) lineage.push(id);
|
|
200
|
+
}
|
|
201
|
+
return revoke(lineage, lineage.length > 1 ? 'cascade' : 'manual', actor);
|
|
202
|
+
};
|
|
203
|
+
|
|
204
|
+
/** Record the end of every open grant whose expiresAt has passed. */
|
|
205
|
+
const expireGrants = (at = now()) => {
|
|
206
|
+
if (settled) return null;
|
|
207
|
+
const cutoff = at.getTime();
|
|
208
|
+
return revoke([...grants].filter(([, g]) => g.expiresAt && Date.parse(g.expiresAt) <= cutoff).map(([id]) => id), 'expired');
|
|
209
|
+
};
|
|
210
|
+
|
|
211
|
+
/** Every non-empty string is scrubbed from here on, whatever its length. */
|
|
212
|
+
const redact = (values = []) => {
|
|
213
|
+
for (const value of values) if (typeof value === 'string' && value && !secrets.includes(value)) secrets.push(value);
|
|
214
|
+
secrets.sort((a, b) => b.length - a.length);
|
|
215
|
+
};
|
|
216
|
+
|
|
217
|
+
append(indexPath, {
|
|
218
|
+
runId,
|
|
219
|
+
startedAt: startedAt.toISOString(),
|
|
220
|
+
adapter: meta.adapter || null,
|
|
221
|
+
label: meta.label || null,
|
|
222
|
+
...(taskId ? { taskId, podId } : podId ? { podId } : {}),
|
|
223
|
+
...(runActor ? { actor: runActor } : {}),
|
|
224
|
+
});
|
|
225
|
+
return {
|
|
226
|
+
runId,
|
|
227
|
+
taskId: taskId ?? null,
|
|
228
|
+
podId: podId ?? null,
|
|
229
|
+
emit,
|
|
230
|
+
redact,
|
|
231
|
+
revokeGrant,
|
|
232
|
+
expireGrants,
|
|
233
|
+
openGrants: () => [...grants.keys()],
|
|
234
|
+
get settled() { return settled; },
|
|
235
|
+
};
|
|
236
|
+
};
|
|
237
|
+
|
|
238
|
+
const listRuns = ({ limit = 50 } = {}) => {
|
|
239
|
+
const byId = new Map();
|
|
240
|
+
for (const line of readJsonl(indexPath)) {
|
|
241
|
+
const row = byId.get(line.runId) || { runId: line.runId, outcome: 'running' };
|
|
242
|
+
Object.assign(row, line);
|
|
243
|
+
byId.set(line.runId, row);
|
|
244
|
+
}
|
|
245
|
+
return [...byId.values()].sort((a, b) => String(b.startedAt).localeCompare(String(a.startedAt))).slice(0, limit);
|
|
246
|
+
};
|
|
247
|
+
|
|
248
|
+
const runPath = (runId) => (RUN_ID.test(String(runId)) ? join(runsDir, `${runId}.jsonl`) : null);
|
|
249
|
+
|
|
250
|
+
const readRun = (runId) => {
|
|
251
|
+
const path = runPath(runId);
|
|
252
|
+
return path && existsSync(path) ? readRecord(readFileSync(path, 'utf8')).map(asEvent) : null;
|
|
253
|
+
};
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* verifyRun(runId) → { ok, errors: [{seq, reason, detail?}], entries, head, settled } | null
|
|
257
|
+
*
|
|
258
|
+
* The chain check from verifyRecord, plus the last line against the head the
|
|
259
|
+
* index recorded at settle (head_mismatch, count_mismatch). The index sits
|
|
260
|
+
* beside the record, so that check catches accidents, not an editor.
|
|
261
|
+
*/
|
|
262
|
+
const verifyRun = (runId) => {
|
|
263
|
+
const path = runPath(runId);
|
|
264
|
+
if (!path || !existsSync(path)) return null;
|
|
265
|
+
const result = verifyRecord(readFileSync(path, 'utf8'));
|
|
266
|
+
const sealed = readJsonl(indexPath).filter((row) => row.runId === runId && row.head).pop();
|
|
267
|
+
if (sealed) {
|
|
268
|
+
if (sealed.entries !== result.entries) result.errors.push({ seq: sealed.entries, reason: 'count_mismatch', detail: `index ${sealed.entries}, file ${result.entries}` });
|
|
269
|
+
else if (sealed.head !== result.head) result.errors.push({ seq: sealed.entries, reason: 'head_mismatch' });
|
|
270
|
+
result.ok = result.errors.length === 0;
|
|
271
|
+
}
|
|
272
|
+
return result;
|
|
273
|
+
};
|
|
274
|
+
|
|
275
|
+
return { root, startRun, listRuns, readRun, verifyRun };
|
|
276
|
+
};
|
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TASK-096 — name a refused model route, instead of reporting the silence it
|
|
3
|
+
* causes.
|
|
4
|
+
*
|
|
5
|
+
* Measured 2026-09-22 against pi 0.84.1 with a route answering 429 (the shape
|
|
6
|
+
* that made a seat look broken for two and a half days): pi runs its own retry
|
|
7
|
+
* ladder, reports the refusal on `message_end`/`auto_retry_end` JSON events, and
|
|
8
|
+
* EXITS 0 with no assistant text. `extractReply` in the pi adapter collected only
|
|
9
|
+
* `type: "error"` events — of which this produces none — so the wrapper logged
|
|
10
|
+
* `no wrapper-post (empty output)` and the cause sat unread in the stream.
|
|
11
|
+
*
|
|
12
|
+
* Two rules govern the text that reaches a seat log:
|
|
13
|
+
*
|
|
14
|
+
* 1. KEEP-LIST, not strip-list (Vera 70876, Wren 70877). A strip-list removes
|
|
15
|
+
* only the shapes we thought of, and a 401 can echo the key back. So a body
|
|
16
|
+
* contributes one NAMED field or nothing at all.
|
|
17
|
+
* 2. Exact-match refusal for credentials. If the kept text contains a value
|
|
18
|
+
* this spawn was handed, the text is dropped — a known value, not a guessed
|
|
19
|
+
* shape (Wren 70877).
|
|
20
|
+
*
|
|
21
|
+
* Two limits follow from rule 2. They are the accepted cost of refusing guessed
|
|
22
|
+
* shapes, named here so they are not rediscovered as defects later (Vera 71082):
|
|
23
|
+
*
|
|
24
|
+
* a. The match is exact and against THIS spawn's values only, so a credential
|
|
25
|
+
* that arrives transformed — base64 inside a proxy error, percent-encoded in
|
|
26
|
+
* a URL echo — does not match and the text is kept. Matching transformed
|
|
27
|
+
* shapes instead would reintroduce the class rule 1 exists to avoid.
|
|
28
|
+
* b. The keep-list can still surface someone ELSE's secret. A third party's key
|
|
29
|
+
* relayed in an upstream error is not detectable here, because the one
|
|
30
|
+
* inventory this module has is the set of values it was itself handed.
|
|
31
|
+
*
|
|
32
|
+
* Scope: this reader was wired into the pi adapter only. TASK-103 extended it
|
|
33
|
+
* to codex and claude, which are NOT silent the way pi was: the claude adapter
|
|
34
|
+
* rejects on a non-zero exit and deliberately reports stdout with it, because a
|
|
35
|
+
* `-p` run writes quota conditions there; the codex adapter rejects on
|
|
36
|
+
* `turnFailedMessage` even at exit 0. Their gap was the opposite one — the text
|
|
37
|
+
* they raised was the raw tail, unscrubbed by the rules above, and carried no
|
|
38
|
+
* HTTP status for the run loop to classify on. `scrubAdapterFailure` below is
|
|
39
|
+
* the shared reader they now both go through, so the two rules cannot drift
|
|
40
|
+
* apart between three adapters.
|
|
41
|
+
*/
|
|
42
|
+
|
|
43
|
+
/** The kept detail is truncated, ellipsis included, to this many characters. */
|
|
44
|
+
export const MAX_REFUSAL_DETAIL = 256;
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* The status an adapter prefixes to its error text: `"429: {…}"` -> `429`.
|
|
48
|
+
* Null for anything that is not a 4xx/5xx, so a message that merely starts with
|
|
49
|
+
* a number ("4040 tokens over budget") is not read as a status.
|
|
50
|
+
*
|
|
51
|
+
* The separator pi uses is NOT fixed — measured 2026-09-22, the same pi 0.84.1
|
|
52
|
+
* emits `"429: {"message":…}"` for a JSON body and `"502 <html>…"` for an
|
|
53
|
+
* HTML one. Both forms are matched here; a parser that required the colon would
|
|
54
|
+
* have kept `502 <html>…` as the whole "body" and then dropped it as non-JSON,
|
|
55
|
+
* reporting the status alone with no sign that the body was never read.
|
|
56
|
+
*/
|
|
57
|
+
const STATUS_PREFIX = /^\s*(\d{3})(?!\d)\s*:?\s*/;
|
|
58
|
+
|
|
59
|
+
export const statusFromError = (value) => {
|
|
60
|
+
const match = STATUS_PREFIX.exec(String(value ?? ''));
|
|
61
|
+
if (!match) return null;
|
|
62
|
+
const status = Number(match[1]);
|
|
63
|
+
return status >= 400 && status <= 599 ? status : null;
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* The body the adapter carried after its status prefix, or the whole text when
|
|
68
|
+
* there is no status to strip. `"429: {"message":"a: b"}"` -> `{"message":"a: b"}`;
|
|
69
|
+
* `"502 <html>"` -> `<html>`.
|
|
70
|
+
*/
|
|
71
|
+
export const bodyFromError = (value) => {
|
|
72
|
+
const text = String(value ?? '');
|
|
73
|
+
if (statusFromError(text) === null) return text.trim();
|
|
74
|
+
return text.replace(STATUS_PREFIX, '').trim();
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
/** The shortest value the refusal-text reader treats as a credential. */
|
|
78
|
+
export const MIN_CREDENTIAL_LENGTH = 8;
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Replace credential values this spawn was handed with a marker.
|
|
82
|
+
*
|
|
83
|
+
* This is the exact-match half, for text that is kept for its whole shape rather
|
|
84
|
+
* than filtered to one field — an adapter's stderr tail, say, whose diagnostic
|
|
85
|
+
* value is in the lines around the secret. Values shorter than eight characters
|
|
86
|
+
* are ignored: redacting a one-character "credential" would corrupt ordinary
|
|
87
|
+
* prose, and no issued credential is that short.
|
|
88
|
+
*
|
|
89
|
+
* The floor belongs to this reader only. The trace store scrubs every value
|
|
90
|
+
* registered with it, whatever its length, and the valet refuses to grant a
|
|
91
|
+
* credential shorter than this (credential_denied), so no granted value is
|
|
92
|
+
* ever below the floor.
|
|
93
|
+
*/
|
|
94
|
+
export const redactKnownCredentials = (text, credentials = []) => {
|
|
95
|
+
let out = String(text ?? '');
|
|
96
|
+
for (const value of credentials) {
|
|
97
|
+
if (typeof value !== 'string' || value.length < MIN_CREDENTIAL_LENGTH) continue;
|
|
98
|
+
out = out.split(value).join('[redacted]');
|
|
99
|
+
}
|
|
100
|
+
return out;
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* The keep-list. Given the raw body an adapter captured, keep at most one named
|
|
105
|
+
* field of it and only when that field is a string:
|
|
106
|
+
*
|
|
107
|
+
* `error.message` first (the OpenAI-shaped error), then a top-level `message`
|
|
108
|
+
* (which is the shape LiteLLM's budget refusal actually uses — the motivating
|
|
109
|
+
* case; a keep-list that kept only `error.message` would name the status and
|
|
110
|
+
* still hide the budget line that started this row).
|
|
111
|
+
*
|
|
112
|
+
* Everything else — HTML, a non-JSON body, a JSON body with neither field, a
|
|
113
|
+
* body echoing a known credential — contributes nothing. Returns null rather
|
|
114
|
+
* than a partial guess.
|
|
115
|
+
*/
|
|
116
|
+
export const keptRefusalDetail = (raw, { credentials = [] } = {}) => {
|
|
117
|
+
let parsed;
|
|
118
|
+
try {
|
|
119
|
+
parsed = JSON.parse(String(raw ?? ''));
|
|
120
|
+
} catch {
|
|
121
|
+
return null;
|
|
122
|
+
}
|
|
123
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return null;
|
|
124
|
+
|
|
125
|
+
const candidate = parsed?.error?.message ?? parsed?.message;
|
|
126
|
+
if (typeof candidate !== 'string') return null;
|
|
127
|
+
|
|
128
|
+
const text = candidate.replace(/\s+/g, ' ').trim();
|
|
129
|
+
if (!text) return null;
|
|
130
|
+
|
|
131
|
+
const redacted = redactKnownCredentials(text, credentials);
|
|
132
|
+
// `[redacted]` appearing is the signal that the body echoed something this
|
|
133
|
+
// spawn was handed. The status is still worth keeping; the text is not.
|
|
134
|
+
if (redacted !== text) return null;
|
|
135
|
+
|
|
136
|
+
return text.length > MAX_REFUSAL_DETAIL
|
|
137
|
+
? `${text.slice(0, MAX_REFUSAL_DETAIL - 1)}…`
|
|
138
|
+
: text;
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Read a pi JSON stream and return the refusal it ended on, or null.
|
|
143
|
+
*
|
|
144
|
+
* `auto_retry_end` is the terminal word — it names the status and the body after
|
|
145
|
+
* the ladder has finished — so it wins over the per-attempt `message_end`
|
|
146
|
+
* events that precede it. A run that refused on its first attempt and never
|
|
147
|
+
* retried still reports through `message_end`.
|
|
148
|
+
*/
|
|
149
|
+
export const readUpstreamRefusal = (stdout, { credentials = [] } = {}) => {
|
|
150
|
+
let status = null;
|
|
151
|
+
let raw = null;
|
|
152
|
+
|
|
153
|
+
for (const line of String(stdout ?? '').split('\n')) {
|
|
154
|
+
if (!line.trim()) continue;
|
|
155
|
+
let event;
|
|
156
|
+
try {
|
|
157
|
+
event = JSON.parse(line);
|
|
158
|
+
} catch {
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
if (event?.type === 'auto_retry_end' && event.success === false) {
|
|
163
|
+
const candidate = event.finalError ?? event.errorMessage;
|
|
164
|
+
const attemptStatus = statusFromError(candidate);
|
|
165
|
+
if (attemptStatus) {
|
|
166
|
+
status = attemptStatus;
|
|
167
|
+
raw = bodyFromError(candidate);
|
|
168
|
+
}
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
const message = event?.message;
|
|
173
|
+
if (
|
|
174
|
+
event?.type === 'message_end'
|
|
175
|
+
&& message?.role === 'assistant'
|
|
176
|
+
&& message.stopReason === 'error'
|
|
177
|
+
&& status === null
|
|
178
|
+
) {
|
|
179
|
+
const attemptStatus = statusFromError(message.errorMessage);
|
|
180
|
+
if (attemptStatus) {
|
|
181
|
+
status = attemptStatus;
|
|
182
|
+
raw = bodyFromError(message.errorMessage);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
if (status === null) return null;
|
|
188
|
+
return { status, detail: keptRefusalDetail(raw, { credentials }) };
|
|
189
|
+
};
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Is this text STRUCTURED upstream data — a JSON object or an array? The gate for
|
|
193
|
+
* the whole-tail fallback below, deliberately NOT `kept === null`, which is true
|
|
194
|
+
* for five different reasons (Vera 71350).
|
|
195
|
+
*
|
|
196
|
+
* An array counts, and that is a correction (Vera 71360, measured): `[{...}]` is
|
|
197
|
+
* structured data with fields this code does not know, which is exactly what rule
|
|
198
|
+
* 1 exists for, and treating it as prose shipped the whole body under exact-match
|
|
199
|
+
* alone. A JSON *string* is the opposite case and does NOT count — `"Too many
|
|
200
|
+
* requests"` is a whole-shape text with no fields to reduce, so it keeps its
|
|
201
|
+
* wording like any other prose.
|
|
202
|
+
*/
|
|
203
|
+
const isStructuredBody = (raw) => {
|
|
204
|
+
try {
|
|
205
|
+
const parsed = JSON.parse(String(raw ?? ''));
|
|
206
|
+
return parsed !== null && typeof parsed === 'object';
|
|
207
|
+
} catch {
|
|
208
|
+
return false;
|
|
209
|
+
}
|
|
210
|
+
};
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* A 3-digit number is read as a status only when the tail NAMES it as one:
|
|
214
|
+
* `status 429`, `HTTP 429`, `code: "429"` — or when it opens the tail or a ` | `
|
|
215
|
+
* segment followed by a colon, which is the shape claude's `stderr | stdout`
|
|
216
|
+
* join produces for a body that leads with its status.
|
|
217
|
+
*
|
|
218
|
+
* A bare `\b(\d{3})\b` scan is not enough, and the false positives are not
|
|
219
|
+
* hypothetical: `500 tokens`, `"max_tokens": 500` and `retry in 500 seconds`
|
|
220
|
+
* all read as 500, and `classifySpawnFailure` tests the status FIELD before the
|
|
221
|
+
* message — so a count misread as a status stands the fleet down on a refusal
|
|
222
|
+
* that never happened, which is worse than no status at all (Vera 71352).
|
|
223
|
+
*/
|
|
224
|
+
const STATUS_NAMED = /\b(?:status(?:_?code)?|code|http)\b\W{0,4}(\d{3})(?!\d)/gi;
|
|
225
|
+
// `HTTP/1.1 429 Too Many Requests` is the most canonical refusal shape there is,
|
|
226
|
+
// and the `\W{0,4}` gap above handles `HTTP 429` but cannot reach a VERSIONED
|
|
227
|
+
// line: the `1` in `/1.1` is a
|
|
228
|
+
// WORD character, so `HTTP` and `429` are not within four non-word characters of
|
|
229
|
+
// each other. It read as `status: null` — an unclassified runtime failure on the
|
|
230
|
+
// single most recognisable shape, which is the misclassification this row exists
|
|
231
|
+
// to remove (Vera 71359). Its own pattern, not a looser gap: widening the gap
|
|
232
|
+
// would start reading `/v1/models 500` as a status.
|
|
233
|
+
const STATUS_LINE = /\bHTTP\/\d(?:\.\d)?\s+(\d{3})(?!\d)/gi;
|
|
234
|
+
const STATUS_COLON = /(?:^|\|\s*)(\d{3})(?!\d)\s*:/g;
|
|
235
|
+
|
|
236
|
+
/** The first 4xx/5xx one of those shapes names, or null. */
|
|
237
|
+
const scanStatuses = (text, pattern) => {
|
|
238
|
+
for (const match of String(text ?? '').matchAll(pattern)) {
|
|
239
|
+
const value = Number(match[1]);
|
|
240
|
+
if (value >= 400 && value <= 599) return value;
|
|
241
|
+
}
|
|
242
|
+
return null;
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Find the status AND the body boundary, because they are the same question.
|
|
247
|
+
*
|
|
248
|
+
* Anchoring the status to a ` | ` segment start is not enough on its own: the
|
|
249
|
+
* body has to be taken from just past that status too, or a mid-tail read
|
|
250
|
+
* isolates the status and then hands the WHOLE joined tail to the body rules,
|
|
251
|
+
* which no longer look like a JSON body and so fall through to "keep it all".
|
|
252
|
+
* That is the difference between dropping a credential-echoing body and shipping
|
|
253
|
+
* the part of it that exact-match cannot see.
|
|
254
|
+
*
|
|
255
|
+
* `STATUS_COLON` is the only shape that moves the boundary, because it is the
|
|
256
|
+
* same shape as the anchored prefix (`429: {body}`), just found at a segment
|
|
257
|
+
* boundary — claude's `stderr | stdout` join. A status that appears inside a
|
|
258
|
+
* body (`{"error":{"code":"429"}}`) or as prose (`HTTP 503`) names the status
|
|
259
|
+
* (`HTTP/1.1 429`) but marks no boundary, so the whole text stays the body.
|
|
260
|
+
*/
|
|
261
|
+
const readStatusAndBody = (text) => {
|
|
262
|
+
const raw = String(text ?? '');
|
|
263
|
+
const prefixed = statusFromError(raw);
|
|
264
|
+
if (prefixed !== null) return { status: prefixed, body: bodyFromError(raw) };
|
|
265
|
+
// Only the colon shape moves the boundary — see the note above. A status LINE
|
|
266
|
+
// names the status but its reason phrase is prose, so the body stays the whole
|
|
267
|
+
// text and the exact-match rule protects it.
|
|
268
|
+
for (const match of raw.matchAll(STATUS_COLON)) {
|
|
269
|
+
const value = Number(match[1]);
|
|
270
|
+
if (value >= 400 && value <= 599) {
|
|
271
|
+
return { status: value, body: raw.slice(match.index + match[0].length).replace(/^\s*:?\s*/, '') };
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
const named = scanStatuses(raw, STATUS_NAMED);
|
|
275
|
+
return { status: named ?? scanStatuses(raw, STATUS_LINE), body: raw };
|
|
276
|
+
};
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* The shared reader for an adapter's own failure tail (TASK-103).
|
|
280
|
+
*
|
|
281
|
+
* The two rules above are about a body an adapter captured, and they apply here
|
|
282
|
+
* too — but the tail is a different shape from pi's refusal, and the difference
|
|
283
|
+
* decides how each rule lands:
|
|
284
|
+
*
|
|
285
|
+
* - The tail is often NOT structured. Claude's `-p` mode writes its terminal
|
|
286
|
+
* condition as prose (`Claude usage limit reached. Your limit will reset at
|
|
287
|
+
* 11:40pm.`), and that line is the whole diagnostic value of reporting
|
|
288
|
+
* stdout at all — 361 consecutive failures on 2026-08-03 carried no reason
|
|
289
|
+
* because of it. So the keep-list applies only when the tail parses as a
|
|
290
|
+
* body; when it does not, the text is KEPT as a whole shape and the
|
|
291
|
+
* exact-match rule is what protects it (`redactKnownCredentials`, which
|
|
292
|
+
* substitutes rather than drops, precisely because a whole-shape text has no
|
|
293
|
+
* second field to fall back to).
|
|
294
|
+
*
|
|
295
|
+
* - A body that IS structured (a JSON object or array) is reduced to one named
|
|
296
|
+
* field, and a body whose field echoed a value this spawn was handed is
|
|
297
|
+
* DROPPED WHOLE — the same trade rule 2 makes everywhere. The fallback to the
|
|
298
|
+
* whole tail is gated on the SHAPE (is it structured at all), never on
|
|
299
|
+
* "the keep-list returned null": that null has
|
|
300
|
+
* four other causes, and one of them is the credential echo this line exists
|
|
301
|
+
* to drop. Gating on `kept === null` would emit the whole body for that case,
|
|
302
|
+
* which is precisely what rule 2 forbids — and exact-match cannot see a
|
|
303
|
+
* transformed shape (limit (a) in the header), so a key that arrived
|
|
304
|
+
* base64'd or percent-encoded would ship (Vera 71350/71351).
|
|
305
|
+
*
|
|
306
|
+
* Returns `{ status, detail }`: `status` is the 4xx/5xx the tail named, or null.
|
|
307
|
+
* The caller decides where the status goes; `classifySpawnFailure` reads it off
|
|
308
|
+
* `error.status` or out of the message text, so both a leading `429: …` prefix
|
|
309
|
+
* and an attached field classify the same way. `detail` is `''` when nothing of
|
|
310
|
+
* the body may be shown — the status is then the whole diagnostic.
|
|
311
|
+
*/
|
|
312
|
+
export const scrubAdapterFailure = (text, { credentials = [], limit = MAX_REFUSAL_DETAIL } = {}) => {
|
|
313
|
+
const raw = String(text ?? '').trim();
|
|
314
|
+
const { status, body } = readStatusAndBody(raw);
|
|
315
|
+
const kept = keptRefusalDetail(body, { credentials });
|
|
316
|
+
const detail = kept ?? (isStructuredBody(body) ? '' : redactKnownCredentials(body, credentials));
|
|
317
|
+
return { status, detail: detail.slice(0, limit) };
|
|
318
|
+
};
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* The Error an adapter raises, built from its own failure tail.
|
|
322
|
+
*
|
|
323
|
+
* `status` is attached as a field, not only written into the text: the run
|
|
324
|
+
* loop's classifier tests `error.status === 429` before it looks at the
|
|
325
|
+
* message, and a refusal that is only readable by regex is one wording away
|
|
326
|
+
* from the misclassification that cost an hour on 2026-08-18 (see
|
|
327
|
+
* spawn-retry.js). The message still names the status so a human log says the
|
|
328
|
+
* same thing as the classifier's decision.
|
|
329
|
+
*/
|
|
330
|
+
export const adapterFailure = (label, text, { credentials = [], exitCode = null, limit } = {}) => {
|
|
331
|
+
const { status, detail } = scrubAdapterFailure(text, { credentials, limit });
|
|
332
|
+
const code = exitCode === null ? '' : ` exited with code ${exitCode}`;
|
|
333
|
+
const named = status === null ? '' : ` (upstream ${status})`;
|
|
334
|
+
const error = new Error(`${label}${code}${named}${detail ? `: ${detail}` : ''}`);
|
|
335
|
+
if (status !== null) error.status = status;
|
|
336
|
+
return error;
|
|
337
|
+
};
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* The seat-log clause. The status is always present; the detail only when the
|
|
341
|
+
* keep-list kept one. `grep 'upstream refused'` finds every refusal, which is
|
|
342
|
+
* the property the old `no wrapper-post (empty output)` line did not have.
|
|
343
|
+
*/
|
|
344
|
+
export const describeUpstreamRefusal = (refusal) => {
|
|
345
|
+
if (!refusal || typeof refusal.status !== 'number') return null;
|
|
346
|
+
return refusal.detail
|
|
347
|
+
? `upstream refused ${refusal.status}: ${refusal.detail}`
|
|
348
|
+
: `upstream refused ${refusal.status}`;
|
|
349
|
+
};
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* The credential values this spawn was handed, for the exact-match check in
|
|
353
|
+
* `redactKnownCredentials`. `envNames` are the adapter's own provider keys; the
|
|
354
|
+
* caller supplies them because only the adapter knows which key its route uses.
|
|
355
|
+
*/
|
|
356
|
+
export const spawnCredentials = (ctx = {}, envNames = []) => {
|
|
357
|
+
const env = ctx.env || process.env || {};
|
|
358
|
+
return [
|
|
359
|
+
ctx.runtimeToken,
|
|
360
|
+
...envNames.map((name) => env[name]),
|
|
361
|
+
].filter((value) => typeof value === 'string' && value.length >= MIN_CREDENTIAL_LENGTH);
|
|
362
|
+
};
|