@tangle-network/agent-app 0.60.215 → 0.60.217
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chat-routes/index.js +1 -1
- package/dist/{chunk-QA2CBATZ.js → chunk-4QUOPWIG.js} +12 -9
- package/dist/chunk-4QUOPWIG.js.map +1 -0
- package/dist/chunk-CD2LQXY6.js +306 -0
- package/dist/chunk-CD2LQXY6.js.map +1 -0
- package/dist/chunk-LRYY3OX7.js +105 -0
- package/dist/chunk-LRYY3OX7.js.map +1 -0
- package/dist/{chunk-LLO436QO.js → chunk-QM7RTTDY.js} +27 -86
- package/dist/chunk-QM7RTTDY.js.map +1 -0
- package/dist/{chunk-KDPTXXOQ.js → chunk-WDY47GBR.js} +812 -115
- package/dist/chunk-WDY47GBR.js.map +1 -0
- package/dist/{chunk-ZRLKB4KB.js → chunk-YWLZOPDS.js} +10 -11
- package/dist/{chunk-ZRLKB4KB.js.map → chunk-YWLZOPDS.js.map} +1 -1
- package/dist/launch-invariants/catalog.d.ts +16 -0
- package/dist/launch-invariants/cli.js +5 -2
- package/dist/launch-invariants/cli.js.map +1 -1
- package/dist/launch-invariants/index.d.ts +1 -1
- package/dist/launch-invariants/index.js +13 -5
- package/dist/launch-invariants/index.js.map +1 -1
- package/dist/launch-invariants/limit-alarms.d.ts +19 -1
- package/dist/launch-invariants/testing.d.ts +0 -3
- package/dist/launch-invariants/testing.js +5 -2
- package/dist/sandbox/index.d.ts +3 -3
- package/dist/sandbox/index.js +1 -1
- package/dist/signoff/app-defaults.d.ts +65 -0
- package/dist/signoff/cache.d.ts +29 -0
- package/dist/signoff/cli.js +75 -4
- package/dist/signoff/cli.js.map +1 -1
- package/dist/signoff/defaults-cli.d.ts +1 -0
- package/dist/signoff/index.js +2 -1
- package/dist/signoff/store.d.ts +2 -0
- package/dist/signoff/types.d.ts +12 -0
- package/dist/signoff/worker-defaults.d.ts +8 -0
- package/dist/turn-health/index.js +12 -291
- package/dist/turn-health/index.js.map +1 -1
- package/dist/turn-trace/index.js +20 -3
- package/dist/turn-trace/index.js.map +1 -1
- package/dist/turn-trace/otel.d.ts +11 -4
- package/package.json +5 -2
- package/dist/chunk-KDPTXXOQ.js.map +0 -1
- package/dist/chunk-LLO436QO.js.map +0 -1
- package/dist/chunk-QA2CBATZ.js.map +0 -1
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
import {
|
|
2
|
+
postSlackAlert
|
|
3
|
+
} from "./chunk-F2VX6TTJ.js";
|
|
4
|
+
|
|
5
|
+
// src/turn-health/classify.ts
|
|
6
|
+
var ARTIFACT_PART_KINDS = /* @__PURE__ */ new Set(["file", "image", "work-product", "plan", "interaction"]);
|
|
7
|
+
var KNOWN_NON_OUTPUT_PART_KINDS = /* @__PURE__ */ new Set([
|
|
8
|
+
"reasoning",
|
|
9
|
+
"step-start",
|
|
10
|
+
"step-finish",
|
|
11
|
+
"source",
|
|
12
|
+
"source-url",
|
|
13
|
+
"data"
|
|
14
|
+
]);
|
|
15
|
+
var SAMPLE_CHARS = 120;
|
|
16
|
+
function asRecord(value) {
|
|
17
|
+
return typeof value === "object" && value !== null ? value : null;
|
|
18
|
+
}
|
|
19
|
+
function nonEmptyString(value) {
|
|
20
|
+
return typeof value === "string" && value.trim().length > 0 ? value : null;
|
|
21
|
+
}
|
|
22
|
+
function isUnparseableJson(value) {
|
|
23
|
+
const trimmed = value.trim();
|
|
24
|
+
if (trimmed.length === 0) return false;
|
|
25
|
+
try {
|
|
26
|
+
JSON.parse(trimmed);
|
|
27
|
+
return false;
|
|
28
|
+
} catch {
|
|
29
|
+
return true;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
var SETTLED_TOOL_STATUSES = /* @__PURE__ */ new Set(["completed", "complete", "success", "done"]);
|
|
33
|
+
var FAILED_TOOL_STATUSES = /* @__PURE__ */ new Set(["error", "failed"]);
|
|
34
|
+
function classifyTurnOutcome(input) {
|
|
35
|
+
const reasons = [];
|
|
36
|
+
if (input.failed) {
|
|
37
|
+
reasons.push({
|
|
38
|
+
kind: "turn_failed",
|
|
39
|
+
reason: nonEmptyString(input.failureReason) ?? "unspecified"
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
const parts = Array.isArray(input.parts) ? input.parts : [];
|
|
43
|
+
let hasVisibleText = nonEmptyString(input.finalText) !== null;
|
|
44
|
+
let artifactCount = 0;
|
|
45
|
+
let toolCalls = 0;
|
|
46
|
+
const opaqueTypes = [];
|
|
47
|
+
let interpretedParts = 0;
|
|
48
|
+
for (const raw of parts) {
|
|
49
|
+
const part = asRecord(raw);
|
|
50
|
+
if (!part) continue;
|
|
51
|
+
const type = typeof part.type === "string" ? part.type : "";
|
|
52
|
+
if (type === "text") {
|
|
53
|
+
interpretedParts += 1;
|
|
54
|
+
if (nonEmptyString(part.text) !== null) hasVisibleText = true;
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
if (ARTIFACT_PART_KINDS.has(type)) {
|
|
58
|
+
interpretedParts += 1;
|
|
59
|
+
artifactCount += 1;
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
if (KNOWN_NON_OUTPUT_PART_KINDS.has(type)) {
|
|
63
|
+
interpretedParts += 1;
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
if (type !== "tool") {
|
|
67
|
+
opaqueTypes.push(type || "(missing type)");
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
interpretedParts += 1;
|
|
71
|
+
toolCalls += 1;
|
|
72
|
+
const tool = nonEmptyString(part.tool) ?? "unknown";
|
|
73
|
+
const state = asRecord(part.state);
|
|
74
|
+
const status = typeof state?.status === "string" ? state.status : "unknown";
|
|
75
|
+
const toolInput = state?.input;
|
|
76
|
+
const inputRecord = asRecord(toolInput);
|
|
77
|
+
const rejection = nonEmptyString(inputRecord?.error) ?? nonEmptyString(state?.error);
|
|
78
|
+
if (rejection && SETTLED_TOOL_STATUSES.has(status)) {
|
|
79
|
+
reasons.push({
|
|
80
|
+
kind: "tool_call_rejected",
|
|
81
|
+
// The payload names the tool the model MEANT to call; the part's own
|
|
82
|
+
// `tool` is the harness's placeholder for a rejected call.
|
|
83
|
+
tool: nonEmptyString(inputRecord?.tool) ?? tool,
|
|
84
|
+
error: rejection.slice(0, 200)
|
|
85
|
+
});
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
if (typeof toolInput === "string" && isUnparseableJson(toolInput)) {
|
|
89
|
+
reasons.push({
|
|
90
|
+
kind: "malformed_tool_call",
|
|
91
|
+
tool,
|
|
92
|
+
inputLength: toolInput.length,
|
|
93
|
+
sample: toolInput.slice(0, SAMPLE_CHARS)
|
|
94
|
+
});
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
if (!SETTLED_TOOL_STATUSES.has(status) && !FAILED_TOOL_STATUSES.has(status)) {
|
|
98
|
+
reasons.push({ kind: "tool_call_no_effect", tool, status });
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
const partsReadable = opaqueTypes.length === 0;
|
|
102
|
+
const uniqueOpaque = [...new Set(opaqueTypes)];
|
|
103
|
+
const unreadable = opaqueTypes.length > 0 && interpretedParts === 0 && !hasVisibleText;
|
|
104
|
+
if (unreadable) {
|
|
105
|
+
reasons.push({ kind: "unreadable_turn", partTypes: uniqueOpaque });
|
|
106
|
+
return {
|
|
107
|
+
healthy: false,
|
|
108
|
+
severity: "warning",
|
|
109
|
+
reasons,
|
|
110
|
+
toolCalls,
|
|
111
|
+
unreadable: true,
|
|
112
|
+
partsReadable: false,
|
|
113
|
+
opaquePartTypes: uniqueOpaque,
|
|
114
|
+
// Structurally 0 here — `unreadable` is only reached when
|
|
115
|
+
// `interpretedParts === 0`. Carried rather than hardcoded so the field
|
|
116
|
+
// has one source on every return path.
|
|
117
|
+
interpretedParts
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
if (input.gated) {
|
|
121
|
+
reasons.push({ kind: "answered_without_model" });
|
|
122
|
+
} else if (!input.failed && !hasVisibleText && artifactCount === 0) {
|
|
123
|
+
reasons.push({
|
|
124
|
+
kind: "empty_completion",
|
|
125
|
+
outputTokens: input.outputTokens ?? null,
|
|
126
|
+
partCount: parts.length,
|
|
127
|
+
...input.durationMs !== void 0 ? { durationMs: input.durationMs } : {}
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
return {
|
|
131
|
+
healthy: reasons.length === 0,
|
|
132
|
+
severity: severityOf(reasons),
|
|
133
|
+
reasons,
|
|
134
|
+
toolCalls,
|
|
135
|
+
unreadable: false,
|
|
136
|
+
partsReadable,
|
|
137
|
+
opaquePartTypes: uniqueOpaque,
|
|
138
|
+
interpretedParts
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
function severityOf(reasons) {
|
|
142
|
+
if (reasons.length === 0) return null;
|
|
143
|
+
const critical = reasons.some(
|
|
144
|
+
(r) => r.kind === "empty_completion" || r.kind === "turn_failed" || r.kind === "tool_call_rejected"
|
|
145
|
+
);
|
|
146
|
+
return critical ? "critical" : "warning";
|
|
147
|
+
}
|
|
148
|
+
function describeReason(reason) {
|
|
149
|
+
switch (reason.kind) {
|
|
150
|
+
case "empty_completion":
|
|
151
|
+
return `completed with NO output (${reason.partCount} parts, outputTokens=${reason.outputTokens ?? "unknown"})`;
|
|
152
|
+
case "malformed_tool_call":
|
|
153
|
+
return `tool \`${reason.tool}\` arguments did not parse (${reason.inputLength} chars): ${reason.sample}`;
|
|
154
|
+
case "tool_call_no_effect":
|
|
155
|
+
return `tool \`${reason.tool}\` left no effect (status=${reason.status})`;
|
|
156
|
+
case "turn_failed":
|
|
157
|
+
return `turn failed: ${reason.reason}`;
|
|
158
|
+
case "tool_call_rejected":
|
|
159
|
+
return `tool \`${reason.tool}\` was REJECTED but settled as completed: ${reason.error}`;
|
|
160
|
+
case "answered_without_model":
|
|
161
|
+
return "answered by a pre-producer gate \u2014 the model never ran";
|
|
162
|
+
case "unreadable_turn":
|
|
163
|
+
return `turn could not be read: every part had an uninterpretable type (${reason.partTypes.join(
|
|
164
|
+
", "
|
|
165
|
+
)})`;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// src/turn-health/sink.ts
|
|
170
|
+
function turnAlert(input) {
|
|
171
|
+
const kinds = [...new Set(input.reasons.map((r) => r.kind))].sort();
|
|
172
|
+
return {
|
|
173
|
+
product: input.product,
|
|
174
|
+
severity: input.severity,
|
|
175
|
+
// Keyed by product + reason kinds ONLY. A blank-completion storm across
|
|
176
|
+
// 200 turns is one incident, not 200 pages.
|
|
177
|
+
key: `turn:${input.product}:${kinds.join("+")}`,
|
|
178
|
+
title: `${input.product}: turn completed but delivered nothing (${kinds.join(", ")})`,
|
|
179
|
+
details: input.reasons.map(describeReason),
|
|
180
|
+
data: {
|
|
181
|
+
kinds,
|
|
182
|
+
...input.threadId ? { threadId: input.threadId } : {},
|
|
183
|
+
...input.turnId ? { turnId: input.turnId } : {},
|
|
184
|
+
...input.model ? { model: input.model } : {}
|
|
185
|
+
},
|
|
186
|
+
at: input.at ?? Date.now()
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
function createWebhookAlertSink(options) {
|
|
190
|
+
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
191
|
+
return {
|
|
192
|
+
async deliver(alert) {
|
|
193
|
+
const icon = alert.severity === "critical" ? ":rotating_light:" : ":warning:";
|
|
194
|
+
const lines = [
|
|
195
|
+
`${icon} *${alert.title}*`,
|
|
196
|
+
...alert.details.map((d) => `\u2022 ${d}`),
|
|
197
|
+
`_${new Date(alert.at).toISOString()}_`
|
|
198
|
+
];
|
|
199
|
+
const response = await fetchImpl(options.webhookUrl, {
|
|
200
|
+
method: "POST",
|
|
201
|
+
headers: { "Content-Type": "application/json" },
|
|
202
|
+
body: JSON.stringify({ text: lines.join("\n") })
|
|
203
|
+
});
|
|
204
|
+
if (!response.ok) {
|
|
205
|
+
throw new Error(`alert webhook responded ${response.status}`);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
function createSlackBotAlertSink(options) {
|
|
211
|
+
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
212
|
+
return {
|
|
213
|
+
async deliver(alert) {
|
|
214
|
+
const icon = alert.severity === "critical" ? ":rotating_light:" : ":warning:";
|
|
215
|
+
const lines = [
|
|
216
|
+
`${icon} *${alert.title}*`,
|
|
217
|
+
...alert.details.map((d) => `\u2022 ${d}`),
|
|
218
|
+
`_${new Date(alert.at).toISOString()}_`
|
|
219
|
+
];
|
|
220
|
+
const outcome = await postSlackAlert({
|
|
221
|
+
token: options.botToken,
|
|
222
|
+
channel: options.channel,
|
|
223
|
+
text: lines.join("\n"),
|
|
224
|
+
attempts: 1,
|
|
225
|
+
// Preserve the public minimal-fetch seam without owning a second Slack
|
|
226
|
+
// request builder or response parser. Real callers use native fetch.
|
|
227
|
+
fetchImpl: options.fetchImpl ? (async (url, init) => {
|
|
228
|
+
const response = await fetchImpl(String(url), {
|
|
229
|
+
method: init?.method ?? "POST",
|
|
230
|
+
headers: Object.fromEntries(new Headers(init?.headers)),
|
|
231
|
+
body: String(init?.body ?? "")
|
|
232
|
+
});
|
|
233
|
+
return new Response(await response.text?.() ?? "", { status: response.status });
|
|
234
|
+
}) : void 0
|
|
235
|
+
});
|
|
236
|
+
if (!outcome.delivered) throw new Error(`Slack alert failed (${outcome.reason}): ${outcome.detail}`);
|
|
237
|
+
}
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
function createConsoleAlertSink(log = console.error) {
|
|
241
|
+
return {
|
|
242
|
+
async deliver(alert) {
|
|
243
|
+
log(
|
|
244
|
+
`[turn-health] ${alert.severity.toUpperCase()} ${alert.title} :: ${alert.details.join(" | ")}`
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
function createMultiAlertSink(sinks) {
|
|
250
|
+
return {
|
|
251
|
+
async deliver(alert) {
|
|
252
|
+
const settled = await Promise.allSettled(sinks.map((s) => s.deliver(alert)));
|
|
253
|
+
const failures = settled.filter((r) => r.status === "rejected");
|
|
254
|
+
if (failures.length === sinks.length && sinks.length > 0) {
|
|
255
|
+
throw new Error("every alert sink failed");
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
function createMemoryThrottleStore() {
|
|
261
|
+
const seen = /* @__PURE__ */ new Map();
|
|
262
|
+
return {
|
|
263
|
+
async lastSentAt(key) {
|
|
264
|
+
return seen.get(key) ?? null;
|
|
265
|
+
},
|
|
266
|
+
async markSent(key, at) {
|
|
267
|
+
seen.set(key, at);
|
|
268
|
+
}
|
|
269
|
+
};
|
|
270
|
+
}
|
|
271
|
+
function createThrottledAlertSink(inner, options) {
|
|
272
|
+
const store = options.store ?? createMemoryThrottleStore();
|
|
273
|
+
return {
|
|
274
|
+
async deliver(alert) {
|
|
275
|
+
const last = await store.lastSentAt(alert.key);
|
|
276
|
+
if (last !== null && alert.at - last < options.windowMs) return;
|
|
277
|
+
await inner.deliver(alert);
|
|
278
|
+
await store.markSent(alert.key, alert.at);
|
|
279
|
+
}
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
function createGuardedAlertSink(inner, onError = (e) => console.error("[turn-health] alert delivery failed", e)) {
|
|
283
|
+
return {
|
|
284
|
+
async deliver(alert) {
|
|
285
|
+
try {
|
|
286
|
+
await inner.deliver(alert);
|
|
287
|
+
} catch (error) {
|
|
288
|
+
onError(error);
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
export {
|
|
295
|
+
classifyTurnOutcome,
|
|
296
|
+
describeReason,
|
|
297
|
+
turnAlert,
|
|
298
|
+
createWebhookAlertSink,
|
|
299
|
+
createSlackBotAlertSink,
|
|
300
|
+
createConsoleAlertSink,
|
|
301
|
+
createMultiAlertSink,
|
|
302
|
+
createMemoryThrottleStore,
|
|
303
|
+
createThrottledAlertSink,
|
|
304
|
+
createGuardedAlertSink
|
|
305
|
+
};
|
|
306
|
+
//# sourceMappingURL=chunk-CD2LQXY6.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/turn-health/classify.ts","../src/turn-health/sink.ts"],"sourcesContent":["/**\n * The classifier for turns that FAIL BY RETURNING SUCCESS.\n *\n * Every failure this module names shipped to a customer with HTTP 200, no\n * thrown error, and no log line anyone read. Three were measured in production\n * in a single week:\n *\n * - a turn settled `{\"outcome\":{\"type\":\"completed\"},\"finalText\":\"\",\n * \"tokenUsage\":{\"outputTokens\":0}}` — the customer saw a blank bubble;\n * - six `submit_proposal` tool calls collapsed into ONE whose arguments were\n * a 1,652-character non-JSON string, so zero proposals persisted and\n * nothing errored (agent-runtime #626);\n * - a thread took 255 user messages over 17 days and produced 2 replies,\n * both of them error text.\n *\n * A conventional health check cannot see any of these, because it probes\n * DEPENDENCIES (is the sandbox reachable, is the router up) and every one of\n * these failures happens with all dependencies green. This classifier probes\n * the OUTCOME instead.\n *\n * It is deliberately pure and structural: it reads a settled turn's own\n * projection, so the SAME function judges a live turn through the\n * `/chat-routes` lifecycle seam and a historical row read back out of the\n * store during a sweep. One definition of \"silently broken\", two call sites.\n */\n\n/** How loudly a reason should be routed. `critical` means a customer got\n * nothing usable; `warning` means the turn degraded but still produced\n * something a human could read. */\nexport type TurnHealthSeverity = 'critical' | 'warning'\n\n/** One specific way a turn returned success while failing.\n *\n * Each variant carries the evidence that identified it, so an alert can name\n * the offending value instead of asserting a verdict the reader has to take\n * on faith. */\nexport type TurnHealthReason =\n /** Settled without error and produced nothing a user can read: no text, and\n * no artifact part (file/image/work-product/plan/interaction). This is the\n * verbatim blank-completion capture. */\n | {\n kind: 'empty_completion'\n outputTokens: number | null\n partCount: number\n durationMs?: number\n }\n /** A tool call whose arguments never parsed. The engine surfaces unparseable\n * arguments as a RAW STRING rather than throwing, so the call is neither\n * dropped nor errored — it silently does nothing. Detecting a string-typed\n * tool input that fails `JSON.parse` is the exact fingerprint of the\n * index-less parallel-tool-call collapse. */\n | {\n kind: 'malformed_tool_call'\n tool: string\n inputLength: number\n /** Leading characters of the offending input, for the alert body. */\n sample: string\n }\n /** A tool call that never reached a terminal state carrying output. The call\n * was issued and then simply produced no effect. */\n | {\n kind: 'tool_call_no_effect'\n tool: string\n status: string\n }\n /** The turn failed outright. Not silent by itself — but it becomes silent\n * the moment nothing is watching, which is how 16 days of\n * `TANGLE_HUB_URL is required` reached customers unnoticed. */\n | { kind: 'turn_failed'; reason: string }\n /** A tool call the harness REJECTED, settled as `completed`.\n *\n * Found live in legal-agent production, and missed by every other rule here:\n * the model called `submit_proposal`, the harness answered \"Model tried to\n * call unavailable tool 'submit_proposal'\", and the part persisted as\n * `{\"tool\":\"invalid\",\"state\":{\"status\":\"completed\",\"input\":{\"error\":\"…\"}}}`.\n *\n * Status is `completed`, the arguments parse cleanly, and the turn has text —\n * so the blank-completion, malformed-argument and no-effect rules all pass it.\n * Six deliverables were requested and silently discarded while the product\n * reported success six times.\n *\n * Detected structurally, on the presence of an `error` in the settled state\n * rather than on any harness's name for a rejected call — `invalid` is one\n * harness's convention and must not be baked into the shell. */\n | {\n kind: 'tool_call_rejected'\n /** The tool the model was trying to reach, when the payload names it. */\n tool: string\n error: string\n }\n /** The turn was answered without the model ever running — a pre-producer gate\n * short-circuited and returned the product's own response.\n *\n * Reported as a `warning`, never `critical`: gating an unready turn is a\n * legitimate design (an intake flow SHOULD answer before spending a model\n * call). What is pathological is the RATE, which only the caller's own\n * threshold can judge — so this variant exists to be COUNTED, and alerting\n * on it is opt-in. It is the per-turn evidence behind a dead tool surface:\n * a gate that answers every turn means the agent never runs at all. */\n | { kind: 'answered_without_model' }\n /** The detector could not read this turn.\n *\n * Every part carried a type outside the vocabulary this classifier\n * understands — which is what field-level encryption at rest looks like from\n * the outside (tax-agent persists `{\"type\":\"__encrypted_parts__\"}`, its own\n * convention, and 32 of its 129 assistant rows are exactly that).\n *\n * This exists because the alternative is the bug this whole module hunts.\n * An unreadable row has no text, no artifact and no tool part, so every\n * other rule here would happily conclude \"nothing wrong\" — a detector\n * reporting health from data it cannot see. Blindness is a finding, not a\n * pass, so it gets its own reason and its own counter. */\n | { kind: 'unreadable_turn'; partTypes: string[] }\n\n/** A settled turn, in the narrowest shape both call sites can supply.\n *\n * Structural on purpose: the lifecycle seam supplies `finalText`/`usage`, a\n * store sweep supplies `content`/`parts` read back from a row, and neither\n * has to import the other's types. */\nexport interface TurnOutcomeInput {\n /** The turn's final assistant text. */\n finalText?: string | null\n /** The persisted assistant parts. Untyped by design — a sweep reads these\n * out of a JSON column and must not be forced to validate them first. */\n parts?: readonly unknown[] | null\n /** Output tokens, when the caller has usage. `null`/absent is unknown, which\n * is NOT the same as zero and is never treated as evidence. */\n outputTokens?: number | null\n /** Set when the turn surfaced a terminal error event. */\n failed?: boolean\n failureReason?: string | null\n durationMs?: number\n /** Set when a pre-producer gate answered this turn and the model never ran.\n * Supplied by `/chat-routes`' lifecycle seam, which stamps `gated` on the\n * completion it now fires for a `contextGate` short-circuit. */\n gated?: boolean\n}\n\n/** The verdict for one turn. `healthy` is exactly `reasons.length === 0`, kept\n * as a field so callers read intent rather than an array length. */\nexport interface TurnHealthVerdict {\n healthy: boolean\n severity: TurnHealthSeverity | null\n reasons: TurnHealthReason[]\n /** How many tool parts this turn carried.\n *\n * Zero is NOT a per-turn defect — plenty of good turns answer from context\n * without touching a tool, and paging on each one would be pure noise. It is\n * reported so a WINDOW can be judged: a product whose deliverable is tool\n * output and which produced zero tool calls across every turn in the\n * lookback has a dead tool surface, and that is the shape no per-turn rule\n * can see.\n *\n * Only meaningful alongside {@link interpretedParts} > 0. Zero tool calls\n * read off a row that persisted NO parts is not an observation about the\n * tool surface — it is the absence of one. */\n toolCalls: number\n /** True when nothing about this turn could be judged — no interpretable part\n * AND no visible text. Callers MUST exclude these from any healthy/unhealthy\n * ratio, because counting an unreadable turn as healthy is how a detector\n * reports green on data it never read. */\n unreadable: boolean\n /** False when this turn carried parts that could not be interpreted.\n *\n * Separate from {@link unreadable} because the two blindnesses have\n * different consequences, and production has a row that is one but not the\n * other: tax-agent persists CIPHERTEXT as `content` alongside an encrypted\n * `parts` blob, so the turn plainly delivered something (there is text) while\n * its tool calls are completely invisible.\n *\n * Any conclusion ABOUT TOOLS — above all the dead-tool-surface verdict — may\n * only be drawn over turns where this is true. Reading \"no tool parts\" off an\n * encrypted blob and declaring the tool surface dead would be the same\n * crime as declaring it healthy: a finding asserted from data never read. */\n partsReadable: boolean\n /** The part types that could not be interpreted. Empty when\n * {@link partsReadable}. Named in the alert so a reader can see EXACTLY what\n * the detector was blind to instead of taking \"unreadable\" on faith. */\n opaquePartTypes: string[]\n /** How many parts this module actually READ (text, artifact, tool, or a known\n * non-output kind).\n *\n * The third blindness, and the one that shipped a false page. `partsReadable`\n * only says nothing was *uninterpretable*; a row that persisted NO parts at\n * all satisfies that vacuously. Production: 97 of tax-agent's 156 assistant\n * rows predate parts persistence entirely and store `[]`, and reading \"zero\n * tool calls\" off them raised a critical `dead_tool_surface` against a\n * product whose own `turn_events` table holds 243 `tool_call` frames.\n *\n * So a tool verdict requires BOTH `partsReadable` AND this being > 0 —\n * evidence that was read, not merely evidence that failed to be unreadable. */\n interpretedParts: number\n}\n\n/** Part kinds that count as something a user actually receives.\n *\n * A tool part is deliberately NOT here. A turn that ran six tools and said\n * nothing, with no artifact to show for it, is the malformed-tool-call\n * disaster — counting a tool chip as output would suppress the very alert\n * this module exists to raise. */\nconst ARTIFACT_PART_KINDS = new Set(['file', 'image', 'work-product', 'plan', 'interaction'])\n\n/** Part types that carry no user-visible output but ARE understood.\n *\n * The distinction matters: a turn made of nothing but `reasoning` parts\n * thought hard and said nothing, which is a real empty completion. A turn made\n * of types this module has never heard of is a turn it cannot read. Only the\n * second is blindness — so these are enumerated rather than lumped in with the\n * unknown.\n *\n * Grounded in what the fleet actually persists, not guessed: a scan of every\n * assistant part in the gtm / legal / tax production tables returns exactly\n * `tool` (138), `text` (69), `reasoning` (26), `step-start` (3),\n * `step-finish` (3) and tax's opaque `__encrypted_parts__` (32). */\nconst KNOWN_NON_OUTPUT_PART_KINDS = new Set([\n 'reasoning',\n 'step-start',\n 'step-finish',\n 'source',\n 'source-url',\n 'data',\n])\n\nconst SAMPLE_CHARS = 120\n\nfunction asRecord(value: unknown): Record<string, unknown> | null {\n return typeof value === 'object' && value !== null ? (value as Record<string, unknown>) : null\n}\n\nfunction nonEmptyString(value: unknown): string | null {\n return typeof value === 'string' && value.trim().length > 0 ? value : null\n}\n\n/** True when a string is not parseable JSON.\n *\n * Only meaningful for tool INPUT, where the engine's contract is that a\n * well-formed call carries an object (or a string that parses into one). A\n * string that fails to parse means the arguments were concatenated or\n * truncated upstream. */\nfunction isUnparseableJson(value: string): boolean {\n const trimmed = value.trim()\n if (trimmed.length === 0) return false\n try {\n JSON.parse(trimmed)\n return false\n } catch {\n return true\n }\n}\n\n/** Tool statuses that mean the call actually landed.\n *\n * Anything else — `pending`, `running`, an unknown string — left no persisted\n * effect by the time the turn settled. */\nconst SETTLED_TOOL_STATUSES = new Set(['completed', 'complete', 'success', 'done'])\n\n/** Tool statuses that ended the call by reporting its failure. The model and\n * the person saw the error, so the call is neither silently rejected nor left\n * without effect: on gtm-agent, 2,527 errored calls (HTTP statuses, missing\n * files, timeouts) in one day's 500 turns were all reported as REJECTED. */\nconst FAILED_TOOL_STATUSES = new Set(['error', 'failed'])\n\n/**\n * Judge one settled turn.\n *\n * Never throws: a malformed `parts` blob is a thing this function REPORTS on,\n * so it must not be a thing it dies on. Telemetry that can crash the turn it\n * measures is worse than no telemetry.\n */\nexport function classifyTurnOutcome(input: TurnOutcomeInput): TurnHealthVerdict {\n const reasons: TurnHealthReason[] = []\n\n if (input.failed) {\n reasons.push({\n kind: 'turn_failed',\n reason: nonEmptyString(input.failureReason) ?? 'unspecified',\n })\n }\n\n const parts = Array.isArray(input.parts) ? input.parts : []\n\n let hasVisibleText = nonEmptyString(input.finalText) !== null\n let artifactCount = 0\n let toolCalls = 0\n // Part types this module could not interpret. Tracked so blindness can be\n // reported as blindness instead of silently reading as health.\n const opaqueTypes: string[] = []\n let interpretedParts = 0\n\n for (const raw of parts) {\n const part = asRecord(raw)\n if (!part) continue\n const type = typeof part.type === 'string' ? part.type : ''\n\n if (type === 'text') {\n interpretedParts += 1\n if (nonEmptyString(part.text) !== null) hasVisibleText = true\n continue\n }\n if (ARTIFACT_PART_KINDS.has(type)) {\n interpretedParts += 1\n artifactCount += 1\n continue\n }\n if (KNOWN_NON_OUTPUT_PART_KINDS.has(type)) {\n interpretedParts += 1\n continue\n }\n if (type !== 'tool') {\n opaqueTypes.push(type || '(missing type)')\n continue\n }\n interpretedParts += 1\n toolCalls += 1\n\n const tool = nonEmptyString(part.tool) ?? 'unknown'\n const state = asRecord(part.state)\n const status = typeof state?.status === 'string' ? state.status : 'unknown'\n\n // The #626 fingerprint: arguments surfaced as a raw string because they\n // failed to parse upstream. Checked before the status gate — a malformed\n // call can still be marked completed, which is precisely why it is silent.\n // A rejected call settles as `completed` and carries its rejection in the\n // state. Checked BEFORE the status gate for the same reason as the\n // malformed case: the status is exactly what makes it silent.\n const toolInput = state?.input\n const inputRecord = asRecord(toolInput)\n const rejection =\n nonEmptyString(inputRecord?.error) ?? nonEmptyString((state as Record<string, unknown>)?.error)\n if (rejection && SETTLED_TOOL_STATUSES.has(status)) {\n reasons.push({\n kind: 'tool_call_rejected',\n // The payload names the tool the model MEANT to call; the part's own\n // `tool` is the harness's placeholder for a rejected call.\n tool: nonEmptyString(inputRecord?.tool) ?? tool,\n error: rejection.slice(0, 200),\n })\n continue\n }\n if (typeof toolInput === 'string' && isUnparseableJson(toolInput)) {\n reasons.push({\n kind: 'malformed_tool_call',\n tool,\n inputLength: toolInput.length,\n sample: toolInput.slice(0, SAMPLE_CHARS),\n })\n continue\n }\n\n if (!SETTLED_TOOL_STATUSES.has(status) && !FAILED_TOOL_STATUSES.has(status)) {\n reasons.push({ kind: 'tool_call_no_effect', tool, status })\n }\n }\n\n // A turn whose parts were ALL uninterpretable cannot be judged. Reported as\n // blindness and returned early, because every rule below would otherwise read\n // \"no text, no artifact\" off data that was simply encrypted and call it a\n // blank completion — a false page that would train readers to ignore this\n // module, which is the same ending as no detector at all.\n const partsReadable = opaqueTypes.length === 0\n const uniqueOpaque = [...new Set(opaqueTypes)]\n const unreadable = opaqueTypes.length > 0 && interpretedParts === 0 && !hasVisibleText\n if (unreadable) {\n reasons.push({ kind: 'unreadable_turn', partTypes: uniqueOpaque })\n return {\n healthy: false,\n severity: 'warning',\n reasons,\n toolCalls,\n unreadable: true,\n partsReadable: false,\n opaquePartTypes: uniqueOpaque,\n // Structurally 0 here — `unreadable` is only reached when\n // `interpretedParts === 0`. Carried rather than hardcoded so the field\n // has one source on every return path.\n interpretedParts,\n }\n }\n\n // A gate answered before the producer ran. Recorded rather than judged: see\n // `answered_without_model`. It also SUPPRESSES the empty-completion rule\n // below — the model producing no text is the expected outcome when the model\n // never ran, and firing `empty_completion` here would page on every healthy\n // intake turn.\n if (input.gated) {\n reasons.push({ kind: 'answered_without_model' })\n } else if (!input.failed && !hasVisibleText && artifactCount === 0) {\n // `outputTokens` is corroborating evidence, never the trigger: a turn can\n // spend tokens on reasoning and still deliver nothing, and a turn with\n // unknown usage can still be perfectly fine.\n reasons.push({\n kind: 'empty_completion',\n outputTokens: input.outputTokens ?? null,\n partCount: parts.length,\n ...(input.durationMs !== undefined ? { durationMs: input.durationMs } : {}),\n })\n }\n\n return {\n healthy: reasons.length === 0,\n severity: severityOf(reasons),\n reasons,\n toolCalls,\n unreadable: false,\n partsReadable,\n opaquePartTypes: uniqueOpaque,\n interpretedParts,\n }\n}\n\n/** `critical` when the customer got nothing usable out of the turn. A\n * malformed tool call alongside readable text is a `warning` — degraded, but\n * a human still received an answer. */\nfunction severityOf(reasons: TurnHealthReason[]): TurnHealthSeverity | null {\n if (reasons.length === 0) return null\n // A rejected tool call is critical even alongside readable prose: the\n // customer got words where they should have got a filed deliverable.\n const critical = reasons.some(\n (r) =>\n r.kind === 'empty_completion' ||\n r.kind === 'turn_failed' ||\n r.kind === 'tool_call_rejected',\n )\n return critical ? 'critical' : 'warning'\n}\n\n/** One-line human summary of a reason, for an alert body. */\nexport function describeReason(reason: TurnHealthReason): string {\n switch (reason.kind) {\n case 'empty_completion':\n return `completed with NO output (${reason.partCount} parts, outputTokens=${\n reason.outputTokens ?? 'unknown'\n })`\n case 'malformed_tool_call':\n return `tool \\`${reason.tool}\\` arguments did not parse (${reason.inputLength} chars): ${reason.sample}`\n case 'tool_call_no_effect':\n return `tool \\`${reason.tool}\\` left no effect (status=${reason.status})`\n case 'turn_failed':\n return `turn failed: ${reason.reason}`\n case 'tool_call_rejected':\n return `tool \\`${reason.tool}\\` was REJECTED but settled as completed: ${reason.error}`\n case 'answered_without_model':\n return 'answered by a pre-producer gate — the model never ran'\n case 'unreadable_turn':\n return `turn could not be read: every part had an uninterpretable type (${reason.partTypes.join(\n ', ',\n )})`\n }\n}\n","/**\n * Where a silent-failure verdict GOES.\n *\n * The detection half is worthless without this half. Every failure this module\n * finds was already visible in the database the whole time — 255 unanswered\n * messages sat in a table for 17 days. What was missing was not the data, it\n * was delivery to a human who had not thought to look.\n *\n * So the sink is a seam, not a channel: the product supplies the transport,\n * and agent-app ships the two shapes the fleet already has credentials for\n * (an ops webhook, and stderr). No new channel is invented here.\n */\n\nimport { postSlackAlert } from '../alerting/slack.js'\nimport { describeReason, type TurnHealthReason, type TurnHealthSeverity } from './classify.js'\n\n/** One deliverable alert. */\nexport interface TurnHealthAlert {\n /** Which product raised it (`projectId`). Alerts from four products land in\n * one channel, so this is what makes the message actionable. */\n product: string\n severity: TurnHealthSeverity\n /** Stable grouping key. Throttling is keyed on this, so it must NOT contain\n * a turn id or a timestamp or every alert is unique and nothing dedupes. */\n key: string\n title: string\n /** Human-readable lines. */\n details: string[]\n /** Structured payload for a machine consumer. */\n data?: Record<string, unknown>\n at: number\n}\n\n/** Deliver an alert. Implementations MUST NOT throw — see\n * {@link createGuardedAlertSink}. */\nexport interface AlertSink {\n deliver(alert: TurnHealthAlert): Promise<void>\n}\n\n/** Build the alert for a set of reasons found on one turn. */\nexport function turnAlert(input: {\n product: string\n severity: TurnHealthSeverity\n reasons: TurnHealthReason[]\n threadId?: string\n turnId?: string\n model?: string\n at?: number\n}): TurnHealthAlert {\n const kinds = [...new Set(input.reasons.map((r) => r.kind))].sort()\n return {\n product: input.product,\n severity: input.severity,\n // Keyed by product + reason kinds ONLY. A blank-completion storm across\n // 200 turns is one incident, not 200 pages.\n key: `turn:${input.product}:${kinds.join('+')}`,\n title: `${input.product}: turn completed but delivered nothing (${kinds.join(', ')})`,\n details: input.reasons.map(describeReason),\n data: {\n kinds,\n ...(input.threadId ? { threadId: input.threadId } : {}),\n ...(input.turnId ? { turnId: input.turnId } : {}),\n ...(input.model ? { model: input.model } : {}),\n },\n at: input.at ?? Date.now(),\n }\n}\n\n// ── transports ────────────────────────────────────────────────────────────\n\n/** Minimal structural fetch, so this module has no lib-dom dependency and can\n * be driven by a fake in tests. */\nexport type FetchLike = (\n url: string,\n init: { method: string; headers: Record<string, string>; body: string },\n) => Promise<{ ok: boolean; status: number; text?(): Promise<string> }>\n\n/**\n * POST to an incoming webhook in the Slack message format.\n *\n * Chosen because the org already runs one (`SLACK_OPS_WEBHOOK_URL`) and\n * gtm-agent's outbound webhook code already speaks this exact shape — the\n * instruction was to route somewhere humans already look, not to stand up a\n * new channel. Discord and most log drains accept the same `{text}` body.\n */\nexport function createWebhookAlertSink(options: {\n webhookUrl: string\n fetchImpl?: FetchLike\n}): AlertSink {\n const fetchImpl = options.fetchImpl ?? (globalThis.fetch as unknown as FetchLike)\n return {\n async deliver(alert) {\n const icon = alert.severity === 'critical' ? ':rotating_light:' : ':warning:'\n const lines = [\n `${icon} *${alert.title}*`,\n ...alert.details.map((d) => `• ${d}`),\n `_${new Date(alert.at).toISOString()}_`,\n ]\n const response = await fetchImpl(options.webhookUrl, {\n method: 'POST',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify({ text: lines.join('\\n') }),\n })\n if (!response.ok) {\n // A dropped alert is a silent failure of the silent-failure detector.\n // It must be loud in the one place that is still working: the log.\n throw new Error(`alert webhook responded ${response.status}`)\n }\n },\n }\n}\n\n/**\n * POST to Slack `chat.postMessage` with a bot token.\n *\n * This is the transport the org actually has. A survey of the four product\n * repos found NO ops alert path of any kind — no incoming webhook, no pager, no\n * notifier — which is the mechanical reason a 17-day outage never reached a\n * human. What does exist is a Slack bot token in the shared secrets store, and\n * `@tangle-network/agent-integrations` already speaks this exact API, so this\n * routes alerts through the channel the org runs rather than standing up a new\n * one.\n *\n * Slack answers `200 OK` with `{\"ok\": false, \"error\": \"...\"}` for an invalid\n * token or channel, so the body is checked and not just the status — a\n * transport that reports success on a rejected post would make the alerter\n * itself a silent failure.\n */\nexport function createSlackBotAlertSink(options: {\n botToken: string\n channel: string\n fetchImpl?: FetchLike\n}): AlertSink {\n const fetchImpl = options.fetchImpl ?? (globalThis.fetch as unknown as FetchLike)\n return {\n async deliver(alert) {\n const icon = alert.severity === 'critical' ? ':rotating_light:' : ':warning:'\n const lines = [\n `${icon} *${alert.title}*`,\n ...alert.details.map((d) => `• ${d}`),\n `_${new Date(alert.at).toISOString()}_`,\n ]\n const outcome = await postSlackAlert({\n token: options.botToken, channel: options.channel, text: lines.join('\\n'), attempts: 1,\n // Preserve the public minimal-fetch seam without owning a second Slack\n // request builder or response parser. Real callers use native fetch.\n fetchImpl: options.fetchImpl ? (async (url, init) => {\n const response = await fetchImpl(String(url), {\n method: init?.method ?? 'POST',\n headers: Object.fromEntries(new Headers(init?.headers)),\n body: String(init?.body ?? ''),\n })\n return new Response(await response.text?.() ?? '', { status: response.status })\n }) : undefined,\n })\n if (!outcome.delivered) throw new Error(`Slack alert failed (${outcome.reason}): ${outcome.detail}`)\n },\n }\n}\n\n/** stderr sink. The zero-config fallback so a product that has not yet been\n * given a webhook still emits something a log search can find. */\nexport function createConsoleAlertSink(log: (message: string) => void = console.error): AlertSink {\n return {\n async deliver(alert) {\n log(\n `[turn-health] ${alert.severity.toUpperCase()} ${alert.title} :: ${alert.details.join(' | ')}`,\n )\n },\n }\n}\n\n/** Fan out to several sinks. One failing transport must not stop the others. */\nexport function createMultiAlertSink(sinks: readonly AlertSink[]): AlertSink {\n return {\n async deliver(alert) {\n const settled = await Promise.allSettled(sinks.map((s) => s.deliver(alert)))\n const failures = settled.filter((r) => r.status === 'rejected')\n if (failures.length === sinks.length && sinks.length > 0) {\n throw new Error('every alert sink failed')\n }\n },\n }\n}\n\n// ── throttling ────────────────────────────────────────────────────────────\n\n/** Records the last time a key was alerted on. A product backs this with KV,\n * D1, or a Durable Object; the in-memory default is correct for a sweep that\n * runs as a single cron invocation. */\nexport interface AlertThrottleStore {\n lastSentAt(key: string): Promise<number | null>\n markSent(key: string, at: number): Promise<void>\n}\n\n/** Process-local throttle store. */\nexport function createMemoryThrottleStore(): AlertThrottleStore {\n const seen = new Map<string, number>()\n return {\n async lastSentAt(key) {\n return seen.get(key) ?? null\n },\n async markSent(key, at) {\n seen.set(key, at)\n },\n }\n}\n\n/**\n * Collapse repeats of the same `key` inside `windowMs`.\n *\n * Deliberately re-alerts once per window rather than going silent after the\n * first: an incident that is still burning must keep saying so. Going quiet\n * after one message is how a 17-day outage stays invisible after someone\n * dismisses the first notification.\n */\nexport function createThrottledAlertSink(\n inner: AlertSink,\n options: { windowMs: number; store?: AlertThrottleStore },\n): AlertSink {\n const store = options.store ?? createMemoryThrottleStore()\n return {\n async deliver(alert) {\n const last = await store.lastSentAt(alert.key)\n if (last !== null && alert.at - last < options.windowMs) return\n await inner.deliver(alert)\n await store.markSent(alert.key, alert.at)\n },\n }\n}\n\n/**\n * Swallow transport errors so telemetry can never fail the turn it measures.\n *\n * Use this at the LIVE lifecycle call site only. A sweep should let the error\n * surface, because a sweep that cannot deliver has done nothing at all and its\n * cron run should go red.\n */\nexport function createGuardedAlertSink(\n inner: AlertSink,\n onError: (error: unknown) => void = (e) => console.error('[turn-health] alert delivery failed', e),\n): AlertSink {\n return {\n async deliver(alert) {\n try {\n await inner.deliver(alert)\n } catch (error) {\n onError(error)\n }\n },\n }\n}\n"],"mappings":";;;;;AAwMA,IAAM,sBAAsB,oBAAI,IAAI,CAAC,QAAQ,SAAS,gBAAgB,QAAQ,aAAa,CAAC;AAc5F,IAAM,8BAA8B,oBAAI,IAAI;AAAA,EAC1C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,eAAe;AAErB,SAAS,SAAS,OAAgD;AAChE,SAAO,OAAO,UAAU,YAAY,UAAU,OAAQ,QAAoC;AAC5F;AAEA,SAAS,eAAe,OAA+B;AACrD,SAAO,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,IAAI,QAAQ;AACxE;AAQA,SAAS,kBAAkB,OAAwB;AACjD,QAAM,UAAU,MAAM,KAAK;AAC3B,MAAI,QAAQ,WAAW,EAAG,QAAO;AACjC,MAAI;AACF,SAAK,MAAM,OAAO;AAClB,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAMA,IAAM,wBAAwB,oBAAI,IAAI,CAAC,aAAa,YAAY,WAAW,MAAM,CAAC;AAMlF,IAAM,uBAAuB,oBAAI,IAAI,CAAC,SAAS,QAAQ,CAAC;AASjD,SAAS,oBAAoB,OAA4C;AAC9E,QAAM,UAA8B,CAAC;AAErC,MAAI,MAAM,QAAQ;AAChB,YAAQ,KAAK;AAAA,MACX,MAAM;AAAA,MACN,QAAQ,eAAe,MAAM,aAAa,KAAK;AAAA,IACjD,CAAC;AAAA,EACH;AAEA,QAAM,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,MAAM,QAAQ,CAAC;AAE1D,MAAI,iBAAiB,eAAe,MAAM,SAAS,MAAM;AACzD,MAAI,gBAAgB;AACpB,MAAI,YAAY;AAGhB,QAAM,cAAwB,CAAC;AAC/B,MAAI,mBAAmB;AAEvB,aAAW,OAAO,OAAO;AACvB,UAAM,OAAO,SAAS,GAAG;AACzB,QAAI,CAAC,KAAM;AACX,UAAM,OAAO,OAAO,KAAK,SAAS,WAAW,KAAK,OAAO;AAEzD,QAAI,SAAS,QAAQ;AACnB,0BAAoB;AACpB,UAAI,eAAe,KAAK,IAAI,MAAM,KAAM,kBAAiB;AACzD;AAAA,IACF;AACA,QAAI,oBAAoB,IAAI,IAAI,GAAG;AACjC,0BAAoB;AACpB,uBAAiB;AACjB;AAAA,IACF;AACA,QAAI,4BAA4B,IAAI,IAAI,GAAG;AACzC,0BAAoB;AACpB;AAAA,IACF;AACA,QAAI,SAAS,QAAQ;AACnB,kBAAY,KAAK,QAAQ,gBAAgB;AACzC;AAAA,IACF;AACA,wBAAoB;AACpB,iBAAa;AAEb,UAAM,OAAO,eAAe,KAAK,IAAI,KAAK;AAC1C,UAAM,QAAQ,SAAS,KAAK,KAAK;AACjC,UAAM,SAAS,OAAO,OAAO,WAAW,WAAW,MAAM,SAAS;AAQlE,UAAM,YAAY,OAAO;AACzB,UAAM,cAAc,SAAS,SAAS;AACtC,UAAM,YACJ,eAAe,aAAa,KAAK,KAAK,eAAgB,OAAmC,KAAK;AAChG,QAAI,aAAa,sBAAsB,IAAI,MAAM,GAAG;AAClD,cAAQ,KAAK;AAAA,QACX,MAAM;AAAA;AAAA;AAAA,QAGN,MAAM,eAAe,aAAa,IAAI,KAAK;AAAA,QAC3C,OAAO,UAAU,MAAM,GAAG,GAAG;AAAA,MAC/B,CAAC;AACD;AAAA,IACF;AACA,QAAI,OAAO,cAAc,YAAY,kBAAkB,SAAS,GAAG;AACjE,cAAQ,KAAK;AAAA,QACX,MAAM;AAAA,QACN;AAAA,QACA,aAAa,UAAU;AAAA,QACvB,QAAQ,UAAU,MAAM,GAAG,YAAY;AAAA,MACzC,CAAC;AACD;AAAA,IACF;AAEA,QAAI,CAAC,sBAAsB,IAAI,MAAM,KAAK,CAAC,qBAAqB,IAAI,MAAM,GAAG;AAC3E,cAAQ,KAAK,EAAE,MAAM,uBAAuB,MAAM,OAAO,CAAC;AAAA,IAC5D;AAAA,EACF;AAOA,QAAM,gBAAgB,YAAY,WAAW;AAC7C,QAAM,eAAe,CAAC,GAAG,IAAI,IAAI,WAAW,CAAC;AAC7C,QAAM,aAAa,YAAY,SAAS,KAAK,qBAAqB,KAAK,CAAC;AACxE,MAAI,YAAY;AACd,YAAQ,KAAK,EAAE,MAAM,mBAAmB,WAAW,aAAa,CAAC;AACjE,WAAO;AAAA,MACL,SAAS;AAAA,MACT,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA,YAAY;AAAA,MACZ,eAAe;AAAA,MACf,iBAAiB;AAAA;AAAA;AAAA;AAAA,MAIjB;AAAA,IACF;AAAA,EACF;AAOA,MAAI,MAAM,OAAO;AACf,YAAQ,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAAA,EACjD,WAAW,CAAC,MAAM,UAAU,CAAC,kBAAkB,kBAAkB,GAAG;AAIlE,YAAQ,KAAK;AAAA,MACX,MAAM;AAAA,MACN,cAAc,MAAM,gBAAgB;AAAA,MACpC,WAAW,MAAM;AAAA,MACjB,GAAI,MAAM,eAAe,SAAY,EAAE,YAAY,MAAM,WAAW,IAAI,CAAC;AAAA,IAC3E,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,SAAS,QAAQ,WAAW;AAAA,IAC5B,UAAU,WAAW,OAAO;AAAA,IAC5B;AAAA,IACA;AAAA,IACA,YAAY;AAAA,IACZ;AAAA,IACA,iBAAiB;AAAA,IACjB;AAAA,EACF;AACF;AAKA,SAAS,WAAW,SAAwD;AAC1E,MAAI,QAAQ,WAAW,EAAG,QAAO;AAGjC,QAAM,WAAW,QAAQ;AAAA,IACvB,CAAC,MACC,EAAE,SAAS,sBACX,EAAE,SAAS,iBACX,EAAE,SAAS;AAAA,EACf;AACA,SAAO,WAAW,aAAa;AACjC;AAGO,SAAS,eAAe,QAAkC;AAC/D,UAAQ,OAAO,MAAM;AAAA,IACnB,KAAK;AACH,aAAO,6BAA6B,OAAO,SAAS,wBAClD,OAAO,gBAAgB,SACzB;AAAA,IACF,KAAK;AACH,aAAO,UAAU,OAAO,IAAI,+BAA+B,OAAO,WAAW,YAAY,OAAO,MAAM;AAAA,IACxG,KAAK;AACH,aAAO,UAAU,OAAO,IAAI,6BAA6B,OAAO,MAAM;AAAA,IACxE,KAAK;AACH,aAAO,gBAAgB,OAAO,MAAM;AAAA,IACtC,KAAK;AACH,aAAO,UAAU,OAAO,IAAI,6CAA6C,OAAO,KAAK;AAAA,IACvF,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO,mEAAmE,OAAO,UAAU;AAAA,QACzF;AAAA,MACF,CAAC;AAAA,EACL;AACF;;;ACxZO,SAAS,UAAU,OAQN;AAClB,QAAM,QAAQ,CAAC,GAAG,IAAI,IAAI,MAAM,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,KAAK;AAClE,SAAO;AAAA,IACL,SAAS,MAAM;AAAA,IACf,UAAU,MAAM;AAAA;AAAA;AAAA,IAGhB,KAAK,QAAQ,MAAM,OAAO,IAAI,MAAM,KAAK,GAAG,CAAC;AAAA,IAC7C,OAAO,GAAG,MAAM,OAAO,2CAA2C,MAAM,KAAK,IAAI,CAAC;AAAA,IAClF,SAAS,MAAM,QAAQ,IAAI,cAAc;AAAA,IACzC,MAAM;AAAA,MACJ;AAAA,MACA,GAAI,MAAM,WAAW,EAAE,UAAU,MAAM,SAAS,IAAI,CAAC;AAAA,MACrD,GAAI,MAAM,SAAS,EAAE,QAAQ,MAAM,OAAO,IAAI,CAAC;AAAA,MAC/C,GAAI,MAAM,QAAQ,EAAE,OAAO,MAAM,MAAM,IAAI,CAAC;AAAA,IAC9C;AAAA,IACA,IAAI,MAAM,MAAM,KAAK,IAAI;AAAA,EAC3B;AACF;AAmBO,SAAS,uBAAuB,SAGzB;AACZ,QAAM,YAAY,QAAQ,aAAc,WAAW;AACnD,SAAO;AAAA,IACL,MAAM,QAAQ,OAAO;AACnB,YAAM,OAAO,MAAM,aAAa,aAAa,qBAAqB;AAClE,YAAM,QAAQ;AAAA,QACZ,GAAG,IAAI,KAAK,MAAM,KAAK;AAAA,QACvB,GAAG,MAAM,QAAQ,IAAI,CAAC,MAAM,UAAK,CAAC,EAAE;AAAA,QACpC,IAAI,IAAI,KAAK,MAAM,EAAE,EAAE,YAAY,CAAC;AAAA,MACtC;AACA,YAAM,WAAW,MAAM,UAAU,QAAQ,YAAY;AAAA,QACnD,QAAQ;AAAA,QACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,QAC9C,MAAM,KAAK,UAAU,EAAE,MAAM,MAAM,KAAK,IAAI,EAAE,CAAC;AAAA,MACjD,CAAC;AACD,UAAI,CAAC,SAAS,IAAI;AAGhB,cAAM,IAAI,MAAM,2BAA2B,SAAS,MAAM,EAAE;AAAA,MAC9D;AAAA,IACF;AAAA,EACF;AACF;AAkBO,SAAS,wBAAwB,SAI1B;AACZ,QAAM,YAAY,QAAQ,aAAc,WAAW;AACnD,SAAO;AAAA,IACL,MAAM,QAAQ,OAAO;AACnB,YAAM,OAAO,MAAM,aAAa,aAAa,qBAAqB;AAClE,YAAM,QAAQ;AAAA,QACZ,GAAG,IAAI,KAAK,MAAM,KAAK;AAAA,QACvB,GAAG,MAAM,QAAQ,IAAI,CAAC,MAAM,UAAK,CAAC,EAAE;AAAA,QACpC,IAAI,IAAI,KAAK,MAAM,EAAE,EAAE,YAAY,CAAC;AAAA,MACtC;AACA,YAAM,UAAU,MAAM,eAAe;AAAA,QACnC,OAAO,QAAQ;AAAA,QAAU,SAAS,QAAQ;AAAA,QAAS,MAAM,MAAM,KAAK,IAAI;AAAA,QAAG,UAAU;AAAA;AAAA;AAAA,QAGrF,WAAW,QAAQ,aAAa,OAAO,KAAK,SAAS;AACnD,gBAAM,WAAW,MAAM,UAAU,OAAO,GAAG,GAAG;AAAA,YAC5C,QAAQ,MAAM,UAAU;AAAA,YACxB,SAAS,OAAO,YAAY,IAAI,QAAQ,MAAM,OAAO,CAAC;AAAA,YACtD,MAAM,OAAO,MAAM,QAAQ,EAAE;AAAA,UAC/B,CAAC;AACD,iBAAO,IAAI,SAAS,MAAM,SAAS,OAAO,KAAK,IAAI,EAAE,QAAQ,SAAS,OAAO,CAAC;AAAA,QAChF,KAAK;AAAA,MACP,CAAC;AACD,UAAI,CAAC,QAAQ,UAAW,OAAM,IAAI,MAAM,uBAAuB,QAAQ,MAAM,MAAM,QAAQ,MAAM,EAAE;AAAA,IACrG;AAAA,EACF;AACF;AAIO,SAAS,uBAAuB,MAAiC,QAAQ,OAAkB;AAChG,SAAO;AAAA,IACL,MAAM,QAAQ,OAAO;AACnB;AAAA,QACE,iBAAiB,MAAM,SAAS,YAAY,CAAC,IAAI,MAAM,KAAK,OAAO,MAAM,QAAQ,KAAK,KAAK,CAAC;AAAA,MAC9F;AAAA,IACF;AAAA,EACF;AACF;AAGO,SAAS,qBAAqB,OAAwC;AAC3E,SAAO;AAAA,IACL,MAAM,QAAQ,OAAO;AACnB,YAAM,UAAU,MAAM,QAAQ,WAAW,MAAM,IAAI,CAAC,MAAM,EAAE,QAAQ,KAAK,CAAC,CAAC;AAC3E,YAAM,WAAW,QAAQ,OAAO,CAAC,MAAM,EAAE,WAAW,UAAU;AAC9D,UAAI,SAAS,WAAW,MAAM,UAAU,MAAM,SAAS,GAAG;AACxD,cAAM,IAAI,MAAM,yBAAyB;AAAA,MAC3C;AAAA,IACF;AAAA,EACF;AACF;AAaO,SAAS,4BAAgD;AAC9D,QAAM,OAAO,oBAAI,IAAoB;AACrC,SAAO;AAAA,IACL,MAAM,WAAW,KAAK;AACpB,aAAO,KAAK,IAAI,GAAG,KAAK;AAAA,IAC1B;AAAA,IACA,MAAM,SAAS,KAAK,IAAI;AACtB,WAAK,IAAI,KAAK,EAAE;AAAA,IAClB;AAAA,EACF;AACF;AAUO,SAAS,yBACd,OACA,SACW;AACX,QAAM,QAAQ,QAAQ,SAAS,0BAA0B;AACzD,SAAO;AAAA,IACL,MAAM,QAAQ,OAAO;AACnB,YAAM,OAAO,MAAM,MAAM,WAAW,MAAM,GAAG;AAC7C,UAAI,SAAS,QAAQ,MAAM,KAAK,OAAO,QAAQ,SAAU;AACzD,YAAM,MAAM,QAAQ,KAAK;AACzB,YAAM,MAAM,SAAS,MAAM,KAAK,MAAM,EAAE;AAAA,IAC1C;AAAA,EACF;AACF;AASO,SAAS,uBACd,OACA,UAAoC,CAAC,MAAM,QAAQ,MAAM,uCAAuC,CAAC,GACtF;AACX,SAAO;AAAA,IACL,MAAM,QAAQ,OAAO;AACnB,UAAI;AACF,cAAM,MAAM,QAAQ,KAAK;AAAA,MAC3B,SAAS,OAAO;AACd,gBAAQ,KAAK;AAAA,MACf;AAAA,IACF;AAAA,EACF;AACF;","names":[]}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
// src/launch-invariants/catalog.ts
|
|
2
|
+
var LAUNCH_BUDGETS = {
|
|
3
|
+
/** Most bytes one query may return in a scheduled job or report route: one sized batch with room for its listing. */
|
|
4
|
+
queryBytes: 8 * 1024 * 1024,
|
|
5
|
+
/** Most the live heap may grow while one scheduled job runs on a day-sized fixture. */
|
|
6
|
+
heapGrowthBytes: 40 * 1024 * 1024,
|
|
7
|
+
/** Most bytes of a large column one sized read returns; a single larger row is read alone. */
|
|
8
|
+
sizedReadBytes: 4 * 1024 * 1024,
|
|
9
|
+
/** Most stored events one long turn may write. */
|
|
10
|
+
turnEvents: 5e3,
|
|
11
|
+
/** A turn whose owner stopped reaches its terminal state within this long. */
|
|
12
|
+
settleWithinMs: 15 * 6e4,
|
|
13
|
+
/** Turn recovery and orphan settlement run at least this often. */
|
|
14
|
+
recoveryEveryMs: 15 * 6e4,
|
|
15
|
+
/** Longest a cached auth lookup may be served while the store cannot answer. */
|
|
16
|
+
authCacheTtlMs: 6e4,
|
|
17
|
+
/** A platform-limit alarm fires at this share of the limit. */
|
|
18
|
+
alarmRatio: 0.8,
|
|
19
|
+
/** A Worker isolate's memory. */
|
|
20
|
+
workerMemoryBytes: 128 * 1024 * 1024,
|
|
21
|
+
/** Rows one D1 query may read before an alarm; a sized read stays far below it (GTM's 1.63 M-event scan overloaded D1). */
|
|
22
|
+
d1RowsPerQuery: 25e4,
|
|
23
|
+
/** A workspace box's disk: the Sandbox SDK's default tier (10 GB). */
|
|
24
|
+
sandboxDiskBytes: 10 * 1024 ** 3,
|
|
25
|
+
/** Platform snapshots per box; the cap GTM hit on 2026-10-10. */
|
|
26
|
+
snapshotCount: 25,
|
|
27
|
+
/** Operator key requests per minute. */
|
|
28
|
+
keyRatePerMinute: 60,
|
|
29
|
+
/** Hours one resource's limit alarm waits before it repeats. */
|
|
30
|
+
alarmRepeatHours: 6,
|
|
31
|
+
/** Cloudflare's gzip limit on a Worker's Total Upload on the paid plan, in KiB; Legal deployed 9,659 KiB (94%) on 2026-10-19. */
|
|
32
|
+
workerUploadGzipKiB: 10 * 1024,
|
|
33
|
+
/** Cloudflare's raw limit on a Worker's Total Upload, in KiB. */
|
|
34
|
+
workerUploadRawKiB: 64 * 1024
|
|
35
|
+
};
|
|
36
|
+
var LAUNCH_INVARIANTS = [
|
|
37
|
+
{
|
|
38
|
+
id: "bounded-reads",
|
|
39
|
+
title: "Bounded reads",
|
|
40
|
+
rule: "No query in a scheduled job or report route returns more than 8 MB; large columns are read in sized batches of at most 4 MB.",
|
|
41
|
+
incident: "GTM turn health read 500 replies with their parts (122 MB of a 128 MB Worker); the reliability report scanned 1.63 M events and overloaded D1."
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
id: "memory-budget",
|
|
45
|
+
title: "Memory budget",
|
|
46
|
+
rule: "Every scheduled job runs on a day-sized fixture with live-heap growth of at most 40 MB, and every scheduled job is covered.",
|
|
47
|
+
incident: "GTM sweeps exceeded the Worker's 128 MB six times; a regex slice kept in a Map pinned whole tool outputs."
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
id: "isolated-jobs",
|
|
51
|
+
title: "Isolated scheduled jobs",
|
|
52
|
+
rule: "Each scheduled invocation runs exactly one job, and turn recovery and orphan settlement run at least every 15 minutes.",
|
|
53
|
+
incident: "At 21:00Z turn health ran out of memory while the hourly sweeps shared one run, and none of the eight queued after it ran."
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
id: "honest-settlement",
|
|
57
|
+
title: "Honest turn settlement",
|
|
58
|
+
rule: "Every turn reaches exactly one terminal state (an answer, a typed failure with Retry, or a Stop counted on its own) within 15 minutes of its owner stopping; failures are attributed to the request time, and a completed run with an aborted tool is a failure.",
|
|
59
|
+
incident: "Stale running rows (51 settled by hand), failure notices 8\u201311 h late, and completions with aborted tools counted as answers."
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
id: "coalesced-events",
|
|
63
|
+
title: "Coalesced turn events",
|
|
64
|
+
rule: "A recorded long turn stores at most 5,000 events: text and reasoning deltas are merged before storage.",
|
|
65
|
+
incident: "A reasoning model wrote 59,659 events in one turn and 1.63 M in a day."
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
id: "auth-survives-d1-stall",
|
|
69
|
+
title: "Auth survives a D1 stall",
|
|
70
|
+
rule: "A key or session verified within 60 s keeps working while D1 errors or stalls; the cache is used only then, never past revoke or expiry, and is keyed by SHA-256.",
|
|
71
|
+
incident: "An overloaded D1 failed sign-in and API-key checks for credentials that were valid moments before."
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
id: "authorize-before-stream",
|
|
75
|
+
title: "Authorization before any stream",
|
|
76
|
+
rule: "A refused request gets a plain response; no stream opens and no product state is touched before authorization resolves.",
|
|
77
|
+
incident: "agent-app 0.60.48\u20130.60.49 opened a progress-first stream before authorize ran."
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
id: "limit-alarms",
|
|
81
|
+
title: "Platform-limit alarms at 80%",
|
|
82
|
+
rule: "Worker memory, D1 rows per query, sandbox disk, snapshot count and key rate each have a declared limit and an alarm that fires at 80%.",
|
|
83
|
+
incident: "19 platform-limit events on 2026-10-10, each found after it failed a customer."
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
id: "pasted-code-reaches-chat",
|
|
87
|
+
title: "Pasted code reaches the chat",
|
|
88
|
+
rule: "An authenticated chat message carrying pasted code (shell, SQL, markup) gets the route's normal response; no app filter, sanitizer or edge rule refuses it.",
|
|
89
|
+
incident: "On 2026-10-11 a chat message containing `cat /etc/passwd` got Cloudflare's 403 block page from gtm.tangle.tools instead of reaching the Worker."
|
|
90
|
+
}
|
|
91
|
+
];
|
|
92
|
+
var LAUNCH_INVARIANT_IDS = LAUNCH_INVARIANTS.map((invariant) => invariant.id);
|
|
93
|
+
function describeVerdict(verdict) {
|
|
94
|
+
const head = `${verdict.pass ? "PASS" : "FAIL"} ${verdict.invariant} \u2014 ${verdict.subject}`;
|
|
95
|
+
return verdict.details.length ? `${head}
|
|
96
|
+
${verdict.details.join("\n ")}` : head;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export {
|
|
100
|
+
LAUNCH_BUDGETS,
|
|
101
|
+
LAUNCH_INVARIANTS,
|
|
102
|
+
LAUNCH_INVARIANT_IDS,
|
|
103
|
+
describeVerdict
|
|
104
|
+
};
|
|
105
|
+
//# sourceMappingURL=chunk-LRYY3OX7.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/launch-invariants/catalog.ts"],"sourcesContent":["/**\n * The launch invariants every agent app holds, and the budgets they are held to.\n *\n * Each one is a failure GTM shipped to production on 2026-10-10 and fixed by\n * hand. They are listed here once so every app checks the same rule with the\n * same numbers, and `agent-app-invariants` reports each by its id.\n */\n\n/** Budgets the invariants are measured against. */\nexport const LAUNCH_BUDGETS = {\n /** Most bytes one query may return in a scheduled job or report route: one sized batch with room for its listing. */\n queryBytes: 8 * 1024 * 1024,\n /** Most the live heap may grow while one scheduled job runs on a day-sized fixture. */\n heapGrowthBytes: 40 * 1024 * 1024,\n /** Most bytes of a large column one sized read returns; a single larger row is read alone. */\n sizedReadBytes: 4 * 1024 * 1024,\n /** Most stored events one long turn may write. */\n turnEvents: 5_000,\n /** A turn whose owner stopped reaches its terminal state within this long. */\n settleWithinMs: 15 * 60_000,\n /** Turn recovery and orphan settlement run at least this often. */\n recoveryEveryMs: 15 * 60_000,\n /** Longest a cached auth lookup may be served while the store cannot answer. */\n authCacheTtlMs: 60_000,\n /** A platform-limit alarm fires at this share of the limit. */\n alarmRatio: 0.8,\n /** A Worker isolate's memory. */\n workerMemoryBytes: 128 * 1024 * 1024,\n /** Rows one D1 query may read before an alarm; a sized read stays far below it (GTM's 1.63 M-event scan overloaded D1). */\n d1RowsPerQuery: 250_000,\n /** A workspace box's disk: the Sandbox SDK's default tier (10 GB). */\n sandboxDiskBytes: 10 * 1024 ** 3,\n /** Platform snapshots per box; the cap GTM hit on 2026-10-10. */\n snapshotCount: 25,\n /** Operator key requests per minute. */\n keyRatePerMinute: 60,\n /** Hours one resource's limit alarm waits before it repeats. */\n alarmRepeatHours: 6,\n /** Cloudflare's gzip limit on a Worker's Total Upload on the paid plan, in KiB; Legal deployed 9,659 KiB (94%) on 2026-10-19. */\n workerUploadGzipKiB: 10 * 1024,\n /** Cloudflare's raw limit on a Worker's Total Upload, in KiB. */\n workerUploadRawKiB: 64 * 1024,\n} as const\n\nexport type LaunchInvariantId =\n | 'bounded-reads'\n | 'memory-budget'\n | 'isolated-jobs'\n | 'honest-settlement'\n | 'coalesced-events'\n | 'auth-survives-d1-stall'\n | 'authorize-before-stream'\n | 'limit-alarms'\n | 'pasted-code-reaches-chat'\n\nexport interface LaunchInvariant {\n id: LaunchInvariantId\n title: string\n /** The rule, as the conformance report states it. */\n rule: string\n /** The production failure the rule answers. */\n incident: string\n}\n\nexport const LAUNCH_INVARIANTS: readonly LaunchInvariant[] = [\n {\n id: 'bounded-reads',\n title: 'Bounded reads',\n rule: 'No query in a scheduled job or report route returns more than 8 MB; large columns are read in sized batches of at most 4 MB.',\n incident: 'GTM turn health read 500 replies with their parts (122 MB of a 128 MB Worker); the reliability report scanned 1.63 M events and overloaded D1.',\n },\n {\n id: 'memory-budget',\n title: 'Memory budget',\n rule: 'Every scheduled job runs on a day-sized fixture with live-heap growth of at most 40 MB, and every scheduled job is covered.',\n incident: 'GTM sweeps exceeded the Worker\\'s 128 MB six times; a regex slice kept in a Map pinned whole tool outputs.',\n },\n {\n id: 'isolated-jobs',\n title: 'Isolated scheduled jobs',\n rule: 'Each scheduled invocation runs exactly one job, and turn recovery and orphan settlement run at least every 15 minutes.',\n incident: 'At 21:00Z turn health ran out of memory while the hourly sweeps shared one run, and none of the eight queued after it ran.',\n },\n {\n id: 'honest-settlement',\n title: 'Honest turn settlement',\n rule: 'Every turn reaches exactly one terminal state (an answer, a typed failure with Retry, or a Stop counted on its own) within 15 minutes of its owner stopping; failures are attributed to the request time, and a completed run with an aborted tool is a failure.',\n incident: 'Stale running rows (51 settled by hand), failure notices 8–11 h late, and completions with aborted tools counted as answers.',\n },\n {\n id: 'coalesced-events',\n title: 'Coalesced turn events',\n rule: 'A recorded long turn stores at most 5,000 events: text and reasoning deltas are merged before storage.',\n incident: 'A reasoning model wrote 59,659 events in one turn and 1.63 M in a day.',\n },\n {\n id: 'auth-survives-d1-stall',\n title: 'Auth survives a D1 stall',\n rule: 'A key or session verified within 60 s keeps working while D1 errors or stalls; the cache is used only then, never past revoke or expiry, and is keyed by SHA-256.',\n incident: 'An overloaded D1 failed sign-in and API-key checks for credentials that were valid moments before.',\n },\n {\n id: 'authorize-before-stream',\n title: 'Authorization before any stream',\n rule: 'A refused request gets a plain response; no stream opens and no product state is touched before authorization resolves.',\n incident: 'agent-app 0.60.48–0.60.49 opened a progress-first stream before authorize ran.',\n },\n {\n id: 'limit-alarms',\n title: 'Platform-limit alarms at 80%',\n rule: 'Worker memory, D1 rows per query, sandbox disk, snapshot count and key rate each have a declared limit and an alarm that fires at 80%.',\n incident: '19 platform-limit events on 2026-10-10, each found after it failed a customer.',\n },\n {\n id: 'pasted-code-reaches-chat',\n title: 'Pasted code reaches the chat',\n rule: 'An authenticated chat message carrying pasted code (shell, SQL, markup) gets the route\\'s normal response; no app filter, sanitizer or edge rule refuses it.',\n incident: 'On 2026-10-11 a chat message containing `cat /etc/passwd` got Cloudflare\\'s 403 block page from gtm.tangle.tools instead of reaching the Worker.',\n },\n]\n\n/** The invariant ids in report order. */\nexport const LAUNCH_INVARIANT_IDS: readonly LaunchInvariantId[] = LAUNCH_INVARIANTS.map((invariant) => invariant.id)\n\n/** One checked outcome of one invariant, for one subject (a job, a route, a scenario). */\nexport interface InvariantVerdict {\n invariant: LaunchInvariantId\n /** What was checked: a job name, a route, a scenario. */\n subject: string\n pass: boolean\n /** Findings, one line each. On a pass, what was proven. */\n details: string[]\n /** Measurements behind the verdict, for the report. */\n data?: Record<string, unknown>\n /**\n * Set when the app recorded this failure as known, with why and who owns the\n * fix. The report counts it as not holding; it does not fail the suite.\n */\n knownFailing?: string\n}\n\n/** A verdict whose message names every finding, for a test runner's failure output. */\nexport function describeVerdict(verdict: InvariantVerdict): string {\n const head = `${verdict.pass ? 'PASS' : 'FAIL'} ${verdict.invariant} — ${verdict.subject}`\n return verdict.details.length ? `${head}\\n ${verdict.details.join('\\n ')}` : head\n}\n"],"mappings":";AASO,IAAM,iBAAiB;AAAA;AAAA,EAE5B,YAAY,IAAI,OAAO;AAAA;AAAA,EAEvB,iBAAiB,KAAK,OAAO;AAAA;AAAA,EAE7B,gBAAgB,IAAI,OAAO;AAAA;AAAA,EAE3B,YAAY;AAAA;AAAA,EAEZ,gBAAgB,KAAK;AAAA;AAAA,EAErB,iBAAiB,KAAK;AAAA;AAAA,EAEtB,gBAAgB;AAAA;AAAA,EAEhB,YAAY;AAAA;AAAA,EAEZ,mBAAmB,MAAM,OAAO;AAAA;AAAA,EAEhC,gBAAgB;AAAA;AAAA,EAEhB,kBAAkB,KAAK,QAAQ;AAAA;AAAA,EAE/B,eAAe;AAAA;AAAA,EAEf,kBAAkB;AAAA;AAAA,EAElB,kBAAkB;AAAA;AAAA,EAElB,qBAAqB,KAAK;AAAA;AAAA,EAE1B,oBAAoB,KAAK;AAC3B;AAsBO,IAAM,oBAAgD;AAAA,EAC3D;AAAA,IACE,IAAI;AAAA,IACJ,OAAO;AAAA,IACP,MAAM;AAAA,IACN,UAAU;AAAA,EACZ;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO;AAAA,IACP,MAAM;AAAA,IACN,UAAU;AAAA,EACZ;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO;AAAA,IACP,MAAM;AAAA,IACN,UAAU;AAAA,EACZ;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO;AAAA,IACP,MAAM;AAAA,IACN,UAAU;AAAA,EACZ;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO;AAAA,IACP,MAAM;AAAA,IACN,UAAU;AAAA,EACZ;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO;AAAA,IACP,MAAM;AAAA,IACN,UAAU;AAAA,EACZ;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO;AAAA,IACP,MAAM;AAAA,IACN,UAAU;AAAA,EACZ;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO;AAAA,IACP,MAAM;AAAA,IACN,UAAU;AAAA,EACZ;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO;AAAA,IACP,MAAM;AAAA,IACN,UAAU;AAAA,EACZ;AACF;AAGO,IAAM,uBAAqD,kBAAkB,IAAI,CAAC,cAAc,UAAU,EAAE;AAoB5G,SAAS,gBAAgB,SAAmC;AACjE,QAAM,OAAO,GAAG,QAAQ,OAAO,SAAS,MAAM,IAAI,QAAQ,SAAS,WAAM,QAAQ,OAAO;AACxF,SAAO,QAAQ,QAAQ,SAAS,GAAG,IAAI;AAAA,IAAO,QAAQ,QAAQ,KAAK,MAAM,CAAC,KAAK;AACjF;","names":[]}
|
|
@@ -1,84 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
/** A turn whose owner stopped reaches its terminal state within this long. */
|
|
12
|
-
settleWithinMs: 15 * 6e4,
|
|
13
|
-
/** Turn recovery and orphan settlement run at least this often. */
|
|
14
|
-
recoveryEveryMs: 15 * 6e4,
|
|
15
|
-
/** Longest a cached auth lookup may be served while the store cannot answer. */
|
|
16
|
-
authCacheTtlMs: 6e4,
|
|
17
|
-
/** A platform-limit alarm fires at this share of the limit. */
|
|
18
|
-
alarmRatio: 0.8
|
|
19
|
-
};
|
|
20
|
-
var LAUNCH_INVARIANTS = [
|
|
21
|
-
{
|
|
22
|
-
id: "bounded-reads",
|
|
23
|
-
title: "Bounded reads",
|
|
24
|
-
rule: "No query in a scheduled job or report route returns more than 8 MB; large columns are read in sized batches of at most 4 MB.",
|
|
25
|
-
incident: "GTM turn health read 500 replies with their parts (122 MB of a 128 MB Worker); the reliability report scanned 1.63 M events and overloaded D1."
|
|
26
|
-
},
|
|
27
|
-
{
|
|
28
|
-
id: "memory-budget",
|
|
29
|
-
title: "Memory budget",
|
|
30
|
-
rule: "Every scheduled job runs on a day-sized fixture with live-heap growth of at most 40 MB, and every scheduled job is covered.",
|
|
31
|
-
incident: "GTM sweeps exceeded the Worker's 128 MB six times; a regex slice kept in a Map pinned whole tool outputs."
|
|
32
|
-
},
|
|
33
|
-
{
|
|
34
|
-
id: "isolated-jobs",
|
|
35
|
-
title: "Isolated scheduled jobs",
|
|
36
|
-
rule: "Each scheduled invocation runs exactly one job, and turn recovery and orphan settlement run at least every 15 minutes.",
|
|
37
|
-
incident: "At 21:00Z turn health ran out of memory while the hourly sweeps shared one run, and none of the eight queued after it ran."
|
|
38
|
-
},
|
|
39
|
-
{
|
|
40
|
-
id: "honest-settlement",
|
|
41
|
-
title: "Honest turn settlement",
|
|
42
|
-
rule: "Every turn reaches exactly one terminal state (an answer, a typed failure with Retry, or a Stop counted on its own) within 15 minutes of its owner stopping; failures are attributed to the request time, and a completed run with an aborted tool is a failure.",
|
|
43
|
-
incident: "Stale running rows (51 settled by hand), failure notices 8\u201311 h late, and completions with aborted tools counted as answers."
|
|
44
|
-
},
|
|
45
|
-
{
|
|
46
|
-
id: "coalesced-events",
|
|
47
|
-
title: "Coalesced turn events",
|
|
48
|
-
rule: "A recorded long turn stores at most 5,000 events: text and reasoning deltas are merged before storage.",
|
|
49
|
-
incident: "A reasoning model wrote 59,659 events in one turn and 1.63 M in a day."
|
|
50
|
-
},
|
|
51
|
-
{
|
|
52
|
-
id: "auth-survives-d1-stall",
|
|
53
|
-
title: "Auth survives a D1 stall",
|
|
54
|
-
rule: "A key or session verified within 60 s keeps working while D1 errors or stalls; the cache is used only then, never past revoke or expiry, and is keyed by SHA-256.",
|
|
55
|
-
incident: "An overloaded D1 failed sign-in and API-key checks for credentials that were valid moments before."
|
|
56
|
-
},
|
|
57
|
-
{
|
|
58
|
-
id: "authorize-before-stream",
|
|
59
|
-
title: "Authorization before any stream",
|
|
60
|
-
rule: "A refused request gets a plain response; no stream opens and no product state is touched before authorization resolves.",
|
|
61
|
-
incident: "agent-app 0.60.48\u20130.60.49 opened a progress-first stream before authorize ran."
|
|
62
|
-
},
|
|
63
|
-
{
|
|
64
|
-
id: "limit-alarms",
|
|
65
|
-
title: "Platform-limit alarms at 80%",
|
|
66
|
-
rule: "Worker memory, D1 rows per query, sandbox disk, snapshot count and key rate each have a declared limit and an alarm that fires at 80%.",
|
|
67
|
-
incident: "19 platform-limit events on 2026-10-10, each found after it failed a customer."
|
|
68
|
-
},
|
|
69
|
-
{
|
|
70
|
-
id: "pasted-code-reaches-chat",
|
|
71
|
-
title: "Pasted code reaches the chat",
|
|
72
|
-
rule: "An authenticated chat message carrying pasted code (shell, SQL, markup) gets the route's normal response; no app filter, sanitizer or edge rule refuses it.",
|
|
73
|
-
incident: "On 2026-10-11 a chat message containing `cat /etc/passwd` got Cloudflare's 403 block page from gtm.tangle.tools instead of reaching the Worker."
|
|
74
|
-
}
|
|
75
|
-
];
|
|
76
|
-
var LAUNCH_INVARIANT_IDS = LAUNCH_INVARIANTS.map((invariant) => invariant.id);
|
|
77
|
-
function describeVerdict(verdict) {
|
|
78
|
-
const head = `${verdict.pass ? "PASS" : "FAIL"} ${verdict.invariant} \u2014 ${verdict.subject}`;
|
|
79
|
-
return verdict.details.length ? `${head}
|
|
80
|
-
${verdict.details.join("\n ")}` : head;
|
|
81
|
-
}
|
|
1
|
+
import {
|
|
2
|
+
LAUNCH_BUDGETS
|
|
3
|
+
} from "./chunk-LRYY3OX7.js";
|
|
4
|
+
import {
|
|
5
|
+
createConsoleAlertSink,
|
|
6
|
+
createMemoryThrottleStore,
|
|
7
|
+
createMultiAlertSink,
|
|
8
|
+
createThrottledAlertSink,
|
|
9
|
+
createWebhookAlertSink
|
|
10
|
+
} from "./chunk-CD2LQXY6.js";
|
|
82
11
|
|
|
83
12
|
// src/launch-invariants/schedule.ts
|
|
84
13
|
function createScheduledDispatch(options) {
|
|
@@ -397,6 +326,20 @@ function formatAmount(value, resource) {
|
|
|
397
326
|
if (UNITS[resource] === "bytes") return `${(value / (1024 * 1024)).toFixed(1)} MB`;
|
|
398
327
|
return Math.round(value).toLocaleString("en-US");
|
|
399
328
|
}
|
|
329
|
+
var DEFAULT_LIMIT_BUDGETS = {
|
|
330
|
+
"worker-memory": LAUNCH_BUDGETS.workerMemoryBytes,
|
|
331
|
+
"d1-rows-per-query": LAUNCH_BUDGETS.d1RowsPerQuery,
|
|
332
|
+
"sandbox-disk": LAUNCH_BUDGETS.sandboxDiskBytes,
|
|
333
|
+
"snapshot-count": LAUNCH_BUDGETS.snapshotCount,
|
|
334
|
+
"key-rate": LAUNCH_BUDGETS.keyRatePerMinute
|
|
335
|
+
};
|
|
336
|
+
function createProductLimitAlarms(options) {
|
|
337
|
+
const sink = options.sink ?? createThrottledAlertSink(createMultiAlertSink([
|
|
338
|
+
createConsoleAlertSink((message) => console.error(message)),
|
|
339
|
+
...options.webhookUrl ? [createWebhookAlertSink({ webhookUrl: options.webhookUrl })] : []
|
|
340
|
+
]), { store: createMemoryThrottleStore(), windowMs: LAUNCH_BUDGETS.alarmRepeatHours * 60 * 60 * 1e3 });
|
|
341
|
+
return createLimitAlarms({ product: options.product, budgets: { ...DEFAULT_LIMIT_BUDGETS, ...options.overrides }, sink });
|
|
342
|
+
}
|
|
400
343
|
function withD1LimitAlarms(d1, alarms, options = {}) {
|
|
401
344
|
const observe = (sql, result) => {
|
|
402
345
|
const meta = result?.meta;
|
|
@@ -477,10 +420,6 @@ async function isEdgeBlock(response) {
|
|
|
477
420
|
}
|
|
478
421
|
|
|
479
422
|
export {
|
|
480
|
-
LAUNCH_BUDGETS,
|
|
481
|
-
LAUNCH_INVARIANTS,
|
|
482
|
-
LAUNCH_INVARIANT_IDS,
|
|
483
|
-
describeVerdict,
|
|
484
423
|
createScheduledDispatch,
|
|
485
424
|
slotsOf,
|
|
486
425
|
cronDailyFirings,
|
|
@@ -489,9 +428,11 @@ export {
|
|
|
489
428
|
checkIsolatedJobs,
|
|
490
429
|
LIMIT_RESOURCES,
|
|
491
430
|
createLimitAlarms,
|
|
431
|
+
DEFAULT_LIMIT_BUDGETS,
|
|
432
|
+
createProductLimitAlarms,
|
|
492
433
|
withD1LimitAlarms,
|
|
493
434
|
estimatedBytes,
|
|
494
435
|
PASTED_CODE_SAMPLES,
|
|
495
436
|
isEdgeBlock
|
|
496
437
|
};
|
|
497
|
-
//# sourceMappingURL=chunk-
|
|
438
|
+
//# sourceMappingURL=chunk-QM7RTTDY.js.map
|