@tangle-network/agent-app 0.60.62 → 0.60.64

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/dist/{DesignCanvasEditor-3DL3PF6P.js → DesignCanvasEditor-UOOPYYYR.js} +3 -3
  2. package/dist/VaultPane-XUAGL5SP.js +7 -0
  3. package/dist/agent-profiles/d1.js +3 -3
  4. package/dist/agent-profiles/index.js +3 -3
  5. package/dist/app-auth/index.js +3 -3
  6. package/dist/assistant/index.js +11 -10
  7. package/dist/assistant/index.js.map +1 -1
  8. package/dist/chat-react/index.js +1 -1
  9. package/dist/chat-routes/index.js +16 -16
  10. package/dist/chat-store/index.js +3 -3
  11. package/dist/{chunk-FAU5FIFD.js → chunk-2FLSODOX.js} +3 -2
  12. package/dist/chunk-2FLSODOX.js.map +1 -0
  13. package/dist/chunk-5UQJ7F3P.js +41 -0
  14. package/dist/chunk-5UQJ7F3P.js.map +1 -0
  15. package/dist/{chunk-YXYQ6MJL.js → chunk-5V6VTO5Q.js} +65 -2
  16. package/dist/chunk-5V6VTO5Q.js.map +1 -0
  17. package/dist/chunk-7UB4K2R2.js +470 -0
  18. package/dist/chunk-7UB4K2R2.js.map +1 -0
  19. package/dist/{chunk-T2IESIJ7.js → chunk-BMZTYGKE.js} +2 -2
  20. package/dist/{chunk-2YLSWZ5W.js → chunk-JBEFTXDO.js} +4 -4
  21. package/dist/{chunk-JGZ7WEPL.js → chunk-JC7YSRBR.js} +15 -1
  22. package/dist/chunk-JC7YSRBR.js.map +1 -0
  23. package/dist/{chunk-N7ZJWESQ.js → chunk-KBTY4AAU.js} +274 -4
  24. package/dist/chunk-KBTY4AAU.js.map +1 -0
  25. package/dist/{chunk-S2G5NAWV.js → chunk-KSITU6BQ.js} +4 -4
  26. package/dist/{chunk-CN5HNTU6.js → chunk-NRG2YHGY.js} +2 -2
  27. package/dist/{chunk-GNA5I5NM.js → chunk-ORFRFCXQ.js} +53 -22
  28. package/dist/chunk-ORFRFCXQ.js.map +1 -0
  29. package/dist/{chunk-4WJREKXX.js → chunk-OUW5AI2G.js} +9 -9
  30. package/dist/chunk-OUW5AI2G.js.map +1 -0
  31. package/dist/{chunk-M36GXE2W.js → chunk-QXKKS3I4.js} +3 -3
  32. package/dist/chunk-RYUTKTOR.js +510 -0
  33. package/dist/chunk-RYUTKTOR.js.map +1 -0
  34. package/dist/{chunk-FWLCPAST.js → chunk-SH56N4PV.js} +3 -3
  35. package/dist/{chunk-QBNV6BJ4.js → chunk-ST3ARHJH.js} +3 -2
  36. package/dist/chunk-ST3ARHJH.js.map +1 -0
  37. package/dist/{chunk-4DIO2LTR.js → chunk-TMOYOMSW.js} +5 -5
  38. package/dist/{chunk-4DIO2LTR.js.map → chunk-TMOYOMSW.js.map} +1 -1
  39. package/dist/{chunk-J7TCMW2J.js → chunk-XMIV374H.js} +2 -2
  40. package/dist/{chunk-ZGUXI5SI.js → chunk-XNKLQG26.js} +5 -5
  41. package/dist/{chunk-XCBDWT5H.js → chunk-Z362IHYL.js} +2 -2
  42. package/dist/{chunk-VUXCM5ZO.js → chunk-ZZACC24K.js} +1 -1
  43. package/dist/{chunk-VUXCM5ZO.js.map → chunk-ZZACC24K.js.map} +1 -1
  44. package/dist/design-canvas-react/index.js +6 -6
  45. package/dist/design-canvas-react/lazy.js +1 -1
  46. package/dist/launch-invariants/auth-lookup-cache.d.ts +57 -0
  47. package/dist/launch-invariants/catalog.d.ts +51 -0
  48. package/dist/launch-invariants/cli.d.ts +10 -0
  49. package/dist/launch-invariants/cli.js +88 -0
  50. package/dist/launch-invariants/cli.js.map +1 -0
  51. package/dist/launch-invariants/conformance.d.ts +49 -0
  52. package/dist/launch-invariants/index.d.ts +19 -0
  53. package/dist/launch-invariants/index.js +253 -0
  54. package/dist/launch-invariants/index.js.map +1 -0
  55. package/dist/launch-invariants/limit-alarms.d.ts +66 -0
  56. package/dist/launch-invariants/schedule.d.ts +114 -0
  57. package/dist/launch-invariants/settlement.d.ts +90 -0
  58. package/dist/launch-invariants/sized-reads.d.ts +50 -0
  59. package/dist/launch-invariants/testing.d.ts +175 -0
  60. package/dist/launch-invariants/testing.js +45 -0
  61. package/dist/launch-invariants/testing.js.map +1 -0
  62. package/dist/preset-cloudflare/index.js +2 -2
  63. package/dist/public-consultation/index.js +7 -7
  64. package/dist/sandbox/index.js +4 -4
  65. package/dist/signoff/cli.js +7 -2
  66. package/dist/signoff/cli.js.map +1 -1
  67. package/dist/signoff/index.d.ts +2 -0
  68. package/dist/signoff/index.js +15 -1
  69. package/dist/signoff/shell-alignment-cli.d.ts +8 -0
  70. package/dist/signoff/shell-alignment.d.ts +82 -0
  71. package/dist/stream/index.d.ts +1 -0
  72. package/dist/stream/index.js +9 -5
  73. package/dist/stream/orphaned-turns.d.ts +56 -0
  74. package/dist/stream/turn-buffer.d.ts +12 -0
  75. package/dist/studio-react/index.js +1 -1
  76. package/dist/teams/invitations-api.js +3 -3
  77. package/dist/theme/brand.generated.d.ts +1 -1
  78. package/dist/theme/index.js +1 -1
  79. package/dist/theme/tokens.css +7 -2
  80. package/dist/tools/index.js +4 -4
  81. package/dist/turn-health/index.js +6 -17
  82. package/dist/turn-health/index.js.map +1 -1
  83. package/dist/turn-stream/index.js +1 -1
  84. package/dist/vault/index.js +1 -1
  85. package/dist/vault/lazy.js +1 -1
  86. package/dist/web-react/index.d.ts +1 -1
  87. package/dist/web-react/index.js +19 -15
  88. package/dist/web-react/lazy-load-boundary.d.ts +19 -1
  89. package/dist/work-product/index.js +2 -2
  90. package/dist/workspace-export/index.js +30 -9
  91. package/dist/workspace-export/index.js.map +1 -1
  92. package/dist/workspace-export/react.js +14 -5
  93. package/dist/workspace-export/react.js.map +1 -1
  94. package/package.json +23 -12
  95. package/dist/VaultPane-A2RGT5PI.js +0 -7
  96. package/dist/chunk-4WJREKXX.js.map +0 -1
  97. package/dist/chunk-FAU5FIFD.js.map +0 -1
  98. package/dist/chunk-GNA5I5NM.js.map +0 -1
  99. package/dist/chunk-JGZ7WEPL.js.map +0 -1
  100. package/dist/chunk-N7ZJWESQ.js.map +0 -1
  101. package/dist/chunk-QBNV6BJ4.js.map +0 -1
  102. package/dist/chunk-YXYQ6MJL.js.map +0 -1
  103. /package/dist/{DesignCanvasEditor-3DL3PF6P.js.map → DesignCanvasEditor-UOOPYYYR.js.map} +0 -0
  104. /package/dist/{VaultPane-A2RGT5PI.js.map → VaultPane-XUAGL5SP.js.map} +0 -0
  105. /package/dist/{chunk-T2IESIJ7.js.map → chunk-BMZTYGKE.js.map} +0 -0
  106. /package/dist/{chunk-2YLSWZ5W.js.map → chunk-JBEFTXDO.js.map} +0 -0
  107. /package/dist/{chunk-S2G5NAWV.js.map → chunk-KSITU6BQ.js.map} +0 -0
  108. /package/dist/{chunk-CN5HNTU6.js.map → chunk-NRG2YHGY.js.map} +0 -0
  109. /package/dist/{chunk-M36GXE2W.js.map → chunk-QXKKS3I4.js.map} +0 -0
  110. /package/dist/{chunk-FWLCPAST.js.map → chunk-SH56N4PV.js.map} +0 -0
  111. /package/dist/{chunk-J7TCMW2J.js.map → chunk-XMIV374H.js.map} +0 -0
  112. /package/dist/{chunk-ZGUXI5SI.js.map → chunk-XNKLQG26.js.map} +0 -0
  113. /package/dist/{chunk-XCBDWT5H.js.map → chunk-Z362IHYL.js.map} +0 -0
@@ -0,0 +1,253 @@
1
+ import {
2
+ LAUNCH_BUDGETS,
3
+ LAUNCH_INVARIANTS,
4
+ LAUNCH_INVARIANT_IDS,
5
+ LIMIT_RESOURCES,
6
+ checkIsolatedJobs,
7
+ createLimitAlarms,
8
+ createScheduledDispatch,
9
+ cronDailyFirings,
10
+ describeVerdict,
11
+ estimatedBytes,
12
+ longestGapMinutes,
13
+ parseWranglerCrons,
14
+ slotsOf,
15
+ withD1LimitAlarms
16
+ } from "../chunk-7UB4K2R2.js";
17
+ import {
18
+ D1_MAX_BOUND_PARAMETERS,
19
+ SIZED_READ_BYTES,
20
+ SIZED_READ_COMPOSITE_ROWS,
21
+ SIZED_READ_ROWS,
22
+ readInSizedBatches,
23
+ sizedBatches
24
+ } from "../chunk-5UQJ7F3P.js";
25
+
26
+ // src/launch-invariants/auth-lookup-cache.ts
27
+ var AUTH_LOOKUP_TTL_MS = 6e4;
28
+ var AUTH_LOOKUP_DEADLINE_MS = 2500;
29
+ var MAX_ENTRIES = 2e3;
30
+ var AuthLookupRefused = class extends Error {
31
+ };
32
+ function createAuthLookupCache(options) {
33
+ const ttlMs = Math.min(options.ttlMs ?? AUTH_LOOKUP_TTL_MS, AUTH_LOOKUP_TTL_MS);
34
+ const deadlineMs = options.deadlineMs ?? AUTH_LOOKUP_DEADLINE_MS;
35
+ const now = options.now ?? (() => Date.now());
36
+ const warn = options.warn ?? ((message, detail) => console.warn(message, detail));
37
+ const entries = /* @__PURE__ */ new Map();
38
+ return {
39
+ async lookup(key, read, lookupOptions) {
40
+ let value;
41
+ try {
42
+ value = await withDeadline(read(), deadlineMs);
43
+ } catch (error) {
44
+ if (error instanceof AuthLookupRefused || error instanceof Response) {
45
+ entries.delete(key);
46
+ throw error;
47
+ }
48
+ const entry = entries.get(key);
49
+ if (entry && now() < entry.until) {
50
+ warn(`[auth-cache] ${options.label}: served the last valid result while the store was unavailable`, {
51
+ error: error instanceof Error ? error.message : String(error),
52
+ remainingMs: entry.until - now()
53
+ });
54
+ return entry.value;
55
+ }
56
+ entries.delete(key);
57
+ throw error;
58
+ }
59
+ if (value === null) {
60
+ entries.delete(key);
61
+ return null;
62
+ }
63
+ const expiresAt = lookupOptions?.expiresAt?.(value);
64
+ const until = Math.min(now() + ttlMs, typeof expiresAt === "number" && expiresAt > 0 ? expiresAt : Number.POSITIVE_INFINITY);
65
+ entries.delete(key);
66
+ entries.set(key, { value, until });
67
+ if (entries.size > MAX_ENTRIES) entries.delete(entries.keys().next().value);
68
+ return value;
69
+ },
70
+ invalidate(matches) {
71
+ for (const [key, entry] of entries) if (matches(entry.value)) entries.delete(key);
72
+ },
73
+ forget(key) {
74
+ entries.delete(key);
75
+ },
76
+ clear() {
77
+ entries.clear();
78
+ },
79
+ keys() {
80
+ return [...entries.keys()];
81
+ }
82
+ };
83
+ }
84
+ async function credentialCacheKey(secret) {
85
+ const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(secret));
86
+ return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
87
+ }
88
+ function withDeadline(promise, ms) {
89
+ let timer;
90
+ const deadline = new Promise((_, reject) => {
91
+ timer = setTimeout(() => reject(new Error(`auth lookup did not answer within ${ms} ms`)), ms);
92
+ });
93
+ return Promise.race([promise, deadline]).finally(() => clearTimeout(timer));
94
+ }
95
+
96
+ // src/launch-invariants/settlement.ts
97
+ function checkTurnSettlement(turns, options) {
98
+ const within = options.settleWithinMs ?? LAUNCH_BUDGETS.settleWithinMs;
99
+ const findings = [];
100
+ const counts = { turns: turns.length, answered: 0, failed: 0, stopped: 0, open: 0, maxDetectionLagMinutes: 0 };
101
+ for (const turn of turns) {
102
+ const label = `turn ${turn.turnId}`;
103
+ if (turn.terminals.length > 1) {
104
+ findings.push(`${label} has ${turn.terminals.length} terminal states (${turn.terminals.map((terminal2) => terminal2.state).join(" then ")}); a turn ends once`);
105
+ }
106
+ const terminal = turn.terminals[0];
107
+ if (!terminal) {
108
+ counts.open += 1;
109
+ if (turn.ownerEndedAt !== null && options.now - turn.ownerEndedAt > within) {
110
+ findings.push(`${label} is still open ${minutes(options.now - turn.ownerEndedAt)} min after its owner stopped; the most is ${minutes(within)}`);
111
+ }
112
+ continue;
113
+ }
114
+ counts[terminal.state] += 1;
115
+ if (turn.ownerEndedAt !== null) {
116
+ const lag = terminal.at - turn.ownerEndedAt;
117
+ counts.maxDetectionLagMinutes = Math.max(counts.maxDetectionLagMinutes, minutes(lag));
118
+ if (lag > within) findings.push(`${label} was settled ${minutes(lag)} min after its owner stopped; the most is ${minutes(within)}`);
119
+ }
120
+ if (terminal.state === "answered" && turn.abortedToolCall) {
121
+ findings.push(`${label} is counted as an answer, but the Sandbox aborted a tool call in it; it stopped mid-work and is a failure`);
122
+ }
123
+ if (terminal.state === "failed") {
124
+ if (!terminal.code) findings.push(`${label} failed without a typed failure code`);
125
+ if (terminal.retryable !== true) findings.push(`${label} failed without offering Retry`);
126
+ if (terminal.attributedTo !== void 0 && terminal.attributedTo !== turn.requestedAt) {
127
+ findings.push(`${label}'s failure is reported at ${new Date(terminal.attributedTo).toISOString()}, not its request time ${new Date(turn.requestedAt).toISOString()}`);
128
+ }
129
+ }
130
+ if (terminal.state === "stopped" && terminal.code && /fail|error/i.test(terminal.code)) {
131
+ findings.push(`${label} was stopped by the person but carries failure code ${terminal.code}; a Stop is counted on its own`);
132
+ }
133
+ }
134
+ return {
135
+ invariant: "honest-settlement",
136
+ subject: options.subject ?? "turns",
137
+ pass: findings.length === 0,
138
+ details: findings.length ? findings : [
139
+ `${counts.turns} turns: ${counts.answered} answered, ${counts.failed} failed with Retry, ${counts.stopped} stopped, ${counts.open} held by a live owner`,
140
+ `longest detection lag ${counts.maxDetectionLagMinutes} min`
141
+ ],
142
+ data: { ...counts }
143
+ };
144
+ }
145
+ function minutes(ms) {
146
+ return Math.round(ms / 6e4 * 10) / 10;
147
+ }
148
+ var SETTLEMENT_SCENARIOS = [
149
+ {
150
+ id: "orphaned-stream",
151
+ story: "The Worker streaming the turn died; its stream row is still running, no admission or Workflow holds it, and its 5-minute lease expired 12 minutes ago.",
152
+ requestedMinutesAgo: 30,
153
+ ownerEndedMinutesAgo: 12,
154
+ expect: "failed"
155
+ },
156
+ {
157
+ id: "abandoned-before-admission",
158
+ story: "The user message was saved 12 minutes ago, then the request died before any completion owner admitted the turn: no reply, no stream, no admission.",
159
+ requestedMinutesAgo: 12,
160
+ ownerEndedMinutesAgo: 12,
161
+ expect: "failed"
162
+ },
163
+ {
164
+ id: "runtime-restarted",
165
+ story: "The admitted run's runtime restarted mid-turn 12 minutes ago: the session still reports running while its execution ledger failed the run (terminal event interrupted-done).",
166
+ requestedMinutesAgo: 40,
167
+ ownerEndedMinutesAgo: 12,
168
+ expect: "failed"
169
+ },
170
+ {
171
+ id: "completed-with-aborted-tool",
172
+ story: 'The run reported completed 12 minutes ago, but a tool part ended with "Tool execution aborted".',
173
+ requestedMinutesAgo: 20,
174
+ ownerEndedMinutesAgo: 12,
175
+ expect: "failed"
176
+ },
177
+ {
178
+ id: "user-stop",
179
+ story: "The person pressed Stop 12 minutes ago while the turn ran.",
180
+ requestedMinutesAgo: 20,
181
+ ownerEndedMinutesAgo: 12,
182
+ expect: "stopped"
183
+ },
184
+ {
185
+ id: "answered",
186
+ story: "The turn answered normally 12 minutes ago.",
187
+ requestedMinutesAgo: 20,
188
+ ownerEndedMinutesAgo: 12,
189
+ expect: "answered"
190
+ },
191
+ {
192
+ id: "live-long-turn",
193
+ story: "A long turn is still running under an owner that renews its lease.",
194
+ requestedMinutesAgo: 90,
195
+ ownerEndedMinutesAgo: null,
196
+ expect: "open"
197
+ }
198
+ ];
199
+ function checkSettlementScenarios(records, options) {
200
+ const scenarios = options.scenarios ?? SETTLEMENT_SCENARIOS;
201
+ const findings = [];
202
+ const present = [];
203
+ for (const scenario of scenarios) {
204
+ const record = records[scenario.id];
205
+ if (!record) {
206
+ findings.push(`scenario ${scenario.id}: no turn record was read back`);
207
+ continue;
208
+ }
209
+ present.push(record);
210
+ const state = record.terminals[0]?.state ?? "open";
211
+ if (state !== scenario.expect) findings.push(`scenario ${scenario.id}: ended ${state}, expected ${scenario.expect} (${scenario.story})`);
212
+ }
213
+ const honest = checkTurnSettlement(present, { now: options.now, subject: "settlement scenarios" });
214
+ if (!honest.pass) findings.push(...honest.details);
215
+ return {
216
+ invariant: "honest-settlement",
217
+ subject: "settlement scenarios",
218
+ pass: findings.length === 0,
219
+ details: findings.length ? findings : [`${scenarios.length} failure scenarios settle to their honest state`, ...honest.details],
220
+ data: honest.data
221
+ };
222
+ }
223
+ export {
224
+ AUTH_LOOKUP_DEADLINE_MS,
225
+ AUTH_LOOKUP_TTL_MS,
226
+ AuthLookupRefused,
227
+ D1_MAX_BOUND_PARAMETERS,
228
+ LAUNCH_BUDGETS,
229
+ LAUNCH_INVARIANTS,
230
+ LAUNCH_INVARIANT_IDS,
231
+ LIMIT_RESOURCES,
232
+ SETTLEMENT_SCENARIOS,
233
+ SIZED_READ_BYTES,
234
+ SIZED_READ_COMPOSITE_ROWS,
235
+ SIZED_READ_ROWS,
236
+ checkIsolatedJobs,
237
+ checkSettlementScenarios,
238
+ checkTurnSettlement,
239
+ createAuthLookupCache,
240
+ createLimitAlarms,
241
+ createScheduledDispatch,
242
+ credentialCacheKey,
243
+ cronDailyFirings,
244
+ describeVerdict,
245
+ estimatedBytes,
246
+ longestGapMinutes,
247
+ parseWranglerCrons,
248
+ readInSizedBatches,
249
+ sizedBatches,
250
+ slotsOf,
251
+ withD1LimitAlarms
252
+ };
253
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../../src/launch-invariants/auth-lookup-cache.ts","../../src/launch-invariants/settlement.ts"],"sourcesContent":["/**\n * The last valid result of each auth lookup (API key, key owner, browser\n * session), served only while the store cannot answer.\n *\n * Every lookup reads the store first, so a revoked or expired credential is\n * refused as soon as the store says so, and that answer drops its entry. When\n * the store errors or stalls past the deadline, a valid result verified less\n * than the TTL ago is served instead: on 2026-10-10 an overloaded D1 failed\n * API-key checks and session reads for requests whose credentials were valid\n * moments before (GTM #1488). The TTL counts from the last successful store\n * read and a served entry never renews it, so a credential revoked during an\n * outage lives at most the TTL past its last verification. Entries are per\n * isolate; revoking or rotating a key must also drop its entry here.\n *\n * The cache serves the verified value itself, never a copy: GTM keyed a\n * WeakMap on the verified key object, a copy missed it, and operator requests\n * answered 401 (GTM #1492).\n */\n\n/** Longest a verified result may be served while the store cannot answer. A larger configured TTL is capped here. */\nexport const AUTH_LOOKUP_TTL_MS = 60_000\n/** A store read slower than this is treated as unavailable; a healthy D1 lookup answers in tens of milliseconds. */\nexport const AUTH_LOOKUP_DEADLINE_MS = 2_500\nconst MAX_ENTRIES = 2_000\n\n/** The store answered that the credential is not valid; never answered from the cache. */\nexport class AuthLookupRefused extends Error {}\n\ninterface Entry<V> {\n value: V\n until: number\n}\n\nexport interface AuthLookupOptions<V> {\n /** When the credential itself expires (Unix ms); the entry never outlives it. */\n expiresAt?: (value: V) => number | null | undefined\n}\n\nexport interface AuthLookupCache<V> {\n /**\n * Read through the store. `read` resolves the valid value or null, or throws\n * {@link AuthLookupRefused} (or a `Response`) when the store refuses; any\n * other failure, or no answer by the deadline, falls back to a fresh entry.\n * `key` must not be the secret: use {@link credentialCacheKey}.\n */\n lookup(key: string, read: () => Promise<V | null>, options?: AuthLookupOptions<V>): Promise<V | null>\n /** Drop every entry whose value matches, as when a key is revoked or rotated. */\n invalidate(matches: (value: V) => boolean): void\n /** Drop one entry by its key. */\n forget(key: string): void\n clear(): void\n /** The keys held, for a check that no secret is stored. */\n keys(): string[]\n}\n\nexport interface AuthLookupCacheOptions {\n /** Names the lookup in the warning logged when a cached result is served. */\n label: string\n /** Capped at {@link AUTH_LOOKUP_TTL_MS}. */\n ttlMs?: number\n deadlineMs?: number\n now?: () => number\n warn?: (message: string, detail: Record<string, unknown>) => void\n}\n\nexport function createAuthLookupCache<V>(options: AuthLookupCacheOptions): AuthLookupCache<V> {\n const ttlMs = Math.min(options.ttlMs ?? AUTH_LOOKUP_TTL_MS, AUTH_LOOKUP_TTL_MS)\n const deadlineMs = options.deadlineMs ?? AUTH_LOOKUP_DEADLINE_MS\n const now = options.now ?? (() => Date.now())\n const warn = options.warn ?? ((message, detail) => console.warn(message, detail))\n const entries = new Map<string, Entry<V>>()\n\n return {\n async lookup(key, read, lookupOptions) {\n let value: V | null\n try {\n value = await withDeadline(read(), deadlineMs)\n } catch (error) {\n if (error instanceof AuthLookupRefused || error instanceof Response) {\n entries.delete(key)\n throw error\n }\n const entry = entries.get(key)\n if (entry && now() < entry.until) {\n warn(`[auth-cache] ${options.label}: served the last valid result while the store was unavailable`, {\n error: error instanceof Error ? error.message : String(error),\n remainingMs: entry.until - now(),\n })\n return entry.value\n }\n entries.delete(key)\n throw error\n }\n if (value === null) {\n entries.delete(key)\n return null\n }\n const expiresAt = lookupOptions?.expiresAt?.(value)\n const until = Math.min(now() + ttlMs, typeof expiresAt === 'number' && expiresAt > 0 ? expiresAt : Number.POSITIVE_INFINITY)\n entries.delete(key)\n entries.set(key, { value, until })\n if (entries.size > MAX_ENTRIES) entries.delete(entries.keys().next().value!)\n return value\n },\n invalidate(matches) {\n for (const [key, entry] of entries) if (matches(entry.value)) entries.delete(key)\n },\n forget(key) {\n entries.delete(key)\n },\n clear() {\n entries.clear()\n },\n keys() {\n return [...entries.keys()]\n },\n }\n}\n\n/** A credential's cache key: its SHA-256 in hex, so the cache never holds the secret. */\nexport async function credentialCacheKey(secret: string): Promise<string> {\n const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(secret))\n return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, '0')).join('')\n}\n\nfunction withDeadline<T>(promise: Promise<T>, ms: number): Promise<T> {\n let timer: ReturnType<typeof setTimeout> | undefined\n const deadline = new Promise<never>((_, reject) => {\n timer = setTimeout(() => reject(new Error(`auth lookup did not answer within ${ms} ms`)), ms)\n })\n return Promise.race([promise, deadline]).finally(() => clearTimeout(timer))\n}\n","/**\n * Honest turn settlement, checked over a product's turn records.\n *\n * A turn ends in exactly one terminal state: an answer, a typed failure the\n * person can Retry, or a Stop the person chose, counted on its own. It gets\n * there within 15 minutes of its owner stopping. A failure belongs to the time\n * the turn was requested, not the time it was noticed, and how late it was\n * noticed is measured. A run that completed with a tool call the Sandbox\n * aborted stopped mid-work, so it is a failure.\n *\n * On 2026-10-10 GTM held turns in five places (turn status, admissions,\n * Workflows, Sandbox sessions and the sidecar tracker) and they disagreed:\n * 51 stream rows stayed `running` after their producers died, failure notices\n * arrived 8–11 hours late, and completions with aborted tools counted as\n * answers. {@link checkTurnSettlement} reads one normalized record per turn,\n * however the product derives it, and names every turn that breaks the rule.\n */\nimport { LAUNCH_BUDGETS, type InvariantVerdict } from './catalog.js'\n\nexport type TurnTerminalState = 'answered' | 'failed' | 'stopped'\n\n/** One turn as its product records it, normalized. Times are Unix ms. */\nexport interface SettledTurnRecord {\n turnId: string\n /** When the request was accepted (the user message was saved). */\n requestedAt: number\n /** Every terminal state recorded for the turn, in order; exactly one is honest. */\n terminals: ReadonlyArray<{\n state: TurnTerminalState\n /** When the terminal state was recorded. */\n at: number\n /** A typed failure code, for `failed`. */\n code?: string\n /** Whether the person is offered Retry, for `failed`. */\n retryable?: boolean\n /** The time the failure is reported under; it must be the request time. */\n attributedTo?: number\n }>\n /** When the turn's owner stopped: for a leased stream, when its lease expired. Null while an owner holds it. */\n ownerEndedAt: number | null\n /** A completed run in which the Sandbox aborted a tool call. */\n abortedToolCall?: boolean\n}\n\nexport interface TurnSettlementCounts {\n turns: number\n answered: number\n failed: number\n stopped: number\n open: number\n /** Most minutes between an owner stopping and its turn's terminal state. */\n maxDetectionLagMinutes: number\n}\n\n/**\n * Check turn records against honest settlement at `now`.\n * A turn still open whose owner has not stopped is fine; one whose owner stopped\n * more than {@link LAUNCH_BUDGETS.settleWithinMs} ago is not.\n */\nexport function checkTurnSettlement(\n turns: readonly SettledTurnRecord[],\n options: { now: number; settleWithinMs?: number; subject?: string },\n): InvariantVerdict {\n const within = options.settleWithinMs ?? LAUNCH_BUDGETS.settleWithinMs\n const findings: string[] = []\n const counts: TurnSettlementCounts = { turns: turns.length, answered: 0, failed: 0, stopped: 0, open: 0, maxDetectionLagMinutes: 0 }\n for (const turn of turns) {\n const label = `turn ${turn.turnId}`\n if (turn.terminals.length > 1) {\n findings.push(`${label} has ${turn.terminals.length} terminal states (${turn.terminals.map((terminal) => terminal.state).join(' then ')}); a turn ends once`)\n }\n const terminal = turn.terminals[0]\n if (!terminal) {\n counts.open += 1\n if (turn.ownerEndedAt !== null && options.now - turn.ownerEndedAt > within) {\n findings.push(`${label} is still open ${minutes(options.now - turn.ownerEndedAt)} min after its owner stopped; the most is ${minutes(within)}`)\n }\n continue\n }\n counts[terminal.state] += 1\n if (turn.ownerEndedAt !== null) {\n const lag = terminal.at - turn.ownerEndedAt\n counts.maxDetectionLagMinutes = Math.max(counts.maxDetectionLagMinutes, minutes(lag))\n if (lag > within) findings.push(`${label} was settled ${minutes(lag)} min after its owner stopped; the most is ${minutes(within)}`)\n }\n if (terminal.state === 'answered' && turn.abortedToolCall) {\n findings.push(`${label} is counted as an answer, but the Sandbox aborted a tool call in it; it stopped mid-work and is a failure`)\n }\n if (terminal.state === 'failed') {\n if (!terminal.code) findings.push(`${label} failed without a typed failure code`)\n if (terminal.retryable !== true) findings.push(`${label} failed without offering Retry`)\n if (terminal.attributedTo !== undefined && terminal.attributedTo !== turn.requestedAt) {\n findings.push(`${label}'s failure is reported at ${new Date(terminal.attributedTo).toISOString()}, not its request time ${new Date(turn.requestedAt).toISOString()}`)\n }\n }\n if (terminal.state === 'stopped' && terminal.code && /fail|error/i.test(terminal.code)) {\n findings.push(`${label} was stopped by the person but carries failure code ${terminal.code}; a Stop is counted on its own`)\n }\n }\n return {\n invariant: 'honest-settlement',\n subject: options.subject ?? 'turns',\n pass: findings.length === 0,\n details: findings.length ? findings : [\n `${counts.turns} turns: ${counts.answered} answered, ${counts.failed} failed with Retry, ${counts.stopped} stopped, ${counts.open} held by a live owner`,\n `longest detection lag ${counts.maxDetectionLagMinutes} min`,\n ],\n data: { ...counts },\n }\n}\n\nfunction minutes(ms: number): number {\n return Math.round((ms / 60_000) * 10) / 10\n}\n\n/**\n * The failure shapes GTM shipped on 2026-10-10, as scenarios a product seeds\n * into its own storage, settles with its own scheduled jobs, and reads back as\n * {@link SettledTurnRecord}s. `expect` is the one honest terminal state.\n */\nexport interface SettlementScenario {\n id: string\n /** What happened to the turn, for the product's seed step. */\n story: string\n /** Minutes before the check's `now` that the request was accepted. */\n requestedMinutesAgo: number\n /**\n * Minutes before `now` that the owner stopped (for a stream, its lease\n * expired), or null for a live owner. Every scenario that stopped did so 12\n * minutes ago, so an app that settles within 15 minutes has settled it by `now`.\n */\n ownerEndedMinutesAgo: number | null\n expect: TurnTerminalState | 'open'\n}\n\nexport const SETTLEMENT_SCENARIOS: readonly SettlementScenario[] = [\n {\n id: 'orphaned-stream',\n story: 'The Worker streaming the turn died; its stream row is still running, no admission or Workflow holds it, and its 5-minute lease expired 12 minutes ago.',\n requestedMinutesAgo: 30,\n ownerEndedMinutesAgo: 12,\n expect: 'failed',\n },\n {\n id: 'abandoned-before-admission',\n story: 'The user message was saved 12 minutes ago, then the request died before any completion owner admitted the turn: no reply, no stream, no admission.',\n requestedMinutesAgo: 12,\n ownerEndedMinutesAgo: 12,\n expect: 'failed',\n },\n {\n id: 'runtime-restarted',\n story: 'The admitted run\\'s runtime restarted mid-turn 12 minutes ago: the session still reports running while its execution ledger failed the run (terminal event interrupted-done).',\n requestedMinutesAgo: 40,\n ownerEndedMinutesAgo: 12,\n expect: 'failed',\n },\n {\n id: 'completed-with-aborted-tool',\n story: 'The run reported completed 12 minutes ago, but a tool part ended with \"Tool execution aborted\".',\n requestedMinutesAgo: 20,\n ownerEndedMinutesAgo: 12,\n expect: 'failed',\n },\n {\n id: 'user-stop',\n story: 'The person pressed Stop 12 minutes ago while the turn ran.',\n requestedMinutesAgo: 20,\n ownerEndedMinutesAgo: 12,\n expect: 'stopped',\n },\n {\n id: 'answered',\n story: 'The turn answered normally 12 minutes ago.',\n requestedMinutesAgo: 20,\n ownerEndedMinutesAgo: 12,\n expect: 'answered',\n },\n {\n id: 'live-long-turn',\n story: 'A long turn is still running under an owner that renews its lease.',\n requestedMinutesAgo: 90,\n ownerEndedMinutesAgo: null,\n expect: 'open',\n },\n]\n\n/**\n * Check a product's settlement of {@link SETTLEMENT_SCENARIOS}: each scenario's\n * record ends in its expected state, and all of them settle honestly. The\n * product seeds each scenario at `now`, runs its own recovery and settlement\n * jobs at `now`, and reads the records back keyed by scenario id.\n */\nexport function checkSettlementScenarios(\n records: Readonly<Record<string, SettledTurnRecord | undefined>>,\n options: { now: number; scenarios?: readonly SettlementScenario[] },\n): InvariantVerdict {\n const scenarios = options.scenarios ?? SETTLEMENT_SCENARIOS\n const findings: string[] = []\n const present: SettledTurnRecord[] = []\n for (const scenario of scenarios) {\n const record = records[scenario.id]\n if (!record) {\n findings.push(`scenario ${scenario.id}: no turn record was read back`)\n continue\n }\n present.push(record)\n const state = record.terminals[0]?.state ?? 'open'\n if (state !== scenario.expect) findings.push(`scenario ${scenario.id}: ended ${state}, expected ${scenario.expect} (${scenario.story})`)\n }\n const honest = checkTurnSettlement(present, { now: options.now, subject: 'settlement scenarios' })\n if (!honest.pass) findings.push(...honest.details)\n return {\n invariant: 'honest-settlement',\n subject: 'settlement scenarios',\n pass: findings.length === 0,\n details: findings.length ? findings : [`${scenarios.length} failure scenarios settle to their honest state`, ...honest.details],\n data: honest.data,\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAoBO,IAAM,qBAAqB;AAE3B,IAAM,0BAA0B;AACvC,IAAM,cAAc;AAGb,IAAM,oBAAN,cAAgC,MAAM;AAAC;AAuCvC,SAAS,sBAAyB,SAAqD;AAC5F,QAAM,QAAQ,KAAK,IAAI,QAAQ,SAAS,oBAAoB,kBAAkB;AAC9E,QAAM,aAAa,QAAQ,cAAc;AACzC,QAAM,MAAM,QAAQ,QAAQ,MAAM,KAAK,IAAI;AAC3C,QAAM,OAAO,QAAQ,SAAS,CAAC,SAAS,WAAW,QAAQ,KAAK,SAAS,MAAM;AAC/E,QAAM,UAAU,oBAAI,IAAsB;AAE1C,SAAO;AAAA,IACL,MAAM,OAAO,KAAK,MAAM,eAAe;AACrC,UAAI;AACJ,UAAI;AACF,gBAAQ,MAAM,aAAa,KAAK,GAAG,UAAU;AAAA,MAC/C,SAAS,OAAO;AACd,YAAI,iBAAiB,qBAAqB,iBAAiB,UAAU;AACnE,kBAAQ,OAAO,GAAG;AAClB,gBAAM;AAAA,QACR;AACA,cAAM,QAAQ,QAAQ,IAAI,GAAG;AAC7B,YAAI,SAAS,IAAI,IAAI,MAAM,OAAO;AAChC,eAAK,gBAAgB,QAAQ,KAAK,kEAAkE;AAAA,YAClG,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,YAC5D,aAAa,MAAM,QAAQ,IAAI;AAAA,UACjC,CAAC;AACD,iBAAO,MAAM;AAAA,QACf;AACA,gBAAQ,OAAO,GAAG;AAClB,cAAM;AAAA,MACR;AACA,UAAI,UAAU,MAAM;AAClB,gBAAQ,OAAO,GAAG;AAClB,eAAO;AAAA,MACT;AACA,YAAM,YAAY,eAAe,YAAY,KAAK;AAClD,YAAM,QAAQ,KAAK,IAAI,IAAI,IAAI,OAAO,OAAO,cAAc,YAAY,YAAY,IAAI,YAAY,OAAO,iBAAiB;AAC3H,cAAQ,OAAO,GAAG;AAClB,cAAQ,IAAI,KAAK,EAAE,OAAO,MAAM,CAAC;AACjC,UAAI,QAAQ,OAAO,YAAa,SAAQ,OAAO,QAAQ,KAAK,EAAE,KAAK,EAAE,KAAM;AAC3E,aAAO;AAAA,IACT;AAAA,IACA,WAAW,SAAS;AAClB,iBAAW,CAAC,KAAK,KAAK,KAAK,QAAS,KAAI,QAAQ,MAAM,KAAK,EAAG,SAAQ,OAAO,GAAG;AAAA,IAClF;AAAA,IACA,OAAO,KAAK;AACV,cAAQ,OAAO,GAAG;AAAA,IACpB;AAAA,IACA,QAAQ;AACN,cAAQ,MAAM;AAAA,IAChB;AAAA,IACA,OAAO;AACL,aAAO,CAAC,GAAG,QAAQ,KAAK,CAAC;AAAA,IAC3B;AAAA,EACF;AACF;AAGA,eAAsB,mBAAmB,QAAiC;AACxE,QAAM,SAAS,MAAM,OAAO,OAAO,OAAO,WAAW,IAAI,YAAY,EAAE,OAAO,MAAM,CAAC;AACrF,SAAO,CAAC,GAAG,IAAI,WAAW,MAAM,CAAC,EAAE,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE,EAAE,SAAS,GAAG,GAAG,CAAC,EAAE,KAAK,EAAE;AAC9F;AAEA,SAAS,aAAgB,SAAqB,IAAwB;AACpE,MAAI;AACJ,QAAM,WAAW,IAAI,QAAe,CAAC,GAAG,WAAW;AACjD,YAAQ,WAAW,MAAM,OAAO,IAAI,MAAM,qCAAqC,EAAE,KAAK,CAAC,GAAG,EAAE;AAAA,EAC9F,CAAC;AACD,SAAO,QAAQ,KAAK,CAAC,SAAS,QAAQ,CAAC,EAAE,QAAQ,MAAM,aAAa,KAAK,CAAC;AAC5E;;;ACxEO,SAAS,oBACd,OACA,SACkB;AAClB,QAAM,SAAS,QAAQ,kBAAkB,eAAe;AACxD,QAAM,WAAqB,CAAC;AAC5B,QAAM,SAA+B,EAAE,OAAO,MAAM,QAAQ,UAAU,GAAG,QAAQ,GAAG,SAAS,GAAG,MAAM,GAAG,wBAAwB,EAAE;AACnI,aAAW,QAAQ,OAAO;AACxB,UAAM,QAAQ,QAAQ,KAAK,MAAM;AACjC,QAAI,KAAK,UAAU,SAAS,GAAG;AAC7B,eAAS,KAAK,GAAG,KAAK,QAAQ,KAAK,UAAU,MAAM,qBAAqB,KAAK,UAAU,IAAI,CAACA,cAAaA,UAAS,KAAK,EAAE,KAAK,QAAQ,CAAC,qBAAqB;AAAA,IAC9J;AACA,UAAM,WAAW,KAAK,UAAU,CAAC;AACjC,QAAI,CAAC,UAAU;AACb,aAAO,QAAQ;AACf,UAAI,KAAK,iBAAiB,QAAQ,QAAQ,MAAM,KAAK,eAAe,QAAQ;AAC1E,iBAAS,KAAK,GAAG,KAAK,kBAAkB,QAAQ,QAAQ,MAAM,KAAK,YAAY,CAAC,6CAA6C,QAAQ,MAAM,CAAC,EAAE;AAAA,MAChJ;AACA;AAAA,IACF;AACA,WAAO,SAAS,KAAK,KAAK;AAC1B,QAAI,KAAK,iBAAiB,MAAM;AAC9B,YAAM,MAAM,SAAS,KAAK,KAAK;AAC/B,aAAO,yBAAyB,KAAK,IAAI,OAAO,wBAAwB,QAAQ,GAAG,CAAC;AACpF,UAAI,MAAM,OAAQ,UAAS,KAAK,GAAG,KAAK,gBAAgB,QAAQ,GAAG,CAAC,6CAA6C,QAAQ,MAAM,CAAC,EAAE;AAAA,IACpI;AACA,QAAI,SAAS,UAAU,cAAc,KAAK,iBAAiB;AACzD,eAAS,KAAK,GAAG,KAAK,2GAA2G;AAAA,IACnI;AACA,QAAI,SAAS,UAAU,UAAU;AAC/B,UAAI,CAAC,SAAS,KAAM,UAAS,KAAK,GAAG,KAAK,sCAAsC;AAChF,UAAI,SAAS,cAAc,KAAM,UAAS,KAAK,GAAG,KAAK,gCAAgC;AACvF,UAAI,SAAS,iBAAiB,UAAa,SAAS,iBAAiB,KAAK,aAAa;AACrF,iBAAS,KAAK,GAAG,KAAK,6BAA6B,IAAI,KAAK,SAAS,YAAY,EAAE,YAAY,CAAC,0BAA0B,IAAI,KAAK,KAAK,WAAW,EAAE,YAAY,CAAC,EAAE;AAAA,MACtK;AAAA,IACF;AACA,QAAI,SAAS,UAAU,aAAa,SAAS,QAAQ,cAAc,KAAK,SAAS,IAAI,GAAG;AACtF,eAAS,KAAK,GAAG,KAAK,uDAAuD,SAAS,IAAI,gCAAgC;AAAA,IAC5H;AAAA,EACF;AACA,SAAO;AAAA,IACL,WAAW;AAAA,IACX,SAAS,QAAQ,WAAW;AAAA,IAC5B,MAAM,SAAS,WAAW;AAAA,IAC1B,SAAS,SAAS,SAAS,WAAW;AAAA,MACpC,GAAG,OAAO,KAAK,WAAW,OAAO,QAAQ,cAAc,OAAO,MAAM,uBAAuB,OAAO,OAAO,aAAa,OAAO,IAAI;AAAA,MACjI,yBAAyB,OAAO,sBAAsB;AAAA,IACxD;AAAA,IACA,MAAM,EAAE,GAAG,OAAO;AAAA,EACpB;AACF;AAEA,SAAS,QAAQ,IAAoB;AACnC,SAAO,KAAK,MAAO,KAAK,MAAU,EAAE,IAAI;AAC1C;AAsBO,IAAM,uBAAsD;AAAA,EACjE;AAAA,IACE,IAAI;AAAA,IACJ,OAAO;AAAA,IACP,qBAAqB;AAAA,IACrB,sBAAsB;AAAA,IACtB,QAAQ;AAAA,EACV;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO;AAAA,IACP,qBAAqB;AAAA,IACrB,sBAAsB;AAAA,IACtB,QAAQ;AAAA,EACV;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO;AAAA,IACP,qBAAqB;AAAA,IACrB,sBAAsB;AAAA,IACtB,QAAQ;AAAA,EACV;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO;AAAA,IACP,qBAAqB;AAAA,IACrB,sBAAsB;AAAA,IACtB,QAAQ;AAAA,EACV;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO;AAAA,IACP,qBAAqB;AAAA,IACrB,sBAAsB;AAAA,IACtB,QAAQ;AAAA,EACV;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO;AAAA,IACP,qBAAqB;AAAA,IACrB,sBAAsB;AAAA,IACtB,QAAQ;AAAA,EACV;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO;AAAA,IACP,qBAAqB;AAAA,IACrB,sBAAsB;AAAA,IACtB,QAAQ;AAAA,EACV;AACF;AAQO,SAAS,yBACd,SACA,SACkB;AAClB,QAAM,YAAY,QAAQ,aAAa;AACvC,QAAM,WAAqB,CAAC;AAC5B,QAAM,UAA+B,CAAC;AACtC,aAAW,YAAY,WAAW;AAChC,UAAM,SAAS,QAAQ,SAAS,EAAE;AAClC,QAAI,CAAC,QAAQ;AACX,eAAS,KAAK,YAAY,SAAS,EAAE,gCAAgC;AACrE;AAAA,IACF;AACA,YAAQ,KAAK,MAAM;AACnB,UAAM,QAAQ,OAAO,UAAU,CAAC,GAAG,SAAS;AAC5C,QAAI,UAAU,SAAS,OAAQ,UAAS,KAAK,YAAY,SAAS,EAAE,WAAW,KAAK,cAAc,SAAS,MAAM,KAAK,SAAS,KAAK,GAAG;AAAA,EACzI;AACA,QAAM,SAAS,oBAAoB,SAAS,EAAE,KAAK,QAAQ,KAAK,SAAS,uBAAuB,CAAC;AACjG,MAAI,CAAC,OAAO,KAAM,UAAS,KAAK,GAAG,OAAO,OAAO;AACjD,SAAO;AAAA,IACL,WAAW;AAAA,IACX,SAAS;AAAA,IACT,MAAM,SAAS,WAAW;AAAA,IAC1B,SAAS,SAAS,SAAS,WAAW,CAAC,GAAG,UAAU,MAAM,mDAAmD,GAAG,OAAO,OAAO;AAAA,IAC9H,MAAM,OAAO;AAAA,EACf;AACF;","names":["terminal"]}
@@ -0,0 +1,66 @@
1
+ /**
2
+ * Alarms that fire at 80% of a platform limit, before the limit fails a customer.
3
+ *
4
+ * On 2026-10-10 GTM recorded 19 platform-limit events (Worker memory, D1,
5
+ * sandbox disk, snapshot count, key rate) and learned of each one after it had
6
+ * failed someone. Each resource here has a declared limit; an observation at
7
+ * or above 80% of it delivers a warning, and at or above the limit a critical
8
+ * alert, through the same {@link AlertSink} the turn-health sweep uses (wrap it
9
+ * in `createThrottledAlertSink` so a hot loop pages once).
10
+ *
11
+ * Where each observation comes from:
12
+ * - `d1-rows-per-query`: {@link withD1LimitAlarms} reads `meta.rows_read` on every query.
13
+ * - `worker-memory`: workerd exposes no heap reading, so the D1 wrapper observes
14
+ * the bytes each response returns (the reads that overran memory today), and a
15
+ * tail consumer reports an `exceededMemory` outcome at the full limit.
16
+ * - `sandbox-disk` and `snapshot-count`: the Sandbox SDK's box usage and snapshot list, read on a schedule.
17
+ * - `key-rate`: the per-key request counter the operator API keeps.
18
+ */
19
+ import type { AlertSink } from '../turn-health/sink.js';
20
+ export type LimitResource = 'worker-memory' | 'd1-rows-per-query' | 'sandbox-disk' | 'snapshot-count' | 'key-rate';
21
+ export declare const LIMIT_RESOURCES: readonly LimitResource[];
22
+ /** The limit of each resource, in its own unit: bytes, rows, bytes, snapshots, requests per minute. */
23
+ export type LimitBudgets = Readonly<Record<LimitResource, number>>;
24
+ export type LimitLevel = 'ok' | 'warning' | 'critical';
25
+ export interface LimitObservation {
26
+ resource: LimitResource;
27
+ used: number;
28
+ limit: number;
29
+ /** used / limit. */
30
+ share: number;
31
+ level: LimitLevel;
32
+ }
33
+ export interface LimitAlarms {
34
+ readonly budgets: LimitBudgets;
35
+ /** Record one reading; a warning or critical reading is delivered before this resolves. */
36
+ observe(resource: LimitResource, used: number, context?: {
37
+ subject?: string;
38
+ detail?: string;
39
+ }): Promise<LimitObservation>;
40
+ }
41
+ export interface LimitAlarmOptions {
42
+ product: string;
43
+ budgets: LimitBudgets;
44
+ sink: AlertSink;
45
+ /** Default {@link LAUNCH_BUDGETS.alarmRatio}; a product cannot raise it past the limit itself. */
46
+ ratio?: number;
47
+ now?: () => number;
48
+ }
49
+ export declare function createLimitAlarms(options: LimitAlarmOptions): LimitAlarms;
50
+ /** The D1 surface the wrapper measures; Cloudflare `D1Database` satisfies it. */
51
+ interface D1Measurable {
52
+ prepare(sql: string): unknown;
53
+ }
54
+ /**
55
+ * The D1 binding with each query's rows read and response size observed.
56
+ * Rows read come from `meta.rows_read` (the rows D1 scanned, not only those
57
+ * returned). The response size is estimated from three sampled rows, so the
58
+ * wrapper never serializes a whole response: that copy would double the memory
59
+ * of the very read it watches.
60
+ */
61
+ export declare function withD1LimitAlarms<D extends D1Measurable>(d1: D, alarms: LimitAlarms, options?: {
62
+ subject?: string;
63
+ }): D;
64
+ /** The serialized size of a result set, from its first, middle and last rows. */
65
+ export declare function estimatedBytes(results: readonly unknown[]): number;
66
+ export {};
@@ -0,0 +1,114 @@
1
+ /**
2
+ * Scheduled jobs that cannot take each other down.
3
+ *
4
+ * On 2026-10-10 at 21:00Z GTM's turn health ran out of memory while the hourly
5
+ * sweeps shared one scheduled run, and none of the eight queued after it ran.
6
+ * Here each scheduled invocation runs exactly one job. A cron can carry a
7
+ * minute list (`0,1,2 * * * *`) with one job per minute, so a Worker keeps a
8
+ * few trigger expressions while every job gets its own invocation, its own
9
+ * memory and its own failure.
10
+ *
11
+ * Cloudflare hands the handler the configured string verbatim as `event.cron`,
12
+ * so dispatch is exact-string; a tick no entry handles is reported as
13
+ * unhandled instead of silently running nothing. {@link checkIsolatedJobs}
14
+ * holds the table against the deployment's configured crons.
15
+ */
16
+ import { type InvariantVerdict } from './catalog.js';
17
+ /** One cron expression and the job it runs, or one job per listed minute. */
18
+ export type ScheduleEntry = {
19
+ cron: string;
20
+ job: string;
21
+ } | {
22
+ cron: string;
23
+ minutes: Readonly<Record<number, string>>;
24
+ };
25
+ /** One scheduled invocation: the cron, the minute for a minute-list entry, and the job it runs. */
26
+ export interface ScheduleSlot {
27
+ cron: string;
28
+ minute?: number;
29
+ job: string;
30
+ }
31
+ export interface ScheduledJobInput<Ctx> {
32
+ cron: string;
33
+ scheduledTime: Date;
34
+ context: Ctx;
35
+ /** 1 for the first try. */
36
+ attempt: number;
37
+ }
38
+ export type ScheduledJob<Ctx> = (input: ScheduledJobInput<Ctx>) => Promise<unknown>;
39
+ export interface ScheduledDispatchOptions<Ctx> {
40
+ entries: readonly ScheduleEntry[];
41
+ jobs: Readonly<Record<string, ScheduledJob<Ctx>>>;
42
+ /** Waits before each retry of a failed job; empty (the default) runs a job once. Jobs that retry must be idempotent. */
43
+ retryDelaysMs?: readonly number[];
44
+ log?: {
45
+ info(message: string, detail?: Record<string, unknown>): void;
46
+ error(message: string, detail?: Record<string, unknown>): void;
47
+ };
48
+ /** Tests pass a no-op. */
49
+ sleep?: (ms: number) => Promise<void>;
50
+ }
51
+ export interface ScheduledJobOutcome {
52
+ job: string;
53
+ ok: boolean;
54
+ attempts: number;
55
+ durationMs: number;
56
+ error?: string;
57
+ }
58
+ export interface ScheduledDispatchResult {
59
+ handled: boolean;
60
+ job: string | null;
61
+ /** Settles when the job's last attempt ends; never rejects. */
62
+ done: Promise<ScheduledJobOutcome | null>;
63
+ }
64
+ export interface ScheduledDispatch<Ctx> {
65
+ readonly entries: readonly ScheduleEntry[];
66
+ /** Every invocation the table can make. */
67
+ slots(): ScheduleSlot[];
68
+ /** The job a tick runs, or null when no entry handles it. */
69
+ jobFor(cron: string, scheduledTime: number | Date): string | null;
70
+ /**
71
+ * Run the tick's one job. With `waitUntil` the job runs in the background and
72
+ * this returns at once; without it, this waits for the job.
73
+ */
74
+ dispatch(event: {
75
+ cron: string;
76
+ scheduledTime: number | Date;
77
+ }, context: Ctx, waitUntil?: (promise: Promise<unknown>) => void): Promise<ScheduledDispatchResult>;
78
+ }
79
+ export declare function createScheduledDispatch<Ctx>(options: ScheduledDispatchOptions<Ctx>): ScheduledDispatch<Ctx>;
80
+ /** Every invocation a schedule table can make. */
81
+ export declare function slotsOf(entries: readonly ScheduleEntry[]): ScheduleSlot[];
82
+ /**
83
+ * The minutes of the day (0–1439, UTC) a cron fires, or null when it does not
84
+ * fire every day (a day-of-month, month or weekday restriction) or cannot be read.
85
+ */
86
+ export declare function cronDailyFirings(cron: string): Set<number> | null;
87
+ /** The longest wait, in minutes, between consecutive firings over a day, or null when they do not fire daily. */
88
+ export declare function longestGapMinutes(firings: Iterable<number>): number | null;
89
+ /** The cron expressions a wrangler config declares, top level and per environment. */
90
+ export interface WranglerCrons {
91
+ top: string[];
92
+ envs: Record<string, string[]>;
93
+ }
94
+ /** Read `[triggers] crons` (and `[env.<name>.triggers]`) from wrangler.toml, or `triggers.crons` from wrangler.json(c). */
95
+ export declare function parseWranglerCrons(source: string, format: 'toml' | 'json'): WranglerCrons;
96
+ /** What one scheduled invocation was observed to run. */
97
+ export interface ObservedSlot {
98
+ cron: string;
99
+ minute?: number;
100
+ /** The jobs the invocation ran; exactly one passes. */
101
+ jobs: readonly string[];
102
+ }
103
+ export interface IsolatedJobsInput {
104
+ /** Every cron the deployment configures, from each wrangler environment. */
105
+ configuredCrons: readonly string[];
106
+ /** What each invocation runs: from {@link slotsOf} for a {@link createScheduledDispatch} table, or observed by driving the app's own handler. */
107
+ slots: readonly ObservedSlot[];
108
+ /** Jobs that recover and settle turns; each must run at least every 15 minutes. */
109
+ recoveryJobs: readonly string[];
110
+ /** Default {@link LAUNCH_BUDGETS.recoveryEveryMs}. */
111
+ everyMs?: number;
112
+ }
113
+ /** One job per invocation, the table and the deployment agree, and recovery runs at least every 15 minutes. */
114
+ export declare function checkIsolatedJobs(input: IsolatedJobsInput): InvariantVerdict;
@@ -0,0 +1,90 @@
1
+ /**
2
+ * Honest turn settlement, checked over a product's turn records.
3
+ *
4
+ * A turn ends in exactly one terminal state: an answer, a typed failure the
5
+ * person can Retry, or a Stop the person chose, counted on its own. It gets
6
+ * there within 15 minutes of its owner stopping. A failure belongs to the time
7
+ * the turn was requested, not the time it was noticed, and how late it was
8
+ * noticed is measured. A run that completed with a tool call the Sandbox
9
+ * aborted stopped mid-work, so it is a failure.
10
+ *
11
+ * On 2026-10-10 GTM held turns in five places (turn status, admissions,
12
+ * Workflows, Sandbox sessions and the sidecar tracker) and they disagreed:
13
+ * 51 stream rows stayed `running` after their producers died, failure notices
14
+ * arrived 8–11 hours late, and completions with aborted tools counted as
15
+ * answers. {@link checkTurnSettlement} reads one normalized record per turn,
16
+ * however the product derives it, and names every turn that breaks the rule.
17
+ */
18
+ import { type InvariantVerdict } from './catalog.js';
19
+ export type TurnTerminalState = 'answered' | 'failed' | 'stopped';
20
+ /** One turn as its product records it, normalized. Times are Unix ms. */
21
+ export interface SettledTurnRecord {
22
+ turnId: string;
23
+ /** When the request was accepted (the user message was saved). */
24
+ requestedAt: number;
25
+ /** Every terminal state recorded for the turn, in order; exactly one is honest. */
26
+ terminals: ReadonlyArray<{
27
+ state: TurnTerminalState;
28
+ /** When the terminal state was recorded. */
29
+ at: number;
30
+ /** A typed failure code, for `failed`. */
31
+ code?: string;
32
+ /** Whether the person is offered Retry, for `failed`. */
33
+ retryable?: boolean;
34
+ /** The time the failure is reported under; it must be the request time. */
35
+ attributedTo?: number;
36
+ }>;
37
+ /** When the turn's owner stopped: for a leased stream, when its lease expired. Null while an owner holds it. */
38
+ ownerEndedAt: number | null;
39
+ /** A completed run in which the Sandbox aborted a tool call. */
40
+ abortedToolCall?: boolean;
41
+ }
42
+ export interface TurnSettlementCounts {
43
+ turns: number;
44
+ answered: number;
45
+ failed: number;
46
+ stopped: number;
47
+ open: number;
48
+ /** Most minutes between an owner stopping and its turn's terminal state. */
49
+ maxDetectionLagMinutes: number;
50
+ }
51
+ /**
52
+ * Check turn records against honest settlement at `now`.
53
+ * A turn still open whose owner has not stopped is fine; one whose owner stopped
54
+ * more than {@link LAUNCH_BUDGETS.settleWithinMs} ago is not.
55
+ */
56
+ export declare function checkTurnSettlement(turns: readonly SettledTurnRecord[], options: {
57
+ now: number;
58
+ settleWithinMs?: number;
59
+ subject?: string;
60
+ }): InvariantVerdict;
61
+ /**
62
+ * The failure shapes GTM shipped on 2026-10-10, as scenarios a product seeds
63
+ * into its own storage, settles with its own scheduled jobs, and reads back as
64
+ * {@link SettledTurnRecord}s. `expect` is the one honest terminal state.
65
+ */
66
+ export interface SettlementScenario {
67
+ id: string;
68
+ /** What happened to the turn, for the product's seed step. */
69
+ story: string;
70
+ /** Minutes before the check's `now` that the request was accepted. */
71
+ requestedMinutesAgo: number;
72
+ /**
73
+ * Minutes before `now` that the owner stopped (for a stream, its lease
74
+ * expired), or null for a live owner. Every scenario that stopped did so 12
75
+ * minutes ago, so an app that settles within 15 minutes has settled it by `now`.
76
+ */
77
+ ownerEndedMinutesAgo: number | null;
78
+ expect: TurnTerminalState | 'open';
79
+ }
80
+ export declare const SETTLEMENT_SCENARIOS: readonly SettlementScenario[];
81
+ /**
82
+ * Check a product's settlement of {@link SETTLEMENT_SCENARIOS}: each scenario's
83
+ * record ends in its expected state, and all of them settle honestly. The
84
+ * product seeds each scenario at `now`, runs its own recovery and settlement
85
+ * jobs at `now`, and reads the records back keyed by scenario id.
86
+ */
87
+ export declare function checkSettlementScenarios(records: Readonly<Record<string, SettledTurnRecord | undefined>>, options: {
88
+ now: number;
89
+ scenarios?: readonly SettlementScenario[];
90
+ }): InvariantVerdict;
@@ -0,0 +1,50 @@
1
+ /**
2
+ * Reads of a large column a bounded batch at a time.
3
+ *
4
+ * On 2026-10-10 GTM's scheduled jobs exceeded the Worker's 128 MB by reading a
5
+ * day of stored turns at once: the outcome sweep held about 93 MB of turn
6
+ * events, the reliability report read 1.63 M events, and turn health read
7
+ * 121.8 MB of message parts in one query. D1 refuses the same read on its side
8
+ * with `D1_ERROR: Memory limit exceeded before EOF`.
9
+ *
10
+ * A caller lists each row's key with the stored size of its large column
11
+ * (SQLite `length(column)`), then reads and handles one batch before it reads
12
+ * the next, keeping only what it concluded (counts, verdicts) rather than the
13
+ * rows. Memory then holds one batch, not the day.
14
+ *
15
+ * Keep what you conclude, not what you read: a value sliced or matched out of a
16
+ * large string can pin the whole parent string in V8 (a regex match kept in a
17
+ * Map held every tool output it came from). Copy a kept substring out, e.g.
18
+ * `new URL(match).href` or `(' ' + text).slice(1)`, before the row goes.
19
+ */
20
+ /** Most bytes of a large column one read returns; a single larger row is read alone. */
21
+ export declare const SIZED_READ_BYTES: number;
22
+ /** D1 binds at most 100 parameters to one query. */
23
+ export declare const D1_MAX_BOUND_PARAMETERS = 100;
24
+ /** Most single-column keys one read binds, leaving room for the query's own parameters. */
25
+ export declare const SIZED_READ_ROWS = 50;
26
+ /** Most two-column keys, such as (turnId, seq), one read binds: two parameters each. */
27
+ export declare const SIZED_READ_COMPOSITE_ROWS = 40;
28
+ /** A row's key and the stored size of its large column. */
29
+ export interface SizedRow<K> {
30
+ key: K;
31
+ /** `length()` of the large column; null when it is NULL. */
32
+ size: number | null | undefined;
33
+ }
34
+ export interface SizedBatchOptions {
35
+ /** Most bytes per batch. Default {@link SIZED_READ_BYTES}. */
36
+ bytes?: number;
37
+ /** Most keys per batch. Default {@link SIZED_READ_ROWS}; `Infinity` for a range read (`seq BETWEEN`). */
38
+ rows?: number;
39
+ }
40
+ /** Consecutive keys whose stored sizes fit one read, in their listed order. */
41
+ export declare function sizedBatches<K>(rows: Iterable<SizedRow<K>>, options?: SizedBatchOptions): K[][];
42
+ /**
43
+ * Read listed rows one sized batch at a time. Each batch is read only after the
44
+ * caller has handled the previous one, so a `for await` loop that keeps its
45
+ * conclusions and drops the rows holds one batch at most.
46
+ */
47
+ export declare function readInSizedBatches<K, R>(rows: Iterable<SizedRow<K>>, read: (keys: K[]) => Promise<R>, options?: SizedBatchOptions): AsyncGenerator<{
48
+ keys: K[];
49
+ value: R;
50
+ }>;