@basein/runner 0.2.12 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/README.md +540 -369
  2. package/dist/auth/client.d.ts +65 -3
  3. package/dist/auth/client.js +305 -53
  4. package/dist/bin/bir-hooks.d.ts +72 -31
  5. package/dist/bin/bir-hooks.js +176 -284
  6. package/dist/bin/bir-proxy.d.ts +17 -2
  7. package/dist/bin/bir-proxy.js +222 -28
  8. package/dist/bin/bir-scenario.d.ts +61 -21
  9. package/dist/bin/bir-scenario.js +593 -122
  10. package/dist/bin/bir.d.ts +12 -6
  11. package/dist/bin/bir.js +2935 -489
  12. package/dist/bin/investigate.js +40 -15
  13. package/dist/bin/scenario-edit.d.ts +7 -1
  14. package/dist/bin/scenario-edit.js +43 -21
  15. package/dist/bin/scenario-liveness.d.ts +194 -0
  16. package/dist/bin/scenario-liveness.js +459 -0
  17. package/dist/bin/scenario-stderr.d.ts +43 -0
  18. package/dist/bin/scenario-stderr.js +76 -0
  19. package/dist/bin/service-read.d.ts +88 -0
  20. package/dist/bin/service-read.js +128 -0
  21. package/dist/bin/setup.d.ts +19 -0
  22. package/dist/bin/setup.js +146 -20
  23. package/dist/config/adapters/claude-code.d.ts +92 -14
  24. package/dist/config/adapters/claude-code.js +291 -101
  25. package/dist/config/adapters/generic.d.ts +4 -1
  26. package/dist/config/adapters/generic.js +13 -14
  27. package/dist/config/generate.d.ts +392 -14
  28. package/dist/config/generate.js +666 -19
  29. package/dist/config/mcp-approval.d.ts +76 -0
  30. package/dist/config/mcp-approval.js +210 -0
  31. package/dist/config/node-path.d.ts +42 -0
  32. package/dist/config/node-path.js +102 -0
  33. package/dist/config/overrides.d.ts +62 -0
  34. package/dist/config/overrides.js +136 -0
  35. package/dist/config/resolve.d.ts +70 -8
  36. package/dist/config/resolve.js +186 -15
  37. package/dist/control/client.d.ts +22 -0
  38. package/dist/control/client.js +26 -7
  39. package/dist/control/correlation.d.ts +23 -5
  40. package/dist/control/correlation.js +0 -0
  41. package/dist/control/daemon.d.ts +400 -18
  42. package/dist/control/daemon.js +1105 -102
  43. package/dist/control/discovery.d.ts +77 -5
  44. package/dist/control/discovery.js +186 -11
  45. package/dist/control/ensure-hook.d.ts +61 -0
  46. package/dist/control/ensure-hook.js +160 -8
  47. package/dist/control/paths.d.ts +26 -0
  48. package/dist/control/paths.js +34 -0
  49. package/dist/control/pin-guard.d.ts +83 -0
  50. package/dist/control/pin-guard.js +534 -0
  51. package/dist/control/recorder-keeper.d.ts +229 -0
  52. package/dist/control/recorder-keeper.js +721 -0
  53. package/dist/control/recorder-main.d.ts +73 -0
  54. package/dist/control/recorder-main.js +778 -0
  55. package/dist/control/server.d.ts +707 -31
  56. package/dist/control/server.js +2787 -302
  57. package/dist/control/transcript.d.ts +21 -1
  58. package/dist/control/transcript.js +378 -112
  59. package/dist/index.d.ts +5 -1
  60. package/dist/index.js +5 -1
  61. package/dist/jsonrpc/framing.d.ts +76 -12
  62. package/dist/jsonrpc/framing.js +216 -59
  63. package/dist/proxy/intercept.d.ts +49 -5
  64. package/dist/proxy/intercept.js +188 -59
  65. package/dist/proxy/relay.d.ts +16 -0
  66. package/dist/proxy/relay.js +77 -5
  67. package/dist/proxy/revive.d.ts +122 -0
  68. package/dist/proxy/revive.js +251 -0
  69. package/dist/proxy/session.d.ts +222 -14
  70. package/dist/proxy/session.js +708 -99
  71. package/dist/record/recorder.d.ts +51 -3
  72. package/dist/record/recorder.js +3 -0
  73. package/dist/record/redact.d.ts +103 -9
  74. package/dist/record/redact.js +659 -30
  75. package/dist/record/remote-recorder.d.ts +327 -32
  76. package/dist/record/remote-recorder.js +1517 -187
  77. package/dist/record/switchable-recorder.d.ts +126 -0
  78. package/dist/record/switchable-recorder.js +351 -0
  79. package/dist/record/truncate.d.ts +40 -1
  80. package/dist/record/truncate.js +111 -4
  81. package/dist/replay/bash-channel.d.ts +35 -0
  82. package/dist/replay/bash-channel.js +133 -0
  83. package/dist/replay/bundle.d.ts +10 -0
  84. package/dist/replay/bundle.js +10 -0
  85. package/dist/replay/controller.d.ts +369 -20
  86. package/dist/replay/controller.js +1176 -107
  87. package/dist/replay/derive.d.ts +1 -0
  88. package/dist/replay/derive.js +81 -24
  89. package/dist/replay/executor.d.ts +51 -16
  90. package/dist/replay/executor.js +114 -41
  91. package/dist/replay/handover.d.ts +54 -1
  92. package/dist/replay/handover.js +55 -5
  93. package/dist/replay/logic.d.ts +62 -10
  94. package/dist/replay/logic.js +140 -22
  95. package/dist/replay/plan.d.ts +201 -27
  96. package/dist/replay/plan.js +581 -141
  97. package/dist/replay/safety.d.ts +72 -0
  98. package/dist/replay/safety.js +303 -0
  99. package/dist/replay/source-run.d.ts +10 -0
  100. package/dist/replay/source-run.js +80 -35
  101. package/dist/replay/targets.d.ts +15 -1
  102. package/dist/replay/targets.js +62 -4
  103. package/dist/replay/tool-error.d.ts +15 -1
  104. package/dist/replay/tool-error.js +33 -1
  105. package/dist/replay/types.d.ts +7 -4
  106. package/dist/upstream/client.d.ts +56 -3
  107. package/dist/upstream/client.js +61 -4
  108. package/dist/upstream/headers.d.ts +43 -0
  109. package/dist/upstream/headers.js +90 -0
  110. package/dist/upstream/http-client.d.ts +213 -22
  111. package/dist/upstream/http-client.js +1034 -168
  112. package/dist/upstream/http-transport.d.ts +143 -0
  113. package/dist/upstream/http-transport.js +391 -0
  114. package/dist/upstream/lazy-client.d.ts +8 -0
  115. package/dist/upstream/lazy-client.js +29 -1
  116. package/dist/upstream/sse.d.ts +78 -0
  117. package/dist/upstream/sse.js +189 -0
  118. package/dist/upstream/stdio-client.d.ts +61 -1
  119. package/dist/upstream/stdio-client.js +356 -33
  120. package/dist/util/atomic-write.d.ts +29 -0
  121. package/dist/util/atomic-write.js +96 -0
  122. package/dist/util/journal.d.ts +15 -0
  123. package/dist/util/journal.js +48 -3
  124. package/dist/util/log.d.ts +10 -0
  125. package/dist/util/log.js +28 -1
  126. package/dist/util/off-switch.d.ts +46 -0
  127. package/dist/util/off-switch.js +91 -0
  128. package/dist/util/service-health.d.ts +133 -0
  129. package/dist/util/service-health.js +217 -0
  130. package/docs/BaseInstRunner.md +694 -621
  131. package/docs/calculatedReplay.md +1625 -1304
  132. package/docs/calculatedReplayGuide.md +1212 -1087
  133. package/docs/installRun.md +1013 -772
  134. package/docs/loginWeb.md +627 -607
  135. package/docs/mcpmark.md +49 -16
  136. package/docs/my-first-sample.md +569 -545
  137. package/docs/quickstart.md +384 -311
  138. package/docs/t-bench.md +406 -394
  139. package/package.json +58 -57
  140. package/production.md +1371 -0
  141. package/scripts/install.ps1 +97 -6
  142. package/scripts/install.sh +104 -7
@@ -8,102 +8,488 @@
8
8
  * PATCH /recordings/runs/:id/steps/:stepId
9
9
  * POST /recordings/runs/:id/finish
10
10
  *
11
- * Four properties matter, and each is load-bearing:
11
+ * The properties that matter, each load-bearing:
12
12
  * - **Client-generated ids** (`run_`/`step_` uuids) so callers get an id
13
13
  * synchronously and never await the network on the hot path.
14
- * - **One ordered promise chain per run**, so steps arrive in the order they
15
- * were recorded even though every send is fire-and-forget.
16
- * - **Best-effort.** A failed send is logged and dropped, never retried inline
17
- * and never thrown at the caller (§10).
18
- * - **One token refresh on 401**, then one retry. Sessions outlive access tokens.
14
+ * - **One ordered lane per run**, so steps arrive in the order they were
15
+ * recorded even though every send is fire-and-forget.
16
+ * - **Best-effort, and invisible when the service is not there.** A failed
17
+ * send is logged once per run and dropped, never thrown at the caller
18
+ * (§10). Every request has a deadline (`BIR_SERVICE_TIMEOUT_MS`), every
19
+ * answer or its absence is reported to the process-wide health verdict
20
+ * ({@link serviceHealth}), and while that verdict says the service is
21
+ * cooling down the calls a hook waits on — `getMatch`, `runCreated`,
22
+ * `matchIntent` — answer "no match" at once instead of making the user wait,
23
+ * and `flush` returns at once, leaving the rest to drain in the background.
24
+ * A run's create that fails its first attempt answers "no match" at once
25
+ * too; its retry is for the recording, never for a hook.
26
+ * - **One probe, not a stampede.** While the breaker is half-open, one
27
+ * queued send finds out for all the lanes; the rest wait for its verdict
28
+ * and then follow it a moment apart, not all at once.
29
+ * - **Bounded.** The lanes are the real backlog (the hook server's step queue
30
+ * drains instantly into them), so they carry the bound: past
31
+ * {@link DEFAULT_MAX_PENDING_SENDS} sends or {@link DEFAULT_MAX_PENDING_BYTES}
32
+ * the oldest unsent *step* is dropped and counted — creates and finishes go
33
+ * last, so the service is not left with runs that never end.
34
+ * - **One token refresh on 401**, shared by every send that meets the same
35
+ * expiry, then one retry. Sessions outlive access tokens.
36
+ * - **It follows credentials.json.** `bir logout` removes the file; a
37
+ * recorder that loaded its session from it stops sending within seconds,
38
+ * rather than recording for the rest of the access token's life.
39
+ * - **It redacts and caps every free-text field it sends** — the prompt, the
40
+ * answer, reasoning, errors — as the last line of defence. Structured tool
41
+ * payloads are redacted by their producers; text is only ever seen here.
19
42
  *
20
43
  * SIMILAR-PROMPT HITS. `POST /recordings/runs` may answer `{ id: null, matched }`,
21
44
  * meaning the server recognised the prompt and did **not** create a run. Every
22
45
  * later step post for that id would then 404, so callers must stop recording —
23
- * {@link RemoteRecorder.getMatch} is how they find out. v1 only stops; replay is v2.
46
+ * {@link RemoteRecorder.getMatch} is how they find out — and this recorder skips
47
+ * them itself too, for the caller that stopped waiting before the answer came.
24
48
  */
25
49
  import { randomUUID } from "node:crypto";
26
- import { loadCredentials, saveCredentials } from "../auth/client.js";
27
- import { logLine, errText } from "../util/log.js";
50
+ import { statSync } from "node:fs";
51
+ import { credentialsPath, describeBody, isAuthResponse, issuedFor, loadCredentials, saveCredentials, } from "../auth/client.js";
52
+ import { errText, logDetail, logEvery, logLine } from "../util/log.js";
53
+ import { isUnhealthyStatus, serviceHealth, serviceTimeoutMs, timeoutSignal, } from "../util/service-health.js";
54
+ import { redactJsonText, redactText } from "./redact.js";
55
+ import { capJsonText, capText, DEFAULT_MAX_PAYLOAD_BYTES, preCut } from "./truncate.js";
56
+ /**
57
+ * Caps for the free text this recorder sends, in UTF-8 bytes. Tool payloads
58
+ * arrive capped at 64 KiB by their producers; text never passed through one,
59
+ * so a pasted 3 MB log used to go out whole — once as the run's input, again
60
+ * in the execution report — and spend the prompt's match budget uploading.
61
+ */
62
+ export const TEXT_CAPS = {
63
+ /** The prompt: the run's `input`, and an execution report's `prompt`. */
64
+ prompt: 32 * 1024,
65
+ /** The final answer, and the run's `finalOutput`. */
66
+ answer: 64 * 1024,
67
+ /** A step's written reasoning. */
68
+ context: 8 * 1024,
69
+ /** A step's thinking text, sent only to be embedded. */
70
+ intent: 8 * 1024,
71
+ /** A tool's error text when it is not JSON. */
72
+ toolError: 8 * 1024,
73
+ /** An execution report's error lines, rendered in a web page. */
74
+ reportError: 2 * 1024,
75
+ };
76
+ /** Queued sends past which the oldest unsent step is dropped (`BIR_RECORDER_MAX_PENDING`). */
77
+ export const DEFAULT_MAX_PENDING_SENDS = 500;
78
+ /** Queued bytes past which the same happens (`BIR_RECORDER_MAX_PENDING_BYTES`). */
79
+ export const DEFAULT_MAX_PENDING_BYTES = 32 * 1024 * 1024;
80
+ /** How long a probe waits when its caller did not say. Its answer is useless after a hook's budget. */
81
+ export const PROBE_TIMEOUT_MS = 5_000;
82
+ /**
83
+ * How long {@link RemoteRecorder.flush} waits when its caller did not say
84
+ * (`BIR_FLUSH_TIMEOUT_MS`). Every caller is a hook or a shutdown, and a
85
+ * healthy service drains a turn's last few sends in well under a second.
86
+ */
87
+ export const DEFAULT_FLUSH_TIMEOUT_MS = 2_000;
88
+ /** How often a waiting flush looks at the breaker. */
89
+ const FLUSH_POLL_MS = 25;
90
+ /** How often, at most, credentials.json is looked at: one `stat`. */
91
+ const CREDENTIALS_CHECK_MS = 2_000;
92
+ /** A settled match (it holds the whole scenario) is kept this long for late readers. */
93
+ const MATCH_TTL_MS = 10 * 60_000;
94
+ /** A run nobody has touched for this long is forgotten entirely. */
95
+ const RUN_STATE_TTL_MS = 6 * 60 * 60_000;
96
+ /** And never more than this many are remembered. */
97
+ const RUN_STATE_MAX = 2_000;
98
+ /** Refresh this long before the access token expires, in the background. */
99
+ const REFRESH_AHEAD_MS = 60_000;
100
+ /** A failed early refresh is not retried sooner than this. */
101
+ const REFRESH_AHEAD_RETRY_MS = 30_000;
102
+ /**
103
+ * The longest one queued send waits for the service. Past it, a send that is
104
+ * still facing a cool-down is given up unsent (counted as dropped): the
105
+ * service has failed every probe for a minute, and sending it anyway is what
106
+ * made every lane hit a dead service at the same moment.
107
+ */
108
+ const COOLDOWN_WAIT_MAX_MS = 60_000;
109
+ /** A cool-down about to end is waited out even past {@link COOLDOWN_WAIT_MAX_MS}. */
110
+ const COOLDOWN_GRACE_MS = 5_000;
111
+ /** Lanes released after a probe succeeds go spread over up to this long. */
112
+ const RELEASE_JITTER_MAX_MS = 2_000;
113
+ /** How often a lane waiting for another lane's probe looks at the verdict. */
114
+ const PROBE_POLL_MS = 1_000;
115
+ /** After a 404 from the probe route (an older service), stop probing for this long. */
116
+ const MATCH_INTENT_UNSUPPORTED_MS = 10 * 60_000;
117
+ /**
118
+ * Statuses a background send is retried once on: the service saying it did
119
+ * not take the request. Not 502/504 — a gateway that gave up may have handed
120
+ * the request to an origin that is still inserting it, and until the service
121
+ * is known to deduplicate ids a repeat could record a step twice.
122
+ */
123
+ const RETRY_STATUSES = new Set([429, 503]);
124
+ /**
125
+ * Network failures that prove the request never reached the service — no
126
+ * connection, no address, or a pooled socket the far end had already closed
127
+ * (undici's "other side closed", before any answer) — so a repeat cannot
128
+ * double it. A reset mid-request is not among them.
129
+ */
130
+ const SAFE_RETRY_CODES = new Set(["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "UND_ERR_SOCKET", "UND_ERR_CONNECT_TIMEOUT"]);
131
+ /** A refresh that failed without an answer to act on is not tried again sooner than this. */
132
+ const REFRESH_RETRY_MS = 30_000;
133
+ /** After a refused refresh, how long to wait for a sibling's newer session to land on disk. */
134
+ const AUTH_RECHECK_MS = 1_500;
135
+ /** While the session is refused, how often it is tried again anyway. */
136
+ const AUTH_REJECTED_RETRY_MS = 10 * 60_000;
137
+ /** Execution reports ride their own lane: a matched turn has no run on the service. */
138
+ const EXECUTIONS = "__executions__";
139
+ /** A send that did not succeed, with the status the service answered, if it answered. */
140
+ export class ServiceError extends Error {
141
+ status;
142
+ constructor(message, status) {
143
+ super(message);
144
+ this.status = status;
145
+ this.name = "ServiceError";
146
+ }
147
+ }
148
+ /** The service's own answer that this refresh token is no good (a JSON 400/401). */
149
+ class RefreshRejected extends ServiceError {
150
+ }
151
+ /** Another account signed in while a send was between attempts: it is not sent under the new one. */
152
+ class AccountChanged extends Error {
153
+ constructor(what) {
154
+ super(`${what}: another account signed in — not sent under it`);
155
+ this.name = "AccountChanged";
156
+ }
157
+ }
158
+ function pend(text, max, json = false) {
159
+ if (typeof text !== "string")
160
+ return undefined;
161
+ const cut = preCut(text, max);
162
+ return { text: cut.text, bytes: cut.bytes, max, json };
163
+ }
164
+ /**
165
+ * The field as sent: capped and redacted. A JSON payload is redacted as JSON
166
+ * ({@link redactJsonText}), so it stays JSON and — when its producer already
167
+ * redacted it — keeps its exact bytes.
168
+ */
169
+ function out(p) {
170
+ if (!p)
171
+ return undefined;
172
+ return p.json ? capJsonText(p.text, p.max, redactJsonText, p.bytes) : capText(p.text, p.max, redactText, p.bytes);
173
+ }
174
+ function sizeOf(...parts) {
175
+ let n = 256;
176
+ for (const p of parts)
177
+ n += p?.text.length ?? 0;
178
+ return n;
179
+ }
180
+ /** A tool error is JSON when a producer serialized it, and prose when it did not. */
181
+ function pendError(text) {
182
+ if (typeof text !== "string")
183
+ return undefined;
184
+ return /^\s*[{[]/.test(text) ? pend(text, DEFAULT_MAX_PAYLOAD_BYTES, true) : pend(text, TEXT_CAPS.toolError);
185
+ }
186
+ function positiveEnv(name, fallback) {
187
+ const n = Number(process.env[name]);
188
+ return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback;
189
+ }
190
+ function reasonOf(err) {
191
+ const cause = err?.cause;
192
+ return cause?.code ?? cause?.message ?? errText(err);
193
+ }
194
+ /** The network error's code: undici's cause, or the first of a happy-eyeballs AggregateError's. */
195
+ function causeCode(err) {
196
+ const cause = err?.cause;
197
+ return cause?.code ?? cause?.errors?.[0]?.code;
198
+ }
199
+ /**
200
+ * An answer that is a web page rather than the API: a captive portal, a
201
+ * website, a CDN challenge. Judged by the body — an empty or plain-text 2xx
202
+ * is the API answering, whatever content type a framework's `res.send("ok")`
203
+ * put on it.
204
+ */
205
+ function isWebPage(text, type) {
206
+ if (/^\s*<(!doctype|html|head|body)\b/i.test(text))
207
+ return true;
208
+ return /html/i.test(type) && /^\s*</.test(text);
209
+ }
210
+ /** A JSON object — the shape of the service's own error answers. */
211
+ function isJsonObject(text) {
212
+ try {
213
+ const v = JSON.parse(text);
214
+ return typeof v === "object" && v !== null && !Array.isArray(v);
215
+ }
216
+ catch {
217
+ return false;
218
+ }
219
+ }
220
+ /** `p`, or a rejection as soon as `signal` aborts. `p` keeps running; its outcome is observed either way. */
221
+ function untilAborted(p, signal) {
222
+ if (signal.aborted) {
223
+ p.catch(() => undefined);
224
+ return Promise.reject(signal.reason);
225
+ }
226
+ return new Promise((resolve, reject) => {
227
+ const onAbort = () => reject(signal.reason);
228
+ signal.addEventListener("abort", onAbort, { once: true });
229
+ p.then((v) => {
230
+ signal.removeEventListener("abort", onAbort);
231
+ resolve(v);
232
+ }, (e) => {
233
+ signal.removeEventListener("abort", onAbort);
234
+ reject(e);
235
+ });
236
+ });
237
+ }
238
+ function detail(text, type) {
239
+ const d = describeBody(text, type);
240
+ return d ? ` — ${d}` : "";
241
+ }
242
+ function sleep(ms, unref = false) {
243
+ return new Promise((resolve) => {
244
+ const timer = setTimeout(resolve, ms);
245
+ if (unref)
246
+ timer.unref?.();
247
+ });
248
+ }
249
+ /** Call a caller's hook; its failure is never the request's. */
250
+ function notify(fn) {
251
+ try {
252
+ fn?.();
253
+ }
254
+ catch {
255
+ /* never */
256
+ }
257
+ }
258
+ /** 250-750 ms: long enough for a load balancer to replace a dead socket, short enough to be unnoticed. */
259
+ function jitterMs() {
260
+ return 250 + Math.floor(Math.random() * 500);
261
+ }
262
+ /** A 429/503's own Retry-After, capped at 2 s; otherwise jitter. */
263
+ function retryDelayMs(res) {
264
+ const header = res.headers.get("retry-after");
265
+ if (header) {
266
+ const seconds = Number(header);
267
+ const ms = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(header) - Date.now();
268
+ if (Number.isFinite(ms))
269
+ return Math.min(2_000, Math.max(0, ms));
270
+ }
271
+ return jitterMs();
272
+ }
273
+ const PAUSE_WHY = {
274
+ signed_out: "credentials.json was removed (bir logout) — nothing more is sent and replay stops until you sign in again",
275
+ auth_rejected: "the service refused this session and no newer one was on disk — run `bir login`; recording resumes on its own (the session is also retried every 10 minutes)",
276
+ other_service: "credentials.json now belongs to a different service address — restart the recorder (or run `bir setup`) to record there",
277
+ };
278
+ const DEAD_WHY = {
279
+ not_found: "the service has no such run (its create never landed) — the rest of this run is skipped",
280
+ create_rejected: "the service refused to create the run — the rest of this run is skipped",
281
+ account_changed: "another account signed in on this machine — the rest of this run is skipped",
282
+ dropped: "the backlog bound dropped this run's create — the rest of it is skipped",
283
+ service_down: "the service failed every probe for a minute, so this run's create was never sent — the rest of it is skipped",
284
+ };
28
285
  export class RemoteRecorder {
29
286
  baseUrl;
30
287
  framework;
288
+ health;
289
+ /** The service these credentials are bound to; never changes for this recorder. */
290
+ binding;
291
+ maxPendingSends;
292
+ maxPendingBytes;
293
+ credentialsCheckMs;
31
294
  session;
32
- /** runId → tail of that run's ordered send chain. */
33
- chains = new Map();
34
- /** runId → the similar-prompt outcome of its create call. */
35
- matches = new Map();
36
- /** runId → what the create call said (not created: a hit, nothing to record). */
37
- created = new Map();
295
+ state = "recording";
296
+ runs = new Map();
297
+ lanes = new Map();
298
+ seq = 0;
299
+ pendingSends = 0;
300
+ pendingBytes = 0;
301
+ droppedSends = 0;
302
+ failedSends = 0;
303
+ backlogFull = false;
304
+ droppedWhileFull = 0;
305
+ lastSweep = 0;
306
+ refreshing;
307
+ /** A failed refresh is not tried again before this (epoch ms). */
308
+ refreshRetryAt = 0;
309
+ authRecheckMs;
310
+ authRetryMs;
311
+ cooldownWaitMaxMs;
312
+ /** When the session was last refused or retried while refused. */
313
+ rejectedAt = 0;
314
+ lastAheadAttempt = 0;
38
315
  /**
39
- * Execution reports ride their own chain: a matched turn has no run on the
40
- * service (that is what a match means), so there is no run chain to append to.
316
+ * The one request out while the breaker is half-open — a lane's own send,
317
+ * or a hook-requested `GET /health` — and when it started.
41
318
  */
42
- executionChainKey = "__executions__";
319
+ probe;
320
+ matchIntentOffUntil = 0;
321
+ /** True once this recorder's session is known to be the one in credentials.json. */
322
+ follow = false;
323
+ credMtime = -1;
324
+ credCheckedAt = 0;
43
325
  constructor(opts) {
44
326
  this.baseUrl = opts.baseUrl.replace(/\/+$/, "");
45
327
  this.session = opts.session;
46
328
  this.framework = opts.framework ?? "baseinstrunner";
329
+ this.health = opts.health ?? serviceHealth;
330
+ this.binding = (opts.session.authUrl ?? this.baseUrl).replace(/\/+$/, "");
331
+ this.maxPendingSends = opts.maxPendingSends ?? positiveEnv("BIR_RECORDER_MAX_PENDING", DEFAULT_MAX_PENDING_SENDS);
332
+ this.maxPendingBytes =
333
+ opts.maxPendingBytes ?? positiveEnv("BIR_RECORDER_MAX_PENDING_BYTES", DEFAULT_MAX_PENDING_BYTES);
334
+ this.credentialsCheckMs = opts.credentialsCheckMs ?? CREDENTIALS_CHECK_MS;
335
+ this.authRecheckMs = opts.authRecheckMs ?? AUTH_RECHECK_MS;
336
+ this.authRetryMs = opts.authRetryMs ?? AUTH_REJECTED_RETRY_MS;
337
+ this.cooldownWaitMaxMs = opts.cooldownWaitMaxMs ?? COOLDOWN_WAIT_MAX_MS;
338
+ // Follow the file only when this session came from it. A synthetic session
339
+ // (BIR_AUTH_DISABLE, tests) has no file, and must not read "no file" as
340
+ // "signed out".
341
+ try {
342
+ const mtime = statSync(credentialsPath()).mtimeMs;
343
+ const stored = loadCredentials();
344
+ if (stored &&
345
+ stored.user.id === opts.session.user.id &&
346
+ issuedFor(stored, this.binding) &&
347
+ (stored.accessToken === opts.session.accessToken || stored.refreshToken === opts.session.refreshToken)) {
348
+ this.follow = true;
349
+ this.credMtime = mtime;
350
+ this.credCheckedAt = Date.now();
351
+ }
352
+ }
353
+ catch {
354
+ /* no file: nothing to follow */
355
+ }
356
+ }
357
+ /**
358
+ * The access token this recorder is using *now*. It changes on every refresh
359
+ * (and when a sibling process's refresh is adopted), so callers that need the
360
+ * token — replay's service derivation, the source-run fetch — must read it
361
+ * through this each time rather than keep the startup session. Undefined once
362
+ * the session is gone (`bir logout`) or was refused.
363
+ */
364
+ currentAccessToken() {
365
+ this.checkCredentials();
366
+ if (this.state !== "recording")
367
+ return undefined;
368
+ return this.session.accessToken || undefined;
369
+ }
370
+ /** The backlog and sending state, for `/health` and `bir doctor`. */
371
+ recorderHealth() {
372
+ return {
373
+ state: this.state,
374
+ pendingSends: this.pendingSends,
375
+ pendingBytes: this.pendingBytes,
376
+ droppedSends: this.droppedSends,
377
+ failedSends: this.failedSends,
378
+ };
47
379
  }
48
380
  startRun(input, metadata, options) {
49
381
  const id = "run_" + randomUUID();
50
- let resolveMatch;
51
- let resolveCreated;
52
- this.matches.set(id, new Promise((r) => {
53
- resolveMatch = r;
54
- }));
55
- this.created.set(id, new Promise((r) => {
56
- resolveCreated = r;
57
- }));
58
- this.enqueue(id, async () => {
59
- try {
60
- const body = await this.post("/recordings/runs", {
61
- id,
62
- framework: this.framework,
63
- input,
64
- metadata,
65
- // A fragment (fallbk.md D7). Omitted, not null, so an older service
66
- // that does not know the field sees the body it always saw.
67
- ...(options?.fallbackOf ? { fallbackOf: options.fallbackOf } : {}),
68
- // A chain with calls is inlined for this runner rather than stopped in
69
- // front of its first call (segmented.md R-CALL-28).
70
- supportsCalls: true,
71
- });
72
- const b = body && typeof body === "object"
73
- ? body
74
- : {};
75
- resolveMatch(b.matched ?? null);
76
- // `id: null` is the service saying it kept a run of its own. Which run
77
- // that was matters: a turn must not count step hits against the
78
- // recording it is itself a repeat of (R-HIT-7).
79
- resolveCreated({ created: b.id !== null, fragmentOf: b.fragmentOf });
80
- }
81
- catch (err) {
82
- resolveMatch(null); // fail-safe: on a create error, record normally
83
- resolveCreated({ created: true });
84
- throw err; // preserve enqueue's dropped-send logging
85
- }
382
+ this.checkCredentials();
383
+ this.sweep();
384
+ const entry = this.newEntry();
385
+ this.runs.set(id, entry);
386
+ if (this.state !== "recording") {
387
+ // Nothing is sent while signed out, so there is nothing to match against
388
+ // and no run on the service: "fresh run", and every later send skipped.
389
+ entry.dead = this.state;
390
+ entry.settle(null, { created: true });
391
+ return id;
392
+ }
393
+ const prompt = pend(input ?? "", TEXT_CAPS.prompt);
394
+ this.enqueue({
395
+ lane: id,
396
+ runId: id,
397
+ kind: "create",
398
+ bytes: sizeOf(prompt),
399
+ // Dropped or skipped before it was sent: a fail-safe "fresh run, no match".
400
+ abandon: () => entry.settle(null, { created: true }),
401
+ send: async () => {
402
+ try {
403
+ const body = await this.request("POST", "/recordings/runs", JSON.stringify({
404
+ id,
405
+ framework: this.framework,
406
+ input: out(prompt),
407
+ metadata,
408
+ // A fragment (fallbk.md D7). Omitted, not null, so an older service
409
+ // that does not know the field sees the body it always saw.
410
+ ...(options?.fallbackOf ? { fallbackOf: options.fallbackOf } : {}),
411
+ // A chain with calls is inlined for this runner rather than stopped in
412
+ // front of its first call (segmented.md R-CALL-28).
413
+ supportsCalls: true,
414
+ }),
415
+ // The id is client-generated, so the service can tell a repeat. A
416
+ // hook waiting on this answer is let go before the retry: a match
417
+ // found by the retry is not the turn's any more.
418
+ { retryTransient: true, onRetry: () => entry.release() });
419
+ const b = body && typeof body === "object"
420
+ ? body
421
+ : {};
422
+ // `id: null` is the service saying it kept a run of its own. Which run
423
+ // that was matters: a turn must not count step hits against the
424
+ // recording it is itself a repeat of (R-HIT-7).
425
+ if (entry.released && b.id === null) {
426
+ logLine("recorder.matched_after_retry", {
427
+ run: id,
428
+ why: "the create's first attempt failed, so the turn went ahead as a fresh one; the service kept its own run, and this one's steps are not sent",
429
+ });
430
+ }
431
+ entry.settle(b.matched ?? null, { created: b.id !== null, fragmentOf: b.fragmentOf });
432
+ }
433
+ catch (err) {
434
+ entry.settle(null, { created: true }); // fail-safe: on a create error, record normally
435
+ throw err; // preserve the dropped-send logging
436
+ }
437
+ },
86
438
  });
87
439
  return id;
88
440
  }
89
- /** What the create call said. "Created" for an id this recorder never opened. */
441
+ /**
442
+ * What the create call said. "Created" for an id this recorder never opened,
443
+ * and — without waiting — while the service is cooling down or the session is
444
+ * gone, or once the create's first attempt has failed: a hook must never wait
445
+ * on a service that just failed to answer, nor on the retry after it.
446
+ */
90
447
  async runCreated(runId) {
91
- const pending = this.created.get(runId);
92
- return pending ? await pending : { created: true };
448
+ const entry = this.runs.get(runId);
449
+ if (!entry)
450
+ return { created: true };
451
+ entry.lastUsed = Date.now();
452
+ if (entry.settled)
453
+ return entry.settled.created;
454
+ if (entry.released || !entry.hot || !this.mayHotWait())
455
+ return { created: true };
456
+ await entry.hot;
457
+ // Settled, or let go when the first attempt failed (then: a fresh run).
458
+ return entry.settled?.created ?? { created: true };
93
459
  }
94
460
  /**
95
461
  * Match a ReAct iteration's reasoning against the caller's scenario intents
96
462
  * (fallbk.md §Runner 4). Awaited, not queued: the caller is a `PreToolUse`
97
463
  * holding a budget, and the answer decides what that hook says. A service that
98
- * predates the route (404) or any failure is a miss.
464
+ * predates the route (404) or any failure is a miss — and so is a service in
465
+ * a cool-down, without a request: the probe runs on every tool call, so a
466
+ * dead service must cost it nothing.
467
+ *
468
+ * `opts.signal`/`opts.timeoutMs` end the request when the caller stops
469
+ * waiting, so an abandoned probe does not hold a socket for minutes.
99
470
  */
100
- async matchIntent(req) {
471
+ async matchIntent(req, opts = {}) {
101
472
  try {
102
- const body = await this.post("/scenarios/match-intent", req);
473
+ if (!this.mayHotWait())
474
+ return null;
475
+ if (Date.now() < this.matchIntentOffUntil)
476
+ return null;
477
+ // The reasoning often quotes what the model just read — a .env, a key.
478
+ const body = {
479
+ ...req,
480
+ text: capText(req.text ?? "", TEXT_CAPS.intent, redactText),
481
+ ...(typeof req.toolInput === "string"
482
+ ? { toolInput: capJsonText(req.toolInput, DEFAULT_MAX_PAYLOAD_BYTES, redactJsonText) }
483
+ : {}),
484
+ };
485
+ const answer = await this.request("POST", "/scenarios/match-intent", JSON.stringify(body), {
486
+ signal: opts.signal,
487
+ timeoutMs: opts.timeoutMs ?? PROBE_TIMEOUT_MS,
488
+ });
103
489
  // The whole answer, not just `matched`: the counting fields are what make
104
490
  // the observe-only period readable (segmented.md R-OUT-10), and an older
105
491
  // server simply sends none of them (R-COMPAT-2).
106
- const b = body && typeof body === "object" ? body : {};
492
+ const b = answer && typeof answer === "object" ? answer : {};
107
493
  return {
108
494
  matched: b.matched ?? null,
109
495
  stepHit: b.stepHit ?? null,
@@ -112,18 +498,45 @@ export class RemoteRecorder {
112
498
  };
113
499
  }
114
500
  catch (err) {
115
- logLine("replay.intent_match_failed", { error: errText(err), why: "treated as a miss" });
501
+ if (err instanceof ServiceError && err.status === 404) {
502
+ // An older service: asking again on the next tool call gets the same 404.
503
+ this.matchIntentOffUntil = Date.now() + MATCH_INTENT_UNSUPPORTED_MS;
504
+ }
505
+ if (!opts.signal?.aborted) {
506
+ // Once per 30 s, not once per tool call: an outage is one fact.
507
+ logEvery("recorder.intent_match_failed", 30_000, "replay.intent_match_failed", {
508
+ error: errText(err),
509
+ why: "treated as a miss",
510
+ });
511
+ }
116
512
  return null;
117
513
  }
118
514
  }
119
- /** Await the similar-prompt match for a run's create call (null = fresh run). */
515
+ /**
516
+ * The similar-prompt match for a run's create call (null = fresh run). Like
517
+ * {@link runCreated}, answers null at once while the service is cooling down,
518
+ * and as soon as the create's first attempt fails — the first prompt of an
519
+ * outage is not held for the retry's sleep (up to 2 s behind a Retry-After).
520
+ */
120
521
  async getMatch(runId) {
121
- const pending = this.matches.get(runId);
122
- return pending ? await pending : null;
522
+ const entry = this.runs.get(runId);
523
+ if (!entry)
524
+ return null;
525
+ entry.lastUsed = Date.now();
526
+ if (entry.settled)
527
+ return entry.settled.match;
528
+ if (entry.released || !entry.hot || !this.mayHotWait())
529
+ return null;
530
+ await entry.hot;
531
+ return entry.settled?.match ?? null;
123
532
  }
124
533
  recordToolSelected(runId, stepIndex, data) {
125
534
  const id = "step_" + randomUUID();
126
- this.enqueue(runId, () =>
535
+ const toolInput = pend(data.toolInput, DEFAULT_MAX_PAYLOAD_BYTES, true);
536
+ const context = pend(data.context, TEXT_CAPS.context);
537
+ const intentText = data.intent ? pend(data.intent.text, TEXT_CAPS.intent) : undefined;
538
+ const intentSource = data.intent?.source;
539
+ this.enqueueRun(runId, "step", sizeOf(toolInput, context, intentText), () =>
127
540
  // `intent` and `metadata` are omitted, not nulled, when absent, so an
128
541
  // older service sees the body it has always seen (segmented.md
129
542
  // R-COMPAT-1).
@@ -132,42 +545,49 @@ export class RemoteRecorder {
132
545
  stepIndex,
133
546
  type: "tool_selected",
134
547
  toolName: data.toolName,
135
- toolInput: data.toolInput,
136
- context: data.context,
137
- intent: data.intent,
548
+ toolInput: out(toolInput),
549
+ context: out(context),
550
+ intent: intentSource ? { text: out(intentText) ?? "", source: intentSource } : undefined,
138
551
  metadata: data.metadata,
139
552
  }));
140
553
  return id;
141
554
  }
142
555
  recordToolResponse(runId, stepIndex, data) {
143
556
  const id = "step_" + randomUUID();
144
- this.enqueue(runId, () => this.post(`/recordings/runs/${runId}/steps`, {
557
+ const toolOutput = pend(data.toolOutput, DEFAULT_MAX_PAYLOAD_BYTES, true);
558
+ const toolError = pendError(data.toolError);
559
+ this.enqueueRun(runId, "step", sizeOf(toolOutput, toolError), () => this.post(`/recordings/runs/${runId}/steps`, {
145
560
  id,
146
561
  stepIndex,
147
562
  type: "tool_response",
148
563
  toolName: data.toolName,
149
- toolOutput: data.toolOutput,
150
- toolError: data.toolError,
564
+ toolOutput: out(toolOutput),
565
+ toolError: out(toolError),
151
566
  }));
152
567
  return id;
153
568
  }
154
569
  recordFinalAnswer(runId, stepIndex, data) {
155
570
  const id = "step_" + randomUUID();
156
- this.enqueue(runId, () => this.post(`/recordings/runs/${runId}/steps`, {
571
+ const answer = pend(data.answer, TEXT_CAPS.answer);
572
+ this.enqueueRun(runId, "step", sizeOf(answer), () => this.post(`/recordings/runs/${runId}/steps`, {
157
573
  id,
158
574
  stepIndex,
159
575
  type: "final_answer",
160
- answer: data.answer,
576
+ answer: out(answer),
161
577
  }));
162
578
  return id;
163
579
  }
164
580
  /** Backfill a step's `context` (the model's reasoning) after the fact. */
165
581
  backfillContext(runId, stepId, context) {
166
- this.enqueue(runId, () => this.patch(`/recordings/runs/${runId}/steps/${stepId}`, { context }));
582
+ const text = pend(context, TEXT_CAPS.context);
583
+ this.enqueueRun(runId, "patch", sizeOf(text), () =>
584
+ // A PATCH that sets one field: repeating it is harmless.
585
+ this.send("PATCH", `/recordings/runs/${runId}/steps/${stepId}`, { context: out(text) }));
167
586
  }
168
587
  finishRun(runId, finalOutput, metrics) {
169
- this.enqueue(runId, () => this.post(`/recordings/runs/${runId}/finish`, {
170
- finalOutput,
588
+ const output = pend(finalOutput, TEXT_CAPS.answer);
589
+ this.enqueueRun(runId, "finish", sizeOf(output), () => this.post(`/recordings/runs/${runId}/finish`, {
590
+ finalOutput: out(output),
171
591
  originalCostUsd: metrics?.costUsd,
172
592
  originalMs: metrics?.durationMs,
173
593
  }));
@@ -175,140 +595,847 @@ export class RemoteRecorder {
175
595
  /**
176
596
  * Report what a matched turn cost (docs/calculatedReplay.md §11.2).
177
597
  *
178
- * Queued on the run's own chain so it lands after that run's last step and so
179
- * `flush` waits for it — a report is the last thing a matched turn produces.
180
- * Best-effort like everything else here: an unredeemed ticket leaves no
181
- * execution row at all, which loses a saving but can never invent one.
598
+ * Queued on the executions lane — a matched turn has no run on the service —
599
+ * so `flush` waits for it. Best-effort like everything else here: an
600
+ * unredeemed ticket leaves no execution row at all, which loses a saving but
601
+ * can never invent one.
182
602
  *
183
603
  * A doubled report (a retry, `Stop` racing `SessionEnd`) is safe by
184
604
  * construction: the ticket *is* the row's id, so the service books once and
185
- * answers `200 { duplicate: true }`.
605
+ * answers `200 { duplicate: true }`. That is also why a report *with* a
606
+ * ticket is the one kind retried after an ambiguous failure.
186
607
  */
187
608
  reportExecution(report) {
188
609
  const costUsd = report.deriveCostUsd + report.sessionCostUsd + report.fallbackCostUsd;
189
- this.enqueue(this.executionChainKey, async () => {
190
- const body = await this.post(`/scenarios/${report.scenarioId}/executions`, {
191
- ticket: report.ticket,
192
- outcome: report.outcome,
193
- costUsd,
194
- deriveCostUsd: report.deriveCostUsd,
195
- sessionCostUsd: report.sessionCostUsd,
196
- fallbackCostUsd: report.fallbackCostUsd,
197
- durationMs: report.durationMs,
198
- stepsPlanned: report.stepsPlanned,
199
- stepsPinned: report.stepsPinned,
200
- measured: report.measured,
201
- pricingVersion: report.pricingVersion,
202
- prompt: report.prompt,
203
- // Per-step verdicts and the headline failure (errorshandling.md). All
204
- // optional on the service side, so an older service ignores them rather
205
- // than rejecting the report — and the costs still book.
206
- steps: report.steps,
207
- error: report.error,
208
- errorStage: report.errorStage,
209
- errorStepIndex: report.errorStepIndex,
210
- errorToolName: report.errorToolName,
211
- fallbackStepIndex: report.fallbackStepIndex,
212
- fallbackKind: report.fallbackKind,
213
- baselineEligible: report.baselineEligible,
214
- sharedWith: report.sharedWith,
215
- // Why nothing ran, when nothing ran. Additive: an older service drops
216
- // the field and books the cost exactly as it always did.
217
- declined: report.declined,
610
+ const prompt = pend(report.prompt, TEXT_CAPS.prompt);
611
+ const error = pend(report.error, TEXT_CAPS.reportError);
612
+ this.enqueue({
613
+ lane: EXECUTIONS,
614
+ kind: "report",
615
+ bytes: sizeOf(prompt, error) + 1024,
616
+ send: async () => {
617
+ const body = await this.request("POST", `/scenarios/${report.scenarioId}/executions`, JSON.stringify({
618
+ ticket: report.ticket,
619
+ outcome: report.outcome,
620
+ costUsd,
621
+ deriveCostUsd: report.deriveCostUsd,
622
+ sessionCostUsd: report.sessionCostUsd,
623
+ fallbackCostUsd: report.fallbackCostUsd,
624
+ durationMs: report.durationMs,
625
+ stepsPlanned: report.stepsPlanned,
626
+ stepsPinned: report.stepsPinned,
627
+ measured: report.measured,
628
+ pricingVersion: report.pricingVersion,
629
+ prompt: out(prompt),
630
+ // Per-step verdicts and the headline failure (errorshandling.md). All
631
+ // optional on the service side, so an older service ignores them rather
632
+ // than rejecting the report — and the costs still book. Errors are
633
+ // shown on a web page, so they are capped and redacted like any text.
634
+ steps: report.steps?.map((s) => typeof s.error === "string" ? { ...s, error: capText(s.error, TEXT_CAPS.reportError, redactText) } : s),
635
+ error: out(error),
636
+ errorStage: report.errorStage,
637
+ errorStepIndex: report.errorStepIndex,
638
+ errorToolName: report.errorToolName,
639
+ fallbackStepIndex: report.fallbackStepIndex,
640
+ fallbackKind: report.fallbackKind,
641
+ baselineEligible: report.baselineEligible,
642
+ sharedWith: report.sharedWith,
643
+ // Why nothing ran, when nothing ran. Additive: an older service drops
644
+ // the field and books the cost exactly as it always did.
645
+ declined: report.declined,
646
+ }), { retryTransient: Boolean(report.ticket) });
647
+ const r = (body ?? {});
648
+ const failed = report.steps?.filter((s) => s.status === "failed").length ?? 0;
649
+ logLine("execution.reported", {
650
+ run: report.runId,
651
+ scenario: report.scenarioId,
652
+ ticket: report.ticket,
653
+ outcome: report.outcome,
654
+ derive: report.deriveCostUsd.toFixed(4),
655
+ session: report.sessionCostUsd.toFixed(4),
656
+ fallback: report.fallbackCostUsd.toFixed(4),
657
+ savedUsd: r.savedUsd ?? undefined,
658
+ duplicate: r.duplicate ? true : undefined,
659
+ measured: report.measured,
660
+ steps: report.steps?.length,
661
+ stepsFailed: failed || undefined,
662
+ baselineEligible: report.baselineEligible === false ? false : undefined,
663
+ sharedWith: report.sharedWith?.length,
664
+ declined: report.declined,
665
+ });
666
+ },
667
+ });
668
+ }
669
+ /**
670
+ * Await every queued send for a run, including its execution report — for at
671
+ * most `opts.timeoutMs` (default {@link DEFAULT_FLUSH_TIMEOUT_MS}, or
672
+ * `BIR_FLUSH_TIMEOUT_MS`), and not at all while the service is cooling down.
673
+ * Either way the sends carry on in the background; a flush behind a dead
674
+ * service must never be what holds a hook or a shutdown.
675
+ *
676
+ * WHY THE BREAKER ENDS THE WAIT. In the common outages — offline, refused,
677
+ * a 5xx — the first send fails in milliseconds and opens the breaker, and
678
+ * every job after it sleeps out the cool-down before it tries (the service
679
+ * is not hammered). A flush that waited for those sleeps paid its whole
680
+ * deadline on every turn of the outage, where the old fail-at-once chain
681
+ * paid tens of milliseconds. Nothing a cooling-down lane holds can land
682
+ * sooner than the cool-down, so the flush stops waiting the moment the
683
+ * breaker opens.
684
+ */
685
+ async flush(runId, opts = {}) {
686
+ const timeoutMs = opts.timeoutMs ?? positiveEnv("BIR_FLUSH_TIMEOUT_MS", DEFAULT_FLUSH_TIMEOUT_MS);
687
+ const outcome = await this.waitForLanes([runId, EXECUTIONS], timeoutMs);
688
+ const entry = this.runs.get(runId);
689
+ // The match holds the whole scenario; nobody asks for it after a flush.
690
+ if (entry?.settled?.match)
691
+ entry.settled = { match: null, created: entry.settled.created };
692
+ if (outcome === "timeout") {
693
+ logLine("recorder.flush_incomplete", {
694
+ run: runId,
695
+ pending: this.pendingSends,
696
+ why: "the service is slow or down — the rest is sent in the background, or dropped past the backlog bound",
218
697
  });
219
- const r = (body ?? {});
220
- const failed = report.steps?.filter((s) => s.status === "failed").length ?? 0;
221
- logLine("execution.reported", {
222
- run: report.runId,
223
- scenario: report.scenarioId,
224
- ticket: report.ticket,
225
- outcome: report.outcome,
226
- derive: report.deriveCostUsd.toFixed(4),
227
- session: report.sessionCostUsd.toFixed(4),
228
- fallback: report.fallbackCostUsd.toFixed(4),
229
- savedUsd: r.savedUsd ?? undefined,
230
- duplicate: r.duplicate ? true : undefined,
231
- measured: report.measured,
232
- steps: report.steps?.length,
233
- stepsFailed: failed || undefined,
234
- baselineEligible: report.baselineEligible === false ? false : undefined,
235
- sharedWith: report.sharedWith?.length,
236
- declined: report.declined,
698
+ }
699
+ else if (outcome === "cooling") {
700
+ // Once per 30 s: an outage is one fact, not one line per turn.
701
+ logEvery("recorder.flush_deferred", 30_000, "recorder.flush_deferred", {
702
+ run: runId,
703
+ pending: this.pendingSends,
704
+ why: "the service is cooling down after a failure — nothing waits on it; the rest is sent in the background once it answers",
237
705
  });
706
+ }
707
+ }
708
+ // ── lanes ────────────────────────────────────────────────────────────────
709
+ newEntry() {
710
+ let resolveHot;
711
+ let resolveCreated;
712
+ const entry = {
713
+ userId: this.session.user.id,
714
+ hot: new Promise((r) => {
715
+ resolveHot = r;
716
+ }),
717
+ created: new Promise((r) => {
718
+ resolveCreated = r;
719
+ }),
720
+ settle: (match, created) => {
721
+ if (entry.settled)
722
+ return;
723
+ entry.settled = { match, created };
724
+ entry.settledAt = Date.now();
725
+ resolveHot();
726
+ resolveCreated(created);
727
+ // Callers that are waiting hold the promises themselves.
728
+ entry.hot = undefined;
729
+ entry.created = undefined;
730
+ },
731
+ release: () => {
732
+ if (entry.settled || entry.released)
733
+ return;
734
+ entry.released = true;
735
+ entry.hot = undefined;
736
+ resolveHot();
737
+ },
738
+ lastUsed: Date.now(),
739
+ failed: 0,
740
+ dropped: 0,
741
+ };
742
+ return entry;
743
+ }
744
+ /** The entry for a run, made on first sight for an id this recorder did not open. */
745
+ entryFor(runId) {
746
+ let entry = this.runs.get(runId);
747
+ if (!entry) {
748
+ entry = this.newEntry();
749
+ entry.settle(null, { created: true });
750
+ this.runs.set(runId, entry);
751
+ }
752
+ entry.lastUsed = Date.now();
753
+ return entry;
754
+ }
755
+ enqueueRun(runId, kind, bytes, send) {
756
+ this.entryFor(runId);
757
+ this.enqueue({
758
+ lane: runId,
759
+ runId,
760
+ kind,
761
+ bytes,
762
+ send: async () => {
763
+ await send();
764
+ },
238
765
  });
239
766
  }
240
- /** Await every queued send for a run, including its execution report. */
241
- async flush(runId) {
242
- await (this.chains.get(runId) ?? Promise.resolve());
243
- this.chains.delete(runId);
244
- this.matches.delete(runId);
245
- this.created.delete(runId);
246
- // Reports live on their own chain (a matched run has no run chain of its own
247
- // — the service created no run), so draining that chain is a separate step.
248
- await (this.chains.get(this.executionChainKey) ?? Promise.resolve());
249
- }
250
- // ── internals ────────────────────────────────────────────────────────────
251
- /** Append to this run's ordered chain; failures are logged, not thrown. */
252
- enqueue(runId, fn) {
253
- const prev = this.chains.get(runId) ?? Promise.resolve();
254
- const next = prev.then(fn).catch((err) => {
255
- // A dropped send means this run's recording is incomplete — something an
256
- // auditor has to be able to see, so it is never verbose-only.
257
- logLine("recorder.send_failed", {
258
- run: runId,
767
+ /** Append to a lane. Synchronous and never throws; the lane drains after the caller returns. */
768
+ enqueue(spec) {
769
+ const run = spec.runId ? this.runs.get(spec.runId) : undefined;
770
+ const job = { ...spec, seq: ++this.seq, userId: run?.userId ?? this.session.user.id };
771
+ if (this.state !== "recording")
772
+ return this.retire(job, run, this.state);
773
+ if (run?.dead)
774
+ return this.retire(job, run, run.dead);
775
+ let lane = this.lanes.get(job.lane);
776
+ if (!lane) {
777
+ lane = { jobs: [], running: false, waiters: [] };
778
+ this.lanes.set(job.lane, lane);
779
+ }
780
+ lane.jobs.push(job);
781
+ this.pendingSends += 1;
782
+ this.pendingBytes += job.bytes;
783
+ this.enforceBound();
784
+ if (!lane.running) {
785
+ lane.running = true;
786
+ const started = lane;
787
+ // Next turn of the event loop: redaction and serialization happen off
788
+ // the hook's own path, after its answer is on the wire.
789
+ setImmediate(() => void this.drain(job.lane, started));
790
+ }
791
+ }
792
+ async drain(key, lane) {
793
+ try {
794
+ for (;;) {
795
+ const job = lane.jobs.shift();
796
+ if (!job)
797
+ break;
798
+ this.pendingSends -= 1;
799
+ this.pendingBytes -= job.bytes;
800
+ this.noteBacklogRecovered();
801
+ await this.execute(job);
802
+ }
803
+ }
804
+ catch {
805
+ /* execute never throws; belt and braces for the lane's own bookkeeping */
806
+ }
807
+ finally {
808
+ if (lane.jobs.length > 0) {
809
+ setImmediate(() => void this.drain(key, lane));
810
+ }
811
+ else {
812
+ lane.running = false;
813
+ if (this.lanes.get(key) === lane)
814
+ this.lanes.delete(key);
815
+ for (const resolve of lane.waiters.splice(0))
816
+ resolve();
817
+ }
818
+ }
819
+ }
820
+ laneIsIdle(key) {
821
+ const lane = this.lanes.get(key);
822
+ return !lane || (!lane.running && lane.jobs.length === 0);
823
+ }
824
+ /**
825
+ * Wait until every lane in `keys` is idle, the breaker opens ("cooling"), or
826
+ * `timeoutMs` passes — whichever is first. The breaker is polled (every
827
+ * {@link FLUSH_POLL_MS}) rather than subscribed to: the verdict is shared by
828
+ * the whole process and has no listeners, and a poll only runs while a flush
829
+ * is actually waiting.
830
+ */
831
+ waitForLanes(keys, timeoutMs) {
832
+ if (keys.every((k) => this.laneIsIdle(k)))
833
+ return Promise.resolve("idle");
834
+ if (!this.health.canWait())
835
+ return Promise.resolve("cooling");
836
+ return new Promise((resolve) => {
837
+ const cancels = [];
838
+ let settled = false;
839
+ const finish = (outcome) => {
840
+ if (settled)
841
+ return;
842
+ settled = true;
843
+ clearTimeout(deadline);
844
+ clearInterval(poll);
845
+ for (const cancel of cancels)
846
+ cancel();
847
+ resolve(outcome);
848
+ };
849
+ // Not unref'd: a caller awaiting this must get its answer even when the
850
+ // only other thing left is a lane asleep in an unref'd cool-down.
851
+ const deadline = setTimeout(() => finish("timeout"), Math.max(0, timeoutMs));
852
+ const poll = setInterval(() => {
853
+ if (!this.health.canWait())
854
+ finish("cooling");
855
+ }, FLUSH_POLL_MS);
856
+ let left = keys.length;
857
+ for (const key of keys) {
858
+ const lane = this.lanes.get(key);
859
+ if (!lane || (!lane.running && lane.jobs.length === 0)) {
860
+ left -= 1;
861
+ continue;
862
+ }
863
+ const waiter = () => {
864
+ left -= 1;
865
+ if (left === 0)
866
+ finish("idle");
867
+ };
868
+ lane.waiters.push(waiter);
869
+ cancels.push(() => {
870
+ const i = lane.waiters.indexOf(waiter);
871
+ if (i >= 0)
872
+ lane.waiters.splice(i, 1);
873
+ });
874
+ }
875
+ if (left === 0)
876
+ finish("idle");
877
+ });
878
+ }
879
+ /** Run one job: decide whether it may still go, wait out a cool-down, send. Never throws. */
880
+ async execute(job) {
881
+ const run = job.runId ? this.runs.get(job.runId) : undefined;
882
+ let endProbe;
883
+ try {
884
+ let why = await this.whyNotSend(job, run);
885
+ if (why)
886
+ return this.retire(job, run, why);
887
+ // Don't hammer a service that just failed: wait out its cool-down, and
888
+ // then let one send (maybe this one) find out whether it came back.
889
+ const wait = await this.waitForService();
890
+ endProbe = wait.endProbe;
891
+ if (wait.giveUp) {
892
+ this.retire(job, run, "service_down");
893
+ // Without its create the rest of the run would 404.
894
+ if (job.kind === "create" && run && job.runId)
895
+ this.markDead(job.runId, run, "service_down");
896
+ return;
897
+ }
898
+ await this.refreshAhead();
899
+ // Both can take a while (a minute of cool-down, a refresh), and a hook
900
+ // may have signed another account in meanwhile: ask again, or a run
901
+ // opened under one account would go out under the next one's token.
902
+ why = await this.whyNotSend(job, run);
903
+ if (why)
904
+ return this.retire(job, run, why);
905
+ await job.send();
906
+ if (job.kind === "finish" && run && job.runId)
907
+ this.summarize(job.runId, run);
908
+ }
909
+ catch (err) {
910
+ if (err instanceof AccountChanged) {
911
+ this.retire(job, run, this.accountChanged(job, run));
912
+ return;
913
+ }
914
+ this.onSendFailed(job, run, err);
915
+ if (job.kind === "finish" && run && job.runId)
916
+ this.summarize(job.runId, run);
917
+ }
918
+ finally {
919
+ endProbe?.();
920
+ }
921
+ }
922
+ async whyNotSend(job, run) {
923
+ this.checkCredentials();
924
+ if (this.state !== "recording")
925
+ return this.state;
926
+ if (job.userId !== undefined && job.userId !== this.session.user.id)
927
+ return this.accountChanged(job, run);
928
+ if (!run || !job.runId)
929
+ return undefined;
930
+ if (run.userId !== this.session.user.id)
931
+ this.markDead(job.runId, run, "account_changed");
932
+ if (run.dead)
933
+ return run.dead;
934
+ if (job.kind !== "create") {
935
+ // The create went first on this lane, so this is already settled — unless
936
+ // it is still being retried, which is worth the wait.
937
+ const created = run.settled?.created ?? (run.created ? await run.created : { created: true });
938
+ if (!created.created) {
939
+ this.markDead(job.runId, run, "not_created");
940
+ return "not_created";
941
+ }
942
+ if (run.dead)
943
+ return run.dead;
944
+ }
945
+ return undefined;
946
+ }
947
+ /**
948
+ * A job queued under one account, about to go out after another signed in:
949
+ * its run is closed (the rest would split it across accounts), and a report
950
+ * is dropped with a line. Returns the reason to retire it with.
951
+ */
952
+ accountChanged(job, run) {
953
+ if (run && job.runId) {
954
+ this.markDead(job.runId, run, "account_changed");
955
+ }
956
+ else {
957
+ logEvery("recorder.report_dropped", 30_000, "recorder.report_dropped", {
958
+ why: "another account signed in on this machine before it was sent — it is not sent under that one",
959
+ });
960
+ }
961
+ return "account_changed";
962
+ }
963
+ /** A job leaves without being sent: counted (unless it was never meant to go), and a create settles. */
964
+ retire(job, run, why) {
965
+ try {
966
+ job.abandon?.();
967
+ }
968
+ catch {
969
+ /* never */
970
+ }
971
+ // A matched turn records nothing by design: nothing was lost.
972
+ if (why !== "not_created") {
973
+ this.droppedSends += 1;
974
+ if (run)
975
+ run.dropped += 1;
976
+ }
977
+ if (job.kind === "finish" && run && job.runId)
978
+ this.summarize(job.runId, run);
979
+ }
980
+ /** Skip everything left for a run, once: its sends would all fail the same way. */
981
+ markDead(runId, run, why) {
982
+ if (run.dead)
983
+ return;
984
+ run.dead = why;
985
+ const lane = this.lanes.get(runId);
986
+ if (lane && lane.jobs.length > 0) {
987
+ const jobs = lane.jobs;
988
+ lane.jobs = [];
989
+ for (const job of jobs) {
990
+ this.pendingSends -= 1;
991
+ this.pendingBytes -= job.bytes;
992
+ this.retire(job, run, why);
993
+ }
994
+ }
995
+ if (why === "not_created") {
996
+ logDetail("recorder.run_not_created", { run: runId, why: "the service matched the prompt and kept its own run" });
997
+ }
998
+ else if (!(why in PAUSE_WHY)) {
999
+ logLine("recorder.run_abandoned", { run: runId, reason: why, why: DEAD_WHY[why] ?? why });
1000
+ }
1001
+ }
1002
+ onSendFailed(job, run, err) {
1003
+ this.failedSends += 1;
1004
+ const status = err instanceof ServiceError ? err.status : undefined;
1005
+ if (run && job.runId) {
1006
+ run.failed += 1;
1007
+ if (job.kind === "create") {
1008
+ // A definite refusal means there is no run; an ambiguous failure (a
1009
+ // timeout, a 5xx) may have created it, and the first step will say.
1010
+ if (status !== undefined && status >= 400 && status < 500 && ![401, 408, 409, 429].includes(status)) {
1011
+ this.markDead(job.runId, run, "create_rejected");
1012
+ }
1013
+ }
1014
+ else if (status === 404) {
1015
+ this.markDead(job.runId, run, "not_found");
1016
+ }
1017
+ // One line per run, not per step: an outage is one fact, and the count
1018
+ // arrives with the run's finish (recorder.run_incomplete).
1019
+ if (run.failed === 1) {
1020
+ logLine("recorder.send_failed", {
1021
+ run: job.runId,
1022
+ why: "this run's recording is incomplete",
1023
+ error: errText(err),
1024
+ });
1025
+ }
1026
+ }
1027
+ else {
1028
+ logEvery(`recorder.send_failed:${job.lane}`, 30_000, "recorder.send_failed", {
1029
+ run: job.lane,
259
1030
  why: "this run's recording is incomplete",
260
1031
  error: errText(err),
261
1032
  });
1033
+ }
1034
+ }
1035
+ /** At a run's finish: say what did not arrive, if anything did not. */
1036
+ summarize(runId, run) {
1037
+ if (run.failed <= 1 && run.dropped === 0)
1038
+ return;
1039
+ logLine("recorder.run_incomplete", {
1040
+ run: runId,
1041
+ failed: run.failed || undefined,
1042
+ dropped: run.dropped || undefined,
1043
+ reason: run.dead,
1044
+ why: "some of this run's steps never reached the service",
262
1045
  });
263
- this.chains.set(runId, next);
264
1046
  }
265
- post(path, body) {
266
- return this.send("POST", path, body);
1047
+ /** Keep the backlog inside its bound: drop the oldest unsent step first. */
1048
+ enforceBound() {
1049
+ while (this.pendingSends > this.maxPendingSends || this.pendingBytes > this.maxPendingBytes) {
1050
+ const victim = this.pickVictim();
1051
+ if (!victim)
1052
+ return;
1053
+ const lane = this.lanes.get(victim.lane);
1054
+ const index = lane ? lane.jobs.indexOf(victim) : -1;
1055
+ if (!lane || index < 0)
1056
+ return;
1057
+ lane.jobs.splice(index, 1);
1058
+ this.pendingSends -= 1;
1059
+ this.pendingBytes -= victim.bytes;
1060
+ this.droppedWhileFull += 1;
1061
+ if (!this.backlogFull) {
1062
+ this.backlogFull = true;
1063
+ logLine("recorder.backlog_full", {
1064
+ pending: this.pendingSends,
1065
+ bytes: this.pendingBytes,
1066
+ why: "the service is not keeping up — the oldest unsent steps are dropped, so those runs are incomplete",
1067
+ });
1068
+ }
1069
+ const run = victim.runId ? this.runs.get(victim.runId) : undefined;
1070
+ this.retire(victim, run, "dropped");
1071
+ // Without its create the rest of the run would 404.
1072
+ if (victim.kind === "create" && run && victim.runId)
1073
+ this.markDead(victim.runId, run, "dropped");
1074
+ }
267
1075
  }
268
- patch(path, body) {
269
- return this.send("PATCH", path, body);
1076
+ /**
1077
+ * The oldest unsent step or patch; failing that, the oldest job of any kind.
1078
+ * Steps first because a run without its middle is still a run, and one
1079
+ * without its create or finish is a run the service can never close.
1080
+ */
1081
+ pickVictim() {
1082
+ let best;
1083
+ for (const lane of this.lanes.values()) {
1084
+ for (const job of lane.jobs) {
1085
+ if (job.kind === "step" || job.kind === "patch") {
1086
+ if (!best || job.seq < best.seq)
1087
+ best = job;
1088
+ break; // lanes are FIFO: the first is the oldest
1089
+ }
1090
+ }
1091
+ }
1092
+ if (best)
1093
+ return best;
1094
+ for (const lane of this.lanes.values()) {
1095
+ const first = lane.jobs[0];
1096
+ if (first && (!best || first.seq < best.seq))
1097
+ best = first;
1098
+ }
1099
+ return best;
270
1100
  }
271
- async send(method, path, body, retried = false) {
272
- const res = await fetch(`${this.baseUrl}${path}`, {
273
- method,
274
- headers: {
275
- "content-type": "application/json",
276
- authorization: `Bearer ${this.session.accessToken}`,
277
- },
278
- body: JSON.stringify(body),
279
- });
280
- if (res.status === 401 && !retried) {
281
- await this.refresh();
282
- return this.send(method, path, body, true);
1101
+ noteBacklogRecovered() {
1102
+ if (!this.backlogFull)
1103
+ return;
1104
+ if (this.pendingSends > this.maxPendingSends / 2 || this.pendingBytes > this.maxPendingBytes / 2)
1105
+ return;
1106
+ this.backlogFull = false;
1107
+ logLine("recorder.backlog_recovered", { dropped: this.droppedWhileFull });
1108
+ this.droppedWhileFull = 0;
1109
+ }
1110
+ /** Forget runs nobody will ask about again, and release matches past their TTL. */
1111
+ sweep() {
1112
+ const now = Date.now();
1113
+ if (now - this.lastSweep < 60_000 && this.runs.size <= RUN_STATE_MAX)
1114
+ return;
1115
+ this.lastSweep = now;
1116
+ for (const [id, entry] of this.runs) {
1117
+ if (this.lanes.has(id))
1118
+ continue;
1119
+ if (entry.settled?.match && entry.settledAt !== undefined && now - entry.settledAt > MATCH_TTL_MS) {
1120
+ entry.settled = { match: null, created: entry.settled.created };
1121
+ }
1122
+ if (entry.settled && now - entry.lastUsed > RUN_STATE_TTL_MS)
1123
+ this.runs.delete(id);
283
1124
  }
284
- const text = await res.text().catch(() => "");
285
- if (!res.ok) {
286
- throw new Error(`HTTP ${res.status} ${method} ${path}${text ? ` — ${text}` : ""}`);
1125
+ if (this.runs.size > RUN_STATE_MAX) {
1126
+ for (const [id, entry] of this.runs) {
1127
+ if (this.runs.size <= RUN_STATE_MAX)
1128
+ break;
1129
+ if (!this.lanes.has(id) && entry.settled)
1130
+ this.runs.delete(id);
1131
+ }
287
1132
  }
288
- if (!text)
289
- return undefined;
1133
+ }
1134
+ // ── the service ──────────────────────────────────────────────────────────
1135
+ /** The breaker is closed: no failure since the last answer. */
1136
+ healthyNow() {
1137
+ return this.health.canWait() && this.health.snapshot().consecutiveFailures === 0;
1138
+ }
1139
+ /**
1140
+ * May a hook wait on the service right now? Only when it is known to be
1141
+ * answering. A hook is never the half-open probe: once the cool-down ends a
1142
+ * background `GET /health` (or a queued send) finds out, and the next hook
1143
+ * waits again.
1144
+ */
1145
+ mayHotWait() {
1146
+ this.checkCredentials();
1147
+ if (this.state !== "recording")
1148
+ return false;
1149
+ if (this.healthyNow())
1150
+ return true;
1151
+ this.probeSoon();
1152
+ return false;
1153
+ }
1154
+ probeSoon() {
1155
+ if (this.probeInFlight() || !this.health.canWait())
1156
+ return;
1157
+ const end = this.beginProbe();
1158
+ this.request("GET", "/health", undefined, { timeoutMs: PROBE_TIMEOUT_MS, anonymous: true })
1159
+ .catch(() => undefined)
1160
+ .finally(end);
1161
+ }
1162
+ /** Take the half-open slot. The returned function gives it back, once. */
1163
+ beginProbe() {
1164
+ let resolve;
1165
+ const flight = { done: new Promise((r) => (resolve = r)), startedAt: Date.now() };
1166
+ this.probe = flight;
1167
+ return () => {
1168
+ if (this.probe === flight)
1169
+ this.probe = undefined;
1170
+ resolve();
1171
+ };
1172
+ }
1173
+ /**
1174
+ * Is a probe out? One older than any request could take (every attempt has
1175
+ * a deadline) is a slot somebody failed to give back, and is not honoured.
1176
+ */
1177
+ probeInFlight() {
1178
+ const p = this.probe;
1179
+ if (!p)
1180
+ return false;
1181
+ if (Date.now() - p.startedAt > 4 * serviceTimeoutMs() + PROBE_TIMEOUT_MS) {
1182
+ this.probe = undefined;
1183
+ return false;
1184
+ }
1185
+ return true;
1186
+ }
1187
+ /** Lanes let go after a successful probe spread over a fraction of the cool-down (at most 2 s). */
1188
+ releaseJitterMs() {
1189
+ const spread = Math.min(RELEASE_JITTER_MAX_MS, Math.floor(this.health.baseCooldown / 2));
1190
+ return Math.floor(Math.random() * Math.max(0, spread));
1191
+ }
1192
+ /**
1193
+ * Wait until a background send may go.
1194
+ *
1195
+ * - The breaker is closed: go at once.
1196
+ * - It is cooling down: sleep until it ends. A send that has waited
1197
+ * {@link COOLDOWN_WAIT_MAX_MS} and still faces more than
1198
+ * {@link COOLDOWN_GRACE_MS} of it is given up (`giveUp`), unsent.
1199
+ * - It is half-open: ONE lane takes the probe slot and goes (`endProbe`
1200
+ * gives the slot back once its send settles); every other lane waits for
1201
+ * that verdict. A success lets them go, each after its own short random
1202
+ * pause; a failure starts a longer cool-down, and they wait that out.
1203
+ *
1204
+ * Before this, every lane with a queued send woke at the same moment and
1205
+ * sent into the dead service together, once per cool-down.
1206
+ */
1207
+ async waitForService() {
1208
+ const started = Date.now();
1209
+ let waited = false;
1210
+ for (;;) {
1211
+ if (this.state !== "recording")
1212
+ return {};
1213
+ if (this.healthyNow()) {
1214
+ if (!waited)
1215
+ return {};
1216
+ // It came back while this lane waited: follow the probe a moment
1217
+ // later, not together with every other lane.
1218
+ waited = false;
1219
+ await sleep(this.releaseJitterMs(), true);
1220
+ continue;
1221
+ }
1222
+ waited = true;
1223
+ if (this.health.canWait()) {
1224
+ if (!this.probeInFlight())
1225
+ return { endProbe: this.beginProbe() };
1226
+ await Promise.race([this.probe?.done, sleep(PROBE_POLL_MS, true)]);
1227
+ continue;
1228
+ }
1229
+ const left = this.cooldownWaitMaxMs - (Date.now() - started);
1230
+ const cooling = (this.health.snapshot().openUntil ?? 0) - Date.now();
1231
+ if (left <= 0 && cooling > Math.min(COOLDOWN_GRACE_MS, this.cooldownWaitMaxMs / 12))
1232
+ return { giveUp: true };
1233
+ // Past the bound, only a cool-down about to end is still waited out.
1234
+ await sleep(Math.min(Math.max(cooling, 25), left > 0 ? left : Number.POSITIVE_INFINITY, 5_000), true);
1235
+ }
1236
+ }
1237
+ /** Renew the access token shortly before it expires — in a lane, never on a hook. */
1238
+ async refreshAhead() {
1239
+ if (this.session.accessExpiresAt - REFRESH_AHEAD_MS > Date.now())
1240
+ return;
1241
+ if (Date.now() - this.lastAheadAttempt < REFRESH_AHEAD_RETRY_MS)
1242
+ return;
1243
+ this.lastAheadAttempt = Date.now();
290
1244
  try {
291
- return JSON.parse(text);
1245
+ await this.refreshSingleFlight();
292
1246
  }
293
1247
  catch {
294
- return undefined;
1248
+ /* the send meets its 401 and says so */
1249
+ }
1250
+ }
1251
+ post(path, body) {
1252
+ return this.send("POST", path, body);
1253
+ }
1254
+ /** A background send: every lane job is safe to repeat once (client ids, a PATCH, a ticket). */
1255
+ send(method, path, body) {
1256
+ return this.request(method, path, JSON.stringify(body), { retryTransient: true });
1257
+ }
1258
+ /**
1259
+ * One request, with a deadline, reported to the health verdict, with the
1260
+ * one-refresh-on-401 rule and — for background sends — one retry after a
1261
+ * transient failure.
1262
+ *
1263
+ * WHICH FAILURES ARE RETRIED. Only those that prove the service did not take
1264
+ * the request: a connection that never happened or a pooled socket the load
1265
+ * balancer had already closed ({@link SAFE_RETRY_CODES}), and a 429 or 503.
1266
+ * Not a 502/504 or a reset mid-request — the origin may have the first
1267
+ * attempt — not a timeout (a hung service would just cost another deadline)
1268
+ * and not while the service is already failing (the breaker's cool-down is
1269
+ * the retry then). Every background body carries a client-generated id — the
1270
+ * run, the step, the execution ticket — or is a PATCH of one field, so even
1271
+ * a repeat whose first attempt did land is recognisable; a 409 on the retry
1272
+ * is taken as that.
1273
+ *
1274
+ * THE ACCOUNT IS FIXED FOR THE REQUEST. A retry or a refresh leaves the event
1275
+ * loop, and a hook may sign another account in meanwhile; the next attempt
1276
+ * then goes nowhere ({@link AccountChanged}) instead of out under that
1277
+ * account's token.
1278
+ *
1279
+ * EVERY ANSWER SAYS WHEN ITS ATTEMPT WAS SENT. `health.ok(started)`: the late
1280
+ * answer of an attempt a hook already gave up on (`slow`) must not end the
1281
+ * cool-down that overrun started.
1282
+ *
1283
+ * `onRetry` runs when the first attempt failed and a retry is about to
1284
+ * sleep: whoever waits on this answer on a hook's behalf can stop now.
1285
+ */
1286
+ async request(method, path, body, opts = {}) {
1287
+ let retried401 = false;
1288
+ let retriedTransient = false;
1289
+ const user = this.session.user.id;
1290
+ for (let attempt = 0;; attempt++) {
1291
+ if (attempt > 0 && !opts.anonymous && this.session.user.id !== user) {
1292
+ throw new AccountChanged(`${method} ${path}`);
1293
+ }
1294
+ const token = this.session.accessToken;
1295
+ const timeoutMs = opts.timeoutMs ?? serviceTimeoutMs();
1296
+ const signal = timeoutSignal(timeoutMs, opts.signal);
1297
+ const started = this.health.stamp();
1298
+ let res;
1299
+ try {
1300
+ res = await fetch(`${this.baseUrl}${path}`, {
1301
+ method,
1302
+ headers: {
1303
+ ...(body !== undefined ? { "content-type": "application/json" } : {}),
1304
+ ...(opts.anonymous ? {} : { authorization: `Bearer ${token}` }),
1305
+ },
1306
+ body,
1307
+ signal,
1308
+ });
1309
+ }
1310
+ catch (err) {
1311
+ if (opts.signal?.aborted)
1312
+ throw err; // the caller stopped waiting; says nothing about the service
1313
+ const timedOut = signal.aborted;
1314
+ const reason = timedOut ? `timed out after ${timeoutMs} ms` : reasonOf(err);
1315
+ const safe = !timedOut && SAFE_RETRY_CODES.has(causeCode(err) ?? "");
1316
+ if (safe && opts.retryTransient && !retriedTransient && this.healthyNow()) {
1317
+ retriedTransient = true;
1318
+ notify(opts.onRetry);
1319
+ await sleep(jitterMs());
1320
+ continue;
1321
+ }
1322
+ this.health.fail(reason);
1323
+ throw new ServiceError(`${method} ${path}: ${reason}`);
1324
+ }
1325
+ if (res.status === 401 && !opts.anonymous) {
1326
+ // Release the socket now rather than at GC, then renew — once.
1327
+ await res.body?.cancel().catch(() => undefined);
1328
+ this.health.ok(started);
1329
+ if (retried401)
1330
+ throw new ServiceError(`HTTP 401 ${method} ${path}`, 401);
1331
+ retried401 = true;
1332
+ // Another send (or a sibling process) already renewed the token this
1333
+ // one used: just resend with the new one. The wait for a renewal is
1334
+ // held to this attempt's own deadline — a probe's caller is a hook.
1335
+ if (this.session.accessToken === token) {
1336
+ try {
1337
+ await untilAborted(this.refreshSingleFlight(), signal);
1338
+ }
1339
+ catch (err) {
1340
+ if (opts.signal?.aborted || !signal.aborted)
1341
+ throw err;
1342
+ throw new ServiceError(`HTTP 401 ${method} ${path}: the token refresh outlasted ${timeoutMs} ms`, 401);
1343
+ }
1344
+ }
1345
+ if (this.state !== "recording")
1346
+ throw new ServiceError(`HTTP 401 ${method} ${path}`, 401);
1347
+ continue;
1348
+ }
1349
+ const type = res.headers.get("content-type") ?? "";
1350
+ let text;
1351
+ try {
1352
+ text = await res.text();
1353
+ }
1354
+ catch (err) {
1355
+ if (opts.signal?.aborted)
1356
+ throw err;
1357
+ const reason = signal.aborted ? `timed out after ${timeoutMs} ms reading the answer` : reasonOf(err);
1358
+ this.health.fail(reason);
1359
+ throw new ServiceError(`${method} ${path}: ${reason}`, res.status);
1360
+ }
1361
+ if (isUnhealthyStatus(res.status)) {
1362
+ if (opts.retryTransient && !retriedTransient && RETRY_STATUSES.has(res.status) && this.healthyNow()) {
1363
+ retriedTransient = true;
1364
+ notify(opts.onRetry);
1365
+ await sleep(retryDelayMs(res));
1366
+ continue;
1367
+ }
1368
+ this.health.fail(`HTTP ${res.status}`);
1369
+ throw new ServiceError(`HTTP ${res.status} ${method} ${path}${detail(text, type)}`, res.status);
1370
+ }
1371
+ // A 2xx web page is a captive portal or a website, and a 403 one a CDN's
1372
+ // challenge — not the API answering. An empty or plain 2xx is: only the
1373
+ // body decides, not a `text/html` a framework put on `res.send("ok")`.
1374
+ if ((res.ok || res.status === 403) && isWebPage(text, type)) {
1375
+ this.health.fail("html");
1376
+ throw new ServiceError(`HTTP ${res.status} ${method} ${path} — the answer was a web page, not the BaseIn API`, res.status);
1377
+ }
1378
+ // Anything else is the service answering, including a 4xx — an older
1379
+ // service's 404 must not look like an outage.
1380
+ this.health.ok(started);
1381
+ if (!res.ok) {
1382
+ if (res.status === 409 && retriedTransient)
1383
+ return undefined; // the first attempt landed
1384
+ throw new ServiceError(`HTTP ${res.status} ${method} ${path}${detail(text, type)}`, res.status);
1385
+ }
1386
+ if (!text)
1387
+ return undefined;
1388
+ try {
1389
+ return JSON.parse(text);
1390
+ }
1391
+ catch {
1392
+ return undefined;
1393
+ }
295
1394
  }
296
1395
  }
1396
+ // ── the session ──────────────────────────────────────────────────────────
1397
+ /**
1398
+ * Refresh once for everyone. Concurrent 401s — a run's create, an execution
1399
+ * report and a probe crossing the expiry together — used to each spend the
1400
+ * rotating refresh token; one won, the others were rejected and their sends
1401
+ * (a create: the whole turn) were dropped.
1402
+ */
1403
+ refreshSingleFlight() {
1404
+ if (!this.refreshing) {
1405
+ this.refreshing = this.refresh().finally(() => {
1406
+ this.refreshing = undefined;
1407
+ });
1408
+ }
1409
+ return this.refreshing;
1410
+ }
297
1411
  /**
298
1412
  * Adopt a newer session written by a sibling process, if there is one.
299
1413
  *
300
1414
  * Returns true when this recorder now holds a usable token it did not have to
301
- * spend a refresh on.
1415
+ * spend a refresh on. Only a session for the same service and the same
1416
+ * account is adopted: the binding is what keeps a token from being sent to
1417
+ * an address it was not issued for, and an account change is a run boundary
1418
+ * ({@link checkCredentials}), never a mid-run swap.
302
1419
  */
303
1420
  adoptStoredSession() {
304
1421
  const stored = loadCredentials();
305
1422
  if (!stored)
306
1423
  return false;
1424
+ if (!issuedFor(stored, this.binding))
1425
+ return false;
1426
+ if (stored.user.id !== this.session.user.id)
1427
+ return false;
307
1428
  if (stored.accessToken === this.session.accessToken)
308
1429
  return false;
309
- if (stored.accessExpiresAt - 30_000 <= Date.now())
1430
+ if (stored.accessExpiresAt - 30_000 <= Date.now()) {
1431
+ // Expired, but its refresh token is newer than ours: spend that one.
1432
+ if (stored.refreshToken !== this.session.refreshToken) {
1433
+ this.session = { ...this.session, refreshToken: stored.refreshToken };
1434
+ }
310
1435
  return false;
311
- this.session = stored;
1436
+ }
1437
+ this.session = { ...stored, authUrl: stored.authUrl ?? this.binding };
1438
+ this.noteFileSeen();
312
1439
  logLine("auth.adopted", { why: "another process had already refreshed" });
313
1440
  return true;
314
1441
  }
@@ -325,37 +1452,139 @@ export class RemoteRecorder {
325
1452
  * The fix is to treat the credentials file as the shared source of truth:
326
1453
  * check it before spending our token, and check it again if our refresh is
327
1454
  * rejected, because the process that beat us has already written a good one.
1455
+ * A rejection with nothing to adopt pauses the recorder (`auth_rejected`)
1456
+ * instead of paying two round trips on every later send; a new sign-in
1457
+ * resumes it, and so does a retry every {@link AUTH_REJECTED_RETRY_MS}.
1458
+ *
1459
+ * WHAT COUNTS AS A REJECTION. Only the service's own JSON 400/401 — the
1460
+ * answer that says this refresh token is spent. A 403 or a web page (a CDN
1461
+ * challenge, a portal) says nothing about the token, and pausing on one
1462
+ * stopped recording for the rest of a daemon's life. And a rejection is
1463
+ * believed only after one more look at the file ~{@link AUTH_RECHECK_MS}
1464
+ * later: a sibling that rotated the token a moment ago may not have written
1465
+ * it yet. Any other failure is not retried for {@link REFRESH_RETRY_MS}, so a
1466
+ * refresh route that keeps failing costs one request per half minute, not
1467
+ * one per send.
328
1468
  */
329
1469
  async refresh() {
330
1470
  if (this.adoptStoredSession())
331
1471
  return;
1472
+ if (Date.now() < this.refreshRetryAt) {
1473
+ throw new ServiceError("refresh failed moments ago — not tried again yet", 401);
1474
+ }
1475
+ const spent = this.session.refreshToken;
332
1476
  try {
333
1477
  await this.refreshOverNetwork();
1478
+ this.refreshRetryAt = 0;
334
1479
  }
335
1480
  catch (err) {
336
1481
  // Someone may have refreshed while we were failing to.
337
1482
  if (this.adoptStoredSession())
338
1483
  return;
1484
+ if (err instanceof RefreshRejected) {
1485
+ await sleep(this.authRecheckMs, true);
1486
+ if (this.adoptStoredSession())
1487
+ return;
1488
+ // Adopted by a hook's credentials check while we waited, or signed
1489
+ // out: either way this refusal is no longer the news.
1490
+ if (this.session.refreshToken !== spent || this.state !== "recording")
1491
+ throw err;
1492
+ this.rejectedAt = Date.now();
1493
+ this.pause("auth_rejected");
1494
+ }
1495
+ else if (this.health.canWait()) {
1496
+ // The service answered, just not with a session (a JSON 403, a 404):
1497
+ // asking again on the next send would get the same. A failure that
1498
+ // opened the breaker needs no hold of its own — the cool-down is it.
1499
+ this.refreshRetryAt = Date.now() + REFRESH_RETRY_MS;
1500
+ }
339
1501
  throw err;
340
1502
  }
341
1503
  }
1504
+ /**
1505
+ * While `auth_rejected`, try the refresh again now and then, in the
1506
+ * background: a refusal the service later takes back (a bad deploy, a
1507
+ * clock), or one the recheck missed, must not need a `bir login` to end.
1508
+ * Never awaited by the caller — a hook's credentials check starts it.
1509
+ */
1510
+ retryRejected() {
1511
+ if (this.state !== "auth_rejected" || this.refreshing)
1512
+ return;
1513
+ if (Date.now() - this.rejectedAt < this.authRetryMs)
1514
+ return;
1515
+ this.rejectedAt = Date.now();
1516
+ this.refreshRetryAt = 0;
1517
+ this.refreshSingleFlight()
1518
+ .then(() => {
1519
+ if (this.state === "auth_rejected")
1520
+ this.resume("the service accepted the session again");
1521
+ })
1522
+ .catch(() => undefined);
1523
+ }
342
1524
  async refreshOverNetwork() {
343
- const res = await fetch(`${this.baseUrl}/auth/refresh`, {
344
- method: "POST",
345
- headers: { "content-type": "application/json" },
346
- body: JSON.stringify({ refreshToken: this.session.refreshToken }),
347
- });
348
- if (!res.ok)
349
- throw new Error(`refresh failed: HTTP ${res.status}`);
350
- const r = (await res.json());
1525
+ const timeoutMs = serviceTimeoutMs();
1526
+ const signal = timeoutSignal(timeoutMs);
1527
+ const started = this.health.stamp();
1528
+ let res;
1529
+ try {
1530
+ res = await fetch(`${this.baseUrl}/auth/refresh`, {
1531
+ method: "POST",
1532
+ headers: { "content-type": "application/json" },
1533
+ body: JSON.stringify({ refreshToken: this.session.refreshToken }),
1534
+ signal,
1535
+ });
1536
+ }
1537
+ catch (err) {
1538
+ const reason = signal.aborted ? `timed out after ${timeoutMs} ms` : reasonOf(err);
1539
+ this.health.fail(reason);
1540
+ throw new ServiceError(`refresh failed: ${reason}`);
1541
+ }
1542
+ const type = res.headers.get("content-type") ?? "";
1543
+ const text = await res.text().catch(() => "");
1544
+ if (!res.ok) {
1545
+ const page = isWebPage(text, type);
1546
+ if (isUnhealthyStatus(res.status))
1547
+ this.health.fail(`HTTP ${res.status}`);
1548
+ else if (page && res.status === 403)
1549
+ this.health.fail("html");
1550
+ else
1551
+ this.health.ok(started);
1552
+ if ((res.status === 400 || res.status === 401) && !page && isJsonObject(text)) {
1553
+ throw new RefreshRejected(`refresh refused: HTTP ${res.status}${detail(text, type)}`, res.status);
1554
+ }
1555
+ throw new ServiceError(`refresh failed: HTTP ${res.status}${page ? " (a web page)" : ""}`, res.status);
1556
+ }
1557
+ let body;
1558
+ try {
1559
+ body = JSON.parse(text);
1560
+ }
1561
+ catch {
1562
+ body = undefined;
1563
+ }
1564
+ if (!isAuthResponse(body)) {
1565
+ // Never saved: the next process would load a session with no token, call
1566
+ // the file corrupt, and the refresh token just rotated would be lost.
1567
+ const page = /html/i.test(type) || /^\s*</.test(text);
1568
+ if (page)
1569
+ this.health.fail("html");
1570
+ else
1571
+ this.health.ok(started);
1572
+ throw new ServiceError(`refresh failed: the answer was not a session${page ? " (a web page)" : ""} — nothing saved`);
1573
+ }
1574
+ this.health.ok(started);
351
1575
  this.session = {
352
- accessToken: r.accessToken,
353
- refreshToken: r.refreshToken,
354
- accessExpiresAt: Date.now() + r.expiresIn * 1000,
355
- user: r.user,
1576
+ accessToken: body.accessToken,
1577
+ refreshToken: body.refreshToken,
1578
+ accessExpiresAt: Date.now() + body.expiresIn * 1000,
1579
+ user: body.user,
1580
+ // Keep the binding. Dropping it here quietly turned off the check that
1581
+ // stops a session being handed to another address, for every process,
1582
+ // from the first refresh on.
1583
+ authUrl: this.binding,
356
1584
  };
357
1585
  try {
358
1586
  saveCredentials(this.session); // persist so the next run reuses it
1587
+ this.noteFileSeen();
359
1588
  }
360
1589
  catch (err) {
361
1590
  // Persisting is a convenience for the *next* process. Letting it throw
@@ -363,5 +1592,106 @@ export class RemoteRecorder {
363
1592
  logLine("auth.save_failed", { error: errText(err), why: "the token still works in-process" });
364
1593
  }
365
1594
  }
1595
+ /** The file now holds this recorder's session: follow it from here on. */
1596
+ noteFileSeen() {
1597
+ this.follow = true;
1598
+ try {
1599
+ this.credMtime = statSync(credentialsPath()).mtimeMs;
1600
+ this.credCheckedAt = Date.now();
1601
+ }
1602
+ catch {
1603
+ /* looked at again on the next check */
1604
+ }
1605
+ }
1606
+ /**
1607
+ * Keep in step with credentials.json, cheaply: one `stat` per
1608
+ * {@link CREDENTIALS_CHECK_MS}, and a read only when it changed.
1609
+ *
1610
+ * - gone → `bir logout`: pause (`signed_out`), drop what is queued.
1611
+ * - another service → pause (`other_service`): never send there.
1612
+ * - another account → adopt it for runs opened from now on; runs already
1613
+ * open are closed, so no run is split across two accounts.
1614
+ * - newer tokens → adopt them (a sibling refreshed, or `bir login`), and
1615
+ * resume if paused.
1616
+ */
1617
+ checkCredentials() {
1618
+ this.retryRejected();
1619
+ if (!this.follow)
1620
+ return;
1621
+ const now = Date.now();
1622
+ if (now - this.credCheckedAt < this.credentialsCheckMs)
1623
+ return;
1624
+ this.credCheckedAt = now;
1625
+ let mtime;
1626
+ try {
1627
+ mtime = statSync(credentialsPath()).mtimeMs;
1628
+ }
1629
+ catch (err) {
1630
+ if (err.code === "ENOENT") {
1631
+ this.credMtime = -1;
1632
+ this.pause("signed_out");
1633
+ }
1634
+ return;
1635
+ }
1636
+ if (mtime === this.credMtime)
1637
+ return;
1638
+ const stored = loadCredentials();
1639
+ if (!stored)
1640
+ return; // unreadable for now: look again next time
1641
+ this.credMtime = mtime;
1642
+ if (!issuedFor(stored, this.binding)) {
1643
+ this.pause("other_service");
1644
+ return;
1645
+ }
1646
+ if (stored.user.id !== this.session.user.id) {
1647
+ this.session = { ...stored, authUrl: stored.authUrl ?? this.binding };
1648
+ this.refreshRetryAt = 0;
1649
+ logLine("recorder.account_changed", {
1650
+ user: stored.user.email,
1651
+ why: "another account signed in on this machine — runs already open stop here, new ones record under it",
1652
+ });
1653
+ if (this.state !== "recording")
1654
+ this.resume();
1655
+ return;
1656
+ }
1657
+ const sameTokens = stored.accessToken === this.session.accessToken && stored.refreshToken === this.session.refreshToken;
1658
+ if (!sameTokens) {
1659
+ this.refreshRetryAt = 0;
1660
+ if (stored.accessExpiresAt - 30_000 > now || this.state !== "recording") {
1661
+ this.session = { ...stored, authUrl: stored.authUrl ?? this.binding };
1662
+ }
1663
+ else if (stored.refreshToken !== this.session.refreshToken) {
1664
+ this.session = { ...this.session, refreshToken: stored.refreshToken };
1665
+ }
1666
+ if (this.state !== "recording")
1667
+ this.resume();
1668
+ }
1669
+ else if (this.state === "signed_out" || this.state === "other_service") {
1670
+ // The same session came back (a restore): nothing was refused.
1671
+ this.resume();
1672
+ }
1673
+ }
1674
+ pause(state) {
1675
+ if (this.state === state)
1676
+ return;
1677
+ this.state = state;
1678
+ let dropped = 0;
1679
+ for (const lane of this.lanes.values()) {
1680
+ const jobs = lane.jobs;
1681
+ lane.jobs = [];
1682
+ for (const job of jobs) {
1683
+ this.pendingSends -= 1;
1684
+ this.pendingBytes -= job.bytes;
1685
+ this.retire(job, job.runId ? this.runs.get(job.runId) : undefined, state);
1686
+ dropped += 1;
1687
+ }
1688
+ }
1689
+ logLine("recorder.paused", { reason: state, dropped: dropped || undefined, why: PAUSE_WHY[state] });
1690
+ }
1691
+ resume(why = "a session is back in credentials.json") {
1692
+ this.state = "recording";
1693
+ this.lastAheadAttempt = 0;
1694
+ logLine("recorder.resumed", { user: this.session.user.email, why });
1695
+ }
366
1696
  }
367
1697
  //# sourceMappingURL=remote-recorder.js.map