@basein/runner 0.2.11 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +540 -350
- package/dist/auth/client.d.ts +65 -3
- package/dist/auth/client.js +305 -53
- package/dist/bin/bir-hooks.d.ts +72 -31
- package/dist/bin/bir-hooks.js +176 -284
- package/dist/bin/bir-proxy.d.ts +17 -2
- package/dist/bin/bir-proxy.js +222 -28
- package/dist/bin/bir-scenario.d.ts +61 -10
- package/dist/bin/bir-scenario.js +704 -135
- package/dist/bin/bir.d.ts +14 -8
- package/dist/bin/bir.js +2937 -488
- package/dist/bin/investigate.js +41 -16
- package/dist/bin/scenario-edit.d.ts +43 -10
- package/dist/bin/scenario-edit.js +281 -80
- package/dist/bin/scenario-liveness.d.ts +194 -0
- package/dist/bin/scenario-liveness.js +459 -0
- package/dist/bin/scenario-stderr.d.ts +43 -0
- package/dist/bin/scenario-stderr.js +76 -0
- package/dist/bin/service-read.d.ts +88 -0
- package/dist/bin/service-read.js +128 -0
- package/dist/bin/setup.d.ts +19 -0
- package/dist/bin/setup.js +146 -20
- package/dist/config/adapters/claude-code.d.ts +92 -14
- package/dist/config/adapters/claude-code.js +291 -101
- package/dist/config/adapters/generic.d.ts +4 -1
- package/dist/config/adapters/generic.js +13 -14
- package/dist/config/generate.d.ts +392 -14
- package/dist/config/generate.js +666 -19
- package/dist/config/mcp-approval.d.ts +76 -0
- package/dist/config/mcp-approval.js +210 -0
- package/dist/config/node-path.d.ts +42 -0
- package/dist/config/node-path.js +102 -0
- package/dist/config/overrides.d.ts +62 -0
- package/dist/config/overrides.js +136 -0
- package/dist/config/resolve.d.ts +70 -8
- package/dist/config/resolve.js +186 -15
- package/dist/control/client.d.ts +22 -0
- package/dist/control/client.js +26 -7
- package/dist/control/correlation.d.ts +23 -5
- package/dist/control/correlation.js +0 -0
- package/dist/control/daemon.d.ts +400 -18
- package/dist/control/daemon.js +1105 -102
- package/dist/control/discovery.d.ts +77 -5
- package/dist/control/discovery.js +186 -11
- package/dist/control/ensure-hook.d.ts +61 -0
- package/dist/control/ensure-hook.js +160 -8
- package/dist/control/paths.d.ts +26 -0
- package/dist/control/paths.js +34 -0
- package/dist/control/pin-guard.d.ts +83 -0
- package/dist/control/pin-guard.js +534 -0
- package/dist/control/recorder-keeper.d.ts +229 -0
- package/dist/control/recorder-keeper.js +721 -0
- package/dist/control/recorder-main.d.ts +73 -0
- package/dist/control/recorder-main.js +778 -0
- package/dist/control/server.d.ts +707 -31
- package/dist/control/server.js +2803 -306
- package/dist/control/transcript.d.ts +21 -1
- package/dist/control/transcript.js +378 -112
- package/dist/index.d.ts +5 -1
- package/dist/index.js +5 -1
- package/dist/jsonrpc/framing.d.ts +76 -12
- package/dist/jsonrpc/framing.js +216 -59
- package/dist/proxy/intercept.d.ts +49 -5
- package/dist/proxy/intercept.js +188 -59
- package/dist/proxy/relay.d.ts +16 -0
- package/dist/proxy/relay.js +77 -5
- package/dist/proxy/revive.d.ts +122 -0
- package/dist/proxy/revive.js +251 -0
- package/dist/proxy/session.d.ts +222 -14
- package/dist/proxy/session.js +708 -99
- package/dist/record/housekeeping.d.ts +71 -0
- package/dist/record/housekeeping.js +417 -0
- package/dist/record/recorder.d.ts +51 -3
- package/dist/record/recorder.js +3 -0
- package/dist/record/redact.d.ts +103 -9
- package/dist/record/redact.js +659 -30
- package/dist/record/remote-recorder.d.ts +327 -32
- package/dist/record/remote-recorder.js +1517 -187
- package/dist/record/switchable-recorder.d.ts +126 -0
- package/dist/record/switchable-recorder.js +351 -0
- package/dist/record/truncate.d.ts +40 -1
- package/dist/record/truncate.js +111 -4
- package/dist/replay/bash-channel.d.ts +35 -0
- package/dist/replay/bash-channel.js +133 -0
- package/dist/replay/bundle.d.ts +10 -0
- package/dist/replay/bundle.js +10 -0
- package/dist/replay/controller.d.ts +371 -20
- package/dist/replay/controller.js +1185 -111
- package/dist/replay/derive.d.ts +1 -0
- package/dist/replay/derive.js +81 -24
- package/dist/replay/executor.d.ts +51 -16
- package/dist/replay/executor.js +114 -41
- package/dist/replay/handover.d.ts +54 -1
- package/dist/replay/handover.js +55 -5
- package/dist/replay/logic.d.ts +62 -10
- package/dist/replay/logic.js +140 -22
- package/dist/replay/plan.d.ts +201 -27
- package/dist/replay/plan.js +581 -141
- package/dist/replay/safety.d.ts +72 -0
- package/dist/replay/safety.js +303 -0
- package/dist/replay/source-run.d.ts +10 -0
- package/dist/replay/source-run.js +80 -35
- package/dist/replay/targets.d.ts +15 -1
- package/dist/replay/targets.js +62 -4
- package/dist/replay/tool-error.d.ts +15 -1
- package/dist/replay/tool-error.js +33 -1
- package/dist/replay/types.d.ts +7 -4
- package/dist/upstream/client.d.ts +56 -3
- package/dist/upstream/client.js +61 -4
- package/dist/upstream/headers.d.ts +43 -0
- package/dist/upstream/headers.js +90 -0
- package/dist/upstream/http-client.d.ts +213 -22
- package/dist/upstream/http-client.js +1034 -168
- package/dist/upstream/http-transport.d.ts +143 -0
- package/dist/upstream/http-transport.js +391 -0
- package/dist/upstream/lazy-client.d.ts +8 -0
- package/dist/upstream/lazy-client.js +29 -1
- package/dist/upstream/sse.d.ts +78 -0
- package/dist/upstream/sse.js +189 -0
- package/dist/upstream/stdio-client.d.ts +61 -1
- package/dist/upstream/stdio-client.js +356 -33
- package/dist/util/atomic-write.d.ts +29 -0
- package/dist/util/atomic-write.js +96 -0
- package/dist/util/journal.d.ts +15 -0
- package/dist/util/journal.js +48 -3
- package/dist/util/log.d.ts +10 -0
- package/dist/util/log.js +28 -1
- package/dist/util/off-switch.d.ts +46 -0
- package/dist/util/off-switch.js +91 -0
- package/dist/util/service-health.d.ts +133 -0
- package/dist/util/service-health.js +217 -0
- package/docs/BaseInstRunner.md +694 -621
- package/docs/calculatedReplay.md +1625 -1244
- package/docs/calculatedReplayGuide.md +1212 -879
- package/docs/installRun.md +1013 -772
- package/docs/loginWeb.md +627 -607
- package/docs/mcpmark.md +49 -16
- package/docs/my-first-sample.md +569 -545
- package/docs/quickstart.md +384 -271
- package/docs/t-bench.md +406 -394
- package/package.json +58 -57
- package/production.md +1371 -0
- package/scripts/install.ps1 +97 -6
- package/scripts/install.sh +104 -7
|
@@ -8,102 +8,488 @@
|
|
|
8
8
|
* PATCH /recordings/runs/:id/steps/:stepId
|
|
9
9
|
* POST /recordings/runs/:id/finish
|
|
10
10
|
*
|
|
11
|
-
*
|
|
11
|
+
* The properties that matter, each load-bearing:
|
|
12
12
|
* - **Client-generated ids** (`run_`/`step_` uuids) so callers get an id
|
|
13
13
|
* synchronously and never await the network on the hot path.
|
|
14
|
-
* - **One ordered
|
|
15
|
-
*
|
|
16
|
-
* - **Best-effort
|
|
17
|
-
* and never thrown at the caller
|
|
18
|
-
*
|
|
14
|
+
* - **One ordered lane per run**, so steps arrive in the order they were
|
|
15
|
+
* recorded even though every send is fire-and-forget.
|
|
16
|
+
* - **Best-effort, and invisible when the service is not there.** A failed
|
|
17
|
+
* send is logged once per run and dropped, never thrown at the caller
|
|
18
|
+
* (§10). Every request has a deadline (`BIR_SERVICE_TIMEOUT_MS`), every
|
|
19
|
+
* answer or its absence is reported to the process-wide health verdict
|
|
20
|
+
* ({@link serviceHealth}), and while that verdict says the service is
|
|
21
|
+
* cooling down the calls a hook waits on — `getMatch`, `runCreated`,
|
|
22
|
+
* `matchIntent` — answer "no match" at once instead of making the user wait,
|
|
23
|
+
* and `flush` returns at once, leaving the rest to drain in the background.
|
|
24
|
+
* A run's create that fails its first attempt answers "no match" at once
|
|
25
|
+
* too; its retry is for the recording, never for a hook.
|
|
26
|
+
* - **One probe, not a stampede.** While the breaker is half-open, one
|
|
27
|
+
* queued send finds out for all the lanes; the rest wait for its verdict
|
|
28
|
+
* and then follow it a moment apart, not all at once.
|
|
29
|
+
* - **Bounded.** The lanes are the real backlog (the hook server's step queue
|
|
30
|
+
* drains instantly into them), so they carry the bound: past
|
|
31
|
+
* {@link DEFAULT_MAX_PENDING_SENDS} sends or {@link DEFAULT_MAX_PENDING_BYTES}
|
|
32
|
+
* the oldest unsent *step* is dropped and counted — creates and finishes go
|
|
33
|
+
* last, so the service is not left with runs that never end.
|
|
34
|
+
* - **One token refresh on 401**, shared by every send that meets the same
|
|
35
|
+
* expiry, then one retry. Sessions outlive access tokens.
|
|
36
|
+
* - **It follows credentials.json.** `bir logout` removes the file; a
|
|
37
|
+
* recorder that loaded its session from it stops sending within seconds,
|
|
38
|
+
* rather than recording for the rest of the access token's life.
|
|
39
|
+
* - **It redacts and caps every free-text field it sends** — the prompt, the
|
|
40
|
+
* answer, reasoning, errors — as the last line of defence. Structured tool
|
|
41
|
+
* payloads are redacted by their producers; text is only ever seen here.
|
|
19
42
|
*
|
|
20
43
|
* SIMILAR-PROMPT HITS. `POST /recordings/runs` may answer `{ id: null, matched }`,
|
|
21
44
|
* meaning the server recognised the prompt and did **not** create a run. Every
|
|
22
45
|
* later step post for that id would then 404, so callers must stop recording —
|
|
23
|
-
* {@link RemoteRecorder.getMatch} is how they find out
|
|
46
|
+
* {@link RemoteRecorder.getMatch} is how they find out — and this recorder skips
|
|
47
|
+
* them itself too, for the caller that stopped waiting before the answer came.
|
|
24
48
|
*/
|
|
25
49
|
import { randomUUID } from "node:crypto";
|
|
26
|
-
import {
|
|
27
|
-
import {
|
|
50
|
+
import { statSync } from "node:fs";
|
|
51
|
+
import { credentialsPath, describeBody, isAuthResponse, issuedFor, loadCredentials, saveCredentials, } from "../auth/client.js";
|
|
52
|
+
import { errText, logDetail, logEvery, logLine } from "../util/log.js";
|
|
53
|
+
import { isUnhealthyStatus, serviceHealth, serviceTimeoutMs, timeoutSignal, } from "../util/service-health.js";
|
|
54
|
+
import { redactJsonText, redactText } from "./redact.js";
|
|
55
|
+
import { capJsonText, capText, DEFAULT_MAX_PAYLOAD_BYTES, preCut } from "./truncate.js";
|
|
56
|
+
/**
|
|
57
|
+
* Caps for the free text this recorder sends, in UTF-8 bytes. Tool payloads
|
|
58
|
+
* arrive capped at 64 KiB by their producers; text never passed through one,
|
|
59
|
+
* so a pasted 3 MB log used to go out whole — once as the run's input, again
|
|
60
|
+
* in the execution report — and spend the prompt's match budget uploading.
|
|
61
|
+
*/
|
|
62
|
+
export const TEXT_CAPS = {
|
|
63
|
+
/** The prompt: the run's `input`, and an execution report's `prompt`. */
|
|
64
|
+
prompt: 32 * 1024,
|
|
65
|
+
/** The final answer, and the run's `finalOutput`. */
|
|
66
|
+
answer: 64 * 1024,
|
|
67
|
+
/** A step's written reasoning. */
|
|
68
|
+
context: 8 * 1024,
|
|
69
|
+
/** A step's thinking text, sent only to be embedded. */
|
|
70
|
+
intent: 8 * 1024,
|
|
71
|
+
/** A tool's error text when it is not JSON. */
|
|
72
|
+
toolError: 8 * 1024,
|
|
73
|
+
/** An execution report's error lines, rendered in a web page. */
|
|
74
|
+
reportError: 2 * 1024,
|
|
75
|
+
};
|
|
76
|
+
/** Queued sends past which the oldest unsent step is dropped (`BIR_RECORDER_MAX_PENDING`). */
|
|
77
|
+
export const DEFAULT_MAX_PENDING_SENDS = 500;
|
|
78
|
+
/** Queued bytes past which the same happens (`BIR_RECORDER_MAX_PENDING_BYTES`). */
|
|
79
|
+
export const DEFAULT_MAX_PENDING_BYTES = 32 * 1024 * 1024;
|
|
80
|
+
/** How long a probe waits when its caller did not say. Its answer is useless after a hook's budget. */
|
|
81
|
+
export const PROBE_TIMEOUT_MS = 5_000;
|
|
82
|
+
/**
|
|
83
|
+
* How long {@link RemoteRecorder.flush} waits when its caller did not say
|
|
84
|
+
* (`BIR_FLUSH_TIMEOUT_MS`). Every caller is a hook or a shutdown, and a
|
|
85
|
+
* healthy service drains a turn's last few sends in well under a second.
|
|
86
|
+
*/
|
|
87
|
+
export const DEFAULT_FLUSH_TIMEOUT_MS = 2_000;
|
|
88
|
+
/** How often a waiting flush looks at the breaker. */
|
|
89
|
+
const FLUSH_POLL_MS = 25;
|
|
90
|
+
/** How often, at most, credentials.json is looked at: one `stat`. */
|
|
91
|
+
const CREDENTIALS_CHECK_MS = 2_000;
|
|
92
|
+
/** A settled match (it holds the whole scenario) is kept this long for late readers. */
|
|
93
|
+
const MATCH_TTL_MS = 10 * 60_000;
|
|
94
|
+
/** A run nobody has touched for this long is forgotten entirely. */
|
|
95
|
+
const RUN_STATE_TTL_MS = 6 * 60 * 60_000;
|
|
96
|
+
/** And never more than this many are remembered. */
|
|
97
|
+
const RUN_STATE_MAX = 2_000;
|
|
98
|
+
/** Refresh this long before the access token expires, in the background. */
|
|
99
|
+
const REFRESH_AHEAD_MS = 60_000;
|
|
100
|
+
/** A failed early refresh is not retried sooner than this. */
|
|
101
|
+
const REFRESH_AHEAD_RETRY_MS = 30_000;
|
|
102
|
+
/**
|
|
103
|
+
* The longest one queued send waits for the service. Past it, a send that is
|
|
104
|
+
* still facing a cool-down is given up unsent (counted as dropped): the
|
|
105
|
+
* service has failed every probe for a minute, and sending it anyway is what
|
|
106
|
+
* made every lane hit a dead service at the same moment.
|
|
107
|
+
*/
|
|
108
|
+
const COOLDOWN_WAIT_MAX_MS = 60_000;
|
|
109
|
+
/** A cool-down about to end is waited out even past {@link COOLDOWN_WAIT_MAX_MS}. */
|
|
110
|
+
const COOLDOWN_GRACE_MS = 5_000;
|
|
111
|
+
/** Lanes released after a probe succeeds go spread over up to this long. */
|
|
112
|
+
const RELEASE_JITTER_MAX_MS = 2_000;
|
|
113
|
+
/** How often a lane waiting for another lane's probe looks at the verdict. */
|
|
114
|
+
const PROBE_POLL_MS = 1_000;
|
|
115
|
+
/** After a 404 from the probe route (an older service), stop probing for this long. */
|
|
116
|
+
const MATCH_INTENT_UNSUPPORTED_MS = 10 * 60_000;
|
|
117
|
+
/**
|
|
118
|
+
* Statuses a background send is retried once on: the service saying it did
|
|
119
|
+
* not take the request. Not 502/504 — a gateway that gave up may have handed
|
|
120
|
+
* the request to an origin that is still inserting it, and until the service
|
|
121
|
+
* is known to deduplicate ids a repeat could record a step twice.
|
|
122
|
+
*/
|
|
123
|
+
const RETRY_STATUSES = new Set([429, 503]);
|
|
124
|
+
/**
|
|
125
|
+
* Network failures that prove the request never reached the service — no
|
|
126
|
+
* connection, no address, or a pooled socket the far end had already closed
|
|
127
|
+
* (undici's "other side closed", before any answer) — so a repeat cannot
|
|
128
|
+
* double it. A reset mid-request is not among them.
|
|
129
|
+
*/
|
|
130
|
+
const SAFE_RETRY_CODES = new Set(["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "UND_ERR_SOCKET", "UND_ERR_CONNECT_TIMEOUT"]);
|
|
131
|
+
/** A refresh that failed without an answer to act on is not tried again sooner than this. */
|
|
132
|
+
const REFRESH_RETRY_MS = 30_000;
|
|
133
|
+
/** After a refused refresh, how long to wait for a sibling's newer session to land on disk. */
|
|
134
|
+
const AUTH_RECHECK_MS = 1_500;
|
|
135
|
+
/** While the session is refused, how often it is tried again anyway. */
|
|
136
|
+
const AUTH_REJECTED_RETRY_MS = 10 * 60_000;
|
|
137
|
+
/** Execution reports ride their own lane: a matched turn has no run on the service. */
|
|
138
|
+
const EXECUTIONS = "__executions__";
|
|
139
|
+
/** A send that did not succeed, with the status the service answered, if it answered. */
|
|
140
|
+
export class ServiceError extends Error {
|
|
141
|
+
status;
|
|
142
|
+
constructor(message, status) {
|
|
143
|
+
super(message);
|
|
144
|
+
this.status = status;
|
|
145
|
+
this.name = "ServiceError";
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
/** The service's own answer that this refresh token is no good (a JSON 400/401). */
|
|
149
|
+
class RefreshRejected extends ServiceError {
|
|
150
|
+
}
|
|
151
|
+
/** Another account signed in while a send was between attempts: it is not sent under the new one. */
|
|
152
|
+
class AccountChanged extends Error {
|
|
153
|
+
constructor(what) {
|
|
154
|
+
super(`${what}: another account signed in — not sent under it`);
|
|
155
|
+
this.name = "AccountChanged";
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
function pend(text, max, json = false) {
|
|
159
|
+
if (typeof text !== "string")
|
|
160
|
+
return undefined;
|
|
161
|
+
const cut = preCut(text, max);
|
|
162
|
+
return { text: cut.text, bytes: cut.bytes, max, json };
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* The field as sent: capped and redacted. A JSON payload is redacted as JSON
|
|
166
|
+
* ({@link redactJsonText}), so it stays JSON and — when its producer already
|
|
167
|
+
* redacted it — keeps its exact bytes.
|
|
168
|
+
*/
|
|
169
|
+
function out(p) {
|
|
170
|
+
if (!p)
|
|
171
|
+
return undefined;
|
|
172
|
+
return p.json ? capJsonText(p.text, p.max, redactJsonText, p.bytes) : capText(p.text, p.max, redactText, p.bytes);
|
|
173
|
+
}
|
|
174
|
+
function sizeOf(...parts) {
|
|
175
|
+
let n = 256;
|
|
176
|
+
for (const p of parts)
|
|
177
|
+
n += p?.text.length ?? 0;
|
|
178
|
+
return n;
|
|
179
|
+
}
|
|
180
|
+
/** A tool error is JSON when a producer serialized it, and prose when it did not. */
|
|
181
|
+
function pendError(text) {
|
|
182
|
+
if (typeof text !== "string")
|
|
183
|
+
return undefined;
|
|
184
|
+
return /^\s*[{[]/.test(text) ? pend(text, DEFAULT_MAX_PAYLOAD_BYTES, true) : pend(text, TEXT_CAPS.toolError);
|
|
185
|
+
}
|
|
186
|
+
function positiveEnv(name, fallback) {
|
|
187
|
+
const n = Number(process.env[name]);
|
|
188
|
+
return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback;
|
|
189
|
+
}
|
|
190
|
+
function reasonOf(err) {
|
|
191
|
+
const cause = err?.cause;
|
|
192
|
+
return cause?.code ?? cause?.message ?? errText(err);
|
|
193
|
+
}
|
|
194
|
+
/** The network error's code: undici's cause, or the first of a happy-eyeballs AggregateError's. */
|
|
195
|
+
function causeCode(err) {
|
|
196
|
+
const cause = err?.cause;
|
|
197
|
+
return cause?.code ?? cause?.errors?.[0]?.code;
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* An answer that is a web page rather than the API: a captive portal, a
|
|
201
|
+
* website, a CDN challenge. Judged by the body — an empty or plain-text 2xx
|
|
202
|
+
* is the API answering, whatever content type a framework's `res.send("ok")`
|
|
203
|
+
* put on it.
|
|
204
|
+
*/
|
|
205
|
+
function isWebPage(text, type) {
|
|
206
|
+
if (/^\s*<(!doctype|html|head|body)\b/i.test(text))
|
|
207
|
+
return true;
|
|
208
|
+
return /html/i.test(type) && /^\s*</.test(text);
|
|
209
|
+
}
|
|
210
|
+
/** A JSON object — the shape of the service's own error answers. */
|
|
211
|
+
function isJsonObject(text) {
|
|
212
|
+
try {
|
|
213
|
+
const v = JSON.parse(text);
|
|
214
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
215
|
+
}
|
|
216
|
+
catch {
|
|
217
|
+
return false;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
/** `p`, or a rejection as soon as `signal` aborts. `p` keeps running; its outcome is observed either way. */
|
|
221
|
+
function untilAborted(p, signal) {
|
|
222
|
+
if (signal.aborted) {
|
|
223
|
+
p.catch(() => undefined);
|
|
224
|
+
return Promise.reject(signal.reason);
|
|
225
|
+
}
|
|
226
|
+
return new Promise((resolve, reject) => {
|
|
227
|
+
const onAbort = () => reject(signal.reason);
|
|
228
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
229
|
+
p.then((v) => {
|
|
230
|
+
signal.removeEventListener("abort", onAbort);
|
|
231
|
+
resolve(v);
|
|
232
|
+
}, (e) => {
|
|
233
|
+
signal.removeEventListener("abort", onAbort);
|
|
234
|
+
reject(e);
|
|
235
|
+
});
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
function detail(text, type) {
|
|
239
|
+
const d = describeBody(text, type);
|
|
240
|
+
return d ? ` — ${d}` : "";
|
|
241
|
+
}
|
|
242
|
+
function sleep(ms, unref = false) {
|
|
243
|
+
return new Promise((resolve) => {
|
|
244
|
+
const timer = setTimeout(resolve, ms);
|
|
245
|
+
if (unref)
|
|
246
|
+
timer.unref?.();
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
/** Call a caller's hook; its failure is never the request's. */
|
|
250
|
+
function notify(fn) {
|
|
251
|
+
try {
|
|
252
|
+
fn?.();
|
|
253
|
+
}
|
|
254
|
+
catch {
|
|
255
|
+
/* never */
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
/** 250-750 ms: long enough for a load balancer to replace a dead socket, short enough to be unnoticed. */
|
|
259
|
+
function jitterMs() {
|
|
260
|
+
return 250 + Math.floor(Math.random() * 500);
|
|
261
|
+
}
|
|
262
|
+
/** A 429/503's own Retry-After, capped at 2 s; otherwise jitter. */
|
|
263
|
+
function retryDelayMs(res) {
|
|
264
|
+
const header = res.headers.get("retry-after");
|
|
265
|
+
if (header) {
|
|
266
|
+
const seconds = Number(header);
|
|
267
|
+
const ms = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(header) - Date.now();
|
|
268
|
+
if (Number.isFinite(ms))
|
|
269
|
+
return Math.min(2_000, Math.max(0, ms));
|
|
270
|
+
}
|
|
271
|
+
return jitterMs();
|
|
272
|
+
}
|
|
273
|
+
const PAUSE_WHY = {
|
|
274
|
+
signed_out: "credentials.json was removed (bir logout) — nothing more is sent and replay stops until you sign in again",
|
|
275
|
+
auth_rejected: "the service refused this session and no newer one was on disk — run `bir login`; recording resumes on its own (the session is also retried every 10 minutes)",
|
|
276
|
+
other_service: "credentials.json now belongs to a different service address — restart the recorder (or run `bir setup`) to record there",
|
|
277
|
+
};
|
|
278
|
+
const DEAD_WHY = {
|
|
279
|
+
not_found: "the service has no such run (its create never landed) — the rest of this run is skipped",
|
|
280
|
+
create_rejected: "the service refused to create the run — the rest of this run is skipped",
|
|
281
|
+
account_changed: "another account signed in on this machine — the rest of this run is skipped",
|
|
282
|
+
dropped: "the backlog bound dropped this run's create — the rest of it is skipped",
|
|
283
|
+
service_down: "the service failed every probe for a minute, so this run's create was never sent — the rest of it is skipped",
|
|
284
|
+
};
|
|
28
285
|
export class RemoteRecorder {
|
|
29
286
|
baseUrl;
|
|
30
287
|
framework;
|
|
288
|
+
health;
|
|
289
|
+
/** The service these credentials are bound to; never changes for this recorder. */
|
|
290
|
+
binding;
|
|
291
|
+
maxPendingSends;
|
|
292
|
+
maxPendingBytes;
|
|
293
|
+
credentialsCheckMs;
|
|
31
294
|
session;
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
295
|
+
state = "recording";
|
|
296
|
+
runs = new Map();
|
|
297
|
+
lanes = new Map();
|
|
298
|
+
seq = 0;
|
|
299
|
+
pendingSends = 0;
|
|
300
|
+
pendingBytes = 0;
|
|
301
|
+
droppedSends = 0;
|
|
302
|
+
failedSends = 0;
|
|
303
|
+
backlogFull = false;
|
|
304
|
+
droppedWhileFull = 0;
|
|
305
|
+
lastSweep = 0;
|
|
306
|
+
refreshing;
|
|
307
|
+
/** A failed refresh is not tried again before this (epoch ms). */
|
|
308
|
+
refreshRetryAt = 0;
|
|
309
|
+
authRecheckMs;
|
|
310
|
+
authRetryMs;
|
|
311
|
+
cooldownWaitMaxMs;
|
|
312
|
+
/** When the session was last refused or retried while refused. */
|
|
313
|
+
rejectedAt = 0;
|
|
314
|
+
lastAheadAttempt = 0;
|
|
38
315
|
/**
|
|
39
|
-
*
|
|
40
|
-
*
|
|
316
|
+
* The one request out while the breaker is half-open — a lane's own send,
|
|
317
|
+
* or a hook-requested `GET /health` — and when it started.
|
|
41
318
|
*/
|
|
42
|
-
|
|
319
|
+
probe;
|
|
320
|
+
matchIntentOffUntil = 0;
|
|
321
|
+
/** True once this recorder's session is known to be the one in credentials.json. */
|
|
322
|
+
follow = false;
|
|
323
|
+
credMtime = -1;
|
|
324
|
+
credCheckedAt = 0;
|
|
43
325
|
constructor(opts) {
|
|
44
326
|
this.baseUrl = opts.baseUrl.replace(/\/+$/, "");
|
|
45
327
|
this.session = opts.session;
|
|
46
328
|
this.framework = opts.framework ?? "baseinstrunner";
|
|
329
|
+
this.health = opts.health ?? serviceHealth;
|
|
330
|
+
this.binding = (opts.session.authUrl ?? this.baseUrl).replace(/\/+$/, "");
|
|
331
|
+
this.maxPendingSends = opts.maxPendingSends ?? positiveEnv("BIR_RECORDER_MAX_PENDING", DEFAULT_MAX_PENDING_SENDS);
|
|
332
|
+
this.maxPendingBytes =
|
|
333
|
+
opts.maxPendingBytes ?? positiveEnv("BIR_RECORDER_MAX_PENDING_BYTES", DEFAULT_MAX_PENDING_BYTES);
|
|
334
|
+
this.credentialsCheckMs = opts.credentialsCheckMs ?? CREDENTIALS_CHECK_MS;
|
|
335
|
+
this.authRecheckMs = opts.authRecheckMs ?? AUTH_RECHECK_MS;
|
|
336
|
+
this.authRetryMs = opts.authRetryMs ?? AUTH_REJECTED_RETRY_MS;
|
|
337
|
+
this.cooldownWaitMaxMs = opts.cooldownWaitMaxMs ?? COOLDOWN_WAIT_MAX_MS;
|
|
338
|
+
// Follow the file only when this session came from it. A synthetic session
|
|
339
|
+
// (BIR_AUTH_DISABLE, tests) has no file, and must not read "no file" as
|
|
340
|
+
// "signed out".
|
|
341
|
+
try {
|
|
342
|
+
const mtime = statSync(credentialsPath()).mtimeMs;
|
|
343
|
+
const stored = loadCredentials();
|
|
344
|
+
if (stored &&
|
|
345
|
+
stored.user.id === opts.session.user.id &&
|
|
346
|
+
issuedFor(stored, this.binding) &&
|
|
347
|
+
(stored.accessToken === opts.session.accessToken || stored.refreshToken === opts.session.refreshToken)) {
|
|
348
|
+
this.follow = true;
|
|
349
|
+
this.credMtime = mtime;
|
|
350
|
+
this.credCheckedAt = Date.now();
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
catch {
|
|
354
|
+
/* no file: nothing to follow */
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
/**
|
|
358
|
+
* The access token this recorder is using *now*. It changes on every refresh
|
|
359
|
+
* (and when a sibling process's refresh is adopted), so callers that need the
|
|
360
|
+
* token — replay's service derivation, the source-run fetch — must read it
|
|
361
|
+
* through this each time rather than keep the startup session. Undefined once
|
|
362
|
+
* the session is gone (`bir logout`) or was refused.
|
|
363
|
+
*/
|
|
364
|
+
currentAccessToken() {
|
|
365
|
+
this.checkCredentials();
|
|
366
|
+
if (this.state !== "recording")
|
|
367
|
+
return undefined;
|
|
368
|
+
return this.session.accessToken || undefined;
|
|
369
|
+
}
|
|
370
|
+
/** The backlog and sending state, for `/health` and `bir doctor`. */
|
|
371
|
+
recorderHealth() {
|
|
372
|
+
return {
|
|
373
|
+
state: this.state,
|
|
374
|
+
pendingSends: this.pendingSends,
|
|
375
|
+
pendingBytes: this.pendingBytes,
|
|
376
|
+
droppedSends: this.droppedSends,
|
|
377
|
+
failedSends: this.failedSends,
|
|
378
|
+
};
|
|
47
379
|
}
|
|
48
380
|
startRun(input, metadata, options) {
|
|
49
381
|
const id = "run_" + randomUUID();
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
this.
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
382
|
+
this.checkCredentials();
|
|
383
|
+
this.sweep();
|
|
384
|
+
const entry = this.newEntry();
|
|
385
|
+
this.runs.set(id, entry);
|
|
386
|
+
if (this.state !== "recording") {
|
|
387
|
+
// Nothing is sent while signed out, so there is nothing to match against
|
|
388
|
+
// and no run on the service: "fresh run", and every later send skipped.
|
|
389
|
+
entry.dead = this.state;
|
|
390
|
+
entry.settle(null, { created: true });
|
|
391
|
+
return id;
|
|
392
|
+
}
|
|
393
|
+
const prompt = pend(input ?? "", TEXT_CAPS.prompt);
|
|
394
|
+
this.enqueue({
|
|
395
|
+
lane: id,
|
|
396
|
+
runId: id,
|
|
397
|
+
kind: "create",
|
|
398
|
+
bytes: sizeOf(prompt),
|
|
399
|
+
// Dropped or skipped before it was sent: a fail-safe "fresh run, no match".
|
|
400
|
+
abandon: () => entry.settle(null, { created: true }),
|
|
401
|
+
send: async () => {
|
|
402
|
+
try {
|
|
403
|
+
const body = await this.request("POST", "/recordings/runs", JSON.stringify({
|
|
404
|
+
id,
|
|
405
|
+
framework: this.framework,
|
|
406
|
+
input: out(prompt),
|
|
407
|
+
metadata,
|
|
408
|
+
// A fragment (fallbk.md D7). Omitted, not null, so an older service
|
|
409
|
+
// that does not know the field sees the body it always saw.
|
|
410
|
+
...(options?.fallbackOf ? { fallbackOf: options.fallbackOf } : {}),
|
|
411
|
+
// A chain with calls is inlined for this runner rather than stopped in
|
|
412
|
+
// front of its first call (segmented.md R-CALL-28).
|
|
413
|
+
supportsCalls: true,
|
|
414
|
+
}),
|
|
415
|
+
// The id is client-generated, so the service can tell a repeat. A
|
|
416
|
+
// hook waiting on this answer is let go before the retry: a match
|
|
417
|
+
// found by the retry is not the turn's any more.
|
|
418
|
+
{ retryTransient: true, onRetry: () => entry.release() });
|
|
419
|
+
const b = body && typeof body === "object"
|
|
420
|
+
? body
|
|
421
|
+
: {};
|
|
422
|
+
// `id: null` is the service saying it kept a run of its own. Which run
|
|
423
|
+
// that was matters: a turn must not count step hits against the
|
|
424
|
+
// recording it is itself a repeat of (R-HIT-7).
|
|
425
|
+
if (entry.released && b.id === null) {
|
|
426
|
+
logLine("recorder.matched_after_retry", {
|
|
427
|
+
run: id,
|
|
428
|
+
why: "the create's first attempt failed, so the turn went ahead as a fresh one; the service kept its own run, and this one's steps are not sent",
|
|
429
|
+
});
|
|
430
|
+
}
|
|
431
|
+
entry.settle(b.matched ?? null, { created: b.id !== null, fragmentOf: b.fragmentOf });
|
|
432
|
+
}
|
|
433
|
+
catch (err) {
|
|
434
|
+
entry.settle(null, { created: true }); // fail-safe: on a create error, record normally
|
|
435
|
+
throw err; // preserve the dropped-send logging
|
|
436
|
+
}
|
|
437
|
+
},
|
|
86
438
|
});
|
|
87
439
|
return id;
|
|
88
440
|
}
|
|
89
|
-
/**
|
|
441
|
+
/**
|
|
442
|
+
* What the create call said. "Created" for an id this recorder never opened,
|
|
443
|
+
* and — without waiting — while the service is cooling down or the session is
|
|
444
|
+
* gone, or once the create's first attempt has failed: a hook must never wait
|
|
445
|
+
* on a service that just failed to answer, nor on the retry after it.
|
|
446
|
+
*/
|
|
90
447
|
async runCreated(runId) {
|
|
91
|
-
const
|
|
92
|
-
|
|
448
|
+
const entry = this.runs.get(runId);
|
|
449
|
+
if (!entry)
|
|
450
|
+
return { created: true };
|
|
451
|
+
entry.lastUsed = Date.now();
|
|
452
|
+
if (entry.settled)
|
|
453
|
+
return entry.settled.created;
|
|
454
|
+
if (entry.released || !entry.hot || !this.mayHotWait())
|
|
455
|
+
return { created: true };
|
|
456
|
+
await entry.hot;
|
|
457
|
+
// Settled, or let go when the first attempt failed (then: a fresh run).
|
|
458
|
+
return entry.settled?.created ?? { created: true };
|
|
93
459
|
}
|
|
94
460
|
/**
|
|
95
461
|
* Match a ReAct iteration's reasoning against the caller's scenario intents
|
|
96
462
|
* (fallbk.md §Runner 4). Awaited, not queued: the caller is a `PreToolUse`
|
|
97
463
|
* holding a budget, and the answer decides what that hook says. A service that
|
|
98
|
-
* predates the route (404) or any failure is a miss
|
|
464
|
+
* predates the route (404) or any failure is a miss — and so is a service in
|
|
465
|
+
* a cool-down, without a request: the probe runs on every tool call, so a
|
|
466
|
+
* dead service must cost it nothing.
|
|
467
|
+
*
|
|
468
|
+
* `opts.signal`/`opts.timeoutMs` end the request when the caller stops
|
|
469
|
+
* waiting, so an abandoned probe does not hold a socket for minutes.
|
|
99
470
|
*/
|
|
100
|
-
async matchIntent(req) {
|
|
471
|
+
async matchIntent(req, opts = {}) {
|
|
101
472
|
try {
|
|
102
|
-
|
|
473
|
+
if (!this.mayHotWait())
|
|
474
|
+
return null;
|
|
475
|
+
if (Date.now() < this.matchIntentOffUntil)
|
|
476
|
+
return null;
|
|
477
|
+
// The reasoning often quotes what the model just read — a .env, a key.
|
|
478
|
+
const body = {
|
|
479
|
+
...req,
|
|
480
|
+
text: capText(req.text ?? "", TEXT_CAPS.intent, redactText),
|
|
481
|
+
...(typeof req.toolInput === "string"
|
|
482
|
+
? { toolInput: capJsonText(req.toolInput, DEFAULT_MAX_PAYLOAD_BYTES, redactJsonText) }
|
|
483
|
+
: {}),
|
|
484
|
+
};
|
|
485
|
+
const answer = await this.request("POST", "/scenarios/match-intent", JSON.stringify(body), {
|
|
486
|
+
signal: opts.signal,
|
|
487
|
+
timeoutMs: opts.timeoutMs ?? PROBE_TIMEOUT_MS,
|
|
488
|
+
});
|
|
103
489
|
// The whole answer, not just `matched`: the counting fields are what make
|
|
104
490
|
// the observe-only period readable (segmented.md R-OUT-10), and an older
|
|
105
491
|
// server simply sends none of them (R-COMPAT-2).
|
|
106
|
-
const b =
|
|
492
|
+
const b = answer && typeof answer === "object" ? answer : {};
|
|
107
493
|
return {
|
|
108
494
|
matched: b.matched ?? null,
|
|
109
495
|
stepHit: b.stepHit ?? null,
|
|
@@ -112,18 +498,45 @@ export class RemoteRecorder {
|
|
|
112
498
|
};
|
|
113
499
|
}
|
|
114
500
|
catch (err) {
|
|
115
|
-
|
|
501
|
+
if (err instanceof ServiceError && err.status === 404) {
|
|
502
|
+
// An older service: asking again on the next tool call gets the same 404.
|
|
503
|
+
this.matchIntentOffUntil = Date.now() + MATCH_INTENT_UNSUPPORTED_MS;
|
|
504
|
+
}
|
|
505
|
+
if (!opts.signal?.aborted) {
|
|
506
|
+
// Once per 30 s, not once per tool call: an outage is one fact.
|
|
507
|
+
logEvery("recorder.intent_match_failed", 30_000, "replay.intent_match_failed", {
|
|
508
|
+
error: errText(err),
|
|
509
|
+
why: "treated as a miss",
|
|
510
|
+
});
|
|
511
|
+
}
|
|
116
512
|
return null;
|
|
117
513
|
}
|
|
118
514
|
}
|
|
119
|
-
/**
|
|
515
|
+
/**
|
|
516
|
+
* The similar-prompt match for a run's create call (null = fresh run). Like
|
|
517
|
+
* {@link runCreated}, answers null at once while the service is cooling down,
|
|
518
|
+
* and as soon as the create's first attempt fails — the first prompt of an
|
|
519
|
+
* outage is not held for the retry's sleep (up to 2 s behind a Retry-After).
|
|
520
|
+
*/
|
|
120
521
|
async getMatch(runId) {
|
|
121
|
-
const
|
|
122
|
-
|
|
522
|
+
const entry = this.runs.get(runId);
|
|
523
|
+
if (!entry)
|
|
524
|
+
return null;
|
|
525
|
+
entry.lastUsed = Date.now();
|
|
526
|
+
if (entry.settled)
|
|
527
|
+
return entry.settled.match;
|
|
528
|
+
if (entry.released || !entry.hot || !this.mayHotWait())
|
|
529
|
+
return null;
|
|
530
|
+
await entry.hot;
|
|
531
|
+
return entry.settled?.match ?? null;
|
|
123
532
|
}
|
|
124
533
|
recordToolSelected(runId, stepIndex, data) {
|
|
125
534
|
const id = "step_" + randomUUID();
|
|
126
|
-
|
|
535
|
+
const toolInput = pend(data.toolInput, DEFAULT_MAX_PAYLOAD_BYTES, true);
|
|
536
|
+
const context = pend(data.context, TEXT_CAPS.context);
|
|
537
|
+
const intentText = data.intent ? pend(data.intent.text, TEXT_CAPS.intent) : undefined;
|
|
538
|
+
const intentSource = data.intent?.source;
|
|
539
|
+
this.enqueueRun(runId, "step", sizeOf(toolInput, context, intentText), () =>
|
|
127
540
|
// `intent` and `metadata` are omitted, not nulled, when absent, so an
|
|
128
541
|
// older service sees the body it has always seen (segmented.md
|
|
129
542
|
// R-COMPAT-1).
|
|
@@ -132,42 +545,49 @@ export class RemoteRecorder {
|
|
|
132
545
|
stepIndex,
|
|
133
546
|
type: "tool_selected",
|
|
134
547
|
toolName: data.toolName,
|
|
135
|
-
toolInput:
|
|
136
|
-
context:
|
|
137
|
-
intent:
|
|
548
|
+
toolInput: out(toolInput),
|
|
549
|
+
context: out(context),
|
|
550
|
+
intent: intentSource ? { text: out(intentText) ?? "", source: intentSource } : undefined,
|
|
138
551
|
metadata: data.metadata,
|
|
139
552
|
}));
|
|
140
553
|
return id;
|
|
141
554
|
}
|
|
142
555
|
recordToolResponse(runId, stepIndex, data) {
|
|
143
556
|
const id = "step_" + randomUUID();
|
|
144
|
-
|
|
557
|
+
const toolOutput = pend(data.toolOutput, DEFAULT_MAX_PAYLOAD_BYTES, true);
|
|
558
|
+
const toolError = pendError(data.toolError);
|
|
559
|
+
this.enqueueRun(runId, "step", sizeOf(toolOutput, toolError), () => this.post(`/recordings/runs/${runId}/steps`, {
|
|
145
560
|
id,
|
|
146
561
|
stepIndex,
|
|
147
562
|
type: "tool_response",
|
|
148
563
|
toolName: data.toolName,
|
|
149
|
-
toolOutput:
|
|
150
|
-
toolError:
|
|
564
|
+
toolOutput: out(toolOutput),
|
|
565
|
+
toolError: out(toolError),
|
|
151
566
|
}));
|
|
152
567
|
return id;
|
|
153
568
|
}
|
|
154
569
|
recordFinalAnswer(runId, stepIndex, data) {
|
|
155
570
|
const id = "step_" + randomUUID();
|
|
156
|
-
|
|
571
|
+
const answer = pend(data.answer, TEXT_CAPS.answer);
|
|
572
|
+
this.enqueueRun(runId, "step", sizeOf(answer), () => this.post(`/recordings/runs/${runId}/steps`, {
|
|
157
573
|
id,
|
|
158
574
|
stepIndex,
|
|
159
575
|
type: "final_answer",
|
|
160
|
-
answer:
|
|
576
|
+
answer: out(answer),
|
|
161
577
|
}));
|
|
162
578
|
return id;
|
|
163
579
|
}
|
|
164
580
|
/** Backfill a step's `context` (the model's reasoning) after the fact. */
|
|
165
581
|
backfillContext(runId, stepId, context) {
|
|
166
|
-
|
|
582
|
+
const text = pend(context, TEXT_CAPS.context);
|
|
583
|
+
this.enqueueRun(runId, "patch", sizeOf(text), () =>
|
|
584
|
+
// A PATCH that sets one field: repeating it is harmless.
|
|
585
|
+
this.send("PATCH", `/recordings/runs/${runId}/steps/${stepId}`, { context: out(text) }));
|
|
167
586
|
}
|
|
168
587
|
finishRun(runId, finalOutput, metrics) {
|
|
169
|
-
|
|
170
|
-
|
|
588
|
+
const output = pend(finalOutput, TEXT_CAPS.answer);
|
|
589
|
+
this.enqueueRun(runId, "finish", sizeOf(output), () => this.post(`/recordings/runs/${runId}/finish`, {
|
|
590
|
+
finalOutput: out(output),
|
|
171
591
|
originalCostUsd: metrics?.costUsd,
|
|
172
592
|
originalMs: metrics?.durationMs,
|
|
173
593
|
}));
|
|
@@ -175,140 +595,847 @@ export class RemoteRecorder {
|
|
|
175
595
|
/**
|
|
176
596
|
* Report what a matched turn cost (docs/calculatedReplay.md §11.2).
|
|
177
597
|
*
|
|
178
|
-
* Queued on the
|
|
179
|
-
* `flush` waits for it
|
|
180
|
-
*
|
|
181
|
-
*
|
|
598
|
+
* Queued on the executions lane — a matched turn has no run on the service —
|
|
599
|
+
* so `flush` waits for it. Best-effort like everything else here: an
|
|
600
|
+
* unredeemed ticket leaves no execution row at all, which loses a saving but
|
|
601
|
+
* can never invent one.
|
|
182
602
|
*
|
|
183
603
|
* A doubled report (a retry, `Stop` racing `SessionEnd`) is safe by
|
|
184
604
|
* construction: the ticket *is* the row's id, so the service books once and
|
|
185
|
-
* answers `200 { duplicate: true }`.
|
|
605
|
+
* answers `200 { duplicate: true }`. That is also why a report *with* a
|
|
606
|
+
* ticket is the one kind retried after an ambiguous failure.
|
|
186
607
|
*/
|
|
187
608
|
reportExecution(report) {
|
|
188
609
|
const costUsd = report.deriveCostUsd + report.sessionCostUsd + report.fallbackCostUsd;
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
610
|
+
const prompt = pend(report.prompt, TEXT_CAPS.prompt);
|
|
611
|
+
const error = pend(report.error, TEXT_CAPS.reportError);
|
|
612
|
+
this.enqueue({
|
|
613
|
+
lane: EXECUTIONS,
|
|
614
|
+
kind: "report",
|
|
615
|
+
bytes: sizeOf(prompt, error) + 1024,
|
|
616
|
+
send: async () => {
|
|
617
|
+
const body = await this.request("POST", `/scenarios/${report.scenarioId}/executions`, JSON.stringify({
|
|
618
|
+
ticket: report.ticket,
|
|
619
|
+
outcome: report.outcome,
|
|
620
|
+
costUsd,
|
|
621
|
+
deriveCostUsd: report.deriveCostUsd,
|
|
622
|
+
sessionCostUsd: report.sessionCostUsd,
|
|
623
|
+
fallbackCostUsd: report.fallbackCostUsd,
|
|
624
|
+
durationMs: report.durationMs,
|
|
625
|
+
stepsPlanned: report.stepsPlanned,
|
|
626
|
+
stepsPinned: report.stepsPinned,
|
|
627
|
+
measured: report.measured,
|
|
628
|
+
pricingVersion: report.pricingVersion,
|
|
629
|
+
prompt: out(prompt),
|
|
630
|
+
// Per-step verdicts and the headline failure (errorshandling.md). All
|
|
631
|
+
// optional on the service side, so an older service ignores them rather
|
|
632
|
+
// than rejecting the report — and the costs still book. Errors are
|
|
633
|
+
// shown on a web page, so they are capped and redacted like any text.
|
|
634
|
+
steps: report.steps?.map((s) => typeof s.error === "string" ? { ...s, error: capText(s.error, TEXT_CAPS.reportError, redactText) } : s),
|
|
635
|
+
error: out(error),
|
|
636
|
+
errorStage: report.errorStage,
|
|
637
|
+
errorStepIndex: report.errorStepIndex,
|
|
638
|
+
errorToolName: report.errorToolName,
|
|
639
|
+
fallbackStepIndex: report.fallbackStepIndex,
|
|
640
|
+
fallbackKind: report.fallbackKind,
|
|
641
|
+
baselineEligible: report.baselineEligible,
|
|
642
|
+
sharedWith: report.sharedWith,
|
|
643
|
+
// Why nothing ran, when nothing ran. Additive: an older service drops
|
|
644
|
+
// the field and books the cost exactly as it always did.
|
|
645
|
+
declined: report.declined,
|
|
646
|
+
}), { retryTransient: Boolean(report.ticket) });
|
|
647
|
+
const r = (body ?? {});
|
|
648
|
+
const failed = report.steps?.filter((s) => s.status === "failed").length ?? 0;
|
|
649
|
+
logLine("execution.reported", {
|
|
650
|
+
run: report.runId,
|
|
651
|
+
scenario: report.scenarioId,
|
|
652
|
+
ticket: report.ticket,
|
|
653
|
+
outcome: report.outcome,
|
|
654
|
+
derive: report.deriveCostUsd.toFixed(4),
|
|
655
|
+
session: report.sessionCostUsd.toFixed(4),
|
|
656
|
+
fallback: report.fallbackCostUsd.toFixed(4),
|
|
657
|
+
savedUsd: r.savedUsd ?? undefined,
|
|
658
|
+
duplicate: r.duplicate ? true : undefined,
|
|
659
|
+
measured: report.measured,
|
|
660
|
+
steps: report.steps?.length,
|
|
661
|
+
stepsFailed: failed || undefined,
|
|
662
|
+
baselineEligible: report.baselineEligible === false ? false : undefined,
|
|
663
|
+
sharedWith: report.sharedWith?.length,
|
|
664
|
+
declined: report.declined,
|
|
665
|
+
});
|
|
666
|
+
},
|
|
667
|
+
});
|
|
668
|
+
}
|
|
669
|
+
/**
|
|
670
|
+
* Await every queued send for a run, including its execution report — for at
|
|
671
|
+
* most `opts.timeoutMs` (default {@link DEFAULT_FLUSH_TIMEOUT_MS}, or
|
|
672
|
+
* `BIR_FLUSH_TIMEOUT_MS`), and not at all while the service is cooling down.
|
|
673
|
+
* Either way the sends carry on in the background; a flush behind a dead
|
|
674
|
+
* service must never be what holds a hook or a shutdown.
|
|
675
|
+
*
|
|
676
|
+
* WHY THE BREAKER ENDS THE WAIT. In the common outages — offline, refused,
|
|
677
|
+
* a 5xx — the first send fails in milliseconds and opens the breaker, and
|
|
678
|
+
* every job after it sleeps out the cool-down before it tries (the service
|
|
679
|
+
* is not hammered). A flush that waited for those sleeps paid its whole
|
|
680
|
+
* deadline on every turn of the outage, where the old fail-at-once chain
|
|
681
|
+
* paid tens of milliseconds. Nothing a cooling-down lane holds can land
|
|
682
|
+
* sooner than the cool-down, so the flush stops waiting the moment the
|
|
683
|
+
* breaker opens.
|
|
684
|
+
*/
|
|
685
|
+
async flush(runId, opts = {}) {
|
|
686
|
+
const timeoutMs = opts.timeoutMs ?? positiveEnv("BIR_FLUSH_TIMEOUT_MS", DEFAULT_FLUSH_TIMEOUT_MS);
|
|
687
|
+
const outcome = await this.waitForLanes([runId, EXECUTIONS], timeoutMs);
|
|
688
|
+
const entry = this.runs.get(runId);
|
|
689
|
+
// The match holds the whole scenario; nobody asks for it after a flush.
|
|
690
|
+
if (entry?.settled?.match)
|
|
691
|
+
entry.settled = { match: null, created: entry.settled.created };
|
|
692
|
+
if (outcome === "timeout") {
|
|
693
|
+
logLine("recorder.flush_incomplete", {
|
|
694
|
+
run: runId,
|
|
695
|
+
pending: this.pendingSends,
|
|
696
|
+
why: "the service is slow or down — the rest is sent in the background, or dropped past the backlog bound",
|
|
218
697
|
});
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
derive: report.deriveCostUsd.toFixed(4),
|
|
227
|
-
session: report.sessionCostUsd.toFixed(4),
|
|
228
|
-
fallback: report.fallbackCostUsd.toFixed(4),
|
|
229
|
-
savedUsd: r.savedUsd ?? undefined,
|
|
230
|
-
duplicate: r.duplicate ? true : undefined,
|
|
231
|
-
measured: report.measured,
|
|
232
|
-
steps: report.steps?.length,
|
|
233
|
-
stepsFailed: failed || undefined,
|
|
234
|
-
baselineEligible: report.baselineEligible === false ? false : undefined,
|
|
235
|
-
sharedWith: report.sharedWith?.length,
|
|
236
|
-
declined: report.declined,
|
|
698
|
+
}
|
|
699
|
+
else if (outcome === "cooling") {
|
|
700
|
+
// Once per 30 s: an outage is one fact, not one line per turn.
|
|
701
|
+
logEvery("recorder.flush_deferred", 30_000, "recorder.flush_deferred", {
|
|
702
|
+
run: runId,
|
|
703
|
+
pending: this.pendingSends,
|
|
704
|
+
why: "the service is cooling down after a failure — nothing waits on it; the rest is sent in the background once it answers",
|
|
237
705
|
});
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
// ── lanes ────────────────────────────────────────────────────────────────
|
|
709
|
+
newEntry() {
|
|
710
|
+
let resolveHot;
|
|
711
|
+
let resolveCreated;
|
|
712
|
+
const entry = {
|
|
713
|
+
userId: this.session.user.id,
|
|
714
|
+
hot: new Promise((r) => {
|
|
715
|
+
resolveHot = r;
|
|
716
|
+
}),
|
|
717
|
+
created: new Promise((r) => {
|
|
718
|
+
resolveCreated = r;
|
|
719
|
+
}),
|
|
720
|
+
settle: (match, created) => {
|
|
721
|
+
if (entry.settled)
|
|
722
|
+
return;
|
|
723
|
+
entry.settled = { match, created };
|
|
724
|
+
entry.settledAt = Date.now();
|
|
725
|
+
resolveHot();
|
|
726
|
+
resolveCreated(created);
|
|
727
|
+
// Callers that are waiting hold the promises themselves.
|
|
728
|
+
entry.hot = undefined;
|
|
729
|
+
entry.created = undefined;
|
|
730
|
+
},
|
|
731
|
+
release: () => {
|
|
732
|
+
if (entry.settled || entry.released)
|
|
733
|
+
return;
|
|
734
|
+
entry.released = true;
|
|
735
|
+
entry.hot = undefined;
|
|
736
|
+
resolveHot();
|
|
737
|
+
},
|
|
738
|
+
lastUsed: Date.now(),
|
|
739
|
+
failed: 0,
|
|
740
|
+
dropped: 0,
|
|
741
|
+
};
|
|
742
|
+
return entry;
|
|
743
|
+
}
|
|
744
|
+
/** The entry for a run, made on first sight for an id this recorder did not open. */
|
|
745
|
+
entryFor(runId) {
|
|
746
|
+
let entry = this.runs.get(runId);
|
|
747
|
+
if (!entry) {
|
|
748
|
+
entry = this.newEntry();
|
|
749
|
+
entry.settle(null, { created: true });
|
|
750
|
+
this.runs.set(runId, entry);
|
|
751
|
+
}
|
|
752
|
+
entry.lastUsed = Date.now();
|
|
753
|
+
return entry;
|
|
754
|
+
}
|
|
755
|
+
enqueueRun(runId, kind, bytes, send) {
|
|
756
|
+
this.entryFor(runId);
|
|
757
|
+
this.enqueue({
|
|
758
|
+
lane: runId,
|
|
759
|
+
runId,
|
|
760
|
+
kind,
|
|
761
|
+
bytes,
|
|
762
|
+
send: async () => {
|
|
763
|
+
await send();
|
|
764
|
+
},
|
|
238
765
|
});
|
|
239
766
|
}
|
|
240
|
-
/**
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
this.
|
|
244
|
-
this.
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
767
|
+
/** Append to a lane. Synchronous and never throws; the lane drains after the caller returns. */
|
|
768
|
+
enqueue(spec) {
|
|
769
|
+
const run = spec.runId ? this.runs.get(spec.runId) : undefined;
|
|
770
|
+
const job = { ...spec, seq: ++this.seq, userId: run?.userId ?? this.session.user.id };
|
|
771
|
+
if (this.state !== "recording")
|
|
772
|
+
return this.retire(job, run, this.state);
|
|
773
|
+
if (run?.dead)
|
|
774
|
+
return this.retire(job, run, run.dead);
|
|
775
|
+
let lane = this.lanes.get(job.lane);
|
|
776
|
+
if (!lane) {
|
|
777
|
+
lane = { jobs: [], running: false, waiters: [] };
|
|
778
|
+
this.lanes.set(job.lane, lane);
|
|
779
|
+
}
|
|
780
|
+
lane.jobs.push(job);
|
|
781
|
+
this.pendingSends += 1;
|
|
782
|
+
this.pendingBytes += job.bytes;
|
|
783
|
+
this.enforceBound();
|
|
784
|
+
if (!lane.running) {
|
|
785
|
+
lane.running = true;
|
|
786
|
+
const started = lane;
|
|
787
|
+
// Next turn of the event loop: redaction and serialization happen off
|
|
788
|
+
// the hook's own path, after its answer is on the wire.
|
|
789
|
+
setImmediate(() => void this.drain(job.lane, started));
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
async drain(key, lane) {
|
|
793
|
+
try {
|
|
794
|
+
for (;;) {
|
|
795
|
+
const job = lane.jobs.shift();
|
|
796
|
+
if (!job)
|
|
797
|
+
break;
|
|
798
|
+
this.pendingSends -= 1;
|
|
799
|
+
this.pendingBytes -= job.bytes;
|
|
800
|
+
this.noteBacklogRecovered();
|
|
801
|
+
await this.execute(job);
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
catch {
|
|
805
|
+
/* execute never throws; belt and braces for the lane's own bookkeeping */
|
|
806
|
+
}
|
|
807
|
+
finally {
|
|
808
|
+
if (lane.jobs.length > 0) {
|
|
809
|
+
setImmediate(() => void this.drain(key, lane));
|
|
810
|
+
}
|
|
811
|
+
else {
|
|
812
|
+
lane.running = false;
|
|
813
|
+
if (this.lanes.get(key) === lane)
|
|
814
|
+
this.lanes.delete(key);
|
|
815
|
+
for (const resolve of lane.waiters.splice(0))
|
|
816
|
+
resolve();
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
laneIsIdle(key) {
|
|
821
|
+
const lane = this.lanes.get(key);
|
|
822
|
+
return !lane || (!lane.running && lane.jobs.length === 0);
|
|
823
|
+
}
|
|
824
|
+
/**
|
|
825
|
+
* Wait until every lane in `keys` is idle, the breaker opens ("cooling"), or
|
|
826
|
+
* `timeoutMs` passes — whichever is first. The breaker is polled (every
|
|
827
|
+
* {@link FLUSH_POLL_MS}) rather than subscribed to: the verdict is shared by
|
|
828
|
+
* the whole process and has no listeners, and a poll only runs while a flush
|
|
829
|
+
* is actually waiting.
|
|
830
|
+
*/
|
|
831
|
+
waitForLanes(keys, timeoutMs) {
|
|
832
|
+
if (keys.every((k) => this.laneIsIdle(k)))
|
|
833
|
+
return Promise.resolve("idle");
|
|
834
|
+
if (!this.health.canWait())
|
|
835
|
+
return Promise.resolve("cooling");
|
|
836
|
+
return new Promise((resolve) => {
|
|
837
|
+
const cancels = [];
|
|
838
|
+
let settled = false;
|
|
839
|
+
const finish = (outcome) => {
|
|
840
|
+
if (settled)
|
|
841
|
+
return;
|
|
842
|
+
settled = true;
|
|
843
|
+
clearTimeout(deadline);
|
|
844
|
+
clearInterval(poll);
|
|
845
|
+
for (const cancel of cancels)
|
|
846
|
+
cancel();
|
|
847
|
+
resolve(outcome);
|
|
848
|
+
};
|
|
849
|
+
// Not unref'd: a caller awaiting this must get its answer even when the
|
|
850
|
+
// only other thing left is a lane asleep in an unref'd cool-down.
|
|
851
|
+
const deadline = setTimeout(() => finish("timeout"), Math.max(0, timeoutMs));
|
|
852
|
+
const poll = setInterval(() => {
|
|
853
|
+
if (!this.health.canWait())
|
|
854
|
+
finish("cooling");
|
|
855
|
+
}, FLUSH_POLL_MS);
|
|
856
|
+
let left = keys.length;
|
|
857
|
+
for (const key of keys) {
|
|
858
|
+
const lane = this.lanes.get(key);
|
|
859
|
+
if (!lane || (!lane.running && lane.jobs.length === 0)) {
|
|
860
|
+
left -= 1;
|
|
861
|
+
continue;
|
|
862
|
+
}
|
|
863
|
+
const waiter = () => {
|
|
864
|
+
left -= 1;
|
|
865
|
+
if (left === 0)
|
|
866
|
+
finish("idle");
|
|
867
|
+
};
|
|
868
|
+
lane.waiters.push(waiter);
|
|
869
|
+
cancels.push(() => {
|
|
870
|
+
const i = lane.waiters.indexOf(waiter);
|
|
871
|
+
if (i >= 0)
|
|
872
|
+
lane.waiters.splice(i, 1);
|
|
873
|
+
});
|
|
874
|
+
}
|
|
875
|
+
if (left === 0)
|
|
876
|
+
finish("idle");
|
|
877
|
+
});
|
|
878
|
+
}
|
|
879
|
+
/** Run one job: decide whether it may still go, wait out a cool-down, send. Never throws. */
|
|
880
|
+
async execute(job) {
|
|
881
|
+
const run = job.runId ? this.runs.get(job.runId) : undefined;
|
|
882
|
+
let endProbe;
|
|
883
|
+
try {
|
|
884
|
+
let why = await this.whyNotSend(job, run);
|
|
885
|
+
if (why)
|
|
886
|
+
return this.retire(job, run, why);
|
|
887
|
+
// Don't hammer a service that just failed: wait out its cool-down, and
|
|
888
|
+
// then let one send (maybe this one) find out whether it came back.
|
|
889
|
+
const wait = await this.waitForService();
|
|
890
|
+
endProbe = wait.endProbe;
|
|
891
|
+
if (wait.giveUp) {
|
|
892
|
+
this.retire(job, run, "service_down");
|
|
893
|
+
// Without its create the rest of the run would 404.
|
|
894
|
+
if (job.kind === "create" && run && job.runId)
|
|
895
|
+
this.markDead(job.runId, run, "service_down");
|
|
896
|
+
return;
|
|
897
|
+
}
|
|
898
|
+
await this.refreshAhead();
|
|
899
|
+
// Both can take a while (a minute of cool-down, a refresh), and a hook
|
|
900
|
+
// may have signed another account in meanwhile: ask again, or a run
|
|
901
|
+
// opened under one account would go out under the next one's token.
|
|
902
|
+
why = await this.whyNotSend(job, run);
|
|
903
|
+
if (why)
|
|
904
|
+
return this.retire(job, run, why);
|
|
905
|
+
await job.send();
|
|
906
|
+
if (job.kind === "finish" && run && job.runId)
|
|
907
|
+
this.summarize(job.runId, run);
|
|
908
|
+
}
|
|
909
|
+
catch (err) {
|
|
910
|
+
if (err instanceof AccountChanged) {
|
|
911
|
+
this.retire(job, run, this.accountChanged(job, run));
|
|
912
|
+
return;
|
|
913
|
+
}
|
|
914
|
+
this.onSendFailed(job, run, err);
|
|
915
|
+
if (job.kind === "finish" && run && job.runId)
|
|
916
|
+
this.summarize(job.runId, run);
|
|
917
|
+
}
|
|
918
|
+
finally {
|
|
919
|
+
endProbe?.();
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
async whyNotSend(job, run) {
|
|
923
|
+
this.checkCredentials();
|
|
924
|
+
if (this.state !== "recording")
|
|
925
|
+
return this.state;
|
|
926
|
+
if (job.userId !== undefined && job.userId !== this.session.user.id)
|
|
927
|
+
return this.accountChanged(job, run);
|
|
928
|
+
if (!run || !job.runId)
|
|
929
|
+
return undefined;
|
|
930
|
+
if (run.userId !== this.session.user.id)
|
|
931
|
+
this.markDead(job.runId, run, "account_changed");
|
|
932
|
+
if (run.dead)
|
|
933
|
+
return run.dead;
|
|
934
|
+
if (job.kind !== "create") {
|
|
935
|
+
// The create went first on this lane, so this is already settled — unless
|
|
936
|
+
// it is still being retried, which is worth the wait.
|
|
937
|
+
const created = run.settled?.created ?? (run.created ? await run.created : { created: true });
|
|
938
|
+
if (!created.created) {
|
|
939
|
+
this.markDead(job.runId, run, "not_created");
|
|
940
|
+
return "not_created";
|
|
941
|
+
}
|
|
942
|
+
if (run.dead)
|
|
943
|
+
return run.dead;
|
|
944
|
+
}
|
|
945
|
+
return undefined;
|
|
946
|
+
}
|
|
947
|
+
/**
|
|
948
|
+
* A job queued under one account, about to go out after another signed in:
|
|
949
|
+
* its run is closed (the rest would split it across accounts), and a report
|
|
950
|
+
* is dropped with a line. Returns the reason to retire it with.
|
|
951
|
+
*/
|
|
952
|
+
accountChanged(job, run) {
|
|
953
|
+
if (run && job.runId) {
|
|
954
|
+
this.markDead(job.runId, run, "account_changed");
|
|
955
|
+
}
|
|
956
|
+
else {
|
|
957
|
+
logEvery("recorder.report_dropped", 30_000, "recorder.report_dropped", {
|
|
958
|
+
why: "another account signed in on this machine before it was sent — it is not sent under that one",
|
|
959
|
+
});
|
|
960
|
+
}
|
|
961
|
+
return "account_changed";
|
|
962
|
+
}
|
|
963
|
+
/** A job leaves without being sent: counted (unless it was never meant to go), and a create settles. */
|
|
964
|
+
retire(job, run, why) {
|
|
965
|
+
try {
|
|
966
|
+
job.abandon?.();
|
|
967
|
+
}
|
|
968
|
+
catch {
|
|
969
|
+
/* never */
|
|
970
|
+
}
|
|
971
|
+
// A matched turn records nothing by design: nothing was lost.
|
|
972
|
+
if (why !== "not_created") {
|
|
973
|
+
this.droppedSends += 1;
|
|
974
|
+
if (run)
|
|
975
|
+
run.dropped += 1;
|
|
976
|
+
}
|
|
977
|
+
if (job.kind === "finish" && run && job.runId)
|
|
978
|
+
this.summarize(job.runId, run);
|
|
979
|
+
}
|
|
980
|
+
/** Skip everything left for a run, once: its sends would all fail the same way. */
|
|
981
|
+
markDead(runId, run, why) {
|
|
982
|
+
if (run.dead)
|
|
983
|
+
return;
|
|
984
|
+
run.dead = why;
|
|
985
|
+
const lane = this.lanes.get(runId);
|
|
986
|
+
if (lane && lane.jobs.length > 0) {
|
|
987
|
+
const jobs = lane.jobs;
|
|
988
|
+
lane.jobs = [];
|
|
989
|
+
for (const job of jobs) {
|
|
990
|
+
this.pendingSends -= 1;
|
|
991
|
+
this.pendingBytes -= job.bytes;
|
|
992
|
+
this.retire(job, run, why);
|
|
993
|
+
}
|
|
994
|
+
}
|
|
995
|
+
if (why === "not_created") {
|
|
996
|
+
logDetail("recorder.run_not_created", { run: runId, why: "the service matched the prompt and kept its own run" });
|
|
997
|
+
}
|
|
998
|
+
else if (!(why in PAUSE_WHY)) {
|
|
999
|
+
logLine("recorder.run_abandoned", { run: runId, reason: why, why: DEAD_WHY[why] ?? why });
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
onSendFailed(job, run, err) {
|
|
1003
|
+
this.failedSends += 1;
|
|
1004
|
+
const status = err instanceof ServiceError ? err.status : undefined;
|
|
1005
|
+
if (run && job.runId) {
|
|
1006
|
+
run.failed += 1;
|
|
1007
|
+
if (job.kind === "create") {
|
|
1008
|
+
// A definite refusal means there is no run; an ambiguous failure (a
|
|
1009
|
+
// timeout, a 5xx) may have created it, and the first step will say.
|
|
1010
|
+
if (status !== undefined && status >= 400 && status < 500 && ![401, 408, 409, 429].includes(status)) {
|
|
1011
|
+
this.markDead(job.runId, run, "create_rejected");
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
else if (status === 404) {
|
|
1015
|
+
this.markDead(job.runId, run, "not_found");
|
|
1016
|
+
}
|
|
1017
|
+
// One line per run, not per step: an outage is one fact, and the count
|
|
1018
|
+
// arrives with the run's finish (recorder.run_incomplete).
|
|
1019
|
+
if (run.failed === 1) {
|
|
1020
|
+
logLine("recorder.send_failed", {
|
|
1021
|
+
run: job.runId,
|
|
1022
|
+
why: "this run's recording is incomplete",
|
|
1023
|
+
error: errText(err),
|
|
1024
|
+
});
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
else {
|
|
1028
|
+
logEvery(`recorder.send_failed:${job.lane}`, 30_000, "recorder.send_failed", {
|
|
1029
|
+
run: job.lane,
|
|
259
1030
|
why: "this run's recording is incomplete",
|
|
260
1031
|
error: errText(err),
|
|
261
1032
|
});
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
/** At a run's finish: say what did not arrive, if anything did not. */
|
|
1036
|
+
summarize(runId, run) {
|
|
1037
|
+
if (run.failed <= 1 && run.dropped === 0)
|
|
1038
|
+
return;
|
|
1039
|
+
logLine("recorder.run_incomplete", {
|
|
1040
|
+
run: runId,
|
|
1041
|
+
failed: run.failed || undefined,
|
|
1042
|
+
dropped: run.dropped || undefined,
|
|
1043
|
+
reason: run.dead,
|
|
1044
|
+
why: "some of this run's steps never reached the service",
|
|
262
1045
|
});
|
|
263
|
-
this.chains.set(runId, next);
|
|
264
1046
|
}
|
|
265
|
-
|
|
266
|
-
|
|
1047
|
+
/** Keep the backlog inside its bound: drop the oldest unsent step first. */
|
|
1048
|
+
enforceBound() {
|
|
1049
|
+
while (this.pendingSends > this.maxPendingSends || this.pendingBytes > this.maxPendingBytes) {
|
|
1050
|
+
const victim = this.pickVictim();
|
|
1051
|
+
if (!victim)
|
|
1052
|
+
return;
|
|
1053
|
+
const lane = this.lanes.get(victim.lane);
|
|
1054
|
+
const index = lane ? lane.jobs.indexOf(victim) : -1;
|
|
1055
|
+
if (!lane || index < 0)
|
|
1056
|
+
return;
|
|
1057
|
+
lane.jobs.splice(index, 1);
|
|
1058
|
+
this.pendingSends -= 1;
|
|
1059
|
+
this.pendingBytes -= victim.bytes;
|
|
1060
|
+
this.droppedWhileFull += 1;
|
|
1061
|
+
if (!this.backlogFull) {
|
|
1062
|
+
this.backlogFull = true;
|
|
1063
|
+
logLine("recorder.backlog_full", {
|
|
1064
|
+
pending: this.pendingSends,
|
|
1065
|
+
bytes: this.pendingBytes,
|
|
1066
|
+
why: "the service is not keeping up — the oldest unsent steps are dropped, so those runs are incomplete",
|
|
1067
|
+
});
|
|
1068
|
+
}
|
|
1069
|
+
const run = victim.runId ? this.runs.get(victim.runId) : undefined;
|
|
1070
|
+
this.retire(victim, run, "dropped");
|
|
1071
|
+
// Without its create the rest of the run would 404.
|
|
1072
|
+
if (victim.kind === "create" && run && victim.runId)
|
|
1073
|
+
this.markDead(victim.runId, run, "dropped");
|
|
1074
|
+
}
|
|
267
1075
|
}
|
|
268
|
-
|
|
269
|
-
|
|
1076
|
+
/**
|
|
1077
|
+
* The oldest unsent step or patch; failing that, the oldest job of any kind.
|
|
1078
|
+
* Steps first because a run without its middle is still a run, and one
|
|
1079
|
+
* without its create or finish is a run the service can never close.
|
|
1080
|
+
*/
|
|
1081
|
+
pickVictim() {
|
|
1082
|
+
let best;
|
|
1083
|
+
for (const lane of this.lanes.values()) {
|
|
1084
|
+
for (const job of lane.jobs) {
|
|
1085
|
+
if (job.kind === "step" || job.kind === "patch") {
|
|
1086
|
+
if (!best || job.seq < best.seq)
|
|
1087
|
+
best = job;
|
|
1088
|
+
break; // lanes are FIFO: the first is the oldest
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
1092
|
+
if (best)
|
|
1093
|
+
return best;
|
|
1094
|
+
for (const lane of this.lanes.values()) {
|
|
1095
|
+
const first = lane.jobs[0];
|
|
1096
|
+
if (first && (!best || first.seq < best.seq))
|
|
1097
|
+
best = first;
|
|
1098
|
+
}
|
|
1099
|
+
return best;
|
|
270
1100
|
}
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
1101
|
+
noteBacklogRecovered() {
|
|
1102
|
+
if (!this.backlogFull)
|
|
1103
|
+
return;
|
|
1104
|
+
if (this.pendingSends > this.maxPendingSends / 2 || this.pendingBytes > this.maxPendingBytes / 2)
|
|
1105
|
+
return;
|
|
1106
|
+
this.backlogFull = false;
|
|
1107
|
+
logLine("recorder.backlog_recovered", { dropped: this.droppedWhileFull });
|
|
1108
|
+
this.droppedWhileFull = 0;
|
|
1109
|
+
}
|
|
1110
|
+
/** Forget runs nobody will ask about again, and release matches past their TTL. */
|
|
1111
|
+
sweep() {
|
|
1112
|
+
const now = Date.now();
|
|
1113
|
+
if (now - this.lastSweep < 60_000 && this.runs.size <= RUN_STATE_MAX)
|
|
1114
|
+
return;
|
|
1115
|
+
this.lastSweep = now;
|
|
1116
|
+
for (const [id, entry] of this.runs) {
|
|
1117
|
+
if (this.lanes.has(id))
|
|
1118
|
+
continue;
|
|
1119
|
+
if (entry.settled?.match && entry.settledAt !== undefined && now - entry.settledAt > MATCH_TTL_MS) {
|
|
1120
|
+
entry.settled = { match: null, created: entry.settled.created };
|
|
1121
|
+
}
|
|
1122
|
+
if (entry.settled && now - entry.lastUsed > RUN_STATE_TTL_MS)
|
|
1123
|
+
this.runs.delete(id);
|
|
283
1124
|
}
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
1125
|
+
if (this.runs.size > RUN_STATE_MAX) {
|
|
1126
|
+
for (const [id, entry] of this.runs) {
|
|
1127
|
+
if (this.runs.size <= RUN_STATE_MAX)
|
|
1128
|
+
break;
|
|
1129
|
+
if (!this.lanes.has(id) && entry.settled)
|
|
1130
|
+
this.runs.delete(id);
|
|
1131
|
+
}
|
|
287
1132
|
}
|
|
288
|
-
|
|
289
|
-
|
|
1133
|
+
}
|
|
1134
|
+
// ── the service ──────────────────────────────────────────────────────────
|
|
1135
|
+
/** The breaker is closed: no failure since the last answer. */
|
|
1136
|
+
healthyNow() {
|
|
1137
|
+
return this.health.canWait() && this.health.snapshot().consecutiveFailures === 0;
|
|
1138
|
+
}
|
|
1139
|
+
/**
|
|
1140
|
+
* May a hook wait on the service right now? Only when it is known to be
|
|
1141
|
+
* answering. A hook is never the half-open probe: once the cool-down ends a
|
|
1142
|
+
* background `GET /health` (or a queued send) finds out, and the next hook
|
|
1143
|
+
* waits again.
|
|
1144
|
+
*/
|
|
1145
|
+
mayHotWait() {
|
|
1146
|
+
this.checkCredentials();
|
|
1147
|
+
if (this.state !== "recording")
|
|
1148
|
+
return false;
|
|
1149
|
+
if (this.healthyNow())
|
|
1150
|
+
return true;
|
|
1151
|
+
this.probeSoon();
|
|
1152
|
+
return false;
|
|
1153
|
+
}
|
|
1154
|
+
probeSoon() {
|
|
1155
|
+
if (this.probeInFlight() || !this.health.canWait())
|
|
1156
|
+
return;
|
|
1157
|
+
const end = this.beginProbe();
|
|
1158
|
+
this.request("GET", "/health", undefined, { timeoutMs: PROBE_TIMEOUT_MS, anonymous: true })
|
|
1159
|
+
.catch(() => undefined)
|
|
1160
|
+
.finally(end);
|
|
1161
|
+
}
|
|
1162
|
+
/** Take the half-open slot. The returned function gives it back, once. */
|
|
1163
|
+
beginProbe() {
|
|
1164
|
+
let resolve;
|
|
1165
|
+
const flight = { done: new Promise((r) => (resolve = r)), startedAt: Date.now() };
|
|
1166
|
+
this.probe = flight;
|
|
1167
|
+
return () => {
|
|
1168
|
+
if (this.probe === flight)
|
|
1169
|
+
this.probe = undefined;
|
|
1170
|
+
resolve();
|
|
1171
|
+
};
|
|
1172
|
+
}
|
|
1173
|
+
/**
|
|
1174
|
+
* Is a probe out? One older than any request could take (every attempt has
|
|
1175
|
+
* a deadline) is a slot somebody failed to give back, and is not honoured.
|
|
1176
|
+
*/
|
|
1177
|
+
probeInFlight() {
|
|
1178
|
+
const p = this.probe;
|
|
1179
|
+
if (!p)
|
|
1180
|
+
return false;
|
|
1181
|
+
if (Date.now() - p.startedAt > 4 * serviceTimeoutMs() + PROBE_TIMEOUT_MS) {
|
|
1182
|
+
this.probe = undefined;
|
|
1183
|
+
return false;
|
|
1184
|
+
}
|
|
1185
|
+
return true;
|
|
1186
|
+
}
|
|
1187
|
+
/** Lanes let go after a successful probe spread over a fraction of the cool-down (at most 2 s). */
|
|
1188
|
+
releaseJitterMs() {
|
|
1189
|
+
const spread = Math.min(RELEASE_JITTER_MAX_MS, Math.floor(this.health.baseCooldown / 2));
|
|
1190
|
+
return Math.floor(Math.random() * Math.max(0, spread));
|
|
1191
|
+
}
|
|
1192
|
+
/**
|
|
1193
|
+
* Wait until a background send may go.
|
|
1194
|
+
*
|
|
1195
|
+
* - The breaker is closed: go at once.
|
|
1196
|
+
* - It is cooling down: sleep until it ends. A send that has waited
|
|
1197
|
+
* {@link COOLDOWN_WAIT_MAX_MS} and still faces more than
|
|
1198
|
+
* {@link COOLDOWN_GRACE_MS} of it is given up (`giveUp`), unsent.
|
|
1199
|
+
* - It is half-open: ONE lane takes the probe slot and goes (`endProbe`
|
|
1200
|
+
* gives the slot back once its send settles); every other lane waits for
|
|
1201
|
+
* that verdict. A success lets them go, each after its own short random
|
|
1202
|
+
* pause; a failure starts a longer cool-down, and they wait that out.
|
|
1203
|
+
*
|
|
1204
|
+
* Before this, every lane with a queued send woke at the same moment and
|
|
1205
|
+
* sent into the dead service together, once per cool-down.
|
|
1206
|
+
*/
|
|
1207
|
+
async waitForService() {
|
|
1208
|
+
const started = Date.now();
|
|
1209
|
+
let waited = false;
|
|
1210
|
+
for (;;) {
|
|
1211
|
+
if (this.state !== "recording")
|
|
1212
|
+
return {};
|
|
1213
|
+
if (this.healthyNow()) {
|
|
1214
|
+
if (!waited)
|
|
1215
|
+
return {};
|
|
1216
|
+
// It came back while this lane waited: follow the probe a moment
|
|
1217
|
+
// later, not together with every other lane.
|
|
1218
|
+
waited = false;
|
|
1219
|
+
await sleep(this.releaseJitterMs(), true);
|
|
1220
|
+
continue;
|
|
1221
|
+
}
|
|
1222
|
+
waited = true;
|
|
1223
|
+
if (this.health.canWait()) {
|
|
1224
|
+
if (!this.probeInFlight())
|
|
1225
|
+
return { endProbe: this.beginProbe() };
|
|
1226
|
+
await Promise.race([this.probe?.done, sleep(PROBE_POLL_MS, true)]);
|
|
1227
|
+
continue;
|
|
1228
|
+
}
|
|
1229
|
+
const left = this.cooldownWaitMaxMs - (Date.now() - started);
|
|
1230
|
+
const cooling = (this.health.snapshot().openUntil ?? 0) - Date.now();
|
|
1231
|
+
if (left <= 0 && cooling > Math.min(COOLDOWN_GRACE_MS, this.cooldownWaitMaxMs / 12))
|
|
1232
|
+
return { giveUp: true };
|
|
1233
|
+
// Past the bound, only a cool-down about to end is still waited out.
|
|
1234
|
+
await sleep(Math.min(Math.max(cooling, 25), left > 0 ? left : Number.POSITIVE_INFINITY, 5_000), true);
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1237
|
+
/** Renew the access token shortly before it expires — in a lane, never on a hook. */
|
|
1238
|
+
async refreshAhead() {
|
|
1239
|
+
if (this.session.accessExpiresAt - REFRESH_AHEAD_MS > Date.now())
|
|
1240
|
+
return;
|
|
1241
|
+
if (Date.now() - this.lastAheadAttempt < REFRESH_AHEAD_RETRY_MS)
|
|
1242
|
+
return;
|
|
1243
|
+
this.lastAheadAttempt = Date.now();
|
|
290
1244
|
try {
|
|
291
|
-
|
|
1245
|
+
await this.refreshSingleFlight();
|
|
292
1246
|
}
|
|
293
1247
|
catch {
|
|
294
|
-
|
|
1248
|
+
/* the send meets its 401 and says so */
|
|
1249
|
+
}
|
|
1250
|
+
}
|
|
1251
|
+
post(path, body) {
|
|
1252
|
+
return this.send("POST", path, body);
|
|
1253
|
+
}
|
|
1254
|
+
/** A background send: every lane job is safe to repeat once (client ids, a PATCH, a ticket). */
|
|
1255
|
+
send(method, path, body) {
|
|
1256
|
+
return this.request(method, path, JSON.stringify(body), { retryTransient: true });
|
|
1257
|
+
}
|
|
1258
|
+
/**
|
|
1259
|
+
* One request, with a deadline, reported to the health verdict, with the
|
|
1260
|
+
* one-refresh-on-401 rule and — for background sends — one retry after a
|
|
1261
|
+
* transient failure.
|
|
1262
|
+
*
|
|
1263
|
+
* WHICH FAILURES ARE RETRIED. Only those that prove the service did not take
|
|
1264
|
+
* the request: a connection that never happened or a pooled socket the load
|
|
1265
|
+
* balancer had already closed ({@link SAFE_RETRY_CODES}), and a 429 or 503.
|
|
1266
|
+
* Not a 502/504 or a reset mid-request — the origin may have the first
|
|
1267
|
+
* attempt — not a timeout (a hung service would just cost another deadline)
|
|
1268
|
+
* and not while the service is already failing (the breaker's cool-down is
|
|
1269
|
+
* the retry then). Every background body carries a client-generated id — the
|
|
1270
|
+
* run, the step, the execution ticket — or is a PATCH of one field, so even
|
|
1271
|
+
* a repeat whose first attempt did land is recognisable; a 409 on the retry
|
|
1272
|
+
* is taken as that.
|
|
1273
|
+
*
|
|
1274
|
+
* THE ACCOUNT IS FIXED FOR THE REQUEST. A retry or a refresh leaves the event
|
|
1275
|
+
* loop, and a hook may sign another account in meanwhile; the next attempt
|
|
1276
|
+
* then goes nowhere ({@link AccountChanged}) instead of out under that
|
|
1277
|
+
* account's token.
|
|
1278
|
+
*
|
|
1279
|
+
* EVERY ANSWER SAYS WHEN ITS ATTEMPT WAS SENT. `health.ok(started)`: the late
|
|
1280
|
+
* answer of an attempt a hook already gave up on (`slow`) must not end the
|
|
1281
|
+
* cool-down that overrun started.
|
|
1282
|
+
*
|
|
1283
|
+
* `onRetry` runs when the first attempt failed and a retry is about to
|
|
1284
|
+
* sleep: whoever waits on this answer on a hook's behalf can stop now.
|
|
1285
|
+
*/
|
|
1286
|
+
async request(method, path, body, opts = {}) {
|
|
1287
|
+
let retried401 = false;
|
|
1288
|
+
let retriedTransient = false;
|
|
1289
|
+
const user = this.session.user.id;
|
|
1290
|
+
for (let attempt = 0;; attempt++) {
|
|
1291
|
+
if (attempt > 0 && !opts.anonymous && this.session.user.id !== user) {
|
|
1292
|
+
throw new AccountChanged(`${method} ${path}`);
|
|
1293
|
+
}
|
|
1294
|
+
const token = this.session.accessToken;
|
|
1295
|
+
const timeoutMs = opts.timeoutMs ?? serviceTimeoutMs();
|
|
1296
|
+
const signal = timeoutSignal(timeoutMs, opts.signal);
|
|
1297
|
+
const started = this.health.stamp();
|
|
1298
|
+
let res;
|
|
1299
|
+
try {
|
|
1300
|
+
res = await fetch(`${this.baseUrl}${path}`, {
|
|
1301
|
+
method,
|
|
1302
|
+
headers: {
|
|
1303
|
+
...(body !== undefined ? { "content-type": "application/json" } : {}),
|
|
1304
|
+
...(opts.anonymous ? {} : { authorization: `Bearer ${token}` }),
|
|
1305
|
+
},
|
|
1306
|
+
body,
|
|
1307
|
+
signal,
|
|
1308
|
+
});
|
|
1309
|
+
}
|
|
1310
|
+
catch (err) {
|
|
1311
|
+
if (opts.signal?.aborted)
|
|
1312
|
+
throw err; // the caller stopped waiting; says nothing about the service
|
|
1313
|
+
const timedOut = signal.aborted;
|
|
1314
|
+
const reason = timedOut ? `timed out after ${timeoutMs} ms` : reasonOf(err);
|
|
1315
|
+
const safe = !timedOut && SAFE_RETRY_CODES.has(causeCode(err) ?? "");
|
|
1316
|
+
if (safe && opts.retryTransient && !retriedTransient && this.healthyNow()) {
|
|
1317
|
+
retriedTransient = true;
|
|
1318
|
+
notify(opts.onRetry);
|
|
1319
|
+
await sleep(jitterMs());
|
|
1320
|
+
continue;
|
|
1321
|
+
}
|
|
1322
|
+
this.health.fail(reason);
|
|
1323
|
+
throw new ServiceError(`${method} ${path}: ${reason}`);
|
|
1324
|
+
}
|
|
1325
|
+
if (res.status === 401 && !opts.anonymous) {
|
|
1326
|
+
// Release the socket now rather than at GC, then renew — once.
|
|
1327
|
+
await res.body?.cancel().catch(() => undefined);
|
|
1328
|
+
this.health.ok(started);
|
|
1329
|
+
if (retried401)
|
|
1330
|
+
throw new ServiceError(`HTTP 401 ${method} ${path}`, 401);
|
|
1331
|
+
retried401 = true;
|
|
1332
|
+
// Another send (or a sibling process) already renewed the token this
|
|
1333
|
+
// one used: just resend with the new one. The wait for a renewal is
|
|
1334
|
+
// held to this attempt's own deadline — a probe's caller is a hook.
|
|
1335
|
+
if (this.session.accessToken === token) {
|
|
1336
|
+
try {
|
|
1337
|
+
await untilAborted(this.refreshSingleFlight(), signal);
|
|
1338
|
+
}
|
|
1339
|
+
catch (err) {
|
|
1340
|
+
if (opts.signal?.aborted || !signal.aborted)
|
|
1341
|
+
throw err;
|
|
1342
|
+
throw new ServiceError(`HTTP 401 ${method} ${path}: the token refresh outlasted ${timeoutMs} ms`, 401);
|
|
1343
|
+
}
|
|
1344
|
+
}
|
|
1345
|
+
if (this.state !== "recording")
|
|
1346
|
+
throw new ServiceError(`HTTP 401 ${method} ${path}`, 401);
|
|
1347
|
+
continue;
|
|
1348
|
+
}
|
|
1349
|
+
const type = res.headers.get("content-type") ?? "";
|
|
1350
|
+
let text;
|
|
1351
|
+
try {
|
|
1352
|
+
text = await res.text();
|
|
1353
|
+
}
|
|
1354
|
+
catch (err) {
|
|
1355
|
+
if (opts.signal?.aborted)
|
|
1356
|
+
throw err;
|
|
1357
|
+
const reason = signal.aborted ? `timed out after ${timeoutMs} ms reading the answer` : reasonOf(err);
|
|
1358
|
+
this.health.fail(reason);
|
|
1359
|
+
throw new ServiceError(`${method} ${path}: ${reason}`, res.status);
|
|
1360
|
+
}
|
|
1361
|
+
if (isUnhealthyStatus(res.status)) {
|
|
1362
|
+
if (opts.retryTransient && !retriedTransient && RETRY_STATUSES.has(res.status) && this.healthyNow()) {
|
|
1363
|
+
retriedTransient = true;
|
|
1364
|
+
notify(opts.onRetry);
|
|
1365
|
+
await sleep(retryDelayMs(res));
|
|
1366
|
+
continue;
|
|
1367
|
+
}
|
|
1368
|
+
this.health.fail(`HTTP ${res.status}`);
|
|
1369
|
+
throw new ServiceError(`HTTP ${res.status} ${method} ${path}${detail(text, type)}`, res.status);
|
|
1370
|
+
}
|
|
1371
|
+
// A 2xx web page is a captive portal or a website, and a 403 one a CDN's
|
|
1372
|
+
// challenge — not the API answering. An empty or plain 2xx is: only the
|
|
1373
|
+
// body decides, not a `text/html` a framework put on `res.send("ok")`.
|
|
1374
|
+
if ((res.ok || res.status === 403) && isWebPage(text, type)) {
|
|
1375
|
+
this.health.fail("html");
|
|
1376
|
+
throw new ServiceError(`HTTP ${res.status} ${method} ${path} — the answer was a web page, not the BaseIn API`, res.status);
|
|
1377
|
+
}
|
|
1378
|
+
// Anything else is the service answering, including a 4xx — an older
|
|
1379
|
+
// service's 404 must not look like an outage.
|
|
1380
|
+
this.health.ok(started);
|
|
1381
|
+
if (!res.ok) {
|
|
1382
|
+
if (res.status === 409 && retriedTransient)
|
|
1383
|
+
return undefined; // the first attempt landed
|
|
1384
|
+
throw new ServiceError(`HTTP ${res.status} ${method} ${path}${detail(text, type)}`, res.status);
|
|
1385
|
+
}
|
|
1386
|
+
if (!text)
|
|
1387
|
+
return undefined;
|
|
1388
|
+
try {
|
|
1389
|
+
return JSON.parse(text);
|
|
1390
|
+
}
|
|
1391
|
+
catch {
|
|
1392
|
+
return undefined;
|
|
1393
|
+
}
|
|
295
1394
|
}
|
|
296
1395
|
}
|
|
1396
|
+
// ── the session ──────────────────────────────────────────────────────────
|
|
1397
|
+
/**
|
|
1398
|
+
* Refresh once for everyone. Concurrent 401s — a run's create, an execution
|
|
1399
|
+
* report and a probe crossing the expiry together — used to each spend the
|
|
1400
|
+
* rotating refresh token; one won, the others were rejected and their sends
|
|
1401
|
+
* (a create: the whole turn) were dropped.
|
|
1402
|
+
*/
|
|
1403
|
+
refreshSingleFlight() {
|
|
1404
|
+
if (!this.refreshing) {
|
|
1405
|
+
this.refreshing = this.refresh().finally(() => {
|
|
1406
|
+
this.refreshing = undefined;
|
|
1407
|
+
});
|
|
1408
|
+
}
|
|
1409
|
+
return this.refreshing;
|
|
1410
|
+
}
|
|
297
1411
|
/**
|
|
298
1412
|
* Adopt a newer session written by a sibling process, if there is one.
|
|
299
1413
|
*
|
|
300
1414
|
* Returns true when this recorder now holds a usable token it did not have to
|
|
301
|
-
* spend a refresh on.
|
|
1415
|
+
* spend a refresh on. Only a session for the same service and the same
|
|
1416
|
+
* account is adopted: the binding is what keeps a token from being sent to
|
|
1417
|
+
* an address it was not issued for, and an account change is a run boundary
|
|
1418
|
+
* ({@link checkCredentials}), never a mid-run swap.
|
|
302
1419
|
*/
|
|
303
1420
|
adoptStoredSession() {
|
|
304
1421
|
const stored = loadCredentials();
|
|
305
1422
|
if (!stored)
|
|
306
1423
|
return false;
|
|
1424
|
+
if (!issuedFor(stored, this.binding))
|
|
1425
|
+
return false;
|
|
1426
|
+
if (stored.user.id !== this.session.user.id)
|
|
1427
|
+
return false;
|
|
307
1428
|
if (stored.accessToken === this.session.accessToken)
|
|
308
1429
|
return false;
|
|
309
|
-
if (stored.accessExpiresAt - 30_000 <= Date.now())
|
|
1430
|
+
if (stored.accessExpiresAt - 30_000 <= Date.now()) {
|
|
1431
|
+
// Expired, but its refresh token is newer than ours: spend that one.
|
|
1432
|
+
if (stored.refreshToken !== this.session.refreshToken) {
|
|
1433
|
+
this.session = { ...this.session, refreshToken: stored.refreshToken };
|
|
1434
|
+
}
|
|
310
1435
|
return false;
|
|
311
|
-
|
|
1436
|
+
}
|
|
1437
|
+
this.session = { ...stored, authUrl: stored.authUrl ?? this.binding };
|
|
1438
|
+
this.noteFileSeen();
|
|
312
1439
|
logLine("auth.adopted", { why: "another process had already refreshed" });
|
|
313
1440
|
return true;
|
|
314
1441
|
}
|
|
@@ -325,37 +1452,139 @@ export class RemoteRecorder {
|
|
|
325
1452
|
* The fix is to treat the credentials file as the shared source of truth:
|
|
326
1453
|
* check it before spending our token, and check it again if our refresh is
|
|
327
1454
|
* rejected, because the process that beat us has already written a good one.
|
|
1455
|
+
* A rejection with nothing to adopt pauses the recorder (`auth_rejected`)
|
|
1456
|
+
* instead of paying two round trips on every later send; a new sign-in
|
|
1457
|
+
* resumes it, and so does a retry every {@link AUTH_REJECTED_RETRY_MS}.
|
|
1458
|
+
*
|
|
1459
|
+
* WHAT COUNTS AS A REJECTION. Only the service's own JSON 400/401 — the
|
|
1460
|
+
* answer that says this refresh token is spent. A 403 or a web page (a CDN
|
|
1461
|
+
* challenge, a portal) says nothing about the token, and pausing on one
|
|
1462
|
+
* stopped recording for the rest of a daemon's life. And a rejection is
|
|
1463
|
+
* believed only after one more look at the file ~{@link AUTH_RECHECK_MS}
|
|
1464
|
+
* later: a sibling that rotated the token a moment ago may not have written
|
|
1465
|
+
* it yet. Any other failure is not retried for {@link REFRESH_RETRY_MS}, so a
|
|
1466
|
+
* refresh route that keeps failing costs one request per half minute, not
|
|
1467
|
+
* one per send.
|
|
328
1468
|
*/
|
|
329
1469
|
async refresh() {
|
|
330
1470
|
if (this.adoptStoredSession())
|
|
331
1471
|
return;
|
|
1472
|
+
if (Date.now() < this.refreshRetryAt) {
|
|
1473
|
+
throw new ServiceError("refresh failed moments ago — not tried again yet", 401);
|
|
1474
|
+
}
|
|
1475
|
+
const spent = this.session.refreshToken;
|
|
332
1476
|
try {
|
|
333
1477
|
await this.refreshOverNetwork();
|
|
1478
|
+
this.refreshRetryAt = 0;
|
|
334
1479
|
}
|
|
335
1480
|
catch (err) {
|
|
336
1481
|
// Someone may have refreshed while we were failing to.
|
|
337
1482
|
if (this.adoptStoredSession())
|
|
338
1483
|
return;
|
|
1484
|
+
if (err instanceof RefreshRejected) {
|
|
1485
|
+
await sleep(this.authRecheckMs, true);
|
|
1486
|
+
if (this.adoptStoredSession())
|
|
1487
|
+
return;
|
|
1488
|
+
// Adopted by a hook's credentials check while we waited, or signed
|
|
1489
|
+
// out: either way this refusal is no longer the news.
|
|
1490
|
+
if (this.session.refreshToken !== spent || this.state !== "recording")
|
|
1491
|
+
throw err;
|
|
1492
|
+
this.rejectedAt = Date.now();
|
|
1493
|
+
this.pause("auth_rejected");
|
|
1494
|
+
}
|
|
1495
|
+
else if (this.health.canWait()) {
|
|
1496
|
+
// The service answered, just not with a session (a JSON 403, a 404):
|
|
1497
|
+
// asking again on the next send would get the same. A failure that
|
|
1498
|
+
// opened the breaker needs no hold of its own — the cool-down is it.
|
|
1499
|
+
this.refreshRetryAt = Date.now() + REFRESH_RETRY_MS;
|
|
1500
|
+
}
|
|
339
1501
|
throw err;
|
|
340
1502
|
}
|
|
341
1503
|
}
|
|
1504
|
+
/**
|
|
1505
|
+
* While `auth_rejected`, try the refresh again now and then, in the
|
|
1506
|
+
* background: a refusal the service later takes back (a bad deploy, a
|
|
1507
|
+
* clock), or one the recheck missed, must not need a `bir login` to end.
|
|
1508
|
+
* Never awaited by the caller — a hook's credentials check starts it.
|
|
1509
|
+
*/
|
|
1510
|
+
retryRejected() {
|
|
1511
|
+
if (this.state !== "auth_rejected" || this.refreshing)
|
|
1512
|
+
return;
|
|
1513
|
+
if (Date.now() - this.rejectedAt < this.authRetryMs)
|
|
1514
|
+
return;
|
|
1515
|
+
this.rejectedAt = Date.now();
|
|
1516
|
+
this.refreshRetryAt = 0;
|
|
1517
|
+
this.refreshSingleFlight()
|
|
1518
|
+
.then(() => {
|
|
1519
|
+
if (this.state === "auth_rejected")
|
|
1520
|
+
this.resume("the service accepted the session again");
|
|
1521
|
+
})
|
|
1522
|
+
.catch(() => undefined);
|
|
1523
|
+
}
|
|
342
1524
|
async refreshOverNetwork() {
|
|
343
|
-
const
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
1525
|
+
const timeoutMs = serviceTimeoutMs();
|
|
1526
|
+
const signal = timeoutSignal(timeoutMs);
|
|
1527
|
+
const started = this.health.stamp();
|
|
1528
|
+
let res;
|
|
1529
|
+
try {
|
|
1530
|
+
res = await fetch(`${this.baseUrl}/auth/refresh`, {
|
|
1531
|
+
method: "POST",
|
|
1532
|
+
headers: { "content-type": "application/json" },
|
|
1533
|
+
body: JSON.stringify({ refreshToken: this.session.refreshToken }),
|
|
1534
|
+
signal,
|
|
1535
|
+
});
|
|
1536
|
+
}
|
|
1537
|
+
catch (err) {
|
|
1538
|
+
const reason = signal.aborted ? `timed out after ${timeoutMs} ms` : reasonOf(err);
|
|
1539
|
+
this.health.fail(reason);
|
|
1540
|
+
throw new ServiceError(`refresh failed: ${reason}`);
|
|
1541
|
+
}
|
|
1542
|
+
const type = res.headers.get("content-type") ?? "";
|
|
1543
|
+
const text = await res.text().catch(() => "");
|
|
1544
|
+
if (!res.ok) {
|
|
1545
|
+
const page = isWebPage(text, type);
|
|
1546
|
+
if (isUnhealthyStatus(res.status))
|
|
1547
|
+
this.health.fail(`HTTP ${res.status}`);
|
|
1548
|
+
else if (page && res.status === 403)
|
|
1549
|
+
this.health.fail("html");
|
|
1550
|
+
else
|
|
1551
|
+
this.health.ok(started);
|
|
1552
|
+
if ((res.status === 400 || res.status === 401) && !page && isJsonObject(text)) {
|
|
1553
|
+
throw new RefreshRejected(`refresh refused: HTTP ${res.status}${detail(text, type)}`, res.status);
|
|
1554
|
+
}
|
|
1555
|
+
throw new ServiceError(`refresh failed: HTTP ${res.status}${page ? " (a web page)" : ""}`, res.status);
|
|
1556
|
+
}
|
|
1557
|
+
let body;
|
|
1558
|
+
try {
|
|
1559
|
+
body = JSON.parse(text);
|
|
1560
|
+
}
|
|
1561
|
+
catch {
|
|
1562
|
+
body = undefined;
|
|
1563
|
+
}
|
|
1564
|
+
if (!isAuthResponse(body)) {
|
|
1565
|
+
// Never saved: the next process would load a session with no token, call
|
|
1566
|
+
// the file corrupt, and the refresh token just rotated would be lost.
|
|
1567
|
+
const page = /html/i.test(type) || /^\s*</.test(text);
|
|
1568
|
+
if (page)
|
|
1569
|
+
this.health.fail("html");
|
|
1570
|
+
else
|
|
1571
|
+
this.health.ok(started);
|
|
1572
|
+
throw new ServiceError(`refresh failed: the answer was not a session${page ? " (a web page)" : ""} — nothing saved`);
|
|
1573
|
+
}
|
|
1574
|
+
this.health.ok(started);
|
|
351
1575
|
this.session = {
|
|
352
|
-
accessToken:
|
|
353
|
-
refreshToken:
|
|
354
|
-
accessExpiresAt: Date.now() +
|
|
355
|
-
user:
|
|
1576
|
+
accessToken: body.accessToken,
|
|
1577
|
+
refreshToken: body.refreshToken,
|
|
1578
|
+
accessExpiresAt: Date.now() + body.expiresIn * 1000,
|
|
1579
|
+
user: body.user,
|
|
1580
|
+
// Keep the binding. Dropping it here quietly turned off the check that
|
|
1581
|
+
// stops a session being handed to another address, for every process,
|
|
1582
|
+
// from the first refresh on.
|
|
1583
|
+
authUrl: this.binding,
|
|
356
1584
|
};
|
|
357
1585
|
try {
|
|
358
1586
|
saveCredentials(this.session); // persist so the next run reuses it
|
|
1587
|
+
this.noteFileSeen();
|
|
359
1588
|
}
|
|
360
1589
|
catch (err) {
|
|
361
1590
|
// Persisting is a convenience for the *next* process. Letting it throw
|
|
@@ -363,5 +1592,106 @@ export class RemoteRecorder {
|
|
|
363
1592
|
logLine("auth.save_failed", { error: errText(err), why: "the token still works in-process" });
|
|
364
1593
|
}
|
|
365
1594
|
}
|
|
1595
|
+
/** The file now holds this recorder's session: follow it from here on. */
|
|
1596
|
+
noteFileSeen() {
|
|
1597
|
+
this.follow = true;
|
|
1598
|
+
try {
|
|
1599
|
+
this.credMtime = statSync(credentialsPath()).mtimeMs;
|
|
1600
|
+
this.credCheckedAt = Date.now();
|
|
1601
|
+
}
|
|
1602
|
+
catch {
|
|
1603
|
+
/* looked at again on the next check */
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
/**
|
|
1607
|
+
* Keep in step with credentials.json, cheaply: one `stat` per
|
|
1608
|
+
* {@link CREDENTIALS_CHECK_MS}, and a read only when it changed.
|
|
1609
|
+
*
|
|
1610
|
+
* - gone → `bir logout`: pause (`signed_out`), drop what is queued.
|
|
1611
|
+
* - another service → pause (`other_service`): never send there.
|
|
1612
|
+
* - another account → adopt it for runs opened from now on; runs already
|
|
1613
|
+
* open are closed, so no run is split across two accounts.
|
|
1614
|
+
* - newer tokens → adopt them (a sibling refreshed, or `bir login`), and
|
|
1615
|
+
* resume if paused.
|
|
1616
|
+
*/
|
|
1617
|
+
checkCredentials() {
|
|
1618
|
+
this.retryRejected();
|
|
1619
|
+
if (!this.follow)
|
|
1620
|
+
return;
|
|
1621
|
+
const now = Date.now();
|
|
1622
|
+
if (now - this.credCheckedAt < this.credentialsCheckMs)
|
|
1623
|
+
return;
|
|
1624
|
+
this.credCheckedAt = now;
|
|
1625
|
+
let mtime;
|
|
1626
|
+
try {
|
|
1627
|
+
mtime = statSync(credentialsPath()).mtimeMs;
|
|
1628
|
+
}
|
|
1629
|
+
catch (err) {
|
|
1630
|
+
if (err.code === "ENOENT") {
|
|
1631
|
+
this.credMtime = -1;
|
|
1632
|
+
this.pause("signed_out");
|
|
1633
|
+
}
|
|
1634
|
+
return;
|
|
1635
|
+
}
|
|
1636
|
+
if (mtime === this.credMtime)
|
|
1637
|
+
return;
|
|
1638
|
+
const stored = loadCredentials();
|
|
1639
|
+
if (!stored)
|
|
1640
|
+
return; // unreadable for now: look again next time
|
|
1641
|
+
this.credMtime = mtime;
|
|
1642
|
+
if (!issuedFor(stored, this.binding)) {
|
|
1643
|
+
this.pause("other_service");
|
|
1644
|
+
return;
|
|
1645
|
+
}
|
|
1646
|
+
if (stored.user.id !== this.session.user.id) {
|
|
1647
|
+
this.session = { ...stored, authUrl: stored.authUrl ?? this.binding };
|
|
1648
|
+
this.refreshRetryAt = 0;
|
|
1649
|
+
logLine("recorder.account_changed", {
|
|
1650
|
+
user: stored.user.email,
|
|
1651
|
+
why: "another account signed in on this machine — runs already open stop here, new ones record under it",
|
|
1652
|
+
});
|
|
1653
|
+
if (this.state !== "recording")
|
|
1654
|
+
this.resume();
|
|
1655
|
+
return;
|
|
1656
|
+
}
|
|
1657
|
+
const sameTokens = stored.accessToken === this.session.accessToken && stored.refreshToken === this.session.refreshToken;
|
|
1658
|
+
if (!sameTokens) {
|
|
1659
|
+
this.refreshRetryAt = 0;
|
|
1660
|
+
if (stored.accessExpiresAt - 30_000 > now || this.state !== "recording") {
|
|
1661
|
+
this.session = { ...stored, authUrl: stored.authUrl ?? this.binding };
|
|
1662
|
+
}
|
|
1663
|
+
else if (stored.refreshToken !== this.session.refreshToken) {
|
|
1664
|
+
this.session = { ...this.session, refreshToken: stored.refreshToken };
|
|
1665
|
+
}
|
|
1666
|
+
if (this.state !== "recording")
|
|
1667
|
+
this.resume();
|
|
1668
|
+
}
|
|
1669
|
+
else if (this.state === "signed_out" || this.state === "other_service") {
|
|
1670
|
+
// The same session came back (a restore): nothing was refused.
|
|
1671
|
+
this.resume();
|
|
1672
|
+
}
|
|
1673
|
+
}
|
|
1674
|
+
pause(state) {
|
|
1675
|
+
if (this.state === state)
|
|
1676
|
+
return;
|
|
1677
|
+
this.state = state;
|
|
1678
|
+
let dropped = 0;
|
|
1679
|
+
for (const lane of this.lanes.values()) {
|
|
1680
|
+
const jobs = lane.jobs;
|
|
1681
|
+
lane.jobs = [];
|
|
1682
|
+
for (const job of jobs) {
|
|
1683
|
+
this.pendingSends -= 1;
|
|
1684
|
+
this.pendingBytes -= job.bytes;
|
|
1685
|
+
this.retire(job, job.runId ? this.runs.get(job.runId) : undefined, state);
|
|
1686
|
+
dropped += 1;
|
|
1687
|
+
}
|
|
1688
|
+
}
|
|
1689
|
+
logLine("recorder.paused", { reason: state, dropped: dropped || undefined, why: PAUSE_WHY[state] });
|
|
1690
|
+
}
|
|
1691
|
+
resume(why = "a session is back in credentials.json") {
|
|
1692
|
+
this.state = "recording";
|
|
1693
|
+
this.lastAheadAttempt = 0;
|
|
1694
|
+
logLine("recorder.resumed", { user: this.session.user.email, why });
|
|
1695
|
+
}
|
|
366
1696
|
}
|
|
367
1697
|
//# sourceMappingURL=remote-recorder.js.map
|