@basein/runner 0.3.25 → 0.3.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -26
- package/dist/auth/client.d.ts +4 -1
- package/dist/auth/client.js +83 -14
- package/dist/bin/bir-hooks.js +4 -2
- package/dist/bin/bir.js +153 -37
- package/dist/bin/investigate.d.ts +13 -0
- package/dist/bin/investigate.js +72 -2
- package/dist/bin/scenario-edit.d.ts +5 -0
- package/dist/bin/scenario-edit.js +17 -1
- package/dist/config/adapters/claude-code.d.ts +51 -7
- package/dist/config/adapters/claude-code.js +105 -18
- package/dist/config/generate.d.ts +61 -13
- package/dist/config/generate.js +108 -20
- package/dist/config/resolve.d.ts +5 -0
- package/dist/config/resolve.js +7 -0
- package/dist/control/correlation.js +0 -0
- package/dist/control/daemon.d.ts +28 -0
- package/dist/control/daemon.js +62 -0
- package/dist/control/paths.d.ts +19 -1
- package/dist/control/paths.js +20 -2
- package/dist/control/pin-guard.js +8 -2
- package/dist/control/recorder-main.js +17 -5
- package/dist/control/server.d.ts +75 -11
- package/dist/control/server.js +440 -77
- package/dist/control/transcript.d.ts +64 -3
- package/dist/control/transcript.js +318 -37
- package/dist/jsonrpc/framing.d.ts +18 -3
- package/dist/jsonrpc/framing.js +57 -4
- package/dist/proxy/intercept.d.ts +4 -0
- package/dist/proxy/intercept.js +4 -0
- package/dist/proxy/relay.d.ts +13 -0
- package/dist/proxy/relay.js +12 -1
- package/dist/proxy/session.d.ts +8 -1
- package/dist/proxy/session.js +67 -10
- package/dist/record/recorder.d.ts +6 -1
- package/dist/record/redact.d.ts +18 -4
- package/dist/record/redact.js +538 -79
- package/dist/record/remote-recorder.d.ts +26 -5
- package/dist/record/remote-recorder.js +174 -18
- package/dist/record/truncate.js +3 -1
- package/dist/replay/bash-channel.d.ts +7 -1
- package/dist/replay/bash-channel.js +9 -3
- package/dist/replay/builtins.d.ts +14 -0
- package/dist/replay/builtins.js +460 -95
- package/dist/replay/bundle.d.ts +15 -11
- package/dist/replay/bundle.js +167 -33
- package/dist/replay/controller.d.ts +92 -1
- package/dist/replay/controller.js +246 -47
- package/dist/replay/derive.js +46 -4
- package/dist/replay/handover.d.ts +13 -0
- package/dist/replay/handover.js +18 -0
- package/dist/replay/plan.d.ts +41 -0
- package/dist/replay/plan.js +171 -3
- package/dist/replay/pricing.d.ts +61 -11
- package/dist/replay/pricing.js +135 -42
- package/dist/replay/reads-only.d.ts +48 -0
- package/dist/replay/reads-only.js +161 -0
- package/dist/replay/safety.js +43 -5
- package/dist/upstream/http-client.d.ts +6 -3
- package/dist/upstream/http-client.js +54 -13
- package/dist/upstream/stdio-client.d.ts +7 -0
- package/dist/upstream/stdio-client.js +1 -1
- package/dist/util/atomic-write.d.ts +9 -1
- package/dist/util/atomic-write.js +9 -6
- package/dist/util/service-health.d.ts +7 -0
- package/dist/util/service-health.js +10 -3
- package/package.json +1 -1
- package/production.md +2 -2
- package/scripts/install.ps1 +14 -2
|
@@ -25,7 +25,17 @@
|
|
|
25
25
|
* too; its retry is for the recording, never for a hook.
|
|
26
26
|
* - **One probe, not a stampede.** While the breaker is half-open, one
|
|
27
27
|
* queued send finds out for all the lanes; the rest wait for its verdict
|
|
28
|
-
* and then follow it a moment apart, not all at once.
|
|
28
|
+
* and then follow it a moment apart, not all at once. When a turn's last
|
|
29
|
+
* send fails and none is queued, one timer for the end of the cool-down
|
|
30
|
+
* asks with a `GET /health` instead, so the next prompt finds the service
|
|
31
|
+
* answering again ({@link CooldownTimer}).
|
|
32
|
+
* - **A send the service takes twice is not given up on one failure.** A
|
|
33
|
+
* run's finish, a step's patch and an execution report with a ticket go
|
|
34
|
+
* back to the head of their lane when a try ends without the service's
|
|
35
|
+
* verdict (a timeout, a reset, a 5xx), {@link REPEATABLE_TRIES} tries in
|
|
36
|
+
* all, paced by the breaker's cool-down and never given up on its wait. A
|
|
37
|
+
* finish lost to one 502 leaves a run that never ends. Every other send is
|
|
38
|
+
* tried once: the service does not deduplicate their ids.
|
|
29
39
|
* - **Bounded.** The lanes are the real backlog (the hook server's step queue
|
|
30
40
|
* drains instantly into them), so they carry the bound: past
|
|
31
41
|
* {@link DEFAULT_MAX_PENDING_SENDS} sends or {@link DEFAULT_MAX_PENDING_BYTES}
|
|
@@ -35,7 +45,8 @@
|
|
|
35
45
|
* expiry, then one retry. Sessions outlive access tokens.
|
|
36
46
|
* - **It follows credentials.json.** `bir logout` removes the file; a
|
|
37
47
|
* recorder that loaded its session from it stops sending within seconds,
|
|
38
|
-
* rather than recording for the rest of the access token's life.
|
|
48
|
+
* rather than recording for the rest of the access token's life. A token
|
|
49
|
+
* refresh answered after the file changed is dropped, not saved over it.
|
|
39
50
|
* - **It redacts and caps every free-text field it sends** — the prompt, the
|
|
40
51
|
* answer, reasoning, errors — as the last line of defence. Structured tool
|
|
41
52
|
* payloads are redacted by their producers; text is only ever seen here.
|
|
@@ -170,6 +181,8 @@ export declare class RemoteRecorder implements Recorder, RecorderHealthReporter
|
|
|
170
181
|
* or a hook-requested `GET /health` — and when it started.
|
|
171
182
|
*/
|
|
172
183
|
private probe?;
|
|
184
|
+
/** Asks again, at the end of a cool-down, when no send is left to ({@link drain}). */
|
|
185
|
+
private readonly idleProbe;
|
|
173
186
|
private matchIntentOffUntil;
|
|
174
187
|
/** True once this recorder's session is known to be the one in credentials.json. */
|
|
175
188
|
private follow;
|
|
@@ -316,7 +329,8 @@ export declare class RemoteRecorder implements Recorder, RecorderHealthReporter
|
|
|
316
329
|
* A doubled report (a retry, `Stop` racing `SessionEnd`) is safe by
|
|
317
330
|
* construction: the ticket *is* the row's id, so the service books once and
|
|
318
331
|
* answers `200 { duplicate: true }`. That is also why a report *with* a
|
|
319
|
-
* ticket is
|
|
332
|
+
* ticket is sent again after an ambiguous failure ({@link REPEATABLE_TRIES}),
|
|
333
|
+
* as a run's finish and a step's patch are; a report without one is not.
|
|
320
334
|
*/
|
|
321
335
|
reportExecution(report: ExecutionReport): void;
|
|
322
336
|
/**
|
|
@@ -352,7 +366,11 @@ export declare class RemoteRecorder implements Recorder, RecorderHealthReporter
|
|
|
352
366
|
* is actually waiting.
|
|
353
367
|
*/
|
|
354
368
|
private waitForLanes;
|
|
355
|
-
/**
|
|
369
|
+
/**
|
|
370
|
+
* Run one job: decide whether it may still go, wait out a cool-down, send.
|
|
371
|
+
* A repeatable job that got no verdict goes back to the head of `lane`, its
|
|
372
|
+
* own, to be run again. Never throws.
|
|
373
|
+
*/
|
|
356
374
|
private execute;
|
|
357
375
|
private whyNotSend;
|
|
358
376
|
/**
|
|
@@ -404,7 +422,10 @@ export declare class RemoteRecorder implements Recorder, RecorderHealthReporter
|
|
|
404
422
|
* - The breaker is closed: go at once.
|
|
405
423
|
* - It is cooling down: sleep until it ends. A send that has waited
|
|
406
424
|
* {@link COOLDOWN_WAIT_MAX_MS} and still faces more than
|
|
407
|
-
* {@link COOLDOWN_GRACE_MS} of it is given up (`giveUp`), unsent
|
|
425
|
+
* {@link COOLDOWN_GRACE_MS} of it is given up (`giveUp`), unsent — unless
|
|
426
|
+
* `mayGiveUp` is false: a repeatable send ({@link REPEATABLE_TRIES}) is
|
|
427
|
+
* bounded by its tries, each one paced by a cool-down, and a finish given
|
|
428
|
+
* up after a minute's outage left a run that never ends.
|
|
408
429
|
* - It is half-open: ONE lane takes the probe slot and goes (`endProbe`
|
|
409
430
|
* gives the slot back once its send settles); every other lane waits for
|
|
410
431
|
* that verdict. A success lets them go, each after its own short random
|
|
@@ -25,7 +25,17 @@
|
|
|
25
25
|
* too; its retry is for the recording, never for a hook.
|
|
26
26
|
* - **One probe, not a stampede.** While the breaker is half-open, one
|
|
27
27
|
* queued send finds out for all the lanes; the rest wait for its verdict
|
|
28
|
-
* and then follow it a moment apart, not all at once.
|
|
28
|
+
* and then follow it a moment apart, not all at once. When a turn's last
|
|
29
|
+
* send fails and none is queued, one timer for the end of the cool-down
|
|
30
|
+
* asks with a `GET /health` instead, so the next prompt finds the service
|
|
31
|
+
* answering again ({@link CooldownTimer}).
|
|
32
|
+
* - **A send the service takes twice is not given up on one failure.** A
|
|
33
|
+
* run's finish, a step's patch and an execution report with a ticket go
|
|
34
|
+
* back to the head of their lane when a try ends without the service's
|
|
35
|
+
* verdict (a timeout, a reset, a 5xx), {@link REPEATABLE_TRIES} tries in
|
|
36
|
+
* all, paced by the breaker's cool-down and never given up on its wait. A
|
|
37
|
+
* finish lost to one 502 leaves a run that never ends. Every other send is
|
|
38
|
+
* tried once: the service does not deduplicate their ids.
|
|
29
39
|
* - **Bounded.** The lanes are the real backlog (the hook server's step queue
|
|
30
40
|
* drains instantly into them), so they carry the bound: past
|
|
31
41
|
* {@link DEFAULT_MAX_PENDING_SENDS} sends or {@link DEFAULT_MAX_PENDING_BYTES}
|
|
@@ -35,7 +45,8 @@
|
|
|
35
45
|
* expiry, then one retry. Sessions outlive access tokens.
|
|
36
46
|
* - **It follows credentials.json.** `bir logout` removes the file; a
|
|
37
47
|
* recorder that loaded its session from it stops sending within seconds,
|
|
38
|
-
* rather than recording for the rest of the access token's life.
|
|
48
|
+
* rather than recording for the rest of the access token's life. A token
|
|
49
|
+
* refresh answered after the file changed is dropped, not saved over it.
|
|
39
50
|
* - **It redacts and caps every free-text field it sends** — the prompt, the
|
|
40
51
|
* answer, reasoning, errors — as the last line of defence. Structured tool
|
|
41
52
|
* payloads are redacted by their producers; text is only ever seen here.
|
|
@@ -49,11 +60,11 @@
|
|
|
49
60
|
* before the answer came.
|
|
50
61
|
*/
|
|
51
62
|
import { randomUUID } from "node:crypto";
|
|
52
|
-
import { statSync } from "node:fs";
|
|
63
|
+
import { existsSync, statSync } from "node:fs";
|
|
53
64
|
import { credentialsPath, describeBody, isAuthResponse, issuedFor, loadCredentials, saveCredentials, } from "../auth/client.js";
|
|
54
65
|
import { errText, logDetail, logEvery, logLine } from "../util/log.js";
|
|
55
66
|
import { serviceHeaders } from "../util/service-headers.js";
|
|
56
|
-
import { isUnhealthyStatus, serviceHealth, serviceTimeoutMs, timeoutSignal, } from "../util/service-health.js";
|
|
67
|
+
import { isUnhealthyStatus, MAX_TIMER_MS, serviceHealth, serviceTimeoutMs, timeoutSignal, } from "../util/service-health.js";
|
|
57
68
|
import { redact, redactJsonText, redactText } from "./redact.js";
|
|
58
69
|
import { capJsonText, capText, DEFAULT_MAX_PAYLOAD_BYTES, preCut, REPORT_PAYLOAD_MAX_BYTES, truncateString, } from "./truncate.js";
|
|
59
70
|
/**
|
|
@@ -141,6 +152,18 @@ const RETRY_STATUSES = new Set([429, 503]);
|
|
|
141
152
|
* double it. A reset mid-request is not among them.
|
|
142
153
|
*/
|
|
143
154
|
const SAFE_RETRY_CODES = new Set(["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "UND_ERR_SOCKET", "UND_ERR_CONNECT_TIMEOUT"]);
|
|
155
|
+
/**
|
|
156
|
+
* How often, in all, a send the service takes twice without harm is tried
|
|
157
|
+
* when a try ends without the service's verdict (a timeout, a reset, a 5xx):
|
|
158
|
+
* a run's finish (an update the service expects more than once), a step's
|
|
159
|
+
* patch (it sets one field), and an execution report with a ticket (the
|
|
160
|
+
* ticket is the row's id, and a repeat is answered `200 { duplicate: true }`).
|
|
161
|
+
* Why the other sends are not repeated — the service does not deduplicate
|
|
162
|
+
* their ids — does not hold for these three, and one lost try cost a run that
|
|
163
|
+
* never ends (the service embeds a run at its finish, so its prompt never
|
|
164
|
+
* matches and it is never calculated) or a turn whose cost is never booked.
|
|
165
|
+
*/
|
|
166
|
+
const REPEATABLE_TRIES = 3;
|
|
144
167
|
/** A refresh that failed without an answer to act on is not tried again sooner than this. */
|
|
145
168
|
const REFRESH_RETRY_MS = 30_000;
|
|
146
169
|
/** After a refused refresh, how long to wait for a sibling's newer session to land on disk. */
|
|
@@ -375,9 +398,10 @@ function pendError(text) {
|
|
|
375
398
|
return undefined;
|
|
376
399
|
return /^\s*[{[]/.test(text) ? pend(text, DEFAULT_MAX_PAYLOAD_BYTES, true) : pend(text, TEXT_CAPS.toolError);
|
|
377
400
|
}
|
|
401
|
+
/** A positive number from the environment, held to what a timer takes ({@link MAX_TIMER_MS}). */
|
|
378
402
|
function positiveEnv(name, fallback) {
|
|
379
403
|
const n = Number(process.env[name]);
|
|
380
|
-
return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback;
|
|
404
|
+
return Number.isFinite(n) && n > 0 ? Math.min(Math.floor(n), MAX_TIMER_MS) : fallback;
|
|
381
405
|
}
|
|
382
406
|
function reasonOf(err) {
|
|
383
407
|
const cause = err?.cause;
|
|
@@ -388,6 +412,15 @@ function causeCode(err) {
|
|
|
388
412
|
const cause = err?.cause;
|
|
389
413
|
return cause?.code ?? cause?.errors?.[0]?.code;
|
|
390
414
|
}
|
|
415
|
+
/**
|
|
416
|
+
* Did a send end without the service's verdict on it? No answer at all (a
|
|
417
|
+
* timeout, a reset, no connection), or a status that is the service being
|
|
418
|
+
* unwell rather than answering (a 5xx, 408, 429). Any other status is its
|
|
419
|
+
* answer, and final.
|
|
420
|
+
*/
|
|
421
|
+
function noVerdict(err) {
|
|
422
|
+
return err instanceof ServiceError && (err.status === undefined || isUnhealthyStatus(err.status));
|
|
423
|
+
}
|
|
391
424
|
/**
|
|
392
425
|
* An answer that is a web page rather than the API: a captive portal, a
|
|
393
426
|
* website, a CDN challenge. Judged by the body — an empty or plain-text 2xx
|
|
@@ -462,6 +495,51 @@ function retryDelayMs(res) {
|
|
|
462
495
|
}
|
|
463
496
|
return jitterMs();
|
|
464
497
|
}
|
|
498
|
+
/**
|
|
499
|
+
* The one timer that asks a failing service again when no send is left to.
|
|
500
|
+
*
|
|
501
|
+
* A failure stays on record until the service next answers, and after a
|
|
502
|
+
* turn's last send the next request is the next prompt's create — whose hook
|
|
503
|
+
* does not wait for its match while a failure is on record. So one 502 at the
|
|
504
|
+
* end of a turn left the next prompt unreplayed, twenty minutes later,
|
|
505
|
+
* although the service had answered everybody else all along: nothing asked
|
|
506
|
+
* it between turns.
|
|
507
|
+
*
|
|
508
|
+
* One timer, for the end of the cool-down, so the service is asked at most
|
|
509
|
+
* once per cool-down; unref'd, so a recorder with nothing to send is never
|
|
510
|
+
* what keeps a process alive; and nothing while no failure is on record.
|
|
511
|
+
*/
|
|
512
|
+
class CooldownTimer {
|
|
513
|
+
health;
|
|
514
|
+
ask;
|
|
515
|
+
timer;
|
|
516
|
+
constructor(health,
|
|
517
|
+
/** Called once the cool-down is over. */
|
|
518
|
+
ask) {
|
|
519
|
+
this.health = health;
|
|
520
|
+
this.ask = ask;
|
|
521
|
+
}
|
|
522
|
+
arm() {
|
|
523
|
+
if (this.timer || this.health.snapshot().consecutiveFailures === 0)
|
|
524
|
+
return;
|
|
525
|
+
const left = (this.health.snapshot().openUntil ?? 0) - Date.now();
|
|
526
|
+
this.timer = setTimeout(() => {
|
|
527
|
+
this.timer = undefined;
|
|
528
|
+
// A send that was already out has failed since, and made the cool-down
|
|
529
|
+
// longer (a turn's finish and its report, one 502 after the other):
|
|
530
|
+
// asking now would be asking into it. Wait that out too.
|
|
531
|
+
if (this.health.canWait())
|
|
532
|
+
this.ask();
|
|
533
|
+
else
|
|
534
|
+
this.arm();
|
|
535
|
+
}, Math.max(left, 25));
|
|
536
|
+
this.timer.unref?.();
|
|
537
|
+
}
|
|
538
|
+
clear() {
|
|
539
|
+
clearTimeout(this.timer);
|
|
540
|
+
this.timer = undefined;
|
|
541
|
+
}
|
|
542
|
+
}
|
|
465
543
|
const PAUSE_WHY = {
|
|
466
544
|
signed_out: "credentials.json was removed (bir logout) — nothing more is sent and replay stops until you sign in again",
|
|
467
545
|
auth_rejected: "the service refused this session and no newer one was on disk — run `bir login`; recording resumes on its own (the session is also retried every 10 minutes)",
|
|
@@ -509,6 +587,8 @@ export class RemoteRecorder {
|
|
|
509
587
|
* or a hook-requested `GET /health` — and when it started.
|
|
510
588
|
*/
|
|
511
589
|
probe;
|
|
590
|
+
/** Asks again, at the end of a cool-down, when no send is left to ({@link drain}). */
|
|
591
|
+
idleProbe;
|
|
512
592
|
matchIntentOffUntil = 0;
|
|
513
593
|
/** True once this recorder's session is known to be the one in credentials.json. */
|
|
514
594
|
follow = false;
|
|
@@ -527,6 +607,7 @@ export class RemoteRecorder {
|
|
|
527
607
|
this.authRecheckMs = opts.authRecheckMs ?? AUTH_RECHECK_MS;
|
|
528
608
|
this.authRetryMs = opts.authRetryMs ?? AUTH_REJECTED_RETRY_MS;
|
|
529
609
|
this.cooldownWaitMaxMs = opts.cooldownWaitMaxMs ?? COOLDOWN_WAIT_MAX_MS;
|
|
610
|
+
this.idleProbe = new CooldownTimer(this.health, () => this.probeIfFailing());
|
|
530
611
|
// Follow the file only when this session came from it. A synthetic session
|
|
531
612
|
// (BIR_AUTH_DISABLE, tests) has no file, and must not read "no file" as
|
|
532
613
|
// "signed out".
|
|
@@ -979,7 +1060,8 @@ export class RemoteRecorder {
|
|
|
979
1060
|
* A doubled report (a retry, `Stop` racing `SessionEnd`) is safe by
|
|
980
1061
|
* construction: the ticket *is* the row's id, so the service books once and
|
|
981
1062
|
* answers `200 { duplicate: true }`. That is also why a report *with* a
|
|
982
|
-
* ticket is
|
|
1063
|
+
* ticket is sent again after an ambiguous failure ({@link REPEATABLE_TRIES}),
|
|
1064
|
+
* as a run's finish and a step's patch are; a report without one is not.
|
|
983
1065
|
*/
|
|
984
1066
|
reportExecution(report) {
|
|
985
1067
|
const costUsd = report.deriveCostUsd + report.sessionCostUsd + report.fallbackCostUsd;
|
|
@@ -999,6 +1081,8 @@ export class RemoteRecorder {
|
|
|
999
1081
|
bytes: sizeOf(prompt, error, answer, ...(steps ?? []).flatMap((s) => [s.toolInput, s.toolOutput])) +
|
|
1000
1082
|
Math.min(callBytes, ANSWER_CALLS_MAX_BYTES + 64 * 1024) +
|
|
1001
1083
|
1024,
|
|
1084
|
+
// Without a ticket nothing tells the service a repeat from a second report.
|
|
1085
|
+
repeatable: Boolean(report.ticket),
|
|
1002
1086
|
send: async () => {
|
|
1003
1087
|
// Shaped here, off the hook's path: the cuts serialize the calls a few times.
|
|
1004
1088
|
const answerCalls = calls ? answerCallsBody(calls) : undefined;
|
|
@@ -1189,6 +1273,7 @@ export class RemoteRecorder {
|
|
|
1189
1273
|
send: async () => {
|
|
1190
1274
|
await send();
|
|
1191
1275
|
},
|
|
1276
|
+
repeatable: kind === "finish" || kind === "patch",
|
|
1192
1277
|
});
|
|
1193
1278
|
}
|
|
1194
1279
|
/** Append to a lane. Synchronous and never throws; the lane drains after the caller returns. */
|
|
@@ -1225,7 +1310,7 @@ export class RemoteRecorder {
|
|
|
1225
1310
|
this.pendingSends -= 1;
|
|
1226
1311
|
this.pendingBytes -= job.bytes;
|
|
1227
1312
|
this.noteBacklogRecovered();
|
|
1228
|
-
await this.execute(job);
|
|
1313
|
+
await this.execute(job, lane);
|
|
1229
1314
|
}
|
|
1230
1315
|
}
|
|
1231
1316
|
catch {
|
|
@@ -1241,6 +1326,10 @@ export class RemoteRecorder {
|
|
|
1241
1326
|
this.lanes.delete(key);
|
|
1242
1327
|
for (const resolve of lane.waiters.splice(0))
|
|
1243
1328
|
resolve();
|
|
1329
|
+
// Its last send may have left a failure on record, and no send is
|
|
1330
|
+
// queued here to find out when the service answers again.
|
|
1331
|
+
if (this.state === "recording")
|
|
1332
|
+
this.idleProbe.arm();
|
|
1244
1333
|
}
|
|
1245
1334
|
}
|
|
1246
1335
|
}
|
|
@@ -1275,7 +1364,7 @@ export class RemoteRecorder {
|
|
|
1275
1364
|
};
|
|
1276
1365
|
// Not unref'd: a caller awaiting this must get its answer even when the
|
|
1277
1366
|
// only other thing left is a lane asleep in an unref'd cool-down.
|
|
1278
|
-
const deadline = setTimeout(() => finish("timeout"), Math.max(0, timeoutMs));
|
|
1367
|
+
const deadline = setTimeout(() => finish("timeout"), Math.min(Math.max(0, timeoutMs), MAX_TIMER_MS));
|
|
1279
1368
|
const poll = setInterval(() => {
|
|
1280
1369
|
if (!this.health.canWait())
|
|
1281
1370
|
finish("cooling");
|
|
@@ -1303,8 +1392,12 @@ export class RemoteRecorder {
|
|
|
1303
1392
|
finish("idle");
|
|
1304
1393
|
});
|
|
1305
1394
|
}
|
|
1306
|
-
/**
|
|
1307
|
-
|
|
1395
|
+
/**
|
|
1396
|
+
* Run one job: decide whether it may still go, wait out a cool-down, send.
|
|
1397
|
+
* A repeatable job that got no verdict goes back to the head of `lane`, its
|
|
1398
|
+
* own, to be run again. Never throws.
|
|
1399
|
+
*/
|
|
1400
|
+
async execute(job, lane) {
|
|
1308
1401
|
const run = job.runId ? this.runs.get(job.runId) : undefined;
|
|
1309
1402
|
let endProbe;
|
|
1310
1403
|
try {
|
|
@@ -1312,8 +1405,9 @@ export class RemoteRecorder {
|
|
|
1312
1405
|
if (why)
|
|
1313
1406
|
return this.retire(job, run, why);
|
|
1314
1407
|
// Don't hammer a service that just failed: wait out its cool-down, and
|
|
1315
|
-
// then let one send (maybe this one) find out whether it came back.
|
|
1316
|
-
|
|
1408
|
+
// then let one send (maybe this one) find out whether it came back. A
|
|
1409
|
+
// repeatable job waits however long that takes: its tries are its bound.
|
|
1410
|
+
const wait = await this.waitForService(!job.repeatable);
|
|
1317
1411
|
endProbe = wait.endProbe;
|
|
1318
1412
|
if (wait.giveUp) {
|
|
1319
1413
|
this.retire(job, run, "service_down");
|
|
@@ -1322,7 +1416,11 @@ export class RemoteRecorder {
|
|
|
1322
1416
|
this.markDead(job.runId, run, "service_down");
|
|
1323
1417
|
return;
|
|
1324
1418
|
}
|
|
1325
|
-
|
|
1419
|
+
// Not once paused: the wait above ends at once then (`bir logout` while
|
|
1420
|
+
// this send slept out a cool-down), and a refresh here would spend the
|
|
1421
|
+
// token of the session that was just signed out.
|
|
1422
|
+
if (this.state === "recording")
|
|
1423
|
+
await this.refreshAhead();
|
|
1326
1424
|
// Both can take a while (a minute of cool-down, a refresh), and a hook
|
|
1327
1425
|
// may have signed another account in meanwhile: ask again, or a run
|
|
1328
1426
|
// opened under one account would go out under the next one's token.
|
|
@@ -1338,6 +1436,26 @@ export class RemoteRecorder {
|
|
|
1338
1436
|
this.retire(job, run, this.accountChanged(job, run));
|
|
1339
1437
|
return;
|
|
1340
1438
|
}
|
|
1439
|
+
if (job.repeatable && noVerdict(err)) {
|
|
1440
|
+
job.tries = (job.tries ?? 0) + 1;
|
|
1441
|
+
if (job.tries < REPEATABLE_TRIES) {
|
|
1442
|
+
// Not lost yet, so not counted as failed. At the head, so nothing
|
|
1443
|
+
// queued behind it goes first; and when it goes is the breaker's
|
|
1444
|
+
// business — the failure just reported opened it, and this job
|
|
1445
|
+
// sleeps out the cool-down like any other (it is then the probe).
|
|
1446
|
+
lane.jobs.unshift(job);
|
|
1447
|
+
this.pendingSends += 1;
|
|
1448
|
+
this.pendingBytes += job.bytes;
|
|
1449
|
+
logDetail("recorder.send_again", {
|
|
1450
|
+
run: job.lane,
|
|
1451
|
+
kind: job.kind,
|
|
1452
|
+
tries: job.tries,
|
|
1453
|
+
error: errText(err),
|
|
1454
|
+
why: "the service did not say whether it took this one, and takes it twice without harm",
|
|
1455
|
+
});
|
|
1456
|
+
return;
|
|
1457
|
+
}
|
|
1458
|
+
}
|
|
1341
1459
|
this.onSendFailed(job, run, err);
|
|
1342
1460
|
if (job.kind === "finish" && run && job.runId)
|
|
1343
1461
|
this.summarize(job.runId, run);
|
|
@@ -1589,7 +1707,11 @@ export class RemoteRecorder {
|
|
|
1589
1707
|
return;
|
|
1590
1708
|
const end = this.beginProbe();
|
|
1591
1709
|
this.request("GET", "/health", undefined, { timeoutMs: PROBE_TIMEOUT_MS, anonymous: true })
|
|
1592
|
-
|
|
1710
|
+
// Still down: ask again at the end of the cool-down this failure started.
|
|
1711
|
+
.catch(() => {
|
|
1712
|
+
if (this.state === "recording")
|
|
1713
|
+
this.idleProbe.arm();
|
|
1714
|
+
})
|
|
1593
1715
|
.finally(end);
|
|
1594
1716
|
}
|
|
1595
1717
|
/** Take the half-open slot. The returned function gives it back, once. */
|
|
@@ -1628,7 +1750,10 @@ export class RemoteRecorder {
|
|
|
1628
1750
|
* - The breaker is closed: go at once.
|
|
1629
1751
|
* - It is cooling down: sleep until it ends. A send that has waited
|
|
1630
1752
|
* {@link COOLDOWN_WAIT_MAX_MS} and still faces more than
|
|
1631
|
-
* {@link COOLDOWN_GRACE_MS} of it is given up (`giveUp`), unsent
|
|
1753
|
+
* {@link COOLDOWN_GRACE_MS} of it is given up (`giveUp`), unsent — unless
|
|
1754
|
+
* `mayGiveUp` is false: a repeatable send ({@link REPEATABLE_TRIES}) is
|
|
1755
|
+
* bounded by its tries, each one paced by a cool-down, and a finish given
|
|
1756
|
+
* up after a minute's outage left a run that never ends.
|
|
1632
1757
|
* - It is half-open: ONE lane takes the probe slot and goes (`endProbe`
|
|
1633
1758
|
* gives the slot back once its send settles); every other lane waits for
|
|
1634
1759
|
* that verdict. A success lets them go, each after its own short random
|
|
@@ -1637,7 +1762,7 @@ export class RemoteRecorder {
|
|
|
1637
1762
|
* Before this, every lane with a queued send woke at the same moment and
|
|
1638
1763
|
* sent into the dead service together, once per cool-down.
|
|
1639
1764
|
*/
|
|
1640
|
-
async waitForService() {
|
|
1765
|
+
async waitForService(mayGiveUp = true) {
|
|
1641
1766
|
const started = Date.now();
|
|
1642
1767
|
let waited = false;
|
|
1643
1768
|
for (;;) {
|
|
@@ -1661,7 +1786,7 @@ export class RemoteRecorder {
|
|
|
1661
1786
|
}
|
|
1662
1787
|
const left = this.cooldownWaitMaxMs - (Date.now() - started);
|
|
1663
1788
|
const cooling = (this.health.snapshot().openUntil ?? 0) - Date.now();
|
|
1664
|
-
if (left <= 0 && cooling > Math.min(COOLDOWN_GRACE_MS, this.cooldownWaitMaxMs / 12))
|
|
1789
|
+
if (mayGiveUp && left <= 0 && cooling > Math.min(COOLDOWN_GRACE_MS, this.cooldownWaitMaxMs / 12))
|
|
1665
1790
|
return { giveUp: true };
|
|
1666
1791
|
// Past the bound, only a cool-down about to end is still waited out.
|
|
1667
1792
|
await sleep(Math.min(Math.max(cooling, 25), left > 0 ? left : Number.POSITIVE_INFINITY, 5_000), true);
|
|
@@ -1962,12 +2087,15 @@ export class RemoteRecorder {
|
|
|
1962
2087
|
const timeoutMs = serviceTimeoutMs();
|
|
1963
2088
|
const signal = timeoutSignal(timeoutMs);
|
|
1964
2089
|
const started = this.health.stamp();
|
|
2090
|
+
// The session this spends: the answer is for it alone, and by the time it
|
|
2091
|
+
// comes credentials.json may hold another, or none.
|
|
2092
|
+
const spent = this.session;
|
|
1965
2093
|
let res;
|
|
1966
2094
|
try {
|
|
1967
2095
|
res = await fetch(`${this.baseUrl}/auth/refresh`, {
|
|
1968
2096
|
method: "POST",
|
|
1969
2097
|
headers: { "content-type": "application/json", ...serviceHeaders() },
|
|
1970
|
-
body: JSON.stringify({ refreshToken:
|
|
2098
|
+
body: JSON.stringify({ refreshToken: spent.refreshToken }),
|
|
1971
2099
|
signal,
|
|
1972
2100
|
});
|
|
1973
2101
|
}
|
|
@@ -2009,6 +2137,33 @@ export class RemoteRecorder {
|
|
|
2009
2137
|
throw new ServiceError(`refresh failed: the answer was not a session${page ? " (a web page)" : ""} — nothing saved`);
|
|
2010
2138
|
}
|
|
2011
2139
|
this.health.ok(started);
|
|
2140
|
+
// THE FILE MAY HAVE MOVED ON while the request was out: `bir logout`
|
|
2141
|
+
// removed it, or somebody signed in. Adopting and saving the answer then
|
|
2142
|
+
// signed the machine back in after the logout, or wrote over the new
|
|
2143
|
+
// sign-in. So the file is read once more, and when it no longer holds the
|
|
2144
|
+
// session this refresh spent, the file decides and the answer is dropped —
|
|
2145
|
+
// the keeper's rule for its own refresh (control/recorder-keeper.ts,
|
|
2146
|
+
// `not_ours`). A file that is there but cannot be read is no sign-in to
|
|
2147
|
+
// protect: it is written over, as before.
|
|
2148
|
+
if (this.follow) {
|
|
2149
|
+
const onDisk = loadCredentials();
|
|
2150
|
+
if (!onDisk && !existsSync(credentialsPath())) {
|
|
2151
|
+
this.credMtime = -1;
|
|
2152
|
+
this.pause("signed_out");
|
|
2153
|
+
return;
|
|
2154
|
+
}
|
|
2155
|
+
if (onDisk && (onDisk.refreshToken !== spent.refreshToken || onDisk.user.id !== spent.user.id)) {
|
|
2156
|
+
logDetail("auth.refresh_superseded", {
|
|
2157
|
+
why: "the sign-in in credentials.json changed while a refresh was out; what is there now decides",
|
|
2158
|
+
});
|
|
2159
|
+
// The credentials check takes it from here — now, whatever the file's
|
|
2160
|
+
// timestamp says and however lately it was looked at.
|
|
2161
|
+
this.credMtime = -1;
|
|
2162
|
+
this.credCheckedAt = 0;
|
|
2163
|
+
this.checkCredentials();
|
|
2164
|
+
return;
|
|
2165
|
+
}
|
|
2166
|
+
}
|
|
2012
2167
|
this.session = {
|
|
2013
2168
|
accessToken: body.accessToken,
|
|
2014
2169
|
refreshToken: body.refreshToken,
|
|
@@ -2112,6 +2267,7 @@ export class RemoteRecorder {
|
|
|
2112
2267
|
if (this.state === state)
|
|
2113
2268
|
return;
|
|
2114
2269
|
this.state = state;
|
|
2270
|
+
this.idleProbe.clear();
|
|
2115
2271
|
let dropped = 0;
|
|
2116
2272
|
for (const lane of this.lanes.values()) {
|
|
2117
2273
|
const jobs = lane.jobs;
|
package/dist/record/truncate.js
CHANGED
|
@@ -159,7 +159,9 @@ export function truncateStrings(value, max, depth = 0) {
|
|
|
159
159
|
if (value && typeof value === "object") {
|
|
160
160
|
const out = {};
|
|
161
161
|
for (const [k, v] of Object.entries(value)) {
|
|
162
|
-
out
|
|
162
|
+
// Defined, not assigned: `out.__proto__ = …` sets the copy's prototype,
|
|
163
|
+
// and a tool's `__proto__` key was missing from the recorded step.
|
|
164
|
+
Object.defineProperty(out, k, { value: truncateStrings(v, max, depth + 1), enumerable: true, writable: true, configurable: true });
|
|
163
165
|
}
|
|
164
166
|
return out;
|
|
165
167
|
}
|
|
@@ -19,7 +19,13 @@
|
|
|
19
19
|
*/
|
|
20
20
|
/** Up to this many characters the text rides inside the command, as a heredoc. */
|
|
21
21
|
export declare const HEREDOC_MAX = 6000;
|
|
22
|
-
/**
|
|
22
|
+
/**
|
|
23
|
+
* Where delivered texts wait to be read. Under `BIR_HOME`, mode 0700.
|
|
24
|
+
*
|
|
25
|
+
* Absolute whatever `BIR_HOME` says: the path is handed to a Bash that runs in
|
|
26
|
+
* the session's folder, not this process's, and a relative one was not found
|
|
27
|
+
* there.
|
|
28
|
+
*/
|
|
23
29
|
export declare function deliveryDir(): string;
|
|
24
30
|
/**
|
|
25
31
|
* The command that prints `text`, or undefined when it could not be prepared —
|
|
@@ -19,15 +19,21 @@
|
|
|
19
19
|
*/
|
|
20
20
|
import { randomUUID } from "node:crypto";
|
|
21
21
|
import { readdirSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
22
|
-
import { join } from "node:path";
|
|
22
|
+
import { join, resolve } from "node:path";
|
|
23
23
|
import { configDir, ensureDir } from "../control/paths.js";
|
|
24
24
|
/** Up to this many characters the text rides inside the command, as a heredoc. */
|
|
25
25
|
export const HEREDOC_MAX = 6_000;
|
|
26
26
|
/** A delivered file is read once, at once; an hour is a generous margin. */
|
|
27
27
|
const MAX_AGE_MS = 60 * 60_000;
|
|
28
|
-
/**
|
|
28
|
+
/**
|
|
29
|
+
* Where delivered texts wait to be read. Under `BIR_HOME`, mode 0700.
|
|
30
|
+
*
|
|
31
|
+
* Absolute whatever `BIR_HOME` says: the path is handed to a Bash that runs in
|
|
32
|
+
* the session's folder, not this process's, and a relative one was not found
|
|
33
|
+
* there.
|
|
34
|
+
*/
|
|
29
35
|
export function deliveryDir() {
|
|
30
|
-
return
|
|
36
|
+
return resolve(configDir(), "tmp");
|
|
31
37
|
}
|
|
32
38
|
/**
|
|
33
39
|
* The command that prints `text`, or undefined when it could not be prepared —
|
|
@@ -163,6 +163,20 @@ export declare function runBuiltin(toolName: string, input: Record<string, unkno
|
|
|
163
163
|
* even be the WSL stub (a different filesystem view entirely). Resolved once.
|
|
164
164
|
*/
|
|
165
165
|
export declare function bashExecutable(): string;
|
|
166
|
+
/**
|
|
167
|
+
* The one short line bash is started with: read the script's file and `eval`
|
|
168
|
+
* it — as the host evals the command it is given. Run that way `$0`, the line
|
|
169
|
+
* numbers and the shell's own error text stay what they were when the script
|
|
170
|
+
* was the `-c` argument (`/usr/bin/bash: line 3: jq: command not found`); run
|
|
171
|
+
* as `bash <file>` every one of them would carry this file's name instead. A
|
|
172
|
+
* file that cannot be read is exit 126: evaluated without the check it is an
|
|
173
|
+
* empty script, which succeeds. And `unread`, a marker unique to the call, goes
|
|
174
|
+
* to stderr first: a command's own 126 (a file that is not executable) is a
|
|
175
|
+
* command that ran, and only the marker tells the two apart ({@link runBash}).
|
|
176
|
+
*
|
|
177
|
+
* Exported for the tests.
|
|
178
|
+
*/
|
|
179
|
+
export declare function readsScript(file: string, unread: string): string;
|
|
166
180
|
/**
|
|
167
181
|
* A glob pattern as a RegExp over `/`-separated relative paths: `**` crosses
|
|
168
182
|
* directories, `*` and `?` stay inside one, `{a,b}` alternates, `[…]` passes
|