@botbuddy/cli 1.50.1 → 1.51.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/agent-session.mjs +7 -4
- package/src/stack-isolation.mjs +29 -4
- package/src/stack.mjs +165 -33
- package/src/wait-core.mjs +61 -8
- package/src/wait.mjs +72 -4
package/package.json
CHANGED
package/src/agent-session.mjs
CHANGED
|
@@ -261,7 +261,7 @@ export async function selfHealAgentSession({
|
|
|
261
261
|
|| (current.worktree && await worktreeRoot(current.worktree) !== registrationWorktree)
|
|
262
262
|
|| (current.agent_session_token === rejectedCredential.token
|
|
263
263
|
&& (current.session_id !== identity.sessionId || (current.session_token_generation ?? null) !== identity.generation))) {
|
|
264
|
-
throw missing("
|
|
264
|
+
throw missing("session_identity_mismatch", "The cached session no longer matches the rejected agent, tenant, worktree or session incarnation.");
|
|
265
265
|
}
|
|
266
266
|
if (current.agent_session_token !== rejectedCredential.token && isFresh(current, now())) {
|
|
267
267
|
return withIdentity({ token: current.agent_session_token, sessionId: sessionId ?? current.session_id, agentId: current.agent_id, source: "cache", ticketSignals: null, stackLeaseCapability: null }, current);
|
|
@@ -368,14 +368,17 @@ export async function selfHealAgentSession({
|
|
|
368
368
|
throw missing("session_mint_invalid", "register_agent did not return an agent_id and session_token");
|
|
369
369
|
}
|
|
370
370
|
if (renewal && (data.agent_id !== current.agent_id
|
|
371
|
-
|| data.tenant_id !== bound.tenant
|
|
371
|
+
|| (data.tenant_id !== undefined && data.tenant_id !== bound.tenant)
|
|
372
372
|
|| (data.tenant !== undefined && data.tenant !== bound.tenant)
|
|
373
|
-
|| (data.worktree !== undefined && data.worktree !== registrationWorktree)
|
|
373
|
+
|| (data.worktree !== undefined && data.worktree !== registrationWorktree))) {
|
|
374
|
+
throw missing("session_identity_mismatch", "register_agent returned a replacement for a different agent, tenant or worktree.");
|
|
375
|
+
}
|
|
376
|
+
if (renewal && (data.tenant_id !== bound.tenant
|
|
374
377
|
|| !AGENT_KEY_RE.test(data.session_token)
|
|
375
378
|
|| data.session_token === current.agent_session_token
|
|
376
379
|
|| typeof data.session_id !== "string" || !data.session_id
|
|
377
380
|
|| (data.session_ttl_seconds !== undefined && (!Number.isFinite(Number(data.session_ttl_seconds)) || Number(data.session_ttl_seconds) <= 0)))) {
|
|
378
|
-
throw missing("session_mint_invalid", "register_agent returned an invalid replacement session
|
|
381
|
+
throw missing("session_mint_invalid", "register_agent returned an invalid replacement session.");
|
|
379
382
|
}
|
|
380
383
|
const issued = now();
|
|
381
384
|
const ttlSeconds = Number.isFinite(Number(data.session_ttl_seconds)) ? Number(data.session_ttl_seconds) : SESSION_TTL_MS / 1000;
|
package/src/stack-isolation.mjs
CHANGED
|
@@ -945,6 +945,26 @@ export function releaseLeasedStackClaim(stackDir, { registryDir = defaultPortReg
|
|
|
945
945
|
}
|
|
946
946
|
}
|
|
947
947
|
|
|
948
|
+
/**
|
|
949
|
+
* BOT-2237 (Codex #1168 P2): whether THIS stack's own port-block claim is still on disk —
|
|
950
|
+
* the verified result a receipt reports after a release attempt (a failed unlink leaves it).
|
|
951
|
+
*/
|
|
952
|
+
export function leasedStackClaimHeld(stackDir, { registryDir = defaultPortRegistryDir(), io = {} } = {}) {
|
|
953
|
+
const read = io.readFile ?? readFileSync;
|
|
954
|
+
// `readStackMarker` deliberately normalizes failures to null for best-effort
|
|
955
|
+
// collection. A release receipt needs the stricter contract: only ENOENT proves
|
|
956
|
+
// this stack has no marker/claim; unreadable or corrupt marker data is unknown.
|
|
957
|
+
let marker = null;
|
|
958
|
+
try { marker = JSON.parse(read(join(stackDir, LEASED_STACK_MARKER), "utf8")); } catch (error) { return error?.code !== "ENOENT"; }
|
|
959
|
+
if (!marker || typeof marker !== "object" || !Number.isInteger(marker.port_base)) return true;
|
|
960
|
+
let claim = null;
|
|
961
|
+
// Codex #1168 R2 P2: only a PROVEN absence (ENOENT) counts as released; an unreadable or
|
|
962
|
+
// corrupt claim file is unverifiable, so it is reported as still held.
|
|
963
|
+
try { claim = JSON.parse(read(portClaimPath(registryDir, marker.port_base), "utf8")); } catch (error) { return error?.code !== "ENOENT"; }
|
|
964
|
+
const owner = marker.invocation_token ?? marker.project_id;
|
|
965
|
+
return Boolean(claim) && (claim.invocation_token ?? claim.project_id) === owner;
|
|
966
|
+
}
|
|
967
|
+
|
|
948
968
|
/**
|
|
949
969
|
* Reserve the blocks an EXPLICIT `--stack-path` occupies (BOT-1798, Codex #928
|
|
950
970
|
* R18). Its project lock only serializes runs that share a project id; two
|
|
@@ -1048,13 +1068,18 @@ export function reservePortBlocks(ports, { registryDir = defaultPortRegistryDir(
|
|
|
1048
1068
|
}
|
|
1049
1069
|
return renewed;
|
|
1050
1070
|
},
|
|
1071
|
+
/** Returns whether none of this invocation's claims remain (BOT-2237: a verified result). */
|
|
1051
1072
|
release: () => {
|
|
1073
|
+
let allReleased = true;
|
|
1052
1074
|
for (const block of claimed) {
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
} catch {
|
|
1075
|
+
let current = null;
|
|
1076
|
+
// Codex #1168 R2 P2: a claim that is gone (ENOENT) is released; one that cannot be read
|
|
1077
|
+
// or parsed is unverifiable — never unlinked blindly, and never reported as released.
|
|
1078
|
+
try { current = JSON.parse(read(portClaimPath(registryDir, block), "utf8")); } catch (error) { if (error?.code !== "ENOENT") allReleased = false; continue; }
|
|
1079
|
+
if (current?.invocation_token !== invocationToken) continue; // superseded: never unlink blindly
|
|
1080
|
+
try { remove(portClaimPath(registryDir, block), { force: true }); } catch { allReleased = false; }
|
|
1057
1081
|
}
|
|
1082
|
+
return allReleased;
|
|
1058
1083
|
},
|
|
1059
1084
|
};
|
|
1060
1085
|
}
|
package/src/stack.mjs
CHANGED
|
@@ -42,7 +42,7 @@ import { AGENT_KEY_RE, readAgentSessionTokenEnv } from "./agent-key.mjs";
|
|
|
42
42
|
import { resolveAgentSessionCredential, selfHealAgentSession } from "./agent-session.mjs";
|
|
43
43
|
import { runDockerCommand, runDockerWorkflow, ADMITTED_DOCKER_VALIDATIONS } from "./docker-hygiene.mjs";
|
|
44
44
|
import { acquireStackLock, lockPathForProject, projectIdFromConfig } from "./stack-file-lock.mjs";
|
|
45
|
-
import { LEASED_STACK_IDLE_RETENTION_SECS, assertLeasedStackIsolated, markLeasedStackReleased, stableRunStackReusable, leasedStackIdentity, materializeRunStackDir, portMapInConfig, refreshLeasedStackRetention, releaseLeasedStackClaim, reservePortBlocks } from "./stack-isolation.mjs";
|
|
45
|
+
import { LEASED_STACK_IDLE_RETENTION_SECS, assertLeasedStackIsolated, leasedStackClaimHeld, markLeasedStackReleased, stableRunStackReusable, leasedStackIdentity, materializeRunStackDir, portMapInConfig, refreshLeasedStackRetention, releaseLeasedStackClaim, reservePortBlocks } from "./stack-isolation.mjs";
|
|
46
46
|
import { measureStackUsage } from "./stack-usage.mjs";
|
|
47
47
|
import { machineUuid } from "./machine-id.mjs";
|
|
48
48
|
import { DEFAULT_BUNDLE_MAX_BYTES, buildStackBundle, uploadStackBundle, validateBundleIncludePath } from "./stack-bundle.mjs";
|
|
@@ -707,7 +707,7 @@ async function claimProjectLock(projectId, lock, subject) {
|
|
|
707
707
|
}
|
|
708
708
|
}
|
|
709
709
|
|
|
710
|
-
export async function resolveRunExecution(cwd, opts, { materialize = materializeRunStackDir, lock = acquireStackLock, identity = leasedStackIdentity, reserveBlocks = reservePortBlocks, releaseClaim = releaseLeasedStackClaim, stableReusable = stableRunStackReusable, markReleased = markLeasedStackReleased, portRegistryDir = null } = {}) {
|
|
710
|
+
export async function resolveRunExecution(cwd, opts, { materialize = materializeRunStackDir, lock = acquireStackLock, identity = leasedStackIdentity, reserveBlocks = reservePortBlocks, releaseClaim = releaseLeasedStackClaim, claimHeld = leasedStackClaimHeld, stableReusable = stableRunStackReusable, markReleased = markLeasedStackReleased, portRegistryDir = null } = {}) {
|
|
711
711
|
const refuseRoot = (execution) => {
|
|
712
712
|
if (execution.stackPath === ".") {
|
|
713
713
|
throw new Error(
|
|
@@ -763,9 +763,12 @@ export async function resolveRunExecution(cwd, opts, { materialize = materialize
|
|
|
763
763
|
// A prepared directory carries no retention marker, so its reservation is
|
|
764
764
|
// renewed here on the batch's own beat (Codex #928 R21 P2).
|
|
765
765
|
renew: (options) => ports.renew(options),
|
|
766
|
+
// BOT-2237: returns the VERIFIED disposition the receipt reports.
|
|
766
767
|
release: ({ reaped = false } = {}) => {
|
|
767
|
-
|
|
768
|
+
let released = false;
|
|
769
|
+
if (reaped) { try { released = ports.release() !== false; } catch { released = false; } }
|
|
768
770
|
held.release();
|
|
771
|
+
return { released };
|
|
769
772
|
},
|
|
770
773
|
};
|
|
771
774
|
}
|
|
@@ -794,13 +797,19 @@ export async function resolveRunExecution(cwd, opts, { materialize = materialize
|
|
|
794
797
|
// or running containers, so the reservation must stand until its retention
|
|
795
798
|
// expires. The project lock is always released — it guards this directory,
|
|
796
799
|
// which no other invocation can target.
|
|
800
|
+
// BOT-2237 (Codex #1168 P2): returns the VERIFIED disposition — whether this stack's
|
|
801
|
+
// claim is gone afterwards — so a failed unlink is never reported as released.
|
|
797
802
|
release: ({ reaped = false } = {}) => {
|
|
803
|
+
const registry = portRegistryDir ? { registryDir: portRegistryDir } : undefined;
|
|
804
|
+
let released = false;
|
|
798
805
|
if (reaped) {
|
|
799
|
-
try { releaseClaim(stackDirFor(execution),
|
|
806
|
+
try { releaseClaim(stackDirFor(execution), registry); } catch { /* verified below */ }
|
|
807
|
+
try { released = !claimHeld(stackDirFor(execution), registry); } catch { released = false; }
|
|
800
808
|
// BOT-1803: the stack is provably gone, so the next run may reuse this directory.
|
|
801
809
|
try { markReleased(stackDirFor(execution)); } catch { /* advisory */ }
|
|
802
810
|
}
|
|
803
811
|
held.release();
|
|
812
|
+
return { released };
|
|
804
813
|
},
|
|
805
814
|
};
|
|
806
815
|
} catch (error) {
|
|
@@ -947,16 +956,22 @@ export function retryingStackCall({
|
|
|
947
956
|
// (runStackLifecycle short-circuits its own await) skip init() — avoiding an extra
|
|
948
957
|
// await point before the first RPC is dispatched.
|
|
949
958
|
let auth = seedAuth;
|
|
950
|
-
|
|
959
|
+
// One renewal per rotation: disarmed by a renewal, re-armed only once a call succeeds again.
|
|
960
|
+
// BOT-2237: a long `stack run` can see its session rotated more than once (a sibling
|
|
961
|
+
// `bb doctor --fix`/`bb wait` self-heal); a renewed credential that WORKED and is later
|
|
962
|
+
// rejected is a new rotation, not the same failure — so it may renew again. A renewal
|
|
963
|
+
// whose credential is itself rejected stays disarmed (never a per-call mint loop).
|
|
964
|
+
let renewalArmed = true;
|
|
951
965
|
return {
|
|
952
966
|
async init() { if (auth === undefined) auth = await authProvider(); return auth; },
|
|
953
967
|
async call(name, args, callOptions = {}) {
|
|
954
968
|
let r = await callTool(name, args, { ...callOptions, auth });
|
|
955
|
-
if (
|
|
956
|
-
|
|
969
|
+
if (renewalArmed && !r?.ok && r?.auth) {
|
|
970
|
+
renewalArmed = false;
|
|
957
971
|
const fresh = await refresh(auth, { cwd: cwd() });
|
|
958
972
|
if (fresh) { auth = fresh; r = await callTool(name, args, { ...callOptions, auth }); }
|
|
959
973
|
}
|
|
974
|
+
if (r?.ok) renewalArmed = true;
|
|
960
975
|
return r;
|
|
961
976
|
},
|
|
962
977
|
get auth() { return auth; },
|
|
@@ -970,7 +985,7 @@ export function retryingStackCall({
|
|
|
970
985
|
// taxonomy so automation sees AUTH/BACKEND, not a CLI-defect INTERNAL. Credential
|
|
971
986
|
// problems are AUTH (3); a transport/server mint failure is BACKEND (5); anything
|
|
972
987
|
// unrecognized stays INTERNAL (7).
|
|
973
|
-
const AUTH_ERROR_CODES = new Set(["client_key_required", "client_key_unavailable", "tenant_binding_required", "session_token_malformed", "session_mint_unauthorized"]);
|
|
988
|
+
const AUTH_ERROR_CODES = new Set(["client_key_required", "client_key_unavailable", "tenant_binding_required", "session_token_malformed", "session_mint_unauthorized", "session_identity_mismatch"]);
|
|
974
989
|
// BOT-1750 (Codex R11 P2): `session_mint_invalid` is an incompatible/malformed SERVER
|
|
975
990
|
// response, not a credential problem — classify it with the transport/server failures.
|
|
976
991
|
const BACKEND_ERROR_CODES = new Set(["session_mint_failed", "session_mint_invalid"]);
|
|
@@ -1384,16 +1399,31 @@ export async function waitForLease(leaseId, isDone, isFailed, { timeoutSec, auth
|
|
|
1384
1399
|
}
|
|
1385
1400
|
|
|
1386
1401
|
/**
|
|
1387
|
-
* BOT-2099 — the platform-incident reasons a park can record. A park
|
|
1388
|
-
* auth or mint error as its outcome
|
|
1389
|
-
* deadline and, if it outlasts it,
|
|
1390
|
-
*
|
|
1402
|
+
* BOT-2099 — the platform-incident reasons a park can record. A park never ends with a raw
|
|
1403
|
+
* auth or mint error as its outcome. A transport outage is retried for the park's whole
|
|
1404
|
+
* remaining deadline and, if it outlasts it, ends as a `timeout` that carries this incident.
|
|
1405
|
+
* An auth failure is bounded tighter (BOT-2237, PARK_AUTH_BUDGET_MS / DEFINITIVE_REAUTH_CODES)
|
|
1406
|
+
* and ends typed as `platform_auth_unavailable`, naming the lease.
|
|
1391
1407
|
*/
|
|
1392
1408
|
export const PARK_INCIDENT_REASONS = Object.freeze({
|
|
1393
1409
|
AUTH: "platform_auth_unavailable", // 401/403 on the wait, or the session re-mint rejected
|
|
1394
1410
|
UNREACHABLE: "platform_unreachable", // transport failure / 5xx / malformed relay response
|
|
1395
1411
|
});
|
|
1396
1412
|
export const PARK_BACKOFF = Object.freeze({ initialMs: 2_000, maxMs: 60_000 });
|
|
1413
|
+
/**
|
|
1414
|
+
* BOT-2237: how long a park keeps retrying an UNRECOVERED auth failure before it ends with a
|
|
1415
|
+
* typed `platform_auth_unavailable`. A short server-side auth blip (BOT-2099 hop 7, about a
|
|
1416
|
+
* minute) still recovers inside it; a session that cannot be re-minted no longer sits out the
|
|
1417
|
+
* whole park deadline — the server's queued-lease liveness (BOT-2013) reaps an unwatched lease
|
|
1418
|
+
* well before an hour-long park ends, so waiting longer only reports a phantom capacity timeout.
|
|
1419
|
+
*/
|
|
1420
|
+
export const PARK_AUTH_BUDGET_MS = 5 * 60_000;
|
|
1421
|
+
// BOT-2237: re-mint refusals that no retry can fix — a locally proven worktree identity fence,
|
|
1422
|
+
// or durable credentials that are absent. A malformed mint response is a backend blip and gets
|
|
1423
|
+
// the regular auth budget; it is not evidence that the lease changed owner.
|
|
1424
|
+
const DEFINITIVE_REAUTH_CODES = new Set([
|
|
1425
|
+
"session_identity_mismatch", "client_key_required", "tenant_binding_required", "session_token_malformed", "agent_tenant_immutable",
|
|
1426
|
+
]);
|
|
1397
1427
|
|
|
1398
1428
|
function abortableSleep(ms, signal) {
|
|
1399
1429
|
return new Promise((resolve) => {
|
|
@@ -1433,10 +1463,11 @@ export async function reauthenticateParkedLease(current, kind, {
|
|
|
1433
1463
|
* head of the queue over a one-second 401 (hop 6) and ending every parked lease in a
|
|
1434
1464
|
* server-side auth outage with `session_mint_unauthorized` (hop 7). Here such a result is
|
|
1435
1465
|
* retried with bounded exponential backoff (PARK_BACKOFF) until the park's OWN deadline:
|
|
1436
|
-
* re-authenticate on an auth failure (`reauth`), re-register the wait
|
|
1437
|
-
*
|
|
1438
|
-
*
|
|
1439
|
-
* thrown by `reauth` are attempts,
|
|
1466
|
+
* re-authenticate on an auth failure (`reauth`), re-register the wait, and — after a
|
|
1467
|
+
* recovery — read the lease directly (`probe`) so a grant or reap that landed during the
|
|
1468
|
+
* outage is never missed (registration does NOT replay a lease's current state). Mint errors
|
|
1469
|
+
* thrown by `reauth` are attempts, except a definitive refusal or an exhausted auth budget
|
|
1470
|
+
* (BOT-2237), which end the park with `{ authUnavailable, code, error }`. The result is waitForLease's shape plus
|
|
1440
1471
|
* `credential` (the headers that finally worked) and, when any blip occurred, `incident`.
|
|
1441
1472
|
*/
|
|
1442
1473
|
export async function parkOnLease(leaseId, isDone, isFailed, {
|
|
@@ -1448,14 +1479,18 @@ export async function parkOnLease(leaseId, isDone, isFailed, {
|
|
|
1448
1479
|
// retryingStackCall) owns the credential; the park re-reads it before every attempt so a
|
|
1449
1480
|
// credential minted during a probe is never overwritten by an older one.
|
|
1450
1481
|
syncCredential = null,
|
|
1482
|
+
authBudgetMs = PARK_AUTH_BUDGET_MS,
|
|
1451
1483
|
} = {}) {
|
|
1452
1484
|
const deadline = deadlineMs ?? now() + timeoutSec * 1000;
|
|
1453
1485
|
let credential = auth;
|
|
1454
1486
|
let incident = null;
|
|
1455
1487
|
let delay = backoff.initialMs;
|
|
1488
|
+
// BOT-2237: when the current unrecovered run of AUTH failures began (null = none open).
|
|
1489
|
+
let authFailingSince = null;
|
|
1456
1490
|
const note = (kind, detail, code = null) => {
|
|
1457
1491
|
const at = new Date(now()).toISOString();
|
|
1458
1492
|
if (!incident) incident = { reason: null, first_at: at, attempts: 0, recovered: false };
|
|
1493
|
+
if (kind === "auth" && authFailingSince == null) authFailingSince = now();
|
|
1459
1494
|
incident.recovered = false; // a new failure re-opens a recovered incident
|
|
1460
1495
|
incident.attempts += 1;
|
|
1461
1496
|
incident.last_at = at;
|
|
@@ -1464,6 +1499,15 @@ export async function parkOnLease(leaseId, isDone, isFailed, {
|
|
|
1464
1499
|
incident.last_error_code = code;
|
|
1465
1500
|
};
|
|
1466
1501
|
const done = (result) => ({ ...result, credential, ...(incident ? { incident } : {}) });
|
|
1502
|
+
const recovered = () => { if (incident) incident.recovered = true; authFailingSince = null; delay = backoff.initialMs; };
|
|
1503
|
+
// BOT-2237: the park cannot re-establish a principal that may observe this lease. End it
|
|
1504
|
+
// typed, naming the lease, instead of parking blind until the deadline and then calling the
|
|
1505
|
+
// outage a capacity timeout.
|
|
1506
|
+
const authUnavailable = (code, detail) => done({
|
|
1507
|
+
authUnavailable: true, code: code ?? "unauthorized",
|
|
1508
|
+
error: `lease ${leaseId}: the parked wait lost its agent session and could not re-authenticate (${code ?? "unauthorized"}: ${String(detail).slice(0, 200)})`,
|
|
1509
|
+
});
|
|
1510
|
+
const authBudgetSpent = () => authFailingSince != null && now() - authFailingSince >= authBudgetMs;
|
|
1467
1511
|
for (;;) {
|
|
1468
1512
|
if (signal?.aborted) return done({ interrupted: true });
|
|
1469
1513
|
const remainingMs = deadline - now();
|
|
@@ -1483,13 +1527,14 @@ export async function parkOnLease(leaseId, isDone, isFailed, {
|
|
|
1483
1527
|
if (!result?.auth && !result?.error) {
|
|
1484
1528
|
// Codex #1057 R5: any attempt that did not fail on auth/transport (a match, a reap, or an
|
|
1485
1529
|
// ordinary timeout on a healthy stream) means the platform recovered.
|
|
1486
|
-
|
|
1530
|
+
recovered();
|
|
1487
1531
|
// A wait that ends on its own timer is a real timeout; never the outage's last word.
|
|
1488
1532
|
if (result?.timeout && incident && now() < deadline) continue;
|
|
1489
1533
|
return incident ? done(result) : result;
|
|
1490
1534
|
}
|
|
1491
1535
|
const kind = result.auth ? "auth" : "transport";
|
|
1492
1536
|
note(kind, result.auth ? `unauthorized (${result.phase ?? "stream"})` : result.error, result.auth ? "unauthorized" : null);
|
|
1537
|
+
if (kind === "auth" && authBudgetSpent()) return authUnavailable(incident.last_error_code, incident.last_error);
|
|
1493
1538
|
log(`${yellow("⚠")} stack: lease ${leaseId} wait interrupted by a platform ${kind === "auth" ? "auth" : "availability"} blip (${incident.last_error}); still parked — retrying in ${Math.round(delay / 1000)}s (attempt ${incident.attempts}).`);
|
|
1494
1539
|
// Back off (bounded) before re-authenticating, then retry; never past the deadline.
|
|
1495
1540
|
for (;;) {
|
|
@@ -1504,17 +1549,25 @@ export async function parkOnLease(leaseId, isDone, isFailed, {
|
|
|
1504
1549
|
break;
|
|
1505
1550
|
} catch (err) {
|
|
1506
1551
|
note("auth", err?.message ?? err, err?.code ?? null);
|
|
1552
|
+
if (DEFINITIVE_REAUTH_CODES.has(err?.code) || authBudgetSpent()) return authUnavailable(err?.code ?? null, err?.message ?? err);
|
|
1507
1553
|
log(`${yellow("⚠")} stack: lease ${leaseId} re-authentication failed (${err?.code ?? err?.message ?? err}); still parked — retrying in ${Math.round(delay / 1000)}s.`);
|
|
1508
1554
|
}
|
|
1509
1555
|
}
|
|
1510
1556
|
if (probe) {
|
|
1557
|
+
let read = null;
|
|
1511
1558
|
try {
|
|
1512
|
-
|
|
1559
|
+
read = await probe(credential);
|
|
1513
1560
|
if (syncCredential) credential = syncCredential() ?? credential;
|
|
1514
|
-
const state = read?.state ?? null;
|
|
1515
|
-
if (state != null && isFailed(state)) { incident.recovered = true; return done({ failed: true, state, ...(read?.reap_reason ? { reap_reason: read.reap_reason } : {}) }); }
|
|
1516
|
-
if (state != null && isDone(state)) { incident.recovered = true; return done({ woke: true, state }); }
|
|
1517
1561
|
} catch { /* the next wait attempt reports the failure */ }
|
|
1562
|
+
// BOT-2237 (lease f0607db4): a re-minted credential that is not the lease's owner can never
|
|
1563
|
+
// observe it — re-registering the wait would park blind until the deadline.
|
|
1564
|
+
if (read?.ownerLost) return authUnavailable(read.code ?? "NOT_LEASE_OWNER", read.message ?? "the re-authenticated agent does not own this lease");
|
|
1565
|
+
const state = read?.state ?? null;
|
|
1566
|
+
// Codex #1168 P1: an authenticated read of the lease proves the outage is over, so a later,
|
|
1567
|
+
// independent rotation starts a fresh auth budget instead of inheriting this one.
|
|
1568
|
+
if (state != null) recovered();
|
|
1569
|
+
if (state != null && isFailed(state)) return done({ failed: true, state, ...(read?.reap_reason ? { reap_reason: read.reap_reason } : {}) });
|
|
1570
|
+
if (state != null && isDone(state)) return done({ woke: true, state });
|
|
1518
1571
|
}
|
|
1519
1572
|
}
|
|
1520
1573
|
}
|
|
@@ -2215,7 +2268,16 @@ export async function cmdUp(opts, {
|
|
|
2215
2268
|
const resilientWait = async (isDone, isFailed) => {
|
|
2216
2269
|
const result = await parkOnLease(leaseId, isDone, isFailed, {
|
|
2217
2270
|
timeoutSec: opts.timeout, auth, wait: waitFn, reauth, ...(sleep ? { sleep } : {}), ...(now ? { now } : {}),
|
|
2218
|
-
|
|
2271
|
+
// Codex #1168 R2 P1: probe with the credential the park just re-minted — `auth`/`call`
|
|
2272
|
+
// are only updated after the park returns, and an old-credential read would 401 and
|
|
2273
|
+
// miss a grant or reap that landed during the outage.
|
|
2274
|
+
probe: async (credential) => {
|
|
2275
|
+
const read = await callTool("get_stack_lease", { lease_id: leaseId }, { auth: credential ?? auth });
|
|
2276
|
+
if (read?.ok && read.data?.success) return read.data;
|
|
2277
|
+
// BOT-2237: a re-authenticated principal that does not own the lease can never observe it.
|
|
2278
|
+
if (read?.ok && read.data?.code === "NOT_LEASE_OWNER") return { ownerLost: true, code: "NOT_LEASE_OWNER", message: read.data.message ?? null };
|
|
2279
|
+
return null;
|
|
2280
|
+
},
|
|
2219
2281
|
});
|
|
2220
2282
|
if (result.credential && result.credential !== auth) {
|
|
2221
2283
|
auth = result.credential;
|
|
@@ -2227,6 +2289,11 @@ export async function cmdUp(opts, {
|
|
|
2227
2289
|
// Codex #1057 R2: a blip that occurred during any wait stays on EVERY terminal receipt
|
|
2228
2290
|
// (recovered or not), so recoveries are not lost from the incident record.
|
|
2229
2291
|
let upIncident = null;
|
|
2292
|
+
// BOT-2237: a park that cannot re-authenticate ends typed, naming the lease.
|
|
2293
|
+
const emitAuthUnavailable = (r) => emitResult(buildReceipt({
|
|
2294
|
+
command: "up", outcome: "error", code: PARK_INCIDENT_REASONS.AUTH, lease_id: leaseId, slot,
|
|
2295
|
+
error: `${r.error}; its server state is unknown to this command — inspect it with \`bb stack status ${leaseId}\` after \`bb doctor --fix\``,
|
|
2296
|
+
}), opts, EXIT.AUTH);
|
|
2230
2297
|
const emitPlain = emitResult;
|
|
2231
2298
|
emitResult = (receipt, o, code) => emitPlain(upIncident && !receipt.platform_incident ? { ...receipt, platform_incident: upIncident } : receipt, o, code);
|
|
2232
2299
|
// BOT-1711 (Codex R13): request_stack_lease returned an EXISTING active lease (reused),
|
|
@@ -2247,6 +2314,7 @@ export async function cmdUp(opts, {
|
|
|
2247
2314
|
// Park zero-poll until the lease leaves the queue (granted in BOT-1187 order).
|
|
2248
2315
|
// BOT-2013: a lease reaped while queued (orphaned_agent, …) is lost, not "left the queue".
|
|
2249
2316
|
const parked = await resilientWait(nonQueued, isReaped);
|
|
2317
|
+
if (parked.authUnavailable) return emitAuthUnavailable(parked);
|
|
2250
2318
|
if (parked.failed) {
|
|
2251
2319
|
return emitResult(buildReceipt({
|
|
2252
2320
|
command: "up", outcome: "error", lease_id: leaseId, state: parked.state, slot,
|
|
@@ -2266,7 +2334,12 @@ export async function cmdUp(opts, {
|
|
|
2266
2334
|
} else if (queueHold) {
|
|
2267
2335
|
process.stderr.write(`${yellow("⚠")} stack: lease ${leaseId} could not keep its queue place (${queueHold.code ?? "error"}: ${queueHold.error})\n`);
|
|
2268
2336
|
}
|
|
2269
|
-
|
|
2337
|
+
// BOT-2237: a park that ended inside an unrecovered platform incident never observed admission.
|
|
2338
|
+
const blind = parked.incident && !parked.incident.recovered;
|
|
2339
|
+
const why = blind
|
|
2340
|
+
? `parked ${opts.timeout}s inside a platform incident (${parked.incident.reason} since ${parked.incident.first_at}), so its admission was never observed`
|
|
2341
|
+
: `parked ${opts.timeout}s without capacity`;
|
|
2342
|
+
return emitResult(buildReceipt({ command: "up", outcome: "timeout", ...(blind ? { code: parked.incident.reason } : {}), lease_id: leaseId, state: "queued", slot, error: why, ...(ledger ? { unit_ledger: ledger } : {}), ...(queueHold ? { queue_hold: queueHold } : {}) }), opts, EXIT.TIMEOUT);
|
|
2270
2343
|
}
|
|
2271
2344
|
if (parked.auth) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, error: "unauthorized on wait stream" }), opts, EXIT.AUTH);
|
|
2272
2345
|
if (parked.error) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, error: parked.error }), opts, EXIT.BACKEND);
|
|
@@ -2367,6 +2440,7 @@ export async function cmdUp(opts, {
|
|
|
2367
2440
|
if (helperClaimed || helperAlreadyActive) {
|
|
2368
2441
|
if (!helperAlreadyActive) {
|
|
2369
2442
|
const active = await resilientWait(isActive, isReaped);
|
|
2443
|
+
if (active.authUnavailable) return emitAuthUnavailable(active);
|
|
2370
2444
|
if (active.timeout) return emitResult(buildReceipt({ command: "up", outcome: "timeout", lease_id: leaseId, state, slot, error: `a Helper claimed the provision job but it did not reach active in ${opts.timeout}s`, ...(active.incident ? { platform_incident: active.incident } : {}) }), opts, EXIT.TIMEOUT);
|
|
2371
2445
|
if (active.auth) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, error: "unauthorized on wait stream" }), opts, EXIT.AUTH);
|
|
2372
2446
|
if (active.failed) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, state: active.state, error: "lease was reaped before it became active" }), opts, EXIT.LEASE_FAILED);
|
|
@@ -2421,6 +2495,7 @@ export async function cmdUp(opts, {
|
|
|
2421
2495
|
}
|
|
2422
2496
|
} else {
|
|
2423
2497
|
const active = await resilientWait(isActive, isReaped);
|
|
2498
|
+
if (active.authUnavailable) return emitAuthUnavailable(active);
|
|
2424
2499
|
if (active.timeout) return emitResult(buildReceipt({ command: "up", outcome: "timeout", lease_id: leaseId, state, slot, error: `provisioning did not reach active in ${opts.timeout}s (Helper may be down — retry with --local-exec)`, ...(active.incident ? { platform_incident: active.incident } : {}) }), opts, EXIT.TIMEOUT);
|
|
2425
2500
|
if (active.auth) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, error: "unauthorized on wait stream" }), opts, EXIT.AUTH);
|
|
2426
2501
|
if (active.failed) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, state: active.state, error: "lease was reaped before it became active" }), opts, EXIT.LEASE_FAILED);
|
|
@@ -2926,12 +3001,20 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
|
|
|
2926
3001
|
// BOT-2099 (Codex #1057 R4): a platform blip recorded by ANY lease wait stays on EVERY
|
|
2927
3002
|
// terminal result (interrupted, unreadable, thrown, …), not only the paths that remember it.
|
|
2928
3003
|
// BOT-2142: where the lease ran (and how it was reached) is on every terminal result too.
|
|
3004
|
+
// A receipt always says what happened to a port-block claim. Before allocation
|
|
3005
|
+
// no claim exists, which is a safe, explicit disposition rather than omission.
|
|
2929
3006
|
// BOT-1803: whether this run re-adopted / re-provisioned an idle lease, and its stack digest.
|
|
2930
|
-
const incidentBox = { incident: null, execution: null, idleLease: null };
|
|
2931
|
-
|
|
3007
|
+
const incidentBox = { incident: null, execution: null, idleLease: null, portBlock: { released: false, reason: "no port-block claim was created before stack allocation" } };
|
|
3008
|
+
let result;
|
|
3009
|
+
try {
|
|
3010
|
+
result = await runStackLifecycleOnce(opts, childArgv, adapters, incidentBox);
|
|
3011
|
+
} catch (error) {
|
|
3012
|
+
result = { exitCode: classifyStackAuthError(error), outcome: "error", error: String(error?.message ?? error) };
|
|
3013
|
+
}
|
|
2932
3014
|
const withIncident = incidentBox.incident && !result?.incident ? { ...result, incident: incidentBox.incident } : result;
|
|
2933
3015
|
const withExecution = incidentBox.execution && !withIncident?.execution ? { ...withIncident, execution: incidentBox.execution } : withIncident;
|
|
2934
|
-
|
|
3016
|
+
const withIdle = incidentBox.idleLease && !withExecution?.idleLease ? { ...withExecution, idleLease: incidentBox.idleLease } : withExecution;
|
|
3017
|
+
return incidentBox.portBlock && !withIdle?.portBlock ? { ...withIdle, portBlock: incidentBox.portBlock } : withIdle;
|
|
2935
3018
|
}
|
|
2936
3019
|
|
|
2937
3020
|
async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
@@ -2947,7 +3030,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
2947
3030
|
// Resolve the initial credential with the same short-circuit as before (an injected
|
|
2948
3031
|
// auth skips the network resolve), then seed the retrying caller so no extra await
|
|
2949
3032
|
// point is introduced before the first RPC.
|
|
2950
|
-
const initialAuth = adapters.auth ?? await stackAuthHeader();
|
|
3033
|
+
const initialAuth = adapters.auth ?? await (adapters.authProvider ?? stackAuthHeader)();
|
|
2951
3034
|
const rpc = retryingStackCall({
|
|
2952
3035
|
callTool: adapters.callTool ?? callToolJson,
|
|
2953
3036
|
refresh: refreshAuth,
|
|
@@ -2971,6 +3054,13 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
2971
3054
|
let parkIncident = null;
|
|
2972
3055
|
// BOT-2099: a best-effort diagnostic read (ledger, persisted Helper reason) never throws.
|
|
2973
3056
|
const readLeaseQuietly = async (id) => { try { return await api.get?.(id) ?? null; } catch { return null; } };
|
|
3057
|
+
// BOT-2237: the typed end of a park that cannot re-authenticate — names the lease, never a
|
|
3058
|
+
// capacity timeout. Exit AUTH: the caller's credential, not the lease, is what failed.
|
|
3059
|
+
const authUnavailableResult = (waited, phase, cleanup) => ({
|
|
3060
|
+
exitCode: EXIT.AUTH, outcome: "error", code: PARK_INCIDENT_REASONS.AUTH, leaseId: leaseId ?? null, cleanup,
|
|
3061
|
+
error: `${waited.error} while the lease was ${phase}; its server state is unknown to this run — inspect it with \`bb stack status ${leaseId}\` after \`bb doctor --fix\``,
|
|
3062
|
+
...(parkIncident ? { incident: parkIncident } : {}),
|
|
3063
|
+
});
|
|
2974
3064
|
const resilientWait = async (leaseIdArg, done, failed, options) => {
|
|
2975
3065
|
const result = await parkOnLease(leaseIdArg, done, failed, {
|
|
2976
3066
|
...options, wait,
|
|
@@ -2979,7 +3069,13 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
2979
3069
|
// credential the RPC layer itself refreshed during the probe is never overwritten.
|
|
2980
3070
|
reauth: async (_current, kind) => { rpc.adopt(await reauth(rpc.auth, kind)); return rpc.auth; },
|
|
2981
3071
|
syncCredential: () => rpc.auth,
|
|
2982
|
-
probe: async () => {
|
|
3072
|
+
probe: async () => {
|
|
3073
|
+
const read = await api.get(leaseIdArg);
|
|
3074
|
+
if (read?.ok && read.data?.success) return read.data;
|
|
3075
|
+
// BOT-2237: the server answered and says this principal does not own the lease.
|
|
3076
|
+
if (read?.ok && read.data?.code === "NOT_LEASE_OWNER") return { ownerLost: true, code: "NOT_LEASE_OWNER", message: read.data.message ?? null };
|
|
3077
|
+
return null;
|
|
3078
|
+
},
|
|
2983
3079
|
...(adapters.sleep ? { sleep: adapters.sleep } : {}),
|
|
2984
3080
|
...(adapters.now ? { now: adapters.now } : {}),
|
|
2985
3081
|
log: stderrLine,
|
|
@@ -3118,7 +3214,12 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3118
3214
|
const postCheck = durableReapFailure(recheck);
|
|
3119
3215
|
if (postCheck) return cleanupResult = postCheck;
|
|
3120
3216
|
return cleanupResult = { ok: false, error: reaped?.timeout ? `signed reap did not arrive within ${opts.reapTimeout}s` : (reaped?.error || "signed reap was not proven") };
|
|
3121
|
-
})()
|
|
3217
|
+
})().catch((error) => {
|
|
3218
|
+
// BOT-2237: a release/read whose session renewal THROWS (e.g. `session_mint_invalid`
|
|
3219
|
+
// once the cache names another agent) is a failed cleanup of THIS lease — never an
|
|
3220
|
+
// exception that escapes the lifecycle and loses the lease id from the receipt.
|
|
3221
|
+
return cleanupResult = { ok: false, error: `cleanup could not run: ${error?.code ? `${error.code}: ` : ""}${error?.message ?? error}` };
|
|
3222
|
+
});
|
|
3122
3223
|
return cleanupPromise;
|
|
3123
3224
|
};
|
|
3124
3225
|
|
|
@@ -3458,14 +3559,25 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3458
3559
|
activeWaitAbort = null;
|
|
3459
3560
|
const incident = parkIncident ? { incident: parkIncident } : {};
|
|
3460
3561
|
if (parked?.interrupted || receivedSignal) { const cleanup = await requestCleanup(); return { exitCode: SIGNAL_EXIT[receivedSignal], outcome: "interrupted", leaseId, cleanup, ...incident }; }
|
|
3562
|
+
if (parked?.authUnavailable) {
|
|
3563
|
+
const cleanup = await requestCleanup();
|
|
3564
|
+
return authUnavailableResult(parked, "queued", cleanup);
|
|
3565
|
+
}
|
|
3461
3566
|
if (parked?.timeout) {
|
|
3462
3567
|
// BOT-2099: say who held the units for the whole park, read before the release.
|
|
3463
3568
|
const final = await readLeaseQuietly(leaseId);
|
|
3464
3569
|
const ledger = final?.ok && final.data?.success ? final.data.ledger ?? null : null;
|
|
3465
3570
|
for (const line of describeUnitLedger(ledger)) stderrLine(`stack: ${line}`);
|
|
3466
3571
|
const cleanup = await requestCleanup();
|
|
3467
|
-
|
|
3468
|
-
|
|
3572
|
+
// BOT-2237: a park that ended inside an unrecovered platform incident never observed the
|
|
3573
|
+
// lease's admission, so it must not claim the host had no capacity.
|
|
3574
|
+
if (parkIncident && !parkIncident.recovered) {
|
|
3575
|
+
return {
|
|
3576
|
+
exitCode: EXIT.TIMEOUT, outcome: "timeout", code: parkIncident.reason, leaseId, cleanup, ...incident, ...(ledger ? { ledger } : {}),
|
|
3577
|
+
error: `lease ${leaseId} parked ${opts.timeout}s inside a platform incident (${parkIncident.reason} since ${parkIncident.first_at}), so its admission was never observed`,
|
|
3578
|
+
};
|
|
3579
|
+
}
|
|
3580
|
+
return { exitCode: EXIT.TIMEOUT, outcome: "timeout", leaseId, error: `parked ${opts.timeout}s without capacity`, cleanup, ...incident, ...(ledger ? { ledger } : {}) };
|
|
3469
3581
|
}
|
|
3470
3582
|
if (parked?.failed) { const cleanup = await requestCleanup(); return { exitCode: EXIT.LEASE_FAILED, outcome: "error", leaseId, error: `lease was ${parked.state} while queued (reap_reason=${parked.reap_reason ?? "unknown"})`, cleanup, ...incident }; }
|
|
3471
3583
|
if (parked?.error || parked?.auth) { const cleanup = await requestCleanup(); return { exitCode: parked?.auth ? EXIT.AUTH : EXIT.LEASE_FAILED, outcome: "error", leaseId, error: parked?.error || "lease did not leave queue", cleanup, ...incident }; }
|
|
@@ -3483,6 +3595,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3483
3595
|
const active = await resilientWait(leaseId, (s) => s === "active", isReaped, { timeoutSec: provisionTimeout, auth: rpc.auth, signal: activeWaitAbort.signal });
|
|
3484
3596
|
activeWaitAbort = null;
|
|
3485
3597
|
if (active?.interrupted || receivedSignal) { const cleanup = await requestCleanup(); return { exitCode: SIGNAL_EXIT[receivedSignal], outcome: "interrupted", leaseId, cleanup }; }
|
|
3598
|
+
if (active?.authUnavailable) { const cleanup = await requestCleanup(); return authUnavailableResult(active, "provisioning", cleanup); }
|
|
3486
3599
|
if (active?.timeout) {
|
|
3487
3600
|
// BOT-1822 belt-and-braces: the zero-poll wait can still miss a live
|
|
3488
3601
|
// `stack_lease` signal (an older server before this fix, a relay hiccup,
|
|
@@ -3602,7 +3715,10 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3602
3715
|
let usage = null;
|
|
3603
3716
|
if (!servedRemote) { try { usage = await measureUsage(usageLease); } catch { usage = null; } }
|
|
3604
3717
|
if (fencing || childResult) return;
|
|
3605
|
-
|
|
3718
|
+
// BOT-2237: a touch whose renewal throws is a failed heartbeat (fence), never an
|
|
3719
|
+
// unhandled rejection from a timer callback.
|
|
3720
|
+
let touched;
|
|
3721
|
+
try { touched = usage ? await api.touch(leaseId, usage) : await api.touch(leaseId); } catch (error) { touched = { ok: false, error: String(error?.message ?? error) }; }
|
|
3606
3722
|
if (touched?.ok && isPreemptedAnswer(touched.data)) {
|
|
3607
3723
|
// BOT-1803 (AC-11): the control plane reclaimed this lease's capacity. Stop the
|
|
3608
3724
|
// child and end typed — never the generic lost-lease wording.
|
|
@@ -3678,7 +3794,21 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3678
3794
|
// runs, so its block stays reserved; so does one this run must not free (retainBlock).
|
|
3679
3795
|
const provenReap = !retainBlock && (!leaseProvisioned
|
|
3680
3796
|
|| (cleanupResult?.ok === true && (cleanupResult.state === "reaped" || cleanupResult.state == null)));
|
|
3681
|
-
execution?.release
|
|
3797
|
+
if (execution?.release) {
|
|
3798
|
+
// BOT-2237 (AC-3): every terminal path says what happened to the host port-block claim,
|
|
3799
|
+
// from the release's VERIFIED result (Codex #1168 P2), never from the reap alone.
|
|
3800
|
+
let disposition = null;
|
|
3801
|
+
let releaseError = null;
|
|
3802
|
+
try { disposition = execution.release({ reaped: provenReap }); } catch (error) { releaseError = error; }
|
|
3803
|
+
const retained = "; the claim is kept until its retention expires";
|
|
3804
|
+
incidentBox.portBlock = !provenReap
|
|
3805
|
+
? { released: false, reason: retainBlock
|
|
3806
|
+
? `a lease on this stack slot may still hold its ports (idle, unreaped, or another live batch)${retained} so its stack is never double-booked`
|
|
3807
|
+
: `reap not proven (${cleanupResult?.error ?? "no cleanup ran"})${retained} so a stack the Helper may still run is never double-booked` }
|
|
3808
|
+
: releaseError || disposition?.released === false
|
|
3809
|
+
? { released: false, reason: `the reap was proven but the claim could not be removed (${releaseError?.message ?? "it is still on disk"})${retained}` }
|
|
3810
|
+
: { released: true };
|
|
3811
|
+
}
|
|
3682
3812
|
for (const [signal, handler] of signalHandlers) signals.off?.(signal, handler);
|
|
3683
3813
|
}
|
|
3684
3814
|
}
|
|
@@ -3718,6 +3848,8 @@ export function runReceiptFields(result) {
|
|
|
3718
3848
|
// incident (reason, window, attempts, recovered) — never as the lease's outcome.
|
|
3719
3849
|
...(result.incident ? { platform_incident: result.incident } : {}),
|
|
3720
3850
|
...(result.ledger ? { unit_ledger: result.ledger } : {}),
|
|
3851
|
+
// BOT-2237: whether this run handed its host port block back, or kept it and why.
|
|
3852
|
+
...(result.portBlock ? { port_block: result.portBlock } : {}),
|
|
3721
3853
|
};
|
|
3722
3854
|
}
|
|
3723
3855
|
|
package/src/wait-core.mjs
CHANGED
|
@@ -1231,6 +1231,20 @@ const nowIso = () => new Date().toISOString();
|
|
|
1231
1231
|
// deadline race, so an unbounded probe would blow past a short --timeout.
|
|
1232
1232
|
const FEED_PROBE_BUDGET_MS = 2000;
|
|
1233
1233
|
|
|
1234
|
+
// BOT-2407: transport failures that cut an established event stream. undici reports a
|
|
1235
|
+
// socket closed mid-body as `TypeError: terminated` with the socket error as its cause
|
|
1236
|
+
// (ENT-5478 prod, 2026-10-15: three exit-7 deaths in 2 h). These are a stream END — the
|
|
1237
|
+
// relay is reconnected from the safe cursor — never an internal error.
|
|
1238
|
+
const STREAM_TRANSPORT_CODES = new Set(["UND_ERR_SOCKET", "UND_ERR_BODY_TIMEOUT", "ECONNRESET", "EPIPE", "ETIMEDOUT"]);
|
|
1239
|
+
export function isStreamTransportError(err) {
|
|
1240
|
+
if (!err || typeof err !== "object") return false;
|
|
1241
|
+
if (err.name === "AbortError") return true;
|
|
1242
|
+
for (let e = err, depth = 0; e && typeof e === "object" && depth < 4; e = e.cause, depth += 1) {
|
|
1243
|
+
if (STREAM_TRANSPORT_CODES.has(e.code)) return true;
|
|
1244
|
+
}
|
|
1245
|
+
return err instanceof TypeError && err.message === "terminated";
|
|
1246
|
+
}
|
|
1247
|
+
|
|
1234
1248
|
/**
|
|
1235
1249
|
* Run one wait to a terminal outcome, driven by an injected `connect(since)`
|
|
1236
1250
|
* that returns an async iterable of parsed SSE frames.
|
|
@@ -1241,7 +1255,9 @@ const FEED_PROBE_BUDGET_MS = 2000;
|
|
|
1241
1255
|
* - reconnects when the stream ends, resuming from the last seen seq, and
|
|
1242
1256
|
* de-duplicates a repeated seq across the reconnect (exactly-once)
|
|
1243
1257
|
* - maps server error frames to AUTH / CURSOR_EXPIRED
|
|
1244
|
-
* - gives up with BACKEND after
|
|
1258
|
+
* - gives up with BACKEND after `maxReconnects` CONSECUTIVE failed connects
|
|
1259
|
+
* (BOT-2351: any delivered frame resets the count; healthy rotations never
|
|
1260
|
+
* exhaust it, so only the wall deadline bounds a healthy wait)
|
|
1245
1261
|
*
|
|
1246
1262
|
* @returns {Promise<{receipt: object, exitCode: number}>}
|
|
1247
1263
|
*/
|
|
@@ -1340,8 +1356,15 @@ export async function runWaitLoop({
|
|
|
1340
1356
|
const pendingReplayBeaconFrames = [];
|
|
1341
1357
|
let pendingReplaySupersession = null;
|
|
1342
1358
|
const degraded = [];
|
|
1359
|
+
// BOT-2351: `reconnects` counts every stream rotation for the receipt (an operator
|
|
1360
|
+
// still sees churn); `failures` counts CONSECUTIVE failed connects — a connect that
|
|
1361
|
+
// threw, or a stream that ended before delivering a single frame — and is reset by any
|
|
1362
|
+
// delivered frame. Only `failures` is bounded by maxReconnects, so a healthy wait whose
|
|
1363
|
+
// relay recycles its stream every few minutes is ended only by its match or its deadline.
|
|
1343
1364
|
let reconnects = 0;
|
|
1344
1365
|
let failures = 0;
|
|
1366
|
+
const noteBackendUnreachable = () => { if (!degraded.includes("backend_unreachable")) degraded.push("backend_unreachable"); };
|
|
1367
|
+
const noteDegraded = (marker) => { if (!degraded.includes(marker)) degraded.push(marker); };
|
|
1345
1368
|
|
|
1346
1369
|
// Internal one-shot alarms (the deadline, the pre-deadline feed probe, reconnect backoff)
|
|
1347
1370
|
// are REAL timers via the default sleep. Each exists only to resolve a race — once the loop
|
|
@@ -1649,9 +1672,9 @@ export async function runWaitLoop({
|
|
|
1649
1672
|
stream = await connect(connectCursor);
|
|
1650
1673
|
} catch (err) {
|
|
1651
1674
|
failures += 1;
|
|
1652
|
-
|
|
1675
|
+
noteBackendUnreachable();
|
|
1653
1676
|
if (failures > maxReconnects) {
|
|
1654
|
-
return finalize("error", { exitCode: EXIT.BACKEND, error: String(err && err.message || err) });
|
|
1677
|
+
return finalize("error", { exitCode: EXIT.BACKEND, error: `reconnect budget exhausted: ${String(err && err.message || err)}` });
|
|
1655
1678
|
}
|
|
1656
1679
|
const waitMs = backoff(failures);
|
|
1657
1680
|
const raced = await Promise.race([
|
|
@@ -1664,6 +1687,8 @@ export async function runWaitLoop({
|
|
|
1664
1687
|
|
|
1665
1688
|
const iterator = stream[Symbol.asyncIterator]();
|
|
1666
1689
|
let streamEnded = false;
|
|
1690
|
+
// BOT-2351: whether this connection delivered anything at all (see `failures`).
|
|
1691
|
+
let deliveredFrame = false;
|
|
1667
1692
|
// BOT-1148 (Codex P2): the pending iterator.next() is HOISTED across loop
|
|
1668
1693
|
// iterations. When a non-frame racer wins (a beacon-stale confirm that a recovery
|
|
1669
1694
|
// then cancels), we must NOT abandon the in-flight read and start a fresh one —
|
|
@@ -1672,13 +1697,29 @@ export async function runWaitLoop({
|
|
|
1672
1697
|
// previous one has actually settled into a frame we consumed.
|
|
1673
1698
|
let pendingNext = null;
|
|
1674
1699
|
while (!streamEnded) {
|
|
1675
|
-
|
|
1700
|
+
// BOT-2407: a rejected read settles into a marker instead of escaping the loop. A
|
|
1701
|
+
// transport cut ends the stream (reconnect); anything else is a typed error receipt.
|
|
1702
|
+
if (!pendingNext) {
|
|
1703
|
+
pendingNext = iterator.next().catch((err) => (
|
|
1704
|
+
isStreamTransportError(err) ? { __streamDropped: err } : { __streamError: err }
|
|
1705
|
+
));
|
|
1706
|
+
}
|
|
1676
1707
|
const racers = [deadlinePromise, pendingNext];
|
|
1677
1708
|
if (staleConfirmPromise) racers.push(staleConfirmPromise);
|
|
1678
1709
|
if (preDeadlineProbePromise && !preDeadlineProbeDone) racers.push(preDeadlineProbePromise);
|
|
1679
1710
|
const raced = await Promise.race(racers);
|
|
1680
1711
|
|
|
1681
1712
|
if (raced && raced.__deadline) return fireReplayGraceExpired();
|
|
1713
|
+
if (raced && raced.__streamDropped) {
|
|
1714
|
+
pendingNext = null;
|
|
1715
|
+
noteDegraded("stream_dropped");
|
|
1716
|
+
streamEnded = true;
|
|
1717
|
+
break;
|
|
1718
|
+
}
|
|
1719
|
+
if (raced && raced.__streamError) {
|
|
1720
|
+
const err = raced.__streamError;
|
|
1721
|
+
return finalize("error", { exitCode: EXIT.INTERNAL, error: `stream_error: ${err instanceof Error ? `${err.name}: ${err.message}` : String(err)}` });
|
|
1722
|
+
}
|
|
1682
1723
|
// BOT-1147 (Codex round-27 P2): the pre-deadline feed probe fired. Fold a
|
|
1683
1724
|
// mid-wait stall/recovery into `degraded` before the imminent timeout receipt,
|
|
1684
1725
|
// then keep waiting — pendingNext stays in flight (hoisted), re-raced next
|
|
@@ -1714,6 +1755,9 @@ export async function runWaitLoop({
|
|
|
1714
1755
|
break;
|
|
1715
1756
|
}
|
|
1716
1757
|
if (!frame) continue;
|
|
1758
|
+
// BOT-2351: the relay answered on this connection, so the backend is reachable —
|
|
1759
|
+
// the consecutive-failure budget starts over.
|
|
1760
|
+
if (!deliveredFrame) { deliveredFrame = true; failures = 0; }
|
|
1717
1761
|
|
|
1718
1762
|
if (frame.event === "connected") {
|
|
1719
1763
|
let connected = {};
|
|
@@ -1968,12 +2012,21 @@ export async function runWaitLoop({
|
|
|
1968
2012
|
await checkpointNonterminalFrame();
|
|
1969
2013
|
}
|
|
1970
2014
|
|
|
1971
|
-
// Stream ended without a match: reconnect from the cursor.
|
|
2015
|
+
// Stream ended without a match: reconnect from the cursor. A stream that delivered
|
|
2016
|
+
// frames was a healthy rotation (the relay recycles streams every few minutes); one
|
|
2017
|
+
// that ended before any frame is a failed connect and counts toward the budget.
|
|
1972
2018
|
reconnects += 1;
|
|
1973
|
-
if (
|
|
1974
|
-
|
|
2019
|
+
if (!deliveredFrame) {
|
|
2020
|
+
failures += 1;
|
|
2021
|
+
if (failures > maxReconnects) {
|
|
2022
|
+
// BOT-2407 AC-4: only consecutive failures spend the budget, so a give-up is always
|
|
2023
|
+
// an outage — name it. A connection that ended before any frame is
|
|
2024
|
+
// `stream_ended_empty`; `backend_unreachable` / `stream_dropped` are already noted.
|
|
2025
|
+
noteDegraded("stream_ended_empty");
|
|
2026
|
+
return finalize("error", { exitCode: EXIT.BACKEND, error: "reconnect budget exhausted" });
|
|
2027
|
+
}
|
|
1975
2028
|
}
|
|
1976
|
-
const waitMs = backoff(
|
|
2029
|
+
const waitMs = backoff(Math.max(1, failures));
|
|
1977
2030
|
const raced = await Promise.race([deadlinePromise, sleepFn(waitMs).then(() => ({ __retry: true }))]);
|
|
1978
2031
|
if (raced && raced.__deadline) return fireReplayGraceExpired();
|
|
1979
2032
|
}
|
package/src/wait.mjs
CHANGED
|
@@ -2192,12 +2192,80 @@ export async function runWait(argv, { recoveryLocalWaitId = null } = {}) {
|
|
|
2192
2192
|
}
|
|
2193
2193
|
process.exit(exitCode);
|
|
2194
2194
|
} catch (err) {
|
|
2195
|
-
|
|
2196
|
-
|
|
2195
|
+
// A checkpoint-store failure is its own typed outcome (wait_checkpoint_write_failed):
|
|
2196
|
+
// the store is what failed, so it is not settled as `internal` (Codex #1190 R2).
|
|
2197
|
+
if (err instanceof WaitCheckpointError) {
|
|
2198
|
+
try { if (checkpoint?.local_wait_id) await store.update(checkpoint.local_wait_id, { status: "interrupted" }); } catch { /* retain original diagnostic state */ }
|
|
2199
|
+
return emitSavedWaitError(err);
|
|
2200
|
+
}
|
|
2201
|
+
const settled = await settleInternalError({
|
|
2202
|
+
err, checkpoint, store, waitSessionId, buildPublicReceipt,
|
|
2203
|
+
finalize: (receipt) => finalizeWait(opts, waitSessionId, receipt),
|
|
2204
|
+
});
|
|
2197
2205
|
process.stderr.write(`bb-wait: ${err && err.stack || err}\n`);
|
|
2198
|
-
|
|
2199
|
-
|
|
2206
|
+
// Receipt first, stdout second, cloud finalization last — as on the normal path.
|
|
2207
|
+
emitReceipt(settled.receipt, { versioned: true });
|
|
2208
|
+
await settled.finalizeCloud();
|
|
2209
|
+
process.exit(settled.exitCode);
|
|
2210
|
+
}
|
|
2211
|
+
}
|
|
2212
|
+
|
|
2213
|
+
/**
|
|
2214
|
+
* BOT-2407 AC-2: the last-resort path for an unexpected throw (other than a
|
|
2215
|
+
* WaitCheckpointError, which the caller emits as its own typed receipt) while a wait runs. It
|
|
2216
|
+
* ends with a durable terminal receipt saying why — `interrupted` with no receipt is
|
|
2217
|
+
* reserved for a killed process — and returns the receipt to print, the exit code, and
|
|
2218
|
+
* `finalizeCloud` to call once the receipt is on stdout.
|
|
2219
|
+
* - The bounded public receipt is built once and is the same object that is persisted,
|
|
2220
|
+
* printed and finalized, so `bb wait resume` replays exactly what stdout carried.
|
|
2221
|
+
* - A registered wait is finalized in the cloud (best effort), so /waits learns why
|
|
2222
|
+
* instead of the row waiting for the reaper.
|
|
2223
|
+
* - A terminal receipt already saved (a match persisted from its frame) stays
|
|
2224
|
+
* authoritative: it is never overwritten by the internal error.
|
|
2225
|
+
* Never throws, and neither does `finalizeCloud`.
|
|
2226
|
+
*/
|
|
2227
|
+
export async function settleInternalError({ err, checkpoint, store, waitSessionId = null, buildPublicReceipt, finalize = null }) {
|
|
2228
|
+
if (checkpoint?.status === "terminal" && checkpoint.terminal_receipt != null) {
|
|
2229
|
+
return {
|
|
2230
|
+
receipt: checkpoint.terminal_receipt,
|
|
2231
|
+
exitCode: Number.isInteger(checkpoint.exit_code) ? checkpoint.exit_code : EXIT.INTERNAL,
|
|
2232
|
+
finalizeCloud: async () => {},
|
|
2233
|
+
};
|
|
2200
2234
|
}
|
|
2235
|
+
const internalReceipt = {
|
|
2236
|
+
schema_version: 1,
|
|
2237
|
+
outcome: "error",
|
|
2238
|
+
error: "internal",
|
|
2239
|
+
detail: err instanceof Error ? `${err.name}: ${err.message}` : String(err),
|
|
2240
|
+
wait_session_id: waitSessionId ?? null,
|
|
2241
|
+
next_cursor: checkpoint?.safe_cursor ?? null,
|
|
2242
|
+
};
|
|
2243
|
+
let receipt = internalReceipt;
|
|
2244
|
+
try { receipt = buildPublicReceipt(internalReceipt); } catch { /* keep the raw receipt */ }
|
|
2245
|
+
const id = checkpoint?.local_wait_id;
|
|
2246
|
+
if (id) {
|
|
2247
|
+
try {
|
|
2248
|
+
await store.update(id, {
|
|
2249
|
+
status: "terminal",
|
|
2250
|
+
terminal_receipt: receipt,
|
|
2251
|
+
exit_code: EXIT.INTERNAL,
|
|
2252
|
+
safe_cursor: receipt.next_cursor ?? checkpoint.safe_cursor ?? null,
|
|
2253
|
+
cloud_finalize_pending: Boolean(waitSessionId),
|
|
2254
|
+
});
|
|
2255
|
+
} catch { /* retain original diagnostic state */ }
|
|
2256
|
+
}
|
|
2257
|
+
const finalizeCloud = async () => {
|
|
2258
|
+
if (!waitSessionId || !finalize) return;
|
|
2259
|
+
try {
|
|
2260
|
+
const finalized = await finalize(receipt);
|
|
2261
|
+
if (id) {
|
|
2262
|
+
await store.update(id, finalized?.delivered
|
|
2263
|
+
? { cloud_finalize_pending: false, cloud_finalized_at: new Date().toISOString(), cloud_finalize_error: null }
|
|
2264
|
+
: { cloud_finalize_pending: true, cloud_finalize_error: finalized?.error ?? null });
|
|
2265
|
+
}
|
|
2266
|
+
} catch { /* the pending marker stays for recovery */ }
|
|
2267
|
+
};
|
|
2268
|
+
return { receipt, exitCode: EXIT.INTERNAL, finalizeCloud };
|
|
2201
2269
|
}
|
|
2202
2270
|
|
|
2203
2271
|
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
|