@botbuddy/cli 1.64.11 → 1.64.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/daemon/bin/darwin-arm64/botbuddy-daemon +0 -0
- package/daemon/bin/linux-x64/botbuddy-daemon +0 -0
- package/daemon-release/darwin-arm64.json +4 -4
- package/daemon-release/linux-x64.json +4 -4
- package/daemon-release/manifest.json +5 -5
- package/package.json +1 -1
- package/src/commands.mjs +33 -8
- package/src/locks.mjs +105 -0
- package/src/pw/run.mjs +18 -6
- package/src/staging-pool/cli.mjs +3 -0
- package/src/staging-pool/errors.mjs +1 -1
- package/src/staging-pool/pool.mjs +39 -5
- package/src/staging-pool/signin-gotrue-totp.mjs +16 -5
|
Binary file
|
|
Binary file
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
|
-
"release_version": "1.64.
|
|
3
|
-
"source_commit": "
|
|
4
|
-
"source_sha256": "
|
|
2
|
+
"release_version": "1.64.13",
|
|
3
|
+
"source_commit": "db2f7f45e8e0444857a2046c18481e107e3ebf58",
|
|
4
|
+
"source_sha256": "1533000de11720445186669ac3bf3eb7c663962f809b4035a52fa4de9c368213",
|
|
5
5
|
"target": "darwin-arm64",
|
|
6
|
-
"sha256": "
|
|
6
|
+
"sha256": "8c2dd7c45a7e1c697fa47e5044a1b1cd6703efbb7fc648e1042a104c784094bb"
|
|
7
7
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
|
-
"release_version": "1.64.
|
|
3
|
-
"source_commit": "
|
|
4
|
-
"source_sha256": "
|
|
2
|
+
"release_version": "1.64.13",
|
|
3
|
+
"source_commit": "db2f7f45e8e0444857a2046c18481e107e3ebf58",
|
|
4
|
+
"source_sha256": "1533000de11720445186669ac3bf3eb7c663962f809b4035a52fa4de9c368213",
|
|
5
5
|
"target": "linux-x64",
|
|
6
|
-
"sha256": "
|
|
6
|
+
"sha256": "81345fbe3fde99c1b41f7d1738f00ef2bf4be9de12fe39e0755f23546ed4ed3f"
|
|
7
7
|
}
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schema_version": 1,
|
|
3
|
-
"release_version": "1.64.
|
|
4
|
-
"source_commit": "
|
|
5
|
-
"source_sha256": "
|
|
3
|
+
"release_version": "1.64.13",
|
|
4
|
+
"source_commit": "db2f7f45e8e0444857a2046c18481e107e3ebf58",
|
|
5
|
+
"source_sha256": "1533000de11720445186669ac3bf3eb7c663962f809b4035a52fa4de9c368213",
|
|
6
6
|
"targets": {
|
|
7
7
|
"darwin-arm64": {
|
|
8
8
|
"goos": "darwin",
|
|
9
9
|
"goarch": "arm64",
|
|
10
10
|
"binary": "daemon/bin/darwin-arm64/botbuddy-daemon",
|
|
11
|
-
"sha256": "
|
|
11
|
+
"sha256": "8c2dd7c45a7e1c697fa47e5044a1b1cd6703efbb7fc648e1042a104c784094bb",
|
|
12
12
|
"mode": "0755"
|
|
13
13
|
},
|
|
14
14
|
"linux-x64": {
|
|
15
15
|
"goos": "linux",
|
|
16
16
|
"goarch": "amd64",
|
|
17
17
|
"binary": "daemon/bin/linux-x64/botbuddy-daemon",
|
|
18
|
-
"sha256": "
|
|
18
|
+
"sha256": "81345fbe3fde99c1b41f7d1738f00ef2bf4be9de12fe39e0755f23546ed4ed3f",
|
|
19
19
|
"mode": "0755"
|
|
20
20
|
}
|
|
21
21
|
}
|
package/package.json
CHANGED
package/src/commands.mjs
CHANGED
|
@@ -10,7 +10,7 @@ import { bootstrapHarnessSession, clearHarnessSession, doctorBootstrapState, for
|
|
|
10
10
|
import { loadConfig, getConfig, clearConfig, getConfigPath, SERVER_URL } from "./config.mjs";
|
|
11
11
|
import { resolveOwnerToken, resolveAgentKey, clearOwnerToken } from "./cli-credentials.mjs";
|
|
12
12
|
import { agentProfileKeyLabel } from "./credential-kinds.mjs";
|
|
13
|
-
import { buildAcquireResourcesPayload, LocksUsageError } from "./locks.mjs";
|
|
13
|
+
import { buildAcquireResourcesPayload, LocksUsageError, classifyLockResponse, lockOutcomeLines, unlockOutcomeLines, UNLOCK_QUEUED_USAGE } from "./locks.mjs";
|
|
14
14
|
import { CallUsageError, discoveryUrlFor, formatDiscovery, parseCallArgs } from "./discovery.mjs";
|
|
15
15
|
import { cmdStack } from "./stack.mjs";
|
|
16
16
|
import { cmdManagedAllocation } from "./managed-allocation.mjs";
|
|
@@ -302,7 +302,14 @@ ${bold("STAGING SIGN-IN")} (BOT-2084)
|
|
|
302
302
|
${bold("LOCKS")}
|
|
303
303
|
locks -m [--host name] Reserve typed local resources, including Playwright MCP lanes
|
|
304
304
|
lock <resource_name> <type> Acquire one named lock (acquire_lock)
|
|
305
|
-
|
|
305
|
+
locks/lock: exit 0 acquired · 2 queued · 1 refused.
|
|
306
|
+
Use a resource only after exit 0; queued means another
|
|
307
|
+
agent holds it and you are in line for it.
|
|
308
|
+
unlock <resource_name> Release one named lock you hold (release_lock); it
|
|
309
|
+
does not withdraw a queued request
|
|
310
|
+
unlock --queued <resource_id> <resource_lock_id>
|
|
311
|
+
Withdraw your queued request (ids from the queued
|
|
312
|
+
lock result; retire_queued_lock_attempt)
|
|
306
313
|
|
|
307
314
|
${bold("DURABLE WORKLOADS")} (${cyan("bb help run")})
|
|
308
315
|
run --session-id <id> --environment <env> -- <command>
|
|
@@ -1713,23 +1720,41 @@ export async function cmdMcp(args, {
|
|
|
1713
1720
|
die(`Unknown mcp subcommand: ${sub}. Run ${cyan("botbuddy mcp --help")}.`);
|
|
1714
1721
|
}
|
|
1715
1722
|
|
|
1716
|
-
|
|
1723
|
+
// BOT-2994: each lock command returns its exit code (0 acquired · 2 queued ·
|
|
1724
|
+
// 1 refused; auth 3 / transport 5 unchanged) and says on stderr what the caller
|
|
1725
|
+
// holds. stdout keeps the server's result exactly as before.
|
|
1726
|
+
async function finishLock(response, lines) {
|
|
1727
|
+
const outcome = classifyLockResponse(response);
|
|
1728
|
+
for (const line of lines(outcome)) console.error(line);
|
|
1729
|
+
return outcome.exit;
|
|
1730
|
+
}
|
|
1731
|
+
|
|
1732
|
+
async function cmdLock(args) {
|
|
1717
1733
|
if (args.length < 2) die("Usage: botbuddy lock <resource_name> <type>");
|
|
1718
|
-
return callTool("acquire_lock", { resource_name: args[0], resource_type: args[1] });
|
|
1734
|
+
return finishLock(await callTool("acquire_lock", { resource_name: args[0], resource_type: args[1] }), lockOutcomeLines);
|
|
1719
1735
|
}
|
|
1720
1736
|
|
|
1721
|
-
function cmdLocks(args) {
|
|
1737
|
+
async function cmdLocks(args) {
|
|
1738
|
+
let payload;
|
|
1722
1739
|
try {
|
|
1723
|
-
|
|
1740
|
+
payload = buildAcquireResourcesPayload(args);
|
|
1724
1741
|
} catch (error) {
|
|
1725
1742
|
if (error instanceof LocksUsageError) die(error.message);
|
|
1726
1743
|
throw error;
|
|
1727
1744
|
}
|
|
1745
|
+
return finishLock(await callTool("acquire_resources", payload), lockOutcomeLines);
|
|
1728
1746
|
}
|
|
1729
1747
|
|
|
1730
|
-
function cmdUnlock(args) {
|
|
1748
|
+
async function cmdUnlock(args) {
|
|
1749
|
+
// BOT-2994: release_lock only releases a lock you HOLD; a queued request is
|
|
1750
|
+
// withdrawn by its exact resource + queued-ledger ids (retire_queued_lock_attempt).
|
|
1751
|
+
if (args[0] === "--queued") {
|
|
1752
|
+
if (args.length !== 3 || !args[1] || !args[2] || args[1].startsWith("-") || args[2].startsWith("-")) die(UNLOCK_QUEUED_USAGE);
|
|
1753
|
+
const response = await callTool("retire_queued_lock_attempt", { resource_id: args[1], resource_lock_id: args[2] });
|
|
1754
|
+
return finishLock(response, (outcome) => unlockOutcomeLines(outcome, { queued: true }));
|
|
1755
|
+
}
|
|
1731
1756
|
if (!args[0]) die("Usage: botbuddy unlock <resource_name>");
|
|
1732
|
-
return callTool("release_lock", { resource_name: args[0] });
|
|
1757
|
+
return finishLock(await callTool("release_lock", { resource_name: args[0] }), (outcome) => unlockOutcomeLines(outcome));
|
|
1733
1758
|
}
|
|
1734
1759
|
|
|
1735
1760
|
function cmdTask(args) {
|
package/src/locks.mjs
CHANGED
|
@@ -152,3 +152,108 @@ export function buildAcquireResourcesPayload(args, options = {}) {
|
|
|
152
152
|
...(environment ? { environment } : {}),
|
|
153
153
|
};
|
|
154
154
|
}
|
|
155
|
+
|
|
156
|
+
// BOT-2994: `bb lock`, `bb locks` and `bb unlock` exit by what the server did, not
|
|
157
|
+
// by whether it answered. Before this every one of them exited 0, so a shell loop
|
|
158
|
+
// that trusted the exit code drove another agent's Playwright lane (ENT-6416).
|
|
159
|
+
// 0 = acquired (or released / withdrawn); 2 = queued behind a holder (you hold
|
|
160
|
+
// nothing yet; wait for the grant or withdraw); 1 = refused or not understood.
|
|
161
|
+
// Auth (3) and transport (5) exits from callTool pass through unchanged.
|
|
162
|
+
export const LOCK_EXIT = Object.freeze({ ACQUIRED: 0, REFUSED: 1, QUEUED: 2 });
|
|
163
|
+
|
|
164
|
+
function parseJson(text) {
|
|
165
|
+
try { return JSON.parse(text); } catch { return undefined; }
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function toolText(response) {
|
|
169
|
+
const content = response?.result?.content;
|
|
170
|
+
if (!Array.isArray(content)) return null;
|
|
171
|
+
const text = content.map((part) => part?.text ?? "").join("\n");
|
|
172
|
+
return text || null;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Classify a lock tool response (acquire_lock, acquire_resources, release_lock,
|
|
177
|
+
* retire_queued_lock_attempt) as returned by callTool: the JSON-RPC body, or a
|
|
178
|
+
* numeric exit when callTool already classified an auth/transport failure.
|
|
179
|
+
* Fails closed: an error flag, `success: false`, or an unreadable result is
|
|
180
|
+
* never "acquired".
|
|
181
|
+
*/
|
|
182
|
+
export function classifyLockResponse(response) {
|
|
183
|
+
if (typeof response === "number") return { exit: response, results: [] };
|
|
184
|
+
const isError = response?.result?.isError === true;
|
|
185
|
+
const text = toolText(response);
|
|
186
|
+
if (text === null) return { exit: LOCK_EXIT.REFUSED, results: [] };
|
|
187
|
+
const parsed = parseJson(text);
|
|
188
|
+
let results;
|
|
189
|
+
if (parsed && Array.isArray(parsed.results)) {
|
|
190
|
+
results = parsed.results.map((r) => ({ ...r, success: r?.success === true }));
|
|
191
|
+
} else if (parsed && typeof parsed === "object" && "success" in parsed) {
|
|
192
|
+
results = [{ ...parsed, resource_name: parsed.resource_name ?? parsed.resource, success: parsed.success === true && !isError }];
|
|
193
|
+
} else {
|
|
194
|
+
// The human success string (acquire_lock / release_lock) or a plain error text.
|
|
195
|
+
results = [{ success: !isError, message: text }];
|
|
196
|
+
}
|
|
197
|
+
const failed = results.filter((r) => !r.success);
|
|
198
|
+
let exit = LOCK_EXIT.ACQUIRED;
|
|
199
|
+
if (failed.some((r) => r.queued !== true)) exit = LOCK_EXIT.REFUSED;
|
|
200
|
+
else if (failed.length) exit = LOCK_EXIT.QUEUED;
|
|
201
|
+
else if (isError || !results.length) exit = LOCK_EXIT.REFUSED;
|
|
202
|
+
return { exit, results };
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function holderLabel(holder) {
|
|
206
|
+
if (!holder || typeof holder !== "object") return null;
|
|
207
|
+
const who = holder.agent ?? holder.agent_name ?? holder.name ?? null;
|
|
208
|
+
if (!who) return null;
|
|
209
|
+
const ticket = holder.ticket_id ? `, ${holder.ticket_id}` : "";
|
|
210
|
+
return `${who}${ticket}`;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// `bb wait 'lock:…,claim=true'` only addresses typed resources
|
|
214
|
+
// (`<subtype>:<host>:<slot>`); an untyped name gets no wait line.
|
|
215
|
+
function claimWaitFor(name) {
|
|
216
|
+
const parts = typeof name === "string" ? name.split(":") : [];
|
|
217
|
+
if (parts.length !== 3 || parts.some((p) => !p)) return null;
|
|
218
|
+
const [subtype, host, slot] = parts;
|
|
219
|
+
return `bb wait 'lock:subtype=${subtype},host=${host},slot=${slot},claim=true' --heartbeat`;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/** The stderr lines that tell the caller what it holds and what to do next. */
|
|
223
|
+
export function lockOutcomeLines({ exit, results }) {
|
|
224
|
+
if (exit === LOCK_EXIT.ACQUIRED || exit > LOCK_EXIT.QUEUED) return [];
|
|
225
|
+
const lines = [`✗ Lock not acquired (exit ${exit}: ${exit === LOCK_EXIT.QUEUED ? "queued" : "refused"}). Do not use ${results.length > 1 ? "these resources" : "this resource"} until a lock command exits 0.`];
|
|
226
|
+
for (const r of results.filter((x) => !x.success)) {
|
|
227
|
+
const name = r.resource_name ?? r.resource ?? "resource";
|
|
228
|
+
const holder = holderLabel(r.holder);
|
|
229
|
+
if (r.queued === true) {
|
|
230
|
+
lines.push(` ${name}: queued${holder ? ` behind ${holder}` : ""}${r.queue_depth != null ? ` (queue depth ${r.queue_depth})` : ""}.`);
|
|
231
|
+
const wait = claimWaitFor(name);
|
|
232
|
+
if (wait) lines.push(` wait to be handed it: ${wait}`);
|
|
233
|
+
if (r.resource_id && r.resource_lock_id) lines.push(` or withdraw the request: bb unlock --queued ${r.resource_id} ${r.resource_lock_id}`);
|
|
234
|
+
} else {
|
|
235
|
+
lines.push(` ${name}: refused${holder ? ` (held by ${holder})` : ""}${r.message ? ` — ${r.message}` : ""}`);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
const held = results.filter((r) => r.success && r.resource_name);
|
|
239
|
+
if (held.length && results.length > 1) {
|
|
240
|
+
lines.push(" Acquired in this batch (release what you will not use):");
|
|
241
|
+
for (const r of held) lines.push(` bb unlock ${r.resource_name}`);
|
|
242
|
+
}
|
|
243
|
+
return lines;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/** `bb unlock` refused: say how a queued caller withdraws instead. */
|
|
247
|
+
export function unlockOutcomeLines({ exit, results }, { queued = false } = {}) {
|
|
248
|
+
if (exit === LOCK_EXIT.ACQUIRED || exit > LOCK_EXIT.QUEUED) return [];
|
|
249
|
+
// The first line is the server's reason; later lines are generic guidance.
|
|
250
|
+
const why = results.find((r) => r.message)?.message?.split("\n")[0];
|
|
251
|
+
if (queued) return [`✗ Queued request not withdrawn${why ? `: ${why}` : ""}.`];
|
|
252
|
+
return [
|
|
253
|
+
`✗ Lock not released${why ? `: ${why}` : ""}.`,
|
|
254
|
+
" bb unlock <name> releases a lock you hold; it does not withdraw a queued request.",
|
|
255
|
+
" To withdraw one, use the ids from the queued lock result: bb unlock --queued <resource_id> <resource_lock_id>",
|
|
256
|
+
];
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
export const UNLOCK_QUEUED_USAGE = "Usage: botbuddy unlock --queued <resource_id> <resource_lock_id>";
|
package/src/pw/run.mjs
CHANGED
|
@@ -92,7 +92,7 @@ async function gate({ env, host, lane, deps }) {
|
|
|
92
92
|
// callerIds/holder let staging-auth cleanup touch only what this caller installed.
|
|
93
93
|
const callerIds = [...selfAgentIds];
|
|
94
94
|
if (status.held && selfAgentIds.has(status.heldBy)) return { allowed: true, coordinator, canonicalHost, holderMatch: "self", holder: status.heldBy, callerIds };
|
|
95
|
-
if (status.held && status.ownerIsCaller === true) return { allowed: true, coordinator, canonicalHost, holderMatch: "owner", holder: status.heldBy ?? null, callerIds };
|
|
95
|
+
if (status.held && status.ownerIsCaller === true) return { allowed: true, coordinator, canonicalHost, holderMatch: "owner", holder: status.heldBy ?? null, holderName: status.heldByName ?? null, callerIds };
|
|
96
96
|
// Distinguish an expired/free lane from one another operator holds so the
|
|
97
97
|
// operator knows whether to re-acquire vs. wait for a handover (BOT-660). Name it
|
|
98
98
|
// by the server's canonical host so a not-held message matches the dashboard even
|
|
@@ -296,6 +296,23 @@ async function runPwInner(argv, deps = {}) {
|
|
|
296
296
|
// Report the host the lock service actually stored (alias-collapsed), so the
|
|
297
297
|
// status view and lane telemetry match the server, not the local string (AC-4).
|
|
298
298
|
const effHost = auth.canonicalHost ?? host;
|
|
299
|
+
// BOT-2994: a lane is driven only by the agent that holds it. The gate also
|
|
300
|
+
// passes a same-owner holder (BOT-1490 owner_is_caller), and every agent one
|
|
301
|
+
// person runs shares that owner, so an agent whose `bb lock` had only queued
|
|
302
|
+
// drove another agent's browser (ENT-6416, 2026-11-03). Resolve an owner-only
|
|
303
|
+
// match to the exact agent once (server whoami); only read-only `status` may
|
|
304
|
+
// proceed without it. `auth` keeps its own staging refusal receipt below.
|
|
305
|
+
let holderMatch = auth.holderMatch;
|
|
306
|
+
if (holderMatch === "owner" && auth.holder && authCtx.resolveCallerId) {
|
|
307
|
+
const me = await authCtx.resolveCallerId().catch(() => null);
|
|
308
|
+
if (me && me === auth.holder) { holderMatch = "self"; authCtx.callerIds = [...authCtx.callerIds, me]; }
|
|
309
|
+
}
|
|
310
|
+
if (holderMatch === "owner" && plan.mode !== "status" && plan.mode !== "auth") {
|
|
311
|
+
const lane = `playwright_lane:${effHost}:${plan.lane}`;
|
|
312
|
+
const who = `${auth.holder ?? "unknown"}${auth.holderName ? ` (${auth.holderName})` : ""}`;
|
|
313
|
+
stderr.write(`bb-pw: lane lock ${lane} is held by ${who}, another agent session of your owner, not this agent session. A lane is driven only by the agent that holds it. Claim a lane from this session with bb locks -m --lane <n> --ticket <ticket> and use it only after that exits 0, or wait to be handed this one: bb wait 'lock:subtype=playwright_lane,host=${effHost},slot=${plan.lane},claim=true' --heartbeat\n`);
|
|
314
|
+
return 3;
|
|
315
|
+
}
|
|
299
316
|
// BOT-1318: resolve the lane's browser-acceptance tracker AFTER the gate (a
|
|
300
317
|
// refused/unverified lane writes no lifecycle) and never for unlocked local mode.
|
|
301
318
|
let tracker = null;
|
|
@@ -312,11 +329,6 @@ async function runPwInner(argv, deps = {}) {
|
|
|
312
329
|
if (tracker && envResolution.error) { stderr.write(`${envResolution.error}\n`); return 2; }
|
|
313
330
|
const environment = envResolution.environment ?? null;
|
|
314
331
|
if (plan.mode === "auth") {
|
|
315
|
-
let holderMatch = auth.holderMatch;
|
|
316
|
-
if (holderMatch === "owner" && auth.holder && authCtx.resolveCallerId) {
|
|
317
|
-
const me = await authCtx.resolveCallerId().catch(() => null);
|
|
318
|
-
if (me && me === auth.holder) { holderMatch = "self"; authCtx.callerIds = [...authCtx.callerIds, me]; }
|
|
319
|
-
}
|
|
320
332
|
if (holderMatch !== "self") return stagingAuth.refuse({ stdout, stderr }, "exact_lane_holder_required", 3, {}, authVars(plan, deps));
|
|
321
333
|
if (tracker) {
|
|
322
334
|
const browserSession = await tracker.status();
|
package/src/staging-pool/cli.mjs
CHANGED
|
@@ -284,7 +284,10 @@ export async function takeForLane({
|
|
|
284
284
|
const g = guidance("pool_empty", guidanceVars(ctx, { persona: entry.alias }));
|
|
285
285
|
effective = { ...health, reason: "pool_empty", agentAction: g.agent_action, remedy: remedyLine(g), message: g.message, cause: g.cause, actor: g.actor_label, next: g.next, doc: g.doc, help: g.help };
|
|
286
286
|
}
|
|
287
|
+
// BOT-2952: the pool's own detail (an Auth outage at take time) survives the health's.
|
|
288
|
+
const poolDetail = error.detail;
|
|
287
289
|
for (const field of GUIDANCE_FIELDS) if (effective[field] !== undefined) error[field] = effective[field];
|
|
290
|
+
if (poolDetail) error.detail = [poolDetail, effective.detail].filter(Boolean).join(" ");
|
|
288
291
|
throw error;
|
|
289
292
|
}
|
|
290
293
|
}
|
|
@@ -135,7 +135,7 @@ export const ERROR_CODES = Object.freeze({
|
|
|
135
135
|
// already asked the human-installed job to refill now, so the agent waits once and retries.
|
|
136
136
|
pool_empty: entry("agent", "take",
|
|
137
137
|
"The staging session pool is healthy but has no ready session for {persona}.",
|
|
138
|
-
"Agents took every ready session since the last tick, or the ready ones were dead or AAL1 (factor rotated) and were discarded; automatic refill creates a durable job before the single retry.",
|
|
138
|
+
"Agents took every ready session since the last tick, or the ready ones were dead or AAL1 (factor rotated) and were discarded, or staging Auth did not answer their liveness check and they were kept (detail says so); automatic refill creates a durable job before the single retry.",
|
|
139
139
|
"Automatic refill is requested by bb pw auth; wait for its refill job, then {retry} once; if it fails, report this code", "retry_once"),
|
|
140
140
|
pool_refill_unavailable: entry("agent", "take",
|
|
141
141
|
"The staging session pool is empty and automatic refill did not return a durable job.",
|
|
@@ -483,6 +483,7 @@ export function createSessionPool({
|
|
|
483
483
|
const entry = (health.personas[persona] ??= {
|
|
484
484
|
belowTargetTicks: 0,
|
|
485
485
|
consecutiveMintFailures: 0,
|
|
486
|
+
backoffMintFailures: 0,
|
|
486
487
|
nextMintAttemptAt: null,
|
|
487
488
|
lastMintError: null,
|
|
488
489
|
});
|
|
@@ -505,20 +506,33 @@ export function createSessionPool({
|
|
|
505
506
|
await mintIntoReady(persona);
|
|
506
507
|
counts[persona] += 1;
|
|
507
508
|
entry.consecutiveMintFailures = 0;
|
|
509
|
+
entry.backoffMintFailures = 0;
|
|
508
510
|
entry.nextMintAttemptAt = null;
|
|
509
511
|
entry.lastMintError = null;
|
|
510
512
|
} catch (error) {
|
|
511
513
|
summary.mintFailures += 1;
|
|
512
514
|
entry.consecutiveMintFailures += 1;
|
|
513
515
|
entry.lastMintError = error?.code ?? "error";
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
516
|
+
// BOT-2952: an outage that never got a password grant created no AAL1 session and
|
|
517
|
+
// ran no MFA verify, so it retries on the next tick and does not lengthen a later
|
|
518
|
+
// backoff. A 429 is Auth asking us to slow down, so it backs off like any other
|
|
519
|
+
// failure, on its own streak.
|
|
520
|
+
const outage =
|
|
521
|
+
error?.transient === true && error?.passwordGrantSucceeded === false && error?.rateLimited !== true;
|
|
522
|
+
if (outage) {
|
|
523
|
+
entry.nextMintAttemptAt = null;
|
|
524
|
+
} else {
|
|
525
|
+
entry.backoffMintFailures = (entry.backoffMintFailures ?? 0) + 1;
|
|
526
|
+
entry.nextMintAttemptAt = new Date(
|
|
527
|
+
now() + nextMintBackoffMs({ failures: entry.backoffMintFailures, intervalMs }),
|
|
528
|
+
).toISOString();
|
|
529
|
+
}
|
|
517
530
|
logger({
|
|
518
531
|
event: "mint_failed",
|
|
519
532
|
persona,
|
|
520
533
|
code: entry.lastMintError,
|
|
521
534
|
message: error?.message ?? String(error),
|
|
535
|
+
outage,
|
|
522
536
|
consecutiveMintFailures: entry.consecutiveMintFailures,
|
|
523
537
|
nextMintAttemptAt: entry.nextMintAttemptAt,
|
|
524
538
|
});
|
|
@@ -567,6 +581,7 @@ export function createSessionPool({
|
|
|
567
581
|
}
|
|
568
582
|
}
|
|
569
583
|
candidates.sort((a, b) => b.verifiedAt - a.verifiedAt);
|
|
584
|
+
let inconclusive = null;
|
|
570
585
|
for (const { slotId } of candidates) {
|
|
571
586
|
const takenDir = join(dirs.taken, slotId);
|
|
572
587
|
if (!(await claim(join(dirs.ready, slotId), takenDir))) continue;
|
|
@@ -582,8 +597,23 @@ export function createSessionPool({
|
|
|
582
597
|
continue;
|
|
583
598
|
}
|
|
584
599
|
const result = await signIn.refresh({ target, storedSession: slot.storedSession, persona });
|
|
600
|
+
if (!result.ok && result.transient) {
|
|
601
|
+
// BOT-2952: Auth did not answer, which says nothing about this slot. It goes back to
|
|
602
|
+
// ready unchanged (the tick does the same, BOT-2947) and the next slot is tried; a
|
|
603
|
+
// persona has at most 4. Nothing unverified is ever handed out.
|
|
604
|
+
inconclusive = result.reason;
|
|
605
|
+
await fs.rm(join(takenDir, "claim.json"), { force: true });
|
|
606
|
+
try {
|
|
607
|
+
await fs.rename(takenDir, join(dirs.ready, slotId));
|
|
608
|
+
} catch (error) {
|
|
609
|
+
// A tick purged it between the claim marker's removal and the rename.
|
|
610
|
+
if (error?.code !== "ENOENT") throw error;
|
|
611
|
+
}
|
|
612
|
+
logger({ event: "take_inconclusive", slotId, persona, reason: result.reason });
|
|
613
|
+
continue;
|
|
614
|
+
}
|
|
585
615
|
if (!result.ok) {
|
|
586
|
-
logger({ event: "take_discard", slotId, persona, reason: result.reason, transient:
|
|
616
|
+
logger({ event: "take_discard", slotId, persona, reason: result.reason, transient: false });
|
|
587
617
|
await removeSlot(takenDir);
|
|
588
618
|
continue;
|
|
589
619
|
}
|
|
@@ -602,7 +632,11 @@ export function createSessionPool({
|
|
|
602
632
|
logger({ event: "take_ok", slotId, persona, owner: meta.owner });
|
|
603
633
|
return { slotId, statePath: join(takenDir, "state.json"), dir: takenDir, meta };
|
|
604
634
|
}
|
|
605
|
-
|
|
635
|
+
const empty = new PoolEmptyError(persona, tickIntervalHint);
|
|
636
|
+
if (inconclusive) {
|
|
637
|
+
empty.detail = `Staging Auth did not answer the liveness check (${inconclusive}); the ready sessions were kept for a retry.`;
|
|
638
|
+
}
|
|
639
|
+
throw empty;
|
|
606
640
|
}
|
|
607
641
|
|
|
608
642
|
async function discardTaken(slotId) {
|
|
@@ -16,12 +16,17 @@ import { assertPersonaAllowed } from "./policy.mjs";
|
|
|
16
16
|
import { decodeJwtClaims, toStoredSession } from "./state-file.mjs";
|
|
17
17
|
|
|
18
18
|
export class SignInError extends Error {
|
|
19
|
-
constructor(code, message, { transient = false, status } = {}) {
|
|
19
|
+
constructor(code, message, { transient = false, status, passwordGrantSucceeded, rateLimited } = {}) {
|
|
20
20
|
super(message);
|
|
21
21
|
this.name = "SignInError";
|
|
22
22
|
this.code = code;
|
|
23
23
|
this.transient = transient;
|
|
24
24
|
this.status = status;
|
|
25
|
+
// BOT-2952: set on mint_retries_exhausted. false = no attempt got a password grant, so no
|
|
26
|
+
// AAL1 session existed and no MFA verify ran (the pool retries next tick, no backoff).
|
|
27
|
+
if (passwordGrantSucceeded !== undefined) this.passwordGrantSucceeded = passwordGrantSucceeded;
|
|
28
|
+
// A 429 on any attempt: Auth answered "slow down", so the pool keeps its backoff.
|
|
29
|
+
if (rateLimited !== undefined) this.rateLimited = rateLimited;
|
|
25
30
|
}
|
|
26
31
|
}
|
|
27
32
|
|
|
@@ -93,9 +98,12 @@ export function createGoTrueTotpSignIn({
|
|
|
93
98
|
async function mint({ target, persona }) {
|
|
94
99
|
assertPersonaAllowed(target, persona);
|
|
95
100
|
let lastFailure = "no attempt made";
|
|
101
|
+
let passwordGrantSucceeded = false;
|
|
102
|
+
let rateLimited = false;
|
|
96
103
|
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
97
|
-
const retry = async (reason, { waitWindow = false } = {}) => {
|
|
104
|
+
const retry = async (reason, { waitWindow = false, response } = {}) => {
|
|
98
105
|
lastFailure = reason;
|
|
106
|
+
if (response?.status === 429) rateLimited = true;
|
|
99
107
|
logger({ event: "mint_retry", persona, attempt, reason });
|
|
100
108
|
if (attempt < maxAttempts) {
|
|
101
109
|
if (waitWindow) await waitForNextTotpWindow();
|
|
@@ -108,7 +116,7 @@ export function createGoTrueTotpSignIn({
|
|
|
108
116
|
});
|
|
109
117
|
if (password.status !== 200) {
|
|
110
118
|
if (isTransient(password)) {
|
|
111
|
-
await retry(`password grant ${describeFailure(password)}
|
|
119
|
+
await retry(`password grant ${describeFailure(password)}`, { response: password });
|
|
112
120
|
continue;
|
|
113
121
|
}
|
|
114
122
|
if (isCredentialRefusal(password)) {
|
|
@@ -120,6 +128,7 @@ export function createGoTrueTotpSignIn({
|
|
|
120
128
|
status: password.status,
|
|
121
129
|
});
|
|
122
130
|
}
|
|
131
|
+
passwordGrantSucceeded = true;
|
|
123
132
|
const aal1Token = password.body?.access_token;
|
|
124
133
|
const factor = (password.body?.user?.factors ?? []).find(
|
|
125
134
|
(candidate) => candidate.factor_type === "totp" && candidate.status === "verified",
|
|
@@ -135,7 +144,7 @@ export function createGoTrueTotpSignIn({
|
|
|
135
144
|
continue;
|
|
136
145
|
}
|
|
137
146
|
if (isTransient(challenge)) {
|
|
138
|
-
await retry(`challenge ${describeFailure(challenge)}
|
|
147
|
+
await retry(`challenge ${describeFailure(challenge)}`, { response: challenge });
|
|
139
148
|
continue;
|
|
140
149
|
}
|
|
141
150
|
throw new SignInError("challenge_failed", `MFA challenge refused: ${describeFailure(challenge)}.`, {
|
|
@@ -158,7 +167,7 @@ export function createGoTrueTotpSignIn({
|
|
|
158
167
|
continue;
|
|
159
168
|
}
|
|
160
169
|
if (isTransient(verify)) {
|
|
161
|
-
await retry(`verify ${describeFailure(verify)}
|
|
170
|
+
await retry(`verify ${describeFailure(verify)}`, { response: verify });
|
|
162
171
|
continue;
|
|
163
172
|
}
|
|
164
173
|
throw new SignInError("verify_failed", `MFA verify refused: ${describeFailure(verify)}.`, { status: verify.status });
|
|
@@ -176,6 +185,8 @@ export function createGoTrueTotpSignIn({
|
|
|
176
185
|
}
|
|
177
186
|
throw new SignInError("mint_retries_exhausted", `Mint gave up after ${maxAttempts} attempts (last: ${lastFailure}).`, {
|
|
178
187
|
transient: true,
|
|
188
|
+
passwordGrantSucceeded,
|
|
189
|
+
rateLimited,
|
|
179
190
|
});
|
|
180
191
|
}
|
|
181
192
|
|