@botbuddy/cli 1.63.13 → 1.63.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/daemon/bin/darwin-arm64/botbuddy-daemon +0 -0
- package/daemon/bin/linux-x64/botbuddy-daemon +0 -0
- package/daemon-release/darwin-arm64.json +4 -4
- package/daemon-release/linux-x64.json +4 -4
- package/daemon-release/manifest.json +5 -5
- package/package.json +1 -1
- package/src/daemon/callback-release.mjs +50 -0
- package/src/daemon/installer.mjs +30 -0
- package/src/fixtures/bot2775-callback-contract.json +32 -0
- package/src/fixtures/bot2779-handler-status-2026-10-27.json +47 -0
- package/src/stack-callback-gateway.mjs +44 -0
- package/src/stack-identified-admission.mjs +17 -0
- package/src/stack-isolation.mjs +57 -3
- package/src/stack-native-callback.mjs +126 -0
- package/src/stack-requester-origin-config.mjs +21 -0
- package/src/stack-requester-origin.mjs +62 -0
- package/src/stack-tunnel.mjs +4 -2
- package/src/stack.mjs +327 -23
- package/src/vendor/smol-toml/LICENSE +24 -0
- package/src/vendor/smol-toml/date.js +156 -0
- package/src/vendor/smol-toml/error.js +67 -0
- package/src/vendor/smol-toml/extract.js +293 -0
- package/src/vendor/smol-toml/index.js +34 -0
- package/src/vendor/smol-toml/parse.js +164 -0
- package/src/vendor/smol-toml/primitive.js +151 -0
- package/src/vendor/smol-toml/stringify.js +219 -0
- package/src/vendor/smol-toml/struct.js +156 -0
- package/src/vendor/smol-toml/util.js +57 -0
- package/src/vendor/smol-toml/vendored.json +20 -0
package/src/stack.mjs
CHANGED
|
@@ -33,7 +33,7 @@ import { existsSync, lstatSync, readFileSync, realpathSync } from "fs";
|
|
|
33
33
|
import { open, unlink, readFile } from "fs/promises";
|
|
34
34
|
import { homedir, tmpdir } from "os";
|
|
35
35
|
import { basename, dirname, join, relative, isAbsolute } from "path";
|
|
36
|
-
import { randomUUID } from "crypto";
|
|
36
|
+
import { randomUUID, createHash } from "crypto";
|
|
37
37
|
import { fileURLToPath } from "url";
|
|
38
38
|
import { callToolJson, CALL_AUTH_ERROR_CODES } from "./api.mjs";
|
|
39
39
|
import { deepLinkNotice, safeWebUrl } from "./deep-link.mjs";
|
|
@@ -43,12 +43,16 @@ import { AGENT_KEY_RE, readAgentSessionTokenEnv } from "./agent-key.mjs";
|
|
|
43
43
|
import { resolveAgentSessionCredential, selfHealAgentSession } from "./agent-session.mjs";
|
|
44
44
|
import { runDockerCommand, runDockerWorkflow, ADMITTED_DOCKER_VALIDATIONS } from "./docker-hygiene.mjs";
|
|
45
45
|
import { acquireStackLock, lockPathForProject, projectIdFromConfig } from "./stack-file-lock.mjs";
|
|
46
|
-
import { LEASED_STACK_IDLE_RETENTION_SECS, bindLeasedStackClaim, recoverLeasedStackClaim, hasLocalLeaseClaim, leasedClaimContext, assertLeasedStackIsolated, leasedStackClaimHeld, markLeasedStackReleased, stableRunStackReusable, leasedStackIdentity, materializeRunStackDir, portMapInConfig, refreshLeasedStackRetention, releaseLeasedStackClaim, reservePortBlocks } from "./stack-isolation.mjs";
|
|
46
|
+
import { LEASED_STACK_IDLE_RETENTION_SECS, assertFreshRunStackAvailable, recordAdmissionRequest, bindLeasedStackClaim, recoverLeasedStackClaim, hasLocalLeaseClaim, leasedClaimContext, assertLeasedStackIsolated, leasedStackClaimHeld, markLeasedStackReleased, stableRunStackReusable, leasedStackIdentity, materializeRunStackDir, portMapInConfig, refreshLeasedStackRetention, releaseLeasedStackClaim, reservePortBlocks } from "./stack-isolation.mjs";
|
|
47
|
+
import { reserveRequesterOrigin } from "./stack-requester-origin.mjs";
|
|
48
|
+
import { identifiedAdmissionAck, identifiedAdmissionDelivery, identifiedAdmissionRecovery } from "./stack-identified-admission.mjs";
|
|
49
|
+
import { patchRequesterAuthOrigin } from "./stack-requester-origin-config.mjs";
|
|
47
50
|
import { measureStackUsage } from "./stack-usage.mjs";
|
|
48
51
|
import { checkReadoptedStackMounts } from "./stack-mount-health.mjs";
|
|
49
52
|
import { machineUuid } from "./machine-id.mjs";
|
|
50
53
|
import { DEFAULT_BUNDLE_MAX_BYTES, buildStackBundle, uploadStackBundle, validateBundleIncludePath } from "./stack-bundle.mjs";
|
|
51
54
|
import { assertSshTarget, establishSshTunnel } from "./stack-tunnel.mjs";
|
|
55
|
+
import { NATIVE_CALLBACK_TRANSPORT, openNativeCallbackTransport } from "./stack-native-callback.mjs";
|
|
52
56
|
import { FORWARD_ATTESTATION_ENV, startForwardAttestation, readForwardAttestation } from "./stack-forward-attestation.mjs";
|
|
53
57
|
import { stackServicesForForwards } from "./stack-service-discovery.mjs";
|
|
54
58
|
import { ENGINE_PUBLISHES_BEYOND_LOOPBACK, engineSocketIsLocal, formatPublishIp, isLoopbackIp, publishIpFix, resolvePublishIp } from "./docker-publish-ip.mjs";
|
|
@@ -57,12 +61,28 @@ import { reportStackRunOutcome } from "./stack-run-outcome.mjs";
|
|
|
57
61
|
|
|
58
62
|
export const STACK_SCHEMA_VERSION = 1;
|
|
59
63
|
|
|
64
|
+
/**
|
|
65
|
+
* BOT-2780: only `stack run --remote --execution-ring beta --reap` reserves a requester
|
|
66
|
+
* origin and allocates through BOT-2499's identified admission. The origin cannot outlive
|
|
67
|
+
* its CLI, so the invocation must own cleanup; stable/auto/local and idle runs are unchanged.
|
|
68
|
+
*/
|
|
69
|
+
export const requesterOriginEnabled = (opts) => opts.execution === "remote" && opts.executionRing === "beta" && opts.reap === true;
|
|
70
|
+
// Exact replays of one identified request after an uncertain answer (lost response, 5xx,
|
|
71
|
+
// ADMISSION_UNCERTAIN, pending delivery). The same request_id can never create a second lease.
|
|
72
|
+
const ADMISSION_REPLAY_DELAYS_MS = [1000, 2000, 4000];
|
|
73
|
+
const ADMISSION_REPLAYABLE = new Set(["ADMISSION_UNCERTAIN", "RETRY_SAME_REQUEST", "HOST_ELIGIBILITY_BUSY"]);
|
|
74
|
+
// The key already has a ledger row (or had one): never replay it blindly or treat it as unallocated.
|
|
75
|
+
const ADMISSION_COMMITTED_UNCERTAIN = new Set(["REQUEST_IDENTITY_MISMATCH", "REQUEST_CLOSED", "ORIGINAL_LEASE_UNAVAILABLE"]);
|
|
76
|
+
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
77
|
+
|
|
60
78
|
/**
|
|
61
79
|
* Codex #1077 P1 / BOT-2184: an sshd with `GatewayPorts yes` binds a `-R` forward on every
|
|
62
80
|
* interface. A remote lease with --callback-port therefore runs the pre-forward GatewayPorts
|
|
63
81
|
* probe (`probeGatewayPorts` in stack-tunnel.mjs) before the real `-R` exists, and refuses
|
|
64
82
|
* with exit 5 CALLBACK_NOT_LOOPBACK when the host would expose it. The code below is kept
|
|
65
83
|
* only so older automation that matched on it still resolves; no path emits it any more.
|
|
84
|
+
* BOT-2775: an explicit beta remote callback skips the -R path and uses the verified native
|
|
85
|
+
* private route (stack-native-callback.mjs); stable/default remote callbacks keep the probe.
|
|
66
86
|
*/
|
|
67
87
|
export const CALLBACK_PORT_UNSUPPORTED_REMOTE = "CALLBACK_PORT_UNSUPPORTED_REMOTE";
|
|
68
88
|
export const DEFAULT_RECEIPT_MAX_BYTES = 10240;
|
|
@@ -119,6 +139,8 @@ export const STACK_HELP = `${bold("botbuddy stack")} — one command for a batch
|
|
|
119
139
|
|
|
120
140
|
${bold("USAGE")}
|
|
121
141
|
botbuddy stack up [options] Request a lease; park if the host is full; hold it active
|
|
142
|
+
botbuddy stack status --admission-request-id <uuid>
|
|
143
|
+
Read an identified admission after a lost response (BOT-2780)
|
|
122
144
|
botbuddy stack status <lease_id> Show a lease's state + connection; within a live remote run,
|
|
123
145
|
include the owner's attested local/remote forwards
|
|
124
146
|
botbuddy stack touch <lease_id> Bump the idle clock (and report measured use) so the reaper doesn't stop it
|
|
@@ -233,12 +255,23 @@ ${bold("run OPTIONS")}
|
|
|
233
255
|
else a remote host with capacity; the bundle digest is sent up front.
|
|
234
256
|
--execution-ring R Capacity rollout ring: stable (default) excludes beta daemon hosts.
|
|
235
257
|
beta is explicit opt-in and requires \`stack run --remote\`.
|
|
258
|
+
With --remote --execution-ring beta --reap (BOT-2780), run reserves a
|
|
259
|
+
loopback requester origin first and writes it as the private stack copy's
|
|
260
|
+
[auth] external_url, so generated auth links (/auth/v1/verify…) reach the
|
|
261
|
+
stack unchanged even when the SSH API forward is remapped; the child gets
|
|
262
|
+
it as connection.requester_origin. The lease is requested through the
|
|
263
|
+
identified admission (exact replay after a lost response, no fallback).
|
|
236
264
|
--callback-port N The caller runs a listener the stack calls back through
|
|
237
|
-
host.docker.internal:N (e.g. 4600): sent as callback_port. On a
|
|
238
|
-
lease run first probes the host with a throwaway reverse forward
|
|
239
|
-
-R 127.0.0.1:N:127.0.0.1:N only when the host keeps it on loopback
|
|
240
|
-
GatewayPorts no/clientspecified); otherwise exit 5
|
|
241
|
-
(BOT-2184).
|
|
265
|
+
host.docker.internal:N (e.g. 4600): sent as callback_port. On a stable or
|
|
266
|
+
auto REMOTE lease run first probes the host with a throwaway reverse forward
|
|
267
|
+
and opens -R 127.0.0.1:N:127.0.0.1:N only when the host keeps it on loopback
|
|
268
|
+
(sshd GatewayPorts no/clientspecified); otherwise exit 5
|
|
269
|
+
CALLBACK_NOT_LOOPBACK (BOT-2184). With --remote --execution-ring beta it
|
|
270
|
+
instead requires a Linux Go runtime with native callbacks (protocol 5): a
|
|
271
|
+
private SSH Unix-socket route to a fixed requester gateway, renewed against
|
|
272
|
+
current owned authorization (BOT-2775). Unknown/older hosts or a backend
|
|
273
|
+
that does not acknowledge it fail HOST_CALLBACK_TRANSPORT_UNSUPPORTED; the
|
|
274
|
+
child never receives the route capability.
|
|
242
275
|
--bundle-max-mb N Remote-lease bundle size cap in MB (default 50, compressed). The bundle
|
|
243
276
|
excludes .bb-stack.json, node_modules, supabase/.temp|.branches and
|
|
244
277
|
untracked-secret dotenv files (.env, .env.local, .env.*.local).
|
|
@@ -299,9 +332,15 @@ ${bold("REMOTE LEASES")} (BOT-2142, stack run)
|
|
|
299
332
|
4 STACK_EXECUTION_MODE_MISMATCH the slot already holds a lease in another mode/host — stack done it first
|
|
300
333
|
(the four codes above create NO lease)
|
|
301
334
|
5 BUNDLE_UPLOAD_FAILED the signed bundle upload failed (the lease is released first)
|
|
302
|
-
5 CALLBACK_NOT_LOOPBACK --callback-port: the GatewayPorts probe (before any real -R)
|
|
303
|
-
post-forward check found the host's -R listener not loopback-only
|
|
335
|
+
5 CALLBACK_NOT_LOOPBACK --callback-port (stable/auto): the GatewayPorts probe (before any real -R)
|
|
336
|
+
or the post-forward check found the host's -R listener not loopback-only
|
|
304
337
|
(sshd GatewayPorts yes), or could not check it (no lsof, no listener)
|
|
338
|
+
5 HOST_CALLBACK_TRANSPORT_UNSUPPORTED --callback-port (beta remote): native callback transport is
|
|
339
|
+
unavailable; needs the installed Linux Go runtime (protocol 5) and backend
|
|
340
|
+
5 CALLBACK_NOT_READY / CALLBACK_CONSUMING_NAMESPACE_UNREACHABLE the native route did not
|
|
341
|
+
answer from the stack's own network before the child (lease reaped)
|
|
342
|
+
6 CALLBACK_TRANSPORT_LOST / CALLBACK_AUTHORIZATION_LOST the native route or its current
|
|
343
|
+
authorization closed; the child is stopped and the lease reaped
|
|
305
344
|
5 CALLBACK_PORT_BUSY --callback-port is already listening on the serving host (another run or
|
|
306
345
|
service), the host refused this run's -R because another run took the
|
|
307
346
|
port first, or another listener shares the port; this run holds no -R —
|
|
@@ -421,6 +460,7 @@ export function parseStackArgs(argv) {
|
|
|
421
460
|
for (let i = 0; i < rest.length; i++) {
|
|
422
461
|
const a = rest[i];
|
|
423
462
|
switch (a) {
|
|
463
|
+
case "--admission-request-id": if (need(a, rest[i + 1])) opts.admissionRequestId = String(rest[++i]).toLowerCase(); break;
|
|
424
464
|
case "--slot": if (need(a, rest[i + 1])) opts.slot = rest[++i]; break;
|
|
425
465
|
case "--host": if (need(a, rest[i + 1])) opts.host = rest[++i]; break;
|
|
426
466
|
case "--repo": if (need(a, rest[i + 1])) opts.repo = rest[++i]; break;
|
|
@@ -588,7 +628,11 @@ export function parseStackArgs(argv) {
|
|
|
588
628
|
errors.push("--stack-path must be a non-empty relative path without '..'");
|
|
589
629
|
}
|
|
590
630
|
const leaseId = positionals[0] ?? null;
|
|
591
|
-
if (
|
|
631
|
+
if (opts.admissionRequestId != null && (!UUID_RE.test(opts.admissionRequestId) ||
|
|
632
|
+
!((command === "status" && !leaseId) || (command === "done" && leaseId && opts.reap && opts.releasePortClaim)))) {
|
|
633
|
+
errors.push("--admission-request-id takes a request UUID with `status` (no <lease_id>) or `done <lease_id> --reap --release-port-claim`");
|
|
634
|
+
}
|
|
635
|
+
if (command && ["status", "touch", "done"].includes(command) && !leaseId && !(command === "status" && opts.admissionRequestId)) {
|
|
592
636
|
errors.push(`${command} needs a <lease_id>`);
|
|
593
637
|
}
|
|
594
638
|
if (["up", "done"].includes(command) && opts.localExec && Boolean(opts.dockerContext) === Boolean(opts.dockerEndpoint)) {
|
|
@@ -865,6 +909,16 @@ export async function resolveRunExecution(cwd, opts, { materialize = materialize
|
|
|
865
909
|
}
|
|
866
910
|
return execution;
|
|
867
911
|
};
|
|
912
|
+
// BOT-2780: the requester origin is only for a fresh CLI-managed stack context. A
|
|
913
|
+
// prepared path, pinned host or fixed nonce could resume a stack whose generated links
|
|
914
|
+
// already name another invocation's listener.
|
|
915
|
+
const originRun = requesterOriginEnabled(opts);
|
|
916
|
+
if (originRun && (opts.host != null || opts.stackNonce != null || (opts.stackPath && opts.stackPath !== ".") ||
|
|
917
|
+
!UUID_RE.test(opts.admissionRequestId ?? "") || !opts.requesterMachineId)) {
|
|
918
|
+
const error = new Error("beta requester origins require a fresh CLI-managed stack context (no --host, --stack-path or fixed nonce)");
|
|
919
|
+
error.code = "REQUESTER_ORIGIN_CONTEXT_REFUSED";
|
|
920
|
+
throw error;
|
|
921
|
+
}
|
|
868
922
|
// An explicitly prepared directory is the caller's to manage, but it is NOT
|
|
869
923
|
// exempt from the isolation contract (BOT-1798, Codex #928 R8 P1): a copied root
|
|
870
924
|
// config — same project_id, or merely the same ports — would make the Helper
|
|
@@ -930,15 +984,30 @@ export async function resolveRunExecution(cwd, opts, { materialize = materialize
|
|
|
930
984
|
const nonce = opts.stackNonce ?? (stableReusable(worktreeRoot, stableDir) ? null : randomUUID());
|
|
931
985
|
const { projectId } = identity({ worktreeRoot, slot: opts.slot, ticket: opts.ticket, nonce });
|
|
932
986
|
const held = await claimProjectLock(projectId, lock, `this batch's isolated stack (${projectId})`);
|
|
987
|
+
let requesterOrigin = null;
|
|
988
|
+
let originContext = null;
|
|
933
989
|
try {
|
|
990
|
+
let configText;
|
|
991
|
+
if (originRun) {
|
|
992
|
+
// BOT-2780: refuse a retained slot context, then reserve the listener BEFORE the
|
|
993
|
+
// private config (and so the bundle digest) is derived. The source config is never
|
|
994
|
+
// changed; only the private materialized copy names the reserved origin.
|
|
995
|
+
assertFreshRunStackAvailable(worktreeRoot, opts.slot);
|
|
996
|
+
originContext = Object.freeze({ request_id: opts.admissionRequestId, worktree: worktreeRoot, slot: opts.slot, machine: opts.requesterMachineId });
|
|
997
|
+
requesterOrigin = await reserveRequesterOrigin({ context: originContext });
|
|
998
|
+
// nosemgrep: javascript.lang.security.audit.path-traversal.path-join-resolve-traversal.path-join-resolve-traversal -- canonical realpath of the current worktree; fixed own source config, never a request-supplied path.
|
|
999
|
+
configText = await patchRequesterAuthOrigin(readFileSync(join(worktreeRoot, "supabase", "config.toml"), "utf8"), requesterOrigin.origin);
|
|
1000
|
+
}
|
|
934
1001
|
const execution = refuseRoot(resolveStackPath(cwd, materialize(
|
|
935
1002
|
cwd,
|
|
936
1003
|
{ slot: opts.slot, repo: opts.repo, ticket: opts.ticket, nonce, idleTtlSecs: opts.idleTtl ?? undefined },
|
|
937
1004
|
// BOT-2403: a functions directory another writer replaced under this stack is reported.
|
|
938
|
-
{ ...(portRegistryDir ? { portRegistryDir } : {}), warn },
|
|
1005
|
+
{ ...(portRegistryDir ? { portRegistryDir } : {}), warn, ...(configText !== undefined ? { configText } : {}) },
|
|
939
1006
|
)));
|
|
940
1007
|
return {
|
|
941
1008
|
...execution,
|
|
1009
|
+
...(requesterOrigin ? { requesterOrigin, originContext, admissionRequestId: opts.admissionRequestId,
|
|
1010
|
+
recordAdmission: (requestId) => recordAdmissionRequest(stackDirFor(execution), requestId, portRegistryDir ? { registryDir: portRegistryDir } : undefined) } : {}),
|
|
942
1011
|
bindLease: (leaseId) => bindLeasedStackClaim(stackDirFor(execution), leaseId, portRegistryDir ? { registryDir: portRegistryDir } : undefined),
|
|
943
1012
|
// Release is called once the batch's signed reap is done, so hand the host
|
|
944
1013
|
// its port block back immediately rather than holding it for the retention
|
|
@@ -951,6 +1020,7 @@ export async function resolveRunExecution(cwd, opts, { materialize = materialize
|
|
|
951
1020
|
// BOT-2237 (Codex #1168 P2): returns the VERIFIED disposition — whether this stack's
|
|
952
1021
|
// claim is gone afterwards — so a failed unlink is never reported as released.
|
|
953
1022
|
release: ({ reaped = false } = {}) => {
|
|
1023
|
+
void requesterOrigin?.close();
|
|
954
1024
|
const registry = portRegistryDir ? { registryDir: portRegistryDir } : undefined;
|
|
955
1025
|
let released = false;
|
|
956
1026
|
if (reaped) {
|
|
@@ -964,6 +1034,7 @@ export async function resolveRunExecution(cwd, opts, { materialize = materialize
|
|
|
964
1034
|
},
|
|
965
1035
|
};
|
|
966
1036
|
} catch (error) {
|
|
1037
|
+
await requesterOrigin?.close();
|
|
967
1038
|
held.release();
|
|
968
1039
|
throw error;
|
|
969
1040
|
}
|
|
@@ -2999,6 +3070,21 @@ export async function cmdStatus(leaseId, opts, {
|
|
|
2999
3070
|
} = {}) {
|
|
3000
3071
|
const auth = await rpc.init();
|
|
3001
3072
|
if (!auth) return emitResult(buildReceipt({ command: "status", outcome: "error", lease_id: leaseId, error: "not authenticated — run `botbuddy login`" }), opts, EXIT.AUTH);
|
|
3073
|
+
if (opts.admissionRequestId) {
|
|
3074
|
+
// BOT-2780: the owner-scoped identified-admission read. It never returns a connection or
|
|
3075
|
+
// capability, and an unknown key is not proof that nothing was admitted.
|
|
3076
|
+
const read = await rpc.call("get_stack_lease_admission", { request_id: opts.admissionRequestId });
|
|
3077
|
+
const base = { command: "status", request_id: opts.admissionRequestId };
|
|
3078
|
+
if (!read.ok) return emitResult(buildReceipt({ ...base, outcome: "error", error: read.error }), opts, read.auth ? EXIT.AUTH : EXIT.BACKEND);
|
|
3079
|
+
const a = read.data ?? {};
|
|
3080
|
+
if (a.success !== true) return emitResult(buildReceipt({ ...base, outcome: "error", code: a.code ?? null, error: a.code ?? "admission unreadable" }), opts, EXIT.BACKEND);
|
|
3081
|
+
return emitResult(buildReceipt({
|
|
3082
|
+
...base, outcome: a.lease_status ?? "unknown", lease_id: a.lease_id ?? null, admission_kind: a.admission_kind ?? null,
|
|
3083
|
+
job_binding: a.job_binding ?? null, provision_job_id: a.provision_job_id ?? null, lease_status: a.lease_status ?? null,
|
|
3084
|
+
closed: a.closed === true, superseded: a.superseded === true, close_reason: a.close_reason ?? null,
|
|
3085
|
+
cleanup_status: a.cleanup_status ?? null, cleanup_evidence: a.cleanup_evidence ?? null,
|
|
3086
|
+
}), opts, 0);
|
|
3087
|
+
}
|
|
3002
3088
|
const got = await rpc.call("get_stack_lease", { lease_id: leaseId });
|
|
3003
3089
|
if (!got.ok) return got.auth
|
|
3004
3090
|
? emitResult(buildReceipt({ command: "status", outcome: "error", lease_id: leaseId, error: got.error }), opts, EXIT.AUTH)
|
|
@@ -3345,9 +3431,19 @@ export async function cmdDone(leaseId, opts, {
|
|
|
3345
3431
|
// provable (cleanup_verified strictly false), ends at once; a raw frame keeps waiting.
|
|
3346
3432
|
(s, p) => (s === "reap_failed" && p?.cleanup_verified === true) || (s === "reaped" && p?.cleanup_verified === false),
|
|
3347
3433
|
{ timeoutSec: opts.reapTimeout ?? 300, auth: rpc.auth });
|
|
3348
|
-
|
|
3434
|
+
// BOT-2780: a claim bound only to an identified admission request is released only when
|
|
3435
|
+
// the owner-scoped admission read proves that request named this exact lease.
|
|
3436
|
+
let requestBinding = null;
|
|
3437
|
+
if (opts.admissionRequestId) {
|
|
3438
|
+
const admission = await call("get_stack_lease_admission", { request_id: opts.admissionRequestId });
|
|
3439
|
+
requestBinding = admission?.ok && admission.data?.success === true && admission.data.superseded !== true && admission.data.lease_id === leaseId
|
|
3440
|
+
? { admission_request_id: opts.admissionRequestId } : false;
|
|
3441
|
+
}
|
|
3442
|
+
if (requestBinding === false) {
|
|
3443
|
+
portBlock = { released: false, reason: `admission ${opts.admissionRequestId} does not name lease ${leaseId}; local claim retained` };
|
|
3444
|
+
} else if (verified.woke && verified.cleanupEvidence) {
|
|
3349
3445
|
state = "reaped";
|
|
3350
|
-
portBlock = await recoverClaim({ ...current.data, lease_id: leaseId }, verified.cleanupEvidence);
|
|
3446
|
+
portBlock = await recoverClaim({ ...current.data, lease_id: leaseId }, { ...verified.cleanupEvidence, ...(requestBinding ?? {}) });
|
|
3351
3447
|
} else portBlock = { released: false, reason: "accepted cleanup proof is unavailable; local claim retained" };
|
|
3352
3448
|
}
|
|
3353
3449
|
} catch (error) {
|
|
@@ -3583,7 +3679,10 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3583
3679
|
incidentBox.outcomeChannel = { get auth() { return rpc.auth; }, callTool: adapters.callTool ?? callToolJson };
|
|
3584
3680
|
const api = adapters.api ?? {
|
|
3585
3681
|
request: (args) => rpc.call("request_stack_lease", args),
|
|
3586
|
-
|
|
3682
|
+
// BOT-2780 (beta requester origin): BOT-2499's identified admission, never a fallback.
|
|
3683
|
+
requestIdentified: (args) => rpc.call("request_stack_lease_identified", args),
|
|
3684
|
+
getAdmission: (args) => rpc.call("get_stack_lease_admission", args),
|
|
3685
|
+
get: (leaseId, { signal } = {}) => rpc.call("get_stack_lease", { lease_id: leaseId }, { signal }),
|
|
3587
3686
|
// BOT-1803: a heartbeat carries the measured client use when there is one.
|
|
3588
3687
|
touch: (leaseId, usage) => rpc.call("touch_stack_lease", { lease_id: leaseId, ...(usage ? { usage } : {}) }),
|
|
3589
3688
|
// BOT-1803: `{ idle: true }` idles an ACTIVE lease instead of reaping it.
|
|
@@ -3651,6 +3750,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3651
3750
|
const buildBundle = adapters.buildBundle ?? buildStackBundle;
|
|
3652
3751
|
const uploadBundle = adapters.uploadBundle ?? uploadStackBundle;
|
|
3653
3752
|
const openTunnel = adapters.openTunnel ?? ((args) => establishSshTunnel(args));
|
|
3753
|
+
const openCallback = adapters.openNativeCallbackTransport ?? openNativeCallbackTransport;
|
|
3654
3754
|
const clock = adapters.clock ?? { setInterval, clearInterval, setTimeout, clearTimeout };
|
|
3655
3755
|
const signals = adapters.signals ?? process;
|
|
3656
3756
|
const provisionTimeout = opts.provisionTimeout ?? opts.timeout;
|
|
@@ -3673,8 +3773,13 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3673
3773
|
let activeWaitAbort = null;
|
|
3674
3774
|
// BOT-2142: the remote lease's forwards, and whether they died under a running child.
|
|
3675
3775
|
let tunnel = null;
|
|
3776
|
+
let nativeCallback = null;
|
|
3777
|
+
let callbackLost = null;
|
|
3778
|
+
let callbackCleanupError = null;
|
|
3676
3779
|
let forwardAttestation = null;
|
|
3677
3780
|
let tunnelLost = null;
|
|
3781
|
+
// BOT-2780: the requester origin closed under a live run (SSH loss or forward failure).
|
|
3782
|
+
let originLost = null;
|
|
3678
3783
|
const placement = { mode: null, hostKey: null, requesterHost: null, transport: "none", forwards: null, bundle: null, placementReason: null, considered: null };
|
|
3679
3784
|
incidentBox.execution = placement;
|
|
3680
3785
|
// BOT-1803: idle-lease bookkeeping for the receipt.
|
|
@@ -3685,8 +3790,12 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3685
3790
|
let retainBlock = false;
|
|
3686
3791
|
// BOT-1803: a heartbeat touch found the lease preempted under the running child.
|
|
3687
3792
|
let preempted = null;
|
|
3793
|
+
// BOT-2780: the requester listener never outlives the run, its tunnel or a reap request.
|
|
3794
|
+
const closeOrigin = async () => { try { await execution?.requesterOrigin?.close(); } catch { /* closed either way */ } };
|
|
3688
3795
|
const closeTunnel = async () => {
|
|
3796
|
+
await closeOrigin();
|
|
3689
3797
|
try {
|
|
3798
|
+
if (nativeCallback) { const route = nativeCallback; nativeCallback = null; try { await route.close(); } catch { callbackCleanupError = "CALLBACK_CLEANUP_UNCERTAIN"; } }
|
|
3690
3799
|
if (forwardAttestation) {
|
|
3691
3800
|
const owner = forwardAttestation;
|
|
3692
3801
|
forwardAttestation = null;
|
|
@@ -3711,6 +3820,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3711
3820
|
if (!leaseId) return Promise.resolve({ ok: true, state: null });
|
|
3712
3821
|
if (cleanupPromise) return cleanupPromise;
|
|
3713
3822
|
cleanupPromise = (async () => {
|
|
3823
|
+
await closeOrigin();
|
|
3714
3824
|
if (!leaseId) return cleanupResult = { ok: true, state: null };
|
|
3715
3825
|
const cleanupAbort = new AbortController();
|
|
3716
3826
|
const cleanupTimer = clock.setTimeout(() => cleanupAbort.abort("reap-timeout"), opts.reapTimeout * 1000);
|
|
@@ -3799,6 +3909,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3799
3909
|
};
|
|
3800
3910
|
|
|
3801
3911
|
const stopChild = (signal = "SIGTERM", forcedCode = null) => {
|
|
3912
|
+
void closeOrigin();
|
|
3802
3913
|
if (!child || childResult) return;
|
|
3803
3914
|
forcedStopCode ??= forcedCode;
|
|
3804
3915
|
childGroupSignal(child, signal);
|
|
@@ -3834,14 +3945,21 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3834
3945
|
const slot = deriveSlot(opts);
|
|
3835
3946
|
// BOT-1798: never "." — materialize/validate an ISOLATED stack dir first.
|
|
3836
3947
|
try {
|
|
3837
|
-
execution = await resolveExecution({ ...opts, slot
|
|
3948
|
+
execution = await resolveExecution({ ...opts, slot,
|
|
3949
|
+
// BOT-2780: one identified admission key per invocation, held through replays.
|
|
3950
|
+
...(requesterOriginEnabled(opts) ? { admissionRequestId: randomUUID(), requesterMachineId: hardwareUuid } : {}),
|
|
3951
|
+
});
|
|
3952
|
+
if (requesterOriginEnabled(opts) && (!execution.requesterOrigin || !execution.admissionRequestId || !execution.originContext)) {
|
|
3953
|
+
await closeOrigin();
|
|
3954
|
+
const error = new Error("requester origin context unavailable"); error.code = "REQUESTER_ORIGIN_CONTEXT_REFUSED"; throw error;
|
|
3955
|
+
}
|
|
3838
3956
|
} catch (error) {
|
|
3839
3957
|
// A slot whose stack another live batch owns is a coordination refusal (the
|
|
3840
3958
|
// same class as the server's duplicate-lease rejection), not bad arguments.
|
|
3841
3959
|
const exitCode = error?.code === "stack_slot_busy" ? EXIT.BACKEND : EXIT.INVALID;
|
|
3842
3960
|
// BOT-2490: retain listener/cooldown refusals in the existing public code field;
|
|
3843
3961
|
// keep the exit mapping and unrelated local-error semantics unchanged.
|
|
3844
|
-
const code = ["stack_port_census_failed", "stack_port_cooldown_failed"].includes(error?.code) ? error.code : null;
|
|
3962
|
+
const code = ["stack_port_census_failed", "stack_port_cooldown_failed"].includes(error?.code) || String(error?.code ?? "").startsWith("REQUESTER_ORIGIN_") ? error.code : null;
|
|
3845
3963
|
return { exitCode, outcome: "error", ...(code ? { code } : {}), error: String(error?.message ?? error) };
|
|
3846
3964
|
}
|
|
3847
3965
|
// BOT-1798 (Codex #928 R13 P2): the queue and provisioning waits can outlast the
|
|
@@ -3920,6 +4038,8 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3920
4038
|
"an idle stack on this slot will be re-provisioned rather than reused.");
|
|
3921
4039
|
}
|
|
3922
4040
|
idleLease.stackDigest = stackDigest;
|
|
4041
|
+
// Runtime-first rollout: only explicit beta callers activate the new transport.
|
|
4042
|
+
const requireNativeCallback = opts.callbackPort != null && opts.executionRing === "beta" && executionRequest !== "local";
|
|
3923
4043
|
const leaseArgs = {
|
|
3924
4044
|
slot, host_key: opts.host || undefined, repo: opts.repo, ticket_id: opts.ticket,
|
|
3925
4045
|
ticket_url: opts.ticketUrl || undefined, pr_id: opts.prId || undefined, pr_url: opts.prUrl || undefined,
|
|
@@ -3934,8 +4054,85 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3934
4054
|
require_stack_digest: true,
|
|
3935
4055
|
// BOT-2269: what the bundle carries beyond the defaults (paths, sha256, key NAMES).
|
|
3936
4056
|
...(bundle?.included?.length ? { bundle_included: bundle.included } : {}),
|
|
3937
|
-
...(opts.callbackPort != null ? { callback_port: opts.callbackPort } : {}),
|
|
4057
|
+
...(opts.callbackPort != null ? { callback_port: opts.callbackPort, ...(requireNativeCallback ? { callback_transport: NATIVE_CALLBACK_TRANSPORT } : {}) } : {}),
|
|
3938
4058
|
};
|
|
4059
|
+
// BOT-2780: the identified admission key and a digest of the immutable origin context.
|
|
4060
|
+
// The intent (including the bundle digest that names the reserved origin) is fingerprinted
|
|
4061
|
+
// server-side; replays of this exact request can never create a second lease.
|
|
4062
|
+
let identifiedVerdict = null;
|
|
4063
|
+
let requestIdentified = null;
|
|
4064
|
+
if (execution.admissionRequestId) {
|
|
4065
|
+
if (!stackDigest || !bundle?.sha256) {
|
|
4066
|
+
return { exitCode: EXIT.INTERNAL, outcome: "error", code: "REQUESTER_ORIGIN_DIGEST_UNAVAILABLE", error: "the requester-origin bundle digest could not be computed; no lease was requested" };
|
|
4067
|
+
}
|
|
4068
|
+
leaseArgs.request_id = execution.admissionRequestId;
|
|
4069
|
+
// Durable request-to-context association before any request can commit (recovery key).
|
|
4070
|
+
try { execution.recordAdmission?.(leaseArgs.request_id); } catch (error) {
|
|
4071
|
+
return { exitCode: EXIT.INTERNAL, outcome: "error", code: "REQUESTER_ORIGIN_CONTEXT_REFUSED", error: `could not record the admission request on this stack's claim (${error?.message ?? error}); no lease was requested` };
|
|
4072
|
+
}
|
|
4073
|
+
leaseArgs.origin_identity_sha256 = createHash("sha256")
|
|
4074
|
+
.update(JSON.stringify({ context: execution.originContext, origin: execution.requesterOrigin.origin })).digest("hex");
|
|
4075
|
+
const replayDelays = adapters.admissionReplayDelaysMs ?? ADMISSION_REPLAY_DELAYS_MS;
|
|
4076
|
+
let replays = 0;
|
|
4077
|
+
let mayHaveCommitted = false;
|
|
4078
|
+
let ownedAck = null;
|
|
4079
|
+
const settle = (code) => {
|
|
4080
|
+
identifiedVerdict = ownedAck ? { kind: "owned", code, ack: ownedAck } : { kind: "uncertain", code, ack: null };
|
|
4081
|
+
};
|
|
4082
|
+
// Returns a legacy-shaped answer: the decorated delivery for a ready ACK, a definite
|
|
4083
|
+
// refusal, or (with identifiedVerdict set) the original answer for the caller to retain.
|
|
4084
|
+
requestIdentified = async () => {
|
|
4085
|
+
for (;;) {
|
|
4086
|
+
let res;
|
|
4087
|
+
try { res = await api.requestIdentified(leaseArgs); } catch (error) { res = { ok: false, status: null, error: String(error?.message ?? error) }; }
|
|
4088
|
+
let replayCode = null;
|
|
4089
|
+
if (!res?.ok) {
|
|
4090
|
+
// A rejected credential, or a readable JSON-RPC protocol error over HTTP 200, was refused
|
|
4091
|
+
// before the tool handler ran: a definite refusal on a first answer. An unreadable body
|
|
4092
|
+
// (`invalid_json`) may follow a commit, so it is replayed like a lost response.
|
|
4093
|
+
if (!mayHaveCommitted && (res?.auth || (res?.status === 200 && res.error !== "invalid_json"))) return res;
|
|
4094
|
+
replayCode = "ADMISSION_UNCERTAIN";
|
|
4095
|
+
} else if (!mayHaveCommitted && typeof res.data?.raw === "string" && /^Unknown tool: request_stack_lease_identified\b/.test(res.data.raw)) {
|
|
4096
|
+
// An older backend without BOT-2499's tool: nothing was admitted, and no fallback.
|
|
4097
|
+
identifiedVerdict = { kind: "unsupported" };
|
|
4098
|
+
return res;
|
|
4099
|
+
} else if (res.data?.success !== true) {
|
|
4100
|
+
const code = typeof res.data?.code === "string" ? res.data.code : "ADMISSION_UNCERTAIN";
|
|
4101
|
+
if (ADMISSION_COMMITTED_UNCERTAIN.has(code)) { settle(code); return res; }
|
|
4102
|
+
if (ADMISSION_REPLAYABLE.has(code)) replayCode = code;
|
|
4103
|
+
// After an answer that may have committed, a later refusal is not proof of absence.
|
|
4104
|
+
else if (mayHaveCommitted) { settle("ADMISSION_UNCERTAIN"); return res; }
|
|
4105
|
+
// A definite, effect-free refusal never carries cleanup authority.
|
|
4106
|
+
else return { ...res, data: { success: false, code, ...(res.data?.message ? { message: res.data.message } : {}),
|
|
4107
|
+
...(res.data?.phase ? { phase: res.data.phase } : {}), ...(res.data?.wait ? { wait: res.data.wait } : {}) } };
|
|
4108
|
+
} else {
|
|
4109
|
+
const ack = identifiedAdmissionAck(res.data, leaseArgs.request_id);
|
|
4110
|
+
// A malformed or foreign ACK is untrusted: never cleanup authority, never adopted.
|
|
4111
|
+
if (!ack || ack.admission_kind !== "created" || (ownedAck && ack.lease_id !== ownedAck.lease_id)) {
|
|
4112
|
+
mayHaveCommitted = true;
|
|
4113
|
+
if (ack && ownedAck && ack.lease_id !== ownedAck.lease_id) ownedAck = null; // two leases for one key: trust neither
|
|
4114
|
+
settle(!ack || ack.admission_kind === "created" ? "ADMISSION_UNCERTAIN" : "ADMISSION_READOPTION_UNSUPPORTED");
|
|
4115
|
+
return res;
|
|
4116
|
+
}
|
|
4117
|
+
const delivery = identifiedAdmissionDelivery(res.data, ack);
|
|
4118
|
+
if (delivery) return { ...res, data: delivery };
|
|
4119
|
+
if (ack.delivery_status === "ready") { mayHaveCommitted = true; settle("ADMISSION_UNCERTAIN"); return res; }
|
|
4120
|
+
ownedAck = ack;
|
|
4121
|
+
replayCode = "ADMISSION_DELIVERY_PENDING";
|
|
4122
|
+
}
|
|
4123
|
+
mayHaveCommitted = mayHaveCommitted || replayCode !== "HOST_ELIGIBILITY_BUSY";
|
|
4124
|
+
if (replays >= replayDelays.length || receivedSignal) {
|
|
4125
|
+
// Only effect-free busy answers so far: still a definite refusal, nothing to retain.
|
|
4126
|
+
if (!mayHaveCommitted) return { ...res, data: { success: false, code: replayCode } };
|
|
4127
|
+
settle(replayCode);
|
|
4128
|
+
return res;
|
|
4129
|
+
}
|
|
4130
|
+
const delay = replayDelays[replays++];
|
|
4131
|
+
stderrLine(`stack: identified admission ${leaseArgs.request_id} is ${replayCode} — replaying the same request in ${delay}ms`);
|
|
4132
|
+
if (delay > 0) await new Promise((resolve) => { clock.setTimeout(resolve, delay); });
|
|
4133
|
+
}
|
|
4134
|
+
};
|
|
4135
|
+
}
|
|
3939
4136
|
// The retrying caller (api → rpc.call) transparently renews+retries the
|
|
3940
4137
|
// FIRST auth rejection among any of request/get/touch/release.
|
|
3941
4138
|
// BOT-2024: a stale-capacity refusal prints phase-aware guidance; with the opt-in
|
|
@@ -3948,11 +4145,51 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3948
4145
|
let idleRetried = false;
|
|
3949
4146
|
for (let attempt = 0; ; attempt++) {
|
|
3950
4147
|
activeWaitAbort = new AbortController();
|
|
3951
|
-
const admission = await requestWithCapacityWait(() => api.request(leaseArgs), opts, {
|
|
4148
|
+
const admission = await requestWithCapacityWait(() => requestIdentified ? requestIdentified() : api.request(leaseArgs), opts, {
|
|
3952
4149
|
waitCapacity: adapters.waitCapacity ?? defaultWaitCapacity, log: stderrLine, signal: activeWaitAbort.signal,
|
|
3953
4150
|
});
|
|
3954
4151
|
activeWaitAbort = null;
|
|
3955
4152
|
request = admission.req;
|
|
4153
|
+
if (identifiedVerdict) {
|
|
4154
|
+
const verdict = identifiedVerdict;
|
|
4155
|
+
if (verdict.kind === "unsupported") {
|
|
4156
|
+
return { exitCode: EXIT.BACKEND, outcome: "error", code: "IDENTIFIED_ADMISSION_UNSUPPORTED",
|
|
4157
|
+
error: "this backend has no identified stack-lease admission (request_stack_lease_identified); a beta requester-origin run never falls back to the legacy allocator" };
|
|
4158
|
+
}
|
|
4159
|
+
if (verdict.kind === "owned") {
|
|
4160
|
+
// The exact ACK proves this lease is ours, but it never became deliverable: reap it.
|
|
4161
|
+
leaseId = verdict.ack.lease_id;
|
|
4162
|
+
leaseProvisioned = true;
|
|
4163
|
+
execution.bindLease?.(leaseId);
|
|
4164
|
+
const cleanup = await requestCleanup();
|
|
4165
|
+
return { exitCode: receivedSignal ? SIGNAL_EXIT[receivedSignal] : cleanup.ok ? EXIT.BACKEND : EXIT.CLEANUP_FAILED,
|
|
4166
|
+
outcome: receivedSignal ? "interrupted" : "error", code: verdict.code, leaseId, cleanup,
|
|
4167
|
+
error: `identified admission ${leaseArgs.request_id} committed lease ${leaseId} but its delivery never became ready; it was released` };
|
|
4168
|
+
}
|
|
4169
|
+
// Possibly committed with no trustworthy ACK. One owner-scoped admission read can still
|
|
4170
|
+
// prove which lease this exact request created: then it is ours to reap normally.
|
|
4171
|
+
let observed = null;
|
|
4172
|
+
try { observed = await api.getAdmission?.({ request_id: leaseArgs.request_id }); } catch { observed = null; }
|
|
4173
|
+
const named = observed?.ok && observed.data?.success === true && observed.data.superseded !== true &&
|
|
4174
|
+
typeof observed.data.lease_id === "string" && UUID_RE.test(observed.data.lease_id) ? observed.data.lease_id : null;
|
|
4175
|
+
if (named) {
|
|
4176
|
+
leaseId = named;
|
|
4177
|
+
leaseProvisioned = true;
|
|
4178
|
+
execution.bindLease?.(leaseId);
|
|
4179
|
+
const cleanup = await requestCleanup();
|
|
4180
|
+
return { exitCode: receivedSignal ? SIGNAL_EXIT[receivedSignal] : cleanup.ok ? EXIT.BACKEND : EXIT.CLEANUP_FAILED,
|
|
4181
|
+
outcome: receivedSignal ? "interrupted" : "error", code: verdict.code, leaseId, cleanup,
|
|
4182
|
+
error: `identified admission ${leaseArgs.request_id} was uncertain; its owner-scoped read names lease ${leaseId}, which was released` };
|
|
4183
|
+
}
|
|
4184
|
+
// Still unknown: keep the port claim (bound to this request) and hand back the key.
|
|
4185
|
+
retainBlock = true;
|
|
4186
|
+
leaseProvisioned = true;
|
|
4187
|
+
return { exitCode: receivedSignal ? SIGNAL_EXIT[receivedSignal] : EXIT.BACKEND, outcome: receivedSignal ? "interrupted" : "error", code: verdict.code,
|
|
4188
|
+
allocationRecovery: identifiedAdmissionRecovery(leaseArgs, null, verdict.ack),
|
|
4189
|
+
error: `identified admission ${leaseArgs.request_id} may have committed; its port claim is retained. Read it with ` +
|
|
4190
|
+
`\`bb stack status --admission-request-id ${leaseArgs.request_id}\`, then release any lease it names with ` +
|
|
4191
|
+
`\`bb stack done <lease-id> --reap --release-port-claim --admission-request-id ${leaseArgs.request_id}\`` };
|
|
4192
|
+
}
|
|
3956
4193
|
|
|
3957
4194
|
// Interrupted DURING the capacity park: the refused request created no lease, so
|
|
3958
4195
|
// there is nothing to release. (A signal during a request that DID admit falls
|
|
@@ -4139,6 +4376,10 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
4139
4376
|
placement.mode = request.data.execution_mode ?? null;
|
|
4140
4377
|
placement.hostKey = request.data.host_key ?? null;
|
|
4141
4378
|
placement.requesterHost = request.data.requester_host ?? null;
|
|
4379
|
+
if (remoteLease && requireNativeCallback && request.data.callback_transport !== NATIVE_CALLBACK_TRANSPORT) {
|
|
4380
|
+
const cleanup = await requestCleanup();
|
|
4381
|
+
return { exitCode: cleanup.ok ? EXIT.BACKEND : EXIT.CLEANUP_FAILED, outcome: "error", code: "HOST_CALLBACK_TRANSPORT_UNSUPPORTED", leaseId, error: "backend did not acknowledge required remote callback transport", cleanup };
|
|
4382
|
+
}
|
|
4142
4383
|
// BOT-2207: why the server placed (or queued) it there, and every host it weighed.
|
|
4143
4384
|
placement.placementReason = request.data.placement_reason ?? null;
|
|
4144
4385
|
placement.considered = Array.isArray(request.data.considered) ? request.data.considered : null;
|
|
@@ -4267,7 +4508,9 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
4267
4508
|
// BOT-2400 (item 1): the park deadline never gives the queue place up. Hold the lease
|
|
4268
4509
|
// for the owner's next run (which re-attaches to it) and exit PARK_DEADLINE; only
|
|
4269
4510
|
// `stack cancel` / `stack done` or the owner's death releases it.
|
|
4270
|
-
|
|
4511
|
+
// BOT-2780: a requester-origin run cannot be resumed by a later invocation (its stack
|
|
4512
|
+
// names this CLI's listener, which closes now), so it releases instead of holding a place.
|
|
4513
|
+
const queueHold = holdPlace && !execution.requesterOrigin ? await holdPlace(leaseId, HOLD_ON_TIMEOUT_MAX_SEC) : null;
|
|
4271
4514
|
if (queueHold) {
|
|
4272
4515
|
// A refused/lost hold reply is not an owner cancellation: it may have
|
|
4273
4516
|
// committed before the reply was lost, or the queued lease may have
|
|
@@ -4406,11 +4649,30 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
4406
4649
|
// BOT-2142: forward every stack port over ONE ssh and hand the child the local ports.
|
|
4407
4650
|
sshTarget = current.data.connection?.ssh_target ?? current.data.ssh_target ?? sshTarget;
|
|
4408
4651
|
try {
|
|
4409
|
-
tunnel = await openTunnel({ connection: current.data.connection, sshTarget, hostKey: placement.hostKey, callbackPort:
|
|
4652
|
+
tunnel = await openTunnel({ connection: current.data.connection, sshTarget, hostKey: placement.hostKey, callbackPort: requireNativeCallback ? null : opts.callbackPort,
|
|
4653
|
+
...(requireNativeCallback ? { reserveLocalPorts: [opts.callbackPort] } : {}) });
|
|
4410
4654
|
} catch (error) {
|
|
4411
4655
|
const cleanup = await requestCleanup();
|
|
4412
4656
|
return { exitCode: EXIT.BACKEND, outcome: "error", code: error?.code ?? "REMOTE_UNREACHABLE", leaseId, error: String(error?.message ?? error), cleanup };
|
|
4413
4657
|
}
|
|
4658
|
+
if (requireNativeCallback) {
|
|
4659
|
+
try {
|
|
4660
|
+
nativeCallback = await openCallback({ current: current.data, leaseId, hostKey: placement.hostKey, callbackPort: opts.callbackPort, requesterMachineId: machineUuidFn(), sshTarget,
|
|
4661
|
+
readCurrent: async (signal) => { let observed; try { observed = await api.get(leaseId, { signal }); } catch (error) { throw Object.assign(new Error(`current callback status unreadable (${error?.message ?? error})`), { transient: true }); } if (!observed?.ok) {
|
|
4662
|
+
// Only an unanswered read is transient (transport, 5xx, 408/429, unreadable body). A 401/403
|
|
4663
|
+
// that survived the auth refresh or a typed JSON-RPC refusal is a definitive answer.
|
|
4664
|
+
const transient = !observed?.auth && (observed?.status == null || observed.status >= 500 || observed.status === 408 || observed.status === 429 || observed.error === "invalid_json");
|
|
4665
|
+
throw Object.assign(new Error(transient ? "current callback status unreadable" : "current callback authorization refused"), transient ? { transient: true } : { code: "CALLBACK_AUTHORIZATION_LOST" });
|
|
4666
|
+
}
|
|
4667
|
+
if (observed.data?.success !== true) throw Object.assign(new Error("current callback authorization unavailable"), { code: "CALLBACK_AUTHORIZATION_LOST" });
|
|
4668
|
+
return observed.data; },
|
|
4669
|
+
});
|
|
4670
|
+
nativeCallback.onExit((exit) => { if (childResult) return; callbackLost = exit; fencing = true; stopChild("SIGTERM", EXIT.LEASE_FAILED); });
|
|
4671
|
+
} catch (error) {
|
|
4672
|
+
await closeTunnel(); const cleanup = await requestCleanup();
|
|
4673
|
+
return { exitCode: cleanup.ok && !callbackCleanupError && error?.code !== "CALLBACK_CLEANUP_UNCERTAIN" ? EXIT.BACKEND : EXIT.CLEANUP_FAILED, outcome: "error", code: error?.code ?? "CALLBACK_NOT_READY", leaseId, error: String(error?.message ?? "callback transport unavailable"), cleanup };
|
|
4674
|
+
}
|
|
4675
|
+
}
|
|
4414
4676
|
placement.transport = "ssh-forward";
|
|
4415
4677
|
placement.forwards = tunnel.forwards.map(({ name, remote, local }) => ({ name, remote, local }));
|
|
4416
4678
|
for (const f of tunnel.forwards.filter((x) => x.remapped)) stderrLine(`stack: forward ${f.name} remapped: local ${f.local} → ${placement.hostKey ?? "remote"} ${f.remote}`);
|
|
@@ -4425,6 +4687,28 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
4425
4687
|
leaseId, sshTarget, forwards: placement.forwards,
|
|
4426
4688
|
isLive: () => Boolean(tunnel && !tunnelLost && !receivedSignal && tunnel.isLive()),
|
|
4427
4689
|
});
|
|
4690
|
+
if (execution.requesterOrigin) {
|
|
4691
|
+
// BOT-2780: bind the reserved listener to exactly this lease's API forward. It is not
|
|
4692
|
+
// another SSH forward: connection.api_url/ports/attestation stay physical-only.
|
|
4693
|
+
try {
|
|
4694
|
+
execution.requesterOrigin.onLost((loss) => { originLost = loss; if (!receivedSignal && !childResult) stopChild("SIGTERM", EXIT.LEASE_FAILED); });
|
|
4695
|
+
execution.requesterOrigin.activate({
|
|
4696
|
+
context: execution.originContext, leaseId, projectId: current.data.connection.project_id, sshTarget,
|
|
4697
|
+
remoteConnection: current.data.connection, forwardedConnection: tunnel.connection, forwards: placement.forwards,
|
|
4698
|
+
isLive: () => Boolean(tunnel && forwardAttestation && !tunnelLost && !receivedSignal && tunnel.isLive()),
|
|
4699
|
+
watchLoss: (callback) => tunnel.onExit(callback),
|
|
4700
|
+
});
|
|
4701
|
+
} catch (error) {
|
|
4702
|
+
await closeTunnel();
|
|
4703
|
+
const cleanup = await requestCleanup();
|
|
4704
|
+
return { exitCode: cleanup.ok ? EXIT.BACKEND : EXIT.CLEANUP_FAILED, outcome: "error", code: error?.code ?? "REQUESTER_ORIGIN_FORWARD_REFUSED", leaseId, error: String(error?.message ?? error), cleanup };
|
|
4705
|
+
}
|
|
4706
|
+
placement.requesterOrigin = execution.requesterOrigin.origin;
|
|
4707
|
+
stderrLine(`stack: requester origin ${execution.requesterOrigin.origin} → ${placement.hostKey ?? "remote"} API (generated auth links use it unchanged)`);
|
|
4708
|
+
childConnection = { ...childConnection, requester_origin: execution.requesterOrigin.origin };
|
|
4709
|
+
}
|
|
4710
|
+
} else if (execution.requesterOrigin) {
|
|
4711
|
+
await closeOrigin();
|
|
4428
4712
|
}
|
|
4429
4713
|
connectionFile = await writeConnection(safeConnectionPath(opts.connectionFile, leaseId), childConnection);
|
|
4430
4714
|
if (receivedSignal) {
|
|
@@ -4461,6 +4745,16 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
4461
4745
|
// there would fail on a missing package.json AFTER the stack was provisioned.
|
|
4462
4746
|
// The child therefore always runs in the (canonicalised) worktree root.
|
|
4463
4747
|
const childCwd = execution.worktreeRoot;
|
|
4748
|
+
if (!tunnelLost && execution.requesterOrigin && servedRemote && (originLost || !execution.requesterOrigin.isLive())) {
|
|
4749
|
+
await closeTunnel();
|
|
4750
|
+
const cleanup = await requestCleanup();
|
|
4751
|
+
return { exitCode: cleanup.ok ? EXIT.LEASE_FAILED : EXIT.CLEANUP_FAILED, outcome: "error", code: originLost?.code ?? "REQUESTER_ORIGIN_TUNNEL_LOST", leaseId, cleanup,
|
|
4752
|
+
error: "the requester origin closed before the child started; the lease was released" };
|
|
4753
|
+
}
|
|
4754
|
+
if (callbackLost || nativeCallback && !nativeCallback.isLive()) {
|
|
4755
|
+
await closeTunnel(); const cleanup = await requestCleanup();
|
|
4756
|
+
return { exitCode: cleanup.ok && !callbackCleanupError ? EXIT.LEASE_FAILED : EXIT.CLEANUP_FAILED, outcome: "error", code: "CALLBACK_TRANSPORT_LOST", leaseId, error: "callback transport closed before child start", cleanup };
|
|
4757
|
+
}
|
|
4464
4758
|
if (tunnelLost) {
|
|
4465
4759
|
// The forwards died between being verified and the child starting.
|
|
4466
4760
|
await closeTunnel();
|
|
@@ -4512,7 +4806,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
4512
4806
|
// BOT-1803: a run that ended on its own (the child exited, success or not) leaves its
|
|
4513
4807
|
// lease IDLE for the next run on this slot — no reap wait. `--reap`, a signal, the
|
|
4514
4808
|
// hard TTL, a fenced/preempted heartbeat and a lost tunnel keep the eager signed reap.
|
|
4515
|
-
const idleOnEnd = !opts.reap && !receivedSignal && !fencing && !tunnelLost && forcedStopCode == null;
|
|
4809
|
+
const idleOnEnd = !opts.reap && !receivedSignal && !fencing && !tunnelLost && !callbackCleanupError && forcedStopCode == null;
|
|
4516
4810
|
const cleanup = await requestCleanup({ idle: idleOnEnd });
|
|
4517
4811
|
if (cleanup?.idle) {
|
|
4518
4812
|
// The stack is still running: keep its directory and host port block reserved well
|
|
@@ -4521,6 +4815,13 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
4521
4815
|
try { execution.renew?.({ idleTtlSecs: LEASED_STACK_IDLE_RETENTION_SECS, idleRetained: true }); } catch { /* advisory */ }
|
|
4522
4816
|
stderrLine(`stack: lease ${leaseId} is idle${cleanup.idle_since ? ` since ${cleanup.idle_since}` : ""} — its stack keeps running for the next run on this slot (\`bb stack done ${leaseId} --reap\` tears it down)`);
|
|
4523
4817
|
}
|
|
4818
|
+
if (originLost && !tunnelLost && !receivedSignal) {
|
|
4819
|
+
return { exitCode: cleanup.ok ? EXIT.LEASE_FAILED : EXIT.CLEANUP_FAILED, outcome: "failed", code: originLost.code, leaseId, childExitCode: childResult.code, cleanup, fenced: true,
|
|
4820
|
+
error: "the requester origin closed while the child was running, so the child was stopped and the lease released" };
|
|
4821
|
+
}
|
|
4822
|
+
if (callbackLost && !receivedSignal) {
|
|
4823
|
+
return { exitCode: cleanup.ok && !callbackCleanupError ? EXIT.LEASE_FAILED : EXIT.CLEANUP_FAILED, outcome: "failed", code: callbackLost.reason === "callback_authorization_lost" ? "CALLBACK_AUTHORIZATION_LOST" : "CALLBACK_TRANSPORT_LOST", leaseId, childExitCode: childResult.code, cleanup, fenced: true, error: "callback transport authorization or runtime context was lost; child stopped" };
|
|
4824
|
+
}
|
|
4524
4825
|
if (tunnelLost && !receivedSignal) {
|
|
4525
4826
|
return tunnelLostResult(tunnelLost, { leaseId, cleanup, childExitCode: childResult.code, fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {}) });
|
|
4526
4827
|
}
|
|
@@ -4532,8 +4833,8 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
4532
4833
|
...(parkIncident ? { incident: parkIncident } : {}),
|
|
4533
4834
|
};
|
|
4534
4835
|
}
|
|
4535
|
-
const exitCode = receivedSignal ? SIGNAL_EXIT[receivedSignal] : forcedStopCode ?? (childResult.code !== 0 ? childResult.code : !cleanup.ok ? EXIT.CLEANUP_FAILED : 0);
|
|
4536
|
-
return { exitCode, outcome: exitCode === 0 ? "completed" : "failed", leaseId, childExitCode: childResult.code, cleanup, fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {}) };
|
|
4836
|
+
const exitCode = receivedSignal ? SIGNAL_EXIT[receivedSignal] : callbackCleanupError ? EXIT.CLEANUP_FAILED : forcedStopCode ?? (childResult.code !== 0 ? childResult.code : callbackCleanupError || !cleanup.ok ? EXIT.CLEANUP_FAILED : 0);
|
|
4837
|
+
return { exitCode, outcome: exitCode === 0 ? "completed" : "failed", leaseId, childExitCode: childResult.code, cleanup, ...(callbackCleanupError ? { code: callbackCleanupError } : {}), fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {}) };
|
|
4537
4838
|
} catch (error) {
|
|
4538
4839
|
const cleanup = await requestCleanup();
|
|
4539
4840
|
// BOT-1750 (Codex R10 P2): a thrown auth/self-heal failure (e.g. a refresh that
|
|
@@ -4617,6 +4918,9 @@ export function runReceiptFields(result) {
|
|
|
4617
4918
|
execution_host_key: result.execution?.hostKey ?? null,
|
|
4618
4919
|
requester_host: result.execution?.requesterHost ?? null,
|
|
4619
4920
|
transport: result.execution?.transport ?? "none",
|
|
4921
|
+
// BOT-2780: the reserved requester origin generated auth links name (never a credential).
|
|
4922
|
+
...(result.execution?.requesterOrigin ? { requester_origin: result.execution.requesterOrigin } : {}),
|
|
4923
|
+
...(result.allocationRecovery ? { allocation_recovery: result.allocationRecovery } : {}),
|
|
4620
4924
|
...(result.execution?.forwards ? { forwards: result.execution.forwards } : {}),
|
|
4621
4925
|
...(result.execution?.bundle ? { bundle: result.execution.bundle } : {}),
|
|
4622
4926
|
// BOT-2269: the opted-in files that shipped in that bundle — paths, sha256, key names.
|