@botbuddy/cli 1.41.1 → 1.42.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/docker-hygiene.mjs +98 -77
- package/src/stack.mjs +73 -3
package/package.json
CHANGED
package/src/docker-hygiene.mjs
CHANGED
|
@@ -12,9 +12,11 @@ import { VERSION } from "./version.mjs";
|
|
|
12
12
|
import { machineUuid } from "./machine-id.mjs";
|
|
13
13
|
|
|
14
14
|
export const SCHEMA_VERSION = 1;
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
export const
|
|
15
|
+
// BOT-2206: preflight applies policy v3's network term (BOT-2202). Docker's built-in
|
|
16
|
+
// default-address-pools are 172.17–31.0.0/16 as /16s (15) + 192.168.0.0/16 as /20s (16).
|
|
17
|
+
export const DEFAULT_NETWORK_LIMIT = 31;
|
|
18
|
+
// The relay's accepted upper bound for `network_limit` (Helper DockerAddressPools).
|
|
19
|
+
const MAX_NETWORK_LIMIT = 1_000_000;
|
|
18
20
|
|
|
19
21
|
// BOT-1681 / BOT-1630 Step 3 — explicit Docker-engine allowlist. The org cut the
|
|
20
22
|
// dev fleet back from OrbStack to Docker Desktop (OrbStack Helper heap runaway,
|
|
@@ -93,7 +95,8 @@ COMMANDS
|
|
|
93
95
|
hygiene Inventory stopped Supabase containers and empty
|
|
94
96
|
supabase_network_* bridge networks. This is a dry-run unless
|
|
95
97
|
--apply is explicitly present.
|
|
96
|
-
preflight Read-only
|
|
98
|
+
preflight Read-only check that the Docker daemon has a network for one more
|
|
99
|
+
managed stack (the server's admission network term).
|
|
97
100
|
|
|
98
101
|
REQUIRED SELECTOR
|
|
99
102
|
--context <name> Explicit Docker context, e.g. orbstack
|
|
@@ -114,10 +117,12 @@ HYGIENE OPTIONS
|
|
|
114
117
|
Without it, no mutation.
|
|
115
118
|
--json Emit exactly one compact machine-readable receipt.
|
|
116
119
|
|
|
117
|
-
PREFLIGHT
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
120
|
+
PREFLIGHT
|
|
121
|
+
Counts user-defined bridge networks against the daemon's network limit, read
|
|
122
|
+
from docker info DefaultAddressPools (absent or malformed: Docker's default
|
|
123
|
+
${DEFAULT_NETWORK_LIMIT}). The default bridge holds one subnet, so a start is
|
|
124
|
+
admitted while bridges + 1 <= limit - 1; it warns when the start would take the
|
|
125
|
+
last network and refuses (exit 2) when none is left. Endpoints are not scored.
|
|
121
126
|
--json Emit exactly one compact machine-readable receipt.
|
|
122
127
|
|
|
123
128
|
Engine allowlist (preflight): OrbStack and Docker Desktop are admitted by
|
|
@@ -169,15 +174,6 @@ const BUILTIN_NETWORKS = new Set(["bridge", "host", "none"]);
|
|
|
169
174
|
// project's resources before its first container reaches running state.
|
|
170
175
|
const ACTIVE_STATES = new Set(["created", "running", "paused", "restarting", "removing"]);
|
|
171
176
|
|
|
172
|
-
function positiveInteger(value, option, errors, { allowZero = false } = {}) {
|
|
173
|
-
const parsed = Number(value);
|
|
174
|
-
if (!Number.isInteger(parsed) || parsed < (allowZero ? 0 : 1)) {
|
|
175
|
-
errors.push(`${option} must be ${allowZero ? "a non-negative" : "a positive"} integer`);
|
|
176
|
-
return null;
|
|
177
|
-
}
|
|
178
|
-
return parsed;
|
|
179
|
-
}
|
|
180
|
-
|
|
181
177
|
export function parseDockerArgs(argv) {
|
|
182
178
|
const [command, ...rest] = argv;
|
|
183
179
|
const errors = [];
|
|
@@ -189,9 +185,6 @@ export function parseDockerArgs(argv) {
|
|
|
189
185
|
candidateIds: [],
|
|
190
186
|
apply: false,
|
|
191
187
|
json: false,
|
|
192
|
-
projectedEndpoints: DEFAULT_PROJECTED_ENDPOINTS,
|
|
193
|
-
warnPressure: DEFAULT_WARN_PRESSURE,
|
|
194
|
-
failPressure: DEFAULT_FAIL_PRESSURE,
|
|
195
188
|
};
|
|
196
189
|
|
|
197
190
|
const needValue = (option, index) => {
|
|
@@ -246,30 +239,12 @@ export function parseDockerArgs(argv) {
|
|
|
246
239
|
}
|
|
247
240
|
break;
|
|
248
241
|
}
|
|
249
|
-
case "--projected-endpoints":
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
}
|
|
255
|
-
break;
|
|
256
|
-
}
|
|
257
|
-
case "--warn-pressure": {
|
|
258
|
-
const value = needValue(arg, i);
|
|
259
|
-
if (value !== null) {
|
|
260
|
-
opts.warnPressure = positiveInteger(value, arg, errors);
|
|
261
|
-
i += 1;
|
|
262
|
-
}
|
|
242
|
+
case "--projected-endpoints":
|
|
243
|
+
case "--warn-pressure":
|
|
244
|
+
case "--fail-pressure":
|
|
245
|
+
errors.push(`${arg} was retired (BOT-2206): preflight now checks Docker networks against the daemon's network limit, like server admission; endpoints are not scored`);
|
|
246
|
+
if (rest[i + 1] !== undefined && !rest[i + 1].startsWith("--")) i += 1;
|
|
263
247
|
break;
|
|
264
|
-
}
|
|
265
|
-
case "--fail-pressure": {
|
|
266
|
-
const value = needValue(arg, i);
|
|
267
|
-
if (value !== null) {
|
|
268
|
-
opts.failPressure = positiveInteger(value, arg, errors);
|
|
269
|
-
i += 1;
|
|
270
|
-
}
|
|
271
|
-
break;
|
|
272
|
-
}
|
|
273
248
|
case "--apply": opts.apply = true; break;
|
|
274
249
|
case "--json": opts.json = true; break;
|
|
275
250
|
case "--help":
|
|
@@ -306,9 +281,6 @@ export function parseDockerArgs(argv) {
|
|
|
306
281
|
if (!opts.lockSlot || opts.lockSlot === "default") errors.push("--apply requires a non-default --lock-slot from the owning stack lifecycle");
|
|
307
282
|
if (opts.candidateIds.length === 0) errors.push("--apply requires at least one exact --candidate-id from the reviewed dry run");
|
|
308
283
|
}
|
|
309
|
-
if (opts.warnPressure !== null && opts.failPressure !== null && opts.failPressure <= opts.warnPressure) {
|
|
310
|
-
errors.push("--fail-pressure must be greater than --warn-pressure");
|
|
311
|
-
}
|
|
312
284
|
}
|
|
313
285
|
|
|
314
286
|
return { command, opts, errors };
|
|
@@ -765,6 +737,7 @@ function validateDockerEngine(runDocker, opts, allowlist = DOCKER_ENGINE_ALLOWLI
|
|
|
765
737
|
|
|
766
738
|
return {
|
|
767
739
|
selector,
|
|
740
|
+
networkLimit: networkLimitFromDockerInfo(info),
|
|
768
741
|
receipt: {
|
|
769
742
|
requested: opts.context ? { type: "context", value: opts.context } : { type: "endpoint", value: opts.endpoint },
|
|
770
743
|
resolved_endpoint: resolvedEndpoint,
|
|
@@ -780,29 +753,73 @@ function validateDockerEngine(runDocker, opts, allowlist = DOCKER_ENGINE_ALLOWLI
|
|
|
780
753
|
};
|
|
781
754
|
}
|
|
782
755
|
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
756
|
+
function isIPv4(address) {
|
|
757
|
+
const octets = address.split(".");
|
|
758
|
+
return octets.length === 4 && octets.every((octet) => /^(0|[1-9][0-9]{0,2})$/.test(octet) && Number(octet) <= 255);
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
/**
|
|
762
|
+
* BOT-2206: how many bridge networks the daemon can allocate — the Helper's
|
|
763
|
+
* DockerAddressPools.networkLimit (0.2.26), so preflight and the census agree. Each
|
|
764
|
+
* IPv4 `DefaultAddressPools` entry yields 2^(Size − prefix) subnets; IPv6 pools never
|
|
765
|
+
* raise it. Absent pools (Docker's built-ins report null) or ANY malformed pool is
|
|
766
|
+
* unknown → Docker's default 31, never a partial sum and never a refusal.
|
|
767
|
+
*/
|
|
768
|
+
export function networkLimitFromDockerInfo(info) {
|
|
769
|
+
const fallback = { network_limit: DEFAULT_NETWORK_LIMIT, network_limit_source: "default" };
|
|
770
|
+
const pools = info?.DefaultAddressPools;
|
|
771
|
+
if (!Array.isArray(pools) || pools.length === 0) return fallback;
|
|
772
|
+
let total = 0;
|
|
773
|
+
let sawIPv4 = false;
|
|
774
|
+
for (const pool of pools) {
|
|
775
|
+
const parts = typeof pool?.Base === "string" ? pool.Base.split("/") : [];
|
|
776
|
+
const size = pool?.Size;
|
|
777
|
+
if (parts.length !== 2 || !/^[0-9]+$/.test(parts[1]) || !Number.isInteger(size)) return fallback;
|
|
778
|
+
const prefix = Number(parts[1]);
|
|
779
|
+
if (parts[0].includes(":")) {
|
|
780
|
+
if (prefix > 128 || size < prefix || size > 128) return fallback;
|
|
781
|
+
continue;
|
|
782
|
+
}
|
|
783
|
+
if (!isIPv4(parts[0]) || prefix > 32 || size < prefix || size > 32) return fallback;
|
|
784
|
+
sawIPv4 = true;
|
|
785
|
+
const bits = size - prefix;
|
|
786
|
+
total = Math.min(MAX_NETWORK_LIMIT, total + (bits >= 20 ? MAX_NETWORK_LIMIT : 2 ** bits));
|
|
787
|
+
}
|
|
788
|
+
return sawIPv4 && total > 0 ? { network_limit: total, network_limit_source: "daemon" } : fallback;
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
/**
|
|
792
|
+
* BOT-2206: policy v3's network term (BOT-2202), evaluated locally. The default
|
|
793
|
+
* `bridge` network holds one subnet, so one more start is admitted while
|
|
794
|
+
* `user bridges + 1 <= network_limit - 1`. It warns when that start takes the last
|
|
795
|
+
* usable network. current/projected_pressure_units keep the server's v3 meaning.
|
|
796
|
+
*/
|
|
797
|
+
export function evaluateNetworks({
|
|
798
|
+
userBridgeNetworks,
|
|
799
|
+
networkLimit = DEFAULT_NETWORK_LIMIT,
|
|
800
|
+
networkLimitSource = "default",
|
|
789
801
|
}) {
|
|
790
|
-
const current = Number(
|
|
791
|
-
const
|
|
792
|
-
const
|
|
802
|
+
const current = Number(userBridgeNetworks);
|
|
803
|
+
const limit = Number(networkLimit);
|
|
804
|
+
const usable = limit - 1;
|
|
805
|
+
const projected = current + 1;
|
|
806
|
+
const status = projected > usable ? "fail" : projected === usable ? "warn" : "ok";
|
|
793
807
|
return {
|
|
794
|
-
metric: "
|
|
795
|
-
|
|
808
|
+
metric: "docker_networks",
|
|
809
|
+
policy: "host_docker_pressure_v1",
|
|
810
|
+
policy_version: 3,
|
|
811
|
+
formula: "user_bridge_networks + 1 <= network_limit - 1",
|
|
812
|
+
network_limit: limit,
|
|
813
|
+
network_limit_source: networkLimitSource,
|
|
814
|
+
// Every network the daemon holds, including the default bridge: `networks n/limit`.
|
|
815
|
+
networks_in_use: current + 1,
|
|
816
|
+
network_free_slots: Math.max(0, usable - current),
|
|
796
817
|
current_pressure_units: current,
|
|
797
818
|
projected_pressure_units: projected,
|
|
798
|
-
projected_endpoints: Number(projectedEndpoints),
|
|
799
|
-
warn_pressure: Number(warn),
|
|
800
|
-
fail_pressure: Number(fail),
|
|
801
819
|
status,
|
|
802
|
-
// BOT-1412: a host-wide
|
|
803
|
-
//
|
|
804
|
-
//
|
|
805
|
-
// eventual capacity signal, rather than retrying Docker locally.
|
|
820
|
+
// BOT-1412: a host-wide refusal is not a per-stack `supabase_local` lock wait.
|
|
821
|
+
// Emit the relay's capacity condition so callers park on the global policy and
|
|
822
|
+
// receive the eventual capacity signal, rather than retrying Docker locally.
|
|
806
823
|
...(status === "fail" ? {
|
|
807
824
|
wait: {
|
|
808
825
|
kind: "global_capacity",
|
|
@@ -810,10 +827,15 @@ export function evaluatePressure({
|
|
|
810
827
|
command: "bb-wait 'capacity:policy=host_docker_pressure_v1,slots>=1'",
|
|
811
828
|
},
|
|
812
829
|
} : {}),
|
|
813
|
-
note: "Conservative pressure proxy; OrbStack does not expose a kernel interface ceiling through the Docker API.",
|
|
814
830
|
};
|
|
815
831
|
}
|
|
816
832
|
|
|
833
|
+
/** `networks 31/31 (default limit)` — the preflight's one-line network verdict. */
|
|
834
|
+
export function formatNetworks(pressure) {
|
|
835
|
+
const source = pressure.network_limit_source === "daemon" ? "daemon address pools" : "Docker default limit";
|
|
836
|
+
return `networks ${pressure.networks_in_use}/${pressure.network_limit} (${source})`;
|
|
837
|
+
}
|
|
838
|
+
|
|
817
839
|
function baseReceipt(command, opts, now) {
|
|
818
840
|
return {
|
|
819
841
|
schema_version: SCHEMA_VERSION,
|
|
@@ -858,7 +880,7 @@ export function buildReliabilityTelemetry(receipt, { dedupeKey } = {}) {
|
|
|
858
880
|
}
|
|
859
881
|
const ghostInventory = (receipt.inventory_errors?.length ?? 0) > 0;
|
|
860
882
|
// A successful target check is not a capacity measurement. If discovery
|
|
861
|
-
// fails before
|
|
883
|
+
// fails before evaluateNetworks produces a decision, suppress the generic
|
|
862
884
|
// observation so the evidence view cannot count a zero-filled false pass.
|
|
863
885
|
// Classified daemon and persistent-inventory faults remain reportable.
|
|
864
886
|
if (receipt.command === "preflight" && !receipt.pressure && !unavailable && !ghostInventory) {
|
|
@@ -1108,13 +1130,12 @@ export function runDockerWorkflow(argv, {
|
|
|
1108
1130
|
}
|
|
1109
1131
|
|
|
1110
1132
|
if (parsed.command === "preflight") {
|
|
1111
|
-
receipt.pressure =
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
warn: parsed.opts.warnPressure,
|
|
1116
|
-
fail: parsed.opts.failPressure,
|
|
1133
|
+
receipt.pressure = evaluateNetworks({
|
|
1134
|
+
userBridgeNetworks: inventory.counts.user_bridge_networks,
|
|
1135
|
+
networkLimit: validated.networkLimit.network_limit,
|
|
1136
|
+
networkLimitSource: validated.networkLimit.network_limit_source,
|
|
1117
1137
|
});
|
|
1138
|
+
const networks = formatNetworks(receipt.pressure);
|
|
1118
1139
|
// BOT-1681: dry-run hygiene is the OrbStack-only remedy. On Docker Desktop it
|
|
1119
1140
|
// refuses, so the pressure warning/refusal must not tell operators to run it;
|
|
1120
1141
|
// point them at the engine-aware recommendation instead.
|
|
@@ -1122,13 +1143,13 @@ export function runDockerWorkflow(argv, {
|
|
|
1122
1143
|
if (receipt.pressure.status === "warn") {
|
|
1123
1144
|
receipt.outcome = "warn";
|
|
1124
1145
|
receipt.warnings.push(pressureEngine === "orbstack"
|
|
1125
|
-
?
|
|
1126
|
-
:
|
|
1146
|
+
? `This start takes the last free Docker network (${networks}); run dry-run hygiene before starting another stack.`
|
|
1147
|
+
: `This start takes the last free Docker network (${networks}); reduce Docker load (see recommendation) before starting another stack.`);
|
|
1127
1148
|
} else if (receipt.pressure.status === "fail") {
|
|
1128
1149
|
receipt.outcome = "refused";
|
|
1129
1150
|
receipt.errors.push(pressureEngine === "orbstack"
|
|
1130
|
-
?
|
|
1131
|
-
:
|
|
1151
|
+
? `No Docker network is free for another stack (${networks}); cleanup or release an owning stack before start.`
|
|
1152
|
+
: `No Docker network is free for another stack (${networks}); release an owning stack (see recommendation) before start.`);
|
|
1132
1153
|
return { exitCode: EXIT.PRESSURE, receipt, json: parsed.opts.json };
|
|
1133
1154
|
}
|
|
1134
1155
|
return { exitCode: EXIT.OK, receipt, json: parsed.opts.json };
|
|
@@ -1275,7 +1296,7 @@ export function formatHumanReceipt(receipt) {
|
|
|
1275
1296
|
}
|
|
1276
1297
|
if (!receipt.inventory_errors?.length) lines.push(" none");
|
|
1277
1298
|
if (receipt.pressure) {
|
|
1278
|
-
lines.push(`
|
|
1299
|
+
lines.push(`Capacity: ${formatNetworks(receipt.pressure)} — ${receipt.pressure.status}; attached endpoints ${receipt.inventory?.attached_endpoints ?? "?"} (not scored)`);
|
|
1279
1300
|
}
|
|
1280
1301
|
lines.push(`Reliability telemetry: ${receipt.telemetry?.status || "not_attempted"}`);
|
|
1281
1302
|
for (const warning of receipt.warnings) lines.push(`Warning: ${warning}`);
|
package/src/stack.mjs
CHANGED
|
@@ -174,6 +174,11 @@ ${bold("REMOTE LEASES")} (BOT-2142, stack run)
|
|
|
174
174
|
The receipt carries execution_mode (local|remote), execution_host_key, requester_host,
|
|
175
175
|
transport (none|ssh-forward), and for a remote lease forwards [{name, remote, local}] + bundle.
|
|
176
176
|
The child also gets BOTBUDDY_STACK_EXECUTION_MODE and BOTBUDDY_STACK_TRANSPORT.
|
|
177
|
+
Requeue (BOT-2207): under auto, a lease that QUEUES on this machine uploads its bundle at once;
|
|
178
|
+
every minute the control plane re-offers it to this owner's other hosts and it may move there
|
|
179
|
+
(placement_reason moved_remote_on_requeue), and run then forwards to the new host. While queued,
|
|
180
|
+
run prints one \`considered: <host> — <verdict>\` line per host weighed (e.g. "census stale 131 s");
|
|
181
|
+
the receipt carries placement_reason and considered[].
|
|
177
182
|
5 REMOTE_UNREACHABLE the host has no ssh_target (an operator must set it), or
|
|
178
183
|
\`ssh <target>\` could not open the forwards (BatchMode: key auth only)
|
|
179
184
|
5 REMOTE_UNSUPPORTED --remote, but the control plane placed the lease locally (pre-remote server)
|
|
@@ -455,6 +460,35 @@ export function describeQueuePosition(d) {
|
|
|
455
460
|
* starved lease names its holders instead of only its position. Other tenants' leases are
|
|
456
461
|
* tenant + state + since; unleased stacks are 5-character prefixes (the server redacts).
|
|
457
462
|
*/
|
|
463
|
+
// BOT-2207: one human line per host the control plane weighed for an \`auto\` lease
|
|
464
|
+
// (request_stack_lease / get_stack_lease \`considered[]\`), e.g. "considered: mac-mini — census
|
|
465
|
+
// stale 131 s". An unknown verdict code is printed as-is.
|
|
466
|
+
const CONSIDERED_VERDICT_TEXT = {
|
|
467
|
+
ADMITS: (c) => `admits (${c.free_slots ?? 0} free)`,
|
|
468
|
+
HOST_CAPACITY_INVENTORY_STALE: (c) => (Number.isFinite(c.snapshot_age_seconds) ? `census stale ${c.snapshot_age_seconds} s` : "no census yet"),
|
|
469
|
+
HOST_CAPACITY_SOURCE_STALE: () => "capacity beacon stale",
|
|
470
|
+
HOST_CAPACITY_WAIT: () => "Docker pressure full",
|
|
471
|
+
HOST_CAPACITY_BUDGET_FULL: () => "slot budget full",
|
|
472
|
+
HOST_CAPACITY_INVENTORY_UNKNOWN: () => "unknown Docker resources in its census",
|
|
473
|
+
TENANT_RESERVATION_FULL: () => "tenant reservation full",
|
|
474
|
+
SLOT_HELD: () => "this slot is already leased there",
|
|
475
|
+
HOST_HELPER_OUTDATED: () => "Helper predates remote leases (protocol < 4)",
|
|
476
|
+
HOST_NO_SSH_TARGET: () => "no ssh_target set",
|
|
477
|
+
HOST_INACTIVE: () => "host inactive",
|
|
478
|
+
};
|
|
479
|
+
|
|
480
|
+
export function describeConsidered(considered) {
|
|
481
|
+
if (!Array.isArray(considered)) return [];
|
|
482
|
+
return considered
|
|
483
|
+
.filter((c) => c && typeof c === "object" && typeof c.host_key === "string")
|
|
484
|
+
.map((c) => {
|
|
485
|
+
const name = (typeof c.display_name === "string" && c.display_name) || c.host_key;
|
|
486
|
+
const where = c.location === "local" ? " (this machine)" : "";
|
|
487
|
+
const text = CONSIDERED_VERDICT_TEXT[c.verdict_code]?.(c) ?? String(c.verdict_code ?? "unknown");
|
|
488
|
+
return `considered: ${name}${where} — ${text}`;
|
|
489
|
+
});
|
|
490
|
+
}
|
|
491
|
+
|
|
458
492
|
export function describeUnitLedger(ledger) {
|
|
459
493
|
if (!ledger || typeof ledger !== "object") return [];
|
|
460
494
|
const cap = ledger.capacity ?? {};
|
|
@@ -2752,7 +2786,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
2752
2786
|
// BOT-2142: the remote lease's forwards, and whether they died under a running child.
|
|
2753
2787
|
let tunnel = null;
|
|
2754
2788
|
let tunnelLost = null;
|
|
2755
|
-
const placement = { mode: null, hostKey: null, requesterHost: null, transport: "none", forwards: null, bundle: null };
|
|
2789
|
+
const placement = { mode: null, hostKey: null, requesterHost: null, transport: "none", forwards: null, bundle: null, placementReason: null, considered: null };
|
|
2756
2790
|
incidentBox.execution = placement;
|
|
2757
2791
|
const closeTunnel = async () => {
|
|
2758
2792
|
if (!tunnel) return;
|
|
@@ -2995,6 +3029,9 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
2995
3029
|
placement.mode = request.data.execution_mode ?? null;
|
|
2996
3030
|
placement.hostKey = request.data.host_key ?? null;
|
|
2997
3031
|
placement.requesterHost = request.data.requester_host ?? null;
|
|
3032
|
+
// BOT-2207: why the server placed (or queued) it there, and every host it weighed.
|
|
3033
|
+
placement.placementReason = request.data.placement_reason ?? null;
|
|
3034
|
+
placement.considered = Array.isArray(request.data.considered) ? request.data.considered : null;
|
|
2998
3035
|
if (executionRequest === "remote" && !remoteLease) {
|
|
2999
3036
|
const cleanup = await requestCleanup();
|
|
3000
3037
|
return {
|
|
@@ -3045,6 +3082,23 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3045
3082
|
}
|
|
3046
3083
|
placement.bundle = { sha256: bundle.sha256, bytes: bundle.size };
|
|
3047
3084
|
stderrLine(`stack: lease ${leaseId} is remote on ${placement.hostKey ?? "?"} — uploaded the ${bundle.size}-byte stack bundle (sha256 ${bundle.sha256.slice(0, 12)}…)`);
|
|
3085
|
+
} else if (bundle && state === "queued" && request.data.bundle_upload_url) {
|
|
3086
|
+
// BOT-2207: the lease queued HERE but may still move: every minute the control plane
|
|
3087
|
+
// re-offers it to the owner's other hosts, and it can only move with its bundle in
|
|
3088
|
+
// storage. Upload now. A failed upload is not fatal — the lease just stays queued here.
|
|
3089
|
+
activeWaitAbort = new AbortController();
|
|
3090
|
+
const uploaded = await uploadBundle(request.data.bundle_upload_url, bundle, { signal: activeWaitAbort.signal });
|
|
3091
|
+
activeWaitAbort = null;
|
|
3092
|
+
if (receivedSignal) {
|
|
3093
|
+
const cleanup = await requestCleanup();
|
|
3094
|
+
return { exitCode: SIGNAL_EXIT[receivedSignal], outcome: "interrupted", leaseId, cleanup };
|
|
3095
|
+
}
|
|
3096
|
+
if (uploaded?.ok) {
|
|
3097
|
+
placement.bundle = { sha256: bundle.sha256, bytes: bundle.size };
|
|
3098
|
+
stderrLine(`stack: lease ${leaseId} queued on ${placement.hostKey ?? "?"} — uploaded the ${bundle.size}-byte stack bundle so it can move to another owned host that admits first`);
|
|
3099
|
+
} else {
|
|
3100
|
+
stderrLine(`${yellow("⚠")} stack: could not upload the stack bundle (${uploaded?.error || "upload failed"}); lease ${leaseId} stays queued on ${placement.hostKey ?? "this host"} and cannot move to another host.`);
|
|
3101
|
+
}
|
|
3048
3102
|
}
|
|
3049
3103
|
if (receivedSignal) {
|
|
3050
3104
|
const cleanup = await requestCleanup();
|
|
@@ -3054,6 +3108,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3054
3108
|
// BOT-2019: say where the lease sits (server dispatch rank, or why it has none) before parking.
|
|
3055
3109
|
stderrLine(`stack: lease ${leaseId} queued — ${describeQueuePosition(request.data)}`);
|
|
3056
3110
|
for (const line of describeUnitLedger(request.data.ledger)) stderrLine(`stack: ${line}`);
|
|
3111
|
+
for (const line of describeConsidered(request.data.considered)) stderrLine(`stack: ${line}`);
|
|
3057
3112
|
activeWaitAbort = new AbortController();
|
|
3058
3113
|
// BOT-2013: every reap state (reaping / reaped / reap_failed) means the queued lease is lost.
|
|
3059
3114
|
// BOT-2099: an auth/availability blip is retried inside the park until its deadline;
|
|
@@ -3123,8 +3178,20 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3123
3178
|
|
|
3124
3179
|
placement.hostKey = current.data.host_key ?? placement.hostKey;
|
|
3125
3180
|
placement.mode = current.data.execution_mode ?? placement.mode ?? (remoteLease ? "remote" : null);
|
|
3181
|
+
placement.placementReason = current.data.placement_reason ?? placement.placementReason;
|
|
3182
|
+
if (Array.isArray(current.data.considered)) placement.considered = current.data.considered;
|
|
3183
|
+
// BOT-2207: a queued auto lease may have been MOVED to another owned host while it waited.
|
|
3184
|
+
// Follow it exactly like a first-time remote placement: forward to the NEW host's ssh_target.
|
|
3185
|
+
const servedRemote = remoteLease || current.data.execution_mode === "remote";
|
|
3186
|
+
if (servedRemote && !remoteLease) {
|
|
3187
|
+
stderrLine(`stack: lease ${leaseId} moved to ${placement.hostKey ?? "another host"} while queued (${placement.placementReason ?? "moved"}) — forwarding to it`);
|
|
3188
|
+
if (opts.callbackPort != null) {
|
|
3189
|
+
const cleanup = await requestCleanup();
|
|
3190
|
+
return { exitCode: EXIT.INVALID, outcome: "error", code: CALLBACK_PORT_UNSUPPORTED_REMOTE, leaseId, cleanup, error: CALLBACK_PORT_REMOTE_REFUSAL };
|
|
3191
|
+
}
|
|
3192
|
+
}
|
|
3126
3193
|
let childConnection = current.data.connection;
|
|
3127
|
-
if (
|
|
3194
|
+
if (servedRemote) {
|
|
3128
3195
|
// BOT-2142: forward every stack port over ONE ssh and hand the child the local ports.
|
|
3129
3196
|
sshTarget = current.data.connection?.ssh_target ?? current.data.ssh_target ?? sshTarget;
|
|
3130
3197
|
try {
|
|
@@ -3162,7 +3229,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3162
3229
|
BOTBUDDY_STACK_CONNECTION_FILE: connectionFile,
|
|
3163
3230
|
// BOT-2142: lets a lane's own checks (e.g. the port-owner preflight) know the
|
|
3164
3231
|
// listeners are SSH forwards to another host, not this machine's Docker.
|
|
3165
|
-
BOTBUDDY_STACK_EXECUTION_MODE:
|
|
3232
|
+
BOTBUDDY_STACK_EXECUTION_MODE: servedRemote ? "remote" : (placement.mode ?? "local"),
|
|
3166
3233
|
BOTBUDDY_STACK_TRANSPORT: placement.transport,
|
|
3167
3234
|
...(leasedTestConfig ? {
|
|
3168
3235
|
BB_INTEGRATION_ENV_FILE: leasedTestConfig.envFile,
|
|
@@ -3262,6 +3329,9 @@ async function cmdRun(opts, childArgv) {
|
|
|
3262
3329
|
transport: result.execution?.transport ?? "none",
|
|
3263
3330
|
...(result.execution?.forwards ? { forwards: result.execution.forwards } : {}),
|
|
3264
3331
|
...(result.execution?.bundle ? { bundle: result.execution.bundle } : {}),
|
|
3332
|
+
// BOT-2207: why it ran where it did (e.g. moved_remote_on_requeue) and every host weighed.
|
|
3333
|
+
...(result.execution?.placementReason ? { placement_reason: result.execution.placementReason } : {}),
|
|
3334
|
+
...(result.execution?.considered ? { considered: result.execution.considered } : {}),
|
|
3265
3335
|
fenced: result.fenced ?? false, error: result.error ?? result.cleanup?.error ?? null,
|
|
3266
3336
|
// BOT-2099: a platform auth/availability blip during the park is reported as an
|
|
3267
3337
|
// incident (reason, window, attempts, recovered) — never as the lease's outcome.
|