@botbuddy/cli 1.64.30 → 1.64.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
1
  {
2
- "release_version": "1.64.30",
3
- "source_commit": "cea90b48937b714dae31540ddf0559943b9d87a2",
4
- "source_sha256": "0ed4fcd3314212258923964ddc437034105fc914901e9bc055668ef650878cd2",
2
+ "release_version": "1.64.32",
3
+ "source_commit": "170ead0f849e941e0076215bba062cb9f7424327",
4
+ "source_sha256": "589ea191f9c830eef71bd941dcd32ea4ba14972537805f12d07bf91231fae7ef",
5
5
  "target": "darwin-arm64",
6
- "sha256": "5b8dff2916c1bd2210c91631d93bbe376a15c4f1c3d1720d5b908cd6e2745385"
6
+ "sha256": "bf4295c5e7d4e6da1019d5c553bc8ab978d36fbbd26acb8c53831c8233c91e0c"
7
7
  }
@@ -1,7 +1,7 @@
1
1
  {
2
- "release_version": "1.64.30",
3
- "source_commit": "cea90b48937b714dae31540ddf0559943b9d87a2",
4
- "source_sha256": "0ed4fcd3314212258923964ddc437034105fc914901e9bc055668ef650878cd2",
2
+ "release_version": "1.64.32",
3
+ "source_commit": "170ead0f849e941e0076215bba062cb9f7424327",
4
+ "source_sha256": "589ea191f9c830eef71bd941dcd32ea4ba14972537805f12d07bf91231fae7ef",
5
5
  "target": "linux-x64",
6
- "sha256": "711103b10bce9107321a8d9ad0e1e6d5cc86158d91234dcebfa952b7f0819f71"
6
+ "sha256": "52d1d1da89ea20d675dbe07a598a2553a52102bcbfe549171005724310e594d7"
7
7
  }
@@ -1,21 +1,21 @@
1
1
  {
2
2
  "schema_version": 1,
3
- "release_version": "1.64.30",
4
- "source_commit": "cea90b48937b714dae31540ddf0559943b9d87a2",
5
- "source_sha256": "0ed4fcd3314212258923964ddc437034105fc914901e9bc055668ef650878cd2",
3
+ "release_version": "1.64.32",
4
+ "source_commit": "170ead0f849e941e0076215bba062cb9f7424327",
5
+ "source_sha256": "589ea191f9c830eef71bd941dcd32ea4ba14972537805f12d07bf91231fae7ef",
6
6
  "targets": {
7
7
  "darwin-arm64": {
8
8
  "goos": "darwin",
9
9
  "goarch": "arm64",
10
10
  "binary": "daemon/bin/darwin-arm64/botbuddy-daemon",
11
- "sha256": "5b8dff2916c1bd2210c91631d93bbe376a15c4f1c3d1720d5b908cd6e2745385",
11
+ "sha256": "bf4295c5e7d4e6da1019d5c553bc8ab978d36fbbd26acb8c53831c8233c91e0c",
12
12
  "mode": "0755"
13
13
  },
14
14
  "linux-x64": {
15
15
  "goos": "linux",
16
16
  "goarch": "amd64",
17
17
  "binary": "daemon/bin/linux-x64/botbuddy-daemon",
18
- "sha256": "711103b10bce9107321a8d9ad0e1e6d5cc86158d91234dcebfa952b7f0819f71",
18
+ "sha256": "52d1d1da89ea20d675dbe07a598a2553a52102bcbfe549171005724310e594d7",
19
19
  "mode": "0755"
20
20
  }
21
21
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@botbuddy/cli",
3
- "version": "1.64.30",
3
+ "version": "1.64.32",
4
4
  "description": "BotBuddy \u2014 Swarm coordination CLI for multi-agent workflows",
5
5
  "type": "module",
6
6
  "bin": {
@@ -0,0 +1,298 @@
1
+ // BOT-3109 — did THIS machine lose its own network while a `bb stack run` failed?
2
+ //
3
+ // A requester-side drop (a cable pulled, Wi-Fi handing over, sleep) ends a remote run
4
+ // with TUNNEL_LOST, CALLBACK_TRANSPORT_LOST or LEASE_HEARTBEAT_UNANSWERED, exactly like
5
+ // a BotBuddy or stack-host outage. This module tells them apart from the requester's own
6
+ // side, and only when it can prove it:
7
+ //
8
+ // - a small sampler reads the primary route every SAMPLE_MS while the run lives
9
+ // (macOS: `route -n get default` + `ipconfig getifaddr`; Linux: `ip -4 route show default`),
10
+ // and when the primary moves, whether the old interface lost its address;
11
+ // - at a candidate failure, `classify()` reads once more and, on macOS, reads configd's own
12
+ // record of interface detaches in the window before the failure.
13
+ //
14
+ // The answer is `changed` (with evidence), `unchanged` or `undecided`. Only `changed`
15
+ // changes what a run reports; the caller keeps today's code for the other two, so an
16
+ // unsupported OS, a missing tool or a probe error never guesses `requester`.
17
+
18
+ import { execFile } from "node:child_process";
19
+
20
+ export const REQUESTER_NETWORK_LOST_CODE = "REQUESTER_NETWORK_LOST";
21
+ /** The terminal codes a requester's own network loss can produce (BOT-3109). */
22
+ export const REQUESTER_NETWORK_CANDIDATE_CODES = Object.freeze(["TUNNEL_LOST", "CALLBACK_TRANSPORT_LOST", "LEASE_HEARTBEAT_UNANSWERED"]);
23
+ /** `=off` disables the probe (the CLI test suites set it; nothing else should). */
24
+ export const REQUESTER_NETWORK_PROBE_ENV = "BOTBUDDY_REQUESTER_NETWORK_PROBE";
25
+ export const REQUESTER_NETWORK_SAMPLE_MS = 15_000;
26
+ /** The control-plane deadlines are 180 s, so a loss that ended a run began at most this long before. */
27
+ export const REQUESTER_NETWORK_LOOKBACK_MS = 300_000;
28
+ /** Evidence just after the failure still counts (the sampler may see the outage a beat late). */
29
+ export const REQUESTER_NETWORK_FAILURE_SLACK_MS = 20_000;
30
+ const PROBE_TIMEOUT_MS = 3000;
31
+ const LOG_TIMEOUT_MS = 6000;
32
+ const MAX_SAMPLES = 64;
33
+
34
+ const IFACE_RE = /^[A-Za-z0-9_.:@-]{1,32}$/;
35
+
36
+ // Every probe child in flight; `stop()` aborts them so a finished run never waits on a probe.
37
+ const inFlight = new Set();
38
+
39
+ /** The probe runner (exported for its abort test). */
40
+ export function execText(file, args, timeout) {
41
+ return new Promise((resolve) => {
42
+ const ac = new AbortController();
43
+ inFlight.add(ac);
44
+ try {
45
+ execFile(file, args, { encoding: "utf8", timeout, maxBuffer: 1 << 20, signal: ac.signal }, (error, stdout, stderr) => {
46
+ inFlight.delete(ac);
47
+ resolve({
48
+ status: error ? (typeof error.code === "number" ? error.code : -1) : 0,
49
+ stdout: typeof stdout === "string" ? stdout : "", stderr: typeof stderr === "string" ? stderr : "", missing: error?.code === "ENOENT",
50
+ });
51
+ });
52
+ } catch {
53
+ inFlight.delete(ac);
54
+ resolve({ status: -1, stdout: "", missing: true });
55
+ }
56
+ });
57
+ }
58
+
59
+ /** Abort every probe child still running (an aborted read is unreadable, never a loss). */
60
+ export function abortRequesterNetworkProbes() {
61
+ for (const ac of inFlight) ac.abort();
62
+ inFlight.clear();
63
+ }
64
+
65
+ const iface = (value) => (typeof value === "string" && IFACE_RE.test(value) ? value : null);
66
+
67
+ /** `route -n get default` → the primary interface, or null when there is no default route. */
68
+ export function parseMacosDefaultRoute(stdout) {
69
+ return iface(/^\s*interface:\s*(\S+)\s*$/m.exec(stdout ?? "")?.[1]);
70
+ }
71
+
72
+ /** `ip -4 route show default` → { interface, gateway } of the first default route, or null. */
73
+ export function parseLinuxDefaultRoute(stdout) {
74
+ const line = String(stdout ?? "").split("\n").find((l) => /^default\b/.test(l.trim()));
75
+ if (!line) return null;
76
+ const dev = iface(/\bdev\s+(\S+)/.exec(line)?.[1]);
77
+ if (!dev) return null;
78
+ return { interface: dev, gateway: /\bvia\s+(\S+)/.exec(line)?.[1] ?? null };
79
+ }
80
+
81
+ /**
82
+ * One reading of this machine's primary network identity. `{ ok: false }` when it cannot be
83
+ * read (unsupported OS, missing tool, timeout); `{ ok: true, interface: null }` when there is
84
+ * provably no primary route — which is itself the loss.
85
+ */
86
+ export async function readPrimaryNetwork({ platform = process.platform, run = execText } = {}) {
87
+ try {
88
+ if (platform === "darwin") {
89
+ const route = await run("/sbin/route", ["-n", "get", "default"], PROBE_TIMEOUT_MS);
90
+ if (route.missing) return { ok: false };
91
+ const primary = parseMacosDefaultRoute(route.stdout);
92
+ // route prints "not in table" (and exits non-zero) when there is no default route; anything else is unreadable.
93
+ if (!primary) return /not in table/i.test(`${route.stdout}\n${route.stderr ?? ""}`) ? { ok: true, interface: null, address: null } : { ok: false };
94
+ const addr = await run("/usr/sbin/ipconfig", ["getifaddr", primary], PROBE_TIMEOUT_MS);
95
+ if (addr.missing) return { ok: false };
96
+ const address = /^\d{1,3}(\.\d{1,3}){3}$/.test(addr.stdout.trim()) ? addr.stdout.trim() : null;
97
+ return { ok: true, interface: primary, address };
98
+ }
99
+ if (platform === "linux") {
100
+ const route = await run("ip", ["-4", "route", "show", "default"], PROBE_TIMEOUT_MS);
101
+ if (route.missing || route.status !== 0) return { ok: false };
102
+ const parsed = parseLinuxDefaultRoute(route.stdout);
103
+ return parsed ? { ok: true, interface: parsed.interface, address: parsed.gateway } : { ok: true, interface: null, address: null };
104
+ }
105
+ } catch { /* an unreadable probe is undecided, never a loss */ }
106
+ return { ok: false };
107
+ }
108
+
109
+ /**
110
+ * Whether `name` still has an IPv4 address right now. `{ ok: true, address: null }` is proof it
111
+ * lost it (macOS `ipconfig getifaddr` exits 1 with no output for an interface without one, or
112
+ * one that detached; Linux `ip -4 -o addr show dev` prints nothing). Anything else is unreadable.
113
+ */
114
+ export async function readInterfaceAddress(name, { platform = process.platform, run = execText } = {}) {
115
+ if (!iface(name)) return { ok: false };
116
+ try {
117
+ if (platform === "darwin") {
118
+ const out = await run("/usr/sbin/ipconfig", ["getifaddr", name], PROBE_TIMEOUT_MS);
119
+ if (out.missing) return { ok: false };
120
+ const text = out.stdout.trim();
121
+ if (out.status === 0 && /^\d{1,3}(\.\d{1,3}){3}$/.test(text)) return { ok: true, address: text };
122
+ return out.status === 1 && text === "" ? { ok: true, address: null } : { ok: false };
123
+ }
124
+ if (platform === "linux") {
125
+ const out = await run("ip", ["-4", "-o", "addr", "show", "dev", name], PROBE_TIMEOUT_MS);
126
+ if (out.missing) return { ok: false };
127
+ // A detached device is an error ("does not exist"): it has no address either.
128
+ if (out.status !== 0) return /does not exist/i.test(out.stderr ?? "") ? { ok: true, address: null } : { ok: false };
129
+ const inet = /\binet\s+(\d{1,3}(?:\.\d{1,3}){3})/.exec(out.stdout)?.[1] ?? null;
130
+ return { ok: true, address: inet };
131
+ }
132
+ } catch { /* unreadable */ }
133
+ return { ok: false };
134
+ }
135
+
136
+ /** Parse configd's timestamp ("2026-11-09 09:19:32.634512+0200"), which carries its own offset. */
137
+ function parseLogTimestamp(value) {
138
+ const m = /^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})(\.\d{1,3})?\d*([+-]\d{2}):?(\d{2})$/.exec(String(value ?? "").trim());
139
+ if (!m) return NaN;
140
+ return Date.parse(`${m[1]}T${m[2]}${m[3] ?? ""}${m[4]}:${m[5]}`);
141
+ }
142
+
143
+ /**
144
+ * configd's `log show --style ndjson` lines → interface-detach events. Only a detach is evidence:
145
+ * a "network changed" line also fires when an interface is merely added (docking while Wi-Fi
146
+ * stays up), so it never counts on its own. A line without a zoned timestamp is skipped.
147
+ */
148
+ export function parseConfigdLog(stdout) {
149
+ const events = [];
150
+ for (const line of String(stdout ?? "").split("\n")) {
151
+ if (!line.startsWith("{")) continue;
152
+ let entry;
153
+ try { entry = JSON.parse(line); } catch { continue; }
154
+ const atMs = parseLogTimestamp(entry?.timestamp);
155
+ if (!Number.isFinite(atMs)) continue;
156
+ const detached = /Process interface detaching:\s*(\S+)/.exec(String(entry?.eventMessage ?? ""));
157
+ if (detached && iface(detached[1])) events.push({ atMs, kind: "interface_detached", interface: detached[1] });
158
+ }
159
+ return events;
160
+ }
161
+
162
+ /** macOS only: configd's interface detaches in `[sinceMs, untilMs]`. `{ ok: false }` when unreadable. */
163
+ export async function readConfigdNetworkEvents({ sinceMs, untilMs = Infinity, nowMs = Date.now(), run = execText } = {}) {
164
+ const secs = Math.max(1, Math.ceil((nowMs - sinceMs) / 1000) + 5);
165
+ try {
166
+ const out = await run("/usr/bin/log", ["show", "--style", "ndjson", "--last", `${secs}s`, "--predicate",
167
+ 'process == "configd" AND eventMessage CONTAINS "interface detaching"'], LOG_TIMEOUT_MS);
168
+ if (out.missing || out.status !== 0) return { ok: false };
169
+ return { ok: true, events: parseConfigdLog(out.stdout).filter((e) => e.atMs >= sinceMs && e.atMs <= untilMs) };
170
+ } catch {
171
+ return { ok: false };
172
+ }
173
+ }
174
+
175
+ /**
176
+ * Pure decision over the samples (oldest first, each `{ atMs, ok, interface, address,
177
+ * previousLost }`), a fresh reading and, on macOS, configd's detaches. Evidence counts only inside
178
+ * `[sinceMs, untilMs]` (the lookback before the failure, and a short slack after it).
179
+ *
180
+ * Proof of a local loss is one of: the primary route disappeared after a sample had one; the
181
+ * primary moved to another interface AND the old one had lost its address when that was seen;
182
+ * the primary interface kept its name but its address changed; configd saw a primary interface
183
+ * detach. A primary that moved while the old one kept its address (docking, a VPN) is not proof:
184
+ * that run is undecided, so today's code stands.
185
+ */
186
+ export function decideRequesterNetwork({ samples = [], current, log = null, sinceMs, untilMs = Infinity, platform }) {
187
+ const readable = samples.filter((s) => s.ok);
188
+ // The reference: the newest readable sample taken before the window opened, else the oldest in it.
189
+ const before = readable.filter((s) => s.atMs < sinceMs);
190
+ const reference = before.length ? before[before.length - 1] : readable.find((s) => s.atMs <= untilMs) ?? null;
191
+ // A machine that never had an IPv4 primary route (IPv6-only, a policy-routed Linux host) gives
192
+ // the probe nothing to compare: never a loss.
193
+ if (!reference?.interface) return { decision: "undecided" };
194
+ const inWindow = (s) => s.atMs >= sinceMs && s.atMs <= untilMs;
195
+ const chain = [...readable.filter((s) => s !== reference && inWindow(s)), ...(current?.ok && inWindow(current) ? [current] : [])];
196
+ const primaries = new Set([reference.interface]);
197
+ let previous = reference;
198
+ let ambiguous = false;
199
+ for (const sample of chain) {
200
+ if (sample.interface == null) {
201
+ if (previous.interface != null) return { decision: "changed", atMs: sample.atMs, evidence: "no_primary_interface", before: previous.interface, after: null };
202
+ } else if (previous.interface == null) {
203
+ // Back from no route: the loss was already counted above.
204
+ } else if (sample.interface !== previous.interface) {
205
+ if (sample.previousLost === true) return { decision: "changed", atMs: sample.atMs, evidence: "primary_interface_changed", before: previous.interface, after: sample.interface };
206
+ ambiguous = true;
207
+ } else if ((sample.address ?? null) !== (previous.address ?? null) && previous.address != null && sample.address != null) {
208
+ return { decision: "changed", atMs: sample.atMs, evidence: "primary_address_changed", before: previous.interface, after: sample.interface };
209
+ }
210
+ if (sample.interface) primaries.add(sample.interface);
211
+ previous = sample;
212
+ }
213
+ if (log?.ok) {
214
+ const detach = log.events.find((e) => e.kind === "interface_detached" && primaries.has(e.interface) && e.atMs >= sinceMs && e.atMs <= untilMs);
215
+ if (detach) return { decision: "changed", atMs: detach.atMs, evidence: "interface_detached", before: detach.interface, after: current?.interface ?? null };
216
+ }
217
+ // "unchanged" needs a fresh readable answer, no ambiguous move and (on macOS) a readable log.
218
+ const logRead = platform !== "darwin" || log?.ok === true;
219
+ return !ambiguous && current?.ok && logRead ? { decision: "unchanged" } : { decision: "undecided" };
220
+ }
221
+
222
+ /**
223
+ * A watch over one run. `start()` takes the first sample and samples every `sampleMs`;
224
+ * `classify({ failedAtMs })` decides over the lookback window before the failure; `stop()` ends
225
+ * sampling. Never throws.
226
+ */
227
+ export function createRequesterNetworkWatch({
228
+ platform = process.platform, env = process.env, now = Date.now, read: readNetwork = readPrimaryNetwork, readAddress: readOldAddress = readInterfaceAddress,
229
+ readLog = readConfigdNetworkEvents, sampleMs = REQUESTER_NETWORK_SAMPLE_MS, lookbackMs = REQUESTER_NETWORK_LOOKBACK_MS,
230
+ } = {}) {
231
+ const disabled = env?.[REQUESTER_NETWORK_PROBE_ENV] === "off" || (platform !== "darwin" && platform !== "linux");
232
+ const samples = [];
233
+ let timer = null;
234
+ let startedAtMs = null;
235
+ let stopped = false;
236
+ // After stop() no probe child starts, so nothing outlives the run.
237
+ const read = (opts) => (stopped ? Promise.resolve({ ok: false }) : readNetwork(opts));
238
+ const readAddress = (name, opts) => (stopped ? Promise.resolve({ ok: false }) : readOldAddress(name, opts));
239
+ const take = async () => {
240
+ const atMs = now();
241
+ let reading;
242
+ try { reading = await read({ platform }); } catch { reading = { ok: false }; }
243
+ const sample = { atMs, ok: reading?.ok === true, interface: reading?.interface ?? null, address: reading?.address ?? null, previousLost: null };
244
+ const last = [...samples].reverse().find((s) => s.ok);
245
+ // The primary moved: did the old one lose its address (a loss) or is it still up (docking, a VPN)?
246
+ if (sample.ok && sample.interface && last?.interface && last.interface !== sample.interface) {
247
+ try {
248
+ const old = await readAddress(last.interface, { platform });
249
+ sample.previousLost = old?.ok === true ? old.address == null : null;
250
+ } catch { sample.previousLost = null; }
251
+ }
252
+ samples.push(sample);
253
+ if (samples.length > MAX_SAMPLES) samples.shift();
254
+ return sample;
255
+ };
256
+ return {
257
+ disabled,
258
+ start() {
259
+ if (disabled || startedAtMs != null) return;
260
+ startedAtMs = now();
261
+ take().catch(() => {});
262
+ timer = setInterval(() => { take().catch(() => {}); }, sampleMs);
263
+ timer.unref?.();
264
+ },
265
+ stop() {
266
+ stopped = true;
267
+ if (timer) clearInterval(timer);
268
+ timer = null;
269
+ if (!disabled) abortRequesterNetworkProbes();
270
+ },
271
+ async classify({ failedAtMs } = {}) {
272
+ if (disabled || startedAtMs == null) return { decision: "undecided" };
273
+ try {
274
+ const nowMs = now();
275
+ // The failure, not this call (cleanup can take minutes): a change after it is not its cause.
276
+ const failAt = Number.isFinite(failedAtMs) ? Math.min(failedAtMs, nowMs) : nowMs;
277
+ const sinceMs = Math.max(startedAtMs, failAt - lookbackMs);
278
+ const untilMs = failAt + REQUESTER_NETWORK_FAILURE_SLACK_MS;
279
+ const history = samples.slice();
280
+ const current = await take();
281
+ const log = platform === "darwin" ? await readLog({ sinceMs, untilMs, nowMs }) : null;
282
+ return decideRequesterNetwork({ samples: history, current, log, sinceMs, untilMs, platform });
283
+ } catch {
284
+ return { decision: "undecided" };
285
+ }
286
+ },
287
+ };
288
+ }
289
+
290
+ /** The one stderr line a requester-side loss prints (BOT-3109). */
291
+ export function requesterNetworkLostMessage(network, underlyingCode) {
292
+ const at = Number.isFinite(network?.atMs) ? new Date(network.atMs).toISOString() : "an unknown time";
293
+ const how = network?.evidence === "no_primary_interface" ? "had no network route"
294
+ : network?.evidence === "interface_detached" ? `lost its network interface${network.before ? ` ${network.before}` : ""}`
295
+ : `changed its primary network${network?.before && network?.after ? ` (${network.before} → ${network.after})` : ""}`;
296
+ return `stack: this machine ${how} at ${at}, so the run ended (${underlyingCode}). BotBuddy and the stack host were not at fault. ` +
297
+ "Check this machine's connection, then re-run the same command.";
298
+ }
@@ -26,7 +26,10 @@ export const STACK_RUN_OUTCOME_TOOL = "record_stack_run_outcome";
26
26
  export const STACK_RUN_OUTCOME_TIMEOUT_MS = 2000;
27
27
  /** `=off` disables the record (the CLI test suites set it; nothing else should). */
28
28
  export const STACK_RUN_OUTCOME_OPT_OUT_ENV = "BOTBUDDY_STACK_RUN_OUTCOME";
29
- export const STACK_RUN_OUTCOME_CATEGORIES = Object.freeze(["platform", "script", "interrupted", "invocation", "released"]);
29
+ export const STACK_RUN_OUTCOME_CATEGORIES = Object.freeze(["platform", "script", "interrupted", "invocation", "released", "requester"]);
30
+ /** BOT-3109: the only code the `requester` category carries, and the codes it can replace. */
31
+ export const REQUESTER_CATEGORY_CODE = "REQUESTER_NETWORK_LOST";
32
+ const REQUESTER_UNDERLYING_CODES = new Set(["TUNNEL_LOST", "CALLBACK_TRANSPORT_LOST", "LEASE_HEARTBEAT_UNANSWERED"]);
30
33
  /**
31
34
  * BOT-2881: a signal carries no sender, so a hold's owner says so first. The hold sets this to an
32
35
  * absolute path when it starts `bb stack run`, and its stop command creates that file before
@@ -71,6 +74,7 @@ function boundedCode(value) {
71
74
  * interrupted — a signal ended the run (operator Ctrl-C / parent kill)
72
75
  * released — the owner of a hold stopped it on purpose (BOT-2881), and its lease and cleanup held
73
76
  * invocation — exit 4: bad arguments or a refused request before any lease
77
+ * requester — this machine's own network dropped while the run lost its transport (BOT-3109)
74
78
  */
75
79
  export function classifyStackRunOutcome(fields = {}, exitCode) {
76
80
  const code = boundedCode(fields.code);
@@ -80,6 +84,8 @@ export function classifyStackRunOutcome(fields = {}, exitCode) {
80
84
  if (fields.stop_signal?.source === STACK_SIGNAL_SOURCE_OWNER_RELEASE && fields.cleanup === false) return { category: "platform", code: "CLEANUP_FAILED" };
81
85
  if (outcome === "released") return { category: "released", code };
82
86
  if (outcome === "interrupted") return { category: "interrupted", code };
87
+ // BOT-3109: set only when the requester network probe proved a local change (never a guess).
88
+ if (code === REQUESTER_CATEGORY_CODE) return { category: "requester", code };
83
89
  if (code && PLATFORM_CODES.has(code)) return { category: "platform", code };
84
90
  // The stdout fence also covers unresolved cleanup. The lifecycle supplies its
85
91
  // original lease-fence fact separately so coincident child exits cannot hide it.
@@ -190,6 +196,8 @@ export function buildStackRunOutcomePayload({ fields = {}, exitCode, runId = ran
190
196
  ? fields.failure_mechanism : null;
191
197
  // BOT-2977: sent only on a stale-context refusal, so every other payload keeps its shape.
192
198
  const fenceScope = code === "STALE_AGENT_CONTEXT" && ["session", "agent"].includes(fields.fence_scope) ? fields.fence_scope : null;
199
+ // BOT-3109: the code a requester network loss replaced; sent only on such a run.
200
+ const underlyingCode = category === "requester" && REQUESTER_UNDERLYING_CODES.has(fields.underlying_code) ? fields.underlying_code : null;
193
201
  return {
194
202
  run_id: runId,
195
203
  lease_id: validStackLeaseId(fields.lease_id),
@@ -215,6 +223,7 @@ export function buildStackRunOutcomePayload({ fields = {}, exitCode, runId = ran
215
223
  ...(callbackAuthorizationReason ? { callback_authorization_reason: callbackAuthorizationReason } : {}),
216
224
  ...(failureMechanism ? { failure_mechanism: failureMechanism } : {}),
217
225
  ...(fenceScope ? { fence_scope: fenceScope } : {}),
226
+ ...(underlyingCode ? { underlying_code: underlyingCode } : {}),
218
227
  };
219
228
  }
220
229
 
@@ -254,6 +263,14 @@ export async function reportStackRunOutcome({ fields, exitCode, runId, opts, env
254
263
  const { [refusedKey]: _omitted, ...legacyPayload } = payload;
255
264
  return await channel.callTool(STACK_RUN_OUTCOME_TOOL, legacyPayload, { auth: channel.auth, signal: ac.signal });
256
265
  }
266
+ // BOT-3109: a server from before the requester category refuses it (or its underlying_code).
267
+ // Record exactly what that server would have got from an older CLI: the platform code.
268
+ const refusedRequester = payload.category === "requester" && payload.underlying_code && typeof first?.data?.message === "string" &&
269
+ (first.data.message.startsWith("unknown key 'underlying_code'") || first.data.message.startsWith("category must be"));
270
+ if (!ac.signal.aborted && refusedRequester && first?.ok && first?.isError && first?.data?.code === "INVALID_STACK_RUN_OUTCOME") {
271
+ const { underlying_code: underlying, ...rest } = payload;
272
+ return await channel.callTool(STACK_RUN_OUTCOME_TOOL, { ...rest, category: "platform", code: underlying }, { auth: channel.auth, signal: ac.signal });
273
+ }
257
274
  return first;
258
275
  });
259
276
  const result = await Promise.race([call.catch(() => null), timeout]);
package/src/stack.mjs CHANGED
@@ -60,6 +60,7 @@ import { stackServicesForForwards } from "./stack-service-discovery.mjs";
60
60
  import { ENGINE_PUBLISHES_BEYOND_LOOPBACK, engineSocketIsLocal, formatPublishIp, isLoopbackIp, publishIpFix, resolvePublishIp } from "./docker-publish-ip.mjs";
61
61
  import { bold, dim, yellow } from "./utils.mjs";
62
62
  import { STACK_RELEASE_INTENT_ENV, classifyStackRunOutcome, observeStackRunSignal, ownerReleaseHeld, reportStackRunOutcome } from "./stack-run-outcome.mjs";
63
+ import { REQUESTER_NETWORK_CANDIDATE_CODES, REQUESTER_NETWORK_LOST_CODE, createRequesterNetworkWatch, requesterNetworkLostMessage } from "./requester-network.mjs";
63
64
 
64
65
  export const STACK_SCHEMA_VERSION = 1;
65
66
 
@@ -416,6 +417,11 @@ ${bold("REMOTE LEASES")} (BOT-2142, stack run)
416
417
  (BUNDLE_UNSAFE_LINK: it resolves outside the stack); no lease is created
417
418
  6 TUNNEL_LOST the forward exited mid-child: the child is stopped, the lease released,
418
419
  park_reason tunnel_lost (its failures were transport, not product)
420
+ 6 REQUESTER_NETWORK_LOST one of TUNNEL_LOST, CALLBACK_TRANSPORT_LOST or LEASE_HEARTBEAT_UNANSWERED,
421
+ while THIS machine's own network changed (macOS: primary interface or IPv4,
422
+ or configd saw it detach; Linux: default route). underlying_code keeps the
423
+ original code, requester_network {lost_at, evidence} says when; recorded as
424
+ category requester, not platform (BOT-3109). Undecided → the original code.
419
425
  Every remote failure after the lease exists still runs \`stack done\` and closes the forwards.
420
426
 
421
427
  ${bold("HOST_CAPACITY_SOURCE_STALE")} (BOT-2024)
@@ -3951,8 +3957,23 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
3951
3957
  // no claim exists, which is a safe, explicit disposition rather than omission.
3952
3958
  // BOT-1803: whether this run re-adopted / re-provisioned an idle lease, and its stack digest.
3953
3959
  const freshBox = () => ({ incident: null, execution: null, idleLease: null, webUrl: null, portBlock: { released: false, reason: "no port-block claim was created before stack allocation" } });
3954
- let incidentBox = freshBox();
3960
+ const incidentBox = freshBox();
3955
3961
  const startedAtMs = (adapters.now ?? Date.now)();
3962
+ // BOT-3109: watch this machine's own primary network for the run's life (null disables).
3963
+ let networkWatch = null;
3964
+ try {
3965
+ networkWatch = adapters.requesterNetwork === undefined ? createRequesterNetworkWatch() : adapters.requesterNetwork;
3966
+ networkWatch?.start?.();
3967
+ } catch { networkWatch = null; }
3968
+ try {
3969
+ return await runStackLifecycleWatched(opts, childArgv, adapters, incidentBox, freshBox, startedAtMs, networkWatch);
3970
+ } finally {
3971
+ try { networkWatch?.stop?.(); } catch { /* a probe never changes the run */ }
3972
+ }
3973
+ }
3974
+
3975
+ async function runStackLifecycleWatched(opts, childArgv, adapters, firstBox, freshBox, startedAtMs, networkWatch) {
3976
+ let incidentBox = firstBox;
3956
3977
  const once = async (attemptOpts = opts) => {
3957
3978
  try {
3958
3979
  return await runStackLifecycleOnce(attemptOpts, childArgv, adapters, incidentBox);
@@ -3984,7 +4005,8 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
3984
4005
  const withPortBlock = incidentBox.portBlock && !withIdle?.portBlock ? { ...withIdle, portBlock: incidentBox.portBlock } : withIdle;
3985
4006
  const withWebUrl = incidentBox.webUrl && withPortBlock && !withPortBlock.webUrl ? { ...withPortBlock, webUrl: incidentBox.webUrl } : withPortBlock;
3986
4007
  const withStopSignal = incidentBox.stopSignal && withWebUrl ? { ...withWebUrl, stopSignal: incidentBox.stopSignal } : withWebUrl;
3987
- const final = incidentBox.callbackOutage && withStopSignal ? { ...withStopSignal, callbackOutage: incidentBox.callbackOutage } : withStopSignal;
4008
+ const withOutage = incidentBox.callbackOutage && withStopSignal ? { ...withStopSignal, callbackOutage: incidentBox.callbackOutage } : withStopSignal;
4009
+ const final = await applyRequesterNetworkLoss(withOutage, networkWatch, { failedAtMs: incidentBox.lossAtMs });
3988
4010
  // BOT-2467: non-enumerable, so the receipt fields and result comparisons never see it.
3989
4011
  if (final && typeof final === "object" && incidentBox.outcomeChannel) {
3990
4012
  Object.defineProperty(final, OUTCOME_CHANNEL, { value: incidentBox.outcomeChannel, enumerable: false });
@@ -3992,6 +4014,36 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
3992
4014
  return final;
3993
4015
  }
3994
4016
 
4017
+ /**
4018
+ * BOT-3109: a run that ended on a transport or control-plane loss while THIS machine's own
4019
+ * network changed is the requester's loss, not BotBuddy's: code REQUESTER_NETWORK_LOST (category
4020
+ * `requester`), the original code kept as `underlyingCode`, and one plain stderr line. Anything
4021
+ * short of a proven change (no change, an undecided or throwing probe, no probe) returns the
4022
+ * result object untouched, so its receipt is byte-identical to before.
4023
+ */
4024
+ export async function applyRequesterNetworkLoss(result, watch, { failedAtMs = null, stderrLine = (line) => process.stderr.write(`${line}\n`) } = {}) {
4025
+ if (!result || typeof result !== "object" || !REQUESTER_NETWORK_CANDIDATE_CODES.includes(result.code) || typeof watch?.classify !== "function") return result;
4026
+ // An unproven teardown (a lease that may be leaked) is never relabelled: it stays the platform failure it is today.
4027
+ // CALLBACK_CLEANUP_UNCERTAIN exits CLEANUP_FAILED with cleanup.ok true, so the exit code is checked too.
4028
+ // A TUNNEL_LOST whose callback close threw keeps exit 6 today, so it says so on the result instead.
4029
+ if (result.cleanup?.ok === false || result.exitCode === EXIT.CLEANUP_FAILED || result.callbackCleanupUncertain === true) return result;
4030
+ let network;
4031
+ try {
4032
+ network = await watch.classify({ failedAtMs });
4033
+ } catch {
4034
+ return result;
4035
+ }
4036
+ if (network?.decision !== "changed" || !Number.isFinite(network.atMs)) return result;
4037
+ const message = requesterNetworkLostMessage(network, result.code);
4038
+ try { stderrLine(message); } catch { /* the receipt still carries it */ }
4039
+ return {
4040
+ ...result, code: REQUESTER_NETWORK_LOST_CODE, underlyingCode: result.code,
4041
+ requesterNetwork: { lost_at: new Date(network.atMs).toISOString(), evidence: network.evidence },
4042
+ // The original diagnostic stays on the receipt, so a wrong attribution can still be seen.
4043
+ error: message, underlyingError: result.error ?? null,
4044
+ };
4045
+ }
4046
+
3995
4047
  /** BOT-2467: where a lifecycle result carries its outcome-record channel. */
3996
4048
  export const OUTCOME_CHANNEL = Symbol.for("botbuddy.stackRunOutcomeChannel");
3997
4049
 
@@ -4316,6 +4368,8 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
4316
4368
  const stopChild = (signal = "SIGTERM", forcedCode = null) => {
4317
4369
  void closeOrigin();
4318
4370
  if (!child || childResult) return;
4371
+ // BOT-3109: when a loss stopped the child, so the requester probe weighs the network then, not after cleanup.
4372
+ if (forcedCode != null && forcedStopCode == null) incidentBox.lossAtMs ??= Date.now();
4319
4373
  forcedStopCode ??= forcedCode;
4320
4374
  childGroupSignal(child, signal);
4321
4375
  const kill = clock.setTimeout(() => childGroupSignal(child, "SIGKILL"), 10_000);
@@ -5173,7 +5227,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
5173
5227
  if (outage.recovered > (previous?.recovered ?? 0)) stderrLine("stack: the control plane answered again; the callback route stayed up");
5174
5228
  },
5175
5229
  });
5176
- nativeCallback.onExit((exit) => { if (childResult) return; callbackLost = exit; fencing = true; stopChild("SIGTERM", EXIT.LEASE_FAILED); });
5230
+ nativeCallback.onExit((exit) => { if (childResult) return; incidentBox.lossAtMs ??= Date.now(); callbackLost = exit; fencing = true; stopChild("SIGTERM", EXIT.LEASE_FAILED); });
5177
5231
  placement.callbackTransport = NATIVE_CALLBACK_TRANSPORT;
5178
5232
  } catch (error) {
5179
5233
  await closeTunnel(); const cleanup = await requestCleanup();
@@ -5188,6 +5242,8 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
5188
5242
  const forwardTarget = sshTarget;
5189
5243
  tunnel.onExit((exit) => {
5190
5244
  if (childResult) return;
5245
+ // BOT-3109: when the loss was seen, even before a child exists (stopChild stamps only a running one).
5246
+ incidentBox.lossAtMs ??= Date.now();
5191
5247
  tunnelLost = { ...exit, target: forwardTarget };
5192
5248
  stopChild("SIGTERM", EXIT.LEASE_FAILED);
5193
5249
  });
@@ -5270,7 +5326,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
5270
5326
  // The forwards died between being verified and the child starting.
5271
5327
  await closeTunnel();
5272
5328
  const cleanup = await requestCleanup();
5273
- return tunnelLostResult(tunnelLost, { leaseId, cleanup, childExitCode: null });
5329
+ return tunnelLostResult(tunnelLost, { leaseId, cleanup, childExitCode: null, ...(callbackCleanupError ? { callbackCleanupUncertain: true } : {}) });
5274
5330
  }
5275
5331
  child = startChild(childArgv, childEnv, childCwd);
5276
5332
  leaseIdleTtlSecs = current.data.idle_ttl_secs ?? opts.idleTtl ?? 1800;
@@ -5385,7 +5441,9 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
5385
5441
  return { exitCode: cleanup.ok && !callbackCleanupError ? EXIT.LEASE_FAILED : EXIT.CLEANUP_FAILED, outcome: "failed", code: authorizationLost ? "CALLBACK_AUTHORIZATION_LOST" : "CALLBACK_TRANSPORT_LOST", callbackCause, ...(callbackAuthorizationReason ? { callbackAuthorizationReason } : {}), leaseId, childExitCode: childResult.code, cleanup, fenced: true, error: `callback transport authorization or runtime context was lost (${callbackCause}${callbackAuthorizationReason ? `: ${callbackAuthorizationReason}` : ""}); child stopped` };
5386
5442
  }
5387
5443
  if (tunnelLost && !receivedSignal) {
5388
- return tunnelLostResult(tunnelLost, { leaseId, cleanup, childExitCode: childResult.code, fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {}) });
5444
+ return tunnelLostResult(tunnelLost, { leaseId, cleanup, childExitCode: childResult.code, fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {}),
5445
+ // BOT-3109: not a receipt field; it keeps an unproven callback close out of the requester relabel.
5446
+ ...(callbackCleanupError ? { callbackCleanupUncertain: true } : {}) });
5389
5447
  }
5390
5448
  if (preempted && !receivedSignal) {
5391
5449
  return {
@@ -5460,6 +5518,10 @@ export function runReceiptFields(result) {
5460
5518
  // BOT-1797: carry the server's typed refusal code (e.g. stack_path_root_refused)
5461
5519
  // so automation can branch on it instead of matching the message text.
5462
5520
  command: "run", outcome: ownerReleaseHeld(result) ? "released" : result.outcome, code: result.code ?? null, lease_id: result.leaseId ?? null,
5521
+ // BOT-3109: a requester-side network loss names the code it replaced and when this machine lost its network.
5522
+ ...(result.code === REQUESTER_NETWORK_LOST_CODE && REQUESTER_NETWORK_CANDIDATE_CODES.includes(result.underlyingCode)
5523
+ ? { underlying_code: result.underlyingCode, underlying_error: result.underlyingError ?? null,
5524
+ ...(result.requesterNetwork ? { requester_network: result.requesterNetwork } : {}) } : {}),
5463
5525
  // BOT-2881: the signal that ended this run and what is known of its source (never on a run no signal touched).
5464
5526
  ...(result.stopSignal ? { stop_signal: result.stopSignal } : {}),
5465
5527
  ...(result.state ? { state: result.state } : {}),