@botbuddy/cli 1.64.29 → 1.64.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/daemon/bin/darwin-arm64/botbuddy-daemon +0 -0
- package/daemon/bin/linux-x64/botbuddy-daemon +0 -0
- package/daemon-release/darwin-arm64.json +4 -4
- package/daemon-release/linux-x64.json +4 -4
- package/daemon-release/manifest.json +5 -5
- package/package.json +1 -1
- package/src/requester-network.mjs +298 -0
- package/src/stack-run-outcome.mjs +26 -6
- package/src/stack.mjs +123 -10
|
Binary file
|
|
Binary file
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
|
-
"release_version": "1.64.
|
|
3
|
-
"source_commit": "
|
|
4
|
-
"source_sha256": "
|
|
2
|
+
"release_version": "1.64.31",
|
|
3
|
+
"source_commit": "1d612675c4e0b580098e3abe63a8bc59cd581c2c",
|
|
4
|
+
"source_sha256": "87837ab1232c23b40ebcaa4e844c9314f811d63471550b866ca66c6aff23febe",
|
|
5
5
|
"target": "darwin-arm64",
|
|
6
|
-
"sha256": "
|
|
6
|
+
"sha256": "fc73087ccd6f503e12dbc011ea74520bc92fc7442009b0674621be1ef3bc7ecd"
|
|
7
7
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
|
-
"release_version": "1.64.
|
|
3
|
-
"source_commit": "
|
|
4
|
-
"source_sha256": "
|
|
2
|
+
"release_version": "1.64.31",
|
|
3
|
+
"source_commit": "1d612675c4e0b580098e3abe63a8bc59cd581c2c",
|
|
4
|
+
"source_sha256": "87837ab1232c23b40ebcaa4e844c9314f811d63471550b866ca66c6aff23febe",
|
|
5
5
|
"target": "linux-x64",
|
|
6
|
-
"sha256": "
|
|
6
|
+
"sha256": "85804a8e6e1cbff0433407337c65954b7feb7e06660a5e4722f94e5f08f57aa5"
|
|
7
7
|
}
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schema_version": 1,
|
|
3
|
-
"release_version": "1.64.
|
|
4
|
-
"source_commit": "
|
|
5
|
-
"source_sha256": "
|
|
3
|
+
"release_version": "1.64.31",
|
|
4
|
+
"source_commit": "1d612675c4e0b580098e3abe63a8bc59cd581c2c",
|
|
5
|
+
"source_sha256": "87837ab1232c23b40ebcaa4e844c9314f811d63471550b866ca66c6aff23febe",
|
|
6
6
|
"targets": {
|
|
7
7
|
"darwin-arm64": {
|
|
8
8
|
"goos": "darwin",
|
|
9
9
|
"goarch": "arm64",
|
|
10
10
|
"binary": "daemon/bin/darwin-arm64/botbuddy-daemon",
|
|
11
|
-
"sha256": "
|
|
11
|
+
"sha256": "fc73087ccd6f503e12dbc011ea74520bc92fc7442009b0674621be1ef3bc7ecd",
|
|
12
12
|
"mode": "0755"
|
|
13
13
|
},
|
|
14
14
|
"linux-x64": {
|
|
15
15
|
"goos": "linux",
|
|
16
16
|
"goarch": "amd64",
|
|
17
17
|
"binary": "daemon/bin/linux-x64/botbuddy-daemon",
|
|
18
|
-
"sha256": "
|
|
18
|
+
"sha256": "85804a8e6e1cbff0433407337c65954b7feb7e06660a5e4722f94e5f08f57aa5",
|
|
19
19
|
"mode": "0755"
|
|
20
20
|
}
|
|
21
21
|
}
|
package/package.json
CHANGED
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
// BOT-3109 — did THIS machine lose its own network while a `bb stack run` failed?
|
|
2
|
+
//
|
|
3
|
+
// A requester-side drop (a cable pulled, Wi-Fi handing over, sleep) ends a remote run
|
|
4
|
+
// with TUNNEL_LOST, CALLBACK_TRANSPORT_LOST or LEASE_HEARTBEAT_UNANSWERED, exactly like
|
|
5
|
+
// a BotBuddy or stack-host outage. This module tells them apart from the requester's own
|
|
6
|
+
// side, and only when it can prove it:
|
|
7
|
+
//
|
|
8
|
+
// - a small sampler reads the primary route every SAMPLE_MS while the run lives
|
|
9
|
+
// (macOS: `route -n get default` + `ipconfig getifaddr`; Linux: `ip -4 route show default`),
|
|
10
|
+
// and when the primary moves, whether the old interface lost its address;
|
|
11
|
+
// - at a candidate failure, `classify()` reads once more and, on macOS, reads configd's own
|
|
12
|
+
// record of interface detaches in the window before the failure.
|
|
13
|
+
//
|
|
14
|
+
// The answer is `changed` (with evidence), `unchanged` or `undecided`. Only `changed`
|
|
15
|
+
// changes what a run reports; the caller keeps today's code for the other two, so an
|
|
16
|
+
// unsupported OS, a missing tool or a probe error never guesses `requester`.
|
|
17
|
+
|
|
18
|
+
import { execFile } from "node:child_process";
|
|
19
|
+
|
|
20
|
+
export const REQUESTER_NETWORK_LOST_CODE = "REQUESTER_NETWORK_LOST";
|
|
21
|
+
/** The terminal codes a requester's own network loss can produce (BOT-3109). */
|
|
22
|
+
export const REQUESTER_NETWORK_CANDIDATE_CODES = Object.freeze(["TUNNEL_LOST", "CALLBACK_TRANSPORT_LOST", "LEASE_HEARTBEAT_UNANSWERED"]);
|
|
23
|
+
/** `=off` disables the probe (the CLI test suites set it; nothing else should). */
|
|
24
|
+
export const REQUESTER_NETWORK_PROBE_ENV = "BOTBUDDY_REQUESTER_NETWORK_PROBE";
|
|
25
|
+
export const REQUESTER_NETWORK_SAMPLE_MS = 15_000;
|
|
26
|
+
/** The control-plane deadlines are 180 s, so a loss that ended a run began at most this long before. */
|
|
27
|
+
export const REQUESTER_NETWORK_LOOKBACK_MS = 300_000;
|
|
28
|
+
/** Evidence just after the failure still counts (the sampler may see the outage a beat late). */
|
|
29
|
+
export const REQUESTER_NETWORK_FAILURE_SLACK_MS = 20_000;
|
|
30
|
+
const PROBE_TIMEOUT_MS = 3000;
|
|
31
|
+
const LOG_TIMEOUT_MS = 6000;
|
|
32
|
+
const MAX_SAMPLES = 64;
|
|
33
|
+
|
|
34
|
+
const IFACE_RE = /^[A-Za-z0-9_.:@-]{1,32}$/;
|
|
35
|
+
|
|
36
|
+
// Every probe child in flight; `stop()` aborts them so a finished run never waits on a probe.
|
|
37
|
+
const inFlight = new Set();
|
|
38
|
+
|
|
39
|
+
/** The probe runner (exported for its abort test). */
|
|
40
|
+
export function execText(file, args, timeout) {
|
|
41
|
+
return new Promise((resolve) => {
|
|
42
|
+
const ac = new AbortController();
|
|
43
|
+
inFlight.add(ac);
|
|
44
|
+
try {
|
|
45
|
+
execFile(file, args, { encoding: "utf8", timeout, maxBuffer: 1 << 20, signal: ac.signal }, (error, stdout, stderr) => {
|
|
46
|
+
inFlight.delete(ac);
|
|
47
|
+
resolve({
|
|
48
|
+
status: error ? (typeof error.code === "number" ? error.code : -1) : 0,
|
|
49
|
+
stdout: typeof stdout === "string" ? stdout : "", stderr: typeof stderr === "string" ? stderr : "", missing: error?.code === "ENOENT",
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
} catch {
|
|
53
|
+
inFlight.delete(ac);
|
|
54
|
+
resolve({ status: -1, stdout: "", missing: true });
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Abort every probe child still running (an aborted read is unreadable, never a loss). */
|
|
60
|
+
export function abortRequesterNetworkProbes() {
|
|
61
|
+
for (const ac of inFlight) ac.abort();
|
|
62
|
+
inFlight.clear();
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const iface = (value) => (typeof value === "string" && IFACE_RE.test(value) ? value : null);
|
|
66
|
+
|
|
67
|
+
/** `route -n get default` → the primary interface, or null when there is no default route. */
|
|
68
|
+
export function parseMacosDefaultRoute(stdout) {
|
|
69
|
+
return iface(/^\s*interface:\s*(\S+)\s*$/m.exec(stdout ?? "")?.[1]);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** `ip -4 route show default` → { interface, gateway } of the first default route, or null. */
|
|
73
|
+
export function parseLinuxDefaultRoute(stdout) {
|
|
74
|
+
const line = String(stdout ?? "").split("\n").find((l) => /^default\b/.test(l.trim()));
|
|
75
|
+
if (!line) return null;
|
|
76
|
+
const dev = iface(/\bdev\s+(\S+)/.exec(line)?.[1]);
|
|
77
|
+
if (!dev) return null;
|
|
78
|
+
return { interface: dev, gateway: /\bvia\s+(\S+)/.exec(line)?.[1] ?? null };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* One reading of this machine's primary network identity. `{ ok: false }` when it cannot be
|
|
83
|
+
* read (unsupported OS, missing tool, timeout); `{ ok: true, interface: null }` when there is
|
|
84
|
+
* provably no primary route — which is itself the loss.
|
|
85
|
+
*/
|
|
86
|
+
export async function readPrimaryNetwork({ platform = process.platform, run = execText } = {}) {
|
|
87
|
+
try {
|
|
88
|
+
if (platform === "darwin") {
|
|
89
|
+
const route = await run("/sbin/route", ["-n", "get", "default"], PROBE_TIMEOUT_MS);
|
|
90
|
+
if (route.missing) return { ok: false };
|
|
91
|
+
const primary = parseMacosDefaultRoute(route.stdout);
|
|
92
|
+
// route prints "not in table" (and exits non-zero) when there is no default route; anything else is unreadable.
|
|
93
|
+
if (!primary) return /not in table/i.test(`${route.stdout}\n${route.stderr ?? ""}`) ? { ok: true, interface: null, address: null } : { ok: false };
|
|
94
|
+
const addr = await run("/usr/sbin/ipconfig", ["getifaddr", primary], PROBE_TIMEOUT_MS);
|
|
95
|
+
if (addr.missing) return { ok: false };
|
|
96
|
+
const address = /^\d{1,3}(\.\d{1,3}){3}$/.test(addr.stdout.trim()) ? addr.stdout.trim() : null;
|
|
97
|
+
return { ok: true, interface: primary, address };
|
|
98
|
+
}
|
|
99
|
+
if (platform === "linux") {
|
|
100
|
+
const route = await run("ip", ["-4", "route", "show", "default"], PROBE_TIMEOUT_MS);
|
|
101
|
+
if (route.missing || route.status !== 0) return { ok: false };
|
|
102
|
+
const parsed = parseLinuxDefaultRoute(route.stdout);
|
|
103
|
+
return parsed ? { ok: true, interface: parsed.interface, address: parsed.gateway } : { ok: true, interface: null, address: null };
|
|
104
|
+
}
|
|
105
|
+
} catch { /* an unreadable probe is undecided, never a loss */ }
|
|
106
|
+
return { ok: false };
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Whether `name` still has an IPv4 address right now. `{ ok: true, address: null }` is proof it
|
|
111
|
+
* lost it (macOS `ipconfig getifaddr` exits 1 with no output for an interface without one, or
|
|
112
|
+
* one that detached; Linux `ip -4 -o addr show dev` prints nothing). Anything else is unreadable.
|
|
113
|
+
*/
|
|
114
|
+
export async function readInterfaceAddress(name, { platform = process.platform, run = execText } = {}) {
|
|
115
|
+
if (!iface(name)) return { ok: false };
|
|
116
|
+
try {
|
|
117
|
+
if (platform === "darwin") {
|
|
118
|
+
const out = await run("/usr/sbin/ipconfig", ["getifaddr", name], PROBE_TIMEOUT_MS);
|
|
119
|
+
if (out.missing) return { ok: false };
|
|
120
|
+
const text = out.stdout.trim();
|
|
121
|
+
if (out.status === 0 && /^\d{1,3}(\.\d{1,3}){3}$/.test(text)) return { ok: true, address: text };
|
|
122
|
+
return out.status === 1 && text === "" ? { ok: true, address: null } : { ok: false };
|
|
123
|
+
}
|
|
124
|
+
if (platform === "linux") {
|
|
125
|
+
const out = await run("ip", ["-4", "-o", "addr", "show", "dev", name], PROBE_TIMEOUT_MS);
|
|
126
|
+
if (out.missing) return { ok: false };
|
|
127
|
+
// A detached device is an error ("does not exist"): it has no address either.
|
|
128
|
+
if (out.status !== 0) return /does not exist/i.test(out.stderr ?? "") ? { ok: true, address: null } : { ok: false };
|
|
129
|
+
const inet = /\binet\s+(\d{1,3}(?:\.\d{1,3}){3})/.exec(out.stdout)?.[1] ?? null;
|
|
130
|
+
return { ok: true, address: inet };
|
|
131
|
+
}
|
|
132
|
+
} catch { /* unreadable */ }
|
|
133
|
+
return { ok: false };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** Parse configd's timestamp ("2026-11-09 09:19:32.634512+0200"), which carries its own offset. */
|
|
137
|
+
function parseLogTimestamp(value) {
|
|
138
|
+
const m = /^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})(\.\d{1,3})?\d*([+-]\d{2}):?(\d{2})$/.exec(String(value ?? "").trim());
|
|
139
|
+
if (!m) return NaN;
|
|
140
|
+
return Date.parse(`${m[1]}T${m[2]}${m[3] ?? ""}${m[4]}:${m[5]}`);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* configd's `log show --style ndjson` lines → interface-detach events. Only a detach is evidence:
|
|
145
|
+
* a "network changed" line also fires when an interface is merely added (docking while Wi-Fi
|
|
146
|
+
* stays up), so it never counts on its own. A line without a zoned timestamp is skipped.
|
|
147
|
+
*/
|
|
148
|
+
export function parseConfigdLog(stdout) {
|
|
149
|
+
const events = [];
|
|
150
|
+
for (const line of String(stdout ?? "").split("\n")) {
|
|
151
|
+
if (!line.startsWith("{")) continue;
|
|
152
|
+
let entry;
|
|
153
|
+
try { entry = JSON.parse(line); } catch { continue; }
|
|
154
|
+
const atMs = parseLogTimestamp(entry?.timestamp);
|
|
155
|
+
if (!Number.isFinite(atMs)) continue;
|
|
156
|
+
const detached = /Process interface detaching:\s*(\S+)/.exec(String(entry?.eventMessage ?? ""));
|
|
157
|
+
if (detached && iface(detached[1])) events.push({ atMs, kind: "interface_detached", interface: detached[1] });
|
|
158
|
+
}
|
|
159
|
+
return events;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** macOS only: configd's interface detaches in `[sinceMs, untilMs]`. `{ ok: false }` when unreadable. */
|
|
163
|
+
export async function readConfigdNetworkEvents({ sinceMs, untilMs = Infinity, nowMs = Date.now(), run = execText } = {}) {
|
|
164
|
+
const secs = Math.max(1, Math.ceil((nowMs - sinceMs) / 1000) + 5);
|
|
165
|
+
try {
|
|
166
|
+
const out = await run("/usr/bin/log", ["show", "--style", "ndjson", "--last", `${secs}s`, "--predicate",
|
|
167
|
+
'process == "configd" AND eventMessage CONTAINS "interface detaching"'], LOG_TIMEOUT_MS);
|
|
168
|
+
if (out.missing || out.status !== 0) return { ok: false };
|
|
169
|
+
return { ok: true, events: parseConfigdLog(out.stdout).filter((e) => e.atMs >= sinceMs && e.atMs <= untilMs) };
|
|
170
|
+
} catch {
|
|
171
|
+
return { ok: false };
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Pure decision over the samples (oldest first, each `{ atMs, ok, interface, address,
|
|
177
|
+
* previousLost }`), a fresh reading and, on macOS, configd's detaches. Evidence counts only inside
|
|
178
|
+
* `[sinceMs, untilMs]` (the lookback before the failure, and a short slack after it).
|
|
179
|
+
*
|
|
180
|
+
* Proof of a local loss is one of: the primary route disappeared after a sample had one; the
|
|
181
|
+
* primary moved to another interface AND the old one had lost its address when that was seen;
|
|
182
|
+
* the primary interface kept its name but its address changed; configd saw a primary interface
|
|
183
|
+
* detach. A primary that moved while the old one kept its address (docking, a VPN) is not proof:
|
|
184
|
+
* that run is undecided, so today's code stands.
|
|
185
|
+
*/
|
|
186
|
+
export function decideRequesterNetwork({ samples = [], current, log = null, sinceMs, untilMs = Infinity, platform }) {
|
|
187
|
+
const readable = samples.filter((s) => s.ok);
|
|
188
|
+
// The reference: the newest readable sample taken before the window opened, else the oldest in it.
|
|
189
|
+
const before = readable.filter((s) => s.atMs < sinceMs);
|
|
190
|
+
const reference = before.length ? before[before.length - 1] : readable.find((s) => s.atMs <= untilMs) ?? null;
|
|
191
|
+
// A machine that never had an IPv4 primary route (IPv6-only, a policy-routed Linux host) gives
|
|
192
|
+
// the probe nothing to compare: never a loss.
|
|
193
|
+
if (!reference?.interface) return { decision: "undecided" };
|
|
194
|
+
const inWindow = (s) => s.atMs >= sinceMs && s.atMs <= untilMs;
|
|
195
|
+
const chain = [...readable.filter((s) => s !== reference && inWindow(s)), ...(current?.ok && inWindow(current) ? [current] : [])];
|
|
196
|
+
const primaries = new Set([reference.interface]);
|
|
197
|
+
let previous = reference;
|
|
198
|
+
let ambiguous = false;
|
|
199
|
+
for (const sample of chain) {
|
|
200
|
+
if (sample.interface == null) {
|
|
201
|
+
if (previous.interface != null) return { decision: "changed", atMs: sample.atMs, evidence: "no_primary_interface", before: previous.interface, after: null };
|
|
202
|
+
} else if (previous.interface == null) {
|
|
203
|
+
// Back from no route: the loss was already counted above.
|
|
204
|
+
} else if (sample.interface !== previous.interface) {
|
|
205
|
+
if (sample.previousLost === true) return { decision: "changed", atMs: sample.atMs, evidence: "primary_interface_changed", before: previous.interface, after: sample.interface };
|
|
206
|
+
ambiguous = true;
|
|
207
|
+
} else if ((sample.address ?? null) !== (previous.address ?? null) && previous.address != null && sample.address != null) {
|
|
208
|
+
return { decision: "changed", atMs: sample.atMs, evidence: "primary_address_changed", before: previous.interface, after: sample.interface };
|
|
209
|
+
}
|
|
210
|
+
if (sample.interface) primaries.add(sample.interface);
|
|
211
|
+
previous = sample;
|
|
212
|
+
}
|
|
213
|
+
if (log?.ok) {
|
|
214
|
+
const detach = log.events.find((e) => e.kind === "interface_detached" && primaries.has(e.interface) && e.atMs >= sinceMs && e.atMs <= untilMs);
|
|
215
|
+
if (detach) return { decision: "changed", atMs: detach.atMs, evidence: "interface_detached", before: detach.interface, after: current?.interface ?? null };
|
|
216
|
+
}
|
|
217
|
+
// "unchanged" needs a fresh readable answer, no ambiguous move and (on macOS) a readable log.
|
|
218
|
+
const logRead = platform !== "darwin" || log?.ok === true;
|
|
219
|
+
return !ambiguous && current?.ok && logRead ? { decision: "unchanged" } : { decision: "undecided" };
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* A watch over one run. `start()` takes the first sample and samples every `sampleMs`;
|
|
224
|
+
* `classify({ failedAtMs })` decides over the lookback window before the failure; `stop()` ends
|
|
225
|
+
* sampling. Never throws.
|
|
226
|
+
*/
|
|
227
|
+
export function createRequesterNetworkWatch({
|
|
228
|
+
platform = process.platform, env = process.env, now = Date.now, read: readNetwork = readPrimaryNetwork, readAddress: readOldAddress = readInterfaceAddress,
|
|
229
|
+
readLog = readConfigdNetworkEvents, sampleMs = REQUESTER_NETWORK_SAMPLE_MS, lookbackMs = REQUESTER_NETWORK_LOOKBACK_MS,
|
|
230
|
+
} = {}) {
|
|
231
|
+
const disabled = env?.[REQUESTER_NETWORK_PROBE_ENV] === "off" || (platform !== "darwin" && platform !== "linux");
|
|
232
|
+
const samples = [];
|
|
233
|
+
let timer = null;
|
|
234
|
+
let startedAtMs = null;
|
|
235
|
+
let stopped = false;
|
|
236
|
+
// After stop() no probe child starts, so nothing outlives the run.
|
|
237
|
+
const read = (opts) => (stopped ? Promise.resolve({ ok: false }) : readNetwork(opts));
|
|
238
|
+
const readAddress = (name, opts) => (stopped ? Promise.resolve({ ok: false }) : readOldAddress(name, opts));
|
|
239
|
+
const take = async () => {
|
|
240
|
+
const atMs = now();
|
|
241
|
+
let reading;
|
|
242
|
+
try { reading = await read({ platform }); } catch { reading = { ok: false }; }
|
|
243
|
+
const sample = { atMs, ok: reading?.ok === true, interface: reading?.interface ?? null, address: reading?.address ?? null, previousLost: null };
|
|
244
|
+
const last = [...samples].reverse().find((s) => s.ok);
|
|
245
|
+
// The primary moved: did the old one lose its address (a loss) or is it still up (docking, a VPN)?
|
|
246
|
+
if (sample.ok && sample.interface && last?.interface && last.interface !== sample.interface) {
|
|
247
|
+
try {
|
|
248
|
+
const old = await readAddress(last.interface, { platform });
|
|
249
|
+
sample.previousLost = old?.ok === true ? old.address == null : null;
|
|
250
|
+
} catch { sample.previousLost = null; }
|
|
251
|
+
}
|
|
252
|
+
samples.push(sample);
|
|
253
|
+
if (samples.length > MAX_SAMPLES) samples.shift();
|
|
254
|
+
return sample;
|
|
255
|
+
};
|
|
256
|
+
return {
|
|
257
|
+
disabled,
|
|
258
|
+
start() {
|
|
259
|
+
if (disabled || startedAtMs != null) return;
|
|
260
|
+
startedAtMs = now();
|
|
261
|
+
take().catch(() => {});
|
|
262
|
+
timer = setInterval(() => { take().catch(() => {}); }, sampleMs);
|
|
263
|
+
timer.unref?.();
|
|
264
|
+
},
|
|
265
|
+
stop() {
|
|
266
|
+
stopped = true;
|
|
267
|
+
if (timer) clearInterval(timer);
|
|
268
|
+
timer = null;
|
|
269
|
+
if (!disabled) abortRequesterNetworkProbes();
|
|
270
|
+
},
|
|
271
|
+
async classify({ failedAtMs } = {}) {
|
|
272
|
+
if (disabled || startedAtMs == null) return { decision: "undecided" };
|
|
273
|
+
try {
|
|
274
|
+
const nowMs = now();
|
|
275
|
+
// The failure, not this call (cleanup can take minutes): a change after it is not its cause.
|
|
276
|
+
const failAt = Number.isFinite(failedAtMs) ? Math.min(failedAtMs, nowMs) : nowMs;
|
|
277
|
+
const sinceMs = Math.max(startedAtMs, failAt - lookbackMs);
|
|
278
|
+
const untilMs = failAt + REQUESTER_NETWORK_FAILURE_SLACK_MS;
|
|
279
|
+
const history = samples.slice();
|
|
280
|
+
const current = await take();
|
|
281
|
+
const log = platform === "darwin" ? await readLog({ sinceMs, untilMs, nowMs }) : null;
|
|
282
|
+
return decideRequesterNetwork({ samples: history, current, log, sinceMs, untilMs, platform });
|
|
283
|
+
} catch {
|
|
284
|
+
return { decision: "undecided" };
|
|
285
|
+
}
|
|
286
|
+
},
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/** The one stderr line a requester-side loss prints (BOT-3109). */
|
|
291
|
+
export function requesterNetworkLostMessage(network, underlyingCode) {
|
|
292
|
+
const at = Number.isFinite(network?.atMs) ? new Date(network.atMs).toISOString() : "an unknown time";
|
|
293
|
+
const how = network?.evidence === "no_primary_interface" ? "had no network route"
|
|
294
|
+
: network?.evidence === "interface_detached" ? `lost its network interface${network.before ? ` ${network.before}` : ""}`
|
|
295
|
+
: `changed its primary network${network?.before && network?.after ? ` (${network.before} → ${network.after})` : ""}`;
|
|
296
|
+
return `stack: this machine ${how} at ${at}, so the run ended (${underlyingCode}). BotBuddy and the stack host were not at fault. ` +
|
|
297
|
+
"Check this machine's connection, then re-run the same command.";
|
|
298
|
+
}
|
|
@@ -26,7 +26,10 @@ export const STACK_RUN_OUTCOME_TOOL = "record_stack_run_outcome";
|
|
|
26
26
|
export const STACK_RUN_OUTCOME_TIMEOUT_MS = 2000;
|
|
27
27
|
/** `=off` disables the record (the CLI test suites set it; nothing else should). */
|
|
28
28
|
export const STACK_RUN_OUTCOME_OPT_OUT_ENV = "BOTBUDDY_STACK_RUN_OUTCOME";
|
|
29
|
-
export const STACK_RUN_OUTCOME_CATEGORIES = Object.freeze(["platform", "script", "interrupted", "invocation", "released"]);
|
|
29
|
+
export const STACK_RUN_OUTCOME_CATEGORIES = Object.freeze(["platform", "script", "interrupted", "invocation", "released", "requester"]);
|
|
30
|
+
/** BOT-3109: the only code the `requester` category carries, and the codes it can replace. */
|
|
31
|
+
export const REQUESTER_CATEGORY_CODE = "REQUESTER_NETWORK_LOST";
|
|
32
|
+
const REQUESTER_UNDERLYING_CODES = new Set(["TUNNEL_LOST", "CALLBACK_TRANSPORT_LOST", "LEASE_HEARTBEAT_UNANSWERED"]);
|
|
30
33
|
/**
|
|
31
34
|
* BOT-2881: a signal carries no sender, so a hold's owner says so first. The hold sets this to an
|
|
32
35
|
* absolute path when it starts `bb stack run`, and its stop command creates that file before
|
|
@@ -71,6 +74,7 @@ function boundedCode(value) {
|
|
|
71
74
|
* interrupted — a signal ended the run (operator Ctrl-C / parent kill)
|
|
72
75
|
* released — the owner of a hold stopped it on purpose (BOT-2881), and its lease and cleanup held
|
|
73
76
|
* invocation — exit 4: bad arguments or a refused request before any lease
|
|
77
|
+
* requester — this machine's own network dropped while the run lost its transport (BOT-3109)
|
|
74
78
|
*/
|
|
75
79
|
export function classifyStackRunOutcome(fields = {}, exitCode) {
|
|
76
80
|
const code = boundedCode(fields.code);
|
|
@@ -80,6 +84,8 @@ export function classifyStackRunOutcome(fields = {}, exitCode) {
|
|
|
80
84
|
if (fields.stop_signal?.source === STACK_SIGNAL_SOURCE_OWNER_RELEASE && fields.cleanup === false) return { category: "platform", code: "CLEANUP_FAILED" };
|
|
81
85
|
if (outcome === "released") return { category: "released", code };
|
|
82
86
|
if (outcome === "interrupted") return { category: "interrupted", code };
|
|
87
|
+
// BOT-3109: set only when the requester network probe proved a local change (never a guess).
|
|
88
|
+
if (code === REQUESTER_CATEGORY_CODE) return { category: "requester", code };
|
|
83
89
|
if (code && PLATFORM_CODES.has(code)) return { category: "platform", code };
|
|
84
90
|
// The stdout fence also covers unresolved cleanup. The lifecycle supplies its
|
|
85
91
|
// original lease-fence fact separately so coincident child exits cannot hide it.
|
|
@@ -188,6 +194,10 @@ export function buildStackRunOutcomePayload({ fields = {}, exitCode, runId = ran
|
|
|
188
194
|
const failureMechanism = code === "REQUESTER_ORIGIN_DIGEST_UNAVAILABLE" &&
|
|
189
195
|
["origin_config_missing", "origin_digest_missing"].includes(fields.failure_mechanism)
|
|
190
196
|
? fields.failure_mechanism : null;
|
|
197
|
+
// BOT-2977: sent only on a stale-context refusal, so every other payload keeps its shape.
|
|
198
|
+
const fenceScope = code === "STALE_AGENT_CONTEXT" && ["session", "agent"].includes(fields.fence_scope) ? fields.fence_scope : null;
|
|
199
|
+
// BOT-3109: the code a requester network loss replaced; sent only on such a run.
|
|
200
|
+
const underlyingCode = category === "requester" && REQUESTER_UNDERLYING_CODES.has(fields.underlying_code) ? fields.underlying_code : null;
|
|
191
201
|
return {
|
|
192
202
|
run_id: runId,
|
|
193
203
|
lease_id: validStackLeaseId(fields.lease_id),
|
|
@@ -212,6 +222,8 @@ export function buildStackRunOutcomePayload({ fields = {}, exitCode, runId = ran
|
|
|
212
222
|
} : {}),
|
|
213
223
|
...(callbackAuthorizationReason ? { callback_authorization_reason: callbackAuthorizationReason } : {}),
|
|
214
224
|
...(failureMechanism ? { failure_mechanism: failureMechanism } : {}),
|
|
225
|
+
...(fenceScope ? { fence_scope: fenceScope } : {}),
|
|
226
|
+
...(underlyingCode ? { underlying_code: underlyingCode } : {}),
|
|
215
227
|
};
|
|
216
228
|
}
|
|
217
229
|
|
|
@@ -244,13 +256,21 @@ export async function reportStackRunOutcome({ fields, exitCode, runId, opts, env
|
|
|
244
256
|
const first = await channel.callTool(STACK_RUN_OUTCOME_TOOL, payload, { auth: channel.auth, signal: ac.signal });
|
|
245
257
|
// Older servers reject unknown keys before writing. Retry only that exact,
|
|
246
258
|
// explicit rejection with the same run id; never retry an uncertain send.
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
first
|
|
251
|
-
const {
|
|
259
|
+
const refusedKey = ["failure_mechanism", "fence_scope"].find((key) => payload[key] &&
|
|
260
|
+
typeof first?.data?.message === "string" && first.data.message.startsWith(`unknown key '${key}'`));
|
|
261
|
+
if (!ac.signal.aborted && refusedKey && first?.ok && first?.isError &&
|
|
262
|
+
first?.data?.code === "INVALID_STACK_RUN_OUTCOME") {
|
|
263
|
+
const { [refusedKey]: _omitted, ...legacyPayload } = payload;
|
|
252
264
|
return await channel.callTool(STACK_RUN_OUTCOME_TOOL, legacyPayload, { auth: channel.auth, signal: ac.signal });
|
|
253
265
|
}
|
|
266
|
+
// BOT-3109: a server from before the requester category refuses it (or its underlying_code).
|
|
267
|
+
// Record exactly what that server would have got from an older CLI: the platform code.
|
|
268
|
+
const refusedRequester = payload.category === "requester" && payload.underlying_code && typeof first?.data?.message === "string" &&
|
|
269
|
+
(first.data.message.startsWith("unknown key 'underlying_code'") || first.data.message.startsWith("category must be"));
|
|
270
|
+
if (!ac.signal.aborted && refusedRequester && first?.ok && first?.isError && first?.data?.code === "INVALID_STACK_RUN_OUTCOME") {
|
|
271
|
+
const { underlying_code: underlying, ...rest } = payload;
|
|
272
|
+
return await channel.callTool(STACK_RUN_OUTCOME_TOOL, { ...rest, category: "platform", code: underlying }, { auth: channel.auth, signal: ac.signal });
|
|
273
|
+
}
|
|
254
274
|
return first;
|
|
255
275
|
});
|
|
256
276
|
const result = await Promise.race([call.catch(() => null), timeout]);
|
package/src/stack.mjs
CHANGED
|
@@ -60,6 +60,7 @@ import { stackServicesForForwards } from "./stack-service-discovery.mjs";
|
|
|
60
60
|
import { ENGINE_PUBLISHES_BEYOND_LOOPBACK, engineSocketIsLocal, formatPublishIp, isLoopbackIp, publishIpFix, resolvePublishIp } from "./docker-publish-ip.mjs";
|
|
61
61
|
import { bold, dim, yellow } from "./utils.mjs";
|
|
62
62
|
import { STACK_RELEASE_INTENT_ENV, classifyStackRunOutcome, observeStackRunSignal, ownerReleaseHeld, reportStackRunOutcome } from "./stack-run-outcome.mjs";
|
|
63
|
+
import { REQUESTER_NETWORK_CANDIDATE_CODES, REQUESTER_NETWORK_LOST_CODE, createRequesterNetworkWatch, requesterNetworkLostMessage } from "./requester-network.mjs";
|
|
63
64
|
|
|
64
65
|
export const STACK_SCHEMA_VERSION = 1;
|
|
65
66
|
|
|
@@ -416,6 +417,11 @@ ${bold("REMOTE LEASES")} (BOT-2142, stack run)
|
|
|
416
417
|
(BUNDLE_UNSAFE_LINK: it resolves outside the stack); no lease is created
|
|
417
418
|
6 TUNNEL_LOST the forward exited mid-child: the child is stopped, the lease released,
|
|
418
419
|
park_reason tunnel_lost (its failures were transport, not product)
|
|
420
|
+
6 REQUESTER_NETWORK_LOST one of TUNNEL_LOST, CALLBACK_TRANSPORT_LOST or LEASE_HEARTBEAT_UNANSWERED,
|
|
421
|
+
while THIS machine's own network changed (macOS: primary interface or IPv4,
|
|
422
|
+
or configd saw it detach; Linux: default route). underlying_code keeps the
|
|
423
|
+
original code, requester_network {lost_at, evidence} says when; recorded as
|
|
424
|
+
category requester, not platform (BOT-3109). Undecided → the original code.
|
|
419
425
|
Every remote failure after the lease exists still runs \`stack done\` and closes the forwards.
|
|
420
426
|
|
|
421
427
|
${bold("HOST_CAPACITY_SOURCE_STALE")} (BOT-2024)
|
|
@@ -1488,13 +1494,52 @@ export const REMOTE_REFUSAL_GUIDANCE = Object.freeze({
|
|
|
1488
1494
|
|
|
1489
1495
|
const AGENT_DEEPLINK_REQUIRED_GUIDANCE = "the control plane refused this stack because its owning agent has no canonical profile or native session link. Reconnect with `bb register`, then retry. No lease was created.";
|
|
1490
1496
|
|
|
1497
|
+
const STALE_CONTEXT_TICKET_RE = /^[A-Z][A-Z0-9]{1,15}-[0-9]{1,12}$/;
|
|
1498
|
+
|
|
1499
|
+
/**
|
|
1500
|
+
* BOT-2977: the one command that re-registers this worktree for the requested
|
|
1501
|
+
* ticket. `bb doctor --fix` registers the ticket its BRANCH names, so it is only
|
|
1502
|
+
* offered when that is the requested ticket. `branchTicket` undefined means the
|
|
1503
|
+
* branch is unknown: say nothing rather than guess. Never retried automatically.
|
|
1504
|
+
*/
|
|
1505
|
+
function staleContextRecovery(requested, branchTicket) {
|
|
1506
|
+
if (branchTicket === undefined || requested === "unknown") return "";
|
|
1507
|
+
const branch = typeof branchTicket === "string" && STALE_CONTEXT_TICKET_RE.test(branchTicket) ? branchTicket : null;
|
|
1508
|
+
if (branch === requested) return `Run \`bb doctor --fix\` in this worktree to re-register it for ${requested}, then retry. `;
|
|
1509
|
+
return `${branch ? `This worktree's branch is for ${branch}` : "This worktree's branch names no ticket"}, so \`bb doctor --fix\` here would not register ${requested}: `
|
|
1510
|
+
+ `run \`bb doctor --fix\` in ${requested}'s worktree, or call register_agent with ticket_id ${requested} in this session, then retry. `;
|
|
1511
|
+
}
|
|
1512
|
+
|
|
1513
|
+
/** BOT-2977: the fence the server applied (`session` for a tier-3 session, else `agent`). */
|
|
1514
|
+
export function staleContextFenceScope(data) {
|
|
1515
|
+
return data?.code === "STALE_AGENT_CONTEXT" && (data.fence_scope === "session" || data.fence_scope === "agent") ? data.fence_scope : null;
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
/** BOT-2977: the ticket this checkout's branch names (null when none), read only for a stale-context refusal. */
|
|
1519
|
+
export async function currentBranchTicket(cwd = process.cwd()) {
|
|
1520
|
+
try {
|
|
1521
|
+
const git = spawnSync("git", ["rev-parse", "--abbrev-ref", "HEAD"], { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 5000 });
|
|
1522
|
+
if (git.status !== 0 || typeof git.stdout !== "string") return undefined;
|
|
1523
|
+
const branch = git.stdout.trim();
|
|
1524
|
+
// A detached HEAD names no branch, so the branch's ticket is unknown, not absent.
|
|
1525
|
+
if (!branch || branch === "HEAD") return undefined;
|
|
1526
|
+
const match = branch.match(/(?:^|\/)\s*(bot|ent)-(\d+)(?:-|$)/i);
|
|
1527
|
+
return match ? `${match[1].toUpperCase()}-${match[2]}` : null;
|
|
1528
|
+
} catch {
|
|
1529
|
+
return undefined;
|
|
1530
|
+
}
|
|
1531
|
+
}
|
|
1532
|
+
|
|
1491
1533
|
/** The refusal text for a receipt: the server's message, else the CLI's guidance, else the code. */
|
|
1492
|
-
export function leaseRefusalError(data) {
|
|
1534
|
+
export function leaseRefusalError(data, { branchTicket } = {}) {
|
|
1493
1535
|
if (data?.code === "STALE_AGENT_CONTEXT") {
|
|
1494
1536
|
// These identifiers are shown only in the local receipt; never copy an
|
|
1495
1537
|
// arbitrary server string into it or into the separate telemetry envelope.
|
|
1496
|
-
const ticket = (value) => typeof value === "string" &&
|
|
1497
|
-
|
|
1538
|
+
const ticket = (value) => typeof value === "string" && STALE_CONTEXT_TICKET_RE.test(value) ? value : "unknown";
|
|
1539
|
+
const requested = ticket(data.requested_ticket_id);
|
|
1540
|
+
return `STALE_AGENT_CONTEXT: registered ticket ${ticket(data.registered_ticket_id)}; requested ticket ${requested}. No lease was created. `
|
|
1541
|
+
+ staleContextRecovery(requested, branchTicket)
|
|
1542
|
+
+ "Run `bb help register_agent` to inspect the supported session registration options before retrying.";
|
|
1498
1543
|
}
|
|
1499
1544
|
if (data?.lease_id && ["STACK_CONTEXT_BIND_FAILED", "STACK_CONTEXT_ALREADY_BOUND"].includes(data?.code)) {
|
|
1500
1545
|
return `${data.message || data.code}; lease ${data.lease_id} is ${data.state ?? "unknown"}. ` +
|
|
@@ -2651,6 +2696,8 @@ async function cmdUpOnce(opts, {
|
|
|
2651
2696
|
machineUuidFn = machineUuid,
|
|
2652
2697
|
assertIsolated = validateLocalExecTarget,
|
|
2653
2698
|
refreshAuth = refreshRejectedCacheAuth,
|
|
2699
|
+
// BOT-2977: the ticket this checkout's branch names, read only for a stale-context refusal.
|
|
2700
|
+
branchTicket = currentBranchTicket,
|
|
2654
2701
|
reportLaneStatus = writeLaneStatus,
|
|
2655
2702
|
waitCapacity = defaultWaitCapacity,
|
|
2656
2703
|
reauth = (current, kind) => reauthenticateParkedLease(current, kind),
|
|
@@ -2791,8 +2838,11 @@ async function cmdUpOnce(opts, {
|
|
|
2791
2838
|
if (!d.success) {
|
|
2792
2839
|
const hint = worktreeRegistrationHint(d.code, auth) ?? stackPathRootRefusalNotice(d.code, d.message);
|
|
2793
2840
|
if (hint) process.stderr.write(`${hint}\n`);
|
|
2841
|
+
const fenceScope = staleContextFenceScope(d);
|
|
2842
|
+
const staleBranchTicket = d.code === "STALE_AGENT_CONTEXT" ? await branchTicket() : undefined;
|
|
2794
2843
|
return emitResult(buildReceipt({
|
|
2795
|
-
command: "up", outcome: "error", code: d.code, ...capacityStaleReceiptFields(d), error: leaseRefusalError(d), slot,
|
|
2844
|
+
command: "up", outcome: "error", code: d.code, ...capacityStaleReceiptFields(d), error: leaseRefusalError(d, { branchTicket: staleBranchTicket }), slot,
|
|
2845
|
+
...(fenceScope ? { fence_scope: fenceScope } : {}),
|
|
2796
2846
|
}), opts, leaseRefusalExit(d.code));
|
|
2797
2847
|
}
|
|
2798
2848
|
let leaseId = d.lease_id;
|
|
@@ -3907,8 +3957,23 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
|
|
|
3907
3957
|
// no claim exists, which is a safe, explicit disposition rather than omission.
|
|
3908
3958
|
// BOT-1803: whether this run re-adopted / re-provisioned an idle lease, and its stack digest.
|
|
3909
3959
|
const freshBox = () => ({ incident: null, execution: null, idleLease: null, webUrl: null, portBlock: { released: false, reason: "no port-block claim was created before stack allocation" } });
|
|
3910
|
-
|
|
3960
|
+
const incidentBox = freshBox();
|
|
3911
3961
|
const startedAtMs = (adapters.now ?? Date.now)();
|
|
3962
|
+
// BOT-3109: watch this machine's own primary network for the run's life (null disables).
|
|
3963
|
+
let networkWatch = null;
|
|
3964
|
+
try {
|
|
3965
|
+
networkWatch = adapters.requesterNetwork === undefined ? createRequesterNetworkWatch() : adapters.requesterNetwork;
|
|
3966
|
+
networkWatch?.start?.();
|
|
3967
|
+
} catch { networkWatch = null; }
|
|
3968
|
+
try {
|
|
3969
|
+
return await runStackLifecycleWatched(opts, childArgv, adapters, incidentBox, freshBox, startedAtMs, networkWatch);
|
|
3970
|
+
} finally {
|
|
3971
|
+
try { networkWatch?.stop?.(); } catch { /* a probe never changes the run */ }
|
|
3972
|
+
}
|
|
3973
|
+
}
|
|
3974
|
+
|
|
3975
|
+
async function runStackLifecycleWatched(opts, childArgv, adapters, firstBox, freshBox, startedAtMs, networkWatch) {
|
|
3976
|
+
let incidentBox = firstBox;
|
|
3912
3977
|
const once = async (attemptOpts = opts) => {
|
|
3913
3978
|
try {
|
|
3914
3979
|
return await runStackLifecycleOnce(attemptOpts, childArgv, adapters, incidentBox);
|
|
@@ -3940,7 +4005,8 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
|
|
|
3940
4005
|
const withPortBlock = incidentBox.portBlock && !withIdle?.portBlock ? { ...withIdle, portBlock: incidentBox.portBlock } : withIdle;
|
|
3941
4006
|
const withWebUrl = incidentBox.webUrl && withPortBlock && !withPortBlock.webUrl ? { ...withPortBlock, webUrl: incidentBox.webUrl } : withPortBlock;
|
|
3942
4007
|
const withStopSignal = incidentBox.stopSignal && withWebUrl ? { ...withWebUrl, stopSignal: incidentBox.stopSignal } : withWebUrl;
|
|
3943
|
-
const
|
|
4008
|
+
const withOutage = incidentBox.callbackOutage && withStopSignal ? { ...withStopSignal, callbackOutage: incidentBox.callbackOutage } : withStopSignal;
|
|
4009
|
+
const final = await applyRequesterNetworkLoss(withOutage, networkWatch, { failedAtMs: incidentBox.lossAtMs });
|
|
3944
4010
|
// BOT-2467: non-enumerable, so the receipt fields and result comparisons never see it.
|
|
3945
4011
|
if (final && typeof final === "object" && incidentBox.outcomeChannel) {
|
|
3946
4012
|
Object.defineProperty(final, OUTCOME_CHANNEL, { value: incidentBox.outcomeChannel, enumerable: false });
|
|
@@ -3948,6 +4014,36 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
|
|
|
3948
4014
|
return final;
|
|
3949
4015
|
}
|
|
3950
4016
|
|
|
4017
|
+
/**
|
|
4018
|
+
* BOT-3109: a run that ended on a transport or control-plane loss while THIS machine's own
|
|
4019
|
+
* network changed is the requester's loss, not BotBuddy's: code REQUESTER_NETWORK_LOST (category
|
|
4020
|
+
* `requester`), the original code kept as `underlyingCode`, and one plain stderr line. Anything
|
|
4021
|
+
* short of a proven change (no change, an undecided or throwing probe, no probe) returns the
|
|
4022
|
+
* result object untouched, so its receipt is byte-identical to before.
|
|
4023
|
+
*/
|
|
4024
|
+
export async function applyRequesterNetworkLoss(result, watch, { failedAtMs = null, stderrLine = (line) => process.stderr.write(`${line}\n`) } = {}) {
|
|
4025
|
+
if (!result || typeof result !== "object" || !REQUESTER_NETWORK_CANDIDATE_CODES.includes(result.code) || typeof watch?.classify !== "function") return result;
|
|
4026
|
+
// An unproven teardown (a lease that may be leaked) is never relabelled: it stays the platform failure it is today.
|
|
4027
|
+
// CALLBACK_CLEANUP_UNCERTAIN exits CLEANUP_FAILED with cleanup.ok true, so the exit code is checked too.
|
|
4028
|
+
// A TUNNEL_LOST whose callback close threw keeps exit 6 today, so it says so on the result instead.
|
|
4029
|
+
if (result.cleanup?.ok === false || result.exitCode === EXIT.CLEANUP_FAILED || result.callbackCleanupUncertain === true) return result;
|
|
4030
|
+
let network;
|
|
4031
|
+
try {
|
|
4032
|
+
network = await watch.classify({ failedAtMs });
|
|
4033
|
+
} catch {
|
|
4034
|
+
return result;
|
|
4035
|
+
}
|
|
4036
|
+
if (network?.decision !== "changed" || !Number.isFinite(network.atMs)) return result;
|
|
4037
|
+
const message = requesterNetworkLostMessage(network, result.code);
|
|
4038
|
+
try { stderrLine(message); } catch { /* the receipt still carries it */ }
|
|
4039
|
+
return {
|
|
4040
|
+
...result, code: REQUESTER_NETWORK_LOST_CODE, underlyingCode: result.code,
|
|
4041
|
+
requesterNetwork: { lost_at: new Date(network.atMs).toISOString(), evidence: network.evidence },
|
|
4042
|
+
// The original diagnostic stays on the receipt, so a wrong attribution can still be seen.
|
|
4043
|
+
error: message, underlyingError: result.error ?? null,
|
|
4044
|
+
};
|
|
4045
|
+
}
|
|
4046
|
+
|
|
3951
4047
|
/** BOT-2467: where a lifecycle result carries its outcome-record channel. */
|
|
3952
4048
|
export const OUTCOME_CHANNEL = Symbol.for("botbuddy.stackRunOutcomeChannel");
|
|
3953
4049
|
|
|
@@ -4272,6 +4368,8 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
4272
4368
|
const stopChild = (signal = "SIGTERM", forcedCode = null) => {
|
|
4273
4369
|
void closeOrigin();
|
|
4274
4370
|
if (!child || childResult) return;
|
|
4371
|
+
// BOT-3109: when a loss stopped the child, so the requester probe weighs the network then, not after cleanup.
|
|
4372
|
+
if (forcedCode != null && forcedStopCode == null) incidentBox.lossAtMs ??= Date.now();
|
|
4275
4373
|
forcedStopCode ??= forcedCode;
|
|
4276
4374
|
childGroupSignal(child, signal);
|
|
4277
4375
|
const kill = clock.setTimeout(() => childGroupSignal(child, "SIGKILL"), 10_000);
|
|
@@ -4762,9 +4860,13 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
4762
4860
|
// an idle mode mismatch) is not proof that its ports are free. Keep the
|
|
4763
4861
|
// claim until owner-safe cleanup is verified; never reap a possibly live batch.
|
|
4764
4862
|
if (request.data?.lease_id) retainBlock = true;
|
|
4863
|
+
// BOT-2977: name the re-registration command; the refusal itself is never retried here.
|
|
4864
|
+
const fenceScope = staleContextFenceScope(request.data);
|
|
4865
|
+
const branchTicket = request.data?.code === "STALE_AGENT_CONTEXT" ? await (adapters.branchTicket ?? currentBranchTicket)() : undefined;
|
|
4765
4866
|
return {
|
|
4766
4867
|
exitCode: leaseRefusalExit(request.data?.code), outcome: "error", code: request.data?.code,
|
|
4767
|
-
...
|
|
4868
|
+
...(fenceScope ? { fenceScope } : {}),
|
|
4869
|
+
...capacityStaleReceiptFields(request.data), error: leaseRefusalError(request.data, { branchTicket }),
|
|
4768
4870
|
...(request.data?.sqlstate ? { sqlstate: request.data.sqlstate } : {}),
|
|
4769
4871
|
...(request.data.lease_id ? { leaseId: request.data.lease_id, state: request.data.state ?? null } : {}),
|
|
4770
4872
|
};
|
|
@@ -5125,7 +5227,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
5125
5227
|
if (outage.recovered > (previous?.recovered ?? 0)) stderrLine("stack: the control plane answered again; the callback route stayed up");
|
|
5126
5228
|
},
|
|
5127
5229
|
});
|
|
5128
|
-
nativeCallback.onExit((exit) => { if (childResult) return; callbackLost = exit; fencing = true; stopChild("SIGTERM", EXIT.LEASE_FAILED); });
|
|
5230
|
+
nativeCallback.onExit((exit) => { if (childResult) return; incidentBox.lossAtMs ??= Date.now(); callbackLost = exit; fencing = true; stopChild("SIGTERM", EXIT.LEASE_FAILED); });
|
|
5129
5231
|
placement.callbackTransport = NATIVE_CALLBACK_TRANSPORT;
|
|
5130
5232
|
} catch (error) {
|
|
5131
5233
|
await closeTunnel(); const cleanup = await requestCleanup();
|
|
@@ -5140,6 +5242,8 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
5140
5242
|
const forwardTarget = sshTarget;
|
|
5141
5243
|
tunnel.onExit((exit) => {
|
|
5142
5244
|
if (childResult) return;
|
|
5245
|
+
// BOT-3109: when the loss was seen, even before a child exists (stopChild stamps only a running one).
|
|
5246
|
+
incidentBox.lossAtMs ??= Date.now();
|
|
5143
5247
|
tunnelLost = { ...exit, target: forwardTarget };
|
|
5144
5248
|
stopChild("SIGTERM", EXIT.LEASE_FAILED);
|
|
5145
5249
|
});
|
|
@@ -5222,7 +5326,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
5222
5326
|
// The forwards died between being verified and the child starting.
|
|
5223
5327
|
await closeTunnel();
|
|
5224
5328
|
const cleanup = await requestCleanup();
|
|
5225
|
-
return tunnelLostResult(tunnelLost, { leaseId, cleanup, childExitCode: null });
|
|
5329
|
+
return tunnelLostResult(tunnelLost, { leaseId, cleanup, childExitCode: null, ...(callbackCleanupError ? { callbackCleanupUncertain: true } : {}) });
|
|
5226
5330
|
}
|
|
5227
5331
|
child = startChild(childArgv, childEnv, childCwd);
|
|
5228
5332
|
leaseIdleTtlSecs = current.data.idle_ttl_secs ?? opts.idleTtl ?? 1800;
|
|
@@ -5337,7 +5441,9 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
5337
5441
|
return { exitCode: cleanup.ok && !callbackCleanupError ? EXIT.LEASE_FAILED : EXIT.CLEANUP_FAILED, outcome: "failed", code: authorizationLost ? "CALLBACK_AUTHORIZATION_LOST" : "CALLBACK_TRANSPORT_LOST", callbackCause, ...(callbackAuthorizationReason ? { callbackAuthorizationReason } : {}), leaseId, childExitCode: childResult.code, cleanup, fenced: true, error: `callback transport authorization or runtime context was lost (${callbackCause}${callbackAuthorizationReason ? `: ${callbackAuthorizationReason}` : ""}); child stopped` };
|
|
5338
5442
|
}
|
|
5339
5443
|
if (tunnelLost && !receivedSignal) {
|
|
5340
|
-
return tunnelLostResult(tunnelLost, { leaseId, cleanup, childExitCode: childResult.code, fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {})
|
|
5444
|
+
return tunnelLostResult(tunnelLost, { leaseId, cleanup, childExitCode: childResult.code, fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {}),
|
|
5445
|
+
// BOT-3109: not a receipt field; it keeps an unproven callback close out of the requester relabel.
|
|
5446
|
+
...(callbackCleanupError ? { callbackCleanupUncertain: true } : {}) });
|
|
5341
5447
|
}
|
|
5342
5448
|
if (preempted && !receivedSignal) {
|
|
5343
5449
|
return {
|
|
@@ -5412,6 +5518,10 @@ export function runReceiptFields(result) {
|
|
|
5412
5518
|
// BOT-1797: carry the server's typed refusal code (e.g. stack_path_root_refused)
|
|
5413
5519
|
// so automation can branch on it instead of matching the message text.
|
|
5414
5520
|
command: "run", outcome: ownerReleaseHeld(result) ? "released" : result.outcome, code: result.code ?? null, lease_id: result.leaseId ?? null,
|
|
5521
|
+
// BOT-3109: a requester-side network loss names the code it replaced and when this machine lost its network.
|
|
5522
|
+
...(result.code === REQUESTER_NETWORK_LOST_CODE && REQUESTER_NETWORK_CANDIDATE_CODES.includes(result.underlyingCode)
|
|
5523
|
+
? { underlying_code: result.underlyingCode, underlying_error: result.underlyingError ?? null,
|
|
5524
|
+
...(result.requesterNetwork ? { requester_network: result.requesterNetwork } : {}) } : {}),
|
|
5415
5525
|
// BOT-2881: the signal that ended this run and what is known of its source (never on a run no signal touched).
|
|
5416
5526
|
...(result.stopSignal ? { stop_signal: result.stopSignal } : {}),
|
|
5417
5527
|
...(result.state ? { state: result.state } : {}),
|
|
@@ -5455,6 +5565,9 @@ export function runReceiptFields(result) {
|
|
|
5455
5565
|
...(result.callbackAuthorizationReason ? { callback_authorization_reason: result.callbackAuthorizationReason } : {}),
|
|
5456
5566
|
...(result.code === "REQUESTER_ORIGIN_DIGEST_UNAVAILABLE" && ["origin_config_missing", "origin_digest_missing"].includes(result.failureMechanism)
|
|
5457
5567
|
? { failure_mechanism: result.failureMechanism } : {}),
|
|
5568
|
+
// BOT-2977: which fence refused a STALE_AGENT_CONTEXT run (session | agent).
|
|
5569
|
+
...(result.code === "STALE_AGENT_CONTEXT" && (result.fenceScope === "session" || result.fenceScope === "agent")
|
|
5570
|
+
? { fence_scope: result.fenceScope } : {}),
|
|
5458
5571
|
// BOT-2957: the control-plane gaps this run's callback route rode through (never on a run that had none).
|
|
5459
5572
|
...(result.callbackOutage ? { callback_outage: result.callbackOutage } : {}),
|
|
5460
5573
|
// BOT-2780: the reserved requester origin generated auth links name (never a credential).
|