@love-moon/conductor-cli 0.7.6 → 0.7.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/package.json +5 -5
- package/src/daemon.js +927 -64
- package/src/handoff-log-mask.js +43 -0
package/src/daemon.js
CHANGED
|
@@ -3,6 +3,7 @@ import path from "node:path";
|
|
|
3
3
|
import os from "node:os";
|
|
4
4
|
import { createRequire } from "node:module";
|
|
5
5
|
import { spawn, spawnSync } from "node:child_process";
|
|
6
|
+
import { randomUUID } from "node:crypto";
|
|
6
7
|
import { fileURLToPath } from "node:url";
|
|
7
8
|
|
|
8
9
|
import dotenv from "dotenv";
|
|
@@ -14,6 +15,7 @@ import {
|
|
|
14
15
|
loadConfig,
|
|
15
16
|
ConfigFileNotFound,
|
|
16
17
|
ProjectContext,
|
|
18
|
+
DurableUpstreamOutboxStore,
|
|
17
19
|
} from "@love-moon/conductor-sdk";
|
|
18
20
|
import { DaemonLogCollector } from "./log-collector.js";
|
|
19
21
|
import { envForExplicitConfigFile } from "./config-env.js";
|
|
@@ -58,7 +60,9 @@ import {
|
|
|
58
60
|
import {
|
|
59
61
|
maskHandoffUrlForLogs,
|
|
60
62
|
maskErrorForLogs,
|
|
63
|
+
redactSecretsForLogs,
|
|
61
64
|
} from "./handoff-log-mask.js";
|
|
65
|
+
import { StringDecoder } from "node:string_decoder";
|
|
62
66
|
|
|
63
67
|
dotenv.config();
|
|
64
68
|
|
|
@@ -447,7 +451,7 @@ function normalizeOptionalString(value) {
|
|
|
447
451
|
// Re-export the handoff-URL masking helpers so existing external imports
|
|
448
452
|
// keep working. Implementation lives in a dependency-free module so unit
|
|
449
453
|
// tests can import it without pulling in conductor-sdk and friends.
|
|
450
|
-
export { maskHandoffUrlForLogs, maskErrorForLogs };
|
|
454
|
+
export { maskHandoffUrlForLogs, maskErrorForLogs, redactSecretsForLogs };
|
|
451
455
|
|
|
452
456
|
function normalizeTerminalResumeStrategy(value) {
|
|
453
457
|
const normalized = normalizeOptionalString(value);
|
|
@@ -628,6 +632,52 @@ function stripPtyTaskScopedEnv(source) {
|
|
|
628
632
|
return env;
|
|
629
633
|
}
|
|
630
634
|
|
|
635
|
+
/**
|
|
636
|
+
* Purge terminal `task_status_update` events left undelivered by a previous run
|
|
637
|
+
* of `taskId` in `taskDir`, before a restart re-uses that same directory.
|
|
638
|
+
*
|
|
639
|
+
* The fire's durable upstream outbox is stored under the fire's cwd, keyed by
|
|
640
|
+
* the `task:<id>` delivery scope. An in-place restart intentionally re-uses the
|
|
641
|
+
* cwd, so without this purge the new run flushes the old run's KILLED/COMPLETED
|
|
642
|
+
* event on startup and kills the task it just resumed. Best-effort: a restart
|
|
643
|
+
* must never fail because of outbox bookkeeping.
|
|
644
|
+
*/
|
|
645
|
+
function dropSupersededTerminalStatusEvents(taskDir, taskId) {
|
|
646
|
+
if (!taskDir || !taskId) return;
|
|
647
|
+
try {
|
|
648
|
+
const store = DurableUpstreamOutboxStore.forProjectPath(taskDir, `task:${taskId}`);
|
|
649
|
+
// Distinguish "the purge failed" from "the purge does not exist here".
|
|
650
|
+
// The CLI loads the INSTALLED copy of conductor-sdk, not this repo's
|
|
651
|
+
// source, so an un-rebuilt or mismatched package silently lacks this
|
|
652
|
+
// method — and a bare try/catch would swallow the resulting TypeError
|
|
653
|
+
// exactly like a benign fs error, leaving every restart unprotected with
|
|
654
|
+
// no way to tell from the log. Say so explicitly instead. We still do not
|
|
655
|
+
// throw: an unprotected restart usually succeeds, whereas refusing to
|
|
656
|
+
// spawn would break restart outright.
|
|
657
|
+
if (typeof store?.dropPendingTerminalStatusEvents !== "function") {
|
|
658
|
+
logError(
|
|
659
|
+
`Cannot purge superseded terminal status events for task ${taskId}: the installed ` +
|
|
660
|
+
`@love-moon/conductor-sdk has no DurableUpstreamOutboxStore.dropPendingTerminalStatusEvents. ` +
|
|
661
|
+
`This restart is unprotected — a terminal status left by the previous run may kill it. ` +
|
|
662
|
+
`Rebuild and reinstall the SDK (make install-cli).`,
|
|
663
|
+
);
|
|
664
|
+
return;
|
|
665
|
+
}
|
|
666
|
+
const dropped = store.dropPendingTerminalStatusEvents();
|
|
667
|
+
if (dropped.length > 0) {
|
|
668
|
+
log(
|
|
669
|
+
`Dropped ${dropped.length} superseded terminal status event(s) for task ${taskId} before restart: ${dropped
|
|
670
|
+
.map((entry) => `${entry.payload?.status ?? "?"}@${entry.createdAt}`)
|
|
671
|
+
.join(", ")}`,
|
|
672
|
+
);
|
|
673
|
+
}
|
|
674
|
+
} catch (error) {
|
|
675
|
+
logError(
|
|
676
|
+
`Failed to purge superseded terminal status events for task ${taskId}: ${error?.message || error}`,
|
|
677
|
+
);
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
|
|
631
681
|
function buildPtyTaskEnv(baseEnv = process.env, launchEnv = {}) {
|
|
632
682
|
const parentEnv = stripPtyTaskScopedEnv(baseEnv);
|
|
633
683
|
const taskLaunchEnv = stripPtyTaskScopedEnv(launchEnv);
|
|
@@ -827,6 +877,11 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
827
877
|
const symlinkSyncFn = deps.symlinkSync || fs.symlinkSync;
|
|
828
878
|
const unlinkSyncFn = deps.unlinkSync || fs.unlinkSync;
|
|
829
879
|
const renameSyncFn = deps.renameSync || fs.renameSync;
|
|
880
|
+
// Used to recover a crashed fire's output from its log file when the daemon
|
|
881
|
+
// could not observe the process directly (tmux mode).
|
|
882
|
+
const openSyncFn = deps.openSync || fs.openSync;
|
|
883
|
+
const readSyncFn = deps.readSync || fs.readSync;
|
|
884
|
+
const closeSyncFn = deps.closeSync || fs.closeSync;
|
|
830
885
|
const createWriteStreamFn = deps.createWriteStream || fs.createWriteStream;
|
|
831
886
|
const fetchFn = deps.fetch || fetch;
|
|
832
887
|
const createRtcPeerConnection = deps.createRtcPeerConnection || null;
|
|
@@ -911,6 +966,231 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
911
966
|
return `${buildFireTmuxSessionPrefix(taskId)}${uniq}`;
|
|
912
967
|
}
|
|
913
968
|
|
|
969
|
+
// Observability for spawned fires (RFC: fork-spawn black box).
|
|
970
|
+
//
|
|
971
|
+
// When a forked/branched task's backend dies during startup, the daemon
|
|
972
|
+
// previously reported a bare `exited with code N` and wrote nothing to the
|
|
973
|
+
// daemon log, so the real cause (backend auth failure, missing CLI, tmux
|
|
974
|
+
// launch error, …) was unrecoverable after the fact — even with DB access
|
|
975
|
+
// the only artifact was a generic `fire_exit`. We keep a bounded tail of the
|
|
976
|
+
// child's output so an abnormal exit can name its own cause.
|
|
977
|
+
//
|
|
978
|
+
// The tail is masked with `maskHandoffUrlForLogs` before it reaches any log
|
|
979
|
+
// or status summary: the handoff prompt embeds a share token, and a backend
|
|
980
|
+
// that echoes its argv on failure would otherwise leak a read-grant for the
|
|
981
|
+
// entire transcript into the daemon log and the task status summary.
|
|
982
|
+
const CHILD_OUTPUT_CAPTURE_LIMIT = 4000;
|
|
983
|
+
const CHILD_OUTPUT_SUMMARY_LIMIT = 400;
|
|
984
|
+
|
|
985
|
+
// Sentinel appended to a tmux-hosted fire's log by the wrapper shell (see
|
|
986
|
+
// `spawnFireProcess`). It is the *only* channel through which the daemon
|
|
987
|
+
// can learn the exit code of a process it does not own. The full marker is
|
|
988
|
+
// `[conductor-fire-exit:<per-spawn nonce>] code=<n>`.
|
|
989
|
+
const FIRE_EXIT_MARKER_PREFIX = "[conductor-fire-exit:";
|
|
990
|
+
|
|
991
|
+
const escapeForRegExp = (value) => String(value).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
992
|
+
|
|
993
|
+
// Per-spawn nonce embedded in the marker. Without it the marker is a fixed
|
|
994
|
+
// string that anything writing to the log can forge — and the log is a
|
|
995
|
+
// verbatim copy of the fire's stdout. A fire that merely *prints* the
|
|
996
|
+
// literal (an agent task reading a conductor.log, or editing this file)
|
|
997
|
+
// would plant `[conductor-fire-exit] code=0` in its own log; if that fire
|
|
998
|
+
// were then SIGKILLed, leaving no genuine marker, the reaper would read the
|
|
999
|
+
// forged one and report COMPLETED for a task that was killed. Dogfooding
|
|
1000
|
+
// this repo makes that near-certain rather than theoretical.
|
|
1001
|
+
//
|
|
1002
|
+
// The nonce also means a previous run's marker can never be mistaken for
|
|
1003
|
+
// this run's, independently of the `logStartOffset` watermark.
|
|
1004
|
+
function buildFireExitMarkerToken() {
|
|
1005
|
+
return randomUUID().replace(/-/g, "").slice(0, 12);
|
|
1006
|
+
}
|
|
1007
|
+
|
|
1008
|
+
// Last exit code recorded by *this run's* wrapper shell in `raw`, or null
|
|
1009
|
+
// when the marker is absent (fire was SIGKILLed / the tmux server died
|
|
1010
|
+
// before the wrapper could run). Anchored at line start so it cannot match
|
|
1011
|
+
// mid-line inside quoted output, and scoped by the run's nonce.
|
|
1012
|
+
function parseFireExitCode(raw, markerToken) {
|
|
1013
|
+
if (!raw || !markerToken) return null;
|
|
1014
|
+
const pattern = new RegExp(
|
|
1015
|
+
`^${escapeForRegExp(FIRE_EXIT_MARKER_PREFIX)}${escapeForRegExp(markerToken)}\\] code=(\\d+)`,
|
|
1016
|
+
"gm",
|
|
1017
|
+
);
|
|
1018
|
+
let match = null;
|
|
1019
|
+
let candidate;
|
|
1020
|
+
// Scan for the *last* match rather than the first: a fresh RegExp per
|
|
1021
|
+
// call, because a shared /g instance carries `lastIndex` between calls.
|
|
1022
|
+
while ((candidate = pattern.exec(raw)) !== null) {
|
|
1023
|
+
match = candidate;
|
|
1024
|
+
}
|
|
1025
|
+
if (!match) return null;
|
|
1026
|
+
const code = Number(match[1]);
|
|
1027
|
+
return Number.isFinite(code) ? code : null;
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
// Read the last `limit` bytes of a log file, never crossing below
|
|
1031
|
+
// `minOffset`. The floor matters because a task's log file is opened with
|
|
1032
|
+
// flags "a" and is therefore *reused* across in-place restarts: without it,
|
|
1033
|
+
// a previous run's trailing bytes (including its exit marker) would be
|
|
1034
|
+
// attributed to the current run. Callers that don't care pass minOffset 0.
|
|
1035
|
+
function readLogTailRaw(logPath, { limit = CHILD_OUTPUT_CAPTURE_LIMIT, minOffset = 0 } = {}) {
|
|
1036
|
+
if (!logPath) return "";
|
|
1037
|
+
try {
|
|
1038
|
+
const stat = statSyncFn(logPath);
|
|
1039
|
+
const size = Number(stat?.size) || 0;
|
|
1040
|
+
// A file smaller than the watermark was truncated or rotated out from
|
|
1041
|
+
// under us, so the watermark no longer refers to anything. Honouring it
|
|
1042
|
+
// would return "" for this record forever — which the reaper reads as
|
|
1043
|
+
// "no exit marker", i.e. a fabricated hard death for a task that may
|
|
1044
|
+
// have finished cleanly. Fall back to reading whatever is there.
|
|
1045
|
+
const floor = size < minOffset ? 0 : Math.max(0, Number(minOffset) || 0);
|
|
1046
|
+
const start = Math.max(floor, size - limit);
|
|
1047
|
+
if (size <= start) return "";
|
|
1048
|
+
const fd = openSyncFn(logPath, "r");
|
|
1049
|
+
try {
|
|
1050
|
+
const length = size - start;
|
|
1051
|
+
const buf = Buffer.alloc(length);
|
|
1052
|
+
readSyncFn(fd, buf, 0, length, start);
|
|
1053
|
+
return buf.toString("utf8");
|
|
1054
|
+
} finally {
|
|
1055
|
+
closeSyncFn(fd);
|
|
1056
|
+
}
|
|
1057
|
+
} catch {
|
|
1058
|
+
return "";
|
|
1059
|
+
}
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
// Size of a log file, or null when it does not exist / cannot be read.
|
|
1063
|
+
// The null case is load-bearing for the reaper: `tee -a` creates the file
|
|
1064
|
+
// the moment it opens it, so a *missing* file means the wrapper never had
|
|
1065
|
+
// a writable log at all (bad cwd, unwritable project dir) — as opposed to
|
|
1066
|
+
// an empty-but-present file, which means the fire really did die before
|
|
1067
|
+
// writing anything.
|
|
1068
|
+
function readLogFileSize(logPath) {
|
|
1069
|
+
if (!logPath) return null;
|
|
1070
|
+
try {
|
|
1071
|
+
return Number(statSyncFn(logPath)?.size) || 0;
|
|
1072
|
+
} catch {
|
|
1073
|
+
return null;
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
// Current size of a log file, or 0 when it doesn't exist / can't be read.
|
|
1078
|
+
// Sampled just before a spawn so later tail reads can be floored at "what
|
|
1079
|
+
// this run wrote".
|
|
1080
|
+
function readLogSize(logPath) {
|
|
1081
|
+
return readLogFileSize(logPath) ?? 0;
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
// How far back to hunt for this run's exit marker, and in what stride.
|
|
1085
|
+
// Bounded so a huge log can never turn one sweep into an unbounded read.
|
|
1086
|
+
const FIRE_EXIT_MARKER_SEARCH_LIMIT = 8 * 1024 * 1024;
|
|
1087
|
+
const FIRE_EXIT_MARKER_SEARCH_CHUNK = 64 * 1024;
|
|
1088
|
+
|
|
1089
|
+
// Find this run's exit code by scanning *backwards* from EOF for its nonce.
|
|
1090
|
+
//
|
|
1091
|
+
// Reading only the last few KB would be wrong: a task's log is not private.
|
|
1092
|
+
// Branch/fork tasks deliberately share a worktree — and therefore a single
|
|
1093
|
+
// `conductor.log` — and a task with no worktree config falls back to the
|
|
1094
|
+
// project directory, so every task in the project appends to the same file.
|
|
1095
|
+
// A neighbouring fire that prints a few KB between our exit and the next
|
|
1096
|
+
// reaper sweep would push our marker out of a fixed tail window. The reaper
|
|
1097
|
+
// would then see "no marker", conclude the fire died hard, and report
|
|
1098
|
+
// KILLED for a task that finished cleanly — quoting the *neighbour's*
|
|
1099
|
+
// output as the cause of death. The marker is this run's identity, not a
|
|
1100
|
+
// property of where it happens to sit in a shared stream, so search for it.
|
|
1101
|
+
function findFireExitCode(logPath, { minOffset = 0, markerToken = "" } = {}) {
|
|
1102
|
+
if (!logPath || !markerToken) return null;
|
|
1103
|
+
let size;
|
|
1104
|
+
try {
|
|
1105
|
+
size = Number(statSyncFn(logPath)?.size) || 0;
|
|
1106
|
+
} catch {
|
|
1107
|
+
return null;
|
|
1108
|
+
}
|
|
1109
|
+
// A file shorter than the watermark was truncated/rotated; the offset no
|
|
1110
|
+
// longer refers to anything, so search the whole file rather than nothing.
|
|
1111
|
+
const floor = size < minOffset ? 0 : Math.max(0, Number(minOffset) || 0);
|
|
1112
|
+
const searchFloor = Math.max(floor, size - FIRE_EXIT_MARKER_SEARCH_LIMIT);
|
|
1113
|
+
let fd;
|
|
1114
|
+
try {
|
|
1115
|
+
fd = openSyncFn(logPath, "r");
|
|
1116
|
+
} catch {
|
|
1117
|
+
return null;
|
|
1118
|
+
}
|
|
1119
|
+
try {
|
|
1120
|
+
let end = size;
|
|
1121
|
+
// Overlap successive chunks by enough to cover a marker split across a
|
|
1122
|
+
// boundary, so a line straddling two reads is still found.
|
|
1123
|
+
const overlap = FIRE_EXIT_MARKER_PREFIX.length + markerToken.length + 32;
|
|
1124
|
+
while (end > searchFloor) {
|
|
1125
|
+
const start = Math.max(searchFloor, end - FIRE_EXIT_MARKER_SEARCH_CHUNK);
|
|
1126
|
+
const length = end - start;
|
|
1127
|
+
const buf = Buffer.alloc(length);
|
|
1128
|
+
readSyncFn(fd, buf, 0, length, start);
|
|
1129
|
+
const code = parseFireExitCode(buf.toString("utf8"), markerToken);
|
|
1130
|
+
if (code !== null) return code;
|
|
1131
|
+
if (start <= searchFloor) break;
|
|
1132
|
+
end = start + overlap;
|
|
1133
|
+
}
|
|
1134
|
+
return null;
|
|
1135
|
+
} catch {
|
|
1136
|
+
return null;
|
|
1137
|
+
} finally {
|
|
1138
|
+
try {
|
|
1139
|
+
closeSyncFn(fd);
|
|
1140
|
+
} catch {
|
|
1141
|
+
// best effort
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
|
|
1146
|
+
// Collapse, redact and clamp a raw output tail so it is safe to put in a
|
|
1147
|
+
// daemon log line or a persisted task status summary.
|
|
1148
|
+
function sanitizeOutputTail(raw, limit = CHILD_OUTPUT_SUMMARY_LIMIT) {
|
|
1149
|
+
// Collapse first, then redact: the mask patterns stop at whitespace, so
|
|
1150
|
+
// a secret split across a newline would otherwise only be half-matched.
|
|
1151
|
+
const collapsed = String(raw || "").replace(/\s+/g, " ").trim();
|
|
1152
|
+
const safe = redactSecretsForLogs(collapsed, [AGENT_TOKEN]).trim();
|
|
1153
|
+
if (!safe) return "";
|
|
1154
|
+
// slice(-(limit - 1)) keeps the ellipsis inside the documented budget.
|
|
1155
|
+
return safe.length > limit ? `…${safe.slice(-(limit - 1))}` : safe;
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
function createChildOutputCapture({ logPath = "", logStartOffset = 0 } = {}) {
|
|
1159
|
+
let buffer = "";
|
|
1160
|
+
// Decode across chunk boundaries: a raw per-chunk toString("utf8") splits
|
|
1161
|
+
// multi-byte characters whenever a chunk ends mid-sequence, which would
|
|
1162
|
+
// put mojibake into the persisted status summary.
|
|
1163
|
+
const decoder = new StringDecoder("utf8");
|
|
1164
|
+
const append = (chunk) => {
|
|
1165
|
+
const text =
|
|
1166
|
+
typeof chunk === "string" ? chunk : decoder.write(Buffer.from(chunk));
|
|
1167
|
+
if (!text) return;
|
|
1168
|
+
buffer += text;
|
|
1169
|
+
if (buffer.length > CHILD_OUTPUT_CAPTURE_LIMIT) {
|
|
1170
|
+
buffer = buffer.slice(-CHILD_OUTPUT_CAPTURE_LIMIT);
|
|
1171
|
+
}
|
|
1172
|
+
};
|
|
1173
|
+
// Attaching an extra `data` listener is safe alongside `.pipe()`: pipe
|
|
1174
|
+
// keeps the stream in flowing mode and additional listeners observe the
|
|
1175
|
+
// same chunks without consuming them.
|
|
1176
|
+
const attach = (stream) => {
|
|
1177
|
+
if (stream && typeof stream.on === "function") {
|
|
1178
|
+
stream.on("data", append);
|
|
1179
|
+
}
|
|
1180
|
+
};
|
|
1181
|
+
|
|
1182
|
+
// In tmux mode the daemon's child is the `tmux new-session` client, and
|
|
1183
|
+
// the fire's own output is redirected to `logPath` *inside* the session
|
|
1184
|
+
// (`… 2>&1 | tee -a <logPath>`). The in-memory buffer therefore only ever
|
|
1185
|
+
// holds tmux's own errors. When it is empty, fall back to the tail of the
|
|
1186
|
+
// log file so the backend's actual crash output is still reported.
|
|
1187
|
+
const readLogTail = () => readLogTailRaw(logPath, { minOffset: logStartOffset });
|
|
1188
|
+
|
|
1189
|
+
const tail = (limit = CHILD_OUTPUT_SUMMARY_LIMIT) =>
|
|
1190
|
+
sanitizeOutputTail(buffer.trim() ? buffer : readLogTail(), limit);
|
|
1191
|
+
return { attach, append, tail };
|
|
1192
|
+
}
|
|
1193
|
+
|
|
914
1194
|
// Spawn the Fire CLI either directly (default) or inside a detached tmux
|
|
915
1195
|
// session (when FIRE_TMUX_MODE_ACTIVE). In tmux mode the returned `child`
|
|
916
1196
|
// is the short-lived `tmux new-session` client; once it exits with code 0
|
|
@@ -924,7 +1204,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
924
1204
|
env,
|
|
925
1205
|
stdio: ["inherit", "pipe", "pipe"],
|
|
926
1206
|
});
|
|
927
|
-
return { child, tmuxSession: null };
|
|
1207
|
+
return { child, tmuxSession: null, exitMarkerToken: "" };
|
|
928
1208
|
}
|
|
929
1209
|
|
|
930
1210
|
const sessionName = buildFireTmuxSessionName(taskId);
|
|
@@ -935,9 +1215,37 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
935
1215
|
// via `tmux a -t <session>` for live observation is useless. With
|
|
936
1216
|
// `tee` the same bytes go to both the pane (visible to whoever
|
|
937
1217
|
// attaches) and the log file (preserved for offline inspection).
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
1218
|
+
//
|
|
1219
|
+
// The trailing exit-marker is what makes a death *inside* the session
|
|
1220
|
+
// observable. The daemon's child is only the `tmux new-session` client,
|
|
1221
|
+
// so once the session exists the daemon has no handle on Fire and never
|
|
1222
|
+
// sees its exit code. `${PIPESTATUS[0]}` recovers Fire's own status from
|
|
1223
|
+
// the pipeline (plain `$?` would be tee's), and appending it to the log
|
|
1224
|
+
// lets the liveness reaper tell "finished normally" from "crashed" —
|
|
1225
|
+
// without which it could only ever guess, and guessing KILLED would
|
|
1226
|
+
// mis-report successful tasks as failures.
|
|
1227
|
+
//
|
|
1228
|
+
// No `exec` in this branch: the wrapper bash must outlive Fire to write
|
|
1229
|
+
// the marker. When Fire is SIGKILLed (or the tmux server dies) the
|
|
1230
|
+
// marker is simply absent, which the reaper reads as "died hard" — the
|
|
1231
|
+
// correct conclusion.
|
|
1232
|
+
//
|
|
1233
|
+
// Each branch owns its own `exec` (or absence of one) and the result is
|
|
1234
|
+
// passed to `bash -c` verbatim. Do NOT re-wrap it in `exec` at the call
|
|
1235
|
+
// site: `exec exec …` is a bash error ("exec: exec: not found", 127),
|
|
1236
|
+
// which in tmux mode means Fire never starts while the tmux client still
|
|
1237
|
+
// exits 0 — a task that hangs at `running` with nothing in the log.
|
|
1238
|
+
const quotedLogPath = logPath ? shellQuoteForBash(logPath) : "";
|
|
1239
|
+
// `\n` before the marker guarantees it starts its own line even when the
|
|
1240
|
+
// fire's last write had no trailing newline — the reader anchors on `^`.
|
|
1241
|
+
const exitMarkerToken = logPath ? buildFireExitMarkerToken() : "";
|
|
1242
|
+
const shellCommand = logPath
|
|
1243
|
+
? `${innerCommandParts.join(" ")} 2>&1 | tee -a ${quotedLogPath}; ` +
|
|
1244
|
+
`__conductor_fire_code=\${PIPESTATUS[0]}; ` +
|
|
1245
|
+
`printf '\\n${FIRE_EXIT_MARKER_PREFIX}${exitMarkerToken}] code=%s\\n' ` +
|
|
1246
|
+
`"$__conductor_fire_code" >> ${quotedLogPath}; ` +
|
|
1247
|
+
`exit $__conductor_fire_code`
|
|
1248
|
+
: `exec ${innerCommandParts.join(" ")}`;
|
|
941
1249
|
|
|
942
1250
|
// Build `-e KEY=VALUE` flags for the new session.
|
|
943
1251
|
//
|
|
@@ -985,7 +1293,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
985
1293
|
cwd,
|
|
986
1294
|
"bash",
|
|
987
1295
|
"-c",
|
|
988
|
-
|
|
1296
|
+
shellCommand,
|
|
989
1297
|
];
|
|
990
1298
|
log(`Spawning Fire via tmux: session=${sessionName} cwd=${cwd}`);
|
|
991
1299
|
const child = spawnFn("tmux", tmuxArgs, {
|
|
@@ -997,7 +1305,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
997
1305
|
if (typeof child.unref === "function") {
|
|
998
1306
|
child.unref();
|
|
999
1307
|
}
|
|
1000
|
-
return { child, tmuxSession: sessionName };
|
|
1308
|
+
return { child, tmuxSession: sessionName, exitMarkerToken };
|
|
1001
1309
|
}
|
|
1002
1310
|
|
|
1003
1311
|
// Async probe: does the named tmux session still exist? Resolves to a
|
|
@@ -1016,34 +1324,45 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1016
1324
|
}
|
|
1017
1325
|
return 5000;
|
|
1018
1326
|
})();
|
|
1019
|
-
|
|
1327
|
+
// Detailed probe: resolves `{ alive, conclusive }`.
|
|
1328
|
+
//
|
|
1329
|
+
// `conclusive` distinguishes "tmux answered, and the session is gone"
|
|
1330
|
+
// (has-session exited non-zero) from "we never got an answer" (spawn error,
|
|
1331
|
+
// tmux binary missing, wedged server hitting the timeout). Both used to
|
|
1332
|
+
// collapse into a bare `false`, which was fine when the only consequence
|
|
1333
|
+
// was dropping a bookkeeping entry. It is NOT fine now that the reaper
|
|
1334
|
+
// turns "not alive" into an authoritative KILLED with a cause of death: one
|
|
1335
|
+
// transient tmux hiccup would otherwise declare every live tmux task dead,
|
|
1336
|
+
// each with a confidently wrong "disappeared without an exit marker".
|
|
1337
|
+
function probeTmuxSession(sessionName) {
|
|
1020
1338
|
return new Promise((resolve) => {
|
|
1021
1339
|
if (!sessionName) {
|
|
1022
|
-
resolve(false);
|
|
1340
|
+
resolve({ alive: false, conclusive: false });
|
|
1023
1341
|
return;
|
|
1024
1342
|
}
|
|
1025
1343
|
let settled = false;
|
|
1026
1344
|
let probe = null;
|
|
1027
1345
|
let timer = null;
|
|
1028
|
-
const settle = (alive) => {
|
|
1346
|
+
const settle = (alive, conclusive) => {
|
|
1029
1347
|
if (settled) return;
|
|
1030
1348
|
settled = true;
|
|
1031
1349
|
if (timer) {
|
|
1032
1350
|
clearTimeout(timer);
|
|
1033
1351
|
timer = null;
|
|
1034
1352
|
}
|
|
1035
|
-
resolve(alive);
|
|
1353
|
+
resolve({ alive, conclusive });
|
|
1036
1354
|
};
|
|
1037
1355
|
try {
|
|
1038
1356
|
probe = spawnFn("tmux", ["has-session", "-t", sessionName], {
|
|
1039
1357
|
stdio: "ignore",
|
|
1040
1358
|
});
|
|
1041
|
-
|
|
1042
|
-
probe.on("
|
|
1359
|
+
// tmux answered: exit 0 = alive, non-zero = genuinely no such session.
|
|
1360
|
+
probe.on("exit", (code) => settle(code === 0, true));
|
|
1361
|
+
// Could not run tmux at all — says nothing about the session.
|
|
1362
|
+
probe.on("error", () => settle(false, false));
|
|
1043
1363
|
timer = setTimeout(() => {
|
|
1044
|
-
// Probe took too long —
|
|
1045
|
-
//
|
|
1046
|
-
// and best-effort kill the stuck child so it doesn't pile up.
|
|
1364
|
+
// Probe took too long — the session state is unknown. Best-effort
|
|
1365
|
+
// kill the stuck child so it doesn't pile up.
|
|
1047
1366
|
try {
|
|
1048
1367
|
if (probe && typeof probe.kill === "function") {
|
|
1049
1368
|
probe.kill("SIGKILL");
|
|
@@ -1054,45 +1373,289 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1054
1373
|
logError(
|
|
1055
1374
|
`tmux has-session probe timed out after ${TMUX_PROBE_TIMEOUT_MS}ms for session ${sessionName}`,
|
|
1056
1375
|
);
|
|
1057
|
-
settle(false);
|
|
1376
|
+
settle(false, false);
|
|
1058
1377
|
}, TMUX_PROBE_TIMEOUT_MS);
|
|
1059
1378
|
if (typeof timer.unref === "function") {
|
|
1060
1379
|
timer.unref();
|
|
1061
1380
|
}
|
|
1062
1381
|
} catch {
|
|
1063
|
-
settle(false);
|
|
1382
|
+
settle(false, false);
|
|
1064
1383
|
}
|
|
1065
1384
|
});
|
|
1066
1385
|
}
|
|
1067
1386
|
|
|
1068
|
-
//
|
|
1069
|
-
//
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1387
|
+
// Boolean form, for the callers that only gate local bookkeeping or a
|
|
1388
|
+
// best-effort kill and are happy to treat "unknown" as "not alive".
|
|
1389
|
+
async function tmuxSessionExists(sessionName) {
|
|
1390
|
+
return (await probeTmuxSession(sessionName)).alive;
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
const BACKEND_STATUS_PROBE_TIMEOUT_MS = (() => {
|
|
1394
|
+
const explicit = Number(config.BACKEND_STATUS_PROBE_TIMEOUT_MS);
|
|
1395
|
+
if (Number.isFinite(explicit) && explicit > 0) {
|
|
1396
|
+
return explicit;
|
|
1397
|
+
}
|
|
1398
|
+
return 10_000;
|
|
1399
|
+
})();
|
|
1400
|
+
|
|
1401
|
+
// Current backend-side status of a task, lowercased, or null when it can't
|
|
1402
|
+
// be determined (HTTP error, unreachable backend, malformed payload).
|
|
1073
1403
|
//
|
|
1074
|
-
//
|
|
1075
|
-
//
|
|
1076
|
-
//
|
|
1077
|
-
//
|
|
1078
|
-
//
|
|
1079
|
-
//
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1404
|
+
// Hard timeout, for the same reason `tmuxSessionExists` has one: this call
|
|
1405
|
+
// is awaited inside the reaper sweep, and the sweep's re-entrancy latch is
|
|
1406
|
+
// only released in a `finally`. A backend that black-holes the connection
|
|
1407
|
+
// would otherwise leave the latch stuck forever — permanently disabling the
|
|
1408
|
+
// only observer of in-session deaths, which is the exact failure this whole
|
|
1409
|
+
// change exists to prevent.
|
|
1410
|
+
async function fetchBackendTaskStatus(taskId) {
|
|
1411
|
+
if (!BACKEND_HTTP || !AGENT_TOKEN || !taskId) return null;
|
|
1412
|
+
// The timeout must cover reading the BODY, not just the headers. `fetch`
|
|
1413
|
+
// resolves as soon as headers arrive, so wrapping only that call leaves
|
|
1414
|
+
// `response.json()` unbounded: a backend that answers 200 and then stalls
|
|
1415
|
+
// the body (half-open proxy, hung route) parks this await forever, the
|
|
1416
|
+
// sweep's re-entrancy latch never clears, and the only observer of
|
|
1417
|
+
// in-session deaths is permanently disabled. `markBackendHttpSuccess` is
|
|
1418
|
+
// likewise deferred until the body actually parsed — crediting the
|
|
1419
|
+
// watchdog for a response we could not read would be a lie.
|
|
1420
|
+
const controller = typeof AbortController !== "undefined" ? new AbortController() : null;
|
|
1421
|
+
try {
|
|
1422
|
+
const status = await withTimeout(
|
|
1423
|
+
(async () => {
|
|
1424
|
+
const response = await fetchFn(`${BACKEND_HTTP}/api/tasks/${taskId}`, {
|
|
1425
|
+
method: "GET",
|
|
1426
|
+
headers: {
|
|
1427
|
+
Authorization: `Bearer ${AGENT_TOKEN}`,
|
|
1428
|
+
Accept: "application/json",
|
|
1429
|
+
},
|
|
1430
|
+
signal: controller?.signal,
|
|
1431
|
+
});
|
|
1432
|
+
if (!response?.ok) return null;
|
|
1433
|
+
const task = await response.json();
|
|
1434
|
+
markBackendHttpSuccess();
|
|
1435
|
+
return String(task?.status || "").trim().toLowerCase() || null;
|
|
1436
|
+
})(),
|
|
1437
|
+
BACKEND_STATUS_PROBE_TIMEOUT_MS,
|
|
1438
|
+
`task status probe for ${taskId}`,
|
|
1439
|
+
);
|
|
1440
|
+
return status;
|
|
1441
|
+
} catch (error) {
|
|
1442
|
+
// Release the socket; withTimeout only stops us waiting, it cannot
|
|
1443
|
+
// cancel the request on its own.
|
|
1444
|
+
try {
|
|
1445
|
+
controller?.abort();
|
|
1446
|
+
} catch {
|
|
1447
|
+
// best effort
|
|
1448
|
+
}
|
|
1449
|
+
logError(`Failed to query backend status for task ${taskId}: ${error?.message || error}`);
|
|
1450
|
+
return null;
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1453
|
+
|
|
1454
|
+
const TERMINAL_BACKEND_TASK_STATUSES = new Set(["completed", "killed"]);
|
|
1455
|
+
|
|
1456
|
+
// A tmux session vanished. Decide what — if anything — to tell the backend.
|
|
1457
|
+
//
|
|
1458
|
+
// This is the fix for "fire dies inside its tmux session and nobody
|
|
1459
|
+
// notices". The daemon's child was only the `tmux new-session` client,
|
|
1460
|
+
// which exited 0 long ago, so there is no exit event to observe. Without a
|
|
1461
|
+
// report here the task sits at `running` until the next reconcile sweep
|
|
1462
|
+
// collects it, and reconcile's `PATCH {status:"killed"}` is rewritten by
|
|
1463
|
+
// the task route into `killing` → `killed`, which the upstream commit path
|
|
1464
|
+
// reads as `killedReason = "user_stopped"`. The user never pressed stop,
|
|
1465
|
+
// and the real cause is lost.
|
|
1466
|
+
//
|
|
1467
|
+
// Two hazards make a naive "session gone → report KILLED" wrong, and both
|
|
1468
|
+
// are handled below:
|
|
1469
|
+
// 1. A session also disappears when fire finished *successfully*.
|
|
1470
|
+
// Reporting KILLED unconditionally would rewrite completed tasks as
|
|
1471
|
+
// failures. We therefore classify from the exit marker the wrapper
|
|
1472
|
+
// shell appends to the log, not from the disappearance itself.
|
|
1473
|
+
// 2. Fire usually reports its own terminal status over its own
|
|
1474
|
+
// websocket. `commitTaskStatusUpdate` has no terminal→terminal guard,
|
|
1475
|
+
// so a late KILLED from us would clobber a COMPLETED from fire. We
|
|
1476
|
+
// ask the backend first and stay silent when the task already reached
|
|
1477
|
+
// a terminal state.
|
|
1478
|
+
//
|
|
1479
|
+
// Note on `suppressedExitStatusReports`: it is deliberately NOT consulted
|
|
1480
|
+
// here. Every path that sets it (refresh_session_inplace via
|
|
1481
|
+
// `stopActiveTaskProcess`) removes the active record synchronously, so the
|
|
1482
|
+
// reaper can never see a record it applies to — but the flag itself is
|
|
1483
|
+
// never cleared in tmux mode, because the tmux client's exit handler
|
|
1484
|
+
// returns early and only the non-early path consumes it. Honouring a flag
|
|
1485
|
+
// that outlives its task would silence a later, unrelated death. The
|
|
1486
|
+
// backend status pre-check below is the guard that actually matters.
|
|
1487
|
+
async function reportDeadTmuxSessionStatus(taskId, record) {
|
|
1488
|
+
if (!record?.projectId) {
|
|
1489
|
+
logError(`Cannot report terminal status for task ${taskId}: no project id on the active record`);
|
|
1490
|
+
return;
|
|
1491
|
+
}
|
|
1492
|
+
// The exit marker only exists because the wrapper shell writes it to
|
|
1493
|
+
// `logPath`. Without one there is no evidence at all, and "no marker"
|
|
1494
|
+
// would be misread as "died hard" — reporting every clean finish as
|
|
1495
|
+
// KILLED. Staying silent restores the old blind spot for this task only,
|
|
1496
|
+
// which is strictly better than manufacturing a wrong cause of death.
|
|
1497
|
+
if (!record.logPath) {
|
|
1498
|
+
logError(
|
|
1499
|
+
`Cannot classify the end of task ${taskId}: no log path on the active record, so no exit marker exists`,
|
|
1500
|
+
);
|
|
1501
|
+
return;
|
|
1502
|
+
}
|
|
1503
|
+
// Same reasoning, one level down. `tee -a` creates the log the instant it
|
|
1504
|
+
// opens it, so a missing file means the wrapper could never write there
|
|
1505
|
+
// (unwritable or vanished cwd) — and `tee` failing does NOT stop the fire,
|
|
1506
|
+
// which runs to completion while the tmux client still exits 0. Treating
|
|
1507
|
+
// that absence as "no exit marker" would report a green task as KILLED
|
|
1508
|
+
// with an invented "killed or OOM" cause. An empty-but-present file is
|
|
1509
|
+
// different: the fire really did die before writing anything.
|
|
1510
|
+
if (readLogFileSize(record.logPath) === null) {
|
|
1511
|
+
logError(
|
|
1512
|
+
`Cannot classify the end of task ${taskId}: log file ${record.logPath} is missing, ` +
|
|
1513
|
+
`so the absence of an exit marker proves nothing`,
|
|
1514
|
+
);
|
|
1515
|
+
return;
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
const rawTail = readLogTailRaw(record.logPath, { minOffset: record.logStartOffset || 0 });
|
|
1519
|
+
// Search the file for the marker rather than hoping it is still inside
|
|
1520
|
+
// the tail window — see `findFireExitCode` for why a shared log makes the
|
|
1521
|
+
// tail unreliable. The tail is still what feeds the human-readable
|
|
1522
|
+
// summary; only the *verdict* needs the exhaustive lookup.
|
|
1523
|
+
const exitCode = findFireExitCode(record.logPath, {
|
|
1524
|
+
minOffset: record.logStartOffset || 0,
|
|
1525
|
+
markerToken: record.exitMarkerToken,
|
|
1526
|
+
});
|
|
1527
|
+
const lifetimeMs = record.spawnedAtMs ? Date.now() - record.spawnedAtMs : null;
|
|
1528
|
+
|
|
1529
|
+
// Classification. `null` exit code means the wrapper shell never got to
|
|
1530
|
+
// write the marker — SIGKILL, OOM kill, or a tmux server crash.
|
|
1531
|
+
const isKilled = exitCode === null || exitCode !== 0;
|
|
1532
|
+
const status = isKilled ? "KILLED" : "COMPLETED";
|
|
1533
|
+
//
|
|
1534
|
+
// The success summary deliberately is NOT the bare word "completed".
|
|
1535
|
+
// The web side treats a summary that merely restates the status as
|
|
1536
|
+
// trivial and drops it — but only when it has to synthesize the event
|
|
1537
|
+
// id. We send our own id (for idempotency), which bypasses that filter,
|
|
1538
|
+
// so a bare "completed" would be persisted and would overwrite
|
|
1539
|
+
// `latest_status_summary` with a word that says nothing. Naming *how* we
|
|
1540
|
+
// found out keeps the event worth its row: it tells the next reader that
|
|
1541
|
+
// fire never published its own status.
|
|
1542
|
+
const baseSummary =
|
|
1543
|
+
exitCode === null
|
|
1544
|
+
? "fire tmux session disappeared without an exit marker (killed or OOM)"
|
|
1545
|
+
: exitCode === 0
|
|
1546
|
+
? "completed; detected by the daemon after the tmux session ended (fire never reported it)"
|
|
1547
|
+
: exitCode === 130 || exitCode === 143
|
|
1548
|
+
? `terminated (exit code ${exitCode})`
|
|
1549
|
+
: `exited with code ${exitCode}`;
|
|
1550
|
+
|
|
1551
|
+
// Ask the backend before overwriting anything: fire normally reports its
|
|
1552
|
+
// own terminal status and we must not downgrade a success.
|
|
1553
|
+
const backendStatus = await fetchBackendTaskStatus(taskId);
|
|
1554
|
+
if (backendStatus && TERMINAL_BACKEND_TASK_STATUSES.has(backendStatus)) {
|
|
1555
|
+
log(
|
|
1556
|
+
`Tmux session ${record.tmuxSession} for task ${taskId} ended; backend already ${backendStatus}, no report needed`,
|
|
1557
|
+
);
|
|
1558
|
+
return;
|
|
1559
|
+
}
|
|
1560
|
+
|
|
1561
|
+
// The await above yields the event loop, and "this task looks dead" is
|
|
1562
|
+
// precisely the trigger for an in-place restart. A `restart_task` landing
|
|
1563
|
+
// in that window finds no active record (we deleted it), spawns a fresh
|
|
1564
|
+
// fire and reports RUNNING — and our stale report would then mark the
|
|
1565
|
+
// *live* replacement as killed, using the previous run's log tail. Worse,
|
|
1566
|
+
// the daemon's own map would say running, so reconcile would never repair
|
|
1567
|
+
// it. Re-check that the task is still absent before speaking.
|
|
1568
|
+
if (activeTaskProcesses.has(taskId)) {
|
|
1569
|
+
log(
|
|
1570
|
+
`Task ${taskId} was restarted while its dead tmux session was being reported; dropping the stale terminal status`,
|
|
1571
|
+
);
|
|
1572
|
+
return;
|
|
1573
|
+
}
|
|
1574
|
+
|
|
1575
|
+
// Strip the marker before it reaches a user-visible summary: its content
|
|
1576
|
+
// is already stated by the classification above, so leaving it in just
|
|
1577
|
+
// appends "[conductor-fire-exit] code=1" to "exited with code 1".
|
|
1578
|
+
const outputTail = isKilled
|
|
1579
|
+
? sanitizeOutputTail(
|
|
1580
|
+
rawTail.replace(
|
|
1581
|
+
new RegExp(`${escapeForRegExp(FIRE_EXIT_MARKER_PREFIX)}[^\\]]*\\] code=\\d+`, "g"),
|
|
1582
|
+
"",
|
|
1583
|
+
),
|
|
1584
|
+
)
|
|
1585
|
+
: "";
|
|
1586
|
+
if (isKilled) {
|
|
1587
|
+
logError(
|
|
1588
|
+
`[tmux-reap] fire died inside its session task=${taskId} ` +
|
|
1589
|
+
`tmux=${record.tmuxSession || "none"} exit=${exitCode === null ? "unknown" : exitCode} ` +
|
|
1590
|
+
`lifetime_ms=${lifetimeMs === null ? "unknown" : lifetimeMs} log=${record.logPath || "none"} ` +
|
|
1591
|
+
`backend_status=${backendStatus || "unknown"} ` +
|
|
1592
|
+
`output_tail=${outputTail ? JSON.stringify(outputTail) : "<empty>"}`,
|
|
1593
|
+
);
|
|
1594
|
+
}
|
|
1595
|
+
const summary = outputTail ? `${baseSummary}: ${outputTail}` : baseSummary;
|
|
1596
|
+
|
|
1597
|
+
client
|
|
1598
|
+
.sendJson({
|
|
1599
|
+
type: "task_status_update",
|
|
1600
|
+
payload: {
|
|
1601
|
+
task_id: taskId,
|
|
1602
|
+
project_id: record.projectId,
|
|
1603
|
+
status,
|
|
1604
|
+
summary,
|
|
1605
|
+
// Idempotency key. Generated per call, so it only dedupes
|
|
1606
|
+
// transport-level redelivery of *this* message — not a second,
|
|
1607
|
+
// independent reaper report (the record deletion above is what
|
|
1608
|
+
// prevents those). It is also what makes the backend persist
|
|
1609
|
+
// `summary` at all: without a client-supplied id the server
|
|
1610
|
+
// synthesizes a fresh one per delivery.
|
|
1611
|
+
status_event_id: randomUUID(),
|
|
1612
|
+
},
|
|
1613
|
+
})
|
|
1614
|
+
.catch((err) => {
|
|
1615
|
+
logError(
|
|
1616
|
+
`Failed to report task status (${status}) for reaped tmux task ${taskId}: ${err?.message || err}`,
|
|
1617
|
+
);
|
|
1618
|
+
});
|
|
1619
|
+
}
|
|
1620
|
+
|
|
1621
|
+
// Walk every tmux-mode entry in `activeTaskProcesses`, remove the ones
|
|
1622
|
+
// whose tmux session no longer exists, and report a terminal status for
|
|
1623
|
+
// them. We don't (and can't reliably) observe the inner Fire process exit
|
|
1624
|
+
// when we never owned it as a child, so this sweep is both what keeps the
|
|
1625
|
+
// active map from leaking across long daemon lifetimes *and* the only
|
|
1626
|
+
// place a death inside the session can be noticed at all.
|
|
1627
|
+
//
|
|
1628
|
+
// Startup race (now guarded — it used to be merely theoretical):
|
|
1629
|
+
// `spawnFireProcess` returns synchronously and we set the active record
|
|
1630
|
+
// before the spawned `tmux new-session -d` client has actually exited,
|
|
1631
|
+
// so there is a window in which an active record exists but
|
|
1632
|
+
// `tmux has-session` returns false because the session has not finished
|
|
1633
|
+
// registering with the tmux server. That window was harmless while the
|
|
1634
|
+
// reaper only cleaned up local bookkeeping; now that it reports terminal
|
|
1635
|
+
// statuses, hitting it would kill a task that started fine. Records are
|
|
1636
|
+
// therefore ignored until they are `TMUX_REAP_GRACE_MS` old.
|
|
1085
1637
|
async function reapDeadTmuxSessionsOnce() {
|
|
1086
1638
|
const candidates = [];
|
|
1639
|
+
const now = Date.now();
|
|
1087
1640
|
for (const [taskId, record] of activeTaskProcesses.entries()) {
|
|
1088
|
-
if (record?.tmuxMode
|
|
1089
|
-
|
|
1090
|
-
|
|
1641
|
+
if (!record?.tmuxMode || !record.tmuxSession) continue;
|
|
1642
|
+
const spawnedAtMs = Number(record.spawnedAtMs) || 0;
|
|
1643
|
+
if (spawnedAtMs && now - spawnedAtMs < TMUX_REAP_GRACE_MS) continue;
|
|
1644
|
+
candidates.push([taskId, record]);
|
|
1091
1645
|
}
|
|
1092
1646
|
for (const [taskId, record] of candidates) {
|
|
1093
|
-
const alive = await
|
|
1094
|
-
//
|
|
1095
|
-
//
|
|
1647
|
+
const { alive, conclusive } = await probeTmuxSession(record.tmuxSession);
|
|
1648
|
+
// An inconclusive probe (tmux missing, wedged server, timeout) is not
|
|
1649
|
+
// evidence of death. Leave the record alone and retry next sweep rather
|
|
1650
|
+
// than cleaning up — and reporting a cause of death — on a guess.
|
|
1651
|
+
if (!alive && !conclusive) {
|
|
1652
|
+
logError(
|
|
1653
|
+
`Could not determine whether tmux session ${record.tmuxSession} for task ${taskId} is alive; leaving it untouched`,
|
|
1654
|
+
);
|
|
1655
|
+
continue;
|
|
1656
|
+
}
|
|
1657
|
+
// Only act if the entry still points to the same record; a concurrent
|
|
1658
|
+
// restart_task may have replaced it while we were probing.
|
|
1096
1659
|
if (!alive && activeTaskProcesses.get(taskId) === record) {
|
|
1097
1660
|
log(
|
|
1098
1661
|
`Tmux session ${record.tmuxSession} for task ${taskId} no longer exists; cleaning up activeTaskProcesses entry`,
|
|
@@ -1102,6 +1665,9 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1102
1665
|
record.stopForceKillTimer = null;
|
|
1103
1666
|
}
|
|
1104
1667
|
activeTaskProcesses.delete(taskId);
|
|
1668
|
+
// Report *after* dropping the local entry so a slow backend round
|
|
1669
|
+
// trip can't make a concurrent stop_task wait on us.
|
|
1670
|
+
await reportDeadTmuxSessionStatus(taskId, record);
|
|
1105
1671
|
}
|
|
1106
1672
|
}
|
|
1107
1673
|
}
|
|
@@ -1442,6 +2008,10 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1442
2008
|
}
|
|
1443
2009
|
}
|
|
1444
2010
|
|
|
2011
|
+
// Monotonic suffix so two symlink swaps inside one daemon process can never
|
|
2012
|
+
// collide on the same temp path.
|
|
2013
|
+
let symlinkSequence = 0;
|
|
2014
|
+
|
|
1445
2015
|
async function ensureTaskWorktreeSymlinks({ projectRepoRoot, projectWorkspacePath, finalCwd }) {
|
|
1446
2016
|
const { symlinkPaths } = readProjectWorktreeSettings(projectWorkspacePath);
|
|
1447
2017
|
for (const configuredPath of symlinkPaths) {
|
|
@@ -1455,6 +2025,27 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1455
2025
|
if (await isGitTrackedWorktreePath({ projectRepoRoot, sourcePath })) {
|
|
1456
2026
|
continue;
|
|
1457
2027
|
}
|
|
2028
|
+
|
|
2029
|
+
// Never link to a source that isn't there. `symlinkSync` does NOT
|
|
2030
|
+
// require its target to exist (POSIX), so a stale `worktree.symlink`
|
|
2031
|
+
// entry — e.g. `xr/android/build/local.properties`, generated locally by
|
|
2032
|
+
// the IDE and never present on this machine — would otherwise make the
|
|
2033
|
+
// daemon *manufacture* a dangling link on the very first worktree prep.
|
|
2034
|
+
// Those links are pure liability: they break tooling that follows them,
|
|
2035
|
+
// and they are what the EEXIST-on-reprepare bug below fed on.
|
|
2036
|
+
//
|
|
2037
|
+
// Note this is the one place where `existsSync` is the RIGHT probe:
|
|
2038
|
+
// here we genuinely care whether the target resolves to something real
|
|
2039
|
+
// (a source that is itself a dangling link is equally useless). The
|
|
2040
|
+
// destination probe further down must use `lstat` instead, for exactly
|
|
2041
|
+
// the opposite reason — see the note there.
|
|
2042
|
+
if (!existsSyncFn(sourcePath)) {
|
|
2043
|
+
logError(
|
|
2044
|
+
`[worktree] skipping symlink for missing source: ${configuredPath} (expected at ${sourcePath})`,
|
|
2045
|
+
);
|
|
2046
|
+
continue;
|
|
2047
|
+
}
|
|
2048
|
+
|
|
1458
2049
|
const linkPath = resolveProjectScopedPath(
|
|
1459
2050
|
finalCwd,
|
|
1460
2051
|
configuredPath,
|
|
@@ -1462,21 +2053,70 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1462
2053
|
);
|
|
1463
2054
|
mkdirSyncFn(path.dirname(linkPath), { recursive: true });
|
|
1464
2055
|
|
|
1465
|
-
|
|
2056
|
+
// NOTE: probe with lstat, never existsSync. existsSync FOLLOWS symlinks,
|
|
2057
|
+
// so a dangling link (source deleted after the worktree was created —
|
|
2058
|
+
// .venv / node_modules / local.properties are all gitignored churn)
|
|
2059
|
+
// reads as "missing", and the symlinkSyncFn below then throws EEXIST.
|
|
2060
|
+
// That made the task permanently un-restartable.
|
|
2061
|
+
let linkStat = null;
|
|
2062
|
+
try {
|
|
2063
|
+
linkStat = lstatSyncFn(linkPath);
|
|
2064
|
+
} catch (error) {
|
|
2065
|
+
if (error?.code !== "ENOENT") {
|
|
2066
|
+
throw error instanceof Error ? error : new Error(String(error));
|
|
2067
|
+
}
|
|
2068
|
+
}
|
|
2069
|
+
|
|
2070
|
+
if (linkStat) {
|
|
2071
|
+
if (!linkStat.isSymbolicLink()) {
|
|
2072
|
+
throw new Error(
|
|
2073
|
+
`worktree symlink destination already exists and is not a symlink: ${linkPath}. ` +
|
|
2074
|
+
`Refusing to replace it because it may hold real data — remove it manually, ` +
|
|
2075
|
+
`or drop "${configuredPath}" from worktree.symlink in .conductor/settings.yaml.`,
|
|
2076
|
+
);
|
|
2077
|
+
}
|
|
2078
|
+
// Compare the link's TARGET, not whether that target resolves. A link
|
|
2079
|
+
// that already points at the right place is correct even when the
|
|
2080
|
+
// source is currently absent.
|
|
2081
|
+
const currentTarget = readlinkSyncFn(linkPath);
|
|
2082
|
+
const currentResolvedTarget = path.resolve(path.dirname(linkPath), currentTarget);
|
|
2083
|
+
if (currentResolvedTarget === sourcePath) {
|
|
2084
|
+
continue;
|
|
2085
|
+
}
|
|
2086
|
+
// Self-heal instead of aborting. A stale link happens whenever the
|
|
2087
|
+
// project moves on disk or its workspace_path binding is edited: every
|
|
2088
|
+
// pre-existing worktree then holds links to the OLD absolute path, and
|
|
2089
|
+
// throwing here made every task in them permanently un-restartable.
|
|
2090
|
+
//
|
|
2091
|
+
// Replacing is safe precisely because this entry is a SYMLINK: it
|
|
2092
|
+
// carries no data of its own, so unlinking destroys nothing. (The
|
|
2093
|
+
// non-symlink case above still throws — a real file or directory here
|
|
2094
|
+
// may hold user data and must never be clobbered.) The source is also
|
|
2095
|
+
// known to exist at this point, thanks to the guard further up, so we
|
|
2096
|
+
// are converging on a link that actually resolves.
|
|
2097
|
+
log(
|
|
2098
|
+
`[worktree] repointing stale symlink ${linkPath}: ${currentResolvedTarget} -> ${sourcePath}`,
|
|
2099
|
+
);
|
|
2100
|
+
// Atomic replace. branch/fork tasks deliberately SHARE one worktree
|
|
2101
|
+
// (identity is keyed on worktreeBranch), so two preparations can run
|
|
2102
|
+
// against this directory concurrently. A plain unlink+symlink leaves a
|
|
2103
|
+
// window where the peer's symlinkSync hits EEXIST — reintroducing the
|
|
2104
|
+
// very failure this function was fixed for. symlink-to-temp + rename
|
|
2105
|
+
// has no such window: rename(2) atomically replaces the entry.
|
|
2106
|
+
const tempLinkPath = `${linkPath}.conductor-tmp-${process.pid}-${symlinkSequence++}`;
|
|
2107
|
+
const relativeTargetForSwap = path.relative(path.dirname(linkPath), sourcePath) || ".";
|
|
2108
|
+
symlinkSyncFn(relativeTargetForSwap, tempLinkPath);
|
|
1466
2109
|
try {
|
|
1467
|
-
|
|
1468
|
-
if (!stat.isSymbolicLink()) {
|
|
1469
|
-
throw new Error(`worktree symlink destination already exists: ${linkPath}`);
|
|
1470
|
-
}
|
|
1471
|
-
const currentTarget = readlinkSyncFn(linkPath);
|
|
1472
|
-
const currentResolvedTarget = path.resolve(path.dirname(linkPath), currentTarget);
|
|
1473
|
-
if (currentResolvedTarget === sourcePath) {
|
|
1474
|
-
continue;
|
|
1475
|
-
}
|
|
1476
|
-
throw new Error(`worktree symlink destination already points elsewhere: ${linkPath}`);
|
|
2110
|
+
renameSyncFn(tempLinkPath, linkPath);
|
|
1477
2111
|
} catch (error) {
|
|
2112
|
+
try {
|
|
2113
|
+
unlinkSyncFn(tempLinkPath);
|
|
2114
|
+
} catch {
|
|
2115
|
+
// best effort: never mask the original failure
|
|
2116
|
+
}
|
|
1478
2117
|
throw error instanceof Error ? error : new Error(String(error));
|
|
1479
2118
|
}
|
|
2119
|
+
continue;
|
|
1480
2120
|
}
|
|
1481
2121
|
|
|
1482
2122
|
const relativeTarget = path.relative(path.dirname(linkPath), sourcePath) || ".";
|
|
@@ -2108,6 +2748,20 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
2108
2748
|
return 30 * 1000;
|
|
2109
2749
|
})();
|
|
2110
2750
|
|
|
2751
|
+
// How long a freshly spawned tmux-mode task is exempt from the reaper.
|
|
2752
|
+
// Covers the window between "we recorded the active task" and "the tmux
|
|
2753
|
+
// server finished registering the session", during which `has-session`
|
|
2754
|
+
// can legitimately answer false for a task that started fine. Since the
|
|
2755
|
+
// reaper now reports terminal statuses, a false positive here would kill a
|
|
2756
|
+
// healthy task, so the exemption is no longer optional. Tests set 0.
|
|
2757
|
+
const TMUX_REAP_GRACE_MS = (() => {
|
|
2758
|
+
const explicit = Number(config.TMUX_REAP_GRACE_MS);
|
|
2759
|
+
if (Number.isFinite(explicit) && explicit >= 0) {
|
|
2760
|
+
return explicit;
|
|
2761
|
+
}
|
|
2762
|
+
return 15 * 1000;
|
|
2763
|
+
})();
|
|
2764
|
+
|
|
2111
2765
|
// --- Auto-update state ---
|
|
2112
2766
|
const VERSION_CHECK_INTERVAL_MS = 6 * 60 * 60 * 1000; // 6 hours
|
|
2113
2767
|
let lastVersionCheckAt = 0;
|
|
@@ -4786,8 +5440,18 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4786
5440
|
|
|
4787
5441
|
let entry = activeTaskProcesses.get(taskId);
|
|
4788
5442
|
if (entry?.tmuxMode && entry.tmuxSession) {
|
|
4789
|
-
|
|
4790
|
-
|
|
5443
|
+
// Only a conclusive "no such session" counts as death. A wedged or
|
|
5444
|
+
// missing tmux answers `false` too, and acting on that would ack the
|
|
5445
|
+
// reclaim as `stale` for a fire that is still running — the backend
|
|
5446
|
+
// then spawns a replacement and two fires share one worktree. It would
|
|
5447
|
+
// also drop the record, so the reaper could never classify the real
|
|
5448
|
+
// death later. When we cannot tell, keep the record and say "alive".
|
|
5449
|
+
const { alive, conclusive } = await probeTmuxSession(entry.tmuxSession);
|
|
5450
|
+
if (!alive && !conclusive) {
|
|
5451
|
+
logError(
|
|
5452
|
+
`Could not determine whether tmux session ${entry.tmuxSession} for task ${taskId} is alive; treating reclaim as still-alive`,
|
|
5453
|
+
);
|
|
5454
|
+
} else if (!alive) {
|
|
4791
5455
|
if (activeTaskProcesses.get(taskId) === entry) {
|
|
4792
5456
|
if (entry.stopForceKillTimer) {
|
|
4793
5457
|
clearTimeout(entry.stopForceKillTimer);
|
|
@@ -4854,6 +5518,40 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4854
5518
|
if ((!processRecord || !processRecord.child) && !ptyRecord) {
|
|
4855
5519
|
log(`Stop requested for task ${taskId}, but no active process found`);
|
|
4856
5520
|
sendStopAck(false);
|
|
5521
|
+
// "Nothing to stop" IS the terminal answer, and the server cannot infer
|
|
5522
|
+
// it: after the app flips the row to `killing` it waits for us to report
|
|
5523
|
+
// a terminal status, and a bare `stop_ack(accepted=false)` is only
|
|
5524
|
+
// command bookkeeping — it never converges the task. Staying silent here
|
|
5525
|
+
// is what strands a task in `killing` forever once its fire has already
|
|
5526
|
+
// died (e.g. the tmux session was reaped before the user pressed Stop).
|
|
5527
|
+
// Mirror the tmux stop path and publish KILLED so the row reaches a
|
|
5528
|
+
// terminal state that the user can then restart from.
|
|
5529
|
+
const terminalProjectId =
|
|
5530
|
+
normalizeOptionalString(payload?.project_id) ||
|
|
5531
|
+
normalizeOptionalString(processRecord?.projectId);
|
|
5532
|
+
if (terminalProjectId) {
|
|
5533
|
+
client
|
|
5534
|
+
.sendJson({
|
|
5535
|
+
type: "task_status_update",
|
|
5536
|
+
payload: {
|
|
5537
|
+
task_id: taskId,
|
|
5538
|
+
project_id: terminalProjectId,
|
|
5539
|
+
status: "KILLED",
|
|
5540
|
+
summary: payload?.reason
|
|
5541
|
+
? `stopped (${payload.reason}); no active process`
|
|
5542
|
+
: "stopped; no active process",
|
|
5543
|
+
},
|
|
5544
|
+
})
|
|
5545
|
+
.catch((err) => {
|
|
5546
|
+
logError(
|
|
5547
|
+
`Failed to report task_status_update(KILLED) for inactive task ${taskId}: ${err?.message || err}`,
|
|
5548
|
+
);
|
|
5549
|
+
});
|
|
5550
|
+
} else {
|
|
5551
|
+
logError(
|
|
5552
|
+
`Cannot report terminal status for inactive task ${taskId}: no project_id in stop_task payload`,
|
|
5553
|
+
);
|
|
5554
|
+
}
|
|
4857
5555
|
// Even when we have no in-memory record, the task may still own a
|
|
4858
5556
|
// tmux session (e.g. the daemon was restarted between spawn and
|
|
4859
5557
|
// stop, or the liveness reaper removed our entry but the session
|
|
@@ -4991,6 +5689,19 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4991
5689
|
: "restart failed";
|
|
4992
5690
|
const scrubbedError = maskErrorForLogs(error);
|
|
4993
5691
|
const summary = `${prefix}: ${scrubbedError?.message || scrubbedError}`;
|
|
5692
|
+
// Always leave a daemon-log trace. Several early rejects (unsupported
|
|
5693
|
+
// backend, cwd resolution, worktree prep) return before the "Restarting
|
|
5694
|
+
// task …" line is ever logged, so without this a failed branch/fork left
|
|
5695
|
+
// the daemon log completely silent and only a generic `fire_exit` in the
|
|
5696
|
+
// backend DB.
|
|
5697
|
+
// Benign, self-explanatory outcomes (orderly shutdown, a refresh that is
|
|
5698
|
+
// already in flight) are logged at normal level so they don't show up as
|
|
5699
|
+
// errors in monitoring; genuine failures still go to stderr.
|
|
5700
|
+
const failureText = `${scrubbedError?.message || scrubbedError}`;
|
|
5701
|
+
const isBenign = /daemon shut(ting)? down|already in progress/i.test(failureText);
|
|
5702
|
+
(isBenign ? log : logError)(
|
|
5703
|
+
`[restart-spawn] failure task=${taskId} mode=${mode || "unknown"}: ${failureText}`,
|
|
5704
|
+
);
|
|
4994
5705
|
if (mode === "refresh_session_inplace") {
|
|
4995
5706
|
rememberCommandRequestAckResult(requestId, false);
|
|
4996
5707
|
}
|
|
@@ -5013,6 +5724,13 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5013
5724
|
project_id: projectId,
|
|
5014
5725
|
status: "KILLED",
|
|
5015
5726
|
summary,
|
|
5727
|
+
// Idempotency key for this transition. The backend persists
|
|
5728
|
+
// `summary` only as a `taskStatusEvent` row, and it keys duplicate
|
|
5729
|
+
// suppression off this id — without one it has to synthesize an id,
|
|
5730
|
+
// which makes a redelivered report indistinguishable from a new
|
|
5731
|
+
// event. Supplying it keeps failure reports both persisted AND
|
|
5732
|
+
// deduplicated.
|
|
5733
|
+
status_event_id: randomUUID(),
|
|
5016
5734
|
},
|
|
5017
5735
|
})
|
|
5018
5736
|
.catch((err) => {
|
|
@@ -5023,8 +5741,14 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5023
5741
|
function reportCreateTaskFailure({ taskId, projectId, requestId, error, sendAck = true }) {
|
|
5024
5742
|
const normalizedTaskId = taskId ? String(taskId) : "";
|
|
5025
5743
|
const normalizedProjectId = projectId ? String(projectId) : "";
|
|
5026
|
-
|
|
5027
|
-
|
|
5744
|
+
// Mirror reportRestartFailure: scrub before the message reaches the log
|
|
5745
|
+
// AND the persisted status summary. An un-scrubbed create failure could
|
|
5746
|
+
// carry the handoff URL or an inherited provider key.
|
|
5747
|
+
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
5748
|
+
const message = redactSecretsForLogs(rawMessage, [AGENT_TOKEN]);
|
|
5749
|
+
logError(
|
|
5750
|
+
`[create-spawn] failure task=${normalizedTaskId || "unknown"}: ${message}`,
|
|
5751
|
+
);
|
|
5028
5752
|
if (sendAck) {
|
|
5029
5753
|
sendAgentCommandAck({
|
|
5030
5754
|
requestId,
|
|
@@ -5046,6 +5770,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5046
5770
|
project_id: normalizedProjectId,
|
|
5047
5771
|
status: "KILLED",
|
|
5048
5772
|
summary: message,
|
|
5773
|
+
status_event_id: randomUUID(),
|
|
5049
5774
|
},
|
|
5050
5775
|
})
|
|
5051
5776
|
.catch((err) => {
|
|
@@ -5304,7 +6029,10 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5304
6029
|
env.CONDUCTOR_BACKEND_URL = BACKEND_HTTP;
|
|
5305
6030
|
}
|
|
5306
6031
|
|
|
5307
|
-
|
|
6032
|
+
// Sampled before the spawn so it is a true "everything after this is
|
|
6033
|
+
// ours" watermark.
|
|
6034
|
+
const logStartOffset = readLogSize(logPath);
|
|
6035
|
+
const { child, tmuxSession, exitMarkerToken } = spawnFireProcess({
|
|
5308
6036
|
taskId,
|
|
5309
6037
|
args,
|
|
5310
6038
|
env,
|
|
@@ -5362,10 +6090,25 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5362
6090
|
log(`New task workspace: ${taskDir}`);
|
|
5363
6091
|
log(`Logs: ${logPath}`);
|
|
5364
6092
|
|
|
6093
|
+
// Same diagnostics contract as the restart/fork path: without this a
|
|
6094
|
+
// create_task whose backend dies at startup is a black box too.
|
|
6095
|
+
const outputCapture = createChildOutputCapture({ logPath, logStartOffset });
|
|
6096
|
+
const spawnedAtMs = Date.now();
|
|
6097
|
+
|
|
5365
6098
|
activeTaskProcesses.set(taskId, {
|
|
5366
6099
|
child,
|
|
5367
6100
|
projectId,
|
|
5368
6101
|
logPath,
|
|
6102
|
+
// Floor for later tail reads and the reaper's exit-marker scan: the
|
|
6103
|
+
// log file is opened with flags "a" and survives in-place restarts,
|
|
6104
|
+
// so bytes written before this run must not be attributed to it.
|
|
6105
|
+
logStartOffset,
|
|
6106
|
+
// Nonce this run's exit marker is tagged with, so the reaper only
|
|
6107
|
+
// trusts a marker written by *this* wrapper shell.
|
|
6108
|
+
exitMarkerToken,
|
|
6109
|
+
// Consumed by the tmux liveness reaper as both a grace-period anchor
|
|
6110
|
+
// and a lifetime figure for diagnostics.
|
|
6111
|
+
spawnedAtMs,
|
|
5369
6112
|
stopForceKillTimer: null,
|
|
5370
6113
|
managedByFireBridge: true,
|
|
5371
6114
|
tmuxSession: tmuxSession || null,
|
|
@@ -5399,13 +6142,20 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5399
6142
|
} else if (child.stderr && typeof child.stderr.on === "function" && logStream) {
|
|
5400
6143
|
child.stderr.on("data", (chunk) => logStream.write(chunk));
|
|
5401
6144
|
}
|
|
6145
|
+
outputCapture.attach(child.stdout);
|
|
6146
|
+
outputCapture.attach(child.stderr);
|
|
5402
6147
|
} else if (child.stderr && typeof child.stderr.on === "function") {
|
|
6148
|
+
outputCapture.attach(child.stderr);
|
|
5403
6149
|
// Capture any error output emitted by the tmux client itself so
|
|
5404
6150
|
// problems during session creation surface in daemon logs.
|
|
5405
6151
|
child.stderr.on("data", (chunk) => {
|
|
5406
6152
|
const text = chunk?.toString?.("utf8") ?? String(chunk ?? "");
|
|
5407
6153
|
if (text.trim()) {
|
|
5408
|
-
logError(
|
|
6154
|
+
logError(
|
|
6155
|
+
// Redact: this is the fire's raw stderr, which can carry the
|
|
6156
|
+
// handoff share token (argv echo) and provider/agent keys.
|
|
6157
|
+
`tmux(${tmuxSession}) stderr: ${redactSecretsForLogs(text.trim(), [AGENT_TOKEN])}`,
|
|
6158
|
+
);
|
|
5409
6159
|
}
|
|
5410
6160
|
});
|
|
5411
6161
|
}
|
|
@@ -5421,6 +6171,20 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5421
6171
|
child.on("exit", (code, signal) => {
|
|
5422
6172
|
const active = activeTaskProcesses.get(taskId);
|
|
5423
6173
|
|
|
6174
|
+
// This spawn was a tmux client but the record is gone: the reaper
|
|
6175
|
+
// (or a stop) already retired this task and, in the reaper's case,
|
|
6176
|
+
// already published its terminal status. Falling through would
|
|
6177
|
+
// delete whatever record a restart has since installed and report a
|
|
6178
|
+
// second, contradictory status — `shouldDaemonReportFireChildTerminal
|
|
6179
|
+
// Status(undefined)` is true, so an `exit(0)` arriving late would
|
|
6180
|
+
// announce COMPLETED over the reaper's verdict. Nothing left to do.
|
|
6181
|
+
if (tmuxSession && !active) {
|
|
6182
|
+
log(
|
|
6183
|
+
`tmux client for task ${taskId} exited after its record was retired (code=${code}, signal=${signal || "null"}); nothing to report`,
|
|
6184
|
+
);
|
|
6185
|
+
return;
|
|
6186
|
+
}
|
|
6187
|
+
|
|
5424
6188
|
// In tmux mode the `tmux new-session -d` client always exits
|
|
5425
6189
|
// shortly after launching the Fire session. A clean exit (code 0,
|
|
5426
6190
|
// no signal) just means the session was successfully created and
|
|
@@ -5484,6 +6248,18 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5484
6248
|
? "completed"
|
|
5485
6249
|
: `exited with code ${code}`;
|
|
5486
6250
|
|
|
6251
|
+
const lifetimeMs = Date.now() - spawnedAtMs;
|
|
6252
|
+
const outputTail = status === "KILLED" ? outputCapture.tail() : "";
|
|
6253
|
+
if (status === "KILLED") {
|
|
6254
|
+
logError(
|
|
6255
|
+
`[create-spawn] abnormal exit task=${taskId} backend=${selectedBackend} ` +
|
|
6256
|
+
`cwd=${taskDir} tmux=${tmuxSession || "none"} exit=${code ?? "null"} ` +
|
|
6257
|
+
`signal=${signal || "null"} lifetime_ms=${lifetimeMs} log=${logPath} ` +
|
|
6258
|
+
`output_tail=${outputTail ? JSON.stringify(outputTail) : "<empty>"}`,
|
|
6259
|
+
);
|
|
6260
|
+
}
|
|
6261
|
+
const reportedSummary = outputTail ? `${summary}: ${outputTail}` : summary;
|
|
6262
|
+
|
|
5487
6263
|
if (!suppressExitStatusReport && shouldDaemonReportFireChildTerminalStatus(active)) {
|
|
5488
6264
|
client
|
|
5489
6265
|
.sendJson({
|
|
@@ -5492,7 +6268,8 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5492
6268
|
task_id: taskId,
|
|
5493
6269
|
project_id: projectId,
|
|
5494
6270
|
status,
|
|
5495
|
-
summary,
|
|
6271
|
+
summary: reportedSummary,
|
|
6272
|
+
status_event_id: randomUUID(),
|
|
5496
6273
|
},
|
|
5497
6274
|
})
|
|
5498
6275
|
.catch((err) => {
|
|
@@ -5610,8 +6387,17 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5610
6387
|
// Probe the actual session and clean up stale entries on demand so
|
|
5611
6388
|
// the restart gating below reflects reality, not the stale record.
|
|
5612
6389
|
if (activeTarget?.tmuxMode && activeTarget.tmuxSession) {
|
|
5613
|
-
|
|
5614
|
-
|
|
6390
|
+
// As in reclaim: only a conclusive answer may clear the record. This
|
|
6391
|
+
// gate is what stops a double spawn, so trusting an inconclusive probe
|
|
6392
|
+
// would start a second fire alongside a live one in the same worktree.
|
|
6393
|
+
const { alive: sessionAlive, conclusive } = await probeTmuxSession(
|
|
6394
|
+
activeTarget.tmuxSession,
|
|
6395
|
+
);
|
|
6396
|
+
if (!sessionAlive && !conclusive) {
|
|
6397
|
+
logError(
|
|
6398
|
+
`Could not determine whether tmux session ${activeTarget.tmuxSession} for task ${normalizedTargetTaskId} is alive; keeping the existing record before restart`,
|
|
6399
|
+
);
|
|
6400
|
+
} else if (
|
|
5615
6401
|
!sessionAlive &&
|
|
5616
6402
|
activeTaskProcesses.get(normalizedTargetTaskId) === activeTarget
|
|
5617
6403
|
) {
|
|
@@ -5946,13 +6732,38 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5946
6732
|
env.CONDUCTOR_BACKEND_URL = BACKEND_HTTP;
|
|
5947
6733
|
}
|
|
5948
6734
|
|
|
5949
|
-
|
|
6735
|
+
// Two hazards below, one root cause: an in-place restart deliberately
|
|
6736
|
+
// re-uses the previous run's working directory, so ANY artifact the last
|
|
6737
|
+
// run left there can be mistaken for this run's own state. Both guards
|
|
6738
|
+
// must run before the spawn.
|
|
6739
|
+
//
|
|
6740
|
+
// (1) The fire's durable upstream outbox lives in that directory. If the
|
|
6741
|
+
// prior run was stopped while its websocket was down, its undelivered
|
|
6742
|
+
// terminal `task_status_update` (KILLED/COMPLETED) is still on disk — and
|
|
6743
|
+
// the run we are about to spawn would flush it during startup, marking the
|
|
6744
|
+
// task killed seconds after resuming and then getting shot down by the
|
|
6745
|
+
// server's `stop_task(task_already_killed)` reply. Purge those superseded
|
|
6746
|
+
// events before the new fire can pick them up.
|
|
6747
|
+
dropSupersededTerminalStatusEvents(taskDir, normalizedTargetTaskId);
|
|
6748
|
+
|
|
6749
|
+
// (2) The log file is opened with `flags:"a"` and re-used as well. Sample
|
|
6750
|
+
// its size first so it is a true "everything after this is ours"
|
|
6751
|
+
// watermark; without it the reaper reads the PREVIOUS run's exit marker.
|
|
6752
|
+
const logStartOffset = readLogSize(logPath);
|
|
6753
|
+
|
|
6754
|
+
const { child, tmuxSession, exitMarkerToken } = spawnFireProcess({
|
|
5950
6755
|
taskId: normalizedTargetTaskId,
|
|
5951
6756
|
args,
|
|
5952
6757
|
env,
|
|
5953
6758
|
cwd: taskDir,
|
|
5954
6759
|
logPath,
|
|
5955
6760
|
});
|
|
6761
|
+
// Bounded tail of the child's output + a spawn timestamp, so an abnormal
|
|
6762
|
+
// exit can report *why* it died and how long it survived. `logPath` lets
|
|
6763
|
+
// the capture recover the fire's own output in tmux mode, where the
|
|
6764
|
+
// daemon's child is only the `tmux new-session` client.
|
|
6765
|
+
const outputCapture = createChildOutputCapture({ logPath, logStartOffset });
|
|
6766
|
+
const spawnedAtMs = Date.now();
|
|
5956
6767
|
|
|
5957
6768
|
let logStream;
|
|
5958
6769
|
try {
|
|
@@ -5983,6 +6794,11 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5983
6794
|
child,
|
|
5984
6795
|
projectId: normalizedProjectId,
|
|
5985
6796
|
logPath,
|
|
6797
|
+
// See the create_task path: watermark for tail reads / exit-marker
|
|
6798
|
+
// scans, the marker nonce, and the reaper's grace-period anchor.
|
|
6799
|
+
logStartOffset,
|
|
6800
|
+
exitMarkerToken,
|
|
6801
|
+
spawnedAtMs,
|
|
5986
6802
|
stopForceKillTimer: null,
|
|
5987
6803
|
managedByFireBridge: true,
|
|
5988
6804
|
tmuxSession: tmuxSession || null,
|
|
@@ -6004,17 +6820,32 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6004
6820
|
} else if (child.stderr && typeof child.stderr.on === "function" && logStream) {
|
|
6005
6821
|
child.stderr.on("data", (chunk) => logStream.write(chunk));
|
|
6006
6822
|
}
|
|
6823
|
+
// Capture independently of logStream: when the log file cannot be
|
|
6824
|
+
// opened (createWriteStream threw) the piping above is skipped
|
|
6825
|
+
// entirely, which is exactly when we most need the tail.
|
|
6826
|
+
outputCapture.attach(child.stdout);
|
|
6827
|
+
outputCapture.attach(child.stderr);
|
|
6007
6828
|
} else if (child.stderr && typeof child.stderr.on === "function") {
|
|
6829
|
+
outputCapture.attach(child.stderr);
|
|
6008
6830
|
child.stderr.on("data", (chunk) => {
|
|
6009
6831
|
const text = chunk?.toString?.("utf8") ?? String(chunk ?? "");
|
|
6010
6832
|
if (text.trim()) {
|
|
6011
|
-
logError(
|
|
6833
|
+
logError(
|
|
6834
|
+
// Redact: this is the fire's raw stderr, which can carry the
|
|
6835
|
+
// handoff share token (argv echo) and provider/agent keys.
|
|
6836
|
+
`tmux(${tmuxSession}) stderr: ${redactSecretsForLogs(text.trim(), [AGENT_TOKEN])}`,
|
|
6837
|
+
);
|
|
6012
6838
|
}
|
|
6013
6839
|
});
|
|
6014
6840
|
}
|
|
6015
6841
|
|
|
6016
6842
|
child.on("error", (err) => {
|
|
6017
|
-
logError(
|
|
6843
|
+
logError(
|
|
6844
|
+
`[fork-spawn] spawn error task=${normalizedTargetTaskId} mode=${normalizedMode} ` +
|
|
6845
|
+
`backend=${selectedBackend} cwd=${taskDir} tmux=${tmuxSession || "none"}: ${
|
|
6846
|
+
maskErrorForLogs(err)?.message || err
|
|
6847
|
+
}`,
|
|
6848
|
+
);
|
|
6018
6849
|
if (logStream) {
|
|
6019
6850
|
const ts = new Date().toLocaleString("sv-SE", { timeZone: "Asia/Shanghai" }).replace(" ", "T");
|
|
6020
6851
|
logStream.write(`[daemon ${ts}] spawn error: ${err.message}\n`);
|
|
@@ -6024,6 +6855,16 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6024
6855
|
child.on("exit", (code, signal) => {
|
|
6025
6856
|
const active = activeTaskProcesses.get(normalizedTargetTaskId);
|
|
6026
6857
|
|
|
6858
|
+
// See the create path: a tmux client exiting after its record was
|
|
6859
|
+
// retired must stay silent, or it would both clobber a replacement
|
|
6860
|
+
// record and publish a status contradicting the reaper's verdict.
|
|
6861
|
+
if (tmuxSession && !active) {
|
|
6862
|
+
log(
|
|
6863
|
+
`tmux client for restarted task ${normalizedTargetTaskId} exited after its record was retired (code=${code}, signal=${signal || "null"}); nothing to report`,
|
|
6864
|
+
);
|
|
6865
|
+
return;
|
|
6866
|
+
}
|
|
6867
|
+
|
|
6027
6868
|
// In tmux mode the `tmux new-session -d` client always exits soon
|
|
6028
6869
|
// after launching the session. A clean exit (code 0, no signal) means
|
|
6029
6870
|
// Fire is now running detached under the tmux server — keep the task
|
|
@@ -6076,6 +6917,26 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6076
6917
|
? "completed"
|
|
6077
6918
|
: `exited with code ${code}`;
|
|
6078
6919
|
|
|
6920
|
+
// Diagnostics for an abnormal exit. This is the line that makes a
|
|
6921
|
+
// "branch task died instantly" incident debuggable: it names the mode,
|
|
6922
|
+
// backend, cwd, tmux session, exit code/signal, how long the child
|
|
6923
|
+
// survived, where its log is, and the tail of what it actually printed.
|
|
6924
|
+
const lifetimeMs = Date.now() - spawnedAtMs;
|
|
6925
|
+
const outputTail = outputCapture.tail();
|
|
6926
|
+
if (status === "KILLED") {
|
|
6927
|
+
logError(
|
|
6928
|
+
`[fork-spawn] abnormal exit task=${normalizedTargetTaskId} mode=${normalizedMode} ` +
|
|
6929
|
+
`backend=${selectedBackend} cwd=${taskDir} tmux=${tmuxSession || "none"} ` +
|
|
6930
|
+
`exit=${code ?? "null"} signal=${signal || "null"} lifetime_ms=${lifetimeMs} ` +
|
|
6931
|
+
`log=${logPath} output_tail=${outputTail ? JSON.stringify(outputTail) : "<empty>"}`,
|
|
6932
|
+
);
|
|
6933
|
+
}
|
|
6934
|
+
// Surface the same tail in the status summary so it reaches the backend
|
|
6935
|
+
// (task_status_events.summary) and the UI, instead of a bare
|
|
6936
|
+
// "exited with code 1" that explains nothing.
|
|
6937
|
+
const reportedSummary =
|
|
6938
|
+
status === "KILLED" && outputTail ? `${summary}: ${outputTail}` : summary;
|
|
6939
|
+
|
|
6079
6940
|
const shouldReportTerminalStatus =
|
|
6080
6941
|
!suppressExitStatusReport &&
|
|
6081
6942
|
(!acceptedRestartAckSent || shouldDaemonReportFireChildTerminalStatus(active));
|
|
@@ -6090,7 +6951,9 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6090
6951
|
task_id: normalizedTargetTaskId,
|
|
6091
6952
|
project_id: normalizedProjectId,
|
|
6092
6953
|
status,
|
|
6093
|
-
summary,
|
|
6954
|
+
summary: reportedSummary,
|
|
6955
|
+
// Idempotency key — see the note in reportRestartFailure.
|
|
6956
|
+
status_event_id: randomUUID(),
|
|
6094
6957
|
},
|
|
6095
6958
|
})
|
|
6096
6959
|
.catch((err) => {
|