@love-moon/conductor-cli 0.7.6 → 0.7.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/daemon.js CHANGED
@@ -3,6 +3,7 @@ import path from "node:path";
3
3
  import os from "node:os";
4
4
  import { createRequire } from "node:module";
5
5
  import { spawn, spawnSync } from "node:child_process";
6
+ import { randomUUID } from "node:crypto";
6
7
  import { fileURLToPath } from "node:url";
7
8
 
8
9
  import dotenv from "dotenv";
@@ -14,6 +15,7 @@ import {
14
15
  loadConfig,
15
16
  ConfigFileNotFound,
16
17
  ProjectContext,
18
+ DurableUpstreamOutboxStore,
17
19
  } from "@love-moon/conductor-sdk";
18
20
  import { DaemonLogCollector } from "./log-collector.js";
19
21
  import { envForExplicitConfigFile } from "./config-env.js";
@@ -58,7 +60,9 @@ import {
58
60
  import {
59
61
  maskHandoffUrlForLogs,
60
62
  maskErrorForLogs,
63
+ redactSecretsForLogs,
61
64
  } from "./handoff-log-mask.js";
65
+ import { StringDecoder } from "node:string_decoder";
62
66
 
63
67
  dotenv.config();
64
68
 
@@ -447,7 +451,7 @@ function normalizeOptionalString(value) {
447
451
  // Re-export the handoff-URL masking helpers so existing external imports
448
452
  // keep working. Implementation lives in a dependency-free module so unit
449
453
  // tests can import it without pulling in conductor-sdk and friends.
450
- export { maskHandoffUrlForLogs, maskErrorForLogs };
454
+ export { maskHandoffUrlForLogs, maskErrorForLogs, redactSecretsForLogs };
451
455
 
452
456
  function normalizeTerminalResumeStrategy(value) {
453
457
  const normalized = normalizeOptionalString(value);
@@ -628,6 +632,52 @@ function stripPtyTaskScopedEnv(source) {
628
632
  return env;
629
633
  }
630
634
 
635
+ /**
636
+ * Purge terminal `task_status_update` events left undelivered by a previous run
637
+ * of `taskId` in `taskDir`, before a restart re-uses that same directory.
638
+ *
639
+ * The fire's durable upstream outbox is stored under the fire's cwd, keyed by
640
+ * the `task:<id>` delivery scope. An in-place restart intentionally re-uses the
641
+ * cwd, so without this purge the new run flushes the old run's KILLED/COMPLETED
642
+ * event on startup and kills the task it just resumed. Best-effort: a restart
643
+ * must never fail because of outbox bookkeeping.
644
+ */
645
+ function dropSupersededTerminalStatusEvents(taskDir, taskId) {
646
+ if (!taskDir || !taskId) return;
647
+ try {
648
+ const store = DurableUpstreamOutboxStore.forProjectPath(taskDir, `task:${taskId}`);
649
+ // Distinguish "the purge failed" from "the purge does not exist here".
650
+ // The CLI loads the INSTALLED copy of conductor-sdk, not this repo's
651
+ // source, so an un-rebuilt or mismatched package silently lacks this
652
+ // method — and a bare try/catch would swallow the resulting TypeError
653
+ // exactly like a benign fs error, leaving every restart unprotected with
654
+ // no way to tell from the log. Say so explicitly instead. We still do not
655
+ // throw: an unprotected restart usually succeeds, whereas refusing to
656
+ // spawn would break restart outright.
657
+ if (typeof store?.dropPendingTerminalStatusEvents !== "function") {
658
+ logError(
659
+ `Cannot purge superseded terminal status events for task ${taskId}: the installed ` +
660
+ `@love-moon/conductor-sdk has no DurableUpstreamOutboxStore.dropPendingTerminalStatusEvents. ` +
661
+ `This restart is unprotected — a terminal status left by the previous run may kill it. ` +
662
+ `Rebuild and reinstall the SDK (make install-cli).`,
663
+ );
664
+ return;
665
+ }
666
+ const dropped = store.dropPendingTerminalStatusEvents();
667
+ if (dropped.length > 0) {
668
+ log(
669
+ `Dropped ${dropped.length} superseded terminal status event(s) for task ${taskId} before restart: ${dropped
670
+ .map((entry) => `${entry.payload?.status ?? "?"}@${entry.createdAt}`)
671
+ .join(", ")}`,
672
+ );
673
+ }
674
+ } catch (error) {
675
+ logError(
676
+ `Failed to purge superseded terminal status events for task ${taskId}: ${error?.message || error}`,
677
+ );
678
+ }
679
+ }
680
+
631
681
  function buildPtyTaskEnv(baseEnv = process.env, launchEnv = {}) {
632
682
  const parentEnv = stripPtyTaskScopedEnv(baseEnv);
633
683
  const taskLaunchEnv = stripPtyTaskScopedEnv(launchEnv);
@@ -827,6 +877,11 @@ export function startDaemon(config = {}, deps = {}) {
827
877
  const symlinkSyncFn = deps.symlinkSync || fs.symlinkSync;
828
878
  const unlinkSyncFn = deps.unlinkSync || fs.unlinkSync;
829
879
  const renameSyncFn = deps.renameSync || fs.renameSync;
880
+ // Used to recover a crashed fire's output from its log file when the daemon
881
+ // could not observe the process directly (tmux mode).
882
+ const openSyncFn = deps.openSync || fs.openSync;
883
+ const readSyncFn = deps.readSync || fs.readSync;
884
+ const closeSyncFn = deps.closeSync || fs.closeSync;
830
885
  const createWriteStreamFn = deps.createWriteStream || fs.createWriteStream;
831
886
  const fetchFn = deps.fetch || fetch;
832
887
  const createRtcPeerConnection = deps.createRtcPeerConnection || null;
@@ -911,6 +966,231 @@ export function startDaemon(config = {}, deps = {}) {
911
966
  return `${buildFireTmuxSessionPrefix(taskId)}${uniq}`;
912
967
  }
913
968
 
969
+ // Observability for spawned fires (RFC: fork-spawn black box).
970
+ //
971
+ // When a forked/branched task's backend dies during startup, the daemon
972
+ // previously reported a bare `exited with code N` and wrote nothing to the
973
+ // daemon log, so the real cause (backend auth failure, missing CLI, tmux
974
+ // launch error, …) was unrecoverable after the fact — even with DB access
975
+ // the only artifact was a generic `fire_exit`. We keep a bounded tail of the
976
+ // child's output so an abnormal exit can name its own cause.
977
+ //
978
+ // The tail is masked with `maskHandoffUrlForLogs` before it reaches any log
979
+ // or status summary: the handoff prompt embeds a share token, and a backend
980
+ // that echoes its argv on failure would otherwise leak a read-grant for the
981
+ // entire transcript into the daemon log and the task status summary.
982
+ const CHILD_OUTPUT_CAPTURE_LIMIT = 4000;
983
+ const CHILD_OUTPUT_SUMMARY_LIMIT = 400;
984
+
985
+ // Sentinel appended to a tmux-hosted fire's log by the wrapper shell (see
986
+ // `spawnFireProcess`). It is the *only* channel through which the daemon
987
+ // can learn the exit code of a process it does not own. The full marker is
988
+ // `[conductor-fire-exit:<per-spawn nonce>] code=<n>`.
989
+ const FIRE_EXIT_MARKER_PREFIX = "[conductor-fire-exit:";
990
+
991
+ const escapeForRegExp = (value) => String(value).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
992
+
993
+ // Per-spawn nonce embedded in the marker. Without it the marker is a fixed
994
+ // string that anything writing to the log can forge — and the log is a
995
+ // verbatim copy of the fire's stdout. A fire that merely *prints* the
996
+ // literal (an agent task reading a conductor.log, or editing this file)
997
+ // would plant `[conductor-fire-exit] code=0` in its own log; if that fire
998
+ // were then SIGKILLed, leaving no genuine marker, the reaper would read the
999
+ // forged one and report COMPLETED for a task that was killed. Dogfooding
1000
+ // this repo makes that near-certain rather than theoretical.
1001
+ //
1002
+ // The nonce also means a previous run's marker can never be mistaken for
1003
+ // this run's, independently of the `logStartOffset` watermark.
1004
+ function buildFireExitMarkerToken() {
1005
+ return randomUUID().replace(/-/g, "").slice(0, 12);
1006
+ }
1007
+
1008
+ // Last exit code recorded by *this run's* wrapper shell in `raw`, or null
1009
+ // when the marker is absent (fire was SIGKILLed / the tmux server died
1010
+ // before the wrapper could run). Anchored at line start so it cannot match
1011
+ // mid-line inside quoted output, and scoped by the run's nonce.
1012
+ function parseFireExitCode(raw, markerToken) {
1013
+ if (!raw || !markerToken) return null;
1014
+ const pattern = new RegExp(
1015
+ `^${escapeForRegExp(FIRE_EXIT_MARKER_PREFIX)}${escapeForRegExp(markerToken)}\\] code=(\\d+)`,
1016
+ "gm",
1017
+ );
1018
+ let match = null;
1019
+ let candidate;
1020
+ // Scan for the *last* match rather than the first: a fresh RegExp per
1021
+ // call, because a shared /g instance carries `lastIndex` between calls.
1022
+ while ((candidate = pattern.exec(raw)) !== null) {
1023
+ match = candidate;
1024
+ }
1025
+ if (!match) return null;
1026
+ const code = Number(match[1]);
1027
+ return Number.isFinite(code) ? code : null;
1028
+ }
1029
+
1030
+ // Read the last `limit` bytes of a log file, never crossing below
1031
+ // `minOffset`. The floor matters because a task's log file is opened with
1032
+ // flags "a" and is therefore *reused* across in-place restarts: without it,
1033
+ // a previous run's trailing bytes (including its exit marker) would be
1034
+ // attributed to the current run. Callers that don't care pass minOffset 0.
1035
+ function readLogTailRaw(logPath, { limit = CHILD_OUTPUT_CAPTURE_LIMIT, minOffset = 0 } = {}) {
1036
+ if (!logPath) return "";
1037
+ try {
1038
+ const stat = statSyncFn(logPath);
1039
+ const size = Number(stat?.size) || 0;
1040
+ // A file smaller than the watermark was truncated or rotated out from
1041
+ // under us, so the watermark no longer refers to anything. Honouring it
1042
+ // would return "" for this record forever — which the reaper reads as
1043
+ // "no exit marker", i.e. a fabricated hard death for a task that may
1044
+ // have finished cleanly. Fall back to reading whatever is there.
1045
+ const floor = size < minOffset ? 0 : Math.max(0, Number(minOffset) || 0);
1046
+ const start = Math.max(floor, size - limit);
1047
+ if (size <= start) return "";
1048
+ const fd = openSyncFn(logPath, "r");
1049
+ try {
1050
+ const length = size - start;
1051
+ const buf = Buffer.alloc(length);
1052
+ readSyncFn(fd, buf, 0, length, start);
1053
+ return buf.toString("utf8");
1054
+ } finally {
1055
+ closeSyncFn(fd);
1056
+ }
1057
+ } catch {
1058
+ return "";
1059
+ }
1060
+ }
1061
+
1062
+ // Size of a log file, or null when it does not exist / cannot be read.
1063
+ // The null case is load-bearing for the reaper: `tee -a` creates the file
1064
+ // the moment it opens it, so a *missing* file means the wrapper never had
1065
+ // a writable log at all (bad cwd, unwritable project dir) — as opposed to
1066
+ // an empty-but-present file, which means the fire really did die before
1067
+ // writing anything.
1068
+ function readLogFileSize(logPath) {
1069
+ if (!logPath) return null;
1070
+ try {
1071
+ return Number(statSyncFn(logPath)?.size) || 0;
1072
+ } catch {
1073
+ return null;
1074
+ }
1075
+ }
1076
+
1077
+ // Current size of a log file, or 0 when it doesn't exist / can't be read.
1078
+ // Sampled just before a spawn so later tail reads can be floored at "what
1079
+ // this run wrote".
1080
+ function readLogSize(logPath) {
1081
+ return readLogFileSize(logPath) ?? 0;
1082
+ }
1083
+
1084
+ // How far back to hunt for this run's exit marker, and in what stride.
1085
+ // Bounded so a huge log can never turn one sweep into an unbounded read.
1086
+ const FIRE_EXIT_MARKER_SEARCH_LIMIT = 8 * 1024 * 1024;
1087
+ const FIRE_EXIT_MARKER_SEARCH_CHUNK = 64 * 1024;
1088
+
1089
+ // Find this run's exit code by scanning *backwards* from EOF for its nonce.
1090
+ //
1091
+ // Reading only the last few KB would be wrong: a task's log is not private.
1092
+ // Branch/fork tasks deliberately share a worktree — and therefore a single
1093
+ // `conductor.log` — and a task with no worktree config falls back to the
1094
+ // project directory, so every task in the project appends to the same file.
1095
+ // A neighbouring fire that prints a few KB between our exit and the next
1096
+ // reaper sweep would push our marker out of a fixed tail window. The reaper
1097
+ // would then see "no marker", conclude the fire died hard, and report
1098
+ // KILLED for a task that finished cleanly — quoting the *neighbour's*
1099
+ // output as the cause of death. The marker is this run's identity, not a
1100
+ // property of where it happens to sit in a shared stream, so search for it.
1101
+ function findFireExitCode(logPath, { minOffset = 0, markerToken = "" } = {}) {
1102
+ if (!logPath || !markerToken) return null;
1103
+ let size;
1104
+ try {
1105
+ size = Number(statSyncFn(logPath)?.size) || 0;
1106
+ } catch {
1107
+ return null;
1108
+ }
1109
+ // A file shorter than the watermark was truncated/rotated; the offset no
1110
+ // longer refers to anything, so search the whole file rather than nothing.
1111
+ const floor = size < minOffset ? 0 : Math.max(0, Number(minOffset) || 0);
1112
+ const searchFloor = Math.max(floor, size - FIRE_EXIT_MARKER_SEARCH_LIMIT);
1113
+ let fd;
1114
+ try {
1115
+ fd = openSyncFn(logPath, "r");
1116
+ } catch {
1117
+ return null;
1118
+ }
1119
+ try {
1120
+ let end = size;
1121
+ // Overlap successive chunks by enough to cover a marker split across a
1122
+ // boundary, so a line straddling two reads is still found.
1123
+ const overlap = FIRE_EXIT_MARKER_PREFIX.length + markerToken.length + 32;
1124
+ while (end > searchFloor) {
1125
+ const start = Math.max(searchFloor, end - FIRE_EXIT_MARKER_SEARCH_CHUNK);
1126
+ const length = end - start;
1127
+ const buf = Buffer.alloc(length);
1128
+ readSyncFn(fd, buf, 0, length, start);
1129
+ const code = parseFireExitCode(buf.toString("utf8"), markerToken);
1130
+ if (code !== null) return code;
1131
+ if (start <= searchFloor) break;
1132
+ end = start + overlap;
1133
+ }
1134
+ return null;
1135
+ } catch {
1136
+ return null;
1137
+ } finally {
1138
+ try {
1139
+ closeSyncFn(fd);
1140
+ } catch {
1141
+ // best effort
1142
+ }
1143
+ }
1144
+ }
1145
+
1146
+ // Collapse, redact and clamp a raw output tail so it is safe to put in a
1147
+ // daemon log line or a persisted task status summary.
1148
+ function sanitizeOutputTail(raw, limit = CHILD_OUTPUT_SUMMARY_LIMIT) {
1149
+ // Collapse first, then redact: the mask patterns stop at whitespace, so
1150
+ // a secret split across a newline would otherwise only be half-matched.
1151
+ const collapsed = String(raw || "").replace(/\s+/g, " ").trim();
1152
+ const safe = redactSecretsForLogs(collapsed, [AGENT_TOKEN]).trim();
1153
+ if (!safe) return "";
1154
+ // slice(-(limit - 1)) keeps the ellipsis inside the documented budget.
1155
+ return safe.length > limit ? `…${safe.slice(-(limit - 1))}` : safe;
1156
+ }
1157
+
1158
+ function createChildOutputCapture({ logPath = "", logStartOffset = 0 } = {}) {
1159
+ let buffer = "";
1160
+ // Decode across chunk boundaries: a raw per-chunk toString("utf8") splits
1161
+ // multi-byte characters whenever a chunk ends mid-sequence, which would
1162
+ // put mojibake into the persisted status summary.
1163
+ const decoder = new StringDecoder("utf8");
1164
+ const append = (chunk) => {
1165
+ const text =
1166
+ typeof chunk === "string" ? chunk : decoder.write(Buffer.from(chunk));
1167
+ if (!text) return;
1168
+ buffer += text;
1169
+ if (buffer.length > CHILD_OUTPUT_CAPTURE_LIMIT) {
1170
+ buffer = buffer.slice(-CHILD_OUTPUT_CAPTURE_LIMIT);
1171
+ }
1172
+ };
1173
+ // Attaching an extra `data` listener is safe alongside `.pipe()`: pipe
1174
+ // keeps the stream in flowing mode and additional listeners observe the
1175
+ // same chunks without consuming them.
1176
+ const attach = (stream) => {
1177
+ if (stream && typeof stream.on === "function") {
1178
+ stream.on("data", append);
1179
+ }
1180
+ };
1181
+
1182
+ // In tmux mode the daemon's child is the `tmux new-session` client, and
1183
+ // the fire's own output is redirected to `logPath` *inside* the session
1184
+ // (`… 2>&1 | tee -a <logPath>`). The in-memory buffer therefore only ever
1185
+ // holds tmux's own errors. When it is empty, fall back to the tail of the
1186
+ // log file so the backend's actual crash output is still reported.
1187
+ const readLogTail = () => readLogTailRaw(logPath, { minOffset: logStartOffset });
1188
+
1189
+ const tail = (limit = CHILD_OUTPUT_SUMMARY_LIMIT) =>
1190
+ sanitizeOutputTail(buffer.trim() ? buffer : readLogTail(), limit);
1191
+ return { attach, append, tail };
1192
+ }
1193
+
914
1194
  // Spawn the Fire CLI either directly (default) or inside a detached tmux
915
1195
  // session (when FIRE_TMUX_MODE_ACTIVE). In tmux mode the returned `child`
916
1196
  // is the short-lived `tmux new-session` client; once it exits with code 0
@@ -924,7 +1204,7 @@ export function startDaemon(config = {}, deps = {}) {
924
1204
  env,
925
1205
  stdio: ["inherit", "pipe", "pipe"],
926
1206
  });
927
- return { child, tmuxSession: null };
1207
+ return { child, tmuxSession: null, exitMarkerToken: "" };
928
1208
  }
929
1209
 
930
1210
  const sessionName = buildFireTmuxSessionName(taskId);
@@ -935,9 +1215,37 @@ export function startDaemon(config = {}, deps = {}) {
935
1215
  // via `tmux a -t <session>` for live observation is useless. With
936
1216
  // `tee` the same bytes go to both the pane (visible to whoever
937
1217
  // attaches) and the log file (preserved for offline inspection).
938
- const redirectedCommand = logPath
939
- ? `${innerCommandParts.join(" ")} 2>&1 | tee -a ${shellQuoteForBash(logPath)}`
940
- : innerCommandParts.join(" ");
1218
+ //
1219
+ // The trailing exit-marker is what makes a death *inside* the session
1220
+ // observable. The daemon's child is only the `tmux new-session` client,
1221
+ // so once the session exists the daemon has no handle on Fire and never
1222
+ // sees its exit code. `${PIPESTATUS[0]}` recovers Fire's own status from
1223
+ // the pipeline (plain `$?` would be tee's), and appending it to the log
1224
+ // lets the liveness reaper tell "finished normally" from "crashed" —
1225
+ // without which it could only ever guess, and guessing KILLED would
1226
+ // mis-report successful tasks as failures.
1227
+ //
1228
+ // No `exec` in this branch: the wrapper bash must outlive Fire to write
1229
+ // the marker. When Fire is SIGKILLed (or the tmux server dies) the
1230
+ // marker is simply absent, which the reaper reads as "died hard" — the
1231
+ // correct conclusion.
1232
+ //
1233
+ // Each branch owns its own `exec` (or absence of one) and the result is
1234
+ // passed to `bash -c` verbatim. Do NOT re-wrap it in `exec` at the call
1235
+ // site: `exec exec …` is a bash error ("exec: exec: not found", 127),
1236
+ // which in tmux mode means Fire never starts while the tmux client still
1237
+ // exits 0 — a task that hangs at `running` with nothing in the log.
1238
+ const quotedLogPath = logPath ? shellQuoteForBash(logPath) : "";
1239
+ // `\n` before the marker guarantees it starts its own line even when the
1240
+ // fire's last write had no trailing newline — the reader anchors on `^`.
1241
+ const exitMarkerToken = logPath ? buildFireExitMarkerToken() : "";
1242
+ const shellCommand = logPath
1243
+ ? `${innerCommandParts.join(" ")} 2>&1 | tee -a ${quotedLogPath}; ` +
1244
+ `__conductor_fire_code=\${PIPESTATUS[0]}; ` +
1245
+ `printf '\\n${FIRE_EXIT_MARKER_PREFIX}${exitMarkerToken}] code=%s\\n' ` +
1246
+ `"$__conductor_fire_code" >> ${quotedLogPath}; ` +
1247
+ `exit $__conductor_fire_code`
1248
+ : `exec ${innerCommandParts.join(" ")}`;
941
1249
 
942
1250
  // Build `-e KEY=VALUE` flags for the new session.
943
1251
  //
@@ -985,7 +1293,7 @@ export function startDaemon(config = {}, deps = {}) {
985
1293
  cwd,
986
1294
  "bash",
987
1295
  "-c",
988
- `exec ${redirectedCommand}`,
1296
+ shellCommand,
989
1297
  ];
990
1298
  log(`Spawning Fire via tmux: session=${sessionName} cwd=${cwd}`);
991
1299
  const child = spawnFn("tmux", tmuxArgs, {
@@ -997,7 +1305,7 @@ export function startDaemon(config = {}, deps = {}) {
997
1305
  if (typeof child.unref === "function") {
998
1306
  child.unref();
999
1307
  }
1000
- return { child, tmuxSession: sessionName };
1308
+ return { child, tmuxSession: sessionName, exitMarkerToken };
1001
1309
  }
1002
1310
 
1003
1311
  // Async probe: does the named tmux session still exist? Resolves to a
@@ -1016,34 +1324,45 @@ export function startDaemon(config = {}, deps = {}) {
1016
1324
  }
1017
1325
  return 5000;
1018
1326
  })();
1019
- function tmuxSessionExists(sessionName) {
1327
+ // Detailed probe: resolves `{ alive, conclusive }`.
1328
+ //
1329
+ // `conclusive` distinguishes "tmux answered, and the session is gone"
1330
+ // (has-session exited non-zero) from "we never got an answer" (spawn error,
1331
+ // tmux binary missing, wedged server hitting the timeout). Both used to
1332
+ // collapse into a bare `false`, which was fine when the only consequence
1333
+ // was dropping a bookkeeping entry. It is NOT fine now that the reaper
1334
+ // turns "not alive" into an authoritative KILLED with a cause of death: one
1335
+ // transient tmux hiccup would otherwise declare every live tmux task dead,
1336
+ // each with a confidently wrong "disappeared without an exit marker".
1337
+ function probeTmuxSession(sessionName) {
1020
1338
  return new Promise((resolve) => {
1021
1339
  if (!sessionName) {
1022
- resolve(false);
1340
+ resolve({ alive: false, conclusive: false });
1023
1341
  return;
1024
1342
  }
1025
1343
  let settled = false;
1026
1344
  let probe = null;
1027
1345
  let timer = null;
1028
- const settle = (alive) => {
1346
+ const settle = (alive, conclusive) => {
1029
1347
  if (settled) return;
1030
1348
  settled = true;
1031
1349
  if (timer) {
1032
1350
  clearTimeout(timer);
1033
1351
  timer = null;
1034
1352
  }
1035
- resolve(alive);
1353
+ resolve({ alive, conclusive });
1036
1354
  };
1037
1355
  try {
1038
1356
  probe = spawnFn("tmux", ["has-session", "-t", sessionName], {
1039
1357
  stdio: "ignore",
1040
1358
  });
1041
- probe.on("exit", (code) => settle(code === 0));
1042
- probe.on("error", () => settle(false));
1359
+ // tmux answered: exit 0 = alive, non-zero = genuinely no such session.
1360
+ probe.on("exit", (code) => settle(code === 0, true));
1361
+ // Could not run tmux at all — says nothing about the session.
1362
+ probe.on("error", () => settle(false, false));
1043
1363
  timer = setTimeout(() => {
1044
- // Probe took too long — assume the session state is unknown,
1045
- // treat as "not alive" (so the reaper falls back to safe cleanup),
1046
- // and best-effort kill the stuck child so it doesn't pile up.
1364
+ // Probe took too long — the session state is unknown. Best-effort
1365
+ // kill the stuck child so it doesn't pile up.
1047
1366
  try {
1048
1367
  if (probe && typeof probe.kill === "function") {
1049
1368
  probe.kill("SIGKILL");
@@ -1054,45 +1373,289 @@ export function startDaemon(config = {}, deps = {}) {
1054
1373
  logError(
1055
1374
  `tmux has-session probe timed out after ${TMUX_PROBE_TIMEOUT_MS}ms for session ${sessionName}`,
1056
1375
  );
1057
- settle(false);
1376
+ settle(false, false);
1058
1377
  }, TMUX_PROBE_TIMEOUT_MS);
1059
1378
  if (typeof timer.unref === "function") {
1060
1379
  timer.unref();
1061
1380
  }
1062
1381
  } catch {
1063
- settle(false);
1382
+ settle(false, false);
1064
1383
  }
1065
1384
  });
1066
1385
  }
1067
1386
 
1068
- // Walk every tmux-mode entry in `activeTaskProcesses` and remove the ones
1069
- // whose tmux session no longer exists. We don't (and can't reliably)
1070
- // observe the inner Fire process exit when we never owned it as a child,
1071
- // so this best-effort sweep is what keeps the active map from leaking
1072
- // across long daemon lifetimes.
1387
+ // Boolean form, for the callers that only gate local bookkeeping or a
1388
+ // best-effort kill and are happy to treat "unknown" as "not alive".
1389
+ async function tmuxSessionExists(sessionName) {
1390
+ return (await probeTmuxSession(sessionName)).alive;
1391
+ }
1392
+
1393
+ const BACKEND_STATUS_PROBE_TIMEOUT_MS = (() => {
1394
+ const explicit = Number(config.BACKEND_STATUS_PROBE_TIMEOUT_MS);
1395
+ if (Number.isFinite(explicit) && explicit > 0) {
1396
+ return explicit;
1397
+ }
1398
+ return 10_000;
1399
+ })();
1400
+
1401
+ // Current backend-side status of a task, lowercased, or null when it can't
1402
+ // be determined (HTTP error, unreachable backend, malformed payload).
1073
1403
  //
1074
- // Theoretical startup race (intentionally not guarded against):
1075
- // `spawnFireProcess` returns synchronously, and we set the active
1076
- // record before the spawned `tmux new-session -d` client has actually
1077
- // exited. There is therefore a microsecond-scale window in which an
1078
- // active record exists but `tmux has-session` may return false because
1079
- // the session has not finished registering with the tmux server. With
1080
- // the default 30s poll interval the chance of hitting this window is
1081
- // ~negligible, and even when hit Fire's own websocket connection
1082
- // subsequently overwrites any stale terminal status the daemon
1083
- // reports. If this ever shows up in production, add a `createdAt`
1084
- // timestamp to the record and a grace period here.
1404
+ // Hard timeout, for the same reason `tmuxSessionExists` has one: this call
1405
+ // is awaited inside the reaper sweep, and the sweep's re-entrancy latch is
1406
+ // only released in a `finally`. A backend that black-holes the connection
1407
+ // would otherwise leave the latch stuck forever permanently disabling the
1408
+ // only observer of in-session deaths, which is the exact failure this whole
1409
+ // change exists to prevent.
1410
+ async function fetchBackendTaskStatus(taskId) {
1411
+ if (!BACKEND_HTTP || !AGENT_TOKEN || !taskId) return null;
1412
+ // The timeout must cover reading the BODY, not just the headers. `fetch`
1413
+ // resolves as soon as headers arrive, so wrapping only that call leaves
1414
+ // `response.json()` unbounded: a backend that answers 200 and then stalls
1415
+ // the body (half-open proxy, hung route) parks this await forever, the
1416
+ // sweep's re-entrancy latch never clears, and the only observer of
1417
+ // in-session deaths is permanently disabled. `markBackendHttpSuccess` is
1418
+ // likewise deferred until the body actually parsed — crediting the
1419
+ // watchdog for a response we could not read would be a lie.
1420
+ const controller = typeof AbortController !== "undefined" ? new AbortController() : null;
1421
+ try {
1422
+ const status = await withTimeout(
1423
+ (async () => {
1424
+ const response = await fetchFn(`${BACKEND_HTTP}/api/tasks/${taskId}`, {
1425
+ method: "GET",
1426
+ headers: {
1427
+ Authorization: `Bearer ${AGENT_TOKEN}`,
1428
+ Accept: "application/json",
1429
+ },
1430
+ signal: controller?.signal,
1431
+ });
1432
+ if (!response?.ok) return null;
1433
+ const task = await response.json();
1434
+ markBackendHttpSuccess();
1435
+ return String(task?.status || "").trim().toLowerCase() || null;
1436
+ })(),
1437
+ BACKEND_STATUS_PROBE_TIMEOUT_MS,
1438
+ `task status probe for ${taskId}`,
1439
+ );
1440
+ return status;
1441
+ } catch (error) {
1442
+ // Release the socket; withTimeout only stops us waiting, it cannot
1443
+ // cancel the request on its own.
1444
+ try {
1445
+ controller?.abort();
1446
+ } catch {
1447
+ // best effort
1448
+ }
1449
+ logError(`Failed to query backend status for task ${taskId}: ${error?.message || error}`);
1450
+ return null;
1451
+ }
1452
+ }
1453
+
1454
+ const TERMINAL_BACKEND_TASK_STATUSES = new Set(["completed", "killed"]);
1455
+
1456
+ // A tmux session vanished. Decide what — if anything — to tell the backend.
1457
+ //
1458
+ // This is the fix for "fire dies inside its tmux session and nobody
1459
+ // notices". The daemon's child was only the `tmux new-session` client,
1460
+ // which exited 0 long ago, so there is no exit event to observe. Without a
1461
+ // report here the task sits at `running` until the next reconcile sweep
1462
+ // collects it, and reconcile's `PATCH {status:"killed"}` is rewritten by
1463
+ // the task route into `killing` → `killed`, which the upstream commit path
1464
+ // reads as `killedReason = "user_stopped"`. The user never pressed stop,
1465
+ // and the real cause is lost.
1466
+ //
1467
+ // Two hazards make a naive "session gone → report KILLED" wrong, and both
1468
+ // are handled below:
1469
+ // 1. A session also disappears when fire finished *successfully*.
1470
+ // Reporting KILLED unconditionally would rewrite completed tasks as
1471
+ // failures. We therefore classify from the exit marker the wrapper
1472
+ // shell appends to the log, not from the disappearance itself.
1473
+ // 2. Fire usually reports its own terminal status over its own
1474
+ // websocket. `commitTaskStatusUpdate` has no terminal→terminal guard,
1475
+ // so a late KILLED from us would clobber a COMPLETED from fire. We
1476
+ // ask the backend first and stay silent when the task already reached
1477
+ // a terminal state.
1478
+ //
1479
+ // Note on `suppressedExitStatusReports`: it is deliberately NOT consulted
1480
+ // here. Every path that sets it (refresh_session_inplace via
1481
+ // `stopActiveTaskProcess`) removes the active record synchronously, so the
1482
+ // reaper can never see a record it applies to — but the flag itself is
1483
+ // never cleared in tmux mode, because the tmux client's exit handler
1484
+ // returns early and only the non-early path consumes it. Honouring a flag
1485
+ // that outlives its task would silence a later, unrelated death. The
1486
+ // backend status pre-check below is the guard that actually matters.
1487
+ async function reportDeadTmuxSessionStatus(taskId, record) {
1488
+ if (!record?.projectId) {
1489
+ logError(`Cannot report terminal status for task ${taskId}: no project id on the active record`);
1490
+ return;
1491
+ }
1492
+ // The exit marker only exists because the wrapper shell writes it to
1493
+ // `logPath`. Without one there is no evidence at all, and "no marker"
1494
+ // would be misread as "died hard" — reporting every clean finish as
1495
+ // KILLED. Staying silent restores the old blind spot for this task only,
1496
+ // which is strictly better than manufacturing a wrong cause of death.
1497
+ if (!record.logPath) {
1498
+ logError(
1499
+ `Cannot classify the end of task ${taskId}: no log path on the active record, so no exit marker exists`,
1500
+ );
1501
+ return;
1502
+ }
1503
+ // Same reasoning, one level down. `tee -a` creates the log the instant it
1504
+ // opens it, so a missing file means the wrapper could never write there
1505
+ // (unwritable or vanished cwd) — and `tee` failing does NOT stop the fire,
1506
+ // which runs to completion while the tmux client still exits 0. Treating
1507
+ // that absence as "no exit marker" would report a green task as KILLED
1508
+ // with an invented "killed or OOM" cause. An empty-but-present file is
1509
+ // different: the fire really did die before writing anything.
1510
+ if (readLogFileSize(record.logPath) === null) {
1511
+ logError(
1512
+ `Cannot classify the end of task ${taskId}: log file ${record.logPath} is missing, ` +
1513
+ `so the absence of an exit marker proves nothing`,
1514
+ );
1515
+ return;
1516
+ }
1517
+
1518
+ const rawTail = readLogTailRaw(record.logPath, { minOffset: record.logStartOffset || 0 });
1519
+ // Search the file for the marker rather than hoping it is still inside
1520
+ // the tail window — see `findFireExitCode` for why a shared log makes the
1521
+ // tail unreliable. The tail is still what feeds the human-readable
1522
+ // summary; only the *verdict* needs the exhaustive lookup.
1523
+ const exitCode = findFireExitCode(record.logPath, {
1524
+ minOffset: record.logStartOffset || 0,
1525
+ markerToken: record.exitMarkerToken,
1526
+ });
1527
+ const lifetimeMs = record.spawnedAtMs ? Date.now() - record.spawnedAtMs : null;
1528
+
1529
+ // Classification. `null` exit code means the wrapper shell never got to
1530
+ // write the marker — SIGKILL, OOM kill, or a tmux server crash.
1531
+ const isKilled = exitCode === null || exitCode !== 0;
1532
+ const status = isKilled ? "KILLED" : "COMPLETED";
1533
+ //
1534
+ // The success summary deliberately is NOT the bare word "completed".
1535
+ // The web side treats a summary that merely restates the status as
1536
+ // trivial and drops it — but only when it has to synthesize the event
1537
+ // id. We send our own id (for idempotency), which bypasses that filter,
1538
+ // so a bare "completed" would be persisted and would overwrite
1539
+ // `latest_status_summary` with a word that says nothing. Naming *how* we
1540
+ // found out keeps the event worth its row: it tells the next reader that
1541
+ // fire never published its own status.
1542
+ const baseSummary =
1543
+ exitCode === null
1544
+ ? "fire tmux session disappeared without an exit marker (killed or OOM)"
1545
+ : exitCode === 0
1546
+ ? "completed; detected by the daemon after the tmux session ended (fire never reported it)"
1547
+ : exitCode === 130 || exitCode === 143
1548
+ ? `terminated (exit code ${exitCode})`
1549
+ : `exited with code ${exitCode}`;
1550
+
1551
+ // Ask the backend before overwriting anything: fire normally reports its
1552
+ // own terminal status and we must not downgrade a success.
1553
+ const backendStatus = await fetchBackendTaskStatus(taskId);
1554
+ if (backendStatus && TERMINAL_BACKEND_TASK_STATUSES.has(backendStatus)) {
1555
+ log(
1556
+ `Tmux session ${record.tmuxSession} for task ${taskId} ended; backend already ${backendStatus}, no report needed`,
1557
+ );
1558
+ return;
1559
+ }
1560
+
1561
+ // The await above yields the event loop, and "this task looks dead" is
1562
+ // precisely the trigger for an in-place restart. A `restart_task` landing
1563
+ // in that window finds no active record (we deleted it), spawns a fresh
1564
+ // fire and reports RUNNING — and our stale report would then mark the
1565
+ // *live* replacement as killed, using the previous run's log tail. Worse,
1566
+ // the daemon's own map would say running, so reconcile would never repair
1567
+ // it. Re-check that the task is still absent before speaking.
1568
+ if (activeTaskProcesses.has(taskId)) {
1569
+ log(
1570
+ `Task ${taskId} was restarted while its dead tmux session was being reported; dropping the stale terminal status`,
1571
+ );
1572
+ return;
1573
+ }
1574
+
1575
+ // Strip the marker before it reaches a user-visible summary: its content
1576
+ // is already stated by the classification above, so leaving it in just
1577
+ // appends "[conductor-fire-exit] code=1" to "exited with code 1".
1578
+ const outputTail = isKilled
1579
+ ? sanitizeOutputTail(
1580
+ rawTail.replace(
1581
+ new RegExp(`${escapeForRegExp(FIRE_EXIT_MARKER_PREFIX)}[^\\]]*\\] code=\\d+`, "g"),
1582
+ "",
1583
+ ),
1584
+ )
1585
+ : "";
1586
+ if (isKilled) {
1587
+ logError(
1588
+ `[tmux-reap] fire died inside its session task=${taskId} ` +
1589
+ `tmux=${record.tmuxSession || "none"} exit=${exitCode === null ? "unknown" : exitCode} ` +
1590
+ `lifetime_ms=${lifetimeMs === null ? "unknown" : lifetimeMs} log=${record.logPath || "none"} ` +
1591
+ `backend_status=${backendStatus || "unknown"} ` +
1592
+ `output_tail=${outputTail ? JSON.stringify(outputTail) : "<empty>"}`,
1593
+ );
1594
+ }
1595
+ const summary = outputTail ? `${baseSummary}: ${outputTail}` : baseSummary;
1596
+
1597
+ client
1598
+ .sendJson({
1599
+ type: "task_status_update",
1600
+ payload: {
1601
+ task_id: taskId,
1602
+ project_id: record.projectId,
1603
+ status,
1604
+ summary,
1605
+ // Idempotency key. Generated per call, so it only dedupes
1606
+ // transport-level redelivery of *this* message — not a second,
1607
+ // independent reaper report (the record deletion above is what
1608
+ // prevents those). It is also what makes the backend persist
1609
+ // `summary` at all: without a client-supplied id the server
1610
+ // synthesizes a fresh one per delivery.
1611
+ status_event_id: randomUUID(),
1612
+ },
1613
+ })
1614
+ .catch((err) => {
1615
+ logError(
1616
+ `Failed to report task status (${status}) for reaped tmux task ${taskId}: ${err?.message || err}`,
1617
+ );
1618
+ });
1619
+ }
1620
+
1621
+ // Walk every tmux-mode entry in `activeTaskProcesses`, remove the ones
1622
+ // whose tmux session no longer exists, and report a terminal status for
1623
+ // them. We don't (and can't reliably) observe the inner Fire process exit
1624
+ // when we never owned it as a child, so this sweep is both what keeps the
1625
+ // active map from leaking across long daemon lifetimes *and* the only
1626
+ // place a death inside the session can be noticed at all.
1627
+ //
1628
+ // Startup race (now guarded — it used to be merely theoretical):
1629
+ // `spawnFireProcess` returns synchronously and we set the active record
1630
+ // before the spawned `tmux new-session -d` client has actually exited,
1631
+ // so there is a window in which an active record exists but
1632
+ // `tmux has-session` returns false because the session has not finished
1633
+ // registering with the tmux server. That window was harmless while the
1634
+ // reaper only cleaned up local bookkeeping; now that it reports terminal
1635
+ // statuses, hitting it would kill a task that started fine. Records are
1636
+ // therefore ignored until they are `TMUX_REAP_GRACE_MS` old.
1085
1637
  async function reapDeadTmuxSessionsOnce() {
1086
1638
  const candidates = [];
1639
+ const now = Date.now();
1087
1640
  for (const [taskId, record] of activeTaskProcesses.entries()) {
1088
- if (record?.tmuxMode && record.tmuxSession) {
1089
- candidates.push([taskId, record]);
1090
- }
1641
+ if (!record?.tmuxMode || !record.tmuxSession) continue;
1642
+ const spawnedAtMs = Number(record.spawnedAtMs) || 0;
1643
+ if (spawnedAtMs && now - spawnedAtMs < TMUX_REAP_GRACE_MS) continue;
1644
+ candidates.push([taskId, record]);
1091
1645
  }
1092
1646
  for (const [taskId, record] of candidates) {
1093
- const alive = await tmuxSessionExists(record.tmuxSession);
1094
- // Only remove the entry if it still points to the same record; a
1095
- // concurrent restart_task may have replaced it while we were probing.
1647
+ const { alive, conclusive } = await probeTmuxSession(record.tmuxSession);
1648
+ // An inconclusive probe (tmux missing, wedged server, timeout) is not
1649
+ // evidence of death. Leave the record alone and retry next sweep rather
1650
+ // than cleaning up — and reporting a cause of death — on a guess.
1651
+ if (!alive && !conclusive) {
1652
+ logError(
1653
+ `Could not determine whether tmux session ${record.tmuxSession} for task ${taskId} is alive; leaving it untouched`,
1654
+ );
1655
+ continue;
1656
+ }
1657
+ // Only act if the entry still points to the same record; a concurrent
1658
+ // restart_task may have replaced it while we were probing.
1096
1659
  if (!alive && activeTaskProcesses.get(taskId) === record) {
1097
1660
  log(
1098
1661
  `Tmux session ${record.tmuxSession} for task ${taskId} no longer exists; cleaning up activeTaskProcesses entry`,
@@ -1102,6 +1665,9 @@ export function startDaemon(config = {}, deps = {}) {
1102
1665
  record.stopForceKillTimer = null;
1103
1666
  }
1104
1667
  activeTaskProcesses.delete(taskId);
1668
+ // Report *after* dropping the local entry so a slow backend round
1669
+ // trip can't make a concurrent stop_task wait on us.
1670
+ await reportDeadTmuxSessionStatus(taskId, record);
1105
1671
  }
1106
1672
  }
1107
1673
  }
@@ -1442,6 +2008,10 @@ export function startDaemon(config = {}, deps = {}) {
1442
2008
  }
1443
2009
  }
1444
2010
 
2011
+ // Monotonic suffix so two symlink swaps inside one daemon process can never
2012
+ // collide on the same temp path.
2013
+ let symlinkSequence = 0;
2014
+
1445
2015
  async function ensureTaskWorktreeSymlinks({ projectRepoRoot, projectWorkspacePath, finalCwd }) {
1446
2016
  const { symlinkPaths } = readProjectWorktreeSettings(projectWorkspacePath);
1447
2017
  for (const configuredPath of symlinkPaths) {
@@ -1455,6 +2025,27 @@ export function startDaemon(config = {}, deps = {}) {
1455
2025
  if (await isGitTrackedWorktreePath({ projectRepoRoot, sourcePath })) {
1456
2026
  continue;
1457
2027
  }
2028
+
2029
+ // Never link to a source that isn't there. `symlinkSync` does NOT
2030
+ // require its target to exist (POSIX), so a stale `worktree.symlink`
2031
+ // entry — e.g. `xr/android/build/local.properties`, generated locally by
2032
+ // the IDE and never present on this machine — would otherwise make the
2033
+ // daemon *manufacture* a dangling link on the very first worktree prep.
2034
+ // Those links are pure liability: they break tooling that follows them,
2035
+ // and they are what the EEXIST-on-reprepare bug below fed on.
2036
+ //
2037
+ // Note this is the one place where `existsSync` is the RIGHT probe:
2038
+ // here we genuinely care whether the target resolves to something real
2039
+ // (a source that is itself a dangling link is equally useless). The
2040
+ // destination probe further down must use `lstat` instead, for exactly
2041
+ // the opposite reason — see the note there.
2042
+ if (!existsSyncFn(sourcePath)) {
2043
+ logError(
2044
+ `[worktree] skipping symlink for missing source: ${configuredPath} (expected at ${sourcePath})`,
2045
+ );
2046
+ continue;
2047
+ }
2048
+
1458
2049
  const linkPath = resolveProjectScopedPath(
1459
2050
  finalCwd,
1460
2051
  configuredPath,
@@ -1462,21 +2053,70 @@ export function startDaemon(config = {}, deps = {}) {
1462
2053
  );
1463
2054
  mkdirSyncFn(path.dirname(linkPath), { recursive: true });
1464
2055
 
1465
- if (existsSyncFn(linkPath)) {
2056
+ // NOTE: probe with lstat, never existsSync. existsSync FOLLOWS symlinks,
2057
+ // so a dangling link (source deleted after the worktree was created —
2058
+ // .venv / node_modules / local.properties are all gitignored churn)
2059
+ // reads as "missing", and the symlinkSyncFn below then throws EEXIST.
2060
+ // That made the task permanently un-restartable.
2061
+ let linkStat = null;
2062
+ try {
2063
+ linkStat = lstatSyncFn(linkPath);
2064
+ } catch (error) {
2065
+ if (error?.code !== "ENOENT") {
2066
+ throw error instanceof Error ? error : new Error(String(error));
2067
+ }
2068
+ }
2069
+
2070
+ if (linkStat) {
2071
+ if (!linkStat.isSymbolicLink()) {
2072
+ throw new Error(
2073
+ `worktree symlink destination already exists and is not a symlink: ${linkPath}. ` +
2074
+ `Refusing to replace it because it may hold real data — remove it manually, ` +
2075
+ `or drop "${configuredPath}" from worktree.symlink in .conductor/settings.yaml.`,
2076
+ );
2077
+ }
2078
+ // Compare the link's TARGET, not whether that target resolves. A link
2079
+ // that already points at the right place is correct even when the
2080
+ // source is currently absent.
2081
+ const currentTarget = readlinkSyncFn(linkPath);
2082
+ const currentResolvedTarget = path.resolve(path.dirname(linkPath), currentTarget);
2083
+ if (currentResolvedTarget === sourcePath) {
2084
+ continue;
2085
+ }
2086
+ // Self-heal instead of aborting. A stale link happens whenever the
2087
+ // project moves on disk or its workspace_path binding is edited: every
2088
+ // pre-existing worktree then holds links to the OLD absolute path, and
2089
+ // throwing here made every task in them permanently un-restartable.
2090
+ //
2091
+ // Replacing is safe precisely because this entry is a SYMLINK: it
2092
+ // carries no data of its own, so unlinking destroys nothing. (The
2093
+ // non-symlink case above still throws — a real file or directory here
2094
+ // may hold user data and must never be clobbered.) The source is also
2095
+ // known to exist at this point, thanks to the guard further up, so we
2096
+ // are converging on a link that actually resolves.
2097
+ log(
2098
+ `[worktree] repointing stale symlink ${linkPath}: ${currentResolvedTarget} -> ${sourcePath}`,
2099
+ );
2100
+ // Atomic replace. branch/fork tasks deliberately SHARE one worktree
2101
+ // (identity is keyed on worktreeBranch), so two preparations can run
2102
+ // against this directory concurrently. A plain unlink+symlink leaves a
2103
+ // window where the peer's symlinkSync hits EEXIST — reintroducing the
2104
+ // very failure this function was fixed for. symlink-to-temp + rename
2105
+ // has no such window: rename(2) atomically replaces the entry.
2106
+ const tempLinkPath = `${linkPath}.conductor-tmp-${process.pid}-${symlinkSequence++}`;
2107
+ const relativeTargetForSwap = path.relative(path.dirname(linkPath), sourcePath) || ".";
2108
+ symlinkSyncFn(relativeTargetForSwap, tempLinkPath);
1466
2109
  try {
1467
- const stat = lstatSyncFn(linkPath);
1468
- if (!stat.isSymbolicLink()) {
1469
- throw new Error(`worktree symlink destination already exists: ${linkPath}`);
1470
- }
1471
- const currentTarget = readlinkSyncFn(linkPath);
1472
- const currentResolvedTarget = path.resolve(path.dirname(linkPath), currentTarget);
1473
- if (currentResolvedTarget === sourcePath) {
1474
- continue;
1475
- }
1476
- throw new Error(`worktree symlink destination already points elsewhere: ${linkPath}`);
2110
+ renameSyncFn(tempLinkPath, linkPath);
1477
2111
  } catch (error) {
2112
+ try {
2113
+ unlinkSyncFn(tempLinkPath);
2114
+ } catch {
2115
+ // best effort: never mask the original failure
2116
+ }
1478
2117
  throw error instanceof Error ? error : new Error(String(error));
1479
2118
  }
2119
+ continue;
1480
2120
  }
1481
2121
 
1482
2122
  const relativeTarget = path.relative(path.dirname(linkPath), sourcePath) || ".";
@@ -2108,6 +2748,20 @@ export function startDaemon(config = {}, deps = {}) {
2108
2748
  return 30 * 1000;
2109
2749
  })();
2110
2750
 
2751
+ // How long a freshly spawned tmux-mode task is exempt from the reaper.
2752
+ // Covers the window between "we recorded the active task" and "the tmux
2753
+ // server finished registering the session", during which `has-session`
2754
+ // can legitimately answer false for a task that started fine. Since the
2755
+ // reaper now reports terminal statuses, a false positive here would kill a
2756
+ // healthy task, so the exemption is no longer optional. Tests set 0.
2757
+ const TMUX_REAP_GRACE_MS = (() => {
2758
+ const explicit = Number(config.TMUX_REAP_GRACE_MS);
2759
+ if (Number.isFinite(explicit) && explicit >= 0) {
2760
+ return explicit;
2761
+ }
2762
+ return 15 * 1000;
2763
+ })();
2764
+
2111
2765
  // --- Auto-update state ---
2112
2766
  const VERSION_CHECK_INTERVAL_MS = 6 * 60 * 60 * 1000; // 6 hours
2113
2767
  let lastVersionCheckAt = 0;
@@ -4786,8 +5440,18 @@ export function startDaemon(config = {}, deps = {}) {
4786
5440
 
4787
5441
  let entry = activeTaskProcesses.get(taskId);
4788
5442
  if (entry?.tmuxMode && entry.tmuxSession) {
4789
- const alive = await tmuxSessionExists(entry.tmuxSession);
4790
- if (!alive) {
5443
+ // Only a conclusive "no such session" counts as death. A wedged or
5444
+ // missing tmux answers `false` too, and acting on that would ack the
5445
+ // reclaim as `stale` for a fire that is still running — the backend
5446
+ // then spawns a replacement and two fires share one worktree. It would
5447
+ // also drop the record, so the reaper could never classify the real
5448
+ // death later. When we cannot tell, keep the record and say "alive".
5449
+ const { alive, conclusive } = await probeTmuxSession(entry.tmuxSession);
5450
+ if (!alive && !conclusive) {
5451
+ logError(
5452
+ `Could not determine whether tmux session ${entry.tmuxSession} for task ${taskId} is alive; treating reclaim as still-alive`,
5453
+ );
5454
+ } else if (!alive) {
4791
5455
  if (activeTaskProcesses.get(taskId) === entry) {
4792
5456
  if (entry.stopForceKillTimer) {
4793
5457
  clearTimeout(entry.stopForceKillTimer);
@@ -4854,6 +5518,40 @@ export function startDaemon(config = {}, deps = {}) {
4854
5518
  if ((!processRecord || !processRecord.child) && !ptyRecord) {
4855
5519
  log(`Stop requested for task ${taskId}, but no active process found`);
4856
5520
  sendStopAck(false);
5521
+ // "Nothing to stop" IS the terminal answer, and the server cannot infer
5522
+ // it: after the app flips the row to `killing` it waits for us to report
5523
+ // a terminal status, and a bare `stop_ack(accepted=false)` is only
5524
+ // command bookkeeping — it never converges the task. Staying silent here
5525
+ // is what strands a task in `killing` forever once its fire has already
5526
+ // died (e.g. the tmux session was reaped before the user pressed Stop).
5527
+ // Mirror the tmux stop path and publish KILLED so the row reaches a
5528
+ // terminal state that the user can then restart from.
5529
+ const terminalProjectId =
5530
+ normalizeOptionalString(payload?.project_id) ||
5531
+ normalizeOptionalString(processRecord?.projectId);
5532
+ if (terminalProjectId) {
5533
+ client
5534
+ .sendJson({
5535
+ type: "task_status_update",
5536
+ payload: {
5537
+ task_id: taskId,
5538
+ project_id: terminalProjectId,
5539
+ status: "KILLED",
5540
+ summary: payload?.reason
5541
+ ? `stopped (${payload.reason}); no active process`
5542
+ : "stopped; no active process",
5543
+ },
5544
+ })
5545
+ .catch((err) => {
5546
+ logError(
5547
+ `Failed to report task_status_update(KILLED) for inactive task ${taskId}: ${err?.message || err}`,
5548
+ );
5549
+ });
5550
+ } else {
5551
+ logError(
5552
+ `Cannot report terminal status for inactive task ${taskId}: no project_id in stop_task payload`,
5553
+ );
5554
+ }
4857
5555
  // Even when we have no in-memory record, the task may still own a
4858
5556
  // tmux session (e.g. the daemon was restarted between spawn and
4859
5557
  // stop, or the liveness reaper removed our entry but the session
@@ -4991,6 +5689,19 @@ export function startDaemon(config = {}, deps = {}) {
4991
5689
  : "restart failed";
4992
5690
  const scrubbedError = maskErrorForLogs(error);
4993
5691
  const summary = `${prefix}: ${scrubbedError?.message || scrubbedError}`;
5692
+ // Always leave a daemon-log trace. Several early rejects (unsupported
5693
+ // backend, cwd resolution, worktree prep) return before the "Restarting
5694
+ // task …" line is ever logged, so without this a failed branch/fork left
5695
+ // the daemon log completely silent and only a generic `fire_exit` in the
5696
+ // backend DB.
5697
+ // Benign, self-explanatory outcomes (orderly shutdown, a refresh that is
5698
+ // already in flight) are logged at normal level so they don't show up as
5699
+ // errors in monitoring; genuine failures still go to stderr.
5700
+ const failureText = `${scrubbedError?.message || scrubbedError}`;
5701
+ const isBenign = /daemon shut(ting)? down|already in progress/i.test(failureText);
5702
+ (isBenign ? log : logError)(
5703
+ `[restart-spawn] failure task=${taskId} mode=${mode || "unknown"}: ${failureText}`,
5704
+ );
4994
5705
  if (mode === "refresh_session_inplace") {
4995
5706
  rememberCommandRequestAckResult(requestId, false);
4996
5707
  }
@@ -5013,6 +5724,13 @@ export function startDaemon(config = {}, deps = {}) {
5013
5724
  project_id: projectId,
5014
5725
  status: "KILLED",
5015
5726
  summary,
5727
+ // Idempotency key for this transition. The backend persists
5728
+ // `summary` only as a `taskStatusEvent` row, and it keys duplicate
5729
+ // suppression off this id — without one it has to synthesize an id,
5730
+ // which makes a redelivered report indistinguishable from a new
5731
+ // event. Supplying it keeps failure reports both persisted AND
5732
+ // deduplicated.
5733
+ status_event_id: randomUUID(),
5016
5734
  },
5017
5735
  })
5018
5736
  .catch((err) => {
@@ -5023,8 +5741,14 @@ export function startDaemon(config = {}, deps = {}) {
5023
5741
  function reportCreateTaskFailure({ taskId, projectId, requestId, error, sendAck = true }) {
5024
5742
  const normalizedTaskId = taskId ? String(taskId) : "";
5025
5743
  const normalizedProjectId = projectId ? String(projectId) : "";
5026
- const message = error instanceof Error ? error.message : String(error);
5027
- logError(`Failed to create task ${normalizedTaskId || "unknown"}: ${message}`);
5744
+ // Mirror reportRestartFailure: scrub before the message reaches the log
5745
+ // AND the persisted status summary. An un-scrubbed create failure could
5746
+ // carry the handoff URL or an inherited provider key.
5747
+ const rawMessage = error instanceof Error ? error.message : String(error);
5748
+ const message = redactSecretsForLogs(rawMessage, [AGENT_TOKEN]);
5749
+ logError(
5750
+ `[create-spawn] failure task=${normalizedTaskId || "unknown"}: ${message}`,
5751
+ );
5028
5752
  if (sendAck) {
5029
5753
  sendAgentCommandAck({
5030
5754
  requestId,
@@ -5046,6 +5770,7 @@ export function startDaemon(config = {}, deps = {}) {
5046
5770
  project_id: normalizedProjectId,
5047
5771
  status: "KILLED",
5048
5772
  summary: message,
5773
+ status_event_id: randomUUID(),
5049
5774
  },
5050
5775
  })
5051
5776
  .catch((err) => {
@@ -5304,7 +6029,10 @@ export function startDaemon(config = {}, deps = {}) {
5304
6029
  env.CONDUCTOR_BACKEND_URL = BACKEND_HTTP;
5305
6030
  }
5306
6031
 
5307
- const { child, tmuxSession } = spawnFireProcess({
6032
+ // Sampled before the spawn so it is a true "everything after this is
6033
+ // ours" watermark.
6034
+ const logStartOffset = readLogSize(logPath);
6035
+ const { child, tmuxSession, exitMarkerToken } = spawnFireProcess({
5308
6036
  taskId,
5309
6037
  args,
5310
6038
  env,
@@ -5362,10 +6090,25 @@ export function startDaemon(config = {}, deps = {}) {
5362
6090
  log(`New task workspace: ${taskDir}`);
5363
6091
  log(`Logs: ${logPath}`);
5364
6092
 
6093
+ // Same diagnostics contract as the restart/fork path: without this a
6094
+ // create_task whose backend dies at startup is a black box too.
6095
+ const outputCapture = createChildOutputCapture({ logPath, logStartOffset });
6096
+ const spawnedAtMs = Date.now();
6097
+
5365
6098
  activeTaskProcesses.set(taskId, {
5366
6099
  child,
5367
6100
  projectId,
5368
6101
  logPath,
6102
+ // Floor for later tail reads and the reaper's exit-marker scan: the
6103
+ // log file is opened with flags "a" and survives in-place restarts,
6104
+ // so bytes written before this run must not be attributed to it.
6105
+ logStartOffset,
6106
+ // Nonce this run's exit marker is tagged with, so the reaper only
6107
+ // trusts a marker written by *this* wrapper shell.
6108
+ exitMarkerToken,
6109
+ // Consumed by the tmux liveness reaper as both a grace-period anchor
6110
+ // and a lifetime figure for diagnostics.
6111
+ spawnedAtMs,
5369
6112
  stopForceKillTimer: null,
5370
6113
  managedByFireBridge: true,
5371
6114
  tmuxSession: tmuxSession || null,
@@ -5399,13 +6142,20 @@ export function startDaemon(config = {}, deps = {}) {
5399
6142
  } else if (child.stderr && typeof child.stderr.on === "function" && logStream) {
5400
6143
  child.stderr.on("data", (chunk) => logStream.write(chunk));
5401
6144
  }
6145
+ outputCapture.attach(child.stdout);
6146
+ outputCapture.attach(child.stderr);
5402
6147
  } else if (child.stderr && typeof child.stderr.on === "function") {
6148
+ outputCapture.attach(child.stderr);
5403
6149
  // Capture any error output emitted by the tmux client itself so
5404
6150
  // problems during session creation surface in daemon logs.
5405
6151
  child.stderr.on("data", (chunk) => {
5406
6152
  const text = chunk?.toString?.("utf8") ?? String(chunk ?? "");
5407
6153
  if (text.trim()) {
5408
- logError(`tmux(${tmuxSession}) stderr: ${text.trim()}`);
6154
+ logError(
6155
+ // Redact: this is the fire's raw stderr, which can carry the
6156
+ // handoff share token (argv echo) and provider/agent keys.
6157
+ `tmux(${tmuxSession}) stderr: ${redactSecretsForLogs(text.trim(), [AGENT_TOKEN])}`,
6158
+ );
5409
6159
  }
5410
6160
  });
5411
6161
  }
@@ -5421,6 +6171,20 @@ export function startDaemon(config = {}, deps = {}) {
5421
6171
  child.on("exit", (code, signal) => {
5422
6172
  const active = activeTaskProcesses.get(taskId);
5423
6173
 
6174
+ // This spawn was a tmux client but the record is gone: the reaper
6175
+ // (or a stop) already retired this task and, in the reaper's case,
6176
+ // already published its terminal status. Falling through would
6177
+ // delete whatever record a restart has since installed and report a
6178
+ // second, contradictory status — `shouldDaemonReportFireChildTerminal
6179
+ // Status(undefined)` is true, so an `exit(0)` arriving late would
6180
+ // announce COMPLETED over the reaper's verdict. Nothing left to do.
6181
+ if (tmuxSession && !active) {
6182
+ log(
6183
+ `tmux client for task ${taskId} exited after its record was retired (code=${code}, signal=${signal || "null"}); nothing to report`,
6184
+ );
6185
+ return;
6186
+ }
6187
+
5424
6188
  // In tmux mode the `tmux new-session -d` client always exits
5425
6189
  // shortly after launching the Fire session. A clean exit (code 0,
5426
6190
  // no signal) just means the session was successfully created and
@@ -5484,6 +6248,18 @@ export function startDaemon(config = {}, deps = {}) {
5484
6248
  ? "completed"
5485
6249
  : `exited with code ${code}`;
5486
6250
 
6251
+ const lifetimeMs = Date.now() - spawnedAtMs;
6252
+ const outputTail = status === "KILLED" ? outputCapture.tail() : "";
6253
+ if (status === "KILLED") {
6254
+ logError(
6255
+ `[create-spawn] abnormal exit task=${taskId} backend=${selectedBackend} ` +
6256
+ `cwd=${taskDir} tmux=${tmuxSession || "none"} exit=${code ?? "null"} ` +
6257
+ `signal=${signal || "null"} lifetime_ms=${lifetimeMs} log=${logPath} ` +
6258
+ `output_tail=${outputTail ? JSON.stringify(outputTail) : "<empty>"}`,
6259
+ );
6260
+ }
6261
+ const reportedSummary = outputTail ? `${summary}: ${outputTail}` : summary;
6262
+
5487
6263
  if (!suppressExitStatusReport && shouldDaemonReportFireChildTerminalStatus(active)) {
5488
6264
  client
5489
6265
  .sendJson({
@@ -5492,7 +6268,8 @@ export function startDaemon(config = {}, deps = {}) {
5492
6268
  task_id: taskId,
5493
6269
  project_id: projectId,
5494
6270
  status,
5495
- summary,
6271
+ summary: reportedSummary,
6272
+ status_event_id: randomUUID(),
5496
6273
  },
5497
6274
  })
5498
6275
  .catch((err) => {
@@ -5610,8 +6387,17 @@ export function startDaemon(config = {}, deps = {}) {
5610
6387
  // Probe the actual session and clean up stale entries on demand so
5611
6388
  // the restart gating below reflects reality, not the stale record.
5612
6389
  if (activeTarget?.tmuxMode && activeTarget.tmuxSession) {
5613
- const sessionAlive = await tmuxSessionExists(activeTarget.tmuxSession);
5614
- if (
6390
+ // As in reclaim: only a conclusive answer may clear the record. This
6391
+ // gate is what stops a double spawn, so trusting an inconclusive probe
6392
+ // would start a second fire alongside a live one in the same worktree.
6393
+ const { alive: sessionAlive, conclusive } = await probeTmuxSession(
6394
+ activeTarget.tmuxSession,
6395
+ );
6396
+ if (!sessionAlive && !conclusive) {
6397
+ logError(
6398
+ `Could not determine whether tmux session ${activeTarget.tmuxSession} for task ${normalizedTargetTaskId} is alive; keeping the existing record before restart`,
6399
+ );
6400
+ } else if (
5615
6401
  !sessionAlive &&
5616
6402
  activeTaskProcesses.get(normalizedTargetTaskId) === activeTarget
5617
6403
  ) {
@@ -5946,13 +6732,38 @@ export function startDaemon(config = {}, deps = {}) {
5946
6732
  env.CONDUCTOR_BACKEND_URL = BACKEND_HTTP;
5947
6733
  }
5948
6734
 
5949
- const { child, tmuxSession } = spawnFireProcess({
6735
+ // Two hazards below, one root cause: an in-place restart deliberately
6736
+ // re-uses the previous run's working directory, so ANY artifact the last
6737
+ // run left there can be mistaken for this run's own state. Both guards
6738
+ // must run before the spawn.
6739
+ //
6740
+ // (1) The fire's durable upstream outbox lives in that directory. If the
6741
+ // prior run was stopped while its websocket was down, its undelivered
6742
+ // terminal `task_status_update` (KILLED/COMPLETED) is still on disk — and
6743
+ // the run we are about to spawn would flush it during startup, marking the
6744
+ // task killed seconds after resuming and then getting shot down by the
6745
+ // server's `stop_task(task_already_killed)` reply. Purge those superseded
6746
+ // events before the new fire can pick them up.
6747
+ dropSupersededTerminalStatusEvents(taskDir, normalizedTargetTaskId);
6748
+
6749
+ // (2) The log file is opened with `flags:"a"` and re-used as well. Sample
6750
+ // its size first so it is a true "everything after this is ours"
6751
+ // watermark; without it the reaper reads the PREVIOUS run's exit marker.
6752
+ const logStartOffset = readLogSize(logPath);
6753
+
6754
+ const { child, tmuxSession, exitMarkerToken } = spawnFireProcess({
5950
6755
  taskId: normalizedTargetTaskId,
5951
6756
  args,
5952
6757
  env,
5953
6758
  cwd: taskDir,
5954
6759
  logPath,
5955
6760
  });
6761
+ // Bounded tail of the child's output + a spawn timestamp, so an abnormal
6762
+ // exit can report *why* it died and how long it survived. `logPath` lets
6763
+ // the capture recover the fire's own output in tmux mode, where the
6764
+ // daemon's child is only the `tmux new-session` client.
6765
+ const outputCapture = createChildOutputCapture({ logPath, logStartOffset });
6766
+ const spawnedAtMs = Date.now();
5956
6767
 
5957
6768
  let logStream;
5958
6769
  try {
@@ -5983,6 +6794,11 @@ export function startDaemon(config = {}, deps = {}) {
5983
6794
  child,
5984
6795
  projectId: normalizedProjectId,
5985
6796
  logPath,
6797
+ // See the create_task path: watermark for tail reads / exit-marker
6798
+ // scans, the marker nonce, and the reaper's grace-period anchor.
6799
+ logStartOffset,
6800
+ exitMarkerToken,
6801
+ spawnedAtMs,
5986
6802
  stopForceKillTimer: null,
5987
6803
  managedByFireBridge: true,
5988
6804
  tmuxSession: tmuxSession || null,
@@ -6004,17 +6820,32 @@ export function startDaemon(config = {}, deps = {}) {
6004
6820
  } else if (child.stderr && typeof child.stderr.on === "function" && logStream) {
6005
6821
  child.stderr.on("data", (chunk) => logStream.write(chunk));
6006
6822
  }
6823
+ // Capture independently of logStream: when the log file cannot be
6824
+ // opened (createWriteStream threw) the piping above is skipped
6825
+ // entirely, which is exactly when we most need the tail.
6826
+ outputCapture.attach(child.stdout);
6827
+ outputCapture.attach(child.stderr);
6007
6828
  } else if (child.stderr && typeof child.stderr.on === "function") {
6829
+ outputCapture.attach(child.stderr);
6008
6830
  child.stderr.on("data", (chunk) => {
6009
6831
  const text = chunk?.toString?.("utf8") ?? String(chunk ?? "");
6010
6832
  if (text.trim()) {
6011
- logError(`tmux(${tmuxSession}) stderr: ${text.trim()}`);
6833
+ logError(
6834
+ // Redact: this is the fire's raw stderr, which can carry the
6835
+ // handoff share token (argv echo) and provider/agent keys.
6836
+ `tmux(${tmuxSession}) stderr: ${redactSecretsForLogs(text.trim(), [AGENT_TOKEN])}`,
6837
+ );
6012
6838
  }
6013
6839
  });
6014
6840
  }
6015
6841
 
6016
6842
  child.on("error", (err) => {
6017
- logError(`Failed to spawn restart CLI for ${normalizedTargetTaskId}: ${err.message}`);
6843
+ logError(
6844
+ `[fork-spawn] spawn error task=${normalizedTargetTaskId} mode=${normalizedMode} ` +
6845
+ `backend=${selectedBackend} cwd=${taskDir} tmux=${tmuxSession || "none"}: ${
6846
+ maskErrorForLogs(err)?.message || err
6847
+ }`,
6848
+ );
6018
6849
  if (logStream) {
6019
6850
  const ts = new Date().toLocaleString("sv-SE", { timeZone: "Asia/Shanghai" }).replace(" ", "T");
6020
6851
  logStream.write(`[daemon ${ts}] spawn error: ${err.message}\n`);
@@ -6024,6 +6855,16 @@ export function startDaemon(config = {}, deps = {}) {
6024
6855
  child.on("exit", (code, signal) => {
6025
6856
  const active = activeTaskProcesses.get(normalizedTargetTaskId);
6026
6857
 
6858
+ // See the create path: a tmux client exiting after its record was
6859
+ // retired must stay silent, or it would both clobber a replacement
6860
+ // record and publish a status contradicting the reaper's verdict.
6861
+ if (tmuxSession && !active) {
6862
+ log(
6863
+ `tmux client for restarted task ${normalizedTargetTaskId} exited after its record was retired (code=${code}, signal=${signal || "null"}); nothing to report`,
6864
+ );
6865
+ return;
6866
+ }
6867
+
6027
6868
  // In tmux mode the `tmux new-session -d` client always exits soon
6028
6869
  // after launching the session. A clean exit (code 0, no signal) means
6029
6870
  // Fire is now running detached under the tmux server — keep the task
@@ -6076,6 +6917,26 @@ export function startDaemon(config = {}, deps = {}) {
6076
6917
  ? "completed"
6077
6918
  : `exited with code ${code}`;
6078
6919
 
6920
+ // Diagnostics for an abnormal exit. This is the line that makes a
6921
+ // "branch task died instantly" incident debuggable: it names the mode,
6922
+ // backend, cwd, tmux session, exit code/signal, how long the child
6923
+ // survived, where its log is, and the tail of what it actually printed.
6924
+ const lifetimeMs = Date.now() - spawnedAtMs;
6925
+ const outputTail = outputCapture.tail();
6926
+ if (status === "KILLED") {
6927
+ logError(
6928
+ `[fork-spawn] abnormal exit task=${normalizedTargetTaskId} mode=${normalizedMode} ` +
6929
+ `backend=${selectedBackend} cwd=${taskDir} tmux=${tmuxSession || "none"} ` +
6930
+ `exit=${code ?? "null"} signal=${signal || "null"} lifetime_ms=${lifetimeMs} ` +
6931
+ `log=${logPath} output_tail=${outputTail ? JSON.stringify(outputTail) : "<empty>"}`,
6932
+ );
6933
+ }
6934
+ // Surface the same tail in the status summary so it reaches the backend
6935
+ // (task_status_events.summary) and the UI, instead of a bare
6936
+ // "exited with code 1" that explains nothing.
6937
+ const reportedSummary =
6938
+ status === "KILLED" && outputTail ? `${summary}: ${outputTail}` : summary;
6939
+
6079
6940
  const shouldReportTerminalStatus =
6080
6941
  !suppressExitStatusReport &&
6081
6942
  (!acceptedRestartAckSent || shouldDaemonReportFireChildTerminalStatus(active));
@@ -6090,7 +6951,9 @@ export function startDaemon(config = {}, deps = {}) {
6090
6951
  task_id: normalizedTargetTaskId,
6091
6952
  project_id: normalizedProjectId,
6092
6953
  status,
6093
- summary,
6954
+ summary: reportedSummary,
6955
+ // Idempotency key — see the note in reportRestartFailure.
6956
+ status_event_id: randomUUID(),
6094
6957
  },
6095
6958
  })
6096
6959
  .catch((err) => {