@love-moon/conductor-cli 0.7.6 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/bin/conductor-task.js +124 -0
- package/bin/conductor.js +3 -2
- package/package.json +5 -5
- package/src/daemon.js +1126 -64
- package/src/entity-helpers.js +27 -2
- package/src/handoff-log-mask.js +43 -0
package/src/daemon.js
CHANGED
|
@@ -3,6 +3,7 @@ import path from "node:path";
|
|
|
3
3
|
import os from "node:os";
|
|
4
4
|
import { createRequire } from "node:module";
|
|
5
5
|
import { spawn, spawnSync } from "node:child_process";
|
|
6
|
+
import { randomUUID } from "node:crypto";
|
|
6
7
|
import { fileURLToPath } from "node:url";
|
|
7
8
|
|
|
8
9
|
import dotenv from "dotenv";
|
|
@@ -14,6 +15,7 @@ import {
|
|
|
14
15
|
loadConfig,
|
|
15
16
|
ConfigFileNotFound,
|
|
16
17
|
ProjectContext,
|
|
18
|
+
DurableUpstreamOutboxStore,
|
|
17
19
|
} from "@love-moon/conductor-sdk";
|
|
18
20
|
import { DaemonLogCollector } from "./log-collector.js";
|
|
19
21
|
import { envForExplicitConfigFile } from "./config-env.js";
|
|
@@ -58,7 +60,9 @@ import {
|
|
|
58
60
|
import {
|
|
59
61
|
maskHandoffUrlForLogs,
|
|
60
62
|
maskErrorForLogs,
|
|
63
|
+
redactSecretsForLogs,
|
|
61
64
|
} from "./handoff-log-mask.js";
|
|
65
|
+
import { StringDecoder } from "node:string_decoder";
|
|
62
66
|
|
|
63
67
|
dotenv.config();
|
|
64
68
|
|
|
@@ -447,7 +451,7 @@ function normalizeOptionalString(value) {
|
|
|
447
451
|
// Re-export the handoff-URL masking helpers so existing external imports
|
|
448
452
|
// keep working. Implementation lives in a dependency-free module so unit
|
|
449
453
|
// tests can import it without pulling in conductor-sdk and friends.
|
|
450
|
-
export { maskHandoffUrlForLogs, maskErrorForLogs };
|
|
454
|
+
export { maskHandoffUrlForLogs, maskErrorForLogs, redactSecretsForLogs };
|
|
451
455
|
|
|
452
456
|
function normalizeTerminalResumeStrategy(value) {
|
|
453
457
|
const normalized = normalizeOptionalString(value);
|
|
@@ -628,6 +632,52 @@ function stripPtyTaskScopedEnv(source) {
|
|
|
628
632
|
return env;
|
|
629
633
|
}
|
|
630
634
|
|
|
635
|
+
/**
|
|
636
|
+
* Purge terminal `task_status_update` events left undelivered by a previous run
|
|
637
|
+
* of `taskId` in `taskDir`, before a restart re-uses that same directory.
|
|
638
|
+
*
|
|
639
|
+
* The fire's durable upstream outbox is stored under the fire's cwd, keyed by
|
|
640
|
+
* the `task:<id>` delivery scope. An in-place restart intentionally re-uses the
|
|
641
|
+
* cwd, so without this purge the new run flushes the old run's KILLED/COMPLETED
|
|
642
|
+
* event on startup and kills the task it just resumed. Best-effort: a restart
|
|
643
|
+
* must never fail because of outbox bookkeeping.
|
|
644
|
+
*/
|
|
645
|
+
function dropSupersededTerminalStatusEvents(taskDir, taskId) {
|
|
646
|
+
if (!taskDir || !taskId) return;
|
|
647
|
+
try {
|
|
648
|
+
const store = DurableUpstreamOutboxStore.forProjectPath(taskDir, `task:${taskId}`);
|
|
649
|
+
// Distinguish "the purge failed" from "the purge does not exist here".
|
|
650
|
+
// The CLI loads the INSTALLED copy of conductor-sdk, not this repo's
|
|
651
|
+
// source, so an un-rebuilt or mismatched package silently lacks this
|
|
652
|
+
// method — and a bare try/catch would swallow the resulting TypeError
|
|
653
|
+
// exactly like a benign fs error, leaving every restart unprotected with
|
|
654
|
+
// no way to tell from the log. Say so explicitly instead. We still do not
|
|
655
|
+
// throw: an unprotected restart usually succeeds, whereas refusing to
|
|
656
|
+
// spawn would break restart outright.
|
|
657
|
+
if (typeof store?.dropPendingTerminalStatusEvents !== "function") {
|
|
658
|
+
logError(
|
|
659
|
+
`Cannot purge superseded terminal status events for task ${taskId}: the installed ` +
|
|
660
|
+
`@love-moon/conductor-sdk has no DurableUpstreamOutboxStore.dropPendingTerminalStatusEvents. ` +
|
|
661
|
+
`This restart is unprotected — a terminal status left by the previous run may kill it. ` +
|
|
662
|
+
`Rebuild and reinstall the SDK (make install-cli).`,
|
|
663
|
+
);
|
|
664
|
+
return;
|
|
665
|
+
}
|
|
666
|
+
const dropped = store.dropPendingTerminalStatusEvents();
|
|
667
|
+
if (dropped.length > 0) {
|
|
668
|
+
log(
|
|
669
|
+
`Dropped ${dropped.length} superseded terminal status event(s) for task ${taskId} before restart: ${dropped
|
|
670
|
+
.map((entry) => `${entry.payload?.status ?? "?"}@${entry.createdAt}`)
|
|
671
|
+
.join(", ")}`,
|
|
672
|
+
);
|
|
673
|
+
}
|
|
674
|
+
} catch (error) {
|
|
675
|
+
logError(
|
|
676
|
+
`Failed to purge superseded terminal status events for task ${taskId}: ${error?.message || error}`,
|
|
677
|
+
);
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
|
|
631
681
|
function buildPtyTaskEnv(baseEnv = process.env, launchEnv = {}) {
|
|
632
682
|
const parentEnv = stripPtyTaskScopedEnv(baseEnv);
|
|
633
683
|
const taskLaunchEnv = stripPtyTaskScopedEnv(launchEnv);
|
|
@@ -827,6 +877,11 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
827
877
|
const symlinkSyncFn = deps.symlinkSync || fs.symlinkSync;
|
|
828
878
|
const unlinkSyncFn = deps.unlinkSync || fs.unlinkSync;
|
|
829
879
|
const renameSyncFn = deps.renameSync || fs.renameSync;
|
|
880
|
+
// Used to recover a crashed fire's output from its log file when the daemon
|
|
881
|
+
// could not observe the process directly (tmux mode).
|
|
882
|
+
const openSyncFn = deps.openSync || fs.openSync;
|
|
883
|
+
const readSyncFn = deps.readSync || fs.readSync;
|
|
884
|
+
const closeSyncFn = deps.closeSync || fs.closeSync;
|
|
830
885
|
const createWriteStreamFn = deps.createWriteStream || fs.createWriteStream;
|
|
831
886
|
const fetchFn = deps.fetch || fetch;
|
|
832
887
|
const createRtcPeerConnection = deps.createRtcPeerConnection || null;
|
|
@@ -911,6 +966,231 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
911
966
|
return `${buildFireTmuxSessionPrefix(taskId)}${uniq}`;
|
|
912
967
|
}
|
|
913
968
|
|
|
969
|
+
// Observability for spawned fires (RFC: fork-spawn black box).
|
|
970
|
+
//
|
|
971
|
+
// When a forked/branched task's backend dies during startup, the daemon
|
|
972
|
+
// previously reported a bare `exited with code N` and wrote nothing to the
|
|
973
|
+
// daemon log, so the real cause (backend auth failure, missing CLI, tmux
|
|
974
|
+
// launch error, …) was unrecoverable after the fact — even with DB access
|
|
975
|
+
// the only artifact was a generic `fire_exit`. We keep a bounded tail of the
|
|
976
|
+
// child's output so an abnormal exit can name its own cause.
|
|
977
|
+
//
|
|
978
|
+
// The tail is masked with `maskHandoffUrlForLogs` before it reaches any log
|
|
979
|
+
// or status summary: the handoff prompt embeds a share token, and a backend
|
|
980
|
+
// that echoes its argv on failure would otherwise leak a read-grant for the
|
|
981
|
+
// entire transcript into the daemon log and the task status summary.
|
|
982
|
+
const CHILD_OUTPUT_CAPTURE_LIMIT = 4000;
|
|
983
|
+
const CHILD_OUTPUT_SUMMARY_LIMIT = 400;
|
|
984
|
+
|
|
985
|
+
// Sentinel appended to a tmux-hosted fire's log by the wrapper shell (see
|
|
986
|
+
// `spawnFireProcess`). It is the *only* channel through which the daemon
|
|
987
|
+
// can learn the exit code of a process it does not own. The full marker is
|
|
988
|
+
// `[conductor-fire-exit:<per-spawn nonce>] code=<n>`.
|
|
989
|
+
const FIRE_EXIT_MARKER_PREFIX = "[conductor-fire-exit:";
|
|
990
|
+
|
|
991
|
+
const escapeForRegExp = (value) => String(value).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
992
|
+
|
|
993
|
+
// Per-spawn nonce embedded in the marker. Without it the marker is a fixed
|
|
994
|
+
// string that anything writing to the log can forge — and the log is a
|
|
995
|
+
// verbatim copy of the fire's stdout. A fire that merely *prints* the
|
|
996
|
+
// literal (an agent task reading a conductor.log, or editing this file)
|
|
997
|
+
// would plant `[conductor-fire-exit] code=0` in its own log; if that fire
|
|
998
|
+
// were then SIGKILLed, leaving no genuine marker, the reaper would read the
|
|
999
|
+
// forged one and report COMPLETED for a task that was killed. Dogfooding
|
|
1000
|
+
// this repo makes that near-certain rather than theoretical.
|
|
1001
|
+
//
|
|
1002
|
+
// The nonce also means a previous run's marker can never be mistaken for
|
|
1003
|
+
// this run's, independently of the `logStartOffset` watermark.
|
|
1004
|
+
function buildFireExitMarkerToken() {
|
|
1005
|
+
return randomUUID().replace(/-/g, "").slice(0, 12);
|
|
1006
|
+
}
|
|
1007
|
+
|
|
1008
|
+
// Last exit code recorded by *this run's* wrapper shell in `raw`, or null
|
|
1009
|
+
// when the marker is absent (fire was SIGKILLed / the tmux server died
|
|
1010
|
+
// before the wrapper could run). Anchored at line start so it cannot match
|
|
1011
|
+
// mid-line inside quoted output, and scoped by the run's nonce.
|
|
1012
|
+
function parseFireExitCode(raw, markerToken) {
|
|
1013
|
+
if (!raw || !markerToken) return null;
|
|
1014
|
+
const pattern = new RegExp(
|
|
1015
|
+
`^${escapeForRegExp(FIRE_EXIT_MARKER_PREFIX)}${escapeForRegExp(markerToken)}\\] code=(\\d+)`,
|
|
1016
|
+
"gm",
|
|
1017
|
+
);
|
|
1018
|
+
let match = null;
|
|
1019
|
+
let candidate;
|
|
1020
|
+
// Scan for the *last* match rather than the first: a fresh RegExp per
|
|
1021
|
+
// call, because a shared /g instance carries `lastIndex` between calls.
|
|
1022
|
+
while ((candidate = pattern.exec(raw)) !== null) {
|
|
1023
|
+
match = candidate;
|
|
1024
|
+
}
|
|
1025
|
+
if (!match) return null;
|
|
1026
|
+
const code = Number(match[1]);
|
|
1027
|
+
return Number.isFinite(code) ? code : null;
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
// Read the last `limit` bytes of a log file, never crossing below
|
|
1031
|
+
// `minOffset`. The floor matters because a task's log file is opened with
|
|
1032
|
+
// flags "a" and is therefore *reused* across in-place restarts: without it,
|
|
1033
|
+
// a previous run's trailing bytes (including its exit marker) would be
|
|
1034
|
+
// attributed to the current run. Callers that don't care pass minOffset 0.
|
|
1035
|
+
function readLogTailRaw(logPath, { limit = CHILD_OUTPUT_CAPTURE_LIMIT, minOffset = 0 } = {}) {
|
|
1036
|
+
if (!logPath) return "";
|
|
1037
|
+
try {
|
|
1038
|
+
const stat = statSyncFn(logPath);
|
|
1039
|
+
const size = Number(stat?.size) || 0;
|
|
1040
|
+
// A file smaller than the watermark was truncated or rotated out from
|
|
1041
|
+
// under us, so the watermark no longer refers to anything. Honouring it
|
|
1042
|
+
// would return "" for this record forever — which the reaper reads as
|
|
1043
|
+
// "no exit marker", i.e. a fabricated hard death for a task that may
|
|
1044
|
+
// have finished cleanly. Fall back to reading whatever is there.
|
|
1045
|
+
const floor = size < minOffset ? 0 : Math.max(0, Number(minOffset) || 0);
|
|
1046
|
+
const start = Math.max(floor, size - limit);
|
|
1047
|
+
if (size <= start) return "";
|
|
1048
|
+
const fd = openSyncFn(logPath, "r");
|
|
1049
|
+
try {
|
|
1050
|
+
const length = size - start;
|
|
1051
|
+
const buf = Buffer.alloc(length);
|
|
1052
|
+
readSyncFn(fd, buf, 0, length, start);
|
|
1053
|
+
return buf.toString("utf8");
|
|
1054
|
+
} finally {
|
|
1055
|
+
closeSyncFn(fd);
|
|
1056
|
+
}
|
|
1057
|
+
} catch {
|
|
1058
|
+
return "";
|
|
1059
|
+
}
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
// Size of a log file, or null when it does not exist / cannot be read.
|
|
1063
|
+
// The null case is load-bearing for the reaper: `tee -a` creates the file
|
|
1064
|
+
// the moment it opens it, so a *missing* file means the wrapper never had
|
|
1065
|
+
// a writable log at all (bad cwd, unwritable project dir) — as opposed to
|
|
1066
|
+
// an empty-but-present file, which means the fire really did die before
|
|
1067
|
+
// writing anything.
|
|
1068
|
+
function readLogFileSize(logPath) {
|
|
1069
|
+
if (!logPath) return null;
|
|
1070
|
+
try {
|
|
1071
|
+
return Number(statSyncFn(logPath)?.size) || 0;
|
|
1072
|
+
} catch {
|
|
1073
|
+
return null;
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
// Current size of a log file, or 0 when it doesn't exist / can't be read.
|
|
1078
|
+
// Sampled just before a spawn so later tail reads can be floored at "what
|
|
1079
|
+
// this run wrote".
|
|
1080
|
+
function readLogSize(logPath) {
|
|
1081
|
+
return readLogFileSize(logPath) ?? 0;
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
// How far back to hunt for this run's exit marker, and in what stride.
|
|
1085
|
+
// Bounded so a huge log can never turn one sweep into an unbounded read.
|
|
1086
|
+
const FIRE_EXIT_MARKER_SEARCH_LIMIT = 8 * 1024 * 1024;
|
|
1087
|
+
const FIRE_EXIT_MARKER_SEARCH_CHUNK = 64 * 1024;
|
|
1088
|
+
|
|
1089
|
+
// Find this run's exit code by scanning *backwards* from EOF for its nonce.
|
|
1090
|
+
//
|
|
1091
|
+
// Reading only the last few KB would be wrong: a task's log is not private.
|
|
1092
|
+
// Branch/fork tasks deliberately share a worktree — and therefore a single
|
|
1093
|
+
// `conductor.log` — and a task with no worktree config falls back to the
|
|
1094
|
+
// project directory, so every task in the project appends to the same file.
|
|
1095
|
+
// A neighbouring fire that prints a few KB between our exit and the next
|
|
1096
|
+
// reaper sweep would push our marker out of a fixed tail window. The reaper
|
|
1097
|
+
// would then see "no marker", conclude the fire died hard, and report
|
|
1098
|
+
// KILLED for a task that finished cleanly — quoting the *neighbour's*
|
|
1099
|
+
// output as the cause of death. The marker is this run's identity, not a
|
|
1100
|
+
// property of where it happens to sit in a shared stream, so search for it.
|
|
1101
|
+
function findFireExitCode(logPath, { minOffset = 0, markerToken = "" } = {}) {
|
|
1102
|
+
if (!logPath || !markerToken) return null;
|
|
1103
|
+
let size;
|
|
1104
|
+
try {
|
|
1105
|
+
size = Number(statSyncFn(logPath)?.size) || 0;
|
|
1106
|
+
} catch {
|
|
1107
|
+
return null;
|
|
1108
|
+
}
|
|
1109
|
+
// A file shorter than the watermark was truncated/rotated; the offset no
|
|
1110
|
+
// longer refers to anything, so search the whole file rather than nothing.
|
|
1111
|
+
const floor = size < minOffset ? 0 : Math.max(0, Number(minOffset) || 0);
|
|
1112
|
+
const searchFloor = Math.max(floor, size - FIRE_EXIT_MARKER_SEARCH_LIMIT);
|
|
1113
|
+
let fd;
|
|
1114
|
+
try {
|
|
1115
|
+
fd = openSyncFn(logPath, "r");
|
|
1116
|
+
} catch {
|
|
1117
|
+
return null;
|
|
1118
|
+
}
|
|
1119
|
+
try {
|
|
1120
|
+
let end = size;
|
|
1121
|
+
// Overlap successive chunks by enough to cover a marker split across a
|
|
1122
|
+
// boundary, so a line straddling two reads is still found.
|
|
1123
|
+
const overlap = FIRE_EXIT_MARKER_PREFIX.length + markerToken.length + 32;
|
|
1124
|
+
while (end > searchFloor) {
|
|
1125
|
+
const start = Math.max(searchFloor, end - FIRE_EXIT_MARKER_SEARCH_CHUNK);
|
|
1126
|
+
const length = end - start;
|
|
1127
|
+
const buf = Buffer.alloc(length);
|
|
1128
|
+
readSyncFn(fd, buf, 0, length, start);
|
|
1129
|
+
const code = parseFireExitCode(buf.toString("utf8"), markerToken);
|
|
1130
|
+
if (code !== null) return code;
|
|
1131
|
+
if (start <= searchFloor) break;
|
|
1132
|
+
end = start + overlap;
|
|
1133
|
+
}
|
|
1134
|
+
return null;
|
|
1135
|
+
} catch {
|
|
1136
|
+
return null;
|
|
1137
|
+
} finally {
|
|
1138
|
+
try {
|
|
1139
|
+
closeSyncFn(fd);
|
|
1140
|
+
} catch {
|
|
1141
|
+
// best effort
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
|
|
1146
|
+
// Collapse, redact and clamp a raw output tail so it is safe to put in a
|
|
1147
|
+
// daemon log line or a persisted task status summary.
|
|
1148
|
+
function sanitizeOutputTail(raw, limit = CHILD_OUTPUT_SUMMARY_LIMIT) {
|
|
1149
|
+
// Collapse first, then redact: the mask patterns stop at whitespace, so
|
|
1150
|
+
// a secret split across a newline would otherwise only be half-matched.
|
|
1151
|
+
const collapsed = String(raw || "").replace(/\s+/g, " ").trim();
|
|
1152
|
+
const safe = redactSecretsForLogs(collapsed, [AGENT_TOKEN]).trim();
|
|
1153
|
+
if (!safe) return "";
|
|
1154
|
+
// slice(-(limit - 1)) keeps the ellipsis inside the documented budget.
|
|
1155
|
+
return safe.length > limit ? `…${safe.slice(-(limit - 1))}` : safe;
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
function createChildOutputCapture({ logPath = "", logStartOffset = 0 } = {}) {
|
|
1159
|
+
let buffer = "";
|
|
1160
|
+
// Decode across chunk boundaries: a raw per-chunk toString("utf8") splits
|
|
1161
|
+
// multi-byte characters whenever a chunk ends mid-sequence, which would
|
|
1162
|
+
// put mojibake into the persisted status summary.
|
|
1163
|
+
const decoder = new StringDecoder("utf8");
|
|
1164
|
+
const append = (chunk) => {
|
|
1165
|
+
const text =
|
|
1166
|
+
typeof chunk === "string" ? chunk : decoder.write(Buffer.from(chunk));
|
|
1167
|
+
if (!text) return;
|
|
1168
|
+
buffer += text;
|
|
1169
|
+
if (buffer.length > CHILD_OUTPUT_CAPTURE_LIMIT) {
|
|
1170
|
+
buffer = buffer.slice(-CHILD_OUTPUT_CAPTURE_LIMIT);
|
|
1171
|
+
}
|
|
1172
|
+
};
|
|
1173
|
+
// Attaching an extra `data` listener is safe alongside `.pipe()`: pipe
|
|
1174
|
+
// keeps the stream in flowing mode and additional listeners observe the
|
|
1175
|
+
// same chunks without consuming them.
|
|
1176
|
+
const attach = (stream) => {
|
|
1177
|
+
if (stream && typeof stream.on === "function") {
|
|
1178
|
+
stream.on("data", append);
|
|
1179
|
+
}
|
|
1180
|
+
};
|
|
1181
|
+
|
|
1182
|
+
// In tmux mode the daemon's child is the `tmux new-session` client, and
|
|
1183
|
+
// the fire's own output is redirected to `logPath` *inside* the session
|
|
1184
|
+
// (`… 2>&1 | tee -a <logPath>`). The in-memory buffer therefore only ever
|
|
1185
|
+
// holds tmux's own errors. When it is empty, fall back to the tail of the
|
|
1186
|
+
// log file so the backend's actual crash output is still reported.
|
|
1187
|
+
const readLogTail = () => readLogTailRaw(logPath, { minOffset: logStartOffset });
|
|
1188
|
+
|
|
1189
|
+
const tail = (limit = CHILD_OUTPUT_SUMMARY_LIMIT) =>
|
|
1190
|
+
sanitizeOutputTail(buffer.trim() ? buffer : readLogTail(), limit);
|
|
1191
|
+
return { attach, append, tail };
|
|
1192
|
+
}
|
|
1193
|
+
|
|
914
1194
|
// Spawn the Fire CLI either directly (default) or inside a detached tmux
|
|
915
1195
|
// session (when FIRE_TMUX_MODE_ACTIVE). In tmux mode the returned `child`
|
|
916
1196
|
// is the short-lived `tmux new-session` client; once it exits with code 0
|
|
@@ -924,7 +1204,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
924
1204
|
env,
|
|
925
1205
|
stdio: ["inherit", "pipe", "pipe"],
|
|
926
1206
|
});
|
|
927
|
-
return { child, tmuxSession: null };
|
|
1207
|
+
return { child, tmuxSession: null, exitMarkerToken: "" };
|
|
928
1208
|
}
|
|
929
1209
|
|
|
930
1210
|
const sessionName = buildFireTmuxSessionName(taskId);
|
|
@@ -935,9 +1215,37 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
935
1215
|
// via `tmux a -t <session>` for live observation is useless. With
|
|
936
1216
|
// `tee` the same bytes go to both the pane (visible to whoever
|
|
937
1217
|
// attaches) and the log file (preserved for offline inspection).
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
1218
|
+
//
|
|
1219
|
+
// The trailing exit-marker is what makes a death *inside* the session
|
|
1220
|
+
// observable. The daemon's child is only the `tmux new-session` client,
|
|
1221
|
+
// so once the session exists the daemon has no handle on Fire and never
|
|
1222
|
+
// sees its exit code. `${PIPESTATUS[0]}` recovers Fire's own status from
|
|
1223
|
+
// the pipeline (plain `$?` would be tee's), and appending it to the log
|
|
1224
|
+
// lets the liveness reaper tell "finished normally" from "crashed" —
|
|
1225
|
+
// without which it could only ever guess, and guessing KILLED would
|
|
1226
|
+
// mis-report successful tasks as failures.
|
|
1227
|
+
//
|
|
1228
|
+
// No `exec` in this branch: the wrapper bash must outlive Fire to write
|
|
1229
|
+
// the marker. When Fire is SIGKILLed (or the tmux server dies) the
|
|
1230
|
+
// marker is simply absent, which the reaper reads as "died hard" — the
|
|
1231
|
+
// correct conclusion.
|
|
1232
|
+
//
|
|
1233
|
+
// Each branch owns its own `exec` (or absence of one) and the result is
|
|
1234
|
+
// passed to `bash -c` verbatim. Do NOT re-wrap it in `exec` at the call
|
|
1235
|
+
// site: `exec exec …` is a bash error ("exec: exec: not found", 127),
|
|
1236
|
+
// which in tmux mode means Fire never starts while the tmux client still
|
|
1237
|
+
// exits 0 — a task that hangs at `running` with nothing in the log.
|
|
1238
|
+
const quotedLogPath = logPath ? shellQuoteForBash(logPath) : "";
|
|
1239
|
+
// `\n` before the marker guarantees it starts its own line even when the
|
|
1240
|
+
// fire's last write had no trailing newline — the reader anchors on `^`.
|
|
1241
|
+
const exitMarkerToken = logPath ? buildFireExitMarkerToken() : "";
|
|
1242
|
+
const shellCommand = logPath
|
|
1243
|
+
? `${innerCommandParts.join(" ")} 2>&1 | tee -a ${quotedLogPath}; ` +
|
|
1244
|
+
`__conductor_fire_code=\${PIPESTATUS[0]}; ` +
|
|
1245
|
+
`printf '\\n${FIRE_EXIT_MARKER_PREFIX}${exitMarkerToken}] code=%s\\n' ` +
|
|
1246
|
+
`"$__conductor_fire_code" >> ${quotedLogPath}; ` +
|
|
1247
|
+
`exit $__conductor_fire_code`
|
|
1248
|
+
: `exec ${innerCommandParts.join(" ")}`;
|
|
941
1249
|
|
|
942
1250
|
// Build `-e KEY=VALUE` flags for the new session.
|
|
943
1251
|
//
|
|
@@ -985,7 +1293,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
985
1293
|
cwd,
|
|
986
1294
|
"bash",
|
|
987
1295
|
"-c",
|
|
988
|
-
|
|
1296
|
+
shellCommand,
|
|
989
1297
|
];
|
|
990
1298
|
log(`Spawning Fire via tmux: session=${sessionName} cwd=${cwd}`);
|
|
991
1299
|
const child = spawnFn("tmux", tmuxArgs, {
|
|
@@ -997,7 +1305,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
997
1305
|
if (typeof child.unref === "function") {
|
|
998
1306
|
child.unref();
|
|
999
1307
|
}
|
|
1000
|
-
return { child, tmuxSession: sessionName };
|
|
1308
|
+
return { child, tmuxSession: sessionName, exitMarkerToken };
|
|
1001
1309
|
}
|
|
1002
1310
|
|
|
1003
1311
|
// Async probe: does the named tmux session still exist? Resolves to a
|
|
@@ -1016,34 +1324,45 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1016
1324
|
}
|
|
1017
1325
|
return 5000;
|
|
1018
1326
|
})();
|
|
1019
|
-
|
|
1327
|
+
// Detailed probe: resolves `{ alive, conclusive }`.
|
|
1328
|
+
//
|
|
1329
|
+
// `conclusive` distinguishes "tmux answered, and the session is gone"
|
|
1330
|
+
// (has-session exited non-zero) from "we never got an answer" (spawn error,
|
|
1331
|
+
// tmux binary missing, wedged server hitting the timeout). Both used to
|
|
1332
|
+
// collapse into a bare `false`, which was fine when the only consequence
|
|
1333
|
+
// was dropping a bookkeeping entry. It is NOT fine now that the reaper
|
|
1334
|
+
// turns "not alive" into an authoritative KILLED with a cause of death: one
|
|
1335
|
+
// transient tmux hiccup would otherwise declare every live tmux task dead,
|
|
1336
|
+
// each with a confidently wrong "disappeared without an exit marker".
|
|
1337
|
+
function probeTmuxSession(sessionName) {
|
|
1020
1338
|
return new Promise((resolve) => {
|
|
1021
1339
|
if (!sessionName) {
|
|
1022
|
-
resolve(false);
|
|
1340
|
+
resolve({ alive: false, conclusive: false });
|
|
1023
1341
|
return;
|
|
1024
1342
|
}
|
|
1025
1343
|
let settled = false;
|
|
1026
1344
|
let probe = null;
|
|
1027
1345
|
let timer = null;
|
|
1028
|
-
const settle = (alive) => {
|
|
1346
|
+
const settle = (alive, conclusive) => {
|
|
1029
1347
|
if (settled) return;
|
|
1030
1348
|
settled = true;
|
|
1031
1349
|
if (timer) {
|
|
1032
1350
|
clearTimeout(timer);
|
|
1033
1351
|
timer = null;
|
|
1034
1352
|
}
|
|
1035
|
-
resolve(alive);
|
|
1353
|
+
resolve({ alive, conclusive });
|
|
1036
1354
|
};
|
|
1037
1355
|
try {
|
|
1038
1356
|
probe = spawnFn("tmux", ["has-session", "-t", sessionName], {
|
|
1039
1357
|
stdio: "ignore",
|
|
1040
1358
|
});
|
|
1041
|
-
|
|
1042
|
-
probe.on("
|
|
1359
|
+
// tmux answered: exit 0 = alive, non-zero = genuinely no such session.
|
|
1360
|
+
probe.on("exit", (code) => settle(code === 0, true));
|
|
1361
|
+
// Could not run tmux at all — says nothing about the session.
|
|
1362
|
+
probe.on("error", () => settle(false, false));
|
|
1043
1363
|
timer = setTimeout(() => {
|
|
1044
|
-
// Probe took too long —
|
|
1045
|
-
//
|
|
1046
|
-
// and best-effort kill the stuck child so it doesn't pile up.
|
|
1364
|
+
// Probe took too long — the session state is unknown. Best-effort
|
|
1365
|
+
// kill the stuck child so it doesn't pile up.
|
|
1047
1366
|
try {
|
|
1048
1367
|
if (probe && typeof probe.kill === "function") {
|
|
1049
1368
|
probe.kill("SIGKILL");
|
|
@@ -1054,45 +1373,289 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1054
1373
|
logError(
|
|
1055
1374
|
`tmux has-session probe timed out after ${TMUX_PROBE_TIMEOUT_MS}ms for session ${sessionName}`,
|
|
1056
1375
|
);
|
|
1057
|
-
settle(false);
|
|
1376
|
+
settle(false, false);
|
|
1058
1377
|
}, TMUX_PROBE_TIMEOUT_MS);
|
|
1059
1378
|
if (typeof timer.unref === "function") {
|
|
1060
1379
|
timer.unref();
|
|
1061
1380
|
}
|
|
1062
1381
|
} catch {
|
|
1063
|
-
settle(false);
|
|
1382
|
+
settle(false, false);
|
|
1383
|
+
}
|
|
1384
|
+
});
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1387
|
+
// Boolean form, for the callers that only gate local bookkeeping or a
|
|
1388
|
+
// best-effort kill and are happy to treat "unknown" as "not alive".
|
|
1389
|
+
async function tmuxSessionExists(sessionName) {
|
|
1390
|
+
return (await probeTmuxSession(sessionName)).alive;
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
const BACKEND_STATUS_PROBE_TIMEOUT_MS = (() => {
|
|
1394
|
+
const explicit = Number(config.BACKEND_STATUS_PROBE_TIMEOUT_MS);
|
|
1395
|
+
if (Number.isFinite(explicit) && explicit > 0) {
|
|
1396
|
+
return explicit;
|
|
1397
|
+
}
|
|
1398
|
+
return 10_000;
|
|
1399
|
+
})();
|
|
1400
|
+
|
|
1401
|
+
// Current backend-side status of a task, lowercased, or null when it can't
|
|
1402
|
+
// be determined (HTTP error, unreachable backend, malformed payload).
|
|
1403
|
+
//
|
|
1404
|
+
// Hard timeout, for the same reason `tmuxSessionExists` has one: this call
|
|
1405
|
+
// is awaited inside the reaper sweep, and the sweep's re-entrancy latch is
|
|
1406
|
+
// only released in a `finally`. A backend that black-holes the connection
|
|
1407
|
+
// would otherwise leave the latch stuck forever — permanently disabling the
|
|
1408
|
+
// only observer of in-session deaths, which is the exact failure this whole
|
|
1409
|
+
// change exists to prevent.
|
|
1410
|
+
async function fetchBackendTaskStatus(taskId) {
|
|
1411
|
+
if (!BACKEND_HTTP || !AGENT_TOKEN || !taskId) return null;
|
|
1412
|
+
// The timeout must cover reading the BODY, not just the headers. `fetch`
|
|
1413
|
+
// resolves as soon as headers arrive, so wrapping only that call leaves
|
|
1414
|
+
// `response.json()` unbounded: a backend that answers 200 and then stalls
|
|
1415
|
+
// the body (half-open proxy, hung route) parks this await forever, the
|
|
1416
|
+
// sweep's re-entrancy latch never clears, and the only observer of
|
|
1417
|
+
// in-session deaths is permanently disabled. `markBackendHttpSuccess` is
|
|
1418
|
+
// likewise deferred until the body actually parsed — crediting the
|
|
1419
|
+
// watchdog for a response we could not read would be a lie.
|
|
1420
|
+
const controller = typeof AbortController !== "undefined" ? new AbortController() : null;
|
|
1421
|
+
try {
|
|
1422
|
+
const status = await withTimeout(
|
|
1423
|
+
(async () => {
|
|
1424
|
+
const response = await fetchFn(`${BACKEND_HTTP}/api/tasks/${taskId}`, {
|
|
1425
|
+
method: "GET",
|
|
1426
|
+
headers: {
|
|
1427
|
+
Authorization: `Bearer ${AGENT_TOKEN}`,
|
|
1428
|
+
Accept: "application/json",
|
|
1429
|
+
},
|
|
1430
|
+
signal: controller?.signal,
|
|
1431
|
+
});
|
|
1432
|
+
if (!response?.ok) return null;
|
|
1433
|
+
const task = await response.json();
|
|
1434
|
+
markBackendHttpSuccess();
|
|
1435
|
+
return String(task?.status || "").trim().toLowerCase() || null;
|
|
1436
|
+
})(),
|
|
1437
|
+
BACKEND_STATUS_PROBE_TIMEOUT_MS,
|
|
1438
|
+
`task status probe for ${taskId}`,
|
|
1439
|
+
);
|
|
1440
|
+
return status;
|
|
1441
|
+
} catch (error) {
|
|
1442
|
+
// Release the socket; withTimeout only stops us waiting, it cannot
|
|
1443
|
+
// cancel the request on its own.
|
|
1444
|
+
try {
|
|
1445
|
+
controller?.abort();
|
|
1446
|
+
} catch {
|
|
1447
|
+
// best effort
|
|
1064
1448
|
}
|
|
1449
|
+
logError(`Failed to query backend status for task ${taskId}: ${error?.message || error}`);
|
|
1450
|
+
return null;
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1453
|
+
|
|
1454
|
+
const TERMINAL_BACKEND_TASK_STATUSES = new Set(["completed", "killed"]);
|
|
1455
|
+
|
|
1456
|
+
// A tmux session vanished. Decide what — if anything — to tell the backend.
|
|
1457
|
+
//
|
|
1458
|
+
// This is the fix for "fire dies inside its tmux session and nobody
|
|
1459
|
+
// notices". The daemon's child was only the `tmux new-session` client,
|
|
1460
|
+
// which exited 0 long ago, so there is no exit event to observe. Without a
|
|
1461
|
+
// report here the task sits at `running` until the next reconcile sweep
|
|
1462
|
+
// collects it, and reconcile's `PATCH {status:"killed"}` is rewritten by
|
|
1463
|
+
// the task route into `killing` → `killed`, which the upstream commit path
|
|
1464
|
+
// reads as `killedReason = "user_stopped"`. The user never pressed stop,
|
|
1465
|
+
// and the real cause is lost.
|
|
1466
|
+
//
|
|
1467
|
+
// Two hazards make a naive "session gone → report KILLED" wrong, and both
|
|
1468
|
+
// are handled below:
|
|
1469
|
+
// 1. A session also disappears when fire finished *successfully*.
|
|
1470
|
+
// Reporting KILLED unconditionally would rewrite completed tasks as
|
|
1471
|
+
// failures. We therefore classify from the exit marker the wrapper
|
|
1472
|
+
// shell appends to the log, not from the disappearance itself.
|
|
1473
|
+
// 2. Fire usually reports its own terminal status over its own
|
|
1474
|
+
// websocket. `commitTaskStatusUpdate` has no terminal→terminal guard,
|
|
1475
|
+
// so a late KILLED from us would clobber a COMPLETED from fire. We
|
|
1476
|
+
// ask the backend first and stay silent when the task already reached
|
|
1477
|
+
// a terminal state.
|
|
1478
|
+
//
|
|
1479
|
+
// Note on `suppressedExitStatusReports`: it is deliberately NOT consulted
|
|
1480
|
+
// here. Every path that sets it (refresh_session_inplace via
|
|
1481
|
+
// `stopActiveTaskProcess`) removes the active record synchronously, so the
|
|
1482
|
+
// reaper can never see a record it applies to — but the flag itself is
|
|
1483
|
+
// never cleared in tmux mode, because the tmux client's exit handler
|
|
1484
|
+
// returns early and only the non-early path consumes it. Honouring a flag
|
|
1485
|
+
// that outlives its task would silence a later, unrelated death. The
|
|
1486
|
+
// backend status pre-check below is the guard that actually matters.
|
|
1487
|
+
async function reportDeadTmuxSessionStatus(taskId, record) {
|
|
1488
|
+
if (!record?.projectId) {
|
|
1489
|
+
logError(`Cannot report terminal status for task ${taskId}: no project id on the active record`);
|
|
1490
|
+
return;
|
|
1491
|
+
}
|
|
1492
|
+
// The exit marker only exists because the wrapper shell writes it to
|
|
1493
|
+
// `logPath`. Without one there is no evidence at all, and "no marker"
|
|
1494
|
+
// would be misread as "died hard" — reporting every clean finish as
|
|
1495
|
+
// KILLED. Staying silent restores the old blind spot for this task only,
|
|
1496
|
+
// which is strictly better than manufacturing a wrong cause of death.
|
|
1497
|
+
if (!record.logPath) {
|
|
1498
|
+
logError(
|
|
1499
|
+
`Cannot classify the end of task ${taskId}: no log path on the active record, so no exit marker exists`,
|
|
1500
|
+
);
|
|
1501
|
+
return;
|
|
1502
|
+
}
|
|
1503
|
+
// Same reasoning, one level down. `tee -a` creates the log the instant it
|
|
1504
|
+
// opens it, so a missing file means the wrapper could never write there
|
|
1505
|
+
// (unwritable or vanished cwd) — and `tee` failing does NOT stop the fire,
|
|
1506
|
+
// which runs to completion while the tmux client still exits 0. Treating
|
|
1507
|
+
// that absence as "no exit marker" would report a green task as KILLED
|
|
1508
|
+
// with an invented "killed or OOM" cause. An empty-but-present file is
|
|
1509
|
+
// different: the fire really did die before writing anything.
|
|
1510
|
+
if (readLogFileSize(record.logPath) === null) {
|
|
1511
|
+
logError(
|
|
1512
|
+
`Cannot classify the end of task ${taskId}: log file ${record.logPath} is missing, ` +
|
|
1513
|
+
`so the absence of an exit marker proves nothing`,
|
|
1514
|
+
);
|
|
1515
|
+
return;
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
const rawTail = readLogTailRaw(record.logPath, { minOffset: record.logStartOffset || 0 });
|
|
1519
|
+
// Search the file for the marker rather than hoping it is still inside
|
|
1520
|
+
// the tail window — see `findFireExitCode` for why a shared log makes the
|
|
1521
|
+
// tail unreliable. The tail is still what feeds the human-readable
|
|
1522
|
+
// summary; only the *verdict* needs the exhaustive lookup.
|
|
1523
|
+
const exitCode = findFireExitCode(record.logPath, {
|
|
1524
|
+
minOffset: record.logStartOffset || 0,
|
|
1525
|
+
markerToken: record.exitMarkerToken,
|
|
1065
1526
|
});
|
|
1527
|
+
const lifetimeMs = record.spawnedAtMs ? Date.now() - record.spawnedAtMs : null;
|
|
1528
|
+
|
|
1529
|
+
// Classification. `null` exit code means the wrapper shell never got to
|
|
1530
|
+
// write the marker — SIGKILL, OOM kill, or a tmux server crash.
|
|
1531
|
+
const isKilled = exitCode === null || exitCode !== 0;
|
|
1532
|
+
const status = isKilled ? "KILLED" : "COMPLETED";
|
|
1533
|
+
//
|
|
1534
|
+
// The success summary deliberately is NOT the bare word "completed".
|
|
1535
|
+
// The web side treats a summary that merely restates the status as
|
|
1536
|
+
// trivial and drops it — but only when it has to synthesize the event
|
|
1537
|
+
// id. We send our own id (for idempotency), which bypasses that filter,
|
|
1538
|
+
// so a bare "completed" would be persisted and would overwrite
|
|
1539
|
+
// `latest_status_summary` with a word that says nothing. Naming *how* we
|
|
1540
|
+
// found out keeps the event worth its row: it tells the next reader that
|
|
1541
|
+
// fire never published its own status.
|
|
1542
|
+
const baseSummary =
|
|
1543
|
+
exitCode === null
|
|
1544
|
+
? "fire tmux session disappeared without an exit marker (killed or OOM)"
|
|
1545
|
+
: exitCode === 0
|
|
1546
|
+
? "completed; detected by the daemon after the tmux session ended (fire never reported it)"
|
|
1547
|
+
: exitCode === 130 || exitCode === 143
|
|
1548
|
+
? `terminated (exit code ${exitCode})`
|
|
1549
|
+
: `exited with code ${exitCode}`;
|
|
1550
|
+
|
|
1551
|
+
// Ask the backend before overwriting anything: fire normally reports its
|
|
1552
|
+
// own terminal status and we must not downgrade a success.
|
|
1553
|
+
const backendStatus = await fetchBackendTaskStatus(taskId);
|
|
1554
|
+
if (backendStatus && TERMINAL_BACKEND_TASK_STATUSES.has(backendStatus)) {
|
|
1555
|
+
log(
|
|
1556
|
+
`Tmux session ${record.tmuxSession} for task ${taskId} ended; backend already ${backendStatus}, no report needed`,
|
|
1557
|
+
);
|
|
1558
|
+
return;
|
|
1559
|
+
}
|
|
1560
|
+
|
|
1561
|
+
// The await above yields the event loop, and "this task looks dead" is
|
|
1562
|
+
// precisely the trigger for an in-place restart. A `restart_task` landing
|
|
1563
|
+
// in that window finds no active record (we deleted it), spawns a fresh
|
|
1564
|
+
// fire and reports RUNNING — and our stale report would then mark the
|
|
1565
|
+
// *live* replacement as killed, using the previous run's log tail. Worse,
|
|
1566
|
+
// the daemon's own map would say running, so reconcile would never repair
|
|
1567
|
+
// it. Re-check that the task is still absent before speaking.
|
|
1568
|
+
if (activeTaskProcesses.has(taskId)) {
|
|
1569
|
+
log(
|
|
1570
|
+
`Task ${taskId} was restarted while its dead tmux session was being reported; dropping the stale terminal status`,
|
|
1571
|
+
);
|
|
1572
|
+
return;
|
|
1573
|
+
}
|
|
1574
|
+
|
|
1575
|
+
// Strip the marker before it reaches a user-visible summary: its content
|
|
1576
|
+
// is already stated by the classification above, so leaving it in just
|
|
1577
|
+
// appends "[conductor-fire-exit] code=1" to "exited with code 1".
|
|
1578
|
+
const outputTail = isKilled
|
|
1579
|
+
? sanitizeOutputTail(
|
|
1580
|
+
rawTail.replace(
|
|
1581
|
+
new RegExp(`${escapeForRegExp(FIRE_EXIT_MARKER_PREFIX)}[^\\]]*\\] code=\\d+`, "g"),
|
|
1582
|
+
"",
|
|
1583
|
+
),
|
|
1584
|
+
)
|
|
1585
|
+
: "";
|
|
1586
|
+
if (isKilled) {
|
|
1587
|
+
logError(
|
|
1588
|
+
`[tmux-reap] fire died inside its session task=${taskId} ` +
|
|
1589
|
+
`tmux=${record.tmuxSession || "none"} exit=${exitCode === null ? "unknown" : exitCode} ` +
|
|
1590
|
+
`lifetime_ms=${lifetimeMs === null ? "unknown" : lifetimeMs} log=${record.logPath || "none"} ` +
|
|
1591
|
+
`backend_status=${backendStatus || "unknown"} ` +
|
|
1592
|
+
`output_tail=${outputTail ? JSON.stringify(outputTail) : "<empty>"}`,
|
|
1593
|
+
);
|
|
1594
|
+
}
|
|
1595
|
+
const summary = outputTail ? `${baseSummary}: ${outputTail}` : baseSummary;
|
|
1596
|
+
|
|
1597
|
+
client
|
|
1598
|
+
.sendJson({
|
|
1599
|
+
type: "task_status_update",
|
|
1600
|
+
payload: {
|
|
1601
|
+
task_id: taskId,
|
|
1602
|
+
project_id: record.projectId,
|
|
1603
|
+
status,
|
|
1604
|
+
summary,
|
|
1605
|
+
// Idempotency key. Generated per call, so it only dedupes
|
|
1606
|
+
// transport-level redelivery of *this* message — not a second,
|
|
1607
|
+
// independent reaper report (the record deletion above is what
|
|
1608
|
+
// prevents those). It is also what makes the backend persist
|
|
1609
|
+
// `summary` at all: without a client-supplied id the server
|
|
1610
|
+
// synthesizes a fresh one per delivery.
|
|
1611
|
+
status_event_id: randomUUID(),
|
|
1612
|
+
},
|
|
1613
|
+
})
|
|
1614
|
+
.catch((err) => {
|
|
1615
|
+
logError(
|
|
1616
|
+
`Failed to report task status (${status}) for reaped tmux task ${taskId}: ${err?.message || err}`,
|
|
1617
|
+
);
|
|
1618
|
+
});
|
|
1066
1619
|
}
|
|
1067
1620
|
|
|
1068
|
-
// Walk every tmux-mode entry in `activeTaskProcesses
|
|
1069
|
-
// whose tmux session no longer exists
|
|
1070
|
-
// observe the inner Fire process exit
|
|
1071
|
-
// so this
|
|
1072
|
-
// across long daemon lifetimes
|
|
1621
|
+
// Walk every tmux-mode entry in `activeTaskProcesses`, remove the ones
|
|
1622
|
+
// whose tmux session no longer exists, and report a terminal status for
|
|
1623
|
+
// them. We don't (and can't reliably) observe the inner Fire process exit
|
|
1624
|
+
// when we never owned it as a child, so this sweep is both what keeps the
|
|
1625
|
+
// active map from leaking across long daemon lifetimes *and* the only
|
|
1626
|
+
// place a death inside the session can be noticed at all.
|
|
1073
1627
|
//
|
|
1074
|
-
//
|
|
1075
|
-
// `spawnFireProcess` returns synchronously
|
|
1076
|
-
//
|
|
1077
|
-
//
|
|
1078
|
-
//
|
|
1079
|
-
//
|
|
1080
|
-
//
|
|
1081
|
-
//
|
|
1082
|
-
//
|
|
1083
|
-
// reports. If this ever shows up in production, add a `createdAt`
|
|
1084
|
-
// timestamp to the record and a grace period here.
|
|
1628
|
+
// Startup race (now guarded — it used to be merely theoretical):
|
|
1629
|
+
// `spawnFireProcess` returns synchronously and we set the active record
|
|
1630
|
+
// before the spawned `tmux new-session -d` client has actually exited,
|
|
1631
|
+
// so there is a window in which an active record exists but
|
|
1632
|
+
// `tmux has-session` returns false because the session has not finished
|
|
1633
|
+
// registering with the tmux server. That window was harmless while the
|
|
1634
|
+
// reaper only cleaned up local bookkeeping; now that it reports terminal
|
|
1635
|
+
// statuses, hitting it would kill a task that started fine. Records are
|
|
1636
|
+
// therefore ignored until they are `TMUX_REAP_GRACE_MS` old.
|
|
1085
1637
|
async function reapDeadTmuxSessionsOnce() {
|
|
1086
1638
|
const candidates = [];
|
|
1639
|
+
const now = Date.now();
|
|
1087
1640
|
for (const [taskId, record] of activeTaskProcesses.entries()) {
|
|
1088
|
-
if (record?.tmuxMode
|
|
1089
|
-
|
|
1090
|
-
|
|
1641
|
+
if (!record?.tmuxMode || !record.tmuxSession) continue;
|
|
1642
|
+
const spawnedAtMs = Number(record.spawnedAtMs) || 0;
|
|
1643
|
+
if (spawnedAtMs && now - spawnedAtMs < TMUX_REAP_GRACE_MS) continue;
|
|
1644
|
+
candidates.push([taskId, record]);
|
|
1091
1645
|
}
|
|
1092
1646
|
for (const [taskId, record] of candidates) {
|
|
1093
|
-
const alive = await
|
|
1094
|
-
//
|
|
1095
|
-
//
|
|
1647
|
+
const { alive, conclusive } = await probeTmuxSession(record.tmuxSession);
|
|
1648
|
+
// An inconclusive probe (tmux missing, wedged server, timeout) is not
|
|
1649
|
+
// evidence of death. Leave the record alone and retry next sweep rather
|
|
1650
|
+
// than cleaning up — and reporting a cause of death — on a guess.
|
|
1651
|
+
if (!alive && !conclusive) {
|
|
1652
|
+
logError(
|
|
1653
|
+
`Could not determine whether tmux session ${record.tmuxSession} for task ${taskId} is alive; leaving it untouched`,
|
|
1654
|
+
);
|
|
1655
|
+
continue;
|
|
1656
|
+
}
|
|
1657
|
+
// Only act if the entry still points to the same record; a concurrent
|
|
1658
|
+
// restart_task may have replaced it while we were probing.
|
|
1096
1659
|
if (!alive && activeTaskProcesses.get(taskId) === record) {
|
|
1097
1660
|
log(
|
|
1098
1661
|
`Tmux session ${record.tmuxSession} for task ${taskId} no longer exists; cleaning up activeTaskProcesses entry`,
|
|
@@ -1102,6 +1665,9 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1102
1665
|
record.stopForceKillTimer = null;
|
|
1103
1666
|
}
|
|
1104
1667
|
activeTaskProcesses.delete(taskId);
|
|
1668
|
+
// Report *after* dropping the local entry so a slow backend round
|
|
1669
|
+
// trip can't make a concurrent stop_task wait on us.
|
|
1670
|
+
await reportDeadTmuxSessionStatus(taskId, record);
|
|
1105
1671
|
}
|
|
1106
1672
|
}
|
|
1107
1673
|
}
|
|
@@ -1277,6 +1843,14 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1277
1843
|
"# (svg/png/jpg/gif/webp/ico/avif) relative to this .conductor/ directory.",
|
|
1278
1844
|
"# Remove this line to use the default folder icon.",
|
|
1279
1845
|
"",
|
|
1846
|
+
"# Optional agent registry used by the task-composer worker/reviewer picker.",
|
|
1847
|
+
"# Agent docs are workspace-relative paths; backend is optional.",
|
|
1848
|
+
"# agents:",
|
|
1849
|
+
"# feature-dev:",
|
|
1850
|
+
"# doc: claw/agents/feature-dev.md",
|
|
1851
|
+
"# description: Implements features end to end.",
|
|
1852
|
+
"# backend: codex",
|
|
1853
|
+
"",
|
|
1280
1854
|
"worktree:",
|
|
1281
1855
|
" sync_branch: false",
|
|
1282
1856
|
" sync_submodules: true",
|
|
@@ -1290,6 +1864,13 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1290
1864
|
].join("\n");
|
|
1291
1865
|
|
|
1292
1866
|
const MAX_PROJECT_ICON_IMAGE_BYTES = 128 * 1024;
|
|
1867
|
+
const MAX_PROJECT_AGENT_ENTRIES = 64;
|
|
1868
|
+
const MAX_PROJECT_AGENT_NAME_LENGTH = 64;
|
|
1869
|
+
const MAX_PROJECT_AGENT_DOC_LENGTH = 512;
|
|
1870
|
+
const MAX_PROJECT_AGENT_DESCRIPTION_LENGTH = 512;
|
|
1871
|
+
const MAX_PROJECT_AGENT_BACKEND_LENGTH = 64;
|
|
1872
|
+
const PROJECT_AGENT_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
1873
|
+
const PROJECT_AGENT_BACKEND_RE = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
1293
1874
|
const PROJECT_ICON_MIME_BY_EXTENSION = {
|
|
1294
1875
|
".svg": "image/svg+xml",
|
|
1295
1876
|
".png": "image/png",
|
|
@@ -1388,6 +1969,118 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1388
1969
|
return null;
|
|
1389
1970
|
}
|
|
1390
1971
|
|
|
1972
|
+
function normalizeProjectAgentSetting(name, value) {
|
|
1973
|
+
const normalizedName = normalizeOptionalString(name);
|
|
1974
|
+
if (
|
|
1975
|
+
!normalizedName ||
|
|
1976
|
+
normalizedName.length > MAX_PROJECT_AGENT_NAME_LENGTH ||
|
|
1977
|
+
!PROJECT_AGENT_NAME_RE.test(normalizedName)
|
|
1978
|
+
) {
|
|
1979
|
+
return null;
|
|
1980
|
+
}
|
|
1981
|
+
|
|
1982
|
+
let docValue = null;
|
|
1983
|
+
let descriptionValue = null;
|
|
1984
|
+
let backendValue = null;
|
|
1985
|
+
if (typeof value === "string") {
|
|
1986
|
+
docValue = value;
|
|
1987
|
+
} else if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
1988
|
+
docValue = value.doc ?? value.path;
|
|
1989
|
+
descriptionValue = value.description;
|
|
1990
|
+
backendValue = value.backend;
|
|
1991
|
+
} else {
|
|
1992
|
+
return null;
|
|
1993
|
+
}
|
|
1994
|
+
|
|
1995
|
+
const doc = normalizeOptionalString(docValue);
|
|
1996
|
+
if (
|
|
1997
|
+
!doc ||
|
|
1998
|
+
doc.length > MAX_PROJECT_AGENT_DOC_LENGTH ||
|
|
1999
|
+
path.isAbsolute(doc) ||
|
|
2000
|
+
/^[A-Za-z]:[\\/]/.test(doc) ||
|
|
2001
|
+
/^[/\\]{2}/.test(doc)
|
|
2002
|
+
) {
|
|
2003
|
+
return null;
|
|
2004
|
+
}
|
|
2005
|
+
const normalizedDoc = path.posix.normalize(doc.split("\\").join("/"));
|
|
2006
|
+
if (normalizedDoc === ".." || normalizedDoc.startsWith("../")) {
|
|
2007
|
+
return null;
|
|
2008
|
+
}
|
|
2009
|
+
|
|
2010
|
+
const description = normalizeOptionalString(descriptionValue);
|
|
2011
|
+
const backend = normalizeOptionalString(backendValue)?.toLowerCase() || null;
|
|
2012
|
+
return {
|
|
2013
|
+
name: normalizedName,
|
|
2014
|
+
doc,
|
|
2015
|
+
description:
|
|
2016
|
+
description && description.length <= MAX_PROJECT_AGENT_DESCRIPTION_LENGTH
|
|
2017
|
+
? description
|
|
2018
|
+
: null,
|
|
2019
|
+
backend:
|
|
2020
|
+
backend &&
|
|
2021
|
+
backend.length <= MAX_PROJECT_AGENT_BACKEND_LENGTH &&
|
|
2022
|
+
PROJECT_AGENT_BACKEND_RE.test(backend)
|
|
2023
|
+
? backend
|
|
2024
|
+
: null,
|
|
2025
|
+
};
|
|
2026
|
+
}
|
|
2027
|
+
|
|
2028
|
+
function normalizeProjectAgentsSetting(agentsNode) {
|
|
2029
|
+
if (!agentsNode || typeof agentsNode !== "object") {
|
|
2030
|
+
return [];
|
|
2031
|
+
}
|
|
2032
|
+
const result = [];
|
|
2033
|
+
const seen = new Set();
|
|
2034
|
+
const push = (entry) => {
|
|
2035
|
+
if (
|
|
2036
|
+
entry &&
|
|
2037
|
+
result.length < MAX_PROJECT_AGENT_ENTRIES &&
|
|
2038
|
+
!seen.has(entry.name)
|
|
2039
|
+
) {
|
|
2040
|
+
seen.add(entry.name);
|
|
2041
|
+
result.push(entry);
|
|
2042
|
+
}
|
|
2043
|
+
};
|
|
2044
|
+
|
|
2045
|
+
if (Array.isArray(agentsNode)) {
|
|
2046
|
+
for (const item of agentsNode) {
|
|
2047
|
+
if (!item || typeof item !== "object" || Array.isArray(item)) {
|
|
2048
|
+
continue;
|
|
2049
|
+
}
|
|
2050
|
+
const keys = Object.keys(item);
|
|
2051
|
+
if (keys.length === 1) {
|
|
2052
|
+
push(normalizeProjectAgentSetting(keys[0], item[keys[0]]));
|
|
2053
|
+
} else if (Object.prototype.hasOwnProperty.call(item, "name")) {
|
|
2054
|
+
push(normalizeProjectAgentSetting(item.name, item));
|
|
2055
|
+
}
|
|
2056
|
+
}
|
|
2057
|
+
return result;
|
|
2058
|
+
}
|
|
2059
|
+
|
|
2060
|
+
for (const [name, value] of Object.entries(agentsNode)) {
|
|
2061
|
+
push(normalizeProjectAgentSetting(name, value));
|
|
2062
|
+
}
|
|
2063
|
+
return result;
|
|
2064
|
+
}
|
|
2065
|
+
|
|
2066
|
+
function readProjectAgentsSetting(projectWorkspacePath) {
|
|
2067
|
+
for (const settingsPath of getProjectSettingsCandidates(projectWorkspacePath)) {
|
|
2068
|
+
if (!existsSyncFn(settingsPath)) {
|
|
2069
|
+
continue;
|
|
2070
|
+
}
|
|
2071
|
+
try {
|
|
2072
|
+
const parsed = yaml.load(readFileSyncFn(settingsPath, "utf8"));
|
|
2073
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
2074
|
+
return [];
|
|
2075
|
+
}
|
|
2076
|
+
return normalizeProjectAgentsSetting(parsed.agents);
|
|
2077
|
+
} catch {
|
|
2078
|
+
return [];
|
|
2079
|
+
}
|
|
2080
|
+
}
|
|
2081
|
+
return [];
|
|
2082
|
+
}
|
|
2083
|
+
|
|
1391
2084
|
function readProjectWorktreeSettings(projectWorkspacePath) {
|
|
1392
2085
|
for (const settingsPath of getProjectSettingsCandidates(projectWorkspacePath)) {
|
|
1393
2086
|
if (!existsSyncFn(settingsPath)) {
|
|
@@ -1442,6 +2135,10 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1442
2135
|
}
|
|
1443
2136
|
}
|
|
1444
2137
|
|
|
2138
|
+
// Monotonic suffix so two symlink swaps inside one daemon process can never
|
|
2139
|
+
// collide on the same temp path.
|
|
2140
|
+
let symlinkSequence = 0;
|
|
2141
|
+
|
|
1445
2142
|
async function ensureTaskWorktreeSymlinks({ projectRepoRoot, projectWorkspacePath, finalCwd }) {
|
|
1446
2143
|
const { symlinkPaths } = readProjectWorktreeSettings(projectWorkspacePath);
|
|
1447
2144
|
for (const configuredPath of symlinkPaths) {
|
|
@@ -1455,6 +2152,27 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1455
2152
|
if (await isGitTrackedWorktreePath({ projectRepoRoot, sourcePath })) {
|
|
1456
2153
|
continue;
|
|
1457
2154
|
}
|
|
2155
|
+
|
|
2156
|
+
// Never link to a source that isn't there. `symlinkSync` does NOT
|
|
2157
|
+
// require its target to exist (POSIX), so a stale `worktree.symlink`
|
|
2158
|
+
// entry — e.g. `xr/android/build/local.properties`, generated locally by
|
|
2159
|
+
// the IDE and never present on this machine — would otherwise make the
|
|
2160
|
+
// daemon *manufacture* a dangling link on the very first worktree prep.
|
|
2161
|
+
// Those links are pure liability: they break tooling that follows them,
|
|
2162
|
+
// and they are what the EEXIST-on-reprepare bug below fed on.
|
|
2163
|
+
//
|
|
2164
|
+
// Note this is the one place where `existsSync` is the RIGHT probe:
|
|
2165
|
+
// here we genuinely care whether the target resolves to something real
|
|
2166
|
+
// (a source that is itself a dangling link is equally useless). The
|
|
2167
|
+
// destination probe further down must use `lstat` instead, for exactly
|
|
2168
|
+
// the opposite reason — see the note there.
|
|
2169
|
+
if (!existsSyncFn(sourcePath)) {
|
|
2170
|
+
logError(
|
|
2171
|
+
`[worktree] skipping symlink for missing source: ${configuredPath} (expected at ${sourcePath})`,
|
|
2172
|
+
);
|
|
2173
|
+
continue;
|
|
2174
|
+
}
|
|
2175
|
+
|
|
1458
2176
|
const linkPath = resolveProjectScopedPath(
|
|
1459
2177
|
finalCwd,
|
|
1460
2178
|
configuredPath,
|
|
@@ -1462,21 +2180,70 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1462
2180
|
);
|
|
1463
2181
|
mkdirSyncFn(path.dirname(linkPath), { recursive: true });
|
|
1464
2182
|
|
|
1465
|
-
|
|
2183
|
+
// NOTE: probe with lstat, never existsSync. existsSync FOLLOWS symlinks,
|
|
2184
|
+
// so a dangling link (source deleted after the worktree was created —
|
|
2185
|
+
// .venv / node_modules / local.properties are all gitignored churn)
|
|
2186
|
+
// reads as "missing", and the symlinkSyncFn below then throws EEXIST.
|
|
2187
|
+
// That made the task permanently un-restartable.
|
|
2188
|
+
let linkStat = null;
|
|
2189
|
+
try {
|
|
2190
|
+
linkStat = lstatSyncFn(linkPath);
|
|
2191
|
+
} catch (error) {
|
|
2192
|
+
if (error?.code !== "ENOENT") {
|
|
2193
|
+
throw error instanceof Error ? error : new Error(String(error));
|
|
2194
|
+
}
|
|
2195
|
+
}
|
|
2196
|
+
|
|
2197
|
+
if (linkStat) {
|
|
2198
|
+
if (!linkStat.isSymbolicLink()) {
|
|
2199
|
+
throw new Error(
|
|
2200
|
+
`worktree symlink destination already exists and is not a symlink: ${linkPath}. ` +
|
|
2201
|
+
`Refusing to replace it because it may hold real data — remove it manually, ` +
|
|
2202
|
+
`or drop "${configuredPath}" from worktree.symlink in .conductor/settings.yaml.`,
|
|
2203
|
+
);
|
|
2204
|
+
}
|
|
2205
|
+
// Compare the link's TARGET, not whether that target resolves. A link
|
|
2206
|
+
// that already points at the right place is correct even when the
|
|
2207
|
+
// source is currently absent.
|
|
2208
|
+
const currentTarget = readlinkSyncFn(linkPath);
|
|
2209
|
+
const currentResolvedTarget = path.resolve(path.dirname(linkPath), currentTarget);
|
|
2210
|
+
if (currentResolvedTarget === sourcePath) {
|
|
2211
|
+
continue;
|
|
2212
|
+
}
|
|
2213
|
+
// Self-heal instead of aborting. A stale link happens whenever the
|
|
2214
|
+
// project moves on disk or its workspace_path binding is edited: every
|
|
2215
|
+
// pre-existing worktree then holds links to the OLD absolute path, and
|
|
2216
|
+
// throwing here made every task in them permanently un-restartable.
|
|
2217
|
+
//
|
|
2218
|
+
// Replacing is safe precisely because this entry is a SYMLINK: it
|
|
2219
|
+
// carries no data of its own, so unlinking destroys nothing. (The
|
|
2220
|
+
// non-symlink case above still throws — a real file or directory here
|
|
2221
|
+
// may hold user data and must never be clobbered.) The source is also
|
|
2222
|
+
// known to exist at this point, thanks to the guard further up, so we
|
|
2223
|
+
// are converging on a link that actually resolves.
|
|
2224
|
+
log(
|
|
2225
|
+
`[worktree] repointing stale symlink ${linkPath}: ${currentResolvedTarget} -> ${sourcePath}`,
|
|
2226
|
+
);
|
|
2227
|
+
// Atomic replace. branch/fork tasks deliberately SHARE one worktree
|
|
2228
|
+
// (identity is keyed on worktreeBranch), so two preparations can run
|
|
2229
|
+
// against this directory concurrently. A plain unlink+symlink leaves a
|
|
2230
|
+
// window where the peer's symlinkSync hits EEXIST — reintroducing the
|
|
2231
|
+
// very failure this function was fixed for. symlink-to-temp + rename
|
|
2232
|
+
// has no such window: rename(2) atomically replaces the entry.
|
|
2233
|
+
const tempLinkPath = `${linkPath}.conductor-tmp-${process.pid}-${symlinkSequence++}`;
|
|
2234
|
+
const relativeTargetForSwap = path.relative(path.dirname(linkPath), sourcePath) || ".";
|
|
2235
|
+
symlinkSyncFn(relativeTargetForSwap, tempLinkPath);
|
|
1466
2236
|
try {
|
|
1467
|
-
|
|
1468
|
-
if (!stat.isSymbolicLink()) {
|
|
1469
|
-
throw new Error(`worktree symlink destination already exists: ${linkPath}`);
|
|
1470
|
-
}
|
|
1471
|
-
const currentTarget = readlinkSyncFn(linkPath);
|
|
1472
|
-
const currentResolvedTarget = path.resolve(path.dirname(linkPath), currentTarget);
|
|
1473
|
-
if (currentResolvedTarget === sourcePath) {
|
|
1474
|
-
continue;
|
|
1475
|
-
}
|
|
1476
|
-
throw new Error(`worktree symlink destination already points elsewhere: ${linkPath}`);
|
|
2237
|
+
renameSyncFn(tempLinkPath, linkPath);
|
|
1477
2238
|
} catch (error) {
|
|
2239
|
+
try {
|
|
2240
|
+
unlinkSyncFn(tempLinkPath);
|
|
2241
|
+
} catch {
|
|
2242
|
+
// best effort: never mask the original failure
|
|
2243
|
+
}
|
|
1478
2244
|
throw error instanceof Error ? error : new Error(String(error));
|
|
1479
2245
|
}
|
|
2246
|
+
continue;
|
|
1480
2247
|
}
|
|
1481
2248
|
|
|
1482
2249
|
const relativeTarget = path.relative(path.dirname(linkPath), sourcePath) || ".";
|
|
@@ -2108,6 +2875,20 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
2108
2875
|
return 30 * 1000;
|
|
2109
2876
|
})();
|
|
2110
2877
|
|
|
2878
|
+
// How long a freshly spawned tmux-mode task is exempt from the reaper.
|
|
2879
|
+
// Covers the window between "we recorded the active task" and "the tmux
|
|
2880
|
+
// server finished registering the session", during which `has-session`
|
|
2881
|
+
// can legitimately answer false for a task that started fine. Since the
|
|
2882
|
+
// reaper now reports terminal statuses, a false positive here would kill a
|
|
2883
|
+
// healthy task, so the exemption is no longer optional. Tests set 0.
|
|
2884
|
+
const TMUX_REAP_GRACE_MS = (() => {
|
|
2885
|
+
const explicit = Number(config.TMUX_REAP_GRACE_MS);
|
|
2886
|
+
if (Number.isFinite(explicit) && explicit >= 0) {
|
|
2887
|
+
return explicit;
|
|
2888
|
+
}
|
|
2889
|
+
return 15 * 1000;
|
|
2890
|
+
})();
|
|
2891
|
+
|
|
2111
2892
|
// --- Auto-update state ---
|
|
2112
2893
|
const VERSION_CHECK_INTERVAL_MS = 6 * 60 * 60 * 1000; // 6 hours
|
|
2113
2894
|
let lastVersionCheckAt = 0;
|
|
@@ -2150,6 +2931,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
2150
2931
|
};
|
|
2151
2932
|
const advertisedCapabilities = [
|
|
2152
2933
|
"project_path_validation",
|
|
2934
|
+
"project_agents_registry",
|
|
2153
2935
|
"restart_daemon",
|
|
2154
2936
|
"refresh_session_inplace",
|
|
2155
2937
|
CUSTOM_COMMANDS_CAPABILITY,
|
|
@@ -4373,6 +5155,11 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4373
5155
|
}
|
|
4374
5156
|
if (event.type === "validate_project_path") {
|
|
4375
5157
|
void handleValidateProjectPath(event.payload);
|
|
5158
|
+
return;
|
|
5159
|
+
}
|
|
5160
|
+
if (event.type === "get_project_agents") {
|
|
5161
|
+
void handleGetProjectAgents(event.payload);
|
|
5162
|
+
return;
|
|
4376
5163
|
}
|
|
4377
5164
|
if (event.type === "ai_manager_request") {
|
|
4378
5165
|
handleAiManagerRequest(client, aiManagerHandlers, event.payload).catch((error) => {
|
|
@@ -4478,6 +5265,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4478
5265
|
lastCommitAt: null,
|
|
4479
5266
|
fileCount: null,
|
|
4480
5267
|
icon: null,
|
|
5268
|
+
agents: [],
|
|
4481
5269
|
error: null,
|
|
4482
5270
|
errorCode: null,
|
|
4483
5271
|
validatedAt,
|
|
@@ -4532,6 +5320,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4532
5320
|
? snapshot.fileCount
|
|
4533
5321
|
: null,
|
|
4534
5322
|
icon: readProjectIconSetting(effectiveWorkspace),
|
|
5323
|
+
agents: readProjectAgentsSetting(effectiveWorkspace),
|
|
4535
5324
|
};
|
|
4536
5325
|
}
|
|
4537
5326
|
} catch (error) {
|
|
@@ -4556,6 +5345,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4556
5345
|
git_remote_url: result.gitRemoteUrl,
|
|
4557
5346
|
file_count: result.fileCount,
|
|
4558
5347
|
icon: result.icon,
|
|
5348
|
+
agents: result.agents,
|
|
4559
5349
|
error: result.error,
|
|
4560
5350
|
error_code: result.errorCode,
|
|
4561
5351
|
validated_at: result.validatedAt,
|
|
@@ -4566,6 +5356,69 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4566
5356
|
}
|
|
4567
5357
|
}
|
|
4568
5358
|
|
|
5359
|
+
async function handleGetProjectAgents(payload) {
|
|
5360
|
+
const requestId = payload?.request_id ? String(payload.request_id).trim() : "";
|
|
5361
|
+
const rawWorkspacePath = payload?.workspace_path ? String(payload.workspace_path).trim() : "";
|
|
5362
|
+
const resolvedAt = new Date().toISOString();
|
|
5363
|
+
|
|
5364
|
+
if (!requestId || !rawWorkspacePath) {
|
|
5365
|
+
logError(`Invalid get_project_agents payload: ${JSON.stringify(payload)}`);
|
|
5366
|
+
return;
|
|
5367
|
+
}
|
|
5368
|
+
|
|
5369
|
+
let result = {
|
|
5370
|
+
workspacePath: null,
|
|
5371
|
+
agents: [],
|
|
5372
|
+
error: null,
|
|
5373
|
+
errorCode: null,
|
|
5374
|
+
};
|
|
5375
|
+
try {
|
|
5376
|
+
const resolvedPath = path.resolve(rawWorkspacePath);
|
|
5377
|
+
if (!existsSyncFn(resolvedPath)) {
|
|
5378
|
+
result = {
|
|
5379
|
+
...result,
|
|
5380
|
+
error: `Workspace path does not exist on daemon ${AGENT_NAME}: ${rawWorkspacePath}`,
|
|
5381
|
+
errorCode: "workspace_not_found",
|
|
5382
|
+
};
|
|
5383
|
+
} else if (!statSyncFn(resolvedPath).isDirectory()) {
|
|
5384
|
+
result = {
|
|
5385
|
+
...result,
|
|
5386
|
+
error: `Workspace path is not a directory on daemon ${AGENT_NAME}: ${rawWorkspacePath}`,
|
|
5387
|
+
errorCode: "workspace_not_directory",
|
|
5388
|
+
};
|
|
5389
|
+
} else {
|
|
5390
|
+
result = {
|
|
5391
|
+
...result,
|
|
5392
|
+
workspacePath: resolvedPath,
|
|
5393
|
+
agents: readProjectAgentsSetting(resolvedPath),
|
|
5394
|
+
};
|
|
5395
|
+
}
|
|
5396
|
+
} catch (error) {
|
|
5397
|
+
result = {
|
|
5398
|
+
...result,
|
|
5399
|
+
error: `Failed to read project agents on daemon ${AGENT_NAME}: ${error?.message || error}`,
|
|
5400
|
+
errorCode: "project_agents_read_failed",
|
|
5401
|
+
};
|
|
5402
|
+
}
|
|
5403
|
+
|
|
5404
|
+
try {
|
|
5405
|
+
await client.sendJson({
|
|
5406
|
+
type: "project_agents_resolved",
|
|
5407
|
+
payload: {
|
|
5408
|
+
request_id: requestId,
|
|
5409
|
+
daemon_host: AGENT_NAME,
|
|
5410
|
+
workspace_path: result.workspacePath,
|
|
5411
|
+
agents: result.agents,
|
|
5412
|
+
error: result.error,
|
|
5413
|
+
error_code: result.errorCode,
|
|
5414
|
+
resolved_at: resolvedAt,
|
|
5415
|
+
},
|
|
5416
|
+
});
|
|
5417
|
+
} catch (error) {
|
|
5418
|
+
logError(`Failed to report project_agents_resolved for ${rawWorkspacePath}: ${error?.message || error}`);
|
|
5419
|
+
}
|
|
5420
|
+
}
|
|
5421
|
+
|
|
4569
5422
|
function stopActiveTaskProcess(
|
|
4570
5423
|
taskId,
|
|
4571
5424
|
{
|
|
@@ -4786,8 +5639,18 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4786
5639
|
|
|
4787
5640
|
let entry = activeTaskProcesses.get(taskId);
|
|
4788
5641
|
if (entry?.tmuxMode && entry.tmuxSession) {
|
|
4789
|
-
|
|
4790
|
-
|
|
5642
|
+
// Only a conclusive "no such session" counts as death. A wedged or
|
|
5643
|
+
// missing tmux answers `false` too, and acting on that would ack the
|
|
5644
|
+
// reclaim as `stale` for a fire that is still running — the backend
|
|
5645
|
+
// then spawns a replacement and two fires share one worktree. It would
|
|
5646
|
+
// also drop the record, so the reaper could never classify the real
|
|
5647
|
+
// death later. When we cannot tell, keep the record and say "alive".
|
|
5648
|
+
const { alive, conclusive } = await probeTmuxSession(entry.tmuxSession);
|
|
5649
|
+
if (!alive && !conclusive) {
|
|
5650
|
+
logError(
|
|
5651
|
+
`Could not determine whether tmux session ${entry.tmuxSession} for task ${taskId} is alive; treating reclaim as still-alive`,
|
|
5652
|
+
);
|
|
5653
|
+
} else if (!alive) {
|
|
4791
5654
|
if (activeTaskProcesses.get(taskId) === entry) {
|
|
4792
5655
|
if (entry.stopForceKillTimer) {
|
|
4793
5656
|
clearTimeout(entry.stopForceKillTimer);
|
|
@@ -4854,6 +5717,40 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4854
5717
|
if ((!processRecord || !processRecord.child) && !ptyRecord) {
|
|
4855
5718
|
log(`Stop requested for task ${taskId}, but no active process found`);
|
|
4856
5719
|
sendStopAck(false);
|
|
5720
|
+
// "Nothing to stop" IS the terminal answer, and the server cannot infer
|
|
5721
|
+
// it: after the app flips the row to `killing` it waits for us to report
|
|
5722
|
+
// a terminal status, and a bare `stop_ack(accepted=false)` is only
|
|
5723
|
+
// command bookkeeping — it never converges the task. Staying silent here
|
|
5724
|
+
// is what strands a task in `killing` forever once its fire has already
|
|
5725
|
+
// died (e.g. the tmux session was reaped before the user pressed Stop).
|
|
5726
|
+
// Mirror the tmux stop path and publish KILLED so the row reaches a
|
|
5727
|
+
// terminal state that the user can then restart from.
|
|
5728
|
+
const terminalProjectId =
|
|
5729
|
+
normalizeOptionalString(payload?.project_id) ||
|
|
5730
|
+
normalizeOptionalString(processRecord?.projectId);
|
|
5731
|
+
if (terminalProjectId) {
|
|
5732
|
+
client
|
|
5733
|
+
.sendJson({
|
|
5734
|
+
type: "task_status_update",
|
|
5735
|
+
payload: {
|
|
5736
|
+
task_id: taskId,
|
|
5737
|
+
project_id: terminalProjectId,
|
|
5738
|
+
status: "KILLED",
|
|
5739
|
+
summary: payload?.reason
|
|
5740
|
+
? `stopped (${payload.reason}); no active process`
|
|
5741
|
+
: "stopped; no active process",
|
|
5742
|
+
},
|
|
5743
|
+
})
|
|
5744
|
+
.catch((err) => {
|
|
5745
|
+
logError(
|
|
5746
|
+
`Failed to report task_status_update(KILLED) for inactive task ${taskId}: ${err?.message || err}`,
|
|
5747
|
+
);
|
|
5748
|
+
});
|
|
5749
|
+
} else {
|
|
5750
|
+
logError(
|
|
5751
|
+
`Cannot report terminal status for inactive task ${taskId}: no project_id in stop_task payload`,
|
|
5752
|
+
);
|
|
5753
|
+
}
|
|
4857
5754
|
// Even when we have no in-memory record, the task may still own a
|
|
4858
5755
|
// tmux session (e.g. the daemon was restarted between spawn and
|
|
4859
5756
|
// stop, or the liveness reaper removed our entry but the session
|
|
@@ -4991,6 +5888,19 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4991
5888
|
: "restart failed";
|
|
4992
5889
|
const scrubbedError = maskErrorForLogs(error);
|
|
4993
5890
|
const summary = `${prefix}: ${scrubbedError?.message || scrubbedError}`;
|
|
5891
|
+
// Always leave a daemon-log trace. Several early rejects (unsupported
|
|
5892
|
+
// backend, cwd resolution, worktree prep) return before the "Restarting
|
|
5893
|
+
// task …" line is ever logged, so without this a failed branch/fork left
|
|
5894
|
+
// the daemon log completely silent and only a generic `fire_exit` in the
|
|
5895
|
+
// backend DB.
|
|
5896
|
+
// Benign, self-explanatory outcomes (orderly shutdown, a refresh that is
|
|
5897
|
+
// already in flight) are logged at normal level so they don't show up as
|
|
5898
|
+
// errors in monitoring; genuine failures still go to stderr.
|
|
5899
|
+
const failureText = `${scrubbedError?.message || scrubbedError}`;
|
|
5900
|
+
const isBenign = /daemon shut(ting)? down|already in progress/i.test(failureText);
|
|
5901
|
+
(isBenign ? log : logError)(
|
|
5902
|
+
`[restart-spawn] failure task=${taskId} mode=${mode || "unknown"}: ${failureText}`,
|
|
5903
|
+
);
|
|
4994
5904
|
if (mode === "refresh_session_inplace") {
|
|
4995
5905
|
rememberCommandRequestAckResult(requestId, false);
|
|
4996
5906
|
}
|
|
@@ -5013,6 +5923,13 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5013
5923
|
project_id: projectId,
|
|
5014
5924
|
status: "KILLED",
|
|
5015
5925
|
summary,
|
|
5926
|
+
// Idempotency key for this transition. The backend persists
|
|
5927
|
+
// `summary` only as a `taskStatusEvent` row, and it keys duplicate
|
|
5928
|
+
// suppression off this id — without one it has to synthesize an id,
|
|
5929
|
+
// which makes a redelivered report indistinguishable from a new
|
|
5930
|
+
// event. Supplying it keeps failure reports both persisted AND
|
|
5931
|
+
// deduplicated.
|
|
5932
|
+
status_event_id: randomUUID(),
|
|
5016
5933
|
},
|
|
5017
5934
|
})
|
|
5018
5935
|
.catch((err) => {
|
|
@@ -5023,8 +5940,14 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5023
5940
|
function reportCreateTaskFailure({ taskId, projectId, requestId, error, sendAck = true }) {
|
|
5024
5941
|
const normalizedTaskId = taskId ? String(taskId) : "";
|
|
5025
5942
|
const normalizedProjectId = projectId ? String(projectId) : "";
|
|
5026
|
-
|
|
5027
|
-
|
|
5943
|
+
// Mirror reportRestartFailure: scrub before the message reaches the log
|
|
5944
|
+
// AND the persisted status summary. An un-scrubbed create failure could
|
|
5945
|
+
// carry the handoff URL or an inherited provider key.
|
|
5946
|
+
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
5947
|
+
const message = redactSecretsForLogs(rawMessage, [AGENT_TOKEN]);
|
|
5948
|
+
logError(
|
|
5949
|
+
`[create-spawn] failure task=${normalizedTaskId || "unknown"}: ${message}`,
|
|
5950
|
+
);
|
|
5028
5951
|
if (sendAck) {
|
|
5029
5952
|
sendAgentCommandAck({
|
|
5030
5953
|
requestId,
|
|
@@ -5046,6 +5969,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5046
5969
|
project_id: normalizedProjectId,
|
|
5047
5970
|
status: "KILLED",
|
|
5048
5971
|
summary: message,
|
|
5972
|
+
status_event_id: randomUUID(),
|
|
5049
5973
|
},
|
|
5050
5974
|
})
|
|
5051
5975
|
.catch((err) => {
|
|
@@ -5304,7 +6228,10 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5304
6228
|
env.CONDUCTOR_BACKEND_URL = BACKEND_HTTP;
|
|
5305
6229
|
}
|
|
5306
6230
|
|
|
5307
|
-
|
|
6231
|
+
// Sampled before the spawn so it is a true "everything after this is
|
|
6232
|
+
// ours" watermark.
|
|
6233
|
+
const logStartOffset = readLogSize(logPath);
|
|
6234
|
+
const { child, tmuxSession, exitMarkerToken } = spawnFireProcess({
|
|
5308
6235
|
taskId,
|
|
5309
6236
|
args,
|
|
5310
6237
|
env,
|
|
@@ -5362,10 +6289,25 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5362
6289
|
log(`New task workspace: ${taskDir}`);
|
|
5363
6290
|
log(`Logs: ${logPath}`);
|
|
5364
6291
|
|
|
6292
|
+
// Same diagnostics contract as the restart/fork path: without this a
|
|
6293
|
+
// create_task whose backend dies at startup is a black box too.
|
|
6294
|
+
const outputCapture = createChildOutputCapture({ logPath, logStartOffset });
|
|
6295
|
+
const spawnedAtMs = Date.now();
|
|
6296
|
+
|
|
5365
6297
|
activeTaskProcesses.set(taskId, {
|
|
5366
6298
|
child,
|
|
5367
6299
|
projectId,
|
|
5368
6300
|
logPath,
|
|
6301
|
+
// Floor for later tail reads and the reaper's exit-marker scan: the
|
|
6302
|
+
// log file is opened with flags "a" and survives in-place restarts,
|
|
6303
|
+
// so bytes written before this run must not be attributed to it.
|
|
6304
|
+
logStartOffset,
|
|
6305
|
+
// Nonce this run's exit marker is tagged with, so the reaper only
|
|
6306
|
+
// trusts a marker written by *this* wrapper shell.
|
|
6307
|
+
exitMarkerToken,
|
|
6308
|
+
// Consumed by the tmux liveness reaper as both a grace-period anchor
|
|
6309
|
+
// and a lifetime figure for diagnostics.
|
|
6310
|
+
spawnedAtMs,
|
|
5369
6311
|
stopForceKillTimer: null,
|
|
5370
6312
|
managedByFireBridge: true,
|
|
5371
6313
|
tmuxSession: tmuxSession || null,
|
|
@@ -5399,13 +6341,20 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5399
6341
|
} else if (child.stderr && typeof child.stderr.on === "function" && logStream) {
|
|
5400
6342
|
child.stderr.on("data", (chunk) => logStream.write(chunk));
|
|
5401
6343
|
}
|
|
6344
|
+
outputCapture.attach(child.stdout);
|
|
6345
|
+
outputCapture.attach(child.stderr);
|
|
5402
6346
|
} else if (child.stderr && typeof child.stderr.on === "function") {
|
|
6347
|
+
outputCapture.attach(child.stderr);
|
|
5403
6348
|
// Capture any error output emitted by the tmux client itself so
|
|
5404
6349
|
// problems during session creation surface in daemon logs.
|
|
5405
6350
|
child.stderr.on("data", (chunk) => {
|
|
5406
6351
|
const text = chunk?.toString?.("utf8") ?? String(chunk ?? "");
|
|
5407
6352
|
if (text.trim()) {
|
|
5408
|
-
logError(
|
|
6353
|
+
logError(
|
|
6354
|
+
// Redact: this is the fire's raw stderr, which can carry the
|
|
6355
|
+
// handoff share token (argv echo) and provider/agent keys.
|
|
6356
|
+
`tmux(${tmuxSession}) stderr: ${redactSecretsForLogs(text.trim(), [AGENT_TOKEN])}`,
|
|
6357
|
+
);
|
|
5409
6358
|
}
|
|
5410
6359
|
});
|
|
5411
6360
|
}
|
|
@@ -5421,6 +6370,20 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5421
6370
|
child.on("exit", (code, signal) => {
|
|
5422
6371
|
const active = activeTaskProcesses.get(taskId);
|
|
5423
6372
|
|
|
6373
|
+
// This spawn was a tmux client but the record is gone: the reaper
|
|
6374
|
+
// (or a stop) already retired this task and, in the reaper's case,
|
|
6375
|
+
// already published its terminal status. Falling through would
|
|
6376
|
+
// delete whatever record a restart has since installed and report a
|
|
6377
|
+
// second, contradictory status — `shouldDaemonReportFireChildTerminal
|
|
6378
|
+
// Status(undefined)` is true, so an `exit(0)` arriving late would
|
|
6379
|
+
// announce COMPLETED over the reaper's verdict. Nothing left to do.
|
|
6380
|
+
if (tmuxSession && !active) {
|
|
6381
|
+
log(
|
|
6382
|
+
`tmux client for task ${taskId} exited after its record was retired (code=${code}, signal=${signal || "null"}); nothing to report`,
|
|
6383
|
+
);
|
|
6384
|
+
return;
|
|
6385
|
+
}
|
|
6386
|
+
|
|
5424
6387
|
// In tmux mode the `tmux new-session -d` client always exits
|
|
5425
6388
|
// shortly after launching the Fire session. A clean exit (code 0,
|
|
5426
6389
|
// no signal) just means the session was successfully created and
|
|
@@ -5484,6 +6447,18 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5484
6447
|
? "completed"
|
|
5485
6448
|
: `exited with code ${code}`;
|
|
5486
6449
|
|
|
6450
|
+
const lifetimeMs = Date.now() - spawnedAtMs;
|
|
6451
|
+
const outputTail = status === "KILLED" ? outputCapture.tail() : "";
|
|
6452
|
+
if (status === "KILLED") {
|
|
6453
|
+
logError(
|
|
6454
|
+
`[create-spawn] abnormal exit task=${taskId} backend=${selectedBackend} ` +
|
|
6455
|
+
`cwd=${taskDir} tmux=${tmuxSession || "none"} exit=${code ?? "null"} ` +
|
|
6456
|
+
`signal=${signal || "null"} lifetime_ms=${lifetimeMs} log=${logPath} ` +
|
|
6457
|
+
`output_tail=${outputTail ? JSON.stringify(outputTail) : "<empty>"}`,
|
|
6458
|
+
);
|
|
6459
|
+
}
|
|
6460
|
+
const reportedSummary = outputTail ? `${summary}: ${outputTail}` : summary;
|
|
6461
|
+
|
|
5487
6462
|
if (!suppressExitStatusReport && shouldDaemonReportFireChildTerminalStatus(active)) {
|
|
5488
6463
|
client
|
|
5489
6464
|
.sendJson({
|
|
@@ -5492,7 +6467,8 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5492
6467
|
task_id: taskId,
|
|
5493
6468
|
project_id: projectId,
|
|
5494
6469
|
status,
|
|
5495
|
-
summary,
|
|
6470
|
+
summary: reportedSummary,
|
|
6471
|
+
status_event_id: randomUUID(),
|
|
5496
6472
|
},
|
|
5497
6473
|
})
|
|
5498
6474
|
.catch((err) => {
|
|
@@ -5610,8 +6586,17 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5610
6586
|
// Probe the actual session and clean up stale entries on demand so
|
|
5611
6587
|
// the restart gating below reflects reality, not the stale record.
|
|
5612
6588
|
if (activeTarget?.tmuxMode && activeTarget.tmuxSession) {
|
|
5613
|
-
|
|
5614
|
-
|
|
6589
|
+
// As in reclaim: only a conclusive answer may clear the record. This
|
|
6590
|
+
// gate is what stops a double spawn, so trusting an inconclusive probe
|
|
6591
|
+
// would start a second fire alongside a live one in the same worktree.
|
|
6592
|
+
const { alive: sessionAlive, conclusive } = await probeTmuxSession(
|
|
6593
|
+
activeTarget.tmuxSession,
|
|
6594
|
+
);
|
|
6595
|
+
if (!sessionAlive && !conclusive) {
|
|
6596
|
+
logError(
|
|
6597
|
+
`Could not determine whether tmux session ${activeTarget.tmuxSession} for task ${normalizedTargetTaskId} is alive; keeping the existing record before restart`,
|
|
6598
|
+
);
|
|
6599
|
+
} else if (
|
|
5615
6600
|
!sessionAlive &&
|
|
5616
6601
|
activeTaskProcesses.get(normalizedTargetTaskId) === activeTarget
|
|
5617
6602
|
) {
|
|
@@ -5946,13 +6931,38 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5946
6931
|
env.CONDUCTOR_BACKEND_URL = BACKEND_HTTP;
|
|
5947
6932
|
}
|
|
5948
6933
|
|
|
5949
|
-
|
|
6934
|
+
// Two hazards below, one root cause: an in-place restart deliberately
|
|
6935
|
+
// re-uses the previous run's working directory, so ANY artifact the last
|
|
6936
|
+
// run left there can be mistaken for this run's own state. Both guards
|
|
6937
|
+
// must run before the spawn.
|
|
6938
|
+
//
|
|
6939
|
+
// (1) The fire's durable upstream outbox lives in that directory. If the
|
|
6940
|
+
// prior run was stopped while its websocket was down, its undelivered
|
|
6941
|
+
// terminal `task_status_update` (KILLED/COMPLETED) is still on disk — and
|
|
6942
|
+
// the run we are about to spawn would flush it during startup, marking the
|
|
6943
|
+
// task killed seconds after resuming and then getting shot down by the
|
|
6944
|
+
// server's `stop_task(task_already_killed)` reply. Purge those superseded
|
|
6945
|
+
// events before the new fire can pick them up.
|
|
6946
|
+
dropSupersededTerminalStatusEvents(taskDir, normalizedTargetTaskId);
|
|
6947
|
+
|
|
6948
|
+
// (2) The log file is opened with `flags:"a"` and re-used as well. Sample
|
|
6949
|
+
// its size first so it is a true "everything after this is ours"
|
|
6950
|
+
// watermark; without it the reaper reads the PREVIOUS run's exit marker.
|
|
6951
|
+
const logStartOffset = readLogSize(logPath);
|
|
6952
|
+
|
|
6953
|
+
const { child, tmuxSession, exitMarkerToken } = spawnFireProcess({
|
|
5950
6954
|
taskId: normalizedTargetTaskId,
|
|
5951
6955
|
args,
|
|
5952
6956
|
env,
|
|
5953
6957
|
cwd: taskDir,
|
|
5954
6958
|
logPath,
|
|
5955
6959
|
});
|
|
6960
|
+
// Bounded tail of the child's output + a spawn timestamp, so an abnormal
|
|
6961
|
+
// exit can report *why* it died and how long it survived. `logPath` lets
|
|
6962
|
+
// the capture recover the fire's own output in tmux mode, where the
|
|
6963
|
+
// daemon's child is only the `tmux new-session` client.
|
|
6964
|
+
const outputCapture = createChildOutputCapture({ logPath, logStartOffset });
|
|
6965
|
+
const spawnedAtMs = Date.now();
|
|
5956
6966
|
|
|
5957
6967
|
let logStream;
|
|
5958
6968
|
try {
|
|
@@ -5983,6 +6993,11 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5983
6993
|
child,
|
|
5984
6994
|
projectId: normalizedProjectId,
|
|
5985
6995
|
logPath,
|
|
6996
|
+
// See the create_task path: watermark for tail reads / exit-marker
|
|
6997
|
+
// scans, the marker nonce, and the reaper's grace-period anchor.
|
|
6998
|
+
logStartOffset,
|
|
6999
|
+
exitMarkerToken,
|
|
7000
|
+
spawnedAtMs,
|
|
5986
7001
|
stopForceKillTimer: null,
|
|
5987
7002
|
managedByFireBridge: true,
|
|
5988
7003
|
tmuxSession: tmuxSession || null,
|
|
@@ -6004,17 +7019,32 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6004
7019
|
} else if (child.stderr && typeof child.stderr.on === "function" && logStream) {
|
|
6005
7020
|
child.stderr.on("data", (chunk) => logStream.write(chunk));
|
|
6006
7021
|
}
|
|
7022
|
+
// Capture independently of logStream: when the log file cannot be
|
|
7023
|
+
// opened (createWriteStream threw) the piping above is skipped
|
|
7024
|
+
// entirely, which is exactly when we most need the tail.
|
|
7025
|
+
outputCapture.attach(child.stdout);
|
|
7026
|
+
outputCapture.attach(child.stderr);
|
|
6007
7027
|
} else if (child.stderr && typeof child.stderr.on === "function") {
|
|
7028
|
+
outputCapture.attach(child.stderr);
|
|
6008
7029
|
child.stderr.on("data", (chunk) => {
|
|
6009
7030
|
const text = chunk?.toString?.("utf8") ?? String(chunk ?? "");
|
|
6010
7031
|
if (text.trim()) {
|
|
6011
|
-
logError(
|
|
7032
|
+
logError(
|
|
7033
|
+
// Redact: this is the fire's raw stderr, which can carry the
|
|
7034
|
+
// handoff share token (argv echo) and provider/agent keys.
|
|
7035
|
+
`tmux(${tmuxSession}) stderr: ${redactSecretsForLogs(text.trim(), [AGENT_TOKEN])}`,
|
|
7036
|
+
);
|
|
6012
7037
|
}
|
|
6013
7038
|
});
|
|
6014
7039
|
}
|
|
6015
7040
|
|
|
6016
7041
|
child.on("error", (err) => {
|
|
6017
|
-
logError(
|
|
7042
|
+
logError(
|
|
7043
|
+
`[fork-spawn] spawn error task=${normalizedTargetTaskId} mode=${normalizedMode} ` +
|
|
7044
|
+
`backend=${selectedBackend} cwd=${taskDir} tmux=${tmuxSession || "none"}: ${
|
|
7045
|
+
maskErrorForLogs(err)?.message || err
|
|
7046
|
+
}`,
|
|
7047
|
+
);
|
|
6018
7048
|
if (logStream) {
|
|
6019
7049
|
const ts = new Date().toLocaleString("sv-SE", { timeZone: "Asia/Shanghai" }).replace(" ", "T");
|
|
6020
7050
|
logStream.write(`[daemon ${ts}] spawn error: ${err.message}\n`);
|
|
@@ -6024,6 +7054,16 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6024
7054
|
child.on("exit", (code, signal) => {
|
|
6025
7055
|
const active = activeTaskProcesses.get(normalizedTargetTaskId);
|
|
6026
7056
|
|
|
7057
|
+
// See the create path: a tmux client exiting after its record was
|
|
7058
|
+
// retired must stay silent, or it would both clobber a replacement
|
|
7059
|
+
// record and publish a status contradicting the reaper's verdict.
|
|
7060
|
+
if (tmuxSession && !active) {
|
|
7061
|
+
log(
|
|
7062
|
+
`tmux client for restarted task ${normalizedTargetTaskId} exited after its record was retired (code=${code}, signal=${signal || "null"}); nothing to report`,
|
|
7063
|
+
);
|
|
7064
|
+
return;
|
|
7065
|
+
}
|
|
7066
|
+
|
|
6027
7067
|
// In tmux mode the `tmux new-session -d` client always exits soon
|
|
6028
7068
|
// after launching the session. A clean exit (code 0, no signal) means
|
|
6029
7069
|
// Fire is now running detached under the tmux server — keep the task
|
|
@@ -6076,6 +7116,26 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6076
7116
|
? "completed"
|
|
6077
7117
|
: `exited with code ${code}`;
|
|
6078
7118
|
|
|
7119
|
+
// Diagnostics for an abnormal exit. This is the line that makes a
|
|
7120
|
+
// "branch task died instantly" incident debuggable: it names the mode,
|
|
7121
|
+
// backend, cwd, tmux session, exit code/signal, how long the child
|
|
7122
|
+
// survived, where its log is, and the tail of what it actually printed.
|
|
7123
|
+
const lifetimeMs = Date.now() - spawnedAtMs;
|
|
7124
|
+
const outputTail = outputCapture.tail();
|
|
7125
|
+
if (status === "KILLED") {
|
|
7126
|
+
logError(
|
|
7127
|
+
`[fork-spawn] abnormal exit task=${normalizedTargetTaskId} mode=${normalizedMode} ` +
|
|
7128
|
+
`backend=${selectedBackend} cwd=${taskDir} tmux=${tmuxSession || "none"} ` +
|
|
7129
|
+
`exit=${code ?? "null"} signal=${signal || "null"} lifetime_ms=${lifetimeMs} ` +
|
|
7130
|
+
`log=${logPath} output_tail=${outputTail ? JSON.stringify(outputTail) : "<empty>"}`,
|
|
7131
|
+
);
|
|
7132
|
+
}
|
|
7133
|
+
// Surface the same tail in the status summary so it reaches the backend
|
|
7134
|
+
// (task_status_events.summary) and the UI, instead of a bare
|
|
7135
|
+
// "exited with code 1" that explains nothing.
|
|
7136
|
+
const reportedSummary =
|
|
7137
|
+
status === "KILLED" && outputTail ? `${summary}: ${outputTail}` : summary;
|
|
7138
|
+
|
|
6079
7139
|
const shouldReportTerminalStatus =
|
|
6080
7140
|
!suppressExitStatusReport &&
|
|
6081
7141
|
(!acceptedRestartAckSent || shouldDaemonReportFireChildTerminalStatus(active));
|
|
@@ -6090,7 +7150,9 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6090
7150
|
task_id: normalizedTargetTaskId,
|
|
6091
7151
|
project_id: normalizedProjectId,
|
|
6092
7152
|
status,
|
|
6093
|
-
summary,
|
|
7153
|
+
summary: reportedSummary,
|
|
7154
|
+
// Idempotency key — see the note in reportRestartFailure.
|
|
7155
|
+
status_event_id: randomUUID(),
|
|
6094
7156
|
},
|
|
6095
7157
|
})
|
|
6096
7158
|
.catch((err) => {
|