@love-moon/conductor-cli 0.7.5 → 0.7.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/bin/conductor-channel.js +3 -4
- package/bin/conductor-chrome.js +3 -2
- package/bin/conductor-config.js +55 -5
- package/bin/conductor-daemon.js +18 -7
- package/bin/conductor-fire.js +23 -19
- package/bin/conductor-send-file.js +2 -3
- package/bin/conductor-serve-ai.js +1 -1
- package/bin/conductor.js +4 -0
- package/package.json +5 -5
- package/src/ai-manager-handlers.js +1 -1
- package/src/cli-update-notifier.js +11 -4
- package/src/conductor-paths.js +61 -0
- package/src/custom-command-handlers.js +4 -2
- package/src/daemon.js +961 -84
- package/src/entity-helpers.js +2 -8
- package/src/fire/resume.js +17 -17
- package/src/handoff-log-mask.js +43 -0
- package/src/runtime-backends.js +2 -4
- package/src/serve-ai/config.js +7 -13
package/src/daemon.js
CHANGED
|
@@ -3,6 +3,7 @@ import path from "node:path";
|
|
|
3
3
|
import os from "node:os";
|
|
4
4
|
import { createRequire } from "node:module";
|
|
5
5
|
import { spawn, spawnSync } from "node:child_process";
|
|
6
|
+
import { randomUUID } from "node:crypto";
|
|
6
7
|
import { fileURLToPath } from "node:url";
|
|
7
8
|
|
|
8
9
|
import dotenv from "dotenv";
|
|
@@ -14,9 +15,15 @@ import {
|
|
|
14
15
|
loadConfig,
|
|
15
16
|
ConfigFileNotFound,
|
|
16
17
|
ProjectContext,
|
|
18
|
+
DurableUpstreamOutboxStore,
|
|
17
19
|
} from "@love-moon/conductor-sdk";
|
|
18
20
|
import { DaemonLogCollector } from "./log-collector.js";
|
|
19
21
|
import { envForExplicitConfigFile } from "./config-env.js";
|
|
22
|
+
import {
|
|
23
|
+
materializeConductorPathEnv,
|
|
24
|
+
resolveConductorConfigPath,
|
|
25
|
+
resolveConductorHome,
|
|
26
|
+
} from "./conductor-paths.js";
|
|
20
27
|
import { createAiManagerHandlers, handleAiManagerRequest } from "./ai-manager-handlers.js";
|
|
21
28
|
import {
|
|
22
29
|
CUSTOM_COMMANDS_CAPABILITY,
|
|
@@ -53,7 +60,9 @@ import {
|
|
|
53
60
|
import {
|
|
54
61
|
maskHandoffUrlForLogs,
|
|
55
62
|
maskErrorForLogs,
|
|
63
|
+
redactSecretsForLogs,
|
|
56
64
|
} from "./handoff-log-mask.js";
|
|
65
|
+
import { StringDecoder } from "node:string_decoder";
|
|
57
66
|
|
|
58
67
|
dotenv.config();
|
|
59
68
|
|
|
@@ -62,8 +71,6 @@ const __dirname = path.dirname(__filename);
|
|
|
62
71
|
const PACKAGE_ROOT = path.join(__dirname, "..");
|
|
63
72
|
const moduleRequire = createRequire(import.meta.url);
|
|
64
73
|
const CLI_PATH = path.resolve(PACKAGE_ROOT, "bin", "conductor-fire.js");
|
|
65
|
-
const DAEMON_LOG_DIR = path.join(os.homedir(), ".conductor", "logs");
|
|
66
|
-
const DAEMON_LOG_PATH = path.join(DAEMON_LOG_DIR, "conductor-daemon.log");
|
|
67
74
|
const CLI_VERSION = (() => {
|
|
68
75
|
try {
|
|
69
76
|
return JSON.parse(fs.readFileSync(path.join(PACKAGE_ROOT, "package.json"), "utf-8")).version;
|
|
@@ -71,6 +78,14 @@ const CLI_VERSION = (() => {
|
|
|
71
78
|
return "unknown";
|
|
72
79
|
}
|
|
73
80
|
})();
|
|
81
|
+
|
|
82
|
+
export function resolveDaemonLogPaths(env = process.env) {
|
|
83
|
+
const logDir = path.join(resolveConductorHome(env), "logs");
|
|
84
|
+
return {
|
|
85
|
+
logDir,
|
|
86
|
+
logPath: path.join(logDir, "conductor-daemon.log"),
|
|
87
|
+
};
|
|
88
|
+
}
|
|
74
89
|
const PLAN_LIMIT_MESSAGES = {
|
|
75
90
|
manual_fire_active_task: "Free plan limit reached: only 1 active fire task is allowed.",
|
|
76
91
|
app_active_task: "Free plan limit reached: only 1 active app task is allowed.",
|
|
@@ -119,8 +134,9 @@ export function probePtyTaskCapability({
|
|
|
119
134
|
|
|
120
135
|
function appendDaemonLog(line) {
|
|
121
136
|
try {
|
|
122
|
-
|
|
123
|
-
fs.
|
|
137
|
+
const { logDir, logPath } = resolveDaemonLogPaths();
|
|
138
|
+
fs.mkdirSync(logDir, { recursive: true });
|
|
139
|
+
fs.appendFileSync(logPath, line);
|
|
124
140
|
} catch {
|
|
125
141
|
// ignore file log errors
|
|
126
142
|
}
|
|
@@ -156,8 +172,7 @@ function sleepSync(ms) {
|
|
|
156
172
|
|
|
157
173
|
function getUserConfig(configFilePath) {
|
|
158
174
|
try {
|
|
159
|
-
const
|
|
160
|
-
const configPath = configFilePath || path.join(home, ".conductor", "config.yaml");
|
|
175
|
+
const configPath = resolveConductorConfigPath(configFilePath);
|
|
161
176
|
if (fs.existsSync(configPath)) {
|
|
162
177
|
const content = fs.readFileSync(configPath, "utf8");
|
|
163
178
|
const parsed = yaml.load(content);
|
|
@@ -179,7 +194,7 @@ function getUserConfig(configFilePath) {
|
|
|
179
194
|
//
|
|
180
195
|
// Resolution order:
|
|
181
196
|
// 1. CONDUCTOR_FIRE_TMUX_MODE env var ("1"/"true"/"on" enable, "0"/"false"/"off" disable)
|
|
182
|
-
// 2. fire_tmux_mode boolean in
|
|
197
|
+
// 2. fire_tmux_mode boolean in the resolved Conductor config.yaml
|
|
183
198
|
// 3. Default: false
|
|
184
199
|
function getFireTmuxModeEnabled(userConfig) {
|
|
185
200
|
const rawEnv = process.env.CONDUCTOR_FIRE_TMUX_MODE;
|
|
@@ -436,7 +451,7 @@ function normalizeOptionalString(value) {
|
|
|
436
451
|
// Re-export the handoff-URL masking helpers so existing external imports
|
|
437
452
|
// keep working. Implementation lives in a dependency-free module so unit
|
|
438
453
|
// tests can import it without pulling in conductor-sdk and friends.
|
|
439
|
-
export { maskHandoffUrlForLogs, maskErrorForLogs };
|
|
454
|
+
export { maskHandoffUrlForLogs, maskErrorForLogs, redactSecretsForLogs };
|
|
440
455
|
|
|
441
456
|
function normalizeTerminalResumeStrategy(value) {
|
|
442
457
|
const normalized = normalizeOptionalString(value);
|
|
@@ -617,6 +632,52 @@ function stripPtyTaskScopedEnv(source) {
|
|
|
617
632
|
return env;
|
|
618
633
|
}
|
|
619
634
|
|
|
635
|
+
/**
|
|
636
|
+
* Purge terminal `task_status_update` events left undelivered by a previous run
|
|
637
|
+
* of `taskId` in `taskDir`, before a restart re-uses that same directory.
|
|
638
|
+
*
|
|
639
|
+
* The fire's durable upstream outbox is stored under the fire's cwd, keyed by
|
|
640
|
+
* the `task:<id>` delivery scope. An in-place restart intentionally re-uses the
|
|
641
|
+
* cwd, so without this purge the new run flushes the old run's KILLED/COMPLETED
|
|
642
|
+
* event on startup and kills the task it just resumed. Best-effort: a restart
|
|
643
|
+
* must never fail because of outbox bookkeeping.
|
|
644
|
+
*/
|
|
645
|
+
function dropSupersededTerminalStatusEvents(taskDir, taskId) {
|
|
646
|
+
if (!taskDir || !taskId) return;
|
|
647
|
+
try {
|
|
648
|
+
const store = DurableUpstreamOutboxStore.forProjectPath(taskDir, `task:${taskId}`);
|
|
649
|
+
// Distinguish "the purge failed" from "the purge does not exist here".
|
|
650
|
+
// The CLI loads the INSTALLED copy of conductor-sdk, not this repo's
|
|
651
|
+
// source, so an un-rebuilt or mismatched package silently lacks this
|
|
652
|
+
// method — and a bare try/catch would swallow the resulting TypeError
|
|
653
|
+
// exactly like a benign fs error, leaving every restart unprotected with
|
|
654
|
+
// no way to tell from the log. Say so explicitly instead. We still do not
|
|
655
|
+
// throw: an unprotected restart usually succeeds, whereas refusing to
|
|
656
|
+
// spawn would break restart outright.
|
|
657
|
+
if (typeof store?.dropPendingTerminalStatusEvents !== "function") {
|
|
658
|
+
logError(
|
|
659
|
+
`Cannot purge superseded terminal status events for task ${taskId}: the installed ` +
|
|
660
|
+
`@love-moon/conductor-sdk has no DurableUpstreamOutboxStore.dropPendingTerminalStatusEvents. ` +
|
|
661
|
+
`This restart is unprotected — a terminal status left by the previous run may kill it. ` +
|
|
662
|
+
`Rebuild and reinstall the SDK (make install-cli).`,
|
|
663
|
+
);
|
|
664
|
+
return;
|
|
665
|
+
}
|
|
666
|
+
const dropped = store.dropPendingTerminalStatusEvents();
|
|
667
|
+
if (dropped.length > 0) {
|
|
668
|
+
log(
|
|
669
|
+
`Dropped ${dropped.length} superseded terminal status event(s) for task ${taskId} before restart: ${dropped
|
|
670
|
+
.map((entry) => `${entry.payload?.status ?? "?"}@${entry.createdAt}`)
|
|
671
|
+
.join(", ")}`,
|
|
672
|
+
);
|
|
673
|
+
}
|
|
674
|
+
} catch (error) {
|
|
675
|
+
logError(
|
|
676
|
+
`Failed to purge superseded terminal status events for task ${taskId}: ${error?.message || error}`,
|
|
677
|
+
);
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
|
|
620
681
|
function buildPtyTaskEnv(baseEnv = process.env, launchEnv = {}) {
|
|
621
682
|
const parentEnv = stripPtyTaskScopedEnv(baseEnv);
|
|
622
683
|
const taskLaunchEnv = stripPtyTaskScopedEnv(launchEnv);
|
|
@@ -669,10 +730,15 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
669
730
|
};
|
|
670
731
|
|
|
671
732
|
let fileConfig;
|
|
733
|
+
const materializedConductorPathEnv = materializeConductorPathEnv(
|
|
734
|
+
config.CONFIG_FILE,
|
|
735
|
+
process.env,
|
|
736
|
+
);
|
|
737
|
+
const effectiveConfigPath = materializedConductorPathEnv.CONDUCTOR_CONFIG;
|
|
672
738
|
const configFileEnv = envForExplicitConfigFile(config.CONFIG_FILE, process.env);
|
|
673
739
|
try {
|
|
674
740
|
fileConfig = loadConfig(config.CONFIG_FILE, { env: configFileEnv });
|
|
675
|
-
log(`Loaded config from ${
|
|
741
|
+
log(`Loaded config from ${effectiveConfigPath}`);
|
|
676
742
|
} catch (err) {
|
|
677
743
|
if (!(err instanceof ConfigFileNotFound)) {
|
|
678
744
|
log(`Failed to load config: ${err.message}`);
|
|
@@ -712,7 +778,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
712
778
|
os.hostname()
|
|
713
779
|
).trim();
|
|
714
780
|
if (!AGENT_NAME) {
|
|
715
|
-
logError(
|
|
781
|
+
logError(`Daemon name is required. Set daemon_name in ${effectiveConfigPath} or CONDUCTOR_DAEMON_NAME.`);
|
|
716
782
|
return exitAndReturn(1);
|
|
717
783
|
}
|
|
718
784
|
const homeDir = process.env.HOME || os.homedir() || "/tmp";
|
|
@@ -811,6 +877,11 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
811
877
|
const symlinkSyncFn = deps.symlinkSync || fs.symlinkSync;
|
|
812
878
|
const unlinkSyncFn = deps.unlinkSync || fs.unlinkSync;
|
|
813
879
|
const renameSyncFn = deps.renameSync || fs.renameSync;
|
|
880
|
+
// Used to recover a crashed fire's output from its log file when the daemon
|
|
881
|
+
// could not observe the process directly (tmux mode).
|
|
882
|
+
const openSyncFn = deps.openSync || fs.openSync;
|
|
883
|
+
const readSyncFn = deps.readSync || fs.readSync;
|
|
884
|
+
const closeSyncFn = deps.closeSync || fs.closeSync;
|
|
814
885
|
const createWriteStreamFn = deps.createWriteStream || fs.createWriteStream;
|
|
815
886
|
const fetchFn = deps.fetch || fetch;
|
|
816
887
|
const createRtcPeerConnection = deps.createRtcPeerConnection || null;
|
|
@@ -895,6 +966,231 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
895
966
|
return `${buildFireTmuxSessionPrefix(taskId)}${uniq}`;
|
|
896
967
|
}
|
|
897
968
|
|
|
969
|
+
// Observability for spawned fires (RFC: fork-spawn black box).
|
|
970
|
+
//
|
|
971
|
+
// When a forked/branched task's backend dies during startup, the daemon
|
|
972
|
+
// previously reported a bare `exited with code N` and wrote nothing to the
|
|
973
|
+
// daemon log, so the real cause (backend auth failure, missing CLI, tmux
|
|
974
|
+
// launch error, …) was unrecoverable after the fact — even with DB access
|
|
975
|
+
// the only artifact was a generic `fire_exit`. We keep a bounded tail of the
|
|
976
|
+
// child's output so an abnormal exit can name its own cause.
|
|
977
|
+
//
|
|
978
|
+
// The tail is masked with `maskHandoffUrlForLogs` before it reaches any log
|
|
979
|
+
// or status summary: the handoff prompt embeds a share token, and a backend
|
|
980
|
+
// that echoes its argv on failure would otherwise leak a read-grant for the
|
|
981
|
+
// entire transcript into the daemon log and the task status summary.
|
|
982
|
+
const CHILD_OUTPUT_CAPTURE_LIMIT = 4000;
|
|
983
|
+
const CHILD_OUTPUT_SUMMARY_LIMIT = 400;
|
|
984
|
+
|
|
985
|
+
// Sentinel appended to a tmux-hosted fire's log by the wrapper shell (see
|
|
986
|
+
// `spawnFireProcess`). It is the *only* channel through which the daemon
|
|
987
|
+
// can learn the exit code of a process it does not own. The full marker is
|
|
988
|
+
// `[conductor-fire-exit:<per-spawn nonce>] code=<n>`.
|
|
989
|
+
const FIRE_EXIT_MARKER_PREFIX = "[conductor-fire-exit:";
|
|
990
|
+
|
|
991
|
+
const escapeForRegExp = (value) => String(value).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
992
|
+
|
|
993
|
+
// Per-spawn nonce embedded in the marker. Without it the marker is a fixed
|
|
994
|
+
// string that anything writing to the log can forge — and the log is a
|
|
995
|
+
// verbatim copy of the fire's stdout. A fire that merely *prints* the
|
|
996
|
+
// literal (an agent task reading a conductor.log, or editing this file)
|
|
997
|
+
// would plant `[conductor-fire-exit] code=0` in its own log; if that fire
|
|
998
|
+
// were then SIGKILLed, leaving no genuine marker, the reaper would read the
|
|
999
|
+
// forged one and report COMPLETED for a task that was killed. Dogfooding
|
|
1000
|
+
// this repo makes that near-certain rather than theoretical.
|
|
1001
|
+
//
|
|
1002
|
+
// The nonce also means a previous run's marker can never be mistaken for
|
|
1003
|
+
// this run's, independently of the `logStartOffset` watermark.
|
|
1004
|
+
function buildFireExitMarkerToken() {
|
|
1005
|
+
return randomUUID().replace(/-/g, "").slice(0, 12);
|
|
1006
|
+
}
|
|
1007
|
+
|
|
1008
|
+
// Last exit code recorded by *this run's* wrapper shell in `raw`, or null
|
|
1009
|
+
// when the marker is absent (fire was SIGKILLed / the tmux server died
|
|
1010
|
+
// before the wrapper could run). Anchored at line start so it cannot match
|
|
1011
|
+
// mid-line inside quoted output, and scoped by the run's nonce.
|
|
1012
|
+
function parseFireExitCode(raw, markerToken) {
|
|
1013
|
+
if (!raw || !markerToken) return null;
|
|
1014
|
+
const pattern = new RegExp(
|
|
1015
|
+
`^${escapeForRegExp(FIRE_EXIT_MARKER_PREFIX)}${escapeForRegExp(markerToken)}\\] code=(\\d+)`,
|
|
1016
|
+
"gm",
|
|
1017
|
+
);
|
|
1018
|
+
let match = null;
|
|
1019
|
+
let candidate;
|
|
1020
|
+
// Scan for the *last* match rather than the first: a fresh RegExp per
|
|
1021
|
+
// call, because a shared /g instance carries `lastIndex` between calls.
|
|
1022
|
+
while ((candidate = pattern.exec(raw)) !== null) {
|
|
1023
|
+
match = candidate;
|
|
1024
|
+
}
|
|
1025
|
+
if (!match) return null;
|
|
1026
|
+
const code = Number(match[1]);
|
|
1027
|
+
return Number.isFinite(code) ? code : null;
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
// Read the last `limit` bytes of a log file, never crossing below
|
|
1031
|
+
// `minOffset`. The floor matters because a task's log file is opened with
|
|
1032
|
+
// flags "a" and is therefore *reused* across in-place restarts: without it,
|
|
1033
|
+
// a previous run's trailing bytes (including its exit marker) would be
|
|
1034
|
+
// attributed to the current run. Callers that don't care pass minOffset 0.
|
|
1035
|
+
function readLogTailRaw(logPath, { limit = CHILD_OUTPUT_CAPTURE_LIMIT, minOffset = 0 } = {}) {
|
|
1036
|
+
if (!logPath) return "";
|
|
1037
|
+
try {
|
|
1038
|
+
const stat = statSyncFn(logPath);
|
|
1039
|
+
const size = Number(stat?.size) || 0;
|
|
1040
|
+
// A file smaller than the watermark was truncated or rotated out from
|
|
1041
|
+
// under us, so the watermark no longer refers to anything. Honouring it
|
|
1042
|
+
// would return "" for this record forever — which the reaper reads as
|
|
1043
|
+
// "no exit marker", i.e. a fabricated hard death for a task that may
|
|
1044
|
+
// have finished cleanly. Fall back to reading whatever is there.
|
|
1045
|
+
const floor = size < minOffset ? 0 : Math.max(0, Number(minOffset) || 0);
|
|
1046
|
+
const start = Math.max(floor, size - limit);
|
|
1047
|
+
if (size <= start) return "";
|
|
1048
|
+
const fd = openSyncFn(logPath, "r");
|
|
1049
|
+
try {
|
|
1050
|
+
const length = size - start;
|
|
1051
|
+
const buf = Buffer.alloc(length);
|
|
1052
|
+
readSyncFn(fd, buf, 0, length, start);
|
|
1053
|
+
return buf.toString("utf8");
|
|
1054
|
+
} finally {
|
|
1055
|
+
closeSyncFn(fd);
|
|
1056
|
+
}
|
|
1057
|
+
} catch {
|
|
1058
|
+
return "";
|
|
1059
|
+
}
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
// Size of a log file, or null when it does not exist / cannot be read.
|
|
1063
|
+
// The null case is load-bearing for the reaper: `tee -a` creates the file
|
|
1064
|
+
// the moment it opens it, so a *missing* file means the wrapper never had
|
|
1065
|
+
// a writable log at all (bad cwd, unwritable project dir) — as opposed to
|
|
1066
|
+
// an empty-but-present file, which means the fire really did die before
|
|
1067
|
+
// writing anything.
|
|
1068
|
+
function readLogFileSize(logPath) {
|
|
1069
|
+
if (!logPath) return null;
|
|
1070
|
+
try {
|
|
1071
|
+
return Number(statSyncFn(logPath)?.size) || 0;
|
|
1072
|
+
} catch {
|
|
1073
|
+
return null;
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
// Current size of a log file, or 0 when it doesn't exist / can't be read.
|
|
1078
|
+
// Sampled just before a spawn so later tail reads can be floored at "what
|
|
1079
|
+
// this run wrote".
|
|
1080
|
+
function readLogSize(logPath) {
|
|
1081
|
+
return readLogFileSize(logPath) ?? 0;
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
// How far back to hunt for this run's exit marker, and in what stride.
|
|
1085
|
+
// Bounded so a huge log can never turn one sweep into an unbounded read.
|
|
1086
|
+
const FIRE_EXIT_MARKER_SEARCH_LIMIT = 8 * 1024 * 1024;
|
|
1087
|
+
const FIRE_EXIT_MARKER_SEARCH_CHUNK = 64 * 1024;
|
|
1088
|
+
|
|
1089
|
+
// Find this run's exit code by scanning *backwards* from EOF for its nonce.
|
|
1090
|
+
//
|
|
1091
|
+
// Reading only the last few KB would be wrong: a task's log is not private.
|
|
1092
|
+
// Branch/fork tasks deliberately share a worktree — and therefore a single
|
|
1093
|
+
// `conductor.log` — and a task with no worktree config falls back to the
|
|
1094
|
+
// project directory, so every task in the project appends to the same file.
|
|
1095
|
+
// A neighbouring fire that prints a few KB between our exit and the next
|
|
1096
|
+
// reaper sweep would push our marker out of a fixed tail window. The reaper
|
|
1097
|
+
// would then see "no marker", conclude the fire died hard, and report
|
|
1098
|
+
// KILLED for a task that finished cleanly — quoting the *neighbour's*
|
|
1099
|
+
// output as the cause of death. The marker is this run's identity, not a
|
|
1100
|
+
// property of where it happens to sit in a shared stream, so search for it.
|
|
1101
|
+
function findFireExitCode(logPath, { minOffset = 0, markerToken = "" } = {}) {
|
|
1102
|
+
if (!logPath || !markerToken) return null;
|
|
1103
|
+
let size;
|
|
1104
|
+
try {
|
|
1105
|
+
size = Number(statSyncFn(logPath)?.size) || 0;
|
|
1106
|
+
} catch {
|
|
1107
|
+
return null;
|
|
1108
|
+
}
|
|
1109
|
+
// A file shorter than the watermark was truncated/rotated; the offset no
|
|
1110
|
+
// longer refers to anything, so search the whole file rather than nothing.
|
|
1111
|
+
const floor = size < minOffset ? 0 : Math.max(0, Number(minOffset) || 0);
|
|
1112
|
+
const searchFloor = Math.max(floor, size - FIRE_EXIT_MARKER_SEARCH_LIMIT);
|
|
1113
|
+
let fd;
|
|
1114
|
+
try {
|
|
1115
|
+
fd = openSyncFn(logPath, "r");
|
|
1116
|
+
} catch {
|
|
1117
|
+
return null;
|
|
1118
|
+
}
|
|
1119
|
+
try {
|
|
1120
|
+
let end = size;
|
|
1121
|
+
// Overlap successive chunks by enough to cover a marker split across a
|
|
1122
|
+
// boundary, so a line straddling two reads is still found.
|
|
1123
|
+
const overlap = FIRE_EXIT_MARKER_PREFIX.length + markerToken.length + 32;
|
|
1124
|
+
while (end > searchFloor) {
|
|
1125
|
+
const start = Math.max(searchFloor, end - FIRE_EXIT_MARKER_SEARCH_CHUNK);
|
|
1126
|
+
const length = end - start;
|
|
1127
|
+
const buf = Buffer.alloc(length);
|
|
1128
|
+
readSyncFn(fd, buf, 0, length, start);
|
|
1129
|
+
const code = parseFireExitCode(buf.toString("utf8"), markerToken);
|
|
1130
|
+
if (code !== null) return code;
|
|
1131
|
+
if (start <= searchFloor) break;
|
|
1132
|
+
end = start + overlap;
|
|
1133
|
+
}
|
|
1134
|
+
return null;
|
|
1135
|
+
} catch {
|
|
1136
|
+
return null;
|
|
1137
|
+
} finally {
|
|
1138
|
+
try {
|
|
1139
|
+
closeSyncFn(fd);
|
|
1140
|
+
} catch {
|
|
1141
|
+
// best effort
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
|
|
1146
|
+
// Collapse, redact and clamp a raw output tail so it is safe to put in a
|
|
1147
|
+
// daemon log line or a persisted task status summary.
|
|
1148
|
+
function sanitizeOutputTail(raw, limit = CHILD_OUTPUT_SUMMARY_LIMIT) {
|
|
1149
|
+
// Collapse first, then redact: the mask patterns stop at whitespace, so
|
|
1150
|
+
// a secret split across a newline would otherwise only be half-matched.
|
|
1151
|
+
const collapsed = String(raw || "").replace(/\s+/g, " ").trim();
|
|
1152
|
+
const safe = redactSecretsForLogs(collapsed, [AGENT_TOKEN]).trim();
|
|
1153
|
+
if (!safe) return "";
|
|
1154
|
+
// slice(-(limit - 1)) keeps the ellipsis inside the documented budget.
|
|
1155
|
+
return safe.length > limit ? `…${safe.slice(-(limit - 1))}` : safe;
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
function createChildOutputCapture({ logPath = "", logStartOffset = 0 } = {}) {
|
|
1159
|
+
let buffer = "";
|
|
1160
|
+
// Decode across chunk boundaries: a raw per-chunk toString("utf8") splits
|
|
1161
|
+
// multi-byte characters whenever a chunk ends mid-sequence, which would
|
|
1162
|
+
// put mojibake into the persisted status summary.
|
|
1163
|
+
const decoder = new StringDecoder("utf8");
|
|
1164
|
+
const append = (chunk) => {
|
|
1165
|
+
const text =
|
|
1166
|
+
typeof chunk === "string" ? chunk : decoder.write(Buffer.from(chunk));
|
|
1167
|
+
if (!text) return;
|
|
1168
|
+
buffer += text;
|
|
1169
|
+
if (buffer.length > CHILD_OUTPUT_CAPTURE_LIMIT) {
|
|
1170
|
+
buffer = buffer.slice(-CHILD_OUTPUT_CAPTURE_LIMIT);
|
|
1171
|
+
}
|
|
1172
|
+
};
|
|
1173
|
+
// Attaching an extra `data` listener is safe alongside `.pipe()`: pipe
|
|
1174
|
+
// keeps the stream in flowing mode and additional listeners observe the
|
|
1175
|
+
// same chunks without consuming them.
|
|
1176
|
+
const attach = (stream) => {
|
|
1177
|
+
if (stream && typeof stream.on === "function") {
|
|
1178
|
+
stream.on("data", append);
|
|
1179
|
+
}
|
|
1180
|
+
};
|
|
1181
|
+
|
|
1182
|
+
// In tmux mode the daemon's child is the `tmux new-session` client, and
|
|
1183
|
+
// the fire's own output is redirected to `logPath` *inside* the session
|
|
1184
|
+
// (`… 2>&1 | tee -a <logPath>`). The in-memory buffer therefore only ever
|
|
1185
|
+
// holds tmux's own errors. When it is empty, fall back to the tail of the
|
|
1186
|
+
// log file so the backend's actual crash output is still reported.
|
|
1187
|
+
const readLogTail = () => readLogTailRaw(logPath, { minOffset: logStartOffset });
|
|
1188
|
+
|
|
1189
|
+
const tail = (limit = CHILD_OUTPUT_SUMMARY_LIMIT) =>
|
|
1190
|
+
sanitizeOutputTail(buffer.trim() ? buffer : readLogTail(), limit);
|
|
1191
|
+
return { attach, append, tail };
|
|
1192
|
+
}
|
|
1193
|
+
|
|
898
1194
|
// Spawn the Fire CLI either directly (default) or inside a detached tmux
|
|
899
1195
|
// session (when FIRE_TMUX_MODE_ACTIVE). In tmux mode the returned `child`
|
|
900
1196
|
// is the short-lived `tmux new-session` client; once it exits with code 0
|
|
@@ -908,7 +1204,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
908
1204
|
env,
|
|
909
1205
|
stdio: ["inherit", "pipe", "pipe"],
|
|
910
1206
|
});
|
|
911
|
-
return { child, tmuxSession: null };
|
|
1207
|
+
return { child, tmuxSession: null, exitMarkerToken: "" };
|
|
912
1208
|
}
|
|
913
1209
|
|
|
914
1210
|
const sessionName = buildFireTmuxSessionName(taskId);
|
|
@@ -919,9 +1215,37 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
919
1215
|
// via `tmux a -t <session>` for live observation is useless. With
|
|
920
1216
|
// `tee` the same bytes go to both the pane (visible to whoever
|
|
921
1217
|
// attaches) and the log file (preserved for offline inspection).
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
1218
|
+
//
|
|
1219
|
+
// The trailing exit-marker is what makes a death *inside* the session
|
|
1220
|
+
// observable. The daemon's child is only the `tmux new-session` client,
|
|
1221
|
+
// so once the session exists the daemon has no handle on Fire and never
|
|
1222
|
+
// sees its exit code. `${PIPESTATUS[0]}` recovers Fire's own status from
|
|
1223
|
+
// the pipeline (plain `$?` would be tee's), and appending it to the log
|
|
1224
|
+
// lets the liveness reaper tell "finished normally" from "crashed" —
|
|
1225
|
+
// without which it could only ever guess, and guessing KILLED would
|
|
1226
|
+
// mis-report successful tasks as failures.
|
|
1227
|
+
//
|
|
1228
|
+
// No `exec` in this branch: the wrapper bash must outlive Fire to write
|
|
1229
|
+
// the marker. When Fire is SIGKILLed (or the tmux server dies) the
|
|
1230
|
+
// marker is simply absent, which the reaper reads as "died hard" — the
|
|
1231
|
+
// correct conclusion.
|
|
1232
|
+
//
|
|
1233
|
+
// Each branch owns its own `exec` (or absence of one) and the result is
|
|
1234
|
+
// passed to `bash -c` verbatim. Do NOT re-wrap it in `exec` at the call
|
|
1235
|
+
// site: `exec exec …` is a bash error ("exec: exec: not found", 127),
|
|
1236
|
+
// which in tmux mode means Fire never starts while the tmux client still
|
|
1237
|
+
// exits 0 — a task that hangs at `running` with nothing in the log.
|
|
1238
|
+
const quotedLogPath = logPath ? shellQuoteForBash(logPath) : "";
|
|
1239
|
+
// `\n` before the marker guarantees it starts its own line even when the
|
|
1240
|
+
// fire's last write had no trailing newline — the reader anchors on `^`.
|
|
1241
|
+
const exitMarkerToken = logPath ? buildFireExitMarkerToken() : "";
|
|
1242
|
+
const shellCommand = logPath
|
|
1243
|
+
? `${innerCommandParts.join(" ")} 2>&1 | tee -a ${quotedLogPath}; ` +
|
|
1244
|
+
`__conductor_fire_code=\${PIPESTATUS[0]}; ` +
|
|
1245
|
+
`printf '\\n${FIRE_EXIT_MARKER_PREFIX}${exitMarkerToken}] code=%s\\n' ` +
|
|
1246
|
+
`"$__conductor_fire_code" >> ${quotedLogPath}; ` +
|
|
1247
|
+
`exit $__conductor_fire_code`
|
|
1248
|
+
: `exec ${innerCommandParts.join(" ")}`;
|
|
925
1249
|
|
|
926
1250
|
// Build `-e KEY=VALUE` flags for the new session.
|
|
927
1251
|
//
|
|
@@ -969,7 +1293,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
969
1293
|
cwd,
|
|
970
1294
|
"bash",
|
|
971
1295
|
"-c",
|
|
972
|
-
|
|
1296
|
+
shellCommand,
|
|
973
1297
|
];
|
|
974
1298
|
log(`Spawning Fire via tmux: session=${sessionName} cwd=${cwd}`);
|
|
975
1299
|
const child = spawnFn("tmux", tmuxArgs, {
|
|
@@ -981,7 +1305,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
981
1305
|
if (typeof child.unref === "function") {
|
|
982
1306
|
child.unref();
|
|
983
1307
|
}
|
|
984
|
-
return { child, tmuxSession: sessionName };
|
|
1308
|
+
return { child, tmuxSession: sessionName, exitMarkerToken };
|
|
985
1309
|
}
|
|
986
1310
|
|
|
987
1311
|
// Async probe: does the named tmux session still exist? Resolves to a
|
|
@@ -1000,34 +1324,45 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1000
1324
|
}
|
|
1001
1325
|
return 5000;
|
|
1002
1326
|
})();
|
|
1003
|
-
|
|
1327
|
+
// Detailed probe: resolves `{ alive, conclusive }`.
|
|
1328
|
+
//
|
|
1329
|
+
// `conclusive` distinguishes "tmux answered, and the session is gone"
|
|
1330
|
+
// (has-session exited non-zero) from "we never got an answer" (spawn error,
|
|
1331
|
+
// tmux binary missing, wedged server hitting the timeout). Both used to
|
|
1332
|
+
// collapse into a bare `false`, which was fine when the only consequence
|
|
1333
|
+
// was dropping a bookkeeping entry. It is NOT fine now that the reaper
|
|
1334
|
+
// turns "not alive" into an authoritative KILLED with a cause of death: one
|
|
1335
|
+
// transient tmux hiccup would otherwise declare every live tmux task dead,
|
|
1336
|
+
// each with a confidently wrong "disappeared without an exit marker".
|
|
1337
|
+
function probeTmuxSession(sessionName) {
|
|
1004
1338
|
return new Promise((resolve) => {
|
|
1005
1339
|
if (!sessionName) {
|
|
1006
|
-
resolve(false);
|
|
1340
|
+
resolve({ alive: false, conclusive: false });
|
|
1007
1341
|
return;
|
|
1008
1342
|
}
|
|
1009
1343
|
let settled = false;
|
|
1010
1344
|
let probe = null;
|
|
1011
1345
|
let timer = null;
|
|
1012
|
-
const settle = (alive) => {
|
|
1346
|
+
const settle = (alive, conclusive) => {
|
|
1013
1347
|
if (settled) return;
|
|
1014
1348
|
settled = true;
|
|
1015
1349
|
if (timer) {
|
|
1016
1350
|
clearTimeout(timer);
|
|
1017
1351
|
timer = null;
|
|
1018
1352
|
}
|
|
1019
|
-
resolve(alive);
|
|
1353
|
+
resolve({ alive, conclusive });
|
|
1020
1354
|
};
|
|
1021
1355
|
try {
|
|
1022
1356
|
probe = spawnFn("tmux", ["has-session", "-t", sessionName], {
|
|
1023
1357
|
stdio: "ignore",
|
|
1024
1358
|
});
|
|
1025
|
-
|
|
1026
|
-
probe.on("
|
|
1359
|
+
// tmux answered: exit 0 = alive, non-zero = genuinely no such session.
|
|
1360
|
+
probe.on("exit", (code) => settle(code === 0, true));
|
|
1361
|
+
// Could not run tmux at all — says nothing about the session.
|
|
1362
|
+
probe.on("error", () => settle(false, false));
|
|
1027
1363
|
timer = setTimeout(() => {
|
|
1028
|
-
// Probe took too long —
|
|
1029
|
-
//
|
|
1030
|
-
// and best-effort kill the stuck child so it doesn't pile up.
|
|
1364
|
+
// Probe took too long — the session state is unknown. Best-effort
|
|
1365
|
+
// kill the stuck child so it doesn't pile up.
|
|
1031
1366
|
try {
|
|
1032
1367
|
if (probe && typeof probe.kill === "function") {
|
|
1033
1368
|
probe.kill("SIGKILL");
|
|
@@ -1038,45 +1373,289 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1038
1373
|
logError(
|
|
1039
1374
|
`tmux has-session probe timed out after ${TMUX_PROBE_TIMEOUT_MS}ms for session ${sessionName}`,
|
|
1040
1375
|
);
|
|
1041
|
-
settle(false);
|
|
1376
|
+
settle(false, false);
|
|
1042
1377
|
}, TMUX_PROBE_TIMEOUT_MS);
|
|
1043
1378
|
if (typeof timer.unref === "function") {
|
|
1044
1379
|
timer.unref();
|
|
1045
1380
|
}
|
|
1046
1381
|
} catch {
|
|
1047
|
-
settle(false);
|
|
1382
|
+
settle(false, false);
|
|
1048
1383
|
}
|
|
1049
1384
|
});
|
|
1050
1385
|
}
|
|
1051
1386
|
|
|
1052
|
-
//
|
|
1053
|
-
//
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1387
|
+
// Boolean form, for the callers that only gate local bookkeeping or a
|
|
1388
|
+
// best-effort kill and are happy to treat "unknown" as "not alive".
|
|
1389
|
+
async function tmuxSessionExists(sessionName) {
|
|
1390
|
+
return (await probeTmuxSession(sessionName)).alive;
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
const BACKEND_STATUS_PROBE_TIMEOUT_MS = (() => {
|
|
1394
|
+
const explicit = Number(config.BACKEND_STATUS_PROBE_TIMEOUT_MS);
|
|
1395
|
+
if (Number.isFinite(explicit) && explicit > 0) {
|
|
1396
|
+
return explicit;
|
|
1397
|
+
}
|
|
1398
|
+
return 10_000;
|
|
1399
|
+
})();
|
|
1400
|
+
|
|
1401
|
+
// Current backend-side status of a task, lowercased, or null when it can't
|
|
1402
|
+
// be determined (HTTP error, unreachable backend, malformed payload).
|
|
1057
1403
|
//
|
|
1058
|
-
//
|
|
1059
|
-
//
|
|
1060
|
-
//
|
|
1061
|
-
//
|
|
1062
|
-
//
|
|
1063
|
-
//
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1404
|
+
// Hard timeout, for the same reason `tmuxSessionExists` has one: this call
|
|
1405
|
+
// is awaited inside the reaper sweep, and the sweep's re-entrancy latch is
|
|
1406
|
+
// only released in a `finally`. A backend that black-holes the connection
|
|
1407
|
+
// would otherwise leave the latch stuck forever — permanently disabling the
|
|
1408
|
+
// only observer of in-session deaths, which is the exact failure this whole
|
|
1409
|
+
// change exists to prevent.
|
|
1410
|
+
async function fetchBackendTaskStatus(taskId) {
|
|
1411
|
+
if (!BACKEND_HTTP || !AGENT_TOKEN || !taskId) return null;
|
|
1412
|
+
// The timeout must cover reading the BODY, not just the headers. `fetch`
|
|
1413
|
+
// resolves as soon as headers arrive, so wrapping only that call leaves
|
|
1414
|
+
// `response.json()` unbounded: a backend that answers 200 and then stalls
|
|
1415
|
+
// the body (half-open proxy, hung route) parks this await forever, the
|
|
1416
|
+
// sweep's re-entrancy latch never clears, and the only observer of
|
|
1417
|
+
// in-session deaths is permanently disabled. `markBackendHttpSuccess` is
|
|
1418
|
+
// likewise deferred until the body actually parsed — crediting the
|
|
1419
|
+
// watchdog for a response we could not read would be a lie.
|
|
1420
|
+
const controller = typeof AbortController !== "undefined" ? new AbortController() : null;
|
|
1421
|
+
try {
|
|
1422
|
+
const status = await withTimeout(
|
|
1423
|
+
(async () => {
|
|
1424
|
+
const response = await fetchFn(`${BACKEND_HTTP}/api/tasks/${taskId}`, {
|
|
1425
|
+
method: "GET",
|
|
1426
|
+
headers: {
|
|
1427
|
+
Authorization: `Bearer ${AGENT_TOKEN}`,
|
|
1428
|
+
Accept: "application/json",
|
|
1429
|
+
},
|
|
1430
|
+
signal: controller?.signal,
|
|
1431
|
+
});
|
|
1432
|
+
if (!response?.ok) return null;
|
|
1433
|
+
const task = await response.json();
|
|
1434
|
+
markBackendHttpSuccess();
|
|
1435
|
+
return String(task?.status || "").trim().toLowerCase() || null;
|
|
1436
|
+
})(),
|
|
1437
|
+
BACKEND_STATUS_PROBE_TIMEOUT_MS,
|
|
1438
|
+
`task status probe for ${taskId}`,
|
|
1439
|
+
);
|
|
1440
|
+
return status;
|
|
1441
|
+
} catch (error) {
|
|
1442
|
+
// Release the socket; withTimeout only stops us waiting, it cannot
|
|
1443
|
+
// cancel the request on its own.
|
|
1444
|
+
try {
|
|
1445
|
+
controller?.abort();
|
|
1446
|
+
} catch {
|
|
1447
|
+
// best effort
|
|
1448
|
+
}
|
|
1449
|
+
logError(`Failed to query backend status for task ${taskId}: ${error?.message || error}`);
|
|
1450
|
+
return null;
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1453
|
+
|
|
1454
|
+
const TERMINAL_BACKEND_TASK_STATUSES = new Set(["completed", "killed"]);
|
|
1455
|
+
|
|
1456
|
+
// A tmux session vanished. Decide what — if anything — to tell the backend.
|
|
1457
|
+
//
|
|
1458
|
+
// This is the fix for "fire dies inside its tmux session and nobody
|
|
1459
|
+
// notices". The daemon's child was only the `tmux new-session` client,
|
|
1460
|
+
// which exited 0 long ago, so there is no exit event to observe. Without a
|
|
1461
|
+
// report here the task sits at `running` until the next reconcile sweep
|
|
1462
|
+
// collects it, and reconcile's `PATCH {status:"killed"}` is rewritten by
|
|
1463
|
+
// the task route into `killing` → `killed`, which the upstream commit path
|
|
1464
|
+
// reads as `killedReason = "user_stopped"`. The user never pressed stop,
|
|
1465
|
+
// and the real cause is lost.
|
|
1466
|
+
//
|
|
1467
|
+
// Two hazards make a naive "session gone → report KILLED" wrong, and both
|
|
1468
|
+
// are handled below:
|
|
1469
|
+
// 1. A session also disappears when fire finished *successfully*.
|
|
1470
|
+
// Reporting KILLED unconditionally would rewrite completed tasks as
|
|
1471
|
+
// failures. We therefore classify from the exit marker the wrapper
|
|
1472
|
+
// shell appends to the log, not from the disappearance itself.
|
|
1473
|
+
// 2. Fire usually reports its own terminal status over its own
|
|
1474
|
+
// websocket. `commitTaskStatusUpdate` has no terminal→terminal guard,
|
|
1475
|
+
// so a late KILLED from us would clobber a COMPLETED from fire. We
|
|
1476
|
+
// ask the backend first and stay silent when the task already reached
|
|
1477
|
+
// a terminal state.
|
|
1478
|
+
//
|
|
1479
|
+
// Note on `suppressedExitStatusReports`: it is deliberately NOT consulted
|
|
1480
|
+
// here. Every path that sets it (refresh_session_inplace via
|
|
1481
|
+
// `stopActiveTaskProcess`) removes the active record synchronously, so the
|
|
1482
|
+
// reaper can never see a record it applies to — but the flag itself is
|
|
1483
|
+
// never cleared in tmux mode, because the tmux client's exit handler
|
|
1484
|
+
// returns early and only the non-early path consumes it. Honouring a flag
|
|
1485
|
+
// that outlives its task would silence a later, unrelated death. The
|
|
1486
|
+
// backend status pre-check below is the guard that actually matters.
|
|
1487
|
+
async function reportDeadTmuxSessionStatus(taskId, record) {
|
|
1488
|
+
if (!record?.projectId) {
|
|
1489
|
+
logError(`Cannot report terminal status for task ${taskId}: no project id on the active record`);
|
|
1490
|
+
return;
|
|
1491
|
+
}
|
|
1492
|
+
// The exit marker only exists because the wrapper shell writes it to
|
|
1493
|
+
// `logPath`. Without one there is no evidence at all, and "no marker"
|
|
1494
|
+
// would be misread as "died hard" — reporting every clean finish as
|
|
1495
|
+
// KILLED. Staying silent restores the old blind spot for this task only,
|
|
1496
|
+
// which is strictly better than manufacturing a wrong cause of death.
|
|
1497
|
+
if (!record.logPath) {
|
|
1498
|
+
logError(
|
|
1499
|
+
`Cannot classify the end of task ${taskId}: no log path on the active record, so no exit marker exists`,
|
|
1500
|
+
);
|
|
1501
|
+
return;
|
|
1502
|
+
}
|
|
1503
|
+
// Same reasoning, one level down. `tee -a` creates the log the instant it
|
|
1504
|
+
// opens it, so a missing file means the wrapper could never write there
|
|
1505
|
+
// (unwritable or vanished cwd) — and `tee` failing does NOT stop the fire,
|
|
1506
|
+
// which runs to completion while the tmux client still exits 0. Treating
|
|
1507
|
+
// that absence as "no exit marker" would report a green task as KILLED
|
|
1508
|
+
// with an invented "killed or OOM" cause. An empty-but-present file is
|
|
1509
|
+
// different: the fire really did die before writing anything.
|
|
1510
|
+
if (readLogFileSize(record.logPath) === null) {
|
|
1511
|
+
logError(
|
|
1512
|
+
`Cannot classify the end of task ${taskId}: log file ${record.logPath} is missing, ` +
|
|
1513
|
+
`so the absence of an exit marker proves nothing`,
|
|
1514
|
+
);
|
|
1515
|
+
return;
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
const rawTail = readLogTailRaw(record.logPath, { minOffset: record.logStartOffset || 0 });
|
|
1519
|
+
// Search the file for the marker rather than hoping it is still inside
|
|
1520
|
+
// the tail window — see `findFireExitCode` for why a shared log makes the
|
|
1521
|
+
// tail unreliable. The tail is still what feeds the human-readable
|
|
1522
|
+
// summary; only the *verdict* needs the exhaustive lookup.
|
|
1523
|
+
const exitCode = findFireExitCode(record.logPath, {
|
|
1524
|
+
minOffset: record.logStartOffset || 0,
|
|
1525
|
+
markerToken: record.exitMarkerToken,
|
|
1526
|
+
});
|
|
1527
|
+
const lifetimeMs = record.spawnedAtMs ? Date.now() - record.spawnedAtMs : null;
|
|
1528
|
+
|
|
1529
|
+
// Classification. `null` exit code means the wrapper shell never got to
|
|
1530
|
+
// write the marker — SIGKILL, OOM kill, or a tmux server crash.
|
|
1531
|
+
const isKilled = exitCode === null || exitCode !== 0;
|
|
1532
|
+
const status = isKilled ? "KILLED" : "COMPLETED";
|
|
1533
|
+
//
|
|
1534
|
+
// The success summary deliberately is NOT the bare word "completed".
|
|
1535
|
+
// The web side treats a summary that merely restates the status as
|
|
1536
|
+
// trivial and drops it — but only when it has to synthesize the event
|
|
1537
|
+
// id. We send our own id (for idempotency), which bypasses that filter,
|
|
1538
|
+
// so a bare "completed" would be persisted and would overwrite
|
|
1539
|
+
// `latest_status_summary` with a word that says nothing. Naming *how* we
|
|
1540
|
+
// found out keeps the event worth its row: it tells the next reader that
|
|
1541
|
+
// fire never published its own status.
|
|
1542
|
+
const baseSummary =
|
|
1543
|
+
exitCode === null
|
|
1544
|
+
? "fire tmux session disappeared without an exit marker (killed or OOM)"
|
|
1545
|
+
: exitCode === 0
|
|
1546
|
+
? "completed; detected by the daemon after the tmux session ended (fire never reported it)"
|
|
1547
|
+
: exitCode === 130 || exitCode === 143
|
|
1548
|
+
? `terminated (exit code ${exitCode})`
|
|
1549
|
+
: `exited with code ${exitCode}`;
|
|
1550
|
+
|
|
1551
|
+
// Ask the backend before overwriting anything: fire normally reports its
|
|
1552
|
+
// own terminal status and we must not downgrade a success.
|
|
1553
|
+
const backendStatus = await fetchBackendTaskStatus(taskId);
|
|
1554
|
+
if (backendStatus && TERMINAL_BACKEND_TASK_STATUSES.has(backendStatus)) {
|
|
1555
|
+
log(
|
|
1556
|
+
`Tmux session ${record.tmuxSession} for task ${taskId} ended; backend already ${backendStatus}, no report needed`,
|
|
1557
|
+
);
|
|
1558
|
+
return;
|
|
1559
|
+
}
|
|
1560
|
+
|
|
1561
|
+
// The await above yields the event loop, and "this task looks dead" is
|
|
1562
|
+
// precisely the trigger for an in-place restart. A `restart_task` landing
|
|
1563
|
+
// in that window finds no active record (we deleted it), spawns a fresh
|
|
1564
|
+
// fire and reports RUNNING — and our stale report would then mark the
|
|
1565
|
+
// *live* replacement as killed, using the previous run's log tail. Worse,
|
|
1566
|
+
// the daemon's own map would say running, so reconcile would never repair
|
|
1567
|
+
// it. Re-check that the task is still absent before speaking.
|
|
1568
|
+
if (activeTaskProcesses.has(taskId)) {
|
|
1569
|
+
log(
|
|
1570
|
+
`Task ${taskId} was restarted while its dead tmux session was being reported; dropping the stale terminal status`,
|
|
1571
|
+
);
|
|
1572
|
+
return;
|
|
1573
|
+
}
|
|
1574
|
+
|
|
1575
|
+
// Strip the marker before it reaches a user-visible summary: its content
|
|
1576
|
+
// is already stated by the classification above, so leaving it in just
|
|
1577
|
+
// appends "[conductor-fire-exit] code=1" to "exited with code 1".
|
|
1578
|
+
const outputTail = isKilled
|
|
1579
|
+
? sanitizeOutputTail(
|
|
1580
|
+
rawTail.replace(
|
|
1581
|
+
new RegExp(`${escapeForRegExp(FIRE_EXIT_MARKER_PREFIX)}[^\\]]*\\] code=\\d+`, "g"),
|
|
1582
|
+
"",
|
|
1583
|
+
),
|
|
1584
|
+
)
|
|
1585
|
+
: "";
|
|
1586
|
+
if (isKilled) {
|
|
1587
|
+
logError(
|
|
1588
|
+
`[tmux-reap] fire died inside its session task=${taskId} ` +
|
|
1589
|
+
`tmux=${record.tmuxSession || "none"} exit=${exitCode === null ? "unknown" : exitCode} ` +
|
|
1590
|
+
`lifetime_ms=${lifetimeMs === null ? "unknown" : lifetimeMs} log=${record.logPath || "none"} ` +
|
|
1591
|
+
`backend_status=${backendStatus || "unknown"} ` +
|
|
1592
|
+
`output_tail=${outputTail ? JSON.stringify(outputTail) : "<empty>"}`,
|
|
1593
|
+
);
|
|
1594
|
+
}
|
|
1595
|
+
const summary = outputTail ? `${baseSummary}: ${outputTail}` : baseSummary;
|
|
1596
|
+
|
|
1597
|
+
client
|
|
1598
|
+
.sendJson({
|
|
1599
|
+
type: "task_status_update",
|
|
1600
|
+
payload: {
|
|
1601
|
+
task_id: taskId,
|
|
1602
|
+
project_id: record.projectId,
|
|
1603
|
+
status,
|
|
1604
|
+
summary,
|
|
1605
|
+
// Idempotency key. Generated per call, so it only dedupes
|
|
1606
|
+
// transport-level redelivery of *this* message — not a second,
|
|
1607
|
+
// independent reaper report (the record deletion above is what
|
|
1608
|
+
// prevents those). It is also what makes the backend persist
|
|
1609
|
+
// `summary` at all: without a client-supplied id the server
|
|
1610
|
+
// synthesizes a fresh one per delivery.
|
|
1611
|
+
status_event_id: randomUUID(),
|
|
1612
|
+
},
|
|
1613
|
+
})
|
|
1614
|
+
.catch((err) => {
|
|
1615
|
+
logError(
|
|
1616
|
+
`Failed to report task status (${status}) for reaped tmux task ${taskId}: ${err?.message || err}`,
|
|
1617
|
+
);
|
|
1618
|
+
});
|
|
1619
|
+
}
|
|
1620
|
+
|
|
1621
|
+
// Walk every tmux-mode entry in `activeTaskProcesses`, remove the ones
|
|
1622
|
+
// whose tmux session no longer exists, and report a terminal status for
|
|
1623
|
+
// them. We don't (and can't reliably) observe the inner Fire process exit
|
|
1624
|
+
// when we never owned it as a child, so this sweep is both what keeps the
|
|
1625
|
+
// active map from leaking across long daemon lifetimes *and* the only
|
|
1626
|
+
// place a death inside the session can be noticed at all.
|
|
1627
|
+
//
|
|
1628
|
+
// Startup race (now guarded — it used to be merely theoretical):
|
|
1629
|
+
// `spawnFireProcess` returns synchronously and we set the active record
|
|
1630
|
+
// before the spawned `tmux new-session -d` client has actually exited,
|
|
1631
|
+
// so there is a window in which an active record exists but
|
|
1632
|
+
// `tmux has-session` returns false because the session has not finished
|
|
1633
|
+
// registering with the tmux server. That window was harmless while the
|
|
1634
|
+
// reaper only cleaned up local bookkeeping; now that it reports terminal
|
|
1635
|
+
// statuses, hitting it would kill a task that started fine. Records are
|
|
1636
|
+
// therefore ignored until they are `TMUX_REAP_GRACE_MS` old.
|
|
1069
1637
|
async function reapDeadTmuxSessionsOnce() {
|
|
1070
1638
|
const candidates = [];
|
|
1639
|
+
const now = Date.now();
|
|
1071
1640
|
for (const [taskId, record] of activeTaskProcesses.entries()) {
|
|
1072
|
-
if (record?.tmuxMode
|
|
1073
|
-
|
|
1074
|
-
|
|
1641
|
+
if (!record?.tmuxMode || !record.tmuxSession) continue;
|
|
1642
|
+
const spawnedAtMs = Number(record.spawnedAtMs) || 0;
|
|
1643
|
+
if (spawnedAtMs && now - spawnedAtMs < TMUX_REAP_GRACE_MS) continue;
|
|
1644
|
+
candidates.push([taskId, record]);
|
|
1075
1645
|
}
|
|
1076
1646
|
for (const [taskId, record] of candidates) {
|
|
1077
|
-
const alive = await
|
|
1078
|
-
//
|
|
1079
|
-
//
|
|
1647
|
+
const { alive, conclusive } = await probeTmuxSession(record.tmuxSession);
|
|
1648
|
+
// An inconclusive probe (tmux missing, wedged server, timeout) is not
|
|
1649
|
+
// evidence of death. Leave the record alone and retry next sweep rather
|
|
1650
|
+
// than cleaning up — and reporting a cause of death — on a guess.
|
|
1651
|
+
if (!alive && !conclusive) {
|
|
1652
|
+
logError(
|
|
1653
|
+
`Could not determine whether tmux session ${record.tmuxSession} for task ${taskId} is alive; leaving it untouched`,
|
|
1654
|
+
);
|
|
1655
|
+
continue;
|
|
1656
|
+
}
|
|
1657
|
+
// Only act if the entry still points to the same record; a concurrent
|
|
1658
|
+
// restart_task may have replaced it while we were probing.
|
|
1080
1659
|
if (!alive && activeTaskProcesses.get(taskId) === record) {
|
|
1081
1660
|
log(
|
|
1082
1661
|
`Tmux session ${record.tmuxSession} for task ${taskId} no longer exists; cleaning up activeTaskProcesses entry`,
|
|
@@ -1086,6 +1665,9 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1086
1665
|
record.stopForceKillTimer = null;
|
|
1087
1666
|
}
|
|
1088
1667
|
activeTaskProcesses.delete(taskId);
|
|
1668
|
+
// Report *after* dropping the local entry so a slow backend round
|
|
1669
|
+
// trip can't make a concurrent stop_task wait on us.
|
|
1670
|
+
await reportDeadTmuxSessionStatus(taskId, record);
|
|
1089
1671
|
}
|
|
1090
1672
|
}
|
|
1091
1673
|
}
|
|
@@ -1426,6 +2008,10 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1426
2008
|
}
|
|
1427
2009
|
}
|
|
1428
2010
|
|
|
2011
|
+
// Monotonic suffix so two symlink swaps inside one daemon process can never
|
|
2012
|
+
// collide on the same temp path.
|
|
2013
|
+
let symlinkSequence = 0;
|
|
2014
|
+
|
|
1429
2015
|
async function ensureTaskWorktreeSymlinks({ projectRepoRoot, projectWorkspacePath, finalCwd }) {
|
|
1430
2016
|
const { symlinkPaths } = readProjectWorktreeSettings(projectWorkspacePath);
|
|
1431
2017
|
for (const configuredPath of symlinkPaths) {
|
|
@@ -1439,6 +2025,27 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1439
2025
|
if (await isGitTrackedWorktreePath({ projectRepoRoot, sourcePath })) {
|
|
1440
2026
|
continue;
|
|
1441
2027
|
}
|
|
2028
|
+
|
|
2029
|
+
// Never link to a source that isn't there. `symlinkSync` does NOT
|
|
2030
|
+
// require its target to exist (POSIX), so a stale `worktree.symlink`
|
|
2031
|
+
// entry — e.g. `xr/android/build/local.properties`, generated locally by
|
|
2032
|
+
// the IDE and never present on this machine — would otherwise make the
|
|
2033
|
+
// daemon *manufacture* a dangling link on the very first worktree prep.
|
|
2034
|
+
// Those links are pure liability: they break tooling that follows them,
|
|
2035
|
+
// and they are what the EEXIST-on-reprepare bug below fed on.
|
|
2036
|
+
//
|
|
2037
|
+
// Note this is the one place where `existsSync` is the RIGHT probe:
|
|
2038
|
+
// here we genuinely care whether the target resolves to something real
|
|
2039
|
+
// (a source that is itself a dangling link is equally useless). The
|
|
2040
|
+
// destination probe further down must use `lstat` instead, for exactly
|
|
2041
|
+
// the opposite reason — see the note there.
|
|
2042
|
+
if (!existsSyncFn(sourcePath)) {
|
|
2043
|
+
logError(
|
|
2044
|
+
`[worktree] skipping symlink for missing source: ${configuredPath} (expected at ${sourcePath})`,
|
|
2045
|
+
);
|
|
2046
|
+
continue;
|
|
2047
|
+
}
|
|
2048
|
+
|
|
1442
2049
|
const linkPath = resolveProjectScopedPath(
|
|
1443
2050
|
finalCwd,
|
|
1444
2051
|
configuredPath,
|
|
@@ -1446,21 +2053,70 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1446
2053
|
);
|
|
1447
2054
|
mkdirSyncFn(path.dirname(linkPath), { recursive: true });
|
|
1448
2055
|
|
|
1449
|
-
|
|
2056
|
+
// NOTE: probe with lstat, never existsSync. existsSync FOLLOWS symlinks,
|
|
2057
|
+
// so a dangling link (source deleted after the worktree was created —
|
|
2058
|
+
// .venv / node_modules / local.properties are all gitignored churn)
|
|
2059
|
+
// reads as "missing", and the symlinkSyncFn below then throws EEXIST.
|
|
2060
|
+
// That made the task permanently un-restartable.
|
|
2061
|
+
let linkStat = null;
|
|
2062
|
+
try {
|
|
2063
|
+
linkStat = lstatSyncFn(linkPath);
|
|
2064
|
+
} catch (error) {
|
|
2065
|
+
if (error?.code !== "ENOENT") {
|
|
2066
|
+
throw error instanceof Error ? error : new Error(String(error));
|
|
2067
|
+
}
|
|
2068
|
+
}
|
|
2069
|
+
|
|
2070
|
+
if (linkStat) {
|
|
2071
|
+
if (!linkStat.isSymbolicLink()) {
|
|
2072
|
+
throw new Error(
|
|
2073
|
+
`worktree symlink destination already exists and is not a symlink: ${linkPath}. ` +
|
|
2074
|
+
`Refusing to replace it because it may hold real data — remove it manually, ` +
|
|
2075
|
+
`or drop "${configuredPath}" from worktree.symlink in .conductor/settings.yaml.`,
|
|
2076
|
+
);
|
|
2077
|
+
}
|
|
2078
|
+
// Compare the link's TARGET, not whether that target resolves. A link
|
|
2079
|
+
// that already points at the right place is correct even when the
|
|
2080
|
+
// source is currently absent.
|
|
2081
|
+
const currentTarget = readlinkSyncFn(linkPath);
|
|
2082
|
+
const currentResolvedTarget = path.resolve(path.dirname(linkPath), currentTarget);
|
|
2083
|
+
if (currentResolvedTarget === sourcePath) {
|
|
2084
|
+
continue;
|
|
2085
|
+
}
|
|
2086
|
+
// Self-heal instead of aborting. A stale link happens whenever the
|
|
2087
|
+
// project moves on disk or its workspace_path binding is edited: every
|
|
2088
|
+
// pre-existing worktree then holds links to the OLD absolute path, and
|
|
2089
|
+
// throwing here made every task in them permanently un-restartable.
|
|
2090
|
+
//
|
|
2091
|
+
// Replacing is safe precisely because this entry is a SYMLINK: it
|
|
2092
|
+
// carries no data of its own, so unlinking destroys nothing. (The
|
|
2093
|
+
// non-symlink case above still throws — a real file or directory here
|
|
2094
|
+
// may hold user data and must never be clobbered.) The source is also
|
|
2095
|
+
// known to exist at this point, thanks to the guard further up, so we
|
|
2096
|
+
// are converging on a link that actually resolves.
|
|
2097
|
+
log(
|
|
2098
|
+
`[worktree] repointing stale symlink ${linkPath}: ${currentResolvedTarget} -> ${sourcePath}`,
|
|
2099
|
+
);
|
|
2100
|
+
// Atomic replace. branch/fork tasks deliberately SHARE one worktree
|
|
2101
|
+
// (identity is keyed on worktreeBranch), so two preparations can run
|
|
2102
|
+
// against this directory concurrently. A plain unlink+symlink leaves a
|
|
2103
|
+
// window where the peer's symlinkSync hits EEXIST — reintroducing the
|
|
2104
|
+
// very failure this function was fixed for. symlink-to-temp + rename
|
|
2105
|
+
// has no such window: rename(2) atomically replaces the entry.
|
|
2106
|
+
const tempLinkPath = `${linkPath}.conductor-tmp-${process.pid}-${symlinkSequence++}`;
|
|
2107
|
+
const relativeTargetForSwap = path.relative(path.dirname(linkPath), sourcePath) || ".";
|
|
2108
|
+
symlinkSyncFn(relativeTargetForSwap, tempLinkPath);
|
|
1450
2109
|
try {
|
|
1451
|
-
|
|
1452
|
-
if (!stat.isSymbolicLink()) {
|
|
1453
|
-
throw new Error(`worktree symlink destination already exists: ${linkPath}`);
|
|
1454
|
-
}
|
|
1455
|
-
const currentTarget = readlinkSyncFn(linkPath);
|
|
1456
|
-
const currentResolvedTarget = path.resolve(path.dirname(linkPath), currentTarget);
|
|
1457
|
-
if (currentResolvedTarget === sourcePath) {
|
|
1458
|
-
continue;
|
|
1459
|
-
}
|
|
1460
|
-
throw new Error(`worktree symlink destination already points elsewhere: ${linkPath}`);
|
|
2110
|
+
renameSyncFn(tempLinkPath, linkPath);
|
|
1461
2111
|
} catch (error) {
|
|
2112
|
+
try {
|
|
2113
|
+
unlinkSyncFn(tempLinkPath);
|
|
2114
|
+
} catch {
|
|
2115
|
+
// best effort: never mask the original failure
|
|
2116
|
+
}
|
|
1462
2117
|
throw error instanceof Error ? error : new Error(String(error));
|
|
1463
2118
|
}
|
|
2119
|
+
continue;
|
|
1464
2120
|
}
|
|
1465
2121
|
|
|
1466
2122
|
const relativeTarget = path.relative(path.dirname(linkPath), sourcePath) || ".";
|
|
@@ -2092,6 +2748,20 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
2092
2748
|
return 30 * 1000;
|
|
2093
2749
|
})();
|
|
2094
2750
|
|
|
2751
|
+
// How long a freshly spawned tmux-mode task is exempt from the reaper.
|
|
2752
|
+
// Covers the window between "we recorded the active task" and "the tmux
|
|
2753
|
+
// server finished registering the session", during which `has-session`
|
|
2754
|
+
// can legitimately answer false for a task that started fine. Since the
|
|
2755
|
+
// reaper now reports terminal statuses, a false positive here would kill a
|
|
2756
|
+
// healthy task, so the exemption is no longer optional. Tests set 0.
|
|
2757
|
+
const TMUX_REAP_GRACE_MS = (() => {
|
|
2758
|
+
const explicit = Number(config.TMUX_REAP_GRACE_MS);
|
|
2759
|
+
if (Number.isFinite(explicit) && explicit >= 0) {
|
|
2760
|
+
return explicit;
|
|
2761
|
+
}
|
|
2762
|
+
return 15 * 1000;
|
|
2763
|
+
})();
|
|
2764
|
+
|
|
2095
2765
|
// --- Auto-update state ---
|
|
2096
2766
|
const VERSION_CHECK_INTERVAL_MS = 6 * 60 * 60 * 1000; // 6 hours
|
|
2097
2767
|
let lastVersionCheckAt = 0;
|
|
@@ -2144,8 +2814,8 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
2144
2814
|
if (advertisedCapabilities.length > 0) {
|
|
2145
2815
|
extraHeaders["x-conductor-capabilities"] = advertisedCapabilities.join(",");
|
|
2146
2816
|
}
|
|
2147
|
-
const aiManagerHandlers = createAiManagerHandlers({ configPath:
|
|
2148
|
-
const customCommandHandlers = createCustomCommandHandlers({ configPath:
|
|
2817
|
+
const aiManagerHandlers = createAiManagerHandlers({ configPath: effectiveConfigPath });
|
|
2818
|
+
const customCommandHandlers = createCustomCommandHandlers({ configPath: effectiveConfigPath });
|
|
2149
2819
|
|
|
2150
2820
|
const client = createWebSocketClient(sdkConfig, {
|
|
2151
2821
|
extraHeaders,
|
|
@@ -2690,15 +3360,16 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
2690
3360
|
|
|
2691
3361
|
let logFd = null;
|
|
2692
3362
|
if (shouldRespawn) {
|
|
3363
|
+
const { logDir: daemonLogDir, logPath: daemonLogPath } = resolveDaemonLogPaths();
|
|
2693
3364
|
try {
|
|
2694
|
-
mkdirSyncFn(
|
|
3365
|
+
mkdirSyncFn(daemonLogDir, { recursive: true });
|
|
2695
3366
|
} catch {
|
|
2696
3367
|
/* ignore */
|
|
2697
3368
|
}
|
|
2698
|
-
logFd = fs.openSync(
|
|
3369
|
+
logFd = fs.openSync(daemonLogPath, "a");
|
|
2699
3370
|
if (!isBackgroundProcess) {
|
|
2700
3371
|
log(
|
|
2701
|
-
`[${reason}] Foreground daemon will be respawned in background. Logs: ${
|
|
3372
|
+
`[${reason}] Foreground daemon will be respawned in background. Logs: ${daemonLogPath}`
|
|
2702
3373
|
);
|
|
2703
3374
|
}
|
|
2704
3375
|
}
|
|
@@ -2719,6 +3390,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
2719
3390
|
stdio: ["ignore", logFd, logFd],
|
|
2720
3391
|
env: {
|
|
2721
3392
|
...process.env,
|
|
3393
|
+
...materializedConductorPathEnv,
|
|
2722
3394
|
CONDUCTOR_LOCK_HANDOFF_TOKEN: handoffToken,
|
|
2723
3395
|
CONDUCTOR_LOCK_HANDOFF_FROM_PID: String(process.pid),
|
|
2724
3396
|
CONDUCTOR_LOCK_HANDOFF_EXPIRES_AT: String(handoffExpiresAt),
|
|
@@ -4768,8 +5440,18 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4768
5440
|
|
|
4769
5441
|
let entry = activeTaskProcesses.get(taskId);
|
|
4770
5442
|
if (entry?.tmuxMode && entry.tmuxSession) {
|
|
4771
|
-
|
|
4772
|
-
|
|
5443
|
+
// Only a conclusive "no such session" counts as death. A wedged or
|
|
5444
|
+
// missing tmux answers `false` too, and acting on that would ack the
|
|
5445
|
+
// reclaim as `stale` for a fire that is still running — the backend
|
|
5446
|
+
// then spawns a replacement and two fires share one worktree. It would
|
|
5447
|
+
// also drop the record, so the reaper could never classify the real
|
|
5448
|
+
// death later. When we cannot tell, keep the record and say "alive".
|
|
5449
|
+
const { alive, conclusive } = await probeTmuxSession(entry.tmuxSession);
|
|
5450
|
+
if (!alive && !conclusive) {
|
|
5451
|
+
logError(
|
|
5452
|
+
`Could not determine whether tmux session ${entry.tmuxSession} for task ${taskId} is alive; treating reclaim as still-alive`,
|
|
5453
|
+
);
|
|
5454
|
+
} else if (!alive) {
|
|
4773
5455
|
if (activeTaskProcesses.get(taskId) === entry) {
|
|
4774
5456
|
if (entry.stopForceKillTimer) {
|
|
4775
5457
|
clearTimeout(entry.stopForceKillTimer);
|
|
@@ -4836,6 +5518,40 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4836
5518
|
if ((!processRecord || !processRecord.child) && !ptyRecord) {
|
|
4837
5519
|
log(`Stop requested for task ${taskId}, but no active process found`);
|
|
4838
5520
|
sendStopAck(false);
|
|
5521
|
+
// "Nothing to stop" IS the terminal answer, and the server cannot infer
|
|
5522
|
+
// it: after the app flips the row to `killing` it waits for us to report
|
|
5523
|
+
// a terminal status, and a bare `stop_ack(accepted=false)` is only
|
|
5524
|
+
// command bookkeeping — it never converges the task. Staying silent here
|
|
5525
|
+
// is what strands a task in `killing` forever once its fire has already
|
|
5526
|
+
// died (e.g. the tmux session was reaped before the user pressed Stop).
|
|
5527
|
+
// Mirror the tmux stop path and publish KILLED so the row reaches a
|
|
5528
|
+
// terminal state that the user can then restart from.
|
|
5529
|
+
const terminalProjectId =
|
|
5530
|
+
normalizeOptionalString(payload?.project_id) ||
|
|
5531
|
+
normalizeOptionalString(processRecord?.projectId);
|
|
5532
|
+
if (terminalProjectId) {
|
|
5533
|
+
client
|
|
5534
|
+
.sendJson({
|
|
5535
|
+
type: "task_status_update",
|
|
5536
|
+
payload: {
|
|
5537
|
+
task_id: taskId,
|
|
5538
|
+
project_id: terminalProjectId,
|
|
5539
|
+
status: "KILLED",
|
|
5540
|
+
summary: payload?.reason
|
|
5541
|
+
? `stopped (${payload.reason}); no active process`
|
|
5542
|
+
: "stopped; no active process",
|
|
5543
|
+
},
|
|
5544
|
+
})
|
|
5545
|
+
.catch((err) => {
|
|
5546
|
+
logError(
|
|
5547
|
+
`Failed to report task_status_update(KILLED) for inactive task ${taskId}: ${err?.message || err}`,
|
|
5548
|
+
);
|
|
5549
|
+
});
|
|
5550
|
+
} else {
|
|
5551
|
+
logError(
|
|
5552
|
+
`Cannot report terminal status for inactive task ${taskId}: no project_id in stop_task payload`,
|
|
5553
|
+
);
|
|
5554
|
+
}
|
|
4839
5555
|
// Even when we have no in-memory record, the task may still own a
|
|
4840
5556
|
// tmux session (e.g. the daemon was restarted between spawn and
|
|
4841
5557
|
// stop, or the liveness reaper removed our entry but the session
|
|
@@ -4973,6 +5689,19 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4973
5689
|
: "restart failed";
|
|
4974
5690
|
const scrubbedError = maskErrorForLogs(error);
|
|
4975
5691
|
const summary = `${prefix}: ${scrubbedError?.message || scrubbedError}`;
|
|
5692
|
+
// Always leave a daemon-log trace. Several early rejects (unsupported
|
|
5693
|
+
// backend, cwd resolution, worktree prep) return before the "Restarting
|
|
5694
|
+
// task …" line is ever logged, so without this a failed branch/fork left
|
|
5695
|
+
// the daemon log completely silent and only a generic `fire_exit` in the
|
|
5696
|
+
// backend DB.
|
|
5697
|
+
// Benign, self-explanatory outcomes (orderly shutdown, a refresh that is
|
|
5698
|
+
// already in flight) are logged at normal level so they don't show up as
|
|
5699
|
+
// errors in monitoring; genuine failures still go to stderr.
|
|
5700
|
+
const failureText = `${scrubbedError?.message || scrubbedError}`;
|
|
5701
|
+
const isBenign = /daemon shut(ting)? down|already in progress/i.test(failureText);
|
|
5702
|
+
(isBenign ? log : logError)(
|
|
5703
|
+
`[restart-spawn] failure task=${taskId} mode=${mode || "unknown"}: ${failureText}`,
|
|
5704
|
+
);
|
|
4976
5705
|
if (mode === "refresh_session_inplace") {
|
|
4977
5706
|
rememberCommandRequestAckResult(requestId, false);
|
|
4978
5707
|
}
|
|
@@ -4995,6 +5724,13 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4995
5724
|
project_id: projectId,
|
|
4996
5725
|
status: "KILLED",
|
|
4997
5726
|
summary,
|
|
5727
|
+
// Idempotency key for this transition. The backend persists
|
|
5728
|
+
// `summary` only as a `taskStatusEvent` row, and it keys duplicate
|
|
5729
|
+
// suppression off this id — without one it has to synthesize an id,
|
|
5730
|
+
// which makes a redelivered report indistinguishable from a new
|
|
5731
|
+
// event. Supplying it keeps failure reports both persisted AND
|
|
5732
|
+
// deduplicated.
|
|
5733
|
+
status_event_id: randomUUID(),
|
|
4998
5734
|
},
|
|
4999
5735
|
})
|
|
5000
5736
|
.catch((err) => {
|
|
@@ -5005,8 +5741,14 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5005
5741
|
function reportCreateTaskFailure({ taskId, projectId, requestId, error, sendAck = true }) {
|
|
5006
5742
|
const normalizedTaskId = taskId ? String(taskId) : "";
|
|
5007
5743
|
const normalizedProjectId = projectId ? String(projectId) : "";
|
|
5008
|
-
|
|
5009
|
-
|
|
5744
|
+
// Mirror reportRestartFailure: scrub before the message reaches the log
|
|
5745
|
+
// AND the persisted status summary. An un-scrubbed create failure could
|
|
5746
|
+
// carry the handoff URL or an inherited provider key.
|
|
5747
|
+
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
5748
|
+
const message = redactSecretsForLogs(rawMessage, [AGENT_TOKEN]);
|
|
5749
|
+
logError(
|
|
5750
|
+
`[create-spawn] failure task=${normalizedTaskId || "unknown"}: ${message}`,
|
|
5751
|
+
);
|
|
5010
5752
|
if (sendAck) {
|
|
5011
5753
|
sendAgentCommandAck({
|
|
5012
5754
|
requestId,
|
|
@@ -5028,6 +5770,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5028
5770
|
project_id: normalizedProjectId,
|
|
5029
5771
|
status: "KILLED",
|
|
5030
5772
|
summary: message,
|
|
5773
|
+
status_event_id: randomUUID(),
|
|
5031
5774
|
},
|
|
5032
5775
|
})
|
|
5033
5776
|
.catch((err) => {
|
|
@@ -5272,15 +6015,13 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5272
6015
|
|
|
5273
6016
|
const env = {
|
|
5274
6017
|
...stripPtyTaskScopedEnv(process.env),
|
|
6018
|
+
...materializedConductorPathEnv,
|
|
5275
6019
|
PWD: taskDir,
|
|
5276
6020
|
CONDUCTOR_PROJECT_ID: projectId,
|
|
5277
6021
|
CONDUCTOR_TASK_ID: taskId,
|
|
5278
6022
|
CONDUCTOR_LAUNCHED_BY_DAEMON: "1",
|
|
5279
6023
|
...(cliCommand ? { CONDUCTOR_CLI_COMMAND: cliCommand } : {}),
|
|
5280
6024
|
};
|
|
5281
|
-
if (config.CONFIG_FILE) {
|
|
5282
|
-
env.CONDUCTOR_CONFIG = config.CONFIG_FILE;
|
|
5283
|
-
}
|
|
5284
6025
|
if (AGENT_TOKEN) {
|
|
5285
6026
|
env.CONDUCTOR_AGENT_TOKEN = AGENT_TOKEN;
|
|
5286
6027
|
}
|
|
@@ -5288,7 +6029,10 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5288
6029
|
env.CONDUCTOR_BACKEND_URL = BACKEND_HTTP;
|
|
5289
6030
|
}
|
|
5290
6031
|
|
|
5291
|
-
|
|
6032
|
+
// Sampled before the spawn so it is a true "everything after this is
|
|
6033
|
+
// ours" watermark.
|
|
6034
|
+
const logStartOffset = readLogSize(logPath);
|
|
6035
|
+
const { child, tmuxSession, exitMarkerToken } = spawnFireProcess({
|
|
5292
6036
|
taskId,
|
|
5293
6037
|
args,
|
|
5294
6038
|
env,
|
|
@@ -5346,10 +6090,25 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5346
6090
|
log(`New task workspace: ${taskDir}`);
|
|
5347
6091
|
log(`Logs: ${logPath}`);
|
|
5348
6092
|
|
|
6093
|
+
// Same diagnostics contract as the restart/fork path: without this a
|
|
6094
|
+
// create_task whose backend dies at startup is a black box too.
|
|
6095
|
+
const outputCapture = createChildOutputCapture({ logPath, logStartOffset });
|
|
6096
|
+
const spawnedAtMs = Date.now();
|
|
6097
|
+
|
|
5349
6098
|
activeTaskProcesses.set(taskId, {
|
|
5350
6099
|
child,
|
|
5351
6100
|
projectId,
|
|
5352
6101
|
logPath,
|
|
6102
|
+
// Floor for later tail reads and the reaper's exit-marker scan: the
|
|
6103
|
+
// log file is opened with flags "a" and survives in-place restarts,
|
|
6104
|
+
// so bytes written before this run must not be attributed to it.
|
|
6105
|
+
logStartOffset,
|
|
6106
|
+
// Nonce this run's exit marker is tagged with, so the reaper only
|
|
6107
|
+
// trusts a marker written by *this* wrapper shell.
|
|
6108
|
+
exitMarkerToken,
|
|
6109
|
+
// Consumed by the tmux liveness reaper as both a grace-period anchor
|
|
6110
|
+
// and a lifetime figure for diagnostics.
|
|
6111
|
+
spawnedAtMs,
|
|
5353
6112
|
stopForceKillTimer: null,
|
|
5354
6113
|
managedByFireBridge: true,
|
|
5355
6114
|
tmuxSession: tmuxSession || null,
|
|
@@ -5383,13 +6142,20 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5383
6142
|
} else if (child.stderr && typeof child.stderr.on === "function" && logStream) {
|
|
5384
6143
|
child.stderr.on("data", (chunk) => logStream.write(chunk));
|
|
5385
6144
|
}
|
|
6145
|
+
outputCapture.attach(child.stdout);
|
|
6146
|
+
outputCapture.attach(child.stderr);
|
|
5386
6147
|
} else if (child.stderr && typeof child.stderr.on === "function") {
|
|
6148
|
+
outputCapture.attach(child.stderr);
|
|
5387
6149
|
// Capture any error output emitted by the tmux client itself so
|
|
5388
6150
|
// problems during session creation surface in daemon logs.
|
|
5389
6151
|
child.stderr.on("data", (chunk) => {
|
|
5390
6152
|
const text = chunk?.toString?.("utf8") ?? String(chunk ?? "");
|
|
5391
6153
|
if (text.trim()) {
|
|
5392
|
-
logError(
|
|
6154
|
+
logError(
|
|
6155
|
+
// Redact: this is the fire's raw stderr, which can carry the
|
|
6156
|
+
// handoff share token (argv echo) and provider/agent keys.
|
|
6157
|
+
`tmux(${tmuxSession}) stderr: ${redactSecretsForLogs(text.trim(), [AGENT_TOKEN])}`,
|
|
6158
|
+
);
|
|
5393
6159
|
}
|
|
5394
6160
|
});
|
|
5395
6161
|
}
|
|
@@ -5405,6 +6171,20 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5405
6171
|
child.on("exit", (code, signal) => {
|
|
5406
6172
|
const active = activeTaskProcesses.get(taskId);
|
|
5407
6173
|
|
|
6174
|
+
// This spawn was a tmux client but the record is gone: the reaper
|
|
6175
|
+
// (or a stop) already retired this task and, in the reaper's case,
|
|
6176
|
+
// already published its terminal status. Falling through would
|
|
6177
|
+
// delete whatever record a restart has since installed and report a
|
|
6178
|
+
// second, contradictory status — `shouldDaemonReportFireChildTerminal
|
|
6179
|
+
// Status(undefined)` is true, so an `exit(0)` arriving late would
|
|
6180
|
+
// announce COMPLETED over the reaper's verdict. Nothing left to do.
|
|
6181
|
+
if (tmuxSession && !active) {
|
|
6182
|
+
log(
|
|
6183
|
+
`tmux client for task ${taskId} exited after its record was retired (code=${code}, signal=${signal || "null"}); nothing to report`,
|
|
6184
|
+
);
|
|
6185
|
+
return;
|
|
6186
|
+
}
|
|
6187
|
+
|
|
5408
6188
|
// In tmux mode the `tmux new-session -d` client always exits
|
|
5409
6189
|
// shortly after launching the Fire session. A clean exit (code 0,
|
|
5410
6190
|
// no signal) just means the session was successfully created and
|
|
@@ -5468,6 +6248,18 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5468
6248
|
? "completed"
|
|
5469
6249
|
: `exited with code ${code}`;
|
|
5470
6250
|
|
|
6251
|
+
const lifetimeMs = Date.now() - spawnedAtMs;
|
|
6252
|
+
const outputTail = status === "KILLED" ? outputCapture.tail() : "";
|
|
6253
|
+
if (status === "KILLED") {
|
|
6254
|
+
logError(
|
|
6255
|
+
`[create-spawn] abnormal exit task=${taskId} backend=${selectedBackend} ` +
|
|
6256
|
+
`cwd=${taskDir} tmux=${tmuxSession || "none"} exit=${code ?? "null"} ` +
|
|
6257
|
+
`signal=${signal || "null"} lifetime_ms=${lifetimeMs} log=${logPath} ` +
|
|
6258
|
+
`output_tail=${outputTail ? JSON.stringify(outputTail) : "<empty>"}`,
|
|
6259
|
+
);
|
|
6260
|
+
}
|
|
6261
|
+
const reportedSummary = outputTail ? `${summary}: ${outputTail}` : summary;
|
|
6262
|
+
|
|
5471
6263
|
if (!suppressExitStatusReport && shouldDaemonReportFireChildTerminalStatus(active)) {
|
|
5472
6264
|
client
|
|
5473
6265
|
.sendJson({
|
|
@@ -5476,7 +6268,8 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5476
6268
|
task_id: taskId,
|
|
5477
6269
|
project_id: projectId,
|
|
5478
6270
|
status,
|
|
5479
|
-
summary,
|
|
6271
|
+
summary: reportedSummary,
|
|
6272
|
+
status_event_id: randomUUID(),
|
|
5480
6273
|
},
|
|
5481
6274
|
})
|
|
5482
6275
|
.catch((err) => {
|
|
@@ -5594,8 +6387,17 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5594
6387
|
// Probe the actual session and clean up stale entries on demand so
|
|
5595
6388
|
// the restart gating below reflects reality, not the stale record.
|
|
5596
6389
|
if (activeTarget?.tmuxMode && activeTarget.tmuxSession) {
|
|
5597
|
-
|
|
5598
|
-
|
|
6390
|
+
// As in reclaim: only a conclusive answer may clear the record. This
|
|
6391
|
+
// gate is what stops a double spawn, so trusting an inconclusive probe
|
|
6392
|
+
// would start a second fire alongside a live one in the same worktree.
|
|
6393
|
+
const { alive: sessionAlive, conclusive } = await probeTmuxSession(
|
|
6394
|
+
activeTarget.tmuxSession,
|
|
6395
|
+
);
|
|
6396
|
+
if (!sessionAlive && !conclusive) {
|
|
6397
|
+
logError(
|
|
6398
|
+
`Could not determine whether tmux session ${activeTarget.tmuxSession} for task ${normalizedTargetTaskId} is alive; keeping the existing record before restart`,
|
|
6399
|
+
);
|
|
6400
|
+
} else if (
|
|
5599
6401
|
!sessionAlive &&
|
|
5600
6402
|
activeTaskProcesses.get(normalizedTargetTaskId) === activeTarget
|
|
5601
6403
|
) {
|
|
@@ -5915,6 +6717,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5915
6717
|
|
|
5916
6718
|
const env = {
|
|
5917
6719
|
...stripPtyTaskScopedEnv(process.env),
|
|
6720
|
+
...materializedConductorPathEnv,
|
|
5918
6721
|
PWD: taskDir,
|
|
5919
6722
|
CONDUCTOR_PROJECT_ID: normalizedProjectId,
|
|
5920
6723
|
CONDUCTOR_TASK_ID: normalizedTargetTaskId,
|
|
@@ -5922,9 +6725,6 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5922
6725
|
...(cliCommand ? { CONDUCTOR_CLI_COMMAND: cliCommand } : {}),
|
|
5923
6726
|
};
|
|
5924
6727
|
env.CONDUCTOR_RESUME_CWD = resolvedResumeCwd;
|
|
5925
|
-
if (config.CONFIG_FILE) {
|
|
5926
|
-
env.CONDUCTOR_CONFIG = config.CONFIG_FILE;
|
|
5927
|
-
}
|
|
5928
6728
|
if (AGENT_TOKEN) {
|
|
5929
6729
|
env.CONDUCTOR_AGENT_TOKEN = AGENT_TOKEN;
|
|
5930
6730
|
}
|
|
@@ -5932,13 +6732,38 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5932
6732
|
env.CONDUCTOR_BACKEND_URL = BACKEND_HTTP;
|
|
5933
6733
|
}
|
|
5934
6734
|
|
|
5935
|
-
|
|
6735
|
+
// Two hazards below, one root cause: an in-place restart deliberately
|
|
6736
|
+
// re-uses the previous run's working directory, so ANY artifact the last
|
|
6737
|
+
// run left there can be mistaken for this run's own state. Both guards
|
|
6738
|
+
// must run before the spawn.
|
|
6739
|
+
//
|
|
6740
|
+
// (1) The fire's durable upstream outbox lives in that directory. If the
|
|
6741
|
+
// prior run was stopped while its websocket was down, its undelivered
|
|
6742
|
+
// terminal `task_status_update` (KILLED/COMPLETED) is still on disk — and
|
|
6743
|
+
// the run we are about to spawn would flush it during startup, marking the
|
|
6744
|
+
// task killed seconds after resuming and then getting shot down by the
|
|
6745
|
+
// server's `stop_task(task_already_killed)` reply. Purge those superseded
|
|
6746
|
+
// events before the new fire can pick them up.
|
|
6747
|
+
dropSupersededTerminalStatusEvents(taskDir, normalizedTargetTaskId);
|
|
6748
|
+
|
|
6749
|
+
// (2) The log file is opened with `flags:"a"` and re-used as well. Sample
|
|
6750
|
+
// its size first so it is a true "everything after this is ours"
|
|
6751
|
+
// watermark; without it the reaper reads the PREVIOUS run's exit marker.
|
|
6752
|
+
const logStartOffset = readLogSize(logPath);
|
|
6753
|
+
|
|
6754
|
+
const { child, tmuxSession, exitMarkerToken } = spawnFireProcess({
|
|
5936
6755
|
taskId: normalizedTargetTaskId,
|
|
5937
6756
|
args,
|
|
5938
6757
|
env,
|
|
5939
6758
|
cwd: taskDir,
|
|
5940
6759
|
logPath,
|
|
5941
6760
|
});
|
|
6761
|
+
// Bounded tail of the child's output + a spawn timestamp, so an abnormal
|
|
6762
|
+
// exit can report *why* it died and how long it survived. `logPath` lets
|
|
6763
|
+
// the capture recover the fire's own output in tmux mode, where the
|
|
6764
|
+
// daemon's child is only the `tmux new-session` client.
|
|
6765
|
+
const outputCapture = createChildOutputCapture({ logPath, logStartOffset });
|
|
6766
|
+
const spawnedAtMs = Date.now();
|
|
5942
6767
|
|
|
5943
6768
|
let logStream;
|
|
5944
6769
|
try {
|
|
@@ -5969,6 +6794,11 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5969
6794
|
child,
|
|
5970
6795
|
projectId: normalizedProjectId,
|
|
5971
6796
|
logPath,
|
|
6797
|
+
// See the create_task path: watermark for tail reads / exit-marker
|
|
6798
|
+
// scans, the marker nonce, and the reaper's grace-period anchor.
|
|
6799
|
+
logStartOffset,
|
|
6800
|
+
exitMarkerToken,
|
|
6801
|
+
spawnedAtMs,
|
|
5972
6802
|
stopForceKillTimer: null,
|
|
5973
6803
|
managedByFireBridge: true,
|
|
5974
6804
|
tmuxSession: tmuxSession || null,
|
|
@@ -5990,17 +6820,32 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5990
6820
|
} else if (child.stderr && typeof child.stderr.on === "function" && logStream) {
|
|
5991
6821
|
child.stderr.on("data", (chunk) => logStream.write(chunk));
|
|
5992
6822
|
}
|
|
6823
|
+
// Capture independently of logStream: when the log file cannot be
|
|
6824
|
+
// opened (createWriteStream threw) the piping above is skipped
|
|
6825
|
+
// entirely, which is exactly when we most need the tail.
|
|
6826
|
+
outputCapture.attach(child.stdout);
|
|
6827
|
+
outputCapture.attach(child.stderr);
|
|
5993
6828
|
} else if (child.stderr && typeof child.stderr.on === "function") {
|
|
6829
|
+
outputCapture.attach(child.stderr);
|
|
5994
6830
|
child.stderr.on("data", (chunk) => {
|
|
5995
6831
|
const text = chunk?.toString?.("utf8") ?? String(chunk ?? "");
|
|
5996
6832
|
if (text.trim()) {
|
|
5997
|
-
logError(
|
|
6833
|
+
logError(
|
|
6834
|
+
// Redact: this is the fire's raw stderr, which can carry the
|
|
6835
|
+
// handoff share token (argv echo) and provider/agent keys.
|
|
6836
|
+
`tmux(${tmuxSession}) stderr: ${redactSecretsForLogs(text.trim(), [AGENT_TOKEN])}`,
|
|
6837
|
+
);
|
|
5998
6838
|
}
|
|
5999
6839
|
});
|
|
6000
6840
|
}
|
|
6001
6841
|
|
|
6002
6842
|
child.on("error", (err) => {
|
|
6003
|
-
logError(
|
|
6843
|
+
logError(
|
|
6844
|
+
`[fork-spawn] spawn error task=${normalizedTargetTaskId} mode=${normalizedMode} ` +
|
|
6845
|
+
`backend=${selectedBackend} cwd=${taskDir} tmux=${tmuxSession || "none"}: ${
|
|
6846
|
+
maskErrorForLogs(err)?.message || err
|
|
6847
|
+
}`,
|
|
6848
|
+
);
|
|
6004
6849
|
if (logStream) {
|
|
6005
6850
|
const ts = new Date().toLocaleString("sv-SE", { timeZone: "Asia/Shanghai" }).replace(" ", "T");
|
|
6006
6851
|
logStream.write(`[daemon ${ts}] spawn error: ${err.message}\n`);
|
|
@@ -6010,6 +6855,16 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6010
6855
|
child.on("exit", (code, signal) => {
|
|
6011
6856
|
const active = activeTaskProcesses.get(normalizedTargetTaskId);
|
|
6012
6857
|
|
|
6858
|
+
// See the create path: a tmux client exiting after its record was
|
|
6859
|
+
// retired must stay silent, or it would both clobber a replacement
|
|
6860
|
+
// record and publish a status contradicting the reaper's verdict.
|
|
6861
|
+
if (tmuxSession && !active) {
|
|
6862
|
+
log(
|
|
6863
|
+
`tmux client for restarted task ${normalizedTargetTaskId} exited after its record was retired (code=${code}, signal=${signal || "null"}); nothing to report`,
|
|
6864
|
+
);
|
|
6865
|
+
return;
|
|
6866
|
+
}
|
|
6867
|
+
|
|
6013
6868
|
// In tmux mode the `tmux new-session -d` client always exits soon
|
|
6014
6869
|
// after launching the session. A clean exit (code 0, no signal) means
|
|
6015
6870
|
// Fire is now running detached under the tmux server — keep the task
|
|
@@ -6062,6 +6917,26 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6062
6917
|
? "completed"
|
|
6063
6918
|
: `exited with code ${code}`;
|
|
6064
6919
|
|
|
6920
|
+
// Diagnostics for an abnormal exit. This is the line that makes a
|
|
6921
|
+
// "branch task died instantly" incident debuggable: it names the mode,
|
|
6922
|
+
// backend, cwd, tmux session, exit code/signal, how long the child
|
|
6923
|
+
// survived, where its log is, and the tail of what it actually printed.
|
|
6924
|
+
const lifetimeMs = Date.now() - spawnedAtMs;
|
|
6925
|
+
const outputTail = outputCapture.tail();
|
|
6926
|
+
if (status === "KILLED") {
|
|
6927
|
+
logError(
|
|
6928
|
+
`[fork-spawn] abnormal exit task=${normalizedTargetTaskId} mode=${normalizedMode} ` +
|
|
6929
|
+
`backend=${selectedBackend} cwd=${taskDir} tmux=${tmuxSession || "none"} ` +
|
|
6930
|
+
`exit=${code ?? "null"} signal=${signal || "null"} lifetime_ms=${lifetimeMs} ` +
|
|
6931
|
+
`log=${logPath} output_tail=${outputTail ? JSON.stringify(outputTail) : "<empty>"}`,
|
|
6932
|
+
);
|
|
6933
|
+
}
|
|
6934
|
+
// Surface the same tail in the status summary so it reaches the backend
|
|
6935
|
+
// (task_status_events.summary) and the UI, instead of a bare
|
|
6936
|
+
// "exited with code 1" that explains nothing.
|
|
6937
|
+
const reportedSummary =
|
|
6938
|
+
status === "KILLED" && outputTail ? `${summary}: ${outputTail}` : summary;
|
|
6939
|
+
|
|
6065
6940
|
const shouldReportTerminalStatus =
|
|
6066
6941
|
!suppressExitStatusReport &&
|
|
6067
6942
|
(!acceptedRestartAckSent || shouldDaemonReportFireChildTerminalStatus(active));
|
|
@@ -6076,7 +6951,9 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6076
6951
|
task_id: normalizedTargetTaskId,
|
|
6077
6952
|
project_id: normalizedProjectId,
|
|
6078
6953
|
status,
|
|
6079
|
-
summary,
|
|
6954
|
+
summary: reportedSummary,
|
|
6955
|
+
// Idempotency key — see the note in reportRestartFailure.
|
|
6956
|
+
status_event_id: randomUUID(),
|
|
6080
6957
|
},
|
|
6081
6958
|
})
|
|
6082
6959
|
.catch((err) => {
|