@love-moon/conductor-cli 0.11.0 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +97 -0
- package/bin/conductor-config.js +4 -2
- package/bin/conductor-daemon-update.js +55 -0
- package/package.json +5 -5
- package/src/daemon-update.js +614 -0
- package/src/daemon.js +676 -29
- package/src/fire-session-registry.js +136 -0
- package/src/guest-daemon.js +8 -1
package/src/daemon.js
CHANGED
|
@@ -24,6 +24,13 @@ import {
|
|
|
24
24
|
resolveConductorConfigPath,
|
|
25
25
|
resolveConductorHome,
|
|
26
26
|
} from "./conductor-paths.js";
|
|
27
|
+
import {
|
|
28
|
+
deleteFireSessionRecord,
|
|
29
|
+
pruneFireSessionRecords,
|
|
30
|
+
readFireSessionRecord,
|
|
31
|
+
resolveFireSessionRegistryDir,
|
|
32
|
+
writeFireSessionRecord,
|
|
33
|
+
} from "./fire-session-registry.js";
|
|
27
34
|
import { createAiManagerHandlers, handleAiManagerRequest } from "./ai-manager-handlers.js";
|
|
28
35
|
import {
|
|
29
36
|
CUSTOM_COMMANDS_CAPABILITY,
|
|
@@ -35,6 +42,12 @@ import {
|
|
|
35
42
|
createRemoteExecHandlers,
|
|
36
43
|
handleRemoteExecRequest,
|
|
37
44
|
} from "./remote-exec-handlers.js";
|
|
45
|
+
import {
|
|
46
|
+
UPDATE_DAEMON_CAPABILITY,
|
|
47
|
+
createDaemonUpdateHandlers,
|
|
48
|
+
handleUpdateDaemonRequest,
|
|
49
|
+
resolveDaemonUpdatePaths,
|
|
50
|
+
} from "./daemon-update.js";
|
|
38
51
|
import { resolveResumeContext } from "./fire/resume.js";
|
|
39
52
|
import {
|
|
40
53
|
filterRuntimeSupportedAllowCliList,
|
|
@@ -106,6 +119,7 @@ const CLI_PATH = path.resolve(PACKAGE_ROOT, "bin", "conductor-fire.js");
|
|
|
106
119
|
// the host daemon, so guests always match the installed version and there is
|
|
107
120
|
// no second update path to keep in sync.
|
|
108
121
|
const DAEMON_LAUNCHER_PATH = path.resolve(PACKAGE_ROOT, "bin", "conductor-daemon.js");
|
|
122
|
+
const DAEMON_UPDATE_SCRIPT_PATH = path.resolve(PACKAGE_ROOT, "bin", "conductor-daemon-update.js");
|
|
109
123
|
const CLI_VERSION = (() => {
|
|
110
124
|
try {
|
|
111
125
|
return JSON.parse(fs.readFileSync(path.join(PACKAGE_ROOT, "package.json"), "utf-8")).version;
|
|
@@ -658,10 +672,102 @@ export function buildFireSpawnArgs({ selectedBackend, initialContent, launchConf
|
|
|
658
672
|
if (effectivePrefill) {
|
|
659
673
|
args.push("--prefill", effectivePrefill);
|
|
660
674
|
}
|
|
675
|
+
const resumeSessionId =
|
|
676
|
+
normalizeOptionalString(launchConfig?.resumeSessionId) ||
|
|
677
|
+
normalizeOptionalString(launchConfig?.resume_session_id);
|
|
678
|
+
if (resumeSessionId) {
|
|
679
|
+
args.push("--resume", resumeSessionId);
|
|
680
|
+
}
|
|
661
681
|
args.push("--");
|
|
662
682
|
return args;
|
|
663
683
|
}
|
|
664
684
|
|
|
685
|
+
// Backends whose local session stores the ai-sdk resume module can enumerate.
|
|
686
|
+
// Kept in sync with the server side of the `list_backend_sessions` RPC.
|
|
687
|
+
const LISTABLE_SESSION_BACKENDS = ["claude", "codex", "kimi"];
|
|
688
|
+
const DEFAULT_BACKEND_SESSION_LIMIT = 20;
|
|
689
|
+
const MAX_BACKEND_SESSION_LIMIT = 50;
|
|
690
|
+
|
|
691
|
+
// Lazy so a daemon running against an older ai-sdk build only fails when the
|
|
692
|
+
// RPC is actually invoked (surfaced as a per-backend error), not at load time.
|
|
693
|
+
async function listSessionsForBackendFromSdk(backendType, options) {
|
|
694
|
+
let sdk = await import("@love-moon/ai-sdk");
|
|
695
|
+
if (typeof sdk.listSessionsForBackend !== "function") {
|
|
696
|
+
// The root index can lag behind the resume module the function lives in;
|
|
697
|
+
// fall back to that entry point before declaring the sdk too old.
|
|
698
|
+
sdk = await import("@love-moon/ai-sdk/dist/resume/index.js").catch(() => sdk);
|
|
699
|
+
}
|
|
700
|
+
if (typeof sdk.listSessionsForBackend !== "function") {
|
|
701
|
+
throw new Error("Session listing is not supported by the installed ai-sdk");
|
|
702
|
+
}
|
|
703
|
+
return sdk.listSessionsForBackend(backendType, options);
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
/**
|
|
707
|
+
* Pure aggregation behind the `list_backend_sessions` RPC: intersect the
|
|
708
|
+
* requested backends with what this daemon supports, list each backend's
|
|
709
|
+
* sessions, and merge them newest-first. A single backend failing lands in
|
|
710
|
+
* `errors` without affecting the others. Exported so tests can drive it with
|
|
711
|
+
* a stubbed `listSessions`.
|
|
712
|
+
*/
|
|
713
|
+
export async function collectBackendSessions({
|
|
714
|
+
requestedBackends = null,
|
|
715
|
+
supportedBackends = [],
|
|
716
|
+
limit,
|
|
717
|
+
listSessions = listSessionsForBackendFromSdk,
|
|
718
|
+
} = {}) {
|
|
719
|
+
const numericLimit = Number(limit);
|
|
720
|
+
const effectiveLimit =
|
|
721
|
+
Number.isFinite(numericLimit) && numericLimit > 0
|
|
722
|
+
? Math.min(Math.floor(numericLimit), MAX_BACKEND_SESSION_LIMIT)
|
|
723
|
+
: DEFAULT_BACKEND_SESSION_LIMIT;
|
|
724
|
+
const listableBackends = LISTABLE_SESSION_BACKENDS.filter((backend) => supportedBackends.includes(backend));
|
|
725
|
+
const errors = [];
|
|
726
|
+
let backends;
|
|
727
|
+
if (Array.isArray(requestedBackends)) {
|
|
728
|
+
backends = [];
|
|
729
|
+
for (const raw of requestedBackends) {
|
|
730
|
+
const backend = typeof raw === "string" ? raw.trim().toLowerCase() : "";
|
|
731
|
+
if (!backend || backends.includes(backend)) {
|
|
732
|
+
continue;
|
|
733
|
+
}
|
|
734
|
+
if (!listableBackends.includes(backend)) {
|
|
735
|
+
errors.push({ backend, message: `Backend "${backend}" does not support session listing on this daemon` });
|
|
736
|
+
continue;
|
|
737
|
+
}
|
|
738
|
+
backends.push(backend);
|
|
739
|
+
}
|
|
740
|
+
} else {
|
|
741
|
+
backends = listableBackends;
|
|
742
|
+
}
|
|
743
|
+
const sessions = [];
|
|
744
|
+
for (const backend of backends) {
|
|
745
|
+
try {
|
|
746
|
+
const listed = await listSessions(backend, { limit: effectiveLimit });
|
|
747
|
+
for (const entry of Array.isArray(listed) ? listed : []) {
|
|
748
|
+
const sessionId = normalizeOptionalString(entry?.sessionId);
|
|
749
|
+
if (!sessionId) {
|
|
750
|
+
continue;
|
|
751
|
+
}
|
|
752
|
+
sessions.push({
|
|
753
|
+
backend,
|
|
754
|
+
session_id: sessionId,
|
|
755
|
+
session_file_path: normalizeOptionalString(entry?.sessionFilePath),
|
|
756
|
+
cwd: normalizeOptionalString(entry?.cwd),
|
|
757
|
+
title: normalizeOptionalString(entry?.title),
|
|
758
|
+
updated_at: Number.isFinite(entry?.updatedAt) ? new Date(entry.updatedAt).toISOString() : null,
|
|
759
|
+
});
|
|
760
|
+
}
|
|
761
|
+
} catch (error) {
|
|
762
|
+
errors.push({ backend, message: error?.message || String(error) });
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
// ISO 8601 strings compare lexicographically; entries without a timestamp
|
|
766
|
+
// sort last.
|
|
767
|
+
sessions.sort((a, b) => (b.updated_at || "").localeCompare(a.updated_at || ""));
|
|
768
|
+
return { sessions, errors };
|
|
769
|
+
}
|
|
770
|
+
|
|
665
771
|
function normalizeBooleanFlag(value) {
|
|
666
772
|
if (typeof value === "boolean") {
|
|
667
773
|
return value;
|
|
@@ -1187,6 +1293,16 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1187
1293
|
log("Fire tmux mode enabled: each Fire process will run in a detached tmux session");
|
|
1188
1294
|
}
|
|
1189
1295
|
|
|
1296
|
+
// Shared by every Fire tmux session name. Also the discriminator that lets
|
|
1297
|
+
// startup adoption tell our sessions apart from the user's own tmux work.
|
|
1298
|
+
const FIRE_TMUX_SESSION_NAMESPACE = "conductor-fire-";
|
|
1299
|
+
|
|
1300
|
+
// Where the per-session hand-off records live. See fire-session-registry.js
|
|
1301
|
+
// for why a fresh daemon cannot reconstruct them from the session alone.
|
|
1302
|
+
const FIRE_SESSION_REGISTRY_DIR = resolveFireSessionRegistryDir(
|
|
1303
|
+
materializedConductorPathEnv.CONDUCTOR_HOME,
|
|
1304
|
+
);
|
|
1305
|
+
|
|
1190
1306
|
// Single-quote a value so it can be embedded inside a `bash -c '...'`
|
|
1191
1307
|
// command. Embedded single-quotes are escaped via the standard `'\\''`
|
|
1192
1308
|
// sequence.
|
|
@@ -1211,7 +1327,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1211
1327
|
.replace(/[^a-zA-Z0-9_-]/g, "_")
|
|
1212
1328
|
.replace(/^_+|_+$/g, "")
|
|
1213
1329
|
.slice(0, 32) || "task";
|
|
1214
|
-
return
|
|
1330
|
+
return `${FIRE_TMUX_SESSION_NAMESPACE}${safe}-`;
|
|
1215
1331
|
}
|
|
1216
1332
|
|
|
1217
1333
|
// Build a unique tmux session name for a Fire spawn.
|
|
@@ -1754,7 +1870,14 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1754
1870
|
// would be misread as "died hard" — reporting every clean finish as
|
|
1755
1871
|
// KILLED. Staying silent restores the old blind spot for this task only,
|
|
1756
1872
|
// which is strictly better than manufacturing a wrong cause of death.
|
|
1757
|
-
|
|
1873
|
+
//
|
|
1874
|
+
// The one exception is a Fire adopted without a hand-off record. There
|
|
1875
|
+
// the silence is not a blind spot we are preserving but one we would be
|
|
1876
|
+
// creating: nothing else watches an adopted session, so staying quiet
|
|
1877
|
+
// parks the task at `running` forever. Speak, and let the backend
|
|
1878
|
+
// pre-check below stop us from downgrading a status fire already
|
|
1879
|
+
// published for itself.
|
|
1880
|
+
if (!record.logPath && !record.adoptedWithoutMetadata) {
|
|
1758
1881
|
logError(
|
|
1759
1882
|
`Cannot classify the end of task ${taskId}: no log path on the active record, so no exit marker exists`,
|
|
1760
1883
|
);
|
|
@@ -1767,7 +1890,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1767
1890
|
// that absence as "no exit marker" would report a green task as KILLED
|
|
1768
1891
|
// with an invented "killed or OOM" cause. An empty-but-present file is
|
|
1769
1892
|
// different: the fire really did die before writing anything.
|
|
1770
|
-
if (readLogFileSize(record.logPath) === null) {
|
|
1893
|
+
if (!record.adoptedWithoutMetadata && readLogFileSize(record.logPath) === null) {
|
|
1771
1894
|
logError(
|
|
1772
1895
|
`Cannot classify the end of task ${taskId}: log file ${record.logPath} is missing, ` +
|
|
1773
1896
|
`so the absence of an exit marker proves nothing`,
|
|
@@ -1775,7 +1898,9 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1775
1898
|
return;
|
|
1776
1899
|
}
|
|
1777
1900
|
|
|
1778
|
-
const rawTail =
|
|
1901
|
+
const rawTail = record.adoptedWithoutMetadata
|
|
1902
|
+
? ""
|
|
1903
|
+
: readLogTailRaw(record.logPath, { minOffset: record.logStartOffset || 0 });
|
|
1779
1904
|
// Search the file for the marker rather than hoping it is still inside
|
|
1780
1905
|
// the tail window — see `findFireExitCode` for why a shared log makes the
|
|
1781
1906
|
// tail unreliable. The tail is still what feeds the human-readable
|
|
@@ -1799,8 +1924,9 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1799
1924
|
// `latest_status_summary` with a word that says nothing. Naming *how* we
|
|
1800
1925
|
// found out keeps the event worth its row: it tells the next reader that
|
|
1801
1926
|
// fire never published its own status.
|
|
1802
|
-
const baseSummary =
|
|
1803
|
-
|
|
1927
|
+
const baseSummary = record.adoptedWithoutMetadata
|
|
1928
|
+
? "fire tmux session ended; it was adopted from a previous daemon without a hand-off record, so the exit could not be classified"
|
|
1929
|
+
: exitCode === null
|
|
1804
1930
|
? "fire tmux session disappeared without an exit marker (killed or OOM)"
|
|
1805
1931
|
: exitCode === 0
|
|
1806
1932
|
? "completed; detected by the daemon after the tmux session ended (fire never reported it)"
|
|
@@ -1811,6 +1937,19 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1811
1937
|
// Ask the backend before overwriting anything: fire normally reports its
|
|
1812
1938
|
// own terminal status and we must not downgrade a success.
|
|
1813
1939
|
const backendStatus = await fetchBackendTaskStatus(taskId);
|
|
1940
|
+
// For a record adopted without metadata there is no exit marker to read,
|
|
1941
|
+
// so the verdict below is unconditionally KILLED and this probe is the
|
|
1942
|
+
// ONLY thing standing between a task that finished cleanly and a
|
|
1943
|
+
// fabricated failure. `fetchBackendTaskStatus` returns null both for "not
|
|
1944
|
+
// terminal" and for "could not ask", so a backend blip during a daemon
|
|
1945
|
+
// upgrade would mark a green task killed. Require a real answer.
|
|
1946
|
+
if (!backendStatus && record.adoptedWithoutMetadata) {
|
|
1947
|
+
logError(
|
|
1948
|
+
`Cannot classify the end of adopted task ${taskId}: the backend did not answer, ` +
|
|
1949
|
+
`and without a hand-off record its exit code is unknowable. Staying silent.`,
|
|
1950
|
+
);
|
|
1951
|
+
return;
|
|
1952
|
+
}
|
|
1814
1953
|
if (backendStatus && TERMINAL_BACKEND_TASK_STATUSES.has(backendStatus)) {
|
|
1815
1954
|
log(
|
|
1816
1955
|
`Tmux session ${record.tmuxSession} for task ${taskId} ended; backend already ${backendStatus}, no report needed`,
|
|
@@ -1925,6 +2064,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1925
2064
|
record.stopForceKillTimer = null;
|
|
1926
2065
|
}
|
|
1927
2066
|
activeTaskProcesses.delete(taskId);
|
|
2067
|
+
forgetFireSessionRecord(record);
|
|
1928
2068
|
// Report *after* dropping the local entry so a slow backend round
|
|
1929
2069
|
// trip can't make a concurrent stop_task wait on us.
|
|
1930
2070
|
await reportDeadTmuxSessionStatus(taskId, record);
|
|
@@ -1956,22 +2096,27 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1956
2096
|
}
|
|
1957
2097
|
}
|
|
1958
2098
|
|
|
1959
|
-
// List every tmux session name visible to the current user.
|
|
1960
|
-
//
|
|
1961
|
-
//
|
|
2099
|
+
// List every tmux session name visible to the current user.
|
|
2100
|
+
//
|
|
2101
|
+
// Returns `{ sessions, conclusive }`. The flag is not decoration: "tmux says
|
|
2102
|
+
// there are no sessions" and "we could not ask tmux" are the same empty
|
|
2103
|
+
// array, and callers that kill on the strength of an empty list would turn
|
|
2104
|
+
// one flaky probe into a mass kill of live Fires — the very regression the
|
|
2105
|
+
// adoption path exists to prevent. `probeTmuxSession` draws the same
|
|
2106
|
+
// distinction for the same reason. Subject to TMUX_PROBE_TIMEOUT_MS.
|
|
1962
2107
|
function listAllTmuxSessions() {
|
|
1963
2108
|
return new Promise((resolve) => {
|
|
1964
2109
|
let settled = false;
|
|
1965
2110
|
let stdout = "";
|
|
1966
2111
|
let timer = null;
|
|
1967
|
-
const settle = (lines) => {
|
|
2112
|
+
const settle = (lines, conclusive) => {
|
|
1968
2113
|
if (settled) return;
|
|
1969
2114
|
settled = true;
|
|
1970
2115
|
if (timer) {
|
|
1971
2116
|
clearTimeout(timer);
|
|
1972
2117
|
timer = null;
|
|
1973
2118
|
}
|
|
1974
|
-
resolve(lines);
|
|
2119
|
+
resolve({ sessions: lines, conclusive });
|
|
1975
2120
|
};
|
|
1976
2121
|
try {
|
|
1977
2122
|
const probe = spawnFn("tmux", ["list-sessions", "-F", "#{session_name}"], {
|
|
@@ -1989,14 +2134,16 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
1989
2134
|
.split("\n")
|
|
1990
2135
|
.map((s) => s.trim())
|
|
1991
2136
|
.filter(Boolean),
|
|
2137
|
+
true,
|
|
1992
2138
|
);
|
|
1993
2139
|
} else {
|
|
1994
|
-
// Non-zero exit
|
|
1995
|
-
//
|
|
1996
|
-
settle([]);
|
|
2140
|
+
// Non-zero exit means "no server running" / "no sessions". tmux
|
|
2141
|
+
// answered, so this is a real, conclusive empty list.
|
|
2142
|
+
settle([], true);
|
|
1997
2143
|
}
|
|
1998
2144
|
});
|
|
1999
|
-
|
|
2145
|
+
// Could not run tmux at all — no information either way.
|
|
2146
|
+
probe.on("error", () => settle([], false));
|
|
2000
2147
|
timer = setTimeout(() => {
|
|
2001
2148
|
try {
|
|
2002
2149
|
if (typeof probe.kill === "function") {
|
|
@@ -2006,13 +2153,13 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
2006
2153
|
// ignore; we already settled
|
|
2007
2154
|
}
|
|
2008
2155
|
logError(`tmux list-sessions probe timed out after ${TMUX_PROBE_TIMEOUT_MS}ms`);
|
|
2009
|
-
settle([]);
|
|
2156
|
+
settle([], false);
|
|
2010
2157
|
}, TMUX_PROBE_TIMEOUT_MS);
|
|
2011
2158
|
if (typeof timer.unref === "function") {
|
|
2012
2159
|
timer.unref();
|
|
2013
2160
|
}
|
|
2014
2161
|
} catch {
|
|
2015
|
-
settle([]);
|
|
2162
|
+
settle([], false);
|
|
2016
2163
|
}
|
|
2017
2164
|
});
|
|
2018
2165
|
}
|
|
@@ -2027,17 +2174,288 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
2027
2174
|
async function killTmuxSessionsForDeletedTask(taskId) {
|
|
2028
2175
|
if (!FIRE_TMUX_MODE_ACTIVE || !taskId) return 0;
|
|
2029
2176
|
const prefix = buildFireTmuxSessionPrefix(taskId);
|
|
2030
|
-
const sessions = await listAllTmuxSessions();
|
|
2177
|
+
const { sessions } = await listAllTmuxSessions();
|
|
2031
2178
|
let killed = 0;
|
|
2032
2179
|
for (const name of sessions) {
|
|
2033
2180
|
if (name.startsWith(prefix)) {
|
|
2034
2181
|
log(`Killing orphaned tmux session ${name} for deleted task ${taskId}`);
|
|
2035
2182
|
killFireTmuxSession(name);
|
|
2183
|
+
forgetFireSessionRecord({ tmuxSession: name });
|
|
2036
2184
|
killed += 1;
|
|
2037
2185
|
}
|
|
2038
2186
|
}
|
|
2039
2187
|
return killed;
|
|
2040
2188
|
}
|
|
2189
|
+
|
|
2190
|
+
// --- tmux Fire hand-off across daemon restarts ---------------------------
|
|
2191
|
+
//
|
|
2192
|
+
// Shutdown in tmux mode deliberately leaves Fires running, but a fresh
|
|
2193
|
+
// daemon starts with an empty `activeTaskProcesses` by definition. Both
|
|
2194
|
+
// stale-task sweeps decide "this task is dead" from that map alone, so
|
|
2195
|
+
// without the adoption path below every survivor of the hand-off is
|
|
2196
|
+
// PATCHed to `killed` seconds after the new daemon connects — destroying
|
|
2197
|
+
// the exact work the hand-off exists to preserve. Regression history:
|
|
2198
|
+
// 2026-07-23, 2026-07-31, 2026-08-31.
|
|
2199
|
+
|
|
2200
|
+
// Persist what a successor daemon needs to adopt this session. Best-effort:
|
|
2201
|
+
// failing here costs post-restart fidelity, never the running Fire.
|
|
2202
|
+
function rememberFireSessionRecord(taskId, record) {
|
|
2203
|
+
if (!FIRE_TMUX_MODE_ACTIVE || !record?.tmuxSession) return;
|
|
2204
|
+
try {
|
|
2205
|
+
writeFireSessionRecord(FIRE_SESSION_REGISTRY_DIR, {
|
|
2206
|
+
taskId,
|
|
2207
|
+
projectId: record.projectId,
|
|
2208
|
+
tmuxSession: record.tmuxSession,
|
|
2209
|
+
logPath: record.logPath,
|
|
2210
|
+
logStartOffset: record.logStartOffset,
|
|
2211
|
+
exitMarkerToken: record.exitMarkerToken,
|
|
2212
|
+
spawnedAtMs: record.spawnedAtMs,
|
|
2213
|
+
daemonName: AGENT_NAME,
|
|
2214
|
+
});
|
|
2215
|
+
} catch (error) {
|
|
2216
|
+
logError(
|
|
2217
|
+
`Failed to persist tmux hand-off record for task ${taskId}: ${error?.message || error}`,
|
|
2218
|
+
);
|
|
2219
|
+
}
|
|
2220
|
+
}
|
|
2221
|
+
|
|
2222
|
+
// Drop the hand-off record once the session it describes is provably gone.
|
|
2223
|
+
// `pruneFireSessionRecords` at startup is the backstop for the paths that
|
|
2224
|
+
// never get to run (SIGKILL, power loss).
|
|
2225
|
+
function forgetFireSessionRecord(record) {
|
|
2226
|
+
if (!record?.tmuxSession) return;
|
|
2227
|
+
try {
|
|
2228
|
+
deleteFireSessionRecord(FIRE_SESSION_REGISTRY_DIR, record.tmuxSession);
|
|
2229
|
+
} catch {
|
|
2230
|
+
// best effort; the startup prune will collect it
|
|
2231
|
+
}
|
|
2232
|
+
}
|
|
2233
|
+
|
|
2234
|
+
function readFireSessionRecordSafe(sessionName) {
|
|
2235
|
+
try {
|
|
2236
|
+
return readFireSessionRecord(FIRE_SESSION_REGISTRY_DIR, sessionName);
|
|
2237
|
+
} catch {
|
|
2238
|
+
return null;
|
|
2239
|
+
}
|
|
2240
|
+
}
|
|
2241
|
+
|
|
2242
|
+
// A session can outlive the Fire inside it (a wrapper shell that never
|
|
2243
|
+
// exited, a pane held open by `remain-on-exit`). Adopting one of those is
|
|
2244
|
+
// worse than killing it: the reaper only ever speaks when a session
|
|
2245
|
+
// disappears, so the task would sit at `running` with nobody left to report
|
|
2246
|
+
// its death. The hand-off record's exit marker is the only evidence that
|
|
2247
|
+
// distinguishes the two, which is why this can only be checked for sessions
|
|
2248
|
+
// we have a record for. Returns the fire's exit code, or null when it is
|
|
2249
|
+
// still running (or when we cannot tell).
|
|
2250
|
+
function readAdoptedFireExitCode(sidecar) {
|
|
2251
|
+
if (!sidecar?.logPath || !sidecar?.exitMarkerToken) return null;
|
|
2252
|
+
return findFireExitCode(sidecar.logPath, {
|
|
2253
|
+
minOffset: sidecar.logStartOffset,
|
|
2254
|
+
markerToken: sidecar.exitMarkerToken,
|
|
2255
|
+
});
|
|
2256
|
+
}
|
|
2257
|
+
|
|
2258
|
+
// `FIRE_TMUX_MODE_ACTIVE` is a one-shot startup snapshot (`tmux -V` via
|
|
2259
|
+
// spawnSync). If that probe failed transiently — fork pressure, a PATH blip
|
|
2260
|
+
// — this process spends its ENTIRE life believing tmux is absent, so both
|
|
2261
|
+
// stale sweeps would skip the tmux lookup entirely and kill every Fire the
|
|
2262
|
+
// previous daemon left in a tmux server that is running perfectly well.
|
|
2263
|
+
// That is the same mass kill by a different door.
|
|
2264
|
+
//
|
|
2265
|
+
// Re-probe before killing. We deliberately do NOT adopt in this state: this
|
|
2266
|
+
// process spawns fires directly and never started the tmux liveness reaper,
|
|
2267
|
+
// so an adopted record would have no watcher. Declining to destroy is
|
|
2268
|
+
// enough — the next restart with a healthy probe adopts them properly.
|
|
2269
|
+
function tmuxFiresMayExistUnseen() {
|
|
2270
|
+
return FIRE_TMUX_MODE_ENABLED && !FIRE_TMUX_MODE_ACTIVE && isTmuxAvailable();
|
|
2271
|
+
}
|
|
2272
|
+
|
|
2273
|
+
// Sessions already identified as orphaned shells and killed. `tmux
|
|
2274
|
+
// kill-session` is asynchronous, so a sweep running moments later still
|
|
2275
|
+
// sees the name in `list-sessions` — and by then the hand-off record is
|
|
2276
|
+
// gone, which would make it look like a live Fire with no metadata and put
|
|
2277
|
+
// the task straight back to `running`.
|
|
2278
|
+
const retiredOrphanFireSessions = new Set();
|
|
2279
|
+
|
|
2280
|
+
// Shared by both adoption entry points: kill the shell, drop the record and
|
|
2281
|
+
// report "not alive" so the caller falls through to its normal dead-task
|
|
2282
|
+
// handling.
|
|
2283
|
+
function retireOrphanedFireSession(sessionName, taskId, exitCode) {
|
|
2284
|
+
retiredOrphanFireSessions.add(sessionName);
|
|
2285
|
+
log(
|
|
2286
|
+
`Tmux session ${sessionName} outlived its Fire for task ${taskId} ` +
|
|
2287
|
+
`(exit marker code=${exitCode}); killing the orphaned shell`,
|
|
2288
|
+
);
|
|
2289
|
+
killFireTmuxSession(sessionName);
|
|
2290
|
+
forgetFireSessionRecord({ tmuxSession: sessionName });
|
|
2291
|
+
}
|
|
2292
|
+
|
|
2293
|
+
// Build an `activeTaskProcesses` entry for a tmux session this daemon did
|
|
2294
|
+
// not spawn. `child` is deliberately absent: the `tmux new-session` client
|
|
2295
|
+
// belonged to the previous daemon and exited long ago.
|
|
2296
|
+
//
|
|
2297
|
+
// This is the one record shape where a truthy `child` does NOT mean "a fire
|
|
2298
|
+
// is running here" — so any code asking that question must go through
|
|
2299
|
+
// `recordHasLiveFire`, never `record.child`. Getting it wrong is not
|
|
2300
|
+
// cosmetic: the restart/create gates read as "nothing running" and spawn a
|
|
2301
|
+
// second fire into a worktree that already has one.
|
|
2302
|
+
function buildAdoptedTmuxRecord({ sessionName, projectId, sidecar }) {
|
|
2303
|
+
return {
|
|
2304
|
+
child: null,
|
|
2305
|
+
projectId: sidecar?.projectId || String(projectId || ""),
|
|
2306
|
+
logPath: sidecar?.logPath || "",
|
|
2307
|
+
logStartOffset: Number(sidecar?.logStartOffset) || 0,
|
|
2308
|
+
exitMarkerToken: sidecar?.exitMarkerToken || "",
|
|
2309
|
+
// Without a record we have no real spawn time. Anchoring at "now" costs
|
|
2310
|
+
// one reaper grace period of delay and is far safer than a 0, which
|
|
2311
|
+
// would make the record instantly reapable.
|
|
2312
|
+
spawnedAtMs: Number(sidecar?.spawnedAtMs) || Date.now(),
|
|
2313
|
+
stopForceKillTimer: null,
|
|
2314
|
+
managedByFireBridge: true,
|
|
2315
|
+
tmuxSession: sessionName,
|
|
2316
|
+
tmuxMode: true,
|
|
2317
|
+
adopted: true,
|
|
2318
|
+
// No record => no logPath and no marker nonce, so the reaper cannot
|
|
2319
|
+
// classify this fire's exit from evidence. Flagged so it reports an
|
|
2320
|
+
// honest KILLED instead of taking the "cannot classify" silent exit,
|
|
2321
|
+
// which would strand the task at `running` with nobody left to speak
|
|
2322
|
+
// for it.
|
|
2323
|
+
adoptedWithoutMetadata: !sidecar,
|
|
2324
|
+
};
|
|
2325
|
+
}
|
|
2326
|
+
|
|
2327
|
+
// Adopt the live tmux Fire belonging to `task`, if there is one. Returns
|
|
2328
|
+
// true when the task must NOT be treated as stale. Called from both kill
|
|
2329
|
+
// paths, which is also the only place a session whose hand-off record is
|
|
2330
|
+
// missing can be adopted at all: the task id is not recoverable from a
|
|
2331
|
+
// session name, but the backend just told us which ids to look for.
|
|
2332
|
+
function adoptLiveTmuxFireForTask(task, sessionNames) {
|
|
2333
|
+
const taskId = String(task?.id || "");
|
|
2334
|
+
if (!taskId) return false;
|
|
2335
|
+
if (activeTaskProcesses.has(taskId) || activePtySessions.has(taskId)) return true;
|
|
2336
|
+
if (!FIRE_TMUX_MODE_ACTIVE) return false;
|
|
2337
|
+
const prefix = buildFireTmuxSessionPrefix(taskId);
|
|
2338
|
+
const candidates = (sessionNames || []).filter(
|
|
2339
|
+
(name) => name.startsWith(prefix) && !retiredOrphanFireSessions.has(name),
|
|
2340
|
+
);
|
|
2341
|
+
if (candidates.length === 0) return false;
|
|
2342
|
+
// Session names truncate the task id to 32 chars, so a record could in
|
|
2343
|
+
// principle have been filed for a different task sharing that prefix.
|
|
2344
|
+
// Trust it only when it names the task we are actually rescuing.
|
|
2345
|
+
const recordFor = (name) => {
|
|
2346
|
+
const stored = readFireSessionRecordSafe(name);
|
|
2347
|
+
return stored && stored.taskId === taskId ? stored : null;
|
|
2348
|
+
};
|
|
2349
|
+
// With more than one candidate (an abandoned session left beside a
|
|
2350
|
+
// respawn) prefer the one we have a record for: adopting blind would pick
|
|
2351
|
+
// by tmux's list order and could watch the wrong session's exit marker.
|
|
2352
|
+
const sessionName = candidates.find((name) => recordFor(name)) || candidates[0];
|
|
2353
|
+
const sidecar = recordFor(sessionName);
|
|
2354
|
+
const exitCode = readAdoptedFireExitCode(sidecar);
|
|
2355
|
+
if (exitCode !== null) {
|
|
2356
|
+
retireOrphanedFireSession(sessionName, taskId, exitCode);
|
|
2357
|
+
return false;
|
|
2358
|
+
}
|
|
2359
|
+
activeTaskProcesses.set(
|
|
2360
|
+
taskId,
|
|
2361
|
+
buildAdoptedTmuxRecord({ sessionName, projectId: task?.project_id, sidecar }),
|
|
2362
|
+
);
|
|
2363
|
+
log(
|
|
2364
|
+
`Task ${taskId} still has a live tmux Fire (session=${sessionName}); adopted instead of killed` +
|
|
2365
|
+
(sidecar ? "" : " — no hand-off record, so its exit can only be classified as killed"),
|
|
2366
|
+
);
|
|
2367
|
+
return true;
|
|
2368
|
+
}
|
|
2369
|
+
|
|
2370
|
+
// Resolves once the startup adoption pass has finished. Both stale sweeps
|
|
2371
|
+
// await it so they can never judge a task before its Fire has had the
|
|
2372
|
+
// chance to be reclaimed.
|
|
2373
|
+
let adoptOrphanedTmuxFiresPromise = null;
|
|
2374
|
+
|
|
2375
|
+
// Startup pass: adopt every Fire session the previous daemon left behind,
|
|
2376
|
+
// then discard hand-off records whose session is gone.
|
|
2377
|
+
async function adoptOrphanedTmuxFires() {
|
|
2378
|
+
if (!FIRE_TMUX_MODE_ACTIVE) return 0;
|
|
2379
|
+
const { sessions, conclusive } = await listAllTmuxSessions();
|
|
2380
|
+
if (!conclusive) {
|
|
2381
|
+
// We could not ask tmux. Adopting nothing is survivable — both stale
|
|
2382
|
+
// sweeps re-probe and adopt whatever they find. Pruning is not: the
|
|
2383
|
+
// prune below deletes every record whose session is absent from this
|
|
2384
|
+
// list, so running it on a blind listing would wipe the
|
|
2385
|
+
// `exitMarkerToken`s of every live Fire on the host, permanently
|
|
2386
|
+
// downgrading them all to `adoptedWithoutMetadata`.
|
|
2387
|
+
logError(
|
|
2388
|
+
"Could not list tmux sessions for Fire adoption; adopting nothing and keeping every hand-off record",
|
|
2389
|
+
);
|
|
2390
|
+
return 0;
|
|
2391
|
+
}
|
|
2392
|
+
|
|
2393
|
+
let adopted = 0;
|
|
2394
|
+
for (const sessionName of sessions) {
|
|
2395
|
+
if (!sessionName.startsWith(FIRE_TMUX_SESSION_NAMESPACE)) continue;
|
|
2396
|
+
if (retiredOrphanFireSessions.has(sessionName)) continue;
|
|
2397
|
+
const sidecar = readFireSessionRecordSafe(sessionName);
|
|
2398
|
+
// No record: we cannot learn the task id from the session name (it is
|
|
2399
|
+
// truncated to 32 chars). Leave it — the kill paths adopt it later,
|
|
2400
|
+
// once the backend supplies the id.
|
|
2401
|
+
if (!sidecar) continue;
|
|
2402
|
+
|
|
2403
|
+
if (pendingTaskStarts.has(sidecar.taskId)) {
|
|
2404
|
+
// A `create_task` for this task is mid-flight and will install its own
|
|
2405
|
+
// record when the spawn completes. Adopting now just gets overwritten
|
|
2406
|
+
// moments later, which is how a session ends up orphaned with its
|
|
2407
|
+
// record still on disk.
|
|
2408
|
+
continue;
|
|
2409
|
+
}
|
|
2410
|
+
|
|
2411
|
+
const tracked = activeTaskProcesses.get(sidecar.taskId);
|
|
2412
|
+
if (tracked) {
|
|
2413
|
+
// A `create_task`/`restart_task` redelivered into the adoption window
|
|
2414
|
+
// can spawn a second session for the same task before we get here.
|
|
2415
|
+
// Keeping both records is the dangerous part: on the NEXT restart
|
|
2416
|
+
// adoption would pick between them by tmux's arbitrary list order and
|
|
2417
|
+
// could end up watching the abandoned session's exit marker while the
|
|
2418
|
+
// live fire's death goes unreported. Drop the loser's record so the
|
|
2419
|
+
// task has exactly one, and say so loudly — two fires in one worktree
|
|
2420
|
+
// is a real problem, just not one we can safely resolve by killing.
|
|
2421
|
+
if (tracked.tmuxSession !== sessionName) {
|
|
2422
|
+
logError(
|
|
2423
|
+
`Task ${sidecar.taskId} has a second tmux session ${sessionName} besides the tracked ` +
|
|
2424
|
+
`${tracked.tmuxSession || "(none)"}; discarding its hand-off record. Two fires may be ` +
|
|
2425
|
+
`sharing one worktree — inspect and kill the stale session manually.`,
|
|
2426
|
+
);
|
|
2427
|
+
forgetFireSessionRecord({ tmuxSession: sessionName });
|
|
2428
|
+
}
|
|
2429
|
+
continue;
|
|
2430
|
+
}
|
|
2431
|
+
|
|
2432
|
+
const exitCode = readAdoptedFireExitCode(sidecar);
|
|
2433
|
+
if (exitCode !== null) {
|
|
2434
|
+
retireOrphanedFireSession(sessionName, sidecar.taskId, exitCode);
|
|
2435
|
+
continue;
|
|
2436
|
+
}
|
|
2437
|
+
|
|
2438
|
+
activeTaskProcesses.set(
|
|
2439
|
+
sidecar.taskId,
|
|
2440
|
+
buildAdoptedTmuxRecord({ sessionName, projectId: sidecar.projectId, sidecar }),
|
|
2441
|
+
);
|
|
2442
|
+
adopted += 1;
|
|
2443
|
+
log(
|
|
2444
|
+
`Adopted tmux Fire task ${sidecar.taskId} (session=${sessionName}) left running by a previous daemon`,
|
|
2445
|
+
);
|
|
2446
|
+
}
|
|
2447
|
+
|
|
2448
|
+
try {
|
|
2449
|
+
pruneFireSessionRecords(FIRE_SESSION_REGISTRY_DIR, sessions);
|
|
2450
|
+
} catch (error) {
|
|
2451
|
+
logError(`Failed to prune tmux hand-off records: ${error?.message || error}`);
|
|
2452
|
+
}
|
|
2453
|
+
|
|
2454
|
+
if (adopted > 0) {
|
|
2455
|
+
log(`Adopted ${adopted} tmux Fire task(s) from a previous daemon`);
|
|
2456
|
+
}
|
|
2457
|
+
return adopted;
|
|
2458
|
+
}
|
|
2041
2459
|
// -------------------------------------------------------------------------
|
|
2042
2460
|
|
|
2043
2461
|
function buildTaskWorktreeRoot(projectWorkspacePath, worktreeBranch) {
|
|
@@ -3337,7 +3755,9 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
3337
3755
|
"restart_daemon",
|
|
3338
3756
|
"refresh_session_inplace",
|
|
3339
3757
|
"task_attachments_v1",
|
|
3758
|
+
"backend_session_list",
|
|
3340
3759
|
CUSTOM_COMMANDS_CAPABILITY,
|
|
3760
|
+
UPDATE_DAEMON_CAPABILITY,
|
|
3341
3761
|
];
|
|
3342
3762
|
if (remoteExecEnabled) {
|
|
3343
3763
|
advertisedCapabilities.push(REMOTE_EXEC_CAPABILITY);
|
|
@@ -3371,8 +3791,43 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
3371
3791
|
if (effectiveCapabilities.length > 0) {
|
|
3372
3792
|
extraHeaders["x-conductor-capabilities"] = effectiveCapabilities.join(",");
|
|
3373
3793
|
}
|
|
3794
|
+
// Built-in "Update Daemon" is a global `npm install -g` followed by a
|
|
3795
|
+
// restart. Refuse up front — with a reason the UI can show — wherever that
|
|
3796
|
+
// is the wrong way to upgrade this install.
|
|
3797
|
+
function resolveDaemonUpdateRefusal() {
|
|
3798
|
+
if (IS_GUEST_DAEMON) {
|
|
3799
|
+
return "a shared guest daemon cannot install a new CLI version on the host machine";
|
|
3800
|
+
}
|
|
3801
|
+
if (installMethod === "homebrew") {
|
|
3802
|
+
return `conductor was installed with Homebrew; run \`${buildUpgradeCommand({ env: process.env })}\` on ${AGENT_NAME} instead`;
|
|
3803
|
+
}
|
|
3804
|
+
if (!autoUpdateSupportedInstall) {
|
|
3805
|
+
return `conductor runs from ${installedPackageRoot}, which is not a global install; upgrade it the same way it was installed`;
|
|
3806
|
+
}
|
|
3807
|
+
return null;
|
|
3808
|
+
}
|
|
3809
|
+
|
|
3374
3810
|
const aiManagerHandlers = (deps.createAiManagerHandlers || createAiManagerHandlers)({ configPath: effectiveConfigPath });
|
|
3375
3811
|
const customCommandHandlers = createCustomCommandHandlers({ configPath: effectiveConfigPath });
|
|
3812
|
+
const daemonUpdatePaths = resolveDaemonUpdatePaths(process.env);
|
|
3813
|
+
const daemonUpdateHandlers = createDaemonUpdateHandlers({
|
|
3814
|
+
statusPath: daemonUpdatePaths.statusPath,
|
|
3815
|
+
logPath: daemonUpdatePaths.logPath,
|
|
3816
|
+
updaterScript: DAEMON_UPDATE_SCRIPT_PATH,
|
|
3817
|
+
refuseReason: resolveDaemonUpdateRefusal(),
|
|
3818
|
+
env: { ...process.env, ...materializedConductorPathEnv },
|
|
3819
|
+
updaterParams: {
|
|
3820
|
+
packageRoot: installedPackageRoot,
|
|
3821
|
+
launcherScript: restartLauncherScript,
|
|
3822
|
+
launcherArgs: restartLauncherArgs,
|
|
3823
|
+
versionCheckScript,
|
|
3824
|
+
versionCheckArgs,
|
|
3825
|
+
daemonPid: process.pid,
|
|
3826
|
+
lockFile: LOCK_FILE,
|
|
3827
|
+
daemonLogPath: resolveDaemonLogPaths().logPath,
|
|
3828
|
+
currentVersion: cliVersion,
|
|
3829
|
+
},
|
|
3830
|
+
});
|
|
3376
3831
|
const remoteExecHandlers = remoteExecEnabled
|
|
3377
3832
|
? createRemoteExecHandlers({ defaultWorkspace: homeDir })
|
|
3378
3833
|
: null;
|
|
@@ -3501,6 +3956,16 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
3501
3956
|
log(`Daemon Name: ${AGENT_NAME}`);
|
|
3502
3957
|
log(`Supported Backends: ${SUPPORTED_BACKENDS.join(", ")}`);
|
|
3503
3958
|
|
|
3959
|
+
// Reclaim the Fires the previous daemon left running. Started here rather
|
|
3960
|
+
// than awaited: connecting must not wait on a wedged tmux server. Both
|
|
3961
|
+
// stale sweeps await `adoptOrphanedTmuxFiresPromise` before judging
|
|
3962
|
+
// anything, which is the ordering that actually matters — they decide
|
|
3963
|
+
// liveness from `activeTaskProcesses`, so a survivor not yet adopted when
|
|
3964
|
+
// they run is a survivor that gets killed.
|
|
3965
|
+
adoptOrphanedTmuxFiresPromise = adoptOrphanedTmuxFires().catch((error) => {
|
|
3966
|
+
logError(`adoptOrphanedTmuxFires failed: ${error?.message || error}`);
|
|
3967
|
+
});
|
|
3968
|
+
|
|
3504
3969
|
client.connect().catch((err) => {
|
|
3505
3970
|
logError(`Failed to connect: ${err}`);
|
|
3506
3971
|
});
|
|
@@ -4284,6 +4749,8 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4284
4749
|
|
|
4285
4750
|
async function recoverStaleTasks() {
|
|
4286
4751
|
try {
|
|
4752
|
+
// Never judge a task before startup adoption has had its say.
|
|
4753
|
+
await adoptOrphanedTmuxFiresPromise;
|
|
4287
4754
|
const response = await fetchFn(`${BACKEND_HTTP}/api/tasks`, {
|
|
4288
4755
|
method: "GET",
|
|
4289
4756
|
headers: {
|
|
@@ -4316,8 +4783,48 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4316
4783
|
return;
|
|
4317
4784
|
}
|
|
4318
4785
|
|
|
4786
|
+
// Startup is the main case the tmux hand-off exists for: the previous
|
|
4787
|
+
// daemon deliberately left its Fires running and this fresh process has
|
|
4788
|
+
// an empty `activeTaskProcesses` by definition, so "not in memory"
|
|
4789
|
+
// proves nothing here. `adoptOrphanedTmuxFires` has already claimed
|
|
4790
|
+
// every session with a hand-off record; this second pass catches the
|
|
4791
|
+
// ones without a record, which only become identifiable now that the
|
|
4792
|
+
// backend has told us which task ids to look for.
|
|
4793
|
+
if (tmuxFiresMayExistUnseen()) {
|
|
4794
|
+
logError(
|
|
4795
|
+
`tmux is reachable but this daemon started without tmux mode active; skipping stale ` +
|
|
4796
|
+
`recovery for ${staleTasks.length} task(s) rather than killing Fires it cannot see`,
|
|
4797
|
+
);
|
|
4798
|
+
return;
|
|
4799
|
+
}
|
|
4800
|
+
const tmuxSessions = FIRE_TMUX_MODE_ACTIVE ? await listAllTmuxSessions() : null;
|
|
4801
|
+
if (tmuxSessions && !tmuxSessions.conclusive) {
|
|
4802
|
+
// An unanswerable `tmux list-sessions` is not evidence that nothing is
|
|
4803
|
+
// running — and this is the one sweep with no second chance, since it
|
|
4804
|
+
// runs once per process. Killing on a blind probe is precisely how a
|
|
4805
|
+
// single wedged tmux server reproduces the 2026-08-31 mass kill.
|
|
4806
|
+
logError(
|
|
4807
|
+
`Could not list tmux sessions; skipping stale recovery for ${staleTasks.length} task(s) rather than killing on a blind probe`,
|
|
4808
|
+
);
|
|
4809
|
+
return;
|
|
4810
|
+
}
|
|
4811
|
+
const deadTasks = staleTasks.filter((task) => {
|
|
4812
|
+
// A spawn already in flight has no record yet either; killing it
|
|
4813
|
+
// would shoot down a Fire this daemon is in the middle of starting.
|
|
4814
|
+
if (pendingTaskStarts.has(String(task?.id || ""))) return false;
|
|
4815
|
+
return !adoptLiveTmuxFireForTask(task, tmuxSessions?.sessions || []);
|
|
4816
|
+
});
|
|
4817
|
+
|
|
4818
|
+
if (deadTasks.length === 0) {
|
|
4819
|
+
return;
|
|
4820
|
+
}
|
|
4821
|
+
|
|
4822
|
+
// Count what the backend actually accepted, not what we attempted: a
|
|
4823
|
+
// 409/500 leaves the task running, and reporting it as recovered has
|
|
4824
|
+
// repeatedly sent readers of this log looking in the wrong place.
|
|
4825
|
+
let killedCount = 0;
|
|
4319
4826
|
await Promise.all(
|
|
4320
|
-
|
|
4827
|
+
deadTasks.map(async (task) => {
|
|
4321
4828
|
const taskId = task?.id;
|
|
4322
4829
|
if (!taskId) return;
|
|
4323
4830
|
const patchResp = await fetchFn(`${BACKEND_HTTP}/api/tasks/${taskId}`, {
|
|
@@ -4332,12 +4839,13 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4332
4839
|
if (!patchResp.ok) {
|
|
4333
4840
|
logError(`Failed to mark stale task ${taskId} as killed: HTTP ${patchResp.status}`);
|
|
4334
4841
|
} else {
|
|
4842
|
+
killedCount += 1;
|
|
4335
4843
|
markBackendHttpSuccess();
|
|
4336
4844
|
}
|
|
4337
4845
|
}),
|
|
4338
4846
|
);
|
|
4339
4847
|
|
|
4340
|
-
log(`Recovered ${
|
|
4848
|
+
log(`Recovered ${killedCount}/${deadTasks.length} stale task(s) to killed`);
|
|
4341
4849
|
} catch (error) {
|
|
4342
4850
|
logError(`recoverStaleTasks error: ${error?.message || error}`);
|
|
4343
4851
|
}
|
|
@@ -4349,6 +4857,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4349
4857
|
|
|
4350
4858
|
async function reconcileAssignedTasks() {
|
|
4351
4859
|
try {
|
|
4860
|
+
await adoptOrphanedTmuxFiresPromise;
|
|
4352
4861
|
const response = await fetchFn(`${BACKEND_HTTP}/api/tasks`, {
|
|
4353
4862
|
method: "GET",
|
|
4354
4863
|
headers: {
|
|
@@ -4383,10 +4892,36 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4383
4892
|
});
|
|
4384
4893
|
|
|
4385
4894
|
let killedCount = 0;
|
|
4895
|
+
// Shutdown deliberately leaves tmux-detached Fires running, and a ws
|
|
4896
|
+
// reconnect does not repopulate `activeTaskProcesses`, so this map
|
|
4897
|
+
// under-reports what is still alive on this host. Ask tmux.
|
|
4898
|
+
if (assigned.length && tmuxFiresMayExistUnseen()) {
|
|
4899
|
+
logError(
|
|
4900
|
+
`tmux is reachable but this daemon started without tmux mode active; skipping reconcile ` +
|
|
4901
|
+
`for ${assigned.length} assigned task(s)`,
|
|
4902
|
+
);
|
|
4903
|
+
return;
|
|
4904
|
+
}
|
|
4905
|
+
const tmuxSessions =
|
|
4906
|
+
FIRE_TMUX_MODE_ACTIVE && assigned.length ? await listAllTmuxSessions() : null;
|
|
4907
|
+
if (tmuxSessions && !tmuxSessions.conclusive) {
|
|
4908
|
+
// Same rule as the startup sweep. Here we do get another chance on the
|
|
4909
|
+
// next reconnect, which is all the more reason not to guess.
|
|
4910
|
+
logError(
|
|
4911
|
+
`Could not list tmux sessions; skipping reconcile for ${assigned.length} assigned task(s)`,
|
|
4912
|
+
);
|
|
4913
|
+
return;
|
|
4914
|
+
}
|
|
4386
4915
|
for (const task of assigned) {
|
|
4387
4916
|
const taskId = String(task?.id || "");
|
|
4388
4917
|
if (!taskId) continue;
|
|
4389
|
-
if (localTaskIds.has(taskId)) {
|
|
4918
|
+
if (localTaskIds.has(taskId) || pendingTaskStarts.has(taskId)) {
|
|
4919
|
+
continue;
|
|
4920
|
+
}
|
|
4921
|
+
// Adopts on the spot rather than merely skipping: a survivor that
|
|
4922
|
+
// dodges the kill but gains no watcher would sit at `running`
|
|
4923
|
+
// forever, because nothing else reports a tmux Fire's death.
|
|
4924
|
+
if (adoptLiveTmuxFireForTask(task, tmuxSessions?.sessions || [])) {
|
|
4390
4925
|
continue;
|
|
4391
4926
|
}
|
|
4392
4927
|
const patchResp = await fetchFn(`${BACKEND_HTTP}/api/tasks/${taskId}`, {
|
|
@@ -4422,7 +4957,12 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4422
4957
|
// half-close). Fires raise their own ws to backend independently — this is
|
|
4423
4958
|
// the daemon's *own* belief about which tasks it is hosting. The backend
|
|
4424
4959
|
// arbitrates between this push and the persisted task state.
|
|
4425
|
-
function pushAgentAliveTasks(reason) {
|
|
4960
|
+
async function pushAgentAliveTasks(reason) {
|
|
4961
|
+
// Adoption may still be running when the ws handshake completes. This is
|
|
4962
|
+
// the only channel that revokes a `killed_reason=daemon_disconnected`
|
|
4963
|
+
// flag, so pushing before the adopted tasks are in the map would leave
|
|
4964
|
+
// them showing as killed in the UI even though this daemon rescued them.
|
|
4965
|
+
await adoptOrphanedTmuxFiresPromise;
|
|
4426
4966
|
const aliveTaskIds = getActiveTaskIds();
|
|
4427
4967
|
if (aliveTaskIds.length === 0) {
|
|
4428
4968
|
return Promise.resolve();
|
|
@@ -5726,6 +6266,10 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5726
6266
|
void handleGetProjectAgents(event.payload);
|
|
5727
6267
|
return;
|
|
5728
6268
|
}
|
|
6269
|
+
if (event.type === "list_backend_sessions") {
|
|
6270
|
+
void handleListBackendSessions(event.payload);
|
|
6271
|
+
return;
|
|
6272
|
+
}
|
|
5729
6273
|
if (event.type === "ai_manager_request") {
|
|
5730
6274
|
// RFC 0035: reads are fine and genuinely useful to a guest (it needs to
|
|
5731
6275
|
// know how much quota the borrowed machine has left). `switch_account` is
|
|
@@ -5795,6 +6339,14 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
5795
6339
|
logError(`Unhandled restart_daemon failure: ${error?.message || error}`);
|
|
5796
6340
|
});
|
|
5797
6341
|
}
|
|
6342
|
+
if (event.type === "update_daemon_request") {
|
|
6343
|
+
if (event?.payload?.action === "start") {
|
|
6344
|
+
log(`[update_daemon] Update requested (current=${cliVersion}); log: ${daemonUpdatePaths.logPath}`);
|
|
6345
|
+
}
|
|
6346
|
+
handleUpdateDaemonRequest(client, daemonUpdateHandlers, event.payload).catch((error) => {
|
|
6347
|
+
logError(`Unhandled update_daemon_request failure: ${error?.message || error}`);
|
|
6348
|
+
});
|
|
6349
|
+
}
|
|
5798
6350
|
}
|
|
5799
6351
|
|
|
5800
6352
|
function markWatchdogHealthy(signal, at = Date.now()) {
|
|
@@ -6096,6 +6648,42 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6096
6648
|
}
|
|
6097
6649
|
}
|
|
6098
6650
|
|
|
6651
|
+
async function handleListBackendSessions(payload) {
|
|
6652
|
+
const requestId = payload?.request_id ? String(payload.request_id).trim() : "";
|
|
6653
|
+
if (!requestId) {
|
|
6654
|
+
logError(`Invalid list_backend_sessions payload: ${JSON.stringify(payload)}`);
|
|
6655
|
+
return;
|
|
6656
|
+
}
|
|
6657
|
+
|
|
6658
|
+
let result;
|
|
6659
|
+
try {
|
|
6660
|
+
result = await collectBackendSessions({
|
|
6661
|
+
requestedBackends: Array.isArray(payload?.backends) ? payload.backends : null,
|
|
6662
|
+
supportedBackends: SUPPORTED_BACKENDS,
|
|
6663
|
+
limit: payload?.limit,
|
|
6664
|
+
listSessions: deps.listBackendSessions,
|
|
6665
|
+
});
|
|
6666
|
+
} catch (error) {
|
|
6667
|
+
result = {
|
|
6668
|
+
sessions: [],
|
|
6669
|
+
errors: [{ backend: "*", message: `Failed to list backend sessions on daemon ${AGENT_NAME}: ${error?.message || error}` }],
|
|
6670
|
+
};
|
|
6671
|
+
}
|
|
6672
|
+
|
|
6673
|
+
try {
|
|
6674
|
+
await client.sendJson({
|
|
6675
|
+
type: "backend_sessions_listed",
|
|
6676
|
+
payload: {
|
|
6677
|
+
request_id: requestId,
|
|
6678
|
+
sessions: result.sessions,
|
|
6679
|
+
errors: result.errors,
|
|
6680
|
+
},
|
|
6681
|
+
});
|
|
6682
|
+
} catch (error) {
|
|
6683
|
+
logError(`Failed to report backend_sessions_listed for ${requestId}: ${error?.message || error}`);
|
|
6684
|
+
}
|
|
6685
|
+
}
|
|
6686
|
+
|
|
6099
6687
|
function stopActiveTaskProcess(
|
|
6100
6688
|
taskId,
|
|
6101
6689
|
{
|
|
@@ -6162,6 +6750,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6162
6750
|
}
|
|
6163
6751
|
|
|
6164
6752
|
activeTaskProcesses.delete(taskId);
|
|
6753
|
+
forgetFireSessionRecord(processRecord);
|
|
6165
6754
|
return true;
|
|
6166
6755
|
}
|
|
6167
6756
|
|
|
@@ -6236,6 +6825,15 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6236
6825
|
return Boolean(record?.forceDaemonTerminalStatusReport) || !Boolean(record?.managedByFireBridge);
|
|
6237
6826
|
}
|
|
6238
6827
|
|
|
6828
|
+
// "Does this record stand for a live fire?" `child` alone is not the answer:
|
|
6829
|
+
// an ADOPTED tmux record has none — the `tmux new-session` client belonged
|
|
6830
|
+
// to the daemon that spawned it — while the fire it points at is very much
|
|
6831
|
+
// alive. Every already-active gate must ask this instead, or it will spawn
|
|
6832
|
+
// a second fire into a worktree that already has one.
|
|
6833
|
+
function recordHasLiveFire(record) {
|
|
6834
|
+
return Boolean(record?.child) || Boolean(record?.tmuxMode && record?.tmuxSession);
|
|
6835
|
+
}
|
|
6836
|
+
|
|
6239
6837
|
// RFC 0029: confirm that a task the backend believes is dead is actually
|
|
6240
6838
|
// still hosting a live fire on this daemon. The check is intentionally
|
|
6241
6839
|
// conservative:
|
|
@@ -6334,6 +6932,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6334
6932
|
entry.stopForceKillTimer = null;
|
|
6335
6933
|
}
|
|
6336
6934
|
activeTaskProcesses.delete(taskId);
|
|
6935
|
+
forgetFireSessionRecord(entry);
|
|
6337
6936
|
}
|
|
6338
6937
|
entry = undefined;
|
|
6339
6938
|
}
|
|
@@ -6391,7 +6990,12 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6391
6990
|
|
|
6392
6991
|
const processRecord = activeTaskProcesses.get(taskId);
|
|
6393
6992
|
const ptyRecord = activePtySessions.get(taskId);
|
|
6394
|
-
|
|
6993
|
+
// `recordHasLiveFire`, not `child`: an adopted tmux record has no child,
|
|
6994
|
+
// and reading it as "nothing to stop" would report the task KILLED to the
|
|
6995
|
+
// backend while leaving the fire running in its session, still writing to
|
|
6996
|
+
// the worktree. Nothing would ever notice — the reaper only speaks when a
|
|
6997
|
+
// session disappears.
|
|
6998
|
+
if (!recordHasLiveFire(processRecord) && !ptyRecord) {
|
|
6395
6999
|
log(`Stop requested for task ${taskId}, but no active process found`);
|
|
6396
7000
|
sendStopAck(false);
|
|
6397
7001
|
// "Nothing to stop" IS the terminal answer, and the server cannot infer
|
|
@@ -6769,9 +7373,12 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6769
7373
|
}
|
|
6770
7374
|
|
|
6771
7375
|
const existingTaskRecord = activeTaskProcesses.get(taskId);
|
|
6772
|
-
if (existingTaskRecord
|
|
7376
|
+
if (recordHasLiveFire(existingTaskRecord) || pendingTaskStarts.has(taskId)) {
|
|
6773
7377
|
log(
|
|
6774
|
-
`Duplicate create_task ignored for ${taskId}: task already active (
|
|
7378
|
+
`Duplicate create_task ignored for ${taskId}: task already active (` +
|
|
7379
|
+
`${existingTaskRecord?.tmuxSession
|
|
7380
|
+
? `tmux=${existingTaskRecord.tmuxSession}`
|
|
7381
|
+
: `pid=${existingTaskRecord?.child?.pid ?? "unknown"}`})`,
|
|
6775
7382
|
);
|
|
6776
7383
|
sendAgentCommandAck({
|
|
6777
7384
|
requestId,
|
|
@@ -6975,7 +7582,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6975
7582
|
const outputCapture = createChildOutputCapture({ logPath, logStartOffset });
|
|
6976
7583
|
const spawnedAtMs = Date.now();
|
|
6977
7584
|
|
|
6978
|
-
|
|
7585
|
+
const activeProcessRecord = {
|
|
6979
7586
|
child,
|
|
6980
7587
|
projectId,
|
|
6981
7588
|
logPath,
|
|
@@ -6993,7 +7600,12 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
6993
7600
|
managedByFireBridge: true,
|
|
6994
7601
|
tmuxSession: tmuxSession || null,
|
|
6995
7602
|
tmuxMode: Boolean(tmuxSession),
|
|
6996
|
-
}
|
|
7603
|
+
};
|
|
7604
|
+
activeTaskProcesses.set(taskId, activeProcessRecord);
|
|
7605
|
+
// Mirror the parts of the record that only exist in this process's
|
|
7606
|
+
// memory, so a successor daemon can adopt this Fire instead of
|
|
7607
|
+
// mistaking the hand-off for a dead task.
|
|
7608
|
+
rememberFireSessionRecord(taskId, activeProcessRecord);
|
|
6997
7609
|
|
|
6998
7610
|
client
|
|
6999
7611
|
.sendJson({
|
|
@@ -7099,6 +7711,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
7099
7711
|
clearTimeout(active.stopForceKillTimer);
|
|
7100
7712
|
}
|
|
7101
7713
|
activeTaskProcesses.delete(taskId);
|
|
7714
|
+
forgetFireSessionRecord(active);
|
|
7102
7715
|
const suppressExitStatusReport = suppressedExitStatusReports.has(taskId);
|
|
7103
7716
|
suppressedExitStatusReports.delete(taskId);
|
|
7104
7717
|
if (logStream) {
|
|
@@ -7289,12 +7902,17 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
7289
7902
|
activeTarget.stopForceKillTimer = null;
|
|
7290
7903
|
}
|
|
7291
7904
|
activeTaskProcesses.delete(normalizedTargetTaskId);
|
|
7905
|
+
forgetFireSessionRecord(activeTarget);
|
|
7292
7906
|
activeTarget = undefined;
|
|
7293
7907
|
}
|
|
7294
7908
|
}
|
|
7295
7909
|
|
|
7910
|
+
// The probe above already cleared the record if the session was
|
|
7911
|
+
// conclusively gone, so a surviving tmuxMode entry means a live session.
|
|
7912
|
+
const targetHasLiveFire = recordHasLiveFire(activeTarget);
|
|
7913
|
+
|
|
7296
7914
|
if (isRefreshSessionInplace) {
|
|
7297
|
-
if (!
|
|
7915
|
+
if (!targetHasLiveFire) {
|
|
7298
7916
|
reportRestartFailure({
|
|
7299
7917
|
taskId: normalizedTargetTaskId,
|
|
7300
7918
|
projectId: normalizedProjectId,
|
|
@@ -7305,7 +7923,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
7305
7923
|
});
|
|
7306
7924
|
return;
|
|
7307
7925
|
}
|
|
7308
|
-
} else if (
|
|
7926
|
+
} else if (targetHasLiveFire) {
|
|
7309
7927
|
// tmux-mode failures point at the tmux session, not a meaningless pid.
|
|
7310
7928
|
const description = activeTarget.tmuxMode
|
|
7311
7929
|
? `task already active in tmux session ${activeTarget.tmuxSession || "(unknown)"} — stop the task before restarting`
|
|
@@ -7455,6 +8073,31 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
7455
8073
|
return;
|
|
7456
8074
|
}
|
|
7457
8075
|
|
|
8076
|
+
if (!resolvedResumeCwd && isForkMode) {
|
|
8077
|
+
// A fork starts a brand-new session and receives its context as a URL, so
|
|
8078
|
+
// it does not need the source task's files. When nothing in the chain
|
|
8079
|
+
// resolves — the normal case for a cross-daemon fork, where the server
|
|
8080
|
+
// drops the source machine's paths and this daemon holds no local path
|
|
8081
|
+
// for the project — land in a fresh workspace dir exactly like
|
|
8082
|
+
// create_task does, instead of killing the successor on arrival.
|
|
8083
|
+
const now = new Date();
|
|
8084
|
+
// Keyed off the task id, not the daemon pid: the run timestamp only has
|
|
8085
|
+
// second granularity and this path never renames the dir afterwards
|
|
8086
|
+
// (create_task does), so two forks landing in the same second would
|
|
8087
|
+
// otherwise share one workspace AND one conductor.log.
|
|
8088
|
+
const taskSuffix =
|
|
8089
|
+
normalizedTargetTaskId.replace(/[^a-zA-Z0-9]/g, "").slice(0, 8) || String(process.pid);
|
|
8090
|
+
resolvedResumeCwd = path.join(
|
|
8091
|
+
WORKSPACE_ROOT,
|
|
8092
|
+
formatWorkspaceDate(now),
|
|
8093
|
+
`${formatWorkspaceRunTimestamp(now)}_fork_${taskSuffix}`,
|
|
8094
|
+
);
|
|
8095
|
+
log(
|
|
8096
|
+
`No local workspace for project ${normalizedProjectId} on this daemon; ` +
|
|
8097
|
+
`forking task ${normalizedTargetTaskId} into ${resolvedResumeCwd}`,
|
|
8098
|
+
);
|
|
8099
|
+
}
|
|
8100
|
+
|
|
7458
8101
|
if (!resolvedResumeCwd) {
|
|
7459
8102
|
reportRestartFailure({
|
|
7460
8103
|
taskId: normalizedTargetTaskId,
|
|
@@ -7690,6 +8333,9 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
7690
8333
|
tmuxMode: Boolean(tmuxSession),
|
|
7691
8334
|
};
|
|
7692
8335
|
activeTaskProcesses.set(normalizedTargetTaskId, activeProcessRecord);
|
|
8336
|
+
// See the create_task path: this is the only copy of `exitMarkerToken`
|
|
8337
|
+
// that survives the current process.
|
|
8338
|
+
rememberFireSessionRecord(normalizedTargetTaskId, activeProcessRecord);
|
|
7693
8339
|
|
|
7694
8340
|
// In tmux mode the Fire process writes directly to logPath via shell
|
|
7695
8341
|
// redirection inside the tmux session, so the daemon does not pipe
|
|
@@ -7780,6 +8426,7 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
7780
8426
|
clearTimeout(active.stopForceKillTimer);
|
|
7781
8427
|
}
|
|
7782
8428
|
activeTaskProcesses.delete(normalizedTargetTaskId);
|
|
8429
|
+
forgetFireSessionRecord(active);
|
|
7783
8430
|
const suppressExitStatusReport = suppressedExitStatusReports.has(normalizedTargetTaskId);
|
|
7784
8431
|
suppressedExitStatusReports.delete(normalizedTargetTaskId);
|
|
7785
8432
|
if (logStream) {
|