omp-conductor 0.19.6 → 0.19.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +17 -1
- package/package.json +1 -1
- package/src/daemon.ts +123 -32
- package/src/doctor.ts +17 -12
- package/src/escalate.ts +39 -21
- package/src/fleet.ts +928 -140
- package/src/store.ts +42 -0
- package/src/types.ts +19 -0
- package/src/verbs/server.ts +92 -11
package/src/fleet.ts
CHANGED
|
@@ -391,7 +391,10 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
391
391
|
// challenge proof never reads it: its acknowledgement is conductor state,
|
|
392
392
|
// so where (or whether) a transcript lives is no longer part of arming
|
|
393
393
|
// (#614).
|
|
394
|
-
const claimed =
|
|
394
|
+
const claimed =
|
|
395
|
+
deps.claimedSessionFile !== undefined
|
|
396
|
+
? deps.claimedSessionFile()
|
|
397
|
+
: claimedOrchestratorSessionFile(named, deps.pidAlive ?? pidAlive);
|
|
395
398
|
|
|
396
399
|
// Prefer the project's live forum topic so arm challenges land where
|
|
397
400
|
// escalations already do (#318), following the bridge's current claim when the
|
|
@@ -399,7 +402,7 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
399
402
|
let sendTopic: number | undefined;
|
|
400
403
|
if (named !== undefined) {
|
|
401
404
|
try {
|
|
402
|
-
sendTopic = resolveProjectTopicId(findProject(loadConfig(), named));
|
|
405
|
+
sendTopic = resolveProjectTopicId(findProject(loadConfig(), named), deps.pidAlive ?? pidAlive);
|
|
403
406
|
} catch {
|
|
404
407
|
/* no project config */
|
|
405
408
|
}
|
|
@@ -907,10 +910,17 @@ export interface WorkerPaneIdentity {
|
|
|
907
910
|
sessionFile?: string;
|
|
908
911
|
}
|
|
909
912
|
|
|
910
|
-
/** A tracked pane, or the explicit reason there is none. Never a silent claim.
|
|
913
|
+
/** A tracked pane, or the explicit reason there is none. Never a silent claim.
|
|
914
|
+
* The tracked variant names the marked workspace the representation lives in,
|
|
915
|
+
* so a caller that just created one can protect it from the same pass's
|
|
916
|
+
* empty-workspace cleanup. `phase` names the step that refused — `"split"`
|
|
917
|
+
* marks the one failure the re-establishment budget spends whole (#998): a
|
|
918
|
+
* terminal that refuses to split will refuse every retry within the pass, so
|
|
919
|
+
* burning the budget once beats leaking an attempt per reconciliation pass
|
|
920
|
+
* against a wall. */
|
|
911
921
|
export type WorkerPaneOutcome =
|
|
912
|
-
| { kind: "tracked"; paneId: string; label: string; pid: number }
|
|
913
|
-
| { kind: "unavailable"; reason: string };
|
|
922
|
+
| { kind: "tracked"; paneId: string; label: string; pid: number; workspaceId?: string }
|
|
923
|
+
| { kind: "unavailable"; reason: string; phase?: "split" };
|
|
914
924
|
|
|
915
925
|
/** One `herdr` invocation, injected so every path is testable with no terminal. */
|
|
916
926
|
export type HerdrRun = (args: readonly string[]) => { ok: boolean; stdout: string; stderr: string };
|
|
@@ -920,6 +930,9 @@ export interface WorkerPaneDeps {
|
|
|
920
930
|
session?: string;
|
|
921
931
|
/** The follower command the pane displays; the CLI by default. */
|
|
922
932
|
viewer?: (identity: WorkerPaneIdentity) => readonly string[];
|
|
933
|
+
/** The durable ownership half of workspace discovery (#1035 review);
|
|
934
|
+
* production wires the conductor store, tests a recorder. */
|
|
935
|
+
ownership?: WorkspaceOwnership;
|
|
923
936
|
}
|
|
924
937
|
|
|
925
938
|
/**
|
|
@@ -932,9 +945,13 @@ export interface WorkerPaneDeps {
|
|
|
932
945
|
/**
|
|
933
946
|
* The external-supervisor source every conductor pane report carries.
|
|
934
947
|
*
|
|
935
|
-
*
|
|
936
|
-
*
|
|
937
|
-
*
|
|
948
|
+
* Written onto every pane conductor reports against. It is no longer what
|
|
949
|
+
* proves a pane is conductor's: production repeatedly handed back panes whose
|
|
950
|
+
* `agent_session` record Herdr had dropped (#1035), so filtering by reported
|
|
951
|
+
* source quietly exempted exactly the panes that needed cleaning. Ownership is
|
|
952
|
+
* structural now — conductor creates representations only inside a workspace
|
|
953
|
+
* it marked itself (see {@link WORKER_WORKSPACE_TOKEN}) — and this source
|
|
954
|
+
* remains on the reports as the display identity of who speaks for a pane.
|
|
938
955
|
*/
|
|
939
956
|
export const WORKER_PANE_SOURCE = "omp-conductor";
|
|
940
957
|
|
|
@@ -950,6 +967,26 @@ export const WORKER_PANE_SOURCE = "omp-conductor";
|
|
|
950
967
|
*/
|
|
951
968
|
export const PANE_REATTEMPT_MAX = 3;
|
|
952
969
|
|
|
970
|
+
/**
|
|
971
|
+
* The workspace metadata token that marks a workspace as conductor's
|
|
972
|
+
* per-project worker surface (#1035).
|
|
973
|
+
*
|
|
974
|
+
* Written once at creation (`workspace report-metadata --token
|
|
975
|
+
* conductor_workers=<project>`) and read back from `workspace list`, so
|
|
976
|
+
* rediscovery survives daemon and Herdr restarts without trusting the label,
|
|
977
|
+
* the sidebar position, or whichever workspace happens to be focused. Only
|
|
978
|
+
* panes inside a marked workspace are ever reconciled here, which is what makes
|
|
979
|
+
* the operator's own panes invisible to this surface by construction — and
|
|
980
|
+
* what makes every pane inside a marked workspace conductor's, however Herdr
|
|
981
|
+
* feels about reporting `agent_session` back.
|
|
982
|
+
*/
|
|
983
|
+
export const WORKER_WORKSPACE_TOKEN = "conductor_workers";
|
|
984
|
+
|
|
985
|
+
/** The dedicated sibling workspace's display label. Never identity. */
|
|
986
|
+
export function workerWorkspaceLabel(project: string): string {
|
|
987
|
+
return `${project}-workers`;
|
|
988
|
+
}
|
|
989
|
+
|
|
953
990
|
export function workerPaneLabel(identity: WorkerPaneIdentity): string {
|
|
954
991
|
return `worker-${identity.project}-${identity.issue}-a${identity.attempt}-${identity.runId.slice(0, 8)}`;
|
|
955
992
|
}
|
|
@@ -997,41 +1034,402 @@ function parsePaneId(stdout: string): string | undefined {
|
|
|
997
1034
|
return typeof paneId === "string" && paneId.trim() !== "" ? paneId.trim() : undefined;
|
|
998
1035
|
}
|
|
999
1036
|
|
|
1037
|
+
/**
|
|
1038
|
+
* The workspace and root-pane ids inside `herdr workspace create`'s answer
|
|
1039
|
+
* (`{"id":"cli:workspace:create","result":{"workspace":{…},"root_pane":{…}}}`).
|
|
1040
|
+
* The root pane is the anchor every later split targets, so a create answer
|
|
1041
|
+
* without one is only half a workspace — the caller cleans it up rather than
|
|
1042
|
+
* guessing at a pane to split.
|
|
1043
|
+
*/
|
|
1044
|
+
function parseCreatedWorkspace(stdout: string): { workspaceId: string; rootPaneId?: string } | undefined {
|
|
1045
|
+
const result = envelopeOf(stdout);
|
|
1046
|
+
if (result === undefined) return undefined;
|
|
1047
|
+
const workspace = result["workspace"];
|
|
1048
|
+
if (workspace === null || typeof workspace !== "object") return undefined;
|
|
1049
|
+
const workspaceId = (workspace as Record<string, unknown>)["workspace_id"];
|
|
1050
|
+
if (typeof workspaceId !== "string" || workspaceId.trim() === "") return undefined;
|
|
1051
|
+
const root = (result as Record<string, unknown>)["root_pane"];
|
|
1052
|
+
const rootPaneId =
|
|
1053
|
+
root !== null && typeof root === "object"
|
|
1054
|
+
? (root as Record<string, unknown>)["pane_id"]
|
|
1055
|
+
: undefined;
|
|
1056
|
+
return {
|
|
1057
|
+
workspaceId: workspaceId.trim(),
|
|
1058
|
+
...(typeof rootPaneId === "string" && rootPaneId.trim() !== "" ? { rootPaneId: rootPaneId.trim() } : {}),
|
|
1059
|
+
};
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
/** The `{result: …}` object of a herdr CLI envelope, or nothing. */
|
|
1063
|
+
|
|
1064
|
+
function envelopeOf(stdout: string): Record<string, unknown> | undefined {
|
|
1065
|
+
const line = firstLine(stdout);
|
|
1066
|
+
if (line === "") return undefined;
|
|
1067
|
+
let payload: unknown;
|
|
1068
|
+
try {
|
|
1069
|
+
payload = JSON.parse(line);
|
|
1070
|
+
} catch {
|
|
1071
|
+
return undefined;
|
|
1072
|
+
}
|
|
1073
|
+
if (payload === null || typeof payload !== "object") return undefined;
|
|
1074
|
+
const result = (payload as Record<string, unknown>)["result"];
|
|
1075
|
+
return result === null || typeof result !== "object" ? undefined : (result as Record<string, unknown>);
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
/** The root-pane id inside a `tab create` answer (`result.root_pane.pane_id`):
|
|
1079
|
+
* a tab answer carries no workspace object, so {@link parseCreatedWorkspace}
|
|
1080
|
+
* cannot read it. */
|
|
1081
|
+
function parseRootPane(stdout: string): string | undefined {
|
|
1082
|
+
const result = envelopeOf(stdout);
|
|
1083
|
+
const root = result?.["root_pane"];
|
|
1084
|
+
const paneId =
|
|
1085
|
+
root !== null && typeof root === "object" ? (root as Record<string, unknown>)["pane_id"] : undefined;
|
|
1086
|
+
return typeof paneId === "string" && paneId.trim() !== "" ? paneId.trim() : undefined;
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
/** Every workspace in a `workspace list` answer, with the tokens ownership is read from. */
|
|
1090
|
+
function parseWorkspaceList(stdout: string): { workspaces: { workspaceId: string; label?: string; tokens: Record<string, string> }[] } | undefined {
|
|
1091
|
+
const result = envelopeOf(stdout);
|
|
1092
|
+
const raw = result?.["workspaces"];
|
|
1093
|
+
if (!Array.isArray(raw)) return undefined;
|
|
1094
|
+
const workspaces: { workspaceId: string; label?: string; tokens: Record<string, string> }[] = [];
|
|
1095
|
+
for (const entry of raw) {
|
|
1096
|
+
if (entry === null || typeof entry !== "object") continue;
|
|
1097
|
+
const w = entry as Record<string, unknown>;
|
|
1098
|
+
if (typeof w["workspace_id"] !== "string" || w["workspace_id"].trim() === "") continue;
|
|
1099
|
+
const tokens: Record<string, string> = {};
|
|
1100
|
+
if (w["tokens"] !== null && typeof w["tokens"] === "object") {
|
|
1101
|
+
for (const [name, value] of Object.entries(w["tokens"] as Record<string, unknown>)) {
|
|
1102
|
+
if (typeof value === "string") tokens[name] = value;
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
workspaces.push({
|
|
1106
|
+
workspaceId: w["workspace_id"],
|
|
1107
|
+
...(typeof w["label"] === "string" ? { label: w["label"] } : {}),
|
|
1108
|
+
tokens,
|
|
1109
|
+
});
|
|
1110
|
+
}
|
|
1111
|
+
return { workspaces };
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
/** One pane record out of any `pane list` answer — global or `--workspace`-scoped. */
|
|
1115
|
+
export interface ListedPane {
|
|
1116
|
+
paneId: string;
|
|
1117
|
+
/** The run id Herdr still reports for this pane, or "" when it reports none. */
|
|
1118
|
+
runId: string;
|
|
1119
|
+
agent?: string;
|
|
1120
|
+
workspaceId?: string;
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
/** The panes of a `pane list` answer, carrying whichever fields reconciliation reads. */
|
|
1124
|
+
function parsePaneRecords(stdout: string): { panes: ListedPane[] } | undefined {
|
|
1125
|
+
const result = envelopeOf(stdout);
|
|
1126
|
+
const raw = result?.["panes"];
|
|
1127
|
+
if (!Array.isArray(raw)) return undefined;
|
|
1128
|
+
const panes: ListedPane[] = [];
|
|
1129
|
+
for (const entry of raw) {
|
|
1130
|
+
if (entry === null || typeof entry !== "object") continue;
|
|
1131
|
+
const p = entry as Record<string, unknown>;
|
|
1132
|
+
if (typeof p["pane_id"] !== "string" || p["pane_id"].trim() === "") continue;
|
|
1133
|
+
const session = p["agent_session"];
|
|
1134
|
+
const source = session !== null && typeof session === "object" ? (session as Record<string, unknown>)["source"] : undefined;
|
|
1135
|
+
const value = session !== null && typeof session === "object" ? (session as Record<string, unknown>)["value"] : undefined;
|
|
1136
|
+
panes.push({
|
|
1137
|
+
paneId: p["pane_id"].trim(),
|
|
1138
|
+
// Ours by construction inside a marked workspace; the reported session id
|
|
1139
|
+
// is a secondary signal only (#1035), because Herdr has been observed to
|
|
1140
|
+
// drop it while the pane lives on.
|
|
1141
|
+
runId: typeof value === "string" && source === WORKER_PANE_SOURCE ? value : "",
|
|
1142
|
+
...(typeof p["agent"] === "string" ? { agent: p["agent"] } : {}),
|
|
1143
|
+
...(typeof p["workspace_id"] === "string" ? { workspaceId: p["workspace_id"] } : {}),
|
|
1144
|
+
});
|
|
1145
|
+
}
|
|
1146
|
+
return { panes };
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
/** One conductor-marked worker workspace, as discovery returns it. */
|
|
1150
|
+
export interface WorkerWorkspace {
|
|
1151
|
+
workspaceId: string;
|
|
1152
|
+
/** The project the marking token binds it to — never derived from the label. */
|
|
1153
|
+
project: string;
|
|
1154
|
+
label?: string;
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
/**
|
|
1158
|
+
* The durable half of worker-workspace discovery (#1035 review).
|
|
1159
|
+
*
|
|
1160
|
+
* Herdr restores a session's workspaces and panes across a server restart but
|
|
1161
|
+
* drops `report-metadata` tokens — probed live: the restored worker workspace
|
|
1162
|
+
* comes back carrying no tokens at all. Token-only discovery would then ignore
|
|
1163
|
+
* the surface conductor created, build a duplicate beside it, and never be
|
|
1164
|
+
* able to reconcile or remove the restored one's stale panes — exactly the
|
|
1165
|
+
* accumulation this issue exists to end. Ownership is therefore two-legged:
|
|
1166
|
+
* the live token AND the conductor-store record of workspace ids the project
|
|
1167
|
+
* created. Either leg proves ownership; the store is the leg that survives.
|
|
1168
|
+
*/
|
|
1169
|
+
export interface WorkspaceOwnership {
|
|
1170
|
+
/** Store-recorded workspace ids for a project, oldest first. */
|
|
1171
|
+
recordedWorkspaces(project: string): readonly string[];
|
|
1172
|
+
rememberWorkspace(project: string, workspaceId: string): void;
|
|
1173
|
+
forgetWorkspace(project: string, workspaceId: string): void;
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1176
|
+
/** Every workspace in a `workspace list` answer, marked or not. */
|
|
1177
|
+
function listWorkspacesRaw(
|
|
1178
|
+
deps: { run?: HerdrRun; session?: string } = {},
|
|
1179
|
+
): {
|
|
1180
|
+
ok: true;
|
|
1181
|
+
workspaces: { workspaceId: string; label?: string; tokens: Record<string, string> }[];
|
|
1182
|
+
} | { ok: false; reason: string } {
|
|
1183
|
+
const run = deps.run ?? realHerdrRun;
|
|
1184
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
1185
|
+
const res = run(["--session", session, "workspace", "list"]);
|
|
1186
|
+
if (!res.ok) {
|
|
1187
|
+
return { ok: false, reason: `herdr workspace list failed: ${firstLine(res.stderr) || "no output"}` };
|
|
1188
|
+
}
|
|
1189
|
+
const parsed = parseWorkspaceList(res.stdout);
|
|
1190
|
+
if (parsed === undefined) return { ok: false, reason: "herdr workspace list was unreadable" };
|
|
1191
|
+
return { ok: true, workspaces: parsed.workspaces };
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
/**
|
|
1195
|
+
* Every conductor-marked worker workspace Herdr currently has, keyed by nothing:
|
|
1196
|
+
* the caller filters and scopes. Discovery reads the token, not the label — a
|
|
1197
|
+
* renamed workspace is still ours, and a lookalike label without the token is
|
|
1198
|
+
* not (#1035).
|
|
1199
|
+
*/
|
|
1200
|
+
export function listWorkerWorkspaces(deps: { run?: HerdrRun; session?: string } = {}): {
|
|
1201
|
+
ok: true;
|
|
1202
|
+
workspaces: WorkerWorkspace[];
|
|
1203
|
+
} | { ok: false; reason: string } {
|
|
1204
|
+
const listed = listWorkspacesRaw(deps);
|
|
1205
|
+
if (!listed.ok) return listed;
|
|
1206
|
+
const workspaces: WorkerWorkspace[] = [];
|
|
1207
|
+
for (const w of listed.workspaces) {
|
|
1208
|
+
const project = w.tokens[WORKER_WORKSPACE_TOKEN];
|
|
1209
|
+
if (project === undefined || project.trim() === "") continue;
|
|
1210
|
+
workspaces.push({
|
|
1211
|
+
workspaceId: w.workspaceId,
|
|
1212
|
+
project: project.trim(),
|
|
1213
|
+
...(w.label === undefined ? {} : { label: w.label }),
|
|
1214
|
+
});
|
|
1215
|
+
}
|
|
1216
|
+
return { ok: true, workspaces };
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
/**
|
|
1220
|
+
* Every worker workspace these projects own on the live Herdr (#1035 review):
|
|
1221
|
+
* token-marked ones plus store-recorded ids that still exist. A recorded id
|
|
1222
|
+
* Herdr no longer lists is dead weight rather than evidence — its workspace is
|
|
1223
|
+
* gone, and reconciliation's removal path is what forgets it.
|
|
1224
|
+
*/
|
|
1225
|
+
export function ownedWorkerWorkspaces(
|
|
1226
|
+
projects: readonly string[],
|
|
1227
|
+
deps: { run?: HerdrRun; session?: string; ownership?: WorkspaceOwnership } = {},
|
|
1228
|
+
): { ok: true; workspaces: WorkerWorkspace[] } | { ok: false; reason: string } {
|
|
1229
|
+
const raw = listWorkspacesRaw(deps);
|
|
1230
|
+
if (!raw.ok) return raw;
|
|
1231
|
+
const scope = new Set(projects);
|
|
1232
|
+
const owned = new Map<string, WorkerWorkspace>();
|
|
1233
|
+
for (const w of raw.workspaces) {
|
|
1234
|
+
const project = w.tokens[WORKER_WORKSPACE_TOKEN]?.trim();
|
|
1235
|
+
if (project === undefined || project === "" || !scope.has(project)) continue;
|
|
1236
|
+
owned.set(w.workspaceId, {
|
|
1237
|
+
workspaceId: w.workspaceId,
|
|
1238
|
+
project,
|
|
1239
|
+
...(w.label === undefined ? {} : { label: w.label }),
|
|
1240
|
+
});
|
|
1241
|
+
// Adopt token-owned surfaces into the durable leg even when no pane needs
|
|
1242
|
+
// opening this pass. Otherwise an upgrade followed by a Herdr restart can
|
|
1243
|
+
// drop the token before ensureWorkerWorkspace ever records the workspace.
|
|
1244
|
+
deps.ownership?.rememberWorkspace(project, w.workspaceId);
|
|
1245
|
+
}
|
|
1246
|
+
for (const project of projects) {
|
|
1247
|
+
for (const id of deps.ownership?.recordedWorkspaces(project) ?? []) {
|
|
1248
|
+
if (owned.has(id)) continue;
|
|
1249
|
+
const hit = raw.workspaces.find((w) => w.workspaceId === id);
|
|
1250
|
+
if (hit === undefined) {
|
|
1251
|
+
// The workspace disappeared outside conductor. Forget the stale id now
|
|
1252
|
+
// so a future Herdr session cannot reuse it as false ownership.
|
|
1253
|
+
deps.ownership?.forgetWorkspace(project, id);
|
|
1254
|
+
continue;
|
|
1255
|
+
}
|
|
1256
|
+
owned.set(id, { workspaceId: id, project, ...(hit.label === undefined ? {} : { label: hit.label }) });
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
return { ok: true, workspaces: [...owned.values()] };
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
export type WorkerWorkspaceResolution =
|
|
1263
|
+
| { kind: "ready"; workspaceId: string; anchorPaneId: string; created?: true }
|
|
1264
|
+
| { kind: "unavailable"; reason: string };
|
|
1265
|
+
|
|
1266
|
+
/**
|
|
1267
|
+
* Find this project's worker workspace or create it — the single door every
|
|
1268
|
+
* split goes through (#1035).
|
|
1269
|
+
*
|
|
1270
|
+
* Identity is two-legged now (#1035 review): the marking token OR the
|
|
1271
|
+
* conductor-store record, so rediscovery after any restart finds the same
|
|
1272
|
+
* workspace whatever Herdr preserved. Duplicates are deliberately NOT
|
|
1273
|
+
* resolved here: closing one blind — without listing its panes or checking
|
|
1274
|
+
* the result — could destroy the only live representation and strand the
|
|
1275
|
+
* durable row pointing at a dead pane. Convergence belongs to
|
|
1276
|
+
* {@link reconcileWorkerPanes}, which attributes every pane first and removes
|
|
1277
|
+
* a duplicate only once nothing living holds it. A discovered workspace with
|
|
1278
|
+
* no panes gets a fresh tab rather than failing forever; a workspace CREATED
|
|
1279
|
+
* by this call is closed again on every later failure, because newly-made
|
|
1280
|
+
* residue is exactly what the acceptance criterion forbids.
|
|
1281
|
+
*/
|
|
1282
|
+
export function ensureWorkerWorkspace(
|
|
1283
|
+
project: string,
|
|
1284
|
+
deps: { run?: HerdrRun; session?: string; ownership?: WorkspaceOwnership } = {},
|
|
1285
|
+
): WorkerWorkspaceResolution {
|
|
1286
|
+
const run = deps.run ?? realHerdrRun;
|
|
1287
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
1288
|
+
const base = ["--session", session];
|
|
1289
|
+
|
|
1290
|
+
let listed = ownedWorkerWorkspaces([project], deps);
|
|
1291
|
+
if (!listed.ok) return { kind: "unavailable", reason: listed.reason };
|
|
1292
|
+
|
|
1293
|
+
for (const existing of [...listed.workspaces].sort((a, b) => (a.workspaceId < b.workspaceId ? -1 : 1))) {
|
|
1294
|
+
const res = run([...base, "pane", "list", "--workspace", existing.workspaceId]);
|
|
1295
|
+
if (!res.ok) {
|
|
1296
|
+
return { kind: "unavailable", reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}` };
|
|
1297
|
+
}
|
|
1298
|
+
const parsed = parsePaneRecords(res.stdout);
|
|
1299
|
+
if (parsed === undefined) return { kind: "unavailable", reason: "herdr pane list was unreadable" };
|
|
1300
|
+
let anchor = parsed.panes[0]?.paneId;
|
|
1301
|
+
if (anchor === undefined) {
|
|
1302
|
+
// No pane to split: give the workspace a fresh tab and use its root pane.
|
|
1303
|
+
const tabbed = run([...base, "tab", "create", "--workspace", existing.workspaceId, "--no-focus"]);
|
|
1304
|
+
if (!tabbed.ok) {
|
|
1305
|
+
return { kind: "unavailable", reason: `herdr tab create failed: ${firstLine(tabbed.stderr) || "no output"}` };
|
|
1306
|
+
}
|
|
1307
|
+
anchor = parseRootPane(tabbed.stdout);
|
|
1308
|
+
if (anchor === undefined) {
|
|
1309
|
+
return { kind: "unavailable", reason: "herdr tab create reported no root pane" };
|
|
1310
|
+
}
|
|
1311
|
+
}
|
|
1312
|
+
// Adopted into the store: if this workspace's token is what Herdr drops
|
|
1313
|
+
// next restart, the record below is what still finds it.
|
|
1314
|
+
deps.ownership?.rememberWorkspace(project, existing.workspaceId);
|
|
1315
|
+
return { kind: "ready", workspaceId: existing.workspaceId, anchorPaneId: anchor };
|
|
1316
|
+
}
|
|
1317
|
+
|
|
1318
|
+
const made = run([
|
|
1319
|
+
...base,
|
|
1320
|
+
"workspace",
|
|
1321
|
+
"create",
|
|
1322
|
+
"--label",
|
|
1323
|
+
workerWorkspaceLabel(project),
|
|
1324
|
+
"--no-focus",
|
|
1325
|
+
]);
|
|
1326
|
+
if (!made.ok) {
|
|
1327
|
+
return { kind: "unavailable", reason: `herdr workspace create failed: ${firstLine(made.stderr) || "no output"}` };
|
|
1328
|
+
}
|
|
1329
|
+
const parsed = parseCreatedWorkspace(made.stdout);
|
|
1330
|
+
if (parsed === undefined) {
|
|
1331
|
+
return { kind: "unavailable", reason: "herdr workspace create reported no workspace id" };
|
|
1332
|
+
}
|
|
1333
|
+
/** This attempt made the workspace, so every exit from here takes it back. */
|
|
1334
|
+
const discardMade = (): void => {
|
|
1335
|
+
run([...base, "workspace", "close", parsed.workspaceId]);
|
|
1336
|
+
deps.ownership?.forgetWorkspace(project, parsed.workspaceId);
|
|
1337
|
+
};
|
|
1338
|
+
// Mark before use: an unmarked workspace would be invisible to token-side
|
|
1339
|
+
// discovery on the next pass — an empty shell accumulating precisely as
|
|
1340
|
+
// #1035 describes. A failure here takes the half-made workspace with it.
|
|
1341
|
+
const marked = run([
|
|
1342
|
+
...base,
|
|
1343
|
+
"workspace",
|
|
1344
|
+
"report-metadata",
|
|
1345
|
+
parsed.workspaceId,
|
|
1346
|
+
"--source",
|
|
1347
|
+
WORKER_PANE_SOURCE,
|
|
1348
|
+
"--token",
|
|
1349
|
+
`${WORKER_WORKSPACE_TOKEN}=${project}`,
|
|
1350
|
+
]);
|
|
1351
|
+
if (!marked.ok) {
|
|
1352
|
+
discardMade();
|
|
1353
|
+
return {
|
|
1354
|
+
kind: "unavailable",
|
|
1355
|
+
reason: `herdr workspace report-metadata failed: ${firstLine(marked.stderr) || "no output"}`,
|
|
1356
|
+
};
|
|
1357
|
+
}
|
|
1358
|
+
if (parsed.rootPaneId === undefined) {
|
|
1359
|
+
// No anchor means no split can ever be targeted; a fresh-made workspace
|
|
1360
|
+
// without one is residue this very attempt must not leave behind
|
|
1361
|
+
// (#1035 review) — so it goes now, not on some later pass.
|
|
1362
|
+
discardMade();
|
|
1363
|
+
return { kind: "unavailable", reason: "herdr workspace create reported no root pane" };
|
|
1364
|
+
}
|
|
1365
|
+
deps.ownership?.rememberWorkspace(project, parsed.workspaceId);
|
|
1366
|
+
return { kind: "ready", workspaceId: parsed.workspaceId, anchorPaneId: parsed.rootPaneId, created: true };
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1000
1369
|
/**
|
|
1001
1370
|
* Create the pane, run the follower in it, and report the child's identity and
|
|
1002
1371
|
* initial state — or say exactly why it could not.
|
|
1003
1372
|
*
|
|
1004
|
-
*
|
|
1005
|
-
*
|
|
1006
|
-
*
|
|
1007
|
-
*
|
|
1008
|
-
*
|
|
1009
|
-
*
|
|
1010
|
-
*
|
|
1373
|
+
* The representation lives in the project's worker workspace (#1035), found or
|
|
1374
|
+
* created first, and the split targets an anchor pane inside it explicitly, so
|
|
1375
|
+
* the operator's own layout is never touched. Every step is checked, and the
|
|
1376
|
+
* first failure returns `unavailable` with the reason and leaves ZERO residue
|
|
1377
|
+
* of this attempt (#1035 review): the pane the split had already created is
|
|
1378
|
+
* closed, and a workspace this call CREATED is closed with it — only surfaces
|
|
1379
|
+
* that already existed are preserved. A partially established representation
|
|
1380
|
+
* is worse than none: it is a pane the operator would read as a tracked
|
|
1381
|
+
* worker, and before #992 it was left behind on every single launch. What
|
|
1382
|
+
* this deliberately does NOT do is decide what a failure means for the launch
|
|
1383
|
+
* — failing closed versus running degraded is #841's policy, and inventing it
|
|
1384
|
+
* here would pre-empt it.
|
|
1011
1385
|
*/
|
|
1012
1386
|
export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDeps = {}): WorkerPaneOutcome {
|
|
1013
1387
|
const run = deps.run ?? realHerdrRun;
|
|
1014
1388
|
const session = deps.session ?? resolveHerdrSession();
|
|
1015
1389
|
const label = workerPaneLabel(identity);
|
|
1390
|
+
const ensured = ensureWorkerWorkspace(identity.project, deps);
|
|
1391
|
+
if (ensured.kind === "unavailable") return ensured;
|
|
1016
1392
|
const base = ["--session", session, "pane"];
|
|
1017
1393
|
|
|
1018
|
-
|
|
1394
|
+
/** Undo everything this attempt built so far — the split's pane once there
|
|
1395
|
+
* is one, and always a workspace this call itself created (#1035 review). */
|
|
1396
|
+
const abandon = (reason: string, phase?: "split", paneId?: string): WorkerPaneOutcome => {
|
|
1397
|
+
if (paneId !== undefined) run([...base, "close", paneId]);
|
|
1398
|
+
if (ensured.created === true) {
|
|
1399
|
+
run(["--session", session, "workspace", "close", ensured.workspaceId]);
|
|
1400
|
+
deps.ownership?.forgetWorkspace(identity.project, ensured.workspaceId);
|
|
1401
|
+
}
|
|
1402
|
+
return { kind: "unavailable", reason, ...(phase === undefined ? {} : { phase }) };
|
|
1403
|
+
};
|
|
1404
|
+
|
|
1405
|
+
// The explicit target is the whole point (#1035): the split names a pane
|
|
1406
|
+
// inside the worker workspace, so whichever pane the operator happens to be
|
|
1407
|
+
// looking at is never touched.
|
|
1408
|
+
const split = run([
|
|
1409
|
+
...base,
|
|
1410
|
+
"split",
|
|
1411
|
+
"--pane",
|
|
1412
|
+
ensured.anchorPaneId,
|
|
1413
|
+
"--direction",
|
|
1414
|
+
"down",
|
|
1415
|
+
"--ratio",
|
|
1416
|
+
"0.3",
|
|
1417
|
+
"--no-focus",
|
|
1418
|
+
]);
|
|
1019
1419
|
if (!split.ok) {
|
|
1020
|
-
return
|
|
1420
|
+
return abandon(
|
|
1421
|
+
`herdr pane split failed: ${firstLine(split.stderr) || "no output"}`,
|
|
1422
|
+
"split",
|
|
1423
|
+
);
|
|
1021
1424
|
}
|
|
1022
1425
|
const paneId = parsePaneId(split.stdout);
|
|
1023
1426
|
if (paneId === undefined) {
|
|
1024
|
-
return
|
|
1427
|
+
return abandon("herdr pane split reported no pane id");
|
|
1025
1428
|
}
|
|
1026
|
-
// The split has created a pane. From here every exit must take it with it.
|
|
1027
|
-
const abandon = (reason: string): WorkerPaneOutcome => {
|
|
1028
|
-
run([...base, "close", paneId]);
|
|
1029
|
-
return { kind: "unavailable", reason };
|
|
1030
|
-
};
|
|
1031
1429
|
|
|
1032
1430
|
const named = run([...base, "rename", paneId, label]);
|
|
1033
1431
|
if (!named.ok) {
|
|
1034
|
-
return abandon(`herdr pane rename failed: ${firstLine(named.stderr) || "no output"}
|
|
1432
|
+
return abandon(`herdr pane rename failed: ${firstLine(named.stderr) || "no output"}`, undefined, paneId);
|
|
1035
1433
|
}
|
|
1036
1434
|
|
|
1037
1435
|
// Identity before display: the pane must be attributable to this exact run
|
|
@@ -1052,12 +1450,16 @@ export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDep
|
|
|
1052
1450
|
WORKER_PANE_SOURCE,
|
|
1053
1451
|
]);
|
|
1054
1452
|
if (!identified.ok) {
|
|
1055
|
-
return abandon(
|
|
1453
|
+
return abandon(
|
|
1454
|
+
`herdr pane report-agent-session failed: ${firstLine(identified.stderr) || "no output"}`,
|
|
1455
|
+
undefined,
|
|
1456
|
+
paneId,
|
|
1457
|
+
);
|
|
1056
1458
|
}
|
|
1057
1459
|
|
|
1058
1460
|
const started = run([...base, "run", paneId, ...deps.viewer?.(identity) ?? defaultViewer(identity)]);
|
|
1059
1461
|
if (!started.ok) {
|
|
1060
|
-
return abandon(`herdr pane run failed: ${firstLine(started.stderr) || "no output"}
|
|
1462
|
+
return abandon(`herdr pane run failed: ${firstLine(started.stderr) || "no output"}`, undefined, paneId);
|
|
1061
1463
|
}
|
|
1062
1464
|
|
|
1063
1465
|
// A worker that has just spawned is working by definition. The ongoing
|
|
@@ -1065,9 +1467,9 @@ export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDep
|
|
|
1065
1467
|
// events — never from the pane's output.
|
|
1066
1468
|
const reported = reportWorkerPaneState(paneId, label, "working", { run, session });
|
|
1067
1469
|
if (!reported.ok) {
|
|
1068
|
-
return abandon(reported.reason);
|
|
1470
|
+
return abandon(reported.reason, undefined, paneId);
|
|
1069
1471
|
}
|
|
1070
|
-
return { kind: "tracked", paneId, label, pid: identity.pid };
|
|
1472
|
+
return { kind: "tracked", paneId, label, pid: identity.pid, workspaceId: ensured.workspaceId };
|
|
1071
1473
|
}
|
|
1072
1474
|
|
|
1073
1475
|
/** Report one lifecycle state for a tracked pane. Monotonic `seq` is the caller's. */
|
|
@@ -1100,12 +1502,11 @@ export function reportWorkerPaneState(
|
|
|
1100
1502
|
}
|
|
1101
1503
|
|
|
1102
1504
|
/**
|
|
1103
|
-
* Hand lifecycle authority back when
|
|
1104
|
-
*
|
|
1105
|
-
*
|
|
1106
|
-
*
|
|
1107
|
-
*
|
|
1108
|
-
* no longer speaks for this agent", which is exactly what is true.
|
|
1505
|
+
* Hand lifecycle authority back when conductor no longer speaks for a pane's
|
|
1506
|
+
* agent. Half of {@link retireWorkerPane}: it says "this agent record is no
|
|
1507
|
+
* longer ours" without removing anything, which is what reconciliation wants
|
|
1508
|
+
* for a pane it is about to close anyway but also stands alone for callers
|
|
1509
|
+
* that only re-report identity (#842's adopt path).
|
|
1109
1510
|
*/
|
|
1110
1511
|
export function releaseWorkerPane(
|
|
1111
1512
|
paneId: string,
|
|
@@ -1131,71 +1532,153 @@ export function releaseWorkerPane(
|
|
|
1131
1532
|
: { ok: false, reason: `herdr pane release-agent failed: ${firstLine(res.stderr) || "no output"}` };
|
|
1132
1533
|
}
|
|
1133
1534
|
|
|
1535
|
+
/** Close one pane outright. The pane holds only ever a follower, so closing
|
|
1536
|
+
* it cannot reach the authoritative worker — that child belongs to the
|
|
1537
|
+
* daemon's process tree, not to any terminal (#1035). */
|
|
1538
|
+
export function closeWorkerPane(
|
|
1539
|
+
paneId: string,
|
|
1540
|
+
deps: { run?: HerdrRun; session?: string } = {},
|
|
1541
|
+
): { ok: true } | { ok: false; reason: string } {
|
|
1542
|
+
const run = deps.run ?? realHerdrRun;
|
|
1543
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
1544
|
+
const res = run(["--session", session, "pane", "close", paneId]);
|
|
1545
|
+
return res.ok
|
|
1546
|
+
? { ok: true }
|
|
1547
|
+
: { ok: false, reason: `herdr pane close failed: ${firstLine(res.stderr) || "no output"}` };
|
|
1548
|
+
}
|
|
1549
|
+
|
|
1550
|
+
/**
|
|
1551
|
+
* Retire one run's visual representation (#1035): hand lifecycle authority
|
|
1552
|
+
* back to Herdr, then close the pane.
|
|
1553
|
+
*
|
|
1554
|
+
* #841 originally stopped at release, and production showed what that leaves:
|
|
1555
|
+
* panes advertising `working` for runs that had settled hours earlier, their
|
|
1556
|
+
* follower long exited, a bare shell where a worker used to be. A settled run
|
|
1557
|
+
* keeps no representation. The close reaches only the follower — the
|
|
1558
|
+
* authoritative child was never in the pane, so nothing here can signal it.
|
|
1559
|
+
*
|
|
1560
|
+
* A close failure is reported, not swallowed: the next reconciliation pass
|
|
1561
|
+
* sees the pane as unclaimed junk and closes it again, so one refusal costs a
|
|
1562
|
+
* pass rather than the representation.
|
|
1563
|
+
*/
|
|
1564
|
+
export function retireWorkerPane(
|
|
1565
|
+
paneId: string,
|
|
1566
|
+
label: string,
|
|
1567
|
+
deps: { run?: HerdrRun; session?: string; seq?: number } = {},
|
|
1568
|
+
): { ok: true } | { ok: false; reason: string } {
|
|
1569
|
+
const released = releaseWorkerPane(paneId, label, deps);
|
|
1570
|
+
if (!released.ok) return released;
|
|
1571
|
+
return closeWorkerPane(paneId, deps);
|
|
1572
|
+
}
|
|
1573
|
+
|
|
1134
1574
|
/**
|
|
1135
|
-
* Hand back the
|
|
1575
|
+
* Hand back the representation a dead worker left behind (#842), retired for
|
|
1576
|
+
* good by closing it (#1035) — but only once the recorded pane is PROVEN still
|
|
1577
|
+
* conductor's (#1035 review).
|
|
1136
1578
|
*
|
|
1137
1579
|
* Called for every run a restart reaps. The recorded pid is deliberately NOT
|
|
1138
1580
|
* consulted for liveness: a `session-host` child dies with the daemon that owned
|
|
1139
1581
|
* its verb socket, so an orphaned row's worker is gone whatever pid it carries —
|
|
1140
1582
|
* and pids are reused, so checking one is how a stranger's process comes to read
|
|
1141
|
-
* as a live worker.
|
|
1142
|
-
*
|
|
1583
|
+
* as a live worker. The same reuse argument applies to pane ids: after a Herdr
|
|
1584
|
+
* restart the stored `w2:pT` may belong to a pane in somebody else's workspace,
|
|
1585
|
+
* so a blind release+close could destroy a stranger's terminal. Ownership is
|
|
1586
|
+
* therefore proven before anything is closed:
|
|
1587
|
+
*
|
|
1588
|
+
* - the pane must be listed inside a workspace conductor OWNS — a
|
|
1589
|
+
* {@link WORKER_WORKSPACE_TOKEN}-marked one, or one the store recorded for
|
|
1590
|
+
* the run's project (#1035 review: Herdr restarts restore workspaces without
|
|
1591
|
+
* their tokens, and a token-only check would leave the restored pane
|
|
1592
|
+
* unretired forever); and
|
|
1143
1593
|
*/
|
|
1144
1594
|
export function releaseOrphanedWorkerPane(
|
|
1145
|
-
run: { paneId?: string; paneLabel?: string },
|
|
1146
|
-
deps: { run?: HerdrRun; session?: string; seq?: number } = {},
|
|
1147
|
-
):
|
|
1595
|
+
run: { paneId?: string; paneLabel?: string; runId?: string; project?: string },
|
|
1596
|
+
deps: { run?: HerdrRun; session?: string; seq?: number; ownership?: WorkspaceOwnership } = {},
|
|
1597
|
+
):
|
|
1598
|
+
| { kind: "none" }
|
|
1599
|
+
| { kind: "released"; paneId: string }
|
|
1600
|
+
| { kind: "failed"; paneId: string; reason: string }
|
|
1601
|
+
| { kind: "unowned"; paneId: string; reason: string } {
|
|
1148
1602
|
if (run.paneId === undefined || run.paneLabel === undefined) return { kind: "none" };
|
|
1149
|
-
const
|
|
1150
|
-
|
|
1603
|
+
const discovered = ownedWorkerWorkspaces(run.project === undefined ? [] : [run.project], deps);
|
|
1604
|
+
if (!discovered.ok) {
|
|
1605
|
+
return { kind: "failed", paneId: run.paneId, reason: discovered.reason };
|
|
1606
|
+
}
|
|
1607
|
+
const runFn = deps.run ?? realHerdrRun;
|
|
1608
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
1609
|
+
let home: string | undefined;
|
|
1610
|
+
let reportedAgent: string | undefined;
|
|
1611
|
+
let reportedSession: string | undefined;
|
|
1612
|
+
for (const ws of discovered.workspaces) {
|
|
1613
|
+
const res = runFn(["--session", session, "pane", "list", "--workspace", ws.workspaceId]);
|
|
1614
|
+
if (!res.ok) {
|
|
1615
|
+
return {
|
|
1616
|
+
kind: "failed",
|
|
1617
|
+
paneId: run.paneId,
|
|
1618
|
+
reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}`,
|
|
1619
|
+
};
|
|
1620
|
+
}
|
|
1621
|
+
const parsed = parsePaneRecords(res.stdout);
|
|
1622
|
+
if (parsed === undefined) {
|
|
1623
|
+
return { kind: "failed", paneId: run.paneId, reason: "herdr pane list was unreadable" };
|
|
1624
|
+
}
|
|
1625
|
+
const found = parsed.panes.find((pane) => pane.paneId === run.paneId);
|
|
1626
|
+
if (found !== undefined && home === undefined) {
|
|
1627
|
+
home = ws.workspaceId;
|
|
1628
|
+
if (found.agent !== undefined) reportedAgent = found.agent;
|
|
1629
|
+
if (found.runId !== "") reportedSession = found.runId;
|
|
1630
|
+
}
|
|
1631
|
+
}
|
|
1632
|
+
if (home === undefined) {
|
|
1633
|
+
// Not in any marked workspace: either already retired or never ours to
|
|
1634
|
+
// touch. Both answers close nothing.
|
|
1635
|
+
return { kind: "none" };
|
|
1636
|
+
}
|
|
1637
|
+
if (
|
|
1638
|
+
(reportedAgent !== undefined && reportedAgent !== run.paneLabel) ||
|
|
1639
|
+
(reportedSession !== undefined && run.runId !== undefined && reportedSession !== run.runId)
|
|
1640
|
+
) {
|
|
1641
|
+
return {
|
|
1642
|
+
kind: "unowned",
|
|
1643
|
+
paneId: run.paneId,
|
|
1644
|
+
reason: `pane ${run.paneId} in marked workspace ${home} reports another identity (agent ${reportedAgent ?? "none"}, session ${reportedSession ?? "none"})`,
|
|
1645
|
+
};
|
|
1646
|
+
}
|
|
1647
|
+
const retired = retireWorkerPane(run.paneId, run.paneLabel, deps);
|
|
1648
|
+
return retired.ok
|
|
1151
1649
|
? { kind: "released", paneId: run.paneId }
|
|
1152
|
-
: { kind: "failed", paneId: run.paneId, reason:
|
|
1650
|
+
: { kind: "failed", paneId: run.paneId, reason: retired.reason };
|
|
1153
1651
|
}
|
|
1154
1652
|
|
|
1155
1653
|
/**
|
|
1156
|
-
* Every
|
|
1157
|
-
* reported under (#841).
|
|
1654
|
+
* Every pane inside a conductor-marked worker workspace (#1035).
|
|
1158
1655
|
*
|
|
1159
|
-
*
|
|
1160
|
-
*
|
|
1161
|
-
*
|
|
1162
|
-
*
|
|
1656
|
+
* Ownership here is structural: conductor created the workspace, marked it,
|
|
1657
|
+
* and splits into it exclusively, so every pane inside it is conductor's
|
|
1658
|
+
* however Herdr feels about reporting `agent_session` back — production has
|
|
1659
|
+
* repeatedly handed back worker panes with that record dropped to null, which
|
|
1660
|
+
* is why the previous source-filtered listing could not see the very panes
|
|
1661
|
+
* that needed cleaning. The reported session id still rides along as a
|
|
1662
|
+
* secondary attribution signal; an empty run id means "unattributable", and
|
|
1663
|
+
* unattributable panes in a marked workspace are exactly the stale shells
|
|
1664
|
+
* this surface exists to converge away.
|
|
1163
1665
|
*/
|
|
1164
1666
|
export function listWorkerPanes(
|
|
1165
1667
|
deps: { run?: HerdrRun; session?: string } = {},
|
|
1166
|
-
): { ok: true; panes:
|
|
1668
|
+
): { ok: true; panes: (ListedPane & { project: string })[] } | { ok: false; reason: string } {
|
|
1669
|
+
const workspaces = listWorkerWorkspaces(deps);
|
|
1670
|
+
if (!workspaces.ok) return workspaces;
|
|
1167
1671
|
const run = deps.run ?? realHerdrRun;
|
|
1168
1672
|
const session = deps.session ?? resolveHerdrSession();
|
|
1169
|
-
const
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
parsed =
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
}
|
|
1179
|
-
const panes = (parsed as { result?: { panes?: unknown[] } }).result?.panes;
|
|
1180
|
-
if (!Array.isArray(panes)) return { ok: false, reason: "herdr pane list carried no pane array" };
|
|
1181
|
-
const owned: { paneId: string; runId: string; label?: string }[] = [];
|
|
1182
|
-
for (const pane of panes) {
|
|
1183
|
-
const p = pane as {
|
|
1184
|
-
pane_id?: unknown;
|
|
1185
|
-
agent?: unknown;
|
|
1186
|
-
agent_session?: { source?: unknown; value?: unknown };
|
|
1187
|
-
};
|
|
1188
|
-
if (typeof p.pane_id !== "string") continue;
|
|
1189
|
-
if (p.agent_session?.source !== WORKER_PANE_SOURCE) continue;
|
|
1190
|
-
const runId = p.agent_session.value;
|
|
1191
|
-
// Ours by source but carrying no run id: an identity nobody can resolve is
|
|
1192
|
-
// not an identity. Reported as a pane with an empty run id so the caller
|
|
1193
|
-
// treats it as stale rather than silently ignoring it.
|
|
1194
|
-
owned.push({
|
|
1195
|
-
paneId: p.pane_id,
|
|
1196
|
-
runId: typeof runId === "string" ? runId : "",
|
|
1197
|
-
...(typeof p.agent === "string" ? { label: p.agent } : {}),
|
|
1198
|
-
});
|
|
1673
|
+
const owned: (ListedPane & { project: string })[] = [];
|
|
1674
|
+
for (const ws of workspaces.workspaces) {
|
|
1675
|
+
const res = run(["--session", session, "pane", "list", "--workspace", ws.workspaceId]);
|
|
1676
|
+
if (!res.ok) {
|
|
1677
|
+
return { ok: false, reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}` };
|
|
1678
|
+
}
|
|
1679
|
+
const parsed = parsePaneRecords(res.stdout);
|
|
1680
|
+
if (parsed === undefined) return { ok: false, reason: "herdr pane list carried no pane array" };
|
|
1681
|
+
for (const pane of parsed.panes) owned.push({ ...pane, project: ws.project });
|
|
1199
1682
|
}
|
|
1200
1683
|
return { ok: true, panes: owned };
|
|
1201
1684
|
}
|
|
@@ -1213,43 +1696,147 @@ export interface LiveWorkerPane {
|
|
|
1213
1696
|
sessionFile?: string;
|
|
1214
1697
|
}
|
|
1215
1698
|
|
|
1699
|
+
export type StaleCause = "settled" | "duplicate" | "unidentified";
|
|
1700
|
+
|
|
1216
1701
|
export type WorkerPaneReconciliation =
|
|
1217
|
-
/** The recorded pane is still there and
|
|
1702
|
+
/** The recorded pane is still there and its follower is alive — nothing done. */
|
|
1218
1703
|
| { kind: "intact"; runId: string; paneId: string }
|
|
1219
|
-
/**
|
|
1704
|
+
/** The pane stands but its follower table could not decide life or death:
|
|
1705
|
+
* no evidence, no destruction (#1035 review). The visual may be stale; the
|
|
1706
|
+
* caller says so once instead of gambling a live worker's representation. */
|
|
1707
|
+
| { kind: "visual-unreadable"; runId: string; paneId: string; reason: string }
|
|
1708
|
+
/** The run had no usable representation; a new one now represents the same
|
|
1709
|
+
* child — a Herdr restart, a missing pane, or a follower that had exited. */
|
|
1220
1710
|
| { kind: "reassociated"; runId: string; paneId: string; label: string }
|
|
1221
1711
|
/** No representation, and the reason. The run keeps working regardless.
|
|
1222
1712
|
* `attempted` marks the ones that spent a re-establishment attempt (#998),
|
|
1223
|
-
* so the caller can bound them; the no-pid case costs nothing.
|
|
1224
|
-
|
|
1713
|
+
* so the caller can bound them; the no-pid case costs nothing. `phase`
|
|
1714
|
+
* carries {@link WorkerPaneOutcome}'s split marker through. */
|
|
1715
|
+
| { kind: "untracked"; runId: string; reason: string; attempted?: true; phase?: "split" }
|
|
1225
1716
|
/** The attempt budget for this run is spent, so nothing was tried this pass
|
|
1226
1717
|
* (#998). Before this existed, a run whose pane could not be created was
|
|
1227
1718
|
* retried on every reconciliation pass forever — and each attempt leaked a
|
|
1228
1719
|
* pane until #992, ending in `ghostty error -2` once enough had piled up. */
|
|
1229
1720
|
| { kind: "attempts-exhausted"; runId: string; attempts: number }
|
|
1230
|
-
/** A conductor pane
|
|
1231
|
-
|
|
1232
|
-
|
|
1721
|
+
/** A conductor-workspace pane with no live run behind it: authority handed
|
|
1722
|
+
* back and the pane closed. `settled` — the run finished and its retirement
|
|
1723
|
+
* never landed; `duplicate` — a second representation of one live run, or
|
|
1724
|
+
* the one a fresh replacement displaced; `unidentified` — a representation
|
|
1725
|
+
* whose run id nobody can resolve any more, the bare shells production
|
|
1726
|
+
* accumulated (#1035). */
|
|
1727
|
+
| { kind: "stale-released"; paneId: string; runId: string; cause: StaleCause }
|
|
1728
|
+
| { kind: "stale-release-failed"; paneId: string; runId: string; reason: string; cause: StaleCause }
|
|
1729
|
+
/** A scoped marked workspace nobody's live run holds any more, emptied by
|
|
1730
|
+
* the cleanup above, is closed (#1035). Every marked workspace converges,
|
|
1731
|
+
* not just one per project (#1035 review). */
|
|
1732
|
+
| { kind: "workspace-removed"; project: string; workspaceId: string }
|
|
1733
|
+
| { kind: "workspace-remove-failed"; project: string; workspaceId: string; reason: string };
|
|
1734
|
+
|
|
1735
|
+
/** The verdict the foreground process table supports about one pane's
|
|
1736
|
+
* read-only follower. Terminal output is never read: this says whether the
|
|
1737
|
+
* *visual* is alive, nothing more, and settlement stays the store's business
|
|
1738
|
+
* (#1035). `unknown` is a real verdict, not an error — recognition of
|
|
1739
|
+
* `omp-conductor` in the table is positive evidence of life, but its absence
|
|
1740
|
+
* is only evidence of death when every entry carries enough identity (name or
|
|
1741
|
+
* argv) to rule it out. Real Herdr answers name + argv; a table with blanked
|
|
1742
|
+
* or missing fields, no shell pid, or no entries at all cannot decide, and a
|
|
1743
|
+
* pane destroyed on a guess we did not have is a live worker gone blind. */
|
|
1744
|
+
export type FollowerPulse =
|
|
1745
|
+
| { pulse: "alive" }
|
|
1746
|
+
| { pulse: "dead" }
|
|
1747
|
+
| { pulse: "unknown"; reason: string };
|
|
1748
|
+
|
|
1749
|
+
export function followerPulseFromProcessInfo(info: ProcessInfo): FollowerPulse {
|
|
1750
|
+
const procs = Array.isArray(info.foreground_processes) ? info.foreground_processes : undefined;
|
|
1751
|
+
if (procs === undefined || procs.length === 0) {
|
|
1752
|
+
return { pulse: "unknown", reason: "process-info carried no foreground process table" };
|
|
1753
|
+
}
|
|
1754
|
+
const shell = typeof info.shell_pid === "number" ? info.shell_pid : undefined;
|
|
1755
|
+
let nonShell = false;
|
|
1756
|
+
for (const proc of procs) {
|
|
1757
|
+
const parts = [proc.name ?? "", proc.argv0 ?? "", ...(proc.argv ?? [])];
|
|
1758
|
+
const isShell = shell !== undefined && proc.pid === shell;
|
|
1759
|
+
if (!isShell && !parts.some((a) => a !== "")) {
|
|
1760
|
+
return { pulse: "unknown", reason: `foreground pid ${proc.pid ?? "unknown"} carries no name or argv` };
|
|
1761
|
+
}
|
|
1762
|
+
if (!isShell) {
|
|
1763
|
+
nonShell = true;
|
|
1764
|
+
if (parts.some((a) => a.toLowerCase() === "omp-conductor" || a.toLowerCase().endsWith("/omp-conductor"))) {
|
|
1765
|
+
return { pulse: "alive" };
|
|
1766
|
+
}
|
|
1767
|
+
}
|
|
1768
|
+
}
|
|
1769
|
+
if (!nonShell && shell === undefined) {
|
|
1770
|
+
return { pulse: "unknown", reason: "process-info named neither a shell nor any identifiable process" };
|
|
1771
|
+
}
|
|
1772
|
+
return { pulse: "dead" };
|
|
1773
|
+
}
|
|
1774
|
+
|
|
1775
|
+
/**
|
|
1776
|
+
* Probe one pane's follower liveness. A probe that cannot be read, or a table
|
|
1777
|
+
* that cannot decide, answers `unknown` rather than dead: destroying a
|
|
1778
|
+
* representation on evidence we do not have is how a healthy worker goes blind
|
|
1779
|
+
* mid-run.
|
|
1780
|
+
*/
|
|
1781
|
+
export function paneFollowerAlive(
|
|
1782
|
+
paneId: string,
|
|
1783
|
+
deps: { run?: HerdrRun; session?: string } = {},
|
|
1784
|
+
): FollowerPulse {
|
|
1785
|
+
const run = deps.run ?? realHerdrRun;
|
|
1786
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
1787
|
+
const res = run(["--session", session, "pane", "process-info", "--pane", paneId]);
|
|
1788
|
+
if (!res.ok) {
|
|
1789
|
+
return {
|
|
1790
|
+
pulse: "unknown",
|
|
1791
|
+
reason: `herdr pane process-info failed: ${firstLine(res.stderr) || "no output"}`,
|
|
1792
|
+
};
|
|
1793
|
+
}
|
|
1794
|
+
try {
|
|
1795
|
+
return followerPulseFromProcessInfo(parseHerdrProcessInfo(res.stdout, paneId));
|
|
1796
|
+
} catch (err) {
|
|
1797
|
+
return { pulse: "unknown", reason: err instanceof Error ? err.message : String(err) };
|
|
1798
|
+
}
|
|
1799
|
+
}
|
|
1800
|
+
|
|
1801
|
+
function firstLine(text: string): string {
|
|
1802
|
+
return text.split("\n", 1)[0]?.trim() ?? "";
|
|
1803
|
+
}
|
|
1804
|
+
/** The kill syscall, injectable so error mapping is testable without one. */
|
|
1805
|
+
export type KillFn = (pid: number, sig: NodeJS.Signals | 0) => void;
|
|
1806
|
+
|
|
1807
|
+
const realKill: KillFn = (pid, sig) => {
|
|
1808
|
+
process.kill(pid, sig);
|
|
1809
|
+
};
|
|
1233
1810
|
|
|
1234
1811
|
/**
|
|
1235
|
-
* Make
|
|
1812
|
+
* Make the conductor-owned worker workspaces agree with the live run set
|
|
1813
|
+
* (#841, reworked by #1035).
|
|
1236
1814
|
*
|
|
1237
1815
|
* Idempotent by construction: a second pass over an already-reconciled fleet
|
|
1238
|
-
* returns `intact` for every live run and finds
|
|
1239
|
-
*
|
|
1816
|
+
* returns `intact` for every live run and finds nothing left to close, so
|
|
1817
|
+
* repeated launch/settlement/restart cycles converge instead of accumulating.
|
|
1240
1818
|
*
|
|
1241
|
-
*
|
|
1819
|
+
* Four rules, and each exists to refuse a specific way this goes wrong:
|
|
1242
1820
|
*
|
|
1243
|
-
* - **
|
|
1244
|
-
*
|
|
1245
|
-
*
|
|
1246
|
-
*
|
|
1247
|
-
*
|
|
1248
|
-
*
|
|
1249
|
-
*
|
|
1250
|
-
*
|
|
1251
|
-
*
|
|
1252
|
-
*
|
|
1821
|
+
* - **Only owned workspaces are touched.** Scope is the projects of the live
|
|
1822
|
+
* set plus the caller's managed list; ownership is the marking token OR the
|
|
1823
|
+
* conductor-store record (#1035 review), so a Herdr restart that drops
|
|
1824
|
+
* tokens still finds the restored surface. A pane in another project's
|
|
1825
|
+
* worker workspace is invisible to this pass, which is what keeps conductor
|
|
1826
|
+
* and veltrosecurity from discovering, reusing, or cleaning each other.
|
|
1827
|
+
* - **Attribution is exact, structural first.** A pane belongs to the live run
|
|
1828
|
+
* the store recorded it under; the session id Herdr reports is the fallback.
|
|
1829
|
+
* Everything else in an owned workspace — a settled run's leftover, a
|
|
1830
|
+
* duplicate of a live run, a representation whose identity Herdr dropped —
|
|
1831
|
+
* is closed, because inside an owned workspace "ours" and "junk" are the
|
|
1832
|
+
* only options and leaving junk is what filled the operator's layout.
|
|
1833
|
+
* - **A live run keeps exactly one living representation.** Duplicates fold
|
|
1834
|
+
* into the recorded pane; a pane whose follower has exited is replaced while
|
|
1835
|
+
* the run is live; a missing pane is recreated — all bounded by the same
|
|
1836
|
+
* re-establishment budget (#998).
|
|
1837
|
+
* - **Nothing here can stop a worker.** Panes hold followers only; the
|
|
1838
|
+
* authoritative child lives in the daemon's process tree and is never
|
|
1839
|
+
* signalled, and worker liveness comes from the caller's live set alone.
|
|
1253
1840
|
*/
|
|
1254
1841
|
export function reconcileWorkerPanes(
|
|
1255
1842
|
live: readonly LiveWorkerPane[],
|
|
@@ -1258,39 +1845,184 @@ export function reconcileWorkerPanes(
|
|
|
1258
1845
|
* and a daemon restart legitimately gets a fresh budget (#998). */
|
|
1259
1846
|
attempts: ReadonlyMap<string, number> = new Map(),
|
|
1260
1847
|
maxAttempts = PANE_REATTEMPT_MAX,
|
|
1848
|
+
/** Projects this pass manages even when they have no live runs: their stale
|
|
1849
|
+
* panes are cleaned and a workspace left empty by that cleanup is removed. */
|
|
1850
|
+
managedProjects: readonly string[] = [],
|
|
1261
1851
|
): { ok: true; outcomes: WorkerPaneReconciliation[] } | { ok: false; reason: string } {
|
|
1262
|
-
const
|
|
1263
|
-
|
|
1264
|
-
const
|
|
1265
|
-
|
|
1852
|
+
const run = deps.run ?? realHerdrRun;
|
|
1853
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
1854
|
+
const base = ["--session", session];
|
|
1855
|
+
|
|
1856
|
+
const scope = new Set([...managedProjects, ...live.map((worker) => worker.project)]);
|
|
1857
|
+
const discovered = ownedWorkerWorkspaces([...scope], deps);
|
|
1858
|
+
if (!discovered.ok) return { ok: false, reason: discovered.reason };
|
|
1859
|
+
// Deterministic order so outcomes and closes are reproducible pass to pass.
|
|
1860
|
+
const scoped = [...discovered.workspaces].sort((a, b) => (a.workspaceId < b.workspaceId ? -1 : 1));
|
|
1861
|
+
|
|
1862
|
+
// Every marked workspace of every scoped project converges (#1035 review):
|
|
1863
|
+
// duplicate token-marked workspaces are cleaned like any other, so a settled
|
|
1864
|
+
// project cannot keep stale shells in a second marked surface nobody tracks.
|
|
1865
|
+
const entries: { ws: WorkerWorkspace; panes: ListedPane[] }[] = [];
|
|
1866
|
+
for (const ws of scoped) {
|
|
1867
|
+
const res = run([...base, "pane", "list", "--workspace", ws.workspaceId]);
|
|
1868
|
+
if (!res.ok) {
|
|
1869
|
+
// An unreadable workspace is not evidence that anything is stale, so the
|
|
1870
|
+
// pass mutates nothing rather than closing on a guess.
|
|
1871
|
+
return { ok: false, reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}` };
|
|
1872
|
+
}
|
|
1873
|
+
const parsed = parsePaneRecords(res.stdout);
|
|
1874
|
+
if (parsed === undefined) return { ok: false, reason: "herdr pane list carried no pane array" };
|
|
1875
|
+
entries.push({ ws, panes: parsed.panes });
|
|
1876
|
+
}
|
|
1877
|
+
|
|
1878
|
+
const liveByRun = new Map(live.map((worker) => [worker.runId, worker]));
|
|
1266
1879
|
const outcomes: WorkerPaneReconciliation[] = [];
|
|
1880
|
+
/** Panes this pass will close, with the identity the outcome names. */
|
|
1881
|
+
const junk = new Map<string, { runId: string; cause: StaleCause; agent?: string }>();
|
|
1882
|
+
/** Panes still standing when cleanup ends, keyed to their workspace — an
|
|
1883
|
+
* occupied workspace survives, whichever marked surface it sits in. */
|
|
1884
|
+
const standingWs = new Map<string, string>();
|
|
1885
|
+
/** Workspaces that gained a representation this pass, read from the tracked
|
|
1886
|
+
* outcomes: their listing predates the new pane, so removal never judges
|
|
1887
|
+
* them on it. */
|
|
1888
|
+
const openedWs = new Set<string>();
|
|
1889
|
+
|
|
1890
|
+
/** One representation per live run: the pane plus the marked workspace that
|
|
1891
|
+
* holds it. Attribution runs across ALL of a project's marked workspaces,
|
|
1892
|
+
* because the recorded pane may sit in any of them. */
|
|
1893
|
+
const held = new Map<string, { pane: ListedPane; wsId: string }>();
|
|
1894
|
+
for (const entry of entries) {
|
|
1895
|
+
const projectLive = live.filter((worker) => worker.project === entry.ws.project);
|
|
1896
|
+
for (const pane of entry.panes) {
|
|
1897
|
+
standingWs.set(pane.paneId, entry.ws.workspaceId);
|
|
1898
|
+
const owner =
|
|
1899
|
+
projectLive.find((worker) => worker.paneId === pane.paneId) ??
|
|
1900
|
+
(pane.runId === "" ? undefined : liveByRun.get(pane.runId));
|
|
1901
|
+
if (owner === undefined || owner.project !== entry.ws.project) {
|
|
1902
|
+
// Not attributable to any live run of this project: a settled run's
|
|
1903
|
+
// leftover (still carrying its old id) or an unidentified shell.
|
|
1904
|
+
junk.set(pane.paneId, {
|
|
1905
|
+
runId: pane.runId,
|
|
1906
|
+
cause: pane.runId === "" ? "unidentified" : "settled",
|
|
1907
|
+
...(pane.agent === undefined ? {} : { agent: pane.agent }),
|
|
1908
|
+
});
|
|
1909
|
+
continue;
|
|
1910
|
+
}
|
|
1911
|
+
const bucket = held.get(owner.runId);
|
|
1912
|
+
if (bucket === undefined) {
|
|
1913
|
+
held.set(owner.runId, { pane, wsId: entry.ws.workspaceId });
|
|
1914
|
+
continue;
|
|
1915
|
+
}
|
|
1916
|
+
// Exactly one focusable representation per live run: the recorded pane
|
|
1917
|
+
// wins so the durable row stays true without a write; the other is a
|
|
1918
|
+
// duplicate (#1035).
|
|
1919
|
+
const keep =
|
|
1920
|
+
bucket.pane.paneId === owner.paneId
|
|
1921
|
+
? bucket
|
|
1922
|
+
: pane.paneId === owner.paneId
|
|
1923
|
+
? { pane, wsId: entry.ws.workspaceId }
|
|
1924
|
+
: bucket;
|
|
1925
|
+
const drop = keep.pane.paneId === bucket.pane.paneId ? pane : bucket.pane;
|
|
1926
|
+
held.set(owner.runId, keep);
|
|
1927
|
+
junk.set(drop.paneId, {
|
|
1928
|
+
runId: owner.runId,
|
|
1929
|
+
cause: "duplicate",
|
|
1930
|
+
...(drop.agent === undefined ? {} : { agent: drop.agent }),
|
|
1931
|
+
});
|
|
1932
|
+
}
|
|
1933
|
+
}
|
|
1267
1934
|
|
|
1268
1935
|
for (const worker of live) {
|
|
1269
|
-
const
|
|
1270
|
-
if (
|
|
1271
|
-
|
|
1936
|
+
const hold = held.get(worker.runId);
|
|
1937
|
+
if (hold === undefined) {
|
|
1938
|
+
// No representation anywhere in the marked workspaces. Only the pid makes
|
|
1939
|
+
// a replacement honest: the pane represents an exact child, so without
|
|
1940
|
+
// one there is nothing to represent and inventing a pane would be the
|
|
1941
|
+
// silent claim this whole surface exists to avoid.
|
|
1942
|
+
if (worker.pid === undefined) {
|
|
1943
|
+
outcomes.push({
|
|
1944
|
+
kind: "untracked",
|
|
1945
|
+
runId: worker.runId,
|
|
1946
|
+
reason: "no session-host pid was ever recorded for this run",
|
|
1947
|
+
});
|
|
1948
|
+
continue;
|
|
1949
|
+
}
|
|
1950
|
+
// The budget, checked before the attempt: an exhausted run is skipped
|
|
1951
|
+
// silently here and reported once by the caller (#998).
|
|
1952
|
+
const spent = attempts.get(worker.runId) ?? 0;
|
|
1953
|
+
if (spent >= maxAttempts) {
|
|
1954
|
+
outcomes.push({ kind: "attempts-exhausted", runId: worker.runId, attempts: spent });
|
|
1955
|
+
continue;
|
|
1956
|
+
}
|
|
1957
|
+
const opened = openWorkerPane(
|
|
1958
|
+
{
|
|
1959
|
+
project: worker.project,
|
|
1960
|
+
issue: worker.issue,
|
|
1961
|
+
attempt: worker.attempt,
|
|
1962
|
+
runId: worker.runId,
|
|
1963
|
+
pid: worker.pid,
|
|
1964
|
+
...(worker.sessionFile === undefined ? {} : { sessionFile: worker.sessionFile }),
|
|
1965
|
+
},
|
|
1966
|
+
deps,
|
|
1967
|
+
);
|
|
1968
|
+
if (opened.kind === "tracked") {
|
|
1969
|
+
if (opened.workspaceId !== undefined) openedWs.add(opened.workspaceId);
|
|
1970
|
+
outcomes.push({
|
|
1971
|
+
kind: "reassociated",
|
|
1972
|
+
runId: worker.runId,
|
|
1973
|
+
paneId: opened.paneId,
|
|
1974
|
+
label: opened.label,
|
|
1975
|
+
});
|
|
1976
|
+
} else {
|
|
1977
|
+
outcomes.push({
|
|
1978
|
+
kind: "untracked",
|
|
1979
|
+
runId: worker.runId,
|
|
1980
|
+
reason: opened.reason,
|
|
1981
|
+
attempted: true,
|
|
1982
|
+
...(opened.phase === undefined ? {} : { phase: opened.phase }),
|
|
1983
|
+
});
|
|
1984
|
+
}
|
|
1985
|
+
continue;
|
|
1986
|
+
}
|
|
1987
|
+
// The pane stands. Whether it still shows anything is the follower's
|
|
1988
|
+
// question, asked only to decide the *visual* — never settlement (#1035).
|
|
1989
|
+
// A probe that cannot be read, or a table that cannot decide, is no
|
|
1990
|
+
// evidence the visual died: it is reported and the pane keeps standing,
|
|
1991
|
+
// because a live worker's representation is never destroyed on a guess.
|
|
1992
|
+
const pulse = paneFollowerAlive(hold.pane.paneId, deps);
|
|
1993
|
+
if (pulse.pulse !== "dead") {
|
|
1994
|
+
outcomes.push(
|
|
1995
|
+
pulse.pulse === "alive"
|
|
1996
|
+
? { kind: "intact", runId: worker.runId, paneId: hold.pane.paneId }
|
|
1997
|
+
: {
|
|
1998
|
+
kind: "visual-unreadable",
|
|
1999
|
+
runId: worker.runId,
|
|
2000
|
+
paneId: hold.pane.paneId,
|
|
2001
|
+
reason: pulse.reason,
|
|
2002
|
+
},
|
|
2003
|
+
);
|
|
1272
2004
|
continue;
|
|
1273
2005
|
}
|
|
1274
|
-
//
|
|
1275
|
-
//
|
|
1276
|
-
//
|
|
1277
|
-
//
|
|
2006
|
+
// The follower has provably exited. A replacement represents the exact
|
|
2007
|
+
// child, so a run with no recorded pid keeps its dead pane rather than
|
|
2008
|
+
// gaining a representation of nobody — reported so the record drops and
|
|
2009
|
+
// the next pass reads the orphan as junk.
|
|
1278
2010
|
if (worker.pid === undefined) {
|
|
1279
2011
|
outcomes.push({
|
|
1280
2012
|
kind: "untracked",
|
|
1281
2013
|
runId: worker.runId,
|
|
1282
|
-
reason: "no session-host pid was ever recorded
|
|
2014
|
+
reason: "follower exited but no session-host pid was ever recorded; not replacing",
|
|
1283
2015
|
});
|
|
1284
2016
|
continue;
|
|
1285
2017
|
}
|
|
1286
|
-
// The budget, checked before the attempt: an exhausted run is skipped
|
|
1287
|
-
// silently here and reported once by the caller (#998).
|
|
1288
2018
|
const spent = attempts.get(worker.runId) ?? 0;
|
|
1289
2019
|
if (spent >= maxAttempts) {
|
|
2020
|
+
// Out of budget: the dead pane keeps its last state until a restart
|
|
2021
|
+
// refreshes the budget or the run settles and retires it outright.
|
|
1290
2022
|
outcomes.push({ kind: "attempts-exhausted", runId: worker.runId, attempts: spent });
|
|
1291
2023
|
continue;
|
|
1292
2024
|
}
|
|
1293
|
-
const
|
|
2025
|
+
const replaced = openWorkerPane(
|
|
1294
2026
|
{
|
|
1295
2027
|
project: worker.project,
|
|
1296
2028
|
issue: worker.issue,
|
|
@@ -1301,36 +2033,89 @@ export function reconcileWorkerPanes(
|
|
|
1301
2033
|
},
|
|
1302
2034
|
deps,
|
|
1303
2035
|
);
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
2036
|
+
if (replaced.kind === "tracked") {
|
|
2037
|
+
if (replaced.workspaceId !== undefined) openedWs.add(replaced.workspaceId);
|
|
2038
|
+
outcomes.push({
|
|
2039
|
+
kind: "reassociated",
|
|
2040
|
+
runId: worker.runId,
|
|
2041
|
+
paneId: replaced.paneId,
|
|
2042
|
+
label: replaced.label,
|
|
2043
|
+
});
|
|
2044
|
+
junk.set(hold.pane.paneId, {
|
|
2045
|
+
runId: worker.runId,
|
|
2046
|
+
cause: "duplicate",
|
|
2047
|
+
...(hold.pane.agent === undefined ? {} : { agent: hold.pane.agent }),
|
|
2048
|
+
});
|
|
2049
|
+
} else {
|
|
2050
|
+
outcomes.push({
|
|
2051
|
+
kind: "untracked",
|
|
2052
|
+
runId: worker.runId,
|
|
2053
|
+
reason: replaced.reason,
|
|
2054
|
+
attempted: true,
|
|
2055
|
+
...(replaced.phase === undefined ? {} : { phase: replaced.phase }),
|
|
2056
|
+
});
|
|
2057
|
+
// The replacement failed; the dead pane has nothing left to show, so it
|
|
2058
|
+
// goes now rather than advertising a worker for another pass. Its record
|
|
2059
|
+
// is dropped either way, so a survivor is junk next pass.
|
|
2060
|
+
closeWorkerPane(hold.pane.paneId, deps);
|
|
2061
|
+
standingWs.delete(hold.pane.paneId);
|
|
2062
|
+
}
|
|
1309
2063
|
}
|
|
1310
2064
|
|
|
1311
|
-
for (const
|
|
1312
|
-
|
|
1313
|
-
|
|
2065
|
+
for (const [paneId, info] of junk) {
|
|
2066
|
+
const released =
|
|
2067
|
+
info.agent === undefined ? ({ ok: true } as const) : releaseWorkerPane(paneId, info.agent, deps);
|
|
2068
|
+
if (released.ok) {
|
|
2069
|
+
const closed = closeWorkerPane(paneId, deps);
|
|
2070
|
+
if (closed.ok) {
|
|
2071
|
+
standingWs.delete(paneId);
|
|
2072
|
+
outcomes.push({ kind: "stale-released", paneId, runId: info.runId, cause: info.cause });
|
|
2073
|
+
continue;
|
|
2074
|
+
}
|
|
2075
|
+
outcomes.push({
|
|
2076
|
+
kind: "stale-release-failed",
|
|
2077
|
+
paneId,
|
|
2078
|
+
runId: info.runId,
|
|
2079
|
+
reason: closed.reason,
|
|
2080
|
+
cause: info.cause,
|
|
2081
|
+
});
|
|
2082
|
+
continue;
|
|
2083
|
+
}
|
|
2084
|
+
outcomes.push({
|
|
2085
|
+
kind: "stale-release-failed",
|
|
2086
|
+
paneId,
|
|
2087
|
+
runId: info.runId,
|
|
2088
|
+
reason: released.reason,
|
|
2089
|
+
cause: info.cause,
|
|
2090
|
+
});
|
|
2091
|
+
}
|
|
2092
|
+
|
|
2093
|
+
// Every scoped marked workspace whose panes the cleanup above emptied, and
|
|
2094
|
+
// that gained no representation this pass, is closed — however many marked
|
|
2095
|
+
// surfaces a project accumulated (#1035 review). A live project's primary
|
|
2096
|
+
// workspace survives because its representation keeps it standing; extras
|
|
2097
|
+
// converge away; repeated cycles hold at zero shells.
|
|
2098
|
+
for (const entry of entries) {
|
|
2099
|
+
if (openedWs.has(entry.ws.workspaceId)) continue;
|
|
2100
|
+
if ([...standingWs.values()].some((wsId) => wsId === entry.ws.workspaceId)) continue;
|
|
2101
|
+
const res = run([...base, "workspace", "close", entry.ws.workspaceId]);
|
|
2102
|
+
if (res.ok) deps.ownership?.forgetWorkspace(entry.ws.project, entry.ws.workspaceId);
|
|
2103
|
+
// A failed close keeps the record: the workspace may still exist, and
|
|
2104
|
+
// forgetting ownership of a live surface is how orphans are manufactured.
|
|
1314
2105
|
outcomes.push(
|
|
1315
|
-
|
|
1316
|
-
? { kind: "
|
|
1317
|
-
: {
|
|
2106
|
+
res.ok
|
|
2107
|
+
? { kind: "workspace-removed", project: entry.ws.project, workspaceId: entry.ws.workspaceId }
|
|
2108
|
+
: {
|
|
2109
|
+
kind: "workspace-remove-failed",
|
|
2110
|
+
project: entry.ws.project,
|
|
2111
|
+
workspaceId: entry.ws.workspaceId,
|
|
2112
|
+
reason: `herdr workspace close failed: ${firstLine(res.stderr) || "no output"}`,
|
|
2113
|
+
},
|
|
1318
2114
|
);
|
|
1319
2115
|
}
|
|
1320
2116
|
return { ok: true, outcomes };
|
|
1321
2117
|
}
|
|
1322
2118
|
|
|
1323
|
-
function firstLine(text: string): string {
|
|
1324
|
-
return text.split("\n", 1)[0]?.trim() ?? "";
|
|
1325
|
-
}
|
|
1326
|
-
|
|
1327
|
-
/** The kill syscall, injectable so error mapping is testable without one. */
|
|
1328
|
-
export type KillFn = (pid: number, sig: NodeJS.Signals | 0) => void;
|
|
1329
|
-
|
|
1330
|
-
const realKill: KillFn = (pid, sig) => {
|
|
1331
|
-
process.kill(pid, sig);
|
|
1332
|
-
};
|
|
1333
|
-
|
|
1334
2119
|
/**
|
|
1335
2120
|
* What a failed `kill` proves.
|
|
1336
2121
|
*
|
|
@@ -2556,10 +3341,13 @@ export function sessionsRoot(): string {
|
|
|
2556
3341
|
* claim-only verdict consumes this — the challenge proof reads conductor
|
|
2557
3342
|
* state, not transcripts (#614).
|
|
2558
3343
|
*/
|
|
2559
|
-
function claimedOrchestratorSessionFile(
|
|
3344
|
+
function claimedOrchestratorSessionFile(
|
|
3345
|
+
named: string | undefined,
|
|
3346
|
+
alive?: (pid: number) => boolean,
|
|
3347
|
+
): string | undefined {
|
|
2560
3348
|
if (named === undefined) return undefined;
|
|
2561
3349
|
try {
|
|
2562
|
-
return resolveClaimedSessionFile(findProject(loadConfig(), named));
|
|
3350
|
+
return resolveClaimedSessionFile(findProject(loadConfig(), named), alive);
|
|
2563
3351
|
} catch {
|
|
2564
3352
|
return undefined;
|
|
2565
3353
|
}
|