omp-conductor 0.19.6 → 0.19.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/fleet.ts CHANGED
@@ -391,7 +391,10 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
391
391
  // challenge proof never reads it: its acknowledgement is conductor state,
392
392
  // so where (or whether) a transcript lives is no longer part of arming
393
393
  // (#614).
394
- const claimed = deps.claimedSessionFile !== undefined ? deps.claimedSessionFile() : claimedOrchestratorSessionFile(named);
394
+ const claimed =
395
+ deps.claimedSessionFile !== undefined
396
+ ? deps.claimedSessionFile()
397
+ : claimedOrchestratorSessionFile(named, deps.pidAlive ?? pidAlive);
395
398
 
396
399
  // Prefer the project's live forum topic so arm challenges land where
397
400
  // escalations already do (#318), following the bridge's current claim when the
@@ -399,7 +402,7 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
399
402
  let sendTopic: number | undefined;
400
403
  if (named !== undefined) {
401
404
  try {
402
- sendTopic = resolveProjectTopicId(findProject(loadConfig(), named));
405
+ sendTopic = resolveProjectTopicId(findProject(loadConfig(), named), deps.pidAlive ?? pidAlive);
403
406
  } catch {
404
407
  /* no project config */
405
408
  }
@@ -907,10 +910,17 @@ export interface WorkerPaneIdentity {
907
910
  sessionFile?: string;
908
911
  }
909
912
 
910
- /** A tracked pane, or the explicit reason there is none. Never a silent claim. */
913
+ /** A tracked pane, or the explicit reason there is none. Never a silent claim.
914
+ * The tracked variant names the marked workspace the representation lives in,
915
+ * so a caller that just created one can protect it from the same pass's
916
+ * empty-workspace cleanup. `phase` names the step that refused — `"split"`
917
+ * marks the one failure the re-establishment budget spends whole (#998): a
918
+ * terminal that refuses to split will refuse every retry within the pass, so
919
+ * burning the budget once beats leaking an attempt per reconciliation pass
920
+ * against a wall. */
911
921
  export type WorkerPaneOutcome =
912
- | { kind: "tracked"; paneId: string; label: string; pid: number }
913
- | { kind: "unavailable"; reason: string };
922
+ | { kind: "tracked"; paneId: string; label: string; pid: number; workspaceId?: string }
923
+ | { kind: "unavailable"; reason: string; phase?: "split" };
914
924
 
915
925
  /** One `herdr` invocation, injected so every path is testable with no terminal. */
916
926
  export type HerdrRun = (args: readonly string[]) => { ok: boolean; stdout: string; stderr: string };
@@ -920,6 +930,9 @@ export interface WorkerPaneDeps {
920
930
  session?: string;
921
931
  /** The follower command the pane displays; the CLI by default. */
922
932
  viewer?: (identity: WorkerPaneIdentity) => readonly string[];
933
+ /** The durable ownership half of workspace discovery (#1035 review);
934
+ * production wires the conductor store, tests a recorder. */
935
+ ownership?: WorkspaceOwnership;
923
936
  }
924
937
 
925
938
  /**
@@ -932,9 +945,13 @@ export interface WorkerPaneDeps {
932
945
  /**
933
946
  * The external-supervisor source every conductor pane report carries.
934
947
  *
935
- * This, not the label, is what proves a pane is conductor's: it is written by
936
- * `openWorkerPane` and read back by {@link listWorkerPanes}, so the two halves
937
- * of reconciliation cannot come to disagree about what "ours" means.
948
+ * Written onto every pane conductor reports against. It is no longer what
949
+ * proves a pane is conductor's: production repeatedly handed back panes whose
950
+ * `agent_session` record Herdr had dropped (#1035), so filtering by reported
951
+ * source quietly exempted exactly the panes that needed cleaning. Ownership is
952
+ * structural now — conductor creates representations only inside a workspace
953
+ * it marked itself (see {@link WORKER_WORKSPACE_TOKEN}) — and this source
954
+ * remains on the reports as the display identity of who speaks for a pane.
938
955
  */
939
956
  export const WORKER_PANE_SOURCE = "omp-conductor";
940
957
 
@@ -950,6 +967,26 @@ export const WORKER_PANE_SOURCE = "omp-conductor";
950
967
  */
951
968
  export const PANE_REATTEMPT_MAX = 3;
952
969
 
970
+ /**
971
+ * The workspace metadata token that marks a workspace as conductor's
972
+ * per-project worker surface (#1035).
973
+ *
974
+ * Written once at creation (`workspace report-metadata --token
975
+ * conductor_workers=<project>`) and read back from `workspace list`, so
976
+ * rediscovery survives daemon and Herdr restarts without trusting the label,
977
+ * the sidebar position, or whichever workspace happens to be focused. Only
978
+ * panes inside a marked workspace are ever reconciled here, which is what makes
979
+ * the operator's own panes invisible to this surface by construction — and
980
+ * what makes every pane inside a marked workspace conductor's, however Herdr
981
+ * feels about reporting `agent_session` back.
982
+ */
983
+ export const WORKER_WORKSPACE_TOKEN = "conductor_workers";
984
+
985
+ /** The dedicated sibling workspace's display label. Never identity. */
986
+ export function workerWorkspaceLabel(project: string): string {
987
+ return `${project}-workers`;
988
+ }
989
+
953
990
  export function workerPaneLabel(identity: WorkerPaneIdentity): string {
954
991
  return `worker-${identity.project}-${identity.issue}-a${identity.attempt}-${identity.runId.slice(0, 8)}`;
955
992
  }
@@ -997,41 +1034,402 @@ function parsePaneId(stdout: string): string | undefined {
997
1034
  return typeof paneId === "string" && paneId.trim() !== "" ? paneId.trim() : undefined;
998
1035
  }
999
1036
 
1037
+ /**
1038
+ * The workspace and root-pane ids inside `herdr workspace create`'s answer
1039
+ * (`{"id":"cli:workspace:create","result":{"workspace":{…},"root_pane":{…}}}`).
1040
+ * The root pane is the anchor every later split targets, so a create answer
1041
+ * without one is only half a workspace — the caller cleans it up rather than
1042
+ * guessing at a pane to split.
1043
+ */
1044
+ function parseCreatedWorkspace(stdout: string): { workspaceId: string; rootPaneId?: string } | undefined {
1045
+ const result = envelopeOf(stdout);
1046
+ if (result === undefined) return undefined;
1047
+ const workspace = result["workspace"];
1048
+ if (workspace === null || typeof workspace !== "object") return undefined;
1049
+ const workspaceId = (workspace as Record<string, unknown>)["workspace_id"];
1050
+ if (typeof workspaceId !== "string" || workspaceId.trim() === "") return undefined;
1051
+ const root = (result as Record<string, unknown>)["root_pane"];
1052
+ const rootPaneId =
1053
+ root !== null && typeof root === "object"
1054
+ ? (root as Record<string, unknown>)["pane_id"]
1055
+ : undefined;
1056
+ return {
1057
+ workspaceId: workspaceId.trim(),
1058
+ ...(typeof rootPaneId === "string" && rootPaneId.trim() !== "" ? { rootPaneId: rootPaneId.trim() } : {}),
1059
+ };
1060
+ }
1061
+
1062
+ /** The `{result: …}` object of a herdr CLI envelope, or nothing. */
1063
+
1064
+ function envelopeOf(stdout: string): Record<string, unknown> | undefined {
1065
+ const line = firstLine(stdout);
1066
+ if (line === "") return undefined;
1067
+ let payload: unknown;
1068
+ try {
1069
+ payload = JSON.parse(line);
1070
+ } catch {
1071
+ return undefined;
1072
+ }
1073
+ if (payload === null || typeof payload !== "object") return undefined;
1074
+ const result = (payload as Record<string, unknown>)["result"];
1075
+ return result === null || typeof result !== "object" ? undefined : (result as Record<string, unknown>);
1076
+ }
1077
+
1078
+ /** The root-pane id inside a `tab create` answer (`result.root_pane.pane_id`):
1079
+ * a tab answer carries no workspace object, so {@link parseCreatedWorkspace}
1080
+ * cannot read it. */
1081
+ function parseRootPane(stdout: string): string | undefined {
1082
+ const result = envelopeOf(stdout);
1083
+ const root = result?.["root_pane"];
1084
+ const paneId =
1085
+ root !== null && typeof root === "object" ? (root as Record<string, unknown>)["pane_id"] : undefined;
1086
+ return typeof paneId === "string" && paneId.trim() !== "" ? paneId.trim() : undefined;
1087
+ }
1088
+
1089
+ /** Every workspace in a `workspace list` answer, with the tokens ownership is read from. */
1090
+ function parseWorkspaceList(stdout: string): { workspaces: { workspaceId: string; label?: string; tokens: Record<string, string> }[] } | undefined {
1091
+ const result = envelopeOf(stdout);
1092
+ const raw = result?.["workspaces"];
1093
+ if (!Array.isArray(raw)) return undefined;
1094
+ const workspaces: { workspaceId: string; label?: string; tokens: Record<string, string> }[] = [];
1095
+ for (const entry of raw) {
1096
+ if (entry === null || typeof entry !== "object") continue;
1097
+ const w = entry as Record<string, unknown>;
1098
+ if (typeof w["workspace_id"] !== "string" || w["workspace_id"].trim() === "") continue;
1099
+ const tokens: Record<string, string> = {};
1100
+ if (w["tokens"] !== null && typeof w["tokens"] === "object") {
1101
+ for (const [name, value] of Object.entries(w["tokens"] as Record<string, unknown>)) {
1102
+ if (typeof value === "string") tokens[name] = value;
1103
+ }
1104
+ }
1105
+ workspaces.push({
1106
+ workspaceId: w["workspace_id"],
1107
+ ...(typeof w["label"] === "string" ? { label: w["label"] } : {}),
1108
+ tokens,
1109
+ });
1110
+ }
1111
+ return { workspaces };
1112
+ }
1113
+
1114
+ /** One pane record out of any `pane list` answer — global or `--workspace`-scoped. */
1115
+ export interface ListedPane {
1116
+ paneId: string;
1117
+ /** The run id Herdr still reports for this pane, or "" when it reports none. */
1118
+ runId: string;
1119
+ agent?: string;
1120
+ workspaceId?: string;
1121
+ }
1122
+
1123
+ /** The panes of a `pane list` answer, carrying whichever fields reconciliation reads. */
1124
+ function parsePaneRecords(stdout: string): { panes: ListedPane[] } | undefined {
1125
+ const result = envelopeOf(stdout);
1126
+ const raw = result?.["panes"];
1127
+ if (!Array.isArray(raw)) return undefined;
1128
+ const panes: ListedPane[] = [];
1129
+ for (const entry of raw) {
1130
+ if (entry === null || typeof entry !== "object") continue;
1131
+ const p = entry as Record<string, unknown>;
1132
+ if (typeof p["pane_id"] !== "string" || p["pane_id"].trim() === "") continue;
1133
+ const session = p["agent_session"];
1134
+ const source = session !== null && typeof session === "object" ? (session as Record<string, unknown>)["source"] : undefined;
1135
+ const value = session !== null && typeof session === "object" ? (session as Record<string, unknown>)["value"] : undefined;
1136
+ panes.push({
1137
+ paneId: p["pane_id"].trim(),
1138
+ // Ours by construction inside a marked workspace; the reported session id
1139
+ // is a secondary signal only (#1035), because Herdr has been observed to
1140
+ // drop it while the pane lives on.
1141
+ runId: typeof value === "string" && source === WORKER_PANE_SOURCE ? value : "",
1142
+ ...(typeof p["agent"] === "string" ? { agent: p["agent"] } : {}),
1143
+ ...(typeof p["workspace_id"] === "string" ? { workspaceId: p["workspace_id"] } : {}),
1144
+ });
1145
+ }
1146
+ return { panes };
1147
+ }
1148
+
1149
+ /** One conductor-marked worker workspace, as discovery returns it. */
1150
+ export interface WorkerWorkspace {
1151
+ workspaceId: string;
1152
+ /** The project the marking token binds it to — never derived from the label. */
1153
+ project: string;
1154
+ label?: string;
1155
+ }
1156
+
1157
+ /**
1158
+ * The durable half of worker-workspace discovery (#1035 review).
1159
+ *
1160
+ * Herdr restores a session's workspaces and panes across a server restart but
1161
+ * drops `report-metadata` tokens — probed live: the restored worker workspace
1162
+ * comes back carrying no tokens at all. Token-only discovery would then ignore
1163
+ * the surface conductor created, build a duplicate beside it, and never be
1164
+ * able to reconcile or remove the restored one's stale panes — exactly the
1165
+ * accumulation this issue exists to end. Ownership is therefore two-legged:
1166
+ * the live token AND the conductor-store record of workspace ids the project
1167
+ * created. Either leg proves ownership; the store is the leg that survives.
1168
+ */
1169
+ export interface WorkspaceOwnership {
1170
+ /** Store-recorded workspace ids for a project, oldest first. */
1171
+ recordedWorkspaces(project: string): readonly string[];
1172
+ rememberWorkspace(project: string, workspaceId: string): void;
1173
+ forgetWorkspace(project: string, workspaceId: string): void;
1174
+ }
1175
+
1176
+ /** Every workspace in a `workspace list` answer, marked or not. */
1177
+ function listWorkspacesRaw(
1178
+ deps: { run?: HerdrRun; session?: string } = {},
1179
+ ): {
1180
+ ok: true;
1181
+ workspaces: { workspaceId: string; label?: string; tokens: Record<string, string> }[];
1182
+ } | { ok: false; reason: string } {
1183
+ const run = deps.run ?? realHerdrRun;
1184
+ const session = deps.session ?? resolveHerdrSession();
1185
+ const res = run(["--session", session, "workspace", "list"]);
1186
+ if (!res.ok) {
1187
+ return { ok: false, reason: `herdr workspace list failed: ${firstLine(res.stderr) || "no output"}` };
1188
+ }
1189
+ const parsed = parseWorkspaceList(res.stdout);
1190
+ if (parsed === undefined) return { ok: false, reason: "herdr workspace list was unreadable" };
1191
+ return { ok: true, workspaces: parsed.workspaces };
1192
+ }
1193
+
1194
+ /**
1195
+ * Every conductor-marked worker workspace Herdr currently has, keyed by nothing:
1196
+ * the caller filters and scopes. Discovery reads the token, not the label — a
1197
+ * renamed workspace is still ours, and a lookalike label without the token is
1198
+ * not (#1035).
1199
+ */
1200
+ export function listWorkerWorkspaces(deps: { run?: HerdrRun; session?: string } = {}): {
1201
+ ok: true;
1202
+ workspaces: WorkerWorkspace[];
1203
+ } | { ok: false; reason: string } {
1204
+ const listed = listWorkspacesRaw(deps);
1205
+ if (!listed.ok) return listed;
1206
+ const workspaces: WorkerWorkspace[] = [];
1207
+ for (const w of listed.workspaces) {
1208
+ const project = w.tokens[WORKER_WORKSPACE_TOKEN];
1209
+ if (project === undefined || project.trim() === "") continue;
1210
+ workspaces.push({
1211
+ workspaceId: w.workspaceId,
1212
+ project: project.trim(),
1213
+ ...(w.label === undefined ? {} : { label: w.label }),
1214
+ });
1215
+ }
1216
+ return { ok: true, workspaces };
1217
+ }
1218
+
1219
+ /**
1220
+ * Every worker workspace these projects own on the live Herdr (#1035 review):
1221
+ * token-marked ones plus store-recorded ids that still exist. A recorded id
1222
+ * Herdr no longer lists is dead weight rather than evidence — its workspace is
1223
+ * gone, and reconciliation's removal path is what forgets it.
1224
+ */
1225
+ export function ownedWorkerWorkspaces(
1226
+ projects: readonly string[],
1227
+ deps: { run?: HerdrRun; session?: string; ownership?: WorkspaceOwnership } = {},
1228
+ ): { ok: true; workspaces: WorkerWorkspace[] } | { ok: false; reason: string } {
1229
+ const raw = listWorkspacesRaw(deps);
1230
+ if (!raw.ok) return raw;
1231
+ const scope = new Set(projects);
1232
+ const owned = new Map<string, WorkerWorkspace>();
1233
+ for (const w of raw.workspaces) {
1234
+ const project = w.tokens[WORKER_WORKSPACE_TOKEN]?.trim();
1235
+ if (project === undefined || project === "" || !scope.has(project)) continue;
1236
+ owned.set(w.workspaceId, {
1237
+ workspaceId: w.workspaceId,
1238
+ project,
1239
+ ...(w.label === undefined ? {} : { label: w.label }),
1240
+ });
1241
+ // Adopt token-owned surfaces into the durable leg even when no pane needs
1242
+ // opening this pass. Otherwise an upgrade followed by a Herdr restart can
1243
+ // drop the token before ensureWorkerWorkspace ever records the workspace.
1244
+ deps.ownership?.rememberWorkspace(project, w.workspaceId);
1245
+ }
1246
+ for (const project of projects) {
1247
+ for (const id of deps.ownership?.recordedWorkspaces(project) ?? []) {
1248
+ if (owned.has(id)) continue;
1249
+ const hit = raw.workspaces.find((w) => w.workspaceId === id);
1250
+ if (hit === undefined) {
1251
+ // The workspace disappeared outside conductor. Forget the stale id now
1252
+ // so a future Herdr session cannot reuse it as false ownership.
1253
+ deps.ownership?.forgetWorkspace(project, id);
1254
+ continue;
1255
+ }
1256
+ owned.set(id, { workspaceId: id, project, ...(hit.label === undefined ? {} : { label: hit.label }) });
1257
+ }
1258
+ }
1259
+ return { ok: true, workspaces: [...owned.values()] };
1260
+ }
1261
+
1262
+ export type WorkerWorkspaceResolution =
1263
+ | { kind: "ready"; workspaceId: string; anchorPaneId: string; created?: true }
1264
+ | { kind: "unavailable"; reason: string };
1265
+
1266
+ /**
1267
+ * Find this project's worker workspace or create it — the single door every
1268
+ * split goes through (#1035).
1269
+ *
1270
+ * Identity is two-legged now (#1035 review): the marking token OR the
1271
+ * conductor-store record, so rediscovery after any restart finds the same
1272
+ * workspace whatever Herdr preserved. Duplicates are deliberately NOT
1273
+ * resolved here: closing one blind — without listing its panes or checking
1274
+ * the result — could destroy the only live representation and strand the
1275
+ * durable row pointing at a dead pane. Convergence belongs to
1276
+ * {@link reconcileWorkerPanes}, which attributes every pane first and removes
1277
+ * a duplicate only once nothing living holds it. A discovered workspace with
1278
+ * no panes gets a fresh tab rather than failing forever; a workspace CREATED
1279
+ * by this call is closed again on every later failure, because newly-made
1280
+ * residue is exactly what the acceptance criterion forbids.
1281
+ */
1282
+ export function ensureWorkerWorkspace(
1283
+ project: string,
1284
+ deps: { run?: HerdrRun; session?: string; ownership?: WorkspaceOwnership } = {},
1285
+ ): WorkerWorkspaceResolution {
1286
+ const run = deps.run ?? realHerdrRun;
1287
+ const session = deps.session ?? resolveHerdrSession();
1288
+ const base = ["--session", session];
1289
+
1290
+ let listed = ownedWorkerWorkspaces([project], deps);
1291
+ if (!listed.ok) return { kind: "unavailable", reason: listed.reason };
1292
+
1293
+ for (const existing of [...listed.workspaces].sort((a, b) => (a.workspaceId < b.workspaceId ? -1 : 1))) {
1294
+ const res = run([...base, "pane", "list", "--workspace", existing.workspaceId]);
1295
+ if (!res.ok) {
1296
+ return { kind: "unavailable", reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}` };
1297
+ }
1298
+ const parsed = parsePaneRecords(res.stdout);
1299
+ if (parsed === undefined) return { kind: "unavailable", reason: "herdr pane list was unreadable" };
1300
+ let anchor = parsed.panes[0]?.paneId;
1301
+ if (anchor === undefined) {
1302
+ // No pane to split: give the workspace a fresh tab and use its root pane.
1303
+ const tabbed = run([...base, "tab", "create", "--workspace", existing.workspaceId, "--no-focus"]);
1304
+ if (!tabbed.ok) {
1305
+ return { kind: "unavailable", reason: `herdr tab create failed: ${firstLine(tabbed.stderr) || "no output"}` };
1306
+ }
1307
+ anchor = parseRootPane(tabbed.stdout);
1308
+ if (anchor === undefined) {
1309
+ return { kind: "unavailable", reason: "herdr tab create reported no root pane" };
1310
+ }
1311
+ }
1312
+ // Adopted into the store: if this workspace's token is what Herdr drops
1313
+ // next restart, the record below is what still finds it.
1314
+ deps.ownership?.rememberWorkspace(project, existing.workspaceId);
1315
+ return { kind: "ready", workspaceId: existing.workspaceId, anchorPaneId: anchor };
1316
+ }
1317
+
1318
+ const made = run([
1319
+ ...base,
1320
+ "workspace",
1321
+ "create",
1322
+ "--label",
1323
+ workerWorkspaceLabel(project),
1324
+ "--no-focus",
1325
+ ]);
1326
+ if (!made.ok) {
1327
+ return { kind: "unavailable", reason: `herdr workspace create failed: ${firstLine(made.stderr) || "no output"}` };
1328
+ }
1329
+ const parsed = parseCreatedWorkspace(made.stdout);
1330
+ if (parsed === undefined) {
1331
+ return { kind: "unavailable", reason: "herdr workspace create reported no workspace id" };
1332
+ }
1333
+ /** This attempt made the workspace, so every exit from here takes it back. */
1334
+ const discardMade = (): void => {
1335
+ run([...base, "workspace", "close", parsed.workspaceId]);
1336
+ deps.ownership?.forgetWorkspace(project, parsed.workspaceId);
1337
+ };
1338
+ // Mark before use: an unmarked workspace would be invisible to token-side
1339
+ // discovery on the next pass — an empty shell accumulating precisely as
1340
+ // #1035 describes. A failure here takes the half-made workspace with it.
1341
+ const marked = run([
1342
+ ...base,
1343
+ "workspace",
1344
+ "report-metadata",
1345
+ parsed.workspaceId,
1346
+ "--source",
1347
+ WORKER_PANE_SOURCE,
1348
+ "--token",
1349
+ `${WORKER_WORKSPACE_TOKEN}=${project}`,
1350
+ ]);
1351
+ if (!marked.ok) {
1352
+ discardMade();
1353
+ return {
1354
+ kind: "unavailable",
1355
+ reason: `herdr workspace report-metadata failed: ${firstLine(marked.stderr) || "no output"}`,
1356
+ };
1357
+ }
1358
+ if (parsed.rootPaneId === undefined) {
1359
+ // No anchor means no split can ever be targeted; a fresh-made workspace
1360
+ // without one is residue this very attempt must not leave behind
1361
+ // (#1035 review) — so it goes now, not on some later pass.
1362
+ discardMade();
1363
+ return { kind: "unavailable", reason: "herdr workspace create reported no root pane" };
1364
+ }
1365
+ deps.ownership?.rememberWorkspace(project, parsed.workspaceId);
1366
+ return { kind: "ready", workspaceId: parsed.workspaceId, anchorPaneId: parsed.rootPaneId, created: true };
1367
+ }
1368
+
1000
1369
  /**
1001
1370
  * Create the pane, run the follower in it, and report the child's identity and
1002
1371
  * initial state — or say exactly why it could not.
1003
1372
  *
1004
- * Every step is checked, and the first failure returns `unavailable` with the
1005
- * reason and closes the pane the split had already created, because a
1006
- * partially established representation is worse than none: it is a pane the
1007
- * operator would read as a tracked worker, and before #992 it was left behind
1008
- * on every single launch. What this deliberately does NOT do is decide what a
1009
- * failure means for the launch failing closed versus running degraded is
1010
- * #841's policy, and inventing it here would pre-empt it.
1373
+ * The representation lives in the project's worker workspace (#1035), found or
1374
+ * created first, and the split targets an anchor pane inside it explicitly, so
1375
+ * the operator's own layout is never touched. Every step is checked, and the
1376
+ * first failure returns `unavailable` with the reason and leaves ZERO residue
1377
+ * of this attempt (#1035 review): the pane the split had already created is
1378
+ * closed, and a workspace this call CREATED is closed with it only surfaces
1379
+ * that already existed are preserved. A partially established representation
1380
+ * is worse than none: it is a pane the operator would read as a tracked
1381
+ * worker, and before #992 it was left behind on every single launch. What
1382
+ * this deliberately does NOT do is decide what a failure means for the launch
1383
+ * — failing closed versus running degraded is #841's policy, and inventing it
1384
+ * here would pre-empt it.
1011
1385
  */
1012
1386
  export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDeps = {}): WorkerPaneOutcome {
1013
1387
  const run = deps.run ?? realHerdrRun;
1014
1388
  const session = deps.session ?? resolveHerdrSession();
1015
1389
  const label = workerPaneLabel(identity);
1390
+ const ensured = ensureWorkerWorkspace(identity.project, deps);
1391
+ if (ensured.kind === "unavailable") return ensured;
1016
1392
  const base = ["--session", session, "pane"];
1017
1393
 
1018
- const split = run([...base, "split", "--direction", "down", "--ratio", "0.3"]);
1394
+ /** Undo everything this attempt built so far — the split's pane once there
1395
+ * is one, and always a workspace this call itself created (#1035 review). */
1396
+ const abandon = (reason: string, phase?: "split", paneId?: string): WorkerPaneOutcome => {
1397
+ if (paneId !== undefined) run([...base, "close", paneId]);
1398
+ if (ensured.created === true) {
1399
+ run(["--session", session, "workspace", "close", ensured.workspaceId]);
1400
+ deps.ownership?.forgetWorkspace(identity.project, ensured.workspaceId);
1401
+ }
1402
+ return { kind: "unavailable", reason, ...(phase === undefined ? {} : { phase }) };
1403
+ };
1404
+
1405
+ // The explicit target is the whole point (#1035): the split names a pane
1406
+ // inside the worker workspace, so whichever pane the operator happens to be
1407
+ // looking at is never touched.
1408
+ const split = run([
1409
+ ...base,
1410
+ "split",
1411
+ "--pane",
1412
+ ensured.anchorPaneId,
1413
+ "--direction",
1414
+ "down",
1415
+ "--ratio",
1416
+ "0.3",
1417
+ "--no-focus",
1418
+ ]);
1019
1419
  if (!split.ok) {
1020
- return { kind: "unavailable", reason: `herdr pane split failed: ${firstLine(split.stderr) || "no output"}` };
1420
+ return abandon(
1421
+ `herdr pane split failed: ${firstLine(split.stderr) || "no output"}`,
1422
+ "split",
1423
+ );
1021
1424
  }
1022
1425
  const paneId = parsePaneId(split.stdout);
1023
1426
  if (paneId === undefined) {
1024
- return { kind: "unavailable", reason: "herdr pane split reported no pane id" };
1427
+ return abandon("herdr pane split reported no pane id");
1025
1428
  }
1026
- // The split has created a pane. From here every exit must take it with it.
1027
- const abandon = (reason: string): WorkerPaneOutcome => {
1028
- run([...base, "close", paneId]);
1029
- return { kind: "unavailable", reason };
1030
- };
1031
1429
 
1032
1430
  const named = run([...base, "rename", paneId, label]);
1033
1431
  if (!named.ok) {
1034
- return abandon(`herdr pane rename failed: ${firstLine(named.stderr) || "no output"}`);
1432
+ return abandon(`herdr pane rename failed: ${firstLine(named.stderr) || "no output"}`, undefined, paneId);
1035
1433
  }
1036
1434
 
1037
1435
  // Identity before display: the pane must be attributable to this exact run
@@ -1052,12 +1450,16 @@ export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDep
1052
1450
  WORKER_PANE_SOURCE,
1053
1451
  ]);
1054
1452
  if (!identified.ok) {
1055
- return abandon(`herdr pane report-agent-session failed: ${firstLine(identified.stderr) || "no output"}`);
1453
+ return abandon(
1454
+ `herdr pane report-agent-session failed: ${firstLine(identified.stderr) || "no output"}`,
1455
+ undefined,
1456
+ paneId,
1457
+ );
1056
1458
  }
1057
1459
 
1058
1460
  const started = run([...base, "run", paneId, ...deps.viewer?.(identity) ?? defaultViewer(identity)]);
1059
1461
  if (!started.ok) {
1060
- return abandon(`herdr pane run failed: ${firstLine(started.stderr) || "no output"}`);
1462
+ return abandon(`herdr pane run failed: ${firstLine(started.stderr) || "no output"}`, undefined, paneId);
1061
1463
  }
1062
1464
 
1063
1465
  // A worker that has just spawned is working by definition. The ongoing
@@ -1065,9 +1467,9 @@ export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDep
1065
1467
  // events — never from the pane's output.
1066
1468
  const reported = reportWorkerPaneState(paneId, label, "working", { run, session });
1067
1469
  if (!reported.ok) {
1068
- return abandon(reported.reason);
1470
+ return abandon(reported.reason, undefined, paneId);
1069
1471
  }
1070
- return { kind: "tracked", paneId, label, pid: identity.pid };
1472
+ return { kind: "tracked", paneId, label, pid: identity.pid, workspaceId: ensured.workspaceId };
1071
1473
  }
1072
1474
 
1073
1475
  /** Report one lifecycle state for a tracked pane. Monotonic `seq` is the caller's. */
@@ -1100,12 +1502,11 @@ export function reportWorkerPaneState(
1100
1502
  }
1101
1503
 
1102
1504
  /**
1103
- * Hand lifecycle authority back when the worker is gone.
1104
- *
1105
- * Release, never close: whether a settled worker's pane is closed, kept, or
1106
- * retained for N settlements is #841's documented policy, and a slice that
1107
- * closed panes here would decide it by accident. Releasing says only "conductor
1108
- * no longer speaks for this agent", which is exactly what is true.
1505
+ * Hand lifecycle authority back when conductor no longer speaks for a pane's
1506
+ * agent. Half of {@link retireWorkerPane}: it says "this agent record is no
1507
+ * longer ours" without removing anything, which is what reconciliation wants
1508
+ * for a pane it is about to close anyway but also stands alone for callers
1509
+ * that only re-report identity (#842's adopt path).
1109
1510
  */
1110
1511
  export function releaseWorkerPane(
1111
1512
  paneId: string,
@@ -1131,71 +1532,153 @@ export function releaseWorkerPane(
1131
1532
  : { ok: false, reason: `herdr pane release-agent failed: ${firstLine(res.stderr) || "no output"}` };
1132
1533
  }
1133
1534
 
1535
+ /** Close one pane outright. The pane holds only ever a follower, so closing
1536
+ * it cannot reach the authoritative worker — that child belongs to the
1537
+ * daemon's process tree, not to any terminal (#1035). */
1538
+ export function closeWorkerPane(
1539
+ paneId: string,
1540
+ deps: { run?: HerdrRun; session?: string } = {},
1541
+ ): { ok: true } | { ok: false; reason: string } {
1542
+ const run = deps.run ?? realHerdrRun;
1543
+ const session = deps.session ?? resolveHerdrSession();
1544
+ const res = run(["--session", session, "pane", "close", paneId]);
1545
+ return res.ok
1546
+ ? { ok: true }
1547
+ : { ok: false, reason: `herdr pane close failed: ${firstLine(res.stderr) || "no output"}` };
1548
+ }
1549
+
1550
+ /**
1551
+ * Retire one run's visual representation (#1035): hand lifecycle authority
1552
+ * back to Herdr, then close the pane.
1553
+ *
1554
+ * #841 originally stopped at release, and production showed what that leaves:
1555
+ * panes advertising `working` for runs that had settled hours earlier, their
1556
+ * follower long exited, a bare shell where a worker used to be. A settled run
1557
+ * keeps no representation. The close reaches only the follower — the
1558
+ * authoritative child was never in the pane, so nothing here can signal it.
1559
+ *
1560
+ * A close failure is reported, not swallowed: the next reconciliation pass
1561
+ * sees the pane as unclaimed junk and closes it again, so one refusal costs a
1562
+ * pass rather than the representation.
1563
+ */
1564
+ export function retireWorkerPane(
1565
+ paneId: string,
1566
+ label: string,
1567
+ deps: { run?: HerdrRun; session?: string; seq?: number } = {},
1568
+ ): { ok: true } | { ok: false; reason: string } {
1569
+ const released = releaseWorkerPane(paneId, label, deps);
1570
+ if (!released.ok) return released;
1571
+ return closeWorkerPane(paneId, deps);
1572
+ }
1573
+
1134
1574
  /**
1135
- * Hand back the pane a dead worker left behind (#842).
1575
+ * Hand back the representation a dead worker left behind (#842), retired for
1576
+ * good by closing it (#1035) — but only once the recorded pane is PROVEN still
1577
+ * conductor's (#1035 review).
1136
1578
  *
1137
1579
  * Called for every run a restart reaps. The recorded pid is deliberately NOT
1138
1580
  * consulted for liveness: a `session-host` child dies with the daemon that owned
1139
1581
  * its verb socket, so an orphaned row's worker is gone whatever pid it carries —
1140
1582
  * and pids are reused, so checking one is how a stranger's process comes to read
1141
- * as a live worker. A run with no recorded pane is a no-op, not a failure: it
1142
- * never had one to release.
1583
+ * as a live worker. The same reuse argument applies to pane ids: after a Herdr
1584
+ * restart the stored `w2:pT` may belong to a pane in somebody else's workspace,
1585
+ * so a blind release+close could destroy a stranger's terminal. Ownership is
1586
+ * therefore proven before anything is closed:
1587
+ *
1588
+ * - the pane must be listed inside a workspace conductor OWNS — a
1589
+ * {@link WORKER_WORKSPACE_TOKEN}-marked one, or one the store recorded for
1590
+ * the run's project (#1035 review: Herdr restarts restore workspaces without
1591
+ * their tokens, and a token-only check would leave the restored pane
1592
+ * unretired forever); and
1143
1593
  */
1144
1594
  export function releaseOrphanedWorkerPane(
1145
- run: { paneId?: string; paneLabel?: string },
1146
- deps: { run?: HerdrRun; session?: string; seq?: number } = {},
1147
- ): { kind: "none" } | { kind: "released"; paneId: string } | { kind: "failed"; paneId: string; reason: string } {
1595
+ run: { paneId?: string; paneLabel?: string; runId?: string; project?: string },
1596
+ deps: { run?: HerdrRun; session?: string; seq?: number; ownership?: WorkspaceOwnership } = {},
1597
+ ):
1598
+ | { kind: "none" }
1599
+ | { kind: "released"; paneId: string }
1600
+ | { kind: "failed"; paneId: string; reason: string }
1601
+ | { kind: "unowned"; paneId: string; reason: string } {
1148
1602
  if (run.paneId === undefined || run.paneLabel === undefined) return { kind: "none" };
1149
- const released = releaseWorkerPane(run.paneId, run.paneLabel, deps);
1150
- return released.ok
1603
+ const discovered = ownedWorkerWorkspaces(run.project === undefined ? [] : [run.project], deps);
1604
+ if (!discovered.ok) {
1605
+ return { kind: "failed", paneId: run.paneId, reason: discovered.reason };
1606
+ }
1607
+ const runFn = deps.run ?? realHerdrRun;
1608
+ const session = deps.session ?? resolveHerdrSession();
1609
+ let home: string | undefined;
1610
+ let reportedAgent: string | undefined;
1611
+ let reportedSession: string | undefined;
1612
+ for (const ws of discovered.workspaces) {
1613
+ const res = runFn(["--session", session, "pane", "list", "--workspace", ws.workspaceId]);
1614
+ if (!res.ok) {
1615
+ return {
1616
+ kind: "failed",
1617
+ paneId: run.paneId,
1618
+ reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}`,
1619
+ };
1620
+ }
1621
+ const parsed = parsePaneRecords(res.stdout);
1622
+ if (parsed === undefined) {
1623
+ return { kind: "failed", paneId: run.paneId, reason: "herdr pane list was unreadable" };
1624
+ }
1625
+ const found = parsed.panes.find((pane) => pane.paneId === run.paneId);
1626
+ if (found !== undefined && home === undefined) {
1627
+ home = ws.workspaceId;
1628
+ if (found.agent !== undefined) reportedAgent = found.agent;
1629
+ if (found.runId !== "") reportedSession = found.runId;
1630
+ }
1631
+ }
1632
+ if (home === undefined) {
1633
+ // Not in any marked workspace: either already retired or never ours to
1634
+ // touch. Both answers close nothing.
1635
+ return { kind: "none" };
1636
+ }
1637
+ if (
1638
+ (reportedAgent !== undefined && reportedAgent !== run.paneLabel) ||
1639
+ (reportedSession !== undefined && run.runId !== undefined && reportedSession !== run.runId)
1640
+ ) {
1641
+ return {
1642
+ kind: "unowned",
1643
+ paneId: run.paneId,
1644
+ reason: `pane ${run.paneId} in marked workspace ${home} reports another identity (agent ${reportedAgent ?? "none"}, session ${reportedSession ?? "none"})`,
1645
+ };
1646
+ }
1647
+ const retired = retireWorkerPane(run.paneId, run.paneLabel, deps);
1648
+ return retired.ok
1151
1649
  ? { kind: "released", paneId: run.paneId }
1152
- : { kind: "failed", paneId: run.paneId, reason: released.reason };
1650
+ : { kind: "failed", paneId: run.paneId, reason: retired.reason };
1153
1651
  }
1154
1652
 
1155
1653
  /**
1156
- * Every conductor-owned pane Herdr currently has, keyed by the run id it was
1157
- * reported under (#841).
1654
+ * Every pane inside a conductor-marked worker workspace (#1035).
1158
1655
  *
1159
- * Read from `pane list`'s own agent-session record, never from a label pattern:
1160
- * the label is what a human reads, and matching on it is exactly how a stale
1161
- * lookalike (a renamed pane, a pane from a previous fleet) gets mistaken for a
1162
- * live worker. `source` proves conductor reported it; `value` is the run id.
1656
+ * Ownership here is structural: conductor created the workspace, marked it,
1657
+ * and splits into it exclusively, so every pane inside it is conductor's
1658
+ * however Herdr feels about reporting `agent_session` back production has
1659
+ * repeatedly handed back worker panes with that record dropped to null, which
1660
+ * is why the previous source-filtered listing could not see the very panes
1661
+ * that needed cleaning. The reported session id still rides along as a
1662
+ * secondary attribution signal; an empty run id means "unattributable", and
1663
+ * unattributable panes in a marked workspace are exactly the stale shells
1664
+ * this surface exists to converge away.
1163
1665
  */
1164
1666
  export function listWorkerPanes(
1165
1667
  deps: { run?: HerdrRun; session?: string } = {},
1166
- ): { ok: true; panes: { paneId: string; runId: string; label?: string }[] } | { ok: false; reason: string } {
1668
+ ): { ok: true; panes: (ListedPane & { project: string })[] } | { ok: false; reason: string } {
1669
+ const workspaces = listWorkerWorkspaces(deps);
1670
+ if (!workspaces.ok) return workspaces;
1167
1671
  const run = deps.run ?? realHerdrRun;
1168
1672
  const session = deps.session ?? resolveHerdrSession();
1169
- const res = run(["--session", session, "pane", "list"]);
1170
- if (!res.ok) {
1171
- return { ok: false, reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}` };
1172
- }
1173
- let parsed: unknown;
1174
- try {
1175
- parsed = JSON.parse(res.stdout);
1176
- } catch (err) {
1177
- return { ok: false, reason: `herdr pane list was unreadable: ${err instanceof Error ? err.message : String(err)}` };
1178
- }
1179
- const panes = (parsed as { result?: { panes?: unknown[] } }).result?.panes;
1180
- if (!Array.isArray(panes)) return { ok: false, reason: "herdr pane list carried no pane array" };
1181
- const owned: { paneId: string; runId: string; label?: string }[] = [];
1182
- for (const pane of panes) {
1183
- const p = pane as {
1184
- pane_id?: unknown;
1185
- agent?: unknown;
1186
- agent_session?: { source?: unknown; value?: unknown };
1187
- };
1188
- if (typeof p.pane_id !== "string") continue;
1189
- if (p.agent_session?.source !== WORKER_PANE_SOURCE) continue;
1190
- const runId = p.agent_session.value;
1191
- // Ours by source but carrying no run id: an identity nobody can resolve is
1192
- // not an identity. Reported as a pane with an empty run id so the caller
1193
- // treats it as stale rather than silently ignoring it.
1194
- owned.push({
1195
- paneId: p.pane_id,
1196
- runId: typeof runId === "string" ? runId : "",
1197
- ...(typeof p.agent === "string" ? { label: p.agent } : {}),
1198
- });
1673
+ const owned: (ListedPane & { project: string })[] = [];
1674
+ for (const ws of workspaces.workspaces) {
1675
+ const res = run(["--session", session, "pane", "list", "--workspace", ws.workspaceId]);
1676
+ if (!res.ok) {
1677
+ return { ok: false, reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}` };
1678
+ }
1679
+ const parsed = parsePaneRecords(res.stdout);
1680
+ if (parsed === undefined) return { ok: false, reason: "herdr pane list carried no pane array" };
1681
+ for (const pane of parsed.panes) owned.push({ ...pane, project: ws.project });
1199
1682
  }
1200
1683
  return { ok: true, panes: owned };
1201
1684
  }
@@ -1213,43 +1696,147 @@ export interface LiveWorkerPane {
1213
1696
  sessionFile?: string;
1214
1697
  }
1215
1698
 
1699
+ export type StaleCause = "settled" | "duplicate" | "unidentified";
1700
+
1216
1701
  export type WorkerPaneReconciliation =
1217
- /** The recorded pane is still there and still carries this run — nothing done. */
1702
+ /** The recorded pane is still there and its follower is alive — nothing done. */
1218
1703
  | { kind: "intact"; runId: string; paneId: string }
1219
- /** Herdr lost the pane (a restart); a new one now represents the same child. */
1704
+ /** The pane stands but its follower table could not decide life or death:
1705
+ * no evidence, no destruction (#1035 review). The visual may be stale; the
1706
+ * caller says so once instead of gambling a live worker's representation. */
1707
+ | { kind: "visual-unreadable"; runId: string; paneId: string; reason: string }
1708
+ /** The run had no usable representation; a new one now represents the same
1709
+ * child — a Herdr restart, a missing pane, or a follower that had exited. */
1220
1710
  | { kind: "reassociated"; runId: string; paneId: string; label: string }
1221
1711
  /** No representation, and the reason. The run keeps working regardless.
1222
1712
  * `attempted` marks the ones that spent a re-establishment attempt (#998),
1223
- * so the caller can bound them; the no-pid case costs nothing. */
1224
- | { kind: "untracked"; runId: string; reason: string; attempted?: true }
1713
+ * so the caller can bound them; the no-pid case costs nothing. `phase`
1714
+ * carries {@link WorkerPaneOutcome}'s split marker through. */
1715
+ | { kind: "untracked"; runId: string; reason: string; attempted?: true; phase?: "split" }
1225
1716
  /** The attempt budget for this run is spent, so nothing was tried this pass
1226
1717
  * (#998). Before this existed, a run whose pane could not be created was
1227
1718
  * retried on every reconciliation pass forever — and each attempt leaked a
1228
1719
  * pane until #992, ending in `ghostty error -2` once enough had piled up. */
1229
1720
  | { kind: "attempts-exhausted"; runId: string; attempts: number }
1230
- /** A conductor pane whose run is not live: handed back to Herdr. */
1231
- | { kind: "stale-released"; paneId: string; runId: string }
1232
- | { kind: "stale-release-failed"; paneId: string; runId: string; reason: string };
1721
+ /** A conductor-workspace pane with no live run behind it: authority handed
1722
+ * back and the pane closed. `settled` the run finished and its retirement
1723
+ * never landed; `duplicate` a second representation of one live run, or
1724
+ * the one a fresh replacement displaced; `unidentified` — a representation
1725
+ * whose run id nobody can resolve any more, the bare shells production
1726
+ * accumulated (#1035). */
1727
+ | { kind: "stale-released"; paneId: string; runId: string; cause: StaleCause }
1728
+ | { kind: "stale-release-failed"; paneId: string; runId: string; reason: string; cause: StaleCause }
1729
+ /** A scoped marked workspace nobody's live run holds any more, emptied by
1730
+ * the cleanup above, is closed (#1035). Every marked workspace converges,
1731
+ * not just one per project (#1035 review). */
1732
+ | { kind: "workspace-removed"; project: string; workspaceId: string }
1733
+ | { kind: "workspace-remove-failed"; project: string; workspaceId: string; reason: string };
1734
+
1735
+ /** The verdict the foreground process table supports about one pane's
1736
+ * read-only follower. Terminal output is never read: this says whether the
1737
+ * *visual* is alive, nothing more, and settlement stays the store's business
1738
+ * (#1035). `unknown` is a real verdict, not an error — recognition of
1739
+ * `omp-conductor` in the table is positive evidence of life, but its absence
1740
+ * is only evidence of death when every entry carries enough identity (name or
1741
+ * argv) to rule it out. Real Herdr answers name + argv; a table with blanked
1742
+ * or missing fields, no shell pid, or no entries at all cannot decide, and a
1743
+ * pane destroyed on a guess we did not have is a live worker gone blind. */
1744
+ export type FollowerPulse =
1745
+ | { pulse: "alive" }
1746
+ | { pulse: "dead" }
1747
+ | { pulse: "unknown"; reason: string };
1748
+
1749
+ export function followerPulseFromProcessInfo(info: ProcessInfo): FollowerPulse {
1750
+ const procs = Array.isArray(info.foreground_processes) ? info.foreground_processes : undefined;
1751
+ if (procs === undefined || procs.length === 0) {
1752
+ return { pulse: "unknown", reason: "process-info carried no foreground process table" };
1753
+ }
1754
+ const shell = typeof info.shell_pid === "number" ? info.shell_pid : undefined;
1755
+ let nonShell = false;
1756
+ for (const proc of procs) {
1757
+ const parts = [proc.name ?? "", proc.argv0 ?? "", ...(proc.argv ?? [])];
1758
+ const isShell = shell !== undefined && proc.pid === shell;
1759
+ if (!isShell && !parts.some((a) => a !== "")) {
1760
+ return { pulse: "unknown", reason: `foreground pid ${proc.pid ?? "unknown"} carries no name or argv` };
1761
+ }
1762
+ if (!isShell) {
1763
+ nonShell = true;
1764
+ if (parts.some((a) => a.toLowerCase() === "omp-conductor" || a.toLowerCase().endsWith("/omp-conductor"))) {
1765
+ return { pulse: "alive" };
1766
+ }
1767
+ }
1768
+ }
1769
+ if (!nonShell && shell === undefined) {
1770
+ return { pulse: "unknown", reason: "process-info named neither a shell nor any identifiable process" };
1771
+ }
1772
+ return { pulse: "dead" };
1773
+ }
1774
+
1775
+ /**
1776
+ * Probe one pane's follower liveness. A probe that cannot be read, or a table
1777
+ * that cannot decide, answers `unknown` rather than dead: destroying a
1778
+ * representation on evidence we do not have is how a healthy worker goes blind
1779
+ * mid-run.
1780
+ */
1781
+ export function paneFollowerAlive(
1782
+ paneId: string,
1783
+ deps: { run?: HerdrRun; session?: string } = {},
1784
+ ): FollowerPulse {
1785
+ const run = deps.run ?? realHerdrRun;
1786
+ const session = deps.session ?? resolveHerdrSession();
1787
+ const res = run(["--session", session, "pane", "process-info", "--pane", paneId]);
1788
+ if (!res.ok) {
1789
+ return {
1790
+ pulse: "unknown",
1791
+ reason: `herdr pane process-info failed: ${firstLine(res.stderr) || "no output"}`,
1792
+ };
1793
+ }
1794
+ try {
1795
+ return followerPulseFromProcessInfo(parseHerdrProcessInfo(res.stdout, paneId));
1796
+ } catch (err) {
1797
+ return { pulse: "unknown", reason: err instanceof Error ? err.message : String(err) };
1798
+ }
1799
+ }
1800
+
1801
+ function firstLine(text: string): string {
1802
+ return text.split("\n", 1)[0]?.trim() ?? "";
1803
+ }
1804
+ /** The kill syscall, injectable so error mapping is testable without one. */
1805
+ export type KillFn = (pid: number, sig: NodeJS.Signals | 0) => void;
1806
+
1807
+ const realKill: KillFn = (pid, sig) => {
1808
+ process.kill(pid, sig);
1809
+ };
1233
1810
 
1234
1811
  /**
1235
- * Make Herdr's conductor-owned panes agree with the live run set (#841).
1812
+ * Make the conductor-owned worker workspaces agree with the live run set
1813
+ * (#841, reworked by #1035).
1236
1814
  *
1237
1815
  * Idempotent by construction: a second pass over an already-reconciled fleet
1238
- * returns `intact` for every live run and finds no stale panes, so repeated
1239
- * daemon or Herdr restarts converge rather than accumulate.
1816
+ * returns `intact` for every live run and finds nothing left to close, so
1817
+ * repeated launch/settlement/restart cycles converge instead of accumulating.
1240
1818
  *
1241
- * Three rules, and each exists to refuse a specific way this goes wrong:
1819
+ * Four rules, and each exists to refuse a specific way this goes wrong:
1242
1820
  *
1243
- * - **A live run's pane is re-created, never duplicated.** Re-association is
1244
- * keyed on the run id Herdr itself reports, so a pane that is still there is
1245
- * left alone. Only a run whose pane Herdr no longer has gets a new one.
1246
- * - **Cleanup is by exact identity, never by name or age.** A pane is stale only
1247
- * when the run id it carries is absent from the live set. A worker's own pane
1248
- * can therefore never be released while its run is live, whatever it is called
1249
- * and however old it is.
1250
- * - **Nothing here can stop a worker.** The only mutation is `release-agent`,
1251
- * which hands lifecycle authority back to Herdr; the authoritative child is
1252
- * never signalled, and its pane is never closed.
1821
+ * - **Only owned workspaces are touched.** Scope is the projects of the live
1822
+ * set plus the caller's managed list; ownership is the marking token OR the
1823
+ * conductor-store record (#1035 review), so a Herdr restart that drops
1824
+ * tokens still finds the restored surface. A pane in another project's
1825
+ * worker workspace is invisible to this pass, which is what keeps conductor
1826
+ * and veltrosecurity from discovering, reusing, or cleaning each other.
1827
+ * - **Attribution is exact, structural first.** A pane belongs to the live run
1828
+ * the store recorded it under; the session id Herdr reports is the fallback.
1829
+ * Everything else in an owned workspace a settled run's leftover, a
1830
+ * duplicate of a live run, a representation whose identity Herdr dropped —
1831
+ * is closed, because inside an owned workspace "ours" and "junk" are the
1832
+ * only options and leaving junk is what filled the operator's layout.
1833
+ * - **A live run keeps exactly one living representation.** Duplicates fold
1834
+ * into the recorded pane; a pane whose follower has exited is replaced while
1835
+ * the run is live; a missing pane is recreated — all bounded by the same
1836
+ * re-establishment budget (#998).
1837
+ * - **Nothing here can stop a worker.** Panes hold followers only; the
1838
+ * authoritative child lives in the daemon's process tree and is never
1839
+ * signalled, and worker liveness comes from the caller's live set alone.
1253
1840
  */
1254
1841
  export function reconcileWorkerPanes(
1255
1842
  live: readonly LiveWorkerPane[],
@@ -1258,39 +1845,184 @@ export function reconcileWorkerPanes(
1258
1845
  * and a daemon restart legitimately gets a fresh budget (#998). */
1259
1846
  attempts: ReadonlyMap<string, number> = new Map(),
1260
1847
  maxAttempts = PANE_REATTEMPT_MAX,
1848
+ /** Projects this pass manages even when they have no live runs: their stale
1849
+ * panes are cleaned and a workspace left empty by that cleanup is removed. */
1850
+ managedProjects: readonly string[] = [],
1261
1851
  ): { ok: true; outcomes: WorkerPaneReconciliation[] } | { ok: false; reason: string } {
1262
- const listed = listWorkerPanes(deps);
1263
- if (!listed.ok) return { ok: false, reason: listed.reason };
1264
- const byRun = new Map(listed.panes.map((pane) => [pane.runId, pane]));
1265
- const liveIds = new Set(live.map((worker) => worker.runId));
1852
+ const run = deps.run ?? realHerdrRun;
1853
+ const session = deps.session ?? resolveHerdrSession();
1854
+ const base = ["--session", session];
1855
+
1856
+ const scope = new Set([...managedProjects, ...live.map((worker) => worker.project)]);
1857
+ const discovered = ownedWorkerWorkspaces([...scope], deps);
1858
+ if (!discovered.ok) return { ok: false, reason: discovered.reason };
1859
+ // Deterministic order so outcomes and closes are reproducible pass to pass.
1860
+ const scoped = [...discovered.workspaces].sort((a, b) => (a.workspaceId < b.workspaceId ? -1 : 1));
1861
+
1862
+ // Every marked workspace of every scoped project converges (#1035 review):
1863
+ // duplicate token-marked workspaces are cleaned like any other, so a settled
1864
+ // project cannot keep stale shells in a second marked surface nobody tracks.
1865
+ const entries: { ws: WorkerWorkspace; panes: ListedPane[] }[] = [];
1866
+ for (const ws of scoped) {
1867
+ const res = run([...base, "pane", "list", "--workspace", ws.workspaceId]);
1868
+ if (!res.ok) {
1869
+ // An unreadable workspace is not evidence that anything is stale, so the
1870
+ // pass mutates nothing rather than closing on a guess.
1871
+ return { ok: false, reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}` };
1872
+ }
1873
+ const parsed = parsePaneRecords(res.stdout);
1874
+ if (parsed === undefined) return { ok: false, reason: "herdr pane list carried no pane array" };
1875
+ entries.push({ ws, panes: parsed.panes });
1876
+ }
1877
+
1878
+ const liveByRun = new Map(live.map((worker) => [worker.runId, worker]));
1266
1879
  const outcomes: WorkerPaneReconciliation[] = [];
1880
+ /** Panes this pass will close, with the identity the outcome names. */
1881
+ const junk = new Map<string, { runId: string; cause: StaleCause; agent?: string }>();
1882
+ /** Panes still standing when cleanup ends, keyed to their workspace — an
1883
+ * occupied workspace survives, whichever marked surface it sits in. */
1884
+ const standingWs = new Map<string, string>();
1885
+ /** Workspaces that gained a representation this pass, read from the tracked
1886
+ * outcomes: their listing predates the new pane, so removal never judges
1887
+ * them on it. */
1888
+ const openedWs = new Set<string>();
1889
+
1890
+ /** One representation per live run: the pane plus the marked workspace that
1891
+ * holds it. Attribution runs across ALL of a project's marked workspaces,
1892
+ * because the recorded pane may sit in any of them. */
1893
+ const held = new Map<string, { pane: ListedPane; wsId: string }>();
1894
+ for (const entry of entries) {
1895
+ const projectLive = live.filter((worker) => worker.project === entry.ws.project);
1896
+ for (const pane of entry.panes) {
1897
+ standingWs.set(pane.paneId, entry.ws.workspaceId);
1898
+ const owner =
1899
+ projectLive.find((worker) => worker.paneId === pane.paneId) ??
1900
+ (pane.runId === "" ? undefined : liveByRun.get(pane.runId));
1901
+ if (owner === undefined || owner.project !== entry.ws.project) {
1902
+ // Not attributable to any live run of this project: a settled run's
1903
+ // leftover (still carrying its old id) or an unidentified shell.
1904
+ junk.set(pane.paneId, {
1905
+ runId: pane.runId,
1906
+ cause: pane.runId === "" ? "unidentified" : "settled",
1907
+ ...(pane.agent === undefined ? {} : { agent: pane.agent }),
1908
+ });
1909
+ continue;
1910
+ }
1911
+ const bucket = held.get(owner.runId);
1912
+ if (bucket === undefined) {
1913
+ held.set(owner.runId, { pane, wsId: entry.ws.workspaceId });
1914
+ continue;
1915
+ }
1916
+ // Exactly one focusable representation per live run: the recorded pane
1917
+ // wins so the durable row stays true without a write; the other is a
1918
+ // duplicate (#1035).
1919
+ const keep =
1920
+ bucket.pane.paneId === owner.paneId
1921
+ ? bucket
1922
+ : pane.paneId === owner.paneId
1923
+ ? { pane, wsId: entry.ws.workspaceId }
1924
+ : bucket;
1925
+ const drop = keep.pane.paneId === bucket.pane.paneId ? pane : bucket.pane;
1926
+ held.set(owner.runId, keep);
1927
+ junk.set(drop.paneId, {
1928
+ runId: owner.runId,
1929
+ cause: "duplicate",
1930
+ ...(drop.agent === undefined ? {} : { agent: drop.agent }),
1931
+ });
1932
+ }
1933
+ }
1267
1934
 
1268
1935
  for (const worker of live) {
1269
- const held = byRun.get(worker.runId);
1270
- if (held !== undefined) {
1271
- outcomes.push({ kind: "intact", runId: worker.runId, paneId: held.paneId });
1936
+ const hold = held.get(worker.runId);
1937
+ if (hold === undefined) {
1938
+ // No representation anywhere in the marked workspaces. Only the pid makes
1939
+ // a replacement honest: the pane represents an exact child, so without
1940
+ // one there is nothing to represent and inventing a pane would be the
1941
+ // silent claim this whole surface exists to avoid.
1942
+ if (worker.pid === undefined) {
1943
+ outcomes.push({
1944
+ kind: "untracked",
1945
+ runId: worker.runId,
1946
+ reason: "no session-host pid was ever recorded for this run",
1947
+ });
1948
+ continue;
1949
+ }
1950
+ // The budget, checked before the attempt: an exhausted run is skipped
1951
+ // silently here and reported once by the caller (#998).
1952
+ const spent = attempts.get(worker.runId) ?? 0;
1953
+ if (spent >= maxAttempts) {
1954
+ outcomes.push({ kind: "attempts-exhausted", runId: worker.runId, attempts: spent });
1955
+ continue;
1956
+ }
1957
+ const opened = openWorkerPane(
1958
+ {
1959
+ project: worker.project,
1960
+ issue: worker.issue,
1961
+ attempt: worker.attempt,
1962
+ runId: worker.runId,
1963
+ pid: worker.pid,
1964
+ ...(worker.sessionFile === undefined ? {} : { sessionFile: worker.sessionFile }),
1965
+ },
1966
+ deps,
1967
+ );
1968
+ if (opened.kind === "tracked") {
1969
+ if (opened.workspaceId !== undefined) openedWs.add(opened.workspaceId);
1970
+ outcomes.push({
1971
+ kind: "reassociated",
1972
+ runId: worker.runId,
1973
+ paneId: opened.paneId,
1974
+ label: opened.label,
1975
+ });
1976
+ } else {
1977
+ outcomes.push({
1978
+ kind: "untracked",
1979
+ runId: worker.runId,
1980
+ reason: opened.reason,
1981
+ attempted: true,
1982
+ ...(opened.phase === undefined ? {} : { phase: opened.phase }),
1983
+ });
1984
+ }
1985
+ continue;
1986
+ }
1987
+ // The pane stands. Whether it still shows anything is the follower's
1988
+ // question, asked only to decide the *visual* — never settlement (#1035).
1989
+ // A probe that cannot be read, or a table that cannot decide, is no
1990
+ // evidence the visual died: it is reported and the pane keeps standing,
1991
+ // because a live worker's representation is never destroyed on a guess.
1992
+ const pulse = paneFollowerAlive(hold.pane.paneId, deps);
1993
+ if (pulse.pulse !== "dead") {
1994
+ outcomes.push(
1995
+ pulse.pulse === "alive"
1996
+ ? { kind: "intact", runId: worker.runId, paneId: hold.pane.paneId }
1997
+ : {
1998
+ kind: "visual-unreadable",
1999
+ runId: worker.runId,
2000
+ paneId: hold.pane.paneId,
2001
+ reason: pulse.reason,
2002
+ },
2003
+ );
1272
2004
  continue;
1273
2005
  }
1274
- // Herdr does not have this run's pane. Only the pid makes a replacement
1275
- // honest: the pane represents an exact child, so without one there is
1276
- // nothing to represent and inventing a pane would be the silent claim this
1277
- // whole surface exists to avoid.
2006
+ // The follower has provably exited. A replacement represents the exact
2007
+ // child, so a run with no recorded pid keeps its dead pane rather than
2008
+ // gaining a representation of nobody reported so the record drops and
2009
+ // the next pass reads the orphan as junk.
1278
2010
  if (worker.pid === undefined) {
1279
2011
  outcomes.push({
1280
2012
  kind: "untracked",
1281
2013
  runId: worker.runId,
1282
- reason: "no session-host pid was ever recorded for this run",
2014
+ reason: "follower exited but no session-host pid was ever recorded; not replacing",
1283
2015
  });
1284
2016
  continue;
1285
2017
  }
1286
- // The budget, checked before the attempt: an exhausted run is skipped
1287
- // silently here and reported once by the caller (#998).
1288
2018
  const spent = attempts.get(worker.runId) ?? 0;
1289
2019
  if (spent >= maxAttempts) {
2020
+ // Out of budget: the dead pane keeps its last state until a restart
2021
+ // refreshes the budget or the run settles and retires it outright.
1290
2022
  outcomes.push({ kind: "attempts-exhausted", runId: worker.runId, attempts: spent });
1291
2023
  continue;
1292
2024
  }
1293
- const opened = openWorkerPane(
2025
+ const replaced = openWorkerPane(
1294
2026
  {
1295
2027
  project: worker.project,
1296
2028
  issue: worker.issue,
@@ -1301,36 +2033,89 @@ export function reconcileWorkerPanes(
1301
2033
  },
1302
2034
  deps,
1303
2035
  );
1304
- outcomes.push(
1305
- opened.kind === "tracked"
1306
- ? { kind: "reassociated", runId: worker.runId, paneId: opened.paneId, label: opened.label }
1307
- : { kind: "untracked", runId: worker.runId, reason: opened.reason, attempted: true },
1308
- );
2036
+ if (replaced.kind === "tracked") {
2037
+ if (replaced.workspaceId !== undefined) openedWs.add(replaced.workspaceId);
2038
+ outcomes.push({
2039
+ kind: "reassociated",
2040
+ runId: worker.runId,
2041
+ paneId: replaced.paneId,
2042
+ label: replaced.label,
2043
+ });
2044
+ junk.set(hold.pane.paneId, {
2045
+ runId: worker.runId,
2046
+ cause: "duplicate",
2047
+ ...(hold.pane.agent === undefined ? {} : { agent: hold.pane.agent }),
2048
+ });
2049
+ } else {
2050
+ outcomes.push({
2051
+ kind: "untracked",
2052
+ runId: worker.runId,
2053
+ reason: replaced.reason,
2054
+ attempted: true,
2055
+ ...(replaced.phase === undefined ? {} : { phase: replaced.phase }),
2056
+ });
2057
+ // The replacement failed; the dead pane has nothing left to show, so it
2058
+ // goes now rather than advertising a worker for another pass. Its record
2059
+ // is dropped either way, so a survivor is junk next pass.
2060
+ closeWorkerPane(hold.pane.paneId, deps);
2061
+ standingWs.delete(hold.pane.paneId);
2062
+ }
1309
2063
  }
1310
2064
 
1311
- for (const pane of listed.panes) {
1312
- if (liveIds.has(pane.runId)) continue;
1313
- const released = releaseWorkerPane(pane.paneId, pane.label ?? "", deps);
2065
+ for (const [paneId, info] of junk) {
2066
+ const released =
2067
+ info.agent === undefined ? ({ ok: true } as const) : releaseWorkerPane(paneId, info.agent, deps);
2068
+ if (released.ok) {
2069
+ const closed = closeWorkerPane(paneId, deps);
2070
+ if (closed.ok) {
2071
+ standingWs.delete(paneId);
2072
+ outcomes.push({ kind: "stale-released", paneId, runId: info.runId, cause: info.cause });
2073
+ continue;
2074
+ }
2075
+ outcomes.push({
2076
+ kind: "stale-release-failed",
2077
+ paneId,
2078
+ runId: info.runId,
2079
+ reason: closed.reason,
2080
+ cause: info.cause,
2081
+ });
2082
+ continue;
2083
+ }
2084
+ outcomes.push({
2085
+ kind: "stale-release-failed",
2086
+ paneId,
2087
+ runId: info.runId,
2088
+ reason: released.reason,
2089
+ cause: info.cause,
2090
+ });
2091
+ }
2092
+
2093
+ // Every scoped marked workspace whose panes the cleanup above emptied, and
2094
+ // that gained no representation this pass, is closed — however many marked
2095
+ // surfaces a project accumulated (#1035 review). A live project's primary
2096
+ // workspace survives because its representation keeps it standing; extras
2097
+ // converge away; repeated cycles hold at zero shells.
2098
+ for (const entry of entries) {
2099
+ if (openedWs.has(entry.ws.workspaceId)) continue;
2100
+ if ([...standingWs.values()].some((wsId) => wsId === entry.ws.workspaceId)) continue;
2101
+ const res = run([...base, "workspace", "close", entry.ws.workspaceId]);
2102
+ if (res.ok) deps.ownership?.forgetWorkspace(entry.ws.project, entry.ws.workspaceId);
2103
+ // A failed close keeps the record: the workspace may still exist, and
2104
+ // forgetting ownership of a live surface is how orphans are manufactured.
1314
2105
  outcomes.push(
1315
- released.ok
1316
- ? { kind: "stale-released", paneId: pane.paneId, runId: pane.runId }
1317
- : { kind: "stale-release-failed", paneId: pane.paneId, runId: pane.runId, reason: released.reason },
2106
+ res.ok
2107
+ ? { kind: "workspace-removed", project: entry.ws.project, workspaceId: entry.ws.workspaceId }
2108
+ : {
2109
+ kind: "workspace-remove-failed",
2110
+ project: entry.ws.project,
2111
+ workspaceId: entry.ws.workspaceId,
2112
+ reason: `herdr workspace close failed: ${firstLine(res.stderr) || "no output"}`,
2113
+ },
1318
2114
  );
1319
2115
  }
1320
2116
  return { ok: true, outcomes };
1321
2117
  }
1322
2118
 
1323
- function firstLine(text: string): string {
1324
- return text.split("\n", 1)[0]?.trim() ?? "";
1325
- }
1326
-
1327
- /** The kill syscall, injectable so error mapping is testable without one. */
1328
- export type KillFn = (pid: number, sig: NodeJS.Signals | 0) => void;
1329
-
1330
- const realKill: KillFn = (pid, sig) => {
1331
- process.kill(pid, sig);
1332
- };
1333
-
1334
2119
  /**
1335
2120
  * What a failed `kill` proves.
1336
2121
  *
@@ -2556,10 +3341,13 @@ export function sessionsRoot(): string {
2556
3341
  * claim-only verdict consumes this — the challenge proof reads conductor
2557
3342
  * state, not transcripts (#614).
2558
3343
  */
2559
- function claimedOrchestratorSessionFile(named: string | undefined): string | undefined {
3344
+ function claimedOrchestratorSessionFile(
3345
+ named: string | undefined,
3346
+ alive?: (pid: number) => boolean,
3347
+ ): string | undefined {
2560
3348
  if (named === undefined) return undefined;
2561
3349
  try {
2562
- return resolveClaimedSessionFile(findProject(loadConfig(), named));
3350
+ return resolveClaimedSessionFile(findProject(loadConfig(), named), alive);
2563
3351
  } catch {
2564
3352
  return undefined;
2565
3353
  }