omp-conductor 0.19.5 → 0.19.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +17 -1
- package/package.json +1 -1
- package/src/cli.ts +2 -0
- package/src/command-help.ts +18 -8
- package/src/command-manifest.ts +22 -5
- package/src/commands/context.ts +1 -0
- package/src/commands/report.ts +63 -0
- package/src/commands/restore-db.ts +1 -1
- package/src/commands/snapshot-db.ts +67 -0
- package/src/commands/upgrade-install.ts +1 -1
- package/src/commands/upgrade.ts +2 -2
- package/src/commands/verb.ts +1 -0
- package/src/commands/watch.ts +14 -2
- package/src/config.ts +17 -0
- package/src/daemon.ts +146 -35
- package/src/doctor.ts +53 -15
- package/src/escalate.ts +39 -21
- package/src/fleet.ts +979 -143
- package/src/orchestrator-tick.ts +14 -3
- package/src/setup-host.ts +10 -10
- package/src/setup-wizard.ts +1 -1
- package/src/status-render.ts +10 -0
- package/src/store.ts +338 -22
- package/src/types.ts +51 -3
- package/src/upgrade-journal.ts +35 -1
- package/src/upgrade-verify.ts +27 -6
- package/src/upgrade.ts +301 -22
- package/src/verbs/actions.ts +6 -1
- package/src/verbs/server.ts +146 -31
- package/systemd/omp-conductor-recover.sh +55 -7
- package/systemd/recover-unit-test.sh +99 -6
package/src/fleet.ts
CHANGED
|
@@ -223,6 +223,46 @@ export function armState(projectName?: string): { path: string; armed: boolean }
|
|
|
223
223
|
return { path, armed: resolveArmState(path, named).armed };
|
|
224
224
|
}
|
|
225
225
|
|
|
226
|
+
export type WatchMonitoring =
|
|
227
|
+
| { monitored: true }
|
|
228
|
+
| { monitored: false; reason: string };
|
|
229
|
+
|
|
230
|
+
/** Whether a durable watch can be observed and wake an orchestrator tick (#1024). */
|
|
231
|
+
export function watchMonitoringState(
|
|
232
|
+
layers: Pick<FleetLayers, "daemon" | "ticks">,
|
|
233
|
+
): WatchMonitoring {
|
|
234
|
+
if (!layers.daemon.running) return { monitored: false, reason: "daemon not running" };
|
|
235
|
+
if (layers.ticks === "armed" || layers.ticks === "ungated") return { monitored: true };
|
|
236
|
+
if (layers.ticks === "disarmed") {
|
|
237
|
+
return {
|
|
238
|
+
monitored: false,
|
|
239
|
+
reason: "ticks disarmed — conditions may still be checked, but no tick can be woken",
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
return { monitored: false, reason: `ticks ${layers.ticks}` };
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/** Cheap watch liveness read: no Herdr, Telegram, healthz, or systemd probes. */
|
|
246
|
+
export function watchMonitoringForProject(projectName: string): WatchMonitoring {
|
|
247
|
+
const daemon = livingDaemon();
|
|
248
|
+
if (daemon === undefined) return { monitored: false, reason: "daemon not running" };
|
|
249
|
+
if (daemon.project !== undefined && daemon.project !== projectName) {
|
|
250
|
+
return { monitored: false, reason: `daemon serves ${daemon.project}, not ${projectName}` };
|
|
251
|
+
}
|
|
252
|
+
const tick = resolveTickConfig(projectName);
|
|
253
|
+
const ticks: TicksLayer =
|
|
254
|
+
tick.kind === "absent"
|
|
255
|
+
? "no-heartbeat-config"
|
|
256
|
+
: tick.kind === "invalid"
|
|
257
|
+
? "invalid-heartbeat-config"
|
|
258
|
+
: tick.config.armedFile === undefined
|
|
259
|
+
? "ungated"
|
|
260
|
+
: armState(projectName).armed
|
|
261
|
+
? "armed"
|
|
262
|
+
: "disarmed";
|
|
263
|
+
return watchMonitoringState({ daemon: { running: true }, ticks });
|
|
264
|
+
}
|
|
265
|
+
|
|
226
266
|
/**
|
|
227
267
|
* Clears the arm gate for this project — and `wasArmed` is the gate the
|
|
228
268
|
* heartbeat reads, not merely one file's presence.
|
|
@@ -351,7 +391,10 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
351
391
|
// challenge proof never reads it: its acknowledgement is conductor state,
|
|
352
392
|
// so where (or whether) a transcript lives is no longer part of arming
|
|
353
393
|
// (#614).
|
|
354
|
-
const claimed =
|
|
394
|
+
const claimed =
|
|
395
|
+
deps.claimedSessionFile !== undefined
|
|
396
|
+
? deps.claimedSessionFile()
|
|
397
|
+
: claimedOrchestratorSessionFile(named, deps.pidAlive ?? pidAlive);
|
|
355
398
|
|
|
356
399
|
// Prefer the project's live forum topic so arm challenges land where
|
|
357
400
|
// escalations already do (#318), following the bridge's current claim when the
|
|
@@ -359,7 +402,7 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
359
402
|
let sendTopic: number | undefined;
|
|
360
403
|
if (named !== undefined) {
|
|
361
404
|
try {
|
|
362
|
-
sendTopic = resolveProjectTopicId(findProject(loadConfig(), named));
|
|
405
|
+
sendTopic = resolveProjectTopicId(findProject(loadConfig(), named), deps.pidAlive ?? pidAlive);
|
|
363
406
|
} catch {
|
|
364
407
|
/* no project config */
|
|
365
408
|
}
|
|
@@ -867,10 +910,17 @@ export interface WorkerPaneIdentity {
|
|
|
867
910
|
sessionFile?: string;
|
|
868
911
|
}
|
|
869
912
|
|
|
870
|
-
/** A tracked pane, or the explicit reason there is none. Never a silent claim.
|
|
913
|
+
/** A tracked pane, or the explicit reason there is none. Never a silent claim.
|
|
914
|
+
* The tracked variant names the marked workspace the representation lives in,
|
|
915
|
+
* so a caller that just created one can protect it from the same pass's
|
|
916
|
+
* empty-workspace cleanup. `phase` names the step that refused — `"split"`
|
|
917
|
+
* marks the one failure the re-establishment budget spends whole (#998): a
|
|
918
|
+
* terminal that refuses to split will refuse every retry within the pass, so
|
|
919
|
+
* burning the budget once beats leaking an attempt per reconciliation pass
|
|
920
|
+
* against a wall. */
|
|
871
921
|
export type WorkerPaneOutcome =
|
|
872
|
-
| { kind: "tracked"; paneId: string; label: string; pid: number }
|
|
873
|
-
| { kind: "unavailable"; reason: string };
|
|
922
|
+
| { kind: "tracked"; paneId: string; label: string; pid: number; workspaceId?: string }
|
|
923
|
+
| { kind: "unavailable"; reason: string; phase?: "split" };
|
|
874
924
|
|
|
875
925
|
/** One `herdr` invocation, injected so every path is testable with no terminal. */
|
|
876
926
|
export type HerdrRun = (args: readonly string[]) => { ok: boolean; stdout: string; stderr: string };
|
|
@@ -880,6 +930,9 @@ export interface WorkerPaneDeps {
|
|
|
880
930
|
session?: string;
|
|
881
931
|
/** The follower command the pane displays; the CLI by default. */
|
|
882
932
|
viewer?: (identity: WorkerPaneIdentity) => readonly string[];
|
|
933
|
+
/** The durable ownership half of workspace discovery (#1035 review);
|
|
934
|
+
* production wires the conductor store, tests a recorder. */
|
|
935
|
+
ownership?: WorkspaceOwnership;
|
|
883
936
|
}
|
|
884
937
|
|
|
885
938
|
/**
|
|
@@ -892,9 +945,13 @@ export interface WorkerPaneDeps {
|
|
|
892
945
|
/**
|
|
893
946
|
* The external-supervisor source every conductor pane report carries.
|
|
894
947
|
*
|
|
895
|
-
*
|
|
896
|
-
*
|
|
897
|
-
*
|
|
948
|
+
* Written onto every pane conductor reports against. It is no longer what
|
|
949
|
+
* proves a pane is conductor's: production repeatedly handed back panes whose
|
|
950
|
+
* `agent_session` record Herdr had dropped (#1035), so filtering by reported
|
|
951
|
+
* source quietly exempted exactly the panes that needed cleaning. Ownership is
|
|
952
|
+
* structural now — conductor creates representations only inside a workspace
|
|
953
|
+
* it marked itself (see {@link WORKER_WORKSPACE_TOKEN}) — and this source
|
|
954
|
+
* remains on the reports as the display identity of who speaks for a pane.
|
|
898
955
|
*/
|
|
899
956
|
export const WORKER_PANE_SOURCE = "omp-conductor";
|
|
900
957
|
|
|
@@ -910,6 +967,26 @@ export const WORKER_PANE_SOURCE = "omp-conductor";
|
|
|
910
967
|
*/
|
|
911
968
|
export const PANE_REATTEMPT_MAX = 3;
|
|
912
969
|
|
|
970
|
+
/**
|
|
971
|
+
* The workspace metadata token that marks a workspace as conductor's
|
|
972
|
+
* per-project worker surface (#1035).
|
|
973
|
+
*
|
|
974
|
+
* Written once at creation (`workspace report-metadata --token
|
|
975
|
+
* conductor_workers=<project>`) and read back from `workspace list`, so
|
|
976
|
+
* rediscovery survives daemon and Herdr restarts without trusting the label,
|
|
977
|
+
* the sidebar position, or whichever workspace happens to be focused. Only
|
|
978
|
+
* panes inside a marked workspace are ever reconciled here, which is what makes
|
|
979
|
+
* the operator's own panes invisible to this surface by construction — and
|
|
980
|
+
* what makes every pane inside a marked workspace conductor's, however Herdr
|
|
981
|
+
* feels about reporting `agent_session` back.
|
|
982
|
+
*/
|
|
983
|
+
export const WORKER_WORKSPACE_TOKEN = "conductor_workers";
|
|
984
|
+
|
|
985
|
+
/** The dedicated sibling workspace's display label. Never identity. */
|
|
986
|
+
export function workerWorkspaceLabel(project: string): string {
|
|
987
|
+
return `${project}-workers`;
|
|
988
|
+
}
|
|
989
|
+
|
|
913
990
|
export function workerPaneLabel(identity: WorkerPaneIdentity): string {
|
|
914
991
|
return `worker-${identity.project}-${identity.issue}-a${identity.attempt}-${identity.runId.slice(0, 8)}`;
|
|
915
992
|
}
|
|
@@ -957,41 +1034,402 @@ function parsePaneId(stdout: string): string | undefined {
|
|
|
957
1034
|
return typeof paneId === "string" && paneId.trim() !== "" ? paneId.trim() : undefined;
|
|
958
1035
|
}
|
|
959
1036
|
|
|
1037
|
+
/**
|
|
1038
|
+
* The workspace and root-pane ids inside `herdr workspace create`'s answer
|
|
1039
|
+
* (`{"id":"cli:workspace:create","result":{"workspace":{…},"root_pane":{…}}}`).
|
|
1040
|
+
* The root pane is the anchor every later split targets, so a create answer
|
|
1041
|
+
* without one is only half a workspace — the caller cleans it up rather than
|
|
1042
|
+
* guessing at a pane to split.
|
|
1043
|
+
*/
|
|
1044
|
+
function parseCreatedWorkspace(stdout: string): { workspaceId: string; rootPaneId?: string } | undefined {
|
|
1045
|
+
const result = envelopeOf(stdout);
|
|
1046
|
+
if (result === undefined) return undefined;
|
|
1047
|
+
const workspace = result["workspace"];
|
|
1048
|
+
if (workspace === null || typeof workspace !== "object") return undefined;
|
|
1049
|
+
const workspaceId = (workspace as Record<string, unknown>)["workspace_id"];
|
|
1050
|
+
if (typeof workspaceId !== "string" || workspaceId.trim() === "") return undefined;
|
|
1051
|
+
const root = (result as Record<string, unknown>)["root_pane"];
|
|
1052
|
+
const rootPaneId =
|
|
1053
|
+
root !== null && typeof root === "object"
|
|
1054
|
+
? (root as Record<string, unknown>)["pane_id"]
|
|
1055
|
+
: undefined;
|
|
1056
|
+
return {
|
|
1057
|
+
workspaceId: workspaceId.trim(),
|
|
1058
|
+
...(typeof rootPaneId === "string" && rootPaneId.trim() !== "" ? { rootPaneId: rootPaneId.trim() } : {}),
|
|
1059
|
+
};
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
/** The `{result: …}` object of a herdr CLI envelope, or nothing. */
|
|
1063
|
+
|
|
1064
|
+
function envelopeOf(stdout: string): Record<string, unknown> | undefined {
|
|
1065
|
+
const line = firstLine(stdout);
|
|
1066
|
+
if (line === "") return undefined;
|
|
1067
|
+
let payload: unknown;
|
|
1068
|
+
try {
|
|
1069
|
+
payload = JSON.parse(line);
|
|
1070
|
+
} catch {
|
|
1071
|
+
return undefined;
|
|
1072
|
+
}
|
|
1073
|
+
if (payload === null || typeof payload !== "object") return undefined;
|
|
1074
|
+
const result = (payload as Record<string, unknown>)["result"];
|
|
1075
|
+
return result === null || typeof result !== "object" ? undefined : (result as Record<string, unknown>);
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
/** The root-pane id inside a `tab create` answer (`result.root_pane.pane_id`):
|
|
1079
|
+
* a tab answer carries no workspace object, so {@link parseCreatedWorkspace}
|
|
1080
|
+
* cannot read it. */
|
|
1081
|
+
function parseRootPane(stdout: string): string | undefined {
|
|
1082
|
+
const result = envelopeOf(stdout);
|
|
1083
|
+
const root = result?.["root_pane"];
|
|
1084
|
+
const paneId =
|
|
1085
|
+
root !== null && typeof root === "object" ? (root as Record<string, unknown>)["pane_id"] : undefined;
|
|
1086
|
+
return typeof paneId === "string" && paneId.trim() !== "" ? paneId.trim() : undefined;
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
/** Every workspace in a `workspace list` answer, with the tokens ownership is read from. */
|
|
1090
|
+
function parseWorkspaceList(stdout: string): { workspaces: { workspaceId: string; label?: string; tokens: Record<string, string> }[] } | undefined {
|
|
1091
|
+
const result = envelopeOf(stdout);
|
|
1092
|
+
const raw = result?.["workspaces"];
|
|
1093
|
+
if (!Array.isArray(raw)) return undefined;
|
|
1094
|
+
const workspaces: { workspaceId: string; label?: string; tokens: Record<string, string> }[] = [];
|
|
1095
|
+
for (const entry of raw) {
|
|
1096
|
+
if (entry === null || typeof entry !== "object") continue;
|
|
1097
|
+
const w = entry as Record<string, unknown>;
|
|
1098
|
+
if (typeof w["workspace_id"] !== "string" || w["workspace_id"].trim() === "") continue;
|
|
1099
|
+
const tokens: Record<string, string> = {};
|
|
1100
|
+
if (w["tokens"] !== null && typeof w["tokens"] === "object") {
|
|
1101
|
+
for (const [name, value] of Object.entries(w["tokens"] as Record<string, unknown>)) {
|
|
1102
|
+
if (typeof value === "string") tokens[name] = value;
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
workspaces.push({
|
|
1106
|
+
workspaceId: w["workspace_id"],
|
|
1107
|
+
...(typeof w["label"] === "string" ? { label: w["label"] } : {}),
|
|
1108
|
+
tokens,
|
|
1109
|
+
});
|
|
1110
|
+
}
|
|
1111
|
+
return { workspaces };
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
/** One pane record out of any `pane list` answer — global or `--workspace`-scoped. */
|
|
1115
|
+
export interface ListedPane {
|
|
1116
|
+
paneId: string;
|
|
1117
|
+
/** The run id Herdr still reports for this pane, or "" when it reports none. */
|
|
1118
|
+
runId: string;
|
|
1119
|
+
agent?: string;
|
|
1120
|
+
workspaceId?: string;
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
/** The panes of a `pane list` answer, carrying whichever fields reconciliation reads. */
|
|
1124
|
+
function parsePaneRecords(stdout: string): { panes: ListedPane[] } | undefined {
|
|
1125
|
+
const result = envelopeOf(stdout);
|
|
1126
|
+
const raw = result?.["panes"];
|
|
1127
|
+
if (!Array.isArray(raw)) return undefined;
|
|
1128
|
+
const panes: ListedPane[] = [];
|
|
1129
|
+
for (const entry of raw) {
|
|
1130
|
+
if (entry === null || typeof entry !== "object") continue;
|
|
1131
|
+
const p = entry as Record<string, unknown>;
|
|
1132
|
+
if (typeof p["pane_id"] !== "string" || p["pane_id"].trim() === "") continue;
|
|
1133
|
+
const session = p["agent_session"];
|
|
1134
|
+
const source = session !== null && typeof session === "object" ? (session as Record<string, unknown>)["source"] : undefined;
|
|
1135
|
+
const value = session !== null && typeof session === "object" ? (session as Record<string, unknown>)["value"] : undefined;
|
|
1136
|
+
panes.push({
|
|
1137
|
+
paneId: p["pane_id"].trim(),
|
|
1138
|
+
// Ours by construction inside a marked workspace; the reported session id
|
|
1139
|
+
// is a secondary signal only (#1035), because Herdr has been observed to
|
|
1140
|
+
// drop it while the pane lives on.
|
|
1141
|
+
runId: typeof value === "string" && source === WORKER_PANE_SOURCE ? value : "",
|
|
1142
|
+
...(typeof p["agent"] === "string" ? { agent: p["agent"] } : {}),
|
|
1143
|
+
...(typeof p["workspace_id"] === "string" ? { workspaceId: p["workspace_id"] } : {}),
|
|
1144
|
+
});
|
|
1145
|
+
}
|
|
1146
|
+
return { panes };
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
/** One conductor-marked worker workspace, as discovery returns it. */
|
|
1150
|
+
export interface WorkerWorkspace {
|
|
1151
|
+
workspaceId: string;
|
|
1152
|
+
/** The project the marking token binds it to — never derived from the label. */
|
|
1153
|
+
project: string;
|
|
1154
|
+
label?: string;
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
/**
|
|
1158
|
+
* The durable half of worker-workspace discovery (#1035 review).
|
|
1159
|
+
*
|
|
1160
|
+
* Herdr restores a session's workspaces and panes across a server restart but
|
|
1161
|
+
* drops `report-metadata` tokens — probed live: the restored worker workspace
|
|
1162
|
+
* comes back carrying no tokens at all. Token-only discovery would then ignore
|
|
1163
|
+
* the surface conductor created, build a duplicate beside it, and never be
|
|
1164
|
+
* able to reconcile or remove the restored one's stale panes — exactly the
|
|
1165
|
+
* accumulation this issue exists to end. Ownership is therefore two-legged:
|
|
1166
|
+
* the live token AND the conductor-store record of workspace ids the project
|
|
1167
|
+
* created. Either leg proves ownership; the store is the leg that survives.
|
|
1168
|
+
*/
|
|
1169
|
+
export interface WorkspaceOwnership {
|
|
1170
|
+
/** Store-recorded workspace ids for a project, oldest first. */
|
|
1171
|
+
recordedWorkspaces(project: string): readonly string[];
|
|
1172
|
+
rememberWorkspace(project: string, workspaceId: string): void;
|
|
1173
|
+
forgetWorkspace(project: string, workspaceId: string): void;
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1176
|
+
/** Every workspace in a `workspace list` answer, marked or not. */
|
|
1177
|
+
function listWorkspacesRaw(
|
|
1178
|
+
deps: { run?: HerdrRun; session?: string } = {},
|
|
1179
|
+
): {
|
|
1180
|
+
ok: true;
|
|
1181
|
+
workspaces: { workspaceId: string; label?: string; tokens: Record<string, string> }[];
|
|
1182
|
+
} | { ok: false; reason: string } {
|
|
1183
|
+
const run = deps.run ?? realHerdrRun;
|
|
1184
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
1185
|
+
const res = run(["--session", session, "workspace", "list"]);
|
|
1186
|
+
if (!res.ok) {
|
|
1187
|
+
return { ok: false, reason: `herdr workspace list failed: ${firstLine(res.stderr) || "no output"}` };
|
|
1188
|
+
}
|
|
1189
|
+
const parsed = parseWorkspaceList(res.stdout);
|
|
1190
|
+
if (parsed === undefined) return { ok: false, reason: "herdr workspace list was unreadable" };
|
|
1191
|
+
return { ok: true, workspaces: parsed.workspaces };
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
/**
|
|
1195
|
+
* Every conductor-marked worker workspace Herdr currently has, keyed by nothing:
|
|
1196
|
+
* the caller filters and scopes. Discovery reads the token, not the label — a
|
|
1197
|
+
* renamed workspace is still ours, and a lookalike label without the token is
|
|
1198
|
+
* not (#1035).
|
|
1199
|
+
*/
|
|
1200
|
+
export function listWorkerWorkspaces(deps: { run?: HerdrRun; session?: string } = {}): {
|
|
1201
|
+
ok: true;
|
|
1202
|
+
workspaces: WorkerWorkspace[];
|
|
1203
|
+
} | { ok: false; reason: string } {
|
|
1204
|
+
const listed = listWorkspacesRaw(deps);
|
|
1205
|
+
if (!listed.ok) return listed;
|
|
1206
|
+
const workspaces: WorkerWorkspace[] = [];
|
|
1207
|
+
for (const w of listed.workspaces) {
|
|
1208
|
+
const project = w.tokens[WORKER_WORKSPACE_TOKEN];
|
|
1209
|
+
if (project === undefined || project.trim() === "") continue;
|
|
1210
|
+
workspaces.push({
|
|
1211
|
+
workspaceId: w.workspaceId,
|
|
1212
|
+
project: project.trim(),
|
|
1213
|
+
...(w.label === undefined ? {} : { label: w.label }),
|
|
1214
|
+
});
|
|
1215
|
+
}
|
|
1216
|
+
return { ok: true, workspaces };
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
/**
|
|
1220
|
+
* Every worker workspace these projects own on the live Herdr (#1035 review):
|
|
1221
|
+
* token-marked ones plus store-recorded ids that still exist. A recorded id
|
|
1222
|
+
* Herdr no longer lists is dead weight rather than evidence — its workspace is
|
|
1223
|
+
* gone, and reconciliation's removal path is what forgets it.
|
|
1224
|
+
*/
|
|
1225
|
+
export function ownedWorkerWorkspaces(
|
|
1226
|
+
projects: readonly string[],
|
|
1227
|
+
deps: { run?: HerdrRun; session?: string; ownership?: WorkspaceOwnership } = {},
|
|
1228
|
+
): { ok: true; workspaces: WorkerWorkspace[] } | { ok: false; reason: string } {
|
|
1229
|
+
const raw = listWorkspacesRaw(deps);
|
|
1230
|
+
if (!raw.ok) return raw;
|
|
1231
|
+
const scope = new Set(projects);
|
|
1232
|
+
const owned = new Map<string, WorkerWorkspace>();
|
|
1233
|
+
for (const w of raw.workspaces) {
|
|
1234
|
+
const project = w.tokens[WORKER_WORKSPACE_TOKEN]?.trim();
|
|
1235
|
+
if (project === undefined || project === "" || !scope.has(project)) continue;
|
|
1236
|
+
owned.set(w.workspaceId, {
|
|
1237
|
+
workspaceId: w.workspaceId,
|
|
1238
|
+
project,
|
|
1239
|
+
...(w.label === undefined ? {} : { label: w.label }),
|
|
1240
|
+
});
|
|
1241
|
+
// Adopt token-owned surfaces into the durable leg even when no pane needs
|
|
1242
|
+
// opening this pass. Otherwise an upgrade followed by a Herdr restart can
|
|
1243
|
+
// drop the token before ensureWorkerWorkspace ever records the workspace.
|
|
1244
|
+
deps.ownership?.rememberWorkspace(project, w.workspaceId);
|
|
1245
|
+
}
|
|
1246
|
+
for (const project of projects) {
|
|
1247
|
+
for (const id of deps.ownership?.recordedWorkspaces(project) ?? []) {
|
|
1248
|
+
if (owned.has(id)) continue;
|
|
1249
|
+
const hit = raw.workspaces.find((w) => w.workspaceId === id);
|
|
1250
|
+
if (hit === undefined) {
|
|
1251
|
+
// The workspace disappeared outside conductor. Forget the stale id now
|
|
1252
|
+
// so a future Herdr session cannot reuse it as false ownership.
|
|
1253
|
+
deps.ownership?.forgetWorkspace(project, id);
|
|
1254
|
+
continue;
|
|
1255
|
+
}
|
|
1256
|
+
owned.set(id, { workspaceId: id, project, ...(hit.label === undefined ? {} : { label: hit.label }) });
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
return { ok: true, workspaces: [...owned.values()] };
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
export type WorkerWorkspaceResolution =
|
|
1263
|
+
| { kind: "ready"; workspaceId: string; anchorPaneId: string; created?: true }
|
|
1264
|
+
| { kind: "unavailable"; reason: string };
|
|
1265
|
+
|
|
1266
|
+
/**
|
|
1267
|
+
* Find this project's worker workspace or create it — the single door every
|
|
1268
|
+
* split goes through (#1035).
|
|
1269
|
+
*
|
|
1270
|
+
* Identity is two-legged now (#1035 review): the marking token OR the
|
|
1271
|
+
* conductor-store record, so rediscovery after any restart finds the same
|
|
1272
|
+
* workspace whatever Herdr preserved. Duplicates are deliberately NOT
|
|
1273
|
+
* resolved here: closing one blind — without listing its panes or checking
|
|
1274
|
+
* the result — could destroy the only live representation and strand the
|
|
1275
|
+
* durable row pointing at a dead pane. Convergence belongs to
|
|
1276
|
+
* {@link reconcileWorkerPanes}, which attributes every pane first and removes
|
|
1277
|
+
* a duplicate only once nothing living holds it. A discovered workspace with
|
|
1278
|
+
* no panes gets a fresh tab rather than failing forever; a workspace CREATED
|
|
1279
|
+
* by this call is closed again on every later failure, because newly-made
|
|
1280
|
+
* residue is exactly what the acceptance criterion forbids.
|
|
1281
|
+
*/
|
|
1282
|
+
export function ensureWorkerWorkspace(
|
|
1283
|
+
project: string,
|
|
1284
|
+
deps: { run?: HerdrRun; session?: string; ownership?: WorkspaceOwnership } = {},
|
|
1285
|
+
): WorkerWorkspaceResolution {
|
|
1286
|
+
const run = deps.run ?? realHerdrRun;
|
|
1287
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
1288
|
+
const base = ["--session", session];
|
|
1289
|
+
|
|
1290
|
+
let listed = ownedWorkerWorkspaces([project], deps);
|
|
1291
|
+
if (!listed.ok) return { kind: "unavailable", reason: listed.reason };
|
|
1292
|
+
|
|
1293
|
+
for (const existing of [...listed.workspaces].sort((a, b) => (a.workspaceId < b.workspaceId ? -1 : 1))) {
|
|
1294
|
+
const res = run([...base, "pane", "list", "--workspace", existing.workspaceId]);
|
|
1295
|
+
if (!res.ok) {
|
|
1296
|
+
return { kind: "unavailable", reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}` };
|
|
1297
|
+
}
|
|
1298
|
+
const parsed = parsePaneRecords(res.stdout);
|
|
1299
|
+
if (parsed === undefined) return { kind: "unavailable", reason: "herdr pane list was unreadable" };
|
|
1300
|
+
let anchor = parsed.panes[0]?.paneId;
|
|
1301
|
+
if (anchor === undefined) {
|
|
1302
|
+
// No pane to split: give the workspace a fresh tab and use its root pane.
|
|
1303
|
+
const tabbed = run([...base, "tab", "create", "--workspace", existing.workspaceId, "--no-focus"]);
|
|
1304
|
+
if (!tabbed.ok) {
|
|
1305
|
+
return { kind: "unavailable", reason: `herdr tab create failed: ${firstLine(tabbed.stderr) || "no output"}` };
|
|
1306
|
+
}
|
|
1307
|
+
anchor = parseRootPane(tabbed.stdout);
|
|
1308
|
+
if (anchor === undefined) {
|
|
1309
|
+
return { kind: "unavailable", reason: "herdr tab create reported no root pane" };
|
|
1310
|
+
}
|
|
1311
|
+
}
|
|
1312
|
+
// Adopted into the store: if this workspace's token is what Herdr drops
|
|
1313
|
+
// next restart, the record below is what still finds it.
|
|
1314
|
+
deps.ownership?.rememberWorkspace(project, existing.workspaceId);
|
|
1315
|
+
return { kind: "ready", workspaceId: existing.workspaceId, anchorPaneId: anchor };
|
|
1316
|
+
}
|
|
1317
|
+
|
|
1318
|
+
const made = run([
|
|
1319
|
+
...base,
|
|
1320
|
+
"workspace",
|
|
1321
|
+
"create",
|
|
1322
|
+
"--label",
|
|
1323
|
+
workerWorkspaceLabel(project),
|
|
1324
|
+
"--no-focus",
|
|
1325
|
+
]);
|
|
1326
|
+
if (!made.ok) {
|
|
1327
|
+
return { kind: "unavailable", reason: `herdr workspace create failed: ${firstLine(made.stderr) || "no output"}` };
|
|
1328
|
+
}
|
|
1329
|
+
const parsed = parseCreatedWorkspace(made.stdout);
|
|
1330
|
+
if (parsed === undefined) {
|
|
1331
|
+
return { kind: "unavailable", reason: "herdr workspace create reported no workspace id" };
|
|
1332
|
+
}
|
|
1333
|
+
/** This attempt made the workspace, so every exit from here takes it back. */
|
|
1334
|
+
const discardMade = (): void => {
|
|
1335
|
+
run([...base, "workspace", "close", parsed.workspaceId]);
|
|
1336
|
+
deps.ownership?.forgetWorkspace(project, parsed.workspaceId);
|
|
1337
|
+
};
|
|
1338
|
+
// Mark before use: an unmarked workspace would be invisible to token-side
|
|
1339
|
+
// discovery on the next pass — an empty shell accumulating precisely as
|
|
1340
|
+
// #1035 describes. A failure here takes the half-made workspace with it.
|
|
1341
|
+
const marked = run([
|
|
1342
|
+
...base,
|
|
1343
|
+
"workspace",
|
|
1344
|
+
"report-metadata",
|
|
1345
|
+
parsed.workspaceId,
|
|
1346
|
+
"--source",
|
|
1347
|
+
WORKER_PANE_SOURCE,
|
|
1348
|
+
"--token",
|
|
1349
|
+
`${WORKER_WORKSPACE_TOKEN}=${project}`,
|
|
1350
|
+
]);
|
|
1351
|
+
if (!marked.ok) {
|
|
1352
|
+
discardMade();
|
|
1353
|
+
return {
|
|
1354
|
+
kind: "unavailable",
|
|
1355
|
+
reason: `herdr workspace report-metadata failed: ${firstLine(marked.stderr) || "no output"}`,
|
|
1356
|
+
};
|
|
1357
|
+
}
|
|
1358
|
+
if (parsed.rootPaneId === undefined) {
|
|
1359
|
+
// No anchor means no split can ever be targeted; a fresh-made workspace
|
|
1360
|
+
// without one is residue this very attempt must not leave behind
|
|
1361
|
+
// (#1035 review) — so it goes now, not on some later pass.
|
|
1362
|
+
discardMade();
|
|
1363
|
+
return { kind: "unavailable", reason: "herdr workspace create reported no root pane" };
|
|
1364
|
+
}
|
|
1365
|
+
deps.ownership?.rememberWorkspace(project, parsed.workspaceId);
|
|
1366
|
+
return { kind: "ready", workspaceId: parsed.workspaceId, anchorPaneId: parsed.rootPaneId, created: true };
|
|
1367
|
+
}
|
|
1368
|
+
|
|
960
1369
|
/**
|
|
961
1370
|
* Create the pane, run the follower in it, and report the child's identity and
|
|
962
1371
|
* initial state — or say exactly why it could not.
|
|
963
1372
|
*
|
|
964
|
-
*
|
|
965
|
-
*
|
|
966
|
-
*
|
|
967
|
-
*
|
|
968
|
-
*
|
|
969
|
-
*
|
|
970
|
-
*
|
|
1373
|
+
* The representation lives in the project's worker workspace (#1035), found or
|
|
1374
|
+
* created first, and the split targets an anchor pane inside it explicitly, so
|
|
1375
|
+
* the operator's own layout is never touched. Every step is checked, and the
|
|
1376
|
+
* first failure returns `unavailable` with the reason and leaves ZERO residue
|
|
1377
|
+
* of this attempt (#1035 review): the pane the split had already created is
|
|
1378
|
+
* closed, and a workspace this call CREATED is closed with it — only surfaces
|
|
1379
|
+
* that already existed are preserved. A partially established representation
|
|
1380
|
+
* is worse than none: it is a pane the operator would read as a tracked
|
|
1381
|
+
* worker, and before #992 it was left behind on every single launch. What
|
|
1382
|
+
* this deliberately does NOT do is decide what a failure means for the launch
|
|
1383
|
+
* — failing closed versus running degraded is #841's policy, and inventing it
|
|
1384
|
+
* here would pre-empt it.
|
|
971
1385
|
*/
|
|
972
1386
|
export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDeps = {}): WorkerPaneOutcome {
|
|
973
1387
|
const run = deps.run ?? realHerdrRun;
|
|
974
1388
|
const session = deps.session ?? resolveHerdrSession();
|
|
975
1389
|
const label = workerPaneLabel(identity);
|
|
1390
|
+
const ensured = ensureWorkerWorkspace(identity.project, deps);
|
|
1391
|
+
if (ensured.kind === "unavailable") return ensured;
|
|
976
1392
|
const base = ["--session", session, "pane"];
|
|
977
1393
|
|
|
978
|
-
|
|
1394
|
+
/** Undo everything this attempt built so far — the split's pane once there
|
|
1395
|
+
* is one, and always a workspace this call itself created (#1035 review). */
|
|
1396
|
+
const abandon = (reason: string, phase?: "split", paneId?: string): WorkerPaneOutcome => {
|
|
1397
|
+
if (paneId !== undefined) run([...base, "close", paneId]);
|
|
1398
|
+
if (ensured.created === true) {
|
|
1399
|
+
run(["--session", session, "workspace", "close", ensured.workspaceId]);
|
|
1400
|
+
deps.ownership?.forgetWorkspace(identity.project, ensured.workspaceId);
|
|
1401
|
+
}
|
|
1402
|
+
return { kind: "unavailable", reason, ...(phase === undefined ? {} : { phase }) };
|
|
1403
|
+
};
|
|
1404
|
+
|
|
1405
|
+
// The explicit target is the whole point (#1035): the split names a pane
|
|
1406
|
+
// inside the worker workspace, so whichever pane the operator happens to be
|
|
1407
|
+
// looking at is never touched.
|
|
1408
|
+
const split = run([
|
|
1409
|
+
...base,
|
|
1410
|
+
"split",
|
|
1411
|
+
"--pane",
|
|
1412
|
+
ensured.anchorPaneId,
|
|
1413
|
+
"--direction",
|
|
1414
|
+
"down",
|
|
1415
|
+
"--ratio",
|
|
1416
|
+
"0.3",
|
|
1417
|
+
"--no-focus",
|
|
1418
|
+
]);
|
|
979
1419
|
if (!split.ok) {
|
|
980
|
-
return
|
|
1420
|
+
return abandon(
|
|
1421
|
+
`herdr pane split failed: ${firstLine(split.stderr) || "no output"}`,
|
|
1422
|
+
"split",
|
|
1423
|
+
);
|
|
981
1424
|
}
|
|
982
1425
|
const paneId = parsePaneId(split.stdout);
|
|
983
1426
|
if (paneId === undefined) {
|
|
984
|
-
return
|
|
1427
|
+
return abandon("herdr pane split reported no pane id");
|
|
985
1428
|
}
|
|
986
|
-
// The split has created a pane. From here every exit must take it with it.
|
|
987
|
-
const abandon = (reason: string): WorkerPaneOutcome => {
|
|
988
|
-
run([...base, "close", paneId]);
|
|
989
|
-
return { kind: "unavailable", reason };
|
|
990
|
-
};
|
|
991
1429
|
|
|
992
1430
|
const named = run([...base, "rename", paneId, label]);
|
|
993
1431
|
if (!named.ok) {
|
|
994
|
-
return abandon(`herdr pane rename failed: ${firstLine(named.stderr) || "no output"}
|
|
1432
|
+
return abandon(`herdr pane rename failed: ${firstLine(named.stderr) || "no output"}`, undefined, paneId);
|
|
995
1433
|
}
|
|
996
1434
|
|
|
997
1435
|
// Identity before display: the pane must be attributable to this exact run
|
|
@@ -1012,12 +1450,16 @@ export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDep
|
|
|
1012
1450
|
WORKER_PANE_SOURCE,
|
|
1013
1451
|
]);
|
|
1014
1452
|
if (!identified.ok) {
|
|
1015
|
-
return abandon(
|
|
1453
|
+
return abandon(
|
|
1454
|
+
`herdr pane report-agent-session failed: ${firstLine(identified.stderr) || "no output"}`,
|
|
1455
|
+
undefined,
|
|
1456
|
+
paneId,
|
|
1457
|
+
);
|
|
1016
1458
|
}
|
|
1017
1459
|
|
|
1018
1460
|
const started = run([...base, "run", paneId, ...deps.viewer?.(identity) ?? defaultViewer(identity)]);
|
|
1019
1461
|
if (!started.ok) {
|
|
1020
|
-
return abandon(`herdr pane run failed: ${firstLine(started.stderr) || "no output"}
|
|
1462
|
+
return abandon(`herdr pane run failed: ${firstLine(started.stderr) || "no output"}`, undefined, paneId);
|
|
1021
1463
|
}
|
|
1022
1464
|
|
|
1023
1465
|
// A worker that has just spawned is working by definition. The ongoing
|
|
@@ -1025,9 +1467,9 @@ export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDep
|
|
|
1025
1467
|
// events — never from the pane's output.
|
|
1026
1468
|
const reported = reportWorkerPaneState(paneId, label, "working", { run, session });
|
|
1027
1469
|
if (!reported.ok) {
|
|
1028
|
-
return abandon(reported.reason);
|
|
1470
|
+
return abandon(reported.reason, undefined, paneId);
|
|
1029
1471
|
}
|
|
1030
|
-
return { kind: "tracked", paneId, label, pid: identity.pid };
|
|
1472
|
+
return { kind: "tracked", paneId, label, pid: identity.pid, workspaceId: ensured.workspaceId };
|
|
1031
1473
|
}
|
|
1032
1474
|
|
|
1033
1475
|
/** Report one lifecycle state for a tracked pane. Monotonic `seq` is the caller's. */
|
|
@@ -1060,12 +1502,11 @@ export function reportWorkerPaneState(
|
|
|
1060
1502
|
}
|
|
1061
1503
|
|
|
1062
1504
|
/**
|
|
1063
|
-
* Hand lifecycle authority back when
|
|
1064
|
-
*
|
|
1065
|
-
*
|
|
1066
|
-
*
|
|
1067
|
-
*
|
|
1068
|
-
* no longer speaks for this agent", which is exactly what is true.
|
|
1505
|
+
* Hand lifecycle authority back when conductor no longer speaks for a pane's
|
|
1506
|
+
* agent. Half of {@link retireWorkerPane}: it says "this agent record is no
|
|
1507
|
+
* longer ours" without removing anything, which is what reconciliation wants
|
|
1508
|
+
* for a pane it is about to close anyway but also stands alone for callers
|
|
1509
|
+
* that only re-report identity (#842's adopt path).
|
|
1069
1510
|
*/
|
|
1070
1511
|
export function releaseWorkerPane(
|
|
1071
1512
|
paneId: string,
|
|
@@ -1091,71 +1532,153 @@ export function releaseWorkerPane(
|
|
|
1091
1532
|
: { ok: false, reason: `herdr pane release-agent failed: ${firstLine(res.stderr) || "no output"}` };
|
|
1092
1533
|
}
|
|
1093
1534
|
|
|
1535
|
+
/** Close one pane outright. The pane holds only ever a follower, so closing
|
|
1536
|
+
* it cannot reach the authoritative worker — that child belongs to the
|
|
1537
|
+
* daemon's process tree, not to any terminal (#1035). */
|
|
1538
|
+
export function closeWorkerPane(
|
|
1539
|
+
paneId: string,
|
|
1540
|
+
deps: { run?: HerdrRun; session?: string } = {},
|
|
1541
|
+
): { ok: true } | { ok: false; reason: string } {
|
|
1542
|
+
const run = deps.run ?? realHerdrRun;
|
|
1543
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
1544
|
+
const res = run(["--session", session, "pane", "close", paneId]);
|
|
1545
|
+
return res.ok
|
|
1546
|
+
? { ok: true }
|
|
1547
|
+
: { ok: false, reason: `herdr pane close failed: ${firstLine(res.stderr) || "no output"}` };
|
|
1548
|
+
}
|
|
1549
|
+
|
|
1550
|
+
/**
|
|
1551
|
+
* Retire one run's visual representation (#1035): hand lifecycle authority
|
|
1552
|
+
* back to Herdr, then close the pane.
|
|
1553
|
+
*
|
|
1554
|
+
* #841 originally stopped at release, and production showed what that leaves:
|
|
1555
|
+
* panes advertising `working` for runs that had settled hours earlier, their
|
|
1556
|
+
* follower long exited, a bare shell where a worker used to be. A settled run
|
|
1557
|
+
* keeps no representation. The close reaches only the follower — the
|
|
1558
|
+
* authoritative child was never in the pane, so nothing here can signal it.
|
|
1559
|
+
*
|
|
1560
|
+
* A close failure is reported, not swallowed: the next reconciliation pass
|
|
1561
|
+
* sees the pane as unclaimed junk and closes it again, so one refusal costs a
|
|
1562
|
+
* pass rather than the representation.
|
|
1563
|
+
*/
|
|
1564
|
+
export function retireWorkerPane(
|
|
1565
|
+
paneId: string,
|
|
1566
|
+
label: string,
|
|
1567
|
+
deps: { run?: HerdrRun; session?: string; seq?: number } = {},
|
|
1568
|
+
): { ok: true } | { ok: false; reason: string } {
|
|
1569
|
+
const released = releaseWorkerPane(paneId, label, deps);
|
|
1570
|
+
if (!released.ok) return released;
|
|
1571
|
+
return closeWorkerPane(paneId, deps);
|
|
1572
|
+
}
|
|
1573
|
+
|
|
1094
1574
|
/**
|
|
1095
|
-
* Hand back the
|
|
1575
|
+
* Hand back the representation a dead worker left behind (#842), retired for
|
|
1576
|
+
* good by closing it (#1035) — but only once the recorded pane is PROVEN still
|
|
1577
|
+
* conductor's (#1035 review).
|
|
1096
1578
|
*
|
|
1097
1579
|
* Called for every run a restart reaps. The recorded pid is deliberately NOT
|
|
1098
1580
|
* consulted for liveness: a `session-host` child dies with the daemon that owned
|
|
1099
1581
|
* its verb socket, so an orphaned row's worker is gone whatever pid it carries —
|
|
1100
1582
|
* and pids are reused, so checking one is how a stranger's process comes to read
|
|
1101
|
-
* as a live worker.
|
|
1102
|
-
*
|
|
1583
|
+
* as a live worker. The same reuse argument applies to pane ids: after a Herdr
|
|
1584
|
+
* restart the stored `w2:pT` may belong to a pane in somebody else's workspace,
|
|
1585
|
+
* so a blind release+close could destroy a stranger's terminal. Ownership is
|
|
1586
|
+
* therefore proven before anything is closed:
|
|
1587
|
+
*
|
|
1588
|
+
* - the pane must be listed inside a workspace conductor OWNS — a
|
|
1589
|
+
* {@link WORKER_WORKSPACE_TOKEN}-marked one, or one the store recorded for
|
|
1590
|
+
* the run's project (#1035 review: Herdr restarts restore workspaces without
|
|
1591
|
+
* their tokens, and a token-only check would leave the restored pane
|
|
1592
|
+
* unretired forever); and
|
|
1103
1593
|
*/
|
|
1104
1594
|
export function releaseOrphanedWorkerPane(
|
|
1105
|
-
run: { paneId?: string; paneLabel?: string },
|
|
1106
|
-
deps: { run?: HerdrRun; session?: string; seq?: number } = {},
|
|
1107
|
-
):
|
|
1595
|
+
run: { paneId?: string; paneLabel?: string; runId?: string; project?: string },
|
|
1596
|
+
deps: { run?: HerdrRun; session?: string; seq?: number; ownership?: WorkspaceOwnership } = {},
|
|
1597
|
+
):
|
|
1598
|
+
| { kind: "none" }
|
|
1599
|
+
| { kind: "released"; paneId: string }
|
|
1600
|
+
| { kind: "failed"; paneId: string; reason: string }
|
|
1601
|
+
| { kind: "unowned"; paneId: string; reason: string } {
|
|
1108
1602
|
if (run.paneId === undefined || run.paneLabel === undefined) return { kind: "none" };
|
|
1109
|
-
const
|
|
1110
|
-
|
|
1603
|
+
const discovered = ownedWorkerWorkspaces(run.project === undefined ? [] : [run.project], deps);
|
|
1604
|
+
if (!discovered.ok) {
|
|
1605
|
+
return { kind: "failed", paneId: run.paneId, reason: discovered.reason };
|
|
1606
|
+
}
|
|
1607
|
+
const runFn = deps.run ?? realHerdrRun;
|
|
1608
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
1609
|
+
let home: string | undefined;
|
|
1610
|
+
let reportedAgent: string | undefined;
|
|
1611
|
+
let reportedSession: string | undefined;
|
|
1612
|
+
for (const ws of discovered.workspaces) {
|
|
1613
|
+
const res = runFn(["--session", session, "pane", "list", "--workspace", ws.workspaceId]);
|
|
1614
|
+
if (!res.ok) {
|
|
1615
|
+
return {
|
|
1616
|
+
kind: "failed",
|
|
1617
|
+
paneId: run.paneId,
|
|
1618
|
+
reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}`,
|
|
1619
|
+
};
|
|
1620
|
+
}
|
|
1621
|
+
const parsed = parsePaneRecords(res.stdout);
|
|
1622
|
+
if (parsed === undefined) {
|
|
1623
|
+
return { kind: "failed", paneId: run.paneId, reason: "herdr pane list was unreadable" };
|
|
1624
|
+
}
|
|
1625
|
+
const found = parsed.panes.find((pane) => pane.paneId === run.paneId);
|
|
1626
|
+
if (found !== undefined && home === undefined) {
|
|
1627
|
+
home = ws.workspaceId;
|
|
1628
|
+
if (found.agent !== undefined) reportedAgent = found.agent;
|
|
1629
|
+
if (found.runId !== "") reportedSession = found.runId;
|
|
1630
|
+
}
|
|
1631
|
+
}
|
|
1632
|
+
if (home === undefined) {
|
|
1633
|
+
// Not in any marked workspace: either already retired or never ours to
|
|
1634
|
+
// touch. Both answers close nothing.
|
|
1635
|
+
return { kind: "none" };
|
|
1636
|
+
}
|
|
1637
|
+
if (
|
|
1638
|
+
(reportedAgent !== undefined && reportedAgent !== run.paneLabel) ||
|
|
1639
|
+
(reportedSession !== undefined && run.runId !== undefined && reportedSession !== run.runId)
|
|
1640
|
+
) {
|
|
1641
|
+
return {
|
|
1642
|
+
kind: "unowned",
|
|
1643
|
+
paneId: run.paneId,
|
|
1644
|
+
reason: `pane ${run.paneId} in marked workspace ${home} reports another identity (agent ${reportedAgent ?? "none"}, session ${reportedSession ?? "none"})`,
|
|
1645
|
+
};
|
|
1646
|
+
}
|
|
1647
|
+
const retired = retireWorkerPane(run.paneId, run.paneLabel, deps);
|
|
1648
|
+
return retired.ok
|
|
1111
1649
|
? { kind: "released", paneId: run.paneId }
|
|
1112
|
-
: { kind: "failed", paneId: run.paneId, reason:
|
|
1650
|
+
: { kind: "failed", paneId: run.paneId, reason: retired.reason };
|
|
1113
1651
|
}
|
|
1114
1652
|
|
|
1115
1653
|
/**
|
|
1116
|
-
* Every
|
|
1117
|
-
* reported under (#841).
|
|
1654
|
+
* Every pane inside a conductor-marked worker workspace (#1035).
|
|
1118
1655
|
*
|
|
1119
|
-
*
|
|
1120
|
-
*
|
|
1121
|
-
*
|
|
1122
|
-
*
|
|
1656
|
+
* Ownership here is structural: conductor created the workspace, marked it,
|
|
1657
|
+
* and splits into it exclusively, so every pane inside it is conductor's
|
|
1658
|
+
* however Herdr feels about reporting `agent_session` back — production has
|
|
1659
|
+
* repeatedly handed back worker panes with that record dropped to null, which
|
|
1660
|
+
* is why the previous source-filtered listing could not see the very panes
|
|
1661
|
+
* that needed cleaning. The reported session id still rides along as a
|
|
1662
|
+
* secondary attribution signal; an empty run id means "unattributable", and
|
|
1663
|
+
* unattributable panes in a marked workspace are exactly the stale shells
|
|
1664
|
+
* this surface exists to converge away.
|
|
1123
1665
|
*/
|
|
1124
1666
|
export function listWorkerPanes(
|
|
1125
1667
|
deps: { run?: HerdrRun; session?: string } = {},
|
|
1126
|
-
): { ok: true; panes:
|
|
1668
|
+
): { ok: true; panes: (ListedPane & { project: string })[] } | { ok: false; reason: string } {
|
|
1669
|
+
const workspaces = listWorkerWorkspaces(deps);
|
|
1670
|
+
if (!workspaces.ok) return workspaces;
|
|
1127
1671
|
const run = deps.run ?? realHerdrRun;
|
|
1128
1672
|
const session = deps.session ?? resolveHerdrSession();
|
|
1129
|
-
const
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
parsed =
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
}
|
|
1139
|
-
const panes = (parsed as { result?: { panes?: unknown[] } }).result?.panes;
|
|
1140
|
-
if (!Array.isArray(panes)) return { ok: false, reason: "herdr pane list carried no pane array" };
|
|
1141
|
-
const owned: { paneId: string; runId: string; label?: string }[] = [];
|
|
1142
|
-
for (const pane of panes) {
|
|
1143
|
-
const p = pane as {
|
|
1144
|
-
pane_id?: unknown;
|
|
1145
|
-
agent?: unknown;
|
|
1146
|
-
agent_session?: { source?: unknown; value?: unknown };
|
|
1147
|
-
};
|
|
1148
|
-
if (typeof p.pane_id !== "string") continue;
|
|
1149
|
-
if (p.agent_session?.source !== WORKER_PANE_SOURCE) continue;
|
|
1150
|
-
const runId = p.agent_session.value;
|
|
1151
|
-
// Ours by source but carrying no run id: an identity nobody can resolve is
|
|
1152
|
-
// not an identity. Reported as a pane with an empty run id so the caller
|
|
1153
|
-
// treats it as stale rather than silently ignoring it.
|
|
1154
|
-
owned.push({
|
|
1155
|
-
paneId: p.pane_id,
|
|
1156
|
-
runId: typeof runId === "string" ? runId : "",
|
|
1157
|
-
...(typeof p.agent === "string" ? { label: p.agent } : {}),
|
|
1158
|
-
});
|
|
1673
|
+
const owned: (ListedPane & { project: string })[] = [];
|
|
1674
|
+
for (const ws of workspaces.workspaces) {
|
|
1675
|
+
const res = run(["--session", session, "pane", "list", "--workspace", ws.workspaceId]);
|
|
1676
|
+
if (!res.ok) {
|
|
1677
|
+
return { ok: false, reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}` };
|
|
1678
|
+
}
|
|
1679
|
+
const parsed = parsePaneRecords(res.stdout);
|
|
1680
|
+
if (parsed === undefined) return { ok: false, reason: "herdr pane list carried no pane array" };
|
|
1681
|
+
for (const pane of parsed.panes) owned.push({ ...pane, project: ws.project });
|
|
1159
1682
|
}
|
|
1160
1683
|
return { ok: true, panes: owned };
|
|
1161
1684
|
}
|
|
@@ -1173,43 +1696,147 @@ export interface LiveWorkerPane {
|
|
|
1173
1696
|
sessionFile?: string;
|
|
1174
1697
|
}
|
|
1175
1698
|
|
|
1699
|
+
export type StaleCause = "settled" | "duplicate" | "unidentified";
|
|
1700
|
+
|
|
1176
1701
|
export type WorkerPaneReconciliation =
|
|
1177
|
-
/** The recorded pane is still there and
|
|
1702
|
+
/** The recorded pane is still there and its follower is alive — nothing done. */
|
|
1178
1703
|
| { kind: "intact"; runId: string; paneId: string }
|
|
1179
|
-
/**
|
|
1704
|
+
/** The pane stands but its follower table could not decide life or death:
|
|
1705
|
+
* no evidence, no destruction (#1035 review). The visual may be stale; the
|
|
1706
|
+
* caller says so once instead of gambling a live worker's representation. */
|
|
1707
|
+
| { kind: "visual-unreadable"; runId: string; paneId: string; reason: string }
|
|
1708
|
+
/** The run had no usable representation; a new one now represents the same
|
|
1709
|
+
* child — a Herdr restart, a missing pane, or a follower that had exited. */
|
|
1180
1710
|
| { kind: "reassociated"; runId: string; paneId: string; label: string }
|
|
1181
1711
|
/** No representation, and the reason. The run keeps working regardless.
|
|
1182
1712
|
* `attempted` marks the ones that spent a re-establishment attempt (#998),
|
|
1183
|
-
* so the caller can bound them; the no-pid case costs nothing.
|
|
1184
|
-
|
|
1713
|
+
* so the caller can bound them; the no-pid case costs nothing. `phase`
|
|
1714
|
+
* carries {@link WorkerPaneOutcome}'s split marker through. */
|
|
1715
|
+
| { kind: "untracked"; runId: string; reason: string; attempted?: true; phase?: "split" }
|
|
1185
1716
|
/** The attempt budget for this run is spent, so nothing was tried this pass
|
|
1186
1717
|
* (#998). Before this existed, a run whose pane could not be created was
|
|
1187
1718
|
* retried on every reconciliation pass forever — and each attempt leaked a
|
|
1188
1719
|
* pane until #992, ending in `ghostty error -2` once enough had piled up. */
|
|
1189
1720
|
| { kind: "attempts-exhausted"; runId: string; attempts: number }
|
|
1190
|
-
/** A conductor pane
|
|
1191
|
-
|
|
1192
|
-
|
|
1721
|
+
/** A conductor-workspace pane with no live run behind it: authority handed
|
|
1722
|
+
* back and the pane closed. `settled` — the run finished and its retirement
|
|
1723
|
+
* never landed; `duplicate` — a second representation of one live run, or
|
|
1724
|
+
* the one a fresh replacement displaced; `unidentified` — a representation
|
|
1725
|
+
* whose run id nobody can resolve any more, the bare shells production
|
|
1726
|
+
* accumulated (#1035). */
|
|
1727
|
+
| { kind: "stale-released"; paneId: string; runId: string; cause: StaleCause }
|
|
1728
|
+
| { kind: "stale-release-failed"; paneId: string; runId: string; reason: string; cause: StaleCause }
|
|
1729
|
+
/** A scoped marked workspace nobody's live run holds any more, emptied by
|
|
1730
|
+
* the cleanup above, is closed (#1035). Every marked workspace converges,
|
|
1731
|
+
* not just one per project (#1035 review). */
|
|
1732
|
+
| { kind: "workspace-removed"; project: string; workspaceId: string }
|
|
1733
|
+
| { kind: "workspace-remove-failed"; project: string; workspaceId: string; reason: string };
|
|
1734
|
+
|
|
1735
|
+
/** The verdict the foreground process table supports about one pane's
|
|
1736
|
+
* read-only follower. Terminal output is never read: this says whether the
|
|
1737
|
+
* *visual* is alive, nothing more, and settlement stays the store's business
|
|
1738
|
+
* (#1035). `unknown` is a real verdict, not an error — recognition of
|
|
1739
|
+
* `omp-conductor` in the table is positive evidence of life, but its absence
|
|
1740
|
+
* is only evidence of death when every entry carries enough identity (name or
|
|
1741
|
+
* argv) to rule it out. Real Herdr answers name + argv; a table with blanked
|
|
1742
|
+
* or missing fields, no shell pid, or no entries at all cannot decide, and a
|
|
1743
|
+
* pane destroyed on a guess we did not have is a live worker gone blind. */
|
|
1744
|
+
export type FollowerPulse =
|
|
1745
|
+
| { pulse: "alive" }
|
|
1746
|
+
| { pulse: "dead" }
|
|
1747
|
+
| { pulse: "unknown"; reason: string };
|
|
1748
|
+
|
|
1749
|
+
export function followerPulseFromProcessInfo(info: ProcessInfo): FollowerPulse {
|
|
1750
|
+
const procs = Array.isArray(info.foreground_processes) ? info.foreground_processes : undefined;
|
|
1751
|
+
if (procs === undefined || procs.length === 0) {
|
|
1752
|
+
return { pulse: "unknown", reason: "process-info carried no foreground process table" };
|
|
1753
|
+
}
|
|
1754
|
+
const shell = typeof info.shell_pid === "number" ? info.shell_pid : undefined;
|
|
1755
|
+
let nonShell = false;
|
|
1756
|
+
for (const proc of procs) {
|
|
1757
|
+
const parts = [proc.name ?? "", proc.argv0 ?? "", ...(proc.argv ?? [])];
|
|
1758
|
+
const isShell = shell !== undefined && proc.pid === shell;
|
|
1759
|
+
if (!isShell && !parts.some((a) => a !== "")) {
|
|
1760
|
+
return { pulse: "unknown", reason: `foreground pid ${proc.pid ?? "unknown"} carries no name or argv` };
|
|
1761
|
+
}
|
|
1762
|
+
if (!isShell) {
|
|
1763
|
+
nonShell = true;
|
|
1764
|
+
if (parts.some((a) => a.toLowerCase() === "omp-conductor" || a.toLowerCase().endsWith("/omp-conductor"))) {
|
|
1765
|
+
return { pulse: "alive" };
|
|
1766
|
+
}
|
|
1767
|
+
}
|
|
1768
|
+
}
|
|
1769
|
+
if (!nonShell && shell === undefined) {
|
|
1770
|
+
return { pulse: "unknown", reason: "process-info named neither a shell nor any identifiable process" };
|
|
1771
|
+
}
|
|
1772
|
+
return { pulse: "dead" };
|
|
1773
|
+
}
|
|
1774
|
+
|
|
1775
|
+
/**
|
|
1776
|
+
* Probe one pane's follower liveness. A probe that cannot be read, or a table
|
|
1777
|
+
* that cannot decide, answers `unknown` rather than dead: destroying a
|
|
1778
|
+
* representation on evidence we do not have is how a healthy worker goes blind
|
|
1779
|
+
* mid-run.
|
|
1780
|
+
*/
|
|
1781
|
+
export function paneFollowerAlive(
|
|
1782
|
+
paneId: string,
|
|
1783
|
+
deps: { run?: HerdrRun; session?: string } = {},
|
|
1784
|
+
): FollowerPulse {
|
|
1785
|
+
const run = deps.run ?? realHerdrRun;
|
|
1786
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
1787
|
+
const res = run(["--session", session, "pane", "process-info", "--pane", paneId]);
|
|
1788
|
+
if (!res.ok) {
|
|
1789
|
+
return {
|
|
1790
|
+
pulse: "unknown",
|
|
1791
|
+
reason: `herdr pane process-info failed: ${firstLine(res.stderr) || "no output"}`,
|
|
1792
|
+
};
|
|
1793
|
+
}
|
|
1794
|
+
try {
|
|
1795
|
+
return followerPulseFromProcessInfo(parseHerdrProcessInfo(res.stdout, paneId));
|
|
1796
|
+
} catch (err) {
|
|
1797
|
+
return { pulse: "unknown", reason: err instanceof Error ? err.message : String(err) };
|
|
1798
|
+
}
|
|
1799
|
+
}
|
|
1800
|
+
|
|
1801
|
+
function firstLine(text: string): string {
|
|
1802
|
+
return text.split("\n", 1)[0]?.trim() ?? "";
|
|
1803
|
+
}
|
|
1804
|
+
/** The kill syscall, injectable so error mapping is testable without one. */
|
|
1805
|
+
export type KillFn = (pid: number, sig: NodeJS.Signals | 0) => void;
|
|
1806
|
+
|
|
1807
|
+
const realKill: KillFn = (pid, sig) => {
|
|
1808
|
+
process.kill(pid, sig);
|
|
1809
|
+
};
|
|
1193
1810
|
|
|
1194
1811
|
/**
|
|
1195
|
-
* Make
|
|
1812
|
+
* Make the conductor-owned worker workspaces agree with the live run set
|
|
1813
|
+
* (#841, reworked by #1035).
|
|
1196
1814
|
*
|
|
1197
1815
|
* Idempotent by construction: a second pass over an already-reconciled fleet
|
|
1198
|
-
* returns `intact` for every live run and finds
|
|
1199
|
-
*
|
|
1816
|
+
* returns `intact` for every live run and finds nothing left to close, so
|
|
1817
|
+
* repeated launch/settlement/restart cycles converge instead of accumulating.
|
|
1200
1818
|
*
|
|
1201
|
-
*
|
|
1819
|
+
* Four rules, and each exists to refuse a specific way this goes wrong:
|
|
1202
1820
|
*
|
|
1203
|
-
* - **
|
|
1204
|
-
*
|
|
1205
|
-
*
|
|
1206
|
-
*
|
|
1207
|
-
*
|
|
1208
|
-
*
|
|
1209
|
-
*
|
|
1210
|
-
*
|
|
1211
|
-
*
|
|
1212
|
-
*
|
|
1821
|
+
* - **Only owned workspaces are touched.** Scope is the projects of the live
|
|
1822
|
+
* set plus the caller's managed list; ownership is the marking token OR the
|
|
1823
|
+
* conductor-store record (#1035 review), so a Herdr restart that drops
|
|
1824
|
+
* tokens still finds the restored surface. A pane in another project's
|
|
1825
|
+
* worker workspace is invisible to this pass, which is what keeps conductor
|
|
1826
|
+
* and veltrosecurity from discovering, reusing, or cleaning each other.
|
|
1827
|
+
* - **Attribution is exact, structural first.** A pane belongs to the live run
|
|
1828
|
+
* the store recorded it under; the session id Herdr reports is the fallback.
|
|
1829
|
+
* Everything else in an owned workspace — a settled run's leftover, a
|
|
1830
|
+
* duplicate of a live run, a representation whose identity Herdr dropped —
|
|
1831
|
+
* is closed, because inside an owned workspace "ours" and "junk" are the
|
|
1832
|
+
* only options and leaving junk is what filled the operator's layout.
|
|
1833
|
+
* - **A live run keeps exactly one living representation.** Duplicates fold
|
|
1834
|
+
* into the recorded pane; a pane whose follower has exited is replaced while
|
|
1835
|
+
* the run is live; a missing pane is recreated — all bounded by the same
|
|
1836
|
+
* re-establishment budget (#998).
|
|
1837
|
+
* - **Nothing here can stop a worker.** Panes hold followers only; the
|
|
1838
|
+
* authoritative child lives in the daemon's process tree and is never
|
|
1839
|
+
* signalled, and worker liveness comes from the caller's live set alone.
|
|
1213
1840
|
*/
|
|
1214
1841
|
export function reconcileWorkerPanes(
|
|
1215
1842
|
live: readonly LiveWorkerPane[],
|
|
@@ -1218,39 +1845,184 @@ export function reconcileWorkerPanes(
|
|
|
1218
1845
|
* and a daemon restart legitimately gets a fresh budget (#998). */
|
|
1219
1846
|
attempts: ReadonlyMap<string, number> = new Map(),
|
|
1220
1847
|
maxAttempts = PANE_REATTEMPT_MAX,
|
|
1848
|
+
/** Projects this pass manages even when they have no live runs: their stale
|
|
1849
|
+
* panes are cleaned and a workspace left empty by that cleanup is removed. */
|
|
1850
|
+
managedProjects: readonly string[] = [],
|
|
1221
1851
|
): { ok: true; outcomes: WorkerPaneReconciliation[] } | { ok: false; reason: string } {
|
|
1222
|
-
const
|
|
1223
|
-
|
|
1224
|
-
const
|
|
1225
|
-
|
|
1852
|
+
const run = deps.run ?? realHerdrRun;
|
|
1853
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
1854
|
+
const base = ["--session", session];
|
|
1855
|
+
|
|
1856
|
+
const scope = new Set([...managedProjects, ...live.map((worker) => worker.project)]);
|
|
1857
|
+
const discovered = ownedWorkerWorkspaces([...scope], deps);
|
|
1858
|
+
if (!discovered.ok) return { ok: false, reason: discovered.reason };
|
|
1859
|
+
// Deterministic order so outcomes and closes are reproducible pass to pass.
|
|
1860
|
+
const scoped = [...discovered.workspaces].sort((a, b) => (a.workspaceId < b.workspaceId ? -1 : 1));
|
|
1861
|
+
|
|
1862
|
+
// Every marked workspace of every scoped project converges (#1035 review):
|
|
1863
|
+
// duplicate token-marked workspaces are cleaned like any other, so a settled
|
|
1864
|
+
// project cannot keep stale shells in a second marked surface nobody tracks.
|
|
1865
|
+
const entries: { ws: WorkerWorkspace; panes: ListedPane[] }[] = [];
|
|
1866
|
+
for (const ws of scoped) {
|
|
1867
|
+
const res = run([...base, "pane", "list", "--workspace", ws.workspaceId]);
|
|
1868
|
+
if (!res.ok) {
|
|
1869
|
+
// An unreadable workspace is not evidence that anything is stale, so the
|
|
1870
|
+
// pass mutates nothing rather than closing on a guess.
|
|
1871
|
+
return { ok: false, reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}` };
|
|
1872
|
+
}
|
|
1873
|
+
const parsed = parsePaneRecords(res.stdout);
|
|
1874
|
+
if (parsed === undefined) return { ok: false, reason: "herdr pane list carried no pane array" };
|
|
1875
|
+
entries.push({ ws, panes: parsed.panes });
|
|
1876
|
+
}
|
|
1877
|
+
|
|
1878
|
+
const liveByRun = new Map(live.map((worker) => [worker.runId, worker]));
|
|
1226
1879
|
const outcomes: WorkerPaneReconciliation[] = [];
|
|
1880
|
+
/** Panes this pass will close, with the identity the outcome names. */
|
|
1881
|
+
const junk = new Map<string, { runId: string; cause: StaleCause; agent?: string }>();
|
|
1882
|
+
/** Panes still standing when cleanup ends, keyed to their workspace — an
|
|
1883
|
+
* occupied workspace survives, whichever marked surface it sits in. */
|
|
1884
|
+
const standingWs = new Map<string, string>();
|
|
1885
|
+
/** Workspaces that gained a representation this pass, read from the tracked
|
|
1886
|
+
* outcomes: their listing predates the new pane, so removal never judges
|
|
1887
|
+
* them on it. */
|
|
1888
|
+
const openedWs = new Set<string>();
|
|
1889
|
+
|
|
1890
|
+
/** One representation per live run: the pane plus the marked workspace that
|
|
1891
|
+
* holds it. Attribution runs across ALL of a project's marked workspaces,
|
|
1892
|
+
* because the recorded pane may sit in any of them. */
|
|
1893
|
+
const held = new Map<string, { pane: ListedPane; wsId: string }>();
|
|
1894
|
+
for (const entry of entries) {
|
|
1895
|
+
const projectLive = live.filter((worker) => worker.project === entry.ws.project);
|
|
1896
|
+
for (const pane of entry.panes) {
|
|
1897
|
+
standingWs.set(pane.paneId, entry.ws.workspaceId);
|
|
1898
|
+
const owner =
|
|
1899
|
+
projectLive.find((worker) => worker.paneId === pane.paneId) ??
|
|
1900
|
+
(pane.runId === "" ? undefined : liveByRun.get(pane.runId));
|
|
1901
|
+
if (owner === undefined || owner.project !== entry.ws.project) {
|
|
1902
|
+
// Not attributable to any live run of this project: a settled run's
|
|
1903
|
+
// leftover (still carrying its old id) or an unidentified shell.
|
|
1904
|
+
junk.set(pane.paneId, {
|
|
1905
|
+
runId: pane.runId,
|
|
1906
|
+
cause: pane.runId === "" ? "unidentified" : "settled",
|
|
1907
|
+
...(pane.agent === undefined ? {} : { agent: pane.agent }),
|
|
1908
|
+
});
|
|
1909
|
+
continue;
|
|
1910
|
+
}
|
|
1911
|
+
const bucket = held.get(owner.runId);
|
|
1912
|
+
if (bucket === undefined) {
|
|
1913
|
+
held.set(owner.runId, { pane, wsId: entry.ws.workspaceId });
|
|
1914
|
+
continue;
|
|
1915
|
+
}
|
|
1916
|
+
// Exactly one focusable representation per live run: the recorded pane
|
|
1917
|
+
// wins so the durable row stays true without a write; the other is a
|
|
1918
|
+
// duplicate (#1035).
|
|
1919
|
+
const keep =
|
|
1920
|
+
bucket.pane.paneId === owner.paneId
|
|
1921
|
+
? bucket
|
|
1922
|
+
: pane.paneId === owner.paneId
|
|
1923
|
+
? { pane, wsId: entry.ws.workspaceId }
|
|
1924
|
+
: bucket;
|
|
1925
|
+
const drop = keep.pane.paneId === bucket.pane.paneId ? pane : bucket.pane;
|
|
1926
|
+
held.set(owner.runId, keep);
|
|
1927
|
+
junk.set(drop.paneId, {
|
|
1928
|
+
runId: owner.runId,
|
|
1929
|
+
cause: "duplicate",
|
|
1930
|
+
...(drop.agent === undefined ? {} : { agent: drop.agent }),
|
|
1931
|
+
});
|
|
1932
|
+
}
|
|
1933
|
+
}
|
|
1227
1934
|
|
|
1228
1935
|
for (const worker of live) {
|
|
1229
|
-
const
|
|
1230
|
-
if (
|
|
1231
|
-
|
|
1936
|
+
const hold = held.get(worker.runId);
|
|
1937
|
+
if (hold === undefined) {
|
|
1938
|
+
// No representation anywhere in the marked workspaces. Only the pid makes
|
|
1939
|
+
// a replacement honest: the pane represents an exact child, so without
|
|
1940
|
+
// one there is nothing to represent and inventing a pane would be the
|
|
1941
|
+
// silent claim this whole surface exists to avoid.
|
|
1942
|
+
if (worker.pid === undefined) {
|
|
1943
|
+
outcomes.push({
|
|
1944
|
+
kind: "untracked",
|
|
1945
|
+
runId: worker.runId,
|
|
1946
|
+
reason: "no session-host pid was ever recorded for this run",
|
|
1947
|
+
});
|
|
1948
|
+
continue;
|
|
1949
|
+
}
|
|
1950
|
+
// The budget, checked before the attempt: an exhausted run is skipped
|
|
1951
|
+
// silently here and reported once by the caller (#998).
|
|
1952
|
+
const spent = attempts.get(worker.runId) ?? 0;
|
|
1953
|
+
if (spent >= maxAttempts) {
|
|
1954
|
+
outcomes.push({ kind: "attempts-exhausted", runId: worker.runId, attempts: spent });
|
|
1955
|
+
continue;
|
|
1956
|
+
}
|
|
1957
|
+
const opened = openWorkerPane(
|
|
1958
|
+
{
|
|
1959
|
+
project: worker.project,
|
|
1960
|
+
issue: worker.issue,
|
|
1961
|
+
attempt: worker.attempt,
|
|
1962
|
+
runId: worker.runId,
|
|
1963
|
+
pid: worker.pid,
|
|
1964
|
+
...(worker.sessionFile === undefined ? {} : { sessionFile: worker.sessionFile }),
|
|
1965
|
+
},
|
|
1966
|
+
deps,
|
|
1967
|
+
);
|
|
1968
|
+
if (opened.kind === "tracked") {
|
|
1969
|
+
if (opened.workspaceId !== undefined) openedWs.add(opened.workspaceId);
|
|
1970
|
+
outcomes.push({
|
|
1971
|
+
kind: "reassociated",
|
|
1972
|
+
runId: worker.runId,
|
|
1973
|
+
paneId: opened.paneId,
|
|
1974
|
+
label: opened.label,
|
|
1975
|
+
});
|
|
1976
|
+
} else {
|
|
1977
|
+
outcomes.push({
|
|
1978
|
+
kind: "untracked",
|
|
1979
|
+
runId: worker.runId,
|
|
1980
|
+
reason: opened.reason,
|
|
1981
|
+
attempted: true,
|
|
1982
|
+
...(opened.phase === undefined ? {} : { phase: opened.phase }),
|
|
1983
|
+
});
|
|
1984
|
+
}
|
|
1985
|
+
continue;
|
|
1986
|
+
}
|
|
1987
|
+
// The pane stands. Whether it still shows anything is the follower's
|
|
1988
|
+
// question, asked only to decide the *visual* — never settlement (#1035).
|
|
1989
|
+
// A probe that cannot be read, or a table that cannot decide, is no
|
|
1990
|
+
// evidence the visual died: it is reported and the pane keeps standing,
|
|
1991
|
+
// because a live worker's representation is never destroyed on a guess.
|
|
1992
|
+
const pulse = paneFollowerAlive(hold.pane.paneId, deps);
|
|
1993
|
+
if (pulse.pulse !== "dead") {
|
|
1994
|
+
outcomes.push(
|
|
1995
|
+
pulse.pulse === "alive"
|
|
1996
|
+
? { kind: "intact", runId: worker.runId, paneId: hold.pane.paneId }
|
|
1997
|
+
: {
|
|
1998
|
+
kind: "visual-unreadable",
|
|
1999
|
+
runId: worker.runId,
|
|
2000
|
+
paneId: hold.pane.paneId,
|
|
2001
|
+
reason: pulse.reason,
|
|
2002
|
+
},
|
|
2003
|
+
);
|
|
1232
2004
|
continue;
|
|
1233
2005
|
}
|
|
1234
|
-
//
|
|
1235
|
-
//
|
|
1236
|
-
//
|
|
1237
|
-
//
|
|
2006
|
+
// The follower has provably exited. A replacement represents the exact
|
|
2007
|
+
// child, so a run with no recorded pid keeps its dead pane rather than
|
|
2008
|
+
// gaining a representation of nobody — reported so the record drops and
|
|
2009
|
+
// the next pass reads the orphan as junk.
|
|
1238
2010
|
if (worker.pid === undefined) {
|
|
1239
2011
|
outcomes.push({
|
|
1240
2012
|
kind: "untracked",
|
|
1241
2013
|
runId: worker.runId,
|
|
1242
|
-
reason: "no session-host pid was ever recorded
|
|
2014
|
+
reason: "follower exited but no session-host pid was ever recorded; not replacing",
|
|
1243
2015
|
});
|
|
1244
2016
|
continue;
|
|
1245
2017
|
}
|
|
1246
|
-
// The budget, checked before the attempt: an exhausted run is skipped
|
|
1247
|
-
// silently here and reported once by the caller (#998).
|
|
1248
2018
|
const spent = attempts.get(worker.runId) ?? 0;
|
|
1249
2019
|
if (spent >= maxAttempts) {
|
|
2020
|
+
// Out of budget: the dead pane keeps its last state until a restart
|
|
2021
|
+
// refreshes the budget or the run settles and retires it outright.
|
|
1250
2022
|
outcomes.push({ kind: "attempts-exhausted", runId: worker.runId, attempts: spent });
|
|
1251
2023
|
continue;
|
|
1252
2024
|
}
|
|
1253
|
-
const
|
|
2025
|
+
const replaced = openWorkerPane(
|
|
1254
2026
|
{
|
|
1255
2027
|
project: worker.project,
|
|
1256
2028
|
issue: worker.issue,
|
|
@@ -1261,36 +2033,89 @@ export function reconcileWorkerPanes(
|
|
|
1261
2033
|
},
|
|
1262
2034
|
deps,
|
|
1263
2035
|
);
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
2036
|
+
if (replaced.kind === "tracked") {
|
|
2037
|
+
if (replaced.workspaceId !== undefined) openedWs.add(replaced.workspaceId);
|
|
2038
|
+
outcomes.push({
|
|
2039
|
+
kind: "reassociated",
|
|
2040
|
+
runId: worker.runId,
|
|
2041
|
+
paneId: replaced.paneId,
|
|
2042
|
+
label: replaced.label,
|
|
2043
|
+
});
|
|
2044
|
+
junk.set(hold.pane.paneId, {
|
|
2045
|
+
runId: worker.runId,
|
|
2046
|
+
cause: "duplicate",
|
|
2047
|
+
...(hold.pane.agent === undefined ? {} : { agent: hold.pane.agent }),
|
|
2048
|
+
});
|
|
2049
|
+
} else {
|
|
2050
|
+
outcomes.push({
|
|
2051
|
+
kind: "untracked",
|
|
2052
|
+
runId: worker.runId,
|
|
2053
|
+
reason: replaced.reason,
|
|
2054
|
+
attempted: true,
|
|
2055
|
+
...(replaced.phase === undefined ? {} : { phase: replaced.phase }),
|
|
2056
|
+
});
|
|
2057
|
+
// The replacement failed; the dead pane has nothing left to show, so it
|
|
2058
|
+
// goes now rather than advertising a worker for another pass. Its record
|
|
2059
|
+
// is dropped either way, so a survivor is junk next pass.
|
|
2060
|
+
closeWorkerPane(hold.pane.paneId, deps);
|
|
2061
|
+
standingWs.delete(hold.pane.paneId);
|
|
2062
|
+
}
|
|
2063
|
+
}
|
|
2064
|
+
|
|
2065
|
+
for (const [paneId, info] of junk) {
|
|
2066
|
+
const released =
|
|
2067
|
+
info.agent === undefined ? ({ ok: true } as const) : releaseWorkerPane(paneId, info.agent, deps);
|
|
2068
|
+
if (released.ok) {
|
|
2069
|
+
const closed = closeWorkerPane(paneId, deps);
|
|
2070
|
+
if (closed.ok) {
|
|
2071
|
+
standingWs.delete(paneId);
|
|
2072
|
+
outcomes.push({ kind: "stale-released", paneId, runId: info.runId, cause: info.cause });
|
|
2073
|
+
continue;
|
|
2074
|
+
}
|
|
2075
|
+
outcomes.push({
|
|
2076
|
+
kind: "stale-release-failed",
|
|
2077
|
+
paneId,
|
|
2078
|
+
runId: info.runId,
|
|
2079
|
+
reason: closed.reason,
|
|
2080
|
+
cause: info.cause,
|
|
2081
|
+
});
|
|
2082
|
+
continue;
|
|
2083
|
+
}
|
|
2084
|
+
outcomes.push({
|
|
2085
|
+
kind: "stale-release-failed",
|
|
2086
|
+
paneId,
|
|
2087
|
+
runId: info.runId,
|
|
2088
|
+
reason: released.reason,
|
|
2089
|
+
cause: info.cause,
|
|
2090
|
+
});
|
|
1269
2091
|
}
|
|
1270
2092
|
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
2093
|
+
// Every scoped marked workspace whose panes the cleanup above emptied, and
|
|
2094
|
+
// that gained no representation this pass, is closed — however many marked
|
|
2095
|
+
// surfaces a project accumulated (#1035 review). A live project's primary
|
|
2096
|
+
// workspace survives because its representation keeps it standing; extras
|
|
2097
|
+
// converge away; repeated cycles hold at zero shells.
|
|
2098
|
+
for (const entry of entries) {
|
|
2099
|
+
if (openedWs.has(entry.ws.workspaceId)) continue;
|
|
2100
|
+
if ([...standingWs.values()].some((wsId) => wsId === entry.ws.workspaceId)) continue;
|
|
2101
|
+
const res = run([...base, "workspace", "close", entry.ws.workspaceId]);
|
|
2102
|
+
if (res.ok) deps.ownership?.forgetWorkspace(entry.ws.project, entry.ws.workspaceId);
|
|
2103
|
+
// A failed close keeps the record: the workspace may still exist, and
|
|
2104
|
+
// forgetting ownership of a live surface is how orphans are manufactured.
|
|
1274
2105
|
outcomes.push(
|
|
1275
|
-
|
|
1276
|
-
? { kind: "
|
|
1277
|
-
: {
|
|
2106
|
+
res.ok
|
|
2107
|
+
? { kind: "workspace-removed", project: entry.ws.project, workspaceId: entry.ws.workspaceId }
|
|
2108
|
+
: {
|
|
2109
|
+
kind: "workspace-remove-failed",
|
|
2110
|
+
project: entry.ws.project,
|
|
2111
|
+
workspaceId: entry.ws.workspaceId,
|
|
2112
|
+
reason: `herdr workspace close failed: ${firstLine(res.stderr) || "no output"}`,
|
|
2113
|
+
},
|
|
1278
2114
|
);
|
|
1279
2115
|
}
|
|
1280
2116
|
return { ok: true, outcomes };
|
|
1281
2117
|
}
|
|
1282
2118
|
|
|
1283
|
-
function firstLine(text: string): string {
|
|
1284
|
-
return text.split("\n", 1)[0]?.trim() ?? "";
|
|
1285
|
-
}
|
|
1286
|
-
|
|
1287
|
-
/** The kill syscall, injectable so error mapping is testable without one. */
|
|
1288
|
-
export type KillFn = (pid: number, sig: NodeJS.Signals | 0) => void;
|
|
1289
|
-
|
|
1290
|
-
const realKill: KillFn = (pid, sig) => {
|
|
1291
|
-
process.kill(pid, sig);
|
|
1292
|
-
};
|
|
1293
|
-
|
|
1294
2119
|
/**
|
|
1295
2120
|
* What a failed `kill` proves.
|
|
1296
2121
|
*
|
|
@@ -2174,7 +2999,7 @@ export async function collectFleetStatus(projectName?: string): Promise<FleetSta
|
|
|
2174
2999
|
telegram,
|
|
2175
3000
|
codeGraph,
|
|
2176
3001
|
brief: briefStatusLine(project),
|
|
2177
|
-
decisions: decisionStatusLine(project.name),
|
|
3002
|
+
decisions: decisionStatusLine(project.name, layers),
|
|
2178
3003
|
failureClasses: failureClassBlock(project.name),
|
|
2179
3004
|
workerPhases,
|
|
2180
3005
|
intake: intakeStatusLine(project.name),
|
|
@@ -2307,7 +3132,10 @@ function failureClassBlock(projectName: string): string | undefined {
|
|
|
2307
3132
|
* watches behind GitHub's checks is not a fleet waiting on its operator, and
|
|
2308
3133
|
* lumping them in made `decisions 3 open` read as three unanswered questions.
|
|
2309
3134
|
*/
|
|
2310
|
-
export function decisionStatusLine(
|
|
3135
|
+
export function decisionStatusLine(
|
|
3136
|
+
projectName: string,
|
|
3137
|
+
layers: Pick<FleetLayers, "daemon" | "ticks">,
|
|
3138
|
+
): string | undefined {
|
|
2311
3139
|
const path = dbPath();
|
|
2312
3140
|
if (!existsSync(path)) return undefined;
|
|
2313
3141
|
let store: Store | undefined;
|
|
@@ -2325,7 +3153,12 @@ export function decisionStatusLine(projectName: string): string | undefined {
|
|
|
2325
3153
|
const hours = Math.max(0, Math.round((Date.now() - oldest.askedAt) / 3_600_000));
|
|
2326
3154
|
line = `decisions ${questions.length} open (oldest ${hours}h)`;
|
|
2327
3155
|
}
|
|
2328
|
-
if (watches.length > 0)
|
|
3156
|
+
if (watches.length > 0) {
|
|
3157
|
+
const monitoring = watchMonitoringState(layers);
|
|
3158
|
+
line += monitoring.monitored
|
|
3159
|
+
? ` · watches ${watches.length} monitored`
|
|
3160
|
+
: ` · watches ${watches.length} durable, unmonitored (${monitoring.reason})`;
|
|
3161
|
+
}
|
|
2329
3162
|
return line;
|
|
2330
3163
|
} catch {
|
|
2331
3164
|
return undefined;
|
|
@@ -2508,10 +3341,13 @@ export function sessionsRoot(): string {
|
|
|
2508
3341
|
* claim-only verdict consumes this — the challenge proof reads conductor
|
|
2509
3342
|
* state, not transcripts (#614).
|
|
2510
3343
|
*/
|
|
2511
|
-
function claimedOrchestratorSessionFile(
|
|
3344
|
+
function claimedOrchestratorSessionFile(
|
|
3345
|
+
named: string | undefined,
|
|
3346
|
+
alive?: (pid: number) => boolean,
|
|
3347
|
+
): string | undefined {
|
|
2512
3348
|
if (named === undefined) return undefined;
|
|
2513
3349
|
try {
|
|
2514
|
-
return resolveClaimedSessionFile(findProject(loadConfig(), named));
|
|
3350
|
+
return resolveClaimedSessionFile(findProject(loadConfig(), named), alive);
|
|
2515
3351
|
} catch {
|
|
2516
3352
|
return undefined;
|
|
2517
3353
|
}
|