omp-conductor 0.18.0 → 0.18.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -1
- package/REFERENCE.md +61 -11
- package/agents/to-spec.md +94 -0
- package/package.json +2 -1
- package/schema/config.schema.json +35 -1
- package/src/admission.ts +204 -75
- package/src/arm-challenge.ts +250 -57
- package/src/ask.ts +268 -7
- package/src/board.ts +17 -3
- package/src/briefs/orchestrator.md +62 -21
- package/src/briefs/to-spec.md +88 -0
- package/src/briefs/worker.md +2 -1
- package/src/cli.ts +124 -1
- package/src/command-help.ts +11 -0
- package/src/command-manifest.ts +38 -5
- package/src/commands/arm.ts +1 -1
- package/src/commands/context.ts +1 -0
- package/src/commands/drain.ts +176 -0
- package/src/commands/extend.ts +6 -10
- package/src/commands/intake.ts +4 -19
- package/src/commands/status.ts +5 -1
- package/src/commands/watch.ts +51 -16
- package/src/commands/worker.ts +9 -10
- package/src/config-schema.ts +43 -6
- package/src/config.ts +65 -9
- package/src/daemon.ts +879 -41
- package/src/dashboard/app.js +4 -1
- package/src/dashboard/server.ts +5 -2
- package/src/decisions.ts +243 -17
- package/src/diff-flags.ts +75 -1
- package/src/doctor.ts +60 -82
- package/src/escalate.ts +31 -14
- package/src/failure-class.ts +28 -2
- package/src/fleet.ts +239 -240
- package/src/gitops.ts +188 -81
- package/src/graph-health.ts +35 -1
- package/src/graph.ts +66 -1
- package/src/harness-loader.ts +59 -0
- package/src/host.ts +242 -2
- package/src/lifecycle.ts +122 -1
- package/src/omp-settings.ts +19 -0
- package/src/omp.ts +183 -21
- package/src/orchestrator-tick.ts +1591 -32
- package/src/orchestrator.ts +12 -0
- package/src/privileged.ts +1 -4
- package/src/release-policy.ts +503 -9
- package/src/session-host.ts +65 -6
- package/src/settlement.ts +69 -17
- package/src/setup-host.ts +1225 -9
- package/src/setup-install.ts +28 -0
- package/src/setup-wizard.ts +154 -3
- package/src/setup.ts +83 -17
- package/src/shell.ts +15 -0
- package/src/status-render.ts +216 -12
- package/src/store.ts +443 -42
- package/src/to-spec.ts +408 -0
- package/src/tracker/github.ts +104 -14
- package/src/types.ts +405 -19
- package/src/upgrade-verify.ts +209 -2
- package/src/upgrade.ts +175 -1
- package/src/verbs/protocol.ts +39 -0
- package/src/verbs/server.ts +765 -56
- package/src/verbs/socket.ts +24 -5
- package/src/worker.ts +12 -2
- package/src/worktree.ts +29 -12
package/src/fleet.ts
CHANGED
|
@@ -15,20 +15,16 @@
|
|
|
15
15
|
|
|
16
16
|
import { spawnSync } from "node:child_process";
|
|
17
17
|
import {
|
|
18
|
-
createReadStream,
|
|
19
18
|
existsSync,
|
|
20
19
|
mkdirSync,
|
|
21
|
-
readdirSync,
|
|
22
20
|
readFileSync,
|
|
23
21
|
rmSync,
|
|
24
|
-
statSync,
|
|
25
22
|
writeFileSync,
|
|
26
23
|
} from "node:fs";
|
|
27
|
-
import { createInterface } from "node:readline";
|
|
28
24
|
import { homedir } from "node:os";
|
|
29
25
|
import { dirname, join, sep } from "node:path";
|
|
30
26
|
import { findProject, loadConfig, resolveArmProof, stateDir } from "./config.ts";
|
|
31
|
-
import {
|
|
27
|
+
import { clearArmTransaction, readArmAcknowledgement, recordArmChallenge } from "./arm-challenge.ts";
|
|
32
28
|
import {
|
|
33
29
|
claimedTelegramTopics,
|
|
34
30
|
lockPidAlive,
|
|
@@ -50,8 +46,24 @@ import { inspectBriefLayout } from "./brief-upgrade.ts";
|
|
|
50
46
|
import { dbPath, openStore } from "./store.ts";
|
|
51
47
|
import { renderBriefForProject } from "./setup.ts";
|
|
52
48
|
import { DEFAULT_ARM_PROOF, type ArmProof, type DaemonStop, type ProjectConfig, type Store } from "./types.ts";
|
|
53
|
-
import { probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
49
|
+
import { DEFAULT_DEPS, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
54
50
|
import { isPaused, setPaused, statusSnapshot, type StatusSnapshot } from "./daemon.ts";
|
|
51
|
+
// The drain surface belongs beside hold/releaseHold on the operator surface:
|
|
52
|
+
// the record is daemon state (implemented next to the pause sentinel), but the
|
|
53
|
+
// later CLI and release #484 children import from here, exactly where they find
|
|
54
|
+
// every other fleet control.
|
|
55
|
+
export {
|
|
56
|
+
cancelDrain,
|
|
57
|
+
consumeDrain,
|
|
58
|
+
createDrain,
|
|
59
|
+
drainPath,
|
|
60
|
+
readDrain,
|
|
61
|
+
type CreateDrainOptions,
|
|
62
|
+
type DrainProblem,
|
|
63
|
+
type DrainRecord,
|
|
64
|
+
type DrainStatus,
|
|
65
|
+
type DrainVerdict,
|
|
66
|
+
} from "./daemon.ts";
|
|
55
67
|
import {
|
|
56
68
|
healthCheck,
|
|
57
69
|
isAlive,
|
|
@@ -59,11 +71,13 @@ import {
|
|
|
59
71
|
probeUnit,
|
|
60
72
|
runSystemctl,
|
|
61
73
|
stopDaemon,
|
|
74
|
+
type HealthCheckResult,
|
|
62
75
|
type StopResult,
|
|
63
76
|
} from "./lifecycle.ts";
|
|
64
77
|
import type { WorkerPausePhase } from "./worker.ts";
|
|
65
78
|
import {
|
|
66
79
|
formatFleetStatus,
|
|
80
|
+
formatGroomingStatus,
|
|
67
81
|
type DaemonProjectHealth,
|
|
68
82
|
type DispatchLayer,
|
|
69
83
|
type FleetDaemonProbe,
|
|
@@ -311,19 +325,10 @@ export interface ArmDeps {
|
|
|
311
325
|
* The orchestrator session file the live omp-telegram claim names for this
|
|
312
326
|
* project, or undefined when there is no (readable) claim. The default
|
|
313
327
|
* resolves the claim the same way the send does; tests inject a fixture.
|
|
314
|
-
*
|
|
315
|
-
*
|
|
328
|
+
* Claim-only feeds it to the plumbing verdict's session-identity checks;
|
|
329
|
+
* the challenge proof never reads a transcript, so it ignores the claim.
|
|
316
330
|
*/
|
|
317
331
|
claimedSessionFile?: () => string | undefined;
|
|
318
|
-
/**
|
|
319
|
-
* Waits for the challenge to appear as a user turn somewhere under the scan
|
|
320
|
-
* directories. The waiter owns transcript discovery — not the caller —
|
|
321
|
-
* because the reply may land in a session that starts *after* the send, so a
|
|
322
|
-
* path resolved before the challenge went out can be the wrong file by the
|
|
323
|
-
* time the operator answers (#142), and because the claimed session file can
|
|
324
|
-
* sit in a different directory than the tick cwd implies (#600).
|
|
325
|
-
*/
|
|
326
|
-
waitForUserTurn?: (dirs: readonly string[], code: string, sentAt: number, timeoutMs: number) => Promise<boolean>;
|
|
327
332
|
now?: () => number;
|
|
328
333
|
sleep?: (ms: number) => Promise<void>;
|
|
329
334
|
timeoutMs?: number;
|
|
@@ -377,6 +382,13 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
377
382
|
// one pane that can ask. The challenge names which one, or the operator is
|
|
378
383
|
// answering a question they cannot attribute.
|
|
379
384
|
const named = tick.config.project ?? projectName;
|
|
385
|
+
// The handshake state key must be exactly what the orchestrator's inbound
|
|
386
|
+
// adapter computes: TickConfig.project, undefined for a legacy unstamped
|
|
387
|
+
// config. `named` may fall back to the CLI argument for the challenge text
|
|
388
|
+
// and config lookups; the state key must not — the adapter has no CLI
|
|
389
|
+
// argument to fall back to, and a mismatched key would make arming wait on
|
|
390
|
+
// an acknowledgement that can never be written.
|
|
391
|
+
const stateKey = tick.config.project;
|
|
380
392
|
|
|
381
393
|
// The arming proof is a declared per-project policy (#613). A config that
|
|
382
394
|
// cannot name the project fails safe to `challenge` — today's authenticated
|
|
@@ -389,15 +401,11 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
389
401
|
/* no project config — keep today's challenge behaviour */
|
|
390
402
|
}
|
|
391
403
|
|
|
392
|
-
//
|
|
393
|
-
//
|
|
394
|
-
//
|
|
395
|
-
//
|
|
396
|
-
//
|
|
397
|
-
// be. A claim outside the session tree arm scans can never be answered, so
|
|
398
|
-
// that is a stop, not five minutes of polling. (Claim-only resolves the
|
|
399
|
-
// claim too — the verdict checks the same session identity — but names the
|
|
400
|
-
// refusal itself rather than throwing the transcript wording.)
|
|
404
|
+
// The orchestrator's live session file per omp-telegram's claim (#600) —
|
|
405
|
+
// input to the claim-only verdict's session-identity checks below. The
|
|
406
|
+
// challenge proof never reads it: its acknowledgement is conductor state,
|
|
407
|
+
// so where (or whether) a transcript lives is no longer part of arming
|
|
408
|
+
// (#614).
|
|
401
409
|
const claimed = deps.claimedSessionFile !== undefined ? deps.claimedSessionFile() : claimedOrchestratorSessionFile(named);
|
|
402
410
|
|
|
403
411
|
// Prefer the project's live forum topic so arm challenges land where
|
|
@@ -450,17 +458,6 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
450
458
|
return { path, alreadyArmed, owner: channel.owner, proof };
|
|
451
459
|
}
|
|
452
460
|
|
|
453
|
-
// The transcript proof needs a session tree to poll. The claim-only verdict
|
|
454
|
-
// needs no such thing — it reads session identity from omp-telegram's own
|
|
455
|
-
// state — so this stop stays on the challenge path only.
|
|
456
|
-
const dirs = armSessionScanDirs(tick.cwd, claimed);
|
|
457
|
-
if (dirs.every((d) => !existsSync(d))) {
|
|
458
|
-
throw new Error(
|
|
459
|
-
`no orchestrator session directory under ${dirs.join(" or ")} — ` +
|
|
460
|
-
`the inbound proof is read from a user turn in a transcript there. Start the pane orchestrator, let it settle, then arm again`,
|
|
461
|
-
);
|
|
462
|
-
}
|
|
463
|
-
|
|
464
461
|
const code = makeChallengeCode();
|
|
465
462
|
const text =
|
|
466
463
|
`Fleet arming check${named === undefined ? "" : ` — project ${named}`}. ` +
|
|
@@ -469,54 +466,38 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
469
466
|
|
|
470
467
|
const send = deps.sendChallenge ?? sendTelegramMessage;
|
|
471
468
|
const timeoutMs = deps.timeoutMs ?? ARM_CHALLENGE_TIMEOUT_MS;
|
|
472
|
-
// Read before the send, not after: a transcript untouched since this instant
|
|
473
|
-
// cannot contain the reply, and that is what the waiter filters on.
|
|
474
469
|
const sentAt = (deps.now ?? Date.now)();
|
|
475
|
-
// The orchestrator
|
|
476
|
-
//
|
|
477
|
-
//
|
|
478
|
-
//
|
|
479
|
-
|
|
470
|
+
// The orchestrator's inbound adapter can only acknowledge an *active*
|
|
471
|
+
// challenge, so the authenticated pending record (hash + expiry, never the
|
|
472
|
+
// code) is written before the challenge goes out and settled the moment
|
|
473
|
+
// this end finishes (#415). The returned id pins the wait below: an
|
|
474
|
+
// acknowledgement can only ever name the currently-pending id, so replacing
|
|
475
|
+
// a challenge makes every prior acknowledgement inert.
|
|
476
|
+
const challengeId = recordArmChallenge(stateKey, code, sentAt, sentAt + timeoutMs);
|
|
480
477
|
try {
|
|
481
478
|
await send(token, channel.owner, text, sendTopic);
|
|
482
479
|
} catch (err) {
|
|
483
|
-
// The challenge never went out, so
|
|
484
|
-
// proof either.
|
|
485
|
-
|
|
480
|
+
// The challenge never went out, so its transaction must not linger as a
|
|
481
|
+
// classifiable proof either.
|
|
482
|
+
clearArmTransaction(stateKey, challengeId);
|
|
486
483
|
throw new Error(
|
|
487
484
|
`arm: outbound sendMessage failed — NOT armed: ${err instanceof Error ? err.message : String(err)}`,
|
|
488
485
|
);
|
|
489
486
|
}
|
|
490
487
|
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
// The claimed session file is named before the bridge/token/chat block: a
|
|
500
|
-
// rotation under the window is visible from this error alone, and on the
|
|
501
|
-
// host this guard exists for the claimed file is the one that held the
|
|
502
|
-
// answer (#600).
|
|
503
|
-
const listing = [
|
|
504
|
-
...(claimed === undefined
|
|
505
|
-
? []
|
|
506
|
-
: [
|
|
507
|
-
`claimed orchestrator session file: ${claimed}` +
|
|
508
|
-
(dirs.includes(dirname(claimed))
|
|
509
|
-
? " (watched)"
|
|
510
|
-
: ` (NOT under any watched dir — a reply there can never be seen)`),
|
|
511
|
-
]),
|
|
512
|
-
`session dir: ${dirs.join(", ")}`,
|
|
513
|
-
...scan.scanned.map((f) => ` watched: ${f}`),
|
|
514
|
-
...scan.ignored.map((f) => ` ignored (stale, last written before the challenge): ${f}`),
|
|
515
|
-
].join("\n");
|
|
488
|
+
// Wait for the orchestrator's own acknowledgement — conductor state written
|
|
489
|
+
// by the inbound user-turn adapter when the real reply lands (#614). No
|
|
490
|
+
// transcript is read: the proof no longer depends on where (or whether) a
|
|
491
|
+
// session file lives, which is exactly the discovery that mis-fired on the
|
|
492
|
+
// host three times (#614). A wrong-project or lookalike reply writes no
|
|
493
|
+
// acknowledgement, so the window simply runs out fail-closed.
|
|
494
|
+
if (!(await waitForArmAcknowledgement(challengeId, timeoutMs, deps))) {
|
|
495
|
+
clearArmTransaction(stateKey, challengeId);
|
|
516
496
|
throw new Error(
|
|
517
|
-
`arm: the challenge never
|
|
518
|
-
|
|
519
|
-
|
|
497
|
+
`arm: the challenge was never acknowledged in time — NOT armed.\n` +
|
|
498
|
+
`The orchestrator's inbound adapter acknowledges the reply when it lands as a user turn; ` +
|
|
499
|
+
`no acknowledgement for challenge ${challengeId} arrived.\n` +
|
|
500
|
+
`Inbound Telegram is not reaching the omp session. Check, in order:\n` +
|
|
520
501
|
` * is the bridge polling? attach and run: /telegram status\n` +
|
|
521
502
|
` * is another process holding this bot token? Telegram allows exactly one\n` +
|
|
522
503
|
` getUpdates consumer and rejects the second with HTTP 409.\n` +
|
|
@@ -525,10 +506,11 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
525
506
|
}
|
|
526
507
|
|
|
527
508
|
writeArmedMarker(path, channel.owner, arm);
|
|
528
|
-
// The
|
|
529
|
-
//
|
|
530
|
-
//
|
|
531
|
-
|
|
509
|
+
// The acknowledgement landed and this project is armed: settling clears this
|
|
510
|
+
// transaction's pending record and acknowledgement — never a newer
|
|
511
|
+
// replacement's — so a later unsolicited lookalike stays inert past this
|
|
512
|
+
// handshake.
|
|
513
|
+
clearArmTransaction(stateKey, challengeId);
|
|
532
514
|
return { path, alreadyArmed, owner: channel.owner, challenge: code, proof };
|
|
533
515
|
}
|
|
534
516
|
|
|
@@ -1053,21 +1035,15 @@ export function parseHerdrAgentList(rawOutput: string): HerdrAgent[] {
|
|
|
1053
1035
|
return out;
|
|
1054
1036
|
}
|
|
1055
1037
|
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
});
|
|
1066
|
-
if (res.error) throw res.error;
|
|
1067
|
-
if (res.status !== 0) {
|
|
1068
|
-
throw new Error((res.stderr ?? res.stdout ?? `process-info exit ${String(res.status)}`).trim());
|
|
1069
|
-
}
|
|
1070
|
-
const raw = (res.stdout ?? "").trim();
|
|
1038
|
+
/**
|
|
1039
|
+
* The `process_info` document out of `herdr pane process-info` output
|
|
1040
|
+
* (#832): every caller that maps the fleet pane's live omp processes starts
|
|
1041
|
+
* from this schema, whether the output carries herdr's CLI `result` envelope
|
|
1042
|
+
* or the bare document. Throws when the output cannot be read; the caller
|
|
1043
|
+
* decides what an unreadable answer means.
|
|
1044
|
+
*/
|
|
1045
|
+
export function parseHerdrProcessInfo(stdout: string, paneId: string): ProcessInfo {
|
|
1046
|
+
const raw = stdout.trim();
|
|
1071
1047
|
if (raw.length === 0) {
|
|
1072
1048
|
throw new Error(`herdr pane process-info printed nothing for ${paneId}`);
|
|
1073
1049
|
}
|
|
@@ -1083,10 +1059,27 @@ async function herdrOmpForegroundPids(paneId: string, deps: PaneStopDeps): Promi
|
|
|
1083
1059
|
if (info === undefined) {
|
|
1084
1060
|
throw new Error(`herdr pane process-info has no process_info for ${paneId} — unrecognized schema`);
|
|
1085
1061
|
}
|
|
1086
|
-
return
|
|
1062
|
+
return info;
|
|
1087
1063
|
}
|
|
1088
1064
|
|
|
1089
|
-
|
|
1065
|
+
async function herdrOmpForegroundPids(paneId: string, deps: PaneStopDeps): Promise<number[]> {
|
|
1066
|
+
const bin = deps.herdrBin ?? "herdr";
|
|
1067
|
+
const session =
|
|
1068
|
+
deps.herdrSession ??
|
|
1069
|
+
resolveHerdrSessionWithBridge({ env: process.env, herdrBin: bin });
|
|
1070
|
+
const res = spawnSync(bin, ["--session", session, "pane", "process-info", "--pane", paneId], {
|
|
1071
|
+
encoding: "utf8",
|
|
1072
|
+
timeout: 8_000,
|
|
1073
|
+
env: process.env,
|
|
1074
|
+
});
|
|
1075
|
+
if (res.error) throw res.error;
|
|
1076
|
+
if (res.status !== 0) {
|
|
1077
|
+
throw new Error((res.stderr ?? res.stdout ?? `process-info exit ${String(res.status)}`).trim());
|
|
1078
|
+
}
|
|
1079
|
+
return ompPidsFromProcessInfo(parseHerdrProcessInfo(res.stdout ?? "", paneId));
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
export interface ProcessInfo {
|
|
1090
1083
|
shell_pid?: number;
|
|
1091
1084
|
foreground_processes?: ForegroundProc[];
|
|
1092
1085
|
}
|
|
@@ -1121,6 +1114,85 @@ function isOmpProcess(proc: ForegroundProc): boolean {
|
|
|
1121
1114
|
return false;
|
|
1122
1115
|
}
|
|
1123
1116
|
|
|
1117
|
+
/**
|
|
1118
|
+
* The omp processes the fleet pane claims, resolved to their start times
|
|
1119
|
+
* (#832), or why they could not be read. Structurally identical to
|
|
1120
|
+
* upgrade-verify's `PaneOmpProbe` — the upgrade engine and the post-restart
|
|
1121
|
+
* verifier pass this straight into the pure verdicts, and the type is
|
|
1122
|
+
* repeated here because the verifier leaf must not import this module (fleet
|
|
1123
|
+
* reaches the daemon, which reaches the verifier).
|
|
1124
|
+
*/
|
|
1125
|
+
export type PaneProbeResult = { starts: readonly number[] } | { problem: string };
|
|
1126
|
+
|
|
1127
|
+
/**
|
|
1128
|
+
* Every omp process the fleet pane currently claims by herdr, resolved to
|
|
1129
|
+
* its start time (#832): the pane is where an *external* orchestrator
|
|
1130
|
+
* session lives, and the only fact that proves it reloaded is the live
|
|
1131
|
+
* process's own start — a pane process still running from before the install
|
|
1132
|
+
* began loaded the pre-upgrade extension. Every unreadable answer is a
|
|
1133
|
+
* `problem`: absence of evidence is never a reload.
|
|
1134
|
+
*
|
|
1135
|
+
* `run` is the injected command runner (the upgrade's scripted seam, the
|
|
1136
|
+
* daemon's `runCommand`), and `startTime` resolves one pid to its start —
|
|
1137
|
+
* the /proc read lives with the verifier's other live-process facts.
|
|
1138
|
+
*/
|
|
1139
|
+
export async function herdrPaneOmpStarts(
|
|
1140
|
+
run: (
|
|
1141
|
+
command: string,
|
|
1142
|
+
args: readonly string[],
|
|
1143
|
+
) => Promise<{ code: number; stdout: string; stderr: string }>,
|
|
1144
|
+
session: string,
|
|
1145
|
+
startTime: (pid: number) => number | undefined,
|
|
1146
|
+
): Promise<PaneProbeResult> {
|
|
1147
|
+
const agents = await run("herdr", ["--session", session, "agent", "list"]);
|
|
1148
|
+
if (agents.code !== 0) {
|
|
1149
|
+
return {
|
|
1150
|
+
problem:
|
|
1151
|
+
`herdr agent list failed: ${agents.stderr.trim() || agents.stdout.trim() || `exit ${agents.code}`}`,
|
|
1152
|
+
};
|
|
1153
|
+
}
|
|
1154
|
+
let parsed: HerdrAgent[];
|
|
1155
|
+
try {
|
|
1156
|
+
parsed = parseHerdrAgentList(agents.stdout);
|
|
1157
|
+
} catch (err) {
|
|
1158
|
+
return { problem: err instanceof Error ? err.message : String(err) };
|
|
1159
|
+
}
|
|
1160
|
+
const starts: number[] = [];
|
|
1161
|
+
for (const agent of parsed) {
|
|
1162
|
+
// A name without a live omp claim is a leftover label, not a pane process.
|
|
1163
|
+
if (agent.agent === undefined) continue;
|
|
1164
|
+
const info = await run("herdr", [
|
|
1165
|
+
"--session",
|
|
1166
|
+
session,
|
|
1167
|
+
"pane",
|
|
1168
|
+
"process-info",
|
|
1169
|
+
"--pane",
|
|
1170
|
+
agent.paneId,
|
|
1171
|
+
]);
|
|
1172
|
+
if (info.code !== 0) {
|
|
1173
|
+
return {
|
|
1174
|
+
problem:
|
|
1175
|
+
`herdr pane process-info for ${agent.paneId} failed: ` +
|
|
1176
|
+
`${info.stderr.trim() || info.stdout.trim() || `exit ${info.code}`}`,
|
|
1177
|
+
};
|
|
1178
|
+
}
|
|
1179
|
+
let pids: number[];
|
|
1180
|
+
try {
|
|
1181
|
+
pids = ompPidsFromProcessInfo(parseHerdrProcessInfo(info.stdout, agent.paneId));
|
|
1182
|
+
} catch (err) {
|
|
1183
|
+
return { problem: err instanceof Error ? err.message : String(err) };
|
|
1184
|
+
}
|
|
1185
|
+
for (const pid of pids) {
|
|
1186
|
+
const startedAt = startTime(pid);
|
|
1187
|
+
if (startedAt === undefined) {
|
|
1188
|
+
return { problem: `cannot read the start time of pane process ${pid} — reload unproven` };
|
|
1189
|
+
}
|
|
1190
|
+
starts.push(startedAt);
|
|
1191
|
+
}
|
|
1192
|
+
}
|
|
1193
|
+
return { starts };
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1124
1196
|
/**
|
|
1125
1197
|
* The project a bare read means, when the config leaves no doubt.
|
|
1126
1198
|
*
|
|
@@ -1355,7 +1427,7 @@ export function workerPhasesFromHealthz(
|
|
|
1355
1427
|
*/
|
|
1356
1428
|
export function classifyDaemonProjectHealth(
|
|
1357
1429
|
record: { project?: string } | undefined,
|
|
1358
|
-
health:
|
|
1430
|
+
health: HealthCheckResult | undefined,
|
|
1359
1431
|
project: string,
|
|
1360
1432
|
): DaemonProjectHealth {
|
|
1361
1433
|
if (record === undefined) return { kind: "stopped" };
|
|
@@ -1365,7 +1437,13 @@ export function classifyDaemonProjectHealth(
|
|
|
1365
1437
|
if (record.project !== undefined && record.project !== project) {
|
|
1366
1438
|
return { kind: "other-project", serves: record.project };
|
|
1367
1439
|
}
|
|
1368
|
-
|
|
1440
|
+
// #685: a timed-out probe is a probe outcome, never a death verdict — the
|
|
1441
|
+
// pid is up but wedged, so it must not read as `not running`. Only a
|
|
1442
|
+
// refusal or other failure (nothing listening, a torn answer) stays
|
|
1443
|
+
// `unreachable`.
|
|
1444
|
+
if (health?.ok !== true) {
|
|
1445
|
+
return health?.failure === "timeout" ? { kind: "unresponsive" } : { kind: "unreachable" };
|
|
1446
|
+
}
|
|
1369
1447
|
try {
|
|
1370
1448
|
const payload = JSON.parse(health.body ?? "null") as unknown;
|
|
1371
1449
|
if (payload === null || typeof payload !== "object") {
|
|
@@ -1415,6 +1493,9 @@ export type FleetStatusReport = StatusSnapshot & {
|
|
|
1415
1493
|
failureClasses: string | undefined;
|
|
1416
1494
|
workerPhases: { issue: number; phase: WorkerPausePhase }[];
|
|
1417
1495
|
intake: string | undefined;
|
|
1496
|
+
/** The project-scoped durable to-spec grooming lifecycle as status lines
|
|
1497
|
+
* (#809), or nothing when there is nothing to report. */
|
|
1498
|
+
grooming: string | undefined;
|
|
1418
1499
|
lastStop: DaemonStop | undefined;
|
|
1419
1500
|
siblings: { project: string; live: number }[];
|
|
1420
1501
|
};
|
|
@@ -1455,7 +1536,12 @@ export async function collectFleetStatus(projectName?: string): Promise<FleetSta
|
|
|
1455
1536
|
};
|
|
1456
1537
|
const healthBody = projectHealth.kind === "ok" ? rawHealth?.body : undefined;
|
|
1457
1538
|
const cached = codeGraphFromHealthz(healthBody, project.name);
|
|
1458
|
-
|
|
1539
|
+
// The runtime half of the code-graph finding (#726): the probe reads the
|
|
1540
|
+
// store-backed per-run observations, so "observed" is always grounded in
|
|
1541
|
+
// dispatched runs, never in the daemon's own process or in mcp.json. The
|
|
1542
|
+
// store stays scoped to the same try/finally as the other reads so a
|
|
1543
|
+
// throwing probe cannot leak its handle.
|
|
1544
|
+
const store = openStore(dbPath());
|
|
1459
1545
|
const workerPhases = [...workerPhasesFromHealthz(healthBody, project.name)].map(
|
|
1460
1546
|
([issue, phase]) => ({ issue, phase }),
|
|
1461
1547
|
);
|
|
@@ -1464,16 +1550,32 @@ export async function collectFleetStatus(projectName?: string): Promise<FleetSta
|
|
|
1464
1550
|
// and rendered identically from either project: the daemon_stops table is
|
|
1465
1551
|
// deliberately not partitioned by project, because the daemon serves every
|
|
1466
1552
|
// project and the uninvolved one must see who stopped it too.
|
|
1467
|
-
|
|
1553
|
+
let codeGraph: CodeGraphHealth;
|
|
1468
1554
|
let lastStop: DaemonStop | undefined;
|
|
1469
1555
|
let siblings: { project: string; live: number }[] = [];
|
|
1556
|
+
let grooming: string | undefined;
|
|
1470
1557
|
try {
|
|
1558
|
+
codeGraph =
|
|
1559
|
+
cached ??
|
|
1560
|
+
(await probeCodeGraph(project, {
|
|
1561
|
+
...DEFAULT_DEPS,
|
|
1562
|
+
graphToolsObservations: () => store.graphToolsObservationCounts(project.name),
|
|
1563
|
+
}));
|
|
1471
1564
|
lastStop = store.latestDaemonStop();
|
|
1472
1565
|
// Shared-daemon visibility (#545): every configured project other than the
|
|
1473
1566
|
// one being viewed, with its live-run count.
|
|
1474
1567
|
siblings = loadConfig()
|
|
1475
1568
|
.projects.filter((p) => p.name !== project.name)
|
|
1476
1569
|
.map((p) => ({ project: p.name, live: store.liveRuns(p.name).length }));
|
|
1570
|
+
// The durable to-spec grooming lifecycle (#809): project-scoped verdict
|
|
1571
|
+
// rows from the store, plus the dispatch snapshot's routed/parked counts —
|
|
1572
|
+
// no tracker read, no second cache, no evidence parsing. Rendered only
|
|
1573
|
+
// when it has something to say.
|
|
1574
|
+
grooming = formatGroomingStatus({
|
|
1575
|
+
records: store.groomingVerdicts(project.name),
|
|
1576
|
+
routed: snapshot.dispatch?.routed ?? 0,
|
|
1577
|
+
parked: snapshot.dispatch?.parked ?? 0,
|
|
1578
|
+
});
|
|
1477
1579
|
} finally {
|
|
1478
1580
|
store.close();
|
|
1479
1581
|
}
|
|
@@ -1491,6 +1593,7 @@ export async function collectFleetStatus(projectName?: string): Promise<FleetSta
|
|
|
1491
1593
|
failureClasses: failureClassBlock(project.name),
|
|
1492
1594
|
workerPhases,
|
|
1493
1595
|
intake: intakeStatusLine(project.name),
|
|
1596
|
+
grooming,
|
|
1494
1597
|
lastStop,
|
|
1495
1598
|
siblings,
|
|
1496
1599
|
};
|
|
@@ -1511,6 +1614,7 @@ export function renderFleetStatusReport(report: FleetStatusReport): string {
|
|
|
1511
1614
|
report.intake,
|
|
1512
1615
|
report.lastStop,
|
|
1513
1616
|
report.siblings,
|
|
1617
|
+
report.grooming,
|
|
1514
1618
|
);
|
|
1515
1619
|
}
|
|
1516
1620
|
|
|
@@ -1768,7 +1872,7 @@ export function sessionDirForCwd(cwd: string): string {
|
|
|
1768
1872
|
return join(home, ".omp", "agent", "sessions", slug.replaceAll("/", "-"));
|
|
1769
1873
|
}
|
|
1770
1874
|
|
|
1771
|
-
/** The session tree
|
|
1875
|
+
/** The omp session tree the claim-only verdict judges session identity against. */
|
|
1772
1876
|
export function sessionsRoot(): string {
|
|
1773
1877
|
return join(homedir(), ".omp", "agent", "sessions");
|
|
1774
1878
|
}
|
|
@@ -1777,8 +1881,9 @@ export function sessionsRoot(): string {
|
|
|
1777
1881
|
* The orchestrator session file omp-telegram's live claim names for this
|
|
1778
1882
|
* project — the transcript a resumed/restored pane actually writes, which can
|
|
1779
1883
|
* live in a different session directory than the tick cwd implies (#600).
|
|
1780
|
-
* Never throws: no claim, or an unreadable config, means undefined
|
|
1781
|
-
*
|
|
1884
|
+
* Never throws: no claim, or an unreadable config, means undefined. Only the
|
|
1885
|
+
* claim-only verdict consumes this — the challenge proof reads conductor
|
|
1886
|
+
* state, not transcripts (#614).
|
|
1782
1887
|
*/
|
|
1783
1888
|
function claimedOrchestratorSessionFile(named: string | undefined): string | undefined {
|
|
1784
1889
|
if (named === undefined) return undefined;
|
|
@@ -1789,35 +1894,6 @@ function claimedOrchestratorSessionFile(named: string | undefined): string | und
|
|
|
1789
1894
|
}
|
|
1790
1895
|
}
|
|
1791
1896
|
|
|
1792
|
-
/**
|
|
1793
|
-
* The session directories `arm` watches for the reply — the cwd-derived one
|
|
1794
|
-
* plus, when the live claim's file lives elsewhere, that file's directory.
|
|
1795
|
-
*
|
|
1796
|
-
* The cwd-derived directory is where a *fresh* session started from the tick
|
|
1797
|
-
* cwd writes; the claimed file's directory is where the *restored* pane keeps
|
|
1798
|
-
* writing (herdr pins it to the original transcript). Watching both covers a
|
|
1799
|
-
* resumed session, a rotated one, and a session that starts after the send.
|
|
1800
|
-
*
|
|
1801
|
-
* A claim outside the session tree is a stop, not a slower failure: arm polls
|
|
1802
|
-
* only transcripts under {@link sessionsRoot}, so a claimed file elsewhere can
|
|
1803
|
-
* never satisfy the challenge, and five minutes of polling cannot discover a
|
|
1804
|
-
* file that is not in the search set. The throw names both paths.
|
|
1805
|
-
*/
|
|
1806
|
-
function armSessionScanDirs(cwd: string, claimed: string | undefined): string[] {
|
|
1807
|
-
const cwdDir = sessionDirForCwd(cwd);
|
|
1808
|
-
if (claimed === undefined) return [cwdDir];
|
|
1809
|
-
const root = sessionsRoot();
|
|
1810
|
-
const claimDir = dirname(claimed);
|
|
1811
|
-
if (claimDir !== root && !claimDir.startsWith(join(root, sep))) {
|
|
1812
|
-
throw new Error(
|
|
1813
|
-
`arm: the orchestrator's claimed session file ${claimed} is outside the session tree arm scans (${root}) — ` +
|
|
1814
|
-
`no transcript there can satisfy the challenge. Resume the pane under ${root}, or start it from ${cwd}; ` +
|
|
1815
|
-
`arm would otherwise poll ${cwdDir} until the timer runs out`,
|
|
1816
|
-
);
|
|
1817
|
-
}
|
|
1818
|
-
return claimDir === cwdDir ? [cwdDir] : [cwdDir, claimDir];
|
|
1819
|
-
}
|
|
1820
|
-
|
|
1821
1897
|
function makeChallengeCode(): string {
|
|
1822
1898
|
const bytes = new Uint8Array(4);
|
|
1823
1899
|
crypto.getRandomValues(bytes);
|
|
@@ -1825,6 +1901,31 @@ function makeChallengeCode(): string {
|
|
|
1825
1901
|
return `FLEET-${hex}`;
|
|
1826
1902
|
}
|
|
1827
1903
|
|
|
1904
|
+
/**
|
|
1905
|
+
* Polls the acknowledgement record for one exact challenge id until the
|
|
1906
|
+
* orchestrator's inbound adapter writes it or the deadline passes (#614).
|
|
1907
|
+
*
|
|
1908
|
+
* The state is a small JSON file re-read every pass, never snapshotted: the
|
|
1909
|
+
* acknowledgement may land at any point in the window, written by the live
|
|
1910
|
+
* orchestrator process. Only a record naming this exact id satisfies the
|
|
1911
|
+
* wait — an acknowledgement cut for a replaced challenge is inert here by
|
|
1912
|
+
* construction, and no transcript anywhere is opened.
|
|
1913
|
+
*/
|
|
1914
|
+
async function waitForArmAcknowledgement(
|
|
1915
|
+
challengeId: string,
|
|
1916
|
+
timeoutMs: number,
|
|
1917
|
+
deps: ArmDeps,
|
|
1918
|
+
): Promise<boolean> {
|
|
1919
|
+
const now = deps.now ?? Date.now;
|
|
1920
|
+
const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
|
|
1921
|
+
const deadline = now() + timeoutMs;
|
|
1922
|
+
for (;;) {
|
|
1923
|
+
if (readArmAcknowledgement(challengeId) !== undefined) return true;
|
|
1924
|
+
if (now() >= deadline) return false;
|
|
1925
|
+
await sleep(5_000);
|
|
1926
|
+
}
|
|
1927
|
+
}
|
|
1928
|
+
|
|
1828
1929
|
/**
|
|
1829
1930
|
* The one armed-marker write both proofs share: same content, same mode, and
|
|
1830
1931
|
* the same restamp of the pre-per-project shared marker the heartbeat still
|
|
@@ -1838,12 +1939,12 @@ function writeArmedMarker(path: string, owner: string, arm: ArmState): void {
|
|
|
1838
1939
|
}
|
|
1839
1940
|
|
|
1840
1941
|
/**
|
|
1841
|
-
* The
|
|
1842
|
-
*
|
|
1843
|
-
*
|
|
1844
|
-
*
|
|
1845
|
-
*
|
|
1846
|
-
*
|
|
1942
|
+
* The session surface the claim-only verdict judges: the tick-cwd-derived
|
|
1943
|
+
* session directory plus, when the live claim's file lives elsewhere in the
|
|
1944
|
+
* tree, that file's directory (#600). An outside-tree claim is not a reason
|
|
1945
|
+
* to stop here — the verdict names that failure itself
|
|
1946
|
+
* (`claim-session-outside` / `dm-owner-unrelated`), so the refusal carries
|
|
1947
|
+
* the fact.
|
|
1847
1948
|
*/
|
|
1848
1949
|
function armVerdictScanDirs(cwd: string, claimed: string | undefined): string[] {
|
|
1849
1950
|
const cwdDir = sessionDirForCwd(cwd);
|
|
@@ -1897,108 +1998,6 @@ async function sendTelegramMessage(
|
|
|
1897
1998
|
await sendTelegram(token, owner, text, { topicId });
|
|
1898
1999
|
}
|
|
1899
2000
|
|
|
1900
|
-
interface SessionScan {
|
|
1901
|
-
seen: boolean;
|
|
1902
|
-
/** `name (mtime <iso>)` for every transcript the last pass actually parsed. */
|
|
1903
|
-
scanned: string[];
|
|
1904
|
-
/** Same shape, for the ones skipped as written before the challenge. */
|
|
1905
|
-
ignored: string[];
|
|
1906
|
-
}
|
|
1907
|
-
|
|
1908
|
-
/**
|
|
1909
|
-
* Polls the session directories — re-read on every pass, never snapshotted —
|
|
1910
|
-
* until the challenge shows up as a user turn or the deadline passes.
|
|
1911
|
-
*
|
|
1912
|
-
* Discovery lives here because the reply can land in a transcript that does not
|
|
1913
|
-
* exist yet when the challenge is sent: a rotated session, or the first one of
|
|
1914
|
-
* a pane started right after arming (#142). And the scan set can be two
|
|
1915
|
-
* directories when the live claim's session file lives outside the tick-cwd
|
|
1916
|
-
* directory: the restored pane keeps writing the claimed file, so its
|
|
1917
|
-
* directory is watched alongside the cwd-derived one (#600).
|
|
1918
|
-
*
|
|
1919
|
-
* Files untouched since just before the send are named, not parsed: an append
|
|
1920
|
-
* bumps mtime, so a transcript older than the challenge cannot hold the reply,
|
|
1921
|
-
* and skipping it keeps a large stale session out of every 5 s pass. A missing
|
|
1922
|
-
* directory is not an error — it can vanish under a rotation and reappear.
|
|
1923
|
-
*/
|
|
1924
|
-
async function waitForChallengeInSessions(
|
|
1925
|
-
dirs: readonly string[],
|
|
1926
|
-
code: string,
|
|
1927
|
-
sentAt: number,
|
|
1928
|
-
timeoutMs: number,
|
|
1929
|
-
deps: ArmDeps,
|
|
1930
|
-
): Promise<SessionScan> {
|
|
1931
|
-
const now = deps.now ?? Date.now;
|
|
1932
|
-
const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
|
|
1933
|
-
const deadline = now() + timeoutMs;
|
|
1934
|
-
for (;;) {
|
|
1935
|
-
const scanned: string[] = [];
|
|
1936
|
-
const ignored: string[] = [];
|
|
1937
|
-
for (const dir of dirs) {
|
|
1938
|
-
let names: string[] = [];
|
|
1939
|
-
try {
|
|
1940
|
-
names = readdirSync(dir);
|
|
1941
|
-
} catch {
|
|
1942
|
-
/* the directory can go away under a rotation; the next pass re-reads it */
|
|
1943
|
-
}
|
|
1944
|
-
for (const name of names.sort()) {
|
|
1945
|
-
if (!name.endsWith(".jsonl")) continue;
|
|
1946
|
-
const path = join(dir, name);
|
|
1947
|
-
let mtimeMs: number;
|
|
1948
|
-
try {
|
|
1949
|
-
mtimeMs = statSync(path).mtimeMs;
|
|
1950
|
-
} catch {
|
|
1951
|
-
continue; /* race */
|
|
1952
|
-
}
|
|
1953
|
-
const label = `${name} (mtime ${new Date(mtimeMs).toISOString()})`;
|
|
1954
|
-
if (mtimeMs < sentAt - 1_000) {
|
|
1955
|
-
ignored.push(label);
|
|
1956
|
-
continue;
|
|
1957
|
-
}
|
|
1958
|
-
scanned.push(label);
|
|
1959
|
-
if (await transcriptHasUserCode(path, code)) return { seen: true, scanned, ignored };
|
|
1960
|
-
}
|
|
1961
|
-
}
|
|
1962
|
-
if (now() >= deadline) return { seen: false, scanned, ignored };
|
|
1963
|
-
await sleep(5_000);
|
|
1964
|
-
}
|
|
1965
|
-
}
|
|
1966
|
-
|
|
1967
|
-
export async function transcriptHasUserCode(path: string, code: string): Promise<boolean> {
|
|
1968
|
-
if (!existsSync(path)) return false;
|
|
1969
|
-
const rl = createInterface({ input: createReadStream(path, { encoding: "utf8" }), crlfDelay: Infinity });
|
|
1970
|
-
try {
|
|
1971
|
-
for await (const line of rl) {
|
|
1972
|
-
if (line.length === 0) continue;
|
|
1973
|
-
let row: unknown;
|
|
1974
|
-
try {
|
|
1975
|
-
row = JSON.parse(line);
|
|
1976
|
-
} catch {
|
|
1977
|
-
continue;
|
|
1978
|
-
}
|
|
1979
|
-
if (row === null || typeof row !== "object") continue;
|
|
1980
|
-
const rec = row as { readonly [key: string]: unknown };
|
|
1981
|
-
if (rec["type"] !== "message") continue;
|
|
1982
|
-
const message = rec["message"];
|
|
1983
|
-
if (message === null || typeof message !== "object") continue;
|
|
1984
|
-
const msg = message as { readonly [key: string]: unknown };
|
|
1985
|
-
if (msg["role"] !== "user") continue;
|
|
1986
|
-
const content = msg["content"];
|
|
1987
|
-
if (!Array.isArray(content)) continue;
|
|
1988
|
-
for (const part of content) {
|
|
1989
|
-
if (part === null || typeof part !== "object") continue;
|
|
1990
|
-
const p = part as { readonly [key: string]: unknown };
|
|
1991
|
-
if (p["type"] === "text" && typeof p["text"] === "string" && p["text"].includes(code)) {
|
|
1992
|
-
return true;
|
|
1993
|
-
}
|
|
1994
|
-
}
|
|
1995
|
-
}
|
|
1996
|
-
} finally {
|
|
1997
|
-
rl.close();
|
|
1998
|
-
}
|
|
1999
|
-
return false;
|
|
2000
|
-
}
|
|
2001
|
-
|
|
2002
2001
|
// ---------------------------------------------------------------------------
|
|
2003
2002
|
// probes
|
|
2004
2003
|
// ---------------------------------------------------------------------------
|