omp-conductor 0.15.13 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +72 -2
- package/package.json +2 -1
- package/schema/config.schema.json +7 -0
- package/src/admission.ts +849 -0
- package/src/ask.ts +47 -0
- package/src/backups.ts +19 -7
- package/src/board.ts +1 -2
- package/src/briefs/orchestrator.md +62 -4
- package/src/cli.ts +26 -0
- package/src/commands/context.ts +3 -0
- package/src/commands/decision.ts +10 -1
- package/src/commands/doctor.ts +2 -0
- package/src/commands/message.ts +8 -1
- package/src/commands/restart.ts +15 -3
- package/src/commands/restore-db.ts +146 -0
- package/src/commands/stop.ts +24 -15
- package/src/commands/tail.ts +204 -44
- package/src/commands/unfreeze.ts +56 -0
- package/src/commands/watch.ts +77 -0
- package/src/config-schema.ts +13 -0
- package/src/config.ts +54 -0
- package/src/daemon.ts +255 -530
- package/src/dashboard/server.ts +2 -1
- package/src/decisions.ts +32 -7
- package/src/depends-on.ts +122 -0
- package/src/doctor.ts +297 -5
- package/src/escalate.ts +191 -19
- package/src/failure-class.ts +47 -0
- package/src/fleet.ts +168 -452
- package/src/gitops.ts +86 -1
- package/src/log.ts +40 -0
- package/src/model-fallback.ts +3 -2
- package/src/omp-settings.ts +114 -0
- package/src/omp.ts +39 -0
- package/src/orchestrator-tick.ts +7 -1
- package/src/reports.ts +124 -12
- package/src/session-host.ts +6 -0
- package/src/setup-wizard.ts +36 -0
- package/src/setup.ts +58 -1
- package/src/status-render.ts +445 -0
- package/src/stop-provenance.ts +53 -0
- package/src/store.ts +352 -11
- package/src/transcript.ts +1 -1
- package/src/types.ts +187 -4
- package/src/unblock.ts +1 -1
- package/src/upgrade-verify.ts +1 -1
- package/src/upgrade.ts +1 -2
- package/src/verbs/server.ts +25 -0
- package/src/worker.ts +358 -10
- package/src/worktree.ts +13 -1
package/src/fleet.ts
CHANGED
|
@@ -26,33 +26,18 @@ import {
|
|
|
26
26
|
} from "node:fs";
|
|
27
27
|
import { createInterface } from "node:readline";
|
|
28
28
|
import { homedir } from "node:os";
|
|
29
|
-
import { dirname, join } from "node:path";
|
|
30
|
-
import { formatZonedMinute } from "./availability.ts";
|
|
29
|
+
import { dirname, join, sep } from "node:path";
|
|
31
30
|
import { findProject, loadConfig, stateDir } from "./config.ts";
|
|
32
31
|
import { clearArmChallenge, recordArmChallenge } from "./arm-challenge.ts";
|
|
33
|
-
import { resolveProjectTopicId, sendTelegram } from "./escalate.ts";
|
|
34
|
-
import {
|
|
32
|
+
import { resolveClaimedSessionFile, resolveProjectTopicId, sendTelegram } from "./escalate.ts";
|
|
33
|
+
import { readPlanUsage, sharedUsageSource } from "./usage.ts";
|
|
35
34
|
import { readApprovalSurface, readDaemonProfile } from "./approval-surface.ts";
|
|
36
35
|
import { inspectBriefLayout } from "./brief-upgrade.ts";
|
|
37
36
|
import { dbPath, openStore } from "./store.ts";
|
|
38
37
|
import { renderBriefForProject } from "./setup.ts";
|
|
39
38
|
import type { DaemonStop, ProjectConfig, Store } from "./types.ts";
|
|
40
|
-
import { settlementFlagSummary } from "./diff-flags.ts";
|
|
41
39
|
import { probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
42
|
-
import {
|
|
43
|
-
import {
|
|
44
|
-
formatBaseHealth,
|
|
45
|
-
formatDispatchSummary,
|
|
46
|
-
formatReleaseGrants,
|
|
47
|
-
formatSalvagedRuns,
|
|
48
|
-
isPaused,
|
|
49
|
-
pauseProvenance,
|
|
50
|
-
pausedAt,
|
|
51
|
-
setPaused,
|
|
52
|
-
statusSnapshot,
|
|
53
|
-
type StatusSnapshot,
|
|
54
|
-
} from "./daemon.ts";
|
|
55
|
-
import { formatOrchestratorDown } from "./orchestrator-down.ts";
|
|
40
|
+
import { isPaused, setPaused, statusSnapshot } from "./daemon.ts";
|
|
56
41
|
import {
|
|
57
42
|
healthCheck,
|
|
58
43
|
isAlive,
|
|
@@ -61,11 +46,19 @@ import {
|
|
|
61
46
|
runSystemctl,
|
|
62
47
|
stopDaemon,
|
|
63
48
|
type StopResult,
|
|
64
|
-
type UnitOwnership,
|
|
65
|
-
SYSTEMD_UNIT,
|
|
66
49
|
} from "./lifecycle.ts";
|
|
67
|
-
import { formatRss, rssBytesFromHealthz } from "./host.ts";
|
|
68
50
|
import type { WorkerPausePhase } from "./worker.ts";
|
|
51
|
+
import {
|
|
52
|
+
formatFleetStatus,
|
|
53
|
+
type DaemonProjectHealth,
|
|
54
|
+
type DispatchLayer,
|
|
55
|
+
type FleetDaemonProbe,
|
|
56
|
+
type FleetLayers,
|
|
57
|
+
type HerdrLayer,
|
|
58
|
+
type PaneLayer,
|
|
59
|
+
type TelegramHealth,
|
|
60
|
+
type TicksLayer,
|
|
61
|
+
} from "./status-render.ts";
|
|
69
62
|
import { fetchRateLimit } from "./tracker/github.ts";
|
|
70
63
|
import {
|
|
71
64
|
LEGACY_ARM_MARKER_DETAIL,
|
|
@@ -294,13 +287,22 @@ export interface ArmResult {
|
|
|
294
287
|
export interface ArmDeps {
|
|
295
288
|
sendChallenge?: (token: string, owner: string, text: string, topicId?: number) => Promise<void>;
|
|
296
289
|
/**
|
|
297
|
-
*
|
|
298
|
-
*
|
|
290
|
+
* The orchestrator session file the live omp-telegram claim names for this
|
|
291
|
+
* project, or undefined when there is no (readable) claim. The default
|
|
292
|
+
* resolves the claim the same way the send does; tests inject a fixture.
|
|
293
|
+
* Absent → arm watches the cwd-derived session directory, the pre-claim
|
|
294
|
+
* behaviour.
|
|
295
|
+
*/
|
|
296
|
+
claimedSessionFile?: () => string | undefined;
|
|
297
|
+
/**
|
|
298
|
+
* Waits for the challenge to appear as a user turn somewhere under the scan
|
|
299
|
+
* directories. The waiter owns transcript discovery — not the caller —
|
|
299
300
|
* because the reply may land in a session that starts *after* the send, so a
|
|
300
301
|
* path resolved before the challenge went out can be the wrong file by the
|
|
301
|
-
* time the operator answers (#142)
|
|
302
|
+
* time the operator answers (#142), and because the claimed session file can
|
|
303
|
+
* sit in a different directory than the tick cwd implies (#600).
|
|
302
304
|
*/
|
|
303
|
-
waitForUserTurn?: (
|
|
305
|
+
waitForUserTurn?: (dirs: readonly string[], code: string, sentAt: number, timeoutMs: number) => Promise<boolean>;
|
|
304
306
|
now?: () => number;
|
|
305
307
|
sleep?: (ms: number) => Promise<void>;
|
|
306
308
|
timeoutMs?: number;
|
|
@@ -339,10 +341,23 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
339
341
|
);
|
|
340
342
|
}
|
|
341
343
|
|
|
342
|
-
|
|
343
|
-
|
|
344
|
+
// One bot, one chat, and — once a host runs more than one fleet — more than
|
|
345
|
+
// one pane that can ask. The challenge names which one, or the operator is
|
|
346
|
+
// answering a question they cannot attribute.
|
|
347
|
+
const named = tick.config.project ?? projectName;
|
|
348
|
+
|
|
349
|
+
// Resolve the orchestrator's live session file *before* the challenge goes
|
|
350
|
+
// out — the same claim the send follows (#600). A pane resumed from a
|
|
351
|
+
// session created elsewhere (herdr pins it to the original transcript)
|
|
352
|
+
// writes a session file outside the directory the tick cwd implies, and a
|
|
353
|
+
// cwd-derived scan would poll the one place the reply is guaranteed not to
|
|
354
|
+
// be. A claim outside the session tree arm scans can never be answered, so
|
|
355
|
+
// that is a stop, not five minutes of polling.
|
|
356
|
+
const claimed = deps.claimedSessionFile !== undefined ? deps.claimedSessionFile() : claimedOrchestratorSessionFile(named);
|
|
357
|
+
const dirs = armSessionScanDirs(tick.cwd, claimed);
|
|
358
|
+
if (dirs.every((d) => !existsSync(d))) {
|
|
344
359
|
throw new Error(
|
|
345
|
-
`no orchestrator session directory
|
|
360
|
+
`no orchestrator session directory under ${dirs.join(" or ")} — ` +
|
|
346
361
|
`the inbound proof is read from a user turn in a transcript there. Start the pane orchestrator, let it settle, then arm again`,
|
|
347
362
|
);
|
|
348
363
|
}
|
|
@@ -354,10 +369,6 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
354
369
|
const arm = resolveArmState(path, tick.config.project ?? projectName);
|
|
355
370
|
const alreadyArmed = arm.armed;
|
|
356
371
|
const code = makeChallengeCode();
|
|
357
|
-
// One bot, one chat, and — once a host runs more than one fleet — more than
|
|
358
|
-
// one pane that can ask. The challenge names which one, or the operator is
|
|
359
|
-
// answering a question they cannot attribute.
|
|
360
|
-
const named = tick.config.project ?? projectName;
|
|
361
372
|
const text =
|
|
362
373
|
`Fleet arming check${named === undefined ? "" : ` — project ${named}`}. ` +
|
|
363
374
|
`Reply to this chat with exactly:\n${code}\n` +
|
|
@@ -398,25 +409,37 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
398
409
|
|
|
399
410
|
const injected = deps.waitForUserTurn;
|
|
400
411
|
const scan: SessionScan = injected
|
|
401
|
-
? { seen: await injected(
|
|
402
|
-
: await waitForChallengeInSessions(
|
|
412
|
+
? { seen: await injected(dirs, code, sentAt, timeoutMs), scanned: [], ignored: [] }
|
|
413
|
+
: await waitForChallengeInSessions(dirs, code, sentAt, timeoutMs, deps);
|
|
403
414
|
if (!scan.seen) {
|
|
404
415
|
// The proof missed its window: a lookalike reply must not stay classifiable
|
|
405
416
|
// after arming gave up, so the record is cleared before the failure lands.
|
|
406
417
|
clearArmChallenge(named);
|
|
418
|
+
// The claimed session file is named before the bridge/token/chat block: a
|
|
419
|
+
// rotation under the window is visible from this error alone, and on the
|
|
420
|
+
// host this guard exists for the claimed file is the one that held the
|
|
421
|
+
// answer (#600).
|
|
407
422
|
const listing = [
|
|
408
|
-
|
|
423
|
+
...(claimed === undefined
|
|
424
|
+
? []
|
|
425
|
+
: [
|
|
426
|
+
`claimed orchestrator session file: ${claimed}` +
|
|
427
|
+
(dirs.includes(dirname(claimed))
|
|
428
|
+
? " (watched)"
|
|
429
|
+
: ` (NOT under any watched dir — a reply there can never be seen)`),
|
|
430
|
+
]),
|
|
431
|
+
`session dir: ${dirs.join(", ")}`,
|
|
409
432
|
...scan.scanned.map((f) => ` watched: ${f}`),
|
|
410
433
|
...scan.ignored.map((f) => ` ignored (stale, last written before the challenge): ${f}`),
|
|
411
434
|
].join("\n");
|
|
412
435
|
throw new Error(
|
|
413
436
|
`arm: the challenge never arrived as a user turn in time — NOT armed.\n` +
|
|
414
|
-
|
|
437
|
+
listing +
|
|
438
|
+
`\nInbound Telegram is not reaching the omp session. Check, in order:\n` +
|
|
415
439
|
` * is the bridge polling? attach and run: /telegram status\n` +
|
|
416
440
|
` * is another process holding this bot token? Telegram allows exactly one\n` +
|
|
417
441
|
` getUpdates consumer and rejects the second with HTTP 409.\n` +
|
|
418
|
-
` * did you reply in the DM with the bot, not another chat?\n
|
|
419
|
-
listing,
|
|
442
|
+
` * did you reply in the DM with the bot, not another chat?\n`,
|
|
420
443
|
);
|
|
421
444
|
}
|
|
422
445
|
|
|
@@ -1023,46 +1046,6 @@ function isOmpProcess(proc: ForegroundProc): boolean {
|
|
|
1023
1046
|
return false;
|
|
1024
1047
|
}
|
|
1025
1048
|
|
|
1026
|
-
// ---------------------------------------------------------------------------
|
|
1027
|
-
// layered status
|
|
1028
|
-
// ---------------------------------------------------------------------------
|
|
1029
|
-
|
|
1030
|
-
export type DispatchLayer = "running" | "paused" | "stopped";
|
|
1031
|
-
export type TicksLayer =
|
|
1032
|
-
| "armed"
|
|
1033
|
-
| "disarmed"
|
|
1034
|
-
| "no-heartbeat-config"
|
|
1035
|
-
| "invalid-heartbeat-config"
|
|
1036
|
-
| "ungated";
|
|
1037
|
-
export type PaneLayer = "live" | "missing" | "unknown";
|
|
1038
|
-
/** `unpinnable`: no tick config, so FLEET_CWD — the only path recovery reads — is unknown. */
|
|
1039
|
-
export type RecoveryLayer = "pinned" | "clear" | "unpinnable";
|
|
1040
|
-
export type HerdrLayer = "active" | "inactive" | "unknown";
|
|
1041
|
-
export type TelegramLayer = "ok" | "degraded" | "down" | "unconfigured" | "unprobed";
|
|
1042
|
-
|
|
1043
|
-
export interface TelegramHealth {
|
|
1044
|
-
kind: TelegramLayer;
|
|
1045
|
-
detail?: string;
|
|
1046
|
-
}
|
|
1047
|
-
|
|
1048
|
-
export interface FleetLayers {
|
|
1049
|
-
dispatch: DispatchLayer;
|
|
1050
|
-
ticks: TicksLayer;
|
|
1051
|
-
ticksDetail?: string;
|
|
1052
|
-
nextTickAt?: string;
|
|
1053
|
-
pane: PaneLayer;
|
|
1054
|
-
paneDetail?: string;
|
|
1055
|
-
recovery: RecoveryLayer;
|
|
1056
|
-
recoveryDetail?: string;
|
|
1057
|
-
herdr: HerdrLayer;
|
|
1058
|
-
herdrDetail?: string;
|
|
1059
|
-
armedPath?: string;
|
|
1060
|
-
tickConfigPath?: string;
|
|
1061
|
-
paneHaltPath?: string;
|
|
1062
|
-
paused: boolean;
|
|
1063
|
-
daemon: { running: boolean; pid?: number; port?: number };
|
|
1064
|
-
}
|
|
1065
|
-
|
|
1066
1049
|
/**
|
|
1067
1050
|
* The project a bare read means, when the config leaves no doubt.
|
|
1068
1051
|
*
|
|
@@ -1270,50 +1253,6 @@ export function workerPhasesFromHealthz(
|
|
|
1270
1253
|
}
|
|
1271
1254
|
|
|
1272
1255
|
|
|
1273
|
-
export function formatCodeGraphHealth(graph: CodeGraphHealth, now = Date.now()): string | undefined {
|
|
1274
|
-
if (!graph.configured) return undefined;
|
|
1275
|
-
const indexed = graph.repos.filter((repo) => repo.index === "present").length;
|
|
1276
|
-
let refresh: string = graph.refresh.result;
|
|
1277
|
-
if (graph.refresh.lastSuccessAt !== undefined) {
|
|
1278
|
-
const ageMs = Math.max(0, now - Date.parse(graph.refresh.lastSuccessAt));
|
|
1279
|
-
refresh = `${graph.refresh.lastSuccessAt} (${Math.max(1, Math.ceil(ageMs / 60_000))}m ago)`;
|
|
1280
|
-
}
|
|
1281
|
-
return [
|
|
1282
|
-
`code graph ${graph.status} ${indexed}/${graph.repos.length} repos indexed`,
|
|
1283
|
-
` indexer ${graph.prerequisites.indexer}`,
|
|
1284
|
-
` MCP mount ${graph.prerequisites.mcpMount}`,
|
|
1285
|
-
` timer ${graph.timer.enabled} / ${graph.timer.active}`,
|
|
1286
|
-
` refresh ${refresh}`,
|
|
1287
|
-
...graph.reasons.map((reason) => ` - ${reason}`),
|
|
1288
|
-
].join("\n");
|
|
1289
|
-
}
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
/**
|
|
1293
|
-
* Project-scoped reading of a living daemon record + `/healthz` body.
|
|
1294
|
-
*
|
|
1295
|
-
* Board and `status` share this so a host running one daemon for several
|
|
1296
|
-
* projects cannot be called healthy for a project it does not serve (#379).
|
|
1297
|
-
* A third interpretation convention is forbidden.
|
|
1298
|
-
*/
|
|
1299
|
-
export type DaemonProjectHealth =
|
|
1300
|
-
| { kind: "stopped" }
|
|
1301
|
-
| { kind: "ok" }
|
|
1302
|
-
| { kind: "unreachable" }
|
|
1303
|
-
| { kind: "other-project"; serves?: string };
|
|
1304
|
-
|
|
1305
|
-
/**
|
|
1306
|
-
* Facts `formatFleetStatus` needs about the living daemon. Pure input — the
|
|
1307
|
-
* caller probes; the formatter never shells out (#379).
|
|
1308
|
-
*/
|
|
1309
|
-
export type FleetDaemonProbe = {
|
|
1310
|
-
project: DaemonProjectHealth;
|
|
1311
|
-
/** `/healthz` body when the probe is project-ok (rss / overlays). */
|
|
1312
|
-
body?: string;
|
|
1313
|
-
/** systemd ownership of the living record pid; omit when unprobed. */
|
|
1314
|
-
unit?: UnitOwnership;
|
|
1315
|
-
};
|
|
1316
|
-
|
|
1317
1256
|
/**
|
|
1318
1257
|
* Decide whether a living record + optional `/healthz` answer serve `project`.
|
|
1319
1258
|
*
|
|
@@ -1372,306 +1311,6 @@ function servedProjectName(payload: object): string | undefined {
|
|
|
1372
1311
|
return undefined;
|
|
1373
1312
|
}
|
|
1374
1313
|
|
|
1375
|
-
function formatDaemonHealthz(probe: FleetDaemonProbe | undefined): string {
|
|
1376
|
-
if (probe === undefined) return "unprobed";
|
|
1377
|
-
switch (probe.project.kind) {
|
|
1378
|
-
case "stopped":
|
|
1379
|
-
return "stopped";
|
|
1380
|
-
case "ok":
|
|
1381
|
-
return "ok";
|
|
1382
|
-
case "unreachable":
|
|
1383
|
-
return "unreachable — the process is up but not serving";
|
|
1384
|
-
case "other-project":
|
|
1385
|
-
return probe.project.serves === undefined
|
|
1386
|
-
? "other-project"
|
|
1387
|
-
: `other-project — serves ${probe.project.serves}`;
|
|
1388
|
-
}
|
|
1389
|
-
}
|
|
1390
|
-
|
|
1391
|
-
/**
|
|
1392
|
-
* Unit line for a living record. The bare unit name alone used to imply
|
|
1393
|
-
* systemd ownership that was never checked (#379); every branch states the
|
|
1394
|
-
* probe result, and `unknown` is never rendered as owned.
|
|
1395
|
-
*/
|
|
1396
|
-
function formatDaemonUnit(unit: UnitOwnership | undefined, recordPid: number): string {
|
|
1397
|
-
if (unit === undefined) return ` unit ${SYSTEMD_UNIT} unprobed`;
|
|
1398
|
-
switch (unit.kind) {
|
|
1399
|
-
case "active":
|
|
1400
|
-
if (unit.pid === recordPid) {
|
|
1401
|
-
return ` unit ${SYSTEMD_UNIT} systemd-owned`;
|
|
1402
|
-
}
|
|
1403
|
-
// Live unit, different MainPID — the record pid is not systemd's.
|
|
1404
|
-
return ` unit ${SYSTEMD_UNIT} unmanaged — MainPID ${unit.pid}`;
|
|
1405
|
-
case "failed":
|
|
1406
|
-
return ` unit ${SYSTEMD_UNIT} unmanaged — unit failed`;
|
|
1407
|
-
case "inactive":
|
|
1408
|
-
return ` unit ${SYSTEMD_UNIT} inactive`;
|
|
1409
|
-
case "unknown":
|
|
1410
|
-
return ` unit ${SYSTEMD_UNIT} unknown — ${unit.reason}`;
|
|
1411
|
-
}
|
|
1412
|
-
}
|
|
1413
|
-
|
|
1414
|
-
/**
|
|
1415
|
-
* The newest daemon stop/restart provenance as status lines (#378).
|
|
1416
|
-
*
|
|
1417
|
-
* Rendered under the daemon block whether the daemon is down or has since
|
|
1418
|
-
* been restarted — the whole point is that the debrief survives the restart.
|
|
1419
|
-
* An unattributed record says so explicitly: the external-signal fallback
|
|
1420
|
-
* records what the receiving daemon knew (that it was unattributed, when,
|
|
1421
|
-
* which projects it served with live counts) rather than guessing a caller.
|
|
1422
|
-
*/
|
|
1423
|
-
function formatLastStop(lastStop: DaemonStop | undefined): string[] {
|
|
1424
|
-
if (lastStop === undefined) return [];
|
|
1425
|
-
const caller = lastStop.unattributed
|
|
1426
|
-
? "unattributed — no mediated request (external signal)"
|
|
1427
|
-
: `pid ${lastStop.callerPid ?? "?"}` +
|
|
1428
|
-
(lastStop.callerUid === undefined ? "" : ` uid ${lastStop.callerUid}`) +
|
|
1429
|
-
(lastStop.role === undefined ? "" : ` (${lastStop.role})`);
|
|
1430
|
-
return [
|
|
1431
|
-
` last stop ${new Date(lastStop.at).toISOString()} ${lastStop.controlPath}`,
|
|
1432
|
-
` caller ${caller}`,
|
|
1433
|
-
` scope ${lastStop.scope}${lastStop.project === undefined ? "" : `: ${lastStop.project}`}`,
|
|
1434
|
-
` affects ${lastStop.affected.map((a) => `${a.project} (${a.live} live)`).join(", ")}`,
|
|
1435
|
-
` reason ${lastStop.reason}`,
|
|
1436
|
-
];
|
|
1437
|
-
}
|
|
1438
|
-
|
|
1439
|
-
export function formatFleetStatus(
|
|
1440
|
-
s: StatusSnapshot,
|
|
1441
|
-
layers: FleetLayers,
|
|
1442
|
-
daemon: FleetDaemonProbe | undefined = undefined,
|
|
1443
|
-
telegram: TelegramHealth = { kind: "unprobed" },
|
|
1444
|
-
now = Date.now(),
|
|
1445
|
-
codeGraph: CodeGraphHealth = { configured: false },
|
|
1446
|
-
brief: string | undefined = undefined,
|
|
1447
|
-
decisions: string | undefined = undefined,
|
|
1448
|
-
failureClasses: string | undefined = undefined,
|
|
1449
|
-
workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
|
|
1450
|
-
intake: string | undefined = undefined,
|
|
1451
|
-
lastStop: DaemonStop | undefined = undefined,
|
|
1452
|
-
): string {
|
|
1453
|
-
const tickLine =
|
|
1454
|
-
layers.ticksDetail === undefined
|
|
1455
|
-
? `ticks ${layers.ticks}`
|
|
1456
|
-
: `ticks ${layers.ticks} (${layers.ticksDetail})`;
|
|
1457
|
-
let nextTickLine: string | undefined;
|
|
1458
|
-
if (layers.nextTickAt !== undefined) {
|
|
1459
|
-
const delta = Date.parse(layers.nextTickAt) - now;
|
|
1460
|
-
const minutes = Math.max(1, Math.ceil(Math.abs(delta) / 60_000));
|
|
1461
|
-
nextTickLine =
|
|
1462
|
-
`next tick ${layers.nextTickAt} ` +
|
|
1463
|
-
`(${delta >= 0 ? `in ${minutes}m` : `overdue by ${minutes}m`})`;
|
|
1464
|
-
}
|
|
1465
|
-
const paneLine =
|
|
1466
|
-
layers.paneDetail === undefined
|
|
1467
|
-
? `pane ${layers.pane}`
|
|
1468
|
-
: `pane ${layers.pane} (${layers.paneDetail})`;
|
|
1469
|
-
const recoveryLine =
|
|
1470
|
-
layers.recoveryDetail === undefined
|
|
1471
|
-
? `recovery ${layers.recovery}`
|
|
1472
|
-
: `recovery ${layers.recovery} (${layers.recoveryDetail})`;
|
|
1473
|
-
const herdrLine =
|
|
1474
|
-
layers.herdrDetail === undefined
|
|
1475
|
-
? `herdr ${layers.herdr}`
|
|
1476
|
-
: `herdr ${layers.herdr} (${layers.herdrDetail})`;
|
|
1477
|
-
const telegramLine =
|
|
1478
|
-
telegram.detail === undefined
|
|
1479
|
-
? `telegram ${telegram.kind}`
|
|
1480
|
-
: `telegram ${telegram.kind} (${telegram.detail})`;
|
|
1481
|
-
|
|
1482
|
-
let daemonBlock: string;
|
|
1483
|
-
if (!layers.daemon.running || layers.daemon.pid === undefined) {
|
|
1484
|
-
daemonBlock = "daemon not running";
|
|
1485
|
-
} else {
|
|
1486
|
-
// Only trust rss from a project-ok body — a foreign payload's bytes are
|
|
1487
|
-
// not this project's daemon facts (#379).
|
|
1488
|
-
const rss =
|
|
1489
|
-
daemon?.project.kind === "ok" ? rssBytesFromHealthz(daemon.body) : undefined;
|
|
1490
|
-
daemonBlock = [
|
|
1491
|
-
"daemon",
|
|
1492
|
-
` pid ${layers.daemon.pid}`,
|
|
1493
|
-
` port ${layers.daemon.port ?? "?"}`,
|
|
1494
|
-
...(rss === undefined ? [] : [` rss ${formatRss(rss)}`]),
|
|
1495
|
-
` healthz ${formatDaemonHealthz(daemon)}`,
|
|
1496
|
-
formatDaemonUnit(daemon?.unit, layers.daemon.pid),
|
|
1497
|
-
].join("\n");
|
|
1498
|
-
}
|
|
1499
|
-
|
|
1500
|
-
const graphBlock = formatCodeGraphHealth(codeGraph, now);
|
|
1501
|
-
|
|
1502
|
-
// Pause provenance (`pause --reason`, an integrity/spend-cap/upgrade pause,
|
|
1503
|
-
// a halt) answers "who stopped the fleet" without opening a file (#185). An
|
|
1504
|
-
// unparseable sentinel — paused but with no datable line 1 — is itself news:
|
|
1505
|
-
// it means a run admitted before an *unknown* pause cannot prove innocence
|
|
1506
|
-
// (#174), so completion mutations fail closed while release gates remain usable.
|
|
1507
|
-
const dispatchLine =
|
|
1508
|
-
layers.dispatch === "paused"
|
|
1509
|
-
? (() => {
|
|
1510
|
-
const prov = pauseProvenance(s.project);
|
|
1511
|
-
if (prov !== undefined) {
|
|
1512
|
-
const reason = prov.reason === undefined ? "" : ` — "${prov.reason}"`;
|
|
1513
|
-
return `dispatch paused (source: ${prov.source}${reason})`;
|
|
1514
|
-
}
|
|
1515
|
-
return isPaused(s.project) && pausedAt(s.project) === undefined
|
|
1516
|
-
? "dispatch paused (unparseable sentinel — new work and completion mutations refused; release gates remain available)"
|
|
1517
|
-
: "dispatch paused";
|
|
1518
|
-
})()
|
|
1519
|
-
: `dispatch ${layers.dispatch}`;
|
|
1520
|
-
|
|
1521
|
-
return [
|
|
1522
|
-
dispatchLine,
|
|
1523
|
-
tickLine,
|
|
1524
|
-
...(nextTickLine === undefined ? [] : [nextTickLine]),
|
|
1525
|
-
paneLine,
|
|
1526
|
-
recoveryLine,
|
|
1527
|
-
herdrLine,
|
|
1528
|
-
telegramLine,
|
|
1529
|
-
...(brief === undefined ? [] : [brief]),
|
|
1530
|
-
...(decisions === undefined ? [] : [decisions]),
|
|
1531
|
-
...(failureClasses === undefined ? [] : [failureClasses]),
|
|
1532
|
-
...(intake === undefined ? [] : [intake]),
|
|
1533
|
-
...(graphBlock === undefined ? [] : [graphBlock]),
|
|
1534
|
-
daemonBlock,
|
|
1535
|
-
...formatLastStop(lastStop),
|
|
1536
|
-
"",
|
|
1537
|
-
formatProjectBody(s, workerPhases, now),
|
|
1538
|
-
].join("\n");
|
|
1539
|
-
}
|
|
1540
|
-
|
|
1541
|
-
function formatAvailabilityStatus(s: StatusSnapshot): string[] {
|
|
1542
|
-
const availability = s.availability;
|
|
1543
|
-
if (availability === undefined) return [];
|
|
1544
|
-
if (availability.mode === "always") {
|
|
1545
|
-
return ["availability 24-hour interrupts (no weekly window)"];
|
|
1546
|
-
}
|
|
1547
|
-
const mode =
|
|
1548
|
-
availability.nextTransitionAt === undefined || availability.timezone === undefined
|
|
1549
|
-
? `${availability.mode}; next transition could not be calculated`
|
|
1550
|
-
: `${availability.mode} until ${formatZonedMinute(availability.nextTransitionAt, availability.timezone)}`;
|
|
1551
|
-
const bypass = availability.bypass.length === 0 ? "none" : availability.bypass.join(", ");
|
|
1552
|
-
return [`availability ${mode}; quiet-hours bypass ${bypass}`];
|
|
1553
|
-
}
|
|
1554
|
-
|
|
1555
|
-
function formatDigestScheduleStatus(s: StatusSnapshot): string[] {
|
|
1556
|
-
const schedule = s.digestSchedule;
|
|
1557
|
-
if (schedule === undefined) return [];
|
|
1558
|
-
if (schedule.mode === "disabled") return ["next digest disabled"];
|
|
1559
|
-
if (schedule.mode === "per-tick") return ["next digest every tick"];
|
|
1560
|
-
if (schedule.mode === "due") return ["next digest due now"];
|
|
1561
|
-
return [
|
|
1562
|
-
schedule.nextAt === undefined
|
|
1563
|
-
? "next digest could not be calculated"
|
|
1564
|
-
: `next digest ${formatZonedMinute(schedule.nextAt, schedule.timezone)}`,
|
|
1565
|
-
];
|
|
1566
|
-
}
|
|
1567
|
-
|
|
1568
|
-
function formatProjectBody(
|
|
1569
|
-
s: StatusSnapshot,
|
|
1570
|
-
workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
|
|
1571
|
-
now = Date.now(),
|
|
1572
|
-
): string {
|
|
1573
|
-
const lines = [
|
|
1574
|
-
`project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
|
|
1575
|
-
...(s.pauseReason === undefined ? [] : [`paused ${s.pauseReason}`]),
|
|
1576
|
-
`config ${s.configPath}`,
|
|
1577
|
-
`state ${s.stateDir}`,
|
|
1578
|
-
// The orchestrator-down degrade row: first-class in the body, present only
|
|
1579
|
-
// while the incident is open, so recovery drops it (#288).
|
|
1580
|
-
...(s.orchestratorDown === undefined
|
|
1581
|
-
? []
|
|
1582
|
-
: formatOrchestratorDown(s.orchestratorDown, now)),
|
|
1583
|
-
...formatAvailabilityStatus(s),
|
|
1584
|
-
...formatDigestScheduleStatus(s),
|
|
1585
|
-
"",
|
|
1586
|
-
"caps",
|
|
1587
|
-
` workers ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
|
|
1588
|
-
` issues today ${s.runsToday}`,
|
|
1589
|
-
s.caps.dailySpendUsd === null
|
|
1590
|
-
? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
|
|
1591
|
-
: ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
|
|
1592
|
-
// Its own row beside the spend row, never folded into it: they are two
|
|
1593
|
-
// independent controls and an operator has to see which one stopped the
|
|
1594
|
-
// fleet (#110).
|
|
1595
|
-
` plan usage ${planUsageLine(s.planUsage)}`,
|
|
1596
|
-
// The tracker's API budget, when the renderer could read it. Absent on a
|
|
1597
|
-
// broken `gh`: one missing row, never a broken report (#188). Next to it,
|
|
1598
|
-
// the refusals the tracker actually observed, so a budget that looks
|
|
1599
|
-
// healthy next to every call being refused is still visible (#198).
|
|
1600
|
-
...(s.github === undefined
|
|
1601
|
-
? []
|
|
1602
|
-
: [
|
|
1603
|
-
` github graphql ${s.github.graphql.remaining}/${s.github.graphql.limit}, core ${s.github.core.remaining}/${s.github.core.limit} (graphql resets ${Math.max(0, Math.round((s.github.graphql.reset * 1000 - Date.now()) / 60_000))}m)` +
|
|
1604
|
-
(s.ghRefusals === undefined || s.ghRefusals.count === 0
|
|
1605
|
-
? ""
|
|
1606
|
-
: ` — ${s.ghRefusals.count} refusal(s) in last 5m (last ${new Date(s.ghRefusals.latestAt ?? Date.now()).toISOString().slice(11, 19)}Z)`),
|
|
1607
|
-
]),
|
|
1608
|
-
// The daemon's own observed github traffic today (#198): a separate row
|
|
1609
|
-
// from the polled budget above, so one missing `status` read never hides
|
|
1610
|
-
// the other. `onCall` counts every spawn including ones that end 304, so
|
|
1611
|
-
// `daemon` is spawns and `daemon-304` is the unbilled subset — billed ≈
|
|
1612
|
-
// difference, and the polled budget row stays the authority (#203).
|
|
1613
|
-
` github calls daemon ${s.ghCallsToday?.find((c) => c.source === "daemon")?.calls ?? 0} today` +
|
|
1614
|
-
((s.ghCallsToday?.find((c) => c.source === "daemon-304")?.calls ?? 0) === 0
|
|
1615
|
-
? ""
|
|
1616
|
-
: ` (${s.ghCallsToday!.find((c) => c.source === "daemon-304")!.calls} free 304s)`),
|
|
1617
|
-
// Label-projection ops the tracker has not applied yet (#201): GitHub is
|
|
1618
|
-
// behind what the store decided, and the operator can see the lag instead
|
|
1619
|
-
// of discovering it as a stale label or a missing one.
|
|
1620
|
-
...(s.labelOps === undefined
|
|
1621
|
-
? []
|
|
1622
|
-
: [
|
|
1623
|
-
` labels projection ${s.labelOps.pending} pending (oldest ${
|
|
1624
|
-
s.labelOps.oldestAgeMs >= 60_000
|
|
1625
|
-
? `${Math.round(s.labelOps.oldestAgeMs / 60_000)}m`
|
|
1626
|
-
: `${Math.max(1, Math.round(s.labelOps.oldestAgeMs / 1000))}s`
|
|
1627
|
-
})`,
|
|
1628
|
-
]),
|
|
1629
|
-
` new worker turns ${s.caps.workerMaxTurns}`,
|
|
1630
|
-
...s.turnOverrides.map(
|
|
1631
|
-
({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`,
|
|
1632
|
-
),
|
|
1633
|
-
` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
|
|
1634
|
-
` failed attempts ${s.caps.maxAttemptsPerIssue}`,
|
|
1635
|
-
` continuations ${s.caps.maxContinuationsPerIssue}`,
|
|
1636
|
-
"",
|
|
1637
|
-
...formatReleaseGrants(s.releaseGrants),
|
|
1638
|
-
"",
|
|
1639
|
-
formatDispatchSummary(s.dispatch),
|
|
1640
|
-
"",
|
|
1641
|
-
];
|
|
1642
|
-
if (s.activeRuns.length === 0) {
|
|
1643
|
-
lines.push("active runs (none)");
|
|
1644
|
-
} else {
|
|
1645
|
-
lines.push("active runs");
|
|
1646
|
-
for (const r of s.activeRuns) {
|
|
1647
|
-
const phase = workerPhases.get(r.issue);
|
|
1648
|
-
const state = phase === "pausing" || phase === "paused" ? phase : r.state;
|
|
1649
|
-
lines.push(
|
|
1650
|
-
` #${r.issue} ${r.repo} ${state} attempt ${r.attempt} ` +
|
|
1651
|
-
`${r.turns}/${r.maxTurns} turns ${r.spendUsd.toFixed(2)} ${r.branch}` +
|
|
1652
|
-
(r.prUrl ? ` ${r.prUrl}` : ""),
|
|
1653
|
-
);
|
|
1654
|
-
// The orchestrator's Duty 1 reads this command, and a flagged run's
|
|
1655
|
-
// escalation is deduplicated after one delivery — so this is where a
|
|
1656
|
-
// flagged PR stays visible for as long as it is still open (#128).
|
|
1657
|
-
const flagged = settlementFlagSummary(r.settlementFlags);
|
|
1658
|
-
if (flagged !== undefined) lines.push(` ${flagged}`);
|
|
1659
|
-
}
|
|
1660
|
-
}
|
|
1661
|
-
lines.push(...formatBaseHealth(s.baseHealth));
|
|
1662
|
-
lines.push(...formatSalvagedRuns(s.salvagedRuns));
|
|
1663
|
-
lines.push(...formatOpenReports(s.openReports));
|
|
1664
|
-
lines.push(...formatDigestBacklog(s.digestBacklog));
|
|
1665
|
-
if (s.liveWorkers > 0) {
|
|
1666
|
-
lines.push(
|
|
1667
|
-
"",
|
|
1668
|
-
`deploy ${s.liveWorkers} live worker(s) — restart salvages dirty trees then orphans the rows; ` +
|
|
1669
|
-
`hold and wait for workers 0/${s.caps.maxConcurrentWorkers} when you can drain instead`,
|
|
1670
|
-
);
|
|
1671
|
-
}
|
|
1672
|
-
return lines.join("\n");
|
|
1673
|
-
}
|
|
1674
|
-
|
|
1675
1314
|
export async function renderStatus(projectName?: string): Promise<string> {
|
|
1676
1315
|
const s = statusSnapshot(projectName);
|
|
1677
1316
|
const layers = fleetLayers(projectName);
|
|
@@ -1716,8 +1355,16 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
1716
1355
|
// project and the uninvolved one must see who stopped it too.
|
|
1717
1356
|
const store = openStore(dbPath());
|
|
1718
1357
|
let lastStop: DaemonStop | undefined;
|
|
1358
|
+
let siblings: { project: string; live: number }[] = [];
|
|
1719
1359
|
try {
|
|
1720
1360
|
lastStop = store.latestDaemonStop();
|
|
1361
|
+
// Shared-daemon visibility (#545): every configured project other than the
|
|
1362
|
+
// one being viewed, with its live-run count. Read from the same open store
|
|
1363
|
+
// as the provenance above, so a status reader can tell "my fleet is idle"
|
|
1364
|
+
// from "the process I am about to stop is busy".
|
|
1365
|
+
siblings = loadConfig()
|
|
1366
|
+
.projects.filter((p) => p.name !== project.name)
|
|
1367
|
+
.map((p) => ({ project: p.name, live: store.liveRuns(p.name).length }));
|
|
1721
1368
|
} finally {
|
|
1722
1369
|
store.close();
|
|
1723
1370
|
}
|
|
@@ -1734,6 +1381,7 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
1734
1381
|
workerPhases,
|
|
1735
1382
|
intakeStatusLine(project.name),
|
|
1736
1383
|
lastStop,
|
|
1384
|
+
siblings,
|
|
1737
1385
|
);
|
|
1738
1386
|
}
|
|
1739
1387
|
|
|
@@ -1793,18 +1441,32 @@ function failureClassBlock(projectName: string): string | undefined {
|
|
|
1793
1441
|
* the ambiguity this ledger exists to remove. Omitted on any throw — a status
|
|
1794
1442
|
* that fails to render because the store is mid-migration is worse than one
|
|
1795
1443
|
* missing a row.
|
|
1444
|
+
*
|
|
1445
|
+
* `decisions N open` counts only rows a human must answer (#459). Watches the
|
|
1446
|
+
* orchestrator set for itself are reported separately — a fleet with three
|
|
1447
|
+
* watches behind GitHub's checks is not a fleet waiting on its operator, and
|
|
1448
|
+
* lumping them in made `decisions 3 open` read as three unanswered questions.
|
|
1796
1449
|
*/
|
|
1797
|
-
function decisionStatusLine(projectName: string): string | undefined {
|
|
1450
|
+
export function decisionStatusLine(projectName: string): string | undefined {
|
|
1798
1451
|
const path = dbPath();
|
|
1799
1452
|
if (!existsSync(path)) return undefined;
|
|
1800
1453
|
let store: Store | undefined;
|
|
1801
1454
|
try {
|
|
1802
1455
|
store = openStore(path);
|
|
1803
1456
|
const open = store.openDecisions(projectName);
|
|
1804
|
-
|
|
1805
|
-
const
|
|
1806
|
-
|
|
1807
|
-
|
|
1457
|
+
const questions = open.filter((d) => d.kind !== "watch");
|
|
1458
|
+
const watches = open.filter((d) => d.kind === "watch");
|
|
1459
|
+
if (questions.length === 0 && watches.length === 0) return "decisions none open";
|
|
1460
|
+
let line: string;
|
|
1461
|
+
if (questions.length === 0) {
|
|
1462
|
+
line = "decisions none open";
|
|
1463
|
+
} else {
|
|
1464
|
+
const oldest = questions[0]!;
|
|
1465
|
+
const hours = Math.max(0, Math.round((Date.now() - oldest.askedAt) / 3_600_000));
|
|
1466
|
+
line = `decisions ${questions.length} open (oldest ${hours}h)`;
|
|
1467
|
+
}
|
|
1468
|
+
if (watches.length > 0) line += ` · watches ${watches.length}`;
|
|
1469
|
+
return line;
|
|
1808
1470
|
} catch {
|
|
1809
1471
|
return undefined;
|
|
1810
1472
|
} finally {
|
|
@@ -1973,6 +1635,56 @@ export function sessionDirForCwd(cwd: string): string {
|
|
|
1973
1635
|
return join(home, ".omp", "agent", "sessions", slug.replaceAll("/", "-"));
|
|
1974
1636
|
}
|
|
1975
1637
|
|
|
1638
|
+
/** The session tree arm scans: every transcript under it, whatever the cwd slug. */
|
|
1639
|
+
export function sessionsRoot(): string {
|
|
1640
|
+
return join(homedir(), ".omp", "agent", "sessions");
|
|
1641
|
+
}
|
|
1642
|
+
|
|
1643
|
+
/**
|
|
1644
|
+
* The orchestrator session file omp-telegram's live claim names for this
|
|
1645
|
+
* project — the transcript a resumed/restored pane actually writes, which can
|
|
1646
|
+
* live in a different session directory than the tick cwd implies (#600).
|
|
1647
|
+
* Never throws: no claim, or an unreadable config, means undefined and arm
|
|
1648
|
+
* falls back to the cwd-derived directory.
|
|
1649
|
+
*/
|
|
1650
|
+
function claimedOrchestratorSessionFile(named: string | undefined): string | undefined {
|
|
1651
|
+
if (named === undefined) return undefined;
|
|
1652
|
+
try {
|
|
1653
|
+
return resolveClaimedSessionFile(findProject(loadConfig(), named));
|
|
1654
|
+
} catch {
|
|
1655
|
+
return undefined;
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
/**
|
|
1660
|
+
* The session directories `arm` watches for the reply — the cwd-derived one
|
|
1661
|
+
* plus, when the live claim's file lives elsewhere, that file's directory.
|
|
1662
|
+
*
|
|
1663
|
+
* The cwd-derived directory is where a *fresh* session started from the tick
|
|
1664
|
+
* cwd writes; the claimed file's directory is where the *restored* pane keeps
|
|
1665
|
+
* writing (herdr pins it to the original transcript). Watching both covers a
|
|
1666
|
+
* resumed session, a rotated one, and a session that starts after the send.
|
|
1667
|
+
*
|
|
1668
|
+
* A claim outside the session tree is a stop, not a slower failure: arm polls
|
|
1669
|
+
* only transcripts under {@link sessionsRoot}, so a claimed file elsewhere can
|
|
1670
|
+
* never satisfy the challenge, and five minutes of polling cannot discover a
|
|
1671
|
+
* file that is not in the search set. The throw names both paths.
|
|
1672
|
+
*/
|
|
1673
|
+
function armSessionScanDirs(cwd: string, claimed: string | undefined): string[] {
|
|
1674
|
+
const cwdDir = sessionDirForCwd(cwd);
|
|
1675
|
+
if (claimed === undefined) return [cwdDir];
|
|
1676
|
+
const root = sessionsRoot();
|
|
1677
|
+
const claimDir = dirname(claimed);
|
|
1678
|
+
if (claimDir !== root && !claimDir.startsWith(join(root, sep))) {
|
|
1679
|
+
throw new Error(
|
|
1680
|
+
`arm: the orchestrator's claimed session file ${claimed} is outside the session tree arm scans (${root}) — ` +
|
|
1681
|
+
`no transcript there can satisfy the challenge. Resume the pane under ${root}, or start it from ${cwd}; ` +
|
|
1682
|
+
`arm would otherwise poll ${cwdDir} until the timer runs out`,
|
|
1683
|
+
);
|
|
1684
|
+
}
|
|
1685
|
+
return claimDir === cwdDir ? [cwdDir] : [cwdDir, claimDir];
|
|
1686
|
+
}
|
|
1687
|
+
|
|
1976
1688
|
function makeChallengeCode(): string {
|
|
1977
1689
|
const bytes = new Uint8Array(4);
|
|
1978
1690
|
crypto.getRandomValues(bytes);
|
|
@@ -1999,20 +1711,23 @@ interface SessionScan {
|
|
|
1999
1711
|
}
|
|
2000
1712
|
|
|
2001
1713
|
/**
|
|
2002
|
-
* Polls the session
|
|
1714
|
+
* Polls the session directories — re-read on every pass, never snapshotted —
|
|
2003
1715
|
* until the challenge shows up as a user turn or the deadline passes.
|
|
2004
1716
|
*
|
|
2005
1717
|
* Discovery lives here because the reply can land in a transcript that does not
|
|
2006
1718
|
* exist yet when the challenge is sent: a rotated session, or the first one of
|
|
2007
|
-
* a pane started right after arming (#142).
|
|
2008
|
-
*
|
|
1719
|
+
* a pane started right after arming (#142). And the scan set can be two
|
|
1720
|
+
* directories when the live claim's session file lives outside the tick-cwd
|
|
1721
|
+
* directory: the restored pane keeps writing the claimed file, so its
|
|
1722
|
+
* directory is watched alongside the cwd-derived one (#600).
|
|
2009
1723
|
*
|
|
2010
1724
|
* Files untouched since just before the send are named, not parsed: an append
|
|
2011
1725
|
* bumps mtime, so a transcript older than the challenge cannot hold the reply,
|
|
2012
|
-
* and skipping it keeps a large stale session out of every 5 s pass.
|
|
1726
|
+
* and skipping it keeps a large stale session out of every 5 s pass. A missing
|
|
1727
|
+
* directory is not an error — it can vanish under a rotation and reappear.
|
|
2013
1728
|
*/
|
|
2014
1729
|
async function waitForChallengeInSessions(
|
|
2015
|
-
|
|
1730
|
+
dirs: readonly string[],
|
|
2016
1731
|
code: string,
|
|
2017
1732
|
sentAt: number,
|
|
2018
1733
|
timeoutMs: number,
|
|
@@ -2024,28 +1739,30 @@ async function waitForChallengeInSessions(
|
|
|
2024
1739
|
for (;;) {
|
|
2025
1740
|
const scanned: string[] = [];
|
|
2026
1741
|
const ignored: string[] = [];
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
names = readdirSync(dir);
|
|
2030
|
-
} catch {
|
|
2031
|
-
/* the directory can go away under a rotation; the next pass re-reads it */
|
|
2032
|
-
}
|
|
2033
|
-
for (const name of names.sort()) {
|
|
2034
|
-
if (!name.endsWith(".jsonl")) continue;
|
|
2035
|
-
const path = join(dir, name);
|
|
2036
|
-
let mtimeMs: number;
|
|
1742
|
+
for (const dir of dirs) {
|
|
1743
|
+
let names: string[] = [];
|
|
2037
1744
|
try {
|
|
2038
|
-
|
|
1745
|
+
names = readdirSync(dir);
|
|
2039
1746
|
} catch {
|
|
2040
|
-
|
|
1747
|
+
/* the directory can go away under a rotation; the next pass re-reads it */
|
|
2041
1748
|
}
|
|
2042
|
-
const
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
1749
|
+
for (const name of names.sort()) {
|
|
1750
|
+
if (!name.endsWith(".jsonl")) continue;
|
|
1751
|
+
const path = join(dir, name);
|
|
1752
|
+
let mtimeMs: number;
|
|
1753
|
+
try {
|
|
1754
|
+
mtimeMs = statSync(path).mtimeMs;
|
|
1755
|
+
} catch {
|
|
1756
|
+
continue; /* race */
|
|
1757
|
+
}
|
|
1758
|
+
const label = `${name} (mtime ${new Date(mtimeMs).toISOString()})`;
|
|
1759
|
+
if (mtimeMs < sentAt - 1_000) {
|
|
1760
|
+
ignored.push(label);
|
|
1761
|
+
continue;
|
|
1762
|
+
}
|
|
1763
|
+
scanned.push(label);
|
|
1764
|
+
if (await transcriptHasUserCode(path, code)) return { seen: true, scanned, ignored };
|
|
2046
1765
|
}
|
|
2047
|
-
scanned.push(label);
|
|
2048
|
-
if (await transcriptHasUserCode(path, code)) return { seen: true, scanned, ignored };
|
|
2049
1766
|
}
|
|
2050
1767
|
if (now() >= deadline) return { seen: false, scanned, ignored };
|
|
2051
1768
|
await sleep(5_000);
|
|
@@ -2167,4 +1884,3 @@ function probeOmpPane(
|
|
|
2167
1884
|
return { kind: "unknown", reason: err instanceof Error ? err.message : String(err) };
|
|
2168
1885
|
}
|
|
2169
1886
|
}
|
|
2170
|
-
|