omp-conductor 0.15.13 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +72 -2
- package/package.json +2 -1
- package/schema/config.schema.json +6 -0
- package/src/admission.ts +745 -0
- package/src/ask.ts +47 -0
- package/src/backups.ts +19 -7
- package/src/board.ts +1 -2
- package/src/briefs/orchestrator.md +62 -4
- package/src/cli.ts +26 -0
- package/src/commands/context.ts +3 -0
- package/src/commands/decision.ts +10 -1
- package/src/commands/doctor.ts +2 -0
- package/src/commands/message.ts +8 -1
- package/src/commands/restart.ts +15 -3
- package/src/commands/restore-db.ts +146 -0
- package/src/commands/stop.ts +24 -15
- package/src/commands/unfreeze.ts +56 -0
- package/src/commands/watch.ts +77 -0
- package/src/config-schema.ts +9 -0
- package/src/config.ts +24 -0
- package/src/daemon.ts +239 -530
- package/src/dashboard/server.ts +2 -1
- package/src/decisions.ts +32 -7
- package/src/depends-on.ts +73 -0
- package/src/doctor.ts +178 -5
- package/src/escalate.ts +114 -15
- package/src/failure-class.ts +47 -0
- package/src/fleet.ts +41 -410
- package/src/gitops.ts +86 -1
- package/src/log.ts +40 -0
- package/src/model-fallback.ts +3 -2
- package/src/omp-settings.ts +114 -0
- package/src/omp.ts +39 -0
- package/src/orchestrator-tick.ts +7 -1
- package/src/reports.ts +124 -12
- package/src/session-host.ts +6 -0
- package/src/setup-wizard.ts +36 -0
- package/src/setup.ts +58 -1
- package/src/status-render.ts +445 -0
- package/src/stop-provenance.ts +53 -0
- package/src/store.ts +352 -11
- package/src/types.ts +187 -4
- package/src/unblock.ts +1 -1
- package/src/upgrade-verify.ts +1 -1
- package/src/upgrade.ts +1 -2
- package/src/verbs/server.ts +25 -0
- package/src/worker.ts +162 -10
package/src/fleet.ts
CHANGED
|
@@ -27,32 +27,17 @@ import {
|
|
|
27
27
|
import { createInterface } from "node:readline";
|
|
28
28
|
import { homedir } from "node:os";
|
|
29
29
|
import { dirname, join } from "node:path";
|
|
30
|
-
import { formatZonedMinute } from "./availability.ts";
|
|
31
30
|
import { findProject, loadConfig, stateDir } from "./config.ts";
|
|
32
31
|
import { clearArmChallenge, recordArmChallenge } from "./arm-challenge.ts";
|
|
33
32
|
import { resolveProjectTopicId, sendTelegram } from "./escalate.ts";
|
|
34
|
-
import {
|
|
33
|
+
import { readPlanUsage, sharedUsageSource } from "./usage.ts";
|
|
35
34
|
import { readApprovalSurface, readDaemonProfile } from "./approval-surface.ts";
|
|
36
35
|
import { inspectBriefLayout } from "./brief-upgrade.ts";
|
|
37
36
|
import { dbPath, openStore } from "./store.ts";
|
|
38
37
|
import { renderBriefForProject } from "./setup.ts";
|
|
39
38
|
import type { DaemonStop, ProjectConfig, Store } from "./types.ts";
|
|
40
|
-
import { settlementFlagSummary } from "./diff-flags.ts";
|
|
41
39
|
import { probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
42
|
-
import {
|
|
43
|
-
import {
|
|
44
|
-
formatBaseHealth,
|
|
45
|
-
formatDispatchSummary,
|
|
46
|
-
formatReleaseGrants,
|
|
47
|
-
formatSalvagedRuns,
|
|
48
|
-
isPaused,
|
|
49
|
-
pauseProvenance,
|
|
50
|
-
pausedAt,
|
|
51
|
-
setPaused,
|
|
52
|
-
statusSnapshot,
|
|
53
|
-
type StatusSnapshot,
|
|
54
|
-
} from "./daemon.ts";
|
|
55
|
-
import { formatOrchestratorDown } from "./orchestrator-down.ts";
|
|
40
|
+
import { isPaused, setPaused, statusSnapshot } from "./daemon.ts";
|
|
56
41
|
import {
|
|
57
42
|
healthCheck,
|
|
58
43
|
isAlive,
|
|
@@ -61,11 +46,19 @@ import {
|
|
|
61
46
|
runSystemctl,
|
|
62
47
|
stopDaemon,
|
|
63
48
|
type StopResult,
|
|
64
|
-
type UnitOwnership,
|
|
65
|
-
SYSTEMD_UNIT,
|
|
66
49
|
} from "./lifecycle.ts";
|
|
67
|
-
import { formatRss, rssBytesFromHealthz } from "./host.ts";
|
|
68
50
|
import type { WorkerPausePhase } from "./worker.ts";
|
|
51
|
+
import {
|
|
52
|
+
formatFleetStatus,
|
|
53
|
+
type DaemonProjectHealth,
|
|
54
|
+
type DispatchLayer,
|
|
55
|
+
type FleetDaemonProbe,
|
|
56
|
+
type FleetLayers,
|
|
57
|
+
type HerdrLayer,
|
|
58
|
+
type PaneLayer,
|
|
59
|
+
type TelegramHealth,
|
|
60
|
+
type TicksLayer,
|
|
61
|
+
} from "./status-render.ts";
|
|
69
62
|
import { fetchRateLimit } from "./tracker/github.ts";
|
|
70
63
|
import {
|
|
71
64
|
LEGACY_ARM_MARKER_DETAIL,
|
|
@@ -1023,46 +1016,6 @@ function isOmpProcess(proc: ForegroundProc): boolean {
|
|
|
1023
1016
|
return false;
|
|
1024
1017
|
}
|
|
1025
1018
|
|
|
1026
|
-
// ---------------------------------------------------------------------------
|
|
1027
|
-
// layered status
|
|
1028
|
-
// ---------------------------------------------------------------------------
|
|
1029
|
-
|
|
1030
|
-
export type DispatchLayer = "running" | "paused" | "stopped";
|
|
1031
|
-
export type TicksLayer =
|
|
1032
|
-
| "armed"
|
|
1033
|
-
| "disarmed"
|
|
1034
|
-
| "no-heartbeat-config"
|
|
1035
|
-
| "invalid-heartbeat-config"
|
|
1036
|
-
| "ungated";
|
|
1037
|
-
export type PaneLayer = "live" | "missing" | "unknown";
|
|
1038
|
-
/** `unpinnable`: no tick config, so FLEET_CWD — the only path recovery reads — is unknown. */
|
|
1039
|
-
export type RecoveryLayer = "pinned" | "clear" | "unpinnable";
|
|
1040
|
-
export type HerdrLayer = "active" | "inactive" | "unknown";
|
|
1041
|
-
export type TelegramLayer = "ok" | "degraded" | "down" | "unconfigured" | "unprobed";
|
|
1042
|
-
|
|
1043
|
-
export interface TelegramHealth {
|
|
1044
|
-
kind: TelegramLayer;
|
|
1045
|
-
detail?: string;
|
|
1046
|
-
}
|
|
1047
|
-
|
|
1048
|
-
export interface FleetLayers {
|
|
1049
|
-
dispatch: DispatchLayer;
|
|
1050
|
-
ticks: TicksLayer;
|
|
1051
|
-
ticksDetail?: string;
|
|
1052
|
-
nextTickAt?: string;
|
|
1053
|
-
pane: PaneLayer;
|
|
1054
|
-
paneDetail?: string;
|
|
1055
|
-
recovery: RecoveryLayer;
|
|
1056
|
-
recoveryDetail?: string;
|
|
1057
|
-
herdr: HerdrLayer;
|
|
1058
|
-
herdrDetail?: string;
|
|
1059
|
-
armedPath?: string;
|
|
1060
|
-
tickConfigPath?: string;
|
|
1061
|
-
paneHaltPath?: string;
|
|
1062
|
-
paused: boolean;
|
|
1063
|
-
daemon: { running: boolean; pid?: number; port?: number };
|
|
1064
|
-
}
|
|
1065
|
-
|
|
1066
1019
|
/**
|
|
1067
1020
|
* The project a bare read means, when the config leaves no doubt.
|
|
1068
1021
|
*
|
|
@@ -1270,50 +1223,6 @@ export function workerPhasesFromHealthz(
|
|
|
1270
1223
|
}
|
|
1271
1224
|
|
|
1272
1225
|
|
|
1273
|
-
export function formatCodeGraphHealth(graph: CodeGraphHealth, now = Date.now()): string | undefined {
|
|
1274
|
-
if (!graph.configured) return undefined;
|
|
1275
|
-
const indexed = graph.repos.filter((repo) => repo.index === "present").length;
|
|
1276
|
-
let refresh: string = graph.refresh.result;
|
|
1277
|
-
if (graph.refresh.lastSuccessAt !== undefined) {
|
|
1278
|
-
const ageMs = Math.max(0, now - Date.parse(graph.refresh.lastSuccessAt));
|
|
1279
|
-
refresh = `${graph.refresh.lastSuccessAt} (${Math.max(1, Math.ceil(ageMs / 60_000))}m ago)`;
|
|
1280
|
-
}
|
|
1281
|
-
return [
|
|
1282
|
-
`code graph ${graph.status} ${indexed}/${graph.repos.length} repos indexed`,
|
|
1283
|
-
` indexer ${graph.prerequisites.indexer}`,
|
|
1284
|
-
` MCP mount ${graph.prerequisites.mcpMount}`,
|
|
1285
|
-
` timer ${graph.timer.enabled} / ${graph.timer.active}`,
|
|
1286
|
-
` refresh ${refresh}`,
|
|
1287
|
-
...graph.reasons.map((reason) => ` - ${reason}`),
|
|
1288
|
-
].join("\n");
|
|
1289
|
-
}
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
/**
|
|
1293
|
-
* Project-scoped reading of a living daemon record + `/healthz` body.
|
|
1294
|
-
*
|
|
1295
|
-
* Board and `status` share this so a host running one daemon for several
|
|
1296
|
-
* projects cannot be called healthy for a project it does not serve (#379).
|
|
1297
|
-
* A third interpretation convention is forbidden.
|
|
1298
|
-
*/
|
|
1299
|
-
export type DaemonProjectHealth =
|
|
1300
|
-
| { kind: "stopped" }
|
|
1301
|
-
| { kind: "ok" }
|
|
1302
|
-
| { kind: "unreachable" }
|
|
1303
|
-
| { kind: "other-project"; serves?: string };
|
|
1304
|
-
|
|
1305
|
-
/**
|
|
1306
|
-
* Facts `formatFleetStatus` needs about the living daemon. Pure input — the
|
|
1307
|
-
* caller probes; the formatter never shells out (#379).
|
|
1308
|
-
*/
|
|
1309
|
-
export type FleetDaemonProbe = {
|
|
1310
|
-
project: DaemonProjectHealth;
|
|
1311
|
-
/** `/healthz` body when the probe is project-ok (rss / overlays). */
|
|
1312
|
-
body?: string;
|
|
1313
|
-
/** systemd ownership of the living record pid; omit when unprobed. */
|
|
1314
|
-
unit?: UnitOwnership;
|
|
1315
|
-
};
|
|
1316
|
-
|
|
1317
1226
|
/**
|
|
1318
1227
|
* Decide whether a living record + optional `/healthz` answer serve `project`.
|
|
1319
1228
|
*
|
|
@@ -1372,306 +1281,6 @@ function servedProjectName(payload: object): string | undefined {
|
|
|
1372
1281
|
return undefined;
|
|
1373
1282
|
}
|
|
1374
1283
|
|
|
1375
|
-
function formatDaemonHealthz(probe: FleetDaemonProbe | undefined): string {
|
|
1376
|
-
if (probe === undefined) return "unprobed";
|
|
1377
|
-
switch (probe.project.kind) {
|
|
1378
|
-
case "stopped":
|
|
1379
|
-
return "stopped";
|
|
1380
|
-
case "ok":
|
|
1381
|
-
return "ok";
|
|
1382
|
-
case "unreachable":
|
|
1383
|
-
return "unreachable — the process is up but not serving";
|
|
1384
|
-
case "other-project":
|
|
1385
|
-
return probe.project.serves === undefined
|
|
1386
|
-
? "other-project"
|
|
1387
|
-
: `other-project — serves ${probe.project.serves}`;
|
|
1388
|
-
}
|
|
1389
|
-
}
|
|
1390
|
-
|
|
1391
|
-
/**
|
|
1392
|
-
* Unit line for a living record. The bare unit name alone used to imply
|
|
1393
|
-
* systemd ownership that was never checked (#379); every branch states the
|
|
1394
|
-
* probe result, and `unknown` is never rendered as owned.
|
|
1395
|
-
*/
|
|
1396
|
-
function formatDaemonUnit(unit: UnitOwnership | undefined, recordPid: number): string {
|
|
1397
|
-
if (unit === undefined) return ` unit ${SYSTEMD_UNIT} unprobed`;
|
|
1398
|
-
switch (unit.kind) {
|
|
1399
|
-
case "active":
|
|
1400
|
-
if (unit.pid === recordPid) {
|
|
1401
|
-
return ` unit ${SYSTEMD_UNIT} systemd-owned`;
|
|
1402
|
-
}
|
|
1403
|
-
// Live unit, different MainPID — the record pid is not systemd's.
|
|
1404
|
-
return ` unit ${SYSTEMD_UNIT} unmanaged — MainPID ${unit.pid}`;
|
|
1405
|
-
case "failed":
|
|
1406
|
-
return ` unit ${SYSTEMD_UNIT} unmanaged — unit failed`;
|
|
1407
|
-
case "inactive":
|
|
1408
|
-
return ` unit ${SYSTEMD_UNIT} inactive`;
|
|
1409
|
-
case "unknown":
|
|
1410
|
-
return ` unit ${SYSTEMD_UNIT} unknown — ${unit.reason}`;
|
|
1411
|
-
}
|
|
1412
|
-
}
|
|
1413
|
-
|
|
1414
|
-
/**
|
|
1415
|
-
* The newest daemon stop/restart provenance as status lines (#378).
|
|
1416
|
-
*
|
|
1417
|
-
* Rendered under the daemon block whether the daemon is down or has since
|
|
1418
|
-
* been restarted — the whole point is that the debrief survives the restart.
|
|
1419
|
-
* An unattributed record says so explicitly: the external-signal fallback
|
|
1420
|
-
* records what the receiving daemon knew (that it was unattributed, when,
|
|
1421
|
-
* which projects it served with live counts) rather than guessing a caller.
|
|
1422
|
-
*/
|
|
1423
|
-
function formatLastStop(lastStop: DaemonStop | undefined): string[] {
|
|
1424
|
-
if (lastStop === undefined) return [];
|
|
1425
|
-
const caller = lastStop.unattributed
|
|
1426
|
-
? "unattributed — no mediated request (external signal)"
|
|
1427
|
-
: `pid ${lastStop.callerPid ?? "?"}` +
|
|
1428
|
-
(lastStop.callerUid === undefined ? "" : ` uid ${lastStop.callerUid}`) +
|
|
1429
|
-
(lastStop.role === undefined ? "" : ` (${lastStop.role})`);
|
|
1430
|
-
return [
|
|
1431
|
-
` last stop ${new Date(lastStop.at).toISOString()} ${lastStop.controlPath}`,
|
|
1432
|
-
` caller ${caller}`,
|
|
1433
|
-
` scope ${lastStop.scope}${lastStop.project === undefined ? "" : `: ${lastStop.project}`}`,
|
|
1434
|
-
` affects ${lastStop.affected.map((a) => `${a.project} (${a.live} live)`).join(", ")}`,
|
|
1435
|
-
` reason ${lastStop.reason}`,
|
|
1436
|
-
];
|
|
1437
|
-
}
|
|
1438
|
-
|
|
1439
|
-
export function formatFleetStatus(
|
|
1440
|
-
s: StatusSnapshot,
|
|
1441
|
-
layers: FleetLayers,
|
|
1442
|
-
daemon: FleetDaemonProbe | undefined = undefined,
|
|
1443
|
-
telegram: TelegramHealth = { kind: "unprobed" },
|
|
1444
|
-
now = Date.now(),
|
|
1445
|
-
codeGraph: CodeGraphHealth = { configured: false },
|
|
1446
|
-
brief: string | undefined = undefined,
|
|
1447
|
-
decisions: string | undefined = undefined,
|
|
1448
|
-
failureClasses: string | undefined = undefined,
|
|
1449
|
-
workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
|
|
1450
|
-
intake: string | undefined = undefined,
|
|
1451
|
-
lastStop: DaemonStop | undefined = undefined,
|
|
1452
|
-
): string {
|
|
1453
|
-
const tickLine =
|
|
1454
|
-
layers.ticksDetail === undefined
|
|
1455
|
-
? `ticks ${layers.ticks}`
|
|
1456
|
-
: `ticks ${layers.ticks} (${layers.ticksDetail})`;
|
|
1457
|
-
let nextTickLine: string | undefined;
|
|
1458
|
-
if (layers.nextTickAt !== undefined) {
|
|
1459
|
-
const delta = Date.parse(layers.nextTickAt) - now;
|
|
1460
|
-
const minutes = Math.max(1, Math.ceil(Math.abs(delta) / 60_000));
|
|
1461
|
-
nextTickLine =
|
|
1462
|
-
`next tick ${layers.nextTickAt} ` +
|
|
1463
|
-
`(${delta >= 0 ? `in ${minutes}m` : `overdue by ${minutes}m`})`;
|
|
1464
|
-
}
|
|
1465
|
-
const paneLine =
|
|
1466
|
-
layers.paneDetail === undefined
|
|
1467
|
-
? `pane ${layers.pane}`
|
|
1468
|
-
: `pane ${layers.pane} (${layers.paneDetail})`;
|
|
1469
|
-
const recoveryLine =
|
|
1470
|
-
layers.recoveryDetail === undefined
|
|
1471
|
-
? `recovery ${layers.recovery}`
|
|
1472
|
-
: `recovery ${layers.recovery} (${layers.recoveryDetail})`;
|
|
1473
|
-
const herdrLine =
|
|
1474
|
-
layers.herdrDetail === undefined
|
|
1475
|
-
? `herdr ${layers.herdr}`
|
|
1476
|
-
: `herdr ${layers.herdr} (${layers.herdrDetail})`;
|
|
1477
|
-
const telegramLine =
|
|
1478
|
-
telegram.detail === undefined
|
|
1479
|
-
? `telegram ${telegram.kind}`
|
|
1480
|
-
: `telegram ${telegram.kind} (${telegram.detail})`;
|
|
1481
|
-
|
|
1482
|
-
let daemonBlock: string;
|
|
1483
|
-
if (!layers.daemon.running || layers.daemon.pid === undefined) {
|
|
1484
|
-
daemonBlock = "daemon not running";
|
|
1485
|
-
} else {
|
|
1486
|
-
// Only trust rss from a project-ok body — a foreign payload's bytes are
|
|
1487
|
-
// not this project's daemon facts (#379).
|
|
1488
|
-
const rss =
|
|
1489
|
-
daemon?.project.kind === "ok" ? rssBytesFromHealthz(daemon.body) : undefined;
|
|
1490
|
-
daemonBlock = [
|
|
1491
|
-
"daemon",
|
|
1492
|
-
` pid ${layers.daemon.pid}`,
|
|
1493
|
-
` port ${layers.daemon.port ?? "?"}`,
|
|
1494
|
-
...(rss === undefined ? [] : [` rss ${formatRss(rss)}`]),
|
|
1495
|
-
` healthz ${formatDaemonHealthz(daemon)}`,
|
|
1496
|
-
formatDaemonUnit(daemon?.unit, layers.daemon.pid),
|
|
1497
|
-
].join("\n");
|
|
1498
|
-
}
|
|
1499
|
-
|
|
1500
|
-
const graphBlock = formatCodeGraphHealth(codeGraph, now);
|
|
1501
|
-
|
|
1502
|
-
// Pause provenance (`pause --reason`, an integrity/spend-cap/upgrade pause,
|
|
1503
|
-
// a halt) answers "who stopped the fleet" without opening a file (#185). An
|
|
1504
|
-
// unparseable sentinel — paused but with no datable line 1 — is itself news:
|
|
1505
|
-
// it means a run admitted before an *unknown* pause cannot prove innocence
|
|
1506
|
-
// (#174), so completion mutations fail closed while release gates remain usable.
|
|
1507
|
-
const dispatchLine =
|
|
1508
|
-
layers.dispatch === "paused"
|
|
1509
|
-
? (() => {
|
|
1510
|
-
const prov = pauseProvenance(s.project);
|
|
1511
|
-
if (prov !== undefined) {
|
|
1512
|
-
const reason = prov.reason === undefined ? "" : ` — "${prov.reason}"`;
|
|
1513
|
-
return `dispatch paused (source: ${prov.source}${reason})`;
|
|
1514
|
-
}
|
|
1515
|
-
return isPaused(s.project) && pausedAt(s.project) === undefined
|
|
1516
|
-
? "dispatch paused (unparseable sentinel — new work and completion mutations refused; release gates remain available)"
|
|
1517
|
-
: "dispatch paused";
|
|
1518
|
-
})()
|
|
1519
|
-
: `dispatch ${layers.dispatch}`;
|
|
1520
|
-
|
|
1521
|
-
return [
|
|
1522
|
-
dispatchLine,
|
|
1523
|
-
tickLine,
|
|
1524
|
-
...(nextTickLine === undefined ? [] : [nextTickLine]),
|
|
1525
|
-
paneLine,
|
|
1526
|
-
recoveryLine,
|
|
1527
|
-
herdrLine,
|
|
1528
|
-
telegramLine,
|
|
1529
|
-
...(brief === undefined ? [] : [brief]),
|
|
1530
|
-
...(decisions === undefined ? [] : [decisions]),
|
|
1531
|
-
...(failureClasses === undefined ? [] : [failureClasses]),
|
|
1532
|
-
...(intake === undefined ? [] : [intake]),
|
|
1533
|
-
...(graphBlock === undefined ? [] : [graphBlock]),
|
|
1534
|
-
daemonBlock,
|
|
1535
|
-
...formatLastStop(lastStop),
|
|
1536
|
-
"",
|
|
1537
|
-
formatProjectBody(s, workerPhases, now),
|
|
1538
|
-
].join("\n");
|
|
1539
|
-
}
|
|
1540
|
-
|
|
1541
|
-
function formatAvailabilityStatus(s: StatusSnapshot): string[] {
|
|
1542
|
-
const availability = s.availability;
|
|
1543
|
-
if (availability === undefined) return [];
|
|
1544
|
-
if (availability.mode === "always") {
|
|
1545
|
-
return ["availability 24-hour interrupts (no weekly window)"];
|
|
1546
|
-
}
|
|
1547
|
-
const mode =
|
|
1548
|
-
availability.nextTransitionAt === undefined || availability.timezone === undefined
|
|
1549
|
-
? `${availability.mode}; next transition could not be calculated`
|
|
1550
|
-
: `${availability.mode} until ${formatZonedMinute(availability.nextTransitionAt, availability.timezone)}`;
|
|
1551
|
-
const bypass = availability.bypass.length === 0 ? "none" : availability.bypass.join(", ");
|
|
1552
|
-
return [`availability ${mode}; quiet-hours bypass ${bypass}`];
|
|
1553
|
-
}
|
|
1554
|
-
|
|
1555
|
-
function formatDigestScheduleStatus(s: StatusSnapshot): string[] {
|
|
1556
|
-
const schedule = s.digestSchedule;
|
|
1557
|
-
if (schedule === undefined) return [];
|
|
1558
|
-
if (schedule.mode === "disabled") return ["next digest disabled"];
|
|
1559
|
-
if (schedule.mode === "per-tick") return ["next digest every tick"];
|
|
1560
|
-
if (schedule.mode === "due") return ["next digest due now"];
|
|
1561
|
-
return [
|
|
1562
|
-
schedule.nextAt === undefined
|
|
1563
|
-
? "next digest could not be calculated"
|
|
1564
|
-
: `next digest ${formatZonedMinute(schedule.nextAt, schedule.timezone)}`,
|
|
1565
|
-
];
|
|
1566
|
-
}
|
|
1567
|
-
|
|
1568
|
-
function formatProjectBody(
|
|
1569
|
-
s: StatusSnapshot,
|
|
1570
|
-
workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
|
|
1571
|
-
now = Date.now(),
|
|
1572
|
-
): string {
|
|
1573
|
-
const lines = [
|
|
1574
|
-
`project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
|
|
1575
|
-
...(s.pauseReason === undefined ? [] : [`paused ${s.pauseReason}`]),
|
|
1576
|
-
`config ${s.configPath}`,
|
|
1577
|
-
`state ${s.stateDir}`,
|
|
1578
|
-
// The orchestrator-down degrade row: first-class in the body, present only
|
|
1579
|
-
// while the incident is open, so recovery drops it (#288).
|
|
1580
|
-
...(s.orchestratorDown === undefined
|
|
1581
|
-
? []
|
|
1582
|
-
: formatOrchestratorDown(s.orchestratorDown, now)),
|
|
1583
|
-
...formatAvailabilityStatus(s),
|
|
1584
|
-
...formatDigestScheduleStatus(s),
|
|
1585
|
-
"",
|
|
1586
|
-
"caps",
|
|
1587
|
-
` workers ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
|
|
1588
|
-
` issues today ${s.runsToday}`,
|
|
1589
|
-
s.caps.dailySpendUsd === null
|
|
1590
|
-
? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
|
|
1591
|
-
: ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
|
|
1592
|
-
// Its own row beside the spend row, never folded into it: they are two
|
|
1593
|
-
// independent controls and an operator has to see which one stopped the
|
|
1594
|
-
// fleet (#110).
|
|
1595
|
-
` plan usage ${planUsageLine(s.planUsage)}`,
|
|
1596
|
-
// The tracker's API budget, when the renderer could read it. Absent on a
|
|
1597
|
-
// broken `gh`: one missing row, never a broken report (#188). Next to it,
|
|
1598
|
-
// the refusals the tracker actually observed, so a budget that looks
|
|
1599
|
-
// healthy next to every call being refused is still visible (#198).
|
|
1600
|
-
...(s.github === undefined
|
|
1601
|
-
? []
|
|
1602
|
-
: [
|
|
1603
|
-
` github graphql ${s.github.graphql.remaining}/${s.github.graphql.limit}, core ${s.github.core.remaining}/${s.github.core.limit} (graphql resets ${Math.max(0, Math.round((s.github.graphql.reset * 1000 - Date.now()) / 60_000))}m)` +
|
|
1604
|
-
(s.ghRefusals === undefined || s.ghRefusals.count === 0
|
|
1605
|
-
? ""
|
|
1606
|
-
: ` — ${s.ghRefusals.count} refusal(s) in last 5m (last ${new Date(s.ghRefusals.latestAt ?? Date.now()).toISOString().slice(11, 19)}Z)`),
|
|
1607
|
-
]),
|
|
1608
|
-
// The daemon's own observed github traffic today (#198): a separate row
|
|
1609
|
-
// from the polled budget above, so one missing `status` read never hides
|
|
1610
|
-
// the other. `onCall` counts every spawn including ones that end 304, so
|
|
1611
|
-
// `daemon` is spawns and `daemon-304` is the unbilled subset — billed ≈
|
|
1612
|
-
// difference, and the polled budget row stays the authority (#203).
|
|
1613
|
-
` github calls daemon ${s.ghCallsToday?.find((c) => c.source === "daemon")?.calls ?? 0} today` +
|
|
1614
|
-
((s.ghCallsToday?.find((c) => c.source === "daemon-304")?.calls ?? 0) === 0
|
|
1615
|
-
? ""
|
|
1616
|
-
: ` (${s.ghCallsToday!.find((c) => c.source === "daemon-304")!.calls} free 304s)`),
|
|
1617
|
-
// Label-projection ops the tracker has not applied yet (#201): GitHub is
|
|
1618
|
-
// behind what the store decided, and the operator can see the lag instead
|
|
1619
|
-
// of discovering it as a stale label or a missing one.
|
|
1620
|
-
...(s.labelOps === undefined
|
|
1621
|
-
? []
|
|
1622
|
-
: [
|
|
1623
|
-
` labels projection ${s.labelOps.pending} pending (oldest ${
|
|
1624
|
-
s.labelOps.oldestAgeMs >= 60_000
|
|
1625
|
-
? `${Math.round(s.labelOps.oldestAgeMs / 60_000)}m`
|
|
1626
|
-
: `${Math.max(1, Math.round(s.labelOps.oldestAgeMs / 1000))}s`
|
|
1627
|
-
})`,
|
|
1628
|
-
]),
|
|
1629
|
-
` new worker turns ${s.caps.workerMaxTurns}`,
|
|
1630
|
-
...s.turnOverrides.map(
|
|
1631
|
-
({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`,
|
|
1632
|
-
),
|
|
1633
|
-
` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
|
|
1634
|
-
` failed attempts ${s.caps.maxAttemptsPerIssue}`,
|
|
1635
|
-
` continuations ${s.caps.maxContinuationsPerIssue}`,
|
|
1636
|
-
"",
|
|
1637
|
-
...formatReleaseGrants(s.releaseGrants),
|
|
1638
|
-
"",
|
|
1639
|
-
formatDispatchSummary(s.dispatch),
|
|
1640
|
-
"",
|
|
1641
|
-
];
|
|
1642
|
-
if (s.activeRuns.length === 0) {
|
|
1643
|
-
lines.push("active runs (none)");
|
|
1644
|
-
} else {
|
|
1645
|
-
lines.push("active runs");
|
|
1646
|
-
for (const r of s.activeRuns) {
|
|
1647
|
-
const phase = workerPhases.get(r.issue);
|
|
1648
|
-
const state = phase === "pausing" || phase === "paused" ? phase : r.state;
|
|
1649
|
-
lines.push(
|
|
1650
|
-
` #${r.issue} ${r.repo} ${state} attempt ${r.attempt} ` +
|
|
1651
|
-
`${r.turns}/${r.maxTurns} turns ${r.spendUsd.toFixed(2)} ${r.branch}` +
|
|
1652
|
-
(r.prUrl ? ` ${r.prUrl}` : ""),
|
|
1653
|
-
);
|
|
1654
|
-
// The orchestrator's Duty 1 reads this command, and a flagged run's
|
|
1655
|
-
// escalation is deduplicated after one delivery — so this is where a
|
|
1656
|
-
// flagged PR stays visible for as long as it is still open (#128).
|
|
1657
|
-
const flagged = settlementFlagSummary(r.settlementFlags);
|
|
1658
|
-
if (flagged !== undefined) lines.push(` ${flagged}`);
|
|
1659
|
-
}
|
|
1660
|
-
}
|
|
1661
|
-
lines.push(...formatBaseHealth(s.baseHealth));
|
|
1662
|
-
lines.push(...formatSalvagedRuns(s.salvagedRuns));
|
|
1663
|
-
lines.push(...formatOpenReports(s.openReports));
|
|
1664
|
-
lines.push(...formatDigestBacklog(s.digestBacklog));
|
|
1665
|
-
if (s.liveWorkers > 0) {
|
|
1666
|
-
lines.push(
|
|
1667
|
-
"",
|
|
1668
|
-
`deploy ${s.liveWorkers} live worker(s) — restart salvages dirty trees then orphans the rows; ` +
|
|
1669
|
-
`hold and wait for workers 0/${s.caps.maxConcurrentWorkers} when you can drain instead`,
|
|
1670
|
-
);
|
|
1671
|
-
}
|
|
1672
|
-
return lines.join("\n");
|
|
1673
|
-
}
|
|
1674
|
-
|
|
1675
1284
|
export async function renderStatus(projectName?: string): Promise<string> {
|
|
1676
1285
|
const s = statusSnapshot(projectName);
|
|
1677
1286
|
const layers = fleetLayers(projectName);
|
|
@@ -1716,8 +1325,16 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
1716
1325
|
// project and the uninvolved one must see who stopped it too.
|
|
1717
1326
|
const store = openStore(dbPath());
|
|
1718
1327
|
let lastStop: DaemonStop | undefined;
|
|
1328
|
+
let siblings: { project: string; live: number }[] = [];
|
|
1719
1329
|
try {
|
|
1720
1330
|
lastStop = store.latestDaemonStop();
|
|
1331
|
+
// Shared-daemon visibility (#545): every configured project other than the
|
|
1332
|
+
// one being viewed, with its live-run count. Read from the same open store
|
|
1333
|
+
// as the provenance above, so a status reader can tell "my fleet is idle"
|
|
1334
|
+
// from "the process I am about to stop is busy".
|
|
1335
|
+
siblings = loadConfig()
|
|
1336
|
+
.projects.filter((p) => p.name !== project.name)
|
|
1337
|
+
.map((p) => ({ project: p.name, live: store.liveRuns(p.name).length }));
|
|
1721
1338
|
} finally {
|
|
1722
1339
|
store.close();
|
|
1723
1340
|
}
|
|
@@ -1734,6 +1351,7 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
1734
1351
|
workerPhases,
|
|
1735
1352
|
intakeStatusLine(project.name),
|
|
1736
1353
|
lastStop,
|
|
1354
|
+
siblings,
|
|
1737
1355
|
);
|
|
1738
1356
|
}
|
|
1739
1357
|
|
|
@@ -1793,18 +1411,32 @@ function failureClassBlock(projectName: string): string | undefined {
|
|
|
1793
1411
|
* the ambiguity this ledger exists to remove. Omitted on any throw — a status
|
|
1794
1412
|
* that fails to render because the store is mid-migration is worse than one
|
|
1795
1413
|
* missing a row.
|
|
1414
|
+
*
|
|
1415
|
+
* `decisions N open` counts only rows a human must answer (#459). Watches the
|
|
1416
|
+
* orchestrator set for itself are reported separately — a fleet with three
|
|
1417
|
+
* watches behind GitHub's checks is not a fleet waiting on its operator, and
|
|
1418
|
+
* lumping them in made `decisions 3 open` read as three unanswered questions.
|
|
1796
1419
|
*/
|
|
1797
|
-
function decisionStatusLine(projectName: string): string | undefined {
|
|
1420
|
+
export function decisionStatusLine(projectName: string): string | undefined {
|
|
1798
1421
|
const path = dbPath();
|
|
1799
1422
|
if (!existsSync(path)) return undefined;
|
|
1800
1423
|
let store: Store | undefined;
|
|
1801
1424
|
try {
|
|
1802
1425
|
store = openStore(path);
|
|
1803
1426
|
const open = store.openDecisions(projectName);
|
|
1804
|
-
|
|
1805
|
-
const
|
|
1806
|
-
|
|
1807
|
-
|
|
1427
|
+
const questions = open.filter((d) => d.kind !== "watch");
|
|
1428
|
+
const watches = open.filter((d) => d.kind === "watch");
|
|
1429
|
+
if (questions.length === 0 && watches.length === 0) return "decisions none open";
|
|
1430
|
+
let line: string;
|
|
1431
|
+
if (questions.length === 0) {
|
|
1432
|
+
line = "decisions none open";
|
|
1433
|
+
} else {
|
|
1434
|
+
const oldest = questions[0]!;
|
|
1435
|
+
const hours = Math.max(0, Math.round((Date.now() - oldest.askedAt) / 3_600_000));
|
|
1436
|
+
line = `decisions ${questions.length} open (oldest ${hours}h)`;
|
|
1437
|
+
}
|
|
1438
|
+
if (watches.length > 0) line += ` · watches ${watches.length}`;
|
|
1439
|
+
return line;
|
|
1808
1440
|
} catch {
|
|
1809
1441
|
return undefined;
|
|
1810
1442
|
} finally {
|
|
@@ -2167,4 +1799,3 @@ function probeOmpPane(
|
|
|
2167
1799
|
return { kind: "unknown", reason: err instanceof Error ? err.message : String(err) };
|
|
2168
1800
|
}
|
|
2169
1801
|
}
|
|
2170
|
-
|
package/src/gitops.ts
CHANGED
|
@@ -96,10 +96,95 @@ export function scrubUserinfo(text: string): string {
|
|
|
96
96
|
return text.replace(URL_USERINFO, "$1***@");
|
|
97
97
|
}
|
|
98
98
|
|
|
99
|
-
function mirrorPath(project: Pick<ProjectConfig, "mirrorRoot">, repo: RepoTarget): string {
|
|
99
|
+
export function mirrorPath(project: Pick<ProjectConfig, "mirrorRoot">, repo: RepoTarget): string {
|
|
100
100
|
return join(project.mirrorRoot, `${repo.name}.git`);
|
|
101
101
|
}
|
|
102
102
|
|
|
103
|
+
/**
|
|
104
|
+
* Everything the file-lane interlock (#555) needs to read one active run's lane.
|
|
105
|
+
*
|
|
106
|
+
* The dispatcher may read worktrees — this is squarely its job, and the issue
|
|
107
|
+
* is explicit that the orchestrator must not be the one guessing file lanes
|
|
108
|
+
* from another run's tree. A live run advertises its lane from its own worktree
|
|
109
|
+
* (`git status --porcelain` plus `git diff --name-only <base>...HEAD`, exactly
|
|
110
|
+
* the two commands the continuation brief already hands every worker), which is
|
|
111
|
+
* the truth rather than the run's issue body — the incident this interlock
|
|
112
|
+
* exists for was an issue *saying* one set of files while its tree wrote
|
|
113
|
+
* another. A worker-finished run (`pushed-pending` / `pushed-green`) no longer
|
|
114
|
+
* has a worktree, but its open PR still reaches for base's files, so its lane
|
|
115
|
+
* is read from the retained branch in the mirror instead.
|
|
116
|
+
*/
|
|
117
|
+
export interface RunLaneInput {
|
|
118
|
+
/** Conductor-created worktree path; `""` when the run has no live checkout. */
|
|
119
|
+
worktree: string;
|
|
120
|
+
/** Tracking ref of the base branch to diff against, e.g. `refs/remotes/origin/main`. */
|
|
121
|
+
baseRef: string;
|
|
122
|
+
/** The run's branch ref in the mirror (e.g. `refs/heads/feat-x`), for a
|
|
123
|
+
* worker-finished run whose worktree has been removed. */
|
|
124
|
+
branchRef?: string;
|
|
125
|
+
/** The repo's bare mirror directory, for the branch-diff read. */
|
|
126
|
+
mirror?: string;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* The set of file paths one active run has touched relative to base, deduped
|
|
131
|
+
* and sorted. Persistable but advisory: the admitting gate treats an unreadable
|
|
132
|
+
* or absent lane as "no overlap" (fail open), so no probe bug can refuse a
|
|
133
|
+
* well-formed issue.
|
|
134
|
+
*/
|
|
135
|
+
export type RunLaneProbe = (input: RunLaneInput) => Promise<string[]>;
|
|
136
|
+
|
|
137
|
+
/** The real one, reading the run's own worktree and/or the mirror branch. */
|
|
138
|
+
export async function probeRunLane(
|
|
139
|
+
input: RunLaneInput,
|
|
140
|
+
exec: Exec = spawnCaptured,
|
|
141
|
+
): Promise<string[]> {
|
|
142
|
+
const files = new Set<string>();
|
|
143
|
+
const add = (raw: string): void => {
|
|
144
|
+
for (const line of raw.split("\n")) {
|
|
145
|
+
const path = line.trim();
|
|
146
|
+
if (path !== "") files.add(path);
|
|
147
|
+
}
|
|
148
|
+
};
|
|
149
|
+
if (input.worktree !== "") {
|
|
150
|
+
const status = await exec(["git", "-C", input.worktree, "status", "--porcelain"], {});
|
|
151
|
+
if (status.code === 0) {
|
|
152
|
+
for (const path of parsePorcelain(status.stdout)) files.add(path);
|
|
153
|
+
}
|
|
154
|
+
const diff = await exec(
|
|
155
|
+
["git", "-C", input.worktree, "diff", "--name-only", `${input.baseRef}...HEAD`],
|
|
156
|
+
{},
|
|
157
|
+
);
|
|
158
|
+
if (diff.code === 0) add(diff.stdout);
|
|
159
|
+
}
|
|
160
|
+
if (input.branchRef !== undefined && input.mirror !== undefined && input.mirror !== "") {
|
|
161
|
+
const diff = await exec(
|
|
162
|
+
["git", "--git-dir", input.mirror, "diff", "--name-only", `${input.baseRef}...${input.branchRef}`],
|
|
163
|
+
{},
|
|
164
|
+
);
|
|
165
|
+
if (diff.code === 0) add(diff.stdout);
|
|
166
|
+
}
|
|
167
|
+
return [...files].sort();
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Parse `git status --porcelain` output into the paths it names. A rename or
|
|
172
|
+
* copy is reported as `XY old -> new`, and the *new* path is the one that
|
|
173
|
+
* occupies the lane (it is the path the other run's future writes would
|
|
174
|
+
* collide with), so the arrow form keeps only its target.
|
|
175
|
+
*/
|
|
176
|
+
function parsePorcelain(stdout: string): string[] {
|
|
177
|
+
const out = new Set<string>();
|
|
178
|
+
for (const raw of stdout.split("\n")) {
|
|
179
|
+
if (raw.length < 3) continue;
|
|
180
|
+
let rest = raw.slice(2).trim();
|
|
181
|
+
const arrow = rest.indexOf(" -> ");
|
|
182
|
+
if (arrow !== -1) rest = rest.slice(arrow + 4).trim();
|
|
183
|
+
if (rest !== "") out.add(rest);
|
|
184
|
+
}
|
|
185
|
+
return [...out];
|
|
186
|
+
}
|
|
187
|
+
|
|
103
188
|
/** GitHub slug parsed off the clone URL, falling back to the routing name. */
|
|
104
189
|
export function repoSlugFor(repo: RepoTarget): string {
|
|
105
190
|
const m = /(?:[:/])([^/:]+\/[^/]+?)(?:\.git)?$/.exec(repo.cloneUrl);
|
package/src/log.ts
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared daemon logging and best-effort escalation delivery.
|
|
3
|
+
*
|
|
4
|
+
* Split out of `daemon.ts` so the extracted admission module (and any future
|
|
5
|
+
* extracted concern) can log and escalate without importing the composition
|
|
6
|
+
* root back — which would be the circular-import failure mode the daemon split
|
|
7
|
+
* (#571) exists to rule out. `daemon.ts` composes; the leaf modules talk only
|
|
8
|
+
* to other leaves.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { escalationIssueRef } from "./escalate.ts";
|
|
12
|
+
import type { Escalation } from "./types.ts";
|
|
13
|
+
|
|
14
|
+
/** One timestamped line to stderr — the daemon's canonical log sink. */
|
|
15
|
+
export function log(msg: string): void {
|
|
16
|
+
process.stderr.write(`[conductor ${new Date().toISOString()}] ${msg}\n`);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** A stable one-line rendering of an unknown thrown value. */
|
|
20
|
+
export function errText(e: unknown): string {
|
|
21
|
+
return e instanceof Error ? (e.stack ?? e.message) : String(e);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Best-effort escalation delivery: never let a failed page take down the
|
|
26
|
+
* pass, and say in the log that it failed. Structural `escalate` only — the
|
|
27
|
+
* caller's full dep object satisfies it without importing anything back.
|
|
28
|
+
*/
|
|
29
|
+
export async function safeEscalate(
|
|
30
|
+
d: { escalate(e: Escalation): Promise<void> },
|
|
31
|
+
e: Escalation,
|
|
32
|
+
): Promise<boolean> {
|
|
33
|
+
try {
|
|
34
|
+
await d.escalate(e);
|
|
35
|
+
return true;
|
|
36
|
+
} catch (err) {
|
|
37
|
+
log(`escalation for ${escalationIssueRef(e.issue)} could not be delivered: ${errText(err)}`);
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
}
|