omp-conductor 0.15.12 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +81 -6
- package/package.json +2 -1
- package/schema/config.schema.json +6 -0
- package/src/admission.ts +745 -0
- package/src/ask.ts +47 -0
- package/src/backups.ts +19 -7
- package/src/board.ts +1 -2
- package/src/briefs/orchestrator.md +62 -4
- package/src/cli.ts +26 -0
- package/src/commands/context.ts +3 -0
- package/src/commands/decision.ts +10 -1
- package/src/commands/doctor.ts +2 -0
- package/src/commands/message.ts +8 -1
- package/src/commands/restart.ts +93 -54
- package/src/commands/restore-db.ts +146 -0
- package/src/commands/stop.ts +66 -34
- package/src/commands/unfreeze.ts +56 -0
- package/src/commands/watch.ts +77 -0
- package/src/config-schema.ts +9 -0
- package/src/config.ts +24 -0
- package/src/daemon.ts +485 -577
- package/src/dashboard/server.ts +2 -1
- package/src/decisions.ts +32 -7
- package/src/depends-on.ts +73 -0
- package/src/doctor.ts +418 -8
- package/src/escalate.ts +122 -15
- package/src/failure-class.ts +47 -0
- package/src/fleet.ts +55 -377
- package/src/gitops.ts +86 -1
- package/src/lifecycle.ts +113 -2
- package/src/log.ts +40 -0
- package/src/model-fallback.ts +3 -2
- package/src/omp-settings.ts +114 -0
- package/src/omp.ts +63 -0
- package/src/orchestrator-down.ts +231 -0
- package/src/orchestrator-tick.ts +14 -1
- package/src/orchestrator.ts +14 -0
- package/src/release-policy.ts +163 -18
- package/src/reports.ts +124 -12
- package/src/session-host.ts +6 -0
- package/src/setup-host.ts +386 -17
- package/src/setup-install.ts +40 -2
- package/src/setup-wizard.ts +314 -113
- package/src/setup.ts +58 -1
- package/src/status-render.ts +445 -0
- package/src/stop-provenance.ts +119 -0
- package/src/store.ts +533 -11
- package/src/types.ts +298 -4
- package/src/unblock.ts +1 -1
- package/src/upgrade-verify.ts +1 -1
- package/src/upgrade.ts +27 -8
- package/src/verbs/protocol.ts +16 -3
- package/src/verbs/server.ts +52 -1
- package/src/wizard-ui.ts +261 -46
- package/src/worker.ts +183 -10
package/src/fleet.ts
CHANGED
|
@@ -27,31 +27,17 @@ import {
|
|
|
27
27
|
import { createInterface } from "node:readline";
|
|
28
28
|
import { homedir } from "node:os";
|
|
29
29
|
import { dirname, join } from "node:path";
|
|
30
|
-
import { formatZonedMinute } from "./availability.ts";
|
|
31
30
|
import { findProject, loadConfig, stateDir } from "./config.ts";
|
|
32
31
|
import { clearArmChallenge, recordArmChallenge } from "./arm-challenge.ts";
|
|
33
32
|
import { resolveProjectTopicId, sendTelegram } from "./escalate.ts";
|
|
34
|
-
import {
|
|
33
|
+
import { readPlanUsage, sharedUsageSource } from "./usage.ts";
|
|
35
34
|
import { readApprovalSurface, readDaemonProfile } from "./approval-surface.ts";
|
|
36
35
|
import { inspectBriefLayout } from "./brief-upgrade.ts";
|
|
37
36
|
import { dbPath, openStore } from "./store.ts";
|
|
38
37
|
import { renderBriefForProject } from "./setup.ts";
|
|
39
|
-
import type { ProjectConfig, Store } from "./types.ts";
|
|
40
|
-
import { settlementFlagSummary } from "./diff-flags.ts";
|
|
38
|
+
import type { DaemonStop, ProjectConfig, Store } from "./types.ts";
|
|
41
39
|
import { probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
42
|
-
import {
|
|
43
|
-
import {
|
|
44
|
-
formatBaseHealth,
|
|
45
|
-
formatDispatchSummary,
|
|
46
|
-
formatReleaseGrants,
|
|
47
|
-
formatSalvagedRuns,
|
|
48
|
-
isPaused,
|
|
49
|
-
pauseProvenance,
|
|
50
|
-
pausedAt,
|
|
51
|
-
setPaused,
|
|
52
|
-
statusSnapshot,
|
|
53
|
-
type StatusSnapshot,
|
|
54
|
-
} from "./daemon.ts";
|
|
40
|
+
import { isPaused, setPaused, statusSnapshot } from "./daemon.ts";
|
|
55
41
|
import {
|
|
56
42
|
healthCheck,
|
|
57
43
|
isAlive,
|
|
@@ -60,11 +46,19 @@ import {
|
|
|
60
46
|
runSystemctl,
|
|
61
47
|
stopDaemon,
|
|
62
48
|
type StopResult,
|
|
63
|
-
type UnitOwnership,
|
|
64
|
-
SYSTEMD_UNIT,
|
|
65
49
|
} from "./lifecycle.ts";
|
|
66
|
-
import { formatRss, rssBytesFromHealthz } from "./host.ts";
|
|
67
50
|
import type { WorkerPausePhase } from "./worker.ts";
|
|
51
|
+
import {
|
|
52
|
+
formatFleetStatus,
|
|
53
|
+
type DaemonProjectHealth,
|
|
54
|
+
type DispatchLayer,
|
|
55
|
+
type FleetDaemonProbe,
|
|
56
|
+
type FleetLayers,
|
|
57
|
+
type HerdrLayer,
|
|
58
|
+
type PaneLayer,
|
|
59
|
+
type TelegramHealth,
|
|
60
|
+
type TicksLayer,
|
|
61
|
+
} from "./status-render.ts";
|
|
68
62
|
import { fetchRateLimit } from "./tracker/github.ts";
|
|
69
63
|
import {
|
|
70
64
|
LEGACY_ARM_MARKER_DETAIL,
|
|
@@ -1022,46 +1016,6 @@ function isOmpProcess(proc: ForegroundProc): boolean {
|
|
|
1022
1016
|
return false;
|
|
1023
1017
|
}
|
|
1024
1018
|
|
|
1025
|
-
// ---------------------------------------------------------------------------
|
|
1026
|
-
// layered status
|
|
1027
|
-
// ---------------------------------------------------------------------------
|
|
1028
|
-
|
|
1029
|
-
export type DispatchLayer = "running" | "paused" | "stopped";
|
|
1030
|
-
export type TicksLayer =
|
|
1031
|
-
| "armed"
|
|
1032
|
-
| "disarmed"
|
|
1033
|
-
| "no-heartbeat-config"
|
|
1034
|
-
| "invalid-heartbeat-config"
|
|
1035
|
-
| "ungated";
|
|
1036
|
-
export type PaneLayer = "live" | "missing" | "unknown";
|
|
1037
|
-
/** `unpinnable`: no tick config, so FLEET_CWD — the only path recovery reads — is unknown. */
|
|
1038
|
-
export type RecoveryLayer = "pinned" | "clear" | "unpinnable";
|
|
1039
|
-
export type HerdrLayer = "active" | "inactive" | "unknown";
|
|
1040
|
-
export type TelegramLayer = "ok" | "degraded" | "down" | "unconfigured" | "unprobed";
|
|
1041
|
-
|
|
1042
|
-
export interface TelegramHealth {
|
|
1043
|
-
kind: TelegramLayer;
|
|
1044
|
-
detail?: string;
|
|
1045
|
-
}
|
|
1046
|
-
|
|
1047
|
-
export interface FleetLayers {
|
|
1048
|
-
dispatch: DispatchLayer;
|
|
1049
|
-
ticks: TicksLayer;
|
|
1050
|
-
ticksDetail?: string;
|
|
1051
|
-
nextTickAt?: string;
|
|
1052
|
-
pane: PaneLayer;
|
|
1053
|
-
paneDetail?: string;
|
|
1054
|
-
recovery: RecoveryLayer;
|
|
1055
|
-
recoveryDetail?: string;
|
|
1056
|
-
herdr: HerdrLayer;
|
|
1057
|
-
herdrDetail?: string;
|
|
1058
|
-
armedPath?: string;
|
|
1059
|
-
tickConfigPath?: string;
|
|
1060
|
-
paneHaltPath?: string;
|
|
1061
|
-
paused: boolean;
|
|
1062
|
-
daemon: { running: boolean; pid?: number; port?: number };
|
|
1063
|
-
}
|
|
1064
|
-
|
|
1065
1019
|
/**
|
|
1066
1020
|
* The project a bare read means, when the config leaves no doubt.
|
|
1067
1021
|
*
|
|
@@ -1269,50 +1223,6 @@ export function workerPhasesFromHealthz(
|
|
|
1269
1223
|
}
|
|
1270
1224
|
|
|
1271
1225
|
|
|
1272
|
-
export function formatCodeGraphHealth(graph: CodeGraphHealth, now = Date.now()): string | undefined {
|
|
1273
|
-
if (!graph.configured) return undefined;
|
|
1274
|
-
const indexed = graph.repos.filter((repo) => repo.index === "present").length;
|
|
1275
|
-
let refresh: string = graph.refresh.result;
|
|
1276
|
-
if (graph.refresh.lastSuccessAt !== undefined) {
|
|
1277
|
-
const ageMs = Math.max(0, now - Date.parse(graph.refresh.lastSuccessAt));
|
|
1278
|
-
refresh = `${graph.refresh.lastSuccessAt} (${Math.max(1, Math.ceil(ageMs / 60_000))}m ago)`;
|
|
1279
|
-
}
|
|
1280
|
-
return [
|
|
1281
|
-
`code graph ${graph.status} ${indexed}/${graph.repos.length} repos indexed`,
|
|
1282
|
-
` indexer ${graph.prerequisites.indexer}`,
|
|
1283
|
-
` MCP mount ${graph.prerequisites.mcpMount}`,
|
|
1284
|
-
` timer ${graph.timer.enabled} / ${graph.timer.active}`,
|
|
1285
|
-
` refresh ${refresh}`,
|
|
1286
|
-
...graph.reasons.map((reason) => ` - ${reason}`),
|
|
1287
|
-
].join("\n");
|
|
1288
|
-
}
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
/**
|
|
1292
|
-
* Project-scoped reading of a living daemon record + `/healthz` body.
|
|
1293
|
-
*
|
|
1294
|
-
* Board and `status` share this so a host running one daemon for several
|
|
1295
|
-
* projects cannot be called healthy for a project it does not serve (#379).
|
|
1296
|
-
* A third interpretation convention is forbidden.
|
|
1297
|
-
*/
|
|
1298
|
-
export type DaemonProjectHealth =
|
|
1299
|
-
| { kind: "stopped" }
|
|
1300
|
-
| { kind: "ok" }
|
|
1301
|
-
| { kind: "unreachable" }
|
|
1302
|
-
| { kind: "other-project"; serves?: string };
|
|
1303
|
-
|
|
1304
|
-
/**
|
|
1305
|
-
* Facts `formatFleetStatus` needs about the living daemon. Pure input — the
|
|
1306
|
-
* caller probes; the formatter never shells out (#379).
|
|
1307
|
-
*/
|
|
1308
|
-
export type FleetDaemonProbe = {
|
|
1309
|
-
project: DaemonProjectHealth;
|
|
1310
|
-
/** `/healthz` body when the probe is project-ok (rss / overlays). */
|
|
1311
|
-
body?: string;
|
|
1312
|
-
/** systemd ownership of the living record pid; omit when unprobed. */
|
|
1313
|
-
unit?: UnitOwnership;
|
|
1314
|
-
};
|
|
1315
|
-
|
|
1316
1226
|
/**
|
|
1317
1227
|
* Decide whether a living record + optional `/healthz` answer serve `project`.
|
|
1318
1228
|
*
|
|
@@ -1371,273 +1281,6 @@ function servedProjectName(payload: object): string | undefined {
|
|
|
1371
1281
|
return undefined;
|
|
1372
1282
|
}
|
|
1373
1283
|
|
|
1374
|
-
function formatDaemonHealthz(probe: FleetDaemonProbe | undefined): string {
|
|
1375
|
-
if (probe === undefined) return "unprobed";
|
|
1376
|
-
switch (probe.project.kind) {
|
|
1377
|
-
case "stopped":
|
|
1378
|
-
return "stopped";
|
|
1379
|
-
case "ok":
|
|
1380
|
-
return "ok";
|
|
1381
|
-
case "unreachable":
|
|
1382
|
-
return "unreachable — the process is up but not serving";
|
|
1383
|
-
case "other-project":
|
|
1384
|
-
return probe.project.serves === undefined
|
|
1385
|
-
? "other-project"
|
|
1386
|
-
: `other-project — serves ${probe.project.serves}`;
|
|
1387
|
-
}
|
|
1388
|
-
}
|
|
1389
|
-
|
|
1390
|
-
/**
|
|
1391
|
-
* Unit line for a living record. The bare unit name alone used to imply
|
|
1392
|
-
* systemd ownership that was never checked (#379); every branch states the
|
|
1393
|
-
* probe result, and `unknown` is never rendered as owned.
|
|
1394
|
-
*/
|
|
1395
|
-
function formatDaemonUnit(unit: UnitOwnership | undefined, recordPid: number): string {
|
|
1396
|
-
if (unit === undefined) return ` unit ${SYSTEMD_UNIT} unprobed`;
|
|
1397
|
-
switch (unit.kind) {
|
|
1398
|
-
case "active":
|
|
1399
|
-
if (unit.pid === recordPid) {
|
|
1400
|
-
return ` unit ${SYSTEMD_UNIT} systemd-owned`;
|
|
1401
|
-
}
|
|
1402
|
-
// Live unit, different MainPID — the record pid is not systemd's.
|
|
1403
|
-
return ` unit ${SYSTEMD_UNIT} unmanaged — MainPID ${unit.pid}`;
|
|
1404
|
-
case "failed":
|
|
1405
|
-
return ` unit ${SYSTEMD_UNIT} unmanaged — unit failed`;
|
|
1406
|
-
case "inactive":
|
|
1407
|
-
return ` unit ${SYSTEMD_UNIT} inactive`;
|
|
1408
|
-
case "unknown":
|
|
1409
|
-
return ` unit ${SYSTEMD_UNIT} unknown — ${unit.reason}`;
|
|
1410
|
-
}
|
|
1411
|
-
}
|
|
1412
|
-
|
|
1413
|
-
export function formatFleetStatus(
|
|
1414
|
-
s: StatusSnapshot,
|
|
1415
|
-
layers: FleetLayers,
|
|
1416
|
-
daemon: FleetDaemonProbe | undefined = undefined,
|
|
1417
|
-
telegram: TelegramHealth = { kind: "unprobed" },
|
|
1418
|
-
now = Date.now(),
|
|
1419
|
-
codeGraph: CodeGraphHealth = { configured: false },
|
|
1420
|
-
brief: string | undefined = undefined,
|
|
1421
|
-
decisions: string | undefined = undefined,
|
|
1422
|
-
failureClasses: string | undefined = undefined,
|
|
1423
|
-
workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
|
|
1424
|
-
intake: string | undefined = undefined,
|
|
1425
|
-
): string {
|
|
1426
|
-
const tickLine =
|
|
1427
|
-
layers.ticksDetail === undefined
|
|
1428
|
-
? `ticks ${layers.ticks}`
|
|
1429
|
-
: `ticks ${layers.ticks} (${layers.ticksDetail})`;
|
|
1430
|
-
let nextTickLine: string | undefined;
|
|
1431
|
-
if (layers.nextTickAt !== undefined) {
|
|
1432
|
-
const delta = Date.parse(layers.nextTickAt) - now;
|
|
1433
|
-
const minutes = Math.max(1, Math.ceil(Math.abs(delta) / 60_000));
|
|
1434
|
-
nextTickLine =
|
|
1435
|
-
`next tick ${layers.nextTickAt} ` +
|
|
1436
|
-
`(${delta >= 0 ? `in ${minutes}m` : `overdue by ${minutes}m`})`;
|
|
1437
|
-
}
|
|
1438
|
-
const paneLine =
|
|
1439
|
-
layers.paneDetail === undefined
|
|
1440
|
-
? `pane ${layers.pane}`
|
|
1441
|
-
: `pane ${layers.pane} (${layers.paneDetail})`;
|
|
1442
|
-
const recoveryLine =
|
|
1443
|
-
layers.recoveryDetail === undefined
|
|
1444
|
-
? `recovery ${layers.recovery}`
|
|
1445
|
-
: `recovery ${layers.recovery} (${layers.recoveryDetail})`;
|
|
1446
|
-
const herdrLine =
|
|
1447
|
-
layers.herdrDetail === undefined
|
|
1448
|
-
? `herdr ${layers.herdr}`
|
|
1449
|
-
: `herdr ${layers.herdr} (${layers.herdrDetail})`;
|
|
1450
|
-
const telegramLine =
|
|
1451
|
-
telegram.detail === undefined
|
|
1452
|
-
? `telegram ${telegram.kind}`
|
|
1453
|
-
: `telegram ${telegram.kind} (${telegram.detail})`;
|
|
1454
|
-
|
|
1455
|
-
let daemonBlock: string;
|
|
1456
|
-
if (!layers.daemon.running || layers.daemon.pid === undefined) {
|
|
1457
|
-
daemonBlock = "daemon not running";
|
|
1458
|
-
} else {
|
|
1459
|
-
// Only trust rss from a project-ok body — a foreign payload's bytes are
|
|
1460
|
-
// not this project's daemon facts (#379).
|
|
1461
|
-
const rss =
|
|
1462
|
-
daemon?.project.kind === "ok" ? rssBytesFromHealthz(daemon.body) : undefined;
|
|
1463
|
-
daemonBlock = [
|
|
1464
|
-
"daemon",
|
|
1465
|
-
` pid ${layers.daemon.pid}`,
|
|
1466
|
-
` port ${layers.daemon.port ?? "?"}`,
|
|
1467
|
-
...(rss === undefined ? [] : [` rss ${formatRss(rss)}`]),
|
|
1468
|
-
` healthz ${formatDaemonHealthz(daemon)}`,
|
|
1469
|
-
formatDaemonUnit(daemon?.unit, layers.daemon.pid),
|
|
1470
|
-
].join("\n");
|
|
1471
|
-
}
|
|
1472
|
-
|
|
1473
|
-
const graphBlock = formatCodeGraphHealth(codeGraph, now);
|
|
1474
|
-
|
|
1475
|
-
// Pause provenance (`pause --reason`, an integrity/spend-cap/upgrade pause,
|
|
1476
|
-
// a halt) answers "who stopped the fleet" without opening a file (#185). An
|
|
1477
|
-
// unparseable sentinel — paused but with no datable line 1 — is itself news:
|
|
1478
|
-
// it means a run admitted before an *unknown* pause cannot prove innocence
|
|
1479
|
-
// (#174), so completion mutations fail closed while release gates remain usable.
|
|
1480
|
-
const dispatchLine =
|
|
1481
|
-
layers.dispatch === "paused"
|
|
1482
|
-
? (() => {
|
|
1483
|
-
const prov = pauseProvenance(s.project);
|
|
1484
|
-
if (prov !== undefined) {
|
|
1485
|
-
const reason = prov.reason === undefined ? "" : ` — "${prov.reason}"`;
|
|
1486
|
-
return `dispatch paused (source: ${prov.source}${reason})`;
|
|
1487
|
-
}
|
|
1488
|
-
return isPaused(s.project) && pausedAt(s.project) === undefined
|
|
1489
|
-
? "dispatch paused (unparseable sentinel — new work and completion mutations refused; release gates remain available)"
|
|
1490
|
-
: "dispatch paused";
|
|
1491
|
-
})()
|
|
1492
|
-
: `dispatch ${layers.dispatch}`;
|
|
1493
|
-
|
|
1494
|
-
return [
|
|
1495
|
-
dispatchLine,
|
|
1496
|
-
tickLine,
|
|
1497
|
-
...(nextTickLine === undefined ? [] : [nextTickLine]),
|
|
1498
|
-
paneLine,
|
|
1499
|
-
recoveryLine,
|
|
1500
|
-
herdrLine,
|
|
1501
|
-
telegramLine,
|
|
1502
|
-
...(brief === undefined ? [] : [brief]),
|
|
1503
|
-
...(decisions === undefined ? [] : [decisions]),
|
|
1504
|
-
...(failureClasses === undefined ? [] : [failureClasses]),
|
|
1505
|
-
...(intake === undefined ? [] : [intake]),
|
|
1506
|
-
...(graphBlock === undefined ? [] : [graphBlock]),
|
|
1507
|
-
daemonBlock,
|
|
1508
|
-
"",
|
|
1509
|
-
formatProjectBody(s, workerPhases),
|
|
1510
|
-
].join("\n");
|
|
1511
|
-
}
|
|
1512
|
-
|
|
1513
|
-
function formatAvailabilityStatus(s: StatusSnapshot): string[] {
|
|
1514
|
-
const availability = s.availability;
|
|
1515
|
-
if (availability === undefined) return [];
|
|
1516
|
-
if (availability.mode === "always") {
|
|
1517
|
-
return ["availability 24-hour interrupts (no weekly window)"];
|
|
1518
|
-
}
|
|
1519
|
-
const mode =
|
|
1520
|
-
availability.nextTransitionAt === undefined || availability.timezone === undefined
|
|
1521
|
-
? `${availability.mode}; next transition could not be calculated`
|
|
1522
|
-
: `${availability.mode} until ${formatZonedMinute(availability.nextTransitionAt, availability.timezone)}`;
|
|
1523
|
-
const bypass = availability.bypass.length === 0 ? "none" : availability.bypass.join(", ");
|
|
1524
|
-
return [`availability ${mode}; quiet-hours bypass ${bypass}`];
|
|
1525
|
-
}
|
|
1526
|
-
|
|
1527
|
-
function formatDigestScheduleStatus(s: StatusSnapshot): string[] {
|
|
1528
|
-
const schedule = s.digestSchedule;
|
|
1529
|
-
if (schedule === undefined) return [];
|
|
1530
|
-
if (schedule.mode === "disabled") return ["next digest disabled"];
|
|
1531
|
-
if (schedule.mode === "per-tick") return ["next digest every tick"];
|
|
1532
|
-
if (schedule.mode === "due") return ["next digest due now"];
|
|
1533
|
-
return [
|
|
1534
|
-
schedule.nextAt === undefined
|
|
1535
|
-
? "next digest could not be calculated"
|
|
1536
|
-
: `next digest ${formatZonedMinute(schedule.nextAt, schedule.timezone)}`,
|
|
1537
|
-
];
|
|
1538
|
-
}
|
|
1539
|
-
|
|
1540
|
-
function formatProjectBody(
|
|
1541
|
-
s: StatusSnapshot,
|
|
1542
|
-
workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
|
|
1543
|
-
): string {
|
|
1544
|
-
const lines = [
|
|
1545
|
-
`project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
|
|
1546
|
-
...(s.pauseReason === undefined ? [] : [`paused ${s.pauseReason}`]),
|
|
1547
|
-
`config ${s.configPath}`,
|
|
1548
|
-
`state ${s.stateDir}`,
|
|
1549
|
-
...formatAvailabilityStatus(s),
|
|
1550
|
-
...formatDigestScheduleStatus(s),
|
|
1551
|
-
"",
|
|
1552
|
-
"caps",
|
|
1553
|
-
` workers ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
|
|
1554
|
-
` issues today ${s.runsToday}`,
|
|
1555
|
-
s.caps.dailySpendUsd === null
|
|
1556
|
-
? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
|
|
1557
|
-
: ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
|
|
1558
|
-
// Its own row beside the spend row, never folded into it: they are two
|
|
1559
|
-
// independent controls and an operator has to see which one stopped the
|
|
1560
|
-
// fleet (#110).
|
|
1561
|
-
` plan usage ${planUsageLine(s.planUsage)}`,
|
|
1562
|
-
// The tracker's API budget, when the renderer could read it. Absent on a
|
|
1563
|
-
// broken `gh`: one missing row, never a broken report (#188). Next to it,
|
|
1564
|
-
// the refusals the tracker actually observed, so a budget that looks
|
|
1565
|
-
// healthy next to every call being refused is still visible (#198).
|
|
1566
|
-
...(s.github === undefined
|
|
1567
|
-
? []
|
|
1568
|
-
: [
|
|
1569
|
-
` github graphql ${s.github.graphql.remaining}/${s.github.graphql.limit}, core ${s.github.core.remaining}/${s.github.core.limit} (graphql resets ${Math.max(0, Math.round((s.github.graphql.reset * 1000 - Date.now()) / 60_000))}m)` +
|
|
1570
|
-
(s.ghRefusals === undefined || s.ghRefusals.count === 0
|
|
1571
|
-
? ""
|
|
1572
|
-
: ` — ${s.ghRefusals.count} refusal(s) in last 5m (last ${new Date(s.ghRefusals.latestAt ?? Date.now()).toISOString().slice(11, 19)}Z)`),
|
|
1573
|
-
]),
|
|
1574
|
-
// The daemon's own observed github traffic today (#198): a separate row
|
|
1575
|
-
// from the polled budget above, so one missing `status` read never hides
|
|
1576
|
-
// the other. `onCall` counts every spawn including ones that end 304, so
|
|
1577
|
-
// `daemon` is spawns and `daemon-304` is the unbilled subset — billed ≈
|
|
1578
|
-
// difference, and the polled budget row stays the authority (#203).
|
|
1579
|
-
` github calls daemon ${s.ghCallsToday?.find((c) => c.source === "daemon")?.calls ?? 0} today` +
|
|
1580
|
-
((s.ghCallsToday?.find((c) => c.source === "daemon-304")?.calls ?? 0) === 0
|
|
1581
|
-
? ""
|
|
1582
|
-
: ` (${s.ghCallsToday!.find((c) => c.source === "daemon-304")!.calls} free 304s)`),
|
|
1583
|
-
// Label-projection ops the tracker has not applied yet (#201): GitHub is
|
|
1584
|
-
// behind what the store decided, and the operator can see the lag instead
|
|
1585
|
-
// of discovering it as a stale label or a missing one.
|
|
1586
|
-
...(s.labelOps === undefined
|
|
1587
|
-
? []
|
|
1588
|
-
: [
|
|
1589
|
-
` labels projection ${s.labelOps.pending} pending (oldest ${
|
|
1590
|
-
s.labelOps.oldestAgeMs >= 60_000
|
|
1591
|
-
? `${Math.round(s.labelOps.oldestAgeMs / 60_000)}m`
|
|
1592
|
-
: `${Math.max(1, Math.round(s.labelOps.oldestAgeMs / 1000))}s`
|
|
1593
|
-
})`,
|
|
1594
|
-
]),
|
|
1595
|
-
` new worker turns ${s.caps.workerMaxTurns}`,
|
|
1596
|
-
...s.turnOverrides.map(
|
|
1597
|
-
({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`,
|
|
1598
|
-
),
|
|
1599
|
-
` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
|
|
1600
|
-
` failed attempts ${s.caps.maxAttemptsPerIssue}`,
|
|
1601
|
-
` continuations ${s.caps.maxContinuationsPerIssue}`,
|
|
1602
|
-
"",
|
|
1603
|
-
...formatReleaseGrants(s.releaseGrants),
|
|
1604
|
-
"",
|
|
1605
|
-
formatDispatchSummary(s.dispatch),
|
|
1606
|
-
"",
|
|
1607
|
-
];
|
|
1608
|
-
if (s.activeRuns.length === 0) {
|
|
1609
|
-
lines.push("active runs (none)");
|
|
1610
|
-
} else {
|
|
1611
|
-
lines.push("active runs");
|
|
1612
|
-
for (const r of s.activeRuns) {
|
|
1613
|
-
const phase = workerPhases.get(r.issue);
|
|
1614
|
-
const state = phase === "pausing" || phase === "paused" ? phase : r.state;
|
|
1615
|
-
lines.push(
|
|
1616
|
-
` #${r.issue} ${r.repo} ${state} attempt ${r.attempt} ` +
|
|
1617
|
-
`${r.turns}/${r.maxTurns} turns ${r.spendUsd.toFixed(2)} ${r.branch}` +
|
|
1618
|
-
(r.prUrl ? ` ${r.prUrl}` : ""),
|
|
1619
|
-
);
|
|
1620
|
-
// The orchestrator's Duty 1 reads this command, and a flagged run's
|
|
1621
|
-
// escalation is deduplicated after one delivery — so this is where a
|
|
1622
|
-
// flagged PR stays visible for as long as it is still open (#128).
|
|
1623
|
-
const flagged = settlementFlagSummary(r.settlementFlags);
|
|
1624
|
-
if (flagged !== undefined) lines.push(` ${flagged}`);
|
|
1625
|
-
}
|
|
1626
|
-
}
|
|
1627
|
-
lines.push(...formatBaseHealth(s.baseHealth));
|
|
1628
|
-
lines.push(...formatSalvagedRuns(s.salvagedRuns));
|
|
1629
|
-
lines.push(...formatOpenReports(s.openReports));
|
|
1630
|
-
lines.push(...formatDigestBacklog(s.digestBacklog));
|
|
1631
|
-
if (s.liveWorkers > 0) {
|
|
1632
|
-
lines.push(
|
|
1633
|
-
"",
|
|
1634
|
-
`deploy ${s.liveWorkers} live worker(s) — restart salvages dirty trees then orphans the rows; ` +
|
|
1635
|
-
`hold and wait for workers 0/${s.caps.maxConcurrentWorkers} when you can drain instead`,
|
|
1636
|
-
);
|
|
1637
|
-
}
|
|
1638
|
-
return lines.join("\n");
|
|
1639
|
-
}
|
|
1640
|
-
|
|
1641
1284
|
export async function renderStatus(projectName?: string): Promise<string> {
|
|
1642
1285
|
const s = statusSnapshot(projectName);
|
|
1643
1286
|
const layers = fleetLayers(projectName);
|
|
@@ -1675,6 +1318,26 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
1675
1318
|
const cached = codeGraphFromHealthz(healthBody, project.name);
|
|
1676
1319
|
const codeGraph = cached ?? (await probeCodeGraph(project));
|
|
1677
1320
|
const workerPhases = workerPhasesFromHealthz(healthBody, project.name);
|
|
1321
|
+
// The newest host-wide stop/restart provenance (#378). Read here — not in
|
|
1322
|
+
// `statusSnapshot`, which is synchronous and belongs to the daemon module —
|
|
1323
|
+
// and rendered identically from either project: the daemon_stops table is
|
|
1324
|
+
// deliberately not partitioned by project, because the daemon serves every
|
|
1325
|
+
// project and the uninvolved one must see who stopped it too.
|
|
1326
|
+
const store = openStore(dbPath());
|
|
1327
|
+
let lastStop: DaemonStop | undefined;
|
|
1328
|
+
let siblings: { project: string; live: number }[] = [];
|
|
1329
|
+
try {
|
|
1330
|
+
lastStop = store.latestDaemonStop();
|
|
1331
|
+
// Shared-daemon visibility (#545): every configured project other than the
|
|
1332
|
+
// one being viewed, with its live-run count. Read from the same open store
|
|
1333
|
+
// as the provenance above, so a status reader can tell "my fleet is idle"
|
|
1334
|
+
// from "the process I am about to stop is busy".
|
|
1335
|
+
siblings = loadConfig()
|
|
1336
|
+
.projects.filter((p) => p.name !== project.name)
|
|
1337
|
+
.map((p) => ({ project: p.name, live: store.liveRuns(p.name).length }));
|
|
1338
|
+
} finally {
|
|
1339
|
+
store.close();
|
|
1340
|
+
}
|
|
1678
1341
|
return formatFleetStatus(
|
|
1679
1342
|
{ ...s, planUsage, github },
|
|
1680
1343
|
layers,
|
|
@@ -1687,6 +1350,8 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
1687
1350
|
failureClassBlock(project.name),
|
|
1688
1351
|
workerPhases,
|
|
1689
1352
|
intakeStatusLine(project.name),
|
|
1353
|
+
lastStop,
|
|
1354
|
+
siblings,
|
|
1690
1355
|
);
|
|
1691
1356
|
}
|
|
1692
1357
|
|
|
@@ -1746,18 +1411,32 @@ function failureClassBlock(projectName: string): string | undefined {
|
|
|
1746
1411
|
* the ambiguity this ledger exists to remove. Omitted on any throw — a status
|
|
1747
1412
|
* that fails to render because the store is mid-migration is worse than one
|
|
1748
1413
|
* missing a row.
|
|
1414
|
+
*
|
|
1415
|
+
* `decisions N open` counts only rows a human must answer (#459). Watches the
|
|
1416
|
+
* orchestrator set for itself are reported separately — a fleet with three
|
|
1417
|
+
* watches behind GitHub's checks is not a fleet waiting on its operator, and
|
|
1418
|
+
* lumping them in made `decisions 3 open` read as three unanswered questions.
|
|
1749
1419
|
*/
|
|
1750
|
-
function decisionStatusLine(projectName: string): string | undefined {
|
|
1420
|
+
export function decisionStatusLine(projectName: string): string | undefined {
|
|
1751
1421
|
const path = dbPath();
|
|
1752
1422
|
if (!existsSync(path)) return undefined;
|
|
1753
1423
|
let store: Store | undefined;
|
|
1754
1424
|
try {
|
|
1755
1425
|
store = openStore(path);
|
|
1756
1426
|
const open = store.openDecisions(projectName);
|
|
1757
|
-
|
|
1758
|
-
const
|
|
1759
|
-
|
|
1760
|
-
|
|
1427
|
+
const questions = open.filter((d) => d.kind !== "watch");
|
|
1428
|
+
const watches = open.filter((d) => d.kind === "watch");
|
|
1429
|
+
if (questions.length === 0 && watches.length === 0) return "decisions none open";
|
|
1430
|
+
let line: string;
|
|
1431
|
+
if (questions.length === 0) {
|
|
1432
|
+
line = "decisions none open";
|
|
1433
|
+
} else {
|
|
1434
|
+
const oldest = questions[0]!;
|
|
1435
|
+
const hours = Math.max(0, Math.round((Date.now() - oldest.askedAt) / 3_600_000));
|
|
1436
|
+
line = `decisions ${questions.length} open (oldest ${hours}h)`;
|
|
1437
|
+
}
|
|
1438
|
+
if (watches.length > 0) line += ` · watches ${watches.length}`;
|
|
1439
|
+
return line;
|
|
1761
1440
|
} catch {
|
|
1762
1441
|
return undefined;
|
|
1763
1442
|
} finally {
|
|
@@ -2120,4 +1799,3 @@ function probeOmpPane(
|
|
|
2120
1799
|
return { kind: "unknown", reason: err instanceof Error ? err.message : String(err) };
|
|
2121
1800
|
}
|
|
2122
1801
|
}
|
|
2123
|
-
|
package/src/gitops.ts
CHANGED
|
@@ -96,10 +96,95 @@ export function scrubUserinfo(text: string): string {
|
|
|
96
96
|
return text.replace(URL_USERINFO, "$1***@");
|
|
97
97
|
}
|
|
98
98
|
|
|
99
|
-
function mirrorPath(project: Pick<ProjectConfig, "mirrorRoot">, repo: RepoTarget): string {
|
|
99
|
+
export function mirrorPath(project: Pick<ProjectConfig, "mirrorRoot">, repo: RepoTarget): string {
|
|
100
100
|
return join(project.mirrorRoot, `${repo.name}.git`);
|
|
101
101
|
}
|
|
102
102
|
|
|
103
|
+
/**
|
|
104
|
+
* Everything the file-lane interlock (#555) needs to read one active run's lane.
|
|
105
|
+
*
|
|
106
|
+
* The dispatcher may read worktrees — this is squarely its job, and the issue
|
|
107
|
+
* is explicit that the orchestrator must not be the one guessing file lanes
|
|
108
|
+
* from another run's tree. A live run advertises its lane from its own worktree
|
|
109
|
+
* (`git status --porcelain` plus `git diff --name-only <base>...HEAD`, exactly
|
|
110
|
+
* the two commands the continuation brief already hands every worker), which is
|
|
111
|
+
* the truth rather than the run's issue body — the incident this interlock
|
|
112
|
+
* exists for was an issue *saying* one set of files while its tree wrote
|
|
113
|
+
* another. A worker-finished run (`pushed-pending` / `pushed-green`) no longer
|
|
114
|
+
* has a worktree, but its open PR still reaches for base's files, so its lane
|
|
115
|
+
* is read from the retained branch in the mirror instead.
|
|
116
|
+
*/
|
|
117
|
+
export interface RunLaneInput {
|
|
118
|
+
/** Conductor-created worktree path; `""` when the run has no live checkout. */
|
|
119
|
+
worktree: string;
|
|
120
|
+
/** Tracking ref of the base branch to diff against, e.g. `refs/remotes/origin/main`. */
|
|
121
|
+
baseRef: string;
|
|
122
|
+
/** The run's branch ref in the mirror (e.g. `refs/heads/feat-x`), for a
|
|
123
|
+
* worker-finished run whose worktree has been removed. */
|
|
124
|
+
branchRef?: string;
|
|
125
|
+
/** The repo's bare mirror directory, for the branch-diff read. */
|
|
126
|
+
mirror?: string;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* The set of file paths one active run has touched relative to base, deduped
|
|
131
|
+
* and sorted. Persistable but advisory: the admitting gate treats an unreadable
|
|
132
|
+
* or absent lane as "no overlap" (fail open), so no probe bug can refuse a
|
|
133
|
+
* well-formed issue.
|
|
134
|
+
*/
|
|
135
|
+
export type RunLaneProbe = (input: RunLaneInput) => Promise<string[]>;
|
|
136
|
+
|
|
137
|
+
/** The real one, reading the run's own worktree and/or the mirror branch. */
|
|
138
|
+
export async function probeRunLane(
|
|
139
|
+
input: RunLaneInput,
|
|
140
|
+
exec: Exec = spawnCaptured,
|
|
141
|
+
): Promise<string[]> {
|
|
142
|
+
const files = new Set<string>();
|
|
143
|
+
const add = (raw: string): void => {
|
|
144
|
+
for (const line of raw.split("\n")) {
|
|
145
|
+
const path = line.trim();
|
|
146
|
+
if (path !== "") files.add(path);
|
|
147
|
+
}
|
|
148
|
+
};
|
|
149
|
+
if (input.worktree !== "") {
|
|
150
|
+
const status = await exec(["git", "-C", input.worktree, "status", "--porcelain"], {});
|
|
151
|
+
if (status.code === 0) {
|
|
152
|
+
for (const path of parsePorcelain(status.stdout)) files.add(path);
|
|
153
|
+
}
|
|
154
|
+
const diff = await exec(
|
|
155
|
+
["git", "-C", input.worktree, "diff", "--name-only", `${input.baseRef}...HEAD`],
|
|
156
|
+
{},
|
|
157
|
+
);
|
|
158
|
+
if (diff.code === 0) add(diff.stdout);
|
|
159
|
+
}
|
|
160
|
+
if (input.branchRef !== undefined && input.mirror !== undefined && input.mirror !== "") {
|
|
161
|
+
const diff = await exec(
|
|
162
|
+
["git", "--git-dir", input.mirror, "diff", "--name-only", `${input.baseRef}...${input.branchRef}`],
|
|
163
|
+
{},
|
|
164
|
+
);
|
|
165
|
+
if (diff.code === 0) add(diff.stdout);
|
|
166
|
+
}
|
|
167
|
+
return [...files].sort();
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Parse `git status --porcelain` output into the paths it names. A rename or
|
|
172
|
+
* copy is reported as `XY old -> new`, and the *new* path is the one that
|
|
173
|
+
* occupies the lane (it is the path the other run's future writes would
|
|
174
|
+
* collide with), so the arrow form keeps only its target.
|
|
175
|
+
*/
|
|
176
|
+
function parsePorcelain(stdout: string): string[] {
|
|
177
|
+
const out = new Set<string>();
|
|
178
|
+
for (const raw of stdout.split("\n")) {
|
|
179
|
+
if (raw.length < 3) continue;
|
|
180
|
+
let rest = raw.slice(2).trim();
|
|
181
|
+
const arrow = rest.indexOf(" -> ");
|
|
182
|
+
if (arrow !== -1) rest = rest.slice(arrow + 4).trim();
|
|
183
|
+
if (rest !== "") out.add(rest);
|
|
184
|
+
}
|
|
185
|
+
return [...out];
|
|
186
|
+
}
|
|
187
|
+
|
|
103
188
|
/** GitHub slug parsed off the clone URL, falling back to the routing name. */
|
|
104
189
|
export function repoSlugFor(repo: RepoTarget): string {
|
|
105
190
|
const m = /(?:[:/])([^/:]+\/[^/]+?)(?:\.git)?$/.exec(repo.cloneUrl);
|