omp-conductor 0.15.13 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +72 -2
- package/package.json +2 -1
- package/schema/config.schema.json +7 -0
- package/src/admission.ts +849 -0
- package/src/ask.ts +47 -0
- package/src/backups.ts +19 -7
- package/src/board.ts +1 -2
- package/src/briefs/orchestrator.md +62 -4
- package/src/cli.ts +26 -0
- package/src/commands/context.ts +3 -0
- package/src/commands/decision.ts +10 -1
- package/src/commands/doctor.ts +2 -0
- package/src/commands/message.ts +8 -1
- package/src/commands/restart.ts +15 -3
- package/src/commands/restore-db.ts +146 -0
- package/src/commands/stop.ts +24 -15
- package/src/commands/tail.ts +204 -44
- package/src/commands/unfreeze.ts +56 -0
- package/src/commands/watch.ts +77 -0
- package/src/config-schema.ts +13 -0
- package/src/config.ts +54 -0
- package/src/daemon.ts +255 -530
- package/src/dashboard/server.ts +2 -1
- package/src/decisions.ts +32 -7
- package/src/depends-on.ts +122 -0
- package/src/doctor.ts +297 -5
- package/src/escalate.ts +191 -19
- package/src/failure-class.ts +47 -0
- package/src/fleet.ts +168 -452
- package/src/gitops.ts +86 -1
- package/src/log.ts +40 -0
- package/src/model-fallback.ts +3 -2
- package/src/omp-settings.ts +114 -0
- package/src/omp.ts +39 -0
- package/src/orchestrator-tick.ts +7 -1
- package/src/reports.ts +124 -12
- package/src/session-host.ts +6 -0
- package/src/setup-wizard.ts +36 -0
- package/src/setup.ts +58 -1
- package/src/status-render.ts +445 -0
- package/src/stop-provenance.ts +53 -0
- package/src/store.ts +352 -11
- package/src/transcript.ts +1 -1
- package/src/types.ts +187 -4
- package/src/unblock.ts +1 -1
- package/src/upgrade-verify.ts +1 -1
- package/src/upgrade.ts +1 -2
- package/src/verbs/server.ts +25 -0
- package/src/worker.ts +358 -10
- package/src/worktree.ts +13 -1
package/src/setup.ts
CHANGED
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
24
24
|
import { homedir } from "node:os";
|
|
25
25
|
import { dirname, join } from "node:path";
|
|
26
|
+
import { parse as parseYaml } from "yaml";
|
|
26
27
|
import {
|
|
27
28
|
COMPOSE_BANNER,
|
|
28
29
|
ORCHESTRATOR_BRIEF_NAME,
|
|
@@ -46,6 +47,7 @@ import {
|
|
|
46
47
|
stateDir,
|
|
47
48
|
} from "./config.ts";
|
|
48
49
|
import { graphProjectPath, graphRepos } from "./graph.ts";
|
|
50
|
+
import { ompSettingsOverlay } from "./omp-settings.ts";
|
|
49
51
|
import {
|
|
50
52
|
CONFIG_VERSION,
|
|
51
53
|
DEFAULT_AUTHORITY,
|
|
@@ -163,6 +165,15 @@ export interface SetupAnswers {
|
|
|
163
165
|
*/
|
|
164
166
|
modelFallbacks?: string[];
|
|
165
167
|
modelFallbackThreshold?: number;
|
|
168
|
+
/**
|
|
169
|
+
* The project's omp settings overlay (#537): an opaque map layered into
|
|
170
|
+
* every worker session via the fleet-owned settings channel. The wizard asks
|
|
171
|
+
* for it as one free-form YAML answer (the only prompt this map gets — the
|
|
172
|
+
* keys inside it are omp's vocabulary, not a list conductor can offer); it
|
|
173
|
+
* also survives an amend of any other area unchanged, like
|
|
174
|
+
* {@link modelFallbacks}.
|
|
175
|
+
*/
|
|
176
|
+
ompSettings?: Record<string, unknown>;
|
|
166
177
|
telegramChatId?: string;
|
|
167
178
|
/** Forum topic for tier-2 Telegram pages; absent keeps flat-chat 0.13 behaviour. */
|
|
168
179
|
telegramTopicId?: number;
|
|
@@ -855,6 +866,9 @@ export function buildProject(a: SetupAnswers): ProjectConfig {
|
|
|
855
866
|
// would pin every run of the project onto a dead provider again (#286).
|
|
856
867
|
...(a.modelFallbacks === undefined ? {} : { modelFallbacks: [...a.modelFallbacks] }),
|
|
857
868
|
...(a.modelFallbackThreshold === undefined ? {} : { modelFallbackThreshold: a.modelFallbackThreshold }),
|
|
869
|
+
// The omp settings overlay is an opaque map the wizard collects as free-form
|
|
870
|
+
// YAML; an unrelated amend must not delete it (#537).
|
|
871
|
+
...(a.ompSettings === undefined ? {} : { ompSettings: a.ompSettings }),
|
|
858
872
|
escalation,
|
|
859
873
|
authority: { ...a.authority },
|
|
860
874
|
// Written out in full, never as the legacy string: the file then says which
|
|
@@ -1061,6 +1075,7 @@ export function answersFromProject(p: ProjectConfig): SetupAnswers {
|
|
|
1061
1075
|
if (p.groomBelow !== undefined) answers.groomBelow = p.groomBelow;
|
|
1062
1076
|
if (p.modelFallbacks !== undefined) answers.modelFallbacks = [...p.modelFallbacks];
|
|
1063
1077
|
if (p.modelFallbackThreshold !== undefined) answers.modelFallbackThreshold = p.modelFallbackThreshold;
|
|
1078
|
+
if (p.ompSettings !== undefined) answers.ompSettings = { ...p.ompSettings };
|
|
1064
1079
|
if (p.escalation.telegramChatId !== undefined) answers.telegramChatId = p.escalation.telegramChatId;
|
|
1065
1080
|
if (p.escalation.telegramTopicId !== undefined) answers.telegramTopicId = p.escalation.telegramTopicId;
|
|
1066
1081
|
if (p.recoveryMerges !== undefined) {
|
|
@@ -1682,19 +1697,47 @@ export function formatGates(gates: readonly { cmd: string; cwd: string }[]): str
|
|
|
1682
1697
|
return gates.map((g) => (g.cwd === "." ? g.cmd : `${g.cmd} @ ${g.cwd}`)).join(", ");
|
|
1683
1698
|
}
|
|
1684
1699
|
|
|
1700
|
+
/**
|
|
1701
|
+
* Parses the wizard's free-form YAML answer for a project's omp settings
|
|
1702
|
+
* overlay (#537) and validates YAML shape only — the same boundary the config
|
|
1703
|
+
* loader enforces. A mapping at the document root is the whole contract:
|
|
1704
|
+
* everything inside it is omp's schema to own, so an unknown key or a
|
|
1705
|
+
* wrong-typed value is omp's to reject, never this module's to understand
|
|
1706
|
+
* (non-string keys are coerced by the YAML parser exactly as omp's own loader
|
|
1707
|
+
* coerces them, so a `true: x` mapping stays YAML-valid on both sides). Blank
|
|
1708
|
+
* input is a valid "no overlay" answer.
|
|
1709
|
+
*/
|
|
1710
|
+
export function parseOmpSettingsYaml(
|
|
1711
|
+
text: string,
|
|
1712
|
+
): { ok: true; value: Record<string, unknown> } | { ok: false; problem: string } {
|
|
1713
|
+
const trimmed = text.trim();
|
|
1714
|
+
if (trimmed.length === 0) return { ok: true, value: {} };
|
|
1715
|
+
let parsed: unknown;
|
|
1716
|
+
try {
|
|
1717
|
+
parsed = parseYaml(trimmed);
|
|
1718
|
+
} catch (err) {
|
|
1719
|
+
return { ok: false, problem: `"${text}" is not valid YAML: ${err instanceof Error ? err.message : String(err)}` };
|
|
1720
|
+
}
|
|
1721
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
1722
|
+
return { ok: false, problem: "the omp settings overlay must be a YAML mapping at the root, e.g. `modelRoles: { worker: \"@slow\" }`" };
|
|
1723
|
+
}
|
|
1724
|
+
return { ok: true, value: parsed as Record<string, unknown> };
|
|
1725
|
+
}
|
|
1726
|
+
|
|
1685
1727
|
/**
|
|
1686
1728
|
* The wizard's questions, grouped as the areas a re-run can amend one of, in the
|
|
1687
1729
|
* order the full interview asks them.
|
|
1688
1730
|
*
|
|
1689
1731
|
* Data rather than a switch so the menu, the CLI's positional area vocabulary,
|
|
1690
1732
|
* ./setup-wizard.ts's `AREA_ASKERS` table and the amend summary all enumerate the
|
|
1691
|
-
* same
|
|
1733
|
+
* same ten areas: an added area fails to compile until it has a name, a current
|
|
1692
1734
|
* value and a set of questions.
|
|
1693
1735
|
*/
|
|
1694
1736
|
export const AMEND_AREA_IDS = [
|
|
1695
1737
|
"tracker",
|
|
1696
1738
|
"gates",
|
|
1697
1739
|
"caps",
|
|
1740
|
+
"omp-settings",
|
|
1698
1741
|
"code-graph",
|
|
1699
1742
|
"authority",
|
|
1700
1743
|
"policy",
|
|
@@ -1765,6 +1808,20 @@ export const AMEND_AREAS: {
|
|
|
1765
1808
|
);
|
|
1766
1809
|
},
|
|
1767
1810
|
},
|
|
1811
|
+
"omp-settings": {
|
|
1812
|
+
// The free-form omp settings overlay (#537), the sibling per-worker knob to
|
|
1813
|
+
// the model: one area no menu offers is a setting only a full re-interview
|
|
1814
|
+
// can reach, so this one has its own row like the caps.
|
|
1815
|
+
name: "omp settings overlay",
|
|
1816
|
+
asks: "the free-form YAML map layered into every worker session's omp settings — omp's schema, not conductor's",
|
|
1817
|
+
// The *effective* overlay — `ompSettings` plus the retry keys derived from
|
|
1818
|
+
// `modelFallbacks` — so the menu says what a worker would actually load,
|
|
1819
|
+
// matching `doctor` and the dispatcher rather than echoing raw config.
|
|
1820
|
+
describe: (p) => {
|
|
1821
|
+
const overlay = ompSettingsOverlay(p);
|
|
1822
|
+
return overlay === undefined ? "no overlay — workers inherit global settings" : JSON.stringify(overlay);
|
|
1823
|
+
},
|
|
1824
|
+
},
|
|
1768
1825
|
"code-graph": {
|
|
1769
1826
|
name: "code graph",
|
|
1770
1827
|
asks: "whether workers query a code-graph index, and the root its one-clone-per-repo lives under",
|
|
@@ -0,0 +1,445 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The status-rendering half of the fleet operator surface (`fleet.ts`
|
|
3
|
+
* composes; this owns the rendered text).
|
|
4
|
+
*
|
|
5
|
+
* Extracted from `fleet.ts` so the region every issue that surfaces something
|
|
6
|
+
* an operator reads must touch is a lane of its own: #459, #582, #318, #283
|
|
7
|
+
* and #484 all render a new line here, and a diff touching only this module
|
|
8
|
+
* must not have to touch the composition root. Behavior is deliberately
|
|
9
|
+
* untouched by the move — this is a move, not a rewrite: the module takes a
|
|
10
|
+
* data structure and returns text, and reads nothing itself.
|
|
11
|
+
*
|
|
12
|
+
* What status rendering is allowed to consult is the point of the module. It
|
|
13
|
+
* does not import `fleet.ts` back, and it owns the structural input it
|
|
14
|
+
* consumes (`FleetLayers`, `TelegramHealth`, `FleetDaemonProbe`) rather than
|
|
15
|
+
* reaching into the module that builds them — the same seam #580 used for
|
|
16
|
+
* `AdmissionDeps`. Adding a field to `fleetLayers` must not break these
|
|
17
|
+
* functions.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { formatZonedMinute } from "./availability.ts";
|
|
21
|
+
import type { DaemonStop } from "./types.ts";
|
|
22
|
+
import { settlementFlagSummary } from "./diff-flags.ts";
|
|
23
|
+
import type { CodeGraphHealth } from "./graph-health.ts";
|
|
24
|
+
import { formatDigestBacklog, formatOpenReports } from "./reports.ts";
|
|
25
|
+
import { formatOrchestratorDown } from "./orchestrator-down.ts";
|
|
26
|
+
import { planUsageLine } from "./usage.ts";
|
|
27
|
+
import { SYSTEMD_UNIT, type UnitOwnership } from "./lifecycle.ts";
|
|
28
|
+
import type { WorkerPausePhase } from "./worker.ts";
|
|
29
|
+
import { formatRss, rssBytesFromHealthz } from "./host.ts";
|
|
30
|
+
import {
|
|
31
|
+
formatBaseHealth,
|
|
32
|
+
formatDispatchSummary,
|
|
33
|
+
formatFreezes,
|
|
34
|
+
formatReleaseGrants,
|
|
35
|
+
formatSalvagedRuns,
|
|
36
|
+
isPaused,
|
|
37
|
+
pausedAt,
|
|
38
|
+
pauseProvenance,
|
|
39
|
+
type StatusSnapshot,
|
|
40
|
+
} from "./daemon.ts";
|
|
41
|
+
|
|
42
|
+
// layered status
|
|
43
|
+
// ---------------------------------------------------------------------------
|
|
44
|
+
|
|
45
|
+
export type DispatchLayer = "running" | "paused" | "stopped";
|
|
46
|
+
export type TicksLayer =
|
|
47
|
+
| "armed"
|
|
48
|
+
| "disarmed"
|
|
49
|
+
| "no-heartbeat-config"
|
|
50
|
+
| "invalid-heartbeat-config"
|
|
51
|
+
| "ungated";
|
|
52
|
+
export type PaneLayer = "live" | "missing" | "unknown";
|
|
53
|
+
/** `unpinnable`: no tick config, so FLEET_CWD — the only path recovery reads — is unknown. */
|
|
54
|
+
export type RecoveryLayer = "pinned" | "clear" | "unpinnable";
|
|
55
|
+
export type HerdrLayer = "active" | "inactive" | "unknown";
|
|
56
|
+
export type TelegramLayer = "ok" | "degraded" | "down" | "unconfigured" | "unprobed";
|
|
57
|
+
|
|
58
|
+
export interface TelegramHealth {
|
|
59
|
+
kind: TelegramLayer;
|
|
60
|
+
detail?: string;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface FleetLayers {
|
|
64
|
+
dispatch: DispatchLayer;
|
|
65
|
+
ticks: TicksLayer;
|
|
66
|
+
ticksDetail?: string;
|
|
67
|
+
nextTickAt?: string;
|
|
68
|
+
pane: PaneLayer;
|
|
69
|
+
paneDetail?: string;
|
|
70
|
+
recovery: RecoveryLayer;
|
|
71
|
+
recoveryDetail?: string;
|
|
72
|
+
herdr: HerdrLayer;
|
|
73
|
+
herdrDetail?: string;
|
|
74
|
+
armedPath?: string;
|
|
75
|
+
tickConfigPath?: string;
|
|
76
|
+
paneHaltPath?: string;
|
|
77
|
+
paused: boolean;
|
|
78
|
+
daemon: { running: boolean; pid?: number; port?: number };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function formatCodeGraphHealth(graph: CodeGraphHealth, now = Date.now()): string | undefined {
|
|
82
|
+
if (!graph.configured) return undefined;
|
|
83
|
+
const indexed = graph.repos.filter((repo) => repo.index === "present").length;
|
|
84
|
+
let refresh: string = graph.refresh.result;
|
|
85
|
+
if (graph.refresh.lastSuccessAt !== undefined) {
|
|
86
|
+
const ageMs = Math.max(0, now - Date.parse(graph.refresh.lastSuccessAt));
|
|
87
|
+
refresh = `${graph.refresh.lastSuccessAt} (${Math.max(1, Math.ceil(ageMs / 60_000))}m ago)`;
|
|
88
|
+
}
|
|
89
|
+
return [
|
|
90
|
+
`code graph ${graph.status} ${indexed}/${graph.repos.length} repos indexed`,
|
|
91
|
+
` indexer ${graph.prerequisites.indexer}`,
|
|
92
|
+
` MCP mount ${graph.prerequisites.mcpMount}`,
|
|
93
|
+
` timer ${graph.timer.enabled} / ${graph.timer.active}`,
|
|
94
|
+
` refresh ${refresh}`,
|
|
95
|
+
...graph.reasons.map((reason) => ` - ${reason}`),
|
|
96
|
+
].join("\n");
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Project-scoped reading of a living daemon record + `/healthz` body.
|
|
102
|
+
*
|
|
103
|
+
* Board and `status` share this so a host running one daemon for several
|
|
104
|
+
* projects cannot be called healthy for a project it does not serve (#379).
|
|
105
|
+
* A third interpretation convention is forbidden.
|
|
106
|
+
*/
|
|
107
|
+
export type DaemonProjectHealth =
|
|
108
|
+
| { kind: "stopped" }
|
|
109
|
+
| { kind: "ok" }
|
|
110
|
+
| { kind: "unreachable" }
|
|
111
|
+
| { kind: "other-project"; serves?: string };
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Facts `formatFleetStatus` needs about the living daemon. Pure input — the
|
|
115
|
+
* caller probes; the formatter never shells out (#379).
|
|
116
|
+
*/
|
|
117
|
+
export type FleetDaemonProbe = {
|
|
118
|
+
project: DaemonProjectHealth;
|
|
119
|
+
/** `/healthz` body when the probe is project-ok (rss / overlays). */
|
|
120
|
+
body?: string;
|
|
121
|
+
/** systemd ownership of the living record pid; omit when unprobed. */
|
|
122
|
+
unit?: UnitOwnership;
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
function formatDaemonHealthz(probe: FleetDaemonProbe | undefined): string {
|
|
126
|
+
if (probe === undefined) return "unprobed";
|
|
127
|
+
switch (probe.project.kind) {
|
|
128
|
+
case "stopped":
|
|
129
|
+
return "stopped";
|
|
130
|
+
case "ok":
|
|
131
|
+
return "ok";
|
|
132
|
+
case "unreachable":
|
|
133
|
+
return "unreachable — the process is up but not serving";
|
|
134
|
+
case "other-project":
|
|
135
|
+
return probe.project.serves === undefined
|
|
136
|
+
? "other-project"
|
|
137
|
+
: `other-project — serves ${probe.project.serves}`;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Unit line for a living record. The bare unit name alone used to imply
|
|
143
|
+
* systemd ownership that was never checked (#379); every branch states the
|
|
144
|
+
* probe result, and `unknown` is never rendered as owned.
|
|
145
|
+
*/
|
|
146
|
+
function formatDaemonUnit(unit: UnitOwnership | undefined, recordPid: number): string {
|
|
147
|
+
if (unit === undefined) return ` unit ${SYSTEMD_UNIT} unprobed`;
|
|
148
|
+
switch (unit.kind) {
|
|
149
|
+
case "active":
|
|
150
|
+
if (unit.pid === recordPid) {
|
|
151
|
+
return ` unit ${SYSTEMD_UNIT} systemd-owned`;
|
|
152
|
+
}
|
|
153
|
+
// Live unit, different MainPID — the record pid is not systemd's.
|
|
154
|
+
return ` unit ${SYSTEMD_UNIT} unmanaged — MainPID ${unit.pid}`;
|
|
155
|
+
case "failed":
|
|
156
|
+
return ` unit ${SYSTEMD_UNIT} unmanaged — unit failed`;
|
|
157
|
+
case "inactive":
|
|
158
|
+
return ` unit ${SYSTEMD_UNIT} inactive`;
|
|
159
|
+
case "unknown":
|
|
160
|
+
return ` unit ${SYSTEMD_UNIT} unknown — ${unit.reason}`;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* The newest daemon stop/restart provenance as status lines (#378).
|
|
166
|
+
*
|
|
167
|
+
* Rendered under the daemon block whether the daemon is down or has since
|
|
168
|
+
* been restarted — the whole point is that the debrief survives the restart.
|
|
169
|
+
* An unattributed record says so explicitly: the external-signal fallback
|
|
170
|
+
* records what the receiving daemon knew (that it was unattributed, when,
|
|
171
|
+
* which projects it served with live counts) rather than guessing a caller.
|
|
172
|
+
*/
|
|
173
|
+
function formatLastStop(lastStop: DaemonStop | undefined): string[] {
|
|
174
|
+
if (lastStop === undefined) return [];
|
|
175
|
+
const caller = lastStop.unattributed
|
|
176
|
+
? "unattributed — no mediated request (external signal)"
|
|
177
|
+
: `pid ${lastStop.callerPid ?? "?"}` +
|
|
178
|
+
(lastStop.callerUid === undefined ? "" : ` uid ${lastStop.callerUid}`) +
|
|
179
|
+
(lastStop.role === undefined ? "" : ` (${lastStop.role})`);
|
|
180
|
+
return [
|
|
181
|
+
` last stop ${new Date(lastStop.at).toISOString()} ${lastStop.controlPath}`,
|
|
182
|
+
` caller ${caller}`,
|
|
183
|
+
` scope ${lastStop.scope}${lastStop.project === undefined ? "" : `: ${lastStop.project}`}`,
|
|
184
|
+
` affects ${lastStop.affected.map((a) => `${a.project} (${a.live} live)`).join(", ")}`,
|
|
185
|
+
` reason ${lastStop.reason}`,
|
|
186
|
+
];
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
export function formatFleetStatus(
|
|
191
|
+
s: StatusSnapshot,
|
|
192
|
+
layers: FleetLayers,
|
|
193
|
+
daemon: FleetDaemonProbe | undefined = undefined,
|
|
194
|
+
telegram: TelegramHealth = { kind: "unprobed" },
|
|
195
|
+
now = Date.now(),
|
|
196
|
+
codeGraph: CodeGraphHealth = { configured: false },
|
|
197
|
+
brief: string | undefined = undefined,
|
|
198
|
+
decisions: string | undefined = undefined,
|
|
199
|
+
failureClasses: string | undefined = undefined,
|
|
200
|
+
workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
|
|
201
|
+
intake: string | undefined = undefined,
|
|
202
|
+
lastStop: DaemonStop | undefined = undefined,
|
|
203
|
+
siblings: { project: string; live: number }[] = [],
|
|
204
|
+
): string {
|
|
205
|
+
const tickLine =
|
|
206
|
+
layers.ticksDetail === undefined
|
|
207
|
+
? `ticks ${layers.ticks}`
|
|
208
|
+
: `ticks ${layers.ticks} (${layers.ticksDetail})`;
|
|
209
|
+
let nextTickLine: string | undefined;
|
|
210
|
+
if (layers.nextTickAt !== undefined) {
|
|
211
|
+
const delta = Date.parse(layers.nextTickAt) - now;
|
|
212
|
+
const minutes = Math.max(1, Math.ceil(Math.abs(delta) / 60_000));
|
|
213
|
+
nextTickLine =
|
|
214
|
+
`next tick ${layers.nextTickAt} ` +
|
|
215
|
+
`(${delta >= 0 ? `in ${minutes}m` : `overdue by ${minutes}m`})`;
|
|
216
|
+
}
|
|
217
|
+
const paneLine =
|
|
218
|
+
layers.paneDetail === undefined
|
|
219
|
+
? `pane ${layers.pane}`
|
|
220
|
+
: `pane ${layers.pane} (${layers.paneDetail})`;
|
|
221
|
+
const recoveryLine =
|
|
222
|
+
layers.recoveryDetail === undefined
|
|
223
|
+
? `recovery ${layers.recovery}`
|
|
224
|
+
: `recovery ${layers.recovery} (${layers.recoveryDetail})`;
|
|
225
|
+
const herdrLine =
|
|
226
|
+
layers.herdrDetail === undefined
|
|
227
|
+
? `herdr ${layers.herdr}`
|
|
228
|
+
: `herdr ${layers.herdr} (${layers.herdrDetail})`;
|
|
229
|
+
const telegramLine =
|
|
230
|
+
telegram.detail === undefined
|
|
231
|
+
? `telegram ${telegram.kind}`
|
|
232
|
+
: `telegram ${telegram.kind} (${telegram.detail})`;
|
|
233
|
+
|
|
234
|
+
let daemonBlock: string;
|
|
235
|
+
if (!layers.daemon.running || layers.daemon.pid === undefined) {
|
|
236
|
+
daemonBlock = "daemon not running";
|
|
237
|
+
} else {
|
|
238
|
+
// Only trust rss from a project-ok body — a foreign payload's bytes are
|
|
239
|
+
// not this project's daemon facts (#379).
|
|
240
|
+
const rss =
|
|
241
|
+
daemon?.project.kind === "ok" ? rssBytesFromHealthz(daemon.body) : undefined;
|
|
242
|
+
daemonBlock = [
|
|
243
|
+
"daemon",
|
|
244
|
+
` pid ${layers.daemon.pid}`,
|
|
245
|
+
` port ${layers.daemon.port ?? "?"}`,
|
|
246
|
+
...(rss === undefined ? [] : [` rss ${formatRss(rss)}`]),
|
|
247
|
+
` healthz ${formatDaemonHealthz(daemon)}`,
|
|
248
|
+
formatDaemonUnit(daemon?.unit, layers.daemon.pid),
|
|
249
|
+
].join("\n");
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
const graphBlock = formatCodeGraphHealth(codeGraph, now);
|
|
253
|
+
|
|
254
|
+
// Pause provenance (`pause --reason`, an integrity/spend-cap/upgrade pause,
|
|
255
|
+
// a halt) answers "who stopped the fleet" without opening a file (#185). An
|
|
256
|
+
// unparseable sentinel — paused but with no datable line 1 — is itself news:
|
|
257
|
+
// it means a run admitted before an *unknown* pause cannot prove innocence
|
|
258
|
+
// (#174), so completion mutations fail closed while release gates remain usable.
|
|
259
|
+
const dispatchLine =
|
|
260
|
+
layers.dispatch === "paused"
|
|
261
|
+
? (() => {
|
|
262
|
+
const prov = pauseProvenance(s.project);
|
|
263
|
+
if (prov !== undefined) {
|
|
264
|
+
const reason = prov.reason === undefined ? "" : ` — "${prov.reason}"`;
|
|
265
|
+
return `dispatch paused (source: ${prov.source}${reason})`;
|
|
266
|
+
}
|
|
267
|
+
return isPaused(s.project) && pausedAt(s.project) === undefined
|
|
268
|
+
? "dispatch paused (unparseable sentinel — new work and completion mutations refused; release gates remain available)"
|
|
269
|
+
: "dispatch paused";
|
|
270
|
+
})()
|
|
271
|
+
: `dispatch ${layers.dispatch}`;
|
|
272
|
+
|
|
273
|
+
return [
|
|
274
|
+
dispatchLine,
|
|
275
|
+
tickLine,
|
|
276
|
+
...(nextTickLine === undefined ? [] : [nextTickLine]),
|
|
277
|
+
paneLine,
|
|
278
|
+
recoveryLine,
|
|
279
|
+
herdrLine,
|
|
280
|
+
telegramLine,
|
|
281
|
+
...(brief === undefined ? [] : [brief]),
|
|
282
|
+
...(decisions === undefined ? [] : [decisions]),
|
|
283
|
+
...(failureClasses === undefined ? [] : [failureClasses]),
|
|
284
|
+
...(intake === undefined ? [] : [intake]),
|
|
285
|
+
...(graphBlock === undefined ? [] : [graphBlock]),
|
|
286
|
+
daemonBlock,
|
|
287
|
+
...formatLastStop(lastStop),
|
|
288
|
+
"",
|
|
289
|
+
formatProjectBody(s, workerPhases, now, siblings),
|
|
290
|
+
].join("\n");
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
function formatAvailabilityStatus(s: StatusSnapshot): string[] {
|
|
295
|
+
const availability = s.availability;
|
|
296
|
+
if (availability === undefined) return [];
|
|
297
|
+
if (availability.mode === "always") {
|
|
298
|
+
return ["availability 24-hour interrupts (no weekly window)"];
|
|
299
|
+
}
|
|
300
|
+
const mode =
|
|
301
|
+
availability.nextTransitionAt === undefined || availability.timezone === undefined
|
|
302
|
+
? `${availability.mode}; next transition could not be calculated`
|
|
303
|
+
: `${availability.mode} until ${formatZonedMinute(availability.nextTransitionAt, availability.timezone)}`;
|
|
304
|
+
const bypass = availability.bypass.length === 0 ? "none" : availability.bypass.join(", ");
|
|
305
|
+
return [`availability ${mode}; quiet-hours bypass ${bypass}`];
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
function formatDigestScheduleStatus(s: StatusSnapshot): string[] {
|
|
310
|
+
const schedule = s.digestSchedule;
|
|
311
|
+
if (schedule === undefined) return [];
|
|
312
|
+
if (schedule.mode === "disabled") return ["next digest disabled"];
|
|
313
|
+
if (schedule.mode === "per-tick") return ["next digest every tick"];
|
|
314
|
+
if (schedule.mode === "due") return ["next digest due now"];
|
|
315
|
+
return [
|
|
316
|
+
schedule.nextAt === undefined
|
|
317
|
+
? "next digest could not be calculated"
|
|
318
|
+
: `next digest ${formatZonedMinute(schedule.nextAt, schedule.timezone)}`,
|
|
319
|
+
];
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* The shared-daemon visibility row (#545). When the daemon also serves sibling
|
|
324
|
+
* projects, tell the reader how many runs each has live, so they can tell "my
|
|
325
|
+
* fleet is idle" from "the process I am about to stop is busy" — the count is
|
|
326
|
+
* the point, so a "shared daemon" line that omits it is the bug re-expressed.
|
|
327
|
+
*/
|
|
328
|
+
function formatSiblingLive(siblings: { project: string; live: number }[]): string | undefined {
|
|
329
|
+
if (siblings.length === 0) return undefined;
|
|
330
|
+
const counts = siblings.map((s) => `${s.project} (${s.live} live)`).join(", ");
|
|
331
|
+
return `shared daemon also serves ${siblings.length} other project(s): ${counts}`;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
function formatProjectBody(
|
|
336
|
+
s: StatusSnapshot,
|
|
337
|
+
workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
|
|
338
|
+
now = Date.now(),
|
|
339
|
+
siblings: { project: string; live: number }[] = [],
|
|
340
|
+
): string {
|
|
341
|
+
const siblingLine = formatSiblingLive(siblings);
|
|
342
|
+
const lines = [
|
|
343
|
+
`project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
|
|
344
|
+
...(s.pauseReason === undefined ? [] : [`paused ${s.pauseReason}`]),
|
|
345
|
+
`config ${s.configPath}`,
|
|
346
|
+
`state ${s.stateDir}`,
|
|
347
|
+
...(siblingLine === undefined ? [] : [siblingLine]),
|
|
348
|
+
// The orchestrator-down degrade row: first-class in the body, present only
|
|
349
|
+
// while the incident is open, so recovery drops it (#288).
|
|
350
|
+
...(s.orchestratorDown === undefined
|
|
351
|
+
? []
|
|
352
|
+
: formatOrchestratorDown(s.orchestratorDown, now)),
|
|
353
|
+
...formatAvailabilityStatus(s),
|
|
354
|
+
...formatDigestScheduleStatus(s),
|
|
355
|
+
"",
|
|
356
|
+
"caps",
|
|
357
|
+
` workers ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
|
|
358
|
+
` issues today ${s.runsToday}`,
|
|
359
|
+
s.caps.dailySpendUsd === null
|
|
360
|
+
? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
|
|
361
|
+
: ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
|
|
362
|
+
// Its own row beside the spend row, never folded into it: they are two
|
|
363
|
+
// independent controls and an operator has to see which one stopped the
|
|
364
|
+
// fleet (#110).
|
|
365
|
+
` plan usage ${planUsageLine(s.planUsage)}`,
|
|
366
|
+
// The tracker's API budget, when the renderer could read it. Absent on a
|
|
367
|
+
// broken `gh`: one missing row, never a broken report (#188). Next to it,
|
|
368
|
+
// the refusals the tracker actually observed, so a budget that looks
|
|
369
|
+
// healthy next to every call being refused is still visible (#198).
|
|
370
|
+
...(s.github === undefined
|
|
371
|
+
? []
|
|
372
|
+
: [
|
|
373
|
+
` github graphql ${s.github.graphql.remaining}/${s.github.graphql.limit}, core ${s.github.core.remaining}/${s.github.core.limit} (graphql resets ${Math.max(0, Math.round((s.github.graphql.reset * 1000 - Date.now()) / 60_000))}m)` +
|
|
374
|
+
(s.ghRefusals === undefined || s.ghRefusals.count === 0
|
|
375
|
+
? ""
|
|
376
|
+
: ` — ${s.ghRefusals.count} refusal(s) in last 5m (last ${new Date(s.ghRefusals.latestAt ?? Date.now()).toISOString().slice(11, 19)}Z)`),
|
|
377
|
+
]),
|
|
378
|
+
// The daemon's own observed github traffic today (#198): a separate row
|
|
379
|
+
// from the polled budget above, so one missing `status` read never hides
|
|
380
|
+
// the other. `onCall` counts every spawn including ones that end 304, so
|
|
381
|
+
// `daemon` is spawns and `daemon-304` is the unbilled subset — billed ≈
|
|
382
|
+
// difference, and the polled budget row stays the authority (#203).
|
|
383
|
+
` github calls daemon ${s.ghCallsToday?.find((c) => c.source === "daemon")?.calls ?? 0} today` +
|
|
384
|
+
((s.ghCallsToday?.find((c) => c.source === "daemon-304")?.calls ?? 0) === 0
|
|
385
|
+
? ""
|
|
386
|
+
: ` (${s.ghCallsToday!.find((c) => c.source === "daemon-304")!.calls} free 304s)`),
|
|
387
|
+
// Label-projection ops the tracker has not applied yet (#201): GitHub is
|
|
388
|
+
// behind what the store decided, and the operator can see the lag instead
|
|
389
|
+
// of discovering it as a stale label or a missing one.
|
|
390
|
+
...(s.labelOps === undefined
|
|
391
|
+
? []
|
|
392
|
+
: [
|
|
393
|
+
` labels projection ${s.labelOps.pending} pending (oldest ${
|
|
394
|
+
s.labelOps.oldestAgeMs >= 60_000
|
|
395
|
+
? `${Math.round(s.labelOps.oldestAgeMs / 60_000)}m`
|
|
396
|
+
: `${Math.max(1, Math.round(s.labelOps.oldestAgeMs / 1000))}s`
|
|
397
|
+
})`,
|
|
398
|
+
]),
|
|
399
|
+
` new worker turns ${s.caps.workerMaxTurns}`,
|
|
400
|
+
...s.turnOverrides.map(
|
|
401
|
+
({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`,
|
|
402
|
+
),
|
|
403
|
+
` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
|
|
404
|
+
` failed attempts ${s.caps.maxAttemptsPerIssue}`,
|
|
405
|
+
` continuations ${s.caps.maxContinuationsPerIssue}`,
|
|
406
|
+
"",
|
|
407
|
+
...formatReleaseGrants(s.releaseGrants),
|
|
408
|
+
"",
|
|
409
|
+
formatDispatchSummary(s.dispatch),
|
|
410
|
+
"",
|
|
411
|
+
];
|
|
412
|
+
if (s.activeRuns.length === 0) {
|
|
413
|
+
lines.push("active runs (none)");
|
|
414
|
+
} else {
|
|
415
|
+
lines.push("active runs");
|
|
416
|
+
for (const r of s.activeRuns) {
|
|
417
|
+
const phase = workerPhases.get(r.issue);
|
|
418
|
+
const state = phase === "pausing" || phase === "paused" ? phase : r.state;
|
|
419
|
+
lines.push(
|
|
420
|
+
` #${r.issue} ${r.repo} ${state} attempt ${r.attempt} ` +
|
|
421
|
+
`${r.turns}/${r.maxTurns} turns ${r.spendUsd.toFixed(2)} ${r.branch}` +
|
|
422
|
+
(r.prUrl ? ` ${r.prUrl}` : ""),
|
|
423
|
+
);
|
|
424
|
+
// The orchestrator's Duty 1 reads this command, and a flagged run's
|
|
425
|
+
// escalation is deduplicated after one delivery — so this is where a
|
|
426
|
+
// flagged PR stays visible for as long as it is still open (#128).
|
|
427
|
+
const flagged = settlementFlagSummary(r.settlementFlags);
|
|
428
|
+
if (flagged !== undefined) lines.push(` ${flagged}`);
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
lines.push(...formatBaseHealth(s.baseHealth));
|
|
432
|
+
lines.push(...formatFreezes(s.freezes));
|
|
433
|
+
lines.push(...formatSalvagedRuns(s.salvagedRuns));
|
|
434
|
+
lines.push(...formatOpenReports(s.openReports));
|
|
435
|
+
lines.push(...formatDigestBacklog(s.digestBacklog));
|
|
436
|
+
if (s.liveWorkers > 0) {
|
|
437
|
+
lines.push(
|
|
438
|
+
"",
|
|
439
|
+
`deploy ${s.liveWorkers} live worker(s) — restart salvages dirty trees then orphans the rows; ` +
|
|
440
|
+
`hold and wait for workers 0/${s.caps.maxConcurrentWorkers} when you can drain instead`,
|
|
441
|
+
);
|
|
442
|
+
}
|
|
443
|
+
return lines.join("\n");
|
|
444
|
+
}
|
|
445
|
+
|
package/src/stop-provenance.ts
CHANGED
|
@@ -37,6 +37,59 @@ export interface StopProvenanceSpec {
|
|
|
37
37
|
* entry for drain-style restarts whose signal may never arrive, and at the
|
|
38
38
|
* lifecycle chokepoint immediately before signalling for every mediated stop).
|
|
39
39
|
*/
|
|
40
|
+
/**
|
|
41
|
+
* One configured project's live workload at a stop/restart refusal window.
|
|
42
|
+
*/
|
|
43
|
+
export interface LiveWorkload {
|
|
44
|
+
project: string;
|
|
45
|
+
/** Live (claimed/running) run count for this project. */
|
|
46
|
+
live: number;
|
|
47
|
+
/** The issues currently live, so a refusal can name exactly what would be orphaned. */
|
|
48
|
+
issues: number[];
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Every configured project's live workload. The refusal half of #545: the
|
|
53
|
+
* shared daemon serves all projects, so a stop/restart must account for every
|
|
54
|
+
* live run — not just the one the request happens to name — and it has to name
|
|
55
|
+
* the issues, not just a count, so an operator can see what `--force` would
|
|
56
|
+
* orphan. Read-only over the store; never writes.
|
|
57
|
+
*/
|
|
58
|
+
export function liveWorkload(
|
|
59
|
+
store: { liveRuns(project: string): readonly { issue: number }[] },
|
|
60
|
+
): LiveWorkload[] {
|
|
61
|
+
const cfg = loadConfig();
|
|
62
|
+
return cfg.projects.map((project) => {
|
|
63
|
+
const runs = store.liveRuns(project.name);
|
|
64
|
+
return { project: project.name, live: runs.length, issues: runs.map((r) => r.issue) };
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* The refusal gate for a stop/restart of the shared daemon (#545).
|
|
70
|
+
*
|
|
71
|
+
* Refuses while ANY configured project — the named one and its siblings alike
|
|
72
|
+
* — has a live run, since the process is global and stopping it orphans every
|
|
73
|
+
* project's workers. Throws with the live project(s) and issue numbers so the
|
|
74
|
+
* operator sees the whole blast radius, never a project-local count.
|
|
75
|
+
*
|
|
76
|
+
* `override` (an explicit `--force`) is the escape hatch that keeps a wedged
|
|
77
|
+
* daemon stoppable: the caller skips this gate on `--force` and records the
|
|
78
|
+
* override in the stop provenance rather than silencing it.
|
|
79
|
+
*/
|
|
80
|
+
export function refuseBusyDaemon(
|
|
81
|
+
workload: LiveWorkload[],
|
|
82
|
+
o: { override: boolean; verb: "stop" | "restart" },
|
|
83
|
+
): void {
|
|
84
|
+
const busy = workload.filter((w) => w.live > 0);
|
|
85
|
+
if (busy.length === 0 || o.override) return;
|
|
86
|
+
const live = busy.map((w) => `project ${w.project}: #${w.issues.join(", #")}`).join("; ");
|
|
87
|
+
throw new Error(
|
|
88
|
+
`refusing to ${o.verb} the shared daemon while work is live — ${live}. ` +
|
|
89
|
+
`Re-run with --force to ${o.verb} anyway (the override is recorded in the daemon-stop provenance).`,
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
|
|
40
93
|
export function buildStopProvenance(spec: StopProvenanceSpec): DaemonStopDraft {
|
|
41
94
|
const cfg = loadConfig();
|
|
42
95
|
const store = openStore(dbPath());
|