omp-conductor 0.18.1 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +106 -41
- package/REFERENCE.md +866 -31
- package/agents/to-spec.md +6 -2
- package/package.json +1 -1
- package/schema/config.schema.json +32 -1
- package/src/admission.ts +212 -26
- package/src/arm-challenge.ts +250 -57
- package/src/ask.ts +288 -1
- package/src/briefs/orchestrator.md +27 -13
- package/src/briefs/to-spec.md +6 -2
- package/src/cli.ts +127 -2
- package/src/command-help.ts +9 -1
- package/src/command-manifest.ts +52 -8
- package/src/commands/arm.ts +6 -2
- package/src/commands/context.ts +2 -0
- package/src/commands/intake.ts +4 -19
- package/src/commands/message.ts +26 -2
- package/src/commands/reconcile-units.ts +104 -0
- package/src/commands/release-composition.ts +232 -0
- package/src/commands/resume.ts +2 -27
- package/src/commands/setup.ts +101 -16
- package/src/commands/stats.ts +11 -30
- package/src/commands/tail.ts +31 -1
- package/src/commands/upgrade.ts +20 -3
- package/src/commands/verb.ts +2 -1
- package/src/commands/watch.ts +4 -17
- package/src/config-schema.ts +38 -6
- package/src/config.ts +103 -8
- package/src/credential-class.ts +366 -0
- package/src/daemon.ts +1368 -529
- package/src/dashboard/app.js +504 -2
- package/src/dashboard/controls.ts +336 -0
- package/src/dashboard/index.html +30 -0
- package/src/dashboard/server.ts +271 -30
- package/src/dashboard/style.css +116 -0
- package/src/dashboard/transcript.ts +173 -0
- package/src/decisions.ts +19 -11
- package/src/doctor.ts +431 -148
- package/src/escalate.ts +22 -11
- package/src/failure-class.ts +59 -0
- package/src/fleet.ts +587 -230
- package/src/host.ts +6 -455
- package/src/omp-settings.ts +19 -0
- package/src/omp.ts +40 -56
- package/src/orchestrator-tick.ts +564 -121
- package/src/pause.ts +233 -0
- package/src/session-host.ts +6 -41
- package/src/settlement.ts +159 -2
- package/src/setup-answers.ts +97 -0
- package/src/setup-host.ts +343 -1160
- package/src/setup-install.ts +204 -27
- package/src/setup-wizard.ts +252 -51
- package/src/setup.ts +87 -4
- package/src/spend-telemetry.ts +117 -0
- package/src/stats.ts +35 -0
- package/src/status-render.ts +485 -19
- package/src/store.ts +1229 -55
- package/src/telegram-freshness.ts +269 -0
- package/src/to-spec.ts +50 -2
- package/src/types.ts +759 -10
- package/src/unblock.ts +22 -0
- package/src/unit-reconcile.ts +303 -0
- package/src/upgrade-verify.ts +8 -1
- package/src/upgrade.ts +299 -12
- package/src/verbs/actions.ts +124 -10
- package/src/verbs/protocol.ts +70 -2
- package/src/verbs/server.ts +485 -11
- package/src/wake.ts +48 -0
- package/src/worker.ts +401 -14
package/src/status-render.ts
CHANGED
|
@@ -18,7 +18,8 @@
|
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
20
|
import { formatZonedMinute } from "./availability.ts";
|
|
21
|
-
import type
|
|
21
|
+
import { spendTelemetryDetail, type SpendTelemetryVerdict } from "./spend-telemetry.ts";
|
|
22
|
+
import type { DaemonStop, GroomingRecord, InstallSurfaceObservation, RunRecord } from "./types.ts";
|
|
22
23
|
import { settlementFlagSummary } from "./diff-flags.ts";
|
|
23
24
|
import type { CodeGraphHealth } from "./graph-health.ts";
|
|
24
25
|
import { formatDigestBacklog, formatOpenReports } from "./reports.ts";
|
|
@@ -37,7 +38,9 @@ import {
|
|
|
37
38
|
pauseProvenance,
|
|
38
39
|
type StatusSnapshot,
|
|
39
40
|
} from "./daemon.ts";
|
|
41
|
+
import { pidAlive } from "./escalate.ts";
|
|
40
42
|
import { formatQuarantinedRuns, formatSalvagedRuns } from "./settlement.ts";
|
|
43
|
+
import { formatVerbLedger } from "./verbs/ledger.ts";
|
|
41
44
|
|
|
42
45
|
// layered status
|
|
43
46
|
// ---------------------------------------------------------------------------
|
|
@@ -52,6 +55,16 @@ export type TicksLayer =
|
|
|
52
55
|
export type PaneLayer = "live" | "missing" | "unknown";
|
|
53
56
|
/** `unpinnable`: no tick config, so FLEET_CWD — the only path recovery reads — is unknown. */
|
|
54
57
|
export type RecoveryLayer = "pinned" | "clear" | "unpinnable";
|
|
58
|
+
/**
|
|
59
|
+
* A token count a human reads at a glance: `102.1k`, not `102105`.
|
|
60
|
+
*
|
|
61
|
+
* Thousands only — a run's output tokens are the one count here, and they land
|
|
62
|
+
* between thousands and low millions, where a single unit reads unambiguously.
|
|
63
|
+
*/
|
|
64
|
+
function formatCount(n: number): string {
|
|
65
|
+
return n < 1_000 ? String(n) : `${(n / 1_000).toFixed(1)}k`;
|
|
66
|
+
}
|
|
67
|
+
|
|
55
68
|
export type HerdrLayer = "active" | "inactive" | "unknown";
|
|
56
69
|
export type TelegramLayer = "ok" | "degraded" | "down" | "unconfigured" | "unprobed";
|
|
57
70
|
|
|
@@ -139,6 +152,14 @@ export type FleetDaemonProbe = {
|
|
|
139
152
|
const UNRESPONSIVE_HEALTHZ =
|
|
140
153
|
`unresponsive (healthz timed out after ${HEALTH_TIMEOUT_MS / 1000}s)`;
|
|
141
154
|
|
|
155
|
+
/** One line of a possibly-multi-line record field, bounded for a status row:
|
|
156
|
+
* the evidence and disposition an adjudicator writes are prose, and status is
|
|
157
|
+
* a scannable board — the full text lives in the durable row and the ledger. */
|
|
158
|
+
function firstLine(text: string, limit = 120): string {
|
|
159
|
+
const line = text.split("\n").find((candidate) => candidate.trim().length > 0)?.trim() ?? "";
|
|
160
|
+
return line.length <= limit ? line : `${line.slice(0, limit - 1)}…`;
|
|
161
|
+
}
|
|
162
|
+
|
|
142
163
|
function formatDaemonHealthz(probe: FleetDaemonProbe | undefined): string {
|
|
143
164
|
if (probe === undefined) return "unprobed";
|
|
144
165
|
switch (probe.project.kind) {
|
|
@@ -206,6 +227,241 @@ function formatLastStop(lastStop: DaemonStop | undefined): string[] {
|
|
|
206
227
|
}
|
|
207
228
|
|
|
208
229
|
|
|
230
|
+
// durable to-spec grooming lifecycle (#809)
|
|
231
|
+
// ---------------------------------------------------------------------------
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* The durable mechanical-hold reasons admission's reconcile writes as blocked
|
|
235
|
+
* rows (#735) — the lane/dependency holds that clear by themselves. Mirrors
|
|
236
|
+
* `BLOCKED_GROOMING_HOLDS` in `store.ts`; the status reads the stored reason
|
|
237
|
+
* strings so both sides of the contract stay on the store vocabulary.
|
|
238
|
+
*/
|
|
239
|
+
const MECHANICAL_GROOMING_REASONS: Record<string, true> = {
|
|
240
|
+
"file-lane": true,
|
|
241
|
+
"depends-on": true,
|
|
242
|
+
};
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* The to-spec refusal classes persisted as blocked rows (#772) — a result
|
|
246
|
+
* that failed validation is a mechanical block, never a verdict. Mirrors the
|
|
247
|
+
* failure kinds of `ToSpecFailure` in `to-spec.ts`.
|
|
248
|
+
*/
|
|
249
|
+
const REFUSED_GROOMING_REASONS: Record<string, true> = {
|
|
250
|
+
malformed: true,
|
|
251
|
+
"missing-source": true,
|
|
252
|
+
"stale-source": true,
|
|
253
|
+
};
|
|
254
|
+
|
|
255
|
+
/** The durable in-flight launch marker (#777) — a batch is running right now.
|
|
256
|
+
* Mirrors `TO_SPEC_IN_FLIGHT_REASON` in `orchestrator-tick.ts`. */
|
|
257
|
+
const GROOMING_IN_FLIGHT_REASON = "in-flight";
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* One project's durable grooming state as status lines, or nothing when there
|
|
261
|
+
* is nothing to report (#809).
|
|
262
|
+
*
|
|
263
|
+
* Renders the state already owned by the `grooming_verdicts` table plus the
|
|
264
|
+
* dispatch snapshot's parked count — each category below is derived from the
|
|
265
|
+
* project-scoped durable rows the caller passes, never from parsing evidence
|
|
266
|
+
* strings and never from a second cache:
|
|
267
|
+
*
|
|
268
|
+
* - `awaiting to-spec` — the routable queue candidates the last dispatch
|
|
269
|
+
* pass saw that no durable row covers: `routed − rows`, clamped at zero so
|
|
270
|
+
* stale rows (candidates since removed from the queue) cannot push it
|
|
271
|
+
* negative. `routed` is the same denominator the tick's grooming trigger
|
|
272
|
+
* reads (`summary.routed >= groomBelow`), and the fleet's queue-digest
|
|
273
|
+
* arithmetic already treats durable rows as current-queue facts
|
|
274
|
+
* (`claimable = routed − knownBlocked`), so this is the operator-facing
|
|
275
|
+
* half of the same subtraction. Parent/epic exclusions and backlog
|
|
276
|
+
* candidates that never entered the queue are tracker-side and not
|
|
277
|
+
* store-knowable; the row lines below carry the durable results either
|
|
278
|
+
* way.
|
|
279
|
+
* - `promotable` / `considered` / `blocked` — completed to-spec results
|
|
280
|
+
* (`blocked` rows whose reason is a groomer verdict or a product-judgement
|
|
281
|
+
* label, e.g. `needs-product-decision`).
|
|
282
|
+
* - `mechanically blocked` — admission's lane/dependency holds.
|
|
283
|
+
* - `refused` — to-spec results that failed validation (`malformed`,
|
|
284
|
+
* `missing-source`, `stale-source`), told apart from the holds so an
|
|
285
|
+
* operator sees whether the runway cannot move or a result cannot be
|
|
286
|
+
* trusted.
|
|
287
|
+
* - `in-flight` — a launched batch is running (#777).
|
|
288
|
+
* - `operator-parked` — the dispatch snapshot's parked count (#507).
|
|
289
|
+
*
|
|
290
|
+
* Rendering is read-only: nothing here reconciles, promotes, retries, or
|
|
291
|
+
* mutates grooming records. Empty categories are omitted — zero-value rows
|
|
292
|
+
* would be noise, and an absent block means a fleet with no grooming state.
|
|
293
|
+
*/
|
|
294
|
+
export interface GroomingStatusInput {
|
|
295
|
+
/** Project-scoped durable rows (`Store.groomingVerdicts`), issue-ascending. */
|
|
296
|
+
records: readonly GroomingRecord[];
|
|
297
|
+
/** Routable queue candidates from the last dispatch pass
|
|
298
|
+
* (`DispatchSummary.routed`). */
|
|
299
|
+
routed: number;
|
|
300
|
+
/** Operator-parked candidates from the last dispatch pass
|
|
301
|
+
* (`DispatchSummary.parked`). */
|
|
302
|
+
parked: number;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/** How many issue identifiers one category line may carry — enough to act on,
|
|
306
|
+
* never a queue dump. Mirrors the dispatch hold sampling. */
|
|
307
|
+
const GROOMING_STATUS_SAMPLE = 5;
|
|
308
|
+
|
|
309
|
+
/** One category line: label, count, and a bounded sample of issue identifiers
|
|
310
|
+
* with their durable reasons where the reason differs from the line label
|
|
311
|
+
* (e.g. `#13 already-done` under `considered`, `#19 malformed` under
|
|
312
|
+
* `refused`). Absent at zero. */
|
|
313
|
+
function groomingStatusLine(
|
|
314
|
+
label: string,
|
|
315
|
+
rows: readonly GroomingRecord[],
|
|
316
|
+
): string | undefined {
|
|
317
|
+
if (rows.length === 0) return undefined;
|
|
318
|
+
const sample = rows
|
|
319
|
+
.slice(0, GROOMING_STATUS_SAMPLE)
|
|
320
|
+
.map((row) => (row.reason === label ? `#${row.issue}` : `#${row.issue} ${row.reason}`));
|
|
321
|
+
const more = rows.length > GROOMING_STATUS_SAMPLE ? ", …" : "";
|
|
322
|
+
return ` ${label.padEnd(21)}${rows.length} ${sample.join(", ")}${more}`;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
export function formatGroomingStatus(input: GroomingStatusInput): string | undefined {
|
|
326
|
+
const { records, routed, parked } = input;
|
|
327
|
+
const promotable = records.filter((row) => row.verdict === "promotable");
|
|
328
|
+
const considered = records.filter((row) => row.verdict === "considered");
|
|
329
|
+
const blockedResults = records.filter(
|
|
330
|
+
(row) =>
|
|
331
|
+
row.verdict === "blocked" &&
|
|
332
|
+
MECHANICAL_GROOMING_REASONS[row.reason] !== true &&
|
|
333
|
+
REFUSED_GROOMING_REASONS[row.reason] !== true &&
|
|
334
|
+
row.reason !== GROOMING_IN_FLIGHT_REASON,
|
|
335
|
+
);
|
|
336
|
+
const mechanical = records.filter(
|
|
337
|
+
(row) => row.verdict === "blocked" && MECHANICAL_GROOMING_REASONS[row.reason] === true,
|
|
338
|
+
);
|
|
339
|
+
const refused = records.filter(
|
|
340
|
+
(row) => row.verdict === "blocked" && REFUSED_GROOMING_REASONS[row.reason] === true,
|
|
341
|
+
);
|
|
342
|
+
const inFlight = records.filter(
|
|
343
|
+
(row) => row.verdict === "blocked" && row.reason === GROOMING_IN_FLIGHT_REASON,
|
|
344
|
+
);
|
|
345
|
+
const awaiting = Math.max(0, routed - records.length);
|
|
346
|
+
const lines: string[] = [];
|
|
347
|
+
if (awaiting > 0) {
|
|
348
|
+
lines.push(` ${"awaiting to-spec".padEnd(21)}${awaiting} (of ${routed} routable)`);
|
|
349
|
+
}
|
|
350
|
+
for (const line of [
|
|
351
|
+
groomingStatusLine("promotable", promotable),
|
|
352
|
+
groomingStatusLine("considered", considered),
|
|
353
|
+
groomingStatusLine("blocked", blockedResults),
|
|
354
|
+
groomingStatusLine("mechanically blocked", mechanical),
|
|
355
|
+
groomingStatusLine("refused", refused),
|
|
356
|
+
groomingStatusLine("in-flight", inFlight),
|
|
357
|
+
]) {
|
|
358
|
+
if (line !== undefined) lines.push(line);
|
|
359
|
+
}
|
|
360
|
+
if (parked > 0) lines.push(` ${"operator-parked".padEnd(21)}${parked}`);
|
|
361
|
+
if (lines.length === 0) return undefined;
|
|
362
|
+
return ["grooming (to-spec)", ...lines].join("\n");
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/**
|
|
366
|
+
* The one install-surface fault the cheap surfaces are allowed to name (#919):
|
|
367
|
+
* two different releases live on one host at the same time.
|
|
368
|
+
*
|
|
369
|
+
* Deliberately narrower than `doctor`'s finding. An absent surface, a `local:`
|
|
370
|
+
* herdr link and a pin whose release could not be verified are all real
|
|
371
|
+
* observations, and all of them are *steady states* someone chose — a warning
|
|
372
|
+
* repeated every fifteen minutes would train an operator to ignore the row that
|
|
373
|
+
* matters. A CLI and an omp plugin on different versions is neither steady nor
|
|
374
|
+
* chosen: it is the 2026-08-22 shape, where the plugin sat on a withdrawn
|
|
375
|
+
* 0.18.1 beside a 0.18.0 daemon for a day with nothing saying so.
|
|
376
|
+
*
|
|
377
|
+
* Pure, so the tick and `status` read the same recorded row and agree by
|
|
378
|
+
* construction rather than by two copies of a rule.
|
|
379
|
+
*/
|
|
380
|
+
export function installSurfaceMismatch(
|
|
381
|
+
observation: InstallSurfaceObservation | undefined,
|
|
382
|
+
): string | undefined {
|
|
383
|
+
if (observation === undefined) return undefined;
|
|
384
|
+
const { cliVersion, ompVersion } = observation;
|
|
385
|
+
if (ompVersion === undefined || ompVersion === cliVersion) return undefined;
|
|
386
|
+
return (
|
|
387
|
+
`installed surfaces disagree: omp plugin ${ompVersion}, CLI/daemon ${cliVersion}` +
|
|
388
|
+
`${observation.herdrSource === undefined ? "" : `, herdr ${observation.herdrSource}`}` +
|
|
389
|
+
` — run \`omp-conductor upgrade --to ${cliVersion}\` so every surface carries one identity`
|
|
390
|
+
);
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/** The `status` row for the recorded observation (#919): nothing on agreement,
|
|
394
|
+
* the mismatch when there is one, and an honest "not observed yet" before the
|
|
395
|
+
* first dispatch pass has looked. Never a probe — the whole point of the
|
|
396
|
+
* recorded row is that reading it spawns nothing. */
|
|
397
|
+
export function installSurfaceStatusLine(
|
|
398
|
+
observation: InstallSurfaceObservation | undefined,
|
|
399
|
+
): string | undefined {
|
|
400
|
+
if (observation === undefined) return "surfaces not observed yet (the next dispatch pass records them)";
|
|
401
|
+
const mismatch = installSurfaceMismatch(observation);
|
|
402
|
+
return mismatch === undefined ? undefined : `surfaces ${mismatch}`;
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
/**
|
|
406
|
+
* The `status` row for the recorded `omp-telegram` versions (#961).
|
|
407
|
+
*
|
|
408
|
+
* Same discipline as the row above: rendered from what the dispatch pass
|
|
409
|
+
* already recorded, so reading it spawns nothing and touches no registry — and
|
|
410
|
+
* silent on agreement, because a row that always prints is a row nobody reads.
|
|
411
|
+
*
|
|
412
|
+
* Deliberately its own line rather than a clause on the `telegram` row. That
|
|
413
|
+
* row's severity ladder carries one remedy at a time (token, then inbound, then
|
|
414
|
+
* approval surface, then profile), and a stale install is a different kind of
|
|
415
|
+
* fact about a different thing: the bot can be perfectly healthy while the
|
|
416
|
+
* plugin driving it predates the contract the brief mandates. Folding it in
|
|
417
|
+
* would either bury it under a louder rung or displace one.
|
|
418
|
+
*/
|
|
419
|
+
export function telegramPluginStatusLine(
|
|
420
|
+
observation: InstallSurfaceObservation | undefined,
|
|
421
|
+
): string | undefined {
|
|
422
|
+
if (observation === undefined) return undefined;
|
|
423
|
+
const { telegramInstalled, telegramDaemon, telegramPublished } = observation;
|
|
424
|
+
// Nothing recorded at all: this pass predates the read, or none answered.
|
|
425
|
+
// Silent rather than "unverified" — a fleet that does not use Telegram must
|
|
426
|
+
// not grow a permanent row about a plugin it does not have.
|
|
427
|
+
if (telegramInstalled === undefined) return undefined;
|
|
428
|
+
if (telegramPublished !== undefined && telegramPublished !== telegramInstalled) {
|
|
429
|
+
return (
|
|
430
|
+
`tg-plugin omp-telegram ${telegramInstalled} installed, ${telegramPublished} published` +
|
|
431
|
+
`${telegramDaemon === undefined || telegramDaemon === telegramInstalled ? "" : `, daemon ${telegramDaemon}`}` +
|
|
432
|
+
" — install it and restart its daemon (#882's target ladder ships in the newer one)"
|
|
433
|
+
);
|
|
434
|
+
}
|
|
435
|
+
if (telegramDaemon !== undefined && telegramDaemon !== telegramInstalled) {
|
|
436
|
+
return (
|
|
437
|
+
`tg-plugin the running omp-telegram daemon is ${telegramDaemon}, installed is ${telegramInstalled}` +
|
|
438
|
+
" — restart it so it serves the installed code"
|
|
439
|
+
);
|
|
440
|
+
}
|
|
441
|
+
return undefined;
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
/**
|
|
445
|
+
* The `status` row qualifying the spend figure above it (#970).
|
|
446
|
+
*
|
|
447
|
+
* Silent when telemetry is healthy, for the same reason the install-surface row
|
|
448
|
+
* is: a row that always prints is a row nobody reads. Loud when the cap's input
|
|
449
|
+
* is going absent, because `spend today $0.64 / $35.00` reads as headroom and
|
|
450
|
+
* the only surface that said otherwise was an on-demand `doctor` run — the exact
|
|
451
|
+
* gap #919 closed for install surfaces, on a control that stops the fleet
|
|
452
|
+
* spending money.
|
|
453
|
+
*
|
|
454
|
+
* The wording is the shared one, so this row and `doctor`'s finding cannot state
|
|
455
|
+
* the same fact differently.
|
|
456
|
+
*/
|
|
457
|
+
export function spendTelemetryStatusLine(
|
|
458
|
+
verdict: SpendTelemetryVerdict | undefined,
|
|
459
|
+
): string | undefined {
|
|
460
|
+
if (verdict === undefined) return undefined;
|
|
461
|
+
const detail = spendTelemetryDetail(verdict);
|
|
462
|
+
return detail === undefined ? undefined : ` spend telemetry ${detail}`;
|
|
463
|
+
}
|
|
464
|
+
|
|
209
465
|
export function formatFleetStatus(
|
|
210
466
|
s: StatusSnapshot,
|
|
211
467
|
layers: FleetLayers,
|
|
@@ -220,6 +476,7 @@ export function formatFleetStatus(
|
|
|
220
476
|
intake: string | undefined = undefined,
|
|
221
477
|
lastStop: DaemonStop | undefined = undefined,
|
|
222
478
|
siblings: { project: string; live: number }[] = [],
|
|
479
|
+
grooming: string | undefined = undefined,
|
|
223
480
|
): string {
|
|
224
481
|
const tickLine =
|
|
225
482
|
layers.ticksDetail === undefined
|
|
@@ -286,7 +543,19 @@ export function formatFleetStatus(
|
|
|
286
543
|
const prov = pauseProvenance(s.project);
|
|
287
544
|
if (prov !== undefined) {
|
|
288
545
|
const reason = prov.reason === undefined ? "" : ` — "${prov.reason}"`;
|
|
289
|
-
|
|
546
|
+
// A fence that declared an owning process is judged against it
|
|
547
|
+
// (#938): a setup transaction's fence outliving the setup is a
|
|
548
|
+
// stalled fleet that reads exactly like a deliberate hold. Never
|
|
549
|
+
// cleared here — status reports, and the operator decides.
|
|
550
|
+
const owner =
|
|
551
|
+
prov.owner === undefined
|
|
552
|
+
? ""
|
|
553
|
+
: pidAlive(prov.owner)
|
|
554
|
+
? `, held by a live process (pid ${prov.owner})`
|
|
555
|
+
: `, ABANDONED — its process (pid ${prov.owner}) is gone; clear it with \`omp-conductor resume${
|
|
556
|
+
s.project === undefined ? "" : ` --project ${s.project}`
|
|
557
|
+
}\``;
|
|
558
|
+
return `dispatch paused (source: ${prov.source}${reason}${owner})`;
|
|
290
559
|
}
|
|
291
560
|
return isPaused(s.project) && pausedAt(s.project) === undefined
|
|
292
561
|
? "dispatch paused (unparseable sentinel — new work and completion mutations refused; release gates remain available)"
|
|
@@ -306,6 +575,7 @@ export function formatFleetStatus(
|
|
|
306
575
|
...(decisions === undefined ? [] : [decisions]),
|
|
307
576
|
...(failureClasses === undefined ? [] : [failureClasses]),
|
|
308
577
|
...(intake === undefined ? [] : [intake]),
|
|
578
|
+
...(grooming === undefined ? [] : [grooming]),
|
|
309
579
|
...(graphBlock === undefined ? [] : [graphBlock]),
|
|
310
580
|
daemonBlock,
|
|
311
581
|
...formatLastStop(lastStop),
|
|
@@ -407,6 +677,15 @@ function formatProjectBody(
|
|
|
407
677
|
...(s.orchestratorDown === undefined
|
|
408
678
|
? []
|
|
409
679
|
: formatOrchestratorDown(s.orchestratorDown, now)),
|
|
680
|
+
// The recorded install-surface observation (#919): silent on agreement,
|
|
681
|
+
// loud on a proven mismatch, honest before the first pass has looked.
|
|
682
|
+
...(installSurfaceStatusLine(s.installSurfaces) === undefined
|
|
683
|
+
? []
|
|
684
|
+
: [installSurfaceStatusLine(s.installSurfaces) as string]),
|
|
685
|
+
// Same recorded pass, same silence-on-agreement rule (#961).
|
|
686
|
+
...(telegramPluginStatusLine(s.installSurfaces) === undefined
|
|
687
|
+
? []
|
|
688
|
+
: [telegramPluginStatusLine(s.installSurfaces) as string]),
|
|
410
689
|
...formatReportingStatus(s),
|
|
411
690
|
...formatAvailabilityStatus(s),
|
|
412
691
|
...formatDigestScheduleStatus(s),
|
|
@@ -414,9 +693,26 @@ function formatProjectBody(
|
|
|
414
693
|
"caps",
|
|
415
694
|
` workers ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
|
|
416
695
|
` issues today ${s.runsToday}`,
|
|
696
|
+
// "estimated" is not hedging (#851): this figure is OMP's own local price
|
|
697
|
+
// for the tokens the session recorded, not the provider's bill. On
|
|
698
|
+
// 2026-08-21 the local estimate read $25.16 while roughly $16 of provider
|
|
699
|
+
// credit actually moved, so a row that called it "spend" was asserting
|
|
700
|
+
// something conductor cannot see. The reservation is shown beside it
|
|
701
|
+
// because that, not spend-to-date, is what the next admission subtracts.
|
|
417
702
|
s.caps.dailySpendUsd === null
|
|
418
|
-
? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
|
|
419
|
-
: ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}
|
|
703
|
+
? ` spend today $${s.spendTodayUsd.toFixed(2)} estimated (no daily cap)`
|
|
704
|
+
: ` spend today $${s.spendTodayUsd.toFixed(2)} estimated / $${s.caps.dailySpendUsd.toFixed(2)}` +
|
|
705
|
+
(s.reservedSpendUsd === undefined || s.reservedSpendUsd === 0
|
|
706
|
+
? ""
|
|
707
|
+
: ` (+$${s.reservedSpendUsd.toFixed(2)} reserved by runs in flight)`),
|
|
708
|
+
// What that figure is built on (#970). Silent when telemetry is healthy;
|
|
709
|
+
// its own row when it is not, because the number above reads as headroom
|
|
710
|
+
// and the cap subtracts exactly it. Measured 2026-08-22: 9 of 12 working
|
|
711
|
+
// runs reported $0.00 and the only surface that could have said so was an
|
|
712
|
+
// on-demand `doctor` run — whose predicate would not have fired either.
|
|
713
|
+
...(spendTelemetryStatusLine(s.spendTelemetry) === undefined
|
|
714
|
+
? []
|
|
715
|
+
: [spendTelemetryStatusLine(s.spendTelemetry) as string]),
|
|
420
716
|
// Its own row beside the spend row, never folded into it: they are two
|
|
421
717
|
// independent controls and an operator has to see which one stopped the
|
|
422
718
|
// fleet (#110).
|
|
@@ -474,11 +770,93 @@ function formatProjectBody(
|
|
|
474
770
|
formatDispatchSummary(s.dispatch),
|
|
475
771
|
"",
|
|
476
772
|
];
|
|
773
|
+
// The two lifecycle kinds are rendered as two blocks (#898). An operator
|
|
774
|
+
// looking at "a branch holding a file" has to be able to tell "a worker is
|
|
775
|
+
// editing this right now" from "durable work nobody is touching, waiting on
|
|
776
|
+
// review, merge or recovery" — the same distinction #899 made load-bearing in
|
|
777
|
+
// admission, where only the first kind occupies files. One block for both
|
|
778
|
+
// read as though every preserved branch had a live editor, which is exactly
|
|
779
|
+
// the misreading that let a settled artifact look like a busy worker.
|
|
780
|
+
//
|
|
781
|
+
// Membership comes from the snapshot's `leasedRunIds` (`Store.leasedRuns`),
|
|
782
|
+
// never from a state check here: a dispatched review revision is a lease on a
|
|
783
|
+
// row whose state still says `pushed-green`, so a renderer that classified by
|
|
784
|
+
// state alone would call a live rewrite "preserved".
|
|
785
|
+
//
|
|
786
|
+
// Membership is the UNION of the run's own live state and the recorded ids,
|
|
787
|
+
// never the ids alone. A live worker is a lease by definition, so the field
|
|
788
|
+
// exists only to ADD the case a state read cannot see — a dispatched revision
|
|
789
|
+
// on a row that still says `pushed-green`. Reading it as the whole truth would
|
|
790
|
+
// let an absent or stale field print a live worker under "worker-free", which
|
|
791
|
+
// is the exact misreading this block exists to stop, and the union can never
|
|
792
|
+
// over-report either: `leasedRuns` is itself live rows plus those revisions.
|
|
793
|
+
//
|
|
794
|
+
// The two state names are spelled here rather than importing `LIVE_STATES`:
|
|
795
|
+
// this module is a pure renderer and does not pull in the store (and its
|
|
796
|
+
// sqlite binding) to read two string literals.
|
|
797
|
+
const leased = new Set(s.leasedRunIds ?? []);
|
|
798
|
+
const isLease = (r: RunRecord): boolean =>
|
|
799
|
+
r.state === "claimed" || r.state === "running" || leased.has(r.id);
|
|
800
|
+
const leases = s.activeRuns.filter((r) => isLease(r));
|
|
801
|
+
const preserved = s.activeRuns.filter((r) => !isLease(r));
|
|
802
|
+
// The per-run annotations both blocks carry: they are facts about the run,
|
|
803
|
+
// not about which lifecycle kind it is, and a flagged or merge-blocked
|
|
804
|
+
// artifact is if anything MORE interesting once nobody is editing it.
|
|
805
|
+
const annotations = (r: RunRecord): string[] => {
|
|
806
|
+
const out: string[] = [];
|
|
807
|
+
// The orchestrator's Duty 1 reads this command, and a flagged run's
|
|
808
|
+
// escalation is deduplicated after one delivery — so this is where a
|
|
809
|
+
// flagged PR stays visible for as long as it is still open (#128).
|
|
810
|
+
const flagged = settlementFlagSummary(r.settlementFlags);
|
|
811
|
+
if (flagged !== undefined) out.push(` ${flagged}`);
|
|
812
|
+
// The exact-head merge blocker (#888): a green PR whose head carries
|
|
813
|
+
// unresolved durable review evidence would otherwise read as merge-ready
|
|
814
|
+
// while `conductor_pr_merge` refuses it. Same rows the verb reads,
|
|
815
|
+
// rendered where Duty 1 already looks.
|
|
816
|
+
// The review-ceiling adjudication for this run's exact head (#874). Matched
|
|
817
|
+
// on PR + head, never on the run: a verdict describes the diff it read, so a
|
|
818
|
+
// run that has since pushed a corrected head must not appear to carry the
|
|
819
|
+
// older head's decision.
|
|
820
|
+
const adjudication =
|
|
821
|
+
r.prUrl === undefined || r.headSha === undefined
|
|
822
|
+
? undefined
|
|
823
|
+
: s.reviewAdjudications?.find(
|
|
824
|
+
(a) => a.prUrl === r.prUrl && a.headSha.toLowerCase() === r.headSha?.toLowerCase(),
|
|
825
|
+
);
|
|
826
|
+
if (adjudication !== undefined) {
|
|
827
|
+
// The role asked for AND the model that actually ran it: those differ
|
|
828
|
+
// exactly when something is misconfigured, and then the state is
|
|
829
|
+
// `unavailable-model` and the difference is the whole finding (#875).
|
|
830
|
+
const launched =
|
|
831
|
+
adjudication.provenance === undefined
|
|
832
|
+
? `role ${adjudication.role} (not launched)`
|
|
833
|
+
: `role ${adjudication.role} → ${adjudication.provenance.model}${
|
|
834
|
+
adjudication.provenance.provider === undefined ? "" : ` (${adjudication.provenance.provider})`
|
|
835
|
+
}`;
|
|
836
|
+
out.push(
|
|
837
|
+
` adjudication ${adjudication.state} ${launched}` +
|
|
838
|
+
(adjudication.evidence === undefined ? "" : ` ${firstLine(adjudication.evidence)}`) +
|
|
839
|
+
(adjudication.disposition === undefined ? "" : ` → ${firstLine(adjudication.disposition)}`),
|
|
840
|
+
);
|
|
841
|
+
}
|
|
842
|
+
const blocked = s.mergeBlockers?.[r.id];
|
|
843
|
+
if (blocked !== undefined)
|
|
844
|
+
out.push(
|
|
845
|
+
` merge blocked: review round ${blocked.round} ${
|
|
846
|
+
blocked.state === "pending"
|
|
847
|
+
? "queued at this head"
|
|
848
|
+
: blocked.state === "crashed"
|
|
849
|
+
? "dispatched and unfinished at this head"
|
|
850
|
+
: "failed at this head"
|
|
851
|
+
} — unresolved findings; push a corrected head, or record a conductor_pr_review_clear for this exact head, before merge`,
|
|
852
|
+
);
|
|
853
|
+
return out;
|
|
854
|
+
};
|
|
477
855
|
if (s.activeRuns.length === 0) {
|
|
478
|
-
lines.push("
|
|
856
|
+
lines.push("mutation leases (none)");
|
|
479
857
|
} else {
|
|
480
|
-
lines.push("
|
|
481
|
-
for (const r of
|
|
858
|
+
lines.push(leases.length === 0 ? "mutation leases (none)" : "mutation leases");
|
|
859
|
+
for (const r of leases) {
|
|
482
860
|
const phase = workerPhases.get(r.issue);
|
|
483
861
|
// A run in a live review round reads `review-revision N`, distinct from
|
|
484
862
|
// a failure and from an ordinary continuation, with the round number
|
|
@@ -489,7 +867,7 @@ function formatProjectBody(
|
|
|
489
867
|
const state = paused
|
|
490
868
|
? phase
|
|
491
869
|
: round !== undefined
|
|
492
|
-
? `review-revision ${round}`
|
|
870
|
+
? `review-revision ${round.round}`
|
|
493
871
|
: r.state;
|
|
494
872
|
// Turn rate and cap projection (#730/#767): a stalled run and a fast one
|
|
495
873
|
// used to render identically as a bare turn count. The rate is the
|
|
@@ -506,24 +884,103 @@ function formatProjectBody(
|
|
|
506
884
|
// elapsed includes banked pause time and its state already says
|
|
507
885
|
// `paused`.
|
|
508
886
|
const elapsedMs = Math.max(0, now - r.startedAt);
|
|
887
|
+
// A review revision resumes the SAME run and session, so `turns` and
|
|
888
|
+
// `startedAt` are cumulative over the whole attempt — and reading them as
|
|
889
|
+
// this round's is how a revision recorded two minutes ago was read as a
|
|
890
|
+
// worker stuck for two hours (#802, #797 on 2026-08-19). Nothing is reset:
|
|
891
|
+
// the attempt totals are the cap evidence and stay exactly as recorded.
|
|
892
|
+
// They are labelled `cumulative` instead, and the round's own age is shown
|
|
893
|
+
// beside them from the durable dispatch instant.
|
|
894
|
+
//
|
|
895
|
+
// The rate and cap projection are deliberately dropped for a live round:
|
|
896
|
+
// both divide cumulative turns by cumulative elapsed, so neither describes
|
|
897
|
+
// the phase the line is naming, and a number attributed to the wrong phase
|
|
898
|
+
// is worse than no number. The attempt's own cap remains visible as
|
|
899
|
+
// `N/M turns cumulative`.
|
|
509
900
|
const progress = paused
|
|
510
901
|
? ""
|
|
511
|
-
:
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
902
|
+
: round !== undefined
|
|
903
|
+
? ` cumulative ${formatDownDuration(elapsedMs)} elapsed cumulative round ${
|
|
904
|
+
round.round
|
|
905
|
+
} dispatched ${formatDownDuration(Math.max(0, now - round.dispatchedAt))} ago`
|
|
906
|
+
: ` ${formatDownDuration(elapsedMs)} elapsed ${(
|
|
907
|
+
(r.turns * 60_000) / Math.max(elapsedMs, 1_000)
|
|
908
|
+
).toFixed(1)} turns/min avg` +
|
|
909
|
+
` projects ${Math.round(
|
|
910
|
+
(r.turns * s.caps.workerWallClockMs) / Math.max(elapsedMs, 1_000),
|
|
911
|
+
)} turns at cap`;
|
|
517
912
|
lines.push(
|
|
518
913
|
` #${r.issue} ${r.repo} ${state} attempt ${r.attempt} ` +
|
|
519
914
|
`${r.turns}/${r.maxTurns} turns${progress} ${r.spendUsd.toFixed(2)} ${r.branch}` +
|
|
520
915
|
(r.prUrl ? ` ${r.prUrl}` : ""),
|
|
521
916
|
);
|
|
522
|
-
//
|
|
523
|
-
//
|
|
524
|
-
//
|
|
525
|
-
|
|
526
|
-
|
|
917
|
+
// What the lease actually holds, named where the operator is already
|
|
918
|
+
// looking: the run this is (an issue can have several attempts, and only
|
|
919
|
+
// one of them holds the lease), how long it has held it, the read the
|
|
920
|
+
// interlock uses to prove occupancy, and the files it occupies.
|
|
921
|
+
//
|
|
922
|
+
// The source is not a guess: `probeRunLane` reads a live checkout when the
|
|
923
|
+
// row has one and the mirror branch when it does not, which is exactly
|
|
924
|
+
// this condition — a dispatched revision on a settled row has no worktree,
|
|
925
|
+
// and its lane comes from the branch.
|
|
926
|
+
//
|
|
927
|
+
// The file list is the durable declaration admission persisted at dispatch
|
|
928
|
+
// (#744) — the half that occupies before anything is written. What the run
|
|
929
|
+
// has additionally touched is a git read, deliberately not taken here:
|
|
930
|
+
// status is a recorded answer, never a subprocess per render (#919). A
|
|
931
|
+
// lease with no declaration says so, because "no files listed" and "no
|
|
932
|
+
// declaration made" invite different actions.
|
|
933
|
+
lines.push(
|
|
934
|
+
` lease run ${r.id.slice(0, 8)} ${formatDownDuration(Math.max(0, now - r.startedAt))} held ` +
|
|
935
|
+
`${r.worktree === "" ? "branch" : "worktree"} ` +
|
|
936
|
+
(r.lane === undefined || r.lane.files.length === 0
|
|
937
|
+
? "no declared lane (probe-only occupancy)"
|
|
938
|
+
: `occupies ${r.lane.files.join(", ")}`),
|
|
939
|
+
);
|
|
940
|
+
// Whether the operator can watch this worker, named rather than absent
|
|
941
|
+
// (#841). A live run with no pane is a fleet running blind, and silence
|
|
942
|
+
// reads identical to "there is nothing to see" — so degraded says so and
|
|
943
|
+
// carries the reason the launch or the last reconcile recorded.
|
|
944
|
+
lines.push(
|
|
945
|
+
r.paneId === undefined
|
|
946
|
+
? ` pane degraded${r.paneUnavailable === undefined ? "" : ` — ${r.paneUnavailable}`}`
|
|
947
|
+
: ` pane ${r.paneId}${r.paneLabel === undefined ? "" : ` ${r.paneLabel}`}`,
|
|
948
|
+
);
|
|
949
|
+
// The fact that explains an expensive-in-time, cheap-in-dollars run
|
|
950
|
+
// (#518). Shown only when the share is disproportionate, because a run
|
|
951
|
+
// reasoning normally needs no annotation and a line printed for every run
|
|
952
|
+
// is a line nobody reads. The threshold is deliberately high: at 80% the
|
|
953
|
+
// turn cap is already unreachable and the wall clock is the only ceiling
|
|
954
|
+
// that can fire, which is the operator's actual decision.
|
|
955
|
+
if (r.outputTokens !== undefined && r.outputTokens > 0 && r.reasoningTokens !== undefined) {
|
|
956
|
+
const share = r.reasoningTokens / r.outputTokens;
|
|
957
|
+
if (share >= 0.8) {
|
|
958
|
+
lines.push(
|
|
959
|
+
` tokens ${Math.round(share * 100)}% of ${formatCount(r.outputTokens)} output tokens ` +
|
|
960
|
+
`were reasoning — deliberating, not stalled; wall clock is the binding cap`,
|
|
961
|
+
);
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
lines.push(...annotations(r));
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
// Preserved artifacts: durable work with no worker and no revision behind it.
|
|
968
|
+
// Rendered without a turn rate or a cap projection on purpose — both describe
|
|
969
|
+
// a session in progress, and printing them beside work nobody is running is
|
|
970
|
+
// how a settled artifact came to read as a busy one. What matters here is how
|
|
971
|
+
// long it has been waiting and on what: a PR to review and merge, or (with no
|
|
972
|
+
// PR) a branch `conductor_pr_recover` can still publish.
|
|
973
|
+
if (preserved.length > 0) {
|
|
974
|
+
lines.push("preserved artifacts (worker-free — nothing is editing these)");
|
|
975
|
+
for (const r of preserved) {
|
|
976
|
+
const since = r.endedAt ?? r.startedAt;
|
|
977
|
+
lines.push(
|
|
978
|
+
` #${r.issue} ${r.repo} ${r.state} attempt ${r.attempt} ` +
|
|
979
|
+
`${formatDownDuration(Math.max(0, now - since))} waiting ${r.turns} turns spent ` +
|
|
980
|
+
`${r.spendUsd.toFixed(2)} ${r.branch}` +
|
|
981
|
+
(r.prUrl === undefined ? " no PR — recoverable with conductor_pr_recover" : ` ${r.prUrl}`),
|
|
982
|
+
);
|
|
983
|
+
lines.push(...annotations(r));
|
|
527
984
|
}
|
|
528
985
|
}
|
|
529
986
|
lines.push(...formatBaseHealth(s.baseHealth));
|
|
@@ -532,6 +989,15 @@ function formatProjectBody(
|
|
|
532
989
|
lines.push(...formatQuarantinedRuns(s.quarantinedRuns));
|
|
533
990
|
lines.push(...formatOpenReports(s.openReports));
|
|
534
991
|
lines.push(...formatDigestBacklog(s.digestBacklog));
|
|
992
|
+
// The mediated-verb ledger (#972). Absent from this renderer since the block
|
|
993
|
+
// was written (#133) — it was wired into `daemon.ts`'s copy, which was
|
|
994
|
+
// already not the live renderer, so the read below was paid for on every
|
|
995
|
+
// status call and dropped. It belongs here: the count is how an operator sees
|
|
996
|
+
// that a mediated act was refused without going to the sqlite file, and
|
|
997
|
+
// #968's refusal class (41% of this fleet's refusals) went unnoticed for a
|
|
998
|
+
// fortnight precisely because nothing on this surface said so. Self-silencing
|
|
999
|
+
// on an empty ledger, so a fresh fleet gains no row.
|
|
1000
|
+
lines.push(...formatVerbLedger(s.verbLedger));
|
|
535
1001
|
if (s.liveWorkers > 0) {
|
|
536
1002
|
lines.push(
|
|
537
1003
|
"",
|