omp-conductor 0.19.2 → 0.19.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +1 -1
- package/package.json +1 -1
- package/schema/config.schema.json +13 -1
- package/src/arm-challenge.ts +92 -16
- package/src/board.ts +50 -9
- package/src/cli.ts +2 -0
- package/src/command-manifest.ts +10 -0
- package/src/commands/arm.ts +30 -2
- package/src/commands/companion.ts +103 -0
- package/src/commands/context.ts +1 -0
- package/src/commands/worker.ts +1 -0
- package/src/companion-view.ts +81 -0
- package/src/config-schema.ts +7 -1
- package/src/config.ts +6 -3
- package/src/daemon.ts +197 -14
- package/src/dashboard/controls.ts +1 -1
- package/src/doctor.ts +126 -7
- package/src/escalate.ts +47 -1
- package/src/failure-class.fixture.json +546 -0
- package/src/failure-class.ts +192 -0
- package/src/fleet.ts +275 -27
- package/src/orchestrator-tick.ts +115 -34
- package/src/settlement.ts +167 -0
- package/src/setup.ts +6 -4
- package/src/spend-telemetry.ts +74 -2
- package/src/status-render.ts +25 -6
- package/src/store.ts +38 -2
- package/src/types.ts +39 -2
- package/src/usage.ts +25 -0
package/src/daemon.ts
CHANGED
|
@@ -38,7 +38,12 @@ import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-
|
|
|
38
38
|
import { graphHint } from "./graph.ts";
|
|
39
39
|
import { hostConstraintsNotice } from "./host.ts";
|
|
40
40
|
import { acquireOnceLease, healthCheck, livingDaemon, probeUnit, SYSTEMD_UNIT } from "./lifecycle.ts";
|
|
41
|
-
import {
|
|
41
|
+
import {
|
|
42
|
+
readTickRequestReason,
|
|
43
|
+
requestImmediateTick,
|
|
44
|
+
resolveTickConfigCwd,
|
|
45
|
+
STALL_MARKER_FILE,
|
|
46
|
+
} from "./orchestrator-tick.ts";
|
|
42
47
|
import { SPEND_SAMPLE_ROWS, SPEND_SAMPLE_RUNS, runDoctor } from "./doctor.ts";
|
|
43
48
|
import { judgeSpendTelemetry, type SpendTelemetryVerdict } from "./spend-telemetry.ts";
|
|
44
49
|
import {
|
|
@@ -62,6 +67,7 @@ import {
|
|
|
62
67
|
herdrPaneOmpStarts,
|
|
63
68
|
openWorkerPane,
|
|
64
69
|
reconcileWorkerPanes,
|
|
70
|
+
PANE_REATTEMPT_MAX,
|
|
65
71
|
releaseOrphanedWorkerPane,
|
|
66
72
|
releaseWorkerPane,
|
|
67
73
|
reportWorkerPaneState,
|
|
@@ -1424,11 +1430,20 @@ export interface WorkerControlRegistry {
|
|
|
1424
1430
|
runId: string,
|
|
1425
1431
|
onPhase?: (phase: WorkerPausePhase) => void,
|
|
1426
1432
|
): WorkerControlSlot;
|
|
1427
|
-
|
|
1433
|
+
/** `source` names who asked (board/cli/dashboard) — recorded as pause
|
|
1434
|
+
* provenance so "who paused this and when" is answerable later (#997). */
|
|
1435
|
+
pause(project: string, issue: number, source: string): Promise<WorkerControlResult>;
|
|
1428
1436
|
resume(project: string, issue: number): WorkerControlResult;
|
|
1429
1437
|
stop(project: string, issue: number, reason: string): Promise<WorkerControlResult>;
|
|
1430
|
-
/** Live runs whose phase is not `running` — what /healthz and the board
|
|
1431
|
-
|
|
1438
|
+
/** Live runs whose phase is not `running` — what /healthz and the board
|
|
1439
|
+
* show — each carrying its pause provenance when one was recorded (#997). */
|
|
1440
|
+
snapshot(project: string): {
|
|
1441
|
+
issue: number;
|
|
1442
|
+
runId: string;
|
|
1443
|
+
phase: WorkerPausePhase;
|
|
1444
|
+
source?: string;
|
|
1445
|
+
pausedAtMs?: number;
|
|
1446
|
+
}[];
|
|
1432
1447
|
}
|
|
1433
1448
|
|
|
1434
1449
|
/** Authoritative controls for sessions owned by this daemon process. */
|
|
@@ -1442,6 +1457,8 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
1442
1457
|
stopError?: string;
|
|
1443
1458
|
finished: PromiseWithResolvers<void>;
|
|
1444
1459
|
onPhase?: (phase: WorkerPausePhase) => void;
|
|
1460
|
+
/** Who asked for the live pause, and when — cleared on resume (#997). */
|
|
1461
|
+
pausedBy?: { source: string; at: number };
|
|
1445
1462
|
}
|
|
1446
1463
|
|
|
1447
1464
|
const active = new Map<string, Entry>();
|
|
@@ -1476,12 +1493,13 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
1476
1493
|
},
|
|
1477
1494
|
};
|
|
1478
1495
|
},
|
|
1479
|
-
async pause(project, issue) {
|
|
1496
|
+
async pause(project, issue, source) {
|
|
1480
1497
|
const entry = active.get(key(project, issue));
|
|
1481
1498
|
if (entry?.control === undefined) return { kind: "not-active" };
|
|
1482
1499
|
try {
|
|
1483
1500
|
await entry.control.pause();
|
|
1484
1501
|
const phase = entry.control.phase();
|
|
1502
|
+
entry.pausedBy = { source, at: Date.now() };
|
|
1485
1503
|
entry.onPhase?.(phase);
|
|
1486
1504
|
return { kind: "ok", runId: entry.runId, phase };
|
|
1487
1505
|
} catch (err) {
|
|
@@ -1498,6 +1516,7 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
1498
1516
|
try {
|
|
1499
1517
|
entry.control.resume();
|
|
1500
1518
|
const phase = entry.control.phase();
|
|
1519
|
+
delete entry.pausedBy;
|
|
1501
1520
|
entry.onPhase?.(phase);
|
|
1502
1521
|
return { kind: "ok", runId: entry.runId, phase };
|
|
1503
1522
|
} catch (err) {
|
|
@@ -1533,11 +1552,25 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
1533
1552
|
return { kind: "stopped", runId: entry.runId, reason: entry.stopReason! };
|
|
1534
1553
|
},
|
|
1535
1554
|
snapshot(project) {
|
|
1536
|
-
const workers: {
|
|
1555
|
+
const workers: {
|
|
1556
|
+
issue: number;
|
|
1557
|
+
runId: string;
|
|
1558
|
+
phase: WorkerPausePhase;
|
|
1559
|
+
source?: string;
|
|
1560
|
+
pausedAtMs?: number;
|
|
1561
|
+
}[] = [];
|
|
1537
1562
|
for (const entry of active.values()) {
|
|
1538
1563
|
if (entry.project !== project || entry.control === undefined) continue;
|
|
1539
1564
|
const phase = entry.control.phase();
|
|
1540
|
-
if (phase
|
|
1565
|
+
if (phase === "running") continue;
|
|
1566
|
+
workers.push({
|
|
1567
|
+
issue: entry.issue,
|
|
1568
|
+
runId: entry.runId,
|
|
1569
|
+
phase,
|
|
1570
|
+
...(entry.pausedBy === undefined
|
|
1571
|
+
? {}
|
|
1572
|
+
: { source: entry.pausedBy.source, pausedAtMs: entry.pausedBy.at }),
|
|
1573
|
+
});
|
|
1541
1574
|
}
|
|
1542
1575
|
return workers;
|
|
1543
1576
|
},
|
|
@@ -2804,6 +2837,7 @@ export async function handleIssue(
|
|
|
2804
2837
|
log(`#${issue} stopped by operator on attempt ${attempt}: ${result.stoppedReason}`);
|
|
2805
2838
|
} else if (state === "blocked") {
|
|
2806
2839
|
swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
|
|
2840
|
+
wakeOrchestratorForBlockedRun(project.name, issue);
|
|
2807
2841
|
await safeEscalate(d, {
|
|
2808
2842
|
tier: 1,
|
|
2809
2843
|
project: project.name,
|
|
@@ -3035,8 +3069,27 @@ export async function handleIssue(
|
|
|
3035
3069
|
* releasing one whose run is not live; the authoritative child is never
|
|
3036
3070
|
* signalled, and no pane is ever closed.
|
|
3037
3071
|
*/
|
|
3072
|
+
/**
|
|
3073
|
+
* Pane re-establishment attempts spent per run, for this daemon process only
|
|
3074
|
+
* (#998).
|
|
3075
|
+
*
|
|
3076
|
+
* Deliberately in memory rather than on the run row: a restart is exactly the
|
|
3077
|
+
* event that makes another attempt worth making (Herdr came back), so a fresh
|
|
3078
|
+
* process legitimately gets a fresh budget, and giving up never marks a run
|
|
3079
|
+
* permanently unrepresentable. Pruned against the live set on every pass, so it
|
|
3080
|
+
* cannot outgrow the fleet.
|
|
3081
|
+
*/
|
|
3082
|
+
const paneAttemptsSpent = new Map<string, number>();
|
|
3083
|
+
|
|
3038
3084
|
function reconcilePanes(d: Deps, project: string, log: (message: string) => void): void {
|
|
3039
3085
|
const live = d.store.liveRuns(project);
|
|
3086
|
+
const liveKeys = new Set(live.map((run) => `${project}\u0000${run.id}`));
|
|
3087
|
+
for (const key of [...paneAttemptsSpent.keys()]) {
|
|
3088
|
+
if (key.startsWith(`${project}\u0000`) && !liveKeys.has(key)) paneAttemptsSpent.delete(key);
|
|
3089
|
+
}
|
|
3090
|
+
const attempts = new Map(
|
|
3091
|
+
live.map((run) => [run.id, paneAttemptsSpent.get(`${project}\u0000${run.id}`) ?? 0]),
|
|
3092
|
+
);
|
|
3040
3093
|
const result = reconcileWorkerPanes(
|
|
3041
3094
|
live.map((run) => ({
|
|
3042
3095
|
runId: run.id,
|
|
@@ -3048,6 +3101,8 @@ function reconcilePanes(d: Deps, project: string, log: (message: string) => void
|
|
|
3048
3101
|
...(run.paneLabel === undefined ? {} : { paneLabel: run.paneLabel }),
|
|
3049
3102
|
...(run.sessionFile === undefined ? {} : { sessionFile: run.sessionFile }),
|
|
3050
3103
|
})),
|
|
3104
|
+
{},
|
|
3105
|
+
attempts,
|
|
3051
3106
|
);
|
|
3052
3107
|
if (!result.ok) {
|
|
3053
3108
|
// An unreadable workspace is not evidence that anything is stale, so nothing
|
|
@@ -3068,9 +3123,33 @@ function reconcilePanes(d: Deps, project: string, log: (message: string) => void
|
|
|
3068
3123
|
});
|
|
3069
3124
|
log(`#${runIssue(live, outcome.runId)} herdr pane re-associated as ${outcome.paneId} after a restart`);
|
|
3070
3125
|
break;
|
|
3071
|
-
case "untracked":
|
|
3126
|
+
case "untracked": {
|
|
3072
3127
|
d.store.updateRun(outcome.runId, { paneId: null, paneLabel: null, paneUnavailable: outcome.reason });
|
|
3073
|
-
|
|
3128
|
+
if (outcome.attempted !== true) {
|
|
3129
|
+
// Cost no attempt (no recorded pid): say so every pass, as before.
|
|
3130
|
+
log(`#${runIssue(live, outcome.runId)} has no herdr pane: ${outcome.reason}`);
|
|
3131
|
+
break;
|
|
3132
|
+
}
|
|
3133
|
+
// #998: a terminal refusing to split will refuse the next one too, and
|
|
3134
|
+
// each retry made the pile worse — spend the whole budget at once.
|
|
3135
|
+
const key = `${project}\u0000${outcome.runId}`;
|
|
3136
|
+
const spent = outcome.reason.includes("pane_split_failed")
|
|
3137
|
+
? PANE_REATTEMPT_MAX
|
|
3138
|
+
: (paneAttemptsSpent.get(key) ?? 0) + 1;
|
|
3139
|
+
paneAttemptsSpent.set(key, spent);
|
|
3140
|
+
// Once per run, at the moment the budget runs out — not once per pass.
|
|
3141
|
+
// Four identical lines a minute for #986 is what made the real cause
|
|
3142
|
+
// (a leaked pane per attempt) hard to see on 2026-08-23.
|
|
3143
|
+
log(
|
|
3144
|
+
spent >= PANE_REATTEMPT_MAX
|
|
3145
|
+
? `#${runIssue(live, outcome.runId)} has no herdr pane after ${spent} attempt(s), not retrying: ${outcome.reason}`
|
|
3146
|
+
: `#${runIssue(live, outcome.runId)} has no herdr pane (attempt ${spent}/${PANE_REATTEMPT_MAX}): ${outcome.reason}`,
|
|
3147
|
+
);
|
|
3148
|
+
break;
|
|
3149
|
+
}
|
|
3150
|
+
case "attempts-exhausted":
|
|
3151
|
+
// Already reported when the budget ran out; the run keeps working with
|
|
3152
|
+
// no pane, and a later restart gets a fresh budget.
|
|
3074
3153
|
break;
|
|
3075
3154
|
case "stale-released":
|
|
3076
3155
|
log(`released stale herdr pane ${outcome.paneId} (run ${outcome.runId || "unidentified"} is not live)`);
|
|
@@ -4128,6 +4207,7 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
4128
4207
|
log(`#${issue} review round ${revision.round} stopped by operator: ${result.stoppedReason}`);
|
|
4129
4208
|
} else if (state === "blocked") {
|
|
4130
4209
|
swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
|
|
4210
|
+
wakeOrchestratorForBlockedRun(project.name, issue);
|
|
4131
4211
|
await safeEscalate(d, {
|
|
4132
4212
|
tier: 1,
|
|
4133
4213
|
project: project.name,
|
|
@@ -5060,6 +5140,53 @@ export function wakeOrchestratorForMetConditions(
|
|
|
5060
5140
|
}
|
|
5061
5141
|
|
|
5062
5142
|
|
|
5143
|
+
/**
|
|
5144
|
+
* Wake the orchestrator because a worker stopped to ask a question (#990).
|
|
5145
|
+
*
|
|
5146
|
+
* `question` is the third-largest class in this ledger — 58 of 839 runs — and
|
|
5147
|
+
* every one of them is a parked worker holding a slot until someone reads the
|
|
5148
|
+
* question. Nothing woke the orchestrator for it: the only wake was a decision
|
|
5149
|
+
* condition being met, so on a fleet with a 1800s heartbeat a tier-1 question
|
|
5150
|
+
* whose answer is one comment could sit unread for half an hour.
|
|
5151
|
+
*
|
|
5152
|
+
* Deliberately narrow. Only the blocked path wakes: a failed run, a cap kill
|
|
5153
|
+
* and a clean settle are the next scheduled tick's business, and waking on
|
|
5154
|
+
* every terminal state turns the heartbeat into a busy loop — which would be a
|
|
5155
|
+
* worse fleet than the one that waits.
|
|
5156
|
+
*
|
|
5157
|
+
* Debounced by the marker itself rather than by a counter: an unconsumed
|
|
5158
|
+
* request already asks for exactly the wake this call wants, so three workers
|
|
5159
|
+
* blocking in one pass write one request and the *first* question's number
|
|
5160
|
+
* survives as the reason. Best effort throughout — a marker that cannot be
|
|
5161
|
+
* written is logged, and the settlement it belongs to is never affected.
|
|
5162
|
+
*/
|
|
5163
|
+
export function wakeOrchestratorForBlockedRun(
|
|
5164
|
+
projectName: string,
|
|
5165
|
+
issue: number,
|
|
5166
|
+
writeLog: (line: string) => void = log,
|
|
5167
|
+
): void {
|
|
5168
|
+
const tickCwd = resolveTickConfigCwd(projectName);
|
|
5169
|
+
if (tickCwd === undefined) {
|
|
5170
|
+
writeLog(
|
|
5171
|
+
`#${issue} is blocked but no tick config cwd — the question waits for the next heartbeat`,
|
|
5172
|
+
);
|
|
5173
|
+
return;
|
|
5174
|
+
}
|
|
5175
|
+
const pending = readTickRequestReason(tickCwd);
|
|
5176
|
+
if (pending !== undefined) {
|
|
5177
|
+
writeLog(`#${issue} is blocked; a tick request is already pending (${pending})`);
|
|
5178
|
+
return;
|
|
5179
|
+
}
|
|
5180
|
+
const reason = `tier1-question #${issue}`;
|
|
5181
|
+
if (requestImmediateTick(tickCwd, reason)) {
|
|
5182
|
+
writeLog(`requested immediate tick at ${tickCwd}: ${reason}`);
|
|
5183
|
+
} else {
|
|
5184
|
+
writeLog(
|
|
5185
|
+
`could not write tick request under ${tickCwd}; #${issue}'s question waits for the next heartbeat`,
|
|
5186
|
+
);
|
|
5187
|
+
}
|
|
5188
|
+
}
|
|
5189
|
+
|
|
5063
5190
|
/**
|
|
5064
5191
|
* The first-tick verification for a fleet-initiated upgrade (#486).
|
|
5065
5192
|
*
|
|
@@ -5803,8 +5930,9 @@ export interface DaemonHealthSnapshot {
|
|
|
5803
5930
|
turnOverrides: TurnOverride[];
|
|
5804
5931
|
dispatch?: DispatchSummary;
|
|
5805
5932
|
codeGraph?: CodeGraphHealth;
|
|
5806
|
-
/** Live workers in a non-running pause phase; absent/empty = nothing paused.
|
|
5807
|
-
|
|
5933
|
+
/** Live workers in a non-running pause phase; absent/empty = nothing paused.
|
|
5934
|
+
* `source`/`pausedAtMs` carry the pause provenance when one was recorded (#997). */
|
|
5935
|
+
workers?: { issue: number; runId: string; phase: WorkerPausePhase; source?: string; pausedAtMs?: number }[];
|
|
5808
5936
|
/**
|
|
5809
5937
|
* The live orchestrator surface, attested by the running daemon (#832):
|
|
5810
5938
|
* which mode this project's fleet is in, and — when the daemon hosts the
|
|
@@ -5961,8 +6089,12 @@ export async function turnLimitResponse(
|
|
|
5961
6089
|
export async function workerControlResponse(
|
|
5962
6090
|
req: Request,
|
|
5963
6091
|
project: string,
|
|
5964
|
-
store: Pick<Store, "latestRun" | "updateRun">,
|
|
6092
|
+
store: Pick<Store, "latestRun" | "updateRun" | "recordMaterialEvent">,
|
|
5965
6093
|
registry: WorkerControlRegistry,
|
|
6094
|
+
/** Journal line for every accepted pause/resume — the live half of the
|
|
6095
|
+
* audit trail #997 asks for; the durable half is the material event. */
|
|
6096
|
+
log: (line: string) => void = () => {},
|
|
6097
|
+
now: () => number = Date.now,
|
|
5966
6098
|
): Promise<Response | undefined> {
|
|
5967
6099
|
const url = new URL(req.url);
|
|
5968
6100
|
const match = /^\/runs\/(\d+)\/(pause|resume|stop)$/.exec(url.pathname);
|
|
@@ -6002,16 +6134,44 @@ export async function workerControlResponse(
|
|
|
6002
6134
|
return Response.json({ error: "reason must be at most 500 characters" }, { status: 400 });
|
|
6003
6135
|
}
|
|
6004
6136
|
}
|
|
6137
|
+
// Who asked. The board, the CLI and the dashboard each name themselves so a
|
|
6138
|
+
// pause is attributable afterwards (#997); an older caller that sends no
|
|
6139
|
+
// source is recorded as such rather than guessed at.
|
|
6140
|
+
const rawSource = Reflect.get(body, "source");
|
|
6141
|
+
if (rawSource !== undefined && (typeof rawSource !== "string" || rawSource.trim() === "" || rawSource.trim().length > 40)) {
|
|
6142
|
+
return Response.json({ error: "source must be a non-empty string of at most 40 characters when present" }, { status: 400 });
|
|
6143
|
+
}
|
|
6144
|
+
const source = typeof rawSource === "string" ? rawSource.trim() : "unattributed";
|
|
6005
6145
|
|
|
6006
6146
|
|
|
6007
6147
|
const issue = Number(match[1]);
|
|
6008
6148
|
const outcome =
|
|
6009
6149
|
action === "pause"
|
|
6010
|
-
? await registry.pause(project, issue)
|
|
6150
|
+
? await registry.pause(project, issue, source)
|
|
6011
6151
|
: action === "resume"
|
|
6012
6152
|
? registry.resume(project, issue)
|
|
6013
6153
|
: await registry.stop(project, issue, reason!);
|
|
6014
6154
|
if (outcome.kind === "ok") {
|
|
6155
|
+
// The transition really happened — write both halves of the audit trail
|
|
6156
|
+
// before answering (#997): one journal line for the live log, one durable
|
|
6157
|
+
// material event the digest ledger keeps after the journal rotates. #986
|
|
6158
|
+
// was parked 14 minutes by an unlogged keypress and nobody could say why.
|
|
6159
|
+
const verb = action === "pause" ? "paused" : "resumed";
|
|
6160
|
+
log(`#${issue} worker ${verb} via ${source} (run ${outcome.runId})`);
|
|
6161
|
+
try {
|
|
6162
|
+
store.recordMaterialEvent({
|
|
6163
|
+
project,
|
|
6164
|
+
category: "worker-control",
|
|
6165
|
+
summary: `#${issue} worker ${verb} via ${source}`,
|
|
6166
|
+
evidence: `run ${outcome.runId}, phase ${outcome.phase}`,
|
|
6167
|
+
occurredAt: now(),
|
|
6168
|
+
recordedAt: now(),
|
|
6169
|
+
});
|
|
6170
|
+
} catch (err) {
|
|
6171
|
+
// The audit must never turn a successful transition into an error
|
|
6172
|
+
// answer, but a swallowed write would be a silent audit gap — log it.
|
|
6173
|
+
log(`#${issue} worker-control audit write failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
6174
|
+
}
|
|
6015
6175
|
return Response.json({ runId: outcome.runId, phase: outcome.phase });
|
|
6016
6176
|
}
|
|
6017
6177
|
if (outcome.kind === "refused") {
|
|
@@ -6089,8 +6249,10 @@ export async function workerControlResponse(
|
|
|
6089
6249
|
|
|
6090
6250
|
export interface DaemonHttpProjectDeps {
|
|
6091
6251
|
project: string;
|
|
6092
|
-
store: Pick<Store, "latestRun" | "setTurnOverride" | "updateRun">;
|
|
6252
|
+
store: Pick<Store, "latestRun" | "setTurnOverride" | "updateRun" | "recordMaterialEvent">;
|
|
6093
6253
|
caps: () => Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">;
|
|
6254
|
+
/** Project-scoped journal line — the live half of the pause audit (#997). */
|
|
6255
|
+
log?: (line: string) => void;
|
|
6094
6256
|
}
|
|
6095
6257
|
|
|
6096
6258
|
export interface DaemonHttpDeps {
|
|
@@ -6163,6 +6325,7 @@ export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promi
|
|
|
6163
6325
|
selected.project,
|
|
6164
6326
|
selected.store,
|
|
6165
6327
|
d.workerControls,
|
|
6328
|
+
selected.log,
|
|
6166
6329
|
);
|
|
6167
6330
|
return workerControl ?? new Response("not found\n", { status: 404 });
|
|
6168
6331
|
}
|
|
@@ -6524,6 +6687,22 @@ export function statusSnapshotFromStore(
|
|
|
6524
6687
|
spendTelemetry: judgeSpendTelemetry(
|
|
6525
6688
|
store.recentSpendSamples?.(p.name, SPEND_SAMPLE_ROWS) ?? [],
|
|
6526
6689
|
SPEND_SAMPLE_RUNS,
|
|
6690
|
+
{
|
|
6691
|
+
// Config and an already-read status only — this snapshot probes nothing
|
|
6692
|
+
// at render time, which is the property the figure above depends on
|
|
6693
|
+
// (#970). The declaration alone is enough to know a dollar cap cannot
|
|
6694
|
+
// fire; the window name rides along when a configured plan cap has
|
|
6695
|
+
// already resolved one, and `doctor` (which may probe) reads it live.
|
|
6696
|
+
declaredSubscription: (p.requireOauthProviders ?? []).length > 0,
|
|
6697
|
+
...(planUsage?.window === undefined
|
|
6698
|
+
? {}
|
|
6699
|
+
: {
|
|
6700
|
+
allowanceWindow:
|
|
6701
|
+
planUsage.window.label === undefined
|
|
6702
|
+
? planUsage.window.id
|
|
6703
|
+
: `${planUsage.window.id} (${planUsage.window.label})`,
|
|
6704
|
+
}),
|
|
6705
|
+
},
|
|
6527
6706
|
),
|
|
6528
6707
|
// Read, never probed: the recorded row is the whole point (#919).
|
|
6529
6708
|
...(() => {
|
|
@@ -7461,6 +7640,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
7461
7640
|
project: runtime.d.project.name,
|
|
7462
7641
|
store,
|
|
7463
7642
|
caps: () => runtime.d.caps,
|
|
7643
|
+
// The same project prefix the runtime's own journal lines carry, so
|
|
7644
|
+
// a pause audit reads like every other daemon line (#997).
|
|
7645
|
+
log: (line: string) =>
|
|
7646
|
+
log(runtimes.length === 1 ? line : `[${runtime.d.project.name}] ${line}`),
|
|
7464
7647
|
})),
|
|
7465
7648
|
turnLimits,
|
|
7466
7649
|
workerControls,
|
|
@@ -189,7 +189,7 @@ export async function dashboardWorkerControl(
|
|
|
189
189
|
body: unknown,
|
|
190
190
|
d: ControlDeps,
|
|
191
191
|
): Promise<ControlOutcome> {
|
|
192
|
-
const payload: Record<string, unknown> = { project: d.project.name };
|
|
192
|
+
const payload: Record<string, unknown> = { project: d.project.name, source: "dashboard" };
|
|
193
193
|
if (action === "stop") {
|
|
194
194
|
const reason = readReason(body);
|
|
195
195
|
if (reason === undefined) {
|
package/src/doctor.ts
CHANGED
|
@@ -64,6 +64,7 @@ import {
|
|
|
64
64
|
import { pauseInstance } from "./pause.ts";
|
|
65
65
|
import type { TelegramHealth } from "./status-render.ts";
|
|
66
66
|
import {
|
|
67
|
+
claimIsLive,
|
|
67
68
|
claimedTelegramTopics,
|
|
68
69
|
lockPidAlive,
|
|
69
70
|
pidAlive,
|
|
@@ -100,7 +101,7 @@ import {
|
|
|
100
101
|
readTickConfig,
|
|
101
102
|
type HerdrAgentList,
|
|
102
103
|
} from "./orchestrator-tick.ts";
|
|
103
|
-
import type { ConductorConfig, ProjectConfig, RepoTarget, RunState } from "./types.ts";
|
|
104
|
+
import type { Caps, ConductorConfig, ProjectConfig, RepoTarget, RunState } from "./types.ts";
|
|
104
105
|
import { DEFAULT_ARM_PROOF, type ArmProof } from "./types.ts";
|
|
105
106
|
import {
|
|
106
107
|
DEFAULT_DEPS as UPGRADE_DEPS,
|
|
@@ -114,7 +115,8 @@ import {
|
|
|
114
115
|
checkTelegramFreshness,
|
|
115
116
|
type TelegramFreshness,
|
|
116
117
|
} from "./telegram-freshness.ts";
|
|
117
|
-
import {
|
|
118
|
+
import { bindingAllowanceWindow, sharedUsageSource } from "./usage.ts";
|
|
119
|
+
import { judgeSpendTelemetry, spendTelemetryDetail, type SpendBilling } from "./spend-telemetry.ts";
|
|
118
120
|
|
|
119
121
|
/** One read of this host's installed surfaces: the three identities, or why
|
|
120
122
|
* they could not be read. A failed read is a finding (`warn`, "unverified"),
|
|
@@ -308,6 +310,14 @@ export interface DoctorDeps {
|
|
|
308
310
|
/** Whether a recorded claim or dm-owner pid is live, with omp-telegram's
|
|
309
311
|
* topics.ts semantics (EPERM is dead). */
|
|
310
312
|
pidAlive?: (pid: number) => boolean;
|
|
313
|
+
/** Whether one claimed topic is live — the single fact `topic-pin` and
|
|
314
|
+
* `telegram-plumbing` both read, so they cannot disagree about a claim
|
|
315
|
+
* (#987). Defaults to {@link claimIsLive}. */
|
|
316
|
+
claimIsLive?: (claim: { pid?: number }) => boolean;
|
|
317
|
+
/** The provider allowance window nearest its ceiling, from `omp usage --json`
|
|
318
|
+
* — what binds a subscription-billed fleet instead of a dollar cap (#984).
|
|
319
|
+
* `undefined` when the provider reports no comparable window. */
|
|
320
|
+
allowanceWindow?: () => Promise<string | undefined>;
|
|
311
321
|
/** Whether a bot.lock owner pid is live, with omp-telegram's api.ts
|
|
312
322
|
* semantics (EPERM is live). Distinct from {@link pidAlive} because the
|
|
313
323
|
* bridge itself uses two rules. */
|
|
@@ -1096,6 +1106,62 @@ async function telegramProbe(probes: Probes, project: ProjectConfig | undefined,
|
|
|
1096
1106
|
);
|
|
1097
1107
|
}
|
|
1098
1108
|
|
|
1109
|
+
/**
|
|
1110
|
+
* Is any economic control actually governing this project? (#985)
|
|
1111
|
+
*
|
|
1112
|
+
* Two exist, and on this fleet neither was active. `caps.planUsage` was `null`,
|
|
1113
|
+
* so the allowance gate that admission already applies — it holds every routed
|
|
1114
|
+
* candidate and escalates once (`admission.ts`) — was configured off. And
|
|
1115
|
+
* `caps.dailySpendUsd` was a ceiling that #984 established cannot fire under
|
|
1116
|
+
* subscription billing, because a subscription request carries no per-request
|
|
1117
|
+
* price. So the fleet ran with no ceiling of any kind, and no surface said so:
|
|
1118
|
+
* `status` reported `unmetered — no plan allowance cap configured`, which reads
|
|
1119
|
+
* as a fact about the plan rather than as a missing control.
|
|
1120
|
+
*
|
|
1121
|
+
* This row states the gap once, per project, with the remedy that closes it.
|
|
1122
|
+
* It is deliberately NOT a new gate: the gate exists and works. What was
|
|
1123
|
+
* missing was anyone being told it is switched off.
|
|
1124
|
+
*/
|
|
1125
|
+
function economicGateProbe(
|
|
1126
|
+
project: ProjectConfig,
|
|
1127
|
+
caps: Caps,
|
|
1128
|
+
subscriptionBilled: boolean,
|
|
1129
|
+
allowanceWindow: string | undefined,
|
|
1130
|
+
): Finding {
|
|
1131
|
+
const allowanceGate = caps.planUsage !== null;
|
|
1132
|
+
// A dollar cap governs only if it can fire, and under subscription billing it
|
|
1133
|
+
// cannot — that is #984's finding, reused here rather than re-derived.
|
|
1134
|
+
const dollarGate = caps.dailySpendUsd !== null && !subscriptionBilled;
|
|
1135
|
+
|
|
1136
|
+
if (allowanceGate) {
|
|
1137
|
+
return passFinding(
|
|
1138
|
+
"economic-gate",
|
|
1139
|
+
`[${project.name}] the provider allowance gate is configured (window ${caps.planUsage?.windowId ?? "?"}, ` +
|
|
1140
|
+
`hold at ${String(Math.round((caps.planUsage?.maxUsedFraction ?? 0) * 100))}% used)` +
|
|
1141
|
+
(dollarGate ? ` beside a $${caps.dailySpendUsd?.toFixed(2) ?? "?"} daily cap` : ""),
|
|
1142
|
+
);
|
|
1143
|
+
}
|
|
1144
|
+
if (dollarGate) {
|
|
1145
|
+
return passFinding(
|
|
1146
|
+
"economic-gate",
|
|
1147
|
+
`[${project.name}] the $${caps.dailySpendUsd?.toFixed(2) ?? "?"} daily spend cap governs; ` +
|
|
1148
|
+
`no provider allowance cap is configured`,
|
|
1149
|
+
);
|
|
1150
|
+
}
|
|
1151
|
+
return warnFinding(
|
|
1152
|
+
"economic-gate",
|
|
1153
|
+
`[${project.name}] no economic control is active: caps.planUsage is null, and ` +
|
|
1154
|
+
(caps.dailySpendUsd === null
|
|
1155
|
+
? "no daily spend cap is set"
|
|
1156
|
+
: `the $${caps.dailySpendUsd.toFixed(2)} daily cap cannot fire on subscription billing`) +
|
|
1157
|
+
` — admission has nothing to weigh and will keep claiming until the provider itself refuses` +
|
|
1158
|
+
(allowanceWindow === undefined ? "" : `; the window that would bind is ${allowanceWindow}`),
|
|
1159
|
+
allowanceWindow === undefined
|
|
1160
|
+
? "run `omp usage --json` on the fleet host and copy an allowance `id` into `caps.planUsage.windowId` (with `maxUsedFraction`), or set a `caps.dailySpendUsd` the billing model can actually spend"
|
|
1161
|
+
: `set caps.planUsage to { windowId: "${allowanceWindow.split(" ")[0] ?? ""}", maxUsedFraction: 0.9 } — the gate is already wired into admission, it is simply switched off`,
|
|
1162
|
+
);
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1099
1165
|
/**
|
|
1100
1166
|
* Spend telemetry, through the shared judgement (#970).
|
|
1101
1167
|
*
|
|
@@ -1105,9 +1171,17 @@ async function telegramProbe(probes: Probes, project: ProjectConfig | undefined,
|
|
|
1105
1171
|
* that had lost a majority of their telemetry — so partial loss, the common
|
|
1106
1172
|
* case, was the invisible one. See `spend-telemetry.ts` for the measurements.
|
|
1107
1173
|
*/
|
|
1108
|
-
function spendProbe(rows: RunSpendRow[], limit: number): Finding {
|
|
1109
|
-
const verdict = judgeSpendTelemetry(rows, limit);
|
|
1174
|
+
function spendProbe(rows: RunSpendRow[], limit: number, billing: SpendBilling = {}): Finding {
|
|
1175
|
+
const verdict = judgeSpendTelemetry(rows, limit, billing);
|
|
1110
1176
|
const detail = spendTelemetryDetail(verdict);
|
|
1177
|
+
// A subscription-billed fleet's zeros are the truth, so this passes and names
|
|
1178
|
+
// the constraint that does bind (#984). It still says the sentence: a reader
|
|
1179
|
+
// who expected a dollar figure needs to know why there isn't one. What it
|
|
1180
|
+
// must never do is warn forever with advice that cannot be acted on — there
|
|
1181
|
+
// is nothing to repair, and a permanent warning is one nobody reads.
|
|
1182
|
+
if (verdict.kind === "subscription") {
|
|
1183
|
+
return passFinding("spend-telemetry", detail ?? "subscription-billed; no USD cap applies");
|
|
1184
|
+
}
|
|
1111
1185
|
if (detail !== undefined) {
|
|
1112
1186
|
return warnFinding(
|
|
1113
1187
|
"spend-telemetry",
|
|
@@ -1296,15 +1370,35 @@ function topicPinProbe(probes: Probes, p: ProjectConfig): Finding {
|
|
|
1296
1370
|
`check omp-telegram's claim registry (threads.json in its state dir) is readable and the bridge is running, then re-run doctor — until then sends keep the pinned topic and degrade to the flat chat on a missing thread (#318)`,
|
|
1297
1371
|
);
|
|
1298
1372
|
}
|
|
1299
|
-
|
|
1373
|
+
// One liveness fact, shared with `telegram-plumbing` (#987). Before this the
|
|
1374
|
+
// registry's rows were all treated as live here, so a pin aimed at a corpse
|
|
1375
|
+
// read PASS while the plumbing row called the same claim dead.
|
|
1376
|
+
const all = result.claims;
|
|
1377
|
+
const claims = all.filter((claim) => probes.claimIsLive(claim));
|
|
1378
|
+
const dead = all.filter((claim) => !probes.claimIsLive(claim));
|
|
1379
|
+
// Named, never dropped: the row is the only index to the remote topic, so
|
|
1380
|
+
// deleting it strands the topic instead of cleaning it up.
|
|
1381
|
+
const deadNote =
|
|
1382
|
+
dead.length === 0
|
|
1383
|
+
? ""
|
|
1384
|
+
: ` (dead claims still recorded, and left alone: ${dead
|
|
1385
|
+
.map((c) => `${c.threadId}${c.pid === undefined ? " no pid" : ` pid ${c.pid}`}`)
|
|
1386
|
+
.join(", ")} — run the bridge's /cleanup to remove those topics)`;
|
|
1300
1387
|
if (claims.length === 0) {
|
|
1388
|
+
if (dead.length > 0) {
|
|
1389
|
+
return warnFinding(
|
|
1390
|
+
"topic-pin",
|
|
1391
|
+
`[${p.name}] pinned topic ${pinned} — every claim in omp-telegram's registry is dead${deadNote}`,
|
|
1392
|
+
`run /cleanup in Telegram so the bridge closes those topics and drops their rows itself, then re-run setup to re-pin escalation.telegramTopicId to a live claim — never delete rows from threads.json by hand, because the row is the only index to the remote topic (#987)`,
|
|
1393
|
+
);
|
|
1394
|
+
}
|
|
1301
1395
|
return passFinding(
|
|
1302
1396
|
"topic-pin",
|
|
1303
1397
|
`[${p.name}] pinned topic ${pinned} — no live claims to compare (the bridge has claimed no topics yet); the pin stands`,
|
|
1304
1398
|
);
|
|
1305
1399
|
}
|
|
1306
1400
|
if (claims.some((claim) => claim.threadId === pinned)) {
|
|
1307
|
-
return passFinding("topic-pin", `[${p.name}] pinned topic ${pinned} is a live claim`);
|
|
1401
|
+
return passFinding("topic-pin", `[${p.name}] pinned topic ${pinned} is a live claim${deadNote}`);
|
|
1308
1402
|
}
|
|
1309
1403
|
const match = resolveProjectClaim(claims, p.name);
|
|
1310
1404
|
if (match.kind === "match") {
|
|
@@ -1924,7 +2018,30 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
|
|
|
1924
2018
|
// reported against another's name.
|
|
1925
2019
|
for (const p of projects) findings.push(fenceProbe(probes, p.name));
|
|
1926
2020
|
}
|
|
1927
|
-
|
|
2021
|
+
// The billing class the spend row needs (#984). Declared only when EVERY
|
|
2022
|
+
// project requires a subscription credential: on a mixed fleet the zeros of
|
|
2023
|
+
// an API-key project are still a fault, and one project's declaration must
|
|
2024
|
+
// not excuse another's telemetry loss.
|
|
2025
|
+
const declaredSubscription =
|
|
2026
|
+
projects.length > 0 && projects.every((p) => (p.requireOauthProviders ?? []).length > 0);
|
|
2027
|
+
const allowanceWindow = probes.allowanceWindow === undefined ? undefined : await probes.allowanceWindow();
|
|
2028
|
+
const billing = {
|
|
2029
|
+
declaredSubscription,
|
|
2030
|
+
...(allowanceWindow === undefined ? {} : { allowanceWindow }),
|
|
2031
|
+
};
|
|
2032
|
+
findings.push(spendProbe(spendRows, SPEND_SAMPLE_RUNS, billing));
|
|
2033
|
+
// Per project, because the caps are: one project's configured ceiling says
|
|
2034
|
+
// nothing about another's (#985).
|
|
2035
|
+
const subscriptionBilled =
|
|
2036
|
+
judgeSpendTelemetry(spendRows, SPEND_SAMPLE_RUNS, billing).kind === "subscription";
|
|
2037
|
+
if (cfg !== undefined) {
|
|
2038
|
+
const defaults = cfg.defaults;
|
|
2039
|
+
for (const p of projects) {
|
|
2040
|
+
findings.push(
|
|
2041
|
+
economicGateProbe(p, resolveCaps(p, defaults), subscriptionBilled, allowanceWindow),
|
|
2042
|
+
);
|
|
2043
|
+
}
|
|
2044
|
+
}
|
|
1928
2045
|
|
|
1929
2046
|
const status: ReportStatus = findings.some((f) => f.status === "fail")
|
|
1930
2047
|
? "fail"
|
|
@@ -2042,6 +2159,8 @@ export function defaultProbes(): Probes {
|
|
|
2042
2159
|
return claimDir === cwdDir ? [cwdDir] : [cwdDir, claimDir];
|
|
2043
2160
|
},
|
|
2044
2161
|
pidAlive,
|
|
2162
|
+
claimIsLive: (claim) => claimIsLive(claim),
|
|
2163
|
+
allowanceWindow: async () => bindingAllowanceWindow(await sharedUsageSource().read()),
|
|
2045
2164
|
lockPidAlive,
|
|
2046
2165
|
lockFresh: (mtimeMs) => Date.now() - mtimeMs < TELEGRAM_LOCK_FRESH_MS,
|
|
2047
2166
|
tickAgentName: (p) => {
|
package/src/escalate.ts
CHANGED
|
@@ -750,6 +750,37 @@ export function pidAlive(
|
|
|
750
750
|
}
|
|
751
751
|
}
|
|
752
752
|
|
|
753
|
+
/**
|
|
754
|
+
* Whether one claimed topic is live — **the** liveness fact about a claim
|
|
755
|
+
* (#987).
|
|
756
|
+
*
|
|
757
|
+
* Two doctor rows used to answer this question separately: `topic-pin` treated
|
|
758
|
+
* every row in the registry as live (the reader returns rows, it never filtered
|
|
759
|
+
* them), while `telegram-plumbing` applied `pidAlive` and called the same claim
|
|
760
|
+
* `claim-dead`. One fact, two rules, opposite answers — and whichever row the
|
|
761
|
+
* reader happened to trust decided whether the fleet looked healthy. A doctor
|
|
762
|
+
* that contradicts itself is worse than a silent one, because it teaches the
|
|
763
|
+
* reader to discount its rows.
|
|
764
|
+
*
|
|
765
|
+
* A claim with no pid is not live: the bridge records a number or nothing, and
|
|
766
|
+
* `telegramPlumbingVerdict` already refuses a pidless claim as dead. `kill` is
|
|
767
|
+
* injectable for the same reason it is on {@link pidAlive}.
|
|
768
|
+
*
|
|
769
|
+
* Being dead is **not** a licence to delete the row. No Bot API call lists
|
|
770
|
+
* forum topics, so a claim row is the only index to a remote topic
|
|
771
|
+
* (`omp-telegram/src/topics.ts:149-151`): dropping it strands the topic with no
|
|
772
|
+
* supported way to remove it — 70 were stranded that way in one day, and every
|
|
773
|
+
* recovery depended on an artifact nothing guarantees. Callers report dead
|
|
774
|
+
* claims; the bridge's own `/cleanup` removes them, remote side first.
|
|
775
|
+
*/
|
|
776
|
+
export function claimIsLive(
|
|
777
|
+
claim: Pick<ClaimedTopic, "pid">,
|
|
778
|
+
kill?: (target: number, signal: number) => void,
|
|
779
|
+
): boolean {
|
|
780
|
+
if (claim.pid === undefined) return false;
|
|
781
|
+
return kill === undefined ? pidAlive(claim.pid) : pidAlive(claim.pid, kill);
|
|
782
|
+
}
|
|
783
|
+
|
|
753
784
|
/**
|
|
754
785
|
* Whether a bot.lock owner pid is a live process as the bridge's `api.ts`
|
|
755
786
|
* judges it (`src/api.ts:pidAlive`): `EPERM` means the process exists but is
|
|
@@ -1073,6 +1104,19 @@ export type TelegramPlumbingScan = { dirs: readonly string[] };
|
|
|
1073
1104
|
* stops its heartbeat while its own pid stays live (#612 ecosystem
|
|
1074
1105
|
* correction).
|
|
1075
1106
|
*/
|
|
1107
|
+
/**
|
|
1108
|
+
* Adapts an injected pid-liveness predicate to the `kill` seam
|
|
1109
|
+
* {@link claimIsLive} owns, so the verdict can read the shared claim fact
|
|
1110
|
+
* without a second pid rule and without changing its own probe shape (#987).
|
|
1111
|
+
* Throwing is how `pidAlive` spells "dead", which is the contract it applies to
|
|
1112
|
+
* whatever this returns.
|
|
1113
|
+
*/
|
|
1114
|
+
function killOf(alive: (pid: number) => boolean): (target: number, signal: number) => void {
|
|
1115
|
+
return (target) => {
|
|
1116
|
+
if (!alive(target)) throw new Error("not alive");
|
|
1117
|
+
};
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1076
1120
|
export function telegramPlumbingVerdict(
|
|
1077
1121
|
sendTopic: number | undefined,
|
|
1078
1122
|
scan: TelegramPlumbingScan | undefined,
|
|
@@ -1085,7 +1129,9 @@ export function telegramPlumbingVerdict(
|
|
|
1085
1129
|
if (registry.kind !== "ok") return { ok: false, reason: "registry-unreadable" };
|
|
1086
1130
|
const claim = registry.claims.find((c) => c.threadId === sendTopic);
|
|
1087
1131
|
if (claim === undefined) return { ok: false, reason: "no-topic-claim" };
|
|
1088
|
-
|
|
1132
|
+
// The shared fact, not a second copy of the pid rule (#987): `topic-pin`
|
|
1133
|
+
// reads the same predicate, so the two rows cannot disagree about a claim.
|
|
1134
|
+
if (!claimIsLive(claim, killOf(alive))) return { ok: false, reason: "claim-dead" };
|
|
1089
1135
|
if (
|
|
1090
1136
|
scan !== undefined &&
|
|
1091
1137
|
claim.sessionFile !== undefined &&
|