omp-conductor 0.19.1 → 0.19.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +1 -1
- package/package.json +2 -2
- package/schema/config.schema.json +13 -1
- package/src/arm-challenge.ts +92 -16
- package/src/board.ts +50 -9
- package/src/check-browser-js.ts +72 -0
- package/src/cli.ts +2 -0
- package/src/command-manifest.ts +10 -0
- package/src/commands/arm.ts +30 -2
- package/src/commands/companion.ts +103 -0
- package/src/commands/context.ts +1 -0
- package/src/commands/worker.ts +1 -0
- package/src/companion-view.ts +81 -0
- package/src/config-schema.ts +7 -1
- package/src/config.ts +6 -3
- package/src/daemon.ts +197 -14
- package/src/dashboard/app.js +4 -4
- package/src/dashboard/controls.ts +1 -1
- package/src/doctor.ts +126 -7
- package/src/escalate.ts +47 -1
- package/src/failure-class.fixture.json +546 -0
- package/src/failure-class.ts +192 -0
- package/src/fleet.ts +275 -27
- package/src/orchestrator-tick.ts +115 -34
- package/src/settlement.ts +167 -0
- package/src/setup.ts +6 -4
- package/src/spend-telemetry.ts +74 -2
- package/src/status-render.ts +25 -6
- package/src/store.ts +38 -2
- package/src/types.ts +39 -2
- package/src/usage.ts +25 -0
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The composed view the recovery companion pane renders (#994).
|
|
3
|
+
*
|
|
4
|
+
* Two defects produced this module, and they were one decision:
|
|
5
|
+
*
|
|
6
|
+
* - The pane re-forked `watch -c -n 60 '<one-shot>'`, and every 60 seconds the
|
|
7
|
+
* foreground-process transition made herdr repaint it — a visible flash. The
|
|
8
|
+
* board pane, one long-lived process holding its terminal, never flashed. So
|
|
9
|
+
* the fix is a process that persists between renders, which is what
|
|
10
|
+
* {@link companionFrame} exists to feed.
|
|
11
|
+
* - It rendered `decision list` alone, which on this fleet meant a whole pane
|
|
12
|
+
* refreshing forever to display `no open decisions` while nine watches, four
|
|
13
|
+
* of them `condition:met`, were invisible in it.
|
|
14
|
+
*
|
|
15
|
+
* The split is deliberate and this is the only place it is composed: operator
|
|
16
|
+
* decisions are rare and already interrupt through Telegram when they matter,
|
|
17
|
+
* while watches are the live state a glanceable pane is *for* — and a met
|
|
18
|
+
* condition is the one row in either list that wants eyes immediately.
|
|
19
|
+
*
|
|
20
|
+
* An empty section is omitted entirely rather than printed with a placeholder.
|
|
21
|
+
* A pane showing nothing is honest; a pane showing `no open decisions` forever
|
|
22
|
+
* is the noise the operator objected to.
|
|
23
|
+
*
|
|
24
|
+
* `decision list` and `watch list` keep their own contracts untouched — this is
|
|
25
|
+
* the pane's composition, not those commands' output.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import type { DecisionRecord } from "./types.ts";
|
|
29
|
+
|
|
30
|
+
/** The marker that makes an actionable watch findable at a glance. Uppercase
|
|
31
|
+
* rather than coloured: the pane inherits whatever palette the terminal has,
|
|
32
|
+
* and the flash this module replaced was chased through an ANSI hypothesis. */
|
|
33
|
+
export const MET_MARKER = "[ACT NOW]";
|
|
34
|
+
|
|
35
|
+
/** The single line shown when neither list has anything. Quiet, and still
|
|
36
|
+
* proof the pane is alive rather than blank because something died. */
|
|
37
|
+
export const IDLE_LINE = "nothing open";
|
|
38
|
+
|
|
39
|
+
/** One row, reduced to what the pane shows. */
|
|
40
|
+
function row(record: DecisionRecord, now: number): string {
|
|
41
|
+
const ageHours = Math.max(0, Math.round((now - record.askedAt) / 3_600_000));
|
|
42
|
+
const condition =
|
|
43
|
+
record.condition === undefined ? "-" : record.conditionMetAt === undefined ? "pending" : "met";
|
|
44
|
+
const met = condition === "met" ? `${MET_MARKER} ` : "";
|
|
45
|
+
return ` ${met}${record.id} ${String(ageHours)}h blocks:${record.blocks ?? "-"} condition:${condition} ${record.question}`;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* The frame to draw, as lines.
|
|
50
|
+
*
|
|
51
|
+
* Decisions above watches, because only a decision wants an answer. Met
|
|
52
|
+
* conditions are hoisted to the top of the watch section: the whole reason to
|
|
53
|
+
* render watches at all is that one of them may be actionable, and a reader who
|
|
54
|
+
* has to scan nine rows for the one that matters is reading a log, not a board.
|
|
55
|
+
*/
|
|
56
|
+
export function companionFrame(
|
|
57
|
+
open: readonly DecisionRecord[],
|
|
58
|
+
now: number = Date.now(),
|
|
59
|
+
): string[] {
|
|
60
|
+
const decisions = open.filter((d) => d.kind !== "watch");
|
|
61
|
+
const watches = open.filter((d) => d.kind === "watch");
|
|
62
|
+
const lines: string[] = [];
|
|
63
|
+
|
|
64
|
+
if (decisions.length > 0) {
|
|
65
|
+
lines.push(`operator decisions (${String(decisions.length)}) — these want an answer`);
|
|
66
|
+
for (const decision of decisions) lines.push(row(decision, now));
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
if (watches.length > 0) {
|
|
70
|
+
const met = watches.filter((w) => w.conditionMetAt !== undefined);
|
|
71
|
+
const pending = watches.filter((w) => w.conditionMetAt === undefined);
|
|
72
|
+
if (lines.length > 0) lines.push("");
|
|
73
|
+
lines.push(
|
|
74
|
+
`watches (${String(watches.length)}) — no answer needed` +
|
|
75
|
+
(met.length === 0 ? "" : `, ${String(met.length)} to act on now`),
|
|
76
|
+
);
|
|
77
|
+
for (const watch of [...met, ...pending]) lines.push(row(watch, now));
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return lines.length === 0 ? [IDLE_LINE] : lines;
|
|
81
|
+
}
|
package/src/config-schema.ts
CHANGED
|
@@ -397,7 +397,13 @@ const projectSchema = z
|
|
|
397
397
|
})
|
|
398
398
|
.strict(),
|
|
399
399
|
queueLabel: z.string().min(1, "must be a non-empty string — it is the human sign-off gate"),
|
|
400
|
-
|
|
400
|
+
// The grooming trigger (#988): an integer ≥ 1, or the literal "always" —
|
|
401
|
+
// groom on demand every tick while ungroomed candidates remain. Unlike the
|
|
402
|
+
// opaque model selectors below, this field's usable vocabulary is closed,
|
|
403
|
+
// so the schema states it and a wrong value fails the load with an
|
|
404
|
+
// authored line instead of silently degrading the fleet's grooming duty
|
|
405
|
+
// to the default.
|
|
406
|
+
groomBelow: z.union([z.number().int().min(1), z.literal("always")]).optional(),
|
|
401
407
|
stateLabels: stateLabelsSchema.optional(),
|
|
402
408
|
routing: routingSchema.optional(),
|
|
403
409
|
caps: capsSchema.optional(),
|
package/src/config.ts
CHANGED
|
@@ -847,6 +847,9 @@ function clauseFor(rel: readonly PropertyKey[], issue: {
|
|
|
847
847
|
const pathStr = dottedPath(rel);
|
|
848
848
|
const found = (at?: readonly PropertyKey[]): string => FOUND(rawAt(root, at ?? fullPath));
|
|
849
849
|
|
|
850
|
+
if (rel.length === 1 && rel[0] === "groomBelow") {
|
|
851
|
+
return `groomBelow must be an integer ≥ 1 or the literal "always", found ${found()}`;
|
|
852
|
+
}
|
|
850
853
|
if (rel.length === 1 && rel[0] === "releasePolicy") {
|
|
851
854
|
return `releasePolicy must be an object mapping release shapes (${RELEASE_SHAPE_LIST}) to ${AUTHORITY_HOLDER_LIST}, or the legacy ${LEGACY_RELEASE_POLICY_LIST}, found ${found()}`;
|
|
852
855
|
}
|
|
@@ -1195,11 +1198,11 @@ function finalizeProject(
|
|
|
1195
1198
|
const label = `project "${name}"`;
|
|
1196
1199
|
const trackerRepo = ((p["tracker"] as Raw | undefined)?.["repo"]) as string;
|
|
1197
1200
|
|
|
1201
|
+
// The zod union already enforced `integer ≥ 1 | "always"`; this only
|
|
1202
|
+
// narrows the type for the assembled ProjectConfig (#988).
|
|
1198
1203
|
const rawGroomBelow = p["groomBelow"];
|
|
1199
1204
|
const groomBelow =
|
|
1200
|
-
typeof rawGroomBelow === "number"
|
|
1201
|
-
? rawGroomBelow
|
|
1202
|
-
: undefined;
|
|
1205
|
+
rawGroomBelow === "always" ? "always" : typeof rawGroomBelow === "number" ? rawGroomBelow : undefined;
|
|
1203
1206
|
|
|
1204
1207
|
const rawRouting = p["routing"] as Raw | undefined;
|
|
1205
1208
|
const rawPrefix = rawRouting?.["labelPrefix"];
|
package/src/daemon.ts
CHANGED
|
@@ -38,7 +38,12 @@ import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-
|
|
|
38
38
|
import { graphHint } from "./graph.ts";
|
|
39
39
|
import { hostConstraintsNotice } from "./host.ts";
|
|
40
40
|
import { acquireOnceLease, healthCheck, livingDaemon, probeUnit, SYSTEMD_UNIT } from "./lifecycle.ts";
|
|
41
|
-
import {
|
|
41
|
+
import {
|
|
42
|
+
readTickRequestReason,
|
|
43
|
+
requestImmediateTick,
|
|
44
|
+
resolveTickConfigCwd,
|
|
45
|
+
STALL_MARKER_FILE,
|
|
46
|
+
} from "./orchestrator-tick.ts";
|
|
42
47
|
import { SPEND_SAMPLE_ROWS, SPEND_SAMPLE_RUNS, runDoctor } from "./doctor.ts";
|
|
43
48
|
import { judgeSpendTelemetry, type SpendTelemetryVerdict } from "./spend-telemetry.ts";
|
|
44
49
|
import {
|
|
@@ -62,6 +67,7 @@ import {
|
|
|
62
67
|
herdrPaneOmpStarts,
|
|
63
68
|
openWorkerPane,
|
|
64
69
|
reconcileWorkerPanes,
|
|
70
|
+
PANE_REATTEMPT_MAX,
|
|
65
71
|
releaseOrphanedWorkerPane,
|
|
66
72
|
releaseWorkerPane,
|
|
67
73
|
reportWorkerPaneState,
|
|
@@ -1424,11 +1430,20 @@ export interface WorkerControlRegistry {
|
|
|
1424
1430
|
runId: string,
|
|
1425
1431
|
onPhase?: (phase: WorkerPausePhase) => void,
|
|
1426
1432
|
): WorkerControlSlot;
|
|
1427
|
-
|
|
1433
|
+
/** `source` names who asked (board/cli/dashboard) — recorded as pause
|
|
1434
|
+
* provenance so "who paused this and when" is answerable later (#997). */
|
|
1435
|
+
pause(project: string, issue: number, source: string): Promise<WorkerControlResult>;
|
|
1428
1436
|
resume(project: string, issue: number): WorkerControlResult;
|
|
1429
1437
|
stop(project: string, issue: number, reason: string): Promise<WorkerControlResult>;
|
|
1430
|
-
/** Live runs whose phase is not `running` — what /healthz and the board
|
|
1431
|
-
|
|
1438
|
+
/** Live runs whose phase is not `running` — what /healthz and the board
|
|
1439
|
+
* show — each carrying its pause provenance when one was recorded (#997). */
|
|
1440
|
+
snapshot(project: string): {
|
|
1441
|
+
issue: number;
|
|
1442
|
+
runId: string;
|
|
1443
|
+
phase: WorkerPausePhase;
|
|
1444
|
+
source?: string;
|
|
1445
|
+
pausedAtMs?: number;
|
|
1446
|
+
}[];
|
|
1432
1447
|
}
|
|
1433
1448
|
|
|
1434
1449
|
/** Authoritative controls for sessions owned by this daemon process. */
|
|
@@ -1442,6 +1457,8 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
1442
1457
|
stopError?: string;
|
|
1443
1458
|
finished: PromiseWithResolvers<void>;
|
|
1444
1459
|
onPhase?: (phase: WorkerPausePhase) => void;
|
|
1460
|
+
/** Who asked for the live pause, and when — cleared on resume (#997). */
|
|
1461
|
+
pausedBy?: { source: string; at: number };
|
|
1445
1462
|
}
|
|
1446
1463
|
|
|
1447
1464
|
const active = new Map<string, Entry>();
|
|
@@ -1476,12 +1493,13 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
1476
1493
|
},
|
|
1477
1494
|
};
|
|
1478
1495
|
},
|
|
1479
|
-
async pause(project, issue) {
|
|
1496
|
+
async pause(project, issue, source) {
|
|
1480
1497
|
const entry = active.get(key(project, issue));
|
|
1481
1498
|
if (entry?.control === undefined) return { kind: "not-active" };
|
|
1482
1499
|
try {
|
|
1483
1500
|
await entry.control.pause();
|
|
1484
1501
|
const phase = entry.control.phase();
|
|
1502
|
+
entry.pausedBy = { source, at: Date.now() };
|
|
1485
1503
|
entry.onPhase?.(phase);
|
|
1486
1504
|
return { kind: "ok", runId: entry.runId, phase };
|
|
1487
1505
|
} catch (err) {
|
|
@@ -1498,6 +1516,7 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
1498
1516
|
try {
|
|
1499
1517
|
entry.control.resume();
|
|
1500
1518
|
const phase = entry.control.phase();
|
|
1519
|
+
delete entry.pausedBy;
|
|
1501
1520
|
entry.onPhase?.(phase);
|
|
1502
1521
|
return { kind: "ok", runId: entry.runId, phase };
|
|
1503
1522
|
} catch (err) {
|
|
@@ -1533,11 +1552,25 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
1533
1552
|
return { kind: "stopped", runId: entry.runId, reason: entry.stopReason! };
|
|
1534
1553
|
},
|
|
1535
1554
|
snapshot(project) {
|
|
1536
|
-
const workers: {
|
|
1555
|
+
const workers: {
|
|
1556
|
+
issue: number;
|
|
1557
|
+
runId: string;
|
|
1558
|
+
phase: WorkerPausePhase;
|
|
1559
|
+
source?: string;
|
|
1560
|
+
pausedAtMs?: number;
|
|
1561
|
+
}[] = [];
|
|
1537
1562
|
for (const entry of active.values()) {
|
|
1538
1563
|
if (entry.project !== project || entry.control === undefined) continue;
|
|
1539
1564
|
const phase = entry.control.phase();
|
|
1540
|
-
if (phase
|
|
1565
|
+
if (phase === "running") continue;
|
|
1566
|
+
workers.push({
|
|
1567
|
+
issue: entry.issue,
|
|
1568
|
+
runId: entry.runId,
|
|
1569
|
+
phase,
|
|
1570
|
+
...(entry.pausedBy === undefined
|
|
1571
|
+
? {}
|
|
1572
|
+
: { source: entry.pausedBy.source, pausedAtMs: entry.pausedBy.at }),
|
|
1573
|
+
});
|
|
1541
1574
|
}
|
|
1542
1575
|
return workers;
|
|
1543
1576
|
},
|
|
@@ -2804,6 +2837,7 @@ export async function handleIssue(
|
|
|
2804
2837
|
log(`#${issue} stopped by operator on attempt ${attempt}: ${result.stoppedReason}`);
|
|
2805
2838
|
} else if (state === "blocked") {
|
|
2806
2839
|
swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
|
|
2840
|
+
wakeOrchestratorForBlockedRun(project.name, issue);
|
|
2807
2841
|
await safeEscalate(d, {
|
|
2808
2842
|
tier: 1,
|
|
2809
2843
|
project: project.name,
|
|
@@ -3035,8 +3069,27 @@ export async function handleIssue(
|
|
|
3035
3069
|
* releasing one whose run is not live; the authoritative child is never
|
|
3036
3070
|
* signalled, and no pane is ever closed.
|
|
3037
3071
|
*/
|
|
3072
|
+
/**
|
|
3073
|
+
* Pane re-establishment attempts spent per run, for this daemon process only
|
|
3074
|
+
* (#998).
|
|
3075
|
+
*
|
|
3076
|
+
* Deliberately in memory rather than on the run row: a restart is exactly the
|
|
3077
|
+
* event that makes another attempt worth making (Herdr came back), so a fresh
|
|
3078
|
+
* process legitimately gets a fresh budget, and giving up never marks a run
|
|
3079
|
+
* permanently unrepresentable. Pruned against the live set on every pass, so it
|
|
3080
|
+
* cannot outgrow the fleet.
|
|
3081
|
+
*/
|
|
3082
|
+
const paneAttemptsSpent = new Map<string, number>();
|
|
3083
|
+
|
|
3038
3084
|
function reconcilePanes(d: Deps, project: string, log: (message: string) => void): void {
|
|
3039
3085
|
const live = d.store.liveRuns(project);
|
|
3086
|
+
const liveKeys = new Set(live.map((run) => `${project}\u0000${run.id}`));
|
|
3087
|
+
for (const key of [...paneAttemptsSpent.keys()]) {
|
|
3088
|
+
if (key.startsWith(`${project}\u0000`) && !liveKeys.has(key)) paneAttemptsSpent.delete(key);
|
|
3089
|
+
}
|
|
3090
|
+
const attempts = new Map(
|
|
3091
|
+
live.map((run) => [run.id, paneAttemptsSpent.get(`${project}\u0000${run.id}`) ?? 0]),
|
|
3092
|
+
);
|
|
3040
3093
|
const result = reconcileWorkerPanes(
|
|
3041
3094
|
live.map((run) => ({
|
|
3042
3095
|
runId: run.id,
|
|
@@ -3048,6 +3101,8 @@ function reconcilePanes(d: Deps, project: string, log: (message: string) => void
|
|
|
3048
3101
|
...(run.paneLabel === undefined ? {} : { paneLabel: run.paneLabel }),
|
|
3049
3102
|
...(run.sessionFile === undefined ? {} : { sessionFile: run.sessionFile }),
|
|
3050
3103
|
})),
|
|
3104
|
+
{},
|
|
3105
|
+
attempts,
|
|
3051
3106
|
);
|
|
3052
3107
|
if (!result.ok) {
|
|
3053
3108
|
// An unreadable workspace is not evidence that anything is stale, so nothing
|
|
@@ -3068,9 +3123,33 @@ function reconcilePanes(d: Deps, project: string, log: (message: string) => void
|
|
|
3068
3123
|
});
|
|
3069
3124
|
log(`#${runIssue(live, outcome.runId)} herdr pane re-associated as ${outcome.paneId} after a restart`);
|
|
3070
3125
|
break;
|
|
3071
|
-
case "untracked":
|
|
3126
|
+
case "untracked": {
|
|
3072
3127
|
d.store.updateRun(outcome.runId, { paneId: null, paneLabel: null, paneUnavailable: outcome.reason });
|
|
3073
|
-
|
|
3128
|
+
if (outcome.attempted !== true) {
|
|
3129
|
+
// Cost no attempt (no recorded pid): say so every pass, as before.
|
|
3130
|
+
log(`#${runIssue(live, outcome.runId)} has no herdr pane: ${outcome.reason}`);
|
|
3131
|
+
break;
|
|
3132
|
+
}
|
|
3133
|
+
// #998: a terminal refusing to split will refuse the next one too, and
|
|
3134
|
+
// each retry made the pile worse — spend the whole budget at once.
|
|
3135
|
+
const key = `${project}\u0000${outcome.runId}`;
|
|
3136
|
+
const spent = outcome.reason.includes("pane_split_failed")
|
|
3137
|
+
? PANE_REATTEMPT_MAX
|
|
3138
|
+
: (paneAttemptsSpent.get(key) ?? 0) + 1;
|
|
3139
|
+
paneAttemptsSpent.set(key, spent);
|
|
3140
|
+
// Once per run, at the moment the budget runs out — not once per pass.
|
|
3141
|
+
// Four identical lines a minute for #986 is what made the real cause
|
|
3142
|
+
// (a leaked pane per attempt) hard to see on 2026-08-23.
|
|
3143
|
+
log(
|
|
3144
|
+
spent >= PANE_REATTEMPT_MAX
|
|
3145
|
+
? `#${runIssue(live, outcome.runId)} has no herdr pane after ${spent} attempt(s), not retrying: ${outcome.reason}`
|
|
3146
|
+
: `#${runIssue(live, outcome.runId)} has no herdr pane (attempt ${spent}/${PANE_REATTEMPT_MAX}): ${outcome.reason}`,
|
|
3147
|
+
);
|
|
3148
|
+
break;
|
|
3149
|
+
}
|
|
3150
|
+
case "attempts-exhausted":
|
|
3151
|
+
// Already reported when the budget ran out; the run keeps working with
|
|
3152
|
+
// no pane, and a later restart gets a fresh budget.
|
|
3074
3153
|
break;
|
|
3075
3154
|
case "stale-released":
|
|
3076
3155
|
log(`released stale herdr pane ${outcome.paneId} (run ${outcome.runId || "unidentified"} is not live)`);
|
|
@@ -4128,6 +4207,7 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
4128
4207
|
log(`#${issue} review round ${revision.round} stopped by operator: ${result.stoppedReason}`);
|
|
4129
4208
|
} else if (state === "blocked") {
|
|
4130
4209
|
swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
|
|
4210
|
+
wakeOrchestratorForBlockedRun(project.name, issue);
|
|
4131
4211
|
await safeEscalate(d, {
|
|
4132
4212
|
tier: 1,
|
|
4133
4213
|
project: project.name,
|
|
@@ -5060,6 +5140,53 @@ export function wakeOrchestratorForMetConditions(
|
|
|
5060
5140
|
}
|
|
5061
5141
|
|
|
5062
5142
|
|
|
5143
|
+
/**
|
|
5144
|
+
* Wake the orchestrator because a worker stopped to ask a question (#990).
|
|
5145
|
+
*
|
|
5146
|
+
* `question` is the third-largest class in this ledger — 58 of 839 runs — and
|
|
5147
|
+
* every one of them is a parked worker holding a slot until someone reads the
|
|
5148
|
+
* question. Nothing woke the orchestrator for it: the only wake was a decision
|
|
5149
|
+
* condition being met, so on a fleet with a 1800s heartbeat a tier-1 question
|
|
5150
|
+
* whose answer is one comment could sit unread for half an hour.
|
|
5151
|
+
*
|
|
5152
|
+
* Deliberately narrow. Only the blocked path wakes: a failed run, a cap kill
|
|
5153
|
+
* and a clean settle are the next scheduled tick's business, and waking on
|
|
5154
|
+
* every terminal state turns the heartbeat into a busy loop — which would be a
|
|
5155
|
+
* worse fleet than the one that waits.
|
|
5156
|
+
*
|
|
5157
|
+
* Debounced by the marker itself rather than by a counter: an unconsumed
|
|
5158
|
+
* request already asks for exactly the wake this call wants, so three workers
|
|
5159
|
+
* blocking in one pass write one request and the *first* question's number
|
|
5160
|
+
* survives as the reason. Best effort throughout — a marker that cannot be
|
|
5161
|
+
* written is logged, and the settlement it belongs to is never affected.
|
|
5162
|
+
*/
|
|
5163
|
+
export function wakeOrchestratorForBlockedRun(
|
|
5164
|
+
projectName: string,
|
|
5165
|
+
issue: number,
|
|
5166
|
+
writeLog: (line: string) => void = log,
|
|
5167
|
+
): void {
|
|
5168
|
+
const tickCwd = resolveTickConfigCwd(projectName);
|
|
5169
|
+
if (tickCwd === undefined) {
|
|
5170
|
+
writeLog(
|
|
5171
|
+
`#${issue} is blocked but no tick config cwd — the question waits for the next heartbeat`,
|
|
5172
|
+
);
|
|
5173
|
+
return;
|
|
5174
|
+
}
|
|
5175
|
+
const pending = readTickRequestReason(tickCwd);
|
|
5176
|
+
if (pending !== undefined) {
|
|
5177
|
+
writeLog(`#${issue} is blocked; a tick request is already pending (${pending})`);
|
|
5178
|
+
return;
|
|
5179
|
+
}
|
|
5180
|
+
const reason = `tier1-question #${issue}`;
|
|
5181
|
+
if (requestImmediateTick(tickCwd, reason)) {
|
|
5182
|
+
writeLog(`requested immediate tick at ${tickCwd}: ${reason}`);
|
|
5183
|
+
} else {
|
|
5184
|
+
writeLog(
|
|
5185
|
+
`could not write tick request under ${tickCwd}; #${issue}'s question waits for the next heartbeat`,
|
|
5186
|
+
);
|
|
5187
|
+
}
|
|
5188
|
+
}
|
|
5189
|
+
|
|
5063
5190
|
/**
|
|
5064
5191
|
* The first-tick verification for a fleet-initiated upgrade (#486).
|
|
5065
5192
|
*
|
|
@@ -5803,8 +5930,9 @@ export interface DaemonHealthSnapshot {
|
|
|
5803
5930
|
turnOverrides: TurnOverride[];
|
|
5804
5931
|
dispatch?: DispatchSummary;
|
|
5805
5932
|
codeGraph?: CodeGraphHealth;
|
|
5806
|
-
/** Live workers in a non-running pause phase; absent/empty = nothing paused.
|
|
5807
|
-
|
|
5933
|
+
/** Live workers in a non-running pause phase; absent/empty = nothing paused.
|
|
5934
|
+
* `source`/`pausedAtMs` carry the pause provenance when one was recorded (#997). */
|
|
5935
|
+
workers?: { issue: number; runId: string; phase: WorkerPausePhase; source?: string; pausedAtMs?: number }[];
|
|
5808
5936
|
/**
|
|
5809
5937
|
* The live orchestrator surface, attested by the running daemon (#832):
|
|
5810
5938
|
* which mode this project's fleet is in, and — when the daemon hosts the
|
|
@@ -5961,8 +6089,12 @@ export async function turnLimitResponse(
|
|
|
5961
6089
|
export async function workerControlResponse(
|
|
5962
6090
|
req: Request,
|
|
5963
6091
|
project: string,
|
|
5964
|
-
store: Pick<Store, "latestRun" | "updateRun">,
|
|
6092
|
+
store: Pick<Store, "latestRun" | "updateRun" | "recordMaterialEvent">,
|
|
5965
6093
|
registry: WorkerControlRegistry,
|
|
6094
|
+
/** Journal line for every accepted pause/resume — the live half of the
|
|
6095
|
+
* audit trail #997 asks for; the durable half is the material event. */
|
|
6096
|
+
log: (line: string) => void = () => {},
|
|
6097
|
+
now: () => number = Date.now,
|
|
5966
6098
|
): Promise<Response | undefined> {
|
|
5967
6099
|
const url = new URL(req.url);
|
|
5968
6100
|
const match = /^\/runs\/(\d+)\/(pause|resume|stop)$/.exec(url.pathname);
|
|
@@ -6002,16 +6134,44 @@ export async function workerControlResponse(
|
|
|
6002
6134
|
return Response.json({ error: "reason must be at most 500 characters" }, { status: 400 });
|
|
6003
6135
|
}
|
|
6004
6136
|
}
|
|
6137
|
+
// Who asked. The board, the CLI and the dashboard each name themselves so a
|
|
6138
|
+
// pause is attributable afterwards (#997); an older caller that sends no
|
|
6139
|
+
// source is recorded as such rather than guessed at.
|
|
6140
|
+
const rawSource = Reflect.get(body, "source");
|
|
6141
|
+
if (rawSource !== undefined && (typeof rawSource !== "string" || rawSource.trim() === "" || rawSource.trim().length > 40)) {
|
|
6142
|
+
return Response.json({ error: "source must be a non-empty string of at most 40 characters when present" }, { status: 400 });
|
|
6143
|
+
}
|
|
6144
|
+
const source = typeof rawSource === "string" ? rawSource.trim() : "unattributed";
|
|
6005
6145
|
|
|
6006
6146
|
|
|
6007
6147
|
const issue = Number(match[1]);
|
|
6008
6148
|
const outcome =
|
|
6009
6149
|
action === "pause"
|
|
6010
|
-
? await registry.pause(project, issue)
|
|
6150
|
+
? await registry.pause(project, issue, source)
|
|
6011
6151
|
: action === "resume"
|
|
6012
6152
|
? registry.resume(project, issue)
|
|
6013
6153
|
: await registry.stop(project, issue, reason!);
|
|
6014
6154
|
if (outcome.kind === "ok") {
|
|
6155
|
+
// The transition really happened — write both halves of the audit trail
|
|
6156
|
+
// before answering (#997): one journal line for the live log, one durable
|
|
6157
|
+
// material event the digest ledger keeps after the journal rotates. #986
|
|
6158
|
+
// was parked 14 minutes by an unlogged keypress and nobody could say why.
|
|
6159
|
+
const verb = action === "pause" ? "paused" : "resumed";
|
|
6160
|
+
log(`#${issue} worker ${verb} via ${source} (run ${outcome.runId})`);
|
|
6161
|
+
try {
|
|
6162
|
+
store.recordMaterialEvent({
|
|
6163
|
+
project,
|
|
6164
|
+
category: "worker-control",
|
|
6165
|
+
summary: `#${issue} worker ${verb} via ${source}`,
|
|
6166
|
+
evidence: `run ${outcome.runId}, phase ${outcome.phase}`,
|
|
6167
|
+
occurredAt: now(),
|
|
6168
|
+
recordedAt: now(),
|
|
6169
|
+
});
|
|
6170
|
+
} catch (err) {
|
|
6171
|
+
// The audit must never turn a successful transition into an error
|
|
6172
|
+
// answer, but a swallowed write would be a silent audit gap — log it.
|
|
6173
|
+
log(`#${issue} worker-control audit write failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
6174
|
+
}
|
|
6015
6175
|
return Response.json({ runId: outcome.runId, phase: outcome.phase });
|
|
6016
6176
|
}
|
|
6017
6177
|
if (outcome.kind === "refused") {
|
|
@@ -6089,8 +6249,10 @@ export async function workerControlResponse(
|
|
|
6089
6249
|
|
|
6090
6250
|
export interface DaemonHttpProjectDeps {
|
|
6091
6251
|
project: string;
|
|
6092
|
-
store: Pick<Store, "latestRun" | "setTurnOverride" | "updateRun">;
|
|
6252
|
+
store: Pick<Store, "latestRun" | "setTurnOverride" | "updateRun" | "recordMaterialEvent">;
|
|
6093
6253
|
caps: () => Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">;
|
|
6254
|
+
/** Project-scoped journal line — the live half of the pause audit (#997). */
|
|
6255
|
+
log?: (line: string) => void;
|
|
6094
6256
|
}
|
|
6095
6257
|
|
|
6096
6258
|
export interface DaemonHttpDeps {
|
|
@@ -6163,6 +6325,7 @@ export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promi
|
|
|
6163
6325
|
selected.project,
|
|
6164
6326
|
selected.store,
|
|
6165
6327
|
d.workerControls,
|
|
6328
|
+
selected.log,
|
|
6166
6329
|
);
|
|
6167
6330
|
return workerControl ?? new Response("not found\n", { status: 404 });
|
|
6168
6331
|
}
|
|
@@ -6524,6 +6687,22 @@ export function statusSnapshotFromStore(
|
|
|
6524
6687
|
spendTelemetry: judgeSpendTelemetry(
|
|
6525
6688
|
store.recentSpendSamples?.(p.name, SPEND_SAMPLE_ROWS) ?? [],
|
|
6526
6689
|
SPEND_SAMPLE_RUNS,
|
|
6690
|
+
{
|
|
6691
|
+
// Config and an already-read status only — this snapshot probes nothing
|
|
6692
|
+
// at render time, which is the property the figure above depends on
|
|
6693
|
+
// (#970). The declaration alone is enough to know a dollar cap cannot
|
|
6694
|
+
// fire; the window name rides along when a configured plan cap has
|
|
6695
|
+
// already resolved one, and `doctor` (which may probe) reads it live.
|
|
6696
|
+
declaredSubscription: (p.requireOauthProviders ?? []).length > 0,
|
|
6697
|
+
...(planUsage?.window === undefined
|
|
6698
|
+
? {}
|
|
6699
|
+
: {
|
|
6700
|
+
allowanceWindow:
|
|
6701
|
+
planUsage.window.label === undefined
|
|
6702
|
+
? planUsage.window.id
|
|
6703
|
+
: `${planUsage.window.id} (${planUsage.window.label})`,
|
|
6704
|
+
}),
|
|
6705
|
+
},
|
|
6527
6706
|
),
|
|
6528
6707
|
// Read, never probed: the recorded row is the whole point (#919).
|
|
6529
6708
|
...(() => {
|
|
@@ -7461,6 +7640,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
7461
7640
|
project: runtime.d.project.name,
|
|
7462
7641
|
store,
|
|
7463
7642
|
caps: () => runtime.d.caps,
|
|
7643
|
+
// The same project prefix the runtime's own journal lines carry, so
|
|
7644
|
+
// a pause audit reads like every other daemon line (#997).
|
|
7645
|
+
log: (line: string) =>
|
|
7646
|
+
log(runtimes.length === 1 ? line : `[${runtime.d.project.name}] ${line}`),
|
|
7464
7647
|
})),
|
|
7465
7648
|
turnLimits,
|
|
7466
7649
|
workerControls,
|
package/src/dashboard/app.js
CHANGED
|
@@ -140,7 +140,7 @@ async function runControl(path, body, describe) {
|
|
|
140
140
|
: JSON.stringify(answer.body);
|
|
141
141
|
showControlResult(
|
|
142
142
|
answer.ok ? `${describe}: ok` : `${describe} refused (${answer.status}): ${detail}`,
|
|
143
|
-
|
|
143
|
+
!answer.ok,
|
|
144
144
|
);
|
|
145
145
|
} catch (err) {
|
|
146
146
|
showControlResult(`${describe} failed: ${String(err.message ?? err)}`, true);
|
|
@@ -188,7 +188,7 @@ controls.addEventListener("click", (event) => {
|
|
|
188
188
|
if (reason === null || reason.trim() === "") return;
|
|
189
189
|
if (
|
|
190
190
|
action === "hold" &&
|
|
191
|
-
|
|
191
|
+
!confirmDestructive(
|
|
192
192
|
"Hold stops new claims AND disarms ticks. Re-arming needs a Telegram challenge answered in the chat. Continue?",
|
|
193
193
|
)
|
|
194
194
|
) {
|
|
@@ -447,7 +447,7 @@ function renderRunControls(issue) {
|
|
|
447
447
|
const reason = window.prompt(`Reason for stopping #${issue} (recorded in the run's report):`);
|
|
448
448
|
if (reason === null || reason.trim() === "") return;
|
|
449
449
|
if (
|
|
450
|
-
|
|
450
|
+
!confirmDestructive(
|
|
451
451
|
`Stop #${issue}? This settles the run terminally, salvages its tree and frees the slot. It cannot be resumed.`,
|
|
452
452
|
)
|
|
453
453
|
) {
|
|
@@ -474,7 +474,7 @@ function renderRunControls(issue) {
|
|
|
474
474
|
);
|
|
475
475
|
button("Unblock --force", "destructive", () => {
|
|
476
476
|
if (
|
|
477
|
-
|
|
477
|
+
!confirmDestructive(
|
|
478
478
|
`Force-unblock #${issue}? This accepts the loss of uncommitted work in its worktree, which may be the only copy.`,
|
|
479
479
|
)
|
|
480
480
|
) {
|
|
@@ -189,7 +189,7 @@ export async function dashboardWorkerControl(
|
|
|
189
189
|
body: unknown,
|
|
190
190
|
d: ControlDeps,
|
|
191
191
|
): Promise<ControlOutcome> {
|
|
192
|
-
const payload: Record<string, unknown> = { project: d.project.name };
|
|
192
|
+
const payload: Record<string, unknown> = { project: d.project.name, source: "dashboard" };
|
|
193
193
|
if (action === "stop") {
|
|
194
194
|
const reason = readReason(body);
|
|
195
195
|
if (reason === undefined) {
|