omp-conductor 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -2
- package/package.json +1 -1
- package/src/board.ts +124 -10
- package/src/briefs/orchestrator.md +18 -2
- package/src/briefs/policy.md +4 -1
- package/src/chain-check.ts +157 -0
- package/src/cli.ts +90 -2
- package/src/config.ts +185 -13
- package/src/daemon.ts +238 -27
- package/src/diff-flags.ts +44 -3
- package/src/digest-schedule.ts +59 -0
- package/src/escalate.ts +17 -0
- package/src/failure-class.ts +25 -0
- package/src/fleet.ts +1 -1
- package/src/gitops.ts +49 -0
- package/src/omp.ts +36 -5
- package/src/orchestrator-tick.ts +73 -3
- package/src/plugin.ts +18 -2
- package/src/reports.ts +5 -6
- package/src/session-host.ts +19 -3
- package/src/setup.ts +39 -8
- package/src/store.ts +73 -2
- package/src/types.ts +110 -4
- package/src/verbs/server.ts +49 -0
- package/src/worker.ts +152 -14
package/src/daemon.ts
CHANGED
|
@@ -35,7 +35,7 @@ import { recordReleaseBlock } from "./release-policy.ts";
|
|
|
35
35
|
import { branchName, effectiveLabels, route } from "./routing.ts";
|
|
36
36
|
import type { Routed, UnroutableReason } from "./routing.ts";
|
|
37
37
|
import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
|
|
38
|
-
import { classifyRun, providerCreditRefusal, type ClassifyFacts } from "./failure-class.ts";
|
|
38
|
+
import { classifyRun, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
|
|
39
39
|
import { projectLabels } from "./label-projection.ts";
|
|
40
40
|
import { dbPath, openStore, utcDay } from "./store.ts";
|
|
41
41
|
import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
|
|
@@ -61,7 +61,14 @@ import type {
|
|
|
61
61
|
Tracker,
|
|
62
62
|
VerbLedgerEntry,
|
|
63
63
|
} from "./types.ts";
|
|
64
|
-
import {
|
|
64
|
+
import {
|
|
65
|
+
type KilledBy,
|
|
66
|
+
type WorkerPauseControl,
|
|
67
|
+
type WorkerPausePhase,
|
|
68
|
+
type WorkerResult,
|
|
69
|
+
renderBrief,
|
|
70
|
+
runWorker,
|
|
71
|
+
} from "./worker.ts";
|
|
65
72
|
import {
|
|
66
73
|
addRunRepo,
|
|
67
74
|
cleanupRetainedWorktree,
|
|
@@ -86,7 +93,7 @@ import {
|
|
|
86
93
|
} from "./verbs/socket.ts";
|
|
87
94
|
import { homedir } from "node:os";
|
|
88
95
|
|
|
89
|
-
import { pushRunBranch, type RunRepoRef } from "./gitops.ts";
|
|
96
|
+
import { pushRunBranch, readBaseChain, type RunRepoRef } from "./gitops.ts";
|
|
90
97
|
import {
|
|
91
98
|
planUsageLine,
|
|
92
99
|
readPlanUsage,
|
|
@@ -108,6 +115,11 @@ const REPORT_DELIVERY_INTERVAL_MS = 30_000;
|
|
|
108
115
|
* means the mirror itself is broken, not unlucky, and the sweep escalates
|
|
109
116
|
* instead of burning a turn-0 run per tick forever (#168, #177). */
|
|
110
117
|
const DISPATCH_INFRA_MAX_STRIKES = 3;
|
|
118
|
+
/** A provider-transient requeue (stream stalled mid-run) is retried, but only a
|
|
119
|
+
* bounded number of times: three aborted streams for one issue means the
|
|
120
|
+
* provider itself is degraded, not unlucky, and the sweep escalates to a
|
|
121
|
+
* human instead of requeueing into a down provider forever (#220). */
|
|
122
|
+
const PROVIDER_TRANSIENT_MAX_STRIKES = 3;
|
|
111
123
|
const DEFAULT_PORT = 8787;
|
|
112
124
|
const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
|
|
113
125
|
|
|
@@ -144,6 +156,7 @@ interface Deps {
|
|
|
144
156
|
usage: UsageSource;
|
|
145
157
|
escalate(e: Escalation): Promise<void>;
|
|
146
158
|
turnLimits: TurnLimitRegistry;
|
|
159
|
+
workerControls: WorkerControlRegistry;
|
|
147
160
|
integrity: IntegrityGate;
|
|
148
161
|
stall: StallGate;
|
|
149
162
|
cleanup?: RetainedCleanupCursor;
|
|
@@ -190,6 +203,7 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
|
|
|
190
203
|
pausedAt,
|
|
191
204
|
log,
|
|
192
205
|
now: () => Date.now(),
|
|
206
|
+
chain: { readBaseChain },
|
|
193
207
|
};
|
|
194
208
|
}
|
|
195
209
|
|
|
@@ -264,6 +278,7 @@ async function watchOrchestrator(d: Deps): Promise<void> {
|
|
|
264
278
|
log(`ERROR: the orchestrator session is not draining its queue — ${verdict.since ?? "no timestamp"}`);
|
|
265
279
|
const delivered = await safeEscalate(d, {
|
|
266
280
|
tier: 2,
|
|
281
|
+
category: "confirmed-failure",
|
|
267
282
|
project: d.project.name,
|
|
268
283
|
issue: NO_ISSUE,
|
|
269
284
|
// Keyed on the marker's own timestamp, not the date. The dedup ledger keys
|
|
@@ -823,6 +838,91 @@ export function createTurnLimitRegistry(
|
|
|
823
838
|
};
|
|
824
839
|
}
|
|
825
840
|
|
|
841
|
+
export type WorkerControlResult =
|
|
842
|
+
| { kind: "ok"; runId: string; phase: WorkerPausePhase }
|
|
843
|
+
| { kind: "refused"; runId: string; error: string }
|
|
844
|
+
| { kind: "not-active" };
|
|
845
|
+
|
|
846
|
+
export interface WorkerControlSlot {
|
|
847
|
+
install(control: WorkerPauseControl): void;
|
|
848
|
+
close(): void;
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
export interface WorkerControlRegistry {
|
|
852
|
+
open(project: string, issue: number, runId: string): WorkerControlSlot;
|
|
853
|
+
pause(project: string, issue: number): Promise<WorkerControlResult>;
|
|
854
|
+
resume(project: string, issue: number): WorkerControlResult;
|
|
855
|
+
/** Live runs whose phase is not `running` — what /healthz and the board show. */
|
|
856
|
+
snapshot(project: string): { issue: number; runId: string; phase: WorkerPausePhase }[];
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
/** Authoritative controls for sessions owned by this daemon process. */
|
|
860
|
+
export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
861
|
+
const active = new Map<
|
|
862
|
+
string,
|
|
863
|
+
{ project: string; issue: number; runId: string; control?: WorkerPauseControl }
|
|
864
|
+
>();
|
|
865
|
+
const key = (project: string, issue: number): string => `${project}\0${issue}`;
|
|
866
|
+
return {
|
|
867
|
+
open(project, issue, runId) {
|
|
868
|
+
const k = key(project, issue);
|
|
869
|
+
if (active.has(k)) throw new Error(`#${issue} already has a live worker controller`);
|
|
870
|
+
const entry = { project, issue, runId } as {
|
|
871
|
+
project: string;
|
|
872
|
+
issue: number;
|
|
873
|
+
runId: string;
|
|
874
|
+
control?: WorkerPauseControl;
|
|
875
|
+
};
|
|
876
|
+
active.set(k, entry);
|
|
877
|
+
return {
|
|
878
|
+
install: (control) => {
|
|
879
|
+
if (active.get(k) === entry) entry.control = control;
|
|
880
|
+
},
|
|
881
|
+
close: () => {
|
|
882
|
+
if (active.get(k) === entry) active.delete(k);
|
|
883
|
+
},
|
|
884
|
+
};
|
|
885
|
+
},
|
|
886
|
+
async pause(project, issue) {
|
|
887
|
+
const entry = active.get(key(project, issue));
|
|
888
|
+
if (entry?.control === undefined) return { kind: "not-active" };
|
|
889
|
+
try {
|
|
890
|
+
await entry.control.pause();
|
|
891
|
+
return { kind: "ok", runId: entry.runId, phase: entry.control.phase() };
|
|
892
|
+
} catch (err) {
|
|
893
|
+
return {
|
|
894
|
+
kind: "refused",
|
|
895
|
+
runId: entry.runId,
|
|
896
|
+
error: err instanceof Error ? err.message : String(err),
|
|
897
|
+
};
|
|
898
|
+
}
|
|
899
|
+
},
|
|
900
|
+
resume(project, issue) {
|
|
901
|
+
const entry = active.get(key(project, issue));
|
|
902
|
+
if (entry?.control === undefined) return { kind: "not-active" };
|
|
903
|
+
try {
|
|
904
|
+
entry.control.resume();
|
|
905
|
+
return { kind: "ok", runId: entry.runId, phase: entry.control.phase() };
|
|
906
|
+
} catch (err) {
|
|
907
|
+
return {
|
|
908
|
+
kind: "refused",
|
|
909
|
+
runId: entry.runId,
|
|
910
|
+
error: err instanceof Error ? err.message : String(err),
|
|
911
|
+
};
|
|
912
|
+
}
|
|
913
|
+
},
|
|
914
|
+
snapshot(project) {
|
|
915
|
+
const workers: { issue: number; runId: string; phase: WorkerPausePhase }[] = [];
|
|
916
|
+
for (const entry of active.values()) {
|
|
917
|
+
if (entry.project !== project || entry.control === undefined) continue;
|
|
918
|
+
const phase = entry.control.phase();
|
|
919
|
+
if (phase !== "running") workers.push({ issue: entry.issue, runId: entry.runId, phase });
|
|
920
|
+
}
|
|
921
|
+
return workers;
|
|
922
|
+
},
|
|
923
|
+
};
|
|
924
|
+
}
|
|
925
|
+
|
|
826
926
|
export async function verifyPushedGreenClaim(
|
|
827
927
|
tracker: Pick<Tracker, "verifyPr">,
|
|
828
928
|
claim: Pick<WorkerResult, "prUrl" | "headSha">,
|
|
@@ -926,6 +1026,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
926
1026
|
const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
|
|
927
1027
|
let worktreePath: string | undefined;
|
|
928
1028
|
let turnLimit: TurnLimitController | undefined;
|
|
1029
|
+
let workerControl: WorkerControlSlot | undefined;
|
|
929
1030
|
// The run's own repository. Hoisted for the same reason `worktreePath` is —
|
|
930
1031
|
// the catch and finally paths have to publish the branch.
|
|
931
1032
|
let runRepo: RunRepoRef | undefined;
|
|
@@ -981,6 +1082,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
981
1082
|
store.enqueueLabelOps(project.name, [{ issue, op: "add", label: inProgress }]);
|
|
982
1083
|
claimed = true;
|
|
983
1084
|
turnLimit = d.turnLimits.open(project.name, issue, runId, caps.workerMaxTurns);
|
|
1085
|
+
workerControl = d.workerControls.open(project.name, issue, runId);
|
|
984
1086
|
|
|
985
1087
|
// A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
|
|
986
1088
|
// an existing path, so a retry — or a tree kept from a failed attempt — has
|
|
@@ -1057,6 +1159,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1057
1159
|
cwd: worktreePath,
|
|
1058
1160
|
caps,
|
|
1059
1161
|
maxTurns: () => turnLimit?.maxTurns() ?? caps.workerMaxTurns,
|
|
1162
|
+
onPauseControl: (control) => workerControl?.install(control),
|
|
1060
1163
|
sessionDir,
|
|
1061
1164
|
// The session's control socket, under the daemon's own state directory —
|
|
1062
1165
|
// a child process of the daemon reaches it directly.
|
|
@@ -1081,6 +1184,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1081
1184
|
onKilled: () => {
|
|
1082
1185
|
turnLimit?.close();
|
|
1083
1186
|
turnLimit = undefined;
|
|
1187
|
+
workerControl?.close();
|
|
1188
|
+
workerControl = undefined;
|
|
1084
1189
|
},
|
|
1085
1190
|
// Recorded the moment the session opens its transcript, not when the run
|
|
1086
1191
|
// ends: `omp-conductor tail` resolves an issue to a file through this row,
|
|
@@ -1093,6 +1198,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1093
1198
|
// PR verification or terminal row writes can leave stale `running` state.
|
|
1094
1199
|
turnLimit?.close();
|
|
1095
1200
|
turnLimit = undefined;
|
|
1201
|
+
workerControl?.close();
|
|
1202
|
+
workerControl = undefined;
|
|
1096
1203
|
}
|
|
1097
1204
|
|
|
1098
1205
|
// A configured model the harness could not honour means this run was done by
|
|
@@ -1110,13 +1217,10 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1110
1217
|
|
|
1111
1218
|
// Read before the row is written so `lastError` carries the provider's own
|
|
1112
1219
|
// message — it names the fix and the URL, which no classification can.
|
|
1113
|
-
const
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
return err === undefined ? undefined : providerCreditRefusal(err);
|
|
1118
|
-
})()
|
|
1119
|
-
: undefined;
|
|
1220
|
+
const sessionErr = state === "failed" || state === "killed" ? readSessionError(result.sessionFile) : undefined;
|
|
1221
|
+
const providerCredit = sessionErr === undefined ? undefined : providerCreditRefusal(sessionErr);
|
|
1222
|
+
const providerTransient =
|
|
1223
|
+
providerCredit !== undefined || sessionErr === undefined ? undefined : providerTransientFault(sessionErr);
|
|
1120
1224
|
|
|
1121
1225
|
// The other half of not believing a worker about its own run (#128). The
|
|
1122
1226
|
// claim being audited is `state: pushed-green`, so the audit runs on the
|
|
@@ -1202,8 +1306,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1202
1306
|
// push: the report of a killed attempt is exactly the one a later
|
|
1203
1307
|
// continuation must pool its disclosures from (#199).
|
|
1204
1308
|
report: result.report,
|
|
1205
|
-
...(providerCredit !== undefined
|
|
1206
|
-
? { lastError: providerCredit }
|
|
1309
|
+
...(providerCredit !== undefined || providerTransient !== undefined
|
|
1310
|
+
? { lastError: providerCredit ?? providerTransient }
|
|
1207
1311
|
: verified.reason === undefined
|
|
1208
1312
|
? {}
|
|
1209
1313
|
: { lastError: verified.reason }),
|
|
@@ -1251,6 +1355,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1251
1355
|
// instead of once per affected run.
|
|
1252
1356
|
await safeEscalate(d, {
|
|
1253
1357
|
tier: 2,
|
|
1358
|
+
category: "fleet-stopped",
|
|
1254
1359
|
project: project.name,
|
|
1255
1360
|
issue: NO_ISSUE,
|
|
1256
1361
|
summary: `Model provider refused for credit — ${project.name} is paused`,
|
|
@@ -1364,6 +1469,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1364
1469
|
// inner settlement guard exists. Latch it before any terminal write or await.
|
|
1365
1470
|
turnLimit?.close();
|
|
1366
1471
|
turnLimit = undefined;
|
|
1472
|
+
workerControl?.close();
|
|
1473
|
+
workerControl = undefined;
|
|
1367
1474
|
const detail = errText(err);
|
|
1368
1475
|
log(`#${issue} errored: ${detail}`);
|
|
1369
1476
|
// A crash lands anywhere, including mid-edit in a tree holding the only
|
|
@@ -1409,6 +1516,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1409
1516
|
// a failure path, and whatever it still held is now a commit on the branch.
|
|
1410
1517
|
} finally {
|
|
1411
1518
|
turnLimit?.close();
|
|
1519
|
+
workerControl?.close();
|
|
1520
|
+
workerControl = undefined;
|
|
1412
1521
|
// The run is over, so its channel is too. Closed here rather than beside
|
|
1413
1522
|
// the session so the crash path closes it as well: a listener left bound
|
|
1414
1523
|
// after its run settled is a socket whose `run-not-live` check is the only
|
|
@@ -1730,6 +1839,7 @@ const DEGRADED_HOLDS: ReadonlySet<AdmissionHoldReason> = new Set([
|
|
|
1730
1839
|
export function summarizeDispatch(
|
|
1731
1840
|
ready: number,
|
|
1732
1841
|
routed: number,
|
|
1842
|
+
claimed: number,
|
|
1733
1843
|
admitted: number,
|
|
1734
1844
|
holds: readonly AdmissionHold[],
|
|
1735
1845
|
completedAt = Date.now(),
|
|
@@ -1745,6 +1855,7 @@ export function summarizeDispatch(
|
|
|
1745
1855
|
completedAt,
|
|
1746
1856
|
ready,
|
|
1747
1857
|
routed,
|
|
1858
|
+
claimed,
|
|
1748
1859
|
admitted,
|
|
1749
1860
|
degraded: holds.some((hold) => DEGRADED_HOLDS.has(hold.reason)),
|
|
1750
1861
|
holds: [...groups]
|
|
@@ -1792,6 +1903,7 @@ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation
|
|
|
1792
1903
|
return {
|
|
1793
1904
|
...base,
|
|
1794
1905
|
tier: 2,
|
|
1906
|
+
category: "fleet-stopped",
|
|
1795
1907
|
// Dated: a meter that breaks again next month is a new incident, not a
|
|
1796
1908
|
// repeat of this one.
|
|
1797
1909
|
summary: `Plan usage source unreadable — ${project} is not claiming new work (${new Date().toISOString().slice(0, 10)})`,
|
|
@@ -1810,6 +1922,7 @@ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation
|
|
|
1810
1922
|
return {
|
|
1811
1923
|
...base,
|
|
1812
1924
|
tier: 2,
|
|
1925
|
+
category: "fleet-stopped",
|
|
1813
1926
|
summary: `Plan usage cap names an unusable window "${plan.cap?.windowId ?? "?"}" — ${project} is not claiming new work`,
|
|
1814
1927
|
detail: [
|
|
1815
1928
|
plan.detail,
|
|
@@ -2284,6 +2397,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2284
2397
|
if (integrity.page) {
|
|
2285
2398
|
const delivered = await safeEscalate(d, {
|
|
2286
2399
|
tier: 2,
|
|
2400
|
+
category: "fleet-stopped",
|
|
2287
2401
|
project: project.name,
|
|
2288
2402
|
issue: NO_ISSUE,
|
|
2289
2403
|
// Dated for the same reason the spend cap is: the dedup key is the
|
|
@@ -2327,6 +2441,10 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2327
2441
|
return pending.length === 0 ? issue : { ...issue, labels: effectiveLabels(issue.labels, pending) };
|
|
2328
2442
|
});
|
|
2329
2443
|
const { routed, unroutable } = route(effective, project);
|
|
2444
|
+
// Ready issues that route() dropped without a reason carry a state label —
|
|
2445
|
+
// claimed by a live or settling run — so the claimed count is subtraction
|
|
2446
|
+
// rather than a routed-state signal (#228).
|
|
2447
|
+
const claimed = effective.length - routed.length - unroutable.length;
|
|
2330
2448
|
const routingHolds: AdmissionHold[] = unroutable.map((u) => ({
|
|
2331
2449
|
issue: u.issue.number,
|
|
2332
2450
|
reason: `unroutable:${u.reason}`,
|
|
@@ -2334,7 +2452,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2334
2452
|
const recordDispatch = (admitted: number, holds: readonly AdmissionHold[]): void => {
|
|
2335
2453
|
store.recordDispatch(
|
|
2336
2454
|
project.name,
|
|
2337
|
-
summarizeDispatch(ready.length, routed.length, admitted, holds),
|
|
2455
|
+
summarizeDispatch(ready.length, routed.length, claimed, admitted, holds),
|
|
2338
2456
|
);
|
|
2339
2457
|
};
|
|
2340
2458
|
|
|
@@ -2368,6 +2486,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2368
2486
|
setPaused(true, { source: "spend-cap", reason: `daily spend reached $${caps.dailySpendUsd}` });
|
|
2369
2487
|
await safeEscalate(d, {
|
|
2370
2488
|
tier: 2,
|
|
2489
|
+
category: "fleet-stopped",
|
|
2371
2490
|
project: project.name,
|
|
2372
2491
|
issue: NO_ISSUE,
|
|
2373
2492
|
// Dated so the same cap pages again tomorrow, but only once per day.
|
|
@@ -2433,6 +2552,8 @@ export interface DaemonHealthSnapshot {
|
|
|
2433
2552
|
rssBytes: number;
|
|
2434
2553
|
dispatch?: DispatchSummary;
|
|
2435
2554
|
codeGraph?: CodeGraphHealth;
|
|
2555
|
+
/** Live workers in a non-running pause phase; absent/empty = nothing paused. */
|
|
2556
|
+
workers?: { issue: number; runId: string; phase: WorkerPausePhase }[];
|
|
2436
2557
|
}
|
|
2437
2558
|
|
|
2438
2559
|
export function daemonHealthSnapshot(
|
|
@@ -2441,6 +2562,7 @@ export function daemonHealthSnapshot(
|
|
|
2441
2562
|
paused = isPaused(),
|
|
2442
2563
|
rssBytes = process.memoryUsage().rss,
|
|
2443
2564
|
codeGraph?: CodeGraphHealth,
|
|
2565
|
+
workerControls?: WorkerControlRegistry,
|
|
2444
2566
|
): DaemonHealthSnapshot {
|
|
2445
2567
|
const dispatch = store.latestDispatch(project);
|
|
2446
2568
|
return {
|
|
@@ -2451,6 +2573,7 @@ export function daemonHealthSnapshot(
|
|
|
2451
2573
|
rssBytes,
|
|
2452
2574
|
...(dispatch === undefined ? {} : { dispatch }),
|
|
2453
2575
|
...(codeGraph?.configured === true ? { codeGraph } : {}),
|
|
2576
|
+
...(workerControls === undefined ? {} : { workers: workerControls.snapshot(project) }),
|
|
2454
2577
|
};
|
|
2455
2578
|
}
|
|
2456
2579
|
|
|
@@ -2515,10 +2638,70 @@ export async function turnLimitResponse(
|
|
|
2515
2638
|
);
|
|
2516
2639
|
}
|
|
2517
2640
|
|
|
2641
|
+
export async function workerControlResponse(
|
|
2642
|
+
req: Request,
|
|
2643
|
+
project: string,
|
|
2644
|
+
store: Pick<Store, "latestRun">,
|
|
2645
|
+
registry: WorkerControlRegistry,
|
|
2646
|
+
): Promise<Response | undefined> {
|
|
2647
|
+
const url = new URL(req.url);
|
|
2648
|
+
const match = /^\/runs\/(\d+)\/(pause|resume)$/.exec(url.pathname);
|
|
2649
|
+
if (req.method !== "PUT" || match === null) return undefined;
|
|
2650
|
+
if (!req.headers.get("content-type")?.startsWith("application/json")) {
|
|
2651
|
+
return Response.json({ error: "content-type must be application/json" }, { status: 415 });
|
|
2652
|
+
}
|
|
2653
|
+
|
|
2654
|
+
let body: unknown;
|
|
2655
|
+
try {
|
|
2656
|
+
body = await req.json();
|
|
2657
|
+
} catch {
|
|
2658
|
+
return Response.json({ error: "request body must be valid JSON" }, { status: 400 });
|
|
2659
|
+
}
|
|
2660
|
+
if (body === null || typeof body !== "object") {
|
|
2661
|
+
return Response.json({ error: "request body must be a JSON object" }, { status: 400 });
|
|
2662
|
+
}
|
|
2663
|
+
const requestedProject = Reflect.get(body, "project");
|
|
2664
|
+
if (typeof requestedProject !== "string" || requestedProject.length === 0) {
|
|
2665
|
+
return Response.json({ error: "project must be a non-empty string" }, { status: 400 });
|
|
2666
|
+
}
|
|
2667
|
+
if (requestedProject !== project) {
|
|
2668
|
+
return Response.json(
|
|
2669
|
+
{ error: `daemon serves project "${project}", not requested project "${requestedProject}"` },
|
|
2670
|
+
{ status: 409 },
|
|
2671
|
+
);
|
|
2672
|
+
}
|
|
2673
|
+
|
|
2674
|
+
const issue = Number(match[1]);
|
|
2675
|
+
const outcome =
|
|
2676
|
+
match[2] === "pause"
|
|
2677
|
+
? await registry.pause(project, issue)
|
|
2678
|
+
: registry.resume(project, issue);
|
|
2679
|
+
if (outcome.kind === "ok") {
|
|
2680
|
+
return Response.json({ runId: outcome.runId, phase: outcome.phase });
|
|
2681
|
+
}
|
|
2682
|
+
if (outcome.kind === "refused") {
|
|
2683
|
+
return Response.json({ error: `#${issue}: ${outcome.error}` }, { status: 409 });
|
|
2684
|
+
}
|
|
2685
|
+
|
|
2686
|
+
const latest = store.latestRun(project, issue);
|
|
2687
|
+
if (latest === undefined) {
|
|
2688
|
+
return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
|
|
2689
|
+
}
|
|
2690
|
+
return Response.json(
|
|
2691
|
+
{
|
|
2692
|
+
error:
|
|
2693
|
+
`#${issue} has no live worker controller; its session already settled ` +
|
|
2694
|
+
`or belongs to another daemon (stored state: ${latest.state})`,
|
|
2695
|
+
},
|
|
2696
|
+
{ status: 409 },
|
|
2697
|
+
);
|
|
2698
|
+
}
|
|
2699
|
+
|
|
2518
2700
|
export interface DaemonHttpDeps {
|
|
2519
2701
|
project: string;
|
|
2520
2702
|
store: Pick<Store, "latestRun">;
|
|
2521
2703
|
turnLimits: TurnLimitRegistry;
|
|
2704
|
+
workerControls: WorkerControlRegistry;
|
|
2522
2705
|
health: () => DaemonHealthSnapshot;
|
|
2523
2706
|
}
|
|
2524
2707
|
|
|
@@ -2526,17 +2709,18 @@ export interface DaemonHttpDeps {
|
|
|
2526
2709
|
* The whole HTTP surface, in one named function so a test can pin what is *not*
|
|
2527
2710
|
* on it.
|
|
2528
2711
|
*
|
|
2529
|
-
*
|
|
2530
|
-
*
|
|
2531
|
-
*
|
|
2532
|
-
*
|
|
2533
|
-
* a test asserts rather than something a reviewer has to notice (#126).
|
|
2712
|
+
* Three route families: the health read, turn-limit control, and worker
|
|
2713
|
+
* pause/resume control. Everything else is 404. The controls mutate only
|
|
2714
|
+
* daemon-owned live sessions; tracker and repository mutations stay on the
|
|
2715
|
+
* authenticated per-run channel described at the `Bun.serve` call (#126).
|
|
2534
2716
|
*/
|
|
2535
2717
|
export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promise<Response> {
|
|
2536
2718
|
const url = new URL(req.url);
|
|
2537
2719
|
if (req.method === "GET" && url.pathname === "/healthz") return Response.json(d.health());
|
|
2538
|
-
const
|
|
2539
|
-
|
|
2720
|
+
const turnLimit = await turnLimitResponse(req, d.project, d.store, d.turnLimits);
|
|
2721
|
+
if (turnLimit !== undefined) return turnLimit;
|
|
2722
|
+
const workerControl = await workerControlResponse(req, d.project, d.store, d.workerControls);
|
|
2723
|
+
return workerControl ?? new Response("not found\n", { status: 404 });
|
|
2540
2724
|
}
|
|
2541
2725
|
|
|
2542
2726
|
export interface StatusSnapshot {
|
|
@@ -2672,7 +2856,7 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
|
|
|
2672
2856
|
if (summary === undefined) return "last dispatch (none recorded)";
|
|
2673
2857
|
const lines = [
|
|
2674
2858
|
`last dispatch ${new Date(summary.completedAt).toISOString()}${summary.degraded ? " DEGRADED" : ""}`,
|
|
2675
|
-
` candidates ${summary.ready} ready / ${summary.routed}
|
|
2859
|
+
` candidates ${summary.ready} ready / ${summary.claimed ?? 0} in flight / ${summary.routed} spare`,
|
|
2676
2860
|
` admitted ${summary.admitted}`,
|
|
2677
2861
|
];
|
|
2678
2862
|
if (summary.holds.length === 0) {
|
|
@@ -3104,6 +3288,28 @@ async function recoverRun(
|
|
|
3104
3288
|
log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
|
|
3105
3289
|
return;
|
|
3106
3290
|
}
|
|
3291
|
+
// Same bound for provider-transient: an issue whose stream keeps stalling
|
|
3292
|
+
// mid-run is requeued free (no attempt, no continuation charged) — but a
|
|
3293
|
+
// provider that aborts three times for one issue is down, and a human has
|
|
3294
|
+
// to check its status before hand-requeueing (#220).
|
|
3295
|
+
if (
|
|
3296
|
+
cls === "provider-transient" &&
|
|
3297
|
+
store.classCountFor(project.name, run.issue, "provider-transient") >= PROVIDER_TRANSIENT_MAX_STRIKES
|
|
3298
|
+
) {
|
|
3299
|
+
await safeEscalate(d, {
|
|
3300
|
+
tier: 1,
|
|
3301
|
+
project: project.name,
|
|
3302
|
+
issue: run.issue,
|
|
3303
|
+
summary: `[provider-transient] #${run.issue}: the provider keeps aborting mid-stream — ${evidence}`,
|
|
3304
|
+
detail: [
|
|
3305
|
+
`The provider aborted the stream for #${run.issue} ${PROVIDER_TRANSIENT_MAX_STRIKES} times without the run ever producing a verdict (0 tokens billed each time).`,
|
|
3306
|
+
"Check provider status before requeueing by hand.",
|
|
3307
|
+
].join("\n"),
|
|
3308
|
+
});
|
|
3309
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
3310
|
+
log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
|
|
3311
|
+
return;
|
|
3312
|
+
}
|
|
3107
3313
|
// Only when the tracker still shows this issue as ours to hand back. An
|
|
3108
3314
|
// issue that is closed, or has no state label, was resolved by another route
|
|
3109
3315
|
// and requeueing it would dispatch work nobody asked for.
|
|
@@ -3521,6 +3727,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3521
3727
|
const turnLimits = createTurnLimitRegistry((runId, maxTurns) => {
|
|
3522
3728
|
store.updateRun(runId, { maxTurns });
|
|
3523
3729
|
});
|
|
3730
|
+
const workerControls = createWorkerControlRegistry();
|
|
3524
3731
|
const d: Deps = {
|
|
3525
3732
|
project,
|
|
3526
3733
|
caps,
|
|
@@ -3531,6 +3738,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3531
3738
|
usage: sharedUsageSource(),
|
|
3532
3739
|
escalate: (e) => escalator.escalate(e),
|
|
3533
3740
|
turnLimits,
|
|
3741
|
+
workerControls,
|
|
3534
3742
|
integrity,
|
|
3535
3743
|
// Fresh per daemon run, like the integrity gate: a restart is entitled to
|
|
3536
3744
|
// page again about a stall that is still on disk.
|
|
@@ -3607,14 +3815,15 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3607
3815
|
process.on("SIGINT", stop);
|
|
3608
3816
|
process.on("SIGTERM", stop);
|
|
3609
3817
|
|
|
3610
|
-
// ── NO MUTATION
|
|
3818
|
+
// ── NO TRACKER OR REPOSITORY MUTATION BELONGS ON THIS PORT ───────────────
|
|
3611
3819
|
//
|
|
3612
3820
|
// This is unauthenticated loopback TCP. Every local user can reach it, it
|
|
3613
3821
|
// carries no credential of any kind, and it cannot tell one caller from
|
|
3614
|
-
// another: `127.0.0.1` is not an identity.
|
|
3615
|
-
//
|
|
3616
|
-
// shape "identity from the payload" takes when nobody is watching
|
|
3617
|
-
// tolerable only because
|
|
3822
|
+
// another: `127.0.0.1` is not an identity. The turn-limit and worker
|
|
3823
|
+
// pause/resume controls trust a body-supplied `project`, which is exactly the
|
|
3824
|
+
// shape "identity from the payload" takes when nobody is watching. Those
|
|
3825
|
+
// controls are tolerable only because they are bounded, live-run-local, and
|
|
3826
|
+
// reversible.
|
|
3618
3827
|
//
|
|
3619
3828
|
// A merge, a push, a release or a label is none of those things. Do not add
|
|
3620
3829
|
// one here, and do not add "just a small one" behind a shared secret either:
|
|
@@ -3630,7 +3839,9 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3630
3839
|
project: project.name,
|
|
3631
3840
|
store,
|
|
3632
3841
|
turnLimits,
|
|
3633
|
-
|
|
3842
|
+
workerControls,
|
|
3843
|
+
health: () =>
|
|
3844
|
+
daemonHealthSnapshot(store, project.name, undefined, undefined, codeGraph, workerControls),
|
|
3634
3845
|
}),
|
|
3635
3846
|
});
|
|
3636
3847
|
log(`serving /healthz on :${server.port}, project ${project.name}`);
|
package/src/diff-flags.ts
CHANGED
|
@@ -266,17 +266,49 @@ const CHANGED_LINE = /^[ \t>]*changed:[ \t]*(.*)$/im;
|
|
|
266
266
|
*/
|
|
267
267
|
const CLAIMED_PATH = /^(?:[\w.@~+-]+\/)+[\w.@~*+-]*$|^[\w.@~+-]*\.[A-Za-z][A-Za-z0-9]{1,7}$/;
|
|
268
268
|
|
|
269
|
+
/** A Python dotted module: `services.rule_purge`, `omp.routing`. Not a path —
|
|
270
|
+
* segments joined by `.` — but a legitimate way for a report to name the file
|
|
271
|
+
* it touched without disclosing the directory it lives in (#224).
|
|
272
|
+
*
|
|
273
|
+
* Every segment is at least two characters so prose abbreviations (`e.g`,
|
|
274
|
+
* `i.e.`, `a.k.a`) stay commentary: they are exactly the prose the path
|
|
275
|
+
* heuristic exists to reject, and each one that slipped through would become
|
|
276
|
+
* an "you claimed a file you never touched" flag on an honest report. */
|
|
277
|
+
const DOTTED_MODULE = /^[A-Za-z_]\w+(?:\.[A-Za-z_]\w+)+$/;
|
|
278
|
+
|
|
279
|
+
/** One-level brace expansion: `a/{b,c}.py` → [`a/b.py`, `a/c.py`]. Braces are
|
|
280
|
+
* how a report compresses adjacent files into one token; each comma-separated
|
|
281
|
+
* alternative is substituted into the skeleton. Nesting is deliberately
|
|
282
|
+
* unsupported — one level matches the spellings reports actually produce. */
|
|
283
|
+
function expandBraces(token: string): string[] {
|
|
284
|
+
const group = /\{[^{}]*\}/.exec(token);
|
|
285
|
+
if (group === null) return [token];
|
|
286
|
+
const alternatives = group[0].slice(1, -1).split(",").filter((a) => a.length > 0);
|
|
287
|
+
if (alternatives.length === 0) return [token];
|
|
288
|
+
const prefix = token.slice(0, group.index);
|
|
289
|
+
const suffix = token.slice(group.index + group[0].length);
|
|
290
|
+
return alternatives.map((alt) => prefix + alt + suffix);
|
|
291
|
+
}
|
|
292
|
+
|
|
269
293
|
/** Every path-shaped token on the report's `changed:` line. An absent line and
|
|
270
294
|
* a line naming nothing are the same answer: nothing was disclosed. */
|
|
271
295
|
export function claimedPaths(report: string): string[] {
|
|
272
296
|
const line = CHANGED_LINE.exec(report)?.[1] ?? "";
|
|
273
297
|
const seen = new Set<string>();
|
|
274
|
-
|
|
298
|
+
// Split on whitespace and semicolons, and on commas *outside* a brace group:
|
|
299
|
+
// the comma in `a/{b,c}.py` is an alternative separator, not a token
|
|
300
|
+
// separator, so a brace-compressed token must survive whole (#224).
|
|
301
|
+
for (const raw of line.split(/[\s;]+|,(?![^{}]*\})/)) {
|
|
275
302
|
const token = raw.replace(/^[`'"([*-]+/, "").replace(/[`'")\].,:;]+$/, "");
|
|
276
303
|
if (token === "" || token === "none") continue;
|
|
277
304
|
const normalised = token.replace(/^\.?\//, "");
|
|
278
|
-
|
|
279
|
-
|
|
305
|
+
// Expansion happens here so every downstream consumer — both `covers`
|
|
306
|
+
// directions — sees plain paths: a brace token dies earlier at the
|
|
307
|
+
// CLAIMED_PATH filter if it is never opened up (#224).
|
|
308
|
+
for (const expanded of expandBraces(normalised)) {
|
|
309
|
+
if (!CLAIMED_PATH.test(expanded) && !DOTTED_MODULE.test(expanded)) continue;
|
|
310
|
+
seen.add(expanded);
|
|
311
|
+
}
|
|
280
312
|
}
|
|
281
313
|
return [...seen];
|
|
282
314
|
}
|
|
@@ -305,6 +337,15 @@ function covers(claim: string, path: string): boolean {
|
|
|
305
337
|
return pattern.test(path) || pattern.test(basename(path));
|
|
306
338
|
}
|
|
307
339
|
if (path.endsWith(`/${claim}`) || claim.endsWith(`/${path}`)) return true;
|
|
340
|
+
// A Python dotted module names a file by import path, not by location:
|
|
341
|
+
// `services.rule_purge` covers `backend/app/services/rule_purge.py`. Only
|
|
342
|
+
// consulted after the path rules, because `foo.bar` is also a valid filename
|
|
343
|
+
// and a literal match must win. Matching several changed files is fine — the
|
|
344
|
+
// check wants evidence the worker knew, not a unique index (#224).
|
|
345
|
+
if (DOTTED_MODULE.test(claim)) {
|
|
346
|
+
const fragment = `${claim.replaceAll(".", "/")}.py`;
|
|
347
|
+
if (path === fragment || path.endsWith(`/${fragment}`)) return true;
|
|
348
|
+
}
|
|
308
349
|
const last = claim.slice(claim.lastIndexOf("/") + 1);
|
|
309
350
|
return !last.includes(".") && path.startsWith(`${claim}/`);
|
|
310
351
|
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* When the daily digest is due, in the zone the operator reads it (#229).
|
|
3
|
+
*
|
|
4
|
+
* Pure by construction: every fact arrives as a timestamp and the policy, and
|
|
5
|
+
* nothing here reads a clock or a file. The digest is "one per day" in the
|
|
6
|
+
* IANA zone it is configured with (host zone when none) — a UTC key would roll
|
|
7
|
+
* the digest over mid-evening for anyone west of Greenwich.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import type { ReportingPolicy } from "./types.ts";
|
|
11
|
+
|
|
12
|
+
/** `YYYY-MM-DD` in an IANA timezone (host zone when `timezone` is absent). */
|
|
13
|
+
export function localDayKey(at: number, timezone?: string): string {
|
|
14
|
+
const parts = new Intl.DateTimeFormat("en-CA", {
|
|
15
|
+
timeZone: timezone,
|
|
16
|
+
year: "numeric",
|
|
17
|
+
month: "2-digit",
|
|
18
|
+
day: "2-digit",
|
|
19
|
+
}).formatToParts(new Date(at));
|
|
20
|
+
const get = (type: "year" | "month" | "day"): string =>
|
|
21
|
+
parts.find((p) => p.type === type)?.value ?? "00";
|
|
22
|
+
return `${get("year")}-${get("month")}-${get("day")}`;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** `HH:MM` wall-clock in the zone, 24h and zero-padded. */
|
|
26
|
+
function localClockAt(at: number, timezone?: string): string {
|
|
27
|
+
return new Intl.DateTimeFormat("en-GB", {
|
|
28
|
+
timeZone: timezone,
|
|
29
|
+
hour: "2-digit",
|
|
30
|
+
minute: "2-digit",
|
|
31
|
+
hourCycle: "h23",
|
|
32
|
+
}).format(new Date(at));
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Whether the digest is due now.
|
|
37
|
+
*
|
|
38
|
+
* - `none` → never.
|
|
39
|
+
* - `per-tick` → always (the orchestrator sends it with every report).
|
|
40
|
+
* - `daily` without `at` → once per day in the configured zone.
|
|
41
|
+
* - `daily` with `at` → on a day it has not already run, once the local clock
|
|
42
|
+
* has passed `at`. A restart after `at` still finds today unsent → one
|
|
43
|
+
* catch-up; a fully missed day is skipped, never sent late.
|
|
44
|
+
*/
|
|
45
|
+
export function digestDue(
|
|
46
|
+
policy: Pick<ReportingPolicy, "digest">,
|
|
47
|
+
lastDigestDayKey: string | undefined,
|
|
48
|
+
now: number,
|
|
49
|
+
): boolean {
|
|
50
|
+
const { cadence, at, timezone } = policy.digest;
|
|
51
|
+
if (cadence === "none") return false;
|
|
52
|
+
if (cadence === "per-tick") return true;
|
|
53
|
+
// daily
|
|
54
|
+
if (at === undefined) {
|
|
55
|
+
return lastDigestDayKey !== localDayKey(now, timezone);
|
|
56
|
+
}
|
|
57
|
+
if (lastDigestDayKey === localDayKey(now, timezone)) return false;
|
|
58
|
+
return localClockAt(now, timezone) >= at;
|
|
59
|
+
}
|
package/src/escalate.ts
CHANGED
|
@@ -213,6 +213,23 @@ export function createEscalator(
|
|
|
213
213
|
if (e.tier === 2 && chatId) {
|
|
214
214
|
const token = readTelegramToken();
|
|
215
215
|
if (token) {
|
|
216
|
+
// A category this policy defers does not page now: it is held here so
|
|
217
|
+
// the digest is the delivery authority for it (#229). A missing
|
|
218
|
+
// `reporting` block means the default (page everything); an explicit
|
|
219
|
+
// list decides each escalation by its category, defaulting `tier2`.
|
|
220
|
+
const category = e.category ?? "tier2";
|
|
221
|
+
const interruptOn = p.reporting?.interruptOn;
|
|
222
|
+
if (interruptOn !== undefined && !interruptOn.includes(category)) {
|
|
223
|
+
store.addHeldNotice({
|
|
224
|
+
project: p.name,
|
|
225
|
+
category,
|
|
226
|
+
summary: e.summary,
|
|
227
|
+
detail: text,
|
|
228
|
+
createdAt: Date.now(),
|
|
229
|
+
});
|
|
230
|
+
store.markNotified(key);
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
216
233
|
// A send failure throws: `markNotified` stays uncalled so the next
|
|
217
234
|
// poll retries instead of writing the event off as delivered. No
|
|
218
235
|
// backoff in here — the dispatcher tick *is* the retry, and an
|