omp-conductor 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/daemon.ts CHANGED
@@ -35,7 +35,7 @@ import { recordReleaseBlock } from "./release-policy.ts";
35
35
  import { branchName, effectiveLabels, route } from "./routing.ts";
36
36
  import type { Routed, UnroutableReason } from "./routing.ts";
37
37
  import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
38
- import { classifyRun, providerCreditRefusal, type ClassifyFacts } from "./failure-class.ts";
38
+ import { classifyRun, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
39
39
  import { projectLabels } from "./label-projection.ts";
40
40
  import { dbPath, openStore, utcDay } from "./store.ts";
41
41
  import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
@@ -61,7 +61,14 @@ import type {
61
61
  Tracker,
62
62
  VerbLedgerEntry,
63
63
  } from "./types.ts";
64
- import { type KilledBy, type WorkerResult, renderBrief, runWorker } from "./worker.ts";
64
+ import {
65
+ type KilledBy,
66
+ type WorkerPauseControl,
67
+ type WorkerPausePhase,
68
+ type WorkerResult,
69
+ renderBrief,
70
+ runWorker,
71
+ } from "./worker.ts";
65
72
  import {
66
73
  addRunRepo,
67
74
  cleanupRetainedWorktree,
@@ -86,7 +93,7 @@ import {
86
93
  } from "./verbs/socket.ts";
87
94
  import { homedir } from "node:os";
88
95
 
89
- import { pushRunBranch, type RunRepoRef } from "./gitops.ts";
96
+ import { pushRunBranch, readBaseChain, type RunRepoRef } from "./gitops.ts";
90
97
  import {
91
98
  planUsageLine,
92
99
  readPlanUsage,
@@ -108,6 +115,11 @@ const REPORT_DELIVERY_INTERVAL_MS = 30_000;
108
115
  * means the mirror itself is broken, not unlucky, and the sweep escalates
109
116
  * instead of burning a turn-0 run per tick forever (#168, #177). */
110
117
  const DISPATCH_INFRA_MAX_STRIKES = 3;
118
+ /** A provider-transient requeue (stream stalled mid-run) is retried, but only a
119
+ * bounded number of times: three aborted streams for one issue means the
120
+ * provider itself is degraded, not unlucky, and the sweep escalates to a
121
+ * human instead of requeueing into a down provider forever (#220). */
122
+ const PROVIDER_TRANSIENT_MAX_STRIKES = 3;
111
123
  const DEFAULT_PORT = 8787;
112
124
  const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
113
125
 
@@ -144,6 +156,7 @@ interface Deps {
144
156
  usage: UsageSource;
145
157
  escalate(e: Escalation): Promise<void>;
146
158
  turnLimits: TurnLimitRegistry;
159
+ workerControls: WorkerControlRegistry;
147
160
  integrity: IntegrityGate;
148
161
  stall: StallGate;
149
162
  cleanup?: RetainedCleanupCursor;
@@ -190,6 +203,7 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
190
203
  pausedAt,
191
204
  log,
192
205
  now: () => Date.now(),
206
+ chain: { readBaseChain },
193
207
  };
194
208
  }
195
209
 
@@ -264,6 +278,7 @@ async function watchOrchestrator(d: Deps): Promise<void> {
264
278
  log(`ERROR: the orchestrator session is not draining its queue — ${verdict.since ?? "no timestamp"}`);
265
279
  const delivered = await safeEscalate(d, {
266
280
  tier: 2,
281
+ category: "confirmed-failure",
267
282
  project: d.project.name,
268
283
  issue: NO_ISSUE,
269
284
  // Keyed on the marker's own timestamp, not the date. The dedup ledger keys
@@ -823,6 +838,91 @@ export function createTurnLimitRegistry(
823
838
  };
824
839
  }
825
840
 
841
+ export type WorkerControlResult =
842
+ | { kind: "ok"; runId: string; phase: WorkerPausePhase }
843
+ | { kind: "refused"; runId: string; error: string }
844
+ | { kind: "not-active" };
845
+
846
+ export interface WorkerControlSlot {
847
+ install(control: WorkerPauseControl): void;
848
+ close(): void;
849
+ }
850
+
851
+ export interface WorkerControlRegistry {
852
+ open(project: string, issue: number, runId: string): WorkerControlSlot;
853
+ pause(project: string, issue: number): Promise<WorkerControlResult>;
854
+ resume(project: string, issue: number): WorkerControlResult;
855
+ /** Live runs whose phase is not `running` — what /healthz and the board show. */
856
+ snapshot(project: string): { issue: number; runId: string; phase: WorkerPausePhase }[];
857
+ }
858
+
859
+ /** Authoritative controls for sessions owned by this daemon process. */
860
+ export function createWorkerControlRegistry(): WorkerControlRegistry {
861
+ const active = new Map<
862
+ string,
863
+ { project: string; issue: number; runId: string; control?: WorkerPauseControl }
864
+ >();
865
+ const key = (project: string, issue: number): string => `${project}\0${issue}`;
866
+ return {
867
+ open(project, issue, runId) {
868
+ const k = key(project, issue);
869
+ if (active.has(k)) throw new Error(`#${issue} already has a live worker controller`);
870
+ const entry = { project, issue, runId } as {
871
+ project: string;
872
+ issue: number;
873
+ runId: string;
874
+ control?: WorkerPauseControl;
875
+ };
876
+ active.set(k, entry);
877
+ return {
878
+ install: (control) => {
879
+ if (active.get(k) === entry) entry.control = control;
880
+ },
881
+ close: () => {
882
+ if (active.get(k) === entry) active.delete(k);
883
+ },
884
+ };
885
+ },
886
+ async pause(project, issue) {
887
+ const entry = active.get(key(project, issue));
888
+ if (entry?.control === undefined) return { kind: "not-active" };
889
+ try {
890
+ await entry.control.pause();
891
+ return { kind: "ok", runId: entry.runId, phase: entry.control.phase() };
892
+ } catch (err) {
893
+ return {
894
+ kind: "refused",
895
+ runId: entry.runId,
896
+ error: err instanceof Error ? err.message : String(err),
897
+ };
898
+ }
899
+ },
900
+ resume(project, issue) {
901
+ const entry = active.get(key(project, issue));
902
+ if (entry?.control === undefined) return { kind: "not-active" };
903
+ try {
904
+ entry.control.resume();
905
+ return { kind: "ok", runId: entry.runId, phase: entry.control.phase() };
906
+ } catch (err) {
907
+ return {
908
+ kind: "refused",
909
+ runId: entry.runId,
910
+ error: err instanceof Error ? err.message : String(err),
911
+ };
912
+ }
913
+ },
914
+ snapshot(project) {
915
+ const workers: { issue: number; runId: string; phase: WorkerPausePhase }[] = [];
916
+ for (const entry of active.values()) {
917
+ if (entry.project !== project || entry.control === undefined) continue;
918
+ const phase = entry.control.phase();
919
+ if (phase !== "running") workers.push({ issue: entry.issue, runId: entry.runId, phase });
920
+ }
921
+ return workers;
922
+ },
923
+ };
924
+ }
925
+
826
926
  export async function verifyPushedGreenClaim(
827
927
  tracker: Pick<Tracker, "verifyPr">,
828
928
  claim: Pick<WorkerResult, "prUrl" | "headSha">,
@@ -926,6 +1026,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
926
1026
  const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
927
1027
  let worktreePath: string | undefined;
928
1028
  let turnLimit: TurnLimitController | undefined;
1029
+ let workerControl: WorkerControlSlot | undefined;
929
1030
  // The run's own repository. Hoisted for the same reason `worktreePath` is —
930
1031
  // the catch and finally paths have to publish the branch.
931
1032
  let runRepo: RunRepoRef | undefined;
@@ -981,6 +1082,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
981
1082
  store.enqueueLabelOps(project.name, [{ issue, op: "add", label: inProgress }]);
982
1083
  claimed = true;
983
1084
  turnLimit = d.turnLimits.open(project.name, issue, runId, caps.workerMaxTurns);
1085
+ workerControl = d.workerControls.open(project.name, issue, runId);
984
1086
 
985
1087
  // A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
986
1088
  // an existing path, so a retry — or a tree kept from a failed attempt — has
@@ -1057,6 +1159,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1057
1159
  cwd: worktreePath,
1058
1160
  caps,
1059
1161
  maxTurns: () => turnLimit?.maxTurns() ?? caps.workerMaxTurns,
1162
+ onPauseControl: (control) => workerControl?.install(control),
1060
1163
  sessionDir,
1061
1164
  // The session's control socket, under the daemon's own state directory —
1062
1165
  // a child process of the daemon reaches it directly.
@@ -1081,6 +1184,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1081
1184
  onKilled: () => {
1082
1185
  turnLimit?.close();
1083
1186
  turnLimit = undefined;
1187
+ workerControl?.close();
1188
+ workerControl = undefined;
1084
1189
  },
1085
1190
  // Recorded the moment the session opens its transcript, not when the run
1086
1191
  // ends: `omp-conductor tail` resolves an issue to a file through this row,
@@ -1093,6 +1198,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1093
1198
  // PR verification or terminal row writes can leave stale `running` state.
1094
1199
  turnLimit?.close();
1095
1200
  turnLimit = undefined;
1201
+ workerControl?.close();
1202
+ workerControl = undefined;
1096
1203
  }
1097
1204
 
1098
1205
  // A configured model the harness could not honour means this run was done by
@@ -1110,13 +1217,10 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1110
1217
 
1111
1218
  // Read before the row is written so `lastError` carries the provider's own
1112
1219
  // message — it names the fix and the URL, which no classification can.
1113
- const providerCredit =
1114
- state === "failed" || state === "killed"
1115
- ? (() => {
1116
- const err = readSessionError(result.sessionFile);
1117
- return err === undefined ? undefined : providerCreditRefusal(err);
1118
- })()
1119
- : undefined;
1220
+ const sessionErr = state === "failed" || state === "killed" ? readSessionError(result.sessionFile) : undefined;
1221
+ const providerCredit = sessionErr === undefined ? undefined : providerCreditRefusal(sessionErr);
1222
+ const providerTransient =
1223
+ providerCredit !== undefined || sessionErr === undefined ? undefined : providerTransientFault(sessionErr);
1120
1224
 
1121
1225
  // The other half of not believing a worker about its own run (#128). The
1122
1226
  // claim being audited is `state: pushed-green`, so the audit runs on the
@@ -1202,8 +1306,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1202
1306
  // push: the report of a killed attempt is exactly the one a later
1203
1307
  // continuation must pool its disclosures from (#199).
1204
1308
  report: result.report,
1205
- ...(providerCredit !== undefined
1206
- ? { lastError: providerCredit }
1309
+ ...(providerCredit !== undefined || providerTransient !== undefined
1310
+ ? { lastError: providerCredit ?? providerTransient }
1207
1311
  : verified.reason === undefined
1208
1312
  ? {}
1209
1313
  : { lastError: verified.reason }),
@@ -1251,6 +1355,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1251
1355
  // instead of once per affected run.
1252
1356
  await safeEscalate(d, {
1253
1357
  tier: 2,
1358
+ category: "fleet-stopped",
1254
1359
  project: project.name,
1255
1360
  issue: NO_ISSUE,
1256
1361
  summary: `Model provider refused for credit — ${project.name} is paused`,
@@ -1364,6 +1469,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1364
1469
  // inner settlement guard exists. Latch it before any terminal write or await.
1365
1470
  turnLimit?.close();
1366
1471
  turnLimit = undefined;
1472
+ workerControl?.close();
1473
+ workerControl = undefined;
1367
1474
  const detail = errText(err);
1368
1475
  log(`#${issue} errored: ${detail}`);
1369
1476
  // A crash lands anywhere, including mid-edit in a tree holding the only
@@ -1409,6 +1516,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1409
1516
  // a failure path, and whatever it still held is now a commit on the branch.
1410
1517
  } finally {
1411
1518
  turnLimit?.close();
1519
+ workerControl?.close();
1520
+ workerControl = undefined;
1412
1521
  // The run is over, so its channel is too. Closed here rather than beside
1413
1522
  // the session so the crash path closes it as well: a listener left bound
1414
1523
  // after its run settled is a socket whose `run-not-live` check is the only
@@ -1730,6 +1839,7 @@ const DEGRADED_HOLDS: ReadonlySet<AdmissionHoldReason> = new Set([
1730
1839
  export function summarizeDispatch(
1731
1840
  ready: number,
1732
1841
  routed: number,
1842
+ claimed: number,
1733
1843
  admitted: number,
1734
1844
  holds: readonly AdmissionHold[],
1735
1845
  completedAt = Date.now(),
@@ -1745,6 +1855,7 @@ export function summarizeDispatch(
1745
1855
  completedAt,
1746
1856
  ready,
1747
1857
  routed,
1858
+ claimed,
1748
1859
  admitted,
1749
1860
  degraded: holds.some((hold) => DEGRADED_HOLDS.has(hold.reason)),
1750
1861
  holds: [...groups]
@@ -1792,6 +1903,7 @@ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation
1792
1903
  return {
1793
1904
  ...base,
1794
1905
  tier: 2,
1906
+ category: "fleet-stopped",
1795
1907
  // Dated: a meter that breaks again next month is a new incident, not a
1796
1908
  // repeat of this one.
1797
1909
  summary: `Plan usage source unreadable — ${project} is not claiming new work (${new Date().toISOString().slice(0, 10)})`,
@@ -1810,6 +1922,7 @@ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation
1810
1922
  return {
1811
1923
  ...base,
1812
1924
  tier: 2,
1925
+ category: "fleet-stopped",
1813
1926
  summary: `Plan usage cap names an unusable window "${plan.cap?.windowId ?? "?"}" — ${project} is not claiming new work`,
1814
1927
  detail: [
1815
1928
  plan.detail,
@@ -2284,6 +2397,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2284
2397
  if (integrity.page) {
2285
2398
  const delivered = await safeEscalate(d, {
2286
2399
  tier: 2,
2400
+ category: "fleet-stopped",
2287
2401
  project: project.name,
2288
2402
  issue: NO_ISSUE,
2289
2403
  // Dated for the same reason the spend cap is: the dedup key is the
@@ -2327,6 +2441,10 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2327
2441
  return pending.length === 0 ? issue : { ...issue, labels: effectiveLabels(issue.labels, pending) };
2328
2442
  });
2329
2443
  const { routed, unroutable } = route(effective, project);
2444
+ // Ready issues that route() dropped without a reason carry a state label —
2445
+ // claimed by a live or settling run — so the claimed count is subtraction
2446
+ // rather than a routed-state signal (#228).
2447
+ const claimed = effective.length - routed.length - unroutable.length;
2330
2448
  const routingHolds: AdmissionHold[] = unroutable.map((u) => ({
2331
2449
  issue: u.issue.number,
2332
2450
  reason: `unroutable:${u.reason}`,
@@ -2334,7 +2452,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2334
2452
  const recordDispatch = (admitted: number, holds: readonly AdmissionHold[]): void => {
2335
2453
  store.recordDispatch(
2336
2454
  project.name,
2337
- summarizeDispatch(ready.length, routed.length, admitted, holds),
2455
+ summarizeDispatch(ready.length, routed.length, claimed, admitted, holds),
2338
2456
  );
2339
2457
  };
2340
2458
 
@@ -2368,6 +2486,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2368
2486
  setPaused(true, { source: "spend-cap", reason: `daily spend reached $${caps.dailySpendUsd}` });
2369
2487
  await safeEscalate(d, {
2370
2488
  tier: 2,
2489
+ category: "fleet-stopped",
2371
2490
  project: project.name,
2372
2491
  issue: NO_ISSUE,
2373
2492
  // Dated so the same cap pages again tomorrow, but only once per day.
@@ -2433,6 +2552,8 @@ export interface DaemonHealthSnapshot {
2433
2552
  rssBytes: number;
2434
2553
  dispatch?: DispatchSummary;
2435
2554
  codeGraph?: CodeGraphHealth;
2555
+ /** Live workers in a non-running pause phase; absent/empty = nothing paused. */
2556
+ workers?: { issue: number; runId: string; phase: WorkerPausePhase }[];
2436
2557
  }
2437
2558
 
2438
2559
  export function daemonHealthSnapshot(
@@ -2441,6 +2562,7 @@ export function daemonHealthSnapshot(
2441
2562
  paused = isPaused(),
2442
2563
  rssBytes = process.memoryUsage().rss,
2443
2564
  codeGraph?: CodeGraphHealth,
2565
+ workerControls?: WorkerControlRegistry,
2444
2566
  ): DaemonHealthSnapshot {
2445
2567
  const dispatch = store.latestDispatch(project);
2446
2568
  return {
@@ -2451,6 +2573,7 @@ export function daemonHealthSnapshot(
2451
2573
  rssBytes,
2452
2574
  ...(dispatch === undefined ? {} : { dispatch }),
2453
2575
  ...(codeGraph?.configured === true ? { codeGraph } : {}),
2576
+ ...(workerControls === undefined ? {} : { workers: workerControls.snapshot(project) }),
2454
2577
  };
2455
2578
  }
2456
2579
 
@@ -2515,10 +2638,70 @@ export async function turnLimitResponse(
2515
2638
  );
2516
2639
  }
2517
2640
 
2641
+ export async function workerControlResponse(
2642
+ req: Request,
2643
+ project: string,
2644
+ store: Pick<Store, "latestRun">,
2645
+ registry: WorkerControlRegistry,
2646
+ ): Promise<Response | undefined> {
2647
+ const url = new URL(req.url);
2648
+ const match = /^\/runs\/(\d+)\/(pause|resume)$/.exec(url.pathname);
2649
+ if (req.method !== "PUT" || match === null) return undefined;
2650
+ if (!req.headers.get("content-type")?.startsWith("application/json")) {
2651
+ return Response.json({ error: "content-type must be application/json" }, { status: 415 });
2652
+ }
2653
+
2654
+ let body: unknown;
2655
+ try {
2656
+ body = await req.json();
2657
+ } catch {
2658
+ return Response.json({ error: "request body must be valid JSON" }, { status: 400 });
2659
+ }
2660
+ if (body === null || typeof body !== "object") {
2661
+ return Response.json({ error: "request body must be a JSON object" }, { status: 400 });
2662
+ }
2663
+ const requestedProject = Reflect.get(body, "project");
2664
+ if (typeof requestedProject !== "string" || requestedProject.length === 0) {
2665
+ return Response.json({ error: "project must be a non-empty string" }, { status: 400 });
2666
+ }
2667
+ if (requestedProject !== project) {
2668
+ return Response.json(
2669
+ { error: `daemon serves project "${project}", not requested project "${requestedProject}"` },
2670
+ { status: 409 },
2671
+ );
2672
+ }
2673
+
2674
+ const issue = Number(match[1]);
2675
+ const outcome =
2676
+ match[2] === "pause"
2677
+ ? await registry.pause(project, issue)
2678
+ : registry.resume(project, issue);
2679
+ if (outcome.kind === "ok") {
2680
+ return Response.json({ runId: outcome.runId, phase: outcome.phase });
2681
+ }
2682
+ if (outcome.kind === "refused") {
2683
+ return Response.json({ error: `#${issue}: ${outcome.error}` }, { status: 409 });
2684
+ }
2685
+
2686
+ const latest = store.latestRun(project, issue);
2687
+ if (latest === undefined) {
2688
+ return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
2689
+ }
2690
+ return Response.json(
2691
+ {
2692
+ error:
2693
+ `#${issue} has no live worker controller; its session already settled ` +
2694
+ `or belongs to another daemon (stored state: ${latest.state})`,
2695
+ },
2696
+ { status: 409 },
2697
+ );
2698
+ }
2699
+
2518
2700
  export interface DaemonHttpDeps {
2519
2701
  project: string;
2520
2702
  store: Pick<Store, "latestRun">;
2521
2703
  turnLimits: TurnLimitRegistry;
2704
+ workerControls: WorkerControlRegistry;
2522
2705
  health: () => DaemonHealthSnapshot;
2523
2706
  }
2524
2707
 
@@ -2526,17 +2709,18 @@ export interface DaemonHttpDeps {
2526
2709
  * The whole HTTP surface, in one named function so a test can pin what is *not*
2527
2710
  * on it.
2528
2711
  *
2529
- * Two routes: a health read, and the turn-limit control. Everything else is
2530
- * 404, and that is the contract see the prohibition at the `Bun.serve` call
2531
- * for why no mutation may be added here. Extracted from the serve callback
2532
- * precisely so "the daemon's HTTP port exposes no mutation route" is something
2533
- * a test asserts rather than something a reviewer has to notice (#126).
2712
+ * Three route families: the health read, turn-limit control, and worker
2713
+ * pause/resume control. Everything else is 404. The controls mutate only
2714
+ * daemon-owned live sessions; tracker and repository mutations stay on the
2715
+ * authenticated per-run channel described at the `Bun.serve` call (#126).
2534
2716
  */
2535
2717
  export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promise<Response> {
2536
2718
  const url = new URL(req.url);
2537
2719
  if (req.method === "GET" && url.pathname === "/healthz") return Response.json(d.health());
2538
- const control = await turnLimitResponse(req, d.project, d.store, d.turnLimits);
2539
- return control ?? new Response("not found\n", { status: 404 });
2720
+ const turnLimit = await turnLimitResponse(req, d.project, d.store, d.turnLimits);
2721
+ if (turnLimit !== undefined) return turnLimit;
2722
+ const workerControl = await workerControlResponse(req, d.project, d.store, d.workerControls);
2723
+ return workerControl ?? new Response("not found\n", { status: 404 });
2540
2724
  }
2541
2725
 
2542
2726
  export interface StatusSnapshot {
@@ -2672,7 +2856,7 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
2672
2856
  if (summary === undefined) return "last dispatch (none recorded)";
2673
2857
  const lines = [
2674
2858
  `last dispatch ${new Date(summary.completedAt).toISOString()}${summary.degraded ? " DEGRADED" : ""}`,
2675
- ` candidates ${summary.ready} ready / ${summary.routed} routed`,
2859
+ ` candidates ${summary.ready} ready / ${summary.claimed ?? 0} in flight / ${summary.routed} spare`,
2676
2860
  ` admitted ${summary.admitted}`,
2677
2861
  ];
2678
2862
  if (summary.holds.length === 0) {
@@ -3104,6 +3288,28 @@ async function recoverRun(
3104
3288
  log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
3105
3289
  return;
3106
3290
  }
3291
+ // Same bound for provider-transient: an issue whose stream keeps stalling
3292
+ // mid-run is requeued free (no attempt, no continuation charged) — but a
3293
+ // provider that aborts three times for one issue is down, and a human has
3294
+ // to check its status before hand-requeueing (#220).
3295
+ if (
3296
+ cls === "provider-transient" &&
3297
+ store.classCountFor(project.name, run.issue, "provider-transient") >= PROVIDER_TRANSIENT_MAX_STRIKES
3298
+ ) {
3299
+ await safeEscalate(d, {
3300
+ tier: 1,
3301
+ project: project.name,
3302
+ issue: run.issue,
3303
+ summary: `[provider-transient] #${run.issue}: the provider keeps aborting mid-stream — ${evidence}`,
3304
+ detail: [
3305
+ `The provider aborted the stream for #${run.issue} ${PROVIDER_TRANSIENT_MAX_STRIKES} times without the run ever producing a verdict (0 tokens billed each time).`,
3306
+ "Check provider status before requeueing by hand.",
3307
+ ].join("\n"),
3308
+ });
3309
+ store.updateRun(run.id, { recoveredAt: Date.now() });
3310
+ log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
3311
+ return;
3312
+ }
3107
3313
  // Only when the tracker still shows this issue as ours to hand back. An
3108
3314
  // issue that is closed, or has no state label, was resolved by another route
3109
3315
  // and requeueing it would dispatch work nobody asked for.
@@ -3521,6 +3727,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3521
3727
  const turnLimits = createTurnLimitRegistry((runId, maxTurns) => {
3522
3728
  store.updateRun(runId, { maxTurns });
3523
3729
  });
3730
+ const workerControls = createWorkerControlRegistry();
3524
3731
  const d: Deps = {
3525
3732
  project,
3526
3733
  caps,
@@ -3531,6 +3738,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3531
3738
  usage: sharedUsageSource(),
3532
3739
  escalate: (e) => escalator.escalate(e),
3533
3740
  turnLimits,
3741
+ workerControls,
3534
3742
  integrity,
3535
3743
  // Fresh per daemon run, like the integrity gate: a restart is entitled to
3536
3744
  // page again about a stall that is still on disk.
@@ -3607,14 +3815,15 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3607
3815
  process.on("SIGINT", stop);
3608
3816
  process.on("SIGTERM", stop);
3609
3817
 
3610
- // ── NO MUTATION ROUTE BELONGS ON THIS PORT ────────────────────────────────
3818
+ // ── NO TRACKER OR REPOSITORY MUTATION BELONGS ON THIS PORT ───────────────
3611
3819
  //
3612
3820
  // This is unauthenticated loopback TCP. Every local user can reach it, it
3613
3821
  // carries no credential of any kind, and it cannot tell one caller from
3614
- // another: `127.0.0.1` is not an identity. `turnLimitResponse` already shows
3615
- // what that costs — it trusts a body-supplied `project`, which is exactly the
3616
- // shape "identity from the payload" takes when nobody is watching, and it is
3617
- // tolerable only because raising a turn ceiling is bounded and reversible.
3822
+ // another: `127.0.0.1` is not an identity. The turn-limit and worker
3823
+ // pause/resume controls trust a body-supplied `project`, which is exactly the
3824
+ // shape "identity from the payload" takes when nobody is watching. Those
3825
+ // controls are tolerable only because they are bounded, live-run-local, and
3826
+ // reversible.
3618
3827
  //
3619
3828
  // A merge, a push, a release or a label is none of those things. Do not add
3620
3829
  // one here, and do not add "just a small one" behind a shared secret either:
@@ -3630,7 +3839,9 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3630
3839
  project: project.name,
3631
3840
  store,
3632
3841
  turnLimits,
3633
- health: () => daemonHealthSnapshot(store, project.name, undefined, undefined, codeGraph),
3842
+ workerControls,
3843
+ health: () =>
3844
+ daemonHealthSnapshot(store, project.name, undefined, undefined, codeGraph, workerControls),
3634
3845
  }),
3635
3846
  });
3636
3847
  log(`serving /healthz on :${server.port}, project ${project.name}`);
package/src/diff-flags.ts CHANGED
@@ -266,17 +266,49 @@ const CHANGED_LINE = /^[ \t>]*changed:[ \t]*(.*)$/im;
266
266
  */
267
267
  const CLAIMED_PATH = /^(?:[\w.@~+-]+\/)+[\w.@~*+-]*$|^[\w.@~+-]*\.[A-Za-z][A-Za-z0-9]{1,7}$/;
268
268
 
269
+ /** A Python dotted module: `services.rule_purge`, `omp.routing`. Not a path —
270
+ * segments joined by `.` — but a legitimate way for a report to name the file
271
+ * it touched without disclosing the directory it lives in (#224).
272
+ *
273
+ * Every segment is at least two characters so prose abbreviations (`e.g`,
274
+ * `i.e.`, `a.k.a`) stay commentary: they are exactly the prose the path
275
+ * heuristic exists to reject, and each one that slipped through would become
276
+ * an "you claimed a file you never touched" flag on an honest report. */
277
+ const DOTTED_MODULE = /^[A-Za-z_]\w+(?:\.[A-Za-z_]\w+)+$/;
278
+
279
+ /** One-level brace expansion: `a/{b,c}.py` → [`a/b.py`, `a/c.py`]. Braces are
280
+ * how a report compresses adjacent files into one token; each comma-separated
281
+ * alternative is substituted into the skeleton. Nesting is deliberately
282
+ * unsupported — one level matches the spellings reports actually produce. */
283
+ function expandBraces(token: string): string[] {
284
+ const group = /\{[^{}]*\}/.exec(token);
285
+ if (group === null) return [token];
286
+ const alternatives = group[0].slice(1, -1).split(",").filter((a) => a.length > 0);
287
+ if (alternatives.length === 0) return [token];
288
+ const prefix = token.slice(0, group.index);
289
+ const suffix = token.slice(group.index + group[0].length);
290
+ return alternatives.map((alt) => prefix + alt + suffix);
291
+ }
292
+
269
293
  /** Every path-shaped token on the report's `changed:` line. An absent line and
270
294
  * a line naming nothing are the same answer: nothing was disclosed. */
271
295
  export function claimedPaths(report: string): string[] {
272
296
  const line = CHANGED_LINE.exec(report)?.[1] ?? "";
273
297
  const seen = new Set<string>();
274
- for (const raw of line.split(/[\s,;]+/)) {
298
+ // Split on whitespace and semicolons, and on commas *outside* a brace group:
299
+ // the comma in `a/{b,c}.py` is an alternative separator, not a token
300
+ // separator, so a brace-compressed token must survive whole (#224).
301
+ for (const raw of line.split(/[\s;]+|,(?![^{}]*\})/)) {
275
302
  const token = raw.replace(/^[`'"([*-]+/, "").replace(/[`'")\].,:;]+$/, "");
276
303
  if (token === "" || token === "none") continue;
277
304
  const normalised = token.replace(/^\.?\//, "");
278
- if (!CLAIMED_PATH.test(normalised)) continue;
279
- seen.add(normalised);
305
+ // Expansion happens here so every downstream consumer — both `covers`
306
+ // directions — sees plain paths: a brace token dies earlier at the
307
+ // CLAIMED_PATH filter if it is never opened up (#224).
308
+ for (const expanded of expandBraces(normalised)) {
309
+ if (!CLAIMED_PATH.test(expanded) && !DOTTED_MODULE.test(expanded)) continue;
310
+ seen.add(expanded);
311
+ }
280
312
  }
281
313
  return [...seen];
282
314
  }
@@ -305,6 +337,15 @@ function covers(claim: string, path: string): boolean {
305
337
  return pattern.test(path) || pattern.test(basename(path));
306
338
  }
307
339
  if (path.endsWith(`/${claim}`) || claim.endsWith(`/${path}`)) return true;
340
+ // A Python dotted module names a file by import path, not by location:
341
+ // `services.rule_purge` covers `backend/app/services/rule_purge.py`. Only
342
+ // consulted after the path rules, because `foo.bar` is also a valid filename
343
+ // and a literal match must win. Matching several changed files is fine — the
344
+ // check wants evidence the worker knew, not a unique index (#224).
345
+ if (DOTTED_MODULE.test(claim)) {
346
+ const fragment = `${claim.replaceAll(".", "/")}.py`;
347
+ if (path === fragment || path.endsWith(`/${fragment}`)) return true;
348
+ }
308
349
  const last = claim.slice(claim.lastIndexOf("/") + 1);
309
350
  return !last.includes(".") && path.startsWith(`${claim}/`);
310
351
  }
@@ -0,0 +1,59 @@
1
+ /**
2
+ * When the daily digest is due, in the zone the operator reads it (#229).
3
+ *
4
+ * Pure by construction: every fact arrives as a timestamp and the policy, and
5
+ * nothing here reads a clock or a file. The digest is "one per day" in the
6
+ * IANA zone it is configured with (host zone when none) — a UTC key would roll
7
+ * the digest over mid-evening for anyone west of Greenwich.
8
+ */
9
+
10
+ import type { ReportingPolicy } from "./types.ts";
11
+
12
+ /** `YYYY-MM-DD` in an IANA timezone (host zone when `timezone` is absent). */
13
+ export function localDayKey(at: number, timezone?: string): string {
14
+ const parts = new Intl.DateTimeFormat("en-CA", {
15
+ timeZone: timezone,
16
+ year: "numeric",
17
+ month: "2-digit",
18
+ day: "2-digit",
19
+ }).formatToParts(new Date(at));
20
+ const get = (type: "year" | "month" | "day"): string =>
21
+ parts.find((p) => p.type === type)?.value ?? "00";
22
+ return `${get("year")}-${get("month")}-${get("day")}`;
23
+ }
24
+
25
+ /** `HH:MM` wall-clock in the zone, 24h and zero-padded. */
26
+ function localClockAt(at: number, timezone?: string): string {
27
+ return new Intl.DateTimeFormat("en-GB", {
28
+ timeZone: timezone,
29
+ hour: "2-digit",
30
+ minute: "2-digit",
31
+ hourCycle: "h23",
32
+ }).format(new Date(at));
33
+ }
34
+
35
+ /**
36
+ * Whether the digest is due now.
37
+ *
38
+ * - `none` → never.
39
+ * - `per-tick` → always (the orchestrator sends it with every report).
40
+ * - `daily` without `at` → once per day in the configured zone.
41
+ * - `daily` with `at` → on a day it has not already run, once the local clock
42
+ * has passed `at`. A restart after `at` still finds today unsent → one
43
+ * catch-up; a fully missed day is skipped, never sent late.
44
+ */
45
+ export function digestDue(
46
+ policy: Pick<ReportingPolicy, "digest">,
47
+ lastDigestDayKey: string | undefined,
48
+ now: number,
49
+ ): boolean {
50
+ const { cadence, at, timezone } = policy.digest;
51
+ if (cadence === "none") return false;
52
+ if (cadence === "per-tick") return true;
53
+ // daily
54
+ if (at === undefined) {
55
+ return lastDigestDayKey !== localDayKey(now, timezone);
56
+ }
57
+ if (lastDigestDayKey === localDayKey(now, timezone)) return false;
58
+ return localClockAt(now, timezone) >= at;
59
+ }
package/src/escalate.ts CHANGED
@@ -213,6 +213,23 @@ export function createEscalator(
213
213
  if (e.tier === 2 && chatId) {
214
214
  const token = readTelegramToken();
215
215
  if (token) {
216
+ // A category this policy defers does not page now: it is held here so
217
+ // the digest is the delivery authority for it (#229). A missing
218
+ // `reporting` block means the default (page everything); an explicit
219
+ // list decides each escalation by its category, defaulting `tier2`.
220
+ const category = e.category ?? "tier2";
221
+ const interruptOn = p.reporting?.interruptOn;
222
+ if (interruptOn !== undefined && !interruptOn.includes(category)) {
223
+ store.addHeldNotice({
224
+ project: p.name,
225
+ category,
226
+ summary: e.summary,
227
+ detail: text,
228
+ createdAt: Date.now(),
229
+ });
230
+ store.markNotified(key);
231
+ return;
232
+ }
216
233
  // A send failure throws: `markNotified` stays uncalled so the next
217
234
  // poll retries instead of writing the event off as delivered. No
218
235
  // backoff in here — the dispatcher tick *is* the retry, and an