omp-conductor 0.19.1 → 0.19.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -84,6 +84,7 @@ import {
84
84
  type FrictionSignal,
85
85
  type GroomingRecord,
86
86
  type HeldNotice,
87
+ type GroomTrigger,
87
88
  type InterruptCategory,
88
89
  type IntakeItem,
89
90
  type IssueState,
@@ -136,7 +137,7 @@ import {
136
137
  type ToSpecResult,
137
138
  } from "./to-spec.ts";
138
139
  import { heldNoticeId } from "./notices.ts";
139
- import { acknowledgeArmReply } from "./arm-challenge.ts";
140
+ import { classifyArmReply } from "./arm-challenge.ts";
140
141
 
141
142
  /** The activation file. Absent means "this is not an orchestrator session". */
142
143
  export const TICK_CONFIG_FILE = ".conductor-tick.json";
@@ -154,6 +155,29 @@ export const TICK_CUSTOM_TYPE = "omp-conductor.tick";
154
155
  */
155
156
  export const ARM_PROOF_CUSTOM_TYPE = "omp-conductor.arm-proof";
156
157
 
158
+ /**
159
+ * The type carried by an answer to a code that was **not** a proof (#991).
160
+ *
161
+ * Deliberately not {@link ARM_PROOF_CUSTOM_TYPE}: that type means "an arming
162
+ * proof landed", and the availability gate and every reader that filters on it
163
+ * would otherwise see a refusal as a proof. A refusal is the opposite fact.
164
+ */
165
+ export const ARM_REPLY_CUSTOM_TYPE = "omp-conductor.arm-reply";
166
+
167
+ /**
168
+ * What the operator is told when their code arrived too late (#991).
169
+ *
170
+ * Never the token, and never a hint about any other project: an expired record
171
+ * is the one thing this session can state about a code it could not accept.
172
+ */
173
+ export const ARM_EXPIRED_REPLY_TEXT =
174
+ "That arming code has expired — nothing was armed. Run `omp-conductor arm` again for a fresh code.";
175
+
176
+ /** What the operator is told when the code matches nothing here (#991). Worded
177
+ * so it cannot be read as a statement about another project's ceremony. */
178
+ export const ARM_UNKNOWN_REPLY_TEXT =
179
+ "No arming is in progress here, so that code was not accepted. Run `omp-conductor arm` to start one.";
180
+
157
181
  /** The model-ready wording for {@link ARM_PROOF_CUSTOM_TYPE}. Kept fixed so the
158
182
  * orchestrator's handling of an arming proof is deterministic, whichever
159
183
  * challenge it was cut for. */
@@ -218,6 +242,19 @@ const PENDING_MESSAGE_GRACE_MS = 60_000;
218
242
  /** Routable candidates below which the queue digest tells the orchestrator to groom (#181). */
219
243
  const DEFAULT_GROOM_BELOW = 4;
220
244
 
245
+ /**
246
+ * Whether the count-based half of the grooming duty fires for one dispatch
247
+ * row's routable count (#988): below a numeric threshold as always, while
248
+ * `"always"` leaves this gate open on purpose — the selectors then answer
249
+ * "is anything left to groom?" from candidate state (ungroomed, unrefused,
250
+ * unparked), and the launch block or no-batch line renders the truthful
251
+ * finding either way. Never substitute a large number for `"always"`: a big
252
+ * queue must not silence the duty.
253
+ */
254
+ function groomingDue(routed: number, groomBelow: GroomTrigger): boolean {
255
+ return groomBelow === "always" || routed < groomBelow;
256
+ }
257
+
221
258
  /**
222
259
  * Written by herdr-conductor `recover.sh` *before* `agent start`, and by the
223
260
  * dispatch daemon when a watched decision condition transitions false→true
@@ -668,7 +705,9 @@ export interface QueueObservation {
668
705
  /**
669
706
  * One line telling the orchestrator the routable queue is running dry (#181),
670
707
  * or `undefined` when healthy — no dispatch recorded yet, or the routable count
671
- * is at/above the grooming trigger. `grooming` is the durable per-issue verdict
708
+ * at/above a numeric grooming trigger (`"always"` renders on every dispatched
709
+ * tick, #988, stating what remains ungroomed instead of a threshold). `grooming`
710
+ * is the durable per-issue verdict
672
711
  * table (#735): `blocked` rows are admission's lane/dependency holds, so the
673
712
  * line tells claimable candidates apart from a runway that cannot move —
674
713
  * instead of inviting Duty 2 to groom work whose last pass held it, which is
@@ -688,7 +727,7 @@ export function queueDigestLine(
688
727
  summary: DispatchSummary | undefined,
689
728
  queueLabel: string,
690
729
  labelPrefix: string,
691
- groomBelow: number,
730
+ groomBelow: GroomTrigger,
692
731
  grooming: readonly GroomingRecord[] = [],
693
732
  queue: QueueObservation | undefined = undefined,
694
733
  /** Observation time for the durability of the grooming rows this line
@@ -710,7 +749,7 @@ function datedQueueDigestLine(
710
749
  summary: DispatchSummary,
711
750
  queueLabel: string,
712
751
  labelPrefix: string,
713
- groomBelow: number,
752
+ groomBelow: GroomTrigger,
714
753
  grooming: readonly GroomingRecord[],
715
754
  now: number,
716
755
  ): string | undefined {
@@ -750,7 +789,7 @@ function datedQueueDigestLine(
750
789
  }
751
790
  return line;
752
791
  }
753
- if (summary.routed >= groomBelow) return undefined;
792
+ if (!groomingDue(summary.routed, groomBelow)) return undefined;
754
793
  return lowQueueTail(summary, groomBelow, grooming, `As of the last dispatch (${dated}): Queue: `, now);
755
794
  }
756
795
 
@@ -766,7 +805,7 @@ function liveQueueDigestLine(
766
805
  queue: QueueObservation,
767
806
  queueLabel: string,
768
807
  labelPrefix: string,
769
- groomBelow: number,
808
+ groomBelow: GroomTrigger,
770
809
  grooming: readonly GroomingRecord[],
771
810
  now: number,
772
811
  ): string | undefined {
@@ -786,7 +825,7 @@ function liveQueueDigestLine(
786
825
  // carrier volume. Four in-flight/parked/unroutable carriers must not
787
826
  // suppress the low-claimable signal, because the to-spec offer gates on
788
827
  // the very same routed count and the two would split.
789
- if (summary.routed >= groomBelow) return undefined;
828
+ if (!groomingDue(summary.routed, groomBelow)) return undefined;
790
829
  const dated = new Date(summary.completedAt).toISOString();
791
830
  const inventory = `Queue: ${queued} open issue${queued === 1 ? "" : "s"} carry "${queueLabel}" right now (tracker ${observed}); `;
792
831
  if (summary.routed === 0) {
@@ -836,7 +875,7 @@ function liveQueueDigestLine(
836
875
  * means a tracker read can never hide these distinctions. */
837
876
  function lowQueueTail(
838
877
  summary: DispatchSummary,
839
- groomBelow: number,
878
+ groomBelow: GroomTrigger,
840
879
  grooming: readonly GroomingRecord[],
841
880
  lead: string,
842
881
  /** The clock the durability of each verdict is judged against — the same
@@ -866,9 +905,19 @@ function lowQueueTail(
866
905
  `${inFlight.length === 0 ? "" : " and the to-spec batch's results land when it settles"}.`;
867
906
  } else if (knownBlocked.length > 0 || inFlight.length > 0) {
868
907
  tail =
869
- `${lead}running low — ${summary.routed} routable candidate(s), below the grooming trigger of ${groomBelow}: ` +
870
- `${claimable} claimable, ${knownBlocked.length} known-blocked (${groomingGroupCounts(knownBlocked)})` +
871
- `${busy} groom only the claimable.`;
908
+ groomBelow === "always"
909
+ ? `${lead}grooming runs every tick ("always") — ${summary.routed} routable candidate(s): ` +
910
+ `${claimable} claimable, ${knownBlocked.length} known-blocked (${groomingGroupCounts(knownBlocked)})` +
911
+ `${busy} — groom the claimable while any remain ungroomed.`
912
+ : `${lead}running low — ${summary.routed} routable candidate(s), below the grooming trigger of ${groomBelow}: ` +
913
+ `${claimable} claimable, ${knownBlocked.length} known-blocked (${groomingGroupCounts(knownBlocked)})` +
914
+ `${busy} — groom only the claimable.`;
915
+ } else if (groomBelow === "always") {
916
+ // No numeric trigger exists to be "below", so the tail states the actual
917
+ // condition and what remains ungroomed (#988) instead of inventing one.
918
+ tail =
919
+ `${lead}grooming runs every tick ("always") — ${summary.routed} routable candidate(s); ` +
920
+ "groom while ungroomed, unrefused, unparked candidates remain.";
872
921
  } else {
873
922
  tail = `${lead}running low — ${summary.routed} routable candidate(s), below the grooming trigger of ${groomBelow}.`;
874
923
  }
@@ -1158,7 +1207,8 @@ export function toSpecCandidateExclusion(
1158
1207
  /**
1159
1208
  * The mechanical half of Duty 2's launch: deterministic, bounded selection of
1160
1209
  * the eligible candidates, smallest issue numbers first, never more than
1161
- * {@link TO_SPEC_BATCH_MAX} per batch, nothing at/above the grooming trigger,
1210
+ * {@link TO_SPEC_BATCH_MAX} per batch, nothing at/above a numeric grooming
1211
+ * trigger (`"always"` opens that gate and lets candidate state decide, #988),
1162
1212
  * nothing before the first dispatch summary exists (queue health unknown —
1163
1213
  * the same gate the queue digest uses). Parked and parent views are honored
1164
1214
  * when the caller supplies them.
@@ -1172,13 +1222,13 @@ export function toSpecCandidateExclusion(
1172
1222
  export function selectToSpecBatch(input: {
1173
1223
  candidates: readonly ToSpecCandidateView[];
1174
1224
  summary: DispatchSummary | undefined;
1175
- groomBelow: number;
1225
+ groomBelow: GroomTrigger;
1176
1226
  grooming: readonly GroomingRecord[];
1177
1227
  active: readonly { issue: number }[];
1178
1228
  now: number;
1179
1229
  }): ToSpecCandidateView[] {
1180
1230
  if (input.summary === undefined) return [];
1181
- if (input.summary.routed >= input.groomBelow) return [];
1231
+ if (!groomingDue(input.summary.routed, input.groomBelow)) return [];
1182
1232
  const groomingByIssue = new Map(input.grooming.map((row) => [row.issue, row]));
1183
1233
  const activeIssues = new Set(input.active.map((run) => run.issue));
1184
1234
  const selected: ToSpecCandidateView[] = [];
@@ -1295,8 +1345,9 @@ function toSpecPoolFromSnapshot(
1295
1345
  * exclusion, ending in the token + block + allowlist, or `undefined` when no
1296
1346
  * batch may launch. All of the following yield `undefined`:
1297
1347
  *
1298
- * - no dispatch row yet, or the routable queue is at/above the grooming
1299
- * trigger (the same gate the queue digest uses);
1348
+ * - no dispatch row yet, or the routable queue is at/above a numeric
1349
+ * grooming trigger (`"always"` opens that gate; candidate state decides,
1350
+ * #988 — the same gate the queue digest uses);
1300
1351
  * - no tracker seam (the snapshot is unavailable);
1301
1352
  * - the snapshot cannot be read — the launch fails closed rather than
1302
1353
  * trusting the model to self-filter parked/parent/epic candidates;
@@ -1311,7 +1362,7 @@ function toSpecPoolFromSnapshot(
1311
1362
  */
1312
1363
  export async function offerToSpecLaunch(input: {
1313
1364
  summary: DispatchSummary | undefined;
1314
- groomBelow: number;
1365
+ groomBelow: GroomTrigger;
1315
1366
  grooming: readonly GroomingRecord[];
1316
1367
  active: readonly { issue: number }[];
1317
1368
  project: ProjectConfig;
@@ -1324,7 +1375,7 @@ export async function offerToSpecLaunch(input: {
1324
1375
  now: number;
1325
1376
  }): Promise<ToSpecLaunchBlock | undefined> {
1326
1377
  if (input.summary === undefined) return undefined;
1327
- if (input.summary.routed >= input.groomBelow) return undefined;
1378
+ if (!groomingDue(input.summary.routed, input.groomBelow)) return undefined;
1328
1379
  const seam = input.trackerSeam;
1329
1380
  if (seam === undefined) return undefined;
1330
1381
  let issues = input.issues;
@@ -1391,7 +1442,7 @@ export async function offerToSpecLaunch(input: {
1391
1442
 
1392
1443
  /**
1393
1444
  * The per-tick launch block. Present exactly when a batch may be launched:
1394
- * the queue is below the grooming trigger (`summary.routed < groomBelow`),
1445
+ * the grooming duty is due (below a numeric trigger, or `"always"` #988),
1395
1446
  * a dispatch row exists, and the mechanical selection produced at least one
1396
1447
  * candidate. The block names the selected candidates as the ONLY batch this
1397
1448
  * tick authorizes and lists every store-proven exclusion the selection
@@ -1399,7 +1450,7 @@ export async function offerToSpecLaunch(input: {
1399
1450
  */
1400
1451
  export function toSpecLaunchBlock(input: {
1401
1452
  summary: DispatchSummary | undefined;
1402
- groomBelow: number;
1453
+ groomBelow: GroomTrigger;
1403
1454
  grooming: readonly GroomingRecord[];
1404
1455
  active: readonly { issue: number }[];
1405
1456
  tracker: string;
@@ -1411,7 +1462,7 @@ export function toSpecLaunchBlock(input: {
1411
1462
  now: number;
1412
1463
  }): ToSpecLaunchBlock | undefined {
1413
1464
  if (input.summary === undefined) return undefined;
1414
- if (input.summary.routed >= input.groomBelow) return undefined;
1465
+ if (!groomingDue(input.summary.routed, input.groomBelow)) return undefined;
1415
1466
  if (input.selected.length === 0) return undefined;
1416
1467
  const exclusions: ToSpecLaunchExclusions = {
1417
1468
  groomed: [],
@@ -1454,10 +1505,18 @@ export function toSpecLaunchBlock(input: {
1454
1505
  title: candidate.title,
1455
1506
  }));
1456
1507
  const candidates = items.map((item) => `- #${item.issue} (${item.routing}) — ${item.title}`).join("\n");
1508
+ // The opening sentence names the condition that authorized this batch:
1509
+ // below a numeric trigger as before, or the always-on mode in its own
1510
+ // words (#988) — never a threshold the configuration does not have.
1511
+ const trigger =
1512
+ input.groomBelow === "always"
1513
+ ? `Grooming is configured \`groomBelow: "always"\` — Duty 2 grooms every tick while ungroomed candidates remain ` +
1514
+ `(${input.summary.routed} routable right now). `
1515
+ : `The routable queue is below the grooming trigger of ${input.groomBelow} (${input.summary.routed} routable). `;
1457
1516
  const lines = [
1458
1517
  `## Bounded to-spec grooming batch (${TO_SPEC_BATCH_MARKER}, #777)`,
1459
1518
  "",
1460
- `The routable queue is below the grooming trigger of ${input.groomBelow} (${input.summary.routed} routable). ` +
1519
+ trigger +
1461
1520
  "Duty 2's finding is now a strict contract: launch EXACTLY ONE native `task` batch this turn with the " +
1462
1521
  `\`${TO_SPEC_AGENT}\` agent — never a second batch, never an improvised scout.`,
1463
1522
  "",
@@ -1503,19 +1562,20 @@ export function toSpecLaunchBlock(input: {
1503
1562
  * not run". Every count comes from the same predicates the selection and the
1504
1563
  * `tool_call` gate apply (#887).
1505
1564
  *
1506
- * `undefined` when no dispatch row exists or the queue is at/above the
1507
- * grooming trigger the same gate the offer itself uses, so this line and a
1508
- * launch block are mutually exclusive.
1565
+ * `undefined` when no dispatch row exists or the queue is at/above a numeric
1566
+ * grooming trigger (`"always"` opens that gate; candidate state decides, #988)
1567
+ * — the same gate the offer itself uses, so this line and a launch block are
1568
+ * mutually exclusive.
1509
1569
  */
1510
1570
  export function toSpecNoBatchLine(input: {
1511
1571
  summary: DispatchSummary | undefined;
1512
- groomBelow: number;
1572
+ groomBelow: GroomTrigger;
1513
1573
  grooming: readonly GroomingRecord[];
1514
1574
  active: readonly { issue: number }[];
1515
1575
  now: number;
1516
1576
  }): string | undefined {
1517
1577
  if (input.summary === undefined) return undefined;
1518
- if (input.summary.routed >= input.groomBelow) return undefined;
1578
+ if (!groomingDue(input.summary.routed, input.groomBelow)) return undefined;
1519
1579
  let durable = 0;
1520
1580
  let refused = 0;
1521
1581
  let inFlight = 0;
@@ -3555,14 +3615,16 @@ async function tick(
3555
3615
  // and the offer retries its own read exactly as it did before.
3556
3616
  //
3557
3617
  // The read is made only for dispatch rows the digest could render:
3558
- // a healthy row (routed at/above the grooming threshold) emits no
3559
- // queue line at all, so no freshness check is spent on it and no
3560
- // stale claim can leak from it. A zero-ready row (or any row below
3561
- // the trigger) is exactly where the old wording lied, so every such
3562
- // tick reads the live queue.
3618
+ // a healthy row (routed at/above a numeric grooming threshold)
3619
+ // emits no queue line at all, so no freshness check is spent on it
3620
+ // and no stale claim can leak from it. A zero-ready row (or any row
3621
+ // below the trigger) is exactly where the old wording lied, so every
3622
+ // such tick reads the live queue. Under `groomBelow: "always"`
3623
+ // (#988) the duty never clears on volume, so every dispatch row
3624
+ // reads the queue and the selectors judge candidate state.
3563
3625
  const needsQueueRead =
3564
3626
  dispatch !== undefined &&
3565
- (dispatch.ready === 0 || dispatch.routed === 0 || dispatch.routed < groomBelow);
3627
+ (dispatch.ready === 0 || dispatch.routed === 0 || groomingDue(dispatch.routed, groomBelow));
3566
3628
  let queueObservation: QueueObservation | undefined;
3567
3629
  let openSnapshot: readonly ReadyIssue[] | undefined;
3568
3630
  if (toSpecTrackerSeam !== undefined && needsQueueRead) {
@@ -4431,7 +4493,8 @@ export default function orchestratorTickExtension(
4431
4493
  ?.filter((part) => part.type === "text" && typeof part.text === "string")
4432
4494
  .map((part) => part.text as string)
4433
4495
  .join(" ") ?? "";
4434
- const proof = acknowledgeArmReply(configuredProject, replyText, Date.now());
4496
+ const verdict = classifyArmReply(configuredProject, replyText, Date.now());
4497
+ const proof = verdict === "matched";
4435
4498
  if (session.activeLocalTick !== undefined) {
4436
4499
  session.activeLocalTick.humanWaiting = true;
4437
4500
  if (proof) session.activeLocalTick.armingProof = true;
@@ -4444,6 +4507,24 @@ export default function orchestratorTickExtension(
4444
4507
  { customType: ARM_PROOF_CUSTOM_TYPE, content: ARM_PROOF_ACK_TEXT, display: true, attribution: "agent" },
4445
4508
  { triggerTurn: true, deliverAs: "steer" },
4446
4509
  );
4510
+ } else if (verdict !== "none") {
4511
+ // A code that matched nothing used to be silent: the waiter kept
4512
+ // waiting and the operator had no idea whether they had been heard,
4513
+ // so the fastest way to be sure of the current code was to scroll
4514
+ // (#991). The answer never echoes the token — that would put the
4515
+ // plaintext in the chat and in any log that captures outbound
4516
+ // messages, defeating the hash-only storage on purpose — and it never
4517
+ // says whether some other project has a live challenge, because
4518
+ // `classifyArmReply` reads only this session's own records.
4519
+ pi.sendMessage(
4520
+ {
4521
+ customType: ARM_REPLY_CUSTOM_TYPE,
4522
+ content: verdict === "expired" ? ARM_EXPIRED_REPLY_TEXT : ARM_UNKNOWN_REPLY_TEXT,
4523
+ display: true,
4524
+ attribution: "agent",
4525
+ },
4526
+ { triggerTurn: true, deliverAs: "steer" },
4527
+ );
4447
4528
  }
4448
4529
  }
4449
4530
  });
package/src/settlement.ts CHANGED
@@ -15,6 +15,8 @@
15
15
  */
16
16
 
17
17
  import { existsSync, readFileSync } from "node:fs";
18
+ import { join } from "node:path";
19
+ import { homedir } from "node:os";
18
20
  import { log, errText, safeEscalate } from "./log.ts";
19
21
  import { hasContinuationBudget } from "./admission.ts";
20
22
  import {
@@ -72,6 +74,10 @@ export interface SettlementDeps {
72
74
  tracker: Tracker;
73
75
  store: Store;
74
76
  escalate(e: Escalation): Promise<void>;
77
+ /** Where the harness writes its per-process session logs, read for the
78
+ * terminal evidence a settling row cannot supply itself (#986). Defaults to
79
+ * the harness's own location — a test points it at a fixture. */
80
+ harnessLogDir?: string;
75
81
  /** Whether the pause sentinel is set, daemon-owned. */
76
82
  isPaused(project?: string): boolean;
77
83
  /** Writes the pause sentinel, daemon-owned. */
@@ -1266,6 +1272,118 @@ function assistantStopError(rec: { readonly [key: string]: unknown }): string |
1266
1272
  return undefined;
1267
1273
  }
1268
1274
 
1275
+ /**
1276
+ * Where the harness writes its per-process session logs (#986).
1277
+ *
1278
+ * The harness's own location, not the conductor's: `OMP_CONDUCTOR_HOME`
1279
+ * relocates conductor state and must not move this, because the file being
1280
+ * read belongs to omp and is named by omp.
1281
+ */
1282
+ export function harnessLogDir(): string {
1283
+ return process.env["OMP_HARNESS_LOG_DIR"] ?? join(homedir(), ".omp", "logs");
1284
+ }
1285
+
1286
+ /** The harness's own maintenance-routing markers, in the session log it writes
1287
+ * per process. These name why a session ended when the row cannot: the run
1288
+ * itself records no error, because from its side nothing failed (#986). */
1289
+ const HARNESS_ROUTE_MARKERS = [
1290
+ "empty-stop-retry-cap",
1291
+ "empty-stop-handled",
1292
+ "bottom-checkCompaction",
1293
+ "context-overflow",
1294
+ ] as const;
1295
+
1296
+ /**
1297
+ * One bounded line of terminal evidence for a run that did not settle cleanly
1298
+ * (#986), or `undefined` when nothing readable says anything.
1299
+ *
1300
+ * Two surfaces, because the interesting deaths leave nothing on the row:
1301
+ *
1302
+ * - **the transcript's own exit record** — `session_exit` carries the reason
1303
+ * and kind the harness disposed the session with;
1304
+ * - **the harness session log for this run's session-host pid** — where the
1305
+ * provider-flake routes live (`empty-stop-retry-cap`). The path is
1306
+ * deterministic (`<logDir>/omp.<YYYY-MM-DD>.<pid>.log`, the harness's own
1307
+ * naming), so this reads exactly one file and never scans a directory.
1308
+ *
1309
+ * Bounded on purpose: this string is rendered in `status` beside a surviving
1310
+ * `unknown`, and an operator reading a board does not want a stack trace.
1311
+ */
1312
+ export function readTerminalEvidence(
1313
+ run: Pick<RunRecord, "sessionFile" | "workerPid" | "startedAt">,
1314
+ logDir: string,
1315
+ ): string | undefined {
1316
+ const parts: string[] = [];
1317
+ const exit = readSessionExit(run.sessionFile);
1318
+ if (exit !== undefined) parts.push(exit);
1319
+ const route = readHarnessRoute(run, logDir);
1320
+ if (route !== undefined) parts.push(route);
1321
+ if (parts.length === 0) return undefined;
1322
+ return parts.join("; ").slice(0, 300);
1323
+ }
1324
+
1325
+ /** The transcript's own last word on how the session ended. */
1326
+ function readSessionExit(sessionFile: string | undefined): string | undefined {
1327
+ if (sessionFile === undefined) return undefined;
1328
+ let text: string;
1329
+ try {
1330
+ text = readFileSync(sessionFile, "utf8");
1331
+ } catch {
1332
+ return undefined;
1333
+ }
1334
+ const lines = text.split("\n");
1335
+ for (let i = lines.length - 1; i >= 0; i -= 1) {
1336
+ const line = lines[i];
1337
+ if (line === undefined || line.length === 0) continue;
1338
+ let row: unknown;
1339
+ try {
1340
+ row = JSON.parse(line) as unknown;
1341
+ } catch {
1342
+ continue;
1343
+ }
1344
+ if (row === null || typeof row !== "object") continue;
1345
+ const rec = row as { readonly [key: string]: unknown };
1346
+ if (rec["customType"] !== "session_exit") continue;
1347
+ const data = rec["data"];
1348
+ if (data === null || typeof data !== "object") continue;
1349
+ const d = data as { readonly [key: string]: unknown };
1350
+ const reason = typeof d["reason"] === "string" ? d["reason"] : undefined;
1351
+ const kind = typeof d["kind"] === "string" ? d["kind"] : undefined;
1352
+ if (reason === undefined && kind === undefined) continue;
1353
+ return `session_exit ${reason ?? "?"}${kind === undefined ? "" : `/${kind}`}`;
1354
+ }
1355
+ return undefined;
1356
+ }
1357
+
1358
+ /**
1359
+ * The last maintenance route the harness logged for this run's session-host
1360
+ * process, or `undefined`. Reads one deterministic path and nothing else: a
1361
+ * settle sweep must never walk a log directory whose size is unbounded.
1362
+ */
1363
+ function readHarnessRoute(
1364
+ run: Pick<RunRecord, "workerPid" | "startedAt">,
1365
+ logDir: string,
1366
+ ): string | undefined {
1367
+ if (run.workerPid === undefined) return undefined;
1368
+ const day = new Date(run.startedAt).toISOString().slice(0, 10);
1369
+ const path = join(logDir, `omp.${day}.${run.workerPid}.log`);
1370
+ let text: string;
1371
+ try {
1372
+ text = readFileSync(path, "utf8");
1373
+ } catch {
1374
+ return undefined;
1375
+ }
1376
+ // Last match wins: the terminal route is the one that ended the session, and
1377
+ // `empty-stop-handled` appearing earlier is a retry that worked.
1378
+ let found: string | undefined;
1379
+ for (const marker of HARNESS_ROUTE_MARKERS) {
1380
+ const at = text.lastIndexOf(marker);
1381
+ if (at === -1) continue;
1382
+ if (found === undefined || at > text.lastIndexOf(found)) found = marker;
1383
+ }
1384
+ return found === undefined ? undefined : `harness route ${found}`;
1385
+ }
1386
+
1269
1387
  export function readSessionError(sessionFile: string | undefined): TranscriptError | undefined {
1270
1388
  if (sessionFile === undefined) return undefined;
1271
1389
  let text: string;
@@ -1328,6 +1446,7 @@ export function readSessionError(sessionFile: string | undefined): TranscriptErr
1328
1446
  */
1329
1447
  export async function classifyAndRecover(d: SettlementDeps): Promise<number> {
1330
1448
  const { project, caps, tracker, store } = d;
1449
+ const logDir = d.harnessLogDir ?? harnessLogDir();
1331
1450
  // Settle recoveries, counted for the pass's dispatch record — a row whose PR
1332
1451
  // merged is settled here when the settle sweep could not establish identity
1333
1452
  // (#497). Every other exit returns 0.
@@ -1346,6 +1465,19 @@ export async function classifyAndRecover(d: SettlementDeps): Promise<number> {
1346
1465
  classifiedRun = { ...run, lastError: sessionError.message };
1347
1466
  }
1348
1467
  }
1468
+ // The evidence the row cannot supply itself (#986): the transcript's exit
1469
+ // record and the harness's own routing for this run's process. Recorded
1470
+ // before classification so the table can read it, and recorded even when
1471
+ // it names nothing actionable — "what was known at settle time" is the
1472
+ // question `status` answers beside a surviving `unknown`, and a blank is
1473
+ // only honest when the sweep genuinely found nothing.
1474
+ if (run.terminalEvidence === undefined) {
1475
+ const evidence = readTerminalEvidence(run, logDir);
1476
+ if (evidence !== undefined) {
1477
+ store.updateRun(run.id, { terminalEvidence: evidence });
1478
+ classifiedRun = { ...classifiedRun, terminalEvidence: evidence };
1479
+ }
1480
+ }
1349
1481
  }
1350
1482
  try {
1351
1483
  if (run.prUrl !== undefined) {
@@ -1515,6 +1647,41 @@ async function recoverRun(
1515
1647
  return;
1516
1648
  }
1517
1649
 
1650
+ // `model-empty-stop`: the provider answered with empty turns until the
1651
+ // harness ended the session. The work is not at fault and the row is not a
1652
+ // failed attempt (the counters exclude the class), so this hands the queue
1653
+ // label back and the next dispatch resumes from whatever the attempt
1654
+ // pushed. Bounded by the continuation budget for the same reason the
1655
+ // wall-clock path is: a provider stuck in that state must reach a human
1656
+ // instead of consuming the issue forever, one free retry at a time.
1657
+ if (cls === "model-empty-stop") {
1658
+ const state = await tracker.issueState(run.issue).catch(() => undefined);
1659
+ if (state !== "open") {
1660
+ log(`#${run.issue} not continued from ${cls}: issue is ${state ?? "unreadable"}`);
1661
+ return;
1662
+ }
1663
+ const continuation = store.continuationsFor(project.name, run.issue);
1664
+ if (hasContinuationBudget(continuation, caps.maxContinuationsPerIssue)) {
1665
+ swapToQueue(d, run.issue, project.stateLabels.failed);
1666
+ store.updateRun(run.id, { recoveredAt: Date.now() });
1667
+ log(`#${run.issue} requeued after a provider empty-stop: ${evidence}`);
1668
+ return;
1669
+ }
1670
+ store.enqueueLabelOps(project.name, [
1671
+ { issue: run.issue, op: "remove", label: project.stateLabels.failed },
1672
+ ]);
1673
+ store.updateRun(run.id, { recoveredAt: Date.now() });
1674
+ await safeEscalate(d, {
1675
+ tier: 1,
1676
+ project: project.name,
1677
+ issue: run.issue,
1678
+ summary: `#${run.issue} keeps dying to provider empty-stops and has no continuation budget left`,
1679
+ detail: `${evidence}\n\nThe provider ended ${continuation} session(s) on this issue by returning empty turns until the harness's retry cap. That is not the work failing, so no implementation attempt was charged — but the continuation budget is spent, and another free retry would just consume the issue. Switching this issue's model is the usual remedy.`,
1680
+ });
1681
+ log(`#${run.issue} escalated: provider empty-stops exhausted the continuation budget`);
1682
+ return;
1683
+ }
1684
+
1518
1685
  // `merge-conflict`: the branch is retained and its PR is open, so #50's
1519
1686
  // continuation guard admits it and the next tick briefs a rebase.
1520
1687
  //
package/src/setup.ts CHANGED
@@ -72,6 +72,7 @@ import {
72
72
  type DigestCadence,
73
73
  type DraftPolicy,
74
74
  type OrchestratorMode,
75
+ type GroomTrigger,
75
76
  type InterruptCategory,
76
77
  type ProjectConfig,
77
78
  type ProjectPolicy,
@@ -161,11 +162,12 @@ export interface SetupAnswers {
161
162
  */
162
163
  workerModel?: string;
163
164
  /**
164
- * Routable-candidate count below which the tick prompt says to groom. The
165
- * wizard never asks for it it is hand-edited so it exists here only to
166
- * survive an amend of some other area (#369).
165
+ * The grooming trigger: a routable-candidate count below which the tick
166
+ * prompt says to groom, or `"always"` to groom on demand every tick (#988).
167
+ * The wizard never asks for it it is hand-edited — so it exists here only
168
+ * to survive an amend of some other area (#369).
167
169
  */
168
- groomBelow?: number;
170
+ groomBelow?: GroomTrigger;
169
171
  /**
170
172
  * Ordered fallback worker models and the provider-failure threshold that
171
173
  * starts them. The wizard never asks for either — they are hand-edited —
@@ -66,7 +66,34 @@ export type SpendTelemetryVerdict =
66
66
  /** A majority of working runs reported nothing. */
67
67
  | { kind: "partial"; worked: number; missing: number }
68
68
  /** Every working run reported nothing — the same fact, stated louder. */
69
- | { kind: "absent"; worked: number };
69
+ | { kind: "absent"; worked: number }
70
+ /**
71
+ * Every working run reported nothing **and this fleet does not bill in
72
+ * dollars** (#984). Not a fault: a subscription request has no per-request
73
+ * price, so the harness reports none and `spendUsd` is permanently 0.00 —
74
+ * accurate rather than missing. The constraint that actually binds is the
75
+ * provider's allowance window, named here.
76
+ */
77
+ | { kind: "subscription"; worked: number; window: string; evidence: "declared" | "observed" };
78
+
79
+ /**
80
+ * What is known about how this project's runs are billed (#984).
81
+ *
82
+ * Two independent grounds, because on a real fleet the declaration is often
83
+ * absent while the evidence is unambiguous:
84
+ *
85
+ * - `declaredSubscription` — the project requires its providers to bill to a
86
+ * subscription credential (`requireOauthProviders`, #852). A statement of
87
+ * intent, so it is believed on its own.
88
+ * - `allowanceWindow` — the provider itself reports an allowance window
89
+ * (`omp usage --json`), which is what a subscription has instead of a price.
90
+ * Positive evidence only: a fleet whose provider reports no window has
91
+ * nothing here, and its zeros stay a fault.
92
+ */
93
+ export interface SpendBilling {
94
+ declaredSubscription?: boolean;
95
+ allowanceWindow?: string;
96
+ }
70
97
 
71
98
  /**
72
99
  * Judge a newest-first sample.
@@ -78,13 +105,29 @@ export type SpendTelemetryVerdict =
78
105
  export function judgeSpendTelemetry(
79
106
  samples: readonly SpendSample[],
80
107
  limit: number,
108
+ billing: SpendBilling = {},
81
109
  ): SpendTelemetryVerdict {
82
110
  const working = samples.filter((row) => row.turns > 0).slice(0, limit);
83
111
  if (working.length < limit) {
84
112
  return { kind: "insufficient", worked: working.length, needed: limit };
85
113
  }
86
114
  const missing = working.filter((row) => row.spendUsd === 0).length;
87
- if (missing === working.length) return { kind: "absent", worked: working.length };
115
+ if (missing === working.length) {
116
+ // Total absence is the only verdict a billing class can reclassify, and
117
+ // that asymmetry is the whole safety property: a metered neighbour proves
118
+ // dollars ARE being reported, so a zero beside one is real telemetry loss
119
+ // no matter how the fleet is billed. `partial` therefore always warns.
120
+ const window = subscriptionWindow(billing);
121
+ if (window !== undefined) {
122
+ return {
123
+ kind: "subscription",
124
+ worked: working.length,
125
+ window: window.window,
126
+ evidence: window.evidence,
127
+ };
128
+ }
129
+ return { kind: "absent", worked: working.length };
130
+ }
88
131
  if (missing / working.length >= SPEND_MISSING_SHARE) {
89
132
  return { kind: "partial", worked: working.length, missing };
90
133
  }
@@ -98,8 +141,37 @@ export function judgeSpendTelemetry(
98
141
 
99
142
  /** The operator-facing sentence, or undefined when there is nothing to say.
100
143
  * Shared so `doctor` and `status` cannot word the same fact differently. */
144
+ /**
145
+ * The allowance window to name, or `undefined` when nothing establishes that
146
+ * this fleet is subscription-billed.
147
+ *
148
+ * A declaration with no readable window still counts — the operator has said
149
+ * how the fleet bills, and a provider that cannot be read is not evidence
150
+ * against it — but it says so rather than inventing a window name.
151
+ */
152
+ function subscriptionWindow(
153
+ billing: SpendBilling,
154
+ ): { window: string; evidence: "declared" | "observed" } | undefined {
155
+ if (billing.allowanceWindow !== undefined && billing.allowanceWindow.trim() !== "") {
156
+ return {
157
+ window: billing.allowanceWindow.trim(),
158
+ evidence: billing.declaredSubscription === true ? "declared" : "observed",
159
+ };
160
+ }
161
+ if (billing.declaredSubscription === true) {
162
+ return { window: "unreadable (omp usage --json reported no window)", evidence: "declared" };
163
+ }
164
+ return undefined;
165
+ }
166
+
101
167
  export function spendTelemetryDetail(verdict: SpendTelemetryVerdict): string | undefined {
102
168
  switch (verdict.kind) {
169
+ case "subscription":
170
+ return (
171
+ `the last ${verdict.worked} working runs recorded $0.00, which is correct: this fleet bills to a ` +
172
+ `subscription (${verdict.evidence}), so requests carry no per-request price and no USD cap can fire` +
173
+ ` — the constraint that binds is the ${verdict.window} allowance`
174
+ );
103
175
  case "absent":
104
176
  return (
105
177
  `the last ${verdict.worked} completed runs that did any work all recorded $0.00 spend` +