omp-conductor 0.19.1 → 0.19.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +1 -1
- package/package.json +2 -2
- package/schema/config.schema.json +13 -1
- package/src/arm-challenge.ts +92 -16
- package/src/board.ts +50 -9
- package/src/check-browser-js.ts +72 -0
- package/src/cli.ts +2 -0
- package/src/command-manifest.ts +10 -0
- package/src/commands/arm.ts +30 -2
- package/src/commands/companion.ts +103 -0
- package/src/commands/context.ts +1 -0
- package/src/commands/worker.ts +1 -0
- package/src/companion-view.ts +81 -0
- package/src/config-schema.ts +7 -1
- package/src/config.ts +6 -3
- package/src/daemon.ts +197 -14
- package/src/dashboard/app.js +4 -4
- package/src/dashboard/controls.ts +1 -1
- package/src/doctor.ts +126 -7
- package/src/escalate.ts +47 -1
- package/src/failure-class.fixture.json +546 -0
- package/src/failure-class.ts +192 -0
- package/src/fleet.ts +275 -27
- package/src/orchestrator-tick.ts +115 -34
- package/src/settlement.ts +167 -0
- package/src/setup.ts +6 -4
- package/src/spend-telemetry.ts +74 -2
- package/src/status-render.ts +25 -6
- package/src/store.ts +38 -2
- package/src/types.ts +39 -2
- package/src/usage.ts +25 -0
package/src/orchestrator-tick.ts
CHANGED
|
@@ -84,6 +84,7 @@ import {
|
|
|
84
84
|
type FrictionSignal,
|
|
85
85
|
type GroomingRecord,
|
|
86
86
|
type HeldNotice,
|
|
87
|
+
type GroomTrigger,
|
|
87
88
|
type InterruptCategory,
|
|
88
89
|
type IntakeItem,
|
|
89
90
|
type IssueState,
|
|
@@ -136,7 +137,7 @@ import {
|
|
|
136
137
|
type ToSpecResult,
|
|
137
138
|
} from "./to-spec.ts";
|
|
138
139
|
import { heldNoticeId } from "./notices.ts";
|
|
139
|
-
import {
|
|
140
|
+
import { classifyArmReply } from "./arm-challenge.ts";
|
|
140
141
|
|
|
141
142
|
/** The activation file. Absent means "this is not an orchestrator session". */
|
|
142
143
|
export const TICK_CONFIG_FILE = ".conductor-tick.json";
|
|
@@ -154,6 +155,29 @@ export const TICK_CUSTOM_TYPE = "omp-conductor.tick";
|
|
|
154
155
|
*/
|
|
155
156
|
export const ARM_PROOF_CUSTOM_TYPE = "omp-conductor.arm-proof";
|
|
156
157
|
|
|
158
|
+
/**
|
|
159
|
+
* The type carried by an answer to a code that was **not** a proof (#991).
|
|
160
|
+
*
|
|
161
|
+
* Deliberately not {@link ARM_PROOF_CUSTOM_TYPE}: that type means "an arming
|
|
162
|
+
* proof landed", and the availability gate and every reader that filters on it
|
|
163
|
+
* would otherwise see a refusal as a proof. A refusal is the opposite fact.
|
|
164
|
+
*/
|
|
165
|
+
export const ARM_REPLY_CUSTOM_TYPE = "omp-conductor.arm-reply";
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* What the operator is told when their code arrived too late (#991).
|
|
169
|
+
*
|
|
170
|
+
* Never the token, and never a hint about any other project: an expired record
|
|
171
|
+
* is the one thing this session can state about a code it could not accept.
|
|
172
|
+
*/
|
|
173
|
+
export const ARM_EXPIRED_REPLY_TEXT =
|
|
174
|
+
"That arming code has expired — nothing was armed. Run `omp-conductor arm` again for a fresh code.";
|
|
175
|
+
|
|
176
|
+
/** What the operator is told when the code matches nothing here (#991). Worded
|
|
177
|
+
* so it cannot be read as a statement about another project's ceremony. */
|
|
178
|
+
export const ARM_UNKNOWN_REPLY_TEXT =
|
|
179
|
+
"No arming is in progress here, so that code was not accepted. Run `omp-conductor arm` to start one.";
|
|
180
|
+
|
|
157
181
|
/** The model-ready wording for {@link ARM_PROOF_CUSTOM_TYPE}. Kept fixed so the
|
|
158
182
|
* orchestrator's handling of an arming proof is deterministic, whichever
|
|
159
183
|
* challenge it was cut for. */
|
|
@@ -218,6 +242,19 @@ const PENDING_MESSAGE_GRACE_MS = 60_000;
|
|
|
218
242
|
/** Routable candidates below which the queue digest tells the orchestrator to groom (#181). */
|
|
219
243
|
const DEFAULT_GROOM_BELOW = 4;
|
|
220
244
|
|
|
245
|
+
/**
|
|
246
|
+
* Whether the count-based half of the grooming duty fires for one dispatch
|
|
247
|
+
* row's routable count (#988): below a numeric threshold as always, while
|
|
248
|
+
* `"always"` leaves this gate open on purpose — the selectors then answer
|
|
249
|
+
* "is anything left to groom?" from candidate state (ungroomed, unrefused,
|
|
250
|
+
* unparked), and the launch block or no-batch line renders the truthful
|
|
251
|
+
* finding either way. Never substitute a large number for `"always"`: a big
|
|
252
|
+
* queue must not silence the duty.
|
|
253
|
+
*/
|
|
254
|
+
function groomingDue(routed: number, groomBelow: GroomTrigger): boolean {
|
|
255
|
+
return groomBelow === "always" || routed < groomBelow;
|
|
256
|
+
}
|
|
257
|
+
|
|
221
258
|
/**
|
|
222
259
|
* Written by herdr-conductor `recover.sh` *before* `agent start`, and by the
|
|
223
260
|
* dispatch daemon when a watched decision condition transitions false→true
|
|
@@ -668,7 +705,9 @@ export interface QueueObservation {
|
|
|
668
705
|
/**
|
|
669
706
|
* One line telling the orchestrator the routable queue is running dry (#181),
|
|
670
707
|
* or `undefined` when healthy — no dispatch recorded yet, or the routable count
|
|
671
|
-
*
|
|
708
|
+
* at/above a numeric grooming trigger (`"always"` renders on every dispatched
|
|
709
|
+
* tick, #988, stating what remains ungroomed instead of a threshold). `grooming`
|
|
710
|
+
* is the durable per-issue verdict
|
|
672
711
|
* table (#735): `blocked` rows are admission's lane/dependency holds, so the
|
|
673
712
|
* line tells claimable candidates apart from a runway that cannot move —
|
|
674
713
|
* instead of inviting Duty 2 to groom work whose last pass held it, which is
|
|
@@ -688,7 +727,7 @@ export function queueDigestLine(
|
|
|
688
727
|
summary: DispatchSummary | undefined,
|
|
689
728
|
queueLabel: string,
|
|
690
729
|
labelPrefix: string,
|
|
691
|
-
groomBelow:
|
|
730
|
+
groomBelow: GroomTrigger,
|
|
692
731
|
grooming: readonly GroomingRecord[] = [],
|
|
693
732
|
queue: QueueObservation | undefined = undefined,
|
|
694
733
|
/** Observation time for the durability of the grooming rows this line
|
|
@@ -710,7 +749,7 @@ function datedQueueDigestLine(
|
|
|
710
749
|
summary: DispatchSummary,
|
|
711
750
|
queueLabel: string,
|
|
712
751
|
labelPrefix: string,
|
|
713
|
-
groomBelow:
|
|
752
|
+
groomBelow: GroomTrigger,
|
|
714
753
|
grooming: readonly GroomingRecord[],
|
|
715
754
|
now: number,
|
|
716
755
|
): string | undefined {
|
|
@@ -750,7 +789,7 @@ function datedQueueDigestLine(
|
|
|
750
789
|
}
|
|
751
790
|
return line;
|
|
752
791
|
}
|
|
753
|
-
if (summary.routed
|
|
792
|
+
if (!groomingDue(summary.routed, groomBelow)) return undefined;
|
|
754
793
|
return lowQueueTail(summary, groomBelow, grooming, `As of the last dispatch (${dated}): Queue: `, now);
|
|
755
794
|
}
|
|
756
795
|
|
|
@@ -766,7 +805,7 @@ function liveQueueDigestLine(
|
|
|
766
805
|
queue: QueueObservation,
|
|
767
806
|
queueLabel: string,
|
|
768
807
|
labelPrefix: string,
|
|
769
|
-
groomBelow:
|
|
808
|
+
groomBelow: GroomTrigger,
|
|
770
809
|
grooming: readonly GroomingRecord[],
|
|
771
810
|
now: number,
|
|
772
811
|
): string | undefined {
|
|
@@ -786,7 +825,7 @@ function liveQueueDigestLine(
|
|
|
786
825
|
// carrier volume. Four in-flight/parked/unroutable carriers must not
|
|
787
826
|
// suppress the low-claimable signal, because the to-spec offer gates on
|
|
788
827
|
// the very same routed count and the two would split.
|
|
789
|
-
if (summary.routed
|
|
828
|
+
if (!groomingDue(summary.routed, groomBelow)) return undefined;
|
|
790
829
|
const dated = new Date(summary.completedAt).toISOString();
|
|
791
830
|
const inventory = `Queue: ${queued} open issue${queued === 1 ? "" : "s"} carry "${queueLabel}" right now (tracker ${observed}); `;
|
|
792
831
|
if (summary.routed === 0) {
|
|
@@ -836,7 +875,7 @@ function liveQueueDigestLine(
|
|
|
836
875
|
* means a tracker read can never hide these distinctions. */
|
|
837
876
|
function lowQueueTail(
|
|
838
877
|
summary: DispatchSummary,
|
|
839
|
-
groomBelow:
|
|
878
|
+
groomBelow: GroomTrigger,
|
|
840
879
|
grooming: readonly GroomingRecord[],
|
|
841
880
|
lead: string,
|
|
842
881
|
/** The clock the durability of each verdict is judged against — the same
|
|
@@ -866,9 +905,19 @@ function lowQueueTail(
|
|
|
866
905
|
`${inFlight.length === 0 ? "" : " and the to-spec batch's results land when it settles"}.`;
|
|
867
906
|
} else if (knownBlocked.length > 0 || inFlight.length > 0) {
|
|
868
907
|
tail =
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
908
|
+
groomBelow === "always"
|
|
909
|
+
? `${lead}grooming runs every tick ("always") — ${summary.routed} routable candidate(s): ` +
|
|
910
|
+
`${claimable} claimable, ${knownBlocked.length} known-blocked (${groomingGroupCounts(knownBlocked)})` +
|
|
911
|
+
`${busy} — groom the claimable while any remain ungroomed.`
|
|
912
|
+
: `${lead}running low — ${summary.routed} routable candidate(s), below the grooming trigger of ${groomBelow}: ` +
|
|
913
|
+
`${claimable} claimable, ${knownBlocked.length} known-blocked (${groomingGroupCounts(knownBlocked)})` +
|
|
914
|
+
`${busy} — groom only the claimable.`;
|
|
915
|
+
} else if (groomBelow === "always") {
|
|
916
|
+
// No numeric trigger exists to be "below", so the tail states the actual
|
|
917
|
+
// condition and what remains ungroomed (#988) instead of inventing one.
|
|
918
|
+
tail =
|
|
919
|
+
`${lead}grooming runs every tick ("always") — ${summary.routed} routable candidate(s); ` +
|
|
920
|
+
"groom while ungroomed, unrefused, unparked candidates remain.";
|
|
872
921
|
} else {
|
|
873
922
|
tail = `${lead}running low — ${summary.routed} routable candidate(s), below the grooming trigger of ${groomBelow}.`;
|
|
874
923
|
}
|
|
@@ -1158,7 +1207,8 @@ export function toSpecCandidateExclusion(
|
|
|
1158
1207
|
/**
|
|
1159
1208
|
* The mechanical half of Duty 2's launch: deterministic, bounded selection of
|
|
1160
1209
|
* the eligible candidates, smallest issue numbers first, never more than
|
|
1161
|
-
* {@link TO_SPEC_BATCH_MAX} per batch, nothing at/above
|
|
1210
|
+
* {@link TO_SPEC_BATCH_MAX} per batch, nothing at/above a numeric grooming
|
|
1211
|
+
* trigger (`"always"` opens that gate and lets candidate state decide, #988),
|
|
1162
1212
|
* nothing before the first dispatch summary exists (queue health unknown —
|
|
1163
1213
|
* the same gate the queue digest uses). Parked and parent views are honored
|
|
1164
1214
|
* when the caller supplies them.
|
|
@@ -1172,13 +1222,13 @@ export function toSpecCandidateExclusion(
|
|
|
1172
1222
|
export function selectToSpecBatch(input: {
|
|
1173
1223
|
candidates: readonly ToSpecCandidateView[];
|
|
1174
1224
|
summary: DispatchSummary | undefined;
|
|
1175
|
-
groomBelow:
|
|
1225
|
+
groomBelow: GroomTrigger;
|
|
1176
1226
|
grooming: readonly GroomingRecord[];
|
|
1177
1227
|
active: readonly { issue: number }[];
|
|
1178
1228
|
now: number;
|
|
1179
1229
|
}): ToSpecCandidateView[] {
|
|
1180
1230
|
if (input.summary === undefined) return [];
|
|
1181
|
-
if (input.summary.routed
|
|
1231
|
+
if (!groomingDue(input.summary.routed, input.groomBelow)) return [];
|
|
1182
1232
|
const groomingByIssue = new Map(input.grooming.map((row) => [row.issue, row]));
|
|
1183
1233
|
const activeIssues = new Set(input.active.map((run) => run.issue));
|
|
1184
1234
|
const selected: ToSpecCandidateView[] = [];
|
|
@@ -1295,8 +1345,9 @@ function toSpecPoolFromSnapshot(
|
|
|
1295
1345
|
* exclusion, ending in the token + block + allowlist, or `undefined` when no
|
|
1296
1346
|
* batch may launch. All of the following yield `undefined`:
|
|
1297
1347
|
*
|
|
1298
|
-
* - no dispatch row yet, or the routable queue is at/above
|
|
1299
|
-
* trigger (
|
|
1348
|
+
* - no dispatch row yet, or the routable queue is at/above a numeric
|
|
1349
|
+
* grooming trigger (`"always"` opens that gate; candidate state decides,
|
|
1350
|
+
* #988 — the same gate the queue digest uses);
|
|
1300
1351
|
* - no tracker seam (the snapshot is unavailable);
|
|
1301
1352
|
* - the snapshot cannot be read — the launch fails closed rather than
|
|
1302
1353
|
* trusting the model to self-filter parked/parent/epic candidates;
|
|
@@ -1311,7 +1362,7 @@ function toSpecPoolFromSnapshot(
|
|
|
1311
1362
|
*/
|
|
1312
1363
|
export async function offerToSpecLaunch(input: {
|
|
1313
1364
|
summary: DispatchSummary | undefined;
|
|
1314
|
-
groomBelow:
|
|
1365
|
+
groomBelow: GroomTrigger;
|
|
1315
1366
|
grooming: readonly GroomingRecord[];
|
|
1316
1367
|
active: readonly { issue: number }[];
|
|
1317
1368
|
project: ProjectConfig;
|
|
@@ -1324,7 +1375,7 @@ export async function offerToSpecLaunch(input: {
|
|
|
1324
1375
|
now: number;
|
|
1325
1376
|
}): Promise<ToSpecLaunchBlock | undefined> {
|
|
1326
1377
|
if (input.summary === undefined) return undefined;
|
|
1327
|
-
if (input.summary.routed
|
|
1378
|
+
if (!groomingDue(input.summary.routed, input.groomBelow)) return undefined;
|
|
1328
1379
|
const seam = input.trackerSeam;
|
|
1329
1380
|
if (seam === undefined) return undefined;
|
|
1330
1381
|
let issues = input.issues;
|
|
@@ -1391,7 +1442,7 @@ export async function offerToSpecLaunch(input: {
|
|
|
1391
1442
|
|
|
1392
1443
|
/**
|
|
1393
1444
|
* The per-tick launch block. Present exactly when a batch may be launched:
|
|
1394
|
-
* the
|
|
1445
|
+
* the grooming duty is due (below a numeric trigger, or `"always"` — #988),
|
|
1395
1446
|
* a dispatch row exists, and the mechanical selection produced at least one
|
|
1396
1447
|
* candidate. The block names the selected candidates as the ONLY batch this
|
|
1397
1448
|
* tick authorizes and lists every store-proven exclusion the selection
|
|
@@ -1399,7 +1450,7 @@ export async function offerToSpecLaunch(input: {
|
|
|
1399
1450
|
*/
|
|
1400
1451
|
export function toSpecLaunchBlock(input: {
|
|
1401
1452
|
summary: DispatchSummary | undefined;
|
|
1402
|
-
groomBelow:
|
|
1453
|
+
groomBelow: GroomTrigger;
|
|
1403
1454
|
grooming: readonly GroomingRecord[];
|
|
1404
1455
|
active: readonly { issue: number }[];
|
|
1405
1456
|
tracker: string;
|
|
@@ -1411,7 +1462,7 @@ export function toSpecLaunchBlock(input: {
|
|
|
1411
1462
|
now: number;
|
|
1412
1463
|
}): ToSpecLaunchBlock | undefined {
|
|
1413
1464
|
if (input.summary === undefined) return undefined;
|
|
1414
|
-
if (input.summary.routed
|
|
1465
|
+
if (!groomingDue(input.summary.routed, input.groomBelow)) return undefined;
|
|
1415
1466
|
if (input.selected.length === 0) return undefined;
|
|
1416
1467
|
const exclusions: ToSpecLaunchExclusions = {
|
|
1417
1468
|
groomed: [],
|
|
@@ -1454,10 +1505,18 @@ export function toSpecLaunchBlock(input: {
|
|
|
1454
1505
|
title: candidate.title,
|
|
1455
1506
|
}));
|
|
1456
1507
|
const candidates = items.map((item) => `- #${item.issue} (${item.routing}) — ${item.title}`).join("\n");
|
|
1508
|
+
// The opening sentence names the condition that authorized this batch:
|
|
1509
|
+
// below a numeric trigger as before, or the always-on mode in its own
|
|
1510
|
+
// words (#988) — never a threshold the configuration does not have.
|
|
1511
|
+
const trigger =
|
|
1512
|
+
input.groomBelow === "always"
|
|
1513
|
+
? `Grooming is configured \`groomBelow: "always"\` — Duty 2 grooms every tick while ungroomed candidates remain ` +
|
|
1514
|
+
`(${input.summary.routed} routable right now). `
|
|
1515
|
+
: `The routable queue is below the grooming trigger of ${input.groomBelow} (${input.summary.routed} routable). `;
|
|
1457
1516
|
const lines = [
|
|
1458
1517
|
`## Bounded to-spec grooming batch (${TO_SPEC_BATCH_MARKER}, #777)`,
|
|
1459
1518
|
"",
|
|
1460
|
-
|
|
1519
|
+
trigger +
|
|
1461
1520
|
"Duty 2's finding is now a strict contract: launch EXACTLY ONE native `task` batch this turn with the " +
|
|
1462
1521
|
`\`${TO_SPEC_AGENT}\` agent — never a second batch, never an improvised scout.`,
|
|
1463
1522
|
"",
|
|
@@ -1503,19 +1562,20 @@ export function toSpecLaunchBlock(input: {
|
|
|
1503
1562
|
* not run". Every count comes from the same predicates the selection and the
|
|
1504
1563
|
* `tool_call` gate apply (#887).
|
|
1505
1564
|
*
|
|
1506
|
-
* `undefined` when no dispatch row exists or the queue is at/above
|
|
1507
|
-
* grooming trigger
|
|
1508
|
-
* launch block are
|
|
1565
|
+
* `undefined` when no dispatch row exists or the queue is at/above a numeric
|
|
1566
|
+
* grooming trigger (`"always"` opens that gate; candidate state decides, #988)
|
|
1567
|
+
* — the same gate the offer itself uses, so this line and a launch block are
|
|
1568
|
+
* mutually exclusive.
|
|
1509
1569
|
*/
|
|
1510
1570
|
export function toSpecNoBatchLine(input: {
|
|
1511
1571
|
summary: DispatchSummary | undefined;
|
|
1512
|
-
groomBelow:
|
|
1572
|
+
groomBelow: GroomTrigger;
|
|
1513
1573
|
grooming: readonly GroomingRecord[];
|
|
1514
1574
|
active: readonly { issue: number }[];
|
|
1515
1575
|
now: number;
|
|
1516
1576
|
}): string | undefined {
|
|
1517
1577
|
if (input.summary === undefined) return undefined;
|
|
1518
|
-
if (input.summary.routed
|
|
1578
|
+
if (!groomingDue(input.summary.routed, input.groomBelow)) return undefined;
|
|
1519
1579
|
let durable = 0;
|
|
1520
1580
|
let refused = 0;
|
|
1521
1581
|
let inFlight = 0;
|
|
@@ -3555,14 +3615,16 @@ async function tick(
|
|
|
3555
3615
|
// and the offer retries its own read exactly as it did before.
|
|
3556
3616
|
//
|
|
3557
3617
|
// The read is made only for dispatch rows the digest could render:
|
|
3558
|
-
// a healthy row (routed at/above
|
|
3559
|
-
// queue line at all, so no freshness check is spent on it
|
|
3560
|
-
// stale claim can leak from it. A zero-ready row (or any row
|
|
3561
|
-
// the trigger) is exactly where the old wording lied, so every
|
|
3562
|
-
// tick reads the live queue.
|
|
3618
|
+
// a healthy row (routed at/above a numeric grooming threshold)
|
|
3619
|
+
// emits no queue line at all, so no freshness check is spent on it
|
|
3620
|
+
// and no stale claim can leak from it. A zero-ready row (or any row
|
|
3621
|
+
// below the trigger) is exactly where the old wording lied, so every
|
|
3622
|
+
// such tick reads the live queue. Under `groomBelow: "always"`
|
|
3623
|
+
// (#988) the duty never clears on volume, so every dispatch row
|
|
3624
|
+
// reads the queue and the selectors judge candidate state.
|
|
3563
3625
|
const needsQueueRead =
|
|
3564
3626
|
dispatch !== undefined &&
|
|
3565
|
-
(dispatch.ready === 0 || dispatch.routed === 0 || dispatch.routed
|
|
3627
|
+
(dispatch.ready === 0 || dispatch.routed === 0 || groomingDue(dispatch.routed, groomBelow));
|
|
3566
3628
|
let queueObservation: QueueObservation | undefined;
|
|
3567
3629
|
let openSnapshot: readonly ReadyIssue[] | undefined;
|
|
3568
3630
|
if (toSpecTrackerSeam !== undefined && needsQueueRead) {
|
|
@@ -4431,7 +4493,8 @@ export default function orchestratorTickExtension(
|
|
|
4431
4493
|
?.filter((part) => part.type === "text" && typeof part.text === "string")
|
|
4432
4494
|
.map((part) => part.text as string)
|
|
4433
4495
|
.join(" ") ?? "";
|
|
4434
|
-
const
|
|
4496
|
+
const verdict = classifyArmReply(configuredProject, replyText, Date.now());
|
|
4497
|
+
const proof = verdict === "matched";
|
|
4435
4498
|
if (session.activeLocalTick !== undefined) {
|
|
4436
4499
|
session.activeLocalTick.humanWaiting = true;
|
|
4437
4500
|
if (proof) session.activeLocalTick.armingProof = true;
|
|
@@ -4444,6 +4507,24 @@ export default function orchestratorTickExtension(
|
|
|
4444
4507
|
{ customType: ARM_PROOF_CUSTOM_TYPE, content: ARM_PROOF_ACK_TEXT, display: true, attribution: "agent" },
|
|
4445
4508
|
{ triggerTurn: true, deliverAs: "steer" },
|
|
4446
4509
|
);
|
|
4510
|
+
} else if (verdict !== "none") {
|
|
4511
|
+
// A code that matched nothing used to be silent: the waiter kept
|
|
4512
|
+
// waiting and the operator had no idea whether they had been heard,
|
|
4513
|
+
// so the fastest way to be sure of the current code was to scroll
|
|
4514
|
+
// (#991). The answer never echoes the token — that would put the
|
|
4515
|
+
// plaintext in the chat and in any log that captures outbound
|
|
4516
|
+
// messages, defeating the hash-only storage on purpose — and it never
|
|
4517
|
+
// says whether some other project has a live challenge, because
|
|
4518
|
+
// `classifyArmReply` reads only this session's own records.
|
|
4519
|
+
pi.sendMessage(
|
|
4520
|
+
{
|
|
4521
|
+
customType: ARM_REPLY_CUSTOM_TYPE,
|
|
4522
|
+
content: verdict === "expired" ? ARM_EXPIRED_REPLY_TEXT : ARM_UNKNOWN_REPLY_TEXT,
|
|
4523
|
+
display: true,
|
|
4524
|
+
attribution: "agent",
|
|
4525
|
+
},
|
|
4526
|
+
{ triggerTurn: true, deliverAs: "steer" },
|
|
4527
|
+
);
|
|
4447
4528
|
}
|
|
4448
4529
|
}
|
|
4449
4530
|
});
|
package/src/settlement.ts
CHANGED
|
@@ -15,6 +15,8 @@
|
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
17
|
import { existsSync, readFileSync } from "node:fs";
|
|
18
|
+
import { join } from "node:path";
|
|
19
|
+
import { homedir } from "node:os";
|
|
18
20
|
import { log, errText, safeEscalate } from "./log.ts";
|
|
19
21
|
import { hasContinuationBudget } from "./admission.ts";
|
|
20
22
|
import {
|
|
@@ -72,6 +74,10 @@ export interface SettlementDeps {
|
|
|
72
74
|
tracker: Tracker;
|
|
73
75
|
store: Store;
|
|
74
76
|
escalate(e: Escalation): Promise<void>;
|
|
77
|
+
/** Where the harness writes its per-process session logs, read for the
|
|
78
|
+
* terminal evidence a settling row cannot supply itself (#986). Defaults to
|
|
79
|
+
* the harness's own location — a test points it at a fixture. */
|
|
80
|
+
harnessLogDir?: string;
|
|
75
81
|
/** Whether the pause sentinel is set, daemon-owned. */
|
|
76
82
|
isPaused(project?: string): boolean;
|
|
77
83
|
/** Writes the pause sentinel, daemon-owned. */
|
|
@@ -1266,6 +1272,118 @@ function assistantStopError(rec: { readonly [key: string]: unknown }): string |
|
|
|
1266
1272
|
return undefined;
|
|
1267
1273
|
}
|
|
1268
1274
|
|
|
1275
|
+
/**
|
|
1276
|
+
* Where the harness writes its per-process session logs (#986).
|
|
1277
|
+
*
|
|
1278
|
+
* The harness's own location, not the conductor's: `OMP_CONDUCTOR_HOME`
|
|
1279
|
+
* relocates conductor state and must not move this, because the file being
|
|
1280
|
+
* read belongs to omp and is named by omp.
|
|
1281
|
+
*/
|
|
1282
|
+
export function harnessLogDir(): string {
|
|
1283
|
+
return process.env["OMP_HARNESS_LOG_DIR"] ?? join(homedir(), ".omp", "logs");
|
|
1284
|
+
}
|
|
1285
|
+
|
|
1286
|
+
/** The harness's own maintenance-routing markers, in the session log it writes
|
|
1287
|
+
* per process. These name why a session ended when the row cannot: the run
|
|
1288
|
+
* itself records no error, because from its side nothing failed (#986). */
|
|
1289
|
+
const HARNESS_ROUTE_MARKERS = [
|
|
1290
|
+
"empty-stop-retry-cap",
|
|
1291
|
+
"empty-stop-handled",
|
|
1292
|
+
"bottom-checkCompaction",
|
|
1293
|
+
"context-overflow",
|
|
1294
|
+
] as const;
|
|
1295
|
+
|
|
1296
|
+
/**
|
|
1297
|
+
* One bounded line of terminal evidence for a run that did not settle cleanly
|
|
1298
|
+
* (#986), or `undefined` when nothing readable says anything.
|
|
1299
|
+
*
|
|
1300
|
+
* Two surfaces, because the interesting deaths leave nothing on the row:
|
|
1301
|
+
*
|
|
1302
|
+
* - **the transcript's own exit record** — `session_exit` carries the reason
|
|
1303
|
+
* and kind the harness disposed the session with;
|
|
1304
|
+
* - **the harness session log for this run's session-host pid** — where the
|
|
1305
|
+
* provider-flake routes live (`empty-stop-retry-cap`). The path is
|
|
1306
|
+
* deterministic (`<logDir>/omp.<YYYY-MM-DD>.<pid>.log`, the harness's own
|
|
1307
|
+
* naming), so this reads exactly one file and never scans a directory.
|
|
1308
|
+
*
|
|
1309
|
+
* Bounded on purpose: this string is rendered in `status` beside a surviving
|
|
1310
|
+
* `unknown`, and an operator reading a board does not want a stack trace.
|
|
1311
|
+
*/
|
|
1312
|
+
export function readTerminalEvidence(
|
|
1313
|
+
run: Pick<RunRecord, "sessionFile" | "workerPid" | "startedAt">,
|
|
1314
|
+
logDir: string,
|
|
1315
|
+
): string | undefined {
|
|
1316
|
+
const parts: string[] = [];
|
|
1317
|
+
const exit = readSessionExit(run.sessionFile);
|
|
1318
|
+
if (exit !== undefined) parts.push(exit);
|
|
1319
|
+
const route = readHarnessRoute(run, logDir);
|
|
1320
|
+
if (route !== undefined) parts.push(route);
|
|
1321
|
+
if (parts.length === 0) return undefined;
|
|
1322
|
+
return parts.join("; ").slice(0, 300);
|
|
1323
|
+
}
|
|
1324
|
+
|
|
1325
|
+
/** The transcript's own last word on how the session ended. */
|
|
1326
|
+
function readSessionExit(sessionFile: string | undefined): string | undefined {
|
|
1327
|
+
if (sessionFile === undefined) return undefined;
|
|
1328
|
+
let text: string;
|
|
1329
|
+
try {
|
|
1330
|
+
text = readFileSync(sessionFile, "utf8");
|
|
1331
|
+
} catch {
|
|
1332
|
+
return undefined;
|
|
1333
|
+
}
|
|
1334
|
+
const lines = text.split("\n");
|
|
1335
|
+
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
|
1336
|
+
const line = lines[i];
|
|
1337
|
+
if (line === undefined || line.length === 0) continue;
|
|
1338
|
+
let row: unknown;
|
|
1339
|
+
try {
|
|
1340
|
+
row = JSON.parse(line) as unknown;
|
|
1341
|
+
} catch {
|
|
1342
|
+
continue;
|
|
1343
|
+
}
|
|
1344
|
+
if (row === null || typeof row !== "object") continue;
|
|
1345
|
+
const rec = row as { readonly [key: string]: unknown };
|
|
1346
|
+
if (rec["customType"] !== "session_exit") continue;
|
|
1347
|
+
const data = rec["data"];
|
|
1348
|
+
if (data === null || typeof data !== "object") continue;
|
|
1349
|
+
const d = data as { readonly [key: string]: unknown };
|
|
1350
|
+
const reason = typeof d["reason"] === "string" ? d["reason"] : undefined;
|
|
1351
|
+
const kind = typeof d["kind"] === "string" ? d["kind"] : undefined;
|
|
1352
|
+
if (reason === undefined && kind === undefined) continue;
|
|
1353
|
+
return `session_exit ${reason ?? "?"}${kind === undefined ? "" : `/${kind}`}`;
|
|
1354
|
+
}
|
|
1355
|
+
return undefined;
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
/**
|
|
1359
|
+
* The last maintenance route the harness logged for this run's session-host
|
|
1360
|
+
* process, or `undefined`. Reads one deterministic path and nothing else: a
|
|
1361
|
+
* settle sweep must never walk a log directory whose size is unbounded.
|
|
1362
|
+
*/
|
|
1363
|
+
function readHarnessRoute(
|
|
1364
|
+
run: Pick<RunRecord, "workerPid" | "startedAt">,
|
|
1365
|
+
logDir: string,
|
|
1366
|
+
): string | undefined {
|
|
1367
|
+
if (run.workerPid === undefined) return undefined;
|
|
1368
|
+
const day = new Date(run.startedAt).toISOString().slice(0, 10);
|
|
1369
|
+
const path = join(logDir, `omp.${day}.${run.workerPid}.log`);
|
|
1370
|
+
let text: string;
|
|
1371
|
+
try {
|
|
1372
|
+
text = readFileSync(path, "utf8");
|
|
1373
|
+
} catch {
|
|
1374
|
+
return undefined;
|
|
1375
|
+
}
|
|
1376
|
+
// Last match wins: the terminal route is the one that ended the session, and
|
|
1377
|
+
// `empty-stop-handled` appearing earlier is a retry that worked.
|
|
1378
|
+
let found: string | undefined;
|
|
1379
|
+
for (const marker of HARNESS_ROUTE_MARKERS) {
|
|
1380
|
+
const at = text.lastIndexOf(marker);
|
|
1381
|
+
if (at === -1) continue;
|
|
1382
|
+
if (found === undefined || at > text.lastIndexOf(found)) found = marker;
|
|
1383
|
+
}
|
|
1384
|
+
return found === undefined ? undefined : `harness route ${found}`;
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1269
1387
|
export function readSessionError(sessionFile: string | undefined): TranscriptError | undefined {
|
|
1270
1388
|
if (sessionFile === undefined) return undefined;
|
|
1271
1389
|
let text: string;
|
|
@@ -1328,6 +1446,7 @@ export function readSessionError(sessionFile: string | undefined): TranscriptErr
|
|
|
1328
1446
|
*/
|
|
1329
1447
|
export async function classifyAndRecover(d: SettlementDeps): Promise<number> {
|
|
1330
1448
|
const { project, caps, tracker, store } = d;
|
|
1449
|
+
const logDir = d.harnessLogDir ?? harnessLogDir();
|
|
1331
1450
|
// Settle recoveries, counted for the pass's dispatch record — a row whose PR
|
|
1332
1451
|
// merged is settled here when the settle sweep could not establish identity
|
|
1333
1452
|
// (#497). Every other exit returns 0.
|
|
@@ -1346,6 +1465,19 @@ export async function classifyAndRecover(d: SettlementDeps): Promise<number> {
|
|
|
1346
1465
|
classifiedRun = { ...run, lastError: sessionError.message };
|
|
1347
1466
|
}
|
|
1348
1467
|
}
|
|
1468
|
+
// The evidence the row cannot supply itself (#986): the transcript's exit
|
|
1469
|
+
// record and the harness's own routing for this run's process. Recorded
|
|
1470
|
+
// before classification so the table can read it, and recorded even when
|
|
1471
|
+
// it names nothing actionable — "what was known at settle time" is the
|
|
1472
|
+
// question `status` answers beside a surviving `unknown`, and a blank is
|
|
1473
|
+
// only honest when the sweep genuinely found nothing.
|
|
1474
|
+
if (run.terminalEvidence === undefined) {
|
|
1475
|
+
const evidence = readTerminalEvidence(run, logDir);
|
|
1476
|
+
if (evidence !== undefined) {
|
|
1477
|
+
store.updateRun(run.id, { terminalEvidence: evidence });
|
|
1478
|
+
classifiedRun = { ...classifiedRun, terminalEvidence: evidence };
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1349
1481
|
}
|
|
1350
1482
|
try {
|
|
1351
1483
|
if (run.prUrl !== undefined) {
|
|
@@ -1515,6 +1647,41 @@ async function recoverRun(
|
|
|
1515
1647
|
return;
|
|
1516
1648
|
}
|
|
1517
1649
|
|
|
1650
|
+
// `model-empty-stop`: the provider answered with empty turns until the
|
|
1651
|
+
// harness ended the session. The work is not at fault and the row is not a
|
|
1652
|
+
// failed attempt (the counters exclude the class), so this hands the queue
|
|
1653
|
+
// label back and the next dispatch resumes from whatever the attempt
|
|
1654
|
+
// pushed. Bounded by the continuation budget for the same reason the
|
|
1655
|
+
// wall-clock path is: a provider stuck in that state must reach a human
|
|
1656
|
+
// instead of consuming the issue forever, one free retry at a time.
|
|
1657
|
+
if (cls === "model-empty-stop") {
|
|
1658
|
+
const state = await tracker.issueState(run.issue).catch(() => undefined);
|
|
1659
|
+
if (state !== "open") {
|
|
1660
|
+
log(`#${run.issue} not continued from ${cls}: issue is ${state ?? "unreadable"}`);
|
|
1661
|
+
return;
|
|
1662
|
+
}
|
|
1663
|
+
const continuation = store.continuationsFor(project.name, run.issue);
|
|
1664
|
+
if (hasContinuationBudget(continuation, caps.maxContinuationsPerIssue)) {
|
|
1665
|
+
swapToQueue(d, run.issue, project.stateLabels.failed);
|
|
1666
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
1667
|
+
log(`#${run.issue} requeued after a provider empty-stop: ${evidence}`);
|
|
1668
|
+
return;
|
|
1669
|
+
}
|
|
1670
|
+
store.enqueueLabelOps(project.name, [
|
|
1671
|
+
{ issue: run.issue, op: "remove", label: project.stateLabels.failed },
|
|
1672
|
+
]);
|
|
1673
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
1674
|
+
await safeEscalate(d, {
|
|
1675
|
+
tier: 1,
|
|
1676
|
+
project: project.name,
|
|
1677
|
+
issue: run.issue,
|
|
1678
|
+
summary: `#${run.issue} keeps dying to provider empty-stops and has no continuation budget left`,
|
|
1679
|
+
detail: `${evidence}\n\nThe provider ended ${continuation} session(s) on this issue by returning empty turns until the harness's retry cap. That is not the work failing, so no implementation attempt was charged — but the continuation budget is spent, and another free retry would just consume the issue. Switching this issue's model is the usual remedy.`,
|
|
1680
|
+
});
|
|
1681
|
+
log(`#${run.issue} escalated: provider empty-stops exhausted the continuation budget`);
|
|
1682
|
+
return;
|
|
1683
|
+
}
|
|
1684
|
+
|
|
1518
1685
|
// `merge-conflict`: the branch is retained and its PR is open, so #50's
|
|
1519
1686
|
// continuation guard admits it and the next tick briefs a rebase.
|
|
1520
1687
|
//
|
package/src/setup.ts
CHANGED
|
@@ -72,6 +72,7 @@ import {
|
|
|
72
72
|
type DigestCadence,
|
|
73
73
|
type DraftPolicy,
|
|
74
74
|
type OrchestratorMode,
|
|
75
|
+
type GroomTrigger,
|
|
75
76
|
type InterruptCategory,
|
|
76
77
|
type ProjectConfig,
|
|
77
78
|
type ProjectPolicy,
|
|
@@ -161,11 +162,12 @@ export interface SetupAnswers {
|
|
|
161
162
|
*/
|
|
162
163
|
workerModel?: string;
|
|
163
164
|
/**
|
|
164
|
-
*
|
|
165
|
-
*
|
|
166
|
-
*
|
|
165
|
+
* The grooming trigger: a routable-candidate count below which the tick
|
|
166
|
+
* prompt says to groom, or `"always"` to groom on demand every tick (#988).
|
|
167
|
+
* The wizard never asks for it — it is hand-edited — so it exists here only
|
|
168
|
+
* to survive an amend of some other area (#369).
|
|
167
169
|
*/
|
|
168
|
-
groomBelow?:
|
|
170
|
+
groomBelow?: GroomTrigger;
|
|
169
171
|
/**
|
|
170
172
|
* Ordered fallback worker models and the provider-failure threshold that
|
|
171
173
|
* starts them. The wizard never asks for either — they are hand-edited —
|
package/src/spend-telemetry.ts
CHANGED
|
@@ -66,7 +66,34 @@ export type SpendTelemetryVerdict =
|
|
|
66
66
|
/** A majority of working runs reported nothing. */
|
|
67
67
|
| { kind: "partial"; worked: number; missing: number }
|
|
68
68
|
/** Every working run reported nothing — the same fact, stated louder. */
|
|
69
|
-
| { kind: "absent"; worked: number }
|
|
69
|
+
| { kind: "absent"; worked: number }
|
|
70
|
+
/**
|
|
71
|
+
* Every working run reported nothing **and this fleet does not bill in
|
|
72
|
+
* dollars** (#984). Not a fault: a subscription request has no per-request
|
|
73
|
+
* price, so the harness reports none and `spendUsd` is permanently 0.00 —
|
|
74
|
+
* accurate rather than missing. The constraint that actually binds is the
|
|
75
|
+
* provider's allowance window, named here.
|
|
76
|
+
*/
|
|
77
|
+
| { kind: "subscription"; worked: number; window: string; evidence: "declared" | "observed" };
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* What is known about how this project's runs are billed (#984).
|
|
81
|
+
*
|
|
82
|
+
* Two independent grounds, because on a real fleet the declaration is often
|
|
83
|
+
* absent while the evidence is unambiguous:
|
|
84
|
+
*
|
|
85
|
+
* - `declaredSubscription` — the project requires its providers to bill to a
|
|
86
|
+
* subscription credential (`requireOauthProviders`, #852). A statement of
|
|
87
|
+
* intent, so it is believed on its own.
|
|
88
|
+
* - `allowanceWindow` — the provider itself reports an allowance window
|
|
89
|
+
* (`omp usage --json`), which is what a subscription has instead of a price.
|
|
90
|
+
* Positive evidence only: a fleet whose provider reports no window has
|
|
91
|
+
* nothing here, and its zeros stay a fault.
|
|
92
|
+
*/
|
|
93
|
+
export interface SpendBilling {
|
|
94
|
+
declaredSubscription?: boolean;
|
|
95
|
+
allowanceWindow?: string;
|
|
96
|
+
}
|
|
70
97
|
|
|
71
98
|
/**
|
|
72
99
|
* Judge a newest-first sample.
|
|
@@ -78,13 +105,29 @@ export type SpendTelemetryVerdict =
|
|
|
78
105
|
export function judgeSpendTelemetry(
|
|
79
106
|
samples: readonly SpendSample[],
|
|
80
107
|
limit: number,
|
|
108
|
+
billing: SpendBilling = {},
|
|
81
109
|
): SpendTelemetryVerdict {
|
|
82
110
|
const working = samples.filter((row) => row.turns > 0).slice(0, limit);
|
|
83
111
|
if (working.length < limit) {
|
|
84
112
|
return { kind: "insufficient", worked: working.length, needed: limit };
|
|
85
113
|
}
|
|
86
114
|
const missing = working.filter((row) => row.spendUsd === 0).length;
|
|
87
|
-
if (missing === working.length)
|
|
115
|
+
if (missing === working.length) {
|
|
116
|
+
// Total absence is the only verdict a billing class can reclassify, and
|
|
117
|
+
// that asymmetry is the whole safety property: a metered neighbour proves
|
|
118
|
+
// dollars ARE being reported, so a zero beside one is real telemetry loss
|
|
119
|
+
// no matter how the fleet is billed. `partial` therefore always warns.
|
|
120
|
+
const window = subscriptionWindow(billing);
|
|
121
|
+
if (window !== undefined) {
|
|
122
|
+
return {
|
|
123
|
+
kind: "subscription",
|
|
124
|
+
worked: working.length,
|
|
125
|
+
window: window.window,
|
|
126
|
+
evidence: window.evidence,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
return { kind: "absent", worked: working.length };
|
|
130
|
+
}
|
|
88
131
|
if (missing / working.length >= SPEND_MISSING_SHARE) {
|
|
89
132
|
return { kind: "partial", worked: working.length, missing };
|
|
90
133
|
}
|
|
@@ -98,8 +141,37 @@ export function judgeSpendTelemetry(
|
|
|
98
141
|
|
|
99
142
|
/** The operator-facing sentence, or undefined when there is nothing to say.
|
|
100
143
|
* Shared so `doctor` and `status` cannot word the same fact differently. */
|
|
144
|
+
/**
|
|
145
|
+
* The allowance window to name, or `undefined` when nothing establishes that
|
|
146
|
+
* this fleet is subscription-billed.
|
|
147
|
+
*
|
|
148
|
+
* A declaration with no readable window still counts — the operator has said
|
|
149
|
+
* how the fleet bills, and a provider that cannot be read is not evidence
|
|
150
|
+
* against it — but it says so rather than inventing a window name.
|
|
151
|
+
*/
|
|
152
|
+
function subscriptionWindow(
|
|
153
|
+
billing: SpendBilling,
|
|
154
|
+
): { window: string; evidence: "declared" | "observed" } | undefined {
|
|
155
|
+
if (billing.allowanceWindow !== undefined && billing.allowanceWindow.trim() !== "") {
|
|
156
|
+
return {
|
|
157
|
+
window: billing.allowanceWindow.trim(),
|
|
158
|
+
evidence: billing.declaredSubscription === true ? "declared" : "observed",
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
if (billing.declaredSubscription === true) {
|
|
162
|
+
return { window: "unreadable (omp usage --json reported no window)", evidence: "declared" };
|
|
163
|
+
}
|
|
164
|
+
return undefined;
|
|
165
|
+
}
|
|
166
|
+
|
|
101
167
|
export function spendTelemetryDetail(verdict: SpendTelemetryVerdict): string | undefined {
|
|
102
168
|
switch (verdict.kind) {
|
|
169
|
+
case "subscription":
|
|
170
|
+
return (
|
|
171
|
+
`the last ${verdict.worked} working runs recorded $0.00, which is correct: this fleet bills to a ` +
|
|
172
|
+
`subscription (${verdict.evidence}), so requests carry no per-request price and no USD cap can fire` +
|
|
173
|
+
` — the constraint that binds is the ${verdict.window} allowance`
|
|
174
|
+
);
|
|
103
175
|
case "absent":
|
|
104
176
|
return (
|
|
105
177
|
`the last ${verdict.worked} completed runs that did any work all recorded $0.00 spend` +
|