omp-conductor 0.20.1 → 0.20.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/schema/config.schema.json +24 -0
- package/src/commands/companion.ts +52 -16
- package/src/commands/daemon.ts +14 -25
- package/src/commands/drain.ts +12 -5
- package/src/commands/worker.ts +5 -0
- package/src/config-schema.ts +8 -0
- package/src/config.ts +17 -2
- package/src/daemon/drain.ts +33 -0
- package/src/daemon/http.ts +28 -0
- package/src/daemon/review.ts +11 -11
- package/src/daemon/supervision.ts +144 -1
- package/src/daemon/tick.ts +46 -15
- package/src/daemon/views.ts +17 -0
- package/src/daemon.ts +2 -1
- package/src/decisions.ts +20 -9
- package/src/diff-flags.ts +186 -31
- package/src/doctor.ts +320 -9
- package/src/escalate.ts +187 -15
- package/src/failure-class.ts +176 -4
- package/src/fleet.ts +444 -50
- package/src/graph-health.ts +17 -1
- package/src/groom.ts +11 -0
- package/src/knowledge.ts +75 -21
- package/src/lifecycle.ts +267 -5
- package/src/orchestrator-tick.ts +8 -2
- package/src/ready-gate.ts +100 -5
- package/src/reports.ts +4 -1
- package/src/settlement.ts +98 -7
- package/src/status-render.ts +27 -2
- package/src/store.ts +50 -13
- package/src/to-spec.ts +52 -0
- package/src/tracker/github.ts +132 -10
- package/src/types.ts +31 -0
- package/src/upgrade.ts +12 -5
- package/src/verbs/server.ts +16 -12
package/src/escalate.ts
CHANGED
|
@@ -20,11 +20,12 @@
|
|
|
20
20
|
* those strings end up in daemon logs and, on the fallback path, in a public
|
|
21
21
|
* issue comment.
|
|
22
22
|
*/
|
|
23
|
-
import { readFileSync, statSync } from "node:fs";
|
|
23
|
+
import { mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs";
|
|
24
24
|
import { homedir } from "node:os";
|
|
25
25
|
import { dirname, join } from "node:path";
|
|
26
26
|
|
|
27
27
|
import { availabilityOpen, interruptDisposition, type InterruptDisposition } from "./availability.ts";
|
|
28
|
+
import { stateDir } from "./config.ts";
|
|
28
29
|
import { heldNoticeId } from "./notices.ts";
|
|
29
30
|
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
30
31
|
import type { Escalation, ProjectConfig, Store, Tracker } from "./types.ts";
|
|
@@ -341,7 +342,10 @@ export function createEscalator(
|
|
|
341
342
|
// the tick re-raises it. A *report* has no such source — it exists
|
|
342
343
|
// once, in the model's head, and nothing regenerates it — which is
|
|
343
344
|
// why that one needed a ledger and this one does not.
|
|
344
|
-
await sendTelegram(token, chatId, text, {
|
|
345
|
+
await sendTelegram(token, chatId, text, {
|
|
346
|
+
topicId: resolveProjectTopicId(p),
|
|
347
|
+
project: { name: p.name, workspaceRoot: p.workspaceRoot },
|
|
348
|
+
});
|
|
345
349
|
store.markNotified(key);
|
|
346
350
|
return;
|
|
347
351
|
}
|
|
@@ -435,6 +439,98 @@ export function readTelegramToken(): string | undefined {
|
|
|
435
439
|
return undefined;
|
|
436
440
|
}
|
|
437
441
|
|
|
442
|
+
/**
|
|
443
|
+
* One flat-chat delivery that happened only because the project's pinned
|
|
444
|
+
* topic was unreachable (#1094).
|
|
445
|
+
*
|
|
446
|
+
* The fallback is a delivery, not a success: the message reached an audience
|
|
447
|
+
* that did not ask for it while the topic the project reads stayed empty, and
|
|
448
|
+
* the only record used to be a journal `warn`. This row is what makes the
|
|
449
|
+
* mis-route visible where the operator looks: `doctor`'s topic-pin finding
|
|
450
|
+
* and the project's `status` render the rows matching the *current* pin, so
|
|
451
|
+
* re-pinning is what retires them. Nothing deletes rows — the file is bounded
|
|
452
|
+
* and a retired row simply stops matching.
|
|
453
|
+
*/
|
|
454
|
+
export interface TelegramMisroute {
|
|
455
|
+
/** The project whose message rode the flat chat. */
|
|
456
|
+
project: string;
|
|
457
|
+
/** The pinned `escalation.telegramTopicId` Telegram refused. */
|
|
458
|
+
staleTopicId: number;
|
|
459
|
+
/** When the flat delivery happened (epoch ms). */
|
|
460
|
+
at: number;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
const TELEGRAM_MISROUTES_FILE = "telegram-misroutes.json";
|
|
464
|
+
/** Diagnostic history, not a ledger: enough to see "again?", never unbounded. */
|
|
465
|
+
const TELEGRAM_MISROUTES_MAX = 20;
|
|
466
|
+
|
|
467
|
+
/**
|
|
468
|
+
* Records one flat delivery beside the conductor state. Best-effort by
|
|
469
|
+
* contract: the delivery has already happened, so a failed breadcrumb must
|
|
470
|
+
* never turn a completed escalation into a thrown one.
|
|
471
|
+
*/
|
|
472
|
+
export function recordTelegramMisroute(projectName: string, staleTopicId: number, at: number = Date.now()): void {
|
|
473
|
+
try {
|
|
474
|
+
const next = [
|
|
475
|
+
...readTelegramMisroutes(),
|
|
476
|
+
{ project: projectName, staleTopicId, at },
|
|
477
|
+
].slice(-TELEGRAM_MISROUTES_MAX);
|
|
478
|
+
const path = join(stateDir(), TELEGRAM_MISROUTES_FILE);
|
|
479
|
+
mkdirSync(stateDir(), { recursive: true });
|
|
480
|
+
const tmp = `${path}.${process.pid}.tmp`;
|
|
481
|
+
writeFileSync(tmp, `${JSON.stringify(next, null, 2)}\n`);
|
|
482
|
+
renameSync(tmp, path);
|
|
483
|
+
} catch {
|
|
484
|
+
// The journal warn still named the degradation; doctor still names the
|
|
485
|
+
// stale pin. Losing the row costs history, never delivery.
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
/**
|
|
490
|
+
* Every recorded mis-route, oldest first. Unreadable or malformed state reads
|
|
491
|
+
* empty — a broken breadcrumb file must not break the diagnostics that read
|
|
492
|
+
* it, and per-row validation keeps a hand-edited entry from poisoning the
|
|
493
|
+
* render (the same rule #1089 applied to the claim registry's fields).
|
|
494
|
+
*/
|
|
495
|
+
export function readTelegramMisroutes(): readonly TelegramMisroute[] {
|
|
496
|
+
let raw: string;
|
|
497
|
+
let parsed: unknown;
|
|
498
|
+
try {
|
|
499
|
+
raw = readFileSync(join(stateDir(), TELEGRAM_MISROUTES_FILE), "utf8");
|
|
500
|
+
parsed = JSON.parse(raw);
|
|
501
|
+
} catch {
|
|
502
|
+
return [];
|
|
503
|
+
}
|
|
504
|
+
if (!Array.isArray(parsed)) return [];
|
|
505
|
+
const rows: TelegramMisroute[] = [];
|
|
506
|
+
for (const entry of parsed) {
|
|
507
|
+
if (entry === null || typeof entry !== "object") continue;
|
|
508
|
+
const project = Reflect.get(entry, "project");
|
|
509
|
+
const staleTopicId = Reflect.get(entry, "staleTopicId");
|
|
510
|
+
const at = Reflect.get(entry, "at");
|
|
511
|
+
if (typeof project !== "string" || project.length === 0) continue;
|
|
512
|
+
if (typeof staleTopicId !== "number" || !Number.isSafeInteger(staleTopicId)) continue;
|
|
513
|
+
if (typeof at !== "number" || !Number.isFinite(at)) continue;
|
|
514
|
+
rows.push({ project, staleTopicId, at });
|
|
515
|
+
}
|
|
516
|
+
return rows.sort((a, b) => a.at - b.at);
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
/**
|
|
520
|
+
* The one-line notice prepended to a flat fallback (#1094): a reader in the
|
|
521
|
+
* main chat must know why the message is there and which fleet sent it, not
|
|
522
|
+
* reverse-engineer it from content — and the pinned id is named so the
|
|
523
|
+
* operator can act without opening the journal. Exported because the tests
|
|
524
|
+
* assert its wording and `status`'s misroute row quotes the same condition.
|
|
525
|
+
*/
|
|
526
|
+
export function staleTopicNotice(topicId: number, projectName?: string): string {
|
|
527
|
+
const who = projectName === undefined ? "This project's" : `[${projectName}] its`;
|
|
528
|
+
return (
|
|
529
|
+
`${who} pinned Telegram topic (escalation.telegramTopicId=${topicId}) is stale — ` +
|
|
530
|
+
`Telegram has no such thread, so this message was delivered to the flat chat instead.`
|
|
531
|
+
);
|
|
532
|
+
}
|
|
533
|
+
|
|
438
534
|
/** One live omp-telegram topic claim, as much of it as conductor reads. */
|
|
439
535
|
export interface ClaimedTopic {
|
|
440
536
|
threadId: number;
|
|
@@ -448,6 +544,13 @@ export interface ClaimedTopic {
|
|
|
448
544
|
pid?: number;
|
|
449
545
|
/** The herdr space the claiming pane sits in, when the bridge captured one. */
|
|
450
546
|
workspaceLabel?: string;
|
|
547
|
+
/**
|
|
548
|
+
* The claiming pane's working directory, when the bridge captured one. The
|
|
549
|
+
* bridge records it on every entry it writes; conductor reads it as the one
|
|
550
|
+
* discriminator it owns for telling a project's own panes apart when several
|
|
551
|
+
* sit in one herdr space (#1089).
|
|
552
|
+
*/
|
|
553
|
+
cwd?: string;
|
|
451
554
|
/**
|
|
452
555
|
* The transcript the claiming pane actually writes, when the bridge captured
|
|
453
556
|
* one. This is the *live* session file — herdr pins a restored pane to it
|
|
@@ -497,7 +600,7 @@ export function resolveProjectTopicId(project: ProjectConfig, alive?: (pid: numb
|
|
|
497
600
|
if (claims.some((claim) => claim.threadId === pinned && claimIsLive(claim, alive === undefined ? undefined : killOf(alive)))) {
|
|
498
601
|
return pinned;
|
|
499
602
|
}
|
|
500
|
-
const match = claimForProject(claims, project.name, alive);
|
|
603
|
+
const match = claimForProject(claims, project.name, alive, project.workspaceRoot);
|
|
501
604
|
if (match === undefined) return pinned;
|
|
502
605
|
warn(
|
|
503
606
|
`escalation.telegramTopicId for ${project.name} is no longer a claimed topic; ` +
|
|
@@ -535,20 +638,36 @@ export type ProjectClaim =
|
|
|
535
638
|
* *no* live claim carries the project's space — several live claims wearing
|
|
536
639
|
* the space is an ambiguity the title cannot resolve, because the titled
|
|
537
640
|
* claim may be a sibling pane, not this project (#626).
|
|
641
|
+
*
|
|
642
|
+
* One same-space ambiguity conductor settles itself (#1089): it provisioned
|
|
643
|
+
* both panes from its own config, so when the caller supplies the project's
|
|
644
|
+
* orchestrator cwd — its workspace root, against the console's
|
|
645
|
+
* `<stateDir>/console/<name>` — exactly one live claimant sitting there is
|
|
646
|
+
* identification, not a guess. Neither or both matching stays the ambiguity.
|
|
538
647
|
*/
|
|
539
648
|
export function resolveProjectClaim(
|
|
540
649
|
claims: readonly ClaimedTopic[],
|
|
541
650
|
projectName: string,
|
|
542
651
|
alive?: (pid: number) => boolean,
|
|
652
|
+
orchestratorCwd?: string,
|
|
543
653
|
): ProjectClaim {
|
|
544
654
|
// Dead claims do not vote: a closed pane's row must not turn this project's
|
|
545
655
|
// one live claim into an ambiguity, and must not answer by title on its own
|
|
546
|
-
// (#987). Ambiguity among genuinely live claims is
|
|
656
|
+
// (#987). Ambiguity among genuinely live claims is narrowed by the one
|
|
657
|
+
// discriminator conductor owns (#1089), never removed (#626).
|
|
547
658
|
const live = claims.filter((claim) => claimIsLive(claim, alive === undefined ? undefined : killOf(alive)));
|
|
548
659
|
const bySpace = live.filter((claim) => claim.workspaceLabel === projectName);
|
|
549
660
|
if (bySpace.length > 0) {
|
|
550
|
-
|
|
551
|
-
|
|
661
|
+
if (bySpace.length === 1) return { kind: "match", claim: bySpace[0]! };
|
|
662
|
+
// Conductor provisions every same-space pane from its own config: the
|
|
663
|
+
// orchestrator pane's cwd is the project's workspace root, the console's
|
|
664
|
+
// is `<stateDir>/console/<name>` (#1089). Exactly one claimant sitting at
|
|
665
|
+
// the orchestrator cwd is identification; neither or both matching stays
|
|
666
|
+
// the coin toss #626 refuses to spin.
|
|
667
|
+
const orchestrator =
|
|
668
|
+
orchestratorCwd === undefined ? [] : bySpace.filter((claim) => claim.cwd === orchestratorCwd);
|
|
669
|
+
return orchestrator.length === 1
|
|
670
|
+
? { kind: "match", claim: orchestrator[0]! }
|
|
552
671
|
: { kind: "ambiguous", claimants: bySpace };
|
|
553
672
|
}
|
|
554
673
|
const byTitle = live.filter((claim) => claim.name === projectName);
|
|
@@ -571,8 +690,9 @@ export function claimForProject(
|
|
|
571
690
|
claims: readonly ClaimedTopic[],
|
|
572
691
|
projectName: string,
|
|
573
692
|
alive?: (pid: number) => boolean,
|
|
693
|
+
orchestratorCwd?: string,
|
|
574
694
|
): ClaimedTopic | undefined {
|
|
575
|
-
const match = resolveProjectClaim(claims, projectName, alive);
|
|
695
|
+
const match = resolveProjectClaim(claims, projectName, alive, orchestratorCwd);
|
|
576
696
|
return match.kind === "match" ? match.claim : undefined;
|
|
577
697
|
}
|
|
578
698
|
|
|
@@ -595,7 +715,7 @@ export function resolveClaimedSessionFile(
|
|
|
595
715
|
): string | undefined {
|
|
596
716
|
const result = claimedTelegramTopics();
|
|
597
717
|
if (result.kind !== "ok" || result.claims.length === 0) return undefined;
|
|
598
|
-
return claimForProject(result.claims, project.name, alive)?.sessionFile;
|
|
718
|
+
return claimForProject(result.claims, project.name, alive, project.workspaceRoot)?.sessionFile;
|
|
599
719
|
}
|
|
600
720
|
|
|
601
721
|
/**
|
|
@@ -699,6 +819,9 @@ export function claimedTelegramTopics(stateDir?: string): ClaimedTopicsResult {
|
|
|
699
819
|
if ("sessionFile" in entry && typeof entry.sessionFile !== "string") {
|
|
700
820
|
return { kind: "unavailable", problem: `${registry}: claim "${id}" has a non-string sessionFile` };
|
|
701
821
|
}
|
|
822
|
+
if ("cwd" in entry && typeof entry.cwd !== "string") {
|
|
823
|
+
return { kind: "unavailable", problem: `${registry}: claim "${id}" has a non-string cwd` };
|
|
824
|
+
}
|
|
702
825
|
if ("pid" in entry && (typeof entry.pid !== "number" || !Number.isFinite(entry.pid))) {
|
|
703
826
|
return { kind: "unavailable", problem: `${registry}: claim "${id}" has a non-numeric pid` };
|
|
704
827
|
}
|
|
@@ -706,14 +829,17 @@ export function claimedTelegramTopics(stateDir?: string): ClaimedTopicsResult {
|
|
|
706
829
|
let workspaceLabel: string | undefined;
|
|
707
830
|
let sessionFile: string | undefined;
|
|
708
831
|
let pid: number | undefined;
|
|
832
|
+
let cwd: string | undefined;
|
|
709
833
|
if ("name" in entry && entry.name.trim() !== "") name = entry.name.trim();
|
|
710
834
|
if ("workspaceLabel" in entry && entry.workspaceLabel.trim() !== "") workspaceLabel = entry.workspaceLabel.trim();
|
|
711
835
|
if ("sessionFile" in entry && entry.sessionFile.trim() !== "") sessionFile = entry.sessionFile.trim();
|
|
836
|
+
if ("cwd" in entry && entry.cwd.trim() !== "") cwd = entry.cwd.trim();
|
|
712
837
|
if ("pid" in entry) pid = entry.pid;
|
|
713
838
|
const claim: ClaimedTopic = { threadId, name };
|
|
714
839
|
if (workspaceLabel !== undefined) claim.workspaceLabel = workspaceLabel;
|
|
715
840
|
if (sessionFile !== undefined) claim.sessionFile = sessionFile;
|
|
716
841
|
if (pid !== undefined) claim.pid = pid;
|
|
842
|
+
if (cwd !== undefined) claim.cwd = cwd;
|
|
717
843
|
out.push(claim);
|
|
718
844
|
}
|
|
719
845
|
return { kind: "ok", claims: out };
|
|
@@ -1213,17 +1339,32 @@ function sessionWithinScan(sessionFile: string, dirs: readonly string[]): boolea
|
|
|
1213
1339
|
* interleave with each other.
|
|
1214
1340
|
*
|
|
1215
1341
|
* Optional `topicId` pins the message to a forum topic (`message_thread_id`).
|
|
1216
|
-
* A definitive missing-thread reject
|
|
1217
|
-
*
|
|
1218
|
-
*
|
|
1219
|
-
*
|
|
1220
|
-
* a flat
|
|
1342
|
+
* A definitive missing-thread reject degrades in two ordered steps (#1094):
|
|
1343
|
+
* first to the project's one live claim, when `project` names who the send
|
|
1344
|
+
* answers to and the registry identifies exactly one (#1089) — a resend into
|
|
1345
|
+
* the project's own current topic. Only when no topic can be resolved does it
|
|
1346
|
+
* retry as a flat chat, prepending a notice that names the stale pin and the
|
|
1347
|
+
* project, and recording a mis-route row for doctor/status (#318's silent
|
|
1348
|
+
* shape). Every degrade re-sends the whole split: parts already accepted went
|
|
1349
|
+
* into a thread Telegram has just told us is gone, so nothing readable is
|
|
1350
|
+
* ever duplicated.
|
|
1221
1351
|
*/
|
|
1222
1352
|
export async function sendTelegram(
|
|
1223
1353
|
token: string,
|
|
1224
1354
|
chatId: string,
|
|
1225
1355
|
text: string,
|
|
1226
|
-
opts?: {
|
|
1356
|
+
opts?: {
|
|
1357
|
+
topicId?: number;
|
|
1358
|
+
/**
|
|
1359
|
+
* Who this send answers to, when it rides a pinned topic (#1094): the
|
|
1360
|
+
* project whose escalation, report, or challenge it carries. Only a
|
|
1361
|
+
* caller that resolved `topicId` for a project passes it, and that pair
|
|
1362
|
+
* is what lets a missing-thread fallback prefer the project's one live
|
|
1363
|
+
* claim, label the flat delivery with the project's name, and record the
|
|
1364
|
+
* mis-route where the operator looks.
|
|
1365
|
+
*/
|
|
1366
|
+
project?: { name: string; workspaceRoot?: string };
|
|
1367
|
+
},
|
|
1227
1368
|
): Promise<number[]> {
|
|
1228
1369
|
const topicId =
|
|
1229
1370
|
opts?.topicId !== undefined && Number.isFinite(opts.topicId) && Number.isSafeInteger(opts.topicId)
|
|
@@ -1244,7 +1385,38 @@ export async function sendTelegram(
|
|
|
1244
1385
|
warn(
|
|
1245
1386
|
`escalation.telegramTopicId=${topicId} is stale (${err.message}); retrying flat chat`,
|
|
1246
1387
|
);
|
|
1247
|
-
|
|
1388
|
+
// #318's flat retry is the last resort, not the first (#1094). When the
|
|
1389
|
+
// claim registry names exactly one live claim for this project — #1089's
|
|
1390
|
+
// cwd discriminator included — the resend follows it: a page landing in
|
|
1391
|
+
// the project's own current topic is not a mis-route at all. A
|
|
1392
|
+
// substitution that itself fails falls through to the labelled flat send
|
|
1393
|
+
// below; delivery outranks tidiness.
|
|
1394
|
+
const project = opts?.project;
|
|
1395
|
+
if (project !== undefined) {
|
|
1396
|
+
try {
|
|
1397
|
+
const registry = claimedTelegramTopics();
|
|
1398
|
+
if (registry.kind === "ok") {
|
|
1399
|
+
const claim = claimForProject(registry.claims, project.name, undefined, project.workspaceRoot);
|
|
1400
|
+
if (claim !== undefined && claim.threadId !== topicId) {
|
|
1401
|
+
warn(`following omp-telegram's live "${project.name}" claim instead of the flat chat`);
|
|
1402
|
+
return await postTelegramParts(token, chatId, parts, claim.threadId);
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
} catch (subErr) {
|
|
1406
|
+
warn(
|
|
1407
|
+
`live-claim substitution failed (${subErr instanceof Error ? subErr.message : String(subErr)}); delivering flat`,
|
|
1408
|
+
);
|
|
1409
|
+
}
|
|
1410
|
+
}
|
|
1411
|
+
// Flat, but never silent (#1094): the delivered text names the stale pin
|
|
1412
|
+
// and the project, so the wrong audience knows what it is reading, and
|
|
1413
|
+
// doctor/status gain the evidence against this pin. The record is written
|
|
1414
|
+
// only after the send resolves — a flat send that threw delivered
|
|
1415
|
+
// nothing, and the next tick retries it as a first attempt.
|
|
1416
|
+
const labelled = `${staleTopicNotice(topicId, project?.name)}\n\n${text}`;
|
|
1417
|
+
const ids = await postTelegramParts(token, chatId, telegramTextParts(labelled), undefined);
|
|
1418
|
+
if (project?.name !== undefined) recordTelegramMisroute(project.name, topicId);
|
|
1419
|
+
return ids;
|
|
1248
1420
|
}
|
|
1249
1421
|
}
|
|
1250
1422
|
|
package/src/failure-class.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* green test per class" possible without a daemon, a tracker or a network.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import type
|
|
18
|
+
import { DEFAULT_CAPS, type Caps, type FailureClass, type RecoveryAction, type RunRecord } from "./types.ts";
|
|
19
19
|
|
|
20
20
|
/** Facts the caller fetched, each only for the rows that need it.
|
|
21
21
|
*
|
|
@@ -202,6 +202,45 @@ export const SPINNING_CAP_CLASSES: readonly FailureClass[] = [
|
|
|
202
202
|
"wall-clock-cap-spinning",
|
|
203
203
|
];
|
|
204
204
|
|
|
205
|
+
/** How much of the wall-clock ceiling the default stall window may take (#1086). */
|
|
206
|
+
export const WALL_CLOCK_STALL_SHARE = 3;
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* The transcript-silence window a stalled run is settled at (#1086): the
|
|
210
|
+
* configured `workerStallSilenceMs`, or a third of the wall-clock ceiling when
|
|
211
|
+
* it is null or unconfigured. Shared by the daemon's progress watch (which
|
|
212
|
+
* settles at this threshold) and the classifier (which recognises a row killed
|
|
213
|
+
* at it), so the two can never disagree about what a stall is.
|
|
214
|
+
*/
|
|
215
|
+
export function stallSilenceMs(
|
|
216
|
+
caps:
|
|
217
|
+
| {
|
|
218
|
+
workerStallSilenceMs?: number | null;
|
|
219
|
+
workerWallClockMs: number;
|
|
220
|
+
}
|
|
221
|
+
| undefined,
|
|
222
|
+
): number {
|
|
223
|
+
const wallClock = caps?.workerWallClockMs ?? DEFAULT_CAPS.workerWallClockMs;
|
|
224
|
+
return caps?.workerStallSilenceMs ?? Math.floor(wallClock / WALL_CLOCK_STALL_SHARE);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* How long a run sat silent before its stall kill, from the row's own facts
|
|
229
|
+
* (#1086). Defined only when the progress watch had observed the run AND the
|
|
230
|
+
* silence reached the configured window — the exact predicate that keeps every
|
|
231
|
+
* other killed-under-ceiling row (a drain, an operator stop) reading as
|
|
232
|
+
* `admin-kill`.
|
|
233
|
+
*/
|
|
234
|
+
export function stallSilence(
|
|
235
|
+
run: RunRecord,
|
|
236
|
+
caps: { workerStallSilenceMs?: number | null; workerWallClockMs: number } | undefined,
|
|
237
|
+
): number | undefined {
|
|
238
|
+
if (run.lastProgressAt === undefined || run.endedAt === undefined) return undefined;
|
|
239
|
+
const silent = run.endedAt - run.lastProgressAt;
|
|
240
|
+
if (silent < stallSilenceMs(caps)) return undefined;
|
|
241
|
+
return silent;
|
|
242
|
+
}
|
|
243
|
+
|
|
205
244
|
/**
|
|
206
245
|
* A stable fingerprint of the infrastructure signature list (#638). The
|
|
207
246
|
* historical reconciliation persists a per-project review cursor stamped with
|
|
@@ -728,10 +767,100 @@ function firstReportContentLine(report: string | undefined): string | undefined
|
|
|
728
767
|
return undefined;
|
|
729
768
|
}
|
|
730
769
|
|
|
770
|
+
/**
|
|
771
|
+
* What the wall-clock ceiling actually buys on this fleet right now (#1063).
|
|
772
|
+
*
|
|
773
|
+
* `caps.workerMaxTurns` and `caps.workerWallClockMs` are independent config
|
|
774
|
+
* values, but they meet at an implicit pace: 0.5 minutes per turn at the
|
|
775
|
+
* shipped defaults. A model slower than that can never reach the turn ceiling,
|
|
776
|
+
* so the budget a groomer sized a slice against is fiction — the run dies of
|
|
777
|
+
* the clock first. This observation is derived from *completed* runs (never
|
|
778
|
+
* from the configured numbers, whose ratio is a constant and tells nobody
|
|
779
|
+
* anything), so `status` and the settlement report can name the real number.
|
|
780
|
+
*/
|
|
781
|
+
export interface ObservedTurnBudget {
|
|
782
|
+
/** Aggregate elapsed minutes ÷ aggregate turns across the sample. */
|
|
783
|
+
minutesPerTurn: number;
|
|
784
|
+
/** Completed runs the figure was derived from. */
|
|
785
|
+
sampleSize: number;
|
|
786
|
+
/** ⌊wall clock ÷ minutesPerTurn⌋ — what the clock buys at that pace. */
|
|
787
|
+
effectiveTurns: number;
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
/** How many completed runs the effective-budget sample spans. */
|
|
791
|
+
export const EFFECTIVE_BUDGET_SAMPLE_RUNS = 20;
|
|
792
|
+
|
|
793
|
+
/** A wall-clock kill below this share of its turn ceiling is a latency
|
|
794
|
+
* verdict; at or above it, both ceilings were nearly exhausted and size is
|
|
795
|
+
* the honest reading (#1063). */
|
|
796
|
+
export const WALL_CLOCK_SIZE_VERDICT_SHARE = 0.7;
|
|
797
|
+
|
|
798
|
+
/** Strips omp selector suffixes (`:thinking`, `:max`) so only the model path
|
|
799
|
+
* is compared. */
|
|
800
|
+
function modelBase(selector: string): string {
|
|
801
|
+
const colon = selector.indexOf(":");
|
|
802
|
+
return colon === -1 ? selector : selector.slice(0, colon);
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
/**
|
|
806
|
+
* Whether one model attribution names the same model as the configured
|
|
807
|
+
* selector, compared as path suffixes: config says
|
|
808
|
+
* `openrouter/stealth/ox-alpha:max` while the transcript's resolved model
|
|
809
|
+
* reads `stealth/ox-alpha`, and strict equality would empty every sample.
|
|
810
|
+
* Role aliases (`@slow`) have no concrete spelling to resolve to from here, so
|
|
811
|
+
* they match themselves only — no sample, rather than an invented one.
|
|
812
|
+
*/
|
|
813
|
+
function sameModel(configured: string, observed: string): boolean {
|
|
814
|
+
if (configured.startsWith("@") || observed.startsWith("@")) return configured === observed;
|
|
815
|
+
const c = modelBase(configured);
|
|
816
|
+
const o = modelBase(observed);
|
|
817
|
+
return c === o || c.endsWith(`/${o}`) || o.endsWith(`/${c}`);
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
/**
|
|
821
|
+
* Derives {@link ObservedTurnBudget} from completed-run rows, newest-first
|
|
822
|
+
* callers' ordering irrelevant. Rows without turns or elapsed time carry no
|
|
823
|
+
* pace and are skipped; when a configured worker model is named, rows written
|
|
824
|
+
* by another model are skipped with them (`resolvedModel` preferred — what
|
|
825
|
+
* actually wrote the messages — falling back to the dispatch record).
|
|
826
|
+
* `undefined` when nothing qualifying remains: absence is the honest answer,
|
|
827
|
+
* never a guess from the configured constants.
|
|
828
|
+
*/
|
|
829
|
+
export function observeTurnBudget(
|
|
830
|
+
samples: readonly Pick<RunRecord, "id" | "turns" | "startedAt" | "endedAt" | "model" | "resolvedModel">[],
|
|
831
|
+
caps: Pick<Caps, "workerWallClockMs">,
|
|
832
|
+
configuredModel?: string,
|
|
833
|
+
excludeRunId?: string,
|
|
834
|
+
): ObservedTurnBudget | undefined {
|
|
835
|
+
let minutes = 0;
|
|
836
|
+
let turns = 0;
|
|
837
|
+
let used = 0;
|
|
838
|
+
for (const s of samples) {
|
|
839
|
+
if (s.id === excludeRunId) continue;
|
|
840
|
+
if (s.turns <= 0 || s.endedAt === undefined || s.endedAt <= s.startedAt) continue;
|
|
841
|
+
if (configuredModel !== undefined) {
|
|
842
|
+
const attribution = s.resolvedModel ?? s.model;
|
|
843
|
+
if (attribution === undefined || !sameModel(configuredModel, attribution)) continue;
|
|
844
|
+
}
|
|
845
|
+
minutes += (s.endedAt - s.startedAt) / 60_000;
|
|
846
|
+
turns += s.turns;
|
|
847
|
+
used += 1;
|
|
848
|
+
}
|
|
849
|
+
if (used === 0 || turns <= 0) return undefined;
|
|
850
|
+
const minutesPerTurn = minutes / turns;
|
|
851
|
+
if (!Number.isFinite(minutesPerTurn) || minutesPerTurn <= 0) return undefined;
|
|
852
|
+
return {
|
|
853
|
+
minutesPerTurn,
|
|
854
|
+
sampleSize: used,
|
|
855
|
+
effectiveTurns: Math.floor(caps.workerWallClockMs / 60_000 / minutesPerTurn),
|
|
856
|
+
};
|
|
857
|
+
}
|
|
858
|
+
|
|
731
859
|
export function classifyRun(
|
|
732
860
|
run: RunRecord,
|
|
733
861
|
facts: ClassifyFacts,
|
|
734
|
-
caps?: Pick<Caps, "workerWallClockMs"
|
|
862
|
+
caps?: Pick<Caps, "workerWallClockMs"> & { workerStallSilenceMs?: number | null },
|
|
863
|
+
observedBudget?: ObservedTurnBudget,
|
|
735
864
|
): Classification {
|
|
736
865
|
const hasArtifacts = run.prUrl !== undefined || run.headSha !== undefined || run.salvageSha !== undefined;
|
|
737
866
|
const providerError =
|
|
@@ -932,6 +1061,30 @@ export function classifyRun(
|
|
|
932
1061
|
}
|
|
933
1062
|
|
|
934
1063
|
if (run.state === "killed") {
|
|
1064
|
+
// A stall the daemon settled (#1086), recognised from the row's own facts
|
|
1065
|
+
// — the watch observed a transcript write instant, and the silence to the
|
|
1066
|
+
// kill reached the configured window. Checked before the cap branches on
|
|
1067
|
+
// purpose: a hung session's turns never move, so this row is under its
|
|
1068
|
+
// ceilings, but even if a config change made it *reach* one mid-hang, the
|
|
1069
|
+
// silence is the operative fact about why it died. The same split as the
|
|
1070
|
+
// caps: artifacts continue, nothing shows escalates.
|
|
1071
|
+
const silentMs = stallSilence(run, caps);
|
|
1072
|
+
if (silentMs !== undefined) {
|
|
1073
|
+
const minutes = Math.round(silentMs / 60_000);
|
|
1074
|
+
const silent =
|
|
1075
|
+
minutes < 60 ? `${minutes}m` : `${Math.floor(minutes / 60)}h${minutes % 60 === 0 ? "" : `${minutes % 60}m`}`;
|
|
1076
|
+
return hasArtifacts
|
|
1077
|
+
? {
|
|
1078
|
+
cls: "progress-stall",
|
|
1079
|
+
recovery: "continue",
|
|
1080
|
+
evidence: `transcript silent ${silent} at turn ${run.turns}/${run.maxTurns}; the session never came back — work to continue from (${run.prUrl ?? run.salvageSha ?? run.headSha})`,
|
|
1081
|
+
}
|
|
1082
|
+
: {
|
|
1083
|
+
cls: "progress-stall",
|
|
1084
|
+
recovery: "escalate",
|
|
1085
|
+
evidence: `transcript silent ${silent} at turn ${run.turns}/${run.maxTurns}, no PR, no commits — $${run.spendUsd.toFixed(2)} spent`,
|
|
1086
|
+
};
|
|
1087
|
+
}
|
|
935
1088
|
if (run.turns >= run.maxTurns) {
|
|
936
1089
|
return hasArtifacts
|
|
937
1090
|
? {
|
|
@@ -950,21 +1103,40 @@ export function classifyRun(
|
|
|
950
1103
|
// worker slot until the ticks ran out (#490). The same split as the turns
|
|
951
1104
|
// path: artifacts mean the next attempt resumes from real work; nothing to
|
|
952
1105
|
// show means a human re-scopes, naming the time and spend consumed.
|
|
1106
|
+
//
|
|
1107
|
+
// #1063: which ceiling was ever reachable? The two caps meet at an implied
|
|
1108
|
+
// pace (wall clock ÷ max turns), so a run killed at the clock far under
|
|
1109
|
+
// its turn budget died of *latency* — re-attempting it as an oversized
|
|
1110
|
+
// slice spends a continuation on the wrong diagnosis. The evidence names
|
|
1111
|
+
// the run's own pace, the latency/size verdict the share of the turn
|
|
1112
|
+
// ceiling implies, and — when the caller supplied one — the effective
|
|
1113
|
+
// turn budget observed across completed runs on this model.
|
|
953
1114
|
const wallClockCap = caps?.workerWallClockMs;
|
|
954
1115
|
const wallClockElapsed =
|
|
955
1116
|
run.endedAt === undefined ? undefined : run.endedAt - run.startedAt;
|
|
956
1117
|
if (wallClockCap !== undefined && wallClockElapsed !== undefined && wallClockElapsed >= wallClockCap) {
|
|
957
1118
|
const clock = `${Math.round(wallClockElapsed / 60_000)}m of ${Math.round(wallClockCap / 60_000)}m`;
|
|
1119
|
+
const pace =
|
|
1120
|
+
run.turns > 0 ? ` at ${((wallClockElapsed / 60_000) / run.turns).toFixed(2)} min/turn` : "";
|
|
1121
|
+
const share = run.maxTurns > 0 ? run.turns / run.maxTurns : 1;
|
|
1122
|
+
const verdict =
|
|
1123
|
+
share < WALL_CLOCK_SIZE_VERDICT_SHARE
|
|
1124
|
+
? `latency verdict: ${run.turns}/${run.maxTurns} turns before the clock — the slice fits, the model is slow`
|
|
1125
|
+
: `size verdict: ${run.turns}/${run.maxTurns} turns — both ceilings nearly exhausted`;
|
|
1126
|
+
const observed =
|
|
1127
|
+
observedBudget === undefined
|
|
1128
|
+
? ""
|
|
1129
|
+
: `; effective ~${observedBudget.effectiveTurns} turns at ${observedBudget.minutesPerTurn.toFixed(2)} min/turn (last ${observedBudget.sampleSize} runs)`;
|
|
958
1130
|
return hasArtifacts
|
|
959
1131
|
? {
|
|
960
1132
|
cls: "wall-clock-cap-progress",
|
|
961
1133
|
recovery: "continue",
|
|
962
|
-
evidence: `wall clock ${clock}
|
|
1134
|
+
evidence: `wall clock ${clock}${pace} — ${verdict}${observed}; work to continue from (${run.prUrl ?? run.salvageSha ?? run.headSha})`,
|
|
963
1135
|
}
|
|
964
1136
|
: {
|
|
965
1137
|
cls: "wall-clock-cap-spinning",
|
|
966
1138
|
recovery: "escalate",
|
|
967
|
-
evidence: `wall clock ${clock}
|
|
1139
|
+
evidence: `wall clock ${clock}${pace} — ${verdict}${observed}; no PR, no commits — $${run.spendUsd.toFixed(2)} spent`,
|
|
968
1140
|
};
|
|
969
1141
|
}
|
|
970
1142
|
// Below its own ceilings, so nothing this worker did ended it: a daemon
|