omp-conductor 0.20.1 → 0.20.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/escalate.ts CHANGED
@@ -20,11 +20,12 @@
20
20
  * those strings end up in daemon logs and, on the fallback path, in a public
21
21
  * issue comment.
22
22
  */
23
- import { readFileSync, statSync } from "node:fs";
23
+ import { mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs";
24
24
  import { homedir } from "node:os";
25
25
  import { dirname, join } from "node:path";
26
26
 
27
27
  import { availabilityOpen, interruptDisposition, type InterruptDisposition } from "./availability.ts";
28
+ import { stateDir } from "./config.ts";
28
29
  import { heldNoticeId } from "./notices.ts";
29
30
  import type { OrchestratorHandle } from "./orchestrator.ts";
30
31
  import type { Escalation, ProjectConfig, Store, Tracker } from "./types.ts";
@@ -341,7 +342,10 @@ export function createEscalator(
341
342
  // the tick re-raises it. A *report* has no such source — it exists
342
343
  // once, in the model's head, and nothing regenerates it — which is
343
344
  // why that one needed a ledger and this one does not.
344
- await sendTelegram(token, chatId, text, { topicId: resolveProjectTopicId(p) });
345
+ await sendTelegram(token, chatId, text, {
346
+ topicId: resolveProjectTopicId(p),
347
+ project: { name: p.name, workspaceRoot: p.workspaceRoot },
348
+ });
345
349
  store.markNotified(key);
346
350
  return;
347
351
  }
@@ -435,6 +439,98 @@ export function readTelegramToken(): string | undefined {
435
439
  return undefined;
436
440
  }
437
441
 
442
+ /**
443
+ * One flat-chat delivery that happened only because the project's pinned
444
+ * topic was unreachable (#1094).
445
+ *
446
+ * The fallback is a delivery, not a success: the message reached an audience
447
+ * that did not ask for it while the topic the project reads stayed empty, and
448
+ * the only record used to be a journal `warn`. This row is what makes the
449
+ * mis-route visible where the operator looks: `doctor`'s topic-pin finding
450
+ * and the project's `status` render the rows matching the *current* pin, so
451
+ * re-pinning is what retires them. Nothing deletes rows — the file is bounded
452
+ * and a retired row simply stops matching.
453
+ */
454
+ export interface TelegramMisroute {
455
+ /** The project whose message rode the flat chat. */
456
+ project: string;
457
+ /** The pinned `escalation.telegramTopicId` Telegram refused. */
458
+ staleTopicId: number;
459
+ /** When the flat delivery happened (epoch ms). */
460
+ at: number;
461
+ }
462
+
463
+ const TELEGRAM_MISROUTES_FILE = "telegram-misroutes.json";
464
+ /** Diagnostic history, not a ledger: enough to see "again?", never unbounded. */
465
+ const TELEGRAM_MISROUTES_MAX = 20;
466
+
467
+ /**
468
+ * Records one flat delivery beside the conductor state. Best-effort by
469
+ * contract: the delivery has already happened, so a failed breadcrumb must
470
+ * never turn a completed escalation into a thrown one.
471
+ */
472
+ export function recordTelegramMisroute(projectName: string, staleTopicId: number, at: number = Date.now()): void {
473
+ try {
474
+ const next = [
475
+ ...readTelegramMisroutes(),
476
+ { project: projectName, staleTopicId, at },
477
+ ].slice(-TELEGRAM_MISROUTES_MAX);
478
+ const path = join(stateDir(), TELEGRAM_MISROUTES_FILE);
479
+ mkdirSync(stateDir(), { recursive: true });
480
+ const tmp = `${path}.${process.pid}.tmp`;
481
+ writeFileSync(tmp, `${JSON.stringify(next, null, 2)}\n`);
482
+ renameSync(tmp, path);
483
+ } catch {
484
+ // The journal warn still named the degradation; doctor still names the
485
+ // stale pin. Losing the row costs history, never delivery.
486
+ }
487
+ }
488
+
489
+ /**
490
+ * Every recorded mis-route, oldest first. Unreadable or malformed state reads
491
+ * empty — a broken breadcrumb file must not break the diagnostics that read
492
+ * it, and per-row validation keeps a hand-edited entry from poisoning the
493
+ * render (the same rule #1089 applied to the claim registry's fields).
494
+ */
495
+ export function readTelegramMisroutes(): readonly TelegramMisroute[] {
496
+ let raw: string;
497
+ let parsed: unknown;
498
+ try {
499
+ raw = readFileSync(join(stateDir(), TELEGRAM_MISROUTES_FILE), "utf8");
500
+ parsed = JSON.parse(raw);
501
+ } catch {
502
+ return [];
503
+ }
504
+ if (!Array.isArray(parsed)) return [];
505
+ const rows: TelegramMisroute[] = [];
506
+ for (const entry of parsed) {
507
+ if (entry === null || typeof entry !== "object") continue;
508
+ const project = Reflect.get(entry, "project");
509
+ const staleTopicId = Reflect.get(entry, "staleTopicId");
510
+ const at = Reflect.get(entry, "at");
511
+ if (typeof project !== "string" || project.length === 0) continue;
512
+ if (typeof staleTopicId !== "number" || !Number.isSafeInteger(staleTopicId)) continue;
513
+ if (typeof at !== "number" || !Number.isFinite(at)) continue;
514
+ rows.push({ project, staleTopicId, at });
515
+ }
516
+ return rows.sort((a, b) => a.at - b.at);
517
+ }
518
+
519
+ /**
520
+ * The one-line notice prepended to a flat fallback (#1094): a reader in the
521
+ * main chat must know why the message is there and which fleet sent it, not
522
+ * reverse-engineer it from content — and the pinned id is named so the
523
+ * operator can act without opening the journal. Exported because the tests
524
+ * assert its wording and `status`'s misroute row quotes the same condition.
525
+ */
526
+ export function staleTopicNotice(topicId: number, projectName?: string): string {
527
+ const who = projectName === undefined ? "This project's" : `[${projectName}] its`;
528
+ return (
529
+ `${who} pinned Telegram topic (escalation.telegramTopicId=${topicId}) is stale — ` +
530
+ `Telegram has no such thread, so this message was delivered to the flat chat instead.`
531
+ );
532
+ }
533
+
438
534
  /** One live omp-telegram topic claim, as much of it as conductor reads. */
439
535
  export interface ClaimedTopic {
440
536
  threadId: number;
@@ -448,6 +544,13 @@ export interface ClaimedTopic {
448
544
  pid?: number;
449
545
  /** The herdr space the claiming pane sits in, when the bridge captured one. */
450
546
  workspaceLabel?: string;
547
+ /**
548
+ * The claiming pane's working directory, when the bridge captured one. The
549
+ * bridge records it on every entry it writes; conductor reads it as the one
550
+ * discriminator it owns for telling a project's own panes apart when several
551
+ * sit in one herdr space (#1089).
552
+ */
553
+ cwd?: string;
451
554
  /**
452
555
  * The transcript the claiming pane actually writes, when the bridge captured
453
556
  * one. This is the *live* session file — herdr pins a restored pane to it
@@ -497,7 +600,7 @@ export function resolveProjectTopicId(project: ProjectConfig, alive?: (pid: numb
497
600
  if (claims.some((claim) => claim.threadId === pinned && claimIsLive(claim, alive === undefined ? undefined : killOf(alive)))) {
498
601
  return pinned;
499
602
  }
500
- const match = claimForProject(claims, project.name, alive);
603
+ const match = claimForProject(claims, project.name, alive, project.workspaceRoot);
501
604
  if (match === undefined) return pinned;
502
605
  warn(
503
606
  `escalation.telegramTopicId for ${project.name} is no longer a claimed topic; ` +
@@ -535,20 +638,36 @@ export type ProjectClaim =
535
638
  * *no* live claim carries the project's space — several live claims wearing
536
639
  * the space is an ambiguity the title cannot resolve, because the titled
537
640
  * claim may be a sibling pane, not this project (#626).
641
+ *
642
+ * One same-space ambiguity conductor settles itself (#1089): it provisioned
643
+ * both panes from its own config, so when the caller supplies the project's
644
+ * orchestrator cwd — its workspace root, against the console's
645
+ * `<stateDir>/console/<name>` — exactly one live claimant sitting there is
646
+ * identification, not a guess. Neither or both matching stays the ambiguity.
538
647
  */
539
648
  export function resolveProjectClaim(
540
649
  claims: readonly ClaimedTopic[],
541
650
  projectName: string,
542
651
  alive?: (pid: number) => boolean,
652
+ orchestratorCwd?: string,
543
653
  ): ProjectClaim {
544
654
  // Dead claims do not vote: a closed pane's row must not turn this project's
545
655
  // one live claim into an ambiguity, and must not answer by title on its own
546
- // (#987). Ambiguity among genuinely live claims is unchanged (#626).
656
+ // (#987). Ambiguity among genuinely live claims is narrowed by the one
657
+ // discriminator conductor owns (#1089), never removed (#626).
547
658
  const live = claims.filter((claim) => claimIsLive(claim, alive === undefined ? undefined : killOf(alive)));
548
659
  const bySpace = live.filter((claim) => claim.workspaceLabel === projectName);
549
660
  if (bySpace.length > 0) {
550
- return bySpace.length === 1
551
- ? { kind: "match", claim: bySpace[0]! }
661
+ if (bySpace.length === 1) return { kind: "match", claim: bySpace[0]! };
662
+ // Conductor provisions every same-space pane from its own config: the
663
+ // orchestrator pane's cwd is the project's workspace root, the console's
664
+ // is `<stateDir>/console/<name>` (#1089). Exactly one claimant sitting at
665
+ // the orchestrator cwd is identification; neither or both matching stays
666
+ // the coin toss #626 refuses to spin.
667
+ const orchestrator =
668
+ orchestratorCwd === undefined ? [] : bySpace.filter((claim) => claim.cwd === orchestratorCwd);
669
+ return orchestrator.length === 1
670
+ ? { kind: "match", claim: orchestrator[0]! }
552
671
  : { kind: "ambiguous", claimants: bySpace };
553
672
  }
554
673
  const byTitle = live.filter((claim) => claim.name === projectName);
@@ -571,8 +690,9 @@ export function claimForProject(
571
690
  claims: readonly ClaimedTopic[],
572
691
  projectName: string,
573
692
  alive?: (pid: number) => boolean,
693
+ orchestratorCwd?: string,
574
694
  ): ClaimedTopic | undefined {
575
- const match = resolveProjectClaim(claims, projectName, alive);
695
+ const match = resolveProjectClaim(claims, projectName, alive, orchestratorCwd);
576
696
  return match.kind === "match" ? match.claim : undefined;
577
697
  }
578
698
 
@@ -595,7 +715,7 @@ export function resolveClaimedSessionFile(
595
715
  ): string | undefined {
596
716
  const result = claimedTelegramTopics();
597
717
  if (result.kind !== "ok" || result.claims.length === 0) return undefined;
598
- return claimForProject(result.claims, project.name, alive)?.sessionFile;
718
+ return claimForProject(result.claims, project.name, alive, project.workspaceRoot)?.sessionFile;
599
719
  }
600
720
 
601
721
  /**
@@ -699,6 +819,9 @@ export function claimedTelegramTopics(stateDir?: string): ClaimedTopicsResult {
699
819
  if ("sessionFile" in entry && typeof entry.sessionFile !== "string") {
700
820
  return { kind: "unavailable", problem: `${registry}: claim "${id}" has a non-string sessionFile` };
701
821
  }
822
+ if ("cwd" in entry && typeof entry.cwd !== "string") {
823
+ return { kind: "unavailable", problem: `${registry}: claim "${id}" has a non-string cwd` };
824
+ }
702
825
  if ("pid" in entry && (typeof entry.pid !== "number" || !Number.isFinite(entry.pid))) {
703
826
  return { kind: "unavailable", problem: `${registry}: claim "${id}" has a non-numeric pid` };
704
827
  }
@@ -706,14 +829,17 @@ export function claimedTelegramTopics(stateDir?: string): ClaimedTopicsResult {
706
829
  let workspaceLabel: string | undefined;
707
830
  let sessionFile: string | undefined;
708
831
  let pid: number | undefined;
832
+ let cwd: string | undefined;
709
833
  if ("name" in entry && entry.name.trim() !== "") name = entry.name.trim();
710
834
  if ("workspaceLabel" in entry && entry.workspaceLabel.trim() !== "") workspaceLabel = entry.workspaceLabel.trim();
711
835
  if ("sessionFile" in entry && entry.sessionFile.trim() !== "") sessionFile = entry.sessionFile.trim();
836
+ if ("cwd" in entry && entry.cwd.trim() !== "") cwd = entry.cwd.trim();
712
837
  if ("pid" in entry) pid = entry.pid;
713
838
  const claim: ClaimedTopic = { threadId, name };
714
839
  if (workspaceLabel !== undefined) claim.workspaceLabel = workspaceLabel;
715
840
  if (sessionFile !== undefined) claim.sessionFile = sessionFile;
716
841
  if (pid !== undefined) claim.pid = pid;
842
+ if (cwd !== undefined) claim.cwd = cwd;
717
843
  out.push(claim);
718
844
  }
719
845
  return { kind: "ok", claims: out };
@@ -1213,17 +1339,32 @@ function sessionWithinScan(sessionFile: string, dirs: readonly string[]): boolea
1213
1339
  * interleave with each other.
1214
1340
  *
1215
1341
  * Optional `topicId` pins the message to a forum topic (`message_thread_id`).
1216
- * A definitive missing-thread reject retries once as a flat chat and warns, so
1217
- * a deleted topic degrades instead of silently losing the page (#318). The
1218
- * flat retry re-sends the whole split: parts already accepted went into a
1219
- * thread Telegram has just told us is gone, so nothing is readable there that
1220
- * a flat resend would duplicate.
1342
+ * A definitive missing-thread reject degrades in two ordered steps (#1094):
1343
+ * first to the project's one live claim, when `project` names who the send
1344
+ * answers to and the registry identifies exactly one (#1089) a resend into
1345
+ * the project's own current topic. Only when no topic can be resolved does it
1346
+ * retry as a flat chat, prepending a notice that names the stale pin and the
1347
+ * project, and recording a mis-route row for doctor/status (#318's silent
1348
+ * shape). Every degrade re-sends the whole split: parts already accepted went
1349
+ * into a thread Telegram has just told us is gone, so nothing readable is
1350
+ * ever duplicated.
1221
1351
  */
1222
1352
  export async function sendTelegram(
1223
1353
  token: string,
1224
1354
  chatId: string,
1225
1355
  text: string,
1226
- opts?: { topicId?: number },
1356
+ opts?: {
1357
+ topicId?: number;
1358
+ /**
1359
+ * Who this send answers to, when it rides a pinned topic (#1094): the
1360
+ * project whose escalation, report, or challenge it carries. Only a
1361
+ * caller that resolved `topicId` for a project passes it, and that pair
1362
+ * is what lets a missing-thread fallback prefer the project's one live
1363
+ * claim, label the flat delivery with the project's name, and record the
1364
+ * mis-route where the operator looks.
1365
+ */
1366
+ project?: { name: string; workspaceRoot?: string };
1367
+ },
1227
1368
  ): Promise<number[]> {
1228
1369
  const topicId =
1229
1370
  opts?.topicId !== undefined && Number.isFinite(opts.topicId) && Number.isSafeInteger(opts.topicId)
@@ -1244,7 +1385,38 @@ export async function sendTelegram(
1244
1385
  warn(
1245
1386
  `escalation.telegramTopicId=${topicId} is stale (${err.message}); retrying flat chat`,
1246
1387
  );
1247
- return await postTelegramParts(token, chatId, parts, undefined);
1388
+ // #318's flat retry is the last resort, not the first (#1094). When the
1389
+ // claim registry names exactly one live claim for this project — #1089's
1390
+ // cwd discriminator included — the resend follows it: a page landing in
1391
+ // the project's own current topic is not a mis-route at all. A
1392
+ // substitution that itself fails falls through to the labelled flat send
1393
+ // below; delivery outranks tidiness.
1394
+ const project = opts?.project;
1395
+ if (project !== undefined) {
1396
+ try {
1397
+ const registry = claimedTelegramTopics();
1398
+ if (registry.kind === "ok") {
1399
+ const claim = claimForProject(registry.claims, project.name, undefined, project.workspaceRoot);
1400
+ if (claim !== undefined && claim.threadId !== topicId) {
1401
+ warn(`following omp-telegram's live "${project.name}" claim instead of the flat chat`);
1402
+ return await postTelegramParts(token, chatId, parts, claim.threadId);
1403
+ }
1404
+ }
1405
+ } catch (subErr) {
1406
+ warn(
1407
+ `live-claim substitution failed (${subErr instanceof Error ? subErr.message : String(subErr)}); delivering flat`,
1408
+ );
1409
+ }
1410
+ }
1411
+ // Flat, but never silent (#1094): the delivered text names the stale pin
1412
+ // and the project, so the wrong audience knows what it is reading, and
1413
+ // doctor/status gain the evidence against this pin. The record is written
1414
+ // only after the send resolves — a flat send that threw delivered
1415
+ // nothing, and the next tick retries it as a first attempt.
1416
+ const labelled = `${staleTopicNotice(topicId, project?.name)}\n\n${text}`;
1417
+ const ids = await postTelegramParts(token, chatId, telegramTextParts(labelled), undefined);
1418
+ if (project?.name !== undefined) recordTelegramMisroute(project.name, topicId);
1419
+ return ids;
1248
1420
  }
1249
1421
  }
1250
1422
 
@@ -15,7 +15,7 @@
15
15
  * green test per class" possible without a daemon, a tracker or a network.
16
16
  */
17
17
 
18
- import type { Caps, FailureClass, RecoveryAction, RunRecord } from "./types.ts";
18
+ import { DEFAULT_CAPS, type Caps, type FailureClass, type RecoveryAction, type RunRecord } from "./types.ts";
19
19
 
20
20
  /** Facts the caller fetched, each only for the rows that need it.
21
21
  *
@@ -202,6 +202,45 @@ export const SPINNING_CAP_CLASSES: readonly FailureClass[] = [
202
202
  "wall-clock-cap-spinning",
203
203
  ];
204
204
 
205
+ /** How much of the wall-clock ceiling the default stall window may take (#1086). */
206
+ export const WALL_CLOCK_STALL_SHARE = 3;
207
+
208
+ /**
209
+ * The transcript-silence window a stalled run is settled at (#1086): the
210
+ * configured `workerStallSilenceMs`, or a third of the wall-clock ceiling when
211
+ * it is null or unconfigured. Shared by the daemon's progress watch (which
212
+ * settles at this threshold) and the classifier (which recognises a row killed
213
+ * at it), so the two can never disagree about what a stall is.
214
+ */
215
+ export function stallSilenceMs(
216
+ caps:
217
+ | {
218
+ workerStallSilenceMs?: number | null;
219
+ workerWallClockMs: number;
220
+ }
221
+ | undefined,
222
+ ): number {
223
+ const wallClock = caps?.workerWallClockMs ?? DEFAULT_CAPS.workerWallClockMs;
224
+ return caps?.workerStallSilenceMs ?? Math.floor(wallClock / WALL_CLOCK_STALL_SHARE);
225
+ }
226
+
227
+ /**
228
+ * How long a run sat silent before its stall kill, from the row's own facts
229
+ * (#1086). Defined only when the progress watch had observed the run AND the
230
+ * silence reached the configured window — the exact predicate that keeps every
231
+ * other killed-under-ceiling row (a drain, an operator stop) reading as
232
+ * `admin-kill`.
233
+ */
234
+ export function stallSilence(
235
+ run: RunRecord,
236
+ caps: { workerStallSilenceMs?: number | null; workerWallClockMs: number } | undefined,
237
+ ): number | undefined {
238
+ if (run.lastProgressAt === undefined || run.endedAt === undefined) return undefined;
239
+ const silent = run.endedAt - run.lastProgressAt;
240
+ if (silent < stallSilenceMs(caps)) return undefined;
241
+ return silent;
242
+ }
243
+
205
244
  /**
206
245
  * A stable fingerprint of the infrastructure signature list (#638). The
207
246
  * historical reconciliation persists a per-project review cursor stamped with
@@ -728,10 +767,100 @@ function firstReportContentLine(report: string | undefined): string | undefined
728
767
  return undefined;
729
768
  }
730
769
 
770
+ /**
771
+ * What the wall-clock ceiling actually buys on this fleet right now (#1063).
772
+ *
773
+ * `caps.workerMaxTurns` and `caps.workerWallClockMs` are independent config
774
+ * values, but they meet at an implicit pace: 0.5 minutes per turn at the
775
+ * shipped defaults. A model slower than that can never reach the turn ceiling,
776
+ * so the budget a groomer sized a slice against is fiction — the run dies of
777
+ * the clock first. This observation is derived from *completed* runs (never
778
+ * from the configured numbers, whose ratio is a constant and tells nobody
779
+ * anything), so `status` and the settlement report can name the real number.
780
+ */
781
+ export interface ObservedTurnBudget {
782
+ /** Aggregate elapsed minutes ÷ aggregate turns across the sample. */
783
+ minutesPerTurn: number;
784
+ /** Completed runs the figure was derived from. */
785
+ sampleSize: number;
786
+ /** ⌊wall clock ÷ minutesPerTurn⌋ — what the clock buys at that pace. */
787
+ effectiveTurns: number;
788
+ }
789
+
790
+ /** How many completed runs the effective-budget sample spans. */
791
+ export const EFFECTIVE_BUDGET_SAMPLE_RUNS = 20;
792
+
793
+ /** A wall-clock kill below this share of its turn ceiling is a latency
794
+ * verdict; at or above it, both ceilings were nearly exhausted and size is
795
+ * the honest reading (#1063). */
796
+ export const WALL_CLOCK_SIZE_VERDICT_SHARE = 0.7;
797
+
798
+ /** Strips omp selector suffixes (`:thinking`, `:max`) so only the model path
799
+ * is compared. */
800
+ function modelBase(selector: string): string {
801
+ const colon = selector.indexOf(":");
802
+ return colon === -1 ? selector : selector.slice(0, colon);
803
+ }
804
+
805
+ /**
806
+ * Whether one model attribution names the same model as the configured
807
+ * selector, compared as path suffixes: config says
808
+ * `openrouter/stealth/ox-alpha:max` while the transcript's resolved model
809
+ * reads `stealth/ox-alpha`, and strict equality would empty every sample.
810
+ * Role aliases (`@slow`) have no concrete spelling to resolve to from here, so
811
+ * they match themselves only — no sample, rather than an invented one.
812
+ */
813
+ function sameModel(configured: string, observed: string): boolean {
814
+ if (configured.startsWith("@") || observed.startsWith("@")) return configured === observed;
815
+ const c = modelBase(configured);
816
+ const o = modelBase(observed);
817
+ return c === o || c.endsWith(`/${o}`) || o.endsWith(`/${c}`);
818
+ }
819
+
820
+ /**
821
+ * Derives {@link ObservedTurnBudget} from completed-run rows, newest-first
822
+ * callers' ordering irrelevant. Rows without turns or elapsed time carry no
823
+ * pace and are skipped; when a configured worker model is named, rows written
824
+ * by another model are skipped with them (`resolvedModel` preferred — what
825
+ * actually wrote the messages — falling back to the dispatch record).
826
+ * `undefined` when nothing qualifying remains: absence is the honest answer,
827
+ * never a guess from the configured constants.
828
+ */
829
+ export function observeTurnBudget(
830
+ samples: readonly Pick<RunRecord, "id" | "turns" | "startedAt" | "endedAt" | "model" | "resolvedModel">[],
831
+ caps: Pick<Caps, "workerWallClockMs">,
832
+ configuredModel?: string,
833
+ excludeRunId?: string,
834
+ ): ObservedTurnBudget | undefined {
835
+ let minutes = 0;
836
+ let turns = 0;
837
+ let used = 0;
838
+ for (const s of samples) {
839
+ if (s.id === excludeRunId) continue;
840
+ if (s.turns <= 0 || s.endedAt === undefined || s.endedAt <= s.startedAt) continue;
841
+ if (configuredModel !== undefined) {
842
+ const attribution = s.resolvedModel ?? s.model;
843
+ if (attribution === undefined || !sameModel(configuredModel, attribution)) continue;
844
+ }
845
+ minutes += (s.endedAt - s.startedAt) / 60_000;
846
+ turns += s.turns;
847
+ used += 1;
848
+ }
849
+ if (used === 0 || turns <= 0) return undefined;
850
+ const minutesPerTurn = minutes / turns;
851
+ if (!Number.isFinite(minutesPerTurn) || minutesPerTurn <= 0) return undefined;
852
+ return {
853
+ minutesPerTurn,
854
+ sampleSize: used,
855
+ effectiveTurns: Math.floor(caps.workerWallClockMs / 60_000 / minutesPerTurn),
856
+ };
857
+ }
858
+
731
859
  export function classifyRun(
732
860
  run: RunRecord,
733
861
  facts: ClassifyFacts,
734
- caps?: Pick<Caps, "workerWallClockMs">,
862
+ caps?: Pick<Caps, "workerWallClockMs"> & { workerStallSilenceMs?: number | null },
863
+ observedBudget?: ObservedTurnBudget,
735
864
  ): Classification {
736
865
  const hasArtifacts = run.prUrl !== undefined || run.headSha !== undefined || run.salvageSha !== undefined;
737
866
  const providerError =
@@ -932,6 +1061,30 @@ export function classifyRun(
932
1061
  }
933
1062
 
934
1063
  if (run.state === "killed") {
1064
+ // A stall the daemon settled (#1086), recognised from the row's own facts
1065
+ // — the watch observed a transcript write instant, and the silence to the
1066
+ // kill reached the configured window. Checked before the cap branches on
1067
+ // purpose: a hung session's turns never move, so this row is under its
1068
+ // ceilings, but even if a config change made it *reach* one mid-hang, the
1069
+ // silence is the operative fact about why it died. The same split as the
1070
+ // caps: artifacts continue, nothing shows escalates.
1071
+ const silentMs = stallSilence(run, caps);
1072
+ if (silentMs !== undefined) {
1073
+ const minutes = Math.round(silentMs / 60_000);
1074
+ const silent =
1075
+ minutes < 60 ? `${minutes}m` : `${Math.floor(minutes / 60)}h${minutes % 60 === 0 ? "" : `${minutes % 60}m`}`;
1076
+ return hasArtifacts
1077
+ ? {
1078
+ cls: "progress-stall",
1079
+ recovery: "continue",
1080
+ evidence: `transcript silent ${silent} at turn ${run.turns}/${run.maxTurns}; the session never came back — work to continue from (${run.prUrl ?? run.salvageSha ?? run.headSha})`,
1081
+ }
1082
+ : {
1083
+ cls: "progress-stall",
1084
+ recovery: "escalate",
1085
+ evidence: `transcript silent ${silent} at turn ${run.turns}/${run.maxTurns}, no PR, no commits — $${run.spendUsd.toFixed(2)} spent`,
1086
+ };
1087
+ }
935
1088
  if (run.turns >= run.maxTurns) {
936
1089
  return hasArtifacts
937
1090
  ? {
@@ -950,21 +1103,40 @@ export function classifyRun(
950
1103
  // worker slot until the ticks ran out (#490). The same split as the turns
951
1104
  // path: artifacts mean the next attempt resumes from real work; nothing to
952
1105
  // show means a human re-scopes, naming the time and spend consumed.
1106
+ //
1107
+ // #1063: which ceiling was ever reachable? The two caps meet at an implied
1108
+ // pace (wall clock ÷ max turns), so a run killed at the clock far under
1109
+ // its turn budget died of *latency* — re-attempting it as an oversized
1110
+ // slice spends a continuation on the wrong diagnosis. The evidence names
1111
+ // the run's own pace, the latency/size verdict the share of the turn
1112
+ // ceiling implies, and — when the caller supplied one — the effective
1113
+ // turn budget observed across completed runs on this model.
953
1114
  const wallClockCap = caps?.workerWallClockMs;
954
1115
  const wallClockElapsed =
955
1116
  run.endedAt === undefined ? undefined : run.endedAt - run.startedAt;
956
1117
  if (wallClockCap !== undefined && wallClockElapsed !== undefined && wallClockElapsed >= wallClockCap) {
957
1118
  const clock = `${Math.round(wallClockElapsed / 60_000)}m of ${Math.round(wallClockCap / 60_000)}m`;
1119
+ const pace =
1120
+ run.turns > 0 ? ` at ${((wallClockElapsed / 60_000) / run.turns).toFixed(2)} min/turn` : "";
1121
+ const share = run.maxTurns > 0 ? run.turns / run.maxTurns : 1;
1122
+ const verdict =
1123
+ share < WALL_CLOCK_SIZE_VERDICT_SHARE
1124
+ ? `latency verdict: ${run.turns}/${run.maxTurns} turns before the clock — the slice fits, the model is slow`
1125
+ : `size verdict: ${run.turns}/${run.maxTurns} turns — both ceilings nearly exhausted`;
1126
+ const observed =
1127
+ observedBudget === undefined
1128
+ ? ""
1129
+ : `; effective ~${observedBudget.effectiveTurns} turns at ${observedBudget.minutesPerTurn.toFixed(2)} min/turn (last ${observedBudget.sampleSize} runs)`;
958
1130
  return hasArtifacts
959
1131
  ? {
960
1132
  cls: "wall-clock-cap-progress",
961
1133
  recovery: "continue",
962
- evidence: `wall clock ${clock} with work to continue from (${run.prUrl ?? run.salvageSha ?? run.headSha})`,
1134
+ evidence: `wall clock ${clock}${pace} ${verdict}${observed}; work to continue from (${run.prUrl ?? run.salvageSha ?? run.headSha})`,
963
1135
  }
964
1136
  : {
965
1137
  cls: "wall-clock-cap-spinning",
966
1138
  recovery: "escalate",
967
- evidence: `wall clock ${clock}, no PR, no commits — $${run.spendUsd.toFixed(2)} spent`,
1139
+ evidence: `wall clock ${clock}${pace} — ${verdict}${observed}; no PR, no commits — $${run.spendUsd.toFixed(2)} spent`,
968
1140
  };
969
1141
  }
970
1142
  // Below its own ceilings, so nothing this worker did ended it: a daemon