omp-conductor 0.20.0 → 0.20.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -62,15 +62,15 @@ export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promi
62
62
  );
63
63
  break;
64
64
  }
65
- // A capped/failed run has no pushed-green settle sweep keeping its row
66
- // honest (#795 review round 1): nothing transitions a `failed` / `killed`
67
- // row when its PR merges or closes, so a PR that changed after the verb
68
- // recorded the round would otherwise be claimed and a worker resumed
69
- // against a dead PR. The settled-green origin keeps its own interlock
70
- // (the settle sweep flips the row and the claim below refuses it), so
71
- // only terminal-origin rounds re-read the reviewed PR fact here: the
72
- // round dispatches only while the PR is still open at the exact reviewed
73
- // head.
65
+ // A capped/failed/stopped run has no pushed-green settle sweep keeping its
66
+ // row honest (#795 review round 1, #1101): nothing transitions a `failed`
67
+ // / `killed` / `stopped` row when its PR merges or closes, so a PR that
68
+ // changed after the verb recorded the round would otherwise be claimed and
69
+ // a worker resumed against a dead PR. The settled-green origin keeps its
70
+ // own interlock (the settle sweep flips the row and the claim below
71
+ // refuses it), so every other revisable origin re-reads the reviewed PR
72
+ // fact here: the round dispatches only while the PR is still open at the
73
+ // exact reviewed head.
74
74
  //
75
75
  // The skip decision is decisive-fact only (review round 2): a definitively
76
76
  // missing PR (`GhPrMissingError` — a corroborated 404, #779), a PR that
@@ -86,7 +86,7 @@ export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promi
86
86
  // #1047's launch decision rests on are only readable here.
87
87
  const originRun = d.store.getRun(revision.runId);
88
88
  const origin = originRun?.state;
89
- if (origin === "failed" || origin === "killed") {
89
+ if (origin === "failed" || origin === "killed" || origin === "stopped") {
90
90
  let prState: PrState | undefined;
91
91
  try {
92
92
  prState = await d.tracker.prState(revision.prUrl);
@@ -161,7 +161,7 @@ export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promi
161
161
  if (!d.store.claimRunForReview(revision.runId)) {
162
162
  d.store.settleReviewRevision(revision.id, "skipped", Date.now());
163
163
  log(
164
- `#${revision.issue} review round ${revision.round} skipped: run ${revision.runId} is no longer pushed-green`,
164
+ `#${revision.issue} review round ${revision.round} skipped: run ${revision.runId} is ${origin ?? "gone"} — no longer reclaimable for review`,
165
165
  );
166
166
  continue;
167
167
  }
@@ -33,7 +33,7 @@ import { reconcileOrphanedRuns } from "../settlement.ts";
33
33
  import { dbPath, openStore, utcDay } from "../store.ts";
34
34
  import { checkTelegramFreshness } from "../telegram-freshness.ts";
35
35
  import { GraphqlBreaker, makeTracker } from "../tracker/github.ts";
36
- import { RELEASE_SHAPES, type ProjectConfig, type ResolvedGrants, type RunRecord } from "../types.ts";
36
+ import { RELEASE_SHAPES, type Escalation, type ProjectConfig, type ResolvedGrants, type RunRecord } from "../types.ts";
37
37
  import { runCommand } from "../upgrade-verify.ts";
38
38
  import { inspectSurfaces } from "../upgrade.ts";
39
39
  import { sharedUsageSource } from "../usage.ts";
@@ -203,6 +203,54 @@ export async function runDispatchLoop(o: DispatchLoopOptions): Promise<void> {
203
203
  }
204
204
  }
205
205
 
206
+ /** What the daemon knows about its tier-1 orchestrator at diversion time. */
207
+ export interface Tier1DiversionFacts {
208
+ /** The project's configured tier-1 transport is the external orchestrator
209
+ * session: issue comments are the *configured* destination, not a
210
+ * fallback. */
211
+ external: boolean;
212
+ /** Why the daemon-owned orchestrator session failed to start, when it did. */
213
+ startError?: string;
214
+ /** Whether the daemon-owned orchestrator session is alive right now
215
+ * (`OrchestratorHandle.alive()` — event-driven, false once the session
216
+ * exited). */
217
+ alive: boolean;
218
+ /** Whether the daemon-owned orchestrator session is mid-turn right now
219
+ * (`OrchestratorHandle.busy()`). */
220
+ busy: boolean;
221
+ }
222
+
223
+ /**
224
+ * The one-line record of a tier-1 escalation that landed on the issue-comment
225
+ * fallback instead of the orchestrator injection (#1068).
226
+ *
227
+ * "while the orchestrator was down" is reserved for what it actually means —
228
+ * a session that failed to start or one that has exited. The #1062 shape was
229
+ * neither: the orchestrator was mid-tick, its session did not take the
230
+ * injection, and a reader of "diverted … while the orchestrator was down"
231
+ * went looking for a dead session that did not exist. When the session is
232
+ * alive the line says where the escalation was delivered and why it was not
233
+ * injected; when the project runs an external orchestrator the comments *are*
234
+ * the configured transport and the line says so instead of calling it a
235
+ * diversion.
236
+ */
237
+ export function tier1DiversionLine(e: Escalation, facts: Tier1DiversionFacts): string {
238
+ const ref = escalationIssueRef(e.issue);
239
+ if (facts.external) {
240
+ return `orchestrator: tier-1 escalation on ${ref} posted as an issue comment (external orchestrator — the configured tier-1 transport)`;
241
+ }
242
+ if (facts.startError !== undefined || !facts.alive) {
243
+ return (
244
+ `orchestrator: tier-1 escalation on ${ref} diverted to issue comments while the orchestrator was down` +
245
+ (facts.startError !== undefined ? ` (start failed: ${facts.startError})` : " (session exited)")
246
+ );
247
+ }
248
+ return (
249
+ `orchestrator: tier-1 escalation on ${ref} posted as an issue comment — the orchestrator session ` +
250
+ `is alive but did not take the injection${facts.busy ? " (mid-turn)" : ""}`
251
+ );
252
+ }
253
+
206
254
  export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
207
255
  // A `--once` tick is still a dispatcher: it settles rows, projects labels,
208
256
  // admits and launches workers, so it must hold the same exclusive daemon/
@@ -391,8 +439,12 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
391
439
  // the outage diverted (#288).
392
440
  store.bumpOrchestratorDiverted(project.name, 1);
393
441
  projectLog(
394
- `orchestrator: tier-1 escalation on ${escalationIssueRef(e.issue)} diverted to issue comments ` +
395
- `while the orchestrator was down`,
442
+ tier1DiversionLine(e, {
443
+ external: project.escalation.orchestrator === "external",
444
+ startError: orchestratorStartError,
445
+ alive: orchestrator?.alive() ?? false,
446
+ busy: orchestrator?.busy() ?? false,
447
+ }),
396
448
  );
397
449
  },
398
450
  );
@@ -16,6 +16,7 @@
16
16
  */
17
17
  import { infraLogSignature, infraSignatureVersion } from "../failure-class.ts";
18
18
  import { errText, log, safeEscalate } from "../log.ts";
19
+ import { GhPrMissingError } from "../tracker/github.ts";
19
20
  import type { BaseHealth, RunRecord, SettlementFlag, WorkflowRun } from "../types.ts";
20
21
  import { cleanupRetainedWorktree, mirrorPathFor, type RetainedWorktreeCleanup } from "../worktree.ts";
21
22
  import { githubRepo, type Deps, type RetainedCleanupCursor } from "./deps.ts";
@@ -393,8 +394,24 @@ export async function cleanupRetainedRuns(
393
394
  terminal = issue === "closed";
394
395
  }
395
396
  } catch (err) {
396
- log(`#${run.issue} retained cleanup deferred: tracker state failed (${errText(err)})`);
397
- continue;
397
+ // The tracker throws exactly one classified error: `GhPrMissingError`,
398
+ // an individual REST 404 it corroborated with a same-repository
399
+ // pulls-list read, so the claimed PR definitively does not exist
400
+ // (#779). That is a fact, not "could not tell": retrying can never
401
+ // conjure the PR, so the missing PR is terminal and the retained row
402
+ // settles in this same pass like a closed one (#1065). The row's
403
+ // phantom URL is cleared with the settlement so a repeat pass neither
404
+ // asks GitHub about it again nor logs about it again. Everything else —
405
+ // a rate limit, a 5xx, a lost network — keeps the deferred retry below,
406
+ // unchanged.
407
+ if (err instanceof GhPrMissingError) {
408
+ store.updateRun(run.id, { prUrl: null });
409
+ log(`#${run.issue} retained cleanup settled: PR ${run.prUrl} does not exist`);
410
+ terminal = true;
411
+ } else {
412
+ log(`#${run.issue} retained cleanup deferred: tracker state failed (${errText(err)})`);
413
+ continue;
414
+ }
398
415
  }
399
416
  if (!terminal) continue;
400
417
 
@@ -14,12 +14,15 @@
14
14
  * pass into the dispatch path's import graph, for two functions whose subject —
15
15
  * "make the orchestrator look at this now" — is already this module's.
16
16
  */
17
- import { existsSync, readFileSync } from "node:fs";
17
+ import { existsSync, readFileSync, statSync } from "node:fs";
18
18
  import { join } from "node:path";
19
19
  import { stateDir } from "../config.ts";
20
+ import { stallSilenceMs } from "../failure-class.ts";
20
21
  import { resolvePaneHaltPath, stopConductorPane } from "../fleet.ts";
21
22
  import { errText, log, safeEscalate } from "../log.ts";
22
23
  import { STALL_MARKER_FILE, readTickRequestReason, requestImmediateTick, resolveTickConfigCwd } from "../orchestrator-tick.ts";
24
+ import { LIVE_STATES } from "../store.ts";
25
+ import { wakeDispatch } from "../wake.ts";
23
26
  import { NO_ISSUE, type Deps, type StallGate } from "./deps.ts";
24
27
  import { markPaged } from "./integrity.ts";
25
28
 
@@ -436,3 +439,143 @@ export function wakeOrchestratorForBlockedRun(
436
439
  );
437
440
  }
438
441
  }
442
+
443
+
444
+ // ----------------------------------------------------------------- worker progress
445
+
446
+ /**
447
+ * Watching live workers for silence, and settling a run that stopped making
448
+ * progress (#1086).
449
+ *
450
+ * The only bounds a worker's own session enforces are the turn ceiling and the
451
+ * wall clock — neither fires until the whole budget is gone, so a session that
452
+ * hangs at minute 44 keeps its slot for another 46 minutes and then dies as a
453
+ * `wall-clock-cap-*` verdict that reads like an oversized slice about work that
454
+ * may already be merged. The signal that separates "thinking" from "gone" is
455
+ * already on disk: a live session writes its transcript continuously, so the
456
+ * file's own mtime is when the worker was last demonstrably alive.
457
+ *
458
+ * Deliberately transcript-only: the row's turn counter rides in the evidence,
459
+ * but it is never the trigger — one turn can legitimately run for minutes on a
460
+ * slow provider, and this fleet has measured 1.5 min/turn.
461
+ *
462
+ * A stall whose PR is already pushed and green settles `pushed-green` — the
463
+ * exact state any delivered run holds — so the settle sweep owns the merge and
464
+ * no continuation is charged for work that landed (#1086). Every other stall
465
+ * settles `killed` over the observed facts; the classifier turns those into
466
+ * `progress-stall`, whose recovery splits on artifacts exactly as the caps do.
467
+ */
468
+ export async function watchWorkerProgress(
469
+ d: Pick<Deps, "project" | "caps" | "store" | "tracker" | "workerControls" | "escalate">,
470
+ now = Date.now(),
471
+ ): Promise<void> {
472
+ const thresholdMs = stallSilenceMs(d.caps);
473
+ // A paused worker banks its wall clock and goes quiet on purpose (#938):
474
+ // silence under pause is the pause working, never a hang.
475
+ const pausedIssues = new Set(
476
+ d.workerControls
477
+ .snapshot(d.project.name)
478
+ .filter((w) => w.phase === "pausing" || w.phase === "paused")
479
+ .map((w) => w.issue),
480
+ );
481
+ for (const run of d.store.liveRuns(d.project.name)) {
482
+ if (pausedIssues.has(run.issue)) continue;
483
+ if (run.sessionFile === undefined) continue;
484
+ let wroteAt: number;
485
+ try {
486
+ wroteAt = Math.trunc(statSync(run.sessionFile).mtimeMs);
487
+ } catch {
488
+ // No transcript yet (claimed, session opening) or none readable: nothing
489
+ // to observe, which is never evidence of a stall.
490
+ continue;
491
+ }
492
+ // Record before judging: `status` renders this instant as the run's
493
+ // silence interval whether or not the threshold fires (#1086).
494
+ if (run.lastProgressAt !== wroteAt) {
495
+ d.store.updateRun(run.id, { lastProgressAt: wroteAt });
496
+ }
497
+ const silentMs = now - wroteAt;
498
+ if (silentMs < thresholdMs) continue;
499
+
500
+ // Re-read before settling: the session can finish between the stat above
501
+ // and here, and a terminal row must never be re-settled over.
502
+ const current = d.store.getRun(run.id);
503
+ if (current === undefined || !LIVE_STATES.includes(current.state)) continue;
504
+ const minutes = Math.round(silentMs / 60_000);
505
+ const silent =
506
+ minutes < 60 ? `${minutes}m` : `${Math.floor(minutes / 60)}h${minutes % 60 === 0 ? "" : `${minutes % 60}m`}`;
507
+ const why = `transcript silent ${silent} at turn ${current.turns}/${current.maxTurns}`;
508
+ log(`#${current.issue} ${why} — settling (silence window ${Math.round(thresholdMs / 60_000)}m)`);
509
+
510
+ // Best-effort abort through the run-control surface, so a session that can
511
+ // still answer exits through its own settlement path. One that cannot is
512
+ // exactly the case this pass exists for: the row settles below regardless,
513
+ // and the re-read guard keeps whichever writer lands second out.
514
+ try {
515
+ void d.workerControls.stop(d.project.name, current.issue, `progress stall: ${why}`).catch(() => {});
516
+ } catch {
517
+ // No live control for this run — the settle below is the only writer.
518
+ }
519
+
520
+ // Work that already landed is delivered work — but only verified green
521
+ // counts: landing `pushed-green` puts the row exactly where any delivered
522
+ // run sits, under the settle sweep's merge watch, with nothing charged.
523
+ let deliveredGreen = false;
524
+ if (current.prUrl !== undefined && current.headSha !== undefined) {
525
+ try {
526
+ deliveredGreen = (await d.tracker.verifyPr(current.prUrl, current.headSha))?.status === "green";
527
+ } catch {
528
+ deliveredGreen = false;
529
+ }
530
+ }
531
+ if (deliveredGreen) {
532
+ d.store.updateRun(current.id, { state: "pushed-green", endedAt: now });
533
+ log(`#${current.issue} stalled but ${current.prUrl} was already green — settled as delivered`);
534
+ await safeEscalate(d, {
535
+ tier: 1,
536
+ project: d.project.name,
537
+ issue: current.issue,
538
+ runId: current.id,
539
+ summary: `#${current.issue} went silent (${why}) with its PR already green — settled as delivered`,
540
+ detail: [
541
+ current.prUrl ?? "(no PR URL)",
542
+ `Branch ${current.branch} at ${current.headSha}.`,
543
+ `Session: ${current.sessionFile ?? "(no transcript)"}`,
544
+ "",
545
+ "The worker never reported, so there is no settlement narrative — the PR's diff is the record.",
546
+ "The slot is released; the settle sweep watches the merge from here.",
547
+ ].join("\n"),
548
+ });
549
+ await wakeDispatchLogged(d.project.name);
550
+ continue;
551
+ }
552
+
553
+ d.store.updateRun(current.id, {
554
+ state: "killed",
555
+ endedAt: now,
556
+ lastError: `progress-stall: ${why}`,
557
+ });
558
+ await safeEscalate(d, {
559
+ tier: 1,
560
+ project: d.project.name,
561
+ issue: current.issue,
562
+ runId: current.id,
563
+ summary: `[progress-stall] #${current.issue} attempt ${current.attempt}: ${why}`,
564
+ detail: [
565
+ `${current.prUrl === undefined ? "No PR" : `PR ${current.prUrl} not verified green`} — the classifier decides continue vs escalate from what attempt ${current.attempt} left.`,
566
+ `Branch ${current.branch}; worktree ${current.worktree === "" ? "(none)" : current.worktree}.`,
567
+ `Session: ${current.sessionFile ?? "(no transcript)"}`,
568
+ ].join("\n"),
569
+ });
570
+ await wakeDispatchLogged(d.project.name);
571
+ }
572
+ }
573
+
574
+ /** Best-effort dispatch wake once a stall settle freed its slot (#1086). */
575
+ async function wakeDispatchLogged(projectName: string): Promise<void> {
576
+ try {
577
+ log(`dispatch wake after stall settle: ${await wakeDispatch(projectName)}`);
578
+ } catch (err) {
579
+ log(`dispatch wake after stall settle failed: ${errText(err)}`);
580
+ }
581
+ }
@@ -42,13 +42,13 @@ import { writeAdmissionAck } from "./ack.ts";
42
42
  import { dispatchAdmissions, summarizeDispatch, summarizeHeldPass, type WorkerPool } from "./admission-pass.ts";
43
43
  import { NO_ISSUE, PACKAGE_SRC_DIR, UNROUTABLE_TEXT, type Deps } from "./deps.ts";
44
44
  import { handleIssue } from "./dispatch.ts";
45
- import { cancelDrain, consumeDrain } from "./drain.ts";
45
+ import { consumeDrain, markDrained } from "./drain.ts";
46
46
  import { dispatchToSpecGrooming } from "./groom-pass.ts";
47
47
  import { INTEGRITY_SAMPLE, checkIntegrity, markPaged, packageManifest } from "./integrity.ts";
48
48
  import { reconcilePanes } from "./panes.ts";
49
49
  import { applyAdjudicationDispositions, dispatchReviewAdjudications, dispatchReviewRevisions } from "./review.ts";
50
50
  import { cleanupRetainedRuns, watchBaseHealth, watchMergedBase } from "./settle-pass.ts";
51
- import { wakeOrchestratorForMetConditions, watchOrchestrator } from "./supervision.ts";
51
+ import { wakeOrchestratorForMetConditions, watchOrchestrator, watchWorkerProgress } from "./supervision.ts";
52
52
 
53
53
  /**
54
54
  * Record what this host has installed, once per dispatch pass (#919).
@@ -388,6 +388,16 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
388
388
  // failure happened. A pause silences claiming, not the operator's right to
389
389
  // know their supervising session stopped reading its queue.
390
390
  await watchOrchestrator(d);
391
+ // The worker-side sibling, and above the pause gate with it: a stalled run
392
+ // holds its slot on a parked fleet exactly as hard as on a busy one, and the
393
+ // #1086 incident was an upgrade drain blocked for 26 minutes by one silent
394
+ // session. Paused workers are skipped inside the pass — their silence is
395
+ // the pause working (#938) — so this gate costs nothing there.
396
+ try {
397
+ await watchWorkerProgress(d);
398
+ } catch (err) {
399
+ log(`worker-progress sweep failed: ${errText(err)}`);
400
+ }
391
401
  // The down incident is reconciled the same place and for the same reason: a
392
402
  // session that has actually died is as much the operator's concern as one
393
403
  // that is wedged, and restarting it is the daemon's restart either way. This
@@ -584,11 +594,16 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
584
594
  // the same admission boundary — settlement above it, nothing claimed below —
585
595
  // but the intent is durable (it survives orchestrator loss) and bounded (the
586
596
  // record carries an absolute deadline, so a crash can never strand
587
- // admission). Three shapes, three behaviours:
597
+ // admission). Reaching an empty active set SUCCEEDS the drain; it does not
598
+ // end it (#1078): the release window the drain exists to create opens at
599
+ // that moment and holds claims down until the deadline — exactly the quiet
600
+ // interval a tick-driven orchestrator needs to cut a release, which the old
601
+ // eight-second auto-clear destroyed before any tick could use it. Four
602
+ // shapes, four behaviours:
588
603
  // - fresh drain with live runs → a held pass, exactly like a pause;
589
- // - fresh drain with nothing left to wait for → the drain is satisfied
590
- // and clears itself, so a completed drain never needs a second operator
591
- // action and this pass proceeds normally;
604
+ // - fresh drain whose active set just reached zero → the opening is marked
605
+ // on the record once and logged once, and this pass admits nothing;
606
+ // - fresh drain already marked drained → the same held pass, silent;
592
607
  // - malformed record → fails closed for THIS pass (it might be a fresh
593
608
  // fence we cannot read), and the same consume removed it, so it can
594
609
  // never become an unbounded permanent drain.
@@ -596,15 +611,17 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
596
611
  if (drain.kind === "active") {
597
612
  // Completion is the ACTIVE set, not the live-worker set: pushed-pending
598
613
  // and pushed-green PRs still make the `runs-settled` release gate fail, so
599
- // a drain that cleared while one remained would admit work on top of a
600
- // batch the releases still see as unfinished (#776 review #2).
601
- if (d.store.activeRuns(d.project.name).length === 0) {
602
- cancelDrain(d.project.name);
603
- log("project drain completed: no active runs remain — drain cleared");
604
- } else {
605
- d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
606
- return;
614
+ // a window declared open over a remaining run would admit work on top of
615
+ // a batch the releases still see as unfinished (#776 review #2).
616
+ if (d.store.activeRuns(d.project.name).length === 0 && drain.drain.drainedAt === undefined) {
617
+ markDrained(d.project.name);
618
+ log(
619
+ `project drain: fleet drained — release window open until ${drain.drain.expiresAt}` +
620
+ " (claims stay paused; admission resumes at the deadline or on drain cancel)",
621
+ );
607
622
  }
623
+ d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
624
+ return;
608
625
  } else if (drain.kind === "error") {
609
626
  log(`project drain record invalid (${drain.problem}) — removed; this pass admits nothing`);
610
627
  d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
@@ -23,6 +23,7 @@ import { availabilityState, type AvailabilityState } from "../availability.ts";
23
23
  import { configPath, findProject, loadConfig, resolveCaps, resolveReleaseGrants, resolveReview, stateDir } from "../config.ts";
24
24
  import { digestScheduleState, type DigestScheduleState } from "../digest-schedule.ts";
25
25
  import { SPEND_SAMPLE_ROWS, SPEND_SAMPLE_RUNS } from "../doctor.ts";
26
+ import { EFFECTIVE_BUDGET_SAMPLE_RUNS, observeTurnBudget, type ObservedTurnBudget } from "../failure-class.ts";
26
27
  import type { CodeGraphHealth } from "../graph-health.ts";
27
28
  import { isPaused, pauseProvenance, setPaused } from "../pause.ts";
28
29
  import { branchName, route } from "../routing.ts";
@@ -189,6 +190,11 @@ export interface StatusSnapshot {
189
190
  */
190
191
  reporting?: ReportingSummary;
191
192
  caps: Caps;
193
+ /** #1063: the turn count the wall-clock ceiling actually buys at this
194
+ * project's observed per-turn latency, with the sample it was derived
195
+ * from. Absent when no qualifying completed run exists — never derived
196
+ * from the configured constants, whose ratio is a constant. */
197
+ workerTurnBudgetObserved?: ObservedTurnBudget;
192
198
  /**
193
199
  * The effective per-shape release grants. On the snapshot rather than re-read
194
200
  * by each renderer because #122 began with a grant nobody had looked at in
@@ -552,6 +558,17 @@ export function statusSnapshotFromStore(
552
558
  const observed = store.installSurfaces();
553
559
  return observed === undefined ? {} : { installSurfaces: observed };
554
560
  })(),
561
+ // #1063, the same read-not-probe discipline as the spend judgement above:
562
+ // derived from completed-run rows on this read, never probed at render
563
+ // time. The renderer decides whether the figure is materially lower.
564
+ ...(() => {
565
+ const budget = observeTurnBudget(
566
+ store.recentLatencySamples(p.name, EFFECTIVE_BUDGET_SAMPLE_RUNS),
567
+ caps,
568
+ p.workerModel,
569
+ );
570
+ return budget === undefined ? {} : { workerTurnBudgetObserved: budget };
571
+ })(),
555
572
  ...(dispatch === undefined ? {} : { dispatch }),
556
573
  ...(planUsage === undefined ? {} : { planUsage }),
557
574
  // Written by the tracker's hooks rather than polled, so the renderer does
package/src/daemon.ts CHANGED
@@ -34,7 +34,7 @@ export type {
34
34
  export { completionLastError, exhaustedSessionReason, verbDeps } from "./daemon/deps.ts";
35
35
 
36
36
  export type { CreateDrainOptions, DrainProblem, DrainRecord, DrainVerdict } from "./daemon/drain.ts";
37
- export { cancelDrain, consumeDrain, createDrain, drainPath, readDrain } from "./daemon/drain.ts";
37
+ export { cancelDrain, consumeDrain, createDrain, drainPath, markDrained, readDrain } from "./daemon/drain.ts";
38
38
 
39
39
  export type { AdmissionAckRecord } from "./daemon/ack.ts";
40
40
  export { admissionAckPath, daemonGeneration, readAdmissionAck, wakeDaemon, writeAdmissionAck } from "./daemon/ack.ts";
@@ -52,6 +52,7 @@ export {
52
52
  wakeOrchestratorForBlockedRun,
53
53
  wakeOrchestratorForMetConditions,
54
54
  watchOrchestrator,
55
+ watchWorkerProgress,
55
56
  } from "./daemon/supervision.ts";
56
57
 
57
58
  export { reconcilePanes } from "./daemon/panes.ts";
@@ -113,8 +114,8 @@ export { daemonHttpResponse, turnLimitResponse, workerControlResponse } from "./
113
114
 
114
115
  export { mineIntakeSignals, runDbSnapshotCadence, tick, upgradeVerifyDepsFor } from "./daemon/tick.ts";
115
116
 
116
- export type { DispatchLoopOptions, DispatchPace } from "./daemon/runtime.ts";
117
- export { createDispatchPace, orchestratorStandingOrders, runDaemon, runDispatchLoop } from "./daemon/runtime.ts";
117
+ export type { DispatchLoopOptions, DispatchPace, Tier1DiversionFacts } from "./daemon/runtime.ts";
118
+ export { createDispatchPace, orchestratorStandingOrders, runDaemon, runDispatchLoop, tier1DiversionLine } from "./daemon/runtime.ts";
118
119
 
119
120
  // ------------------------------------------------------------------- pause
120
121
  // The sentinel itself lives in `pause.ts` (#938): this module imports
package/src/decisions.ts CHANGED
@@ -45,26 +45,35 @@ const NPM_SPEC = /^(@[^/@\s]+\/)?[^@\s]+@[^\s]+$/;
45
45
  const GREEN_CHECK_STATES: Record<string, true> = { success: true, neutral: true };
46
46
 
47
47
  /**
48
- * The run states a review revision may start from (#795) — the single
48
+ * The run states a review revision may start from (#795, #1101) — the single
49
49
  * definition shared by `conductor_pr_review` and the `pr-review-ready` watch,
50
50
  * so the verb's gate and the condition can never name different sets.
51
51
  *
52
52
  * A revision round resumes the exact run whose row owns the PR, so the
53
- * revisable states are exactly the terminal runs that pushed one: a settled
54
- * `pushed-green` row, or a `failed` / `killed` row — a run that capped or
55
- * failed *after* pushing a green PR. The PR is the durable artefact, the
56
- * exact-head green verification is the gate on "green at the reviewed SHA",
57
- * and a terminal row proves no worker is in flight, so findings are returned
53
+ * revisable states are exactly the terminal runs that own the named PR: a
54
+ * settled `pushed-green` row, a `failed` / `killed` row — a run that capped or
55
+ * failed *after* pushing a green PR and, since #1101, a `stopped` row that
56
+ * pushed one. Ownership is not carried by the state alone: selection runs
57
+ * through `runsForProjectPr`, so only a row that itself recorded the reviewed
58
+ * PR can ever reach this gate. The PR is the durable artefact, the exact-head
59
+ * green verification is the gate on "green at the reviewed SHA", and a
60
+ * terminal row proves no worker is in flight, so findings are returned
58
61
  * without the close-PR → unblock → continuation dance.
59
62
  *
63
+ * A `stopped` row has no settle sweep keeping it honest (nothing transitions a
64
+ * stopped row when its PR merges or closes), so like `failed` / `killed` its
65
+ * rounds re-read PR-open and green-at-head decisively before the claim — the
66
+ * dispatch pass refuses to wake a worker against a dead or moved PR.
67
+ *
60
68
  * Closed on purpose: a live row (`running` / `claimed`) is already doing its
61
69
  * own work, a `pushed-pending` PR is not green yet, and a `blocked` /
62
- * `orphaned` / `stopped` / `merged` row is not work returned for revision.
70
+ * `orphaned` / `merged` row is not work returned for revision.
63
71
  */
64
72
  export const REVISABLE_RUN_STATES: Record<string, true> = {
65
73
  "pushed-green": true,
66
74
  failed: true,
67
75
  killed: true,
76
+ stopped: true,
68
77
  };
69
78
 
70
79
  /**
@@ -92,8 +101,10 @@ export const PR_LOOKUP_WINDOW_MS = 30 * 24 * 60 * 60_000;
92
101
  * `running`, and a watch keyed only to checks woke the orchestrator before
93
102
  * `conductor_pr_review` was actionable), `pushed-pending` checks are still
94
103
  * settling, `blocked` may resume, `orphaned` is reconciled back to live at
95
- * startup, and `merged` means the PR lifecycle is over. When every row is
96
- * `stopped`, the newest one answers and the predicate fails closed.
104
+ * startup, and `merged` means the PR lifecycle is over. When every row of the
105
+ * history is `stopped`, the newest one answers and since #1101 it answers
106
+ * `ready` when it owns the named PR: stopping a worker that had already pushed
107
+ * must leave the PR reviewable rather than stranded.
97
108
  *
98
109
  * `no-owner` and `not-revisable` both fail closed: a review can never act, so
99
110
  * a watch must not wake, even when the checks are green.