omp-conductor 0.20.0 → 0.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/schema/config.schema.json +24 -0
- package/src/admission.ts +58 -14
- package/src/arm-challenge.ts +54 -3
- package/src/briefs/console.md +10 -5
- package/src/commands/arm.ts +7 -5
- package/src/commands/companion.ts +52 -16
- package/src/commands/drain.ts +12 -5
- package/src/commands/worker.ts +5 -0
- package/src/config-schema.ts +8 -0
- package/src/config.ts +17 -2
- package/src/daemon/drain.ts +33 -0
- package/src/daemon/groom-pass.ts +16 -6
- package/src/daemon/http.ts +28 -0
- package/src/daemon/review.ts +11 -11
- package/src/daemon/runtime.ts +55 -3
- package/src/daemon/settle-pass.ts +19 -2
- package/src/daemon/supervision.ts +144 -1
- package/src/daemon/tick.ts +31 -14
- package/src/daemon/views.ts +17 -0
- package/src/daemon.ts +4 -3
- package/src/decisions.ts +20 -9
- package/src/diff-flags.ts +204 -28
- package/src/doctor.ts +27 -9
- package/src/escalate.ts +187 -15
- package/src/failure-class.ts +358 -5
- package/src/fleet.ts +55 -25
- package/src/graph-health.ts +17 -1
- package/src/groom.ts +11 -0
- package/src/orchestrator-tick.ts +302 -24
- package/src/reports.ts +4 -1
- package/src/settlement.ts +133 -12
- package/src/status-render.ts +38 -9
- package/src/store.ts +64 -15
- package/src/to-spec.ts +285 -24
- package/src/tracker/github.ts +132 -10
- package/src/types.ts +49 -3
- package/src/verbs/server.ts +16 -12
- package/src/worker.ts +149 -35
package/src/daemon/review.ts
CHANGED
|
@@ -62,15 +62,15 @@ export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promi
|
|
|
62
62
|
);
|
|
63
63
|
break;
|
|
64
64
|
}
|
|
65
|
-
// A capped/failed run has no pushed-green settle sweep keeping its
|
|
66
|
-
// honest (#795 review round 1): nothing transitions a `failed`
|
|
67
|
-
// row when its PR merges or closes, so a PR that
|
|
68
|
-
// recorded the round would otherwise be claimed and
|
|
69
|
-
// against a dead PR. The settled-green origin keeps its
|
|
70
|
-
// (the settle sweep flips the row and the claim below
|
|
71
|
-
//
|
|
72
|
-
// round dispatches only while the PR is still open at the
|
|
73
|
-
// head.
|
|
65
|
+
// A capped/failed/stopped run has no pushed-green settle sweep keeping its
|
|
66
|
+
// row honest (#795 review round 1, #1101): nothing transitions a `failed`
|
|
67
|
+
// / `killed` / `stopped` row when its PR merges or closes, so a PR that
|
|
68
|
+
// changed after the verb recorded the round would otherwise be claimed and
|
|
69
|
+
// a worker resumed against a dead PR. The settled-green origin keeps its
|
|
70
|
+
// own interlock (the settle sweep flips the row and the claim below
|
|
71
|
+
// refuses it), so every other revisable origin re-reads the reviewed PR
|
|
72
|
+
// fact here: the round dispatches only while the PR is still open at the
|
|
73
|
+
// exact reviewed head.
|
|
74
74
|
//
|
|
75
75
|
// The skip decision is decisive-fact only (review round 2): a definitively
|
|
76
76
|
// missing PR (`GhPrMissingError` — a corroborated 404, #779), a PR that
|
|
@@ -86,7 +86,7 @@ export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promi
|
|
|
86
86
|
// #1047's launch decision rests on are only readable here.
|
|
87
87
|
const originRun = d.store.getRun(revision.runId);
|
|
88
88
|
const origin = originRun?.state;
|
|
89
|
-
if (origin === "failed" || origin === "killed") {
|
|
89
|
+
if (origin === "failed" || origin === "killed" || origin === "stopped") {
|
|
90
90
|
let prState: PrState | undefined;
|
|
91
91
|
try {
|
|
92
92
|
prState = await d.tracker.prState(revision.prUrl);
|
|
@@ -161,7 +161,7 @@ export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promi
|
|
|
161
161
|
if (!d.store.claimRunForReview(revision.runId)) {
|
|
162
162
|
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
163
163
|
log(
|
|
164
|
-
`#${revision.issue} review round ${revision.round} skipped: run ${revision.runId} is no longer
|
|
164
|
+
`#${revision.issue} review round ${revision.round} skipped: run ${revision.runId} is ${origin ?? "gone"} — no longer reclaimable for review`,
|
|
165
165
|
);
|
|
166
166
|
continue;
|
|
167
167
|
}
|
package/src/daemon/runtime.ts
CHANGED
|
@@ -33,7 +33,7 @@ import { reconcileOrphanedRuns } from "../settlement.ts";
|
|
|
33
33
|
import { dbPath, openStore, utcDay } from "../store.ts";
|
|
34
34
|
import { checkTelegramFreshness } from "../telegram-freshness.ts";
|
|
35
35
|
import { GraphqlBreaker, makeTracker } from "../tracker/github.ts";
|
|
36
|
-
import { RELEASE_SHAPES, type ProjectConfig, type ResolvedGrants, type RunRecord } from "../types.ts";
|
|
36
|
+
import { RELEASE_SHAPES, type Escalation, type ProjectConfig, type ResolvedGrants, type RunRecord } from "../types.ts";
|
|
37
37
|
import { runCommand } from "../upgrade-verify.ts";
|
|
38
38
|
import { inspectSurfaces } from "../upgrade.ts";
|
|
39
39
|
import { sharedUsageSource } from "../usage.ts";
|
|
@@ -203,6 +203,54 @@ export async function runDispatchLoop(o: DispatchLoopOptions): Promise<void> {
|
|
|
203
203
|
}
|
|
204
204
|
}
|
|
205
205
|
|
|
206
|
+
/** What the daemon knows about its tier-1 orchestrator at diversion time. */
|
|
207
|
+
export interface Tier1DiversionFacts {
|
|
208
|
+
/** The project's configured tier-1 transport is the external orchestrator
|
|
209
|
+
* session: issue comments are the *configured* destination, not a
|
|
210
|
+
* fallback. */
|
|
211
|
+
external: boolean;
|
|
212
|
+
/** Why the daemon-owned orchestrator session failed to start, when it did. */
|
|
213
|
+
startError?: string;
|
|
214
|
+
/** Whether the daemon-owned orchestrator session is alive right now
|
|
215
|
+
* (`OrchestratorHandle.alive()` — event-driven, false once the session
|
|
216
|
+
* exited). */
|
|
217
|
+
alive: boolean;
|
|
218
|
+
/** Whether the daemon-owned orchestrator session is mid-turn right now
|
|
219
|
+
* (`OrchestratorHandle.busy()`). */
|
|
220
|
+
busy: boolean;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* The one-line record of a tier-1 escalation that landed on the issue-comment
|
|
225
|
+
* fallback instead of the orchestrator injection (#1068).
|
|
226
|
+
*
|
|
227
|
+
* "while the orchestrator was down" is reserved for what it actually means —
|
|
228
|
+
* a session that failed to start or one that has exited. The #1062 shape was
|
|
229
|
+
* neither: the orchestrator was mid-tick, its session did not take the
|
|
230
|
+
* injection, and a reader of "diverted … while the orchestrator was down"
|
|
231
|
+
* went looking for a dead session that did not exist. When the session is
|
|
232
|
+
* alive the line says where the escalation was delivered and why it was not
|
|
233
|
+
* injected; when the project runs an external orchestrator the comments *are*
|
|
234
|
+
* the configured transport and the line says so instead of calling it a
|
|
235
|
+
* diversion.
|
|
236
|
+
*/
|
|
237
|
+
export function tier1DiversionLine(e: Escalation, facts: Tier1DiversionFacts): string {
|
|
238
|
+
const ref = escalationIssueRef(e.issue);
|
|
239
|
+
if (facts.external) {
|
|
240
|
+
return `orchestrator: tier-1 escalation on ${ref} posted as an issue comment (external orchestrator — the configured tier-1 transport)`;
|
|
241
|
+
}
|
|
242
|
+
if (facts.startError !== undefined || !facts.alive) {
|
|
243
|
+
return (
|
|
244
|
+
`orchestrator: tier-1 escalation on ${ref} diverted to issue comments while the orchestrator was down` +
|
|
245
|
+
(facts.startError !== undefined ? ` (start failed: ${facts.startError})` : " (session exited)")
|
|
246
|
+
);
|
|
247
|
+
}
|
|
248
|
+
return (
|
|
249
|
+
`orchestrator: tier-1 escalation on ${ref} posted as an issue comment — the orchestrator session ` +
|
|
250
|
+
`is alive but did not take the injection${facts.busy ? " (mid-turn)" : ""}`
|
|
251
|
+
);
|
|
252
|
+
}
|
|
253
|
+
|
|
206
254
|
export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
207
255
|
// A `--once` tick is still a dispatcher: it settles rows, projects labels,
|
|
208
256
|
// admits and launches workers, so it must hold the same exclusive daemon/
|
|
@@ -391,8 +439,12 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
391
439
|
// the outage diverted (#288).
|
|
392
440
|
store.bumpOrchestratorDiverted(project.name, 1);
|
|
393
441
|
projectLog(
|
|
394
|
-
|
|
395
|
-
|
|
442
|
+
tier1DiversionLine(e, {
|
|
443
|
+
external: project.escalation.orchestrator === "external",
|
|
444
|
+
startError: orchestratorStartError,
|
|
445
|
+
alive: orchestrator?.alive() ?? false,
|
|
446
|
+
busy: orchestrator?.busy() ?? false,
|
|
447
|
+
}),
|
|
396
448
|
);
|
|
397
449
|
},
|
|
398
450
|
);
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
*/
|
|
17
17
|
import { infraLogSignature, infraSignatureVersion } from "../failure-class.ts";
|
|
18
18
|
import { errText, log, safeEscalate } from "../log.ts";
|
|
19
|
+
import { GhPrMissingError } from "../tracker/github.ts";
|
|
19
20
|
import type { BaseHealth, RunRecord, SettlementFlag, WorkflowRun } from "../types.ts";
|
|
20
21
|
import { cleanupRetainedWorktree, mirrorPathFor, type RetainedWorktreeCleanup } from "../worktree.ts";
|
|
21
22
|
import { githubRepo, type Deps, type RetainedCleanupCursor } from "./deps.ts";
|
|
@@ -393,8 +394,24 @@ export async function cleanupRetainedRuns(
|
|
|
393
394
|
terminal = issue === "closed";
|
|
394
395
|
}
|
|
395
396
|
} catch (err) {
|
|
396
|
-
|
|
397
|
-
|
|
397
|
+
// The tracker throws exactly one classified error: `GhPrMissingError`,
|
|
398
|
+
// an individual REST 404 it corroborated with a same-repository
|
|
399
|
+
// pulls-list read, so the claimed PR definitively does not exist
|
|
400
|
+
// (#779). That is a fact, not "could not tell": retrying can never
|
|
401
|
+
// conjure the PR, so the missing PR is terminal and the retained row
|
|
402
|
+
// settles in this same pass like a closed one (#1065). The row's
|
|
403
|
+
// phantom URL is cleared with the settlement so a repeat pass neither
|
|
404
|
+
// asks GitHub about it again nor logs about it again. Everything else —
|
|
405
|
+
// a rate limit, a 5xx, a lost network — keeps the deferred retry below,
|
|
406
|
+
// unchanged.
|
|
407
|
+
if (err instanceof GhPrMissingError) {
|
|
408
|
+
store.updateRun(run.id, { prUrl: null });
|
|
409
|
+
log(`#${run.issue} retained cleanup settled: PR ${run.prUrl} does not exist`);
|
|
410
|
+
terminal = true;
|
|
411
|
+
} else {
|
|
412
|
+
log(`#${run.issue} retained cleanup deferred: tracker state failed (${errText(err)})`);
|
|
413
|
+
continue;
|
|
414
|
+
}
|
|
398
415
|
}
|
|
399
416
|
if (!terminal) continue;
|
|
400
417
|
|
|
@@ -14,12 +14,15 @@
|
|
|
14
14
|
* pass into the dispatch path's import graph, for two functions whose subject —
|
|
15
15
|
* "make the orchestrator look at this now" — is already this module's.
|
|
16
16
|
*/
|
|
17
|
-
import { existsSync, readFileSync } from "node:fs";
|
|
17
|
+
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
18
18
|
import { join } from "node:path";
|
|
19
19
|
import { stateDir } from "../config.ts";
|
|
20
|
+
import { stallSilenceMs } from "../failure-class.ts";
|
|
20
21
|
import { resolvePaneHaltPath, stopConductorPane } from "../fleet.ts";
|
|
21
22
|
import { errText, log, safeEscalate } from "../log.ts";
|
|
22
23
|
import { STALL_MARKER_FILE, readTickRequestReason, requestImmediateTick, resolveTickConfigCwd } from "../orchestrator-tick.ts";
|
|
24
|
+
import { LIVE_STATES } from "../store.ts";
|
|
25
|
+
import { wakeDispatch } from "../wake.ts";
|
|
23
26
|
import { NO_ISSUE, type Deps, type StallGate } from "./deps.ts";
|
|
24
27
|
import { markPaged } from "./integrity.ts";
|
|
25
28
|
|
|
@@ -436,3 +439,143 @@ export function wakeOrchestratorForBlockedRun(
|
|
|
436
439
|
);
|
|
437
440
|
}
|
|
438
441
|
}
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
// ----------------------------------------------------------------- worker progress
|
|
445
|
+
|
|
446
|
+
/**
|
|
447
|
+
* Watching live workers for silence, and settling a run that stopped making
|
|
448
|
+
* progress (#1086).
|
|
449
|
+
*
|
|
450
|
+
* The only bounds a worker's own session enforces are the turn ceiling and the
|
|
451
|
+
* wall clock — neither fires until the whole budget is gone, so a session that
|
|
452
|
+
* hangs at minute 44 keeps its slot for another 46 minutes and then dies as a
|
|
453
|
+
* `wall-clock-cap-*` verdict that reads like an oversized slice about work that
|
|
454
|
+
* may already be merged. The signal that separates "thinking" from "gone" is
|
|
455
|
+
* already on disk: a live session writes its transcript continuously, so the
|
|
456
|
+
* file's own mtime is when the worker was last demonstrably alive.
|
|
457
|
+
*
|
|
458
|
+
* Deliberately transcript-only: the row's turn counter rides in the evidence,
|
|
459
|
+
* but it is never the trigger — one turn can legitimately run for minutes on a
|
|
460
|
+
* slow provider, and this fleet has measured 1.5 min/turn.
|
|
461
|
+
*
|
|
462
|
+
* A stall whose PR is already pushed and green settles `pushed-green` — the
|
|
463
|
+
* exact state any delivered run holds — so the settle sweep owns the merge and
|
|
464
|
+
* no continuation is charged for work that landed (#1086). Every other stall
|
|
465
|
+
* settles `killed` over the observed facts; the classifier turns those into
|
|
466
|
+
* `progress-stall`, whose recovery splits on artifacts exactly as the caps do.
|
|
467
|
+
*/
|
|
468
|
+
export async function watchWorkerProgress(
|
|
469
|
+
d: Pick<Deps, "project" | "caps" | "store" | "tracker" | "workerControls" | "escalate">,
|
|
470
|
+
now = Date.now(),
|
|
471
|
+
): Promise<void> {
|
|
472
|
+
const thresholdMs = stallSilenceMs(d.caps);
|
|
473
|
+
// A paused worker banks its wall clock and goes quiet on purpose (#938):
|
|
474
|
+
// silence under pause is the pause working, never a hang.
|
|
475
|
+
const pausedIssues = new Set(
|
|
476
|
+
d.workerControls
|
|
477
|
+
.snapshot(d.project.name)
|
|
478
|
+
.filter((w) => w.phase === "pausing" || w.phase === "paused")
|
|
479
|
+
.map((w) => w.issue),
|
|
480
|
+
);
|
|
481
|
+
for (const run of d.store.liveRuns(d.project.name)) {
|
|
482
|
+
if (pausedIssues.has(run.issue)) continue;
|
|
483
|
+
if (run.sessionFile === undefined) continue;
|
|
484
|
+
let wroteAt: number;
|
|
485
|
+
try {
|
|
486
|
+
wroteAt = Math.trunc(statSync(run.sessionFile).mtimeMs);
|
|
487
|
+
} catch {
|
|
488
|
+
// No transcript yet (claimed, session opening) or none readable: nothing
|
|
489
|
+
// to observe, which is never evidence of a stall.
|
|
490
|
+
continue;
|
|
491
|
+
}
|
|
492
|
+
// Record before judging: `status` renders this instant as the run's
|
|
493
|
+
// silence interval whether or not the threshold fires (#1086).
|
|
494
|
+
if (run.lastProgressAt !== wroteAt) {
|
|
495
|
+
d.store.updateRun(run.id, { lastProgressAt: wroteAt });
|
|
496
|
+
}
|
|
497
|
+
const silentMs = now - wroteAt;
|
|
498
|
+
if (silentMs < thresholdMs) continue;
|
|
499
|
+
|
|
500
|
+
// Re-read before settling: the session can finish between the stat above
|
|
501
|
+
// and here, and a terminal row must never be re-settled over.
|
|
502
|
+
const current = d.store.getRun(run.id);
|
|
503
|
+
if (current === undefined || !LIVE_STATES.includes(current.state)) continue;
|
|
504
|
+
const minutes = Math.round(silentMs / 60_000);
|
|
505
|
+
const silent =
|
|
506
|
+
minutes < 60 ? `${minutes}m` : `${Math.floor(minutes / 60)}h${minutes % 60 === 0 ? "" : `${minutes % 60}m`}`;
|
|
507
|
+
const why = `transcript silent ${silent} at turn ${current.turns}/${current.maxTurns}`;
|
|
508
|
+
log(`#${current.issue} ${why} — settling (silence window ${Math.round(thresholdMs / 60_000)}m)`);
|
|
509
|
+
|
|
510
|
+
// Best-effort abort through the run-control surface, so a session that can
|
|
511
|
+
// still answer exits through its own settlement path. One that cannot is
|
|
512
|
+
// exactly the case this pass exists for: the row settles below regardless,
|
|
513
|
+
// and the re-read guard keeps whichever writer lands second out.
|
|
514
|
+
try {
|
|
515
|
+
void d.workerControls.stop(d.project.name, current.issue, `progress stall: ${why}`).catch(() => {});
|
|
516
|
+
} catch {
|
|
517
|
+
// No live control for this run — the settle below is the only writer.
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
// Work that already landed is delivered work — but only verified green
|
|
521
|
+
// counts: landing `pushed-green` puts the row exactly where any delivered
|
|
522
|
+
// run sits, under the settle sweep's merge watch, with nothing charged.
|
|
523
|
+
let deliveredGreen = false;
|
|
524
|
+
if (current.prUrl !== undefined && current.headSha !== undefined) {
|
|
525
|
+
try {
|
|
526
|
+
deliveredGreen = (await d.tracker.verifyPr(current.prUrl, current.headSha))?.status === "green";
|
|
527
|
+
} catch {
|
|
528
|
+
deliveredGreen = false;
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
if (deliveredGreen) {
|
|
532
|
+
d.store.updateRun(current.id, { state: "pushed-green", endedAt: now });
|
|
533
|
+
log(`#${current.issue} stalled but ${current.prUrl} was already green — settled as delivered`);
|
|
534
|
+
await safeEscalate(d, {
|
|
535
|
+
tier: 1,
|
|
536
|
+
project: d.project.name,
|
|
537
|
+
issue: current.issue,
|
|
538
|
+
runId: current.id,
|
|
539
|
+
summary: `#${current.issue} went silent (${why}) with its PR already green — settled as delivered`,
|
|
540
|
+
detail: [
|
|
541
|
+
current.prUrl ?? "(no PR URL)",
|
|
542
|
+
`Branch ${current.branch} at ${current.headSha}.`,
|
|
543
|
+
`Session: ${current.sessionFile ?? "(no transcript)"}`,
|
|
544
|
+
"",
|
|
545
|
+
"The worker never reported, so there is no settlement narrative — the PR's diff is the record.",
|
|
546
|
+
"The slot is released; the settle sweep watches the merge from here.",
|
|
547
|
+
].join("\n"),
|
|
548
|
+
});
|
|
549
|
+
await wakeDispatchLogged(d.project.name);
|
|
550
|
+
continue;
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
d.store.updateRun(current.id, {
|
|
554
|
+
state: "killed",
|
|
555
|
+
endedAt: now,
|
|
556
|
+
lastError: `progress-stall: ${why}`,
|
|
557
|
+
});
|
|
558
|
+
await safeEscalate(d, {
|
|
559
|
+
tier: 1,
|
|
560
|
+
project: d.project.name,
|
|
561
|
+
issue: current.issue,
|
|
562
|
+
runId: current.id,
|
|
563
|
+
summary: `[progress-stall] #${current.issue} attempt ${current.attempt}: ${why}`,
|
|
564
|
+
detail: [
|
|
565
|
+
`${current.prUrl === undefined ? "No PR" : `PR ${current.prUrl} not verified green`} — the classifier decides continue vs escalate from what attempt ${current.attempt} left.`,
|
|
566
|
+
`Branch ${current.branch}; worktree ${current.worktree === "" ? "(none)" : current.worktree}.`,
|
|
567
|
+
`Session: ${current.sessionFile ?? "(no transcript)"}`,
|
|
568
|
+
].join("\n"),
|
|
569
|
+
});
|
|
570
|
+
await wakeDispatchLogged(d.project.name);
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
/** Best-effort dispatch wake once a stall settle freed its slot (#1086). */
|
|
575
|
+
async function wakeDispatchLogged(projectName: string): Promise<void> {
|
|
576
|
+
try {
|
|
577
|
+
log(`dispatch wake after stall settle: ${await wakeDispatch(projectName)}`);
|
|
578
|
+
} catch (err) {
|
|
579
|
+
log(`dispatch wake after stall settle failed: ${errText(err)}`);
|
|
580
|
+
}
|
|
581
|
+
}
|
package/src/daemon/tick.ts
CHANGED
|
@@ -42,13 +42,13 @@ import { writeAdmissionAck } from "./ack.ts";
|
|
|
42
42
|
import { dispatchAdmissions, summarizeDispatch, summarizeHeldPass, type WorkerPool } from "./admission-pass.ts";
|
|
43
43
|
import { NO_ISSUE, PACKAGE_SRC_DIR, UNROUTABLE_TEXT, type Deps } from "./deps.ts";
|
|
44
44
|
import { handleIssue } from "./dispatch.ts";
|
|
45
|
-
import {
|
|
45
|
+
import { consumeDrain, markDrained } from "./drain.ts";
|
|
46
46
|
import { dispatchToSpecGrooming } from "./groom-pass.ts";
|
|
47
47
|
import { INTEGRITY_SAMPLE, checkIntegrity, markPaged, packageManifest } from "./integrity.ts";
|
|
48
48
|
import { reconcilePanes } from "./panes.ts";
|
|
49
49
|
import { applyAdjudicationDispositions, dispatchReviewAdjudications, dispatchReviewRevisions } from "./review.ts";
|
|
50
50
|
import { cleanupRetainedRuns, watchBaseHealth, watchMergedBase } from "./settle-pass.ts";
|
|
51
|
-
import { wakeOrchestratorForMetConditions, watchOrchestrator } from "./supervision.ts";
|
|
51
|
+
import { wakeOrchestratorForMetConditions, watchOrchestrator, watchWorkerProgress } from "./supervision.ts";
|
|
52
52
|
|
|
53
53
|
/**
|
|
54
54
|
* Record what this host has installed, once per dispatch pass (#919).
|
|
@@ -388,6 +388,16 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
388
388
|
// failure happened. A pause silences claiming, not the operator's right to
|
|
389
389
|
// know their supervising session stopped reading its queue.
|
|
390
390
|
await watchOrchestrator(d);
|
|
391
|
+
// The worker-side sibling, and above the pause gate with it: a stalled run
|
|
392
|
+
// holds its slot on a parked fleet exactly as hard as on a busy one, and the
|
|
393
|
+
// #1086 incident was an upgrade drain blocked for 26 minutes by one silent
|
|
394
|
+
// session. Paused workers are skipped inside the pass — their silence is
|
|
395
|
+
// the pause working (#938) — so this gate costs nothing there.
|
|
396
|
+
try {
|
|
397
|
+
await watchWorkerProgress(d);
|
|
398
|
+
} catch (err) {
|
|
399
|
+
log(`worker-progress sweep failed: ${errText(err)}`);
|
|
400
|
+
}
|
|
391
401
|
// The down incident is reconciled the same place and for the same reason: a
|
|
392
402
|
// session that has actually died is as much the operator's concern as one
|
|
393
403
|
// that is wedged, and restarting it is the daemon's restart either way. This
|
|
@@ -584,11 +594,16 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
584
594
|
// the same admission boundary — settlement above it, nothing claimed below —
|
|
585
595
|
// but the intent is durable (it survives orchestrator loss) and bounded (the
|
|
586
596
|
// record carries an absolute deadline, so a crash can never strand
|
|
587
|
-
// admission).
|
|
597
|
+
// admission). Reaching an empty active set SUCCEEDS the drain; it does not
|
|
598
|
+
// end it (#1078): the release window the drain exists to create opens at
|
|
599
|
+
// that moment and holds claims down until the deadline — exactly the quiet
|
|
600
|
+
// interval a tick-driven orchestrator needs to cut a release, which the old
|
|
601
|
+
// eight-second auto-clear destroyed before any tick could use it. Four
|
|
602
|
+
// shapes, four behaviours:
|
|
588
603
|
// - fresh drain with live runs → a held pass, exactly like a pause;
|
|
589
|
-
// - fresh drain
|
|
590
|
-
//
|
|
591
|
-
//
|
|
604
|
+
// - fresh drain whose active set just reached zero → the opening is marked
|
|
605
|
+
// on the record once and logged once, and this pass admits nothing;
|
|
606
|
+
// - fresh drain already marked drained → the same held pass, silent;
|
|
592
607
|
// - malformed record → fails closed for THIS pass (it might be a fresh
|
|
593
608
|
// fence we cannot read), and the same consume removed it, so it can
|
|
594
609
|
// never become an unbounded permanent drain.
|
|
@@ -596,15 +611,17 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
596
611
|
if (drain.kind === "active") {
|
|
597
612
|
// Completion is the ACTIVE set, not the live-worker set: pushed-pending
|
|
598
613
|
// and pushed-green PRs still make the `runs-settled` release gate fail, so
|
|
599
|
-
// a
|
|
600
|
-
// batch the releases still see as unfinished (#776 review #2).
|
|
601
|
-
if (d.store.activeRuns(d.project.name).length === 0) {
|
|
602
|
-
|
|
603
|
-
log(
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
614
|
+
// a window declared open over a remaining run would admit work on top of
|
|
615
|
+
// a batch the releases still see as unfinished (#776 review #2).
|
|
616
|
+
if (d.store.activeRuns(d.project.name).length === 0 && drain.drain.drainedAt === undefined) {
|
|
617
|
+
markDrained(d.project.name);
|
|
618
|
+
log(
|
|
619
|
+
`project drain: fleet drained — release window open until ${drain.drain.expiresAt}` +
|
|
620
|
+
" (claims stay paused; admission resumes at the deadline or on drain cancel)",
|
|
621
|
+
);
|
|
607
622
|
}
|
|
623
|
+
d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
|
|
624
|
+
return;
|
|
608
625
|
} else if (drain.kind === "error") {
|
|
609
626
|
log(`project drain record invalid (${drain.problem}) — removed; this pass admits nothing`);
|
|
610
627
|
d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
|
package/src/daemon/views.ts
CHANGED
|
@@ -23,6 +23,7 @@ import { availabilityState, type AvailabilityState } from "../availability.ts";
|
|
|
23
23
|
import { configPath, findProject, loadConfig, resolveCaps, resolveReleaseGrants, resolveReview, stateDir } from "../config.ts";
|
|
24
24
|
import { digestScheduleState, type DigestScheduleState } from "../digest-schedule.ts";
|
|
25
25
|
import { SPEND_SAMPLE_ROWS, SPEND_SAMPLE_RUNS } from "../doctor.ts";
|
|
26
|
+
import { EFFECTIVE_BUDGET_SAMPLE_RUNS, observeTurnBudget, type ObservedTurnBudget } from "../failure-class.ts";
|
|
26
27
|
import type { CodeGraphHealth } from "../graph-health.ts";
|
|
27
28
|
import { isPaused, pauseProvenance, setPaused } from "../pause.ts";
|
|
28
29
|
import { branchName, route } from "../routing.ts";
|
|
@@ -189,6 +190,11 @@ export interface StatusSnapshot {
|
|
|
189
190
|
*/
|
|
190
191
|
reporting?: ReportingSummary;
|
|
191
192
|
caps: Caps;
|
|
193
|
+
/** #1063: the turn count the wall-clock ceiling actually buys at this
|
|
194
|
+
* project's observed per-turn latency, with the sample it was derived
|
|
195
|
+
* from. Absent when no qualifying completed run exists — never derived
|
|
196
|
+
* from the configured constants, whose ratio is a constant. */
|
|
197
|
+
workerTurnBudgetObserved?: ObservedTurnBudget;
|
|
192
198
|
/**
|
|
193
199
|
* The effective per-shape release grants. On the snapshot rather than re-read
|
|
194
200
|
* by each renderer because #122 began with a grant nobody had looked at in
|
|
@@ -552,6 +558,17 @@ export function statusSnapshotFromStore(
|
|
|
552
558
|
const observed = store.installSurfaces();
|
|
553
559
|
return observed === undefined ? {} : { installSurfaces: observed };
|
|
554
560
|
})(),
|
|
561
|
+
// #1063, the same read-not-probe discipline as the spend judgement above:
|
|
562
|
+
// derived from completed-run rows on this read, never probed at render
|
|
563
|
+
// time. The renderer decides whether the figure is materially lower.
|
|
564
|
+
...(() => {
|
|
565
|
+
const budget = observeTurnBudget(
|
|
566
|
+
store.recentLatencySamples(p.name, EFFECTIVE_BUDGET_SAMPLE_RUNS),
|
|
567
|
+
caps,
|
|
568
|
+
p.workerModel,
|
|
569
|
+
);
|
|
570
|
+
return budget === undefined ? {} : { workerTurnBudgetObserved: budget };
|
|
571
|
+
})(),
|
|
555
572
|
...(dispatch === undefined ? {} : { dispatch }),
|
|
556
573
|
...(planUsage === undefined ? {} : { planUsage }),
|
|
557
574
|
// Written by the tracker's hooks rather than polled, so the renderer does
|
package/src/daemon.ts
CHANGED
|
@@ -34,7 +34,7 @@ export type {
|
|
|
34
34
|
export { completionLastError, exhaustedSessionReason, verbDeps } from "./daemon/deps.ts";
|
|
35
35
|
|
|
36
36
|
export type { CreateDrainOptions, DrainProblem, DrainRecord, DrainVerdict } from "./daemon/drain.ts";
|
|
37
|
-
export { cancelDrain, consumeDrain, createDrain, drainPath, readDrain } from "./daemon/drain.ts";
|
|
37
|
+
export { cancelDrain, consumeDrain, createDrain, drainPath, markDrained, readDrain } from "./daemon/drain.ts";
|
|
38
38
|
|
|
39
39
|
export type { AdmissionAckRecord } from "./daemon/ack.ts";
|
|
40
40
|
export { admissionAckPath, daemonGeneration, readAdmissionAck, wakeDaemon, writeAdmissionAck } from "./daemon/ack.ts";
|
|
@@ -52,6 +52,7 @@ export {
|
|
|
52
52
|
wakeOrchestratorForBlockedRun,
|
|
53
53
|
wakeOrchestratorForMetConditions,
|
|
54
54
|
watchOrchestrator,
|
|
55
|
+
watchWorkerProgress,
|
|
55
56
|
} from "./daemon/supervision.ts";
|
|
56
57
|
|
|
57
58
|
export { reconcilePanes } from "./daemon/panes.ts";
|
|
@@ -113,8 +114,8 @@ export { daemonHttpResponse, turnLimitResponse, workerControlResponse } from "./
|
|
|
113
114
|
|
|
114
115
|
export { mineIntakeSignals, runDbSnapshotCadence, tick, upgradeVerifyDepsFor } from "./daemon/tick.ts";
|
|
115
116
|
|
|
116
|
-
export type { DispatchLoopOptions, DispatchPace } from "./daemon/runtime.ts";
|
|
117
|
-
export { createDispatchPace, orchestratorStandingOrders, runDaemon, runDispatchLoop } from "./daemon/runtime.ts";
|
|
117
|
+
export type { DispatchLoopOptions, DispatchPace, Tier1DiversionFacts } from "./daemon/runtime.ts";
|
|
118
|
+
export { createDispatchPace, orchestratorStandingOrders, runDaemon, runDispatchLoop, tier1DiversionLine } from "./daemon/runtime.ts";
|
|
118
119
|
|
|
119
120
|
// ------------------------------------------------------------------- pause
|
|
120
121
|
// The sentinel itself lives in `pause.ts` (#938): this module imports
|
package/src/decisions.ts
CHANGED
|
@@ -45,26 +45,35 @@ const NPM_SPEC = /^(@[^/@\s]+\/)?[^@\s]+@[^\s]+$/;
|
|
|
45
45
|
const GREEN_CHECK_STATES: Record<string, true> = { success: true, neutral: true };
|
|
46
46
|
|
|
47
47
|
/**
|
|
48
|
-
* The run states a review revision may start from (#795) — the single
|
|
48
|
+
* The run states a review revision may start from (#795, #1101) — the single
|
|
49
49
|
* definition shared by `conductor_pr_review` and the `pr-review-ready` watch,
|
|
50
50
|
* so the verb's gate and the condition can never name different sets.
|
|
51
51
|
*
|
|
52
52
|
* A revision round resumes the exact run whose row owns the PR, so the
|
|
53
|
-
* revisable states are exactly the terminal runs that
|
|
54
|
-
* `pushed-green` row,
|
|
55
|
-
* failed *after* pushing a green PR
|
|
56
|
-
*
|
|
57
|
-
*
|
|
53
|
+
* revisable states are exactly the terminal runs that own the named PR: a
|
|
54
|
+
* settled `pushed-green` row, a `failed` / `killed` row — a run that capped or
|
|
55
|
+
* failed *after* pushing a green PR — and, since #1101, a `stopped` row that
|
|
56
|
+
* pushed one. Ownership is not carried by the state alone: selection runs
|
|
57
|
+
* through `runsForProjectPr`, so only a row that itself recorded the reviewed
|
|
58
|
+
* PR can ever reach this gate. The PR is the durable artefact, the exact-head
|
|
59
|
+
* green verification is the gate on "green at the reviewed SHA", and a
|
|
60
|
+
* terminal row proves no worker is in flight, so findings are returned
|
|
58
61
|
* without the close-PR → unblock → continuation dance.
|
|
59
62
|
*
|
|
63
|
+
* A `stopped` row has no settle sweep keeping it honest (nothing transitions a
|
|
64
|
+
* stopped row when its PR merges or closes), so like `failed` / `killed` its
|
|
65
|
+
* rounds re-read PR-open and green-at-head decisively before the claim — the
|
|
66
|
+
* dispatch pass refuses to wake a worker against a dead or moved PR.
|
|
67
|
+
*
|
|
60
68
|
* Closed on purpose: a live row (`running` / `claimed`) is already doing its
|
|
61
69
|
* own work, a `pushed-pending` PR is not green yet, and a `blocked` /
|
|
62
|
-
* `orphaned` / `
|
|
70
|
+
* `orphaned` / `merged` row is not work returned for revision.
|
|
63
71
|
*/
|
|
64
72
|
export const REVISABLE_RUN_STATES: Record<string, true> = {
|
|
65
73
|
"pushed-green": true,
|
|
66
74
|
failed: true,
|
|
67
75
|
killed: true,
|
|
76
|
+
stopped: true,
|
|
68
77
|
};
|
|
69
78
|
|
|
70
79
|
/**
|
|
@@ -92,8 +101,10 @@ export const PR_LOOKUP_WINDOW_MS = 30 * 24 * 60 * 60_000;
|
|
|
92
101
|
* `running`, and a watch keyed only to checks woke the orchestrator before
|
|
93
102
|
* `conductor_pr_review` was actionable), `pushed-pending` checks are still
|
|
94
103
|
* settling, `blocked` may resume, `orphaned` is reconciled back to live at
|
|
95
|
-
* startup, and `merged` means the PR lifecycle is over. When every row
|
|
96
|
-
* `stopped`, the newest one answers and
|
|
104
|
+
* startup, and `merged` means the PR lifecycle is over. When every row of the
|
|
105
|
+
* history is `stopped`, the newest one answers — and since #1101 it answers
|
|
106
|
+
* `ready` when it owns the named PR: stopping a worker that had already pushed
|
|
107
|
+
* must leave the PR reviewable rather than stranded.
|
|
97
108
|
*
|
|
98
109
|
* `no-owner` and `not-revisable` both fail closed: a review can never act, so
|
|
99
110
|
* a watch must not wake, even when the checks are green.
|