omp-conductor 0.18.0 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/REFERENCE.md +60 -10
- package/agents/to-spec.md +90 -0
- package/package.json +2 -1
- package/schema/config.schema.json +29 -0
- package/src/admission.ts +204 -75
- package/src/ask.ts +268 -7
- package/src/board.ts +17 -3
- package/src/briefs/orchestrator.md +42 -14
- package/src/briefs/to-spec.md +84 -0
- package/src/briefs/worker.md +2 -1
- package/src/cli.ts +2 -0
- package/src/command-help.ts +11 -0
- package/src/command-manifest.ts +22 -0
- package/src/commands/context.ts +1 -0
- package/src/commands/drain.ts +176 -0
- package/src/commands/extend.ts +6 -10
- package/src/commands/status.ts +5 -1
- package/src/commands/watch.ts +50 -2
- package/src/commands/worker.ts +9 -10
- package/src/config-schema.ts +24 -0
- package/src/config.ts +42 -1
- package/src/daemon.ts +965 -36
- package/src/dashboard/app.js +4 -1
- package/src/dashboard/server.ts +5 -2
- package/src/decisions.ts +235 -17
- package/src/diff-flags.ts +75 -1
- package/src/doctor.ts +52 -0
- package/src/escalate.ts +9 -3
- package/src/failure-class.ts +28 -2
- package/src/fleet.ts +146 -22
- package/src/gitops.ts +188 -81
- package/src/graph-health.ts +35 -1
- package/src/graph.ts +66 -1
- package/src/harness-loader.ts +59 -0
- package/src/host.ts +567 -2
- package/src/lifecycle.ts +122 -1
- package/src/omp.ts +227 -20
- package/src/orchestrator-tick.ts +1386 -15
- package/src/orchestrator.ts +12 -0
- package/src/privileged.ts +1 -4
- package/src/release-policy.ts +503 -9
- package/src/session-host.ts +99 -5
- package/src/settlement.ts +69 -17
- package/src/setup-host.ts +1205 -6
- package/src/setup-install.ts +28 -0
- package/src/setup-wizard.ts +13 -2
- package/src/setup.ts +29 -13
- package/src/shell.ts +15 -0
- package/src/status-render.ts +78 -11
- package/src/store.ts +443 -42
- package/src/to-spec.ts +387 -0
- package/src/tracker/github.ts +104 -14
- package/src/types.ts +343 -13
- package/src/upgrade-verify.ts +209 -2
- package/src/upgrade.ts +175 -1
- package/src/verbs/protocol.ts +39 -0
- package/src/verbs/server.ts +730 -56
- package/src/verbs/socket.ts +24 -5
- package/src/worker.ts +25 -2
- package/src/worktree.ts +29 -12
package/src/daemon.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* here and enforced before anything is claimed.
|
|
9
9
|
*/
|
|
10
10
|
import { createHash } from "node:crypto";
|
|
11
|
-
import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
11
|
+
import { chmodSync, closeSync, constants, existsSync, fchownSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, readlinkSync, renameSync, rmSync, statSync, writeFileSync, type Dirent } from "node:fs";
|
|
12
12
|
import { dirname, join, relative } from "node:path";
|
|
13
13
|
import {
|
|
14
14
|
configPath,
|
|
@@ -36,12 +36,20 @@ import {
|
|
|
36
36
|
import { createEscalator, escalationIssueRef } from "./escalate.ts";
|
|
37
37
|
import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
38
38
|
import { graphHint } from "./graph.ts";
|
|
39
|
+
import {
|
|
40
|
+
hostConstraintsNotice,
|
|
41
|
+
resolveWorkerIdentity,
|
|
42
|
+
WORKER_ACCOUNT,
|
|
43
|
+
type WorkerIdentity,
|
|
44
|
+
type WorkerIdentityResolution,
|
|
45
|
+
} from "./host.ts";
|
|
39
46
|
import { acquireOnceLease, healthCheck, livingDaemon, probeUnit, SYSTEMD_UNIT } from "./lifecycle.ts";
|
|
40
47
|
import { requestImmediateTick, resolveTickConfigCwd, STALL_MARKER_FILE } from "./orchestrator-tick.ts";
|
|
41
48
|
import { runDoctor } from "./doctor.ts";
|
|
42
49
|
import { appendJournal, launchTransientUnit, readUpgradeJournal } from "./upgrade-journal.ts";
|
|
43
50
|
import { runCommand, verifyPendingUpgrade, type UpgradeVerifyDeps } from "./upgrade-verify.ts";
|
|
44
|
-
import {
|
|
51
|
+
import { processStartTimeMs } from "./upgrade-verify.ts";
|
|
52
|
+
import { fleetLayers, herdrPaneOmpStarts, resolveHerdrSession } from "./fleet.ts";
|
|
45
53
|
import { startOrchestrator } from "./orchestrator.ts";
|
|
46
54
|
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
47
55
|
import {
|
|
@@ -66,11 +74,13 @@ import type { Routed, UnroutableReason } from "./routing.ts";
|
|
|
66
74
|
import {
|
|
67
75
|
admitCandidates,
|
|
68
76
|
effectiveLane,
|
|
77
|
+
effectiveModel,
|
|
69
78
|
hasContinuationBudget,
|
|
70
79
|
hasFailedAttemptBudget,
|
|
71
80
|
laneEcho,
|
|
72
81
|
} from "./admission.ts";
|
|
73
|
-
import type { Admission, AdmissionHold
|
|
82
|
+
import type { Admission, AdmissionHold } from "./admission.ts";
|
|
83
|
+
import type { EffectiveModel, FileLane } from "./types.ts";
|
|
74
84
|
import { log, errText, safeEscalate } from "./log.ts";
|
|
75
85
|
import {
|
|
76
86
|
adoptSalvagedPrs,
|
|
@@ -107,7 +117,7 @@ import {
|
|
|
107
117
|
snapshotDb,
|
|
108
118
|
utcDay,
|
|
109
119
|
} from "./store.ts";
|
|
110
|
-
import { GraphqlBreaker, makeTracker, type RateLimitStatus } from "./tracker/github.ts";
|
|
120
|
+
import { GhPrMissingError, GraphqlBreaker, makeTracker, type RateLimitStatus } from "./tracker/github.ts";
|
|
111
121
|
import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES } from "./types.ts";
|
|
112
122
|
import type {
|
|
113
123
|
BaseFreeze,
|
|
@@ -118,6 +128,7 @@ import type {
|
|
|
118
128
|
DispatchSummary,
|
|
119
129
|
DigestBacklog,
|
|
120
130
|
Escalation,
|
|
131
|
+
InterruptCategory,
|
|
121
132
|
IssueComment,
|
|
122
133
|
IssueSnapshot,
|
|
123
134
|
MergedPrInfo,
|
|
@@ -125,13 +136,16 @@ import type {
|
|
|
125
136
|
ReleaseShape,
|
|
126
137
|
OrchestratorIncident,
|
|
127
138
|
PrState,
|
|
139
|
+
PrVerification,
|
|
128
140
|
ProjectConfig,
|
|
129
141
|
ReadyIssue,
|
|
142
|
+
ReportScope,
|
|
130
143
|
ReportingPolicy,
|
|
131
144
|
RepoTarget,
|
|
132
145
|
ReportRecord,
|
|
133
146
|
ResolvedGrants,
|
|
134
147
|
FailureClass,
|
|
148
|
+
HostConstraints,
|
|
135
149
|
RecoveryAction,
|
|
136
150
|
ReviewPolicy,
|
|
137
151
|
ReviewRevisionOutcome,
|
|
@@ -170,6 +184,7 @@ import {
|
|
|
170
184
|
import { githubVerbActions } from "./verbs/actions.ts";
|
|
171
185
|
import { formatVerbLedger, STATUS_LEDGER_SCAN } from "./verbs/ledger.ts";
|
|
172
186
|
import {
|
|
187
|
+
isHeadMismatch,
|
|
173
188
|
listenVerbChannel,
|
|
174
189
|
type VerbActions,
|
|
175
190
|
type VerbDeps,
|
|
@@ -244,6 +259,10 @@ export interface DrainSignal {
|
|
|
244
259
|
interface Deps {
|
|
245
260
|
project: ProjectConfig;
|
|
246
261
|
caps: Caps;
|
|
262
|
+
/** The typed host-constraints block (#721), re-resolved at the tick
|
|
263
|
+
* boundary like `project`/`caps`, and rendered into every brief the tick
|
|
264
|
+
* dispatches. Optional so tests that never exercise it can omit it. */
|
|
265
|
+
host?: HostConstraints;
|
|
247
266
|
tracker: Tracker;
|
|
248
267
|
store: Store;
|
|
249
268
|
/**
|
|
@@ -267,6 +286,33 @@ interface Deps {
|
|
|
267
286
|
workerControls: WorkerControlRegistry;
|
|
268
287
|
/** Session seam for lifecycle integration tests; production uses the real harness. */
|
|
269
288
|
workerDeps?: RunWorkerDeps;
|
|
289
|
+
/**
|
|
290
|
+
* Resolves the dedicated worker identity (#798) — the account worker sessions
|
|
291
|
+
* run under, with its uid/gid/home, the transition launcher and a live harness
|
|
292
|
+
* binding (#828).
|
|
293
|
+
*
|
|
294
|
+
* Called **at every worker launch**, never cached: every input is host state
|
|
295
|
+
* that can change under a running daemon. A reboot can start this service
|
|
296
|
+
* before systemd has mounted the harness binding, and an operator can install
|
|
297
|
+
* the account or re-run `setup host` at any time — a verdict taken once at
|
|
298
|
+
* startup would hold the whole fleet closed until somebody thought to restart
|
|
299
|
+
* the daemon, which is the outage #828 exists to end rather than relocate.
|
|
300
|
+
* Resolution is a handful of `stat` calls and one `/etc/passwd` read; a
|
|
301
|
+
* dispatch does far more than that before it reaches this gate.
|
|
302
|
+
*
|
|
303
|
+
* Every worker dispatch refuses to launch on an unresolved identity (fail
|
|
304
|
+
* closed — an unbound worker is indistinguishable from an operator shell),
|
|
305
|
+
* and this is how the dispatcher secures the run's sockets and grants the
|
|
306
|
+
* run's paths to the account. Absent entirely, dispatch fails closed naming
|
|
307
|
+
* the missing account.
|
|
308
|
+
*/
|
|
309
|
+
workerIdentity?: () => WorkerIdentityResolution;
|
|
310
|
+
/**
|
|
311
|
+
* Re-owns a run's working paths (worktree + session dir) under the worker
|
|
312
|
+
* identity before the session launches. Wired by `runDaemon` to the
|
|
313
|
+
* recursive chown; a test fixture leaves it unset so no test chowns.
|
|
314
|
+
*/
|
|
315
|
+
grantWorkerPaths?: (identity: WorkerIdentity, worktreePath: string, sessionDir: string) => void;
|
|
270
316
|
integrity: IntegrityGate;
|
|
271
317
|
stall: StallGate;
|
|
272
318
|
/**
|
|
@@ -454,6 +500,121 @@ export function checkStall(gate: StallGate, marker: string, now = Date.now()): S
|
|
|
454
500
|
* could destroy work an operator would rather read first — the same refusal to
|
|
455
501
|
* guess that the recovery plugin is built on.
|
|
456
502
|
*/
|
|
503
|
+
// --------------------------------------------------------- worker identity (#798) --
|
|
504
|
+
|
|
505
|
+
/**
|
|
506
|
+
* Re-own a run's working paths (worktree, session dir) under the worker
|
|
507
|
+
* identity before the session launches. The daemon runs this as root; the
|
|
508
|
+
* worker identity is granted its own run paths by ownership, never by
|
|
509
|
+
* loosened modes on the daemon's.
|
|
510
|
+
*
|
|
511
|
+
* The walk never re-owns through a path a worker-uid process could
|
|
512
|
+
* re-resolve: every ownership change is `fchownSync` on a descriptor opened
|
|
513
|
+
* with `O_NOFOLLOW`, and every descriptor is verified — via `/proc/self/fd`'s
|
|
514
|
+
* kernel-resolved path — to still sit under the directory the walk opened.
|
|
515
|
+
* A directory entry raced into a symlink therefore either fails `O_NOFOLLOW`
|
|
516
|
+
* at the final component or resolves outside the verified parent and is
|
|
517
|
+
* skipped; it can never carry the chown to an external target. Symlinks
|
|
518
|
+
* within the tree are left untouched (the worker manages entries through its
|
|
519
|
+
* parent directories, and git recreates links on checkout), as are anything
|
|
520
|
+
* unopenable — fifos, sockets, devices — which git never creates. A missing
|
|
521
|
+
* or racing entry is not this dispatch's problem — the next dispatch re-owns
|
|
522
|
+
* whatever survives; a tree already owned by the worker identity — the
|
|
523
|
+
* resume and review-revision re-ownership of a tree a prior dispatch granted
|
|
524
|
+
* — is the only tree a worker-uid process could have modified, and is
|
|
525
|
+
* skipped outright rather than walked.
|
|
526
|
+
*/
|
|
527
|
+
export function chownRecursive(root: string, uid: number, gid: number): void {
|
|
528
|
+
try {
|
|
529
|
+
const current = lstatSync(root);
|
|
530
|
+
if (current.uid === uid && current.gid === gid) return;
|
|
531
|
+
} catch {
|
|
532
|
+
// A missing or racing root is the caller's own existence check; the next
|
|
533
|
+
// dispatch re-owns whatever survives.
|
|
534
|
+
return;
|
|
535
|
+
}
|
|
536
|
+
const rootFd = openNoFollowDir(root);
|
|
537
|
+
try {
|
|
538
|
+
walkDir(rootFd, root, uid, gid);
|
|
539
|
+
} finally {
|
|
540
|
+
try {
|
|
541
|
+
closeSync(rootFd);
|
|
542
|
+
} catch {
|
|
543
|
+
// Already closed by a raced-away walk; nothing to do.
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
/** Open a directory without following a final-component symlink. */
|
|
549
|
+
function openNoFollowDir(path: string): number {
|
|
550
|
+
return openSync(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
/** The kernel-resolved path of an open descriptor: what the object actually is. */
|
|
554
|
+
function fdRealPath(fd: number): string | undefined {
|
|
555
|
+
try {
|
|
556
|
+
return readlinkSync(`/proc/self/fd/${fd}`);
|
|
557
|
+
} catch {
|
|
558
|
+
return undefined;
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
function walkDir(dirFd: number, dirPath: string, uid: number, gid: number): void {
|
|
563
|
+
try {
|
|
564
|
+
fchownSync(dirFd, uid, gid);
|
|
565
|
+
} catch {
|
|
566
|
+
// Raced away; the next dispatch re-owns what survives.
|
|
567
|
+
}
|
|
568
|
+
const dirReal = fdRealPath(dirFd);
|
|
569
|
+
if (dirReal === undefined) return;
|
|
570
|
+
let entries: Dirent[];
|
|
571
|
+
try {
|
|
572
|
+
entries = readdirSync(dirPath, { withFileTypes: true });
|
|
573
|
+
} catch {
|
|
574
|
+
return;
|
|
575
|
+
}
|
|
576
|
+
for (const entry of entries) {
|
|
577
|
+
const childPath = join(dirPath, entry.name);
|
|
578
|
+
try {
|
|
579
|
+
if (entry.isDirectory()) {
|
|
580
|
+
const childFd = openNoFollowDir(childPath);
|
|
581
|
+
try {
|
|
582
|
+
// The entry is genuinely beneath the directory this fd owns only
|
|
583
|
+
// when the kernel resolves the opened object to a path under it.
|
|
584
|
+
// Anything else — a name swapped to a symlink, a foreign listing
|
|
585
|
+
// read through a replaced parent — is refused, never re-resolved.
|
|
586
|
+
const childReal = fdRealPath(childFd);
|
|
587
|
+
if (childReal === undefined || !childReal.startsWith(`${dirReal}/`)) continue;
|
|
588
|
+
walkDir(childFd, childPath, uid, gid);
|
|
589
|
+
} finally {
|
|
590
|
+
try {
|
|
591
|
+
closeSync(childFd);
|
|
592
|
+
} catch {
|
|
593
|
+
// Raced away; nothing to close.
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
continue;
|
|
597
|
+
}
|
|
598
|
+
// Symlinks stay untouched (never followed, never chowned), and so do
|
|
599
|
+
// entries a read-only open cannot name safely (fifos, sockets, devices).
|
|
600
|
+
if (entry.isSymbolicLink() || !entry.isFile()) continue;
|
|
601
|
+
const fd = openSync(childPath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
602
|
+
try {
|
|
603
|
+
const real = fdRealPath(fd);
|
|
604
|
+
if (real !== undefined && real.startsWith(`${dirReal}/`)) fchownSync(fd, uid, gid);
|
|
605
|
+
} finally {
|
|
606
|
+
try {
|
|
607
|
+
closeSync(fd);
|
|
608
|
+
} catch {
|
|
609
|
+
// Raced away; nothing to close.
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
} catch {
|
|
613
|
+
// O_NOFOLLOW refusal (a symlink swapped onto the name), or gone: nothing to chown.
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
|
|
457
618
|
export async function watchOrchestrator(d: Deps, now = Date.now()): Promise<void> {
|
|
458
619
|
const marker = join(stateDir(), STALL_MARKER_FILE);
|
|
459
620
|
const repeat = d.stall.paged;
|
|
@@ -572,6 +733,29 @@ export function pauseProvenance(
|
|
|
572
733
|
}
|
|
573
734
|
}
|
|
574
735
|
|
|
736
|
+
/**
|
|
737
|
+
* One pause sentinel FILE read as an instance identity: who set it, why, and
|
|
738
|
+
* the creation instant, all from that exact file. An unreadable or malformed
|
|
739
|
+
* file is undefined — the caller may treat it as absence.
|
|
740
|
+
*/
|
|
741
|
+
function pauseInstanceAt(
|
|
742
|
+
path: string,
|
|
743
|
+
): { source: string; reason?: string; since: number } | undefined {
|
|
744
|
+
try {
|
|
745
|
+
const [line1, line2] = readFileSync(path, "utf8").split("\n");
|
|
746
|
+
const since = Date.parse(line1?.trim() ?? "");
|
|
747
|
+
if (!Number.isFinite(since)) return undefined;
|
|
748
|
+
if (line2 === undefined) return undefined;
|
|
749
|
+
const match = /^source=(\S+)(?: reason="(.*)")?$/.exec(line2.trim());
|
|
750
|
+
if (match === null) return undefined;
|
|
751
|
+
const source = match[1]!;
|
|
752
|
+
const reason = match[2];
|
|
753
|
+
return { source, since, ...(reason === undefined ? {} : { reason }) };
|
|
754
|
+
} catch {
|
|
755
|
+
return undefined;
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
|
|
575
759
|
/**
|
|
576
760
|
* One pause sentinel read as a single identity: who set it, why, and the
|
|
577
761
|
* creation instant, all from the SAME file that was selected. Unlike pairing
|
|
@@ -591,19 +775,34 @@ export function pauseInstance(
|
|
|
591
775
|
: [pausedPath(project), pausedPath()];
|
|
592
776
|
const path = paths.find((candidate) => existsSync(candidate));
|
|
593
777
|
if (path === undefined) return undefined;
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
778
|
+
return pauseInstanceAt(path);
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
/**
|
|
782
|
+
* Compare-and-clear one pause sentinel (#780 review): remove `path` only while
|
|
783
|
+
* it still holds exactly the `expected` instance — same source, same reason,
|
|
784
|
+
* same creation instant — as read by {@link pauseInstance}. A hold or pause
|
|
785
|
+
* that replaced or recreated the sentinel between the read and the clear is a
|
|
786
|
+
* newer instance (writes always re-stamp `since`), so it is never destroyed:
|
|
787
|
+
* returning false keeps the newer fence in force. Scoped strictly to one
|
|
788
|
+
* caller-provided path, so auto-expiry can clear the per-project spend-cap
|
|
789
|
+
* sentinel without ever touching the legacy global sentinel.
|
|
790
|
+
*/
|
|
791
|
+
export function clearPauseIfUnchanged(
|
|
792
|
+
path: string,
|
|
793
|
+
expected: { source: string; reason?: string; since: number },
|
|
794
|
+
): boolean {
|
|
795
|
+
const current = pauseInstanceAt(path);
|
|
796
|
+
if (current === undefined) return false;
|
|
797
|
+
if (
|
|
798
|
+
current.source !== expected.source ||
|
|
799
|
+
current.since !== expected.since ||
|
|
800
|
+
current.reason !== expected.reason
|
|
801
|
+
) {
|
|
802
|
+
return false;
|
|
606
803
|
}
|
|
804
|
+
rmSync(path, { force: true });
|
|
805
|
+
return true;
|
|
607
806
|
}
|
|
608
807
|
|
|
609
808
|
/**
|
|
@@ -640,6 +839,173 @@ export function setPaused(
|
|
|
640
839
|
}
|
|
641
840
|
}
|
|
642
841
|
|
|
842
|
+
// ------------------------------------------------------------------- drain
|
|
843
|
+
// (#484 slice 1) A project drain is a durable, self-expiring admission fence:
|
|
844
|
+
// the same boundary as a pause — settlement above it, nothing claimed below —
|
|
845
|
+
// but recorded with an absolute deadline, so an orchestrator crash can never
|
|
846
|
+
// strand admission. The record is a JSON file under the state directory (like
|
|
847
|
+
// the pause sentinel), scoped to exactly one configured project, and replaced
|
|
848
|
+
// atomically on create. Human CLI wording and release-verb coupling are later
|
|
849
|
+
// #484 children.
|
|
850
|
+
|
|
851
|
+
/** The persisted shape of one project drain. Every field is validated when a
|
|
852
|
+
* record is read — a record that cannot be trusted is never a fence. */
|
|
853
|
+
export interface DrainRecord {
|
|
854
|
+
/** The configured project this drain fences; must match the reader. */
|
|
855
|
+
project: string;
|
|
856
|
+
/** ISO instant the drain intent was recorded. */
|
|
857
|
+
createdAt: string;
|
|
858
|
+
/** Absolute ISO deadline: admission resumes automatically at or after it. */
|
|
859
|
+
expiresAt: string;
|
|
860
|
+
/** Purpose recorded at creation (release window, maintenance…). */
|
|
861
|
+
reason?: string;
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
/** Create-time options for {@link createDrain}. */
|
|
865
|
+
export interface CreateDrainOptions {
|
|
866
|
+
/** Absolute epoch-ms deadline — a drain must always expire on its own. */
|
|
867
|
+
expiresAt: number;
|
|
868
|
+
/** Purpose, persisted on the record and shown in structured status. */
|
|
869
|
+
reason?: string;
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
/** Why a drain record could not be trusted, named deterministically. */
|
|
873
|
+
export type DrainProblem =
|
|
874
|
+
| "unparseable-json"
|
|
875
|
+
| "invalid-record"
|
|
876
|
+
| "invalid-project"
|
|
877
|
+
| "invalid-created-at"
|
|
878
|
+
| "invalid-expires-at"
|
|
879
|
+
| "expiry-not-future"
|
|
880
|
+
| "invalid-reason";
|
|
881
|
+
|
|
882
|
+
/** The verdict of one drain read. `error` is the malformed-record case: the
|
|
883
|
+
* caller may fail its pass closed, and the dispatch-side
|
|
884
|
+
* {@link consumeDrain} removes the record in the same transition, so it can
|
|
885
|
+
* never become a permanent drain. */
|
|
886
|
+
export type DrainVerdict =
|
|
887
|
+
| { kind: "active"; drain: DrainRecord }
|
|
888
|
+
| { kind: "inactive" }
|
|
889
|
+
| { kind: "error"; problem: DrainProblem };
|
|
890
|
+
|
|
891
|
+
/** Where this project's drain record lives. Project-scoped by construction:
|
|
892
|
+
* a record present at one project's path never fences another project. */
|
|
893
|
+
export function drainPath(project: string): string {
|
|
894
|
+
return join(stateDir(), `drain-${project}.json`);
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
/**
|
|
898
|
+
* Records a bounded drain intent for `project`, replacing any prior drain of
|
|
899
|
+
* the same project atomically (tmp + rename, exactly like the admission ack).
|
|
900
|
+
* The record is a file, so it survives orchestrator and daemon loss; the
|
|
901
|
+
* absolute `expiresAt` is what stops it from ever stranding admission.
|
|
902
|
+
*/
|
|
903
|
+
export function createDrain(
|
|
904
|
+
project: string,
|
|
905
|
+
opts: CreateDrainOptions,
|
|
906
|
+
now = Date.now(),
|
|
907
|
+
): void {
|
|
908
|
+
if (project === "") {
|
|
909
|
+
throw new Error("drain project must not be empty");
|
|
910
|
+
}
|
|
911
|
+
if (!Number.isFinite(opts.expiresAt) || opts.expiresAt <= now) {
|
|
912
|
+
throw new Error("drain expiresAt must be a finite epoch-ms timestamp in the future");
|
|
913
|
+
}
|
|
914
|
+
if (opts.reason !== undefined && typeof opts.reason !== "string") {
|
|
915
|
+
throw new Error("drain reason must be a string");
|
|
916
|
+
}
|
|
917
|
+
const record: DrainRecord = {
|
|
918
|
+
project,
|
|
919
|
+
createdAt: new Date(now).toISOString(),
|
|
920
|
+
expiresAt: new Date(opts.expiresAt).toISOString(),
|
|
921
|
+
...(opts.reason === undefined ? {} : { reason: opts.reason }),
|
|
922
|
+
};
|
|
923
|
+
const path = drainPath(project);
|
|
924
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
925
|
+
const tmp = `${path}.${process.pid.toString(36)}.${Date.now().toString(36)}.tmp`;
|
|
926
|
+
writeFileSync(tmp, `${JSON.stringify(record, null, 2)}\n`);
|
|
927
|
+
renameSync(tmp, path);
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
/**
|
|
931
|
+
* One drained state read with no side effects — this never touches the record
|
|
932
|
+
* on disk. That purity is what lets observational surfaces live off it: a
|
|
933
|
+
* status/dashboard/health read must not consume a malformed record before a
|
|
934
|
+
* dispatch pass fails closed on it, or the pass would read absent and admit.
|
|
935
|
+
* The dispatch side performs the actual cleanup transition through
|
|
936
|
+
* {@link consumeDrain}. Callers that only need to *see* the state (including
|
|
937
|
+
* the claim path, which refuses but must not unbind its own pass) use this.
|
|
938
|
+
*/
|
|
939
|
+
export function readDrain(project: string, now = Date.now()): DrainVerdict {
|
|
940
|
+
const path = drainPath(project);
|
|
941
|
+
if (!existsSync(path)) return { kind: "inactive" };
|
|
942
|
+
let raw: unknown;
|
|
943
|
+
try {
|
|
944
|
+
raw = JSON.parse(readFileSync(path, "utf8"));
|
|
945
|
+
} catch {
|
|
946
|
+
return { kind: "error", problem: "unparseable-json" };
|
|
947
|
+
}
|
|
948
|
+
if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
|
|
949
|
+
return { kind: "error", problem: "invalid-record" };
|
|
950
|
+
}
|
|
951
|
+
const rec = raw as Record<string, unknown>;
|
|
952
|
+
if (typeof rec["project"] !== "string" || rec["project"] === "") {
|
|
953
|
+
return { kind: "error", problem: "invalid-project" };
|
|
954
|
+
}
|
|
955
|
+
if (typeof rec["createdAt"] !== "string" || Number.isNaN(Date.parse(rec["createdAt"]))) {
|
|
956
|
+
return { kind: "error", problem: "invalid-created-at" };
|
|
957
|
+
}
|
|
958
|
+
if (typeof rec["expiresAt"] !== "string" || Number.isNaN(Date.parse(rec["expiresAt"]))) {
|
|
959
|
+
return { kind: "error", problem: "invalid-expires-at" };
|
|
960
|
+
}
|
|
961
|
+
if (Date.parse(rec["expiresAt"] as string) <= Date.parse(rec["createdAt"] as string)) {
|
|
962
|
+
return { kind: "error", problem: "expiry-not-future" };
|
|
963
|
+
}
|
|
964
|
+
const reason = rec["reason"];
|
|
965
|
+
if (reason !== undefined && typeof reason !== "string") {
|
|
966
|
+
return { kind: "error", problem: "invalid-reason" };
|
|
967
|
+
}
|
|
968
|
+
if (rec["project"] !== project) {
|
|
969
|
+
// A record persisted at this project's path but naming another project is
|
|
970
|
+
// either a copy or a rename mishap; it fences nobody (that project's drain
|
|
971
|
+
// lives at its own path).
|
|
972
|
+
return { kind: "inactive" };
|
|
973
|
+
}
|
|
974
|
+
if (Date.parse(rec["expiresAt"] as string) <= now) {
|
|
975
|
+
return { kind: "inactive" };
|
|
976
|
+
}
|
|
977
|
+
const drain: DrainRecord = {
|
|
978
|
+
project: rec["project"],
|
|
979
|
+
createdAt: rec["createdAt"],
|
|
980
|
+
expiresAt: rec["expiresAt"],
|
|
981
|
+
...(reason === undefined ? {} : { reason }),
|
|
982
|
+
};
|
|
983
|
+
return { kind: "active", drain };
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
/**
|
|
987
|
+
* The dispatch-side drain read: the same verdict as {@link readDrain}, and the
|
|
988
|
+
* one place a stale or untrustworthy record is cleared as a side effect —
|
|
989
|
+
* expired and wrong-project records are removed (bounded stale-state cleanup
|
|
990
|
+
* on the next pass), and a malformed record is removed in the very transition
|
|
991
|
+
* that fails the pass closed, so it can never become an unbounded permanent
|
|
992
|
+
* drain. Only dispatch callers use this: an observational read here would let
|
|
993
|
+
* a status/health reader consume the malformed marker before the pass that
|
|
994
|
+
* must fail closed on it ever ran.
|
|
995
|
+
*/
|
|
996
|
+
export function consumeDrain(project: string, now = Date.now()): DrainVerdict {
|
|
997
|
+
const verdict = readDrain(project, now);
|
|
998
|
+
if (verdict.kind !== "active") {
|
|
999
|
+
rmSync(drainPath(project), { force: true });
|
|
1000
|
+
}
|
|
1001
|
+
return verdict;
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
/** Removes this project's drain, idempotently — a second cancel is a no-op. */
|
|
1005
|
+
export function cancelDrain(project: string): void {
|
|
1006
|
+
rmSync(drainPath(project), { force: true });
|
|
1007
|
+
}
|
|
1008
|
+
|
|
643
1009
|
// ----------------------------------------------------------------- admission
|
|
644
1010
|
// acknowledgement (#651, review #3)
|
|
645
1011
|
//
|
|
@@ -1023,6 +1389,22 @@ function laneBlock(lane: FileLane | undefined): string {
|
|
|
1023
1389
|
return ["## File lane (as parsed)", "", body, "", ""].join("\n");
|
|
1024
1390
|
}
|
|
1025
1391
|
|
|
1392
|
+
/**
|
|
1393
|
+
* The declared model as a brief section (#535): the selector the dispatch
|
|
1394
|
+
* will launch on — the same parse admission carried, never a second read —
|
|
1395
|
+
* or the explicit fail-open note, so the worker reads the run's model on the
|
|
1396
|
+
* brief itself rather than inferring it. Deliberately only the selector,
|
|
1397
|
+
* never the source line: the declaration's prose already renders in the body
|
|
1398
|
+
* or Discussion.
|
|
1399
|
+
*/
|
|
1400
|
+
function modelBlock(model: EffectiveModel | undefined): string {
|
|
1401
|
+
const body =
|
|
1402
|
+
model === undefined
|
|
1403
|
+
? "_no model declared — the project's workerModel (or harness default) is in effect_"
|
|
1404
|
+
: `\`${model.model}\``;
|
|
1405
|
+
return ["## Model (as parsed)", "", body, "", ""].join("\n");
|
|
1406
|
+
}
|
|
1407
|
+
|
|
1026
1408
|
/**
|
|
1027
1409
|
* What an orphan-resumed worker is told about the file lane on top of the
|
|
1028
1410
|
* continuation notice (#608). The original brief already in the transcript
|
|
@@ -1100,6 +1482,22 @@ export async function buildBrief(
|
|
|
1100
1482
|
* resolved from the rendered thread itself.
|
|
1101
1483
|
*/
|
|
1102
1484
|
lane?: FileLane;
|
|
1485
|
+
/**
|
|
1486
|
+
* The effective `Model:` declaration admission resolved for this
|
|
1487
|
+
* candidate (#535). When carried, the brief echoes exactly this
|
|
1488
|
+
* selector — the same value dispatch launches on — instead of
|
|
1489
|
+
* recomputing from the dispatch-time comment read, so the run's model is
|
|
1490
|
+
* one value on every surface. Absent (unit-level callers), the model is
|
|
1491
|
+
* resolved from the rendered thread itself.
|
|
1492
|
+
*/
|
|
1493
|
+
model?: EffectiveModel;
|
|
1494
|
+
/**
|
|
1495
|
+
* The typed host-constraints block the brief renders (#721), re-read with
|
|
1496
|
+
* the config at the tick boundary so an operator edit applies on the next
|
|
1497
|
+
* dispatch. Absent renders no host-constraints section at all — the
|
|
1498
|
+
* brief is byte-for-byte what it always was.
|
|
1499
|
+
*/
|
|
1500
|
+
host?: HostConstraints;
|
|
1103
1501
|
} = {},
|
|
1104
1502
|
): Promise<string> {
|
|
1105
1503
|
// Read per dispatch rather than caching: editing the brief then takes effect
|
|
@@ -1149,6 +1547,11 @@ export async function buildBrief(
|
|
|
1149
1547
|
// itself. Both the discussion renderer and the parsed-lane section draw on
|
|
1150
1548
|
// the same value, so the brief shows one lane on every surface (#608, #724).
|
|
1151
1549
|
const lane = opts.lane ?? (comments === "unread" ? undefined : effectiveLane(r.issue.body, comments));
|
|
1550
|
+
// The effective model declaration admission resolved (or resolves) for this
|
|
1551
|
+
// candidate: the carried admission snapshot when dispatch has one, else the
|
|
1552
|
+
// thread itself. Dispatch launches on the same value, so the brief shows
|
|
1553
|
+
// one model on every surface (#535).
|
|
1554
|
+
const model = opts.model ?? (comments === "unread" ? undefined : effectiveModel(r.issue.body, comments));
|
|
1152
1555
|
return renderBrief(template, {
|
|
1153
1556
|
ISSUE_NUMBER: String(r.issue.number),
|
|
1154
1557
|
ISSUE_TITLE: r.issue.title,
|
|
@@ -1159,12 +1562,18 @@ export async function buildBrief(
|
|
|
1159
1562
|
ACCEPTANCE_CRITERIA: acceptanceCriteria(r.issue),
|
|
1160
1563
|
ISSUE_COMMENTS: renderDiscussion(comments, lane),
|
|
1161
1564
|
FILE_LANE: laneBlock(lane),
|
|
1565
|
+
MODEL: modelBlock(model),
|
|
1162
1566
|
GATES: gatesBlock(r.repo),
|
|
1163
1567
|
// The guarded shell suites and their sanctioned parse-only check, derived
|
|
1164
1568
|
// from SHARED_HOST_SCRIPTS so the brief and the refusal share one list
|
|
1165
1569
|
// (#687). Non-empty whenever the guard has paths, so the placeholder line
|
|
1166
1570
|
// always renders to a line.
|
|
1167
1571
|
SHARED_HOST_NOTICE: sharedHostBriefNotice(),
|
|
1572
|
+
// The typed host-constraints paragraph (#721): derived cores/RAM folded
|
|
1573
|
+
// into the operator's description, the non-interactive PATH, and the
|
|
1574
|
+
// routed repo's convention. Empty when the config names none — no
|
|
1575
|
+
// section, no placeholder text.
|
|
1576
|
+
HOST_CONSTRAINTS: hostConstraintsNotice(opts.host, repoSlug(r.repo)),
|
|
1168
1577
|
// The brief's code-graph paragraph: the exact `project` key for a
|
|
1169
1578
|
// configured repo, or an explicit "no graph" statement for an
|
|
1170
1579
|
// unconfigured one — never silence, because a worker that knows there is
|
|
@@ -1523,11 +1932,39 @@ function orphanResumeVerdict(
|
|
|
1523
1932
|
return { kind: "resume", prior };
|
|
1524
1933
|
}
|
|
1525
1934
|
|
|
1935
|
+
/**
|
|
1936
|
+
* The worker identity for one launch, resolved now — or a throw naming the host
|
|
1937
|
+
* change that is missing (#798/#828).
|
|
1938
|
+
*
|
|
1939
|
+
* Every input is host state a running daemon does not control: the account can
|
|
1940
|
+
* be created after startup, and systemd can bring this service up before it has
|
|
1941
|
+
* mounted the harness binding a worker resolves through. So the verdict is taken
|
|
1942
|
+
* per launch. A daemon that cached one at startup would hold the whole fleet
|
|
1943
|
+
* closed on a boot race until somebody restarted it by hand — the outage #828
|
|
1944
|
+
* exists to end, not to relocate.
|
|
1945
|
+
*
|
|
1946
|
+
* The throw lands in the caller's dispatch catch, which settles the run failed
|
|
1947
|
+
* with this reason and escalates. It classifies as a start failure, so a host
|
|
1948
|
+
* fault charges the issue no implementation attempt.
|
|
1949
|
+
*/
|
|
1950
|
+
function launchIdentity(d: Pick<Deps, "workerIdentity">, launching: string): WorkerIdentity {
|
|
1951
|
+
const resolution: WorkerIdentityResolution = d.workerIdentity?.() ?? {
|
|
1952
|
+
ok: false,
|
|
1953
|
+
reason: `the ${WORKER_ACCOUNT} account is not installed on this host`,
|
|
1954
|
+
};
|
|
1955
|
+
if (resolution.ok) return resolution.identity;
|
|
1956
|
+
throw new Error(
|
|
1957
|
+
`worker identity unavailable: ${resolution.reason} — refusing to launch an unbound ${launching}; ` +
|
|
1958
|
+
"run `omp-conductor setup host` to install the dedicated worker identity",
|
|
1959
|
+
);
|
|
1960
|
+
}
|
|
1961
|
+
|
|
1526
1962
|
export async function handleIssue(
|
|
1527
1963
|
d: Deps,
|
|
1528
1964
|
r: Routed,
|
|
1529
1965
|
attempt: number,
|
|
1530
1966
|
admittedLane?: FileLane,
|
|
1967
|
+
admittedModel?: EffectiveModel,
|
|
1531
1968
|
): Promise<void> {
|
|
1532
1969
|
const { project, caps, tracker, store } = d;
|
|
1533
1970
|
const issue = r.issue.number;
|
|
@@ -1689,6 +2126,24 @@ export async function handleIssue(
|
|
|
1689
2126
|
return;
|
|
1690
2127
|
}
|
|
1691
2128
|
|
|
2129
|
+
// The claim-side of the project drain fence (#484): the tick's gate sits
|
|
2130
|
+
// above routing, so a drain created after that gate can still land
|
|
2131
|
+
// mid-pass — the claim re-checks and refuses while a fresh drain is in
|
|
2132
|
+
// force. An invalid record fails closed the same way. The read here is
|
|
2133
|
+
// observational on purpose: the claim refuses but must not consume the
|
|
2134
|
+
// marker, or the first refused claim would unbind the rest of its own
|
|
2135
|
+
// pass; the tick's consumeDrain removes it on the next pass, so no claim
|
|
2136
|
+
// path can ever be blocked permanently by it.
|
|
2137
|
+
const claimDrain = readDrain(d.project.name);
|
|
2138
|
+
if (claimDrain.kind === "active" || claimDrain.kind === "error") {
|
|
2139
|
+
log(
|
|
2140
|
+
claimDrain.kind === "error"
|
|
2141
|
+
? `#${issue} not claimed: drain record invalid (${claimDrain.problem})`
|
|
2142
|
+
: `#${issue} not claimed: project drain in effect`,
|
|
2143
|
+
);
|
|
2144
|
+
return;
|
|
2145
|
+
}
|
|
2146
|
+
|
|
1692
2147
|
// Claim on the STORE first, before anything that can fail. The run row —
|
|
1693
2148
|
// not the label — is the crash-safe guard against double dispatch: rows
|
|
1694
2149
|
// are local, written before any network call, and the startup orphan
|
|
@@ -1716,13 +2171,21 @@ export async function handleIssue(
|
|
|
1716
2171
|
// unconfigured project — and today's dispatch is byte for byte what it
|
|
1717
2172
|
// has always been.
|
|
1718
2173
|
const chainConfigured = (project.modelFallbacks?.length ?? 0) > 0;
|
|
2174
|
+
// A `Model:` declaration admission resolved for this candidate (#535) is
|
|
2175
|
+
// this issue's workerModel: the orchestrator names a tier when it
|
|
2176
|
+
// promotes, and dispatch launches on that selector exactly as if the
|
|
2177
|
+
// project's `workerModel` were the declared value. Absent a declaration,
|
|
2178
|
+
// today's `project.workerModel` is unchanged, and the failover chain
|
|
2179
|
+
// (#286) keeps its semantics in both cases — it is the same resolution,
|
|
2180
|
+
// one different primary.
|
|
2181
|
+
const declaredModel = admittedModel?.model;
|
|
1719
2182
|
const choice = resolveDispatchModel({
|
|
1720
|
-
workerModel: project.workerModel,
|
|
2183
|
+
workerModel: declaredModel ?? project.workerModel,
|
|
1721
2184
|
modelFallbacks: project.modelFallbacks,
|
|
1722
2185
|
threshold: project.modelFallbackThreshold ?? DEFAULT_MODEL_FALLBACK_THRESHOLD,
|
|
1723
2186
|
streak: chainFacts.streak,
|
|
1724
2187
|
});
|
|
1725
|
-
const clause = fallbackClause(choice, chainFacts, project.workerModel);
|
|
2188
|
+
const clause = fallbackClause(choice, chainFacts, declaredModel ?? project.workerModel);
|
|
1726
2189
|
|
|
1727
2190
|
// #536: an orphan-clean requeue resumes the interrupted session instead of
|
|
1728
2191
|
// re-reading the repo from zero. Decided here, before this attempt's row
|
|
@@ -1768,6 +2231,13 @@ export async function handleIssue(
|
|
|
1768
2231
|
// above fired. `undefined` for a fresh dispatch — the store maps that to
|
|
1769
2232
|
// NULL, so a fresh row simply never carries the field.
|
|
1770
2233
|
resumedFromRunId: resuming?.id,
|
|
2234
|
+
// #744: the file-lane declaration admission resolved for this candidate
|
|
2235
|
+
// is persisted on the row, so lane occupancy survives across dispatch
|
|
2236
|
+
// passes — a later pass knows what this run *intends* to touch, not only
|
|
2237
|
+
// what it has touched so far. This is the exact `Admission.lane` the gate
|
|
2238
|
+
// enforced and the brief rendered, never a re-parse. Absent for a run
|
|
2239
|
+
// with no declaration (fail open), exactly as it was admitted.
|
|
2240
|
+
lane: admittedLane,
|
|
1771
2241
|
});
|
|
1772
2242
|
// #434: carry a terminal predecessor's open PR onto the continuation row.
|
|
1773
2243
|
// The claim itself stays synchronous — `handleIssue` claims at the top of
|
|
@@ -1806,6 +2276,14 @@ export async function handleIssue(
|
|
|
1806
2276
|
if (await settleStopBeforeSession()) return;
|
|
1807
2277
|
if (await settleDrainBeforeSession()) return;
|
|
1808
2278
|
|
|
2279
|
+
// The worker identity is this run's launch gate (#798): a worker session
|
|
2280
|
+
// that cannot be launched under the dedicated unprivileged account is
|
|
2281
|
+
// indistinguishable from an operator shell, so dispatch refuses before any
|
|
2282
|
+
// tree or session is created. Resolved here rather than read off a startup
|
|
2283
|
+
// verdict, so a host that gained its account — or its harness binding
|
|
2284
|
+
// (#828) — after the daemon came up dispatches on the next tick.
|
|
2285
|
+
const identity = launchIdentity(d, "worker session");
|
|
2286
|
+
|
|
1809
2287
|
// A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
|
|
1810
2288
|
// an existing path, so a retry — or a tree kept from a failed attempt — has
|
|
1811
2289
|
// to be cleared first. Both helpers are pure path math, and removeWorktree
|
|
@@ -1886,7 +2364,13 @@ export async function handleIssue(
|
|
|
1886
2364
|
runRepoPath: worktreePath,
|
|
1887
2365
|
branch,
|
|
1888
2366
|
},
|
|
1889
|
-
{
|
|
2367
|
+
{
|
|
2368
|
+
...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }),
|
|
2369
|
+
// A worker-run channel is secured to the worker identity (#798): the
|
|
2370
|
+
// child that must connect to it runs as that uid, and the peer verdict
|
|
2371
|
+
// expects that uid on the wire rather than the daemon's.
|
|
2372
|
+
channelOwner: { uid: identity.uid, gid: identity.gid },
|
|
2373
|
+
},
|
|
1890
2374
|
);
|
|
1891
2375
|
if (await settleStopBeforeSession()) return;
|
|
1892
2376
|
if (await settleDrainBeforeSession()) return;
|
|
@@ -1943,6 +2427,8 @@ export async function handleIssue(
|
|
|
1943
2427
|
: {}),
|
|
1944
2428
|
comments,
|
|
1945
2429
|
lane: admittedLane,
|
|
2430
|
+
model: admittedModel,
|
|
2431
|
+
host: d.host,
|
|
1946
2432
|
});
|
|
1947
2433
|
}
|
|
1948
2434
|
if (await settleStopBeforeSession()) return;
|
|
@@ -1950,6 +2436,13 @@ export async function handleIssue(
|
|
|
1950
2436
|
|
|
1951
2437
|
const repoSlug = githubRepo(r.repo.cloneUrl);
|
|
1952
2438
|
|
|
2439
|
+
// The run's working paths are granted to the worker identity by ownership
|
|
2440
|
+
// (#798) — the worktree the session edits and the session directory it
|
|
2441
|
+
// writes its transcript and settings into. The chown lands here, after
|
|
2442
|
+
// provisioning and the verb socket, so the session starts on a tree it
|
|
2443
|
+
// owns; the worker identity never inherits anything from root's trees.
|
|
2444
|
+
d.grantWorkerPaths?.(identity, worktreePath, sessionDir);
|
|
2445
|
+
|
|
1953
2446
|
let result: WorkerResult;
|
|
1954
2447
|
try {
|
|
1955
2448
|
result = await runWorker({
|
|
@@ -1979,6 +2472,7 @@ export async function handleIssue(
|
|
|
1979
2472
|
onChildLog: (line) => {
|
|
1980
2473
|
log(`#${issue} ${line}`);
|
|
1981
2474
|
},
|
|
2475
|
+
workerIdentity: identity,
|
|
1982
2476
|
...(choice.model === undefined ? {} : { model: choice.model }),
|
|
1983
2477
|
// The fleet-owned omp settings overlay (#537): the staged YAML the
|
|
1984
2478
|
// session loads through `Settings.init({ configFiles: [<path>] })` —
|
|
@@ -2061,6 +2555,12 @@ export async function handleIssue(
|
|
|
2061
2555
|
// The claimed-proof check compares the PR's Verified commands
|
|
2062
2556
|
// against what this run's session actually recorded.
|
|
2063
2557
|
sessionFile: result.sessionFile,
|
|
2558
|
+
// The effective file lane admission resolved for this run at
|
|
2559
|
+
// dispatch — the same value the brief rendered, pre-dispatch
|
|
2560
|
+
// comment declarations included (#608, #744). The audit flags a
|
|
2561
|
+
// diff that escapes it, so a widened lane is named on evidence
|
|
2562
|
+
// rather than found by reading the PR's file list by hand (#739).
|
|
2563
|
+
lane: admittedLane,
|
|
2064
2564
|
})
|
|
2065
2565
|
: undefined;
|
|
2066
2566
|
if (result.state === "pushed-green" && audit?.truncated) {
|
|
@@ -2170,6 +2670,10 @@ export async function handleIssue(
|
|
|
2170
2670
|
...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
|
|
2171
2671
|
...(result.headSha === undefined ? {} : { headSha: result.headSha }),
|
|
2172
2672
|
sessionFile: result.sessionFile,
|
|
2673
|
+
// The code-graph session observation (#726): what the run's own session
|
|
2674
|
+
// registry held at start, persisted with the rest of the run's facts.
|
|
2675
|
+
// Absent only when the session surface did not record one.
|
|
2676
|
+
...(result.graphTools === undefined ? {} : { graphTools: result.graphTools }),
|
|
2173
2677
|
// Every terminal state persists the worker's report — with the `changed:`
|
|
2174
2678
|
// file list derived from the PR's diff where one could be read — not
|
|
2175
2679
|
// just a green push: a stopped attempt's partial report is still part of
|
|
@@ -2441,6 +2945,91 @@ export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promi
|
|
|
2441
2945
|
);
|
|
2442
2946
|
break;
|
|
2443
2947
|
}
|
|
2948
|
+
// A capped/failed run has no pushed-green settle sweep keeping its row
|
|
2949
|
+
// honest (#795 review round 1): nothing transitions a `failed` / `killed`
|
|
2950
|
+
// row when its PR merges or closes, so a PR that changed after the verb
|
|
2951
|
+
// recorded the round would otherwise be claimed and a worker resumed
|
|
2952
|
+
// against a dead PR. The settled-green origin keeps its own interlock
|
|
2953
|
+
// (the settle sweep flips the row and the claim below refuses it), so
|
|
2954
|
+
// only terminal-origin rounds re-read the reviewed PR fact here: the
|
|
2955
|
+
// round dispatches only while the PR is still open at the exact reviewed
|
|
2956
|
+
// head.
|
|
2957
|
+
//
|
|
2958
|
+
// The skip decision is decisive-fact only (review round 2): a definitively
|
|
2959
|
+
// missing PR (`GhPrMissingError` — a corroborated 404, #779), a PR that
|
|
2960
|
+
// is definitively not open, and a head that has definitively MOVED are
|
|
2961
|
+
// settled skipped, exactly like any other row that moved. Everything
|
|
2962
|
+
// transient or unreadable — a bare 404 that could not be corroborated, an
|
|
2963
|
+
// undefined answer, a pending check run, a same-head red check that a
|
|
2964
|
+
// rerun may clear, a throwing tracker — stays PENDING for the next tick:
|
|
2965
|
+
// settling it would permanently discard the orchestrator's findings and
|
|
2966
|
+
// consume the round.
|
|
2967
|
+
const origin = d.store.getRun(revision.runId)?.state;
|
|
2968
|
+
if (origin === "failed" || origin === "killed") {
|
|
2969
|
+
let prState: PrState | undefined;
|
|
2970
|
+
try {
|
|
2971
|
+
prState = await d.tracker.prState(revision.prUrl);
|
|
2972
|
+
} catch (err) {
|
|
2973
|
+
if (err instanceof GhPrMissingError) {
|
|
2974
|
+
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2975
|
+
log(
|
|
2976
|
+
`#${revision.issue} review round ${revision.round} skipped: ${revision.prUrl} does not exist — the reviewed PR is gone`,
|
|
2977
|
+
);
|
|
2978
|
+
continue;
|
|
2979
|
+
}
|
|
2980
|
+
log(
|
|
2981
|
+
`#${revision.issue} review round ${revision.round} held: PR state re-check failed (${errText(err)}) — retrying next tick`,
|
|
2982
|
+
);
|
|
2983
|
+
continue;
|
|
2984
|
+
}
|
|
2985
|
+
if (prState === undefined) {
|
|
2986
|
+
log(
|
|
2987
|
+
`#${revision.issue} review round ${revision.round} held: ${revision.prUrl} state could not be re-read — retrying next tick`,
|
|
2988
|
+
);
|
|
2989
|
+
continue;
|
|
2990
|
+
}
|
|
2991
|
+
if (prState !== "open") {
|
|
2992
|
+
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2993
|
+
log(
|
|
2994
|
+
`#${revision.issue} review round ${revision.round} skipped: ${revision.prUrl} is ${prState}, not open — a review round resumes an open PR only`,
|
|
2995
|
+
);
|
|
2996
|
+
continue;
|
|
2997
|
+
}
|
|
2998
|
+
let verification: PrVerification | undefined;
|
|
2999
|
+
try {
|
|
3000
|
+
verification = await d.tracker.verifyPr(revision.prUrl, revision.headSha);
|
|
3001
|
+
} catch (err) {
|
|
3002
|
+
log(
|
|
3003
|
+
`#${revision.issue} review round ${revision.round} held: the reviewed-head check failed (${errText(err)}) — retrying next tick`,
|
|
3004
|
+
);
|
|
3005
|
+
continue;
|
|
3006
|
+
}
|
|
3007
|
+
if (verification === undefined || verification.status === "pending") {
|
|
3008
|
+
log(
|
|
3009
|
+
`#${revision.issue} review round ${revision.round} held: ${revision.prUrl} at ${revision.headSha} is ` +
|
|
3010
|
+
`${verification?.status ?? "unverifiable"} — retrying next tick`,
|
|
3011
|
+
);
|
|
3012
|
+
continue;
|
|
3013
|
+
}
|
|
3014
|
+
if (verification.status === "failed" && isHeadMismatch(verification.reason)) {
|
|
3015
|
+
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3016
|
+
log(
|
|
3017
|
+
`#${revision.issue} review round ${revision.round} skipped: ${revision.prUrl} no longer stands at ` +
|
|
3018
|
+
`${revision.headSha} (${verification.reason}) — the reviewed head moved`,
|
|
3019
|
+
);
|
|
3020
|
+
continue;
|
|
3021
|
+
}
|
|
3022
|
+
// Anything else — a same-head red check a rerun may clear, an unknown
|
|
3023
|
+
// verdict — is not definitive: the round stays pending for the next
|
|
3024
|
+
// tick rather than discarding the findings.
|
|
3025
|
+
if (verification.status !== "green") {
|
|
3026
|
+
log(
|
|
3027
|
+
`#${revision.issue} review round ${revision.round} held: ${revision.prUrl} at ${revision.headSha} is ` +
|
|
3028
|
+
`${verification.status} — retrying next tick`,
|
|
3029
|
+
);
|
|
3030
|
+
continue;
|
|
3031
|
+
}
|
|
3032
|
+
}
|
|
2444
3033
|
if (!d.store.claimRunForReview(revision.runId)) {
|
|
2445
3034
|
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2446
3035
|
log(
|
|
@@ -2601,18 +3190,84 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
2601
3190
|
// and shutdown fences cover the whole wake window, exactly as they do for a
|
|
2602
3191
|
// fresh claim in `handleIssue` (#374). The original run's entries were
|
|
2603
3192
|
// closed at its settle, so reopening by issue is safe.
|
|
3193
|
+
//
|
|
3194
|
+
// #747: the round gets its own bounded allowance instead of the original
|
|
3195
|
+
// attempt's remainder. The resumed session continues the run's own counter
|
|
3196
|
+
// — the attempt's spent turns travel with the session — so a ceiling of
|
|
3197
|
+
// `run.maxTurns` would leave the round only `run.maxTurns - run.turns`,
|
|
3198
|
+
// and returning a PR from a near-cap run could not finish. Open at the
|
|
3199
|
+
// run's ceiling, then raise it by the round's own allowance — the run's
|
|
3200
|
+
// ceiling again, capped by the project's hard ceiling — through the same
|
|
3201
|
+
// override path `omp-conductor extend` uses, so the raise is persisted on
|
|
3202
|
+
// the row (the run's budget stays explainable in status) while `turns`
|
|
3203
|
+
// keeps accumulating the run's true cost: nothing is reset.
|
|
2604
3204
|
turnLimit = d.turnLimits.open(project.name, issue, runId, run.maxTurns);
|
|
3205
|
+
const revisionAllowance = Math.min(run.maxTurns, caps.workerMaxTurnsCeiling);
|
|
3206
|
+
const revisionCeiling = run.turns + revisionAllowance;
|
|
3207
|
+
const revisionRaise = d.turnLimits.extend(project.name, issue, revisionCeiling);
|
|
3208
|
+
if (revisionRaise.kind === "extended") {
|
|
3209
|
+
log(
|
|
3210
|
+
`#${issue} review round ${revision.round} turn ceiling ${run.maxTurns} → ${revisionCeiling} ` +
|
|
3211
|
+
`(+${revisionAllowance} round allowance on ${run.turns} spent)`,
|
|
3212
|
+
);
|
|
3213
|
+
}
|
|
2605
3214
|
workerControl = d.workerControls.open(project.name, issue, runId);
|
|
2606
3215
|
if (await settleStopBeforeSession()) return;
|
|
2607
3216
|
if (await settleDrainBeforeSession()) return;
|
|
2608
3217
|
|
|
3218
|
+
// The same launch gate as a fresh claim (#798): a revision worker is a
|
|
3219
|
+
// worker, and an unbound one is an operator shell — refuse before any tree
|
|
3220
|
+
// or session is created. Thrown inside the try so the ordinary dispatch
|
|
3221
|
+
// catch settles it: the run row is returned to terminal `failed` with the
|
|
3222
|
+
// reason, the revision row is settled with it, and the still-green PR stays
|
|
3223
|
+
// open for a healthy retry.
|
|
2609
3224
|
try {
|
|
3225
|
+
const identity = launchIdentity(d, "review-revision worker");
|
|
2610
3226
|
// Reattach the run's own branch at the same per-issue path the run used:
|
|
2611
|
-
//
|
|
2612
|
-
//
|
|
2613
|
-
//
|
|
2614
|
-
//
|
|
3227
|
+
// a `pushed-green` settle removed the worktree, so provisioning is the
|
|
3228
|
+
// same continuation reattach as a normal re-claim. A capped/failed run's
|
|
3229
|
+
// tree was RETAINED at its terminal settle (`tree: "keep"`, #795) — the
|
|
3230
|
+
// reviewed PR is green, so the branch already holds the work and the kept
|
|
3231
|
+
// tree holds nothing the branch does not, unless its salvage failed. Clear
|
|
3232
|
+
// it the way restart recovery does (#692): re-attempt the salvage through
|
|
3233
|
+
// `tree: "remove"`, and refuse the removal if that still fails — the tree
|
|
3234
|
+
// is then the only copy of work and is never destroyed. A provisioning
|
|
3235
|
+
// failure restores the row — the PR is still green and still open, and
|
|
3236
|
+
// only the wake failed — and says so.
|
|
2615
3237
|
try {
|
|
3238
|
+
if (run.worktree !== "" && existsSync(run.worktree)) {
|
|
3239
|
+
const keptSettlement = await settleWorktree({
|
|
3240
|
+
issue: run.issue,
|
|
3241
|
+
attempt: run.attempt,
|
|
3242
|
+
ending: `review round ${revision.round} resume`,
|
|
3243
|
+
worktree: run.worktree,
|
|
3244
|
+
branch: run.branch,
|
|
3245
|
+
publish: (branch) => pushRunBranch(project, { repo, runRepoPath: run.worktree, branch }),
|
|
3246
|
+
tree: "remove",
|
|
3247
|
+
mirrorPath,
|
|
3248
|
+
});
|
|
3249
|
+
if (keptSettlement.retained) {
|
|
3250
|
+
log(
|
|
3251
|
+
`#${issue} review round ${revision.round} retained worktree ${run.worktree}: ` +
|
|
3252
|
+
"its salvage failed, so the tree is the only copy of work and will not be removed",
|
|
3253
|
+
);
|
|
3254
|
+
store.updateRun(runId, {
|
|
3255
|
+
state: "pushed-green",
|
|
3256
|
+
endedAt: Date.now(),
|
|
3257
|
+
lastError: `review round ${revision.round} could not clear the run's retained worktree: ${keptSettlement.lines.join(" ")}`,
|
|
3258
|
+
});
|
|
3259
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3260
|
+
await safeEscalate(d, {
|
|
3261
|
+
tier: 1,
|
|
3262
|
+
project: project.name,
|
|
3263
|
+
issue,
|
|
3264
|
+
runId,
|
|
3265
|
+
summary: `#${issue} review round ${revision.round} could not be dispatched — the run's retained worktree is the only copy of work`,
|
|
3266
|
+
detail: [revision.prUrl, "", ...keptSettlement.lines].join("\n"),
|
|
3267
|
+
});
|
|
3268
|
+
return;
|
|
3269
|
+
}
|
|
3270
|
+
}
|
|
2616
3271
|
const provisioned = await addRunRepo(repo, project.mirrorRoot, project.workspaceRoot, issue, branch);
|
|
2617
3272
|
worktreePath = provisioned.path;
|
|
2618
3273
|
runRepo = { repo, runRepoPath: worktreePath, branch };
|
|
@@ -2680,7 +3335,10 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
2680
3335
|
runRepoPath: worktreePath,
|
|
2681
3336
|
branch,
|
|
2682
3337
|
},
|
|
2683
|
-
{
|
|
3338
|
+
{
|
|
3339
|
+
...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }),
|
|
3340
|
+
channelOwner: { uid: identity.uid, gid: identity.gid },
|
|
3341
|
+
},
|
|
2684
3342
|
);
|
|
2685
3343
|
if (await settleStopBeforeSession()) return;
|
|
2686
3344
|
if (await settleDrainBeforeSession()) return;
|
|
@@ -2695,6 +3353,11 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
2695
3353
|
|
|
2696
3354
|
log(`#${issue} review round ${revision.round} → resuming ${branch} (${priorSessionFile})`);
|
|
2697
3355
|
|
|
3356
|
+
// The revision worker gets its own tree ownership, exactly as a fresh
|
|
3357
|
+
// claim does (#798): the resumed worktree and session directory belong to
|
|
3358
|
+
// the worker identity before its session starts.
|
|
3359
|
+
d.grantWorkerPaths?.(identity, worktreePath, sessionDir);
|
|
3360
|
+
|
|
2698
3361
|
let result: WorkerResult;
|
|
2699
3362
|
try {
|
|
2700
3363
|
result = await runWorker({
|
|
@@ -2719,6 +3382,7 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
2719
3382
|
onChildLog: (line) => {
|
|
2720
3383
|
log(`#${issue} ${line}`);
|
|
2721
3384
|
},
|
|
3385
|
+
workerIdentity: identity,
|
|
2722
3386
|
// The continuation stays on the model the green run used (#286).
|
|
2723
3387
|
...(run.model === undefined ? {} : { model: run.model }),
|
|
2724
3388
|
...(ompSettingsFile === undefined ? {} : { ompSettingsFile }),
|
|
@@ -2813,6 +3477,7 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
2813
3477
|
...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
|
|
2814
3478
|
...(result.headSha === undefined ? {} : { headSha: result.headSha }),
|
|
2815
3479
|
sessionFile: result.sessionFile,
|
|
3480
|
+
...(result.graphTools === undefined ? {} : { graphTools: result.graphTools }),
|
|
2816
3481
|
report: finalReport,
|
|
2817
3482
|
...settlement?.patch,
|
|
2818
3483
|
};
|
|
@@ -3021,8 +3686,12 @@ export async function reconcileCrashedReviewRevisions(d: Deps): Promise<ReviewRe
|
|
|
3021
3686
|
}
|
|
3022
3687
|
// A revision the previous daemon never claimed: the run is still a
|
|
3023
3688
|
// settled green row and the round is still pending, so the ordinary
|
|
3024
|
-
// dispatch pass wakes it on the next tick untouched.
|
|
3025
|
-
|
|
3689
|
+
// dispatch pass wakes it on the next tick untouched. Same for a
|
|
3690
|
+
// capped/failed run whose review round is pending (#795): a `failed` /
|
|
3691
|
+
// `killed` row is terminal too — no process died on this round, the
|
|
3692
|
+
// revision was never claimed, and the ordinary dispatch pass claims it
|
|
3693
|
+
// exactly like a pending round on a pushed-green row.
|
|
3694
|
+
if (run.state === "pushed-green" || run.state === "failed" || run.state === "killed") continue;
|
|
3026
3695
|
// The run a previous daemon claimed for this round and died on: the
|
|
3027
3696
|
// orphan sweep just marked it `orphaned` (salvaging the tree to the
|
|
3028
3697
|
// branch), so restore it to the reviewable state the verb recorded.
|
|
@@ -3609,6 +4278,7 @@ export function summarizeDispatch(
|
|
|
3609
4278
|
holds: readonly AdmissionHold[],
|
|
3610
4279
|
completedAt = Date.now(),
|
|
3611
4280
|
settled = 0,
|
|
4281
|
+
parked = 0,
|
|
3612
4282
|
): DispatchSummary {
|
|
3613
4283
|
const groups = new Map<AdmissionHoldReason, { count: number; issues: number[]; details: string[] }>();
|
|
3614
4284
|
for (const hold of holds) {
|
|
@@ -3638,6 +4308,10 @@ export function summarizeDispatch(
|
|
|
3638
4308
|
...(group.details.length === 0 ? {} : { details: group.details }),
|
|
3639
4309
|
})),
|
|
3640
4310
|
settled,
|
|
4311
|
+
// Omitted at zero so a pass with nothing parked keeps the pre-#507 record
|
|
4312
|
+
// shape byte for byte — old persisted rows lack the key and readers use
|
|
4313
|
+
// `?? 0` either way.
|
|
4314
|
+
...(parked === 0 ? {} : { parked }),
|
|
3641
4315
|
};
|
|
3642
4316
|
}
|
|
3643
4317
|
|
|
@@ -3770,6 +4444,13 @@ export function upgradeVerifyDepsFor(d: Pick<Deps, "project" | "store">): Upgrad
|
|
|
3770
4444
|
);
|
|
3771
4445
|
return launched.ok === true ? { ok: true, unit: launched.unit } : { ok: false, stderr: launched.stderr };
|
|
3772
4446
|
},
|
|
4447
|
+
herdrSession: resolveHerdrSession(process.env),
|
|
4448
|
+
// The durable pane probe for the post-restart verifier (#832): every omp
|
|
4449
|
+
// process the fleet pane currently claims by herdr, resolved to its start
|
|
4450
|
+
// time — the evidence an external (pane-owned) orchestrator actually
|
|
4451
|
+
// restarted after the install began, through the same shared probe as the
|
|
4452
|
+
// in-process upgrade.
|
|
4453
|
+
probePaneOmp: (session) => herdrPaneOmpStarts(runCommand, session, processStartTimeMs),
|
|
3773
4454
|
log,
|
|
3774
4455
|
now: () => Date.now(),
|
|
3775
4456
|
};
|
|
@@ -3870,6 +4551,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3870
4551
|
}
|
|
3871
4552
|
d.project = fresh;
|
|
3872
4553
|
d.caps = freshCaps;
|
|
4554
|
+
d.host = cfg.host;
|
|
3873
4555
|
d.deliveryPolicyValid = true;
|
|
3874
4556
|
} catch (err) {
|
|
3875
4557
|
log(`config reload failed (${errText(err)}) — retaining boot values but blocking autonomous delivery`);
|
|
@@ -4030,6 +4712,37 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4030
4712
|
log(`decision condition pass failed: ${errText(err)}`);
|
|
4031
4713
|
});
|
|
4032
4714
|
|
|
4715
|
+
// A spend-cap pause is self-expiring (#780). The spend gate below persists a
|
|
4716
|
+
// pause once today's spend reaches the cap, but once that latch is on disk
|
|
4717
|
+
// every later pass returns at the pause gate and never reaches the spend
|
|
4718
|
+
// check again — so the fleet stayed stopped after the rolling window reset
|
|
4719
|
+
// until an operator ran `resume`. The same measurement that closed the gate
|
|
4720
|
+
// reopens it: clear a per-project spend-cap pause when the current
|
|
4721
|
+
// rolling-window spend is below the cap (or no cap is configured at all —
|
|
4722
|
+
// the gate that justified the pause is gone). Two guards keep the clear from
|
|
4723
|
+
// ever overriding another stop: only the exact instance that was read is
|
|
4724
|
+
// removed (a hold that replaced the sentinel meanwhile is untouched, review
|
|
4725
|
+
// #780), and only the per-project sentinel is ever considered — the legacy
|
|
4726
|
+
// global sentinel, which may carry an operator/integrity/setup hold, is
|
|
4727
|
+
// never removed, so the pause gate below still observes it.
|
|
4728
|
+
const spendPausePath = pausedPath(d.project.name);
|
|
4729
|
+
const spendPause = pauseInstanceAt(spendPausePath);
|
|
4730
|
+
if (spendPause?.source === "spend-cap") {
|
|
4731
|
+
const spent = d.store.spendSince(d.project.name, startOfToday());
|
|
4732
|
+
if (
|
|
4733
|
+
(d.caps.dailySpendUsd === null || spent < d.caps.dailySpendUsd) &&
|
|
4734
|
+
clearPauseIfUnchanged(spendPausePath, spendPause)
|
|
4735
|
+
) {
|
|
4736
|
+
log(
|
|
4737
|
+
`spend-cap pause cleared: $${spent.toFixed(2)} ` +
|
|
4738
|
+
(d.caps.dailySpendUsd === null
|
|
4739
|
+
? "(no daily cap configured)"
|
|
4740
|
+
: `below the $${d.caps.dailySpendUsd.toFixed(2)} daily cap`) +
|
|
4741
|
+
" — dispatch resumes",
|
|
4742
|
+
);
|
|
4743
|
+
}
|
|
4744
|
+
}
|
|
4745
|
+
|
|
4033
4746
|
// A paused fleet claims nothing. Checked first so pausing takes effect on the
|
|
4034
4747
|
// next tick without signalling the process. But the pass still ran, and the
|
|
4035
4748
|
// operator has to be able to see it: record it as a held pass — the work the
|
|
@@ -4050,6 +4763,37 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4050
4763
|
return;
|
|
4051
4764
|
}
|
|
4052
4765
|
|
|
4766
|
+
// The project drain is the self-expiring sibling of the pause fence (#484):
|
|
4767
|
+
// the same admission boundary — settlement above it, nothing claimed below —
|
|
4768
|
+
// but the intent is durable (it survives orchestrator loss) and bounded (the
|
|
4769
|
+
// record carries an absolute deadline, so a crash can never strand
|
|
4770
|
+
// admission). Three shapes, three behaviours:
|
|
4771
|
+
// - fresh drain with live runs → a held pass, exactly like a pause;
|
|
4772
|
+
// - fresh drain with nothing left to wait for → the drain is satisfied
|
|
4773
|
+
// and clears itself, so a completed drain never needs a second operator
|
|
4774
|
+
// action and this pass proceeds normally;
|
|
4775
|
+
// - malformed record → fails closed for THIS pass (it might be a fresh
|
|
4776
|
+
// fence we cannot read), and the same consume removed it, so it can
|
|
4777
|
+
// never become an unbounded permanent drain.
|
|
4778
|
+
const drain = consumeDrain(d.project.name);
|
|
4779
|
+
if (drain.kind === "active") {
|
|
4780
|
+
// Completion is the ACTIVE set, not the live-worker set: pushed-pending
|
|
4781
|
+
// and pushed-green PRs still make the `runs-settled` release gate fail, so
|
|
4782
|
+
// a drain that cleared while one remained would admit work on top of a
|
|
4783
|
+
// batch the releases still see as unfinished (#776 review #2).
|
|
4784
|
+
if (d.store.activeRuns(d.project.name).length === 0) {
|
|
4785
|
+
cancelDrain(d.project.name);
|
|
4786
|
+
log("project drain completed: no active runs remain — drain cleared");
|
|
4787
|
+
} else {
|
|
4788
|
+
d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
|
|
4789
|
+
return;
|
|
4790
|
+
}
|
|
4791
|
+
} else if (drain.kind === "error") {
|
|
4792
|
+
log(`project drain record invalid (${drain.problem}) — removed; this pass admits nothing`);
|
|
4793
|
+
d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
|
|
4794
|
+
return;
|
|
4795
|
+
}
|
|
4796
|
+
|
|
4053
4797
|
const { project, caps, store } = d;
|
|
4054
4798
|
|
|
4055
4799
|
// "Nobody patches the running conductor" is a hard boundary in both briefs —
|
|
@@ -4154,12 +4898,28 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4154
4898
|
const dropped = effective.filter(
|
|
4155
4899
|
(issue) => !isEligible(issue, project) && issue.labels.some((l) => stateLabels.has(l)),
|
|
4156
4900
|
);
|
|
4901
|
+
// The operator's park label is not a stale lifecycle label: the queue query
|
|
4902
|
+
// still returns a parked-and-queued issue, route() drops it as ineligible,
|
|
4903
|
+
// and the claim gate holds it as `issue-parked` when the park lands mid-pass
|
|
4904
|
+
// (#734). Counting it as stale-lifecycle below would summon the orchestrator
|
|
4905
|
+
// to reconcile a deliberate decision. Parked is its own population, derived
|
|
4906
|
+
// from the same isEligible read the gate uses, so the number status renders
|
|
4907
|
+
// cannot disagree with what admission would hold (#507).
|
|
4908
|
+
const parkedCandidates = effective.filter(
|
|
4909
|
+
(issue) => !isEligible(issue, project) && issue.labels.includes(project.stateLabels.backlog),
|
|
4910
|
+
);
|
|
4911
|
+
const parkedNumbers = new Set(parkedCandidates.map((issue) => issue.number));
|
|
4157
4912
|
let claimed = 0;
|
|
4913
|
+
let parked = 0;
|
|
4158
4914
|
const lifecycleHolds: AdmissionHold[] = [];
|
|
4159
4915
|
for (const issue of dropped) {
|
|
4160
4916
|
const newest = store.latestRun(project.name, issue.number);
|
|
4161
4917
|
if (newest !== undefined && ACTIVE_STATES.includes(newest.state)) {
|
|
4162
4918
|
claimed += 1;
|
|
4919
|
+
} else if (parkedNumbers.has(issue.number)) {
|
|
4920
|
+
// A park never kills a live run; a parked issue with no run is inventory
|
|
4921
|
+
// the operator is deliberately holding, not reconciliation work.
|
|
4922
|
+
parked += 1;
|
|
4163
4923
|
} else {
|
|
4164
4924
|
lifecycleHolds.push({ issue: issue.number, reason: "stale-lifecycle" });
|
|
4165
4925
|
}
|
|
@@ -4173,7 +4933,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4173
4933
|
const recordDispatch = (admitted: number, holds: readonly AdmissionHold[]): void => {
|
|
4174
4934
|
store.recordDispatch(
|
|
4175
4935
|
project.name,
|
|
4176
|
-
summarizeDispatch(ready.length, routed.length, claimed, admitted, holds, Date.now(), settled),
|
|
4936
|
+
summarizeDispatch(ready.length, routed.length, claimed, admitted, holds, Date.now(), settled, parked),
|
|
4177
4937
|
);
|
|
4178
4938
|
};
|
|
4179
4939
|
|
|
@@ -4218,7 +4978,8 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4218
4978
|
summary: `Daily spend cap reached on ${new Date().toISOString().slice(0, 10)} — ${project.name} is paused`,
|
|
4219
4979
|
detail: [
|
|
4220
4980
|
`Spent $${spent.toFixed(2)} of the $${caps.dailySpendUsd.toFixed(2)} daily cap.`,
|
|
4221
|
-
"
|
|
4981
|
+
"Dispatch stays paused until today's spend falls below the cap, then resumes",
|
|
4982
|
+
"automatically on the next pass — `omp-conductor resume` only speeds that up.",
|
|
4222
4983
|
].join("\n"),
|
|
4223
4984
|
});
|
|
4224
4985
|
recordDispatch(0, [
|
|
@@ -4266,7 +5027,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4266
5027
|
log(`dispatching ${pass.admitted.map((a) => `#${a.r.issue.number}`).join(" ")}`);
|
|
4267
5028
|
await dispatchAdmissions(
|
|
4268
5029
|
pass.admitted,
|
|
4269
|
-
(a) => handleIssue(d, a.r, a.attempt, a.lane),
|
|
5030
|
+
(a) => handleIssue(d, a.r, a.attempt, a.lane, a.model),
|
|
4270
5031
|
workers,
|
|
4271
5032
|
);
|
|
4272
5033
|
|
|
@@ -4294,6 +5055,23 @@ export interface DaemonHealthSnapshot {
|
|
|
4294
5055
|
codeGraph?: CodeGraphHealth;
|
|
4295
5056
|
/** Live workers in a non-running pause phase; absent/empty = nothing paused. */
|
|
4296
5057
|
workers?: { issue: number; runId: string; phase: WorkerPausePhase }[];
|
|
5058
|
+
/**
|
|
5059
|
+
* The live orchestrator surface, attested by the running daemon (#832):
|
|
5060
|
+
* which mode this project's fleet is in, and — when the daemon hosts the
|
|
5061
|
+
* orchestrator session itself — the extension version that session *loaded*
|
|
5062
|
+
* and the transcript it resumed. This is the read the upgrade's
|
|
5063
|
+
* session-reload verification checks: a restarted daemon whose orchestrator
|
|
5064
|
+
* child came up on the installed release answers `loaded` equal to that
|
|
5065
|
+
* release; one that never reloaded answers an older version; one that
|
|
5066
|
+
* failed to start answers `mode: "failed"`. `external` means the pane owns
|
|
5067
|
+
* the session and the daemon cannot attest it from here.
|
|
5068
|
+
*/
|
|
5069
|
+
orchestrator?: {
|
|
5070
|
+
mode: "embedded" | "external" | "failed";
|
|
5071
|
+
loaded?: string;
|
|
5072
|
+
sessionFile?: string;
|
|
5073
|
+
alive?: boolean;
|
|
5074
|
+
};
|
|
4297
5075
|
}
|
|
4298
5076
|
|
|
4299
5077
|
export interface DaemonHealth {
|
|
@@ -4309,6 +5087,7 @@ export function daemonHealthSnapshot(
|
|
|
4309
5087
|
paused = isPaused(project),
|
|
4310
5088
|
codeGraph?: CodeGraphHealth,
|
|
4311
5089
|
workerControls?: WorkerControlRegistry,
|
|
5090
|
+
orchestrator?: DaemonHealthSnapshot["orchestrator"],
|
|
4312
5091
|
): DaemonHealthSnapshot {
|
|
4313
5092
|
const dispatch = store.latestDispatch(project);
|
|
4314
5093
|
return {
|
|
@@ -4320,6 +5099,7 @@ export function daemonHealthSnapshot(
|
|
|
4320
5099
|
...(dispatch === undefined ? {} : { dispatch }),
|
|
4321
5100
|
...(codeGraph?.configured === true ? { codeGraph } : {}),
|
|
4322
5101
|
...(workerControls === undefined ? {} : { workers: workerControls.snapshot(project) }),
|
|
5102
|
+
...(orchestrator === undefined ? {} : { orchestrator }),
|
|
4323
5103
|
};
|
|
4324
5104
|
}
|
|
4325
5105
|
|
|
@@ -4637,6 +5417,55 @@ export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promi
|
|
|
4637
5417
|
return workerControl ?? new Response("not found\n", { status: 404 });
|
|
4638
5418
|
}
|
|
4639
5419
|
|
|
5420
|
+
/**
|
|
5421
|
+
* The effective reporting surface, derived from the configured policy — never
|
|
5422
|
+
* from the legacy preset name alone: an explicit policy without the
|
|
5423
|
+
* `scopePreset` back-annotation must present the same truth from `interruptOn`
|
|
5424
|
+
* and the digest cadence (#633).
|
|
5425
|
+
*/
|
|
5426
|
+
export interface ReportingSummary {
|
|
5427
|
+
/** The legacy preset this policy came from, when the config back-annotates one. */
|
|
5428
|
+
scopePreset?: ReportScope;
|
|
5429
|
+
/** Categories allowed to interrupt the operator's phone. */
|
|
5430
|
+
interruptOn: InterruptCategory[];
|
|
5431
|
+
/** Where non-interrupting outcomes accumulate: the effective digest config. */
|
|
5432
|
+
digest: ReportingPolicy["digest"];
|
|
5433
|
+
}
|
|
5434
|
+
|
|
5435
|
+
/**
|
|
5436
|
+
* The effective reporting summary. The loader always materialises the policy,
|
|
5437
|
+
* so `undefined` means the default (`DEFAULT_REPORT_POLICY`); the preset name
|
|
5438
|
+
* is carried only when the policy actually back-annotates one, so a rendered
|
|
5439
|
+
* status names the preset without ever letting a stale or absent preset
|
|
5440
|
+
* misstate what the policy does (#633).
|
|
5441
|
+
*/
|
|
5442
|
+
export function reportingSummary(policy: ReportingPolicy | undefined): ReportingSummary {
|
|
5443
|
+
const effective = policy ?? DEFAULT_REPORT_POLICY;
|
|
5444
|
+
return {
|
|
5445
|
+
...(effective.scopePreset === undefined ? {} : { scopePreset: effective.scopePreset }),
|
|
5446
|
+
interruptOn: [...effective.interruptOn],
|
|
5447
|
+
digest: { ...effective.digest },
|
|
5448
|
+
};
|
|
5449
|
+
}
|
|
5450
|
+
|
|
5451
|
+
/**
|
|
5452
|
+
* The active drain as structured status exposes it: the creation instant, the
|
|
5453
|
+
* absolute deadline, the purpose, and the live-run count the drain is waiting
|
|
5454
|
+
* to reach zero. Only present on the snapshot while the record is fresh.
|
|
5455
|
+
*/
|
|
5456
|
+
export interface DrainStatus {
|
|
5457
|
+
/** Epoch-ms instant the drain intent was recorded. */
|
|
5458
|
+
since: number;
|
|
5459
|
+
/** Absolute epoch-ms deadline — admission resumes automatically after it. */
|
|
5460
|
+
expiresAt: number;
|
|
5461
|
+
/** Purpose recorded at creation. */
|
|
5462
|
+
reason?: string;
|
|
5463
|
+
/** Runs still in the active set (live workers plus pushed-state PRs) — the
|
|
5464
|
+
* count a drain waits to reach zero, matching the `runs-settled` release
|
|
5465
|
+
* gate (#776 review #2). */
|
|
5466
|
+
remainingRuns: number;
|
|
5467
|
+
}
|
|
5468
|
+
|
|
4640
5469
|
export interface StatusSnapshot {
|
|
4641
5470
|
project: string;
|
|
4642
5471
|
configPath: string;
|
|
@@ -4648,10 +5477,27 @@ export interface StatusSnapshot {
|
|
|
4648
5477
|
* reading like a mistake (#220).
|
|
4649
5478
|
*/
|
|
4650
5479
|
pauseReason?: string;
|
|
5480
|
+
/**
|
|
5481
|
+
* The project's active self-expiring drain (#484): present only while a
|
|
5482
|
+
* fresh, valid drain record exists. Paused and drained are deliberately two
|
|
5483
|
+
* fields — a drain is bounded and self-clearing where a pause is not, and
|
|
5484
|
+
* the structured surface has to tell them apart without reading the pass
|
|
5485
|
+
* history. Human CLI wording over it is a later #484 child.
|
|
5486
|
+
*/
|
|
5487
|
+
drain?: DrainStatus;
|
|
4651
5488
|
/** Mechanical operator availability at the moment this snapshot was read. */
|
|
4652
5489
|
availability?: AvailabilityState;
|
|
4653
5490
|
/** Next digest opportunity under the same predicate that gates submission. */
|
|
4654
5491
|
digestSchedule?: DigestScheduleState;
|
|
5492
|
+
/**
|
|
5493
|
+
* The effective reporting policy: what interrupts, where everything else
|
|
5494
|
+
* goes, and the legacy preset name when one is back-annotated. On the
|
|
5495
|
+
* snapshot so the human `status`, the dashboard API and the CLI all read
|
|
5496
|
+
* the same truth about the reporting surface without opening config — a
|
|
5497
|
+
* routine outcome that reads as "Telegram is broken" unless the policy says
|
|
5498
|
+
* it is digest-only (#633).
|
|
5499
|
+
*/
|
|
5500
|
+
reporting?: ReportingSummary;
|
|
4655
5501
|
caps: Caps;
|
|
4656
5502
|
/**
|
|
4657
5503
|
* The effective per-shape release grants. On the snapshot rather than re-read
|
|
@@ -4773,6 +5619,19 @@ export function statusSnapshotFromStore(
|
|
|
4773
5619
|
// Read once: the provenance read touches the filesystem, and the renderer
|
|
4774
5620
|
// should never pay for it twice per status.
|
|
4775
5621
|
const reason = pauseProvenance(p.name)?.reason;
|
|
5622
|
+
// Same cost discipline as the pause read: the drain record is a file read,
|
|
5623
|
+
// and the active-drain view is only built when one is actually fresh. The
|
|
5624
|
+
// read is observational — it never mutates the record — so a status read
|
|
5625
|
+
// cannot consume a malformed marker the next dispatch pass still has to fail
|
|
5626
|
+
// closed on; the dispatch consume owns cleanup (#776 review #2).
|
|
5627
|
+
const drain = readDrain(p.name);
|
|
5628
|
+
// The live run list feeds the worker-capacity row; the drain's
|
|
5629
|
+
// remaining-runs count waits on the ACTIVE set (live workers plus
|
|
5630
|
+
// pushed-state PRs), the same population the `runs-settled` release gate
|
|
5631
|
+
// reads, so the two can never disagree about when a batch is finished
|
|
5632
|
+
// (#776 review #2).
|
|
5633
|
+
const live = store.liveRuns(p.name);
|
|
5634
|
+
const active = store.activeRuns(p.name);
|
|
4776
5635
|
// The live review-revision rounds, read from the same durable rows the
|
|
4777
5636
|
// restart recovery uses: a run whose revision is dispatched is read as
|
|
4778
5637
|
// `review-revision N` while its worker is live (#692).
|
|
@@ -4786,12 +5645,23 @@ export function statusSnapshotFromStore(
|
|
|
4786
5645
|
stateDir: stateDir(),
|
|
4787
5646
|
paused: isPaused(p.name),
|
|
4788
5647
|
...(reason === undefined ? {} : { pauseReason: reason }),
|
|
5648
|
+
...(drain.kind === "active"
|
|
5649
|
+
? {
|
|
5650
|
+
drain: {
|
|
5651
|
+
since: Date.parse(drain.drain.createdAt),
|
|
5652
|
+
expiresAt: Date.parse(drain.drain.expiresAt),
|
|
5653
|
+
...(drain.drain.reason === undefined ? {} : { reason: drain.drain.reason }),
|
|
5654
|
+
remainingRuns: active.length,
|
|
5655
|
+
} satisfies DrainStatus,
|
|
5656
|
+
}
|
|
5657
|
+
: {}),
|
|
4789
5658
|
availability: availabilityState(p.reporting, now),
|
|
4790
5659
|
digestSchedule: digestScheduleState(p.reporting ?? DEFAULT_REPORT_POLICY, lastDigestDay, now),
|
|
5660
|
+
reporting: reportingSummary(p.reporting ?? DEFAULT_REPORT_POLICY),
|
|
4791
5661
|
caps,
|
|
4792
5662
|
releaseGrants: resolveReleaseGrants(p),
|
|
4793
5663
|
review: resolveReview(p),
|
|
4794
|
-
activeRuns:
|
|
5664
|
+
activeRuns: active,
|
|
4795
5665
|
reviewRounds,
|
|
4796
5666
|
salvagedRuns: store.salvagedRuns(p.name),
|
|
4797
5667
|
quarantinedRuns: store.quarantinedRuns(p.name),
|
|
@@ -4799,7 +5669,7 @@ export function statusSnapshotFromStore(
|
|
|
4799
5669
|
openReports: store.openReports(p.name),
|
|
4800
5670
|
digestBacklog: store.digestBacklog(p.name),
|
|
4801
5671
|
verbLedger: store.verbLedger(p.name, { limit: STATUS_LEDGER_SCAN }),
|
|
4802
|
-
liveWorkers:
|
|
5672
|
+
liveWorkers: live.length,
|
|
4803
5673
|
runsToday: store.runsStartedSince(p.name, since),
|
|
4804
5674
|
spendTodayUsd: store.spendSince(p.name, since),
|
|
4805
5675
|
...(dispatch === undefined ? {} : { dispatch }),
|
|
@@ -4846,6 +5716,14 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
|
|
|
4846
5716
|
` candidates ${summary.ready} ready / ${summary.claimed ?? 0} in flight / ${summary.routed} spare`,
|
|
4847
5717
|
` admitted ${summary.admitted}`,
|
|
4848
5718
|
);
|
|
5719
|
+
// The operator's park label, counted from the same eligibility read the
|
|
5720
|
+
// claim gate uses (#507): "0 claimable, 12 parked" and "0 claimable,
|
|
5721
|
+
// nothing to do" demand opposite orchestrator responses and must not
|
|
5722
|
+
// render alike. Omitted at zero so an empty queue stays the old shape.
|
|
5723
|
+
const parked = summary.parked ?? 0;
|
|
5724
|
+
if (parked > 0) {
|
|
5725
|
+
lines.push(` parked ${parked} — operator-held, never claimed`);
|
|
5726
|
+
}
|
|
4849
5727
|
if (summary.holds.length === 0) {
|
|
4850
5728
|
lines.push(" held 0");
|
|
4851
5729
|
} else {
|
|
@@ -5372,6 +6250,25 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
5372
6250
|
const store = openStore(dbPath());
|
|
5373
6251
|
const verbPeerReader = peerCredentialReader();
|
|
5374
6252
|
const verbDir = ensureVerbSocketDir(stateDir());
|
|
6253
|
+
// The worker identity (#798). Probed once here for the startup banner only —
|
|
6254
|
+
// every dispatch re-resolves it through {@link launchIdentity}, because the
|
|
6255
|
+
// account, setpriv and the harness binding (#828) are all host state that can
|
|
6256
|
+
// arrive after this process did. A host that cannot establish it keeps its
|
|
6257
|
+
// control plane running and fails each worker launch closed with the reason,
|
|
6258
|
+
// so the operator hears a concrete host change instead of a fleet that
|
|
6259
|
+
// silently ran workers as root.
|
|
6260
|
+
const identityAtStartup = resolveWorkerIdentity();
|
|
6261
|
+
if (identityAtStartup.ok) {
|
|
6262
|
+
log(
|
|
6263
|
+
`worker identity: ${identityAtStartup.identity.account} uid=${identityAtStartup.identity.uid} ` +
|
|
6264
|
+
`gid=${identityAtStartup.identity.gid} home=${identityAtStartup.identity.home}`,
|
|
6265
|
+
);
|
|
6266
|
+
} else {
|
|
6267
|
+
log(
|
|
6268
|
+
"worker identity unavailable — worker dispatch fails closed until the host provides it " +
|
|
6269
|
+
`(re-checked at every launch, so no restart is needed once it does): ${identityAtStartup.reason}`,
|
|
6270
|
+
);
|
|
6271
|
+
}
|
|
5375
6272
|
const integrity: IntegrityGate = { baseline: packageManifest(), paged: false };
|
|
5376
6273
|
const usage = sharedUsageSource();
|
|
5377
6274
|
const turnLimits = createTurnLimitRegistry((runId, maxTurns) => {
|
|
@@ -5398,6 +6295,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
5398
6295
|
log(projects.length === 1 ? message : `[${project.name}] ${message}`);
|
|
5399
6296
|
};
|
|
5400
6297
|
const caps = resolveCaps(project, cfg.defaults);
|
|
6298
|
+
const host = cfg.host;
|
|
5401
6299
|
// One transient-server-error breaker per project (#642): admission's
|
|
5402
6300
|
// GraphQL checks and the orchestrator mutation commands share it, so a 503
|
|
5403
6301
|
// observed by either side gates both instead of one provider outage being
|
|
@@ -5476,7 +6374,11 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
5476
6374
|
recordReleaseBlock(project.name, "orchestrator", shape, context),
|
|
5477
6375
|
});
|
|
5478
6376
|
const transcript = orchestrator.sessionFile();
|
|
5479
|
-
|
|
6377
|
+
const loaded = orchestrator.extensionVersion();
|
|
6378
|
+
projectLog(
|
|
6379
|
+
`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}` +
|
|
6380
|
+
`${loaded === undefined ? "" : ` · loaded omp-conductor ${loaded}`}`,
|
|
6381
|
+
);
|
|
5480
6382
|
} catch (err) {
|
|
5481
6383
|
orchestratorStartError = errText(err);
|
|
5482
6384
|
projectLog(
|
|
@@ -5529,6 +6431,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
5529
6431
|
const d: Deps = {
|
|
5530
6432
|
project,
|
|
5531
6433
|
caps,
|
|
6434
|
+
host,
|
|
5532
6435
|
tracker,
|
|
5533
6436
|
store,
|
|
5534
6437
|
drain,
|
|
@@ -5544,6 +6447,16 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
5544
6447
|
probeCriticalBase: (repo, markers, branch) =>
|
|
5545
6448
|
probeCriticalBase(project, repo, branch, markers),
|
|
5546
6449
|
probeWorktreeLane: (input) => probeRunLane(input),
|
|
6450
|
+
// The live host, per launch — never the startup verdict above.
|
|
6451
|
+
workerIdentity: () => resolveWorkerIdentity(),
|
|
6452
|
+
// Grant each run's paths to the worker identity by ownership, before the
|
|
6453
|
+
// session that must edit them starts (#798). The recursive chown runs as
|
|
6454
|
+
// root from the dispatch path; the worker identity owns its checkout and
|
|
6455
|
+
// transcript and nothing else.
|
|
6456
|
+
grantWorkerPaths: (identity, worktreePath, sessionDir) => {
|
|
6457
|
+
chownRecursive(worktreePath, identity.uid, identity.gid);
|
|
6458
|
+
chownRecursive(sessionDir, identity.uid, identity.gid);
|
|
6459
|
+
},
|
|
5547
6460
|
// A cross-repo Depends-on prerequisite reads through the same GitHub
|
|
5548
6461
|
// credential/accounting seams as the project tracker — a fresh tracker
|
|
5549
6462
|
// scoped to the referenced repo, reusing the daemon's gh hooks so API
|
|
@@ -5718,15 +6631,31 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
5718
6631
|
},
|
|
5719
6632
|
health: () =>
|
|
5720
6633
|
daemonHealth(
|
|
5721
|
-
runtimes.map((runtime) =>
|
|
5722
|
-
|
|
6634
|
+
runtimes.map((runtime) => {
|
|
6635
|
+
const orch = runtime.orchestrator;
|
|
6636
|
+
const mode = runtime.d.project.escalation.orchestrator;
|
|
6637
|
+
return daemonHealthSnapshot(
|
|
5723
6638
|
store,
|
|
5724
6639
|
runtime.d.project.name,
|
|
5725
6640
|
isPaused(runtime.d.project.name),
|
|
5726
6641
|
runtime.codeGraph,
|
|
5727
6642
|
workerControls,
|
|
5728
|
-
|
|
5729
|
-
|
|
6643
|
+
orch === undefined
|
|
6644
|
+
? // External mode is the pane's session: the daemon hosts no
|
|
6645
|
+
// child to attest. An embedded orchestrator that failed to
|
|
6646
|
+
// start is a live outage of the surface the upgrade must
|
|
6647
|
+
// promise reloaded (#832) — say so out loud.
|
|
6648
|
+
{ mode: mode === "external" ? "external" : "failed" }
|
|
6649
|
+
: {
|
|
6650
|
+
mode: "embedded",
|
|
6651
|
+
...(orch.extensionVersion() === undefined
|
|
6652
|
+
? {}
|
|
6653
|
+
: { loaded: orch.extensionVersion() }),
|
|
6654
|
+
...(orch.sessionFile() === undefined ? {} : { sessionFile: orch.sessionFile() }),
|
|
6655
|
+
alive: orch.alive(),
|
|
6656
|
+
},
|
|
6657
|
+
);
|
|
6658
|
+
}),
|
|
5730
6659
|
),
|
|
5731
6660
|
}),
|
|
5732
6661
|
});
|