omp-conductor 0.3.24 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +775 -49
- package/package.json +1 -1
- package/src/approval-surface.ts +36 -1
- package/src/board.ts +30 -6
- package/src/briefs/orchestrator.md +114 -20
- package/src/briefs/policy.md +48 -27
- package/src/briefs/worker.md +82 -31
- package/src/cli.ts +164 -4
- package/src/config.ts +669 -16
- package/src/confinement.ts +506 -25
- package/src/credentials.ts +2029 -0
- package/src/daemon.ts +1213 -78
- package/src/diff-flags.ts +696 -0
- package/src/escalate.ts +136 -9
- package/src/fleet.ts +80 -4
- package/src/omp.ts +471 -15
- package/src/orchestrator-tick.ts +98 -25
- package/src/orchestrator.ts +32 -5
- package/src/plugin.ts +267 -15
- package/src/release-policy.ts +191 -29
- package/src/reports.ts +440 -0
- package/src/session-host.ts +307 -0
- package/src/setup-host.ts +19 -1
- package/src/setup.ts +207 -18
- package/src/store.ts +555 -3
- package/src/tracker/github.ts +45 -0
- package/src/types.ts +863 -10
- package/src/unblock.ts +53 -3
- package/src/usage.ts +726 -0
- package/src/verbs/actions.ts +142 -0
- package/src/verbs/client.ts +207 -0
- package/src/verbs/ledger.ts +77 -0
- package/src/verbs/protocol.ts +465 -0
- package/src/verbs/server.ts +1098 -0
- package/src/verbs/socket.ts +446 -0
- package/src/worker.ts +36 -7
- package/src/worktree.ts +227 -120
- package/systemd/omp-conductor.service.example +96 -8
package/src/daemon.ts
CHANGED
|
@@ -8,9 +8,24 @@
|
|
|
8
8
|
* here and enforced before anything is claimed.
|
|
9
9
|
*/
|
|
10
10
|
import { createHash } from "node:crypto";
|
|
11
|
-
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
11
|
+
import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
12
12
|
import { dirname, join, relative } from "node:path";
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
configPath,
|
|
15
|
+
findProject,
|
|
16
|
+
loadConfig,
|
|
17
|
+
migrateCredentialsOnDisk,
|
|
18
|
+
resolveCaps,
|
|
19
|
+
resolveCredentials,
|
|
20
|
+
resolveReleaseGrants,
|
|
21
|
+
stateDir,
|
|
22
|
+
sharedRoot,
|
|
23
|
+
} from "./config.ts";
|
|
24
|
+
import {
|
|
25
|
+
analyseSettlement,
|
|
26
|
+
formatSettlementFlags,
|
|
27
|
+
settlementFlagSummary,
|
|
28
|
+
} from "./diff-flags.ts";
|
|
14
29
|
import { createEscalator, escalationIssueRef } from "./escalate.ts";
|
|
15
30
|
import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
16
31
|
import { graphHint } from "./graph.ts";
|
|
@@ -18,11 +33,13 @@ import { livingDaemon } from "./lifecycle.ts";
|
|
|
18
33
|
import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
|
|
19
34
|
import { startOrchestrator } from "./orchestrator.ts";
|
|
20
35
|
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
36
|
+
import { createReportOutbox, formatOpenReports } from "./reports.ts";
|
|
21
37
|
import { recordReleaseBlock } from "./release-policy.ts";
|
|
22
38
|
import { branchName, route } from "./routing.ts";
|
|
23
39
|
import type { Routed, UnroutableReason } from "./routing.ts";
|
|
24
40
|
import { dbPath, openStore } from "./store.ts";
|
|
25
41
|
import { makeTracker } from "./tracker/github.ts";
|
|
42
|
+
import { RELEASE_SHAPES } from "./types.ts";
|
|
26
43
|
import type {
|
|
27
44
|
AdmissionHoldReason,
|
|
28
45
|
Caps,
|
|
@@ -30,30 +47,84 @@ import type {
|
|
|
30
47
|
Escalation,
|
|
31
48
|
OpenCloser,
|
|
32
49
|
PrState,
|
|
50
|
+
CredentialIsolation,
|
|
33
51
|
ProjectConfig,
|
|
34
52
|
ReadyIssue,
|
|
35
53
|
RepoTarget,
|
|
54
|
+
ReportRecord,
|
|
55
|
+
ResolvedGrants,
|
|
36
56
|
RunRecord,
|
|
37
57
|
RunState,
|
|
58
|
+
SettlementFlag,
|
|
38
59
|
Store,
|
|
39
60
|
Tracker,
|
|
61
|
+
VerbLedgerEntry,
|
|
40
62
|
} from "./types.ts";
|
|
63
|
+
import { CONDUCTOR_GROUPS } from "./types.ts";
|
|
41
64
|
import { type KilledBy, type WorkerResult, renderBrief, runWorker } from "./worker.ts";
|
|
42
65
|
import {
|
|
43
|
-
|
|
66
|
+
addRunRepo,
|
|
44
67
|
cleanupRetainedWorktree,
|
|
45
68
|
mirrorPathFor,
|
|
46
69
|
removeWorktree,
|
|
47
70
|
salvageWip,
|
|
48
71
|
type RetainedWorktreeCleanup,
|
|
72
|
+
type RunPublisher,
|
|
49
73
|
type SalvageOutcome,
|
|
50
74
|
worktreePathFor,
|
|
51
75
|
} from "./worktree.ts";
|
|
76
|
+
import { githubVerbActions } from "./verbs/actions.ts";
|
|
77
|
+
import { formatVerbLedger, STATUS_LEDGER_SCAN } from "./verbs/ledger.ts";
|
|
78
|
+
import { listenVerbChannel } from "./verbs/server.ts";
|
|
79
|
+
import type { VerbActions, VerbDeps, VerbListener } from "./verbs/server.ts";
|
|
80
|
+
import {
|
|
81
|
+
ensureVerbSocketDir,
|
|
82
|
+
peerCredentialReader,
|
|
83
|
+
transportBanner,
|
|
84
|
+
verbSocketPath,
|
|
85
|
+
type PeerReader,
|
|
86
|
+
} from "./verbs/socket.ts";
|
|
87
|
+
import { homedir } from "node:os";
|
|
88
|
+
|
|
89
|
+
import {
|
|
90
|
+
applyRunOwnership,
|
|
91
|
+
applyMirrorReadAccess,
|
|
92
|
+
auditMcpRoots,
|
|
93
|
+
mirrorTraversalRefusal,
|
|
94
|
+
credentialReadRefusal,
|
|
95
|
+
spawnCaptured,
|
|
96
|
+
boundaryRefusal,
|
|
97
|
+
buildSessionBoundary,
|
|
98
|
+
createSlotPool,
|
|
99
|
+
describeBoundary,
|
|
100
|
+
liveDaemonCredentials,
|
|
101
|
+
mcpRefusal,
|
|
102
|
+
mechanismSatisfies,
|
|
103
|
+
openRunPr,
|
|
104
|
+
probeHost,
|
|
105
|
+
pushRunBranch,
|
|
106
|
+
verifyDaemonAccess,
|
|
107
|
+
type HostProbe,
|
|
108
|
+
type RunRepoRef,
|
|
109
|
+
type SessionBoundary,
|
|
110
|
+
type SlotPool,
|
|
111
|
+
} from "./credentials.ts";
|
|
112
|
+
import {
|
|
113
|
+
planUsageLine,
|
|
114
|
+
readPlanUsage,
|
|
115
|
+
sharedUsageSource,
|
|
116
|
+
type PlanUsageStatus,
|
|
117
|
+
type UsageSource,
|
|
118
|
+
} from "./usage.ts";
|
|
52
119
|
|
|
53
120
|
/** Long enough that the tracker is not polled raw, short enough that a human
|
|
54
121
|
* who labels an issue sees it picked up within a coffee break. */
|
|
55
122
|
const TICK_INTERVAL_MS = 5 * 60_000;
|
|
56
123
|
const GRAPH_HEALTH_INTERVAL_MS = 60_000;
|
|
124
|
+
/** Report delivery runs on its own timer, not the five-minute dispatch tick: a
|
|
125
|
+
* report an operator is waiting on must not sit in the outbox for the length of
|
|
126
|
+
* a poll interval, and delivery is owed even while claiming is paused (#123). */
|
|
127
|
+
const REPORT_DELIVERY_INTERVAL_MS = 30_000;
|
|
57
128
|
const DEFAULT_PORT = 8787;
|
|
58
129
|
const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
|
|
59
130
|
|
|
@@ -75,16 +146,95 @@ export interface DaemonOpts {
|
|
|
75
146
|
|
|
76
147
|
/** Everything one tick touches, resolved once at startup so a tick never
|
|
77
148
|
* re-reads config mid-flight and changes its own limits underneath itself. */
|
|
149
|
+
/**
|
|
150
|
+
* The credential boundary this fleet is running behind (#125), resolved once at
|
|
151
|
+
* startup like every other dep so a tick cannot change its own protection.
|
|
152
|
+
*
|
|
153
|
+
* `paged` is the same one-page-per-episode gate the integrity tripwire uses: a
|
|
154
|
+
* host that cannot build the boundary the operator asked for holds *every*
|
|
155
|
+
* candidate on *every* tick, and paging per five minutes forever is paging
|
|
156
|
+
* nobody reads.
|
|
157
|
+
*/
|
|
158
|
+
export interface FleetBoundary {
|
|
159
|
+
isolation: CredentialIsolation;
|
|
160
|
+
probe: HostProbe;
|
|
161
|
+
slots: SlotPool;
|
|
162
|
+
paged: boolean;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* True when the operator asked for a boundary this host cannot build.
|
|
167
|
+
*
|
|
168
|
+
* Asks {@link mechanismSatisfies} rather than testing for `none`, because the
|
|
169
|
+
* interesting refusal is the near miss: a host offering `group-mode` against a
|
|
170
|
+
* `per-run` config used to dispatch happily under the weaker boundary, which is
|
|
171
|
+
* the silent downgrade #125 forbids.
|
|
172
|
+
*/
|
|
173
|
+
export function boundaryRefusesDispatch(boundary: FleetBoundary | undefined): boolean {
|
|
174
|
+
return boundary !== undefined && !mechanismSatisfies(boundary.isolation, boundary.probe.mechanism);
|
|
175
|
+
}
|
|
176
|
+
|
|
78
177
|
interface Deps {
|
|
79
178
|
project: ProjectConfig;
|
|
80
179
|
caps: Caps;
|
|
180
|
+
/**
|
|
181
|
+
* Optional only so a test can build a `Deps` without a host probe; production
|
|
182
|
+
* always resolves one in `runDaemon`. Absent means the A1 shape — sessions
|
|
183
|
+
* are still child processes, but they run as the daemon's own user and no
|
|
184
|
+
* security claim is made for them.
|
|
185
|
+
*/
|
|
186
|
+
boundary?: FleetBoundary;
|
|
81
187
|
tracker: Tracker;
|
|
82
188
|
store: Store;
|
|
189
|
+
/** Provider-reported plan allowance, cached with a TTL. Resolved once at
|
|
190
|
+
* startup like every other dep so a tick cannot swap its own meter. */
|
|
191
|
+
usage: UsageSource;
|
|
83
192
|
escalate(e: Escalation): Promise<void>;
|
|
84
193
|
turnLimits: TurnLimitRegistry;
|
|
85
194
|
integrity: IntegrityGate;
|
|
86
195
|
stall: StallGate;
|
|
87
196
|
cleanup?: RetainedCleanupCursor;
|
|
197
|
+
/**
|
|
198
|
+
* Reads the connecting uid off a verb socket (#126). Resolved once at startup
|
|
199
|
+
* so the mechanism is logged before the first socket exists; absent on a host
|
|
200
|
+
* exposing no peer-credential call, where the `0600` socket under the
|
|
201
|
+
* daemon-owned `0711` parent is the whole boundary and `status` says so.
|
|
202
|
+
*/
|
|
203
|
+
verbPeerReader?: PeerReader;
|
|
204
|
+
/**
|
|
205
|
+
* The privileged half the verbs call once their checks pass. Optional only so
|
|
206
|
+
* a test can build a `Deps` without a repository; `runDaemon` always wires
|
|
207
|
+
* the real one.
|
|
208
|
+
*/
|
|
209
|
+
verbActions?: VerbActions;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* The daemon-side view a verb call decides against (#126).
|
|
214
|
+
*
|
|
215
|
+
* `project` is a thunk, not the resolved value the rest of a tick uses. That is
|
|
216
|
+
* the whole "fail closed on an unreadable config" rule: a config read once at
|
|
217
|
+
* boot can never become unreadable, and it cannot pick up an operator who has
|
|
218
|
+
* just taken merge authority back either. Every verb pays one file read for
|
|
219
|
+
* the property that its answer reflects the config as it is *now*.
|
|
220
|
+
*
|
|
221
|
+
* `fleetStop` is read the same way and for the same reason — inside the verb,
|
|
222
|
+
* after the model decided to call it. `hold` and `halt` both set the pause
|
|
223
|
+
* sentinel alongside disarming ticks, so this one read covers both.
|
|
224
|
+
*/
|
|
225
|
+
export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbActions">): VerbDeps {
|
|
226
|
+
return {
|
|
227
|
+
project: () => findProject(loadConfig(), d.project.name),
|
|
228
|
+
store: d.store,
|
|
229
|
+
tracker: d.tracker,
|
|
230
|
+
actions: d.verbActions ?? githubVerbActions(d.project),
|
|
231
|
+
fleetStop: () =>
|
|
232
|
+
isPaused()
|
|
233
|
+
? "claiming is paused for this fleet (omp-conductor pause, hold or halt)"
|
|
234
|
+
: undefined,
|
|
235
|
+
log,
|
|
236
|
+
now: () => Date.now(),
|
|
237
|
+
};
|
|
88
238
|
}
|
|
89
239
|
|
|
90
240
|
// ---------------------------------------------------------------- paths & pause
|
|
@@ -387,16 +537,30 @@ async function safeEscalate(d: Pick<Deps, "escalate">, e: Escalation): Promise<b
|
|
|
387
537
|
* `checkIntegrity` is — this wording is the whole thing a human acts on, so it
|
|
388
538
|
* is worth a test holding it, and the sha in it is the only pointer to work
|
|
389
539
|
* that no longer has any other copy.
|
|
540
|
+
*
|
|
541
|
+
* `retained` is not cosmetic. These lines used to promise a tree "kept for
|
|
542
|
+
* inspection" unconditionally, which was true only because salvage ran only on
|
|
543
|
+
* the paths that keep one. A blocked run's tree is removed the moment its work
|
|
544
|
+
* is safely on the branch, and sending an operator to a path this process just
|
|
545
|
+
* deleted is the same class of mistake as #118 itself.
|
|
390
546
|
*/
|
|
391
|
-
export function salvageLines(
|
|
392
|
-
|
|
547
|
+
export function salvageLines(
|
|
548
|
+
outcome: SalvageOutcome,
|
|
549
|
+
worktree: string,
|
|
550
|
+
retained: boolean,
|
|
551
|
+
): string[] {
|
|
552
|
+
const fate = retained
|
|
553
|
+
? `Worktree kept for inspection: ${worktree}`
|
|
554
|
+
: `Worktree removed: ${worktree}`;
|
|
393
555
|
|
|
394
|
-
if (outcome.kind === "nothing") return [`${
|
|
556
|
+
if (outcome.kind === "nothing") return [`${fate} — nothing uncommitted to salvage`];
|
|
395
557
|
|
|
396
558
|
if (outcome.kind === "failed") {
|
|
397
559
|
return [
|
|
398
560
|
`WIP SALVAGE FAILED: ${outcome.error}`,
|
|
399
|
-
`Uncommitted work in ${worktree} is the only copy of it,
|
|
561
|
+
`Uncommitted work in ${worktree} is the only copy of it, so the tree was kept.`,
|
|
562
|
+
"This issue is held out of dispatch until the tree is recovered by hand and",
|
|
563
|
+
"`omp-conductor unblock <n> --force` records that you accepted it.",
|
|
400
564
|
];
|
|
401
565
|
}
|
|
402
566
|
|
|
@@ -415,35 +579,144 @@ export function salvageLines(outcome: SalvageOutcome, worktree: string): string[
|
|
|
415
579
|
: `${count}; new: ${outcome.newPaths.slice(0, 12).join(", ")}${
|
|
416
580
|
outcome.newPaths.length > 12 ? `, … +${outcome.newPaths.length - 12} more` : ""
|
|
417
581
|
}`;
|
|
418
|
-
return [where, manifest,
|
|
582
|
+
return [where, manifest, fate];
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
/** Everything a settled run has to record and say about its worktree. */
|
|
586
|
+
export interface WorktreeSettlement {
|
|
587
|
+
outcome: SalvageOutcome;
|
|
588
|
+
/** Whether the tree still exists now the run is over. */
|
|
589
|
+
retained: boolean;
|
|
590
|
+
/** Escalation lines naming where the work went. */
|
|
591
|
+
lines: string[];
|
|
592
|
+
/** Row fields recording the durable ref, or the failure that blocks a re-claim. */
|
|
593
|
+
patch: Pick<RunRecord, "salvageSha" | "salvageError">;
|
|
419
594
|
}
|
|
420
595
|
|
|
421
596
|
/**
|
|
422
|
-
*
|
|
423
|
-
*
|
|
597
|
+
* Decides what becomes of a finished run's worktree: save the work, then keep
|
|
598
|
+
* or remove the tree, then say which.
|
|
424
599
|
*
|
|
425
|
-
*
|
|
426
|
-
*
|
|
427
|
-
*
|
|
428
|
-
*
|
|
429
|
-
*
|
|
600
|
+
* One function because the two halves are one decision and splitting them is
|
|
601
|
+
* how #118 happened — the removal at the end of dispatch had no idea whether
|
|
602
|
+
* anything had been saved, and the salvage at the top of the failure branch had
|
|
603
|
+
* no idea the blocked branch fell through to a `--force` removal.
|
|
604
|
+
*
|
|
605
|
+
* A salvage that *fails* retains the tree whatever the caller asked for. There
|
|
606
|
+
* was real work, git refused to commit it, and the tree is now the only copy in
|
|
607
|
+
* existence: deleting it on schedule would be the data loss this whole path
|
|
608
|
+
* exists to prevent. The issue is held out of dispatch until an operator says
|
|
609
|
+
* otherwise, because the next attempt's `worktree remove --force` would finish
|
|
610
|
+
* the job (see `admitCandidates`).
|
|
611
|
+
*
|
|
612
|
+
* Exported so a test can drive the real decision against a real git tree.
|
|
430
613
|
*/
|
|
431
|
-
async function
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
614
|
+
export async function settleWorktree(
|
|
615
|
+
args: {
|
|
616
|
+
issue: number;
|
|
617
|
+
attempt: number;
|
|
618
|
+
/** Clause for the commit subject: "killed by the turns cap", "blocked …". */
|
|
619
|
+
ending: string;
|
|
620
|
+
worktree: string;
|
|
621
|
+
/** The run's branch, so the pre-removal publish names the right ref. */
|
|
622
|
+
branch: string;
|
|
623
|
+
/**
|
|
624
|
+
* Publishes the run branch on the privileged side. Required rather than
|
|
625
|
+
* optional: since #125 the run's commits live in a repository of its own,
|
|
626
|
+
* so a removal that did not publish first would delete the only copy —
|
|
627
|
+
* which is #121's data loss with one extra step. `undefined` is a visible
|
|
628
|
+
* decision at the call site, never an omission.
|
|
629
|
+
*/
|
|
630
|
+
publish: RunPublisher | undefined;
|
|
631
|
+
} & (
|
|
632
|
+
| /** Terminal-failure and orphan trees are evidence, and are kept even when clean. */
|
|
633
|
+
{ tree: "keep" }
|
|
634
|
+
| { tree: "remove"; mirrorPath: string }
|
|
635
|
+
),
|
|
636
|
+
): Promise<WorktreeSettlement> {
|
|
637
|
+
const { issue, attempt, ending, worktree, branch, publish } = args;
|
|
638
|
+
const outcome = await salvageWip(worktree, issue, attempt, ending, publish);
|
|
639
|
+
const retained = args.tree === "keep" || outcome.kind === "failed";
|
|
640
|
+
if (!retained && args.tree === "remove") {
|
|
641
|
+
// Before the removal, always — not only when salvage found something. A run
|
|
642
|
+
// that *committed* and could not publish has its work in its own repository
|
|
643
|
+
// and nowhere else, and salvage never sees a committed tree because it is
|
|
644
|
+
// clean. The mirror fetch inside `publish` is what preserves it; the push
|
|
645
|
+
// to GitHub can fail (no network, protected ref) and the work still lives.
|
|
646
|
+
const published = await publish?.(branch);
|
|
647
|
+
if (published !== undefined && !published.ok) {
|
|
648
|
+
log(`#${issue} publish before removal failed, work is preserved in the mirror: ${published.stderr}`);
|
|
649
|
+
}
|
|
650
|
+
await removeWorktree(args.mirrorPath, worktree);
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
const lines = salvageLines(outcome, worktree, retained);
|
|
438
654
|
log(`#${issue} salvage: ${lines.join(" ")}`);
|
|
439
|
-
return
|
|
655
|
+
return {
|
|
656
|
+
outcome,
|
|
657
|
+
retained,
|
|
658
|
+
lines,
|
|
659
|
+
patch:
|
|
660
|
+
outcome.kind === "salvaged"
|
|
661
|
+
? { salvageSha: outcome.sha }
|
|
662
|
+
: outcome.kind === "failed"
|
|
663
|
+
? { salvageError: outcome.error }
|
|
664
|
+
: {},
|
|
665
|
+
};
|
|
440
666
|
}
|
|
441
667
|
|
|
442
|
-
/**
|
|
668
|
+
/**
|
|
669
|
+
* The probe a session gets when no boundary was resolved at all — a test, or a
|
|
670
|
+
* `--once` run built without one. Named rather than inlined so the "no
|
|
671
|
+
* mechanism, no claim" shape is one object every call site shares.
|
|
672
|
+
*/
|
|
673
|
+
const NO_BOUNDARY_PROBE: HostProbe = {
|
|
674
|
+
mechanism: "none",
|
|
675
|
+
reasons: ["no host probe was run"],
|
|
676
|
+
residuals: [],
|
|
677
|
+
boundingSetDropped: false,
|
|
678
|
+
slots: [],
|
|
679
|
+
};
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* Credential paths the macOS profile denies outright.
|
|
683
|
+
*
|
|
684
|
+
* Derived from the operator's real home rather than from the session's
|
|
685
|
+
* redirected one: the point is that the *operator's* `gh` config and keys stay
|
|
686
|
+
* unreachable even when a session names their absolute path, which is exactly
|
|
687
|
+
* the probe `GH_CONFIG_DIR=<operator home>/.config/gh gh auth status` in #125.
|
|
688
|
+
*/
|
|
689
|
+
function credentialDenyRoots(): string[] {
|
|
690
|
+
const home = homedir();
|
|
691
|
+
return [join(home, ".ssh"), join(home, ".config", "gh"), join(home, ".gnupg"), join(home, ".aws")];
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
function credentialDenyFiles(): string[] {
|
|
695
|
+
const home = homedir();
|
|
696
|
+
return [
|
|
697
|
+
join(home, ".git-credentials"),
|
|
698
|
+
join(home, ".npmrc"),
|
|
699
|
+
join(home, ".netrc"),
|
|
700
|
+
// Both git config locations, because a token does not only live in
|
|
701
|
+
// `.git-credentials`: `url.https://<token>@github.com/.insteadOf` is a
|
|
702
|
+
// perfectly ordinary way to wire one up, and an ordinary `.gitconfig` is
|
|
703
|
+
// 0644. Hardening chmods it, but hardening is not the claim — this list is
|
|
704
|
+
// what the empirical recheck actually asks the slot principal about, so a
|
|
705
|
+
// file missing from here is a credential nobody verified was out of reach.
|
|
706
|
+
join(home, ".gitconfig"),
|
|
707
|
+
join(home, ".config", "git", "config"),
|
|
708
|
+
];
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
/**
|
|
712
|
+
* How a run's end is named — in the salvage commit, and to whoever reads it.
|
|
713
|
+
* The whole clause, not a bare reason: a graceful block was not killed by
|
|
714
|
+
* anything, and the commit subject is read during recovery.
|
|
715
|
+
*/
|
|
443
716
|
function endedBy(killedBy: KilledBy | undefined): string {
|
|
444
|
-
if (killedBy === "turns") return "the turns cap";
|
|
445
|
-
if (killedBy === "wallclock") return "the wall-clock cap";
|
|
446
|
-
return "a failed run";
|
|
717
|
+
if (killedBy === "turns") return "killed by the turns cap";
|
|
718
|
+
if (killedBy === "wallclock") return "killed by the wall-clock cap";
|
|
719
|
+
return "killed by a failed run";
|
|
447
720
|
}
|
|
448
721
|
|
|
449
722
|
/**
|
|
@@ -460,7 +733,7 @@ export async function buildBrief(
|
|
|
460
733
|
r: Routed,
|
|
461
734
|
branch: string,
|
|
462
735
|
worktree: string,
|
|
463
|
-
opts: { continuation?: boolean; defaultBranch?: string } = {},
|
|
736
|
+
opts: { continuation?: boolean; defaultBranch?: string; salvagedSha?: string } = {},
|
|
464
737
|
): Promise<string> {
|
|
465
738
|
// Read per dispatch rather than caching: editing the brief then takes effect
|
|
466
739
|
// on the next issue instead of needing a daemon restart.
|
|
@@ -474,6 +747,14 @@ export async function buildBrief(
|
|
|
474
747
|
"",
|
|
475
748
|
`You are **resuming** issue #${r.issue.number}. Branch \`${branch}\` already exists`,
|
|
476
749
|
"and was reattached with prior commits (and possibly a salvaged WIP tip).",
|
|
750
|
+
...(opts.salvagedSha === undefined
|
|
751
|
+
? []
|
|
752
|
+
: [
|
|
753
|
+
"",
|
|
754
|
+
`The previous attempt's uncommitted work was preserved for you as commit`,
|
|
755
|
+
`\`${opts.salvagedSha}\` on this branch. It is the tip you are continuing from,`,
|
|
756
|
+
"and it is the only copy of that work — do not reset past it or force-push over it.",
|
|
757
|
+
]),
|
|
477
758
|
"Before writing anything:",
|
|
478
759
|
"",
|
|
479
760
|
"```bash",
|
|
@@ -587,6 +868,33 @@ export async function verifyPushedGreenClaim(
|
|
|
587
868
|
};
|
|
588
869
|
}
|
|
589
870
|
|
|
871
|
+
/**
|
|
872
|
+
* Audit a worker's own account of its work against the pull request it pushed.
|
|
873
|
+
*
|
|
874
|
+
* The thin half of the split #85 established: this fetches, {@link
|
|
875
|
+
* analyseSettlement} decides. It runs beside {@link verifyPushedGreenClaim} and
|
|
876
|
+
* shares none of its authority — that function decides a run's state, this one
|
|
877
|
+
* cannot, by construction. It returns evidence and the caller appends it.
|
|
878
|
+
*
|
|
879
|
+
* Returns undefined when the audit could not run at all: no PR to read, or a
|
|
880
|
+
* tracker that could not produce a diff. Undefined is *not* an empty flag list.
|
|
881
|
+
* An empty list means "read the whole diff, found nothing"; undefined means
|
|
882
|
+
* "did not look", and the report says so rather than presenting silence as a
|
|
883
|
+
* clean bill.
|
|
884
|
+
*/
|
|
885
|
+
export async function collectSettlementFlags(
|
|
886
|
+
tracker: Pick<Tracker, "prDiff">,
|
|
887
|
+
claim: { prUrl?: string; report: string; issueText: string },
|
|
888
|
+
): Promise<{ flags: SettlementFlag[]; truncated: boolean } | undefined> {
|
|
889
|
+
if (claim.prUrl === undefined) return undefined;
|
|
890
|
+
const diff = await tracker.prDiff(claim.prUrl);
|
|
891
|
+
if (diff === undefined) return undefined;
|
|
892
|
+
return {
|
|
893
|
+
flags: analyseSettlement({ report: claim.report, issueText: claim.issueText, diff }),
|
|
894
|
+
truncated: diff.truncated,
|
|
895
|
+
};
|
|
896
|
+
}
|
|
897
|
+
|
|
590
898
|
/**
|
|
591
899
|
* One attempt at one issue, from claim to terminal state. Everything is inside
|
|
592
900
|
* a single try/catch so that a bad issue costs its own run and nothing else.
|
|
@@ -599,11 +907,35 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
599
907
|
|
|
600
908
|
let claimed = false;
|
|
601
909
|
let run: RunRecord | undefined;
|
|
602
|
-
// Hoisted out of the try so the catch path can still name the tree:
|
|
603
|
-
// mid-dispatch is one of the
|
|
604
|
-
//
|
|
910
|
+
// Hoisted out of the try so the catch path can still name and save the tree:
|
|
911
|
+
// a crash mid-dispatch is one of the ends whose uncommitted work has to be
|
|
912
|
+
// salvaged too, and it is the path least likely to have committed first.
|
|
913
|
+
const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
|
|
605
914
|
let worktreePath: string | undefined;
|
|
606
915
|
let turnLimit: TurnLimitController | undefined;
|
|
916
|
+
// #125: the run's own repository, the slot it holds, and the boundary its
|
|
917
|
+
// session runs behind. Hoisted for the same reason `worktreePath` is — the
|
|
918
|
+
// catch and finally paths have to release the slot and publish the branch.
|
|
919
|
+
let runRepo: RunRepoRef | undefined;
|
|
920
|
+
let slotIndex: number | undefined;
|
|
921
|
+
let boundary: SessionBoundary | undefined;
|
|
922
|
+
// #126: the run's own verb socket. Hoisted like the slot, because the catch
|
|
923
|
+
// and finally paths have to close it — a socket outliving its run is a
|
|
924
|
+
// channel nobody is authenticating any more.
|
|
925
|
+
let verbListener: VerbListener | undefined;
|
|
926
|
+
const credentials = resolveCredentials(project);
|
|
927
|
+
|
|
928
|
+
/**
|
|
929
|
+
* Publishes the run's branch on the privileged side: run repo → mirror →
|
|
930
|
+
* GitHub, fast-forward only. The worker holds no credential and never
|
|
931
|
+
* performs a network git operation itself (#125 item 3); this is the only
|
|
932
|
+
* route its commits take out, and it is also what stops a per-run repository
|
|
933
|
+
* from being the *only* copy when the tree is removed.
|
|
934
|
+
*/
|
|
935
|
+
const publish: RunPublisher = async () => {
|
|
936
|
+
if (runRepo === undefined) return { ok: false, stderr: "the run repository was never provisioned" };
|
|
937
|
+
return pushRunBranch(project, runRepo);
|
|
938
|
+
};
|
|
607
939
|
|
|
608
940
|
try {
|
|
609
941
|
// Claim on the tracker FIRST, before any local work. The label — not the
|
|
@@ -612,6 +944,10 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
612
944
|
// issue out, and a human decides what to do with the orphan.
|
|
613
945
|
await tracker.addLabel(issue, inProgress);
|
|
614
946
|
claimed = true;
|
|
947
|
+
// Read before this attempt's own row exists, so `latestRun` still means the
|
|
948
|
+
// attempt whose work this one inherits.
|
|
949
|
+
const priorSalvage = store.latestRun(project.name, issue)?.salvageSha;
|
|
950
|
+
|
|
615
951
|
|
|
616
952
|
run = store.createRun({
|
|
617
953
|
project: project.name,
|
|
@@ -629,15 +965,14 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
629
965
|
const runId = run.id;
|
|
630
966
|
turnLimit = d.turnLimits.open(project.name, issue, runId, caps.workerMaxTurns);
|
|
631
967
|
|
|
632
|
-
// A run's tree is <workspaceRoot>/<issue> and
|
|
968
|
+
// A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
|
|
633
969
|
// an existing path, so a retry — or a tree kept from a failed attempt — has
|
|
634
970
|
// to be cleared first. Both helpers are pure path math and removeWorktree
|
|
635
971
|
// tolerates a mirror or tree that is not there yet, so this is safe on a
|
|
636
|
-
// first attempt.
|
|
972
|
+
// first attempt. addRunRepo does its own ensureMirror; calling it here too
|
|
637
973
|
// would cost a second network fetch per attempt.
|
|
638
|
-
const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
|
|
639
974
|
await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
|
|
640
|
-
const provisioned = await
|
|
975
|
+
const provisioned = await addRunRepo(
|
|
641
976
|
r.repo,
|
|
642
977
|
project.mirrorRoot,
|
|
643
978
|
project.workspaceRoot,
|
|
@@ -645,12 +980,157 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
645
980
|
branch,
|
|
646
981
|
);
|
|
647
982
|
worktreePath = provisioned.path;
|
|
983
|
+
runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
|
|
648
984
|
|
|
649
985
|
// The SDK names the transcript itself, so the daemon supplies the parent
|
|
650
986
|
// directory and learns the real path back from the result. Inventing one
|
|
651
987
|
// here would put a file that never gets written into an escalation.
|
|
652
|
-
|
|
988
|
+
//
|
|
989
|
+
// Per run rather than one shared directory: under `uid-pool` the session
|
|
990
|
+
// writes its own transcript as its slot principal, and a directory shared
|
|
991
|
+
// with every other run would let each read the others'. The daemon still
|
|
992
|
+
// reads all of them — it owns the group.
|
|
993
|
+
//
|
|
994
|
+
// Under `uid-pool` the session runs as a *different uid*, so every tree it
|
|
995
|
+
// has to reach lives outside the private state directory: that one stays
|
|
996
|
+
// 0700 because it holds conductor.db and the WAL SQLite keeps recreating,
|
|
997
|
+
// and making it searchable to let a slot in would publish fleet history to
|
|
998
|
+
// every local account. `sandbox-exec` and `none` keep the old paths — same
|
|
999
|
+
// uid, nothing to traverse into (#125).
|
|
1000
|
+
const runTreeRoot = d.boundary?.probe.mechanism === "uid-pool" ? sharedRoot() : stateDir();
|
|
1001
|
+
if (runTreeRoot !== stateDir()) {
|
|
1002
|
+
mkdirSync(runTreeRoot, { recursive: true });
|
|
1003
|
+
chmodSync(runTreeRoot, 0o711);
|
|
1004
|
+
}
|
|
1005
|
+
const sessionDir = join(runTreeRoot, "sessions", `run-${String(runId)}`);
|
|
1006
|
+
const envRoot = join(runTreeRoot, "boundaries", `run-${String(runId)}`);
|
|
653
1007
|
mkdirSync(sessionDir, { recursive: true });
|
|
1008
|
+
|
|
1009
|
+
// ---- the credential boundary for this run (#125) --------------------
|
|
1010
|
+
slotIndex = d.boundary?.slots.acquire();
|
|
1011
|
+
boundary = buildSessionBoundary({
|
|
1012
|
+
probe: d.boundary?.probe ?? NO_BOUNDARY_PROBE,
|
|
1013
|
+
isolation: d.boundary?.isolation ?? "none",
|
|
1014
|
+
role: "worker",
|
|
1015
|
+
...(slotIndex === undefined ? {} : { slot: slotIndex }),
|
|
1016
|
+
envRoot,
|
|
1017
|
+
writeRoots: [worktreePath, sessionDir],
|
|
1018
|
+
// The workspace root, denied wholesale and then re-granted for this run's
|
|
1019
|
+
// own checkout by the trailing allow in `sandboxProfile`. On `uid-pool`
|
|
1020
|
+
// the modes already do this; on macOS the profile is the only thing that
|
|
1021
|
+
// does, because sandbox-exec does not change the uid.
|
|
1022
|
+
//
|
|
1023
|
+
// `mirrorRoot` is deliberately NOT denied. The run repository borrows its
|
|
1024
|
+
// objects from the mirror through git alternates, so denying it would
|
|
1025
|
+
// break every git command in the checkout — and a run being able to read
|
|
1026
|
+
// another run's objects out of that shared store is the residual this
|
|
1027
|
+
// design accepts and documents, not one it pretends to have closed. The
|
|
1028
|
+
// mirror stays unwritable, because it is not in `writeRoots`.
|
|
1029
|
+
denyReadRoots: [...credentialDenyRoots(), project.workspaceRoot],
|
|
1030
|
+
denyReadFiles: credentialDenyFiles(),
|
|
1031
|
+
...(credentials.readToken === undefined ? {} : { readToken: credentials.readToken }),
|
|
1032
|
+
});
|
|
1033
|
+
|
|
1034
|
+
if (boundary.principal !== undefined) {
|
|
1035
|
+
// Ownership handoff, then proof. `applyRunOwnership` makes the tree
|
|
1036
|
+
// `<slot>:conductor-daemon 2770`, which is also the point of no return:
|
|
1037
|
+
// if this daemon is not an *effective* member of that group it has just
|
|
1038
|
+
// locked itself out of push, salvage and reclaim, and the modes look
|
|
1039
|
+
// perfectly correct while it does. So it writes, reads and removes a
|
|
1040
|
+
// probe file rather than inspecting them (#125).
|
|
1041
|
+
// Both gids come from the probe's own `getent`, never from the daemon's
|
|
1042
|
+
// `egid`. They are different groups: the shipped unit runs
|
|
1043
|
+
// `User=fleet`/`Group=fleet` with `conductor-daemon` merely
|
|
1044
|
+
// supplementary, so `egid` is `fleet` — handing that to
|
|
1045
|
+
// `applyRunOwnership` would make every run repo `<slot>:fleet`, readable
|
|
1046
|
+
// by anyone else in `fleet`, while the probe had validated a group the
|
|
1047
|
+
// modes never used. A uid-pool probe always resolves both, so their
|
|
1048
|
+
// absence here is a contradiction rather than a default to paper over.
|
|
1049
|
+
const { daemonGid, runsGid } = d.boundary?.probe ?? {};
|
|
1050
|
+
if (daemonGid === undefined || runsGid === undefined) {
|
|
1051
|
+
throw new Error(
|
|
1052
|
+
`the credential boundary reported a per-run principal without resolving ` +
|
|
1053
|
+
`${CONDUCTOR_GROUPS.daemon}/${CONDUCTOR_GROUPS.runs} gids — refusing to hand the tree to a group ` +
|
|
1054
|
+
`nobody verified.`,
|
|
1055
|
+
);
|
|
1056
|
+
}
|
|
1057
|
+
applyRunOwnership(worktreePath, boundary.principal, daemonGid);
|
|
1058
|
+
applyRunOwnership(sessionDir, boundary.principal, daemonGid);
|
|
1059
|
+
// The session's own HOME, TMPDIR and gh-config tree.
|
|
1060
|
+
// `prepareSessionEnvRoot` creates them daemon-owned 0700, so without this
|
|
1061
|
+
// the slot principal cannot write its own home and the harness fails at
|
|
1062
|
+
// session startup — which reads like a broken install rather than a
|
|
1063
|
+
// permissions handoff that was one directory short.
|
|
1064
|
+
applyRunOwnership(envRoot, boundary.principal, daemonGid);
|
|
1065
|
+
// The claim, checked rather than assumed. On Linux the principal
|
|
1066
|
+
// separates by DAC alone, so whether the operator's credentials are
|
|
1067
|
+
// actually out of reach depends on modes this package never set. Ask the
|
|
1068
|
+
// slot itself, and refuse the dispatch if the answer is yes.
|
|
1069
|
+
const leaking = await credentialReadRefusal(
|
|
1070
|
+
spawnCaptured,
|
|
1071
|
+
boundary.principal,
|
|
1072
|
+
{ boundingSet: d.boundary?.probe.boundingSetDropped ?? false },
|
|
1073
|
+
[...credentialDenyRoots(), ...credentialDenyFiles()],
|
|
1074
|
+
);
|
|
1075
|
+
if (leaking !== undefined) throw new Error(leaking);
|
|
1076
|
+
// The other half of the layout, and it is not optional: the run's git dir
|
|
1077
|
+
// borrows the mirror's objects through `alternates`, so a mirror the slot
|
|
1078
|
+
// principal cannot read makes the checkout look corrupt rather than
|
|
1079
|
+
// forbidden.
|
|
1080
|
+
//
|
|
1081
|
+
// Both trees are checked first, and the check REFUSES rather than
|
|
1082
|
+
// widening the state directory — see `mirrorTraversalRefusal`. A slot
|
|
1083
|
+
// that cannot reach its own checkout is just as broken as one that cannot
|
|
1084
|
+
// read the mirror, and the shipped defaults put both under the private
|
|
1085
|
+
// root.
|
|
1086
|
+
const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
|
|
1087
|
+
for (const tree of [mirrorPath, worktreePath]) {
|
|
1088
|
+
const unreachable = mirrorTraversalRefusal(tree, stateDir());
|
|
1089
|
+
if (unreachable !== undefined) throw new Error(unreachable);
|
|
1090
|
+
}
|
|
1091
|
+
applyMirrorReadAccess(mirrorPath, runsGid);
|
|
1092
|
+
const denied = verifyDaemonAccess(worktreePath);
|
|
1093
|
+
if (denied !== undefined) throw new Error(denied);
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
// An MCP server carrying its own PAT re-opens the hole the principal
|
|
1097
|
+
// closes, so this is a refusal and not a warning. Audited here, after
|
|
1098
|
+
// provisioning, because both roots the harness will discover — the run's
|
|
1099
|
+
// own checkout and the agent principal's config root — exist only now.
|
|
1100
|
+
const mcp = mcpRefusal(
|
|
1101
|
+
auditMcpRoots({ agentHome: boundary.env["HOME"] ?? homedir(), cwd: worktreePath }),
|
|
1102
|
+
);
|
|
1103
|
+
if (mcp !== undefined) throw new Error(mcp);
|
|
1104
|
+
|
|
1105
|
+
// ---- the run's mutation channel (#126) -------------------------------
|
|
1106
|
+
// A shared, daemon-owned 0711 parent with one 0600 socket per run, never a
|
|
1107
|
+
// per-run *directory*: a directory owned by the run principal would hand
|
|
1108
|
+
// back the power to unlink a sibling's socket and bind an impostor in its
|
|
1109
|
+
// place, which is the whole thing the layout buys. `listenVerbChannel`
|
|
1110
|
+
// validates every component before binding and throws rather than
|
|
1111
|
+
// degrading, so a tampered path refuses this dispatch instead of running
|
|
1112
|
+
// the worker with a channel nobody can vouch for.
|
|
1113
|
+
verbListener = await listenVerbChannel(
|
|
1114
|
+
verbDeps(d),
|
|
1115
|
+
{
|
|
1116
|
+
kind: "run",
|
|
1117
|
+
// `runTreeRoot`, not the private state dir: the slot has to *connect*
|
|
1118
|
+
// to this socket, and a 0700 ancestor makes it unreachable however
|
|
1119
|
+
// correct the socket's own mode is. `ensureVerbSocketDir` still owns
|
|
1120
|
+
// the parent and still runs its anti-rebind checks there (#126).
|
|
1121
|
+
path: verbSocketPath(ensureVerbSocketDir(runTreeRoot), `run-${String(issue)}`),
|
|
1122
|
+
project: project.name,
|
|
1123
|
+
role: "worker",
|
|
1124
|
+
runId,
|
|
1125
|
+
issue,
|
|
1126
|
+
repo: r.repo,
|
|
1127
|
+
runRepoPath: worktreePath,
|
|
1128
|
+
branch,
|
|
1129
|
+
...(boundary.principal === undefined ? {} : { principal: boundary.principal }),
|
|
1130
|
+
},
|
|
1131
|
+
{ ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }) },
|
|
1132
|
+
);
|
|
1133
|
+
|
|
654
1134
|
store.updateRun(runId, { worktree: worktreePath, state: "running" });
|
|
655
1135
|
|
|
656
1136
|
log(
|
|
@@ -664,13 +1144,27 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
664
1144
|
brief: await buildBrief(project, r, branch, worktreePath, {
|
|
665
1145
|
continuation: provisioned.reattached,
|
|
666
1146
|
defaultBranch: r.repo.defaultBranch,
|
|
1147
|
+
...(provisioned.reattached && priorSalvage !== undefined
|
|
1148
|
+
? { salvagedSha: priorSalvage }
|
|
1149
|
+
: {}),
|
|
667
1150
|
}),
|
|
668
1151
|
cwd: worktreePath,
|
|
669
1152
|
caps,
|
|
670
1153
|
maxTurns: () => turnLimit?.maxTurns() ?? caps.workerMaxTurns,
|
|
671
1154
|
sessionDir,
|
|
1155
|
+
boundary,
|
|
1156
|
+
// Inside the run's own boundary root, so a slot principal can reach it
|
|
1157
|
+
// by traversal. A socket under the daemon's 0700 home would be
|
|
1158
|
+
// unreachable to the very process it exists for (#125).
|
|
1159
|
+
// Derived from the hoisted envRoot for the same reason, and because
|
|
1160
|
+
// that tree has already been handed to the slot principal.
|
|
1161
|
+
socketPath: join(envRoot, "ipc.sock"),
|
|
1162
|
+
verbSocketPath: verbListener.path,
|
|
1163
|
+
onChildLog: (line) => {
|
|
1164
|
+
log(`#${issue} ${line}`);
|
|
1165
|
+
},
|
|
672
1166
|
...(project.workerModel === undefined ? {} : { model: project.workerModel }),
|
|
673
|
-
|
|
1167
|
+
releaseGrants: resolveReleaseGrants(project),
|
|
674
1168
|
onReleaseBlocked: (shape) => recordReleaseBlock(project.name, "worker", shape),
|
|
675
1169
|
onTurn: (n) => store.updateRun(runId, { turns: n }),
|
|
676
1170
|
onSpend: (usd) => store.updateRun(runId, { spendUsd: usd }),
|
|
@@ -689,6 +1183,14 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
689
1183
|
// PR verification or terminal row writes can leave stale `running` state.
|
|
690
1184
|
turnLimit?.close();
|
|
691
1185
|
turnLimit = undefined;
|
|
1186
|
+
// The slot IS the principal, so holding it past the session means the
|
|
1187
|
+
// next run could be handed the same uid while this one's tree still
|
|
1188
|
+
// exists — two live runs able to write each other's checkout, which is
|
|
1189
|
+
// the cross-run property this whole change buys.
|
|
1190
|
+
if (slotIndex !== undefined) {
|
|
1191
|
+
d.boundary?.slots.release(slotIndex);
|
|
1192
|
+
slotIndex = undefined;
|
|
1193
|
+
}
|
|
692
1194
|
}
|
|
693
1195
|
|
|
694
1196
|
// A configured model the harness could not honour means this run was done by
|
|
@@ -703,8 +1205,67 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
703
1205
|
? await verifyPushedGreenClaim(tracker, result)
|
|
704
1206
|
: { state: result.state };
|
|
705
1207
|
const state = verified.state;
|
|
706
|
-
|
|
707
|
-
|
|
1208
|
+
|
|
1209
|
+
// The other half of not believing a worker about its own run (#128). The
|
|
1210
|
+
// claim being audited is `state: pushed-green`, so the audit runs on the
|
|
1211
|
+
// worker's claim rather than on what verification made of it: a claim that
|
|
1212
|
+
// GitHub then contradicted is exactly the report whose `changed:` line is
|
|
1213
|
+
// worth reading twice. Advisory throughout — nothing below reads `flags`
|
|
1214
|
+
// when deciding `state`, and the ordering here makes that checkable.
|
|
1215
|
+
const audit =
|
|
1216
|
+
result.state === "pushed-green"
|
|
1217
|
+
? await collectSettlementFlags(tracker, {
|
|
1218
|
+
prUrl: result.prUrl,
|
|
1219
|
+
report: result.report,
|
|
1220
|
+
issueText: `${r.issue.title}\n${r.issue.body}`,
|
|
1221
|
+
})
|
|
1222
|
+
: undefined;
|
|
1223
|
+
if (result.state === "pushed-green") {
|
|
1224
|
+
if (audit === undefined) log(`#${issue} settlement audit skipped: no readable PR diff`);
|
|
1225
|
+
else if (audit.truncated) log(`#${issue} settlement audit read a truncated PR diff`);
|
|
1226
|
+
}
|
|
1227
|
+
const auditLines =
|
|
1228
|
+
audit === undefined ? [] : formatSettlementFlags(audit.flags, { truncated: audit.truncated });
|
|
1229
|
+
|
|
1230
|
+
const finalReport = [
|
|
1231
|
+
...(verified.reason === undefined ? [] : [verified.reason, ""]),
|
|
1232
|
+
...(auditLines.length === 0 ? [] : [...auditLines, ""]),
|
|
1233
|
+
result.report,
|
|
1234
|
+
].join("\n");
|
|
1235
|
+
|
|
1236
|
+
// What becomes of the tree, decided once, before any label or page. A
|
|
1237
|
+
// `pushed-*` run is the only end that does not salvage: its deliverable is
|
|
1238
|
+
// already on a remote branch, whatever is left loose in the tree is by the
|
|
1239
|
+
// worker's own account not part of it, and appending a WIP commit would
|
|
1240
|
+
// turn the green PR this daemon just verified red. Every other end is
|
|
1241
|
+
// continuable, so its tree is treated as work.
|
|
1242
|
+
const settlement =
|
|
1243
|
+
state === "pushed-green" || state === "pushed-pending" || state === "merged"
|
|
1244
|
+
? undefined
|
|
1245
|
+
: await settleWorktree({
|
|
1246
|
+
issue,
|
|
1247
|
+
attempt,
|
|
1248
|
+
ending:
|
|
1249
|
+
state === "blocked" ? "blocked for an operator decision" : endedBy(result.killedBy),
|
|
1250
|
+
worktree: worktreePath,
|
|
1251
|
+
branch,
|
|
1252
|
+
publish,
|
|
1253
|
+
...(state === "failed" || state === "killed"
|
|
1254
|
+
? ({ tree: "keep" } as const)
|
|
1255
|
+
: ({ tree: "remove", mirrorPath } as const)),
|
|
1256
|
+
});
|
|
1257
|
+
if (settlement === undefined) {
|
|
1258
|
+
// A `pushed-*` end does not salvage — its deliverable is already on a
|
|
1259
|
+
// remote branch — but its repository still goes away here, so the branch
|
|
1260
|
+
// is published first for the same reason `settleWorktree` does it: the
|
|
1261
|
+
// run repo is no longer a view of the mirror, and anything it holds that
|
|
1262
|
+
// never reached the mirror dies with the directory.
|
|
1263
|
+
const published = await publish(branch);
|
|
1264
|
+
if (!published.ok) {
|
|
1265
|
+
log(`#${issue} publish before removal failed, work is preserved in the mirror: ${published.stderr}`);
|
|
1266
|
+
}
|
|
1267
|
+
await removeWorktree(mirrorPath, worktreePath);
|
|
1268
|
+
}
|
|
708
1269
|
|
|
709
1270
|
store.updateRun(runId, {
|
|
710
1271
|
state,
|
|
@@ -715,8 +1276,14 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
715
1276
|
headSha: result.headSha,
|
|
716
1277
|
sessionFile: result.sessionFile,
|
|
717
1278
|
...(verified.reason === undefined ? {} : { lastError: verified.reason }),
|
|
1279
|
+
...settlement?.patch,
|
|
1280
|
+
...(audit === undefined || audit.flags.length === 0
|
|
1281
|
+
? {}
|
|
1282
|
+
: { settlementFlags: audit.flags }),
|
|
718
1283
|
});
|
|
719
1284
|
|
|
1285
|
+
const salvaged = settlement?.lines ?? [];
|
|
1286
|
+
|
|
720
1287
|
if (state === "blocked") {
|
|
721
1288
|
await swapLabel(tracker, issue, inProgress, project.stateLabels.blocked);
|
|
722
1289
|
await safeEscalate(d, {
|
|
@@ -725,7 +1292,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
725
1292
|
issue,
|
|
726
1293
|
runId,
|
|
727
1294
|
summary: `#${issue} is blocked on attempt ${attempt} and needs a decision`,
|
|
728
|
-
detail: [`${r.issue.title}`, r.issue.url, "", result.report].join("\n"),
|
|
1295
|
+
detail: [`${r.issue.title}`, r.issue.url, "", ...salvaged, "", result.report].join("\n"),
|
|
729
1296
|
});
|
|
730
1297
|
} else if (state === "failed" || state === "killed") {
|
|
731
1298
|
// A turns cap consumes the independent continuation budget, not an
|
|
@@ -735,7 +1302,6 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
735
1302
|
const continueTurns =
|
|
736
1303
|
result.killedBy === "turns" &&
|
|
737
1304
|
hasContinuationBudget(continuation, caps.maxContinuationsPerIssue);
|
|
738
|
-
const salvaged = await salvage(issue, attempt, endedBy(result.killedBy), worktreePath);
|
|
739
1305
|
|
|
740
1306
|
if (continueTurns) {
|
|
741
1307
|
await tracker.removeLabel(issue, inProgress);
|
|
@@ -789,13 +1355,45 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
789
1355
|
// A verified or still-pending PR keeps the in-progress label until its
|
|
790
1356
|
// checks or merge settle, preventing another worker from duplicating it.
|
|
791
1357
|
log(`#${issue} ${state}${result.prUrl ? ` ${result.prUrl}` : ""}`);
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
1358
|
+
// A green run's report is the only one nothing ever delivers: it raises no
|
|
1359
|
+
// escalation, and the row it settles into says nothing about what the
|
|
1360
|
+
// worker wrote. That is fine while the report is believed — and it is
|
|
1361
|
+
// exactly wrong once something has stopped believing it. A flagged
|
|
1362
|
+
// settlement is the one case where a `pushed-green` run has news, and it
|
|
1363
|
+
// has it *now*, while the PR is still open and nobody has merged it (#128).
|
|
1364
|
+
//
|
|
1365
|
+
// Tier 1, because the judgement is the orchestrator's: it can read the
|
|
1366
|
+
// diff, it can ask, and it can promote to a human. The state is
|
|
1367
|
+
// deliberately unchanged either way — the run below this line settles
|
|
1368
|
+
// `pushed-green` whether it was flagged or not.
|
|
1369
|
+
//
|
|
1370
|
+
// Findings only. A diff too large to read in full is noted in the log and
|
|
1371
|
+
// in the report, but it is not news for a human: paging "0 flags, could
|
|
1372
|
+
// not read it all" is exactly the kind of line that teaches an
|
|
1373
|
+
// orchestrator to skim past this escalation the next time it carries one.
|
|
1374
|
+
const found = audit?.flags ?? [];
|
|
1375
|
+
if (found.length > 0) {
|
|
1376
|
+
await safeEscalate(d, {
|
|
1377
|
+
tier: 1,
|
|
1378
|
+
project: project.name,
|
|
1379
|
+
issue,
|
|
1380
|
+
runId,
|
|
1381
|
+
summary:
|
|
1382
|
+
`#${issue} settled ${state} on attempt ${attempt} with ` +
|
|
1383
|
+
`${found.length} settlement audit flag(s)`,
|
|
1384
|
+
detail: [
|
|
1385
|
+
`${r.issue.title}`,
|
|
1386
|
+
r.issue.url,
|
|
1387
|
+
result.prUrl ?? "(no PR URL)",
|
|
1388
|
+
"",
|
|
1389
|
+
"The run is NOT blocked and its state is unchanged. These are advisory",
|
|
1390
|
+
"findings about the worker's own account of its work — judge them, then",
|
|
1391
|
+
"merge, ask, or close as you would have anyway.",
|
|
1392
|
+
"",
|
|
1393
|
+
finalReport,
|
|
1394
|
+
].join("\n"),
|
|
1395
|
+
});
|
|
1396
|
+
}
|
|
799
1397
|
}
|
|
800
1398
|
} catch (err) {
|
|
801
1399
|
// Dispatch setup can fail after the controller opens but before runWorker's
|
|
@@ -804,8 +1402,34 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
804
1402
|
turnLimit = undefined;
|
|
805
1403
|
const detail = errText(err);
|
|
806
1404
|
log(`#${issue} errored: ${detail}`);
|
|
1405
|
+
// Released here as well: a dispatch that failed before `runWorker` never
|
|
1406
|
+
// reached the `finally` above, and a leaked slot permanently shrinks the
|
|
1407
|
+
// pool until the daemon restarts.
|
|
1408
|
+
if (slotIndex !== undefined) {
|
|
1409
|
+
d.boundary?.slots.release(slotIndex);
|
|
1410
|
+
slotIndex = undefined;
|
|
1411
|
+
}
|
|
1412
|
+
// A crash lands anywhere, including mid-edit in a tree holding the only
|
|
1413
|
+
// copy of real work. Nothing else on this path so much as looks at it.
|
|
1414
|
+
const settlement =
|
|
1415
|
+
worktreePath === undefined
|
|
1416
|
+
? undefined
|
|
1417
|
+
: await settleWorktree({
|
|
1418
|
+
issue,
|
|
1419
|
+
attempt,
|
|
1420
|
+
ending: "killed by a dispatch error",
|
|
1421
|
+
worktree: worktreePath,
|
|
1422
|
+
branch,
|
|
1423
|
+
publish,
|
|
1424
|
+
tree: "keep",
|
|
1425
|
+
});
|
|
807
1426
|
if (run) {
|
|
808
|
-
store.updateRun(run.id, {
|
|
1427
|
+
store.updateRun(run.id, {
|
|
1428
|
+
state: "failed",
|
|
1429
|
+
endedAt: Date.now(),
|
|
1430
|
+
lastError: detail,
|
|
1431
|
+
...settlement?.patch,
|
|
1432
|
+
});
|
|
809
1433
|
}
|
|
810
1434
|
if (claimed) {
|
|
811
1435
|
// Leaving the issue stuck as in-progress would hide it from both the
|
|
@@ -816,13 +1440,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
816
1440
|
log(`#${issue} could not be relabelled: ${errText(relabelErr)}`);
|
|
817
1441
|
}
|
|
818
1442
|
}
|
|
819
|
-
// A crash lands anywhere, including mid-edit in a tree holding the only
|
|
820
|
-
// copy of real work. Nothing else on this path so much as looks at it.
|
|
821
|
-
const salvaged =
|
|
822
|
-
worktreePath === undefined
|
|
823
|
-
? []
|
|
824
|
-
: await salvage(issue, attempt, "a dispatch error", worktreePath);
|
|
825
1443
|
|
|
1444
|
+
const salvaged = settlement?.lines ?? [];
|
|
826
1445
|
await safeEscalate(d, {
|
|
827
1446
|
tier: 1,
|
|
828
1447
|
project: project.name,
|
|
@@ -835,6 +1454,18 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
835
1454
|
// a failure path, and whatever it still held is now a commit on the branch.
|
|
836
1455
|
} finally {
|
|
837
1456
|
turnLimit?.close();
|
|
1457
|
+
// The run is over, so its channel is too. Closed here rather than beside
|
|
1458
|
+
// the session so the crash path closes it as well: a listener left bound
|
|
1459
|
+
// after its run settled is a socket whose `run-not-live` check is the only
|
|
1460
|
+
// thing standing between a stale child and a push.
|
|
1461
|
+
if (verbListener !== undefined) {
|
|
1462
|
+
try {
|
|
1463
|
+
await verbListener.close();
|
|
1464
|
+
} catch (err) {
|
|
1465
|
+
log(`#${issue} verb socket ${verbListener.path} did not close cleanly: ${errText(err)}`);
|
|
1466
|
+
}
|
|
1467
|
+
verbListener = undefined;
|
|
1468
|
+
}
|
|
838
1469
|
}
|
|
839
1470
|
}
|
|
840
1471
|
|
|
@@ -1178,6 +1809,74 @@ export function summarizeDispatch(
|
|
|
1178
1809
|
};
|
|
1179
1810
|
}
|
|
1180
1811
|
|
|
1812
|
+
/**
|
|
1813
|
+
* What a held plan-usage gate says to a human, if anything.
|
|
1814
|
+
*
|
|
1815
|
+
* Three different problems hide behind one hold, and they want different
|
|
1816
|
+
* tiers. Reaching the threshold is the guard *working*: tier 1, because the
|
|
1817
|
+
* fleet resumes on its own at the provider's reset and nobody needs to get
|
|
1818
|
+
* out of bed. Everything else — a window nothing reports, a window that
|
|
1819
|
+
* resolves to two allowances, a meter that has been unreadable for half an
|
|
1820
|
+
* hour — is dispatch stopped with no self-recovery, which is tier 2.
|
|
1821
|
+
*
|
|
1822
|
+
* Each summary carries the fact that will change when the situation does (the
|
|
1823
|
+
* reset instant, the configured id, the date), because the escalation ledger
|
|
1824
|
+
* dedupes on the summary: a stable one pages once and then goes quiet, which
|
|
1825
|
+
* is right for a repeated tick and wrong for the next window.
|
|
1826
|
+
*/
|
|
1827
|
+
function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation | undefined {
|
|
1828
|
+
const base = { project, issue: NO_ISSUE };
|
|
1829
|
+
if (plan.state === "at-cap") {
|
|
1830
|
+
const window = plan.window?.id ?? plan.cap?.windowId ?? "the configured window";
|
|
1831
|
+
const resets =
|
|
1832
|
+
plan.resetsAt === undefined
|
|
1833
|
+
? new Date().toISOString().slice(0, 10)
|
|
1834
|
+
: new Date(plan.resetsAt).toISOString();
|
|
1835
|
+
return {
|
|
1836
|
+
...base,
|
|
1837
|
+
tier: 1,
|
|
1838
|
+
summary: `Plan allowance cap reached on ${window} — ${project} is not claiming new work (window ${resets})`,
|
|
1839
|
+
detail: [
|
|
1840
|
+
plan.detail,
|
|
1841
|
+
"Running workers finish normally; only new claims are held.",
|
|
1842
|
+
"Dispatch resumes by itself once the provider reports the window reset or usage below the threshold —",
|
|
1843
|
+
"no `resume` needed. Raise `caps.planUsage.maxUsedFraction` only if you mean to spend the rest.",
|
|
1844
|
+
].join("\n"),
|
|
1845
|
+
};
|
|
1846
|
+
}
|
|
1847
|
+
if (plan.state === "blind") {
|
|
1848
|
+
return {
|
|
1849
|
+
...base,
|
|
1850
|
+
tier: 2,
|
|
1851
|
+
// Dated: a meter that breaks again next month is a new incident, not a
|
|
1852
|
+
// repeat of this one.
|
|
1853
|
+
summary: `Plan usage source unreadable — ${project} is not claiming new work (${new Date().toISOString().slice(0, 10)})`,
|
|
1854
|
+
detail: [
|
|
1855
|
+
plan.detail,
|
|
1856
|
+
"The guard admitted work while the failure looked transient and has now stopped.",
|
|
1857
|
+
"Check `omp usage --json` on the fleet host, or set `caps.planUsage` to null if this fleet is unmetered.",
|
|
1858
|
+
].join("\n"),
|
|
1859
|
+
};
|
|
1860
|
+
}
|
|
1861
|
+
if (
|
|
1862
|
+
plan.state === "window-missing" ||
|
|
1863
|
+
plan.state === "window-ambiguous" ||
|
|
1864
|
+
plan.state === "window-uncomparable"
|
|
1865
|
+
) {
|
|
1866
|
+
return {
|
|
1867
|
+
...base,
|
|
1868
|
+
tier: 2,
|
|
1869
|
+
summary: `Plan usage cap names an unusable window "${plan.cap?.windowId ?? "?"}" — ${project} is not claiming new work`,
|
|
1870
|
+
detail: [
|
|
1871
|
+
plan.detail,
|
|
1872
|
+
"Run `omp usage --json` and copy an allowance `id` into `caps.planUsage.windowId`,",
|
|
1873
|
+
"or set `caps.planUsage` to null if this fleet is unmetered.",
|
|
1874
|
+
].join("\n"),
|
|
1875
|
+
};
|
|
1876
|
+
}
|
|
1877
|
+
return undefined;
|
|
1878
|
+
}
|
|
1879
|
+
|
|
1181
1880
|
/**
|
|
1182
1881
|
* Which routed candidates get a worker this tick — in queue order, never more
|
|
1183
1882
|
* than `slots` of them. Every non-admission receives a stable reason code.
|
|
@@ -1192,7 +1891,7 @@ export function summarizeDispatch(
|
|
|
1192
1891
|
* that grows a field has no business breaking these tests.
|
|
1193
1892
|
*/
|
|
1194
1893
|
export async function admitCandidates(
|
|
1195
|
-
d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate">,
|
|
1894
|
+
d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate" | "usage" | "boundary">,
|
|
1196
1895
|
routed: Routed[],
|
|
1197
1896
|
slots: number,
|
|
1198
1897
|
): Promise<AdmissionPass> {
|
|
@@ -1204,6 +1903,50 @@ export async function admitCandidates(
|
|
|
1204
1903
|
holds.push({ issue, reason });
|
|
1205
1904
|
};
|
|
1206
1905
|
|
|
1906
|
+
// The fail-closed the issue asks for, and it only ever fires because the
|
|
1907
|
+
// operator asked for a boundary by name. Placed above every other gate,
|
|
1908
|
+
// including the tracker calls: a fleet that must not dispatch must not spend
|
|
1909
|
+
// API budget discovering it, and it must not claim an issue it will refuse.
|
|
1910
|
+
//
|
|
1911
|
+
// Deliberately NOT a downgrade to group-mode. That is a different, weaker
|
|
1912
|
+
// claim, and silently substituting it is how an operator ends up believing
|
|
1913
|
+
// they have a boundary they do not have (#125).
|
|
1914
|
+
if (boundaryRefusesDispatch(d.boundary) && d.boundary !== undefined) {
|
|
1915
|
+
const refusal = boundaryRefusal(d.boundary.probe);
|
|
1916
|
+
for (const r of routed) hold(r.issue.number, "credential-boundary");
|
|
1917
|
+
log(`credential boundary holding ${String(routed.length)} candidate(s): ${refusal}`);
|
|
1918
|
+
if (!d.boundary.paged) {
|
|
1919
|
+
d.boundary.paged = true;
|
|
1920
|
+
await safeEscalate(d, {
|
|
1921
|
+
tier: 2,
|
|
1922
|
+
project: project.name,
|
|
1923
|
+
issue: NO_ISSUE,
|
|
1924
|
+
summary: `${project.name} cannot dispatch: credentials.isolation is "per-run" and this host offers no mechanism`,
|
|
1925
|
+
detail: [refusal, "", "Nothing is dispatched until this is resolved. No work has been claimed."].join("\n"),
|
|
1926
|
+
});
|
|
1927
|
+
}
|
|
1928
|
+
return { admitted: [], holds };
|
|
1929
|
+
}
|
|
1930
|
+
|
|
1931
|
+
// The plan allowance is a fleet-wide question, so it is asked once per pass
|
|
1932
|
+
// and answers for every candidate — unlike every gate below it, which is
|
|
1933
|
+
// per-issue. It sits here rather than beside the spend cap in `tick` for one
|
|
1934
|
+
// reason: the spend cap *pauses the daemon* and waits for a human, and a
|
|
1935
|
+
// weekly plan window resets by itself. A guard that demanded `resume` after
|
|
1936
|
+
// every rollover would cost more operator attention than the guard saves
|
|
1937
|
+
// (#110). Already-running workers are untouched and settle normally.
|
|
1938
|
+
//
|
|
1939
|
+
// Placed after the cheap local busy-set read and before the first tracker
|
|
1940
|
+
// call, so a held fleet spends no GitHub API budget discovering it is held.
|
|
1941
|
+
const plan = await readPlanUsage(caps.planUsage, d.usage);
|
|
1942
|
+
if (plan.blocking) {
|
|
1943
|
+
for (const r of routed) hold(r.issue.number, "plan-usage-cap");
|
|
1944
|
+
log(`plan usage gate holding ${String(routed.length)} candidate(s): ${plan.detail}`);
|
|
1945
|
+
const escalation = planUsageEscalation(project.name, plan);
|
|
1946
|
+
if (escalation !== undefined) await safeEscalate(d, escalation);
|
|
1947
|
+
return { admitted: [], holds };
|
|
1948
|
+
}
|
|
1949
|
+
|
|
1207
1950
|
// parent -> blocking issue. Seeded from active runs (including pushed-green),
|
|
1208
1951
|
// then extended by candidates admitted earlier in this same pass so two
|
|
1209
1952
|
// siblings never both clear the gate in one tick.
|
|
@@ -1278,6 +2021,32 @@ export async function admitCandidates(
|
|
|
1278
2021
|
continue;
|
|
1279
2022
|
}
|
|
1280
2023
|
|
|
2024
|
+
// Fail closed on work that exists only in a run repo. `addRunRepo` clears
|
|
2025
|
+
// the tree at <workspaceRoot>/<issue> before it provisions, so admitting
|
|
2026
|
+
// this issue is what finally destroys the copy the salvage could not save
|
|
2027
|
+
// (#118). Nothing here can recover it — git already refused once — so the
|
|
2028
|
+
// only safe move is to refuse the claim and keep saying why until an
|
|
2029
|
+
// operator has looked and run `unblock --force`.
|
|
2030
|
+
const newest = store.latestRun(project.name, issue);
|
|
2031
|
+
if (newest?.salvageError !== undefined && newest.salvageAckAt === undefined) {
|
|
2032
|
+
hold(issue, "unsalvaged-wip");
|
|
2033
|
+
await safeEscalate(d, {
|
|
2034
|
+
tier: 1,
|
|
2035
|
+
project: project.name,
|
|
2036
|
+
issue,
|
|
2037
|
+
summary: `#${issue} is holding unsalvaged work and will not be re-claimed`,
|
|
2038
|
+
detail: [
|
|
2039
|
+
r.issue.title,
|
|
2040
|
+
r.issue.url,
|
|
2041
|
+
`Attempt ${newest.attempt} could not commit its uncommitted changes: ${newest.salvageError}`,
|
|
2042
|
+
`The only copy is the worktree ${newest.worktree === "" ? "(path not recorded)" : newest.worktree}.`,
|
|
2043
|
+
"Dispatch is held because claiming this issue removes that tree.",
|
|
2044
|
+
"Recover it by hand, then `omp-conductor unblock <n> --force` to release the hold.",
|
|
2045
|
+
].join("\n"),
|
|
2046
|
+
});
|
|
2047
|
+
continue;
|
|
2048
|
+
}
|
|
2049
|
+
|
|
1281
2050
|
// Soft concurrency per epic: at most one in-flight child of a given parent.
|
|
1282
2051
|
// No parent means today's concurrent admission. Cheap local filters already
|
|
1283
2052
|
// ran; this sits before the open-PR API call so a held sibling frees the
|
|
@@ -1666,25 +2435,83 @@ export async function turnLimitResponse(
|
|
|
1666
2435
|
);
|
|
1667
2436
|
}
|
|
1668
2437
|
|
|
2438
|
+
export interface DaemonHttpDeps {
|
|
2439
|
+
project: string;
|
|
2440
|
+
store: Pick<Store, "latestRun">;
|
|
2441
|
+
turnLimits: TurnLimitRegistry;
|
|
2442
|
+
health: () => DaemonHealthSnapshot;
|
|
2443
|
+
}
|
|
2444
|
+
|
|
2445
|
+
/**
|
|
2446
|
+
* The whole HTTP surface, in one named function so a test can pin what is *not*
|
|
2447
|
+
* on it.
|
|
2448
|
+
*
|
|
2449
|
+
* Two routes: a health read, and the turn-limit control. Everything else is
|
|
2450
|
+
* 404, and that is the contract — see the prohibition at the `Bun.serve` call
|
|
2451
|
+
* for why no mutation may be added here. Extracted from the serve callback
|
|
2452
|
+
* precisely so "the daemon's HTTP port exposes no mutation route" is something
|
|
2453
|
+
* a test asserts rather than something a reviewer has to notice (#126).
|
|
2454
|
+
*/
|
|
2455
|
+
export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promise<Response> {
|
|
2456
|
+
const url = new URL(req.url);
|
|
2457
|
+
if (req.method === "GET" && url.pathname === "/healthz") return Response.json(d.health());
|
|
2458
|
+
const control = await turnLimitResponse(req, d.project, d.store, d.turnLimits);
|
|
2459
|
+
return control ?? new Response("not found\n", { status: 404 });
|
|
2460
|
+
}
|
|
2461
|
+
|
|
1669
2462
|
export interface StatusSnapshot {
|
|
1670
2463
|
project: string;
|
|
1671
2464
|
configPath: string;
|
|
1672
2465
|
stateDir: string;
|
|
1673
2466
|
paused: boolean;
|
|
1674
2467
|
caps: Caps;
|
|
2468
|
+
/**
|
|
2469
|
+
* The effective per-shape release grants. On the snapshot rather than re-read
|
|
2470
|
+
* by each renderer because #122 began with a grant nobody had looked at in
|
|
2471
|
+
* weeks: `status` names them so a stale one is visible without opening either
|
|
2472
|
+
* the config or the brief.
|
|
2473
|
+
*/
|
|
2474
|
+
releaseGrants: ResolvedGrants;
|
|
1675
2475
|
/** Occupied issues: live workers plus green PRs awaiting a human merge. */
|
|
1676
2476
|
activeRuns: RunRecord[];
|
|
2477
|
+
/** Newest attempts holding a preserved WIP tip, or a tree that is still the
|
|
2478
|
+
* only copy of work the daemon could not save. */
|
|
2479
|
+
salvagedRuns: RunRecord[];
|
|
2480
|
+
/** Reports the operator has not provably received: pending, in-flight with an
|
|
2481
|
+
* unknown outcome, or written off. An empty list is the only honest way to
|
|
2482
|
+
* say "everything authored this cycle actually went out" (#123). */
|
|
2483
|
+
openReports: ReportRecord[];
|
|
2484
|
+
/**
|
|
2485
|
+
* The most recent conductor-verb calls and how the daemon decided them
|
|
2486
|
+
* (#126). On `status` rather than only behind `omp-conductor ledger` because
|
|
2487
|
+
* a refused merge is news: it means a session tried to do something the
|
|
2488
|
+
* config does not let it, and an operator who has to know to go looking is an
|
|
2489
|
+
* operator who finds out from the tracker instead.
|
|
2490
|
+
*/
|
|
2491
|
+
verbLedger: VerbLedgerEntry[];
|
|
1677
2492
|
/** Runs backed by a worker process — the number capacity compares against. */
|
|
1678
2493
|
liveWorkers: number;
|
|
1679
2494
|
runsToday: number;
|
|
1680
2495
|
spendTodayUsd: number;
|
|
1681
2496
|
dispatch?: DispatchSummary;
|
|
2497
|
+
/**
|
|
2498
|
+
* Latest plan-allowance verdict, when the caller read one. Optional because
|
|
2499
|
+
* this snapshot is built synchronously off the store while the provider read
|
|
2500
|
+
* is I/O: a renderer that did not do that work must say "not read" rather
|
|
2501
|
+
* than print a percentage nobody measured (#110).
|
|
2502
|
+
*/
|
|
2503
|
+
planUsage?: PlanUsageStatus;
|
|
1682
2504
|
}
|
|
1683
2505
|
|
|
1684
2506
|
/** Builds a status reading from an already-open store. Long-lived operator
|
|
1685
2507
|
* surfaces use this path so a one-second refresh does not repeatedly open and
|
|
1686
2508
|
* initialise SQLite connections. */
|
|
1687
|
-
export function statusSnapshotFromStore(
|
|
2509
|
+
export function statusSnapshotFromStore(
|
|
2510
|
+
p: ProjectConfig,
|
|
2511
|
+
caps: Caps,
|
|
2512
|
+
store: Store,
|
|
2513
|
+
planUsage?: PlanUsageStatus,
|
|
2514
|
+
): StatusSnapshot {
|
|
1688
2515
|
const since = startOfToday();
|
|
1689
2516
|
const dispatch = store.latestDispatch(p.name);
|
|
1690
2517
|
return {
|
|
@@ -1693,11 +2520,16 @@ export function statusSnapshotFromStore(p: ProjectConfig, caps: Caps, store: Sto
|
|
|
1693
2520
|
stateDir: stateDir(),
|
|
1694
2521
|
paused: isPaused(),
|
|
1695
2522
|
caps,
|
|
2523
|
+
releaseGrants: resolveReleaseGrants(p),
|
|
1696
2524
|
activeRuns: store.activeRuns(p.name),
|
|
2525
|
+
salvagedRuns: store.salvagedRuns(p.name),
|
|
2526
|
+
openReports: store.openReports(p.name),
|
|
2527
|
+
verbLedger: store.verbLedger(p.name, { limit: STATUS_LEDGER_SCAN }),
|
|
1697
2528
|
liveWorkers: store.liveRuns(p.name).length,
|
|
1698
2529
|
runsToday: store.runsStartedSince(p.name, since),
|
|
1699
2530
|
spendTodayUsd: store.spendSince(p.name, since),
|
|
1700
2531
|
...(dispatch === undefined ? {} : { dispatch }),
|
|
2532
|
+
...(planUsage === undefined ? {} : { planUsage }),
|
|
1701
2533
|
};
|
|
1702
2534
|
}
|
|
1703
2535
|
|
|
@@ -1737,6 +2569,50 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
|
|
|
1737
2569
|
return lines.join("\n");
|
|
1738
2570
|
}
|
|
1739
2571
|
|
|
2572
|
+
/**
|
|
2573
|
+
* The WIP block: every issue whose newest attempt left work behind, and
|
|
2574
|
+
* whether that work is safe.
|
|
2575
|
+
*
|
|
2576
|
+
* Blocked runs used to be invisible here, which is exactly how #118 stayed
|
|
2577
|
+
* invisible for a full attempt cycle — the operator saw a blocked issue and had
|
|
2578
|
+
* no way to tell "stopped with 34 uncommitted files" from "stopped clean".
|
|
2579
|
+
* A preserved line is informational; an UNSALVAGED line is an alarm, and it
|
|
2580
|
+
* names the directory because that directory is the work.
|
|
2581
|
+
*/
|
|
2582
|
+
export function formatSalvagedRuns(runs: readonly RunRecord[]): string[] {
|
|
2583
|
+
if (runs.length === 0) return [];
|
|
2584
|
+
const lines = ["", "wip"];
|
|
2585
|
+
for (const r of runs) {
|
|
2586
|
+
lines.push(
|
|
2587
|
+
r.salvageError !== undefined && r.salvageAckAt === undefined
|
|
2588
|
+
? ` #${r.issue} UNSALVAGED ${r.worktree === "" ? "(path not recorded)" : r.worktree} — ` +
|
|
2589
|
+
`only copy, dispatch held (${r.salvageError})`
|
|
2590
|
+
: r.salvageError !== undefined
|
|
2591
|
+
? ` #${r.issue} accepted as lost attempt ${r.attempt} (${r.salvageError})`
|
|
2592
|
+
: ` #${r.issue} preserved ${r.salvageSha ?? "?"} on ${r.branch} (attempt ${r.attempt}, ${r.state})`,
|
|
2593
|
+
);
|
|
2594
|
+
}
|
|
2595
|
+
return lines;
|
|
2596
|
+
}
|
|
2597
|
+
|
|
2598
|
+
/**
|
|
2599
|
+
* The grant table, named shape by shape.
|
|
2600
|
+
*
|
|
2601
|
+
* One renderer for both status surfaces (this one and the fleet view), because
|
|
2602
|
+
* #122 began with a POLICY grant that no longer matched anyone's intent and went
|
|
2603
|
+
* unnoticed: a stale grant has to be visible from `status` alone, and two
|
|
2604
|
+
* renderers would eventually show it in only one of them.
|
|
2605
|
+
*/
|
|
2606
|
+
export function formatReleaseGrants(grants: ResolvedGrants): string[] {
|
|
2607
|
+
const granted = RELEASE_SHAPES.filter((shape) => grants[shape] === "orchestrator");
|
|
2608
|
+
return [
|
|
2609
|
+
granted.length === 0
|
|
2610
|
+
? "release no shape granted — every release/deploy tool call is blocked"
|
|
2611
|
+
: `release granted to the orchestrator: ${granted.join(", ")}`,
|
|
2612
|
+
...RELEASE_SHAPES.map((shape) => ` ${shape.padEnd(19)}${grants[shape]}`),
|
|
2613
|
+
];
|
|
2614
|
+
}
|
|
2615
|
+
|
|
1740
2616
|
export function formatStatus(s: StatusSnapshot): string {
|
|
1741
2617
|
const lines = [
|
|
1742
2618
|
`project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
|
|
@@ -1749,11 +2625,17 @@ export function formatStatus(s: StatusSnapshot): string {
|
|
|
1749
2625
|
s.caps.dailySpendUsd === null
|
|
1750
2626
|
? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
|
|
1751
2627
|
: ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
|
|
2628
|
+
// Its own row, never folded into the spend row: they are two independent
|
|
2629
|
+
// controls, and an operator has to be able to see which one stopped the
|
|
2630
|
+
// fleet (#110).
|
|
2631
|
+
` plan usage ${planUsageLine(s.planUsage)}`,
|
|
1752
2632
|
` new worker turns ${s.caps.workerMaxTurns}`,
|
|
1753
2633
|
` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
|
|
1754
2634
|
` failed attempts ${s.caps.maxAttemptsPerIssue}`,
|
|
1755
2635
|
` continuations ${s.caps.maxContinuationsPerIssue}`,
|
|
1756
2636
|
"",
|
|
2637
|
+
...formatReleaseGrants(s.releaseGrants),
|
|
2638
|
+
"",
|
|
1757
2639
|
formatDispatchSummary(s.dispatch),
|
|
1758
2640
|
"",
|
|
1759
2641
|
];
|
|
@@ -1767,8 +2649,16 @@ export function formatStatus(s: StatusSnapshot): string {
|
|
|
1767
2649
|
`${r.turns}/${r.maxTurns} turns $${r.spendUsd.toFixed(2)} ${r.branch}` +
|
|
1768
2650
|
(r.prUrl ? ` ${r.prUrl}` : ""),
|
|
1769
2651
|
);
|
|
2652
|
+
// Its escalation was deduplicated the moment it was delivered, so this
|
|
2653
|
+
// line is the only place a flagged run stays visible while its PR waits
|
|
2654
|
+
// for a merge — which is precisely the window the flag is about (#128).
|
|
2655
|
+
const flagged = settlementFlagSummary(r.settlementFlags);
|
|
2656
|
+
if (flagged !== undefined) lines.push(` ${flagged}`);
|
|
1770
2657
|
}
|
|
1771
2658
|
}
|
|
2659
|
+
lines.push(...formatSalvagedRuns(s.salvagedRuns));
|
|
2660
|
+
lines.push(...formatOpenReports(s.openReports));
|
|
2661
|
+
lines.push(...formatVerbLedger(s.verbLedger));
|
|
1772
2662
|
// Deploy hint: a restart while workers are live orphans them (salvage runs
|
|
1773
2663
|
// first — #35). Prefer pause + drain to zero live workers when you can wait.
|
|
1774
2664
|
if (s.liveWorkers > 0) {
|
|
@@ -1866,6 +2756,13 @@ export function prepareConductor(): void {
|
|
|
1866
2756
|
export async function reconcileOrphanedRuns(
|
|
1867
2757
|
store: Store,
|
|
1868
2758
|
project: string,
|
|
2759
|
+
/**
|
|
2760
|
+
* Resolves the privileged publisher for one orphaned run. Optional because a
|
|
2761
|
+
* test driving the row transitions has no repo to publish to; production
|
|
2762
|
+
* always passes it, and without it a salvaged WIP commit stays local — which
|
|
2763
|
+
* is the half of #121 that reaches a human.
|
|
2764
|
+
*/
|
|
2765
|
+
publish?: (run: RunRecord) => RunPublisher,
|
|
1869
2766
|
): Promise<RunRecord[]> {
|
|
1870
2767
|
// Live runs only: `pushed-green` holds no process, so it cannot be orphaned by
|
|
1871
2768
|
// a process dying — it is finished work waiting on a human merge.
|
|
@@ -1873,12 +2770,25 @@ export async function reconcileOrphanedRuns(
|
|
|
1873
2770
|
const endedAt = Date.now();
|
|
1874
2771
|
for (const r of stale) {
|
|
1875
2772
|
// Salvage before the row flips: the worktree path is on the record, and
|
|
1876
|
-
// salvageWip is a no-op for a missing/clean tree.
|
|
1877
|
-
//
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
2773
|
+
// salvageWip is a no-op for a missing/clean tree. The clause matches the
|
|
2774
|
+
// cap-kill wording so triage reads the same either way, and the tree is
|
|
2775
|
+
// kept because an orphan's remains are the orchestrator's drain-duty call.
|
|
2776
|
+
const settlement =
|
|
2777
|
+
r.worktree === ""
|
|
2778
|
+
? undefined
|
|
2779
|
+
: await settleWorktree({
|
|
2780
|
+
issue: r.issue,
|
|
2781
|
+
attempt: r.attempt,
|
|
2782
|
+
ending: "killed by a daemon restart",
|
|
2783
|
+
worktree: r.worktree,
|
|
2784
|
+
branch: r.branch,
|
|
2785
|
+
// An orphan's tree is kept, so its commits are not about to be
|
|
2786
|
+
// deleted — but a WIP salvage still has to reach GitHub, which is
|
|
2787
|
+
// #121's whole point and is now the daemon's hop to make.
|
|
2788
|
+
publish: publish?.(r),
|
|
2789
|
+
tree: "keep",
|
|
2790
|
+
});
|
|
2791
|
+
store.updateRun(r.id, { state: "orphaned", endedAt, ...settlement?.patch });
|
|
1882
2792
|
}
|
|
1883
2793
|
return stale;
|
|
1884
2794
|
}
|
|
@@ -1886,12 +2796,63 @@ export async function reconcileOrphanedRuns(
|
|
|
1886
2796
|
// ------------------------------------------------------------------- the daemon
|
|
1887
2797
|
|
|
1888
2798
|
export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
2799
|
+
// Before anything reads the config for real: a file written before #125 gains
|
|
2800
|
+
// an explicit `credentials.isolation` here, so the answer lives on disk
|
|
2801
|
+
// rather than being inherited from a default nobody chose. Non-fatal by
|
|
2802
|
+
// design — a config the daemon cannot rewrite still loads as `"none"` and
|
|
2803
|
+
// still dispatches, because an outage caused by a security feature is worse
|
|
2804
|
+
// than the day before it shipped.
|
|
2805
|
+
try {
|
|
2806
|
+
const migration = migrateCredentialsOnDisk();
|
|
2807
|
+
if (migration.migrated.length > 0) {
|
|
2808
|
+
log(
|
|
2809
|
+
`migrated ${migration.migrated.join(", ")} to an explicit credentials.isolation: "none" in ` +
|
|
2810
|
+
`${migration.path} — this fleet is UNPROTECTED: sessions run as this daemon's user and can reach ` +
|
|
2811
|
+
`its GitHub credential. Set credentials.isolation to "per-run" once the host is provisioned (README).`,
|
|
2812
|
+
);
|
|
2813
|
+
}
|
|
2814
|
+
} catch (err) {
|
|
2815
|
+
log(`could not persist the credentials migration (${errText(err)}) — continuing with isolation "none"`);
|
|
2816
|
+
}
|
|
2817
|
+
|
|
1889
2818
|
const cfg = loadConfig();
|
|
1890
2819
|
const project = findProject(cfg, o.project);
|
|
1891
2820
|
const caps = resolveCaps(project, cfg.defaults);
|
|
1892
2821
|
const store = openStore(dbPath());
|
|
1893
2822
|
const tracker = makeTracker(project);
|
|
1894
2823
|
|
|
2824
|
+
// The host capability probe, once, at startup. It states the mechanism
|
|
2825
|
+
// rather than guessing, and it proves the launcher works against a trivial
|
|
2826
|
+
// child before reporting one available — a mechanism whose argv the host
|
|
2827
|
+
// rejects looks identical here and then fails every dispatch at run time.
|
|
2828
|
+
const credentials = resolveCredentials(project);
|
|
2829
|
+
const probe = await probeHost({ slots: caps.maxConcurrentWorkers });
|
|
2830
|
+
const boundary: FleetBoundary = {
|
|
2831
|
+
isolation: credentials.isolation,
|
|
2832
|
+
probe,
|
|
2833
|
+
slots: createSlotPool(caps.maxConcurrentWorkers),
|
|
2834
|
+
paged: false,
|
|
2835
|
+
};
|
|
2836
|
+
const described = describeBoundary(credentials.isolation, probe);
|
|
2837
|
+
log(`credential boundary: ${described.headline}`);
|
|
2838
|
+
for (const line of described.detail) log(`credential boundary: ${line}`);
|
|
2839
|
+
|
|
2840
|
+
// #126's transport, stated at startup rather than guessed at first use. The
|
|
2841
|
+
// banner names what this host can actually enforce — whether the kernel will
|
|
2842
|
+
// vouch for a caller's uid, and whether runs get distinct principals at all —
|
|
2843
|
+
// because "peer credentials asserted" is a claim, and a claim nobody printed
|
|
2844
|
+
// is one nobody can check against the host it is running on.
|
|
2845
|
+
const verbPeerReader = peerCredentialReader();
|
|
2846
|
+
const verbActions = githubVerbActions(project);
|
|
2847
|
+
let orchestratorVerbs: VerbListener | undefined;
|
|
2848
|
+
log(
|
|
2849
|
+
`verb transport: ${transportBanner(
|
|
2850
|
+
ensureVerbSocketDir(stateDir()),
|
|
2851
|
+
verbPeerReader,
|
|
2852
|
+
probe.mechanism === "uid-pool",
|
|
2853
|
+
)}`,
|
|
2854
|
+
);
|
|
2855
|
+
|
|
1895
2856
|
// Recorded here, before a single tick runs, so that the deploy an operator
|
|
1896
2857
|
// *means* to do never trips the tripwire: installing a new build and
|
|
1897
2858
|
// restarting the unit re-records this from the new files. What it catches is
|
|
@@ -1906,7 +2867,17 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
1906
2867
|
// daemon must not orphan that daemon's real, live workers).
|
|
1907
2868
|
const alive = livingDaemon();
|
|
1908
2869
|
if (alive === undefined || alive.pid === process.pid) {
|
|
1909
|
-
|
|
2870
|
+
// The orphan path salvages, and #121 requires that salvage to reach
|
|
2871
|
+
// GitHub. Since the run's commits now live in its own repository, the hop
|
|
2872
|
+
// is the daemon's: resolve the routed repo by name off the run row.
|
|
2873
|
+
const orphanPublisher = (r: RunRecord): RunPublisher => {
|
|
2874
|
+
const repo = project.routing.repos[r.repo];
|
|
2875
|
+
return async (branch) =>
|
|
2876
|
+
repo === undefined
|
|
2877
|
+
? { ok: false, stderr: `run ${String(r.id)} names repo "${r.repo}", which this project no longer routes` }
|
|
2878
|
+
: pushRunBranch(project, { repo, runRepoPath: r.worktree, branch });
|
|
2879
|
+
};
|
|
2880
|
+
for (const r of await reconcileOrphanedRuns(store, project.name, orphanPublisher)) {
|
|
1910
2881
|
log(
|
|
1911
2882
|
`#${r.issue} orphaned by a previous daemon (attempt ${r.attempt}, was ${r.state}, worktree ${r.worktree}) — ` +
|
|
1912
2883
|
`slot freed; the ${project.stateLabels.inProgress} label stays until the orchestrator triages what the worker left`,
|
|
@@ -1919,6 +2890,9 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
1919
2890
|
// Standing orders. The orchestrator holds none of this file's context, so
|
|
1920
2891
|
// everything it needs to act — which tracker, which labels, what the fleet
|
|
1921
2892
|
// does — has to be said once, in words.
|
|
2893
|
+
const releaseGrants = resolveReleaseGrants(project);
|
|
2894
|
+
const grantedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] === "orchestrator");
|
|
2895
|
+
const deniedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] !== "orchestrator");
|
|
1922
2896
|
const brief = [
|
|
1923
2897
|
`You are the omp-conductor orchestrator for project "${project.name}".`,
|
|
1924
2898
|
`Tracker: ${project.tracker.repo}. Pass --repo ${project.tracker.repo} to every gh command:`,
|
|
@@ -1939,10 +2913,14 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
1939
2913
|
"a time, freshness-checked against the base branch, per the Releases section of your POLICY.md."
|
|
1940
2914
|
: "You never edit product code, push a branch, or merge a PR — a worker session edits and pushes, and a " +
|
|
1941
2915
|
"human merges.",
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
2916
|
+
// Named shape by shape rather than as one policy word, so a stale grant is
|
|
2917
|
+
// legible in the transcript instead of only in the config file — the #122
|
|
2918
|
+
// incident began with a grant that no longer matched anyone's intent.
|
|
2919
|
+
`Release tool gate: ${
|
|
2920
|
+
grantedShapes.length === 0
|
|
2921
|
+
? "every release and deploy shape is mechanically blocked for you"
|
|
2922
|
+
: `you may invoke ${grantedShapes.join(", ")} — and only by the procedure in your POLICY.md`
|
|
2923
|
+
}.` + (deniedShapes.length === 0 ? "" : ` Blocked: ${deniedShapes.join(", ")}.`),
|
|
1946
2924
|
"Handle each escalation below before the next one.",
|
|
1947
2925
|
].join("\n");
|
|
1948
2926
|
|
|
@@ -1960,10 +2938,87 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
1960
2938
|
log("orchestrator: external — tier-1 escalations post as issue comments for the external session's drain duty");
|
|
1961
2939
|
} else {
|
|
1962
2940
|
try {
|
|
2941
|
+
// Its own principal, distinct from every worker slot. The orchestrator
|
|
2942
|
+
// reads the state directory and its briefs and must have no read or
|
|
2943
|
+
// write access to any run checkout — so `workspaceRoot` and the mirror
|
|
2944
|
+
// are denied outright here, on top of the #127 tool-layer jail. Two
|
|
2945
|
+
// layers because they fail differently: the jail refuses a structured
|
|
2946
|
+
// tool call, the principal refuses the syscall `bash` would make.
|
|
2947
|
+
// Same rule as a worker's, for the same reason: under `uid-pool` the
|
|
2948
|
+
// orchestrator is its own uid, so every tree it must reach lives outside
|
|
2949
|
+
// the 0700 private state directory. Its principal is in `conductor-runs`
|
|
2950
|
+
// and deliberately NOT in the daemon group, so it could not traverse
|
|
2951
|
+
// there even once — and widening the state directory is the one fix that
|
|
2952
|
+
// is not available, because that is where conductor.db lives.
|
|
2953
|
+
const orchTreeRoot = probe.mechanism === "uid-pool" ? sharedRoot() : stateDir();
|
|
2954
|
+
if (orchTreeRoot !== stateDir()) {
|
|
2955
|
+
mkdirSync(orchTreeRoot, { recursive: true });
|
|
2956
|
+
chmodSync(orchTreeRoot, 0o711);
|
|
2957
|
+
}
|
|
2958
|
+
const orchEnvRoot = join(orchTreeRoot, "boundaries", "orchestrator");
|
|
2959
|
+
const orchCwd = join(orchTreeRoot, "orchestrator");
|
|
2960
|
+
mkdirSync(orchCwd, { recursive: true });
|
|
2961
|
+
const orchestratorBoundary = buildSessionBoundary({
|
|
2962
|
+
probe,
|
|
2963
|
+
isolation: credentials.isolation,
|
|
2964
|
+
role: "orchestrator",
|
|
2965
|
+
envRoot: orchEnvRoot,
|
|
2966
|
+
writeRoots: [orchCwd],
|
|
2967
|
+
denyReadRoots: [...credentialDenyRoots(), project.workspaceRoot, project.mirrorRoot],
|
|
2968
|
+
denyReadFiles: credentialDenyFiles(),
|
|
2969
|
+
...(credentials.readToken === undefined ? {} : { readToken: credentials.readToken }),
|
|
2970
|
+
});
|
|
2971
|
+
if (orchestratorBoundary.principal !== undefined) {
|
|
2972
|
+
const gid = probe.daemonGid;
|
|
2973
|
+
if (gid === undefined) {
|
|
2974
|
+
throw new Error(
|
|
2975
|
+
`the orchestrator principal was allocated without a resolved ${CONDUCTOR_GROUPS.daemon} gid`,
|
|
2976
|
+
);
|
|
2977
|
+
}
|
|
2978
|
+
applyRunOwnership(orchEnvRoot, orchestratorBoundary.principal, gid);
|
|
2979
|
+
applyRunOwnership(orchCwd, orchestratorBoundary.principal, gid);
|
|
2980
|
+
// The same empirical gate a worker gets. The orchestrator is the
|
|
2981
|
+
// session with merge and release authority, so "it cannot reach the
|
|
2982
|
+
// credential" matters more here, not less — and it was the half that
|
|
2983
|
+
// was never checked. `sharedGroup: false` because it is launched with
|
|
2984
|
+
// no supplementary group at all, so the probe has to run under exactly
|
|
2985
|
+
// that identity rather than a worker's.
|
|
2986
|
+
const orchLeaking = await credentialReadRefusal(
|
|
2987
|
+
spawnCaptured,
|
|
2988
|
+
orchestratorBoundary.principal,
|
|
2989
|
+
{ boundingSet: probe.boundingSetDropped, sharedGroup: false },
|
|
2990
|
+
[...credentialDenyRoots(), ...credentialDenyFiles()],
|
|
2991
|
+
);
|
|
2992
|
+
if (orchLeaking !== undefined) throw new Error(orchLeaking);
|
|
2993
|
+
}
|
|
2994
|
+
// A third socket, distinct from every run's, in the same daemon-owned
|
|
2995
|
+
// 0711 parent. This is what makes "merge authority is the orchestrator's"
|
|
2996
|
+
// a property of the channel: the daemon knows which session is speaking
|
|
2997
|
+
// because of where the connection arrived, and no payload can move a
|
|
2998
|
+
// worker's call onto this one (#126).
|
|
2999
|
+
orchestratorVerbs = await listenVerbChannel(
|
|
3000
|
+
verbDeps({ project, store, tracker, verbActions }),
|
|
3001
|
+
{
|
|
3002
|
+
kind: "orchestrator",
|
|
3003
|
+
path: verbSocketPath(ensureVerbSocketDir(orchTreeRoot), "orchestrator"),
|
|
3004
|
+
project: project.name,
|
|
3005
|
+
role: "orchestrator",
|
|
3006
|
+
...(orchestratorBoundary.principal === undefined
|
|
3007
|
+
? {}
|
|
3008
|
+
: { principal: orchestratorBoundary.principal }),
|
|
3009
|
+
},
|
|
3010
|
+
{ ...(verbPeerReader === undefined ? {} : { peerReader: verbPeerReader }) },
|
|
3011
|
+
);
|
|
1963
3012
|
orchestrator = await startOrchestrator({
|
|
1964
|
-
cwd:
|
|
3013
|
+
cwd: orchCwd,
|
|
1965
3014
|
brief,
|
|
1966
|
-
|
|
3015
|
+
releaseGrants,
|
|
3016
|
+
boundary: orchestratorBoundary,
|
|
3017
|
+
socketPath: join(orchEnvRoot, "ipc.sock"),
|
|
3018
|
+
verbSocketPath: orchestratorVerbs.path,
|
|
3019
|
+
onChildLog: (line) => {
|
|
3020
|
+
log(`orchestrator ${line}`);
|
|
3021
|
+
},
|
|
1967
3022
|
onReleaseBlocked: (shape) => recordReleaseBlock(project.name, "orchestrator", shape),
|
|
1968
3023
|
});
|
|
1969
3024
|
const transcript = orchestrator.sessionFile();
|
|
@@ -1979,14 +3034,45 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
1979
3034
|
}
|
|
1980
3035
|
|
|
1981
3036
|
const escalator = createEscalator(project, tracker, store, orchestrator);
|
|
3037
|
+
|
|
3038
|
+
// Report delivery is the daemon's, not the model's (#123). Built beside the
|
|
3039
|
+
// escalator because a report nobody can deliver pages through it, and driven
|
|
3040
|
+
// on its own timer rather than inside `tick()`: a paused fleet claims nothing
|
|
3041
|
+
// but still owes its operator the report it was handed, and five minutes is a
|
|
3042
|
+
// long time to sit on a page.
|
|
3043
|
+
const outbox = createReportOutbox({
|
|
3044
|
+
project,
|
|
3045
|
+
store,
|
|
3046
|
+
escalate: (e) => escalator.escalate(e),
|
|
3047
|
+
log,
|
|
3048
|
+
});
|
|
3049
|
+
|
|
3050
|
+
// Every row still `sending` when a daemon boots belonged to a process that is
|
|
3051
|
+
// gone, so its outcome will never be learned — retry it and say so in the
|
|
3052
|
+
// message. Guarded exactly like the orphan reconciliation above and for the
|
|
3053
|
+
// same reason: a row belonging to a *live* daemon is genuinely in flight, and
|
|
3054
|
+
// stealing it would page the operator twice for one report.
|
|
3055
|
+
if (alive === undefined || alive.pid === process.pid) {
|
|
3056
|
+
for (const r of outbox.recover(Date.now())) {
|
|
3057
|
+
log(
|
|
3058
|
+
`report ${r.id} was left mid-send by a previous daemon (attempt ${r.attempts}) — ` +
|
|
3059
|
+
`retrying; its message will say it may be a repeat`,
|
|
3060
|
+
);
|
|
3061
|
+
}
|
|
3062
|
+
}
|
|
3063
|
+
|
|
1982
3064
|
const turnLimits = createTurnLimitRegistry((runId, maxTurns) => {
|
|
1983
3065
|
store.updateRun(runId, { maxTurns });
|
|
1984
3066
|
});
|
|
1985
3067
|
const d: Deps = {
|
|
1986
3068
|
project,
|
|
1987
3069
|
caps,
|
|
3070
|
+
boundary,
|
|
1988
3071
|
tracker,
|
|
1989
3072
|
store,
|
|
3073
|
+
// Process-wide, so a `status` served off this daemon's own HTTP surface
|
|
3074
|
+
// reuses the tick's reading instead of shelling out again.
|
|
3075
|
+
usage: sharedUsageSource(),
|
|
1990
3076
|
escalate: (e) => escalator.escalate(e),
|
|
1991
3077
|
turnLimits,
|
|
1992
3078
|
integrity,
|
|
@@ -1994,13 +3080,20 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
1994
3080
|
// page again about a stall that is still on disk.
|
|
1995
3081
|
stall: { paged: false },
|
|
1996
3082
|
cleanup: { next: 0 },
|
|
3083
|
+
...(verbPeerReader === undefined ? {} : { verbPeerReader }),
|
|
3084
|
+
verbActions,
|
|
1997
3085
|
};
|
|
1998
3086
|
|
|
1999
3087
|
if (o.once) {
|
|
2000
3088
|
try {
|
|
2001
3089
|
await tick(d);
|
|
3090
|
+
// A single tick still owes the outbox a pass: a drill that leaves a
|
|
3091
|
+
// report undelivered teaches an operator the wrong thing about the
|
|
3092
|
+
// mechanism it is drilling.
|
|
3093
|
+
await outbox.deliverDue();
|
|
2002
3094
|
} finally {
|
|
2003
3095
|
await orchestrator?.dispose();
|
|
3096
|
+
await orchestratorVerbs?.close();
|
|
2004
3097
|
store.close();
|
|
2005
3098
|
}
|
|
2006
3099
|
return;
|
|
@@ -2027,6 +3120,25 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
2027
3120
|
refreshCodeGraph();
|
|
2028
3121
|
const graphTimer = setInterval(refreshCodeGraph, GRAPH_HEALTH_INTERVAL_MS);
|
|
2029
3122
|
|
|
3123
|
+
// Overlap-guarded like the graph probe: a pass that is still waiting on
|
|
3124
|
+
// Telegram must not have a second pass started on top of it, or one report
|
|
3125
|
+
// would be claimed, reclaimed and sent twice by this process alone.
|
|
3126
|
+
let reportPass: Promise<void> | undefined;
|
|
3127
|
+
const drainReports = (): void => {
|
|
3128
|
+
if (reportPass !== undefined) return;
|
|
3129
|
+
reportPass = outbox
|
|
3130
|
+
.deliverDue()
|
|
3131
|
+
.then(() => {})
|
|
3132
|
+
.catch((err: unknown) => {
|
|
3133
|
+
log(`report delivery pass failed: ${errText(err)}`);
|
|
3134
|
+
})
|
|
3135
|
+
.finally(() => {
|
|
3136
|
+
reportPass = undefined;
|
|
3137
|
+
});
|
|
3138
|
+
};
|
|
3139
|
+
drainReports();
|
|
3140
|
+
const reportTimer = setInterval(drainReports, REPORT_DELIVERY_INTERVAL_MS);
|
|
3141
|
+
|
|
2030
3142
|
const workers = createWorkerPool();
|
|
2031
3143
|
let stopping = false;
|
|
2032
3144
|
let wake: (() => void) | undefined;
|
|
@@ -2039,17 +3151,31 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
2039
3151
|
process.on("SIGINT", stop);
|
|
2040
3152
|
process.on("SIGTERM", stop);
|
|
2041
3153
|
|
|
3154
|
+
// ── NO MUTATION ROUTE BELONGS ON THIS PORT ────────────────────────────────
|
|
3155
|
+
//
|
|
3156
|
+
// This is unauthenticated loopback TCP. Every local user can reach it, it
|
|
3157
|
+
// carries no credential of any kind, and it cannot tell one caller from
|
|
3158
|
+
// another: `127.0.0.1` is not an identity. `turnLimitResponse` already shows
|
|
3159
|
+
// what that costs — it trusts a body-supplied `project`, which is exactly the
|
|
3160
|
+
// shape "identity from the payload" takes when nobody is watching, and it is
|
|
3161
|
+
// tolerable only because raising a turn ceiling is bounded and reversible.
|
|
3162
|
+
//
|
|
3163
|
+
// A merge, a push, a release or a label is none of those things. Do not add
|
|
3164
|
+
// one here, and do not add "just a small one" behind a shared secret either:
|
|
3165
|
+
// a secret readable by the process that would be attacking you is not
|
|
3166
|
+
// authentication. Mutations go over the per-run unix sockets in
|
|
3167
|
+
// `verbs/socket.ts`, where the kernel says who the caller is and the daemon
|
|
3168
|
+
// derives project, run and role from the channel rather than the body (#126).
|
|
2042
3169
|
const server = Bun.serve({
|
|
2043
3170
|
hostname: "127.0.0.1",
|
|
2044
3171
|
port: o.port ?? DEFAULT_PORT,
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
},
|
|
3172
|
+
fetch: (req) =>
|
|
3173
|
+
daemonHttpResponse(req, {
|
|
3174
|
+
project: project.name,
|
|
3175
|
+
store,
|
|
3176
|
+
turnLimits,
|
|
3177
|
+
health: () => daemonHealthSnapshot(store, project.name, undefined, undefined, codeGraph),
|
|
3178
|
+
}),
|
|
2053
3179
|
});
|
|
2054
3180
|
log(`serving /healthz on :${server.port}, project ${project.name}`);
|
|
2055
3181
|
|
|
@@ -2076,11 +3202,20 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
2076
3202
|
process.off("SIGINT", stop);
|
|
2077
3203
|
process.off("SIGTERM", stop);
|
|
2078
3204
|
clearInterval(graphTimer);
|
|
3205
|
+
clearInterval(reportTimer);
|
|
3206
|
+
// Before the store closes, like the orchestrator below: a pass mid-send has
|
|
3207
|
+
// a `markReportDelivered` still to write, and losing that write is exactly
|
|
3208
|
+
// how a delivered report comes back as an ambiguous one on the next boot.
|
|
3209
|
+
await reportPass;
|
|
2079
3210
|
await workers.drain();
|
|
2080
3211
|
await server.stop(true);
|
|
2081
3212
|
// Before the store closes: a queued injection that rejects on the way out
|
|
2082
3213
|
// falls back to an issue comment, and that path writes the dedup marker.
|
|
2083
3214
|
await orchestrator?.dispose();
|
|
3215
|
+
// After the session it belongs to is gone. A bound socket outliving its
|
|
3216
|
+
// orchestrator is a channel accepting merges for a session that no longer
|
|
3217
|
+
// exists.
|
|
3218
|
+
await orchestratorVerbs?.close();
|
|
2084
3219
|
store.close();
|
|
2085
3220
|
log("stopped");
|
|
2086
3221
|
// A handled SIGTERM still leaves some runtimes with a non-zero default
|