omp-conductor 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +549 -234
- package/package.json +8 -5
- package/schema/config.schema.json +609 -0
- package/src/availability.ts +165 -0
- package/src/board.ts +19 -32
- package/src/brief-upgrade.ts +1 -1
- package/src/briefs/orchestrator.md +72 -31
- package/src/briefs/policy.md +48 -36
- package/src/briefs/probes/gates.md +51 -0
- package/src/briefs/probes/project-context.md +59 -0
- package/src/briefs/probes/release-procedure.md +81 -0
- package/src/cli.ts +356 -212
- package/src/config-schema.ts +352 -0
- package/src/config.ts +1037 -679
- package/src/confinement.ts +54 -0
- package/src/daemon.ts +644 -390
- package/src/diff-flags.ts +73 -4
- package/src/digest-schedule.ts +92 -24
- package/src/escalate.ts +89 -22
- package/src/fleet.ts +351 -46
- package/src/generate-schema.ts +21 -0
- package/src/graph.ts +3 -3
- package/src/host.ts +16 -0
- package/src/omp.ts +21 -1
- package/src/orchestrator-tick.ts +732 -56
- package/src/privileged.ts +264 -0
- package/src/reports.ts +203 -6
- package/src/session-host.ts +3 -0
- package/src/setup-host.ts +209 -24
- package/src/setup-install.ts +320 -0
- package/src/setup-probe.ts +412 -0
- package/src/setup-wizard.ts +1946 -0
- package/src/setup.ts +457 -53
- package/src/store.ts +610 -98
- package/src/tracker/github.ts +43 -5
- package/src/types.ts +153 -14
- package/src/upgrade.ts +44 -10
- package/src/verbs/actions.ts +131 -13
- package/src/verbs/server.ts +40 -18
- package/src/wizard-ui.ts +249 -0
- package/src/worker.ts +24 -7
- package/skills/conductor-onboarding/SKILL.md +0 -748
- package/skills/conductor-update/SKILL.md +0 -51
- package/src/plugin.ts +0 -1495
package/src/daemon.ts
CHANGED
|
@@ -18,19 +18,26 @@ import {
|
|
|
18
18
|
resolveReleaseGrants,
|
|
19
19
|
stateDir,
|
|
20
20
|
} from "./config.ts";
|
|
21
|
+
import { availabilityState, type AvailabilityState } from "./availability.ts";
|
|
21
22
|
import {
|
|
22
23
|
analyseSettlement,
|
|
23
24
|
formatSettlementFlags,
|
|
24
25
|
settlementFlagSummary,
|
|
25
26
|
} from "./diff-flags.ts";
|
|
27
|
+
import { digestScheduleState, type DigestScheduleState } from "./digest-schedule.ts";
|
|
26
28
|
import { createEscalator, escalationIssueRef } from "./escalate.ts";
|
|
27
29
|
import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
28
30
|
import { graphHint } from "./graph.ts";
|
|
29
31
|
import { livingDaemon } from "./lifecycle.ts";
|
|
30
|
-
import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
|
|
32
|
+
import { requestImmediateTick, resolveTickConfigCwd, STALL_MARKER_FILE } from "./orchestrator-tick.ts";
|
|
31
33
|
import { startOrchestrator } from "./orchestrator.ts";
|
|
32
34
|
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
33
|
-
import {
|
|
35
|
+
import {
|
|
36
|
+
createReportOutbox,
|
|
37
|
+
enqueueAvailableHeldNotices,
|
|
38
|
+
formatOpenReports,
|
|
39
|
+
type ReportOutbox,
|
|
40
|
+
} from "./reports.ts";
|
|
34
41
|
import {
|
|
35
42
|
recordReleaseBlock,
|
|
36
43
|
type ReleaseBlockContext,
|
|
@@ -42,11 +49,13 @@ import { classifyRun, providerCreditRefusal, providerTransientFault, type Classi
|
|
|
42
49
|
import { projectLabels } from "./label-projection.ts";
|
|
43
50
|
import { dbPath, LIVE_STATES, openStore, utcDay } from "./store.ts";
|
|
44
51
|
import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
|
|
45
|
-
import { RELEASE_SHAPES } from "./types.ts";
|
|
52
|
+
import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES } from "./types.ts";
|
|
46
53
|
import type {
|
|
54
|
+
BaseHealth,
|
|
47
55
|
AdmissionHoldReason,
|
|
48
56
|
Caps,
|
|
49
57
|
DispatchSummary,
|
|
58
|
+
DigestBacklog,
|
|
50
59
|
Escalation,
|
|
51
60
|
IssueSnapshot,
|
|
52
61
|
MergedPrInfo,
|
|
@@ -167,6 +176,9 @@ interface Deps {
|
|
|
167
176
|
caps: Caps;
|
|
168
177
|
tracker: Tracker;
|
|
169
178
|
store: Store;
|
|
179
|
+
/** False after a live config reload fails; autonomous delivery then holds
|
|
180
|
+
* fail-closed until a later tick validates the config again. */
|
|
181
|
+
deliveryPolicyValid?: boolean;
|
|
170
182
|
/** Provider-reported plan allowance, cached with a TTL. Resolved once at
|
|
171
183
|
* startup like every other dep so a tick cannot swap its own meter. */
|
|
172
184
|
usage: UsageSource;
|
|
@@ -215,10 +227,10 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
|
|
|
215
227
|
tracker: d.tracker,
|
|
216
228
|
actions: d.verbActions ?? githubVerbActions(d.project),
|
|
217
229
|
fleetStop: () =>
|
|
218
|
-
isPaused()
|
|
219
|
-
? "claiming is paused for this fleet (omp-conductor
|
|
230
|
+
isPaused(d.project.name)
|
|
231
|
+
? "claiming is paused for this fleet (omp-conductor hold or stop)"
|
|
220
232
|
: undefined,
|
|
221
|
-
pausedAt,
|
|
233
|
+
pausedAt: () => pausedAt(d.project.name),
|
|
222
234
|
log,
|
|
223
235
|
now: () => Date.now(),
|
|
224
236
|
chain: { readBaseChain },
|
|
@@ -364,80 +376,92 @@ export async function watchOrchestrator(d: Deps, now = Date.now()): Promise<void
|
|
|
364
376
|
}
|
|
365
377
|
|
|
366
378
|
/**
|
|
367
|
-
* Pause is a file rather than process state on purpose: `omp-conductor
|
|
368
|
-
*
|
|
379
|
+
* Pause is a file rather than process state on purpose: `omp-conductor hold`
|
|
380
|
+
* runs in a different process from the daemon, and a flag
|
|
369
381
|
* on disk needs no IPC and survives a restart. A daemon that crashed while
|
|
370
382
|
* paused comes back paused.
|
|
371
383
|
*/
|
|
372
|
-
export function
|
|
373
|
-
return
|
|
384
|
+
export function pausedPath(project?: string): string {
|
|
385
|
+
return join(stateDir(), project === undefined ? "paused" : `paused-${project}`);
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
function activePausePaths(project?: string): string[] {
|
|
389
|
+
const paths = project === undefined ? [pausedPath()] : [pausedPath(project), pausedPath()];
|
|
390
|
+
return paths.filter((path) => existsSync(path));
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
export function isPaused(project?: string): boolean {
|
|
394
|
+
return activePausePaths(project).length !== 0;
|
|
374
395
|
}
|
|
375
396
|
|
|
376
397
|
/**
|
|
377
|
-
* The epoch-ms timestamp at which the current pause began
|
|
378
|
-
* sentinel
|
|
379
|
-
*
|
|
380
|
-
*
|
|
381
|
-
* fail-closed because a run admitted before an *unknown* pause cannot be proven
|
|
382
|
-
* innocent. {@link isPaused} is the authority on *whether*; this answers *since
|
|
383
|
-
* when*.
|
|
398
|
+
* The epoch-ms timestamp at which the current pause began. A project pause also
|
|
399
|
+
* observes the legacy bare sentinel, which pauses every project. If any active
|
|
400
|
+
* sentinel is unreadable or unparseable, the timestamp is unknown so callers
|
|
401
|
+
* continue to fail closed.
|
|
384
402
|
*/
|
|
385
|
-
export function pausedAt(): number | undefined {
|
|
386
|
-
const
|
|
387
|
-
if (
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
403
|
+
export function pausedAt(project?: string): number | undefined {
|
|
404
|
+
const paths = activePausePaths(project);
|
|
405
|
+
if (paths.length === 0) return undefined;
|
|
406
|
+
const times: number[] = [];
|
|
407
|
+
for (const path of paths) {
|
|
408
|
+
try {
|
|
409
|
+
const first = readFileSync(path, "utf8").split("\n")[0]?.trim();
|
|
410
|
+
if (first === undefined || first === "") return undefined;
|
|
411
|
+
const time = Date.parse(first);
|
|
412
|
+
if (Number.isNaN(time)) return undefined;
|
|
413
|
+
times.push(time);
|
|
414
|
+
} catch {
|
|
415
|
+
return undefined;
|
|
416
|
+
}
|
|
397
417
|
}
|
|
418
|
+
return Math.min(...times);
|
|
398
419
|
}
|
|
399
420
|
|
|
400
421
|
/**
|
|
401
|
-
* Who paused the
|
|
402
|
-
*
|
|
403
|
-
* file has no (parseable) line 2. Provenance lives on its own line so line 1
|
|
404
|
-
* stays a pure ISO timestamp that {@link pausedAt} can `Date.parse`; the `armed
|
|
405
|
-
* <ISO> owner=<id>` marker `armTicks` writes is the same one-key-per-line
|
|
406
|
-
* precedent.
|
|
422
|
+
* Who paused the project and why. Per-project provenance wins when both its
|
|
423
|
+
* sentinel and the legacy all-project sentinel are active.
|
|
407
424
|
*/
|
|
408
|
-
export function pauseProvenance(
|
|
409
|
-
|
|
410
|
-
|
|
425
|
+
export function pauseProvenance(
|
|
426
|
+
project?: string,
|
|
427
|
+
): { source: string; reason?: string } | undefined {
|
|
428
|
+
const paths =
|
|
429
|
+
project === undefined
|
|
430
|
+
? [pausedPath()]
|
|
431
|
+
: [pausedPath(project), pausedPath()];
|
|
432
|
+
const path = paths.find((candidate) => existsSync(candidate));
|
|
433
|
+
if (path === undefined) return undefined;
|
|
411
434
|
try {
|
|
412
|
-
const second = readFileSync(
|
|
435
|
+
const second = readFileSync(path, "utf8").split("\n")[1];
|
|
413
436
|
if (second === undefined) return undefined;
|
|
414
437
|
const match = /^source=(\S+)(?: reason="(.*)")?$/.exec(second.trim());
|
|
415
438
|
if (match === null) return undefined;
|
|
416
|
-
const source = match[1]!;
|
|
439
|
+
const source = match[1]!;
|
|
417
440
|
const reason = match[2];
|
|
418
441
|
return { source, ...(reason === undefined ? {} : { reason }) };
|
|
419
442
|
} catch {
|
|
420
|
-
// Unreadable sentinel: no provenance to name. Completion mutations still
|
|
421
|
-
// fail closed because the pause time is unknown.
|
|
422
443
|
return undefined;
|
|
423
444
|
}
|
|
424
445
|
}
|
|
425
446
|
|
|
426
|
-
export function setPaused(
|
|
427
|
-
|
|
447
|
+
export function setPaused(
|
|
448
|
+
v: boolean,
|
|
449
|
+
why?: { source: string; reason?: string },
|
|
450
|
+
project?: string,
|
|
451
|
+
): void {
|
|
452
|
+
const path = pausedPath(project);
|
|
428
453
|
if (v) {
|
|
429
|
-
mkdirSync(dirname(
|
|
454
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
430
455
|
const line1 = `${new Date().toISOString()}\n`;
|
|
431
456
|
if (why === undefined) {
|
|
432
|
-
writeFileSync(
|
|
457
|
+
writeFileSync(path, line1);
|
|
433
458
|
} else {
|
|
434
|
-
// Quotes are stripped before embedding so a reason cannot break out of
|
|
435
|
-
// the `reason="..."` field of line 2.
|
|
436
459
|
const reason = why.reason === undefined ? "" : ` reason="${why.reason.replaceAll('"', "")}"`;
|
|
437
|
-
writeFileSync(
|
|
460
|
+
writeFileSync(path, `${line1}source=${why.source}${reason}\n`);
|
|
438
461
|
}
|
|
439
462
|
} else {
|
|
440
|
-
rmSync(
|
|
463
|
+
rmSync(path, { force: true });
|
|
464
|
+
if (project !== undefined) rmSync(pausedPath(), { force: true });
|
|
441
465
|
}
|
|
442
466
|
}
|
|
443
467
|
|
|
@@ -663,8 +687,8 @@ async function reactToProviderCredit(
|
|
|
663
687
|
sessionFile: string | undefined,
|
|
664
688
|
): Promise<void> {
|
|
665
689
|
const { project } = d;
|
|
666
|
-
const alreadyPaused = isPaused();
|
|
667
|
-
if (!alreadyPaused) setPaused(true, { source: "provider-credit", reason: message });
|
|
690
|
+
const alreadyPaused = isPaused(project.name);
|
|
691
|
+
if (!alreadyPaused) setPaused(true, { source: "provider-credit", reason: message }, project.name);
|
|
668
692
|
log(
|
|
669
693
|
`#${issue} provider refused for credit — dispatch ${alreadyPaused ? "remains paused" : "paused"}: ${message}`,
|
|
670
694
|
);
|
|
@@ -1402,12 +1426,15 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1402
1426
|
});
|
|
1403
1427
|
if (await settleStopBeforeSession()) return;
|
|
1404
1428
|
|
|
1429
|
+
const repoSlug = githubRepo(r.repo.cloneUrl);
|
|
1430
|
+
|
|
1405
1431
|
let result: WorkerResult;
|
|
1406
1432
|
try {
|
|
1407
1433
|
result = await runWorker({
|
|
1408
1434
|
brief,
|
|
1409
1435
|
cwd: worktreePath,
|
|
1410
1436
|
caps,
|
|
1437
|
+
...(repoSlug === undefined ? {} : { repoSlug }),
|
|
1411
1438
|
maxTurns: () => turnLimit?.maxTurns() ?? maxTurns,
|
|
1412
1439
|
onPauseControl: (control) => {
|
|
1413
1440
|
workerSessionInstalled = true;
|
|
@@ -1804,12 +1831,12 @@ export interface Settlement {
|
|
|
1804
1831
|
*
|
|
1805
1832
|
* - `merged` — the work landed. That is what `merged` was reserved for.
|
|
1806
1833
|
* - `closed` — a human read the work and said no. Leaving it `pushed-green`
|
|
1807
|
-
*
|
|
1808
|
-
*
|
|
1809
|
-
*
|
|
1810
|
-
*
|
|
1811
|
-
*
|
|
1812
|
-
*
|
|
1834
|
+
* forever is a lie; `failed` records that it did not land and releases the
|
|
1835
|
+
* busy guard, so an issue a human re-queues can be attempted again. A row
|
|
1836
|
+
* that had reached `pushed-green` or `pushed-pending` is classified
|
|
1837
|
+
* `returned-for-revision` at settlement. A review decision asks for another
|
|
1838
|
+
* implementation pass, not a failure, so it consumes the continuation budget
|
|
1839
|
+
* instead of the failed-attempt budget.
|
|
1813
1840
|
* - `open`, and undefined — nothing changes. Undefined is "could not tell": a
|
|
1814
1841
|
* flaky network, a revoked token, a deleted PR. Settling on it would record a
|
|
1815
1842
|
* merge that never happened, and the next tick asks again for free. An
|
|
@@ -1944,15 +1971,24 @@ export async function watchMergedBase(d: Pick<Deps, "project" | "tracker" | "sto
|
|
|
1944
1971
|
|
|
1945
1972
|
let workflows;
|
|
1946
1973
|
try {
|
|
1947
|
-
workflows = await d.tracker.workflowRunsAt(repoIdentity, run.mergeSha
|
|
1974
|
+
workflows = await d.tracker.workflowRunsAt(repoIdentity, run.mergeSha, {
|
|
1975
|
+
event: "push",
|
|
1976
|
+
branch: run.baseRef,
|
|
1977
|
+
});
|
|
1948
1978
|
} catch (err) {
|
|
1949
1979
|
log(`#${run.issue} base check unavailable (${errText(err)}) — retrying next tick`);
|
|
1950
1980
|
continue;
|
|
1951
1981
|
}
|
|
1952
|
-
if (workflows === undefined
|
|
1982
|
+
if (workflows === undefined) {
|
|
1953
1983
|
log(`#${run.issue} base check unavailable for ${run.mergeSha} — retrying next tick`);
|
|
1954
1984
|
continue;
|
|
1955
1985
|
}
|
|
1986
|
+
if (workflows.length === 0) {
|
|
1987
|
+
log(
|
|
1988
|
+
`#${run.issue} base check: no push-triggered run yet for ${run.mergeSha} — retrying next tick`,
|
|
1989
|
+
);
|
|
1990
|
+
continue;
|
|
1991
|
+
}
|
|
1956
1992
|
if (workflows.some((workflow) => workflow.status !== "completed")) continue;
|
|
1957
1993
|
|
|
1958
1994
|
const failed = workflows.find(
|
|
@@ -2016,6 +2052,110 @@ export async function watchMergedBase(d: Pick<Deps, "project" | "tracker" | "sto
|
|
|
2016
2052
|
}
|
|
2017
2053
|
}
|
|
2018
2054
|
|
|
2055
|
+
/**
|
|
2056
|
+
* Refresh current base-branch health at each recently merged repository's live
|
|
2057
|
+
* head. This is status and release-gate evidence only; the per-merge audit
|
|
2058
|
+
* above remains the sole path that attributes and escalates a regression.
|
|
2059
|
+
*/
|
|
2060
|
+
export async function watchBaseHealth(
|
|
2061
|
+
d: Pick<Deps, "project" | "tracker" | "store">,
|
|
2062
|
+
): Promise<void> {
|
|
2063
|
+
const now = Date.now();
|
|
2064
|
+
const previousByRepo = new Map(
|
|
2065
|
+
d.store.baseHealth(d.project.name).map((row) => [row.repo, row] as const),
|
|
2066
|
+
);
|
|
2067
|
+
for (const { repo, baseRef } of d.store.mergedRepoBranches(
|
|
2068
|
+
d.project.name,
|
|
2069
|
+
now - BASE_STATUS_WINDOW_MS,
|
|
2070
|
+
)) {
|
|
2071
|
+
const target = d.project.routing.repos[repo];
|
|
2072
|
+
if (target === undefined) {
|
|
2073
|
+
log(`base health skipped: routed repository ${repo} is no longer configured`);
|
|
2074
|
+
continue;
|
|
2075
|
+
}
|
|
2076
|
+
const identity = githubRepo(target.cloneUrl);
|
|
2077
|
+
if (identity === undefined) {
|
|
2078
|
+
log(`base health skipped: routed repository ${repo} has no GitHub identity`);
|
|
2079
|
+
continue;
|
|
2080
|
+
}
|
|
2081
|
+
const branch = baseRef ?? target.defaultBranch;
|
|
2082
|
+
|
|
2083
|
+
let head: string | undefined;
|
|
2084
|
+
try {
|
|
2085
|
+
head = await d.tracker.branchHead(identity, branch);
|
|
2086
|
+
} catch (err) {
|
|
2087
|
+
log(`base ${repo}/${branch} head unavailable (${errText(err)}) — keeping previous health`);
|
|
2088
|
+
continue;
|
|
2089
|
+
}
|
|
2090
|
+
if (head === undefined) {
|
|
2091
|
+
log(`base ${repo}/${branch} head unavailable — keeping previous health`);
|
|
2092
|
+
continue;
|
|
2093
|
+
}
|
|
2094
|
+
|
|
2095
|
+
const previous = previousByRepo.get(repo);
|
|
2096
|
+
if (
|
|
2097
|
+
previous?.branch === branch &&
|
|
2098
|
+
previous.headSha === head &&
|
|
2099
|
+
(previous.verdict === "green" || previous.verdict === "red")
|
|
2100
|
+
) {
|
|
2101
|
+
continue;
|
|
2102
|
+
}
|
|
2103
|
+
|
|
2104
|
+
let runs;
|
|
2105
|
+
try {
|
|
2106
|
+
runs = await d.tracker.workflowRunsAt(identity, head, { event: "push", branch });
|
|
2107
|
+
} catch (err) {
|
|
2108
|
+
log(`base ${repo}/${branch} workflows unavailable (${errText(err)}) — keeping previous health`);
|
|
2109
|
+
continue;
|
|
2110
|
+
}
|
|
2111
|
+
if (runs === undefined) {
|
|
2112
|
+
log(`base ${repo}/${branch} workflows unavailable — keeping previous health`);
|
|
2113
|
+
continue;
|
|
2114
|
+
}
|
|
2115
|
+
|
|
2116
|
+
let verdict: BaseHealth["verdict"];
|
|
2117
|
+
let detail: string | undefined;
|
|
2118
|
+
if (runs.length === 0) {
|
|
2119
|
+
verdict = "unknown";
|
|
2120
|
+
detail = `no push-triggered workflow run for ${head.slice(0, 8)}`;
|
|
2121
|
+
} else if (runs.some((run) => run.status !== "completed")) {
|
|
2122
|
+
verdict = "pending";
|
|
2123
|
+
} else {
|
|
2124
|
+
const failed = runs.find(
|
|
2125
|
+
(run) =>
|
|
2126
|
+
run.conclusion !== undefined &&
|
|
2127
|
+
FAILING_WORKFLOW_CONCLUSIONS.has(run.conclusion),
|
|
2128
|
+
);
|
|
2129
|
+
if (failed !== undefined) {
|
|
2130
|
+
verdict = "red";
|
|
2131
|
+
detail = `${failed.name} failed at ${head.slice(0, 8)} — ${failed.url}`;
|
|
2132
|
+
} else if (
|
|
2133
|
+
runs.some(
|
|
2134
|
+
(run) =>
|
|
2135
|
+
run.conclusion === undefined ||
|
|
2136
|
+
!SUCCESSFUL_WORKFLOW_CONCLUSIONS.has(run.conclusion),
|
|
2137
|
+
)
|
|
2138
|
+
) {
|
|
2139
|
+
verdict = "pending";
|
|
2140
|
+
} else {
|
|
2141
|
+
verdict = "green";
|
|
2142
|
+
}
|
|
2143
|
+
}
|
|
2144
|
+
|
|
2145
|
+
const health: BaseHealth = {
|
|
2146
|
+
repo,
|
|
2147
|
+
branch,
|
|
2148
|
+
headSha: head,
|
|
2149
|
+
verdict,
|
|
2150
|
+
runsCount: runs.length,
|
|
2151
|
+
checkedAt: now,
|
|
2152
|
+
...(detail === undefined ? {} : { detail }),
|
|
2153
|
+
};
|
|
2154
|
+
d.store.upsertBaseHealth(d.project.name, health);
|
|
2155
|
+
previousByRepo.set(repo, health);
|
|
2156
|
+
}
|
|
2157
|
+
}
|
|
2158
|
+
|
|
2019
2159
|
export async function settlePushedGreen(
|
|
2020
2160
|
d: Pick<Deps, "project" | "tracker" | "store">,
|
|
2021
2161
|
): Promise<void> {
|
|
@@ -2080,7 +2220,11 @@ export async function settlePushedGreen(
|
|
|
2080
2220
|
baseCheck: "pending",
|
|
2081
2221
|
}),
|
|
2082
2222
|
};
|
|
2083
|
-
if (settlement.state === "failed")
|
|
2223
|
+
if (settlement.state === "failed") {
|
|
2224
|
+
patch.lastError = settlement.reason;
|
|
2225
|
+
patch.failureClass = "returned-for-revision";
|
|
2226
|
+
patch.recoveryAction = "none";
|
|
2227
|
+
}
|
|
2084
2228
|
store.updateRun(run.id, patch);
|
|
2085
2229
|
log(`#${run.issue} settled: ${settlement.reason}`);
|
|
2086
2230
|
continue;
|
|
@@ -2808,6 +2952,40 @@ export async function dispatchAdmissions(
|
|
|
2808
2952
|
|
|
2809
2953
|
// ----------------------------------------------------------------------- a tick
|
|
2810
2954
|
|
|
2955
|
+
/**
|
|
2956
|
+
* After a false→true condition transition, poke the orchestrator heartbeat so
|
|
2957
|
+
* it does not wait a full interval (#329). Best-effort: missing tick config or
|
|
2958
|
+
* a failed write leaves the row flagged in the ledger for the next heartbeat.
|
|
2959
|
+
*/
|
|
2960
|
+
export function wakeOrchestratorForMetConditions(
|
|
2961
|
+
projectName: string,
|
|
2962
|
+
met: readonly { id: string; condition?: string }[],
|
|
2963
|
+
writeLog: (line: string) => void = log,
|
|
2964
|
+
): void {
|
|
2965
|
+
if (met.length === 0) return;
|
|
2966
|
+
for (const decision of met) {
|
|
2967
|
+
writeLog(
|
|
2968
|
+
`decision ${decision.id} condition met (${decision.condition ?? "?"}) — requesting orchestrator tick`,
|
|
2969
|
+
);
|
|
2970
|
+
}
|
|
2971
|
+
const tickCwd = resolveTickConfigCwd(projectName);
|
|
2972
|
+
if (tickCwd === undefined) {
|
|
2973
|
+
writeLog(
|
|
2974
|
+
`decision condition met for ${met.map((m) => m.id).join(", ")} but no tick config cwd — heartbeat will surface them on its next interval`,
|
|
2975
|
+
);
|
|
2976
|
+
return;
|
|
2977
|
+
}
|
|
2978
|
+
const ids = met.map((m) => m.id).join(",");
|
|
2979
|
+
const reason = `condition-met ${ids}`;
|
|
2980
|
+
if (requestImmediateTick(tickCwd, reason)) {
|
|
2981
|
+
writeLog(`requested immediate tick at ${tickCwd}: ${reason}`);
|
|
2982
|
+
} else {
|
|
2983
|
+
writeLog(
|
|
2984
|
+
`could not write tick request under ${tickCwd}; conditions ${ids} wait for the next heartbeat`,
|
|
2985
|
+
);
|
|
2986
|
+
}
|
|
2987
|
+
}
|
|
2988
|
+
|
|
2811
2989
|
export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
2812
2990
|
// A config edit takes effect on the next tick, not the next daemon restart
|
|
2813
2991
|
// (#170). Re-resolve the project and its caps at the tick boundary so a tick
|
|
@@ -2823,8 +3001,26 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2823
3001
|
}
|
|
2824
3002
|
d.project = fresh;
|
|
2825
3003
|
d.caps = freshCaps;
|
|
3004
|
+
d.deliveryPolicyValid = true;
|
|
2826
3005
|
} catch (err) {
|
|
2827
|
-
log(`config reload failed (${errText(err)}) —
|
|
3006
|
+
log(`config reload failed (${errText(err)}) — retaining boot values but blocking autonomous delivery`);
|
|
3007
|
+
d.deliveryPolicyValid = false;
|
|
3008
|
+
}
|
|
3009
|
+
|
|
3010
|
+
// Availability-held notices are already durable. Once the freshly reloaded
|
|
3011
|
+
// policy opens (or newly allows their category), atomically hand a bounded
|
|
3012
|
+
// batch to the report outbox. Pauses do not suppress delivery.
|
|
3013
|
+
if (d.deliveryPolicyValid !== false) {
|
|
3014
|
+
try {
|
|
3015
|
+
const catchUp = enqueueAvailableHeldNotices(d.project, d.store, Date.now());
|
|
3016
|
+
if (catchUp !== undefined && !catchUp.deduped) {
|
|
3017
|
+
log(`availability catch-up ${catchUp.report.id} queued for ${d.project.name}`);
|
|
3018
|
+
}
|
|
3019
|
+
} catch (err) {
|
|
3020
|
+
// The daily digest may have claimed the same rows from another process
|
|
3021
|
+
// between selection and association. Either way the ledger still owns them.
|
|
3022
|
+
log(`availability catch-up handoff deferred (${errText(err)}) — retrying next tick`);
|
|
3023
|
+
}
|
|
2828
3024
|
}
|
|
2829
3025
|
|
|
2830
3026
|
// Before the pause check, deliberately. This one is not about dispatch: the
|
|
@@ -2844,6 +3040,11 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2844
3040
|
} catch (err) {
|
|
2845
3041
|
log(`base-branch check sweep failed: ${errText(err)}`);
|
|
2846
3042
|
}
|
|
3043
|
+
try {
|
|
3044
|
+
await watchBaseHealth(d);
|
|
3045
|
+
} catch (err) {
|
|
3046
|
+
log(`current base-health sweep failed: ${errText(err)}`);
|
|
3047
|
+
}
|
|
2847
3048
|
try {
|
|
2848
3049
|
await adoptSalvagedPrs(d);
|
|
2849
3050
|
} catch (err) {
|
|
@@ -2884,11 +3085,14 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2884
3085
|
// Ledger maintenance, above the pause gate for the same reason the stall watch
|
|
2885
3086
|
// is: a paused fleet still owes its operator the questions it asked, and a
|
|
2886
3087
|
// condition that came true while dispatch was parked is exactly the thing the
|
|
2887
|
-
// orchestrator has to see
|
|
3088
|
+
// orchestrator has to see promptly (#329).
|
|
2888
3089
|
//
|
|
2889
3090
|
// Expiry is synchronous (one UPDATE); condition evaluation is fire-and-forget
|
|
2890
3091
|
// because it shells out to `gh` and `npm`, and a registry that hangs must cost
|
|
2891
|
-
// one unevaluated condition rather than the tick.
|
|
3092
|
+
// one unevaluated condition rather than the tick. A false→true transition
|
|
3093
|
+
// writes the same immediate-tick poke recover uses so the heartbeat does not
|
|
3094
|
+
// wait a full interval; repeated sweeps while the condition stays true never
|
|
3095
|
+
// re-enter `met` (store mark is idempotent).
|
|
2892
3096
|
for (const expired of d.store.expireDueDecisions(d.project.name, Date.now())) {
|
|
2893
3097
|
log(`decision ${expired.id} expired unanswered after seven days: ${expired.question}`);
|
|
2894
3098
|
}
|
|
@@ -2900,9 +3104,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2900
3104
|
Date.now,
|
|
2901
3105
|
)
|
|
2902
3106
|
.then((met) => {
|
|
2903
|
-
|
|
2904
|
-
log(`decision ${decision.id} condition met (${decision.condition ?? "?"}) — surfacing on the next tick`);
|
|
2905
|
-
}
|
|
3107
|
+
wakeOrchestratorForMetConditions(d.project.name, met);
|
|
2906
3108
|
})
|
|
2907
3109
|
.catch((err: unknown) => {
|
|
2908
3110
|
log(`decision condition pass failed: ${errText(err)}`);
|
|
@@ -2910,7 +3112,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2910
3112
|
|
|
2911
3113
|
// A paused fleet claims nothing. Checked first so pausing takes effect on the
|
|
2912
3114
|
// next tick without signalling the process.
|
|
2913
|
-
if (isPaused()) return;
|
|
3115
|
+
if (isPaused(d.project.name)) return;
|
|
2914
3116
|
|
|
2915
3117
|
const { project, caps, store } = d;
|
|
2916
3118
|
|
|
@@ -2937,7 +3139,11 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2937
3139
|
`ERROR: the installed conductor changed under this daemon — ${integrity.diff.length} file(s) differ ` +
|
|
2938
3140
|
`(${shown.join(", ")}${integrity.diff.length > shown.length ? ", …" : ""}) — pausing`,
|
|
2939
3141
|
);
|
|
2940
|
-
setPaused(
|
|
3142
|
+
setPaused(
|
|
3143
|
+
true,
|
|
3144
|
+
{ source: "integrity", reason: "installed package changed under the daemon" },
|
|
3145
|
+
project.name,
|
|
3146
|
+
);
|
|
2941
3147
|
if (integrity.page) {
|
|
2942
3148
|
const delivered = await safeEscalate(d, {
|
|
2943
3149
|
tier: 2,
|
|
@@ -3027,7 +3233,11 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3027
3233
|
// operator opted out — turns and wall-clock still brake every run (#46).
|
|
3028
3234
|
const spent = store.spendSince(project.name, since);
|
|
3029
3235
|
if (caps.dailySpendUsd !== null && spent >= caps.dailySpendUsd) {
|
|
3030
|
-
setPaused(
|
|
3236
|
+
setPaused(
|
|
3237
|
+
true,
|
|
3238
|
+
{ source: "spend-cap", reason: `daily spend reached $${caps.dailySpendUsd}` },
|
|
3239
|
+
project.name,
|
|
3240
|
+
);
|
|
3031
3241
|
await safeEscalate(d, {
|
|
3032
3242
|
tier: 2,
|
|
3033
3243
|
category: "fleet-stopped",
|
|
@@ -3037,7 +3247,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3037
3247
|
summary: `Daily spend cap reached on ${new Date().toISOString().slice(0, 10)} — ${project.name} is paused`,
|
|
3038
3248
|
detail: [
|
|
3039
3249
|
`Spent $${spent.toFixed(2)} of the $${caps.dailySpendUsd.toFixed(2)} daily cap.`,
|
|
3040
|
-
"No further work will be claimed until `omp-conductor resume
|
|
3250
|
+
"No further work will be claimed until `omp-conductor resume`.",
|
|
3041
3251
|
].join("\n"),
|
|
3042
3252
|
});
|
|
3043
3253
|
recordDispatch(0, [
|
|
@@ -3088,25 +3298,29 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3088
3298
|
// --------------------------------------------------------------- read-only views
|
|
3089
3299
|
|
|
3090
3300
|
export interface DaemonHealthSnapshot {
|
|
3091
|
-
ok:
|
|
3301
|
+
ok: boolean;
|
|
3092
3302
|
paused: boolean;
|
|
3093
3303
|
activeRuns: number;
|
|
3094
3304
|
project: string;
|
|
3095
3305
|
/** One-shot issue ceilings waiting for the next claim. */
|
|
3096
3306
|
turnOverrides: TurnOverride[];
|
|
3097
|
-
/** Resident set of this daemon; workers are in-process omp sessions. */
|
|
3098
|
-
rssBytes: number;
|
|
3099
3307
|
dispatch?: DispatchSummary;
|
|
3100
3308
|
codeGraph?: CodeGraphHealth;
|
|
3101
3309
|
/** Live workers in a non-running pause phase; absent/empty = nothing paused. */
|
|
3102
3310
|
workers?: { issue: number; runId: string; phase: WorkerPausePhase }[];
|
|
3103
3311
|
}
|
|
3104
3312
|
|
|
3313
|
+
export interface DaemonHealth {
|
|
3314
|
+
ok: boolean;
|
|
3315
|
+
/** Resident set of this daemon; workers are in-process omp sessions. */
|
|
3316
|
+
rssBytes: number;
|
|
3317
|
+
projects: DaemonHealthSnapshot[];
|
|
3318
|
+
}
|
|
3319
|
+
|
|
3105
3320
|
export function daemonHealthSnapshot(
|
|
3106
3321
|
store: Store,
|
|
3107
3322
|
project: string,
|
|
3108
|
-
paused = isPaused(),
|
|
3109
|
-
rssBytes = process.memoryUsage().rss,
|
|
3323
|
+
paused = isPaused(project),
|
|
3110
3324
|
codeGraph?: CodeGraphHealth,
|
|
3111
3325
|
workerControls?: WorkerControlRegistry,
|
|
3112
3326
|
): DaemonHealthSnapshot {
|
|
@@ -3117,13 +3331,19 @@ export function daemonHealthSnapshot(
|
|
|
3117
3331
|
activeRuns: store.activeRuns(project).length,
|
|
3118
3332
|
turnOverrides: store.listTurnOverrides(project),
|
|
3119
3333
|
project,
|
|
3120
|
-
rssBytes,
|
|
3121
3334
|
...(dispatch === undefined ? {} : { dispatch }),
|
|
3122
3335
|
...(codeGraph?.configured === true ? { codeGraph } : {}),
|
|
3123
3336
|
...(workerControls === undefined ? {} : { workers: workerControls.snapshot(project) }),
|
|
3124
3337
|
};
|
|
3125
3338
|
}
|
|
3126
3339
|
|
|
3340
|
+
export function daemonHealth(
|
|
3341
|
+
projects: DaemonHealthSnapshot[],
|
|
3342
|
+
rssBytes = process.memoryUsage().rss,
|
|
3343
|
+
): DaemonHealth {
|
|
3344
|
+
return { ok: projects.every((project) => project.ok), rssBytes, projects };
|
|
3345
|
+
}
|
|
3346
|
+
|
|
3127
3347
|
const TURN_OVERRIDE_STATES: ReadonlySet<RunState> = new Set([
|
|
3128
3348
|
"failed",
|
|
3129
3349
|
"killed",
|
|
@@ -3329,13 +3549,17 @@ export async function workerControlResponse(
|
|
|
3329
3549
|
);
|
|
3330
3550
|
}
|
|
3331
3551
|
|
|
3332
|
-
export interface
|
|
3552
|
+
export interface DaemonHttpProjectDeps {
|
|
3333
3553
|
project: string;
|
|
3334
3554
|
store: Pick<Store, "latestRun" | "setTurnOverride">;
|
|
3335
3555
|
caps: () => Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">;
|
|
3556
|
+
}
|
|
3557
|
+
|
|
3558
|
+
export interface DaemonHttpDeps {
|
|
3559
|
+
projects: readonly DaemonHttpProjectDeps[];
|
|
3336
3560
|
turnLimits: TurnLimitRegistry;
|
|
3337
3561
|
workerControls: WorkerControlRegistry;
|
|
3338
|
-
health: () =>
|
|
3562
|
+
health: () => DaemonHealth;
|
|
3339
3563
|
}
|
|
3340
3564
|
|
|
3341
3565
|
/**
|
|
@@ -3350,9 +3574,47 @@ export interface DaemonHttpDeps {
|
|
|
3350
3574
|
export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promise<Response> {
|
|
3351
3575
|
const url = new URL(req.url);
|
|
3352
3576
|
if (req.method === "GET" && url.pathname === "/healthz") return Response.json(d.health());
|
|
3353
|
-
|
|
3577
|
+
|
|
3578
|
+
let selected = d.projects[0];
|
|
3579
|
+
if (
|
|
3580
|
+
req.method === "PUT" &&
|
|
3581
|
+
/^\/runs\/\d+\/(?:turn-limit|pause|resume|stop)$/.test(url.pathname) &&
|
|
3582
|
+
req.headers.get("content-type")?.startsWith("application/json")
|
|
3583
|
+
) {
|
|
3584
|
+
try {
|
|
3585
|
+
const body = await req.clone().json();
|
|
3586
|
+
if (body !== null && typeof body === "object") {
|
|
3587
|
+
const requested = Reflect.get(body, "project");
|
|
3588
|
+
if (typeof requested === "string" && requested.length > 0) {
|
|
3589
|
+
selected = d.projects.find(({ project }) => project === requested);
|
|
3590
|
+
if (selected === undefined) {
|
|
3591
|
+
return Response.json(
|
|
3592
|
+
{ error: `daemon does not serve requested project "${requested}"` },
|
|
3593
|
+
{ status: 409 },
|
|
3594
|
+
);
|
|
3595
|
+
}
|
|
3596
|
+
}
|
|
3597
|
+
}
|
|
3598
|
+
} catch {
|
|
3599
|
+
// The route handler below owns the public malformed-body response.
|
|
3600
|
+
}
|
|
3601
|
+
}
|
|
3602
|
+
if (selected === undefined) return new Response("not found\n", { status: 404 });
|
|
3603
|
+
|
|
3604
|
+
const turnLimit = await turnLimitResponse(
|
|
3605
|
+
req,
|
|
3606
|
+
selected.project,
|
|
3607
|
+
selected.store,
|
|
3608
|
+
d.turnLimits,
|
|
3609
|
+
selected.caps(),
|
|
3610
|
+
);
|
|
3354
3611
|
if (turnLimit !== undefined) return turnLimit;
|
|
3355
|
-
const workerControl = await workerControlResponse(
|
|
3612
|
+
const workerControl = await workerControlResponse(
|
|
3613
|
+
req,
|
|
3614
|
+
selected.project,
|
|
3615
|
+
selected.store,
|
|
3616
|
+
d.workerControls,
|
|
3617
|
+
);
|
|
3356
3618
|
return workerControl ?? new Response("not found\n", { status: 404 });
|
|
3357
3619
|
}
|
|
3358
3620
|
|
|
@@ -3367,6 +3629,10 @@ export interface StatusSnapshot {
|
|
|
3367
3629
|
* reading like a mistake (#220).
|
|
3368
3630
|
*/
|
|
3369
3631
|
pauseReason?: string;
|
|
3632
|
+
/** Mechanical operator availability at the moment this snapshot was read. */
|
|
3633
|
+
availability?: AvailabilityState;
|
|
3634
|
+
/** Next digest opportunity under the same predicate that gates submission. */
|
|
3635
|
+
digestSchedule?: DigestScheduleState;
|
|
3370
3636
|
caps: Caps;
|
|
3371
3637
|
/**
|
|
3372
3638
|
* The effective per-shape release grants. On the snapshot rather than re-read
|
|
@@ -3386,6 +3652,9 @@ export interface StatusSnapshot {
|
|
|
3386
3652
|
* unknown outcome, or written off. An empty list is the only honest way to
|
|
3387
3653
|
* say "everything authored this cycle actually went out" (#123). */
|
|
3388
3654
|
openReports: ReportRecord[];
|
|
3655
|
+
/** Ordinary outcomes and deferred escalations not yet associated with an
|
|
3656
|
+
* accepted digest report. */
|
|
3657
|
+
digestBacklog: DigestBacklog;
|
|
3389
3658
|
/**
|
|
3390
3659
|
* The most recent conductor-verb calls and how the daemon decided them
|
|
3391
3660
|
* (#126). On `status` rather than only behind `omp-conductor ledger` because
|
|
@@ -3393,8 +3662,8 @@ export interface StatusSnapshot {
|
|
|
3393
3662
|
* config does not let it, and an operator who has to know to go looking is an
|
|
3394
3663
|
* operator who finds out from the tracker instead.
|
|
3395
3664
|
*/
|
|
3396
|
-
/**
|
|
3397
|
-
|
|
3665
|
+
/** Current live-head push-workflow verdict per recently merged repository. */
|
|
3666
|
+
baseHealth: BaseHealth[];
|
|
3398
3667
|
verbLedger: VerbLedgerEntry[];
|
|
3399
3668
|
/** Runs backed by a worker process — the number capacity compares against. */
|
|
3400
3669
|
liveWorkers: number;
|
|
@@ -3441,25 +3710,32 @@ export function statusSnapshotFromStore(
|
|
|
3441
3710
|
store: Store,
|
|
3442
3711
|
planUsage?: PlanUsageStatus,
|
|
3443
3712
|
): StatusSnapshot {
|
|
3713
|
+
const now = Date.now();
|
|
3714
|
+
const lastDigestKey = store.lastDigestDedupeKey(p.name);
|
|
3715
|
+
const lastDigestDay =
|
|
3716
|
+
lastDigestKey === undefined ? undefined : lastDigestKey.slice("digest:".length);
|
|
3444
3717
|
const since = startOfToday();
|
|
3445
3718
|
const dispatch = store.latestDispatch(p.name);
|
|
3446
3719
|
const labelOpsPending = store.countPendingLabelOps(p.name);
|
|
3447
3720
|
const oldestLabelOpAt = store.oldestPendingLabelOpAt(p.name);
|
|
3448
3721
|
// Read once: the provenance read touches the filesystem, and the renderer
|
|
3449
3722
|
// should never pay for it twice per status.
|
|
3450
|
-
const reason = pauseProvenance()?.reason;
|
|
3723
|
+
const reason = pauseProvenance(p.name)?.reason;
|
|
3451
3724
|
return {
|
|
3452
3725
|
project: p.name,
|
|
3453
3726
|
configPath: configPath(),
|
|
3454
3727
|
stateDir: stateDir(),
|
|
3455
|
-
paused: isPaused(),
|
|
3728
|
+
paused: isPaused(p.name),
|
|
3456
3729
|
...(reason === undefined ? {} : { pauseReason: reason }),
|
|
3730
|
+
availability: availabilityState(p.reporting, now),
|
|
3731
|
+
digestSchedule: digestScheduleState(p.reporting ?? DEFAULT_REPORT_POLICY, lastDigestDay, now),
|
|
3457
3732
|
caps,
|
|
3458
3733
|
releaseGrants: resolveReleaseGrants(p),
|
|
3459
3734
|
activeRuns: store.activeRuns(p.name),
|
|
3460
3735
|
salvagedRuns: store.salvagedRuns(p.name),
|
|
3461
3736
|
turnOverrides: store.listTurnOverrides(p.name),
|
|
3462
3737
|
openReports: store.openReports(p.name),
|
|
3738
|
+
digestBacklog: store.digestBacklog(p.name),
|
|
3463
3739
|
verbLedger: store.verbLedger(p.name, { limit: STATUS_LEDGER_SCAN }),
|
|
3464
3740
|
liveWorkers: store.liveRuns(p.name).length,
|
|
3465
3741
|
runsToday: store.runsStartedSince(p.name, since),
|
|
@@ -3468,12 +3744,12 @@ export function statusSnapshotFromStore(
|
|
|
3468
3744
|
...(planUsage === undefined ? {} : { planUsage }),
|
|
3469
3745
|
// Written by the tracker's hooks rather than polled, so the renderer does
|
|
3470
3746
|
// not re-read GitHub to know it is being refused (#198).
|
|
3471
|
-
ghRefusals: store.ghRefusalsSince?.(
|
|
3747
|
+
ghRefusals: store.ghRefusalsSince?.(now - 5 * 60_000),
|
|
3472
3748
|
ghCallsToday: store.ghCallsToday?.(utcDay()),
|
|
3473
3749
|
...(labelOpsPending === 0 || oldestLabelOpAt === undefined
|
|
3474
3750
|
? {}
|
|
3475
|
-
: { labelOps: { pending: labelOpsPending, oldestAgeMs:
|
|
3476
|
-
|
|
3751
|
+
: { labelOps: { pending: labelOpsPending, oldestAgeMs: now - oldestLabelOpAt } }),
|
|
3752
|
+
baseHealth: store.baseHealth(p.name),
|
|
3477
3753
|
};
|
|
3478
3754
|
}
|
|
3479
3755
|
|
|
@@ -3556,18 +3832,22 @@ export function formatReleaseGrants(grants: ResolvedGrants): string[] {
|
|
|
3556
3832
|
...RELEASE_SHAPES.map((shape) => ` ${shape.padEnd(19)}${grants[shape]}`),
|
|
3557
3833
|
];
|
|
3558
3834
|
}
|
|
3559
|
-
export function
|
|
3560
|
-
return
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3835
|
+
export function formatBaseHealth(rows: readonly BaseHealth[]): string[] {
|
|
3836
|
+
return rows.map((row) => {
|
|
3837
|
+
const head = row.headSha.slice(0, 8);
|
|
3838
|
+
if (row.verdict === "green") {
|
|
3839
|
+
return `base ${row.repo}/${row.branch} green (${row.runsCount} run(s)) at ${head}`;
|
|
3840
|
+
}
|
|
3841
|
+
if (row.verdict === "red") {
|
|
3842
|
+
return `base ${row.repo}/${row.branch} RED — ${row.detail ?? `workflow failed at ${head}`}`;
|
|
3843
|
+
}
|
|
3844
|
+
if (row.verdict === "pending") {
|
|
3845
|
+
return `base ${row.repo}/${row.branch} pending (${row.runsCount} run(s)) at ${head}`;
|
|
3846
|
+
}
|
|
3847
|
+
return (
|
|
3848
|
+
`base ${row.repo}/${row.branch} unknown — ` +
|
|
3849
|
+
(row.detail ?? `no push-triggered workflow run for ${head}`)
|
|
3850
|
+
);
|
|
3571
3851
|
});
|
|
3572
3852
|
}
|
|
3573
3853
|
|
|
@@ -3618,7 +3898,7 @@ export function formatStatus(s: StatusSnapshot): string {
|
|
|
3618
3898
|
if (flagged !== undefined) lines.push(` ${flagged}`);
|
|
3619
3899
|
}
|
|
3620
3900
|
}
|
|
3621
|
-
lines.push(...
|
|
3901
|
+
lines.push(...formatBaseHealth(s.baseHealth));
|
|
3622
3902
|
lines.push(...formatSalvagedRuns(s.salvagedRuns));
|
|
3623
3903
|
lines.push(...formatOpenReports(s.openReports));
|
|
3624
3904
|
lines.push(...formatVerbLedger(s.verbLedger));
|
|
@@ -3645,7 +3925,7 @@ export interface QueuePreview {
|
|
|
3645
3925
|
|
|
3646
3926
|
/**
|
|
3647
3927
|
* Exactly what the next tick would pick up, computed without touching a single
|
|
3648
|
-
* label, run row or worktree. This is what makes
|
|
3928
|
+
* label, run row or worktree. This is what makes `omp-conductor setup` honest: the
|
|
3649
3929
|
* dry run is the same routing code the loop uses, not a description of it.
|
|
3650
3930
|
*/
|
|
3651
3931
|
export async function previewProject(
|
|
@@ -3661,7 +3941,7 @@ export async function previewProject(
|
|
|
3661
3941
|
`open issues in ${p.tracker.repo} labelled "${p.queueLabel}", ` +
|
|
3662
3942
|
`minus anything already labelled ${states}, ` +
|
|
3663
3943
|
`routed by one "${p.routing.labelPrefix}<repo>" label`,
|
|
3664
|
-
paused: isPaused(),
|
|
3944
|
+
paused: isPaused(p.name),
|
|
3665
3945
|
ready: routed.map((r) => ({
|
|
3666
3946
|
number: r.issue.number,
|
|
3667
3947
|
title: r.issue.title,
|
|
@@ -3688,9 +3968,9 @@ export async function previewQueue(project?: string): Promise<QueuePreview> {
|
|
|
3688
3968
|
* Setup calls this immediately after consent. Every later setup error therefore
|
|
3689
3969
|
* leaves the fleet paused instead of exposing a partially written runtime.
|
|
3690
3970
|
*/
|
|
3691
|
-
export function prepareConductor(): void {
|
|
3971
|
+
export function prepareConductor(project?: string): void {
|
|
3692
3972
|
openStore(dbPath()).close();
|
|
3693
|
-
setPaused(true, { source: "setup" });
|
|
3973
|
+
setPaused(true, { source: "setup" }, project);
|
|
3694
3974
|
}
|
|
3695
3975
|
|
|
3696
3976
|
/** Bounded per tick: each row costs tracker calls to gather facts for. */
|
|
@@ -4211,283 +4491,264 @@ export async function reconcileOrphanedRuns(
|
|
|
4211
4491
|
|
|
4212
4492
|
// ------------------------------------------------------------------- the daemon
|
|
4213
4493
|
|
|
4494
|
+
interface ProjectRuntime {
|
|
4495
|
+
d: Deps;
|
|
4496
|
+
outbox: ReportOutbox;
|
|
4497
|
+
orchestrator?: OrchestratorHandle;
|
|
4498
|
+
orchestratorVerbs?: VerbListener;
|
|
4499
|
+
codeGraph: CodeGraphHealth;
|
|
4500
|
+
graphProbe?: Promise<void>;
|
|
4501
|
+
reportPass?: Promise<void>;
|
|
4502
|
+
}
|
|
4503
|
+
|
|
4504
|
+
function orchestratorStandingOrders(project: ProjectConfig): {
|
|
4505
|
+
brief: string;
|
|
4506
|
+
releaseGrants: ResolvedGrants;
|
|
4507
|
+
} {
|
|
4508
|
+
const releaseGrants = resolveReleaseGrants(project);
|
|
4509
|
+
const grantedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] === "orchestrator");
|
|
4510
|
+
const deniedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] !== "orchestrator");
|
|
4511
|
+
return {
|
|
4512
|
+
releaseGrants,
|
|
4513
|
+
brief: [
|
|
4514
|
+
`You are the omp-conductor orchestrator for project "${project.name}".`,
|
|
4515
|
+
`Tracker: ${project.tracker.repo}. Pass --repo ${project.tracker.repo} to every gh command:`,
|
|
4516
|
+
"this working directory is the conductor's state directory, not a checkout.",
|
|
4517
|
+
`Labels: queue=${project.queueLabel}, running=${project.stateLabels.inProgress}, ` +
|
|
4518
|
+
`blocked=${project.stateLabels.blocked}, failed=${project.stateLabels.failed}.`,
|
|
4519
|
+
"The dispatcher claims queue-labelled issues, runs one worker session per attempt in its own",
|
|
4520
|
+
"worktree under hard turn/wallclock/spend caps, and escalates to you when a worker blocks or",
|
|
4521
|
+
"fails twice, its gates stay red, its branch conflicts, or a tripwire fires.",
|
|
4522
|
+
"Your job when that happens: re-brief the issue (comment what the next worker must do",
|
|
4523
|
+
`differently, then put ${project.queueLabel} back on it), file follow-up issues, or promote to`,
|
|
4524
|
+
"tier 2 and let the human decide.",
|
|
4525
|
+
project.authority.merge === "orchestrator"
|
|
4526
|
+
? "You never edit product code or push a branch — a worker session does that. Merging is yours: one PR at " +
|
|
4527
|
+
"a time, freshness-checked against the base branch, per the Releases section of your POLICY.md."
|
|
4528
|
+
: "You never edit product code, push a branch, or merge a PR — a worker session edits and pushes, and a " +
|
|
4529
|
+
"human merges.",
|
|
4530
|
+
`Release tool gate: ${
|
|
4531
|
+
grantedShapes.length === 0
|
|
4532
|
+
? "every release and deploy shape is mechanically blocked for you"
|
|
4533
|
+
: `you may invoke ${grantedShapes.join(", ")} — and only by the procedure in your POLICY.md`
|
|
4534
|
+
}.` + (deniedShapes.length === 0 ? "" : ` Blocked: ${deniedShapes.join(", ")}.`),
|
|
4535
|
+
"Handle each escalation below before the next one.",
|
|
4536
|
+
].join("\n"),
|
|
4537
|
+
};
|
|
4538
|
+
}
|
|
4539
|
+
|
|
4214
4540
|
export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
4215
4541
|
const cfg = loadConfig();
|
|
4216
|
-
const
|
|
4217
|
-
const caps = resolveCaps(project, cfg.defaults);
|
|
4542
|
+
const projects = o.project === undefined ? cfg.projects : [findProject(cfg, o.project)];
|
|
4218
4543
|
const store = openStore(dbPath());
|
|
4219
|
-
// The tracker's single `gh` funnel is bound to the store so the operator sees
|
|
4220
|
-
// observed truth (#198): every spawn is counted per UTC day, and every
|
|
4221
|
-
// rate-limit refusal is recorded for `status`'s 5m window. Board's ad-hoc
|
|
4222
|
-
// trackers and the polled `fetchRateLimit` probe are deliberately not bound —
|
|
4223
|
-
// this row is the daemon's own traffic.
|
|
4224
|
-
const tracker = makeTracker(project, undefined, {
|
|
4225
|
-
onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
|
|
4226
|
-
onRefusal: (at) => store.recordGhRefusal?.(at),
|
|
4227
|
-
// A conditional 304 revalidation is a spawn but not a billed read (#203);
|
|
4228
|
-
// counted separately so `status`'s call row keeps telling the truth once
|
|
4229
|
-
// most spawns are free revalidations.
|
|
4230
|
-
onNotModified: () => store.bumpGhCalls?.(utcDay(), "daemon-304"),
|
|
4231
|
-
});
|
|
4232
|
-
|
|
4233
|
-
// #126's transport, stated at startup rather than guessed at first use. The
|
|
4234
|
-
// banner names what this host can actually enforce — whether the kernel will
|
|
4235
|
-
// vouch for a caller's uid, and whether runs get distinct principals at all —
|
|
4236
|
-
// because "peer credentials asserted" is a claim, and a claim nobody printed
|
|
4237
|
-
// is one nobody can check against the host it is running on.
|
|
4238
4544
|
const verbPeerReader = peerCredentialReader();
|
|
4239
|
-
const
|
|
4240
|
-
let orchestratorVerbs: VerbListener | undefined;
|
|
4241
|
-
log(`verb transport: ${transportBanner(ensureVerbSocketDir(stateDir()), verbPeerReader)}`);
|
|
4242
|
-
|
|
4243
|
-
// Recorded here, before a single tick runs, so that the deploy an operator
|
|
4244
|
-
// *means* to do never trips the tripwire: installing a new build and
|
|
4245
|
-
// restarting the unit re-records this from the new files. What it catches is
|
|
4246
|
-
// the other thing — the package changing while the daemon that dispatches
|
|
4247
|
-
// work is holding it open, whether that is a worker that wandered out of its
|
|
4248
|
-
// worktree or a human editing the live install "just to test something".
|
|
4545
|
+
const verbDir = ensureVerbSocketDir(stateDir());
|
|
4249
4546
|
const integrity: IntegrityGate = { baseline: packageManifest(), paged: false };
|
|
4547
|
+
const usage = sharedUsageSource();
|
|
4548
|
+
const turnLimits = createTurnLimitRegistry((runId, maxTurns) => {
|
|
4549
|
+
store.updateRun(runId, { maxTurns });
|
|
4550
|
+
});
|
|
4551
|
+
const workerControls = createWorkerControlRegistry();
|
|
4552
|
+
const workers = createWorkerPool();
|
|
4553
|
+
const alive = livingDaemon();
|
|
4554
|
+
const runtimes: ProjectRuntime[] = [];
|
|
4555
|
+
|
|
4556
|
+
log(`verb transport: ${transportBanner(verbDir, verbPeerReader)}`);
|
|
4250
4557
|
log(`package integrity baseline: ${integrity.baseline.size} files under ${import.meta.dir}`);
|
|
4251
4558
|
|
|
4252
|
-
|
|
4253
|
-
|
|
4254
|
-
|
|
4255
|
-
const alive = livingDaemon();
|
|
4256
|
-
if (alive === undefined || alive.pid === process.pid) {
|
|
4257
|
-
// The orphan path salvages, and #121 requires that salvage to reach
|
|
4258
|
-
// GitHub. Since the run's commits now live in its own repository, the hop
|
|
4259
|
-
// is the daemon's: resolve the routed repo by name off the run row.
|
|
4260
|
-
const orphanPublisher = (r: RunRecord): RunPublisher => {
|
|
4261
|
-
const repo = project.routing.repos[r.repo];
|
|
4262
|
-
return async (branch) =>
|
|
4263
|
-
repo === undefined
|
|
4264
|
-
? { ok: false, stderr: `run ${String(r.id)} names repo "${r.repo}", which this project no longer routes` }
|
|
4265
|
-
: pushRunBranch(project, { repo, runRepoPath: r.worktree, branch });
|
|
4559
|
+
for (const project of projects) {
|
|
4560
|
+
const projectLog = (message: string): void => {
|
|
4561
|
+
log(projects.length === 1 ? message : `[${project.name}] ${message}`);
|
|
4266
4562
|
};
|
|
4267
|
-
|
|
4268
|
-
|
|
4269
|
-
|
|
4270
|
-
|
|
4271
|
-
)
|
|
4563
|
+
const caps = resolveCaps(project, cfg.defaults);
|
|
4564
|
+
const tracker = makeTracker(project, undefined, {
|
|
4565
|
+
onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
|
|
4566
|
+
onRefusal: (at) => store.recordGhRefusal?.(at),
|
|
4567
|
+
onNotModified: () => store.bumpGhCalls?.(utcDay(), "daemon-304"),
|
|
4568
|
+
});
|
|
4569
|
+
const verbActions = githubVerbActions(project);
|
|
4570
|
+
|
|
4571
|
+
if (alive === undefined || alive.pid === process.pid) {
|
|
4572
|
+
const orphanPublisher = (run: RunRecord): RunPublisher => {
|
|
4573
|
+
const repo = project.routing.repos[run.repo];
|
|
4574
|
+
return async (branch) =>
|
|
4575
|
+
repo === undefined
|
|
4576
|
+
? {
|
|
4577
|
+
ok: false,
|
|
4578
|
+
stderr:
|
|
4579
|
+
`run ${String(run.id)} names repo "${run.repo}", ` +
|
|
4580
|
+
`which project "${project.name}" no longer routes`,
|
|
4581
|
+
}
|
|
4582
|
+
: pushRunBranch(project, { repo, runRepoPath: run.worktree, branch });
|
|
4583
|
+
};
|
|
4584
|
+
for (const run of await reconcileOrphanedRuns(store, project.name, orphanPublisher)) {
|
|
4585
|
+
projectLog(
|
|
4586
|
+
`#${run.issue} orphaned by a previous daemon (attempt ${run.attempt}, was ${run.state}, ` +
|
|
4587
|
+
`worktree ${run.worktree}) — slot freed; the ${project.stateLabels.inProgress} label stays ` +
|
|
4588
|
+
"until the orchestrator triages what the worker left",
|
|
4589
|
+
);
|
|
4590
|
+
}
|
|
4591
|
+
} else if (runtimes.length === 0) {
|
|
4592
|
+
log(`skipping orphan reconciliation: daemon pid ${alive.pid} is alive and owns the active runs`);
|
|
4272
4593
|
}
|
|
4273
|
-
} else {
|
|
4274
|
-
log(`skipping orphan reconciliation: daemon pid ${alive.pid} is alive and owns the active runs`);
|
|
4275
|
-
}
|
|
4276
4594
|
|
|
4277
|
-
|
|
4278
|
-
|
|
4279
|
-
|
|
4280
|
-
|
|
4281
|
-
|
|
4282
|
-
|
|
4283
|
-
const brief = [
|
|
4284
|
-
`You are the omp-conductor orchestrator for project "${project.name}".`,
|
|
4285
|
-
`Tracker: ${project.tracker.repo}. Pass --repo ${project.tracker.repo} to every gh command:`,
|
|
4286
|
-
"this working directory is the conductor's state directory, not a checkout.",
|
|
4287
|
-
`Labels: queue=${project.queueLabel}, running=${project.stateLabels.inProgress}, ` +
|
|
4288
|
-
`blocked=${project.stateLabels.blocked}, failed=${project.stateLabels.failed}.`,
|
|
4289
|
-
"The dispatcher claims queue-labelled issues, runs one worker session per attempt in its own",
|
|
4290
|
-
"worktree under hard turn/wallclock/spend caps, and escalates to you when a worker blocks or",
|
|
4291
|
-
"fails twice, its gates stay red, its branch conflicts, or a tripwire fires.",
|
|
4292
|
-
"Your job when that happens: re-brief the issue (comment what the next worker must do",
|
|
4293
|
-
`differently, then put ${project.queueLabel} back on it), file follow-up issues, or promote to`,
|
|
4294
|
-
"tier 2 and let the human decide.",
|
|
4295
|
-
// Worded from `authority.merge` rather than fixed, so the standing orders
|
|
4296
|
-
// and the Releases section of the rendered brief cannot disagree about who
|
|
4297
|
-
// is holding the merge button. The daemon still merges nothing itself.
|
|
4298
|
-
project.authority.merge === "orchestrator"
|
|
4299
|
-
? "You never edit product code or push a branch — a worker session does that. Merging is yours: one PR at " +
|
|
4300
|
-
"a time, freshness-checked against the base branch, per the Releases section of your POLICY.md."
|
|
4301
|
-
: "You never edit product code, push a branch, or merge a PR — a worker session edits and pushes, and a " +
|
|
4302
|
-
"human merges.",
|
|
4303
|
-
// Named shape by shape rather than as one policy word, so a stale grant is
|
|
4304
|
-
// legible in the transcript instead of only in the config file — the #122
|
|
4305
|
-
// incident began with a grant that no longer matched anyone's intent.
|
|
4306
|
-
`Release tool gate: ${
|
|
4307
|
-
grantedShapes.length === 0
|
|
4308
|
-
? "every release and deploy shape is mechanically blocked for you"
|
|
4309
|
-
: `you may invoke ${grantedShapes.join(", ")} — and only by the procedure in your POLICY.md`
|
|
4310
|
-
}.` + (deniedShapes.length === 0 ? "" : ` Blocked: ${deniedShapes.join(", ")}.`),
|
|
4311
|
-
"Handle each escalation below before the next one.",
|
|
4312
|
-
].join("\n");
|
|
4313
|
-
|
|
4314
|
-
// One orchestrator per daemon run, not per tick: it is a persistent session
|
|
4315
|
-
// whose whole value is remembering what it has already escalated, and a fresh
|
|
4316
|
-
// one every five minutes would remember nothing. Its cwd is the state
|
|
4317
|
-
// directory, deliberately not a checkout — the orchestrator re-briefs workers
|
|
4318
|
-
// and talks to the tracker, it does not edit product code.
|
|
4319
|
-
let orchestrator: OrchestratorHandle | undefined;
|
|
4320
|
-
if (project.escalation.orchestrator === "external") {
|
|
4321
|
-
// An operator already runs the brain — typically a visible TUI session that
|
|
4322
|
-
// drains `blocked`/`failed` off the tracker as one of its standing duties.
|
|
4323
|
-
// Starting a second one here would re-triage the same issues from a
|
|
4324
|
-
// transcript nobody is watching, and the two would undo each other.
|
|
4325
|
-
log("orchestrator: external — tier-1 escalations post as issue comments for the external session's drain duty");
|
|
4326
|
-
} else {
|
|
4327
|
-
try {
|
|
4328
|
-
// The orchestrator is a child process of this daemon running as its own
|
|
4329
|
-
// user. Nothing mechanically stops it reading a run checkout — what holds
|
|
4330
|
-
// it is its brief and the verb ledger (#143).
|
|
4331
|
-
const orchTreeRoot = stateDir();
|
|
4332
|
-
const orchCwd = join(orchTreeRoot, "orchestrator");
|
|
4333
|
-
mkdirSync(orchCwd, { recursive: true });
|
|
4334
|
-
// A third socket, distinct from every run's, in the same daemon-owned
|
|
4335
|
-
// 0711 parent. It makes the orchestrator's authority a property of the
|
|
4336
|
-
// channel rather than of a payload: no argument list can move a worker's
|
|
4337
|
-
// call onto this one (#126).
|
|
4338
|
-
//
|
|
4339
|
-
// It does NOT authenticate the session. Sessions share this daemon's uid,
|
|
4340
|
-
// so one could list this directory and connect here, and the ledger would
|
|
4341
|
-
// record the orchestrator's role because that is the channel's. See
|
|
4342
|
-
// conductor#163 for the pid binding that would close it.
|
|
4343
|
-
orchestratorVerbs = await listenVerbChannel(
|
|
4344
|
-
verbDeps({ project, store, tracker, verbActions }),
|
|
4345
|
-
{
|
|
4346
|
-
kind: "orchestrator",
|
|
4347
|
-
path: verbSocketPath(ensureVerbSocketDir(orchTreeRoot), "orchestrator"),
|
|
4348
|
-
project: project.name,
|
|
4349
|
-
role: "orchestrator",
|
|
4350
|
-
},
|
|
4351
|
-
{ ...(verbPeerReader === undefined ? {} : { peerReader: verbPeerReader }) },
|
|
4352
|
-
);
|
|
4353
|
-
orchestrator = await startOrchestrator({
|
|
4354
|
-
cwd: orchCwd,
|
|
4355
|
-
brief,
|
|
4356
|
-
releaseGrants,
|
|
4357
|
-
socketPath: join(orchCwd, "ipc.sock"),
|
|
4358
|
-
verbSocketPath: orchestratorVerbs.path,
|
|
4359
|
-
// The orchestrator's channel is the one that matters most: it is the
|
|
4360
|
-
// session authorised to merge, so an unbound channel here is a worker's
|
|
4361
|
-
// route to that authority (#163).
|
|
4362
|
-
onSpawn: (pid) => {
|
|
4363
|
-
orchestratorVerbs?.bindPid(pid);
|
|
4364
|
-
},
|
|
4365
|
-
onChildLog: (line) => {
|
|
4366
|
-
log(`orchestrator ${line}`);
|
|
4367
|
-
},
|
|
4368
|
-
onReleaseBlocked: (shape, context) =>
|
|
4369
|
-
recordReleaseBlock(project.name, "orchestrator", shape, context),
|
|
4370
|
-
});
|
|
4371
|
-
const transcript = orchestrator.sessionFile();
|
|
4372
|
-
log(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
|
|
4373
|
-
} catch (err) {
|
|
4374
|
-
// Loudly, but not fatally: tier-1 escalations degrade to issue comments,
|
|
4375
|
-
// which a human still reads. A dispatcher that refuses to run because its
|
|
4376
|
-
// re-briefing channel is down helps nobody.
|
|
4377
|
-
log(
|
|
4378
|
-
`WARNING: orchestrator session failed to start; tier-1 escalations will fall back to issue comments: ${errText(err)}`,
|
|
4595
|
+
const { brief, releaseGrants } = orchestratorStandingOrders(project);
|
|
4596
|
+
let orchestrator: OrchestratorHandle | undefined;
|
|
4597
|
+
let orchestratorVerbs: VerbListener | undefined;
|
|
4598
|
+
if (project.escalation.orchestrator === "external") {
|
|
4599
|
+
projectLog(
|
|
4600
|
+
"orchestrator: external — tier-1 escalations post as issue comments for the external session's drain duty",
|
|
4379
4601
|
);
|
|
4602
|
+
} else {
|
|
4603
|
+
try {
|
|
4604
|
+
const orchCwd = join(
|
|
4605
|
+
stateDir(),
|
|
4606
|
+
projects.length === 1 ? "orchestrator" : `orchestrator-${project.name}`,
|
|
4607
|
+
);
|
|
4608
|
+
mkdirSync(orchCwd, { recursive: true });
|
|
4609
|
+
orchestratorVerbs = await listenVerbChannel(
|
|
4610
|
+
verbDeps({ project, store, tracker, verbActions }),
|
|
4611
|
+
{
|
|
4612
|
+
kind: "orchestrator",
|
|
4613
|
+
path: verbSocketPath(verbDir, `orchestrator-${project.name}`),
|
|
4614
|
+
project: project.name,
|
|
4615
|
+
role: "orchestrator",
|
|
4616
|
+
},
|
|
4617
|
+
{ ...(verbPeerReader === undefined ? {} : { peerReader: verbPeerReader }) },
|
|
4618
|
+
);
|
|
4619
|
+
orchestrator = await startOrchestrator({
|
|
4620
|
+
cwd: orchCwd,
|
|
4621
|
+
brief,
|
|
4622
|
+
releaseGrants,
|
|
4623
|
+
socketPath: join(orchCwd, "ipc.sock"),
|
|
4624
|
+
verbSocketPath: orchestratorVerbs.path,
|
|
4625
|
+
onSpawn: (pid) => {
|
|
4626
|
+
orchestratorVerbs?.bindPid(pid);
|
|
4627
|
+
},
|
|
4628
|
+
onChildLog: (line) => {
|
|
4629
|
+
projectLog(`orchestrator ${line}`);
|
|
4630
|
+
},
|
|
4631
|
+
onReleaseBlocked: (shape, context) =>
|
|
4632
|
+
recordReleaseBlock(project.name, "orchestrator", shape, context),
|
|
4633
|
+
});
|
|
4634
|
+
const transcript = orchestrator.sessionFile();
|
|
4635
|
+
projectLog(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
|
|
4636
|
+
} catch (err) {
|
|
4637
|
+
projectLog(
|
|
4638
|
+
"WARNING: orchestrator session failed to start; tier-1 escalations will fall back to issue " +
|
|
4639
|
+
`comments: ${errText(err)}`,
|
|
4640
|
+
);
|
|
4641
|
+
await orchestratorVerbs?.close();
|
|
4642
|
+
orchestratorVerbs = undefined;
|
|
4643
|
+
}
|
|
4380
4644
|
}
|
|
4381
|
-
}
|
|
4382
|
-
|
|
4383
|
-
const escalator = createEscalator(project, tracker, store, orchestrator);
|
|
4384
4645
|
|
|
4385
|
-
|
|
4386
|
-
|
|
4387
|
-
|
|
4388
|
-
|
|
4389
|
-
|
|
4390
|
-
|
|
4391
|
-
|
|
4392
|
-
|
|
4393
|
-
|
|
4394
|
-
|
|
4395
|
-
|
|
4396
|
-
|
|
4397
|
-
|
|
4398
|
-
|
|
4399
|
-
|
|
4400
|
-
|
|
4401
|
-
|
|
4402
|
-
|
|
4403
|
-
|
|
4404
|
-
|
|
4405
|
-
|
|
4406
|
-
`
|
|
4407
|
-
|
|
4646
|
+
let runtimeDeps: Deps | undefined;
|
|
4647
|
+
const currentProject = (): ProjectConfig => runtimeDeps?.project ?? project;
|
|
4648
|
+
const deliveryPolicyValid = (): boolean => runtimeDeps?.deliveryPolicyValid === true;
|
|
4649
|
+
const escalator = createEscalator(
|
|
4650
|
+
currentProject,
|
|
4651
|
+
tracker,
|
|
4652
|
+
store,
|
|
4653
|
+
orchestrator,
|
|
4654
|
+
Date.now,
|
|
4655
|
+
deliveryPolicyValid,
|
|
4656
|
+
);
|
|
4657
|
+
const outbox = createReportOutbox({
|
|
4658
|
+
project: currentProject,
|
|
4659
|
+
store,
|
|
4660
|
+
escalate: (event) => escalator.escalate(event),
|
|
4661
|
+
log: projectLog,
|
|
4662
|
+
deliveryAllowed: deliveryPolicyValid,
|
|
4663
|
+
});
|
|
4664
|
+
if (alive === undefined || alive.pid === process.pid) {
|
|
4665
|
+
for (const report of outbox.recover(Date.now())) {
|
|
4666
|
+
projectLog(
|
|
4667
|
+
`report ${report.id} was left mid-send by a previous daemon (attempt ${report.attempts}) — ` +
|
|
4668
|
+
"retrying; its message will say it may be a repeat",
|
|
4669
|
+
);
|
|
4670
|
+
}
|
|
4408
4671
|
}
|
|
4409
|
-
}
|
|
4410
4672
|
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
|
|
4415
|
-
|
|
4416
|
-
|
|
4417
|
-
|
|
4418
|
-
|
|
4419
|
-
|
|
4420
|
-
|
|
4421
|
-
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4673
|
+
const d: Deps = {
|
|
4674
|
+
project,
|
|
4675
|
+
caps,
|
|
4676
|
+
tracker,
|
|
4677
|
+
store,
|
|
4678
|
+
deliveryPolicyValid: false,
|
|
4679
|
+
usage,
|
|
4680
|
+
escalate: (event) => escalator.escalate(event),
|
|
4681
|
+
turnLimits,
|
|
4682
|
+
workerControls,
|
|
4683
|
+
integrity,
|
|
4684
|
+
stall: { paged: false },
|
|
4685
|
+
cleanup: { next: 0 },
|
|
4686
|
+
...(verbPeerReader === undefined ? {} : { verbPeerReader }),
|
|
4687
|
+
verbActions,
|
|
4688
|
+
};
|
|
4689
|
+
runtimeDeps = d;
|
|
4690
|
+
runtimes.push({
|
|
4691
|
+
d,
|
|
4692
|
+
outbox,
|
|
4693
|
+
...(orchestrator === undefined ? {} : { orchestrator }),
|
|
4694
|
+
...(orchestratorVerbs === undefined ? {} : { orchestratorVerbs }),
|
|
4695
|
+
codeGraph: pendingCodeGraph(project),
|
|
4696
|
+
});
|
|
4697
|
+
}
|
|
4434
4698
|
|
|
4435
4699
|
if (o.once) {
|
|
4436
4700
|
try {
|
|
4437
|
-
await tick(d);
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
// mechanism it is drilling.
|
|
4441
|
-
await outbox.deliverDue();
|
|
4701
|
+
for (const runtime of runtimes) await tick(runtime.d, workers);
|
|
4702
|
+
await workers.drain();
|
|
4703
|
+
for (const runtime of runtimes) await runtime.outbox.deliverDue();
|
|
4442
4704
|
} finally {
|
|
4443
|
-
|
|
4444
|
-
|
|
4705
|
+
for (const runtime of runtimes.toReversed()) {
|
|
4706
|
+
await runtime.orchestrator?.dispose();
|
|
4707
|
+
await runtime.orchestratorVerbs?.close();
|
|
4708
|
+
}
|
|
4445
4709
|
store.close();
|
|
4446
4710
|
}
|
|
4447
4711
|
return;
|
|
4448
4712
|
}
|
|
4449
4713
|
|
|
4450
|
-
|
|
4451
|
-
|
|
4452
|
-
|
|
4453
|
-
let codeGraph = pendingCodeGraph(project);
|
|
4454
|
-
let graphProbe: Promise<void> | undefined;
|
|
4455
|
-
const refreshCodeGraph = (): void => {
|
|
4456
|
-
if (graphProbe !== undefined) return;
|
|
4457
|
-
graphProbe = probeCodeGraph(project)
|
|
4714
|
+
const refreshCodeGraph = (runtime: ProjectRuntime): void => {
|
|
4715
|
+
if (runtime.graphProbe !== undefined) return;
|
|
4716
|
+
runtime.graphProbe = probeCodeGraph(runtime.d.project)
|
|
4458
4717
|
.then((health) => {
|
|
4459
|
-
codeGraph = health;
|
|
4718
|
+
runtime.codeGraph = health;
|
|
4460
4719
|
})
|
|
4461
4720
|
.catch(() => {
|
|
4462
|
-
log(
|
|
4721
|
+
log(
|
|
4722
|
+
`[${runtime.d.project.name}] code-graph health probe failed unexpectedly; ` +
|
|
4723
|
+
"retaining the previous bounded result",
|
|
4724
|
+
);
|
|
4463
4725
|
})
|
|
4464
4726
|
.finally(() => {
|
|
4465
|
-
graphProbe = undefined;
|
|
4727
|
+
runtime.graphProbe = undefined;
|
|
4466
4728
|
});
|
|
4467
4729
|
};
|
|
4468
|
-
refreshCodeGraph();
|
|
4469
|
-
const graphTimer = setInterval(
|
|
4470
|
-
|
|
4471
|
-
|
|
4472
|
-
|
|
4473
|
-
|
|
4474
|
-
|
|
4475
|
-
|
|
4476
|
-
if (reportPass !== undefined) return;
|
|
4477
|
-
reportPass = outbox
|
|
4730
|
+
for (const runtime of runtimes) refreshCodeGraph(runtime);
|
|
4731
|
+
const graphTimer = setInterval(() => {
|
|
4732
|
+
for (const runtime of runtimes) refreshCodeGraph(runtime);
|
|
4733
|
+
}, GRAPH_HEALTH_INTERVAL_MS);
|
|
4734
|
+
|
|
4735
|
+
const drainReports = (runtime: ProjectRuntime): void => {
|
|
4736
|
+
if (runtime.reportPass !== undefined) return;
|
|
4737
|
+
runtime.reportPass = runtime.outbox
|
|
4478
4738
|
.deliverDue()
|
|
4479
4739
|
.then(() => {})
|
|
4480
4740
|
.catch((err: unknown) => {
|
|
4481
|
-
log(`report delivery pass failed: ${errText(err)}`);
|
|
4741
|
+
log(`[${runtime.d.project.name}] report delivery pass failed: ${errText(err)}`);
|
|
4482
4742
|
})
|
|
4483
4743
|
.finally(() => {
|
|
4484
|
-
reportPass = undefined;
|
|
4744
|
+
runtime.reportPass = undefined;
|
|
4485
4745
|
});
|
|
4486
4746
|
};
|
|
4487
|
-
drainReports();
|
|
4488
|
-
const reportTimer = setInterval(
|
|
4747
|
+
for (const runtime of runtimes) drainReports(runtime);
|
|
4748
|
+
const reportTimer = setInterval(() => {
|
|
4749
|
+
for (const runtime of runtimes) drainReports(runtime);
|
|
4750
|
+
}, REPORT_DELIVERY_INTERVAL_MS);
|
|
4489
4751
|
|
|
4490
|
-
const workers = createWorkerPool();
|
|
4491
4752
|
let stopping = false;
|
|
4492
4753
|
let wake: (() => void) | undefined;
|
|
4493
4754
|
const stop = (): void => {
|
|
@@ -4499,54 +4760,58 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
4499
4760
|
process.on("SIGINT", stop);
|
|
4500
4761
|
process.on("SIGTERM", stop);
|
|
4501
4762
|
|
|
4502
|
-
//
|
|
4503
|
-
//
|
|
4504
|
-
//
|
|
4505
|
-
// carries no credential of any kind, and it cannot tell one caller from
|
|
4506
|
-
// another: `127.0.0.1` is not an identity. The turn-limit and worker
|
|
4507
|
-
// pause/resume controls trust a body-supplied `project`, which is exactly the
|
|
4508
|
-
// shape "identity from the payload" takes when nobody is watching. They stay
|
|
4509
|
-
// tolerable only because every effect is bounded: pauses are reversible,
|
|
4510
|
-
// live extensions touch one controller, and a persisted next-attempt
|
|
4511
|
-
// override can only raise the project base up to its configured ceiling and
|
|
4512
|
-
// is consumed by one claim.
|
|
4513
|
-
//
|
|
4514
|
-
// A merge, a push, a release or a label is none of those things. Do not add
|
|
4515
|
-
// one here, and do not add "just a small one" behind a shared secret either:
|
|
4516
|
-
// a secret readable by the process that would be attacking you is not
|
|
4517
|
-
// authentication. Mutations go over the per-run unix sockets in
|
|
4518
|
-
// `verbs/socket.ts`, where the kernel says who the caller is and the daemon
|
|
4519
|
-
// derives project, run and role from the channel rather than the body (#126).
|
|
4763
|
+
// This unauthenticated loopback surface exposes liveness and bounded live-run
|
|
4764
|
+
// controls only. Repository and tracker mutations stay on credentialled verb
|
|
4765
|
+
// sockets, where project and role come from the channel rather than a payload.
|
|
4520
4766
|
const server = Bun.serve({
|
|
4521
4767
|
hostname: "127.0.0.1",
|
|
4522
4768
|
port: o.port ?? DEFAULT_PORT,
|
|
4523
4769
|
fetch: (req) =>
|
|
4524
4770
|
daemonHttpResponse(req, {
|
|
4525
|
-
|
|
4526
|
-
|
|
4527
|
-
|
|
4771
|
+
projects: runtimes.map((runtime) => ({
|
|
4772
|
+
project: runtime.d.project.name,
|
|
4773
|
+
store,
|
|
4774
|
+
caps: () => runtime.d.caps,
|
|
4775
|
+
})),
|
|
4528
4776
|
turnLimits,
|
|
4529
4777
|
workerControls,
|
|
4530
4778
|
health: () =>
|
|
4531
|
-
|
|
4779
|
+
daemonHealth(
|
|
4780
|
+
runtimes.map((runtime) =>
|
|
4781
|
+
daemonHealthSnapshot(
|
|
4782
|
+
store,
|
|
4783
|
+
runtime.d.project.name,
|
|
4784
|
+
isPaused(runtime.d.project.name),
|
|
4785
|
+
runtime.codeGraph,
|
|
4786
|
+
workerControls,
|
|
4787
|
+
),
|
|
4788
|
+
),
|
|
4789
|
+
),
|
|
4532
4790
|
}),
|
|
4533
4791
|
});
|
|
4534
|
-
log(
|
|
4792
|
+
log(
|
|
4793
|
+
projects.length === 1
|
|
4794
|
+
? `serving /healthz on :${server.port}, project ${projects[0]!.name}`
|
|
4795
|
+
: `serving /healthz on :${server.port}, projects ${projects.map(({ name }) => name).join(", ")}`,
|
|
4796
|
+
);
|
|
4535
4797
|
|
|
4536
4798
|
try {
|
|
4537
4799
|
while (!stopping) {
|
|
4538
|
-
|
|
4539
|
-
|
|
4540
|
-
|
|
4541
|
-
|
|
4542
|
-
|
|
4543
|
-
|
|
4800
|
+
for (const runtime of runtimes) {
|
|
4801
|
+
try {
|
|
4802
|
+
await tick(runtime.d, workers);
|
|
4803
|
+
} catch (err) {
|
|
4804
|
+
log(
|
|
4805
|
+
`${projects.length === 1 ? "" : `[${runtime.d.project.name}] `}` +
|
|
4806
|
+
`tick failed: ${errText(err)}`,
|
|
4807
|
+
);
|
|
4808
|
+
}
|
|
4544
4809
|
}
|
|
4545
4810
|
if (stopping) break;
|
|
4546
4811
|
await new Promise<void>((resolve) => {
|
|
4547
|
-
const
|
|
4812
|
+
const timer = setTimeout(resolve, TICK_INTERVAL_MS);
|
|
4548
4813
|
wake = () => {
|
|
4549
|
-
clearTimeout(
|
|
4814
|
+
clearTimeout(timer);
|
|
4550
4815
|
resolve();
|
|
4551
4816
|
};
|
|
4552
4817
|
});
|
|
@@ -4557,26 +4822,15 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
4557
4822
|
process.off("SIGTERM", stop);
|
|
4558
4823
|
clearInterval(graphTimer);
|
|
4559
4824
|
clearInterval(reportTimer);
|
|
4560
|
-
|
|
4561
|
-
// a `markReportDelivered` still to write, and losing that write is exactly
|
|
4562
|
-
// how a delivered report comes back as an ambiguous one on the next boot.
|
|
4563
|
-
await reportPass;
|
|
4825
|
+
for (const runtime of runtimes) await runtime.reportPass;
|
|
4564
4826
|
await workers.drain();
|
|
4565
4827
|
await server.stop(true);
|
|
4566
|
-
|
|
4567
|
-
|
|
4568
|
-
|
|
4569
|
-
|
|
4570
|
-
// orchestrator is a channel accepting merges for a session that no longer
|
|
4571
|
-
// exists.
|
|
4572
|
-
await orchestratorVerbs?.close();
|
|
4828
|
+
for (const runtime of runtimes.toReversed()) {
|
|
4829
|
+
await runtime.orchestrator?.dispose();
|
|
4830
|
+
await runtime.orchestratorVerbs?.close();
|
|
4831
|
+
}
|
|
4573
4832
|
store.close();
|
|
4574
4833
|
log("stopped");
|
|
4575
|
-
// A handled SIGTERM still leaves some runtimes with a non-zero default
|
|
4576
|
-
// (historically 128+signal). Under systemd `Restart=on-failure` that looks
|
|
4577
|
-
// like a crash and the unit comes straight back — the exact failure mode
|
|
4578
|
-
// `omp-conductor stop` hit on the reference fleet. Force success so a
|
|
4579
|
-
// graceful drain is not a restart.
|
|
4580
4834
|
process.exitCode = 0;
|
|
4581
4835
|
}
|
|
4582
4836
|
}
|