omp-conductor 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -19
- package/package.json +1 -1
- package/src/board.ts +1 -1
- package/src/briefs/orchestrator.md +32 -2
- package/src/cli.ts +104 -23
- package/src/config.ts +10 -2
- package/src/daemon.ts +818 -131
- package/src/diff-flags.ts +4 -0
- package/src/escalate.ts +5 -5
- package/src/failure-class.ts +7 -5
- package/src/fleet.ts +7 -2
- package/src/omp.ts +8 -4
- package/src/orchestrator-tick.ts +35 -20
- package/src/orchestrator.ts +3 -2
- package/src/plugin.ts +28 -2
- package/src/release-policy.ts +66 -6
- package/src/session-host.ts +4 -3
- package/src/setup.ts +4 -2
- package/src/store.ts +190 -29
- package/src/tracker/github.ts +261 -56
- package/src/types.ts +141 -28
- package/src/verbs/actions.ts +127 -15
- package/src/verbs/protocol.ts +7 -6
- package/src/verbs/server.ts +95 -13
- package/src/worker.ts +33 -5
- package/src/worktree.ts +5 -0
package/src/verbs/server.ts
CHANGED
|
@@ -265,6 +265,10 @@ export interface ReleaseFacts {
|
|
|
265
265
|
openPrs: number;
|
|
266
266
|
/** Queue depth, or `undefined` when the tracker could not be read. */
|
|
267
267
|
queueDepth: number | undefined;
|
|
268
|
+
/** Newest observed verdict for the released routed repository. */
|
|
269
|
+
baseCheck?: RunRecord["baseCheck"];
|
|
270
|
+
/** Evidence attached to a newest red verdict. */
|
|
271
|
+
redBase?: string;
|
|
268
272
|
}
|
|
269
273
|
|
|
270
274
|
export function releaseRequirementRefusal(
|
|
@@ -286,6 +290,19 @@ export function releaseRequirementRefusal(
|
|
|
286
290
|
return `policy.release.requires includes queue-drained and ${facts.queueDepth} issue(s) are still queued`;
|
|
287
291
|
}
|
|
288
292
|
}
|
|
293
|
+
if (requirement === "base-branch-green") {
|
|
294
|
+
if (facts.baseCheck === "green") continue;
|
|
295
|
+
if (facts.baseCheck === "pending") {
|
|
296
|
+
return "policy.release.requires includes base-branch-green and the newest merge's base workflows are still pending";
|
|
297
|
+
}
|
|
298
|
+
if (facts.baseCheck === "red" || facts.baseCheck === "red-preexisting") {
|
|
299
|
+
return (
|
|
300
|
+
"policy.release.requires includes base-branch-green and the newest observed base branch is red" +
|
|
301
|
+
(facts.redBase === undefined ? "" : `: ${facts.redBase}`)
|
|
302
|
+
);
|
|
303
|
+
}
|
|
304
|
+
return "policy.release.requires includes base-branch-green and no green base-branch verdict is available; refusing rather than assuming the base is healthy";
|
|
305
|
+
}
|
|
289
306
|
if (requirement === "epic-children-closed") {
|
|
290
307
|
return "policy.release.requires includes epic-children-closed, which nothing in a release request names an epic for. The daemon cannot settle it, so it refuses: take the requirement off this project's policy, or cut this release by hand.";
|
|
291
308
|
}
|
|
@@ -423,19 +440,32 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
|
|
|
423
440
|
}
|
|
424
441
|
}
|
|
425
442
|
|
|
426
|
-
// 4. Stop always wins, and is re-read here
|
|
427
|
-
//
|
|
428
|
-
//
|
|
429
|
-
//
|
|
430
|
-
//
|
|
431
|
-
//
|
|
432
|
-
|
|
433
|
-
|
|
443
|
+
// 4. Stop always wins for work-starting mutations, and is re-read here
|
|
444
|
+
// rather than remembered. Reads are exempt. A run admitted before the
|
|
445
|
+
// pause may finish its own mutations (#174). Orchestrator completion
|
|
446
|
+
// verbs resolve their target run inside the handler, where they can prove
|
|
447
|
+
// that run also predates the pause (#251). Release remains available:
|
|
448
|
+
// its authority, grant, and runs-settled gates still fail closed below.
|
|
449
|
+
const orchestratorCompletion =
|
|
450
|
+
channel.kind === "orchestrator" &&
|
|
451
|
+
(verb === "conductor_pr_merge" ||
|
|
452
|
+
verb === "conductor_pr_update_branch" ||
|
|
453
|
+
verb === "conductor_pr_update" ||
|
|
454
|
+
verb === "conductor_label");
|
|
455
|
+
const stopped =
|
|
456
|
+
spec.mutating && verb !== "conductor_release" && !orchestratorCompletion
|
|
457
|
+
? deps.fleetStop()
|
|
458
|
+
: undefined;
|
|
434
459
|
if (stopped !== undefined) {
|
|
435
460
|
const at = deps.pausedAt();
|
|
436
461
|
const admittedBeforePause = run !== undefined && at !== undefined && run.startedAt < at;
|
|
437
462
|
if (!admittedBeforePause) {
|
|
438
|
-
return refuse(
|
|
463
|
+
return refuse(
|
|
464
|
+
"fleet-paused",
|
|
465
|
+
`refused: ${stopped}. The pause refuses new claims and work-starting mutations. ` +
|
|
466
|
+
"Completion verbs for runs admitted before the pause (conductor_pr_merge, conductor_pr_update_branch, " +
|
|
467
|
+
"conductor_pr_update, conductor_label) and conductor_release remain available.",
|
|
468
|
+
);
|
|
439
469
|
}
|
|
440
470
|
}
|
|
441
471
|
|
|
@@ -599,7 +629,7 @@ async function prCreateVerb(
|
|
|
599
629
|
* How far back to look is the recent-history cutoff `status` already uses: a
|
|
600
630
|
* pull request older than that is not one an orchestrator is mid-flight on.
|
|
601
631
|
*/
|
|
602
|
-
const PR_LOOKUP_WINDOW_MS = 30 * 24 * 60 * 60_000;
|
|
632
|
+
export const PR_LOOKUP_WINDOW_MS = 30 * 24 * 60 * 60_000;
|
|
603
633
|
|
|
604
634
|
function runForPr(deps: VerbDeps, project: string, prUrl: string): RunRecord | undefined {
|
|
605
635
|
return deps.store
|
|
@@ -607,6 +637,31 @@ function runForPr(deps: VerbDeps, project: string, prUrl: string): RunRecord | u
|
|
|
607
637
|
.find((candidate) => candidate.prUrl === prUrl);
|
|
608
638
|
}
|
|
609
639
|
|
|
640
|
+
/** Refuse an orchestrator completion mutation that cannot prove a pre-pause run. */
|
|
641
|
+
function orchestratorPauseRefusal(
|
|
642
|
+
deps: VerbDeps,
|
|
643
|
+
channel: VerbChannel,
|
|
644
|
+
target: RunRecord | undefined,
|
|
645
|
+
refuse: Refuse,
|
|
646
|
+
): Verdict | undefined {
|
|
647
|
+
if (channel.kind !== "orchestrator") return undefined;
|
|
648
|
+
const stopped = deps.fleetStop();
|
|
649
|
+
if (stopped === undefined) return undefined;
|
|
650
|
+
const at = deps.pausedAt();
|
|
651
|
+
if (target !== undefined && at !== undefined && target.startedAt < at) return undefined;
|
|
652
|
+
const why =
|
|
653
|
+
target === undefined
|
|
654
|
+
? "this issue has no recorded run admitted before it."
|
|
655
|
+
: at === undefined
|
|
656
|
+
? "the pause timestamp cannot prove when this run started."
|
|
657
|
+
: "this run started after it.";
|
|
658
|
+
return refuse(
|
|
659
|
+
"fleet-paused",
|
|
660
|
+
`refused: ${stopped}. Under pause, this verb applies only to runs admitted before the pause; ${why}`,
|
|
661
|
+
target?.issue,
|
|
662
|
+
);
|
|
663
|
+
}
|
|
664
|
+
|
|
610
665
|
async function prUpdateBranchVerb(
|
|
611
666
|
deps: VerbDeps,
|
|
612
667
|
project: ProjectConfig,
|
|
@@ -638,6 +693,9 @@ async function prUpdateBranchVerb(
|
|
|
638
693
|
}
|
|
639
694
|
}
|
|
640
695
|
|
|
696
|
+
const paused = orchestratorPauseRefusal(deps, channel, target, refuse);
|
|
697
|
+
if (paused !== undefined) return paused;
|
|
698
|
+
|
|
641
699
|
let state: PrState | undefined;
|
|
642
700
|
try {
|
|
643
701
|
state = await deps.tracker.prState(prUrl);
|
|
@@ -691,6 +749,9 @@ async function prMergeVerb(
|
|
|
691
749
|
return refuse("pr-not-this-run", `refused: ${prUrl} is not a pull request any run in ${project.name} opened.`);
|
|
692
750
|
}
|
|
693
751
|
|
|
752
|
+
const paused = orchestratorPauseRefusal(deps, channel, target, refuse);
|
|
753
|
+
if (paused !== undefined) return paused;
|
|
754
|
+
|
|
694
755
|
// Taken before any network call, so two concurrent callers contend here
|
|
695
756
|
// rather than both spending a `gh` round trip and racing at the merge.
|
|
696
757
|
const holderId = randomUUID();
|
|
@@ -814,6 +875,14 @@ async function labelVerb(
|
|
|
814
875
|
);
|
|
815
876
|
}
|
|
816
877
|
|
|
878
|
+
const paused = orchestratorPauseRefusal(
|
|
879
|
+
deps,
|
|
880
|
+
channel,
|
|
881
|
+
deps.store.latestRun(project.name, ref.issue),
|
|
882
|
+
refuse,
|
|
883
|
+
);
|
|
884
|
+
if (paused !== undefined) return paused;
|
|
885
|
+
|
|
817
886
|
const label = String(args["label"]);
|
|
818
887
|
const vocabulary = labelVocabulary(project);
|
|
819
888
|
if (vocabulary.lifecycle.includes(label)) {
|
|
@@ -908,10 +977,19 @@ async function releaseVerb(
|
|
|
908
977
|
queueDepth = undefined;
|
|
909
978
|
}
|
|
910
979
|
}
|
|
980
|
+
const wantsBase = policy.release.requires.includes("base-branch-green");
|
|
981
|
+
const latestBase = wantsBase
|
|
982
|
+
? deps.store.latestBaseChecks(project.name, 0).find((run) => run.repo === repoName)
|
|
983
|
+
: undefined;
|
|
984
|
+
const redBase = latestBase?.settlementFlags?.find(
|
|
985
|
+
(flag) => flag.kind === "base-branch-red",
|
|
986
|
+
)?.detail;
|
|
911
987
|
const unmet = releaseRequirementRefusal(policy.release.requires, {
|
|
912
988
|
unsettledRuns: active.length,
|
|
913
989
|
openPrs: active.filter((r) => r.prUrl !== undefined).length,
|
|
914
990
|
queueDepth,
|
|
991
|
+
...(latestBase?.baseCheck === undefined ? {} : { baseCheck: latestBase.baseCheck }),
|
|
992
|
+
...(redBase === undefined ? {} : { redBase }),
|
|
915
993
|
});
|
|
916
994
|
if (unmet !== undefined) return refuse("release-not-granted", `refused: ${unmet}`);
|
|
917
995
|
|
|
@@ -991,10 +1069,11 @@ async function prStatusVerb(
|
|
|
991
1069
|
issue = target.issue;
|
|
992
1070
|
}
|
|
993
1071
|
|
|
994
|
-
const
|
|
1072
|
+
const askedHead = args["headSha"];
|
|
1073
|
+
const expectedHead = typeof askedHead === "string" ? askedHead : undefined;
|
|
995
1074
|
let verification: PrVerification | undefined;
|
|
996
1075
|
try {
|
|
997
|
-
verification = await deps.tracker.verifyPr(prUrl,
|
|
1076
|
+
verification = await deps.tracker.verifyPr(prUrl, expectedHead);
|
|
998
1077
|
} catch (err) {
|
|
999
1078
|
verification = undefined;
|
|
1000
1079
|
deps.log(`verb pr_status could not read ${prUrl}: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -1007,7 +1086,7 @@ async function prStatusVerb(
|
|
|
1007
1086
|
issue,
|
|
1008
1087
|
);
|
|
1009
1088
|
}
|
|
1010
|
-
return allow(`${prUrl} at ${headSha}: ${verification.status} — ${verification.reason}`, undefined, issue);
|
|
1089
|
+
return allow(`${prUrl} at ${verification.headSha}: ${verification.status} — ${verification.reason}`, undefined, issue);
|
|
1011
1090
|
}
|
|
1012
1091
|
|
|
1013
1092
|
/**
|
|
@@ -1054,6 +1133,9 @@ async function prUpdateVerb(
|
|
|
1054
1133
|
if (target === undefined) {
|
|
1055
1134
|
return refuse("pr-not-this-run", `refused: ${asked} is not a pull request any run in ${project.name} opened.`);
|
|
1056
1135
|
}
|
|
1136
|
+
|
|
1137
|
+
const paused = orchestratorPauseRefusal(deps, channel, target, refuse);
|
|
1138
|
+
if (paused !== undefined) return paused;
|
|
1057
1139
|
prUrl = asked;
|
|
1058
1140
|
issue = target.issue;
|
|
1059
1141
|
}
|
package/src/worker.ts
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
import { createSession, disposeSession } from "./omp.ts";
|
|
14
|
+
import type { ReleaseBlockContext } from "./release-policy.ts";
|
|
14
15
|
import type { Caps, ReleaseShape, ResolvedGrants, RunState } from "./types.ts";
|
|
15
16
|
|
|
16
17
|
/** Structured evidence fields from the worker's final report. */
|
|
@@ -47,6 +48,9 @@ export interface WorkerPauseControl {
|
|
|
47
48
|
/** Returns a parked session to work with a continuation prompt. Throws
|
|
48
49
|
* when the phase is not `paused`. */
|
|
49
50
|
resume(): void;
|
|
51
|
+
/** Terminally ends this run for an operator-supplied reason. The run is
|
|
52
|
+
* recorded as an administrative kill and cannot be resumed. */
|
|
53
|
+
stop(reason: string): void;
|
|
50
54
|
}
|
|
51
55
|
|
|
52
56
|
/** What a resumed session is told. One literal so tests can pin it. */
|
|
@@ -78,7 +82,7 @@ export interface WorkerOpts {
|
|
|
78
82
|
*/
|
|
79
83
|
releaseGrants?: ResolvedGrants;
|
|
80
84
|
/** Durable audit sink for rejected release/deploy calls. */
|
|
81
|
-
onReleaseBlocked?: (shape: ReleaseShape) => void;
|
|
85
|
+
onReleaseBlocked?: (shape: ReleaseShape, context: ReleaseBlockContext) => void;
|
|
82
86
|
/** The child's pid, the instant it exists. See {@link VerbListener.bindPid}. */
|
|
83
87
|
onSpawn?: (pid: number) => void;
|
|
84
88
|
/** Control socket for that child, beside the run's own session directory. */
|
|
@@ -135,6 +139,8 @@ export interface WorkerResult {
|
|
|
135
139
|
spendUsd: number;
|
|
136
140
|
report: string;
|
|
137
141
|
killedBy?: KilledBy;
|
|
142
|
+
/** Present only when an operator terminally stopped this run. */
|
|
143
|
+
stoppedReason?: string;
|
|
138
144
|
/**
|
|
139
145
|
* Transcript the session actually opened, absent if it opened none. Recorded
|
|
140
146
|
* per run because it is the only readable evidence left once the worktree is
|
|
@@ -280,6 +286,7 @@ export async function runWorker(
|
|
|
280
286
|
// merged (#217).
|
|
281
287
|
let claim: { prUrl: string; headSha: string } | undefined;
|
|
282
288
|
let killedBy: KilledBy | undefined;
|
|
289
|
+
let stoppedReason: string | undefined;
|
|
283
290
|
// Canceler for the armed wall clock; invoked on every exit path below.
|
|
284
291
|
let cancelWallClock: (() => void) | undefined;
|
|
285
292
|
let wallClockRemainingMs = workerWallClockMs;
|
|
@@ -309,7 +316,7 @@ export async function runWorker(
|
|
|
309
316
|
};
|
|
310
317
|
|
|
311
318
|
const kill = (by: KilledBy) => {
|
|
312
|
-
if (killedBy !== undefined) return;
|
|
319
|
+
if (killedBy !== undefined || stoppedReason !== undefined) return;
|
|
313
320
|
killedBy = by;
|
|
314
321
|
o.onKilled?.(by);
|
|
315
322
|
clearWallClock();
|
|
@@ -331,7 +338,7 @@ export async function runWorker(
|
|
|
331
338
|
resumeWaiter = undefined;
|
|
332
339
|
pauseRequested = Promise.withResolvers<void>();
|
|
333
340
|
}
|
|
334
|
-
return killedBy !== undefined || done ? undefined : prompt;
|
|
341
|
+
return killedBy !== undefined || stoppedReason !== undefined || done ? undefined : prompt;
|
|
335
342
|
};
|
|
336
343
|
|
|
337
344
|
o.onPauseControl?.({
|
|
@@ -388,6 +395,16 @@ export async function runWorker(
|
|
|
388
395
|
armWallClock();
|
|
389
396
|
resumeWaiter?.resolve(RESUME_PROMPT);
|
|
390
397
|
},
|
|
398
|
+
stop: (reason) => {
|
|
399
|
+
if (killedBy !== undefined || done) {
|
|
400
|
+
throw new Error("the run is settling; nothing left to stop");
|
|
401
|
+
}
|
|
402
|
+
stoppedReason = reason;
|
|
403
|
+
clearWallClock();
|
|
404
|
+
done = true;
|
|
405
|
+
settle();
|
|
406
|
+
session.abort();
|
|
407
|
+
},
|
|
391
408
|
});
|
|
392
409
|
|
|
393
410
|
session.on("turn_start", () => {
|
|
@@ -464,7 +481,7 @@ export async function runWorker(
|
|
|
464
481
|
} catch (cause) {
|
|
465
482
|
// Our own abort surfaces here on some paths: a cap kill (existing
|
|
466
483
|
// behavior) or an operator park (new). Anything else is a real failure.
|
|
467
|
-
if (killedBy === undefined && resumeWaiter === undefined) throw cause;
|
|
484
|
+
if (killedBy === undefined && stoppedReason === undefined && resumeWaiter === undefined) throw cause;
|
|
468
485
|
}
|
|
469
486
|
next = undefined;
|
|
470
487
|
if (killedBy !== undefined) break;
|
|
@@ -483,7 +500,7 @@ export async function runWorker(
|
|
|
483
500
|
}
|
|
484
501
|
} catch (cause) {
|
|
485
502
|
// Our own abort surfaces here on some paths; that is a kill, not a crash.
|
|
486
|
-
if (killedBy === undefined) {
|
|
503
|
+
if (killedBy === undefined && stoppedReason === undefined) {
|
|
487
504
|
const detail = cause instanceof Error ? cause.message : String(cause);
|
|
488
505
|
return withSessionFacts({
|
|
489
506
|
state: "failed",
|
|
@@ -504,6 +521,17 @@ export async function runWorker(
|
|
|
504
521
|
}
|
|
505
522
|
}
|
|
506
523
|
|
|
524
|
+
if (stoppedReason !== undefined) {
|
|
525
|
+
return withSessionFacts({
|
|
526
|
+
state: "stopped",
|
|
527
|
+
turns,
|
|
528
|
+
spendUsd,
|
|
529
|
+
report,
|
|
530
|
+
stoppedReason,
|
|
531
|
+
...(claim === undefined ? {} : { prUrl: claim.prUrl, headSha: claim.headSha }),
|
|
532
|
+
});
|
|
533
|
+
}
|
|
534
|
+
|
|
507
535
|
if (killedBy !== undefined) {
|
|
508
536
|
// The PR and head are facts the session already established, so they
|
|
509
537
|
// survive the kill. Without them `shouldContinueAfterTurnsCap` sees no
|
package/src/worktree.ts
CHANGED
|
@@ -317,6 +317,11 @@ async function ensureMirrorUnlocked(repo: RepoTarget, mirrorRoot: string): Promi
|
|
|
317
317
|
throw err;
|
|
318
318
|
}
|
|
319
319
|
await configureMirror(mirrorPath);
|
|
320
|
+
// `clone --mirror` writes upstream heads under `refs/heads/*`; the safe
|
|
321
|
+
// normal-clone refspec above uses `refs/remotes/origin/*`. Populate that
|
|
322
|
+
// namespace before returning so first-use callers see the same fresh mirror
|
|
323
|
+
// shape as every later refresh.
|
|
324
|
+
await refreshMirror(mirrorPath);
|
|
320
325
|
return mirrorPath;
|
|
321
326
|
}
|
|
322
327
|
|