omp-conductor 0.10.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +73 -19
- package/package.json +1 -1
- package/src/board.ts +125 -11
- package/src/briefs/orchestrator.md +50 -4
- package/src/briefs/policy.md +4 -1
- package/src/chain-check.ts +157 -0
- package/src/cli.ts +188 -19
- package/src/config.ts +195 -15
- package/src/daemon.ts +1017 -119
- package/src/diff-flags.ts +48 -3
- package/src/digest-schedule.ts +59 -0
- package/src/escalate.ts +17 -0
- package/src/failure-class.ts +31 -4
- package/src/fleet.ts +8 -3
- package/src/gitops.ts +49 -0
- package/src/omp.ts +44 -9
- package/src/orchestrator-tick.ts +94 -9
- package/src/orchestrator.ts +3 -2
- package/src/plugin.ts +46 -4
- package/src/release-policy.ts +66 -6
- package/src/reports.ts +5 -6
- package/src/session-host.ts +23 -6
- package/src/setup.ts +43 -10
- package/src/store.ts +263 -31
- package/src/tracker/github.ts +261 -56
- package/src/types.ts +251 -32
- package/src/verbs/actions.ts +127 -15
- package/src/verbs/protocol.ts +7 -6
- package/src/verbs/server.ts +144 -13
- package/src/worker.ts +183 -17
- package/src/worktree.ts +5 -0
package/src/verbs/protocol.ts
CHANGED
|
@@ -293,9 +293,10 @@ export const VERB_SPECS: Readonly<Record<VerbName, VerbSpec>> = {
|
|
|
293
293
|
mutating: false,
|
|
294
294
|
allowedRoles: ["worker", "orchestrator"],
|
|
295
295
|
description:
|
|
296
|
-
"Read one pull request's live state: is it open
|
|
297
|
-
"
|
|
298
|
-
"
|
|
296
|
+
"Read one pull request's live state and current head: is it open and are its checks green. " +
|
|
297
|
+
"Optionally compare that head with one the caller already observed. This is the daemon's own " +
|
|
298
|
+
"merge-gate verdict, not a summary of it. A poll, not a watcher — call it again after waiting. " +
|
|
299
|
+
"A worker may omit prUrl and gets its own run's.",
|
|
299
300
|
args: {
|
|
300
301
|
prUrl: {
|
|
301
302
|
type: "string",
|
|
@@ -304,10 +305,10 @@ export const VERB_SPECS: Readonly<Record<VerbName, VerbSpec>> = {
|
|
|
304
305
|
},
|
|
305
306
|
headSha: {
|
|
306
307
|
type: "string",
|
|
307
|
-
required:
|
|
308
|
+
required: false,
|
|
308
309
|
description:
|
|
309
|
-
"
|
|
310
|
-
"moved, the answer names both shas.",
|
|
310
|
+
"Optional: an expected head to compare against; omit it to read the current head. If the " +
|
|
311
|
+
"branch has moved, the answer names both shas.",
|
|
311
312
|
},
|
|
312
313
|
},
|
|
313
314
|
roleRefusalText: (role) => `conductor_pr_status is not open to a ${role} session.`,
|
package/src/verbs/server.ts
CHANGED
|
@@ -42,6 +42,8 @@ import { randomUUID } from "node:crypto";
|
|
|
42
42
|
import { createServer, type Server, type Socket } from "node:net";
|
|
43
43
|
|
|
44
44
|
import { resolvePolicy, resolveReleaseGrants } from "../config.ts";
|
|
45
|
+
import { chainEntriesFromDiff, chainViolations } from "../chain-check.ts";
|
|
46
|
+
import type { readBaseChain as readBaseChainType } from "../gitops.ts";
|
|
45
47
|
import { releaseRefusal } from "../release-policy.ts";
|
|
46
48
|
import { LIVE_STATES } from "../store.ts";
|
|
47
49
|
import type {
|
|
@@ -181,6 +183,12 @@ export interface VerbDeps {
|
|
|
181
183
|
pausedAt: () => number | undefined;
|
|
182
184
|
log: (message: string) => void;
|
|
183
185
|
now: () => number;
|
|
186
|
+
/**
|
|
187
|
+
* Read-only access to the ordered-migration-chain guard. A read seam, kept
|
|
188
|
+
* off `VerbActions` (the mutation surface) on purpose: `prMergeVerb` consults
|
|
189
|
+
* whether the merge would fork the chain, it never changes it (#227).
|
|
190
|
+
*/
|
|
191
|
+
chain: { readBaseChain: typeof readBaseChainType };
|
|
184
192
|
}
|
|
185
193
|
|
|
186
194
|
/**
|
|
@@ -257,6 +265,10 @@ export interface ReleaseFacts {
|
|
|
257
265
|
openPrs: number;
|
|
258
266
|
/** Queue depth, or `undefined` when the tracker could not be read. */
|
|
259
267
|
queueDepth: number | undefined;
|
|
268
|
+
/** Newest observed verdict for the released routed repository. */
|
|
269
|
+
baseCheck?: RunRecord["baseCheck"];
|
|
270
|
+
/** Evidence attached to a newest red verdict. */
|
|
271
|
+
redBase?: string;
|
|
260
272
|
}
|
|
261
273
|
|
|
262
274
|
export function releaseRequirementRefusal(
|
|
@@ -278,6 +290,19 @@ export function releaseRequirementRefusal(
|
|
|
278
290
|
return `policy.release.requires includes queue-drained and ${facts.queueDepth} issue(s) are still queued`;
|
|
279
291
|
}
|
|
280
292
|
}
|
|
293
|
+
if (requirement === "base-branch-green") {
|
|
294
|
+
if (facts.baseCheck === "green") continue;
|
|
295
|
+
if (facts.baseCheck === "pending") {
|
|
296
|
+
return "policy.release.requires includes base-branch-green and the newest merge's base workflows are still pending";
|
|
297
|
+
}
|
|
298
|
+
if (facts.baseCheck === "red" || facts.baseCheck === "red-preexisting") {
|
|
299
|
+
return (
|
|
300
|
+
"policy.release.requires includes base-branch-green and the newest observed base branch is red" +
|
|
301
|
+
(facts.redBase === undefined ? "" : `: ${facts.redBase}`)
|
|
302
|
+
);
|
|
303
|
+
}
|
|
304
|
+
return "policy.release.requires includes base-branch-green and no green base-branch verdict is available; refusing rather than assuming the base is healthy";
|
|
305
|
+
}
|
|
281
306
|
if (requirement === "epic-children-closed") {
|
|
282
307
|
return "policy.release.requires includes epic-children-closed, which nothing in a release request names an epic for. The daemon cannot settle it, so it refuses: take the requirement off this project's policy, or cut this release by hand.";
|
|
283
308
|
}
|
|
@@ -415,19 +440,32 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
|
|
|
415
440
|
}
|
|
416
441
|
}
|
|
417
442
|
|
|
418
|
-
// 4. Stop always wins, and is re-read here
|
|
419
|
-
//
|
|
420
|
-
//
|
|
421
|
-
//
|
|
422
|
-
//
|
|
423
|
-
//
|
|
424
|
-
|
|
425
|
-
|
|
443
|
+
// 4. Stop always wins for work-starting mutations, and is re-read here
|
|
444
|
+
// rather than remembered. Reads are exempt. A run admitted before the
|
|
445
|
+
// pause may finish its own mutations (#174). Orchestrator completion
|
|
446
|
+
// verbs resolve their target run inside the handler, where they can prove
|
|
447
|
+
// that run also predates the pause (#251). Release remains available:
|
|
448
|
+
// its authority, grant, and runs-settled gates still fail closed below.
|
|
449
|
+
const orchestratorCompletion =
|
|
450
|
+
channel.kind === "orchestrator" &&
|
|
451
|
+
(verb === "conductor_pr_merge" ||
|
|
452
|
+
verb === "conductor_pr_update_branch" ||
|
|
453
|
+
verb === "conductor_pr_update" ||
|
|
454
|
+
verb === "conductor_label");
|
|
455
|
+
const stopped =
|
|
456
|
+
spec.mutating && verb !== "conductor_release" && !orchestratorCompletion
|
|
457
|
+
? deps.fleetStop()
|
|
458
|
+
: undefined;
|
|
426
459
|
if (stopped !== undefined) {
|
|
427
460
|
const at = deps.pausedAt();
|
|
428
461
|
const admittedBeforePause = run !== undefined && at !== undefined && run.startedAt < at;
|
|
429
462
|
if (!admittedBeforePause) {
|
|
430
|
-
return refuse(
|
|
463
|
+
return refuse(
|
|
464
|
+
"fleet-paused",
|
|
465
|
+
`refused: ${stopped}. The pause refuses new claims and work-starting mutations. ` +
|
|
466
|
+
"Completion verbs for runs admitted before the pause (conductor_pr_merge, conductor_pr_update_branch, " +
|
|
467
|
+
"conductor_pr_update, conductor_label) and conductor_release remain available.",
|
|
468
|
+
);
|
|
431
469
|
}
|
|
432
470
|
}
|
|
433
471
|
|
|
@@ -591,7 +629,7 @@ async function prCreateVerb(
|
|
|
591
629
|
* How far back to look is the recent-history cutoff `status` already uses: a
|
|
592
630
|
* pull request older than that is not one an orchestrator is mid-flight on.
|
|
593
631
|
*/
|
|
594
|
-
const PR_LOOKUP_WINDOW_MS = 30 * 24 * 60 * 60_000;
|
|
632
|
+
export const PR_LOOKUP_WINDOW_MS = 30 * 24 * 60 * 60_000;
|
|
595
633
|
|
|
596
634
|
function runForPr(deps: VerbDeps, project: string, prUrl: string): RunRecord | undefined {
|
|
597
635
|
return deps.store
|
|
@@ -599,6 +637,31 @@ function runForPr(deps: VerbDeps, project: string, prUrl: string): RunRecord | u
|
|
|
599
637
|
.find((candidate) => candidate.prUrl === prUrl);
|
|
600
638
|
}
|
|
601
639
|
|
|
640
|
+
/** Refuse an orchestrator completion mutation that cannot prove a pre-pause run. */
|
|
641
|
+
function orchestratorPauseRefusal(
|
|
642
|
+
deps: VerbDeps,
|
|
643
|
+
channel: VerbChannel,
|
|
644
|
+
target: RunRecord | undefined,
|
|
645
|
+
refuse: Refuse,
|
|
646
|
+
): Verdict | undefined {
|
|
647
|
+
if (channel.kind !== "orchestrator") return undefined;
|
|
648
|
+
const stopped = deps.fleetStop();
|
|
649
|
+
if (stopped === undefined) return undefined;
|
|
650
|
+
const at = deps.pausedAt();
|
|
651
|
+
if (target !== undefined && at !== undefined && target.startedAt < at) return undefined;
|
|
652
|
+
const why =
|
|
653
|
+
target === undefined
|
|
654
|
+
? "this issue has no recorded run admitted before it."
|
|
655
|
+
: at === undefined
|
|
656
|
+
? "the pause timestamp cannot prove when this run started."
|
|
657
|
+
: "this run started after it.";
|
|
658
|
+
return refuse(
|
|
659
|
+
"fleet-paused",
|
|
660
|
+
`refused: ${stopped}. Under pause, this verb applies only to runs admitted before the pause; ${why}`,
|
|
661
|
+
target?.issue,
|
|
662
|
+
);
|
|
663
|
+
}
|
|
664
|
+
|
|
602
665
|
async function prUpdateBranchVerb(
|
|
603
666
|
deps: VerbDeps,
|
|
604
667
|
project: ProjectConfig,
|
|
@@ -630,6 +693,9 @@ async function prUpdateBranchVerb(
|
|
|
630
693
|
}
|
|
631
694
|
}
|
|
632
695
|
|
|
696
|
+
const paused = orchestratorPauseRefusal(deps, channel, target, refuse);
|
|
697
|
+
if (paused !== undefined) return paused;
|
|
698
|
+
|
|
633
699
|
let state: PrState | undefined;
|
|
634
700
|
try {
|
|
635
701
|
state = await deps.tracker.prState(prUrl);
|
|
@@ -683,6 +749,9 @@ async function prMergeVerb(
|
|
|
683
749
|
return refuse("pr-not-this-run", `refused: ${prUrl} is not a pull request any run in ${project.name} opened.`);
|
|
684
750
|
}
|
|
685
751
|
|
|
752
|
+
const paused = orchestratorPauseRefusal(deps, channel, target, refuse);
|
|
753
|
+
if (paused !== undefined) return paused;
|
|
754
|
+
|
|
686
755
|
// Taken before any network call, so two concurrent callers contend here
|
|
687
756
|
// rather than both spending a `gh` round trip and racing at the merge.
|
|
688
757
|
const holderId = randomUUID();
|
|
@@ -728,6 +797,47 @@ async function prMergeVerb(
|
|
|
728
797
|
);
|
|
729
798
|
}
|
|
730
799
|
|
|
800
|
+
const repoTarget = Object.values(project.routing.repos).find((r) => r.name === target.repo);
|
|
801
|
+
const chainDir = repoTarget?.migrations?.dir;
|
|
802
|
+
if (chainDir !== undefined && repoTarget !== undefined) {
|
|
803
|
+
// Checked against the base tip at merge time, not against a stale advisory:
|
|
804
|
+
// the author hit two merged PRs leaving main with two Alembic heads (#227).
|
|
805
|
+
const diff = await deps.tracker.prDiff(prUrl).catch(() => undefined);
|
|
806
|
+
if (diff === undefined) {
|
|
807
|
+
return refuse(
|
|
808
|
+
"chain-unreadable",
|
|
809
|
+
`refused: the diff of ${prUrl} could not be read, so the migration chain in ${chainDir} cannot be verified. Failing closed: a wrong merge here is a silent production schema drift (#227).`,
|
|
810
|
+
target.issue,
|
|
811
|
+
);
|
|
812
|
+
}
|
|
813
|
+
const { added, changed, deleted } = chainEntriesFromDiff(diff.files, chainDir);
|
|
814
|
+
if (added.length + changed.length + deleted.length > 0) {
|
|
815
|
+
if (diff.truncated) {
|
|
816
|
+
return refuse(
|
|
817
|
+
"chain-unreadable",
|
|
818
|
+
`refused: the diff of ${prUrl} was truncated and it touches ${chainDir} — the chain cannot be verified from a partial diff.`,
|
|
819
|
+
target.issue,
|
|
820
|
+
);
|
|
821
|
+
}
|
|
822
|
+
const base = await deps.chain.readBaseChain(project, repoTarget, chainDir);
|
|
823
|
+
if (!base.ok) {
|
|
824
|
+
return refuse(
|
|
825
|
+
"chain-unreadable",
|
|
826
|
+
`refused: could not read ${chainDir} at the tip of ${repoTarget.defaultBranch} (${base.stderr}).`,
|
|
827
|
+
target.issue,
|
|
828
|
+
);
|
|
829
|
+
}
|
|
830
|
+
const violations = chainViolations({ base: base.entries, added, changed, deleted });
|
|
831
|
+
if (violations.length > 0) {
|
|
832
|
+
return refuse(
|
|
833
|
+
"chain-conflict",
|
|
834
|
+
`refused: merging ${prUrl} would corrupt the ordered migration chain in ${chainDir} — ${violations.join("; ")}. Rebase onto the current tip, regenerate the migration, and call again.`,
|
|
835
|
+
target.issue,
|
|
836
|
+
);
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
}
|
|
840
|
+
|
|
731
841
|
const outcome = await deps.actions.mergePr(prUrl, headSha);
|
|
732
842
|
if (!outcome.ok) {
|
|
733
843
|
return refuse("action-failed", `refused: gh could not merge:\n${outcome.stderr}`, target.issue);
|
|
@@ -765,6 +875,14 @@ async function labelVerb(
|
|
|
765
875
|
);
|
|
766
876
|
}
|
|
767
877
|
|
|
878
|
+
const paused = orchestratorPauseRefusal(
|
|
879
|
+
deps,
|
|
880
|
+
channel,
|
|
881
|
+
deps.store.latestRun(project.name, ref.issue),
|
|
882
|
+
refuse,
|
|
883
|
+
);
|
|
884
|
+
if (paused !== undefined) return paused;
|
|
885
|
+
|
|
768
886
|
const label = String(args["label"]);
|
|
769
887
|
const vocabulary = labelVocabulary(project);
|
|
770
888
|
if (vocabulary.lifecycle.includes(label)) {
|
|
@@ -859,10 +977,19 @@ async function releaseVerb(
|
|
|
859
977
|
queueDepth = undefined;
|
|
860
978
|
}
|
|
861
979
|
}
|
|
980
|
+
const wantsBase = policy.release.requires.includes("base-branch-green");
|
|
981
|
+
const latestBase = wantsBase
|
|
982
|
+
? deps.store.latestBaseChecks(project.name, 0).find((run) => run.repo === repoName)
|
|
983
|
+
: undefined;
|
|
984
|
+
const redBase = latestBase?.settlementFlags?.find(
|
|
985
|
+
(flag) => flag.kind === "base-branch-red",
|
|
986
|
+
)?.detail;
|
|
862
987
|
const unmet = releaseRequirementRefusal(policy.release.requires, {
|
|
863
988
|
unsettledRuns: active.length,
|
|
864
989
|
openPrs: active.filter((r) => r.prUrl !== undefined).length,
|
|
865
990
|
queueDepth,
|
|
991
|
+
...(latestBase?.baseCheck === undefined ? {} : { baseCheck: latestBase.baseCheck }),
|
|
992
|
+
...(redBase === undefined ? {} : { redBase }),
|
|
866
993
|
});
|
|
867
994
|
if (unmet !== undefined) return refuse("release-not-granted", `refused: ${unmet}`);
|
|
868
995
|
|
|
@@ -942,10 +1069,11 @@ async function prStatusVerb(
|
|
|
942
1069
|
issue = target.issue;
|
|
943
1070
|
}
|
|
944
1071
|
|
|
945
|
-
const
|
|
1072
|
+
const askedHead = args["headSha"];
|
|
1073
|
+
const expectedHead = typeof askedHead === "string" ? askedHead : undefined;
|
|
946
1074
|
let verification: PrVerification | undefined;
|
|
947
1075
|
try {
|
|
948
|
-
verification = await deps.tracker.verifyPr(prUrl,
|
|
1076
|
+
verification = await deps.tracker.verifyPr(prUrl, expectedHead);
|
|
949
1077
|
} catch (err) {
|
|
950
1078
|
verification = undefined;
|
|
951
1079
|
deps.log(`verb pr_status could not read ${prUrl}: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -958,7 +1086,7 @@ async function prStatusVerb(
|
|
|
958
1086
|
issue,
|
|
959
1087
|
);
|
|
960
1088
|
}
|
|
961
|
-
return allow(`${prUrl} at ${headSha}: ${verification.status} — ${verification.reason}`, undefined, issue);
|
|
1089
|
+
return allow(`${prUrl} at ${verification.headSha}: ${verification.status} — ${verification.reason}`, undefined, issue);
|
|
962
1090
|
}
|
|
963
1091
|
|
|
964
1092
|
/**
|
|
@@ -1005,6 +1133,9 @@ async function prUpdateVerb(
|
|
|
1005
1133
|
if (target === undefined) {
|
|
1006
1134
|
return refuse("pr-not-this-run", `refused: ${asked} is not a pull request any run in ${project.name} opened.`);
|
|
1007
1135
|
}
|
|
1136
|
+
|
|
1137
|
+
const paused = orchestratorPauseRefusal(deps, channel, target, refuse);
|
|
1138
|
+
if (paused !== undefined) return paused;
|
|
1008
1139
|
prUrl = asked;
|
|
1009
1140
|
issue = target.issue;
|
|
1010
1141
|
}
|
package/src/worker.ts
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
import { createSession, disposeSession } from "./omp.ts";
|
|
14
|
+
import type { ReleaseBlockContext } from "./release-policy.ts";
|
|
14
15
|
import type { Caps, ReleaseShape, ResolvedGrants, RunState } from "./types.ts";
|
|
15
16
|
|
|
16
17
|
/** Structured evidence fields from the worker's final report. */
|
|
@@ -25,6 +26,11 @@ const STATE_LINE_PATTERN = /^state:\s*\S+\s*$/im;
|
|
|
25
26
|
/** `{{KEY}}` placeholders in a brief template. */
|
|
26
27
|
const PLACEHOLDER_PATTERN = /\{\{([A-Za-z0-9_]+)\}\}/g;
|
|
27
28
|
|
|
29
|
+
function scheduleWallClock(callback: () => void, delayMs: number): () => void {
|
|
30
|
+
const timer = setTimeout(callback, delayMs);
|
|
31
|
+
return () => clearTimeout(timer);
|
|
32
|
+
}
|
|
33
|
+
|
|
28
34
|
/**
|
|
29
35
|
* Which ceiling stopped a run. Only ever set alongside `state: "killed"`: the
|
|
30
36
|
* turn counter caught a loop, or the wall clock caught a session that was stuck
|
|
@@ -32,6 +38,27 @@ const PLACEHOLDER_PATTERN = /\{\{([A-Za-z0-9_]+)\}\}/g;
|
|
|
32
38
|
*/
|
|
33
39
|
export type KilledBy = "turns" | "wallclock";
|
|
34
40
|
|
|
41
|
+
export type WorkerPausePhase = "running" | "pausing" | "paused";
|
|
42
|
+
|
|
43
|
+
export interface WorkerPauseControl {
|
|
44
|
+
phase(): WorkerPausePhase;
|
|
45
|
+
/** Cooperative park; resolves when the harness is idle. Rejects when the
|
|
46
|
+
* run is settling, cap-killed, or already pausing/paused. */
|
|
47
|
+
pause(): Promise<void>;
|
|
48
|
+
/** Returns a parked session to work with a continuation prompt. Throws
|
|
49
|
+
* when the phase is not `paused`. */
|
|
50
|
+
resume(): void;
|
|
51
|
+
/** Terminally ends this run for an operator-supplied reason. The run is
|
|
52
|
+
* recorded as an administrative kill and cannot be resumed. */
|
|
53
|
+
stop(reason: string): void;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** What a resumed session is told. One literal so tests can pin it. */
|
|
57
|
+
export const RESUME_PROMPT =
|
|
58
|
+
"The operator paused this session and has now resumed it. Continue exactly where you left off: " +
|
|
59
|
+
"re-check the outcome of your last action before repeating it, then keep working your original " +
|
|
60
|
+
"brief to the same report contract.";
|
|
61
|
+
|
|
35
62
|
export interface WorkerOpts {
|
|
36
63
|
brief: string;
|
|
37
64
|
cwd: string;
|
|
@@ -55,7 +82,7 @@ export interface WorkerOpts {
|
|
|
55
82
|
*/
|
|
56
83
|
releaseGrants?: ResolvedGrants;
|
|
57
84
|
/** Durable audit sink for rejected release/deploy calls. */
|
|
58
|
-
onReleaseBlocked?: (shape: ReleaseShape) => void;
|
|
85
|
+
onReleaseBlocked?: (shape: ReleaseShape, context: ReleaseBlockContext) => void;
|
|
59
86
|
/** The child's pid, the instant it exists. See {@link VerbListener.bindPid}. */
|
|
60
87
|
onSpawn?: (pid: number) => void;
|
|
61
88
|
/** Control socket for that child, beside the run's own session directory. */
|
|
@@ -87,15 +114,21 @@ export interface WorkerOpts {
|
|
|
87
114
|
* would be a path that fails to open rather than an absence.
|
|
88
115
|
*/
|
|
89
116
|
onSessionFile?: (path: string) => void;
|
|
117
|
+
/**
|
|
118
|
+
* Installs the run's operator pause controller the moment the session
|
|
119
|
+
* exists; absent = no pause surface (tests, one-shot callers).
|
|
120
|
+
*/
|
|
121
|
+
onPauseControl?: (control: WorkerPauseControl) => void;
|
|
90
122
|
}
|
|
91
123
|
|
|
92
124
|
/**
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
* never pass it and a test can hand over a fake without a live peer dependency.
|
|
125
|
+
* Session creation plus the clock seam needed to prove wall-clock behavior
|
|
126
|
+
* without sleeping. Production callers use the defaults.
|
|
96
127
|
*/
|
|
97
128
|
export interface RunWorkerDeps {
|
|
98
129
|
createSession: typeof createSession;
|
|
130
|
+
now?: () => number;
|
|
131
|
+
schedule?: (callback: () => void, delayMs: number) => () => void;
|
|
99
132
|
}
|
|
100
133
|
|
|
101
134
|
export interface WorkerResult {
|
|
@@ -106,6 +139,8 @@ export interface WorkerResult {
|
|
|
106
139
|
spendUsd: number;
|
|
107
140
|
report: string;
|
|
108
141
|
killedBy?: KilledBy;
|
|
142
|
+
/** Present only when an operator terminally stopped this run. */
|
|
143
|
+
stoppedReason?: string;
|
|
109
144
|
/**
|
|
110
145
|
* Transcript the session actually opened, absent if it opened none. Recorded
|
|
111
146
|
* per run because it is the only readable evidence left once the worktree is
|
|
@@ -203,6 +238,8 @@ export async function runWorker(
|
|
|
203
238
|
): Promise<WorkerResult> {
|
|
204
239
|
const { workerWallClockMs } = o.caps;
|
|
205
240
|
const maxTurns = o.maxTurns ?? (() => o.caps.workerMaxTurns);
|
|
241
|
+
const now = deps.now ?? Date.now;
|
|
242
|
+
const schedule = deps.schedule ?? scheduleWallClock;
|
|
206
243
|
|
|
207
244
|
const session = await deps.createSession({
|
|
208
245
|
cwd: o.cwd,
|
|
@@ -249,31 +286,127 @@ export async function runWorker(
|
|
|
249
286
|
// merged (#217).
|
|
250
287
|
let claim: { prUrl: string; headSha: string } | undefined;
|
|
251
288
|
let killedBy: KilledBy | undefined;
|
|
252
|
-
|
|
253
|
-
|
|
289
|
+
let stoppedReason: string | undefined;
|
|
290
|
+
// Canceler for the armed wall clock; invoked on every exit path below.
|
|
291
|
+
let cancelWallClock: (() => void) | undefined;
|
|
292
|
+
let wallClockRemainingMs = workerWallClockMs;
|
|
293
|
+
let wallClockArmedAt = now();
|
|
294
|
+
let pausePhase: WorkerPausePhase = "running";
|
|
295
|
+
let resumeWaiter: PromiseWithResolvers<string> | undefined;
|
|
296
|
+
// A terminal agent_end or cap has settled the run.
|
|
297
|
+
let done = false;
|
|
254
298
|
// Resolved by the first terminal `agent_end`, and by every cap kill. Only
|
|
255
299
|
// ever awaited when the harness has already said it is not finished.
|
|
256
300
|
const { promise: settled, resolve: settle } = Promise.withResolvers<void>();
|
|
301
|
+
// Resolves whenever a pause request must interrupt the existing
|
|
302
|
+
// non-terminal agent_end wait. Re-armed after its resume prompt is consumed.
|
|
303
|
+
let pauseRequested = Promise.withResolvers<void>();
|
|
257
304
|
// Set by a non-terminal `agent_end`: the harness will resume this session.
|
|
258
305
|
let resuming = false;
|
|
259
306
|
|
|
260
307
|
const clearWallClock = () => {
|
|
261
|
-
if (
|
|
262
|
-
|
|
263
|
-
|
|
308
|
+
if (cancelWallClock === undefined) return;
|
|
309
|
+
cancelWallClock();
|
|
310
|
+
cancelWallClock = undefined;
|
|
311
|
+
};
|
|
312
|
+
|
|
313
|
+
const armWallClock = () => {
|
|
314
|
+
wallClockArmedAt = now();
|
|
315
|
+
cancelWallClock = schedule(() => kill("wallclock"), wallClockRemainingMs);
|
|
264
316
|
};
|
|
265
317
|
|
|
266
318
|
const kill = (by: KilledBy) => {
|
|
267
|
-
if (killedBy !== undefined) return;
|
|
319
|
+
if (killedBy !== undefined || stoppedReason !== undefined) return;
|
|
268
320
|
killedBy = by;
|
|
269
321
|
o.onKilled?.(by);
|
|
270
322
|
clearWallClock();
|
|
271
323
|
session.abort();
|
|
272
324
|
// An aborted session may never reach a terminal `agent_end`. The cap is the
|
|
273
325
|
// outcome now, so nothing may still be waiting for one.
|
|
326
|
+
done = true;
|
|
274
327
|
settle();
|
|
275
328
|
};
|
|
276
329
|
|
|
330
|
+
const takeResumePrompt = async (): Promise<string | undefined> => {
|
|
331
|
+
const waiter = resumeWaiter;
|
|
332
|
+
if (waiter === undefined) return undefined;
|
|
333
|
+
const prompt = await Promise.race([
|
|
334
|
+
waiter.promise,
|
|
335
|
+
settled.then(() => undefined),
|
|
336
|
+
]);
|
|
337
|
+
if (resumeWaiter === waiter) {
|
|
338
|
+
resumeWaiter = undefined;
|
|
339
|
+
pauseRequested = Promise.withResolvers<void>();
|
|
340
|
+
}
|
|
341
|
+
return killedBy !== undefined || stoppedReason !== undefined || done ? undefined : prompt;
|
|
342
|
+
};
|
|
343
|
+
|
|
344
|
+
o.onPauseControl?.({
|
|
345
|
+
phase: () => pausePhase,
|
|
346
|
+
pause: async () => {
|
|
347
|
+
if (killedBy !== undefined || done) {
|
|
348
|
+
throw new Error("the run is settling; nothing left to pause");
|
|
349
|
+
}
|
|
350
|
+
if (pausePhase !== "running") throw new Error(`the worker is already ${pausePhase}`);
|
|
351
|
+
if (resumeWaiter !== undefined) {
|
|
352
|
+
throw new Error("the worker resume is still starting");
|
|
353
|
+
}
|
|
354
|
+
pausePhase = "pausing";
|
|
355
|
+
resumeWaiter = Promise.withResolvers<string>();
|
|
356
|
+
pauseRequested.resolve();
|
|
357
|
+
// Bank the remaining wall clock before the abort: a slow drain must not
|
|
358
|
+
// be cap-killed mid-park (#238 acceptance 3).
|
|
359
|
+
wallClockRemainingMs = Math.max(
|
|
360
|
+
1_000,
|
|
361
|
+
wallClockRemainingMs - (now() - wallClockArmedAt),
|
|
362
|
+
);
|
|
363
|
+
clearWallClock();
|
|
364
|
+
try {
|
|
365
|
+
await session.park();
|
|
366
|
+
} catch (err) {
|
|
367
|
+
pausePhase = "running";
|
|
368
|
+
if (killedBy !== undefined || done) {
|
|
369
|
+
resumeWaiter = undefined;
|
|
370
|
+
throw err;
|
|
371
|
+
}
|
|
372
|
+
// The abort may already have unwound prompt() even though park itself
|
|
373
|
+
// failed. Wake that path with the same defensive continuation rather
|
|
374
|
+
// than leaving the worker hung on an orphaned waiter.
|
|
375
|
+
armWallClock();
|
|
376
|
+
resumeWaiter?.resolve(RESUME_PROMPT);
|
|
377
|
+
throw err;
|
|
378
|
+
}
|
|
379
|
+
if (killedBy !== undefined || done) {
|
|
380
|
+
pausePhase = "running";
|
|
381
|
+
resumeWaiter = undefined;
|
|
382
|
+
throw new Error("the run settled while pausing");
|
|
383
|
+
}
|
|
384
|
+
pausePhase = "paused";
|
|
385
|
+
},
|
|
386
|
+
resume: () => {
|
|
387
|
+
if (pausePhase !== "paused") {
|
|
388
|
+
throw new Error(
|
|
389
|
+
pausePhase === "pausing"
|
|
390
|
+
? "still pausing — wait until it reports paused"
|
|
391
|
+
: "the worker is not paused",
|
|
392
|
+
);
|
|
393
|
+
}
|
|
394
|
+
pausePhase = "running";
|
|
395
|
+
armWallClock();
|
|
396
|
+
resumeWaiter?.resolve(RESUME_PROMPT);
|
|
397
|
+
},
|
|
398
|
+
stop: (reason) => {
|
|
399
|
+
if (killedBy !== undefined || done) {
|
|
400
|
+
throw new Error("the run is settling; nothing left to stop");
|
|
401
|
+
}
|
|
402
|
+
stoppedReason = reason;
|
|
403
|
+
clearWallClock();
|
|
404
|
+
done = true;
|
|
405
|
+
settle();
|
|
406
|
+
session.abort();
|
|
407
|
+
},
|
|
408
|
+
});
|
|
409
|
+
|
|
277
410
|
session.on("turn_start", () => {
|
|
278
411
|
// The documented watchdog signal, and the honest one: `turn_start` fires
|
|
279
412
|
// exactly once per turn, whereas one turn can emit several assistant
|
|
@@ -326,6 +459,8 @@ export async function runWorker(
|
|
|
326
459
|
// all — is a finished run.
|
|
327
460
|
const isTerminal = field(event, "isTerminal");
|
|
328
461
|
if (shouldComplete(typeof isTerminal === "boolean" ? { isTerminal } : {})) {
|
|
462
|
+
if (pausePhase !== "running") return;
|
|
463
|
+
done = true;
|
|
329
464
|
settle();
|
|
330
465
|
return;
|
|
331
466
|
}
|
|
@@ -336,17 +471,36 @@ export async function runWorker(
|
|
|
336
471
|
// callback does not drop the handle itself: every exit runs `clearWallClock()`
|
|
337
472
|
// exactly once instead, and clearing an already-fired handle is a documented
|
|
338
473
|
// no-op — cheaper than assuming a fired timer holds nothing.
|
|
339
|
-
|
|
474
|
+
armWallClock();
|
|
340
475
|
|
|
341
476
|
try {
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
477
|
+
let next: string | undefined = o.brief;
|
|
478
|
+
while (next !== undefined) {
|
|
479
|
+
try {
|
|
480
|
+
await session.prompt(next);
|
|
481
|
+
} catch (cause) {
|
|
482
|
+
// Our own abort surfaces here on some paths: a cap kill (existing
|
|
483
|
+
// behavior) or an operator park (new). Anything else is a real failure.
|
|
484
|
+
if (killedBy === undefined && stoppedReason === undefined && resumeWaiter === undefined) throw cause;
|
|
485
|
+
}
|
|
486
|
+
next = undefined;
|
|
487
|
+
if (killedBy !== undefined) break;
|
|
488
|
+
if (resumeWaiter === undefined && resuming && !done) {
|
|
489
|
+
// A non-terminal agent_end ordinarily waits for the harness to finish
|
|
490
|
+
// later. A pause request must wake that wait so resume can prompt the
|
|
491
|
+
// same session instead of hanging behind the old settlement promise.
|
|
492
|
+
await Promise.race([settled, pauseRequested.promise]);
|
|
493
|
+
}
|
|
494
|
+
if (killedBy !== undefined) break;
|
|
495
|
+
if (resumeWaiter !== undefined) {
|
|
496
|
+
next = await takeResumePrompt();
|
|
497
|
+
if (next === undefined) break;
|
|
498
|
+
continue;
|
|
499
|
+
}
|
|
500
|
+
}
|
|
347
501
|
} catch (cause) {
|
|
348
502
|
// Our own abort surfaces here on some paths; that is a kill, not a crash.
|
|
349
|
-
if (killedBy === undefined) {
|
|
503
|
+
if (killedBy === undefined && stoppedReason === undefined) {
|
|
350
504
|
const detail = cause instanceof Error ? cause.message : String(cause);
|
|
351
505
|
return withSessionFacts({
|
|
352
506
|
state: "failed",
|
|
@@ -356,6 +510,7 @@ export async function runWorker(
|
|
|
356
510
|
});
|
|
357
511
|
}
|
|
358
512
|
} finally {
|
|
513
|
+
done = true;
|
|
359
514
|
// Runs on every exit, including the early return above: a live timer keeps
|
|
360
515
|
// the dispatcher process alive long after the run it was guarding.
|
|
361
516
|
clearWallClock();
|
|
@@ -366,6 +521,17 @@ export async function runWorker(
|
|
|
366
521
|
}
|
|
367
522
|
}
|
|
368
523
|
|
|
524
|
+
if (stoppedReason !== undefined) {
|
|
525
|
+
return withSessionFacts({
|
|
526
|
+
state: "stopped",
|
|
527
|
+
turns,
|
|
528
|
+
spendUsd,
|
|
529
|
+
report,
|
|
530
|
+
stoppedReason,
|
|
531
|
+
...(claim === undefined ? {} : { prUrl: claim.prUrl, headSha: claim.headSha }),
|
|
532
|
+
});
|
|
533
|
+
}
|
|
534
|
+
|
|
369
535
|
if (killedBy !== undefined) {
|
|
370
536
|
// The PR and head are facts the session already established, so they
|
|
371
537
|
// survive the kill. Without them `shouldContinueAfterTurnsCap` sees no
|
package/src/worktree.ts
CHANGED
|
@@ -317,6 +317,11 @@ async function ensureMirrorUnlocked(repo: RepoTarget, mirrorRoot: string): Promi
|
|
|
317
317
|
throw err;
|
|
318
318
|
}
|
|
319
319
|
await configureMirror(mirrorPath);
|
|
320
|
+
// `clone --mirror` writes upstream heads under `refs/heads/*`; the safe
|
|
321
|
+
// normal-clone refspec above uses `refs/remotes/origin/*`. Populate that
|
|
322
|
+
// namespace before returning so first-use callers see the same fresh mirror
|
|
323
|
+
// shape as every later refresh.
|
|
324
|
+
await refreshMirror(mirrorPath);
|
|
320
325
|
return mirrorPath;
|
|
321
326
|
}
|
|
322
327
|
|