omp-conductor 0.3.17 → 0.3.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +344 -179
- package/package.json +1 -1
- package/skills/conductor-onboarding/SKILL.md +43 -50
- package/skills/conductor-update/SKILL.md +15 -165
- package/src/board.ts +734 -0
- package/src/brief-upgrade.ts +102 -19
- package/src/briefs/orchestrator.md +32 -16
- package/src/briefs/worker.md +7 -2
- package/src/cli.ts +136 -53
- package/src/config.ts +22 -6
- package/src/daemon.ts +673 -126
- package/src/fleet.ts +64 -6
- package/src/graph-health.ts +296 -0
- package/src/graph.ts +6 -6
- package/src/omp.ts +15 -4
- package/src/orchestrator-tick.ts +157 -10
- package/src/orchestrator.ts +8 -1
- package/src/plugin.ts +173 -45
- package/src/release-policy.ts +202 -0
- package/src/setup-host.ts +285 -0
- package/src/setup.ts +24 -20
- package/src/store.ts +373 -13
- package/src/tracker/github.ts +211 -22
- package/src/transcript.ts +45 -0
- package/src/types.ts +145 -2
- package/src/unblock.ts +26 -14
- package/src/upgrade.ts +327 -0
- package/src/worker.ts +67 -30
- package/src/worktree.ts +66 -0
- package/systemd/omp-conductor.service.example +5 -3
package/src/unblock.ts
CHANGED
|
@@ -22,10 +22,9 @@
|
|
|
22
22
|
* one event that happens outside every run, so no member fits it — folding it
|
|
23
23
|
* into `merged` or `killed` would make `status` describe a run that never
|
|
24
24
|
* reached either. Eligibility is read off the tracker's labels and never off a
|
|
25
|
-
* run row, so the store has nothing to say here. Leaving
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
* exists to stop.
|
|
25
|
+
* run row, so the store has nothing to say here. Leaving history alone keeps
|
|
26
|
+
* both budgets honest: a block consumes an operational continuation, while a
|
|
27
|
+
* real implementation failure consumes the separate failed-attempt budget.
|
|
29
28
|
*/
|
|
30
29
|
|
|
31
30
|
import { LIVE_STATES } from "./store.ts";
|
|
@@ -35,8 +34,10 @@ import type { Caps, ProjectConfig, RunRecord, Store, Tracker } from "./types.ts"
|
|
|
35
34
|
export interface UnblockOutcome {
|
|
36
35
|
/** State labels the tracker was asked to drop. */
|
|
37
36
|
cleared: string[];
|
|
38
|
-
/**
|
|
37
|
+
/** Total run segments, retained for history and sequence numbering. */
|
|
39
38
|
attemptsUsed: number;
|
|
39
|
+
failuresUsed: number;
|
|
40
|
+
continuationsUsed: number;
|
|
40
41
|
/** Newest attempt, when the store has one for this issue at all. */
|
|
41
42
|
latest?: RunRecord;
|
|
42
43
|
}
|
|
@@ -72,6 +73,8 @@ export async function unblockIssue(
|
|
|
72
73
|
return {
|
|
73
74
|
cleared,
|
|
74
75
|
attemptsUsed: store.attemptsFor(project.name, issue),
|
|
76
|
+
failuresUsed: store.failuresFor(project.name, issue),
|
|
77
|
+
continuationsUsed: store.continuationsFor(project.name, issue),
|
|
75
78
|
...(latest === undefined ? {} : { latest }),
|
|
76
79
|
};
|
|
77
80
|
}
|
|
@@ -82,17 +85,21 @@ export async function unblockIssue(
|
|
|
82
85
|
* and a spent attempt budget makes the next tick escalate rather than dispatch.
|
|
83
86
|
* Either promised blindly would send someone away believing work had resumed.
|
|
84
87
|
*/
|
|
85
|
-
export function formatUnblock(
|
|
88
|
+
export function formatUnblock(
|
|
89
|
+
issue: number,
|
|
90
|
+
o: UnblockOutcome,
|
|
91
|
+
project: ProjectConfig,
|
|
92
|
+
caps: Caps,
|
|
93
|
+
): string {
|
|
86
94
|
const latest = o.latest;
|
|
87
95
|
const lines = [`#${issue}: cleared ${o.cleared.join(", ")}`];
|
|
88
96
|
|
|
89
97
|
if (latest === undefined) {
|
|
90
|
-
lines.push("
|
|
98
|
+
lines.push(" runs none recorded — the labels were cleared anyway; eligibility is read off the tracker");
|
|
91
99
|
} else {
|
|
92
|
-
lines.push(
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
);
|
|
100
|
+
lines.push(` runs ${o.attemptsUsed}, newest ${latest.state}`);
|
|
101
|
+
lines.push(` failures ${o.failuresUsed} of ${caps.maxAttemptsPerIssue}`);
|
|
102
|
+
lines.push(` continuations ${o.continuationsUsed} of ${caps.maxContinuationsPerIssue}`);
|
|
96
103
|
}
|
|
97
104
|
|
|
98
105
|
if (latest !== undefined && LIVE_STATES.includes(latest.state)) {
|
|
@@ -100,10 +107,15 @@ export function formatUnblock(issue: number, o: UnblockOutcome, project: Project
|
|
|
100
107
|
` in flight attempt ${latest.attempt} is ${latest.state}, so the issue keeps ` +
|
|
101
108
|
`"${project.stateLabels.inProgress}" until it ends — nothing is re-claimed before then`,
|
|
102
109
|
);
|
|
103
|
-
} else if (o.
|
|
110
|
+
} else if (o.failuresUsed >= caps.maxAttemptsPerIssue) {
|
|
111
|
+
lines.push(
|
|
112
|
+
` next tick not eligible: all ${caps.maxAttemptsPerIssue} failed attempts are spent. ` +
|
|
113
|
+
"Rewrite the issue or raise maxAttemptsPerIssue.",
|
|
114
|
+
);
|
|
115
|
+
} else if (o.continuationsUsed > caps.maxContinuationsPerIssue) {
|
|
104
116
|
lines.push(
|
|
105
|
-
` next tick not eligible:
|
|
106
|
-
"
|
|
117
|
+
` next tick not eligible: the ${caps.maxContinuationsPerIssue}-continuation budget was exceeded. ` +
|
|
118
|
+
"Inspect progress or raise maxContinuationsPerIssue.",
|
|
107
119
|
);
|
|
108
120
|
} else {
|
|
109
121
|
lines.push(` next tick eligible again, as long as the issue still carries "${project.queueLabel}"`);
|
package/src/upgrade.ts
ADDED
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { inspectBriefLayout, type BriefLayout } from "./brief-upgrade.ts";
|
|
3
|
+
import { setPaused, statusSnapshot } from "./daemon.ts";
|
|
4
|
+
import { fleetLayers, type DispatchLayer, type FleetLayers } from "./fleet.ts";
|
|
5
|
+
import { livingDaemon, restartDaemon } from "./lifecycle.ts";
|
|
6
|
+
import { findProject, loadConfig } from "./config.ts";
|
|
7
|
+
import { renderBriefForProject } from "./setup.ts";
|
|
8
|
+
|
|
9
|
+
const PACKAGE = "omp-conductor";
|
|
10
|
+
const HERDR_PLUGIN = "herdr-conductor";
|
|
11
|
+
const HERDR_SOURCE = "TerrifiedBug/conductor/herdr";
|
|
12
|
+
const HERDR_UNIT = "herdr-fleet.service";
|
|
13
|
+
const DRAIN_POLL_MS = 5_000;
|
|
14
|
+
const RECOVERY_POLL_MS = 2_000;
|
|
15
|
+
const RECOVERY_ATTEMPTS = 30;
|
|
16
|
+
|
|
17
|
+
export interface UpgradeCommandResult {
|
|
18
|
+
code: number;
|
|
19
|
+
stdout: string;
|
|
20
|
+
stderr: string;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface UpgradeOptions {
|
|
24
|
+
version?: string;
|
|
25
|
+
project?: string;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export interface UpgradeResult {
|
|
29
|
+
previousVersion: string;
|
|
30
|
+
version: string;
|
|
31
|
+
gitHead: string;
|
|
32
|
+
alreadyCurrent: boolean;
|
|
33
|
+
dispatch: DispatchLayer;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface UpgradeDeps {
|
|
37
|
+
run(command: string, args: readonly string[]): Promise<UpgradeCommandResult>;
|
|
38
|
+
snapshot(project?: string): { liveWorkers: number };
|
|
39
|
+
layers(project?: string): FleetLayers;
|
|
40
|
+
brief(project?: string): { kind: BriefLayout["kind"]; current: boolean };
|
|
41
|
+
daemonIdentity(): { running: boolean; project?: string };
|
|
42
|
+
setPaused(value: boolean): void;
|
|
43
|
+
restartDaemon(): Promise<void>;
|
|
44
|
+
sleep(ms: number): Promise<void>;
|
|
45
|
+
env: NodeJS.ProcessEnv;
|
|
46
|
+
log(message: string): void;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async function runCommand(command: string, args: readonly string[]): Promise<UpgradeCommandResult> {
|
|
50
|
+
try {
|
|
51
|
+
const child = Bun.spawn([command, ...args], {
|
|
52
|
+
stdin: "ignore",
|
|
53
|
+
stdout: "pipe",
|
|
54
|
+
stderr: "pipe",
|
|
55
|
+
env: process.env,
|
|
56
|
+
});
|
|
57
|
+
const stdout = new Response(child.stdout).text();
|
|
58
|
+
const stderr = new Response(child.stderr).text();
|
|
59
|
+
const code = await child.exited;
|
|
60
|
+
return { code, stdout: await stdout, stderr: await stderr };
|
|
61
|
+
} catch (err) {
|
|
62
|
+
return { code: 127, stdout: "", stderr: err instanceof Error ? err.message : String(err) };
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const DEFAULT_DEPS: UpgradeDeps = {
|
|
67
|
+
run: runCommand,
|
|
68
|
+
snapshot: statusSnapshot,
|
|
69
|
+
layers: fleetLayers,
|
|
70
|
+
brief: (projectName) => {
|
|
71
|
+
const project = findProject(loadConfig(), projectName);
|
|
72
|
+
const layout = inspectBriefLayout(project.workspaceRoot, renderBriefForProject(project));
|
|
73
|
+
if (layout.kind !== "overlay") return { kind: layout.kind, current: false };
|
|
74
|
+
const policy = readFileSync(layout.policyPath, "utf8");
|
|
75
|
+
const live = readFileSync(layout.orchestratorPath, "utf8");
|
|
76
|
+
return { kind: layout.kind, current: live === renderBriefForProject(project, policy) };
|
|
77
|
+
},
|
|
78
|
+
daemonIdentity: () => {
|
|
79
|
+
const daemon = livingDaemon();
|
|
80
|
+
return daemon === undefined ? { running: false } : { running: true, project: daemon.project };
|
|
81
|
+
},
|
|
82
|
+
setPaused,
|
|
83
|
+
restartDaemon: async () => {
|
|
84
|
+
await restartDaemon({});
|
|
85
|
+
},
|
|
86
|
+
sleep: Bun.sleep,
|
|
87
|
+
env: process.env,
|
|
88
|
+
log: (message) => process.stdout.write(`${message}\n`),
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
function commandLine(command: string, args: readonly string[]): string {
|
|
92
|
+
return [command, ...args].join(" ");
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
async function mustRun(
|
|
96
|
+
deps: UpgradeDeps,
|
|
97
|
+
command: string,
|
|
98
|
+
args: readonly string[],
|
|
99
|
+
): Promise<UpgradeCommandResult> {
|
|
100
|
+
const ran = await deps.run(command, args);
|
|
101
|
+
if (ran.code === 0) return ran;
|
|
102
|
+
const detail = ran.stderr.trim() || ran.stdout.trim() || `exit ${ran.code}`;
|
|
103
|
+
throw new Error(`${commandLine(command, args)} failed: ${detail}`);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function parseRegistry(raw: string): { version: string; gitHead: string } {
|
|
107
|
+
let value: unknown;
|
|
108
|
+
try {
|
|
109
|
+
value = JSON.parse(raw);
|
|
110
|
+
} catch {
|
|
111
|
+
throw new Error("npm returned invalid release metadata");
|
|
112
|
+
}
|
|
113
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
114
|
+
throw new Error("npm returned invalid release metadata");
|
|
115
|
+
}
|
|
116
|
+
const version = Reflect.get(value, "version");
|
|
117
|
+
const gitHead = Reflect.get(value, "gitHead");
|
|
118
|
+
if (typeof version !== "string" || version.length === 0) {
|
|
119
|
+
throw new Error("npm release metadata has no version");
|
|
120
|
+
}
|
|
121
|
+
if (typeof gitHead !== "string" || !/^[0-9a-f]{40}$/i.test(gitHead)) {
|
|
122
|
+
throw new Error(`npm release ${version} has no full gitHead`);
|
|
123
|
+
}
|
|
124
|
+
return { version, gitHead };
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function ompPluginVersion(raw: string): string | undefined {
|
|
128
|
+
try {
|
|
129
|
+
const parsed = JSON.parse(raw) as { npm?: Array<{ name?: unknown; version?: unknown }> };
|
|
130
|
+
const plugin = parsed.npm?.find((entry) => entry.name === PACKAGE);
|
|
131
|
+
return typeof plugin?.version === "string" ? plugin.version : undefined;
|
|
132
|
+
} catch {
|
|
133
|
+
return undefined;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function herdrPluginSource(raw: string): string | undefined {
|
|
138
|
+
const line = raw.split("\n").find((candidate) => /^- herdr-conductor\b/.test(candidate.trim()));
|
|
139
|
+
return line?.match(/\[([^\]]+)\]\s*$/)?.[1];
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
async function inspectSurfaces(deps: UpgradeDeps): Promise<{
|
|
143
|
+
cliVersion: string;
|
|
144
|
+
ompVersion?: string;
|
|
145
|
+
herdrSource?: string;
|
|
146
|
+
}> {
|
|
147
|
+
const session = deps.env["HERDR_SESSION"] || "fleet";
|
|
148
|
+
const [cli, omp, herdr] = await Promise.all([
|
|
149
|
+
mustRun(deps, "omp-conductor", ["--version"]),
|
|
150
|
+
mustRun(deps, "omp", ["plugin", "list", "--json"]),
|
|
151
|
+
mustRun(deps, "herdr", ["--session", session, "plugin", "list"]),
|
|
152
|
+
]);
|
|
153
|
+
return {
|
|
154
|
+
cliVersion: cli.stdout.trim(),
|
|
155
|
+
ompVersion: ompPluginVersion(omp.stdout),
|
|
156
|
+
herdrSource: herdrPluginSource(herdr.stdout),
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function expectedHerdrSource(gitHead: string): string {
|
|
161
|
+
return `github:${HERDR_SOURCE}@${gitHead}`;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function surfacesCurrent(
|
|
165
|
+
surfaces: Awaited<ReturnType<typeof inspectSurfaces>>,
|
|
166
|
+
version: string,
|
|
167
|
+
gitHead: string,
|
|
168
|
+
): boolean {
|
|
169
|
+
return (
|
|
170
|
+
surfaces.cliVersion === version &&
|
|
171
|
+
surfaces.ompVersion === version &&
|
|
172
|
+
surfaces.herdrSource === expectedHerdrSource(gitHead)
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
async function waitForDrain(deps: UpgradeDeps, project?: string): Promise<void> {
|
|
177
|
+
let last = -1;
|
|
178
|
+
while (true) {
|
|
179
|
+
const workers = deps.snapshot(project).liveWorkers;
|
|
180
|
+
if (workers === 0) return;
|
|
181
|
+
if (workers !== last) deps.log(`waiting for ${workers} live worker(s) to finish`);
|
|
182
|
+
last = workers;
|
|
183
|
+
await deps.sleep(DRAIN_POLL_MS);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function recoveryProblem(
|
|
188
|
+
layers: FleetLayers,
|
|
189
|
+
initial: FleetLayers,
|
|
190
|
+
liveWorkers: number,
|
|
191
|
+
): string | undefined {
|
|
192
|
+
if (layers.ticks !== initial.ticks) return `ticks changed from ${initial.ticks} to ${layers.ticks}`;
|
|
193
|
+
if (layers.recovery !== initial.recovery) {
|
|
194
|
+
return `recovery changed from ${initial.recovery} to ${layers.recovery}`;
|
|
195
|
+
}
|
|
196
|
+
if (initial.herdr === "active" && layers.herdr !== "active") return `Herdr is ${layers.herdr}`;
|
|
197
|
+
if (initial.herdr === "active" && initial.recovery === "clear" && layers.pane !== "live") {
|
|
198
|
+
return `orchestrator pane is ${layers.pane}`;
|
|
199
|
+
}
|
|
200
|
+
if (initial.dispatch !== "stopped" && !layers.daemon.running) return "dispatch daemon is not running";
|
|
201
|
+
if (initial.dispatch === "stopped" && layers.daemon.running) return "stopped dispatch daemon was started";
|
|
202
|
+
if (initial.dispatch !== "stopped" && !layers.paused) return "dispatch resumed before verification";
|
|
203
|
+
if (liveWorkers !== 0) return `${liveWorkers} worker(s) appeared during verification`;
|
|
204
|
+
return undefined;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
async function waitForRecovery(
|
|
208
|
+
deps: UpgradeDeps,
|
|
209
|
+
initial: FleetLayers,
|
|
210
|
+
project?: string,
|
|
211
|
+
): Promise<void> {
|
|
212
|
+
let problem = "recovery did not settle";
|
|
213
|
+
for (let attempt = 0; attempt < RECOVERY_ATTEMPTS; attempt += 1) {
|
|
214
|
+
problem = recoveryProblem(deps.layers(project), initial, deps.snapshot(project).liveWorkers) ?? "";
|
|
215
|
+
if (problem === "") return;
|
|
216
|
+
await deps.sleep(RECOVERY_POLL_MS);
|
|
217
|
+
}
|
|
218
|
+
throw new Error(`upgrade verification failed: ${problem}`);
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
async function upgradeBrief(
|
|
222
|
+
deps: UpgradeDeps,
|
|
223
|
+
kind: BriefLayout["kind"],
|
|
224
|
+
project?: string,
|
|
225
|
+
): Promise<void> {
|
|
226
|
+
const selected = project === undefined ? [] : ["--project", project];
|
|
227
|
+
if (kind === "legacy-handwritten") {
|
|
228
|
+
await mustRun(deps, "omp-conductor", ["brief-upgrade", "--retrofit", "--apply", ...selected]);
|
|
229
|
+
}
|
|
230
|
+
await mustRun(deps, "omp-conductor", ["brief-upgrade", "--migrate", "--apply", ...selected]);
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
export async function upgradeConductor(
|
|
234
|
+
options: UpgradeOptions = {},
|
|
235
|
+
overrides: Partial<UpgradeDeps> = {},
|
|
236
|
+
): Promise<UpgradeResult> {
|
|
237
|
+
const deps: UpgradeDeps = { ...DEFAULT_DEPS, ...overrides };
|
|
238
|
+
if (deps.env["HERDR_ENV"] !== undefined) {
|
|
239
|
+
throw new Error("run omp-conductor upgrade from a shell outside the target Herdr session");
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
const requested = options.version === undefined ? `${PACKAGE}@latest` : `${PACKAGE}@${options.version}`;
|
|
243
|
+
const daemon = deps.daemonIdentity();
|
|
244
|
+
if (daemon.running && options.project !== undefined && daemon.project !== options.project) {
|
|
245
|
+
const active = daemon.project === undefined ? "an unrecorded project" : daemon.project;
|
|
246
|
+
throw new Error(
|
|
247
|
+
`active daemon serves ${active}; refusing to upgrade --project ${options.project} while it is running`,
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
const project = options.project ?? daemon.project;
|
|
251
|
+
|
|
252
|
+
const release = parseRegistry(
|
|
253
|
+
(await mustRun(deps, "npm", ["view", requested, "version", "gitHead", "--json"])).stdout,
|
|
254
|
+
);
|
|
255
|
+
const initial = deps.layers(project);
|
|
256
|
+
const surfaces = await inspectSurfaces(deps);
|
|
257
|
+
const brief = deps.brief(project);
|
|
258
|
+
const installNeeded = !surfacesCurrent(surfaces, release.version, release.gitHead);
|
|
259
|
+
if (!installNeeded && brief.current) {
|
|
260
|
+
return {
|
|
261
|
+
previousVersion: surfaces.cliVersion,
|
|
262
|
+
...release,
|
|
263
|
+
alreadyCurrent: true,
|
|
264
|
+
dispatch: initial.dispatch,
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
if (brief.kind === "missing") throw new Error("no ORCHESTRATOR.md exists for the configured project");
|
|
269
|
+
if (initial.herdr === "unknown") throw new Error("cannot determine whether herdr-fleet.service is active");
|
|
270
|
+
|
|
271
|
+
if (initial.dispatch === "running") deps.setPaused(true);
|
|
272
|
+
await waitForDrain(deps, project);
|
|
273
|
+
|
|
274
|
+
deps.log(`upgrading conductor ${surfaces.cliVersion} → ${release.version}`);
|
|
275
|
+
if (installNeeded) {
|
|
276
|
+
await mustRun(deps, "bun", ["add", "-g", `${PACKAGE}@${release.version}`]);
|
|
277
|
+
await mustRun(deps, "omp", ["plugin", "install", `${PACKAGE}@${release.version}`]);
|
|
278
|
+
if (surfaces.herdrSource?.startsWith("local:")) {
|
|
279
|
+
await mustRun(deps, "herdr", ["plugin", "unlink", HERDR_PLUGIN]);
|
|
280
|
+
}
|
|
281
|
+
await mustRun(deps, "herdr", [
|
|
282
|
+
"plugin",
|
|
283
|
+
"install",
|
|
284
|
+
HERDR_SOURCE,
|
|
285
|
+
"--ref",
|
|
286
|
+
release.gitHead,
|
|
287
|
+
"--yes",
|
|
288
|
+
]);
|
|
289
|
+
}
|
|
290
|
+
await upgradeBrief(deps, brief.kind, project);
|
|
291
|
+
|
|
292
|
+
if (initial.herdr === "active") {
|
|
293
|
+
await mustRun(deps, "systemctl", ["restart", HERDR_UNIT]);
|
|
294
|
+
}
|
|
295
|
+
if (initial.dispatch !== "stopped") {
|
|
296
|
+
await deps.restartDaemon();
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
const installed = await inspectSurfaces(deps);
|
|
300
|
+
if (!surfacesCurrent(installed, release.version, release.gitHead)) {
|
|
301
|
+
throw new Error(
|
|
302
|
+
`upgrade verification failed: cli=${installed.cliVersion}, omp=${installed.ompVersion ?? "missing"}, ` +
|
|
303
|
+
`herdr=${installed.herdrSource ?? "missing"}`,
|
|
304
|
+
);
|
|
305
|
+
}
|
|
306
|
+
if (initial.herdr === "active") {
|
|
307
|
+
const active = await mustRun(deps, "systemctl", ["is-active", HERDR_UNIT]);
|
|
308
|
+
if (active.stdout.trim() !== "active") throw new Error(`${HERDR_UNIT} is not active after restart`);
|
|
309
|
+
}
|
|
310
|
+
await waitForRecovery(deps, initial, project);
|
|
311
|
+
await deps.sleep(1_000);
|
|
312
|
+
await waitForRecovery(deps, initial, project);
|
|
313
|
+
|
|
314
|
+
if (initial.dispatch === "running") {
|
|
315
|
+
deps.setPaused(false);
|
|
316
|
+
if (deps.layers(project).dispatch !== "running") {
|
|
317
|
+
throw new Error("upgrade installed successfully, but dispatch did not resume");
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
return {
|
|
322
|
+
previousVersion: surfaces.cliVersion,
|
|
323
|
+
...release,
|
|
324
|
+
alreadyCurrent: false,
|
|
325
|
+
dispatch: initial.dispatch,
|
|
326
|
+
};
|
|
327
|
+
}
|
package/src/worker.ts
CHANGED
|
@@ -11,10 +11,14 @@
|
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
import { createSession, disposeSession } from "./omp.ts";
|
|
14
|
-
import type {
|
|
14
|
+
import type { ReleaseShape } from "./release-policy.ts";
|
|
15
|
+
import type { Caps, ReleasePolicy, RunState } from "./types.ts";
|
|
15
16
|
|
|
16
|
-
/**
|
|
17
|
-
const PR_URL_PATTERN =
|
|
17
|
+
/** Structured evidence fields from the worker's final report. */
|
|
18
|
+
const PR_URL_PATTERN = /^pr:\s*(https:\/\/github\.com\/\S+\/pull\/\d+)\s*$/im;
|
|
19
|
+
const HEAD_SHA_PATTERN = /^head:\s*([0-9a-f]{40})\s*$/im;
|
|
20
|
+
const PUSHED_GREEN_PATTERN = /^state:\s*pushed-green\s*$/im;
|
|
21
|
+
const BLOCKED_PATTERN = /^state:\s*blocked\s*$/im;
|
|
18
22
|
|
|
19
23
|
/** `{{KEY}}` placeholders in a brief template. */
|
|
20
24
|
const PLACEHOLDER_PATTERN = /\{\{([A-Za-z0-9_]+)\}\}/g;
|
|
@@ -42,7 +46,20 @@ export interface WorkerOpts {
|
|
|
42
46
|
* the harness to pick, which is what an unconfigured project wants.
|
|
43
47
|
*/
|
|
44
48
|
model?: string;
|
|
49
|
+
/** Effective release/deploy gate for this session. */
|
|
50
|
+
releasePolicy?: ReleasePolicy;
|
|
51
|
+
/** Durable audit sink for rejected release/deploy calls. */
|
|
52
|
+
onReleaseBlocked?: (shape: ReleaseShape) => void;
|
|
53
|
+
/**
|
|
54
|
+
* Reads the effective ceiling at each turn boundary. Omitted, the configured
|
|
55
|
+
* startup cap remains fixed for the run.
|
|
56
|
+
*/
|
|
57
|
+
maxTurns?: () => number;
|
|
58
|
+
/** Synchronous cap latch; called before the session abort begins. */
|
|
59
|
+
onKilled?: (by: KilledBy) => void;
|
|
45
60
|
onTurn?: (n: number) => void;
|
|
61
|
+
/** Cumulative USD spend, reported as each cost-bearing message finishes. */
|
|
62
|
+
onSpend?: (usd: number) => void;
|
|
46
63
|
/**
|
|
47
64
|
* The transcript path, handed over the moment the session opens it rather
|
|
48
65
|
* than at the end with {@link WorkerResult.sessionFile}. Both report the same
|
|
@@ -66,6 +83,7 @@ export interface RunWorkerDeps {
|
|
|
66
83
|
export interface WorkerResult {
|
|
67
84
|
state: RunState;
|
|
68
85
|
prUrl?: string;
|
|
86
|
+
headSha?: string;
|
|
69
87
|
turns: number;
|
|
70
88
|
spendUsd: number;
|
|
71
89
|
report: string;
|
|
@@ -101,23 +119,30 @@ export function renderBrief(template: string, vars: Record<string, string>): str
|
|
|
101
119
|
}
|
|
102
120
|
|
|
103
121
|
/**
|
|
104
|
-
* Read the run's outcome
|
|
122
|
+
* Read the run's structured outcome evidence from its final report.
|
|
105
123
|
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
124
|
+
* A textual `pushed-green` claim is not success by itself. The exact state line
|
|
125
|
+
* must carry both a PR URL and the head SHA observed after CI; the daemon then
|
|
126
|
+
* asks the tracker to verify those facts independently. Missing or malformed
|
|
127
|
+
* evidence fails closed.
|
|
109
128
|
*/
|
|
110
|
-
export function deriveResult(report: string): {
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
129
|
+
export function deriveResult(report: string): {
|
|
130
|
+
state: RunState;
|
|
131
|
+
prUrl?: string;
|
|
132
|
+
headSha?: string;
|
|
133
|
+
} {
|
|
134
|
+
const prUrl = PR_URL_PATTERN.exec(report)?.[1];
|
|
135
|
+
const headSha = HEAD_SHA_PATTERN.exec(report)?.[1]?.toLowerCase();
|
|
136
|
+
if (PUSHED_GREEN_PATTERN.test(report) && prUrl !== undefined && headSha !== undefined) {
|
|
137
|
+
return { state: "pushed-green", prUrl, headSha };
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const state: RunState = BLOCKED_PATTERN.test(report) ? "blocked" : "failed";
|
|
141
|
+
return {
|
|
142
|
+
state,
|
|
143
|
+
...(prUrl === undefined ? {} : { prUrl }),
|
|
144
|
+
...(headSha === undefined ? {} : { headSha }),
|
|
145
|
+
};
|
|
121
146
|
}
|
|
122
147
|
|
|
123
148
|
/**
|
|
@@ -145,8 +170,8 @@ export async function runWorker(
|
|
|
145
170
|
o: WorkerOpts,
|
|
146
171
|
deps: RunWorkerDeps = { createSession },
|
|
147
172
|
): Promise<WorkerResult> {
|
|
148
|
-
|
|
149
|
-
const
|
|
173
|
+
const { workerWallClockMs } = o.caps;
|
|
174
|
+
const maxTurns = o.maxTurns ?? (() => o.caps.workerMaxTurns);
|
|
150
175
|
|
|
151
176
|
const session = await deps.createSession({
|
|
152
177
|
cwd: o.cwd,
|
|
@@ -154,6 +179,8 @@ export async function runWorker(
|
|
|
154
179
|
...(o.model === undefined ? {} : { model: o.model }),
|
|
155
180
|
// Prevention half of #24: structured file tools cannot leave this worktree.
|
|
156
181
|
confineToCwd: true,
|
|
182
|
+
...(o.releasePolicy === undefined ? {} : { releasePolicy: o.releasePolicy }),
|
|
183
|
+
...(o.onReleaseBlocked === undefined ? {} : { onReleaseBlocked: o.onReleaseBlocked }),
|
|
157
184
|
});
|
|
158
185
|
|
|
159
186
|
// Before the first turn, not after the last: a caller that only learns the
|
|
@@ -195,6 +222,7 @@ export async function runWorker(
|
|
|
195
222
|
const kill = (by: KilledBy) => {
|
|
196
223
|
if (killedBy !== undefined) return;
|
|
197
224
|
killedBy = by;
|
|
225
|
+
o.onKilled?.(by);
|
|
198
226
|
clearWallClock();
|
|
199
227
|
session.abort();
|
|
200
228
|
// An aborted session may never reach a terminal `agent_end`. The cap is the
|
|
@@ -208,7 +236,7 @@ export async function runWorker(
|
|
|
208
236
|
// `message_end`s and would burn the cap on a run that is behaving.
|
|
209
237
|
turns += 1;
|
|
210
238
|
o.onTurn?.(turns);
|
|
211
|
-
if (turns >
|
|
239
|
+
if (turns > maxTurns()) kill("turns");
|
|
212
240
|
});
|
|
213
241
|
|
|
214
242
|
session.on("message_end", (event) => {
|
|
@@ -223,16 +251,25 @@ export async function runWorker(
|
|
|
223
251
|
// transcripts, 2026-08-07). The earlier agent_end.telemetry path never
|
|
224
252
|
// fired, so every run recorded $0 and the daily cap was theater (#46).
|
|
225
253
|
const cost = costUsdFromMessage(message);
|
|
226
|
-
if (cost !== undefined)
|
|
254
|
+
if (cost !== undefined) {
|
|
255
|
+
spendUsd += cost;
|
|
256
|
+
o.onSpend?.(spendUsd);
|
|
257
|
+
}
|
|
227
258
|
});
|
|
228
259
|
|
|
229
260
|
session.on("agent_end", (event) => {
|
|
230
261
|
// Fallback for harnesses that only attach cost on the terminal event.
|
|
231
262
|
const estimated = field(field(field(event, "telemetry"), "cost"), "estimatedUsd");
|
|
232
|
-
if (
|
|
263
|
+
if (
|
|
264
|
+
spendUsd === 0 &&
|
|
265
|
+
typeof estimated === "number" &&
|
|
266
|
+
Number.isFinite(estimated) &&
|
|
267
|
+
estimated > 0
|
|
268
|
+
) {
|
|
233
269
|
// Prefer message totals when both exist — do not double-count a run that
|
|
234
270
|
// already accumulated per-message costs.
|
|
235
|
-
|
|
271
|
+
spendUsd = estimated;
|
|
272
|
+
o.onSpend?.(spendUsd);
|
|
236
273
|
}
|
|
237
274
|
|
|
238
275
|
// Anything that is not literally `false` — including garbage or nothing at
|
|
@@ -283,12 +320,12 @@ export async function runWorker(
|
|
|
283
320
|
return withSessionFacts({ state: "killed", turns, spendUsd, report, killedBy });
|
|
284
321
|
}
|
|
285
322
|
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
);
|
|
323
|
+
return withSessionFacts({
|
|
324
|
+
...deriveResult(report),
|
|
325
|
+
turns,
|
|
326
|
+
spendUsd,
|
|
327
|
+
report,
|
|
328
|
+
});
|
|
292
329
|
}
|
|
293
330
|
|
|
294
331
|
/**
|
package/src/worktree.ts
CHANGED
|
@@ -608,3 +608,69 @@ export async function removeWorktree(
|
|
|
608
608
|
|
|
609
609
|
await git(["worktree", "prune"], mirrorPath);
|
|
610
610
|
}
|
|
611
|
+
|
|
612
|
+
export type RetainedWorktreeCleanup =
|
|
613
|
+
| { kind: "removed" }
|
|
614
|
+
| { kind: "retained"; reason: "dirty" | "unpushed" | "unknown"; detail: string };
|
|
615
|
+
|
|
616
|
+
/**
|
|
617
|
+
* Reap a terminal run's tree and local mirror branch without deleting the only
|
|
618
|
+
* copy of work. Tracker state is proved by the caller; this function proves the
|
|
619
|
+
* local half after refreshing remote refs. Any ambiguity retains everything.
|
|
620
|
+
*/
|
|
621
|
+
export async function cleanupRetainedWorktree(
|
|
622
|
+
mirrorPath: string,
|
|
623
|
+
worktreePath: string,
|
|
624
|
+
branch: string,
|
|
625
|
+
): Promise<RetainedWorktreeCleanup> {
|
|
626
|
+
if (!existsSync(mirrorPath)) {
|
|
627
|
+
return existsSync(worktreePath)
|
|
628
|
+
? { kind: "retained", reason: "unknown", detail: "mirror is missing" }
|
|
629
|
+
: { kind: "removed" };
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
try {
|
|
633
|
+
if (existsSync(worktreePath)) {
|
|
634
|
+
const dirty = await git(["status", "--porcelain"], worktreePath);
|
|
635
|
+
if (dirty !== "") {
|
|
636
|
+
return { kind: "retained", reason: "dirty", detail: "worktree has uncommitted changes" };
|
|
637
|
+
}
|
|
638
|
+
const actual = await git(["rev-parse", "--abbrev-ref", "HEAD"], worktreePath);
|
|
639
|
+
if (actual !== branch) {
|
|
640
|
+
return {
|
|
641
|
+
kind: "retained",
|
|
642
|
+
reason: "unknown",
|
|
643
|
+
detail: `worktree is on ${actual}, expected ${branch}`,
|
|
644
|
+
};
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
// A deleted remote branch can make a pushed commit look local-only until
|
|
649
|
+
// the default branch is fetched. Failure is ambiguity, never permission.
|
|
650
|
+
await git(["fetch", "--prune", "origin"], mirrorPath);
|
|
651
|
+
|
|
652
|
+
const ref = `refs/heads/${branch}`;
|
|
653
|
+
if (await gitSucceeds(["show-ref", "--verify", "--quiet", ref], mirrorPath)) {
|
|
654
|
+
const unique = await git(["rev-list", ref, "--not", "--remotes"], mirrorPath);
|
|
655
|
+
if (unique !== "") {
|
|
656
|
+
return {
|
|
657
|
+
kind: "retained",
|
|
658
|
+
reason: "unpushed",
|
|
659
|
+
detail: `${branch} has commits absent from every remote ref`,
|
|
660
|
+
};
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
await removeWorktree(mirrorPath, worktreePath);
|
|
665
|
+
if (await gitSucceeds(["show-ref", "--verify", "--quiet", ref], mirrorPath)) {
|
|
666
|
+
await git(["branch", "-D", branch], mirrorPath);
|
|
667
|
+
}
|
|
668
|
+
return { kind: "removed" };
|
|
669
|
+
} catch (err) {
|
|
670
|
+
return {
|
|
671
|
+
kind: "retained",
|
|
672
|
+
reason: "unknown",
|
|
673
|
+
detail: err instanceof Error ? err.message : String(err),
|
|
674
|
+
};
|
|
675
|
+
}
|
|
676
|
+
}
|
|
@@ -16,8 +16,10 @@
|
|
|
16
16
|
# fleet on a ≤8 GB box.
|
|
17
17
|
#
|
|
18
18
|
# Install:
|
|
19
|
-
# sudo
|
|
20
|
-
# sudo systemctl daemon-reload
|
|
19
|
+
# sudo install -m 0644 omp-conductor.service.example /etc/systemd/system/omp-conductor.service
|
|
20
|
+
# sudo systemctl daemon-reload
|
|
21
|
+
# sudo systemctl enable omp-conductor.service
|
|
22
|
+
# sudo systemctl restart omp-conductor.service
|
|
21
23
|
|
|
22
24
|
[Unit]
|
|
23
25
|
Description=omp-conductor dispatch daemon
|
|
@@ -38,7 +40,7 @@ WorkingDirectory=/home/fleet
|
|
|
38
40
|
|
|
39
41
|
# Foreground daemon so systemd tracks MainPID. `omp-conductor start` backgrounds;
|
|
40
42
|
# under a unit, call `daemon` directly.
|
|
41
|
-
ExecStart=/home/fleet/.local/bin/omp-conductor daemon --port
|
|
43
|
+
ExecStart=/home/fleet/.local/bin/omp-conductor daemon --port 8787
|
|
42
44
|
|
|
43
45
|
Restart=on-failure
|
|
44
46
|
# Handled SIGTERM exits 143; without this, Restart=on-failure brings the unit
|