@kal-elsam/kairo-runtime 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -11
- package/global-template/components/catalog.json +4 -1
- package/global-template/components/orchestrator/extensions/pi/kairo-minion.js +604 -0
- package/package.json +1 -1
- package/scripts/cockpit-smoke.mjs +4 -4
- package/src/cli.js +34 -2
- package/src/global/adapters/pi.js +1 -1
- package/src/global/global-doctor.js +2 -0
- package/src/global/ink/cockpit/primitives.js +96 -31
- package/src/global/ink/cockpit-alerts.js +36 -0
- package/src/global/ink/cockpit-changes.js +59 -35
- package/src/global/ink/cockpit-control-center.js +129 -51
- package/src/global/ink/cockpit-controller.js +58 -11
- package/src/global/ink/cockpit-focus.js +4 -2
- package/src/global/ink/cockpit-models.js +89 -47
- package/src/global/ink/cockpit-palette.js +98 -0
- package/src/global/ink/cockpit-path-label.js +19 -0
- package/src/global/ink/cockpit-recovery.js +78 -15
- package/src/global/ink/cockpit-reviews.js +14 -10
- package/src/global/ink/cockpit-runs.js +13 -4
- package/src/global/ink/cockpit-settings.js +194 -0
- package/src/global/ink/cockpit-views.js +88 -57
- package/src/global/ink/orchestrator-app.js +137 -46
- package/src/global/ink/orchestrator-state.js +24 -14
- package/src/global/ink/use-orchestrator-data.js +58 -0
- package/src/global/paths.js +3 -0
- package/src/global/runtime/alerts/alert-store.js +216 -0
- package/src/global/runtime/alerts/alert-types.js +59 -0
- package/src/global/runtime/alerts/alert-validate.js +117 -0
- package/src/global/runtime/execution-adapters/pi.js +12 -2
- package/src/global/runtime/monitor/monitor-cli.js +62 -0
- package/src/global/runtime/monitor/monitor-platform.js +95 -0
- package/src/global/runtime/monitor/monitor.js +249 -0
- package/src/global/runtime/orchestration/index.js +23 -0
- package/src/global/runtime/orchestration/orch-receipts.js +234 -0
- package/src/global/runtime/orchestration/orch-types.js +173 -0
- package/src/global/runtime/orchestration/orch-validate.js +63 -0
- package/src/global/runtime/run-cli.js +1 -0
- package/src/global/runtime/run-manager.js +59 -4
- package/src/global/runtime/run-strategy.js +71 -0
- package/src/global/runtime/run-supervisor.js +22 -2
- package/src/global/runtime/run-types.js +5 -1
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { mkdir, unlink, writeFile } from "node:fs/promises";
|
|
3
|
+
import { homedir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { execFile as execFileCb } from "node:child_process";
|
|
6
|
+
import { promisify } from "node:util";
|
|
7
|
+
|
|
8
|
+
const execFile = promisify(execFileCb);
|
|
9
|
+
export const MONITOR_LABEL = "local.kairo.monitor";
|
|
10
|
+
|
|
11
|
+
export function resolveMonitorPlatform(platform = process.platform) {
|
|
12
|
+
if (platform === "darwin") {
|
|
13
|
+
const agentsDir = join(homedir(), "Library", "LaunchAgents");
|
|
14
|
+
return {
|
|
15
|
+
id: "darwin", supportsAutostart: true, supportsNotify: true,
|
|
16
|
+
agentsDir, plistPath: join(agentsDir, `${MONITOR_LABEL}.plist`)
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
return { id: platform, supportsAutostart: false, supportsNotify: platform === "linux" };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function gui() { return `gui/${process.getuid?.() ?? 501}`; }
|
|
23
|
+
|
|
24
|
+
function plist({ nodePath, cliEntry, homeDir, intervalSec }) {
|
|
25
|
+
const e = (v) => String(v).replaceAll("&", "&").replaceAll("<", "<")
|
|
26
|
+
.replaceAll(">", ">").replaceAll("\"", """);
|
|
27
|
+
const log = join(homeDir, ".harness", "monitor");
|
|
28
|
+
const n = Math.max(60, Number(intervalSec) || 300);
|
|
29
|
+
return `<?xml version="1.0" encoding="UTF-8"?>
|
|
30
|
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
31
|
+
<plist version="1.0"><dict>
|
|
32
|
+
<key>Label</key><string>${MONITOR_LABEL}</string>
|
|
33
|
+
<key>ProgramArguments</key><array><string>${e(nodePath)}</string><string>${e(cliEntry)}</string><string>monitor</string><string>tick</string></array>
|
|
34
|
+
<key>StartInterval</key><integer>${n}</integer><key>RunAtLoad</key><true/>
|
|
35
|
+
<key>EnvironmentVariables</key><dict><key>HARNESS_HOME</key><string>${e(homeDir)}</string></dict>
|
|
36
|
+
<key>StandardOutPath</key><string>${e(join(log, "out.log"))}</string>
|
|
37
|
+
<key>StandardErrorPath</key><string>${e(join(log, "err.log"))}</string>
|
|
38
|
+
</dict></plist>`;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export async function notifyNewAlert({
|
|
42
|
+
title, body, platform = resolveMonitorPlatform(), execFileImpl = execFile
|
|
43
|
+
} = {}) {
|
|
44
|
+
const t = String(title ?? "Kairo").slice(0, 80);
|
|
45
|
+
const b = String(body ?? "").slice(0, 180).replace(/[\r\n]+/g, " ");
|
|
46
|
+
try {
|
|
47
|
+
if (platform.id === "darwin") {
|
|
48
|
+
await execFileImpl("osascript", [
|
|
49
|
+
"-e", `display notification ${JSON.stringify(b)} with title ${JSON.stringify(t)}`
|
|
50
|
+
], { shell: false, timeout: 5000 });
|
|
51
|
+
return { sent: true };
|
|
52
|
+
}
|
|
53
|
+
if (platform.id === "linux") {
|
|
54
|
+
await execFileImpl("notify-send", [t, b], { shell: false, timeout: 5000 });
|
|
55
|
+
return { sent: true };
|
|
56
|
+
}
|
|
57
|
+
} catch { /* degrade */ }
|
|
58
|
+
return { sent: false };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export async function installAutostart({
|
|
62
|
+
homeDir, platform = resolveMonitorPlatform(), nodePath, cliEntry,
|
|
63
|
+
intervalSec = 300, execFileImpl = execFile
|
|
64
|
+
} = {}) {
|
|
65
|
+
if (!platform.supportsAutostart) {
|
|
66
|
+
return {
|
|
67
|
+
supported: false, configured: false, loaded: false, installed: false,
|
|
68
|
+
detail: `unsupported on ${platform.id}`
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
await mkdir(platform.agentsDir, { recursive: true });
|
|
72
|
+
await mkdir(join(homeDir, ".harness", "monitor"), { recursive: true });
|
|
73
|
+
await writeFile(platform.plistPath, plist({ nodePath, cliEntry, homeDir, intervalSec }));
|
|
74
|
+
try {
|
|
75
|
+
await execFileImpl("launchctl", ["bootout", `${gui()}/${MONITOR_LABEL}`], { shell: false }).catch(() => {});
|
|
76
|
+
await execFileImpl("launchctl", ["bootstrap", gui(), platform.plistPath], { shell: false });
|
|
77
|
+
return { supported: true, configured: true, loaded: true, installed: true, detail: "LaunchAgent loaded" };
|
|
78
|
+
} catch (error) {
|
|
79
|
+
return {
|
|
80
|
+
supported: true, configured: true, loaded: false, installed: false,
|
|
81
|
+
detail: `plist configured; not loaded (${error?.message ?? error})`
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export async function removeAutostart({
|
|
87
|
+
platform = resolveMonitorPlatform(), execFileImpl = execFile
|
|
88
|
+
} = {}) {
|
|
89
|
+
if (!platform.supportsAutostart) {
|
|
90
|
+
return { supported: false, configured: false, loaded: false, installed: false };
|
|
91
|
+
}
|
|
92
|
+
await execFileImpl("launchctl", ["bootout", `${gui()}/${MONITOR_LABEL}`], { shell: false }).catch(() => {});
|
|
93
|
+
if (platform.plistPath && existsSync(platform.plistPath)) await unlink(platform.plistPath).catch(() => {});
|
|
94
|
+
return { supported: true, configured: false, loaded: false, installed: false };
|
|
95
|
+
}
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { mkdir, readFile } from "node:fs/promises";
|
|
3
|
+
import { harnessHomePaths } from "../../paths.js";
|
|
4
|
+
import { detectGlobalDrift, hasRepairableDrift } from "../../drift.js";
|
|
5
|
+
import { writeAtomicJson } from "../write-atomic-json.js";
|
|
6
|
+
import { listAlerts, saveAlert } from "../alerts/alert-store.js";
|
|
7
|
+
import { ALERT_SEVERITIES, ALERT_STATES } from "../alerts/alert-types.js";
|
|
8
|
+
import { listRunRecords } from "../run-store.js";
|
|
9
|
+
import { isRunAlive } from "../run-liveness.js";
|
|
10
|
+
import { RUN_STATES, isActiveRunState } from "../run-types.js";
|
|
11
|
+
import {
|
|
12
|
+
installAutostart, notifyNewAlert, removeAutostart, resolveMonitorPlatform
|
|
13
|
+
} from "./monitor-platform.js";
|
|
14
|
+
|
|
15
|
+
export {
|
|
16
|
+
installAutostart, notifyNewAlert, removeAutostart, resolveMonitorPlatform
|
|
17
|
+
} from "./monitor-platform.js";
|
|
18
|
+
|
|
19
|
+
const SOURCE = "monitor";
|
|
20
|
+
const INTERVAL = 300;
|
|
21
|
+
|
|
22
|
+
export class MonitorStateError extends Error {
|
|
23
|
+
constructor(message, { code = "corrupt_monitor_state" } = {}) {
|
|
24
|
+
super(message);
|
|
25
|
+
this.name = "MonitorStateError";
|
|
26
|
+
this.code = code;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function defaultMonitorState() {
|
|
31
|
+
return {
|
|
32
|
+
version: 1, enabled: false, intervalSec: INTERVAL,
|
|
33
|
+
lastTickAt: null, lastTick: null,
|
|
34
|
+
autostart: {
|
|
35
|
+
platform: null, installed: false, supported: false, configured: false, loaded: false
|
|
36
|
+
},
|
|
37
|
+
updatedAt: null
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function assertMonitorState(raw) {
|
|
42
|
+
if (!raw || typeof raw !== "object" || raw.version !== 1 || typeof raw.enabled !== "boolean") {
|
|
43
|
+
throw new MonitorStateError("Monitor state schema invalid.");
|
|
44
|
+
}
|
|
45
|
+
if (!Number.isFinite(raw.intervalSec) || raw.intervalSec < 1) {
|
|
46
|
+
throw new MonitorStateError("Monitor state.intervalSec invalid.");
|
|
47
|
+
}
|
|
48
|
+
const a = raw.autostart;
|
|
49
|
+
if (!a || typeof a !== "object"
|
|
50
|
+
|| typeof a.supported !== "boolean"
|
|
51
|
+
|| typeof a.configured !== "boolean"
|
|
52
|
+
|| typeof a.loaded !== "boolean"
|
|
53
|
+
|| typeof a.installed !== "boolean") {
|
|
54
|
+
throw new MonitorStateError("Monitor state.autostart invalid.");
|
|
55
|
+
}
|
|
56
|
+
if (a.loaded && !a.configured) {
|
|
57
|
+
throw new MonitorStateError("Monitor state.autostart loaded requires configured.");
|
|
58
|
+
}
|
|
59
|
+
if (a.installed !== a.loaded) {
|
|
60
|
+
throw new MonitorStateError("Monitor state.autostart installed must equal loaded.");
|
|
61
|
+
}
|
|
62
|
+
if (!a.supported && (a.configured || a.loaded || a.installed)) {
|
|
63
|
+
throw new MonitorStateError("Monitor state.autostart unsupported with lifecycle flags.");
|
|
64
|
+
}
|
|
65
|
+
if (!raw.enabled && (a.configured || a.loaded || a.installed)) {
|
|
66
|
+
throw new MonitorStateError("Monitor disabled with active autostart lifecycle.");
|
|
67
|
+
}
|
|
68
|
+
return {
|
|
69
|
+
...defaultMonitorState(),
|
|
70
|
+
...raw,
|
|
71
|
+
version: 1,
|
|
72
|
+
enabled: raw.enabled,
|
|
73
|
+
intervalSec: raw.intervalSec,
|
|
74
|
+
autostart: { ...defaultMonitorState().autostart, ...a }
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export async function readMonitorState(homeDir, { repair = false } = {}) {
|
|
79
|
+
const path = harnessHomePaths(homeDir).monitorStatePath;
|
|
80
|
+
if (!existsSync(path)) return defaultMonitorState();
|
|
81
|
+
try {
|
|
82
|
+
return assertMonitorState(JSON.parse(await readFile(path, "utf8")));
|
|
83
|
+
} catch (error) {
|
|
84
|
+
if (repair) return defaultMonitorState();
|
|
85
|
+
if (error instanceof MonitorStateError) throw error;
|
|
86
|
+
throw new MonitorStateError("Monitor state unreadable.");
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export async function writeMonitorState(homeDir, patch, { repair = false } = {}) {
|
|
91
|
+
const { monitorDir, monitorStatePath } = harnessHomePaths(homeDir);
|
|
92
|
+
await mkdir(monitorDir, { recursive: true });
|
|
93
|
+
const next = {
|
|
94
|
+
...await readMonitorState(homeDir, { repair }), ...patch,
|
|
95
|
+
updatedAt: new Date().toISOString()
|
|
96
|
+
};
|
|
97
|
+
await writeAtomicJson(monitorStatePath, next);
|
|
98
|
+
return next;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
async function raise(homeDir, input, notifyImpl) {
|
|
102
|
+
const result = await saveAlert({ ...input, source: SOURCE }, { homeDir });
|
|
103
|
+
if (!result.deduped) {
|
|
104
|
+
await notifyImpl({ title: "Kairo", body: `${result.alert.severity} · ${result.alert.title}` });
|
|
105
|
+
}
|
|
106
|
+
return result;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function autostartRecord(platform, a) {
|
|
110
|
+
return {
|
|
111
|
+
platform: platform.id, supported: a.supported,
|
|
112
|
+
configured: Boolean(a.configured), loaded: Boolean(a.loaded),
|
|
113
|
+
installed: Boolean(a.loaded), detail: a.detail ?? null
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export async function runMonitorTick(homeDir, deps = {}) {
|
|
118
|
+
const {
|
|
119
|
+
notifyImpl = notifyNewAlert, detectDriftImpl = detectGlobalDrift,
|
|
120
|
+
listRunsImpl = listRunRecords, isRunAliveImpl = isRunAlive,
|
|
121
|
+
packageRoot = null, workspaceRoot = null
|
|
122
|
+
} = deps;
|
|
123
|
+
const raised = [];
|
|
124
|
+
await mkdir(harnessHomePaths(homeDir).monitorDir, { recursive: true });
|
|
125
|
+
try {
|
|
126
|
+
const paths = harnessHomePaths(homeDir);
|
|
127
|
+
const state = existsSync(paths.statePath)
|
|
128
|
+
? JSON.parse(await readFile(paths.statePath, "utf8")) : null;
|
|
129
|
+
if (hasRepairableDrift(await detectDriftImpl({
|
|
130
|
+
homeDir, paths, state, packageRoot, workspaceRoot, context: { homeDir }
|
|
131
|
+
}))) {
|
|
132
|
+
raised.push(await raise(homeDir, {
|
|
133
|
+
kind: "monitor.drift", title: "Managed configuration drift",
|
|
134
|
+
summary: "Managed configs drifted. Run kairo sync.", severity: ALERT_SEVERITIES.HIGH
|
|
135
|
+
}, notifyImpl));
|
|
136
|
+
}
|
|
137
|
+
} catch {
|
|
138
|
+
raised.push(await raise(homeDir, {
|
|
139
|
+
kind: "monitor.drift", title: "Governance scan unavailable",
|
|
140
|
+
summary: "Monitor could not complete the drift scan.", severity: ALERT_SEVERITIES.MEDIUM
|
|
141
|
+
}, notifyImpl));
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
let runsOk = true; let dead = 0; let failed = 0;
|
|
145
|
+
try {
|
|
146
|
+
for (const run of await listRunsImpl(homeDir, { limit: 40 })) {
|
|
147
|
+
if (isActiveRunState(run.state) && !(await isRunAliveImpl(homeDir, run))) dead += 1;
|
|
148
|
+
if (run.state === RUN_STATES.FAILED) failed += 1;
|
|
149
|
+
}
|
|
150
|
+
} catch {
|
|
151
|
+
runsOk = false;
|
|
152
|
+
raised.push(await raise(homeDir, {
|
|
153
|
+
kind: "monitor.runs-unavailable", title: "Run monitoring unavailable",
|
|
154
|
+
summary: "Monitor could not inspect agent run health this tick.",
|
|
155
|
+
severity: ALERT_SEVERITIES.MEDIUM
|
|
156
|
+
}, notifyImpl));
|
|
157
|
+
}
|
|
158
|
+
if (runsOk && dead > 0) {
|
|
159
|
+
raised.push(await raise(homeDir, {
|
|
160
|
+
kind: "run.orphaned", title: "Orphaned agent run",
|
|
161
|
+
summary: `${dead} active run(s) have no live process.`, severity: ALERT_SEVERITIES.HIGH
|
|
162
|
+
}, notifyImpl));
|
|
163
|
+
}
|
|
164
|
+
if (runsOk && failed > 0) {
|
|
165
|
+
raised.push(await raise(homeDir, {
|
|
166
|
+
kind: "run.failed", title: "Agent run failed",
|
|
167
|
+
summary: `${failed} failed run(s) need attention.`, severity: ALERT_SEVERITIES.MEDIUM
|
|
168
|
+
}, notifyImpl));
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const lastTick = {
|
|
172
|
+
raised: raised.length,
|
|
173
|
+
created: raised.filter((r) => !r.deduped).length,
|
|
174
|
+
deduped: raised.filter((r) => r.deduped).length,
|
|
175
|
+
complete: runsOk, runs: runsOk ? "ok" : "unavailable"
|
|
176
|
+
};
|
|
177
|
+
return {
|
|
178
|
+
state: await writeMonitorState(homeDir, {
|
|
179
|
+
lastTickAt: new Date().toISOString(), lastTick
|
|
180
|
+
}),
|
|
181
|
+
raised
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
export async function enableMonitor(homeDir, {
|
|
186
|
+
cliEntry, nodePath = process.execPath, platform = resolveMonitorPlatform(), intervalSec = INTERVAL
|
|
187
|
+
} = {}) {
|
|
188
|
+
const autostart = await installAutostart({ homeDir, platform, nodePath, cliEntry, intervalSec });
|
|
189
|
+
try {
|
|
190
|
+
return await writeMonitorState(homeDir, {
|
|
191
|
+
enabled: true, intervalSec, autostart: autostartRecord(platform, autostart)
|
|
192
|
+
}, { repair: true });
|
|
193
|
+
} catch (error) {
|
|
194
|
+
if (autostart.loaded) await removeAutostart({ platform });
|
|
195
|
+
throw error;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
export async function disableMonitor(homeDir, { platform = resolveMonitorPlatform() } = {}) {
|
|
200
|
+
return writeMonitorState(homeDir, {
|
|
201
|
+
enabled: false, autostart: autostartRecord(platform, await removeAutostart({ platform }))
|
|
202
|
+
}, { repair: true });
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
export async function getMonitorStatus(homeDir, { platform = resolveMonitorPlatform() } = {}) {
|
|
206
|
+
try {
|
|
207
|
+
const state = await readMonitorState(homeDir);
|
|
208
|
+
let openAlerts = 0;
|
|
209
|
+
try {
|
|
210
|
+
openAlerts = (await listAlerts({ homeDir, state: ALERT_STATES.OPEN })).length;
|
|
211
|
+
} catch { openAlerts = null; }
|
|
212
|
+
return {
|
|
213
|
+
available: true, corrupt: false, enabled: state.enabled, intervalSec: state.intervalSec,
|
|
214
|
+
lastTickAt: state.lastTickAt, lastTick: state.lastTick, autostart: state.autostart,
|
|
215
|
+
platform: platform.id,
|
|
216
|
+
notify: { supported: platform.supportsNotify, backend: platform.id }, openAlerts
|
|
217
|
+
};
|
|
218
|
+
} catch (error) {
|
|
219
|
+
if (error?.code !== "corrupt_monitor_state") throw error;
|
|
220
|
+
return {
|
|
221
|
+
available: false, corrupt: true, enabled: null, intervalSec: null,
|
|
222
|
+
lastTickAt: null, lastTick: null, autostart: null, platform: platform.id,
|
|
223
|
+
notify: { supported: platform.supportsNotify, backend: platform.id },
|
|
224
|
+
openAlerts: null, error: error.message
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
export async function monitorDoctorCheck(homeDir) {
|
|
230
|
+
const s = await getMonitorStatus(homeDir);
|
|
231
|
+
if (s.corrupt || s.available === false) {
|
|
232
|
+
return {
|
|
233
|
+
name: "monitor", status: "stale", category: "monitor",
|
|
234
|
+
detail: "corrupt state — run kairo monitor disable to repair"
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
if (s.enabled && s.autostart?.supported && !s.autostart?.loaded) {
|
|
238
|
+
return {
|
|
239
|
+
name: "monitor", status: "stale", category: "monitor",
|
|
240
|
+
detail: "enabled but autostart not loaded"
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
return {
|
|
244
|
+
name: "monitor", status: "ok", category: "monitor",
|
|
245
|
+
detail: s.enabled
|
|
246
|
+
? `enabled · last ${s.lastTickAt ?? "none"} · autostart ${s.autostart?.loaded ? "loaded" : "off"}`
|
|
247
|
+
: "disabled (opt-in · kairo monitor enable)"
|
|
248
|
+
};
|
|
249
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import { harnessHomePaths } from "../../paths.js";
|
|
3
|
+
|
|
4
|
+
export {
|
|
5
|
+
RUN_STRATEGIES, DAG_NODE_STATES, DAG_TERMINAL_STATES, ORCH_LIMITS, ORCH_ERROR_CODES,
|
|
6
|
+
OrchContractError, createTaskId, isTerminalDagState, normalizeRunStrategy,
|
|
7
|
+
createOrchLineage, createBudgetUsage, createDagNode, createMinionBrief,
|
|
8
|
+
createMinionResult, digestAllowlisted
|
|
9
|
+
} from "./orch-types.js";
|
|
10
|
+
export { assertOrchReceiptSecretFree, FORBIDDEN_KEYS, walkForbiddenKeys } from "./orch-validate.js";
|
|
11
|
+
export {
|
|
12
|
+
orchPaths, buildOrchReceipt, saveOrchReceipt, loadOrchReceipt,
|
|
13
|
+
createOrchState, saveOrchState, loadOrchState, terminalizeOrchNodes,
|
|
14
|
+
updateOrchState, applyMinionDagUpdate, finalizeOrchState, reconcileOrchState
|
|
15
|
+
} from "./orch-receipts.js";
|
|
16
|
+
|
|
17
|
+
export const KAIRO_MINION_RELATIVE_ASSET =
|
|
18
|
+
"components/orchestrator/extensions/pi/kairo-minion.js";
|
|
19
|
+
|
|
20
|
+
/** Materialized extension path under ~/.harness (never Pi global auto-discover). */
|
|
21
|
+
export function resolveKairoMinionExtensionPath(homeDir) {
|
|
22
|
+
return join(harnessHomePaths(homeDir).root, KAIRO_MINION_RELATIVE_ASSET);
|
|
23
|
+
}
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { mkdir, readFile } from "node:fs/promises";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { runPaths } from "../../paths.js";
|
|
5
|
+
import { writeAtomicJson } from "../write-atomic-json.js";
|
|
6
|
+
import {
|
|
7
|
+
DAG_NODE_STATES, ORCH_ERROR_CODES, OrchContractError, RUN_STRATEGIES,
|
|
8
|
+
createBudgetUsage, createDagNode, createMinionResult, createOrchLineage,
|
|
9
|
+
digestAllowlisted, isTerminalDagState, normalizeRunStrategy
|
|
10
|
+
} from "./orch-types.js";
|
|
11
|
+
import { assertOrchReceiptSecretFree, walkForbiddenKeys } from "./orch-validate.js";
|
|
12
|
+
|
|
13
|
+
export function orchPaths(homeDir, rootRunId) {
|
|
14
|
+
if (typeof rootRunId !== "string" || !rootRunId) {
|
|
15
|
+
throw new OrchContractError("rootRunId is required for orchestration paths.", {
|
|
16
|
+
code: ORCH_ERROR_CODES.INVALID_LINEAGE
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
const { runDir } = runPaths(homeDir, rootRunId);
|
|
20
|
+
const orchDir = join(runDir, "orchestration");
|
|
21
|
+
return {
|
|
22
|
+
runDir, orchDir,
|
|
23
|
+
receiptPath: join(orchDir, "receipt.json"),
|
|
24
|
+
statePath: join(orchDir, "state.json")
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function normalizeNodes(nodes = []) {
|
|
29
|
+
return nodes.map((node) => {
|
|
30
|
+
const { objective, ...rest } = node;
|
|
31
|
+
return createDagNode({
|
|
32
|
+
...rest,
|
|
33
|
+
budget: createBudgetUsage(rest.budget ?? {}),
|
|
34
|
+
objectiveDigest: rest.objectiveDigest
|
|
35
|
+
?? (objective ? digestAllowlisted({ objective }) : null)
|
|
36
|
+
});
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function buildOrchReceipt({
|
|
41
|
+
rootRunId, strategy = RUN_STRATEGIES.ORCHESTRATED, lineage = null,
|
|
42
|
+
nodes = [], results = [], cliVersion = null, createdAt = null, recovered = false
|
|
43
|
+
} = {}) {
|
|
44
|
+
const normalizedStrategy = normalizeRunStrategy(strategy);
|
|
45
|
+
const normalizedLineage = createOrchLineage(lineage ?? { rootRunId, parentRunId: null, depth: 0 });
|
|
46
|
+
if (normalizedLineage.rootRunId !== rootRunId) {
|
|
47
|
+
throw new OrchContractError("lineage.rootRunId must match receipt rootRunId.", {
|
|
48
|
+
code: ORCH_ERROR_CODES.INVALID_LINEAGE
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
return assertOrchReceiptSecretFree({
|
|
52
|
+
version: 1, strategy: normalizedStrategy, rootRunId, lineage: normalizedLineage,
|
|
53
|
+
nodes: normalizeNodes(nodes),
|
|
54
|
+
results: results.map((entry) => createMinionResult(entry)),
|
|
55
|
+
createdAt: createdAt ?? new Date().toISOString(),
|
|
56
|
+
cliVersion,
|
|
57
|
+
recovered: Boolean(recovered)
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export async function saveOrchReceipt(receipt, { homeDir } = {}) {
|
|
62
|
+
const sanitized = assertOrchReceiptSecretFree(receipt);
|
|
63
|
+
const { orchDir, receiptPath } = orchPaths(homeDir, sanitized.rootRunId);
|
|
64
|
+
await mkdir(orchDir, { recursive: true });
|
|
65
|
+
try {
|
|
66
|
+
await writeAtomicJson(receiptPath, sanitized, { createExclusive: true });
|
|
67
|
+
} catch (error) {
|
|
68
|
+
if (error?.code === "EEXIST") {
|
|
69
|
+
throw new OrchContractError(`Orchestration receipt already exists: ${sanitized.rootRunId}`, {
|
|
70
|
+
code: ORCH_ERROR_CODES.RECEIPT_EXISTS,
|
|
71
|
+
details: { rootRunId: sanitized.rootRunId, path: receiptPath }
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
throw error;
|
|
75
|
+
}
|
|
76
|
+
return { path: receiptPath, receipt: sanitized };
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export async function loadOrchReceipt(rootRunId, { homeDir } = {}) {
|
|
80
|
+
const { receiptPath } = orchPaths(homeDir, rootRunId);
|
|
81
|
+
if (!existsSync(receiptPath)) {
|
|
82
|
+
throw new OrchContractError(`Orchestration receipt not found: ${rootRunId}`, {
|
|
83
|
+
code: ORCH_ERROR_CODES.INVALID_LINEAGE, details: { rootRunId }
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
return assertOrchReceiptSecretFree(JSON.parse(await readFile(receiptPath, "utf8")));
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function createOrchState({
|
|
90
|
+
rootRunId, strategy = RUN_STRATEGIES.ORCHESTRATED, lineage = null,
|
|
91
|
+
nodes = [], results = [], cliVersion = null, updatedAt = null
|
|
92
|
+
} = {}) {
|
|
93
|
+
const normalizedStrategy = normalizeRunStrategy(strategy);
|
|
94
|
+
if (normalizedStrategy !== RUN_STRATEGIES.ORCHESTRATED) {
|
|
95
|
+
throw new OrchContractError("Orchestration state requires strategy orchestrated.", {
|
|
96
|
+
code: ORCH_ERROR_CODES.INVALID_STRATEGY
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
const normalizedLineage = createOrchLineage(lineage ?? { rootRunId, parentRunId: null, depth: 0 });
|
|
100
|
+
if (normalizedLineage.rootRunId !== rootRunId) {
|
|
101
|
+
throw new OrchContractError("lineage.rootRunId must match state rootRunId.", {
|
|
102
|
+
code: ORCH_ERROR_CODES.INVALID_LINEAGE
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
const state = {
|
|
106
|
+
version: 1, strategy: normalizedStrategy, rootRunId, lineage: normalizedLineage,
|
|
107
|
+
nodes: normalizeNodes(nodes),
|
|
108
|
+
results: (results ?? []).map((entry) => createMinionResult(entry)),
|
|
109
|
+
cliVersion: cliVersion ?? null,
|
|
110
|
+
updatedAt: updatedAt ?? new Date().toISOString()
|
|
111
|
+
};
|
|
112
|
+
walkForbiddenKeys(state);
|
|
113
|
+
return state;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export async function saveOrchState(state, { homeDir } = {}) {
|
|
117
|
+
const sanitized = createOrchState(state);
|
|
118
|
+
const { orchDir, statePath } = orchPaths(homeDir, sanitized.rootRunId);
|
|
119
|
+
await mkdir(orchDir, { recursive: true });
|
|
120
|
+
await writeAtomicJson(statePath, sanitized);
|
|
121
|
+
return { path: statePath, state: sanitized };
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export async function loadOrchState(rootRunId, { homeDir } = {}) {
|
|
125
|
+
const { statePath } = orchPaths(homeDir, rootRunId);
|
|
126
|
+
if (!existsSync(statePath)) {
|
|
127
|
+
throw new OrchContractError(`Orchestration state not found: ${rootRunId}`, {
|
|
128
|
+
code: ORCH_ERROR_CODES.INVALID_NODE, details: { rootRunId }
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
try {
|
|
132
|
+
return createOrchState(JSON.parse(await readFile(statePath, "utf8")));
|
|
133
|
+
} catch {
|
|
134
|
+
throw new OrchContractError(`Corrupt orchestration state: ${rootRunId}`, {
|
|
135
|
+
code: ORCH_ERROR_CODES.INVALID_NODE, details: { rootRunId }
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export function terminalizeOrchNodes(nodes, { recovered = false } = {}) {
|
|
141
|
+
return (nodes ?? []).map((node) => {
|
|
142
|
+
if (isTerminalDagState(node.state)) return createDagNode(node);
|
|
143
|
+
return createDagNode({
|
|
144
|
+
...node,
|
|
145
|
+
state: recovered ? DAG_NODE_STATES.CANCELLED : DAG_NODE_STATES.COMPLETED,
|
|
146
|
+
error: recovered ? (node.error ?? { code: "interrupted" }) : node.error
|
|
147
|
+
});
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
const orchWriteLocks = new Map();
|
|
152
|
+
|
|
153
|
+
function withOrchWriteLock(rootRunId, work) {
|
|
154
|
+
const previous = orchWriteLocks.get(rootRunId) ?? Promise.resolve();
|
|
155
|
+
const next = previous.then(work);
|
|
156
|
+
orchWriteLocks.set(rootRunId, next.catch(() => {}));
|
|
157
|
+
return next;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Serialized load → mutate → save for concurrent minion DAG updates. */
|
|
161
|
+
export async function updateOrchState(rootRunId, mutator, { homeDir } = {}) {
|
|
162
|
+
return withOrchWriteLock(rootRunId, async () => {
|
|
163
|
+
const current = await loadOrchState(rootRunId, { homeDir });
|
|
164
|
+
return saveOrchState(await mutator(current), { homeDir });
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** Upsert one depth-1 node by taskId; append/replace MinionResult on completed. */
|
|
169
|
+
export async function applyMinionDagUpdate(rootRunId, {
|
|
170
|
+
homeDir, taskId, parentTaskId, attempt = 0, state,
|
|
171
|
+
objectiveDigest = null, result = null, error = null
|
|
172
|
+
} = {}) {
|
|
173
|
+
return updateOrchState(rootRunId, (current) => {
|
|
174
|
+
const rootTaskId = current.lineage?.taskId;
|
|
175
|
+
if (!rootTaskId || taskId === rootTaskId || parentTaskId !== rootTaskId) {
|
|
176
|
+
throw new OrchContractError("Minion taskId/parentTaskId must honor supervisor rootTaskId.", {
|
|
177
|
+
code: ORCH_ERROR_CODES.INVALID_NODE, details: { taskId, parentTaskId, rootTaskId }
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
const node = createDagNode({
|
|
181
|
+
taskId, parentTaskId, depth: 1, state, attempt,
|
|
182
|
+
objectiveDigest: objectiveDigest ?? null, error: error ?? null
|
|
183
|
+
});
|
|
184
|
+
const nodes = [...current.nodes];
|
|
185
|
+
const idx = nodes.findIndex((entry) => entry.taskId === taskId);
|
|
186
|
+
if (idx >= 0) {
|
|
187
|
+
nodes[idx] = createDagNode({
|
|
188
|
+
...nodes[idx], ...node,
|
|
189
|
+
objectiveDigest: node.objectiveDigest ?? nodes[idx].objectiveDigest
|
|
190
|
+
});
|
|
191
|
+
} else {
|
|
192
|
+
nodes.push(node);
|
|
193
|
+
}
|
|
194
|
+
let results = current.results;
|
|
195
|
+
if (state === DAG_NODE_STATES.COMPLETED && result) {
|
|
196
|
+
const sealed = createMinionResult(result);
|
|
197
|
+
results = [...results.filter((entry) => entry.taskId !== taskId), sealed];
|
|
198
|
+
}
|
|
199
|
+
return { ...current, nodes, results, updatedAt: new Date().toISOString() };
|
|
200
|
+
}, { homeDir });
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
export async function finalizeOrchState(rootRunId, { homeDir, recovered = false } = {}) {
|
|
204
|
+
return withOrchWriteLock(rootRunId, async () => {
|
|
205
|
+
const { receiptPath } = orchPaths(homeDir, rootRunId);
|
|
206
|
+
if (existsSync(receiptPath)) {
|
|
207
|
+
return { path: receiptPath, receipt: await loadOrchReceipt(rootRunId, { homeDir }), idempotent: true };
|
|
208
|
+
}
|
|
209
|
+
const state = await loadOrchState(rootRunId, { homeDir });
|
|
210
|
+
const nodes = terminalizeOrchNodes(state.nodes, { recovered });
|
|
211
|
+
await saveOrchState({ ...state, nodes, updatedAt: new Date().toISOString() }, { homeDir });
|
|
212
|
+
try {
|
|
213
|
+
const saved = await saveOrchReceipt(buildOrchReceipt({
|
|
214
|
+
rootRunId: state.rootRunId, strategy: state.strategy, lineage: state.lineage,
|
|
215
|
+
nodes, results: state.results, cliVersion: state.cliVersion, recovered
|
|
216
|
+
}), { homeDir });
|
|
217
|
+
return { ...saved, idempotent: false };
|
|
218
|
+
} catch (error) {
|
|
219
|
+
if (error?.code === ORCH_ERROR_CODES.RECEIPT_EXISTS) {
|
|
220
|
+
return { path: receiptPath, receipt: await loadOrchReceipt(rootRunId, { homeDir }), idempotent: true };
|
|
221
|
+
}
|
|
222
|
+
throw error;
|
|
223
|
+
}
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
export async function reconcileOrchState(rootRunId, { homeDir } = {}) {
|
|
228
|
+
const { statePath, receiptPath } = orchPaths(homeDir, rootRunId);
|
|
229
|
+
if (existsSync(receiptPath)) {
|
|
230
|
+
return { rootRunId, path: receiptPath, receipt: await loadOrchReceipt(rootRunId, { homeDir }), idempotent: true };
|
|
231
|
+
}
|
|
232
|
+
if (!existsSync(statePath)) return null;
|
|
233
|
+
return finalizeOrchState(rootRunId, { homeDir, recovered: true });
|
|
234
|
+
}
|