@awebai/oats 0.68.1 → 0.69.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/oats.mjs +102 -22
- package/docs/capabilities.md +49 -5
- package/docs/configuration.md +68 -2
- package/docs/desktop-cli-api.md +355 -12
- package/docs/desktop-instance-start.md +88 -3
- package/docs/desktop.md +108 -10
- package/docs/execution-targets.md +514 -62
- package/docs/harness-trust.md +3 -1
- package/docs/implementation.md +28 -11
- package/docs/integrations.md +1 -1
- package/docs/oats-local.schema.json +2 -2
- package/docs/official-catalog.md +4 -3
- package/docs/packages.md +7 -7
- package/docs/release-notes/v0.69.0.md +658 -0
- package/docs/release-notes/v0.69.1.md +25 -0
- package/docs/servers.md +44 -5
- package/docs/souls-and-instances.md +11 -7
- package/docs/tui.md +77 -12
- package/docs/upgrading.md +12 -7
- package/docs/workspaces.md +1 -1
- package/lib/core.mjs +345 -89
- package/lib/file-pieces.mjs +49 -0
- package/lib/instance-events.mjs +25 -4
- package/lib/launch-prompt-fixtures/README.md +62 -0
- package/lib/launch-prompt-fixtures/resume-confirmation.json +32 -0
- package/lib/launch-prompt-fixtures.mjs +49 -0
- package/lib/launch-prompt-resume.mjs +55 -0
- package/lib/login-environment.mjs +27 -9
- package/lib/schedule.mjs +3 -3
- package/lib/servers.mjs +117 -23
- package/lib/session-end.mjs +82 -0
- package/lib/session-input.mjs +43 -9
- package/lib/session-resume-contract.mjs +32 -0
- package/lib/session-resume.mjs +310 -0
- package/lib/session-seen.mjs +163 -0
- package/lib/session-viewer.mjs +147 -44
- package/lib/soul-composition.mjs +2 -1
- package/lib/start-env.mjs +110 -0
- package/lib/tmux-server-scope.mjs +139 -14
- package/lib/workspace.mjs +38 -4
- package/package-catalog.json +10 -4
- package/package.json +1 -1
- package/packages/client/cli-adapter.mjs +9 -0
- package/packages/client/cli-locator.mjs +3 -2
- package/packages/client/deployment-data.mjs +7 -0
- package/packages/client/deployment-panel.mjs +3 -0
- package/packages/client/file-output-exec.mjs +98 -0
- package/packages/client/launch-prompt-outcome.mjs +25 -13
- package/packages/client/own-codes.mjs +2 -0
- package/packages/client/session-attach.mjs +35 -5
- package/packages/client/session-resume-cli.mjs +64 -0
- package/packages/client/session-resume-trigger.mjs +167 -0
- package/packages/client/session-resume.mjs +332 -0
- package/packages/client/tmux-status.mjs +30 -7
- package/packages/client/waiting-on-you.mjs +11 -4
- package/packages/client/workspace-step.mjs +17 -5
- package/packages/record/lib/last-conversation.mjs +123 -0
- package/packages/record/lib/session-roots.mjs +10 -1
- package/packages/tui/lib/client.mjs +1 -1
- package/packages/tui/lib/main.mjs +5 -3
- package/packages/tui/lib/model.mjs +45 -12
- package/packages/tui/lib/store.mjs +8 -7
- package/packages/tui/lib/term/text.mjs +12 -9
- package/packages/tui/lib/views/frame.mjs +72 -7
- package/packages/tui/lib/views/help.mjs +8 -3
- package/packages/tui/lib/views/instances.mjs +3 -2
- package/skills/oats-getting-started/SKILL.md +117 -4
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
/** `oats session resume` (feature session-resume, #1559): the machine sweep. After a shutdown, a
|
|
2
|
+
* reboot or a crash, every launched instance home whose session was lost (status `sessionEnd: lost`)
|
|
3
|
+
* on every local deployment is started again with `session start --resume`, in its own harness
|
|
4
|
+
* conversation; running homes are left alone, stopped and exited ones stay stopped.
|
|
5
|
+
*
|
|
6
|
+
* - The machine's deployments come from the registry, read only (readDeploymentsDocument, never the
|
|
7
|
+
* first-use import that listDeployments may write); `deployment` narrows the sweep to one
|
|
8
|
+
* directory, registered or not. Nothing opts a deployment out: stopping an instance with OATS
|
|
9
|
+
* is what keeps it down.
|
|
10
|
+
* - Only a loss the kernel can vouch for is resumed (lib/session-seen.mjs resumeEligible, from what
|
|
11
|
+
* local listings recorded of the home): seen running for its current start, not judged by an
|
|
12
|
+
* earlier sweep, first seen lost less than 24 hours ago, and, within one boot, lost less than 24
|
|
13
|
+
* hours after it was last seen running. Any other is `skipped`, `not-seen-recently`. A sweep that
|
|
14
|
+
* is not a plan judges each lost home it answered (`judgedBy`), so it is answered once.
|
|
15
|
+
* - Each home starts as a child process of this kernel (`session start --resume --home <home>
|
|
16
|
+
* --json`, its cwd the home's deployment), so each start resolves its own deployment's scope; at
|
|
17
|
+
* most RESUME_CONCURRENCY at once. A per-home failure is a row, never the sweep's failure.
|
|
18
|
+
* - One sweep per machine: a mkdir lock under OATS_HOME_DIR (default ~/.oats) `session-resume/`,
|
|
19
|
+
* holding its pid and boot id; a lock whose pid is gone, or taken in another boot, is stale and
|
|
20
|
+
* taken over. A second sweep answers E_RESUME_BUSY.
|
|
21
|
+
* - No retry storm. A loss the sweep answered is judged (above) and not started again, whatever the
|
|
22
|
+
* boot; one note per boot (`note.json`) backs that: a refused resume is noted, and a later sweep
|
|
23
|
+
* in the same boot reports it again (`refused-this-boot`) without starting it, also when its
|
|
24
|
+
* judgement was lost (a home whose start id changed since is not matched); and a home is started by sweeps at most RESUME_LIMIT
|
|
25
|
+
* times in a boot (counted by home: each start makes a new start id), then reported
|
|
26
|
+
* `keeps-getting-lost` without a start. A start counts when it launched something (`resumed`,
|
|
27
|
+
* `fresh`, `blocked`), noted as soon as that start has answered, so a sweep that is killed while
|
|
28
|
+
* other starts run has counted the ones that answered. `session start --resume` run by hand is
|
|
29
|
+
* neither bounded nor counted.
|
|
30
|
+
* Without a boot id (neither Linux nor macOS) nothing is noted or counted: each sweep tries again.
|
|
31
|
+
* The lock and the note are kernel-private, not a contract. `plan` predicts from the note, starting
|
|
32
|
+
* nothing, taking no lock and writing no note. */
|
|
33
|
+
import { execFile } from "node:child_process";
|
|
34
|
+
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
35
|
+
import { join, resolve } from "node:path";
|
|
36
|
+
import { oatsError } from "./errors.mjs";
|
|
37
|
+
import { deploymentState, readDeploymentsDocument } from "./deployments.mjs";
|
|
38
|
+
import { listInstances, missingLaunchEnvRefs, resumeConversation } from "./core.mjs";
|
|
39
|
+
import { checkStartEnv, fromEnvNames, launchEnvChangedMessage, launchEnvMissingMessage, launchEnvUnverifiedWarning } from "./start-env.mjs";
|
|
40
|
+
import { FRESH_REASON, NOT_SWEPT, OUTCOME, RESUME_WAITING, ROW_REASON, SESSION_END } from "./session-resume-contract.mjs";
|
|
41
|
+
import { bootId, homeKey, judgeLosses, readSeen, resumeEligible, resumeStateDir } from "./session-seen.mjs";
|
|
42
|
+
import { isAgentScopeWarning } from "./tmux-server-scope.mjs";
|
|
43
|
+
|
|
44
|
+
export { bootId, resumeStateDir };
|
|
45
|
+
|
|
46
|
+
export const RESUME_SCHEMA = 1;
|
|
47
|
+
/** At most this many starts run at once. */
|
|
48
|
+
export const RESUME_CONCURRENCY = 4;
|
|
49
|
+
/** At most this many sweep starts of one home in one boot. */
|
|
50
|
+
export const RESUME_LIMIT = 3;
|
|
51
|
+
/** An acquirer's lock without an owner record yet is its own for this long (the gap between mkdir and
|
|
52
|
+
* the owner's write); older, it is a crashed acquirer's. */
|
|
53
|
+
const OWNERLESS_GRACE_MS = 60_000;
|
|
54
|
+
|
|
55
|
+
const lockDir = () => join(resumeStateDir(), "lock");
|
|
56
|
+
const notePath = () => join(resumeStateDir(), "note.json");
|
|
57
|
+
|
|
58
|
+
function pidAlive(pid) {
|
|
59
|
+
if (!Number.isSafeInteger(pid) || pid < 1) return false;
|
|
60
|
+
try { process.kill(pid, 0); return true; } catch (e) { return e.code === "EPERM"; }
|
|
61
|
+
}
|
|
62
|
+
const readJson = (path) => { try { return JSON.parse(readFileSync(path, "utf8")); } catch { return undefined; } };
|
|
63
|
+
|
|
64
|
+
/** Take the machine's resume lock → its release function. Busy: E_RESUME_BUSY. A lock whose owner's
|
|
65
|
+
* pid is gone, whose owner was recorded in another boot, or that has had no owner for
|
|
66
|
+
* OWNERLESS_GRACE_MS is stale: it is taken over under a second mkdir (`lock.takeover`), so two
|
|
67
|
+
* sweeps that both find it stale cannot both take it. */
|
|
68
|
+
export function acquireResumeLock({ boot = bootId(), now = () => Date.now() } = {}) {
|
|
69
|
+
const dir = lockDir(), takeover = `${dir}.takeover`;
|
|
70
|
+
mkdirSync(resumeStateDir(), { recursive: true, mode: 0o700 });
|
|
71
|
+
const busy = (why) => oatsError("E_RESUME_BUSY", `another session resume on this machine holds ${dir} (${why}); nothing was started`);
|
|
72
|
+
const stale = () => {
|
|
73
|
+
const owner = readJson(join(dir, "owner.json"));
|
|
74
|
+
if (!owner) { try { return now() - statSync(dir).mtimeMs > OWNERLESS_GRACE_MS ? "no owner" : null; } catch { return "gone"; } }
|
|
75
|
+
if (boot && owner.bootId && owner.bootId !== boot) return `taken in another boot by pid ${owner.pid}`;
|
|
76
|
+
return pidAlive(owner.pid) ? null : `its owner pid ${owner.pid} is gone`;
|
|
77
|
+
};
|
|
78
|
+
const take = () => { mkdirSync(dir); writeFileSync(join(dir, "owner.json"), JSON.stringify({ pid: process.pid, bootId: boot, at: new Date(now()).toISOString() }) + "\n", { mode: 0o600 }); };
|
|
79
|
+
try { take(); }
|
|
80
|
+
catch (e) {
|
|
81
|
+
if (e.code !== "EEXIST") throw e;
|
|
82
|
+
if (!stale()) { const owner = readJson(join(dir, "owner.json")); throw busy(owner ? `pid ${owner.pid} holds it` : "its owner is not recorded yet"); }
|
|
83
|
+
try { mkdirSync(takeover); }
|
|
84
|
+
catch (t) {
|
|
85
|
+
if (t.code !== "EEXIST") throw t;
|
|
86
|
+
// A takeover that crashed leaves its directory: after the grace it is gone for good.
|
|
87
|
+
let old = false; try { old = now() - statSync(takeover).mtimeMs > OWNERLESS_GRACE_MS; } catch { /* removed meanwhile */ }
|
|
88
|
+
if (!old) throw busy("another sweep is taking over a stale lock");
|
|
89
|
+
rmSync(takeover, { recursive: true, force: true });
|
|
90
|
+
try { mkdirSync(takeover); } catch { throw busy("another sweep is taking over a stale lock"); }
|
|
91
|
+
}
|
|
92
|
+
try {
|
|
93
|
+
// Read again under the takeover: another sweep may have taken it meanwhile.
|
|
94
|
+
if (existsSync(dir) && !stale()) throw busy("another sweep took it over");
|
|
95
|
+
const aside = `${dir}.stale-${process.pid}-${now()}`;
|
|
96
|
+
try { renameSync(dir, aside); rmSync(aside, { recursive: true, force: true }); } catch (r) { if (r.code !== "ENOENT") throw r; }
|
|
97
|
+
take();
|
|
98
|
+
} finally { rmSync(takeover, { recursive: true, force: true }); }
|
|
99
|
+
}
|
|
100
|
+
return () => { try { if (readJson(join(dir, "owner.json"))?.pid === process.pid) rmSync(dir, { recursive: true, force: true }); } catch { /* best effort */ } };
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** This boot's note: `{ bootId, refused: [{ home, startId, code, message, at }], resumes: { <home>: n } }`,
|
|
104
|
+
* each home by its real path (lib/session-seen.mjs homeKey);
|
|
105
|
+
* empty for another boot's note, an unreadable one, or no boot id. */
|
|
106
|
+
export function readBootNote(boot = bootId()) {
|
|
107
|
+
const note = boot ? readJson(notePath()) : undefined;
|
|
108
|
+
if (!note || note.bootId !== boot) return { bootId: boot, refused: [], resumes: {} };
|
|
109
|
+
const resumes = {};
|
|
110
|
+
if (note.resumes && typeof note.resumes === "object" && !Array.isArray(note.resumes)) {
|
|
111
|
+
for (const [home, n] of Object.entries(note.resumes)) if (Number.isSafeInteger(n) && n > 0) resumes[home] = n;
|
|
112
|
+
}
|
|
113
|
+
return { bootId: boot, refused: Array.isArray(note.refused) ? note.refused.filter((r) => r && typeof r.home === "string") : [], resumes };
|
|
114
|
+
}
|
|
115
|
+
/** Note a sweep's `refusals` (each replacing its home's earlier entry) and the `launched` homes (each
|
|
116
|
+
* counted once more) for this boot; another boot's note is replaced whole. Nothing is written
|
|
117
|
+
* without a boot id. */
|
|
118
|
+
export function noteSweep({ refusals = [], launched = [] }, boot = bootId()) {
|
|
119
|
+
if (!boot || (!refusals.length && !launched.length)) return;
|
|
120
|
+
const note = readBootNote(boot);
|
|
121
|
+
const homes = new Set(refusals.map((r) => r.home));
|
|
122
|
+
const resumes = { ...note.resumes };
|
|
123
|
+
for (const home of launched) resumes[home] = (resumes[home] ?? 0) + 1;
|
|
124
|
+
const path = notePath();
|
|
125
|
+
mkdirSync(resumeStateDir(), { recursive: true, mode: 0o700 });
|
|
126
|
+
const tmp = `${path}.${process.pid}.tmp`;
|
|
127
|
+
try {
|
|
128
|
+
writeFileSync(tmp, JSON.stringify({ bootId: boot, refused: [...note.refused.filter((r) => !homes.has(r.home)), ...refusals], resumes }, null, 2) + "\n", { mode: 0o600 });
|
|
129
|
+
renameSync(tmp, path);
|
|
130
|
+
} catch (e) { rmSync(tmp, { force: true }); throw e; }
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** The deployments a sweep considers: the one `deployment` names (E_BAD_ARGS when it is not a
|
|
134
|
+
* deployment directory), else the registry's live rows, sorted: a row `oats deployments remove` took
|
|
135
|
+
* out (a tombstone, `removed: true`) is never swept. Registry errors (E_DEPLOYMENTS_*) throw. */
|
|
136
|
+
function sweptDeployments(deployment) {
|
|
137
|
+
if (deployment !== undefined) {
|
|
138
|
+
const dir = resolve(deployment);
|
|
139
|
+
if (deploymentState(dir) !== "ok") throw oatsError("E_BAD_ARGS", `--deployment ${dir} is not a deployment directory (no oats-local.yaml there); nothing was started`);
|
|
140
|
+
return [dir];
|
|
141
|
+
}
|
|
142
|
+
return readDeploymentsDocument().deployments.filter((row) => row && row.removed !== true).map((row) => row.path).sort();
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** One deployment: its entry for `deployments[]` and, when it is swept, its homes' rows to act on. */
|
|
146
|
+
function readDeployment(dir) {
|
|
147
|
+
const state = deploymentState(dir);
|
|
148
|
+
if (state !== "ok") return { entry: { deployment: dir, state, swept: false, reason: NOT_SWEPT.NOT_OK } };
|
|
149
|
+
let rows;
|
|
150
|
+
try {
|
|
151
|
+
// The status rows `oats status --dir <deployment> --json` lists (listInstances), from the
|
|
152
|
+
// deployment's own instance root.
|
|
153
|
+
const root = join(dir, "agents");
|
|
154
|
+
rows = existsSync(root) ? listInstances(root).flatMap((a) => (a.instances || []).map((i) => ({ deployment: dir, ...i }))) : [];
|
|
155
|
+
} catch (e) {
|
|
156
|
+
return { entry: { deployment: dir, state, swept: false, reason: NOT_SWEPT.UNREADABLE, code: typeof e?.code === "string" && e.code.startsWith("E_") ? e.code : "E_RESUME_UNREADABLE", message: String(e?.message ?? e) } };
|
|
157
|
+
}
|
|
158
|
+
return { entry: { deployment: dir, state, swept: true }, rows };
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** The reasons of the rows a sweep answers without judging the loss, so that a later boot's sweep
|
|
162
|
+
* may resume it: none. A loop guard's refusal (`keeps-getting-lost`) judges the loss like any
|
|
163
|
+
* refusal, and nothing brings it back automatically, a reboot included (decided for #1567); naming
|
|
164
|
+
* it here would let the next boot retry it. */
|
|
165
|
+
const NOT_JUDGED = new Set([]);
|
|
166
|
+
/** The outcomes of a start that launched something: the ones RESUME_LIMIT counts. */
|
|
167
|
+
const LAUNCHED = new Set([OUTCOME.RESUMED, OUTCOME.FRESH, OUTCOME.BLOCKED]);
|
|
168
|
+
const base = (row) => ({ deployment: row.deployment, instance: row.instance, home: row.home });
|
|
169
|
+
|
|
170
|
+
/** A start's answered warnings, as strings. */
|
|
171
|
+
const startWarnings = (answer) => answer?.ok === true && Array.isArray(answer.result?.warnings) ? answer.result.warnings.filter((w) => typeof w === "string") : [];
|
|
172
|
+
/** The warnings of a start's answer that are said of the host, not of its home: today only the
|
|
173
|
+
* agent-scope warning (isAgentScopeWarning), which every start on a host that cannot scope repeats.
|
|
174
|
+
* The sweep says each once, in its own `warnings`; the rows leave them out (rowFromStart). */
|
|
175
|
+
export function hostWarnings(answer) {
|
|
176
|
+
return startWarnings(answer).filter(isAgentScopeWarning);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** What a start's JSON answer means for the sweep's row: its warnings without the host's (hostWarnings). */
|
|
180
|
+
export function rowFromStart(row, answer) {
|
|
181
|
+
if (answer?.ok === true) {
|
|
182
|
+
const r = answer.result ?? {}, resume = r.resume ?? {};
|
|
183
|
+
const warnings = startWarnings(answer).filter((w) => !isAgentScopeWarning(w));
|
|
184
|
+
const extra = warnings.length ? { warnings } : {};
|
|
185
|
+
if (resume.resumed === true) {
|
|
186
|
+
if (resume.waiting === RESUME_WAITING.RESUME_CONFIRMATION) return { ...base(row), outcome: OUTCOME.BLOCKED, reason: ROW_REASON.RESUME_CONFIRMATION, ...(resume.sessionId ? { sessionId: resume.sessionId } : {}), ...extra };
|
|
187
|
+
return { ...base(row), outcome: OUTCOME.RESUMED, ...(resume.sessionId ? { sessionId: resume.sessionId } : {}), ...extra };
|
|
188
|
+
}
|
|
189
|
+
return { ...base(row), outcome: OUTCOME.FRESH, reason: resume.reason ?? FRESH_REASON.NO_CONVERSATION, ...extra };
|
|
190
|
+
}
|
|
191
|
+
const code = typeof answer?.error?.code === "string" ? answer.error.code : "E_SESSION_START_FAILED";
|
|
192
|
+
const message = String(answer?.error?.message ?? "the start gave no answer");
|
|
193
|
+
if (code === "E_SPAWN_INCOMPLETE") return { ...base(row), outcome: OUTCOME.BLOCKED, reason: ROW_REASON.LAUNCH_PROMPT, code, ...(answer.error.details?.launchPrompts ? { launchPrompts: answer.error.details.launchPrompts } : {}) };
|
|
194
|
+
if (code === "E_SESSION_START_BUSY") return { ...base(row), outcome: OUTCOME.SKIPPED, reason: ROW_REASON.START_BUSY, code };
|
|
195
|
+
if (code === "E_RESUME_NOT_LOST" || code === "E_SESSION_RUNNING") return { ...base(row), outcome: OUTCOME.SKIPPED, reason: ROW_REASON.NOT_LOST, code };
|
|
196
|
+
if (code === "E_SESSION_UNKNOWN") return { ...base(row), outcome: OUTCOME.SKIPPED, reason: ROW_REASON.UNKNOWN, code, message };
|
|
197
|
+
return { ...base(row), outcome: OUTCOME.REFUSED, reason: ROW_REASON.START_REFUSED, code, message };
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** The environment of a start the sweep runs: this one's, without an instance identity, so a sweep
|
|
201
|
+
* run from inside an instance starts each home as the home itself, in its deployment. */
|
|
202
|
+
function startEnvironment(env) {
|
|
203
|
+
const out = { ...env };
|
|
204
|
+
for (const k of ["OATS_INSTANCE", "OATS_INSTANCE_HOME", "OATS_HOME", "PI_AGENT_INSTANCE", "PI_AGENT_HOME"]) delete out[k];
|
|
205
|
+
return out;
|
|
206
|
+
}
|
|
207
|
+
function runStart({ execPath, bin, env }, row) {
|
|
208
|
+
return new Promise((done) => {
|
|
209
|
+
execFile(execPath, [bin, "session", "start", "--resume", "--home", row.home, "--json"], { cwd: row.deployment, env: startEnvironment(env), encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }, (error, stdout) => {
|
|
210
|
+
let answer;
|
|
211
|
+
try { answer = JSON.parse(String(stdout).trim().split("\n").pop()); } catch { answer = undefined; }
|
|
212
|
+
if (!answer || typeof answer !== "object") answer = { ok: false, error: { code: "E_SESSION_START_FAILED", message: `the start exited${error?.code !== undefined ? ` with status ${error.code}` : ""}${error?.signal ? ` on ${error.signal}` : ""} without an answer` } };
|
|
213
|
+
done(answer);
|
|
214
|
+
});
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
/** `fn` over `items`, at most `limit` at once; once `stop()` holds no item is taken any more, and the
|
|
218
|
+
* answer waits for every one already taken. */
|
|
219
|
+
async function inBatches(items, limit, fn, stop = () => false) {
|
|
220
|
+
const out = new Array(items.length);
|
|
221
|
+
let next = 0;
|
|
222
|
+
const worker = async () => { while (!stop() && next < items.length) { const i = next++; out[i] = await fn(items[i]); } };
|
|
223
|
+
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
|
224
|
+
return out;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/** The sweep → `{ schemaVersion, plan, deployments, rows, warnings? }` (docs/desktop-cli-api.md):
|
|
228
|
+
* `warnings`, only when there is one, holds each distinct host-wide warning of its starts once. `kernel`:
|
|
229
|
+
* `{ execPath, bin }`, how a start is run; `env` the environment each start gets. `start` (tests):
|
|
230
|
+
* row → Promise of the start's JSON answer, in place of the child process. */
|
|
231
|
+
export async function resumeSessions({ deployment, plan = false, kernel, env = process.env, boot = bootId(), start = (row) => runStart({ ...kernel, env }, row) } = {}) {
|
|
232
|
+
const release = plan ? null : acquireResumeLock({ boot });
|
|
233
|
+
try {
|
|
234
|
+
const deployments = [], candidates = [], rows = [], seen = new Map();
|
|
235
|
+
for (const dir of sweptDeployments(deployment)) {
|
|
236
|
+
const { entry, rows: found } = readDeployment(dir);
|
|
237
|
+
deployments.push(entry);
|
|
238
|
+
// What the kernel has seen of each home (lib/session-seen.mjs), read after the deployment's own
|
|
239
|
+
// listing recorded what it saw now.
|
|
240
|
+
const { homes } = found ? readSeen(join(dir, "agents")) : { homes: {} };
|
|
241
|
+
for (const [home, record] of Object.entries(homes)) seen.set(home, record);
|
|
242
|
+
for (const row of found ?? []) {
|
|
243
|
+
if (row.running === true) rows.push({ ...base(row), outcome: OUTCOME.ALREADY_RUNNING });
|
|
244
|
+
else if (row.running === null) rows.push({ ...base(row), outcome: OUTCOME.SKIPPED, reason: ROW_REASON.UNKNOWN });
|
|
245
|
+
else if (row.sessionEnd === SESSION_END.LOST) candidates.push(row);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
const note = readBootNote(boot), now = Date.now();
|
|
249
|
+
const toStart = [];
|
|
250
|
+
// Precedence, the first that applies: not seen recently, refused this boot, keeps getting lost,
|
|
251
|
+
// started. The record and the note name a home by its real path (homeKey), whatever spelling the
|
|
252
|
+
// sweep was given; a row shows the path it listed.
|
|
253
|
+
for (const row of candidates) {
|
|
254
|
+
const startId = row.startId ?? null, key = homeKey(row.home);
|
|
255
|
+
const prior = note.refused.find((r) => r.home === key && (r.startId ?? null) === startId);
|
|
256
|
+
const fit = resumeEligible(seen.get(key), row, { now });
|
|
257
|
+
if (!fit.eligible) rows.push({ ...base(row), outcome: OUTCOME.SKIPPED, reason: ROW_REASON.NOT_SEEN_RECENTLY, message: `${row.instance} is not resumed automatically: ${fit.why}; start it by hand (oats session start --resume --home ${row.home}) if it should run` });
|
|
258
|
+
else if (prior) rows.push({ ...base(row), outcome: OUTCOME.REFUSED, reason: ROW_REASON.REFUSED_THIS_BOOT, code: prior.code, message: prior.message });
|
|
259
|
+
else if ((note.resumes[key] ?? 0) >= RESUME_LIMIT) rows.push({ ...base(row), outcome: OUTCOME.REFUSED, reason: ROW_REASON.KEEPS_GETTING_LOST, message: `${row.instance} was resumed ${note.resumes[key]} times in this boot and was lost again each time; no sweep starts this loss again: start it by hand (oats session start --resume --home ${row.home})` });
|
|
260
|
+
else if (plan) rows.push(predicted(row, env));
|
|
261
|
+
else toStart.push(row);
|
|
262
|
+
}
|
|
263
|
+
// Each start's answer is noted as it comes, under the lock (noteSweep is synchronous: two answers
|
|
264
|
+
// never interleave in it), before the sweep waits on the others. A note that cannot be written
|
|
265
|
+
// takes no more home; the lock is held until every start already running has answered, and the
|
|
266
|
+
// failure is the sweep's after that.
|
|
267
|
+
let noteFailure = null;
|
|
268
|
+
const hostWide = new Set();
|
|
269
|
+
const started = (await inBatches(toStart, RESUME_CONCURRENCY, async (row) => {
|
|
270
|
+
const answer = await start(row);
|
|
271
|
+
for (const w of hostWarnings(answer)) hostWide.add(w);
|
|
272
|
+
const answered = rowFromStart(row, answer), key = homeKey(row.home);
|
|
273
|
+
try {
|
|
274
|
+
noteSweep({
|
|
275
|
+
refusals: answered.outcome === OUTCOME.REFUSED ? [{ home: key, startId: row.startId ?? null, code: answered.code, message: answered.message, at: new Date().toISOString() }] : [],
|
|
276
|
+
launched: LAUNCHED.has(answered.outcome) ? [key] : [],
|
|
277
|
+
}, boot);
|
|
278
|
+
} catch (e) { noteFailure ??= e; }
|
|
279
|
+
return answered;
|
|
280
|
+
}, () => noteFailure !== null)).filter(Boolean);
|
|
281
|
+
rows.push(...started);
|
|
282
|
+
// Each lost home this sweep answered is judged: no later sweep resumes this loss (a status read
|
|
283
|
+
// never uses one up). A plan judges nothing.
|
|
284
|
+
if (!plan) {
|
|
285
|
+
const lost = new Set(candidates.map((c) => c.home)), byDeployment = new Map();
|
|
286
|
+
for (const r of rows) if (lost.has(r.home) && !NOT_JUDGED.has(r.reason)) byDeployment.set(r.deployment, [...(byDeployment.get(r.deployment) ?? []), { home: r.home, startId: candidates.find((c) => c.home === r.home).startId ?? null, outcome: r.outcome }]);
|
|
287
|
+
for (const [dir, judged] of byDeployment) judgeLosses(join(dir, "agents"), judged, { boot });
|
|
288
|
+
}
|
|
289
|
+
if (noteFailure) throw noteFailure;
|
|
290
|
+
rows.sort((a, b) => (a.deployment < b.deployment ? -1 : a.deployment > b.deployment ? 1 : a.home < b.home ? -1 : a.home > b.home ? 1 : 0));
|
|
291
|
+
return { schemaVersion: RESUME_SCHEMA, plan: !!plan, deployments, rows, ...(hostWide.size ? { warnings: [...hostWide].sort() } : {}) };
|
|
292
|
+
} finally { release?.(); }
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/** `--plan`'s prediction for a lost home, from what its start would check, read only: refused for a
|
|
296
|
+
* home with no recipe, a reference not set in `env` (the environment each start gets) or one whose
|
|
297
|
+
* value changed since the start it would bring back (lib/start-env.mjs, the start's own check), else
|
|
298
|
+
* resumed or fresh as the conversation lookup reads now, with the start's warning when its launch
|
|
299
|
+
* environment has no record to check. */
|
|
300
|
+
function predicted(row, env) {
|
|
301
|
+
const refused = (code, message) => ({ ...base(row), outcome: OUTCOME.REFUSED, reason: ROW_REASON.START_REFUSED, code, message });
|
|
302
|
+
if (!(row.launch && typeof row.launch === "object")) return refused("E_LAUNCH_LEGACY", `${row.instance} records no launch recipe (an earlier kernel spawned it): re-spawn it from the deployment`);
|
|
303
|
+
const missing = missingLaunchEnvRefs(row.launch.env, env);
|
|
304
|
+
if (missing.length) return refused("E_LAUNCH_ENV_MISSING", launchEnvMissingMessage(missing));
|
|
305
|
+
const fit = checkStartEnv(row.home, row.startId ?? null, fromEnvNames(row.launch.env), env);
|
|
306
|
+
if (fit.changed) return refused("E_LAUNCH_ENV_CHANGED", launchEnvChangedMessage(fit.changed));
|
|
307
|
+
const extra = fit.unverified ? { warnings: [launchEnvUnverifiedWarning(row.instance)] } : {};
|
|
308
|
+
const found = resumeConversation(row.home, row);
|
|
309
|
+
return found.sessionId ? { ...base(row), outcome: OUTCOME.RESUMED, sessionId: found.sessionId, ...extra } : { ...base(row), outcome: OUTCOME.FRESH, reason: found.reason, ...extra };
|
|
310
|
+
}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/** What the kernel has seen of each launched home's session (feature session-resume, #1559;
|
|
2
|
+
* awebai/oats#1567 item 3): the record the sweep reads so that it resumes only a loss it can vouch
|
|
3
|
+
* for. Kernel-private, not a contract, under OATS_HOME_DIR beside the per-boot note: one file per
|
|
4
|
+
* deployment, `session-resume/seen/<first 16 hex of sha256(instances root)>.json`,
|
|
5
|
+
* { deployment, root, homes: { <home's real path>: { startId, lastSeenRunningAt, lastSeenRunningBoot,
|
|
6
|
+
* firstNotRunningAt, firstNotRunningBoot, judgedBy } } }
|
|
7
|
+
* (times ISO; boots bootId's, null without one; `judgedBy` `{ at, boot, outcome }` or null).
|
|
8
|
+
*
|
|
9
|
+
* - Observations have one producer, a local `listInstances` (`oats status`, the sweep's own read
|
|
10
|
+
* and every other local listing): recordSeen. A status routed to another machine reads nothing
|
|
11
|
+
* here; a start writes nothing.
|
|
12
|
+
* - `judgedBy` has one writer, a sweep that is not a plan (judgeLosses), once its rows are known:
|
|
13
|
+
* a status read never uses up a loss.
|
|
14
|
+
* - Never in the way of what reads it: an unreadable file is an empty record (which resumes
|
|
15
|
+
* nothing), every write error is swallowed and nothing is printed. A write is one atomic rename
|
|
16
|
+
* of the whole file, made only when something changed; two writers race harmlessly (the last
|
|
17
|
+
* rename wins whole).
|
|
18
|
+
* - Whether a loss is resumed is one pure function, resumeEligible.
|
|
19
|
+
* Dependency-free. */
|
|
20
|
+
import { execFileSync } from "node:child_process";
|
|
21
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
22
|
+
import { mkdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
23
|
+
import { homedir } from "node:os";
|
|
24
|
+
import { dirname, join, resolve } from "node:path";
|
|
25
|
+
|
|
26
|
+
/** A loss first seen this long ago or more is not resumed. */
|
|
27
|
+
export const FIRST_SEEN_MAX_MS = 24 * 60 * 60_000;
|
|
28
|
+
/** Within one boot, a loss first seen this long or more after the last running look is not resumed. */
|
|
29
|
+
export const WITHIN_BOOT_GAP_MS = 24 * 60 * 60_000;
|
|
30
|
+
/** A running home's last running look is rewritten at most this often in one boot. */
|
|
31
|
+
export const RUNNING_REFRESH_MS = 10 * 60_000;
|
|
32
|
+
|
|
33
|
+
const oatsHomeDir = () => process.env.OATS_HOME_DIR || join(homedir(), ".oats");
|
|
34
|
+
/** The kernel-private directory of the sweep's lock, its per-boot note and this record. */
|
|
35
|
+
export const resumeStateDir = () => join(oatsHomeDir(), "session-resume");
|
|
36
|
+
|
|
37
|
+
/** This boot's identity, or null where the kernel cannot read one: Linux's boot id, macOS's boot
|
|
38
|
+
* time (`sysctl -n kern.boottime`, its `sec`/`usec`). */
|
|
39
|
+
export function bootId({ platform = process.platform, read = (p) => readFileSync(p, "utf8"), sysctl = () => execFileSync("sysctl", ["-n", "kern.boottime"], { encoding: "utf8", timeout: 5000, stdio: ["ignore", "pipe", "ignore"] }) } = {}) {
|
|
40
|
+
try {
|
|
41
|
+
if (platform === "linux") return read("/proc/sys/kernel/random/boot_id").trim() || null;
|
|
42
|
+
if (platform === "darwin") { const m = /sec = (\d+), usec = (\d+)/.exec(sysctl()); return m ? `${m[1]}.${m[2]}` : null; }
|
|
43
|
+
} catch { /* unknown */ }
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
let processBoot;
|
|
47
|
+
/** bootId, read once per process: a process never outlives its boot. */
|
|
48
|
+
export const thisBoot = () => (processBoot === undefined ? (processBoot = bootId()) : processBoot);
|
|
49
|
+
|
|
50
|
+
const realOrResolved = (p) => { try { return realpathSync(p); } catch { return resolve(p); } };
|
|
51
|
+
/** The key of `home` in a record: its real path, so a deployment reached through a symlink (an
|
|
52
|
+
* explicit `--deployment` spelled so, a cwd under one) reads and writes the same entry. */
|
|
53
|
+
export const homeKey = (home) => realOrResolved(home);
|
|
54
|
+
/** The record file of the deployment whose instances root is `root` (its real path, the same way). */
|
|
55
|
+
export function seenFile(root) {
|
|
56
|
+
return join(resumeStateDir(), "seen", `${createHash("sha256").update(realOrResolved(root)).digest("hex").slice(0, 16)}.json`);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** The record of `root` → `{ homes, readable }`: no file is an empty, readable record; a file that
|
|
60
|
+
* cannot be read or parsed is an empty, unreadable one (rewritten whole by the next change). */
|
|
61
|
+
export function readSeen(root) {
|
|
62
|
+
let text;
|
|
63
|
+
try { text = readFileSync(seenFile(root), "utf8"); }
|
|
64
|
+
catch (e) { return { homes: {}, readable: e?.code === "ENOENT" }; }
|
|
65
|
+
try {
|
|
66
|
+
const doc = JSON.parse(text);
|
|
67
|
+
if (!doc || typeof doc !== "object" || !doc.homes || typeof doc.homes !== "object" || Array.isArray(doc.homes)) return { homes: {}, readable: false };
|
|
68
|
+
const homes = {};
|
|
69
|
+
for (const [home, r] of Object.entries(doc.homes)) if (r && typeof r === "object" && !Array.isArray(r)) homes[home] = r;
|
|
70
|
+
return { homes, readable: true };
|
|
71
|
+
} catch { return { homes: {}, readable: false }; }
|
|
72
|
+
}
|
|
73
|
+
function writeSeen(root, homes) {
|
|
74
|
+
const path = seenFile(root), tmp = `${path}.${process.pid}.${randomBytes(4).toString("hex")}.tmp`;
|
|
75
|
+
try {
|
|
76
|
+
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
77
|
+
const real = realOrResolved(root);
|
|
78
|
+
writeFileSync(tmp, JSON.stringify({ deployment: dirname(real), root: real, homes }, null, 2) + "\n", { mode: 0o600 });
|
|
79
|
+
renameSync(tmp, path);
|
|
80
|
+
} catch { try { rmSync(tmp, { force: true }); } catch { /* nothing to remove */ } }
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const startIdOf = (row) => (typeof row?.startId === "string" && row.startId ? row.startId : null);
|
|
84
|
+
|
|
85
|
+
/** The record after one listing of `rows` (status rows) at `now` in `boot` → `{ homes, changed }`.
|
|
86
|
+
* Per row:
|
|
87
|
+
* - running, another start or no record: a new record, seen running now (a new start resets it);
|
|
88
|
+
* - running, the same start: its last running look moves to now when it is RUNNING_REFRESH_MS old
|
|
89
|
+
* or more, or from another boot;
|
|
90
|
+
* - not running, launched, the same start, no loss seen yet: the loss is first seen now (once);
|
|
91
|
+
* - running unknown (null): unchanged.
|
|
92
|
+
* A home no longer listed is dropped. Pure. */
|
|
93
|
+
export function nextSeen(homes, rows, { now, boot }) {
|
|
94
|
+
const at = new Date(now).toISOString(), out = {};
|
|
95
|
+
let changed = false;
|
|
96
|
+
for (const row of rows) {
|
|
97
|
+
if (typeof row?.home !== "string") continue;
|
|
98
|
+
const prior = homes[row.home], startId = startIdOf(row);
|
|
99
|
+
let next = prior;
|
|
100
|
+
if (row.running === true) {
|
|
101
|
+
if (!prior || prior.startId !== startId) next = { startId, lastSeenRunningAt: at, lastSeenRunningBoot: boot, firstNotRunningAt: null, firstNotRunningBoot: null, judgedBy: null };
|
|
102
|
+
else if (!(now - Date.parse(prior.lastSeenRunningAt) < RUNNING_REFRESH_MS) || prior.lastSeenRunningBoot !== boot) next = { ...prior, lastSeenRunningAt: at, lastSeenRunningBoot: boot };
|
|
103
|
+
} else if (row.running === false && row.launched === true && prior && prior.startId === startId && !prior.firstNotRunningAt) {
|
|
104
|
+
next = { ...prior, firstNotRunningAt: at, firstNotRunningBoot: boot };
|
|
105
|
+
}
|
|
106
|
+
if (next) out[row.home] = next;
|
|
107
|
+
if (next !== prior) changed = true;
|
|
108
|
+
}
|
|
109
|
+
if (Object.keys(homes).some((home) => !Object.hasOwn(out, home))) changed = true;
|
|
110
|
+
return { homes: out, changed };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Record one local listing of `root`'s homes (listInstances, its every row). Never throws, never
|
|
114
|
+
* prints: what reads the listing answers the same whatever happens here. */
|
|
115
|
+
export function recordSeen(root, rows, { now = Date.now(), boot = thisBoot() } = {}) {
|
|
116
|
+
try {
|
|
117
|
+
const before = readSeen(root);
|
|
118
|
+
const { homes, changed } = nextSeen(before.homes, rows.map((row) => ({ ...row, home: typeof row?.home === "string" ? homeKey(row.home) : row?.home })), { now, boot });
|
|
119
|
+
if (changed || !before.readable) writeSeen(root, homes);
|
|
120
|
+
} catch { /* the record is the sweep's, never the reader's problem */ }
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** A sweep's answers for the lost homes of `root` (`[{ home, startId, outcome }]`), written once
|
|
124
|
+
* into each home's record as `judgedBy`, when the record is of that start and not judged yet: the
|
|
125
|
+
* file is read again just before the write, so a status write in between is kept. A `judgedBy` lost
|
|
126
|
+
* to a status write that renames over this one is harmless: a resumed home has a new start id; a
|
|
127
|
+
* refused one is still held by the per-boot refusal note; a skipped or blocked one is judged the
|
|
128
|
+
* same again. Never throws. */
|
|
129
|
+
export function judgeLosses(root, judged, { now = Date.now(), boot = thisBoot() } = {}) {
|
|
130
|
+
try {
|
|
131
|
+
const { homes } = readSeen(root);
|
|
132
|
+
let changed = false;
|
|
133
|
+
for (const { home, startId, outcome } of judged) {
|
|
134
|
+
const key = homeKey(home), r = homes[key];
|
|
135
|
+
if (!r || r.startId !== (startId ?? null) || r.judgedBy) continue;
|
|
136
|
+
homes[key] = { ...r, judgedBy: { at: new Date(now).toISOString(), boot, outcome } };
|
|
137
|
+
changed = true;
|
|
138
|
+
}
|
|
139
|
+
if (changed) writeSeen(root, homes);
|
|
140
|
+
} catch { /* best effort: see above */ }
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** Whether the sweep resumes the lost home of `row` (a status row) on its `record`, at `now` in
|
|
144
|
+
* `boot` → `{ eligible: true }` or `{ eligible: false, why }`. All four must hold:
|
|
145
|
+
* 1. the record saw it running for its current start;
|
|
146
|
+
* 2. no earlier sweep judged this loss (`judgedBy`);
|
|
147
|
+
* 3. the loss was first seen less than FIRST_SEEN_MAX_MS ago;
|
|
148
|
+
* 4. within one boot (both looks in the same boot, or either without a boot id), the last running
|
|
149
|
+
* look is less than WITHIN_BOOT_GAP_MS before the first not-running look. Across a reboot it
|
|
150
|
+
* does not apply.
|
|
151
|
+
* The rule reads the boots the record holds, not this one's. Pure. */
|
|
152
|
+
export function resumeEligible(record, row, { now }) {
|
|
153
|
+
const no = (why) => ({ eligible: false, why });
|
|
154
|
+
const lastRunning = Date.parse(record?.lastSeenRunningAt);
|
|
155
|
+
if (!record || record.startId !== startIdOf(row) || !Number.isFinite(lastRunning)) return no("OATS never saw it running for its current start");
|
|
156
|
+
if (record.judgedBy) return no(`an earlier sweep already answered this loss (${record.judgedBy.outcome ?? "answered"}${record.judgedBy.at ? ` at ${record.judgedBy.at}` : ""})`);
|
|
157
|
+
const firstLost = Date.parse(record.firstNotRunningAt);
|
|
158
|
+
if (!Number.isFinite(firstLost)) return no("OATS has not seen it stop since it last saw it running");
|
|
159
|
+
if (now - firstLost >= FIRST_SEEN_MAX_MS) return no(`its loss was first seen ${record.firstNotRunningAt}, 24 hours ago or more`);
|
|
160
|
+
const oneBoot = !record.lastSeenRunningBoot || !record.firstNotRunningBoot || record.lastSeenRunningBoot === record.firstNotRunningBoot;
|
|
161
|
+
if (oneBoot && firstLost - lastRunning >= WITHIN_BOOT_GAP_MS) return no(`in the boot it was lost in, it was last seen running ${record.lastSeenRunningAt}, 24 hours or more before its loss was seen`);
|
|
162
|
+
return { eligible: true };
|
|
163
|
+
}
|