@awebai/oats 0.69.12 → 0.69.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +216 -178
- package/bin/oats.mjs +254 -201
- package/docs/capabilities.md +58 -13
- package/docs/configuration.md +63 -5
- package/docs/container.md +291 -0
- package/docs/design/2026-09-23-workspace-module-contracts.md +2 -0
- package/docs/desktop-cli-api.md +590 -55
- package/docs/desktop-instance-start.md +4 -1
- package/docs/desktop.md +58 -12
- package/docs/execution-targets.md +84 -9
- package/docs/first-team.md +4 -3
- package/docs/implementation.md +99 -32
- package/docs/integrations.md +3 -2
- package/docs/knowledge-and-learning.md +72 -0
- package/docs/oats-local.schema.json +11 -2
- package/docs/official-catalog.md +39 -5
- package/docs/packages.md +93 -14
- package/docs/release-lane.md +2 -1
- package/docs/release-notes/v0.69.13.md +767 -0
- package/docs/release-notes/v0.69.14.md +83 -0
- package/docs/release-notes/v0.69.15.md +444 -0
- package/docs/schedules.md +159 -5
- package/docs/servers.md +241 -7
- package/docs/souls-and-instances.md +223 -14
- package/docs/tui.md +1 -1
- package/docs/upgrading.md +36 -5
- package/docs/workspaces.md +31 -5
- package/lib/claim.mjs +20 -5
- package/lib/command-flags.mjs +240 -0
- package/lib/core.mjs +1004 -633
- package/lib/deployments.mjs +4 -3
- package/lib/dir-lock.mjs +145 -33
- package/lib/errors.mjs +3 -2
- package/lib/harness-token.mjs +110 -0
- package/lib/host-run.mjs +218 -0
- package/lib/host-stop-child.mjs +21 -0
- package/lib/hot-reads.mjs +4 -3
- package/lib/instance-events.mjs +28 -6
- package/lib/instance-git.mjs +165 -14
- package/lib/instance-inspect.mjs +6 -1
- package/lib/instance-lifecycle.mjs +4 -3
- package/lib/instance-listing.mjs +114 -43
- package/lib/instance-resolution.mjs +56 -3
- package/lib/instance-upgrade.mjs +16 -20
- package/lib/kernel-features.mjs +4 -1
- package/lib/launch-env-exec.sh +7 -0
- package/lib/launch-env.mjs +74 -0
- package/lib/launch-preference.mjs +19 -12
- package/lib/member-clone.mjs +7 -1
- package/lib/packages.mjs +74 -2
- package/lib/read-call.mjs +4 -1
- package/lib/reader.mjs +3 -2
- package/lib/refused-text.mjs +3 -2
- package/lib/remote.mjs +126 -31
- package/lib/retained-after-retire.mjs +2 -7
- package/lib/retire-git-forms.mjs +41 -0
- package/lib/retire-incomplete.mjs +96 -0
- package/lib/schedule.mjs +239 -21
- package/lib/servers.mjs +161 -28
- package/lib/session-end.mjs +28 -8
- package/lib/session-resume.mjs +63 -56
- package/lib/session-seen.mjs +3 -14
- package/lib/start-env.mjs +13 -7
- package/lib/status-words.mjs +11 -4
- package/lib/system-tools.mjs +1 -36
- package/lib/teams.mjs +11 -4
- package/lib/upgrade-state.mjs +13 -2
- package/lib/worktree-hooks.mjs +6 -175
- package/lib/worktree.mjs +68 -10
- package/package-catalog.json +1 -1
- package/package.json +5 -1
- package/packages/client/instance-lifecycle.mjs +1 -1
- package/packages/client/lifecycle-contract.mjs +3 -1
- package/packages/client/session-resume-cli.mjs +2 -1
- package/packages/client/session-start.mjs +10 -7
- package/packages/client/workspace-cli.mjs +5 -1
- package/packages/record/README.md +56 -21
- package/packages/record/docs/capture-service.md +8 -5
- package/packages/record/docs/turn-record-sot.md +9 -0
- package/packages/record/lib/capture-health.mjs +6 -4
- package/packages/record/lib/capture-lock.mjs +31 -27
- package/packages/record/lib/capture-retained-scratch.mjs +4 -3
- package/packages/record/lib/capture-scratch.mjs +2 -1
- package/packages/record/lib/derived-state.mjs +4 -2
- package/packages/record/lib/index-db.mjs +4 -2
- package/packages/record/lib/lock-holder.mjs +21 -0
- package/packages/record/lib/private-files.mjs +18 -0
- package/packages/record/lib/process-identity.mjs +310 -0
- package/packages/record/lib/store.mjs +31 -25
- package/share/oats-login-receive +204 -0
- package/skills/oats-getting-started/SKILL.md +93 -26
- package/lib/tmux-config.mjs +0 -35
package/lib/servers.mjs
CHANGED
|
@@ -17,10 +17,11 @@
|
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
19
|
import { execFile, execFileSync, spawnSync } from "node:child_process";
|
|
20
|
-
import { createHash, randomBytes } from "node:crypto";
|
|
20
|
+
import { createHash, randomBytes, randomUUID } from "node:crypto";
|
|
21
21
|
import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
|
22
22
|
import { homedir } from "node:os";
|
|
23
23
|
import { basename, join, resolve } from "node:path";
|
|
24
|
+
import { isDeepStrictEqual } from "node:util";
|
|
24
25
|
import { parseStrictJson } from "./canonical-json.mjs";
|
|
25
26
|
import { noteRuntimeName } from "./deprecation.mjs";
|
|
26
27
|
import { readWaitingClaim, validWaitingMessage } from "./instance-events.mjs";
|
|
@@ -33,12 +34,14 @@ import { shellWord } from "./shell-word.mjs";
|
|
|
33
34
|
import { DETACH_KEY_FEATURE, requireDetachKey } from "./session-viewer.mjs";
|
|
34
35
|
import { LOGIN_CAPTURE_MAX_BYTES, LOGIN_SHELLS, loginPathEmitter, quoteFor, readLoginFrame } from "./login-environment.mjs";
|
|
35
36
|
import { compareSemver } from "./semver.mjs";
|
|
37
|
+
import { SPAWN_TASK_STDIN_FEATURE } from "./kernel-features.mjs";
|
|
36
38
|
import { sameRepo } from "./repo-moves.mjs";
|
|
37
39
|
import { OATS_PACKAGE_NAME } from "./version.mjs";
|
|
38
40
|
export { compareSemver } from "./semver.mjs";
|
|
39
41
|
|
|
40
42
|
const OATS_HOME_DIR = () => process.env.OATS_HOME_DIR || join(homedir(), ".oats");
|
|
41
43
|
export const SERVERS_FILE = () => join(OATS_HOME_DIR(), "servers.json");
|
|
44
|
+
export const SERVER_OPERATOR_ROUTES_FILE = () => join(OATS_HOME_DIR(), "server-operator-routes.json");
|
|
42
45
|
export const REMOTE_SNAPSHOT_DIR = () => join(OATS_HOME_DIR(), "remote");
|
|
43
46
|
|
|
44
47
|
/** Every ssh invocation is non-interactive: a host that needs a password or a
|
|
@@ -131,6 +134,28 @@ export function readServers() {
|
|
|
131
134
|
return out;
|
|
132
135
|
}
|
|
133
136
|
|
|
137
|
+
/** Operator routes stay out of servers.json so an older kernel can still read every agent route. */
|
|
138
|
+
export function readServerOperatorRoutes() {
|
|
139
|
+
const file = SERVER_OPERATOR_ROUTES_FILE();
|
|
140
|
+
if (!existsSync(file)) return {};
|
|
141
|
+
let routes;
|
|
142
|
+
try { routes = JSON.parse(readFileSync(file, "utf8")); }
|
|
143
|
+
catch { throw serverError("E_SERVERS_UNREADABLE", `${file} is not valid JSON`); }
|
|
144
|
+
validateOperatorRoutes(routes);
|
|
145
|
+
return routes;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export function validateOperatorSshHost(alias) {
|
|
149
|
+
if (typeof alias !== "string" || !SSH_HOST_RE.test(alias)) throw serverError("E_SERVER_INVALID", "operatorSshHost must be an OpenSSH host alias or host name (no user@, no options)");
|
|
150
|
+
}
|
|
151
|
+
function validateOperatorRoutes(routes) {
|
|
152
|
+
if (!routes || typeof routes !== "object" || Array.isArray(routes)) throw serverError("E_SERVERS_UNREADABLE", `${SERVER_OPERATOR_ROUTES_FILE()} must map server ids to { operatorSshHost: <alias> }`);
|
|
153
|
+
for (const [id, entry] of Object.entries(routes)) {
|
|
154
|
+
if (!ID_RE.test(id) || !entry || typeof entry !== "object" || Array.isArray(entry) || Object.keys(entry).length !== 1 || !Object.hasOwn(entry, "operatorSshHost")) throw serverError("E_SERVER_INVALID", `${SERVER_OPERATOR_ROUTES_FILE()}: expected a server id and { operatorSshHost: <alias> }`);
|
|
155
|
+
validateOperatorSshHost(entry.operatorSshHost);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
134
159
|
export function writeServers(servers) {
|
|
135
160
|
const file = SERVERS_FILE();
|
|
136
161
|
// A registration whose target changes (server add --replace) is probed afresh.
|
|
@@ -144,15 +169,24 @@ export function writeServers(servers) {
|
|
|
144
169
|
}
|
|
145
170
|
|
|
146
171
|
/** Every read-modify-write of the registry: under one short lock, over the file as it is NOW (never a
|
|
147
|
-
* copy read before a network call), written only when `change` altered it. `change(servers)`
|
|
148
|
-
* mutates its
|
|
149
|
-
|
|
172
|
+
* copy read before a network call), written only when `change` altered it. `change(servers, routes)`
|
|
173
|
+
* mutates its arguments and returns what the caller needs. With `operatorRoutes`, the sidecar is
|
|
174
|
+
* read and written under this same lock; agent-only updates never need to read it. */
|
|
175
|
+
export function updateServers(change, { operatorRoutes = false } = {}) {
|
|
150
176
|
const lock = join(OATS_HOME_DIR(), "servers.lock");
|
|
151
177
|
return withDirLock(lock, "the server registry", () => {
|
|
152
178
|
const servers = readServers();
|
|
153
|
-
const
|
|
154
|
-
const
|
|
155
|
-
|
|
179
|
+
const routes = operatorRoutes ? readServerOperatorRoutes() : undefined;
|
|
180
|
+
const before = structuredClone(servers);
|
|
181
|
+
const routesBefore = JSON.stringify(routes);
|
|
182
|
+
const out = change(servers, routes);
|
|
183
|
+
if (routes) validateOperatorRoutes(routes);
|
|
184
|
+
// Key ordering alone must not rewrite the legacy registry when only an operator route changes.
|
|
185
|
+
if (!isDeepStrictEqual(servers, before)) writeServers(servers);
|
|
186
|
+
if (JSON.stringify(routes) !== routesBefore) {
|
|
187
|
+
mkdirSync(OATS_HOME_DIR(), { recursive: true });
|
|
188
|
+
writeFileSync(SERVER_OPERATOR_ROUTES_FILE(), JSON.stringify(routes, null, 2) + "\n", { mode: 0o600 });
|
|
189
|
+
}
|
|
156
190
|
return out;
|
|
157
191
|
}, { retryMs: 5000, busy: (why) => serverError("E_SERVERS_BUSY", `the server registry is locked (${why}); if no oats process is changing it, remove ${lock} and retry; nothing was changed`) });
|
|
158
192
|
}
|
|
@@ -160,13 +194,15 @@ export function updateServers(change) {
|
|
|
160
194
|
/** Record the workspace key a host reported for registration `id`, only if the registration is still
|
|
161
195
|
* the one that was asked (same host, workspace, oats path and PATH), so a registration changed or
|
|
162
196
|
* removed meanwhile is never overwritten or brought back. → "recorded" | "same" | "changed" |
|
|
163
|
-
* { mismatch: <the recorded key> }
|
|
164
|
-
|
|
197
|
+
* { mismatch: <the recorded key> }. `replace` is the explicit server add --replace remedy: a
|
|
198
|
+
* preserved key may be replaced only while it is still the key this request observed. */
|
|
199
|
+
export function learnWorkspaceKey(id, asked, reported, { replace = false } = {}) {
|
|
165
200
|
return updateServers((servers) => {
|
|
166
201
|
const current = servers[id];
|
|
167
202
|
if (!current || JSON.stringify(targetOf(current)) !== JSON.stringify(targetOf(asked))) return "changed";
|
|
168
203
|
if (sameRepo(current.workspaceKey, reported)) return "same";
|
|
169
|
-
if (current.workspaceKey
|
|
204
|
+
if (replace && current.workspaceKey !== asked.workspaceKey) return "changed";
|
|
205
|
+
if (current.workspaceKey && !replace) return { mismatch: current.workspaceKey };
|
|
170
206
|
current.workspaceKey = reported;
|
|
171
207
|
return "recorded";
|
|
172
208
|
});
|
|
@@ -193,7 +229,7 @@ export function validateServer(id, s) {
|
|
|
193
229
|
// The canonical key of the workspace the host deployment realizes, as its own `status --json` reported it.
|
|
194
230
|
if (s.workspaceKey !== undefined && (typeof s.workspaceKey !== "string" || !WORKSPACE_KEY_RE.test(s.workspaceKey))) throw serverError("E_SERVER_INVALID", `server ${id}: workspaceKey must be a canonical workspace key (as the host's oats status --json reports it)`);
|
|
195
231
|
for (const k of Object.keys(s)) {
|
|
196
|
-
if (!["label", "sshHost", "workspace", "oatsPath", "path", "workspaceKey"].includes(k)) throw serverError("E_SERVER_INVALID", `server ${id}: unknown field ${JSON.stringify(k)} (a registration holds label, sshHost, workspace, oatsPath, path, workspaceKey and nothing else — never keys or passwords)`);
|
|
232
|
+
if (!["label", "sshHost", "workspace", "oatsPath", "path", "workspaceKey"].includes(k)) throw serverError("E_SERVER_INVALID", `server ${id}: unknown field ${JSON.stringify(k)} (a registration holds label, sshHost, workspace, oatsPath, path, workspaceKey and nothing else — never keys or passwords; operator routes belong in server-operator-routes.json)`);
|
|
197
233
|
}
|
|
198
234
|
return true;
|
|
199
235
|
}
|
|
@@ -220,9 +256,8 @@ export function workspaceMismatch(id, server, reported) {
|
|
|
220
256
|
|
|
221
257
|
export function getServer(id) {
|
|
222
258
|
const servers = readServers();
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
return { id, ...s };
|
|
259
|
+
if (!Object.hasOwn(servers, id)) throw serverError("E_SERVER_UNKNOWN", `no server registered as ${JSON.stringify(id)} (oats server list)`);
|
|
260
|
+
return { id, ...servers[id] };
|
|
226
261
|
}
|
|
227
262
|
|
|
228
263
|
// ------------------------------------------------------------------ quoting
|
|
@@ -284,6 +319,13 @@ function sshLineArgv(target, cmd, { control = false } = {}) {
|
|
|
284
319
|
return ["ssh", ...SSH_OPTS, ...(control && !controlPathProblem() ? controlOpts() : NO_CONTROL_OPTS), "--", target.sshHost, cmd];
|
|
285
320
|
}
|
|
286
321
|
|
|
322
|
+
/** The forced-command operator key must never reuse the agent route's authenticated master.
|
|
323
|
+
* Only receiver command words belong here; credential bytes go exclusively on stdin. */
|
|
324
|
+
export function operatorSshArgv(alias, words) {
|
|
325
|
+
validateOperatorSshHost(alias);
|
|
326
|
+
return ["ssh", ...SSH_OPTS, ...NO_CONTROL_OPTS, "--", alias, ...words.map(remoteQuote)];
|
|
327
|
+
}
|
|
328
|
+
|
|
287
329
|
// ------------------------------------------------------------------ running
|
|
288
330
|
|
|
289
331
|
/** Parse the remote kernel's JSON envelope; anything else on stdout is a
|
|
@@ -612,10 +654,21 @@ export function listSnapshots(serverId) {
|
|
|
612
654
|
|
|
613
655
|
// ------------------------------------------------------------------- routes
|
|
614
656
|
|
|
657
|
+
/** What a host must advertise for a routed spawn to be bound to its preview and a key: the preview
|
|
658
|
+
* with its decision, the apply that takes one, and the key's recovery before placement. */
|
|
659
|
+
const KEYED_SPAWN_FEATURES = ["spawn-preview-2", "spawn-apply-2", "spawn-idempotency-2"];
|
|
660
|
+
|
|
615
661
|
/** Route one lifecycle command to a server. Returns the remote envelope,
|
|
616
662
|
* having kept the local snapshot store in step with it:
|
|
617
663
|
* - spawn: compatibility checked first, a snapshot written on success; a
|
|
618
664
|
* --preview runs the same checks, needs spawn-preview-2 and writes nothing;
|
|
665
|
+
* a spawn with no decision and no key of its own is previewed on the host
|
|
666
|
+
* and applied bound to that decision and a key (KEYED_SPAWN_FEATURES); a
|
|
667
|
+
* `--task <text>` does not travel as an argument, where it would be in the
|
|
668
|
+
* command line of the local ssh and of the host's oats: each spawn call
|
|
669
|
+
* carries `--task-file -` in its place and the text on its standard input
|
|
670
|
+
* (SPAWN_TASK_STDIN_FEATURE). A host without the feature gets the task as
|
|
671
|
+
* an argument, and the answer carries a warning that says so;
|
|
619
672
|
* - retire: the target comes from the instance's snapshot when one exists
|
|
620
673
|
* (the registry may have changed since), the snapshot is removed only
|
|
621
674
|
* when the remote kernel reports the home gone;
|
|
@@ -647,6 +700,23 @@ export function routeCommand(serverId, cmd, oatsArgs, io = {}) {
|
|
|
647
700
|
throw serverError("E_ROUTE_CHANGED", `server ${serverId} now points at ${target.sshHost}:${target.workspace}, but ${priorTargets.length} saved route${priorTargets.length === 1 ? "" : "s"} for it (${priorTargets.map((s) => s.instance).join(", ")}) target ${old.sshHost}:${old.workspace}; spawning would overwrite them. Keep the old registration and add a new server id for the new target, or retire those instances first`);
|
|
648
701
|
}
|
|
649
702
|
const remote = checkRemote(target, io);
|
|
703
|
+
// For a host that reads a task from its standard input (SPAWN_TASK_STDIN_FEATURE), the task
|
|
704
|
+
// leaves the arguments here, before any check reads them or a spawn call is built. The first
|
|
705
|
+
// --task is the one a spawn reads; every one is taken out. A host without the feature gets
|
|
706
|
+
// the arguments as given, and its answer says where the task was readable.
|
|
707
|
+
let task, taskAsArgument;
|
|
708
|
+
const taskAt = oatsArgs.indexOf("--task");
|
|
709
|
+
if (taskAt >= 0 && oatsArgs[taskAt + 1] !== undefined && !remote.features.includes(SPAWN_TASK_STDIN_FEATURE)) {
|
|
710
|
+
taskAsArgument = `remote oats ${remote.version} at ${target.sshHost} does not advertise ${SPAWN_TASK_STDIN_FEATURE}, so the task was passed as an argument, visible in this machine's process list (the ssh command) and the host's while the spawn runs; upgrade the host`;
|
|
711
|
+
} else if (taskAt >= 0 && oatsArgs[taskAt + 1] !== undefined) {
|
|
712
|
+
task = Buffer.from(oatsArgs[taskAt + 1]);
|
|
713
|
+
const kept = [];
|
|
714
|
+
for (let i = 0; i < oatsArgs.length; i++) {
|
|
715
|
+
if (oatsArgs[i] !== "--task" || oatsArgs[i + 1] === undefined) kept.push(oatsArgs[i]);
|
|
716
|
+
else { if (i === taskAt) kept.push("--task-file", "-"); i++; }
|
|
717
|
+
}
|
|
718
|
+
oatsArgs = kept;
|
|
719
|
+
}
|
|
650
720
|
// A saved route is keyed by name under its server id. An explicit name
|
|
651
721
|
// (--name, or --instance) that already has one here would overwrite that
|
|
652
722
|
// route on success: refuse before the mutation (the roster shows the
|
|
@@ -676,7 +746,50 @@ export function routeCommand(serverId, cmd, oatsArgs, io = {}) {
|
|
|
676
746
|
if (oatsArgs.includes("--background") || oatsArgs.includes("--no-background")) requireRemoteFeature(remote, target, "instance-background");
|
|
677
747
|
// A host without instance-same-work would refuse the flag as unknown: it is never sent there.
|
|
678
748
|
if (oatsArgs.includes("--same-work")) requireRemoteFeature(remote, target, "instance-same-work");
|
|
679
|
-
const
|
|
749
|
+
const spawnOnHost = (args) => runRemote(target, json(withScope(["spawn", ...hostHarnessArgs(remote, args)])), { ...io, input: task });
|
|
750
|
+
// A spawn that brings neither a decision nor a key is bound here: the host's preview decides, and
|
|
751
|
+
// the apply carries that decision and a key of this request, so the same spawn sent again with
|
|
752
|
+
// both answers the instance this one created. A host that does not advertise the keyed spawn
|
|
753
|
+
// gets the spawn as given, and its answer says what that costs.
|
|
754
|
+
const given = (flag) => oatsArgs.some((a) => a === flag || a.startsWith(`${flag}=`));
|
|
755
|
+
const suggesting = oatsArgs.includes("--suggest-name");
|
|
756
|
+
let applyArgs = oatsArgs, bound, unbound, refusal;
|
|
757
|
+
if (!preview && !given("--expect-decision") && !given("--idempotency-key")) {
|
|
758
|
+
// The decision binds a name, so a suggested one is chosen by the preview and applied as --name.
|
|
759
|
+
const missing = [...KEYED_SPAWN_FEATURES, ...(suggesting ? ["spawn-suggest-name", "spawn-name"] : [])].filter((f) => !remote.features.includes(f));
|
|
760
|
+
if (missing.length) unbound = `remote oats ${remote.version} at ${target.sshHost} does not advertise ${missing.join(", ")}: if the link drops while it spawns, the spawn cannot be retried safely (the instance may exist there, and a second run would create another); check oats status --server ${serverId} first, or upgrade oats there`;
|
|
761
|
+
else {
|
|
762
|
+
const previewed = spawnOnHost([...oatsArgs, "--preview"]);
|
|
763
|
+
const shown = previewed.envelope.result;
|
|
764
|
+
// Anything but a preview with its decision is the host's answer to this spawn (a refusal).
|
|
765
|
+
if (!(previewed.envelope.ok && shown?.preview === true && typeof shown.decision?.revision === "string")) refusal = previewed;
|
|
766
|
+
else {
|
|
767
|
+
bound = { decision: shown.decision, idempotencyKey: randomUUID(), naming: shown.naming };
|
|
768
|
+
applyArgs = [...(suggesting ? [...oatsArgs.filter((a) => a !== "--suggest-name"), "--name", shown.instance] : oatsArgs), "--expect-decision", shown.decision.revision, "--idempotency-key", bound.idempotencyKey];
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
let answer = refusal;
|
|
773
|
+
try { answer ??= spawnOnHost(applyArgs); }
|
|
774
|
+
catch (e) {
|
|
775
|
+
// The link failed under a bound apply: the host may have started it. The decision and the key
|
|
776
|
+
// travel with the failure, so the spawn sent again lands on that one instance.
|
|
777
|
+
if (bound && e.code === "E_SSH" && e.details?.sshStarted !== false) {
|
|
778
|
+
const fix = `run the same spawn again with ${suggesting ? `--name ${bound.decision.instance} in place of --suggest-name, and ` : ""}--expect-decision ${bound.decision.revision} --idempotency-key ${bound.idempotencyKey}: it answers the instance this attempt created there, or creates it`;
|
|
779
|
+
e.message = `${e.message}; the host may have started the spawn: ${fix}`;
|
|
780
|
+
e.details = { ...e.details, decision: bound.decision, idempotencyKey: bound.idempotencyKey, fix };
|
|
781
|
+
}
|
|
782
|
+
throw e;
|
|
783
|
+
}
|
|
784
|
+
const { stderr } = answer;
|
|
785
|
+
let { envelope } = answer;
|
|
786
|
+
if (envelope.ok && envelope.result) {
|
|
787
|
+
const added = [unbound, taskAsArgument].filter(Boolean);
|
|
788
|
+
const warnings = added.length ? { warnings: [...(envelope.result.warnings || []), ...added] } : {};
|
|
789
|
+
// How the preview chose the name is the spawn's own answer to --suggest-name.
|
|
790
|
+
const naming = bound?.naming && envelope.result.naming === undefined ? { naming: bound.naming } : {};
|
|
791
|
+
envelope = { ...envelope, result: { ...envelope.result, ...warnings, ...naming } };
|
|
792
|
+
}
|
|
680
793
|
if (preview && envelope.ok && envelope.result?.preview === true && envelope.result.spawnPreviewApi === 2) {
|
|
681
794
|
// The host's preview with the two fields a routed spawn adds; never a
|
|
682
795
|
// `snapshot` key, which on the apply path says what was (or was not) saved.
|
|
@@ -1263,13 +1376,19 @@ export function scheduleRemote(serverId, oatsArgs, io = {}) {
|
|
|
1263
1376
|
const setsHostCaps = oatsArgs[0] === "host" && oatsArgs[1] === "install" && oatsArgs.some((arg) =>
|
|
1264
1377
|
["--max-concurrent", "--triggers-max-concurrent"].some((name) => arg === name || arg.startsWith(`${name}=`)));
|
|
1265
1378
|
const capsUnsupported = () => serverError("E_REMOTE_INCOMPATIBLE", `remote oats at ${target.sshHost} (${serverId}) does not confirm required features schedule and schedule-host-caps; upgrade oats on that destination before setting or resetting host caps`);
|
|
1379
|
+
// `host install --register-only` installs no timer. A host that does not know the option
|
|
1380
|
+
// ignores it and installs one, so it is sent only to a host that advertises it.
|
|
1381
|
+
const registersOnly = oatsArgs[0] === "host" && oatsArgs[1] === "install" && oatsArgs.includes("--register-only");
|
|
1382
|
+
const registerOnlyUnsupported = () => serverError("E_REMOTE_INCOMPATIBLE", `remote oats at ${target.sshHost} (${serverId}) does not confirm required features schedule and schedule-host-register-only, so --register-only was not sent: a host without it ignores the option and installs a timer; upgrade oats on that destination`);
|
|
1266
1383
|
let remote;
|
|
1267
1384
|
try {
|
|
1268
1385
|
remote = checkRemote(target, io);
|
|
1386
|
+
if (registersOnly && !["schedule", "schedule-host-register-only"].every((feature) => remote.features.includes(feature))) throw registerOnlyUnsupported();
|
|
1269
1387
|
if (!remote.features.includes("schedule")) throw serverError("E_REMOTE_INCOMPATIBLE", `remote oats ${remote.version} at ${target.sshHost} does not advertise schedules; upgrade it there`);
|
|
1270
1388
|
} catch (e) {
|
|
1271
|
-
// Unknown probe formats cannot establish
|
|
1272
|
-
// failures distinct, and preserve the existing contract for
|
|
1389
|
+
// Unknown probe formats cannot establish support for an option. Keep transport
|
|
1390
|
+
// failures distinct, and preserve the existing contract for calls without one.
|
|
1391
|
+
if (registersOnly && ["E_REMOTE_ENVELOPE", "E_REMOTE_INCOMPATIBLE"].includes(e.code)) throw registerOnlyUnsupported();
|
|
1273
1392
|
if (setsHostCaps && ["E_REMOTE_ENVELOPE", "E_REMOTE_INCOMPATIBLE"].includes(e.code)) throw capsUnsupported();
|
|
1274
1393
|
throw e;
|
|
1275
1394
|
}
|
|
@@ -1442,15 +1561,17 @@ function relayedFailure(envelope, what) {
|
|
|
1442
1561
|
|
|
1443
1562
|
/**
|
|
1444
1563
|
* options: { id, sshHost, workspaceRef, dir (absolute, or ~/… resolved by the host), oatsPath?, path?,
|
|
1445
|
-
* label?, installOats?, replace?, localVersion (this kernel's: the version installed) }
|
|
1564
|
+
* label?, operatorSshHost?, noOperatorSsh?, installOats?, replace?, localVersion (this kernel's: the version installed) }
|
|
1446
1565
|
* io: { execFileSync, now, readinessBudgetMs } (the transport and clock; tests inject them).
|
|
1447
1566
|
* → { id, ready, registration, steps, human }; a failed step throws its code with
|
|
1448
1567
|
* details { ...the step's details, steps }.
|
|
1449
1568
|
*/
|
|
1450
1569
|
export function connectServer(options, io = {}) {
|
|
1451
|
-
const { id, sshHost, workspaceRef, dir, oatsPath, path, label, installOats = false, replace = false, localVersion } = options;
|
|
1570
|
+
const { id, sshHost, workspaceRef, dir, oatsPath, path, label, operatorSshHost, noOperatorSsh = false, installOats = false, replace = false, localVersion } = options;
|
|
1452
1571
|
const now = io.now || Date.now;
|
|
1453
1572
|
validateServer(id, { sshHost, workspace: "/", ...(oatsPath ? { oatsPath } : {}), ...(path ? { path } : {}), ...(label ? { label } : {}) });
|
|
1573
|
+
if (operatorSshHost !== undefined) validateOperatorSshHost(operatorSshHost);
|
|
1574
|
+
if (operatorSshHost !== undefined && noOperatorSsh) throw serverError("E_BAD_ARGS", "choose --operator-ssh or --no-operator-ssh, not both");
|
|
1454
1575
|
if (typeof dir !== "string" || !(dir === "~" || dir.startsWith("~/") || dir.startsWith("/"))) throw serverError("E_BAD_ARGS", `--dir must be an absolute path or ~/… on the host (got ${JSON.stringify(dir)})`);
|
|
1455
1576
|
io = { ...io, serverId: id, input: undefined };
|
|
1456
1577
|
const target = { sshHost, workspace: dir, oatsPath: oatsPath || "oats", ...(path ? { path } : {}) };
|
|
@@ -1501,7 +1622,7 @@ export function connectServer(options, io = {}) {
|
|
|
1501
1622
|
const searched = login ? `the PATH of ${sshHost}'s login shell (${basename(login.shell)} -l -i)` : path ? "--path and ssh's PATH" : "ssh's PATH";
|
|
1502
1623
|
const where = login ? `so that its login shell finds it (there, ${runnable([login.shell, "-l", "-i", "-c", "command -v npm"])} prints its path)`
|
|
1503
1624
|
: path ? "where --path or a non-interactive ssh command finds it" : "where a non-interactive ssh command finds it";
|
|
1504
|
-
return { status: "needs-human", detail: `${what}, and no npm on ${searched}`, remedy: `install Node.js 22+ (npm) on ${sshHost} ${where}, then check again (run the same connect); from the CLI, --path <the directory holding npm> names it instead` };
|
|
1625
|
+
return { status: "needs-human", detail: `${what}, and no npm on ${searched}`, remedy: `install Node.js 22.5+ (npm) on ${sshHost} ${where}, then check again (run the same connect); from the CLI, --path <the directory holding npm> names it instead` };
|
|
1505
1626
|
}
|
|
1506
1627
|
const install = ["npm", "install", "-g", `${OATS_PACKAGE_NAME}@${localVersion}`];
|
|
1507
1628
|
if (!installOats) return { status: "needs-human", detail: what, remedy: `on ${sshHost}: ${runnable(install)} (or pass --install-oats)` };
|
|
@@ -1551,10 +1672,18 @@ export function connectServer(options, io = {}) {
|
|
|
1551
1672
|
|
|
1552
1673
|
// One guarded update of the registry as it is now. The requested id comes first: an id registered for
|
|
1553
1674
|
// another target is refused (or replaced), never quietly answered by a twin that serves this target.
|
|
1554
|
-
run("register", () => updateServers((servers) => {
|
|
1675
|
+
run("register", () => updateServers((servers, routes) => {
|
|
1555
1676
|
const abs = check.dir;
|
|
1556
1677
|
const given = { ...(oatsPath ? { oatsPath } : {}), ...(path ? { path } : {}), ...(label ? { label } : {}) };
|
|
1557
1678
|
const entry = { sshHost, workspace: abs, ...found, ...given, workspaceKey: reportedKey };
|
|
1679
|
+
const operatorAt = (regId) => Object.hasOwn(routes, regId) ? routes[regId] : undefined;
|
|
1680
|
+
const operatorChange = (regId) => {
|
|
1681
|
+
const before = operatorAt(regId)?.operatorSshHost;
|
|
1682
|
+
if (operatorSshHost !== undefined) routes[regId] = { operatorSshHost };
|
|
1683
|
+
else if (noOperatorSsh) delete routes[regId];
|
|
1684
|
+
return before !== operatorAt(regId)?.operatorSshHost;
|
|
1685
|
+
};
|
|
1686
|
+
const joined = (regId) => ({ ...servers[regId], ...operatorAt(regId) });
|
|
1558
1687
|
const sameTarget = (s) => s.sshHost === sshHost && s.workspace === abs;
|
|
1559
1688
|
/** A registration of this very target: its key confirmed, or recorded; never contradicted. What
|
|
1560
1689
|
* this run found on the host fills only what it lacks. */
|
|
@@ -1562,30 +1691,34 @@ export function connectServer(options, io = {}) {
|
|
|
1562
1691
|
const existing = servers[regId];
|
|
1563
1692
|
if (existing.workspaceKey && !sameRepo(existing.workspaceKey, reportedKey)) throw workspaceMismatch(regId, existing, reportedKey);
|
|
1564
1693
|
registeredAs = regId;
|
|
1565
|
-
registration = existing;
|
|
1566
1694
|
const filled = Object.keys(found).filter((k) => existing[k] === undefined || existing[k] === null).sort();
|
|
1567
1695
|
for (const k of filled) existing[k] = found[k];
|
|
1568
1696
|
const keyed = !existing.workspaceKey;
|
|
1569
1697
|
if (keyed) existing.workspaceKey = reportedKey;
|
|
1570
|
-
|
|
1571
|
-
|
|
1698
|
+
const operatorDiffers = operatorSshHost !== undefined ? operatorSshHost !== operatorAt(regId)?.operatorSshHost : noOperatorSsh && operatorAt(regId) !== undefined;
|
|
1699
|
+
if (!replace && operatorDiffers) detail = [detail, "operator route differs from the registration and was not changed (--replace rewrites it)"].filter(Boolean).join("; ");
|
|
1700
|
+
const operatorChanged = replace && operatorChange(regId);
|
|
1701
|
+
registration = joined(regId);
|
|
1702
|
+
if (!filled.length && !keyed && !operatorChanged) return { status: "ok", ...(detail ? { detail } : {}) };
|
|
1703
|
+
return { status: "done", detail: [detail, filled.length ? `recorded ${filled.join(", ")}` : null, keyed ? `recorded workspace ${reportedKey}` : null, operatorChanged ? "updated operator route" : null].filter(Boolean).join("; ") };
|
|
1572
1704
|
};
|
|
1573
|
-
const mine = servers[id];
|
|
1705
|
+
const mine = Object.hasOwn(servers, id) ? servers[id] : undefined;
|
|
1574
1706
|
if (mine && sameTarget(mine)) {
|
|
1575
1707
|
// A value given differs; one only found here differs only under --replace.
|
|
1576
1708
|
const differ = ["oatsPath", "path", "label"].filter((k) => entry[k] !== undefined && entry[k] !== mine[k] && (Object.hasOwn(given, k) || replace));
|
|
1577
1709
|
if (!differ.length || !replace) return keep(id, differ.length ? `${differ.join(", ")} differ from the registration and were not changed (--replace rewrites it)` : undefined);
|
|
1578
1710
|
if (mine.workspaceKey && !sameRepo(mine.workspaceKey, reportedKey)) throw workspaceMismatch(id, mine, reportedKey);
|
|
1579
|
-
servers[id] = entry; registration =
|
|
1711
|
+
servers[id] = entry; operatorChange(id); registration = joined(id);
|
|
1580
1712
|
return { status: "done", detail: `updated ${differ.join(", ")}` };
|
|
1581
1713
|
}
|
|
1582
1714
|
if (mine && !replace) throw Object.assign(serverError("E_SERVER_EXISTS", `server ${id} is already registered for ${mine.sshHost}:${mine.workspace} (pass --replace to point it at ${sshHost}:${abs}; instances spawned through it keep their saved routes)`), { details: { registered: { sshHost: mine.sshHost, workspace: mine.workspace } } });
|
|
1583
1715
|
const twin = Object.keys(servers).find((other) => other !== id && sameTarget(servers[other]));
|
|
1584
1716
|
if (twin && !mine) return keep(twin, `${sshHost}:${abs} is already registered as ${twin}; not registered again`);
|
|
1585
1717
|
validateServer(id, entry);
|
|
1586
|
-
|
|
1718
|
+
if (!mine) delete routes[id];
|
|
1719
|
+
servers[id] = entry; operatorChange(id); registration = joined(id);
|
|
1587
1720
|
return { status: "done", detail: [mine ? `replaced ${mine.sshHost}:${mine.workspace}` : `registered ${id}`, twin ? `${sshHost}:${abs} is also registered as ${twin}` : null].filter(Boolean).join("; ") };
|
|
1588
|
-
}));
|
|
1721
|
+
}, { operatorRoutes: true }));
|
|
1589
1722
|
|
|
1590
1723
|
// Every soul the host deployment lists (disabled ones skipped), one routed readiness each, within one
|
|
1591
1724
|
// budget; each failing or unknown required item is a human line, said once for the souls sharing it.
|
package/lib/session-end.mjs
CHANGED
|
@@ -7,6 +7,9 @@
|
|
|
7
7
|
* - `.oats-stop.json`: `oats instance stop` / `oats session stop`, a retire and a restart (before
|
|
8
8
|
* they end the session), `{ instance, at, stop, retained, startId }`; `startId` (0.69) is the
|
|
9
9
|
* start the stop ended, null when the home had none.
|
|
10
|
+
* A stop with the host (`oats host run` when the host stops: lib/host-run.mjs) writes the same
|
|
11
|
+
* record with `resume: true` and `reason: "host-stop"`, before it signals the harness: OATS
|
|
12
|
+
* ended that session only because the host was going down, and a resume brings it back.
|
|
10
13
|
* A reboot or a killed tmux server leaves neither file for the start that was running: that is a
|
|
11
14
|
* lost session, the one a resume brings back. Dependency-free: status, start --resume and the
|
|
12
15
|
* schedule runner read the files through here. */
|
|
@@ -16,6 +19,8 @@ import { SESSION_END } from "./session-resume-contract.mjs";
|
|
|
16
19
|
|
|
17
20
|
export const START_EXITED_FILE = ".oats-start-exited";
|
|
18
21
|
export const STOP_RECEIPT_FILE = ".oats-stop.json";
|
|
22
|
+
/** `reason` in the stop record of a stop with the host. */
|
|
23
|
+
export const HOST_STOP_REASON = "host-stop";
|
|
19
24
|
const RECORD_MAX = 64 * 1024;
|
|
20
25
|
/** A shell's exit status from 128 up is a signal's (128 + n): the harness was killed, not ended. */
|
|
21
26
|
const SIGNALLED = 128;
|
|
@@ -59,9 +64,21 @@ export function currentStartTime(meta) {
|
|
|
59
64
|
return Date.parse(typeof last === "string" ? last : meta?.createdAt);
|
|
60
65
|
}
|
|
61
66
|
|
|
67
|
+
/** The stop record of `home` when it is of `meta`'s current start (`meta.startId`), else null: it
|
|
68
|
+
* names this start (`startId` equal), or, a record without a start id, was written after this
|
|
69
|
+
* start began. */
|
|
70
|
+
function stopOfCurrentStart(home, meta) {
|
|
71
|
+
const startId = typeof meta?.startId === "string" && meta.startId ? meta.startId : null;
|
|
72
|
+
const stop = readStopReceipt(home);
|
|
73
|
+
if (!stop) return null;
|
|
74
|
+
if (typeof stop.startId === "string" && stop.startId) return stop.startId === startId ? stop : null;
|
|
75
|
+
return Date.parse(stop.at) > currentStartTime(meta) ? stop : null;
|
|
76
|
+
}
|
|
77
|
+
|
|
62
78
|
/** How the session of `meta`'s current start (`meta.startId`) ended, from the home's files alone:
|
|
63
|
-
* - `stopped`: the stop
|
|
64
|
-
*
|
|
79
|
+
* - `stopped`: the stop record is of this start (stopOfCurrentStart) and asks for no resume;
|
|
80
|
+
* - `lost`, whatever the exit marker says: the stop record is of this start and says
|
|
81
|
+
* `resume: true` (a stop with the host): the harness ended because the host did;
|
|
65
82
|
* - `exited`: the exit marker names this start with a status below 128, or in its legacy form
|
|
66
83
|
* (no status: a kernel before 0.69 wrote it only when the harness ended on its own window);
|
|
67
84
|
* - `lost`: anything else — a marker with a signal's status (128 and up), or no file naming this
|
|
@@ -71,13 +88,16 @@ export function currentStartTime(meta) {
|
|
|
71
88
|
* The caller reads it only for a launched home that does not run. */
|
|
72
89
|
export function sessionEndOf(home, meta) {
|
|
73
90
|
const startId = typeof meta?.startId === "string" && meta.startId ? meta.startId : null;
|
|
74
|
-
const
|
|
75
|
-
|
|
76
|
-
if (stop) {
|
|
77
|
-
if (typeof stop.startId === "string" && stop.startId) { if (stop.startId === startId) return SESSION_END.STOPPED; }
|
|
78
|
-
else if (Date.parse(stop.at) > since) return SESSION_END.STOPPED;
|
|
79
|
-
}
|
|
91
|
+
const stop = stopOfCurrentStart(home, meta);
|
|
92
|
+
if (stop) return stop.resume === true ? SESSION_END.LOST : SESSION_END.STOPPED;
|
|
80
93
|
const exited = startId ? readStartExited(home) : null;
|
|
81
94
|
if (exited && exited.startId === startId) return exited.status !== null && exited.status >= SIGNALLED ? SESSION_END.LOST : SESSION_END.EXITED;
|
|
82
95
|
return SESSION_END.LOST;
|
|
83
96
|
}
|
|
97
|
+
|
|
98
|
+
/** Why the session of `meta`'s current start reads `lost`, where the home says: HOST_STOP_REASON
|
|
99
|
+
* for a stop with the host; null for every other loss, and for a session that is not lost. */
|
|
100
|
+
export function lostReasonOf(home, meta) {
|
|
101
|
+
const stop = stopOfCurrentStart(home, meta);
|
|
102
|
+
return stop?.resume === true && stop.reason === HOST_STOP_REASON ? HOST_STOP_REASON : null;
|
|
103
|
+
}
|
package/lib/session-resume.mjs
CHANGED
|
@@ -15,9 +15,11 @@
|
|
|
15
15
|
* - Each home starts as a child process of this kernel (`session start --resume --home <home>
|
|
16
16
|
* --json`, its cwd the home's deployment), so each start resolves its own deployment's scope; at
|
|
17
17
|
* most RESUME_CONCURRENCY at once. A per-home failure is a row, never the sweep's failure.
|
|
18
|
-
* - One sweep per machine:
|
|
19
|
-
*
|
|
20
|
-
* taken
|
|
18
|
+
* - One sweep per machine: the lock `session-resume/lock` under OATS_HOME_DIR (default ~/.oats),
|
|
19
|
+
* a claim (lib/claim.mjs, through lib/dir-lock.mjs) that names its holder by pid, process start
|
|
20
|
+
* and boot id; a lock whose holder is gone, or that was taken in another boot, is taken over at
|
|
21
|
+
* once. A second sweep answers E_RESUME_BUSY; a lock that needs a person, E_RESUME_LOCK_STUCK,
|
|
22
|
+
* which `oats doctor` reports too (acquireResumeLock, resumeLockProblem).
|
|
21
23
|
* - No retry storm. A loss the sweep answered is judged (above) and not started again, whatever the
|
|
22
24
|
* boot; one note per boot (`note.json`) backs that: a refused resume is noted, and a later sweep
|
|
23
25
|
* in the same boot reports it again (`refused-this-boot`) without starting it, also when its
|
|
@@ -31,14 +33,15 @@
|
|
|
31
33
|
* The lock and the note are kernel-private, not a contract. `plan` predicts from the note, starting
|
|
32
34
|
* nothing, taking no lock and writing no note. */
|
|
33
35
|
import { execFile } from "node:child_process";
|
|
34
|
-
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync,
|
|
36
|
+
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
35
37
|
import { join, resolve } from "node:path";
|
|
36
38
|
import { oatsError } from "./errors.mjs";
|
|
37
39
|
import { deploymentState, readDeploymentsDocument } from "./deployments.mjs";
|
|
38
|
-
import {
|
|
39
|
-
import {
|
|
40
|
+
import { acquireDirLock, dirLockHeld, stuckLockRemedy } from "./dir-lock.mjs";
|
|
41
|
+
import { homeClaimHolder, listInstances, launchEnvRefs, missingLaunchEnvRefs, owedRetireSaid, resumeConversation } from "./core.mjs";
|
|
42
|
+
import { checkStartEnv, launchEnvChangedMessage, launchEnvMissingMessage, launchEnvUnverifiedWarning } from "./start-env.mjs";
|
|
40
43
|
import { FRESH_REASON, NOT_SWEPT, OUTCOME, RESUME_WAITING, ROW_REASON, SESSION_END } from "./session-resume-contract.mjs";
|
|
41
|
-
import { bootId, homeKey, judgeLosses, readSeen, resumeEligible, resumeStateDir } from "./session-seen.mjs";
|
|
44
|
+
import { bootId, homeKey, judgeLosses, readSeen, resumeEligible, resumeStateDir, takenInAnotherBoot } from "./session-seen.mjs";
|
|
42
45
|
import { isAgentScopeWarning } from "./tmux-server-scope.mjs";
|
|
43
46
|
|
|
44
47
|
export { bootId, resumeStateDir };
|
|
@@ -48,56 +51,54 @@ export const RESUME_SCHEMA = 1;
|
|
|
48
51
|
export const RESUME_CONCURRENCY = 4;
|
|
49
52
|
/** At most this many sweep starts of one home in one boot. */
|
|
50
53
|
export const RESUME_LIMIT = 3;
|
|
51
|
-
|
|
52
|
-
* the owner's write); older, it is a crashed acquirer's. */
|
|
53
|
-
const OWNERLESS_GRACE_MS = 60_000;
|
|
54
|
-
|
|
55
|
-
const lockDir = () => join(resumeStateDir(), "lock");
|
|
54
|
+
const lockPath = () => join(resumeStateDir(), "lock");
|
|
56
55
|
const notePath = () => join(resumeStateDir(), "note.json");
|
|
57
56
|
|
|
58
|
-
function pidAlive(pid) {
|
|
59
|
-
if (!Number.isSafeInteger(pid) || pid < 1) return false;
|
|
60
|
-
try { process.kill(pid, 0); return true; } catch (e) { return e.code === "EPERM"; }
|
|
61
|
-
}
|
|
62
57
|
const readJson = (path) => { try { return JSON.parse(readFileSync(path, "utf8")); } catch { return undefined; } };
|
|
63
58
|
|
|
64
|
-
/**
|
|
65
|
-
*
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
59
|
+
/** A sweep's lock `path` that needs a person, found as lib/dir-lock.mjs says it (`why`, `found`) →
|
|
60
|
+
* what is said of it and what to do. Exported for its test. */
|
|
61
|
+
export function resumeLockStuck(path, why, found) {
|
|
62
|
+
return {
|
|
63
|
+
lock: path, found,
|
|
64
|
+
said: `the session resume lock ${path} cannot be taken (${why})`,
|
|
65
|
+
remedy: stuckLockRemedy(path, found, "oats session resume"),
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Take the machine's resume lock → its release function. The lock is a claim (lib/dir-lock.mjs
|
|
70
|
+
* acquireDirLock) whose record also names the boot it was taken in.
|
|
71
|
+
* - A sweep that runs holds it: E_RESUME_BUSY, at once.
|
|
72
|
+
* - A holder that is gone (exited, killed, its pid another process's) is taken over at once. So
|
|
73
|
+
* is one that took the lock in another boot (takenInAnotherBoot).
|
|
74
|
+
* - A lock that needs a person is E_RESUME_LOCK_STUCK, with what was found and what to do, its
|
|
75
|
+
* `details` `{ lock, found }` (`found`: lib/dir-lock.mjs), and is left as it is: a holder
|
|
76
|
+
* whose liveness cannot be read, a record that does not parse, a gone holder's record that
|
|
77
|
+
* cannot be taken over, a lock taken again each time it was found free, this process's own
|
|
78
|
+
* start unreadable, and a directory at the lock's path, the lock of an earlier kernel, which
|
|
79
|
+
* is never judged and never removed. */
|
|
80
|
+
export function acquireResumeLock({ boot = bootId() } = {}) {
|
|
81
|
+
const path = lockPath();
|
|
70
82
|
mkdirSync(resumeStateDir(), { recursive: true, mode: 0o700 });
|
|
71
|
-
const busy = (why
|
|
72
|
-
|
|
73
|
-
const
|
|
74
|
-
|
|
75
|
-
if (boot && owner.bootId && owner.bootId !== boot) return `taken in another boot by pid ${owner.pid}`;
|
|
76
|
-
return pidAlive(owner.pid) ? null : `its owner pid ${owner.pid} is gone`;
|
|
83
|
+
const busy = (why, { found }) => {
|
|
84
|
+
if (found === "running") return oatsError("E_RESUME_BUSY", `another session resume on this machine holds ${path} (${why}); nothing was started`);
|
|
85
|
+
const stuck = resumeLockStuck(path, why, found);
|
|
86
|
+
return Object.assign(oatsError("E_RESUME_LOCK_STUCK", `${stuck.said}; ${stuck.remedy}; nothing was started`), { details: { lock: path, found } });
|
|
77
87
|
};
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
try {
|
|
93
|
-
// Read again under the takeover: another sweep may have taken it meanwhile.
|
|
94
|
-
if (existsSync(dir) && !stale()) throw busy("another sweep took it over");
|
|
95
|
-
const aside = `${dir}.stale-${process.pid}-${now()}`;
|
|
96
|
-
try { renameSync(dir, aside); rmSync(aside, { recursive: true, force: true }); } catch (r) { if (r.code !== "ENOENT") throw r; }
|
|
97
|
-
take();
|
|
98
|
-
} finally { rmSync(takeover, { recursive: true, force: true }); }
|
|
99
|
-
}
|
|
100
|
-
return () => { try { if (readJson(join(dir, "owner.json"))?.pid === process.pid) rmSync(dir, { recursive: true, force: true }); } catch { /* best effort */ } };
|
|
88
|
+
return acquireDirLock(path, { bootId: boot }, { busy, holderGone: takenInAnotherBoot(boot) });
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export const RESUME_LOCK_STUCK = "resume-lock-stuck";
|
|
92
|
+
/** `oats doctor`'s `resume-lock-stuck` problem: the machine's resume lock when it needs a person
|
|
93
|
+
* → `{ code, message, remedy }`, or null. The lock is read where a sweep would take it
|
|
94
|
+
* (lib/dir-lock.mjs dirLockHeld, with the same boot rule), and nothing is taken or changed. Every
|
|
95
|
+
* state acquireResumeLock answers E_RESUME_LOCK_STUCK for a lock it finds is one; a free lock, a
|
|
96
|
+
* sweep that runs and a lock a gone sweep left, which the next sweep takes over, are not. */
|
|
97
|
+
export function resumeLockProblem({ boot = bootId() } = {}) {
|
|
98
|
+
const path = lockPath(), held = dirLockHeld(path, { holderGone: takenInAnotherBoot(boot) });
|
|
99
|
+
if (!held || held.found === "running") return null;
|
|
100
|
+
const stuck = resumeLockStuck(path, held.why, held.found);
|
|
101
|
+
return { code: RESUME_LOCK_STUCK, message: `${stuck.said}: oats session resume starts nothing on this machine while it is there`, remedy: stuck.remedy };
|
|
101
102
|
}
|
|
102
103
|
|
|
103
104
|
/** This boot's note: `{ bootId, refused: [{ home, startId, code, message, at }], resumes: { <home>: n } }`,
|
|
@@ -208,16 +209,21 @@ function startEnvironment(env) {
|
|
|
208
209
|
for (const k of ["OATS_INSTANCE", "OATS_INSTANCE_HOME", "OATS_HOME", "PI_AGENT_INSTANCE", "PI_AGENT_HOME"]) delete out[k];
|
|
209
210
|
return out;
|
|
210
211
|
}
|
|
211
|
-
|
|
212
|
+
/** Run this kernel (`execPath` `bin`) with `argv` as a child process → its JSON answer: the last
|
|
213
|
+
* line of its stdout. A child that printed none is answered as a failure envelope, `code`, that
|
|
214
|
+
* says how `what` exited. The child gets `env` without an instance identity (startEnvironment).
|
|
215
|
+
* Never rejects. */
|
|
216
|
+
export function kernelAnswer({ execPath, bin, env }, argv, { cwd, code, what }) {
|
|
212
217
|
return new Promise((done) => {
|
|
213
|
-
execFile(execPath, [bin,
|
|
218
|
+
execFile(execPath, [bin, ...argv], { cwd, env: startEnvironment(env), encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }, (error, stdout) => {
|
|
214
219
|
let answer;
|
|
215
220
|
try { answer = JSON.parse(String(stdout).trim().split("\n").pop()); } catch { answer = undefined; }
|
|
216
|
-
if (!answer || typeof answer !== "object") answer = { ok: false, error: { code
|
|
221
|
+
if (!answer || typeof answer !== "object") answer = { ok: false, error: { code, message: `${what} exited${error?.code !== undefined ? ` with status ${error.code}` : ""}${error?.signal ? ` on ${error.signal}` : ""} without an answer` } };
|
|
217
222
|
done(answer);
|
|
218
223
|
});
|
|
219
224
|
});
|
|
220
225
|
}
|
|
226
|
+
const runStart = (kernel, row) => kernelAnswer(kernel, ["session", "start", "--resume", "--home", row.home, "--json"], { cwd: row.deployment, code: "E_SESSION_START_FAILED", what: "the start" });
|
|
221
227
|
/** `fn` over `items`, at most `limit` at once; once `stop()` holds no item is taken any more, and the
|
|
222
228
|
* answer waits for every one already taken. */
|
|
223
229
|
async function inBatches(items, limit, fn, stop = () => false) {
|
|
@@ -329,8 +335,9 @@ function predicted(row, env) {
|
|
|
329
335
|
const refused = (code, message) => ({ ...base(row), outcome: OUTCOME.REFUSED, reason: ROW_REASON.START_REFUSED, code, message });
|
|
330
336
|
if (!(row.launch && typeof row.launch === "object")) return refused("E_LAUNCH_LEGACY", `${row.instance} records no launch recipe (an earlier kernel spawned it): re-spawn it from the deployment`);
|
|
331
337
|
const missing = missingLaunchEnvRefs(row.launch.env, env);
|
|
332
|
-
if (missing.length) return refused("E_LAUNCH_ENV_MISSING", launchEnvMissingMessage(missing));
|
|
333
|
-
const
|
|
338
|
+
if (missing.length) return refused("E_LAUNCH_ENV_MISSING", launchEnvMissingMessage(missing, row.launch.harness));
|
|
339
|
+
const refs = launchEnvRefs(row.launch, env);
|
|
340
|
+
const fit = checkStartEnv(row.home, row.startId ?? null, refs.map((r) => r.source), Object.fromEntries(refs.map((r) => [r.source, r.value])));
|
|
334
341
|
if (fit.changed) return refused("E_LAUNCH_ENV_CHANGED", launchEnvChangedMessage(fit.changed));
|
|
335
342
|
const extra = fit.unverified ? { warnings: [launchEnvUnverifiedWarning(row.instance)] } : {};
|
|
336
343
|
const found = resumeConversation(row.home, row);
|