@awebai/oats 0.69.14 → 0.69.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/oats.mjs +79 -18
- package/docs/capabilities.md +54 -12
- package/docs/container.md +57 -12
- package/docs/desktop-cli-api.md +215 -19
- package/docs/execution-targets.md +15 -3
- package/docs/first-team.md +4 -3
- package/docs/implementation.md +18 -8
- package/docs/release-notes/v0.69.15.md +444 -0
- package/docs/schedules.md +133 -2
- package/docs/servers.md +60 -2
- package/docs/souls-and-instances.md +87 -7
- package/docs/upgrading.md +17 -5
- package/lib/claim.mjs +2 -3
- package/lib/command-flags.mjs +4 -2
- package/lib/core.mjs +215 -72
- package/lib/dir-lock.mjs +17 -0
- package/lib/errors.mjs +2 -1
- package/lib/host-run.mjs +218 -0
- package/lib/host-stop-child.mjs +21 -0
- package/lib/instance-git.mjs +32 -6
- package/lib/instance-lifecycle.mjs +4 -3
- package/lib/instance-listing.mjs +35 -6
- package/lib/kernel-features.mjs +4 -1
- package/lib/read-call.mjs +1 -1
- package/lib/remote.mjs +17 -14
- package/lib/schedule.mjs +237 -20
- package/lib/servers.mjs +83 -5
- package/lib/session-end.mjs +28 -8
- package/lib/session-resume.mjs +10 -18
- package/lib/session-seen.mjs +4 -64
- package/lib/status-words.mjs +11 -4
- package/lib/system-tools.mjs +1 -36
- package/lib/worktree-hooks.mjs +6 -175
- package/lib/worktree.mjs +19 -10
- package/package.json +4 -1
- package/packages/record/README.md +56 -21
- package/packages/record/docs/capture-service.md +8 -5
- package/packages/record/docs/turn-record-sot.md +9 -0
- package/packages/record/lib/capture-health.mjs +6 -4
- package/packages/record/lib/capture-lock.mjs +31 -27
- package/packages/record/lib/capture-retained-scratch.mjs +4 -3
- package/packages/record/lib/capture-scratch.mjs +2 -1
- package/packages/record/lib/derived-state.mjs +4 -2
- package/packages/record/lib/index-db.mjs +4 -2
- package/packages/record/lib/lock-holder.mjs +21 -0
- package/packages/record/lib/private-files.mjs +18 -0
- package/packages/record/lib/process-identity.mjs +310 -0
- package/packages/record/lib/store.mjs +31 -25
- package/skills/oats-getting-started/SKILL.md +1 -1
package/lib/schedule.mjs
CHANGED
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
* `unknown`, its job blocked until `reconcile` proves what happened; it keeps
|
|
23
23
|
* its host slot too, unless it is a command whose process exit was observed. */
|
|
24
24
|
import { runScheduleCommand } from "./schedule-command.mjs";
|
|
25
|
-
import { closeSync, constants as fsConstants, existsSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
|
25
|
+
import { closeSync, constants as fsConstants, existsSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
26
26
|
import { createHash } from "node:crypto";
|
|
27
27
|
import { homedir } from "node:os";
|
|
28
28
|
import { basename, dirname, isAbsolute, join, resolve, sep } from "node:path";
|
|
@@ -295,11 +295,29 @@ function withRegistryLock(fn) { return withDirLock(join(hostScheduleDir(), "regi
|
|
|
295
295
|
/** Definitions and state are read-modify-write; CRUD serializes here briefly. */
|
|
296
296
|
export function withScopeLock(ws, fn) { return withDirLock(join(stateDir(ws), "scope.lock"), `schedules of ${ws}`, fn, { retryMs: 3000 }); }
|
|
297
297
|
|
|
298
|
+
/** A job's slot: the directory `locks/<id>` and its record `owner.json`, which stands for the run
|
|
299
|
+
* that holds it until that run's instance ends (no process mutex: every taker and releaser holds
|
|
300
|
+
* the host lock, withHostLock). The slot is never there without its record. Its record is written
|
|
301
|
+
* in a directory beside the slots (slotStaging), which is then renamed to the slot; a release
|
|
302
|
+
* renames the slot back there before removing it. A taker or releaser killed at any step leaves
|
|
303
|
+
* either the slot with its record or no slot, and at most that directory, which the next taker or
|
|
304
|
+
* releaser of the slot removes. An entry already at the slot is held, whatever it is, and left as
|
|
305
|
+
* it is: a directory without its record (an earlier kernel made the directory first) reads as
|
|
306
|
+
* unreadable (jobLockInfo). */
|
|
307
|
+
const slotStaging = (ws, id) => join(stateDir(ws), `.slot-${id}`);
|
|
298
308
|
export function acquireJobLock(ws, id, owner) {
|
|
299
|
-
const dir = lockDir(ws, id);
|
|
309
|
+
const dir = lockDir(ws, id), staging = slotStaging(ws, id);
|
|
300
310
|
mkdirSync(dirname(dir), { recursive: true });
|
|
301
|
-
try {
|
|
302
|
-
|
|
311
|
+
try { lstatSync(dir); return false; } catch (e) { if (e.code !== "ENOENT") throw e; }
|
|
312
|
+
rmSync(staging, { recursive: true, force: true });
|
|
313
|
+
mkdirSync(staging);
|
|
314
|
+
try {
|
|
315
|
+
writeFileSync(join(staging, "owner.json"), JSON.stringify({ ...owner, pid: process.pid, at: new Date().toISOString() }, null, 2) + "\n");
|
|
316
|
+
renameSync(staging, dir);
|
|
317
|
+
} catch (e) {
|
|
318
|
+
try { rmSync(staging, { recursive: true, force: true }); } catch { /* the next taker of the slot removes it */ }
|
|
319
|
+
throw e;
|
|
320
|
+
}
|
|
303
321
|
return true;
|
|
304
322
|
}
|
|
305
323
|
export function jobLockInfo(ws, id) {
|
|
@@ -307,7 +325,112 @@ export function jobLockInfo(ws, id) {
|
|
|
307
325
|
if (!existsSync(dir)) return null;
|
|
308
326
|
try { return JSON.parse(readFileSync(join(dir, "owner.json"), "utf8")); } catch { return { unreadable: true }; }
|
|
309
327
|
}
|
|
310
|
-
|
|
328
|
+
/** `oats doctor`'s problem for a job's slot whose record cannot be read. */
|
|
329
|
+
export const SCHEDULE_SLOT_UNREADABLE = "schedule-slot-unreadable";
|
|
330
|
+
/** A schedule's id as a person types it, from the key its state and slot are stored under: a
|
|
331
|
+
* workspace schedule's key is <member>~<id> (scheduleStateKey), a local one is its id. */
|
|
332
|
+
const shownScheduleId = (key) => key.replace("~", "/");
|
|
333
|
+
/** The slot of the job `key` in `ws` when its record cannot be read (jobLockInfo: a directory
|
|
334
|
+
* without its record, a record that does not parse, an entry that is no directory) → `{ said,
|
|
335
|
+
* remedy }`, the words the tick, reconcile, run-now and doctor give for it; else null. Nobody
|
|
336
|
+
* can tell from such a slot whether the run it stands for still runs, so it is held and left as
|
|
337
|
+
* it is until a person has checked and reconcile --clear releases it. `dir` is the `--dir`
|
|
338
|
+
* argument the commands of the remedy need from another deployment. */
|
|
339
|
+
function unreadableSlot(ws, key, dir = "") {
|
|
340
|
+
if (!jobLockInfo(ws, key)?.unreadable) return null;
|
|
341
|
+
const id = shownScheduleId(key);
|
|
342
|
+
return { said: `the slot ${lockDir(ws, key)} has no readable record`, remedy: `check oats schedule show ${id}${dir}; once its last run is not running, run oats schedule reconcile ${id} --clear${dir} to release the slot` };
|
|
343
|
+
}
|
|
344
|
+
/** `oats doctor`'s problem for a slot whose schedule is not defined. */
|
|
345
|
+
export const SCHEDULE_SLOT_ORPHANED = "schedule-slot-orphaned";
|
|
346
|
+
const ISO_INSTANT = /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}Z$/;
|
|
347
|
+
/** The slot of `key` in `ws` when its schedule is gone (scheduleAbsence), whatever its record →
|
|
348
|
+
* `{ found, said, remedy }`, the words reconcile and doctor give for it; else null (no slot). No
|
|
349
|
+
* tick reads such a slot, and every tick counts it against maxConcurrent (liveLocks): it may stand
|
|
350
|
+
* for a run of a schedule removed while it ran, so it is held until a person has checked and
|
|
351
|
+
* reconcile --clear releases it. `found` says what the record names: the run's due minute, and
|
|
352
|
+
* the home of its instance when the record names one (a wake's); a record that cannot be read
|
|
353
|
+
* names nothing. `absent` says how the schedule is known to be gone. Only what has the shape the
|
|
354
|
+
* kernel writes is shown. */
|
|
355
|
+
function orphanSlot(ws, key, dir = "", absent = "is not defined") {
|
|
356
|
+
const record = jobLockInfo(ws, key);
|
|
357
|
+
if (!record) return null;
|
|
358
|
+
const id = shownScheduleId(key), release = `run oats schedule reconcile ${id} --clear${dir} to release the slot`;
|
|
359
|
+
const unnamed = `once no run of schedule ${id} is running (oats status lists the instances it started), ${release}`;
|
|
360
|
+
let found, remedy = unnamed;
|
|
361
|
+
if (record.unreadable) found = "has no readable record";
|
|
362
|
+
else {
|
|
363
|
+
const due = typeof record.scheduledFor === "string" && ISO_INSTANT.test(record.scheduledFor) ? record.scheduledFor : "?";
|
|
364
|
+
const home = typeof record.home === "string" && isAbsolute(record.home) && !/[\x00-\x1f\x7f]/.test(record.home) ? record.home : null;
|
|
365
|
+
found = `holds the run due ${due} of ${home ? `the instance at ${home}` : "an instance its record does not name"}`;
|
|
366
|
+
if (home) remedy = `once oats status shows the instance at ${home} is not running, ${release}`;
|
|
367
|
+
}
|
|
368
|
+
return { found, said: `the slot ${lockDir(ws, key)} of schedule ${id}, which ${absent}, ${found}`, remedy };
|
|
369
|
+
}
|
|
370
|
+
/** A path a snapshot names that may be shown as it is. */
|
|
371
|
+
const SHOWN_PATH = /^[A-Za-z0-9._@+/-]+$/;
|
|
372
|
+
/** Whether the schedule whose slot or state is `key` is defined, from what this host has already
|
|
373
|
+
* read and nothing more (no discovery, no network): `defs` (readDefinitions) for a local schedule,
|
|
374
|
+
* and for a workspace one (<member>~<id>) the saved snapshot `snapshot` (readSnapshot) →
|
|
375
|
+
* { state: "defined" } it is defined, or nothing read says otherwise (a workspace one
|
|
376
|
+
* whose snapshot cannot be read)
|
|
377
|
+
* { state: "gone", absent, takenAt? }
|
|
378
|
+
* local: not in oats-schedules.json. Workspace: as of the snapshot's
|
|
379
|
+
* `takenAt`, its member was read at a commit, the snapshot has no
|
|
380
|
+
* problem of that member that could hide a schedule, and no
|
|
381
|
+
* schedule has its id. A workspace schedule that is disabled,
|
|
382
|
+
* untrusted or placed elsewhere still has its entry, so it is
|
|
383
|
+
* defined; `absent` is the clause that says how it is known gone
|
|
384
|
+
* { state: "member-absent", takenAt, member }
|
|
385
|
+
* its member is not in the snapshot at a commit: removed from the
|
|
386
|
+
* workspace, or not read (an unconfirmed member is skipped silently)
|
|
387
|
+
* { state: "member-problem", takenAt, member, path }
|
|
388
|
+
* a problem of its member that has no listed schedule beside it: a
|
|
389
|
+
* schedule file that could not be read or parsed (`path`, which
|
|
390
|
+
* could hold this id: a file's id may be other than its name), or
|
|
391
|
+
* a member that could not be listed (`path` null) */
|
|
392
|
+
function scheduleAbsence(key, { defs, snapshot }) {
|
|
393
|
+
if (!key.includes("~")) return defs.jobs[key] ? { state: "defined" } : { state: "gone", absent: "is not defined" };
|
|
394
|
+
if (!snapshot) return { state: "defined" };
|
|
395
|
+
const id = shownScheduleId(key), member = key.slice(0, key.indexOf("~"));
|
|
396
|
+
const schedules = snapshot.schedules;
|
|
397
|
+
if (schedules.some((e) => e?.id === id)) return { state: "defined" };
|
|
398
|
+
const takenAt = typeof snapshot.takenAt === "string" && ISO_INSTANT.test(snapshot.takenAt) ? snapshot.takenAt : "an unknown time";
|
|
399
|
+
const members = snapshot.members && typeof snapshot.members === "object" ? snapshot.members : {};
|
|
400
|
+
const memberKey = Object.keys(members).find((k) => members[k]?.name === member && typeof members[k]?.commit === "string" && members[k].commit !== "");
|
|
401
|
+
if (!memberKey) return { state: "member-absent", takenAt, member };
|
|
402
|
+
const file = (p) => (typeof p?.path === "string" ? p.path.split("#")[0] : "");
|
|
403
|
+
const listed = (p) => schedules.some((e) => e?.origin?.repoKey === p.repoKey && e.origin.path === file(p));
|
|
404
|
+
const hiding = (Array.isArray(snapshot.problems) ? snapshot.problems : []).find((p) => p && (p.kind === undefined || p.kind === "schedule")
|
|
405
|
+
&& (p.repoKey === memberKey || (file(p) === "" && !Object.hasOwn(members, p.repoKey))) && !listed(p));
|
|
406
|
+
if (hiding) return { state: "member-problem", takenAt, member, path: file(hiding) === "" ? null : SHOWN_PATH.test(file(hiding)) ? file(hiding) : "a schedule file" };
|
|
407
|
+
return { state: "gone", takenAt, absent: `as of ${takenAt} is not in the workspace (run oats automations refresh if that is stale)` };
|
|
408
|
+
}
|
|
409
|
+
/** The attempt reconcile resolves for the job state `js`: its persisted attempt, or the unknown
|
|
410
|
+
* last run an earlier kernel kept in its place. */
|
|
411
|
+
const attemptOf = (js) => js.attempt || (js.lastRun?.outcome === "unknown" ? { scheduledFor: js.lastRun.scheduledFor, startedAt: js.lastRun.startedAt, ...(js.lastRun.execution !== undefined ? { execution: js.lastRun.execution } : {}) } : undefined);
|
|
412
|
+
/** The last run of `js` once reconcile resolves its `attempt` as launching nothing: cleared by the
|
|
413
|
+
* operator (`clear`), or no instance found. */
|
|
414
|
+
/** The retained state `js` of the job `key` whose schedule is gone and that holds no slot →
|
|
415
|
+
* `{ said, remedy }`, the words reconcile and doctor give for it: what is retained (a launch
|
|
416
|
+
* attempt with no recorded result, when there is one), and the reconcile --clear that clears it. */
|
|
417
|
+
function orphanState(key, js, absent, dir = "") {
|
|
418
|
+
const id = shownScheduleId(key), attempt = attemptOf(js);
|
|
419
|
+
const due = typeof attempt?.scheduledFor === "string" && ISO_INSTANT.test(attempt.scheduledFor) ? attempt.scheduledFor : "?";
|
|
420
|
+
return { said: `schedule ${id}, which ${absent}, has retained state${attempt ? ` with a launch attempt for ${due} that has no recorded result` : ""}`,
|
|
421
|
+
remedy: `once no run of schedule ${id} is running (oats status lists the instances it started), run oats schedule reconcile ${id} --clear${dir} to clear its state` };
|
|
422
|
+
}
|
|
423
|
+
const launchFailedRun = (js, attempt, kind, clear) => ({ scheduledFor: attempt.scheduledFor, startedAt: js.lastRun?.startedAt || attempt.startedAt, kind, launched: false, outcome: "launch-failed", error: clear ? "cleared by the operator after checking by hand" : "no instance found in the scope's roster for the unrecorded attempt" });
|
|
424
|
+
/** Whether `key` names an entry directly among a scope's slots (lockDir), and nothing beyond them. */
|
|
425
|
+
const slotName = (key) => typeof key === "string" && key !== "" && key !== "." && key !== ".." && !/[/\\\0]/.test(key);
|
|
426
|
+
export function releaseJobLock(ws, id) {
|
|
427
|
+
const staging = slotStaging(ws, id);
|
|
428
|
+
rmSync(staging, { recursive: true, force: true });
|
|
429
|
+
try { renameSync(lockDir(ws, id), staging); } catch (e) { if (e.code === "ENOENT") return; throw e; }
|
|
430
|
+
// The slot is released once it is renamed: what could not be removed of it is removed by the
|
|
431
|
+
// next taker or releaser of the slot.
|
|
432
|
+
try { rmSync(staging, { recursive: true, force: true }); } catch { /* removed next time */ }
|
|
433
|
+
}
|
|
311
434
|
function liveLocks(wsList) {
|
|
312
435
|
let n = 0;
|
|
313
436
|
for (const ws of wsList) {
|
|
@@ -907,7 +1030,8 @@ export function tickWorkspace(ws, { now = new Date(), firedAt = now, io, reg, ws
|
|
|
907
1030
|
// `candidates` (the host tick) means the host already observed every scope.
|
|
908
1031
|
// Triggers (kind "trigger") live in the same file and are evaluated by tickTriggers, not here.
|
|
909
1032
|
for (const [id, def] of Object.entries(defs.jobs)) if (def && def.kind === "trigger") delete defs.jobs[id];
|
|
910
|
-
|
|
1033
|
+
// A job whose slot cannot be read is not observed: it is reported invalid below, on its own.
|
|
1034
|
+
if (!candidates) for (const [id, def] of Object.entries(defs.jobs)) if (!unreadableSlot(ws, id)) observePrevious(ws, id, def, stateOf(id), io, now, { mutate: !dryRun });
|
|
911
1035
|
if (observeOnly) { if (!dryRun) writeState(ws, st); return considered; }
|
|
912
1036
|
// Least recently LAUNCHED first: only an actual harness launch counts, a
|
|
913
1037
|
// skipped or pending job keeps its place at the front of the line.
|
|
@@ -918,6 +1042,8 @@ export function tickWorkspace(ws, { now = new Date(), firedAt = now, io, reg, ws
|
|
|
918
1042
|
let due, cronError;
|
|
919
1043
|
const invalidShape = shapeError(def), herdr = invalidShape ? null : storedHerdrBackend(ws, id, def);
|
|
920
1044
|
const shape = invalidShape ?? herdr?.message;
|
|
1045
|
+
const slot = shape ? null : unreadableSlot(ws, id);
|
|
1046
|
+
if (slot) { record(id, "invalid", { error: `${slot.said}; ${slot.remedy}`, errorCode: "E_SCHEDULE_INVALID" }); continue; }
|
|
921
1047
|
if (!shape) validateActiveExecution(js, jobLockInfo(ws, id)); // Also covers host pre-observed candidates.
|
|
922
1048
|
if (shape) { cronError = shape; due = false; }
|
|
923
1049
|
else { try { due = only ? true : dueAt(def, minute); } catch (e) { cronError = e.message; due = false; } }
|
|
@@ -1086,6 +1212,8 @@ export function runNow(ws, qid, { now = new Date(), io, force = false, ctx = nul
|
|
|
1086
1212
|
return withHostLock(() => {
|
|
1087
1213
|
const st = readState(ws);
|
|
1088
1214
|
const js = st.jobs[id] || (st.jobs[id] = {});
|
|
1215
|
+
const slot = unreadableSlot(ws, id);
|
|
1216
|
+
if (slot) throw scheduleError("E_SCHEDULE_INVALID", `schedule ${id}: ${slot.said}; ${slot.remedy}`);
|
|
1089
1217
|
observePrevious(ws, id, def, js, io, now, { mutate: true });
|
|
1090
1218
|
writeState(ws, st);
|
|
1091
1219
|
if (js.attempt) throw scheduleError("E_SCHEDULE_UNRESOLVED", `schedule ${id} has a launch attempt without a recorded result; run oats schedule reconcile ${id} first`);
|
|
@@ -1110,12 +1238,21 @@ export function reconcile(ws, qid, { io, now = new Date(), clear = false, ctx =
|
|
|
1110
1238
|
const id = scheduleKeyOf(qid);
|
|
1111
1239
|
const defs = effectiveDefinitions(ws, ctx);
|
|
1112
1240
|
const def = defs.jobs[id];
|
|
1241
|
+
if (!def && slotName(id) && (jobLockInfo(ws, id) || readState(ws).jobs[id]) && scheduleAbsence(id, { defs: readDefinitions(ws), snapshot: readSnapshot(ws) }).state === "gone") return reconcileOrphanSlot(ws, qid, id, { clear, ctx });
|
|
1113
1242
|
if (!def || def.kind === "trigger") throw notHere(ws, qid, ctx);
|
|
1114
1243
|
const describeHere = () => describe(ws, qid, io, { ctx });
|
|
1115
1244
|
return withHostLock(() => {
|
|
1116
1245
|
const st = readState(ws);
|
|
1117
1246
|
const js = st.jobs[id] || (st.jobs[id] = {});
|
|
1118
|
-
const attempt =
|
|
1247
|
+
const attempt = attemptOf(js);
|
|
1248
|
+
// A slot whose record cannot be read is released only once a person has checked (--clear);
|
|
1249
|
+
// an attempt beside it is then reconciled as any other.
|
|
1250
|
+
const slot = unreadableSlot(ws, id);
|
|
1251
|
+
if (slot && !clear) return { schedule: describeHere(), reconciled: "unknown", remedy: `${slot.said}; ${slot.remedy}` };
|
|
1252
|
+
if (slot) {
|
|
1253
|
+
releaseJobLock(ws, id);
|
|
1254
|
+
if (!attempt) return { schedule: describeHere(), reconciled: "cleared" };
|
|
1255
|
+
}
|
|
1119
1256
|
if (!attempt) return { schedule: describeHere(), reconciled: "nothing to reconcile" };
|
|
1120
1257
|
try { validateAttemptExecution(attempt, jobLockInfo(ws, id)); }
|
|
1121
1258
|
catch (error) { throw scheduleError("E_SCHEDULE_INVALID", error.message); }
|
|
@@ -1146,7 +1283,7 @@ export function reconcile(ws, qid, { io, now = new Date(), clear = false, ctx =
|
|
|
1146
1283
|
js.lastRun = { scheduledFor: attempt.scheduledFor, startedAt: js.lastRun?.startedAt || attempt.startedAt, kind: attemptKind, launched: true, ...adopted, outcome: seen.outcome, ...(seen.error ? { error: seen.error } : {}), ...(seen.note ? { note: seen.note } : {}) };
|
|
1147
1284
|
if (seen.outcome !== "ended") { if (!jobLockInfo(ws, id)) acquireJobLock(ws, id, { scheduledFor: attempt.scheduledFor, reconciled: true }); } else releaseJobLock(ws, id);
|
|
1148
1285
|
} else {
|
|
1149
|
-
js.lastRun =
|
|
1286
|
+
js.lastRun = launchFailedRun(js, attempt, attemptKind, clear);
|
|
1150
1287
|
releaseJobLock(ws, id);
|
|
1151
1288
|
}
|
|
1152
1289
|
writeState(ws, st);
|
|
@@ -1154,6 +1291,37 @@ export function reconcile(ws, qid, { io, now = new Date(), clear = false, ctx =
|
|
|
1154
1291
|
});
|
|
1155
1292
|
}
|
|
1156
1293
|
|
|
1294
|
+
/** Reconcile what a schedule that is gone (scheduleAbsence) left: its slot (orphanSlot), its
|
|
1295
|
+
* retained state (orphanState), or both, read again under the host lock, as every reconcile.
|
|
1296
|
+
* Without `clear` it answers what it found and changes nothing. With `clear` it releases the slot,
|
|
1297
|
+
* resolves the job's retained attempt as cleared (as for a defined job: launchFailedRun) and
|
|
1298
|
+
* removes the job's state, and says which of the three it cleared (`cleared`). `schedule` is
|
|
1299
|
+
* null: there is no schedule to describe. */
|
|
1300
|
+
function reconcileOrphanSlot(ws, qid, id, { clear, ctx }) {
|
|
1301
|
+
return withHostLock(() => {
|
|
1302
|
+
const absence = scheduleAbsence(id, { defs: readDefinitions(ws), snapshot: readSnapshot(ws) });
|
|
1303
|
+
const st = readState(ws), js = st.jobs[id];
|
|
1304
|
+
const slot = absence.state === "gone" ? orphanSlot(ws, id, "", absence.absent) : null;
|
|
1305
|
+
const words = slot ?? (absence.state === "gone" && js ? orphanState(id, js, absence.absent) : null);
|
|
1306
|
+
if (!words) throw notHere(ws, qid, ctx);
|
|
1307
|
+
if (!clear) return { schedule: null, reconciled: "unknown", remedy: `${words.said}; ${words.remedy}` };
|
|
1308
|
+
const attempt = js ? attemptOf(js) : undefined;
|
|
1309
|
+
if (attempt) {
|
|
1310
|
+
try { validateAttemptExecution(attempt, jobLockInfo(ws, id)); }
|
|
1311
|
+
catch (error) { throw scheduleError("E_SCHEDULE_INVALID", error.message); }
|
|
1312
|
+
}
|
|
1313
|
+
const cleared = [];
|
|
1314
|
+
if (slot) { releaseJobLock(ws, id); cleared.push("slot"); }
|
|
1315
|
+
if (js) {
|
|
1316
|
+
if (attempt) { delete js.attempt; js.lastRun = launchFailedRun(js, attempt, js.lastRun?.kind ?? null, true); cleared.push("attempt"); }
|
|
1317
|
+
delete st.jobs[id];
|
|
1318
|
+
writeState(ws, st);
|
|
1319
|
+
cleared.push("state");
|
|
1320
|
+
}
|
|
1321
|
+
return { schedule: null, reconciled: "cleared", cleared };
|
|
1322
|
+
});
|
|
1323
|
+
}
|
|
1324
|
+
|
|
1157
1325
|
/** Whether a spawn job's runs are named `<agent>-<purpose>`, so a home's absence proves a lost
|
|
1158
1326
|
* receipt launched nothing: only for a bare name of a member soul under the agents root, whose
|
|
1159
1327
|
* agent name is that name. Spawn names a run after the soul's stem (core.mjs
|
|
@@ -1175,6 +1343,17 @@ function namedHomes(ws, js) {
|
|
|
1175
1343
|
* authorization. In particular, do not call placementOf here: it probes gh. */
|
|
1176
1344
|
function workspaceReconcileRemedy(id, snapshot, local, localError) {
|
|
1177
1345
|
const preserve = "Preserve the retained state and inspect its effects with the deployment owner; this diagnostic does not clear attempts or slots.";
|
|
1346
|
+
const { excluded, unknown } = workspacePlacement(id, snapshot, local, localError);
|
|
1347
|
+
if (excluded) return `Retained-state reconciliation is currently unavailable under this placement: ${excluded}. ${preserve}`;
|
|
1348
|
+
if (unknown) return `Workspace authorization cannot be verified offline: ${unknown}. ${preserve}`;
|
|
1349
|
+
return `Current workspace authorization is unverified offline; reconciliation requires current host, owner and trust authorization. ${preserve}`;
|
|
1350
|
+
}
|
|
1351
|
+
/** Whether this host runs the workspace schedule `id`, as far as it can tell offline from the saved
|
|
1352
|
+
* `snapshot` and its own oats-local.yaml (`local`, or `localError`), without asking gh who it is
|
|
1353
|
+
* → { excluded }: it does not (the clause says why: disabled, not named, placed on another host,
|
|
1354
|
+
* untrusted, not in the snapshot); { unknown }: that cannot be read (the clause says why); or {}:
|
|
1355
|
+
* nothing offline excludes it, and reconcile still checks its owner. */
|
|
1356
|
+
function workspacePlacement(id, snapshot, local, localError) {
|
|
1178
1357
|
let unknown, excluded;
|
|
1179
1358
|
const entries = snapshot?.schedules;
|
|
1180
1359
|
const matches = entries?.filter((entry) => entry?.id === id) || [];
|
|
@@ -1192,14 +1371,13 @@ function workspaceReconcileRemedy(id, snapshot, local, localError) {
|
|
|
1192
1371
|
else if (entry.runsOn !== host.name) excluded = `the saved schedule is assigned elsewhere (${entry.runsOn}); this host is ${host.name}`;
|
|
1193
1372
|
else if (!trusted(host, id)) excluded = "the schedule is untrusted by local automations.trust";
|
|
1194
1373
|
}
|
|
1195
|
-
|
|
1196
|
-
if (unknown) return `Workspace authorization cannot be verified offline: ${unknown}. ${preserve}`;
|
|
1197
|
-
return `Current workspace authorization is unverified offline; reconciliation requires current host, owner and trust authorization. ${preserve}`;
|
|
1374
|
+
return { excluded, unknown };
|
|
1198
1375
|
}
|
|
1199
1376
|
|
|
1200
|
-
/** `oats doctor`: retained launch attempts and unknown last observations,
|
|
1201
|
-
*
|
|
1202
|
-
*
|
|
1377
|
+
/** `oats doctor`: retained launch attempts and unknown last observations, and the slots whose
|
|
1378
|
+
* record cannot be read (SCHEDULE_SLOT_UNREADABLE, unreadableSlot), in this deployment and the
|
|
1379
|
+
* other scopes sharing its slots. Reads saved state only; workspace guidance cannot promise
|
|
1380
|
+
* authorization or an executable recovery. */
|
|
1203
1381
|
export function unresolvedScheduleAttempts(dep, { now = new Date() } = {}) {
|
|
1204
1382
|
const problems = [], information = [];
|
|
1205
1383
|
const here = resolve(dep);
|
|
@@ -1207,9 +1385,10 @@ export function unresolvedScheduleAttempts(dep, { now = new Date() } = {}) {
|
|
|
1207
1385
|
try { scopes = [here, ...readRegistry().workspaces.filter((w) => resolve(w) !== here && existsSync(w))]; }
|
|
1208
1386
|
catch (e) { information.push(`the host schedule registry could not be read, so only this deployment's schedules were checked: ${e.message}`); }
|
|
1209
1387
|
for (const ws of scopes) {
|
|
1210
|
-
|
|
1388
|
+
const dir = ws === here ? "" : ` --dir ${/^[\w./@%+=:,-]+$/.test(ws) ? ws : shq(ws)}`;
|
|
1389
|
+
let defs, st, unread = null;
|
|
1211
1390
|
try { defs = readDefinitions(ws); st = readState(ws); }
|
|
1212
|
-
catch (e) {
|
|
1391
|
+
catch (e) { unread = e; }
|
|
1213
1392
|
// Read the last discovery snapshot directly: placement and refresh would
|
|
1214
1393
|
// require credentials/network, and unresolved work matters even when offline.
|
|
1215
1394
|
const snapshot = readSnapshot(ws);
|
|
@@ -1220,6 +1399,40 @@ export function unresolvedScheduleAttempts(dep, { now = new Date() } = {}) {
|
|
|
1220
1399
|
if (resolve(dirname(found.path)) !== resolve(ws)) throw scheduleError("E_LOCAL_MISSING", "this scope has no local configuration");
|
|
1221
1400
|
local = found.local;
|
|
1222
1401
|
} catch (e) { localError = e; }
|
|
1402
|
+
// Every slot there is, as liveLocks counts them: the tick reports an unreadable one of a
|
|
1403
|
+
// defined job invalid, and reads none of a schedule that is not defined. Definitions that
|
|
1404
|
+
// cannot be read say nothing of which it is.
|
|
1405
|
+
const slots = join(stateDir(ws), "locks");
|
|
1406
|
+
let slotEntries = [];
|
|
1407
|
+
try { slotEntries = readdirSync(slots, { withFileTypes: true }).filter((e) => e.isDirectory()).map((e) => e.name).sort(); }
|
|
1408
|
+
catch (e) { if (e.code !== "ENOENT") information.push(`the schedule slots of ${ws} could not be read: ${e.message}`); }
|
|
1409
|
+
for (const key of slotEntries) {
|
|
1410
|
+
const id = shownScheduleId(key), path = lockDir(ws, key);
|
|
1411
|
+
const absence = unread !== null ? { state: "defined" } : scheduleAbsence(key, { defs, snapshot });
|
|
1412
|
+
const where = ws === here ? "" : ` (in ${ws})`;
|
|
1413
|
+
const orphan = absence.state === "defined" ? null : orphanSlot(ws, key, dir, absence.absent);
|
|
1414
|
+
const slot = absence.state === "defined" ? unreadableSlot(ws, key, dir) : null;
|
|
1415
|
+
const counted = `its slot ${path} ${orphan?.found}, and the slot counts against maxConcurrent`;
|
|
1416
|
+
const untold = `which is not in the workspace's snapshot of ${absence.takenAt}`;
|
|
1417
|
+
if (absence.state === "gone") problems.push({ code: SCHEDULE_SLOT_ORPHANED, severity: "warning", scope: ws, id,
|
|
1418
|
+
message: `schedule ${id}${where}, which ${absence.absent}: ${counted}`, remedy: orphan.remedy });
|
|
1419
|
+
if (absence.state === "member-absent") problems.push({ code: SCHEDULE_SLOT_ORPHANED, severity: "warning", scope: ws, id,
|
|
1420
|
+
message: `schedule ${id}${where}, ${untold}: its member ${absence.member} is not in the workspace, or was not read, so whether the schedule is gone cannot be told; ${counted}`,
|
|
1421
|
+
remedy: `check oats status for a run of schedule ${id}; once none is running, remove the slot by hand: rm -r -- ${shq(path)}` });
|
|
1422
|
+
if (absence.state === "member-problem") problems.push({ code: SCHEDULE_SLOT_ORPHANED, severity: "warning", scope: ws, id,
|
|
1423
|
+
message: `schedule ${id}${where}, ${untold}: ${absence.path ? `member ${absence.member} has a schedule file that could not be read (${absence.path}), which could be this schedule` : `member ${absence.member} could not be listed`}, so whether it is gone cannot be told; ${counted}`,
|
|
1424
|
+
remedy: `${absence.path ? `fix ${absence.path} in member ${absence.member}` : `make member ${absence.member} readable`} and run oats automations refresh; doctor then says whether schedule ${id} is gone` });
|
|
1425
|
+
// A workspace schedule this host does not run, or cannot tell it runs, is refused by reconcile
|
|
1426
|
+
// here: its slot is removed by hand once nothing of it runs.
|
|
1427
|
+
const placement = slot && key.includes("~") ? workspacePlacement(id, snapshot, local, localError) : {};
|
|
1428
|
+
const byHand = `once oats status shows no run of schedule ${id} here, remove the slot by hand: rm -r -- ${shq(path)}`;
|
|
1429
|
+
if (slot) problems.push({ code: SCHEDULE_SLOT_UNREADABLE, severity: "warning", scope: ws, id,
|
|
1430
|
+
message: placement.excluded ? `schedule ${id}${where}: its slot ${path} has no readable record, and the slot counts against maxConcurrent; this host does not run the schedule (${placement.excluded}), so reconcile here refuses it`
|
|
1431
|
+
: placement.unknown ? `schedule ${id}${where}: its slot ${path} has no readable record, and the slot counts against maxConcurrent; whether this host runs the schedule cannot be told offline (${placement.unknown})`
|
|
1432
|
+
: `schedule ${id}${where}: its slot ${path} has no readable record; the tick reports this job invalid and starts nothing of it, and the slot counts against maxConcurrent`,
|
|
1433
|
+
remedy: placement.excluded || placement.unknown ? byHand : slot.remedy });
|
|
1434
|
+
}
|
|
1435
|
+
if (unread) { information.push(`the schedules of ${ws} could not be read: ${unread.message}`); continue; }
|
|
1223
1436
|
for (const [key, js] of Object.entries(st.jobs)) {
|
|
1224
1437
|
if (!js?.attempt && js?.lastRun?.outcome !== "unknown") continue;
|
|
1225
1438
|
// A workspace schedule's state key is <member>~<id> (scheduleStateKey); a local one is its id.
|
|
@@ -1235,23 +1448,27 @@ export function unresolvedScheduleAttempts(dep, { now = new Date() } = {}) {
|
|
|
1235
1448
|
const holdsSlot = !!lock;
|
|
1236
1449
|
const error = js.attempt?.error ?? js.lastRun?.error ?? null;
|
|
1237
1450
|
const unproven = (kind === "command" || kind === "operation") && namedHomes(ws, js).length !== 1;
|
|
1238
|
-
const dir = ws === here ? "" : ` --dir ${/^[\w./@%+=:,-]+$/.test(ws) ? ws : shq(ws)}`;
|
|
1239
1451
|
let unavailable = localError ? `local configuration is invalid or unreadable (${localError.code || "read error"})` : null;
|
|
1240
1452
|
if (!workspace && !unavailable) {
|
|
1241
1453
|
unavailable = !SCHEDULE_ID_RE.test(id) ? "the stored schedule ID is invalid" : shapeError(defs.jobs[key]);
|
|
1242
1454
|
if (!unavailable && defs.jobs[key].id !== undefined && defs.jobs[key].id !== key) unavailable = "the stored definition has a different ID";
|
|
1243
1455
|
try { validateAttemptExecution(js.attempt || { execution: js.lastRun?.execution }, lock); }
|
|
1244
1456
|
catch { unavailable ||= "the retained attempt or lock uses unsupported captured execution state"; }
|
|
1245
|
-
if (lock?.unreadable) unavailable ||= "job lock custody is unreadable";
|
|
1246
1457
|
}
|
|
1247
|
-
const
|
|
1458
|
+
const slot = workspace || unavailable ? null : unreadableSlot(ws, key, dir);
|
|
1459
|
+
// A workspace schedule that is gone: reconcile --clear resolves its slot and state.
|
|
1460
|
+
const absence = workspace ? scheduleAbsence(key, { defs, snapshot }) : null;
|
|
1461
|
+
const orphan = absence?.state === "gone" ? orphanSlot(ws, key, dir, absence.absent) ?? orphanState(key, js, absence.absent, dir) : null;
|
|
1462
|
+
const remedy = orphan ? orphan.remedy
|
|
1463
|
+
: workspace ? workspaceReconcileRemedy(id, snapshot, local, localError)
|
|
1248
1464
|
: unavailable ? `Doctor does not recommend reconciling this job because ${unavailable}; preserve the retained state and inspect its effects with the deployment owner.`
|
|
1465
|
+
: slot ? slot.remedy
|
|
1249
1466
|
: `oats schedule reconcile ${id}${unproven ? " --clear" : ""}${dir}`;
|
|
1250
1467
|
const subject = js.attempt ? `${kind ? `a ${kind}` : "a"} launch attempt for ${scheduledFor} has no recorded result`
|
|
1251
1468
|
: `the last ${kind || "schedule"} run for ${scheduledFor} has an unknown observation`;
|
|
1252
1469
|
const blocking = js.attempt ? "This persisted attempt blocks another run until it is reconciled"
|
|
1253
1470
|
: "There is no persisted launch attempt blocking execution; an eligible tick may re-observe the last run, without a guarantee of clearance";
|
|
1254
|
-
const guidance = workspace || unavailable ? remedy : unproven ? `Its effects can't be proven: check the roster and the host by hand, then run ${remedy}` : `Run ${remedy}`;
|
|
1471
|
+
const guidance = orphan ? `${orphan.said}; ${remedy}` : workspace || unavailable ? remedy : slot ? `${slot.said}: ${remedy}` : unproven ? `Its effects can't be proven: check the roster and the host by hand, then run ${remedy}` : `Run ${remedy}`;
|
|
1255
1472
|
const message = `schedule ${id}${ws === here ? "" : ` (in ${ws})`}: ${subject} (started ${startedAt}${ageSeconds === null ? "" : `, ${formatAge(ageSeconds)} ago`})${error ? `: ${error}` : ""}. ${blocking}, and ${holdsSlot ? "it holds a host slot (counted against maxConcurrent)" : "it holds no host slot"}. ${guidance}`;
|
|
1256
1473
|
problems.push({ code: "schedule-unresolved", severity: "warning", scope: ws, id, kind, scheduledFor, startedAt, ageSeconds, holdsSlot, exited: js.attempt?.exited === true, error, remedy, message });
|
|
1257
1474
|
}
|
package/lib/servers.mjs
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
19
|
import { execFile, execFileSync, spawnSync } from "node:child_process";
|
|
20
|
-
import { createHash, randomBytes } from "node:crypto";
|
|
20
|
+
import { createHash, randomBytes, randomUUID } from "node:crypto";
|
|
21
21
|
import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
|
22
22
|
import { homedir } from "node:os";
|
|
23
23
|
import { basename, join, resolve } from "node:path";
|
|
@@ -34,6 +34,7 @@ import { shellWord } from "./shell-word.mjs";
|
|
|
34
34
|
import { DETACH_KEY_FEATURE, requireDetachKey } from "./session-viewer.mjs";
|
|
35
35
|
import { LOGIN_CAPTURE_MAX_BYTES, LOGIN_SHELLS, loginPathEmitter, quoteFor, readLoginFrame } from "./login-environment.mjs";
|
|
36
36
|
import { compareSemver } from "./semver.mjs";
|
|
37
|
+
import { SPAWN_TASK_STDIN_FEATURE } from "./kernel-features.mjs";
|
|
37
38
|
import { sameRepo } from "./repo-moves.mjs";
|
|
38
39
|
import { OATS_PACKAGE_NAME } from "./version.mjs";
|
|
39
40
|
export { compareSemver } from "./semver.mjs";
|
|
@@ -653,10 +654,21 @@ export function listSnapshots(serverId) {
|
|
|
653
654
|
|
|
654
655
|
// ------------------------------------------------------------------- routes
|
|
655
656
|
|
|
657
|
+
/** What a host must advertise for a routed spawn to be bound to its preview and a key: the preview
|
|
658
|
+
* with its decision, the apply that takes one, and the key's recovery before placement. */
|
|
659
|
+
const KEYED_SPAWN_FEATURES = ["spawn-preview-2", "spawn-apply-2", "spawn-idempotency-2"];
|
|
660
|
+
|
|
656
661
|
/** Route one lifecycle command to a server. Returns the remote envelope,
|
|
657
662
|
* having kept the local snapshot store in step with it:
|
|
658
663
|
* - spawn: compatibility checked first, a snapshot written on success; a
|
|
659
664
|
* --preview runs the same checks, needs spawn-preview-2 and writes nothing;
|
|
665
|
+
* a spawn with no decision and no key of its own is previewed on the host
|
|
666
|
+
* and applied bound to that decision and a key (KEYED_SPAWN_FEATURES); a
|
|
667
|
+
* `--task <text>` does not travel as an argument, where it would be in the
|
|
668
|
+
* command line of the local ssh and of the host's oats: each spawn call
|
|
669
|
+
* carries `--task-file -` in its place and the text on its standard input
|
|
670
|
+
* (SPAWN_TASK_STDIN_FEATURE). A host without the feature gets the task as
|
|
671
|
+
* an argument, and the answer carries a warning that says so;
|
|
660
672
|
* - retire: the target comes from the instance's snapshot when one exists
|
|
661
673
|
* (the registry may have changed since), the snapshot is removed only
|
|
662
674
|
* when the remote kernel reports the home gone;
|
|
@@ -688,6 +700,23 @@ export function routeCommand(serverId, cmd, oatsArgs, io = {}) {
|
|
|
688
700
|
throw serverError("E_ROUTE_CHANGED", `server ${serverId} now points at ${target.sshHost}:${target.workspace}, but ${priorTargets.length} saved route${priorTargets.length === 1 ? "" : "s"} for it (${priorTargets.map((s) => s.instance).join(", ")}) target ${old.sshHost}:${old.workspace}; spawning would overwrite them. Keep the old registration and add a new server id for the new target, or retire those instances first`);
|
|
689
701
|
}
|
|
690
702
|
const remote = checkRemote(target, io);
|
|
703
|
+
// For a host that reads a task from its standard input (SPAWN_TASK_STDIN_FEATURE), the task
|
|
704
|
+
// leaves the arguments here, before any check reads them or a spawn call is built. The first
|
|
705
|
+
// --task is the one a spawn reads; every one is taken out. A host without the feature gets
|
|
706
|
+
// the arguments as given, and its answer says where the task was readable.
|
|
707
|
+
let task, taskAsArgument;
|
|
708
|
+
const taskAt = oatsArgs.indexOf("--task");
|
|
709
|
+
if (taskAt >= 0 && oatsArgs[taskAt + 1] !== undefined && !remote.features.includes(SPAWN_TASK_STDIN_FEATURE)) {
|
|
710
|
+
taskAsArgument = `remote oats ${remote.version} at ${target.sshHost} does not advertise ${SPAWN_TASK_STDIN_FEATURE}, so the task was passed as an argument, visible in this machine's process list (the ssh command) and the host's while the spawn runs; upgrade the host`;
|
|
711
|
+
} else if (taskAt >= 0 && oatsArgs[taskAt + 1] !== undefined) {
|
|
712
|
+
task = Buffer.from(oatsArgs[taskAt + 1]);
|
|
713
|
+
const kept = [];
|
|
714
|
+
for (let i = 0; i < oatsArgs.length; i++) {
|
|
715
|
+
if (oatsArgs[i] !== "--task" || oatsArgs[i + 1] === undefined) kept.push(oatsArgs[i]);
|
|
716
|
+
else { if (i === taskAt) kept.push("--task-file", "-"); i++; }
|
|
717
|
+
}
|
|
718
|
+
oatsArgs = kept;
|
|
719
|
+
}
|
|
691
720
|
// A saved route is keyed by name under its server id. An explicit name
|
|
692
721
|
// (--name, or --instance) that already has one here would overwrite that
|
|
693
722
|
// route on success: refuse before the mutation (the roster shows the
|
|
@@ -717,7 +746,50 @@ export function routeCommand(serverId, cmd, oatsArgs, io = {}) {
|
|
|
717
746
|
if (oatsArgs.includes("--background") || oatsArgs.includes("--no-background")) requireRemoteFeature(remote, target, "instance-background");
|
|
718
747
|
// A host without instance-same-work would refuse the flag as unknown: it is never sent there.
|
|
719
748
|
if (oatsArgs.includes("--same-work")) requireRemoteFeature(remote, target, "instance-same-work");
|
|
720
|
-
const
|
|
749
|
+
const spawnOnHost = (args) => runRemote(target, json(withScope(["spawn", ...hostHarnessArgs(remote, args)])), { ...io, input: task });
|
|
750
|
+
// A spawn that brings neither a decision nor a key is bound here: the host's preview decides, and
|
|
751
|
+
// the apply carries that decision and a key of this request, so the same spawn sent again with
|
|
752
|
+
// both answers the instance this one created. A host that does not advertise the keyed spawn
|
|
753
|
+
// gets the spawn as given, and its answer says what that costs.
|
|
754
|
+
const given = (flag) => oatsArgs.some((a) => a === flag || a.startsWith(`${flag}=`));
|
|
755
|
+
const suggesting = oatsArgs.includes("--suggest-name");
|
|
756
|
+
let applyArgs = oatsArgs, bound, unbound, refusal;
|
|
757
|
+
if (!preview && !given("--expect-decision") && !given("--idempotency-key")) {
|
|
758
|
+
// The decision binds a name, so a suggested one is chosen by the preview and applied as --name.
|
|
759
|
+
const missing = [...KEYED_SPAWN_FEATURES, ...(suggesting ? ["spawn-suggest-name", "spawn-name"] : [])].filter((f) => !remote.features.includes(f));
|
|
760
|
+
if (missing.length) unbound = `remote oats ${remote.version} at ${target.sshHost} does not advertise ${missing.join(", ")}: if the link drops while it spawns, the spawn cannot be retried safely (the instance may exist there, and a second run would create another); check oats status --server ${serverId} first, or upgrade oats there`;
|
|
761
|
+
else {
|
|
762
|
+
const previewed = spawnOnHost([...oatsArgs, "--preview"]);
|
|
763
|
+
const shown = previewed.envelope.result;
|
|
764
|
+
// Anything but a preview with its decision is the host's answer to this spawn (a refusal).
|
|
765
|
+
if (!(previewed.envelope.ok && shown?.preview === true && typeof shown.decision?.revision === "string")) refusal = previewed;
|
|
766
|
+
else {
|
|
767
|
+
bound = { decision: shown.decision, idempotencyKey: randomUUID(), naming: shown.naming };
|
|
768
|
+
applyArgs = [...(suggesting ? [...oatsArgs.filter((a) => a !== "--suggest-name"), "--name", shown.instance] : oatsArgs), "--expect-decision", shown.decision.revision, "--idempotency-key", bound.idempotencyKey];
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
let answer = refusal;
|
|
773
|
+
try { answer ??= spawnOnHost(applyArgs); }
|
|
774
|
+
catch (e) {
|
|
775
|
+
// The link failed under a bound apply: the host may have started it. The decision and the key
|
|
776
|
+
// travel with the failure, so the spawn sent again lands on that one instance.
|
|
777
|
+
if (bound && e.code === "E_SSH" && e.details?.sshStarted !== false) {
|
|
778
|
+
const fix = `run the same spawn again with ${suggesting ? `--name ${bound.decision.instance} in place of --suggest-name, and ` : ""}--expect-decision ${bound.decision.revision} --idempotency-key ${bound.idempotencyKey}: it answers the instance this attempt created there, or creates it`;
|
|
779
|
+
e.message = `${e.message}; the host may have started the spawn: ${fix}`;
|
|
780
|
+
e.details = { ...e.details, decision: bound.decision, idempotencyKey: bound.idempotencyKey, fix };
|
|
781
|
+
}
|
|
782
|
+
throw e;
|
|
783
|
+
}
|
|
784
|
+
const { stderr } = answer;
|
|
785
|
+
let { envelope } = answer;
|
|
786
|
+
if (envelope.ok && envelope.result) {
|
|
787
|
+
const added = [unbound, taskAsArgument].filter(Boolean);
|
|
788
|
+
const warnings = added.length ? { warnings: [...(envelope.result.warnings || []), ...added] } : {};
|
|
789
|
+
// How the preview chose the name is the spawn's own answer to --suggest-name.
|
|
790
|
+
const naming = bound?.naming && envelope.result.naming === undefined ? { naming: bound.naming } : {};
|
|
791
|
+
envelope = { ...envelope, result: { ...envelope.result, ...warnings, ...naming } };
|
|
792
|
+
}
|
|
721
793
|
if (preview && envelope.ok && envelope.result?.preview === true && envelope.result.spawnPreviewApi === 2) {
|
|
722
794
|
// The host's preview with the two fields a routed spawn adds; never a
|
|
723
795
|
// `snapshot` key, which on the apply path says what was (or was not) saved.
|
|
@@ -1304,13 +1376,19 @@ export function scheduleRemote(serverId, oatsArgs, io = {}) {
|
|
|
1304
1376
|
const setsHostCaps = oatsArgs[0] === "host" && oatsArgs[1] === "install" && oatsArgs.some((arg) =>
|
|
1305
1377
|
["--max-concurrent", "--triggers-max-concurrent"].some((name) => arg === name || arg.startsWith(`${name}=`)));
|
|
1306
1378
|
const capsUnsupported = () => serverError("E_REMOTE_INCOMPATIBLE", `remote oats at ${target.sshHost} (${serverId}) does not confirm required features schedule and schedule-host-caps; upgrade oats on that destination before setting or resetting host caps`);
|
|
1379
|
+
// `host install --register-only` installs no timer. A host that does not know the option
|
|
1380
|
+
// ignores it and installs one, so it is sent only to a host that advertises it.
|
|
1381
|
+
const registersOnly = oatsArgs[0] === "host" && oatsArgs[1] === "install" && oatsArgs.includes("--register-only");
|
|
1382
|
+
const registerOnlyUnsupported = () => serverError("E_REMOTE_INCOMPATIBLE", `remote oats at ${target.sshHost} (${serverId}) does not confirm required features schedule and schedule-host-register-only, so --register-only was not sent: a host without it ignores the option and installs a timer; upgrade oats on that destination`);
|
|
1307
1383
|
let remote;
|
|
1308
1384
|
try {
|
|
1309
1385
|
remote = checkRemote(target, io);
|
|
1386
|
+
if (registersOnly && !["schedule", "schedule-host-register-only"].every((feature) => remote.features.includes(feature))) throw registerOnlyUnsupported();
|
|
1310
1387
|
if (!remote.features.includes("schedule")) throw serverError("E_REMOTE_INCOMPATIBLE", `remote oats ${remote.version} at ${target.sshHost} does not advertise schedules; upgrade it there`);
|
|
1311
1388
|
} catch (e) {
|
|
1312
|
-
// Unknown probe formats cannot establish
|
|
1313
|
-
// failures distinct, and preserve the existing contract for
|
|
1389
|
+
// Unknown probe formats cannot establish support for an option. Keep transport
|
|
1390
|
+
// failures distinct, and preserve the existing contract for calls without one.
|
|
1391
|
+
if (registersOnly && ["E_REMOTE_ENVELOPE", "E_REMOTE_INCOMPATIBLE"].includes(e.code)) throw registerOnlyUnsupported();
|
|
1314
1392
|
if (setsHostCaps && ["E_REMOTE_ENVELOPE", "E_REMOTE_INCOMPATIBLE"].includes(e.code)) throw capsUnsupported();
|
|
1315
1393
|
throw e;
|
|
1316
1394
|
}
|
|
@@ -1544,7 +1622,7 @@ export function connectServer(options, io = {}) {
|
|
|
1544
1622
|
const searched = login ? `the PATH of ${sshHost}'s login shell (${basename(login.shell)} -l -i)` : path ? "--path and ssh's PATH" : "ssh's PATH";
|
|
1545
1623
|
const where = login ? `so that its login shell finds it (there, ${runnable([login.shell, "-l", "-i", "-c", "command -v npm"])} prints its path)`
|
|
1546
1624
|
: path ? "where --path or a non-interactive ssh command finds it" : "where a non-interactive ssh command finds it";
|
|
1547
|
-
return { status: "needs-human", detail: `${what}, and no npm on ${searched}`, remedy: `install Node.js 22+ (npm) on ${sshHost} ${where}, then check again (run the same connect); from the CLI, --path <the directory holding npm> names it instead` };
|
|
1625
|
+
return { status: "needs-human", detail: `${what}, and no npm on ${searched}`, remedy: `install Node.js 22.5+ (npm) on ${sshHost} ${where}, then check again (run the same connect); from the CLI, --path <the directory holding npm> names it instead` };
|
|
1548
1626
|
}
|
|
1549
1627
|
const install = ["npm", "install", "-g", `${OATS_PACKAGE_NAME}@${localVersion}`];
|
|
1550
1628
|
if (!installOats) return { status: "needs-human", detail: what, remedy: `on ${sshHost}: ${runnable(install)} (or pass --install-oats)` };
|
package/lib/session-end.mjs
CHANGED
|
@@ -7,6 +7,9 @@
|
|
|
7
7
|
* - `.oats-stop.json`: `oats instance stop` / `oats session stop`, a retire and a restart (before
|
|
8
8
|
* they end the session), `{ instance, at, stop, retained, startId }`; `startId` (0.69) is the
|
|
9
9
|
* start the stop ended, null when the home had none.
|
|
10
|
+
* A stop with the host (`oats host run` when the host stops: lib/host-run.mjs) writes the same
|
|
11
|
+
* record with `resume: true` and `reason: "host-stop"`, before it signals the harness: OATS
|
|
12
|
+
* ended that session only because the host was going down, and a resume brings it back.
|
|
10
13
|
* A reboot or a killed tmux server leaves neither file for the start that was running: that is a
|
|
11
14
|
* lost session, the one a resume brings back. Dependency-free: status, start --resume and the
|
|
12
15
|
* schedule runner read the files through here. */
|
|
@@ -16,6 +19,8 @@ import { SESSION_END } from "./session-resume-contract.mjs";
|
|
|
16
19
|
|
|
17
20
|
export const START_EXITED_FILE = ".oats-start-exited";
|
|
18
21
|
export const STOP_RECEIPT_FILE = ".oats-stop.json";
|
|
22
|
+
/** `reason` in the stop record of a stop with the host. */
|
|
23
|
+
export const HOST_STOP_REASON = "host-stop";
|
|
19
24
|
const RECORD_MAX = 64 * 1024;
|
|
20
25
|
/** A shell's exit status from 128 up is a signal's (128 + n): the harness was killed, not ended. */
|
|
21
26
|
const SIGNALLED = 128;
|
|
@@ -59,9 +64,21 @@ export function currentStartTime(meta) {
|
|
|
59
64
|
return Date.parse(typeof last === "string" ? last : meta?.createdAt);
|
|
60
65
|
}
|
|
61
66
|
|
|
67
|
+
/** The stop record of `home` when it is of `meta`'s current start (`meta.startId`), else null: it
|
|
68
|
+
* names this start (`startId` equal), or, a record without a start id, was written after this
|
|
69
|
+
* start began. */
|
|
70
|
+
function stopOfCurrentStart(home, meta) {
|
|
71
|
+
const startId = typeof meta?.startId === "string" && meta.startId ? meta.startId : null;
|
|
72
|
+
const stop = readStopReceipt(home);
|
|
73
|
+
if (!stop) return null;
|
|
74
|
+
if (typeof stop.startId === "string" && stop.startId) return stop.startId === startId ? stop : null;
|
|
75
|
+
return Date.parse(stop.at) > currentStartTime(meta) ? stop : null;
|
|
76
|
+
}
|
|
77
|
+
|
|
62
78
|
/** How the session of `meta`'s current start (`meta.startId`) ended, from the home's files alone:
|
|
63
|
-
* - `stopped`: the stop
|
|
64
|
-
*
|
|
79
|
+
* - `stopped`: the stop record is of this start (stopOfCurrentStart) and asks for no resume;
|
|
80
|
+
* - `lost`, whatever the exit marker says: the stop record is of this start and says
|
|
81
|
+
* `resume: true` (a stop with the host): the harness ended because the host did;
|
|
65
82
|
* - `exited`: the exit marker names this start with a status below 128, or in its legacy form
|
|
66
83
|
* (no status: a kernel before 0.69 wrote it only when the harness ended on its own window);
|
|
67
84
|
* - `lost`: anything else — a marker with a signal's status (128 and up), or no file naming this
|
|
@@ -71,13 +88,16 @@ export function currentStartTime(meta) {
|
|
|
71
88
|
* The caller reads it only for a launched home that does not run. */
|
|
72
89
|
export function sessionEndOf(home, meta) {
|
|
73
90
|
const startId = typeof meta?.startId === "string" && meta.startId ? meta.startId : null;
|
|
74
|
-
const
|
|
75
|
-
|
|
76
|
-
if (stop) {
|
|
77
|
-
if (typeof stop.startId === "string" && stop.startId) { if (stop.startId === startId) return SESSION_END.STOPPED; }
|
|
78
|
-
else if (Date.parse(stop.at) > since) return SESSION_END.STOPPED;
|
|
79
|
-
}
|
|
91
|
+
const stop = stopOfCurrentStart(home, meta);
|
|
92
|
+
if (stop) return stop.resume === true ? SESSION_END.LOST : SESSION_END.STOPPED;
|
|
80
93
|
const exited = startId ? readStartExited(home) : null;
|
|
81
94
|
if (exited && exited.startId === startId) return exited.status !== null && exited.status >= SIGNALLED ? SESSION_END.LOST : SESSION_END.EXITED;
|
|
82
95
|
return SESSION_END.LOST;
|
|
83
96
|
}
|
|
97
|
+
|
|
98
|
+
/** Why the session of `meta`'s current start reads `lost`, where the home says: HOST_STOP_REASON
|
|
99
|
+
* for a stop with the host; null for every other loss, and for a session that is not lost. */
|
|
100
|
+
export function lostReasonOf(home, meta) {
|
|
101
|
+
const stop = stopOfCurrentStart(home, meta);
|
|
102
|
+
return stop?.resume === true && stop.reason === HOST_STOP_REASON ? HOST_STOP_REASON : null;
|
|
103
|
+
}
|