@awebai/oats 0.69.14 → 0.69.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/bin/oats.mjs +79 -18
  2. package/docs/capabilities.md +54 -12
  3. package/docs/container.md +57 -12
  4. package/docs/desktop-cli-api.md +215 -19
  5. package/docs/execution-targets.md +15 -3
  6. package/docs/first-team.md +4 -3
  7. package/docs/implementation.md +18 -8
  8. package/docs/release-notes/v0.69.15.md +444 -0
  9. package/docs/schedules.md +133 -2
  10. package/docs/servers.md +60 -2
  11. package/docs/souls-and-instances.md +87 -7
  12. package/docs/upgrading.md +17 -5
  13. package/lib/claim.mjs +2 -3
  14. package/lib/command-flags.mjs +4 -2
  15. package/lib/core.mjs +215 -72
  16. package/lib/dir-lock.mjs +17 -0
  17. package/lib/errors.mjs +2 -1
  18. package/lib/host-run.mjs +218 -0
  19. package/lib/host-stop-child.mjs +21 -0
  20. package/lib/instance-git.mjs +32 -6
  21. package/lib/instance-lifecycle.mjs +4 -3
  22. package/lib/instance-listing.mjs +35 -6
  23. package/lib/kernel-features.mjs +4 -1
  24. package/lib/read-call.mjs +1 -1
  25. package/lib/remote.mjs +17 -14
  26. package/lib/schedule.mjs +237 -20
  27. package/lib/servers.mjs +83 -5
  28. package/lib/session-end.mjs +28 -8
  29. package/lib/session-resume.mjs +10 -18
  30. package/lib/session-seen.mjs +4 -64
  31. package/lib/status-words.mjs +11 -4
  32. package/lib/system-tools.mjs +1 -36
  33. package/lib/worktree-hooks.mjs +6 -175
  34. package/lib/worktree.mjs +19 -10
  35. package/package.json +4 -1
  36. package/packages/record/README.md +56 -21
  37. package/packages/record/docs/capture-service.md +8 -5
  38. package/packages/record/docs/turn-record-sot.md +9 -0
  39. package/packages/record/lib/capture-health.mjs +6 -4
  40. package/packages/record/lib/capture-lock.mjs +31 -27
  41. package/packages/record/lib/capture-retained-scratch.mjs +4 -3
  42. package/packages/record/lib/capture-scratch.mjs +2 -1
  43. package/packages/record/lib/derived-state.mjs +4 -2
  44. package/packages/record/lib/index-db.mjs +4 -2
  45. package/packages/record/lib/lock-holder.mjs +21 -0
  46. package/packages/record/lib/private-files.mjs +18 -0
  47. package/packages/record/lib/process-identity.mjs +310 -0
  48. package/packages/record/lib/store.mjs +31 -25
  49. package/skills/oats-getting-started/SKILL.md +1 -1
package/lib/schedule.mjs CHANGED
@@ -22,7 +22,7 @@
22
22
  * `unknown`, its job blocked until `reconcile` proves what happened; it keeps
23
23
  * its host slot too, unless it is a command whose process exit was observed. */
24
24
  import { runScheduleCommand } from "./schedule-command.mjs";
25
- import { closeSync, constants as fsConstants, existsSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, realpathSync, rmSync, writeFileSync } from "node:fs";
25
+ import { closeSync, constants as fsConstants, existsSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
26
26
  import { createHash } from "node:crypto";
27
27
  import { homedir } from "node:os";
28
28
  import { basename, dirname, isAbsolute, join, resolve, sep } from "node:path";
@@ -295,11 +295,29 @@ function withRegistryLock(fn) { return withDirLock(join(hostScheduleDir(), "regi
295
295
  /** Definitions and state are read-modify-write; CRUD serializes here briefly. */
296
296
  export function withScopeLock(ws, fn) { return withDirLock(join(stateDir(ws), "scope.lock"), `schedules of ${ws}`, fn, { retryMs: 3000 }); }
297
297
 
298
+ /** A job's slot: the directory `locks/<id>` and its record `owner.json`, which stands for the run
299
+ * that holds it until that run's instance ends (no process mutex: every taker and releaser holds
300
+ * the host lock, withHostLock). The slot is never there without its record. Its record is written
301
+ * in a directory beside the slots (slotStaging), which is then renamed to the slot; a release
302
+ * renames the slot back there before removing it. A taker or releaser killed at any step leaves
303
+ * either the slot with its record or no slot, and at most that directory, which the next taker or
304
+ * releaser of the slot removes. An entry already at the slot is held, whatever it is, and left as
305
+ * it is: a directory without its record (an earlier kernel made the directory first) reads as
306
+ * unreadable (jobLockInfo). */
307
+ const slotStaging = (ws, id) => join(stateDir(ws), `.slot-${id}`);
298
308
  export function acquireJobLock(ws, id, owner) {
299
- const dir = lockDir(ws, id);
309
+ const dir = lockDir(ws, id), staging = slotStaging(ws, id);
300
310
  mkdirSync(dirname(dir), { recursive: true });
301
- try { mkdirSync(dir); } catch (e) { if (e.code === "EEXIST") return false; throw e; }
302
- writeFileSync(join(dir, "owner.json"), JSON.stringify({ ...owner, pid: process.pid, at: new Date().toISOString() }, null, 2) + "\n");
311
+ try { lstatSync(dir); return false; } catch (e) { if (e.code !== "ENOENT") throw e; }
312
+ rmSync(staging, { recursive: true, force: true });
313
+ mkdirSync(staging);
314
+ try {
315
+ writeFileSync(join(staging, "owner.json"), JSON.stringify({ ...owner, pid: process.pid, at: new Date().toISOString() }, null, 2) + "\n");
316
+ renameSync(staging, dir);
317
+ } catch (e) {
318
+ try { rmSync(staging, { recursive: true, force: true }); } catch { /* the next taker of the slot removes it */ }
319
+ throw e;
320
+ }
303
321
  return true;
304
322
  }
305
323
  export function jobLockInfo(ws, id) {
@@ -307,7 +325,112 @@ export function jobLockInfo(ws, id) {
307
325
  if (!existsSync(dir)) return null;
308
326
  try { return JSON.parse(readFileSync(join(dir, "owner.json"), "utf8")); } catch { return { unreadable: true }; }
309
327
  }
310
- export function releaseJobLock(ws, id) { rmSync(lockDir(ws, id), { recursive: true, force: true }); }
328
+ /** `oats doctor`'s problem for a job's slot whose record cannot be read. */
329
+ export const SCHEDULE_SLOT_UNREADABLE = "schedule-slot-unreadable";
330
+ /** A schedule's id as a person types it, from the key its state and slot are stored under: a
331
+ * workspace schedule's key is <member>~<id> (scheduleStateKey), a local one is its id. */
332
+ const shownScheduleId = (key) => key.replace("~", "/");
333
+ /** The slot of the job `key` in `ws` when its record cannot be read (jobLockInfo: a directory
334
+ * without its record, a record that does not parse, an entry that is no directory) → `{ said,
335
+ * remedy }`, the words the tick, reconcile, run-now and doctor give for it; else null. Nobody
336
+ * can tell from such a slot whether the run it stands for still runs, so it is held and left as
337
+ * it is until a person has checked and reconcile --clear releases it. `dir` is the `--dir`
338
+ * argument the commands of the remedy need from another deployment. */
339
+ function unreadableSlot(ws, key, dir = "") {
340
+ if (!jobLockInfo(ws, key)?.unreadable) return null;
341
+ const id = shownScheduleId(key);
342
+ return { said: `the slot ${lockDir(ws, key)} has no readable record`, remedy: `check oats schedule show ${id}${dir}; once its last run is not running, run oats schedule reconcile ${id} --clear${dir} to release the slot` };
343
+ }
344
+ /** `oats doctor`'s problem for a slot whose schedule is not defined. */
345
+ export const SCHEDULE_SLOT_ORPHANED = "schedule-slot-orphaned";
346
+ const ISO_INSTANT = /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}Z$/;
347
+ /** The slot of `key` in `ws` when its schedule is gone (scheduleAbsence), whatever its record →
348
+ * `{ found, said, remedy }`, the words reconcile and doctor give for it; else null (no slot). No
349
+ * tick reads such a slot, and every tick counts it against maxConcurrent (liveLocks): it may stand
350
+ * for a run of a schedule removed while it ran, so it is held until a person has checked and
351
+ * reconcile --clear releases it. `found` says what the record names: the run's due minute, and
352
+ * the home of its instance when the record names one (a wake's); a record that cannot be read
353
+ * names nothing. `absent` says how the schedule is known to be gone. Only what has the shape the
354
+ * kernel writes is shown. */
355
+ function orphanSlot(ws, key, dir = "", absent = "is not defined") {
356
+ const record = jobLockInfo(ws, key);
357
+ if (!record) return null;
358
+ const id = shownScheduleId(key), release = `run oats schedule reconcile ${id} --clear${dir} to release the slot`;
359
+ const unnamed = `once no run of schedule ${id} is running (oats status lists the instances it started), ${release}`;
360
+ let found, remedy = unnamed;
361
+ if (record.unreadable) found = "has no readable record";
362
+ else {
363
+ const due = typeof record.scheduledFor === "string" && ISO_INSTANT.test(record.scheduledFor) ? record.scheduledFor : "?";
364
+ const home = typeof record.home === "string" && isAbsolute(record.home) && !/[\x00-\x1f\x7f]/.test(record.home) ? record.home : null;
365
+ found = `holds the run due ${due} of ${home ? `the instance at ${home}` : "an instance its record does not name"}`;
366
+ if (home) remedy = `once oats status shows the instance at ${home} is not running, ${release}`;
367
+ }
368
+ return { found, said: `the slot ${lockDir(ws, key)} of schedule ${id}, which ${absent}, ${found}`, remedy };
369
+ }
370
+ /** A path a snapshot names that may be shown as it is. */
371
+ const SHOWN_PATH = /^[A-Za-z0-9._@+/-]+$/;
372
+ /** Whether the schedule whose slot or state is `key` is defined, from what this host has already
373
+ * read and nothing more (no discovery, no network): `defs` (readDefinitions) for a local schedule,
374
+ * and for a workspace one (<member>~<id>) the saved snapshot `snapshot` (readSnapshot) →
375
+ * { state: "defined" } it is defined, or nothing read says otherwise (a workspace one
376
+ * whose snapshot cannot be read)
377
+ * { state: "gone", absent, takenAt? }
378
+ * local: not in oats-schedules.json. Workspace: as of the snapshot's
379
+ * `takenAt`, its member was read at a commit, the snapshot has no
380
+ * problem of that member that could hide a schedule, and no
381
+ * schedule has its id. A workspace schedule that is disabled,
382
+ * untrusted or placed elsewhere still has its entry, so it is
383
+ * defined; `absent` is the clause that says how it is known gone
384
+ * { state: "member-absent", takenAt, member }
385
+ * its member is not in the snapshot at a commit: removed from the
386
+ * workspace, or not read (an unconfirmed member is skipped silently)
387
+ * { state: "member-problem", takenAt, member, path }
388
+ * a problem of its member that has no listed schedule beside it: a
389
+ * schedule file that could not be read or parsed (`path`, which
390
+ * could hold this id: a file's id may be other than its name), or
391
+ * a member that could not be listed (`path` null) */
392
+ function scheduleAbsence(key, { defs, snapshot }) {
393
+ if (!key.includes("~")) return defs.jobs[key] ? { state: "defined" } : { state: "gone", absent: "is not defined" };
394
+ if (!snapshot) return { state: "defined" };
395
+ const id = shownScheduleId(key), member = key.slice(0, key.indexOf("~"));
396
+ const schedules = snapshot.schedules;
397
+ if (schedules.some((e) => e?.id === id)) return { state: "defined" };
398
+ const takenAt = typeof snapshot.takenAt === "string" && ISO_INSTANT.test(snapshot.takenAt) ? snapshot.takenAt : "an unknown time";
399
+ const members = snapshot.members && typeof snapshot.members === "object" ? snapshot.members : {};
400
+ const memberKey = Object.keys(members).find((k) => members[k]?.name === member && typeof members[k]?.commit === "string" && members[k].commit !== "");
401
+ if (!memberKey) return { state: "member-absent", takenAt, member };
402
+ const file = (p) => (typeof p?.path === "string" ? p.path.split("#")[0] : "");
403
+ const listed = (p) => schedules.some((e) => e?.origin?.repoKey === p.repoKey && e.origin.path === file(p));
404
+ const hiding = (Array.isArray(snapshot.problems) ? snapshot.problems : []).find((p) => p && (p.kind === undefined || p.kind === "schedule")
405
+ && (p.repoKey === memberKey || (file(p) === "" && !Object.hasOwn(members, p.repoKey))) && !listed(p));
406
+ if (hiding) return { state: "member-problem", takenAt, member, path: file(hiding) === "" ? null : SHOWN_PATH.test(file(hiding)) ? file(hiding) : "a schedule file" };
407
+ return { state: "gone", takenAt, absent: `as of ${takenAt} is not in the workspace (run oats automations refresh if that is stale)` };
408
+ }
409
+ /** The attempt reconcile resolves for the job state `js`: its persisted attempt, or the unknown
410
+ * last run an earlier kernel kept in its place. */
411
+ const attemptOf = (js) => js.attempt || (js.lastRun?.outcome === "unknown" ? { scheduledFor: js.lastRun.scheduledFor, startedAt: js.lastRun.startedAt, ...(js.lastRun.execution !== undefined ? { execution: js.lastRun.execution } : {}) } : undefined);
412
+ /** The last run of `js` once reconcile resolves its `attempt` as launching nothing: cleared by the
413
+ * operator (`clear`), or no instance found. */
414
+ /** The retained state `js` of the job `key` whose schedule is gone and that holds no slot →
415
+ * `{ said, remedy }`, the words reconcile and doctor give for it: what is retained (a launch
416
+ * attempt with no recorded result, when there is one), and the reconcile --clear that clears it. */
417
+ function orphanState(key, js, absent, dir = "") {
418
+ const id = shownScheduleId(key), attempt = attemptOf(js);
419
+ const due = typeof attempt?.scheduledFor === "string" && ISO_INSTANT.test(attempt.scheduledFor) ? attempt.scheduledFor : "?";
420
+ return { said: `schedule ${id}, which ${absent}, has retained state${attempt ? ` with a launch attempt for ${due} that has no recorded result` : ""}`,
421
+ remedy: `once no run of schedule ${id} is running (oats status lists the instances it started), run oats schedule reconcile ${id} --clear${dir} to clear its state` };
422
+ }
423
+ const launchFailedRun = (js, attempt, kind, clear) => ({ scheduledFor: attempt.scheduledFor, startedAt: js.lastRun?.startedAt || attempt.startedAt, kind, launched: false, outcome: "launch-failed", error: clear ? "cleared by the operator after checking by hand" : "no instance found in the scope's roster for the unrecorded attempt" });
424
+ /** Whether `key` names an entry directly among a scope's slots (lockDir), and nothing beyond them. */
425
+ const slotName = (key) => typeof key === "string" && key !== "" && key !== "." && key !== ".." && !/[/\\\0]/.test(key);
426
+ export function releaseJobLock(ws, id) {
427
+ const staging = slotStaging(ws, id);
428
+ rmSync(staging, { recursive: true, force: true });
429
+ try { renameSync(lockDir(ws, id), staging); } catch (e) { if (e.code === "ENOENT") return; throw e; }
430
+ // The slot is released once it is renamed: what could not be removed of it is removed by the
431
+ // next taker or releaser of the slot.
432
+ try { rmSync(staging, { recursive: true, force: true }); } catch { /* removed next time */ }
433
+ }
311
434
  function liveLocks(wsList) {
312
435
  let n = 0;
313
436
  for (const ws of wsList) {
@@ -907,7 +1030,8 @@ export function tickWorkspace(ws, { now = new Date(), firedAt = now, io, reg, ws
907
1030
  // `candidates` (the host tick) means the host already observed every scope.
908
1031
  // Triggers (kind "trigger") live in the same file and are evaluated by tickTriggers, not here.
909
1032
  for (const [id, def] of Object.entries(defs.jobs)) if (def && def.kind === "trigger") delete defs.jobs[id];
910
- if (!candidates) for (const [id, def] of Object.entries(defs.jobs)) observePrevious(ws, id, def, stateOf(id), io, now, { mutate: !dryRun });
1033
+ // A job whose slot cannot be read is not observed: it is reported invalid below, on its own.
1034
+ if (!candidates) for (const [id, def] of Object.entries(defs.jobs)) if (!unreadableSlot(ws, id)) observePrevious(ws, id, def, stateOf(id), io, now, { mutate: !dryRun });
911
1035
  if (observeOnly) { if (!dryRun) writeState(ws, st); return considered; }
912
1036
  // Least recently LAUNCHED first: only an actual harness launch counts, a
913
1037
  // skipped or pending job keeps its place at the front of the line.
@@ -918,6 +1042,8 @@ export function tickWorkspace(ws, { now = new Date(), firedAt = now, io, reg, ws
918
1042
  let due, cronError;
919
1043
  const invalidShape = shapeError(def), herdr = invalidShape ? null : storedHerdrBackend(ws, id, def);
920
1044
  const shape = invalidShape ?? herdr?.message;
1045
+ const slot = shape ? null : unreadableSlot(ws, id);
1046
+ if (slot) { record(id, "invalid", { error: `${slot.said}; ${slot.remedy}`, errorCode: "E_SCHEDULE_INVALID" }); continue; }
921
1047
  if (!shape) validateActiveExecution(js, jobLockInfo(ws, id)); // Also covers host pre-observed candidates.
922
1048
  if (shape) { cronError = shape; due = false; }
923
1049
  else { try { due = only ? true : dueAt(def, minute); } catch (e) { cronError = e.message; due = false; } }
@@ -1086,6 +1212,8 @@ export function runNow(ws, qid, { now = new Date(), io, force = false, ctx = nul
1086
1212
  return withHostLock(() => {
1087
1213
  const st = readState(ws);
1088
1214
  const js = st.jobs[id] || (st.jobs[id] = {});
1215
+ const slot = unreadableSlot(ws, id);
1216
+ if (slot) throw scheduleError("E_SCHEDULE_INVALID", `schedule ${id}: ${slot.said}; ${slot.remedy}`);
1089
1217
  observePrevious(ws, id, def, js, io, now, { mutate: true });
1090
1218
  writeState(ws, st);
1091
1219
  if (js.attempt) throw scheduleError("E_SCHEDULE_UNRESOLVED", `schedule ${id} has a launch attempt without a recorded result; run oats schedule reconcile ${id} first`);
@@ -1110,12 +1238,21 @@ export function reconcile(ws, qid, { io, now = new Date(), clear = false, ctx =
1110
1238
  const id = scheduleKeyOf(qid);
1111
1239
  const defs = effectiveDefinitions(ws, ctx);
1112
1240
  const def = defs.jobs[id];
1241
+ if (!def && slotName(id) && (jobLockInfo(ws, id) || readState(ws).jobs[id]) && scheduleAbsence(id, { defs: readDefinitions(ws), snapshot: readSnapshot(ws) }).state === "gone") return reconcileOrphanSlot(ws, qid, id, { clear, ctx });
1113
1242
  if (!def || def.kind === "trigger") throw notHere(ws, qid, ctx);
1114
1243
  const describeHere = () => describe(ws, qid, io, { ctx });
1115
1244
  return withHostLock(() => {
1116
1245
  const st = readState(ws);
1117
1246
  const js = st.jobs[id] || (st.jobs[id] = {});
1118
- const attempt = js.attempt || (js.lastRun?.outcome === "unknown" ? { scheduledFor: js.lastRun.scheduledFor, startedAt: js.lastRun.startedAt, ...(js.lastRun.execution !== undefined ? { execution: js.lastRun.execution } : {}) } : undefined);
1247
+ const attempt = attemptOf(js);
1248
+ // A slot whose record cannot be read is released only once a person has checked (--clear);
1249
+ // an attempt beside it is then reconciled as any other.
1250
+ const slot = unreadableSlot(ws, id);
1251
+ if (slot && !clear) return { schedule: describeHere(), reconciled: "unknown", remedy: `${slot.said}; ${slot.remedy}` };
1252
+ if (slot) {
1253
+ releaseJobLock(ws, id);
1254
+ if (!attempt) return { schedule: describeHere(), reconciled: "cleared" };
1255
+ }
1119
1256
  if (!attempt) return { schedule: describeHere(), reconciled: "nothing to reconcile" };
1120
1257
  try { validateAttemptExecution(attempt, jobLockInfo(ws, id)); }
1121
1258
  catch (error) { throw scheduleError("E_SCHEDULE_INVALID", error.message); }
@@ -1146,7 +1283,7 @@ export function reconcile(ws, qid, { io, now = new Date(), clear = false, ctx =
1146
1283
  js.lastRun = { scheduledFor: attempt.scheduledFor, startedAt: js.lastRun?.startedAt || attempt.startedAt, kind: attemptKind, launched: true, ...adopted, outcome: seen.outcome, ...(seen.error ? { error: seen.error } : {}), ...(seen.note ? { note: seen.note } : {}) };
1147
1284
  if (seen.outcome !== "ended") { if (!jobLockInfo(ws, id)) acquireJobLock(ws, id, { scheduledFor: attempt.scheduledFor, reconciled: true }); } else releaseJobLock(ws, id);
1148
1285
  } else {
1149
- js.lastRun = { scheduledFor: attempt.scheduledFor, startedAt: js.lastRun?.startedAt || attempt.startedAt, kind: attemptKind, launched: false, outcome: "launch-failed", error: clear ? "cleared by the operator after checking by hand" : "no instance found in the scope's roster for the unrecorded attempt" };
1286
+ js.lastRun = launchFailedRun(js, attempt, attemptKind, clear);
1150
1287
  releaseJobLock(ws, id);
1151
1288
  }
1152
1289
  writeState(ws, st);
@@ -1154,6 +1291,37 @@ export function reconcile(ws, qid, { io, now = new Date(), clear = false, ctx =
1154
1291
  });
1155
1292
  }
1156
1293
 
1294
+ /** Reconcile what a schedule that is gone (scheduleAbsence) left: its slot (orphanSlot), its
1295
+ * retained state (orphanState), or both, read again under the host lock, as every reconcile.
1296
+ * Without `clear` it answers what it found and changes nothing. With `clear` it releases the slot,
1297
+ * resolves the job's retained attempt as cleared (as for a defined job: launchFailedRun) and
1298
+ * removes the job's state, and says which of the three it cleared (`cleared`). `schedule` is
1299
+ * null: there is no schedule to describe. */
1300
+ function reconcileOrphanSlot(ws, qid, id, { clear, ctx }) {
1301
+ return withHostLock(() => {
1302
+ const absence = scheduleAbsence(id, { defs: readDefinitions(ws), snapshot: readSnapshot(ws) });
1303
+ const st = readState(ws), js = st.jobs[id];
1304
+ const slot = absence.state === "gone" ? orphanSlot(ws, id, "", absence.absent) : null;
1305
+ const words = slot ?? (absence.state === "gone" && js ? orphanState(id, js, absence.absent) : null);
1306
+ if (!words) throw notHere(ws, qid, ctx);
1307
+ if (!clear) return { schedule: null, reconciled: "unknown", remedy: `${words.said}; ${words.remedy}` };
1308
+ const attempt = js ? attemptOf(js) : undefined;
1309
+ if (attempt) {
1310
+ try { validateAttemptExecution(attempt, jobLockInfo(ws, id)); }
1311
+ catch (error) { throw scheduleError("E_SCHEDULE_INVALID", error.message); }
1312
+ }
1313
+ const cleared = [];
1314
+ if (slot) { releaseJobLock(ws, id); cleared.push("slot"); }
1315
+ if (js) {
1316
+ if (attempt) { delete js.attempt; js.lastRun = launchFailedRun(js, attempt, js.lastRun?.kind ?? null, true); cleared.push("attempt"); }
1317
+ delete st.jobs[id];
1318
+ writeState(ws, st);
1319
+ cleared.push("state");
1320
+ }
1321
+ return { schedule: null, reconciled: "cleared", cleared };
1322
+ });
1323
+ }
1324
+
1157
1325
  /** Whether a spawn job's runs are named `<agent>-<purpose>`, so a home's absence proves a lost
1158
1326
  * receipt launched nothing: only for a bare name of a member soul under the agents root, whose
1159
1327
  * agent name is that name. Spawn names a run after the soul's stem (core.mjs
@@ -1175,6 +1343,17 @@ function namedHomes(ws, js) {
1175
1343
  * authorization. In particular, do not call placementOf here: it probes gh. */
1176
1344
  function workspaceReconcileRemedy(id, snapshot, local, localError) {
1177
1345
  const preserve = "Preserve the retained state and inspect its effects with the deployment owner; this diagnostic does not clear attempts or slots.";
1346
+ const { excluded, unknown } = workspacePlacement(id, snapshot, local, localError);
1347
+ if (excluded) return `Retained-state reconciliation is currently unavailable under this placement: ${excluded}. ${preserve}`;
1348
+ if (unknown) return `Workspace authorization cannot be verified offline: ${unknown}. ${preserve}`;
1349
+ return `Current workspace authorization is unverified offline; reconciliation requires current host, owner and trust authorization. ${preserve}`;
1350
+ }
1351
+ /** Whether this host runs the workspace schedule `id`, as far as it can tell offline from the saved
1352
+ * `snapshot` and its own oats-local.yaml (`local`, or `localError`), without asking gh who it is
1353
+ * → { excluded }: it does not (the clause says why: disabled, not named, placed on another host,
1354
+ * untrusted, not in the snapshot); { unknown }: that cannot be read (the clause says why); or {}:
1355
+ * nothing offline excludes it, and reconcile still checks its owner. */
1356
+ function workspacePlacement(id, snapshot, local, localError) {
1178
1357
  let unknown, excluded;
1179
1358
  const entries = snapshot?.schedules;
1180
1359
  const matches = entries?.filter((entry) => entry?.id === id) || [];
@@ -1192,14 +1371,13 @@ function workspaceReconcileRemedy(id, snapshot, local, localError) {
1192
1371
  else if (entry.runsOn !== host.name) excluded = `the saved schedule is assigned elsewhere (${entry.runsOn}); this host is ${host.name}`;
1193
1372
  else if (!trusted(host, id)) excluded = "the schedule is untrusted by local automations.trust";
1194
1373
  }
1195
- if (excluded) return `Retained-state reconciliation is currently unavailable under this placement: ${excluded}. ${preserve}`;
1196
- if (unknown) return `Workspace authorization cannot be verified offline: ${unknown}. ${preserve}`;
1197
- return `Current workspace authorization is unverified offline; reconciliation requires current host, owner and trust authorization. ${preserve}`;
1374
+ return { excluded, unknown };
1198
1375
  }
1199
1376
 
1200
- /** `oats doctor`: retained launch attempts and unknown last observations, in this
1201
- * deployment and the other scopes sharing its slots. Reads saved state only;
1202
- * workspace guidance cannot promise authorization or an executable recovery. */
1377
+ /** `oats doctor`: retained launch attempts and unknown last observations, and the slots whose
1378
+ * record cannot be read (SCHEDULE_SLOT_UNREADABLE, unreadableSlot), in this deployment and the
1379
+ * other scopes sharing its slots. Reads saved state only; workspace guidance cannot promise
1380
+ * authorization or an executable recovery. */
1203
1381
  export function unresolvedScheduleAttempts(dep, { now = new Date() } = {}) {
1204
1382
  const problems = [], information = [];
1205
1383
  const here = resolve(dep);
@@ -1207,9 +1385,10 @@ export function unresolvedScheduleAttempts(dep, { now = new Date() } = {}) {
1207
1385
  try { scopes = [here, ...readRegistry().workspaces.filter((w) => resolve(w) !== here && existsSync(w))]; }
1208
1386
  catch (e) { information.push(`the host schedule registry could not be read, so only this deployment's schedules were checked: ${e.message}`); }
1209
1387
  for (const ws of scopes) {
1210
- let defs, st;
1388
+ const dir = ws === here ? "" : ` --dir ${/^[\w./@%+=:,-]+$/.test(ws) ? ws : shq(ws)}`;
1389
+ let defs, st, unread = null;
1211
1390
  try { defs = readDefinitions(ws); st = readState(ws); }
1212
- catch (e) { information.push(`the schedules of ${ws} could not be read: ${e.message}`); continue; }
1391
+ catch (e) { unread = e; }
1213
1392
  // Read the last discovery snapshot directly: placement and refresh would
1214
1393
  // require credentials/network, and unresolved work matters even when offline.
1215
1394
  const snapshot = readSnapshot(ws);
@@ -1220,6 +1399,40 @@ export function unresolvedScheduleAttempts(dep, { now = new Date() } = {}) {
1220
1399
  if (resolve(dirname(found.path)) !== resolve(ws)) throw scheduleError("E_LOCAL_MISSING", "this scope has no local configuration");
1221
1400
  local = found.local;
1222
1401
  } catch (e) { localError = e; }
1402
+ // Every slot there is, as liveLocks counts them: the tick reports an unreadable one of a
1403
+ // defined job invalid, and reads none of a schedule that is not defined. Definitions that
1404
+ // cannot be read say nothing of which it is.
1405
+ const slots = join(stateDir(ws), "locks");
1406
+ let slotEntries = [];
1407
+ try { slotEntries = readdirSync(slots, { withFileTypes: true }).filter((e) => e.isDirectory()).map((e) => e.name).sort(); }
1408
+ catch (e) { if (e.code !== "ENOENT") information.push(`the schedule slots of ${ws} could not be read: ${e.message}`); }
1409
+ for (const key of slotEntries) {
1410
+ const id = shownScheduleId(key), path = lockDir(ws, key);
1411
+ const absence = unread !== null ? { state: "defined" } : scheduleAbsence(key, { defs, snapshot });
1412
+ const where = ws === here ? "" : ` (in ${ws})`;
1413
+ const orphan = absence.state === "defined" ? null : orphanSlot(ws, key, dir, absence.absent);
1414
+ const slot = absence.state === "defined" ? unreadableSlot(ws, key, dir) : null;
1415
+ const counted = `its slot ${path} ${orphan?.found}, and the slot counts against maxConcurrent`;
1416
+ const untold = `which is not in the workspace's snapshot of ${absence.takenAt}`;
1417
+ if (absence.state === "gone") problems.push({ code: SCHEDULE_SLOT_ORPHANED, severity: "warning", scope: ws, id,
1418
+ message: `schedule ${id}${where}, which ${absence.absent}: ${counted}`, remedy: orphan.remedy });
1419
+ if (absence.state === "member-absent") problems.push({ code: SCHEDULE_SLOT_ORPHANED, severity: "warning", scope: ws, id,
1420
+ message: `schedule ${id}${where}, ${untold}: its member ${absence.member} is not in the workspace, or was not read, so whether the schedule is gone cannot be told; ${counted}`,
1421
+ remedy: `check oats status for a run of schedule ${id}; once none is running, remove the slot by hand: rm -r -- ${shq(path)}` });
1422
+ if (absence.state === "member-problem") problems.push({ code: SCHEDULE_SLOT_ORPHANED, severity: "warning", scope: ws, id,
1423
+ message: `schedule ${id}${where}, ${untold}: ${absence.path ? `member ${absence.member} has a schedule file that could not be read (${absence.path}), which could be this schedule` : `member ${absence.member} could not be listed`}, so whether it is gone cannot be told; ${counted}`,
1424
+ remedy: `${absence.path ? `fix ${absence.path} in member ${absence.member}` : `make member ${absence.member} readable`} and run oats automations refresh; doctor then says whether schedule ${id} is gone` });
1425
+ // A workspace schedule this host does not run, or cannot tell it runs, is refused by reconcile
1426
+ // here: its slot is removed by hand once nothing of it runs.
1427
+ const placement = slot && key.includes("~") ? workspacePlacement(id, snapshot, local, localError) : {};
1428
+ const byHand = `once oats status shows no run of schedule ${id} here, remove the slot by hand: rm -r -- ${shq(path)}`;
1429
+ if (slot) problems.push({ code: SCHEDULE_SLOT_UNREADABLE, severity: "warning", scope: ws, id,
1430
+ message: placement.excluded ? `schedule ${id}${where}: its slot ${path} has no readable record, and the slot counts against maxConcurrent; this host does not run the schedule (${placement.excluded}), so reconcile here refuses it`
1431
+ : placement.unknown ? `schedule ${id}${where}: its slot ${path} has no readable record, and the slot counts against maxConcurrent; whether this host runs the schedule cannot be told offline (${placement.unknown})`
1432
+ : `schedule ${id}${where}: its slot ${path} has no readable record; the tick reports this job invalid and starts nothing of it, and the slot counts against maxConcurrent`,
1433
+ remedy: placement.excluded || placement.unknown ? byHand : slot.remedy });
1434
+ }
1435
+ if (unread) { information.push(`the schedules of ${ws} could not be read: ${unread.message}`); continue; }
1223
1436
  for (const [key, js] of Object.entries(st.jobs)) {
1224
1437
  if (!js?.attempt && js?.lastRun?.outcome !== "unknown") continue;
1225
1438
  // A workspace schedule's state key is <member>~<id> (scheduleStateKey); a local one is its id.
@@ -1235,23 +1448,27 @@ export function unresolvedScheduleAttempts(dep, { now = new Date() } = {}) {
1235
1448
  const holdsSlot = !!lock;
1236
1449
  const error = js.attempt?.error ?? js.lastRun?.error ?? null;
1237
1450
  const unproven = (kind === "command" || kind === "operation") && namedHomes(ws, js).length !== 1;
1238
- const dir = ws === here ? "" : ` --dir ${/^[\w./@%+=:,-]+$/.test(ws) ? ws : shq(ws)}`;
1239
1451
  let unavailable = localError ? `local configuration is invalid or unreadable (${localError.code || "read error"})` : null;
1240
1452
  if (!workspace && !unavailable) {
1241
1453
  unavailable = !SCHEDULE_ID_RE.test(id) ? "the stored schedule ID is invalid" : shapeError(defs.jobs[key]);
1242
1454
  if (!unavailable && defs.jobs[key].id !== undefined && defs.jobs[key].id !== key) unavailable = "the stored definition has a different ID";
1243
1455
  try { validateAttemptExecution(js.attempt || { execution: js.lastRun?.execution }, lock); }
1244
1456
  catch { unavailable ||= "the retained attempt or lock uses unsupported captured execution state"; }
1245
- if (lock?.unreadable) unavailable ||= "job lock custody is unreadable";
1246
1457
  }
1247
- const remedy = workspace ? workspaceReconcileRemedy(id, snapshot, local, localError)
1458
+ const slot = workspace || unavailable ? null : unreadableSlot(ws, key, dir);
1459
+ // A workspace schedule that is gone: reconcile --clear resolves its slot and state.
1460
+ const absence = workspace ? scheduleAbsence(key, { defs, snapshot }) : null;
1461
+ const orphan = absence?.state === "gone" ? orphanSlot(ws, key, dir, absence.absent) ?? orphanState(key, js, absence.absent, dir) : null;
1462
+ const remedy = orphan ? orphan.remedy
1463
+ : workspace ? workspaceReconcileRemedy(id, snapshot, local, localError)
1248
1464
  : unavailable ? `Doctor does not recommend reconciling this job because ${unavailable}; preserve the retained state and inspect its effects with the deployment owner.`
1465
+ : slot ? slot.remedy
1249
1466
  : `oats schedule reconcile ${id}${unproven ? " --clear" : ""}${dir}`;
1250
1467
  const subject = js.attempt ? `${kind ? `a ${kind}` : "a"} launch attempt for ${scheduledFor} has no recorded result`
1251
1468
  : `the last ${kind || "schedule"} run for ${scheduledFor} has an unknown observation`;
1252
1469
  const blocking = js.attempt ? "This persisted attempt blocks another run until it is reconciled"
1253
1470
  : "There is no persisted launch attempt blocking execution; an eligible tick may re-observe the last run, without a guarantee of clearance";
1254
- const guidance = workspace || unavailable ? remedy : unproven ? `Its effects can't be proven: check the roster and the host by hand, then run ${remedy}` : `Run ${remedy}`;
1471
+ const guidance = orphan ? `${orphan.said}; ${remedy}` : workspace || unavailable ? remedy : slot ? `${slot.said}: ${remedy}` : unproven ? `Its effects can't be proven: check the roster and the host by hand, then run ${remedy}` : `Run ${remedy}`;
1255
1472
  const message = `schedule ${id}${ws === here ? "" : ` (in ${ws})`}: ${subject} (started ${startedAt}${ageSeconds === null ? "" : `, ${formatAge(ageSeconds)} ago`})${error ? `: ${error}` : ""}. ${blocking}, and ${holdsSlot ? "it holds a host slot (counted against maxConcurrent)" : "it holds no host slot"}. ${guidance}`;
1256
1473
  problems.push({ code: "schedule-unresolved", severity: "warning", scope: ws, id, kind, scheduledFor, startedAt, ageSeconds, holdsSlot, exited: js.attempt?.exited === true, error, remedy, message });
1257
1474
  }
package/lib/servers.mjs CHANGED
@@ -17,7 +17,7 @@
17
17
  */
18
18
 
19
19
  import { execFile, execFileSync, spawnSync } from "node:child_process";
20
- import { createHash, randomBytes } from "node:crypto";
20
+ import { createHash, randomBytes, randomUUID } from "node:crypto";
21
21
  import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
22
22
  import { homedir } from "node:os";
23
23
  import { basename, join, resolve } from "node:path";
@@ -34,6 +34,7 @@ import { shellWord } from "./shell-word.mjs";
34
34
  import { DETACH_KEY_FEATURE, requireDetachKey } from "./session-viewer.mjs";
35
35
  import { LOGIN_CAPTURE_MAX_BYTES, LOGIN_SHELLS, loginPathEmitter, quoteFor, readLoginFrame } from "./login-environment.mjs";
36
36
  import { compareSemver } from "./semver.mjs";
37
+ import { SPAWN_TASK_STDIN_FEATURE } from "./kernel-features.mjs";
37
38
  import { sameRepo } from "./repo-moves.mjs";
38
39
  import { OATS_PACKAGE_NAME } from "./version.mjs";
39
40
  export { compareSemver } from "./semver.mjs";
@@ -653,10 +654,21 @@ export function listSnapshots(serverId) {
653
654
 
654
655
  // ------------------------------------------------------------------- routes
655
656
 
657
+ /** What a host must advertise for a routed spawn to be bound to its preview and a key: the preview
658
+ * with its decision, the apply that takes one, and the key's recovery before placement. */
659
+ const KEYED_SPAWN_FEATURES = ["spawn-preview-2", "spawn-apply-2", "spawn-idempotency-2"];
660
+
656
661
  /** Route one lifecycle command to a server. Returns the remote envelope,
657
662
  * having kept the local snapshot store in step with it:
658
663
  * - spawn: compatibility checked first, a snapshot written on success; a
659
664
  * --preview runs the same checks, needs spawn-preview-2 and writes nothing;
665
+ * a spawn with no decision and no key of its own is previewed on the host
666
+ * and applied bound to that decision and a key (KEYED_SPAWN_FEATURES); a
667
+ * `--task <text>` does not travel as an argument, where it would be in the
668
+ * command line of the local ssh and of the host's oats: each spawn call
669
+ * carries `--task-file -` in its place and the text on its standard input
670
+ * (SPAWN_TASK_STDIN_FEATURE). A host without the feature gets the task as
671
+ * an argument, and the answer carries a warning that says so;
660
672
  * - retire: the target comes from the instance's snapshot when one exists
661
673
  * (the registry may have changed since), the snapshot is removed only
662
674
  * when the remote kernel reports the home gone;
@@ -688,6 +700,23 @@ export function routeCommand(serverId, cmd, oatsArgs, io = {}) {
688
700
  throw serverError("E_ROUTE_CHANGED", `server ${serverId} now points at ${target.sshHost}:${target.workspace}, but ${priorTargets.length} saved route${priorTargets.length === 1 ? "" : "s"} for it (${priorTargets.map((s) => s.instance).join(", ")}) target ${old.sshHost}:${old.workspace}; spawning would overwrite them. Keep the old registration and add a new server id for the new target, or retire those instances first`);
689
701
  }
690
702
  const remote = checkRemote(target, io);
703
+ // For a host that reads a task from its standard input (SPAWN_TASK_STDIN_FEATURE), the task
704
+ // leaves the arguments here, before any check reads them or a spawn call is built. The first
705
+ // --task is the one a spawn reads; every one is taken out. A host without the feature gets
706
+ // the arguments as given, and its answer says where the task was readable.
707
+ let task, taskAsArgument;
708
+ const taskAt = oatsArgs.indexOf("--task");
709
+ if (taskAt >= 0 && oatsArgs[taskAt + 1] !== undefined && !remote.features.includes(SPAWN_TASK_STDIN_FEATURE)) {
710
+ taskAsArgument = `remote oats ${remote.version} at ${target.sshHost} does not advertise ${SPAWN_TASK_STDIN_FEATURE}, so the task was passed as an argument, visible in this machine's process list (the ssh command) and the host's while the spawn runs; upgrade the host`;
711
+ } else if (taskAt >= 0 && oatsArgs[taskAt + 1] !== undefined) {
712
+ task = Buffer.from(oatsArgs[taskAt + 1]);
713
+ const kept = [];
714
+ for (let i = 0; i < oatsArgs.length; i++) {
715
+ if (oatsArgs[i] !== "--task" || oatsArgs[i + 1] === undefined) kept.push(oatsArgs[i]);
716
+ else { if (i === taskAt) kept.push("--task-file", "-"); i++; }
717
+ }
718
+ oatsArgs = kept;
719
+ }
691
720
  // A saved route is keyed by name under its server id. An explicit name
692
721
  // (--name, or --instance) that already has one here would overwrite that
693
722
  // route on success: refuse before the mutation (the roster shows the
@@ -717,7 +746,50 @@ export function routeCommand(serverId, cmd, oatsArgs, io = {}) {
717
746
  if (oatsArgs.includes("--background") || oatsArgs.includes("--no-background")) requireRemoteFeature(remote, target, "instance-background");
718
747
  // A host without instance-same-work would refuse the flag as unknown: it is never sent there.
719
748
  if (oatsArgs.includes("--same-work")) requireRemoteFeature(remote, target, "instance-same-work");
720
- const { envelope, stderr } = runRemote(target, json(withScope(["spawn", ...hostHarnessArgs(remote, oatsArgs)])), io);
749
+ const spawnOnHost = (args) => runRemote(target, json(withScope(["spawn", ...hostHarnessArgs(remote, args)])), { ...io, input: task });
750
+ // A spawn that brings neither a decision nor a key is bound here: the host's preview decides, and
751
+ // the apply carries that decision and a key of this request, so the same spawn sent again with
752
+ // both answers the instance this one created. A host that does not advertise the keyed spawn
753
+ // gets the spawn as given, and its answer says what that costs.
754
+ const given = (flag) => oatsArgs.some((a) => a === flag || a.startsWith(`${flag}=`));
755
+ const suggesting = oatsArgs.includes("--suggest-name");
756
+ let applyArgs = oatsArgs, bound, unbound, refusal;
757
+ if (!preview && !given("--expect-decision") && !given("--idempotency-key")) {
758
+ // The decision binds a name, so a suggested one is chosen by the preview and applied as --name.
759
+ const missing = [...KEYED_SPAWN_FEATURES, ...(suggesting ? ["spawn-suggest-name", "spawn-name"] : [])].filter((f) => !remote.features.includes(f));
760
+ if (missing.length) unbound = `remote oats ${remote.version} at ${target.sshHost} does not advertise ${missing.join(", ")}: if the link drops while it spawns, the spawn cannot be retried safely (the instance may exist there, and a second run would create another); check oats status --server ${serverId} first, or upgrade oats there`;
761
+ else {
762
+ const previewed = spawnOnHost([...oatsArgs, "--preview"]);
763
+ const shown = previewed.envelope.result;
764
+ // Anything but a preview with its decision is the host's answer to this spawn (a refusal).
765
+ if (!(previewed.envelope.ok && shown?.preview === true && typeof shown.decision?.revision === "string")) refusal = previewed;
766
+ else {
767
+ bound = { decision: shown.decision, idempotencyKey: randomUUID(), naming: shown.naming };
768
+ applyArgs = [...(suggesting ? [...oatsArgs.filter((a) => a !== "--suggest-name"), "--name", shown.instance] : oatsArgs), "--expect-decision", shown.decision.revision, "--idempotency-key", bound.idempotencyKey];
769
+ }
770
+ }
771
+ }
772
+ let answer = refusal;
773
+ try { answer ??= spawnOnHost(applyArgs); }
774
+ catch (e) {
775
+ // The link failed under a bound apply: the host may have started it. The decision and the key
776
+ // travel with the failure, so the spawn sent again lands on that one instance.
777
+ if (bound && e.code === "E_SSH" && e.details?.sshStarted !== false) {
778
+ const fix = `run the same spawn again with ${suggesting ? `--name ${bound.decision.instance} in place of --suggest-name, and ` : ""}--expect-decision ${bound.decision.revision} --idempotency-key ${bound.idempotencyKey}: it answers the instance this attempt created there, or creates it`;
779
+ e.message = `${e.message}; the host may have started the spawn: ${fix}`;
780
+ e.details = { ...e.details, decision: bound.decision, idempotencyKey: bound.idempotencyKey, fix };
781
+ }
782
+ throw e;
783
+ }
784
+ const { stderr } = answer;
785
+ let { envelope } = answer;
786
+ if (envelope.ok && envelope.result) {
787
+ const added = [unbound, taskAsArgument].filter(Boolean);
788
+ const warnings = added.length ? { warnings: [...(envelope.result.warnings || []), ...added] } : {};
789
+ // How the preview chose the name is the spawn's own answer to --suggest-name.
790
+ const naming = bound?.naming && envelope.result.naming === undefined ? { naming: bound.naming } : {};
791
+ envelope = { ...envelope, result: { ...envelope.result, ...warnings, ...naming } };
792
+ }
721
793
  if (preview && envelope.ok && envelope.result?.preview === true && envelope.result.spawnPreviewApi === 2) {
722
794
  // The host's preview with the two fields a routed spawn adds; never a
723
795
  // `snapshot` key, which on the apply path says what was (or was not) saved.
@@ -1304,13 +1376,19 @@ export function scheduleRemote(serverId, oatsArgs, io = {}) {
1304
1376
  const setsHostCaps = oatsArgs[0] === "host" && oatsArgs[1] === "install" && oatsArgs.some((arg) =>
1305
1377
  ["--max-concurrent", "--triggers-max-concurrent"].some((name) => arg === name || arg.startsWith(`${name}=`)));
1306
1378
  const capsUnsupported = () => serverError("E_REMOTE_INCOMPATIBLE", `remote oats at ${target.sshHost} (${serverId}) does not confirm required features schedule and schedule-host-caps; upgrade oats on that destination before setting or resetting host caps`);
1379
+ // `host install --register-only` installs no timer. A host that does not know the option
1380
+ // ignores it and installs one, so it is sent only to a host that advertises it.
1381
+ const registersOnly = oatsArgs[0] === "host" && oatsArgs[1] === "install" && oatsArgs.includes("--register-only");
1382
+ const registerOnlyUnsupported = () => serverError("E_REMOTE_INCOMPATIBLE", `remote oats at ${target.sshHost} (${serverId}) does not confirm required features schedule and schedule-host-register-only, so --register-only was not sent: a host without it ignores the option and installs a timer; upgrade oats on that destination`);
1307
1383
  let remote;
1308
1384
  try {
1309
1385
  remote = checkRemote(target, io);
1386
+ if (registersOnly && !["schedule", "schedule-host-register-only"].every((feature) => remote.features.includes(feature))) throw registerOnlyUnsupported();
1310
1387
  if (!remote.features.includes("schedule")) throw serverError("E_REMOTE_INCOMPATIBLE", `remote oats ${remote.version} at ${target.sshHost} does not advertise schedules; upgrade it there`);
1311
1388
  } catch (e) {
1312
- // Unknown probe formats cannot establish cap support. Keep transport
1313
- // failures distinct, and preserve the existing contract for no-cap calls.
1389
+ // Unknown probe formats cannot establish support for an option. Keep transport
1390
+ // failures distinct, and preserve the existing contract for calls without one.
1391
+ if (registersOnly && ["E_REMOTE_ENVELOPE", "E_REMOTE_INCOMPATIBLE"].includes(e.code)) throw registerOnlyUnsupported();
1314
1392
  if (setsHostCaps && ["E_REMOTE_ENVELOPE", "E_REMOTE_INCOMPATIBLE"].includes(e.code)) throw capsUnsupported();
1315
1393
  throw e;
1316
1394
  }
@@ -1544,7 +1622,7 @@ export function connectServer(options, io = {}) {
1544
1622
  const searched = login ? `the PATH of ${sshHost}'s login shell (${basename(login.shell)} -l -i)` : path ? "--path and ssh's PATH" : "ssh's PATH";
1545
1623
  const where = login ? `so that its login shell finds it (there, ${runnable([login.shell, "-l", "-i", "-c", "command -v npm"])} prints its path)`
1546
1624
  : path ? "where --path or a non-interactive ssh command finds it" : "where a non-interactive ssh command finds it";
1547
- return { status: "needs-human", detail: `${what}, and no npm on ${searched}`, remedy: `install Node.js 22+ (npm) on ${sshHost} ${where}, then check again (run the same connect); from the CLI, --path <the directory holding npm> names it instead` };
1625
+ return { status: "needs-human", detail: `${what}, and no npm on ${searched}`, remedy: `install Node.js 22.5+ (npm) on ${sshHost} ${where}, then check again (run the same connect); from the CLI, --path <the directory holding npm> names it instead` };
1548
1626
  }
1549
1627
  const install = ["npm", "install", "-g", `${OATS_PACKAGE_NAME}@${localVersion}`];
1550
1628
  if (!installOats) return { status: "needs-human", detail: what, remedy: `on ${sshHost}: ${runnable(install)} (or pass --install-oats)` };
@@ -7,6 +7,9 @@
7
7
  * - `.oats-stop.json`: `oats instance stop` / `oats session stop`, a retire and a restart (before
8
8
  * they end the session), `{ instance, at, stop, retained, startId }`; `startId` (0.69) is the
9
9
  * start the stop ended, null when the home had none.
10
+ * A stop with the host (`oats host run` when the host stops: lib/host-run.mjs) writes the same
11
+ * record with `resume: true` and `reason: "host-stop"`, before it signals the harness: OATS
12
+ * ended that session only because the host was going down, and a resume brings it back.
10
13
  * A reboot or a killed tmux server leaves neither file for the start that was running: that is a
11
14
  * lost session, the one a resume brings back. Dependency-free: status, start --resume and the
12
15
  * schedule runner read the files through here. */
@@ -16,6 +19,8 @@ import { SESSION_END } from "./session-resume-contract.mjs";
16
19
 
17
20
  export const START_EXITED_FILE = ".oats-start-exited";
18
21
  export const STOP_RECEIPT_FILE = ".oats-stop.json";
22
+ /** `reason` in the stop record of a stop with the host. */
23
+ export const HOST_STOP_REASON = "host-stop";
19
24
  const RECORD_MAX = 64 * 1024;
20
25
  /** A shell's exit status from 128 up is a signal's (128 + n): the harness was killed, not ended. */
21
26
  const SIGNALLED = 128;
@@ -59,9 +64,21 @@ export function currentStartTime(meta) {
59
64
  return Date.parse(typeof last === "string" ? last : meta?.createdAt);
60
65
  }
61
66
 
67
+ /** The stop record of `home` when it is of `meta`'s current start (`meta.startId`), else null: it
68
+ * names this start (`startId` equal), or, a record without a start id, was written after this
69
+ * start began. */
70
+ function stopOfCurrentStart(home, meta) {
71
+ const startId = typeof meta?.startId === "string" && meta.startId ? meta.startId : null;
72
+ const stop = readStopReceipt(home);
73
+ if (!stop) return null;
74
+ if (typeof stop.startId === "string" && stop.startId) return stop.startId === startId ? stop : null;
75
+ return Date.parse(stop.at) > currentStartTime(meta) ? stop : null;
76
+ }
77
+
62
78
  /** How the session of `meta`'s current start (`meta.startId`) ended, from the home's files alone:
63
- * - `stopped`: the stop receipt names this start (`startId` equal), or, a receipt without a start
64
- * id, was written after this start began;
79
+ * - `stopped`: the stop record is of this start (stopOfCurrentStart) and asks for no resume;
80
+ * - `lost`, whatever the exit marker says: the stop record is of this start and says
81
+ * `resume: true` (a stop with the host): the harness ended because the host did;
65
82
  * - `exited`: the exit marker names this start with a status below 128, or in its legacy form
66
83
  * (no status: a kernel before 0.69 wrote it only when the harness ended on its own window);
67
84
  * - `lost`: anything else — a marker with a signal's status (128 and up), or no file naming this
@@ -71,13 +88,16 @@ export function currentStartTime(meta) {
71
88
  * The caller reads it only for a launched home that does not run. */
72
89
  export function sessionEndOf(home, meta) {
73
90
  const startId = typeof meta?.startId === "string" && meta.startId ? meta.startId : null;
74
- const since = currentStartTime(meta);
75
- const stop = readStopReceipt(home);
76
- if (stop) {
77
- if (typeof stop.startId === "string" && stop.startId) { if (stop.startId === startId) return SESSION_END.STOPPED; }
78
- else if (Date.parse(stop.at) > since) return SESSION_END.STOPPED;
79
- }
91
+ const stop = stopOfCurrentStart(home, meta);
92
+ if (stop) return stop.resume === true ? SESSION_END.LOST : SESSION_END.STOPPED;
80
93
  const exited = startId ? readStartExited(home) : null;
81
94
  if (exited && exited.startId === startId) return exited.status !== null && exited.status >= SIGNALLED ? SESSION_END.LOST : SESSION_END.EXITED;
82
95
  return SESSION_END.LOST;
83
96
  }
97
+
98
+ /** Why the session of `meta`'s current start reads `lost`, where the home says: HOST_STOP_REASON
99
+ * for a stop with the host; null for every other loss, and for a session that is not lost. */
100
+ export function lostReasonOf(home, meta) {
101
+ const stop = stopOfCurrentStart(home, meta);
102
+ return stop?.resume === true && stop.reason === HOST_STOP_REASON ? HOST_STOP_REASON : null;
103
+ }