@awebai/oats 0.69.12 → 0.69.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/README.md +216 -178
  2. package/bin/oats.mjs +254 -201
  3. package/docs/capabilities.md +58 -13
  4. package/docs/configuration.md +63 -5
  5. package/docs/container.md +291 -0
  6. package/docs/design/2026-09-23-workspace-module-contracts.md +2 -0
  7. package/docs/desktop-cli-api.md +590 -55
  8. package/docs/desktop-instance-start.md +4 -1
  9. package/docs/desktop.md +58 -12
  10. package/docs/execution-targets.md +84 -9
  11. package/docs/first-team.md +4 -3
  12. package/docs/implementation.md +99 -32
  13. package/docs/integrations.md +3 -2
  14. package/docs/knowledge-and-learning.md +72 -0
  15. package/docs/oats-local.schema.json +11 -2
  16. package/docs/official-catalog.md +39 -5
  17. package/docs/packages.md +93 -14
  18. package/docs/release-lane.md +2 -1
  19. package/docs/release-notes/v0.69.13.md +767 -0
  20. package/docs/release-notes/v0.69.14.md +83 -0
  21. package/docs/release-notes/v0.69.15.md +444 -0
  22. package/docs/schedules.md +159 -5
  23. package/docs/servers.md +241 -7
  24. package/docs/souls-and-instances.md +223 -14
  25. package/docs/tui.md +1 -1
  26. package/docs/upgrading.md +36 -5
  27. package/docs/workspaces.md +31 -5
  28. package/lib/claim.mjs +20 -5
  29. package/lib/command-flags.mjs +240 -0
  30. package/lib/core.mjs +1004 -633
  31. package/lib/deployments.mjs +4 -3
  32. package/lib/dir-lock.mjs +145 -33
  33. package/lib/errors.mjs +3 -2
  34. package/lib/harness-token.mjs +110 -0
  35. package/lib/host-run.mjs +218 -0
  36. package/lib/host-stop-child.mjs +21 -0
  37. package/lib/hot-reads.mjs +4 -3
  38. package/lib/instance-events.mjs +28 -6
  39. package/lib/instance-git.mjs +165 -14
  40. package/lib/instance-inspect.mjs +6 -1
  41. package/lib/instance-lifecycle.mjs +4 -3
  42. package/lib/instance-listing.mjs +114 -43
  43. package/lib/instance-resolution.mjs +56 -3
  44. package/lib/instance-upgrade.mjs +16 -20
  45. package/lib/kernel-features.mjs +4 -1
  46. package/lib/launch-env-exec.sh +7 -0
  47. package/lib/launch-env.mjs +74 -0
  48. package/lib/launch-preference.mjs +19 -12
  49. package/lib/member-clone.mjs +7 -1
  50. package/lib/packages.mjs +74 -2
  51. package/lib/read-call.mjs +4 -1
  52. package/lib/reader.mjs +3 -2
  53. package/lib/refused-text.mjs +3 -2
  54. package/lib/remote.mjs +126 -31
  55. package/lib/retained-after-retire.mjs +2 -7
  56. package/lib/retire-git-forms.mjs +41 -0
  57. package/lib/retire-incomplete.mjs +96 -0
  58. package/lib/schedule.mjs +239 -21
  59. package/lib/servers.mjs +161 -28
  60. package/lib/session-end.mjs +28 -8
  61. package/lib/session-resume.mjs +63 -56
  62. package/lib/session-seen.mjs +3 -14
  63. package/lib/start-env.mjs +13 -7
  64. package/lib/status-words.mjs +11 -4
  65. package/lib/system-tools.mjs +1 -36
  66. package/lib/teams.mjs +11 -4
  67. package/lib/upgrade-state.mjs +13 -2
  68. package/lib/worktree-hooks.mjs +6 -175
  69. package/lib/worktree.mjs +68 -10
  70. package/package-catalog.json +1 -1
  71. package/package.json +5 -1
  72. package/packages/client/instance-lifecycle.mjs +1 -1
  73. package/packages/client/lifecycle-contract.mjs +3 -1
  74. package/packages/client/session-resume-cli.mjs +2 -1
  75. package/packages/client/session-start.mjs +10 -7
  76. package/packages/client/workspace-cli.mjs +5 -1
  77. package/packages/record/README.md +56 -21
  78. package/packages/record/docs/capture-service.md +8 -5
  79. package/packages/record/docs/turn-record-sot.md +9 -0
  80. package/packages/record/lib/capture-health.mjs +6 -4
  81. package/packages/record/lib/capture-lock.mjs +31 -27
  82. package/packages/record/lib/capture-retained-scratch.mjs +4 -3
  83. package/packages/record/lib/capture-scratch.mjs +2 -1
  84. package/packages/record/lib/derived-state.mjs +4 -2
  85. package/packages/record/lib/index-db.mjs +4 -2
  86. package/packages/record/lib/lock-holder.mjs +21 -0
  87. package/packages/record/lib/private-files.mjs +18 -0
  88. package/packages/record/lib/process-identity.mjs +310 -0
  89. package/packages/record/lib/store.mjs +31 -25
  90. package/share/oats-login-receive +204 -0
  91. package/skills/oats-getting-started/SKILL.md +93 -26
  92. package/lib/tmux-config.mjs +0 -35
package/lib/schedule.mjs CHANGED
@@ -22,7 +22,7 @@
22
22
  * `unknown`, its job blocked until `reconcile` proves what happened; it keeps
23
23
  * its host slot too, unless it is a command whose process exit was observed. */
24
24
  import { runScheduleCommand } from "./schedule-command.mjs";
25
- import { closeSync, constants as fsConstants, existsSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, realpathSync, rmSync, writeFileSync } from "node:fs";
25
+ import { closeSync, constants as fsConstants, existsSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
26
26
  import { createHash } from "node:crypto";
27
27
  import { homedir } from "node:os";
28
28
  import { basename, dirname, isAbsolute, join, resolve, sep } from "node:path";
@@ -284,7 +284,8 @@ export function writeHostState(st) { writeJson(join(hostScheduleDir(), "state.js
284
284
 
285
285
  // ---------------------------------------------------------------- locks
286
286
 
287
- /** Schedule locks (lib/dir-lock.mjs): a holder still there after the retry is E_SCHEDULER_BUSY. */
287
+ /** Schedule locks (lib/dir-lock.mjs): a holder still running after the retry is E_SCHEDULER_BUSY, and so is
288
+ * a lock that cannot be judged; one left by a process that is gone is taken over. `dir` is the lock's file. */
288
289
  function withDirLock(dir, what, fn, { retryMs = 0 } = {}) {
289
290
  return withSharedDirLock(dir, what, fn, { retryMs, busy: (why) => scheduleError("E_SCHEDULER_BUSY", `${what} is locked (${why}). If no oats schedule process is running, remove ${dir} and retry; nothing was changed.`) });
290
291
  }
@@ -294,11 +295,29 @@ function withRegistryLock(fn) { return withDirLock(join(hostScheduleDir(), "regi
294
295
  /** Definitions and state are read-modify-write; CRUD serializes here briefly. */
295
296
  export function withScopeLock(ws, fn) { return withDirLock(join(stateDir(ws), "scope.lock"), `schedules of ${ws}`, fn, { retryMs: 3000 }); }
296
297
 
298
+ /** A job's slot: the directory `locks/<id>` and its record `owner.json`, which stands for the run
299
+ * that holds it until that run's instance ends (no process mutex: every taker and releaser holds
300
+ * the host lock, withHostLock). The slot is never there without its record. Its record is written
301
+ * in a directory beside the slots (slotStaging), which is then renamed to the slot; a release
302
+ * renames the slot back there before removing it. A taker or releaser killed at any step leaves
303
+ * either the slot with its record or no slot, and at most that directory, which the next taker or
304
+ * releaser of the slot removes. An entry already at the slot is held, whatever it is, and left as
305
+ * it is: a directory without its record (an earlier kernel made the directory first) reads as
306
+ * unreadable (jobLockInfo). */
307
+ const slotStaging = (ws, id) => join(stateDir(ws), `.slot-${id}`);
297
308
  export function acquireJobLock(ws, id, owner) {
298
- const dir = lockDir(ws, id);
309
+ const dir = lockDir(ws, id), staging = slotStaging(ws, id);
299
310
  mkdirSync(dirname(dir), { recursive: true });
300
- try { mkdirSync(dir); } catch (e) { if (e.code === "EEXIST") return false; throw e; }
301
- writeFileSync(join(dir, "owner.json"), JSON.stringify({ ...owner, pid: process.pid, at: new Date().toISOString() }, null, 2) + "\n");
311
+ try { lstatSync(dir); return false; } catch (e) { if (e.code !== "ENOENT") throw e; }
312
+ rmSync(staging, { recursive: true, force: true });
313
+ mkdirSync(staging);
314
+ try {
315
+ writeFileSync(join(staging, "owner.json"), JSON.stringify({ ...owner, pid: process.pid, at: new Date().toISOString() }, null, 2) + "\n");
316
+ renameSync(staging, dir);
317
+ } catch (e) {
318
+ try { rmSync(staging, { recursive: true, force: true }); } catch { /* the next taker of the slot removes it */ }
319
+ throw e;
320
+ }
302
321
  return true;
303
322
  }
304
323
  export function jobLockInfo(ws, id) {
@@ -306,7 +325,112 @@ export function jobLockInfo(ws, id) {
306
325
  if (!existsSync(dir)) return null;
307
326
  try { return JSON.parse(readFileSync(join(dir, "owner.json"), "utf8")); } catch { return { unreadable: true }; }
308
327
  }
309
- export function releaseJobLock(ws, id) { rmSync(lockDir(ws, id), { recursive: true, force: true }); }
328
+ /** `oats doctor`'s problem for a job's slot whose record cannot be read. */
329
+ export const SCHEDULE_SLOT_UNREADABLE = "schedule-slot-unreadable";
330
+ /** A schedule's id as a person types it, from the key its state and slot are stored under: a
331
+ * workspace schedule's key is <member>~<id> (scheduleStateKey), a local one is its id. */
332
+ const shownScheduleId = (key) => key.replace("~", "/");
333
+ /** The slot of the job `key` in `ws` when its record cannot be read (jobLockInfo: a directory
334
+ * without its record, a record that does not parse, an entry that is no directory) → `{ said,
335
+ * remedy }`, the words the tick, reconcile, run-now and doctor give for it; else null. Nobody
336
+ * can tell from such a slot whether the run it stands for still runs, so it is held and left as
337
+ * it is until a person has checked and reconcile --clear releases it. `dir` is the `--dir`
338
+ * argument the commands of the remedy need from another deployment. */
339
+ function unreadableSlot(ws, key, dir = "") {
340
+ if (!jobLockInfo(ws, key)?.unreadable) return null;
341
+ const id = shownScheduleId(key);
342
+ return { said: `the slot ${lockDir(ws, key)} has no readable record`, remedy: `check oats schedule show ${id}${dir}; once its last run is not running, run oats schedule reconcile ${id} --clear${dir} to release the slot` };
343
+ }
344
+ /** `oats doctor`'s problem for a slot whose schedule is not defined. */
345
+ export const SCHEDULE_SLOT_ORPHANED = "schedule-slot-orphaned";
346
+ const ISO_INSTANT = /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}Z$/;
347
+ /** The slot of `key` in `ws` when its schedule is gone (scheduleAbsence), whatever its record →
348
+ * `{ found, said, remedy }`, the words reconcile and doctor give for it; else null (no slot). No
349
+ * tick reads such a slot, and every tick counts it against maxConcurrent (liveLocks): it may stand
350
+ * for a run of a schedule removed while it ran, so it is held until a person has checked and
351
+ * reconcile --clear releases it. `found` says what the record names: the run's due minute, and
352
+ * the home of its instance when the record names one (a wake's); a record that cannot be read
353
+ * names nothing. `absent` says how the schedule is known to be gone. Only what has the shape the
354
+ * kernel writes is shown. */
355
+ function orphanSlot(ws, key, dir = "", absent = "is not defined") {
356
+ const record = jobLockInfo(ws, key);
357
+ if (!record) return null;
358
+ const id = shownScheduleId(key), release = `run oats schedule reconcile ${id} --clear${dir} to release the slot`;
359
+ const unnamed = `once no run of schedule ${id} is running (oats status lists the instances it started), ${release}`;
360
+ let found, remedy = unnamed;
361
+ if (record.unreadable) found = "has no readable record";
362
+ else {
363
+ const due = typeof record.scheduledFor === "string" && ISO_INSTANT.test(record.scheduledFor) ? record.scheduledFor : "?";
364
+ const home = typeof record.home === "string" && isAbsolute(record.home) && !/[\x00-\x1f\x7f]/.test(record.home) ? record.home : null;
365
+ found = `holds the run due ${due} of ${home ? `the instance at ${home}` : "an instance its record does not name"}`;
366
+ if (home) remedy = `once oats status shows the instance at ${home} is not running, ${release}`;
367
+ }
368
+ return { found, said: `the slot ${lockDir(ws, key)} of schedule ${id}, which ${absent}, ${found}`, remedy };
369
+ }
370
+ /** A path a snapshot names that may be shown as it is. */
371
+ const SHOWN_PATH = /^[A-Za-z0-9._@+/-]+$/;
372
+ /** Whether the schedule whose slot or state is `key` is defined, from what this host has already
373
+ * read and nothing more (no discovery, no network): `defs` (readDefinitions) for a local schedule,
374
+ * and for a workspace one (<member>~<id>) the saved snapshot `snapshot` (readSnapshot) →
375
+ * { state: "defined" } it is defined, or nothing read says otherwise (a workspace one
376
+ * whose snapshot cannot be read)
377
+ * { state: "gone", absent, takenAt? }
378
+ * local: not in oats-schedules.json. Workspace: as of the snapshot's
379
+ * `takenAt`, its member was read at a commit, the snapshot has no
380
+ * problem of that member that could hide a schedule, and no
381
+ * schedule has its id. A workspace schedule that is disabled,
382
+ * untrusted or placed elsewhere still has its entry, so it is
383
+ * defined; `absent` is the clause that says how it is known gone
384
+ * { state: "member-absent", takenAt, member }
385
+ * its member is not in the snapshot at a commit: removed from the
386
+ * workspace, or not read (an unconfirmed member is skipped silently)
387
+ * { state: "member-problem", takenAt, member, path }
388
+ * a problem of its member that has no listed schedule beside it: a
389
+ * schedule file that could not be read or parsed (`path`, which
390
+ * could hold this id: a file's id may be other than its name), or
391
+ * a member that could not be listed (`path` null) */
392
+ function scheduleAbsence(key, { defs, snapshot }) {
393
+ if (!key.includes("~")) return defs.jobs[key] ? { state: "defined" } : { state: "gone", absent: "is not defined" };
394
+ if (!snapshot) return { state: "defined" };
395
+ const id = shownScheduleId(key), member = key.slice(0, key.indexOf("~"));
396
+ const schedules = snapshot.schedules;
397
+ if (schedules.some((e) => e?.id === id)) return { state: "defined" };
398
+ const takenAt = typeof snapshot.takenAt === "string" && ISO_INSTANT.test(snapshot.takenAt) ? snapshot.takenAt : "an unknown time";
399
+ const members = snapshot.members && typeof snapshot.members === "object" ? snapshot.members : {};
400
+ const memberKey = Object.keys(members).find((k) => members[k]?.name === member && typeof members[k]?.commit === "string" && members[k].commit !== "");
401
+ if (!memberKey) return { state: "member-absent", takenAt, member };
402
+ const file = (p) => (typeof p?.path === "string" ? p.path.split("#")[0] : "");
403
+ const listed = (p) => schedules.some((e) => e?.origin?.repoKey === p.repoKey && e.origin.path === file(p));
404
+ const hiding = (Array.isArray(snapshot.problems) ? snapshot.problems : []).find((p) => p && (p.kind === undefined || p.kind === "schedule")
405
+ && (p.repoKey === memberKey || (file(p) === "" && !Object.hasOwn(members, p.repoKey))) && !listed(p));
406
+ if (hiding) return { state: "member-problem", takenAt, member, path: file(hiding) === "" ? null : SHOWN_PATH.test(file(hiding)) ? file(hiding) : "a schedule file" };
407
+ return { state: "gone", takenAt, absent: `as of ${takenAt} is not in the workspace (run oats automations refresh if that is stale)` };
408
+ }
409
+ /** The attempt reconcile resolves for the job state `js`: its persisted attempt, or the unknown
410
+ * last run an earlier kernel kept in its place. */
411
+ const attemptOf = (js) => js.attempt || (js.lastRun?.outcome === "unknown" ? { scheduledFor: js.lastRun.scheduledFor, startedAt: js.lastRun.startedAt, ...(js.lastRun.execution !== undefined ? { execution: js.lastRun.execution } : {}) } : undefined);
412
+ /** The last run of `js` once reconcile resolves its `attempt` as launching nothing: cleared by the
413
+ * operator (`clear`), or no instance found. */
414
+ /** The retained state `js` of the job `key` whose schedule is gone and that holds no slot →
415
+ * `{ said, remedy }`, the words reconcile and doctor give for it: what is retained (a launch
416
+ * attempt with no recorded result, when there is one), and the reconcile --clear that clears it. */
417
+ function orphanState(key, js, absent, dir = "") {
418
+ const id = shownScheduleId(key), attempt = attemptOf(js);
419
+ const due = typeof attempt?.scheduledFor === "string" && ISO_INSTANT.test(attempt.scheduledFor) ? attempt.scheduledFor : "?";
420
+ return { said: `schedule ${id}, which ${absent}, has retained state${attempt ? ` with a launch attempt for ${due} that has no recorded result` : ""}`,
421
+ remedy: `once no run of schedule ${id} is running (oats status lists the instances it started), run oats schedule reconcile ${id} --clear${dir} to clear its state` };
422
+ }
423
+ const launchFailedRun = (js, attempt, kind, clear) => ({ scheduledFor: attempt.scheduledFor, startedAt: js.lastRun?.startedAt || attempt.startedAt, kind, launched: false, outcome: "launch-failed", error: clear ? "cleared by the operator after checking by hand" : "no instance found in the scope's roster for the unrecorded attempt" });
424
+ /** Whether `key` names an entry directly among a scope's slots (lockDir), and nothing beyond them. */
425
+ const slotName = (key) => typeof key === "string" && key !== "" && key !== "." && key !== ".." && !/[/\\\0]/.test(key);
426
+ export function releaseJobLock(ws, id) {
427
+ const staging = slotStaging(ws, id);
428
+ rmSync(staging, { recursive: true, force: true });
429
+ try { renameSync(lockDir(ws, id), staging); } catch (e) { if (e.code === "ENOENT") return; throw e; }
430
+ // The slot is released once it is renamed: what could not be removed of it is removed by the
431
+ // next taker or releaser of the slot.
432
+ try { rmSync(staging, { recursive: true, force: true }); } catch { /* removed next time */ }
433
+ }
310
434
  function liveLocks(wsList) {
311
435
  let n = 0;
312
436
  for (const ws of wsList) {
@@ -906,7 +1030,8 @@ export function tickWorkspace(ws, { now = new Date(), firedAt = now, io, reg, ws
906
1030
  // `candidates` (the host tick) means the host already observed every scope.
907
1031
  // Triggers (kind "trigger") live in the same file and are evaluated by tickTriggers, not here.
908
1032
  for (const [id, def] of Object.entries(defs.jobs)) if (def && def.kind === "trigger") delete defs.jobs[id];
909
- if (!candidates) for (const [id, def] of Object.entries(defs.jobs)) observePrevious(ws, id, def, stateOf(id), io, now, { mutate: !dryRun });
1033
+ // A job whose slot cannot be read is not observed: it is reported invalid below, on its own.
1034
+ if (!candidates) for (const [id, def] of Object.entries(defs.jobs)) if (!unreadableSlot(ws, id)) observePrevious(ws, id, def, stateOf(id), io, now, { mutate: !dryRun });
910
1035
  if (observeOnly) { if (!dryRun) writeState(ws, st); return considered; }
911
1036
  // Least recently LAUNCHED first: only an actual harness launch counts, a
912
1037
  // skipped or pending job keeps its place at the front of the line.
@@ -917,6 +1042,8 @@ export function tickWorkspace(ws, { now = new Date(), firedAt = now, io, reg, ws
917
1042
  let due, cronError;
918
1043
  const invalidShape = shapeError(def), herdr = invalidShape ? null : storedHerdrBackend(ws, id, def);
919
1044
  const shape = invalidShape ?? herdr?.message;
1045
+ const slot = shape ? null : unreadableSlot(ws, id);
1046
+ if (slot) { record(id, "invalid", { error: `${slot.said}; ${slot.remedy}`, errorCode: "E_SCHEDULE_INVALID" }); continue; }
920
1047
  if (!shape) validateActiveExecution(js, jobLockInfo(ws, id)); // Also covers host pre-observed candidates.
921
1048
  if (shape) { cronError = shape; due = false; }
922
1049
  else { try { due = only ? true : dueAt(def, minute); } catch (e) { cronError = e.message; due = false; } }
@@ -1085,6 +1212,8 @@ export function runNow(ws, qid, { now = new Date(), io, force = false, ctx = nul
1085
1212
  return withHostLock(() => {
1086
1213
  const st = readState(ws);
1087
1214
  const js = st.jobs[id] || (st.jobs[id] = {});
1215
+ const slot = unreadableSlot(ws, id);
1216
+ if (slot) throw scheduleError("E_SCHEDULE_INVALID", `schedule ${id}: ${slot.said}; ${slot.remedy}`);
1088
1217
  observePrevious(ws, id, def, js, io, now, { mutate: true });
1089
1218
  writeState(ws, st);
1090
1219
  if (js.attempt) throw scheduleError("E_SCHEDULE_UNRESOLVED", `schedule ${id} has a launch attempt without a recorded result; run oats schedule reconcile ${id} first`);
@@ -1109,12 +1238,21 @@ export function reconcile(ws, qid, { io, now = new Date(), clear = false, ctx =
1109
1238
  const id = scheduleKeyOf(qid);
1110
1239
  const defs = effectiveDefinitions(ws, ctx);
1111
1240
  const def = defs.jobs[id];
1241
+ if (!def && slotName(id) && (jobLockInfo(ws, id) || readState(ws).jobs[id]) && scheduleAbsence(id, { defs: readDefinitions(ws), snapshot: readSnapshot(ws) }).state === "gone") return reconcileOrphanSlot(ws, qid, id, { clear, ctx });
1112
1242
  if (!def || def.kind === "trigger") throw notHere(ws, qid, ctx);
1113
1243
  const describeHere = () => describe(ws, qid, io, { ctx });
1114
1244
  return withHostLock(() => {
1115
1245
  const st = readState(ws);
1116
1246
  const js = st.jobs[id] || (st.jobs[id] = {});
1117
- const attempt = js.attempt || (js.lastRun?.outcome === "unknown" ? { scheduledFor: js.lastRun.scheduledFor, startedAt: js.lastRun.startedAt, ...(js.lastRun.execution !== undefined ? { execution: js.lastRun.execution } : {}) } : undefined);
1247
+ const attempt = attemptOf(js);
1248
+ // A slot whose record cannot be read is released only once a person has checked (--clear);
1249
+ // an attempt beside it is then reconciled as any other.
1250
+ const slot = unreadableSlot(ws, id);
1251
+ if (slot && !clear) return { schedule: describeHere(), reconciled: "unknown", remedy: `${slot.said}; ${slot.remedy}` };
1252
+ if (slot) {
1253
+ releaseJobLock(ws, id);
1254
+ if (!attempt) return { schedule: describeHere(), reconciled: "cleared" };
1255
+ }
1118
1256
  if (!attempt) return { schedule: describeHere(), reconciled: "nothing to reconcile" };
1119
1257
  try { validateAttemptExecution(attempt, jobLockInfo(ws, id)); }
1120
1258
  catch (error) { throw scheduleError("E_SCHEDULE_INVALID", error.message); }
@@ -1145,7 +1283,7 @@ export function reconcile(ws, qid, { io, now = new Date(), clear = false, ctx =
1145
1283
  js.lastRun = { scheduledFor: attempt.scheduledFor, startedAt: js.lastRun?.startedAt || attempt.startedAt, kind: attemptKind, launched: true, ...adopted, outcome: seen.outcome, ...(seen.error ? { error: seen.error } : {}), ...(seen.note ? { note: seen.note } : {}) };
1146
1284
  if (seen.outcome !== "ended") { if (!jobLockInfo(ws, id)) acquireJobLock(ws, id, { scheduledFor: attempt.scheduledFor, reconciled: true }); } else releaseJobLock(ws, id);
1147
1285
  } else {
1148
- js.lastRun = { scheduledFor: attempt.scheduledFor, startedAt: js.lastRun?.startedAt || attempt.startedAt, kind: attemptKind, launched: false, outcome: "launch-failed", error: clear ? "cleared by the operator after checking by hand" : "no instance found in the scope's roster for the unrecorded attempt" };
1286
+ js.lastRun = launchFailedRun(js, attempt, attemptKind, clear);
1149
1287
  releaseJobLock(ws, id);
1150
1288
  }
1151
1289
  writeState(ws, st);
@@ -1153,6 +1291,37 @@ export function reconcile(ws, qid, { io, now = new Date(), clear = false, ctx =
1153
1291
  });
1154
1292
  }
1155
1293
 
1294
+ /** Reconcile what a schedule that is gone (scheduleAbsence) left: its slot (orphanSlot), its
1295
+ * retained state (orphanState), or both, read again under the host lock, as every reconcile.
1296
+ * Without `clear` it answers what it found and changes nothing. With `clear` it releases the slot,
1297
+ * resolves the job's retained attempt as cleared (as for a defined job: launchFailedRun) and
1298
+ * removes the job's state, and says which of the three it cleared (`cleared`). `schedule` is
1299
+ * null: there is no schedule to describe. */
1300
+ function reconcileOrphanSlot(ws, qid, id, { clear, ctx }) {
1301
+ return withHostLock(() => {
1302
+ const absence = scheduleAbsence(id, { defs: readDefinitions(ws), snapshot: readSnapshot(ws) });
1303
+ const st = readState(ws), js = st.jobs[id];
1304
+ const slot = absence.state === "gone" ? orphanSlot(ws, id, "", absence.absent) : null;
1305
+ const words = slot ?? (absence.state === "gone" && js ? orphanState(id, js, absence.absent) : null);
1306
+ if (!words) throw notHere(ws, qid, ctx);
1307
+ if (!clear) return { schedule: null, reconciled: "unknown", remedy: `${words.said}; ${words.remedy}` };
1308
+ const attempt = js ? attemptOf(js) : undefined;
1309
+ if (attempt) {
1310
+ try { validateAttemptExecution(attempt, jobLockInfo(ws, id)); }
1311
+ catch (error) { throw scheduleError("E_SCHEDULE_INVALID", error.message); }
1312
+ }
1313
+ const cleared = [];
1314
+ if (slot) { releaseJobLock(ws, id); cleared.push("slot"); }
1315
+ if (js) {
1316
+ if (attempt) { delete js.attempt; js.lastRun = launchFailedRun(js, attempt, js.lastRun?.kind ?? null, true); cleared.push("attempt"); }
1317
+ delete st.jobs[id];
1318
+ writeState(ws, st);
1319
+ cleared.push("state");
1320
+ }
1321
+ return { schedule: null, reconciled: "cleared", cleared };
1322
+ });
1323
+ }
1324
+
1156
1325
  /** Whether a spawn job's runs are named `<agent>-<purpose>`, so a home's absence proves a lost
1157
1326
  * receipt launched nothing: only for a bare name of a member soul under the agents root, whose
1158
1327
  * agent name is that name. Spawn names a run after the soul's stem (core.mjs
@@ -1174,6 +1343,17 @@ function namedHomes(ws, js) {
1174
1343
  * authorization. In particular, do not call placementOf here: it probes gh. */
1175
1344
  function workspaceReconcileRemedy(id, snapshot, local, localError) {
1176
1345
  const preserve = "Preserve the retained state and inspect its effects with the deployment owner; this diagnostic does not clear attempts or slots.";
1346
+ const { excluded, unknown } = workspacePlacement(id, snapshot, local, localError);
1347
+ if (excluded) return `Retained-state reconciliation is currently unavailable under this placement: ${excluded}. ${preserve}`;
1348
+ if (unknown) return `Workspace authorization cannot be verified offline: ${unknown}. ${preserve}`;
1349
+ return `Current workspace authorization is unverified offline; reconciliation requires current host, owner and trust authorization. ${preserve}`;
1350
+ }
1351
+ /** Whether this host runs the workspace schedule `id`, as far as it can tell offline from the saved
1352
+ * `snapshot` and its own oats-local.yaml (`local`, or `localError`), without asking gh who it is
1353
+ * → { excluded }: it does not (the clause says why: disabled, not named, placed on another host,
1354
+ * untrusted, not in the snapshot); { unknown }: that cannot be read (the clause says why); or {}:
1355
+ * nothing offline excludes it, and reconcile still checks its owner. */
1356
+ function workspacePlacement(id, snapshot, local, localError) {
1177
1357
  let unknown, excluded;
1178
1358
  const entries = snapshot?.schedules;
1179
1359
  const matches = entries?.filter((entry) => entry?.id === id) || [];
@@ -1191,14 +1371,13 @@ function workspaceReconcileRemedy(id, snapshot, local, localError) {
1191
1371
  else if (entry.runsOn !== host.name) excluded = `the saved schedule is assigned elsewhere (${entry.runsOn}); this host is ${host.name}`;
1192
1372
  else if (!trusted(host, id)) excluded = "the schedule is untrusted by local automations.trust";
1193
1373
  }
1194
- if (excluded) return `Retained-state reconciliation is currently unavailable under this placement: ${excluded}. ${preserve}`;
1195
- if (unknown) return `Workspace authorization cannot be verified offline: ${unknown}. ${preserve}`;
1196
- return `Current workspace authorization is unverified offline; reconciliation requires current host, owner and trust authorization. ${preserve}`;
1374
+ return { excluded, unknown };
1197
1375
  }
1198
1376
 
1199
- /** `oats doctor`: retained launch attempts and unknown last observations, in this
1200
- * deployment and the other scopes sharing its slots. Reads saved state only;
1201
- * workspace guidance cannot promise authorization or an executable recovery. */
1377
+ /** `oats doctor`: retained launch attempts and unknown last observations, and the slots whose
1378
+ * record cannot be read (SCHEDULE_SLOT_UNREADABLE, unreadableSlot), in this deployment and the
1379
+ * other scopes sharing its slots. Reads saved state only; workspace guidance cannot promise
1380
+ * authorization or an executable recovery. */
1202
1381
  export function unresolvedScheduleAttempts(dep, { now = new Date() } = {}) {
1203
1382
  const problems = [], information = [];
1204
1383
  const here = resolve(dep);
@@ -1206,9 +1385,10 @@ export function unresolvedScheduleAttempts(dep, { now = new Date() } = {}) {
1206
1385
  try { scopes = [here, ...readRegistry().workspaces.filter((w) => resolve(w) !== here && existsSync(w))]; }
1207
1386
  catch (e) { information.push(`the host schedule registry could not be read, so only this deployment's schedules were checked: ${e.message}`); }
1208
1387
  for (const ws of scopes) {
1209
- let defs, st;
1388
+ const dir = ws === here ? "" : ` --dir ${/^[\w./@%+=:,-]+$/.test(ws) ? ws : shq(ws)}`;
1389
+ let defs, st, unread = null;
1210
1390
  try { defs = readDefinitions(ws); st = readState(ws); }
1211
- catch (e) { information.push(`the schedules of ${ws} could not be read: ${e.message}`); continue; }
1391
+ catch (e) { unread = e; }
1212
1392
  // Read the last discovery snapshot directly: placement and refresh would
1213
1393
  // require credentials/network, and unresolved work matters even when offline.
1214
1394
  const snapshot = readSnapshot(ws);
@@ -1219,6 +1399,40 @@ export function unresolvedScheduleAttempts(dep, { now = new Date() } = {}) {
1219
1399
  if (resolve(dirname(found.path)) !== resolve(ws)) throw scheduleError("E_LOCAL_MISSING", "this scope has no local configuration");
1220
1400
  local = found.local;
1221
1401
  } catch (e) { localError = e; }
1402
+ // Every slot there is, as liveLocks counts them: the tick reports an unreadable one of a
1403
+ // defined job invalid, and reads none of a schedule that is not defined. Definitions that
1404
+ // cannot be read say nothing of which it is.
1405
+ const slots = join(stateDir(ws), "locks");
1406
+ let slotEntries = [];
1407
+ try { slotEntries = readdirSync(slots, { withFileTypes: true }).filter((e) => e.isDirectory()).map((e) => e.name).sort(); }
1408
+ catch (e) { if (e.code !== "ENOENT") information.push(`the schedule slots of ${ws} could not be read: ${e.message}`); }
1409
+ for (const key of slotEntries) {
1410
+ const id = shownScheduleId(key), path = lockDir(ws, key);
1411
+ const absence = unread !== null ? { state: "defined" } : scheduleAbsence(key, { defs, snapshot });
1412
+ const where = ws === here ? "" : ` (in ${ws})`;
1413
+ const orphan = absence.state === "defined" ? null : orphanSlot(ws, key, dir, absence.absent);
1414
+ const slot = absence.state === "defined" ? unreadableSlot(ws, key, dir) : null;
1415
+ const counted = `its slot ${path} ${orphan?.found}, and the slot counts against maxConcurrent`;
1416
+ const untold = `which is not in the workspace's snapshot of ${absence.takenAt}`;
1417
+ if (absence.state === "gone") problems.push({ code: SCHEDULE_SLOT_ORPHANED, severity: "warning", scope: ws, id,
1418
+ message: `schedule ${id}${where}, which ${absence.absent}: ${counted}`, remedy: orphan.remedy });
1419
+ if (absence.state === "member-absent") problems.push({ code: SCHEDULE_SLOT_ORPHANED, severity: "warning", scope: ws, id,
1420
+ message: `schedule ${id}${where}, ${untold}: its member ${absence.member} is not in the workspace, or was not read, so whether the schedule is gone cannot be told; ${counted}`,
1421
+ remedy: `check oats status for a run of schedule ${id}; once none is running, remove the slot by hand: rm -r -- ${shq(path)}` });
1422
+ if (absence.state === "member-problem") problems.push({ code: SCHEDULE_SLOT_ORPHANED, severity: "warning", scope: ws, id,
1423
+ message: `schedule ${id}${where}, ${untold}: ${absence.path ? `member ${absence.member} has a schedule file that could not be read (${absence.path}), which could be this schedule` : `member ${absence.member} could not be listed`}, so whether it is gone cannot be told; ${counted}`,
1424
+ remedy: `${absence.path ? `fix ${absence.path} in member ${absence.member}` : `make member ${absence.member} readable`} and run oats automations refresh; doctor then says whether schedule ${id} is gone` });
1425
+ // A workspace schedule this host does not run, or cannot tell it runs, is refused by reconcile
1426
+ // here: its slot is removed by hand once nothing of it runs.
1427
+ const placement = slot && key.includes("~") ? workspacePlacement(id, snapshot, local, localError) : {};
1428
+ const byHand = `once oats status shows no run of schedule ${id} here, remove the slot by hand: rm -r -- ${shq(path)}`;
1429
+ if (slot) problems.push({ code: SCHEDULE_SLOT_UNREADABLE, severity: "warning", scope: ws, id,
1430
+ message: placement.excluded ? `schedule ${id}${where}: its slot ${path} has no readable record, and the slot counts against maxConcurrent; this host does not run the schedule (${placement.excluded}), so reconcile here refuses it`
1431
+ : placement.unknown ? `schedule ${id}${where}: its slot ${path} has no readable record, and the slot counts against maxConcurrent; whether this host runs the schedule cannot be told offline (${placement.unknown})`
1432
+ : `schedule ${id}${where}: its slot ${path} has no readable record; the tick reports this job invalid and starts nothing of it, and the slot counts against maxConcurrent`,
1433
+ remedy: placement.excluded || placement.unknown ? byHand : slot.remedy });
1434
+ }
1435
+ if (unread) { information.push(`the schedules of ${ws} could not be read: ${unread.message}`); continue; }
1222
1436
  for (const [key, js] of Object.entries(st.jobs)) {
1223
1437
  if (!js?.attempt && js?.lastRun?.outcome !== "unknown") continue;
1224
1438
  // A workspace schedule's state key is <member>~<id> (scheduleStateKey); a local one is its id.
@@ -1234,23 +1448,27 @@ export function unresolvedScheduleAttempts(dep, { now = new Date() } = {}) {
1234
1448
  const holdsSlot = !!lock;
1235
1449
  const error = js.attempt?.error ?? js.lastRun?.error ?? null;
1236
1450
  const unproven = (kind === "command" || kind === "operation") && namedHomes(ws, js).length !== 1;
1237
- const dir = ws === here ? "" : ` --dir ${/^[\w./@%+=:,-]+$/.test(ws) ? ws : shq(ws)}`;
1238
1451
  let unavailable = localError ? `local configuration is invalid or unreadable (${localError.code || "read error"})` : null;
1239
1452
  if (!workspace && !unavailable) {
1240
1453
  unavailable = !SCHEDULE_ID_RE.test(id) ? "the stored schedule ID is invalid" : shapeError(defs.jobs[key]);
1241
1454
  if (!unavailable && defs.jobs[key].id !== undefined && defs.jobs[key].id !== key) unavailable = "the stored definition has a different ID";
1242
1455
  try { validateAttemptExecution(js.attempt || { execution: js.lastRun?.execution }, lock); }
1243
1456
  catch { unavailable ||= "the retained attempt or lock uses unsupported captured execution state"; }
1244
- if (lock?.unreadable) unavailable ||= "job lock custody is unreadable";
1245
1457
  }
1246
- const remedy = workspace ? workspaceReconcileRemedy(id, snapshot, local, localError)
1458
+ const slot = workspace || unavailable ? null : unreadableSlot(ws, key, dir);
1459
+ // A workspace schedule that is gone: reconcile --clear resolves its slot and state.
1460
+ const absence = workspace ? scheduleAbsence(key, { defs, snapshot }) : null;
1461
+ const orphan = absence?.state === "gone" ? orphanSlot(ws, key, dir, absence.absent) ?? orphanState(key, js, absence.absent, dir) : null;
1462
+ const remedy = orphan ? orphan.remedy
1463
+ : workspace ? workspaceReconcileRemedy(id, snapshot, local, localError)
1247
1464
  : unavailable ? `Doctor does not recommend reconciling this job because ${unavailable}; preserve the retained state and inspect its effects with the deployment owner.`
1465
+ : slot ? slot.remedy
1248
1466
  : `oats schedule reconcile ${id}${unproven ? " --clear" : ""}${dir}`;
1249
1467
  const subject = js.attempt ? `${kind ? `a ${kind}` : "a"} launch attempt for ${scheduledFor} has no recorded result`
1250
1468
  : `the last ${kind || "schedule"} run for ${scheduledFor} has an unknown observation`;
1251
1469
  const blocking = js.attempt ? "This persisted attempt blocks another run until it is reconciled"
1252
1470
  : "There is no persisted launch attempt blocking execution; an eligible tick may re-observe the last run, without a guarantee of clearance";
1253
- const guidance = workspace || unavailable ? remedy : unproven ? `Its effects can't be proven: check the roster and the host by hand, then run ${remedy}` : `Run ${remedy}`;
1471
+ const guidance = orphan ? `${orphan.said}; ${remedy}` : workspace || unavailable ? remedy : slot ? `${slot.said}: ${remedy}` : unproven ? `Its effects can't be proven: check the roster and the host by hand, then run ${remedy}` : `Run ${remedy}`;
1254
1472
  const message = `schedule ${id}${ws === here ? "" : ` (in ${ws})`}: ${subject} (started ${startedAt}${ageSeconds === null ? "" : `, ${formatAge(ageSeconds)} ago`})${error ? `: ${error}` : ""}. ${blocking}, and ${holdsSlot ? "it holds a host slot (counted against maxConcurrent)" : "it holds no host slot"}. ${guidance}`;
1255
1473
  problems.push({ code: "schedule-unresolved", severity: "warning", scope: ws, id, kind, scheduledFor, startedAt, ageSeconds, holdsSlot, exited: js.attempt?.exited === true, error, remedy, message });
1256
1474
  }