@awebai/oats 0.69.17 → 0.69.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/oats.mjs +14 -13
- package/docs/capabilities.md +8 -1
- package/docs/container.md +345 -18
- package/docs/design/2026-09-23-workspace-module-contracts.md +8 -2
- package/docs/design/2026-09-27-team-model-v2.md +2 -2
- package/docs/design/2026-09-28-automations-trust.md +1 -1
- package/docs/desktop-cli-api.md +100 -53
- package/docs/desktop.md +53 -24
- package/docs/execution-targets.md +66 -48
- package/docs/implementation.md +8 -7
- package/docs/integrations.md +2 -2
- package/docs/knowledge.md +5 -5
- package/docs/official-catalog.md +7 -7
- package/docs/packages.md +13 -13
- package/docs/release-lane.md +30 -23
- package/docs/release-notes/v0.69.18.md +358 -0
- package/docs/release-notes/v0.69.19.md +172 -0
- package/docs/servers.md +5 -3
- package/docs/souls-and-instances.md +51 -27
- package/docs/trust-model.md +119 -0
- package/docs/tui.md +8 -8
- package/docs/upgrading.md +45 -19
- package/docs/workspaces.md +3 -3
- package/injects/work-directory.md +27 -2
- package/injects/work-workspace.md +32 -6
- package/lib/command-flags.mjs +1 -1
- package/lib/core.mjs +26 -11
- package/lib/errors.mjs +1 -1
- package/lib/home-owner.mjs +25 -0
- package/lib/host-run-admission.mjs +21 -0
- package/lib/host-run-child.mjs +31 -0
- package/lib/host-run-work.mjs +150 -0
- package/lib/host-run.mjs +43 -29
- package/lib/instance-naming.mjs +26 -44
- package/lib/packages.mjs +0 -43
- package/lib/pi-bridge.mjs +18 -6
- package/lib/schedule.mjs +5 -3
- package/lib/servers.mjs +4 -3
- package/lib/session-end.mjs +9 -0
- package/lib/session-resume-contract.mjs +1 -1
- package/lib/session-resume.mjs +144 -51
- package/lib/session-seen.mjs +16 -11
- package/lib/triggers.mjs +3 -1
- package/lib/worktree.mjs +5 -2
- package/package-catalog.json +3 -3
- package/package.json +4 -2
- package/packages/client/above-tested-notice.mjs +30 -0
- package/packages/client/above-tested.mjs +225 -0
- package/packages/client/cli-locator.mjs +67 -52
- package/packages/client/deployment-label.mjs +4 -4
- package/packages/client/instance-name.mjs +5 -29
- package/packages/client/instance-presentation.mjs +2 -2
- package/packages/client/instance-tree.mjs +18 -7
- package/packages/client/session-start.mjs +31 -3
- package/packages/client/workspace-choices.mjs +1 -1
- package/packages/client/workspace-views.mjs +1 -1
- package/packages/record/README.md +10 -8
- package/packages/record/lib/process-identity.mjs +14 -36
- package/packages/tui/lib/read.mjs +19 -3
- package/skills/oats-getting-started/SKILL.md +33 -4
package/lib/instance-naming.mjs
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/** The suggested instance name (feature spawn-suggest-name, awebai/oats#1421 §3): `oats spawn <soul>
|
|
2
2
|
* --preview --suggest-name` asks the soul's own harness, in one isolated non-interactive call, for a
|
|
3
|
-
*
|
|
4
|
-
*
|
|
3
|
+
* name for the task, and names the instance with the model's whole `<work>-<role>`: nothing appends a
|
|
4
|
+
* role, and nothing reads the soul's name. A failure is never an error: the preview numbers the
|
|
5
|
+
* instance as before and says why (`naming.reason`, a closed set).
|
|
5
6
|
*
|
|
6
7
|
* The call starts nothing that runs code: no MCP server, hook, plugin, extension or project setting
|
|
7
8
|
* (the maintainer's condition). NAMING_CALLS holds the argv of each harness where that is proven
|
|
@@ -18,7 +19,7 @@ import { join } from "node:path";
|
|
|
18
19
|
import { killGroup } from "./process-group.mjs";
|
|
19
20
|
|
|
20
21
|
/** The prompt's fixed words (the maintainer's wording); the task follows. */
|
|
21
|
-
export const NAMING_PROMPT = "Name an agent for this task:
|
|
22
|
+
export const NAMING_PROMPT = "Name an agent for this task: <work>-<role>, at most five lowercase words joined by hyphens. First a few words for the work, then, last, one word for the agent's role as the task gives it; if the task gives no role, one word for what the agent does in it. Reply with the name only.";
|
|
22
23
|
/** The task text the prompt carries, in characters (code points). */
|
|
23
24
|
export const NAMING_TASK_MAX = 4000;
|
|
24
25
|
/** The call's limits: the time after which its process group is killed, and the stdout read. */
|
|
@@ -28,55 +29,37 @@ export const NAMING_OUTPUT_MAX = 4096;
|
|
|
28
29
|
export const NAMING_REASONS = Object.freeze(["no-task", "unsupported-harness", "timeout", "harness-failed", "invalid-reply"]);
|
|
29
30
|
/** Instance names are at most this long (lib/core.mjs MAX_INSTANCE_NAME, the messaging alias's limit). */
|
|
30
31
|
const NAME_MAX = 64;
|
|
31
|
-
/**
|
|
32
|
-
const WORDS_MAX =
|
|
32
|
+
/** A name keeps at most this many words, the role included. */
|
|
33
|
+
const WORDS_MAX = 5;
|
|
33
34
|
|
|
34
35
|
/** The prompt for `task`: the fixed words, then the task, cut to NAMING_TASK_MAX characters. */
|
|
35
36
|
export function namingPrompt(task) {
|
|
36
37
|
return `${NAMING_PROMPT}\n\nTask:\n${Array.from(String(task)).slice(0, NAMING_TASK_MAX).join("")}`;
|
|
37
38
|
}
|
|
38
39
|
|
|
39
|
-
/** The role of a soul name: its last `-`-separated word after any package qualifier (what follows the
|
|
40
|
-
* last `/` or `--`); '' when there is none. The clients' rule (packages/client/instance-name.mjs
|
|
41
|
-
* soulRole), kept here because the kernel imports nothing from packages/: test/instance-naming.test.mjs
|
|
42
|
-
* pins it to the clients' cases (test/client-instance-name.test.mjs ROLE_CASES). */
|
|
43
|
-
export function soulRole(soulName) {
|
|
44
|
-
if (typeof soulName !== "string") return "";
|
|
45
|
-
const slash = soulName.lastIndexOf("/"), dashes = soulName.lastIndexOf("--");
|
|
46
|
-
const bare = soulName.slice(Math.max(slash < 0 ? 0 : slash + 1, dashes < 0 ? 0 : dashes + 2));
|
|
47
|
-
return bare.slice(bare.lastIndexOf("-") + 1);
|
|
48
|
-
}
|
|
49
|
-
|
|
50
40
|
/** lib/core.mjs slug() without its "agent" fallback: lowercase letters and digits, single dashes. */
|
|
51
41
|
const slugWords = (text) => String(text).toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").split("-").filter(Boolean);
|
|
52
42
|
|
|
53
|
-
/** The
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
|
|
43
|
+
/** The name in a model's `reply`: its first non-empty line as slug words, the last of them the role,
|
|
44
|
+
* always kept. Over WORDS_MAX words, or over NAME_MAX characters with any suffix, the descriptive words
|
|
45
|
+
* nearest the role are dropped, and the first word's characters are cut only when it is the one
|
|
46
|
+
* descriptive word left. Unique by `-2`, `-3`, … at the end while `taken(name)` (an instance or a soul
|
|
47
|
+
* of the deployment, or a live window: what an explicit name is refused for). null when the line holds
|
|
48
|
+
* fewer than two words (no work and role), or when the role and suffix leave no room for one
|
|
49
|
+
* character of the first word: an invalid reply. */
|
|
50
|
+
export function nameFromReply(reply, taken) {
|
|
58
51
|
const line = String(reply ?? "").split(/\r?\n/).find((l) => l.trim()) ?? "";
|
|
59
52
|
const words = slugWords(line);
|
|
60
|
-
|
|
61
|
-
const
|
|
62
|
-
|
|
63
|
-
while (kept.length && kept.join("-").length > budget) kept.pop();
|
|
64
|
-
return kept;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
/** The name for `words` and `role`: `<descriptive>-<role>`, made unique by `-2`, `-3`, … while
|
|
68
|
-
* `taken(name)` (an instance or a soul of the deployment, or a live window: what an explicit name
|
|
69
|
-
* is refused for). A suffix that would pass NAME_MAX drops a descriptive word first, and cuts the
|
|
70
|
-
* last one only when one word is left. null for no words. */
|
|
71
|
-
export function uniqueName(words, role, taken) {
|
|
72
|
-
if (!words.length) return null;
|
|
73
|
-
let kept = [...words];
|
|
53
|
+
if (words.length < 2) return null;
|
|
54
|
+
const role = words.pop();
|
|
55
|
+
let kept = words.slice(0, WORDS_MAX - 1);
|
|
74
56
|
for (let i = 1; ; i++) {
|
|
75
57
|
const suffix = i === 1 ? "" : `-${i}`;
|
|
76
|
-
const fixed =
|
|
58
|
+
const fixed = role.length + 1 + suffix.length;
|
|
59
|
+
if (fixed + 1 > NAME_MAX) return null;
|
|
77
60
|
while (kept.length > 1 && kept.join("-").length + fixed > NAME_MAX) kept.pop();
|
|
78
|
-
if (kept.
|
|
79
|
-
const name = `${kept.join("-")}
|
|
61
|
+
if (kept[0].length + fixed > NAME_MAX) kept = [kept[0].slice(0, NAME_MAX - fixed)];
|
|
62
|
+
const name = `${kept.join("-")}-${role}${suffix}`;
|
|
80
63
|
if (!taken(name)) return name;
|
|
81
64
|
}
|
|
82
65
|
}
|
|
@@ -174,14 +157,13 @@ export async function namingCall({ harness, bin, instanceModel, env, task, timeo
|
|
|
174
157
|
return { answer: { source: "suggested", ms: ms(), harness, model }, reply: outcome.stdout };
|
|
175
158
|
}
|
|
176
159
|
|
|
177
|
-
/** The suggested name for `task`: the call, then the name
|
|
178
|
-
*
|
|
179
|
-
*
|
|
180
|
-
export async function suggestInstanceName({
|
|
160
|
+
/** The suggested name for `task`: the call, then the name in its reply (nameFromReply), unique by
|
|
161
|
+
* `taken`. → Promise of `{ naming, name? }`: `name` only when `naming.source` is "suggested". Never
|
|
162
|
+
* rejects. */
|
|
163
|
+
export async function suggestInstanceName({ taken, ...call }) {
|
|
181
164
|
const { answer, reply } = await namingCall(call);
|
|
182
165
|
if (answer.source !== "suggested") return { naming: answer };
|
|
183
|
-
const
|
|
184
|
-
const name = uniqueName(descriptiveWords(reply, role), role, taken);
|
|
166
|
+
const name = nameFromReply(reply, taken);
|
|
185
167
|
if (!name) return { naming: numbered("invalid-reply", `${call.harness}'s reply holds no name`, answer.ms) };
|
|
186
168
|
return { naming: answer, name };
|
|
187
169
|
}
|
package/lib/packages.mjs
CHANGED
|
@@ -36,10 +36,6 @@
|
|
|
36
36
|
*
|
|
37
37
|
* A lock written before 0.26.0 may still carry an `approved` record per entry:
|
|
38
38
|
* it is read with the field ignored, and the next write drops it.
|
|
39
|
-
*
|
|
40
|
-
* Deprecated shims: every name the 0.24 kernel still imports from this module
|
|
41
|
-
* is exported below as a thin function throwing E_REMOVED so nothing breaks at
|
|
42
|
-
* import time; callers are deleted in the next phase.
|
|
43
39
|
*/
|
|
44
40
|
import { existsSync, mkdtempSync, readFileSync, rmSync, lstatSync } from "node:fs";
|
|
45
41
|
import { tmpdir } from "node:os";
|
|
@@ -68,7 +64,6 @@ const SOUL_NAME_RE = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
|
|
68
64
|
/** The one symlink a soul source may carry: its CLAUDE.md → AGENTS.md alias (at the soul's root).
|
|
69
65
|
* Sync and spawn fetch a package soul with this same predicate, so their digests agree. */
|
|
70
66
|
export const SOUL_ALIAS_SYMLINK = (p) => p === "CLAUDE.md";
|
|
71
|
-
const CONTRACT_DOC = "docs/design/2026-09-23-workspace-module-contracts.md";
|
|
72
67
|
|
|
73
68
|
/** `oatsError` with details attached as BOTH `e.provenance` (today's field) and
|
|
74
69
|
* `e.details` (the contract's name) so callers can read either. */
|
|
@@ -657,41 +652,3 @@ export function packageProviding(lock, capName) {
|
|
|
657
652
|
}
|
|
658
653
|
return { id: providers[0], entry: lock.packages[providers[0]] };
|
|
659
654
|
}
|
|
660
|
-
|
|
661
|
-
// ---------- deprecated shims (phase A only; callers are deleted next phase) ----------
|
|
662
|
-
|
|
663
|
-
const removed = (name) => function removedByWorkspaceModel() {
|
|
664
|
-
throw oatsError("E_REMOVED", `${name} was removed by the workspace model; see ${CONTRACT_DOC}`, { name, contract: CONTRACT_DOC });
|
|
665
|
-
};
|
|
666
|
-
|
|
667
|
-
export const aggregateMissingRequirements = removed("aggregateMissingRequirements");
|
|
668
|
-
export const applyConfigMerge = removed("applyConfigMerge");
|
|
669
|
-
export const applyFromOasScope = removed("applyFromOasScope");
|
|
670
|
-
export const beginRunJournal = removed("beginRunJournal");
|
|
671
|
-
export const capabilityRuntimeTargets = removed("capabilityRuntimeTargets");
|
|
672
|
-
export const commandOnPath = removed("commandOnPath");
|
|
673
|
-
export const discoverMigrationScopes = removed("discoverMigrationScopes");
|
|
674
|
-
export const discoverOasScopes = removed("discoverOasScopes");
|
|
675
|
-
export const discoverWorkspaceScopes = removed("discoverWorkspaceScopes");
|
|
676
|
-
export const lockedPackageCapabilities = removed("lockedPackageCapabilities");
|
|
677
|
-
export const normalizeRequirement = removed("normalizeRequirement");
|
|
678
|
-
export const oasRenameMap = removed("oasRenameMap");
|
|
679
|
-
export const packageSpecIdentity = removed("packageSpecIdentity");
|
|
680
|
-
export const planConfigMerge = removed("planConfigMerge");
|
|
681
|
-
export const planFromOasScope = removed("planFromOasScope");
|
|
682
|
-
export const readAdoptedTemplate = removed("readAdoptedTemplate");
|
|
683
|
-
export const requirementInstallPlan = removed("requirementInstallPlan");
|
|
684
|
-
export const runRequirementInstall = removed("runRequirementInstall");
|
|
685
|
-
export const runtimePackageInstalled = removed("runtimePackageInstalled");
|
|
686
|
-
export const runtimePackageStatus = removed("runtimePackageStatus");
|
|
687
|
-
export const selectConfigTemplate = removed("selectConfigTemplate");
|
|
688
|
-
export const splitConfigLines = removed("splitConfigLines");
|
|
689
|
-
export const transformOasConfigText = removed("transformOasConfigText");
|
|
690
|
-
export const validateConfigTemplate = removed("validateConfigTemplate");
|
|
691
|
-
export const writeAdoptedTemplate = removed("writeAdoptedTemplate");
|
|
692
|
-
export const adoptedTemplateDir = removed("adoptedTemplateDir");
|
|
693
|
-
/** Was a constant table; any property access throws E_REMOVED. */
|
|
694
|
-
export const REQUIREMENT_MANAGERS = new Proxy(Object.freeze({}), {
|
|
695
|
-
get(_t, prop) { if (typeof prop === "symbol" || prop === "then") return undefined; removed("REQUIREMENT_MANAGERS")(); },
|
|
696
|
-
has() { removed("REQUIREMENT_MANAGERS")(); },
|
|
697
|
-
});
|
package/lib/pi-bridge.mjs
CHANGED
|
@@ -103,18 +103,30 @@ export function planCliUpdate(installed, registry) {
|
|
|
103
103
|
return { installed, registry, target: registry, ahead: false, step: { argv, display: argv.join(" ") } };
|
|
104
104
|
}
|
|
105
105
|
|
|
106
|
-
/** The
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
|
|
106
|
+
/** The last OATS Desktop with the hard CLI ceiling: it refuses every CLI newer than its own minor version.
|
|
107
|
+
* Every later Desktop accepts such a CLI and holds back what it was not tested with (oats-project/oats#1879,
|
|
108
|
+
* option 3). The release lane corrects this one value if another 0.69 patch ships before the soft ceiling;
|
|
109
|
+
* test/release-packaging.test.mjs and test/above-tested-wording.test.mjs read it from here. */
|
|
110
|
+
export const LAST_HARD_CEILING_DESKTOP = "0.69.17";
|
|
111
|
+
|
|
112
|
+
/** The line `oats update` prints under its steps when the CLI's install (`cli`, from planCliUpdate) moves
|
|
113
|
+
* it to another minor or major version. A Desktop accepts a newer CLI and holds back every change it was
|
|
114
|
+
* not tested with until it is updated (docs/upgrading.md, "Either order works"); its core sentence is the
|
|
115
|
+
* Desktop's own notice for the same version, written here because the kernel imports nothing of its
|
|
116
|
+
* clients (packages/client/above-tested-notice.mjs `heldBackSentence`; test/above-tested-wording.test.mjs
|
|
117
|
+
* holds the two equal). An older Desktop, one with the hard ceiling, refuses the CLI instead, which the
|
|
118
|
+
* line says after the core. Fixed text, whether or not a Desktop is installed here: one on another host,
|
|
119
|
+
* or installed another way, would not be seen. Prerelease and build metadata do not count (0.56.0-rc.1 to
|
|
111
120
|
* 0.56.0 says nothing); an installed version that cannot be read is another minor.
|
|
112
121
|
* → the line, or null when there is no install or it stays within the minor. */
|
|
113
122
|
export function desktopFirstLine(cli) {
|
|
114
123
|
if (!cli.step) return null;
|
|
115
124
|
const from = parseVersion(cli.installed), to = parseVersion(cli.target);
|
|
116
125
|
if (from && to && from.major === to.major && from.minor === to.minor) return null;
|
|
117
|
-
|
|
126
|
+
const minor = to ? `${to.major}.${to.minor}` : cli.target;
|
|
127
|
+
return `An OATS Desktop older than ${minor} holds actions back with this CLI. `
|
|
128
|
+
+ `Stop, Remove, schedules, triggers and every other change except Spawn, Start, Restart, colour and background are off until you install OATS Desktop ${cli.target}. `
|
|
129
|
+
+ `(An OATS Desktop of ${LAST_HARD_CEILING_DESKTOP} or older refuses this CLI until then.)`;
|
|
118
130
|
}
|
|
119
131
|
|
|
120
132
|
/** The lookup `oats update` makes before any step, as arguments: the bridge's exact version on
|
package/lib/schedule.mjs
CHANGED
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
* succeeded. A launch whose side effects cannot be confirmed stays
|
|
22
22
|
* `unknown`, its job blocked until `reconcile` proves what happened; it keeps
|
|
23
23
|
* its host slot too, unless it is a command whose process exit was observed. */
|
|
24
|
+
import { hostRunAdmissionOpen, hostRunChildArgs } from "./host-run-admission.mjs";
|
|
24
25
|
import { runScheduleCommand } from "./schedule-command.mjs";
|
|
25
26
|
import { closeSync, constants as fsConstants, existsSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
26
27
|
import { createHash } from "node:crypto";
|
|
@@ -717,7 +718,7 @@ function spawnPreview(ws, def, { root, discover }, io) {
|
|
|
717
718
|
if (context.error) throw scheduleError(context.error.code, context.error.message);
|
|
718
719
|
if (String(def.agent).startsWith("-")) throw scheduleError("E_SCHEDULE_INVALID", `agent: ${JSON.stringify(def.agent)} is not a soul name`, { field: "agent" });
|
|
719
720
|
const argv = ["spawn", def.agent, `--dir=${context.dir}`, ...(discover ? [] : [`--agents-root=${root}`]), "--preview", "--json", ...spawnSelectorArgs(def)];
|
|
720
|
-
const r = runScheduleCommand(process.execPath,
|
|
721
|
+
const r = runScheduleCommand(process.execPath, hostRunChildArgs(io?.oatsBin || OATS_BIN, argv, true), { cwd: context.dir, timeout: io?.commandTimeoutMs || COMMAND_TIMEOUT_MS, graceMs: io?.commandGraceMs, maxBuffer: 16 * 1024 * 1024, env: childEnv() });
|
|
721
722
|
const unconfirmed = (message) => scheduleError("E_SPAWN_FAILED", message, { unconfirmed: true });
|
|
722
723
|
if (r.interrupted === true) throw unconfirmed("the spawn preview was interrupted; its result is unconfirmed");
|
|
723
724
|
if (r.error?.code === "ETIMEDOUT" || (r.status === null && ["SIGTERM", "SIGKILL"].includes(r.signal))) throw unconfirmed("the spawn preview timed out; its result is unconfirmed");
|
|
@@ -751,7 +752,7 @@ function spawnViaCli(ws, root, agent, opts, context, io) {
|
|
|
751
752
|
writeFileSync(taskFile, opts.task, { mode: 0o600 });
|
|
752
753
|
argv.push(`--task-file=${taskFile}`);
|
|
753
754
|
let r;
|
|
754
|
-
try { r = runScheduleCommand(process.execPath,
|
|
755
|
+
try { r = runScheduleCommand(process.execPath, hostRunChildArgs(io?.oatsBin || OATS_BIN, argv, true), { cwd: context.dir, timeout: io?.commandTimeoutMs || COMMAND_TIMEOUT_MS, graceMs: io?.commandGraceMs, maxBuffer: 16 * 1024 * 1024, env: childEnv() }); }
|
|
755
756
|
finally { try { rmSync(taskFile, { force: true }); } catch { /* best effort */ } }
|
|
756
757
|
const interrupted = r.interrupted === true;
|
|
757
758
|
const timedOut = r.error?.code === "ETIMEDOUT" || (!interrupted && r.status === null && ["SIGTERM", "SIGKILL"].includes(r.signal));
|
|
@@ -836,7 +837,7 @@ function launchCommand(ws, def, io) {
|
|
|
836
837
|
let envelope, timedOut = false, interrupted = false, raw = "", exit;
|
|
837
838
|
if (io?.command) { envelope = io.command({ cwd: def.cwd, argv }); exit = { status: null, signal: null }; }
|
|
838
839
|
else {
|
|
839
|
-
const r = runScheduleCommand(process.execPath,
|
|
840
|
+
const r = runScheduleCommand(process.execPath, hostRunChildArgs(io?.oatsBin || OATS_BIN, argv, true), { cwd: def.cwd, timeout: io?.commandTimeoutMs || COMMAND_TIMEOUT_MS, graceMs: io?.commandGraceMs, maxBuffer: 16 * 1024 * 1024, env: childEnv() });
|
|
840
841
|
interrupted = r.interrupted === true;
|
|
841
842
|
timedOut = r.error?.code === "ETIMEDOUT" || (!interrupted && r.status === null && ["SIGTERM", "SIGKILL"].includes(r.signal));
|
|
842
843
|
if (r.status !== null || r.signal !== null) exit = { status: r.status, signal: r.signal };
|
|
@@ -1037,6 +1038,7 @@ export function tickWorkspace(ws, { now = new Date(), firedAt = now, io, reg, ws
|
|
|
1037
1038
|
// skipped or pending job keeps its place at the front of the line.
|
|
1038
1039
|
const order = (candidates || Object.keys(defs.jobs)).filter((id) => Object.prototype.hasOwnProperty.call(defs.jobs, id) && (!only || id === only)).sort((a, b) => launchOrderKey(st.jobs[a]).localeCompare(launchOrderKey(st.jobs[b])) || a.localeCompare(b));
|
|
1039
1040
|
for (const id of order) {
|
|
1041
|
+
if (!dryRun && !hostRunAdmissionOpen()) break;
|
|
1040
1042
|
const def = defs.jobs[id];
|
|
1041
1043
|
const js = stateOf(id);
|
|
1042
1044
|
let due, cronError;
|
package/lib/servers.mjs
CHANGED
|
@@ -772,9 +772,10 @@ export function routeCommand(serverId, cmd, oatsArgs, io = {}) {
|
|
|
772
772
|
let answer = refusal;
|
|
773
773
|
try { answer ??= spawnOnHost(applyArgs); }
|
|
774
774
|
catch (e) {
|
|
775
|
-
//
|
|
776
|
-
//
|
|
777
|
-
|
|
775
|
+
// A bound apply that came back with no answer, because the link failed or the host's oats
|
|
776
|
+
// ended without one: the host may have started it. The decision and the key travel with the
|
|
777
|
+
// failure, so the spawn sent again lands on that one instance.
|
|
778
|
+
if (bound && ((e.code === "E_SSH" && e.details?.sshStarted !== false) || e.code === "E_REMOTE_ENVELOPE")) {
|
|
778
779
|
const fix = `run the same spawn again with ${suggesting ? `--name ${bound.decision.instance} in place of --suggest-name, and ` : ""}--expect-decision ${bound.decision.revision} --idempotency-key ${bound.idempotencyKey}: it answers the instance this attempt created there, or creates it`;
|
|
779
780
|
e.message = `${e.message}; the host may have started the spawn: ${fix}`;
|
|
780
781
|
e.details = { ...e.details, decision: bound.decision, idempotencyKey: bound.idempotencyKey, fix };
|
package/lib/session-end.mjs
CHANGED
|
@@ -101,3 +101,12 @@ export function lostReasonOf(home, meta) {
|
|
|
101
101
|
const stop = stopOfCurrentStart(home, meta);
|
|
102
102
|
return stop?.resume === true && stop.reason === HOST_STOP_REASON ? HOST_STOP_REASON : null;
|
|
103
103
|
}
|
|
104
|
+
|
|
105
|
+
/** A current-start administrative stop's time, or null when the receipt cannot establish it.
|
|
106
|
+
* A future, pre-start or unreadable timestamp cannot relax the sweep's eligibility or budget. */
|
|
107
|
+
export function hostStoppedAt(home, meta, { now = Date.now() } = {}) {
|
|
108
|
+
const stop = stopOfCurrentStart(home, meta);
|
|
109
|
+
if (stop?.resume !== true || stop.reason !== HOST_STOP_REASON || typeof stop.at !== "string") return null;
|
|
110
|
+
const at = Date.parse(stop.at), startedAt = currentStartTime(meta);
|
|
111
|
+
return Number.isFinite(at) && Number.isFinite(startedAt) && at >= startedAt && at <= now ? at : null;
|
|
112
|
+
}
|
|
@@ -25,7 +25,7 @@ export const OUTCOME = Object.freeze({ RESUMED: "resumed", FRESH: "fresh", ALREA
|
|
|
25
25
|
export const ROW_REASON = Object.freeze({
|
|
26
26
|
UNKNOWN: "unknown", START_BUSY: "start-busy", NOT_LOST: "not-lost", NOT_SEEN_RECENTLY: "not-seen-recently",
|
|
27
27
|
RESUME_CONFIRMATION: RESUME_WAITING.RESUME_CONFIRMATION, LAUNCH_PROMPT: "launch-prompt",
|
|
28
|
-
START_REFUSED: "start-refused",
|
|
28
|
+
START_REFUSED: "start-refused", START_REFUSED_RETRYING: "start-refused-retrying", KEEPS_GETTING_LOST: "keeps-getting-lost",
|
|
29
29
|
LIFECYCLE_BUSY: "lifecycle-busy",
|
|
30
30
|
});
|
|
31
31
|
|
package/lib/session-resume.mjs
CHANGED
|
@@ -10,7 +10,8 @@
|
|
|
10
10
|
* - Only a loss the kernel can vouch for is resumed (lib/session-seen.mjs resumeEligible, from what
|
|
11
11
|
* local listings recorded of the home): seen running for its current start, not judged by an
|
|
12
12
|
* earlier sweep, first seen lost less than 24 hours ago, and, within one boot, lost less than 24
|
|
13
|
-
* hours after it was last seen running. Any other is `skipped`, `not-seen-recently`. A
|
|
13
|
+
* hours after it was last seen running. Any other is `skipped`, `not-seen-recently`. A home the
|
|
14
|
+
* sweep started is recorded as seen running for its new start. A sweep that
|
|
14
15
|
* is not a plan judges each lost home it answered (`judgedBy`), so it is answered once.
|
|
15
16
|
* - Each home starts as a child process of this kernel (`session start --resume --home <home>
|
|
16
17
|
* --json`, its cwd the home's deployment), so each start resolves its own deployment's scope; at
|
|
@@ -20,18 +21,35 @@
|
|
|
20
21
|
* and boot id; a lock whose holder is gone, or that was taken in another boot, is taken over at
|
|
21
22
|
* once. A second sweep answers E_RESUME_BUSY; a lock that needs a person, E_RESUME_LOCK_STUCK,
|
|
22
23
|
* which `oats doctor` reports too (acquireResumeLock, resumeLockProblem).
|
|
23
|
-
* - No retry storm
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
* `keeps-getting-lost` without a start
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
24
|
+
* - No retry storm, and no loss given up for a cause that passes.
|
|
25
|
+
* A start counts when it launched something (`resumed`, `fresh`, `blocked`), noted as soon as
|
|
26
|
+
* that start has answered, so a sweep that is killed while other starts run has counted the
|
|
27
|
+
* ones that answered. A home is started by sweeps at most RESUME_LIMIT times in
|
|
28
|
+
* RESUME_WINDOW_MS (counted by home: each start makes a new start id), then reported
|
|
29
|
+
* `keeps-getting-lost` without a start, with `nextAttemptAt`: when the oldest counted start
|
|
30
|
+
* leaves the window a sweep starts it again.
|
|
31
|
+
* That is not for ever: a home that uses up its starts in RESUME_SPENT_HOURS hours in a row
|
|
32
|
+
* is answered `keeps-getting-lost` with no `nextAttemptAt`, and that loss is judged: no sweep
|
|
33
|
+
* starts it again, and a person does. Hours are in a row when each begins less than
|
|
34
|
+
* RESUME_WINDOW_MS after the one before it cleared; only an hour without one begins the
|
|
35
|
+
* count again. No start does, whoever makes it: a person may start a home the sweep gave up,
|
|
36
|
+
* and if it is lost again in a used-up hour it is given up again at once.
|
|
37
|
+
* A refused start launched nothing: it is not counted, and it does not judge the loss. It is
|
|
38
|
+
* noted, and a sweep tries that home and start again no sooner than RETRY_FIRST_MS after the
|
|
39
|
+
* refusal, doubled at each further refusal up to RETRY_CEILING_MS; until then a sweep reports
|
|
40
|
+
* it `start-refused-retrying` with the refusal's `code` and `message` and `nextAttemptAt`,
|
|
41
|
+
* without a start. The retries end when the loss is no longer one the kernel vouches for
|
|
42
|
+
* (first seen 24 hours ago: `not-seen-recently`).
|
|
43
|
+
* The note is one boot's (`note.json`): another boot's is replaced whole, so a reboot starts
|
|
44
|
+
* the count and the delays again. `session start --resume` run by hand is neither bounded
|
|
45
|
+
* nor counted.
|
|
46
|
+
* A current-start host-stop receipt identifies an administrative restore: a successful one
|
|
47
|
+
* bypasses the crash ceiling without spending or clearing the carried budget. A blocked or
|
|
48
|
+
* failed restore keeps its failure accounting, and eligibility checks still hold.
|
|
32
49
|
* Without a boot id (neither Linux nor macOS) nothing is noted or counted: each sweep tries again.
|
|
33
50
|
* The lock and the note are kernel-private, not a contract. `plan` predicts from the note, starting
|
|
34
51
|
* nothing, taking no lock and writing no note. */
|
|
52
|
+
import { hostRunAdmissionOpen, hostRunChildArgs } from "./host-run-admission.mjs";
|
|
35
53
|
import { execFile } from "node:child_process";
|
|
36
54
|
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
37
55
|
import { join, resolve } from "node:path";
|
|
@@ -41,16 +59,25 @@ import { acquireDirLock, dirLockHeld, stuckLockRemedy } from "./dir-lock.mjs";
|
|
|
41
59
|
import { homeClaimHolder, listInstances, launchEnvRefs, missingLaunchEnvRefs, owedRetireSaid, resumeConversation } from "./core.mjs";
|
|
42
60
|
import { checkStartEnv, launchEnvChangedMessage, launchEnvMissingMessage, launchEnvUnverifiedWarning } from "./start-env.mjs";
|
|
43
61
|
import { FRESH_REASON, NOT_SWEPT, OUTCOME, RESUME_WAITING, ROW_REASON, SESSION_END } from "./session-resume-contract.mjs";
|
|
44
|
-
import { bootId, homeKey, judgeLosses, readSeen, resumeEligible, resumeStateDir, takenInAnotherBoot } from "./session-seen.mjs";
|
|
62
|
+
import { bootId, homeKey, judgeLosses, readSeen, recordSeen, resumeEligible, resumeStateDir, sameBoot, takenInAnotherBoot } from "./session-seen.mjs";
|
|
45
63
|
import { isAgentScopeWarning } from "./tmux-server-scope.mjs";
|
|
64
|
+
import { hostStoppedAt as readHostStoppedAt } from "./session-end.mjs";
|
|
46
65
|
|
|
47
66
|
export { bootId, resumeStateDir };
|
|
48
67
|
|
|
49
68
|
export const RESUME_SCHEMA = 1;
|
|
50
69
|
/** At most this many starts run at once. */
|
|
51
70
|
export const RESUME_CONCURRENCY = 4;
|
|
52
|
-
/** At most this many sweep starts of one home in
|
|
71
|
+
/** At most this many sweep starts of one home in RESUME_WINDOW_MS. */
|
|
53
72
|
export const RESUME_LIMIT = 3;
|
|
73
|
+
/** The span over which a home's sweep starts are counted. */
|
|
74
|
+
export const RESUME_WINDOW_MS = 60 * 60_000;
|
|
75
|
+
/** A home that uses up its starts in this many hours in a row is not started by a sweep again. */
|
|
76
|
+
export const RESUME_SPENT_HOURS = 3;
|
|
77
|
+
/** How long after a refused start a sweep tries it again: this after the first refusal, doubled at
|
|
78
|
+
* each further one, never more than RETRY_CEILING_MS. */
|
|
79
|
+
export const RETRY_FIRST_MS = 60_000;
|
|
80
|
+
export const RETRY_CEILING_MS = 15 * 60_000;
|
|
54
81
|
const lockPath = () => join(resumeStateDir(), "lock");
|
|
55
82
|
const notePath = () => join(resumeStateDir(), "note.json");
|
|
56
83
|
|
|
@@ -101,32 +128,66 @@ export function resumeLockProblem({ boot = bootId() } = {}) {
|
|
|
101
128
|
return { code: RESUME_LOCK_STUCK, message: `${stuck.said}: oats session resume starts nothing on this machine while it is there`, remedy: stuck.remedy };
|
|
102
129
|
}
|
|
103
130
|
|
|
104
|
-
/** This boot's note: `{ bootId, refused: [{ home, startId, code, message, at
|
|
105
|
-
*
|
|
106
|
-
*
|
|
107
|
-
|
|
131
|
+
/** This boot's note as it is stored: `{ bootId, refused: [{ home, startId, code, message, at,
|
|
132
|
+
* attempts }], starts: { <home>: [<ISO time of a sweep start that launched something>] },
|
|
133
|
+
* spent: { <home>: { hours, since } } }` (`since` the oldest counted start of the home's last
|
|
134
|
+
* used-up hour, `hours` how many such hours in a row that makes), each home by its real path
|
|
135
|
+
* (lib/session-seen.mjs homeKey); empty for another boot's note, an unreadable one, or no boot id. */
|
|
136
|
+
function readNote(boot) {
|
|
108
137
|
const note = boot ? readJson(notePath()) : undefined;
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
138
|
+
const map = (v) => (v && typeof v === "object" && !Array.isArray(v) ? v : {});
|
|
139
|
+
if (!note || !sameBoot(note.bootId, boot)) return { bootId: boot, refused: [], starts: {}, spent: {} };
|
|
140
|
+
const starts = {};
|
|
141
|
+
if (note.starts && typeof note.starts === "object" && !Array.isArray(note.starts)) {
|
|
142
|
+
for (const [home, times] of Object.entries(note.starts)) if (Array.isArray(times)) starts[home] = times.filter((t) => typeof t === "string" && Number.isFinite(Date.parse(t)));
|
|
113
143
|
}
|
|
114
|
-
|
|
144
|
+
const spent = Object.fromEntries(Object.entries(map(note.spent)).filter(([, v]) => v && Number.isSafeInteger(v.hours) && v.hours > 0 && Number.isFinite(Date.parse(v.since))));
|
|
145
|
+
return { bootId: boot, refused: Array.isArray(note.refused) ? note.refused.filter((r) => r && typeof r.home === "string") : [], starts, spent };
|
|
115
146
|
}
|
|
116
|
-
/**
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
147
|
+
/** The used-up hour that begins with the counted start `since`, after the home's earlier mark
|
|
148
|
+
* `mark` (a note's `spent` entry, or none) → `{ hours, since }`: the same hour again is the mark
|
|
149
|
+
* itself; an hour that begins less than RESUME_WINDOW_MS after the marked one cleared is one
|
|
150
|
+
* more in a row; any other is the first. */
|
|
151
|
+
export function spentHours(mark, since) {
|
|
152
|
+
if (mark && mark.since === since) return { hours: mark.hours, since };
|
|
153
|
+
const inARow = mark && Date.parse(since) - (Date.parse(mark.since) + RESUME_WINDOW_MS) < RESUME_WINDOW_MS;
|
|
154
|
+
return { hours: inARow ? mark.hours + 1 : 1, since };
|
|
155
|
+
}
|
|
156
|
+
/** The times in `times` that are inside the counting window at `now`, oldest first. */
|
|
157
|
+
const inWindow = (times = [], now) => times.filter((t) => now - Date.parse(t) < RESUME_WINDOW_MS).sort();
|
|
158
|
+
/** This boot's note: `{ bootId, refused, resumes: { <home>: n } }`, `refused` as it is stored
|
|
159
|
+
* (readNote) and `n` the sweep starts of the home counted at `now`: those of the last
|
|
160
|
+
* RESUME_WINDOW_MS. */
|
|
161
|
+
export function readBootNote(boot = bootId(), now = Date.now()) {
|
|
162
|
+
const note = readNote(boot), resumes = {};
|
|
163
|
+
for (const [home, times] of Object.entries(note.starts)) { const n = inWindow(times, now).length; if (n > 0) resumes[home] = n; }
|
|
164
|
+
return { bootId: boot, refused: note.refused, resumes };
|
|
165
|
+
}
|
|
166
|
+
/** When a sweep tries the refused start `prior` (a note's entry) again, in ms: RETRY_FIRST_MS
|
|
167
|
+
* after its refusal, doubled for each refusal before it, at most RETRY_CEILING_MS. An entry
|
|
168
|
+
* whose time does not read is due at once. */
|
|
169
|
+
export function retryDueAt(prior) {
|
|
170
|
+
const at = Date.parse(prior?.at);
|
|
171
|
+
if (!Number.isFinite(at)) return 0;
|
|
172
|
+
const refusals = Number.isSafeInteger(prior.attempts) && prior.attempts > 0 ? prior.attempts : 1;
|
|
173
|
+
return at + Math.min(RETRY_FIRST_MS * 2 ** Math.min(refusals - 1, 30), RETRY_CEILING_MS);
|
|
174
|
+
}
|
|
175
|
+
/** Note a sweep's `refusals` (each replacing its home's earlier entry), the `launched` homes (each
|
|
176
|
+
* a start at `now`, and the starts that left the counting window dropped) and the used-up hours
|
|
177
|
+
* it marked (`spent`: { <home>: { hours, since } }) for this boot; another boot's note is
|
|
178
|
+
* replaced whole. Nothing is written without a boot id. */
|
|
179
|
+
export function noteSweep({ refusals = [], launched = [], spent = {} }, boot = bootId(), now = Date.now()) {
|
|
180
|
+
if (!boot || (!refusals.length && !launched.length && !Object.keys(spent).length)) return;
|
|
181
|
+
const note = readNote(boot);
|
|
122
182
|
const homes = new Set(refusals.map((r) => r.home));
|
|
123
|
-
const
|
|
124
|
-
for (const home of
|
|
183
|
+
const starts = {};
|
|
184
|
+
for (const [home, times] of Object.entries(note.starts)) { const kept = inWindow(times, now); if (kept.length) starts[home] = kept; }
|
|
185
|
+
for (const home of launched) starts[home] = [...(starts[home] ?? []), new Date(now).toISOString()];
|
|
125
186
|
const path = notePath();
|
|
126
187
|
mkdirSync(resumeStateDir(), { recursive: true, mode: 0o700 });
|
|
127
188
|
const tmp = `${path}.${process.pid}.tmp`;
|
|
128
189
|
try {
|
|
129
|
-
writeFileSync(tmp, JSON.stringify({ bootId: boot, refused: [...note.refused.filter((r) => !homes.has(r.home)), ...refusals],
|
|
190
|
+
writeFileSync(tmp, JSON.stringify({ bootId: boot, refused: [...note.refused.filter((r) => !homes.has(r.home)), ...refusals], starts, spent: { ...note.spent, ...spent } }, null, 2) + "\n", { mode: 0o600 });
|
|
130
191
|
renameSync(tmp, path);
|
|
131
192
|
} catch (e) { rmSync(tmp, { force: true }); throw e; }
|
|
132
193
|
}
|
|
@@ -161,11 +222,13 @@ function readDeployment(dir) {
|
|
|
161
222
|
|
|
162
223
|
/** The reasons of the rows a sweep answers without judging the loss, so that a later sweep may
|
|
163
224
|
* resume it: a home OATS was stopping, upgrading or retiring (`lifecycle-busy`), whose next state
|
|
164
|
-
* is that command's to decide
|
|
165
|
-
*
|
|
166
|
-
*
|
|
167
|
-
|
|
168
|
-
|
|
225
|
+
* is that command's to decide; a start that was refused, which launched nothing and says how the
|
|
226
|
+
* host was at that moment (`start-refused`, and `start-refused-retrying` while its delay runs);
|
|
227
|
+
* and a home whose starts of the last RESUME_WINDOW_MS used up the budget (`keeps-getting-lost`),
|
|
228
|
+
* which a sweep starts again when the window has moved on; unless it did so RESUME_SPENT_HOURS
|
|
229
|
+
* hours in a row (the sweep's `givenUp`), which is judged. */
|
|
230
|
+
const NOT_JUDGED = new Set([ROW_REASON.LIFECYCLE_BUSY, ROW_REASON.START_REFUSED, ROW_REASON.START_REFUSED_RETRYING, ROW_REASON.KEEPS_GETTING_LOST]);
|
|
231
|
+
/** Outcomes that can consume the crash budget; a successful host-stop restore is exempt below. */
|
|
169
232
|
const LAUNCHED = new Set([OUTCOME.RESUMED, OUTCOME.FRESH, OUTCOME.BLOCKED]);
|
|
170
233
|
const base = (row) => ({ deployment: row.deployment, instance: row.instance, home: row.home });
|
|
171
234
|
|
|
@@ -215,7 +278,7 @@ function startEnvironment(env) {
|
|
|
215
278
|
* Never rejects. */
|
|
216
279
|
export function kernelAnswer({ execPath, bin, env }, argv, { cwd, code, what }) {
|
|
217
280
|
return new Promise((done) => {
|
|
218
|
-
execFile(execPath,
|
|
281
|
+
execFile(execPath, hostRunChildArgs(bin, argv), { cwd, env: startEnvironment(env), encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }, (error, stdout) => {
|
|
219
282
|
let answer;
|
|
220
283
|
try { answer = JSON.parse(String(stdout).trim().split("\n").pop()); } catch { answer = undefined; }
|
|
221
284
|
if (!answer || typeof answer !== "object") answer = { ok: false, error: { code, message: `${what} exited${error?.code !== undefined ? ` with status ${error.code}` : ""}${error?.signal ? ` on ${error.signal}` : ""} without an answer` } };
|
|
@@ -257,10 +320,11 @@ export async function resumeSessions({ deployment, plan = false, kernel, env = p
|
|
|
257
320
|
else if (row.sessionEnd === SESSION_END.LOST || (row.retirePending && row.launched === true && row.running === false)) candidates.push(row);
|
|
258
321
|
}
|
|
259
322
|
}
|
|
260
|
-
const
|
|
261
|
-
const toStart = [];
|
|
262
|
-
|
|
263
|
-
//
|
|
323
|
+
const now = Date.now(), note = readNote(boot);
|
|
324
|
+
const toStart = [], hostStopped = new Set(), refusedBefore = new Map(), marked = {}, givenUp = new Set();
|
|
325
|
+
const iso = (ms) => new Date(ms).toISOString();
|
|
326
|
+
// Precedence, the first that applies: not seen recently, refused and not yet due again, keeps
|
|
327
|
+
// getting lost, started. The record and the note name a home by its real path (homeKey), whatever spelling the
|
|
264
328
|
// sweep was given; a row shows the path it listed.
|
|
265
329
|
// A home OATS is stopping, upgrading or retiring, or whose retire is owed, comes first: nothing
|
|
266
330
|
// is started there. This read is only a filter (the sweep is not atomic with it): the start's
|
|
@@ -268,15 +332,30 @@ export async function resumeSessions({ deployment, plan = false, kernel, env = p
|
|
|
268
332
|
for (const row of candidates) {
|
|
269
333
|
const startId = row.startId ?? null, key = homeKey(row.home);
|
|
270
334
|
const prior = note.refused.find((r) => r.home === key && (r.startId ?? null) === startId);
|
|
271
|
-
const
|
|
335
|
+
const record = seen.get(key), at = readHostStoppedAt(row.home, row, { now });
|
|
336
|
+
const hostStoppedAt = at !== null && at <= Date.parse(record?.firstNotRunningAt) ? at : null;
|
|
337
|
+
const fit = resumeEligible(record, row, { now, hostStoppedAt });
|
|
272
338
|
const busy = lifecycleBusy(row);
|
|
339
|
+
// A recorded stop of this start with the host is administrative, not another crash.
|
|
340
|
+
// Keep the crash budget, but neither charge a successful restore nor let it prevent one.
|
|
341
|
+
if (hostStoppedAt !== null) hostStopped.add(key);
|
|
273
342
|
if (busy) rows.push({ ...base(row), outcome: OUTCOME.SKIPPED, reason: ROW_REASON.LIFECYCLE_BUSY, message: busy });
|
|
274
343
|
else if (!fit.eligible) rows.push({ ...base(row), outcome: OUTCOME.SKIPPED, reason: ROW_REASON.NOT_SEEN_RECENTLY, message: `${row.instance} is not resumed automatically: ${fit.why}; start it by hand (oats session start --resume --home ${row.home}) if it should run` });
|
|
275
|
-
else if (prior) rows.push({ ...base(row), outcome: OUTCOME.REFUSED, reason: ROW_REASON.
|
|
276
|
-
else if ((
|
|
344
|
+
else if (prior && now < retryDueAt(prior)) rows.push({ ...base(row), outcome: OUTCOME.REFUSED, reason: ROW_REASON.START_REFUSED_RETRYING, code: prior.code, message: prior.message, nextAttemptAt: iso(retryDueAt(prior)) });
|
|
345
|
+
else if (!hostStopped.has(key) && inWindow(note.starts[key], now).length >= RESUME_LIMIT) {
|
|
346
|
+
const counted = inWindow(note.starts[key], now), again = iso(Date.parse(counted[0]) + RESUME_WINDOW_MS);
|
|
347
|
+
const hour = spentHours(note.spent[key], counted[0]);
|
|
348
|
+
if (!plan && (note.spent[key]?.since !== hour.since || note.spent[key]?.hours !== hour.hours)) marked[key] = hour;
|
|
349
|
+
const by = `start it by hand (oats session start --resume --home ${row.home})`;
|
|
350
|
+
if (hour.hours >= RESUME_SPENT_HOURS) {
|
|
351
|
+
givenUp.add(row.home);
|
|
352
|
+
rows.push({ ...base(row), outcome: OUTCOME.REFUSED, reason: ROW_REASON.KEEPS_GETTING_LOST, message: `${row.instance} was resumed ${counted.length} times an hour for ${hour.hours} hours in a row and was lost again each time; no sweep starts it again: ${by}` });
|
|
353
|
+
} else rows.push({ ...base(row), outcome: OUTCOME.REFUSED, reason: ROW_REASON.KEEPS_GETTING_LOST, nextAttemptAt: again, message: `${row.instance} was resumed ${counted.length} times in the last hour and was lost again each time; a sweep starts it again after ${again}, or ${by}` });
|
|
354
|
+
}
|
|
277
355
|
else if (plan) rows.push(predicted(row, env));
|
|
278
|
-
else toStart.push(row);
|
|
356
|
+
else { toStart.push(row); if (prior) refusedBefore.set(key, prior); }
|
|
279
357
|
}
|
|
358
|
+
if (Object.keys(marked).length) noteSweep({ spent: marked }, boot, now);
|
|
280
359
|
// Each start's answer is noted as it comes, under the lock (noteSweep is synchronous: two answers
|
|
281
360
|
// never interleave in it), before the sweep waits on the others. A note that cannot be written
|
|
282
361
|
// takes no more home; the lock is held until every start already running has answered, and the
|
|
@@ -289,24 +368,38 @@ export async function resumeSessions({ deployment, plan = false, kernel, env = p
|
|
|
289
368
|
// A home removed between the read and its start (a retire finished in between) is not lost:
|
|
290
369
|
// whatever its start answered.
|
|
291
370
|
const gone = answer?.ok !== true && !existsSync(row.home);
|
|
292
|
-
const answered = gone ? { ...base(row), outcome: OUTCOME.SKIPPED, reason: ROW_REASON.NOT_LOST, ...(typeof answer?.error?.code === "string" ? { code: answer.error.code } : {}) } : rowFromStart(row, answer);
|
|
293
371
|
const key = homeKey(row.home);
|
|
372
|
+
// A refusal is noted with how many times this home and start were refused, which sets when a
|
|
373
|
+
// sweep tries it again; the row says when.
|
|
374
|
+
const refusal = { home: key, startId: row.startId ?? null, at: iso(Date.now()), attempts: (refusedBefore.get(key)?.attempts ?? 0) + 1 };
|
|
375
|
+
const from = gone ? { ...base(row), outcome: OUTCOME.SKIPPED, reason: ROW_REASON.NOT_LOST, ...(typeof answer?.error?.code === "string" ? { code: answer.error.code } : {}) } : rowFromStart(row, answer);
|
|
376
|
+
const answered = from.outcome === OUTCOME.REFUSED && boot ? { ...from, nextAttemptAt: iso(retryDueAt(refusal)) } : from;
|
|
294
377
|
try {
|
|
295
378
|
noteSweep({
|
|
296
|
-
refusals: answered.outcome === OUTCOME.REFUSED ? [{
|
|
297
|
-
launched: LAUNCHED.has(answered.outcome) ? [key] : [],
|
|
379
|
+
refusals: answered.outcome === OUTCOME.REFUSED ? [{ ...refusal, code: answered.code, message: answered.message }] : [],
|
|
380
|
+
launched: LAUNCHED.has(answered.outcome) && (!hostStopped.has(key) || answered.outcome === OUTCOME.BLOCKED) ? [key] : [],
|
|
298
381
|
}, boot);
|
|
299
382
|
} catch (e) { noteFailure ??= e; }
|
|
300
383
|
return answered;
|
|
301
|
-
}, () => noteFailure !== null)).filter(Boolean);
|
|
384
|
+
}, () => noteFailure !== null || !hostRunAdmissionOpen())).filter(Boolean);
|
|
302
385
|
rows.push(...started);
|
|
303
|
-
// Each lost home this sweep answered is judged
|
|
304
|
-
// never uses one up). A plan judges nothing.
|
|
386
|
+
// Each lost home this sweep answered is judged, but for the rows of NOT_JUDGED: no later sweep
|
|
387
|
+
// resumes a judged loss (a status read never uses one up). A plan judges nothing. Judge the
|
|
388
|
+
// lost start before recording its replacement: a new start resets the record and its judgement.
|
|
305
389
|
if (!plan) {
|
|
306
390
|
const lost = new Set(candidates.map((c) => c.home)), byDeployment = new Map();
|
|
307
|
-
for (const r of rows) if (lost.has(r.home) && !NOT_JUDGED.has(r.reason)) byDeployment.set(r.deployment, [...(byDeployment.get(r.deployment) ?? []), { home: r.home, startId: candidates.find((c) => c.home === r.home).startId ?? null, outcome: r.outcome }]);
|
|
391
|
+
for (const r of rows) if (lost.has(r.home) && (!NOT_JUDGED.has(r.reason) || givenUp.has(r.home))) byDeployment.set(r.deployment, [...(byDeployment.get(r.deployment) ?? []), { home: r.home, startId: candidates.find((c) => c.home === r.home).startId ?? null, outcome: r.outcome }]);
|
|
308
392
|
for (const [dir, judged] of byDeployment) judgeLosses(join(dir, "agents"), judged, { boot });
|
|
309
393
|
}
|
|
394
|
+
// A home this sweep started is recorded as seen running for its new start. Without this,
|
|
395
|
+
// a home lost again before the next listing would have no record for its start: no later
|
|
396
|
+
// sweep would resume it, and its starts would never be counted.
|
|
397
|
+
const launchedIn = new Map();
|
|
398
|
+
for (const r of started) if (r.outcome === OUTCOME.RESUMED || r.outcome === OUTCOME.FRESH) launchedIn.set(r.deployment, (launchedIn.get(r.deployment) ?? new Set()).add(r.home));
|
|
399
|
+
for (const [dir, homes] of launchedIn) {
|
|
400
|
+
const root = join(dir, "agents");
|
|
401
|
+
try { recordSeen(root, listInstances(root).flatMap((a) => a.instances || []).map((i) => (homes.has(i.home) ? { ...i, running: true } : i)), { boot }); } catch { /* the next listing records it */ }
|
|
402
|
+
}
|
|
310
403
|
if (noteFailure) throw noteFailure;
|
|
311
404
|
rows.sort((a, b) => (a.deployment < b.deployment ? -1 : a.deployment > b.deployment ? 1 : a.home < b.home ? -1 : a.home > b.home ? 1 : 0));
|
|
312
405
|
return { schemaVersion: RESUME_SCHEMA, plan: !!plan, deployments, rows, ...(hostWide.size ? { warnings: [...hostWide].sort() } : {}) };
|