@awebai/oats 0.69.22 → 0.69.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/oats.mjs +185 -38
- package/docs/capabilities.md +365 -7
- package/docs/capability-manifest.schema.json +37 -1
- package/docs/configuration.md +27 -7
- package/docs/container.md +206 -228
- package/docs/design/2026-11-19-sandboxes.md +1352 -0
- package/docs/design/2026-11-20-provider-home-lifecycle.md +320 -0
- package/docs/design/README.md +7 -0
- package/docs/desktop-cli-api.md +673 -81
- package/docs/desktop.md +241 -65
- package/docs/execution-targets.md +186 -3
- package/docs/first-team.md +11 -0
- package/docs/implementation.md +6 -2
- package/docs/integrations.md +3 -3
- package/docs/knowledge-capability-authoring.md +10 -7
- package/docs/knowledge-reference/acceptance.md +21 -19
- package/docs/knowledge-reference/adoption.md +18 -4
- package/docs/knowledge-reference/model.md +11 -9
- package/docs/knowledge-reference/package-craft.md +2 -2
- package/docs/knowledge-reference/proposal.md +127 -0
- package/docs/knowledge-reference/provider-mapping.md +10 -10
- package/docs/knowledge-reference/reader-capture.md +26 -22
- package/docs/knowledge-theory.md +1 -1
- package/docs/knowledge.md +25 -16
- package/docs/official-catalog.md +4 -4
- package/docs/packages.md +13 -13
- package/docs/release-notes/v0.69.23.md +231 -0
- package/docs/release-notes/v0.69.24.md +264 -0
- package/docs/servers.md +10 -6
- package/docs/souls-and-instances.md +96 -22
- package/docs/tui.md +25 -4
- package/docs/upgrading.md +19 -5
- package/docs/workspaces.md +14 -6
- package/lib/argv.mjs +6 -2
- package/lib/capability-contract.mjs +2 -2
- package/lib/capability-show.mjs +11 -1
- package/lib/command-flags.mjs +40 -3
- package/lib/compile-cache.mjs +1 -1
- package/lib/core.mjs +1201 -195
- package/lib/errors.mjs +4 -1
- package/lib/hook-env.mjs +106 -0
- package/lib/host-run.mjs +11 -3
- package/lib/instance-board.mjs +337 -0
- package/lib/instance-events.mjs +48 -10
- package/lib/instance-inspect.mjs +44 -0
- package/lib/instance-lifecycle.mjs +34 -18
- package/lib/instance-listing.mjs +113 -16
- package/lib/instance-related-work.mjs +143 -0
- package/lib/kernel-features.mjs +13 -1
- package/lib/node-version.mjs +9 -0
- package/lib/provider-call-child.mjs +60 -0
- package/lib/provider-resume.mjs +83 -0
- package/lib/provider-test-seams.mjs +20 -0
- package/lib/read-call.mjs +2 -0
- package/lib/reserved-launch-env.mjs +8 -1
- package/lib/retire-admission.mjs +5 -1
- package/lib/schedule.mjs +14 -4
- package/lib/servers.mjs +151 -17
- package/lib/session-inspect.mjs +104 -13
- package/lib/session-providers.mjs +528 -0
- package/lib/session-resume.mjs +68 -15
- package/lib/stable-node.mjs +3 -0
- package/lib/triggers.mjs +3 -1
- package/package-catalog.json +3 -3
- package/package.json +2 -1
- package/packages/client/capability-list.mjs +21 -0
- package/packages/client/cli-adapter.mjs +19 -4
- package/packages/client/cli-locator.mjs +5 -1
- package/packages/client/deployment-data.mjs +27 -3
- package/packages/client/deployment-panel.mjs +27 -4
- package/packages/client/home-board.mjs +33 -19
- package/packages/client/inspect-boundary.mjs +18 -8
- package/packages/client/instance-acts.mjs +30 -3
- package/packages/client/instance-board.mjs +206 -0
- package/packages/client/instance-presentation.mjs +55 -0
- package/packages/client/instance-tree.mjs +34 -4
- package/packages/client/keymap.mjs +5 -0
- package/packages/client/liveness.mjs +15 -4
- package/packages/client/mail-contract.mjs +130 -0
- package/packages/client/own-codes.mjs +5 -1
- package/packages/client/panel-views.mjs +7 -7
- package/packages/client/related-work.mjs +84 -0
- package/packages/client/remote-roster.mjs +9 -3
- package/packages/client/session-backends.mjs +31 -0
- package/packages/client/session-start.mjs +18 -6
- package/packages/client/spawn-apply-cli.mjs +5 -3
- package/packages/client/spawn-apply-contract.mjs +25 -2
- package/packages/client/spawn-apply.mjs +5 -1
- package/packages/client/spawn-messages.mjs +3 -0
- package/packages/client/terminal-contract.mjs +4 -0
- package/packages/client/waiting-on-you.mjs +12 -2
- package/packages/client/workspace-cli.mjs +13 -2
- package/packages/tui/lib/model.mjs +14 -3
- package/packages/tui/lib/store.mjs +12 -3
- package/skills/oats-getting-started/SKILL.md +11 -8
- package/docs/knowledge-reference/harvester.md +0 -109
- package/lib/instance-same-work.mjs +0 -67
- package/packages/client/same-work.mjs +0 -49
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/** The "related work" flag (feature instance-related-work; the same-work flag of awebai/oats#1577, renamed):
|
|
2
|
+
* an instance's declared "related work" with the instance it is bound to, its parent or its sibling, which
|
|
3
|
+
* clients draw as a line between the two. A home file of its own, `<home>/.oats-related-work.json`, so that a
|
|
4
|
+
* start, restart, stop or resume, which rewrite instance.json, never lose it; only `oats instance
|
|
5
|
+
* related-work … off` or retirement removes it. The kernel never sets it on its own: only `oats spawn
|
|
6
|
+
* --related-work` and the verb do (and their deprecated aliases, `--same-work` and `oats instance same-work`).
|
|
7
|
+
*
|
|
8
|
+
* The flag is BOUND to the relation it was set against: the home's recorded `parentInstance` when it has
|
|
9
|
+
* one, else its recorded `siblingInstance` (relatedWorkBinding, the one choice). The record names that
|
|
10
|
+
* instance and the kind of edge, and the flag is on only while that edge is still the home's
|
|
11
|
+
* (relatedWorkHeld, the one rule). A relink that changes the relation (a retire's splice, a `--relation
|
|
12
|
+
* parent` spawn re-pointing its anchor) leaves the record bound to the old name, so the flag reads off and
|
|
13
|
+
* nothing errors. A sibling binding names the sibling the home records: for a sibling of a root, that root;
|
|
14
|
+
* for a sibling of a sibling, the anchor it was spawned beside (lib/core.mjs records the anchor itself).
|
|
15
|
+
*
|
|
16
|
+
* The record is `{ relatedWorkApi: 1, parent | sibling, at }`, exactly one of `parent` and `sibling`,
|
|
17
|
+
* written atomically (writeThroughPrivateFile) at mode 0600; present means on. A reader takes a regular file
|
|
18
|
+
* only, without following a link, up to RECORD_MAX bytes: anything that is not a valid record reads as no
|
|
19
|
+
* binding (null), never an error.
|
|
20
|
+
*
|
|
21
|
+
* The compatibility layer with the same-work flag it replaced (item 4 of the related-work contract), all of
|
|
22
|
+
* it in this module and its callers named here: the old home file `.oats-same-work.json`,
|
|
23
|
+
* `{ sameWorkApi: 1, parent, at }`, reads as a parent binding while nothing stands at the new name
|
|
24
|
+
* (readRelatedWork); the write that creates the new file removes the old one, and a removal takes both
|
|
25
|
+
* (writeRelatedWork). Both names are kernel-owned (lib/capability-contract.mjs KERNEL_HOME_NAME_PREFIXES,
|
|
26
|
+
* lib/core.mjs KERNEL_HOME_RECEIPTS, the manifest schema). The old row field `sameWork: true` stands beside
|
|
27
|
+
* `relatedWork: true` for a parent binding (lib/instance-listing.mjs), and the old spellings are aliases
|
|
28
|
+
* (lib/command-flags.mjs SAME_WORK_ALIASES). Dependency-free: the status rows, spawn and the command all go
|
|
29
|
+
* through here. */
|
|
30
|
+
import { lstatSync, rmSync } from "node:fs";
|
|
31
|
+
import { join } from "node:path";
|
|
32
|
+
import { readFileHead } from "./bounded-read.mjs";
|
|
33
|
+
import { writeThroughPrivateFile } from "./write-through-private-file.mjs";
|
|
34
|
+
|
|
35
|
+
/** The record's own version. */
|
|
36
|
+
export const RELATED_WORK_API = 1;
|
|
37
|
+
/** The home file, kernel-owned: in KERNEL_HOME_NAME_PREFIXES (lib/capability-contract.mjs), the retire
|
|
38
|
+
* fingerprint's receipts (lib/core.mjs KERNEL_HOME_RECEIPTS) and the manifest schema. */
|
|
39
|
+
export const RELATED_WORK_FILE = ".oats-related-work.json";
|
|
40
|
+
/** The file of the same-work flag the new one replaced: read while the new one is absent, never written,
|
|
41
|
+
* removed by the write that creates the new one and by a removal; kernel-owned. */
|
|
42
|
+
export const SAME_WORK_FILE = ".oats-same-work.json";
|
|
43
|
+
/** The old record's own version. */
|
|
44
|
+
const SAME_WORK_API = 1;
|
|
45
|
+
/** A record larger than this is not one. */
|
|
46
|
+
export const RECORD_MAX = 4096;
|
|
47
|
+
|
|
48
|
+
const isName = (v) => typeof v === "string" && v !== "";
|
|
49
|
+
|
|
50
|
+
/** The binding a home with lineage `{ parentInstance, siblingInstance }` (its instance.json, or a spawn's
|
|
51
|
+
* lineage before it is written) takes: `{ parent }` when it records a parent, else `{ sibling }` when it
|
|
52
|
+
* records a sibling, else null (no flag can be set). The one choice: spawn, the verb and the route ask it. */
|
|
53
|
+
export function relatedWorkBinding(lineage) {
|
|
54
|
+
if (isName(lineage?.parentInstance)) return { parent: lineage.parentInstance };
|
|
55
|
+
if (isName(lineage?.siblingInstance)) return { sibling: lineage.siblingInstance };
|
|
56
|
+
return null;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** The JSON object in the file at `path`, or null: no file, not a regular file (a link is never followed),
|
|
60
|
+
* over RECORD_MAX bytes, unreadable, not a JSON object. Never throws. */
|
|
61
|
+
function readRecord(path) {
|
|
62
|
+
try {
|
|
63
|
+
const { bytes, size } = readFileHead(path, RECORD_MAX + 1);
|
|
64
|
+
if (size > RECORD_MAX) return null;
|
|
65
|
+
const record = JSON.parse(bytes.toString("utf8"));
|
|
66
|
+
return record && typeof record === "object" && !Array.isArray(record) ? record : null;
|
|
67
|
+
} catch { return null; }
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Whether anything at all stands at `path` (a file, a link, a directory, one that can't be read). */
|
|
71
|
+
function standsAt(path) {
|
|
72
|
+
try { lstatSync(path); return true; } catch (e) { return e?.code !== "ENOENT"; }
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** The binding `home`'s record holds, `{ parent }` or `{ sibling }`, or null. The new record, while anything
|
|
76
|
+
* stands at its name: null when it is not a record (readRecord), another relatedWorkApi, or not exactly one
|
|
77
|
+
* of a non-empty string `parent` and `sibling`. With nothing at the new name, the old same-work record, as
|
|
78
|
+
* a parent binding: null when it is not a record, another sameWorkApi, or no non-empty string `parent`.
|
|
79
|
+
* The one read of both files. Never throws. */
|
|
80
|
+
export function readRelatedWork(home) {
|
|
81
|
+
const path = join(home, RELATED_WORK_FILE);
|
|
82
|
+
if (!standsAt(path)) {
|
|
83
|
+
const old = readRecord(join(home, SAME_WORK_FILE));
|
|
84
|
+
return old?.sameWorkApi === SAME_WORK_API && isName(old.parent) ? { parent: old.parent } : null;
|
|
85
|
+
}
|
|
86
|
+
const record = readRecord(path);
|
|
87
|
+
if (!record || record.relatedWorkApi !== RELATED_WORK_API) return null;
|
|
88
|
+
const hasParent = Object.hasOwn(record, "parent"), hasSibling = Object.hasOwn(record, "sibling");
|
|
89
|
+
if (hasParent === hasSibling) return null;
|
|
90
|
+
if (hasParent) return isName(record.parent) ? { parent: record.parent } : null;
|
|
91
|
+
return isName(record.sibling) ? { sibling: record.sibling } : null;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const sameBinding = (a, b) => !!a && !!b && a.parent === b.parent && a.sibling === b.sibling;
|
|
95
|
+
|
|
96
|
+
/** Remove whatever stands at `path` (a file, a link, a directory), and nothing when nothing does. */
|
|
97
|
+
const removeAt = (path) => { if (standsAt(path)) rmSync(path, { recursive: true, force: true }); };
|
|
98
|
+
|
|
99
|
+
/** Bind `home`'s flag to `binding` (`{ parent: <name> }` or `{ sibling: <name> }`), or remove it when
|
|
100
|
+
* `binding` is null. → `{ changed }`. A home already holding `binding` (readRelatedWork: from either file)
|
|
101
|
+
* is no change: nothing is written. Otherwise the new record is written, and the old same-work file, if
|
|
102
|
+
* anything stands at its name, is removed after it. A removal takes whatever stands at either name (an
|
|
103
|
+
* unreadable file, a link, a directory), and is a change only when a valid binding was read. The caller
|
|
104
|
+
* decided `binding` (relatedWorkBinding of the home's record) and holds the right to change the home. */
|
|
105
|
+
export function writeRelatedWork(home, binding, { now = () => new Date() } = {}) {
|
|
106
|
+
if (binding !== null) {
|
|
107
|
+
const keys = binding && typeof binding === "object" && !Array.isArray(binding) ? Object.keys(binding) : [];
|
|
108
|
+
if (keys.length !== 1 || !["parent", "sibling"].includes(keys[0]) || !isName(binding[keys[0]])) {
|
|
109
|
+
throw new TypeError("writeRelatedWork: binding is { parent: <name> }, { sibling: <name> } or null");
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
const path = join(home, RELATED_WORK_FILE), oldPath = join(home, SAME_WORK_FILE);
|
|
113
|
+
const before = readRelatedWork(home);
|
|
114
|
+
if (binding !== null && sameBinding(before, binding)) return { changed: false };
|
|
115
|
+
if (binding === null) {
|
|
116
|
+
removeAt(path);
|
|
117
|
+
removeAt(oldPath);
|
|
118
|
+
return { changed: before !== null };
|
|
119
|
+
}
|
|
120
|
+
// A directory at the name is no record, and a rename cannot replace it: it goes first. A link or a file
|
|
121
|
+
// is replaced by the rename itself.
|
|
122
|
+
let present;
|
|
123
|
+
try { present = lstatSync(path); } catch (e) { if (e?.code !== "ENOENT") throw e; }
|
|
124
|
+
if (present?.isDirectory()) rmSync(path, { recursive: true, force: true });
|
|
125
|
+
const bytes = JSON.stringify({ relatedWorkApi: RELATED_WORK_API, ...binding, at: now().toISOString() }) + "\n";
|
|
126
|
+
writeThroughPrivateFile(path, `${path}.tmp-${process.pid}`, bytes, 0o600);
|
|
127
|
+
// The new record stands: the old one would never be read again.
|
|
128
|
+
removeAt(oldPath);
|
|
129
|
+
return { changed: true };
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** THE rule: the binding of `home` that holds against the row's current lineage `{ parentInstance,
|
|
133
|
+
* siblingInstance }`, or null. A parent binding holds only while its name is the current `parentInstance`;
|
|
134
|
+
* a sibling binding only while the row records no parent and its name is the current `siblingInstance`. */
|
|
135
|
+
export function relatedWorkHeld(home, lineage) {
|
|
136
|
+
const bound = readRelatedWork(home);
|
|
137
|
+
return bound !== null && sameBinding(bound, relatedWorkBinding(lineage)) ? bound : null;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** Whether `home` does related work, by the rule (relatedWorkHeld). */
|
|
141
|
+
export function relatedWorkOf(home, lineage) {
|
|
142
|
+
return relatedWorkHeld(home, lineage) !== null;
|
|
143
|
+
}
|
package/lib/kernel-features.mjs
CHANGED
|
@@ -8,6 +8,18 @@ import { SESSION_RESUME_FEATURE } from "./session-resume-contract.mjs";
|
|
|
8
8
|
/** `oats spawn --task-file -` reads the task from standard input: how a routed spawn's task reaches
|
|
9
9
|
* its host (lib/servers.mjs routeCommand); a host that does not list this gets it as an argument. */
|
|
10
10
|
export const SPAWN_TASK_STDIN_FEATURE = "spawn-task-stdin";
|
|
11
|
-
|
|
11
|
+
/** Session providers (lib/session-providers.mjs): a session backend a capability supplies, beside the
|
|
12
|
+
* kernel's built-in ones. A host that lists this takes `--backend <provider>`, records and answers
|
|
13
|
+
* provider homes; one that does not refuses such a home's session verbs. */
|
|
14
|
+
export const SESSION_PROVIDERS_FEATURE = "session-providers";
|
|
15
|
+
/** The kernel's built-in session backends: `sessionBackends` in `oats version --json`. A session
|
|
16
|
+
* provider is never listed there: it is a capability's, named by the homes and souls that have it. */
|
|
17
|
+
export const BUILTIN_SESSION_BACKENDS = Object.freeze(["tmux"]);
|
|
18
|
+
/** A session provider's name, and what `--backend` takes. */
|
|
19
|
+
export const SESSION_PROVIDER_NAME_RE = /^[a-z0-9][a-z0-9-]{0,39}$/;
|
|
20
|
+
export const KERNEL_FEATURES = Object.freeze(["retire-home", "session-start", "session-restart", "launch-config", "schedule", "schedule-host-caps", "session-upload", "operations", "instance-git", "instance-git-remote", "souls-declarations", "lifecycle-plans", "retire-retention", "readiness", "spawn-preview", "instance-events", "instance-events-2", "schedule-history", "schedule-read-2", "spawn-preview-2", "spawn-idempotency", "spawn-idempotency-2", "spawn-apply-2", "workspace-v2", "instance-modules", "spawn-provider-payload", "served-identity", "packages-no-approval", "spawn-name", "settings-origins", "team-model-3", "settings-declared", "capabilities-private", "layers-from", "harness", "package-souls", "triggers", "automations", "desktop-facts", "launch-preference", "preview-composed-from", "observe-max-age", "spawn-preview-max-age", "launch-config-default", "capability-show", "capture-file", "workspace-identity", "server-connect", "capability-route", "servers-per-workspace", "operator-default-soul", "waiting-on-you", "automation-descriptions", "souls-capabilities", "soul-composed-instructions", "teams-conditional-default", "server-probe-features", "worktree-event", "trigger-sources", "session-attach-detach-key", "lifecycle-kernel-codes", "schedule-run-warnings", "lifecycle-claim-stop", "launch-prompt-dev-channel", "soul-launch", "retire-plan-own-worktree", "spawn-kernel-codes", "soul-skills", "roster-workspace-key", "spawn-apply-max-age", "status-retire-state", "session-inspect-size", "session-inspect-running", "deployments-list", "soul-composed-skills", "instance-color", "spawn-suggest-name", "instance-upgrade", "session-attach-no-status-line", "session-attach-prefix-scroll", "instance-suggest-name", "spawn-apply-suggest-name", SESSION_RESUME_FEATURE, "instance-background", "instance-pin", "instance-same-work", "instance-related-work", "warm-reader", "status-liveness", "lifecycle-claim-start", "wake-hold", "spawn-attribution", "workspace-tasks-payload", "settings-committed-only", "schedule-host-register-only", SPAWN_TASK_STDIN_FEATURE, "instance-board", SESSION_PROVIDERS_FEATURE]);
|
|
21
|
+
/** The agent's board (feature instance-board, lib/instance-board.mjs): `boardApi` in the probe, the board
|
|
22
|
+
* file and the board commands' answers. */
|
|
23
|
+
export const BOARD_API = 1;
|
|
12
24
|
/** The warm reader's protocol (`oats reader --stdio`, lib/reader.mjs): `readerApi` in the probe and in the reader's hello. */
|
|
13
25
|
export const READER_API = 1;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** CLI startup's Node floor. No imports or effects: this must load before the kernel. */
|
|
2
|
+
export function nodeVersionFailure(found) {
|
|
3
|
+
if (Number(found.split(".")[0]) >= 22) return null;
|
|
4
|
+
return { schemaVersion: 1, ok: false, error: {
|
|
5
|
+
code: "E_NODE_UNSUPPORTED",
|
|
6
|
+
message: `oats needs Node.js 22 or later; this is Node.js ${found}`,
|
|
7
|
+
details: { found, minimum: "22" },
|
|
8
|
+
} };
|
|
9
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/** Private entry point of lib/session-providers.mjs (callSessionProvider); no CLI or provider
|
|
2
|
+
* contract. The kernel runs every command of a session provider through this process, started in
|
|
3
|
+
* a process group of its own, of which it is the leader:
|
|
4
|
+
*
|
|
5
|
+
* node provider-call-child.mjs <group file | -> <provider script> [arguments…]
|
|
6
|
+
*
|
|
7
|
+
* - With a group file, it records who it is there first (`{ pid, processStart }`, created
|
|
8
|
+
* exclusively), outside the home, and only then runs the provider. A kernel that finds the call
|
|
9
|
+
* of a kernel that was killed ends this group by that record before it settles anything the
|
|
10
|
+
* call may still do (lib/core.mjs endProviderCall). A file that is already there is that
|
|
11
|
+
* kernel's word that the call is void: the provider is not run. Nor is it once the call's own
|
|
12
|
+
* record (the group file's name without `.group`) is gone: that kernel has ended the call and
|
|
13
|
+
* taken its records away, the void one last.
|
|
14
|
+
* - The provider's script runs in this group, with this process's stdin, stdout and stderr: its
|
|
15
|
+
* request and its answer pass through untouched. A provider runs its own helpers in the group
|
|
16
|
+
* and never detaches them.
|
|
17
|
+
* - A signal to end it (the kernel's timeout sends SIGTERM) ends the whole group, by SIGKILL.
|
|
18
|
+
*
|
|
19
|
+
* It exits as the provider's script did. 125: the call was void or could not be recorded, and
|
|
20
|
+
* nothing was run. 126: the script could not be started. */
|
|
21
|
+
import { spawn } from "node:child_process";
|
|
22
|
+
import { existsSync, linkSync, rmSync, writeFileSync } from "node:fs";
|
|
23
|
+
import { processStartToken } from "../packages/record/lib/process-identity.mjs";
|
|
24
|
+
|
|
25
|
+
const [groupFile, script, ...args] = process.argv.slice(2);
|
|
26
|
+
const endGroup = () => process.kill(0, "SIGKILL");
|
|
27
|
+
// A test's seam (reserved launch environment): this leader starts late, as on a loaded host.
|
|
28
|
+
const late = Number(process.env.OATS_TEST_PROVIDER_LEADER_DELAY_MS);
|
|
29
|
+
delete process.env.OATS_TEST_PROVIDER_LEADER_DELAY_MS;
|
|
30
|
+
if (Number.isFinite(late) && late > 0) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.min(late, 10_000));
|
|
31
|
+
for (const signal of ["SIGTERM", "SIGINT", "SIGHUP"]) process.on(signal, endGroup);
|
|
32
|
+
|
|
33
|
+
if (groupFile !== "-") {
|
|
34
|
+
const processStart = processStartToken(process.pid);
|
|
35
|
+
// Without its start time the record could not tell this group from a later one of the same id.
|
|
36
|
+
if (!processStart) process.exit(125);
|
|
37
|
+
// Published whole, by an exclusive link from a file of its own: a reader never finds it half
|
|
38
|
+
// written, whenever this process ends (lib/core.mjs publishExclusive, the same rule).
|
|
39
|
+
const own = `${groupFile}.${process.pid}.tmp`;
|
|
40
|
+
try {
|
|
41
|
+
writeFileSync(own, `${JSON.stringify({ pid: process.pid, processStart })}\n`, { flag: "wx", mode: 0o600 });
|
|
42
|
+
linkSync(own, groupFile);
|
|
43
|
+
} catch { rmSync(own, { force: true }); process.exit(125); } // already there (void), or not writable: nothing is run
|
|
44
|
+
rmSync(own, { force: true });
|
|
45
|
+
if (!existsSync(groupFile.slice(0, -".group".length))) { rmSync(groupFile, { force: true }); process.exit(125); }
|
|
46
|
+
}
|
|
47
|
+
// A test's seam (reserved launch environment): the kernel that waits for this command ends
|
|
48
|
+
// while its provider's command runs, shortly after it started and read its request
|
|
49
|
+
// (lib/provider-test-seams.mjs).
|
|
50
|
+
const endsKernel = process.env.OATS_TEST_PROVIDER_KILL_KERNEL !== undefined;
|
|
51
|
+
delete process.env.OATS_TEST_PROVIDER_KILL_KERNEL;
|
|
52
|
+
const kernel = process.ppid;
|
|
53
|
+
const child = spawn(process.execPath, [script, ...args], { stdio: "inherit" });
|
|
54
|
+
if (endsKernel) setTimeout(() => process.kill(kernel, "SIGKILL"), 150);
|
|
55
|
+
child.on("error", () => process.exit(126));
|
|
56
|
+
child.on("close", (code, signal) => {
|
|
57
|
+
if (!signal) process.exit(code ?? 1);
|
|
58
|
+
for (const handled of ["SIGTERM", "SIGINT", "SIGHUP"]) process.removeAllListeners(handled);
|
|
59
|
+
process.kill(process.pid, signal);
|
|
60
|
+
});
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/** The resume cap of a home on a session provider (lib/session-providers.mjs;
|
|
2
|
+
* docs/execution-targets.md, "Lost sessions, and the resume"). A
|
|
3
|
+
* provider home's session that is gone, and that OATS did not stop, is a lost session, and whatever
|
|
4
|
+
* brings it back is a resume: the machine sweep's notice (`oats session resume`), `session start
|
|
5
|
+
* --resume`, and any input to it (a mail the broker delivers, a scheduled wake, a person's
|
|
6
|
+
* `session input`), since the provider restarts the session on that input. The cap is the kernel's
|
|
7
|
+
* own, counted here BEFORE the provider's input is called, so no path to the provider bypasses it:
|
|
8
|
+
* at most RESUME_LIMIT resumes of one home in RESUME_WINDOW_MS, in one boot, as the sweep bounds a
|
|
9
|
+
* tmux home's starts (lib/session-resume.mjs). At the cap the resume is refused
|
|
10
|
+
* E_SESSION_NOT_RUNNING, nothing is sent, and a mail stays unread.
|
|
11
|
+
*
|
|
12
|
+
* One file per home under OATS_HOME_DIR (`session-resume/provider-resumes/<sha256 of the home's
|
|
13
|
+
* real path>.json`): `{ bootId, home, resumes: [<ISO time>] }`. Another boot's file counts
|
|
14
|
+
* nothing. Without a boot id (neither Linux nor macOS) nothing is counted, as for the sweep.
|
|
15
|
+
* Kernel-private, not a contract. A leaf: `session input` reads it without loading the sweep. */
|
|
16
|
+
import { createHash } from "node:crypto";
|
|
17
|
+
import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
18
|
+
import { join } from "node:path";
|
|
19
|
+
import { oatsError } from "./errors.mjs";
|
|
20
|
+
import { bootId, homeKey, resumeStateDir, sameBoot } from "./session-seen.mjs";
|
|
21
|
+
|
|
22
|
+
/** The resume notice of a home on a session provider: one line that asserts only what holds for
|
|
23
|
+
* every such instance, whatever ended its session. It is sent as the session's input. */
|
|
24
|
+
export const PROVIDER_RESUME_NOTICE = "OATS resumed this session after it was interrupted. Continue from where you were.";
|
|
25
|
+
/** How long the resume of a lost session waits for the processes of its last run to leave the home,
|
|
26
|
+
* before it refuses: `waitMs` is the caller's (150 s for a start, none for an input). */
|
|
27
|
+
export const RESUME_QUIET_WAIT_MS = 150_000;
|
|
28
|
+
/** That wait as this process runs it. OATS_TEST_PROVIDER_RESUME_WAIT_MS (a test's seam) shortens a
|
|
29
|
+
* resume that waits: a value above the caller's own wait is that wait, so the seam never makes a
|
|
30
|
+
* resume wait longer; and a resume that does not wait (an input) still does not. */
|
|
31
|
+
export function resumeQuietWait(waitMs = RESUME_QUIET_WAIT_MS) {
|
|
32
|
+
const seam = Number(process.env.OATS_TEST_PROVIDER_RESUME_WAIT_MS);
|
|
33
|
+
return waitMs > 0 && seam > 0 ? Math.min(seam, waitMs) : waitMs;
|
|
34
|
+
}
|
|
35
|
+
/** At most this many resumes of one home in RESUME_WINDOW_MS. */
|
|
36
|
+
export const RESUME_LIMIT = 3;
|
|
37
|
+
/** The span over which a home's resumes are counted. */
|
|
38
|
+
export const RESUME_WINDOW_MS = 60 * 60_000;
|
|
39
|
+
|
|
40
|
+
const fileOf = (home) => join(resumeStateDir(), "provider-resumes", `${createHash("sha256").update(homeKey(home)).digest("hex").slice(0, 32)}.json`);
|
|
41
|
+
/** The lock a resume of `home` is counted under (lib/dir-lock.mjs): the caller's to take around
|
|
42
|
+
* countProviderResume and its `undo`, so two counts never read the same file. Not taken here:
|
|
43
|
+
* this module is a leaf the hot reads load. */
|
|
44
|
+
export const resumeCountLock = (home) => `${fileOf(home)}.lock`;
|
|
45
|
+
function readResumes(home, boot, now) {
|
|
46
|
+
let doc;
|
|
47
|
+
try { doc = JSON.parse(readFileSync(fileOf(home), "utf8")); } catch { return []; }
|
|
48
|
+
if (!doc || typeof doc !== "object" || !sameBoot(doc.bootId, boot) || !Array.isArray(doc.resumes)) return [];
|
|
49
|
+
return doc.resumes.filter((t) => typeof t === "string" && Number.isFinite(Date.parse(t)) && now - Date.parse(t) < RESUME_WINDOW_MS).sort();
|
|
50
|
+
}
|
|
51
|
+
function writeResumes(home, boot, resumes) {
|
|
52
|
+
const path = fileOf(home), tmp = `${path}.${process.pid}.tmp`;
|
|
53
|
+
mkdirSync(join(resumeStateDir(), "provider-resumes"), { recursive: true, mode: 0o700 });
|
|
54
|
+
try { writeFileSync(tmp, JSON.stringify({ bootId: boot, home: homeKey(home), resumes }, null, 2) + "\n", { mode: 0o600 }); renameSync(tmp, path); }
|
|
55
|
+
catch (e) { rmSync(tmp, { force: true }); throw e; }
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** How many resumes of `home` are counted now → `{ count, limit, nextAttemptAt }`: `nextAttemptAt`
|
|
59
|
+
* (an ISO time) when the cap is reached, the instant its oldest counted resume leaves the window. */
|
|
60
|
+
export function providerResumes(home, { boot = bootId(), now = Date.now() } = {}) {
|
|
61
|
+
const resumes = boot ? readResumes(home, boot, now) : [];
|
|
62
|
+
return { count: resumes.length, limit: RESUME_LIMIT, nextAttemptAt: resumes.length >= RESUME_LIMIT ? new Date(Date.parse(resumes[0]) + RESUME_WINDOW_MS).toISOString() : null };
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Count one resume of the provider home `home`, before the provider's input is called → `{ undo }`:
|
|
66
|
+
* `undo()` takes the count back, only for an input the provider itself then refused (nothing was
|
|
67
|
+
* sent: a notice that was not sent is no resume). An input whose call failed, or answered outside
|
|
68
|
+
* the rules, keeps its count: the provider may have restarted the session on it. Throws E_SESSION_NOT_RUNNING at the cap, having counted nothing;
|
|
69
|
+
* `details.resumeCap` is `{ limit, windowMs, nextAttemptAt }`. `instance` names the home in the
|
|
70
|
+
* refusal. A count that cannot be written refuses the resume (E_SESSION_NOT_RUNNING too): an
|
|
71
|
+
* uncounted resume would be one the cap never sees. */
|
|
72
|
+
export function countProviderResume(home, instance, { boot = bootId(), now = Date.now() } = {}) {
|
|
73
|
+
if (!boot) return { undo() {} };
|
|
74
|
+
const resumes = readResumes(home, boot, now);
|
|
75
|
+
if (resumes.length >= RESUME_LIMIT) {
|
|
76
|
+
const nextAttemptAt = new Date(Date.parse(resumes[0]) + RESUME_WINDOW_MS).toISOString();
|
|
77
|
+
throw Object.assign(oatsError("E_SESSION_NOT_RUNNING", `${instance} lost its session and was resumed ${resumes.length} times in the last hour, and was lost again each time: it is not resumed again before ${nextAttemptAt}, and nothing was sent. Start it by hand (oats session restart --home ${home}) if it should run`), { details: { resumeCap: { limit: RESUME_LIMIT, windowMs: RESUME_WINDOW_MS, nextAttemptAt } } });
|
|
78
|
+
}
|
|
79
|
+
const at = new Date(now).toISOString();
|
|
80
|
+
try { writeResumes(home, boot, [...resumes, at]); }
|
|
81
|
+
catch (e) { throw oatsError("E_SESSION_NOT_RUNNING", `${instance} lost its session, and its resume could not be counted (${e?.code ?? e?.message ?? e}); nothing was sent`); }
|
|
82
|
+
return { undo() { try { const kept = readResumes(home, boot, now); const i = kept.lastIndexOf(at); if (i >= 0) { kept.splice(i, 1); writeResumes(home, boot, kept); } } catch { /* the count stands: a resume too many is counted, never one too few */ } } };
|
|
83
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/** Test seams for the lifecycle of a provider home (docs/design/2026-11-20-provider-home-lifecycle.md,
|
|
2
|
+
* "The holder died" and "A provider's command outlives the kernel that ran it"). No effect unless
|
|
3
|
+
* OATS_TEST_PROVIDER_DIE_AT is set; reserved launch environment (lib/reserved-launch-env.mjs):
|
|
4
|
+
* never a launch's to set.
|
|
5
|
+
*
|
|
6
|
+
* OATS_TEST_PROVIDER_DIE_AT names one point, and the kernel dies there, by SIGKILL, as a killed
|
|
7
|
+
* command does:
|
|
8
|
+
* - a point between two writes (`mark:receipt-written`, `stop:stoppedAt-written`, …: lib/core.mjs);
|
|
9
|
+
* - `call:<verb>:before` and `call:<verb>:after`: right before and right after a provider's
|
|
10
|
+
* command (lib/session-providers.mjs callSessionProvider);
|
|
11
|
+
* - `call:<verb>:during`: while the provider's command runs. The kernel cannot kill itself
|
|
12
|
+
* inside a synchronous wait, so the leader of the command's process group does it, before it
|
|
13
|
+
* runs the provider (lib/provider-call-child.mjs, OATS_TEST_PROVIDER_KILL_KERNEL). */
|
|
14
|
+
export function dieAt(point) {
|
|
15
|
+
if (process.env.OATS_TEST_PROVIDER_DIE_AT !== point) return;
|
|
16
|
+
process.kill(process.pid, "SIGKILL");
|
|
17
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10_000); // the signal is on its way
|
|
18
|
+
}
|
|
19
|
+
/** Whether this kernel is to die while the provider's `verb` command runs. */
|
|
20
|
+
export const diesDuring = (verb) => process.env.OATS_TEST_PROVIDER_DIE_AT === `call:${verb}:during`;
|
package/lib/read-call.mjs
CHANGED
|
@@ -25,6 +25,7 @@ import { resetRuntimeNames, runtimeNameWarning } from "./deprecation.mjs";
|
|
|
25
25
|
import { OWN_NAME_FAILURES } from "./errors.mjs";
|
|
26
26
|
import { activateLocalInputs, localRevision, resetLocalInputs } from "./local-inputs.mjs";
|
|
27
27
|
import { createReadSession, MAX_AGE_LIMIT, READ_REMOTE_BUDGET_MS, resetGitProbes, resetSshCommand } from "./remote.mjs";
|
|
28
|
+
import { resetKernelNode } from "./stable-node.mjs";
|
|
28
29
|
import { observeResolvedDeployment, resetResolvedDeployment, settleResolvedDeployment } from "./resolved-deployment.mjs";
|
|
29
30
|
|
|
30
31
|
/** Every module-level binding the reads' import closure writes, and what a call does with it: `reset`
|
|
@@ -38,6 +39,7 @@ export const READ_STATE = [
|
|
|
38
39
|
{ module: "lib/remote.mjs", binding: "gitVersions", how: "reset", reset: resetGitProbes },
|
|
39
40
|
{ module: "lib/remote.mjs", binding: "protocolPins", how: "reset", reset: resetGitProbes },
|
|
40
41
|
{ module: "lib/resolved-deployment.mjs", binding: "reported", how: "reset", reset: resetResolvedDeployment },
|
|
42
|
+
{ module: "lib/stable-node.mjs", binding: "kernel", how: "reset", reset: resetKernelNode },
|
|
41
43
|
{ module: "lib/resolved-deployment.mjs", binding: "observer", how: "scoped", why: "set and settled by the call when it is given `deployments`; else the CLI's own, settled at exit" },
|
|
42
44
|
{ module: "lib/remote.mjs", binding: "liveChildren", how: "safe", why: "a registry of live git process groups, each leaving on 'close': empty once a call's session is closed, never cleared (that would lose exit reaping)" },
|
|
43
45
|
{ module: "lib/remote.mjs", binding: "cacheLocks", how: "safe", why: "an in-process mutex per cache repo; an entry is removed when its holder releases it" },
|
|
@@ -21,4 +21,11 @@ export const RESERVED_LAUNCH_ENV = new Set(["OATS_INSTANCE", "OATS_INSTANCE_HOME
|
|
|
21
21
|
// lib/system-tools.mjs).
|
|
22
22
|
"OATS_TEST_CGROUP", "OATS_TEST_SYSTEMD_BOOTED", "OATS_TEST_SYSTEM_TOOL_DIRS",
|
|
23
23
|
// The test seam of #1421 §3: the naming call's time limit (lib/instance-naming.mjs NAMING_TIMEOUT_MS).
|
|
24
|
-
"OATS_TEST_NAMING_TIMEOUT_MS"
|
|
24
|
+
"OATS_TEST_NAMING_TIMEOUT_MS",
|
|
25
|
+
// The test seams of the session-provider contract: the timeout of every provider call
|
|
26
|
+
// (lib/session-providers.mjs sessionProviderTimeout) and how long a lost session's resume waits
|
|
27
|
+
// for its last run's processes to leave the home (lib/provider-resume.mjs resumeQuietWait). Each
|
|
28
|
+
// only shortens: a value above the kernel's own limit is that limit.
|
|
29
|
+
"OATS_TEST_SESSION_PROVIDER_TIMEOUT_MS", "OATS_TEST_PROVIDER_RESUME_WAIT_MS",
|
|
30
|
+
// The test seam of a provider home's write orders: the point at which the kernel dies (lib/core.mjs dieAt).
|
|
31
|
+
"OATS_TEST_PROVIDER_DIE_AT", "OATS_TEST_PROVIDER_LEADER_DELAY_MS", "OATS_TEST_PROVIDER_KILL_KERNEL"]);
|
package/lib/retire-admission.mjs
CHANGED
|
@@ -4,6 +4,7 @@ import { existsSync, readFileSync } from "node:fs";
|
|
|
4
4
|
import { join } from "node:path";
|
|
5
5
|
import { directoryRollbackPath, isCapturedHome, readHomeRecord, unreadableRecordSaid } from "./instance-listing.mjs";
|
|
6
6
|
import { isPlainObject } from "./session-inspect.mjs";
|
|
7
|
+
import { recordsSessionProvider } from "./session-providers.mjs";
|
|
7
8
|
import { processLiveness } from "./worktree-hooks.mjs";
|
|
8
9
|
import { lifecycleBusy, oatsError } from "./errors.mjs";
|
|
9
10
|
|
|
@@ -160,7 +161,10 @@ export function retireCleanupAdmission(home, name, { force = false } = {}, readW
|
|
|
160
161
|
// Compensation metadata is the SPAWN's, from whichever record survived — never
|
|
161
162
|
// a failed retry's report, which says nothing about what still needs undoing.
|
|
162
163
|
if (!liveMeta?.capabilityMeta && quarantine?.cleanup?.capabilityMeta) meta.capabilityMeta = quarantine.cleanup.capabilityMeta;
|
|
163
|
-
|
|
164
|
+
// A home on a session provider keeps what its own record says of its launch: its receipt is the
|
|
165
|
+
// authority for it, and a rollback marker is written `launched: false` whatever the provider had
|
|
166
|
+
// answered by then, which would set the record against the receipt and refuse every retire.
|
|
167
|
+
if (quarantine && typeof quarantine.cleanup.launched === "boolean" && !recordsSessionProvider(liveMeta)) {
|
|
164
168
|
meta.launched = quarantine.cleanup.launched;
|
|
165
169
|
meta.tmux = quarantine.cleanup.tmux;
|
|
166
170
|
meta.sessionTarget = quarantine.cleanup.sessionTarget;
|
package/lib/schedule.mjs
CHANGED
|
@@ -35,6 +35,7 @@ import { herdrSettingRemoved } from "./errors.mjs";
|
|
|
35
35
|
import { withDirLock as withSharedDirLock } from "./dir-lock.mjs";
|
|
36
36
|
import { pollSourced, tickTriggers } from "./triggers.mjs";
|
|
37
37
|
import { SCHEDULE_NAME_RE, HOST_NAME_RE, readSnapshot, hostOf, trusted, parseOwner, MODEL_RE, automationError, baseRow, localEntry, localId, soulOriginOf, splitId, validateDescription } from "./automations.mjs";
|
|
38
|
+
import { SESSION_PROVIDER_NAME_RE } from "./kernel-features.mjs";
|
|
38
39
|
import { MAX_INSTANCE_NAME, shq, shellWord, findAgent, findInstanceHomes, inspectInstanceSession, inputInstanceSession, sessionServerGone, startInstanceSession, retirePendingMarkerPath } from "./core.mjs";
|
|
39
40
|
import { writeThroughPrivateFile } from "./write-through-private-file.mjs";
|
|
40
41
|
import { readStartExited } from "./session-end.mjs";
|
|
@@ -487,7 +488,9 @@ export function validateDefinition(ws, def, { checkAgent = true, backendSource,
|
|
|
487
488
|
}
|
|
488
489
|
if (def.repo !== undefined) { if (typeof def.repo !== "string" || !def.repo.trim() || def.repo.startsWith("-")) throw scheduleError("E_SCHEDULE_INVALID", "repo: the work repository path, as oats spawn --repo", { field: "repo" }); out.repo = def.repo; }
|
|
489
490
|
if (def.backend === "herdr") throw herdrBackendRefusal(backendSource ?? `${definitionsPath(ws)} sets jobs.${id}.backend`);
|
|
490
|
-
|
|
491
|
+
// tmux, or a session provider of the soul's capabilities (lib/session-providers.mjs): a name here;
|
|
492
|
+
// the run's spawn resolves it against the instance's modules (E_BACKEND_UNKNOWN, E_BACKEND_HARNESS).
|
|
493
|
+
if (def.backend !== undefined) { if (typeof def.backend !== "string" || !SESSION_PROVIDER_NAME_RE.test(def.backend)) throw scheduleError("E_SCHEDULE_INVALID", "backend: tmux, or a session provider's name (1 to 40 lowercase letters, digits and dashes)", { field: "backend" }); out.backend = def.backend; }
|
|
491
494
|
if (def.purpose !== undefined) { if (typeof def.purpose !== "string" || !/^[a-z0-9-]{1,40}$/.test(def.purpose)) throw scheduleError("E_SCHEDULE_INVALID", "purpose: lowercase letters, digits and dashes", { field: "purpose" }); out.purpose = def.purpose; }
|
|
492
495
|
// A run spawns <agent>-<purpose|id>-<YYYYMMDDHHMM>; instance names are at most
|
|
493
496
|
// MAX_INSTANCE_NAME characters, so refuse here what would fail at every firing.
|
|
@@ -646,8 +649,12 @@ export function observeHome(home, io) {
|
|
|
646
649
|
const retiring = existsSync(retirePendingMarkerPath(home));
|
|
647
650
|
try {
|
|
648
651
|
const s = (io?.inspect || inspectInstanceSession)(home);
|
|
649
|
-
|
|
652
|
+
// A home on a session provider (its inspect answer carries `session`): a wake never starts it
|
|
653
|
+
// (performWake). Its session in error (`shell`) is no startup phase: an input recovers it.
|
|
654
|
+
const provider = s.session !== undefined && s.session !== null ? { provider: true } : {};
|
|
655
|
+
if (s.present && s.state !== "shell") return { outcome: "active", ...provider, ...(retiring ? { note: "retiring" } : {}) };
|
|
650
656
|
if (retiring) return { outcome: "active", note: "retiring: home present, teardown pending" };
|
|
657
|
+
if (provider.provider) return { outcome: "stopped", ...provider };
|
|
651
658
|
// A shell right after `session start` is that launch's startup phase
|
|
652
659
|
// until its wrapper writes the exit marker for the launch id; only then
|
|
653
660
|
// (or with no start receipt at all) is a shell a finished harness.
|
|
@@ -915,7 +922,10 @@ function performWake(def, io, { startIfStopped = true, canStart = true, reserve,
|
|
|
915
922
|
if (seen.outcome === "ended") return { kind: "wake", action: "skipped", reason: "home is gone", pending: false };
|
|
916
923
|
if (seen.outcome === "unknown") return { kind: "wake", action: "skipped", reason: `cannot observe the session: ${seen.error}`, pending: true };
|
|
917
924
|
if (seen.outcome === "starting") return { kind: "wake", action: "skipped", reason: "session is starting; delivery pending", pending: true };
|
|
918
|
-
|
|
925
|
+
// A home on a session provider is never started by a wake: the wake is its session's input, which
|
|
926
|
+
// the kernel refuses (E_SESSION_NOT_RUNNING, before any provider call) for a home OATS stopped,
|
|
927
|
+
// and which the provider answers for any other.
|
|
928
|
+
if (seen.outcome === "stopped" && !seen.provider) {
|
|
919
929
|
if (!startIfStopped) return { kind: "wake", action: "skipped", reason: "session stopped; it is started again only at the next due minute; delivery pending", pending: true };
|
|
920
930
|
if (!canStart) return { kind: "wake", action: "skipped", reason: "host busy: starting this session needs a launch slot; delivery pending", pending: true };
|
|
921
931
|
// The slot is persisted BEFORE the start call and kept on a start
|
|
@@ -945,7 +955,7 @@ function performWake(def, io, { startIfStopped = true, canStart = true, reserve,
|
|
|
945
955
|
// Delivery custody: name the instance ONLY when the input result names it; the home's current incarnation at record time.
|
|
946
956
|
return { kind: "wake", action: "delivered", pending: false, ...(typeof r?.instance === "string" ? { instance: r.instance } : {}), home: def.home, incarnation: incarnationOfHome(def.home) };
|
|
947
957
|
}
|
|
948
|
-
catch (e) { return { kind: "wake", action: "skipped", reason: `input refused: ${e.message}`, pending: true }; }
|
|
958
|
+
catch (e) { return { kind: "wake", action: "skipped", reason: `input refused: ${e.message}`, ...(seen.provider && typeof e?.code === "string" ? { errorCode: e.code } : {}), pending: true }; }
|
|
949
959
|
}
|
|
950
960
|
|
|
951
961
|
/** A held run as the tick records it: `held.since` is the first skip's time, kept on
|