@awebai/oats 0.62.0 → 0.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/oats.mjs +1 -1
- package/docs/desktop-cli-api.md +30 -2
- package/docs/execution-targets.md +40 -1
- package/docs/official-catalog.md +1 -1
- package/docs/packages.md +5 -5
- package/docs/release-lane.md +13 -0
- package/docs/release-notes/v0.63.0.md +135 -0
- package/docs/souls-and-instances.md +6 -1
- package/docs/upgrading.md +5 -3
- package/docs/workspaces.md +1 -1
- package/lib/core.mjs +71 -21
- package/lib/tree-copy.mjs +188 -7
- package/package-catalog.json +1 -1
- package/package.json +1 -1
- package/packages/client/cli-locator.mjs +2 -2
- package/packages/record/bin/capture.mjs +59 -17
- package/packages/record/bin/setup.mjs +2 -1
- package/packages/record/docs/capture-service.md +89 -5
- package/packages/record/lib/capture-command-operation.mjs +2 -2
- package/packages/record/lib/capture-command-output.mjs +3 -2
- package/packages/record/lib/capture-command.mjs +20 -3
- package/packages/record/lib/capture-hook-runner.mjs +1 -1
- package/packages/record/lib/capture-index-reset.mjs +1 -0
- package/packages/record/lib/capture-lock.mjs +67 -33
- package/packages/record/lib/capture-pass.mjs +22 -8
- package/packages/record/lib/capture-run.mjs +1 -1
- package/packages/record/lib/capture-service-runner.mjs +1 -1
- package/packages/record/lib/capture-supervisor.mjs +24 -5
- package/packages/record/lib/capture-work-session.mjs +7 -4
- package/packages/record/lib/capture-worker.mjs +20 -4
- package/packages/record/lib/derived-state.mjs +30 -2
- package/packages/record/lib/index-db.mjs +118 -49
- package/packages/record/lib/store.mjs +13 -6
- package/skills/oats-getting-started/SKILL.md +1 -1
package/lib/tree-copy.mjs
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
/** Tree mechanics only: the kernel's catchable recursive copy (spawn, retire and
|
|
2
|
-
* work recovery)
|
|
3
|
-
|
|
4
|
-
|
|
2
|
+
* work recovery), and a bounded comparison of a copy with its source. No
|
|
3
|
+
* acquisition, selection, trust, lock or lifecycle policy. */
|
|
4
|
+
import fs, {
|
|
5
|
+
chmodSync, closeSync, constants, copyFileSync, fstatSync, lstatSync, mkdirSync, opendirSync, openSync, readSync, readdirSync, readlinkSync, symlinkSync,
|
|
5
6
|
} from "node:fs";
|
|
6
7
|
import { sep } from "node:path";
|
|
8
|
+
import { performance } from "node:perf_hooks";
|
|
7
9
|
import { oatsError } from "./errors.mjs";
|
|
8
10
|
|
|
9
11
|
const SEPARATOR = Buffer.from(sep);
|
|
@@ -36,22 +38,201 @@ export const entriesAsBytes = (dir) => readdirSync(pathBytes(dir), { encoding: "
|
|
|
36
38
|
* - symlinks are recreated VERBATIM — never followed, never rewritten — because
|
|
37
39
|
* the bytes about to be hashed must be the bytes the author wrote. The target
|
|
38
40
|
* is read and written as bytes: read as text, a target that is not valid
|
|
39
|
-
* UTF-8 would be recreated as another one
|
|
41
|
+
* UTF-8 would be recreated as another one. A link keeps its own mode, which
|
|
42
|
+
* the exact digest holds: where the new link's mode is not the source's
|
|
43
|
+
* (macOS gives a link the umask's mode; Linux every link 0777), it is set
|
|
44
|
+
* with lchmod, on the link and never on its target. A platform with no
|
|
45
|
+
* lchmod cannot, and the copy is refused (linkModeUnsupported) rather than
|
|
46
|
+
* left with another mode;
|
|
40
47
|
* - FIFOs, sockets and device nodes are rejected fail-closed: they are not
|
|
41
48
|
* distributable content, and copying them has no defined meaning here;
|
|
42
49
|
* - directory modes are applied AFTER their children, so a read-only source
|
|
43
50
|
* directory cannot block writing its own contents. */
|
|
44
|
-
export function copyTreeSafe(src, dest) {
|
|
51
|
+
export function copyTreeSafe(src, dest, { lchmod = fs.lchmodSync } = {}) {
|
|
45
52
|
const st = lstatSync(src);
|
|
46
|
-
if (st.isSymbolicLink()) { symlinkSync(readlinkSync(src, "buffer"), dest); return; }
|
|
53
|
+
if (st.isSymbolicLink()) { symlinkSync(readlinkSync(src, "buffer"), dest); copyLinkMode(dest, st.mode & 0o7777, lchmod); return; }
|
|
47
54
|
if (st.isFile()) { copyFileSync(src, dest); chmodSync(dest, st.mode & 0o7777); return; }
|
|
48
55
|
if (!st.isDirectory()) {
|
|
49
56
|
throw oatsError("invalid-source", `${src.toString()} is not a regular file, directory or symlink (${st.isFIFO() ? "FIFO" : st.isSocket() ? "socket" : st.isBlockDevice() || st.isCharacterDevice() ? "device node" : "unsupported file type"}) — package and capability trees carry distributable content only`);
|
|
50
57
|
}
|
|
51
58
|
mkdirSync(dest, { recursive: true });
|
|
52
|
-
for (const e of entriesAsBytes(src)) copyTreeSafe(joinBytes(src, e.name), joinBytes(dest, e.name));
|
|
59
|
+
for (const e of entriesAsBytes(src)) copyTreeSafe(joinBytes(src, e.name), joinBytes(dest, e.name), { lchmod });
|
|
53
60
|
chmodSync(dest, st.mode & 0o7777);
|
|
54
61
|
}
|
|
62
|
+
/** Give the link `dest`, just made, the permission bits `mode` of its source: nothing to do when it
|
|
63
|
+
* has them already, else lchmod (`lchmod`: fs.lchmodSync, which only a platform that can set a
|
|
64
|
+
* link's mode has), read back. A mode that cannot be set is refused, never dropped. */
|
|
65
|
+
function copyLinkMode(dest, mode, lchmod) {
|
|
66
|
+
if ((lstatSync(dest).mode & 0o7777) === mode) return;
|
|
67
|
+
if (typeof lchmod !== "function") throw linkModeUnsupported(dest, mode, "this platform cannot set a link's mode (no lchmod)");
|
|
68
|
+
lchmod(dest, mode);
|
|
69
|
+
const now = lstatSync(dest).mode & 0o7777;
|
|
70
|
+
if (now !== mode) throw linkModeUnsupported(dest, mode, `lchmod left it ${now.toString(8).padStart(4, "0")}`);
|
|
71
|
+
}
|
|
72
|
+
const linkModeUnsupported = (dest, mode, why) =>
|
|
73
|
+
new Error(`cannot copy the link ${pathBytes(dest).toString()} with its mode ${mode.toString(8).padStart(4, "0")}: ${why}; the copy would differ from its source`);
|
|
74
|
+
|
|
75
|
+
/** How far treeDisagreement goes before it stops: differences listed, directory
|
|
76
|
+
* entries read (on both sides, together), file bytes read (on both sides,
|
|
77
|
+
* together) and milliseconds. */
|
|
78
|
+
export const TREE_DISAGREEMENT_LIMITS = Object.freeze({ discrepancies: 20, entries: 200_000, bytes: 64 * 1024 * 1024, ms: 5000 });
|
|
79
|
+
const COMPARE_CHUNK = 64 * 1024;
|
|
80
|
+
// A file is opened as the regular file lstat saw: a name that became a link is not followed
|
|
81
|
+
// (O_NOFOLLOW), and one that became a FIFO does not block the open (O_NONBLOCK).
|
|
82
|
+
const COMPARE_OPEN = constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK;
|
|
55
83
|
|
|
84
|
+
/** A relative path, as bytes, in text that reads back one way only: each
|
|
85
|
+
* printable ASCII byte as itself (space included), `\` as `\\`, and every
|
|
86
|
+
* other byte (a control character, a byte of a UTF-8 sequence or of no valid
|
|
87
|
+
* one) as `\xHH`. */
|
|
88
|
+
export function escapedPath(bytes) {
|
|
89
|
+
let out = "";
|
|
90
|
+
for (const b of pathBytes(bytes)) out += b === 0x5c ? "\\\\" : b >= 0x20 && b <= 0x7e ? String.fromCharCode(b) : `\\x${b.toString(16).padStart(2, "0")}`;
|
|
91
|
+
return out;
|
|
92
|
+
}
|
|
56
93
|
|
|
94
|
+
/** Where a copy differs from its source, for a diagnosis only: never a verdict.
|
|
95
|
+
* It is read when an exact comparison of the two has already refused, and
|
|
96
|
+
* what it reads is the two trees as they are NOW, so a source that changed
|
|
97
|
+
* since its bytes were recorded can show differences that are not the ones
|
|
98
|
+
* the recorded comparison saw, or none.
|
|
99
|
+
*
|
|
100
|
+
* Both trees are walked together, names as bytes in the order of those bytes,
|
|
101
|
+
* with lstat: an entry that is a link is compared as a link, never followed.
|
|
102
|
+
* `excludePaths` (relative paths, as bytes)
|
|
103
|
+
* are passed over on both sides, exactly. The roots' own modes are not
|
|
104
|
+
* compared. Each difference is `{ path, kind }`: `path` relative to the roots,
|
|
105
|
+
* escaped (escapedPath), and `kind` one of `missing` (in the source, not the
|
|
106
|
+
* copy), `extra` (in the copy, not the source), `type`, `mode` (permission
|
|
107
|
+
* bits), `link-target` and `file-content`. Below a `missing`, `extra` or
|
|
108
|
+
* `type` entry nothing is listed. One entry can have several kinds, in that
|
|
109
|
+
* order. Nothing else is said: no content, no link target, no absolute path.
|
|
110
|
+
*
|
|
111
|
+
* The trees may change while they are read, and every read is by path (Node has
|
|
112
|
+
* no call relative to an open directory). So what is read is checked against
|
|
113
|
+
* what lstat saw: a file is opened without following a link or waiting on a
|
|
114
|
+
* FIFO and must be the regular file lstat saw (device, inode, size); a
|
|
115
|
+
* directory must be the directory lstat saw (device, inode) before its
|
|
116
|
+
* listing, after it, and when the search leaves it, also by a stop below it.
|
|
117
|
+
* A directory that fails its check stops the search as `unreadable`, and
|
|
118
|
+
* nothing noted below it is reported. A replacement that is undone before the
|
|
119
|
+
* replaced directory is checked again is not detected: what was read through
|
|
120
|
+
* it can then be compared and named. Every path named is relative to the
|
|
121
|
+
* roots.
|
|
122
|
+
*
|
|
123
|
+
* The work is bounded (`limits`, TREE_DISAGREEMENT_LIMITS): a directory's
|
|
124
|
+
* entries count as they are read, so no listing is read beyond the budget
|
|
125
|
+
* before the search stops; only what was read is sorted. Files are compared
|
|
126
|
+
* in chunks, sizes first, and never read whole. The time budget is checked
|
|
127
|
+
* between filesystem calls; it does not interrupt one. → `{ entries, complete }`,
|
|
128
|
+
* with `stoppedBy` when it stopped early: `discrepancies` (more than it
|
|
129
|
+
* lists), `entries`, `bytes`, `time`, or `unreadable` when a tree could not
|
|
130
|
+
* be read (it may have changed while it was read). It never throws. An
|
|
131
|
+
* answer with `complete: true` and no entries found nothing that differs now. */
|
|
132
|
+
export function treeDisagreement(source, copy, { excludePaths = [], limits = TREE_DISAGREEMENT_LIMITS, clock = () => performance.now(), io = { lstat: lstatSync, opendir: opendirSync } } = {}) {
|
|
133
|
+
const excluded = new Set(excludePaths.map((path) => pathBytes(path).toString("hex")));
|
|
134
|
+
const entries = [];
|
|
135
|
+
const started = clock();
|
|
136
|
+
let looked = 0, read = 0;
|
|
137
|
+
const STOP = Symbol("stop");
|
|
138
|
+
const stop = (reason) => { throw { [STOP]: reason }; };
|
|
139
|
+
const inTime = () => { if (clock() - started > limits.ms) stop("time"); };
|
|
140
|
+
const note = (rel, kind) => {
|
|
141
|
+
if (entries.length === limits.discrepancies) stop("discrepancies");
|
|
142
|
+
entries.push({ path: escapedPath(rel), kind });
|
|
143
|
+
};
|
|
144
|
+
const kindOf = (st) => (st.isSymbolicLink() ? "link" : st.isFile() ? "file" : st.isDirectory() ? "dir" : "other");
|
|
145
|
+
/** `path` opened as the regular file `st` (its lstat) describes, else the search stops: the tree
|
|
146
|
+
* changed while it was read. */
|
|
147
|
+
const openAsSeen = (path, st) => {
|
|
148
|
+
const fd = openSync(path, COMPARE_OPEN);
|
|
149
|
+
const now = fstatSync(fd);
|
|
150
|
+
if (!now.isFile() || now.ino !== st.ino || now.dev !== st.dev || now.size !== st.size) { closeSync(fd); stop("unreadable"); }
|
|
151
|
+
return fd;
|
|
152
|
+
};
|
|
153
|
+
const sameBytes = (a, sa, b, sb) => {
|
|
154
|
+
const size = sa.size;
|
|
155
|
+
const fa = openAsSeen(a, sa);
|
|
156
|
+
try {
|
|
157
|
+
const fb = openAsSeen(b, sb);
|
|
158
|
+
try {
|
|
159
|
+
const ba = Buffer.allocUnsafe(COMPARE_CHUNK), bb = Buffer.allocUnsafe(COMPARE_CHUNK);
|
|
160
|
+
for (let at = 0; at < size;) {
|
|
161
|
+
const want = Math.min(COMPARE_CHUNK, size - at);
|
|
162
|
+
if (read + 2 * want > limits.bytes) stop("bytes");
|
|
163
|
+
inTime();
|
|
164
|
+
const na = readSync(fa, ba, 0, want, at), nb = readSync(fb, bb, 0, want, at);
|
|
165
|
+
read += na + nb;
|
|
166
|
+
if (na !== nb || na === 0 || !ba.subarray(0, na).equals(bb.subarray(0, nb))) return false;
|
|
167
|
+
at += na;
|
|
168
|
+
}
|
|
169
|
+
return true;
|
|
170
|
+
} finally { closeSync(fb); }
|
|
171
|
+
} finally { closeSync(fa); }
|
|
172
|
+
};
|
|
173
|
+
/** Whether `path` is still the directory `st` (its lstat) describes. */
|
|
174
|
+
const sameDirectory = (path, st) => {
|
|
175
|
+
let now;
|
|
176
|
+
try { now = io.lstat(path); } catch { return false; }
|
|
177
|
+
return now.isDirectory() && now.dev === st.dev && now.ino === st.ino;
|
|
178
|
+
};
|
|
179
|
+
/** The names of the directory `dir` (lstat `st`), as bytes in their order, each counted as it is
|
|
180
|
+
* read: the listing stops with the search once the budget is spent. */
|
|
181
|
+
const listing = (dir, st) => {
|
|
182
|
+
const names = [];
|
|
183
|
+
const d = io.opendir(dir, { encoding: "buffer" });
|
|
184
|
+
try {
|
|
185
|
+
for (let e = d.readSync(); e !== null; e = d.readSync()) {
|
|
186
|
+
if (++looked > limits.entries) stop("entries");
|
|
187
|
+
inTime();
|
|
188
|
+
names.push(e.name);
|
|
189
|
+
}
|
|
190
|
+
} finally { d.closeSync(); }
|
|
191
|
+
if (!sameDirectory(dir, st)) stop("unreadable");
|
|
192
|
+
return names.sort(Buffer.compare);
|
|
193
|
+
};
|
|
194
|
+
/** One directory of each tree, compared. Leaving it, by its end or by a stop below it, it is
|
|
195
|
+
* checked again: if either is no longer the directory lstat saw, everything noted below it was
|
|
196
|
+
* read through a replacement and is dropped, and the search stops as `unreadable`. */
|
|
197
|
+
const walk = (a, sa0, b, sb0, rel) => {
|
|
198
|
+
const mark = entries.length;
|
|
199
|
+
const unchanged = () => sameDirectory(a, sa0) && sameDirectory(b, sb0);
|
|
200
|
+
try { compareDirectories(a, sa0, b, sb0, rel); } catch (e) {
|
|
201
|
+
if (!unchanged()) { entries.length = mark; stop("unreadable"); }
|
|
202
|
+
throw e;
|
|
203
|
+
}
|
|
204
|
+
if (!unchanged()) { entries.length = mark; stop("unreadable"); }
|
|
205
|
+
};
|
|
206
|
+
const compareDirectories = (a, sa0, b, sb0, rel) => {
|
|
207
|
+
const na = listing(a, sa0), nb = listing(b, sb0);
|
|
208
|
+
for (let i = 0, j = 0; i < na.length || j < nb.length;) {
|
|
209
|
+
const order = i === na.length ? 1 : j === nb.length ? -1 : Buffer.compare(na[i], nb[j]);
|
|
210
|
+
const inA = order <= 0, inB = order >= 0;
|
|
211
|
+
const name = inA ? na[i] : nb[j];
|
|
212
|
+
if (inA) i++;
|
|
213
|
+
if (inB) j++;
|
|
214
|
+
const childRel = rel.length ? Buffer.concat([rel, SEPARATOR, name]) : name;
|
|
215
|
+
if (excluded.has(childRel.toString("hex"))) continue;
|
|
216
|
+
inTime();
|
|
217
|
+
if (!inB) { note(childRel, "missing"); continue; }
|
|
218
|
+
if (!inA) { note(childRel, "extra"); continue; }
|
|
219
|
+
const pa = joinBytes(a, name), pb = joinBytes(b, name);
|
|
220
|
+
const sa = io.lstat(pa), sb = io.lstat(pb);
|
|
221
|
+
const kind = kindOf(sa);
|
|
222
|
+
if (kind !== kindOf(sb)) { note(childRel, "type"); continue; }
|
|
223
|
+
if ((sa.mode & 0o7777) !== (sb.mode & 0o7777)) note(childRel, "mode");
|
|
224
|
+
if (kind === "link" && !readlinkSync(pa, "buffer").equals(readlinkSync(pb, "buffer"))) note(childRel, "link-target");
|
|
225
|
+
else if (kind === "file" && (sa.size !== sb.size || !sameBytes(pa, sa, pb, sb))) note(childRel, "file-content");
|
|
226
|
+
else if (kind === "dir") walk(pa, sa, pb, sb, childRel);
|
|
227
|
+
}
|
|
228
|
+
};
|
|
229
|
+
try {
|
|
230
|
+
const ra = io.lstat(pathBytes(source)), rb = io.lstat(pathBytes(copy));
|
|
231
|
+
if (!ra.isDirectory() || !rb.isDirectory()) stop("unreadable");
|
|
232
|
+
walk(pathBytes(source), ra, pathBytes(copy), rb, Buffer.alloc(0));
|
|
233
|
+
return { entries, complete: true };
|
|
234
|
+
} catch (e) {
|
|
235
|
+
return { entries, complete: false, stoppedBy: e && typeof e === "object" && STOP in e ? e[STOP] : "unreadable" };
|
|
236
|
+
}
|
|
237
|
+
}
|
|
57
238
|
|
package/package-catalog.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@awebai/oats",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.63.0",
|
|
4
4
|
"description": "OATS (Open Agent Team Specification) — durable souls, disposable instances, targetable capability packages, and the runtime-neutral oats CLI/kernel.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"agents",
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
// Every candidate is canonicalized to an absolute executable and accepted
|
|
14
14
|
// ONLY if executable and `<bin> version --json` returns the v1 probe:
|
|
15
15
|
// {"schemaVersion":1,"name":"@awebai/oats","version":"0.24.x","desktopApi":1}
|
|
16
|
-
// Desktop accepts desktopApi === 1 and semver >=0.26.0 <0.
|
|
16
|
+
// Desktop accepts desktopApi === 1 and semver >=0.26.0 <0.64.0 (the band is
|
|
17
17
|
// spelled ONCE, in ACCEPT_RANGE below — this line only paraphrases it).
|
|
18
18
|
// API version — not source adjacency — is authoritative.
|
|
19
19
|
//
|
|
@@ -52,7 +52,7 @@ export const DESKTOP_API = 1;
|
|
|
52
52
|
// carry a `description` for every kind and `<kind> update <id> --description`
|
|
53
53
|
// sets one (feature `automation-descriptions`, which gates the Desktop's
|
|
54
54
|
// summary writes). The Desktop v1 surface is otherwise unchanged.
|
|
55
|
-
export const ACCEPT_RANGE = { min: [0, 26, 0], maxExclusive: [0,
|
|
55
|
+
export const ACCEPT_RANGE = { min: [0, 26, 0], maxExclusive: [0, 64, 0] };
|
|
56
56
|
/** The band as humans read it — derived, never hand-spelled, so the probe
|
|
57
57
|
* rejection reason, the backend's /api status and the degradation card can
|
|
58
58
|
* never disagree with the numbers actually enforced above. */
|
|
@@ -33,6 +33,8 @@ import { acquireCaptureLock } from "../lib/capture-lock.mjs";
|
|
|
33
33
|
import { captureHealthForDoctor } from "../lib/capture-health.mjs";
|
|
34
34
|
import { captureDefaultRootPlan, resolveCaptureDefaultRoots } from "../lib/capture-default-roots.mjs";
|
|
35
35
|
import { captureCommandPass, reportLockInitializationFailure, reportReclaimedLock, releaseDetail } from "../lib/capture-command-output.mjs";
|
|
36
|
+
import { captureCommandEntry } from "../lib/capture-command.mjs";
|
|
37
|
+
import { captureSignalPoll } from "../lib/capture-supervisor.mjs";
|
|
36
38
|
import { captureWatchQueue } from "../lib/capture-watch.mjs";
|
|
37
39
|
import { launchCaptureHook } from "../lib/capture-hook.mjs";
|
|
38
40
|
|
|
@@ -48,6 +50,31 @@ function loadIgnoreForCommand(recordRoot) {
|
|
|
48
50
|
}
|
|
49
51
|
}
|
|
50
52
|
|
|
53
|
+
// The exact commands (--home, --file, --reset-cursors) run in this process.
|
|
54
|
+
// TERM/INT while one runs is observed only at its safe points: before the
|
|
55
|
+
// lock, after taking it, before each format's batch of exact files and before
|
|
56
|
+
// the index update. Discovery, one format's batch, the index update and the
|
|
57
|
+
// receipt summaries are synchronous and cannot be preempted, so a signal
|
|
58
|
+
// waits for the segment it arrived in; no response time is promised. An
|
|
59
|
+
// interruption at a safe point unwinds through the lock's release.
|
|
60
|
+
class CaptureInterrupted extends Error {}
|
|
61
|
+
let signalled = null;
|
|
62
|
+
const onSignal = signal => { signalled ??= signal; };
|
|
63
|
+
/** Run `fn` with TERM/INT observed, from before its lock until it returns. */
|
|
64
|
+
async function observingSignals(fn) {
|
|
65
|
+
process.on("SIGTERM", onSignal); process.on("SIGINT", onSignal);
|
|
66
|
+
try { return await fn(); }
|
|
67
|
+
finally { process.off("SIGTERM", onSignal); process.off("SIGINT", onSignal); }
|
|
68
|
+
}
|
|
69
|
+
/** A safe point: a signal pending from synchronous work dispatches during
|
|
70
|
+
* the poll this yields to, and an observed one ends the command here.
|
|
71
|
+
* `after` says what had already been done, so the answer never implies it
|
|
72
|
+
* was not. */
|
|
73
|
+
async function safePoint(after) {
|
|
74
|
+
await captureSignalPoll();
|
|
75
|
+
if (signalled) throw new CaptureInterrupted(`capture interrupted by ${signalled}${after ? ` after ${after}` : ""}`);
|
|
76
|
+
}
|
|
77
|
+
|
|
51
78
|
// Every flag this program understands. An unknown flag is rejected rather
|
|
52
79
|
// than ignored, because the fall-through from argument parsing is pass() —
|
|
53
80
|
// a WRITE. `capture --help` was once a full reconciliation pass under a
|
|
@@ -242,7 +269,7 @@ if (args.health) {
|
|
|
242
269
|
if (args["reset-cursors"] || args["retry-set-aside"]) {
|
|
243
270
|
try {
|
|
244
271
|
const result = args["reset-cursors"]
|
|
245
|
-
? withCaptureLock(() => { resetSourceCursors(root, owner, args.source); return { reset: true, sources: args.source ? [args.source] : CAPTURE_SOURCES }; })
|
|
272
|
+
? await observingSignals(() => withCaptureLock(() => { resetSourceCursors(root, owner, args.source); return { reset: true, sources: args.source ? [args.source] : CAPTURE_SOURCES }; }, "reset-cursors", "the cursor reset was committed"))
|
|
246
273
|
: await pass({ retry: true });
|
|
247
274
|
if (result.skipped) throw new Error("capture lock held; recovery was not performed");
|
|
248
275
|
if (process.exitCode) throw new Error("recovery failed to release its capture lock");
|
|
@@ -309,11 +336,13 @@ function log(...parts) {
|
|
|
309
336
|
* in the finally, and a release that did not happen is said so on stderr
|
|
310
337
|
* with the operator recovery and a nonzero exit status. Likewise an owner
|
|
311
338
|
* record that could not be written: the lock never silently outlives the
|
|
312
|
-
* pass that created it.
|
|
313
|
-
|
|
339
|
+
* pass that created it. `command` labels the owner record; `done` names
|
|
340
|
+
* what `fn` committed, for an interruption observed after it returned. */
|
|
341
|
+
async function withCaptureLock(fn, command, done) {
|
|
314
342
|
let lock;
|
|
343
|
+
await safePoint();
|
|
315
344
|
try {
|
|
316
|
-
lock = acquireCaptureLock(root);
|
|
345
|
+
lock = acquireCaptureLock(root, { command, caller: { entry: captureCommandEntry(), trigger: "manual" } });
|
|
317
346
|
// Said by the pass that removed it, never quiet, whether or not it then took the lock: a pass that
|
|
318
347
|
// died holding the lock is worth knowing about.
|
|
319
348
|
if (lock.reclaimed) reportReclaimedLock({ path: lock.path, ...lock.reclaimed });
|
|
@@ -331,7 +360,9 @@ function withCaptureLock(fn) {
|
|
|
331
360
|
}
|
|
332
361
|
try {
|
|
333
362
|
let result, failure;
|
|
334
|
-
|
|
363
|
+
// The callback's last segment is synchronous too: poll once more before
|
|
364
|
+
// its result can become a success answer.
|
|
365
|
+
try { cleanupCaptureScratch(root, lock); await safePoint(); result = await fn(); await safePoint(done); return result; }
|
|
335
366
|
catch (err) { failure = err; throw err; }
|
|
336
367
|
finally {
|
|
337
368
|
try { writeCaptureStatus(root, { pending: Boolean(failure || result?.pending || result?.held || result?.incomplete || result?.failed) }); }
|
|
@@ -365,14 +396,15 @@ async function pass({ retry = false, trigger = "manual" } = {}) {
|
|
|
365
396
|
loadIgnoreForCommand(root);
|
|
366
397
|
// Recovery JSON keeps its existing small shape. Never read a later status
|
|
367
398
|
// file after releasing the lock to reconstruct this pass's counters.
|
|
368
|
-
return captureCommandPass(root, { owner, plan: commandPlan(), indexEnabled: !args["no-index"], trigger, retrySetAside: retry }, { log });
|
|
399
|
+
return captureCommandPass(root, { owner, plan: commandPlan(), indexEnabled: !args["no-index"], trigger, retrySetAside: retry }, { log, entry: captureCommandEntry() });
|
|
369
400
|
}
|
|
370
401
|
|
|
371
402
|
/** A final capture of exact files, one batch per format, under the lock: counts into `outcome`, issues into
|
|
372
403
|
* `issues`, then the index unless --no-index. → withCaptureLock's answer (`outcome`, or a skip). */
|
|
373
|
-
function captureUnderLock(recordStore, recordOwner, batches, ignore, outcome, issues) {
|
|
374
|
-
return withCaptureLock(() => {
|
|
404
|
+
function captureUnderLock(recordStore, recordOwner, batches, ignore, outcome, issues, command) {
|
|
405
|
+
return withCaptureLock(async () => {
|
|
375
406
|
for (const [format, files] of batches) {
|
|
407
|
+
await safePoint();
|
|
376
408
|
const r = captureSessions(recordStore, { owner: recordOwner, files, format, ignore, final: true });
|
|
377
409
|
outcome.appended += r.appended;
|
|
378
410
|
outcome.held += r.held;
|
|
@@ -381,6 +413,7 @@ function captureUnderLock(recordStore, recordOwner, batches, ignore, outcome, is
|
|
|
381
413
|
issues.push(...r.issues);
|
|
382
414
|
}
|
|
383
415
|
if (!args["no-index"]) { // an earlier append-only pass may have left unindexed turns
|
|
416
|
+
await safePoint();
|
|
384
417
|
const index = new RecordIndex(recordStore);
|
|
385
418
|
try {
|
|
386
419
|
index.update();
|
|
@@ -389,7 +422,7 @@ function captureUnderLock(recordStore, recordOwner, batches, ignore, outcome, is
|
|
|
389
422
|
}
|
|
390
423
|
}
|
|
391
424
|
return outcome;
|
|
392
|
-
});
|
|
425
|
+
}, command, "every batch completed");
|
|
393
426
|
}
|
|
394
427
|
|
|
395
428
|
/** The outcome vocabulary: complete only with no failure, skip, hold or incomplete tail (and, for --home, no
|
|
@@ -437,7 +470,7 @@ function fileKind(stat) {
|
|
|
437
470
|
}
|
|
438
471
|
|
|
439
472
|
/** → the exit status. */
|
|
440
|
-
function fileCapture() {
|
|
473
|
+
async function fileCapture() {
|
|
441
474
|
const json = Boolean(args.json);
|
|
442
475
|
const fail = (code, error, extra = {}) => {
|
|
443
476
|
if (json) console.log(JSON.stringify({ ...extra, status: "failed", complete: false, code, error }, null, 2));
|
|
@@ -502,15 +535,19 @@ function fileCapture() {
|
|
|
502
535
|
const outcome = { appended: 0, skipped: false, held: 0, incomplete: 0, failed: 0, ignored: 0 };
|
|
503
536
|
const issues = [];
|
|
504
537
|
try {
|
|
505
|
-
Object.assign(outcome, captureUnderLock(fileStore, fileOwner, [[args.format, [{ path, pinned }]]], ignore, outcome, issues));
|
|
538
|
+
Object.assign(outcome, await captureUnderLock(fileStore, fileOwner, [[args.format, [{ path, pinned }]]], ignore, outcome, issues, "file"));
|
|
506
539
|
} catch (err) {
|
|
507
540
|
// A failed pass may have appended part of the file: never claim a count for it.
|
|
508
|
-
|
|
541
|
+
// An interruption stops only between whole batches, so its count is confirmed.
|
|
542
|
+
return fail("E_CAPTURE_FAILED", err.message || String(err), { ...where, sha256, appended: err instanceof CaptureInterrupted ? outcome.appended : null });
|
|
509
543
|
}
|
|
510
544
|
if (process.exitCode) return fail("E_CAPTURE_FAILED", "capture lock release failed; see stderr for recovery", { ...where, sha256 });
|
|
511
545
|
|
|
512
546
|
const stream = `${fileOwner}~${fmt.source}.${sessionId}`;
|
|
513
547
|
const turns = withReceiptSummaries(fileStore, summary => summary(stream));
|
|
548
|
+
// The summaries are synchronous: a signal during them is still answered.
|
|
549
|
+
try { await safePoint("every batch completed"); }
|
|
550
|
+
catch (err) { return fail("E_CAPTURE_FAILED", err.message, { ...where, sha256, appended: outcome.appended }); }
|
|
514
551
|
const status = statusOf(outcome);
|
|
515
552
|
const { lock: _lock, ...counts } = outcome;
|
|
516
553
|
const receipt = {
|
|
@@ -524,7 +561,7 @@ function fileCapture() {
|
|
|
524
561
|
} finally { pinned.close(); }
|
|
525
562
|
}
|
|
526
563
|
|
|
527
|
-
if (args.file !== undefined || args.format !== undefined || args.json) return finish(fileCapture
|
|
564
|
+
if (args.file !== undefined || args.format !== undefined || args.json) return finish(await observingSignals(fileCapture));
|
|
528
565
|
|
|
529
566
|
if (args["install-hint"]) {
|
|
530
567
|
const self = new URL("./capture-hook.mjs", import.meta.url).pathname;
|
|
@@ -565,7 +602,7 @@ if (args.status) {
|
|
|
565
602
|
// as the OKF harvester can name what it read without timestamps, which tie
|
|
566
603
|
// and which late capture appends behind. Output is JSON, always: this mode
|
|
567
604
|
// exists for programs.
|
|
568
|
-
if (args.home) {
|
|
605
|
+
if (args.home) await observingSignals(async () => {
|
|
569
606
|
warnOnStrangerOwner();
|
|
570
607
|
const unattributed = [];
|
|
571
608
|
let found = [];
|
|
@@ -588,7 +625,7 @@ if (args.home) {
|
|
|
588
625
|
if (!formats.has(s.source)) formats.set(s.source, []);
|
|
589
626
|
formats.get(s.source).push(s);
|
|
590
627
|
}
|
|
591
|
-
Object.assign(outcome, captureUnderLock(store, owner, formats, ignore, outcome, issues));
|
|
628
|
+
Object.assign(outcome, await captureUnderLock(store, owner, formats, ignore, outcome, issues, "home"));
|
|
592
629
|
withReceiptSummaries(store, summary => {
|
|
593
630
|
for (const s of found) {
|
|
594
631
|
const stream = `${owner}~${s.source}.${s.sessionId}`;
|
|
@@ -608,13 +645,18 @@ if (args.home) {
|
|
|
608
645
|
});
|
|
609
646
|
}
|
|
610
647
|
});
|
|
648
|
+
// Summaries are synchronous: a signal that arrived during them is still a
|
|
649
|
+
// signal this command received before answering.
|
|
650
|
+
await safePoint("every batch completed");
|
|
611
651
|
} catch (err) {
|
|
612
652
|
error = err.message || String(err);
|
|
613
653
|
console.error(`capture pass failed: ${error}`);
|
|
614
654
|
outcome.failed++;
|
|
615
655
|
// captureSessions can throw after appending part of a directory. Do not
|
|
616
656
|
// claim zero (or a complete count) for a partially performed failed pass.
|
|
617
|
-
|
|
657
|
+
// An interruption stops only between whole format batches: its count of
|
|
658
|
+
// completed batches is confirmed.
|
|
659
|
+
if (!(err instanceof CaptureInterrupted)) outcome.appended = null;
|
|
618
660
|
process.exitCode = 1;
|
|
619
661
|
}
|
|
620
662
|
if (process.exitCode && !outcome.failed) {
|
|
@@ -626,7 +668,7 @@ if (args.home) {
|
|
|
626
668
|
...(error ? { error } : {}), ...(issues.length ? { issues } : {}), ...(unattributed.length ? { unattributed } : {}),
|
|
627
669
|
}, null, 2));
|
|
628
670
|
// Let stdout drain naturally, including large session-boundary receipts.
|
|
629
|
-
} else {
|
|
671
|
+
}); else {
|
|
630
672
|
|
|
631
673
|
warnOnStrangerOwner();
|
|
632
674
|
try {
|
|
@@ -29,6 +29,7 @@ import { dirname, isAbsolute, join, resolve } from "node:path";
|
|
|
29
29
|
import { fileURLToPath } from "node:url";
|
|
30
30
|
import { captureDefaultRootPlan } from "../lib/capture-default-roots.mjs";
|
|
31
31
|
import { captureCommandPass } from "../lib/capture-command-output.mjs";
|
|
32
|
+
import { captureCommandEntry } from "../lib/capture-command.mjs";
|
|
32
33
|
import { IgnoreError, loadIgnore } from "../lib/ignore.mjs";
|
|
33
34
|
import { preflightCaptureService, applyCaptureService, captureSetupSnapshot, recheckCaptureSetup, writeCaptureSetup } from "../lib/capture-service-install.mjs";
|
|
34
35
|
import { captureRemedyCommand } from "../lib/capture-health.mjs";
|
|
@@ -249,7 +250,7 @@ try {
|
|
|
249
250
|
console.log("running first capture pass (the initial one can take minutes)...");
|
|
250
251
|
captureStarted = true;
|
|
251
252
|
try {
|
|
252
|
-
const pass = await captureCommandPass(root, { owner, plan, indexEnabled: true, trigger: "setup", retrySetAside: false });
|
|
253
|
+
const pass = await captureCommandPass(root, { owner, plan, indexEnabled: true, trigger: "setup", retrySetAside: false }, { entry: captureCommandEntry() });
|
|
253
254
|
summary.firstPass = pass.skipped ? "skipped: another pass holds the record root" : `appended ${pass.appended}, pending ${pass.pending ? "yes" : "no"}`;
|
|
254
255
|
} catch (error) { summary.firstPass = `failed: ${oneLine(error.message)}`; throw error; }
|
|
255
256
|
}
|
|
@@ -148,6 +148,58 @@ older moved-aside copy that is a directory is not removed: each later
|
|
|
148
148
|
move-aside reports it, and an operator removes it. A move that fails leaves the
|
|
149
149
|
file in place and fails the pass.
|
|
150
150
|
|
|
151
|
+
### Stopping capture and the lock owner record
|
|
152
|
+
|
|
153
|
+
SIGTERM and SIGINT stop a supervised pass (manual, watch, hook, setup, service
|
|
154
|
+
and retry) cooperatively. They are observed from before the root lock is taken
|
|
155
|
+
until after it is released. The first one ends admission: no further worker,
|
|
156
|
+
continuation or finalization starts. A worker not yet started is killed at
|
|
157
|
+
once. A started worker is asked to stop through its own lock owner record,
|
|
158
|
+
which it already rereads between bounded units, so it stops at its next unit
|
|
159
|
+
boundary and writes its own `failed`/`interrupted` receipt with the counts it
|
|
160
|
+
committed; prior `lastSuccess` is kept. A worker that has already sent its
|
|
161
|
+
result keeps it. The worker catches TERM/INT itself as well, so a signal sent
|
|
162
|
+
to the caller's whole process group (a terminal's Ctrl-C, `timeout`, a service
|
|
163
|
+
manager's control group) does not kill it mid-unit; it too ends the worker as
|
|
164
|
+
interrupted at its next check. Either way the worker gets one grace of 5 seconds, capped by
|
|
165
|
+
the pass's absolute deadline, and a worker still running then is killed and
|
|
166
|
+
reaped; later signals neither shorten nor extend the grace. A killed worker
|
|
167
|
+
leaves its unfinished `running` receipt, never a finished one, and the next
|
|
168
|
+
pass records it as `lastFailure` `interrupted` and continues from the durable
|
|
169
|
+
checkpoints. The command exits 1. An earlier failure cause observed in the
|
|
170
|
+
same pass is reported instead of the interruption. SIGKILL, or any death that
|
|
171
|
+
runs no cleanup, leaves the lock to the next pass's reclaim of a dead owner
|
|
172
|
+
(see below); there is no cleanup guarantee for it.
|
|
173
|
+
|
|
174
|
+
The exact commands, `--home`, `--file` and `--reset-cursors`, run in the
|
|
175
|
+
calling process, and their work is synchronous. They observe SIGTERM and SIGINT
|
|
176
|
+
only at safe points: before taking the lock, after taking it, before each
|
|
177
|
+
format's batch of exact files, before the index update, and once the work
|
|
178
|
+
under the lock (and the receipt summaries) is done, before any answer. Discovery, a whole
|
|
179
|
+
format's batch, the index update and the receipt summaries cannot be
|
|
180
|
+
preempted, so a signal waits for the segment it arrived in. No response time is
|
|
181
|
+
promised, and these commands have no deadline. An interruption at a safe point
|
|
182
|
+
releases the lock and answers the command's existing receipt, `failed` with the
|
|
183
|
+
error `capture interrupted by SIGTERM` (or SIGINT) and exit 1. `appended`
|
|
184
|
+
counts only the batches that completed, so it is confirmed; an interruption
|
|
185
|
+
observed after all of them says `after every batch completed`, and one after a
|
|
186
|
+
cursor reset says `after the cursor reset was committed`. These commands
|
|
187
|
+
write no run receipt, so a killed exact command leaves no `running` run behind:
|
|
188
|
+
the next pass reclaims its lock and records only `reclaimedLock`.
|
|
189
|
+
|
|
190
|
+
The lock's `owner.json` holds the owner's `pid`, a per-acquisition `nonce`,
|
|
191
|
+
`startedAt` and `host`, and, while a supervised pass runs, its `worker`'s pid,
|
|
192
|
+
nonce, start and host. A lock taken by this release also says who took it:
|
|
193
|
+
`command` (`pass`, `retry`, `home`, `file` or `reset-cursors`), `caller`
|
|
194
|
+
(`entry`, the program: `oats`, `turn-record`, `hook-runner` or
|
|
195
|
+
`service-runner`, left out when the command runs in another program; `trigger`,
|
|
196
|
+
from `manual`, `watch`, `hook`, `setup` and `service`) and `ppid`, the owner's immediate parent process. A cooperative stop
|
|
197
|
+
adds `stop: true`. These are fixed labels and process facts. No argv,
|
|
198
|
+
environment or shell text is recorded, and nothing names an external caller or
|
|
199
|
+
why it stopped the command. The fields are diagnostic only: liveness, release
|
|
200
|
+
and reclaim read only the pid, nonce, start, host and worker, so records from
|
|
201
|
+
earlier releases, without them, are read and reclaimed exactly as before.
|
|
202
|
+
|
|
151
203
|
### Offline installation and run diagnosis
|
|
152
204
|
|
|
153
205
|
The package owns `captureHealth` and `turn-record capture --health [--json]`.
|
|
@@ -236,10 +288,12 @@ Until setup converts it, the hook line from before this release,
|
|
|
236
288
|
with a 120-second timeout, still works against this release: it is one
|
|
237
289
|
ordinary quiet foreground pass under the root lock, attributed `manual`, up to
|
|
238
290
|
its 300-second deadline. A harness that stops it at 120 seconds with SIGTERM or
|
|
239
|
-
SIGINT
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
291
|
+
SIGINT stops it cooperatively: the worker ends at its next unit boundary, or is
|
|
292
|
+
killed after a 5-second grace, the pass ends as interrupted and the lock is
|
|
293
|
+
released ([stopping capture](#stopping-capture-and-the-lock-owner-record)).
|
|
294
|
+
With SIGKILL the supervisor dies holding the lock, which the next pass reclaims
|
|
295
|
+
as a dead owner. Either way the next pass continues from the durable
|
|
296
|
+
checkpoints.
|
|
243
297
|
Setup's pending-activation marker (`<root>/capture-service-pending.json`, see
|
|
244
298
|
[setup service migration](#setup-service-migration)) produces
|
|
245
299
|
`capture-service-pending` from file evidence only: beside setup's own marked
|
|
@@ -453,6 +507,34 @@ bounded transactions. Ordinary passes seek new journal bytes without parsing
|
|
|
453
507
|
consumed prefixes. Inconsistent identities or cursors invalidate derived state;
|
|
454
508
|
recovery can cost a historical scan. No journal format or turn ID changes.
|
|
455
509
|
|
|
510
|
+
The synchronous index update (`RecordIndex.update`, run by `capture --home`,
|
|
511
|
+
`--file` and `recall --reindex`) uses one `capture.db` connection for the pass.
|
|
512
|
+
It first takes a census without locks. A stream is current, and is skipped
|
|
513
|
+
without a stream lock or validation, only when all of the following hold:
|
|
514
|
+
|
|
515
|
+
- its complete validation checkpoint names the journal's exact current version;
|
|
516
|
+
- the search index's byte cursor and epoch match that checkpoint;
|
|
517
|
+
- both count tables agree with it.
|
|
518
|
+
|
|
519
|
+
Every other stream is a candidate. That includes a stream whose evidence is
|
|
520
|
+
missing, partial or disagreeing, backlog left by `--no-index` passes, and
|
|
521
|
+
streams that other writers changed. Each candidate is validated again under its
|
|
522
|
+
stream lock before any write.
|
|
523
|
+
|
|
524
|
+
A candidate that proves the union stale resets it atomically, and selection
|
|
525
|
+
restarts against the empty index. Stale means the journal is missing or
|
|
526
|
+
replaced, a new epoch, an invalid cursor, or a legacy count disagreement. A
|
|
527
|
+
journal that changes between that check and catch-up refuses the pass with the
|
|
528
|
+
existing retry error. A skipped stream's cursor is read again when its turn
|
|
529
|
+
comes, so a reset by another index connection is caught up, not skipped. The
|
|
530
|
+
pass refuses if `capture.db` was replaced or removed while it held the
|
|
531
|
+
connection.
|
|
532
|
+
|
|
533
|
+
The census is not an atomic snapshot of concurrent writers. A change made after
|
|
534
|
+
a stream was observed is indexed by a later pass. A missing checkpoint still
|
|
535
|
+
costs one historical validation of the stream, and the reset it implies then
|
|
536
|
+
reindexes the union.
|
|
537
|
+
|
|
456
538
|
Search-index reset remains one atomic transaction. Ordinary tables use SQLite's
|
|
457
539
|
truncate clear; the content-storing FTS table is dropped and recreated using the
|
|
458
540
|
owning schema instead of deleting text rows individually. Readers retain the
|
|
@@ -734,7 +816,9 @@ after the owned child exits; signals target the spawned child handle, never a
|
|
|
734
816
|
PID inferred from an abandoned lock or a process group. Continuation retains the
|
|
735
817
|
same root lock: it replaces only the exact previously registered worker identity
|
|
736
818
|
after positive death proof. Reused or uncertain PIDs prevent replacement.
|
|
737
|
-
Workers check their parent and lock identity between work units
|
|
819
|
+
Workers check their parent and lock identity between work units; the same read
|
|
820
|
+
carries the supervisor's cooperative stop request, which the supervisor
|
|
821
|
+
publishes into that owner record exactly as it publishes a worker. The supervisor never writes status
|
|
738
822
|
on behalf of a crashed worker. A shared monotonic deadline includes process
|
|
739
823
|
startup and lock publication, with a separate cooperative drain boundary.
|
|
740
824
|
Launch arguments fix the heap policy and exclude inherited Node flags and
|
|
@@ -23,8 +23,8 @@ export function captureCommandOperation(start) {
|
|
|
23
23
|
issues: { sources: options.plan.sources, retry: options.retrySetAside } });
|
|
24
24
|
while (true) {
|
|
25
25
|
const result = session.step();
|
|
26
|
-
if (result.resetRequired) return session.yieldWorker({ resetRequired: true });
|
|
27
|
-
if (result.finalize) return receipt(session.finish({ reason: "deadline" }));
|
|
26
|
+
if (result.resetRequired) return receipt(session.yieldWorker({ resetRequired: true }));
|
|
27
|
+
if (result.finalize) return receipt(session.finish({ reason: start.budget.sample() ?? "deadline" }));
|
|
28
28
|
if (result.stopped) return receipt(session.yieldWorker());
|
|
29
29
|
if (result.idle) return receipt(session.complete());
|
|
30
30
|
}
|
|
@@ -33,9 +33,10 @@ export function releaseDetail(r) {
|
|
|
33
33
|
: `${r.error ? `${r.error}; ` : ""}${r.recovery}`;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
export async function captureCommandPass(root, options, { log = console.log, error = console.error, run = runCaptureCommand } = {}) {
|
|
36
|
+
export async function captureCommandPass(root, options, { log = console.log, error = console.error, run = runCaptureCommand, entry } = {}) {
|
|
37
37
|
let result;
|
|
38
|
-
|
|
38
|
+
// Every foreground command attributes its lock, naming its entry when known.
|
|
39
|
+
try { result = await run(root, options, { entry, attribute: true }); }
|
|
39
40
|
catch (failure) {
|
|
40
41
|
reportLockInitializationFailure(failure, error);
|
|
41
42
|
reportReclaimedLock(failure.reclaimedLock, error);
|