@awebai/oats 0.62.0 → 0.64.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -1
- package/bin/oats.mjs +58 -23
- package/docs/desktop-cli-api.md +73 -3
- package/docs/execution-targets.md +45 -2
- package/docs/first-team.md +6 -0
- package/docs/official-catalog.md +1 -1
- package/docs/packages.md +5 -5
- package/docs/release-lane.md +13 -0
- package/docs/release-notes/v0.63.0.md +135 -0
- package/docs/release-notes/v0.64.0.md +105 -0
- package/docs/souls-and-instances.md +32 -7
- package/docs/upgrading.md +6 -3
- package/docs/workspaces.md +1 -1
- package/lib/core.mjs +100 -28
- package/lib/session-input.mjs +12 -2
- package/lib/tree-copy.mjs +188 -7
- package/package-catalog.json +1 -1
- package/package.json +1 -1
- package/packages/client/cli-locator.mjs +2 -2
- package/packages/client/instance-tree.mjs +53 -3
- package/packages/client/roster-sections.mjs +22 -6
- package/packages/record/bin/capture.mjs +59 -17
- package/packages/record/bin/setup.mjs +2 -1
- package/packages/record/docs/capture-service.md +89 -5
- package/packages/record/lib/capture-command-operation.mjs +2 -2
- package/packages/record/lib/capture-command-output.mjs +3 -2
- package/packages/record/lib/capture-command.mjs +20 -3
- package/packages/record/lib/capture-hook-runner.mjs +1 -1
- package/packages/record/lib/capture-index-reset.mjs +1 -0
- package/packages/record/lib/capture-lock.mjs +67 -33
- package/packages/record/lib/capture-pass.mjs +22 -8
- package/packages/record/lib/capture-run.mjs +1 -1
- package/packages/record/lib/capture-service-runner.mjs +1 -1
- package/packages/record/lib/capture-supervisor.mjs +24 -5
- package/packages/record/lib/capture-work-session.mjs +7 -4
- package/packages/record/lib/capture-worker.mjs +20 -4
- package/packages/record/lib/derived-state.mjs +30 -2
- package/packages/record/lib/index-db.mjs +118 -49
- package/packages/record/lib/store.mjs +13 -6
- package/skills/oats-getting-started/SKILL.md +438 -170
|
@@ -148,6 +148,58 @@ older moved-aside copy that is a directory is not removed: each later
|
|
|
148
148
|
move-aside reports it, and an operator removes it. A move that fails leaves the
|
|
149
149
|
file in place and fails the pass.
|
|
150
150
|
|
|
151
|
+
### Stopping capture and the lock owner record
|
|
152
|
+
|
|
153
|
+
SIGTERM and SIGINT stop a supervised pass (manual, watch, hook, setup, service
|
|
154
|
+
and retry) cooperatively. They are observed from before the root lock is taken
|
|
155
|
+
until after it is released. The first one ends admission: no further worker,
|
|
156
|
+
continuation or finalization starts. A worker not yet started is killed at
|
|
157
|
+
once. A started worker is asked to stop through its own lock owner record,
|
|
158
|
+
which it already rereads between bounded units, so it stops at its next unit
|
|
159
|
+
boundary and writes its own `failed`/`interrupted` receipt with the counts it
|
|
160
|
+
committed; prior `lastSuccess` is kept. A worker that has already sent its
|
|
161
|
+
result keeps it. The worker catches TERM/INT itself as well, so a signal sent
|
|
162
|
+
to the caller's whole process group (a terminal's Ctrl-C, `timeout`, a service
|
|
163
|
+
manager's control group) does not kill it mid-unit; it too ends the worker as
|
|
164
|
+
interrupted at its next check. Either way the worker gets one grace of 5 seconds, capped by
|
|
165
|
+
the pass's absolute deadline, and a worker still running then is killed and
|
|
166
|
+
reaped; later signals neither shorten nor extend the grace. A killed worker
|
|
167
|
+
leaves its unfinished `running` receipt, never a finished one, and the next
|
|
168
|
+
pass records it as `lastFailure` `interrupted` and continues from the durable
|
|
169
|
+
checkpoints. The command exits 1. An earlier failure cause observed in the
|
|
170
|
+
same pass is reported instead of the interruption. SIGKILL, or any death that
|
|
171
|
+
runs no cleanup, leaves the lock to the next pass's reclaim of a dead owner
|
|
172
|
+
(see below); there is no cleanup guarantee for it.
|
|
173
|
+
|
|
174
|
+
The exact commands, `--home`, `--file` and `--reset-cursors`, run in the
|
|
175
|
+
calling process, and their work is synchronous. They observe SIGTERM and SIGINT
|
|
176
|
+
only at safe points: before taking the lock, after taking it, before each
|
|
177
|
+
format's batch of exact files, before the index update, and once the work
|
|
178
|
+
under the lock (and the receipt summaries) is done, before any answer. Discovery, a whole
|
|
179
|
+
format's batch, the index update and the receipt summaries cannot be
|
|
180
|
+
preempted, so a signal waits for the segment it arrived in. No response time is
|
|
181
|
+
promised, and these commands have no deadline. An interruption at a safe point
|
|
182
|
+
releases the lock and answers the command's existing receipt, `failed` with the
|
|
183
|
+
error `capture interrupted by SIGTERM` (or SIGINT) and exit 1. `appended`
|
|
184
|
+
counts only the batches that completed, so it is confirmed; an interruption
|
|
185
|
+
observed after all of them says `after every batch completed`, and one after a
|
|
186
|
+
cursor reset says `after the cursor reset was committed`. These commands
|
|
187
|
+
write no run receipt, so a killed exact command leaves no `running` run behind:
|
|
188
|
+
the next pass reclaims its lock and records only `reclaimedLock`.
|
|
189
|
+
|
|
190
|
+
The lock's `owner.json` holds the owner's `pid`, a per-acquisition `nonce`,
|
|
191
|
+
`startedAt` and `host`, and, while a supervised pass runs, its `worker`'s pid,
|
|
192
|
+
nonce, start and host. A lock taken by this release also says who took it:
|
|
193
|
+
`command` (`pass`, `retry`, `home`, `file` or `reset-cursors`), `caller`
|
|
194
|
+
(`entry`, the program: `oats`, `turn-record`, `hook-runner` or
|
|
195
|
+
`service-runner`, left out when the command runs in another program; `trigger`,
|
|
196
|
+
from `manual`, `watch`, `hook`, `setup` and `service`) and `ppid`, the owner's immediate parent process. A cooperative stop
|
|
197
|
+
adds `stop: true`. These are fixed labels and process facts. No argv,
|
|
198
|
+
environment or shell text is recorded, and nothing names an external caller or
|
|
199
|
+
why it stopped the command. The fields are diagnostic only: liveness, release
|
|
200
|
+
and reclaim read only the pid, nonce, start, host and worker, so records from
|
|
201
|
+
earlier releases, without them, are read and reclaimed exactly as before.
|
|
202
|
+
|
|
151
203
|
### Offline installation and run diagnosis
|
|
152
204
|
|
|
153
205
|
The package owns `captureHealth` and `turn-record capture --health [--json]`.
|
|
@@ -236,10 +288,12 @@ Until setup converts it, the hook line from before this release,
|
|
|
236
288
|
with a 120-second timeout, still works against this release: it is one
|
|
237
289
|
ordinary quiet foreground pass under the root lock, attributed `manual`, up to
|
|
238
290
|
its 300-second deadline. A harness that stops it at 120 seconds with SIGTERM or
|
|
239
|
-
SIGINT
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
291
|
+
SIGINT stops it cooperatively: the worker ends at its next unit boundary, or is
|
|
292
|
+
killed after a 5-second grace, the pass ends as interrupted and the lock is
|
|
293
|
+
released ([stopping capture](#stopping-capture-and-the-lock-owner-record)).
|
|
294
|
+
With SIGKILL the supervisor dies holding the lock, which the next pass reclaims
|
|
295
|
+
as a dead owner. Either way the next pass continues from the durable
|
|
296
|
+
checkpoints.
|
|
243
297
|
Setup's pending-activation marker (`<root>/capture-service-pending.json`, see
|
|
244
298
|
[setup service migration](#setup-service-migration)) produces
|
|
245
299
|
`capture-service-pending` from file evidence only: beside setup's own marked
|
|
@@ -453,6 +507,34 @@ bounded transactions. Ordinary passes seek new journal bytes without parsing
|
|
|
453
507
|
consumed prefixes. Inconsistent identities or cursors invalidate derived state;
|
|
454
508
|
recovery can cost a historical scan. No journal format or turn ID changes.
|
|
455
509
|
|
|
510
|
+
The synchronous index update (`RecordIndex.update`, run by `capture --home`,
|
|
511
|
+
`--file` and `recall --reindex`) uses one `capture.db` connection for the pass.
|
|
512
|
+
It first takes a census without locks. A stream is current, and is skipped
|
|
513
|
+
without a stream lock or validation, only when all of the following hold:
|
|
514
|
+
|
|
515
|
+
- its complete validation checkpoint names the journal's exact current version;
|
|
516
|
+
- the search index's byte cursor and epoch match that checkpoint;
|
|
517
|
+
- both count tables agree with it.
|
|
518
|
+
|
|
519
|
+
Every other stream is a candidate. That includes a stream whose evidence is
|
|
520
|
+
missing, partial or disagreeing, backlog left by `--no-index` passes, and
|
|
521
|
+
streams that other writers changed. Each candidate is validated again under its
|
|
522
|
+
stream lock before any write.
|
|
523
|
+
|
|
524
|
+
A candidate that proves the union stale resets it atomically, and selection
|
|
525
|
+
restarts against the empty index. Stale means the journal is missing or
|
|
526
|
+
replaced, a new epoch, an invalid cursor, or a legacy count disagreement. A
|
|
527
|
+
journal that changes between that check and catch-up refuses the pass with the
|
|
528
|
+
existing retry error. A skipped stream's cursor is read again when its turn
|
|
529
|
+
comes, so a reset by another index connection is caught up, not skipped. The
|
|
530
|
+
pass refuses if `capture.db` was replaced or removed while it held the
|
|
531
|
+
connection.
|
|
532
|
+
|
|
533
|
+
The census is not an atomic snapshot of concurrent writers. A change made after
|
|
534
|
+
a stream was observed is indexed by a later pass. A missing checkpoint still
|
|
535
|
+
costs one historical validation of the stream, and the reset it implies then
|
|
536
|
+
reindexes the union.
|
|
537
|
+
|
|
456
538
|
Search-index reset remains one atomic transaction. Ordinary tables use SQLite's
|
|
457
539
|
truncate clear; the content-storing FTS table is dropped and recreated using the
|
|
458
540
|
owning schema instead of deleting text rows individually. Readers retain the
|
|
@@ -734,7 +816,9 @@ after the owned child exits; signals target the spawned child handle, never a
|
|
|
734
816
|
PID inferred from an abandoned lock or a process group. Continuation retains the
|
|
735
817
|
same root lock: it replaces only the exact previously registered worker identity
|
|
736
818
|
after positive death proof. Reused or uncertain PIDs prevent replacement.
|
|
737
|
-
Workers check their parent and lock identity between work units
|
|
819
|
+
Workers check their parent and lock identity between work units; the same read
|
|
820
|
+
carries the supervisor's cooperative stop request, which the supervisor
|
|
821
|
+
publishes into that owner record exactly as it publishes a worker. The supervisor never writes status
|
|
738
822
|
on behalf of a crashed worker. A shared monotonic deadline includes process
|
|
739
823
|
startup and lock publication, with a separate cooperative drain boundary.
|
|
740
824
|
Launch arguments fix the heap policy and exclude inherited Node flags and
|
|
@@ -23,8 +23,8 @@ export function captureCommandOperation(start) {
|
|
|
23
23
|
issues: { sources: options.plan.sources, retry: options.retrySetAside } });
|
|
24
24
|
while (true) {
|
|
25
25
|
const result = session.step();
|
|
26
|
-
if (result.resetRequired) return session.yieldWorker({ resetRequired: true });
|
|
27
|
-
if (result.finalize) return receipt(session.finish({ reason: "deadline" }));
|
|
26
|
+
if (result.resetRequired) return receipt(session.yieldWorker({ resetRequired: true }));
|
|
27
|
+
if (result.finalize) return receipt(session.finish({ reason: start.budget.sample() ?? "deadline" }));
|
|
28
28
|
if (result.stopped) return receipt(session.yieldWorker());
|
|
29
29
|
if (result.idle) return receipt(session.complete());
|
|
30
30
|
}
|
|
@@ -33,9 +33,10 @@ export function releaseDetail(r) {
|
|
|
33
33
|
: `${r.error ? `${r.error}; ` : ""}${r.recovery}`;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
export async function captureCommandPass(root, options, { log = console.log, error = console.error, run = runCaptureCommand } = {}) {
|
|
36
|
+
export async function captureCommandPass(root, options, { log = console.log, error = console.error, run = runCaptureCommand, entry } = {}) {
|
|
37
37
|
let result;
|
|
38
|
-
|
|
38
|
+
// Every foreground command attributes its lock, naming its entry when known.
|
|
39
|
+
try { result = await run(root, options, { entry, attribute: true }); }
|
|
39
40
|
catch (failure) {
|
|
40
41
|
reportLockInitializationFailure(failure, error);
|
|
41
42
|
reportReclaimedLock(failure.reclaimedLock, error);
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// Private command caller. Root discovery runs only in the admitted worker;
|
|
2
2
|
// callers cannot choose its executable, entry path, heap or pass allowances.
|
|
3
3
|
import { createHash } from "node:crypto";
|
|
4
|
-
import {
|
|
4
|
+
import { realpathSync } from "node:fs";
|
|
5
|
+
import { basename, isAbsolute, resolve } from "node:path";
|
|
5
6
|
import { fileURLToPath } from "node:url";
|
|
6
7
|
import { captureDefaultRootPlan } from "./capture-default-roots.mjs";
|
|
7
8
|
import { superviseCapturePass } from "./capture-pass.mjs";
|
|
@@ -25,7 +26,22 @@ export function captureCommandSelection(options) {
|
|
|
25
26
|
return createHash("sha256").update(JSON.stringify(options.plan)).digest("hex");
|
|
26
27
|
}
|
|
27
28
|
|
|
28
|
-
|
|
29
|
+
/** The fixed label of the program a command runs in: the kernel's `oats`
|
|
30
|
+
* (which imports the record bins) or one of the record's own bins,
|
|
31
|
+
* `turn-record`; undefined for any other program (a test runner, an
|
|
32
|
+
* embedding host), which is never guessed. Only the entry's file name is
|
|
33
|
+
* compared; nothing of argv is recorded. */
|
|
34
|
+
const RECORD_BINS = new Set(["turn-record.mjs", "capture.mjs", "setup.mjs"]);
|
|
35
|
+
export function captureCommandEntry(entry = process.argv[1] ?? "") {
|
|
36
|
+
let path = entry;
|
|
37
|
+
try { path = realpathSync(entry); } catch { /* the name as given */ }
|
|
38
|
+
const name = basename(path);
|
|
39
|
+
return name === "oats.mjs" ? "oats" : RECORD_BINS.has(name) ? "turn-record" : undefined;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** `entry` labels the calling program in the lock owner record (see
|
|
43
|
+
* captureLockAttribution); `attribute: false` leaves the record without. */
|
|
44
|
+
export async function runCaptureCommand(root, options, { entry, attribute = entry !== undefined } = {}) {
|
|
29
45
|
if (typeof root !== "string" || !root || root.includes("\0")) throw new Error("invalid capture command root");
|
|
30
46
|
const selected = captureCommandOptions(options);
|
|
31
47
|
// Resolve relative root spelling against the caller, never a child's cwd.
|
|
@@ -35,7 +51,8 @@ export async function runCaptureCommand(root, options) {
|
|
|
35
51
|
try { result = await superviseCapturePass(absolute, selected, {
|
|
36
52
|
workerPath: fileURLToPath(new URL("./capture-command-worker.mjs", import.meta.url)),
|
|
37
53
|
acquire: path => {
|
|
38
|
-
const lock = acquireCaptureLock(path
|
|
54
|
+
const lock = acquireCaptureLock(path, !attribute ? {} : {
|
|
55
|
+
command: selected.retrySetAside ? "retry" : "pass", caller: { entry, trigger: selected.trigger } });
|
|
39
56
|
if (lock.reclaimed) reclaimedLock = { path: lock.path, ...lock.reclaimed };
|
|
40
57
|
return lock;
|
|
41
58
|
},
|
|
@@ -8,7 +8,7 @@ try {
|
|
|
8
8
|
if (!request || Object.keys(request).some(key => !["root", "options"].includes(key))) throw new Error("invalid capture hook request");
|
|
9
9
|
const options = captureCommandOptions(request.options);
|
|
10
10
|
if (options.trigger !== "hook" || options.retrySetAside) throw new Error("invalid capture hook intent");
|
|
11
|
-
const result = await runCaptureCommand(request.root, options);
|
|
11
|
+
const result = await runCaptureCommand(request.root, options, { entry: "hook-runner" });
|
|
12
12
|
process.exitCode = result.exitCode;
|
|
13
13
|
} catch {
|
|
14
14
|
// stdout/stderr are deliberately detached from the harness. Only admitted
|
|
@@ -89,6 +89,7 @@ export class CaptureIndexReset {
|
|
|
89
89
|
// No capture.db transaction spans this atomic index operation. The token
|
|
90
90
|
// commits with the reset, so recovery cannot erase index work performed
|
|
91
91
|
// after that commit just because sweep publication subsequently failed.
|
|
92
|
+
this.index.statements.clear(); // prepared against the tables the reset recreates
|
|
92
93
|
this.index.db.exec("BEGIN");
|
|
93
94
|
try {
|
|
94
95
|
resetIndexTables(this.index.db);
|
|
@@ -63,6 +63,22 @@ const readOwner = (dir) => {
|
|
|
63
63
|
};
|
|
64
64
|
export const readCaptureLockOwner = root => readOwner(captureLockPath(root));
|
|
65
65
|
|
|
66
|
+
// Diagnostic attribution in a fresh owner record: which command took the
|
|
67
|
+
// lock, from which entry and trigger, and the immediate parent process. Fixed
|
|
68
|
+
// labels and direct process facts only, never argv, environment or a guessed
|
|
69
|
+
// external caller; an entry that is not known is left out. It never confers authority: identity, liveness and reclaim
|
|
70
|
+
// read only pid, nonce, startedAt, host and worker.
|
|
71
|
+
const COMMANDS = new Set(["pass", "home", "file", "reset-cursors", "retry"]);
|
|
72
|
+
const ENTRIES = new Set(["oats", "turn-record", "hook-runner", "service-runner"]);
|
|
73
|
+
const TRIGGERS = new Set(["manual", "watch", "hook", "setup", "service"]);
|
|
74
|
+
export function captureLockAttribution({ command, caller, ppid = process.ppid } = {}) {
|
|
75
|
+
if (command === undefined && caller === undefined) return {};
|
|
76
|
+
if (!COMMANDS.has(command) || caller?.entry !== undefined && !ENTRIES.has(caller.entry) || !TRIGGERS.has(caller?.trigger) ||
|
|
77
|
+
Object.keys(caller).some(key => key !== "entry" && key !== "trigger")) throw new Error("invalid capture lock attribution");
|
|
78
|
+
return { command, caller: { ...(caller.entry !== undefined ? { entry: caller.entry } : {}), trigger: caller.trigger },
|
|
79
|
+
...(validPid(ppid) ? { ppid } : {}) };
|
|
80
|
+
}
|
|
81
|
+
|
|
66
82
|
function validPid(pid) { return Number.isInteger(pid) && pid > 0 && pid <= 2147483647; }
|
|
67
83
|
function workerDead(owner, { host, liveness }) {
|
|
68
84
|
if (!Object.hasOwn(owner, "worker")) return true;
|
|
@@ -141,8 +157,9 @@ export function recoveryInstruction(dir, owner, liveness) {
|
|
|
141
157
|
* operator recovery followed by a new pass in the same long-lived process).
|
|
142
158
|
*
|
|
143
159
|
* `io` exists for fault injection in tests only. */
|
|
144
|
-
export function acquireCaptureLock(root, { now = Date.now, pid = process.pid, liveness = holderLiveness, host = hostname(), reclaimHostless = RECLAIM_HOSTLESS_RECORDS, io = {} } = {}) {
|
|
160
|
+
export function acquireCaptureLock(root, { now = Date.now, pid = process.pid, liveness = holderLiveness, host = hostname(), reclaimHostless = RECLAIM_HOSTLESS_RECORDS, io = {}, command, caller } = {}) {
|
|
145
161
|
if (!validPid(pid)) throw new Error("capture lock owner must have a positive process ID");
|
|
162
|
+
const attribution = captureLockAttribution({ command, caller });
|
|
146
163
|
const fs = { writeFileSync, rmSync, openSync, closeSync, lstatSync, fsyncSync, renameSync, ...io };
|
|
147
164
|
const dir = captureLockPath(root);
|
|
148
165
|
const nonce = randomBytes(8).toString("hex");
|
|
@@ -180,7 +197,7 @@ export function acquireCaptureLock(root, { now = Date.now, pid = process.pid, li
|
|
|
180
197
|
// a record-less replacement look like the directory we created.
|
|
181
198
|
directoryFd = fs.openSync(dir, "r");
|
|
182
199
|
identity = fstatSync(directoryFd);
|
|
183
|
-
fs.writeFileSync(join(dir, "owner.json"), JSON.stringify({ pid, nonce, startedAt: new Date(now()).toISOString(), host }), { mode: 0o600 });
|
|
200
|
+
fs.writeFileSync(join(dir, "owner.json"), JSON.stringify({ pid, nonce, startedAt: new Date(now()).toISOString(), host, ...attribution }), { mode: 0o600 });
|
|
184
201
|
} catch (err) {
|
|
185
202
|
// Ownership was proven by the mkdir, not by the moment of cleanup: the
|
|
186
203
|
// directory is removed only if it is still ours (same inode) and holds
|
|
@@ -205,6 +222,44 @@ export function acquireCaptureLock(root, { now = Date.now, pid = process.pid, li
|
|
|
205
222
|
} finally {
|
|
206
223
|
if (directoryFd !== undefined) fs.closeSync(directoryFd);
|
|
207
224
|
}
|
|
225
|
+
const ownCurrent = message => {
|
|
226
|
+
const current = readOwner(dir), st = fs.lstatSync(dir);
|
|
227
|
+
if (st.dev !== identity.dev || st.ino !== identity.ino || current?.pid !== pid || current?.nonce !== nonce) throw new Error(message);
|
|
228
|
+
return current;
|
|
229
|
+
};
|
|
230
|
+
// Durably replace this acquisition's owner record with `next`, only while it
|
|
231
|
+
// is still exactly `current` in the directory this call created.
|
|
232
|
+
const replaceOwner = (current, next, what, label) => {
|
|
233
|
+
const temp = join(dir, `${label}-${nonce}.tmp`); let fd, dirFd, rootFd, temporary, published = false;
|
|
234
|
+
try {
|
|
235
|
+
dirFd = fs.openSync(dir, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
236
|
+
const pinned = fstatSync(dirFd);
|
|
237
|
+
if (pinned.dev !== identity.dev || pinned.ino !== identity.ino) throw new Error(`capture lock changed before ${what}`);
|
|
238
|
+
fd = fs.openSync(temp, "wx", 0o600);
|
|
239
|
+
fs.writeFileSync(fd, JSON.stringify(next));
|
|
240
|
+
fs.fsyncSync(fd); temporary = fstatSync(fd, { bigint: true });
|
|
241
|
+
const again = readOwner(dir), named = fs.lstatSync(dir);
|
|
242
|
+
if (named.dev !== identity.dev || named.ino !== identity.ino || JSON.stringify(again) !== JSON.stringify(current) || !sameFile(temporary, fs.lstatSync(temp, { bigint: true }))) throw new Error(`capture lock changed during ${what}`);
|
|
243
|
+
fs.renameSync(temp, join(dir, "owner.json"));
|
|
244
|
+
published = true;
|
|
245
|
+
// Publish file bytes, their rename, and the lock directory's parent
|
|
246
|
+
// entry before GO. A failure is not permission to start the worker.
|
|
247
|
+
fs.fsyncSync(dirFd);
|
|
248
|
+
rootFd = fs.openSync(root, "r"); fs.fsyncSync(rootFd);
|
|
249
|
+
} finally {
|
|
250
|
+
// Never remove a path another caller replaced, even after a failed
|
|
251
|
+
// publication. An unpublished/unverifiable file is conservatively kept.
|
|
252
|
+
if (!published && temporary) {
|
|
253
|
+
try {
|
|
254
|
+
const named = fs.lstatSync(dir);
|
|
255
|
+
if (named.dev === identity.dev && named.ino === identity.ino && sameFile(temporary, fs.lstatSync(temp, { bigint: true }))) unlinkSync(temp);
|
|
256
|
+
} catch { /* absent or unknown: preserve */ }
|
|
257
|
+
}
|
|
258
|
+
if (fd !== undefined) fs.closeSync(fd);
|
|
259
|
+
if (rootFd !== undefined) fs.closeSync(rootFd);
|
|
260
|
+
if (dirFd !== undefined) fs.closeSync(dirFd);
|
|
261
|
+
}
|
|
262
|
+
};
|
|
208
263
|
return {
|
|
209
264
|
path: dir,
|
|
210
265
|
nonce,
|
|
@@ -213,8 +268,7 @@ export function acquireCaptureLock(root, { now = Date.now, pid = process.pid, li
|
|
|
213
268
|
// A dead supervisor's lock cannot then be reclaimed while its worker lives.
|
|
214
269
|
registerWorker(worker, { previous } = {}) {
|
|
215
270
|
if (!validPid(worker?.pid) || worker.pid === pid || typeof worker.nonce !== "string" || !worker.nonce || worker.nonce.length > 128 || typeof worker.startedAt !== "string" || !Number.isFinite(Date.parse(worker.startedAt))) throw new Error("invalid capture worker identity");
|
|
216
|
-
const current =
|
|
217
|
-
if (st.dev !== identity.dev || st.ino !== identity.ino || current?.pid !== pid || current?.nonce !== nonce) throw new Error("capture lock changed before worker registration");
|
|
271
|
+
const current = ownCurrent("capture lock changed before worker registration");
|
|
218
272
|
// Continuation never drops the root lock between children. Replace only
|
|
219
273
|
// the caller's exact previous publication, after positive death proof;
|
|
220
274
|
// reused PIDs and uncertain liveness preserve the old lock unchanged.
|
|
@@ -224,35 +278,15 @@ export function acquireCaptureLock(root, { now = Date.now, pid = process.pid, li
|
|
|
224
278
|
throw new Error("previous capture worker is not proven dead and unchanged");
|
|
225
279
|
}
|
|
226
280
|
} else if (previous) throw new Error("capture lock changed before worker continuation");
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
if (named.dev !== identity.dev || named.ino !== identity.ino || JSON.stringify(again) !== JSON.stringify(current) || !sameFile(temporary, fs.lstatSync(temp, { bigint: true }))) throw new Error("capture lock changed during worker registration");
|
|
237
|
-
fs.renameSync(temp, join(dir, "owner.json"));
|
|
238
|
-
published = true;
|
|
239
|
-
// Publish file bytes, their rename, and the lock directory's parent
|
|
240
|
-
// entry before GO. A failure is not permission to start the worker.
|
|
241
|
-
fs.fsyncSync(dirFd);
|
|
242
|
-
rootFd = fs.openSync(root, "r"); fs.fsyncSync(rootFd);
|
|
243
|
-
} finally {
|
|
244
|
-
// Never remove a path another caller replaced, even after a failed
|
|
245
|
-
// registration. An unpublished/unverifiable file is conservatively kept.
|
|
246
|
-
if (!published && temporary) {
|
|
247
|
-
try {
|
|
248
|
-
const named = fs.lstatSync(dir);
|
|
249
|
-
if (named.dev === identity.dev && named.ino === identity.ino && sameFile(temporary, fs.lstatSync(temp, { bigint: true }))) unlinkSync(temp);
|
|
250
|
-
} catch { /* absent or unknown: preserve */ }
|
|
251
|
-
}
|
|
252
|
-
if (fd !== undefined) fs.closeSync(fd);
|
|
253
|
-
if (rootFd !== undefined) fs.closeSync(rootFd);
|
|
254
|
-
if (dirFd !== undefined) fs.closeSync(dirFd);
|
|
255
|
-
}
|
|
281
|
+
replaceOwner(current, { ...current, worker: { pid: worker.pid, nonce: worker.nonce, startedAt: worker.startedAt, host } }, "worker registration", "worker");
|
|
282
|
+
},
|
|
283
|
+
// A cooperative stop request for this acquisition's worker, which reads the
|
|
284
|
+
// owner record between bounded units. It is not a lock release, changes no
|
|
285
|
+
// identity field and confers nothing: reclaim and release ignore it.
|
|
286
|
+
requestStop() {
|
|
287
|
+
const current = ownCurrent("capture lock changed before stop request");
|
|
288
|
+
if (current.stop === true) return;
|
|
289
|
+
replaceOwner(current, { ...current, stop: true }, "stop request", "stop");
|
|
256
290
|
},
|
|
257
291
|
release: () => {
|
|
258
292
|
if (!existsSync(dir)) return { released: false, reason: "gone" };
|
|
@@ -7,7 +7,7 @@ import { CAPTURE_POLICY } from "./capture-budget.mjs";
|
|
|
7
7
|
import { acquireCaptureLock } from "./capture-lock.mjs";
|
|
8
8
|
import { cleanupCaptureScratch } from "./capture-scratch.mjs";
|
|
9
9
|
import { setAsideInvalidCaptureStatus } from "./capture-status.mjs";
|
|
10
|
-
import { captureMonotonicMs, superviseCapture } from "./capture-supervisor.mjs";
|
|
10
|
+
import { captureMonotonicMs, captureSignalPoll, superviseCapture } from "./capture-supervisor.mjs";
|
|
11
11
|
|
|
12
12
|
// Internal scheduling choices, not public tuning or portable timing promises.
|
|
13
13
|
const ALLOWANCES = Object.freeze({ workMs: 60_000, finalizeMs: 70_000, hardMs: 75_000,
|
|
@@ -23,11 +23,26 @@ export async function superviseCapturePass(root, options, {
|
|
|
23
23
|
!Number.isSafeInteger(maxStarts) || maxStarts < 2 || maxStarts > 8) throw new Error("invalid internal capture scheduling allowances");
|
|
24
24
|
const started = clock(), deadline = started + policy.deadlineSeconds * 1000;
|
|
25
25
|
const drainAt = Math.max(started, deadline - drainSeconds * 1000), runId = randomUUID();
|
|
26
|
+
// TERM/INT is observed from before acquisition until after release: it ends
|
|
27
|
+
// admission of further workers; the current one is stopped by its supervisor.
|
|
28
|
+
let interrupted = false;
|
|
29
|
+
const onSignal = () => { interrupted = true; };
|
|
30
|
+
process.on("SIGINT", onSignal); process.on("SIGTERM", onSignal);
|
|
31
|
+
try {
|
|
32
|
+
return await superviseAcquiredPass(root, options, { policy, clock, acquire, allowances, transport,
|
|
33
|
+
started, deadline, drainAt, runId, interrupted: () => interrupted });
|
|
34
|
+
} finally { process.off("SIGINT", onSignal); process.off("SIGTERM", onSignal); }
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
async function superviseAcquiredPass(root, options, { policy, clock, acquire, allowances, transport,
|
|
38
|
+
started, deadline, drainAt, runId, interrupted: signalled }) {
|
|
39
|
+
const { workMs, finalizeMs, hardMs, finalReserveMs, finalDrainMs, maxStarts } = allowances;
|
|
26
40
|
const lock = acquire(root);
|
|
27
41
|
if (!lock.release) return { outcome: "overlap", reason: null, exitCode: 0, starts: 0, held: lock.held };
|
|
28
42
|
let starts = 0, registered, publicationFailed = false, interrupted = false, firstFailure = null, failureMessage;
|
|
29
43
|
let previous = null, result = { outcome: "partial", reason: "deadline", exitCode: 0 };
|
|
30
|
-
|
|
44
|
+
// A signal pending from synchronous preparation dispatches only at a poll.
|
|
45
|
+
const polled = async () => { await captureSignalPoll(); interrupted ||= signalled(); return interrupted; };
|
|
31
46
|
const attempt = async finalize => {
|
|
32
47
|
const launch = clock();
|
|
33
48
|
starts++;
|
|
@@ -62,13 +77,12 @@ export async function superviseCapturePass(root, options, {
|
|
|
62
77
|
// per-call scratch whose allocating process is proven dead (killed workers).
|
|
63
78
|
cleanupCaptureScratch(root, lock);
|
|
64
79
|
invalidStatus = setAsideInvalidCaptureStatus(root);
|
|
65
|
-
process.on("SIGINT", onSignal); process.on("SIGTERM", onSignal);
|
|
66
80
|
// Keep one start and time for a worker to reconcile/write final facts.
|
|
67
81
|
// A normal child is admitted only with its full allowance; in particular,
|
|
68
82
|
// atomic reset must not be admitted as a shortened last-minute operation.
|
|
69
|
-
while (!
|
|
83
|
+
while (!await polled() && starts < maxStarts - 1 && clock() + hardMs + finalReserveMs <= deadline) {
|
|
70
84
|
result = await attempt(false);
|
|
71
|
-
if (publicationFailed ||
|
|
85
|
+
if (publicationFailed || await polled()) break;
|
|
72
86
|
// A worker may finish an honestly failed pass after committing healthy
|
|
73
87
|
// independent work and exhausting ready work. That explicit terminal
|
|
74
88
|
// result needs no idle retries; an unexplained crash still continues.
|
|
@@ -79,7 +93,7 @@ export async function superviseCapturePass(root, options, {
|
|
|
79
93
|
// On a crash the next child reconciles durable admission, then selects
|
|
80
94
|
// independent ready work. There is NO same-pass retry/diagnostic class.
|
|
81
95
|
}
|
|
82
|
-
if (!
|
|
96
|
+
if (!await polled() && !publicationFailed && starts < maxStarts && clock() < deadline) {
|
|
83
97
|
result = await attempt(true);
|
|
84
98
|
// Finalization cannot certify a sweep that this continuation transport
|
|
85
99
|
// stopped early. The worker likewise must not manufacture lastSuccess.
|
|
@@ -87,14 +101,14 @@ export async function superviseCapturePass(root, options, {
|
|
|
87
101
|
result = { outcome: "failed", reason: "io", exitCode: 1 }; firstFailure ??= "io";
|
|
88
102
|
}
|
|
89
103
|
}
|
|
90
|
-
|
|
104
|
+
// An earlier observed failure cause of this pass is kept, not replaced.
|
|
105
|
+
if (await polled() && !firstFailure) return ended({ outcome: "failed", reason: "interrupted", exitCode: 1, starts });
|
|
91
106
|
if (firstFailure) return failed();
|
|
92
107
|
return ended({ ...result, starts });
|
|
93
108
|
} catch (error) {
|
|
94
109
|
if (invalidStatus) error.invalidStatus ??= invalidStatus;
|
|
95
110
|
throw error;
|
|
96
111
|
} finally {
|
|
97
|
-
process.off("SIGINT", onSignal); process.off("SIGTERM", onSignal);
|
|
98
112
|
const released = lock.release();
|
|
99
113
|
if (!released.released) throw Object.assign(new Error("capture pass could not release its lock"), { release: released,
|
|
100
114
|
...(invalidStatus ? { invalidStatus } : {}) });
|
|
@@ -8,7 +8,7 @@ import { CAPTURE_COUNT_FIELDS, captureReclaimedLock, emptyCaptureCounts, emptyCa
|
|
|
8
8
|
const reasons = new Set(["deadline", "rss", "heap", "interrupted", "input", "io"]);
|
|
9
9
|
const messages = Object.freeze({ deadline: "Capture worker exceeded its admitted time allowance.",
|
|
10
10
|
rss: "Capture worker exceeded its sampled memory allowance.", heap: "Capture worker aborted at its heap limit.",
|
|
11
|
-
interrupted: "Capture execution
|
|
11
|
+
interrupted: "Capture execution was interrupted.",
|
|
12
12
|
input: "Capture inputs remain pending, set aside or failed.", io: "Capture could not complete an operational step." });
|
|
13
13
|
const summarize = run => Object.fromEntries(["id", "trigger", "pid", "startedAt", "finishedAt", "counts", "peakRssBytes"].map(key => [key, structuredClone(run[key])]));
|
|
14
14
|
const failureSummary = (run, reason, at) => ({ ...summarize({ ...run, finishedAt: at }), reason,
|
|
@@ -11,7 +11,7 @@ try {
|
|
|
11
11
|
// here: the registered worker resolves the explicit plan after admission.
|
|
12
12
|
loadIgnore(request.root);
|
|
13
13
|
await captureCommandPass(request.root, { owner: request.owner, plan: request.plan,
|
|
14
|
-
trigger: "service", indexEnabled: true, retrySetAside: false });
|
|
14
|
+
trigger: "service", indexEnabled: true, retrySetAside: false }, { entry: "service-runner" });
|
|
15
15
|
} catch (error) {
|
|
16
16
|
console.error(error instanceof IgnoreError ? error.message : `capture service failed: ${error.message}`);
|
|
17
17
|
process.exitCode = 1;
|
|
@@ -8,6 +8,9 @@ import { acquireCaptureLock } from "./capture-lock.mjs";
|
|
|
8
8
|
// hrtime uses the same monotonic origin in both local processes. Startup and
|
|
9
9
|
// lock publication consume this budget; the child never starts a new 300s clock.
|
|
10
10
|
export const captureMonotonicMs = () => Number(process.hrtime.bigint() / 1000000n);
|
|
11
|
+
// A signal received during synchronous work stays pending until a poll phase.
|
|
12
|
+
// One setImmediate can resume before that phase; two hops allow one between.
|
|
13
|
+
export const captureSignalPoll = () => new Promise(resolve => setImmediate(() => setImmediate(resolve)));
|
|
11
14
|
export function captureWorkerEnvironment(env = process.env) {
|
|
12
15
|
const out = {};
|
|
13
16
|
for (const key of ["HOME", "PATH", "TMPDIR", "TMP", "TEMP", "XDG_CACHE_HOME", "XDG_CONFIG_HOME", "LANG", "LC_ALL", "TZ", "USERPROFILE", "SYSTEMROOT"])
|
|
@@ -39,9 +42,15 @@ export function validWorkerResult(value) {
|
|
|
39
42
|
|
|
40
43
|
/** Internal seams are for isolated tests, not public tuning. Only the worker
|
|
41
44
|
* writes run/status facts. The supervisor owns admission, process lifetime and
|
|
42
|
-
* its command result; a forced death leaves an unfinished worker receipt.
|
|
45
|
+
* its command result; a forced death leaves an unfinished worker receipt.
|
|
46
|
+
*
|
|
47
|
+
* TERM/INT is a cooperative stop: a started worker is asked, through its own
|
|
48
|
+
* owner record, to stop at its next bounded check and write its interrupted
|
|
49
|
+
* receipt. The first signal starts one grace (`graceMs`, capped by the
|
|
50
|
+
* absolute deadline) after which the exact child is killed; later signals
|
|
51
|
+
* neither shorten nor extend it. A child not yet started is killed at once. */
|
|
43
52
|
export async function superviseCapture(root, options, {
|
|
44
|
-
workerPath, policy = CAPTURE_POLICY, drainSeconds = 5,
|
|
53
|
+
workerPath, policy = CAPTURE_POLICY, drainSeconds = 5, graceMs = 5000,
|
|
45
54
|
executable = process.execPath, env = process.env, clock = captureMonotonicMs,
|
|
46
55
|
acquire = acquireCaptureLock, spawnChild = spawn,
|
|
47
56
|
window, pass,
|
|
@@ -54,14 +63,23 @@ export async function superviseCapture(root, options, {
|
|
|
54
63
|
const finalizeAt = Math.min(deadline, window?.finalizeAt ?? deadline);
|
|
55
64
|
const lock = acquire(root);
|
|
56
65
|
if (!lock.release) return { outcome: "overlap", reason: null, exitCode: 0, held: lock.held };
|
|
57
|
-
let child, timer, finish, forced, result, ready = false, diagnostics = "", outputBytes = 0;
|
|
66
|
+
let child, timer, grace, finish, forced, result, admitted = false, ready = false, diagnostics = "", outputBytes = 0;
|
|
58
67
|
const stop = reason => {
|
|
59
68
|
forced ??= reason;
|
|
60
69
|
// ChildProcess owns this exact child until exit/reaping. Never derive a
|
|
61
70
|
// process group, or signal a PID recovered from a persisted lock record.
|
|
62
71
|
if (child?.pid > 0 && child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
|
|
63
72
|
};
|
|
64
|
-
const interrupted = () =>
|
|
73
|
+
const interrupted = () => {
|
|
74
|
+
if (grace !== undefined || forced) return;
|
|
75
|
+
if (!admitted) { stop("interrupted"); return; }
|
|
76
|
+
// A worker that already sent its result keeps it; it still must exit
|
|
77
|
+
// within the same grace.
|
|
78
|
+
if (!result) {
|
|
79
|
+
try { lock.requestStop(); } catch { stop("interrupted"); return; }
|
|
80
|
+
}
|
|
81
|
+
grace = setTimeout(() => stop("interrupted"), Math.max(0, Math.min(deadline, clock() + graceMs) - clock()));
|
|
82
|
+
};
|
|
65
83
|
try {
|
|
66
84
|
process.on("SIGTERM", interrupted); process.on("SIGINT", interrupted);
|
|
67
85
|
child = spawnChild(executable, [`--max-old-space-size=${policy.heapMiB}`, "--no-warnings",
|
|
@@ -88,6 +106,7 @@ export async function superviseCapture(root, options, {
|
|
|
88
106
|
try {
|
|
89
107
|
lock.registerWorker(message);
|
|
90
108
|
if (clock() >= deadline) { stop("deadline"); return; }
|
|
109
|
+
admitted = true;
|
|
91
110
|
child.send({ type: "start", root, options, policy, parent: { pid: process.pid, nonce: lock.nonce },
|
|
92
111
|
workerNonce: message.nonce, drainAt, finalizeAt, hardAt: deadline, pass,
|
|
93
112
|
reclaimedLock: lock.reclaimed ?? null }, error => { if (error) stop("io"); });
|
|
@@ -108,7 +127,7 @@ export async function superviseCapture(root, options, {
|
|
|
108
127
|
if (finish) await finish;
|
|
109
128
|
return { outcome: "failed", reason: "io", exitCode: 1 };
|
|
110
129
|
} finally {
|
|
111
|
-
clearTimeout(timer);
|
|
130
|
+
clearTimeout(timer); clearTimeout(grace);
|
|
112
131
|
process.off("SIGTERM", interrupted); process.off("SIGINT", interrupted);
|
|
113
132
|
const released = lock.release();
|
|
114
133
|
// Cleanup failure is an operational error even after a successful worker.
|
|
@@ -200,9 +200,10 @@ export class CaptureWorkSession {
|
|
|
200
200
|
|
|
201
201
|
// Yield/stop never infer completion. The owning worker calls complete() only
|
|
202
202
|
// when it wants a final all-stage decision; idle is merely a scheduling hint.
|
|
203
|
+
// An observed interruption ends the run as interrupted, finalization included.
|
|
203
204
|
yieldWorker({ resetRequired = false } = {}) {
|
|
204
|
-
if (this.start.pass.finalize
|
|
205
|
-
|
|
205
|
+
if (this.start.pass.finalize || this.start.budget.reason === "interrupted")
|
|
206
|
+
return this.finish({ reason: this.start.budget.reason ?? "deadline" });
|
|
206
207
|
this.run.publish({ pending: true, peakRssBytes: this.start.budget.peakRssBytes });
|
|
207
208
|
return { outcome: "partial", reason: this.start.budget.reason ?? "deadline", continuation: true,
|
|
208
209
|
...(resetRequired ? { resetRequired: true } : {}) };
|
|
@@ -215,8 +216,10 @@ export class CaptureWorkSession {
|
|
|
215
216
|
}
|
|
216
217
|
|
|
217
218
|
complete(options) {
|
|
218
|
-
|
|
219
|
-
|
|
219
|
+
// The sampled reason (interrupted, rss, deadline) is kept; finalization
|
|
220
|
+
// without one reports its allowance.
|
|
221
|
+
const stopped = this.start.budget.sample();
|
|
222
|
+
if (this.start.pass.finalize || stopped !== null) return this.finish({ reason: stopped ?? "deadline" });
|
|
220
223
|
const audit = this.indexEnabled ? auditCaptureCompletion(this, options) : auditCaptureSourceRetirement(this, options);
|
|
221
224
|
if (audit.stopped || audit.needsWork) return this.yieldWorker();
|
|
222
225
|
// Normally these facts entered the run during replacement construction.
|
|
@@ -7,6 +7,19 @@ import { captureMonotonicMs, captureWorkerMessage, validWorkerResult } from "./c
|
|
|
7
7
|
|
|
8
8
|
export async function runCaptureWorker(operation) {
|
|
9
9
|
if (!process.send || !process.connected) throw new Error("capture worker requires its supervisor");
|
|
10
|
+
// TERM/INT sent to the caller's whole process group (a terminal's Ctrl-C,
|
|
11
|
+
// timeout(1), a service manager's control group) reaches this worker too.
|
|
12
|
+
// Caught, it cannot end the worker mid-unit; the supervisor, which received
|
|
13
|
+
// the same signal, owns the stop request and the grace. A signal this worker
|
|
14
|
+
// observes itself also ends it as interrupted at its next check.
|
|
15
|
+
let signalled = false;
|
|
16
|
+
const onSignal = () => { signalled = true; };
|
|
17
|
+
process.on("SIGTERM", onSignal); process.on("SIGINT", onSignal);
|
|
18
|
+
try { return await superviseOperation(operation, () => signalled); }
|
|
19
|
+
finally { process.off("SIGTERM", onSignal); process.off("SIGINT", onSignal); }
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
async function superviseOperation(operation, signalled) {
|
|
10
23
|
const nonce = randomBytes(16).toString("hex"), parentPid = process.ppid;
|
|
11
24
|
const start = await new Promise((resolve, reject) => {
|
|
12
25
|
const disconnected = () => { cleanup(); reject(new Error("capture supervisor disconnected before start")); };
|
|
@@ -26,15 +39,18 @@ export async function runCaptureWorker(operation) {
|
|
|
26
39
|
if (error) { cleanup(); reject(new Error("capture worker could not announce readiness")); }
|
|
27
40
|
});
|
|
28
41
|
});
|
|
29
|
-
|
|
30
|
-
|
|
42
|
+
// The same owner read between bounded units also carries the supervisor's
|
|
43
|
+
// cooperative stop request; either way this worker stops as interrupted.
|
|
44
|
+
const owned = () => {
|
|
45
|
+
if (!process.connected || process.ppid !== parentPid) return null;
|
|
31
46
|
const owner = readCaptureLockOwner(start.root);
|
|
32
47
|
return owner?.pid === parentPid && owner.nonce === start.parent.nonce &&
|
|
33
|
-
owner.worker?.pid === process.pid && owner.worker.nonce === nonce;
|
|
48
|
+
owner.worker?.pid === process.pid && owner.worker.nonce === nonce ? owner : null;
|
|
34
49
|
};
|
|
50
|
+
const alive = () => { if (signalled()) return false; const owner = owned(); return owner !== null && owner.stop !== true; };
|
|
35
51
|
// Publication must already be durable before the first operation, including
|
|
36
52
|
// its first status write. A callback must check again between bounded units.
|
|
37
|
-
if (!
|
|
53
|
+
if (!owned()) throw new Error("capture worker lock publication is missing");
|
|
38
54
|
const budget = new CaptureBudget({ policy: start.policy, clock: captureMonotonicMs,
|
|
39
55
|
drainAt: start.drainAt, supervisorAlive: alive });
|
|
40
56
|
let result;
|