@awebai/oats 0.69.14 → 0.69.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/oats.mjs +79 -18
- package/docs/capabilities.md +57 -14
- package/docs/container.md +111 -17
- package/docs/desktop-cli-api.md +215 -19
- package/docs/desktop.md +3 -3
- package/docs/execution-targets.md +15 -3
- package/docs/first-team.md +4 -3
- package/docs/implementation.md +19 -9
- package/docs/integrations.md +9 -8
- package/docs/knowledge-and-learning.md +2 -2
- package/docs/knowledge-theory.md +17 -15
- package/docs/knowledge.md +202 -179
- package/docs/official-catalog.md +4 -4
- package/docs/packages.md +13 -13
- package/docs/release-notes/v0.69.15.md +444 -0
- package/docs/release-notes/v0.69.16.md +169 -0
- package/docs/schedules.md +168 -34
- package/docs/servers.md +64 -3
- package/docs/souls-and-instances.md +100 -19
- package/docs/upgrading.md +26 -6
- package/docs/workspaces.md +2 -2
- package/lib/claim.mjs +2 -3
- package/lib/command-flags.mjs +4 -2
- package/lib/core.mjs +215 -72
- package/lib/dir-lock.mjs +17 -0
- package/lib/errors.mjs +2 -1
- package/lib/host-run.mjs +218 -0
- package/lib/host-stop-child.mjs +21 -0
- package/lib/instance-git.mjs +32 -6
- package/lib/instance-lifecycle.mjs +4 -3
- package/lib/instance-listing.mjs +35 -6
- package/lib/kernel-features.mjs +4 -1
- package/lib/read-call.mjs +1 -1
- package/lib/remote.mjs +17 -14
- package/lib/schedule.mjs +237 -20
- package/lib/servers.mjs +83 -5
- package/lib/session-end.mjs +28 -8
- package/lib/session-resume.mjs +10 -18
- package/lib/session-seen.mjs +4 -64
- package/lib/status-words.mjs +11 -4
- package/lib/system-tools.mjs +1 -36
- package/lib/worktree-hooks.mjs +6 -175
- package/lib/worktree.mjs +19 -10
- package/package-catalog.json +3 -3
- package/package.json +4 -1
- package/packages/record/README.md +56 -21
- package/packages/record/docs/capture-service.md +8 -5
- package/packages/record/docs/turn-record-sot.md +9 -0
- package/packages/record/lib/capture-health.mjs +6 -4
- package/packages/record/lib/capture-lock.mjs +31 -27
- package/packages/record/lib/capture-retained-scratch.mjs +4 -3
- package/packages/record/lib/capture-scratch.mjs +2 -1
- package/packages/record/lib/derived-state.mjs +4 -2
- package/packages/record/lib/index-db.mjs +4 -2
- package/packages/record/lib/lock-holder.mjs +21 -0
- package/packages/record/lib/private-files.mjs +18 -0
- package/packages/record/lib/process-identity.mjs +310 -0
- package/packages/record/lib/store.mjs +31 -25
- package/skills/oats-getting-started/SKILL.md +9 -6
|
@@ -258,15 +258,18 @@ Appends and merges run under a per-stream lockfile (`streams/<id>/.lock`),
|
|
|
258
258
|
because the torn-tail repair is a read-truncate-write sequence and hooks,
|
|
259
259
|
watchers and manual passes can fire concurrently for the same owner.
|
|
260
260
|
|
|
261
|
-
The lock carries an **owner token** — the holder's pid, its hostname,
|
|
262
|
-
random nonce
|
|
263
|
-
|
|
264
|
-
that
|
|
261
|
+
The lock carries an **owner token** — the holder's pid, its hostname, a
|
|
262
|
+
random nonce, and the boot it was taken in and the holder's process start
|
|
263
|
+
(`bootId`, `processStart`) — written by hard-linking a fully-written temp
|
|
264
|
+
file into place, so a lock that exists is always readable in full. Two
|
|
265
|
+
guarantees rest on that token:
|
|
265
266
|
|
|
266
267
|
- **A live holder is never stolen from, at any age.** Staleness is proven
|
|
267
268
|
from the holder's liveness, not from how long ago the lock was created.
|
|
268
|
-
A contender that finds the holder
|
|
269
|
-
|
|
269
|
+
A contender that finds the holder running (its pid alive with the
|
|
270
|
+
recorded start, in this boot) waits, and fails with a timeout naming the
|
|
271
|
+
holder — it does not reclaim, whatever host the token names. A holder
|
|
272
|
+
whose start cannot be read is waited for too. (The
|
|
270
273
|
earlier design judged a lock stale by age while the waiter's timeout
|
|
271
274
|
started when the contender arrived; a contender arriving late therefore
|
|
272
275
|
reclaimed a lock whose holder was still inside its critical section, and
|
|
@@ -276,9 +279,19 @@ that token:
|
|
|
276
279
|
reclaimed cannot delete the replacement lock out from under its new
|
|
277
280
|
owner.
|
|
278
281
|
|
|
279
|
-
**Crash recovery is immediate, not timed**: a lock whose
|
|
280
|
-
|
|
281
|
-
|
|
282
|
+
**Crash recovery is immediate, not timed**: a lock whose holder is gone is
|
|
283
|
+
provably stale and is reclaimed at once, without waiting out any threshold.
|
|
284
|
+
A holder is gone when its pid is gone, when its pid runs with another start
|
|
285
|
+
than the recorded one (the pid went to another process), or when the lock
|
|
286
|
+
was taken in another boot. A run of a container counts as a boot, so a lock
|
|
287
|
+
left on a volume by a container that was stopped is reclaimed by the next
|
|
288
|
+
one, whatever its hostname and whichever process holds the pid there. The
|
|
289
|
+
kernel judges its own claims by the same rule, with the same code
|
|
290
|
+
(`lib/process-identity.mjs`).
|
|
291
|
+
|
|
292
|
+
A token that names neither boot nor start, as an earlier version writes
|
|
293
|
+
it, is judged as that version judges it: by its pid alone when it names
|
|
294
|
+
this host, and by the lock's age (`lockStaleMs`) when it names another.
|
|
282
295
|
|
|
283
296
|
A lock that cannot be read at all (anything but "not there" — a permission
|
|
284
297
|
error, a directory in its place) is not retried: acquisition fails at once
|
|
@@ -288,8 +301,9 @@ lock judged stale whose removal keeps failing — is bounded by
|
|
|
288
301
|
`lockTimeoutMs` and reports why on timeout.
|
|
289
302
|
|
|
290
303
|
`lockStaleMs` (30 s by default) is the fallback for the two cases where
|
|
291
|
-
liveness cannot be checked here: a lock
|
|
292
|
-
sync can copy one in) and a lock
|
|
304
|
+
liveness cannot be checked here: a lock that names neither boot nor start
|
|
305
|
+
and was created on **another host** (file sync can copy one in) and a lock
|
|
306
|
+
with **no readable token** (written by an
|
|
293
307
|
older version of this package, or by hand). Those are still reclaimed by
|
|
294
308
|
age. `lockTimeoutMs` (10 s) is how long a contender waits; the two
|
|
295
309
|
thresholds are independent, and neither ordering of them is required or
|
|
@@ -297,8 +311,9 @@ assumed.
|
|
|
297
311
|
|
|
298
312
|
Honest limits:
|
|
299
313
|
|
|
300
|
-
- **Pid reuse
|
|
301
|
-
|
|
314
|
+
- **Pid reuse, for a token that names neither boot nor start.** Such a
|
|
315
|
+
lock, left by a process whose pid was
|
|
316
|
+
later reused by an unrelated live process, reads as held forever, and
|
|
302
317
|
contenders on that stream time out until it is deleted by hand. The
|
|
303
318
|
timeout message names the lock path and the recorded holder. This is the
|
|
304
319
|
chosen direction of failure: refusing to write is recoverable, writing
|
|
@@ -311,8 +326,15 @@ Honest limits:
|
|
|
311
326
|
detection, not an exclusion: a replacement landing in the window between
|
|
312
327
|
the recheck and the unlink is not seen and is removed. Nothing here
|
|
313
328
|
defends against that window.
|
|
314
|
-
- **
|
|
315
|
-
|
|
329
|
+
- **One machine at a time writes a record root.** A record root must not
|
|
330
|
+
be shared by two machines. A boot is one machine's:
|
|
331
|
+
a lock taken on another machine was taken in another boot and is
|
|
332
|
+
reclaimed at once, while its holder may be running there. The stream lock
|
|
333
|
+
and the capture lock serialize the processes of one machine (or one
|
|
334
|
+
container run), never two machines that write one directory (NFS).
|
|
335
|
+
- **Sync tools may copy a `.lock` file.** That cannot corrupt data: a
|
|
336
|
+
synced-in lock from another machine is reclaimed at once, or, when it
|
|
337
|
+
names neither boot nor start, judged by age, so it can delay a local
|
|
316
338
|
`mergeStreamCopy` on that stream by up to 30 s — excluding `.lock` from
|
|
317
339
|
sync patterns remains the right configuration.
|
|
318
340
|
|
|
@@ -322,16 +344,29 @@ expose) — the guarantee is OS-crash-level, not power-loss-level.
|
|
|
322
344
|
|
|
323
345
|
**One capture pass at a time.** A pass takes the record root's
|
|
324
346
|
`.capture.lock` directory, whose `owner.json` records the pid, a nonce, the
|
|
325
|
-
start time
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
347
|
+
start time, the host, and the boot the lock was taken in and the holder's
|
|
348
|
+
process start (`bootId`, `processStart`). A pass that finds the lock held
|
|
349
|
+
skips and says so, naming the holder (the next pass catches up). A pass
|
|
350
|
+
that is killed (a hook or caller timeout, a container stop) runs no
|
|
351
|
+
cleanup, so the next pass reclaims a lock whose recorded owner is dead, and
|
|
352
|
+
its registered worker too:
|
|
353
|
+
|
|
354
|
+
- An owner, a worker or a reclaimer is dead when its pid is gone, when its
|
|
355
|
+
pid runs with another start than the recorded one (the pid went to
|
|
356
|
+
another process), or when its record was taken in another boot. A run of
|
|
357
|
+
a container counts as a boot: a lock left on a volume by a container that
|
|
358
|
+
was stopped is reclaimed by the first pass of the next one, whatever its
|
|
359
|
+
hostname and whichever process holds the pid there. The host a record
|
|
360
|
+
names does not enter this judgement. One whose start cannot be read is
|
|
361
|
+
unknowable, and never dead.
|
|
362
|
+
- A record that names neither boot nor start, as an earlier version writes
|
|
363
|
+
it, is judged as that version judges it: dead only when it names this
|
|
364
|
+
host (or no host, below) and its pid is gone.
|
|
330
365
|
- Reclaimers are serialized by a guard, `.capture.lock.reclaim`, taken by
|
|
331
366
|
exclusive create. Under it the record is read again and removed only if it
|
|
332
367
|
still belongs to that dead owner.
|
|
333
368
|
- A live or unknowable owner, an owner-less (initializing) lock and another
|
|
334
|
-
host's lock are never touched.
|
|
369
|
+
host's lock that names neither boot nor start are never touched.
|
|
335
370
|
- A guard left by a reclaimer that died is never removed. It is named, with
|
|
336
371
|
the exact recovery.
|
|
337
372
|
- Records that name no host predate host recording and live under this
|
|
@@ -275,8 +275,9 @@ A failure newer than success and current unresolved stream failures produce
|
|
|
275
275
|
beyond its recorded deadline also warns, without inventing its termination cause.
|
|
276
276
|
Before that deadline, death evidence requires an unchanged local lock registration
|
|
277
277
|
whose worker PID matches the receipt and whose acquisition predates the run,
|
|
278
|
-
with both supervisor and worker positively dead
|
|
279
|
-
|
|
278
|
+
with both supervisor and worker positively dead (each judged by the boot and
|
|
279
|
+
process start its record names). Missing/off-host/hostless,
|
|
280
|
+
changed or uncertain registration and live PIDs supply no death proof.
|
|
280
281
|
The receipt is reread before accepting that observation; health never repairs,
|
|
281
282
|
reclaims, signals termination, writes counters or invents a finished receipt.
|
|
282
283
|
Owned legacy watcher definitions additionally produce `capture-setup-required`,
|
|
@@ -810,12 +811,14 @@ transaction, and identical state values do not produce redundant row updates.
|
|
|
810
811
|
|
|
811
812
|
The private supervisor transport admits one child at a time under the root lock. The child
|
|
812
813
|
announces its nonce and stays idle until the supervisor durably publishes its
|
|
813
|
-
identity in that lock and
|
|
814
|
-
|
|
814
|
+
identity in that lock, with the boot and the child's process start, and sends
|
|
815
|
+
permission to start. A worker that runs, or whose liveness is unknown,
|
|
816
|
+
prevents reclamation after supervisor death; one whose PID is gone or runs with
|
|
817
|
+
another start, or whose record is of another boot, does not. Release occurs only
|
|
815
818
|
after the owned child exits; signals target the spawned child handle, never a
|
|
816
819
|
PID inferred from an abandoned lock or a process group. Continuation retains the
|
|
817
820
|
same root lock: it replaces only the exact previously registered worker identity
|
|
818
|
-
after positive death proof.
|
|
821
|
+
after positive death proof. A worker of uncertain liveness prevents replacement.
|
|
819
822
|
Workers check their parent and lock identity between work units; the same read
|
|
820
823
|
carries the supervisor's cooperative stop request, which the supervisor
|
|
821
824
|
publishes into that owner record exactly as it publishes a worker. The supervisor never writes status
|
|
@@ -202,6 +202,15 @@ at the center of the model.
|
|
|
202
202
|
Turns reference them as `sha256:<hex>`.
|
|
203
203
|
- **index/** (SQLite metadata/FTS, later vectors) is cache. Deleting it loses
|
|
204
204
|
nothing; any replica can rebuild it from streams + objects.
|
|
205
|
+
- **Permissions.** The record holds conversations, so what the store creates
|
|
206
|
+
is for its owner alone: directories with mode `0700` (the root, `streams/`
|
|
207
|
+
and each stream, `objects/` and its fan-out, `index/`) and files with mode
|
|
208
|
+
`0600` (journals, objects, quarantined copies, the index databases). The
|
|
209
|
+
mode is given at creation and the store never changes the permissions of
|
|
210
|
+
what already exists: a root, a directory or a file made beforehand keeps
|
|
211
|
+
the permissions it has until its owner tightens them
|
|
212
|
+
(`chmod -R go-rwx <record-root>`). A capture pass's lock that comes before
|
|
213
|
+
the store creates the root the same way.
|
|
205
214
|
|
|
206
215
|
## Synchronization contract
|
|
207
216
|
|
|
@@ -6,7 +6,8 @@ import { readCaptureStatus } from "./capture-status.mjs";
|
|
|
6
6
|
import { readCurrentIssueBacklog } from "./capture-issues.mjs";
|
|
7
7
|
import { emptyFailureBacklog } from "./capture-failures.mjs";
|
|
8
8
|
import { captureInstallation } from "./capture-health-installation.mjs";
|
|
9
|
-
import {
|
|
9
|
+
import { readCaptureLockOwner, shellQuote } from "./capture-lock.mjs";
|
|
10
|
+
import { holderLiveness } from "./lock-holder.mjs";
|
|
10
11
|
|
|
11
12
|
const setAsideRemedy = (command, root) => `Restore the captured prefix and run ${command} capture --retry-set-aside; or add the source to ${root}/ignore (later turns from that session will not be captured). Retry revalidates; it never authorizes a changed prefix.`;
|
|
12
13
|
export const SET_ASIDE_REMEDY = setAsideRemedy("turn-record", "<root>");
|
|
@@ -23,15 +24,16 @@ const time = value => typeof value === "string" ? Date.parse(value) : NaN;
|
|
|
23
24
|
function deadExecution(root, run, host, liveness, now) {
|
|
24
25
|
// A receipt has no host field. Never signal-probe its PID without matching
|
|
25
26
|
// local registration. Both owner and worker must be dead: a live supervisor
|
|
26
|
-
// can still replace a child and finish this same pass.
|
|
27
|
-
// and changed/foreign/hostless
|
|
27
|
+
// can still replace a child and finish this same pass. A process that runs or
|
|
28
|
+
// cannot be read (holderLiveness judges each record), and changed/foreign/hostless
|
|
29
|
+
// lock records, supply no death evidence.
|
|
28
30
|
const owner = readCaptureLockOwner(root), worker = owner?.worker;
|
|
29
31
|
if (owner?.host !== host || worker?.host !== host || worker.pid !== run.pid ||
|
|
30
32
|
!Number.isInteger(owner.pid) || owner.pid <= 0 || owner.pid === worker.pid ||
|
|
31
33
|
typeof owner.nonce !== "string" || !owner.nonce || typeof worker.nonce !== "string" || !worker.nonce ||
|
|
32
34
|
!(time(owner.startedAt) <= time(run.startedAt)) || !(time(worker.startedAt) >= time(owner.startedAt)) ||
|
|
33
35
|
!(time(worker.startedAt) <= now)) return false;
|
|
34
|
-
if (liveness(owner
|
|
36
|
+
if (liveness(owner) !== "dead" || liveness(worker) !== "dead") return false;
|
|
35
37
|
return JSON.stringify(readCaptureLockOwner(root)) === JSON.stringify(owner) &&
|
|
36
38
|
JSON.stringify(readCaptureStatus(root, { now })?.lastRun) === JSON.stringify(run);
|
|
37
39
|
}
|
|
@@ -4,13 +4,19 @@
|
|
|
4
4
|
// and the next pass catches up, since reconciliation is idempotent.
|
|
5
5
|
//
|
|
6
6
|
// The lock is a DIRECTORY: mkdir is atomic and a directory is never
|
|
7
|
-
// observable half-created. The owner record (pid, nonce, start time, host
|
|
7
|
+
// observable half-created. The owner record (pid, nonce, start time, host,
|
|
8
|
+
// and the boot and process start its holder is judged by)
|
|
8
9
|
// is written inside it after the mkdir. A live, unknowable or still
|
|
9
10
|
// initializing (owner-less) lock refuses the pass and names the holder and
|
|
10
11
|
// the operator recovery; it is never stolen.
|
|
11
12
|
//
|
|
12
|
-
// A lock whose recorded owner AND registered worker are DEAD
|
|
13
|
-
// killed mid-pass by a hook or caller timeout runs no finally).
|
|
13
|
+
// A lock whose recorded owner AND registered worker are DEAD is reclaimed (a capture
|
|
14
|
+
// killed mid-pass by a hook or caller timeout runs no finally). A record that
|
|
15
|
+
// names its holder's boot or start (`bootId`, `processStart`) is judged by them
|
|
16
|
+
// (lock-holder.mjs holderLiveness), whatever host it names: a container that is created again
|
|
17
|
+
// has another hostname, its pids start low again, and a lock on a volume
|
|
18
|
+
// outlives it. A record that names neither is dead only on this host, by its
|
|
19
|
+
// pid alone. Records
|
|
14
20
|
// without host predate host recording and live under this user's home, so
|
|
15
21
|
// they count as this host's (RECLAIM_HOSTLESS_RECORDS). Reclaimers
|
|
16
22
|
// are serialized by a guard, `<lock>.reclaim` (exclusive create): under it
|
|
@@ -21,13 +27,15 @@
|
|
|
21
27
|
// Acquire never waits: it reclaims once or skips, and the next pass catches up.
|
|
22
28
|
// A supervisor publishes its idle child's identity before permitting work.
|
|
23
29
|
// The worker is part of the same lock, not a second independently reclaimed
|
|
24
|
-
// lock. A
|
|
25
|
-
// when the supervisor died. Callers own child termination/reaping; this module
|
|
30
|
+
// lock. A child that runs, or whose liveness cannot be read, prevents release
|
|
31
|
+
// and reclamation even when the supervisor died. Callers own child termination/reaping; this module
|
|
26
32
|
// never sends a terminating signal or infers a process group.
|
|
27
33
|
import { closeSync, constants, existsSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readSync, renameSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
|
|
34
|
+
import { makePrivateDir } from "./private-files.mjs";
|
|
28
35
|
import { randomBytes } from "node:crypto";
|
|
29
36
|
import { hostname } from "node:os";
|
|
30
37
|
import { join } from "node:path";
|
|
38
|
+
import { holderIdentity, holderLiveness, namesIdentity, validPid } from "./lock-holder.mjs";
|
|
31
39
|
|
|
32
40
|
/** Whether an owner record that names no host is reclaimed when its pid is dead here: yes. Records without
|
|
33
41
|
* host predate host recording and live under this user's home, so they are this host's. */
|
|
@@ -35,12 +43,6 @@ export const RECLAIM_HOSTLESS_RECORDS = true;
|
|
|
35
43
|
|
|
36
44
|
export function captureLockPath(root) { return join(root, ".capture.lock"); }
|
|
37
45
|
|
|
38
|
-
/** "alive" | "dead" | "unknown" for an owner pid ("unknown" = exists but not signalable). */
|
|
39
|
-
export function holderLiveness(pid) {
|
|
40
|
-
if (!Number.isInteger(pid) || pid <= 0 || pid > 2147483647) return "unknown";
|
|
41
|
-
try { process.kill(pid, 0); return "alive"; } catch (e) { return e.code === "ESRCH" ? "dead" : "unknown"; }
|
|
42
|
-
}
|
|
43
|
-
|
|
44
46
|
const sameFile = (a, b) => a.dev === b.dev && a.ino === b.ino && a.size === b.size && a.mtimeNs === b.mtimeNs && a.ctimeNs === b.ctimeNs;
|
|
45
47
|
const readJson = (path) => {
|
|
46
48
|
let fd;
|
|
@@ -67,7 +69,7 @@ export const readCaptureLockOwner = root => readOwner(captureLockPath(root));
|
|
|
67
69
|
// lock, from which entry and trigger, and the immediate parent process. Fixed
|
|
68
70
|
// labels and direct process facts only, never argv, environment or a guessed
|
|
69
71
|
// external caller; an entry that is not known is left out. It never confers authority: identity, liveness and reclaim
|
|
70
|
-
// read only pid, nonce, startedAt, host and worker.
|
|
72
|
+
// read only pid, nonce, startedAt, host, bootId, processStart and worker.
|
|
71
73
|
const COMMANDS = new Set(["pass", "home", "file", "reset-cursors", "retry"]);
|
|
72
74
|
const ENTRIES = new Set(["oats", "turn-record", "hook-runner", "service-runner"]);
|
|
73
75
|
const TRIGGERS = new Set(["manual", "watch", "hook", "setup", "service"]);
|
|
@@ -79,20 +81,21 @@ export function captureLockAttribution({ command, caller, ppid = process.ppid }
|
|
|
79
81
|
...(validPid(ppid) ? { ppid } : {}) };
|
|
80
82
|
}
|
|
81
83
|
|
|
82
|
-
function validPid(pid) { return Number.isInteger(pid) && pid > 0 && pid <= 2147483647; }
|
|
83
84
|
function workerDead(owner, { host, liveness }) {
|
|
84
85
|
if (!Object.hasOwn(owner, "worker")) return true;
|
|
85
86
|
const worker = owner.worker;
|
|
86
|
-
return validPid(worker?.pid) && worker.pid !== owner.pid && worker.host === host && typeof worker.nonce === "string" &&
|
|
87
|
+
return validPid(worker?.pid) && worker.pid !== owner.pid && (namesIdentity(worker) || worker.host === host) && typeof worker.nonce === "string" &&
|
|
87
88
|
worker.nonce.length > 0 && worker.nonce.length <= 128 && typeof worker.startedAt === "string" &&
|
|
88
|
-
Number.isFinite(Date.parse(worker.startedAt)) && liveness(worker
|
|
89
|
+
Number.isFinite(Date.parse(worker.startedAt)) && liveness(worker) === "dead";
|
|
89
90
|
}
|
|
90
91
|
|
|
91
|
-
/** Whether `owner` is a record
|
|
92
|
+
/** Whether `owner` is a record whose process is dead, and its worker's: the only lock this module reclaims.
|
|
93
|
+
* A record that names its boot or start is judged wherever it was written; one that names neither only
|
|
94
|
+
* when it is this host's. */
|
|
92
95
|
function deadHere(owner, { host, liveness, reclaimHostless }) {
|
|
93
96
|
if (!owner || !validPid(owner.pid)) return false;
|
|
94
|
-
const here = owner.host === undefined ? reclaimHostless : owner.host === host;
|
|
95
|
-
return here && liveness(owner
|
|
97
|
+
const here = namesIdentity(owner) || (owner.host === undefined ? reclaimHostless : owner.host === host);
|
|
98
|
+
return here && liveness(owner) === "dead" && workerDead(owner, { host, liveness });
|
|
96
99
|
}
|
|
97
100
|
|
|
98
101
|
/** Remove the lock `dir` held by the dead `owner`, serialized by the guard `<dir>.reclaim`. → { removed }
|
|
@@ -135,7 +138,7 @@ export function recoveryInstruction(dir, owner, liveness) {
|
|
|
135
138
|
/** Try to take the root's capture lock. Returns { path, release, reclaimed? }
|
|
136
139
|
* when taken, or { path, held: { pid, startedAt, liveness, recovery, guard? },
|
|
137
140
|
* reclaimed? } when a lock is held. A lock it did not create is removed only
|
|
138
|
-
* when its recorded owner
|
|
141
|
+
* when its recorded owner and worker are dead, under the reclaim guard (the
|
|
139
142
|
* header); `reclaimed: { pid, startedAt }` says THIS call removed it (whether
|
|
140
143
|
* or not it then won the lock), and `held.guard` names a guard a dead
|
|
141
144
|
* reclaimer left. Every other lock is left alone.
|
|
@@ -163,7 +166,7 @@ export function acquireCaptureLock(root, { now = Date.now, pid = process.pid, li
|
|
|
163
166
|
const fs = { writeFileSync, rmSync, openSync, closeSync, lstatSync, fsyncSync, renameSync, ...io };
|
|
164
167
|
const dir = captureLockPath(root);
|
|
165
168
|
const nonce = randomBytes(8).toString("hex");
|
|
166
|
-
|
|
169
|
+
makePrivateDir(root); // the store creates the root lazily; the lock may come first
|
|
167
170
|
let reclaimed;
|
|
168
171
|
for (let attempt = 0; ; attempt++) {
|
|
169
172
|
try { mkdirSync(dir, { mode: 0o700 }); break; }
|
|
@@ -171,8 +174,9 @@ export function acquireCaptureLock(root, { now = Date.now, pid = process.pid, li
|
|
|
171
174
|
if (e.code !== "EEXIST") throw e;
|
|
172
175
|
const owner = readOwner(dir);
|
|
173
176
|
const opts = { host, liveness, reclaimHostless };
|
|
174
|
-
|
|
175
|
-
|
|
177
|
+
// A record with this pass's own pid that names another boot or start is another process's.
|
|
178
|
+
if (attempt === 0 && (owner?.pid !== pid || namesIdentity(owner)) && deadHere(owner, opts)) {
|
|
179
|
+
const r = reclaimDeadLock(dir, owner, { pid, nonce, host, ...holderIdentity(pid) }, opts);
|
|
176
180
|
if (r.abandoned) {
|
|
177
181
|
const { guard, pid: reclaimer } = r.abandoned;
|
|
178
182
|
return { path: dir, held: { pid: owner.pid, startedAt: owner.startedAt, liveness: "dead", guard,
|
|
@@ -186,7 +190,7 @@ export function acquireCaptureLock(root, { now = Date.now, pid = process.pid, li
|
|
|
186
190
|
// Gone between the mkdir and this read (a release or a reclaim): try once more. A directory that
|
|
187
191
|
// exists without a record is initializing or mid-removal, and is reported so.
|
|
188
192
|
if (!now && attempt === 0 && !existsSync(dir)) continue;
|
|
189
|
-
const live = now ? (now.pid === pid ? "alive" : liveness(now
|
|
193
|
+
const live = now ? (now.pid === pid && !namesIdentity(now) ? "alive" : liveness(now)) : "unknown";
|
|
190
194
|
return { path: dir, ...(reclaimed ? { reclaimed } : {}), held: { pid: now?.pid, startedAt: now?.startedAt, liveness: live, recovery: recoveryInstruction(dir, now, live) } };
|
|
191
195
|
}
|
|
192
196
|
}
|
|
@@ -197,7 +201,7 @@ export function acquireCaptureLock(root, { now = Date.now, pid = process.pid, li
|
|
|
197
201
|
// a record-less replacement look like the directory we created.
|
|
198
202
|
directoryFd = fs.openSync(dir, "r");
|
|
199
203
|
identity = fstatSync(directoryFd);
|
|
200
|
-
fs.writeFileSync(join(dir, "owner.json"), JSON.stringify({ pid, nonce, startedAt: new Date(now()).toISOString(), host, ...attribution }), { mode: 0o600 });
|
|
204
|
+
fs.writeFileSync(join(dir, "owner.json"), JSON.stringify({ pid, nonce, startedAt: new Date(now()).toISOString(), host, ...holderIdentity(pid), ...attribution }), { mode: 0o600 });
|
|
201
205
|
} catch (err) {
|
|
202
206
|
// Ownership was proven by the mkdir, not by the moment of cleanup: the
|
|
203
207
|
// directory is removed only if it is still ours (same inode) and holds
|
|
@@ -271,14 +275,14 @@ export function acquireCaptureLock(root, { now = Date.now, pid = process.pid, li
|
|
|
271
275
|
const current = ownCurrent("capture lock changed before worker registration");
|
|
272
276
|
// Continuation never drops the root lock between children. Replace only
|
|
273
277
|
// the caller's exact previous publication, after positive death proof;
|
|
274
|
-
//
|
|
278
|
+
// a worker that runs, or of uncertain liveness, preserves the old lock unchanged.
|
|
275
279
|
if (Object.hasOwn(current, "worker")) {
|
|
276
280
|
if (!previous || current.worker?.pid !== previous.pid || current.worker?.nonce !== previous.nonce ||
|
|
277
281
|
current.worker?.startedAt !== previous.startedAt || !workerDead(current, { host, liveness })) {
|
|
278
282
|
throw new Error("previous capture worker is not proven dead and unchanged");
|
|
279
283
|
}
|
|
280
284
|
} else if (previous) throw new Error("capture lock changed before worker continuation");
|
|
281
|
-
replaceOwner(current, { ...current, worker: { pid: worker.pid, nonce: worker.nonce, startedAt: worker.startedAt, host } }, "worker registration", "worker");
|
|
285
|
+
replaceOwner(current, { ...current, worker: { pid: worker.pid, nonce: worker.nonce, startedAt: worker.startedAt, host, ...holderIdentity(worker.pid) } }, "worker registration", "worker");
|
|
282
286
|
},
|
|
283
287
|
// A cooperative stop request for this acquisition's worker, which reads the
|
|
284
288
|
// owner record between bounded units. It is not a lock release, changes no
|
|
@@ -319,7 +323,7 @@ export function acquireCaptureLock(root, { now = Date.now, pid = process.pid, li
|
|
|
319
323
|
}
|
|
320
324
|
// Our own lock could not be removed. We are the holder and, as far as
|
|
321
325
|
// this process can tell, alive; the operator gets a conditional line.
|
|
322
|
-
const live = pid === process.pid ? "alive" : liveness(
|
|
326
|
+
const live = pid === process.pid ? "alive" : liveness(cur);
|
|
323
327
|
const recovery = `${dir} is still held by pid ${pid} (this pass, ${live} when it reported this); its removal failed${error ? ` (${error.message})` : ""}; once that process has exited (ps -p ${pid}), remove the lock with: rm -r -- ${shellQuote(dir)} and rerun`;
|
|
324
328
|
return { released: false, reason: "remove-failed", liveness: live, ...(error ? { error: error.message } : {}), recovery };
|
|
325
329
|
},
|
|
@@ -7,7 +7,8 @@ import { randomBytes, createHash } from "node:crypto";
|
|
|
7
7
|
import { hostname } from "node:os";
|
|
8
8
|
import { join } from "node:path";
|
|
9
9
|
import { captureContext } from "./capture-context.mjs";
|
|
10
|
-
import {
|
|
10
|
+
import { readCaptureLockOwner } from "./capture-lock.mjs";
|
|
11
|
+
import { holderLiveness } from "./lock-holder.mjs";
|
|
11
12
|
import { captureScratchRoot, captureScratchDirectory, readCaptureScratchMarker,
|
|
12
13
|
sameCaptureScratchMarker } from "./capture-scratch.mjs";
|
|
13
14
|
|
|
@@ -153,11 +154,11 @@ export function openRetainedCaptureScratch(store, stream, key, start, { cleanup
|
|
|
153
154
|
try {
|
|
154
155
|
const mine = marker.value.pid === process.pid && marker.value.nonce === start.workerNonce;
|
|
155
156
|
if (!mine) {
|
|
156
|
-
if (liveness(marker.value
|
|
157
|
+
if (liveness(marker.value) !== "dead") throw new Error("retained capture owner is not proven dead");
|
|
157
158
|
marker = publish(path, ownerValue(key, claim.id, start, claim.owner), () => {
|
|
158
159
|
assertReference(store.root, state, key, claim, path, marker, start);
|
|
159
160
|
payloadAt(path, claim, { allowMissing: cleanup || claim.phase === "deleting" });
|
|
160
|
-
if (liveness(marker.value
|
|
161
|
+
if (liveness(marker.value) !== "dead") throw new Error("retained capture owner death became uncertain");
|
|
161
162
|
});
|
|
162
163
|
}
|
|
163
164
|
// Publication may precede this transaction's commit. A successor accepts
|
|
@@ -5,6 +5,7 @@ import { closeSync, constants, fstatSync, lstatSync, mkdirSync, mkdtempSync, ope
|
|
|
5
5
|
import { randomBytes } from "node:crypto";
|
|
6
6
|
import { hostname } from "node:os";
|
|
7
7
|
import { join } from "node:path";
|
|
8
|
+
import { makePrivateDir } from "./private-files.mjs";
|
|
8
9
|
|
|
9
10
|
const sameObject = (a, b) => a.dev === b.dev && a.ino === b.ino;
|
|
10
11
|
const sameFile = (a, b) => sameObject(a, b) && a.size === b.size && a.mtimeNs === b.mtimeNs && a.ctimeNs === b.ctimeNs;
|
|
@@ -15,7 +16,7 @@ function directory(path) {
|
|
|
15
16
|
}
|
|
16
17
|
function privateRoot(root, create) {
|
|
17
18
|
const path = join(root, ".capture-tmp");
|
|
18
|
-
if (create) {
|
|
19
|
+
if (create) { makePrivateDir(root); try { mkdirSync(path, { mode: 0o700 }); } catch (e) { if (e.code !== "EEXIST") throw e; } }
|
|
19
20
|
const stat = directory(path);
|
|
20
21
|
if (stat.dev !== statSync(root, { bigint: true }).dev) throw new Error(`capture scratch must be on the record root volume: ${path}`);
|
|
21
22
|
if ((stat.mode & 0o077n) !== 0n || (process.getuid && stat.uid !== BigInt(process.getuid()))) throw new Error(`capture scratch directory must be private and owned by this user: ${path}`);
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
// Local accelerators only. Journal bytes remain authority; removing this
|
|
2
2
|
// database costs bounded rebuild work, never changes a turn or its ID.
|
|
3
|
-
import {
|
|
3
|
+
import { statSync, unlinkSync } from "node:fs";
|
|
4
4
|
import { join } from "node:path";
|
|
5
5
|
import { keysetRows } from "./sqlite-rows.mjs";
|
|
6
|
+
import { ensurePrivateFile, makePrivateDir } from "./private-files.mjs";
|
|
6
7
|
import { createRequire } from "node:module";
|
|
7
8
|
const require = createRequire(import.meta.url);
|
|
8
9
|
|
|
@@ -31,9 +32,10 @@ function fileIdentity(path) {
|
|
|
31
32
|
export class DerivedState {
|
|
32
33
|
constructor(root) {
|
|
33
34
|
const { DatabaseSync } = require("node:sqlite");
|
|
34
|
-
|
|
35
|
+
makePrivateDir(join(root, "index"));
|
|
35
36
|
this.path = join(root, "index", "capture.db");
|
|
36
37
|
const before = fileIdentity(this.path);
|
|
38
|
+
ensurePrivateFile(this.path);
|
|
37
39
|
this.db = new DatabaseSync(this.path);
|
|
38
40
|
try {
|
|
39
41
|
this.db.exec("PRAGMA busy_timeout = 60000; PRAGMA cache_size = -2048; PRAGMA temp_store = FILE");
|
|
@@ -11,10 +11,11 @@
|
|
|
11
11
|
// verbatim record); text is extracted from that line, so a hit names the
|
|
12
12
|
// turn AND the event's line in the original transcript.
|
|
13
13
|
|
|
14
|
-
import { closeSync, existsSync,
|
|
14
|
+
import { closeSync, existsSync, openSync, readSync, rmSync, unlinkSync } from "node:fs";
|
|
15
15
|
import { join } from "node:path";
|
|
16
16
|
import { keysetRows } from "./sqlite-rows.mjs";
|
|
17
17
|
import { captureContext } from "./capture-context.mjs";
|
|
18
|
+
import { ensurePrivateFile, makePrivateDir } from "./private-files.mjs";
|
|
18
19
|
import { DerivedState, fileVersion, sameVersion } from "./derived-state.mjs";
|
|
19
20
|
import { validJournalCheckpoint } from "./store.mjs";
|
|
20
21
|
import { INDEX_SCHEMA_VERSION as SCHEMA_VERSION, TEXT_SCHEMA, resetIndexTables } from "./index-reset.mjs";
|
|
@@ -110,9 +111,10 @@ export class RecordIndex {
|
|
|
110
111
|
const { DatabaseSync } = require("node:sqlite");
|
|
111
112
|
this.store = store;
|
|
112
113
|
const dir = join(store.root, "index");
|
|
113
|
-
|
|
114
|
+
makePrivateDir(dir);
|
|
114
115
|
const path = join(dir, "turns.db");
|
|
115
116
|
const open = () => {
|
|
117
|
+
ensurePrivateFile(path);
|
|
116
118
|
const db = new DatabaseSync(path);
|
|
117
119
|
// Concurrent index users (watcher pass, manual reindex, recall)
|
|
118
120
|
// wait for each other instead of dying on "database is locked".
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// The holder a lock record names, and whether it is gone: one reading for the
|
|
2
|
+
// capture lock (its owner, its worker, its reclaim guard: capture-lock.mjs)
|
|
3
|
+
// and the stream lock (store.mjs). A record names its holder by pid, the boot
|
|
4
|
+
// it was taken in and the holder's process start, and is judged by the
|
|
5
|
+
// rule the kernel judges its cache write lock by (process-identity.mjs
|
|
6
|
+
// identityHolderState).
|
|
7
|
+
import { identityHolderState, namesIdentity, processStartToken, thisBoot, validPid } from "./process-identity.mjs";
|
|
8
|
+
|
|
9
|
+
export { namesIdentity, validPid };
|
|
10
|
+
|
|
11
|
+
/** What a record names its holder `pid` by beside the pid: this boot and the pid's start (null when it
|
|
12
|
+
* cannot be read). */
|
|
13
|
+
export const holderIdentity = (pid) => ({ bootId: thisBoot(), processStart: processStartToken(pid) });
|
|
14
|
+
|
|
15
|
+
const LIVENESS = { alive: "alive", gone: "dead", unknown: "unknown" };
|
|
16
|
+
/** "alive" | "dead" | "unknown" for the process a lock record names: identityHolderState in this
|
|
17
|
+
* package's words. One that names its boot or start is dead when taken in another boot, or when
|
|
18
|
+
* its pid is gone, runs with another start (a reused pid) or is a zombie; unknown when its start
|
|
19
|
+
* cannot be read. One that names neither is judged by its pid alone: dead when the pid is gone.
|
|
20
|
+
* A record without a process ID is unknown, whatever else it names. */
|
|
21
|
+
export const holderLiveness = (holder) => LIVENESS[identityHolderState(holder)];
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// The record store holds agents' conversations, so what it creates is for its
|
|
2
|
+
// owner alone: files 0600, directories 0700. A mode applies when the file or
|
|
3
|
+
// directory is created; one that exists keeps the permissions it has.
|
|
4
|
+
import { closeSync, mkdirSync, openSync } from "node:fs";
|
|
5
|
+
|
|
6
|
+
export const PRIVATE_FILE = 0o600;
|
|
7
|
+
export const PRIVATE_DIR = 0o700;
|
|
8
|
+
|
|
9
|
+
/** Create `path` and its missing ancestors, each private. */
|
|
10
|
+
export function makePrivateDir(path) {
|
|
11
|
+
mkdirSync(path, { recursive: true, mode: PRIVATE_DIR });
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/** Create `path` as an empty private file when it is absent, for an opener
|
|
15
|
+
* that takes no mode (SQLite, whose journal files take the database's). */
|
|
16
|
+
export function ensurePrivateFile(path) {
|
|
17
|
+
closeSync(openSync(path, "a", PRIVATE_FILE));
|
|
18
|
+
}
|