@awebai/oats 0.61.0 → 0.62.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/oats.mjs +26 -11
- package/docs/capabilities.md +1 -1
- package/docs/execution-targets.md +49 -3
- package/docs/implementation.md +2 -1
- package/docs/named-test-runs.md +382 -0
- package/docs/official-catalog.md +1 -1
- package/docs/packages.md +5 -5
- package/docs/release-notes/v0.62.0.md +223 -0
- package/docs/upgrading.md +133 -11
- package/docs/workspaces.md +1 -1
- package/lib/core.mjs +2 -2
- package/lib/doctor-capture.mjs +35 -0
- package/lib/session-viewer.mjs +86 -7
- package/package-catalog.json +1 -1
- package/package.json +1 -1
- package/packages/client/cli-locator.mjs +2 -2
- package/packages/record/README.md +87 -13
- package/packages/record/bin/capture-hook.mjs +8 -0
- package/packages/record/bin/capture.mjs +143 -124
- package/packages/record/bin/recall.mjs +8 -3
- package/packages/record/bin/setup.mjs +115 -225
- package/packages/record/bin/turn-record.mjs +2 -2
- package/packages/record/docs/capture-service.md +1443 -28
- package/packages/record/docs/capture-status.schema.json +429 -12
- package/packages/record/lib/capture-admission.mjs +82 -0
- package/packages/record/lib/capture-aw-unit.mjs +137 -0
- package/packages/record/lib/capture-aw.mjs +31 -12
- package/packages/record/lib/capture-budget.mjs +31 -0
- package/packages/record/lib/capture-cc.mjs +39 -6
- package/packages/record/lib/capture-cleanup-queue.mjs +135 -0
- package/packages/record/lib/capture-cleanup-work.mjs +77 -0
- package/packages/record/lib/capture-command-operation.mjs +37 -0
- package/packages/record/lib/capture-command-output.mjs +61 -0
- package/packages/record/lib/capture-command-worker.mjs +6 -0
- package/packages/record/lib/capture-command.mjs +48 -0
- package/packages/record/lib/capture-completion.mjs +152 -0
- package/packages/record/lib/capture-context.mjs +50 -0
- package/packages/record/lib/capture-default-roots.mjs +30 -0
- package/packages/record/lib/capture-discovery.mjs +114 -0
- package/packages/record/lib/capture-failures.mjs +45 -0
- package/packages/record/lib/capture-health-doctor.mjs +13 -0
- package/packages/record/lib/capture-health-installation.mjs +117 -0
- package/packages/record/lib/capture-health.mjs +116 -0
- package/packages/record/lib/capture-hook-ownership.mjs +58 -0
- package/packages/record/lib/capture-hook-runner.mjs +17 -0
- package/packages/record/lib/capture-hook.mjs +26 -0
- package/packages/record/lib/capture-index-inventory.mjs +205 -0
- package/packages/record/lib/capture-index-outcomes.mjs +113 -0
- package/packages/record/lib/capture-index-reset.mjs +109 -0
- package/packages/record/lib/capture-index-work.mjs +77 -0
- package/packages/record/lib/capture-issue-preparation.mjs +73 -0
- package/packages/record/lib/capture-issues.mjs +15 -2
- package/packages/record/lib/capture-lock.mjs +102 -16
- package/packages/record/lib/capture-native-unit.mjs +322 -0
- package/packages/record/lib/capture-pass.mjs +102 -0
- package/packages/record/lib/capture-recovery-snapshot.mjs +182 -0
- package/packages/record/lib/capture-retained-scratch.mjs +213 -0
- package/packages/record/lib/capture-retry.mjs +102 -0
- package/packages/record/lib/capture-run.mjs +125 -0
- package/packages/record/lib/capture-scratch.mjs +12 -3
- package/packages/record/lib/capture-service-definition.mjs +104 -0
- package/packages/record/lib/capture-service-install.mjs +362 -0
- package/packages/record/lib/capture-service-ownership.mjs +137 -0
- package/packages/record/lib/capture-service-runner.mjs +18 -0
- package/packages/record/lib/capture-small-snapshot.mjs +85 -0
- package/packages/record/lib/capture-source-discovery.mjs +319 -0
- package/packages/record/lib/capture-source-queue.mjs +199 -0
- package/packages/record/lib/capture-source-work.mjs +85 -0
- package/packages/record/lib/capture-state.mjs +16 -71
- package/packages/record/lib/capture-status.mjs +171 -0
- package/packages/record/lib/capture-supervisor.mjs +117 -0
- package/packages/record/lib/capture-watch.mjs +32 -0
- package/packages/record/lib/capture-work-session.mjs +238 -0
- package/packages/record/lib/capture-worker.mjs +58 -0
- package/packages/record/lib/derived-state.mjs +28 -15
- package/packages/record/lib/file-lines.mjs +22 -6
- package/packages/record/lib/formats.mjs +4 -3
- package/packages/record/lib/index-capture.mjs +97 -0
- package/packages/record/lib/index-db.mjs +3 -17
- package/packages/record/lib/index-reset.mjs +19 -0
- package/packages/record/lib/store.mjs +36 -11
- package/packages/record/lib/transcript-prefix.mjs +125 -0
- package/skills/oats-getting-started/SKILL.md +1 -1
- package/docs/macos-test-runs.md +0 -246
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
// Private command preparation. Page mutations and their resume key commit
|
|
2
|
+
// together; this is retry intent, never source/custody or completion proof.
|
|
3
|
+
import { basename } from "node:path";
|
|
4
|
+
import { SESSION_FORMATS } from "./formats.mjs";
|
|
5
|
+
import { loadIgnore } from "./ignore.mjs";
|
|
6
|
+
|
|
7
|
+
const formats = ["cc", "pi", "codex", "aw"];
|
|
8
|
+
export class CaptureIssuePreparation {
|
|
9
|
+
constructor(state, queue, { runId, sources, retry = false }) {
|
|
10
|
+
if (typeof runId !== "string" || !runId || typeof retry !== "boolean" ||
|
|
11
|
+
!Array.isArray(sources) || new Set(sources).size !== sources.length || sources.some(source => !formats.includes(source)) ||
|
|
12
|
+
retry && sources.length !== formats.length)
|
|
13
|
+
throw new Error("invalid capture issue preparation");
|
|
14
|
+
this.state = state; this.queue = queue; this.runId = runId; this.retry = retry;
|
|
15
|
+
this.sources = formats.filter(source => sources.includes(source));
|
|
16
|
+
this.scope = queue.current().id;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
current() {
|
|
20
|
+
if (this.queue.current().id !== this.scope) throw new Error("capture issue preparation scope changed");
|
|
21
|
+
const row = this.state.get("capture-issue-preparation", this.queue.owner);
|
|
22
|
+
if (!row || row.runId !== this.runId) return null;
|
|
23
|
+
if (row.scope !== this.scope || row.retry !== this.retry || JSON.stringify(row.sources) !== JSON.stringify(this.sources) ||
|
|
24
|
+
row.after !== null && typeof row.after !== "string" || typeof row.complete !== "boolean")
|
|
25
|
+
throw new Error("capture issue preparation selection changed");
|
|
26
|
+
return row;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
complete() { return this.current()?.complete === true; }
|
|
30
|
+
|
|
31
|
+
advance(root, { stopped = () => false, finalize = false, limit = 128 } = {}) {
|
|
32
|
+
if (this.state.depth || !Number.isSafeInteger(limit) || limit < 1 || limit > 128)
|
|
33
|
+
throw new Error("capture issue preparation requires a bounded standalone transaction");
|
|
34
|
+
if (finalize || stopped()) return { complete: false, stopped: true, checked: 0 };
|
|
35
|
+
if (this.complete()) return { complete: true, checked: 0 };
|
|
36
|
+
// Privacy failure must propagate; it cannot authorize opening inputs.
|
|
37
|
+
const ignore = loadIgnore(root);
|
|
38
|
+
return this.state.transaction(() => {
|
|
39
|
+
const row = this.current() ?? { runId: this.runId, scope: this.scope, retry: this.retry,
|
|
40
|
+
sources: this.sources, after: null, complete: false };
|
|
41
|
+
const page = this.state.db.prepare(`SELECT key,value FROM state WHERE kind='set-aside'
|
|
42
|
+
${row.after === null ? "" : "AND key>?"} ORDER BY key LIMIT ?`)
|
|
43
|
+
.all(...(row.after === null ? [limit] : [row.after, limit]));
|
|
44
|
+
let checked = 0;
|
|
45
|
+
for (const { key, value } of page) {
|
|
46
|
+
if (stopped()) break;
|
|
47
|
+
const issue = JSON.parse(value), fmt = SESSION_FORMATS[issue.source];
|
|
48
|
+
if (!formats.includes(issue.source) || typeof issue.path !== "string")
|
|
49
|
+
throw new Error("invalid capture issue during preparation");
|
|
50
|
+
// Preserve native public reconciliation for missing/ignored inputs,
|
|
51
|
+
// but do not clear another owner or an excluded format's evidence.
|
|
52
|
+
const ignored = fmt && issue.owner === this.queue.owner && this.sources.includes(issue.source) &&
|
|
53
|
+
ignore.ignores(issue.path, [basename(issue.path), fmt.sessionId(issue.path), ...(fmt.ignoreKeys?.(issue.path) ?? [])]);
|
|
54
|
+
if (this.retry) {
|
|
55
|
+
// The existing explicit all-input retry marks the root's issues,
|
|
56
|
+
// including other owners, without changing their source selection.
|
|
57
|
+
// Run identity stays stable across replacement and partial pages.
|
|
58
|
+
this.state.set("source-retry", key, this.runId);
|
|
59
|
+
}
|
|
60
|
+
if (ignored) this.state.delete("set-aside", key);
|
|
61
|
+
else if (this.retry) this.state.set("set-aside", key, { ...issue, retry: true });
|
|
62
|
+
if ((ignored || this.retry) && issue.owner === this.queue.owner && this.sources.includes(issue.source)) {
|
|
63
|
+
this.state.db.prepare("UPDATE capture_sources SET outcome='ready' WHERE scope=? AND source=? AND path=? AND outcome='set-aside'")
|
|
64
|
+
.run(this.scope, issue.source, issue.path);
|
|
65
|
+
}
|
|
66
|
+
row.after = key; checked++;
|
|
67
|
+
}
|
|
68
|
+
row.complete = checked === page.length && page.length < limit;
|
|
69
|
+
this.state.set("capture-issue-preparation", this.queue.owner, row);
|
|
70
|
+
return { complete: row.complete, stopped: stopped(), checked };
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
}
|
|
@@ -2,6 +2,7 @@ import { statSync } from "node:fs";
|
|
|
2
2
|
import { spawnSync } from "node:child_process";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
import { join } from "node:path";
|
|
5
|
+
import { validFailure, validFailureBacklog } from "./capture-failures.mjs";
|
|
5
6
|
// Feature landed separately on the two release lines; 23.0/23.1 lack it.
|
|
6
7
|
export function supportsReadOnlySqlite(version) {
|
|
7
8
|
const match = /^(\d+)\.(\d+)\.(\d+)$/.exec(version);
|
|
@@ -33,8 +34,20 @@ export function readIssueBacklog(db, { pending = false, validate = false } = {})
|
|
|
33
34
|
}
|
|
34
35
|
return { source, path, offset, reason, ...(line === undefined ? {} : { line }) };
|
|
35
36
|
});
|
|
37
|
+
// Both inventories share this read snapshot, but have independent samples.
|
|
38
|
+
// Private generation/admission fields never enter the public projection.
|
|
39
|
+
const failedCount = Number(db.prepare("SELECT count(*) AS n FROM state WHERE kind = 'capture-failed'").get().n);
|
|
40
|
+
const failed = db.prepare("SELECT value FROM state WHERE kind = 'capture-failed' ORDER BY key LIMIT 100").all()
|
|
41
|
+
.map(({ value }) => {
|
|
42
|
+
const { stream, offset, reason, attempts, nextRetryAt } = JSON.parse(value);
|
|
43
|
+
const row = { stream, offset, reason, attempts, nextRetryAt };
|
|
44
|
+
if (!validFailure(row)) throw new Error("malformed capture failure row");
|
|
45
|
+
return row;
|
|
46
|
+
});
|
|
47
|
+
const deferred = db.prepare("SELECT 1 AS found FROM state WHERE kind = 'capture-retry' LIMIT 1").get()?.found === 1;
|
|
36
48
|
db.exec("COMMIT");
|
|
37
|
-
return { pending: pending || count > 0, setAsideCount: count, setAside, truncated: count > sample.length
|
|
49
|
+
return { pending: pending || count > 0 || failedCount > 0 || deferred, setAsideCount: count, setAside, truncated: count > sample.length,
|
|
50
|
+
failedCount, failed, failedTruncated: failedCount > failed.length };
|
|
38
51
|
} catch (err) { db.exec("ROLLBACK"); throw err; }
|
|
39
52
|
}
|
|
40
53
|
|
|
@@ -59,6 +72,6 @@ export function readCurrentIssueBacklog(root) {
|
|
|
59
72
|
const b = response.backlog;
|
|
60
73
|
if (!b || typeof b.pending !== "boolean" || !Number.isSafeInteger(b.setAsideCount) || b.setAsideCount < 0 ||
|
|
61
74
|
!Array.isArray(b.setAside) || b.setAside.length !== Math.min(b.setAsideCount, 100) ||
|
|
62
|
-
b.truncated !== (b.setAsideCount > 100)) throw new Error("malformed capture database reader output");
|
|
75
|
+
b.truncated !== (b.setAsideCount > 100) || !validFailureBacklog(b)) throw new Error("malformed capture database reader output");
|
|
63
76
|
return b;
|
|
64
77
|
}
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
// initializing (owner-less) lock refuses the pass and names the holder and
|
|
10
10
|
// the operator recovery; it is never stolen.
|
|
11
11
|
//
|
|
12
|
-
// A lock whose recorded owner
|
|
12
|
+
// A lock whose recorded owner AND registered worker are DEAD on this host is reclaimed (a capture
|
|
13
13
|
// killed mid-pass by a hook or caller timeout runs no finally). Records
|
|
14
14
|
// without host predate host recording and live under this user's home, so
|
|
15
15
|
// they count as this host's (RECLAIM_HOSTLESS_RECORDS). Reclaimers
|
|
@@ -19,10 +19,12 @@
|
|
|
19
19
|
// meanwhile. A guard whose holder died is never removed (that would race
|
|
20
20
|
// exactly as removing the lock does); it is named with its recovery.
|
|
21
21
|
// Acquire never waits: it reclaims once or skips, and the next pass catches up.
|
|
22
|
-
//
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
|
|
22
|
+
// A supervisor publishes its idle child's identity before permitting work.
|
|
23
|
+
// The worker is part of the same lock, not a second independently reclaimed
|
|
24
|
+
// lock. A live/reused/unknown child PID prevents release and reclamation even
|
|
25
|
+
// when the supervisor died. Callers own child termination/reaping; this module
|
|
26
|
+
// never sends a terminating signal or infers a process group.
|
|
27
|
+
import { closeSync, constants, existsSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readSync, renameSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
|
|
26
28
|
import { randomBytes } from "node:crypto";
|
|
27
29
|
import { hostname } from "node:os";
|
|
28
30
|
import { join } from "node:path";
|
|
@@ -35,18 +37,46 @@ export function captureLockPath(root) { return join(root, ".capture.lock"); }
|
|
|
35
37
|
|
|
36
38
|
/** "alive" | "dead" | "unknown" for an owner pid ("unknown" = exists but not signalable). */
|
|
37
39
|
export function holderLiveness(pid) {
|
|
38
|
-
if (!Number.isInteger(pid) || pid <= 0) return "unknown";
|
|
39
|
-
try { process.kill(pid, 0); return "alive"; } catch (e) { return e.code === "
|
|
40
|
+
if (!Number.isInteger(pid) || pid <= 0 || pid > 2147483647) return "unknown";
|
|
41
|
+
try { process.kill(pid, 0); return "alive"; } catch (e) { return e.code === "ESRCH" ? "dead" : "unknown"; }
|
|
40
42
|
}
|
|
41
43
|
|
|
42
|
-
const
|
|
43
|
-
const
|
|
44
|
+
const sameFile = (a, b) => a.dev === b.dev && a.ino === b.ino && a.size === b.size && a.mtimeNs === b.mtimeNs && a.ctimeNs === b.ctimeNs;
|
|
45
|
+
const readJson = (path) => {
|
|
46
|
+
let fd;
|
|
47
|
+
try {
|
|
48
|
+
const before = lstatSync(path, { bigint: true });
|
|
49
|
+
if (!before.isFile() || before.isSymbolicLink() || before.size > 4096n) return undefined;
|
|
50
|
+
fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW);
|
|
51
|
+
const opened = fstatSync(fd, { bigint: true });
|
|
52
|
+
if (!sameFile(before, opened)) return undefined;
|
|
53
|
+
const bytes = Buffer.alloc(Number(opened.size)); let offset = 0;
|
|
54
|
+
while (offset < bytes.length) { const n = readSync(fd, bytes, offset, bytes.length - offset, offset); if (!n) return undefined; offset += n; }
|
|
55
|
+
if (!sameFile(opened, fstatSync(fd, { bigint: true })) || !sameFile(opened, lstatSync(path, { bigint: true }))) return undefined;
|
|
56
|
+
return JSON.parse(bytes.toString("utf8"));
|
|
57
|
+
} catch { return undefined; }
|
|
58
|
+
finally { if (fd !== undefined) closeSync(fd); }
|
|
59
|
+
};
|
|
60
|
+
const readOwner = (dir) => {
|
|
61
|
+
try { const st = lstatSync(dir); return st.isDirectory() && !st.isSymbolicLink() ? readJson(join(dir, "owner.json")) : undefined; }
|
|
62
|
+
catch { return undefined; }
|
|
63
|
+
};
|
|
64
|
+
export const readCaptureLockOwner = root => readOwner(captureLockPath(root));
|
|
65
|
+
|
|
66
|
+
function validPid(pid) { return Number.isInteger(pid) && pid > 0 && pid <= 2147483647; }
|
|
67
|
+
function workerDead(owner, { host, liveness }) {
|
|
68
|
+
if (!Object.hasOwn(owner, "worker")) return true;
|
|
69
|
+
const worker = owner.worker;
|
|
70
|
+
return validPid(worker?.pid) && worker.pid !== owner.pid && worker.host === host && typeof worker.nonce === "string" &&
|
|
71
|
+
worker.nonce.length > 0 && worker.nonce.length <= 128 && typeof worker.startedAt === "string" &&
|
|
72
|
+
Number.isFinite(Date.parse(worker.startedAt)) && liveness(worker.pid) === "dead";
|
|
73
|
+
}
|
|
44
74
|
|
|
45
75
|
/** Whether `owner` is a record of this host whose process is dead: the only lock this module reclaims. */
|
|
46
76
|
function deadHere(owner, { host, liveness, reclaimHostless }) {
|
|
47
|
-
if (!owner || !
|
|
77
|
+
if (!owner || !validPid(owner.pid)) return false;
|
|
48
78
|
const here = owner.host === undefined ? reclaimHostless : owner.host === host;
|
|
49
|
-
return here && liveness(owner.pid) === "dead";
|
|
79
|
+
return here && liveness(owner.pid) === "dead" && workerDead(owner, { host, liveness });
|
|
50
80
|
}
|
|
51
81
|
|
|
52
82
|
/** Remove the lock `dir` held by the dead `owner`, serialized by the guard `<dir>.reclaim`. → { removed }
|
|
@@ -60,7 +90,7 @@ function reclaimDeadLock(dir, owner, me, opts) {
|
|
|
60
90
|
const g = readJson(guard);
|
|
61
91
|
return g && deadHere(g, opts) ? { abandoned: { guard, pid: g.pid } } : { removed: false };
|
|
62
92
|
}
|
|
63
|
-
const same = (o) => o && o.pid === owner.pid && o.nonce === owner.nonce && o.startedAt === owner.startedAt;
|
|
93
|
+
const same = (o) => o && o.pid === owner.pid && o.nonce === owner.nonce && o.startedAt === owner.startedAt && JSON.stringify(o.worker) === JSON.stringify(owner.worker);
|
|
64
94
|
let removed = false;
|
|
65
95
|
try {
|
|
66
96
|
const now = readOwner(dir);
|
|
@@ -79,6 +109,7 @@ export function shellQuote(s) { return "'" + String(s).replace(/'/g, "'\\''") +
|
|
|
79
109
|
/** The operator's recovery line for a lock that is not ours. */
|
|
80
110
|
export function recoveryInstruction(dir, owner, liveness) {
|
|
81
111
|
const remove = `rm -r -- ${shellQuote(dir)}`;
|
|
112
|
+
if (owner && Object.hasOwn(owner, "worker")) return `${dir} records supervisor pid ${owner.pid} (${liveness}) and worker pid ${owner.worker?.pid ?? "unknown"}; wait for both to finish. If interrupted, stop capture triggers and verify BOTH processes are gone before removing the lock with: ${remove} and rerun`;
|
|
82
113
|
if (!owner?.pid) return `${dir} is held by a pass that has not written its owner record yet (initializing, or killed before it could); stop capture triggers (hooks, launchd), verify no capture process is running (pgrep -f capture.mjs), then remove the lock with: ${remove} and rerun`;
|
|
83
114
|
const who = `pid ${owner.pid} (started ${owner.startedAt || "?"}, now ${liveness})`;
|
|
84
115
|
if (liveness === "alive") return `${dir} is held by ${who}; let it finish, the next pass catches up`;
|
|
@@ -103,7 +134,7 @@ export function recoveryInstruction(dir, owner, liveness) {
|
|
|
103
134
|
* saying what happened. And `release()` never throws: it answers
|
|
104
135
|
* `{ released: true }` only when the lock is verifiably gone, otherwise
|
|
105
136
|
* `{ released: false, reason: "gone" | "unknown-owner" | "not-owner" |
|
|
106
|
-
* "remove-failed", ... }` with the actual observation, never a guess.
|
|
137
|
+
* "remove-failed" | "worker-active", ... }` with the actual observation, never a guess.
|
|
107
138
|
*
|
|
108
139
|
* The owner record carries a per-acquisition nonce, so a release kept from
|
|
109
140
|
* an earlier acquisition cannot erase a later one by the same pid (an
|
|
@@ -111,13 +142,14 @@ export function recoveryInstruction(dir, owner, liveness) {
|
|
|
111
142
|
*
|
|
112
143
|
* `io` exists for fault injection in tests only. */
|
|
113
144
|
export function acquireCaptureLock(root, { now = Date.now, pid = process.pid, liveness = holderLiveness, host = hostname(), reclaimHostless = RECLAIM_HOSTLESS_RECORDS, io = {} } = {}) {
|
|
114
|
-
|
|
145
|
+
if (!validPid(pid)) throw new Error("capture lock owner must have a positive process ID");
|
|
146
|
+
const fs = { writeFileSync, rmSync, openSync, closeSync, lstatSync, fsyncSync, renameSync, ...io };
|
|
115
147
|
const dir = captureLockPath(root);
|
|
116
148
|
const nonce = randomBytes(8).toString("hex");
|
|
117
149
|
mkdirSync(root, { recursive: true }); // the store creates the root lazily; the lock may come first
|
|
118
150
|
let reclaimed;
|
|
119
151
|
for (let attempt = 0; ; attempt++) {
|
|
120
|
-
try { mkdirSync(dir); break; }
|
|
152
|
+
try { mkdirSync(dir, { mode: 0o700 }); break; }
|
|
121
153
|
catch (e) {
|
|
122
154
|
if (e.code !== "EEXIST") throw e;
|
|
123
155
|
const owner = readOwner(dir);
|
|
@@ -148,7 +180,7 @@ export function acquireCaptureLock(root, { now = Date.now, pid = process.pid, li
|
|
|
148
180
|
// a record-less replacement look like the directory we created.
|
|
149
181
|
directoryFd = fs.openSync(dir, "r");
|
|
150
182
|
identity = fstatSync(directoryFd);
|
|
151
|
-
fs.writeFileSync(join(dir, "owner.json"), JSON.stringify({ pid, nonce, startedAt: new Date(now()).toISOString(), host }));
|
|
183
|
+
fs.writeFileSync(join(dir, "owner.json"), JSON.stringify({ pid, nonce, startedAt: new Date(now()).toISOString(), host }), { mode: 0o600 });
|
|
152
184
|
} catch (err) {
|
|
153
185
|
// Ownership was proven by the mkdir, not by the moment of cleanup: the
|
|
154
186
|
// directory is removed only if it is still ours (same inode) and holds
|
|
@@ -175,12 +207,66 @@ export function acquireCaptureLock(root, { now = Date.now, pid = process.pid, li
|
|
|
175
207
|
}
|
|
176
208
|
return {
|
|
177
209
|
path: dir,
|
|
210
|
+
nonce,
|
|
178
211
|
...(reclaimed ? { reclaimed } : {}),
|
|
212
|
+
// The child must remain idle until this durable publication succeeds.
|
|
213
|
+
// A dead supervisor's lock cannot then be reclaimed while its worker lives.
|
|
214
|
+
registerWorker(worker, { previous } = {}) {
|
|
215
|
+
if (!validPid(worker?.pid) || worker.pid === pid || typeof worker.nonce !== "string" || !worker.nonce || worker.nonce.length > 128 || typeof worker.startedAt !== "string" || !Number.isFinite(Date.parse(worker.startedAt))) throw new Error("invalid capture worker identity");
|
|
216
|
+
const current = readOwner(dir), st = fs.lstatSync(dir);
|
|
217
|
+
if (st.dev !== identity.dev || st.ino !== identity.ino || current?.pid !== pid || current?.nonce !== nonce) throw new Error("capture lock changed before worker registration");
|
|
218
|
+
// Continuation never drops the root lock between children. Replace only
|
|
219
|
+
// the caller's exact previous publication, after positive death proof;
|
|
220
|
+
// reused PIDs and uncertain liveness preserve the old lock unchanged.
|
|
221
|
+
if (Object.hasOwn(current, "worker")) {
|
|
222
|
+
if (!previous || current.worker?.pid !== previous.pid || current.worker?.nonce !== previous.nonce ||
|
|
223
|
+
current.worker?.startedAt !== previous.startedAt || !workerDead(current, { host, liveness })) {
|
|
224
|
+
throw new Error("previous capture worker is not proven dead and unchanged");
|
|
225
|
+
}
|
|
226
|
+
} else if (previous) throw new Error("capture lock changed before worker continuation");
|
|
227
|
+
const temp = join(dir, `worker-${nonce}.tmp`); let fd, dirFd, rootFd, temporary, published = false;
|
|
228
|
+
try {
|
|
229
|
+
dirFd = fs.openSync(dir, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
230
|
+
const pinned = fstatSync(dirFd);
|
|
231
|
+
if (pinned.dev !== identity.dev || pinned.ino !== identity.ino) throw new Error("capture lock changed before worker registration");
|
|
232
|
+
fd = fs.openSync(temp, "wx", 0o600);
|
|
233
|
+
fs.writeFileSync(fd, JSON.stringify({ ...current, worker: { pid: worker.pid, nonce: worker.nonce, startedAt: worker.startedAt, host } }));
|
|
234
|
+
fs.fsyncSync(fd); temporary = fstatSync(fd, { bigint: true });
|
|
235
|
+
const again = readOwner(dir), named = fs.lstatSync(dir);
|
|
236
|
+
if (named.dev !== identity.dev || named.ino !== identity.ino || JSON.stringify(again) !== JSON.stringify(current) || !sameFile(temporary, fs.lstatSync(temp, { bigint: true }))) throw new Error("capture lock changed during worker registration");
|
|
237
|
+
fs.renameSync(temp, join(dir, "owner.json"));
|
|
238
|
+
published = true;
|
|
239
|
+
// Publish file bytes, their rename, and the lock directory's parent
|
|
240
|
+
// entry before GO. A failure is not permission to start the worker.
|
|
241
|
+
fs.fsyncSync(dirFd);
|
|
242
|
+
rootFd = fs.openSync(root, "r"); fs.fsyncSync(rootFd);
|
|
243
|
+
} finally {
|
|
244
|
+
// Never remove a path another caller replaced, even after a failed
|
|
245
|
+
// registration. An unpublished/unverifiable file is conservatively kept.
|
|
246
|
+
if (!published && temporary) {
|
|
247
|
+
try {
|
|
248
|
+
const named = fs.lstatSync(dir);
|
|
249
|
+
if (named.dev === identity.dev && named.ino === identity.ino && sameFile(temporary, fs.lstatSync(temp, { bigint: true }))) unlinkSync(temp);
|
|
250
|
+
} catch { /* absent or unknown: preserve */ }
|
|
251
|
+
}
|
|
252
|
+
if (fd !== undefined) fs.closeSync(fd);
|
|
253
|
+
if (rootFd !== undefined) fs.closeSync(rootFd);
|
|
254
|
+
if (dirFd !== undefined) fs.closeSync(dirFd);
|
|
255
|
+
}
|
|
256
|
+
},
|
|
179
257
|
release: () => {
|
|
180
258
|
if (!existsSync(dir)) return { released: false, reason: "gone" };
|
|
181
259
|
const cur = readOwner(dir);
|
|
182
260
|
if (!cur) return { released: false, reason: "unknown-owner", recovery: recoveryInstruction(dir, undefined, "unknown") };
|
|
183
261
|
if (cur.pid !== pid || cur.nonce !== nonce) return { released: false, reason: "not-owner", owner: cur };
|
|
262
|
+
try {
|
|
263
|
+
const named = fs.lstatSync(dir);
|
|
264
|
+
if (!named.isDirectory() || named.dev !== identity.dev || named.ino !== identity.ino) return { released: false, reason: "not-owner", owner: cur };
|
|
265
|
+
} catch (err) {
|
|
266
|
+
return { released: false, reason: "remove-failed", error: err.message,
|
|
267
|
+
recovery: "could not verify capture lock identity before removal; inspect the filesystem error and rerun capture" };
|
|
268
|
+
}
|
|
269
|
+
if (!workerDead(cur, { host, liveness })) return { released: false, reason: "worker-active", recovery: recoveryInstruction(dir, cur, "alive") };
|
|
184
270
|
let error;
|
|
185
271
|
try { fs.rmSync(dir, { recursive: true, force: true }); } catch (e) { error = e; }
|
|
186
272
|
let remains;
|
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
// Private bounded native command work. Public captureSessions and exact
|
|
2
|
+
// receipts retain their synchronous complete-pass contracts. The scheduler
|
|
3
|
+
// supplies a stream lock/transaction and advances to another input after this
|
|
4
|
+
// unit; a partial result is never a completed sweep.
|
|
5
|
+
import { closeSync, constants, fstatSync, openSync } from "node:fs";
|
|
6
|
+
import { basename } from "node:path";
|
|
7
|
+
import { randomUUID } from "node:crypto";
|
|
8
|
+
import { captureContext } from "./capture-context.mjs";
|
|
9
|
+
import { CAPTURE_POLICY } from "./capture-budget.mjs";
|
|
10
|
+
import { completeNativeLineDigests, completeNativeLines, eventTurnCore } from "./capture-cc.mjs";
|
|
11
|
+
import { finishTurn } from "./canonical.mjs";
|
|
12
|
+
import { SESSION_FORMATS } from "./formats.mjs";
|
|
13
|
+
import { loadIgnore } from "./ignore.mjs";
|
|
14
|
+
import { fileVersion, sameVersion, statVersion } from "./derived-state.mjs";
|
|
15
|
+
import { assertProtectedDescriptor, digest, sourceChunks } from "./session-snapshot.mjs";
|
|
16
|
+
import { guardCapturedPath } from "./native-history.mjs";
|
|
17
|
+
import { RecordSizeError } from "./file-lines.mjs";
|
|
18
|
+
import { advanceRecoverySnapshot } from "./capture-recovery-snapshot.mjs";
|
|
19
|
+
import { advanceTranscriptPrefix } from "./transcript-prefix.mjs";
|
|
20
|
+
import { prepareRetainedCaptureCleanup } from "./capture-retained-scratch.mjs";
|
|
21
|
+
import { smallCommandSnapshot, SMALL_SNAPSHOT_BYTES } from "./capture-small-snapshot.mjs";
|
|
22
|
+
|
|
23
|
+
const nonnegative = n => Number.isSafeInteger(n) && n >= 0;
|
|
24
|
+
const validCursor = (value, size) => value && nonnegative(value.bytes) && value.bytes <= size &&
|
|
25
|
+
nonnegative(value.line) && typeof value.lastTs === "string";
|
|
26
|
+
|
|
27
|
+
export function captureNativeUnit(store, { owner, path, format = "cc", capturedPi, ignore,
|
|
28
|
+
start, maxRecords = 2000, maxBytes = 8388608, maxRecordBytes = CAPTURE_POLICY.maxRecordBytes,
|
|
29
|
+
stopped = () => false, work } = {}) {
|
|
30
|
+
const fmt = SESSION_FORMATS[format];
|
|
31
|
+
if (!fmt) throw new Error("unknown native command format");
|
|
32
|
+
const sessionId = fmt.sessionId(path), stream = `${owner}~${fmt.source}.${sessionId}`, key = `${stream}:${path}`;
|
|
33
|
+
const context = captureContext(store), state = context?.state;
|
|
34
|
+
if (!state?.depth || !context.held.has(stream)) throw new Error("native command unit requires its stream work group");
|
|
35
|
+
if (!Number.isSafeInteger(maxRecords) || maxRecords < 1 || maxRecords > 2000 ||
|
|
36
|
+
!Number.isSafeInteger(maxBytes) || maxBytes < 1 || maxBytes > context.batchBytes ||
|
|
37
|
+
!Number.isSafeInteger(maxRecordBytes) || maxRecordBytes < 1 || maxRecordBytes > CAPTURE_POLICY.maxRecordBytes) {
|
|
38
|
+
throw new Error("invalid native command unit budget");
|
|
39
|
+
}
|
|
40
|
+
const result = { stream, key, appended: 0, bytesCommitted: 0, complete: false, pending: false };
|
|
41
|
+
if ((ignore ?? loadIgnore(store.root)).ignores(path, [basename(path), sessionId, ...(fmt.ignoreKeys?.(path) ?? [])])) {
|
|
42
|
+
state.delete("set-aside", key);
|
|
43
|
+
// Ignore is evaluated before custody/source access. An extant retained
|
|
44
|
+
// allocation is housekeeping, not authority to reopen an excluded source.
|
|
45
|
+
return { ...result, complete: true, ignored: true };
|
|
46
|
+
}
|
|
47
|
+
const blocked = state.get("set-aside", key);
|
|
48
|
+
if (blocked && !blocked.retry) return { ...result, setAside: blocked };
|
|
49
|
+
if (capturedPi && fmt.source !== "pi") throw new Error("protected capture proof requires pi format");
|
|
50
|
+
guardCapturedPath(path, capturedPi);
|
|
51
|
+
const fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK | (capturedPi ? constants.O_NOFOLLOW : 0));
|
|
52
|
+
let staged;
|
|
53
|
+
const stagedChunks = function* (from, end) {
|
|
54
|
+
for (const bytes of staged.snapshot.chunks(from, end)) {
|
|
55
|
+
if (work) work.stagedReadBytes = (work.stagedReadBytes ?? 0) + bytes.length;
|
|
56
|
+
yield bytes;
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
const stagedWitness = offset => {
|
|
60
|
+
if (work) work.stagedProbeBytes = (work.stagedProbeBytes ?? 0) + Math.min(4096, offset);
|
|
61
|
+
return digest(Buffer.concat([...staged.snapshot.chunks(Math.max(0, offset - 4096), offset)]));
|
|
62
|
+
};
|
|
63
|
+
try {
|
|
64
|
+
const observe = () => {
|
|
65
|
+
assertProtectedDescriptor(fd, path, capturedPi);
|
|
66
|
+
if (!fstatSync(fd).isFile()) throw new Error("native source is not a regular file");
|
|
67
|
+
const value = statVersion(fstatSync(fd, { bigint: true }));
|
|
68
|
+
if (!sameVersion(value, fileVersion(path))) throw new Error("native source descriptor differs from its named file");
|
|
69
|
+
return value;
|
|
70
|
+
};
|
|
71
|
+
const version = observe();
|
|
72
|
+
const chunks = function* (from, end) {
|
|
73
|
+
for (const bytes of sourceChunks(fd, from, end, path, capturedPi)) {
|
|
74
|
+
// Sampling between reads does not turn an incomplete line into a
|
|
75
|
+
// checkpoint. The outer unit stops at a complete-record boundary.
|
|
76
|
+
stopped();
|
|
77
|
+
if (work) work.sourceBytes = (work.sourceBytes ?? 0) + bytes.length;
|
|
78
|
+
yield bytes;
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
const witness = (from, read = chunks) => digest(Buffer.concat([...read(Math.max(0, from - 4096), from)]));
|
|
82
|
+
const probe = offset => {
|
|
83
|
+
const read = function* (from, end) {
|
|
84
|
+
for (const bytes of sourceChunks(fd, from, end, path, capturedPi)) {
|
|
85
|
+
if (work) work.sourceProbeBytes = (work.sourceProbeBytes ?? 0) + bytes.length;
|
|
86
|
+
yield bytes;
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
return witness(offset, read);
|
|
90
|
+
};
|
|
91
|
+
const generation = JSON.stringify([state.get("reset", `${owner}:${fmt.source}`) ?? "", state.get("source-retry", key) ?? ""]);
|
|
92
|
+
// Ordinary public cursors use the reset value directly. Command cursors
|
|
93
|
+
// retain it too; the private job additionally binds an explicit retry.
|
|
94
|
+
const reset = state.get("reset", `${owner}:${fmt.source}`) ?? "";
|
|
95
|
+
const journalVersion = fileVersion(store.journalPath(stream));
|
|
96
|
+
let cursor = state.get(fmt.source, key);
|
|
97
|
+
const matchesSource = value => value?.version && (sameVersion(value.version, version) ||
|
|
98
|
+
value.version.dev === version.dev && value.version.ino === version.ino && version.size > value.version.size &&
|
|
99
|
+
value.bytes <= version.size && probe(value.bytes) === value.witness);
|
|
100
|
+
let job = state.get("native-unit", key);
|
|
101
|
+
const jobValid = job && job.generation === generation && sameVersion(job.journalVersion, journalVersion) &&
|
|
102
|
+
validCursor(job, version.size) && nonnegative(job.end) && job.bytes <= job.end && job.end <= version.size &&
|
|
103
|
+
matchesSource(job);
|
|
104
|
+
if (!jobValid) { state.delete("native-unit", key); job = null; }
|
|
105
|
+
const cursorValid = validCursor(cursor, version.size) && cursor.generation === reset &&
|
|
106
|
+
sameVersion(cursor.journalVersion, journalVersion) && matchesSource(cursor) && !blocked;
|
|
107
|
+
const recover = !job && !cursorValid && (journalVersion?.size ?? 0) > 0;
|
|
108
|
+
const allocation = state.get("recovery-allocation", key);
|
|
109
|
+
if (allocation?.phase === "deleting") {
|
|
110
|
+
prepareRetainedCaptureCleanup(store, stream, key, start);
|
|
111
|
+
return { ...result, cleanup: true, phase: "cleanup" };
|
|
112
|
+
}
|
|
113
|
+
if (job?.waiting && job.end === version.size && sameVersion(job.version, version)) {
|
|
114
|
+
return { ...result, pending: true, phase: job.scan ? "timestamp" : "capture", reason: job.waiting };
|
|
115
|
+
}
|
|
116
|
+
if (!job && !allocation && cursorValid && cursor.bytes === version.size) {
|
|
117
|
+
return { ...result, complete: true, unchanged: true, phase: "captured" };
|
|
118
|
+
}
|
|
119
|
+
// Journal validation can itself need several units. Never let appendBatch
|
|
120
|
+
// fall back to an unbounded validation inside the first source append.
|
|
121
|
+
const validated = store.validateJournal(stream, state, { maxRecords, maxBytes, stopped, work: work?.journal });
|
|
122
|
+
if (!validated.complete) return { ...result, phase: "journal" };
|
|
123
|
+
const unitWitness = probe(version.size);
|
|
124
|
+
let read = chunks, verify = () => {
|
|
125
|
+
const after = observe();
|
|
126
|
+
if (!sameVersion(version, after) && !(version.dev === after.dev && version.ino === after.ino &&
|
|
127
|
+
after.size > version.size && probe(version.size) === unitWitness)) throw new Error("native source changed during unit");
|
|
128
|
+
};
|
|
129
|
+
if (recover || allocation || job?.snapshot) {
|
|
130
|
+
let small = state.get("small-stage", key);
|
|
131
|
+
if (!small || small.generation !== generation || !sameVersion(small.journalVersion, journalVersion) ||
|
|
132
|
+
!nonnegative(small.size) || small.size > SMALL_SNAPSHOT_BYTES || small.size > version.size ||
|
|
133
|
+
typeof small.id !== "string" || !matchesSource({ ...small, bytes: small.size })) small = null;
|
|
134
|
+
const end = job?.end ?? small?.size ?? version.size;
|
|
135
|
+
// A killed unit has durable admission/retry evidence but cannot have
|
|
136
|
+
// published an in-memory small-copy outcome. Its next due single-stream
|
|
137
|
+
// retry uses retained copying rather than repeating that overhead forever.
|
|
138
|
+
const retain = allocation || state.get("native-retain", key) || context.admission?.retry || end > SMALL_SNAPSHOT_BYTES;
|
|
139
|
+
if (retain) {
|
|
140
|
+
staged = advanceRecoverySnapshot(store, stream, key, path, start, { capturedPi, generation, fixedEnd: end, maxBytes, stopped, work });
|
|
141
|
+
} else {
|
|
142
|
+
// This observation fixes the sweep end even if copying cannot finish.
|
|
143
|
+
// It is NOT a digest/proof checkpoint: reuse below always requires a
|
|
144
|
+
// complete fresh copy/hash before advanceTranscriptPrefix sees its id.
|
|
145
|
+
small ??= { id: randomUUID(), size: end };
|
|
146
|
+
small = { ...small, generation, journalVersion, version, witness: probe(end) };
|
|
147
|
+
state.set("small-stage", key, small);
|
|
148
|
+
staged = smallCommandSnapshot(store.root, fd, path, end, { capturedPi, stopped, work });
|
|
149
|
+
if (staged.complete) {
|
|
150
|
+
// The nonce binds source observation/reset history; digest+size bind
|
|
151
|
+
// bytes. Equal content under a changed identity still restarts proof.
|
|
152
|
+
staged.snapshot.id = `${small.id}:${staged.snapshot.id}`;
|
|
153
|
+
}
|
|
154
|
+
if (!staged.complete || stopped()) {
|
|
155
|
+
state.set("native-retain", key, true);
|
|
156
|
+
state.delete("native-unit", key);
|
|
157
|
+
return { ...result, phase: "snapshot", retainedFallback: true };
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
if (!staged.complete) {
|
|
161
|
+
if (staged.cleanup) state.delete("native-unit", key);
|
|
162
|
+
return { ...result, ...staged, phase: staged.phase ?? "snapshot" };
|
|
163
|
+
}
|
|
164
|
+
if (!job || job.snapshot !== staged.snapshot.id) {
|
|
165
|
+
const priorProof = state.get("prefix-proof", key);
|
|
166
|
+
// The captured prefix is proved by streamed per-line digests, so a
|
|
167
|
+
// record captured before the raw-record limit existed is compared, not
|
|
168
|
+
// refused. The limit applies to capture beyond the proven prefix.
|
|
169
|
+
const proof = advanceTranscriptPrefix(store, stream, { key, snapshot: staged.snapshot, maxRecords, maxBytes, stopped,
|
|
170
|
+
work: work?.prefix, lines: from => completeNativeLineDigests(stagedChunks(from), {}) });
|
|
171
|
+
staged.verify();
|
|
172
|
+
if (proof.conflict) {
|
|
173
|
+
// Journals preserve session/line, not the path that first supplied
|
|
174
|
+
// those bytes. A mismatch can also be a rewrite of this same file;
|
|
175
|
+
// neither a cursor nor today's discovery proves a second source.
|
|
176
|
+
const reason = `${proof.conflict.reason}: refused input ${JSON.stringify(path)} conflicts with stream ${JSON.stringify(stream)} at native line ${proof.conflict.line}, source byte ${proof.conflict.offset}; establishing source path unavailable`;
|
|
177
|
+
const row = { owner, source: fmt.source, path, ...proof.conflict, reason, retry: false };
|
|
178
|
+
state.set("set-aside", key, row);
|
|
179
|
+
const cleanup = !staged.small;
|
|
180
|
+
staged.close(); staged = null;
|
|
181
|
+
state.delete("native-unit", key); state.delete("small-stage", key);
|
|
182
|
+
if (cleanup) prepareRetainedCaptureCleanup(store, stream, key, start);
|
|
183
|
+
return { ...result, setAside: row, cleanup };
|
|
184
|
+
}
|
|
185
|
+
if (!proof.complete) {
|
|
186
|
+
// A full restage that leaves no allowance for proof progress must not
|
|
187
|
+
// recur indefinitely. One committed switch selects resumable copying.
|
|
188
|
+
if (staged.small && stopped() && priorProof &&
|
|
189
|
+
["phase", "journalBytes", "bytes", "line"].every(k => proof[k] === priorProof[k])) {
|
|
190
|
+
state.set("native-retain", key, true);
|
|
191
|
+
}
|
|
192
|
+
return { ...result, phase: "prefix" };
|
|
193
|
+
}
|
|
194
|
+
job = { bytes: proof.bytes, line: proof.line, lastTs: proof.lastTs, end: staged.snapshot.size,
|
|
195
|
+
snapshot: staged.snapshot.id, small: Boolean(staged.small), generation, journalVersion, version, witness: stagedWitness(proof.bytes) };
|
|
196
|
+
state.delete("set-aside", key);
|
|
197
|
+
}
|
|
198
|
+
} else if (!job) {
|
|
199
|
+
cursor = cursorValid ? cursor : { bytes: 0, line: 0, lastTs: "" };
|
|
200
|
+
job = { bytes: cursor.bytes, line: cursor.line, lastTs: cursor.lastTs, end: version.size,
|
|
201
|
+
generation, journalVersion, version, witness: probe(cursor.bytes) };
|
|
202
|
+
}
|
|
203
|
+
if (staged) { read = stagedChunks; verify = staged.verify; }
|
|
204
|
+
// A raw record over the limit is set aside at its exact start, after the
|
|
205
|
+
// complete records before it were committed. Later records wait for an
|
|
206
|
+
// edited source or an ignore of the whole input. A staged job's copy and
|
|
207
|
+
// fixed end retire with it; a retry reproves and returns here.
|
|
208
|
+
const setAsideOversized = offset => {
|
|
209
|
+
state.delete("native-unit", key); state.delete("native-retain", key); state.delete("small-stage", key);
|
|
210
|
+
if (job.snapshot && !job.small) {
|
|
211
|
+
staged.close(); staged = null;
|
|
212
|
+
prepareRetainedCaptureCleanup(store, stream, key, start);
|
|
213
|
+
result.cleanup = true;
|
|
214
|
+
}
|
|
215
|
+
const row = { owner, source: fmt.source, path, offset, reason: "record-too-large", retry: false };
|
|
216
|
+
state.set("set-aside", key, row);
|
|
217
|
+
return { ...result, setAside: row };
|
|
218
|
+
};
|
|
219
|
+
if (job.waiting) {
|
|
220
|
+
if (job.end === version.size) return { ...result, pending: true, phase: job.scan ? "timestamp" : "capture", reason: job.waiting };
|
|
221
|
+
if (job.snapshot) {
|
|
222
|
+
staged.close(); staged = null;
|
|
223
|
+
if (!job.small) prepareRetainedCaptureCleanup(store, stream, key, start);
|
|
224
|
+
state.delete("native-unit", key); state.delete("small-stage", key);
|
|
225
|
+
return { ...result, cleanup: !job.small, phase: "cleanup" };
|
|
226
|
+
}
|
|
227
|
+
// An exhausted fixed end was pending, so the next source growth is a
|
|
228
|
+
// new bounded sweep. Keep its timestamp frontier, not its old EOF.
|
|
229
|
+
job.end = version.size; delete job.waiting;
|
|
230
|
+
}
|
|
231
|
+
const checkpoint = () => {
|
|
232
|
+
verify();
|
|
233
|
+
job.version = staged && !staged.small ? state.get("recovery-stage", key).observedVersion : version;
|
|
234
|
+
job.witness = staged ? stagedWitness(job.bytes) : probe(job.bytes);
|
|
235
|
+
job.journalVersion = fileVersion(store.journalPath(stream));
|
|
236
|
+
state.set("native-unit", key, job);
|
|
237
|
+
state.set(fmt.source, key, { bytes: job.bytes, line: job.line, lastTs: job.lastTs,
|
|
238
|
+
version: job.version, witness: job.witness, generation: reset, journalVersion: job.journalVersion });
|
|
239
|
+
if (job.small) {
|
|
240
|
+
state.set("small-stage", key, { ...state.get("small-stage", key), journalVersion: job.journalVersion });
|
|
241
|
+
} else if (job.snapshot) {
|
|
242
|
+
// Only our successful fsynced append under this stream lock can carry
|
|
243
|
+
// the proven fixed snapshot into the next exact journal generation.
|
|
244
|
+
const stage = state.get("recovery-stage", key);
|
|
245
|
+
if (stage?.allocation !== job.snapshot) throw new Error("native unit lost its staged proof");
|
|
246
|
+
state.set("recovery-stage", key, { ...stage, journalVersion: job.journalVersion });
|
|
247
|
+
}
|
|
248
|
+
};
|
|
249
|
+
let used = 0, records = 0;
|
|
250
|
+
const stop = () => records >= maxRecords || used >= maxBytes || stopped();
|
|
251
|
+
const walk = {};
|
|
252
|
+
const lines = from => completeNativeLines(read(from, job.end), walk, { maxBytes: maxRecordBytes });
|
|
253
|
+
let scanStart = job.bytes;
|
|
254
|
+
try {
|
|
255
|
+
if (!job.lastTs && job.bytes < job.end) {
|
|
256
|
+
const scan = job.scan ?? { bytes: job.bytes, line: job.line };
|
|
257
|
+
if (!validCursor({ ...scan, lastTs: "" }, job.end) || scan.bytes < job.bytes) throw new Error("invalid native timestamp scan");
|
|
258
|
+
scanStart = scan.bytes;
|
|
259
|
+
const iterator = lines(scan.bytes);
|
|
260
|
+
try {
|
|
261
|
+
while (!stop()) {
|
|
262
|
+
const next = iterator.next(); if (next.done) break;
|
|
263
|
+
const line = next.value; scan.bytes += line.bytes; scan.line++; used += line.bytes; records++;
|
|
264
|
+
if (line.ts) { job.lastTs = line.ts; delete job.scan; break; }
|
|
265
|
+
}
|
|
266
|
+
} finally { iterator.return(); }
|
|
267
|
+
if (!job.lastTs) {
|
|
268
|
+
job.scan = scan;
|
|
269
|
+
if (scan.bytes === job.end || walk.reason) job.waiting = walk.reason ?? "unstamped";
|
|
270
|
+
checkpoint();
|
|
271
|
+
// No stamp yet is pending, not a successful empty capture. Preserve
|
|
272
|
+
// the scan frontier so later growth does not rescan this prefix.
|
|
273
|
+
return { ...result, phase: "timestamp", pending: scan.bytes === job.end || !!walk.reason, reason: walk.reason ?? "unstamped" };
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
// Timestamp search is work too; it cannot be followed by another full
|
|
277
|
+
// batch after exhausting this unit's allowance.
|
|
278
|
+
const turns = [], from = job.bytes, iterator = lines(from);
|
|
279
|
+
scanStart = from;
|
|
280
|
+
let oversized = null;
|
|
281
|
+
try {
|
|
282
|
+
while (!stop()) {
|
|
283
|
+
const next = iterator.next(); if (next.done) break;
|
|
284
|
+
const line = next.value;
|
|
285
|
+
if (turns.length && used + line.bytes > maxBytes) break;
|
|
286
|
+
if (line.ts) job.lastTs = line.ts;
|
|
287
|
+
job.bytes += line.bytes; job.line++; used += line.bytes; records++;
|
|
288
|
+
turns.push(finishTurn(eventTurnCore({ owner, source: fmt.source, sessionId, line: line.text, lineNo: job.line, ts: job.lastTs })));
|
|
289
|
+
}
|
|
290
|
+
} catch (error) {
|
|
291
|
+
if (!(error instanceof RecordSizeError)) throw error;
|
|
292
|
+
// The raw record was rejected before allocation/projection. Every
|
|
293
|
+
// complete stamped line before it is still processable: append and
|
|
294
|
+
// checkpoint them through the ordinary commit below, then set aside.
|
|
295
|
+
oversized = from + error.offset;
|
|
296
|
+
if (oversized !== job.bytes) throw new Error("native oversized record does not follow the consumed lines");
|
|
297
|
+
} finally { iterator.return(); }
|
|
298
|
+
verify();
|
|
299
|
+
if (turns.length) result.appended = store.appendUniqueBatch(stream, turns);
|
|
300
|
+
result.bytesCommitted = job.bytes - from;
|
|
301
|
+
if (walk.reason) job.waiting = walk.reason;
|
|
302
|
+
checkpoint();
|
|
303
|
+
if (oversized !== null) return setAsideOversized(oversized);
|
|
304
|
+
if (job.small && job.bytes < job.end && !records && stopped()) state.set("native-retain", key, true);
|
|
305
|
+
if (job.bytes === job.end) {
|
|
306
|
+
state.delete("native-unit", key); state.delete("native-retain", key); state.delete("small-stage", key);
|
|
307
|
+
if (job.snapshot && !job.small) {
|
|
308
|
+
staged.close(); staged = null;
|
|
309
|
+
prepareRetainedCaptureCleanup(store, stream, key, start);
|
|
310
|
+
result.cleanup = true;
|
|
311
|
+
}
|
|
312
|
+
return { ...result, complete: true, phase: "captured" };
|
|
313
|
+
}
|
|
314
|
+
return { ...result, phase: "capture", pending: !!walk.reason, reason: walk.reason ?? null };
|
|
315
|
+
} catch (error) {
|
|
316
|
+
if (!(error instanceof RecordSizeError)) throw error;
|
|
317
|
+
// The offending raw record was rejected before allocation/projection
|
|
318
|
+
// while searching for the first timestamp: no turn precedes it here.
|
|
319
|
+
return setAsideOversized(scanStart + error.offset);
|
|
320
|
+
}
|
|
321
|
+
} finally { try { staged?.close?.(); } finally { closeSync(fd); } }
|
|
322
|
+
}
|