@awebai/oats 0.61.0 → 0.62.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/oats.mjs +26 -11
- package/docs/capabilities.md +1 -1
- package/docs/execution-targets.md +49 -3
- package/docs/implementation.md +2 -1
- package/docs/named-test-runs.md +382 -0
- package/docs/official-catalog.md +1 -1
- package/docs/packages.md +5 -5
- package/docs/release-notes/v0.62.0.md +223 -0
- package/docs/upgrading.md +133 -11
- package/docs/workspaces.md +1 -1
- package/lib/core.mjs +2 -2
- package/lib/doctor-capture.mjs +35 -0
- package/lib/session-viewer.mjs +86 -7
- package/package-catalog.json +1 -1
- package/package.json +1 -1
- package/packages/client/cli-locator.mjs +2 -2
- package/packages/record/README.md +87 -13
- package/packages/record/bin/capture-hook.mjs +8 -0
- package/packages/record/bin/capture.mjs +143 -124
- package/packages/record/bin/recall.mjs +8 -3
- package/packages/record/bin/setup.mjs +115 -225
- package/packages/record/bin/turn-record.mjs +2 -2
- package/packages/record/docs/capture-service.md +1443 -28
- package/packages/record/docs/capture-status.schema.json +429 -12
- package/packages/record/lib/capture-admission.mjs +82 -0
- package/packages/record/lib/capture-aw-unit.mjs +137 -0
- package/packages/record/lib/capture-aw.mjs +31 -12
- package/packages/record/lib/capture-budget.mjs +31 -0
- package/packages/record/lib/capture-cc.mjs +39 -6
- package/packages/record/lib/capture-cleanup-queue.mjs +135 -0
- package/packages/record/lib/capture-cleanup-work.mjs +77 -0
- package/packages/record/lib/capture-command-operation.mjs +37 -0
- package/packages/record/lib/capture-command-output.mjs +61 -0
- package/packages/record/lib/capture-command-worker.mjs +6 -0
- package/packages/record/lib/capture-command.mjs +48 -0
- package/packages/record/lib/capture-completion.mjs +152 -0
- package/packages/record/lib/capture-context.mjs +50 -0
- package/packages/record/lib/capture-default-roots.mjs +30 -0
- package/packages/record/lib/capture-discovery.mjs +114 -0
- package/packages/record/lib/capture-failures.mjs +45 -0
- package/packages/record/lib/capture-health-doctor.mjs +13 -0
- package/packages/record/lib/capture-health-installation.mjs +117 -0
- package/packages/record/lib/capture-health.mjs +116 -0
- package/packages/record/lib/capture-hook-ownership.mjs +58 -0
- package/packages/record/lib/capture-hook-runner.mjs +17 -0
- package/packages/record/lib/capture-hook.mjs +26 -0
- package/packages/record/lib/capture-index-inventory.mjs +205 -0
- package/packages/record/lib/capture-index-outcomes.mjs +113 -0
- package/packages/record/lib/capture-index-reset.mjs +109 -0
- package/packages/record/lib/capture-index-work.mjs +77 -0
- package/packages/record/lib/capture-issue-preparation.mjs +73 -0
- package/packages/record/lib/capture-issues.mjs +15 -2
- package/packages/record/lib/capture-lock.mjs +102 -16
- package/packages/record/lib/capture-native-unit.mjs +322 -0
- package/packages/record/lib/capture-pass.mjs +102 -0
- package/packages/record/lib/capture-recovery-snapshot.mjs +182 -0
- package/packages/record/lib/capture-retained-scratch.mjs +213 -0
- package/packages/record/lib/capture-retry.mjs +102 -0
- package/packages/record/lib/capture-run.mjs +125 -0
- package/packages/record/lib/capture-scratch.mjs +12 -3
- package/packages/record/lib/capture-service-definition.mjs +104 -0
- package/packages/record/lib/capture-service-install.mjs +362 -0
- package/packages/record/lib/capture-service-ownership.mjs +137 -0
- package/packages/record/lib/capture-service-runner.mjs +18 -0
- package/packages/record/lib/capture-small-snapshot.mjs +85 -0
- package/packages/record/lib/capture-source-discovery.mjs +319 -0
- package/packages/record/lib/capture-source-queue.mjs +199 -0
- package/packages/record/lib/capture-source-work.mjs +85 -0
- package/packages/record/lib/capture-state.mjs +16 -71
- package/packages/record/lib/capture-status.mjs +171 -0
- package/packages/record/lib/capture-supervisor.mjs +117 -0
- package/packages/record/lib/capture-watch.mjs +32 -0
- package/packages/record/lib/capture-work-session.mjs +238 -0
- package/packages/record/lib/capture-worker.mjs +58 -0
- package/packages/record/lib/derived-state.mjs +28 -15
- package/packages/record/lib/file-lines.mjs +22 -6
- package/packages/record/lib/formats.mjs +4 -3
- package/packages/record/lib/index-capture.mjs +97 -0
- package/packages/record/lib/index-db.mjs +3 -17
- package/packages/record/lib/index-reset.mjs +19 -0
- package/packages/record/lib/store.mjs +36 -11
- package/packages/record/lib/transcript-prefix.mjs +125 -0
- package/skills/oats-getting-started/SKILL.md +1 -1
- package/docs/macos-test-runs.md +0 -246
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
// Deliberately small readers for the templates this package has actually
|
|
2
|
+
// written. Unsupported directives, expansions, markers or XML are foreign,
|
|
3
|
+
// never permission inferred from a filename or a mention of capture.
|
|
4
|
+
import { basename, isAbsolute, join } from "node:path";
|
|
5
|
+
import { captureServiceIntervalText, captureServiceLabel, captureServiceUnit, captureServiceRequest } from "./capture-service-definition.mjs";
|
|
6
|
+
import { CAPTURE_POLICY } from "./capture-budget.mjs";
|
|
7
|
+
|
|
8
|
+
const owner = value => /^[A-Za-z0-9._-]+$/.test(value);
|
|
9
|
+
const node = value => typeof value === "string" && isAbsolute(value) && ["node", "nodejs"].includes(basename(value));
|
|
10
|
+
const exactKeys = (value, keys) => Object.keys(value).sort().join("|") === [...keys].sort().join("|");
|
|
11
|
+
function legacy(argv) {
|
|
12
|
+
return argv?.length === 5 && node(argv[0]) && isAbsolute(argv[1]) && argv[1].endsWith("/bin/capture.mjs") &&
|
|
13
|
+
argv[2] === "--watch" && argv[3] === "--owner" && owner(argv[4]);
|
|
14
|
+
}
|
|
15
|
+
function periodic(argv) {
|
|
16
|
+
if (argv?.length !== 3 || typeof argv[0] !== "string" || !isAbsolute(argv[0]) || !isAbsolute(argv[1]) || !argv[1].endsWith("/lib/capture-service-runner.mjs")) return null;
|
|
17
|
+
const value = JSON.parse(argv[2]);
|
|
18
|
+
const request = captureServiceRequest(value);
|
|
19
|
+
return JSON.stringify(value) === JSON.stringify(request) ? request : null;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function captureServiceArgvOwnership(argv) {
|
|
23
|
+
try { return legacy(argv) ? "legacy" : periodic(argv) ? "periodic" : "foreign"; }
|
|
24
|
+
catch { return "foreign"; }
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function serviceSystemdArgs(line) {
|
|
28
|
+
const words = []; let i = 0;
|
|
29
|
+
while (i < line.length) {
|
|
30
|
+
while (/[ \t]/.test(line[i] ?? "")) i++;
|
|
31
|
+
if (i === line.length) break;
|
|
32
|
+
let word = "";
|
|
33
|
+
if (line[i] === '"') {
|
|
34
|
+
i++;
|
|
35
|
+
while (line[i] !== '"') {
|
|
36
|
+
if (i >= line.length) throw new Error("unterminated systemd word");
|
|
37
|
+
if (line[i] === "\\") { i++; if (!['"', "\\"].includes(line[i])) throw new Error("unsupported systemd escape"); }
|
|
38
|
+
word += line[i++];
|
|
39
|
+
}
|
|
40
|
+
i++;
|
|
41
|
+
if (i < line.length && !/[ \t]/.test(line[i])) throw new Error("joined systemd word");
|
|
42
|
+
} else {
|
|
43
|
+
while (i < line.length && !/[ \t]/.test(line[i])) word += line[i++];
|
|
44
|
+
if (!/^[A-Za-z0-9_./:@+,-]+$/.test(word)) throw new Error("unsupported bare systemd word");
|
|
45
|
+
}
|
|
46
|
+
if (/%/.test(word.replace(/%%/g, ""))) throw new Error("systemd specifier");
|
|
47
|
+
if (words.length) {
|
|
48
|
+
if (/\$/.test(word.replace(/\$\$/g, ""))) throw new Error("systemd environment expansion");
|
|
49
|
+
word = word.replace(/\$\$/g, "$");
|
|
50
|
+
}
|
|
51
|
+
words.push(word.replace(/%%/g, "%"));
|
|
52
|
+
}
|
|
53
|
+
return words;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function ini(text) {
|
|
57
|
+
const values = {}; let section = null;
|
|
58
|
+
for (const raw of text.split(/\r?\n/)) {
|
|
59
|
+
const line = raw.trim();
|
|
60
|
+
if (!line || /^[#;]/.test(line)) continue;
|
|
61
|
+
const heading = /^\[([A-Za-z]+)\]$/.exec(line);
|
|
62
|
+
if (heading) { section = heading[1]; continue; }
|
|
63
|
+
const item = /^([A-Za-z]+)=(.*)$/.exec(line);
|
|
64
|
+
if (!section || !item || line.endsWith("\\")) throw new Error("unsupported unit directive");
|
|
65
|
+
const key = `${section}.${item[1]}`;
|
|
66
|
+
if (Object.hasOwn(values, key)) throw new Error("duplicate unit directive");
|
|
67
|
+
values[key] = item[2];
|
|
68
|
+
}
|
|
69
|
+
return values;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function xmlText(raw) {
|
|
73
|
+
if (/[<>]/.test(raw) || /&(?!amp;|lt;|gt;|quot;|apos;|#13;)/.test(raw)) throw new Error("unsupported XML text");
|
|
74
|
+
return raw.replace(/ /g, "\r").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"')
|
|
75
|
+
.replace(/'/g, "'").replace(/&/g, "&");
|
|
76
|
+
}
|
|
77
|
+
function plist(text) {
|
|
78
|
+
if (/[\u0000-\u0008\u000b\u000c\u000e-\u001f\ufffe\uffff]/u.test(text)) throw new Error("invalid XML character");
|
|
79
|
+
for (const comment of text.matchAll(/<!--([^]*?)-->/g)) if (comment[1].includes("--")) throw new Error("invalid XML comment");
|
|
80
|
+
let rest = text.replace(/\r\n?/g, "\n").trim();
|
|
81
|
+
rest = rest.replace(/^<\?xml version="1\.0" encoding="UTF-8"\?>\s*/, "")
|
|
82
|
+
.replace(/^<!DOCTYPE plist PUBLIC "-\/\/Apple\/\/DTD PLIST 1\.0\/\/EN" "http:\/\/www\.apple\.com\/DTDs\/PropertyList-1\.0\.dtd">\s*/, "")
|
|
83
|
+
.replace(/<!--[^]*?-->/g, "").trim();
|
|
84
|
+
const body = /^<plist version="1\.0">\s*<dict>([^]*)<\/dict>\s*<\/plist>$/.exec(rest);
|
|
85
|
+
if (!body) throw new Error("unsupported plist");
|
|
86
|
+
rest = body[1]; const values = {};
|
|
87
|
+
while (rest.trim()) {
|
|
88
|
+
const key = /^\s*<key>([^<]*)<\/key>\s*/.exec(rest);
|
|
89
|
+
if (!key || Object.hasOwn(values, key[1])) throw new Error("invalid plist key");
|
|
90
|
+
rest = rest.slice(key[0].length);
|
|
91
|
+
const scalar = /^(?:<string>([^]*?)<\/string>|<integer>(\d+)<\/integer>|<(true|false)\s*\/>)/.exec(rest);
|
|
92
|
+
if (scalar) {
|
|
93
|
+
values[key[1]] = scalar[1] !== undefined ? xmlText(scalar[1]) : scalar[2] !== undefined ? Number(scalar[2]) : scalar[3] === "true";
|
|
94
|
+
rest = rest.slice(scalar[0].length); continue;
|
|
95
|
+
}
|
|
96
|
+
const array = /^<array>([^]*?)<\/array>/.exec(rest);
|
|
97
|
+
if (!array) throw new Error("unsupported plist value");
|
|
98
|
+
let members = array[1]; const args = [];
|
|
99
|
+
while (members.trim()) {
|
|
100
|
+
const item = /^\s*<string>([^]*?)<\/string>/.exec(members);
|
|
101
|
+
if (!item) throw new Error("unsupported plist array");
|
|
102
|
+
args.push(xmlText(item[1])); members = members.slice(item[0].length);
|
|
103
|
+
}
|
|
104
|
+
values[key[1]] = args; rest = rest.slice(array[0].length);
|
|
105
|
+
}
|
|
106
|
+
return values;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// intervalSeconds: the definition module's internal test seam, default the policy.
|
|
110
|
+
export function captureServiceOwnership(text, kind, { intervalSeconds = CAPTURE_POLICY.intervalSeconds } = {}) {
|
|
111
|
+
try {
|
|
112
|
+
const markers = text.match(/oats-turn-record-service:[^\s<>]*/g) ?? [];
|
|
113
|
+
if (markers.length > 1 || (markers.length && markers[0] !== "oats-turn-record-service:1")) return "foreign";
|
|
114
|
+
const marked = markers.length === 1;
|
|
115
|
+
if (marked && !(kind === "plist" ? /<!-- oats-turn-record-service:1 -->/ : /^# oats-turn-record-service:1\s*$/m).test(text)) return "foreign";
|
|
116
|
+
if (kind === "plist") {
|
|
117
|
+
const p = plist(text);
|
|
118
|
+
const common = ["Label", "ProgramArguments", "RunAtLoad", "StandardOutPath", "StandardErrorPath"];
|
|
119
|
+
if (p.Label !== captureServiceLabel || p.RunAtLoad !== true || typeof p.StandardOutPath !== "string" ||
|
|
120
|
+
p.StandardOutPath !== p.StandardErrorPath || !isAbsolute(p.StandardOutPath)) return "foreign";
|
|
121
|
+
if (!marked && exactKeys(p, [...common, "KeepAlive", "ThrottleInterval"]) && p.KeepAlive === true && p.ThrottleInterval === 60 && legacy(p.ProgramArguments)) return "legacy";
|
|
122
|
+
const request = marked && exactKeys(p, [...common, "StartInterval"]) && p.StartInterval === intervalSeconds && periodic(p.ProgramArguments);
|
|
123
|
+
return request && p.StandardOutPath === join(request.plan.home, "Library", "Logs", "turn-record-capture.log") ? "periodic" : "foreign";
|
|
124
|
+
}
|
|
125
|
+
const p = ini(text);
|
|
126
|
+
if (kind === "timer") return marked && exactKeys(p, ["Unit.Description", "Timer.OnBootSec", "Timer.OnUnitActiveSec", "Timer.Unit", "Install.WantedBy"]) &&
|
|
127
|
+
p["Unit.Description"] === "turn-record periodic capture" && p["Timer.OnBootSec"] === "1min" && p["Timer.OnUnitActiveSec"] === captureServiceIntervalText(intervalSeconds) &&
|
|
128
|
+
p["Timer.Unit"] === captureServiceUnit && p["Install.WantedBy"] === "timers.target" ? "periodic" : "foreign";
|
|
129
|
+
const argv = serviceSystemdArgs(p["Service.ExecStart"] ?? "");
|
|
130
|
+
if (!marked && exactKeys(p, ["Unit.Description", "Service.ExecStart", "Service.Restart", "Service.RestartSec", "Install.WantedBy"]) &&
|
|
131
|
+
p["Unit.Description"] === "turn-record capture watcher" && p["Service.Restart"] === "always" && p["Service.RestartSec"] === "60" &&
|
|
132
|
+
p["Install.WantedBy"] === "default.target" && legacy(argv)) return "legacy";
|
|
133
|
+
return marked && exactKeys(p, ["Unit.Description", "Service.Type", "Service.ExecStart", "Service.MemoryMax", "Service.RuntimeMaxSec"]) &&
|
|
134
|
+
p["Unit.Description"] === "turn-record bounded capture pass" && p["Service.Type"] === "exec" && p["Service.MemoryMax"] === "2G" &&
|
|
135
|
+
p["Service.RuntimeMaxSec"] === "300" && periodic(argv) ? "periodic" : "foreign";
|
|
136
|
+
} catch { return "foreign"; }
|
|
137
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// Fixed, finite installed-service supervisor entry. It neither schedules the
|
|
2
|
+
// next pass nor installs/inspects a manager; the root lock owns child lifetime.
|
|
3
|
+
import { captureServiceRequest } from "./capture-service-definition.mjs";
|
|
4
|
+
import { captureCommandPass } from "./capture-command-output.mjs";
|
|
5
|
+
import { IgnoreError, loadIgnore } from "./ignore.mjs";
|
|
6
|
+
|
|
7
|
+
try {
|
|
8
|
+
if (process.argv.length !== 3) throw new Error("invalid private capture service arguments");
|
|
9
|
+
const request = captureServiceRequest(JSON.parse(process.argv[2]));
|
|
10
|
+
// Match foreground setup/manual privacy ordering. No source discovery runs
|
|
11
|
+
// here: the registered worker resolves the explicit plan after admission.
|
|
12
|
+
loadIgnore(request.root);
|
|
13
|
+
await captureCommandPass(request.root, { owner: request.owner, plan: request.plan,
|
|
14
|
+
trigger: "service", indexEnabled: true, retrySetAside: false });
|
|
15
|
+
} catch (error) {
|
|
16
|
+
console.error(error instanceof IgnoreError ? error.message : `capture service failed: ${error.message}`);
|
|
17
|
+
process.exitCode = 1;
|
|
18
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
// A disposable command snapshot, eligible by TOTAL fixed end, never suffix
|
|
2
|
+
// length. Reopening a later worker's source is not proof: it must finish a new
|
|
3
|
+
// chunked copy/hash and match the full digest+size before reusing comparison.
|
|
4
|
+
import { createHash } from "node:crypto";
|
|
5
|
+
import { closeSync, fstatSync, openSync, writeSync } from "node:fs";
|
|
6
|
+
import { join } from "node:path";
|
|
7
|
+
import { createCaptureScratch } from "./capture-scratch.mjs";
|
|
8
|
+
import { fileVersion, sameVersion, statVersion } from "./derived-state.mjs";
|
|
9
|
+
import { assertProtectedDescriptor, sourceChunks } from "./session-snapshot.mjs";
|
|
10
|
+
import { fileChunks } from "./file-lines.mjs";
|
|
11
|
+
|
|
12
|
+
export const SMALL_SNAPSHOT_BYTES = 8388608;
|
|
13
|
+
|
|
14
|
+
export function smallCommandSnapshot(root, fd, path, size, { capturedPi, stopped = () => false, work } = {}) {
|
|
15
|
+
if (!Number.isSafeInteger(size) || size < 0 || size > SMALL_SNAPSHOT_BYTES) {
|
|
16
|
+
throw new Error("small command snapshot exceeds its total fixed-end eligibility");
|
|
17
|
+
}
|
|
18
|
+
const observe = () => {
|
|
19
|
+
assertProtectedDescriptor(fd, path, capturedPi);
|
|
20
|
+
const stat = fstatSync(fd, { bigint: true });
|
|
21
|
+
if (!stat.isFile()) throw new Error("small snapshot source is not a regular file");
|
|
22
|
+
const version = statVersion(stat);
|
|
23
|
+
if (!sameVersion(version, fileVersion(path))) throw new Error("small snapshot descriptor differs from its named source");
|
|
24
|
+
return version;
|
|
25
|
+
};
|
|
26
|
+
const before = observe();
|
|
27
|
+
let observed = before;
|
|
28
|
+
if (before.size < size) throw new Error("small snapshot source shrank before staging");
|
|
29
|
+
if (stopped()) return { complete: false };
|
|
30
|
+
const scratch = createCaptureScratch(root, "snapshot");
|
|
31
|
+
let stagedFd, keep = false;
|
|
32
|
+
try {
|
|
33
|
+
stagedFd = openSync(join(scratch.path, "snapshot"), "wx+", 0o600);
|
|
34
|
+
const hash = createHash("sha256");
|
|
35
|
+
let copied = 0;
|
|
36
|
+
const iterator = sourceChunks(fd, 0, size, path, capturedPi);
|
|
37
|
+
try {
|
|
38
|
+
while (copied < size) {
|
|
39
|
+
if (stopped()) return { complete: false };
|
|
40
|
+
const next = iterator.next();
|
|
41
|
+
if (next.done) throw new Error("small snapshot short read");
|
|
42
|
+
const bytes = next.value;
|
|
43
|
+
if (work) work.sourceBytes = (work.sourceBytes ?? 0) + bytes.length;
|
|
44
|
+
// Bytes are copied, never joined into lines: a captured record of any
|
|
45
|
+
// size stages; the raw-record limit belongs to capture beyond the prefix.
|
|
46
|
+
hash.update(bytes);
|
|
47
|
+
for (let written = 0; written < bytes.length;) {
|
|
48
|
+
const count = writeSync(stagedFd, bytes, written, bytes.length - written, copied + written);
|
|
49
|
+
if (!count) throw new Error("small snapshot short write");
|
|
50
|
+
written += count;
|
|
51
|
+
}
|
|
52
|
+
copied += bytes.length;
|
|
53
|
+
if (work) work.stagedWriteBytes = (work.stagedWriteBytes ?? 0) + bytes.length;
|
|
54
|
+
}
|
|
55
|
+
} finally { iterator.return(); }
|
|
56
|
+
const digest = hash.digest("hex"), payload = statVersion(fstatSync(stagedFd, { bigint: true }));
|
|
57
|
+
const verify = () => {
|
|
58
|
+
if (!sameVersion(payload, statVersion(fstatSync(stagedFd, { bigint: true })))) throw new Error("small snapshot payload changed");
|
|
59
|
+
const after = observe();
|
|
60
|
+
if (sameVersion(observed, after)) return;
|
|
61
|
+
if (after.dev !== observed.dev || after.ino !== observed.ino || after.size <= observed.size) throw new Error("small snapshot source changed");
|
|
62
|
+
// Growth while copying/verifying requires a complete fixed-end digest,
|
|
63
|
+
// not the ordinary incremental cursor's bounded boundary witness.
|
|
64
|
+
const check = createHash("sha256");
|
|
65
|
+
for (const bytes of sourceChunks(fd, 0, size, path, capturedPi)) {
|
|
66
|
+
if (work) work.sourceVerificationBytes = (work.sourceVerificationBytes ?? 0) + bytes.length;
|
|
67
|
+
check.update(bytes);
|
|
68
|
+
}
|
|
69
|
+
if (check.digest("hex") !== digest || !sameVersion(after, observe())) throw new Error("small snapshot source changed during verification");
|
|
70
|
+
observed = after;
|
|
71
|
+
};
|
|
72
|
+
verify(); keep = true; let closed = false;
|
|
73
|
+
return { complete: true, small: true, snapshot: { id: `sha256:${digest}:${size}`, size,
|
|
74
|
+
*chunks(from = 0, end = size) {
|
|
75
|
+
if (!Number.isSafeInteger(from) || from < 0 || !Number.isSafeInteger(end) || end < from || end > size) throw new Error("invalid small snapshot range");
|
|
76
|
+
verify(); let read = 0;
|
|
77
|
+
for (const bytes of fileChunks(stagedFd, { start: from, end, size: 65536 })) { read += bytes.length; yield bytes; }
|
|
78
|
+
if (read !== end - from) throw new Error("small snapshot payload short read");
|
|
79
|
+
verify();
|
|
80
|
+
} }, verify,
|
|
81
|
+
close() { if (!closed) { closed = true; try { closeSync(stagedFd); } finally { scratch.close(); } } } };
|
|
82
|
+
} finally {
|
|
83
|
+
if (!keep) { try { if (stagedFd !== undefined) closeSync(stagedFd); } finally { scratch.close(); } }
|
|
84
|
+
}
|
|
85
|
+
}
|
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
// Private source-plan bridge. The caller supplies explicit roots or a stable
|
|
2
|
+
// default-root selection. Resolution/enumeration never opens source content.
|
|
3
|
+
// Pages and their positions commit together; enumeration never grants success.
|
|
4
|
+
import { readdirSync, statSync } from "node:fs";
|
|
5
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
6
|
+
import { isAbsolute, join } from "node:path";
|
|
7
|
+
import { NativeCaptureDiscovery } from "./capture-discovery.mjs";
|
|
8
|
+
import { SESSION_FORMATS } from "./formats.mjs";
|
|
9
|
+
import { awStream } from "./capture-aw.mjs";
|
|
10
|
+
import { guardCapturedPath } from "./native-history.mjs";
|
|
11
|
+
import { FAILURE_REASONS, retryAfterAttempt } from "./capture-failures.mjs";
|
|
12
|
+
import { CaptureStop } from "./capture-budget.mjs";
|
|
13
|
+
import { captureDefaultRootPlan, resolveCaptureDefaultRoots } from "./capture-default-roots.mjs";
|
|
14
|
+
|
|
15
|
+
const sources = ["cc", "pi", "codex", "aw"];
|
|
16
|
+
const attemptsKind = "capture-discovery-attempt";
|
|
17
|
+
const text = value => typeof value === "string" && value.length > 0;
|
|
18
|
+
const identity = stat => ({ dev: String(stat.dev), ino: String(stat.ino) });
|
|
19
|
+
const sameIdentity = (a, b) => a?.dev === b?.dev && a?.ino === b?.ino;
|
|
20
|
+
const absolute = path => typeof path === "string" && isAbsolute(path) && !path.includes("\0");
|
|
21
|
+
const validIdentity = value => value && typeof value.dev === "string" && typeof value.ino === "string";
|
|
22
|
+
const validName = name => typeof name === "string" && name && name !== "." && name !== ".." && !/[\/\\\0]/.test(name);
|
|
23
|
+
const validFailure = row => row && typeof row.runId === "string" && row.runId &&
|
|
24
|
+
typeof row.attemptedRun === "string" && row.attemptedRun && FAILURE_REASONS.includes(row.reason) &&
|
|
25
|
+
Number.isSafeInteger(row.attempts) && row.attempts > 0 && typeof row.nextRetryAt === "string" &&
|
|
26
|
+
Number.isFinite(Date.parse(row.nextRetryAt)) && new Date(row.nextRetryAt).toISOString() === row.nextRetryAt;
|
|
27
|
+
|
|
28
|
+
// AW uses sorted code-unit basenames, not native localeCompare ordering. A
|
|
29
|
+
// cached name array is metadata proportional to the directory, not content.
|
|
30
|
+
class AwDiscovery {
|
|
31
|
+
constructor(path, checkpoint) {
|
|
32
|
+
this.path = path;
|
|
33
|
+
this.cursor = structuredClone(checkpoint ?? { after: null, identity: null, complete: false });
|
|
34
|
+
const row = this.cursor;
|
|
35
|
+
if (!row || row.after !== null && !validName(row.after) || typeof row.complete !== "boolean" ||
|
|
36
|
+
row.identity !== null && (!row.identity || typeof row.identity.dev !== "string" || typeof row.identity.ino !== "string"))
|
|
37
|
+
throw new Error("invalid AW discovery checkpoint");
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
advance({ limit, stopped, work }) {
|
|
41
|
+
const row = this.cursor, files = []; let visited = 0;
|
|
42
|
+
if (row.complete) return { files, visited, complete: true, checkpoint: structuredClone(row) };
|
|
43
|
+
guardCapturedPath(this.path);
|
|
44
|
+
let before;
|
|
45
|
+
try { before = statSync(this.path, { bigint: true }); }
|
|
46
|
+
catch (error) {
|
|
47
|
+
if (error.code !== "ENOENT" || row.identity !== null || row.after !== null) throw error;
|
|
48
|
+
row.complete = true;
|
|
49
|
+
return { files, visited, complete: true, checkpoint: structuredClone(row) };
|
|
50
|
+
}
|
|
51
|
+
if (!before.isDirectory() || row.identity && !sameIdentity(row.identity, identity(before)))
|
|
52
|
+
throw new Error("AW discovery directory identity changed");
|
|
53
|
+
if (!this.names) {
|
|
54
|
+
this.names = readdirSync(this.path).sort();
|
|
55
|
+
if (!sameIdentity(identity(before), identity(statSync(this.path, { bigint: true }))))
|
|
56
|
+
throw new Error("AW discovery directory changed during enumeration");
|
|
57
|
+
if (work) { work.directoriesRead = (work.directoriesRead ?? 0) + 1;
|
|
58
|
+
work.directoryEntriesRead = (work.directoryEntriesRead ?? 0) + this.names.length; }
|
|
59
|
+
let low = 0, high = this.names.length;
|
|
60
|
+
while (low < high) {
|
|
61
|
+
const mid = Math.floor((low + high) / 2);
|
|
62
|
+
if (row.after !== null && this.names[mid] <= row.after) low = mid + 1; else high = mid;
|
|
63
|
+
}
|
|
64
|
+
this.position = low;
|
|
65
|
+
}
|
|
66
|
+
row.identity = identity(before);
|
|
67
|
+
while (this.position < this.names.length && visited < limit && !stopped()) {
|
|
68
|
+
const name = this.names[this.position++]; row.after = name; visited++;
|
|
69
|
+
if (name.endsWith(".jsonl")) {
|
|
70
|
+
const path = join(this.path, name); guardCapturedPath(path); files.push(path);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
if (!sameIdentity(row.identity, identity(statSync(this.path, { bigint: true }))))
|
|
74
|
+
throw new Error("AW discovery directory changed during enumeration");
|
|
75
|
+
row.complete = this.position === this.names.length;
|
|
76
|
+
if (work) work.entriesVisited = (work.entriesVisited ?? 0) + visited;
|
|
77
|
+
return { files, visited, complete: row.complete, checkpoint: structuredClone(row) };
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
close() { this.names = null; }
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export class CaptureSourceDiscovery {
|
|
84
|
+
constructor(queue, roots) {
|
|
85
|
+
if (roots?.kind === "default-roots") {
|
|
86
|
+
if (Object.keys(roots).some(key => !["kind", "home", "cwd", "env", "sources"].includes(key)))
|
|
87
|
+
throw new Error("invalid capture discovery root plan");
|
|
88
|
+
this.defaults = captureDefaultRootPlan(roots);
|
|
89
|
+
} else {
|
|
90
|
+
if (!Array.isArray(roots) || roots.some(root => !root || !sources.includes(root.source) ||
|
|
91
|
+
!absolute(root.path) || Object.keys(root).some(key => !["source", "path"].includes(key))))
|
|
92
|
+
throw new Error("invalid capture discovery root plan");
|
|
93
|
+
this.roots = sources.map(source => roots.filter(root => root.source === source).map(root => root.path));
|
|
94
|
+
}
|
|
95
|
+
this.queue = queue;
|
|
96
|
+
this.plan = createHash("sha256").update(JSON.stringify(this.defaults ?? this.roots)).digest("hex");
|
|
97
|
+
this.readers = new Map();
|
|
98
|
+
this.current();
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
current() {
|
|
102
|
+
const sweep = this.queue.current();
|
|
103
|
+
let row = sweep.discovery;
|
|
104
|
+
if (row === null && !sweep.discovered) row = { v: this.defaults ? 2 : 1, plan: this.plan, next: 0,
|
|
105
|
+
groups: sources.map((source, index) => this.defaults ?
|
|
106
|
+
{ root: 0, checkpoint: null, complete: !this.defaults.sources.includes(source),
|
|
107
|
+
roots: this.defaults.sources.includes(source) ? null : [] } :
|
|
108
|
+
{ root: 0, checkpoint: null, complete: this.roots[index].length === 0 }) };
|
|
109
|
+
if (!row || row.v !== (this.defaults ? 2 : 1) || row.plan !== this.plan ||
|
|
110
|
+
!Number.isSafeInteger(row.next) || row.next < 0 || row.next >= sources.length ||
|
|
111
|
+
!Array.isArray(row.groups) || row.groups.length !== sources.length || row.groups.some((group, index) => !this.validGroup(group, index)) ||
|
|
112
|
+
sweep.discovered && row.groups.some(group => !group.complete))
|
|
113
|
+
throw new Error("capture source discovery plan or checkpoint changed");
|
|
114
|
+
return structuredClone(row);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
healthyPending() { return this.current().groups.some(group => !group.complete && !group.failure); }
|
|
118
|
+
|
|
119
|
+
validGroup(group, index) {
|
|
120
|
+
if (!group || !Number.isSafeInteger(group.root) || group.root < 0 ||
|
|
121
|
+
group.failure != null && (group.complete || !validFailure(group.failure)) ||
|
|
122
|
+
group.checkpoint !== null && (group.complete || typeof group.checkpoint !== "object")) return false;
|
|
123
|
+
if (this.defaults) {
|
|
124
|
+
if (group.roots === null) return this.defaults.sources.includes(sources[index]) &&
|
|
125
|
+
group.root === 0 && group.checkpoint === null && group.complete === false;
|
|
126
|
+
if (!Array.isArray(group.roots) || group.roots.some(root => !root || !absolute(root.path) ||
|
|
127
|
+
!(validIdentity(root.identity) || sources[index] === "aw" && root.identity === null)) ||
|
|
128
|
+
!this.defaults.sources.includes(sources[index]) && group.roots.length !== 0) return false;
|
|
129
|
+
}
|
|
130
|
+
const count = this.defaults ? group.roots.length : this.roots[index].length;
|
|
131
|
+
return group.root <= count && group.complete === (group.root === count);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
pathFor(group, index) {
|
|
135
|
+
return this.defaults ? group.roots === null ? this.defaults.home : group.roots[group.root]?.path : this.roots[index][group.root];
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// Resolving one format can enumerate/sort home-directory metadata. Keep it
|
|
139
|
+
// outside the admission transaction but inside the worker's bounded life.
|
|
140
|
+
resolveRoots(source) {
|
|
141
|
+
return resolveCaptureDefaultRoots(this.defaults, source).map(path => {
|
|
142
|
+
guardCapturedPath(path);
|
|
143
|
+
try {
|
|
144
|
+
const stat = statSync(path, { bigint: true });
|
|
145
|
+
if (!stat.isDirectory()) throw new Error("capture default root is not a directory");
|
|
146
|
+
return { path, identity: identity(stat) };
|
|
147
|
+
} catch (error) {
|
|
148
|
+
if (source === "aw" && error.code === "ENOENT") return { path, identity: null };
|
|
149
|
+
throw error;
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
active() {
|
|
155
|
+
const attempt = this.queue.state.get(attemptsKind, this.queue.owner);
|
|
156
|
+
if (attempt !== null && (![attempt?.id, attempt?.runId, attempt?.workerNonce].every(text) ||
|
|
157
|
+
attempt.scope !== this.queue.current().id || attempt.plan !== this.plan ||
|
|
158
|
+
!Number.isSafeInteger(attempt.index) || attempt.index < 0 || attempt.index >= sources.length ||
|
|
159
|
+
attempt.source !== sources[attempt.index] || !text(attempt.path) || !this.validGroup(attempt.group, attempt.index) ||
|
|
160
|
+
attempt.group.complete !== false ||
|
|
161
|
+
attempt.phase !== (this.defaults && attempt.group.roots === null ? "roots" : undefined) ||
|
|
162
|
+
attempt.path !== this.pathFor(attempt.group, attempt.index)))
|
|
163
|
+
throw new Error("invalid capture discovery admission");
|
|
164
|
+
return attempt;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
admitted(attempt) {
|
|
168
|
+
if (JSON.stringify(this.active()) !== JSON.stringify(attempt) ||
|
|
169
|
+
JSON.stringify(this.current().groups[attempt.index]) !== JSON.stringify(attempt.group))
|
|
170
|
+
throw new Error("capture discovery admission or checkpoint changed");
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
failed(attempt, reason, now) {
|
|
174
|
+
if (!FAILURE_REASONS.includes(reason)) throw new Error("invalid discovery interruption reason");
|
|
175
|
+
this.queue.state.transaction(() => {
|
|
176
|
+
this.admitted(attempt);
|
|
177
|
+
const row = this.current(), group = row.groups[attempt.index];
|
|
178
|
+
const attempts = (group.failure?.attempts ?? 0) + 1;
|
|
179
|
+
group.failure = { runId: attempt.runId, attemptedRun: attempt.runId, reason,
|
|
180
|
+
attempts, nextRetryAt: retryAfterAttempt(attempts, now) };
|
|
181
|
+
row.next = (attempt.index + 1) % sources.length;
|
|
182
|
+
this.queue.add([], row);
|
|
183
|
+
this.queue.state.set("capture-discovery-failure", "last", { runId: attempt.runId, reason,
|
|
184
|
+
scope: attempt.scope, source: attempt.source, path: attempt.path });
|
|
185
|
+
this.queue.state.delete(attemptsKind, this.queue.owner);
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// Owning bootstrap has registered the new worker and ended its predecessor.
|
|
190
|
+
// Never infer stream/frame attribution from this metadata-only operation.
|
|
191
|
+
reconcile(start, { reason = "interrupted", now = Date.now() } = {}) {
|
|
192
|
+
if (this.queue.state.depth || !text(start?.workerNonce) || !text(start.pass?.runId) ||
|
|
193
|
+
!Number.isSafeInteger(now) || now < 0) throw new Error("invalid discovery replacement");
|
|
194
|
+
const attempt = this.active();
|
|
195
|
+
if (!attempt) return null;
|
|
196
|
+
if (attempt.workerNonce === start.workerNonce) throw new Error("capture discovery worker has not been replaced");
|
|
197
|
+
this.failed(attempt, reason, now);
|
|
198
|
+
this.clear();
|
|
199
|
+
return attempt;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
publish(inputs, row, complete, attempt) {
|
|
203
|
+
if (!attempt) return this.queue.add(inputs, row, { complete });
|
|
204
|
+
this.queue.state.transaction(() => {
|
|
205
|
+
this.admitted(attempt);
|
|
206
|
+
this.queue.add(inputs, row, { complete });
|
|
207
|
+
this.queue.state.delete(attemptsKind, this.queue.owner);
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// Only the owning session supplies run identity. Direct readers keep their
|
|
212
|
+
// refusal/rollback contract; discovery errors never become stream failures.
|
|
213
|
+
advance({ limit = 128, stopped = () => false, work, runId = null, workerNonce = null, now = Date.now(), retry = false } = {}) {
|
|
214
|
+
if (this.closed || this.queue.state.depth || !Number.isSafeInteger(limit) || limit < 1 || limit > 128 || typeof stopped !== "function")
|
|
215
|
+
throw new Error("invalid capture source discovery unit");
|
|
216
|
+
if (typeof retry !== "boolean" || (runId === null ? workerNonce !== null : !text(runId) || !text(workerNonce)) ||
|
|
217
|
+
retry && runId === null || !Number.isSafeInteger(now) || now < 0)
|
|
218
|
+
throw new Error("invalid capture discovery failure context");
|
|
219
|
+
if (this.active()) throw new Error("unreconciled capture discovery admission");
|
|
220
|
+
if (stopped()) return { visited: 0, added: 0, stopped: true, complete: false };
|
|
221
|
+
const row = this.current();
|
|
222
|
+
if (this.queue.current().discovered) return { visited: 0, added: 0, complete: true };
|
|
223
|
+
let index = -1;
|
|
224
|
+
for (let offset = 0; offset < sources.length; offset++) {
|
|
225
|
+
const candidate = (row.next + offset) % sources.length;
|
|
226
|
+
const group = row.groups[candidate], failure = group.failure;
|
|
227
|
+
if (!group.complete && (retry ? failure && failure.attemptedRun !== runId && now >= Date.parse(failure.nextRetryAt) : !failure)) {
|
|
228
|
+
index = candidate; break;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
if (index < 0) {
|
|
232
|
+
if (row.groups.some(group => !group.complete)) return { visited: 0, added: 0, complete: false, deferred: true };
|
|
233
|
+
this.queue.add([], row, { complete: true });
|
|
234
|
+
return { visited: 0, added: 0, complete: true };
|
|
235
|
+
}
|
|
236
|
+
const group = row.groups[index], source = sources[index], path = this.pathFor(group, index);
|
|
237
|
+
const resolving = this.defaults && group.roots === null;
|
|
238
|
+
if (resolving && runId === null) throw new Error("capture default-root resolution requires owned discovery admission");
|
|
239
|
+
const attempt = runId === null ? null : { id: randomUUID(), runId, workerNonce,
|
|
240
|
+
scope: this.queue.current().id, plan: this.plan, index, source, path, group: structuredClone(group),
|
|
241
|
+
...(resolving ? { phase: "roots" } : {}) };
|
|
242
|
+
if (attempt) this.queue.state.transaction(() => {
|
|
243
|
+
if (this.active() || JSON.stringify(this.current()) !== JSON.stringify(row))
|
|
244
|
+
throw new Error("capture discovery changed before admission");
|
|
245
|
+
this.queue.state.set(attemptsKind, this.queue.owner, attempt);
|
|
246
|
+
});
|
|
247
|
+
try {
|
|
248
|
+
if (resolving) {
|
|
249
|
+
let resolved;
|
|
250
|
+
try { resolved = this.resolveRoots(source); }
|
|
251
|
+
catch (error) {
|
|
252
|
+
if (attempt && !(error instanceof CaptureStop)) this.failed(attempt, "io", now);
|
|
253
|
+
throw error;
|
|
254
|
+
}
|
|
255
|
+
group.roots = resolved; group.complete = resolved.length === 0;
|
|
256
|
+
delete group.failure;
|
|
257
|
+
row.next = (index + 1) % sources.length;
|
|
258
|
+
const complete = row.groups.every(group => group.complete);
|
|
259
|
+
this.publish([], row, complete, attempt);
|
|
260
|
+
return { source, resolvedRoots: true, visited: 0, added: 0, complete };
|
|
261
|
+
}
|
|
262
|
+
let cached = this.readers.get(source);
|
|
263
|
+
const checkpoint = JSON.stringify(group.checkpoint);
|
|
264
|
+
const selectedIdentity = this.defaults ? group.roots[group.root].identity : null;
|
|
265
|
+
if (!cached || cached.root !== group.root || cached.checkpoint !== checkpoint) {
|
|
266
|
+
cached?.reader.close();
|
|
267
|
+
// Carry selection's witness into the first reader, not only a stat
|
|
268
|
+
// before construction: the reader also checks it around enumeration.
|
|
269
|
+
const initial = group.checkpoint ?? (selectedIdentity ? source === "aw" ?
|
|
270
|
+
{ after: null, identity: selectedIdentity, complete: false } :
|
|
271
|
+
{ format: source, root: path, complete: false, stack: [{ name: null, after: null, identity: selectedIdentity }] } : null);
|
|
272
|
+
const reader = source === "aw" ? new AwDiscovery(path, initial) : new NativeCaptureDiscovery(source, path, initial);
|
|
273
|
+
cached = { root: group.root, checkpoint, reader }; this.readers.set(source, cached);
|
|
274
|
+
}
|
|
275
|
+
let result;
|
|
276
|
+
try {
|
|
277
|
+
// A previously refused root is not a fresh optional default, even when
|
|
278
|
+
// its first page failed before establishing a directory identity.
|
|
279
|
+
if (retry || selectedIdentity) {
|
|
280
|
+
guardCapturedPath(path);
|
|
281
|
+
const stat = statSync(path, { bigint: true });
|
|
282
|
+
if (!stat.isDirectory()) throw new Error("capture discovery failed root is not a directory");
|
|
283
|
+
if (selectedIdentity && !sameIdentity(selectedIdentity, identity(stat)))
|
|
284
|
+
throw new Error("capture default-root directory identity changed");
|
|
285
|
+
}
|
|
286
|
+
result = cached.reader.advance({ limit, stopped, work });
|
|
287
|
+
}
|
|
288
|
+
catch (error) {
|
|
289
|
+
if (attempt && !(error instanceof CaptureStop)) {
|
|
290
|
+
// Keep the exact rejected checkpoint/root. Publishing the failure and
|
|
291
|
+
// rotation together lets a replacement reach safe independent work.
|
|
292
|
+
// A storage failure here still propagates: it is not a root refusal.
|
|
293
|
+
this.failed(attempt, "io", now);
|
|
294
|
+
}
|
|
295
|
+
throw error;
|
|
296
|
+
}
|
|
297
|
+
const inputs = result.files.map(path => ({ source, path,
|
|
298
|
+
stream: source === "aw" ? awStream(this.queue.owner) : `${this.queue.owner}~${source}.${SESSION_FORMATS[source].sessionId(path)}` }));
|
|
299
|
+
if (result.complete) {
|
|
300
|
+
group.root++; group.checkpoint = null;
|
|
301
|
+
group.complete = group.root === (this.defaults ? group.roots.length : this.roots[index].length);
|
|
302
|
+
delete group.failure;
|
|
303
|
+
} else group.checkpoint = result.checkpoint;
|
|
304
|
+
// Partial retry remains pending and receives at most one unit per pass.
|
|
305
|
+
if (group.failure) group.failure.attemptedRun = runId;
|
|
306
|
+
row.next = (index + 1) % sources.length;
|
|
307
|
+
const complete = row.groups.every(group => group.complete);
|
|
308
|
+
// A transaction failure must discard the advanced volatile walker. The
|
|
309
|
+
// next attempt reloads the last durable page, with idempotent insertion.
|
|
310
|
+
this.publish(inputs, row, complete, attempt);
|
|
311
|
+
cached.checkpoint = JSON.stringify(group.checkpoint);
|
|
312
|
+
if (result.complete) { cached.reader.close(); this.readers.delete(source); }
|
|
313
|
+
return { source, visited: result.visited, added: inputs.length, complete };
|
|
314
|
+
} catch (error) { this.clear(); throw error; }
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
clear() { for (const { reader } of this.readers.values()) reader.close(); this.readers.clear(); }
|
|
318
|
+
close() { this.clear(); this.closed = true; }
|
|
319
|
+
}
|