@awebai/oats 0.53.0 → 0.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/oats.mjs +17 -7
- package/docs/design/2026-09-23-workspace-module-contracts.md +3 -0
- package/docs/desktop-cli-api.md +44 -12
- package/docs/implementation.md +12 -4
- package/docs/integrations.md +2 -2
- package/docs/knowledge.md +14 -2
- package/docs/official-catalog.md +5 -5
- package/docs/packages.md +14 -14
- package/docs/release-notes/v0.53.0.md +14 -0
- package/docs/release-notes/v0.54.0.md +142 -0
- package/docs/release-notes/v0.55.0.md +158 -0
- package/docs/servers.md +13 -3
- package/docs/souls-and-instances.md +2 -1
- package/docs/upgrading.md +14 -2
- package/docs/workspaces.md +122 -2
- package/lib/packages.mjs +5 -1
- package/lib/remote.mjs +255 -76
- package/lib/servers.mjs +82 -34
- package/lib/workspace.mjs +27 -9
- package/package-catalog.json +4 -4
- package/package.json +1 -1
- package/packages/client/cli-locator.mjs +2 -2
- package/packages/client/workspace-views.mjs +4 -3
- package/packages/record/README.md +54 -13
- package/packages/record/bin/capture.mjs +107 -53
- package/packages/record/docs/capture-service.md +193 -0
- package/packages/record/docs/capture-status.schema.json +100 -0
- package/packages/record/lib/capture-aw.mjs +96 -94
- package/packages/record/lib/capture-cc.mjs +109 -187
- package/packages/record/lib/capture-db-reader.mjs +39 -0
- package/packages/record/lib/capture-issues.mjs +64 -0
- package/packages/record/lib/capture-scratch.mjs +111 -0
- package/packages/record/lib/capture-state.mjs +105 -0
- package/packages/record/lib/derived-state.mjs +60 -0
- package/packages/record/lib/formats.mjs +31 -31
- package/packages/record/lib/index-db.mjs +74 -34
- package/packages/record/lib/receipt-summary.mjs +62 -0
- package/packages/record/lib/session-snapshot.mjs +56 -1
- package/packages/record/lib/sqlite-rows.mjs +14 -0
- package/packages/record/lib/store.mjs +122 -43
- package/packages/record/lib/transcript-prefix.mjs +40 -0
- package/skills/oats-getting-started/SKILL.md +8 -2
|
@@ -12,8 +12,8 @@
|
|
|
12
12
|
// original transcript is reconstructible by concatenating body.line.
|
|
13
13
|
//
|
|
14
14
|
// Incremental by source byte offset (cache under index/, derived state:
|
|
15
|
-
// losing it
|
|
16
|
-
//
|
|
15
|
+
// losing it requires comparison against the journal's positional prefix
|
|
16
|
+
// before resuming, never blind re-appending). Reconciliation is the capture: hooks and watchers
|
|
17
17
|
// only decide when to run it.
|
|
18
18
|
//
|
|
19
19
|
// Files matching the record's ignore list (`<root>/ignore`, see
|
|
@@ -21,15 +21,17 @@
|
|
|
21
21
|
// un-ignoring a file later makes the next pass capture it normally.
|
|
22
22
|
|
|
23
23
|
import { isUtf8 } from "node:buffer";
|
|
24
|
-
import { closeSync, constants, fstatSync,
|
|
25
|
-
import { basename
|
|
24
|
+
import { closeSync, constants, fstatSync, openSync } from "node:fs";
|
|
25
|
+
import { basename } from "node:path";
|
|
26
26
|
|
|
27
27
|
import { finishTurn } from "./canonical.mjs";
|
|
28
28
|
import { jsonlLines, SESSION_FORMATS } from "./formats.mjs";
|
|
29
29
|
import { loadIgnore } from "./ignore.mjs";
|
|
30
|
-
import { assertIdentity, assertProtectedDescriptor, digest, identity,
|
|
30
|
+
import { assertIdentity, assertProtectedDescriptor, digest, identity, sourceChunks, stageSnapshot, verifySnapshot } from "./session-snapshot.mjs";
|
|
31
31
|
import { guardCapturedPath } from "./native-history.mjs";
|
|
32
|
-
import {
|
|
32
|
+
import { bufferLines } from "./file-lines.mjs";
|
|
33
|
+
import { DerivedState, fileVersion, sameVersion, statVersion } from "./derived-state.mjs";
|
|
34
|
+
import { compareTranscriptPrefix } from "./transcript-prefix.mjs";
|
|
33
35
|
import { isDeepStrictEqual } from "node:util";
|
|
34
36
|
|
|
35
37
|
export const SESSION_STREAM_SOURCE = "cc";
|
|
@@ -79,141 +81,22 @@ export function eventTurnCore({ owner, source, sessionId, line, lineNo, ts }) {
|
|
|
79
81
|
};
|
|
80
82
|
}
|
|
81
83
|
|
|
82
|
-
// Offset cache: `<stream>:<absolute source path>` -> { bytes consumed,
|
|
83
|
-
// lines emitted, last carried timestamp }. Derived state under index/.
|
|
84
|
-
function offsetsPath(store) {
|
|
85
|
-
return join(store.root, "index", "capture-offsets.json");
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
function loadOffsets(store) {
|
|
89
|
-
try {
|
|
90
|
-
return JSON.parse(readFileSync(offsetsPath(store), "utf8"));
|
|
91
|
-
} catch (err) {
|
|
92
|
-
// This cache is derived, so malformed JSON can be rebuilt. An I/O
|
|
93
|
-
// failure is different: never hide an unreadable cache as missing.
|
|
94
|
-
if (err.code === "ENOENT" || err instanceof SyntaxError) return {};
|
|
95
|
-
throw err;
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
function saveOffsets(store, offsets) {
|
|
100
|
-
const path = offsetsPath(store);
|
|
101
|
-
mkdirSync(dirname(path), { recursive: true });
|
|
102
|
-
writeFileSync(path, JSON.stringify(offsets));
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
// Read bytes of `path` in [start, size).
|
|
106
|
-
function readFrom(path, start, size) {
|
|
107
|
-
const fd = openSync(path, "r");
|
|
108
|
-
try {
|
|
109
|
-
const buf = Buffer.alloc(size - start);
|
|
110
|
-
let done = 0;
|
|
111
|
-
while (done < buf.length) {
|
|
112
|
-
const n = readSync(fd, buf, done, buf.length - done, start + done);
|
|
113
|
-
if (n === 0) throw new Error(`short read of session/journal source: ${path}`);
|
|
114
|
-
done += n;
|
|
115
|
-
}
|
|
116
|
-
return buf.subarray(0, done);
|
|
117
|
-
} finally {
|
|
118
|
-
closeSync(fd);
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
// The journal's last whole turn, read from its tail without parsing the whole
|
|
123
|
-
// file (backward scan, doubling window). null when the journal is missing or
|
|
124
|
-
// holds none.
|
|
125
|
-
function lastJournalTurn(store, streamId) {
|
|
126
|
-
const path = store.journalPath(streamId);
|
|
127
|
-
let size;
|
|
128
|
-
try {
|
|
129
|
-
size = statSync(path).size;
|
|
130
|
-
} catch (err) {
|
|
131
|
-
if (err.code === "ENOENT") return null;
|
|
132
|
-
throw err;
|
|
133
|
-
}
|
|
134
|
-
let window = 64 * 1024;
|
|
135
|
-
while (true) {
|
|
136
|
-
const start = Math.max(0, size - window);
|
|
137
|
-
const tail = readFrom(path, start, size);
|
|
138
|
-
// Last complete line: ignore a torn final line (no trailing newline).
|
|
139
|
-
const text = tail.toString("utf8");
|
|
140
|
-
const endsClean = text.endsWith("\n");
|
|
141
|
-
const parts = text.split("\n").filter((l) => l.trim() !== "");
|
|
142
|
-
const candidates = endsClean ? parts : parts.slice(0, -1);
|
|
143
|
-
for (let i = candidates.length - 1; i >= 0; i--) {
|
|
144
|
-
// The first line of the window may be a fragment; only trust a line
|
|
145
|
-
// we know is whole (preceded by a newline inside the window, or the
|
|
146
|
-
// window covers the whole file).
|
|
147
|
-
if (i === 0 && start > 0) break;
|
|
148
|
-
try {
|
|
149
|
-
return JSON.parse(candidates[i]);
|
|
150
|
-
} catch {
|
|
151
|
-
continue; // fragment or torn line: look further back
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
if (start === 0) return null;
|
|
155
|
-
window *= 2;
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
// Rebuild a lost offset from the journal: its last turn knows the line
|
|
160
|
-
// number it came from; walk the source to that line's byte offset.
|
|
161
|
-
// Honest limit: an in-place REWRITE of already-captured lines is not
|
|
162
|
-
// detected (only growth is; a shrink triggers a rescan via the size
|
|
163
|
-
// check in the caller). Transcript writers are append-only in practice.
|
|
164
|
-
// Both are read in bounded memory: the journal's last turn from its tail, the
|
|
165
|
-
// source's lines counted chunk by chunk (awebai/oats#456).
|
|
166
|
-
// `last` is the journal's last turn (lastJournalTurn), null when it has none.
|
|
167
|
-
function offsetFromJournal(last, sourcePath, final, sourceBytes) {
|
|
168
|
-
if (!last) return { bytes: 0, line: 0, lastTs: "" };
|
|
169
|
-
const lastLine = last.provenance?.origin?.line ?? 0;
|
|
170
|
-
const { line, offset } = sourceBytes ? lineOffset([sourceBytes], lastLine) : lineOffsetOfFile(sourcePath, lastLine);
|
|
171
|
-
if (final && line < lastLine) throw new Error(`session source is shorter than its captured journal: ${sourcePath}`);
|
|
172
|
-
return { bytes: offset, line, lastTs: last.ts ?? "" };
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
// The byte offset just past the `lastLine`-th complete line of the bytes in
|
|
176
|
-
// `chunks` (in order), and how many complete lines it found (fewer when the
|
|
177
|
-
// source is shorter).
|
|
178
|
-
function lineOffset(chunks, lastLine) {
|
|
179
|
-
let line = 0, offset = 0;
|
|
180
|
-
for (const chunk of chunks) {
|
|
181
|
-
let from = 0;
|
|
182
|
-
while (line < lastLine) {
|
|
183
|
-
const nl = chunk.indexOf(10, from);
|
|
184
|
-
if (nl === -1) break;
|
|
185
|
-
line++;
|
|
186
|
-
from = nl + 1;
|
|
187
|
-
}
|
|
188
|
-
offset += line < lastLine ? chunk.length : from;
|
|
189
|
-
if (line >= lastLine) break;
|
|
190
|
-
}
|
|
191
|
-
return { line, offset };
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
function lineOffsetOfFile(path, lastLine) {
|
|
195
|
-
const fd = openSync(path, "r");
|
|
196
|
-
try { return lineOffset(fileChunks(fd), lastLine); }
|
|
197
|
-
finally { closeSync(fd); }
|
|
198
|
-
}
|
|
199
|
-
|
|
200
84
|
// Source bytes of turns held before they are appended: what a pass keeps of a
|
|
201
85
|
// transcript at once, whatever its size.
|
|
202
86
|
const FLUSH_BYTES = 8 * 1024 * 1024;
|
|
203
87
|
|
|
204
|
-
// The COMPLETE lines of `
|
|
88
|
+
// The COMPLETE lines of `chunks`, one at a time, with their stamps:
|
|
205
89
|
// { text, ts, bytes }. `walk` records where the walk ended (`scanned`, bytes)
|
|
206
90
|
// and why it stopped early (`reason`): a torn tail, invalid UTF-8 (decoding
|
|
207
91
|
// replacement characters would change both the verbatim line and its byte
|
|
208
92
|
// offset, possibly treating a later fragment as a record) or a line beyond
|
|
209
93
|
// V8's string limit.
|
|
210
|
-
function* completeLines(
|
|
94
|
+
function* completeLines(chunks, walk) {
|
|
211
95
|
let scanned = 0;
|
|
212
96
|
walk.scanned = 0;
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
const bytes = chunk.subarray(scanned, nl);
|
|
97
|
+
for (const line of bufferLines(chunks)) {
|
|
98
|
+
if (line[line.length - 1] !== 10) { walk.reason = "torn-tail"; return; }
|
|
99
|
+
const bytes = line.subarray(0, -1);
|
|
217
100
|
if (!isUtf8(bytes)) { walk.reason = "invalid-utf8"; return; }
|
|
218
101
|
let text;
|
|
219
102
|
try { text = bytes.toString("utf8"); }
|
|
@@ -222,7 +105,7 @@ function* completeLines(chunk, walk) {
|
|
|
222
105
|
walk.reason = "oversized-line";
|
|
223
106
|
return;
|
|
224
107
|
}
|
|
225
|
-
const lineBytes =
|
|
108
|
+
const lineBytes = line.length;
|
|
226
109
|
let ts = "";
|
|
227
110
|
if (text.trim() !== "") {
|
|
228
111
|
try {
|
|
@@ -240,7 +123,7 @@ function* completeLines(chunk, walk) {
|
|
|
240
123
|
|
|
241
124
|
// One reconciliation pass for one format: one turn per NEW complete line
|
|
242
125
|
// of every session file under `roots`. Unstamped leading lines are held
|
|
243
|
-
// until the file shows its first timestamp (then they carry it
|
|
126
|
+
// until the file shows its first timestamp (then they carry it backward),
|
|
244
127
|
// so every turn is stamped and ts stays a pure function of the source.
|
|
245
128
|
// `files` accepts discovery entries (including their attribution snapshot) or
|
|
246
129
|
// explicit paths for callers not making home-attribution claims. It pins a set,
|
|
@@ -250,14 +133,17 @@ function* completeLines(chunk, walk) {
|
|
|
250
133
|
// stability through the pass. The caller must quiesce writers for retirement;
|
|
251
134
|
// a performed pass is a snapshot, not a promise about future writes.
|
|
252
135
|
//
|
|
253
|
-
// A file given as `{ path, pinned
|
|
136
|
+
// A file given as `{ path, pinned }` is PINNED (stageSnapshot or legacy { bytes, stat }): its caller opened it once and read these bytes from
|
|
254
137
|
// that descriptor (`capture --file`). It is captured from them as a final pass, never opened again, so the
|
|
255
138
|
// turns are exactly those bytes; `path` keys its offsets as for any other file.
|
|
256
|
-
export function captureSessions(store, { owner, roots, files, format = "cc", ignore = null, final = false }) {
|
|
139
|
+
export function captureSessions(store, { owner, roots, files, format = "cc", ignore = null, final = false, collectIssues = true }) {
|
|
257
140
|
const fmt = SESSION_FORMATS[format];
|
|
258
141
|
if (!fmt) throw new Error(`unknown session format ${format}`);
|
|
259
142
|
const ign = ignore ?? loadIgnore(store.root);
|
|
260
|
-
const offsets =
|
|
143
|
+
const offsets = new DerivedState(store.root);
|
|
144
|
+
try {
|
|
145
|
+
offsets.db.exec("CREATE TEMP TABLE captured_streams (stream TEXT PRIMARY KEY)");
|
|
146
|
+
const markStream = offsets.db.prepare("INSERT OR IGNORE INTO captured_streams VALUES (?)");
|
|
261
147
|
let sessions = 0;
|
|
262
148
|
let appended = 0;
|
|
263
149
|
let unchanged = 0;
|
|
@@ -265,9 +151,14 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
|
|
|
265
151
|
let ignored = 0;
|
|
266
152
|
let incomplete = 0;
|
|
267
153
|
const issues = []; // source metadata only, never native record contents
|
|
268
|
-
|
|
154
|
+
let streams = 0;
|
|
155
|
+
const issue = (value) => { if (collectIssues) issues.push(value); };
|
|
269
156
|
|
|
270
|
-
|
|
157
|
+
// Reconcile exclusions even when a blocked input has disappeared from discovery.
|
|
158
|
+
for (const { key, value } of offsets.entries("set-aside")) {
|
|
159
|
+
if (value.owner === owner && value.source === fmt.source && ign.ignores(value.path, [basename(value.path), fmt.sessionId(value.path), ...(fmt.ignoreKeys?.(value.path) ?? [])])) offsets.delete("set-aside", key);
|
|
160
|
+
}
|
|
161
|
+
for (const file of files ?? (fmt.iterateFiles ?? fmt.listFiles)(roots)) {
|
|
271
162
|
const path = typeof file === "string" ? file : file.path;
|
|
272
163
|
const pinned = typeof file === "string" ? null : file.pinned ?? null;
|
|
273
164
|
const expected = typeof file === "string" ? null : file.snapshot;
|
|
@@ -280,53 +171,85 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
|
|
|
280
171
|
ignored++;
|
|
281
172
|
continue; // never opened: nothing stored, nothing remembered
|
|
282
173
|
}
|
|
283
|
-
const
|
|
174
|
+
const streamId = `${owner}~${fmt.source}.${sessionId}`;
|
|
175
|
+
const offKey = `${streamId}:${path}`;
|
|
176
|
+
const blocked = offsets.get("set-aside", offKey);
|
|
177
|
+
if (blocked && !blocked.retry) {
|
|
178
|
+
incomplete++;
|
|
179
|
+
issue({ source: fmt.source, path, reason: blocked.reason, offset: blocked.offset, line: blocked.line });
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
const fd = pinned ? null : openSync(path, constants.O_RDONLY | constants.O_NONBLOCK | (capturedPi ? constants.O_NOFOLLOW : 0));
|
|
183
|
+
let staged;
|
|
284
184
|
try {
|
|
285
185
|
if (capturedPi) assertProtectedDescriptor(fd, path, capturedPi);
|
|
286
186
|
const stat = pinned ? pinned.stat : fstatSync(fd);
|
|
187
|
+
const version = statVersion(pinned ? pinned.stat : fstatSync(fd, { bigint: true }));
|
|
287
188
|
const snapshot = { ...identity(stat), ...(capturedPi ? { capturedPi } : {}) };
|
|
288
189
|
if (expected) assertIdentity(stat, expected, path); // BEFORE reading bytes
|
|
289
190
|
// Final home capture stages a descriptor-pinned snapshot. All attribution
|
|
290
191
|
// and stability checks precede the first append, never a post-write alarm.
|
|
291
|
-
|
|
292
|
-
|
|
192
|
+
staged = !pinned && (final || expected || capturedPi) ? stageSnapshot(fd, stat, path, capturedPi, store.root) : null;
|
|
193
|
+
let source = pinned ?? staged;
|
|
194
|
+
const chunks = (start = 0, end = stat.size) => source
|
|
195
|
+
? source.bytes ? [source.bytes.subarray(start, end)] : source.chunks(start, end)
|
|
196
|
+
: sourceChunks(fd, start, end, path, capturedPi);
|
|
197
|
+
const witness = (end) => {
|
|
198
|
+
// At most 4 KiB; identity and this boundary are witnesses, not proof of
|
|
199
|
+
// the unobserved prefix. See the append-only input contract.
|
|
200
|
+
const bytes = Buffer.concat([...chunks(Math.max(0, end - 4096), end)]);
|
|
201
|
+
return digest(bytes);
|
|
202
|
+
};
|
|
203
|
+
if (expected && (source.bytes ? digest(source.bytes.subarray(0, expected.size)) : source.prefixHash(expected.size)) !== expected.hash) {
|
|
293
204
|
throw new Error(`session source content changed since attribution: ${path}`);
|
|
294
205
|
}
|
|
295
|
-
if (
|
|
206
|
+
if (source) snapshot.hash = source.bytes ? digest(source.bytes) : source.hash;
|
|
296
207
|
const verifySource = () => {
|
|
297
208
|
if (pinned) return; // the bytes are the source: nothing is read from the path again
|
|
298
209
|
guardCapturedPath(path, capturedPi);
|
|
299
|
-
if (
|
|
210
|
+
if (source) verifySnapshot(fd, path, snapshot);
|
|
300
211
|
};
|
|
301
212
|
verifySource();
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
213
|
+
let state = offsets.get(fmt.source, offKey);
|
|
214
|
+
const generation = offsets.get("reset", `${owner}:${fmt.source}`) ?? "";
|
|
215
|
+
const journalVersion = fileVersion(store.journalPath(streamId));
|
|
216
|
+
if (state && (!Number.isSafeInteger(state.bytes) || state.bytes < 0 || state.bytes > stat.size ||
|
|
217
|
+
!Number.isSafeInteger(state.line) || state.line < 0 || typeof state.lastTs !== "string")) state = null;
|
|
218
|
+
const prior = state?.version;
|
|
219
|
+
const sameInput = prior && sameVersion(prior, version);
|
|
220
|
+
const appendGrowth = prior && prior.dev === version.dev && prior.ino === version.ino && version.size > prior.size &&
|
|
221
|
+
state.bytes <= version.size && witness(state.bytes) === state.witness;
|
|
222
|
+
const recover = !state || state.generation !== generation || !sameVersion(state.journalVersion, journalVersion) ||
|
|
223
|
+
(!sameInput && !appendGrowth) || blocked;
|
|
224
|
+
if (recover) {
|
|
225
|
+
// A complete comparison precedes ANY append from this input. Pin the
|
|
226
|
+
// bytes on disk so a later batch cannot reopen a mutable source.
|
|
227
|
+
if (!source && journalVersion?.size) {
|
|
228
|
+
staged = stageSnapshot(fd, stat, path, capturedPi, store.root);
|
|
229
|
+
source = staged; snapshot.hash = staged.hash;
|
|
230
|
+
}
|
|
231
|
+
state = compareTranscriptPrefix(store, streamId, offsets, completeLines(chunks(), {}));
|
|
232
|
+
verifySource();
|
|
233
|
+
if (state.conflict) {
|
|
234
|
+
const row = { owner, source: fmt.source, path, ...state.conflict, retry: false };
|
|
235
|
+
offsets.set("set-aside", offKey, row);
|
|
236
|
+
incomplete++;
|
|
237
|
+
issue({ source: fmt.source, path, ...state.conflict });
|
|
238
|
+
continue;
|
|
239
|
+
}
|
|
240
|
+
offsets.delete("set-aside", offKey);
|
|
320
241
|
}
|
|
321
|
-
|
|
242
|
+
const checkpoint = (value) => offsets.set(fmt.source, offKey, {
|
|
243
|
+
...value, version, witness: witness(value.bytes), generation,
|
|
244
|
+
journalVersion: fileVersion(store.journalPath(streamId)),
|
|
245
|
+
});
|
|
322
246
|
if (stat.size <= state.bytes) {
|
|
323
247
|
unchanged++;
|
|
324
|
-
|
|
248
|
+
checkpoint(state);
|
|
325
249
|
verifySource();
|
|
326
250
|
continue;
|
|
327
251
|
}
|
|
328
252
|
|
|
329
|
-
const chunk = sourceBytes ? sourceBytes.subarray(state.bytes) : readRange(fd, state.bytes, stat.size, path, capturedPi);
|
|
330
253
|
// Every turn needs a stamp. Leading lines before the file's first stamp
|
|
331
254
|
// carry it backward (deterministic: the file's first stamp is invariant
|
|
332
255
|
// however capture is scheduled); if the file has shown no stamp at all
|
|
@@ -336,13 +259,13 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
|
|
|
336
259
|
const stopped = (walk) => {
|
|
337
260
|
if (!walk.reason) return;
|
|
338
261
|
incomplete++;
|
|
339
|
-
|
|
262
|
+
issue({ source: fmt.source, path, reason: walk.reason, offset: state.bytes + walk.scanned });
|
|
340
263
|
};
|
|
341
264
|
let lastTs = state.lastTs ?? "";
|
|
342
265
|
if (!lastTs) {
|
|
343
266
|
const scan = {};
|
|
344
267
|
let any = false;
|
|
345
|
-
for (const l of completeLines(
|
|
268
|
+
for (const l of completeLines(chunks(state.bytes), scan)) {
|
|
346
269
|
any = true;
|
|
347
270
|
if (l.ts) { lastTs = l.ts; break; }
|
|
348
271
|
}
|
|
@@ -350,7 +273,7 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
|
|
|
350
273
|
stopped(scan);
|
|
351
274
|
if (any) {
|
|
352
275
|
held++;
|
|
353
|
-
|
|
276
|
+
issue({ source: fmt.source, path, reason: "unstamped", offset: state.bytes });
|
|
354
277
|
}
|
|
355
278
|
verifySource();
|
|
356
279
|
continue; // do not advance; retry when a stamp exists
|
|
@@ -359,23 +282,24 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
|
|
|
359
282
|
verifySource(); // parsing/recovery may take time; still no journal write yet
|
|
360
283
|
// Turns flush to the journal in bounded batches, so memory stays flat
|
|
361
284
|
// however large the backlog. A crash between flushes cannot
|
|
362
|
-
// duplicate:
|
|
363
|
-
//
|
|
285
|
+
// duplicate: checkpoints follow each append and the durable ID lookup
|
|
286
|
+
// reconciles a crash between journal fsync and the source checkpoint.
|
|
364
287
|
let fresh = [];
|
|
365
288
|
let freshBytes = 0;
|
|
366
289
|
const flush = () => {
|
|
367
290
|
if (fresh.length === 0) return;
|
|
368
291
|
verifySource(); // EVERY batch; earlier committed batches are never rolled back
|
|
369
|
-
store.
|
|
370
|
-
streams.
|
|
371
|
-
appended +=
|
|
292
|
+
const added = store.appendUniqueBatch(streamId, fresh);
|
|
293
|
+
if (added) streams += Number(markStream.run(streamId).changes);
|
|
294
|
+
appended += added;
|
|
295
|
+
checkpoint({ bytes: state.bytes + consumed, line: lineNo, lastTs });
|
|
372
296
|
fresh = [];
|
|
373
297
|
freshBytes = 0;
|
|
374
298
|
};
|
|
375
299
|
let lineNo = state.line;
|
|
376
300
|
let consumed = 0;
|
|
377
301
|
const walk = {};
|
|
378
|
-
for (const l of completeLines(
|
|
302
|
+
for (const l of completeLines(chunks(state.bytes), walk)) {
|
|
379
303
|
if (l.ts) lastTs = l.ts;
|
|
380
304
|
lineNo++;
|
|
381
305
|
consumed += l.bytes;
|
|
@@ -387,22 +311,14 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
|
|
|
387
311
|
),
|
|
388
312
|
);
|
|
389
313
|
freshBytes += l.bytes;
|
|
390
|
-
if (freshBytes >= FLUSH_BYTES) flush();
|
|
314
|
+
if (fresh.length >= 2000 || freshBytes >= FLUSH_BYTES) flush();
|
|
391
315
|
}
|
|
392
316
|
stopped(walk);
|
|
393
|
-
const grew = fresh.length > 0 || consumed > 0;
|
|
394
317
|
flush();
|
|
395
|
-
|
|
396
|
-
// Persist after every file that advanced, not once per pass: a crash
|
|
397
|
-
// later in the pass must not leave THIS file's on-disk offset stale
|
|
398
|
-
// (a stale-but-present offset skips journal recovery, and the rescan
|
|
399
|
-
// re-appends duplicate turn lines — logically deduped by id, but
|
|
400
|
-
// wasted append-only bytes).
|
|
401
|
-
if (grew) saveOffsets(store, offsets);
|
|
318
|
+
checkpoint({ bytes: state.bytes + consumed, line: lineNo, lastTs });
|
|
402
319
|
verifySource();
|
|
403
|
-
} finally { if (fd !== null) closeSync(fd); }
|
|
320
|
+
} finally { try { staged?.close(); } finally { if (fd !== null) closeSync(fd); } }
|
|
404
321
|
}
|
|
405
|
-
saveOffsets(store, offsets);
|
|
406
322
|
return {
|
|
407
323
|
sessions,
|
|
408
324
|
appended,
|
|
@@ -412,20 +328,26 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
|
|
|
412
328
|
incomplete,
|
|
413
329
|
complete: held === 0 && incomplete === 0,
|
|
414
330
|
issues,
|
|
415
|
-
streams
|
|
331
|
+
streams,
|
|
416
332
|
stream: `${owner}~${fmt.source}.*`,
|
|
417
333
|
};
|
|
334
|
+
} finally { offsets.close(); }
|
|
418
335
|
}
|
|
419
336
|
|
|
420
337
|
// One pass over every known format at its default roots. The ignore list
|
|
421
338
|
// is loaded once and shared across formats.
|
|
422
|
-
export function captureAllSessions(store, { owner, ignore = null }) {
|
|
339
|
+
export function captureAllSessions(store, { owner, ignore = null, collectIssues = true }) {
|
|
423
340
|
const ign = ignore ?? loadIgnore(store.root);
|
|
424
341
|
const results = [];
|
|
425
342
|
for (const format of Object.keys(SESSION_FORMATS)) {
|
|
426
343
|
const roots = SESSION_FORMATS[format].defaultRoots();
|
|
427
|
-
if (roots.length === 0)
|
|
428
|
-
|
|
344
|
+
if (roots.length === 0) {
|
|
345
|
+
// Missing defaults must not keep an ignored, previously blocked path
|
|
346
|
+
// active merely because it no longer appears in filesystem discovery.
|
|
347
|
+
captureSessions(store, { owner, roots, format, ignore: ign, collectIssues: false });
|
|
348
|
+
continue;
|
|
349
|
+
}
|
|
350
|
+
results.push(captureSessions(store, { owner, roots, format, ignore: ign, collectIssues }));
|
|
429
351
|
}
|
|
430
352
|
return results;
|
|
431
353
|
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { closeSync, constants, fstatSync, openSync, readSync } from "node:fs";
|
|
2
|
+
import { createRequire } from "node:module";
|
|
3
|
+
import { pathToFileURL } from "node:url";
|
|
4
|
+
import { readIssueBacklog, supportsReadOnlySqlite } from "./capture-issues.mjs";
|
|
5
|
+
const require = createRequire(import.meta.url);
|
|
6
|
+
|
|
7
|
+
// Supported capture writers use rollback journaling. SQLite may create WAL
|
|
8
|
+
// sidecars even for a readOnly connection; refuse that mode before SQLite opens
|
|
9
|
+
// it. This is not protection against an external actor racing a journal-mode
|
|
10
|
+
// change or file replacement. No supported writer performs either operation.
|
|
11
|
+
export function readCaptureDatabase(path) {
|
|
12
|
+
if (!supportsReadOnlySqlite(process.versions.node)) throw new Error("read-only capture health requires newer Node with read-only SQLite (22.13+ on 22.x, 23.2+ on 23.x, or newer major)");
|
|
13
|
+
let fd;
|
|
14
|
+
try { fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK); }
|
|
15
|
+
catch (err) { if (err.code === "ENOENT") return null; throw err; }
|
|
16
|
+
try {
|
|
17
|
+
if (!fstatSync(fd).isFile()) throw new Error("capture.db is not a regular file");
|
|
18
|
+
const header = Buffer.alloc(100);
|
|
19
|
+
if (readSync(fd, header, 0, header.length, 0) !== header.length ||
|
|
20
|
+
header.subarray(0, 16).toString() !== "SQLite format 3\0") throw new Error("invalid capture.db header");
|
|
21
|
+
if (header[18] === 2 || header[19] === 2) throw new Error("WAL capture.db is unsupported by the read-only health reader; use a supported rollback-journal database");
|
|
22
|
+
if (header[18] !== 1 || header[19] !== 1) throw new Error("unsupported capture.db journal format");
|
|
23
|
+
} finally { closeSync(fd); }
|
|
24
|
+
|
|
25
|
+
let DatabaseSync;
|
|
26
|
+
try { ({ DatabaseSync } = require("node:sqlite")); }
|
|
27
|
+
catch (err) { throw new Error(`read-only SQLite is unavailable; rerun health with Node 22.13+ on 22.x, 23.2+ on 23.x, or newer major: ${err.message}`); }
|
|
28
|
+
const db = new DatabaseSync(path, { open: false, readOnly: true });
|
|
29
|
+
db.open();
|
|
30
|
+
try {
|
|
31
|
+
db.exec("PRAGMA busy_timeout = 250; PRAGMA query_only = ON; PRAGMA cache_size = -2048; PRAGMA temp_store = FILE");
|
|
32
|
+
return readIssueBacklog(db, { validate: true });
|
|
33
|
+
} finally { db.close(); }
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) {
|
|
37
|
+
try { console.log(JSON.stringify({ backlog: readCaptureDatabase(process.argv[2]) })); }
|
|
38
|
+
catch (err) { console.log(JSON.stringify({ error: err.message })); process.exitCode = 1; }
|
|
39
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { statSync } from "node:fs";
|
|
2
|
+
import { spawnSync } from "node:child_process";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
// Feature landed separately on the two release lines; 23.0/23.1 lack it.
|
|
6
|
+
export function supportsReadOnlySqlite(version) {
|
|
7
|
+
const match = /^(\d+)\.(\d+)\.(\d+)$/.exec(version);
|
|
8
|
+
if (!match) return false;
|
|
9
|
+
const major = Number(match[1]), minor = Number(match[2]);
|
|
10
|
+
return major > 23 || major === 23 && minor >= 2 || major === 22 && minor >= 12;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
// The command snapshot and live health diagnosis use the same bounded
|
|
14
|
+
// projection. This function owns one read transaction, not the capture lock.
|
|
15
|
+
export function readIssueBacklog(db, { pending = false, validate = false } = {}) {
|
|
16
|
+
db.exec("BEGIN");
|
|
17
|
+
try {
|
|
18
|
+
if (validate) {
|
|
19
|
+
if (db.prepare("PRAGMA user_version").get().user_version !== 0) throw new Error("unsupported capture database version");
|
|
20
|
+
const columns = db.prepare("PRAGMA table_info(state)").all();
|
|
21
|
+
if (columns.length !== 3 || columns.some((column, i) =>
|
|
22
|
+
column.name !== ["kind", "key", "value"][i] || column.type !== "TEXT" || column.pk !== [1, 2, 0][i] ||
|
|
23
|
+
column.notnull !== [0, 0, 1][i])) throw new Error("unsupported capture issue-table schema");
|
|
24
|
+
}
|
|
25
|
+
const count = Number(db.prepare("SELECT count(*) AS n FROM state WHERE kind = 'set-aside'").get().n);
|
|
26
|
+
const sample = db.prepare(`SELECT value FROM state WHERE kind = 'set-aside'
|
|
27
|
+
ORDER BY json_extract(value, '$.source'), json_extract(value, '$.path'), key LIMIT 100`).all();
|
|
28
|
+
const setAside = sample.map(({ value }) => {
|
|
29
|
+
const { source, path, offset, reason, line } = JSON.parse(value);
|
|
30
|
+
if (!["aw", "cc", "pi", "codex"].includes(source) || typeof path !== "string" || typeof reason !== "string" ||
|
|
31
|
+
!Number.isSafeInteger(offset) || offset < 0 || (line !== undefined && (!Number.isSafeInteger(line) || line <= 0))) {
|
|
32
|
+
throw new Error("malformed capture issue row");
|
|
33
|
+
}
|
|
34
|
+
return { source, path, offset, reason, ...(line === undefined ? {} : { line }) };
|
|
35
|
+
});
|
|
36
|
+
db.exec("COMMIT");
|
|
37
|
+
return { pending: pending || count > 0, setAsideCount: count, setAside, truncated: count > sample.length };
|
|
38
|
+
} catch (err) { db.exec("ROLLBACK"); throw err; }
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// A separate process prevents a raw header descriptor close from releasing
|
|
42
|
+
// process-owned POSIX locks held by another SQLite connection in this caller.
|
|
43
|
+
// It loads no capture writers, retains SQLite locking, and inherits no hooks.
|
|
44
|
+
export function readCurrentIssueBacklog(root) {
|
|
45
|
+
const path = join(root, "index", "capture.db");
|
|
46
|
+
try { if (!statSync(path).isFile()) throw new Error("capture.db is not a regular file"); }
|
|
47
|
+
catch (err) { if (err.code === "ENOENT") return null; throw err; }
|
|
48
|
+
if (!supportsReadOnlySqlite(process.versions.node)) throw new Error("read-only capture health needs newer Node with read-only SQLite: use Node 22.13+ on 22.x, 23.2+ on 23.x, or a newer major version");
|
|
49
|
+
const env = Object.fromEntries(["HOME", "TMPDIR", "XDG_CACHE_HOME"].filter(key => process.env[key] !== undefined).map(key => [key, process.env[key]]));
|
|
50
|
+
const result = spawnSync(process.execPath, [
|
|
51
|
+
...process.execArgv.filter(arg => arg === "--experimental-sqlite"),
|
|
52
|
+
fileURLToPath(new URL("./capture-db-reader.mjs", import.meta.url)), path,
|
|
53
|
+
], { env, encoding: "utf8", maxBuffer: 1024 * 1024, timeout: 10000, killSignal: "SIGKILL" });
|
|
54
|
+
if (result.error) throw result.error;
|
|
55
|
+
let response;
|
|
56
|
+
try { response = JSON.parse(result.stdout); }
|
|
57
|
+
catch { throw new Error(`capture database reader failed (${result.signal ?? result.status}): ${result.stderr?.trim() || "no valid response"}`); }
|
|
58
|
+
if (result.status !== 0 || response.error) throw new Error(response.error ?? "capture database reader failed");
|
|
59
|
+
const b = response.backlog;
|
|
60
|
+
if (!b || typeof b.pending !== "boolean" || !Number.isSafeInteger(b.setAsideCount) || b.setAsideCount < 0 ||
|
|
61
|
+
!Array.isArray(b.setAside) || b.setAside.length !== Math.min(b.setAsideCount, 100) ||
|
|
62
|
+
b.truncated !== (b.setAsideCount > 100)) throw new Error("malformed capture database reader output");
|
|
63
|
+
return b;
|
|
64
|
+
}
|