@awebai/oats 0.54.0 → 0.55.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/bin/oats.mjs +5 -5
  2. package/docs/desktop-cli-api.md +15 -4
  3. package/docs/integrations.md +2 -2
  4. package/docs/knowledge.md +14 -2
  5. package/docs/official-catalog.md +4 -4
  6. package/docs/packages.md +10 -10
  7. package/docs/release-notes/v0.55.0.md +158 -0
  8. package/docs/release-notes/v0.55.1.md +45 -0
  9. package/docs/servers.md +13 -3
  10. package/docs/workspaces.md +1 -1
  11. package/lib/servers.mjs +82 -34
  12. package/package-catalog.json +3 -3
  13. package/package.json +1 -1
  14. package/packages/client/cli-locator.mjs +2 -2
  15. package/packages/client/workspace-views.mjs +4 -3
  16. package/packages/record/README.md +58 -13
  17. package/packages/record/bin/capture.mjs +107 -53
  18. package/packages/record/bin/recall.mjs +3 -2
  19. package/packages/record/docs/capture-service.md +193 -0
  20. package/packages/record/docs/capture-status.schema.json +100 -0
  21. package/packages/record/lib/capture-aw.mjs +96 -94
  22. package/packages/record/lib/capture-cc.mjs +109 -187
  23. package/packages/record/lib/capture-db-reader.mjs +39 -0
  24. package/packages/record/lib/capture-issues.mjs +64 -0
  25. package/packages/record/lib/capture-scratch.mjs +111 -0
  26. package/packages/record/lib/capture-state.mjs +105 -0
  27. package/packages/record/lib/derived-state.mjs +60 -0
  28. package/packages/record/lib/formats.mjs +31 -31
  29. package/packages/record/lib/index-db.mjs +74 -34
  30. package/packages/record/lib/receipt-summary.mjs +62 -0
  31. package/packages/record/lib/session-snapshot.mjs +56 -1
  32. package/packages/record/lib/sqlite-rows.mjs +14 -0
  33. package/packages/record/lib/store.mjs +122 -43
  34. package/packages/record/lib/transcript-prefix.mjs +40 -0
  35. package/skills/oats-getting-started/SKILL.md +1 -1
@@ -12,8 +12,8 @@
12
12
  // original transcript is reconstructible by concatenating body.line.
13
13
  //
14
14
  // Incremental by source byte offset (cache under index/, derived state:
15
- // losing it is recovered from the journal's own last line number, never
16
- // by re-appending). Reconciliation is the capture: hooks and watchers
15
+ // losing it requires comparison against the journal's positional prefix
16
+ // before resuming, never blind re-appending). Reconciliation is the capture: hooks and watchers
17
17
  // only decide when to run it.
18
18
  //
19
19
  // Files matching the record's ignore list (`<root>/ignore`, see
@@ -21,15 +21,17 @@
21
21
  // un-ignoring a file later makes the next pass capture it normally.
22
22
 
23
23
  import { isUtf8 } from "node:buffer";
24
- import { closeSync, constants, fstatSync, mkdirSync, openSync, readFileSync, readSync, statSync, writeFileSync } from "node:fs";
25
- import { basename, dirname, join } from "node:path";
24
+ import { closeSync, constants, fstatSync, openSync } from "node:fs";
25
+ import { basename } from "node:path";
26
26
 
27
27
  import { finishTurn } from "./canonical.mjs";
28
28
  import { jsonlLines, SESSION_FORMATS } from "./formats.mjs";
29
29
  import { loadIgnore } from "./ignore.mjs";
30
- import { assertIdentity, assertProtectedDescriptor, digest, identity, readRange, verifySnapshot } from "./session-snapshot.mjs";
30
+ import { assertIdentity, assertProtectedDescriptor, digest, identity, sourceChunks, stageSnapshot, verifySnapshot } from "./session-snapshot.mjs";
31
31
  import { guardCapturedPath } from "./native-history.mjs";
32
- import { fileChunks } from "./file-lines.mjs";
32
+ import { bufferLines } from "./file-lines.mjs";
33
+ import { DerivedState, fileVersion, sameVersion, statVersion } from "./derived-state.mjs";
34
+ import { compareTranscriptPrefix } from "./transcript-prefix.mjs";
33
35
  import { isDeepStrictEqual } from "node:util";
34
36
 
35
37
  export const SESSION_STREAM_SOURCE = "cc";
@@ -79,141 +81,22 @@ export function eventTurnCore({ owner, source, sessionId, line, lineNo, ts }) {
79
81
  };
80
82
  }
81
83
 
82
- // Offset cache: `<stream>:<absolute source path>` -> { bytes consumed,
83
- // lines emitted, last carried timestamp }. Derived state under index/.
84
- function offsetsPath(store) {
85
- return join(store.root, "index", "capture-offsets.json");
86
- }
87
-
88
- function loadOffsets(store) {
89
- try {
90
- return JSON.parse(readFileSync(offsetsPath(store), "utf8"));
91
- } catch (err) {
92
- // This cache is derived, so malformed JSON can be rebuilt. An I/O
93
- // failure is different: never hide an unreadable cache as missing.
94
- if (err.code === "ENOENT" || err instanceof SyntaxError) return {};
95
- throw err;
96
- }
97
- }
98
-
99
- function saveOffsets(store, offsets) {
100
- const path = offsetsPath(store);
101
- mkdirSync(dirname(path), { recursive: true });
102
- writeFileSync(path, JSON.stringify(offsets));
103
- }
104
-
105
- // Read bytes of `path` in [start, size).
106
- function readFrom(path, start, size) {
107
- const fd = openSync(path, "r");
108
- try {
109
- const buf = Buffer.alloc(size - start);
110
- let done = 0;
111
- while (done < buf.length) {
112
- const n = readSync(fd, buf, done, buf.length - done, start + done);
113
- if (n === 0) throw new Error(`short read of session/journal source: ${path}`);
114
- done += n;
115
- }
116
- return buf.subarray(0, done);
117
- } finally {
118
- closeSync(fd);
119
- }
120
- }
121
-
122
- // The journal's last whole turn, read from its tail without parsing the whole
123
- // file (backward scan, doubling window). null when the journal is missing or
124
- // holds none.
125
- function lastJournalTurn(store, streamId) {
126
- const path = store.journalPath(streamId);
127
- let size;
128
- try {
129
- size = statSync(path).size;
130
- } catch (err) {
131
- if (err.code === "ENOENT") return null;
132
- throw err;
133
- }
134
- let window = 64 * 1024;
135
- while (true) {
136
- const start = Math.max(0, size - window);
137
- const tail = readFrom(path, start, size);
138
- // Last complete line: ignore a torn final line (no trailing newline).
139
- const text = tail.toString("utf8");
140
- const endsClean = text.endsWith("\n");
141
- const parts = text.split("\n").filter((l) => l.trim() !== "");
142
- const candidates = endsClean ? parts : parts.slice(0, -1);
143
- for (let i = candidates.length - 1; i >= 0; i--) {
144
- // The first line of the window may be a fragment; only trust a line
145
- // we know is whole (preceded by a newline inside the window, or the
146
- // window covers the whole file).
147
- if (i === 0 && start > 0) break;
148
- try {
149
- return JSON.parse(candidates[i]);
150
- } catch {
151
- continue; // fragment or torn line: look further back
152
- }
153
- }
154
- if (start === 0) return null;
155
- window *= 2;
156
- }
157
- }
158
-
159
- // Rebuild a lost offset from the journal: its last turn knows the line
160
- // number it came from; walk the source to that line's byte offset.
161
- // Honest limit: an in-place REWRITE of already-captured lines is not
162
- // detected (only growth is; a shrink triggers a rescan via the size
163
- // check in the caller). Transcript writers are append-only in practice.
164
- // Both are read in bounded memory: the journal's last turn from its tail, the
165
- // source's lines counted chunk by chunk (awebai/oats#456).
166
- // `last` is the journal's last turn (lastJournalTurn), null when it has none.
167
- function offsetFromJournal(last, sourcePath, final, sourceBytes) {
168
- if (!last) return { bytes: 0, line: 0, lastTs: "" };
169
- const lastLine = last.provenance?.origin?.line ?? 0;
170
- const { line, offset } = sourceBytes ? lineOffset([sourceBytes], lastLine) : lineOffsetOfFile(sourcePath, lastLine);
171
- if (final && line < lastLine) throw new Error(`session source is shorter than its captured journal: ${sourcePath}`);
172
- return { bytes: offset, line, lastTs: last.ts ?? "" };
173
- }
174
-
175
- // The byte offset just past the `lastLine`-th complete line of the bytes in
176
- // `chunks` (in order), and how many complete lines it found (fewer when the
177
- // source is shorter).
178
- function lineOffset(chunks, lastLine) {
179
- let line = 0, offset = 0;
180
- for (const chunk of chunks) {
181
- let from = 0;
182
- while (line < lastLine) {
183
- const nl = chunk.indexOf(10, from);
184
- if (nl === -1) break;
185
- line++;
186
- from = nl + 1;
187
- }
188
- offset += line < lastLine ? chunk.length : from;
189
- if (line >= lastLine) break;
190
- }
191
- return { line, offset };
192
- }
193
-
194
- function lineOffsetOfFile(path, lastLine) {
195
- const fd = openSync(path, "r");
196
- try { return lineOffset(fileChunks(fd), lastLine); }
197
- finally { closeSync(fd); }
198
- }
199
-
200
84
  // Source bytes of turns held before they are appended: what a pass keeps of a
201
85
  // transcript at once, whatever its size.
202
86
  const FLUSH_BYTES = 8 * 1024 * 1024;
203
87
 
204
- // The COMPLETE lines of `chunk`, one at a time, with their stamps:
88
+ // The COMPLETE lines of `chunks`, one at a time, with their stamps:
205
89
  // { text, ts, bytes }. `walk` records where the walk ended (`scanned`, bytes)
206
90
  // and why it stopped early (`reason`): a torn tail, invalid UTF-8 (decoding
207
91
  // replacement characters would change both the verbatim line and its byte
208
92
  // offset, possibly treating a later fragment as a record) or a line beyond
209
93
  // V8's string limit.
210
- function* completeLines(chunk, walk) {
94
+ function* completeLines(chunks, walk) {
211
95
  let scanned = 0;
212
96
  walk.scanned = 0;
213
- while (scanned < chunk.length) {
214
- const nl = chunk.indexOf(10, scanned);
215
- if (nl === -1) { walk.reason = "torn-tail"; return; }
216
- const bytes = chunk.subarray(scanned, nl);
97
+ for (const line of bufferLines(chunks)) {
98
+ if (line[line.length - 1] !== 10) { walk.reason = "torn-tail"; return; }
99
+ const bytes = line.subarray(0, -1);
217
100
  if (!isUtf8(bytes)) { walk.reason = "invalid-utf8"; return; }
218
101
  let text;
219
102
  try { text = bytes.toString("utf8"); }
@@ -222,7 +105,7 @@ function* completeLines(chunk, walk) {
222
105
  walk.reason = "oversized-line";
223
106
  return;
224
107
  }
225
- const lineBytes = nl - scanned + 1;
108
+ const lineBytes = line.length;
226
109
  let ts = "";
227
110
  if (text.trim() !== "") {
228
111
  try {
@@ -240,7 +123,7 @@ function* completeLines(chunk, walk) {
240
123
 
241
124
  // One reconciliation pass for one format: one turn per NEW complete line
242
125
  // of every session file under `roots`. Unstamped leading lines are held
243
- // until the file shows its first timestamp (then they carry it forward),
126
+ // until the file shows its first timestamp (then they carry it backward),
244
127
  // so every turn is stamped and ts stays a pure function of the source.
245
128
  // `files` accepts discovery entries (including their attribution snapshot) or
246
129
  // explicit paths for callers not making home-attribution claims. It pins a set,
@@ -250,14 +133,17 @@ function* completeLines(chunk, walk) {
250
133
  // stability through the pass. The caller must quiesce writers for retirement;
251
134
  // a performed pass is a snapshot, not a promise about future writes.
252
135
  //
253
- // A file given as `{ path, pinned: { bytes, stat } }` is PINNED: its caller opened it once and read these bytes from
136
+ // A file given as `{ path, pinned }` is PINNED (stageSnapshot or legacy { bytes, stat }): its caller opened it once and read these bytes from
254
137
  // that descriptor (`capture --file`). It is captured from them as a final pass, never opened again, so the
255
138
  // turns are exactly those bytes; `path` keys its offsets as for any other file.
256
- export function captureSessions(store, { owner, roots, files, format = "cc", ignore = null, final = false }) {
139
+ export function captureSessions(store, { owner, roots, files, format = "cc", ignore = null, final = false, collectIssues = true }) {
257
140
  const fmt = SESSION_FORMATS[format];
258
141
  if (!fmt) throw new Error(`unknown session format ${format}`);
259
142
  const ign = ignore ?? loadIgnore(store.root);
260
- const offsets = loadOffsets(store);
143
+ const offsets = new DerivedState(store.root);
144
+ try {
145
+ offsets.db.exec("CREATE TEMP TABLE captured_streams (stream TEXT PRIMARY KEY)");
146
+ const markStream = offsets.db.prepare("INSERT OR IGNORE INTO captured_streams VALUES (?)");
261
147
  let sessions = 0;
262
148
  let appended = 0;
263
149
  let unchanged = 0;
@@ -265,9 +151,14 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
265
151
  let ignored = 0;
266
152
  let incomplete = 0;
267
153
  const issues = []; // source metadata only, never native record contents
268
- const streams = new Set();
154
+ let streams = 0;
155
+ const issue = (value) => { if (collectIssues) issues.push(value); };
269
156
 
270
- for (const file of files ?? fmt.listFiles(roots)) {
157
+ // Reconcile exclusions even when a blocked input has disappeared from discovery.
158
+ for (const { key, value } of offsets.entries("set-aside")) {
159
+ if (value.owner === owner && value.source === fmt.source && ign.ignores(value.path, [basename(value.path), fmt.sessionId(value.path), ...(fmt.ignoreKeys?.(value.path) ?? [])])) offsets.delete("set-aside", key);
160
+ }
161
+ for (const file of files ?? (fmt.iterateFiles ?? fmt.listFiles)(roots)) {
271
162
  const path = typeof file === "string" ? file : file.path;
272
163
  const pinned = typeof file === "string" ? null : file.pinned ?? null;
273
164
  const expected = typeof file === "string" ? null : file.snapshot;
@@ -280,53 +171,85 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
280
171
  ignored++;
281
172
  continue; // never opened: nothing stored, nothing remembered
282
173
  }
283
- const fd = pinned ? null : openSync(path, capturedPi ? constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK : "r");
174
+ const streamId = `${owner}~${fmt.source}.${sessionId}`;
175
+ const offKey = `${streamId}:${path}`;
176
+ const blocked = offsets.get("set-aside", offKey);
177
+ if (blocked && !blocked.retry) {
178
+ incomplete++;
179
+ issue({ source: fmt.source, path, reason: blocked.reason, offset: blocked.offset, line: blocked.line });
180
+ continue;
181
+ }
182
+ const fd = pinned ? null : openSync(path, constants.O_RDONLY | constants.O_NONBLOCK | (capturedPi ? constants.O_NOFOLLOW : 0));
183
+ let staged;
284
184
  try {
285
185
  if (capturedPi) assertProtectedDescriptor(fd, path, capturedPi);
286
186
  const stat = pinned ? pinned.stat : fstatSync(fd);
187
+ const version = statVersion(pinned ? pinned.stat : fstatSync(fd, { bigint: true }));
287
188
  const snapshot = { ...identity(stat), ...(capturedPi ? { capturedPi } : {}) };
288
189
  if (expected) assertIdentity(stat, expected, path); // BEFORE reading bytes
289
190
  // Final home capture stages a descriptor-pinned snapshot. All attribution
290
191
  // and stability checks precede the first append, never a post-write alarm.
291
- const sourceBytes = pinned ? pinned.bytes : final || expected || capturedPi ? readRange(fd, 0, stat.size, path, capturedPi) : undefined;
292
- if (expected && digest(sourceBytes.subarray(0, expected.size)) !== expected.hash) {
192
+ staged = !pinned && (final || expected || capturedPi) ? stageSnapshot(fd, stat, path, capturedPi, store.root) : null;
193
+ let source = pinned ?? staged;
194
+ const chunks = (start = 0, end = stat.size) => source
195
+ ? source.bytes ? [source.bytes.subarray(start, end)] : source.chunks(start, end)
196
+ : sourceChunks(fd, start, end, path, capturedPi);
197
+ const witness = (end) => {
198
+ // At most 4 KiB; identity and this boundary are witnesses, not proof of
199
+ // the unobserved prefix. See the append-only input contract.
200
+ const bytes = Buffer.concat([...chunks(Math.max(0, end - 4096), end)]);
201
+ return digest(bytes);
202
+ };
203
+ if (expected && (source.bytes ? digest(source.bytes.subarray(0, expected.size)) : source.prefixHash(expected.size)) !== expected.hash) {
293
204
  throw new Error(`session source content changed since attribution: ${path}`);
294
205
  }
295
- if (sourceBytes) snapshot.hash = digest(sourceBytes);
206
+ if (source) snapshot.hash = source.bytes ? digest(source.bytes) : source.hash;
296
207
  const verifySource = () => {
297
208
  if (pinned) return; // the bytes are the source: nothing is read from the path again
298
209
  guardCapturedPath(path, capturedPi);
299
- if (sourceBytes) verifySnapshot(fd, path, snapshot);
210
+ if (source) verifySnapshot(fd, path, snapshot);
300
211
  };
301
212
  verifySource();
302
- const streamId = `${owner}~${fmt.source}.${sessionId}`;
303
- // Keyed by stream, not by source path: the same source captured under
304
- // two owners must not share offset state (owner is part of the stream).
305
- const offKey = `${streamId}:${path}`;
306
- let state = offsets[offKey];
307
- if (state && stat.size < state.bytes) state = null; // source shrank: rescan, no guesswork
308
- // The offset cache is derived state and can disagree with the journal:
309
- // behind it when another pass appended meanwhile (re-appending would
310
- // duplicate lines), ahead of it when a pass raced a stream wipe and
311
- // saved offsets for appends that landed in an unlinked inode (skipping
312
- // would lose lines — this happened during the live migration). The
313
- // journal is the truth; before appending anything, any disagreement
314
- // rebuilds the offset from it. Background passes check on growth;
315
- // final passes also verify unchanged files before confirming capture.
316
- let journalLast; // the journal's last turn, read from its tail once, when needed
317
- const lastTurn = () => (journalLast === undefined ? (journalLast = lastJournalTurn(store, streamId)) : journalLast);
318
- if (state && (final || stat.size > state.bytes) && state.line !== (lastTurn()?.provenance?.origin?.line ?? 0)) {
319
- state = null;
213
+ let state = offsets.get(fmt.source, offKey);
214
+ const generation = offsets.get("reset", `${owner}:${fmt.source}`) ?? "";
215
+ const journalVersion = fileVersion(store.journalPath(streamId));
216
+ if (state && (!Number.isSafeInteger(state.bytes) || state.bytes < 0 || state.bytes > stat.size ||
217
+ !Number.isSafeInteger(state.line) || state.line < 0 || typeof state.lastTs !== "string")) state = null;
218
+ const prior = state?.version;
219
+ const sameInput = prior && sameVersion(prior, version);
220
+ const appendGrowth = prior && prior.dev === version.dev && prior.ino === version.ino && version.size > prior.size &&
221
+ state.bytes <= version.size && witness(state.bytes) === state.witness;
222
+ const recover = !state || state.generation !== generation || !sameVersion(state.journalVersion, journalVersion) ||
223
+ (!sameInput && !appendGrowth) || blocked;
224
+ if (recover) {
225
+ // A complete comparison precedes ANY append from this input. Pin the
226
+ // bytes on disk so a later batch cannot reopen a mutable source.
227
+ if (!source && journalVersion?.size) {
228
+ staged = stageSnapshot(fd, stat, path, capturedPi, store.root);
229
+ source = staged; snapshot.hash = staged.hash;
230
+ }
231
+ state = compareTranscriptPrefix(store, streamId, offsets, completeLines(chunks(), {}));
232
+ verifySource();
233
+ if (state.conflict) {
234
+ const row = { owner, source: fmt.source, path, ...state.conflict, retry: false };
235
+ offsets.set("set-aside", offKey, row);
236
+ incomplete++;
237
+ issue({ source: fmt.source, path, ...state.conflict });
238
+ continue;
239
+ }
240
+ offsets.delete("set-aside", offKey);
320
241
  }
321
- if (!state) state = offsetFromJournal(lastTurn(), path, final, sourceBytes);
242
+ const checkpoint = (value) => offsets.set(fmt.source, offKey, {
243
+ ...value, version, witness: witness(value.bytes), generation,
244
+ journalVersion: fileVersion(store.journalPath(streamId)),
245
+ });
322
246
  if (stat.size <= state.bytes) {
323
247
  unchanged++;
324
- offsets[offKey] = state;
248
+ checkpoint(state);
325
249
  verifySource();
326
250
  continue;
327
251
  }
328
252
 
329
- const chunk = sourceBytes ? sourceBytes.subarray(state.bytes) : readRange(fd, state.bytes, stat.size, path, capturedPi);
330
253
  // Every turn needs a stamp. Leading lines before the file's first stamp
331
254
  // carry it backward (deterministic: the file's first stamp is invariant
332
255
  // however capture is scheduled); if the file has shown no stamp at all
@@ -336,13 +259,13 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
336
259
  const stopped = (walk) => {
337
260
  if (!walk.reason) return;
338
261
  incomplete++;
339
- issues.push({ source: fmt.source, path, reason: walk.reason, offset: state.bytes + walk.scanned });
262
+ issue({ source: fmt.source, path, reason: walk.reason, offset: state.bytes + walk.scanned });
340
263
  };
341
264
  let lastTs = state.lastTs ?? "";
342
265
  if (!lastTs) {
343
266
  const scan = {};
344
267
  let any = false;
345
- for (const l of completeLines(chunk, scan)) {
268
+ for (const l of completeLines(chunks(state.bytes), scan)) {
346
269
  any = true;
347
270
  if (l.ts) { lastTs = l.ts; break; }
348
271
  }
@@ -350,7 +273,7 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
350
273
  stopped(scan);
351
274
  if (any) {
352
275
  held++;
353
- issues.push({ source: fmt.source, path, reason: "unstamped", offset: state.bytes });
276
+ issue({ source: fmt.source, path, reason: "unstamped", offset: state.bytes });
354
277
  }
355
278
  verifySource();
356
279
  continue; // do not advance; retry when a stamp exists
@@ -359,23 +282,24 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
359
282
  verifySource(); // parsing/recovery may take time; still no journal write yet
360
283
  // Turns flush to the journal in bounded batches, so memory stays flat
361
284
  // however large the backlog. A crash between flushes cannot
362
- // duplicate: this file's offset is saved only after its final flush,
363
- // and a lost offset rebuilds from the journal's own last line number.
285
+ // duplicate: checkpoints follow each append and the durable ID lookup
286
+ // reconciles a crash between journal fsync and the source checkpoint.
364
287
  let fresh = [];
365
288
  let freshBytes = 0;
366
289
  const flush = () => {
367
290
  if (fresh.length === 0) return;
368
291
  verifySource(); // EVERY batch; earlier committed batches are never rolled back
369
- store.appendBatch(streamId, fresh);
370
- streams.add(streamId);
371
- appended += fresh.length;
292
+ const added = store.appendUniqueBatch(streamId, fresh);
293
+ if (added) streams += Number(markStream.run(streamId).changes);
294
+ appended += added;
295
+ checkpoint({ bytes: state.bytes + consumed, line: lineNo, lastTs });
372
296
  fresh = [];
373
297
  freshBytes = 0;
374
298
  };
375
299
  let lineNo = state.line;
376
300
  let consumed = 0;
377
301
  const walk = {};
378
- for (const l of completeLines(chunk, walk)) {
302
+ for (const l of completeLines(chunks(state.bytes), walk)) {
379
303
  if (l.ts) lastTs = l.ts;
380
304
  lineNo++;
381
305
  consumed += l.bytes;
@@ -387,22 +311,14 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
387
311
  ),
388
312
  );
389
313
  freshBytes += l.bytes;
390
- if (freshBytes >= FLUSH_BYTES) flush();
314
+ if (fresh.length >= 2000 || freshBytes >= FLUSH_BYTES) flush();
391
315
  }
392
316
  stopped(walk);
393
- const grew = fresh.length > 0 || consumed > 0;
394
317
  flush();
395
- offsets[offKey] = { bytes: state.bytes + consumed, line: lineNo, lastTs };
396
- // Persist after every file that advanced, not once per pass: a crash
397
- // later in the pass must not leave THIS file's on-disk offset stale
398
- // (a stale-but-present offset skips journal recovery, and the rescan
399
- // re-appends duplicate turn lines — logically deduped by id, but
400
- // wasted append-only bytes).
401
- if (grew) saveOffsets(store, offsets);
318
+ checkpoint({ bytes: state.bytes + consumed, line: lineNo, lastTs });
402
319
  verifySource();
403
- } finally { if (fd !== null) closeSync(fd); }
320
+ } finally { try { staged?.close(); } finally { if (fd !== null) closeSync(fd); } }
404
321
  }
405
- saveOffsets(store, offsets);
406
322
  return {
407
323
  sessions,
408
324
  appended,
@@ -412,20 +328,26 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
412
328
  incomplete,
413
329
  complete: held === 0 && incomplete === 0,
414
330
  issues,
415
- streams: streams.size,
331
+ streams,
416
332
  stream: `${owner}~${fmt.source}.*`,
417
333
  };
334
+ } finally { offsets.close(); }
418
335
  }
419
336
 
420
337
  // One pass over every known format at its default roots. The ignore list
421
338
  // is loaded once and shared across formats.
422
- export function captureAllSessions(store, { owner, ignore = null }) {
339
+ export function captureAllSessions(store, { owner, ignore = null, collectIssues = true }) {
423
340
  const ign = ignore ?? loadIgnore(store.root);
424
341
  const results = [];
425
342
  for (const format of Object.keys(SESSION_FORMATS)) {
426
343
  const roots = SESSION_FORMATS[format].defaultRoots();
427
- if (roots.length === 0) continue;
428
- results.push(captureSessions(store, { owner, roots, format, ignore: ign }));
344
+ if (roots.length === 0) {
345
+ // Missing defaults must not keep an ignored, previously blocked path
346
+ // active merely because it no longer appears in filesystem discovery.
347
+ captureSessions(store, { owner, roots, format, ignore: ign, collectIssues: false });
348
+ continue;
349
+ }
350
+ results.push(captureSessions(store, { owner, roots, format, ignore: ign, collectIssues }));
429
351
  }
430
352
  return results;
431
353
  }
@@ -0,0 +1,39 @@
1
+ import { closeSync, constants, fstatSync, openSync, readSync } from "node:fs";
2
+ import { createRequire } from "node:module";
3
+ import { pathToFileURL } from "node:url";
4
+ import { readIssueBacklog, supportsReadOnlySqlite } from "./capture-issues.mjs";
5
+ const require = createRequire(import.meta.url);
6
+
7
+ // Supported capture writers use rollback journaling. SQLite may create WAL
8
+ // sidecars even for a readOnly connection; refuse that mode before SQLite opens
9
+ // it. This is not protection against an external actor racing a journal-mode
10
+ // change or file replacement. No supported writer performs either operation.
11
+ export function readCaptureDatabase(path) {
12
+ if (!supportsReadOnlySqlite(process.versions.node)) throw new Error("read-only capture health requires newer Node with read-only SQLite (22.13+ on 22.x, 23.2+ on 23.x, or newer major)");
13
+ let fd;
14
+ try { fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK); }
15
+ catch (err) { if (err.code === "ENOENT") return null; throw err; }
16
+ try {
17
+ if (!fstatSync(fd).isFile()) throw new Error("capture.db is not a regular file");
18
+ const header = Buffer.alloc(100);
19
+ if (readSync(fd, header, 0, header.length, 0) !== header.length ||
20
+ header.subarray(0, 16).toString() !== "SQLite format 3\0") throw new Error("invalid capture.db header");
21
+ if (header[18] === 2 || header[19] === 2) throw new Error("WAL capture.db is unsupported by the read-only health reader; use a supported rollback-journal database");
22
+ if (header[18] !== 1 || header[19] !== 1) throw new Error("unsupported capture.db journal format");
23
+ } finally { closeSync(fd); }
24
+
25
+ let DatabaseSync;
26
+ try { ({ DatabaseSync } = require("node:sqlite")); }
27
+ catch (err) { throw new Error(`read-only SQLite is unavailable; rerun health with Node 22.13+ on 22.x, 23.2+ on 23.x, or newer major: ${err.message}`); }
28
+ const db = new DatabaseSync(path, { open: false, readOnly: true });
29
+ db.open();
30
+ try {
31
+ db.exec("PRAGMA busy_timeout = 250; PRAGMA query_only = ON; PRAGMA cache_size = -2048; PRAGMA temp_store = FILE");
32
+ return readIssueBacklog(db, { validate: true });
33
+ } finally { db.close(); }
34
+ }
35
+
36
+ if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) {
37
+ try { console.log(JSON.stringify({ backlog: readCaptureDatabase(process.argv[2]) })); }
38
+ catch (err) { console.log(JSON.stringify({ error: err.message })); process.exitCode = 1; }
39
+ }
@@ -0,0 +1,64 @@
1
+ import { statSync } from "node:fs";
2
+ import { spawnSync } from "node:child_process";
3
+ import { fileURLToPath } from "node:url";
4
+ import { join } from "node:path";
5
+ // Feature landed separately on the two release lines; 23.0/23.1 lack it.
6
+ export function supportsReadOnlySqlite(version) {
7
+ const match = /^(\d+)\.(\d+)\.(\d+)$/.exec(version);
8
+ if (!match) return false;
9
+ const major = Number(match[1]), minor = Number(match[2]);
10
+ return major > 23 || major === 23 && minor >= 2 || major === 22 && minor >= 12;
11
+ }
12
+
13
+ // The command snapshot and live health diagnosis use the same bounded
14
+ // projection. This function owns one read transaction, not the capture lock.
15
+ export function readIssueBacklog(db, { pending = false, validate = false } = {}) {
16
+ db.exec("BEGIN");
17
+ try {
18
+ if (validate) {
19
+ if (db.prepare("PRAGMA user_version").get().user_version !== 0) throw new Error("unsupported capture database version");
20
+ const columns = db.prepare("PRAGMA table_info(state)").all();
21
+ if (columns.length !== 3 || columns.some((column, i) =>
22
+ column.name !== ["kind", "key", "value"][i] || column.type !== "TEXT" || column.pk !== [1, 2, 0][i] ||
23
+ column.notnull !== [0, 0, 1][i])) throw new Error("unsupported capture issue-table schema");
24
+ }
25
+ const count = Number(db.prepare("SELECT count(*) AS n FROM state WHERE kind = 'set-aside'").get().n);
26
+ const sample = db.prepare(`SELECT value FROM state WHERE kind = 'set-aside'
27
+ ORDER BY json_extract(value, '$.source'), json_extract(value, '$.path'), key LIMIT 100`).all();
28
+ const setAside = sample.map(({ value }) => {
29
+ const { source, path, offset, reason, line } = JSON.parse(value);
30
+ if (!["aw", "cc", "pi", "codex"].includes(source) || typeof path !== "string" || typeof reason !== "string" ||
31
+ !Number.isSafeInteger(offset) || offset < 0 || (line !== undefined && (!Number.isSafeInteger(line) || line <= 0))) {
32
+ throw new Error("malformed capture issue row");
33
+ }
34
+ return { source, path, offset, reason, ...(line === undefined ? {} : { line }) };
35
+ });
36
+ db.exec("COMMIT");
37
+ return { pending: pending || count > 0, setAsideCount: count, setAside, truncated: count > sample.length };
38
+ } catch (err) { db.exec("ROLLBACK"); throw err; }
39
+ }
40
+
41
+ // A separate process prevents a raw header descriptor close from releasing
42
+ // process-owned POSIX locks held by another SQLite connection in this caller.
43
+ // It loads no capture writers, retains SQLite locking, and inherits no hooks.
44
+ export function readCurrentIssueBacklog(root) {
45
+ const path = join(root, "index", "capture.db");
46
+ try { if (!statSync(path).isFile()) throw new Error("capture.db is not a regular file"); }
47
+ catch (err) { if (err.code === "ENOENT") return null; throw err; }
48
+ if (!supportsReadOnlySqlite(process.versions.node)) throw new Error("read-only capture health needs newer Node with read-only SQLite: use Node 22.13+ on 22.x, 23.2+ on 23.x, or a newer major version");
49
+ const env = Object.fromEntries(["HOME", "TMPDIR", "XDG_CACHE_HOME"].filter(key => process.env[key] !== undefined).map(key => [key, process.env[key]]));
50
+ const result = spawnSync(process.execPath, [
51
+ ...process.execArgv.filter(arg => arg === "--experimental-sqlite"),
52
+ fileURLToPath(new URL("./capture-db-reader.mjs", import.meta.url)), path,
53
+ ], { env, encoding: "utf8", maxBuffer: 1024 * 1024, timeout: 10000, killSignal: "SIGKILL" });
54
+ if (result.error) throw result.error;
55
+ let response;
56
+ try { response = JSON.parse(result.stdout); }
57
+ catch { throw new Error(`capture database reader failed (${result.signal ?? result.status}): ${result.stderr?.trim() || "no valid response"}`); }
58
+ if (result.status !== 0 || response.error) throw new Error(response.error ?? "capture database reader failed");
59
+ const b = response.backlog;
60
+ if (!b || typeof b.pending !== "boolean" || !Number.isSafeInteger(b.setAsideCount) || b.setAsideCount < 0 ||
61
+ !Array.isArray(b.setAside) || b.setAside.length !== Math.min(b.setAsideCount, 100) ||
62
+ b.truncated !== (b.setAsideCount > 100)) throw new Error("malformed capture database reader output");
63
+ return b;
64
+ }