@awebai/oats 0.54.0 → 0.55.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/bin/oats.mjs +5 -5
  2. package/docs/desktop-cli-api.md +15 -4
  3. package/docs/integrations.md +2 -2
  4. package/docs/knowledge.md +14 -2
  5. package/docs/official-catalog.md +4 -4
  6. package/docs/packages.md +10 -10
  7. package/docs/release-notes/v0.55.0.md +158 -0
  8. package/docs/release-notes/v0.55.1.md +45 -0
  9. package/docs/servers.md +13 -3
  10. package/docs/workspaces.md +1 -1
  11. package/lib/servers.mjs +82 -34
  12. package/package-catalog.json +3 -3
  13. package/package.json +1 -1
  14. package/packages/client/cli-locator.mjs +2 -2
  15. package/packages/client/workspace-views.mjs +4 -3
  16. package/packages/record/README.md +58 -13
  17. package/packages/record/bin/capture.mjs +107 -53
  18. package/packages/record/bin/recall.mjs +3 -2
  19. package/packages/record/docs/capture-service.md +193 -0
  20. package/packages/record/docs/capture-status.schema.json +100 -0
  21. package/packages/record/lib/capture-aw.mjs +96 -94
  22. package/packages/record/lib/capture-cc.mjs +109 -187
  23. package/packages/record/lib/capture-db-reader.mjs +39 -0
  24. package/packages/record/lib/capture-issues.mjs +64 -0
  25. package/packages/record/lib/capture-scratch.mjs +111 -0
  26. package/packages/record/lib/capture-state.mjs +105 -0
  27. package/packages/record/lib/derived-state.mjs +60 -0
  28. package/packages/record/lib/formats.mjs +31 -31
  29. package/packages/record/lib/index-db.mjs +74 -34
  30. package/packages/record/lib/receipt-summary.mjs +62 -0
  31. package/packages/record/lib/session-snapshot.mjs +56 -1
  32. package/packages/record/lib/sqlite-rows.mjs +14 -0
  33. package/packages/record/lib/store.mjs +122 -43
  34. package/packages/record/lib/transcript-prefix.mjs +40 -0
  35. package/skills/oats-getting-started/SKILL.md +1 -1
@@ -20,12 +20,16 @@ import process from "node:process";
20
20
  import { RecordStore } from "../lib/store.mjs";
21
21
  import { captureAllSessions, captureSessions } from "../lib/capture-cc.mjs";
22
22
  import { sessionsForHome } from "../lib/sessions-for-home.mjs";
23
- import { jsonlLines, SESSION_FORMATS } from "../lib/formats.mjs";
23
+ import { SESSION_FORMATS } from "../lib/formats.mjs";
24
24
  import { cwdOfLine } from "../lib/sessions-for-home.mjs";
25
- import { digest, readRange } from "../lib/session-snapshot.mjs";
25
+ import { stageSnapshot } from "../lib/session-snapshot.mjs";
26
+ import { cleanupCaptureScratch } from "../lib/capture-scratch.mjs";
27
+ import { bufferLines } from "../lib/file-lines.mjs";
26
28
  import { captureAwLogs, defaultCommLogDir } from "../lib/capture-aw.mjs";
29
+ import { withReceiptSummaries } from "../lib/receipt-summary.mjs";
27
30
  import { RecordIndex } from "../lib/index-db.mjs";
28
31
  import { IgnoreError, ignoreFilePath, loadIgnore } from "../lib/ignore.mjs";
32
+ import { CAPTURE_SOURCES, captureHealth, resetSourceCursors, retrySetAside, writeCaptureStatus } from "../lib/capture-state.mjs";
29
33
  import { acquireCaptureLock } from "../lib/capture-lock.mjs";
30
34
 
31
35
  // Fail closed but actionably: an unreadable ignore file must stop capture,
@@ -45,8 +49,11 @@ function loadIgnoreOrExit(recordRoot) {
45
49
  // hostname-derived owner, which forked the whole record into a second owner
46
50
  // namespace: 571 duplicate journals from one typo. Parsing must refuse what
47
51
  // it does not understand before anything can be written.
48
- const VALUE_FLAGS = new Set(["root", "owner", "home", "file", "format"]);
52
+ const VALUE_FLAGS = new Set(["root", "owner", "home", "file", "format", "source"]);
49
53
  const BOOL_FLAGS = new Set([
54
+ "reset-cursors",
55
+ "retry-set-aside",
56
+ "health",
50
57
  "watch",
51
58
  "status",
52
59
  "install-hint",
@@ -67,6 +74,12 @@ const USAGE = `capture — land sessions and aw client logs in the turn record.
67
74
  capture --watch pass now, then re-pass on filesystem change
68
75
  (debounced) and every 15 minutes regardless
69
76
  capture --status show store/stream summary, capture nothing
77
+ capture --health [--json] read capture set-aside diagnosis, capture nothing
78
+ capture --reset-cursors [--source aw|cc|pi|codex]
79
+ reset selected source cursors and exit; the next
80
+ pass reprocesses them with journal deduplication
81
+ capture --retry-set-aside revalidate set-aside inputs in one reconciliation;
82
+ does not authorize changing captured transcripts
70
83
  capture --home <dir> capture the sessions that ran inside <dir> (an
71
84
  OATS instance home) and print them as JSON:
72
85
  thread, stream, turn count, first/last turn id;
@@ -146,6 +159,10 @@ function parseArgs(argv) {
146
159
  let args;
147
160
  try {
148
161
  args = parseArgs(process.argv.slice(2));
162
+ const recovery = ["reset-cursors", "retry-set-aside", "health"].filter(key => args[key]);
163
+ if (recovery.length > 1) throw new UsageError("choose one recovery or health operation");
164
+ if (args.source !== undefined && (!args["reset-cursors"] || !CAPTURE_SOURCES.includes(args.source))) throw new UsageError("--source requires --reset-cursors and aw|cc|pi|codex");
165
+ if (recovery.length && ["home", "file", "format", "watch", "status", "install-hint", "current-roots", "sessions-only", "aw-only"].some(key => args[key])) throw new UsageError("recovery/health cannot be combined with another capture mode");
149
166
  } catch (err) {
150
167
  if (!(err instanceof UsageError)) throw err;
151
168
  // --file --json answers JSON even for a usage error: it is for programs.
@@ -165,9 +182,43 @@ if (args.help) {
165
182
  const root = args.root ?? process.env.TURN_RECORD_ROOT ?? join(homedir(), ".turn-record");
166
183
  const ownerExplicit = args.owner !== undefined || process.env.TURN_RECORD_OWNER !== undefined;
167
184
  const owner = args.owner ?? process.env.TURN_RECORD_OWNER ?? hostname().split(".")[0];
168
- const quiet = Boolean(args.quiet);
185
+ const quiet = Boolean(args.quiet || args.json && (args["reset-cursors"] || args["retry-set-aside"]));
169
186
  const store = new RecordStore(root, { owner });
170
187
 
188
+ if (args.health) {
189
+ try {
190
+ const health = captureHealth(root);
191
+ if (args.json) console.log(JSON.stringify(health, null, 2));
192
+ else {
193
+ console.log(`capture: ${health.status}`);
194
+ for (const row of health.rows) console.log(`${row.code}: ${row.message}`);
195
+ for (const entry of health.backlog.setAside) console.log(`${entry.source}: ${entry.path}:${entry.line ?? "?"} offset ${entry.offset}: ${entry.reason}`);
196
+ if (health.backlog.truncated) console.log("showing first 100 set-aside inputs");
197
+ }
198
+ } catch (err) {
199
+ if (args.json) console.log(JSON.stringify({ status: "failed", error: err.message }));
200
+ else console.error(`capture: ${err.message}`);
201
+ process.exitCode = 1;
202
+ }
203
+ process.exit(process.exitCode ?? 0);
204
+ }
205
+ if (args["reset-cursors"] || args["retry-set-aside"]) {
206
+ try {
207
+ const result = args["reset-cursors"]
208
+ ? withCaptureLock(() => { resetSourceCursors(root, owner, args.source); return { reset: true, sources: args.source ? [args.source] : CAPTURE_SOURCES }; })
209
+ : pass({ retry: true });
210
+ if (result.skipped) throw new Error("capture lock held; recovery was not performed");
211
+ if (process.exitCode) throw new Error("recovery failed to release its capture lock");
212
+ if (args.json) console.log(JSON.stringify(result));
213
+ else if (args["reset-cursors"]) log(`capture: reset source cursors (${result.sources.join(", ")})`);
214
+ } catch (err) {
215
+ if (args.json) console.log(JSON.stringify({ status: "failed", complete: false, error: err.message }));
216
+ else console.error(`capture: ${err.message}`);
217
+ process.exitCode = 1;
218
+ }
219
+ process.exit(process.exitCode ?? 0);
220
+ }
221
+
171
222
  // The owner decides the stream namespace, and turn ids hash it into the
172
223
  // canonical core — so the same conversation captured under two owners yields
173
224
  // two sets of ids for identical content, which the index cannot dedupe. When
@@ -178,21 +229,18 @@ const store = new RecordStore(root, { owner });
178
229
  // root, no other owners, and nothing is printed.
179
230
  function warnOnStrangerOwner() {
180
231
  if (ownerExplicit) return;
181
- let others;
232
+ let example;
182
233
  try {
183
- others = new Set(
184
- store
185
- .listStreams()
186
- .map((id) => id.slice(0, id.indexOf("~")))
187
- .filter(Boolean),
188
- );
189
- } catch {
190
- return; // unreadable or absent root: not this function's business
191
- }
192
- if (others.size === 0 || others.has(owner)) return;
234
+ for (const stream of store.iterateStreams()) {
235
+ const existing = stream.slice(0, stream.indexOf("~"));
236
+ if (existing === owner) return;
237
+ if (existing) example ??= existing;
238
+ }
239
+ } catch { return; }
240
+ if (!example) return;
193
241
  console.error(
194
242
  `capture: writing as owner "${owner}" (from hostname), but this record holds ` +
195
- `streams only under ${[...others].sort().map((o) => `"${o}"`).join(", ")}. ` +
243
+ `streams under another owner such as "${example}". ` +
196
244
  `Pass --owner or set TURN_RECORD_OWNER if that is not what you meant — ` +
197
245
  `capturing under a second owner duplicates the record rather than extending it.`,
198
246
  );
@@ -240,7 +288,16 @@ function withCaptureLock(fn) {
240
288
  return { appended: 0, skipped: true, lock: lock.held };
241
289
  }
242
290
  try {
243
- return fn();
291
+ let result, failure;
292
+ try { cleanupCaptureScratch(root, lock); result = fn(); return result; }
293
+ catch (err) { failure = err; throw err; }
294
+ finally {
295
+ try { writeCaptureStatus(root, { pending: Boolean(failure || result?.pending || result?.held || result?.incomplete || result?.failed) }); }
296
+ catch (err) {
297
+ if (failure) throw new Error(`${failure.message}; capture status persistence also failed: ${err.message}`, { cause: failure });
298
+ throw err;
299
+ }
300
+ }
244
301
  } finally {
245
302
  const r = lock.release();
246
303
  if (!r.released) {
@@ -256,15 +313,17 @@ function withCaptureLock(fn) {
256
313
  }
257
314
  }
258
315
 
259
- function pass() {
316
+ function pass({ retry = false } = {}) {
260
317
  // The privacy loader fails closed by exiting; it runs before the lock is
261
318
  // taken so that exit never leaves the lock behind.
262
319
  const ignore = loadIgnoreOrExit(root);
263
320
  return withCaptureLock(() => {
264
- const out = { appended: 0 };
321
+ if (retry) retrySetAside(root);
322
+ const out = { appended: 0, pending: false };
265
323
  if (!args["aw-only"]) {
266
- for (const r of captureAllSessions(store, { owner, ignore })) {
324
+ for (const r of captureAllSessions(store, { owner, ignore, collectIssues: false })) {
267
325
  out.appended += r.appended;
326
+ out.pending ||= !r.complete;
268
327
  const extras = [r.ignored ? `${r.ignored} ignored` : "", r.held ? `${r.held} held` : "", r.incomplete ? `${r.incomplete} incomplete` : ""]
269
328
  .filter(Boolean)
270
329
  .join(", ");
@@ -279,7 +338,7 @@ function pass() {
279
338
  let awFailed = 0;
280
339
  let awFiles = 0;
281
340
  let awIgnored = 0;
282
- for (const r of captureAwLogs(store, { owner, ignore })) {
341
+ for (const r of captureAwLogs(store, { owner, ignore, collect: false })) {
283
342
  if (r.ignored) {
284
343
  awIgnored++;
285
344
  continue;
@@ -288,6 +347,7 @@ function pass() {
288
347
  awEntries += r.entries;
289
348
  awAppended += r.appended;
290
349
  awFailed += r.failed;
350
+ out.pending ||= r.failed > 0;
291
351
  out.appended += r.appended;
292
352
  if (r.failed) console.error(`aw-log ${r.account}: ${r.failed} entries failed to project`);
293
353
  }
@@ -302,7 +362,7 @@ function pass() {
302
362
  const index = new RecordIndex(store);
303
363
  try {
304
364
  index.update();
305
- log(`index: updated (${JSON.stringify(index.counts())})`);
365
+ log("index: updated");
306
366
  } finally {
307
367
  index.close();
308
368
  }
@@ -335,12 +395,6 @@ function captureUnderLock(recordStore, recordOwner, batches, ignore, outcome, is
335
395
  });
336
396
  }
337
397
 
338
- /** A stream's turns that no tombstone hides. A tombstoned turn is hidden everywhere; a boundary naming one
339
- * would be refused by recall, so boundaries come from the visible turns only. */
340
- function visibleTurns(recordStore, stream, claims = recordStore.tombstoneClaims()) {
341
- return recordStore.readStream(stream).filter((t) => !recordStore.claimHides(claims, t));
342
- }
343
-
344
398
  /** The outcome vocabulary: complete only with no failure, skip, hold or incomplete tail (and, for --home, no
345
399
  * unattributed source). */
346
400
  function statusOf(outcome, unattributed = 0) {
@@ -366,21 +420,18 @@ function isFormat(found, stated) {
366
420
  /** Records (non-blank lines, an undecodable one included) and the formats whose session header a COMPLETE
367
421
  * line carries. A pi or codex file is settled at its header; a cc file is judged whole. A line its writer
368
422
  * never terminated is no header. */
369
- function sessionHeaders(bytes, stated) {
423
+ function sessionHeaders(chunks, stated) {
370
424
  let records = 0;
371
425
  const found = new Set();
372
- const terminated = bytes.length > 0 && bytes[bytes.length - 1] === 10;
373
- const take = (text, complete) => {
426
+ for (const line of bufferLines(chunks)) {
427
+ const complete = line[line.length - 1] === 10;
428
+ let text = null;
429
+ try { text = (complete ? line.subarray(0, -1) : line).toString("utf8"); }
430
+ catch (err) { if (err.code !== "ERR_STRING_TOO_LONG") throw err; }
374
431
  if (text === null || text.trim() !== "") records++;
375
432
  if (complete && text !== null) for (const source of HEADER_ORDER) if (cwdOfLine(source, text) !== undefined) found.add(source);
376
- };
377
- let pending;
378
- for (const { text } of jsonlLines(bytes)) {
379
- if (pending !== undefined) take(pending, true);
380
- if (stated !== "cc" && found.has(stated)) return { records, found };
381
- pending = text;
433
+ if (stated !== "cc" && found.has(stated)) break;
382
434
  }
383
- if (pending !== undefined) take(pending, terminated);
384
435
  return { records, found };
385
436
  }
386
437
 
@@ -430,17 +481,18 @@ function fileCapture() {
430
481
  if (err.code === "ELOOP" || err.code === "EMLINK") return fail("E_NOT_REGULAR_FILE", `${path} is a symbolic link; pass the file it names`, where);
431
482
  return fail("E_FILE_UNREADABLE", `cannot open ${path}: ${err.message}`, where);
432
483
  }
433
- let stat, bytes;
484
+ let stat, pinned;
434
485
  try {
435
486
  stat = fstatSync(fd);
436
487
  if (!stat.isFile()) return fail("E_NOT_REGULAR_FILE", `${path} is ${fileKind(stat)}, not a regular file`, where);
437
- bytes = readRange(fd, 0, stat.size, path);
488
+ pinned = stageSnapshot(fd, stat, path, undefined, root);
438
489
  } catch (err) {
439
490
  return fail("E_FILE_UNREADABLE", `cannot read ${path}: ${err.message}`, where);
440
491
  } finally { closeSync(fd); }
441
492
 
442
- const { records, found } = sessionHeaders(bytes, args.format);
443
- if (!records) return fail("E_NO_TURNS", `${path} has no records${bytes.length ? " (only blank lines)" : " (it is empty)"}`, where);
493
+ try {
494
+ const { records, found } = sessionHeaders(pinned.chunks(), args.format);
495
+ if (!records) return fail("E_NO_TURNS", `${path} has no records${stat.size ? " (only blank lines)" : " (it is empty)"}`, where);
444
496
  if (!isFormat(found, args.format)) {
445
497
  const other = HEADER_ORDER.find((f) => f !== args.format && isFormat(found, f));
446
498
  return fail("E_FORMAT", other
@@ -448,12 +500,12 @@ function fileCapture() {
448
500
  : `${path} has no ${args.format} session header (${SESSION_HEADER[args.format]})`, where);
449
501
  }
450
502
 
451
- const sha256 = digest(bytes);
503
+ const sha256 = pinned.hash;
452
504
  const fileStore = new RecordStore(root, { owner: fileOwner });
453
505
  const outcome = { appended: 0, skipped: false, held: 0, incomplete: 0, failed: 0, ignored: 0 };
454
506
  const issues = [];
455
507
  try {
456
- Object.assign(outcome, captureUnderLock(fileStore, fileOwner, [[args.format, [{ path, pinned: { bytes, stat } }]]], ignore, outcome, issues));
508
+ Object.assign(outcome, captureUnderLock(fileStore, fileOwner, [[args.format, [{ path, pinned }]]], ignore, outcome, issues));
457
509
  } catch (err) {
458
510
  // A failed pass may have appended part of the file: never claim a count for it.
459
511
  return fail("E_CAPTURE_FAILED", err.message || String(err), { ...where, sha256, appended: null });
@@ -461,17 +513,18 @@ function fileCapture() {
461
513
  if (process.exitCode) return fail("E_CAPTURE_FAILED", "capture lock release failed; see stderr for recovery", { ...where, sha256 });
462
514
 
463
515
  const stream = `${fileOwner}~${fmt.source}.${sessionId}`;
464
- const turns = visibleTurns(fileStore, stream);
516
+ const turns = withReceiptSummaries(fileStore, summary => summary(stream));
465
517
  const status = statusOf(outcome);
466
518
  const { lock: _lock, ...counts } = outcome;
467
519
  const receipt = {
468
520
  ...where, instance, thread: `${fmt.source}:session:${sessionId}`, stream, sessionId,
469
- turns: turns.length, firstTurnId: turns[0]?.id ?? null, lastTurnId: turns.at(-1)?.id ?? null,
521
+ turns: turns.count, firstTurnId: turns.first?.id ?? null, lastTurnId: turns.last?.id ?? null,
470
522
  ...counts, status, complete: status === "complete", sha256, ...(issues.length ? { issues } : {}),
471
523
  };
472
524
  if (json) console.log(JSON.stringify(receipt, null, 2));
473
525
  else console.log(`capture --file: ${status}, ${receipt.turns} turns (${outcome.appended} new) in ${stream}, sha256 ${sha256}`);
474
526
  return 0;
527
+ } finally { pinned.close(); }
475
528
  }
476
529
 
477
530
  if (args.file !== undefined || args.format !== undefined || args.json) process.exit(fileCapture());
@@ -535,11 +588,11 @@ if (args.home) {
535
588
  formats.get(s.source).push(s);
536
589
  }
537
590
  Object.assign(outcome, captureUnderLock(store, owner, formats, ignore, outcome, issues));
538
- const claims = store.tombstoneClaims();
591
+ withReceiptSummaries(store, summary => {
539
592
  for (const s of found) {
540
593
  const stream = `${owner}~${s.source}.${s.sessionId}`;
541
- const turns = visibleTurns(store, stream, claims);
542
- if (!turns.length) continue; // ignored by rule, nothing capturable yet, or all hidden
594
+ const turns = summary(stream);
595
+ if (!turns.count) continue; // ignored by rule, nothing capturable yet, or all hidden
543
596
  sessions.push({
544
597
  thread: s.thread,
545
598
  source: s.source,
@@ -547,12 +600,13 @@ if (args.home) {
547
600
  path: s.path,
548
601
  cwd: s.cwd,
549
602
  stream,
550
- turns: turns.length,
551
- firstTurnId: turns[0].id,
552
- lastTurnId: turns[turns.length - 1].id,
553
- lastTs: turns[turns.length - 1].ts,
603
+ turns: turns.count,
604
+ firstTurnId: turns.first.id,
605
+ lastTurnId: turns.last.id,
606
+ lastTs: turns.last.ts,
554
607
  });
555
608
  }
609
+ });
556
610
  } catch (err) {
557
611
  error = err.message || String(err);
558
612
  console.error(`capture pass failed: ${error}`);
@@ -61,8 +61,9 @@ try {
61
61
  console.error(`no turn ${args.show}`);
62
62
  process.exitCode = 1; return;
63
63
  }
64
- // Tombstoned turns are hidden from tool output, id lookup included.
65
- if (store.hiddenIds(byId).has(args.show)) {
64
+ // Session targets are absent from the bulk union: evaluate the claims
65
+ // against the resolved turn so id lookup respects their tombstones too.
66
+ if (store.claimHides(store.tombstoneClaims(byId), turn)) {
66
67
  console.error(`turn ${args.show} is tombstoned`);
67
68
  process.exitCode = 1; return;
68
69
  }
@@ -0,0 +1,193 @@
1
+ # Capture algorithms and recovery
2
+
3
+ The record package owns source cursors, journal validation, indexing and capture
4
+ status. This version changes those algorithms; it does not install a bounded
5
+ runner or change service scheduling, hooks, root registration or doctor.
6
+
7
+ ## Input contract
8
+
9
+ Input logs are append-only: capture reads added content, rereads on replacement,
10
+ shrink or checkpoint mismatch, cannot see arbitrary edits to an already-read
11
+ prefix that leave those checks intact, and `capture --reset-cursors` is the
12
+ remedy. This assumption applies to aweb logs and cc/pi/codex transcripts. It
13
+ relies on the native writers appending to existing transcript files.
14
+
15
+ The background cursor qualifies file identity, size, modification/change times
16
+ and a SHA-256 witness of at most 4 KiB before its byte boundary. An unchanged
17
+ version needs no historical content scan. Growth with the same identity and
18
+ witness resumes from the boundary. The witness does **not** prove that the
19
+ unobserved prefix is untouched. Explicit `captureCommLog` and
20
+ `captureInteractionLog` library calls retain their full-reprojection default.
21
+
22
+ A complete aweb entry refused by projection is counted when encountered and
23
+ consumed; later unchanged passes do not retry it. Use
24
+ `capture --reset-cursors --source aw` to request another projection attempt.
25
+ Malformed complete lines are similarly skipped once per scan. Unfinished final
26
+ lines keep their boundary for later growth; source read errors and failed durable
27
+ appends do not become successful consumed checkpoints.
28
+
29
+ Detected changes have two outcomes:
30
+
31
+ - Aweb logs reproject from the beginning. Persistent journal IDs suppress
32
+ already captured turns; changed projected content adds a different turn once.
33
+ Replay does not retract an earlier turn.
34
+ - Native transcripts compare their captured positional prefix against a private
35
+ staged snapshot before any append from that input. Equal content, including
36
+ under a new file identity, resumes at the captured boundary. A conflict sets
37
+ the input aside with source, path, line, byte offset and reason. Other inputs
38
+ can continue. This preserves rollback: older writers recover their offsets
39
+ from the physical last transcript turn, so appending a revised earlier line
40
+ would make those writers duplicate the remaining history.
41
+
42
+ ## Operator recovery
43
+
44
+ `turn-record capture --reset-cursors [--source aw|cc|pi|codex]` uses the selected
45
+ `--root` and `--owner` (or their existing environment defaults). Omit `--source`
46
+ to select all four. It takes the capture lock nonblockingly, atomically commits
47
+ source reset generations, then exits **without a capture pass**. The next pass
48
+ reprocesses those sources, including when a source cursor was lost. Sources,
49
+ journals, journal-ID lookup, journal-validation checkpoints, the search index,
50
+ ignore rules, tombstones and active set-aside entries are preserved. Held or
51
+ unknown locks, invalid arguments and failed writes refuse success.
52
+
53
+ For a conflicting native prefix, either:
54
+
55
+ 1. Restore the captured prefix and run `turn-record capture --retry-set-aside`.
56
+ Retry rechecks the prefix; it never authorizes a changed one.
57
+ 2. Add the source path or session ID to `<root>/ignore`. **Later turns from that
58
+ session will not be captured.** Reconciliation applies ignore before opening
59
+ the source, including missing or unreadable sources, and removes the ignored
60
+ input from active set-aside entries. Removing the ignore rule later does not
61
+ authorize the conflict.
62
+
63
+ Retry takes the same nonblocking capture lock, marks all current entries eligible
64
+ for revalidation, then performs one ordinary reconciliation while retaining the
65
+ lock. An interruption after marking leaves the warning active. It is not yet a
66
+ policy-bounded pass. Resetting cursors does not replace retry or clear a refusal.
67
+
68
+ `capture --status` keeps its existing store summary. `capture --health [--json]`
69
+ reads current active set-aside rows from `index/capture.db`, without capturing,
70
+ rebuilding, creating an absent root/database, or contacting a service manager.
71
+ The shared `capture-status.json` is the atomic **last command snapshot**: direct
72
+ synchronous library calls update active issues but do not rewrite that snapshot
73
+ or acquire an implicit root capture lock. Health gets historical run fields from
74
+ the snapshot and never advances them from current issues. A database with no
75
+ snapshot still exposes its active issues.
76
+
77
+ Both command snapshots and health use the same transactional count/sample
78
+ projection: at most 100 deterministic entries, total count and `truncated`.
79
+ Text output has the same bound and explains both remedies. Absent state on a
80
+ never-installed root is not a warning or a reason to create files. A missing
81
+ current database cannot erase a known nonempty command snapshot. Malformed or
82
+ unreadable status is reported as `capture-status-invalid`; an unavailable current
83
+ issue reader fails the command rather than returning successful stale/empty
84
+ health JSON.
85
+
86
+ Reading an existing database requires Node's read-only SQLite support: 22.12+
87
+ on the 22.x line (22.13+ avoids needing `--experimental-sqlite`), 23.2+ on 23.x,
88
+ or a newer major. Normal capture retains its declared runtime floor. Missing
89
+ SQLite support, unsupported schemas, lock/recovery errors, and WAL databases
90
+ fail health without migration or repair. Supported `capture.db` writers retain
91
+ rollback journaling. The health reader rejects WAL from its fixed header before
92
+ opening SQLite; it never creates WAL sidecars, switches mode, recovers a hot
93
+ journal or uses immutable/no-lock shortcuts. This does not protect against an
94
+ external actor racing a mode change or file replacement.
95
+
96
+ Header validation and the read-only transaction run in one private child using
97
+ the current Node executable, only the explicit SQLite runtime flag, and a clean
98
+ environment. It has a ten-second deadline, a 1 MiB output cap and no retries.
99
+ This costs one short process per health read and keeps a raw descriptor close
100
+ from releasing POSIX locks belonging to another SQLite connection in the caller.
101
+ Command snapshots reuse their owned connection for the shared projection.
102
+
103
+ The shared schema is [capture-status.schema.json](capture-status.schema.json).
104
+ In this algorithm-only phase `policy`, `lastRun`, `lastSuccess`, `lastFailure`
105
+ and `lastProgressAt` are null: evidence has not been supplied. They never mean
106
+ that a bounded pass, service installation or successful sweep occurred. Health
107
+ `status: "ok"` with empty rows means no reported set-aside fault, not service
108
+ liveness. Service supervision will extend this same mechanism with actual policy
109
+ and run evidence. Status replacement is atomic with private file permissions;
110
+ persistence failure is an operation failure, not a fabricated success.
111
+
112
+ ## Resource model
113
+
114
+ Source and journal reads use 64 KiB chunks. Batches commit at 2,000 records or
115
+ 8 MiB of accumulated raw bytes; a single larger record is still supported and
116
+ can exceed that batch target. SQLite connections use a 2 MiB page-cache target
117
+ and disk-backed temporary tables. Metadata query pages hold at most 128 rows;
118
+ keyset queries support the declared Node 22.5 SQLite API without rescanning
119
+ prior pages. Content-processing memory therefore depends on fixed buffers,
120
+ bounded batches/caches and the largest processable individual record, rather
121
+ than total source bytes or all historical IDs. These
122
+ are algorithm settings, **not** a memory/deadline policy or record-size limit.
123
+
124
+ Native root and path discovery retain in-memory metadata inventories, sorted by
125
+ the existing per-directory `localeCompare` order with stable ties and depth-first
126
+ traversal. Aweb account logs retain their code-unit sorted basename order. Their memory grows with file/profile counts, separately from streamed
127
+ content processing; metadata enumeration and sorting still run on unchanged
128
+ passes. Public list APIs retain their full sorted-array output. Per-input
129
+ cursors in `index/capture.db` replace whole-map JSON loads and rewrites. The
130
+ background caller requests capture counts without collecting a receipt inventory;
131
+ it does not run the explicit full-index count query merely to print progress.
132
+ Explicit API and receipt defaults retain complete issue/source/session
133
+ inventories, so their metadata memory may grow with the output they must return.
134
+ They do not retain whole source contents or journals to compute counts and
135
+ first/last visible IDs; receipt summaries use file-backed tombstone claims.
136
+
137
+ Final/attributed/Pi-custody capture and changed-transcript comparison can require
138
+ a complete verified snapshot before mutation. Those paths stream the source to
139
+ a private temporary file under `<root>/.capture-tmp`, hash it while reading,
140
+ and append from the same pinned bytes. Receipt claims databases use that same
141
+ record-root scratch area. Staging costs one source-sized temporary copy and is
142
+ removed on ordinary success or failure. These files are colocated on the record
143
+ root volume, not the system temp volume; the chosen record volume may itself
144
+ be memory-backed. Full receipt verification and recovery may read historical bytes;
145
+ this necessary proof work is distinct from unchanged/small-growth accounting.
146
+
147
+ Journal IDs and validation checkpoints are accelerators, never authority.
148
+ A validation checkpoint qualifies an exact device/inode/size/mtime/ctime version
149
+ under the stream lock. Unknown replacement, shrink or mutation forces bounded
150
+ validation before append. Journal fsync precedes the derived commit, so an
151
+ interruption leaves an old checkpoint that triggers validation. Validation
152
+ commits bounded progress; search rows and byte/count cursors commit together in
153
+ bounded transactions. Ordinary passes seek new journal bytes without parsing
154
+ consumed prefixes. Inconsistent identities or cursors invalidate derived state;
155
+ recovery can cost a historical scan. No journal format or turn ID changes.
156
+
157
+ Scratch directories are private per synchronous caller (0700 directories, 0600
158
+ owner/payload files), with a same-host PID and nonce marker. No supported path
159
+ transfers their ownership or open descriptors to a subprocess. A command holding
160
+ the capture lock removes recognized allocations only after positively proving
161
+ the local allocator dead and rechecking directory and marker identity. Public
162
+ synchronous callers still do not acquire an implicit capture lock; their live
163
+ allocations are preserved by command cleanup. Live/reused PIDs, unknown or foreign
164
+ owners, permission failures, links, special files and incomplete/missing markers
165
+ are preserved. A kill before ownership publication can leave an unowned directory.
166
+ For such leftovers, stop all capture/library users, establish that no caller is
167
+ using the specific allocation, and remove only that confirmed orphan; do not
168
+ blanket-delete the scratch area while callers may be active. No age-only cleanup.
169
+
170
+ ## First-upgrade cost
171
+
172
+ The first capture after upgrade must reconcile existing journals and rebuild the
173
+ search index. Per-input capture checkpoints and per-stream index transactions
174
+ still incur substantial synchronous commit costs on storage where sync is costly.
175
+ Connection/transaction amortization is unresolved by these algorithm changes;
176
+ bounded memory and incremental content work do not imply a cheap first pass.
177
+ The bounded service must separately qualify first-upgrade and unchanged passes
178
+ beside actual v0.53.0 on a Linux runner, including filesystem/runtime details,
179
+ connection/transaction/sync counts, resumable prefix proof and deadline progress.
180
+
181
+ ## Downgrade and re-upgrade
182
+
183
+ The supported path is v0.53.0 → this version → v0.53.0 → this version, including
184
+ appends in each phase. Known legacy `index/capture-offsets.json` and
185
+ `index/capture-aw-seen.json` maps are invalidated before appends can make them
186
+ unsafe; invalidation failure refuses the append. Unknown root files are left
187
+ alone. Old capture rebuilds its offsets and IDs from the unchanged journals.
188
+ The separate new byte-cursor table leaves the old search-index count schema and
189
+ old named-column writes compatible. Old appends invalidate exact journal
190
+ versions and mismatched index counts on re-upgrade, requiring derived rebuilds.
191
+ Deleting the derived caches has the same recovery cost. No journal-byte migration
192
+ or rollback is required. Native-prefix set-aside is essential to that guarantee;
193
+ it does not rewrite history or append a sentinel or duplicate terminal turn.
@@ -0,0 +1,100 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "title": "Capture status (algorithm phase)",
4
+ "type": "object",
5
+ "additionalProperties": false,
6
+ "required": [
7
+ "schemaVersion",
8
+ "policy",
9
+ "lastRun",
10
+ "lastSuccess",
11
+ "lastFailure",
12
+ "lastProgressAt",
13
+ "backlog"
14
+ ],
15
+ "properties": {
16
+ "schemaVersion": {
17
+ "const": 1
18
+ },
19
+ "policy": {
20
+ "type": "null",
21
+ "description": "No policy or run evidence supplied in the algorithm-only phase."
22
+ },
23
+ "lastRun": {
24
+ "type": "null",
25
+ "description": "No policy or run evidence supplied in the algorithm-only phase."
26
+ },
27
+ "lastSuccess": {
28
+ "type": "null",
29
+ "description": "No policy or run evidence supplied in the algorithm-only phase."
30
+ },
31
+ "lastFailure": {
32
+ "type": "null",
33
+ "description": "No policy or run evidence supplied in the algorithm-only phase."
34
+ },
35
+ "lastProgressAt": {
36
+ "type": "null",
37
+ "description": "No policy or run evidence supplied in the algorithm-only phase."
38
+ },
39
+ "backlog": {
40
+ "type": "object",
41
+ "additionalProperties": false,
42
+ "required": [
43
+ "pending",
44
+ "setAsideCount",
45
+ "setAside",
46
+ "truncated"
47
+ ],
48
+ "properties": {
49
+ "pending": {
50
+ "type": "boolean"
51
+ },
52
+ "setAsideCount": {
53
+ "type": "integer",
54
+ "minimum": 0
55
+ },
56
+ "setAside": {
57
+ "type": "array",
58
+ "maxItems": 100,
59
+ "items": {
60
+ "type": "object",
61
+ "additionalProperties": false,
62
+ "required": [
63
+ "source",
64
+ "path",
65
+ "offset",
66
+ "reason"
67
+ ],
68
+ "properties": {
69
+ "source": {
70
+ "enum": [
71
+ "aw",
72
+ "cc",
73
+ "pi",
74
+ "codex"
75
+ ]
76
+ },
77
+ "path": {
78
+ "type": "string"
79
+ },
80
+ "offset": {
81
+ "type": "integer",
82
+ "minimum": 0
83
+ },
84
+ "reason": {
85
+ "type": "string"
86
+ },
87
+ "line": {
88
+ "type": "integer",
89
+ "minimum": 1
90
+ }
91
+ }
92
+ }
93
+ },
94
+ "truncated": {
95
+ "type": "boolean"
96
+ }
97
+ }
98
+ }
99
+ }
100
+ }