@awebai/oats 0.54.0 → 0.55.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/bin/oats.mjs +5 -5
  2. package/docs/desktop-cli-api.md +15 -4
  3. package/docs/integrations.md +2 -2
  4. package/docs/knowledge.md +14 -2
  5. package/docs/official-catalog.md +4 -4
  6. package/docs/packages.md +10 -10
  7. package/docs/release-notes/v0.55.0.md +158 -0
  8. package/docs/servers.md +13 -3
  9. package/docs/workspaces.md +1 -1
  10. package/lib/servers.mjs +82 -34
  11. package/package-catalog.json +3 -3
  12. package/package.json +1 -1
  13. package/packages/client/cli-locator.mjs +2 -2
  14. package/packages/client/workspace-views.mjs +4 -3
  15. package/packages/record/README.md +54 -13
  16. package/packages/record/bin/capture.mjs +107 -53
  17. package/packages/record/docs/capture-service.md +193 -0
  18. package/packages/record/docs/capture-status.schema.json +100 -0
  19. package/packages/record/lib/capture-aw.mjs +96 -94
  20. package/packages/record/lib/capture-cc.mjs +109 -187
  21. package/packages/record/lib/capture-db-reader.mjs +39 -0
  22. package/packages/record/lib/capture-issues.mjs +64 -0
  23. package/packages/record/lib/capture-scratch.mjs +111 -0
  24. package/packages/record/lib/capture-state.mjs +105 -0
  25. package/packages/record/lib/derived-state.mjs +60 -0
  26. package/packages/record/lib/formats.mjs +31 -31
  27. package/packages/record/lib/index-db.mjs +74 -34
  28. package/packages/record/lib/receipt-summary.mjs +62 -0
  29. package/packages/record/lib/session-snapshot.mjs +56 -1
  30. package/packages/record/lib/sqlite-rows.mjs +14 -0
  31. package/packages/record/lib/store.mjs +122 -43
  32. package/packages/record/lib/transcript-prefix.mjs +40 -0
  33. package/skills/oats-getting-started/SKILL.md +1 -1
@@ -13,7 +13,7 @@
13
13
  // Every candidate is canonicalized to an absolute executable and accepted
14
14
  // ONLY if executable and `<bin> version --json` returns the v1 probe:
15
15
  // {"schemaVersion":1,"name":"@awebai/oats","version":"0.24.x","desktopApi":1}
16
- // Desktop accepts desktopApi === 1 and semver >=0.25.8 <0.55.0 (the band is
16
+ // Desktop accepts desktopApi === 1 and semver >=0.25.8 <0.56.0 (the band is
17
17
  // spelled ONCE, in ACCEPT_RANGE below — this line only paraphrases it).
18
18
  // API version — not source adjacency — is authoritative.
19
19
  //
@@ -54,7 +54,7 @@ export const DESKTOP_API = 1;
54
54
  // carry a `description` for every kind and `<kind> update <id> --description`
55
55
  // sets one (feature `automation-descriptions`, which gates the Desktop's
56
56
  // summary writes). The Desktop v1 surface is otherwise unchanged.
57
- export const ACCEPT_RANGE = { min: [0, 25, 8], maxExclusive: [0, 55, 0] };
57
+ export const ACCEPT_RANGE = { min: [0, 25, 8], maxExclusive: [0, 56, 0] };
58
58
  /** The band as humans read it — derived, never hand-spelled, so the probe
59
59
  * rejection reason, the backend's /api status and the degradation card can
60
60
  * never disagree with the numbers actually enforced above. */
@@ -60,8 +60,9 @@ const REASONS = {
60
60
  standalone: () => ({ short: 'Local teams only', detail: 'Teams are local only on this host (standalone).', fix: [] }),
61
61
  };
62
62
 
63
- /** True when a failed probe's message is ssh's own timeout: the kernel reports an ssh that timed out
64
- * as E_SSH with ssh's stderr and no code of its own, so ssh's text is the only signal (a documented limit). */
63
+ /** True when a failed probe's message is a timeout: the kernel reports an ssh that timed out as E_SSH,
64
+ * with ssh's own stderr or, when its own deadline ended the call, "timed out: no answer within N s", and
65
+ * no code of its own, so the text is the only signal (a documented limit). */
65
66
  const TIMED_OUT = /timed out|timeout/i;
66
67
 
67
68
  /** The client's own sentence for a deployment whose last read failed and whose last observation is kept. */
@@ -121,7 +122,7 @@ export function deploymentReasonParts(d, terms) {
121
122
  if (d.rosterError) return { short: 'Not reached', detail: `${machine} was not reached: ${d.rosterError}`, fix: [] };
122
123
  if (d.probe && d.probe.ok !== true) {
123
124
  const code = d.probe.error?.code, message = typeof d.probe.error?.message === 'string' ? d.probe.error.message : '';
124
- if (code === 'E_ROSTER_BUDGET' || code === 'E_CLI_TIMEOUT' || TIMED_OUT.test(message)) return { short: 'Timed out', detail: `${machine} timed out; it is tried again on the next read.`, fix: [] };
125
+ if (code === 'E_CLI_TIMEOUT' || TIMED_OUT.test(message)) return { short: 'Timed out', detail: `${machine} timed out; it is tried again on the next read.`, fix: [] };
125
126
  if (code === 'E_SSH') {
126
127
  const host = d.sshHost || machine;
127
128
  return { short: 'ssh needs a prompt', detail: `${machine} needs ssh to connect without a prompt; run \`ssh ${host}\` once in a terminal.`,
@@ -101,6 +101,32 @@ their shape, quoted or not, and replaces their command; a command it cannot
101
101
  read as its own is left alone and a new hook is added beside it.
102
102
  Stream owner: `--owner`, else `$TURN_RECORD_OWNER`, else the short hostname.
103
103
 
104
+ Transcript capture reads 64 KiB chunks and frames complete native records,
105
+ including UTF-8 characters split across chunks. It appends batches of at most
106
+ 8 MiB of source bytes plus the last record and checkpoints after each batch.
107
+ One very large native record can still require memory proportional to that
108
+ record; capture does not impose a new record-size policy.
109
+ Final, attributed and protected Pi capture stage the descriptor's snapshot on
110
+ private scratch under `<root>/.capture-tmp` before the first append, preserving
111
+ attribution and custody checks without retaining the source in process memory.
112
+ Exact `--file` capture and receipt claims databases use the same record-root
113
+ volume, which may itself be memory-backed. Ordinary success/failure cleans each
114
+ caller's allocation; lock-held cleanup reclaims only proven dead local owners.
115
+ See [scratch ownership and upgrade costs](docs/capture-service.md#resource-model)
116
+ for preserved uncertain leftovers and the unresolved synchronous first-pass cost.
117
+ Explicit final proofs may read the full snapshot; their output inventories
118
+ remain proportional to the issues and sessions the receipt must return.
119
+
120
+ The search index checkpoints byte positions and turn counts in the same
121
+ transaction as its indexed rows, at most 2,000 journal lines or 8 MiB plus
122
+ the last record per transaction. Known journal history is qualified by a
123
+ durable file-version checkpoint (device, inode, size and nanosecond modification
124
+ and change timestamps). Owner appends preserve its epoch after fsync; an
125
+ unknown write, replacement or shrink requires bounded journal validation and
126
+ invalidates dependent index cursors. Interrupted validation and indexing resume
127
+ from their last committed batch. Initial/recovery validation reads historical
128
+ content; unchanged normal passes do not.
129
+
104
130
  ## Ignoring sessions and paths (privacy)
105
131
 
106
132
  `<root>/ignore` is a plain text file of glob patterns, one per line (blank
@@ -249,19 +275,30 @@ into an orphaned inode until it restarts.
249
275
 
250
276
  ## Memory
251
277
 
252
- A capture pass runs in bounded memory, whatever the size of the record.
253
- Journals are streamed, never parsed whole: the first append to a stream
254
- validates its journal one line at a time. Comm logs are read in chunks, and
255
- the aw dedupe looks up only the changed log's own turn ids, in one streamed
256
- read of the journal. A session's new lines are walked, never collected, and
257
- appended in 8 MB batches. A lost offset is rebuilt from the journal's tail.
258
-
259
- So the heap a pass needs grows with one changed comm log's entry count, not
260
- with the record. A host job can keep a small `--max-old-space-size`;
261
- `packages/record/test/capture-bounded-memory.test.mjs` captures a record four
262
- times its 64 MB heap. One cost is not on the heap: the new bytes of one
263
- session are read as one buffer, so resident memory grows with a single
264
- session's backlog.
278
+ Content processing uses fixed read chunks, bounded batches and SQLite caches,
279
+ plus memory proportional to the largest processable native record. Journals,
280
+ transcript tails and comm logs are streamed; persistent ID lookup avoids keeping
281
+ history IDs or a changed log's candidate IDs in memory. Per-input offsets live
282
+ in derived database rows rather than a rewritten global JSON map. Initial
283
+ capture, recovery validation and explicit final proofs may read all applicable
284
+ content; ordinary unchanged and suffix-growth passes use only new content and
285
+ bounded checkpoint probes. The streaming and work-count tests pin these distinct
286
+ paths; the small-heap regression remains supplementary evidence.
287
+
288
+ Filesystem discovery is a separate cost: root/path metadata inventories may grow
289
+ with file and profile counts. Native roots and each directory retain their
290
+ existing `localeCompare` sorting and depth-first traversal, including stable ties.
291
+ Aweb account logs retain code-unit sorted basename order. Public
292
+ list APIs return full arrays. Metadata enumeration, sorting and per-input probes
293
+ still run on unchanged passes; this is not a constant-memory or new-byte-only
294
+ claim for the whole pass. Explicit receipts likewise retain the issue/session
295
+ inventories their public contract returns. Background capture suppresses those
296
+ output inventories without suppressing error counts.
297
+
298
+ Verified snapshots and prefix comparisons use private record-root scratch, which
299
+ can expand with source size; journal and derived-index disk space also grow with
300
+ captured content. Staging is cleaned on success and failure. No new heap, record
301
+ size, elapsed-time or service policy limit is imposed by these algorithms.
265
302
 
266
303
  ## Known costs, accepted for v1
267
304
 
@@ -341,3 +378,7 @@ Anything that binds nothing is an error,
341
378
  | `E_NO_TURNS` | 1 | no records: an empty file, or only blank lines |
342
379
  | `E_FORMAT` | 1 | records, but no session header of the stated format; the message names the format whose header it does carry |
343
380
  | `E_CAPTURE_FAILED` | 1 | the pass failed (`appended: null`: part of the file may be in the record) |
381
+
382
+ See [capture algorithms and recovery](docs/capture-service.md) for the shared
383
+ append-only input assumption, reset/retry commands, bounded status inventory,
384
+ resource accounting and v0.53.0 downgrade/re-upgrade contract.
@@ -20,12 +20,16 @@ import process from "node:process";
20
20
  import { RecordStore } from "../lib/store.mjs";
21
21
  import { captureAllSessions, captureSessions } from "../lib/capture-cc.mjs";
22
22
  import { sessionsForHome } from "../lib/sessions-for-home.mjs";
23
- import { jsonlLines, SESSION_FORMATS } from "../lib/formats.mjs";
23
+ import { SESSION_FORMATS } from "../lib/formats.mjs";
24
24
  import { cwdOfLine } from "../lib/sessions-for-home.mjs";
25
- import { digest, readRange } from "../lib/session-snapshot.mjs";
25
+ import { stageSnapshot } from "../lib/session-snapshot.mjs";
26
+ import { cleanupCaptureScratch } from "../lib/capture-scratch.mjs";
27
+ import { bufferLines } from "../lib/file-lines.mjs";
26
28
  import { captureAwLogs, defaultCommLogDir } from "../lib/capture-aw.mjs";
29
+ import { withReceiptSummaries } from "../lib/receipt-summary.mjs";
27
30
  import { RecordIndex } from "../lib/index-db.mjs";
28
31
  import { IgnoreError, ignoreFilePath, loadIgnore } from "../lib/ignore.mjs";
32
+ import { CAPTURE_SOURCES, captureHealth, resetSourceCursors, retrySetAside, writeCaptureStatus } from "../lib/capture-state.mjs";
29
33
  import { acquireCaptureLock } from "../lib/capture-lock.mjs";
30
34
 
31
35
  // Fail closed but actionably: an unreadable ignore file must stop capture,
@@ -45,8 +49,11 @@ function loadIgnoreOrExit(recordRoot) {
45
49
  // hostname-derived owner, which forked the whole record into a second owner
46
50
  // namespace: 571 duplicate journals from one typo. Parsing must refuse what
47
51
  // it does not understand before anything can be written.
48
- const VALUE_FLAGS = new Set(["root", "owner", "home", "file", "format"]);
52
+ const VALUE_FLAGS = new Set(["root", "owner", "home", "file", "format", "source"]);
49
53
  const BOOL_FLAGS = new Set([
54
+ "reset-cursors",
55
+ "retry-set-aside",
56
+ "health",
50
57
  "watch",
51
58
  "status",
52
59
  "install-hint",
@@ -67,6 +74,12 @@ const USAGE = `capture — land sessions and aw client logs in the turn record.
67
74
  capture --watch pass now, then re-pass on filesystem change
68
75
  (debounced) and every 15 minutes regardless
69
76
  capture --status show store/stream summary, capture nothing
77
+ capture --health [--json] read capture set-aside diagnosis, capture nothing
78
+ capture --reset-cursors [--source aw|cc|pi|codex]
79
+ reset selected source cursors and exit; the next
80
+ pass reprocesses them with journal deduplication
81
+ capture --retry-set-aside revalidate set-aside inputs in one reconciliation;
82
+ does not authorize changing captured transcripts
70
83
  capture --home <dir> capture the sessions that ran inside <dir> (an
71
84
  OATS instance home) and print them as JSON:
72
85
  thread, stream, turn count, first/last turn id;
@@ -146,6 +159,10 @@ function parseArgs(argv) {
146
159
  let args;
147
160
  try {
148
161
  args = parseArgs(process.argv.slice(2));
162
+ const recovery = ["reset-cursors", "retry-set-aside", "health"].filter(key => args[key]);
163
+ if (recovery.length > 1) throw new UsageError("choose one recovery or health operation");
164
+ if (args.source !== undefined && (!args["reset-cursors"] || !CAPTURE_SOURCES.includes(args.source))) throw new UsageError("--source requires --reset-cursors and aw|cc|pi|codex");
165
+ if (recovery.length && ["home", "file", "format", "watch", "status", "install-hint", "current-roots", "sessions-only", "aw-only"].some(key => args[key])) throw new UsageError("recovery/health cannot be combined with another capture mode");
149
166
  } catch (err) {
150
167
  if (!(err instanceof UsageError)) throw err;
151
168
  // --file --json answers JSON even for a usage error: it is for programs.
@@ -165,9 +182,43 @@ if (args.help) {
165
182
  const root = args.root ?? process.env.TURN_RECORD_ROOT ?? join(homedir(), ".turn-record");
166
183
  const ownerExplicit = args.owner !== undefined || process.env.TURN_RECORD_OWNER !== undefined;
167
184
  const owner = args.owner ?? process.env.TURN_RECORD_OWNER ?? hostname().split(".")[0];
168
- const quiet = Boolean(args.quiet);
185
+ const quiet = Boolean(args.quiet || args.json && (args["reset-cursors"] || args["retry-set-aside"]));
169
186
  const store = new RecordStore(root, { owner });
170
187
 
188
+ if (args.health) {
189
+ try {
190
+ const health = captureHealth(root);
191
+ if (args.json) console.log(JSON.stringify(health, null, 2));
192
+ else {
193
+ console.log(`capture: ${health.status}`);
194
+ for (const row of health.rows) console.log(`${row.code}: ${row.message}`);
195
+ for (const entry of health.backlog.setAside) console.log(`${entry.source}: ${entry.path}:${entry.line ?? "?"} offset ${entry.offset}: ${entry.reason}`);
196
+ if (health.backlog.truncated) console.log("showing first 100 set-aside inputs");
197
+ }
198
+ } catch (err) {
199
+ if (args.json) console.log(JSON.stringify({ status: "failed", error: err.message }));
200
+ else console.error(`capture: ${err.message}`);
201
+ process.exitCode = 1;
202
+ }
203
+ process.exit(process.exitCode ?? 0);
204
+ }
205
+ if (args["reset-cursors"] || args["retry-set-aside"]) {
206
+ try {
207
+ const result = args["reset-cursors"]
208
+ ? withCaptureLock(() => { resetSourceCursors(root, owner, args.source); return { reset: true, sources: args.source ? [args.source] : CAPTURE_SOURCES }; })
209
+ : pass({ retry: true });
210
+ if (result.skipped) throw new Error("capture lock held; recovery was not performed");
211
+ if (process.exitCode) throw new Error("recovery failed to release its capture lock");
212
+ if (args.json) console.log(JSON.stringify(result));
213
+ else if (args["reset-cursors"]) log(`capture: reset source cursors (${result.sources.join(", ")})`);
214
+ } catch (err) {
215
+ if (args.json) console.log(JSON.stringify({ status: "failed", complete: false, error: err.message }));
216
+ else console.error(`capture: ${err.message}`);
217
+ process.exitCode = 1;
218
+ }
219
+ process.exit(process.exitCode ?? 0);
220
+ }
221
+
171
222
  // The owner decides the stream namespace, and turn ids hash it into the
172
223
  // canonical core — so the same conversation captured under two owners yields
173
224
  // two sets of ids for identical content, which the index cannot dedupe. When
@@ -178,21 +229,18 @@ const store = new RecordStore(root, { owner });
178
229
  // root, no other owners, and nothing is printed.
179
230
  function warnOnStrangerOwner() {
180
231
  if (ownerExplicit) return;
181
- let others;
232
+ let example;
182
233
  try {
183
- others = new Set(
184
- store
185
- .listStreams()
186
- .map((id) => id.slice(0, id.indexOf("~")))
187
- .filter(Boolean),
188
- );
189
- } catch {
190
- return; // unreadable or absent root: not this function's business
191
- }
192
- if (others.size === 0 || others.has(owner)) return;
234
+ for (const stream of store.iterateStreams()) {
235
+ const existing = stream.slice(0, stream.indexOf("~"));
236
+ if (existing === owner) return;
237
+ if (existing) example ??= existing;
238
+ }
239
+ } catch { return; }
240
+ if (!example) return;
193
241
  console.error(
194
242
  `capture: writing as owner "${owner}" (from hostname), but this record holds ` +
195
- `streams only under ${[...others].sort().map((o) => `"${o}"`).join(", ")}. ` +
243
+ `streams under another owner such as "${example}". ` +
196
244
  `Pass --owner or set TURN_RECORD_OWNER if that is not what you meant — ` +
197
245
  `capturing under a second owner duplicates the record rather than extending it.`,
198
246
  );
@@ -240,7 +288,16 @@ function withCaptureLock(fn) {
240
288
  return { appended: 0, skipped: true, lock: lock.held };
241
289
  }
242
290
  try {
243
- return fn();
291
+ let result, failure;
292
+ try { cleanupCaptureScratch(root, lock); result = fn(); return result; }
293
+ catch (err) { failure = err; throw err; }
294
+ finally {
295
+ try { writeCaptureStatus(root, { pending: Boolean(failure || result?.pending || result?.held || result?.incomplete || result?.failed) }); }
296
+ catch (err) {
297
+ if (failure) throw new Error(`${failure.message}; capture status persistence also failed: ${err.message}`, { cause: failure });
298
+ throw err;
299
+ }
300
+ }
244
301
  } finally {
245
302
  const r = lock.release();
246
303
  if (!r.released) {
@@ -256,15 +313,17 @@ function withCaptureLock(fn) {
256
313
  }
257
314
  }
258
315
 
259
- function pass() {
316
+ function pass({ retry = false } = {}) {
260
317
  // The privacy loader fails closed by exiting; it runs before the lock is
261
318
  // taken so that exit never leaves the lock behind.
262
319
  const ignore = loadIgnoreOrExit(root);
263
320
  return withCaptureLock(() => {
264
- const out = { appended: 0 };
321
+ if (retry) retrySetAside(root);
322
+ const out = { appended: 0, pending: false };
265
323
  if (!args["aw-only"]) {
266
- for (const r of captureAllSessions(store, { owner, ignore })) {
324
+ for (const r of captureAllSessions(store, { owner, ignore, collectIssues: false })) {
267
325
  out.appended += r.appended;
326
+ out.pending ||= !r.complete;
268
327
  const extras = [r.ignored ? `${r.ignored} ignored` : "", r.held ? `${r.held} held` : "", r.incomplete ? `${r.incomplete} incomplete` : ""]
269
328
  .filter(Boolean)
270
329
  .join(", ");
@@ -279,7 +338,7 @@ function pass() {
279
338
  let awFailed = 0;
280
339
  let awFiles = 0;
281
340
  let awIgnored = 0;
282
- for (const r of captureAwLogs(store, { owner, ignore })) {
341
+ for (const r of captureAwLogs(store, { owner, ignore, collect: false })) {
283
342
  if (r.ignored) {
284
343
  awIgnored++;
285
344
  continue;
@@ -288,6 +347,7 @@ function pass() {
288
347
  awEntries += r.entries;
289
348
  awAppended += r.appended;
290
349
  awFailed += r.failed;
350
+ out.pending ||= r.failed > 0;
291
351
  out.appended += r.appended;
292
352
  if (r.failed) console.error(`aw-log ${r.account}: ${r.failed} entries failed to project`);
293
353
  }
@@ -302,7 +362,7 @@ function pass() {
302
362
  const index = new RecordIndex(store);
303
363
  try {
304
364
  index.update();
305
- log(`index: updated (${JSON.stringify(index.counts())})`);
365
+ log("index: updated");
306
366
  } finally {
307
367
  index.close();
308
368
  }
@@ -335,12 +395,6 @@ function captureUnderLock(recordStore, recordOwner, batches, ignore, outcome, is
335
395
  });
336
396
  }
337
397
 
338
- /** A stream's turns that no tombstone hides. A tombstoned turn is hidden everywhere; a boundary naming one
339
- * would be refused by recall, so boundaries come from the visible turns only. */
340
- function visibleTurns(recordStore, stream, claims = recordStore.tombstoneClaims()) {
341
- return recordStore.readStream(stream).filter((t) => !recordStore.claimHides(claims, t));
342
- }
343
-
344
398
  /** The outcome vocabulary: complete only with no failure, skip, hold or incomplete tail (and, for --home, no
345
399
  * unattributed source). */
346
400
  function statusOf(outcome, unattributed = 0) {
@@ -366,21 +420,18 @@ function isFormat(found, stated) {
366
420
  /** Records (non-blank lines, an undecodable one included) and the formats whose session header a COMPLETE
367
421
  * line carries. A pi or codex file is settled at its header; a cc file is judged whole. A line its writer
368
422
  * never terminated is no header. */
369
- function sessionHeaders(bytes, stated) {
423
+ function sessionHeaders(chunks, stated) {
370
424
  let records = 0;
371
425
  const found = new Set();
372
- const terminated = bytes.length > 0 && bytes[bytes.length - 1] === 10;
373
- const take = (text, complete) => {
426
+ for (const line of bufferLines(chunks)) {
427
+ const complete = line[line.length - 1] === 10;
428
+ let text = null;
429
+ try { text = (complete ? line.subarray(0, -1) : line).toString("utf8"); }
430
+ catch (err) { if (err.code !== "ERR_STRING_TOO_LONG") throw err; }
374
431
  if (text === null || text.trim() !== "") records++;
375
432
  if (complete && text !== null) for (const source of HEADER_ORDER) if (cwdOfLine(source, text) !== undefined) found.add(source);
376
- };
377
- let pending;
378
- for (const { text } of jsonlLines(bytes)) {
379
- if (pending !== undefined) take(pending, true);
380
- if (stated !== "cc" && found.has(stated)) return { records, found };
381
- pending = text;
433
+ if (stated !== "cc" && found.has(stated)) break;
382
434
  }
383
- if (pending !== undefined) take(pending, terminated);
384
435
  return { records, found };
385
436
  }
386
437
 
@@ -430,17 +481,18 @@ function fileCapture() {
430
481
  if (err.code === "ELOOP" || err.code === "EMLINK") return fail("E_NOT_REGULAR_FILE", `${path} is a symbolic link; pass the file it names`, where);
431
482
  return fail("E_FILE_UNREADABLE", `cannot open ${path}: ${err.message}`, where);
432
483
  }
433
- let stat, bytes;
484
+ let stat, pinned;
434
485
  try {
435
486
  stat = fstatSync(fd);
436
487
  if (!stat.isFile()) return fail("E_NOT_REGULAR_FILE", `${path} is ${fileKind(stat)}, not a regular file`, where);
437
- bytes = readRange(fd, 0, stat.size, path);
488
+ pinned = stageSnapshot(fd, stat, path, undefined, root);
438
489
  } catch (err) {
439
490
  return fail("E_FILE_UNREADABLE", `cannot read ${path}: ${err.message}`, where);
440
491
  } finally { closeSync(fd); }
441
492
 
442
- const { records, found } = sessionHeaders(bytes, args.format);
443
- if (!records) return fail("E_NO_TURNS", `${path} has no records${bytes.length ? " (only blank lines)" : " (it is empty)"}`, where);
493
+ try {
494
+ const { records, found } = sessionHeaders(pinned.chunks(), args.format);
495
+ if (!records) return fail("E_NO_TURNS", `${path} has no records${stat.size ? " (only blank lines)" : " (it is empty)"}`, where);
444
496
  if (!isFormat(found, args.format)) {
445
497
  const other = HEADER_ORDER.find((f) => f !== args.format && isFormat(found, f));
446
498
  return fail("E_FORMAT", other
@@ -448,12 +500,12 @@ function fileCapture() {
448
500
  : `${path} has no ${args.format} session header (${SESSION_HEADER[args.format]})`, where);
449
501
  }
450
502
 
451
- const sha256 = digest(bytes);
503
+ const sha256 = pinned.hash;
452
504
  const fileStore = new RecordStore(root, { owner: fileOwner });
453
505
  const outcome = { appended: 0, skipped: false, held: 0, incomplete: 0, failed: 0, ignored: 0 };
454
506
  const issues = [];
455
507
  try {
456
- Object.assign(outcome, captureUnderLock(fileStore, fileOwner, [[args.format, [{ path, pinned: { bytes, stat } }]]], ignore, outcome, issues));
508
+ Object.assign(outcome, captureUnderLock(fileStore, fileOwner, [[args.format, [{ path, pinned }]]], ignore, outcome, issues));
457
509
  } catch (err) {
458
510
  // A failed pass may have appended part of the file: never claim a count for it.
459
511
  return fail("E_CAPTURE_FAILED", err.message || String(err), { ...where, sha256, appended: null });
@@ -461,17 +513,18 @@ function fileCapture() {
461
513
  if (process.exitCode) return fail("E_CAPTURE_FAILED", "capture lock release failed; see stderr for recovery", { ...where, sha256 });
462
514
 
463
515
  const stream = `${fileOwner}~${fmt.source}.${sessionId}`;
464
- const turns = visibleTurns(fileStore, stream);
516
+ const turns = withReceiptSummaries(fileStore, summary => summary(stream));
465
517
  const status = statusOf(outcome);
466
518
  const { lock: _lock, ...counts } = outcome;
467
519
  const receipt = {
468
520
  ...where, instance, thread: `${fmt.source}:session:${sessionId}`, stream, sessionId,
469
- turns: turns.length, firstTurnId: turns[0]?.id ?? null, lastTurnId: turns.at(-1)?.id ?? null,
521
+ turns: turns.count, firstTurnId: turns.first?.id ?? null, lastTurnId: turns.last?.id ?? null,
470
522
  ...counts, status, complete: status === "complete", sha256, ...(issues.length ? { issues } : {}),
471
523
  };
472
524
  if (json) console.log(JSON.stringify(receipt, null, 2));
473
525
  else console.log(`capture --file: ${status}, ${receipt.turns} turns (${outcome.appended} new) in ${stream}, sha256 ${sha256}`);
474
526
  return 0;
527
+ } finally { pinned.close(); }
475
528
  }
476
529
 
477
530
  if (args.file !== undefined || args.format !== undefined || args.json) process.exit(fileCapture());
@@ -535,11 +588,11 @@ if (args.home) {
535
588
  formats.get(s.source).push(s);
536
589
  }
537
590
  Object.assign(outcome, captureUnderLock(store, owner, formats, ignore, outcome, issues));
538
- const claims = store.tombstoneClaims();
591
+ withReceiptSummaries(store, summary => {
539
592
  for (const s of found) {
540
593
  const stream = `${owner}~${s.source}.${s.sessionId}`;
541
- const turns = visibleTurns(store, stream, claims);
542
- if (!turns.length) continue; // ignored by rule, nothing capturable yet, or all hidden
594
+ const turns = summary(stream);
595
+ if (!turns.count) continue; // ignored by rule, nothing capturable yet, or all hidden
543
596
  sessions.push({
544
597
  thread: s.thread,
545
598
  source: s.source,
@@ -547,12 +600,13 @@ if (args.home) {
547
600
  path: s.path,
548
601
  cwd: s.cwd,
549
602
  stream,
550
- turns: turns.length,
551
- firstTurnId: turns[0].id,
552
- lastTurnId: turns[turns.length - 1].id,
553
- lastTs: turns[turns.length - 1].ts,
603
+ turns: turns.count,
604
+ firstTurnId: turns.first.id,
605
+ lastTurnId: turns.last.id,
606
+ lastTs: turns.last.ts,
554
607
  });
555
608
  }
609
+ });
556
610
  } catch (err) {
557
611
  error = err.message || String(err);
558
612
  console.error(`capture pass failed: ${error}`);
@@ -0,0 +1,193 @@
1
+ # Capture algorithms and recovery
2
+
3
+ The record package owns source cursors, journal validation, indexing and capture
4
+ status. This version changes those algorithms; it does not install a bounded
5
+ runner or change service scheduling, hooks, root registration or doctor.
6
+
7
+ ## Input contract
8
+
9
+ Input logs are append-only: capture reads added content, rereads on replacement,
10
+ shrink or checkpoint mismatch, cannot see arbitrary edits to an already-read
11
+ prefix that leave those checks intact, and `capture --reset-cursors` is the
12
+ remedy. This assumption applies to aweb logs and cc/pi/codex transcripts. It
13
+ relies on the native writers appending to existing transcript files.
14
+
15
+ The background cursor qualifies file identity, size, modification/change times
16
+ and a SHA-256 witness of at most 4 KiB before its byte boundary. An unchanged
17
+ version needs no historical content scan. Growth with the same identity and
18
+ witness resumes from the boundary. The witness does **not** prove that the
19
+ unobserved prefix is untouched. Explicit `captureCommLog` and
20
+ `captureInteractionLog` library calls retain their full-reprojection default.
21
+
22
+ A complete aweb entry refused by projection is counted when encountered and
23
+ consumed; later unchanged passes do not retry it. Use
24
+ `capture --reset-cursors --source aw` to request another projection attempt.
25
+ Malformed complete lines are similarly skipped once per scan. Unfinished final
26
+ lines keep their boundary for later growth; source read errors and failed durable
27
+ appends do not become successful consumed checkpoints.
28
+
29
+ Detected changes have two outcomes:
30
+
31
+ - Aweb logs reproject from the beginning. Persistent journal IDs suppress
32
+ already captured turns; changed projected content adds a different turn once.
33
+ Replay does not retract an earlier turn.
34
+ - Native transcripts compare their captured positional prefix against a private
35
+ staged snapshot before any append from that input. Equal content, including
36
+ under a new file identity, resumes at the captured boundary. A conflict sets
37
+ the input aside with source, path, line, byte offset and reason. Other inputs
38
+ can continue. This preserves rollback: older writers recover their offsets
39
+ from the physical last transcript turn, so appending a revised earlier line
40
+ would make those writers duplicate the remaining history.
41
+
42
+ ## Operator recovery
43
+
44
+ `turn-record capture --reset-cursors [--source aw|cc|pi|codex]` uses the selected
45
+ `--root` and `--owner` (or their existing environment defaults). Omit `--source`
46
+ to select all four. It takes the capture lock nonblockingly, atomically commits
47
+ source reset generations, then exits **without a capture pass**. The next pass
48
+ reprocesses those sources, including when a source cursor was lost. Sources,
49
+ journals, journal-ID lookup, journal-validation checkpoints, the search index,
50
+ ignore rules, tombstones and active set-aside entries are preserved. Held or
51
+ unknown locks, invalid arguments and failed writes refuse success.
52
+
53
+ For a conflicting native prefix, either:
54
+
55
+ 1. Restore the captured prefix and run `turn-record capture --retry-set-aside`.
56
+ Retry rechecks the prefix; it never authorizes a changed one.
57
+ 2. Add the source path or session ID to `<root>/ignore`. **Later turns from that
58
+ session will not be captured.** Reconciliation applies ignore before opening
59
+ the source, including missing or unreadable sources, and removes the ignored
60
+ input from active set-aside entries. Removing the ignore rule later does not
61
+ authorize the conflict.
62
+
63
+ Retry takes the same nonblocking capture lock, marks all current entries eligible
64
+ for revalidation, then performs one ordinary reconciliation while retaining the
65
+ lock. An interruption after marking leaves the warning active. It is not yet a
66
+ policy-bounded pass. Resetting cursors does not replace retry or clear a refusal.
67
+
68
+ `capture --status` keeps its existing store summary. `capture --health [--json]`
69
+ reads current active set-aside rows from `index/capture.db`, without capturing,
70
+ rebuilding, creating an absent root/database, or contacting a service manager.
71
+ The shared `capture-status.json` is the atomic **last command snapshot**: direct
72
+ synchronous library calls update active issues but do not rewrite that snapshot
73
+ or acquire an implicit root capture lock. Health gets historical run fields from
74
+ the snapshot and never advances them from current issues. A database with no
75
+ snapshot still exposes its active issues.
76
+
77
+ Both command snapshots and health use the same transactional count/sample
78
+ projection: at most 100 deterministic entries, total count and `truncated`.
79
+ Text output has the same bound and explains both remedies. Absent state on a
80
+ never-installed root is not a warning or a reason to create files. A missing
81
+ current database cannot erase a known nonempty command snapshot. Malformed or
82
+ unreadable status is reported as `capture-status-invalid`; an unavailable current
83
+ issue reader fails the command rather than returning successful stale/empty
84
+ health JSON.
85
+
86
+ Reading an existing database requires Node's read-only SQLite support: 22.12+
87
+ on the 22.x line (22.13+ avoids needing `--experimental-sqlite`), 23.2+ on 23.x,
88
+ or a newer major. Normal capture retains its declared runtime floor. Missing
89
+ SQLite support, unsupported schemas, lock/recovery errors, and WAL databases
90
+ fail health without migration or repair. Supported `capture.db` writers retain
91
+ rollback journaling. The health reader rejects WAL from its fixed header before
92
+ opening SQLite; it never creates WAL sidecars, switches mode, recovers a hot
93
+ journal or uses immutable/no-lock shortcuts. This does not protect against an
94
+ external actor racing a mode change or file replacement.
95
+
96
+ Header validation and the read-only transaction run in one private child using
97
+ the current Node executable, only the explicit SQLite runtime flag, and a clean
98
+ environment. It has a ten-second deadline, a 1 MiB output cap and no retries.
99
+ This costs one short process per health read and keeps a raw descriptor close
100
+ from releasing POSIX locks belonging to another SQLite connection in the caller.
101
+ Command snapshots reuse their owned connection for the shared projection.
102
+
103
+ The shared schema is [capture-status.schema.json](capture-status.schema.json).
104
+ In this algorithm-only phase `policy`, `lastRun`, `lastSuccess`, `lastFailure`
105
+ and `lastProgressAt` are null: evidence has not been supplied. They never mean
106
+ that a bounded pass, service installation or successful sweep occurred. Health
107
+ `status: "ok"` with empty rows means no reported set-aside fault, not service
108
+ liveness. Service supervision will extend this same mechanism with actual policy
109
+ and run evidence. Status replacement is atomic with private file permissions;
110
+ persistence failure is an operation failure, not a fabricated success.
111
+
112
+ ## Resource model
113
+
114
+ Source and journal reads use 64 KiB chunks. Batches commit at 2,000 records or
115
+ 8 MiB of accumulated raw bytes; a single larger record is still supported and
116
+ can exceed that batch target. SQLite connections use a 2 MiB page-cache target
117
+ and disk-backed temporary tables. Metadata query pages hold at most 128 rows;
118
+ keyset queries support the declared Node 22.5 SQLite API without rescanning
119
+ prior pages. Content-processing memory therefore depends on fixed buffers,
120
+ bounded batches/caches and the largest processable individual record, rather
121
+ than total source bytes or all historical IDs. These
122
+ are algorithm settings, **not** a memory/deadline policy or record-size limit.
123
+
124
+ Native root and path discovery retain in-memory metadata inventories, sorted by
125
+ the existing per-directory `localeCompare` order with stable ties and depth-first
126
+ traversal. Aweb account logs retain their code-unit sorted basename order. Their memory grows with file/profile counts, separately from streamed
127
+ content processing; metadata enumeration and sorting still run on unchanged
128
+ passes. Public list APIs retain their full sorted-array output. Per-input
129
+ cursors in `index/capture.db` replace whole-map JSON loads and rewrites. The
130
+ background caller requests capture counts without collecting a receipt inventory;
131
+ it does not run the explicit full-index count query merely to print progress.
132
+ Explicit API and receipt defaults retain complete issue/source/session
133
+ inventories, so their metadata memory may grow with the output they must return.
134
+ They do not retain whole source contents or journals to compute counts and
135
+ first/last visible IDs; receipt summaries use file-backed tombstone claims.
136
+
137
+ Final/attributed/Pi-custody capture and changed-transcript comparison can require
138
+ a complete verified snapshot before mutation. Those paths stream the source to
139
+ a private temporary file under `<root>/.capture-tmp`, hash it while reading,
140
+ and append from the same pinned bytes. Receipt claims databases use that same
141
+ record-root scratch area. Staging costs one source-sized temporary copy and is
142
+ removed on ordinary success or failure. These files are colocated on the record
143
+ root volume, not the system temp volume; the chosen record volume may itself
144
+ be memory-backed. Full receipt verification and recovery may read historical bytes;
145
+ this necessary proof work is distinct from unchanged/small-growth accounting.
146
+
147
+ Journal IDs and validation checkpoints are accelerators, never authority.
148
+ A validation checkpoint qualifies an exact device/inode/size/mtime/ctime version
149
+ under the stream lock. Unknown replacement, shrink or mutation forces bounded
150
+ validation before append. Journal fsync precedes the derived commit, so an
151
+ interruption leaves an old checkpoint that triggers validation. Validation
152
+ commits bounded progress; search rows and byte/count cursors commit together in
153
+ bounded transactions. Ordinary passes seek new journal bytes without parsing
154
+ consumed prefixes. Inconsistent identities or cursors invalidate derived state;
155
+ recovery can cost a historical scan. No journal format or turn ID changes.
156
+
157
+ Scratch directories are private per synchronous caller (0700 directories, 0600
158
+ owner/payload files), with a same-host PID and nonce marker. No supported path
159
+ transfers their ownership or open descriptors to a subprocess. A command holding
160
+ the capture lock removes recognized allocations only after positively proving
161
+ the local allocator dead and rechecking directory and marker identity. Public
162
+ synchronous callers still do not acquire an implicit capture lock; their live
163
+ allocations are preserved by command cleanup. Live/reused PIDs, unknown or foreign
164
+ owners, permission failures, links, special files and incomplete/missing markers
165
+ are preserved. A kill before ownership publication can leave an unowned directory.
166
+ For such leftovers, stop all capture/library users, establish that no caller is
167
+ using the specific allocation, and remove only that confirmed orphan; do not
168
+ blanket-delete the scratch area while callers may be active. No age-only cleanup.
169
+
170
+ ## First-upgrade cost
171
+
172
+ The first capture after upgrade must reconcile existing journals and rebuild the
173
+ search index. Per-input capture checkpoints and per-stream index transactions
174
+ still incur substantial synchronous commit costs on storage where sync is costly.
175
+ Connection/transaction amortization is unresolved by these algorithm changes;
176
+ bounded memory and incremental content work do not imply a cheap first pass.
177
+ The bounded service must separately qualify first-upgrade and unchanged passes
178
+ beside actual v0.53.0 on a Linux runner, including filesystem/runtime details,
179
+ connection/transaction/sync counts, resumable prefix proof and deadline progress.
180
+
181
+ ## Downgrade and re-upgrade
182
+
183
+ The supported path is v0.53.0 → this version → v0.53.0 → this version, including
184
+ appends in each phase. Known legacy `index/capture-offsets.json` and
185
+ `index/capture-aw-seen.json` maps are invalidated before appends can make them
186
+ unsafe; invalidation failure refuses the append. Unknown root files are left
187
+ alone. Old capture rebuilds its offsets and IDs from the unchanged journals.
188
+ The separate new byte-cursor table leaves the old search-index count schema and
189
+ old named-column writes compatible. Old appends invalidate exact journal
190
+ versions and mismatched index counts on re-upgrade, requiring derived rebuilds.
191
+ Deleting the derived caches has the same recovery cost. No journal-byte migration
192
+ or rollback is required. Native-prefix set-aside is essential to that guarantee;
193
+ it does not rewrite history or append a sentinel or duplicate terminal turn.