@awebai/oats 0.35.1 → 0.35.2

package/docs/release-notes/v0.35.2.md

@@ -0,0 +1,20 @@
+# OATS 0.35.2
+
+## Fixed
+
+- **A capture pass runs in bounded memory, whatever the record's size**
+  (awebai/oats#456). On a host with a 2.6 GB turn record, the periodic pass
+  (`capture.mjs` under `--max-old-space-size=256`) died within seconds every
+  time it ran. Each death left a dead-owner capture lock, the cause of #437.
+  The pass loaded whole files:
+  - the first append to a stream parsed its whole journal;
+  - the aw dedupe read the whole aw journal into a set of ids;
+  - every changed comm log was read whole (a log over 512 MB failed even
+    with a large heap);
+  - a lost session offset re-read the whole journal and transcript;
+  - a session's new lines were all collected before any was appended.
+
+  Each is now streamed. On a 1.1 GB synthetic record the same pass peaks at
+  160 MB resident instead of about 2 GB. A host whose capture job was turned
+  off can turn it back on: the first pass after a long gap captures the
+  backlog in bounded memory.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@awebai/oats",
-  "version": "0.35.1",
+  "version": "0.35.2",
   "description": "OATS (Open Agent Team Specification) — durable souls, disposable instances, targetable capability packages, and the runtime-neutral oats CLI/kernel.",
   "keywords": [
     "agents",

package/packages/record/README.md

@@ -226,6 +226,22 @@ this package, restart any long-running `capture --watch` process — a
 daemon holding the old database file open would otherwise keep indexing
 into an orphaned inode until it restarts.
 
+## Memory
+
+A capture pass runs in bounded memory, whatever the size of the record.
+Journals are streamed, never parsed whole: the first append to a stream
+validates its journal one line at a time. Comm logs are read in chunks, and
+the aw dedupe looks up only the changed log's own turn ids, in one streamed
+read of the journal. A session's new lines are walked, never collected, and
+appended in 8 MB batches. A lost offset is rebuilt from the journal's tail.
+
+So the heap a pass needs grows with one changed comm log's entry count, not
+with the record. A host job can keep a small `--max-old-space-size`;
+`packages/record/test/capture-bounded-memory.test.mjs` captures a record four
+times its 64 MB heap. One cost is not on the heap: the new bytes of one
+session are read as one buffer, so resident memory grows with a single
+session's backlog.
+
 ## Known costs, accepted for v1
 
 - Deletion via tombstone is eventual: an offline replica retains bytes until

package/packages/record/lib/capture-aw.mjs

@@ -9,9 +9,14 @@
 // account; account and file identity live in each turn's provenance.
 // Projection is deterministic, so the same entry captured on two machines
 // dedupes by id. Reconciliation is the capture: scan, project, append what
-// is new, batched with a single fsync per pass.
+// is new, in bounded batches.
+//
+// Memory is bounded by one changed log's ids, never by the record or a log's
+// bytes (awebai/oats#456): a log is read in chunks, and what the stream
+// already holds is looked up for that log's ids alone, in one streamed read
+// of the journal, never as the whole stream's id set.
 
-import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { closeSync, existsSync, fstatSync, mkdirSync, openSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
 import { basename, dirname, join } from "node:path";
 import { homedir } from "node:os";
 
@@ -20,6 +25,7 @@ import {
   projectInteractionLogEntry,
 } from "./project-aweb.mjs";
 import { loadIgnore } from "./ignore.mjs";
+import { bufferLines, fileChunks } from "./file-lines.mjs";
 
 export function defaultCommLogDir(home = homedir()) {
   return join(home, ".config", "aw", "logs");
@@ -37,42 +43,78 @@ export function listCommLogs(dir = defaultCommLogDir()) {
     .map((name) => join(dir, name));
 }
 
-function readEntries(path) {
-  const entries = [];
-  let skipped = 0;
-  const text = readFileSync(path, "utf8");
-  const lines = text.split("\n");
-  for (let i = 0; i < lines.length; i++) {
-    const line = lines[i];
-    if (line.trim() === "") continue;
+// The entries of the first `end` bytes of a JSONL log, one at a time, read
+// in chunks from `fd` (a comm log can exceed V8's maximum string length).
+// A final line without its newline is still an entry when it parses; when it
+// does not, it is a torn tail the client is still writing, never counted. An
+// interior bad line is skipped and counted (`onSkipped`), never fatal.
+function* logEntries(fd, end, onSkipped) {
+  for (const line of bufferLines(fileChunks(fd, { end }))) {
+    const final = line[line.length - 1] !== 10;
+    let entry;
     try {
-      entries.push(JSON.parse(line));
+      // Inside the try: a line beyond V8's string limit is a bad line like any other.
+      const text = line.toString("utf8");
+      if (text.trim() === "") continue;
+      entry = JSON.parse(text);
     } catch {
-      // A torn final line is expected while the client is writing; an
-      // interior bad line is skipped and counted, never fatal to capture.
-      if (i < lines.length - 1) skipped++;
+      if (!final) onSkipped();
+      continue;
     }
+    yield entry;
   }
-  return { entries, skipped };
 }
 
-function projectFile(entries, project) {
-  const turns = [];
-  let failed = 0;
-  for (const entry of entries) {
-    try {
-      turns.push(project(entry));
-    } catch {
-      // One unprojectable entry must not abort the pass, but it fails
-      // visibly: counted here and reported by the caller.
-      failed++;
+const APPEND_BATCH = 2000;
+
+// One log file into `streamId`: project every entry, append the turns the
+// stream lacks. With `knownIds` (a caller's own set, kept up to date), that
+// set decides; without, the log is read twice over the same prefix (its size
+// when opened: the client may be appending): once for its turn ids, which one
+// streamed read of the journal narrows to those already there, and once to
+// append the rest in batches.
+function captureLogFile(store, { streamId, path, project, knownIds }) {
+  const fd = openSync(path, "r");
+  try {
+    const end = fstatSync(fd).size;
+    let entries = 0, skipped = 0, failed = 0;
+    function* turns() {
+      entries = 0; skipped = 0; failed = 0;
+      for (const entry of logEntries(fd, end, () => skipped++)) {
+        entries++;
+        let turn;
+        try { turn = project(entry); }
+        catch {
+          // One unprojectable entry must not abort the pass, but it fails
+          // visibly: counted here and reported by the caller.
+          failed++;
+          continue;
+        }
+        yield turn;
+      }
     }
-  }
-  return { turns, failed };
+    let known = knownIds;
+    if (!known) {
+      const candidates = new Set();
+      for (const turn of turns()) candidates.add(turn.id);
+      known = store.idsAmong(streamId, candidates);
+    }
+    let appended = 0, batch = [];
+    const flush = () => { store.appendBatch(streamId, batch); appended += batch.length; batch = []; };
+    for (const turn of turns()) {
+      if (known.has(turn.id)) continue;
+      known.add(turn.id);
+      batch.push(turn);
+      if (batch.length >= APPEND_BATCH) flush();
+    }
+    if (batch.length) flush();
+    return { entries, appended, skipped, failed };
+  } finally { closeSync(fd); }
 }
 
 // Capture one comm-log file into `<owner>~aw`. The account name is the
-// filename stem. `knownIds` carries the stream's ids across files in a pass.
+// filename stem. `knownIds`, when given, is the caller's own id set, kept up to
+// date, and decides instead of a read of the journal.
 //
 // Deliberately does NOT consult the ignore list: this is an explicit
 // "capture this file" command, and the caller has named the file. The
@@ -81,26 +123,8 @@ function projectFile(entries, project) {
 export function captureCommLog(store, { owner, path, knownIds = null }) {
   const account = basename(path, ".jsonl");
   const streamId = awStream(owner);
-  const ids = knownIds ?? new Set(store.readStream(streamId).map((t) => t.id));
-  const { entries, skipped } = readEntries(path);
-  const { turns, failed } = projectFile(entries, (e) =>
-    projectCommLogEntry(e, { selfName: account }),
-  );
-  const fresh = [];
-  for (const turn of turns) {
-    if (ids.has(turn.id)) continue;
-    ids.add(turn.id);
-    fresh.push(turn);
-  }
-  store.appendBatch(streamId, fresh);
-  return {
-    account,
-    stream: streamId,
-    entries: entries.length,
-    appended: fresh.length,
-    skipped,
-    failed,
-  };
+  const r = captureLogFile(store, { streamId, path, knownIds, project: (e) => projectCommLogEntry(e, { selfName: account }) });
+  return { account, stream: streamId, ...r };
 }
 
 // Capture one workspace interaction log into `<owner>~aw`. Like
@@ -108,19 +132,8 @@ export function captureCommLog(store, { owner, path, knownIds = null }) {
 // explicit per-file command; pass-level entry points enforce the policy.
 export function captureInteractionLog(store, { owner, path, selfName, workspace, knownIds = null }) {
   const streamId = awStream(owner);
-  const ids = knownIds ?? new Set(store.readStream(streamId).map((t) => t.id));
-  const { entries, skipped } = readEntries(path);
-  const { turns, failed } = projectFile(entries, (e) =>
-    projectInteractionLogEntry(e, { selfName, workspace }),
-  );
-  const fresh = [];
-  for (const turn of turns) {
-    if (ids.has(turn.id)) continue;
-    ids.add(turn.id);
-    fresh.push(turn);
-  }
-  store.appendBatch(streamId, fresh);
-  return { stream: streamId, entries: entries.length, appended: fresh.length, skipped, failed };
+  const r = captureLogFile(store, { streamId, path, knownIds, project: (e) => projectInteractionLogEntry(e, { selfName, workspace }) });
+  return { stream: streamId, ...r };
 }
 
 // Derived seen-files cache (same pattern as capture-cc): skip a log file
@@ -143,17 +156,15 @@ function saveSeenCache(store, cache) {
   writeFileSync(path, JSON.stringify(cache));
 }
 
-// One reconciliation pass over every default comm log. The stream's known
-// ids are read once and shared across files; unchanged files are skipped.
+// One reconciliation pass over every default comm log. Unchanged files are
+// skipped; each changed one looks up its own ids in the stream.
 //
 // Files matching the record's ignore list (`<root>/ignore`, see ignore.mjs)
 // are skipped before being opened — no turns, no seen-cache entry — and
 // reported as `{ account, path, ignored: true }` so a pass stays visible
 // about what it refused to read.
 export function captureAwLogs(store, { owner, commLogDir, ignore = null } = {}) {
-  const streamId = awStream(owner);
   const ign = ignore ?? loadIgnore(store.root);
-  let knownIds = null; // lazy: only read the journal if some file changed
   const seen = loadSeenCache(store);
   const results = [];
   for (const path of listCommLogs(commLogDir ?? defaultCommLogDir())) {
@@ -170,8 +181,7 @@ export function captureAwLogs(store, { owner, commLogDir, ignore = null } = {})
     }
     const prev = seen[path];
     if (prev && prev.size === stat.size && prev.mtimeMs === stat.mtimeMs) continue;
-    if (!knownIds) knownIds = new Set(store.readStream(streamId).map((t) => t.id));
-    results.push(captureCommLog(store, { owner, path, knownIds }));
+    results.push(captureCommLog(store, { owner, path }));
     seen[path] = { size: stat.size, mtimeMs: stat.mtimeMs };
   }
   saveSeenCache(store, seen);

package/packages/record/lib/capture-cc.mjs

@@ -29,6 +29,7 @@ import { jsonlLines, SESSION_FORMATS } from "./formats.mjs";
 import { loadIgnore } from "./ignore.mjs";
 import { assertIdentity, assertProtectedDescriptor, digest, identity, readRange, verifySnapshot } from "./session-snapshot.mjs";
 import { guardCapturedPath } from "./native-history.mjs";
+import { fileChunks } from "./file-lines.mjs";
 import { isDeepStrictEqual } from "node:util";
 
 export const SESSION_STREAM_SOURCE = "cc";
@@ -118,16 +119,16 @@ function readFrom(path, start, size) {
   }
 }
 
-// The journal's last captured source line, read from its tail without
-// parsing the whole file (backward scan, doubling window). 0 when the
-// journal is missing or empty.
-function lastJournalLine(store, streamId) {
+// The journal's last whole turn, read from its tail without parsing the whole
+// file (backward scan, doubling window). null when the journal is missing or
+// holds none.
+function lastJournalTurn(store, streamId) {
   const path = store.journalPath(streamId);
   let size;
   try {
     size = statSync(path).size;
   } catch (err) {
-    if (err.code === "ENOENT") return 0;
+    if (err.code === "ENOENT") return null;
     throw err;
   }
   let window = 64 * 1024;
@@ -145,12 +146,12 @@ function lastJournalLine(store, streamId) {
       // window covers the whole file).
       if (i === 0 && start > 0) break;
       try {
-        return JSON.parse(candidates[i]).provenance?.origin?.line ?? 0;
+        return JSON.parse(candidates[i]);
       } catch {
         continue; // fragment or torn line: look further back
       }
     }
-    if (start === 0) return 0;
+    if (start === 0) return null;
     window *= 2;
   }
 }
@@ -160,24 +161,83 @@ function lastJournalLine(store, streamId) {
 // Honest limit: an in-place REWRITE of already-captured lines is not
 // detected (only growth is; a shrink triggers a rescan via the size
 // check in the caller). Transcript writers are append-only in practice.
-function offsetFromJournal(store, streamId, sourcePath, final, sourceBytes) {
-  const turns = store.readStream(streamId);
-  if (turns.length === 0) return { bytes: 0, line: 0, lastTs: "" };
-  const last = turns[turns.length - 1];
+// Both are read in bounded memory: the journal's last turn from its tail, the
+// source's lines counted chunk by chunk (awebai/oats#456).
+// `last` is the journal's last turn (lastJournalTurn), null when it has none.
+function offsetFromJournal(last, sourcePath, final, sourceBytes) {
+  if (!last) return { bytes: 0, line: 0, lastTs: "" };
   const lastLine = last.provenance?.origin?.line ?? 0;
-  const bytes = sourceBytes ?? readFileSync(sourcePath);
-  let line = 0;
-  let offset = 0;
-  while (line < lastLine && offset < bytes.length) {
-    const nl = bytes.indexOf(10, offset);
-    if (nl === -1) break;
-    line++;
-    offset = nl + 1;
-  }
+  const { line, offset } = sourceBytes ? lineOffset([sourceBytes], lastLine) : lineOffsetOfFile(sourcePath, lastLine);
   if (final && line < lastLine) throw new Error(`session source is shorter than its captured journal: ${sourcePath}`);
   return { bytes: offset, line, lastTs: last.ts ?? "" };
 }
 
+// The byte offset just past the `lastLine`-th complete line of the bytes in
+// `chunks` (in order), and how many complete lines it found (fewer when the
+// source is shorter).
+function lineOffset(chunks, lastLine) {
+  let line = 0, offset = 0;
+  for (const chunk of chunks) {
+    let from = 0;
+    while (line < lastLine) {
+      const nl = chunk.indexOf(10, from);
+      if (nl === -1) break;
+      line++;
+      from = nl + 1;
+    }
+    offset += line < lastLine ? chunk.length : from;
+    if (line >= lastLine) break;
+  }
+  return { line, offset };
+}
+
+function lineOffsetOfFile(path, lastLine) {
+  const fd = openSync(path, "r");
+  try { return lineOffset(fileChunks(fd), lastLine); }
+  finally { closeSync(fd); }
+}
+
+// Source bytes of turns held before they are appended: what a pass keeps of a
+// transcript at once, whatever its size.
+const FLUSH_BYTES = 8 * 1024 * 1024;
+
+// The COMPLETE lines of `chunk`, one at a time, with their stamps:
+// { text, ts, bytes }. `walk` records where the walk ended (`scanned`, bytes)
+// and why it stopped early (`reason`): a torn tail, invalid UTF-8 (decoding
+// replacement characters would change both the verbatim line and its byte
+// offset, possibly treating a later fragment as a record) or a line beyond
+// V8's string limit.
+function* completeLines(chunk, walk) {
+  let scanned = 0;
+  walk.scanned = 0;
+  while (scanned < chunk.length) {
+    const nl = chunk.indexOf(10, scanned);
+    if (nl === -1) { walk.reason = "torn-tail"; return; }
+    const bytes = chunk.subarray(scanned, nl);
+    if (!isUtf8(bytes)) { walk.reason = "invalid-utf8"; return; }
+    let text;
+    try { text = bytes.toString("utf8"); }
+    catch (err) {
+      if (err.code !== "ERR_STRING_TOO_LONG") throw err;
+      walk.reason = "oversized-line";
+      return;
+    }
+    const lineBytes = nl - scanned + 1;
+    let ts = "";
+    if (text.trim() !== "") {
+      try {
+        const d = JSON.parse(text);
+        if (typeof d?.timestamp === "string") ts = d.timestamp;
+      } catch {
+        /* unparseable native line: captured verbatim */
+      }
+    }
+    scanned += lineBytes;
+    walk.scanned = scanned;
+    yield { text, ts, bytes: lineBytes };
+  }
+}
+
 // One reconciliation pass for one format: one turn per NEW complete line
 // of every session file under `roots`. Unstamped leading lines are held
 // until the file shows its first timestamp (then they carry it forward),
@@ -253,10 +313,12 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
     // journal is the truth; before appending anything, any disagreement
     // rebuilds the offset from it. Background passes check on growth;
     // final passes also verify unchanged files before confirming capture.
-    if (state && (final || stat.size > state.bytes) && state.line !== lastJournalLine(store, streamId)) {
+    let journalLast; // the journal's last turn, read from its tail once, when needed
+    const lastTurn = () => (journalLast === undefined ? (journalLast = lastJournalTurn(store, streamId)) : journalLast);
+    if (state && (final || stat.size > state.bytes) && state.line !== (lastTurn()?.provenance?.origin?.line ?? 0)) {
       state = null;
     }
-    if (!state) state = offsetFromJournal(store, streamId, path, final, sourceBytes);
+    if (!state) state = offsetFromJournal(lastTurn(), path, final, sourceBytes);
     if (stat.size <= state.bytes) {
       unchanged++;
       offsets[offKey] = state;
@@ -265,62 +327,38 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
     }
 
     const chunk = sourceBytes ? sourceBytes.subarray(state.bytes) : readRange(fd, state.bytes, stat.size, path, capturedPi);
-    // Phase 1: collect the COMPLETE lines of the chunk with their stamps.
-    const lines = [];
-    let scanned = 0;
-    let reason;
-    while (scanned < chunk.length) {
-      const nl = chunk.indexOf(10, scanned);
-      if (nl === -1) { reason = "torn-tail"; break; }
-      const bytes = chunk.subarray(scanned, nl);
-      // Decoding replacement characters would change both the verbatim line
-      // and its byte offset, possibly treating a later fragment as a record.
-      if (!isUtf8(bytes)) { reason = "invalid-utf8"; break; }
-      let text;
-      try { text = bytes.toString("utf8"); }
-      catch (err) {
-        if (err.code !== "ERR_STRING_TOO_LONG") throw err;
-        reason = "oversized-line";
-        break;
-      }
-      const lineBytes = nl - scanned + 1;
-      let ts = "";
-      if (text.trim() !== "") {
-        try {
-          const d = JSON.parse(text);
-          if (typeof d?.timestamp === "string") ts = d.timestamp;
-        } catch {
-          /* unparseable native line: captured verbatim below */
-        }
-      }
-      lines.push({ text, ts, bytes: lineBytes });
-      scanned += lineBytes;
-    }
-    if (reason) {
+    // Every turn needs a stamp. Leading lines before the file's first stamp
+    // carry it backward (deterministic: the file's first stamp is invariant
+    // however capture is scheduled); if the file has shown no stamp at all
+    // yet, hold everything for a later pass. The lines are walked, never
+    // collected: a first capture of a huge transcript is one file's worth of
+    // NEW lines, far more than the heap (awebai/oats#456).
+    const stopped = (walk) => {
+      if (!walk.reason) return;
       incomplete++;
-      issues.push({ source: fmt.source, path, reason, offset: state.bytes + scanned });
-    }
-    // Phase 2: every turn needs a stamp. Leading lines before the file's
-    // first stamp carry it backward (deterministic: the file's first
-    // stamp is invariant however capture is scheduled); if the file has
-    // shown no stamp at all yet, hold everything for a later pass.
+      issues.push({ source: fmt.source, path, reason: walk.reason, offset: state.bytes + walk.scanned });
+    };
     let lastTs = state.lastTs ?? "";
     if (!lastTs) {
-      const first = lines.find((l) => l.ts);
-      if (!first) {
-        if (lines.length > 0) {
+      const scan = {};
+      let any = false;
+      for (const l of completeLines(chunk, scan)) {
+        any = true;
+        if (l.ts) { lastTs = l.ts; break; }
+      }
+      if (!lastTs) {
+        stopped(scan);
+        if (any) {
           held++;
           issues.push({ source: fmt.source, path, reason: "unstamped", offset: state.bytes });
         }
         verifySource();
         continue; // do not advance; retry when a stamp exists
       }
-      lastTs = first.ts;
     }
     verifySource(); // parsing/recovery may take time; still no journal write yet
     // Turns flush to the journal in bounded batches, so memory stays flat
-    // however large the backlog (a first capture of a huge transcript is
-    // one file's worth of NEW lines). A crash between flushes cannot
+    // however large the backlog. A crash between flushes cannot
     // duplicate: this file's offset is saved only after its final flush,
     // and a lost offset rebuilds from the journal's own last line number.
     let fresh = [];
@@ -336,7 +374,8 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
     };
     let lineNo = state.line;
     let consumed = 0;
-    for (const l of lines) {
+    const walk = {};
+    for (const l of completeLines(chunk, walk)) {
       if (l.ts) lastTs = l.ts;
       lineNo++;
       consumed += l.bytes;
@@ -348,8 +387,9 @@ export function captureSessions(store, { owner, roots, files, format = "cc", ign
         ),
       );
       freshBytes += l.bytes;
-      if (freshBytes >= 64 * 1024 * 1024) flush();
+      if (freshBytes >= FLUSH_BYTES) flush();
     }
+    stopped(walk);
     const grew = fresh.length > 0 || consumed > 0;
     flush();
     offsets[offKey] = { bytes: state.bytes + consumed, line: lineNo, lastTs };

package/packages/record/lib/file-lines.mjs

@@ -0,0 +1,38 @@
+// Reading a file in bounded memory: its bytes in chunks, and its lines one at
+// a time. Journals, comm logs and transcripts can be far larger than the heap
+// (awebai/oats#456), so no reader here ever holds more than one chunk and one
+// line.
+
+import { readSync } from "node:fs";
+
+// The bytes of `fd` from `start` up to `end` (default: the end of the file), in
+// chunks of at most `size` bytes. Each chunk is a fresh buffer, so a caller may
+// keep one past the next read.
+export function* fileChunks(fd, { start = 0, end = Infinity, size = 1 << 20 } = {}) {
+  for (let position = start; position < end; ) {
+    const chunk = Buffer.allocUnsafe(Math.min(size, end - position));
+    const n = readSync(fd, chunk, 0, chunk.length, position);
+    if (n === 0) return;
+    position += n;
+    yield chunk.subarray(0, n);
+  }
+}
+
+// The lines of `chunks`, one at a time: each complete line WITH its newline,
+// then a final fragment without one, if the bytes end mid-line. A caller tells
+// them apart by the last byte. Fragments are joined only at a newline, so a
+// long line is copied once, not as each chunk arrives.
+export function* bufferLines(chunks) {
+  let pieces = [], size = 0;
+  for (const chunk of chunks) {
+    let from = 0;
+    for (let nl = chunk.indexOf(10, from); nl >= 0; nl = chunk.indexOf(10, from)) {
+      pieces.push(chunk.subarray(from, nl + 1));
+      size += nl + 1 - from;
+      yield pieces.length === 1 ? pieces[0] : Buffer.concat(pieces, size);
+      pieces = []; size = 0; from = nl + 1;
+    }
+    if (from < chunk.length) { pieces.push(chunk.subarray(from)); size += chunk.length - from; }
+  }
+  if (size > 0) yield pieces.length === 1 ? pieces[0] : Buffer.concat(pieces, size);
+}

package/packages/record/lib/store.mjs

@@ -13,6 +13,7 @@
 //   - the effective record is the id-deduplicated union across streams,
 //     minus turns hidden by valid tombstones.
 
+import { bufferLines, fileChunks } from "./file-lines.mjs";
 import {
   appendFileSync,
   closeSync,
@@ -413,34 +414,33 @@ export class RecordStore {
     let fd;
     try { fd = openSync(this.journalPath(streamId), "r"); }
     catch (e) { if (e.code === "ENOENT") return; throw e; }
-    let pieces = [], size = 0, lineStart = 0;
+    let lineStart = 0;
     try {
-      for (;;) {
-        const chunk = Buffer.allocUnsafe(65536);
-        const n = readSync(fd, chunk, 0, chunk.length, null);
-        if (n === 0) break;
-        let from = 0;
-        for (let nl = chunk.indexOf(10, from); nl >= 0 && nl < n; nl = chunk.indexOf(10, from)) {
-          pieces.push(chunk.subarray(from, nl + 1));
-          size += nl + 1 - from;
-          const line = pieces.length === 1 ? pieces[0] : Buffer.concat(pieces, size);
-          let parsed;
-          try { parsed = parseJournal(line); }
-          catch (e) {
-            if (e instanceof StoreError) throw new StoreError(`corrupt interior journal line at byte ${lineStart}`);
-            throw e;
-          }
-          lineStart += size;
-          pieces = []; size = 0; from = nl + 1;
-          yield* parsed.turns;
+      for (const line of bufferLines(fileChunks(fd, { size: 65536 }))) {
+        // A final fragment without a newline is a torn tail, even if its
+        // JSON is valid. Readers leave it for the owner's append repair.
+        if (line[line.length - 1] !== 10) return;
+        let parsed;
+        try { parsed = parseJournal(line); }
+        catch (e) {
+          if (e instanceof StoreError) throw new StoreError(`corrupt interior journal line at byte ${lineStart}`);
+          throw e;
         }
-        if (from < n) { pieces.push(chunk.subarray(from, n)); size += n - from; }
+        lineStart += line.length;
+        yield* parsed.turns;
       }
-      // A final fragment without a newline is a torn tail, even if its
-      // JSON is valid. Readers leave it for the owner's append repair.
     } finally { closeSync(fd); }
   }
 
+  // The ids among `candidates` that the stream holds, in one streamed read of
+  // its journal: a dedupe for a few ids never builds the whole stream's set.
+  idsAmong(streamId, candidates) {
+    const found = new Set();
+    if (candidates.size === 0) return found;
+    for (const turn of this.iterateStream(streamId)) if (candidates.has(turn.id)) found.add(turn.id);
+    return found;
+  }
+
   // Is this a session-content stream (`<owner>~<source>.<session-id>`)?
   // Their journals hold whole conversations and can be large, so bulk
   // reads exclude them unless asked; access them per-thread instead.
@@ -548,12 +548,14 @@ export class RecordStore {
     }
     const path = this.journalPath(streamId);
     this.withStreamLock(streamId, () => {
-      // First append to this stream in this instance: parse the whole
+      // First append to this stream in this instance: read the whole
       // journal, so interior corruption throws here instead of silently
       // collecting appends behind the damage. A torn tail is tolerated
-      // (parseJournal treats it as final-line-torn) and repaired below.
+      // (iterateStream leaves it, as parseJournal treats it as
+      // final-line-torn) and repaired below. Streamed, one line at a time:
+      // a journal can be far larger than the heap (awebai/oats#456).
       if (!this.validatedStreams.has(streamId) && existsSync(path)) {
-        parseJournal(readFileSync(path));
+        for (const _turn of this.iterateStream(streamId)) { /* validated as read */ }
       }
       this.validatedStreams.add(streamId);
       repairTail(path);