omnirush 0.3.3 → 0.4.2

package/assets/extensions/omnirush/collector-lib.ts

@@ -22,7 +22,9 @@
 //      node_modules, build outputs, .git (full reproducibility) — with
 //      ONLY the secrets denylist (.env*, keys, credentials, private
 //      keys) excluded. Binaries are captured (base64), symlinks are
-//      stored as records (path + target) so replay recreates them, and
+//      stored as {path, content: <target>, encoding: "symlink"} so the
+//      backend's text-content validation accepts them and replay
+//      recreates them, and
 //      the POSIX mode/executable bit rides each file entry for replay
 //      (stored-and-ignored on Windows). Redaction also scrubs
 //      credentials embedded in URLs (e.g. .git/config remotes).
@@ -50,6 +52,8 @@ import { dirname, join, relative, resolve, sep } from "node:path";
 import { promisify } from "node:util";
 import { zstdCompress as zstdCompressCb } from "node:zlib";
 
+import { isRetryableStatus, retryAttempts, withRetries } from "./retry";
+
 const execFileAsync = promisify(execFile);
 
 // --- limits ---------------------------------------------------------------
@@ -65,10 +69,31 @@ export const MAX_SESSION_BYTES = 100 * 1024 * 1024 * 1024;
 // byte size + sha256) so replay reassembles the original deterministically.
 export const MAX_SLICE_BYTES = 4 * 1024 * 1024;
 // Whole-read ceiling. Files up to this size are read and processed as
-// one string (worst case: binary -> base64 inflates 4/3 to ~427 MiB
-// chars, comfortably under the runtime's ~512 MiB MAX_STRING_LENGTH).
-// Bigger files are streamed in fixed raw chunks and sliced.
-const WHOLE_READ_BYTES = 320 * 1024 * 1024;
+// one string; bigger files stream in fixed raw chunks and slice.
+// Deliberately tiny (issue #6): a whole read holds buffer + utf8 string
+// + redaction copy + base64 simultaneously (~4-5x file size), so even a
+// modest ceiling spikes RSS by that multiple per file. 8 MiB bounds the
+// worst case to ~40 MiB; anything bigger takes the streaming path with
+// identical output.
+const WHOLE_READ_BYTES = 8 * 1024 * 1024;
+// Text-segment cap for newline-less files (see the streaming reader).
+const PENDING_TEXT_CAP_BYTES = 32 * 1024 * 1024;
+// Change-journal memory ceiling (issue #6, Windows 8-9 GiB OOM): the
+// journal holds every changed file's content between debounced change
+// uploads. At HIGH_WATER an upload is forced immediately (normal
+// backpressure); at the HARD ceiling — a backend down for a long time —
+// the OLDEST records are dropped, loudly (warn + journal.dropped trace),
+// because losing replay data beats crashing the whole session.
+// Soft disk cap for the on-disk change journal: when unacked journal
+// bytes exceed it (a backend down for a very long time), capture stops
+// loudly instead of filling the disk. The end snapshot still ships.
+export const JOURNAL_DISK_CAP_BYTES = 1024 * 1024 * 1024;
+/**
+ * Non-enumerable per-record field carrying the raw serialized line
+ * length (for journal ack accounting) — invisible to JSON.stringify so
+ * the changes.json sidecar never includes it.
+ */
+export const RAW_BYTES_FIELD = Symbol("omnirushJournalRawBytes");
 // Multi-file parts target this compressed size (the backend body rail
 // was live-verified at >= 31.5 MiB compressed, so 15 MiB leaves ample
 // headroom for any ingress in front of the manager).
@@ -81,8 +106,10 @@ export const PART_UNCOMPRESSED_STEP = 8 * 1024 * 1024;
 // Flush a part when a check shows at least this much compressed payload.
 export const PART_READY_COMPRESSED_BYTES = 12 * 1024 * 1024;
 // Absolute uncompressed ceiling for one part batch — bounds resident
-// memory even for highly-compressible content.
-export const PART_BATCH_UNCOMPRESSED_MAX = 48 * 1024 * 1024;
+// memory even for highly-compressible content (incompressible batches
+// never hit the compressed-size thresholds, so this is what actually
+// stops accumulation; the payload build doubles it transiently).
+export const PART_BATCH_UNCOMPRESSED_MAX = 24 * 1024 * 1024;
 // Change/trace parts flush by PLAIN size so their __agent__/changes.json
 // / trace.json entries (single file entries whose content grows with the
 // batch) stay small and parts upload promptly.
@@ -105,13 +132,13 @@ type SnapshotType = "start" | "change" | "trace" | "end";
 
 export type CollectorFile = {
   path: string;
-  /** utf8 text (default) or base64 when encoding is "base64". Absent for
-   *  symlink records, which carry `target` instead. */
-  content?: string;
-  /** Absent = utf8 text. "base64" for binary content. */
-  encoding?: "base64";
-  /** Symlink record: the link target as stored on disk. */
-  target?: string;
+  /** utf8 text by default. For binaries: base64. For symlinks: the link
+   *  target verbatim (the backend requires text content on every file
+   *  entry, so symlink records are text entries with a marker). */
+  content: string;
+  /** Absent = utf8 text. "base64" for binary content, "symlink" for
+   *  symlink records (content = link target). */
+  encoding?: "base64" | "symlink";
   /** POSIX mode bits (octal string, e.g. "755"). Absent for symlinks. */
   mode?: string;
 };
@@ -127,8 +154,7 @@ export type ChangeJournalEntry = {
   path: string;
   status: "present" | "deleted";
   content?: string;
-  encoding?: "base64";
-  target?: string;
+  encoding?: "base64" | "symlink";
   mode?: string;
   /** Present when the record carries one slice of a bigger file. */
   slice?: { index: number; total: number; bytes: number; sha256: string };
@@ -175,8 +201,28 @@ type SessionState = {
    *  fabricate deletions inside the workspace. */
   knownPaths: Set<string>;
   trace: TraceEvent[];
-  /** Ordered change records — the replayable change log. */
-  changeJournal: ChangeJournalEntry[];
+  /**
+   * The change journal lives ON DISK (ndjson sidecar, one JSON record
+   * per line) — never as an in-memory array (issue #6: the array held
+   * every changed file's content string and grew to multiple GB).
+   * Captures APPEND to the file; change uploads stream the unacked
+   * region and compact the acked prefix away. Appends during an upload
+   * simply land past the read window (bounded by `journalUploadEof`) —
+   * no gating, no memory growth.
+   */
+  journalPath: string | null;
+  /** File offset of the first not-yet-uploaded record. */
+  journalAckOffset: number;
+  /** Bytes appended so far (soft disk cap accounting). */
+  journalAppendedBytes: number;
+  /** Serializes journal appends. */
+  journalAppendTail: Promise<void>;
+  /** Set while a change upload reads the journal. */
+  journalUploadEof: number | null;
+  /** True once the soft disk cap stopped captures (warned once). */
+  journalCaptureStopped: boolean;
+  /** Last journal signature per path (consecutive-duplicate suppression). */
+  journalLastByPath: Map<string, string>;
   changeCaptureTail: Promise<void>;
   ready: Promise<void>;
   tail: Promise<void>;
@@ -393,7 +439,7 @@ export function selectTraceEvents(
  * removed on purpose: `git ls-files` drops anything .gitignored (build
  * outputs, node_modules), which broke the whole-project guarantee.
  */
-async function listWorkspaceFiles(root: string): Promise<string[]> {
+export async function listWorkspaceFiles(root: string): Promise<string[]> {
   return walkWorkspace(root);
 }
 
@@ -527,19 +573,23 @@ function sliceManifest(
 type ContentChunk = { content: string; encoding?: "base64" };
 
 /**
- * Read a regular file into <= ~8 MiB content chunks: small files in one
- * whole read (fully redacted), big files streamed in fixed raw chunks —
- * strings stay far below the runtime string limit regardless of file
- * size. Binary detection happens on the first chunk and applies to the
- * whole file.
+ * Stream a regular file as <= ~8 MiB content chunks, yielding each chunk
+ * the moment it is produced — a file's content is NEVER materialized as
+ * one string regardless of size (issue #6: the old array version held
+ * every chunk at once). Small files are a single whole read; big files
+ * stream in fixed raw chunks. Binary detection happens on the first
+ * chunk and applies to the whole file; text is redacted per
+ * line-aligned segment.
  */
-async function readContentChunks(absolute: string, size: number): Promise<ContentChunk[]> {
+async function* iterateContentChunks(absolute: string, size: number): AsyncGenerator<ContentChunk> {
   if (size <= WHOLE_READ_BYTES) {
     const buffer = await readFile(absolute);
     if (isBinary(buffer)) {
-      return [{ content: buffer.toString("base64"), encoding: "base64" }];
+      yield { content: buffer.toString("base64"), encoding: "base64" };
+      return;
     }
-    return [{ content: redactCollectorText(buffer.toString("utf8")).text }];
+    yield { content: redactCollectorText(buffer.toString("utf8")).text };
+    return;
   }
   const { open } = await import("node:fs/promises");
   const handle = await open(absolute, "r");
@@ -548,112 +598,139 @@ async function readContentChunks(absolute: string, size: number): Promise<Conten
     const buffer = Buffer.alloc(chunkBytes);
     const first = await handle.read(buffer, 0, chunkBytes, null);
     const binary = isBinary(buffer.subarray(0, first.bytesRead));
-    const chunks: ContentChunk[] = [];
     if (binary) {
       // Binaries carry no redactable text: base64 each raw chunk.
-      chunks.push({ content: buffer.subarray(0, first.bytesRead).toString("base64"), encoding: "base64" });
+      yield { content: buffer.subarray(0, first.bytesRead).toString("base64"), encoding: "base64" };
       let read = 0;
       while ((read = (await handle.read(buffer, 0, chunkBytes, null)).bytesRead) > 0) {
-        chunks.push({ content: buffer.subarray(0, read).toString("base64"), encoding: "base64" });
+        yield { content: buffer.subarray(0, read).toString("base64"), encoding: "base64" };
       }
-    } else {
-      // Text: decode incrementally (StringDecoder absorbs multibyte
-      // sequences split across chunk boundaries), emit only up to the
-      // last complete line, carry the remainder, and redact each
-      // line-aligned segment.
-      const { StringDecoder } = await import("node:string_decoder");
-      const decoder = new StringDecoder("utf8");
-      let pending = decoder.write(buffer.subarray(0, first.bytesRead));
-      let read = 0;
-      while ((read = (await handle.read(buffer, 0, chunkBytes, null)).bytesRead) > 0) {
-        pending += decoder.write(buffer.subarray(0, read));
+      return;
+    }
+    // Text: decode incrementally (StringDecoder absorbs multibyte
+    // sequences split across chunk boundaries), emit only up to the
+    // last complete line, carry the remainder, and redact each
+    // line-aligned segment. A pathological newline-less file must not
+    // grow `pending` forever: at PENDING_TEXT_CAP_BYTES the segment is
+    // flushed mid-line (line-anchored redaction weakens at that rare
+    // cut; unbounded memory is worse).
+    const { StringDecoder } = await import("node:string_decoder");
+    const decoder = new StringDecoder("utf8");
+    let pending = decoder.write(buffer.subarray(0, first.bytesRead));
+    let read = 0;
+    while ((read = (await handle.read(buffer, 0, chunkBytes, null)).bytesRead) > 0) {
+      pending += decoder.write(buffer.subarray(0, read));
+      for (;;) {
         const lastNl = pending.lastIndexOf("\n");
         if (lastNl >= 0) {
-          chunks.push({ content: redactCollectorText(pending.slice(0, lastNl + 1)).text });
+          yield { content: redactCollectorText(pending.slice(0, lastNl + 1)).text };
           pending = pending.slice(lastNl + 1);
+          continue;
         }
+        if (pending.length >= PENDING_TEXT_CAP_BYTES) {
+          yield { content: redactCollectorText(pending).text };
+          pending = "";
+        }
+        break;
       }
-      pending += decoder.end();
-      if (pending) chunks.push({ content: redactCollectorText(pending).text });
     }
-    return chunks;
+    pending += decoder.end();
+    if (pending) yield { content: redactCollectorText(pending).text };
   } finally {
     await handle.close();
   }
 }
 
 /**
- * Read one workspace entry into CollectorFile entries. Regular files:
+ * Stream one workspace entry as CollectorFile entries. Regular files:
  * text is redacted, binaries base64-encoded ("encoding": "base64"), and
  * the POSIX mode rides along (octal string) so replay can restore the
  * executable bit (stored-and-ignored on Windows). Content bigger than
- * MAX_SLICE_BYTES is sliced (see buildFileEntries). Symlinks become
- * records { path, target } (target verbatim; replay recreates the link).
- * Returns [] when the entry vanished or is a special file.
+ * MAX_SLICE_BYTES is sliced on the fly — slices are yielded the moment
+ * they exist while only the tiny sha256 descriptors accumulate, then
+ * the manifest is yielded LAST (replay unions manifests by index, so
+ * position never matters). Symlinks become text entries
+ * { path, content: <target>, encoding: "symlink" } — the backend's
+ * per-entry validation requires text content, and replay recreates the
+ * actual link from the target. Yields nothing when the entry vanished
+ * or is a special file.
  */
-async function readWorkspaceEntries(root: string, path: string): Promise<CollectorFile[]> {
+async function* iterateWorkspaceEntries(root: string, path: string): AsyncGenerator<CollectorFile> {
+  let absolute = "";
   try {
-    const absolute = resolve(root, path);
-    if (portablePath(root, absolute).startsWith("../")) return [];
+    absolute = resolve(root, path);
+    if (portablePath(root, absolute).startsWith("../")) return;
     const file = await lstat(absolute);
     if (file.isSymbolicLink()) {
       const target = await readlink(absolute);
-      return [{ path, target }];
+      yield { path, content: target, encoding: "symlink" };
+      return;
     }
-    if (!file.isFile()) return [];
+    if (!file.isFile()) return;
     const mode = file.mode & 0o777;
-    const chunks: ContentChunk[] = await readContentChunks(absolute, file.size);
-    if (chunks.length === 1) {
-      const chunk = chunks[0];
-      return buildFileEntries({
-        path,
-        content: chunk.content,
-        ...(chunk.encoding ? { encoding: chunk.encoding } : {}),
-        ...(mode ? { mode: mode.toString(8) } : {}),
-      });
+    const modeField = mode ? { mode: mode.toString(8) } : {};
+    if (file.size <= WHOLE_READ_BYTES) {
+      // Small file: whole read, real path preserved; buildFileEntries
+      // slices (manifest + parts) when the encoded content exceeds
+      // MAX_SLICE_BYTES — same shape as always.
+      for await (const chunk of iterateContentChunks(absolute, file.size)) {
+        for (const entry of buildFileEntries({
+          path,
+          content: chunk.content,
+          ...(chunk.encoding ? { encoding: chunk.encoding } : {}),
+          ...modeField,
+        })) {
+          yield entry;
+        }
+      }
+      return;
     }
-    // Big file: streamed chunks -> one logical entry per chunk (already
-    // <= ~8 MiB for raw chunks; base64 inflates 4/3, so sub-slice those),
-    // plus the manifest for deterministic reassembly.
-    const encoding = chunks[0].encoding;
-    const entries: CollectorFile[] = [];
-    const sliceContents = chunks.flatMap((chunk) => sliceString(chunk.content));
-    entries.push({
+    // Big file (streamed): slices are yielded the moment they exist;
+    // only the tiny sha256 descriptors accumulate. The reassembly
+    // manifest is yielded LAST (replay unions manifests by index, so
+    // position never matters).
+    let index = 0;
+    const descriptors: Array<{ index: number; bytes: number; sha256: string }> = [];
+    let sawEncoding: "base64" | undefined;
+    for await (const chunk of iterateContentChunks(absolute, file.size)) {
+      sawEncoding ??= chunk.encoding;
+      for (const slice of sliceString(chunk.content)) {
+        descriptors.push({ index, bytes: Buffer.byteLength(slice), sha256: sha256Hex(slice) });
+        yield {
+          path: slicePartPath(path, index),
+          content: slice,
+          ...(sawEncoding ? { encoding: sawEncoding } : {}),
+        };
+        index += 1;
+      }
+    }
+    if (index === 0) return;
+    yield {
       path: sliceManifestPath(path),
       content: JSON.stringify(sliceManifest(
-        {
-          path,
-          ...(encoding ? { encoding } : {}),
-          ...(mode ? { mode: mode.toString(8) } : {}),
-        },
-        sliceContents.map((slice, index) => ({
-          index,
-          bytes: Buffer.byteLength(slice),
-          sha256: sha256Hex(slice),
-        })),
+        { path, ...(sawEncoding ? { encoding: sawEncoding } : {}), ...modeField },
+        descriptors,
       )),
-    });
-    sliceContents.forEach((slice, index) => {
-      entries.push({
-        path: slicePartPath(path, index),
-        content: slice,
-        ...(encoding ? { encoding } : {}),
-      });
-    });
-    return entries;
+    };
+    return;
   } catch {
     // Workspaces are live; races are expected and retried by the next snapshot.
-    return [];
+    return;
   }
 }
 
-export async function collectFiles(
+/**
+ * Streaming file collection: yields workspace files (metadata entry
+ * first) one at a time so the one-shot path never holds the whole tree's
+ * content in memory (issue #6). Same order, denylist, and byte budget
+ * as the compat collectFiles drain below.
+ */
+export async function* iterateCollectFiles(
   root: string,
   byteLimit: number,
   workspaceId: string,
   session?: Pick<SessionState, "id" | "segment" | "resumed">,
-): Promise<CollectorFile[]> {
-  const files: CollectorFile[] = [];
+): AsyncGenerator<CollectorFile> {
   let used = 0;
   const metadata = JSON.stringify({
     workspace_id: workspaceId,
@@ -665,19 +742,32 @@ export async function collectFiles(
     root_name: root.split(sep).filter(Boolean).at(-1) ?? "workspace",
     git: await gitMetadata(root),
   });
-  files.push({ path: `${AGENT_DIR}/workspace.json`, content: metadata });
+  yield { path: `${AGENT_DIR}/workspace.json`, content: metadata };
   used += Buffer.byteLength(metadata);
 
   const paths = (await listWorkspaceFiles(root)).sort(comparePaths);
   for (const path of paths) {
     if (used >= byteLimit) break;
-    for (const file of await readWorkspaceEntries(root, path)) {
-      const size = Buffer.byteLength(file.content ?? file.target ?? "");
+    for await (const file of iterateWorkspaceEntries(root, path)) {
+      const size = Buffer.byteLength(file.content ?? "");
       if (used + size > byteLimit) continue;
-      files.push(file);
+      yield file;
       used += size;
     }
   }
+}
+
+/** Compat drain of iterateCollectFiles (tests + tooling). */
+export async function collectFiles(
+  root: string,
+  byteLimit: number,
+  workspaceId: string,
+  session?: Pick<SessionState, "id" | "segment" | "resumed">,
+): Promise<CollectorFile[]> {
+  const files: CollectorFile[] = [];
+  for await (const file of iterateCollectFiles(root, byteLimit, workspaceId, session)) {
+    files.push(file);
+  }
   return files;
 }
 
@@ -721,14 +811,11 @@ export type EnvelopePart = {
 };
 
 /**
- * Deterministic part split for a full files array (used by `omnirush
- * collect` and the tests): sort files by path, accumulate in order, keep
- * each part <= maxCompressed compressed. A single file bigger than one
- * part gets its own part; file content is never split across parts. A
- * lone file that cannot fit even the hard rail is skipped with a warning.
- * Envelope sequence numbers are state.sequence + 1 + partIndex.
+ * Deterministic part split for a full files array (compat drain of
+ * iterateEnvelopeParts, used by the tests): sort files by path,
+ * accumulate in order, keep each part <= maxCompressed compressed.
  */
-export function buildEnvelopeParts(
+export async function buildEnvelopeParts(
   state: Pick<SessionState, "id" | "segment" | "resumed"> & { sequence?: number },
   snapshotType: SnapshotType,
   files: CollectorFile[],
@@ -741,28 +828,34 @@ export function buildEnvelopeParts(
   const sorted = files
     .flatMap((file) => buildFileEntries(file))
     .sort((left, right) => comparePaths(left.path, right.path));
-  return packEnvelopeParts(state, snapshotType, sorted, maxCompressed, log);
+  const parts: EnvelopePart[] = [];
+  for await (const part of iterateEnvelopeParts(state, snapshotType, sorted, maxCompressed, log)) {
+    parts.push(part);
+  }
+  return parts;
 }
 
 /**
- * Shared greedy packer: split an ordered file list into compressed parts,
- * preserving path order across parts. Used by buildEnvelopeParts (pure,
- * whole list in memory) and mirrored by the collector's streaming packer
- * (packAndUpload, which never holds more than one part in memory).
+ * Streaming packer for the one-shot path: yields compressed parts in
+ * order so callers can upload-and-discard instead of holding the whole
+ * workspace's parts (payload + compressed buffers) in memory (issue #6).
+ * Same greedy algorithm as before: split at the compressed target, a
+ * lone file may use the hard rail, lone-over-rail files are skipped
+ * loudly. File content is never split across parts.
  */
-export async function packEnvelopeParts(
+export async function* iterateEnvelopeParts(
   state: Pick<SessionState, "id" | "segment" | "resumed"> & { sequence?: number },
   snapshotType: SnapshotType,
-  sortedFiles: CollectorFile[],
+  sortedFiles: Iterable<CollectorFile> | AsyncIterable<CollectorFile>,
   maxCompressed = MAX_PART_COMPRESSED_BYTES,
   log?: (message: string, attributes?: Record<string, unknown>) => void,
-): Promise<EnvelopePart[]> {
+): AsyncGenerator<EnvelopePart> {
   const hard = Math.max(maxCompressed, MAX_PART_COMPRESSED_HARD);
   const measure = async (batch: CollectorFile[], partIndex: number): Promise<EnvelopePart> => {
     const payload = buildEnvelopePayload({ ...state, sequence: (state.sequence ?? 0) + partIndex }, snapshotType, batch);
     return { files: batch, payload, compressed: await compressZstd(payload) };
   };
-  const parts: EnvelopePart[] = [];
+  let partsYielded = 0;
   let batch: CollectorFile[] = [];
   const fit = async (batchToFit: CollectorFile[], partIndex: number) => {
     let measured = await measure(batchToFit, partIndex);
@@ -778,21 +871,22 @@ export async function packEnvelopeParts(
       const lone = batchToFit.pop()!;
       log?.("OmniRush file exceeds the upload limits and cannot be shipped as a single part; skipped", {
         path: lone.path,
-        bytes: Buffer.byteLength(lone.content ?? lone.target ?? ""),
+        bytes: Buffer.byteLength(lone.content ?? ""),
       });
       return { measured: null, carry };
     }
     return { measured, carry };
   };
-  for (const file of sortedFiles) {
+  for await (const file of sortedFiles) {
     batch.push(file);
-    const partIndex = parts.length;
+    const partIndex = partsYielded;
     const estimate = batch.reduce((total, f) => total + Buffer.byteLength(f.content), 0);
     if (estimate < PART_UNCOMPRESSED_STEP) continue;
     const { measured, carry } = await fit(batch, partIndex);
     if (measured
       && (measured.compressed.length >= PART_READY_COMPRESSED_BYTES || estimate >= PART_BATCH_UNCOMPRESSED_MAX)) {
-      parts.push(measured);
+      partsYielded += 1;
+      yield measured;
       batch = carry;
     } else if (measured) {
       batch = [...batch, ...carry];
@@ -801,15 +895,37 @@ export async function packEnvelopeParts(
     }
   }
   while (batch.length > 0) {
-    const { measured, carry } = await fit(batch, parts.length);
-    if (measured) parts.push(measured);
+    const { measured, carry } = await fit(batch, partsYielded);
+    if (measured) {
+      partsYielded += 1;
+      yield measured;
+    }
     if (carry.length === 0) break;
     batch = carry;
   }
-  return parts;
 }
 
 export class WorkspaceCollector {
+  /**
+   * Bounded first-chunk read of an error response body for diagnostics:
+   * at most 512 bytes, decoded and whitespace-collapsed. Never throws.
+   * Error bodies are small JSON from the manager; the bound exists so a
+   * pathological response cannot balloon a log line (or memory).
+   */
+  private static async errorBodySnippet(response: any): Promise<string> {
+    try {
+      const reader = response?.body?.getReader?.();
+      if (!reader) return "";
+      const { value } = await reader.read();
+      reader.cancel().catch(() => undefined);
+      if (!value) return "";
+      const text = Buffer.from(value).toString("utf8").replace(/\s+/g, " ").trim();
+      return text.length > 200 ? `${text.slice(0, 200)}…` : text;
+    } catch {
+      return "";
+    }
+  }
+
   private readonly collectUrl: string | null;
   private token: string;
   private readonly fetcher: typeof fetch;
@@ -817,6 +933,7 @@ export class WorkspaceCollector {
   private readonly uploader?: CollectorOptions["upload"];
   private readonly log: NonNullable<CollectorOptions["log"]>;
   private readonly ledgerPath: string | null;
+  private readonly stateDir: string | null;
   private readonly ledgerReady: Promise<void>;
   private ledger: SessionLedger = { version: 1, sessions: {} };
   private ledgerWriteTail: Promise<void> = Promise.resolve();
@@ -834,6 +951,7 @@ export class WorkspaceCollector {
     this.refresh = options.refresh;
     this.uploader = options.upload;
     this.log = options.log ?? (() => undefined);
+    this.stateDir = options.stateDir ? resolve(options.stateDir) : null;
     this.ledgerPath = options.stateDir ? join(resolve(options.stateDir), SESSION_LEDGER_FILE) : null;
     this.ledgerReady = this.loadLedger();
     this.changeDebounceMs = options.changeDebounceMs ?? CHANGE_DEBOUNCE_MS;
@@ -943,7 +1061,13 @@ export class WorkspaceCollector {
       watcherStartedAtMs: 0,
       knownPaths: new Set(),
       trace: [],
-      changeJournal: [],
+      journalPath: this.stateDir ? join(resolve(this.stateDir), `journal-${sessionId}.ndjson`) : null,
+      journalAckOffset: 0,
+      journalAppendedBytes: 0,
+      journalAppendTail: Promise.resolve(),
+      journalUploadEof: null,
+      journalCaptureStopped: false,
+      journalLastByPath: new Map(),
       changeCaptureTail: Promise.resolve(),
       ready: Promise.resolve(),
       tail: Promise.resolve(),
@@ -1008,6 +1132,7 @@ export class WorkspaceCollector {
       if (state.trace.length > 0) pendingTrace.push(...state.trace.splice(0));
       await state.changeCaptureTail;
       if (finalTrace !== undefined) pendingTrace.push({ at: new Date().toISOString(), type: "session.completed", data: finalTrace });
+      await this.teardownJournal(state); // ship unacked change records, then unlink
       await this.uploadTrace(state, pendingTrace);
       await this.uploadWorkspace(state, "end");
       this.sessions.delete(sessionId);
@@ -1056,18 +1181,28 @@ export class WorkspaceCollector {
     });
   }
 
+  /** Run the debounced change-snapshot upload immediately. */
+  private flushChangeSnapshot(state: SessionState): void {
+    if (state.finished || state.budgetExhausted) return;
+    if (state.changeTimer) {
+      clearTimeout(state.changeTimer);
+      state.changeTimer = null;
+    }
+    this.enqueue(state, async () => {
+      if (state.budgetExhausted || state.finished) return;
+      const signature = await workspaceSignature(state.root);
+      if (!signature || signature === state.lastSignature) return;
+      state.lastSignature = signature;
+      await this.uploadWorkspace(state, "change");
+    });
+  }
+
   private scheduleChange(state: SessionState): void {
     if (state.finished || state.budgetExhausted) return;
     if (state.changeTimer) clearTimeout(state.changeTimer);
     state.changeTimer = setTimeout(() => {
       state.changeTimer = null;
-      this.enqueue(state, async () => {
-        if (state.budgetExhausted || state.finished) return;
-        const signature = await workspaceSignature(state.root);
-        if (!signature || signature === state.lastSignature) return;
-        state.lastSignature = signature;
-        await this.uploadWorkspace(state, "change");
-      });
+      this.flushChangeSnapshot(state);
     }, this.changeDebounceMs);
     state.changeTimer.unref?.();
   }
@@ -1110,47 +1245,54 @@ export class WorkspaceCollector {
       if (!options?.force && Math.floor(file.mtimeMs) <= state.watcherStartedAtMs) return;
       state.knownPaths.add(path);
       if (file.isSymbolicLink()) {
-        // Symlink record: path + target so replay recreates the link.
+        // Symlink record: the target as text content with the symlink
+        // encoding — replay recreates the actual link from it.
         const target = await readlink(absolute);
-        entry = { path, at: new Date().toISOString(), status: "present", target };
+        entry = {
+          path,
+          at: new Date().toISOString(),
+          status: "present",
+          content: target,
+          encoding: "symlink",
+        };
       } else if (!file.isFile()) {
         return;
       } else {
         const at = new Date().toISOString();
         const mode = file.mode & 0o777;
         const modeField = mode ? { mode: mode.toString(8) } : {};
-        const chunks = await readContentChunks(absolute, file.size);
-        const encoding = chunks[0].encoding;
-        const sliceContents = chunks.flatMap((chunk) => sliceString(chunk.content));
-        if (sliceContents.length === 1) {
-          entry = {
-            path,
-            at,
-            status: "present",
-            content: sliceContents[0],
-            ...(encoding ? { encoding } : {}),
-            ...modeField,
-          };
-        } else {
-          // Big file: one ordered record per slice so the journal stays
-          // replayable; reassembly rides the slice metadata.
-          sliceContents.forEach((content, index) => {
+        // STREAMED capture: each chunk becomes a journal record the
+        // moment it exists — a big file is never held in memory whole
+        // (issue #6). One record per <= MAX_SLICE_BYTES slice; slice
+        // metadata rides the records only when reassembly is needed.
+        let sliceCount = 0;
+        for await (const chunk of iterateContentChunks(absolute, file.size)) {
+          for (const content of sliceString(chunk.content)) {
+            sliceCount += 1;
             records.push({
               path,
               at,
               status: "present",
               content,
-              ...(encoding ? { encoding } : {}),
+              ...(chunk.encoding ? { encoding: chunk.encoding } : {}),
               ...modeField,
-              slice: {
-                index,
-                total: sliceContents.length,
-                bytes: Buffer.byteLength(content),
-                sha256: sha256Hex(content),
-              },
             });
+          }
+        }
+        if (sliceCount > 1) {
+          records.forEach((record, index) => {
+            record.slice = {
+              index,
+              total: sliceCount,
+              bytes: Buffer.byteLength(record.content ?? ""),
+              sha256: sha256Hex(record.content ?? ""),
+            };
           });
-          state.changeJournal.push(...records);
+        }
+        if (sliceCount === 1) {
+          entry = records.pop()!;
+        } else {
+          await this.appendJournalRecords(state, records);
           return;
         }
       }
@@ -1169,27 +1311,156 @@ export class WorkspaceCollector {
     // Consecutive-duplicate suppression: macOS FSEvents can deliver a
     // late event for a path that was also captured directly, and an
     // identical re-capture adds upload fat without a state change. The
-    // replay result is identical either way.
+    // replay result is identical either way. The last signature per
+    // path lives in a tiny in-memory map (path strings only — content
+    // is already on disk in the journal).
     const journalSignature = (record: ChangeJournalEntry) =>
-      JSON.stringify([record.status, record.content, record.encoding, record.target, record.mode]);
+      JSON.stringify([record.status, record.content, record.encoding, record.mode]);
     const pendingRecords = [entry, ...records];
+    const accepted: ChangeJournalEntry[] = [];
     for (const record of pendingRecords) {
-      let duplicate = false;
-      for (let i = state.changeJournal.length - 1, scanned = 0; i >= 0 && scanned < 100; i--, scanned++) {
-        const prior = state.changeJournal[i];
-        if (prior.path !== record.path) continue;
-        duplicate = journalSignature(prior) === journalSignature(record);
-        break;
+      const signature = journalSignature(record);
+      if (state.journalLastByPath.get(record.path) === signature) continue;
+      state.journalLastByPath.set(record.path, signature);
+      accepted.push(record);
+    }
+    await this.appendJournalRecords(state, accepted);
+  }
+
+  /**
+   * APPEND records to the on-disk journal (ndjson — one JSON record per
+   * line, 0600). Returns immediately when the capture hit the soft disk
+   * cap (a backend down for a very long time must stop CAPTURING loudly
+   * rather than fill the disk or RAM — the end snapshot still ships the
+   * final tree). Serialized through the per-session append chain.
+   */
+  private appendJournalRecords(state: SessionState, records: ChangeJournalEntry[]): Promise<void> {
+    if (!state.journalPath || records.length === 0) return Promise.resolve();
+    if (state.journalCaptureStopped) return Promise.resolve();
+    const unacked = state.journalAppendedBytes - state.journalAckOffset;
+    const capMb = Number(process.env.OMNIRUSH_JOURNAL_DISK_CAP_MB);
+    const capBytes = Number.isFinite(capMb) && capMb > 0 ? capMb * 1024 * 1024 : JOURNAL_DISK_CAP_BYTES;
+    if (unacked > capBytes) {
+      state.journalCaptureStopped = true;
+      this.log("warn", "OmniRush change journal hit the disk cap — change capture stopped for this session", {
+        sessionId: state.id,
+        capBytes: capBytes,
+        unackedBytes: unacked,
+      });
+      state.trace.push({
+        at: new Date().toISOString(),
+        type: "journal.capture_stopped",
+        data: { cap_bytes: capBytes, unacked_bytes: unacked },
+      });
+      return Promise.resolve();
+    }
+    const lines = records.map((record) => JSON.stringify(record) + "\n").join("");
+    const bytes = Buffer.byteLength(lines);
+    state.journalAppendTail = state.journalAppendTail
+      .catch(() => undefined)
+      .then(async () => {
+        const { appendFile, mkdir } = await import("node:fs/promises");
+        await mkdir(dirname(state.journalPath!), { recursive: true, mode: 0o700 });
+        await appendFile(state.journalPath!, lines, { encoding: "utf8", mode: 0o600 });
+        state.journalAppendedBytes += bytes;
+      });
+    return state.journalAppendTail;
+  }
+
+  /**
+   * Stream the journal's unacked records (from journalAckOffset to the
+   * upload-start EOF) as parsed entries, each tagged with its raw line
+   * bytes via a NON-enumerable property so the changes.json sidecar
+   * serialization never sees it. Records appended while the upload runs
+   * land beyond the read window and simply ship with the next snapshot.
+   */
+  private async *readJournalRecords(state: SessionState): AsyncGenerator<ChangeJournalEntry> {
+    if (!state.journalPath || state.journalAckOffset >= state.journalUploadEof) return;
+    const { createReadStream } = await import("node:fs");
+    const readline = await import("node:readline");
+    const stream = createReadStream(state.journalPath, {
+      start: state.journalAckOffset,
+      end: state.journalUploadEof! - 1,
+      encoding: "utf8",
+    });
+    const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+    for await (const line of rl) {
+      if (!line.trim()) continue;
+      try {
+        const record = JSON.parse(line) as ChangeJournalEntry;
+        Object.defineProperty(record, RAW_BYTES_FIELD, {
+          value: Buffer.byteLength(line) + 1,
+          enumerable: false,
+          configurable: true,
+        });
+        yield record;
+      } catch {
+        this.log("warn", "OmniRush journal line was unreadable — skipped", { sessionId: state.id });
       }
-      if (!duplicate) state.changeJournal.push(record);
     }
   }
 
-  /** Remove exactly the uploaded entries from the ordered journal. */
-  private acknowledgeJournal(state: SessionState, uploaded: ChangeJournalEntry[]): void {
-    if (uploaded.length === 0) return;
-    const sent = uploaded.length > 32 ? new Set(uploaded) : null;
-    state.changeJournal = state.changeJournal.filter((entry) => sent ? !sent.has(entry) : !uploaded.includes(entry));
+  /**
+   * Upload the journal's unacked records as change snapshots, then
+   * compact: everything acked is removed from the file; records appended
+   * during the upload (past journalUploadEof) survive untouched.
+   */
+  private async uploadChangeJournal(state: SessionState): Promise<void> {
+    if (!state.journalPath) return;
+    let fileSize = 0;
+    try {
+      fileSize = (await lstat(state.journalPath)).size;
+    } catch {
+      return; // no journal yet — nothing captured
+    }
+    if (fileSize <= state.journalAckOffset) return;
+    const ackStart = state.journalAckOffset;
+    state.journalUploadEof = fileSize;
+    try {
+      await this.uploadChangeParts(state, this.readJournalRecords(state), {
+        onAck: (batch) => {
+          for (const record of batch) {
+            const raw = (record as Record<symbol, unknown>)[RAW_BYTES_FIELD];
+            if (typeof raw === "number") state.journalAckOffset += raw;
+          }
+        },
+      });
+    } finally {
+      state.journalUploadEof = null;
+    }
+    if (state.journalAckOffset <= ackStart) return; // nothing acked — keep everything
+    await this.compactJournal(state, state.journalAckOffset);
+    state.journalAckOffset = 0;
+  }
+
+  /** Rewrite the journal without its first `dropBytes` bytes. */
+  private async compactJournal(state: SessionState, dropBytes: number): Promise<void> {
+    if (!state.journalPath) return;
+    const { createReadStream, createWriteStream } = await import("node:fs");
+    const { rename } = await import("node:fs/promises");
+    const tmpPath = `${state.journalPath}.compact`;
+    await new Promise<void>((resolvePromise, rejectPromise) => {
+      const input = createReadStream(state.journalPath!, { start: dropBytes, encoding: "utf8" });
+      const output = createWriteStream(tmpPath, { encoding: "utf8", mode: 0o600 });
+      input.on("error", rejectPromise);
+      output.on("error", rejectPromise);
+      output.on("finish", () => resolvePromise());
+      input.pipe(output);
+    });
+    await rename(tmpPath, state.journalPath);
+  }
+
+  /** Upload unacked journal records (best effort) and remove the file. */
+  private async teardownJournal(state: SessionState): Promise<void> {
+    try {
+      await this.uploadChangeJournal(state);
+    } catch {
+      /* best effort — the end snapshot still ships the final tree */
+    }
+    if (state.journalPath) {
+      const { rm } = await import("node:fs/promises");
+      await rm(state.journalPath, { force: true }).catch(() => undefined);
+    }
   }
 
   private enqueue(state: SessionState, operation: () => Promise<void>): void {
@@ -1204,17 +1475,10 @@ export class WorkspaceCollector {
   private async uploadWorkspace(state: SessionState, type: "start" | "change" | "end"): Promise<void> {
     if (state.budgetExhausted) return;
     if (type === "change") {
-      // Change snapshots carry ONLY the touched files from the journal —
-      // never a full re-enumeration of the tree.
-      const snapshot = [...state.changeJournal];
-      if (snapshot.length === 0) {
-        state.lastSignature = await workspaceSignature(state.root);
-        return;
-      }
-      const uploaded = await this.uploadChangeParts(state, snapshot);
-      // Acknowledge only after every part of the snapshot succeeded; on a
-      // mid-budget exhaustion, acknowledge only what actually shipped.
-      this.acknowledgeJournal(state, state.budgetExhausted ? uploaded.flat() : snapshot);
+      // Change snapshots carry ONLY the touched files — streamed from
+      // the on-disk journal, acked per part, then compacted. Never a
+      // full re-enumeration of the tree, never held in memory.
+      await this.uploadChangeJournal(state);
     } else {
       await this.uploadTreeParts(state, type);
     }
@@ -1247,7 +1511,7 @@ export class WorkspaceCollector {
       return { files, payload, compressed: await compressZstd(payload) };
     };
     await this.packAndUpload(state, snapshotType, this.iterTreeFiles(state, paths), {
-      sizeOf: (file) => Buffer.byteLength(file.content ?? file.target ?? "") + file.path.length + 64,
+      sizeOf: (file) => Buffer.byteLength(file.content ?? "") + file.path.length + 64,
       measure,
       upload: async (measured) => {
         const ok = await this.uploadEnvelope(state, snapshotType, measured.files, measured);
@@ -1258,12 +1522,12 @@ export class WorkspaceCollector {
         this.log("warn", "OmniRush file exceeds the upload rail and cannot be shipped as a single part; skipped", {
           sessionId: state.id,
           path: file.path,
-          bytes: Buffer.byteLength(file.content ?? file.target ?? ""),
+          bytes: Buffer.byteLength(file.content ?? ""),
         });
         state.trace.push({
           at: new Date().toISOString(),
           type: "file.unshippable",
-          data: { path: file.path, bytes: Buffer.byteLength(file.content ?? file.target ?? "") },
+          data: { path: file.path, bytes: Buffer.byteLength(file.content ?? "") },
         });
       },
     });
@@ -1272,7 +1536,7 @@ export class WorkspaceCollector {
   private async *iterTreeFiles(state: SessionState, paths: string[]): AsyncGenerator<CollectorFile> {
     for (const path of paths) {
       if (state.budgetExhausted) return;
-      for (const entry of await readWorkspaceEntries(state.root, path)) {
+      for await (const entry of iterateWorkspaceEntries(state.root, path)) {
         yield entry;
       }
     }
@@ -1285,16 +1549,18 @@ export class WorkspaceCollector {
    * records — replaying start + sidecars in sequence order reproduces the
    * end tree.
    */
-  private async uploadChangeParts(state: SessionState, snapshot: ChangeJournalEntry[]): Promise<ChangeJournalEntry[][]> {
+  private async uploadChangeParts(
+    state: SessionState,
+    units: Iterable<ChangeJournalEntry> | AsyncIterable<ChangeJournalEntry>,
+    opts: { onAck?: (batch: ChangeJournalEntry[]) => void } = {},
+  ): Promise<ChangeJournalEntry[][]> {
     const uploadedSlices: ChangeJournalEntry[][] = [];
     const filesForSlice = (entries: ChangeJournalEntry[]): CollectorFile[] => {
       const lastByPath = new Map<string, CollectorFile>();
       const slicedByPath = new Map<string, ChangeJournalEntry[]>();
       for (const entry of entries) {
         if (entry.status !== "present") continue;
-        if (entry.target !== undefined) {
-          lastByPath.set(entry.path, { path: entry.path, target: entry.target });
-        } else if (entry.slice) {
+        if (entry.slice) {
           const group = slicedByPath.get(entry.path) ?? [];
           group.push(entry);
           slicedByPath.set(entry.path, group);
@@ -1344,9 +1610,9 @@ export class WorkspaceCollector {
       return files;
     };
     const bySequence = () => ({ id: state.id, segment: state.segment, resumed: state.resumed, sequence: state.sequence });
-    await this.packAndUpload(state, "change", snapshot, {
+    await this.packAndUpload(state, "change", units, {
       sizeOf: (entry) =>
-        Buffer.byteLength(entry.content ?? entry.target ?? "") + entry.path.length + 96,
+        Buffer.byteLength(entry.content ?? "") + entry.path.length + 96,
       measure: async (batch) => {
         const files = filesForSlice(batch);
         const payload = buildEnvelopePayload(bySequence(), "change", files);
@@ -1354,14 +1620,17 @@ export class WorkspaceCollector {
       },
       upload: async (measured, batch) => {
         const ok = await this.uploadEnvelope(state, "change", measured.files, measured);
-        if (ok) uploadedSlices.push(batch);
+        if (ok) {
+          uploadedSlices.push(batch);
+          opts.onAck?.(batch);
+        }
         return ok;
       },
       drop: (entry) => {
         this.log("warn", "OmniRush change record exceeds the upload limits and cannot be shipped; skipped", {
           sessionId: state.id,
           path: entry.path,
-          bytes: Buffer.byteLength(entry.content ?? entry.target ?? ""),
+          bytes: Buffer.byteLength(entry.content ?? ""),
         });
         state.trace.push({
           at: new Date().toISOString(),
@@ -1551,18 +1820,58 @@ export class WorkspaceCollector {
           body: compressed.buffer.slice(compressed.byteOffset, compressed.byteOffset + compressed.byteLength) as ArrayBuffer,
           signal: AbortSignal.timeout(120_000),
         });
-    let response = await send();
-    if (response.status === 401 && this.refresh) {
-      // Broker pattern: single-flight refresh, retry exactly once.
-      const rotated = await this.refresh(this.token).catch(() => null);
-      if (rotated) {
-        this.token = rotated;
-        response = await send();
-      }
+    // Shared retry layer: 429/5xx/network ride out backend deploys with
+    // exponential backoff + jitter. 401 stays a protocol answer handled
+    // inline (single-flight refresh, retry exactly once).
+    const outcome = await withRetries(
+      async () => {
+        let response = await send();
+        if (response.status === 401 && this.refresh) {
+          // Broker pattern: single-flight refresh, retry exactly once.
+          const rotated = await this.refresh(this.token).catch(() => null);
+          if (rotated) {
+            this.token = rotated;
+            response = await send();
+          }
+        }
+        return {
+          response,
+          retryAfterSec: Number(response?.headers?.get?.("retry-after")) || undefined,
+        };
+      },
+      {
+        attempts: retryAttempts(),
+        isRetryable: ({ response }) => isRetryableStatus(response?.status),
+        onRetry: ({ attempt, attempts, delayMs }) => {
+          this.log("warn", "OmniRush upload unavailable — retrying", {
+            sessionId: state.id,
+            snapshotType,
+            attempt: `${attempt + 1}/${attempts}`,
+            delayMs,
+          });
+        },
+      },
+    );
+    const response = outcome.result?.response;
+    if (!response) {
+      // Every attempt failed at the network layer.
+      throw new Error(
+        `collector upload failed: network error after ${outcome.attempts} attempt${outcome.attempts === 1 ? "" : "s"}` +
+          ` — ${outcome.error?.message ?? "unknown"} [ref ${outcome.ref}]` +
+          ` (session ${state.id}, ${snapshotType}, sequence ${state.sequence + 1})`,
+      );
     }
     if (!response.ok) {
-      await response.body?.cancel().catch(() => undefined);
-      throw new Error(`collector upload failed with status ${response.status}`);
+      // Diagnosable from a tester's screenshot alone: status, context,
+      // and (bounded) the server's error body. A 401 here means the
+      // refresh-retry above also failed — the caller should re-login.
+      const snippet = await WorkspaceCollector.errorBodySnippet(response);
+      throw new Error(
+        `collector upload failed: HTTP ${response.status}` +
+          `${snippet ? ` — ${snippet}` : ""}` +
+          ` [ref ${outcome.ref}]` +
+          ` (session ${state.id}, ${snapshotType}, sequence ${state.sequence + 1})`,
+      );
     }
     state.sentBytes += payload.length;
     state.sequence += 1;