omnirush 0.4.1 → 0.5.0

package/assets/extensions/omnirush/collector-lib.ts

@@ -52,6 +52,9 @@ import { dirname, join, relative, resolve, sep } from "node:path";
 import { promisify } from "node:util";
 import { zstdCompress as zstdCompressCb } from "node:zlib";
 
+import { isRetryableStatus, retryAttempts, withRetries } from "./retry";
+import { buildTraceArtifact, parsePiSession } from "./trace-format";
+
 const execFileAsync = promisify(execFile);
 
 // --- limits ---------------------------------------------------------------
@@ -67,10 +70,31 @@ export const MAX_SESSION_BYTES = 100 * 1024 * 1024 * 1024;
 // byte size + sha256) so replay reassembles the original deterministically.
 export const MAX_SLICE_BYTES = 4 * 1024 * 1024;
 // Whole-read ceiling. Files up to this size are read and processed as
-// one string (worst case: binary -> base64 inflates 4/3 to ~427 MiB
-// chars, comfortably under the runtime's ~512 MiB MAX_STRING_LENGTH).
-// Bigger files are streamed in fixed raw chunks and sliced.
-const WHOLE_READ_BYTES = 320 * 1024 * 1024;
+// one string; bigger files stream in fixed raw chunks and slice.
+// Deliberately tiny (issue #6): a whole read holds buffer + utf8 string
+// + redaction copy + base64 simultaneously (~4-5x file size), so even a
+// modest ceiling spikes RSS by that multiple per file. 8 MiB bounds the
+// worst case to ~40 MiB; anything bigger takes the streaming path with
+// identical output.
+const WHOLE_READ_BYTES = 8 * 1024 * 1024;
+// Text-segment cap for newline-less files (see the streaming reader).
+const PENDING_TEXT_CAP_BYTES = 32 * 1024 * 1024;
+// Change-journal memory ceiling (issue #6, Windows 8-9 GiB OOM): the
+// journal holds every changed file's content between debounced change
+// uploads. At HIGH_WATER an upload is forced immediately (normal
+// backpressure); at the HARD ceiling — a backend down for a long time —
+// the OLDEST records are dropped, loudly (warn + journal.dropped trace),
+// because losing replay data beats crashing the whole session.
+// Soft disk cap for the on-disk change journal: when unacked journal
+// bytes exceed it (a backend down for a very long time), capture stops
+// loudly instead of filling the disk. The end snapshot still ships.
+export const JOURNAL_DISK_CAP_BYTES = 1024 * 1024 * 1024;
+/**
+ * Non-enumerable per-record field carrying the raw serialized line
+ * length (for journal ack accounting) — invisible to JSON.stringify so
+ * the changes.json sidecar never includes it.
+ */
+export const RAW_BYTES_FIELD = Symbol("omnirushJournalRawBytes");
 // Multi-file parts target this compressed size (the backend body rail
 // was live-verified at >= 31.5 MiB compressed, so 15 MiB leaves ample
 // headroom for any ingress in front of the manager).
@@ -83,8 +107,10 @@ export const PART_UNCOMPRESSED_STEP = 8 * 1024 * 1024;
 // Flush a part when a check shows at least this much compressed payload.
 export const PART_READY_COMPRESSED_BYTES = 12 * 1024 * 1024;
 // Absolute uncompressed ceiling for one part batch — bounds resident
-// memory even for highly-compressible content.
-export const PART_BATCH_UNCOMPRESSED_MAX = 48 * 1024 * 1024;
+// memory even for highly-compressible content (incompressible batches
+// never hit the compressed-size thresholds, so this is what actually
+// stops accumulation; the payload build doubles it transiently).
+export const PART_BATCH_UNCOMPRESSED_MAX = 24 * 1024 * 1024;
 // Change/trace parts flush by PLAIN size so their __agent__/changes.json
 // / trace.json entries (single file entries whose content grows with the
 // batch) stay small and parts upload promptly.
@@ -141,6 +167,9 @@ export type SessionLedgerRecord = {
   sentBytes?: number;
   lastMessageId?: string;
   lastSeenAt: string;
+  /** Per-session-FILE transcript capture state (issue: trace format spec).
+   *  Keys are absolute .jsonl paths; a size/mtime change re-captures. */
+  traces?: Record<string, { size: number; mtimeMs: number }>;
 };
 
 export type SessionLedger = {
@@ -176,8 +205,30 @@ type SessionState = {
    *  fabricate deletions inside the workspace. */
   knownPaths: Set<string>;
   trace: TraceEvent[];
-  /** Ordered change records — the replayable change log. */
-  changeJournal: ChangeJournalEntry[];
+  /**
+   * The change journal lives ON DISK (ndjson sidecar, one JSON record
+   * per line) — never as an in-memory array (issue #6: the array held
+   * every changed file's content string and grew to multiple GB).
+   * Captures APPEND to the file; change uploads stream the unacked
+   * region and compact the acked prefix away. Appends during an upload
+   * simply land past the read window (bounded by `journalUploadEof`) —
+   * no gating, no memory growth.
+   */
+  journalPath: string | null;
+  /** File offset of the first not-yet-uploaded record. */
+  journalAckOffset: number;
+  /** Bytes appended so far (soft disk cap accounting). */
+  journalAppendedBytes: number;
+  /** Serializes journal appends. */
+  journalAppendTail: Promise<void>;
+  /** Set while a change upload reads the journal. */
+  journalUploadEof: number | null;
+  /** True once the soft disk cap stopped captures (warned once). */
+  journalCaptureStopped: boolean;
+  /** Last journal signature per path (consecutive-duplicate suppression). */
+  journalLastByPath: Map<string, string>;
+  /** Transcript capture state carried across runs via the ledger. */
+  transcripts: Record<string, { size: number; mtimeMs: number }>;
   changeCaptureTail: Promise<void>;
   ready: Promise<void>;
   tail: Promise<void>;
@@ -186,6 +237,13 @@ type SessionState = {
 export type CollectorOptions = {
   gatewayUrl?: string;
   accessToken?: string;
+  /** pi agent config dir (~/.pi/agent): session transcripts live under
+   *  <agentDir>/sessions/<cwd-slug>/. Required for trace capture. */
+  agentDir?: string;
+  /** Resolves the signed-in user id for trace headers (best effort). */
+  identityProvider?: () => Promise<{ userId: string | null }>;
+  /** Stable client identifier for trace headers (e.g. hostname). */
+  clientId?: string;
   fetch?: typeof fetch;
   /** Returns the rotated access token, or null when refresh failed. */
   refresh?: (tokenUsed: string) => Promise<string | null>;
@@ -394,7 +452,7 @@ export function selectTraceEvents(
  * removed on purpose: `git ls-files` drops anything .gitignored (build
  * outputs, node_modules), which broke the whole-project guarantee.
  */
-async function listWorkspaceFiles(root: string): Promise<string[]> {
+export async function listWorkspaceFiles(root: string): Promise<string[]> {
   return walkWorkspace(root);
 }
 
@@ -528,19 +586,23 @@ function sliceManifest(
 type ContentChunk = { content: string; encoding?: "base64" };
 
 /**
- * Read a regular file into <= ~8 MiB content chunks: small files in one
- * whole read (fully redacted), big files streamed in fixed raw chunks —
- * strings stay far below the runtime string limit regardless of file
- * size. Binary detection happens on the first chunk and applies to the
- * whole file.
+ * Stream a regular file as <= ~8 MiB content chunks, yielding each chunk
+ * the moment it is produced — a file's content is NEVER materialized as
+ * one string regardless of size (issue #6: the old array version held
+ * every chunk at once). Small files are a single whole read; big files
+ * stream in fixed raw chunks. Binary detection happens on the first
+ * chunk and applies to the whole file; text is redacted per
+ * line-aligned segment.
  */
-async function readContentChunks(absolute: string, size: number): Promise<ContentChunk[]> {
+async function* iterateContentChunks(absolute: string, size: number): AsyncGenerator<ContentChunk> {
   if (size <= WHOLE_READ_BYTES) {
     const buffer = await readFile(absolute);
     if (isBinary(buffer)) {
-      return [{ content: buffer.toString("base64"), encoding: "base64" }];
+      yield { content: buffer.toString("base64"), encoding: "base64" };
+      return;
     }
-    return [{ content: redactCollectorText(buffer.toString("utf8")).text }];
+    yield { content: redactCollectorText(buffer.toString("utf8")).text };
+    return;
   }
   const { open } = await import("node:fs/promises");
   const handle = await open(absolute, "r");
@@ -549,114 +611,139 @@ async function readContentChunks(absolute: string, size: number): Promise<Conten
     const buffer = Buffer.alloc(chunkBytes);
     const first = await handle.read(buffer, 0, chunkBytes, null);
     const binary = isBinary(buffer.subarray(0, first.bytesRead));
-    const chunks: ContentChunk[] = [];
     if (binary) {
       // Binaries carry no redactable text: base64 each raw chunk.
-      chunks.push({ content: buffer.subarray(0, first.bytesRead).toString("base64"), encoding: "base64" });
+      yield { content: buffer.subarray(0, first.bytesRead).toString("base64"), encoding: "base64" };
       let read = 0;
       while ((read = (await handle.read(buffer, 0, chunkBytes, null)).bytesRead) > 0) {
-        chunks.push({ content: buffer.subarray(0, read).toString("base64"), encoding: "base64" });
+        yield { content: buffer.subarray(0, read).toString("base64"), encoding: "base64" };
       }
-    } else {
-      // Text: decode incrementally (StringDecoder absorbs multibyte
-      // sequences split across chunk boundaries), emit only up to the
-      // last complete line, carry the remainder, and redact each
-      // line-aligned segment.
-      const { StringDecoder } = await import("node:string_decoder");
-      const decoder = new StringDecoder("utf8");
-      let pending = decoder.write(buffer.subarray(0, first.bytesRead));
-      let read = 0;
-      while ((read = (await handle.read(buffer, 0, chunkBytes, null)).bytesRead) > 0) {
-        pending += decoder.write(buffer.subarray(0, read));
+      return;
+    }
+    // Text: decode incrementally (StringDecoder absorbs multibyte
+    // sequences split across chunk boundaries), emit only up to the
+    // last complete line, carry the remainder, and redact each
+    // line-aligned segment. A pathological newline-less file must not
+    // grow `pending` forever: at PENDING_TEXT_CAP_BYTES the segment is
+    // flushed mid-line (line-anchored redaction weakens at that rare
+    // cut; unbounded memory is worse).
+    const { StringDecoder } = await import("node:string_decoder");
+    const decoder = new StringDecoder("utf8");
+    let pending = decoder.write(buffer.subarray(0, first.bytesRead));
+    let read = 0;
+    while ((read = (await handle.read(buffer, 0, chunkBytes, null)).bytesRead) > 0) {
+      pending += decoder.write(buffer.subarray(0, read));
+      for (;;) {
         const lastNl = pending.lastIndexOf("\n");
         if (lastNl >= 0) {
-          chunks.push({ content: redactCollectorText(pending.slice(0, lastNl + 1)).text });
+          yield { content: redactCollectorText(pending.slice(0, lastNl + 1)).text };
           pending = pending.slice(lastNl + 1);
+          continue;
+        }
+        if (pending.length >= PENDING_TEXT_CAP_BYTES) {
+          yield { content: redactCollectorText(pending).text };
+          pending = "";
         }
+        break;
       }
-      pending += decoder.end();
-      if (pending) chunks.push({ content: redactCollectorText(pending).text });
     }
-    return chunks;
+    pending += decoder.end();
+    if (pending) yield { content: redactCollectorText(pending).text };
   } finally {
     await handle.close();
   }
 }
 
 /**
- * Read one workspace entry into CollectorFile entries. Regular files:
+ * Stream one workspace entry as CollectorFile entries. Regular files:
  * text is redacted, binaries base64-encoded ("encoding": "base64"), and
  * the POSIX mode rides along (octal string) so replay can restore the
  * executable bit (stored-and-ignored on Windows). Content bigger than
- * MAX_SLICE_BYTES is sliced (see buildFileEntries). Symlinks become text
- * entries { path, content: <target>, encoding: "symlink" } — the
- * backend's per-entry validation requires text content, and replay
- * recreates the actual link from the target. Returns [] when the entry
- * vanished or is a special file.
+ * MAX_SLICE_BYTES is sliced on the fly — slices are yielded the moment
+ * they exist while only the tiny sha256 descriptors accumulate, then
+ * the manifest is yielded LAST (replay unions manifests by index, so
+ * position never matters). Symlinks become text entries
+ * { path, content: <target>, encoding: "symlink" } — the backend's
+ * per-entry validation requires text content, and replay recreates the
+ * actual link from the target. Yields nothing when the entry vanished
+ * or is a special file.
  */
-async function readWorkspaceEntries(root: string, path: string): Promise<CollectorFile[]> {
+async function* iterateWorkspaceEntries(root: string, path: string): AsyncGenerator<CollectorFile> {
+  let absolute = "";
   try {
-    const absolute = resolve(root, path);
-    if (portablePath(root, absolute).startsWith("../")) return [];
+    absolute = resolve(root, path);
+    if (portablePath(root, absolute).startsWith("../")) return;
     const file = await lstat(absolute);
     if (file.isSymbolicLink()) {
       const target = await readlink(absolute);
-      return [{ path, content: target, encoding: "symlink" }];
+      yield { path, content: target, encoding: "symlink" };
+      return;
     }
-    if (!file.isFile()) return [];
+    if (!file.isFile()) return;
     const mode = file.mode & 0o777;
-    const chunks: ContentChunk[] = await readContentChunks(absolute, file.size);
-    if (chunks.length === 1) {
-      const chunk = chunks[0];
-      return buildFileEntries({
-        path,
-        content: chunk.content,
-        ...(chunk.encoding ? { encoding: chunk.encoding } : {}),
-        ...(mode ? { mode: mode.toString(8) } : {}),
-      });
+    const modeField = mode ? { mode: mode.toString(8) } : {};
+    if (file.size <= WHOLE_READ_BYTES) {
+      // Small file: whole read, real path preserved; buildFileEntries
+      // slices (manifest + parts) when the encoded content exceeds
+      // MAX_SLICE_BYTES — same shape as always.
+      for await (const chunk of iterateContentChunks(absolute, file.size)) {
+        for (const entry of buildFileEntries({
+          path,
+          content: chunk.content,
+          ...(chunk.encoding ? { encoding: chunk.encoding } : {}),
+          ...modeField,
+        })) {
+          yield entry;
+        }
+      }
+      return;
+    }
+    // Big file (streamed): slices are yielded the moment they exist;
+    // only the tiny sha256 descriptors accumulate. The reassembly
+    // manifest is yielded LAST (replay unions manifests by index, so
+    // position never matters).
+    let index = 0;
+    const descriptors: Array<{ index: number; bytes: number; sha256: string }> = [];
+    let sawEncoding: "base64" | undefined;
+    for await (const chunk of iterateContentChunks(absolute, file.size)) {
+      sawEncoding ??= chunk.encoding;
+      for (const slice of sliceString(chunk.content)) {
+        descriptors.push({ index, bytes: Buffer.byteLength(slice), sha256: sha256Hex(slice) });
+        yield {
+          path: slicePartPath(path, index),
+          content: slice,
+          ...(sawEncoding ? { encoding: sawEncoding } : {}),
+        };
+        index += 1;
+      }
     }
-    // Big file: streamed chunks -> one logical entry per chunk (already
-    // <= ~8 MiB for raw chunks; base64 inflates 4/3, so sub-slice those),
-    // plus the manifest for deterministic reassembly.
-    const encoding = chunks[0].encoding;
-    const entries: CollectorFile[] = [];
-    const sliceContents = chunks.flatMap((chunk) => sliceString(chunk.content));
-    entries.push({
+    if (index === 0) return;
+    yield {
       path: sliceManifestPath(path),
       content: JSON.stringify(sliceManifest(
-        {
-          path,
-          ...(encoding ? { encoding } : {}),
-          ...(mode ? { mode: mode.toString(8) } : {}),
-        },
-        sliceContents.map((slice, index) => ({
-          index,
-          bytes: Buffer.byteLength(slice),
-          sha256: sha256Hex(slice),
-        })),
+        { path, ...(sawEncoding ? { encoding: sawEncoding } : {}), ...modeField },
+        descriptors,
       )),
-    });
-    sliceContents.forEach((slice, index) => {
-      entries.push({
-        path: slicePartPath(path, index),
-        content: slice,
-        ...(encoding ? { encoding } : {}),
-      });
-    });
-    return entries;
+    };
+    return;
   } catch {
     // Workspaces are live; races are expected and retried by the next snapshot.
-    return [];
+    return;
   }
 }
 
-export async function collectFiles(
+/**
+ * Streaming file collection: yields workspace files (metadata entry
+ * first) one at a time so the one-shot path never holds the whole tree's
+ * content in memory (issue #6). Same order, denylist, and byte budget
+ * as the compat collectFiles drain below.
+ */
+export async function* iterateCollectFiles(
   root: string,
   byteLimit: number,
   workspaceId: string,
   session?: Pick<SessionState, "id" | "segment" | "resumed">,
-): Promise<CollectorFile[]> {
-  const files: CollectorFile[] = [];
+): AsyncGenerator<CollectorFile> {
   let used = 0;
   const metadata = JSON.stringify({
     workspace_id: workspaceId,
@@ -668,19 +755,32 @@ export async function collectFiles(
     root_name: root.split(sep).filter(Boolean).at(-1) ?? "workspace",
     git: await gitMetadata(root),
   });
-  files.push({ path: `${AGENT_DIR}/workspace.json`, content: metadata });
+  yield { path: `${AGENT_DIR}/workspace.json`, content: metadata };
   used += Buffer.byteLength(metadata);
 
   const paths = (await listWorkspaceFiles(root)).sort(comparePaths);
   for (const path of paths) {
     if (used >= byteLimit) break;
-    for (const file of await readWorkspaceEntries(root, path)) {
+    for await (const file of iterateWorkspaceEntries(root, path)) {
       const size = Buffer.byteLength(file.content ?? "");
       if (used + size > byteLimit) continue;
-      files.push(file);
+      yield file;
       used += size;
     }
   }
+}
+
+/** Compat drain of iterateCollectFiles (tests + tooling). */
+export async function collectFiles(
+  root: string,
+  byteLimit: number,
+  workspaceId: string,
+  session?: Pick<SessionState, "id" | "segment" | "resumed">,
+): Promise<CollectorFile[]> {
+  const files: CollectorFile[] = [];
+  for await (const file of iterateCollectFiles(root, byteLimit, workspaceId, session)) {
+    files.push(file);
+  }
   return files;
 }
 
@@ -724,14 +824,11 @@ export type EnvelopePart = {
 };
 
 /**
- * Deterministic part split for a full files array (used by `omnirush
- * collect` and the tests): sort files by path, accumulate in order, keep
- * each part <= maxCompressed compressed. A single file bigger than one
- * part gets its own part; file content is never split across parts. A
- * lone file that cannot fit even the hard rail is skipped with a warning.
- * Envelope sequence numbers are state.sequence + 1 + partIndex.
+ * Deterministic part split for a full files array (compat drain of
+ * iterateEnvelopeParts, used by the tests): sort files by path,
+ * accumulate in order, keep each part <= maxCompressed compressed.
  */
-export function buildEnvelopeParts(
+export async function buildEnvelopeParts(
   state: Pick<SessionState, "id" | "segment" | "resumed"> & { sequence?: number },
   snapshotType: SnapshotType,
   files: CollectorFile[],
@@ -744,28 +841,34 @@ export function buildEnvelopeParts(
   const sorted = files
     .flatMap((file) => buildFileEntries(file))
     .sort((left, right) => comparePaths(left.path, right.path));
-  return packEnvelopeParts(state, snapshotType, sorted, maxCompressed, log);
+  const parts: EnvelopePart[] = [];
+  for await (const part of iterateEnvelopeParts(state, snapshotType, sorted, maxCompressed, log)) {
+    parts.push(part);
+  }
+  return parts;
 }
 
 /**
- * Shared greedy packer: split an ordered file list into compressed parts,
- * preserving path order across parts. Used by buildEnvelopeParts (pure,
- * whole list in memory) and mirrored by the collector's streaming packer
- * (packAndUpload, which never holds more than one part in memory).
+ * Streaming packer for the one-shot path: yields compressed parts in
+ * order so callers can upload-and-discard instead of holding the whole
+ * workspace's parts (payload + compressed buffers) in memory (issue #6).
+ * Same greedy algorithm as before: split at the compressed target, a
+ * lone file may use the hard rail, lone-over-rail files are skipped
+ * loudly. File content is never split across parts.
  */
-export async function packEnvelopeParts(
+export async function* iterateEnvelopeParts(
   state: Pick<SessionState, "id" | "segment" | "resumed"> & { sequence?: number },
   snapshotType: SnapshotType,
-  sortedFiles: CollectorFile[],
+  sortedFiles: Iterable<CollectorFile> | AsyncIterable<CollectorFile>,
   maxCompressed = MAX_PART_COMPRESSED_BYTES,
   log?: (message: string, attributes?: Record<string, unknown>) => void,
-): Promise<EnvelopePart[]> {
+): AsyncGenerator<EnvelopePart> {
   const hard = Math.max(maxCompressed, MAX_PART_COMPRESSED_HARD);
   const measure = async (batch: CollectorFile[], partIndex: number): Promise<EnvelopePart> => {
     const payload = buildEnvelopePayload({ ...state, sequence: (state.sequence ?? 0) + partIndex }, snapshotType, batch);
     return { files: batch, payload, compressed: await compressZstd(payload) };
   };
-  const parts: EnvelopePart[] = [];
+  let partsYielded = 0;
   let batch: CollectorFile[] = [];
   const fit = async (batchToFit: CollectorFile[], partIndex: number) => {
     let measured = await measure(batchToFit, partIndex);
@@ -787,15 +890,16 @@ export async function packEnvelopeParts(
     }
     return { measured, carry };
   };
-  for (const file of sortedFiles) {
+  for await (const file of sortedFiles) {
     batch.push(file);
-    const partIndex = parts.length;
+    const partIndex = partsYielded;
     const estimate = batch.reduce((total, f) => total + Buffer.byteLength(f.content), 0);
     if (estimate < PART_UNCOMPRESSED_STEP) continue;
     const { measured, carry } = await fit(batch, partIndex);
     if (measured
       && (measured.compressed.length >= PART_READY_COMPRESSED_BYTES || estimate >= PART_BATCH_UNCOMPRESSED_MAX)) {
-      parts.push(measured);
+      partsYielded += 1;
+      yield measured;
       batch = carry;
     } else if (measured) {
       batch = [...batch, ...carry];
@@ -804,12 +908,14 @@ export async function packEnvelopeParts(
     }
   }
   while (batch.length > 0) {
-    const { measured, carry } = await fit(batch, parts.length);
-    if (measured) parts.push(measured);
+    const { measured, carry } = await fit(batch, partsYielded);
+    if (measured) {
+      partsYielded += 1;
+      yield measured;
+    }
     if (carry.length === 0) break;
     batch = carry;
   }
-  return parts;
 }
 
 export class WorkspaceCollector {
@@ -840,6 +946,12 @@ export class WorkspaceCollector {
   private readonly uploader?: CollectorOptions["upload"];
   private readonly log: NonNullable<CollectorOptions["log"]>;
   private readonly ledgerPath: string | null;
+  private readonly stateDir: string | null;
+  private readonly agentDir: string | null;
+  private readonly identityProvider?: CollectorOptions["identityProvider"];
+  private readonly clientId?: string;
+  private identity: { userId: string | null } | null = null;
+  private identityTried = false;
   private readonly ledgerReady: Promise<void>;
   private ledger: SessionLedger = { version: 1, sessions: {} };
   private ledgerWriteTail: Promise<void> = Promise.resolve();
@@ -857,6 +969,10 @@ export class WorkspaceCollector {
     this.refresh = options.refresh;
     this.uploader = options.upload;
     this.log = options.log ?? (() => undefined);
+    this.stateDir = options.stateDir ? resolve(options.stateDir) : null;
+    this.agentDir = options.agentDir ? resolve(options.agentDir) : null;
+    this.identityProvider = options.identityProvider;
+    this.clientId = options.clientId;
     this.ledgerPath = options.stateDir ? join(resolve(options.stateDir), SESSION_LEDGER_FILE) : null;
     this.ledgerReady = this.loadLedger();
     this.changeDebounceMs = options.changeDebounceMs ?? CHANGE_DEBOUNCE_MS;
@@ -898,6 +1014,9 @@ export class WorkspaceCollector {
     state.sentBytes = previous?.sentBytes ?? 0;
     state.lastMessageId = previous?.lastMessageId;
     state.resumed = Boolean(previous);
+    if (previous?.traces && typeof previous.traces === "object") {
+      state.transcripts = { ...previous.traces };
+    }
     if (state.resumed) {
       state.trace.push({
         at: new Date().toISOString(),
@@ -922,6 +1041,7 @@ export class WorkspaceCollector {
       nextSequence: state.sequence,
       sentBytes: state.sentBytes,
       ...(state.lastMessageId ? { lastMessageId: state.lastMessageId } : {}),
+      ...(Object.keys(state.transcripts).length > 0 ? { traces: state.transcripts } : {}),
       lastSeenAt: new Date().toISOString(),
     };
     await this.saveLedger();
@@ -966,7 +1086,14 @@ export class WorkspaceCollector {
       watcherStartedAtMs: 0,
       knownPaths: new Set(),
       trace: [],
-      changeJournal: [],
+      journalPath: this.stateDir ? join(resolve(this.stateDir), `journal-${sessionId}.ndjson`) : null,
+      journalAckOffset: 0,
+      journalAppendedBytes: 0,
+      journalAppendTail: Promise.resolve(),
+      journalUploadEof: null,
+      journalCaptureStopped: false,
+      journalLastByPath: new Map(),
+      transcripts: {},
       changeCaptureTail: Promise.resolve(),
       ready: Promise.resolve(),
       tail: Promise.resolve(),
@@ -1031,6 +1158,7 @@ export class WorkspaceCollector {
       if (state.trace.length > 0) pendingTrace.push(...state.trace.splice(0));
       await state.changeCaptureTail;
       if (finalTrace !== undefined) pendingTrace.push({ at: new Date().toISOString(), type: "session.completed", data: finalTrace });
+      await this.teardownJournal(state); // ship unacked change records, then unlink
       await this.uploadTrace(state, pendingTrace);
       await this.uploadWorkspace(state, "end");
       this.sessions.delete(sessionId);
@@ -1079,18 +1207,28 @@ export class WorkspaceCollector {
     });
   }
 
+  /** Run the debounced change-snapshot upload immediately. */
+  private flushChangeSnapshot(state: SessionState): void {
+    if (state.finished || state.budgetExhausted) return;
+    if (state.changeTimer) {
+      clearTimeout(state.changeTimer);
+      state.changeTimer = null;
+    }
+    this.enqueue(state, async () => {
+      if (state.budgetExhausted || state.finished) return;
+      const signature = await workspaceSignature(state.root);
+      if (!signature || signature === state.lastSignature) return;
+      state.lastSignature = signature;
+      await this.uploadWorkspace(state, "change");
+    });
+  }
+
   private scheduleChange(state: SessionState): void {
     if (state.finished || state.budgetExhausted) return;
     if (state.changeTimer) clearTimeout(state.changeTimer);
     state.changeTimer = setTimeout(() => {
       state.changeTimer = null;
-      this.enqueue(state, async () => {
-        if (state.budgetExhausted || state.finished) return;
-        const signature = await workspaceSignature(state.root);
-        if (!signature || signature === state.lastSignature) return;
-        state.lastSignature = signature;
-        await this.uploadWorkspace(state, "change");
-      });
+      this.flushChangeSnapshot(state);
     }, this.changeDebounceMs);
     state.changeTimer.unref?.();
   }
@@ -1149,38 +1287,38 @@ export class WorkspaceCollector {
         const at = new Date().toISOString();
         const mode = file.mode & 0o777;
         const modeField = mode ? { mode: mode.toString(8) } : {};
-        const chunks = await readContentChunks(absolute, file.size);
-        const encoding = chunks[0].encoding;
-        const sliceContents = chunks.flatMap((chunk) => sliceString(chunk.content));
-        if (sliceContents.length === 1) {
-          entry = {
-            path,
-            at,
-            status: "present",
-            content: sliceContents[0],
-            ...(encoding ? { encoding } : {}),
-            ...modeField,
-          };
-        } else {
-          // Big file: one ordered record per slice so the journal stays
-          // replayable; reassembly rides the slice metadata.
-          sliceContents.forEach((content, index) => {
+        // STREAMED capture: each chunk becomes a journal record the
+        // moment it exists — a big file is never held in memory whole
+        // (issue #6). One record per <= MAX_SLICE_BYTES slice; slice
+        // metadata rides the records only when reassembly is needed.
+        let sliceCount = 0;
+        for await (const chunk of iterateContentChunks(absolute, file.size)) {
+          for (const content of sliceString(chunk.content)) {
+            sliceCount += 1;
             records.push({
               path,
               at,
               status: "present",
               content,
-              ...(encoding ? { encoding } : {}),
+              ...(chunk.encoding ? { encoding: chunk.encoding } : {}),
               ...modeField,
-              slice: {
-                index,
-                total: sliceContents.length,
-                bytes: Buffer.byteLength(content),
-                sha256: sha256Hex(content),
-              },
             });
+          }
+        }
+        if (sliceCount > 1) {
+          records.forEach((record, index) => {
+            record.slice = {
+              index,
+              total: sliceCount,
+              bytes: Buffer.byteLength(record.content ?? ""),
+              sha256: sha256Hex(record.content ?? ""),
+            };
           });
-          state.changeJournal.push(...records);
+        }
+        if (sliceCount === 1) {
+          entry = records.pop()!;
+        } else {
+          await this.appendJournalRecords(state, records);
           return;
         }
       }
@@ -1199,27 +1337,156 @@ export class WorkspaceCollector {
     // Consecutive-duplicate suppression: macOS FSEvents can deliver a
     // late event for a path that was also captured directly, and an
     // identical re-capture adds upload fat without a state change. The
-    // replay result is identical either way.
+    // replay result is identical either way. The last signature per
+    // path lives in a tiny in-memory map (path strings only — content
+    // is already on disk in the journal).
     const journalSignature = (record: ChangeJournalEntry) =>
       JSON.stringify([record.status, record.content, record.encoding, record.mode]);
     const pendingRecords = [entry, ...records];
+    const accepted: ChangeJournalEntry[] = [];
     for (const record of pendingRecords) {
-      let duplicate = false;
-      for (let i = state.changeJournal.length - 1, scanned = 0; i >= 0 && scanned < 100; i--, scanned++) {
-        const prior = state.changeJournal[i];
-        if (prior.path !== record.path) continue;
-        duplicate = journalSignature(prior) === journalSignature(record);
-        break;
+      const signature = journalSignature(record);
+      if (state.journalLastByPath.get(record.path) === signature) continue;
+      state.journalLastByPath.set(record.path, signature);
+      accepted.push(record);
+    }
+    await this.appendJournalRecords(state, accepted);
+  }
+
+  /**
+   * APPEND records to the on-disk journal (ndjson — one JSON record per
+   * line, 0600). Returns immediately when the capture hit the soft disk
+   * cap (a backend down for a very long time must stop CAPTURING loudly
+   * rather than fill the disk or RAM — the end snapshot still ships the
+   * final tree). Serialized through the per-session append chain.
+   */
+  private appendJournalRecords(state: SessionState, records: ChangeJournalEntry[]): Promise<void> {
+    if (!state.journalPath || records.length === 0) return Promise.resolve();
+    if (state.journalCaptureStopped) return Promise.resolve();
+    const unacked = state.journalAppendedBytes - state.journalAckOffset;
+    const capMb = Number(process.env.OMNIRUSH_JOURNAL_DISK_CAP_MB);
+    const capBytes = Number.isFinite(capMb) && capMb > 0 ? capMb * 1024 * 1024 : JOURNAL_DISK_CAP_BYTES;
+    if (unacked > capBytes) {
+      state.journalCaptureStopped = true;
+      this.log("warn", "OmniRush change journal hit the disk cap — change capture stopped for this session", {
+        sessionId: state.id,
+        capBytes: capBytes,
+        unackedBytes: unacked,
+      });
+      state.trace.push({
+        at: new Date().toISOString(),
+        type: "journal.capture_stopped",
+        data: { cap_bytes: capBytes, unacked_bytes: unacked },
+      });
+      return Promise.resolve();
+    }
+    const lines = records.map((record) => JSON.stringify(record) + "\n").join("");
+    const bytes = Buffer.byteLength(lines);
+    state.journalAppendTail = state.journalAppendTail
+      .catch(() => undefined)
+      .then(async () => {
+        const { appendFile, mkdir } = await import("node:fs/promises");
+        await mkdir(dirname(state.journalPath!), { recursive: true, mode: 0o700 });
+        await appendFile(state.journalPath!, lines, { encoding: "utf8", mode: 0o600 });
+        state.journalAppendedBytes += bytes;
+      });
+    return state.journalAppendTail;
+  }
+
+  /**
+   * Stream the journal's unacked records (from journalAckOffset to the
+   * upload-start EOF) as parsed entries, each tagged with its raw line
+   * bytes via a NON-enumerable property so the changes.json sidecar
+   * serialization never sees it. Records appended while the upload runs
+   * land beyond the read window and simply ship with the next snapshot.
+   */
+  private async *readJournalRecords(state: SessionState): AsyncGenerator<ChangeJournalEntry> {
+    if (!state.journalPath || state.journalAckOffset >= state.journalUploadEof) return;
+    const { createReadStream } = await import("node:fs");
+    const readline = await import("node:readline");
+    const stream = createReadStream(state.journalPath, {
+      start: state.journalAckOffset,
+      end: state.journalUploadEof! - 1,
+      encoding: "utf8",
+    });
+    const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+    for await (const line of rl) {
+      if (!line.trim()) continue;
+      try {
+        const record = JSON.parse(line) as ChangeJournalEntry;
+        Object.defineProperty(record, RAW_BYTES_FIELD, {
+          value: Buffer.byteLength(line) + 1,
+          enumerable: false,
+          configurable: true,
+        });
+        yield record;
+      } catch {
+        this.log("warn", "OmniRush journal line was unreadable — skipped", { sessionId: state.id });
       }
-      if (!duplicate) state.changeJournal.push(record);
     }
   }
 
-  /** Remove exactly the uploaded entries from the ordered journal. */
-  private acknowledgeJournal(state: SessionState, uploaded: ChangeJournalEntry[]): void {
-    if (uploaded.length === 0) return;
-    const sent = uploaded.length > 32 ? new Set(uploaded) : null;
-    state.changeJournal = state.changeJournal.filter((entry) => sent ? !sent.has(entry) : !uploaded.includes(entry));
+  /**
+   * Upload the journal's unacked records as change snapshots, then
+   * compact: everything acked is removed from the file; records appended
+   * during the upload (past journalUploadEof) survive untouched.
+   */
+  private async uploadChangeJournal(state: SessionState): Promise<void> {
+    if (!state.journalPath) return;
+    let fileSize = 0;
+    try {
+      fileSize = (await lstat(state.journalPath)).size;
+    } catch {
+      return; // no journal yet — nothing captured
+    }
+    if (fileSize <= state.journalAckOffset) return;
+    const ackStart = state.journalAckOffset;
+    state.journalUploadEof = fileSize;
+    try {
+      await this.uploadChangeParts(state, this.readJournalRecords(state), {
+        onAck: (batch) => {
+          for (const record of batch) {
+            const raw = (record as Record<symbol, unknown>)[RAW_BYTES_FIELD];
+            if (typeof raw === "number") state.journalAckOffset += raw;
+          }
+        },
+      });
+    } finally {
+      state.journalUploadEof = null;
+    }
+    if (state.journalAckOffset <= ackStart) return; // nothing acked — keep everything
+    await this.compactJournal(state, state.journalAckOffset);
+    state.journalAckOffset = 0;
+  }
+
+  /** Rewrite the journal without its first `dropBytes` bytes. */
+  private async compactJournal(state: SessionState, dropBytes: number): Promise<void> {
+    if (!state.journalPath) return;
+    const { createReadStream, createWriteStream } = await import("node:fs");
+    const { rename } = await import("node:fs/promises");
+    const tmpPath = `${state.journalPath}.compact`;
+    await new Promise<void>((resolvePromise, rejectPromise) => {
+      const input = createReadStream(state.journalPath!, { start: dropBytes, encoding: "utf8" });
+      const output = createWriteStream(tmpPath, { encoding: "utf8", mode: 0o600 });
+      input.on("error", rejectPromise);
+      output.on("error", rejectPromise);
+      output.on("finish", () => resolvePromise());
+      input.pipe(output);
+    });
+    await rename(tmpPath, state.journalPath);
+  }
+
+  /** Upload unacked journal records (best effort) and remove the file. */
+  private async teardownJournal(state: SessionState): Promise<void> {
+    try {
+      await this.uploadChangeJournal(state);
+    } catch {
+      /* best effort — the end snapshot still ships the final tree */
+    }
+    if (state.journalPath) {
+      const { rm } = await import("node:fs/promises");
+      await rm(state.journalPath, { force: true }).catch(() => undefined);
+    }
   }
 
   private enqueue(state: SessionState, operation: () => Promise<void>): void {
@@ -1234,17 +1501,10 @@ export class WorkspaceCollector {
   private async uploadWorkspace(state: SessionState, type: "start" | "change" | "end"): Promise<void> {
     if (state.budgetExhausted) return;
     if (type === "change") {
-      // Change snapshots carry ONLY the touched files from the journal —
-      // never a full re-enumeration of the tree.
-      const snapshot = [...state.changeJournal];
-      if (snapshot.length === 0) {
-        state.lastSignature = await workspaceSignature(state.root);
-        return;
-      }
-      const uploaded = await this.uploadChangeParts(state, snapshot);
-      // Acknowledge only after every part of the snapshot succeeded; on a
-      // mid-budget exhaustion, acknowledge only what actually shipped.
-      this.acknowledgeJournal(state, state.budgetExhausted ? uploaded.flat() : snapshot);
+      // Change snapshots carry ONLY the touched files — streamed from
+      // the on-disk journal, acked per part, then compacted. Never a
+      // full re-enumeration of the tree, never held in memory.
+      await this.uploadChangeJournal(state);
     } else {
       await this.uploadTreeParts(state, type);
     }
@@ -1302,7 +1562,7 @@ export class WorkspaceCollector {
   private async *iterTreeFiles(state: SessionState, paths: string[]): AsyncGenerator<CollectorFile> {
     for (const path of paths) {
       if (state.budgetExhausted) return;
-      for (const entry of await readWorkspaceEntries(state.root, path)) {
+      for await (const entry of iterateWorkspaceEntries(state.root, path)) {
         yield entry;
       }
     }
@@ -1315,7 +1575,11 @@ export class WorkspaceCollector {
    * records — replaying start + sidecars in sequence order reproduces the
    * end tree.
    */
-  private async uploadChangeParts(state: SessionState, snapshot: ChangeJournalEntry[]): Promise<ChangeJournalEntry[][]> {
+  private async uploadChangeParts(
+    state: SessionState,
+    units: Iterable<ChangeJournalEntry> | AsyncIterable<ChangeJournalEntry>,
+    opts: { onAck?: (batch: ChangeJournalEntry[]) => void } = {},
+  ): Promise<ChangeJournalEntry[][]> {
     const uploadedSlices: ChangeJournalEntry[][] = [];
     const filesForSlice = (entries: ChangeJournalEntry[]): CollectorFile[] => {
       const lastByPath = new Map<string, CollectorFile>();
@@ -1372,7 +1636,7 @@ export class WorkspaceCollector {
       return files;
     };
     const bySequence = () => ({ id: state.id, segment: state.segment, resumed: state.resumed, sequence: state.sequence });
-    await this.packAndUpload(state, "change", snapshot, {
+    await this.packAndUpload(state, "change", units, {
       sizeOf: (entry) =>
         Buffer.byteLength(entry.content ?? "") + entry.path.length + 96,
       measure: async (batch) => {
@@ -1382,7 +1646,10 @@ export class WorkspaceCollector {
       },
       upload: async (measured, batch) => {
         const ok = await this.uploadEnvelope(state, "change", measured.files, measured);
-        if (ok) uploadedSlices.push(batch);
+        if (ok) {
+          uploadedSlices.push(batch);
+          opts.onAck?.(batch);
+        }
         return ok;
       },
       drop: (entry) => {
@@ -1487,7 +1754,12 @@ export class WorkspaceCollector {
 
   /** Chunked trace upload: event subsets per part, ordered by sequence. */
   private async uploadTrace(state: SessionState, traceEvents: TraceEvent[]): Promise<void> {
-    if (state.budgetExhausted || traceEvents.length === 0) return;
+    if (state.budgetExhausted || traceEvents.length === 0) {
+      // No lifecycle events this round — transcript artifacts may still
+      // be due (new or changed session files).
+      await this.uploadSessionTranscripts(state);
+      return;
+    }
     const remaining = this.sessionBudgetBytes - state.sentBytes;
     if (remaining <= 1024) {
       this.exhaustSessionBudget(state);
@@ -1538,6 +1810,128 @@ export class WorkspaceCollector {
         });
       },
     }, { checkBytes: PART_SIDECAR_PLAIN_MAX, flushPlainBytes: PART_SIDECAR_PLAIN_MAX });
+    // Full-conversation transcript artifacts (trace format spec) ride
+    // the same trace uploads — lifecycle events stay in trace.json, the
+    // transcripts land as __agent__/traces/<session>.jsonl.
+    await this.uploadSessionTranscripts(state);
+  }
+
+  /**
+   * The pi agent writes the FULL session transcript as JSONL under
+   * <agentDir>/sessions/<cwd-slug>/<ts>_<session-id>.jsonl — one file
+   * per run, SUBAGENTS as their own files in the same dir. Every file
+   * (new or changed since the last capture) becomes its own trace
+   * artifact (header + full conversation, redacted) uploaded as
+   * __agent__/traces/<session>.jsonl and linked to this session via
+   * parent_session_id. Detection state rides the persistent ledger.
+   */
+  private async uploadSessionTranscripts(state: SessionState): Promise<void> {
+    if (!this.agentDir || !state.id) return;
+    const { readdir, stat } = await import("node:fs/promises");
+    const sessionsRoot = join(this.agentDir, "sessions");
+    let sessionDir: string | null = null;
+    try {
+      for (const entry of await readdir(sessionsRoot, { withFileTypes: true })) {
+        if (!entry.isDirectory()) continue;
+        const dir = join(sessionsRoot, entry.name);
+        const files = await readdir(dir).catch(() => [] as string[]);
+        if (files.some((f) => f.endsWith(`_${state.id}.jsonl`))) {
+          sessionDir = dir;
+          break;
+        }
+      }
+    } catch {
+      return; // no sessions dir — nothing to capture
+    }
+    if (!sessionDir) return;
+
+    const artifacts: Array<{ path: string; content: string; fileKey: string; size: number; mtimeMs: number }> = [];
+    let files: string[] = [];
+    try {
+      files = (await readdir(sessionDir)).filter((f) => f.endsWith(".jsonl")).sort();
+    } catch {
+      return;
+    }
+    for (const file of files) {
+      const filePath = join(sessionDir, file);
+      let size = 0;
+      let mtimeMs = 0;
+      try {
+        const stats = await stat(filePath);
+        size = stats.size;
+        mtimeMs = stats.mtimeMs;
+      } catch {
+        continue;
+      }
+      const known = state.transcripts[filePath];
+      if (known && known.size === size && known.mtimeMs === mtimeMs) continue;
+      let text: string;
+      try {
+        const { readFile } = await import("node:fs/promises");
+        text = await readFile(filePath, "utf8");
+      } catch {
+        continue;
+      }
+      const parsed = parsePiSession(text);
+      const artifactSessionId = parsed.sessionId ?? file.replace(/^\d{4}-\d{2}-\d{2}T[\d-]+Z_/, "").replace(/\.jsonl$/, "");
+      if (!artifactSessionId || parsed.messageRecords.length === 0) {
+        // Still mark empty/HEAD-only files seen so they are not retried.
+        state.transcripts[filePath] = { size, mtimeMs };
+        continue;
+      }
+      const identity = await this.resolveIdentity();
+      const artifact = buildTraceArtifact({
+        sessionId: artifactSessionId,
+        parentSessionId: state.id,
+        userId: identity?.userId ?? null,
+        clientId: this.clientId ?? null,
+        agentId: "omnirush-cli",
+        parsed,
+        redact: (value) => redactCollectorText(value).text,
+      });
+      artifacts.push({
+        path: `${AGENT_DIR}/traces/${artifactSessionId}.jsonl`,
+        content: JSON.stringify(artifact),
+        fileKey: filePath,
+        size,
+        mtimeMs,
+      });
+      state.transcripts[filePath] = { size, mtimeMs };
+    }
+    if (artifacts.length === 0) {
+      await this.persistSession(state).catch(() => undefined);
+      return;
+    }
+    const bySequence = () => ({ id: state.id, segment: state.segment, resumed: state.resumed, sequence: state.sequence });
+    await this.packAndUpload(state, "trace", artifacts, {
+      sizeOf: (artifact) => Buffer.byteLength(artifact.content) + artifact.path.length + 64,
+      measure: async (batch) => {
+        const payload = buildEnvelopePayload(bySequence(), "trace", batch);
+        return { files: batch, payload, compressed: await compressZstd(payload) };
+      },
+      upload: async (measured) => this.uploadEnvelope(state, "trace", measured.files, measured),
+      drop: (artifact) => {
+        this.log("warn", "OmniRush transcript artifact exceeds the upload limits; skipped", {
+          sessionId: state.id,
+          path: artifact.path,
+          bytes: Buffer.byteLength(artifact.content),
+        });
+      },
+    }, { checkBytes: PART_SIDECAR_PLAIN_MAX, flushPlainBytes: PART_SIDECAR_PLAIN_MAX });
+    await this.persistSession(state).catch(() => undefined);
+  }
+
+  /** Best-effort /device/me identity, fetched once per collector. */
+  private async resolveIdentity(): Promise<{ userId: string | null } | null> {
+    if (this.identityTried) return this.identity;
+    this.identityTried = true;
+    if (!this.identityProvider) return null;
+    try {
+      this.identity = await this.identityProvider();
+    } catch {
+      this.identity = null;
+    }
+    return this.identity;
   }
 
   /**
@@ -1579,14 +1973,46 @@ export class WorkspaceCollector {
           body: compressed.buffer.slice(compressed.byteOffset, compressed.byteOffset + compressed.byteLength) as ArrayBuffer,
           signal: AbortSignal.timeout(120_000),
         });
-    let response = await send();
-    if (response.status === 401 && this.refresh) {
-      // Broker pattern: single-flight refresh, retry exactly once.
-      const rotated = await this.refresh(this.token).catch(() => null);
-      if (rotated) {
-        this.token = rotated;
-        response = await send();
-      }
+    // Shared retry layer: 429/5xx/network ride out backend deploys with
+    // exponential backoff + jitter. 401 stays a protocol answer handled
+    // inline (single-flight refresh, retry exactly once).
+    const outcome = await withRetries(
+      async () => {
+        let response = await send();
+        if (response.status === 401 && this.refresh) {
+          // Broker pattern: single-flight refresh, retry exactly once.
+          const rotated = await this.refresh(this.token).catch(() => null);
+          if (rotated) {
+            this.token = rotated;
+            response = await send();
+          }
+        }
+        return {
+          response,
+          retryAfterSec: Number(response?.headers?.get?.("retry-after")) || undefined,
+        };
+      },
+      {
+        attempts: retryAttempts(),
+        isRetryable: ({ response }) => isRetryableStatus(response?.status),
+        onRetry: ({ attempt, attempts, delayMs }) => {
+          this.log("warn", "OmniRush upload unavailable — retrying", {
+            sessionId: state.id,
+            snapshotType,
+            attempt: `${attempt + 1}/${attempts}`,
+            delayMs,
+          });
+        },
+      },
+    );
+    const response = outcome.result?.response;
+    if (!response) {
+      // Every attempt failed at the network layer.
+      throw new Error(
+        `collector upload failed: network error after ${outcome.attempts} attempt${outcome.attempts === 1 ? "" : "s"}` +
+          ` — ${outcome.error?.message ?? "unknown"} [ref ${outcome.ref}]` +
+          ` (session ${state.id}, ${snapshotType}, sequence ${state.sequence + 1})`,
+      );
     }
     if (!response.ok) {
       // Diagnosable from a tester's screenshot alone: status, context,
@@ -1596,6 +2022,7 @@ export class WorkspaceCollector {
       throw new Error(
         `collector upload failed: HTTP ${response.status}` +
           `${snippet ? ` — ${snippet}` : ""}` +
+          ` [ref ${outcome.ref}]` +
           ` (session ${state.id}, ${snapshotType}, sequence ${state.sequence + 1})`,
       );
     }