@vellumai/assistant 0.6.4 → 0.6.6

package/src/runtime/migrations/vbundle-streaming-validator.ts

@@ -0,0 +1,244 @@
+/**
+ * Streaming validation primitives for `.vbundle` archives.
+ *
+ * The non-streaming `validateVBundle` decompresses the entire archive into
+ * memory and walks the tar buffer to compute per-file SHA-256s. That is fine
+ * for small bundles but peaks at 2x the decompressed size in RAM — an 8 GB
+ * bundle OOMs a 3 GB pod.
+ *
+ * This module lets a caller validate a bundle while streaming:
+ * - `readAndValidateManifest` consumes the first tar entry (which must be
+ *   `manifest.json`), validates the schema, and verifies the self-referencing
+ *   `manifest_sha256` against the canonicalized JSON.
+ * - `createHashVerifier` returns a passthrough `Transform` that hashes bytes
+ *   flowing through it and errors the pipeline if the final digest or byte
+ *   count does not match the expected values from the manifest.
+ *
+ * Together, these let a consumer pipe every subsequent tar entry through a
+ * hash verifier before writing it to disk, without ever buffering the full
+ * bundle.
+ */
+
+import { createHash } from "node:crypto";
+import { Transform, type TransformCallback } from "node:stream";
+
+import type { StreamedTarEntry } from "./vbundle-tar-stream.js";
+import {
+  computeManifestSha256,
+  ManifestSchema,
+  type ManifestType,
+} from "./vbundle-validator.js";
+
+// ---------------------------------------------------------------------------
+// Public types
+// ---------------------------------------------------------------------------
+
+export interface ManifestReadResult {
+  manifest: ManifestType;
+  /** Fast lookup from archive path -> expected sha256 + size (from manifest.files). */
+  expected: Map<string, { sha256: string; size: number }>;
+}
+
+/**
+ * All failure modes produced by this module. Every throw/error includes a
+ * stable `code` string so callers can branch on the failure kind without
+ * string-matching the message.
+ */
+export class StreamingValidationError extends Error {
+  public readonly code: string;
+  public readonly archivePath?: string;
+
+  constructor(code: string, message: string, archivePath?: string) {
+    super(message);
+    this.name = "StreamingValidationError";
+    this.code = code;
+    if (archivePath !== undefined) {
+      this.archivePath = archivePath;
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Manifest validation
+// ---------------------------------------------------------------------------
+
+// Manifests are metadata only — typically tens to hundreds of KB even for
+// huge bundles. A 1 MiB cap is comfortably above realistic sizes and
+// protects against a malicious archive whose "manifest" is actually a
+// multi-GB stream intended to OOM the validator.
+const MANIFEST_MAX_BYTES = 1 * 1024 * 1024;
+
+/**
+ * Drain the first tar entry — which MUST be `manifest.json` — and run the
+ * full manifest-level validation pipeline:
+ *   1. Entry name check.
+ *   2. Size cap (1 MiB).
+ *   3. JSON parse.
+ *   4. Zod schema validation.
+ *   5. Self-referencing `manifest_sha256` verification against the
+ *      canonicalized JSON (minus that field).
+ *
+ * On success, returns the parsed manifest plus a `Map` keyed by archive
+ * path that callers consult as each subsequent entry streams past.
+ *
+ * On failure, throws a `StreamingValidationError` with a distinct `code`
+ * for every failure mode.
+ */
+export async function readAndValidateManifest(
+  first: StreamedTarEntry,
+): Promise<ManifestReadResult> {
+  if (first.header.name !== "manifest.json") {
+    // Drain the body so the underlying tar extractor isn't left dangling
+    // on backpressure before the caller reports the error.
+    first.body.resume();
+    throw new StreamingValidationError(
+      "manifest_not_first",
+      `Expected manifest.json as the first tar entry, got "${first.header.name}"`,
+    );
+  }
+
+  // Drain the entry body into a Buffer, enforcing the size cap as we go.
+  // The moment we cross the cap we destroy the entry stream — this signals
+  // the tar extractor (and therefore gunzip + upstream source) to abort,
+  // so a malicious archive whose "manifest" is a multi-GB decompressed
+  // stream can't force us to read through all of it before rejecting.
+  const chunks: Buffer[] = [];
+  let total = 0;
+  for await (const chunk of first.body) {
+    const buf = chunk instanceof Buffer ? chunk : Buffer.from(chunk);
+    total += buf.length;
+    if (total > MANIFEST_MAX_BYTES) {
+      first.body.destroy();
+      throw new StreamingValidationError(
+        "manifest_too_large",
+        `manifest.json exceeds ${MANIFEST_MAX_BYTES} byte limit (read ${total} bytes before aborting)`,
+      );
+    }
+    chunks.push(buf);
+  }
+
+  const bodyBuf = Buffer.concat(chunks, total);
+
+  let manifestRaw: unknown;
+  try {
+    manifestRaw = JSON.parse(bodyBuf.toString("utf8"));
+  } catch (err) {
+    throw new StreamingValidationError(
+      "manifest_malformed",
+      `manifest.json is not valid JSON: ${
+        err instanceof Error ? err.message : String(err)
+      }`,
+    );
+  }
+
+  const parseResult = ManifestSchema.safeParse(manifestRaw);
+  if (!parseResult.success) {
+    const issues = parseResult.error.issues
+      .map((i) => `${i.path.join(".") || "<root>"}: ${i.message}`)
+      .join("; ");
+    throw new StreamingValidationError(
+      "manifest_schema",
+      `manifest.json failed schema validation: ${issues}`,
+    );
+  }
+
+  const manifest = parseResult.data;
+
+  // Recompute the self-referencing checksum using the exact canonicalization
+  // that vbundle-validator.ts uses. Any drift here would silently reject
+  // valid bundles produced by buildVBundle.
+  const computed = computeManifestSha256(manifestRaw);
+  if (computed !== manifest.manifest_sha256) {
+    throw new StreamingValidationError(
+      "manifest_sha256",
+      `Manifest checksum mismatch: expected ${manifest.manifest_sha256}, computed ${computed}`,
+    );
+  }
+
+  const expected = new Map<string, { sha256: string; size: number }>();
+  for (const file of manifest.files) {
+    if (expected.has(file.path)) {
+      throw new StreamingValidationError(
+        "manifest_duplicate_path",
+        `Manifest contains duplicate entry for path: ${file.path}`,
+      );
+    }
+    expected.set(file.path, { sha256: file.sha256, size: file.size });
+  }
+
+  return { manifest, expected };
+}
+
+// ---------------------------------------------------------------------------
+// Per-entry hash + size verifier
+// ---------------------------------------------------------------------------
+
+/**
+ * Create a passthrough `Transform` that:
+ *   - forwards every chunk unchanged (identity transform for correct input),
+ *   - incrementally SHA-256s the byte stream,
+ *   - on `_flush`, errors the pipeline if the final digest or total byte
+ *     count differs from `expected`.
+ *
+ * Errors are emitted as `StreamingValidationError` with `code` set to
+ * `"entry_hash"` or `"entry_size"` and `archivePath` populated so callers
+ * can surface which file failed.
+ *
+ * Consumers should pipe the entry body through this transform before
+ * writing to disk — that way a bad payload is caught before the byte
+ * reaches storage rather than after a whole 8 GB write completes.
+ */
+export function createHashVerifier(expected: {
+  sha256: string;
+  size: number;
+  archivePath: string;
+}): Transform {
+  const hash = createHash("sha256");
+  let bytes = 0;
+
+  return new Transform({
+    transform(
+      chunk: Buffer | string,
+      encoding: BufferEncoding,
+      callback: TransformCallback,
+    ) {
+      try {
+        const buf =
+          typeof chunk === "string" ? Buffer.from(chunk, encoding) : chunk;
+        hash.update(buf);
+        bytes += buf.length;
+        callback(null, buf);
+      } catch (err) {
+        callback(err instanceof Error ? err : new Error(String(err)));
+      }
+    },
+    flush(callback: TransformCallback) {
+      // Size check first — a wrong size is a sharper signal than a hash
+      // collision, and a truncated payload frequently triggers both.
+      if (bytes !== expected.size) {
+        callback(
+          new StreamingValidationError(
+            "entry_size",
+            `Size mismatch for ${expected.archivePath}: expected ${expected.size} bytes, got ${bytes}`,
+            expected.archivePath,
+          ),
+        );
+        return;
+      }
+
+      const digest = hash.digest("hex");
+      if (digest !== expected.sha256) {
+        callback(
+          new StreamingValidationError(
+            "entry_hash",
+            `Checksum mismatch for ${expected.archivePath}: expected ${expected.sha256}, computed ${digest}`,
+            expected.archivePath,
+          ),
+        );
+        return;
+      }
+
+      callback();
+    },
+  });
+}

package/src/runtime/migrations/vbundle-tar-stream.ts

@@ -0,0 +1,217 @@
+/**
+ * Streaming tar reader for `.vbundle` archives.
+ *
+ * A `.vbundle` is a gzip-compressed tar. This module gunzips the incoming
+ * `Readable`, pipes it through `tar-stream`'s push-style extractor, and
+ * adapts the `(header, stream, next)` event into a consumer-friendly async
+ * generator that yields one entry at a time.
+ *
+ * Memory invariant: each entry's body is surfaced as a `Readable`, never a
+ * pre-buffered blob. Callers MUST fully consume (or explicitly `resume()`)
+ * each body stream before advancing the outer generator — otherwise the
+ * underlying tar extractor will stall waiting on backpressure.
+ */
+
+import type { Readable } from "node:stream";
+import { createGunzip } from "node:zlib";
+
+import { extract as tarExtract } from "tar-stream";
+
+// ---------------------------------------------------------------------------
+// Public types
+// ---------------------------------------------------------------------------
+
+/** Subset of tar-stream's header surface that vbundle consumers care about. */
+export interface StreamedTarHeader {
+  name: string;
+  size: number;
+  type: "file" | "directory" | "pax-header" | "other";
+}
+
+export interface StreamedTarEntry {
+  header: StreamedTarHeader;
+  /**
+   * The entry body. Must be fully consumed (drained via for-await, piped,
+   * or explicitly `.resume()`'d) before the generator is advanced again.
+   */
+  body: Readable;
+}
+
+// ---------------------------------------------------------------------------
+// Implementation
+// ---------------------------------------------------------------------------
+
+interface PendingEntry {
+  header: StreamedTarHeader;
+  body: Readable;
+  next: (err?: Error | null) => void;
+}
+
+function normalizeHeaderType(
+  type: string | undefined,
+): StreamedTarHeader["type"] {
+  switch (type) {
+    case "file":
+      return "file";
+    case "directory":
+      return "directory";
+    case "pax-header":
+      return "pax-header";
+    default:
+      return "other";
+  }
+}
+
+/**
+ * Stream a `.vbundle` archive as an async sequence of tar entries.
+ *
+ * Errors from the upstream source, gunzip, or tar-stream extractor are
+ * surfaced by throwing from the generator.
+ *
+ * Early termination (caller `break`s or the for-await loop throws) destroys
+ * the upstream source and the tar extractor so that sockets and file
+ * descriptors are released promptly.
+ */
+export async function* parseVBundleStream(
+  source: Readable,
+): AsyncGenerator<StreamedTarEntry, void, void> {
+  const gunzip = createGunzip();
+  const extractor = tarExtract();
+
+  // Terminal error captured from any stage in the pipeline. Consumed by the
+  // generator's internal pump; re-thrown on the next `yield`.
+  let pipelineError: Error | null = null;
+  let finished = false;
+
+  // Single-slot mailbox: the extractor pushes one entry here, and the
+  // generator consumes it. We gate the push-style API by holding the tar
+  // extractor's `next` callback until the caller is done with the body.
+  let waiter: {
+    resolve: (entry: PendingEntry | null) => void;
+    reject: (err: Error) => void;
+  } | null = null;
+  let pending: PendingEntry | null = null;
+
+  function pushEntry(entry: PendingEntry): void {
+    if (waiter) {
+      const w = waiter;
+      waiter = null;
+      w.resolve(entry);
+    } else {
+      pending = entry;
+    }
+  }
+
+  function pushError(err: Error): void {
+    if (pipelineError) return;
+    pipelineError = err;
+    if (waiter) {
+      const w = waiter;
+      waiter = null;
+      w.reject(err);
+    }
+  }
+
+  function pushFinish(): void {
+    finished = true;
+    if (waiter) {
+      const w = waiter;
+      waiter = null;
+      w.resolve(null);
+    }
+  }
+
+  extractor.on("entry", (header, body, next) => {
+    // Avoid unhandled "error" on body streams destroyed mid-flight; the
+    // extractor itself propagates the real error to its "error" listener.
+    body.on("error", () => {});
+    pushEntry({
+      header: {
+        name: header.name,
+        size: header.size,
+        type: normalizeHeaderType(header.type),
+      },
+      body,
+      next,
+    });
+  });
+
+  extractor.on("error", (err: Error) => {
+    pushError(err);
+  });
+
+  extractor.on("finish", () => {
+    pushFinish();
+  });
+
+  gunzip.on("error", (err: Error) => {
+    pushError(err);
+    extractor.destroy(err);
+  });
+
+  source.on("error", (err: Error) => {
+    pushError(err);
+    gunzip.destroy(err);
+    extractor.destroy(err);
+  });
+
+  // Kick off the pipeline. We wire stages with `.pipe(..., { end: true })`
+  // so that clean EOF from the source triggers `finish` on the extractor.
+  source.pipe(gunzip).pipe(extractor);
+
+  function nextEntry(): Promise<PendingEntry | null> {
+    if (pipelineError) return Promise.reject(pipelineError);
+    if (pending) {
+      const p = pending;
+      pending = null;
+      return Promise.resolve(p);
+    }
+    if (finished) return Promise.resolve(null);
+    return new Promise<PendingEntry | null>((resolve, reject) => {
+      waiter = { resolve, reject };
+    });
+  }
+
+  try {
+    while (true) {
+      const entry = await nextEntry();
+      if (entry === null) return;
+
+      let advanced = false;
+      const advance = (err?: Error | null): void => {
+        if (advanced) return;
+        advanced = true;
+        entry.next(err ?? null);
+      };
+
+      // When the consumer finishes (or abandons) the body, release the tar
+      // extractor so the next entry can flow.
+      entry.body.once("end", () => advance());
+      entry.body.once("close", () => advance());
+      entry.body.once("error", (err: Error) => advance(err));
+
+      try {
+        yield { header: entry.header, body: entry.body };
+      } catch (err) {
+        // Caller threw (or re-threw) inside the for-await loop. Propagate
+        // after cleanup in the finally below.
+        advance(err instanceof Error ? err : new Error(String(err)));
+        throw err;
+      }
+
+      // If the consumer neither consumed nor destroyed the body, drain it
+      // for them so the extractor can advance.
+      if (!advanced) {
+        entry.body.resume();
+      }
+    }
+  } finally {
+    // Early termination (break, throw, or natural completion): tear down the
+    // pipeline so we don't leak the socket/file descriptor underneath.
+    if (!finished || pipelineError) {
+      source.destroy();
+      gunzip.destroy();
+      extractor.destroy();
+    }
+  }
+}

package/src/runtime/migrations/vbundle-validator.ts

@@ -29,7 +29,7 @@ const ManifestFileEntry = z.object({
   size: z.number().int().nonnegative(),
 });
 
-const ManifestSchema = z.object({
+export const ManifestSchema = z.object({
   schema_version: z.string(),
   created_at: z.string(),
   source: z.string().optional(),
@@ -176,7 +176,7 @@ function sha256Hex(data: Uint8Array | string): string {
  * Canonicalize a JSON object by sorting keys recursively, then SHA-256 hash it.
  * This matches the platform's canonicalization approach.
  */
-function canonicalizeJson(obj: unknown): string {
+export function canonicalizeJson(obj: unknown): string {
   return JSON.stringify(obj, (_key, value) => {
     if (value && typeof value === "object" && !Array.isArray(value)) {
       const sorted: Record<string, unknown> = {};
@@ -189,6 +189,18 @@ function canonicalizeJson(obj: unknown): string {
   });
 }
 
+/**
+ * Recompute the `manifest_sha256` field for a manifest object. Strips the
+ * `manifest_sha256` property, canonicalizes the remaining JSON, and returns
+ * the SHA-256 hex digest. Centralized here so the streaming validator and
+ * the in-memory validator agree on the exact canonicalization.
+ */
+export function computeManifestSha256(manifest: unknown): string {
+  const copy = { ...(manifest as Record<string, unknown>) };
+  delete copy.manifest_sha256;
+  return sha256Hex(canonicalizeJson(copy));
+}
+
 // ---------------------------------------------------------------------------
 // Core validation
 // ---------------------------------------------------------------------------
@@ -300,10 +312,7 @@ export function validateVBundle(data: Uint8Array): VBundleValidationResult {
   // Step 5: Verify manifest checksum
   // The manifest_sha256 field is the SHA-256 of the canonicalized JSON
   // with the manifest_sha256 field itself excluded.
-  const manifestForChecksum = { ...(manifestRaw as Record<string, unknown>) };
-  delete manifestForChecksum.manifest_sha256;
-  const canonicalized = canonicalizeJson(manifestForChecksum);
-  const computedManifestSha256 = sha256Hex(canonicalized);
+  const computedManifestSha256 = computeManifestSha256(manifestRaw);
 
   if (computedManifestSha256 !== manifest.manifest_sha256) {
     errors.push({

package/src/runtime/routes/__tests__/home-feed-routes.test.ts

@@ -33,7 +33,10 @@ const addedMessages: Array<{
 }> = [];
 let createConversationShouldThrow = false;
 
+const realConversationCrud =
+  await import("../../../memory/conversation-crud.js");
 mock.module("../../../memory/conversation-crud.js", () => ({
+  ...realConversationCrud,
   createConversation: (opts: unknown) => {
     if (createConversationShouldThrow) {
       throw new Error("synthetic createConversation failure");
@@ -51,10 +54,37 @@ mock.module("../../../memory/conversation-crud.js", () => ({
     addedMessages.push({ conversationId, role, content });
     return { id: `msg-${addedMessages.length}` };
   },
+  // Stub the message reader surface transitively required by other
+  // modules that route through conversation-crud. The home-feed route
+  // paths don't consult it directly, but Bun's ESM mock needs the named
+  // export to exist so transitive `import { getMessages }` resolves.
+  getMessages: () => [],
+  getMessagesPaginated: () => ({ messages: [], hasMore: false }),
+  getMessageById: () => null,
+}));
+
+// Stub the rollup producer so the on-visit refresh trigger inside
+// handleGetHomeFeed doesn't try to resolve a real provider or touch
+// relationship state. The route calls the producer fire-and-forget,
+// so tests observe the trigger via call-count on this spy rather
+// than awaiting a return value.
+//
+// Default skip reason is `empty_items` — a real LLM attempt that
+// returned nothing to consolidate. Using a real-run skip means the
+// debounce gate holds firm in the default case (matching production
+// semantics); individual tests override with `no_provider` etc. to
+// exercise the rollback path.
+const rollupProducerSpy = mock<() => Promise<unknown>>(async () => ({
+  wroteCount: 0,
+  skippedReason: "empty_items",
+}));
+mock.module("../../../home/rollup-producer.js", () => ({
+  runRollupProducer: rollupProducerSpy,
 }));
 
 // Dynamic imports so module mocks are wired before evaluation.
 const {
+  __resetOnVisitRefreshStateForTests,
   computeGreeting,
   formatRelativeTime,
   handleGetHomeFeed,
@@ -125,6 +155,8 @@ beforeEach(() => {
   origWorkspaceDir = process.env.VELLUM_WORKSPACE_DIR;
   process.env.VELLUM_WORKSPACE_DIR = workspaceDir;
   publishSpy.mockClear();
+  rollupProducerSpy.mockClear();
+  __resetOnVisitRefreshStateForTests();
   createdConversations.length = 0;
   addedMessages.length = 0;
   createConversationShouldThrow = false;
@@ -322,6 +354,85 @@ describe("handleGetHomeFeed", () => {
     expect(body.contextBanner.timeAwayLabel).toBe("2 hours ago");
   });
 
+  test("fires the rollup producer fire-and-forget on the first GET", async () => {
+    const res = await handleGetHomeFeed(
+      new Request("http://localhost/v1/home/feed?timeAwaySeconds=0"),
+    );
+    expect(res.status).toBe(200);
+    // Yield a microtask so the fire-and-forget call reaches its
+    // first await point.
+    await Promise.resolve();
+    expect(rollupProducerSpy).toHaveBeenCalledTimes(1);
+  });
+
+  test("does NOT refire the rollup when the debounce window has not elapsed", async () => {
+    await handleGetHomeFeed(
+      new Request("http://localhost/v1/home/feed?timeAwaySeconds=0"),
+    );
+    await Promise.resolve();
+    expect(rollupProducerSpy).toHaveBeenCalledTimes(1);
+
+    // A second GET milliseconds later should NOT re-trigger the
+    // rollup — the 10-minute debounce prevents aggressive pollers or
+    // multiple panels from firing repeat refreshes.
+    await handleGetHomeFeed(
+      new Request("http://localhost/v1/home/feed?timeAwaySeconds=0"),
+    );
+    await handleGetHomeFeed(
+      new Request("http://localhost/v1/home/feed?timeAwaySeconds=0"),
+    );
+    await Promise.resolve();
+    expect(rollupProducerSpy).toHaveBeenCalledTimes(1);
+  });
+
+  test("debounce is rolled back when the producer skips before the LLM call", async () => {
+    // Simulate the daemon-boot race: the first GET fires the
+    // producer but the provider registry isn't ready yet, so the
+    // producer short-circuits with `no_provider`. A second GET a
+    // moment later must still be allowed to fire — otherwise Home
+    // stays stale for the full 10-minute debounce window while the
+    // user is actively trying to refresh.
+    rollupProducerSpy.mockImplementationOnce(async () => ({
+      wroteCount: 0,
+      skippedReason: "no_provider",
+    }));
+
+    await handleGetHomeFeed(
+      new Request("http://localhost/v1/home/feed?timeAwaySeconds=0"),
+    );
+    // Let the fire-and-forget `.then()` that performs the rollback
+    // run before we issue the second GET. Two microtask ticks
+    // because the chain is runRollupProducer -> .then handler.
+    await Promise.resolve();
+    await Promise.resolve();
+    expect(rollupProducerSpy).toHaveBeenCalledTimes(1);
+
+    // Second GET — producer is now ready. Gate must have been
+    // rolled back so this GET re-fires the producer.
+    await handleGetHomeFeed(
+      new Request("http://localhost/v1/home/feed?timeAwaySeconds=0"),
+    );
+    await Promise.resolve();
+    expect(rollupProducerSpy).toHaveBeenCalledTimes(2);
+  });
+
+  test("rollup producer failure does not turn the GET into an error", async () => {
+    // Even if the rollup producer rejects, the GET must still return
+    // 200 with the cached feed — the refresh is fire-and-forget.
+    rollupProducerSpy.mockImplementationOnce(async () => {
+      throw new Error("synthetic rollup failure");
+    });
+
+    const res = await handleGetHomeFeed(
+      new Request("http://localhost/v1/home/feed?timeAwaySeconds=0"),
+    );
+    expect(res.status).toBe(200);
+    // Drain the rejected promise so it doesn't leak into the next
+    // test as an unhandled rejection.
+    await Promise.resolve();
+    await Promise.resolve();
+  });
+
   test("newCount counts only status=new after filtering", async () => {
     writeFeedFile([
       makeItem({ id: "a", status: "new" }),