@vellumai/assistant 0.6.3 → 0.6.5

package/src/runtime/migrations/vbundle-streaming-validator.ts

@@ -0,0 +1,244 @@
+/**
+ * Streaming validation primitives for `.vbundle` archives.
+ *
+ * The non-streaming `validateVBundle` decompresses the entire archive into
+ * memory and walks the tar buffer to compute per-file SHA-256s. That is fine
+ * for small bundles but peaks at 2x the decompressed size in RAM — an 8 GB
+ * bundle OOMs a 3 GB pod.
+ *
+ * This module lets a caller validate a bundle while streaming:
+ * - `readAndValidateManifest` consumes the first tar entry (which must be
+ *   `manifest.json`), validates the schema, and verifies the self-referencing
+ *   `manifest_sha256` against the canonicalized JSON.
+ * - `createHashVerifier` returns a passthrough `Transform` that hashes bytes
+ *   flowing through it and errors the pipeline if the final digest or byte
+ *   count does not match the expected values from the manifest.
+ *
+ * Together, these let a consumer pipe every subsequent tar entry through a
+ * hash verifier before writing it to disk, without ever buffering the full
+ * bundle.
+ */
+
+import { createHash } from "node:crypto";
+import { Transform, type TransformCallback } from "node:stream";
+
+import type { StreamedTarEntry } from "./vbundle-tar-stream.js";
+import {
+  computeManifestSha256,
+  ManifestSchema,
+  type ManifestType,
+} from "./vbundle-validator.js";
+
+// ---------------------------------------------------------------------------
+// Public types
+// ---------------------------------------------------------------------------
+
+export interface ManifestReadResult {
+  manifest: ManifestType;
+  /** Fast lookup from archive path -> expected sha256 + size (from manifest.files). */
+  expected: Map<string, { sha256: string; size: number }>;
+}
+
+/**
+ * All failure modes produced by this module. Every throw/error includes a
+ * stable `code` string so callers can branch on the failure kind without
+ * string-matching the message.
+ */
+export class StreamingValidationError extends Error {
+  public readonly code: string;
+  public readonly archivePath?: string;
+
+  constructor(code: string, message: string, archivePath?: string) {
+    super(message);
+    this.name = "StreamingValidationError";
+    this.code = code;
+    if (archivePath !== undefined) {
+      this.archivePath = archivePath;
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Manifest validation
+// ---------------------------------------------------------------------------
+
+// Manifests are metadata only — typically tens to hundreds of KB even for
+// huge bundles. A 1 MiB cap is comfortably above realistic sizes and
+// protects against a malicious archive whose "manifest" is actually a
+// multi-GB stream intended to OOM the validator.
+const MANIFEST_MAX_BYTES = 1 * 1024 * 1024;
+
+/**
+ * Drain the first tar entry — which MUST be `manifest.json` — and run the
+ * full manifest-level validation pipeline:
+ *   1. Entry name check.
+ *   2. Size cap (1 MiB).
+ *   3. JSON parse.
+ *   4. Zod schema validation.
+ *   5. Self-referencing `manifest_sha256` verification against the
+ *      canonicalized JSON (minus that field).
+ *
+ * On success, returns the parsed manifest plus a `Map` keyed by archive
+ * path that callers consult as each subsequent entry streams past.
+ *
+ * On failure, throws a `StreamingValidationError` with a distinct `code`
+ * for every failure mode.
+ */
+export async function readAndValidateManifest(
+  first: StreamedTarEntry,
+): Promise<ManifestReadResult> {
+  if (first.header.name !== "manifest.json") {
+    // Drain the body so the underlying tar extractor isn't left dangling
+    // on backpressure before the caller reports the error.
+    first.body.resume();
+    throw new StreamingValidationError(
+      "manifest_not_first",
+      `Expected manifest.json as the first tar entry, got "${first.header.name}"`,
+    );
+  }
+
+  // Drain the entry body into a Buffer, enforcing the size cap as we go.
+  // The moment we cross the cap we destroy the entry stream — this signals
+  // the tar extractor (and therefore gunzip + upstream source) to abort,
+  // so a malicious archive whose "manifest" is a multi-GB decompressed
+  // stream can't force us to read through all of it before rejecting.
+  const chunks: Buffer[] = [];
+  let total = 0;
+  for await (const chunk of first.body) {
+    const buf = chunk instanceof Buffer ? chunk : Buffer.from(chunk);
+    total += buf.length;
+    if (total > MANIFEST_MAX_BYTES) {
+      first.body.destroy();
+      throw new StreamingValidationError(
+        "manifest_too_large",
+        `manifest.json exceeds ${MANIFEST_MAX_BYTES} byte limit (read ${total} bytes before aborting)`,
+      );
+    }
+    chunks.push(buf);
+  }
+
+  const bodyBuf = Buffer.concat(chunks, total);
+
+  let manifestRaw: unknown;
+  try {
+    manifestRaw = JSON.parse(bodyBuf.toString("utf8"));
+  } catch (err) {
+    throw new StreamingValidationError(
+      "manifest_malformed",
+      `manifest.json is not valid JSON: ${
+        err instanceof Error ? err.message : String(err)
+      }`,
+    );
+  }
+
+  const parseResult = ManifestSchema.safeParse(manifestRaw);
+  if (!parseResult.success) {
+    const issues = parseResult.error.issues
+      .map((i) => `${i.path.join(".") || "<root>"}: ${i.message}`)
+      .join("; ");
+    throw new StreamingValidationError(
+      "manifest_schema",
+      `manifest.json failed schema validation: ${issues}`,
+    );
+  }
+
+  const manifest = parseResult.data;
+
+  // Recompute the self-referencing checksum using the exact canonicalization
+  // that vbundle-validator.ts uses. Any drift here would silently reject
+  // valid bundles produced by buildVBundle.
+  const computed = computeManifestSha256(manifestRaw);
+  if (computed !== manifest.manifest_sha256) {
+    throw new StreamingValidationError(
+      "manifest_sha256",
+      `Manifest checksum mismatch: expected ${manifest.manifest_sha256}, computed ${computed}`,
+    );
+  }
+
+  const expected = new Map<string, { sha256: string; size: number }>();
+  for (const file of manifest.files) {
+    if (expected.has(file.path)) {
+      throw new StreamingValidationError(
+        "manifest_duplicate_path",
+        `Manifest contains duplicate entry for path: ${file.path}`,
+      );
+    }
+    expected.set(file.path, { sha256: file.sha256, size: file.size });
+  }
+
+  return { manifest, expected };
+}
+
+// ---------------------------------------------------------------------------
+// Per-entry hash + size verifier
+// ---------------------------------------------------------------------------
+
+/**
+ * Create a passthrough `Transform` that:
+ *   - forwards every chunk unchanged (identity transform for correct input),
+ *   - incrementally SHA-256s the byte stream,
+ *   - on `_flush`, errors the pipeline if the final digest or total byte
+ *     count differs from `expected`.
+ *
+ * Errors are emitted as `StreamingValidationError` with `code` set to
+ * `"entry_hash"` or `"entry_size"` and `archivePath` populated so callers
+ * can surface which file failed.
+ *
+ * Consumers should pipe the entry body through this transform before
+ * writing to disk — that way a bad payload is caught before the byte
+ * reaches storage rather than after a whole 8 GB write completes.
+ */
+export function createHashVerifier(expected: {
+  sha256: string;
+  size: number;
+  archivePath: string;
+}): Transform {
+  const hash = createHash("sha256");
+  let bytes = 0;
+
+  return new Transform({
+    transform(
+      chunk: Buffer | string,
+      encoding: BufferEncoding,
+      callback: TransformCallback,
+    ) {
+      try {
+        const buf =
+          typeof chunk === "string" ? Buffer.from(chunk, encoding) : chunk;
+        hash.update(buf);
+        bytes += buf.length;
+        callback(null, buf);
+      } catch (err) {
+        callback(err instanceof Error ? err : new Error(String(err)));
+      }
+    },
+    flush(callback: TransformCallback) {
+      // Size check first — a wrong size is a sharper signal than a hash
+      // collision, and a truncated payload frequently triggers both.
+      if (bytes !== expected.size) {
+        callback(
+          new StreamingValidationError(
+            "entry_size",
+            `Size mismatch for ${expected.archivePath}: expected ${expected.size} bytes, got ${bytes}`,
+            expected.archivePath,
+          ),
+        );
+        return;
+      }
+
+      const digest = hash.digest("hex");
+      if (digest !== expected.sha256) {
+        callback(
+          new StreamingValidationError(
+            "entry_hash",
+            `Checksum mismatch for ${expected.archivePath}: expected ${expected.sha256}, computed ${digest}`,
+            expected.archivePath,
+          ),
+        );
+        return;
+      }
+
+      callback();
+    },
+  });
+}

package/src/runtime/migrations/vbundle-tar-stream.ts

@@ -0,0 +1,217 @@
+/**
+ * Streaming tar reader for `.vbundle` archives.
+ *
+ * A `.vbundle` is a gzip-compressed tar. This module gunzips the incoming
+ * `Readable`, pipes it through `tar-stream`'s push-style extractor, and
+ * adapts the `(header, stream, next)` event into a consumer-friendly async
+ * generator that yields one entry at a time.
+ *
+ * Memory invariant: each entry's body is surfaced as a `Readable`, never a
+ * pre-buffered blob. Callers MUST fully consume (or explicitly `resume()`)
+ * each body stream before advancing the outer generator — otherwise the
+ * underlying tar extractor will stall waiting on backpressure.
+ */
+
+import type { Readable } from "node:stream";
+import { createGunzip } from "node:zlib";
+
+import { extract as tarExtract } from "tar-stream";
+
+// ---------------------------------------------------------------------------
+// Public types
+// ---------------------------------------------------------------------------
+
+/** Subset of tar-stream's header surface that vbundle consumers care about. */
+export interface StreamedTarHeader {
+  name: string;
+  size: number;
+  type: "file" | "directory" | "pax-header" | "other";
+}
+
+export interface StreamedTarEntry {
+  header: StreamedTarHeader;
+  /**
+   * The entry body. Must be fully consumed (drained via for-await, piped,
+   * or explicitly `.resume()`'d) before the generator is advanced again.
+   */
+  body: Readable;
+}
+
+// ---------------------------------------------------------------------------
+// Implementation
+// ---------------------------------------------------------------------------
+
+interface PendingEntry {
+  header: StreamedTarHeader;
+  body: Readable;
+  next: (err?: Error | null) => void;
+}
+
+function normalizeHeaderType(
+  type: string | undefined,
+): StreamedTarHeader["type"] {
+  switch (type) {
+    case "file":
+      return "file";
+    case "directory":
+      return "directory";
+    case "pax-header":
+      return "pax-header";
+    default:
+      return "other";
+  }
+}
+
+/**
+ * Stream a `.vbundle` archive as an async sequence of tar entries.
+ *
+ * Errors from the upstream source, gunzip, or tar-stream extractor are
+ * surfaced by throwing from the generator.
+ *
+ * Early termination (caller `break`s or the for-await loop throws) destroys
+ * the upstream source and the tar extractor so that sockets and file
+ * descriptors are released promptly.
+ */
+export async function* parseVBundleStream(
+  source: Readable,
+): AsyncGenerator<StreamedTarEntry, void, void> {
+  const gunzip = createGunzip();
+  const extractor = tarExtract();
+
+  // Terminal error captured from any stage in the pipeline. Consumed by the
+  // generator's internal pump; re-thrown on the next `yield`.
+  let pipelineError: Error | null = null;
+  let finished = false;
+
+  // Single-slot mailbox: the extractor pushes one entry here, and the
+  // generator consumes it. We gate the push-style API by holding the tar
+  // extractor's `next` callback until the caller is done with the body.
+  let waiter: {
+    resolve: (entry: PendingEntry | null) => void;
+    reject: (err: Error) => void;
+  } | null = null;
+  let pending: PendingEntry | null = null;
+
+  function pushEntry(entry: PendingEntry): void {
+    if (waiter) {
+      const w = waiter;
+      waiter = null;
+      w.resolve(entry);
+    } else {
+      pending = entry;
+    }
+  }
+
+  function pushError(err: Error): void {
+    if (pipelineError) return;
+    pipelineError = err;
+    if (waiter) {
+      const w = waiter;
+      waiter = null;
+      w.reject(err);
+    }
+  }
+
+  function pushFinish(): void {
+    finished = true;
+    if (waiter) {
+      const w = waiter;
+      waiter = null;
+      w.resolve(null);
+    }
+  }
+
+  extractor.on("entry", (header, body, next) => {
+    // Avoid unhandled "error" on body streams destroyed mid-flight; the
+    // extractor itself propagates the real error to its "error" listener.
+    body.on("error", () => {});
+    pushEntry({
+      header: {
+        name: header.name,
+        size: header.size,
+        type: normalizeHeaderType(header.type),
+      },
+      body,
+      next,
+    });
+  });
+
+  extractor.on("error", (err: Error) => {
+    pushError(err);
+  });
+
+  extractor.on("finish", () => {
+    pushFinish();
+  });
+
+  gunzip.on("error", (err: Error) => {
+    pushError(err);
+    extractor.destroy(err);
+  });
+
+  source.on("error", (err: Error) => {
+    pushError(err);
+    gunzip.destroy(err);
+    extractor.destroy(err);
+  });
+
+  // Kick off the pipeline. We wire stages with `.pipe(..., { end: true })`
+  // so that clean EOF from the source triggers `finish` on the extractor.
+  source.pipe(gunzip).pipe(extractor);
+
+  function nextEntry(): Promise<PendingEntry | null> {
+    if (pipelineError) return Promise.reject(pipelineError);
+    if (pending) {
+      const p = pending;
+      pending = null;
+      return Promise.resolve(p);
+    }
+    if (finished) return Promise.resolve(null);
+    return new Promise<PendingEntry | null>((resolve, reject) => {
+      waiter = { resolve, reject };
+    });
+  }
+
+  try {
+    while (true) {
+      const entry = await nextEntry();
+      if (entry === null) return;
+
+      let advanced = false;
+      const advance = (err?: Error | null): void => {
+        if (advanced) return;
+        advanced = true;
+        entry.next(err ?? null);
+      };
+
+      // When the consumer finishes (or abandons) the body, release the tar
+      // extractor so the next entry can flow.
+      entry.body.once("end", () => advance());
+      entry.body.once("close", () => advance());
+      entry.body.once("error", (err: Error) => advance(err));
+
+      try {
+        yield { header: entry.header, body: entry.body };
+      } catch (err) {
+        // Caller threw (or re-threw) inside the for-await loop. Propagate
+        // after cleanup in the finally below.
+        advance(err instanceof Error ? err : new Error(String(err)));
+        throw err;
+      }
+
+      // If the consumer neither consumed nor destroyed the body, drain it
+      // for them so the extractor can advance.
+      if (!advanced) {
+        entry.body.resume();
+      }
+    }
+  } finally {
+    // Early termination (break, throw, or natural completion): tear down the
+    // pipeline so we don't leak the socket/file descriptor underneath.
+    if (!finished || pipelineError) {
+      source.destroy();
+      gunzip.destroy();
+      extractor.destroy();
+    }
+  }
+}

package/src/runtime/migrations/vbundle-validator.ts

@@ -29,7 +29,7 @@ const ManifestFileEntry = z.object({
   size: z.number().int().nonnegative(),
 });
 
-const ManifestSchema = z.object({
+export const ManifestSchema = z.object({
   schema_version: z.string(),
   created_at: z.string(),
   source: z.string().optional(),
@@ -176,7 +176,7 @@ function sha256Hex(data: Uint8Array | string): string {
  * Canonicalize a JSON object by sorting keys recursively, then SHA-256 hash it.
  * This matches the platform's canonicalization approach.
  */
-function canonicalizeJson(obj: unknown): string {
+export function canonicalizeJson(obj: unknown): string {
   return JSON.stringify(obj, (_key, value) => {
     if (value && typeof value === "object" && !Array.isArray(value)) {
       const sorted: Record<string, unknown> = {};
@@ -189,6 +189,18 @@ function canonicalizeJson(obj: unknown): string {
   });
 }
 
+/**
+ * Recompute the `manifest_sha256` field for a manifest object. Strips the
+ * `manifest_sha256` property, canonicalizes the remaining JSON, and returns
+ * the SHA-256 hex digest. Centralized here so the streaming validator and
+ * the in-memory validator agree on the exact canonicalization.
+ */
+export function computeManifestSha256(manifest: unknown): string {
+  const copy = { ...(manifest as Record<string, unknown>) };
+  delete copy.manifest_sha256;
+  return sha256Hex(canonicalizeJson(copy));
+}
+
 // ---------------------------------------------------------------------------
 // Core validation
 // ---------------------------------------------------------------------------
@@ -300,10 +312,7 @@ export function validateVBundle(data: Uint8Array): VBundleValidationResult {
   // Step 5: Verify manifest checksum
   // The manifest_sha256 field is the SHA-256 of the canonicalized JSON
   // with the manifest_sha256 field itself excluded.
-  const manifestForChecksum = { ...(manifestRaw as Record<string, unknown>) };
-  delete manifestForChecksum.manifest_sha256;
-  const canonicalized = canonicalizeJson(manifestForChecksum);
-  const computedManifestSha256 = sha256Hex(canonicalized);
+  const computedManifestSha256 = computeManifestSha256(manifestRaw);
 
   if (computedManifestSha256 !== manifest.manifest_sha256) {
     errors.push({

package/src/runtime/pending-interactions.ts

@@ -52,17 +52,6 @@ export interface PendingInteraction {
   confirmationDetails?: ConfirmationDetails;
   /** For ACP permissions: resolves directly without a Conversation object. */
   directResolve?: (decision: UserDecision) => void;
-  /**
-   * For host_browser interactions originating outside an agent loop
-   * (e.g. the `assistant browser chrome relay` CLI shim that POSTs to
-   * /v1/browser-cdp). Resolves the CDP round-trip directly without
-   * touching a Conversation. When set, /v1/host-browser-result invokes
-   * this instead of `interaction.conversation.resolveHostBrowser`.
-   */
-  directBrowserResolve?: (response: {
-    content: string;
-    isError: boolean;
-  }) => void;
 }
 
 const pending = new Map<string, PendingInteraction>();