@mycadet/assistant 0.12.11 → 0.13.0

package/src/runtime/migrations/__tests__/vbundle-streaming-validator.test.ts

@@ -1,598 +0,0 @@
-/**
- * Tests for the streaming validator primitives:
- * - `readAndValidateManifest` consumes the first tar entry and runs the
- *   full manifest validation pipeline.
- * - `createHashVerifier` is a passthrough Transform that aborts with a
- *   typed error on digest/size mismatch.
- *
- * Happy-path fixtures are built with the existing `buildVBundle` helper.
- * Negative-path fixtures (non-manifest first, oversize manifest, malformed
- * JSON, schema fail, sha mismatch) are hand-constructed because
- * `buildVBundle` always produces valid, manifest-first archives.
- */
-
-import { createHash } from "node:crypto";
-import { Readable } from "node:stream";
-import { pipeline } from "node:stream/promises";
-import { gzipSync } from "node:zlib";
-import { describe, expect, test } from "bun:test";
-
-import { buildVBundle } from "../vbundle-builder.js";
-import {
-  createHashVerifier,
-  readAndValidateManifest,
-  StreamingValidationError,
-} from "../vbundle-streaming-validator.js";
-import {
-  parseVBundleStream,
-  type StreamedTarEntry,
-} from "../vbundle-tar-stream.js";
-import {
-  computeLegacyManifestSha256,
-  computeManifestChecksum,
-  MANIFEST_MAX_BYTES,
-} from "../vbundle-validator.js";
-import { defaultV1Options } from "./v1-test-helpers.js";
-
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-
-const BLOCK_SIZE = 512;
-
-function readableFromBuffer(buf: Uint8Array): Readable {
-  return Readable.from([Buffer.from(buf)]);
-}
-
-/** Minimal hand-rolled tar entry builder (ustar regular file). */
-function buildTarEntry(name: string, data: Uint8Array): Uint8Array {
-  const encoder = new TextEncoder();
-  const nameBytes = encoder.encode(name);
-  if (nameBytes.length > 100) {
-    throw new Error(`test helper: name too long (${nameBytes.length} bytes)`);
-  }
-
-  const header = new Uint8Array(BLOCK_SIZE);
-  header.set(nameBytes, 0);
-
-  const writeOctal = (offset: number, length: number, value: number) => {
-    const str = value.toString(8).padStart(length - 1, "0");
-    for (let i = 0; i < str.length; i++) {
-      header[offset + i] = str.charCodeAt(i);
-    }
-    header[offset + length - 1] = 0;
-  };
-
-  writeOctal(100, 8, 0o644); // mode
-  writeOctal(108, 8, 0); // uid
-  writeOctal(116, 8, 0); // gid
-  writeOctal(124, 12, data.length); // size
-  writeOctal(136, 12, Math.floor(Date.now() / 1000)); // mtime
-  header[156] = "0".charCodeAt(0); // typeflag = regular file
-
-  const magic = encoder.encode("ustar\0");
-  header.set(magic, 257);
-  header[263] = "0".charCodeAt(0);
-  header[264] = "0".charCodeAt(0);
-
-  // Header checksum: sum of all bytes with the checksum field treated
-  // as 8 ASCII spaces.
-  let sum = 0;
-  for (let i = 0; i < BLOCK_SIZE; i++) {
-    sum += i >= 148 && i < 156 ? 0x20 : header[i];
-  }
-  writeOctal(148, 7, sum);
-  header[155] = 0x20;
-
-  // Pad data out to the next 512-byte boundary.
-  const remainder = data.length % BLOCK_SIZE;
-  const padded =
-    remainder === 0
-      ? data
-      : (() => {
-          const out = new Uint8Array(data.length + (BLOCK_SIZE - remainder));
-          out.set(data, 0);
-          return out;
-        })();
-
-  const entry = new Uint8Array(header.length + padded.length);
-  entry.set(header, 0);
-  entry.set(padded, header.length);
-  return entry;
-}
-
-/** Hand-build a gzipped tar archive from the given entries (no manifest injected). */
-function buildRawVBundle(
-  entries: Array<{ name: string; data: Uint8Array }>,
-): Uint8Array {
-  const parts: Uint8Array[] = entries.map((e) => buildTarEntry(e.name, e.data));
-  // End-of-archive marker: two zero blocks.
-  parts.push(new Uint8Array(BLOCK_SIZE * 2));
-
-  const total = parts.reduce((n, p) => n + p.length, 0);
-  const tar = new Uint8Array(total);
-  let offset = 0;
-  for (const p of parts) {
-    tar.set(p, offset);
-    offset += p.length;
-  }
-  return gzipSync(tar);
-}
-
-/** Fetch the first entry of a streaming archive; drain+close the iterator after. */
-async function firstEntryOf(
-  archive: Uint8Array,
-): Promise<{ entry: StreamedTarEntry; drainRest: () => Promise<void> }> {
-  const iter = parseVBundleStream(readableFromBuffer(archive));
-  const first = await iter.next();
-  if (first.done || !first.value) {
-    throw new Error("archive contained no entries");
-  }
-  const drainRest = async () => {
-    for await (const rest of iter) {
-      rest.body.resume();
-    }
-  };
-  return { entry: first.value, drainRest };
-}
-
-// ---------------------------------------------------------------------------
-// readAndValidateManifest — happy path
-// ---------------------------------------------------------------------------
-
-describe("readAndValidateManifest — happy path", () => {
-  test("parses manifest and populates expected map from manifest.contents", async () => {
-    const fileA = new TextEncoder().encode("alpha payload\n");
-    const fileB = new TextEncoder().encode("beta payload\n");
-    const { archive, manifest } = buildVBundle({
-      files: [
-        {
-          path: "data/db/assistant.db",
-          data: new Uint8Array(),
-        },
-        { path: "workspace/a.txt", data: fileA },
-        { path: "workspace/b.txt", data: fileB },
-      ],
-      ...defaultV1Options(),
-    });
-
-    const { entry, drainRest } = await firstEntryOf(archive);
-    const result = await readAndValidateManifest(entry);
-    await drainRest();
-
-    expect(result.manifest.schema_version).toBe(manifest.schema_version);
-    // Includes the synthetic data/db/assistant.db entry plus a/b.txt.
-    expect(result.manifest.contents).toHaveLength(3);
-    expect(result.manifest.checksum).toBe(manifest.checksum);
-
-    expect(result.expected.size).toBe(3);
-    const expectA = result.expected.get("workspace/a.txt");
-    expect(expectA?.size).toBe(fileA.length);
-    expect(expectA?.sha256).toBe(
-      manifest.contents.find((f) => f.path === "workspace/a.txt")?.sha256,
-    );
-    const expectB = result.expected.get("workspace/b.txt");
-    expect(expectB?.size).toBe(fileB.length);
-    expect(expectB?.sha256).toBe(
-      manifest.contents.find((f) => f.path === "workspace/b.txt")?.sha256,
-    );
-  });
-});
-
-// ---------------------------------------------------------------------------
-// readAndValidateManifest — negative paths
-// ---------------------------------------------------------------------------
-
-describe("readAndValidateManifest — negative paths", () => {
-  test("throws manifest_not_first when first entry is not manifest.json", async () => {
-    const archive = buildRawVBundle([
-      { name: "workspace/a.txt", data: new TextEncoder().encode("hello") },
-    ]);
-    const { entry, drainRest } = await firstEntryOf(archive);
-
-    let err: StreamingValidationError | null = null;
-    try {
-      await readAndValidateManifest(entry);
-    } catch (e) {
-      err = e as StreamingValidationError;
-    }
-    await drainRest();
-
-    expect(err).toBeInstanceOf(StreamingValidationError);
-    expect(err?.code).toBe("manifest_not_first");
-  });
-
-  test("throws manifest_too_large and fails fast before draining the whole body", async () => {
-    // Fake a tar entry whose body would emit 10x the cap if fully drained.
-    // The validator must destroy() the stream and throw the moment the
-    // running byte count crosses MANIFEST_MAX_BYTES; it must NOT keep
-    // pulling chunks.
-    //
-    // We count bytes emitted via a _read implementation, and after the
-    // throw assert both that destroy() fired and that far less than the
-    // full body was ever pulled out of the stream.
-    const CHUNK = MANIFEST_MAX_BYTES / 2;
-    const TOTAL_CHUNKS = 20;
-    let chunksEmitted = 0;
-    let bytesEmitted = 0;
-    const body = new Readable({
-      read() {
-        if (chunksEmitted >= TOTAL_CHUNKS) {
-          this.push(null);
-          return;
-        }
-        const buf = Buffer.alloc(CHUNK, 0x20);
-        chunksEmitted += 1;
-        bytesEmitted += buf.length;
-        this.push(buf);
-      },
-    });
-
-    const entry: StreamedTarEntry = {
-      header: {
-        name: "manifest.json",
-        size: CHUNK * TOTAL_CHUNKS,
-        type: "file",
-      },
-      body,
-    };
-
-    let err: StreamingValidationError | null = null;
-    try {
-      await readAndValidateManifest(entry);
-    } catch (e) {
-      err = e as StreamingValidationError;
-    }
-
-    expect(err).toBeInstanceOf(StreamingValidationError);
-    expect(err?.code).toBe("manifest_too_large");
-    // Fail-fast assertions: destroy() was called, and we didn't drain past
-    // the cap + one chunk. If the validator had drained to EOF we'd see all
-    // 20 chunks here.
-    expect(body.destroyed).toBe(true);
-    expect(chunksEmitted).toBeLessThanOrEqual(3);
-    expect(bytesEmitted).toBeLessThan(2 * MANIFEST_MAX_BYTES);
-  });
-
-  test("throws bundle_too_many_entries when manifest.contents exceeds maxEntries", async () => {
-    const { archive } = buildVBundle({
-      files: [
-        { path: "data/db/assistant.db", data: new Uint8Array() },
-        { path: "workspace/a.txt", data: new TextEncoder().encode("a") },
-        { path: "workspace/b.txt", data: new TextEncoder().encode("b") },
-      ],
-      ...defaultV1Options(),
-    });
-    const { entry, drainRest } = await firstEntryOf(archive);
-
-    let err: StreamingValidationError | null = null;
-    try {
-      await readAndValidateManifest(entry, { maxEntries: 2 });
-    } catch (e) {
-      err = e as StreamingValidationError;
-    }
-    await drainRest();
-
-    expect(err).toBeInstanceOf(StreamingValidationError);
-    expect(err?.code).toBe("bundle_too_many_entries");
-  });
-
-  test("throws manifest_malformed when manifest body is not valid JSON", async () => {
-    const archive = buildRawVBundle([
-      { name: "manifest.json", data: new TextEncoder().encode("{not-json") },
-    ]);
-    const { entry, drainRest } = await firstEntryOf(archive);
-
-    let err: StreamingValidationError | null = null;
-    try {
-      await readAndValidateManifest(entry);
-    } catch (e) {
-      err = e as StreamingValidationError;
-    }
-    await drainRest();
-
-    expect(err).toBeInstanceOf(StreamingValidationError);
-    expect(err?.code).toBe("manifest_malformed");
-  });
-
-  test("throws manifest_schema when a required field is missing", async () => {
-    // Valid JSON but missing `files`, `manifest_sha256`, etc.
-    const bogus = new TextEncoder().encode(
-      JSON.stringify({ schema_version: "1.0", created_at: "now" }),
-    );
-    const archive = buildRawVBundle([{ name: "manifest.json", data: bogus }]);
-    const { entry, drainRest } = await firstEntryOf(archive);
-
-    let err: StreamingValidationError | null = null;
-    try {
-      await readAndValidateManifest(entry);
-    } catch (e) {
-      err = e as StreamingValidationError;
-    }
-    await drainRest();
-
-    expect(err).toBeInstanceOf(StreamingValidationError);
-    expect(err?.code).toBe("manifest_schema");
-  });
-
-  test("throws manifest_sha256 when the declared digest doesn't match canonical JSON", async () => {
-    const badManifest = {
-      schema_version: 1,
-      bundle_id: "00000000-0000-4000-8000-000000000000",
-      created_at: new Date().toISOString(),
-      assistant: { id: "self", name: "Test", runtime_version: "0.0.0-test" },
-      origin: { mode: "self-hosted-local" },
-      compatibility: {
-        min_runtime_version: "0.0.0-test",
-        max_runtime_version: null,
-      },
-      contents: [
-        {
-          path: "data/db/assistant.db",
-          sha256:
-            "1111111111111111111111111111111111111111111111111111111111111111",
-          size_bytes: 10,
-        },
-      ],
-      // Deliberately wrong digest. The canonical hash of a v1 manifest
-      // won't match this, regardless of ordering.
-      checksum:
-        "0000000000000000000000000000000000000000000000000000000000000000",
-      secrets_redacted: false,
-      export_options: {
-        include_logs: false,
-        include_browser_state: false,
-        include_memory_vectors: false,
-      },
-    };
-    const archive = buildRawVBundle([
-      {
-        name: "manifest.json",
-        data: new TextEncoder().encode(JSON.stringify(badManifest)),
-      },
-    ]);
-    const { entry, drainRest } = await firstEntryOf(archive);
-
-    let err: StreamingValidationError | null = null;
-    try {
-      await readAndValidateManifest(entry);
-    } catch (e) {
-      err = e as StreamingValidationError;
-    }
-    await drainRest();
-
-    expect(err).toBeInstanceOf(StreamingValidationError);
-    expect(err?.code).toBe("manifest_sha256");
-  });
-
-  test("throws manifest_duplicate_path when the same archive path appears twice", async () => {
-    const baseManifest = {
-      schema_version: 1,
-      bundle_id: "00000000-0000-4000-8000-000000000001",
-      created_at: new Date().toISOString(),
-      assistant: { id: "self", name: "Test", runtime_version: "0.0.0-test" },
-      origin: { mode: "self-hosted-local" },
-      compatibility: {
-        min_runtime_version: "0.0.0-test",
-        max_runtime_version: null,
-      },
-      contents: [
-        {
-          path: "data/db/assistant.db",
-          sha256:
-            "1111111111111111111111111111111111111111111111111111111111111111",
-          size_bytes: 10,
-        },
-        {
-          // Deliberately duplicate path — malicious bundle could exploit
-          // this to bypass per-entry integrity checks if we silently
-          // collapsed.
-          path: "data/db/assistant.db",
-          sha256:
-            "2222222222222222222222222222222222222222222222222222222222222222",
-          size_bytes: 20,
-        },
-      ],
-      checksum: "",
-      secrets_redacted: false,
-      export_options: {
-        include_logs: false,
-        include_browser_state: false,
-        include_memory_vectors: false,
-      },
-    };
-    baseManifest.checksum = computeManifestChecksum(baseManifest);
-
-    const archive = buildRawVBundle([
-      {
-        name: "manifest.json",
-        data: new TextEncoder().encode(JSON.stringify(baseManifest)),
-      },
-    ]);
-    const { entry, drainRest } = await firstEntryOf(archive);
-
-    let err: StreamingValidationError | null = null;
-    try {
-      await readAndValidateManifest(entry);
-    } catch (e) {
-      err = e as StreamingValidationError;
-    }
-    await drainRest();
-
-    expect(err).toBeInstanceOf(StreamingValidationError);
-    expect(err?.code).toBe("manifest_duplicate_path");
-    expect(err?.message).toContain("data/db/assistant.db");
-  });
-});
-
-// ---------------------------------------------------------------------------
-// createHashVerifier
-// ---------------------------------------------------------------------------
-
-describe("createHashVerifier — identity + integrity", () => {
-  const payload = Buffer.from(
-    "the quick brown fox jumps over the lazy dog".repeat(100),
-    "utf8",
-  );
-  // Precomputed digest for the payload above.
-  const payloadSha = createHash("sha256").update(payload).digest("hex");
-
-  test("is an identity Transform for correct inputs", async () => {
-    const verifier = createHashVerifier({
-      sha256: payloadSha,
-      size: payload.length,
-      archivePath: "workspace/ok.txt",
-    });
-
-    // Feed the payload in two chunks to exercise multi-call _transform.
-    const half = payload.length >>> 1;
-    const source = Readable.from([
-      payload.subarray(0, half),
-      payload.subarray(half),
-    ]);
-
-    const collected: Buffer[] = [];
-    verifier.on("data", (chunk: Buffer) => collected.push(chunk));
-
-    await pipeline(source, verifier);
-
-    const out = Buffer.concat(collected);
-    expect(out.length).toBe(payload.length);
-    expect(out.equals(payload)).toBe(true);
-  });
-
-  test("errors with code entry_hash on digest mismatch", async () => {
-    const verifier = createHashVerifier({
-      sha256:
-        // Wrong digest.
-        "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
-      size: payload.length,
-      archivePath: "workspace/bad-hash.txt",
-    });
-
-    let err: StreamingValidationError | null = null;
-    try {
-      await pipeline(
-        Readable.from([payload]),
-        verifier,
-        async function* (source: AsyncIterable<Buffer>) {
-          // Drain the transform so _flush runs.
-          for await (const _chunk of source) {
-            // discard
-          }
-        },
-      );
-    } catch (e) {
-      err = e as StreamingValidationError;
-    }
-
-    expect(err).toBeInstanceOf(StreamingValidationError);
-    expect(err?.code).toBe("entry_hash");
-    expect(err?.archivePath).toBe("workspace/bad-hash.txt");
-  });
-
-  test("errors with code entry_size on byte-count mismatch", async () => {
-    // Right digest, wrong declared size.
-    const verifier = createHashVerifier({
-      sha256: payloadSha,
-      size: payload.length + 1,
-      archivePath: "workspace/bad-size.txt",
-    });
-
-    let err: StreamingValidationError | null = null;
-    try {
-      await pipeline(
-        Readable.from([payload]),
-        verifier,
-        async function* (source: AsyncIterable<Buffer>) {
-          for await (const _chunk of source) {
-            // discard
-          }
-        },
-      );
-    } catch (e) {
-      err = e as StreamingValidationError;
-    }
-
-    expect(err).toBeInstanceOf(StreamingValidationError);
-    expect(err?.code).toBe("entry_size");
-    expect(err?.archivePath).toBe("workspace/bad-size.txt");
-  });
-});
-
-// ---------------------------------------------------------------------------
-// readAndValidateManifest — legacy fallback (backwards compatibility)
-// ---------------------------------------------------------------------------
-
-describe("readAndValidateManifest — legacy fallback", () => {
-  test("accepts a valid legacy six-field manifest and translates to v1 shape", async () => {
-    const dbBytes = new TextEncoder().encode("legacy-db");
-    const dbSha = createHash("sha256").update(dbBytes).digest("hex");
-
-    const legacyManifest: Record<string, unknown> = {
-      schema_version: "1.0",
-      created_at: new Date().toISOString(),
-      source: "runtime-export",
-      description: "legacy fixture",
-      files: [
-        { path: "data/db/assistant.db", sha256: dbSha, size: dbBytes.length },
-      ],
-      manifest_sha256: "",
-    };
-    legacyManifest.manifest_sha256 =
-      computeLegacyManifestSha256(legacyManifest);
-
-    const archive = buildRawVBundle([
-      {
-        name: "manifest.json",
-        data: new TextEncoder().encode(JSON.stringify(legacyManifest)),
-      },
-      { name: "data/db/assistant.db", data: dbBytes },
-    ]);
-
-    const { entry, drainRest } = await firstEntryOf(archive);
-    const result = await readAndValidateManifest(entry);
-    await drainRest();
-
-    expect(result.manifest.schema_version).toBe(1);
-    expect(result.manifest.contents).toHaveLength(1);
-    expect(result.manifest.contents[0]?.path).toBe("data/db/assistant.db");
-    expect(result.manifest.contents[0]?.size_bytes).toBe(dbBytes.length);
-    expect(result.expected.get("data/db/assistant.db")?.sha256).toBe(dbSha);
-  });
-
-  test("throws manifest_sha256 on a legacy manifest with a wrong checksum", async () => {
-    const dbBytes = new TextEncoder().encode("legacy-db");
-    const dbSha = createHash("sha256").update(dbBytes).digest("hex");
-
-    const badLegacy = {
-      schema_version: "1.0",
-      created_at: new Date().toISOString(),
-      files: [
-        { path: "data/db/assistant.db", sha256: dbSha, size: dbBytes.length },
-      ],
-      // Deliberately wrong checksum.
-      manifest_sha256:
-        "0000000000000000000000000000000000000000000000000000000000000000",
-    };
-
-    const archive = buildRawVBundle([
-      {
-        name: "manifest.json",
-        data: new TextEncoder().encode(JSON.stringify(badLegacy)),
-      },
-    ]);
-    const { entry, drainRest } = await firstEntryOf(archive);
-
-    let err: StreamingValidationError | null = null;
-    try {
-      await readAndValidateManifest(entry);
-    } catch (e) {
-      err = e as StreamingValidationError;
-    }
-    await drainRest();
-
-    expect(err).toBeInstanceOf(StreamingValidationError);
-    expect(err?.code).toBe("manifest_sha256");
-  });
-});