@vellumai/assistant 0.5.16 → 0.6.1

package/src/__tests__/memory-upsert-concurrency.test.ts

@@ -16,7 +16,7 @@
  * processes and is not tested here.
  */
 
-import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test";
+import { beforeEach, describe, expect, mock, test } from "bun:test";
 
 import { eq } from "drizzle-orm";
 
@@ -59,27 +59,17 @@ mock.module("../config/loader.js", () => ({
   invalidateConfigCache: () => {},
 }));
 
-import { getDb, initializeDb, resetDb } from "../memory/db.js";
+import { getDb, initializeDb } from "../memory/db.js";
 import { indexMessageNow } from "../memory/indexer.js";
-import {
-  conversations,
-  memoryItems,
-  memorySegments,
-  messages,
-} from "../memory/schema.js";
+import { conversations, memorySegments, messages } from "../memory/schema.js";
 
 // Initialize DB once for the entire file. Each test cleans its own tables.
 initializeDb();
 
-afterAll(() => {
-  resetDb();
-});
-
 function resetTables() {
   const db = getDb();
-  db.run("DELETE FROM memory_item_sources");
   db.run("DELETE FROM memory_embeddings");
-  db.run("DELETE FROM memory_items");
+  db.run("DELETE FROM memory_graph_nodes");
   db.run("DELETE FROM memory_segments");
   db.run("DELETE FROM memory_jobs");
   db.run("DELETE FROM messages");
@@ -559,101 +549,3 @@ describe("memory segment job atomicity under repeated indexer invocations", () =
   });
 });
 
-// ─────────────────────────────────────────────────────────────────────────────
-// Test suite: memory_items fingerprint uniqueness under race conditions
-// ─────────────────────────────────────────────────────────────────────────────
-
-describe("memory_items fingerprint uniqueness under race conditions", () => {
-  beforeEach(() => {
-    resetTables();
-  });
-
-  test("duplicate inserts with identical fingerprints produce exactly one row", () => {
-    // The memory_items table has a unique constraint on (fingerprint, scope_id).
-    // Two sequential inserts for the same fingerprint simulate duplicate extractor
-    // runs.  Only one INSERT must land; the second must be absorbed by ON CONFLICT.
-    const db = getDb();
-    const now = Date.now();
-    const fingerprint = "fp-race-unique-test-concurrency";
-    const scopeId = "default";
-
-    // Use raw SQL to replicate what the items-extractor would do when a second
-    // run tries to INSERT the same fingerprint that already exists.
-    const raw = (db as unknown as { $client: import("bun:sqlite").Database })
-      .$client;
-
-    raw.run(`
-      INSERT INTO memory_items (
-        id, kind, subject, statement, status, confidence, importance,
-        fingerprint, verification_state, scope_id, first_seen_at, last_seen_at
-      ) VALUES (
-        'item-race-1', 'preference', 'code style', 'I prefer tabs over spaces.',
-        'active', 0.8, 0.6, '${fingerprint}', 'user_reported', '${scopeId}',
-        ${now}, ${now}
-      )
-    `);
-
-    // Second "worker" tries to insert the same fingerprint — must not create a
-    // duplicate.  INSERT OR IGNORE / ON CONFLICT DO NOTHING is the expected
-    // behavior for the unique constraint.
-    expect(() => {
-      raw.run(`
-        INSERT OR IGNORE INTO memory_items (
-          id, kind, subject, statement, status, confidence, importance,
-          fingerprint, verification_state, scope_id, first_seen_at, last_seen_at
-        ) VALUES (
-          'item-race-2', 'preference', 'code style', 'I prefer tabs over spaces.',
-          'active', 0.8, 0.6, '${fingerprint}', 'user_reported', '${scopeId}',
-          ${now + 1}, ${now + 1}
-        )
-      `);
-    }).not.toThrow();
-
-    const rows = db
-      .select()
-      .from(memoryItems)
-      .all()
-      .filter((r) => r.fingerprint === fingerprint);
-
-    // Only the first insert must have landed.
-    expect(rows).toHaveLength(1);
-    expect(rows[0].id).toBe("item-race-1");
-  });
-
-  test("bare INSERT without IGNORE throws on duplicate fingerprint+scopeId", () => {
-    // Verify the DB-level unique constraint is actually enforced so that any code
-    // path that accidentally omits ON CONFLICT will fail loudly rather than silently
-    // producing inconsistent state.
-    const db = getDb();
-    const now = Date.now();
-    const fingerprint = "fp-constraint-enforcement-test";
-
-    const raw = (db as unknown as { $client: import("bun:sqlite").Database })
-      .$client;
-
-    raw.run(`
-      INSERT INTO memory_items (
-        id, kind, subject, statement, status, confidence, importance,
-        fingerprint, verification_state, scope_id, first_seen_at, last_seen_at
-      ) VALUES (
-        'item-constraint-a', 'preference', 'editor', 'I use VS Code.',
-        'active', 0.9, 0.7, '${fingerprint}', 'user_reported', 'default',
-        ${now}, ${now}
-      )
-    `);
-
-    // A bare INSERT (no ON CONFLICT) for the same fingerprint+scope_id must throw.
-    expect(() => {
-      raw.run(`
-        INSERT INTO memory_items (
-          id, kind, subject, statement, status, confidence, importance,
-          fingerprint, verification_state, scope_id, first_seen_at, last_seen_at
-        ) VALUES (
-          'item-constraint-b', 'preference', 'editor', 'I use VS Code.',
-          'active', 0.9, 0.7, '${fingerprint}', 'user_reported', 'default',
-          ${now + 1}, ${now + 1}
-        )
-      `);
-    }).toThrow();
-  });
-});

package/src/__tests__/migration-export-streaming.test.ts

@@ -0,0 +1,304 @@
+/**
+ * Integration tests for the streaming vbundle export builder.
+ *
+ * Tests cover:
+ * - Format parity: streaming builder produces archives identical to sync builder
+ * - SHA-256 integrity: manifest checksums match actual file content
+ * - Temp file lifecycle: cleanup removes the temp file
+ * - Cleanup idempotency: calling cleanup twice doesn't throw
+ * - Round-trip: streamExportVBundle -> validateVBundle succeeds
+ */
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { gunzipSync } from "node:zlib";
+import { beforeAll, describe, expect, mock, test } from "bun:test";
+
+const testDir = process.env.VELLUM_WORKSPACE_DIR!;
+const testDbDir = join(testDir, "data", "db");
+const testDbPath = join(testDbDir, "assistant.db");
+const testConfigPath = join(testDir, "config.json");
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+mock.module("../permissions/trust-store.js", () => ({
+  getAllRules: () => [],
+  isStarterBundleAccepted: () => false,
+  clearCache: () => {},
+}));
+
+mock.module("../config/loader.js", () => ({
+  getConfig: () => ({
+    ui: {},
+    model: "test",
+    provider: "test",
+    memory: { enabled: false },
+    rateLimit: { maxRequestsPerMinute: 0 },
+    secretDetection: { enabled: false },
+  }),
+}));
+
+mock.module("../config/env.js", () => ({
+  isHttpAuthDisabled: () => true,
+  hasUngatedHttpAuthDisabled: () => false,
+  getGatewayInternalBaseUrl: () => "http://127.0.0.1:7830",
+  getGatewayPort: () => 7830,
+  getRuntimeHttpPort: () => 7821,
+  getRuntimeHttpHost: () => "127.0.0.1",
+  getRuntimeGatewayOriginSecret: () => undefined,
+  getIngressPublicBaseUrl: () => undefined,
+  setIngressPublicBaseUrl: () => {},
+}));
+
+import {
+  buildExportVBundle,
+  streamExportVBundle,
+} from "../runtime/migrations/vbundle-builder.js";
+import { validateVBundle } from "../runtime/migrations/vbundle-validator.js";
+
+// Test fixture data: a minimal SQLite header to simulate a real database file
+const SQLITE_HEADER = new Uint8Array([
+  0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74,
+  0x20, 0x33, 0x00,
+]);
+const TEST_CONFIG = { provider: "anthropic", model: "test-model" };
+
+// Generate a synthetic large file (~10 MB of repeated data)
+const LARGE_FILE_SIZE = 10 * 1024 * 1024; // 10 MB
+const LARGE_FILE_NAME = "large-test-data.txt";
+
+function generateLargeFileContent(): Buffer {
+  const line =
+    "This is a line of repeated test data for streaming export verification.\n";
+  const lineBuffer = Buffer.from(line);
+  const buf = Buffer.alloc(LARGE_FILE_SIZE);
+  for (let offset = 0; offset < LARGE_FILE_SIZE; offset += lineBuffer.length) {
+    const remaining = LARGE_FILE_SIZE - offset;
+    lineBuffer.copy(buf, offset, 0, Math.min(lineBuffer.length, remaining));
+  }
+  return buf;
+}
+
+beforeAll(() => {
+  // Write test fixture files so the export reads real data
+  mkdirSync(testDbDir, { recursive: true });
+  writeFileSync(testDbPath, SQLITE_HEADER);
+  writeFileSync(testConfigPath, JSON.stringify(TEST_CONFIG, null, 2));
+  writeFileSync(join(testDir, LARGE_FILE_NAME), generateLargeFileContent());
+});
+
+// ---------------------------------------------------------------------------
+// Tar parsing helper (mirrors vbundle-validator's internal parser)
+// ---------------------------------------------------------------------------
+
+interface TarEntry {
+  name: string;
+  data: Uint8Array;
+}
+
+function parseTarEntries(gzippedData: Uint8Array): TarEntry[] {
+  const tarData = gunzipSync(gzippedData);
+  const entries: TarEntry[] = [];
+  let offset = 0;
+  const BLOCK_SIZE = 512;
+
+  while (offset + BLOCK_SIZE <= tarData.length) {
+    const header = tarData.subarray(offset, offset + BLOCK_SIZE);
+    if (header.every((b) => b === 0)) break;
+
+    let end = 0;
+    while (end < 100 && header[end] !== 0) end++;
+    const name = new TextDecoder().decode(header.subarray(0, end));
+
+    let sizeEnd = 124;
+    while (sizeEnd < 136 && header[sizeEnd] !== 0) sizeEnd++;
+    const sizeStr = new TextDecoder().decode(header.subarray(124, sizeEnd));
+    const size = parseInt(sizeStr, 8) || 0;
+
+    const dataStart = offset + BLOCK_SIZE;
+    const data = tarData.subarray(dataStart, dataStart + size);
+    const dataBlocks = Math.ceil(size / BLOCK_SIZE);
+
+    if (header[156] === "0".charCodeAt(0) || header[156] === 0) {
+      entries.push({ name, data: new Uint8Array(data) });
+    }
+
+    offset = dataStart + dataBlocks * BLOCK_SIZE;
+  }
+
+  return entries;
+}
+
+function sha256Hex(data: Uint8Array): string {
+  return createHash("sha256").update(data).digest("hex");
+}
+
+// ---------------------------------------------------------------------------
+// Streaming export tests
+// ---------------------------------------------------------------------------
+
+describe("streamExportVBundle", () => {
+  test("returns a temp file that exists on disk", async () => {
+    const result = await streamExportVBundle({
+      workspaceDir: testDir,
+    });
+
+    try {
+      expect(existsSync(result.tempPath)).toBe(true);
+      expect(result.size).toBeGreaterThan(0);
+    } finally {
+      await result.cleanup();
+    }
+  });
+
+  test("manifest contains correct SHA-256 checksums for all files", async () => {
+    const result = await streamExportVBundle({
+      workspaceDir: testDir,
+    });
+
+    try {
+      const archiveData = new Uint8Array(readFileSync(result.tempPath));
+      const entries = parseTarEntries(archiveData);
+
+      for (const fileEntry of result.manifest.files) {
+        const tarEntry = entries.find((e) => e.name === fileEntry.path);
+        expect(tarEntry).toBeDefined();
+        expect(sha256Hex(tarEntry!.data)).toBe(fileEntry.sha256);
+      }
+    } finally {
+      await result.cleanup();
+    }
+  });
+
+  test("decompressed archive produces same files as buildExportVBundle", async () => {
+    const streamResult = await streamExportVBundle({
+      workspaceDir: testDir,
+    });
+
+    try {
+      const syncResult = buildExportVBundle({
+        workspaceDir: testDir,
+      });
+
+      const streamArchiveData = new Uint8Array(
+        readFileSync(streamResult.tempPath),
+      );
+      const streamEntries = parseTarEntries(streamArchiveData);
+      const syncEntries = parseTarEntries(syncResult.archive);
+
+      // Filter out manifest.json — compare only data files
+      const streamFileEntries = streamEntries
+        .filter((e) => e.name !== "manifest.json")
+        .sort((a, b) => a.name.localeCompare(b.name));
+      const syncFileEntries = syncEntries
+        .filter((e) => e.name !== "manifest.json")
+        .sort((a, b) => a.name.localeCompare(b.name));
+
+      // Same set of files
+      expect(streamFileEntries.map((e) => e.name)).toEqual(
+        syncFileEntries.map((e) => e.name),
+      );
+
+      // Same content (verified via checksums)
+      for (let i = 0; i < streamFileEntries.length; i++) {
+        expect(sha256Hex(streamFileEntries[i].data)).toBe(
+          sha256Hex(syncFileEntries[i].data),
+        );
+      }
+
+      // Same manifest file entries (excluding timing fields)
+      const streamManifestFiles = streamResult.manifest.files
+        .slice()
+        .sort((a, b) => a.path.localeCompare(b.path));
+      const syncManifestFiles = syncResult.manifest.files
+        .slice()
+        .sort((a, b) => a.path.localeCompare(b.path));
+
+      expect(streamManifestFiles.map((f) => f.path)).toEqual(
+        syncManifestFiles.map((f) => f.path),
+      );
+      expect(streamManifestFiles.map((f) => f.sha256)).toEqual(
+        syncManifestFiles.map((f) => f.sha256),
+      );
+      expect(streamManifestFiles.map((f) => f.size)).toEqual(
+        syncManifestFiles.map((f) => f.size),
+      );
+    } finally {
+      await streamResult.cleanup();
+    }
+  });
+
+  test("cleanup removes the temp file", async () => {
+    const result = await streamExportVBundle({
+      workspaceDir: testDir,
+    });
+
+    expect(existsSync(result.tempPath)).toBe(true);
+    await result.cleanup();
+    expect(existsSync(result.tempPath)).toBe(false);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Round-trip test
+// ---------------------------------------------------------------------------
+
+describe("streamExportVBundle round-trip", () => {
+  test("streaming archive passes validateVBundle with matching manifest", async () => {
+    const result = await streamExportVBundle({
+      workspaceDir: testDir,
+    });
+
+    try {
+      const archiveData = new Uint8Array(readFileSync(result.tempPath));
+      const validationResult = validateVBundle(archiveData);
+
+      expect(validationResult.is_valid).toBe(true);
+      expect(validationResult.errors).toHaveLength(0);
+      expect(validationResult.manifest).toBeDefined();
+
+      // Verify manifest fields match
+      expect(validationResult.manifest!.schema_version).toBe(
+        result.manifest.schema_version,
+      );
+      expect(validationResult.manifest!.manifest_sha256).toBe(
+        result.manifest.manifest_sha256,
+      );
+
+      // Verify file entries match
+      const validatedFiles = validationResult
+        .manifest!.files.slice()
+        .sort((a, b) => a.path.localeCompare(b.path));
+      const resultFiles = result.manifest.files
+        .slice()
+        .sort((a, b) => a.path.localeCompare(b.path));
+
+      expect(validatedFiles).toEqual(resultFiles);
+    } finally {
+      await result.cleanup();
+    }
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Cleanup idempotency test
+// ---------------------------------------------------------------------------
+
+describe("streamExportVBundle cleanup", () => {
+  test("calling cleanup twice does not throw", async () => {
+    const result = await streamExportVBundle({
+      workspaceDir: testDir,
+    });
+
+    await result.cleanup();
+    // Second call should not throw
+    await result.cleanup();
+
+    expect(existsSync(result.tempPath)).toBe(false);
+  });
+});

package/src/__tests__/migration-import-commit-http.test.ts

@@ -355,10 +355,10 @@ describe("handleMigrationImport", () => {
     expect(writtenData).toEqual(newDbData);
   });
 
-  test("workspace is cleared before restore — files are created fresh", async () => {
-    // Only new-format bundles (workspace/ prefix) trigger workspace clearing.
-    // With clearing, existing files are removed before writing, so all files
-    // in the bundle are "created" (not "overwritten") and no backups are made.
+  test("workspace is cleared before restore — preserved dirs are overwritten", async () => {
+    // New-format bundles (workspace/ prefix) trigger workspace clearing, but
+    // data/db/ is preserved during clearing to avoid destroying the database
+    // if the import fails partway. The DB file is overwritten (with backup).
     const newDbData = new Uint8Array([0x01, 0x02, 0x03]);
     const vbundle = createValidVBundle([
       { path: "workspace/data/db/assistant.db", data: newDbData },
@@ -373,17 +373,18 @@ describe("handleMigrationImport", () => {
     const body = (await res.json()) as ImportCommitResponse;
 
     expect(body.success).toBe(true);
-    expect(body.summary.backups_created).toBe(0);
+    expect(body.summary.backups_created).toBe(1);
 
     const dbFile = body.files.find(
       (f) => f.path === "workspace/data/db/assistant.db",
     );
     expect(dbFile).toBeDefined();
-    expect(dbFile!.action).toBe("created");
+    expect(dbFile!.action).toBe("overwritten");
   });
 
-  test("reports all files as created after workspace clear", async () => {
-    // New-format workspace/ paths trigger clearing — both files are fresh.
+  test("workspace clear: preserved dirs overwritten, cleared dirs created", async () => {
+    // data/db/ is preserved during workspace clearing → DB is "overwritten".
+    // config.json lives at the workspace root which IS cleared → "created".
     const newDbData = new Uint8Array([0xaa, 0xbb]);
     const newConfigData = new TextEncoder().encode('{"provider":"openai"}');
     const vbundle = createValidVBundle([
@@ -408,8 +409,8 @@ describe("handleMigrationImport", () => {
       (f) => f.path === "workspace/config.json",
     );
 
-    // Both are "created" because workspace was cleared before writing
-    expect(dbFile!.action).toBe("created");
+    // data/db/ preserved during clearing → overwritten; config.json cleared → created
+    expect(dbFile!.action).toBe("overwritten");
     expect(configFile!.action).toBe("created");
   });
 

package/src/__tests__/mock-fetch.ts

@@ -0,0 +1,87 @@
+/**
+ * Global test utility for mocking `globalThis.fetch`.
+ *
+ * Usage:
+ *   mockFetch("/some/path", { method: "POST" }, { body: { ok: true }, status: 200 });
+ *
+ * Entries are consumed in order — the first match wins and is removed, so
+ * register multiple responses for the same path to simulate sequences.
+ */
+
+interface MockedResponse {
+  body?: unknown;
+  status: number;
+}
+
+interface MockFetchEntry {
+  path: string;
+  init: Partial<RequestInit>;
+  response: MockedResponse | Response;
+}
+
+const entries: MockFetchEntry[] = [];
+const calls: { path: string; init: RequestInit }[] = [];
+let originalFetch: typeof globalThis.fetch | undefined;
+
+export function mockFetch(
+  path: string,
+  init: Partial<RequestInit>,
+  response: MockedResponse | Response,
+): void {
+  if (!originalFetch) {
+    originalFetch = globalThis.fetch;
+  }
+
+  entries.push({ path, init, response });
+
+  globalThis.fetch = (async (
+    input: RequestInfo | URL,
+    actualInit?: RequestInit,
+  ) => {
+    const url = String(input);
+    calls.push({ path: url, init: actualInit ?? {} });
+
+    const idx = entries.findIndex((e) => {
+      if (!url.includes(e.path)) return false;
+      for (const [key, val] of Object.entries(e.init)) {
+        if (
+          (actualInit as Record<string, unknown> | undefined)?.[key] !== val
+        ) {
+          return false;
+        }
+      }
+      return true;
+    });
+
+    if (idx === -1) {
+      return new Response(JSON.stringify({ detail: "No mock matched" }), {
+        status: 500,
+      });
+    }
+
+    const entry = entries[idx];
+    entries.splice(idx, 1);
+
+    if (entry.response instanceof Response) {
+      return entry.response;
+    }
+
+    return new Response(JSON.stringify(entry.response.body ?? null), {
+      status: entry.response.status,
+      headers: { "Content-Type": "application/json" },
+    });
+  }) as unknown as typeof globalThis.fetch;
+}
+
+export function getMockFetchCalls(): { path: string; init: RequestInit }[] {
+  return calls;
+}
+
+export function resetMockFetch(): void {
+  if (originalFetch) {
+    globalThis.fetch = originalFetch;
+    originalFetch = undefined;
+  }
+  entries.length = 0;
+  calls.length = 0;
+}

package/src/__tests__/non-member-access-request.test.ts

@@ -7,7 +7,7 @@
  * 3. Create a guardian approval request for the access request
  * 4. Deduplicate: don't create duplicate requests for repeated messages
  */
-import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test";
+import { beforeEach, describe, expect, mock, test } from "bun:test";
 
 // ---------------------------------------------------------------------------
 // Test isolation: in-memory SQLite via temp directory
@@ -66,17 +66,13 @@ import {
   listCanonicalGuardianDeliveries,
   listCanonicalGuardianRequests,
 } from "../memory/canonical-guardian-store.js";
-import { getDb, initializeDb, resetDb } from "../memory/db.js";
+import { getDb, initializeDb } from "../memory/db.js";
 import { notifyGuardianOfAccessRequest } from "../runtime/access-request-helper.js";
 import { ensureVellumGuardianBinding } from "../runtime/guardian-vellum-migration.js";
 import { handleChannelInbound } from "../runtime/routes/channel-routes.js";
 
 initializeDb();
 
-afterAll(() => {
-  resetDb();
-});
-
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------