@vellumai/assistant 0.4.56 → 0.5.0

package/src/runtime/migrations/vbundle-builder.ts

@@ -3,12 +3,15 @@
  *
  * A .vbundle is a gzip-compressed tar archive containing:
  * - manifest.json: metadata with schema_version, checksums, and bundle info
- * - data/db/assistant.db: the SQLite database with conversations and memory
- * - config/settings.json: the assistant configuration
+ * - workspace/: the entire ~/.vellum/workspace/ directory tree (DB, config,
+ *   skills, hooks, prompts, attachments, etc.) — excluding large/regenerable
+ *   dirs (embedding-models/, data/qdrant/)
+ * - trust/trust.json: trust rules (optional, lives in protected/ outside workspace)
  */
 
 import { createHash } from "node:crypto";
-import { existsSync, readFileSync } from "node:fs";
+import { existsSync, lstatSync, readdirSync, readFileSync } from "node:fs";
+import { join, relative } from "node:path";
 import { gzipSync } from "node:zlib";
 
 import type {
@@ -107,12 +110,72 @@ function computeHeaderChecksum(header: Uint8Array): number {
   return sum;
 }
 
-function createTarEntry(name: string, data: Uint8Array): Uint8Array {
-  const header = new Uint8Array(BLOCK_SIZE);
+/**
+ * Build a PAX extended header entry for paths that exceed the 100-byte ustar
+ * limit. The PAX entry is a special tar record (typeflag 'x') whose body
+ * contains "key=value" records. The following data entry uses a truncated name
+ * in its ustar header, but tar extractors use the PAX path attribute instead.
+ */
+function createPaxPathEntry(name: string): Uint8Array {
   const encoder = new TextEncoder();
 
-  // File name (0-99)
+  // Build PAX payload: "<length> path=<name>\n"
+  // The length field includes itself, the space, and the trailing newline.
+  const record = `path=${name}\n`;
+  // Start with a guess for the decimal length prefix
+  let prefix = `${record.length + 2} `; // +2 for prefix digit + space (min)
+  let full = `${prefix}${record}`;
+  // Iterate until the length prefix is self-consistent
+  while (new TextEncoder().encode(full).length !== Number.parseInt(prefix)) {
+    prefix = `${new TextEncoder().encode(full).length} `;
+    full = `${prefix}${record}`;
+  }
+  const paxData = encoder.encode(full);
+
+  // Build a ustar header for the PAX entry itself
+  const header = new Uint8Array(BLOCK_SIZE);
+
+  // Use a synthetic name for the PAX header entry
+  const paxName = encoder.encode("PaxHeader/entry");
+  header.set(paxName.subarray(0, 100), 0);
+
+  writeOctal(header, 100, 8, 0o644);
+  writeOctal(header, 108, 8, 0);
+  writeOctal(header, 116, 8, 0);
+  writeOctal(header, 124, 12, paxData.length);
+  writeOctal(header, 136, 12, Math.floor(Date.now() / 1000));
+
+  // Type flag 'x' = PAX extended header for the next entry
+  header[156] = "x".charCodeAt(0);
+
+  const magic = encoder.encode("ustar\0");
+  header.set(magic, 257);
+  header[263] = "0".charCodeAt(0);
+  header[264] = "0".charCodeAt(0);
+
+  const checksum = computeHeaderChecksum(header);
+  writeOctal(header, 148, 7, checksum);
+  header[155] = 0x20;
+
+  const paddedData = padToBlock(paxData);
+  const result = new Uint8Array(header.length + paddedData.length);
+  result.set(header, 0);
+  result.set(paddedData, header.length);
+  return result;
+}
+
+function createTarEntry(name: string, data: Uint8Array): Uint8Array {
+  const encoder = new TextEncoder();
   const nameBytes = encoder.encode(name);
+
+  // If the name exceeds 100 bytes, emit a PAX extended header first
+  // so that the full path is preserved in the archive.
+  const needsPax = nameBytes.length > 100;
+  const paxEntry = needsPax ? createPaxPathEntry(name) : null;
+
+  const header = new Uint8Array(BLOCK_SIZE);
+
+  // File name (0-99) — truncated if >100 bytes; PAX header carries the full name
   header.set(nameBytes.subarray(0, 100), 0);
 
   // File mode (100-107): 0644
@@ -148,10 +211,18 @@ function createTarEntry(name: string, data: Uint8Array): Uint8Array {
 
   // Combine header + padded data
   const paddedData = padToBlock(data);
-  const result = new Uint8Array(header.length + paddedData.length);
-  result.set(header, 0);
-  result.set(paddedData, header.length);
-  return result;
+  const fileEntry = new Uint8Array(header.length + paddedData.length);
+  fileEntry.set(header, 0);
+  fileEntry.set(paddedData, header.length);
+
+  if (paxEntry) {
+    const result = new Uint8Array(paxEntry.length + fileEntry.length);
+    result.set(paxEntry, 0);
+    result.set(fileEntry, paxEntry.length);
+    return result;
+  }
+
+  return fileEntry;
 }
 
 function createTarArchive(
@@ -230,24 +301,119 @@ export function buildVBundle(options: BuildVBundleOptions): BuildVBundleResult {
   return { archive, manifest };
 }
 
+// ---------------------------------------------------------------------------
+// Directory walker — recursively collects files for archive inclusion
+// ---------------------------------------------------------------------------
+
+interface WalkDirectoryOptions {
+  /** Include binary files (files containing null bytes). Default: false. */
+  includeBinary?: boolean;
+  /** Directory names to skip (matched against immediate child name). */
+  skipDirs?: string[];
+}
+
+/**
+ * Recursively walk a directory and return all non-symlink files as
+ * VBundleFileEntry objects with paths prefixed by `archivePrefix`.
+ *
+ * By default, binary files (detected via null-byte heuristic in the first
+ * 8 KB) are skipped. Pass `includeBinary: true` to include them.
+ */
+function walkDirectory(
+  dir: string,
+  archivePrefix: string,
+  options: WalkDirectoryOptions = {},
+): VBundleFileEntry[] {
+  const { includeBinary = false, skipDirs = [] } = options;
+  const entries: VBundleFileEntry[] = [];
+
+  function walk(currentDir: string): void {
+    const dirEntries = readdirSync(currentDir, { withFileTypes: true });
+    for (const entry of dirEntries) {
+      const fullPath = join(currentDir, entry.name);
+
+      // Skip symlinks
+      const stat = lstatSync(fullPath);
+      if (stat.isSymbolicLink()) continue;
+
+      if (stat.isDirectory()) {
+        // Check skip list against the relative path from the walk root
+        const relDir = relative(dir, fullPath);
+        if (skipDirs.some((s) => relDir === s || relDir.startsWith(s + "/"))) {
+          continue;
+        }
+        walk(fullPath);
+      } else if (stat.isFile()) {
+        // Skip SQLite auxiliary files — these are ephemeral and race-prone
+        // with the live DB connection. The WAL is checkpointed before the
+        // walk, so the main .db file has all committed rows.
+        if (
+          entry.name.endsWith(".db-wal") ||
+          entry.name.endsWith(".db-shm") ||
+          entry.name.endsWith(".db-journal")
+        ) {
+          continue;
+        }
+
+        const data = new Uint8Array(readFileSync(fullPath));
+
+        // Skip binary files unless explicitly included
+        if (!includeBinary) {
+          const checkLength = Math.min(data.length, 8192);
+          let isBinary = false;
+          for (let i = 0; i < checkLength; i++) {
+            if (data[i] === 0) {
+              isBinary = true;
+              break;
+            }
+          }
+          if (isBinary) continue;
+        }
+
+        const relativePath = relative(dir, fullPath);
+        entries.push({
+          path: `${archivePrefix}/${relativePath}`,
+          data,
+        });
+      }
+    }
+  }
+
+  walk(dir);
+  return entries;
+}
+
 // ---------------------------------------------------------------------------
 // Export builder — reads real data from disk
 // ---------------------------------------------------------------------------
 
 export interface BuildExportVBundleOptions {
-  /** Path to the SQLite database file (e.g. ~/.vellum/workspace/data/db/assistant.db). */
-  dbPath: string;
-  /** Path to the config file (e.g. ~/.vellum/workspace/config.json). */
-  configPath: string;
   /** Source identifier. Defaults to "runtime-export". */
   source?: string;
   /** Human-readable description. */
   description?: string;
+  /** Absolute path to trust.json. If provided and the file exists, it is included in the archive. */
+  trustPath?: string;
+  /**
+   * Absolute path to the hooks directory (~/.vellum/hooks/).
+   * Hooks live outside the workspace, so they must be included explicitly.
+   * Included in the archive under the "hooks/" prefix for backward compat.
+   */
+  hooksDir?: string;
+  /**
+   * Absolute path to the workspace directory (~/.vellum/workspace/).
+   * When provided and exists, the entire directory tree is walked and
+   * included in the archive under the "workspace/" prefix, skipping
+   * large/regenerable dirs (embedding-models/, data/qdrant/).
+   * Binary files (SQLite DB, attachments) are included.
+   */
+  workspaceDir?: string;
   /**
    * Optional callback to checkpoint the WAL before reading the database file.
    * In WAL mode, committed rows may live in the -wal file and not yet be
    * flushed to the main .db file. Callers should pass a function that runs
    * PRAGMA wal_checkpoint(TRUNCATE) on the live database connection.
+   * Called before the workspace walk so the DB file is up to date.
    */
   checkpoint?: () => void;
 }
@@ -255,18 +421,20 @@ export interface BuildExportVBundleOptions {
 /**
  * Build a .vbundle archive populated with real assistant data.
  *
- * Reads the actual SQLite database (which contains all conversations,
- * messages, memory segments, embeddings, and other assistant state) and
- * the config file from disk. Returns a complete, self-validating archive
- * ready for migration import.
+ * Walks the entire workspace directory (~/.vellum/workspace/) and includes
+ * all files in the archive, skipping only large/regenerable directories
+ * (embedding-models/, data/qdrant/). Binary files (SQLite DB, attachments)
+ * are included. Trust rules (in protected/, outside workspace) are handled
+ * separately.
  *
- * Falls back gracefully: if the database does not exist, an empty file is
- * included. If the config does not exist, an empty JSON object is used.
+ * The WAL is checkpointed before the walk so the exported DB file contains
+ * all committed rows.
  */
 export function buildExportVBundle(
   options: BuildExportVBundleOptions,
 ): BuildVBundleResult {
-  const { dbPath, configPath, source, description, checkpoint } = options;
+  const { source, description, checkpoint, trustPath, workspaceDir, hooksDir } =
+    options;
 
   // Flush WAL to the main database file before reading so the export
   // captures all committed rows (SQLite WAL mode keeps recent writes
@@ -275,20 +443,32 @@ export function buildExportVBundle(
     checkpoint();
   }
 
-  const dbData = existsSync(dbPath)
-    ? new Uint8Array(readFileSync(dbPath))
-    : new Uint8Array(0);
+  const files: VBundleFileEntry[] = [];
+
+  // Walk the entire workspace directory, including binary files (DB,
+  // attachments) but skipping large/regenerable subdirectories.
+  if (workspaceDir && existsSync(workspaceDir) && lstatSync(workspaceDir).isDirectory()) {
+    files.push(
+      ...walkDirectory(workspaceDir, "workspace", {
+        includeBinary: true,
+        skipDirs: ["embedding-models", "data/qdrant"],
+      }),
+    );
+  }
+
+  // Include hooks directory if it exists (lives at ~/.vellum/hooks/, outside workspace).
+  if (hooksDir && existsSync(hooksDir) && lstatSync(hooksDir).isDirectory()) {
+    files.push(...walkDirectory(hooksDir, "hooks"));
+  }
 
-  // Read the config file (settings, provider config, feature flags, etc.).
-  const configData = existsSync(configPath)
-    ? new Uint8Array(readFileSync(configPath))
-    : new TextEncoder().encode("{}");
+  // Include trust rules if the file exists (lives in protected/, outside workspace).
+  if (trustPath && existsSync(trustPath)) {
+    const trustData = new Uint8Array(readFileSync(trustPath));
+    files.push({ path: "trust/trust.json", data: trustData });
+  }
 
   return buildVBundle({
-    files: [
-      { path: "data/db/assistant.db", data: dbData },
-      { path: "config/settings.json", data: configData },
-    ],
+    files,
     source: source ?? "runtime-export",
     description: description ?? "Runtime export bundle",
   });

package/src/runtime/migrations/vbundle-import-analyzer.ts

@@ -15,9 +15,18 @@
 
 import { createHash } from "node:crypto";
 import { existsSync, readFileSync, statSync } from "node:fs";
+import { join, resolve } from "node:path";
 
 import type { ManifestType } from "./vbundle-validator.js";
 
+/** Only these prompt filenames are accepted during import. */
+const ALLOWED_PROMPT_FILENAMES = new Set([
+  "IDENTITY.md",
+  "SOUL.md",
+  "USER.md",
+  "UPDATES.md",
+]);
+
 // ---------------------------------------------------------------------------
 // Public types
 // ---------------------------------------------------------------------------
@@ -79,19 +88,76 @@ export interface PathResolver {
 
 export class DefaultPathResolver implements PathResolver {
   constructor(
-    private dbPath: string,
-    private configPath: string,
+    private protectedDir?: string,
+    private workspaceDir?: string,
+    private hooksDir?: string,
   ) {}
 
   resolve(archivePath: string): string | null {
-    switch (archivePath) {
-      case "data/db/assistant.db":
-        return this.dbPath;
-      case "config/settings.json":
-        return this.configPath;
-      default:
+    // New format: workspace/ prefix — maps directly into the workspace dir
+    if (archivePath.startsWith("workspace/") && this.workspaceDir) {
+      const relPath = archivePath.slice("workspace/".length);
+      if (!relPath) return null;
+      const resolved = resolve(this.workspaceDir, relPath);
+      const wsRoot = resolve(this.workspaceDir);
+      // Path traversal containment
+      if (resolved !== wsRoot && !resolved.startsWith(wsRoot + "/")) {
+        return null;
+      }
+      return resolved;
+    }
+
+    // Backward compat: old bundle formats with specific archive paths
+    if (archivePath === "data/db/assistant.db" && this.workspaceDir) {
+      return join(this.workspaceDir, "data", "db", "assistant.db");
+    }
+    if (archivePath === "config/settings.json" && this.workspaceDir) {
+      return join(this.workspaceDir, "config.json");
+    }
+    if (archivePath === "trust/trust.json" && this.protectedDir) {
+      return join(this.protectedDir, "trust.json");
+    }
+    if (archivePath.startsWith("skills/") && this.workspaceDir) {
+      const resolved = resolve(
+        this.workspaceDir,
+        "skills",
+        archivePath.slice("skills/".length),
+      );
+      const skillsRoot = resolve(this.workspaceDir, "skills");
+      if (resolved !== skillsRoot && !resolved.startsWith(skillsRoot + "/")) {
+        return null;
+      }
+      return resolved;
+    }
+    if (archivePath.startsWith("prompts/") && this.workspaceDir) {
+      // Old bundles stored prompts as prompts/IDENTITY.md etc — these map
+      // to the workspace root (e.g. workspace/IDENTITY.md).
+      // Only accepted prompt filenames resolve — unknown entries are
+      // skipped so they cannot trigger workspace clearing.
+      const filename = archivePath.slice("prompts/".length);
+      if (!ALLOWED_PROMPT_FILENAMES.has(filename)) {
         return null;
+      }
+      const resolved = resolve(this.workspaceDir, filename);
+      const wsRoot = resolve(this.workspaceDir);
+      if (resolved !== wsRoot && !resolved.startsWith(wsRoot + "/")) {
+        return null;
+      }
+      return resolved;
     }
+    if (archivePath.startsWith("hooks/") && this.hooksDir) {
+      const resolved = resolve(
+        this.hooksDir,
+        archivePath.slice("hooks/".length),
+      );
+      const hooksRoot = resolve(this.hooksDir);
+      if (resolved !== hooksRoot && !resolved.startsWith(hooksRoot + "/")) {
+        return null;
+      }
+      return resolved;
+    }
+
+    return null;
   }
 }
 

package/src/runtime/migrations/vbundle-importer.ts

@@ -17,10 +17,12 @@ import {
   copyFileSync,
   existsSync,
   mkdirSync,
+  readdirSync,
   readFileSync,
+  rmSync,
   writeFileSync,
 } from "node:fs";
-import { dirname } from "node:path";
+import { dirname, join } from "node:path";
 
 import type { PathResolver } from "./vbundle-import-analyzer.js";
 import type { ManifestType, VBundleTarEntry } from "./vbundle-validator.js";
@@ -113,6 +115,12 @@ export interface ImportCommitOptions {
   preValidatedManifest?: ManifestType;
   /** Pre-parsed tar entries from a prior validateVBundle call. */
   preValidatedEntries?: Map<string, VBundleTarEntry>;
+  /**
+   * Absolute path to the workspace directory. When set and the bundle
+   * contains workspace/ entries, the workspace is cleared (except
+   * skip dirs) before writing to ensure an exact-match restore.
+   */
+  workspaceDir?: string;
 }
 
 /**
@@ -128,6 +136,7 @@ export function commitImport(options: ImportCommitOptions): ImportCommitResult {
     pathResolver,
     preValidatedManifest,
     preValidatedEntries,
+    workspaceDir,
   } = options;
 
   let manifest: ManifestType;
@@ -154,6 +163,62 @@ export function commitImport(options: ImportCommitOptions): ImportCommitResult {
     entryMap = validation.entries;
   }
 
+  // Directories to preserve when clearing the workspace (large/regenerable).
+  const WORKSPACE_SKIP_DIRS = new Set(["embedding-models"]);
+  // data/qdrant is nested — we skip "qdrant" inside "data/"
+  const DATA_SKIP_DIRS = new Set(["qdrant"]);
+
+  // Step 1b: Clear the workspace directory before restore if the bundle
+  // contains new-format workspace/ entries. This ensures an exact-match
+  // restore with no stale files left behind. Skips embedding-models/ and
+  // data/qdrant/ (large, regenerable).
+  //
+  // Only new-format bundles (workspace/ prefix) trigger clearing. Old-format
+  // bundles (skills/, hooks/, data/db/*, config/*) wrote specific files
+  // without clearing — preserving that behavior avoids wiping workspace
+  // data when importing legacy bundles.
+  //
+  // Gate on resolution: at least one workspace/ entry must resolve to a
+  // valid disk path. This prevents path-traversal entries (e.g.
+  // "workspace/../../etc/passwd") from triggering a workspace purge while
+  // resolving to nothing.
+  const hasWorkspaceEntries = manifest.files.some(
+    (f) => f.path.startsWith("workspace/") && !!pathResolver.resolve(f.path),
+  );
+
+  if (hasWorkspaceEntries && workspaceDir && existsSync(workspaceDir)) {
+    try {
+      // Clear workspace contents selectively, preserving skip dirs
+      const topEntries = readdirSync(workspaceDir, { withFileTypes: true });
+      for (const entry of topEntries) {
+        if (WORKSPACE_SKIP_DIRS.has(entry.name)) continue;
+
+        const entryPath = join(workspaceDir, entry.name);
+        if (entry.name === "data" && entry.isDirectory()) {
+          // Inside data/, preserve qdrant/ but clear everything else
+          const dataEntries = readdirSync(entryPath, { withFileTypes: true });
+          for (const dataEntry of dataEntries) {
+            if (DATA_SKIP_DIRS.has(dataEntry.name)) continue;
+            rmSync(join(entryPath, dataEntry.name), {
+              recursive: true,
+              force: true,
+            });
+          }
+        } else {
+          rmSync(entryPath, { recursive: true, force: true });
+        }
+      }
+    } catch (err) {
+      return {
+        ok: false,
+        reason: "write_failed",
+        message: `Failed to clear workspace directory "${workspaceDir}": ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      };
+    }
+  }
+
   // Step 2: Write files to disk with backups
   const importedFiles: ImportedFileReport[] = [];
   const warnings: string[] = [];

package/src/runtime/migrations/vbundle-validator.ts

@@ -3,8 +3,9 @@
  *
  * A .vbundle is a gzip-compressed tar archive containing:
  * - manifest.json: metadata with schema_version, checksums, and bundle info
- * - data/db/assistant.db: the SQLite database
- * - Additional config files as declared in the manifest
+ * - workspace/: the entire workspace directory tree (new format), OR
+ *   data/db/assistant.db + config/settings.json (old format)
+ * - trust/trust.json: trust rules (optional)
  *
  * Validation steps:
  * 1. Archive structure: valid gzip tar with required entries
@@ -124,6 +125,17 @@ function parseTar(buffer: Uint8Array): TarEntry[] {
       continue;
     }
 
+    // PAX extended header (type 'x') — extract path= attribute for next entry
+    if (typeFlag === "x") {
+      const paxText = new TextDecoder().decode(data);
+      const pathMatch = paxText.match(/\d+ path=([^\n]+)\n/);
+      if (pathMatch) {
+        longName = pathMatch[1];
+      }
+      offset = dataStart + dataBlocks * BLOCK_SIZE;
+      continue;
+    }
+
     // Regular file or hard link
     if (typeFlag === "0" || typeFlag === "\0" || typeFlag === "") {
       entries.push({ name: normalizePath(name), data, size });
@@ -181,7 +193,9 @@ function canonicalizeJson(obj: unknown): string {
 // Core validation
 // ---------------------------------------------------------------------------
 
-const REQUIRED_ENTRIES = ["manifest.json", "data/db/assistant.db"];
+// Only manifest.json is structurally required. The DB and config live under
+// workspace/ (new format) or data/db/ + config/ (old format) — both are valid.
+const REQUIRED_ENTRIES = ["manifest.json"];
 
 // 2 GB — must accommodate large but valid migrations from buildExportVBundle()
 const MAX_DECOMPRESSED_SIZE = 2 * 1024 * 1024 * 1024;

package/src/runtime/routes/approval-strategies/guardian-callback-strategy.ts

@@ -637,7 +637,7 @@ async function handleAccessRequestApproval(
     void emitNotificationSignal({
       sourceEventName: "ingress.trusted_contact.guardian_decision",
       sourceChannel: approval.channel as NotificationSourceChannel,
-      sourceSessionId: approval.conversationId,
+      sourceContextId: approval.conversationId,
       attentionHints: {
         requiresAction: false,
         urgency: "medium",
@@ -651,7 +651,7 @@ async function handleAccessRequestApproval(
     void emitNotificationSignal({
       sourceEventName: "ingress.trusted_contact.denied",
       sourceChannel: approval.channel as NotificationSourceChannel,
-      sourceSessionId: approval.conversationId,
+      sourceContextId: approval.conversationId,
       attentionHints: {
         requiresAction: false,
         urgency: "low",
@@ -719,7 +719,7 @@ async function handleAccessRequestApproval(
     void emitNotificationSignal({
       sourceEventName: "ingress.trusted_contact.guardian_decision",
       sourceChannel: approval.channel as NotificationSourceChannel,
-      sourceSessionId: approval.conversationId,
+      sourceContextId: approval.conversationId,
       attentionHints: {
         requiresAction: false,
         urgency: "medium",
@@ -745,7 +745,7 @@ async function handleAccessRequestApproval(
     void emitNotificationSignal({
       sourceEventName: "ingress.trusted_contact.verification_sent",
       sourceChannel: approval.channel as NotificationSourceChannel,
-      sourceSessionId: approval.conversationId,
+      sourceContextId: approval.conversationId,
       attentionHints: {
         requiresAction: false,
         urgency: "low",

package/src/runtime/routes/attachment-routes.ts

@@ -12,8 +12,8 @@ import {
 import { httpError } from "../http-errors.js";
 import type { RouteDefinition } from "../http-router.js";
 
-/** 75 MB — base64-encoded 50 MB attachment ≈ 67 MB plus JSON wrapper overhead. */
-const MAX_UPLOAD_BODY_BYTES = 75 * 1024 * 1024;
+/** 150 MB — base64-encoded 100 MB attachment ≈ 134 MB plus JSON wrapper overhead. */
+const MAX_UPLOAD_BODY_BYTES = 150 * 1024 * 1024;
 
 export async function handleUploadAttachment(req: Request): Promise<Response> {
   const rawBody = await req.arrayBuffer();