@vellumai/assistant 0.6.4 → 0.6.6

package/src/runtime/migrations/gcs-signed-url.ts

@@ -0,0 +1,162 @@
+/**
+ * Validator for Google Cloud Storage signed URLs.
+ *
+ * Used by the migrations import path to confirm that a URL the daemon is
+ * about to `fetch()` is, in fact, a signed GCS object URL (and not an
+ * attacker-chosen host, scheme, or path).
+ *
+ * This is a pure function — no I/O, no side effects. The caller decides
+ * what to do with the validation result.
+ *
+ * What we check:
+ *   - The string parses as a URL.
+ *   - The scheme is `https:` (no `http:`, `file:`, `data:`, etc.).
+ *   - The hostname is exactly `storage.googleapis.com`.
+ *   - No explicit port is present (the default HTTPS port is required;
+ *     WHATWG URL normalizes `:443` to an empty port string for HTTPS).
+ *   - The URL carries a signature query param — either `X-Goog-Signature`
+ *     (V4 signing) or `Signature` (V2 signing). If neither is present
+ *     the URL is not a signed URL and we refuse it.
+ *   - The pathname does not contain `..` segments (traversal guard).
+ *
+ * On success we return the hostname and pathname for logging/telemetry.
+ * We deliberately do NOT return the full URL or the query string,
+ * because the signature is sensitive and should not end up in logs.
+ */
+
+export type GcsUrlValidation =
+  | { ok: true; host: string; path: string }
+  | { ok: false; reason: string };
+
+const EXPECTED_HOST = "storage.googleapis.com";
+const DEFAULT_ALLOWED_HOSTS: readonly string[] = [EXPECTED_HOST];
+
+export interface ValidateGcsSignedUrlOptions {
+  /**
+   * Allowlisted hosts. Defaults to `["storage.googleapis.com"]` — the only
+   * production value. Test-only callers can widen this to point at a local
+   * HTTP fixture (the `scheme` check also relaxes to accept `http:` for
+   * non-default hosts). Production code MUST NOT pass a wider list.
+   */
+  allowedHosts?: readonly string[];
+}
+
+export function validateGcsSignedUrl(
+  raw: string,
+  options?: ValidateGcsSignedUrlOptions,
+): GcsUrlValidation {
+  let parsed: URL;
+  try {
+    parsed = new URL(raw);
+  } catch {
+    return { ok: false, reason: "invalid_url" };
+  }
+
+  const allowedHosts = options?.allowedHosts ?? DEFAULT_ALLOWED_HOSTS;
+  const isDefaultHostList =
+    allowedHosts.length === 1 && allowedHosts[0] === EXPECTED_HOST;
+
+  // When the allowlist is the production default (GCS only), require HTTPS.
+  // When a caller passes a non-default allowlist (tests pointing at a local
+  // HTTP server), allow http too — same-process tests cannot reasonably
+  // stand up HTTPS. We still refuse non-http(s) schemes (file:, data:, etc).
+  if (isDefaultHostList) {
+    if (parsed.protocol !== "https:") {
+      return { ok: false, reason: "scheme" };
+    }
+  } else {
+    if (parsed.protocol !== "https:" && parsed.protocol !== "http:") {
+      return { ok: false, reason: "scheme" };
+    }
+  }
+
+  if (!allowedHosts.includes(parsed.hostname)) {
+    return { ok: false, reason: "host" };
+  }
+
+  // Reject any explicit port. GCS signed URLs use the default HTTPS
+  // port (443), and WHATWG URL normalizes `:443` to an empty port
+  // string for HTTPS — so a correctly-issued signed URL always has
+  // `parsed.port === ""`. A non-empty port indicates an attacker is
+  // trying to redirect the fetch to a non-default port (SSRF vector).
+  //
+  // For the non-default (test) allowlist, an explicit port is expected
+  // (local servers bind to arbitrary ephemeral ports), so the check is
+  // skipped in that case.
+  if (isDefaultHostList && parsed.port !== "") {
+    return { ok: false, reason: "port" };
+  }
+
+  const hasV4 = parsed.searchParams.has("X-Goog-Signature");
+  const hasV2 = parsed.searchParams.has("Signature");
+  if (!hasV4 && !hasV2) {
+    return { ok: false, reason: "missing_signature" };
+  }
+
+  // Defense-in-depth: reject any `..` path segment. The WHATWG URL
+  // parser silently normalizes `/bucket/../foo` to `/foo`, which would
+  // hide a traversal attempt — so inspect the *raw* input before
+  // normalization in addition to the parsed pathname.
+  if (hasTraversalSegment(parsed.pathname) || hasTraversalInRawPath(raw)) {
+    return { ok: false, reason: "path_traversal" };
+  }
+
+  return { ok: true, host: parsed.hostname, path: parsed.pathname };
+}
+
+function hasTraversalSegment(pathname: string): boolean {
+  for (const segment of pathname.split("/")) {
+    if (segment === "..") return true;
+  }
+  return false;
+}
+
+/**
+ * Look for `..` path segments in the raw input before URL normalization.
+ * We slice off the scheme+authority and stop at the first `?` or `#`,
+ * then examine each segment split on both `/` and `\` (the WHATWG URL
+ * parser treats `\` as equivalent to `/` for special schemes like
+ * `https:`, so a backslash-delimited traversal would be normalized
+ * away by `new URL()`).
+ *
+ * Inside each segment we also handle percent-encoded separators
+ * (`%2F` = `/`, `%5C` = `\`): we pre-normalize those before the
+ * top-level split, and we also re-split each decoded segment on `/`
+ * and `\` to catch cases like `%2e%2e%2fother` where the encoded
+ * slash hides a traversal within a single raw segment.
+ */
+function hasTraversalInRawPath(raw: string): boolean {
+  const schemeEnd = raw.indexOf("://");
+  if (schemeEnd === -1) return false;
+  const afterScheme = raw.slice(schemeEnd + 3);
+  const pathStart = afterScheme.search(/[\/\\]/);
+  if (pathStart === -1) return false;
+  let path = afterScheme.slice(pathStart);
+  const queryIdx = path.indexOf("?");
+  if (queryIdx !== -1) path = path.slice(0, queryIdx);
+  const hashIdx = path.indexOf("#");
+  if (hashIdx !== -1) path = path.slice(0, hashIdx);
+
+  // Normalize percent-encoded forms of `/` and `\` so that encoded
+  // separators participate in the segment split.
+  const normalized = path.replace(/%2[fF]/g, "/").replace(/%5[cC]/g, "\\");
+
+  for (const segment of normalized.split(/[\/\\]/)) {
+    if (segment === "..") return true;
+    // Percent-decoded forms of ".." — e.g. "%2E%2E", ".%2e", "%2e.".
+    // Also re-split on `/` and `\` to catch encoded separators that
+    // weren't covered by the top-level normalization (e.g. a decoded
+    // segment `../other`).
+    let decoded: string;
+    try {
+      decoded = decodeURIComponent(segment);
+    } catch {
+      // Ignore malformed percent-encoding; URL parser would handle it.
+      continue;
+    }
+    for (const sub of decoded.split(/[\/\\]/)) {
+      if (sub === "..") return true;
+    }
+  }
+  return false;
+}

package/src/runtime/migrations/vbundle-builder.ts

@@ -4,7 +4,7 @@
  * A .vbundle is a gzip-compressed tar archive containing:
  * - manifest.json: metadata with schema_version, checksums, and bundle info
  * - workspace/: the entire ~/.vellum/workspace/ directory tree (DB, config,
- *   skills, hooks, prompts, attachments, etc.) — excluding large/regenerable
+ *   skills, prompts, attachments, etc.) — excluding large/regenerable
  *   dirs (embedding-models/, data/qdrant/)
  * - trust/trust.json: trust rules (optional, lives in protected/ outside workspace)
  */
@@ -429,15 +429,6 @@ export interface BuildExportVBundleOptions {
   description?: string;
   /** Absolute path to trust.json. If provided and the file exists, it is included in the archive. */
   trustPath?: string;
-  /**
-   * Absolute path to the hooks directory. Previously hooks lived outside the
-   * workspace at ~/.vellum/hooks/ and needed explicit inclusion. Now hooks
-   * live under workspace (~/.vellum/workspace/hooks/) and are included in
-   * the workspace walk. Only pass this for backward-compat scenarios where
-   * hooks are still outside the workspace; otherwise omit to avoid double
-   * export. Included in the archive under the "hooks/" prefix.
-   */
-  hooksDir?: string;
   /**
    * Absolute path to the workspace directory (~/.vellum/workspace/).
    * When provided and exists, the entire directory tree is walked and
@@ -479,7 +470,6 @@ export function buildExportVBundle(
     checkpoint,
     trustPath,
     workspaceDir,
-    hooksDir,
     credentials,
   } = options;
 
@@ -515,11 +505,6 @@ export function buildExportVBundle(
     configEntry.data = new TextEncoder().encode(sanitized);
   }
 
-  // Include hooks directory if it exists (lives at ~/.vellum/hooks/, outside workspace).
-  if (hooksDir && existsSync(hooksDir) && lstatSync(hooksDir).isDirectory()) {
-    files.push(...walkDirectory(hooksDir, "hooks"));
-  }
-
   // Include trust rules if the file exists (lives in protected/, outside workspace).
   if (trustPath && existsSync(trustPath)) {
     const trustData = new Uint8Array(readFileSync(trustPath));
@@ -820,7 +805,6 @@ export async function streamExportVBundle(
     checkpoint,
     trustPath,
     workspaceDir,
-    hooksDir,
     credentials,
   } = options;
 
@@ -845,11 +829,6 @@ export async function streamExportVBundle(
     );
   }
 
-  // Include hooks directory if it exists
-  if (hooksDir && existsSync(hooksDir) && lstatSync(hooksDir).isDirectory()) {
-    allFileMetadata.push(...walkDirectoryForMetadata(hooksDir, "hooks"));
-  }
-
   // Include trust rules if the file exists
   if (trustPath && existsSync(trustPath)) {
     const trustStat = lstatSync(trustPath);

package/src/runtime/migrations/vbundle-importer.ts

@@ -28,13 +28,59 @@ import { sanitizeConfigForTransfer } from "../../config/sanitize-for-transfer.js
 import { isGuardianPersonaCustomized } from "../../prompts/persona-resolver.js";
 import { getLogger } from "../../util/logger.js";
 import type { PathResolver } from "./vbundle-import-analyzer.js";
+import { mergeMetadataPreservingVellum } from "./vbundle-metadata-merge.js";
 import type { ManifestType, VBundleTarEntry } from "./vbundle-validator.js";
 import { validateVBundle } from "./vbundle-validator.js";
 
 const log = getLogger("vbundle-importer");
 
 /** Archive path for the legacy guardian user persona file. */
-const LEGACY_USER_MD_ARCHIVE_PATH = "prompts/USER.md";
+export const LEGACY_USER_MD_ARCHIVE_PATH = "prompts/USER.md";
+
+/**
+ * Archive paths recognized as JSON config files that must be run through
+ * `sanitizeConfigForTransfer` before writing to disk. Exported so the
+ * streaming importer can apply the same defense-in-depth treatment.
+ */
+export const CONFIG_ARCHIVE_PATHS: ReadonlySet<string> = new Set([
+  "workspace/config.json",
+  "config/settings.json",
+]);
+
+/**
+ * Archive path for the credential metadata file. On import, bundle contents
+ * must be merged with the target's live `vellum:*` entries so the gateway's
+ * `readServiceCredentials` can still locate the platform API key after a
+ * local→platform teleport. Both importers consult this constant.
+ */
+export const CREDENTIAL_METADATA_ARCHIVE_PATH =
+  "workspace/data/credentials/metadata.json";
+
+/**
+ * Paths inside the workspace directory that must be preserved across an
+ * import when the bundle does not carry entries for them.
+ *
+ * Each entry is a path RELATIVE to the workspace root. Two kinds of live
+ * data warrant carry-over:
+ *
+ * - `embedding-models` / `deprecated`: large local caches / quarantine
+ *   directories that are never shipped inside bundles but are expensive
+ *   or impossible to reconstruct from an import.
+ * - `data/db` / `data/qdrant`: user-critical state (SQLite assistant DB;
+ *   Qdrant vector store). If the bundle omits them — e.g. a partial
+ *   bundle covering only prompts/config — the live copies must survive.
+ *
+ * Both the buffer-based `commitImport` (which selectively clears the
+ * workspace in place) and the streaming importer (which swaps the
+ * workspace with a freshly-populated temp tree) consult this list so
+ * their behavior stays in sync.
+ */
+export const WORKSPACE_PRESERVE_PATHS: readonly string[] = [
+  "embedding-models",
+  "deprecated",
+  "data/db",
+  "data/qdrant",
+];
 
 // ---------------------------------------------------------------------------
 // Public types
@@ -171,10 +217,19 @@ export function commitImport(options: ImportCommitOptions): ImportCommitResult {
     entryMap = validation.entries;
   }
 
-  // Directories to preserve when clearing the workspace.
-  const WORKSPACE_SKIP_DIRS = new Set(["embedding-models", "deprecated"]);
-  // data/qdrant and data/db are nested — we skip them inside "data/"
-  const DATA_SKIP_DIRS = new Set(["qdrant", "db"]);
+  // Directories to preserve when clearing the workspace. Derived from the
+  // shared WORKSPACE_PRESERVE_PATHS list so the streaming importer's
+  // carry-over logic and this in-place clear stay in sync.
+  const WORKSPACE_SKIP_DIRS = new Set<string>();
+  const DATA_SKIP_DIRS = new Set<string>();
+  for (const rel of WORKSPACE_PRESERVE_PATHS) {
+    const parts = rel.split("/");
+    if (parts.length === 1) {
+      WORKSPACE_SKIP_DIRS.add(parts[0]);
+    } else if (parts.length === 2 && parts[0] === "data") {
+      DATA_SKIP_DIRS.add(parts[1]);
+    }
+  }
 
   // Step 1b: Clear the workspace directory before restore if the bundle
   // contains new-format workspace/ entries. This ensures an exact-match
@@ -195,6 +250,30 @@ export function commitImport(options: ImportCommitOptions): ImportCommitResult {
     (f) => f.path.startsWith("workspace/") && !!pathResolver.resolve(f.path),
   );
 
+  // Capture the target's credential metadata BEFORE the workspace clear
+  // runs. Step 1b wipes `data/credentials/`, so reading live metadata
+  // later (during the per-file write loop) would always miss. The merge
+  // helper needs this content to preserve the target's platform-identity
+  // (`vellum:*`) entries across the overwrite.
+  let liveCredentialMetadataJson: string | null = null;
+  const credentialMetadataDiskPath = pathResolver.resolve(
+    CREDENTIAL_METADATA_ARCHIVE_PATH,
+  );
+  if (credentialMetadataDiskPath && existsSync(credentialMetadataDiskPath)) {
+    try {
+      liveCredentialMetadataJson = readFileSync(
+        credentialMetadataDiskPath,
+        "utf-8",
+      );
+    } catch (err) {
+      log.warn(
+        { err, path: credentialMetadataDiskPath },
+        "Failed to read live credential metadata before import; vellum:* entries may not be preserved",
+      );
+    }
+  }
+
+  let workspaceWasCleared = false;
   if (hasWorkspaceEntries && workspaceDir && existsSync(workspaceDir)) {
     try {
       // Clear workspace contents selectively, preserving skip dirs
@@ -218,6 +297,7 @@ export function commitImport(options: ImportCommitOptions): ImportCommitResult {
           rmSync(entryPath, { recursive: true, force: true });
         }
       }
+      workspaceWasCleared = true;
     } catch (err) {
       return {
         ok: false,
@@ -358,15 +438,47 @@ export function commitImport(options: ImportCommitOptions): ImportCommitResult {
 
     // Sanitize config files to strip environment-specific fields (defense-in-depth)
     let dataToWrite: Uint8Array = archiveEntry.data;
-    if (
-      fileEntry.path === "workspace/config.json" ||
-      fileEntry.path === "config/settings.json"
-    ) {
+    if (CONFIG_ARCHIVE_PATHS.has(fileEntry.path)) {
       const configJson = new TextDecoder().decode(archiveEntry.data);
       const sanitized = sanitizeConfigForTransfer(configJson);
       dataToWrite = new TextEncoder().encode(sanitized);
     }
 
+    // Preserve target's `vellum:*` metadata entries across the overwrite.
+    // Django's post-hatch provisioning writes these on the target via
+    // POST /v1/secrets; a naive overwrite of the bundle's metadata.json
+    // would wipe them and break the gateway's vellum credential read.
+    // We use the snapshot captured BEFORE the workspace clear because
+    // Step 1b may have already removed the live file.
+    if (fileEntry.path === CREDENTIAL_METADATA_ARCHIVE_PATH) {
+      const bundleJson = new TextDecoder().decode(archiveEntry.data);
+      const merged = mergeMetadataPreservingVellum(
+        bundleJson,
+        liveCredentialMetadataJson,
+      );
+      dataToWrite = new TextEncoder().encode(merged);
+    }
+
+    // If we're about to replace a SQLite main database file, remove any
+    // pre-existing `.db-wal`/`.db-shm`/`.db-journal` siblings at the
+    // target. Those auxiliary files are only valid as a pair with the
+    // exact `.db` that wrote them; leaving them alongside a replacement
+    // DB causes SQLite to replay incompatible WAL frames on the first
+    // open and report "database disk image is malformed". The exporter
+    // already checkpointed the source WAL into the main DB before the
+    // bundle was built, so dropping the sibling aux files doesn't lose
+    // data from the source workspace.
+    if (diskPath.endsWith(".db")) {
+      for (const suffix of [".db-wal", ".db-shm", ".db-journal"]) {
+        const auxPath = `${diskPath.slice(0, -".db".length)}${suffix}`;
+        try {
+          rmSync(auxPath, { force: true });
+        } catch {
+          /* best effort — if the aux file doesn't exist we're fine */
+        }
+      }
+    }
+
     // Write the file
     try {
       writeFileSync(diskPath, dataToWrite);
@@ -416,6 +528,39 @@ export function commitImport(options: ImportCommitOptions): ImportCommitResult {
     });
   }
 
+  // If Step 1b actually cleared the workspace AND the bundle did not
+  // carry a metadata.json entry, the target's vellum:* entries were
+  // wiped along with the `data/credentials/` directory. Restore them by
+  // writing a minimal file containing just the preserved entries so the
+  // gateway can still locate the platform API key. When Step 1b did NOT
+  // run (e.g. workspaceDir unset) the live metadata.json is still on
+  // disk untouched — we must not rewrite it here or we would drop the
+  // non-vellum entries the caller chose to keep.
+  const bundleHadMetadata = manifest.files.some(
+    (f) => f.path === CREDENTIAL_METADATA_ARCHIVE_PATH,
+  );
+  if (
+    workspaceWasCleared &&
+    !bundleHadMetadata &&
+    liveCredentialMetadataJson &&
+    credentialMetadataDiskPath
+  ) {
+    const merged = mergeMetadataPreservingVellum(
+      JSON.stringify({ version: 5, credentials: [] }),
+      liveCredentialMetadataJson,
+    );
+    try {
+      mkdirSync(dirname(credentialMetadataDiskPath), { recursive: true });
+      writeFileSync(credentialMetadataDiskPath, merged);
+    } catch (err) {
+      warnings.push(
+        `Failed to restore preserved vellum:* credential metadata: ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      );
+    }
+  }
+
   // Build final report
   const report: ImportCommitReport = {
     success: true,

package/src/runtime/migrations/vbundle-metadata-merge.ts

@@ -0,0 +1,124 @@
+/**
+ * Merge helper for `data/credentials/metadata.json` on bundle import.
+ *
+ * The credential metadata file lists the credentials known to the assistant
+ * (service, field, policy, timestamps) and is used by the gateway's
+ * `readServiceCredentials` to decide whether a service is "configured".
+ * The VELLUM spec requires all four `vellum:*` fields (`platform_base_url`,
+ * `assistant_api_key`, `platform_assistant_id`, `webhook_secret`) to be
+ * present in metadata before the gateway will even look up their values in
+ * CES.
+ *
+ * On a local→platform teleport, the bundle carries the SOURCE's metadata
+ * (no `vellum:*` entries, since the source is local), and a naive overwrite
+ * would wipe out the `vellum:*` entries that Django's post-hatch
+ * provisioning just wrote on the TARGET. This module merges bundle and live
+ * metadata with one rule:
+ *
+ *   - Drop `service === "vellum"` entries the bundle tries to ship
+ *     (defense-in-depth — they represent the source's identity, not the
+ *     target's). This mirrors the CES-side filter in migration-routes.ts.
+ *   - Preserve every `service === "vellum"` entry the target already has.
+ *   - Import bundle entries for every other service normally (user OAuth,
+ *     channel credentials).
+ *
+ * Malformed input (missing file, unparseable JSON, unrecognized schema) is
+ * handled by the callers: they should treat "no live metadata" as no
+ * preservation needed, and leave the bundle's file untouched if its schema
+ * can't be merged cleanly.
+ */
+
+const VELLUM_SERVICE = "vellum";
+
+interface MetadataRecord {
+  credentialId: string;
+  service: string;
+  field: string;
+  [key: string]: unknown;
+}
+
+interface MetadataFile {
+  version?: number;
+  credentials?: unknown[];
+  [key: string]: unknown;
+}
+
+function isRecord(value: unknown): value is MetadataRecord {
+  if (typeof value !== "object" || value == null) return false;
+  const r = value as Record<string, unknown>;
+  return (
+    typeof r.credentialId === "string" &&
+    typeof r.service === "string" &&
+    typeof r.field === "string"
+  );
+}
+
+function parseMetadata(json: string | null | undefined): MetadataFile | null {
+  if (!json) return null;
+  try {
+    const parsed: unknown = JSON.parse(json);
+    if (typeof parsed !== "object" || parsed == null || Array.isArray(parsed)) {
+      return null;
+    }
+    return parsed as MetadataFile;
+  } catch {
+    return null;
+  }
+}
+
+function extractVellumRecords(file: MetadataFile | null): MetadataRecord[] {
+  if (!file || !Array.isArray(file.credentials)) return [];
+  return file.credentials
+    .filter(isRecord)
+    .filter((r) => r.service === VELLUM_SERVICE);
+}
+
+/**
+ * Merge the bundle's metadata.json content with any `vellum:*` entries
+ * present in the target's live metadata.json.
+ *
+ * Returns the merged JSON string, preserving the bundle's schema version
+ * and formatting (2-space indent). If the bundle's JSON is unparseable the
+ * original input is returned unchanged — we never want to corrupt the
+ * bundle's file by emitting an empty or restructured payload.
+ *
+ * If the live JSON is unparseable or missing, the bundle's file is returned
+ * verbatim (no preservation possible — nothing to preserve).
+ */
+export function mergeMetadataPreservingVellum(
+  bundleJson: string,
+  liveJson: string | null,
+): string {
+  const bundle = parseMetadata(bundleJson);
+  if (!bundle) return bundleJson;
+
+  const preservedVellum = extractVellumRecords(parseMetadata(liveJson));
+
+  const bundleCredentials = Array.isArray(bundle.credentials)
+    ? bundle.credentials.filter(isRecord)
+    : [];
+
+  // Drop any `service === "vellum"` entries from the bundle (defense-in-depth).
+  const filteredBundle = bundleCredentials.filter(
+    (r) => r.service !== VELLUM_SERVICE,
+  );
+
+  // Union: bundle non-vellum entries + target vellum entries. If the
+  // preserved list happens to collide with a bundle entry on credentialId,
+  // the preserved version wins (it belongs to the target's live identity).
+  const merged = [...filteredBundle, ...preservedVellum];
+
+  const output: MetadataFile = {
+    ...bundle,
+    credentials: merged,
+  };
+
+  return JSON.stringify(output, null, 2);
+}
+
+/** @internal For direct use by tests. */
+export const _internal = {
+  VELLUM_SERVICE,
+  parseMetadata,
+  extractVellumRecords,
+};