@vellumai/assistant 0.5.6 → 0.5.8

package/src/workspace/migrations/018-rekey-compound-credential-keys.ts

@@ -0,0 +1,184 @@
+import { credentialKey } from "../../security/credential-key.js";
+import { getLogger } from "../../util/logger.js";
+import type { WorkspaceMigration } from "./types.js";
+
+const log = getLogger("workspace-migrations");
+const CREDENTIAL_PREFIX = "credential/";
+
+/**
+ * Re-key compound credential storage keys from the old indexOf-based split
+ * to the new lastIndexOf-based split.
+ *
+ * The old code split "integration:google:access_token" at the first colon:
+ *   service = "integration", field = "google:access_token"
+ *   → key = "credential/integration/google:access_token"
+ *
+ * The new code splits at the last colon:
+ *   service = "integration:google", field = "access_token"
+ *   → key = "credential/integration:google/access_token"
+ *
+ * Detection heuristic: if the field portion of a stored key contains a colon,
+ * it was stored with the old indexOf logic and needs re-keying. Simple
+ * service:field names (single colon) produce the same key with both methods
+ * and don't need migration.
+ */
+export const rekeyCompoundCredentialKeysMigration: WorkspaceMigration = {
+  id: "018-rekey-compound-credential-keys",
+  description:
+    "Re-key compound credential keys from indexOf to lastIndexOf split format",
+
+  async run(_workspaceDir: string): Promise<void> {
+    const {
+      listSecureKeysAsync,
+      getSecureKeyAsync,
+      setSecureKeyAsync,
+      deleteSecureKeyAsync,
+    } = await import("../../security/secure-keys.js");
+
+    const { accounts, unreachable } = await listSecureKeysAsync();
+    if (unreachable) {
+      throw new Error(
+        "Credential store unreachable — migration will be retried on next startup",
+      );
+    }
+
+    let migratedCount = 0;
+    let failedCount = 0;
+
+    for (const account of accounts) {
+      if (!account.startsWith(CREDENTIAL_PREFIX)) continue;
+
+      const rest = account.slice(CREDENTIAL_PREFIX.length);
+      const slashIdx = rest.indexOf("/");
+      if (slashIdx < 1 || slashIdx >= rest.length - 1) continue;
+
+      const oldService = rest.slice(0, slashIdx);
+      const oldField = rest.slice(slashIdx + 1);
+
+      // Only migrate keys where the field contains a colon — these were
+      // stored using the old indexOf(":") split and need re-keying.
+      if (!oldField.includes(":")) continue;
+
+      // Reconstruct the original "service:field" name and re-split with lastIndexOf
+      const originalName = `${oldService}:${oldField}`;
+      const lastColonIdx = originalName.lastIndexOf(":");
+      const newService = originalName.slice(0, lastColonIdx);
+      const newField = originalName.slice(lastColonIdx + 1);
+      const newKey = credentialKey(newService, newField);
+
+      // Skip if the key format didn't actually change
+      if (account === newKey) continue;
+
+      // Skip if the new key already exists (idempotent — may have been
+      // partially migrated or the user already stored under the new format)
+      const existingNewValue = await getSecureKeyAsync(newKey);
+      if (existingNewValue !== undefined) {
+        // New key exists — just clean up the old orphaned key
+        await deleteSecureKeyAsync(account);
+        log.info(
+          { oldKey: account, newKey },
+          "Deleted orphaned old-format credential key (new key already exists)",
+        );
+        migratedCount++;
+        continue;
+      }
+
+      const value = await getSecureKeyAsync(account);
+      if (value === undefined) continue;
+
+      // Write new key first, then delete old key (crash-safe order)
+      const stored = await setSecureKeyAsync(newKey, value);
+      if (!stored) {
+        log.warn(
+          { oldKey: account, newKey },
+          "Failed to write re-keyed credential — skipping",
+        );
+        failedCount++;
+        continue;
+      }
+
+      await deleteSecureKeyAsync(account);
+      migratedCount++;
+      log.info({ oldKey: account, newKey }, "Re-keyed compound credential");
+    }
+
+    if (migratedCount > 0 || failedCount > 0) {
+      log.info(
+        { migratedCount, failedCount },
+        "Compound credential key migration complete",
+      );
+    }
+  },
+
+  async down(_workspaceDir: string): Promise<void> {
+    // Reverse: re-key from lastIndexOf format back to indexOf format.
+    // Keys where the service contains ":" were migrated from old format.
+    const {
+      listSecureKeysAsync,
+      getSecureKeyAsync,
+      setSecureKeyAsync,
+      deleteSecureKeyAsync,
+    } = await import("../../security/secure-keys.js");
+
+    const { accounts, unreachable } = await listSecureKeysAsync();
+    if (unreachable) {
+      throw new Error(
+        "Credential store unreachable — rollback will be retried on next startup",
+      );
+    }
+
+    let rolledBackCount = 0;
+    let failedCount = 0;
+
+    for (const account of accounts) {
+      if (!account.startsWith(CREDENTIAL_PREFIX)) continue;
+
+      const rest = account.slice(CREDENTIAL_PREFIX.length);
+      const slashIdx = rest.indexOf("/");
+      if (slashIdx < 1 || slashIdx >= rest.length - 1) continue;
+
+      const service = rest.slice(0, slashIdx);
+      const field = rest.slice(slashIdx + 1);
+
+      // Only rollback keys where the service contains ":" — these are in
+      // the new lastIndexOf format and need reverting to indexOf format.
+      if (!service.includes(":")) continue;
+
+      // Reconstruct the original name and re-split with indexOf (old format)
+      const originalName = `${service}:${field}`;
+      const firstColonIdx = originalName.indexOf(":");
+      const oldService = originalName.slice(0, firstColonIdx);
+      const oldField = originalName.slice(firstColonIdx + 1);
+      const oldKey = credentialKey(oldService, oldField);
+
+      if (account === oldKey) continue;
+
+      const value = await getSecureKeyAsync(account);
+      if (value === undefined) continue;
+
+      const stored = await setSecureKeyAsync(oldKey, value);
+      if (!stored) {
+        log.warn(
+          { newKey: account, oldKey },
+          "Failed to rollback re-keyed credential — skipping",
+        );
+        failedCount++;
+        continue;
+      }
+
+      await deleteSecureKeyAsync(account);
+      rolledBackCount++;
+      log.info(
+        { newKey: account, oldKey },
+        "Rolled back compound credential key",
+      );
+    }
+
+    if (rolledBackCount > 0 || failedCount > 0) {
+      log.info(
+        { rolledBackCount, failedCount },
+        "Compound credential key rollback complete",
+      );
+    }
+  },
+};

package/src/workspace/migrations/019-scope-journal-to-guardian.ts

@@ -0,0 +1,103 @@
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  renameSync,
+  rmdirSync,
+  statSync,
+} from "node:fs";
+import { join } from "node:path";
+
+import { desc, eq } from "drizzle-orm";
+
+import { getDb } from "../../memory/db.js";
+import { contacts } from "../../memory/schema/contacts.js";
+import type { WorkspaceMigration } from "./types.js";
+
+export const scopeJournalToGuardianMigration: WorkspaceMigration = {
+  id: "019-scope-journal-to-guardian",
+  description:
+    "Move root journal entries into per-user subdirectory for guardian",
+
+  run(workspaceDir: string): void {
+    const journalDir = join(workspaceDir, "journal");
+    if (!existsSync(journalDir)) return;
+
+    // Find .md files in the root journal directory (not in subdirs)
+    let entries: string[];
+    try {
+      entries = readdirSync(journalDir);
+    } catch {
+      return;
+    }
+    const mdFiles = entries.filter((f) => {
+      if (!f.endsWith(".md") || f.toLowerCase() === "readme.md") return false;
+      try {
+        return statSync(join(journalDir, f)).isFile();
+      } catch {
+        return false;
+      }
+    });
+    if (mdFiles.length === 0) return;
+
+    // Resolve guardian user slug (same pattern as 017-seed-persona-dirs)
+    let slug = "guardian";
+    try {
+      const db = getDb();
+      const guardian = db
+        .select()
+        .from(contacts)
+        .where(eq(contacts.role, "guardian"))
+        .orderBy(desc(contacts.createdAt))
+        .limit(1)
+        .get();
+      if (guardian?.userFile) {
+        slug = guardian.userFile.replace(/\.md$/, "");
+      }
+    } catch {
+      // DB not ready — use fallback "guardian"
+    }
+
+    // Create per-user directory and move files (renameSync preserves birthtimes)
+    const destDir = join(journalDir, slug);
+    mkdirSync(destDir, { recursive: true });
+    for (const f of mdFiles) {
+      const src = join(journalDir, f);
+      const dest = join(destDir, f);
+      if (!existsSync(dest)) {
+        renameSync(src, dest);
+      }
+    }
+  },
+
+  down(workspaceDir: string): void {
+    const journalDir = join(workspaceDir, "journal");
+    if (!existsSync(journalDir)) return;
+    let entries: string[];
+    try {
+      entries = readdirSync(journalDir);
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      const subdir = join(journalDir, entry);
+      try {
+        if (!statSync(subdir).isDirectory()) continue;
+      } catch {
+        continue;
+      }
+      for (const f of readdirSync(subdir)) {
+        if (!f.endsWith(".md")) continue;
+        const dest = join(journalDir, f);
+        if (!existsSync(dest)) {
+          renameSync(join(subdir, f), dest);
+        }
+      }
+      try {
+        rmdirSync(subdir);
+      } catch {
+        // not empty — leave it
+      }
+    }
+  },
+};

package/src/workspace/migrations/migrate-to-workspace-volume.ts

@@ -2,19 +2,25 @@
  * Workspace migration: Migrate workspace data from /data to /workspace volume.
  *
  * In the old Docker volume layout, workspace data lived at
- * `$BASE_DATA_DIR/.vellum/workspace`. In the new layout, WORKSPACE_DIR points
+ * `$BASE_DATA_DIR/.vellum/workspace`. In the new layout, VELLUM_WORKSPACE_DIR points
  * to a dedicated volume (e.g. `/workspace`). On first boot with the new layout,
  * this migration copies existing workspace data from the old location to the
  * new volume so nothing is lost.
  *
  * Idempotent:
- * - Skips if WORKSPACE_DIR is not set (non-Docker or old layout).
+ * - Skips if VELLUM_WORKSPACE_DIR is not set (non-Docker or old layout).
  * - Skips if the workspace volume already has data (config.json exists).
  * - Skips if the sentinel file exists (already migrated).
  * - Skips if the old workspace directory doesn't exist or is empty.
  */
 
-import { cpSync, existsSync, readdirSync, writeFileSync } from "node:fs";
+import {
+  cpSync,
+  existsSync,
+  readdirSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
 import { join } from "node:path";
 
 import {
@@ -28,12 +34,28 @@ const SENTINEL_FILENAME = ".workspace-volume-migrated";
 export const migrateToWorkspaceVolumeMigration: WorkspaceMigration = {
   id: "014-migrate-to-workspace-volume",
   description:
-    "Copy workspace data from old /data/.vellum/workspace to new WORKSPACE_DIR volume on first boot",
+    "Copy workspace data from old /data/.vellum/workspace to new VELLUM_WORKSPACE_DIR volume on first boot",
+
+  down(workspaceDir: string): void {
+    // This migration copies data between volumes. Actually reversing the copy
+    // (deleting data from the workspace volume) is dangerous and could cause
+    // data loss. Instead, we just remove the sentinel file so the migration
+    // will re-run and re-evaluate on next startup.
+    const sentinelPath = join(workspaceDir, SENTINEL_FILENAME);
+    if (existsSync(sentinelPath)) {
+      try {
+        unlinkSync(sentinelPath);
+      } catch {
+        // Best-effort — the migration runner's checkpoint removal will
+        // also ensure the migration re-runs.
+      }
+    }
+  },
 
   run(workspaceDir: string): void {
     const workspaceDirOverride = getWorkspaceDirOverride();
 
-    // Only relevant when WORKSPACE_DIR is explicitly set (Docker with separate volume)
+    // Only relevant when VELLUM_WORKSPACE_DIR is explicitly set (Docker with separate volume)
     if (!workspaceDirOverride) return;
 
     const sentinelPath = join(workspaceDir, SENTINEL_FILENAME);

package/src/workspace/migrations/registry.ts

@@ -10,6 +10,12 @@ import { appDirRenameMigration } from "./010-app-dir-rename.js";
 import { backfillInstallationIdMigration } from "./011-backfill-installation-id.js";
 import { renameConversationDiskViewDirsMigration } from "./012-rename-conversation-disk-view-dirs.js";
 import { repairConversationDiskViewMigration } from "./013-repair-conversation-disk-view.js";
+import { migrateCredentialsToKeychainMigration } from "./015-migrate-credentials-to-keychain.js";
+import { extractFeatureFlagsToProtectedMigration } from "./016-extract-feature-flags-to-protected.js";
+import { migrateCredentialsFromKeychainMigration } from "./016-migrate-credentials-from-keychain.js";
+import { seedPersonaDirsMigration } from "./017-seed-persona-dirs.js";
+import { rekeyCompoundCredentialKeysMigration } from "./018-rekey-compound-credential-keys.js";
+import { scopeJournalToGuardianMigration } from "./019-scope-journal-to-guardian.js";
 import { migrateToWorkspaceVolumeMigration } from "./migrate-to-workspace-volume.js";
 import type { WorkspaceMigration } from "./types.js";
 
@@ -31,4 +37,10 @@ export const WORKSPACE_MIGRATIONS: WorkspaceMigration[] = [
   renameConversationDiskViewDirsMigration,
   repairConversationDiskViewMigration,
   migrateToWorkspaceVolumeMigration,
+  migrateCredentialsToKeychainMigration,
+  migrateCredentialsFromKeychainMigration,
+  seedPersonaDirsMigration,
+  extractFeatureFlagsToProtectedMigration,
+  rekeyCompoundCredentialKeysMigration,
+  scopeJournalToGuardianMigration,
 ];

package/src/workspace/migrations/runner.ts

@@ -7,10 +7,16 @@ import type { WorkspaceMigration } from "./types.js";
 
 const log = getLogger("workspace-migrations");
 
+export function getLastWorkspaceMigrationId(
+  migrations: WorkspaceMigration[],
+): string | null {
+  return migrations.length > 0 ? migrations[migrations.length - 1].id : null;
+}
+
 export type CheckpointFile = {
   applied: Record<
     string,
-    { appliedAt: string; status?: "started" | "completed" }
+    { appliedAt: string; status?: "started" | "completed" | "rolling_back" }
   >;
 };
 
@@ -73,7 +79,7 @@ export async function runWorkspaceMigrations(
   const checkpoints = loadCheckpoints(workspaceDir);
 
   for (const [id, entry] of Object.entries(checkpoints.applied)) {
-    if (entry.status === "started") {
+    if (entry.status === "started" || entry.status === "rolling_back") {
       log.warn(
         `Workspace migration "${id}" was interrupted during a previous run; will re-run`,
       );
@@ -115,3 +121,101 @@ export async function runWorkspaceMigrations(
     saveCheckpoints(workspaceDir, checkpoints);
   }
 }
+
+/**
+ * Roll back workspace (filesystem) migrations in reverse order, stopping before
+ * the target migration.
+ *
+ * Migrations after `targetMigrationId` in the registry array are reversed in
+ * reverse order; the target migration itself is kept applied.
+ *
+ * **Usage**: Pass the full migrations array (typically `WORKSPACE_MIGRATIONS`
+ * from `registry.ts`) and the ID of the migration you want to roll back *to*.
+ * For example, `rollbackWorkspaceMigrations(dir, migrations, "010-app-dir-rename")`
+ * rolls back all applied migrations that appear after `010-app-dir-rename` in
+ * the registry.
+ *
+ * **Checkpoint state**: Each rolled-back migration's entry is deleted from the
+ * `.workspace-migrations.json` checkpoint file. If the process crashes
+ * mid-rollback, the `"rolling_back"` marker is detected and cleared by
+ * `runWorkspaceMigrations` on the next startup (it re-runs interrupted
+ * migrations).
+ *
+ * **Warning — data loss**: Every workspace migration must define a `down()`
+ * method (enforced at the type level), but some rollbacks are lossy (e.g.,
+ * file deletions or format conversions that discard the original cannot fully
+ * restore prior state). Review each migration's `down()` implementation
+ * before calling this function.
+ *
+ * **Important**: Stop the assistant before running rollbacks. Rolling back
+ * workspace migrations while the assistant is running may cause file conflicts,
+ * stale caches, or data corruption.
+ *
+ * @param workspaceDir  The workspace directory path (e.g., `~/.vellum/workspace`).
+ * @param migrations  The full ordered array of workspace migrations (from `WORKSPACE_MIGRATIONS`).
+ * @param targetMigrationId  The migration ID to roll back to (exclusive — all
+ *   migrations after this one are reversed).
+ */
+export async function rollbackWorkspaceMigrations(
+  workspaceDir: string,
+  migrations: WorkspaceMigration[],
+  targetMigrationId: string,
+): Promise<void> {
+  // Find the index of the target migration
+  const targetIndex = migrations.findIndex((m) => m.id === targetMigrationId);
+  if (targetIndex === -1) {
+    throw new Error(
+      `Target migration "${targetMigrationId}" not found in the migrations array`,
+    );
+  }
+
+  // Collect migrations that come after the target, in reverse order
+  const migrationsToRollback = migrations.slice(targetIndex + 1).reverse();
+  if (migrationsToRollback.length === 0) {
+    log.info("No migrations to roll back");
+    return;
+  }
+
+  const checkpoints = loadCheckpoints(workspaceDir);
+
+  for (const migration of migrationsToRollback) {
+    // Only roll back migrations that have been fully applied.
+    // Legacy checkpoints may not have a status field (just appliedAt) — treat
+    // missing/undefined status as completed, matching runWorkspaceMigrations behavior.
+    const entry = checkpoints.applied[migration.id];
+    if (
+      !entry ||
+      entry.status === "started" ||
+      entry.status === "rolling_back"
+    ) {
+      continue;
+    }
+
+    log.info(
+      `Rolling back workspace migration: ${migration.id} — ${migration.description}`,
+    );
+
+    // Mark as rolling_back before execution (for crash recovery)
+    checkpoints.applied[migration.id] = {
+      appliedAt: checkpoints.applied[migration.id]!.appliedAt,
+      status: "rolling_back",
+    };
+    saveCheckpoints(workspaceDir, checkpoints);
+
+    try {
+      await migration.down(workspaceDir);
+    } catch (error) {
+      log.error(
+        { migrationId: migration.id, error },
+        `Workspace migration rollback failed: ${migration.id}`,
+      );
+      throw error;
+    }
+
+    // Remove the migration entry from checkpoints
+    delete checkpoints.applied[migration.id];
+    saveCheckpoints(workspaceDir, checkpoints);
+
+    log.info(`Rolled back workspace migration: ${migration.id}`);
+  }
+}

package/src/workspace/migrations/types.ts

@@ -8,4 +8,8 @@ export interface WorkspaceMigration {
    *  Must be idempotent — safe to re-run if it was interrupted.
    *  Both synchronous and asynchronous migrations are supported. */
   run(workspaceDir: string): void | Promise<void>;
+  /** Reverse the migration. Receives the workspace directory path.
+   *  Must be idempotent — safe to re-run if it was interrupted.
+   *  Both synchronous and asynchronous rollbacks are supported. */
+  down(workspaceDir: string): void | Promise<void>;
 }

package/src/workspace/provider-commit-message-generator.ts

@@ -52,15 +52,7 @@ const KEYLESS_PROVIDERS = new Set(["ollama"]);
 const deterministicProvider = new DefaultCommitMessageProvider();
 
 function getProviderCandidates(config: ReturnType<typeof getConfig>): string[] {
-  const order = Array.isArray(config.providerOrder) ? config.providerOrder : [];
-  const seen = new Set<string>();
-  const out: string[] = [];
-  for (const name of [config.services.inference.provider, ...order]) {
-    if (seen.has(name)) continue;
-    seen.add(name);
-    out.push(name);
-  }
-  return out;
+  return [config.services.inference.provider];
 }
 
 function buildDeterministicResult(
@@ -121,7 +113,7 @@ export class ProviderCommitMessageGenerator {
 
     // ── Fallback check order (canonical) ──────────────────────────────
     // 1. disabled
-    // 2. resolve configured provider via fail-open selection:
+    // 2. resolve configured provider:
     //    - missing_provider_api_key OR provider_not_initialized
     // 3. selected-provider API key preflight (except keyless providers)
     // 4. breaker_open
@@ -138,7 +130,7 @@ export class ProviderCommitMessageGenerator {
       return buildDeterministicResult(context, "disabled");
     }
 
-    // Step 2: Resolve configured provider using fail-open semantics.
+    // Step 2: Resolve configured provider.
     // If nothing is resolvable, differentiate likely missing-key cases from
     // true registry/init failures.
     const resolved = await resolveConfiguredProvider();
@@ -168,18 +160,17 @@ export class ProviderCommitMessageGenerator {
     }
 
     const provider = resolved.provider;
-    const selectedProviderName = resolved.selectedProviderName;
+    const providerName = resolved.configuredProviderName;
 
-    // Step 2b: API key preflight for the selected provider (skip keyless).
-    if (!KEYLESS_PROVIDERS.has(selectedProviderName)) {
-      const providerApiKey = await getSecureKeyAsync(selectedProviderName);
+    // Step 2b: API key preflight for the configured provider (skip keyless).
+    if (!KEYLESS_PROVIDERS.has(providerName)) {
+      const providerApiKey = await getSecureKeyAsync(providerName);
       if (!providerApiKey) {
         log.debug(
           {
-            selectedProvider: selectedProviderName,
-            configuredProvider: config.services.inference.provider,
+            provider: providerName,
           },
-          "Selected provider API key missing; falling back to deterministic",
+          "Provider API key missing; falling back to deterministic",
         );
         return buildDeterministicResult(context, "missing_provider_api_key");
       }
@@ -211,13 +202,13 @@ export class ProviderCommitMessageGenerator {
 
     // Step 5: Fast model preflight — resolve before any provider call
     const fastModel =
-      llmConfig.providerFastModelOverrides[selectedProviderName] ??
-      PROVIDER_DEFAULT_FAST_MODELS[selectedProviderName];
+      llmConfig.providerFastModelOverrides[providerName] ??
+      PROVIDER_DEFAULT_FAST_MODELS[providerName];
 
     if (!fastModel) {
       log.debug(
         {
-          provider: selectedProviderName,
+          provider: providerName,
           configuredProvider: config.services.inference.provider,
         },
         "No fast model resolvable for provider; falling back to deterministic",