@vellumai/assistant 0.8.1 → 0.8.2

package/src/workspace/migrations/084-remove-legacy-skills-index.ts

@@ -0,0 +1,276 @@
+import * as fs from "node:fs";
+import {
+  basename,
+  dirname,
+  isAbsolute,
+  join,
+  normalize,
+  relative,
+  resolve,
+} from "node:path";
+
+import { getLogger } from "../../util/logger.js";
+import type { WorkspaceMigration } from "./types.js";
+
+const log = getLogger("workspace-migration-084-remove-legacy-skills-index");
+
+function isNotFoundError(err: unknown): boolean {
+  return (
+    typeof err === "object" &&
+    err !== null &&
+    "code" in err &&
+    err.code === "ENOENT"
+  );
+}
+
+function isInsideDirectory(rootDir: string, candidatePath: string): boolean {
+  const rootRealPath = fs.realpathSync(rootDir);
+  const candidateRealPath = fs.realpathSync(candidatePath);
+  const relativePath = relative(rootRealPath, candidateRealPath);
+  return (
+    relativePath === "" ||
+    (!relativePath.startsWith("..") && !isAbsolute(relativePath))
+  );
+}
+
+function parseLegacySkillIndexEntry(line: string): string | null {
+  const match = line.match(/^\s*[-*]\s+(.+?)\s*$/);
+  if (!match) return null;
+
+  let entry = match[1].trim();
+  const markdownLink = entry.match(/^\[.+?\]\((.+?)\)$/);
+  if (markdownLink) {
+    entry = markdownLink[1].trim();
+  } else {
+    entry = entry.split(/\s+/)[0]?.trim() ?? "";
+  }
+
+  entry = entry.replace(/^`|`$/g, "");
+  if (!entry || entry.includes("\0") || isAbsolute(entry)) return null;
+
+  const normalized = normalize(entry);
+  if (
+    normalized === "." ||
+    normalized.startsWith("..") ||
+    isAbsolute(normalized)
+  ) {
+    return null;
+  }
+
+  if (basename(normalized).toLowerCase() === "skill.md") {
+    const skillDir = dirname(normalized);
+    return skillDir === "." ? null : skillDir;
+  }
+
+  return normalized;
+}
+
+function parseLegacySkillIndexEntries(contents: string): string[] {
+  const entries = new Set<string>();
+  for (const line of contents.split(/\r?\n/)) {
+    const entry = parseLegacySkillIndexEntry(line);
+    if (entry) entries.add(entry);
+  }
+  return [...entries];
+}
+
+function skillFileContentsMatch(
+  sourceDir: string,
+  destinationDir: string,
+): boolean {
+  try {
+    return (
+      fs.readFileSync(join(sourceDir, "SKILL.md"), "utf-8") ===
+      fs.readFileSync(join(destinationDir, "SKILL.md"), "utf-8")
+    );
+  } catch {
+    return false;
+  }
+}
+
+function topLevelPreservationName(relativeSkillDir: string): string {
+  return relativeSkillDir
+    .split(/[\\/]+/)
+    .filter(Boolean)
+    .join("__");
+}
+
+function getPreservationDestinationDir(
+  skillsDir: string,
+  sourceDir: string,
+  relativeSkillDir: string,
+): string | null {
+  const skillName = basename(relativeSkillDir);
+  if (!skillName || skillName === relativeSkillDir) return null;
+
+  const alternateName = topLevelPreservationName(relativeSkillDir);
+  const candidateNames = [
+    skillName,
+    ...(alternateName && alternateName !== skillName ? [alternateName] : []),
+  ];
+
+  for (const candidateName of candidateNames) {
+    const destinationDir = join(skillsDir, candidateName);
+    if (!fs.existsSync(destinationDir)) return destinationDir;
+    if (skillFileContentsMatch(sourceDir, destinationDir)) {
+      log.info(
+        { destinationDir, sourceDir },
+        "Nested indexed skill already preserved at top-level skills directory",
+      );
+      return null;
+    }
+  }
+
+  const baseName =
+    alternateName && alternateName !== skillName
+      ? alternateName
+      : `legacy__${skillName}`;
+  for (let suffix = 2; ; suffix += 1) {
+    const destinationDir = join(skillsDir, `${baseName}-${suffix}`);
+    if (!fs.existsSync(destinationDir)) return destinationDir;
+    if (skillFileContentsMatch(sourceDir, destinationDir)) {
+      log.info(
+        { destinationDir, sourceDir },
+        "Nested indexed skill already preserved at top-level skills directory",
+      );
+      return null;
+    }
+  }
+}
+
+function preserveNestedIndexedSkill(
+  tempRootDir: string,
+  skillsDir: string,
+  relativeSkillDir: string,
+): void {
+  const sourceDir = resolve(skillsDir, relativeSkillDir);
+  let destinationDir: string | null = null;
+
+  try {
+    if (!fs.existsSync(sourceDir)) return;
+    if (!isInsideDirectory(skillsDir, sourceDir)) {
+      log.warn(
+        { relativeSkillDir, sourceDir },
+        "Skipping nested indexed skill that resolves outside skills root",
+      );
+      return;
+    }
+
+    const sourceStat = fs.lstatSync(sourceDir);
+    if (!sourceStat.isDirectory()) return;
+
+    const skillFilePath = join(sourceDir, "SKILL.md");
+    if (!fs.existsSync(skillFilePath)) return;
+    if (!isInsideDirectory(sourceDir, skillFilePath)) {
+      log.warn(
+        { skillFilePath, sourceDir },
+        "Skipping nested indexed skill with SKILL.md outside skill directory",
+      );
+      return;
+    }
+
+    const skillFileStat = fs.lstatSync(skillFilePath);
+    if (!skillFileStat.isFile()) return;
+
+    destinationDir = getPreservationDestinationDir(
+      skillsDir,
+      sourceDir,
+      relativeSkillDir,
+    );
+    if (!destinationDir) return;
+
+    fs.mkdirSync(tempRootDir, { recursive: true });
+    const tempDir = join(tempRootDir, basename(destinationDir));
+    if (fs.existsSync(tempDir)) {
+      fs.rmSync(tempDir, { recursive: true, force: true });
+    }
+
+    fs.cpSync(sourceDir, tempDir, {
+      dereference: false,
+      errorOnExist: true,
+      force: false,
+      recursive: true,
+    });
+
+    if (fs.existsSync(destinationDir)) {
+      fs.rmSync(tempDir, { recursive: true, force: true });
+      if (skillFileContentsMatch(sourceDir, destinationDir)) return;
+      log.warn(
+        { destinationDir, sourceDir },
+        "Skipping nested indexed skill preservation because destination appeared during copy",
+      );
+      return;
+    }
+
+    fs.renameSync(tempDir, destinationDir);
+    log.info(
+      { destinationDir, sourceDir },
+      "Preserved nested indexed skill at top-level skills directory",
+    );
+  } catch (err) {
+    if (isNotFoundError(err)) return;
+    log.warn(
+      { err, relativeSkillDir, sourceDir, destinationDir },
+      "Failed to preserve nested indexed skill",
+    );
+    throw err;
+  }
+}
+
+function preserveNestedIndexedSkills(
+  workspaceDir: string,
+  skillsDir: string,
+  indexPath: string,
+): void {
+  const contents = fs.readFileSync(indexPath, "utf-8");
+  const tempRootDir = join(
+    workspaceDir,
+    ".workspace-migration-084-remove-legacy-skills-index",
+  );
+  for (const relativeSkillDir of parseLegacySkillIndexEntries(contents)) {
+    preserveNestedIndexedSkill(tempRootDir, skillsDir, relativeSkillDir);
+  }
+  if (fs.existsSync(tempRootDir)) {
+    fs.rmSync(tempRootDir, { recursive: true, force: true });
+  }
+}
+
+export const removeLegacySkillsIndexMigration: WorkspaceMigration = {
+  id: "084-remove-legacy-skills-index",
+  description: "Remove legacy workspace skills/SKILLS.md index file",
+  retryFailedCheckpoint: true,
+
+  run(workspaceDir: string): void {
+    const skillsDir = join(workspaceDir, "skills");
+    const indexPath = join(skillsDir, "SKILLS.md");
+
+    try {
+      const stat = fs.lstatSync(indexPath);
+      if (!stat.isFile() && !stat.isSymbolicLink()) {
+        log.warn(
+          { path: indexPath },
+          "Legacy SKILLS.md path is not a file; leaving it in place",
+        );
+        return;
+      }
+
+      if (stat.isFile()) {
+        preserveNestedIndexedSkills(workspaceDir, skillsDir, indexPath);
+      }
+
+      fs.unlinkSync(indexPath);
+      log.info({ path: indexPath }, "Removed legacy skills index file");
+    } catch (err) {
+      if (isNotFoundError(err)) return;
+      log.warn(
+        { err, path: indexPath },
+        "Failed to remove legacy skills index file",
+      );
+      throw err;
+    }
+  },
+
+  down(_workspaceDir: string): void {
+    // Forward-only: SKILLS.md is no longer a supported skill catalog format.
+  },
+};

package/src/workspace/migrations/085-memory-v2-bm25-b-reembed-disabled-v2-pages.ts

@@ -0,0 +1,137 @@
+import { randomUUID } from "node:crypto";
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { Database } from "bun:sqlite";
+
+import type { WorkspaceMigration } from "./types.js";
+
+/**
+ * Follow-up to `075-memory-v2-bm25-b-default-reembed`. Migration 075 shipped
+ * in v0.8.1 with no gating beyond "db exists", so workspaces with no v2
+ * pages still recorded a checkpoint entry and a queued reembed job. We
+ * cannot edit 075 to add gating retroactively — the runner skips any
+ * already-checkpointed id — so this migration re-runs the enqueue with the
+ * gating we want now.
+ *
+ * Two gates:
+ *
+ * 1. `hasConceptPages` — only enqueue if `memory/concepts/` actually has a
+ *    `.md` page. Workspaces that never wrote a v2 page have nothing to
+ *    reembed.
+ * 2. `isMemoryV2Disabled` — skip when `memory.v2.enabled` is explicitly
+ *    `false`. The worker does not currently gate `memory_v2_reembed`
+ *    dispatch on the config flag, so enqueueing for a workspace that
+ *    intentionally disabled v2 would immediately re-embed pages and hit
+ *    the embedding backend against the user's intent.
+ *
+ * When either gate fails, we also DELETE any pending `memory_v2_reembed`
+ * job that 075 may have enqueued in the same startup sweep. For workspaces
+ * that never checkpointed 075 (upgrades from pre-v0.8.1, first-boot sweeps),
+ * 075 runs first and unconditionally enqueues a job; without the explicit
+ * cancellation here, 075's job would survive 085's gate and execute against
+ * the user's intent.
+ */
+export const memoryV2Bm25BReembedDisabledV2PagesMigration: WorkspaceMigration =
+  {
+    id: "085-memory-v2-bm25-b-reembed-disabled-v2-pages",
+    description:
+      "Re-enqueue memory_v2_reembed for workspaces with v2 pages, gated on v2 not being explicitly disabled",
+
+    run(workspaceDir: string): void {
+      const dbPath = join(workspaceDir, "data", "db", "assistant.db");
+      if (!existsSync(dbPath)) return;
+
+      let db: Database;
+      try {
+        db = new Database(dbPath);
+      } catch {
+        return;
+      }
+
+      try {
+        const tableRow = db
+          .query(
+            `SELECT name FROM sqlite_master WHERE type='table' AND name='memory_jobs'`,
+          )
+          .get();
+        if (!tableRow) return;
+
+        // If either gate fails, cancel any pending reembed job that 075 may
+        // have enqueued in this same startup sweep. Leave 'running' jobs
+        // alone — the worker is already mid-flight and canceling would orphan
+        // its state. Only 'pending' jobs are safe to delete here.
+        if (
+          isMemoryV2Disabled(workspaceDir) ||
+          !hasConceptPages(workspaceDir)
+        ) {
+          db.query(
+            `DELETE FROM memory_jobs WHERE type='memory_v2_reembed' AND status='pending'`,
+          ).run();
+          return;
+        }
+
+        const existing = db
+          .query(
+            `SELECT id FROM memory_jobs WHERE type='memory_v2_reembed' AND status IN ('pending','running') LIMIT 1`,
+          )
+          .get();
+        if (existing) return;
+
+        const now = Date.now();
+        db.query(
+          `INSERT INTO memory_jobs
+             (id, type, payload, status, attempts, deferrals, run_after, last_error, created_at, updated_at)
+           VALUES (?, 'memory_v2_reembed', '{}', 'pending', 0, 0, ?, NULL, ?, ?)`,
+        ).run(randomUUID(), now, now, now);
+      } finally {
+        db.close();
+      }
+    },
+
+    down(_workspaceDir: string): void {
+      // Forward-only: the reembed is a one-shot data refresh.
+    },
+  };
+
+/**
+ * Returns true only when `memory.v2.enabled` is explicitly set to `false`
+ * in the workspace `config.json`. Missing/unparseable config falls through
+ * to the schema default (enabled), matching `MemoryV2ConfigSchema`.
+ */
+function isMemoryV2Disabled(workspaceDir: string): boolean {
+  const configPath = join(workspaceDir, "config.json");
+  if (!existsSync(configPath)) return false;
+  try {
+    const raw = JSON.parse(readFileSync(configPath, "utf-8"));
+    const memory = (raw as { memory?: { v2?: { enabled?: unknown } } })?.memory;
+    return memory?.v2?.enabled === false;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Returns true when `memory/concepts/` contains any `.md` file. Walks the
+ * tree iteratively so we bail on the first hit — pages can be nested in
+ * subdirectories (e.g. `memory/concepts/people/alice.md`).
+ */
+function hasConceptPages(workspaceDir: string): boolean {
+  const stack = [join(workspaceDir, "memory", "concepts")];
+  while (stack.length > 0) {
+    const dir = stack.pop()!;
+    let entries;
+    try {
+      entries = readdirSync(dir, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    for (const entry of entries) {
+      if (entry.isDirectory()) {
+        stack.push(join(dir, entry.name));
+      } else if (entry.isFile() && entry.name.endsWith(".md")) {
+        return true;
+      }
+    }
+  }
+  return false;
+}

package/src/workspace/migrations/086-revert-stale-gemini-mis-rewrites.ts

@@ -0,0 +1,198 @@
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import type { WorkspaceMigration } from "./types.js";
+
+/**
+ * Revert mis-rewrites by migration 057 where a non-Gemini fragment was
+ * incorrectly treated as Gemini.
+ *
+ * Migration 057's `inferProvider` helper hardcodes `model === "gemini-3-flash"`
+ * as Gemini even when no `provider` is set, but `resolveCallSiteConfig` only
+ * infers providers via the catalog and `gemini-3-flash` is not a catalog
+ * entry. The runtime resolver therefore leaves such fragments inheriting from
+ * lower layers, so a workspace like `llm.default = {provider: "ollama"}` with
+ * `llm.callSites.recall = {model: "gemini-3-flash"}` (a user-named local
+ * model) resolves as Ollama at runtime — but 057 rewrites that recall model
+ * to `gemini-3-flash-preview`, flipping the catalog-based provider inference
+ * to Gemini and corrupting the user's intent.
+ *
+ * For each call-site or profile fragment that currently looks like a 057
+ * rewrite output (model in the replacement set, no explicit `provider`), this
+ * migration recomputes the effective provider via proper catalog-based
+ * inference using only the other layers. If that effective provider is
+ * explicitly non-Gemini, the fragment's model is reverted to
+ * `gemini-3-flash`, restoring the user's pre-057 state.
+ */
+export const revertStaleGeminiMisRewritesMigration: WorkspaceMigration = {
+  id: "086-revert-stale-gemini-mis-rewrites",
+  description:
+    "Revert 057 mis-rewrites of gemini-3-flash in non-Gemini fragment contexts",
+  run(workspaceDir: string): void {
+    const configPath = join(workspaceDir, "config.json");
+    if (!existsSync(configPath)) return;
+
+    let config: Record<string, unknown>;
+    try {
+      const raw = JSON.parse(readFileSync(configPath, "utf-8"));
+      if (!raw || typeof raw !== "object" || Array.isArray(raw)) return;
+      config = raw as Record<string, unknown>;
+    } catch {
+      return;
+    }
+
+    const llm = readObject(config.llm);
+    if (llm === null) return;
+
+    const defaultBlock = readObject(llm.default);
+    const defaultProvider = inferLayerProvider(defaultBlock);
+    const profiles = readObject(llm.profiles);
+    const activeProfileName =
+      typeof llm.activeProfile === "string" ? llm.activeProfile : undefined;
+
+    let changed = false;
+
+    // Pass 1: revert profile candidates first so that pass 2's call-site
+    // evaluation sees the reverted profile state. A call-site that references
+    // a profile candidate would otherwise infer `siteProfileProvider = gemini`
+    // from the profile's pre-revert rewritten model, masking the call-site's
+    // own need to revert.
+    if (profiles !== null) {
+      for (const rawProfile of Object.values(profiles)) {
+        const profile = readObject(rawProfile);
+        if (profile === null) continue;
+        if (!isRevertCandidate(profile)) continue;
+        if (isExplicitlyNonGemini(defaultProvider)) {
+          profile.model = STALE_MODEL;
+          changed = true;
+        }
+      }
+    }
+
+    // Compute the active-profile provider after pass 1 so it reflects any
+    // reversion of the active profile itself.
+    const activeProfileBlock =
+      profiles !== null && activeProfileName !== undefined
+        ? readObject(profiles[activeProfileName])
+        : null;
+    const activeProfileProvider = inferLayerProvider(activeProfileBlock);
+
+    // Pass 2: call-site candidates. `inferLayerProvider` is re-read from
+    // `profiles` per call site, so it observes pass 1's reversions.
+    const callSites = readObject(llm.callSites);
+    if (callSites !== null) {
+      for (const [site, rawConfig] of Object.entries(callSites)) {
+        const callSiteConfig = readObject(rawConfig);
+        if (callSiteConfig === null) continue;
+        if (!isRevertCandidate(callSiteConfig)) continue;
+        const siteProfileName =
+          typeof callSiteConfig.profile === "string"
+            ? callSiteConfig.profile
+            : undefined;
+        const siteProfileBlock =
+          profiles !== null && siteProfileName !== undefined
+            ? readObject(profiles[siteProfileName])
+            : null;
+        const siteProfileProvider = inferLayerProvider(siteProfileBlock);
+
+        const effective = effectiveProviderExcludingSite({
+          callSite: site,
+          siteProfileProvider,
+          activeProfileProvider,
+          defaultProvider,
+        });
+        if (isExplicitlyNonGemini(effective)) {
+          callSiteConfig.model = STALE_MODEL;
+          changed = true;
+        }
+      }
+    }
+
+    if (!changed) return;
+
+    writeFileSync(configPath, JSON.stringify(config, null, 2) + "\n");
+  },
+  down(_workspaceDir: string): void {
+    // Forward-only: re-applying the broken rewrite would reintroduce the bug.
+  },
+};
+
+// ---------------------------------------------------------------------------
+// Helpers — self-contained per workspace migrations AGENTS.md
+// ---------------------------------------------------------------------------
+
+const STALE_MODEL = "gemini-3-flash";
+
+// Models 057 writes when rewriting. Any fragment now holding one of these
+// values without an explicit provider is a potential mis-rewrite candidate.
+const REPLACEMENT_MODELS = new Set<string>([
+  "gemini-3-flash-preview",
+  "gemini-3.1-flash-lite-preview",
+]);
+
+// Subset of the Gemini provider catalog used for per-layer catalog inference.
+// Mirrors `getCatalogProviderForModel` for Gemini entries only — for other
+// providers we treat a bare model as "provider unknown" and fall through to
+// lower layers, which is safe because the revert only fires when at least one
+// layer below carries an explicit non-Gemini provider.
+const GEMINI_CATALOG_MODELS = new Set<string>([
+  "gemini-3.1-pro-preview",
+  "gemini-3.1-pro-preview-customtools",
+  "gemini-3-flash-preview",
+  "gemini-3.1-flash-lite-preview",
+  "gemini-2.5-flash",
+  "gemini-2.5-flash-lite",
+  "gemini-2.5-pro",
+]);
+
+function isRevertCandidate(block: Record<string, unknown>): boolean {
+  if (typeof block.provider === "string") return false;
+  return typeof block.model === "string" && REPLACEMENT_MODELS.has(block.model);
+}
+
+function inferLayerProvider(
+  block: Record<string, unknown> | null,
+): string | undefined {
+  if (block === null) return undefined;
+  if (typeof block.provider === "string") return block.provider;
+  if (
+    typeof block.model === "string" &&
+    GEMINI_CATALOG_MODELS.has(block.model)
+  ) {
+    return "gemini";
+  }
+  return undefined;
+}
+
+function isExplicitlyNonGemini(provider: string | undefined): boolean {
+  return provider !== undefined && provider !== "gemini";
+}
+
+// Mirrors `resolveCallSiteConfig`'s layered provider resolution, omitting the
+// candidate call-site fragment itself (its provider is undefined and its
+// model is the replacement value, so including it would always say "gemini"
+// via catalog inference — defeating the revert check).
+function effectiveProviderExcludingSite(args: {
+  callSite: string;
+  siteProfileProvider: string | undefined;
+  activeProfileProvider: string | undefined;
+  defaultProvider: string | undefined;
+}): string | undefined {
+  const {
+    callSite,
+    siteProfileProvider,
+    activeProfileProvider,
+    defaultProvider,
+  } = args;
+  if (callSite === "mainAgent") {
+    return activeProfileProvider ?? siteProfileProvider ?? defaultProvider;
+  }
+  return siteProfileProvider ?? activeProfileProvider ?? defaultProvider;
+}
+
+function readObject(value: unknown): Record<string, unknown> | null {
+  if (value === null || typeof value !== "object" || Array.isArray(value)) {
+    return null;
+  }
+  return value as Record<string, unknown>;
+}

package/src/workspace/migrations/registry.ts

@@ -80,6 +80,10 @@ import { homeFeedNotificationOnlyMigration } from "./079-home-feed-notification-
 import { restrictVercelApiTokenMetadataMigration } from "./080-restrict-vercel-api-token-metadata.js";
 import { backfillBashAllowedToolsForInjectionCredentialsMigration } from "./081-backfill-bash-allowed-tools-for-injection-credentials.js";
 import { backfillManagedProfileLabelsMigration } from "./082-backfill-managed-profile-labels.js";
+import { systemPromptPrefixToFileMigration } from "./083-system-prompt-prefix-to-file.js";
+import { removeLegacySkillsIndexMigration } from "./084-remove-legacy-skills-index.js";
+import { memoryV2Bm25BReembedDisabledV2PagesMigration } from "./085-memory-v2-bm25-b-reembed-disabled-v2-pages.js";
+import { revertStaleGeminiMisRewritesMigration } from "./086-revert-stale-gemini-mis-rewrites.js";
 import { migrateToWorkspaceVolumeMigration } from "./migrate-to-workspace-volume.js";
 import type { WorkspaceMigration } from "./types.js";
 
@@ -171,4 +175,8 @@ export const WORKSPACE_MIGRATIONS: WorkspaceMigration[] = [
   restrictVercelApiTokenMetadataMigration,
   backfillBashAllowedToolsForInjectionCredentialsMigration,
   backfillManagedProfileLabelsMigration,
+  systemPromptPrefixToFileMigration,
+  removeLegacySkillsIndexMigration,
+  memoryV2Bm25BReembedDisabledV2PagesMigration,
+  revertStaleGeminiMisRewritesMigration,
 ];