@vellumai/assistant 0.5.1 → 0.5.3

package/src/permissions/checker.ts

@@ -2,12 +2,15 @@ import { createHash } from "node:crypto";
 import { homedir } from "node:os";
 import { dirname, resolve } from "node:path";
 
+import { isAssistantFeatureFlagEnabled } from "../config/assistant-feature-flags.js";
 import { getConfig } from "../config/loader.js";
-import { resolveSkillSelector } from "../config/skills.js";
+import { loadSkillCatalog, resolveSkillSelector } from "../config/skills.js";
+import { indexCatalogById } from "../skills/include-graph.js";
 import {
   isSkillSourcePath,
   normalizeFilePath,
 } from "../skills/path-classifier.js";
+import { computeTransitiveSkillVersionHash } from "../skills/transitive-version-hash.js";
 import { computeSkillVersionHash } from "../skills/version-hash.js";
 import type { ManifestOverride } from "../tools/execution-target.js";
 import {
@@ -144,6 +147,7 @@ const LOW_RISK_PROGRAMS = new Set([
   "du",
   "df",
   "assistant",
+  "vellum",
 ]);
 
 // High-risk shell programs / patterns
@@ -197,6 +201,32 @@ const LOW_RISK_GIT_SUBCOMMANDS = new Set([
   "reflog",
 ]);
 
+// Mutating assistant/vellum CLI subcommands that should be escalated to Medium
+// risk. Most assistant/vellum subcommands are read-only and stay Low risk.
+// This mirrors the git subcommand pattern — only known mutating operations
+// get escalated.
+const MEDIUM_RISK_CLI_SUBCOMMANDS = new Set([
+  "credentials",
+  "config",
+  "bash",
+  "trust",
+  "autonomy",
+  "contacts",
+  "mcp",
+  "keys",
+  "wake",
+  "sleep",
+  "hatch",
+  "retire",
+  "clean",
+  "setup",
+  "upgrade",
+  "recover",
+  "login",
+  "use",
+  "pair",
+]);
+
 // Commands that wrap another program — the real program appears as the first
 // non-flag argument.  When one of these is the segment program we look through
 // its args to find the effective program (e.g. `env curl …` → curl).
@@ -219,15 +249,40 @@ const WRAPPER_PROGRAMS = new Set([
 // value of -u) as the wrapped program instead of `echo`.
 const ENV_VALUE_FLAGS = new Set(["-u", "--unset", "-C", "--chdir"]);
 
+// `git` global flags that consume the next positional argument as their value.
+// Without this, `git -C status commit` would incorrectly identify `status`
+// (the directory path) as the subcommand instead of `commit`.
+const GIT_VALUE_FLAGS = new Set([
+  "-C",
+  "-c",
+  "--git-dir",
+  "--work-tree",
+  "--namespace",
+  "--super-prefix",
+  "--config-env",
+]);
+
 /**
- * Return the first non-flag argument from an argument list.
- * Flags are arguments that start with `-`.  This is used to skip global
- * options (e.g. `--verbose`, `-h`) when extracting the subcommand from
- * CLIs like `git`, `vellum`, and `assistant`.
+ * Return the first non-flag argument from an argument list, optionally
+ * skipping value-taking flags.  Flags are arguments that start with `-`.
+ * This is used to skip global options (e.g. `--verbose`, `-h`, `-C <path>`)
+ * when extracting the subcommand from CLIs like `git`, `vellum`, and
+ * `assistant`.
+ *
+ * When `valueFlags` is provided, any flag in that set causes the next
+ * argument to be skipped as well (it is the flag's value, not a positional).
  */
-function firstPositionalArg(args: string[]): string | undefined {
-  for (const arg of args) {
-    if (!arg.startsWith("-")) return arg;
+function firstPositionalArg(
+  args: string[],
+  valueFlags?: Set<string>,
+): string | undefined {
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg.startsWith("-")) {
+      if (valueFlags?.has(arg)) i++; // skip the next arg (the flag's value)
+      continue;
+    }
+    return arg;
   }
   return undefined;
 }
@@ -300,6 +355,34 @@ function resolveSkillIdAndHash(
   }
 }
 
+/**
+ * Check whether a skill (by id) has parsed inline command expansions.
+ * Returns false when the skill is not found in the catalog.
+ */
+function hasInlineExpansions(skillId: string): boolean {
+  const catalog = loadSkillCatalog();
+  const skill = catalog.find((s) => s.id === skillId);
+  return (
+    skill?.inlineCommandExpansions != null &&
+    skill.inlineCommandExpansions.length > 0
+  );
+}
+
+/**
+ * Compute the transitive version hash for a skill, returning `undefined`
+ * when computation fails (missing includes, cycle, etc.). The permission
+ * layer falls back to the any-version candidate in that case.
+ */
+function computeTransitiveHashSafe(skillId: string): string | undefined {
+  try {
+    const catalog = loadSkillCatalog();
+    const index = indexCatalogById(catalog);
+    return computeTransitiveSkillVersionHash(skillId, index);
+  } catch {
+    return undefined;
+  }
+}
+
 function canonicalizeWebFetchUrl(parsed: URL): URL {
   parsed.hash = "";
   parsed.username = "";
@@ -381,13 +464,39 @@ async function buildCommandCandidates(
       targets.push("");
     } else {
       const resolved = resolveSkillIdAndHash(rawSelector);
-      if (resolved && resolved.versionHash) {
-        // Version-specific candidate lets rules pin to an exact skill version
-        targets.push(`${resolved.id}@${resolved.versionHash}`);
+
+      // When the resolved skill contains inline command expansions and the
+      // feature flag is on, emit skill_load_dynamic: candidates so the
+      // higher-priority default ask rule catches them instead of falling
+      // through to the permissive skill_load:* allow rule.
+      const config = getConfig();
+      const inlineEnabled = isAssistantFeatureFlagEnabled(
+        "feature_flags.inline-skill-commands.enabled",
+        config,
+      );
+
+      if (resolved && inlineEnabled && hasInlineExpansions(resolved.id)) {
+        const transitiveHash = computeTransitiveHashSafe(resolved.id);
+        if (transitiveHash) {
+          targets.push(`skill_load_dynamic:${resolved.id}@${transitiveHash}`);
+        }
+        targets.push(`skill_load_dynamic:${resolved.id}`);
+        // Don't fall through to skill_load:* — dynamic skills use their own
+        // candidate namespace so the default ask rule applies.
+      } else {
+        if (resolved && resolved.versionHash) {
+          // Version-specific candidate lets rules pin to an exact skill version
+          targets.push(`${resolved.id}@${resolved.versionHash}`);
+        }
+        targets.push(rawSelector);
       }
-      targets.push(rawSelector);
     }
-    return [...new Set(targets)].map((target) => `${toolName}:${target}`);
+
+    // Dynamic candidates use skill_load_dynamic: prefix; normal ones use skill_load:
+    return [...new Set(targets)].map((target) => {
+      if (target.startsWith("skill_load_dynamic:")) return target;
+      return `${toolName}:${target}`;
+    });
   }
 
   if (
@@ -652,7 +761,7 @@ async function classifyRiskUncached(
       }
 
       if (prog === "git") {
-        const subcommand = firstPositionalArg(seg.args);
+        const subcommand = firstPositionalArg(seg.args, GIT_VALUE_FLAGS);
         if (subcommand && LOW_RISK_GIT_SUBCOMMANDS.has(subcommand)) {
           // Stay at current risk
           continue;
@@ -662,6 +771,17 @@ async function classifyRiskUncached(
         continue;
       }
 
+      if (prog === "vellum" || prog === "assistant") {
+        const subcommand = firstPositionalArg(seg.args);
+        if (subcommand && MEDIUM_RISK_CLI_SUBCOMMANDS.has(subcommand)) {
+          // Known mutating subcommands are medium
+          maxRisk = RiskLevel.Medium;
+          continue;
+        }
+        // Read-only / unknown subcommands stay at current risk
+        continue;
+      }
+
       if (!LOW_RISK_PROGRAMS.has(prog)) {
         // Unknown program → medium
         if (maxRisk === RiskLevel.Low) {
@@ -1021,6 +1141,32 @@ function skillLoadAllowlistStrategy(
 
   if (rawSelector) {
     const resolved = resolveSkillIdAndHash(rawSelector);
+
+    // Check whether this is a dynamic (inline-command) skill load
+    const config = getConfig();
+    const inlineEnabled = isAssistantFeatureFlagEnabled(
+      "feature_flags.inline-skill-commands.enabled",
+      config,
+    );
+
+    if (resolved && inlineEnabled && hasInlineExpansions(resolved.id)) {
+      const transitiveHash = computeTransitiveHashSafe(resolved.id);
+      const options: AllowlistOption[] = [];
+      if (transitiveHash) {
+        options.push({
+          label: `${resolved.id}@${transitiveHash}`,
+          description: "This exact version (pinned)",
+          pattern: `skill_load_dynamic:${resolved.id}@${transitiveHash}`,
+        });
+      }
+      options.push({
+        label: resolved.id,
+        description: "This skill (any version)",
+        pattern: `skill_load_dynamic:${resolved.id}`,
+      });
+      return options;
+    }
+
     if (resolved && resolved.versionHash) {
       return [
         {

package/src/permissions/defaults.ts

@@ -198,6 +198,19 @@ export function getDefaultRuleTemplates(): DefaultRuleTemplate[] {
     })),
   );
 
+  // Inline-command skill loads use a distinct candidate namespace
+  // (skill_load_dynamic:*) so they prompt by default instead of falling
+  // through to the permissive skill_load:* allow rule below. The higher
+  // priority ensures this rule wins when both could match.
+  const skillLoadDynamicRule: DefaultRuleTemplate = {
+    id: "default:ask-skill_load_dynamic-global",
+    tool: "skill_load",
+    pattern: "skill_load_dynamic:*",
+    scope: "everywhere",
+    decision: "ask",
+    priority: 200,
+  };
+
   const skillLoadRule: DefaultRuleTemplate = {
     id: "default:allow-skill_load-global",
     tool: "skill_load",
@@ -294,6 +307,7 @@ export function getDefaultRuleTemplates(): DefaultRuleTemplate[] {
     bootstrapDeleteRule,
     updatesDeleteRule,
     ...skillSourceMutationRules,
+    skillLoadDynamicRule,
     skillLoadRule,
     skillExecuteRule,
     browserNavigateRule,

package/src/prompts/templates/BOOTSTRAP.md

@@ -208,6 +208,8 @@ Once you've completed Phase 1 and made reasonable progress through Phase 2, you'
 
 If you still haven't shown the two suggestions (Phase 2 step 4), do that before wrapping.
 
+When you're confident onboarding is complete, delete `BOOTSTRAP.md` so it doesn't re-trigger on the next conversation.
+
 ---
 
 _Good luck out there. Make it count._

package/src/providers/anthropic/client.ts

@@ -16,6 +16,9 @@ import type {
 
 const log = getLogger("anthropic-client");
 
+/** Validation-specific timeout (10s) so a stalled network doesn't block key submission. */
+const VALIDATION_TIMEOUT_MS = 10_000;
+
 /**
  * Validate an Anthropic API key by making a lightweight GET /v1/models call.
  * Returns `{ valid: true }` on success or `{ valid: false, reason: string }` on failure.
@@ -24,7 +27,11 @@ export async function validateAnthropicApiKey(
   apiKey: string,
 ): Promise<{ valid: true } | { valid: false; reason: string }> {
   try {
-    const client = new Anthropic({ apiKey });
+    const client = new Anthropic({
+      apiKey,
+      timeout: VALIDATION_TIMEOUT_MS,
+      maxRetries: 0,
+    });
     await client.models.list({ limit: 1 });
     return { valid: true };
   } catch (error) {

package/src/providers/failover.ts

@@ -133,7 +133,10 @@ export class FailoverProvider implements Provider {
           );
           health.unhealthySince = null;
         }
-        return response;
+        return {
+          ...response,
+          actualProvider: response.actualProvider ?? provider.name,
+        };
       } catch (error) {
         lastError = error;
 

package/src/providers/gemini/client.ts

@@ -3,6 +3,7 @@ import { ApiError, GoogleGenAI } from "@google/genai";
 
 import { SYSTEM_PROMPT_CACHE_BOUNDARY } from "../../prompts/cache-boundary.js";
 import { ProviderError } from "../../util/errors.js";
+import { getLogger } from "../../util/logger.js";
 import { createStreamTimeout } from "../stream-timeout.js";
 import type {
   ContentBlock,
@@ -13,6 +14,55 @@ import type {
   ToolDefinition,
 } from "../types.js";
 
+const log = getLogger("gemini-client");
+
+/** Validation-specific timeout (10s) so a stalled network doesn't block key submission. */
+const VALIDATION_TIMEOUT_MS = 10_000;
+
+/**
+ * Validate a Gemini API key by making a lightweight models.list() call.
+ * Returns `{ valid: true }` on success or `{ valid: false, reason: string }` on failure.
+ */
+export async function validateGeminiApiKey(
+  apiKey: string,
+): Promise<{ valid: true } | { valid: false; reason: string }> {
+  try {
+    const client = new GoogleGenAI({ apiKey });
+    await client.models.list({
+      config: {
+        pageSize: 1,
+        httpOptions: { timeout: VALIDATION_TIMEOUT_MS },
+      },
+    });
+    return { valid: true };
+  } catch (error) {
+    if (error instanceof ApiError) {
+      if (error.status === 401) {
+        return { valid: false, reason: "API key is invalid or expired." };
+      }
+      if (error.status === 403) {
+        return {
+          valid: false,
+          reason: `Gemini API error (${error.status}): ${error.message}`,
+        };
+      }
+      // Transient errors (429, 5xx, etc.) — validation is inconclusive,
+      // allow the key to be stored rather than blocking the user.
+      log.warn(
+        { status: error.status },
+        "Gemini API returned a transient error during key validation — allowing key storage",
+      );
+      return { valid: true };
+    }
+    // Network errors — validation is inconclusive, allow key storage.
+    log.warn(
+      { error: error instanceof Error ? error.message : String(error) },
+      "Network error during Gemini key validation — allowing key storage",
+    );
+    return { valid: true };
+  }
+}
+
 export interface GeminiProviderOptions {
   streamTimeoutMs?: number;
   /** When set, routes requests through the managed proxy at this base URL. */

package/src/providers/model-catalog.ts

@@ -0,0 +1,92 @@
+export interface CatalogModel {
+  id: string;
+  displayName: string;
+}
+
+export interface ProviderCatalogEntry {
+  id: string;
+  displayName: string;
+  models: CatalogModel[];
+  defaultModel: string;
+  apiKeyUrl?: string;
+  apiKeyPlaceholder?: string;
+}
+
+/** Single source of truth for all inference provider metadata and models. */
+export const PROVIDER_CATALOG: ProviderCatalogEntry[] = [
+  {
+    id: "anthropic",
+    displayName: "Anthropic",
+    models: [
+      { id: "claude-opus-4-6", displayName: "Claude Opus 4.6" },
+      { id: "claude-sonnet-4-6", displayName: "Claude Sonnet 4.6" },
+      { id: "claude-haiku-4-5-20251001", displayName: "Claude Haiku 4.5" },
+    ],
+    defaultModel: "claude-opus-4-6",
+    apiKeyUrl: "https://console.anthropic.com/settings/keys",
+    apiKeyPlaceholder: "sk-ant-api03-...",
+  },
+  {
+    id: "openai",
+    displayName: "OpenAI",
+    models: [
+      { id: "gpt-5.4", displayName: "GPT-5.4" },
+      { id: "gpt-5.2", displayName: "GPT-5.2" },
+      { id: "gpt-5.4-mini", displayName: "GPT-5.4 Mini" },
+      { id: "gpt-5.4-nano", displayName: "GPT-5.4 Nano" },
+    ],
+    defaultModel: "gpt-5.4",
+    apiKeyUrl: "https://platform.openai.com/api-keys",
+    apiKeyPlaceholder: "sk-proj-...",
+  },
+  {
+    id: "gemini",
+    displayName: "Google Gemini",
+    models: [
+      { id: "gemini-3-flash", displayName: "Gemini 3 Flash" },
+      { id: "gemini-3-pro", displayName: "Gemini 3 Pro" },
+    ],
+    defaultModel: "gemini-3-flash",
+    apiKeyUrl: "https://aistudio.google.com/apikey",
+    apiKeyPlaceholder: "AIza...",
+  },
+  {
+    id: "ollama",
+    displayName: "Ollama",
+    models: [
+      { id: "llama3.2", displayName: "Llama 3.2" },
+      { id: "mistral", displayName: "Mistral" },
+    ],
+    defaultModel: "llama3.2",
+  },
+  {
+    id: "fireworks",
+    displayName: "Fireworks",
+    models: [
+      {
+        id: "accounts/fireworks/models/kimi-k2p5",
+        displayName: "Kimi K2.5",
+      },
+    ],
+    defaultModel: "accounts/fireworks/models/kimi-k2p5",
+    apiKeyUrl: "https://fireworks.ai/account/api-keys",
+    apiKeyPlaceholder: "fw_...",
+  },
+  {
+    id: "openrouter",
+    displayName: "OpenRouter",
+    models: [
+      { id: "x-ai/grok-4", displayName: "Grok 4" },
+      { id: "x-ai/grok-4.20-beta", displayName: "Grok 4.20 Beta" },
+    ],
+    defaultModel: "x-ai/grok-4",
+    apiKeyUrl: "https://openrouter.ai/keys",
+    apiKeyPlaceholder: "sk-or-v1-...",
+  },
+];
+
+/** Check if a model ID is in the catalog for a given provider. */
+export function isModelInCatalog(provider: string, modelId: string): boolean {
+  const entry = PROVIDER_CATALOG.find((p) => p.id === provider);
+  return entry?.models.some((m) => m.id === modelId) ?? false;
+}

package/src/providers/model-intents.ts

@@ -1,20 +1,16 @@
+import { isModelInCatalog, PROVIDER_CATALOG } from "./model-catalog.js";
 import type { ModelIntent } from "./types.js";
 
-const PROVIDER_DEFAULT_MODELS = {
-  anthropic: "claude-opus-4-6",
-  openai: "gpt-5.2",
-  gemini: "gemini-3-flash",
-  ollama: "llama3.2",
-  fireworks: "accounts/fireworks/models/kimi-k2p5",
-  openrouter: "x-ai/grok-4",
-} as const;
-
-type KnownProviderName = keyof typeof PROVIDER_DEFAULT_MODELS;
+/**
+ * Derived from PROVIDER_CATALOG — single source of truth for default models.
+ * Each provider's `defaultModel` in the catalog populates this map.
+ */
+export const PROVIDER_DEFAULT_MODELS: Record<string, string> =
+  Object.fromEntries(
+    PROVIDER_CATALOG.map((entry) => [entry.id, entry.defaultModel]),
+  );
 
-const PROVIDER_MODEL_INTENTS: Record<
-  KnownProviderName,
-  Record<ModelIntent, string>
-> = {
+const PROVIDER_MODEL_INTENTS: Record<string, Record<ModelIntent, string>> = {
   anthropic: {
     "latency-optimized": "claude-haiku-4-5-20251001",
     "quality-optimized": "claude-opus-4-6",
@@ -47,29 +43,42 @@ const PROVIDER_MODEL_INTENTS: Record<
   },
 };
 
+const FALLBACK_DEFAULT_MODEL = "claude-opus-4-6";
+
 const MODEL_INTENTS = new Set<ModelIntent>([
   "latency-optimized",
   "quality-optimized",
   "vision-optimized",
 ]);
 
+// ── Consistency validation ───────────────────────────────────────────
+// Eagerly verify that every model ID referenced by PROVIDER_MODEL_INTENTS
+// exists in PROVIDER_CATALOG, catching drift at module-load time rather
+// than at runtime when a user picks a model.
+for (const [provider, intents] of Object.entries(PROVIDER_MODEL_INTENTS)) {
+  for (const [intent, modelId] of Object.entries(intents)) {
+    if (!isModelInCatalog(provider, modelId)) {
+      throw new Error(
+        `PROVIDER_MODEL_INTENTS[${provider}][${intent}] references model "${modelId}" ` +
+          `which is not in PROVIDER_CATALOG. Update model-catalog.ts or model-intents.ts.`,
+      );
+    }
+  }
+}
+
 export function isModelIntent(value: unknown): value is ModelIntent {
   return typeof value === "string" && MODEL_INTENTS.has(value as ModelIntent);
 }
 
 export function getProviderDefaultModel(providerName: string): string {
-  const knownProvider = providerName as KnownProviderName;
-  return (
-    PROVIDER_DEFAULT_MODELS[knownProvider] ?? PROVIDER_DEFAULT_MODELS.anthropic
-  );
+  return PROVIDER_DEFAULT_MODELS[providerName] ?? FALLBACK_DEFAULT_MODEL;
 }
 
 export function resolveModelIntent(
   providerName: string,
   intent: ModelIntent,
 ): string {
-  const knownProvider = providerName as KnownProviderName;
-  const providerIntentModels = PROVIDER_MODEL_INTENTS[knownProvider];
+  const providerIntentModels = PROVIDER_MODEL_INTENTS[providerName];
   if (providerIntentModels) {
     return providerIntentModels[intent];
   }

package/src/providers/openai/client.ts

@@ -2,6 +2,7 @@ import OpenAI from "openai";
 
 import { SYSTEM_PROMPT_CACHE_BOUNDARY } from "../../prompts/cache-boundary.js";
 import { ProviderError } from "../../util/errors.js";
+import { getLogger } from "../../util/logger.js";
 import { extractRetryAfterMs } from "../../util/retry.js";
 import { escapeXmlAttr } from "../../util/xml.js";
 import { createStreamTimeout } from "../stream-timeout.js";
@@ -14,6 +15,54 @@ import type {
   ToolDefinition,
 } from "../types.js";
 
+const log = getLogger("openai-client");
+
+/** Validation-specific timeout (10s) so a stalled network doesn't block key submission. */
+const VALIDATION_TIMEOUT_MS = 10_000;
+
+/**
+ * Validate an OpenAI API key by making a lightweight GET /v1/models call.
+ * Returns `{ valid: true }` on success or `{ valid: false, reason: string }` on failure.
+ */
+export async function validateOpenAIApiKey(
+  apiKey: string,
+): Promise<{ valid: true } | { valid: false; reason: string }> {
+  try {
+    const client = new OpenAI({
+      apiKey,
+      timeout: VALIDATION_TIMEOUT_MS,
+      maxRetries: 0,
+    });
+    await client.models.list();
+    return { valid: true };
+  } catch (error) {
+    if (error instanceof OpenAI.APIError) {
+      if (error.status === 401) {
+        return { valid: false, reason: "API key is invalid or expired." };
+      }
+      if (error.status === 403) {
+        return {
+          valid: false,
+          reason: `OpenAI API error (${error.status}): ${error.message}`,
+        };
+      }
+      // Transient errors (429, 5xx, etc.) — validation is inconclusive,
+      // allow the key to be stored rather than blocking the user.
+      log.warn(
+        { status: error.status },
+        "OpenAI API returned a transient error during key validation — allowing key storage",
+      );
+      return { valid: true };
+    }
+    // Network errors — validation is inconclusive, allow key storage.
+    log.warn(
+      { error: error instanceof Error ? error.message : String(error) },
+      "Network error during OpenAI key validation — allowing key storage",
+    );
+    return { valid: true };
+  }
+}
+
 export interface OpenAICompatibleProviderOptions {
   baseURL?: string;
   providerName?: string;

package/src/providers/types.ts

@@ -93,6 +93,8 @@ export type ModelIntent =
 export interface ProviderResponse {
   content: ContentBlock[];
   model: string;
+  /** Provider that actually produced this response, which may differ from a wrapper provider name. */
+  actualProvider?: string;
   usage: {
     /** Total input tokens (input_tokens + cache_creation + cache_read). */
     inputTokens: number;