@vellumai/assistant 0.9.0 → 0.10.0-staging.2

package/src/config/seed-inference-profiles.ts

@@ -27,7 +27,10 @@ type ManagedProfileTemplate = Omit<
   ProfileEntry,
   "provider" | "model" | "provider_connection"
 > & {
-  intent: ModelIntent;
+  // Exactly one of `intent` or `model` must be set. `intent` resolves the
+  // model from the catalog at seed time; `model` pins an explicit model id.
+  intent?: ModelIntent;
+  model?: string;
   provider: NonNullable<ProfileEntry["provider"]>;
   connectionName: string;
 };
@@ -38,17 +41,20 @@ type ManagedProfileTemplate = Omit<
  * (`preserveProfileNames`) take precedence when present.
  */
 const MANAGED_PROFILE_TEMPLATES: Record<string, ManagedProfileTemplate> = {
+  // Served by MiniMax M3 on Fireworks via managed platform inference: a strong
+  // open model at a lower price point than the managed Anthropic route.
   balanced: {
     intent: "balanced",
-    provider: "anthropic",
-    connectionName: "anthropic-managed",
+    provider: "fireworks",
+    connectionName: "fireworks-managed",
     source: "managed",
     label: "Balanced",
     description: "Good balance of quality, cost, and speed",
-    maxTokens: 16000,
+    maxTokens: 32000,
     effort: "high",
     thinking: { enabled: true, streamThinking: true },
     contextWindow: { maxInputTokens: DEFAULT_CONTEXT_WINDOW_MAX_INPUT_TOKENS },
+    topP: 0.95,
   },
   "quality-optimized": {
     intent: "quality-optimized",
@@ -61,6 +67,9 @@ const MANAGED_PROFILE_TEMPLATES: Record<string, ManagedProfileTemplate> = {
     effort: "high",
     thinking: { enabled: true, streamThinking: true },
     contextWindow: { maxInputTokens: DEFAULT_CONTEXT_WINDOW_MAX_INPUT_TOKENS },
+    // This is the advisor's own (strongest) profile: when it's also the chat
+    // profile there's nothing stronger to consult, so the advisor defaults off.
+    advisorEnabled: false,
   },
   "cost-optimized": {
     intent: "latency-optimized",
@@ -74,20 +83,6 @@ const MANAGED_PROFILE_TEMPLATES: Record<string, ManagedProfileTemplate> = {
     thinking: { enabled: false, streamThinking: false },
     contextWindow: { maxInputTokens: DEFAULT_CONTEXT_WINDOW_MAX_INPUT_TOKENS },
   },
-  // Open-weight economy option: MiniMax M3 served by Fireworks via managed
-  // platform inference.
-  "balanced-economy": {
-    intent: "balanced",
-    provider: "fireworks",
-    connectionName: "fireworks-managed",
-    source: "managed",
-    label: "Balanced Economy",
-    description: "Strong open model (MiniMax M3) at a lower price point",
-    maxTokens: 32000,
-    effort: "high",
-    thinking: { enabled: true, streamThinking: true },
-    contextWindow: { maxInputTokens: DEFAULT_CONTEXT_WINDOW_MAX_INPUT_TOKENS },
-  },
 };
 
 /**
@@ -144,8 +139,37 @@ const USER_PROFILE_TEMPLATES: Record<string, ManagedProfileTemplate> = {
  */
 export const AUTO_PROFILE_KEY = "auto";
 
+export const OS_BETA_PROFILE_KEY = "os-beta";
+export const OS_BETA_FEATURE_FLAG_KEY = "os-beta";
+
+/**
+ * Flag-gated managed profile. NOT in MANAGED_PROFILE_TEMPLATES, so the
+ * unconditional boot seed never creates it. Reconciled in/out by
+ * the flag-gated profile reconcile based on the `os-beta` feature flag.
+ * Balanced-parity defaults; GLM 5.2 pinned explicitly via `model`.
+ */
+export const OS_BETA_PROFILE_TEMPLATE: ManagedProfileTemplate = {
+  model: "accounts/fireworks/models/glm-5p2",
+  provider: "fireworks",
+  connectionName: "fireworks-managed",
+  source: "managed",
+  label: "OS Beta",
+  description: "Open-source frontier model (GLM 5.2), in beta",
+  maxTokens: 32000,
+  effort: "high",
+  thinking: { enabled: true, streamThinking: true },
+  contextWindow: { maxInputTokens: DEFAULT_CONTEXT_WINDOW_MAX_INPUT_TOKENS },
+};
+
+// Membership here marks a name as managed. The route layer applies managed
+// restrictions (blocking model/provider edits and deletion) only to entries
+// whose on-disk `source` is `managed`, so a user-owned profile sharing one of
+// these names is not locked. `OS_BETA_PROFILE_KEY` is flag-gated: it is
+// materialized by the flag-gated profile reconcile, which refuses to touch a
+// same-named user profile.
 export const MANAGED_PROFILE_NAMES = new Set([
   ...Object.keys(MANAGED_PROFILE_TEMPLATES),
+  OS_BETA_PROFILE_KEY,
   AUTO_PROFILE_KEY,
 ]);
 
@@ -167,10 +191,17 @@ export type SeedInferenceProfilesOptions = {
  *
  * Runs on every daemon startup. Two responsibilities:
  *
- * 1. **Managed profiles** (`balanced`, `quality-optimized`, `cost-optimized`):
- *    overwritten on every boot so Vellum can push model/config updates to
- *    customers. Each carries `provider_connection: "anthropic-managed"`.
- *    Platform overlays (`preserveProfileNames`) take precedence.
+ * 1. **Managed profiles** (`balanced`, `quality-optimized`,
+ *    `cost-optimized`): reconciled from the code templates on every boot —
+ *    on-platform and off-platform alike — so Vellum can push model/config
+ *    updates to customers in a release without a workspace migration. The
+ *    templates own all profile content; `label`, `status`, `advisorEnabled`,
+ *    and `topP` are user overrides that survive reseeds. Of those, only
+ *    `label`, `status`, and `topP` are editable through the managed PUT route
+ *    allowlist; `advisorEnabled` is edited via the generic config path but is
+ *    still preserved here so it survives reboots.
+ *    Platform overlays (`preserveProfileNames`) take precedence for the boot
+ *    they are supplied.
  *
  * 2. **User profiles** (`custom-balanced`, `custom-quality-optimized`,
  *    `custom-cost-optimized`): materialized once at hatch time for
@@ -208,24 +239,31 @@ export function seedInferenceProfiles(
   // subsequent boot.
   const isByokMode = !isPlatform;
 
-  // 1. Managed profiles. Off-platform: overwrite on every boot so Vellum can
-  //    push model/config updates in new releases. On-platform: insert only if
-  //    absent — the platform controls profiles through overlays, and the
-  //    overlay fragment is authoritative even when it omits fields the local
-  //    template carries (e.g. an overlay supplying only provider/model/label
-  //    must not get its maxTokens/thinking polluted from the template). The
-  //    legacy migration-052 backfill that seeds label-less Anthropic
-  //    defaults is healed by workspace migration 082
-  //    (`backfill-managed-profile-labels`) rather than the seeder, so
-  //    this skip path stays simple.
+  // 1. Managed profiles. Reconciled from the code templates on every boot —
+  //    on-platform and off-platform alike — so Vellum can push model/config
+  //    updates in new releases just by editing `MANAGED_PROFILE_TEMPLATES` /
+  //    `model-intents.ts` and shipping a release, with no workspace migration.
+  //    The templates are the single source of truth for profile *content*
+  //    (model, maxTokens, effort, thinking, description, provider/connection).
+  //
+  //    Platform overlays (`preserveProfileNames`) still take precedence for the
+  //    boot they are supplied: a profile named in the overlay is skipped here so
+  //    the overlay fragment lands verbatim and is never polluted by template
+  //    fields it omits. The overlay is a one-time hatch input (archived after
+  //    its first merge), so on subsequent boots the templates reconcile content
+  //    as usual.
   //
-  //    Two user-editable fields survive the overwrite: `label` (display
-  //    rename) and `status` (active/disabled toggle). The PUT route
-  //    `/v1/config/llm/profiles/:name` lets users patch these on managed
-  //    profiles without duplicating; we have to honor those edits across
-  //    reseeds or they'd silently revert on every boot. Carry by
-  //    key-presence rather than truthiness so an explicit `null` (user
-  //    cleared the label) survives too.
+  //    A whitelist of user-editable fields survives the reconcile: `label`
+  //    (display rename), `status` (active/disabled toggle), `advisorEnabled`
+  //    (per-profile advisor toggle), and `topP` (sampling override) — the only
+  //    fields a user may override. The managed PUT route
+  //    `/v1/config/llm/profiles/:name` lets users patch `label`, `status`, and
+  //    `topP` on managed profiles without duplicating (its editable allowlist,
+  //    `MANAGED_PROFILE_EDITABLE_KEYS`, deliberately excludes `advisorEnabled`,
+  //    which is set through the generic config path). We honor every one of
+  //    these overrides across reseeds or they'd silently revert on every boot.
+  //    Carry by key-presence rather than truthiness so an explicit `null` (user
+  //    cleared the field) survives too.
   //
   //    BYOK seed defaults (off-platform only):
   //      • label: " (Managed)" suffix disambiguates managed profile labels
@@ -249,7 +287,6 @@ export function seedInferenceProfiles(
 
   for (const [name, template] of Object.entries(MANAGED_PROFILE_TEMPLATES)) {
     if (preservedProfileNames.has(name)) continue;
-    if (isPlatform && readObject(profiles[name]) !== null) continue;
 
     const previous = readObject(profiles[name]);
     const effectiveTemplate: ManagedProfileTemplate = isByokMode
@@ -281,6 +318,18 @@ export function seedInferenceProfiles(
             : previous.label;
       }
       if ("status" in previous) next.status = previous.status;
+      // The per-profile advisor toggle is a user override — preserve it across
+      // reseeds so a user's choice survives reboots (the template value only
+      // seeds the initial default, e.g. off for quality-optimized).
+      if ("advisorEnabled" in previous) {
+        next.advisorEnabled = previous.advisorEnabled;
+      }
+      // `topP` is user-editable on managed profiles (see the managed-profile
+      // editable allowlist on the PUT route) — preserve a user override across
+      // reseeds, including an explicit `null` clear, or it would silently revert
+      // to the template value on every boot. Carry by key-presence (not
+      // truthiness) so `null` survives too.
+      if ("topP" in previous) next.topP = previous.topP;
     }
     profiles[name] = next as ProfileEntry;
   }
@@ -364,6 +413,17 @@ export function seedInferenceProfiles(
     }
   }
 
+  // Advisor profile: default to the strongest managed profile when unset, so
+  // the advisor consults `quality-optimized` out of the box. Guarded on
+  // existence so it never names a missing profile (superRefine rejects that);
+  // off-platform/BYOK installs can repoint it at one of their own profiles.
+  if (
+    readString(llm.advisorProfile) === undefined &&
+    readObject(profiles["quality-optimized"]) !== null
+  ) {
+    llm.advisorProfile = "quality-optimized";
+  }
+
   // Profile ordering — ensure all seeded profiles appear in the order array.
   // "auto" is prepended so it appears first in the picker.
   const profileOrder = Array.isArray(llm.profileOrder)
@@ -405,21 +465,26 @@ export function seedInferenceProfiles(
   saveRawConfig(config);
 }
 
-function materializeProfile(
+export function materializeProfile(
   template: ManagedProfileTemplate,
   provider: NonNullable<ProfileEntry["provider"]>,
   connectionName: string,
 ): ProfileEntry {
-  const { intent, provider: _p, connectionName: _c, ...rest } = template;
+  const { intent, model, provider: _p, connectionName: _c, ...rest } = template;
+  const resolvedModel =
+    model ?? (intent ? resolveModelIntent(provider, intent) : undefined);
+  if (!resolvedModel) {
+    throw new Error("ManagedProfileTemplate requires `intent` or `model`");
+  }
   return {
     ...rest,
     provider,
     provider_connection: connectionName,
-    model: resolveModelIntent(provider, intent),
+    model: resolvedModel,
   };
 }
 
-function readObject(value: unknown): Record<string, unknown> | null {
+export function readObject(value: unknown): Record<string, unknown> | null {
   return value !== null && typeof value === "object" && !Array.isArray(value)
     ? (value as Record<string, unknown>)
     : null;

package/src/config/skills.ts

@@ -47,6 +47,7 @@ const VellumMetadataSchema = z
     "activation-hints": z.array(z.string()).optional(),
     "avoid-when": z.array(z.string()).optional(),
     category: z.string().optional(),
+    "always-candidate": z.boolean().optional(),
   })
   .passthrough();
 
@@ -109,6 +110,13 @@ export interface SkillSummary {
   avoidWhen?: string[];
   /** Category slug declared in frontmatter, used as a fallback when the skill is not in the Vellum catalog. */
   category?: string;
+  /**
+   * When true, this skill is pinned into the memory-v3 selector's stable-prefix
+   * candidate pool every turn (so the selector can choose it even when no
+   * retrieval lane surfaces it). For cross-cutting capabilities whose relevance
+   * the model must judge, not embedding similarity.
+   */
+  alwaysCandidate?: boolean;
   /** Parsed inline command expansion descriptors (`!\`command\``) found in the skill body. */
   inlineCommandExpansions?: InlineCommandExpansion[];
 }
@@ -227,6 +235,7 @@ interface ParsedFrontmatter {
   activationHints?: string[];
   avoidWhen?: string[];
   category?: string;
+  alwaysCandidate?: boolean;
   inlineCommandExpansions?: InlineCommandExpansion[];
 }
 
@@ -342,6 +351,11 @@ function parseFrontmatter(
       ? vellum.category.trim()
       : undefined;
 
+  const alwaysCandidate =
+    typeof vellum?.["always-candidate"] === "boolean"
+      ? vellum["always-candidate"]
+      : undefined;
+
   const strippedBody = stripCommentLines(body);
 
   // Parse inline command expansions from the body (after frontmatter/comment stripping)
@@ -366,6 +380,7 @@ function parseFrontmatter(
     activationHints,
     avoidWhen,
     category,
+    alwaysCandidate,
     inlineCommandExpansions,
   };
 }
@@ -523,6 +538,7 @@ function readSkillFromDirectory(
       activationHints: parsed.activationHints,
       avoidWhen: parsed.avoidWhen,
       category: parsed.category,
+      alwaysCandidate: parsed.alwaysCandidate,
       inlineCommandExpansions: parsed.inlineCommandExpansions,
     };
   } catch (err) {
@@ -576,6 +592,7 @@ function readBundledSkillFromDirectory(
       activationHints: parsed.activationHints,
       avoidWhen: parsed.avoidWhen,
       category: parsed.category,
+      alwaysCandidate: parsed.alwaysCandidate,
       inlineCommandExpansions: parsed.inlineCommandExpansions,
     };
   } catch (err) {
@@ -637,6 +654,7 @@ function loadBundledSkills(): SkillSummary[] {
       activationHints: skill.activationHints,
       avoidWhen: skill.avoidWhen,
       category: skill.category,
+      alwaysCandidate: skill.alwaysCandidate,
       inlineCommandExpansions: skill.inlineCommandExpansions,
     });
   }
@@ -764,6 +782,7 @@ function skillSummaryFromDefinition(
     activationHints: skill.activationHints,
     avoidWhen: skill.avoidWhen,
     category: skill.category,
+    alwaysCandidate: skill.alwaysCandidate,
     inlineCommandExpansions: skill.inlineCommandExpansions,
   };
 }
@@ -818,6 +837,7 @@ export function loadSkillCatalog(
             activationHints: parsed.activationHints,
             avoidWhen: parsed.avoidWhen,
             category: parsed.category,
+            alwaysCandidate: parsed.alwaysCandidate,
             inlineCommandExpansions: parsed.inlineCommandExpansions,
           });
         } catch (err) {
@@ -954,6 +974,7 @@ export function loadSkillCatalog(
           activationHints: parsed.activationHints,
           avoidWhen: parsed.avoidWhen,
           category: parsed.category,
+          alwaysCandidate: parsed.alwaysCandidate,
           inlineCommandExpansions: parsed.inlineCommandExpansions,
         };
 

package/src/config/sync-gated-profiles.ts

@@ -0,0 +1,220 @@
+import { isDeepStrictEqual } from "node:util";
+
+import { getLogger } from "../util/logger.js";
+import { isAssistantFeatureFlagEnabled } from "./assistant-feature-flags.js";
+import {
+  getConfigReadOnly,
+  invalidateConfigCache,
+  loadRawConfig,
+  saveRawConfig,
+} from "./loader.js";
+import type { ProfileEntry } from "./schemas/llm.js";
+import {
+  materializeProfile,
+  OS_BETA_FEATURE_FLAG_KEY,
+  OS_BETA_PROFILE_KEY,
+  OS_BETA_PROFILE_TEMPLATE,
+  readObject,
+} from "./seed-inference-profiles.js";
+
+const log = getLogger("sync-gated-profiles");
+
+/**
+ * Reconcile flag-gated managed profiles against the current feature-flag state.
+ *
+ * `seedInferenceProfiles()` runs synchronously at boot before feature flags are
+ * available, so the OS Beta profile (GLM 5.2 / fireworks-managed) is materialized
+ * here once flags have loaded. When the `os-beta` flag is on, the managed profile
+ * is created (ordered right after `balanced`); when it is off, a previously
+ * managed entry is removed with `profileOrder` / `activeProfile` / `advisorProfile`
+ * fallbacks. The reconcile is idempotent and never touches a user-owned profile of
+ * the same name.
+ *
+ * Returns whether the on-disk config changed.
+ */
+export function reconcileFlagGatedProfiles(): boolean {
+  const config = loadRawConfig();
+
+  if (config.llm == null || typeof config.llm !== "object") {
+    config.llm = {};
+  }
+  const llm = config.llm as Record<string, unknown>;
+
+  if (llm.profiles == null || typeof llm.profiles !== "object") {
+    llm.profiles = {};
+  }
+  const profiles = llm.profiles as Record<string, Record<string, unknown>>;
+
+  const profileOrder = Array.isArray(llm.profileOrder)
+    ? (llm.profileOrder as string[])
+    : [];
+  llm.profileOrder = profileOrder;
+
+  // The resolver reads flag state from the gateway-populated override cache and
+  // ignores the config argument; pass the read-only config for signature parity
+  // without mutating disk before the reconcile decision is made.
+  const enabled = isAssistantFeatureFlagEnabled(
+    OS_BETA_FEATURE_FLAG_KEY,
+    getConfigReadOnly(),
+  );
+
+  const isPlatform =
+    process.env.IS_PLATFORM === "true" || process.env.IS_PLATFORM === "1";
+  const isByokMode = !isPlatform;
+
+  const previous = readObject(profiles[OS_BETA_PROFILE_KEY]);
+
+  // Never clobber a user-owned profile that happens to be named `os-beta`. The
+  // entry is ours to manage only when it is absent or already managed; a
+  // user-sourced entry of the same name is left untouched on every path.
+  const isOursToManage = previous == null || previous.source === "managed";
+  if (!isOursToManage) {
+    return false;
+  }
+
+  const changed = enabled
+    ? enableProfile(profiles, profileOrder, previous, isByokMode)
+    : disableProfile(llm, profiles, profileOrder, previous);
+
+  if (changed) {
+    saveRawConfig(config);
+    invalidateConfigCache();
+    log.info(
+      { profile: OS_BETA_PROFILE_KEY, enabled },
+      "Reconciled flag-gated profile",
+    );
+  }
+  return changed;
+}
+
+function enableProfile(
+  profiles: Record<string, Record<string, unknown>>,
+  profileOrder: string[],
+  previous: Record<string, unknown> | null,
+  isByokMode: boolean,
+): boolean {
+  const effectiveTemplate = isByokMode
+    ? {
+        ...OS_BETA_PROFILE_TEMPLATE,
+        label: `${OS_BETA_PROFILE_TEMPLATE.label} (Managed)`,
+      }
+    : OS_BETA_PROFILE_TEMPLATE;
+  const next = materializeProfile(
+    effectiveTemplate,
+    OS_BETA_PROFILE_TEMPLATE.provider,
+    OS_BETA_PROFILE_TEMPLATE.connectionName,
+  ) as Record<string, unknown>;
+
+  // BYOK installs seed managed profiles disabled: the platform-auth
+  // `fireworks-managed` connection backing this profile isn't usable until the
+  // user enables it, so a fresh OS Beta entry starts disabled to avoid offering
+  // an unusable route. A user's own status override (preserved below) wins on
+  // later reconciles.
+  if (isByokMode && !previous) {
+    next.status = "disabled";
+  }
+
+  if (previous) {
+    // The only fields a user may override on a managed profile. Carry `label`
+    // by key-presence so an explicit null (user cleared it) survives too.
+    if ("label" in previous) next.label = previous.label;
+    if ("status" in previous) next.status = previous.status;
+    if ("advisorEnabled" in previous) {
+      next.advisorEnabled = previous.advisorEnabled;
+    }
+  }
+
+  let changed = false;
+  if (!previous || !isDeepStrictEqual(previous, next)) {
+    profiles[OS_BETA_PROFILE_KEY] = next as ProfileEntry;
+    changed = true;
+  }
+
+  if (!profileOrder.includes(OS_BETA_PROFILE_KEY)) {
+    const balancedIndex = profileOrder.indexOf("balanced");
+    if (balancedIndex >= 0) {
+      profileOrder.splice(balancedIndex + 1, 0, OS_BETA_PROFILE_KEY);
+    } else {
+      profileOrder.push(OS_BETA_PROFILE_KEY);
+    }
+    changed = true;
+  }
+
+  return changed;
+}
+
+// `MixSchema = z.array(MixArmSchema).min(2)` in schemas/llm.ts: mixes require
+// >= 2 arms. A mix that drops below this is invalid and cannot be kept.
+const MIX_MIN_ARMS = 2;
+
+function disableProfile(
+  llm: Record<string, unknown>,
+  profiles: Record<string, Record<string, unknown>>,
+  profileOrder: string[],
+  previous: Record<string, unknown> | null,
+): boolean {
+  if (!previous) return false;
+
+  delete profiles[OS_BETA_PROFILE_KEY];
+
+  // The removal closure: every name here is absent from `profiles` once the
+  // closure settles, so the written config can never reference one. A mix that
+  // loses arms below the >= 2 minimum is itself invalid, so it joins the set
+  // and the loop runs to a fixpoint to resolve any references that cascade.
+  const removed = new Set<string>([OS_BETA_PROFILE_KEY]);
+  let cascading = true;
+  while (cascading) {
+    cascading = false;
+    for (const [name, profile] of Object.entries(profiles)) {
+      if (removed.has(name)) continue;
+      if (!Array.isArray(profile.mix)) continue;
+      const arms = profile.mix as unknown[];
+      const kept = arms.filter((arm) => {
+        const armProfile = readObject(arm)?.profile;
+        return typeof armProfile !== "string" || !removed.has(armProfile);
+      });
+      if (kept.length === arms.length) continue;
+      if (kept.length >= MIX_MIN_ARMS) {
+        profile.mix = kept;
+      } else {
+        delete profiles[name];
+        removed.add(name);
+      }
+      cascading = true;
+    }
+  }
+
+  llm.profileOrder = profileOrder.filter((name) => !removed.has(name));
+
+  if (typeof llm.activeProfile === "string" && removed.has(llm.activeProfile)) {
+    llm.activeProfile = "balanced";
+  }
+  if (
+    typeof llm.advisorProfile === "string" &&
+    removed.has(llm.advisorProfile)
+  ) {
+    if (readObject(profiles["quality-optimized"]) !== null) {
+      llm.advisorProfile = "quality-optimized";
+    } else {
+      delete llm.advisorProfile;
+    }
+  }
+
+  // Clear any call-site `profile` reference to a removed profile; other override
+  // fields on the entry stay intact (an empty override object is valid).
+  const callSites = readObject(llm.callSites);
+  if (callSites) {
+    for (const entry of Object.values(callSites)) {
+      const site = readObject(entry);
+      if (
+        site &&
+        typeof site.profile === "string" &&
+        removed.has(site.profile)
+      ) {
+        delete site.profile;
+      }
+    }
+  }
+
+  return true;
+}