@vellumai/assistant 0.11.3 → 0.11.4-dev.202608190019.b94dbf2

package/src/config/__tests__/balanced-model-experiment.test.ts

@@ -0,0 +1,278 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { afterEach, describe, expect, test } from "bun:test";
+
+import type { z } from "zod";
+
+import { setOverridesForTesting } from "../../__tests__/feature-flag-test-helpers.js";
+import {
+  catalogMaxOutputTokens,
+  isModelInCatalog,
+} from "../../providers/model-catalog.js";
+import { getManagedUpstream } from "../../providers/vellum-model-routing.js";
+import { BALANCED_MODEL_EXPERIMENT_FLAG_KEY } from "../balanced-model-experiment.js";
+import {
+  CODE_DEFAULT_PROFILE_ENTRIES,
+  getEffectiveProfilesForProvider,
+  resolveDefaultProfileForProvider,
+} from "../default-profile-catalog.js";
+import { clearCachedOverrides } from "../feature-flag-cache.js";
+import { resolveCallSiteConfig } from "../llm-resolver.js";
+import {
+  type DefaultProviderConfig,
+  LLMSchema,
+  type ProfileEntry,
+} from "../schemas/llm.js";
+
+// The `experiment-balanced-model-2026-08-06` arm repoints the model the managed
+// Balanced profile resolves to. Both the runtime resolver and the client-facing
+// profile listing go through `defaultProfileBodyForProvider`, so both are
+// exercised here: a disagreement between them shows the user one model beside a
+// profile that runs another.
+
+const FLAG = BALANCED_MODEL_EXPERIMENT_FLAG_KEY;
+const SHIPPED_MODEL = "gpt-5.6-luna";
+const GLM_MODEL = "accounts/fireworks/models/glm-5p2";
+
+function setArm(value: boolean | string): void {
+  setOverridesForTesting({ [FLAG]: value });
+}
+
+const managed: DefaultProviderConfig = { provider: "vellum" };
+
+function llmWithActiveBalanced(
+  overrides: Record<string, unknown> = {},
+): z.infer<typeof LLMSchema> {
+  return LLMSchema.parse({
+    profiles: {},
+    profileOrder: [],
+    activeProfile: "balanced",
+    defaultProvider: managed,
+    ...overrides,
+  });
+}
+
+afterEach(() => {
+  clearCachedOverrides();
+});
+
+describe("balanced-model experiment arms", () => {
+  test("terra repoints mainAgent at gpt-5.6-terra on the managed connection", () => {
+    setArm("terra");
+    const resolved = resolveCallSiteConfig(
+      "mainAgent",
+      llmWithActiveBalanced(),
+    );
+    expect(resolved.model).toBe("gpt-5.6-terra");
+    expect(resolved.provider).toBe("vellum");
+    expect(getManagedUpstream("gpt-5.6-terra")).toBe("openai");
+
+    const entry = resolveDefaultProfileForProvider(
+      undefined,
+      "balanced",
+      managed,
+    );
+    expect(entry?.model).toBe("gpt-5.6-terra");
+    expect(entry?.provider).toBe("vellum");
+    // Routing-identity providers derive their connection per request.
+    expect(entry?.provider_connection).toBeUndefined();
+    expect(entry?.source).toBe("managed");
+  });
+
+  test("glm-5p2 repoints mainAgent at GLM 5.2 within the model's output cap", () => {
+    setArm("glm-5p2");
+    const resolved = resolveCallSiteConfig(
+      "mainAgent",
+      llmWithActiveBalanced(),
+    );
+    expect(resolved.model).toBe(GLM_MODEL);
+    expect(resolved.provider).toBe("vellum");
+
+    const upstream = getManagedUpstream(GLM_MODEL);
+    expect(upstream).toBe("fireworks");
+    const cap = catalogMaxOutputTokens(upstream as string, GLM_MODEL);
+    expect(cap).toBeDefined();
+    expect(resolved.maxTokens).toBeLessThanOrEqual(cap as number);
+    // Only the model moves: the shipped token budget stands.
+    expect(resolved.maxTokens).toBe(
+      CODE_DEFAULT_PROFILE_ENTRIES.balanced.maxTokens as number,
+    );
+  });
+
+  test("an arm changes only the model of the managed balanced body", () => {
+    setArm("terra");
+    const entry = resolveDefaultProfileForProvider(
+      undefined,
+      "balanced",
+      managed,
+    );
+    const shipped = CODE_DEFAULT_PROFILE_ENTRIES.balanced;
+    expect(entry).toEqual({ ...shipped, model: "gpt-5.6-terra" });
+  });
+
+  test("the other default profiles are untouched by an arm", () => {
+    setArm("glm-5p2");
+    for (const key of [
+      "quality-optimized",
+      "cost-optimized",
+      "latency-optimized",
+    ]) {
+      expect(
+        resolveDefaultProfileForProvider(undefined, key, managed)?.model,
+      ).toBe(CODE_DEFAULT_PROFILE_ENTRIES[key].model);
+    }
+  });
+
+  test("every pinned arm model is managed-routable and in its upstream catalog", () => {
+    for (const model of ["gpt-5.6-terra", GLM_MODEL]) {
+      const upstream = getManagedUpstream(model);
+      expect(upstream).not.toBeNull();
+      expect(isModelInCatalog(upstream as string, model)).toBe(true);
+    }
+  });
+});
+
+describe("balanced-model experiment fallbacks", () => {
+  const shippedCases: [string, () => void][] = [
+    ["the flag is unset", () => clearCachedOverrides()],
+    ["no override is present", () => setOverridesForTesting({})],
+    ["the arm is control", () => setArm("control")],
+    ["the arm is an unknown string", () => setArm("gpt-9-does-not-exist")],
+    ["the arm is the empty string", () => setArm("")],
+    ["the value is boolean true", () => setArm(true)],
+    ["the value is boolean false", () => setArm(false)],
+    // The arm is remote input, so it can name an Object.prototype member. The
+    // pin table is a Map, which has no such members to inherit.
+    ["the arm is constructor", () => setArm("constructor")],
+    ["the arm is toString", () => setArm("toString")],
+    ["the arm is valueOf", () => setArm("valueOf")],
+    ["the arm is __proto__", () => setArm("__proto__")],
+    ["the arm is hasOwnProperty", () => setArm("hasOwnProperty")],
+  ];
+
+  for (const [label, seed] of shippedCases) {
+    test(`balanced stays on the shipped model when ${label}`, () => {
+      seed();
+      expect(
+        resolveDefaultProfileForProvider(undefined, "balanced", managed)?.model,
+      ).toBe(SHIPPED_MODEL);
+      expect(
+        resolveCallSiteConfig("mainAgent", llmWithActiveBalanced()).model,
+      ).toBe(SHIPPED_MODEL);
+    });
+  }
+
+  test("the shipped catalog body is the control arm", () => {
+    expect(CODE_DEFAULT_PROFILE_ENTRIES.balanced.model).toBe(SHIPPED_MODEL);
+  });
+
+  test("the registry default is control, which the override-only flag read assumes", () => {
+    const registryPath = join(
+      import.meta.dirname,
+      "..",
+      "..",
+      "..",
+      "..",
+      "meta",
+      "feature-flags",
+      "feature-flag-registry.json",
+    );
+    const registry = JSON.parse(readFileSync(registryPath, "utf-8")) as {
+      flags: {
+        key: string;
+        scope: string;
+        defaultEnabled: boolean | string;
+        values?: string[];
+      }[];
+    };
+    const flag = registry.flags.find((entry) => entry.key === FLAG);
+    expect(flag).toBeDefined();
+    expect(flag?.scope).toBe("assistant");
+    expect(flag?.defaultEnabled).toBe("control");
+    expect(flag?.values).toEqual(["control", "terra", "glm-5p2"]);
+  });
+});
+
+describe("balanced-model experiment boundaries", () => {
+  test("a user-owned balanced shadow wins over the arm", () => {
+    setArm("terra");
+    const profiles: Record<string, ProfileEntry> = {
+      balanced: { source: "user", provider: "openai", model: "gpt-5.5" },
+    };
+    expect(
+      resolveDefaultProfileForProvider(profiles, "balanced", managed),
+    ).toEqual(profiles.balanced);
+    expect(
+      resolveCallSiteConfig("mainAgent", llmWithActiveBalanced({ profiles }))
+        .model,
+    ).toBe("gpt-5.5");
+  });
+
+  test("a managed-source stub still resolves to the arm's body", () => {
+    setArm("terra");
+    const profiles: Record<string, ProfileEntry> = {
+      balanced: { source: "managed", label: "My Balanced" },
+    };
+    const entry = resolveDefaultProfileForProvider(
+      profiles,
+      "balanced",
+      managed,
+    );
+    expect(entry?.model).toBe("gpt-5.6-terra");
+    expect(entry?.label).toBe("My Balanced");
+  });
+
+  test("the chatgpt and BYOK columns stay out of the experiment", () => {
+    setArm("terra");
+    for (const provider of ["anthropic", "openai", "chatgpt"] as const) {
+      const armed = resolveDefaultProfileForProvider(undefined, "balanced", {
+        provider,
+      });
+      clearCachedOverrides();
+      const shipped = resolveDefaultProfileForProvider(undefined, "balanced", {
+        provider,
+      });
+      expect(armed).toEqual(shipped as ProfileEntry);
+      expect(armed?.provider).toBe(provider);
+      setArm("terra");
+    }
+  });
+
+  test("an install with no defaultProvider still picks up the arm", () => {
+    setArm("glm-5p2");
+    expect(
+      resolveDefaultProfileForProvider(undefined, "balanced", null)?.model,
+    ).toBe(GLM_MODEL);
+  });
+});
+
+describe("client-facing profile listing", () => {
+  test("reports the arm's model so the UI matches what runs", () => {
+    setArm("terra");
+    const profiles = getEffectiveProfilesForProvider(undefined, managed);
+    expect(profiles.balanced?.model).toBe("gpt-5.6-terra");
+    expect(profiles["quality-optimized"]?.model).toBe(
+      CODE_DEFAULT_PROFILE_ENTRIES["quality-optimized"].model,
+    );
+  });
+
+  test("reports the shipped model on the control arm", () => {
+    setArm("control");
+    expect(
+      getEffectiveProfilesForProvider(undefined, managed).balanced?.model,
+    ).toBe(SHIPPED_MODEL);
+  });
+
+  test("agrees with the runtime resolver on every arm", () => {
+    for (const arm of ["control", "terra", "glm-5p2", "nonsense"]) {
+      setArm(arm);
+      const listed = getEffectiveProfilesForProvider(undefined, managed)
+        .balanced?.model;
+      const resolved = resolveCallSiteConfig(
+        "mainAgent",
+        llmWithActiveBalanced(),
+      ).model;
+      expect(listed).toBe(resolved as string);
+    }
+  });
+});

package/src/config/__tests__/default-profile-catalog.test.ts

@@ -7,6 +7,7 @@ import {
   CODE_DEFAULT_PROFILE_ENTRIES,
   getEffectiveProfile,
   getEffectiveProfiles,
+  getEffectiveProfilesForProvider,
   PROFILE_IMPLS,
   resolveDefaultProfileForProvider,
 } from "../default-profile-catalog.js";
@@ -22,6 +23,7 @@ import {
 import {
   type DefaultProviderConfig,
   type LLMCallSite,
+  LLMCallSiteEnum,
   LLMSchema,
   type ProfileEntry,
 } from "../schemas/llm.js";
@@ -245,7 +247,10 @@ describe("resolver integration", () => {
       },
     });
     const body = CODE_DEFAULT_PROFILE_ENTRIES["latency-optimized"]!;
-    for (const callSite of ["voiceFrontDoor", "voiceFrontDecision"] as const) {
+    for (const callSite of [
+      "voiceFrontDoor",
+      "voiceProgressNarration",
+    ] as const) {
       const resolved = resolveCallSiteConfig(callSite, llm);
       expect(resolved.model).toBe(body.model as string);
       expect(resolved.model).not.toBe("claude-opus-4-6");
@@ -265,6 +270,12 @@ describe("resolver integration", () => {
 });
 
 describe("schema validation", () => {
+  test("voiceFrontDoor is the only front callsite", () => {
+    expect(
+      LLMCallSiteEnum.options.filter((callSite) => callSite.includes("Front")),
+    ).toEqual(["voiceFrontDoor"]);
+  });
+
   test("always-available default names are valid references; os-beta only when materialized", () => {
     expect(() => LLMSchema.parse({ activeProfile: "balanced" })).not.toThrow();
     expect(() =>
@@ -323,7 +334,7 @@ describe("resolveDefaultProfileForProvider", () => {
         expect(typeof entry?.model).toBe("string");
         expect(entry?.provider).toBeDefined();
         // Identity columns stamp no connection; BYOK columns always do.
-        if (entry?.provider === "vellum") {
+        if (entry?.provider === "vellum" || entry?.provider === "chatgpt") {
           expect(entry?.provider_connection).toBeUndefined();
         } else {
           expect(entry?.provider_connection).toBeDefined();
@@ -333,6 +344,27 @@ describe("resolveDefaultProfileForProvider", () => {
     }
   });
 
+  test("the chatgpt column resolves Codex-pinned models with no connection stamp", () => {
+    const byKey: Record<string, string> = {
+      balanced: "gpt-5.6-luna",
+      "quality-optimized": "gpt-5.6-sol",
+      "cost-optimized": "gpt-5.6-luna",
+      "latency-optimized": "gpt-5.6-luna",
+    };
+    const effective = getEffectiveProfilesForProvider(undefined, dp("chatgpt"));
+    for (const [key, model] of Object.entries(byKey)) {
+      const entry = effective[key];
+      expect(entry?.provider).toBe("chatgpt");
+      expect(entry?.model).toBe(model);
+      expect(entry?.provider_connection).toBeUndefined();
+      expect(entry?.source).toBe("managed");
+    }
+    // Cost and Speed both opt fully out of reasoning.
+    expect(effective["cost-optimized"]?.effort).toBe("none");
+    expect(effective["latency-optimized"]?.effort).toBe("none");
+    expect(effective.balanced?.thinking?.enabled).toBe(true);
+  });
+
   test("a provider without a named matrix column materializes from the shared BYOK templates", () => {
     const entry = resolveDefaultProfileForProvider(
       undefined,

package/src/config/__tests__/default-provider.test.ts

@@ -157,6 +157,12 @@ describe("resolveDefaultConnectionName", () => {
     );
   });
 
+  test("chatgpt resolves to the canonical subscription connection name", () => {
+    expect(resolveDefaultConnectionName({ provider: "chatgpt" })).toBe(
+      "chatgpt-subscription",
+    );
+  });
+
   test("every other provider resolves to its personal connection", () => {
     expect(resolveDefaultConnectionName({ provider: "anthropic" })).toBe(
       "anthropic-personal",
@@ -221,7 +227,6 @@ describe("getDefaultProvider / setDefaultProvider", () => {
   test("setDefaultProvider validates the provider before writing", () => {
     expect(() =>
       setDefaultProvider({
-        // @ts-expect-error deliberately invalid for the test
         provider: "not-a-provider",
       }),
     ).toThrow();

package/src/config/__tests__/profile-materialization.test.ts

@@ -1,4 +1,17 @@
-import { describe, expect, test } from "bun:test";
+import { describe, expect, mock, test } from "bun:test";
+
+// Entry-name folds are gated on the named row existing with an agreeing
+// kind, so the tests control the row store directly.
+const connectionRows = new Map<string, { name: string; provider: string }>([
+  ["anthropic-personal", { name: "anthropic-personal", provider: "anthropic" }],
+]);
+mock.module("../../persistence/db-connection.js", () => ({
+  getDb: () => ({}),
+}));
+mock.module("../../providers/inference/connections.js", () => ({
+  getConnection: (_db: unknown, name: string) =>
+    connectionRows.get(name) ?? null,
+}));
 
 import { VELLUM_MANAGED_CONNECTION_NAME } from "../../providers/vellum-model-routing.js";
 import { completeCustomProfile } from "../profile-materialization.js";
@@ -28,8 +41,10 @@ describe("completeCustomProfile", () => {
       model: "claude-fable-5",
     });
     expect(completed.model).toBe("claude-fable-5");
-    expect(completed.provider).toBe("anthropic");
-    expect(completed.provider_connection).toBe("anthropic-personal");
+    // The default's explicit binding is inherited IN the provider value
+    // (the entries-model representation), never as a stamped binding.
+    expect(completed.provider).toBe("anthropic-personal");
+    expect(completed.provider_connection).toBeUndefined();
     expect(completed.maxTokens).toBe(64000);
     expect(completed.effort).toBe("max");
     expect(completed.speed).toBe("standard");
@@ -78,12 +93,12 @@ describe("completeCustomProfile", () => {
     expect(completed.contextWindow?.overflowRecovery?.enabled).toBe(true);
   });
 
-  test("keeps the inherited provider when it serves the profile's model", () => {
+  test("inherits the default's binding as the entry name when the provider serves the model", () => {
     const completed = completeCustomProfile(fullDefault, {
       model: "claude-fable-5",
     });
-    expect(completed.provider).toBe("anthropic");
-    expect(completed.provider_connection).toBe("anthropic-personal");
+    expect(completed.provider).toBe("anthropic-personal");
+    expect(completed.provider_connection).toBeUndefined();
   });
 
   test("stamps the catalog owner for a model the default provider does not serve, and drops the default's connection", () => {
@@ -123,36 +138,77 @@ describe("completeCustomProfile", () => {
     expect(completed.provider_connection).toBeUndefined();
   });
 
-  test("inherits the vellum managed connection across a provider change, but only onto managed-routable providers", () => {
+  test("a managed default's binding becomes the routing identity, never a stamped field", () => {
     const managedDefault = LLMConfigBase.parse({
       ...fullDefault,
       provider_connection: VELLUM_MANAGED_CONNECTION_NAME,
     });
     const implied = completeCustomProfile(managedDefault, { model: "gpt-5.5" });
-    expect(implied.provider).toBe("openai");
-    expect(implied.provider_connection).toBe(VELLUM_MANAGED_CONNECTION_NAME);
+    expect(implied.provider).toBe("vellum");
+    expect(implied.provider_connection).toBeUndefined();
 
     const explicit = completeCustomProfile(managedDefault, {
       provider: "openai",
       model: "gpt-5.4",
     });
-    expect(explicit.provider_connection).toBe(VELLUM_MANAGED_CONNECTION_NAME);
+    expect(explicit.provider).toBe("vellum");
+    expect(explicit.provider_connection).toBeUndefined();
+  });
 
-    // The vellum connection can't route a non-managed provider; baking it in
-    // would fail dispatch's mismatch path instead of auto-resolving.
-    const nonRoutable = completeCustomProfile(managedDefault, {
-      provider: "openrouter",
-      model: "minimax/minimax-m3",
+  test("inherits the binding as the entry name even for a model unknown to the catalog", () => {
+    const completed = completeCustomProfile(fullDefault, {
+      model: "totally-custom-model",
     });
-    expect(nonRoutable.provider_connection).toBeUndefined();
+    expect(completed.provider).toBe("anthropic-personal");
+    expect(completed.provider_connection).toBeUndefined();
   });
 
-  test("keeps the inherited provider for a model unknown to the catalog", () => {
-    const completed = completeCustomProfile(fullDefault, {
+  test("a dangling default binding passes through as the legacy field", () => {
+    const dangling = LLMConfigBase.parse({
+      ...fullDefault,
+      provider_connection: "deleted-row",
+    });
+    const completed = completeCustomProfile(dangling, {
+      model: "claude-fable-5",
+    });
+    // Folding an unverifiable binding would hide it from the collapse
+    // migration's dangling recovery; the legacy field keeps it visible.
+    expect(completed.provider).toBe("anthropic");
+    expect(completed.provider_connection).toBe("deleted-row");
+  });
+
+  test("a kind-disagreeing default binding passes through as the legacy field", () => {
+    connectionRows.set("mislabeled", {
+      name: "mislabeled",
+      provider: "openai",
+    });
+    try {
+      const mismatched = LLMConfigBase.parse({
+        ...fullDefault,
+        provider_connection: "mislabeled",
+      });
+      const completed = completeCustomProfile(mismatched, {
+        model: "claude-fable-5",
+      });
+      expect(completed.provider).toBe("anthropic");
+      expect(completed.provider_connection).toBe("mislabeled");
+    } finally {
+      connectionRows.delete("mislabeled");
+    }
+  });
+
+  test("a managed default binding is not inherited when the identity cannot serve the model", () => {
+    const managedDefault = LLMConfigBase.parse({
+      ...fullDefault,
+      provider_connection: VELLUM_MANAGED_CONNECTION_NAME,
+    });
+    const completed = completeCustomProfile(managedDefault, {
       model: "totally-custom-model",
     });
+    // Dispatch auto-resolves by vendor instead of pinning an unservable
+    // managed route.
     expect(completed.provider).toBe("anthropic");
-    expect(completed.provider_connection).toBe("anthropic-personal");
+    expect(completed.provider_connection).toBeUndefined();
   });
 
   test("passes mix profiles through untouched", () => {

package/src/config/assistant-feature-flags.ts

@@ -59,6 +59,12 @@ function loadDefaultsRegistry(): FeatureFlagDefaultsRegistry {
     // Works in Docker / packaged builds where the repo-root `meta/` dir
     // is not available.
     join(thisDir, REGISTRY_FILENAME),
+    ...(process.platform === "win32"
+      ? [
+          // Packaged Windows CLI runtime: assets live beside the executable.
+          join(dirname(process.execPath), REGISTRY_FILENAME),
+        ]
+      : []),
     // Packaged macOS app layout: the daemon binary lives at
     // <App>.app/Contents/MacOS/vellum-daemon and the registry is copied
     // to <App>.app/Contents/Resources/ by build.sh. In bun --compile
@@ -190,11 +196,11 @@ const DEFAULT_INIT_RETRY_BACKOFFS_MS: readonly number[] = [
  * intentionally simulate an unreachable gateway and want immediate
  * fallback without waiting through the production schedule).
  *
- * No-ops when the cache is already populated — callers that want to
- * refresh must call `clearFeatureFlagOverridesCache()` first. This lets
- * tests preseed flag state via `setOverridesForTesting()` (in
- * `__tests__/feature-flag-test-helpers.ts`) without the gateway IPC call
- * clobbering their setup.
+ * No-ops when the cache is already populated — this is the load-time
+ * populate, and `refreshOverridesFromGateway()` is what re-reads a cache that
+ * already holds values. The no-op also lets tests preseed flag state via
+ * `setOverridesForTesting()` (in `__tests__/feature-flag-test-helpers.ts`)
+ * without the gateway IPC call clobbering their setup.
  *
  * Resolves `true` when the override cache is populated from the gateway and
  * `false` when the cache is left unset (exhausted retries / unreachable
@@ -274,10 +280,13 @@ function loadOverrides(): Record<string, boolean | string> {
  * Invalidate the cached overrides so the next call to
  * `isAssistantFeatureFlagEnabled` re-reads from the gateway.
  *
- * Called by `refreshOverridesFromGateway()` when the gateway pushes a
- * `feature_flags_changed` event, and by tests between cases to reset
- * module state. (Tests typically call `setOverridesForTesting()` from
- * `__tests__/feature-flag-test-helpers.ts`, which combines clear + seed.)
+ * Used by tests between cases to reset module state. (Tests typically call
+ * `setOverridesForTesting()` from `__tests__/feature-flag-test-helpers.ts`,
+ * which combines clear + seed.)
+ *
+ * Not a step in refreshing from the gateway: `refreshOverridesFromGateway()`
+ * swaps the cache atomically so flag values never read as unset while a
+ * refresh is in flight.
  */
 export function clearFeatureFlagOverridesCache(): void {
   clearCachedOverrides();
@@ -286,18 +295,30 @@ export function clearFeatureFlagOverridesCache(): void {
 /**
  * Re-fetch feature flag overrides from the gateway.
  *
- * Clears the cached overrides and re-runs the gateway IPC fetch without
- * retries (the gateway is known to be up because it just pushed an event).
- * Called by the gateway flag listener when a `feature_flags_changed` event
- * arrives.
+ * Runs the gateway IPC fetch without retries (the gateway is known to be up
+ * because it just pushed an event) and replaces the cache only once that fetch
+ * succeeds. Called by the gateway flag listener when a `feature_flags_changed`
+ * event arrives and on reconnect.
+ *
+ * The swap is atomic on purpose. The previously fetched values stay readable
+ * for the whole call, so a sync flag read that lands mid-refresh sees the last
+ * known values rather than an empty cache, and a failed fetch leaves them in
+ * place rather than dropping every flag to its registry default until the next
+ * event. A flag that selects behavior rather than gating it (an experiment
+ * arm, a kill switch) therefore cannot flip to its default because one IPC
+ * call timed out.
  *
  * Resolves `true` when the cache was repopulated from the gateway, `false`
  * when the fetch came back empty/failed — callers gate persisted-state
  * reconciliation on a `true` result.
  */
 export async function refreshOverridesFromGateway(): Promise<boolean> {
-  clearFeatureFlagOverridesCache();
-  return initFeatureFlagOverrides({ retryBackoffsMs: [] });
+  const gatewayOverrides = await fetchOverridesFromGateway();
+  if (Object.keys(gatewayOverrides).length === 0) {
+    return false;
+  }
+  setCachedOverrides(gatewayOverrides, { fromGateway: true });
+  return true;
 }
 
 // ---------------------------------------------------------------------------

package/src/config/balanced-model-experiment.ts

@@ -0,0 +1,35 @@
+/**
+ * Flag read for the managed Balanced profile's model A/B test.
+ *
+ * `experiment-balanced-model-2026-08-06` is a multivariate LaunchDarkly flag
+ * whose arm repoints the model the managed (`vellum`) implementation of the
+ * `balanced` default profile resolves to. The arm to model pins live beside
+ * the profile bodies in `default-profile-catalog.ts`, which validates their
+ * managed routability at load time; this module owns only the flag read.
+ *
+ * The read goes straight to the override cache rather than through
+ * `assistant-feature-flags.ts`. `feature-flag-cache.ts` is a stdlib-only leaf,
+ * so the profile catalog stays free of the pino logger and the gateway IPC
+ * client that resolver would pull into its import graph, and the registry
+ * declares `defaultEnabled: "control"` for this flag, which is the same
+ * shipped body an absent override already resolves to. That equivalence is
+ * pinned by `__tests__/balanced-model-experiment.test.ts` so it cannot drift.
+ */
+
+import { getCachedOverrides } from "./feature-flag-cache.js";
+
+export const BALANCED_MODEL_EXPERIMENT_FLAG_KEY =
+  "experiment-balanced-model-2026-08-06";
+
+/**
+ * The experiment arm in force, or `undefined` when the flag resolves to
+ * anything that is not an arm name: unset, a boolean, or the empty string.
+ * `control` and any arm this build does not know are returned as-is and miss
+ * the pin table, which is what keeps a stale or malformed LaunchDarkly value
+ * on the shipped model rather than stranding an install on one that does not
+ * exist.
+ */
+export function getBalancedModelExperimentArm(): string | undefined {
+  const value = getCachedOverrides()?.[BALANCED_MODEL_EXPERIMENT_FLAG_KEY];
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}

package/src/config/bundled-skills/acp/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: acp
-description: Spawn external coding agents via the Agent Client Protocol (ACP)
+description: Set up, authenticate, and run external coding agents (Claude Code, Codex) via the Agent Client Protocol
 compatibility: "Designed for Vellum personal assistants"
 metadata:
   emoji: "🔗"
@@ -8,13 +8,12 @@ metadata:
     display-name: "ACP"
     category: "development"
     activation-hints:
-      - "User asks to use Claude Code or Codex to do something"
-      - "User wants to delegate a coding task to Claude Code, Codex, or another ACP agent"
-      - "User wants to hand a coding task to another agent and check on it later"
-      - "User wants to spawn an external coding agent that runs autonomously and streams results back"
-      - "User mentions ACP, claude-agent-acp, codex-acp, or running multiple coding agents in parallel"
+      - "User wants to set up, install, configure, authenticate, or connect Claude Code or Codex"
+      - "User asks to use Claude Code or Codex, or delegate a coding task to an ACP agent"
+      - "User wants an agent to work autonomously and report back later"
+      - "User mentions ACP, claude-agent-acp, or codex-acp"
     avoid-when:
-      - "Task is small enough to do inline with the assistant's own tools - no need for an external agent"
+      - "The task is small enough to do inline"
 ---
 
 ACP agent orchestration - spawn external coding agents (Claude Code, Codex) to work on tasks via the Agent Client Protocol. Each agent runs as its own subprocess speaking ACP over stdio and streams results back into the conversation.