@vellumai/assistant 0.11.7-dev.202609010224.44cd29e → 0.11.7-dev.202609010322.2bc955e

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.11.7-dev.202609010224.44cd29e",
+  "version": "0.11.7-dev.202609010322.2bc955e",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/agent-loop-callsite-precedence.test.ts

@@ -175,6 +175,22 @@ describe("AgentLoop — call-site precedence", () => {
 
   test("call-site thinking wins over conversation default when callSite is set", async () => {
     setLlmConfig({
+      // Pinned to a model that can actually be told not to think. Without an
+      // active profile the call site resolves the shipped Balanced body, and
+      // the disabled shape asserted below only exists for models whose
+      // reasoning can be turned off: the normalizer drops a disabled thinking
+      // config for adaptive-thinking-only models (they always reason and 4xx
+      // the opt-out), so this test would assert the Balanced pin's capabilities
+      // rather than call-site precedence.
+      profiles: {
+        "test-anthropic": {
+          source: "user",
+          provider: "anthropic",
+          model: "claude-sonnet-5",
+        },
+      },
+      profileOrder: ["test-anthropic"],
+      activeProfile: "test-anthropic",
       default: {
         provider: "anthropic",
         model: "claude-default",

package/src/__tests__/managed-profile-guard.test.ts

@@ -1118,7 +1118,7 @@ describe("code-owned default profiles — wire view and write normalization", ()
     });
     const response = (await getRoute.handler({})) as Record<string, any>;
     const wireBalanced = response.llm.profiles.balanced;
-    expect(wireBalanced.model).toBe("accounts/fireworks/models/glm-5p2");
+    expect(wireBalanced.model).toBe("accounts/fireworks/models/glm-5p3-flash");
     expect(wireBalanced.provider).toBe("vellum");
     expect(wireBalanced.provider_connection).toBeUndefined();
     expect(wireBalanced.status).toBe("disabled");
@@ -1366,7 +1366,7 @@ describe("code-owned default profiles — echoes over stale on-disk bodies", ()
     const response = (await getRoute.handler({})) as Record<string, any>;
     // The wire view serves catalog content, not the stale body.
     expect(response.llm.profiles.balanced.model).toBe(
-      "accounts/fireworks/models/glm-5p2",
+      "accounts/fireworks/models/glm-5p3-flash",
     );
     const result = await patchRoute.handler({
       body: { llm: { profiles: response.llm.profiles } },

package/src/config/__tests__/balanced-model-experiment.test.ts

@@ -24,15 +24,16 @@ import {
   type ProfileEntry,
 } from "../schemas/llm.js";
 
-// The `experiment-balanced-model-2026-08-06` arm repoints the model the managed
+// The `experiment-balanced-model-2026-08-31` arm repoints the model the managed
 // Balanced profile resolves to. Both the runtime resolver and the client-facing
 // profile listing go through `defaultProfileBodyForProvider`, so both are
 // exercised here: a disagreement between them shows the user one model beside a
 // profile that runs another.
 
 const FLAG = BALANCED_MODEL_EXPERIMENT_FLAG_KEY;
-const SHIPPED_MODEL = "accounts/fireworks/models/glm-5p2";
-const GLM_MODEL = SHIPPED_MODEL;
+const SHIPPED_MODEL = "accounts/fireworks/models/glm-5p3-flash";
+const GLM_53_MODEL = "accounts/fireworks/models/glm-5p3";
+const GLM_52_MODEL = "accounts/fireworks/models/glm-5p2";
 
 function setArm(value: boolean | string): void {
   setOverridesForTesting({ [FLAG]: value });
@@ -57,22 +58,22 @@ afterEach(() => {
 });
 
 describe("balanced-model experiment arms", () => {
-  test("terra repoints mainAgent at gpt-5.6-terra on the managed connection", () => {
-    setArm("terra");
+  test("glm-5p3 repoints mainAgent at GLM 5.3 on the managed connection", () => {
+    setArm("glm-5p3");
     const resolved = resolveCallSiteConfig(
       "mainAgent",
       llmWithActiveBalanced(),
     );
-    expect(resolved.model).toBe("gpt-5.6-terra");
+    expect(resolved.model).toBe(GLM_53_MODEL);
     expect(resolved.provider).toBe("vellum");
-    expect(getManagedUpstream("gpt-5.6-terra")).toBe("openai");
+    expect(getManagedUpstream(GLM_53_MODEL)).toBe("fireworks");
 
     const entry = resolveDefaultProfileForProvider(
       undefined,
       "balanced",
       managed,
     );
-    expect(entry?.model).toBe("gpt-5.6-terra");
+    expect(entry?.model).toBe(GLM_53_MODEL);
     expect(entry?.provider).toBe("vellum");
     // Routing-identity providers derive their connection per request.
     expect(entry?.provider_connection).toBeUndefined();
@@ -85,12 +86,12 @@ describe("balanced-model experiment arms", () => {
       "mainAgent",
       llmWithActiveBalanced(),
     );
-    expect(resolved.model).toBe(GLM_MODEL);
+    expect(resolved.model).toBe(GLM_52_MODEL);
     expect(resolved.provider).toBe("vellum");
 
-    const upstream = getManagedUpstream(GLM_MODEL);
+    const upstream = getManagedUpstream(GLM_52_MODEL);
     expect(upstream).toBe("fireworks");
-    const cap = catalogMaxOutputTokens(upstream as string, GLM_MODEL);
+    const cap = catalogMaxOutputTokens(upstream as string, GLM_52_MODEL);
     expect(cap).toBeDefined();
     expect(resolved.maxTokens).toBeLessThanOrEqual(cap as number);
     // The shipped token budget stands.
@@ -99,19 +100,26 @@ describe("balanced-model experiment arms", () => {
     );
   });
 
+  test("the glm-5p3-flash arm names the shipped pin", () => {
+    setArm("glm-5p3-flash");
+    expect(
+      resolveDefaultProfileForProvider(undefined, "balanced", managed)?.model,
+    ).toBe(SHIPPED_MODEL);
+  });
+
   test("an arm changes only the model of the managed balanced body", () => {
-    setArm("terra");
+    setArm("glm-5p2");
     const entry = resolveDefaultProfileForProvider(
       undefined,
       "balanced",
       managed,
     );
     const shipped = CODE_DEFAULT_PROFILE_ENTRIES.balanced;
-    expect(entry).toEqual({ ...shipped, model: "gpt-5.6-terra" });
+    expect(entry).toEqual({ ...shipped, model: GLM_52_MODEL });
   });
 
   test("the other default profiles are untouched by an arm", () => {
-    setArm("terra");
+    setArm("glm-5p2");
     for (const key of [
       "quality-optimized",
       "cost-optimized",
@@ -124,7 +132,7 @@ describe("balanced-model experiment arms", () => {
   });
 
   test("every pinned arm model is managed-routable and in its upstream catalog", () => {
-    for (const model of ["gpt-5.6-terra", GLM_MODEL]) {
+    for (const model of [SHIPPED_MODEL, GLM_53_MODEL, GLM_52_MODEL]) {
       const upstream = getManagedUpstream(model);
       expect(upstream).not.toBeNull();
       expect(isModelInCatalog(upstream as string, model)).toBe(true);
@@ -136,7 +144,10 @@ describe("balanced-model experiment fallbacks", () => {
   const shippedCases: [string, () => void][] = [
     ["the flag is unset", () => clearCachedOverrides()],
     ["no override is present", () => setOverridesForTesting({})],
+    // Arms of the retired 2026-08-06 experiment are no longer pinned, so a
+    // stale value cannot strand an install on a model this test no longer runs.
     ["the arm is control", () => setArm("control")],
+    ["the arm is terra", () => setArm("terra")],
     ["the arm is an unknown string", () => setArm("gpt-9-does-not-exist")],
     ["the arm is the empty string", () => setArm("")],
     ["the value is boolean true", () => setArm(true)],
@@ -162,11 +173,11 @@ describe("balanced-model experiment fallbacks", () => {
     });
   }
 
-  test("the shipped catalog body is the control arm", () => {
+  test("the shipped catalog body is the glm-5p3-flash arm", () => {
     expect(CODE_DEFAULT_PROFILE_ENTRIES.balanced.model).toBe(SHIPPED_MODEL);
   });
 
-  test("the registry default is control, which the override-only flag read assumes", () => {
+  test("the registry default names the shipped pin, which the override-only flag read assumes", () => {
     const registryPath = join(
       import.meta.dirname,
       "..",
@@ -188,14 +199,35 @@ describe("balanced-model experiment fallbacks", () => {
     const flag = registry.flags.find((entry) => entry.key === FLAG);
     expect(flag).toBeDefined();
     expect(flag?.scope).toBe("assistant");
-    expect(flag?.defaultEnabled).toBe("control");
-    expect(flag?.values).toEqual(["control", "terra", "glm-5p2"]);
+    expect(flag?.defaultEnabled).toBe("glm-5p3-flash");
+    expect(flag?.values).toEqual(["glm-5p3-flash", "glm-5p3", "glm-5p2"]);
+  });
+
+  test("the retired experiment's flag is gone from the registry", () => {
+    const registryPath = join(
+      import.meta.dirname,
+      "..",
+      "..",
+      "..",
+      "..",
+      "meta",
+      "feature-flags",
+      "feature-flag-registry.json",
+    );
+    const registry = JSON.parse(readFileSync(registryPath, "utf-8")) as {
+      flags: { key: string }[];
+    };
+    expect(
+      registry.flags.some(
+        (entry) => entry.key === "experiment-balanced-model-2026-08-06",
+      ),
+    ).toBe(false);
   });
 });
 
 describe("balanced-model experiment boundaries", () => {
   test("a user-owned balanced shadow wins over the arm", () => {
-    setArm("terra");
+    setArm("glm-5p2");
     const profiles: Record<string, ProfileEntry> = {
       balanced: { source: "user", provider: "openai", model: "gpt-5.5" },
     };
@@ -209,7 +241,7 @@ describe("balanced-model experiment boundaries", () => {
   });
 
   test("a managed-source stub still resolves to the arm's body", () => {
-    setArm("terra");
+    setArm("glm-5p2");
     const profiles: Record<string, ProfileEntry> = {
       balanced: { source: "managed", label: "My Balanced" },
     };
@@ -218,12 +250,12 @@ describe("balanced-model experiment boundaries", () => {
       "balanced",
       managed,
     );
-    expect(entry?.model).toBe("gpt-5.6-terra");
+    expect(entry?.model).toBe(GLM_52_MODEL);
     expect(entry?.label).toBe("My Balanced");
   });
 
   test("the chatgpt and BYOK columns stay out of the experiment", () => {
-    setArm("terra");
+    setArm("glm-5p2");
     for (const provider of ["anthropic", "openai", "chatgpt"] as const) {
       const armed = resolveDefaultProfileForProvider(undefined, "balanced", {
         provider,
@@ -234,37 +266,37 @@ describe("balanced-model experiment boundaries", () => {
       });
       expect(armed).toEqual(shipped as ProfileEntry);
       expect(armed?.provider).toBe(provider);
-      setArm("terra");
+      setArm("glm-5p2");
     }
   });
 
   test("an install with no defaultProvider still picks up the arm", () => {
-    setArm("terra");
+    setArm("glm-5p2");
     expect(
       resolveDefaultProfileForProvider(undefined, "balanced", null)?.model,
-    ).toBe("gpt-5.6-terra");
+    ).toBe(GLM_52_MODEL);
   });
 });
 
 describe("client-facing profile listing", () => {
   test("reports the arm's model so the UI matches what runs", () => {
-    setArm("terra");
+    setArm("glm-5p2");
     const profiles = getEffectiveProfilesForProvider(undefined, managed);
-    expect(profiles.balanced?.model).toBe("gpt-5.6-terra");
+    expect(profiles.balanced?.model).toBe(GLM_52_MODEL);
     expect(profiles["quality-optimized"]?.model).toBe(
       CODE_DEFAULT_PROFILE_ENTRIES["quality-optimized"].model,
     );
   });
 
-  test("reports the shipped model on the control arm", () => {
-    setArm("control");
+  test("reports the shipped model on an arm this build does not know", () => {
+    setArm("terra");
     expect(
       getEffectiveProfilesForProvider(undefined, managed).balanced?.model,
     ).toBe(SHIPPED_MODEL);
   });
 
   test("agrees with the runtime resolver on every arm", () => {
-    for (const arm of ["control", "terra", "glm-5p2", "nonsense"]) {
+    for (const arm of ["glm-5p3-flash", "glm-5p3", "glm-5p2", "nonsense"]) {
       setArm(arm);
       const listed = getEffectiveProfilesForProvider(undefined, managed)
         .balanced?.model;

package/src/config/__tests__/default-profile-catalog.test.ts

@@ -59,9 +59,9 @@ describe("getEffectiveProfiles", () => {
     }
   });
 
-  test("the managed Balanced profile routes GLM 5.2 through Fireworks", () => {
+  test("the managed Balanced profile routes GLM 5.3 Flash through Fireworks", () => {
     const balanced = CODE_DEFAULT_PROFILE_ENTRIES.balanced;
-    expect(balanced.model).toBe("accounts/fireworks/models/glm-5p2");
+    expect(balanced.model).toBe("accounts/fireworks/models/glm-5p3-flash");
     expect(resolveRoutingIdentity(balanced.provider, balanced.model)).toEqual({
       connectionName: "vellum",
       expectedProvider: "fireworks",

package/src/config/balanced-model-experiment.ts

@@ -1,33 +1,39 @@
 /**
  * Flag read for the managed Balanced profile's model A/B test.
  *
- * `experiment-balanced-model-2026-08-06` is a multivariate LaunchDarkly flag
+ * `experiment-balanced-model-2026-08-31` is a multivariate LaunchDarkly flag
  * whose arm repoints the model the managed (`vellum`) implementation of the
  * `balanced` default profile resolves to. The arm to model pins live beside
  * the profile bodies in `default-profile-catalog.ts`, which validates their
  * managed routability at load time; this module owns only the flag read.
  *
+ * A flag key per experiment, rather than new variations on the previous one,
+ * is what keeps a readout honest: the activation cohort query keys a user on
+ * their first-ever assignment for a flag, so every user the 2026-08-06 test
+ * already stamped would stay pinned to that arm forever and never enter a new
+ * arm on the same key.
+ *
  * The read goes straight to the override cache rather than through
  * `assistant-feature-flags.ts`. `feature-flag-cache.ts` is a stdlib-only leaf,
  * so the profile catalog stays free of the pino logger and the gateway IPC
  * client that resolver would pull into its import graph, and the registry
- * declares `defaultEnabled: "control"` for this flag, which is the same
- * shipped body an absent override already resolves to. That equivalence is
- * pinned by `__tests__/balanced-model-experiment.test.ts` so it cannot drift.
+ * declares `defaultEnabled: "glm-5p3-flash"` for this flag, which pins the same
+ * model the shipped body an absent override resolves to already carries. That
+ * equivalence is pinned by `__tests__/balanced-model-experiment.test.ts` so it
+ * cannot drift.
  */
 
 import { getCachedOverrides } from "./feature-flag-cache.js";
 
 export const BALANCED_MODEL_EXPERIMENT_FLAG_KEY =
-  "experiment-balanced-model-2026-08-06";
+  "experiment-balanced-model-2026-08-31";
 
 /**
  * The experiment arm in force, or `undefined` when the flag resolves to
  * anything that is not an arm name: unset, a boolean, or the empty string.
- * `control` and any arm this build does not know are returned as-is and miss
- * the pin table, which is what keeps a stale or malformed LaunchDarkly value
- * on the shipped model rather than stranding an install on one that does not
- * exist.
+ * Any arm this build does not know is returned as-is and misses the pin table,
+ * which is what keeps a stale or malformed LaunchDarkly value on the shipped
+ * model rather than stranding an install on one that does not exist.
  */
 export function getBalancedModelExperimentArm(): string | undefined {
   const value = getCachedOverrides()?.[BALANCED_MODEL_EXPERIMENT_FLAG_KEY];

package/src/config/default-profile-catalog.ts

@@ -86,7 +86,7 @@ type ProfileImpls = Record<DefaultProfileKey, DefaultProfileTemplate>;
  */
 const VELLUM_PROFILE_IMPLS: ProfileImpls = {
   balanced: {
-    model: "accounts/fireworks/models/glm-5p2",
+    model: "accounts/fireworks/models/glm-5p3-flash",
     provider: "vellum",
     source: "managed",
     label: "Balanced",
@@ -241,25 +241,26 @@ const BACKUP_PROFILE_IMPLS: Record<BackupProfileKey, DefaultProfileTemplate> = {
 };
 
 /**
- * Arm to managed model pin for the `experiment-balanced-model-2026-08-06` A/B
+ * Arm to managed model pin for the `experiment-balanced-model-2026-08-31` A/B
  * test (`balanced-model-experiment.ts` owns the flag read). An arm repoints
  * the model of the managed (`vellum`) implementation of `balanced` and nothing
  * else: effort, thinking, token budget, label and description all stay on the
  * shipped body, and the `chatgpt` and BYOK columns are untouched because those
  * installs run the provider their user chose and sit outside the experiment.
  *
- * `control` is absent by design. It, an arm this build does not know, and an
- * unset flag all resolve to the shipped body, so no LaunchDarkly value can
- * strand an install on a model that is not pinned here. A `Map` rather than an
- * object literal keeps that true for every string LaunchDarkly can send: the
- * arm is remote input, and an object lookup would resolve `constructor` or
- * `toString` to an inherited `Object.prototype` member instead of missing.
+ * An arm this build does not know, and an unset flag, both resolve to the
+ * shipped body, so no LaunchDarkly value can strand an install on a model that
+ * is not pinned here. A `Map` rather than an object literal keeps that true for
+ * every string LaunchDarkly can send: the arm is remote input, and an object
+ * lookup would resolve `constructor` or `toString` to an inherited
+ * `Object.prototype` member instead of missing.
  *
- * `glm-5p2` names the same model as the shipped pin and stays in the table so
- * the arm keeps its meaning if the shipped pin moves again.
+ * `glm-5p3-flash` names the same model as the shipped pin and stays in the
+ * table so the arm keeps its meaning if the shipped pin moves again.
  */
 const BALANCED_EXPERIMENT_MODELS = new Map<string, string>([
-  ["terra", "gpt-5.6-terra"],
+  ["glm-5p3-flash", "accounts/fireworks/models/glm-5p3-flash"],
+  ["glm-5p3", "accounts/fireworks/models/glm-5p3"],
   ["glm-5p2", "accounts/fireworks/models/glm-5p2"],
 ]);
 

package/src/config/feature-flag-registry.json

@@ -382,13 +382,13 @@
       "defaultEnabled": true
     },
     {
-      "id": "experiment-balanced-model-2026-08-06",
+      "id": "experiment-balanced-model-2026-08-31",
       "scope": "assistant",
-      "key": "experiment-balanced-model-2026-08-06",
+      "key": "experiment-balanced-model-2026-08-31",
       "label": "Experiment: Balanced Profile Model",
-      "description": "Multivariate experiment on the model the managed Balanced inference profile resolves to. control = the shipped pin (accounts/fireworks/models/glm-5p2); terra = gpt-5.6-terra; glm-5p2 = accounts/fireworks/models/glm-5p2. Only the managed (vellum) implementation moves: ChatGPT-subscription and BYOK installs run the provider the user chose and are outside the experiment. Rollout and targeting are managed in the LaunchDarkly dashboard; any value that is not an arm name resolves to control.",
-      "defaultEnabled": "control",
-      "values": ["control", "terra", "glm-5p2"]
+      "description": "Multivariate experiment on the model the managed Balanced inference profile resolves to. glm-5p3-flash = the shipped pin (accounts/fireworks/models/glm-5p3-flash); glm-5p3 = accounts/fireworks/models/glm-5p3; glm-5p2 = accounts/fireworks/models/glm-5p2. Replaces experiment-balanced-model-2026-08-06, which ended on 2026-08-31: the activation cohort keys a user on their first-ever assignment per flag, so a new arm on the old key could never enrol a user the old test already stamped. Only the managed (vellum) implementation moves: ChatGPT-subscription and BYOK installs run the provider the user chose and are outside the experiment. Rollout and targeting are managed in the LaunchDarkly dashboard; any value that is not an arm name resolves to the shipped pin.",
+      "defaultEnabled": "glm-5p3-flash",
+      "values": ["glm-5p3-flash", "glm-5p3", "glm-5p2"]
     },
     {
       "id": "ios-avatar-app-icon",