@vellumai/assistant 0.11.6 → 0.11.7-staging.2

package/src/__tests__/compactor-call-site-logging.test.ts

@@ -194,3 +194,64 @@ describe("compactor records llm_request_logs with call_site=compactionAgent", ()
     expect(recordRequestLogCalls[0]!.callSite).toBe("compactionAgent");
   });
 });
+
+describe("compaction result reports what actually served the summary", () => {
+  beforeEach(() => {
+    recordRequestLogCalls.length = 0;
+  });
+
+  test("rerouted summary carries the backup provider and profile", async () => {
+    // `RetryProvider` escalated the summary call to a backup profile on
+    // another provider and stamped the response, so the result must report
+    // the backup even though the compactor resolved `primaryProfile`.
+    const provider: Provider = {
+      name: "openai",
+      sendMessage: async () => ({
+        content: [{ type: "text", text: compactionResponse }],
+        model: "claude-sonnet-5",
+        actualProvider: "anthropic",
+        actualInferenceProfile: "backupProfile",
+        usage: { inputTokens: 100, outputTokens: 50 },
+        stopReason: "end_turn",
+        rawRequest: RAW_REQUEST,
+        rawResponse: RAW_RESPONSE,
+      }),
+    };
+
+    const result = await runAssistantDrivenCompaction({
+      ...args(provider),
+      overrideProfile: "primaryProfile",
+    });
+
+    expect(result.summaryModel).toBe("claude-sonnet-5");
+    expect(result.summaryActualProvider).toBe("anthropic");
+    expect(result.summaryActualInferenceProfile).toBe("backupProfile");
+    // The primary's resolution is still reported, just outranked downstream.
+    expect(result.summaryOverrideProfile).toBe("primaryProfile");
+    // Same value the request log recorded: one source of truth, not two.
+    expect(recordRequestLogCalls[0]!.provider).toBe("anthropic");
+  });
+
+  test("non-rerouted summary carries the configured provider and no profile", async () => {
+    const provider: Provider = {
+      name: "openai",
+      sendMessage: async () => ({
+        content: [{ type: "text", text: compactionResponse }],
+        model: "mock-model",
+        usage: { inputTokens: 100, outputTokens: 50 },
+        stopReason: "end_turn",
+        rawRequest: RAW_REQUEST,
+        rawResponse: RAW_RESPONSE,
+      }),
+    };
+
+    const result = await runAssistantDrivenCompaction({
+      ...args(provider),
+      overrideProfile: "primaryProfile",
+    });
+
+    expect(result.summaryActualProvider).toBe("openai");
+    expect(result.summaryActualInferenceProfile).toBeUndefined();
+    expect(result.summaryOverrideProfile).toBe("primaryProfile");
+  });
+});

package/src/__tests__/config-loader-backfill.test.ts

@@ -1411,6 +1411,71 @@ describe("loadConfig startup behavior", () => {
     expect(effective.balanced?.provider_connection).toBe("anthropic-personal");
   });
 
+  test("an active managed backup profile survives a reload", () => {
+    // A managed backup resolves from the code catalog without being
+    // materialized in `llm.profiles`, so selecting one persists nothing but
+    // the reference. If the schema did not treat backup keys as
+    // always-available references, the strip-and-reparse in `loadConfig`
+    // would drop `activeProfile` and the seeder would reset the user's
+    // chat-model selection to `balanced` on the next boot.
+    writeConfig({
+      llm: {
+        defaultProvider: { provider: "vellum" },
+        profiles: {},
+        activeProfile: "balanced-backup",
+      },
+    });
+
+    const config = loadConfig();
+    expect(config.llm.activeProfile).toBe("balanced-backup");
+
+    mergeDefaultConfigAndSeedInferenceProfiles();
+
+    const raw = JSON.parse(readFileSync(CONFIG_PATH, "utf-8"));
+    expect(raw.llm.activeProfile).toBe("balanced-backup");
+    expect(raw.llm.profiles["balanced-backup"]).toBeUndefined();
+  });
+
+  test("an active managed backup profile is repaired on a BYOK install", () => {
+    // Backups live on the managed column alone, so under a BYOK default
+    // provider the selection names a profile that resolves to nothing. The
+    // schema rejects the reference on load and the seeder, which resolves
+    // availability through the same provider-aware view, repairs the stored
+    // selection instead of preserving a dead pointer.
+    writeConfig({
+      llm: {
+        defaultProvider: { provider: "anthropic" },
+        profiles: {},
+        activeProfile: "balanced-backup",
+      },
+    });
+
+    const config = loadConfig();
+    expect(config.llm.activeProfile).toBeUndefined();
+
+    mergeDefaultConfigAndSeedInferenceProfiles();
+
+    const raw = JSON.parse(readFileSync(CONFIG_PATH, "utf-8"));
+    expect(raw.llm.activeProfile).toBe("balanced");
+  });
+
+  test("a call site pinned to a managed backup profile survives a reload", () => {
+    writeConfig({
+      llm: {
+        defaultProvider: { provider: "vellum" },
+        profiles: {},
+        activeProfile: "balanced",
+        callSites: { recall: { profile: "cost-optimized-backup" } },
+      },
+    });
+
+    const config = loadConfig();
+    expect(config.llm.callSites.recall?.profile).toBe("cost-optimized-backup");
+
+    const raw = JSON.parse(readFileSync(CONFIG_PATH, "utf-8"));
+    expect(raw.llm.callSites.recall.profile).toBe("cost-optimized-backup");
+  });
+
   test("still quarantines corrupt JSON", () => {
     // Corrupt-config quarantine is a recovery path: the broken file is
     // renamed to `config.json.corrupt-<ts>.json` and the daemon proceeds
@@ -1800,6 +1865,10 @@ describe("seedInferenceProfiles BYOK-mode default profiles", () => {
       "quality-optimized",
       "cost-optimized",
       "latency-optimized",
+      "balanced-backup",
+      "quality-optimized-backup",
+      "cost-optimized-backup",
+      "latency-optimized-backup",
       "my-custom",
     ]);
   });

package/src/__tests__/config-schema.test.ts

@@ -959,8 +959,7 @@ describe("AssistantConfigSchema", () => {
         },
       },
       flux: {
-        turnEnd: { enabled: false },
-        model: "flux-general-en",
+        turnEnd: { enabled: true },
         eotThreshold: 0.7,
         eotTimeoutMs: 5000,
       },
@@ -989,7 +988,7 @@ describe("AssistantConfigSchema", () => {
     expect(result.liveVoice.vad.maxTurnDurationMs).toBe(30000);
     expect(result.liveVoice.maxSessionDurationSeconds).toBe(900);
     // A partial liveVoice override leaves Flux turn-end disabled by default.
-    expect(result.liveVoice.flux.turnEnd.enabled).toBe(false);
+    expect(result.liveVoice.flux.turnEnd.enabled).toBe(true);
   });
 
   test("accepts a liveVoice.vad.bargeInMinSpeechMs of 0 (guard disabled)", () => {

package/src/__tests__/conversation-agent-loop.test.ts

@@ -23,6 +23,10 @@ import { resetPluginRegistryAndRegisterDefaults } from "../plugins/defaults/inde
 import { registerPlugin } from "../plugins/registry.js";
 import type { Message, Provider, ToolDefinition } from "../providers/types.js";
 import { ContextOverflowError } from "../providers/types.js";
+import {
+  resolveUsageAttribution,
+  type UsageAttributionInput,
+} from "../usage/attribution.js";
 import { getWorkspaceDir } from "../util/platform.js";
 import { setConfig } from "./helpers/set-config.js";
 
@@ -1763,6 +1767,108 @@ describe("session-agent-loop", () => {
       expect(mainAgentCall?.[3]).toBe("gpt-4.1-2026-03-01");
     });
 
+    test("attributes a fallback serve to the profile that actually served", async () => {
+      // The turn resolved to the primary managed profile, the request hit an
+      // outage-shaped failure, and `RetryProvider` escalated to a backup
+      // profile on a different provider, stamping `actualProvider` and
+      // `actualInferenceProfile` on the response it returns.
+      seedLlmConfig({
+        profiles: {
+          ...disabledCatalogDefaultProfiles,
+          backupProfile: { provider: "anthropic", model: "claude-sonnet-5" },
+        },
+      });
+
+      const ctx = makeCtx({
+        providerResponses: [
+          {
+            content: [{ type: "text", text: "Hi there." }],
+            model: "claude-sonnet-5",
+            usage: { inputTokens: 12, outputTokens: 3 },
+            stopReason: "end_turn",
+            actualProvider: "anthropic",
+            actualInferenceProfile: "backupProfile",
+          },
+        ],
+        provider: {
+          name: "openai",
+          sendMessage: async () => ({
+            content: [{ type: "text", text: "title" }],
+            model: "mock",
+            usage: { inputTokens: 0, outputTokens: 0 },
+            stopReason: "end_turn",
+          }),
+        } as unknown as Conversation["provider"],
+      });
+
+      await runAgentLoopImpl(ctx, "hello", "msg-1", () => {});
+
+      const mainAgentCall = recordUsageMock.mock.calls.find(
+        (call) => (call as unknown[])[5] === "main_agent",
+      ) as unknown[] | undefined;
+
+      expect(mainAgentCall).toBeDefined();
+      // All three attribution facets of the row must describe the same call.
+      expect(mainAgentCall?.[0]).toMatchObject({ providerName: "anthropic" });
+      expect(mainAgentCall?.[3]).toBe("claude-sonnet-5");
+      const attribution = mainAgentCall?.[12] as UsageAttributionInput;
+      expect(attribution.callSite).toBe("mainAgent");
+      expect(attribution.overrideProfile).toBe("backupProfile");
+      expect(attribution.forceOverrideProfile).toBe(true);
+      // `inference_profile` on the persisted row is this snapshot's applied
+      // profile, so resolve it the way `recordUsage` does and assert the
+      // column value itself rather than only the input that feeds it.
+      expect(resolveUsageAttribution(attribution).appliedProfile).toBe(
+        "backupProfile",
+      );
+    });
+
+    test("leaves a non-fallback turn attributed to the conversation's own profile", async () => {
+      // No reroute: the response carries no `actualInferenceProfile`, so the
+      // attribution input stays exactly what the conversation resolved and
+      // carries no forced override.
+      seedLlmConfig({
+        profiles: {
+          ...disabledCatalogDefaultProfiles,
+          backupProfile: { provider: "anthropic", model: "claude-sonnet-5" },
+        },
+      });
+
+      const ctx = makeCtx({
+        providerResponses: [
+          {
+            content: [{ type: "text", text: "Hi there." }],
+            model: "gpt-4.1-2026-03-01",
+            usage: { inputTokens: 12, outputTokens: 3 },
+            stopReason: "end_turn",
+          },
+        ],
+        provider: {
+          name: "openai",
+          sendMessage: async () => ({
+            content: [{ type: "text", text: "title" }],
+            model: "mock",
+            usage: { inputTokens: 0, outputTokens: 0 },
+            stopReason: "end_turn",
+          }),
+        } as unknown as Conversation["provider"],
+      });
+
+      await runAgentLoopImpl(ctx, "hello", "msg-1", () => {});
+
+      const mainAgentCall = recordUsageMock.mock.calls.find(
+        (call) => (call as unknown[])[5] === "main_agent",
+      ) as unknown[] | undefined;
+
+      expect(mainAgentCall).toBeDefined();
+      expect(mainAgentCall?.[0]).toMatchObject({ providerName: "openai" });
+      expect(mainAgentCall?.[3]).toBe("gpt-4.1-2026-03-01");
+      expect(mainAgentCall?.[12]).toEqual({
+        callSite: "mainAgent",
+        overrideProfile: null,
+      });
+    });
+
     test("persists the served model onto the assistant row's metadata at finalize", async () => {
       const events: AssistantEvent[] = [];
 
@@ -4037,6 +4143,138 @@ describe("session-agent-loop", () => {
       );
     });
 
+    test("rerouted compaction records usage under the profile that actually served", async () => {
+      const ctx = makeCtx();
+
+      await applyCompactionResult(
+        ctx,
+        {
+          messages: [
+            { role: "user", content: [{ type: "text", text: "summary" }] },
+          ],
+          compactedPersistedMessages: 4,
+          previousEstimatedInputTokens: 12000,
+          estimatedInputTokens: 3000,
+          maxInputTokens: 100000,
+          thresholdTokens: 80000,
+          compactedMessages: 4,
+          summaryCalls: 1,
+          summaryInputTokens: 100,
+          summaryOutputTokens: 20,
+          summaryModel: "claude-sonnet-5",
+          summaryText: "summary",
+          summaryCallSite: "compactionAgent",
+          // What the compactor resolved before the call...
+          summaryOverrideProfile: "primaryProfile",
+          // ...and what the reroute actually served.
+          summaryActualProvider: "anthropic",
+          summaryActualInferenceProfile: "backupProfile",
+        },
+        () => {},
+        "req-1",
+      );
+
+      const compactorCall = recordUsageMock.mock.calls.find(
+        (call) => (call as unknown[])[5] === "context_compactor",
+      ) as unknown[] | undefined;
+
+      expect(compactorCall).toBeDefined();
+      // All three attribution facets of the row describe the same call.
+      expect(compactorCall?.[0]).toMatchObject({ providerName: "anthropic" });
+      expect(compactorCall?.[3]).toBe("claude-sonnet-5");
+      expect(compactorCall?.[12]).toEqual({
+        callSite: "compactionAgent",
+        overrideProfile: "backupProfile",
+        forceOverrideProfile: true,
+      });
+    });
+
+    test("non-rerouted compaction keeps the compactor's own attribution", async () => {
+      const ctx = makeCtx();
+
+      await applyCompactionResult(
+        ctx,
+        {
+          messages: [
+            { role: "user", content: [{ type: "text", text: "summary" }] },
+          ],
+          compactedPersistedMessages: 4,
+          previousEstimatedInputTokens: 12000,
+          estimatedInputTokens: 3000,
+          maxInputTokens: 100000,
+          thresholdTokens: 80000,
+          compactedMessages: 4,
+          summaryCalls: 1,
+          summaryInputTokens: 100,
+          summaryOutputTokens: 20,
+          summaryModel: "mock-model",
+          summaryText: "summary",
+          summaryCallSite: "compactionAgent",
+          summaryOverrideProfile: "primaryProfile",
+          summaryActualProvider: "openai",
+        },
+        () => {},
+        "req-1",
+      );
+
+      const compactorCall = recordUsageMock.mock.calls.find(
+        (call) => (call as unknown[])[5] === "context_compactor",
+      ) as unknown[] | undefined;
+
+      expect(compactorCall).toBeDefined();
+      expect(compactorCall?.[0]).toMatchObject({ providerName: "openai" });
+      expect(compactorCall?.[3]).toBe("mock-model");
+      expect(compactorCall?.[12]).toEqual({
+        callSite: "compactionAgent",
+        overrideProfile: "primaryProfile",
+      });
+    });
+
+    test("compaction that served no call falls back to the conversation's provider", async () => {
+      // The degraded shape: `emptyResult` paths record no served attribution
+      // because no call happened. Provider must fall back to the
+      // conversation's own, and the attribution input must be byte-identical
+      // to what it was before served attribution existed.
+      const ctx = makeCtx();
+
+      await applyCompactionResult(
+        ctx,
+        {
+          messages: [
+            { role: "user", content: [{ type: "text", text: "summary" }] },
+          ],
+          compactedPersistedMessages: 4,
+          previousEstimatedInputTokens: 12000,
+          estimatedInputTokens: 3000,
+          maxInputTokens: 100000,
+          thresholdTokens: 80000,
+          compactedMessages: 4,
+          summaryCalls: 1,
+          summaryInputTokens: 100,
+          summaryOutputTokens: 20,
+          summaryModel: "mock-model",
+          summaryText: "summary",
+          summaryCallSite: "compactionAgent",
+          summaryOverrideProfile: "primaryProfile",
+        },
+        () => {},
+        "req-1",
+      );
+
+      const compactorCall = recordUsageMock.mock.calls.find(
+        (call) => (call as unknown[])[5] === "context_compactor",
+      ) as unknown[] | undefined;
+
+      expect(compactorCall).toBeDefined();
+      expect(compactorCall?.[0]).toMatchObject({
+        providerName: "mock-provider",
+      });
+      expect(compactorCall?.[12]).toEqual({
+        callSite: "compactionAgent",
+        overrideProfile: "primaryProfile",
+      });
+    });
+
     test("applyCompactionResult advances the persisted count from the trusted in-context boundary", async () => {
       // Trusted views slice past the already-compacted prefix, so a further
       // compaction advances the persisted count from the mirrored DB boundary.

package/src/__tests__/conversation-attachments.test.ts

@@ -28,7 +28,6 @@ mock.module("../permissions/types.js", () => ({
     Medium: "medium",
     High: "high",
   },
-  isAllowDecision: () => true,
 }));
 
 import type { AssistantAttachmentDraft } from "../daemon/assistant-attachments.js";

package/src/__tests__/conversation-error.test.ts

@@ -1136,6 +1136,50 @@ describe("classifyConversationError", () => {
   });
 
   describe("reason-driven classification (ProviderError.reason)", () => {
+    it("classifies reason=request_shape_unsupported as a friendly capability-mismatch message", () => {
+      // GIVEN a chat-template 400 that the provider stamped with the
+      // semantic reason (Together serving MiniMax M3)
+      const err = new ProviderError(
+        "Together AI API error (400): Failed to apply chat template: invalid operation: object is not callable (in chat:22)",
+        "together",
+        400,
+        { reason: "request_shape_unsupported" },
+      );
+
+      // WHEN it is classified
+      const result = classifyConversationError(err, baseCtx);
+
+      // THEN the user sees the capability-mismatch copy, not the raw
+      // template error
+      expect(result.code).toBe("PROVIDER_API");
+      expect(result.errorCategory).toBe("request_shape_unsupported");
+      expect(result.retryable).toBe(false);
+      expect(result.userMessage).toContain(
+        "couldn't process the request format",
+      );
+      expect(result.userMessage).not.toContain("chat template");
+    });
+
+    it("classifies a reason-less chat-template 400 via the message ladder", () => {
+      // GIVEN a chat-template 400 whose ProviderError carries no semantic
+      // reason (e.g. thrown by a non-OpenAI-compatible wrapper)
+      const err = new ProviderError(
+        "Together AI API error (400): Failed to apply chat template: invalid operation: object is not callable (in chat:22)",
+        "together",
+        400,
+      );
+
+      // WHEN it is classified
+      const result = classifyConversationError(err, baseCtx);
+
+      // THEN the regex ladder still lands on the capability-mismatch copy
+      expect(result.code).toBe("PROVIDER_API");
+      expect(result.errorCategory).toBe("request_shape_unsupported");
+      expect(result.userMessage).toContain(
+        "couldn't process the request format",
+      );
+    });
+
     it("classifies reason=model_restricted on the skew-safe PROVIDER_API code with a specific errorCategory", () => {
       const err = new ProviderError(
         "Vercel AI Gateway API error (403): Model claude-opus-4 is restricted on your plan [type=no_providers_available]",

package/src/__tests__/conversation-evictor.test.ts

@@ -9,6 +9,13 @@ mock.module("../subagent/index.js", () => ({
   getSubagentManager: () => ({
     abortAllForParent: (id: string) => abortedParents.push(id),
     getChildrenOf: () => protectedChildren,
+    hasActiveChildren: () =>
+      protectedChildren.some(
+        (c) =>
+          c.status === "running" ||
+          c.status === "pending" ||
+          c.status === "awaiting_input",
+      ),
   }),
 }));
 
@@ -236,6 +243,18 @@ describe("ConversationEvictor", () => {
       expect(sessions.has("a")).toBe(true);
       expect(s1.disposed).toBe(false);
     });
+
+    test("skips conversations with awaiting_input subagents", () => {
+      protectedChildren = [{ status: "awaiting_input" }];
+      const s1 = createMockSession();
+      sessions.set("a", s1);
+
+      const result = evictor.sweep();
+
+      expect(result.skipped).toBe(1);
+      expect(sessions.has("a")).toBe(true);
+      expect(s1.disposed).toBe(false);
+    });
   });
 
   describe("remove()", () => {