@vellumai/assistant 0.8.11 → 0.8.12-staging.2

package/src/providers/anthropic/client.ts

@@ -1,5 +1,6 @@
 import Anthropic from "@anthropic-ai/sdk";
 
+import { SYSTEM_PROMPT_CACHE_BOUNDARY } from "../../prompts/cache-boundary.js";
 import { isAbortReason } from "../../util/abort-reasons.js";
 import { ProviderError } from "../../util/errors.js";
 import { getLogger } from "../../util/logger.js";
@@ -809,6 +810,12 @@ export class AnthropicProvider implements Provider {
     const mutableLatestUserMessage =
       (config as Record<string, unknown> | undefined)
         ?.mutableLatestUserMessage === true;
+    // Full prompt-caching opt-out: send no cache breakpoints at all and strip
+    // caller-stamped block-level markers. Resolved per call site (see
+    // `disableCache` in the LLM config schema) for one-shot prompts where
+    // every breakpoint is a paid cache write with no future read.
+    const disableCache =
+      (config as Record<string, unknown> | undefined)?.disableCache === true;
     let sentMessages: Anthropic.MessageParam[] | undefined;
     const startedAt = Date.now();
     // Hoisted so the catch block can distinguish our inner stream timeout
@@ -998,6 +1005,7 @@ export class AnthropicProvider implements Provider {
         cacheTtl: _cacheTtl,
         disableTurnStartCache: _disableTurnStartCache,
         mutableLatestUserMessage: _mutableLatestUserMessage,
+        disableCache: _disableCache,
         max_tokens: callerMaxTokens,
         usageAttributionHeaders,
         ...restConfig
@@ -1054,19 +1062,34 @@ export class AnthropicProvider implements Provider {
       };
 
       if (systemPrompt) {
-        // The whole system prompt is rendered as a single cached
-        // block.  A 1-hour cache TTL is used (when supported by the
-        // model) so the breakpoint survives turn gaps that exceed the
-        // default 5-minute window.
-        params.system = [
-          {
+        // The system prompt may carry a cache boundary (placed by the
+        // section pipeline — see `prompts/sections.ts`) splitting it into
+        // a stable-prefix block and a volatile-suffix block, each with
+        // its own breakpoint so a volatile-section change doesn't
+        // re-create the stable prefix.  A 1-hour cache TTL is used (when
+        // supported by the model) so the breakpoints survive turn gaps
+        // that exceed the default 5-minute window.
+        params.system = systemPrompt
+          .split(SYSTEM_PROMPT_CACHE_BOUNDARY)
+          .filter((text) => text.length > 0)
+          .map((text) => ({
             type: "text" as const,
-            text: systemPrompt,
-            cache_control: cacheControl,
-          },
-        ];
+            text,
+            ...(disableCache ? {} : { cache_control: cacheControl }),
+          }));
+        if (params.system.length === 0) delete params.system;
       }
 
+      // Tools precede the system blocks in the cached prefix, so the first
+      // system breakpoint already covers the tool definitions.  When the
+      // system prompt is split into two blocks, skip the explicit last-tool
+      // breakpoint to stay within Anthropic's 4-breakpoint budget; with a
+      // single (or no) system block the tool breakpoint is kept.
+      const systemBlockCount = Array.isArray(params.system)
+        ? params.system.length
+        : 0;
+      const applyToolCacheControl = systemBlockCount < 2;
+
       if (tools && tools.length > 0) {
         if (
           this.useNativeWebSearch &&
@@ -1077,7 +1100,9 @@ export class AnthropicProvider implements Provider {
             name: t.name,
             description: t.description,
             input_schema: t.input_schema as Anthropic.Tool["input_schema"],
-            ...(i === otherTools.length - 1
+            ...(applyToolCacheControl &&
+            !disableCache &&
+            i === otherTools.length - 1
               ? { cache_control: cacheControl }
               : {}),
           }));
@@ -1092,7 +1117,9 @@ export class AnthropicProvider implements Provider {
             name: t.name,
             description: t.description,
             input_schema: t.input_schema as Anthropic.Tool["input_schema"],
-            ...(i === tools.length - 1 ? { cache_control: cacheControl } : {}),
+            ...(applyToolCacheControl && !disableCache && i === tools.length - 1
+              ? { cache_control: cacheControl }
+              : {}),
           }));
         }
       }
@@ -1140,6 +1167,7 @@ export class AnthropicProvider implements Provider {
         mutableLatestUserMessage && turnStartIdx === msgs.length - 1;
       if (
         turnStartIdx >= 0 &&
+        !disableCache &&
         !disableTurnStartCache &&
         !skipVolatileTurnStartAnchor
       ) {
@@ -1156,7 +1184,11 @@ export class AnthropicProvider implements Provider {
       // cache_creation tokens per new turn). Skipped during tool-use loops
       // where the current turn-start already covers the same prefix and a
       // second anchor would blow the 4-breakpoint budget.
-      if (turnStartIdx === msgs.length - 1 && turnStartIdx > 0) {
+      if (
+        !disableCache &&
+        turnStartIdx === msgs.length - 1 &&
+        turnStartIdx > 0
+      ) {
         const prevTurnAnchorIdx = findUserTextMsgIdx(turnStartIdx - 1);
         if (prevTurnAnchorIdx >= 0)
           applyCacheControlToLastBlock(prevTurnAnchorIdx);
@@ -1168,7 +1200,11 @@ export class AnthropicProvider implements Provider {
       // cheaply without conflicting with the 1h breakpoints above.
       // Skip thinking/redacted_thinking blocks — Anthropic doesn't allow
       // cache_control on those types.
-      if (turnStartIdx >= 0 && turnStartIdx < sentMessages.length - 1) {
+      if (
+        !disableCache &&
+        turnStartIdx >= 0 &&
+        turnStartIdx < sentMessages.length - 1
+      ) {
         const lastMsg = sentMessages[sentMessages.length - 1];
         if (Array.isArray(lastMsg.content) && lastMsg.content.length > 0) {
           const NON_CACHEABLE_TYPES = new Set([
@@ -1193,11 +1229,12 @@ export class AnthropicProvider implements Provider {
         }
       }
 
-      // Cache-breakpoint accounting: system(1) + tools(1) + turn-start(1) +
-      // (tail OR prev-turn-anchor)(1) = 4 — exactly Anthropic's per-request
-      // cap.  Tail and prev-turn-anchor are mutually exclusive (the latter
-      // only fires when turn-start is the last message, which suppresses
-      // the tail), so the total can't drift past 4.
+      // Cache-breakpoint accounting: system(≤2) + tools(1, only when the
+      // system is a single block or absent) + turn-start(1) +
+      // (tail OR prev-turn-anchor)(1) ≤ 4 — Anthropic's per-request cap.
+      // Tail and prev-turn-anchor are mutually exclusive (the latter only
+      // fires when turn-start is the last message, which suppresses the
+      // tail), so the total can't drift past 4.
 
       // Strip orphaned UTF-16 surrogates so the Anthropic JSON parser never
       // sees invalid strings produced by upstream surrogate-splitting `.slice()` calls.
@@ -1208,19 +1245,28 @@ export class AnthropicProvider implements Provider {
         sentMessages = params.messages;
       }
 
-      // Haiku does not support the extended-cache-ttl beta, so it must never
-      // receive a `ttl` on any cache_control. The client's own breakpoints
-      // already omit it for Haiku, but a caller can stamp a `ttl` on message
-      // blocks before the provider sees them — strip it here so the request
-      // stays valid on Haiku models.
-      if (isHaiku) {
+      // Callers can stamp `cache_control` on message blocks before the
+      // provider sees them. Two repairs apply:
+      // - `disableCache`: strip the marker entirely — this call opted out of
+      //   prompt caching, and a leftover block-level marker would still incur
+      //   a cache write.
+      // - Haiku: strip only the `ttl` field — Haiku does not support the
+      //   extended-cache-ttl beta, so a `ttl` would make the request invalid.
+      //   The client's own breakpoints already omit it for Haiku.
+      if (disableCache || isHaiku) {
         for (const msg of sentMessages) {
           if (!Array.isArray(msg.content)) continue;
           for (const block of msg.content) {
             if (typeof block === "string") continue;
-            const cc = (block as { cache_control?: { ttl?: unknown } })
-              .cache_control;
-            if (cc && "ttl" in cc) delete cc.ttl;
+            const blockRecord = block as {
+              cache_control?: { ttl?: unknown };
+            };
+            if (!blockRecord.cache_control) continue;
+            if (disableCache) {
+              delete blockRecord.cache_control;
+            } else if ("ttl" in blockRecord.cache_control) {
+              delete blockRecord.cache_control.ttl;
+            }
           }
         }
       }

package/src/providers/gemini/client.ts

@@ -5,6 +5,7 @@ import {
   THINKING_LEVELS,
   type ThinkingLevel as ThinkingLevelName,
 } from "../../config/schemas/llm.js";
+import { SYSTEM_PROMPT_CACHE_BOUNDARY } from "../../prompts/cache-boundary.js";
 import { isAbortReason } from "../../util/abort-reasons.js";
 import { ProviderError } from "../../util/errors.js";
 import { getLogger } from "../../util/logger.js";
@@ -314,7 +315,10 @@ export class GeminiProvider implements Provider {
       const geminiConfig: genai.GenerateContentConfig = {};
 
       if (systemPrompt) {
-        geminiConfig.systemInstruction = systemPrompt;
+        geminiConfig.systemInstruction = systemPrompt.replaceAll(
+          SYSTEM_PROMPT_CACHE_BOUNDARY,
+          "\n\n",
+        );
       }
       if (maxTokens) {
         geminiConfig.maxOutputTokens = maxTokens;

package/src/providers/minimax/client.ts

@@ -101,6 +101,15 @@ export class MinimaxProvider extends OpenAIChatCompletionsProvider {
       providerName: "minimax",
       providerLabel: "MiniMax",
       streamTimeoutMs: options.streamTimeoutMs,
+      // Without reasoning_split, MiniMax embeds reasoning in `content`
+      // wrapped in <think>...</think> tags (and also mirrors it into
+      // reasoning deltas), so raw tags leak into user-visible text. With it,
+      // reasoning arrives only via `reasoning_content`/`reasoning_details`,
+      // which the base provider already parses into thinking blocks.
+      extraCreateParams: { reasoning_split: true },
+      // MiniMax models reason between tool calls (interleaved thinking) and
+      // expect prior-turn reasoning replayed on multi-turn requests.
+      assistantReasoningField: "reasoning_content",
     });
   }
 }

package/src/providers/model-catalog.ts

@@ -691,6 +691,24 @@ const RAW_PROVIDER_CATALOG: ProviderCatalogEntry[] = [
           outputPer1mTokens: 2.5,
         },
       },
+      {
+        id: "accounts/fireworks/models/minimax-m3",
+        displayName: "MiniMax M3",
+        // The model supports 1M context, but Fireworks serves it with a
+        // 512K (524,288-token) window; advertise the served limit.
+        contextWindowTokens: 524288,
+        maxOutputTokens: 512000,
+        supportsThinking: true,
+        supportsCaching: true,
+        supportsVision: true,
+        supportsToolUse: true,
+        maxEffort: "high",
+        pricing: {
+          inputPer1mTokens: 0.3,
+          outputPer1mTokens: 1.2,
+          cacheReadPer1mTokens: 0.06,
+        },
+      },
       {
         id: "accounts/fireworks/models/minimax-m2p7",
         displayName: "MiniMax M2.7",
@@ -1241,6 +1259,16 @@ const RAW_PROVIDER_CATALOG: ProviderCatalogEntry[] = [
       linkLabel: "Open MiniMax Dashboard",
     },
     models: [
+      {
+        id: "MiniMax-M3",
+        displayName: "MiniMax M3",
+        contextWindowTokens: 1000000,
+        maxOutputTokens: 512000,
+        supportsThinking: true,
+        supportsCaching: true,
+        supportsVision: true,
+        supportsToolUse: true,
+      },
       {
         id: "MiniMax-M2.7",
         displayName: "MiniMax M2.7",

package/src/providers/model-intents.ts

@@ -13,7 +13,7 @@ const PROVIDER_MODEL_INTENTS: Record<string, Record<ModelIntent, string>> = {
   anthropic: {
     balanced: "claude-sonnet-4-6",
     "latency-optimized": "claude-haiku-4-5-20251001",
-    "quality-optimized": "claude-opus-4-8",
+    "quality-optimized": "claude-fable-5",
     "vision-optimized": "claude-opus-4-6",
   },
   openai: {
@@ -35,7 +35,7 @@ const PROVIDER_MODEL_INTENTS: Record<string, Record<ModelIntent, string>> = {
     "vision-optimized": "llama3.2",
   },
   fireworks: {
-    balanced: "accounts/fireworks/models/kimi-k2p6",
+    balanced: "accounts/fireworks/models/minimax-m3",
     "latency-optimized": "accounts/fireworks/models/kimi-k2p5",
     "quality-optimized": "accounts/fireworks/models/kimi-k2p6",
     "vision-optimized": "accounts/fireworks/models/kimi-k2p6",
@@ -43,7 +43,7 @@ const PROVIDER_MODEL_INTENTS: Record<string, Record<ModelIntent, string>> = {
   openrouter: {
     balanced: "anthropic/claude-sonnet-4.6",
     "latency-optimized": "anthropic/claude-haiku-4.5",
-    "quality-optimized": "anthropic/claude-opus-4.8",
+    "quality-optimized": "anthropic/claude-fable-5",
     "vision-optimized": "anthropic/claude-opus-4.6",
   },
 };

package/src/providers/openai/chat-completions-provider.ts

@@ -1,5 +1,6 @@
 import OpenAI from "openai";
 
+import { SYSTEM_PROMPT_CACHE_BOUNDARY } from "../../prompts/cache-boundary.js";
 import { isAbortReason } from "../../util/abort-reasons.js";
 import { ProviderError } from "../../util/errors.js";
 import { extractRetryAfterMs } from "../../util/retry.js";
@@ -17,6 +18,7 @@ import {
   ContextOverflowError,
   extractOverflowTokensFromMessage,
 } from "../types.js";
+import { wrapUnparseableToolArgs } from "../unparseable-tool-args.js";
 
 /**
  * Detect OpenAI-compatible context-overflow signals on an `OpenAI.APIError`.
@@ -634,7 +636,7 @@ export class OpenAIChatCompletionsProvider implements Provider {
         try {
           input = JSON.parse(tc.args);
         } catch {
-          input = { _raw: tc.args };
+          input = wrapUnparseableToolArgs(tc.args);
         }
         content.push({
           type: "tool_use",
@@ -797,7 +799,7 @@ export class OpenAIChatCompletionsProvider implements Provider {
     if (systemPrompt) {
       result.push({
         role: "system",
-        content: systemPrompt,
+        content: systemPrompt.replaceAll(SYSTEM_PROMPT_CACHE_BOUNDARY, "\n\n"),
       });
     }
 

package/src/providers/openai/responses-provider.ts

@@ -1,5 +1,6 @@
 import OpenAI from "openai";
 
+import { SYSTEM_PROMPT_CACHE_BOUNDARY } from "../../prompts/cache-boundary.js";
 import { isAbortReason } from "../../util/abort-reasons.js";
 import { ProviderError } from "../../util/errors.js";
 import { getLogger } from "../../util/logger.js";
@@ -14,6 +15,7 @@ import type {
   SendMessageOptions,
 } from "../types.js";
 import { ContextOverflowError } from "../types.js";
+import { wrapUnparseableToolArgs } from "../unparseable-tool-args.js";
 import { detectOpenAICompatibleContextOverflow } from "./chat-completions-provider.js";
 
 const log = getLogger("openai-responses");
@@ -216,7 +218,10 @@ export class OpenAIResponsesProvider implements Provider {
       };
 
       if (systemPrompt) {
-        params.instructions = systemPrompt;
+        params.instructions = systemPrompt.replaceAll(
+          SYSTEM_PROMPT_CACHE_BOUNDARY,
+          "\n\n",
+        );
       }
 
       if (maxTokens && !this.codexSubscription) {
@@ -461,7 +466,7 @@ export class OpenAIResponsesProvider implements Provider {
         try {
           input = JSON.parse(tc.args);
         } catch {
-          input = { _raw: tc.args };
+          input = wrapUnparseableToolArgs(tc.args);
         }
         content.push({
           type: "tool_use",

package/src/providers/retry.ts

@@ -268,6 +268,14 @@ function normalizeSendMessageOptions(
     if (nextConfig.thinking === undefined && resolved.thinking !== undefined) {
       nextConfig.thinking = resolved.thinking;
     }
+    // Not a wire field: consumed (and stripped) by provider clients that
+    // implement prompt caching, like `cacheTtl` / `disableTurnStartCache`.
+    if (
+      nextConfig.disableCache === undefined &&
+      resolved.disableCache !== undefined
+    ) {
+      nextConfig.disableCache = resolved.disableCache;
+    }
     // Forward OpenRouter-only routing preferences so `OpenRouterProvider` can
     // translate `openrouter.only` into the wire-format `provider: { only: [...] }`
     // body field on both the OpenAI-compat and Anthropic-compat endpoints.

package/src/providers/types.ts

@@ -222,6 +222,17 @@ export interface SendMessageConfig {
    * behavior.
    */
   mutableLatestUserMessage?: boolean;
+  /**
+   * When true, the provider sends no prompt-cache breakpoints at all (and
+   * strips any block-level `cache_control` markers callers stamped on
+   * messages). For one-shot call sites whose prompts are unique per call or
+   * whose call cadence exceeds the cache TTL, every breakpoint is a paid
+   * cache write that will never be read — opting out saves the write
+   * premium. Resolved per call site via `resolveCallSiteConfig` (see
+   * `disableCache` in the LLM config schema); a per-call explicit value
+   * wins. Default false — existing behavior.
+   */
+  disableCache?: boolean;
   [key: string]: unknown;
 }
 

package/src/providers/unparseable-tool-args.ts

@@ -0,0 +1,56 @@
+/**
+ * Marker shape for tool-call arguments that failed JSON parsing.
+ *
+ * Streaming providers accumulate tool-call argument deltas as a string and
+ * parse the result once the stream completes. Some models/gateways emit
+ * truncated or malformed argument JSON (e.g. MiniMax M3 via OpenRouter cutting
+ * the stream mid-object). The provider cannot drop the tool_use block — the
+ * protocol requires a paired tool_result for every tool_use id — so it wraps
+ * the raw string under this marker key instead.
+ *
+ * The tool execution layer detects the marker and rejects the invocation with
+ * an error tool result, so the model sees the failure and can retry, instead
+ * of a tool executing with garbage input.
+ */
+const UNPARSEABLE_TOOL_ARGS_KEY = "_raw";
+
+/** Wrap raw, unparseable tool-call argument text in the marker shape. */
+export function wrapUnparseableToolArgs(raw: string): Record<string, unknown> {
+  return { [UNPARSEABLE_TOOL_ARGS_KEY]: raw };
+}
+
+/**
+ * Detect input produced by {@link wrapUnparseableToolArgs}: exactly one key,
+ * the marker key, holding the raw argument string. The exact-shape check
+ * avoids false positives on hypothetical legitimate inputs that merely
+ * contain a `_raw` field among others.
+ */
+export function isUnparseableToolArgs(
+  input: Record<string, unknown>,
+): input is { _raw: string } {
+  const keys = Object.keys(input);
+  return (
+    keys.length === 1 &&
+    keys[0] === UNPARSEABLE_TOOL_ARGS_KEY &&
+    typeof input[UNPARSEABLE_TOOL_ARGS_KEY] === "string"
+  );
+}
+
+/**
+ * Build the error message returned to the model when it sends unparseable
+ * tool arguments. Includes a bounded prefix of what was received so the model
+ * can see where its output was cut off or malformed.
+ */
+export function unparseableToolArgsMessage(
+  toolName: string,
+  raw: string,
+): string {
+  const PREVIEW_LIMIT = 200;
+  const preview =
+    raw.length > PREVIEW_LIMIT ? `${raw.slice(0, PREVIEW_LIMIT)}…` : raw;
+  return (
+    `Error: the arguments for "${toolName}" were not valid JSON — the argument stream was malformed or truncated, so the tool was NOT executed. ` +
+    `Received: ${preview || "(empty)"}\n` +
+    `Retry the call with complete, valid JSON arguments.`
+  );
+}

package/src/runtime/AGENTS.md

@@ -20,6 +20,12 @@ SSE handlers built on `ReadableStream` shed slow subscribers when `controller.de
 
 GET handlers must be safe and side-effect-free — they must not enqueue background jobs, mutate database state, or trigger writes. If a feature needs server-initiated work in response to a client request, use an explicit POST endpoint or a push-based flow (SSE event → client refetch). See [RFC 9110 §9.2.1 — Safe Methods](https://httpwg.org/specs/rfc9110.html#safe.methods).
 
+Accepted exceptions (stale-while-revalidate caches): a GET handler may kick off a bounded, fire-and-forget background refresh of a generated-content cache when no fresh cache exists, provided the handler itself stays read-only and returns immediately with cached/fallback copy, the refresh is single-flight (concurrent GETs share one regeneration), and a TTL bounds regeneration frequency. Current instances:
+
+- `GET /v1/identity/intro` — refreshes the generated greeting cache; the background prompt may only depend on static identity/soul context plus caller-supplied local hour/minute.
+- `GET /v1/home/feed` — refreshes the personalized home greeting and suggested-prompt caches via `revalidateHomeContentInBackground()`, which publishes `home_feed_updated` when fresh content lands so clients refetch. This is intentional: home content is generated on demand (when a user actually views Home), never at daemon startup or on a timer.
+- `GET /v1/conversation-starters` — enqueues a `generate_conversation_starters` memory job when the starter set is stale, cooldown-gated and deduped against in-flight jobs.
+
 ### Approvals (confirmations, secrets, trust rules)
 
 Approvals are **orthogonal to message sending**. The assistant asks for approval whenever it needs one — this is a separate concern from how a message enters the system.

package/src/runtime/__tests__/agent-wake.test.ts

@@ -173,7 +173,7 @@ mock.module("../../config/loader.js", () => ({
   getNestedValue: () => undefined,
   setNestedValue: () => {},
   API_KEY_PROVIDERS: [],
-  _appendQuarantineBulletin: () => {},
+  _writeQuarantineNotice: () => {},
   invalidateConfigCache: () => {},
 }));
 
@@ -837,7 +837,7 @@ describe("wakeAgentForOpportunity", () => {
   });
 
   test("applies caller-supplied trustContext to the target before the agent loop runs", async () => {
-    // Background system jobs (memory consolidation, update-bulletin) need
+    // Background system jobs (e.g. memory consolidation) need
     // guardian trust to clear the side-effect approval gate. The wake must
     // call setTrustContext BEFORE agentLoop.run so the per-turn snapshot
     // captures the elevated trust.

package/src/runtime/agent-wake.ts

@@ -66,7 +66,7 @@ import {
 import {
   recordCompactionEndBestEffort,
   recordCompactionStartBestEffort,
-} from "../memory/compaction-log-writer-clickhouse.js";
+} from "../memory/compaction-log-store-clickhouse.js";
 import { getConversationOverrideProfile } from "../memory/conversation-crud.js";
 import {
   buildProviderErrorResponsePayload,
@@ -167,10 +167,10 @@ export interface WakeOptions {
 }
 
 /**
- * Reason a wake returned `invoked: false`. Callers (CLI, update-bulletin
- * job) need to distinguish "conversation doesn't exist" from "conversation
- * exists but stayed busy past the wait-until-idle timeout" — the former is
- * a user-visible error, the latter is an expected transient condition.
+ * Reason a wake returned `invoked: false`. Callers (e.g. the CLI) need to
+ * distinguish "conversation doesn't exist" from "conversation exists but
+ * stayed busy past the wait-until-idle timeout" — the former is a
+ * user-visible error, the latter is an expected transient condition.
  */
 export type WakeSkipReason =
   | "not_found"

package/src/runtime/background-job-runner.ts

@@ -3,7 +3,7 @@
  *
  * `runBackgroundJob()` consolidates the bootstrap → processMessage → timeout
  * pattern that every background producer (heartbeat, filing, scheduler, memory
- * consolidation, watcher, update-bulletin, subagent, sequence) has been
+ * consolidation, watcher, subagent, sequence) has been
  * open-coding. Wrapping it here lets us:
  *
  *  - apply a single timeout policy
@@ -182,7 +182,7 @@ export async function runBackgroundJob(
   // failed" rows visible in the sidebar the moment a real user hatches the
   // assistant — see `pre-first-message-gate.ts` for the rationale.
   //
-  // Service-level callers (heartbeat, update-bulletin) are expected to gate
+  // Service-level callers (e.g. heartbeat) are expected to gate
   // earlier and never reach this point; reaching the gate here means a
   // caller either forgot to gate or deliberately opted in via
   // `allowPreFirstUserMessage`. We log at `info` (not `warn`) because the