@pensar/apex 2.5.0 → 2.6.1-canary.025b1ecf

package/src/core/ai/caching.test.ts

@@ -0,0 +1,179 @@
+import type { ModelMessage } from "ai";
+import { describe, expect, it } from "vitest";
+import {
+  cacheBreakpointFor,
+  withCachedLastMessage,
+  withCachedSystemPrompt,
+} from "./caching";
+import { AVAILABLE_MODELS } from "./models";
+
+const BEDROCK_BREAKPOINT = { bedrock: { cachePoint: { type: "default" } } };
+const ANTHROPIC_BREAKPOINT = {
+  anthropic: { cacheControl: { type: "ephemeral" } },
+};
+
+describe("cacheBreakpointFor", () => {
+  it.each([
+    "global.anthropic.claude-opus-4-6-v1",
+    "global.anthropic.claude-opus-4-8",
+    "us.anthropic.claude-haiku-4-5-20251001-v1:0",
+    "anthropic.claude-sonnet-4-6-v1",
+  ])("uses the bedrock cachePoint namespace for %s", (model) => {
+    expect(cacheBreakpointFor(model)).toEqual(BEDROCK_BREAKPOINT);
+  });
+
+  it("never emits an anthropic key on Bedrock", () => {
+    // The bug this guards: @ai-sdk/amazon-bedrock only reads
+    // providerOptions.bedrock.cachePoint, so an `anthropic` key is dropped with
+    // no error and every request bills the full prefix.
+    expect(
+      cacheBreakpointFor("global.anthropic.claude-opus-4-6-v1"),
+    ).not.toHaveProperty("anthropic");
+  });
+
+  it.each([
+    "us.deepseek.r1-v1:0",
+    "us.amazon.nova-lite-v1:0",
+    "amazon.titan-text-express-v1",
+    "anthropic.claude-v2",
+    "anthropic.claude-v2:1",
+    "anthropic.claude-instant-v1",
+    "global.anthropic.claude-3-opus-20240229-v1:0",
+    "us.anthropic.claude-3-haiku-20240307-v1:0",
+    "anthropic.claude-3-5-sonnet-20240620-v1:0",
+    "us.anthropic.claude-3-5-sonnet-20240620-v1:0",
+    "global.anthropic.claude-3-5-sonnet-20240620-v1:0",
+    // Rest of the claude-3 line: cache support was uneven and all of it is
+    // end-of-life on Bedrock, so none of it may carry a cache point.
+    "global.anthropic.claude-3-5-sonnet-20241022-v2:0",
+    "us.anthropic.claude-3-5-haiku-20241022-v1:0",
+    "us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+  ])("returns undefined for Bedrock model %s", (model) => {
+    // Bedrock rejects a cache point outright on models that don't support
+    // prompt caching, so these must stay uncached rather than 400.
+    expect(cacheBreakpointFor(model)).toBeUndefined();
+  });
+
+  it("caches every Claude 4+ id in the registry and no earlier one", () => {
+    // Guards the generation boundary wholesale rather than by sampled id:
+    // 3.x put the generation first (claude-3-5-sonnet-…), 4.x puts the tier
+    // first (claude-sonnet-4-…), and only the latter may carry a cache point.
+    const bedrockClaude = AVAILABLE_MODELS.filter(
+      (m) => m.provider === "bedrock" && m.id.includes("anthropic.claude"),
+    );
+    expect(bedrockClaude.length).toBeGreaterThan(0);
+    for (const { id } of bedrockClaude) {
+      const base = id.replace(/^(?:[a-z]{2,6}\.)?anthropic\./, "");
+      const isGen4Plus = /^claude-(?:opus|sonnet|haiku)-\d/.test(base);
+      expect({ id, cached: cacheBreakpointFor(id) !== undefined }).toEqual({
+        id,
+        cached: isGen4Plus,
+      });
+    }
+  });
+
+  it("uses the anthropic namespace for direct Anthropic models", () => {
+    expect(cacheBreakpointFor("claude-haiku-4-5")).toEqual(
+      ANTHROPIC_BREAKPOINT,
+    );
+  });
+
+  it("uses the anthropic namespace for Pensar gateway models", () => {
+    // The gateway hand-builds native Anthropic cache_control blocks — see
+    // providers/pensarFormatters.ts.
+    expect(cacheBreakpointFor("pensar:anthropic.claude-opus-4-8")).toEqual(
+      ANTHROPIC_BREAKPOINT,
+    );
+  });
+
+  it.each([
+    "gpt-5.6-sol",
+    "some-unknown-model",
+  ])("returns undefined for %s", (model) => {
+    expect(cacheBreakpointFor(model)).toBeUndefined();
+  });
+});
+
+describe("withCachedSystemPrompt", () => {
+  const messages: ModelMessage[] = [{ role: "user", content: "hi" }];
+
+  it("prepends a system message carrying the breakpoint", () => {
+    const result = withCachedSystemPrompt(
+      "be helpful",
+      messages,
+      BEDROCK_BREAKPOINT,
+    );
+    expect(result[0]).toEqual({
+      role: "system",
+      content: "be helpful",
+      providerOptions: BEDROCK_BREAKPOINT,
+    });
+    expect(result.slice(1)).toEqual(messages);
+  });
+
+  it("omits providerOptions when the model has no breakpoint", () => {
+    const result = withCachedSystemPrompt("be helpful", messages, undefined);
+    expect(result[0]).not.toHaveProperty("providerOptions");
+  });
+});
+
+describe("withCachedLastMessage", () => {
+  it("marks only the last message", () => {
+    const result = withCachedLastMessage(
+      [
+        { role: "user", content: "first" },
+        { role: "assistant", content: "second" },
+        { role: "user", content: "third" },
+      ],
+      BEDROCK_BREAKPOINT,
+    );
+    expect(result[0].providerOptions).toBeUndefined();
+    expect(result[1].providerOptions).toBeUndefined();
+    expect(result[2].providerOptions).toEqual(BEDROCK_BREAKPOINT);
+  });
+
+  it("merges with providerOptions already on the message", () => {
+    const result = withCachedLastMessage(
+      [
+        {
+          role: "user",
+          content: "hi",
+          providerOptions: { anthropic: { signature: "abc" } },
+        },
+      ],
+      BEDROCK_BREAKPOINT,
+    );
+    expect(result[0].providerOptions).toEqual({
+      anthropic: { signature: "abc" },
+      ...BEDROCK_BREAKPOINT,
+    });
+  });
+
+  it("keeps sibling options within the same provider namespace", () => {
+    const result = withCachedLastMessage(
+      [
+        {
+          role: "user",
+          content: "hi",
+          providerOptions: { bedrock: { citations: { enabled: true } } },
+        },
+      ],
+      BEDROCK_BREAKPOINT,
+    );
+    expect(result[0].providerOptions).toEqual({
+      bedrock: {
+        citations: { enabled: true },
+        cachePoint: { type: "default" },
+      },
+    });
+  });
+
+  it("is a no-op without a breakpoint", () => {
+    const messages: ModelMessage[] = [{ role: "user", content: "hi" }];
+    expect(withCachedLastMessage(messages, undefined)).toBe(messages);
+  });
+
+  it("is a no-op on an empty list", () => {
+    expect(withCachedLastMessage([], BEDROCK_BREAKPOINT)).toEqual([]);
+  });
+});

package/src/core/ai/caching.ts

@@ -0,0 +1,119 @@
+import type { ModelMessage } from "ai";
+import type { AIModel } from "./ai";
+import { getModelInfo } from "./models";
+
+type CacheBreakpoint = NonNullable<ModelMessage["providerOptions"]>;
+
+const ANTHROPIC_CACHE_CONTROL: CacheBreakpoint = {
+  anthropic: { cacheControl: { type: "ephemeral" as const } },
+};
+
+/**
+ * Bedrock's Converse API namespaces the breakpoint under `bedrock.cachePoint`.
+ * Provider options are keyed by provider id, so an `anthropic` key is dropped
+ * with no error and no warning — which is how caching stayed silently off on
+ * every Bedrock model. Confirmed against live Bedrock: `anthropic.cacheControl`
+ * reports cacheWrite/cacheRead of 0, `bedrock.cachePoint` caches the prefix.
+ */
+const BEDROCK_CACHE_POINT: CacheBreakpoint = {
+  bedrock: { cachePoint: { type: "default" as const } },
+};
+
+/**
+ * Legacy Claude on Bedrock — v2, instant, and the whole `claude-3-…` line.
+ * Cache-point support was uneven across it (3.5 Sonnet v1 never had it), and
+ * that is now moot: every `claude-3-…` id is end-of-life on Bedrock and fails
+ * with "This model version has reached the end of its life" before caching is
+ * reached. Denying the line wholesale beats enumerating per-snapshot support
+ * for models that can no longer be invoked.
+ *
+ * The generation boundary is legible in the id itself — 3.x puts the generation
+ * first (`claude-3-5-sonnet-…`), 4.x onward puts the tier first
+ * (`claude-sonnet-4-…`) — so this stays a deny-list: a new Claude picks caching
+ * up the moment `generate:models` adds it, rather than silently losing it,
+ * which is the failure this whole fix is about.
+ */
+const BEDROCK_UNCACHEABLE_CLAUDE = /^claude-(?:v2|instant|3[-.])/;
+
+/**
+ * Bedrock *rejects* a cache point on models that don't support prompt caching
+ * ("You invoked an unsupported model or your request did not allow prompt
+ * caching" — observed on `us.deepseek.r1-v1:0`), so this gate is load-bearing:
+ * non-Claude Bedrock models must stay uncached rather than fail.
+ */
+function bedrockSupportsCachePoint(modelId: string): boolean {
+  // Strip the cross-region inference-profile prefix (`global.`, `us.`, `eu.`, …).
+  const base = modelId.replace(/^(?:[a-z]{2,6}\.)?anthropic\./, "");
+  return base.startsWith("claude-") && !BEDROCK_UNCACHEABLE_CLAUDE.test(base);
+}
+
+/**
+ * Provider options marking a cache breakpoint for `model`, or `undefined` when
+ * the model has no breakpoint we can express.
+ */
+export function cacheBreakpointFor(
+  model: AIModel,
+): CacheBreakpoint | undefined {
+  const { provider } = getModelInfo(model);
+  switch (provider) {
+    case "anthropic":
+    // The Pensar gateway speaks the native Anthropic Messages API and reads
+    // this same key — see providers/pensarFormatters.ts.
+    case "pensar":
+      return ANTHROPIC_CACHE_CONTROL;
+    case "bedrock":
+      return bedrockSupportsCachePoint(model) ? BEDROCK_CACHE_POINT : undefined;
+    default:
+      return undefined;
+  }
+}
+
+/**
+ * Prepend a system message carrying a cache breakpoint to the messages array.
+ * This ensures tools + system prompt are cached on every request.
+ *
+ * The system prompt is placed as a system-role message at the start of
+ * the messages array with providerOptions containing the breakpoint, so the
+ * provider caches the entire prefix (tools + system) and reads it from cache
+ * on subsequent turns (~90% cost reduction).
+ */
+export function withCachedSystemPrompt(
+  system: string,
+  messages: ModelMessage[],
+  breakpoint: CacheBreakpoint | undefined,
+): ModelMessage[] {
+  return [
+    {
+      role: "system" as const,
+      content: system,
+      ...(breakpoint ? { providerOptions: breakpoint } : {}),
+    },
+    ...messages,
+  ];
+}
+
+/**
+ * Add a cache breakpoint to the last message for incremental conversation
+ * caching. On each turn, everything up to (and including) the last message is
+ * read from cache, so only the new content after the breakpoint is billed at
+ * full price.
+ */
+export function withCachedLastMessage(
+  messages: ModelMessage[],
+  breakpoint: CacheBreakpoint | undefined,
+): ModelMessage[] {
+  if (messages.length === 0 || !breakpoint) return messages;
+
+  return messages.map((msg, i) => {
+    if (i === messages.length - 1) {
+      // Merge inside the provider's namespace — a top-level spread would drop
+      // any sibling options the message already carries for that provider.
+      const merged = { ...msg.providerOptions };
+      for (const [provider, options] of Object.entries(breakpoint)) {
+        merged[provider] = { ...merged[provider], ...options };
+      }
+      return { ...msg, providerOptions: merged };
+    }
+    return msg;
+  });
+}

package/src/core/ai/compactionEvidence.test.ts

@@ -0,0 +1,52 @@
+import { describe, expect, it, vi } from "vitest";
+import { encodeCompactionEvidence } from "./compactionEvidence";
+
+describe("bounded compaction evidence", () => {
+  it("matches JSON for text, escaping, arrays and absent optional fields", () => {
+    const value = {
+      text: 'hello\n"world" 🐴',
+      absent: undefined,
+      parts: [null, true, 2, { output: "🐴" }],
+    };
+    const json = JSON.stringify(value);
+    expect(encodeCompactionEvidence(value, Buffer.byteLength(json))).toEqual({
+      status: "available",
+      json,
+      bytes: Buffer.byteLength(json),
+    });
+    expect(
+      encodeCompactionEvidence(value, Buffer.byteLength(json) - 1),
+    ).toMatchObject({ status: "truncated", reason: "byte_limit", bytes: 0 });
+  });
+
+  it("bounds wide and deep traversal without invoking getters or custom serializers", () => {
+    expect(
+      encodeCompactionEvidence(
+        Array.from({ length: 20_000 }, () => 0),
+        1_000_000,
+      ),
+    ).toMatchObject({ status: "truncated", reason: "structure_limit" });
+    let deep: unknown = "end";
+    for (let i = 0; i < 40; i++) deep = { nested: deep };
+    expect(encodeCompactionEvidence(deep, 10_000)).toMatchObject({
+      status: "truncated",
+      reason: "structure_limit",
+    });
+    const getter = vi.fn();
+    const value = Object.defineProperty({}, "secret", {
+      enumerable: true,
+      get: getter,
+    });
+    expect(encodeCompactionEvidence(value, 100)).toMatchObject({
+      status: "unavailable",
+      reason: "unsupported_value",
+    });
+    expect(getter).not.toHaveBeenCalled();
+    const toJSON = vi.fn();
+    expect(encodeCompactionEvidence({ toJSON }, 100)).toMatchObject({
+      status: "unavailable",
+      reason: "unsupported_value",
+    });
+    expect(toJSON).not.toHaveBeenCalled();
+  });
+});

package/src/core/ai/compactionEvidence.ts

@@ -0,0 +1,101 @@
+const MAX_NODES = 10_000;
+const MAX_DEPTH = 32;
+
+export const CONTEXT_EVIDENCE_BYTES = 2 * 1024 * 1024;
+export const RESULT_EVIDENCE_BYTES = 256 * 1024;
+export const MAX_RESULT_EVIDENCE = 256;
+
+export type EvidenceStatus =
+  | "available"
+  | "truncated"
+  | "unavailable"
+  | "failed";
+
+class EvidenceLimit extends Error {
+  constructor(
+    readonly status: EvidenceStatus,
+    readonly reason: string,
+  ) {
+    super(reason);
+  }
+}
+
+// Bound traversal and allocations before serialization, including huge strings.
+export function encodeCompactionEvidence(value: unknown, maxBytes: number) {
+  const chunks: string[] = [];
+  const ancestors = new Set<object>();
+  let bytes = 0;
+  let nodes = 0;
+  const append = (text: string) => {
+    bytes += Buffer.byteLength(text);
+    if (bytes > maxBytes) throw new EvidenceLimit("truncated", "byte_limit");
+    chunks.push(text);
+  };
+  const string = (text: string) => {
+    if (text.length > maxBytes - bytes)
+      throw new EvidenceLimit("truncated", "byte_limit");
+    append(JSON.stringify(text));
+  };
+  const visit = (item: unknown, depth: number): void => {
+    if (++nodes > MAX_NODES || depth > MAX_DEPTH)
+      throw new EvidenceLimit("truncated", "structure_limit");
+    if (typeof item === "string") {
+      string(item);
+      return;
+    }
+    if (
+      item === null ||
+      typeof item === "boolean" ||
+      typeof item === "number"
+    ) {
+      append(JSON.stringify(item));
+      return;
+    }
+    if (typeof item !== "object" || ancestors.has(item))
+      throw new EvidenceLimit("unavailable", "unsupported_value");
+    const array = Array.isArray(item);
+    const prototype = Object.getPrototypeOf(item);
+    if (!array && prototype !== Object.prototype && prototype !== null)
+      throw new EvidenceLimit("unavailable", "unsupported_value");
+    ancestors.add(item);
+    append(array ? "[" : "{");
+    let count = 0;
+    const property = (key: string) => {
+      if (++nodes > MAX_NODES)
+        throw new EvidenceLimit("truncated", "structure_limit");
+      const descriptor = Object.getOwnPropertyDescriptor(item, key);
+      if (!descriptor || !("value" in descriptor))
+        throw new EvidenceLimit("unavailable", "unsupported_value");
+      if (!array && descriptor.value === undefined) return;
+      if (count++) append(",");
+      if (!array) {
+        string(key);
+        append(":");
+      }
+      visit(
+        descriptor.value === undefined ? null : descriptor.value,
+        depth + 1,
+      );
+    };
+    if (array) {
+      for (let i = 0; i < item.length; i++) property(String(i));
+    } else {
+      for (const key in item) {
+        if (Object.hasOwn(item, key)) property(key);
+      }
+    }
+    append(array ? "]" : "}");
+    ancestors.delete(item);
+  };
+  try {
+    visit(value, 0);
+    return { status: "available" as const, json: chunks.join(""), bytes };
+  } catch (error) {
+    return {
+      status: error instanceof EvidenceLimit ? error.status : "failed",
+      reason:
+        error instanceof EvidenceLimit ? error.reason : "serialization_failed",
+      bytes: 0,
+    };
+  }
+}