@intentface/latch-core 0.9.1

package/dist/prompt-caching.js

@@ -0,0 +1,165 @@
+/**
+ * Prompt-caching plans: which provider options a turn should set so the model
+ * provider caches the stable prompt prefix (tool definitions, system prompt,
+ * conversation history) instead of re-billing it every step.
+ *
+ * Kept provider-generic in core: the PLATFORM decides which provider a model id
+ * belongs to (it owns the catalog) and calls `promptCachingPlanFor` from its
+ * `RuntimeConfig.resolvePromptCaching` hook; core only applies the plan in
+ * `buildTurn`. Providers not named here (e.g. Google: implicit caching only)
+ * get no plan and are untouched.
+ *
+ * Anthropic layout — three of the four allowed `cache_control` breakpoints:
+ *  1. The LAST function tool definition. Tools render before the system prompt
+ *     in the request, so this keeps the (often large, MCP-heavy) tool prefix
+ *     cached even when the system block changes mid-conversation (e.g. a
+ *     memory-index write).
+ *  2. The system message that holds the agent instructions + memory block. The
+ *     day-granular date line is a SEPARATE, unmarked system message after it,
+ *     so the daily date flip never invalidates this breakpoint.
+ *  3. A request-level (top-level) `cache_control` — Anthropic's automatic
+ *     caching: the API places a breakpoint on the last cacheable block of the
+ *     request. The AI SDK re-sends call-level providerOptions on every step of
+ *     the tool loop, so this breakpoint ROLLS to the newest tool result with no
+ *     per-step code. Note: top-level cache_control is a first-party Claude API
+ *     feature (not Bedrock/Vertex); block-level markers work everywhere. Also
+ *     note the SDK's client-side breakpoint validator counts only block-level
+ *     markers, so the request-level one is invisible to it — harmless.
+ *
+ * TTL is left at the provider default (5m). Agents whose turns arrive slower
+ * than that won't benefit — that's surfaced by the platform's caching lint,
+ * not worked around here.
+ */
+/** A cache-control marker in the shape the Anthropic provider expects. */
+const EPHEMERAL = { type: "ephemeral" };
+/**
+ * The caching plan for a provider family (`"anthropic"`, `"openai"`, …).
+ * `cacheKeySuffix` scopes OpenAI's cache routing key beyond the agent name —
+ * pass a per-principal key for agents whose system prompt is per-principal
+ * (e.g. memory-enabled agents), so structurally different prefixes don't
+ * co-route to one shard.
+ */
+export function promptCachingPlanFor(provider, opts) {
+    if (provider === "anthropic") {
+        return {
+            requestProviderOptions: { anthropic: { cacheControl: { ...EPHEMERAL } } },
+            systemProviderOptions: { anthropic: { cacheControl: { ...EPHEMERAL } } },
+            toolProviderOptions: { anthropic: { cacheControl: { ...EPHEMERAL } } },
+        };
+    }
+    if (provider === "openai") {
+        const key = opts.cacheKeySuffix
+            ? `latch:agent:${opts.agent}:${opts.cacheKeySuffix}`
+            : `latch:agent:${opts.agent}`;
+        // OpenAI caps prompt_cache_key at 64 chars and 400s the whole turn past
+        // it — agent names and principal keys are unbounded, so an over-long key
+        // collapses to a deterministic hash of the full key material (routing
+        // only needs stability, not readability).
+        const bounded = key.length <= 64 ? key : `latch:${fnv1a64(key)}`;
+        return { requestProviderOptions: { openai: { promptCacheKey: bounded } } };
+    }
+    return undefined;
+}
+/**
+ * The DEFAULT caching plan, used when the runtime has no `resolvePromptCaching`
+ * hook: derive the provider family from the model object itself — every AI SDK
+ * `LanguageModel` carries a `provider` string (`"anthropic.messages"`,
+ * `"openai.responses"`, …) — and apply {@link promptCachingPlanFor}. Measured
+ * on a real MCP-heavy agent, this is a ~6x difference in unit cost (a ~32k-token
+ * tool catalogue re-billed every step vs read from cache), which is not a
+ * default a customer should have to discover by reading source.
+ *
+ * Deliberately conservative: only FIRST-PARTY Anthropic/OpenAI provider ids
+ * match (Bedrock/Vertex ids don't contain these substrings), because the
+ * Anthropic plan's request-level `cache_control` is a first-party feature.
+ * Unknown providers get no plan and are untouched. Opt out entirely with
+ * `resolvePromptCaching: () => undefined`, or per turn with
+ * `promptCaching: false`.
+ */
+export function defaultPromptCachingPlan(model, agent) {
+    const provider = typeof model === "object" && model !== null
+        ? String(model.provider ?? "")
+        : "";
+    const family = provider.includes("anthropic")
+        ? "anthropic"
+        : provider.includes("openai")
+            ? "openai"
+            : undefined;
+    return promptCachingPlanFor(family, { agent });
+}
+/** FNV-1a 64-bit over a string — dependency-free stable fingerprint. */
+function fnv1a64(input) {
+    let hash = 0xcbf29ce484222325n;
+    const prime = 0x100000001b3n;
+    for (let i = 0; i < input.length; i++) {
+        hash ^= BigInt(input.charCodeAt(i));
+        hash = (hash * prime) & 0xffffffffffffffffn;
+    }
+    return hash.toString(16).padStart(16, "0");
+}
+/**
+ * One-level-deep merge of providerOptions objects (per provider key), so e.g.
+ * `{ anthropic: { thinking } }` from reasoning and `{ anthropic: { cacheControl } }`
+ * from caching coexist — a plain spread would clobber one with the other. Later
+ * sources win on conflicting inner keys. Returns undefined when nothing to merge.
+ */
+export function mergeProviderOptions(...sources) {
+    const present = sources.filter((s) => !!s);
+    if (present.length === 0)
+        return undefined;
+    if (present.length === 1)
+        return present[0];
+    const out = {};
+    for (const source of present) {
+        for (const [provider, options] of Object.entries(source)) {
+            const prev = out[provider];
+            out[provider] =
+                prev && typeof prev === "object" && options && typeof options === "object"
+                    ? { ...prev, ...options }
+                    : options;
+        }
+    }
+    return out;
+}
+/**
+ * Stable fingerprint of a toolset (FNV-1a 64-bit over the sorted names) —
+ * equality is all that matters (same hash ⇔ same tool names), so a
+ * dependency-free non-cryptographic hash beats pulling in node:crypto (core
+ * stays runtime-agnostic). Stored on the run row to detect toolset churn
+ * across a chat's turns, one of the main prompt-cache invalidators.
+ */
+export function toolsetHash(names) {
+    return fnv1a64([...names].sort().join(","));
+}
+/**
+ * Mark the LAST active function tool with the given providerOptions (in place).
+ * Skips provider-executed tools (`type: "provider"`, e.g. Anthropic's native
+ * web_search/web_fetch) — the provider silently drops `cache_control` on those —
+ * and, when `activeNames` is given, tools the turn filtered out via
+ * `activeTools` (a marker on an unsent tool would vanish). Returns the marked
+ * tool name, or undefined when no eligible tool exists.
+ */
+export function markLastFunctionTool(tools, providerOptions, activeNames) {
+    const active = activeNames ? new Set(activeNames) : undefined;
+    const names = Object.keys(tools);
+    for (let i = names.length - 1; i >= 0; i--) {
+        const name = names[i];
+        if (active && !active.has(name))
+            continue;
+        const t = tools[name];
+        if (!t || typeof t !== "object")
+            continue;
+        // "provider" (and any future provider-executed variant) can't carry
+        // cache_control; everything else (function/dynamic) serializes as a
+        // client tool definition and can.
+        if (typeof t.type === "string" && t.type.startsWith("provider"))
+            continue;
+        tools[name] = {
+            ...t,
+            providerOptions: mergeProviderOptions(t.providerOptions, providerOptions),
+        };
+        return name;
+    }
+    return undefined;
+}
+//# sourceMappingURL=prompt-caching.js.map

package/dist/prompt-caching.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt-caching.js","sourceRoot":"","sources":["../src/prompt-caching.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,0EAA0E;AAC1E,MAAM,SAAS,GAAG,EAAE,IAAI,EAAE,WAAW,EAAW,CAAC;AAwBjD;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAClC,QAA4B,EAC5B,IAAgD;IAEhD,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;QAC7B,OAAO;YACL,sBAAsB,EAAE,EAAE,SAAS,EAAE,EAAE,YAAY,EAAE,EAAE,GAAG,SAAS,EAAE,EAAE,EAAE;YACzE,qBAAqB,EAAE,EAAE,SAAS,EAAE,EAAE,YAAY,EAAE,EAAE,GAAG,SAAS,EAAE,EAAE,EAAE;YACxE,mBAAmB,EAAE,EAAE,SAAS,EAAE,EAAE,YAAY,EAAE,EAAE,GAAG,SAAS,EAAE,EAAE,EAAE;SACvE,CAAC;IACJ,CAAC;IACD,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,IAAI,CAAC,cAAc;YAC7B,CAAC,CAAC,eAAe,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,cAAc,EAAE;YACpD,CAAC,CAAC,eAAe,IAAI,CAAC,KAAK,EAAE,CAAC;QAChC,wEAAwE;QACxE,yEAAyE;QACzE,sEAAsE;QACtE,0CAA0C;QAC1C,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACjE,OAAO,EAAE,sBAAsB,EAAE,EAAE,MAAM,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IAC7E,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,wBAAwB,CACtC,KAAc,EACd,KAAa;IAEb,MAAM,QAAQ,GACZ,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QACzC,CAAC,CAAC,MAAM,CAAE,KAAgC,CAAC,QAAQ,IAAI,EAAE,CAAC;QAC1D,CAAC,CAAC,EAAE,CAAC;IACT,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC;QAC3C,CAAC,CAAC,WAAW;QACb,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAC3B,CAAC,CAAC,QAAQ;YACV,CAAC,CAAC,SAAS,CAAC;IAChB,OAAO,oBAAoB,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;AACjD,CAAC;AAED,wEAAwE;AACxE,SAAS,OAAO,CAAC,KAAa;IAC5B,IAAI,IAAI,GAAG,mBAAmB,CAAC;IAC/B,MAAM,KAAK,GAAG,cAAc,CAAC;IAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,IAAI,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,GAAG,mBAAmB,CAAC;IAC9C,CAAC;IACD,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;AAC7C,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAClC,GAAG,OAAmD;IAEtD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAgC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IAC5C,MAAM,GAAG,GAA4B,EAAE,CAAC;IACxC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC3B,GAAG,CAAC,QAAQ,CAAC;gBACX,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ;oBACxE,CAAC,CAAC,EAAE,GAAI,IAAe,EAAE,GAAI,OAAkB,EAAE;oBACjD,CAAC,CAAC,OAAO,CAAC;QAChB,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,KAAuB;IACjD,OAAO,OAAO,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAClC,KAA8B,EAC9B,eAAwD,EACxD,WAA+B;IAE/B,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QACvB,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAS;QAC1C,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,CAAiE,CAAC;QACtF,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,SAAS;QAC1C,oEAAoE;QACpE,oEAAoE;QACpE,kCAAkC;QAClC,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;YAAE,SAAS;QAC1E,KAAK,CAAC,IAAI,CAAC,GAAG;YACZ,GAAI,CAAY;YAChB,eAAe,EAAE,oBAAoB,CAAC,CAAC,CAAC,eAAe,EAAE,eAAe,CAAC;SAC1E,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}