omnigateway 0.4.11 → 0.4.12

package/bin/omni.js

@@ -1931,16 +1931,51 @@ function hasVendorCacheControl(req) {
   const anthropic2 = req.vendor?.anthropic;
   return typeof anthropic2 === "object" && anthropic2 !== null && "cache_control" in anthropic2;
 }
-function addAutoCacheBreakpoint(body, req, note2) {
+var CACHEABLE_WIRE_BLOCKS = new Set(["text", "image", "tool_use", "tool_result", "document"]);
+function lastCacheableHistoryBlock(messages) {
+  for (let m = messages.length - 1;m >= 0; m--) {
+    const message = messages[m];
+    if (!isRecord(message) || !Array.isArray(message.content))
+      continue;
+    for (let b = message.content.length - 1;b >= 0; b--) {
+      const block = message.content[b];
+      if (!isRecord(block))
+        continue;
+      if (typeof block.type === "string" && CACHEABLE_WIRE_BLOCKS.has(block.type))
+        return block;
+    }
+  }
+  return;
+}
+function addAutoCacheBreakpoints(body, req, note2) {
   if (estimateCachedInputTokens(req) !== 0 || hasVendorCacheControl(req))
     return;
-  if (estimateInputTokens({ ...req, messages: [] }) < AUTO_CACHE_MIN_TOKENS)
+  let markedPrefix = 0;
+  const worthAMarker = (prefix) => prefix - markedPrefix >= AUTO_CACHE_MIN_TOKENS;
+  let stablePrefixMarked = false;
+  const toolsPrefix = estimateInputTokens({ ...req, messages: [], system: [] });
+  const lastTool = body.tools?.at(-1);
+  if (lastTool !== undefined && worthAMarker(toolsPrefix)) {
+    lastTool.cache_control = { type: "ephemeral" };
+    markedPrefix = toolsPrefix;
+    stablePrefixMarked = true;
+  }
+  const systemPrefix = estimateInputTokens({ ...req, messages: [] });
+  const lastSystem = body.system?.at(-1);
+  if (lastSystem !== undefined && worthAMarker(systemPrefix)) {
+    lastSystem.cache_control = { type: "ephemeral" };
+    markedPrefix = systemPrefix;
+    stablePrefixMarked = true;
+  }
+  if (stablePrefixMarked)
+    note2("anthropic:cache-breakpoint-added");
+  if (!worthAMarker(estimateInputTokens(req)))
     return;
-  const target = body.system?.at(-1) ?? body.tools?.at(-1);
-  if (target === undefined)
+  const lastHistory = lastCacheableHistoryBlock(body.messages);
+  if (lastHistory === undefined)
     return;
-  target.cache_control = { type: "ephemeral" };
-  note2("anthropic:cache-breakpoint-added");
+  lastHistory.cache_control = { type: "ephemeral" };
+  note2("anthropic:history-cache-breakpoint-added");
 }
 function toWire(req, model, opts) {
   const cloak = opts.cloak ?? null;
@@ -1992,7 +2027,7 @@ function toWire(req, model, opts) {
   if (req.toolChoice !== undefined)
     body.tool_choice = encodeToolChoice(req.toolChoice, cloak);
   if (opts.autoCache === true)
-    addAutoCacheBreakpoint(body, req, note2);
+    addAutoCacheBreakpoints(body, req, note2);
   if (req.reasoning !== undefined) {
     switch (req.reasoning.mode) {
       case "adaptive":

package/gateway.js

@@ -7293,16 +7293,51 @@ function hasVendorCacheControl(req) {
   const anthropic2 = req.vendor?.anthropic;
   return typeof anthropic2 === "object" && anthropic2 !== null && "cache_control" in anthropic2;
 }
-function addAutoCacheBreakpoint(body, req, note) {
+var CACHEABLE_WIRE_BLOCKS = new Set(["text", "image", "tool_use", "tool_result", "document"]);
+function lastCacheableHistoryBlock(messages) {
+  for (let m = messages.length - 1;m >= 0; m--) {
+    const message = messages[m];
+    if (!isRecord(message) || !Array.isArray(message.content))
+      continue;
+    for (let b = message.content.length - 1;b >= 0; b--) {
+      const block = message.content[b];
+      if (!isRecord(block))
+        continue;
+      if (typeof block.type === "string" && CACHEABLE_WIRE_BLOCKS.has(block.type))
+        return block;
+    }
+  }
+  return;
+}
+function addAutoCacheBreakpoints(body, req, note) {
   if (estimateCachedInputTokens(req) !== 0 || hasVendorCacheControl(req))
     return;
-  if (estimateInputTokens({ ...req, messages: [] }) < AUTO_CACHE_MIN_TOKENS)
+  let markedPrefix = 0;
+  const worthAMarker = (prefix) => prefix - markedPrefix >= AUTO_CACHE_MIN_TOKENS;
+  let stablePrefixMarked = false;
+  const toolsPrefix = estimateInputTokens({ ...req, messages: [], system: [] });
+  const lastTool = body.tools?.at(-1);
+  if (lastTool !== undefined && worthAMarker(toolsPrefix)) {
+    lastTool.cache_control = { type: "ephemeral" };
+    markedPrefix = toolsPrefix;
+    stablePrefixMarked = true;
+  }
+  const systemPrefix = estimateInputTokens({ ...req, messages: [] });
+  const lastSystem = body.system?.at(-1);
+  if (lastSystem !== undefined && worthAMarker(systemPrefix)) {
+    lastSystem.cache_control = { type: "ephemeral" };
+    markedPrefix = systemPrefix;
+    stablePrefixMarked = true;
+  }
+  if (stablePrefixMarked)
+    note("anthropic:cache-breakpoint-added");
+  if (!worthAMarker(estimateInputTokens(req)))
     return;
-  const target = body.system?.at(-1) ?? body.tools?.at(-1);
-  if (target === undefined)
+  const lastHistory = lastCacheableHistoryBlock(body.messages);
+  if (lastHistory === undefined)
     return;
-  target.cache_control = { type: "ephemeral" };
-  note("anthropic:cache-breakpoint-added");
+  lastHistory.cache_control = { type: "ephemeral" };
+  note("anthropic:history-cache-breakpoint-added");
 }
 function toWire(req, model, opts) {
   const cloak = opts.cloak ?? null;
@@ -7354,7 +7389,7 @@ function toWire(req, model, opts) {
   if (req.toolChoice !== undefined)
     body.tool_choice = encodeToolChoice(req.toolChoice, cloak);
   if (opts.autoCache === true)
-    addAutoCacheBreakpoint(body, req, note);
+    addAutoCacheBreakpoints(body, req, note);
   if (req.reasoning !== undefined) {
     switch (req.reasoning.mode) {
       case "adaptive":

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omnigateway",
-  "version": "0.4.11",
+  "version": "0.4.12",
   "description": "Self-hosted AI gateway with Anthropic- and OpenAI-compatible APIs, an admin console, and a CLI",
   "license": "MIT",
   "author": "Harismawan <mail@harismawan.com>",