omnigateway 0.4.10 → 0.4.12

package/bin/omni.js

@@ -233,6 +233,7 @@ var RETRYABLE = {
   BAD_REQUEST: false,
   CONFLICT: false,
   CONTENT_FILTER: false,
+  FINGERPRINT_REFUSED: false,
   NO_CANDIDATES: false,
   ALL_CANDIDATES_FAILED: false,
   INTERNAL: false
@@ -249,6 +250,7 @@ class GatewayError extends Error {
   provider;
   upstreamStatus;
   retryAfterMs;
+  degradations;
   constructor(code, message, opts) {
     super(message, opts?.cause === undefined ? undefined : { cause: opts.cause });
     this.name = "GatewayError";
@@ -257,6 +259,7 @@ class GatewayError extends Error {
     this.provider = opts?.provider;
     this.upstreamStatus = opts?.status;
     this.retryAfterMs = opts?.retryAfterMs;
+    this.degradations = [...opts?.degradations ?? []];
   }
 }
 // packages/ir/src/logger.ts
@@ -1290,6 +1293,82 @@ function parseRecord(record) {
 `) };
 }
 
+// packages/providers/src/anthropic/cloak.ts
+var ALREADY_CLOAKED = /^[A-Z][A-Za-z0-9]*$/;
+var MCP_PREFIX = "mcp__";
+function isExempt(name) {
+  return ALREADY_CLOAKED.test(name) || name.startsWith(MCP_PREFIX);
+}
+var MAX_ALIAS = 128;
+var SUFFIX_LENGTH = 6;
+function aliasFor(name) {
+  const joined = name.split(/[^A-Za-z0-9]+/).filter((s) => s.length > 0).map((s) => s.charAt(0).toUpperCase() + s.slice(1)).join("");
+  if (joined === "")
+    return { base: "Tool", forced: true };
+  if (joined.length > MAX_ALIAS) {
+    return { base: joined.slice(0, MAX_ALIAS - SUFFIX_LENGTH), forced: true };
+  }
+  return { base: joined, forced: false };
+}
+function clientToolNames(request2) {
+  const renamable = new Set;
+  const reserved = new Set;
+  const add = (name) => {
+    (isExempt(name) ? reserved : renamable).add(name);
+  };
+  for (const tool of request2.tools ?? []) {
+    if (tool.provider === "custom")
+      add(tool.name);
+  }
+  for (const message of request2.messages) {
+    for (const block of message.content) {
+      if (block.type === "toolUse")
+        add(block.name);
+    }
+  }
+  if (request2.toolChoice?.type === "tool")
+    add(request2.toolChoice.name);
+  return { renamable: [...renamable], reserved: [...reserved] };
+}
+function suffixFor(name) {
+  const digest = Bun.hash.xxHash64(Buffer.from(name, "utf8")) & 0xffffffn;
+  const hex = digest.toString(16).padStart(SUFFIX_LENGTH, "0");
+  return hex.charAt(0).toUpperCase() + hex.slice(1);
+}
+function buildToolCloak(request2) {
+  const { renamable, reserved } = clientToolNames(request2);
+  const claims = new Map;
+  const claim = (name) => {
+    claims.set(name, (claims.get(name) ?? 0) + 1);
+  };
+  for (const tool of request2.tools ?? []) {
+    if (tool.provider === "anthropic" && tool.name !== "")
+      claim(tool.name);
+  }
+  for (const name of reserved)
+    claim(name);
+  const bases = new Map;
+  for (const name of renamable) {
+    const derived = aliasFor(name);
+    bases.set(name, derived);
+    claim(derived.base);
+  }
+  const toWire = new Map;
+  const fromWire = new Map;
+  for (const [name, { base, forced }] of bases) {
+    const alias = forced || (claims.get(base) ?? 0) > 1 ? base + suffixFor(name) : base;
+    toWire.set(name, alias);
+    fromWire.set(alias, name);
+  }
+  return toWire.size === 0 ? null : { toWire, fromWire };
+}
+function cloakName(cloak, name) {
+  return cloak?.toWire.get(name) ?? name;
+}
+function uncloakName(cloak, name) {
+  return cloak?.fromWire.get(name) ?? name;
+}
+
 // packages/providers/src/anthropic/tools.ts
 var COMMON = ["cache_control", "strict", "defer_loading", "allowed_callers"];
 var COMMON_WITH_EXAMPLES = [...COMMON, "input_examples"];
@@ -1524,6 +1603,14 @@ var ERROR_TYPE = {
   invalid_request_error: "BAD_REQUEST",
   api_error: "UPSTREAM"
 };
+var FINGERPRINT_PHRASES = ["out of extra usage", "draw from extra usage"];
+function isFingerprintMessage(message) {
+  const lowered = message.toLowerCase();
+  return FINGERPRINT_PHRASES.some((phrase) => lowered.includes(phrase));
+}
+function isFingerprintRefusal(type, message) {
+  return type === "invalid_request_error" && isFingerprintMessage(message);
+}
 function json(data) {
   try {
     const v = JSON.parse(data);
@@ -1532,7 +1619,8 @@ function json(data) {
     return null;
   }
 }
-async function* decodeAnthropic(messages) {
+async function* decodeAnthropic(messages, opts = {}) {
+  const cloak = opts.cloak ?? null;
   let inputTokens = 0;
   let cacheReadTokens = 0;
   let cacheWriteTokens = 0;
@@ -1578,7 +1666,11 @@ async function* decodeAnthropic(messages) {
           yield {
             type: "blockStart",
             index,
-            block: { type: "toolUse", id: String(cb.id), name: String(cb.name) }
+            block: {
+              type: "toolUse",
+              id: String(cb.id),
+              name: uncloakName(cloak, String(cb.name))
+            }
           };
         else if (cb.type !== undefined && ANTHROPIC_NATIVE_BLOCK_TYPES.has(cb.type)) {
           nativeBlocks.add(index);
@@ -1681,13 +1773,10 @@ async function* decodeAnthropic(messages) {
         break;
       case "error": {
         terminal = true;
-        const code = ERROR_TYPE[String(d.error?.type)] ?? "UPSTREAM";
-        yield {
-          type: "error",
-          code,
-          message: String(d.error?.message ?? "upstream error"),
-          retryable: RETRYABLE[code]
-        };
+        const type = String(d.error?.type);
+        const message = String(d.error?.message ?? "upstream error");
+        const code = isFingerprintRefusal(type, message) ? "FINGERPRINT_REFUSED" : ERROR_TYPE[type] ?? "UPSTREAM";
+        yield { type: "error", code, message, retryable: RETRYABLE[code] };
         break;
       }
       default:
@@ -1714,7 +1803,7 @@ function wireCacheControl(c) {
 function isReplayableThinking(b) {
   return b.type !== "thinking" || b.signature !== undefined && b.signature !== "";
 }
-function encodeBlock(b) {
+function encodeBlock(b, cloak) {
   const cache = wireCacheControl(cacheControlOf(b));
   switch (b.type) {
     case "text":
@@ -1733,7 +1822,13 @@ function encodeBlock(b) {
     case "thinking":
       return { type: "thinking", thinking: b.text, signature: b.signature };
     case "toolUse":
-      return { type: "tool_use", id: b.id, name: b.name, input: b.input, ...cache };
+      return {
+        type: "tool_use",
+        id: b.id,
+        name: cloakName(cloak, b.name),
+        input: b.input,
+        ...cache
+      };
     case "toolResult":
       return {
         type: "tool_result",
@@ -1746,12 +1841,12 @@ function encodeBlock(b) {
       return { ...b.data, type: b.blockType, ...cache };
   }
 }
-function encodeSystemTurn(content) {
+function encodeSystemTurn(content, cloak) {
   if (content.every((block) => block.type === "text")) {
     return content.map((block) => block.text).join(`
 `);
   }
-  return content.map(encodeBlock);
+  return content.map((block) => encodeBlock(block, cloak));
 }
 function systemCacheControl(req) {
   const cacheable = req.messages.flatMap((message) => message.content.flatMap((block) => block.type === "thinking" ? [] : [{ role: message.role, block }]));
@@ -1763,7 +1858,7 @@ function systemCacheControl(req) {
     lost: markedSystemBlocks.length > (promoted === undefined ? 0 : 1)
   };
 }
-function encodeTool(t) {
+function encodeTool(t, cloak) {
   if (t.provider === "anthropic") {
     return {
       type: t.type,
@@ -1773,14 +1868,14 @@ function encodeTool(t) {
     };
   }
   return {
-    name: t.name,
+    name: cloakName(cloak, t.name),
     ...t.description === undefined ? {} : { description: t.description },
     input_schema: t.inputSchema,
     ...t.options ?? {},
     ...wireCacheControl(t.cacheControl)
   };
 }
-function encodeToolChoice(c) {
+function encodeToolChoice(c, cloak) {
   switch (c.type) {
     case "auto":
       return { type: "auto" };
@@ -1789,7 +1884,7 @@ function encodeToolChoice(c) {
     case "none":
       return { type: "none" };
     case "tool":
-      return { type: "tool", name: c.name };
+      return { type: "tool", name: cloakName(cloak, c.name) };
   }
 }
 function isRecord(value) {
@@ -1831,7 +1926,59 @@ function stripUnsupportedEdits(body, note2) {
   }
   body.context_management = { ...config, edits: kept };
 }
+var AUTO_CACHE_MIN_TOKENS = 1024;
+function hasVendorCacheControl(req) {
+  const anthropic2 = req.vendor?.anthropic;
+  return typeof anthropic2 === "object" && anthropic2 !== null && "cache_control" in anthropic2;
+}
+var CACHEABLE_WIRE_BLOCKS = new Set(["text", "image", "tool_use", "tool_result", "document"]);
+function lastCacheableHistoryBlock(messages) {
+  for (let m = messages.length - 1;m >= 0; m--) {
+    const message = messages[m];
+    if (!isRecord(message) || !Array.isArray(message.content))
+      continue;
+    for (let b = message.content.length - 1;b >= 0; b--) {
+      const block = message.content[b];
+      if (!isRecord(block))
+        continue;
+      if (typeof block.type === "string" && CACHEABLE_WIRE_BLOCKS.has(block.type))
+        return block;
+    }
+  }
+  return;
+}
+function addAutoCacheBreakpoints(body, req, note2) {
+  if (estimateCachedInputTokens(req) !== 0 || hasVendorCacheControl(req))
+    return;
+  let markedPrefix = 0;
+  const worthAMarker = (prefix) => prefix - markedPrefix >= AUTO_CACHE_MIN_TOKENS;
+  let stablePrefixMarked = false;
+  const toolsPrefix = estimateInputTokens({ ...req, messages: [], system: [] });
+  const lastTool = body.tools?.at(-1);
+  if (lastTool !== undefined && worthAMarker(toolsPrefix)) {
+    lastTool.cache_control = { type: "ephemeral" };
+    markedPrefix = toolsPrefix;
+    stablePrefixMarked = true;
+  }
+  const systemPrefix = estimateInputTokens({ ...req, messages: [] });
+  const lastSystem = body.system?.at(-1);
+  if (lastSystem !== undefined && worthAMarker(systemPrefix)) {
+    lastSystem.cache_control = { type: "ephemeral" };
+    markedPrefix = systemPrefix;
+    stablePrefixMarked = true;
+  }
+  if (stablePrefixMarked)
+    note2("anthropic:cache-breakpoint-added");
+  if (!worthAMarker(estimateInputTokens(req)))
+    return;
+  const lastHistory = lastCacheableHistoryBlock(body.messages);
+  if (lastHistory === undefined)
+    return;
+  lastHistory.cache_control = { type: "ephemeral" };
+  note2("anthropic:history-cache-breakpoint-added");
+}
 function toWire(req, model, opts) {
+  const cloak = opts.cloak ?? null;
   const degradations = [];
   const note2 = (d) => {
     if (!degradations.includes(d))
@@ -1842,6 +1989,8 @@ function toWire(req, model, opts) {
     system = [{ type: "text", text: OAUTH_IDENTITY }, ...system ?? []];
     note2("anthropic:oauth-system-prefix");
   }
+  if (cloak !== null)
+    note2("anthropic:tool-names-cloaked");
   const systemCache = systemCacheControl(req);
   if (systemCache.lost)
     note2("anthropic:system-turn-cache-control-dropped");
@@ -1854,9 +2003,9 @@ function toWire(req, model, opts) {
           note2("anthropic:unsigned-thinking-dropped");
         if (replayable.length === 0)
           return [];
-        return [{ role: m.role, content: replayable.map(encodeBlock) }];
+        return [{ role: m.role, content: replayable.map((b) => encodeBlock(b, cloak)) }];
       }
-      return [{ role: m.role, content: encodeSystemTurn(m.content) }];
+      return [{ role: m.role, content: encodeSystemTurn(m.content, cloak) }];
     }),
     max_tokens: req.maxTokens ?? 4096,
     stream: req.stream,
@@ -1874,9 +2023,11 @@ function toWire(req, model, opts) {
   if (req.stopSequences !== undefined)
     body.stop_sequences = req.stopSequences;
   if (req.tools !== undefined)
-    body.tools = req.tools.map(encodeTool);
+    body.tools = req.tools.map((t) => encodeTool(t, cloak));
   if (req.toolChoice !== undefined)
-    body.tool_choice = encodeToolChoice(req.toolChoice);
+    body.tool_choice = encodeToolChoice(req.toolChoice, cloak);
+  if (opts.autoCache === true)
+    addAutoCacheBreakpoints(body, req, note2);
   if (req.reasoning !== undefined) {
     switch (req.reasoning.mode) {
       case "adaptive":
@@ -1919,7 +2070,12 @@ var anthropicAdapter = {
   capabilities: PROVIDER_CAPABILITIES.anthropic,
   async send(req) {
     const oauth = req.credentials.accessToken !== null;
-    const { body, degradations } = toWire(req.request, req.model, { oauth });
+    const cloak = oauth ? buildToolCloak(req.request) : null;
+    const { body, degradations } = toWire(req.request, req.model, {
+      oauth,
+      cloak,
+      autoCache: req.autoCache === true
+    });
     const withSystem = {
       ...body,
       system: applyAnthropicSystem(body.system ?? []),
@@ -1963,11 +2119,25 @@ var anthropicAdapter = {
       body: bodyString,
       signal: req.signal
     });
-    if (res.status < 200 || res.status >= 300)
-      throw await httpError(res, "anthropic");
+    if (res.status < 200 || res.status >= 300) {
+      const error = await httpError(res, "anthropic");
+      if (res.status === 400 && isFingerprintMessage(error.message)) {
+        throw new GatewayError("FINGERPRINT_REFUSED", error.message, {
+          provider: "anthropic",
+          degradations: notes,
+          ...error.upstreamStatus === undefined ? {} : { status: error.upstreamStatus },
+          ...error.retryAfterMs === undefined ? {} : { retryAfterMs: error.retryAfterMs }
+        });
+      }
+      throw error;
+    }
     if (res.body === null)
       throw new GatewayError("UPSTREAM", "empty response body", { provider: "anthropic" });
-    return { events: decodeAnthropic(parseSse(res.body)), degradations: notes };
+    return {
+      events: decodeAnthropic(parseSse(res.body), { cloak }),
+      degradations: notes,
+      ...cloak === null ? {} : { cloakedTools: cloak.toWire.size }
+    };
   }
 };
 // packages/providers/src/kimi/decode.ts
@@ -17998,6 +18168,7 @@ var settingsSchema = exports_external.object({
   logRetentionDays: exports_external.number().int().min(1),
   quotaPollIntervalMs: exports_external.number().int().min(0),
   rtkEnabled: exports_external.boolean(),
+  autoCacheEnabled: exports_external.boolean(),
   bodyLoggingEnabled: exports_external.boolean(),
   bodyLoggingCaptureStreamChunks: exports_external.boolean(),
   snapshotKeepLatest: retentionSchema.shape.keepLatest.optional(),
@@ -18225,6 +18396,7 @@ var DEFAULT_SETTINGS = {
   logRetentionDays: 30,
   quotaPollIntervalMs: 300000,
   rtkEnabled: false,
+  autoCacheEnabled: true,
   bodyLoggingEnabled: false,
   bodyLoggingCaptureStreamChunks: false,
   snapshotKeepLatest: 5,
@@ -18821,6 +18993,7 @@ function createConfigRepo(db, emit2 = () => {}) {
           ...DEFAULT_SETTINGS,
           ...stored,
           rtkEnabled: stored.rtkEnabled === true,
+          autoCacheEnabled: stored.autoCacheEnabled === undefined ? true : stored.autoCacheEnabled === true,
           bodyLoggingEnabled: stored.bodyLoggingEnabled === true,
           bodyLoggingCaptureStreamChunks: stored.bodyLoggingCaptureStreamChunks === true,
           weights: knownWeights(stored.weights)