@zerowidth/workbench-sdk 2.5.0 → 2.6.0

package/nodes/system-prompt/system-prompt.process.js

@@ -56,30 +56,39 @@ export default async ({inputs, settings, config}) => {
   // Combine chained content with base content
   let fullContent = chainedContent ? `${chainedContent}\n\n${baseContent}` : baseContent;
 
-  // Create message object
-  let message = {
-    role: "system",
-    content: [
-      {
-        type: "text",
-        text: fullContent
+  const fill = (text) =>
+    text.replace(/\{\{(.*?)\}\}/g, (match, p1) => {
+      // look for a variable with the key p1
+      let variable = variables.find(variable => Object.keys(variable).find(key => key === p1));
+      if(variable) {
+        return renderVariable(variable[p1]);
       }
-    ]
-  };
+      return match;
+    });
 
-  // Process variables
-  message.content[0].text = message.content[0].text.replace(/\{\{(.*?)\}\}/g, (match, p1) => {
-    // look for a variable with the key p1
-    let variable = variables.find(variable => Object.keys(variable).find(key => key === p1));
-    if(variable) {
-      return renderVariable(variable[p1]);
+  // Values filled in can change from run to run (the time, memory, search
+  // results); the text before the first of them doesn't. The block records
+  // that length as `cache_prefix_length`, so the model client can cache
+  // the prompt up to there for a model that caches on request. The client
+  // removes the hint before any model sees it.
+  let firstFilled = -1;
+  for (const m of fullContent.matchAll(/\{\{(.*?)\}\}/g)) {
+    if (variables.some(variable => Object.keys(variable).includes(m[1]))) {
+      firstFilled = m.index;
+      break;
     }
-    return match;
-  });
-  
+  }
+
+  const text = firstFilled < 0 ? fullContent : fullContent.slice(0, firstFilled) + fill(fullContent.slice(firstFilled));
+  const prefix = firstFilled < 0 ? text.length : firstFilled;
+  const message = {
+    role: "system",
+    content: [{ type: "text", text, ...(text.slice(0, prefix).trim() ? { cache_prefix_length: prefix } : {}) }],
+  };
+
   // Return the message and string prompt
   return {
     message: message,
-    prompt: message.content[0].text
+    prompt: text
   };
 }; 

package/nodes/truncate-old-tool-responses/truncate-old-tool-responses.config.json

@@ -20,6 +20,14 @@
       "required": false,
       "default": 20
     },
+    {
+      "name": "step",
+      "display_name": "Step",
+      "type": "number",
+      "description": "Move the cutoff only in steps of this many messages, so earlier messages stay the same from turn to turn and a model's prompt cache keeps matching them. 1 moves it every message.",
+      "required": false,
+      "default": 1
+    },
     {
       "name": "placeholder",
       "display_name": "Placeholder",

package/nodes/truncate-old-tool-responses/truncate-old-tool-responses.process.js

@@ -1,14 +1,17 @@
 export default async ({ inputs, settings, config }) => {
   const messages = inputs.messages;
   const keepRecent = Math.max(0, Math.floor(Number(inputs.keep_recent ?? 20)));
+  const step = Math.max(1, Math.floor(Number(inputs.step ?? 1)) || 1);
   const placeholder = inputs.placeholder ?? "[Truncated]";
 
   if (!Array.isArray(messages)) {
     throw new Error("Messages input must be an array");
   }
 
+  // The cutoff moves in whole steps, so between steps the earlier
+  // messages are byte-for-byte what they were last turn (cacheable).
   const total = messages.length;
-  const cutoffIndex = total - keepRecent;
+  const cutoffIndex = Math.floor(Math.max(0, total - keepRecent) / step) * step;
 
   const result = [];
   let truncatedCount = 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zerowidth/workbench-sdk",
-  "version": "2.5.0",
+  "version": "2.6.0",
   "dependencies": {
     "adm-zip": "^0.5.16",
     "ajv": "^8.17.1",

package/src/integrations/openrouter.js

@@ -1,5 +1,19 @@
 import OpenAI, { AzureOpenAI } from 'openai';
 import { emitAPICallEvent } from '../utilities/sanitizeAPICall.js';
+import { applyPromptCache, stripCacheHints } from '../utilities/promptCache.js';
+
+/**
+ * Prompt-cache counts from a usage block, in OpenAI's shape
+ * (`prompt_tokens_details`), which OpenRouter normalizes every provider
+ * to. Both are part of `prompt_tokens`, not on top of it.
+ */
+export function readCacheUsage(usage) {
+    const details = usage?.prompt_tokens_details || {};
+    return {
+        cached_tokens: Number(details.cached_tokens) || 0,
+        cache_write_tokens: Number(details.cache_write_tokens) || 0,
+    };
+}
 
 export default class OpenRouterIntegration {
     constructor(apiKey, options = {}) {
@@ -72,6 +86,15 @@ export default class OpenRouterIntegration {
               return clean;
             });
 
+            // Prompt-cache marks for models that need them (promptCache.js).
+            // Only the platform endpoint understands them; the SDK's own
+            // cache flags are removed for every endpoint.
+            const cacheConfig = engineConfig || this._engineConfig;
+            payload.messages = applyPromptCache(payload.messages, {
+                model,
+                enabled: this.dialect === 'openrouter' && cacheConfig?.promptCache !== false,
+            });
+
         } else if (prompt) {
             payload.prompt = prompt;
         } else {
@@ -151,7 +174,11 @@ export default class OpenRouterIntegration {
             let usage = {
               prompt_tokens: 0,
               completion_tokens: 0,
-              total_tokens: 0
+              total_tokens: 0,
+              // Of prompt_tokens: read from the provider's prompt cache, and
+              // written to it. Both 0 when the provider reports neither.
+              cached_tokens: 0,
+              cache_write_tokens: 0
             }
 
             // Authoritative cost from OpenRouter usage accounting (final usage chunk)
@@ -240,6 +267,9 @@ export default class OpenRouterIntegration {
                     usage.prompt_tokens += chunk.usage.prompt_tokens || 0;
                     usage.completion_tokens += chunk.usage.completion_tokens || 0;
                     usage.total_tokens += chunk.usage.total_tokens || 0;
+                    const cache = readCacheUsage(chunk.usage);
+                    usage.cached_tokens += cache.cached_tokens;
+                    usage.cache_write_tokens += cache.cache_write_tokens;
                     if (typeof chunk.usage.cost === 'number') apiCost = chunk.usage.cost;
                     if (chunk.usage.cost_details) apiCostDetails = chunk.usage.cost_details;
                   }
@@ -413,7 +443,9 @@ export default class OpenRouterIntegration {
         const data = await res.json();
         const choice = data.choices?.[0];
         const message = choice?.message || {};
-        const usage = data.usage || { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 };
+        const usage = data.usage
+            ? { ...data.usage, ...readCacheUsage(data.usage) }
+            : { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0, cached_tokens: 0, cache_write_tokens: 0 };
 
         let costData = null;
         if (nodeConfig) {
@@ -547,10 +579,20 @@ export default class OpenRouterIntegration {
             outputCost = 0;
         }
 
+        // How much of the input came from (or went into) the provider's
+        // prompt cache. The total above already reflects what that cost.
+        const cachedTokens = usage.cached_tokens || 0;
+        const cacheWriteTokens = usage.cache_write_tokens || 0;
         return {
             totalCost: Number(totalCost.toFixed(8)),
             itemizedCosts: [
-                { label: "Input Tokens", cost: Number(inputCost.toFixed(8)), tokens: promptTokens },
+                {
+                    label: "Input Tokens",
+                    cost: Number(inputCost.toFixed(8)),
+                    tokens: promptTokens,
+                    ...(cachedTokens > 0 && { cached_tokens: cachedTokens }),
+                    ...(cacheWriteTokens > 0 && { cache_write_tokens: cacheWriteTokens })
+                },
                 { label: "Output Tokens", cost: Number(outputCost.toFixed(8)), tokens: completionTokens }
             ]
         };
@@ -803,7 +845,8 @@ export default class OpenRouterIntegration {
             throw new Error('questions must be a non-empty object mapping question ids to { type, instructions, criteria }');
         }
 
-        const payload = { model, state, questions };
+        // A conversation's cache hints are for chat models only.
+        const payload = { model, state: stripCacheHints(state), questions };
 
         const url = `${this.client.baseURL}/systemone`;
         const headers = {

package/src/utilities/promptCache.js

@@ -0,0 +1,142 @@
+/**
+ * Prompt caching for models that need to be told where to cache.
+ *
+ * Anthropic models cache a request's prefix only up to blocks marked
+ * `cache_control: { type: "ephemeral" }`, at most four per request.
+ * Everything before a mark (tools, then the system prompt, then the
+ * messages) is reused by the next call that starts the same way, read
+ * at a fraction of the input price. Other providers OpenRouter serves
+ * cache a repeated prefix on their own.
+ *
+ * Where marks go, for a model that takes them:
+ *   - the end of the system prompt's fixed part. The System Prompt node
+ *     records it as `cache_prefix_length` on its text block (the text
+ *     before the first variable it fills in); the block is split there
+ *     and the first part marked, so a value that changes per run (the
+ *     time, memory, search results) doesn't spoil the cache;
+ *   - marks the caller placed itself (`cache_control` on a block);
+ *   - the last message, whatever its role, so everything so far
+ *     (earlier turns, and this turn's tool calls and results) is reused
+ *     by the next call.
+ * At most four: the earliest three are kept, then the last message.
+ * A mark that comes before a one-hour mark gets the one-hour lifetime
+ * too, since Anthropic needs longer-lived marks to come first.
+ *
+ * For any other model, or with caching off, the hint and every mark are
+ * removed, and the system prompt goes out exactly as it was written.
+ */
+
+const MAX_MARKS = 4;
+const MARK = { type: "ephemeral" };
+
+/** Models that cache only where they're told to. */
+export function takesCacheMarks(model) {
+  return typeof model === "string" && model.startsWith("anthropic/");
+}
+
+const isText = (b) => b && typeof b === "object" && b.type === "text" && typeof b.text === "string";
+
+function strip(block) {
+  if (!block || typeof block !== "object") return block;
+  if (!("cache_control" in block) && !("cache_prefix_length" in block)) return block;
+  const { cache_control, cache_prefix_length, ...rest } = block;
+  return rest;
+}
+
+/** One message's blocks, with the System Prompt node's hint turned into
+ *  a split and a mark. */
+function splitAtPrefix(blocks) {
+  return blocks.flatMap((b) => {
+    if (!isText(b) || typeof b.cache_prefix_length !== "number") return [b];
+    const { cache_prefix_length: at, ...block } = b;
+    if (at <= 0 || !b.text.slice(0, at).trim()) return [block];
+    if (at >= b.text.length) return [{ ...block, cache_control: block.cache_control ?? MARK }];
+    return [
+      { type: "text", text: b.text.slice(0, at), cache_control: MARK },
+      { ...block, text: b.text.slice(at) },
+    ];
+  });
+}
+
+const stripMessage = (m) =>
+  m && typeof m === "object" && Array.isArray(m.content) ? { ...m, content: m.content.map(strip) } : m;
+
+/**
+ * A message or conversation with every cache mark and hint removed, for
+ * anything that sends one somewhere other than a chat model (a decision
+ * model's `state`). Anything else passes through. Never mutates the input.
+ */
+export function stripCacheHints(value) {
+  return Array.isArray(value) ? value.map(stripMessage) : stripMessage(value);
+}
+
+/**
+ * The messages to send, with cache marks placed for a model that takes
+ * them and removed for any other. Never mutates the input.
+ */
+export function applyPromptCache(messages, { model, enabled = true } = {}) {
+  if (!Array.isArray(messages)) return messages;
+  const marking = enabled && takesCacheMarks(model);
+
+  if (!marking) {
+    return messages.map(stripMessage);
+  }
+
+  // Split the hinted blocks, then keep the earliest three marks.
+  let kept = 0;
+  const out = messages.map((m) => {
+    if (!m || !Array.isArray(m.content)) return m;
+    const blocks = splitAtPrefix(m.content).map((b) => {
+      if (!b || !b.cache_control) return b;
+      // A mark on an empty block is refused by the provider.
+      if (kept < MAX_MARKS - 1 && !(isText(b) && !b.text)) {
+        kept++;
+        return b;
+      }
+      return strip(b);
+    });
+    return { ...m, content: blocks };
+  });
+
+  // The last message's last text block.
+  const last = out.length - 1;
+  const m = out[last];
+  if (m && m.role !== "system") {
+    const blocks =
+      typeof m.content === "string"
+        ? [{ type: "text", text: m.content }]
+        : Array.isArray(m.content)
+          ? [...m.content]
+          : null;
+    if (blocks && !blocks.some((b) => b && b.cache_control)) {
+      for (let j = blocks.length - 1; j >= 0; j--) {
+        if (isText(blocks[j]) && blocks[j].text) {
+          blocks[j] = { ...blocks[j], cache_control: MARK };
+          out[last] = { ...m, content: blocks };
+          break;
+        }
+      }
+    }
+  }
+
+  // Longer-lived marks must come first: anything before a one-hour mark
+  // lives an hour too.
+  let lastHourAt = -1;
+  out.forEach((msg, i) => {
+    if (Array.isArray(msg?.content) && msg.content.some((b) => b?.cache_control?.ttl === "1h")) lastHourAt = i;
+  });
+  if (lastHourAt > 0) {
+    for (let i = 0; i < lastHourAt; i++) {
+      const msg = out[i];
+      if (!Array.isArray(msg?.content)) continue;
+      if (!msg.content.some((b) => b?.cache_control && b.cache_control.ttl !== "1h")) continue;
+      out[i] = {
+        ...msg,
+        content: msg.content.map((b) =>
+          b?.cache_control && b.cache_control.ttl !== "1h" ? { ...b, cache_control: { ...b.cache_control, ttl: "1h" } } : b,
+        ),
+      };
+    }
+  }
+  return out;
+}