@columenlabs/lumen-ai 0.1.201 → 0.1.202

package/dist/agent/agent.js

@@ -16,7 +16,7 @@ import { Think, } from '@cloudflare/think';
 import { createCompactFunction } from 'agents/sessions';
 import { generateText } from 'ai';
 import { CONNECTOR_GROUP, checkConnectorUrl, connectorGroup, connectorId, connectorSource, } from './connectors.js';
-import { aiModel, planFallback, turnSettings } from './models.js';
+import { aiModel, cachedTurn, planFallback, turnSettings } from './models.js';
 import { screenMapOf, screenNews, screenOutcomesOf, screenRules, screenTool, } from './screen.js';
 import { LAZY_AT, TOOL_SEARCH, lazyTools } from './search.js';
 import { chunkEvents, itemsOf } from './stream.js';
@@ -522,6 +522,7 @@ export class LumenAgent extends Think {
             this.#turn.lazy = lazy;
         }
         const turn = this.#turn;
+        const cached = cachedTurn(system, withLead(ctx.messages, `${this.#news()}${this.#screenNews()}${plan.lead ?? ''}`));
         return {
             model: this.#model(plan, () => {
                 const fb = plan.fallback;
@@ -531,8 +532,9 @@ export class LumenAgent extends Think {
                 turn.main = new UsageMeter(fb.model, fb.label, false);
                 turn.meters.push(turn.main);
             }),
-            instructions: system,
-            messages: withLead(ctx.messages, `${this.#news()}${this.#screenNews()}${plan.lead ?? ''}`),
+            // Think types instructions as a string; the SDK it hands them to takes a system message too.
+            instructions: cached.instructions,
+            messages: cached.messages,
             tools,
             ...(lazy ? { activeTools: lazy.active() } : {}),
             ...(plan.first ? { toolChoice: { type: 'tool', toolName: plan.first } } : {}),

package/dist/agent/models.d.ts

@@ -1,5 +1,5 @@
 import type { AssistantEffort } from '@columenlabs/lumen-ui-types/assistant';
-import type { JSONValue, LanguageModel } from 'ai';
+import type { JSONValue, LanguageModel, ModelMessage, SystemModelMessage } from 'ai';
 import type { ModelDef } from '../assistant/models.js';
 /** The key a model runs on. `plan`: an OpenAI access token from a ChatGPT plan sign-in. */
 export interface ModelKey {
@@ -24,6 +24,14 @@ export interface AiModelOptions {
 export declare function planFetch(base?: typeof fetch, name?: string): typeof fetch;
 /** The AI SDK model a tier runs: its provider's client on the payer's key. */
 export declare function aiModel(model: ModelDef, key: ModelKey, o?: AiModelOptions): LanguageModel;
+/**
+ * A turn's instructions and messages with Claude's two lasting cache breakpoints: the instructions
+ * as a system message that carries one, and the message before the newest person's message.
+ */
+export declare function cachedTurn(instructions: string, messages: ModelMessage[]): {
+    instructions: SystemModelMessage;
+    messages: ModelMessage[];
+};
 /** A tier's effort and output cap as a turn's settings, with Claude's prompt cache on. */
 export declare function turnSettings(model: ModelDef, o: {
     effort: AssistantEffort;

package/dist/agent/models.js

@@ -88,12 +88,40 @@ export function aiModel(model, key, o = {}) {
     return createOpenAI({ apiKey: key.apiKey, ...(f ? { fetch: f } : {}) }).responses(model.id);
 }
 /**
- * Claude's prompt cache: one breakpoint on the request, which Claude moves to the newest block on
- * each call, so a turn's later steps and the next turn read the tools, instructions and chat so
- * far from cache (a tenth of the input price) instead of paying for them again. Sent whatever the
- * tier's provider, so a plan's fallback to Claude caches too; other providers ignore it.
+ * Claude's prompt cache. The request's own breakpoint follows the newest block, so a turn's later
+ * steps read the steps before them. It alone never helps the next turn: the bearings put before
+ * the newest message are gone from it by then, so that prefix never comes again. `cachedTurn`
+ * adds two breakpoints that do: after the instructions (tools and instructions) and on the last
+ * message before the newest question (the chat so far). Sent whatever the tier's provider, so a
+ * plan's fallback to Claude caches too; other providers ignore it.
  */
 const CLAUDE_CACHE = { cacheControl: { type: 'ephemeral' } };
+const CACHED = { anthropic: CLAUDE_CACHE };
+/**
+ * A turn's instructions and messages with Claude's two lasting cache breakpoints: the instructions
+ * as a system message that carries one, and the message before the newest person's message.
+ */
+export function cachedTurn(instructions, messages) {
+    let i = messages.length - 1;
+    while (i >= 0 && messages[i]?.role !== 'user')
+        i -= 1;
+    const before = i - 1;
+    const marked = before >= 0
+        ? messages.map((m, n) => n === before
+            ? {
+                ...m,
+                providerOptions: {
+                    ...m.providerOptions,
+                    anthropic: { ...m.providerOptions?.anthropic, ...CLAUDE_CACHE },
+                },
+            }
+            : m)
+        : messages;
+    return {
+        instructions: { role: 'system', content: instructions, providerOptions: CACHED },
+        messages: marked,
+    };
+}
 /** A tier's effort and output cap as a turn's settings, with Claude's prompt cache on. */
 export function turnSettings(model, o) {
     const out = o.plan || !o.out ? {} : { maxOutputTokens: o.out };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@columenlabs/lumen-ai",
-  "version": "0.1.201",
+  "version": "0.1.202",
   "description": "Lumen's assistant and agent parts: a chat assistant's server (Claude tiers, tools over MCP, the web, hand off, spend caps), serving an MCP server from a Worker, read-only tool sets and metered tool calls.",
   "license": "MIT",
   "type": "module",
@@ -52,8 +52,8 @@
     "@ai-sdk/openai": "4.0.108",
     "@anthropic-ai/sdk": "0.133.0",
     "@cloudflare/think": "0.23.0",
-    "@columenlabs/lumen-ops": "0.1.201",
-    "@columenlabs/lumen-ui-types": "0.1.201",
+    "@columenlabs/lumen-ops": "0.1.202",
+    "@columenlabs/lumen-ui-types": "0.1.202",
     "agents": "0.32.0",
     "ai": "7.0.162",
     "just-bash": "3.6.0",