@columenlabs/lumen-ai 0.1.200 → 0.1.202

package/dist/agent/agent.js

@@ -16,7 +16,7 @@ import { Think, } from '@cloudflare/think';
 import { createCompactFunction } from 'agents/sessions';
 import { generateText } from 'ai';
 import { CONNECTOR_GROUP, checkConnectorUrl, connectorGroup, connectorId, connectorSource, } from './connectors.js';
-import { aiModel, planFallback, turnSettings } from './models.js';
+import { aiModel, cachedTurn, planFallback, turnSettings } from './models.js';
 import { screenMapOf, screenNews, screenOutcomesOf, screenRules, screenTool, } from './screen.js';
 import { LAZY_AT, TOOL_SEARCH, lazyTools } from './search.js';
 import { chunkEvents, itemsOf } from './stream.js';
@@ -79,6 +79,7 @@ export function chatUsage(parts, context) {
                 read: was.read + p.read,
                 out: was.out + p.out,
                 usd: was.usd + p.usd,
+                ...(was.runs || p.runs ? { runs: (was.runs ?? 0) + (p.runs ?? 0) } : {}),
             }
             : { ...p });
     }
@@ -521,6 +522,7 @@ export class LumenAgent extends Think {
             this.#turn.lazy = lazy;
         }
         const turn = this.#turn;
+        const cached = cachedTurn(system, withLead(ctx.messages, `${this.#news()}${this.#screenNews()}${plan.lead ?? ''}`));
         return {
             model: this.#model(plan, () => {
                 const fb = plan.fallback;
@@ -530,8 +532,9 @@ export class LumenAgent extends Think {
                 turn.main = new UsageMeter(fb.model, fb.label, false);
                 turn.meters.push(turn.main);
             }),
-            instructions: system,
-            messages: withLead(ctx.messages, `${this.#news()}${this.#screenNews()}${plan.lead ?? ''}`),
+            // Think types instructions as a string; the SDK it hands them to takes a system message too.
+            instructions: cached.instructions,
+            messages: cached.messages,
             tools,
             ...(lazy ? { activeTools: lazy.active() } : {}),
             ...(plan.first ? { toolChoice: { type: 'tool', toolName: plan.first } } : {}),

package/dist/agent/models.d.ts

@@ -1,5 +1,5 @@
 import type { AssistantEffort } from '@columenlabs/lumen-ui-types/assistant';
-import type { JSONValue, LanguageModel } from 'ai';
+import type { JSONValue, LanguageModel, ModelMessage, SystemModelMessage } from 'ai';
 import type { ModelDef } from '../assistant/models.js';
 /** The key a model runs on. `plan`: an OpenAI access token from a ChatGPT plan sign-in. */
 export interface ModelKey {
@@ -24,6 +24,14 @@ export interface AiModelOptions {
 export declare function planFetch(base?: typeof fetch, name?: string): typeof fetch;
 /** The AI SDK model a tier runs: its provider's client on the payer's key. */
 export declare function aiModel(model: ModelDef, key: ModelKey, o?: AiModelOptions): LanguageModel;
+/**
+ * A turn's instructions and messages with Claude's two lasting cache breakpoints: the instructions
+ * as a system message that carries one, and the message before the newest person's message.
+ */
+export declare function cachedTurn(instructions: string, messages: ModelMessage[]): {
+    instructions: SystemModelMessage;
+    messages: ModelMessage[];
+};
 /** A tier's effort and output cap as a turn's settings, with Claude's prompt cache on. */
 export declare function turnSettings(model: ModelDef, o: {
     effort: AssistantEffort;

package/dist/agent/models.js

@@ -88,12 +88,40 @@ export function aiModel(model, key, o = {}) {
     return createOpenAI({ apiKey: key.apiKey, ...(f ? { fetch: f } : {}) }).responses(model.id);
 }
 /**
- * Claude's prompt cache: one breakpoint on the request, which Claude moves to the newest block on
- * each call, so a turn's later steps and the next turn read the tools, instructions and chat so
- * far from cache (a tenth of the input price) instead of paying for them again. Sent whatever the
- * tier's provider, so a plan's fallback to Claude caches too; other providers ignore it.
+ * Claude's prompt cache. The request's own breakpoint follows the newest block, so a turn's later
+ * steps read the steps before them. It alone never helps the next turn: the bearings put before
+ * the newest message are gone from it by then, so that prefix never comes again. `cachedTurn`
+ * adds two breakpoints that do: after the instructions (tools and instructions) and on the last
+ * message before the newest question (the chat so far). Sent whatever the tier's provider, so a
+ * plan's fallback to Claude caches too; other providers ignore it.
  */
 const CLAUDE_CACHE = { cacheControl: { type: 'ephemeral' } };
+const CACHED = { anthropic: CLAUDE_CACHE };
+/**
+ * A turn's instructions and messages with Claude's two lasting cache breakpoints: the instructions
+ * as a system message that carries one, and the message before the newest person's message.
+ */
+export function cachedTurn(instructions, messages) {
+    let i = messages.length - 1;
+    while (i >= 0 && messages[i]?.role !== 'user')
+        i -= 1;
+    const before = i - 1;
+    const marked = before >= 0
+        ? messages.map((m, n) => n === before
+            ? {
+                ...m,
+                providerOptions: {
+                    ...m.providerOptions,
+                    anthropic: { ...m.providerOptions?.anthropic, ...CLAUDE_CACHE },
+                },
+            }
+            : m)
+        : messages;
+    return {
+        instructions: { role: 'system', content: instructions, providerOptions: CACHED },
+        messages: marked,
+    };
+}
 /** A tier's effort and output cap as a turn's settings, with Claude's prompt cache on. */
 export function turnSettings(model, o) {
     const out = o.plan || !o.out ? {} : { maxOutputTokens: o.out };

package/dist/agent/usage.d.ts

@@ -32,5 +32,7 @@ export declare class UsageMeter {
         read: number;
         usd: number;
     };
+    /** Marks this part as one hand-off to a higher tier; parts of the same label add up their runs. */
+    handOff(): this;
     part(): UsagePart;
 }

package/dist/agent/usage.js

@@ -40,6 +40,11 @@ export class UsageMeter {
             this.p.usd += t.usd;
         return t;
     }
+    /** Marks this part as one hand-off to a higher tier; parts of the same label add up their runs. */
+    handOff() {
+        this.p.runs = 1;
+        return this;
+    }
     part() {
         return { ...this.p };
     }

package/dist/assistant/engine.js

@@ -426,7 +426,7 @@ export class Assistant {
                     return {
                         error: `${tier.label} runs on ${provider} and no ${provider} key is set for whoever pays this answer: answer yourself.`,
                     };
-                const meter = tt.meter(sub.model, sub.meter.label, !!key.plan);
+                const meter = tt.meter(sub.model, sub.meter.label, !!key.plan).handOff();
                 c.counts.handoffs += 1;
                 if (this.tiers.indexOf(tier) > this.tiers.indexOf(c.counts.end))
                     c.counts.end = tier;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@columenlabs/lumen-ai",
-  "version": "0.1.200",
+  "version": "0.1.202",
   "description": "Lumen's assistant and agent parts: a chat assistant's server (Claude tiers, tools over MCP, the web, hand off, spend caps), serving an MCP server from a Worker, read-only tool sets and metered tool calls.",
   "license": "MIT",
   "type": "module",
@@ -52,8 +52,8 @@
     "@ai-sdk/openai": "4.0.108",
     "@anthropic-ai/sdk": "0.133.0",
     "@cloudflare/think": "0.23.0",
-    "@columenlabs/lumen-ops": "0.1.200",
-    "@columenlabs/lumen-ui-types": "0.1.200",
+    "@columenlabs/lumen-ops": "0.1.202",
+    "@columenlabs/lumen-ui-types": "0.1.202",
     "agents": "0.32.0",
     "ai": "7.0.162",
     "just-bash": "3.6.0",