@columenlabs/lumen-ai 0.1.210 → 0.1.211

package/README.md

@@ -237,6 +237,8 @@ Drafts and writes are cards in the chat's own storage (`lumen_cards` in the agen
 
 **Lazy tools.** Past `LAZY_AT` (30) tools, a turn sends the model its core tools and `tool_search`, with one catalog line per group in the instructions. By default the core tools are the apps' finds, the hand off and the advisor. `plan.lazy` sets `{ at, core }`. A search matches the words in a tool's name and description and turns on up to 8 tools from the next step on.
 
+**Images.** `AskInput.images` (base64 PNG, JPEG, WebP or GIF; at most `ASSISTANT_IMAGES_MAX` (4) and `ASSISTANT_IMAGE_CHARS` together) go into the user message as file parts (`imageParts`), so the model sees them and the chat shows them again. `LumenAssistant images` lets the person paste (Cmd+V) or drop images on the box; the composer scales each to 1568 px on its long edge. An app's `POST /chat` passes `images` on in its `AskInput`.
+
 **The panel.** `LumenAssistant` reads a Think chat with no change. `ask(input, sink)` runs one answer and sends the panel's own events (`AssistantEvent`: the chat, the text, each tool line, cards, hand-off sections, advice, the use) to `sink`. The app's `POST /chat` returns `askResponse((sink) => chat.ask(input, sink), { signal, stop: () => chat.stop() })`, the same server-sent events `assistantRoutes` sends. `view()` gives a stored chat as the panel's items, and `usage()` gives the chat's use so far. A card's id is its tool call's, so the panel's Save, Dismiss and Undo go to `saveCard`, `dismissCard` and `undoCard`. The app keeps the list of chats, and each chat is its own agent, such as a facet of the person's store. `chunkEvents` and `itemsOf` are the mapping, for an app that streams some other way.
 
 ```ts

package/dist/agent/agent.d.ts

@@ -12,7 +12,7 @@
  * An `Assistant` gives both: `assistant.agentTurn(caller, input)` returns a plan and its settle.
  */
 import { type ChatResponseResult, type PrepareStepContext, type Session, type StepConfig, type StepContext, Think, type ThinkModel, type TurnConfig, type TurnContext } from '@cloudflare/think';
-import type { AssistantChat, AssistantEffort, AssistantEvent, AssistantRefNote, AssistantScreenMap, AssistantScreenOutcome, ChatUsage, UsagePart } from '@columenlabs/lumen-ui-types/assistant';
+import { type AssistantChat, type AssistantEffort, type AssistantEvent, type AssistantImage, type AssistantRefNote, type AssistantScreenMap, type AssistantScreenOutcome, type ChatUsage, type UsagePart } from '@columenlabs/lumen-ui-types/assistant';
 import { type ModelMessage, type ToolSet } from 'ai';
 import type { ModelDef } from '../assistant/models.js';
 import { type AgentConnector, type ConnectorAt } from './connectors.js';
@@ -116,6 +116,8 @@ export declare function withLead(messages: ModelMessage[], lead: string | undefi
 /** One message the person sent, with the page, refs and tier the composer added. */
 export interface AskInput {
     message: string;
+    /** Images to look at with the message; kept to the allowed types, count and size. */
+    images?: AssistantImage[];
     context?: {
         label: string;
         note?: unknown;
@@ -128,6 +130,15 @@ export interface AskInput {
     /** How earlier screen calls ended in the browser. */
     outcomes?: AssistantScreenOutcome[];
 }
+/**
+ * The images a message may carry, as file parts: allowed types only, at most `ASSISTANT_IMAGES_MAX`,
+ * and no more than `ASSISTANT_IMAGE_CHARS` of base64 together; the rest are left out.
+ */
+export declare function imageParts(images: unknown): {
+    type: 'file';
+    mediaType: string;
+    url: string;
+}[];
 /** Where an answer's events go: the app's route, over RPC. */
 export interface AskSink {
     event(e: AssistantEvent): void | Promise<void>;

package/dist/agent/agent.js

@@ -13,6 +13,7 @@
  */
 import { anthropic } from '@ai-sdk/anthropic';
 import { Think, } from '@cloudflare/think';
+import { ASSISTANT_IMAGES_MAX, ASSISTANT_IMAGE_CHARS, ASSISTANT_IMAGE_TYPES, } from '@columenlabs/lumen-ui-types/assistant';
 import { createCompactFunction } from 'agents/sessions';
 import { generateText } from 'ai';
 import { CONNECTOR_GROUP, checkConnectorUrl, connectorGroup, connectorId, connectorSource, } from './connectors.js';
@@ -66,6 +67,26 @@ export function withLead(messages, lead) {
         : [{ type: 'text', text: lead }, ...m.content];
     return [...messages.slice(0, i), { ...m, content }, ...messages.slice(i + 1)];
 }
+/**
+ * The images a message may carry, as file parts: allowed types only, at most `ASSISTANT_IMAGES_MAX`,
+ * and no more than `ASSISTANT_IMAGE_CHARS` of base64 together; the rest are left out.
+ */
+export function imageParts(images) {
+    if (!Array.isArray(images))
+        return [];
+    const out = [];
+    let chars = 0;
+    for (const i of images.slice(0, ASSISTANT_IMAGES_MAX)) {
+        const type = ASSISTANT_IMAGE_TYPES.find((t) => t === i?.media_type);
+        if (!type || typeof i?.data !== 'string' || !/^[A-Za-z0-9+/]+=*$/.test(i.data))
+            continue;
+        if (chars + i.data.length > ASSISTANT_IMAGE_CHARS)
+            break;
+        chars += i.data.length;
+        out.push({ type: 'file', mediaType: type, url: `data:${type};base64,${i.data}` });
+    }
+    return out;
+}
 /** Usage parts added up: same label, one part. */
 export function chatUsage(parts, context) {
     const by = new Map();
@@ -174,7 +195,9 @@ export class LumenAgent extends Think {
             kindOf: (n) => this.#turn?.tools.info.find((i) => i.name === n)?.kind,
         });
         const map = screenMapOf(input.screen);
-        this.#input = { ...input, ...(map ? { screen: map } : {}) };
+        // The images go in the message's parts; the turn's plan reads the rest.
+        const { images: _images, ...asked } = input;
+        this.#input = { ...asked, ...(map ? { screen: map } : {}) };
         if (!map)
             delete this.#input.screen;
         let failed = false;
@@ -182,7 +205,7 @@ export class LumenAgent extends Think {
             await this.chat({
                 id: `u_${crypto.randomUUID().replaceAll('-', '').slice(0, 16)}`,
                 role: 'user',
-                parts: [{ type: 'text', text: input.message }],
+                parts: [{ type: 'text', text: input.message }, ...imageParts(input.images)],
                 metadata: meta,
             }, {
                 onStart: () => undefined,

package/dist/agent/stream.js

@@ -138,10 +138,14 @@ export function itemsOf(messages, cards, screens = new Map()) {
                 .map((p) => String(p.text ?? ''))
                 .join('');
             const meta = (obj(m.metadata) ?? {});
+            const images = m.parts
+                .filter((p) => p.type === 'file' && typeof p.url === 'string' && p.url.startsWith('data:image/'))
+                .map((p) => p.url);
             items.push({
                 kind: 'user',
                 text,
                 at: meta.at ?? 0,
+                ...(images.length ? { images } : {}),
                 ...(meta.context?.label ? { context: meta.context.label } : {}),
                 ...(meta.refs?.length ? { refs: meta.refs } : {}),
                 ...(meta.tier ? { tier: meta.tier } : {}),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@columenlabs/lumen-ai",
-  "version": "0.1.210",
+  "version": "0.1.211",
   "description": "Lumen's assistant and agent parts: a chat assistant's server (Claude tiers, tools over MCP, the web, hand off, spend caps), serving an MCP server from a Worker, read-only tool sets and metered tool calls.",
   "license": "MIT",
   "type": "module",
@@ -52,8 +52,8 @@
     "@ai-sdk/openai": "4.0.108",
     "@anthropic-ai/sdk": "0.133.0",
     "@cloudflare/think": "0.23.0",
-    "@columenlabs/lumen-ops": "0.1.210",
-    "@columenlabs/lumen-ui-types": "0.1.210",
+    "@columenlabs/lumen-ops": "0.1.211",
+    "@columenlabs/lumen-ui-types": "0.1.211",
     "agents": "0.32.0",
     "ai": "7.0.162",
     "just-bash": "3.6.0",