@columenlabs/lumen-ai 0.1.148 → 0.1.150

package/README.md

@@ -102,7 +102,7 @@ app.all('/api/assistant/*', (c) => routes(c.req.raw));
 - **Web.** `fetch` is a plain GET from the Worker (10 s, 2 MB, never the app's own hosts or private addresses). ScrapeQL's own tools join over its MCP API once an admin pastes a Cloudflare Access service token (`PUT /web/scrapeql`). A Worker on ScrapeQL's own Cloudflare account passes its service binding instead, `scrapeql: { url: 'https://scrapeql/mcp', fetch: env.SCRAPEQL.fetch.bind(env.SCRAPEQL) }`: no token is stored, the tools are always on and Settings shows the link with a Test.
 - **Hand off and advice.** The `escalate` tool hands one task to a higher tier (the person sets next, any or only when asked); Anthropic's advisor tool lets a higher tier send a plan, at most two an answer.
 - **Chats** stream as server-sent events (`POST /chat`); closing the request stops the answer and keeps what was said. Past 150k tokens, or on `/compact`, the history sent to the model becomes a summary. Run `assistant.purge()` once a day to drop chats 30 days after their last message.
-- **Background work** an app runs for a person (a score, a summary) is `assistant.complete(caller, { task, system, prompt, maxTokens })`: one answer on the task's tier (`tasks`, so the person picks its model in Settings ▸ Models), no chat and no tools, billed to their payer and counted in their spend and allowance like a chat turn. It throws `AssistantError` `not_connected`, `app_cap` or `own_cap` when nobody can pay, so the caller waits and tries later.
+- **Background work** an app runs for a person (a score, a summary) is `assistant.complete(caller, { task, system, prompt, images?, maxTokens })`: one answer on the task's tier (`tasks`, so the person picks its model in Settings ▸ Models), no chat and no tools (`images`, base64 with their media type, are read with the prompt: a scanned page, a screenshot), billed to their payer and counted in their spend and allowance like a chat turn. It throws `AssistantError` `not_connected`, `app_cap` or `own_cap` when nobody can pay, so the caller waits and tries later.
 
 ### Commands, `@` and `#`
 

package/dist/assistant/engine.d.ts

@@ -17,6 +17,11 @@ export interface AssistantCommandDef {
     prompt: string | ((args: string, caller: Caller) => string | Promise<string>);
 }
 /** Who is asking: their id (the store's scope for their things) and whether they are an admin. */
+/** A photo `complete` reads with its prompt: base64 bytes and their media type (`image/jpeg`). */
+export interface CompleteImage {
+    mediaType: string;
+    data: string;
+}
 export interface Caller {
     id: string;
     admin: boolean;
@@ -257,6 +262,8 @@ export declare class Assistant {
         task: string;
         system: string;
         prompt: string;
+        /** Photos read with the prompt (a scanned page, a screenshot). */
+        images?: CompleteImage[];
         maxTokens?: number;
         signal?: AbortSignal;
     }): Promise<{

package/dist/assistant/engine.js

@@ -1702,7 +1702,7 @@ export class Assistant {
      * events like the chat loop, because a ChatGPT plan sign-in answers only streamed calls, takes no
      * output cap and may send its words only as deltas.
      */
-    async openaiOnce(clients, s, instructions, input, max, signal) {
+    async openaiOnce(clients, s, instructions, input, max, signal, images) {
         const client = clients.openai;
         const plan = !!clients.openaiPlan;
         let text = '';
@@ -1711,7 +1711,22 @@ export class Assistant {
             const stream = await client.responses.create({
                 model: s.model.id,
                 instructions,
-                input,
+                // A plan sign-in takes only a list of items, never bare text ("Input must be a list").
+                input: [
+                    {
+                        role: 'user',
+                        content: images?.length
+                            ? [
+                                { type: 'input_text', text: input },
+                                ...images.map((i) => ({
+                                    type: 'input_image',
+                                    image_url: `data:${i.mediaType};base64,${i.data}`,
+                                    detail: 'auto',
+                                })),
+                            ]
+                            : input,
+                    },
+                ],
                 store: false,
                 stream: true,
                 ...(plan ? {} : { max_output_tokens: max }),
@@ -1803,14 +1818,31 @@ export class Assistant {
         let text = '';
         try {
             if (s.model.provider === 'openai' && clients.openai) {
-                text = await this.openaiOnce(clients, s, o.system, o.prompt, max, o.signal);
+                text = await this.openaiOnce(clients, s, o.system, o.prompt, max, o.signal, o.images);
             }
             else if (clients.anthropic) {
                 const msg = await clients.anthropic.messages.create({
                     model: s.model.id,
                     max_tokens: max,
                     system: o.system,
-                    messages: [{ role: 'user', content: o.prompt }],
+                    messages: [
+                        {
+                            role: 'user',
+                            content: o.images?.length
+                                ? [
+                                    ...o.images.map((i) => ({
+                                        type: 'image',
+                                        source: {
+                                            type: 'base64',
+                                            media_type: i.mediaType,
+                                            data: i.data,
+                                        },
+                                    })),
+                                    { type: 'text', text: o.prompt },
+                                ]
+                                : o.prompt,
+                        },
+                    ],
                     ...effortParams(s.effort),
                 }, o.signal ? { signal: o.signal } : undefined);
                 s.meter.add(msg.usage);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@columenlabs/lumen-ai",
-  "version": "0.1.148",
+  "version": "0.1.150",
   "description": "Lumen's assistant and agent parts: a chat assistant's server (Claude tiers, tools over MCP, the web, hand off, spend caps), serving an MCP server from a Worker, read-only tool sets and metered tool calls.",
   "license": "MIT",
   "type": "module",
@@ -41,8 +41,8 @@
   },
   "dependencies": {
     "@anthropic-ai/sdk": "0.133.0",
-    "@columenlabs/lumen-ops": "0.1.148",
-    "@columenlabs/lumen-ui-types": "0.1.148",
+    "@columenlabs/lumen-ops": "0.1.150",
+    "@columenlabs/lumen-ui-types": "0.1.150",
     "openai": "7.32.0",
     "zod": "^4.3.6"
   },