@columenlabs/lumen-ai 0.1.225 → 0.1.227

package/README.md

@@ -104,6 +104,7 @@ app.all('/api/assistant/*', (c) => routes(c.req.raw));
 - **Chats** run on the agent (`./agent`, `LumenAgent`, below): the app routes `POST /chat` and each chat's reads, cards and Undo to its own agent, and `assistantRoutes` answers `POST /chat` with 410 `moved`. Chats saved by the old turn loop stay readable (`GET /chats`, `GET|DELETE /chats/:id`) until `assistant.purge()` drops them 30 days after their last message.
 - **Screen moves**: while the person's Settings ▸ Screen is Move (the default), the panel sends its page map (`AssistantMessage.screen`: routes, nav pages, Settings sections, cards on screen, rows to open) and `LumenAgent` gives the model a `screen` tool (`./agent` `screen.ts`). Each call's moves (go, settings, chart, open, point) are checked against the map, kept in the chat and streamed as a `screen` event; the panel runs them through `AssistantScreenProvider` (mounted by `LumenShell`) with an Undo chip, and their outcomes go back with the next message (`outcomes`). Nothing is saved by a move.
 - **Background work** an app runs for a person (a score, a summary) is `assistant.complete(caller, { task, system, prompt, images?, maxTokens })`: one answer on the task's tier (`tasks`, so the person picks its model in Settings ▸ Models), no chat and no tools (`images`, base64 with their media type, are read with the prompt: a scanned page, a screenshot), billed to their payer and counted in their spend and allowance like a chat turn. `maxTokens` (default 1,000) goes up to `COMPLETE_MAX_TOKENS` (32,000, thinking included); the answer is streamed, so a long one keeps its connection, and `cut: true` says it stopped at the cap before it ended (the app can ask again, shorter, or continue). It throws `AssistantError` `not_connected`, `app_cap` or `own_cap` when nobody can pay, so the caller waits and tries later. It returns the answer's `usd`, `model` and `usage` (tokens in, cache reads, out, per model). `effort` sets how hard the model thinks on that one call (`low` to `max`; on an OpenAI or xAI model it is the request's reasoning effort), over what the person saved for the tier. `run: { id, title }` makes the calls that share an `id` one line of the person's usage history (every round of one rebuild); without it a task's calls that day share a line. The month's spend also adds up per task and model (`spend:YYYY-MM` `tasks`: answers, calls, tokens, cost and time, a chat answer as `chat`), so an app's admin page can show what each kind of work costs; `tag` counts one call under another name when a task does more than one kind of work (a first try, a repair).
+- **Images** an app makes for a person are `assistant.image(caller, { task, prompt, images?, size?, quality?, run? })`: one image on the payer's OpenAI key (`IMAGE_MODEL`, gpt-image-2; a ChatGPT plan sign-in cannot make one), from `images` when given (a reference, a sketch), billed and capped like `complete` at the model's list price per token. It returns the PNG as base64 with its `usd`. A chat tool that returns `image: { url, alt?, cost? }` shows as an image card in the panel (`StepItem.image`); `LumenAssistant` `imageTool` names that tool (its "Making the image" box while it runs, Make another) and `modes` adds composer toggles (Make an image) whose id goes with the message as `use`.
 - **Usage history.** Each chat (`agentTurn` with `chat`) and each run of background work is one line of the person's month (`usage:YYYY-MM`, the newest 300): its title, first and last use, tokens, cost and its parts per tier and model. `GET /usage?month=YYYY-MM` returns it beside that month's spend and the months that have any; `LumenAssistant`'s footer shows this month's total and opens it (`UsageHistoryView`).
 
 ### Commands, `@` and `#`

package/dist/agent/agent.d.ts

@@ -12,7 +12,7 @@
  * An `Assistant` gives both: `assistant.agentTurn(caller, input)` returns a plan and its settle.
  */
 import { type ChatResponseResult, type PrepareStepContext, type Session, type StepConfig, type StepContext, Think, type ThinkModel, type TurnConfig, type TurnContext } from '@cloudflare/think';
-import { type AssistantChat, type AssistantEffort, type AssistantEvent, type AssistantImage, type AssistantRefNote, type AssistantScreenMap, type AssistantScreenOutcome, type ChatUsage, type UsagePart } from '@columenlabs/lumen-ui-types/assistant';
+import { type AssistantChat, type AssistantEffort, type AssistantEvent, type AssistantImage, type AssistantQuote, type AssistantRefNote, type AssistantScreenMap, type AssistantScreenOutcome, type ChatUsage, type UsagePart } from '@columenlabs/lumen-ui-types/assistant';
 import { type ModelMessage, type ToolSet } from 'ai';
 import type { ModelDef } from '../assistant/models.js';
 import { type AgentConnector, type ConnectorAt } from './connectors.js';
@@ -134,6 +134,10 @@ export interface AskInput {
         note?: unknown;
     };
     refs?: AssistantRefNote[];
+    /** The piece of the chat this message replies to (Reply on selected text). */
+    quote?: AssistantQuote;
+    /** A composer mode the person turned on for this message, by id; the app's plan reads it. */
+    use?: string;
     tier?: string;
     time_zone?: string;
     /** What the browser can show (Screen on): the `screen` tool and its rules join the turn. */
@@ -141,6 +145,14 @@ export interface AskInput {
     /** How earlier screen calls ended in the browser. */
     outcomes?: AssistantScreenOutcome[];
 }
+/** A quote as sent, checked: its text cut to `ASSISTANT_QUOTE_MAX`, and whose message it came from. */
+export declare function quoteOf(q: unknown): AssistantQuote | undefined;
+/**
+ * The words the model reads before a message that replies to a quote: which message it came from
+ * and the quote itself. It is the message's first text part; the panel shows the quote from the
+ * message's metadata instead, so `itemsOf` leaves this part out.
+ */
+export declare function quoteLead(q: AssistantQuote): string;
 /**
  * The images a message may carry, as file parts: allowed types only, at most `ASSISTANT_IMAGES_MAX`,
  * and no more than `ASSISTANT_IMAGE_CHARS` of base64 together; the rest are left out.

package/dist/agent/agent.js

@@ -13,7 +13,7 @@
  */
 import { anthropic } from '@ai-sdk/anthropic';
 import { Think, } from '@cloudflare/think';
-import { ASSISTANT_IMAGES_MAX, ASSISTANT_IMAGE_CHARS, ASSISTANT_IMAGE_TYPES, } from '@columenlabs/lumen-ui-types/assistant';
+import { ASSISTANT_IMAGES_MAX, ASSISTANT_IMAGE_CHARS, ASSISTANT_IMAGE_TYPES, ASSISTANT_QUOTE_MAX, } from '@columenlabs/lumen-ui-types/assistant';
 import { createCompactFunction } from 'agents/sessions';
 import { generateText } from 'ai';
 import { CONNECTOR_GROUP, checkConnectorUrl, connectorGroup, connectorId, connectorSource, } from './connectors.js';
@@ -73,6 +73,27 @@ export function withLead(messages, lead) {
         : [{ type: 'text', text: lead }, ...m.content];
     return [...messages.slice(0, i), { ...m, content }, ...messages.slice(i + 1)];
 }
+/** A quote as sent, checked: its text cut to `ASSISTANT_QUOTE_MAX`, and whose message it came from. */
+export function quoteOf(q) {
+    const o = q;
+    const text = typeof o?.text === 'string' ? o.text.trim().slice(0, ASSISTANT_QUOTE_MAX) : '';
+    if (!text)
+        return undefined;
+    return { text, from: o?.from === 'user' ? 'user' : 'assistant' };
+}
+/**
+ * The words the model reads before a message that replies to a quote: which message it came from
+ * and the quote itself. It is the message's first text part; the panel shows the quote from the
+ * message's metadata instead, so `itemsOf` leaves this part out.
+ */
+export function quoteLead(q) {
+    const whose = q.from === 'user' ? 'their own earlier message' : 'your earlier answer';
+    const lines = q.text
+        .split('\n')
+        .map((l) => `> ${l}`)
+        .join('\n');
+    return `The person replies to this part of ${whose}:\n${lines}\n\n`;
+}
 /**
  * The images a message may carry, as file parts: allowed types only, at most `ASSISTANT_IMAGES_MAX`,
  * and no more than `ASSISTANT_IMAGE_CHARS` of base64 together; the rest are left out.
@@ -209,10 +230,12 @@ export class LumenAgent extends Think {
         this.#put('at', at);
         await sink.event({ event: 'chat', data: { id: this.name, title } });
         this.#outcomes(screenOutcomesOf(input.outcomes));
+        const quote = quoteOf(input.quote);
         const meta = {
             at,
             ...(input.context ? { context: { label: input.context.label } } : {}),
             ...(input.refs?.length ? { refs: input.refs } : {}),
+            ...(quote ? { quote } : {}),
             ...(input.tier ? { tier: input.tier } : {}),
         };
         const toEvents = chunkEvents({
@@ -237,7 +260,11 @@ export class LumenAgent extends Think {
             await this.chat({
                 id: userId,
                 role: 'user',
-                parts: [{ type: 'text', text: input.message }, ...imageParts(input.images)],
+                parts: [
+                    ...(quote ? [{ type: 'text', text: quoteLead(quote) }] : []),
+                    { type: 'text', text: input.message },
+                    ...imageParts(input.images),
+                ],
                 metadata: meta,
             }, {
                 onStart: () => undefined,

package/dist/agent/stream.d.ts

@@ -3,7 +3,7 @@
  * panel's `AssistantEvent`s, and a stored chat's messages and cards as its `ChatItem`s. The panel
  * (`@columenlabs/lumen-ui`'s `LumenAssistant`) reads a Think chat the same way as before.
  */
-import type { AssistantEvent, AssistantRefNote, ChatItem } from '@columenlabs/lumen-ui-types/assistant';
+import type { AssistantEvent, AssistantQuote, AssistantRefNote, ChatItem } from '@columenlabs/lumen-ui-types/assistant';
 import type { AgentCard } from './agent.js';
 import type { ScreenCall } from './screen.js';
 /** A tool's name as a line's title: `notes__find_notes` → `find notes`. */
@@ -40,6 +40,8 @@ export interface AskMeta {
         label: string;
     };
     refs?: AssistantRefNote[];
+    /** The piece of the chat the message replies to; its words to the model are the first text part. */
+    quote?: AssistantQuote;
     tier?: string;
 }
 /** A stored chat as the panel's items: the messages in order, each tool call as it ended. */

package/dist/agent/stream.js

@@ -6,6 +6,18 @@ export function stepTitle(name) {
     return own.replace(/_/g, ' ');
 }
 const obj = (v) => v && typeof v === 'object' && !Array.isArray(v) ? v : null;
+/** A tool's `image` output as the panel's card: a same-site path or an https address only. */
+function stepImage(v) {
+    const i = obj(v);
+    const url = typeof i?.url === 'string' ? i.url : '';
+    if (!/^(\/[^/\\]|https:\/\/)/.test(url))
+        return undefined;
+    return {
+        url,
+        ...(typeof i?.alt === 'string' ? { alt: i.alt.slice(0, 300) } : {}),
+        ...(typeof i?.cost === 'string' ? { cost: i.cost.slice(0, 40) } : {}),
+    };
+}
 /** What one finished tool call shows as: a line, a card, a hand-off section or nothing. */
 function settledCall(id, c, out, ms, cards, screens = new Map()) {
     const o = obj(out.output);
@@ -52,6 +64,7 @@ function settledCall(id, c, out, ms, cards, screens = new Map()) {
     }
     const error = out.error ?? (o && typeof o.error === 'string' ? o.error : undefined);
     const rows = rowCount(o && 'result' in o ? o.result : out.output);
+    const image = stepImage(o?.image);
     const web = webStep(c.name, c.input, out.output, !!error || obj(out.output)?.ok === false);
     const step = {
         kind: 'step',
@@ -66,6 +79,7 @@ function settledCall(id, c, out, ms, cards, screens = new Map()) {
             ? { write: { undo: true, status: card?.status === 'undone' ? 'undone' : 'done' } }
             : {}),
         ...(web ? { web } : {}),
+        ...(image && !error ? { image } : {}),
     };
     return [{ event: 'step', data: step }];
 }
@@ -137,11 +151,12 @@ export function itemsOf(messages, cards, screens = new Map(), steers = {}) {
     for (const m of messages) {
         if (m.role === 'user') {
             asked = m.id;
+            const meta = (obj(m.metadata) ?? {});
             const text = m.parts
                 .filter((p) => p.type === 'text')
+                .slice(meta.quote ? 1 : 0)
                 .map((p) => String(p.text ?? ''))
                 .join('');
-            const meta = (obj(m.metadata) ?? {});
             const images = m.parts
                 .filter((p) => p.type === 'file' && typeof p.url === 'string' && p.url.startsWith('data:image/'))
                 .map((p) => p.url);
@@ -152,6 +167,7 @@ export function itemsOf(messages, cards, screens = new Map(), steers = {}) {
                 ...(images.length ? { images } : {}),
                 ...(meta.context?.label ? { context: meta.context.label } : {}),
                 ...(meta.refs?.length ? { refs: meta.refs } : {}),
+                ...(meta.quote ? { quote: meta.quote } : {}),
                 ...(meta.tier ? { tier: meta.tier } : {}),
             });
             continue;

package/dist/assistant/engine.d.ts

@@ -19,6 +19,17 @@ export interface AssistantCommandDef {
 }
 /** Who is asking: their id (the store's scope for their things) and whether they are an admin. */
 /** A photo `complete` reads with its prompt: base64 bytes and their media type (`image/jpeg`). */
+/**
+ * The model `image` runs and its list price per million tokens (OpenAI, gpt-image-2: text in $5,
+ * image in $8, image out $30). `fallbackUsd` is a medium square image, for an answer with no usage.
+ */
+export declare const IMAGE_MODEL: {
+    id: string;
+    text: number;
+    image: number;
+    out: number;
+    fallbackUsd: number;
+};
 export interface CompleteImage {
     mediaType: string;
     data: string;
@@ -334,6 +345,31 @@ export declare class Assistant {
         cut: boolean;
         usage: UsagePart;
     }>;
+    /**
+     * One image made as the person, on the payer's OpenAI key (a plan sign-in cannot make one),
+     * billed and capped like `complete`: the price is the model's list price per token, read from the
+     * answer's usage. With `images` the model works from them (a reference, a sketch). Throws
+     * `AssistantError` with `not_connected`, `app_cap` or `own_cap` when nobody can pay, or `upstream`
+     * when the model fails. Returns the PNG as base64.
+     */
+    image(caller: Caller, o: {
+        task: string;
+        prompt: string;
+        images?: CompleteImage[];
+        /** Wide, tall or square (default). */
+        size?: '1024x1024' | '1536x1024' | '1024x1536';
+        quality?: 'low' | 'medium' | 'high';
+        signal?: AbortSignal;
+        run?: {
+            id: string;
+            title: string;
+        };
+    }): Promise<{
+        data: string;
+        mediaType: 'image/png';
+        usd: number;
+        model: string;
+    }>;
     /** Starts a person's ChatGPT plan sign-in: the address to open in a new tab. */
     startSignin(caller: Caller, provider: AssistantProvider): Promise<{
         url: string;

package/dist/assistant/engine.js

@@ -10,7 +10,7 @@
 import Anthropic from '@anthropic-ai/sdk';
 import { HAND_OFF_WHYS } from '@columenlabs/lumen-ui-types/assistant';
 import { generateText, jsonSchema, stepCountIs, tool } from 'ai';
-import OpenAI from 'openai';
+import OpenAI, { toFile } from 'openai';
 import { aiModel, turnSettings } from '../agent/models.js';
 import { bearings } from './bearings.js';
 import { CHATGPT_REDIRECT, SigninError, finishChatgpt, startChatgpt, testChatgpt, } from './chatgpt.js';
@@ -19,6 +19,13 @@ import { DEFAULT_CAP, State, keyDocName, keyState, planDocName, waitText, } from
 import { Docs } from './store.js';
 import { toolName, withDeadline } from './tools.js';
 import { scrapeqlTools, webFetchTools } from './web.js';
+/** Who is asking: their id (the store's scope for their things) and whether they are an admin. */
+/** A photo `complete` reads with its prompt: base64 bytes and their media type (`image/jpeg`). */
+/**
+ * The model `image` runs and its list price per million tokens (OpenAI, gpt-image-2: text in $5,
+ * image in $8, image out $30). `fallbackUsd` is a medium square image, for an answer with no usage.
+ */
+export const IMAGE_MODEL = { id: 'gpt-image-2', text: 5, image: 8, out: 30, fallbackUsd: 0.06 };
 const DOLLARS = { one: 'dollar', many: 'dollars', usd: 1 };
 const round2 = (n) => Math.round(n * 100) / 100;
 /** A request may ask for at most this many times the starter or monthly cap. */
@@ -990,6 +997,87 @@ export class Assistant {
         }
         return { text, usd: s.meter.usd, model: s.model.id, cut, usage: s.meter.part() };
     }
+    // ---------------------------------------------------------------- images
+    /**
+     * One image made as the person, on the payer's OpenAI key (a plan sign-in cannot make one),
+     * billed and capped like `complete`: the price is the model's list price per token, read from the
+     * answer's usage. With `images` the model works from them (a reference, a sketch). Throws
+     * `AssistantError` with `not_connected`, `app_cap` or `own_cap` when nobody can pay, or `upstream`
+     * when the model fails. Returns the PNG as base64.
+     */
+    async image(caller, o) {
+        const began = Date.now();
+        const { paying, wait } = await this.state.paying(caller.id, 'openai');
+        if (!paying) {
+            const code = wait ?? 'not_connected';
+            throw new AssistantError(waitText(code, 'openai', !!this.o.allowance), code, 402);
+        }
+        const key = paying.keys.openai;
+        if (!key || paying.plans.openai)
+            throw new AssistantError(waitText('not_connected', 'openai'), 'not_connected', 402);
+        const m = IMAGE_MODEL;
+        const client = this.openai(key);
+        const common = {
+            model: m.id,
+            prompt: o.prompt.slice(0, 4000),
+            size: o.size ?? '1024x1024',
+            quality: o.quality ?? 'medium',
+            n: 1,
+        };
+        let usd = 0;
+        let tokens = { in: 0, out: 0 };
+        try {
+            const refs = (o.images ?? []).slice(0, 4);
+            const res = refs.length
+                ? await client.images.edit({
+                    ...common,
+                    image: await Promise.all(refs.map((r, i) => toFile(Uint8Array.from(atob(r.data), (c) => c.charCodeAt(0)), `ref-${i + 1}.${r.mediaType.split('/')[1] ?? 'png'}`, { type: r.mediaType }))),
+                }, o.signal ? { signal: o.signal } : undefined)
+                : await client.images.generate(common, o.signal ? { signal: o.signal } : undefined);
+            const u = res.usage;
+            if (u) {
+                const text = u.input_tokens_details?.text_tokens ?? u.input_tokens;
+                const pics = u.input_tokens_details?.image_tokens ?? 0;
+                tokens = { in: u.input_tokens, out: u.output_tokens };
+                usd = (text * m.text + pics * m.image + u.output_tokens * m.out) / 1e6;
+            }
+            else
+                usd = m.fallbackUsd;
+            const data = res.data?.[0]?.b64_json;
+            if (!data)
+                throw new AssistantError('The image model sent no image.', 'upstream', 502);
+            return { data, mediaType: 'image/png', usd, model: m.id };
+        }
+        catch (e) {
+            if (e instanceof AssistantError)
+                throw e;
+            throw new AssistantError(`The image model did not answer: ${String(e?.message ?? e).slice(0, 200)}`, 'upstream', 502);
+        }
+        finally {
+            const part = {
+                label: 'Image',
+                calls: 1,
+                in: tokens.in,
+                read: 0,
+                out: tokens.out,
+                usd,
+                model: m.id,
+            };
+            await this.state.addSpend(caller.id, paying.payer, [part], {
+                task: o.task,
+                ms: Date.now() - began,
+            });
+            const task = this.tasks.find((t) => t.id === o.task);
+            await this.state
+                .addUsage(caller.id, {
+                id: `task:${o.task}:${o.run?.id ?? new Date().toISOString().slice(0, 10)}`,
+                kind: 'task',
+                task: o.task,
+                title: o.run?.title ?? task?.label ?? o.task,
+            }, [part])
+                .catch(() => undefined);
+        }
+    }
     // ---------------------------------------------------------------- keys
     /** Starts a person's ChatGPT plan sign-in: the address to open in a new tab. */
     async startSignin(caller, provider) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@columenlabs/lumen-ai",
-  "version": "0.1.225",
+  "version": "0.1.227",
   "description": "Lumen's assistant and agent parts: a chat assistant's server (Claude tiers, tools over MCP, the web, hand off, spend caps), serving an MCP server from a Worker, read-only tool sets and metered tool calls.",
   "license": "MIT",
   "type": "module",
@@ -52,8 +52,8 @@
     "@ai-sdk/openai": "4.0.108",
     "@anthropic-ai/sdk": "0.133.0",
     "@cloudflare/think": "0.23.0",
-    "@columenlabs/lumen-ops": "0.1.225",
-    "@columenlabs/lumen-ui-types": "0.1.225",
+    "@columenlabs/lumen-ops": "0.1.227",
+    "@columenlabs/lumen-ui-types": "0.1.227",
     "agents": "0.32.0",
     "ai": "7.0.162",
     "just-bash": "3.6.0",