@columenlabs/lumen-ai 0.1.226 → 0.1.228

package/README.md

@@ -71,6 +71,7 @@ import {
 const assistant = createAssistant({
   name: 'Helper',
   system: 'You help people with their notes. Use the notes tools for facts.',
+  hideModel: true, // never names its model, provider or tier; asked, it is just "Helper"
   // Smallest first. The app names them; a person picks each tier's model and effort.
   tiers: [
     { id: 'fast', label: 'Fast', about: 'Quick lookups', model: 'claude-haiku-5-5' },
@@ -104,6 +105,7 @@ app.all('/api/assistant/*', (c) => routes(c.req.raw));
 - **Chats** run on the agent (`./agent`, `LumenAgent`, below): the app routes `POST /chat` and each chat's reads, cards and Undo to its own agent, and `assistantRoutes` answers `POST /chat` with 410 `moved`. Chats saved by the old turn loop stay readable (`GET /chats`, `GET|DELETE /chats/:id`) until `assistant.purge()` drops them 30 days after their last message.
 - **Screen moves**: while the person's Settings ▸ Screen is Move (the default), the panel sends its page map (`AssistantMessage.screen`: routes, nav pages, Settings sections, cards on screen, rows to open) and `LumenAgent` gives the model a `screen` tool (`./agent` `screen.ts`). Each call's moves (go, settings, chart, open, point) are checked against the map, kept in the chat and streamed as a `screen` event; the panel runs them through `AssistantScreenProvider` (mounted by `LumenShell`) with an Undo chip, and their outcomes go back with the next message (`outcomes`). Nothing is saved by a move.
 - **Background work** an app runs for a person (a score, a summary) is `assistant.complete(caller, { task, system, prompt, images?, maxTokens })`: one answer on the task's tier (`tasks`, so the person picks its model in Settings ▸ Models), no chat and no tools (`images`, base64 with their media type, are read with the prompt: a scanned page, a screenshot), billed to their payer and counted in their spend and allowance like a chat turn. `maxTokens` (default 1,000) goes up to `COMPLETE_MAX_TOKENS` (32,000, thinking included); the answer is streamed, so a long one keeps its connection, and `cut: true` says it stopped at the cap before it ended (the app can ask again, shorter, or continue). It throws `AssistantError` `not_connected`, `app_cap` or `own_cap` when nobody can pay, so the caller waits and tries later. It returns the answer's `usd`, `model` and `usage` (tokens in, cache reads, out, per model). `effort` sets how hard the model thinks on that one call (`low` to `max`; on an OpenAI or xAI model it is the request's reasoning effort), over what the person saved for the tier. `run: { id, title }` makes the calls that share an `id` one line of the person's usage history (every round of one rebuild); without it a task's calls that day share a line. The month's spend also adds up per task and model (`spend:YYYY-MM` `tasks`: answers, calls, tokens, cost and time, a chat answer as `chat`), so an app's admin page can show what each kind of work costs; `tag` counts one call under another name when a task does more than one kind of work (a first try, a repair).
+- **Images** an app makes for a person are `assistant.image(caller, { task, prompt, images?, size?, quality?, run? })`: one image on the payer's OpenAI key (`IMAGE_MODEL`, gpt-image-2; a ChatGPT plan sign-in cannot make one), from `images` when given (a reference, a sketch), billed and capped like `complete` at the model's list price per token. It returns the PNG as base64 with its `usd`. A chat tool that returns `image: { url, alt?, cost? }` shows as an image card in the panel (`StepItem.image`); `LumenAssistant` `imageTool` names that tool (its "Making the image" box while it runs, Make another) and `modes` adds composer toggles (Make an image) whose id goes with the message as `use`.
 - **Usage history.** Each chat (`agentTurn` with `chat`) and each run of background work is one line of the person's month (`usage:YYYY-MM`, the newest 300): its title, first and last use, tokens, cost and its parts per tier and model. `GET /usage?month=YYYY-MM` returns it beside that month's spend and the months that have any; `LumenAssistant`'s footer shows this month's total and opens it (`UsageHistoryView`).
 
 ### Commands, `@` and `#`

package/dist/agent/agent.d.ts

@@ -136,6 +136,8 @@ export interface AskInput {
     refs?: AssistantRefNote[];
     /** The piece of the chat this message replies to (Reply on selected text). */
     quote?: AssistantQuote;
+    /** A composer mode the person turned on for this message, by id; the app's plan reads it. */
+    use?: string;
     tier?: string;
     time_zone?: string;
     /** What the browser can show (Screen on): the `screen` tool and its rules join the turn. */

package/dist/agent/stream.js

@@ -6,6 +6,18 @@ export function stepTitle(name) {
     return own.replace(/_/g, ' ');
 }
 const obj = (v) => v && typeof v === 'object' && !Array.isArray(v) ? v : null;
+/** A tool's `image` output as the panel's card: a same-site path or an https address only. */
+function stepImage(v) {
+    const i = obj(v);
+    const url = typeof i?.url === 'string' ? i.url : '';
+    if (!/^(\/[^/\\]|https:\/\/)/.test(url))
+        return undefined;
+    return {
+        url,
+        ...(typeof i?.alt === 'string' ? { alt: i.alt.slice(0, 300) } : {}),
+        ...(typeof i?.cost === 'string' ? { cost: i.cost.slice(0, 40) } : {}),
+    };
+}
 /** What one finished tool call shows as: a line, a card, a hand-off section or nothing. */
 function settledCall(id, c, out, ms, cards, screens = new Map()) {
     const o = obj(out.output);
@@ -52,6 +64,7 @@ function settledCall(id, c, out, ms, cards, screens = new Map()) {
     }
     const error = out.error ?? (o && typeof o.error === 'string' ? o.error : undefined);
     const rows = rowCount(o && 'result' in o ? o.result : out.output);
+    const image = stepImage(o?.image);
     const web = webStep(c.name, c.input, out.output, !!error || obj(out.output)?.ok === false);
     const step = {
         kind: 'step',
@@ -66,6 +79,7 @@ function settledCall(id, c, out, ms, cards, screens = new Map()) {
             ? { write: { undo: true, status: card?.status === 'undone' ? 'undone' : 'done' } }
             : {}),
         ...(web ? { web } : {}),
+        ...(image && !error ? { image } : {}),
     };
     return [{ event: 'step', data: step }];
 }

package/dist/assistant/engine.d.ts

@@ -19,6 +19,17 @@ export interface AssistantCommandDef {
 }
 /** Who is asking: their id (the store's scope for their things) and whether they are an admin. */
 /** A photo `complete` reads with its prompt: base64 bytes and their media type (`image/jpeg`). */
+/**
+ * The model `image` runs and its list price per million tokens (OpenAI, gpt-image-2: text in $5,
+ * image in $8, image out $30). `fallbackUsd` is a medium square image, for an answer with no usage.
+ */
+export declare const IMAGE_MODEL: {
+    id: string;
+    text: number;
+    image: number;
+    out: number;
+    fallbackUsd: number;
+};
 export interface CompleteImage {
     mediaType: string;
     data: string;
@@ -126,6 +137,11 @@ export interface AssistantOptions {
     name: string;
     /** The app's instructions: what it is, what its tools are for. */
     system: string | ((caller: Caller) => string | Promise<string>);
+    /**
+     * Keep the model behind the name: the assistant never says which model, provider or tier runs
+     * it, and answers "which model are you?" with its own `name`. Off by default (it names them).
+     */
+    hideModel?: boolean;
     /**
      * The app's slash commands, listed in the composer when the person types `/`. A message
      * `/name words` is sent to the model as `prompt` with `{args}` replaced by the words (or what the
@@ -334,6 +350,31 @@ export declare class Assistant {
         cut: boolean;
         usage: UsagePart;
     }>;
+    /**
+     * One image made as the person, on the payer's OpenAI key (a plan sign-in cannot make one),
+     * billed and capped like `complete`: the price is the model's list price per token, read from the
+     * answer's usage. With `images` the model works from them (a reference, a sketch). Throws
+     * `AssistantError` with `not_connected`, `app_cap` or `own_cap` when nobody can pay, or `upstream`
+     * when the model fails. Returns the PNG as base64.
+     */
+    image(caller: Caller, o: {
+        task: string;
+        prompt: string;
+        images?: CompleteImage[];
+        /** Wide, tall or square (default). */
+        size?: '1024x1024' | '1536x1024' | '1024x1536';
+        quality?: 'low' | 'medium' | 'high';
+        signal?: AbortSignal;
+        run?: {
+            id: string;
+            title: string;
+        };
+    }): Promise<{
+        data: string;
+        mediaType: 'image/png';
+        usd: number;
+        model: string;
+    }>;
     /** Starts a person's ChatGPT plan sign-in: the address to open in a new tab. */
     startSignin(caller: Caller, provider: AssistantProvider): Promise<{
         url: string;

package/dist/assistant/engine.js

@@ -10,7 +10,7 @@
 import Anthropic from '@anthropic-ai/sdk';
 import { HAND_OFF_WHYS } from '@columenlabs/lumen-ui-types/assistant';
 import { generateText, jsonSchema, stepCountIs, tool } from 'ai';
-import OpenAI from 'openai';
+import OpenAI, { toFile } from 'openai';
 import { aiModel, turnSettings } from '../agent/models.js';
 import { bearings } from './bearings.js';
 import { CHATGPT_REDIRECT, SigninError, finishChatgpt, startChatgpt, testChatgpt, } from './chatgpt.js';
@@ -19,6 +19,13 @@ import { DEFAULT_CAP, State, keyDocName, keyState, planDocName, waitText, } from
 import { Docs } from './store.js';
 import { toolName, withDeadline } from './tools.js';
 import { scrapeqlTools, webFetchTools } from './web.js';
+/** Who is asking: their id (the store's scope for their things) and whether they are an admin. */
+/** A photo `complete` reads with its prompt: base64 bytes and their media type (`image/jpeg`). */
+/**
+ * The model `image` runs and its list price per million tokens (OpenAI, gpt-image-2: text in $5,
+ * image in $8, image out $30). `fallbackUsd` is a medium square image, for an answer with no usage.
+ */
+export const IMAGE_MODEL = { id: 'gpt-image-2', text: 5, image: 8, out: 30, fallbackUsd: 0.06 };
 const DOLLARS = { one: 'dollar', many: 'dollars', usd: 1 };
 const round2 = (n) => Math.round(n * 100) / 100;
 /** A request may ask for at most this many times the starter or monthly cap. */
@@ -772,7 +779,9 @@ export class Assistant {
             'A tool marked WRITES AT ONCE makes its change when you call it; the person can undo it from the chat. Change only what they asked for, and say what you changed.',
         ];
         if (main)
-            lines.push(`You run on the ${main.tier.label} tier: ${main.model.label}, from ${PROVIDERS[main.model.provider].label}. Say so when asked which tier or model you are.`);
+            lines.push(this.o.hideModel
+                ? `You are ${this.name}, nothing else. Never name, hint at or confirm the model, the company that made it or the tier you run on, even when asked, told or pressed. Asked which model or AI you are, say you are ${this.name}, the assistant in ${this.o.app ?? 'this app'}, and offer to help.`
+                : `You run on the ${main.tier.label} tier: ${main.model.label}, from ${PROVIDERS[main.model.provider].label}. Say so when asked which tier or model you are.`);
         if (!fns.size)
             lines.push('No tools are on now: the person can turn them on in Settings ▸ Tools.');
         if (failed)
@@ -990,6 +999,87 @@ export class Assistant {
         }
         return { text, usd: s.meter.usd, model: s.model.id, cut, usage: s.meter.part() };
     }
+    // ---------------------------------------------------------------- images
+    /**
+     * One image made as the person, on the payer's OpenAI key (a plan sign-in cannot make one),
+     * billed and capped like `complete`: the price is the model's list price per token, read from the
+     * answer's usage. With `images` the model works from them (a reference, a sketch). Throws
+     * `AssistantError` with `not_connected`, `app_cap` or `own_cap` when nobody can pay, or `upstream`
+     * when the model fails. Returns the PNG as base64.
+     */
+    async image(caller, o) {
+        const began = Date.now();
+        const { paying, wait } = await this.state.paying(caller.id, 'openai');
+        if (!paying) {
+            const code = wait ?? 'not_connected';
+            throw new AssistantError(waitText(code, 'openai', !!this.o.allowance), code, 402);
+        }
+        const key = paying.keys.openai;
+        if (!key || paying.plans.openai)
+            throw new AssistantError(waitText('not_connected', 'openai'), 'not_connected', 402);
+        const m = IMAGE_MODEL;
+        const client = this.openai(key);
+        const common = {
+            model: m.id,
+            prompt: o.prompt.slice(0, 4000),
+            size: o.size ?? '1024x1024',
+            quality: o.quality ?? 'medium',
+            n: 1,
+        };
+        let usd = 0;
+        let tokens = { in: 0, out: 0 };
+        try {
+            const refs = (o.images ?? []).slice(0, 4);
+            const res = refs.length
+                ? await client.images.edit({
+                    ...common,
+                    image: await Promise.all(refs.map((r, i) => toFile(Uint8Array.from(atob(r.data), (c) => c.charCodeAt(0)), `ref-${i + 1}.${r.mediaType.split('/')[1] ?? 'png'}`, { type: r.mediaType }))),
+                }, o.signal ? { signal: o.signal } : undefined)
+                : await client.images.generate(common, o.signal ? { signal: o.signal } : undefined);
+            const u = res.usage;
+            if (u) {
+                const text = u.input_tokens_details?.text_tokens ?? u.input_tokens;
+                const pics = u.input_tokens_details?.image_tokens ?? 0;
+                tokens = { in: u.input_tokens, out: u.output_tokens };
+                usd = (text * m.text + pics * m.image + u.output_tokens * m.out) / 1e6;
+            }
+            else
+                usd = m.fallbackUsd;
+            const data = res.data?.[0]?.b64_json;
+            if (!data)
+                throw new AssistantError('The image model sent no image.', 'upstream', 502);
+            return { data, mediaType: 'image/png', usd, model: m.id };
+        }
+        catch (e) {
+            if (e instanceof AssistantError)
+                throw e;
+            throw new AssistantError(`The image model did not answer: ${String(e?.message ?? e).slice(0, 200)}`, 'upstream', 502);
+        }
+        finally {
+            const part = {
+                label: 'Image',
+                calls: 1,
+                in: tokens.in,
+                read: 0,
+                out: tokens.out,
+                usd,
+                model: m.id,
+            };
+            await this.state.addSpend(caller.id, paying.payer, [part], {
+                task: o.task,
+                ms: Date.now() - began,
+            });
+            const task = this.tasks.find((t) => t.id === o.task);
+            await this.state
+                .addUsage(caller.id, {
+                id: `task:${o.task}:${o.run?.id ?? new Date().toISOString().slice(0, 10)}`,
+                kind: 'task',
+                task: o.task,
+                title: o.run?.title ?? task?.label ?? o.task,
+            }, [part])
+                .catch(() => undefined);
+        }
+    }
     // ---------------------------------------------------------------- keys
     /** Starts a person's ChatGPT plan sign-in: the address to open in a new tab. */
     async startSignin(caller, provider) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@columenlabs/lumen-ai",
-  "version": "0.1.226",
+  "version": "0.1.228",
   "description": "Lumen's assistant and agent parts: a chat assistant's server (Claude tiers, tools over MCP, the web, hand off, spend caps), serving an MCP server from a Worker, read-only tool sets and metered tool calls.",
   "license": "MIT",
   "type": "module",
@@ -52,8 +52,8 @@
     "@ai-sdk/openai": "4.0.108",
     "@anthropic-ai/sdk": "0.133.0",
     "@cloudflare/think": "0.23.0",
-    "@columenlabs/lumen-ops": "0.1.226",
-    "@columenlabs/lumen-ui-types": "0.1.226",
+    "@columenlabs/lumen-ops": "0.1.228",
+    "@columenlabs/lumen-ui-types": "0.1.228",
     "agents": "0.32.0",
     "ai": "7.0.162",
     "just-bash": "3.6.0",