@columenlabs/lumen-ui 0.1.210 → 0.1.211

package/dist/assistant/lumen-assistant.d.ts

@@ -1,4 +1,4 @@
-import type { AllowanceUnit, AssistantMessage, AssistantRefNote, AssistantStatus } from '@columenlabs/lumen-ui-types/assistant';
+import { type AllowanceUnit, type AssistantImage, type AssistantMessage, type AssistantRefNote, type AssistantStatus } from '@columenlabs/lumen-ui-types/assistant';
 import { type LumenOption } from '../lumen-selector.js';
 import type { AssistantAdapter } from './client.js';
 import { type AssistantSource } from './sources.js';
@@ -27,6 +27,11 @@ export interface LumenAssistantProps {
     commands?: AssistantCommand[];
     /** What `#` lists everywhere; a page adds its own with `useAssistantSource`. */
     sources?: readonly AssistantSource[];
+    /**
+     * Images can go with a message: pasted (Cmd+V / Ctrl+V) or dropped on the box, shown as chips
+     * until it is sent. Turn it on when the app's chat route passes `images` on to the agent.
+     */
+    images?: boolean;
 }
 /** A slash command: typing `/` in the composer lists them. */
 export interface AssistantCommand {
@@ -42,21 +47,24 @@ export interface AssistantCommand {
     run: (args: string) => string | undefined | Promise<string | undefined>;
 }
 /** The assistant's chat: header, messages and composer. */
-export declare function LumenAssistant({ client, context, onSettings, keepDays, onClose, chat: outside, commands, sources, }: LumenAssistantProps): import("react").JSX.Element;
+export declare function LumenAssistant({ client, context, onSettings, keepDays, onClose, chat: outside, commands, sources, images, }: LumenAssistantProps): import("react").JSX.Element;
 /** An allowance in its unit: `$4.20`, or `350 credits`. */
 export declare const allowanceAmount: (u: AllowanceUnit, n: number) => string;
 /** What the composer sends with the text: the tier picked with `@` and the things pointed at with `#`. */
 export interface ComposerExtra {
     tier?: string;
     refs?: AssistantRefNote[];
+    images?: AssistantImage[];
 }
 /**
  * The message box: Enter sends, Shift+Enter breaks a line; Stop while an answer runs. Three
  * triggers list what can follow them as it is typed: `/` at the start a command, `@` a tier for
  * this message, `#` a thing to point at (the `sources`); a picked tier or thing shows as a chip.
  */
-export declare function LumenComposer({ name, busy, queued, disabled, commands, tiers, sources, page, onSend, onStop, }: {
+export declare function LumenComposer({ name, busy, queued, disabled, commands, tiers, sources, page, images: canAttach, onSend, onStop, }: {
     name: string;
+    /** Images can be pasted or dropped on the box (`LumenAssistant` `images`). */
+    images?: boolean;
     /** The page's name for `#` This page; the document's title by default. */
     page?: string;
     busy: boolean;

package/dist/assistant/lumen-assistant.js

@@ -1,4 +1,10 @@
 import { Fragment as _Fragment, jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
+// LumenAssistant: the chat that fills LumenAssistantPanel, for any app's assistant served by
+// `@columenlabs/lumen-ai/assistant`. A header with the tier, Chats and New chat; the chat itself
+// (the person's messages, the answer as it streams, each tool call as a line, drafted changes as
+// cards to Save or Dismiss, work handed to a higher tier in its own section); and the composer,
+// with Stop, a queued message and the chat's cost and context under it.
+import { ASSISTANT_IMAGES_MAX, ASSISTANT_IMAGE_CHARS, } from '@columenlabs/lumen-ui-types/assistant';
 import { useQuery, useQueryClient } from '@tanstack/react-query';
 import { ArrowUpIcon, CheckIcon, ChevronRightIcon, CircleAlertIcon, CopyIcon, CornerDownRightIcon, CrosshairIcon, FileTextIcon, LightbulbIcon, MonitorIcon, PencilIcon, PlusIcon, ReceiptIcon, SparklesIcon, SquareIcon, TriangleAlertIcon, Undo2Icon, WrenchIcon, XIcon, ZapIcon, } from 'lucide-react';
 import { Fragment, createContext, useContext, useEffect, useRef, useState, } from 'react';
@@ -42,7 +48,7 @@ export function useAssistantStatus(client) {
 export { usd };
 const seconds = (ms) => (ms < 1000 ? `${ms} ms` : `${(ms / 1000).toFixed(1)} s`);
 /** The assistant's chat: header, messages and composer. */
-export function LumenAssistant({ client, context, onSettings, keepDays = 30, onClose, chat: outside, commands = [], sources = [], }) {
+export function LumenAssistant({ client, context, onSettings, keepDays = 30, onClose, chat: outside, commands = [], sources = [], images = false, }) {
     const own = useAssistantChat(client);
     const chat = outside ?? own;
     const qc = useQueryClient();
@@ -179,7 +185,7 @@ export function LumenAssistant({ client, context, onSettings, keepDays = 30, onC
                             id: c.id,
                             title: c.title,
                             updated_at: new Date(c.at).toISOString(),
-                        })) ?? (chats.isFetched ? [] : null), value: chat.chatId ?? undefined, note: `Chats are kept ${keepDays} days after their last message.`, onRefresh: () => void chats.refetch(), onOpen: (c) => void chat.open(c.id) }), _jsx(IconButton, { icon: CopyIcon, label: "Copy chat", disabled: !chat.items.length, onClick: () => copyMarkdown(chatMarkdown(chat.title, chat.items, name), 'Chat') }), _jsx(IconButton, { icon: PlusIcon, label: "New chat", disabled: !chat.items.length, onClick: chat.reset }), onClose && (_jsx(IconButton, { icon: XIcon, label: `Close ${s?.name ?? 'assistant'}`, onClick: onClose }))] }), _jsx(AssistantName.Provider, { value: name, children: _jsx(Screen.Provider, { value: controls, children: _jsx(Messages, { chat: chat, status: s, onSettings: onSettings }) }) }), _jsx(LumenComposer, { name: s?.name ?? 'the assistant', busy: chat.busy, queued: chat.queued, disabled: !s || !!s.wait, commands: allCommands, tiers: tiers, sources: allSources, page: context?.label, onSend: (text, more) => chat.send(text, { ...extra(), ...more }), onStop: chat.stop }), _jsx(UsageLine, { usage: chat.usage, client: client, spent: s?.spend.usd })] }));
+                        })) ?? (chats.isFetched ? [] : null), value: chat.chatId ?? undefined, note: `Chats are kept ${keepDays} days after their last message.`, onRefresh: () => void chats.refetch(), onOpen: (c) => void chat.open(c.id) }), _jsx(IconButton, { icon: CopyIcon, label: "Copy chat", disabled: !chat.items.length, onClick: () => copyMarkdown(chatMarkdown(chat.title, chat.items, name), 'Chat') }), _jsx(IconButton, { icon: PlusIcon, label: "New chat", disabled: !chat.items.length, onClick: chat.reset }), onClose && (_jsx(IconButton, { icon: XIcon, label: `Close ${s?.name ?? 'assistant'}`, onClick: onClose }))] }), _jsx(AssistantName.Provider, { value: name, children: _jsx(Screen.Provider, { value: controls, children: _jsx(Messages, { chat: chat, status: s, onSettings: onSettings }) }) }), _jsx(LumenComposer, { name: s?.name ?? 'the assistant', busy: chat.busy, queued: chat.queued, disabled: !s || !!s.wait, commands: allCommands, tiers: tiers, sources: allSources, page: context?.label, images: images, onSend: (text, more) => chat.send(text, { ...extra(), ...more }), onStop: chat.stop }), _jsx(UsageLine, { usage: chat.usage, client: client, spent: s?.spend.usd })] }));
 }
 function Messages({ chat, status, onSettings, }) {
     const end = useRef(null);
@@ -253,7 +259,7 @@ function Item({ item, onSettle, busy, }) {
     const tierName = useContext(TierNames);
     switch (item.kind) {
         case 'user':
-            return (_jsxs("div", { className: "ml-8 self-end rounded-lg bg-muted px-3 py-2 text-sm whitespace-pre-wrap", children: [(item.tier || item.refs?.length) && (_jsxs("span", { className: "mb-1 flex flex-wrap gap-1", children: [item.tier && _jsx(ComposerChip, { sign: "@", label: tierName(item.tier) }), item.refs?.map((r, i) => (_jsx(ComposerChip, { sign: "#", label: r.label }, `${r.kind}:${r.label}:${i}`)))] })), item.text, item.context && (_jsxs("span", { className: "mt-1 block text-muted-foreground text-xs", children: ["on ", item.context] }))] }));
+            return (_jsxs("div", { className: "ml-8 self-end rounded-lg bg-muted px-3 py-2 text-sm whitespace-pre-wrap", children: [(item.tier || item.refs?.length) && (_jsxs("span", { className: "mb-1 flex flex-wrap gap-1", children: [item.tier && _jsx(ComposerChip, { sign: "@", label: tierName(item.tier) }), item.refs?.map((r, i) => (_jsx(ComposerChip, { sign: "#", label: r.label }, `${r.kind}:${r.label}:${i}`)))] })), item.images?.length ? (_jsx("span", { className: "mb-1 flex flex-wrap gap-1.5", children: item.images.map((src, i) => (_jsx("img", { src: src, alt: `Pasted ${i + 1}`, className: "max-h-32 max-w-40 rounded-md border object-cover" }, `${i}:${src.length}`))) })) : null, item.text, item.context && (_jsxs("span", { className: "mt-1 block text-muted-foreground text-xs", children: ["on ", item.context] }))] }));
         case 'text':
             return (_jsxs("div", { className: "prose-sm text-sm leading-relaxed [&_code]:font-code [&_code]:text-xs [&_li]:ml-4 [&_ol]:list-decimal [&_p]:my-1.5 [&_ul]:list-disc", children: [_jsx(ChatMarkdown, { children: item.text }), item.live && (_jsx("span", { className: "ml-0.5 inline-block h-3.5 w-1.5 animate-pulse bg-foreground/60 align-middle" }))] }));
         case 'step':
@@ -420,17 +426,49 @@ function HandOff({ item, onSettle, busy, }) {
 function ComposerChip({ sign, label, onRemove, }) {
     return (_jsxs("span", { className: "inline-flex max-w-full items-center gap-1 rounded-md border bg-background px-1.5 py-0.5 text-xs", children: [_jsx("span", { className: "text-muted-foreground", children: sign }), _jsx("span", { className: "truncate", children: label }), onRemove && (_jsx("button", { type: "button", "aria-label": `Remove ${label}`, className: "text-muted-foreground hover:text-foreground", onClick: onRemove, children: _jsx(XIcon, { className: "size-3" }) }))] }));
 }
+/** The longest edge an image is sent at (what the models read best; larger only costs more). */
+const IMAGE_EDGE = 1568;
+/**
+ * A pasted or dropped image, ready to send: scaled to `IMAGE_EDGE` on a canvas and saved as JPEG
+ * (PNG when it has few pixels to lose), as base64. `null` when the browser cannot read it.
+ */
+async function readImage(file) {
+    try {
+        const bmp = await createImageBitmap(file);
+        const k = Math.min(1, IMAGE_EDGE / Math.max(bmp.width, bmp.height));
+        const canvas = document.createElement('canvas');
+        canvas.width = Math.max(1, Math.round(bmp.width * k));
+        canvas.height = Math.max(1, Math.round(bmp.height * k));
+        const g = canvas.getContext('2d');
+        if (!g)
+            return null;
+        g.fillStyle = '#fff';
+        g.fillRect(0, 0, canvas.width, canvas.height);
+        g.drawImage(bmp, 0, 0, canvas.width, canvas.height);
+        bmp.close();
+        const png = canvas.toDataURL('image/png');
+        const url = png.length < 400_000 ? png : canvas.toDataURL('image/jpeg', 0.85);
+        const [head, data] = url.split(',');
+        const media_type = head?.includes('png') ? 'image/png' : 'image/jpeg';
+        return data ? { media_type, data } : null;
+    }
+    catch {
+        return null;
+    }
+}
+const imageUrl = (i) => `data:${i.media_type};base64,${i.data}`;
 /**
  * The message box: Enter sends, Shift+Enter breaks a line; Stop while an answer runs. Three
  * triggers list what can follow them as it is typed: `/` at the start a command, `@` a tier for
  * this message, `#` a thing to point at (the `sources`); a picked tier or thing shows as a chip.
  */
-export function LumenComposer({ name, busy, queued, disabled, commands = [], tiers = [], sources = [], page, onSend, onStop, }) {
+export function LumenComposer({ name, busy, queued, disabled, commands = [], tiers = [], sources = [], page, images: canAttach = false, onSend, onStop, }) {
     const [text, setText] = useState('');
     const [caret, setCaret] = useState(0);
     const [closed, setClosed] = useState(null);
     const [tier, setTier] = useState(null);
     const [refs, setRefs] = useState([]);
+    const [images, setImages] = useState([]);
     const list = useRef(null);
     const box = useRef(null);
     const selection = usePageSelection();
@@ -464,12 +502,40 @@ export function LumenComposer({ name, busy, queued, disabled, commands = [], tie
     const extra = () => ({
         ...(tier ? { tier } : {}),
         ...(refs.length ? { refs } : {}),
+        ...(images.length ? { images } : {}),
     });
     const clear = () => {
         setText('');
         setCaret(0);
         setTier(null);
         setRefs([]);
+        setImages([]);
+    };
+    /** Adds the image files among `files`; says so when one is left out. Returns whether any was an image. */
+    const attach = (files) => {
+        const picked = [...files].filter((f) => f.type.startsWith('image/'));
+        if (!canAttach || !picked.length)
+            return false;
+        void (async () => {
+            const read = (await Promise.all(picked.map(readImage))).filter((x) => !!x);
+            if (read.length < picked.length)
+                lumenToast.error("That image couldn't be read. Try a PNG or JPEG.");
+            setImages((p) => {
+                const next = [...p];
+                let chars = next.reduce((n, x) => n + x.data.length, 0);
+                for (const img of read) {
+                    if (next.length >= ASSISTANT_IMAGES_MAX ||
+                        chars + img.data.length > ASSISTANT_IMAGE_CHARS) {
+                        lumenToast.error(`Up to ${ASSISTANT_IMAGES_MAX} images go with one message.`);
+                        break;
+                    }
+                    next.push(img);
+                    chars += img.data.length;
+                }
+                return next;
+            });
+        })();
+        return true;
     };
     const run = async (c, args) => {
         const keep = extra();
@@ -654,10 +720,20 @@ export function LumenComposer({ name, busy, queued, disabled, commands = [], tie
                     const r = sourceItems.get(o.value)?.();
                     if (r)
                         setRefs((p) => [...p, r]);
-                }, onClose: () => setClosed(`${text}|${caret}`) })), (tierLabel || refs.length > 0) && (_jsxs("div", { className: "mb-2 flex flex-wrap gap-1", children: [tierLabel && _jsx(ComposerChip, { sign: "@", label: tierLabel, onRemove: () => setTier(null) }), refs.map((r, i) => (_jsx(ComposerChip, { sign: "#", label: r.label, onRemove: () => setRefs((p) => p.filter((_, j) => j !== i)) }, `${r.kind}:${r.label}:${i}`)))] })), _jsxs("div", { className: "relative", children: [_jsx(Textarea, { ref: box, "aria-label": `Message ${name}`, placeholder: `Ask ${name}…${hints.length ? ` (${hints.join(' ')})` : ''}`, className: "max-h-48 min-h-11 resize-none pr-11", value: text, disabled: disabled, onChange: (e) => {
+                }, onClose: () => setClosed(`${text}|${caret}`) })), images.length > 0 && (_jsx("div", { className: "mb-2 flex flex-wrap gap-1.5", "aria-label": "Images going with your message", children: images.map((img, i) => (_jsxs("span", { className: "relative", children: [_jsx("img", { src: imageUrl(img), alt: `Pasted ${i + 1}`, className: "size-12 rounded-md border object-cover" }), _jsx("button", { type: "button", "aria-label": `Remove image ${i + 1}`, className: "absolute -top-1.5 -right-1.5 grid size-4 place-items-center rounded-full border bg-background text-muted-foreground hover:text-foreground", onClick: () => setImages((p) => p.filter((_, j) => j !== i)), children: _jsx(XIcon, { className: "size-3" }) })] }, `${i}:${img.data.length}`))) })), (tierLabel || refs.length > 0) && (_jsxs("div", { className: "mb-2 flex flex-wrap gap-1", children: [tierLabel && _jsx(ComposerChip, { sign: "@", label: tierLabel, onRemove: () => setTier(null) }), refs.map((r, i) => (_jsx(ComposerChip, { sign: "#", label: r.label, onRemove: () => setRefs((p) => p.filter((_, j) => j !== i)) }, `${r.kind}:${r.label}:${i}`)))] })), _jsxs("div", { className: "relative", children: [_jsx(Textarea, { ref: box, "aria-label": `Message ${name}`, placeholder: `Ask ${name}…${hints.length ? ` (${hints.join(' ')})` : ''}`, className: "max-h-48 min-h-11 resize-none pr-11", value: text, disabled: disabled, onChange: (e) => {
                             setText(e.target.value);
                             setCaret(e.target.selectionStart);
-                        }, onSelect: (e) => setCaret(e.currentTarget.selectionStart), onKeyDown: (e) => {
+                        }, onSelect: (e) => setCaret(e.currentTarget.selectionStart), onPaste: (e) => {
+                            // An image on the clipboard becomes a chip; text pastes as usual.
+                            if (!e.clipboardData.getData('text/plain') && attach(e.clipboardData.files))
+                                e.preventDefault();
+                        }, onDragOver: (e) => {
+                            if (canAttach && [...e.dataTransfer.items].some((x) => x.type.startsWith('image/')))
+                                e.preventDefault();
+                        }, onDrop: (e) => {
+                            if (attach(e.dataTransfer.files))
+                                e.preventDefault();
+                        }, onKeyDown: (e) => {
                             if (picking && list.current?.key(e))
                                 return;
                             if (e.key === 'Enter' && !e.shiftKey && !e.nativeEvent.isComposing) {

package/dist/assistant/use-assistant-chat.js

@@ -168,6 +168,9 @@ export function useAssistantChat(client) {
                     ...(extra?.context ? { context: extra.context.label } : {}),
                     ...(extra?.refs?.length ? { refs: extra.refs } : {}),
                     ...(extra?.tier ? { tier: extra.tier } : {}),
+                    ...(extra?.images?.length
+                        ? { images: extra.images.map((i) => `data:${i.media_type};base64,${i.data}`) }
+                        : {}),
                 },
             ]);
         // Every screen outcome not yet sent goes with this message; kept again if it fails.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@columenlabs/lumen-ui",
-  "version": "0.1.210",
+  "version": "0.1.211",
   "description": "Lumen: shared React components (frame, views, widgets, charts, drawers).",
   "license": "MIT",
   "type": "module",
@@ -49,7 +49,7 @@
     "@codemirror/lang-javascript": "6.2.5",
     "@codemirror/lang-json": "6.0.2",
     "@codemirror/lang-sql": "6.10.0",
-    "@columenlabs/lumen-ui-types": "0.1.210",
+    "@columenlabs/lumen-ui-types": "0.1.211",
     "@dnd-kit/core": "6.3.1",
     "@dnd-kit/sortable": "10.0.0",
     "@dnd-kit/utilities": "3.2.2",