@columenlabs/lumen-ai 0.1.189 → 0.1.191

package/README.md

@@ -102,6 +102,7 @@ app.all('/api/assistant/*', (c) => routes(c.req.raw));
 - **Web.** `fetch` is a plain GET from the Worker (10 s, 2 MB, never the app's own hosts or private addresses). ScrapeQL's own tools join over its MCP API once an admin pastes a Cloudflare Access service token (`PUT /web/scrapeql`). A Worker on ScrapeQL's own Cloudflare account passes its service binding instead, `scrapeql: { url: 'https://scrapeql/mcp', fetch: env.SCRAPEQL.fetch.bind(env.SCRAPEQL) }`: no token is stored, the tools are always on and Settings shows the link with a Test.
 - **Hand off and advice.** The `escalate` tool hands one task to a higher tier (the person sets next, any or only when asked); Anthropic's advisor tool lets a higher tier send a plan, at most two an answer.
 - **Chats** run on the agent (`./agent`, `LumenAgent`, below): the app routes `POST /chat` and each chat's reads, cards and Undo to its own agent, and `assistantRoutes` answers `POST /chat` with 410 `moved`. Chats saved by the old turn loop stay readable (`GET /chats`, `GET|DELETE /chats/:id`) until `assistant.purge()` drops them 30 days after their last message.
+- **Screen moves**: while the person's Settings ▸ Screen is Move (the default), the panel sends its page map (`AssistantMessage.screen`: routes, nav pages, Settings sections, cards on screen, rows to open) and `LumenAgent` gives the model a `screen` tool (`./agent` `screen.ts`). Each call's moves (go, settings, chart, open, point) are checked against the map, kept in the chat and streamed as a `screen` event; the panel runs them through `AssistantScreenProvider` (mounted by `LumenShell`) with an Undo chip, and their outcomes go back with the next message (`outcomes`). Nothing is saved by a move.
 - **Background work** an app runs for a person (a score, a summary) is `assistant.complete(caller, { task, system, prompt, images?, maxTokens })`: one answer on the task's tier (`tasks`, so the person picks its model in Settings ▸ Models), no chat and no tools (`images`, base64 with their media type, are read with the prompt: a scanned page, a screenshot), billed to their payer and counted in their spend and allowance like a chat turn. It throws `AssistantError` `not_connected`, `app_cap` or `own_cap` when nobody can pay, so the caller waits and tries later.
 
 ### Commands, `@` and `#`
@@ -251,3 +252,24 @@ class Chat extends LumenAgent<Env> {
   protected undoWrite(card: AgentCard) { return assistant.undoChange(caller, card); }
 }
 ```
+
+## Workspaces: tabs over recipes, run again on demand
+
+`@columenlabs/lumen-ai/workspaces` is the server the kit's `WorkspacesSection` and `WorkspacePage` call. A workspace is a person's page of tabs; each tab is a table over a **frame**: the rows one **recipe** pulled, the recipe itself and the person's hand edits. A recipe is one call to one of the product's read tools (`{kind: 'call', fn: 'jobs.list', args}`), so running it again needs no model. Nothing runs on a timer: ↻ on a tab runs its recipe, ↻ in the header runs every tab one after another, each within 15 s (`RUN_MS`); a failed run keeps the last rows and says why.
+
+```ts
+import { createWorkspaces, doSql, workspaceMigrations, workspaceRoutes } from '@columenlabs/lumen-ai/workspaces';
+
+// In the store that holds the person's data (a Durable Object's SQLite, or D1 with d1Sql):
+for (const q of workspaceMigrations({ spaces: 'app_ws_spaces', frames: 'app_ws_frames' })) sql.exec(q);
+const store = createWorkspaces({
+  db: doSql(sql),
+  tables: { spaces: 'app_ws_spaces', frames: 'app_ws_frames' },
+  run: (recipe, { owner, signal }) => callReadTool(owner, recipe, signal), // the product's own tools, as the person
+});
+
+// In the Worker, at the assistant's api:
+const routes = workspaceRoutes({ base: '/api/assistant', owner: callerOf, store: () => store });
+```
+
+Budget: one frame is one row (at most 1.5 MB of rows, 10,000 rows, 50 columns), so a run writes one row; every read has a LIMIT (30 workspaces, 12 tabs each, 100 frames). Columns are typed from their values (number, date, text); a list of words, or short text from a small set, becomes a tags column (chips). The kit's `WorkspacesProvider` takes `recipes` to offer + New tab from a recipe in the tab bar.

package/dist/agent/agent.d.ts

@@ -12,7 +12,7 @@
  * An `Assistant` gives both: `assistant.agentTurn(caller, input)` returns a plan and its settle.
  */
 import { type ChatResponseResult, type PrepareStepContext, type Session, type StepConfig, type StepContext, Think, type ThinkModel, type TurnConfig, type TurnContext } from '@cloudflare/think';
-import type { AssistantChat, AssistantEffort, AssistantEvent, AssistantRefNote, ChatUsage, UsagePart } from '@columenlabs/lumen-ui-types/assistant';
+import type { AssistantChat, AssistantEffort, AssistantEvent, AssistantRefNote, AssistantScreenMap, AssistantScreenOutcome, ChatUsage, UsagePart } from '@columenlabs/lumen-ui-types/assistant';
 import { type ModelMessage, type ToolSet } from 'ai';
 import type { ModelDef } from '../assistant/models.js';
 import { type AgentConnector, type ConnectorAt } from './connectors.js';
@@ -72,6 +72,8 @@ export interface AgentTurnPlan {
     };
     /** Dollars the payer has left; at or past it the model gets no more tools. */
     left?: number;
+    /** The person lets the assistant move their screen: the `screen` tool joins when the browser sent its map. */
+    screen?: boolean;
     /** Most model calls in one answer; 8. */
     maxSteps?: number;
     /** Fetch for provider calls (a metered one). */
@@ -121,6 +123,10 @@ export interface AskInput {
     refs?: AssistantRefNote[];
     tier?: string;
     time_zone?: string;
+    /** What the browser can show (Screen on): the `screen` tool and its rules join the turn. */
+    screen?: AssistantScreenMap;
+    /** How earlier screen calls ended in the browser. */
+    outcomes?: AssistantScreenOutcome[];
 }
 /** Where an answer's events go: the app's route, over RPC. */
 export interface AskSink {

package/dist/agent/agent.js

@@ -17,6 +17,7 @@ import { createCompactFunction } from 'agents/sessions';
 import { generateText } from 'ai';
 import { CONNECTOR_GROUP, checkConnectorUrl, connectorGroup, connectorId, connectorSource, } from './connectors.js';
 import { aiModel, planFallback, turnSettings } from './models.js';
+import { screenMapOf, screenNews, screenOutcomesOf, screenRules, screenTool, } from './screen.js';
 import { LAZY_AT, TOOL_SEARCH, lazyTools } from './search.js';
 import { chunkEvents, itemsOf } from './stream.js';
 import { openAiTools, } from './tools.js';
@@ -136,7 +137,7 @@ export class LumenAgent extends Think {
             id,
             title: this.#get('title') ?? '',
             at: this.#get('at') ?? 0,
-            items: itemsOf(await this.getMessages(), this.cards()),
+            items: itemsOf(await this.getMessages(), this.cards(), this.#screens()),
             usage: this.usage(),
         };
     }
@@ -158,18 +159,23 @@ export class LumenAgent extends Think {
         }
         this.#put('at', at);
         await sink.event({ event: 'chat', data: { id: this.name, title } });
+        this.#outcomes(screenOutcomesOf(input.outcomes));
         const meta = {
             at,
             ...(input.context ? { context: { label: input.context.label } } : {}),
             ...(input.refs?.length ? { refs: input.refs } : {}),
             ...(input.tier ? { tier: input.tier } : {}),
         };
-        const map = chunkEvents({
+        const toEvents = chunkEvents({
             ...(this.#last?.plan.advisor ? { advisor: this.#last.plan.advisor.label } : {}),
             cards: () => new Map(this.cards().map((c) => [c.id, c])),
+            screens: () => this.#screens(),
             kindOf: (n) => this.#turn?.tools.info.find((i) => i.name === n)?.kind,
         });
-        this.#input = input;
+        const map = screenMapOf(input.screen);
+        this.#input = { ...input, ...(map ? { screen: map } : {}) };
+        if (!map)
+            delete this.#input.screen;
         let failed = false;
         try {
             await this.chat({
@@ -187,7 +193,7 @@ export class LumenAgent extends Think {
                     catch {
                         return;
                     }
-                    for (const e of map(chunk))
+                    for (const e of toEvents(chunk))
                         await sink.event(e);
                 },
                 onDone: () => undefined,
@@ -240,6 +246,54 @@ export class LumenAgent extends Think {
         this
             .sql `UPDATE lumen_cards SET status = ${status}, result = COALESCE(${o.result === undefined ? null : JSON.stringify(o.result).slice(0, 20_000)}, result), error = ${o.error ?? null}, told = 0 WHERE id = ${id}`;
     }
+    #screensTabled = false;
+    #screenTable() {
+        if (!this.#screensTabled) {
+            this
+                .sql `CREATE TABLE IF NOT EXISTS lumen_screens (id TEXT PRIMARY KEY, label TEXT NOT NULL, moves TEXT NOT NULL, status TEXT NOT NULL, text TEXT, told INTEGER NOT NULL DEFAULT 1, at INTEGER NOT NULL)`;
+            this.#screensTabled = true;
+        }
+    }
+    /** The chat's screen calls (at most 200), by id. */
+    #screens() {
+        this.#screenTable();
+        const rows = this.sql `SELECT id, label, moves, status, text FROM lumen_screens ORDER BY at DESC LIMIT 200`;
+        return new Map(rows.map((r) => [
+            r.id,
+            {
+                id: r.id,
+                label: r.label,
+                moves: JSON.parse(r.moves),
+                status: r.status,
+                ...(r.text ? { text: r.text } : {}),
+            },
+        ]));
+    }
+    /** How screen calls ended in the browser: kept, and told to the model with its next turn. */
+    #outcomes(list) {
+        if (!list.length)
+            return;
+        this.#screenTable();
+        for (const o of list)
+            this
+                .sql `UPDATE lumen_screens SET status = ${o.status}, text = ${o.text ?? null}, told = 0 WHERE id = ${o.id} AND (status != ${o.status} OR COALESCE(text, '') != ${o.text ?? ''})`;
+    }
+    /** Screen calls whose end the model has not heard, as its note; marks them told. */
+    #screenNews() {
+        this.#screenTable();
+        const rows = this.sql `SELECT id, label, moves, status, text FROM lumen_screens WHERE told = 0 ORDER BY at LIMIT 20`;
+        if (!rows.length)
+            return '';
+        for (const r of rows)
+            this.sql `UPDATE lumen_screens SET told = 1 WHERE id = ${r.id}`;
+        return screenNews(rows.map((r) => ({
+            id: r.id,
+            label: r.label,
+            moves: JSON.parse(r.moves),
+            status: r.status,
+            ...(r.text ? { text: r.text } : {}),
+        })));
+    }
     #connectorClient() {
         return {
             listTools: () => this.getMcpServers().tools,
@@ -436,7 +490,19 @@ export class LumenAgent extends Think {
                 maxUses: plan.advisor.maxUses,
                 maxTokens: plan.advisor.maxTokens,
             });
-        const tools = { ...opened.tools, ...more, ...advisor };
+        const shown = plan.screen ? this.#input?.screen : undefined;
+        const screen = shown
+            ? screenTool(shown, (c) => {
+                this.#screenTable();
+                this
+                    .sql `INSERT OR REPLACE INTO lumen_screens (id, label, moves, status, told, at) VALUES (${c.id}, ${c.label}, ${JSON.stringify(c.moves)}, 'sent', 1, ${Date.now()})`;
+            })
+            : {};
+        if (shown) {
+            system = `${system}\n\n${screenRules(shown)}`;
+            this.#turn.system = system;
+        }
+        const tools = { ...opened.tools, ...more, ...advisor, ...screen };
         let lazy;
         if (Object.keys(tools).length > (plan.lazy?.at ?? LAZY_AT)) {
             const groups = new Map(opened.info.map((i) => [i.name, i.group]));
@@ -444,6 +510,7 @@ export class LumenAgent extends Think {
                 ...opened.info.filter((i) => i.find).map((i) => i.name),
                 ...Object.keys(more ?? {}),
                 ...Object.keys(advisor),
+                ...Object.keys(screen),
             ];
             if (plan.first && !core.includes(plan.first))
                 core.push(plan.first);
@@ -464,7 +531,7 @@ export class LumenAgent extends Think {
                 turn.meters.push(turn.main);
             }),
             instructions: system,
-            messages: withLead(ctx.messages, `${this.#news()}${plan.lead ?? ''}`),
+            messages: withLead(ctx.messages, `${this.#news()}${this.#screenNews()}${plan.lead ?? ''}`),
             tools,
             ...(lazy ? { activeTools: lazy.active() } : {}),
             ...(plan.first ? { toolChoice: { type: 'tool', toolName: plan.first } } : {}),

package/dist/agent/index.d.ts

@@ -7,6 +7,7 @@ export * from './agent.js';
 export * from './connectors.js';
 export * from './models.js';
 export * from './respond.js';
+export * from './screen.js';
 export * from './search.js';
 export * from './stream.js';
 export * from './tools.js';

package/dist/agent/index.js

@@ -7,6 +7,7 @@ export * from './agent.js';
 export * from './connectors.js';
 export * from './models.js';
 export * from './respond.js';
+export * from './screen.js';
 export * from './search.js';
 export * from './stream.js';
 export * from './tools.js';

package/dist/agent/screen.d.ts

@@ -0,0 +1,63 @@
+/**
+ * The assistant moves the person's screen: one tool, `screen`, takes a short list of moves the
+ * browser runs (`@columenlabs/lumen-ui`'s `AssistantScreenProvider`): a page of the app, Settings
+ * at a section (pulsing a row, never changing it), a symbol's chart, a row's details, a card
+ * pointed at. The browser sends its page map with each message; every move is checked against it
+ * here before the call reaches the screen, and the browser checks again. How each call ended comes
+ * back with the next message, so the model never guesses. Settings ▸ Screen: Off leaves the tool out.
+ */
+import type { AssistantScreenMap, AssistantScreenMove, AssistantScreenOutcome } from '@columenlabs/lumen-ui-types/assistant';
+import { type ToolSet } from 'ai';
+import { z } from 'zod';
+export declare const screenInput: z.ZodObject<{
+    label: z.ZodPipe<z.ZodString, z.ZodTransform<string, string>>;
+    moves: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
+        go: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        settings: z.ZodString;
+        row: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>, z.ZodObject<{
+        chart: z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>;
+        indicators: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
+            label: z.ZodString;
+            f: z.ZodString;
+            where: z.ZodOptional<z.ZodEnum<{
+                overlay: "overlay";
+                pane: "pane";
+            }>>;
+            refs: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
+        }, z.core.$strict>]>>>;
+    }, z.core.$strict>, z.ZodObject<{
+        open: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        point: z.ZodString;
+    }, z.core.$strict>]>>;
+}, z.core.$strip>;
+/** Bounds on a page map from the browser, so a bad one costs nothing. */
+export declare function screenMapOf(v: unknown): AssistantScreenMap | undefined;
+/** Outcomes from the browser, checked. */
+export declare function screenOutcomesOf(v: unknown): AssistantScreenOutcome[];
+/**
+ * Each move checked against the page map: an unknown page, Settings section or card is an error
+ * back to the model in the same turn. A card after `go` is checked by the browser alone.
+ */
+export declare function checkMoves(moves: AssistantScreenMove[], map: AssistantScreenMap): string | null;
+/** The rules and this app's page map, for the instructions. */
+export declare function screenRules(map: AssistantScreenMap): string;
+/** One screen call as the chat keeps it. */
+export interface ScreenCall {
+    id: string;
+    label: string;
+    moves: AssistantScreenMove[];
+    status: 'sent' | AssistantScreenOutcome['status'];
+    text?: string;
+}
+/** The calls the model has not heard about since the browser ran them or the person pressed Undo. */
+export declare function screenNews(calls: ScreenCall[]): string;
+export declare const screenId: () => string;
+/** The `screen` tool for one turn: checks the moves, keeps the call and answers with its id. */
+export declare function screenTool(map: AssistantScreenMap, keep: (call: {
+    id: string;
+    label: string;
+    moves: AssistantScreenMove[];
+}) => void): ToolSet;

package/dist/agent/screen.js

@@ -0,0 +1,178 @@
+import { indicatorSpecIssue, indicatorSpecSchema } from '@columenlabs/lumen-ui-types/chart-formula';
+import { tool } from 'ai';
+import { z } from 'zod';
+const ticker = z
+    .string()
+    .trim()
+    .regex(/^[A-Za-z0-9.^$=:/-]{1,24}$/);
+const move = z.union([
+    z.object({ go: z.string().trim().min(1).max(600) }).strict(),
+    z
+        .object({
+        settings: z.string().trim().min(1).max(80),
+        row: z.string().trim().min(1).max(120).optional(),
+    })
+        .strict(),
+    z
+        .object({
+        chart: z.union([ticker, z.array(ticker).min(1).max(8)]),
+        indicators: z.array(indicatorSpecSchema).max(8).optional(),
+    })
+        .strict(),
+    z.object({ open: z.string().trim().min(1).max(200) }).strict(),
+    z.object({ point: z.string().trim().min(1).max(200) }).strict(),
+]);
+export const screenInput = z.object({
+    label: z
+        .string()
+        .describe('What the screen will show, in a few words, e.g. "Settings · Email".')
+        .transform((s) => s.replace(/\s+/g, ' ').trim().slice(0, 80)),
+    moves: z
+        .array(move)
+        .min(1)
+        .max(8)
+        .describe('In order. Each is one of: {go: "/path"} (a page; fill path params), {settings: "<section id>", row?: "<row label>"}, {chart: "SYM"} or {chart: ["A", "B"]} (up to 8 on one chart), {open: "<row title>"} (the row\'s details in the view on screen), {point: "<card id>"} (scroll to a card and highlight it).'),
+});
+/** Bounds on a page map from the browser, so a bad one costs nothing. */
+export function screenMapOf(v) {
+    if (!v || typeof v !== 'object')
+        return undefined;
+    const o = v;
+    const list = (x, n, f) => (Array.isArray(x) ? x : [])
+        .slice(0, n)
+        .flatMap((i) => (i && typeof i === 'object' ? [f(i)] : []))
+        .filter((i) => i !== null);
+    const str = (x, n) => (typeof x === 'string' && x ? x.slice(0, n) : null);
+    return {
+        routes: (Array.isArray(o.routes) ? o.routes : [])
+            .slice(0, 200)
+            .flatMap((r) => (typeof r === 'string' ? [r.slice(0, 300)] : [])),
+        pages: list(o.pages, 300, (p) => {
+            const path = str(p.path, 300);
+            const title = str(p.title, 120);
+            const section = str(p.section, 80);
+            return path && title ? { path, title, ...(section ? { section } : {}) } : null;
+        }),
+        settings: list(o.settings, 60, (p) => {
+            const id = str(p.id, 80);
+            const title = str(p.title, 120);
+            return id && title ? { id, title } : null;
+        }),
+        cards: list(o.cards, 60, (p) => {
+            const id = str(p.id, 200);
+            const title = str(p.title, 200) ?? '';
+            const kind = str(p.kind, 40);
+            return id ? { id, title, ...(kind ? { kind } : {}) } : null;
+        }),
+        rows: (Array.isArray(o.rows) ? o.rows : [])
+            .slice(0, 30)
+            .flatMap((r) => (typeof r === 'string' && r ? [r.slice(0, 120)] : [])),
+    };
+}
+/** Outcomes from the browser, checked. */
+export function screenOutcomesOf(v) {
+    return (Array.isArray(v) ? v : []).slice(0, 50).flatMap((o) => {
+        const x = o;
+        if (!x || typeof x.id !== 'string' || !/^sc_[a-f0-9]{16}$/.test(x.id))
+            return [];
+        if (!['done', 'failed', 'waiting', 'undone'].includes(String(x.status)))
+            return [];
+        return [
+            {
+                id: x.id,
+                status: x.status,
+                ...(typeof x.text === 'string' && x.text ? { text: x.text.slice(0, 1200) } : {}),
+            },
+        ];
+    });
+}
+/** The pattern's segments match the path's: `$param` matches any one segment. */
+function matches(pattern, path) {
+    const a = pattern.split('/').filter(Boolean);
+    const b = path.split('/').filter(Boolean);
+    if (a.length !== b.length)
+        return false;
+    return a.every((s, i) => s.startsWith('$') || s === b[i]);
+}
+/**
+ * Each move checked against the page map: an unknown page, Settings section or card is an error
+ * back to the model in the same turn. A card after `go` is checked by the browser alone.
+ */
+export function checkMoves(moves, map) {
+    let moved = false;
+    for (const m of moves) {
+        if ('go' in m) {
+            const path = m.go.split(/[?#]/)[0] ?? '';
+            if (!path.startsWith('/') || /\$/.test(path))
+                return `go "${m.go}": give a path of this app with its params filled`;
+            if (!map.routes.some((r) => matches(r, path)))
+                return `go "${m.go}": no such page. Routes: ${map.routes.join(', ')}`;
+            moved = true;
+        }
+        else if ('chart' in m) {
+            for (const spec of m.indicators ?? []) {
+                const wrong = indicatorSpecIssue(spec);
+                if (wrong)
+                    return `chart indicators: ${wrong}`;
+            }
+        }
+        else if ('settings' in m) {
+            if (!map.settings.some((s) => s.id === m.settings))
+                return `settings "${m.settings}": no such section. Sections: ${map.settings.map((s) => s.id).join(', ')}`;
+        }
+        else if ('point' in m && !moved) {
+            const id = m.point.replace(/^card:/, '');
+            if (!map.cards.some((c) => c.id === id))
+                return `point "${m.point}": no such card on this page. Cards: ${map.cards.map((c) => c.id).join(', ') || 'none'}`;
+        }
+    }
+    return null;
+}
+/** The rules and this app's page map, for the instructions. */
+export function screenRules(map) {
+    const pages = map.pages
+        .map((p) => `${p.path} (${p.section ? `${p.section} ▸ ` : ''}${p.title})`)
+        .join('; ');
+    return `Screen: with the screen tool you move the person's screen in their browser.
+- Use it when the person asks to see, open, show, go to or be taken somewhere ("show me", "take me to", "where do I set X"). Otherwise answer in words.
+- One call with every move, in order. After go, the next moves act on the new page; open and point act on the page on screen after the moves before them.
+- Moves only change the screen: nothing is saved. Never say something was saved or changed. Say in one line what you put on screen.
+- Settings: you open a section and point at a row; you never flip a switch or change a value. Say which one the person taps.
+- A line "Your screen moves since your last answer" tells you how each call ended (done, failed and why, waiting for the person's tap, or undone) and what the view shows now: answer from it.
+Routes: ${map.routes.join(', ') || '(none)'}
+Pages: ${pages || '(none listed)'}
+Settings sections: ${map.settings.map((s) => `${s.id} (${s.title})`).join(', ') || '(none)'}
+Cards on this page: ${map.cards.map((c) => `${c.id} (${c.title})`).join(', ') || '(none)'}${map.rows?.length ? `\nRows you can open here: ${map.rows.join('; ')}` : ''}`;
+}
+const SAID = {
+    sent: 'sent; no word back yet',
+    done: 'done',
+    failed: 'failed',
+    waiting: 'not run yet: the person was mid-edit; it runs when they tap it',
+    undone: 'undone by the person',
+};
+/** The calls the model has not heard about since the browser ran them or the person pressed Undo. */
+export function screenNews(calls) {
+    if (!calls.length)
+        return '';
+    const lines = calls.map((c) => `- "${c.label}" ${JSON.stringify(c.moves).slice(0, 300)}: ${SAID[c.status]}${c.text ? `. ${c.status === 'failed' ? 'Why' : 'On screen now'}: ${c.text}` : ''}`);
+    return `Your screen moves since your last answer:\n${lines.join('\n')}\n\n`;
+}
+export const screenId = () => `sc_${crypto.randomUUID().replaceAll('-', '').slice(0, 16)}`;
+/** The `screen` tool for one turn: checks the moves, keeps the call and answers with its id. */
+export function screenTool(map, keep) {
+    return {
+        screen: tool({
+            description: "Move the person's screen: go to a page, open Settings at a section, open a symbol's chart, open a row's details or point at a card. Moves run in order in their browser at once, with an Undo. Use the routes, pages, Settings sections and cards listed under Screen. The outcome comes with the person's next message.",
+            inputSchema: screenInput,
+            execute: async ({ label, moves }) => {
+                const wrong = checkMoves(moves, map);
+                if (wrong)
+                    return { error: wrong };
+                const id = screenId();
+                keep({ id, label, moves: moves });
+                return { screen: id, label, note: 'Sent to the screen. Say in one line what it shows.' };
+            },
+        }),
+    };
+}

package/dist/agent/stream.d.ts

@@ -5,6 +5,7 @@
  */
 import type { AssistantEvent, AssistantRefNote, ChatItem } from '@columenlabs/lumen-ui-types/assistant';
 import type { AgentCard } from './agent.js';
+import type { ScreenCall } from './screen.js';
 /** A tool's name as a line's title: `notes__find_notes` → `find notes`. */
 export declare function stepTitle(name: string): string;
 /**
@@ -14,6 +15,8 @@ export declare function stepTitle(name: string): string;
 export declare function chunkEvents(o?: {
     advisor?: string;
     cards?: () => Map<string, AgentCard>;
+    /** The chat's screen calls by id: a `screen` tool's result names its call. */
+    screens?: () => Map<string, ScreenCall>;
     /** A tool's kind this turn: a draft shows as its card only, with no line before it. */
     kindOf?: (name: string) => 'read' | 'draft' | 'write' | undefined;
 }): (chunk: {
@@ -39,5 +42,5 @@ export interface AskMeta {
     tier?: string;
 }
 /** A stored chat as the panel's items: the messages in order, each tool call as it ended. */
-export declare function itemsOf(messages: StoredMessage[], cards: AgentCard[]): ChatItem[];
+export declare function itemsOf(messages: StoredMessage[], cards: AgentCard[], screens?: Map<string, ScreenCall>): ChatItem[];
 export {};

package/dist/agent/stream.js

@@ -6,8 +6,24 @@ export function stepTitle(name) {
 }
 const obj = (v) => v && typeof v === 'object' && !Array.isArray(v) ? v : null;
 /** What one finished tool call shows as: a line, a card, a hand-off section or nothing. */
-function settledCall(id, c, out, ms, cards) {
+function settledCall(id, c, out, ms, cards, screens = new Map()) {
     const o = obj(out.output);
+    if (c.name === 'screen') {
+        // A call the page map refused goes back to the model only; the person sees the one that ran.
+        const call = typeof o?.screen === 'string' ? screens.get(o.screen) : undefined;
+        if (!call)
+            return [];
+        const item = {
+            kind: 'screen',
+            id: call.id,
+            label: call.label,
+            moves: call.moves,
+            ...(call.status !== 'sent'
+                ? { outcome: { status: call.status, ...(call.text ? { text: call.text } : {}) } }
+                : {}),
+        };
+        return [{ event: 'screen', data: item }];
+    }
     const card = cards.get(id);
     if (card?.kind === 'draft' || (o && typeof o.card === 'string')) {
         const draft = {
@@ -73,7 +89,7 @@ export function chunkEvents(o = {}) {
                     advising = true;
                     return [{ event: 'advice', data: { tier: 'advisor', label: o.advisor ?? 'Advice' } }];
                 }
-                if (name === 'escalate' || o.kindOf?.(name) === 'draft')
+                if (name === 'escalate' || name === 'screen' || o.kindOf?.(name) === 'draft')
                     return [];
                 return [{ event: 'step_start', data: { id, tool: name, title: stepTitle(name) } }];
             }
@@ -93,7 +109,7 @@ export function chunkEvents(o = {}) {
                     advising = false;
                     return [{ event: 'advice_done', data: error ? { error } : {} }];
                 }
-                return settledCall(id, c, error !== undefined ? { error } : { output: chunk.output }, Date.now() - c.t0, o.cards?.() ?? new Map());
+                return settledCall(id, c, error !== undefined ? { error } : { output: chunk.output }, Date.now() - c.t0, o.cards?.() ?? new Map(), c.name === 'screen' ? (o.screens?.() ?? new Map()) : undefined);
             }
             case 'error':
                 return [
@@ -112,7 +128,7 @@ export function chunkEvents(o = {}) {
     };
 }
 /** A stored chat as the panel's items: the messages in order, each tool call as it ended. */
-export function itemsOf(messages, cards) {
+export function itemsOf(messages, cards, screens = new Map()) {
     const byId = new Map(cards.map((c) => [c.id, c]));
     const items = [];
     for (const m of messages) {
@@ -161,7 +177,7 @@ export function itemsOf(messages, cards) {
                     : null;
             if (!ended)
                 continue;
-            for (const e of settledCall(p.toolCallId, { name: tool, input: p.input, t0: 0 }, ended, 0, byId)) {
+            for (const e of settledCall(p.toolCallId, { name: tool, input: p.input, t0: 0 }, ended, 0, byId, screens)) {
                 if (e.event === 'handoff')
                     items.push({ ...e.data, items: [] });
                 else if (e.event === 'handoff_done') {
@@ -169,7 +185,7 @@ export function itemsOf(messages, cards) {
                     if (h?.kind === 'handoff')
                         h.usage = e.data.usage;
                 }
-                else if (e.event === 'step' || e.event === 'draft') {
+                else if (e.event === 'step' || e.event === 'draft' || e.event === 'screen') {
                     const { handoff: _h, ...item } = e.data;
                     items.push(item);
                 }

package/dist/assistant/engine.js

@@ -284,6 +284,7 @@ export class Assistant {
             ...(fallback ? { fallback } : {}),
             left: paying.left,
             maxSteps: MAX_ROUNDS,
+            screen: (p.screen ?? 'move') === 'move',
             ...(this.o.fetch ? { fetch: this.o.fetch } : {}),
             ...(handTo && reach.length ? { first: 'escalate' } : {}),
             ...(advisor
@@ -551,6 +552,7 @@ export class Assistant {
             })),
             hand_off: p.hand_off ?? 'next',
             advice: p.advice && (p.advice === 'off' || this.tierById(p.advice)) ? p.advice : 'off',
+            screen: p.screen ?? 'move',
             web: {
                 fetch: p.web ?? true,
                 scrapeql: this.o.web?.scrapeql

package/dist/assistant/routes.js

@@ -5,7 +5,7 @@
  *   app.all('/api/assistant/*', (c) => assistant(c.req.raw));   // Hono
  *
  *   GET    /status                  what the panel and Settings show
- *   PUT    /settings                the person's tiers, tasks, hand off, advice, tool access, web, payer
+ *   PUT    /settings                the person's tiers, tasks, hand off, advice, screen, tool access, web, payer
  *   PUT    /app {cap_usd}           admin: what each person may spend a month on the app's keys
  *   PUT    /key {api_key}           admin: the app's key of `?provider=` (anthropic by default,
  *                                   or openai), tested before it is kept
@@ -30,7 +30,7 @@
  * form or fetch is refused before it reaches a rule. Keys and tokens are write-only: no route
  * returns one, only a hint (`sk-ant-…abcd`).
  */
-import { ASSISTANT_ACCESS, ASSISTANT_EFFORTS, ASSISTANT_OWN_WAYS, ASSISTANT_PAYERS, HAND_OFF_POLICIES, } from '@columenlabs/lumen-ui-types/assistant';
+import { ASSISTANT_ACCESS, ASSISTANT_EFFORTS, ASSISTANT_OWN_WAYS, ASSISTANT_PAYERS, ASSISTANT_SCREEN, HAND_OFF_POLICIES, } from '@columenlabs/lumen-ui-types/assistant';
 import { AssistantError } from './engine.js';
 import { PROVIDERS } from './models.js';
 import { keyDocName, planDocName, } from './state.js';
@@ -89,6 +89,8 @@ function settingsChange(a, groups, b) {
         return 'No such hand off.';
     if (p.advice !== undefined && p.advice !== 'off' && !tierIds.has(p.advice))
         return 'Advice is off or a tier.';
+    if (p.screen !== undefined && !oneOf(ASSISTANT_SCREEN, p.screen))
+        return 'screen is off or move.';
     if (p.access !== undefined) {
         if (!p.access || typeof p.access !== 'object')
             return 'access is {group: off | read | draft}.';
@@ -127,6 +129,7 @@ function settingsChange(a, groups, b) {
             ...(p.tasks !== undefined ? { tasks: { ...(d.tasks ?? {}), ...p.tasks } } : {}),
             ...(p.hand_off !== undefined ? { hand_off: p.hand_off } : {}),
             ...(p.advice !== undefined ? { advice: p.advice } : {}),
+            ...(p.screen !== undefined ? { screen: p.screen } : {}),
             ...(p.access !== undefined ? { access: { ...(d.access ?? {}), ...p.access } } : {}),
             ...(p.web !== undefined ? { web: p.web.fetch } : {}),
             ...(p.pay !== undefined ? { pay: p.pay } : {}),

package/dist/assistant/state.d.ts

@@ -19,7 +19,7 @@
  * tier runs on and money left this month, else the other; with neither the assistant waits and says
  * why. A cap covers every provider of its payer.
  */
-import type { AllowanceRequest, AssistantAccess, AssistantEffort, AssistantOwnWay, AssistantPayer, AssistantProvider, AssistantWait, HandOffPolicy, UsagePart } from '@columenlabs/lumen-ui-types/assistant';
+import type { AllowanceRequest, AssistantAccess, AssistantEffort, AssistantOwnWay, AssistantPayer, AssistantProvider, AssistantScreenSetting, AssistantWait, HandOffPolicy, UsagePart } from '@columenlabs/lumen-ui-types/assistant';
 import { type ChatgptTokens } from './chatgpt.js';
 import { type Docs } from './store.js';
 export interface KeyTest {
@@ -59,6 +59,8 @@ export interface PersonDoc {
     tasks?: Record<string, string>;
     hand_off?: HandOffPolicy;
     advice?: string;
+    /** Whether the assistant may move the screen; Move when unset. */
+    screen?: AssistantScreenSetting;
     access?: Record<string, AssistantAccess>;
     web?: boolean;
     pay?: AssistantPayer;