@meetopenbot/openbot 1.6.2 → 1.7.0

package/dist/build-tools.js

@@ -8,6 +8,7 @@ import { threadStatusTools } from "./tools/thread-status.js";
 import { threadTitleTools } from "./tools/thread-title.js";
 import { previewTools } from "./tools/preview.js";
 import { variablesRequestTools } from "./tools/variables-request.js";
+import { renderWidgetTools } from "./tools/render-widget.js";
 import { wrapToolsWithApproval } from "./tools/approval.js";
 export const DEFAULT_APPROVAL_ACTIONS = ["update_channel_spec", "start_thread"];
 /**
@@ -26,6 +27,7 @@ export function buildOpenbotTools(args) {
         ...threadTitleTools,
         ...previewTools,
         ...variablesRequestTools,
+        ...renderWidgetTools,
         ...args.context.tools,
     };
     const approvalConfig = args.context.config?.approval;

package/dist/history.js

@@ -1,3 +1,4 @@
+import { compactRenderWidgetInput, renderedWidgetReplyText } from './tools/render-widget.js';
 function asReasoningPart(part) {
     const text = typeof part.text === 'string' ? part.text : '';
     if (!text.trim() && !part.providerOptions)
@@ -201,6 +202,12 @@ export function eventsToModelMessages(events, options) {
                 }
                 break;
             }
+            case 'client:ui:widget:response': {
+                const reply = renderedWidgetReplyText(event.data);
+                if (reply)
+                    messages.push({ role: 'user', content: reply });
+                break;
+            }
             default:
                 // Handle tool calls (action:*)
                 if (event.type.startsWith('action:') && !event.type.endsWith(':result')) {
@@ -212,7 +219,9 @@ export function eventsToModelMessages(events, options) {
                         type: 'tool-call',
                         toolCallId,
                         toolName,
-                        input: event.data,
+                        input: toolName === 'render_widget'
+                            ? compactRenderWidgetInput(event.data)
+                            : event.data,
                     };
                     const last = messages[messages.length - 1];
                     if (last && last.role === 'assistant') {

package/dist/index.js

@@ -12,6 +12,7 @@ import { threadStatusTools } from './tools/thread-status.js';
 import { threadTitleTools } from './tools/thread-title.js';
 import { previewTools } from './tools/preview.js';
 import { variablesRequestTools } from './tools/variables-request.js';
+import { renderWidgetTools } from './tools/render-widget.js';
 import { buildOpenbotTools } from './build-tools.js';
 export const OPENBOT_PLUGIN_ID = '@meetopenbot/openbot';
 const modelField = await modelConfigField({
@@ -53,6 +54,7 @@ export const openbotPlugin = defineOpenbotPlugin({
         ...threadTitleTools,
         ...previewTools,
         ...variablesRequestTools,
+        ...renderWidgetTools,
     },
     async *run(args) {
         const { context } = args;

package/dist/runtime.js

@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
 import { generateText, hasToolCall, streamText, stepCountIs } from 'ai';
 import { BYOK_ENV_KEYS, isAuthErrorMessage, isCreditsErrorMessage, fetchModelRegistry, getProviderModelOptions, listApiKeyProviders, PROVIDER_API_KEY_LINKS, } from '@meetopenbot/plugin-sdk';
 import { eventsToModelMessages } from './history.js';
+import { RENDER_WIDGET_METADATA_TYPE, settledRenderedWidget } from './tools/render-widget.js';
 import { addAttachmentParts, chooseTurnCandidates, eventsHaveAttachments, modelSupportsImages, } from './images.js';
 import { buildContext } from './context.js';
 import { OPENBOT_SYSTEM_PROMPT, withGithubGitInstructions, withUserInstructions, } from './system-prompt.js';
@@ -438,6 +439,18 @@ export async function* handleOpenbotWidgetResponse(args, options) {
     const { host, storage } = options;
     let configuredModelString = options.model ?? AUTO_MODEL_ID;
     const state = args.state;
+    // A click on a card the model rendered is the human's next message: the
+    // harness already stored it, and history replays it as a user turn.
+    if (metadata?.type === RENDER_WIDGET_METADATA_TYPE) {
+        if (storage) {
+            const events = await storage.getEvents({ channelId: state.channelId ?? '', threadId });
+            const settled = settledRenderedWidget(events, event.data);
+            if (settled)
+                yield settled;
+        }
+        yield* runOpenbotTurn(args, options);
+        return;
+    }
     if (host.isCloudSystemAgent(state.agentId) && options.authMode === 'credits')
         return;
     if (metadata?.type === 'api_provider_selection') {
@@ -459,7 +472,6 @@ export async function* handleOpenbotWidgetResponse(args, options) {
                 body: `${label} provider was selected.`,
                 state: 'submitted',
                 display: 'collapsed',
-                disabled: true,
                 actions: [],
             },
             meta: { agentId: state.agentId, threadId },
@@ -536,7 +548,6 @@ export async function* handleOpenbotWidgetResponse(args, options) {
                 body: `${modelId} was selected.`,
                 state: 'submitted',
                 display: 'collapsed',
-                disabled: true,
                 actions: [],
             },
             meta: { agentId: state.agentId, threadId },

package/dist/system-prompt.js

@@ -28,6 +28,12 @@ export const OPENBOT_SYSTEM_PROMPT = [
     "- The current list is injected into context each turn as `## TODOS`; call `todo_read` only if you need an explicit refresh.",
     "- Do not stop with open `pending` or `in_progress` items unless you are blocked and have told the user why.",
     "",
+    "# WIDGETS",
+    "- Call `render_widget` when your own answer is structured: 3+ similar records (Spaces, agents, options, statuses), a side-by-side comparison, a decision with fixed choices, or a few fields you need from the human. Use plain text for everything else.",
+    "- Never use it to restate or re-format what a specialist already said in this thread; that rule from Close still applies.",
+    "- Do not repeat the widget's contents in text. A click or form submission arrives as the human's next message, quoting what they picked.",
+    "- Forms never ask for secrets; use `request_variables` for those.",
+    "",
     "# JOB TITLE",
     "- Unnamed jobs get a system hint to call `set_thread_title` once with a concise, human-readable title.",
     "- Call `set_thread_title` again only if the topic clearly changes.",
@@ -54,6 +60,7 @@ export const OPENBOT_SYSTEM_PROMPT = [
     "- You see an image only when it is shown to you. You cannot open other files (PDF, documents, spreadsheets, archives, code).",
     "- Never claim you read a file or saw an image you were not shown.",
     "- When a coding agent should use a file, `ask_agent` them with the exact full path(s) and ask them to open the file there. Do not paraphrase or shorten the path.",
+    "- When one agent's result is a file another agent needs (for example a draft a coding agent wrote that Resend should send), pass the path exactly as the first agent reported it and ask the next agent to read it there. Do not paste or retype the file's contents yourself.",
     "- If an image was not shown to you and no specialist fits, say the current model cannot view images and suggest picking a vision-capable model.",
     "",
     "# CLOUD COMPUTER",

package/dist/tools/render-widget.js

@@ -0,0 +1,317 @@
+import { randomUUID } from 'node:crypto';
+import z from 'zod';
+import { uiWidgetIssue, } from '@meetopenbot/plugin-sdk';
+/**
+ * `render_widget` — the coordinator shows structured data as an inline card
+ * instead of markdown. Clicks and form submits come back as a new turn
+ * (`handleOpenbotWidgetResponse` → `history.ts`), tagged by this metadata type.
+ */
+export const RENDER_WIDGET_METADATA_TYPE = 'render_widget';
+const MAX_COLUMNS = 12;
+const MAX_ROWS = 200;
+const MAX_ITEMS = 50;
+const MAX_ACTIONS = 6;
+/** Ids the client runs itself (navigate, open_url, …); a model action must not collide. */
+const RESERVED_ACTION_IDS = new Set(['open_preview', 'open_url', 'navigate', 'start_in_space']);
+const actionSchema = z.object({
+    id: z.string().min(1).max(64).describe('Stable id returned to you when clicked, e.g. "approve".'),
+    label: z.string().min(1).max(40).describe('Button text.'),
+    variant: z.enum(['primary', 'secondary', 'danger']).optional(),
+});
+const cellSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]);
+const inputSchema = z.object({
+    kind: z
+        .enum(['table', 'list', 'choice', 'form', 'media'])
+        .describe('table: rows × columns of data. list: records with status/badge and optional per-item buttons. choice: a question answered with buttons. form: structured input. media: images/videos/files.'),
+    title: z.string().min(1).max(120),
+    description: z.string().max(400).optional().describe('One short line under the title.'),
+    summary: z
+        .string()
+        .min(1)
+        .max(400)
+        .describe('One or two sentences on what the widget shows. Kept in history in place of the full data, so include what you will need to refer back to.'),
+    columns: z
+        .array(z.object({
+        id: z.string().min(1),
+        label: z.string(),
+        align: z.enum(['left', 'right', 'center']).optional().describe('Use right for numbers.'),
+    }))
+        .max(MAX_COLUMNS)
+        .optional()
+        .describe('table only.'),
+    rows: z
+        .array(z.array(cellSchema))
+        .max(MAX_ROWS)
+        .optional()
+        .describe('table only. Each row lists one cell per column, in column order.'),
+    items: z
+        .array(z.object({
+        id: z.string().min(1),
+        label: z.string().min(1),
+        description: z.string().optional(),
+        badge: z.string().optional(),
+        status: z.string().optional(),
+        statusVariant: z.enum(['default', 'success', 'warning', 'danger', 'info']).optional(),
+        actions: z.array(actionSchema).max(3).optional(),
+    }))
+        .max(MAX_ITEMS)
+        .optional()
+        .describe('list only.'),
+    fields: z
+        .array(z.object({
+        id: z.string().min(1),
+        label: z.string().min(1),
+        type: z.enum(['text', 'textarea', 'number', 'boolean', 'select', 'multiselect', 'date']),
+        description: z.string().optional(),
+        placeholder: z.string().optional(),
+        required: z.boolean().optional(),
+        options: z.array(z.object({ label: z.string(), value: z.string() })).optional(),
+    }))
+        .max(12)
+        .optional()
+        .describe('form only. Never ask for secrets.'),
+    submitLabel: z.string().max(40).optional().describe('form only.'),
+    media: z
+        .array(z.object({
+        type: z.enum(['image', 'video', 'audio', 'file']),
+        url: z.string().url(),
+        title: z.string().optional(),
+        alt: z.string().optional(),
+    }))
+        .max(20)
+        .optional()
+        .describe('media only. Public https URLs.'),
+    actions: z
+        .array(actionSchema)
+        .max(MAX_ACTIONS)
+        .optional()
+        .describe('Buttons under the widget. Required for choice.'),
+});
+function actionIdError(actions = []) {
+    const reserved = actions.find((action) => RESERVED_ACTION_IDS.has(action.id));
+    return reserved ? `Action id "${reserved.id}" is reserved; pick another id.` : null;
+}
+function labelsById(input) {
+    const labels = {};
+    for (const action of input.actions ?? [])
+        labels[action.id] = action.label;
+    for (const item of input.items ?? []) {
+        for (const action of item.actions ?? [])
+            labels[action.id] = action.label;
+    }
+    return labels;
+}
+/** Validate the flat tool input for its kind and map it onto the shared widget shape. */
+export function buildRenderedWidget(input, widgetId) {
+    const base = {
+        widgetId,
+        title: input.title,
+        ...(input.description ? { description: input.description } : {}),
+        display: 'expanded',
+        state: 'open',
+        metadata: {
+            type: RENDER_WIDGET_METADATA_TYPE,
+            title: input.title,
+            summary: input.summary,
+            labels: labelsById(input),
+        },
+    };
+    const actions = input.actions?.length ? input.actions : undefined;
+    const reserved = actionIdError(input.actions) ??
+        actionIdError((input.items ?? []).flatMap((item) => item.actions ?? []));
+    if (reserved)
+        return { error: reserved };
+    let widget;
+    switch (input.kind) {
+        case 'table': {
+            const columns = input.columns ?? [];
+            const rows = input.rows ?? [];
+            if (columns.length === 0)
+                return { error: 'table needs at least one column.' };
+            const badRow = rows.findIndex((row) => row.length !== columns.length);
+            if (badRow !== -1) {
+                return {
+                    error: `table row ${badRow} has ${rows[badRow].length} cells; expected ${columns.length} (one per column).`,
+                };
+            }
+            widget = {
+                ...base,
+                kind: 'table',
+                columns,
+                rows: rows.map((row) => Object.fromEntries(columns.map((column, i) => [column.id, row[i]]))),
+                ...(actions ? { actions } : {}),
+            };
+            break;
+        }
+        case 'list': {
+            if (!input.items?.length)
+                return { error: 'list needs at least one item.' };
+            widget = {
+                ...base,
+                kind: 'list',
+                items: input.items.map((item) => ({
+                    ...item,
+                    ...(item.actions?.length
+                        ? {
+                            actions: item.actions.map((action) => ({
+                                ...action,
+                                value: { itemId: item.id, itemLabel: item.label },
+                            })),
+                        }
+                        : {}),
+                })),
+                ...(actions ? { actions } : {}),
+            };
+            break;
+        }
+        case 'choice':
+            if (!actions)
+                return { error: 'choice needs at least one action.' };
+            widget = { ...base, kind: 'choice', actions };
+            break;
+        case 'form':
+            if (!input.fields?.length)
+                return { error: 'form needs at least one field.' };
+            widget = {
+                ...base,
+                kind: 'form',
+                fields: input.fields,
+                ...(input.submitLabel ? { submitLabel: input.submitLabel } : {}),
+            };
+            break;
+        case 'media':
+            if (!input.media?.length)
+                return { error: 'media needs at least one item.' };
+            widget = {
+                ...base,
+                kind: 'media',
+                items: input.media,
+                layout: input.media.length > 1 ? 'grid' : 'single',
+                ...(actions ? { actions } : {}),
+            };
+            break;
+    }
+    const issue = uiWidgetIssue(widget);
+    return issue ? { error: issue } : { widget };
+}
+function asRecord(value) {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? value
+        : undefined;
+}
+/**
+ * How a click on a `render_widget` card reads in the model's history, or
+ * undefined for widget responses that came from anything else.
+ */
+export function renderedWidgetReplyText(data) {
+    const metadata = asRecord(data.metadata);
+    if (metadata?.type !== RENDER_WIDGET_METADATA_TYPE)
+        return undefined;
+    const title = typeof metadata.title === 'string' ? metadata.title : 'widget';
+    const values = { ...(data.values ?? {}) };
+    if (data.actionId === 'submit' && !asRecord(metadata.labels)?.submit) {
+        return `[Widget "${title}"] Submitted: ${JSON.stringify(values)}`;
+    }
+    const label = asRecord(metadata.labels)?.[data.actionId];
+    const clicked = typeof label === 'string' ? label : data.actionId;
+    const itemLabel = values.itemLabel;
+    delete values.itemId;
+    delete values.itemLabel;
+    const target = typeof itemLabel === 'string' ? ` on "${itemLabel}"` : '';
+    const extra = Object.keys(values).length > 0 ? ` with ${JSON.stringify(values)}` : '';
+    return `[Widget "${title}"] Clicked "${clicked}"${target}${extra} (action id: ${data.actionId})`;
+}
+/**
+ * A choice or form answers once: re-emit it as submitted, with the picked
+ * button highlighted or the submitted values filled in. Lists, tables and
+ * media stay open for further clicks.
+ */
+export function settledRenderedWidget(events, response) {
+    for (let i = events.length - 1; i >= 0; i--) {
+        const event = events[i];
+        if (event.type !== 'client:ui:widget' || event.data.widgetId !== response.widgetId)
+            continue;
+        const widget = event.data;
+        if (widget.metadata?.type !== RENDER_WIDGET_METADATA_TYPE)
+            return undefined;
+        if (widget.kind === 'choice') {
+            return {
+                ...event,
+                data: {
+                    ...widget,
+                    state: 'submitted',
+                    actions: widget.actions.map((action) => ({
+                        ...action,
+                        variant: action.id === response.actionId ? 'primary' : 'secondary',
+                    })),
+                },
+            };
+        }
+        if (widget.kind === 'form') {
+            const values = response.values ?? {};
+            return {
+                ...event,
+                data: {
+                    ...widget,
+                    state: 'submitted',
+                    fields: widget.fields.map((field) => field.id in values ? { ...field, defaultValue: values[field.id] } : field),
+                },
+            };
+        }
+        return undefined;
+    }
+    return undefined;
+}
+/** History stand-in for a `render_widget` call: what it showed, without the data. */
+export function compactRenderWidgetInput(input) {
+    const record = asRecord(input);
+    if (!record)
+        return input;
+    const { kind, title, summary } = record;
+    return { kind, title, summary };
+}
+function isInteractive(widget) {
+    if (widget.kind === 'choice' || widget.kind === 'form')
+        return true;
+    if (widget.actions?.length)
+        return true;
+    return widget.kind === 'list' && widget.items.some((item) => item.actions?.length);
+}
+async function renderWidget(rawArgs, ctx) {
+    const parsed = inputSchema.safeParse(rawArgs ?? {});
+    if (!parsed.success) {
+        const message = parsed.error.issues[0]?.message ?? 'Invalid render_widget input.';
+        return { success: false, error: message, output: message };
+    }
+    const threadId = ctx.threadId ?? ctx.state.threadId;
+    const built = buildRenderedWidget(parsed.data, ctx.toolCallId ?? `render:${randomUUID()}`);
+    if ('error' in built) {
+        return { success: false, error: built.error, output: built.error };
+    }
+    await ctx.emit({
+        type: 'client:ui:widget',
+        data: built.widget,
+        meta: {
+            agentId: ctx.agentId,
+            ...(threadId ? { threadId } : {}),
+            ...(ctx.toolCallId ? { toolCallId: ctx.toolCallId } : {}),
+        },
+    });
+    const output = isInteractive(built.widget)
+        ? 'Shown to the human. Do not repeat its contents in text. Their click or submission arrives as their next message; end your turn unless you have something new to add.'
+        : 'Shown to the human. Do not repeat its contents in text; add only a short takeaway if useful.';
+    return { success: true, widgetId: built.widget.widgetId, output };
+}
+export const toolDefinitions = {
+    render_widget: {
+        description: 'Show structured data to the human as an inline card instead of markdown: a table, a list of records with status, a choice with buttons, a short form, or media. Use it when the answer is a collection of 3+ similar records, a comparison, or a decision or input you need from the human. Do not use it for prose, single facts, or to restate what a specialist already said in this thread.',
+        inputSchema,
+    },
+};
+export const tools = {
+    render_widget: {
+        ...toolDefinitions.render_widget,
+        execute: renderWidget,
+    },
+};
+export const renderWidgetTools = tools;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@meetopenbot/openbot",
-  "version": "1.6.2",
+  "version": "1.7.0",
   "description": "OpenBot coordinator runtime: ask specialists, route work into Spaces, and track todos.",
   "type": "module",
   "main": "./dist/index.js",
@@ -25,7 +25,7 @@
     "@ai-sdk/openai": "^4.0.72",
     "ai": "^7.0.111",
     "zod": "^4.3.5",
-    "@meetopenbot/plugin-sdk": "^2.1.0"
+    "@meetopenbot/plugin-sdk": "^2.3.0"
   },
   "devDependencies": {
     "@types/node": "^20.10.1",