@meetopenbot/openbot 1.6.3 → 1.7.0

package/dist/build-tools.js

@@ -8,6 +8,7 @@ import { threadStatusTools } from "./tools/thread-status.js";
 import { threadTitleTools } from "./tools/thread-title.js";
 import { previewTools } from "./tools/preview.js";
 import { variablesRequestTools } from "./tools/variables-request.js";
+import { renderWidgetTools } from "./tools/render-widget.js";
 import { wrapToolsWithApproval } from "./tools/approval.js";
 export const DEFAULT_APPROVAL_ACTIONS = ["update_channel_spec", "start_thread"];
 /**
@@ -26,6 +27,7 @@ export function buildOpenbotTools(args) {
         ...threadTitleTools,
         ...previewTools,
         ...variablesRequestTools,
+        ...renderWidgetTools,
         ...args.context.tools,
     };
     const approvalConfig = args.context.config?.approval;

package/dist/history.js

@@ -1,3 +1,4 @@
+import { compactRenderWidgetInput, renderedWidgetReplyText } from './tools/render-widget.js';
 function asReasoningPart(part) {
     const text = typeof part.text === 'string' ? part.text : '';
     if (!text.trim() && !part.providerOptions)
@@ -201,6 +202,12 @@ export function eventsToModelMessages(events, options) {
                 }
                 break;
             }
+            case 'client:ui:widget:response': {
+                const reply = renderedWidgetReplyText(event.data);
+                if (reply)
+                    messages.push({ role: 'user', content: reply });
+                break;
+            }
             default:
                 // Handle tool calls (action:*)
                 if (event.type.startsWith('action:') && !event.type.endsWith(':result')) {
@@ -212,7 +219,9 @@ export function eventsToModelMessages(events, options) {
                         type: 'tool-call',
                         toolCallId,
                         toolName,
-                        input: event.data,
+                        input: toolName === 'render_widget'
+                            ? compactRenderWidgetInput(event.data)
+                            : event.data,
                     };
                     const last = messages[messages.length - 1];
                     if (last && last.role === 'assistant') {

package/dist/index.js

@@ -12,6 +12,7 @@ import { threadStatusTools } from './tools/thread-status.js';
 import { threadTitleTools } from './tools/thread-title.js';
 import { previewTools } from './tools/preview.js';
 import { variablesRequestTools } from './tools/variables-request.js';
+import { renderWidgetTools } from './tools/render-widget.js';
 import { buildOpenbotTools } from './build-tools.js';
 export const OPENBOT_PLUGIN_ID = '@meetopenbot/openbot';
 const modelField = await modelConfigField({
@@ -53,6 +54,7 @@ export const openbotPlugin = defineOpenbotPlugin({
         ...threadTitleTools,
         ...previewTools,
         ...variablesRequestTools,
+        ...renderWidgetTools,
     },
     async *run(args) {
         const { context } = args;

package/dist/runtime.js

@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
 import { generateText, hasToolCall, streamText, stepCountIs } from 'ai';
 import { BYOK_ENV_KEYS, isAuthErrorMessage, isCreditsErrorMessage, fetchModelRegistry, getProviderModelOptions, listApiKeyProviders, PROVIDER_API_KEY_LINKS, } from '@meetopenbot/plugin-sdk';
 import { eventsToModelMessages } from './history.js';
+import { RENDER_WIDGET_METADATA_TYPE, settledRenderedWidget } from './tools/render-widget.js';
 import { addAttachmentParts, chooseTurnCandidates, eventsHaveAttachments, modelSupportsImages, } from './images.js';
 import { buildContext } from './context.js';
 import { OPENBOT_SYSTEM_PROMPT, withGithubGitInstructions, withUserInstructions, } from './system-prompt.js';
@@ -438,6 +439,18 @@ export async function* handleOpenbotWidgetResponse(args, options) {
     const { host, storage } = options;
     let configuredModelString = options.model ?? AUTO_MODEL_ID;
     const state = args.state;
+    // A click on a card the model rendered is the human's next message: the
+    // harness already stored it, and history replays it as a user turn.
+    if (metadata?.type === RENDER_WIDGET_METADATA_TYPE) {
+        if (storage) {
+            const events = await storage.getEvents({ channelId: state.channelId ?? '', threadId });
+            const settled = settledRenderedWidget(events, event.data);
+            if (settled)
+                yield settled;
+        }
+        yield* runOpenbotTurn(args, options);
+        return;
+    }
     if (host.isCloudSystemAgent(state.agentId) && options.authMode === 'credits')
         return;
     if (metadata?.type === 'api_provider_selection') {
@@ -459,7 +472,6 @@ export async function* handleOpenbotWidgetResponse(args, options) {
                 body: `${label} provider was selected.`,
                 state: 'submitted',
                 display: 'collapsed',
-                disabled: true,
                 actions: [],
             },
             meta: { agentId: state.agentId, threadId },
@@ -536,7 +548,6 @@ export async function* handleOpenbotWidgetResponse(args, options) {
                 body: `${modelId} was selected.`,
                 state: 'submitted',
                 display: 'collapsed',
-                disabled: true,
                 actions: [],
             },
             meta: { agentId: state.agentId, threadId },

package/dist/system-prompt.js

@@ -28,6 +28,12 @@ export const OPENBOT_SYSTEM_PROMPT = [
     "- The current list is injected into context each turn as `## TODOS`; call `todo_read` only if you need an explicit refresh.",
     "- Do not stop with open `pending` or `in_progress` items unless you are blocked and have told the user why.",
     "",
+    "# WIDGETS",
+    "- Call `render_widget` when your own answer is structured: 3+ similar records (Spaces, agents, options, statuses), a side-by-side comparison, a decision with fixed choices, or a few fields you need from the human. Use plain text for everything else.",
+    "- Never use it to restate or re-format what a specialist already said in this thread; that rule from Close still applies.",
+    "- Do not repeat the widget's contents in text. A click or form submission arrives as the human's next message, quoting what they picked.",
+    "- Forms never ask for secrets; use `request_variables` for those.",
+    "",
     "# JOB TITLE",
     "- Unnamed jobs get a system hint to call `set_thread_title` once with a concise, human-readable title.",
     "- Call `set_thread_title` again only if the topic clearly changes.",

package/dist/tools/render-widget.js

@@ -0,0 +1,317 @@
+import { randomUUID } from 'node:crypto';
+import z from 'zod';
+import { uiWidgetIssue, } from '@meetopenbot/plugin-sdk';
+/**
+ * `render_widget` — the coordinator shows structured data as an inline card
+ * instead of markdown. Clicks and form submits come back as a new turn
+ * (`handleOpenbotWidgetResponse` → `history.ts`), tagged by this metadata type.
+ */
+export const RENDER_WIDGET_METADATA_TYPE = 'render_widget';
+const MAX_COLUMNS = 12;
+const MAX_ROWS = 200;
+const MAX_ITEMS = 50;
+const MAX_ACTIONS = 6;
+/** Ids the client runs itself (navigate, open_url, …); a model action must not collide. */
+const RESERVED_ACTION_IDS = new Set(['open_preview', 'open_url', 'navigate', 'start_in_space']);
+const actionSchema = z.object({
+    id: z.string().min(1).max(64).describe('Stable id returned to you when clicked, e.g. "approve".'),
+    label: z.string().min(1).max(40).describe('Button text.'),
+    variant: z.enum(['primary', 'secondary', 'danger']).optional(),
+});
+const cellSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]);
+const inputSchema = z.object({
+    kind: z
+        .enum(['table', 'list', 'choice', 'form', 'media'])
+        .describe('table: rows × columns of data. list: records with status/badge and optional per-item buttons. choice: a question answered with buttons. form: structured input. media: images/videos/files.'),
+    title: z.string().min(1).max(120),
+    description: z.string().max(400).optional().describe('One short line under the title.'),
+    summary: z
+        .string()
+        .min(1)
+        .max(400)
+        .describe('One or two sentences on what the widget shows. Kept in history in place of the full data, so include what you will need to refer back to.'),
+    columns: z
+        .array(z.object({
+        id: z.string().min(1),
+        label: z.string(),
+        align: z.enum(['left', 'right', 'center']).optional().describe('Use right for numbers.'),
+    }))
+        .max(MAX_COLUMNS)
+        .optional()
+        .describe('table only.'),
+    rows: z
+        .array(z.array(cellSchema))
+        .max(MAX_ROWS)
+        .optional()
+        .describe('table only. Each row lists one cell per column, in column order.'),
+    items: z
+        .array(z.object({
+        id: z.string().min(1),
+        label: z.string().min(1),
+        description: z.string().optional(),
+        badge: z.string().optional(),
+        status: z.string().optional(),
+        statusVariant: z.enum(['default', 'success', 'warning', 'danger', 'info']).optional(),
+        actions: z.array(actionSchema).max(3).optional(),
+    }))
+        .max(MAX_ITEMS)
+        .optional()
+        .describe('list only.'),
+    fields: z
+        .array(z.object({
+        id: z.string().min(1),
+        label: z.string().min(1),
+        type: z.enum(['text', 'textarea', 'number', 'boolean', 'select', 'multiselect', 'date']),
+        description: z.string().optional(),
+        placeholder: z.string().optional(),
+        required: z.boolean().optional(),
+        options: z.array(z.object({ label: z.string(), value: z.string() })).optional(),
+    }))
+        .max(12)
+        .optional()
+        .describe('form only. Never ask for secrets.'),
+    submitLabel: z.string().max(40).optional().describe('form only.'),
+    media: z
+        .array(z.object({
+        type: z.enum(['image', 'video', 'audio', 'file']),
+        url: z.string().url(),
+        title: z.string().optional(),
+        alt: z.string().optional(),
+    }))
+        .max(20)
+        .optional()
+        .describe('media only. Public https URLs.'),
+    actions: z
+        .array(actionSchema)
+        .max(MAX_ACTIONS)
+        .optional()
+        .describe('Buttons under the widget. Required for choice.'),
+});
+function actionIdError(actions = []) {
+    const reserved = actions.find((action) => RESERVED_ACTION_IDS.has(action.id));
+    return reserved ? `Action id "${reserved.id}" is reserved; pick another id.` : null;
+}
+function labelsById(input) {
+    const labels = {};
+    for (const action of input.actions ?? [])
+        labels[action.id] = action.label;
+    for (const item of input.items ?? []) {
+        for (const action of item.actions ?? [])
+            labels[action.id] = action.label;
+    }
+    return labels;
+}
+/** Validate the flat tool input for its kind and map it onto the shared widget shape. */
+export function buildRenderedWidget(input, widgetId) {
+    const base = {
+        widgetId,
+        title: input.title,
+        ...(input.description ? { description: input.description } : {}),
+        display: 'expanded',
+        state: 'open',
+        metadata: {
+            type: RENDER_WIDGET_METADATA_TYPE,
+            title: input.title,
+            summary: input.summary,
+            labels: labelsById(input),
+        },
+    };
+    const actions = input.actions?.length ? input.actions : undefined;
+    const reserved = actionIdError(input.actions) ??
+        actionIdError((input.items ?? []).flatMap((item) => item.actions ?? []));
+    if (reserved)
+        return { error: reserved };
+    let widget;
+    switch (input.kind) {
+        case 'table': {
+            const columns = input.columns ?? [];
+            const rows = input.rows ?? [];
+            if (columns.length === 0)
+                return { error: 'table needs at least one column.' };
+            const badRow = rows.findIndex((row) => row.length !== columns.length);
+            if (badRow !== -1) {
+                return {
+                    error: `table row ${badRow} has ${rows[badRow].length} cells; expected ${columns.length} (one per column).`,
+                };
+            }
+            widget = {
+                ...base,
+                kind: 'table',
+                columns,
+                rows: rows.map((row) => Object.fromEntries(columns.map((column, i) => [column.id, row[i]]))),
+                ...(actions ? { actions } : {}),
+            };
+            break;
+        }
+        case 'list': {
+            if (!input.items?.length)
+                return { error: 'list needs at least one item.' };
+            widget = {
+                ...base,
+                kind: 'list',
+                items: input.items.map((item) => ({
+                    ...item,
+                    ...(item.actions?.length
+                        ? {
+                            actions: item.actions.map((action) => ({
+                                ...action,
+                                value: { itemId: item.id, itemLabel: item.label },
+                            })),
+                        }
+                        : {}),
+                })),
+                ...(actions ? { actions } : {}),
+            };
+            break;
+        }
+        case 'choice':
+            if (!actions)
+                return { error: 'choice needs at least one action.' };
+            widget = { ...base, kind: 'choice', actions };
+            break;
+        case 'form':
+            if (!input.fields?.length)
+                return { error: 'form needs at least one field.' };
+            widget = {
+                ...base,
+                kind: 'form',
+                fields: input.fields,
+                ...(input.submitLabel ? { submitLabel: input.submitLabel } : {}),
+            };
+            break;
+        case 'media':
+            if (!input.media?.length)
+                return { error: 'media needs at least one item.' };
+            widget = {
+                ...base,
+                kind: 'media',
+                items: input.media,
+                layout: input.media.length > 1 ? 'grid' : 'single',
+                ...(actions ? { actions } : {}),
+            };
+            break;
+    }
+    const issue = uiWidgetIssue(widget);
+    return issue ? { error: issue } : { widget };
+}
+function asRecord(value) {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? value
+        : undefined;
+}
+/**
+ * How a click on a `render_widget` card reads in the model's history, or
+ * undefined for widget responses that came from anything else.
+ */
+export function renderedWidgetReplyText(data) {
+    const metadata = asRecord(data.metadata);
+    if (metadata?.type !== RENDER_WIDGET_METADATA_TYPE)
+        return undefined;
+    const title = typeof metadata.title === 'string' ? metadata.title : 'widget';
+    const values = { ...(data.values ?? {}) };
+    if (data.actionId === 'submit' && !asRecord(metadata.labels)?.submit) {
+        return `[Widget "${title}"] Submitted: ${JSON.stringify(values)}`;
+    }
+    const label = asRecord(metadata.labels)?.[data.actionId];
+    const clicked = typeof label === 'string' ? label : data.actionId;
+    const itemLabel = values.itemLabel;
+    delete values.itemId;
+    delete values.itemLabel;
+    const target = typeof itemLabel === 'string' ? ` on "${itemLabel}"` : '';
+    const extra = Object.keys(values).length > 0 ? ` with ${JSON.stringify(values)}` : '';
+    return `[Widget "${title}"] Clicked "${clicked}"${target}${extra} (action id: ${data.actionId})`;
+}
+/**
+ * A choice or form answers once: re-emit it as submitted, with the picked
+ * button highlighted or the submitted values filled in. Lists, tables and
+ * media stay open for further clicks.
+ */
+export function settledRenderedWidget(events, response) {
+    for (let i = events.length - 1; i >= 0; i--) {
+        const event = events[i];
+        if (event.type !== 'client:ui:widget' || event.data.widgetId !== response.widgetId)
+            continue;
+        const widget = event.data;
+        if (widget.metadata?.type !== RENDER_WIDGET_METADATA_TYPE)
+            return undefined;
+        if (widget.kind === 'choice') {
+            return {
+                ...event,
+                data: {
+                    ...widget,
+                    state: 'submitted',
+                    actions: widget.actions.map((action) => ({
+                        ...action,
+                        variant: action.id === response.actionId ? 'primary' : 'secondary',
+                    })),
+                },
+            };
+        }
+        if (widget.kind === 'form') {
+            const values = response.values ?? {};
+            return {
+                ...event,
+                data: {
+                    ...widget,
+                    state: 'submitted',
+                    fields: widget.fields.map((field) => field.id in values ? { ...field, defaultValue: values[field.id] } : field),
+                },
+            };
+        }
+        return undefined;
+    }
+    return undefined;
+}
+/** History stand-in for a `render_widget` call: what it showed, without the data. */
+export function compactRenderWidgetInput(input) {
+    const record = asRecord(input);
+    if (!record)
+        return input;
+    const { kind, title, summary } = record;
+    return { kind, title, summary };
+}
+function isInteractive(widget) {
+    if (widget.kind === 'choice' || widget.kind === 'form')
+        return true;
+    if (widget.actions?.length)
+        return true;
+    return widget.kind === 'list' && widget.items.some((item) => item.actions?.length);
+}
+async function renderWidget(rawArgs, ctx) {
+    const parsed = inputSchema.safeParse(rawArgs ?? {});
+    if (!parsed.success) {
+        const message = parsed.error.issues[0]?.message ?? 'Invalid render_widget input.';
+        return { success: false, error: message, output: message };
+    }
+    const threadId = ctx.threadId ?? ctx.state.threadId;
+    const built = buildRenderedWidget(parsed.data, ctx.toolCallId ?? `render:${randomUUID()}`);
+    if ('error' in built) {
+        return { success: false, error: built.error, output: built.error };
+    }
+    await ctx.emit({
+        type: 'client:ui:widget',
+        data: built.widget,
+        meta: {
+            agentId: ctx.agentId,
+            ...(threadId ? { threadId } : {}),
+            ...(ctx.toolCallId ? { toolCallId: ctx.toolCallId } : {}),
+        },
+    });
+    const output = isInteractive(built.widget)
+        ? 'Shown to the human. Do not repeat its contents in text. Their click or submission arrives as their next message; end your turn unless you have something new to add.'
+        : 'Shown to the human. Do not repeat its contents in text; add only a short takeaway if useful.';
+    return { success: true, widgetId: built.widget.widgetId, output };
+}
+export const toolDefinitions = {
+    render_widget: {
+        description: 'Show structured data to the human as an inline card instead of markdown: a table, a list of records with status, a choice with buttons, a short form, or media. Use it when the answer is a collection of 3+ similar records, a comparison, or a decision or input you need from the human. Do not use it for prose, single facts, or to restate what a specialist already said in this thread.',
+        inputSchema,
+    },
+};
+export const tools = {
+    render_widget: {
+        ...toolDefinitions.render_widget,
+        execute: renderWidget,
+    },
+};
+export const renderWidgetTools = tools;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@meetopenbot/openbot",
-  "version": "1.6.3",
+  "version": "1.7.0",
   "description": "OpenBot coordinator runtime: ask specialists, route work into Spaces, and track todos.",
   "type": "module",
   "main": "./dist/index.js",
@@ -25,7 +25,7 @@
     "@ai-sdk/openai": "^4.0.72",
     "ai": "^7.0.111",
     "zod": "^4.3.5",
-    "@meetopenbot/plugin-sdk": "^2.2.0"
+    "@meetopenbot/plugin-sdk": "^2.3.0"
   },
   "devDependencies": {
     "@types/node": "^20.10.1",