openbot 0.2.10 → 0.2.11

package/dist/plugins/approval/index.js

@@ -1,186 +1,99 @@
 import { generateId } from "melony";
-import { ui } from "@melony/ui-kit/server";
+import { uiEvent } from "../../ui/block.js";
 import { widgets } from "../../ui/widgets/index.js";
-const DEFAULT_REDACTED_KEY_PATTERNS = [
-    /toolcallid/i,
-    /content/i,
-    /stdout/i,
-    /stderr/i,
-    /password/i,
-    /secret/i,
-    /token/i,
-    /api[_-]?key/i,
-    /authorization/i,
-    /cookie/i,
-];
-const MAX_VALUE_LENGTH = 240;
-const MAX_DETAILS = 8;
-function serializeValue(value) {
-    if (value === undefined || value === null)
-        return "-";
-    const serialized = typeof value === "string"
-        ? value
-        : typeof value === "number" || typeof value === "boolean"
-            ? String(value)
-            : JSON.stringify(value);
-    if (serialized.length <= MAX_VALUE_LENGTH)
-        return serialized;
-    return `${serialized.slice(0, MAX_VALUE_LENGTH - 3)}...`;
-}
-function buildActionLabel(eventType) {
-    const action = eventType.startsWith("action:") ? eventType.slice("action:".length) : eventType;
-    return action.replace(/([A-Z])/g, " $1").replace(/^./, (c) => c.toUpperCase()).trim();
-}
-function toTitleCaseKey(key) {
-    return key
-        .replace(/([A-Z])/g, " $1")
-        .replace(/[_-]+/g, " ")
-        .replace(/^./, (c) => c.toUpperCase())
-        .trim();
-}
-function isRedactedKey(key, hiddenKeys = []) {
-    if (hiddenKeys.includes(key))
-        return true;
-    return DEFAULT_REDACTED_KEY_PATTERNS.some((pattern) => pattern.test(key));
-}
-function sanitizePayload(value, hiddenKeys = []) {
-    if (Array.isArray(value)) {
-        return value.map((item) => sanitizePayload(item, hiddenKeys));
-    }
-    if (value && typeof value === "object") {
-        const obj = value;
-        const sanitizedEntries = Object.entries(obj).map(([key, v]) => {
-            if (isRedactedKey(key, hiddenKeys))
-                return [key, "[REDACTED]"];
-            return [key, sanitizePayload(v, hiddenKeys)];
-        });
-        return Object.fromEntries(sanitizedEntries);
-    }
-    if (typeof value === "string" && value.length > 1000) {
-        return `${value.slice(0, 997)}...`;
-    }
-    return value;
-}
-function summarizeData(data = {}, hiddenKeys = []) {
-    const safeData = sanitizePayload(data, hiddenKeys);
-    return JSON.stringify(safeData, null, 2);
-}
-function isRenderableDetailValue(value) {
-    return value !== undefined && value !== null;
-}
-function deriveDetailEntries(data, rule) {
-    const hiddenKeys = rule.hiddenKeys ?? [];
-    if (rule.detailKeys?.length) {
-        return rule.detailKeys
-            .filter((key) => key in data)
-            .filter((key) => !isRedactedKey(key, hiddenKeys))
-            .filter((key) => isRenderableDetailValue(data[key]))
-            .map((key) => ({
-            label: toTitleCaseKey(key),
-            value: serializeValue(data[key]),
-        }))
-            .slice(0, MAX_DETAILS);
-    }
-    return Object.entries(data)
-        .filter(([key]) => !isRedactedKey(key, hiddenKeys))
-        .filter(([_, value]) => isRenderableDetailValue(value))
-        .slice(0, MAX_DETAILS)
-        .map(([key, value]) => ({
-        label: toTitleCaseKey(key),
-        value: serializeValue(value),
-    }));
+function getActionName(eventType) {
+    return eventType.startsWith("action:")
+        ? eventType.slice("action:".length)
+        : eventType;
 }
 function buildApprovalData(event, rule) {
-    const eventType = event.type;
-    const data = (event.data ?? {});
-    const details = [
-        { label: "Action", value: buildActionLabel(eventType) },
-        { label: "Event", value: eventType },
-        ...deriveDetailEntries(data, rule),
-    ];
+    const actionName = getActionName(String(event.type));
+    const toolCallId = event?.data?.toolCallId;
+    const data = event?.data ?? {};
     return {
-        summary: rule.message || "The agent wants to execute an action. Review details before approving.",
-        details,
-        rawPayload: summarizeData(data, rule.hiddenKeys ?? []),
+        summary: rule.message ??
+            "The agent requested a protected action. Approve to continue or deny to block it.",
+        details: [
+            { label: "Action", value: actionName },
+            ...(toolCallId ? [{ label: "Tool call", value: String(toolCallId) }] : []),
+        ],
+        rawPayload: JSON.stringify(data, null, 2),
     };
 }
 /**
- * Approval Plugin for OpenBot.
- * Intercepts specific actions and requires user approval before proceeding.
- * Optimized using the new melony intercept() feature.
+ * Minimal approval gate:
+ * - Intercept protected action events.
+ * - Suspend current request and show Approve/Deny UI.
+ * - Resume only when user sends action:approve or action:deny.
  */
 export const approvalPlugin = (options) => (builder) => {
-    const { rules = [] } = options;
-    // Register an interceptor that runs before any handlers.
-    // This is the correct way to handle HITL/Approval in Melony.
+    const rules = options?.rules ?? [];
     builder.intercept(async (event, { state, suspend }) => {
-        // Skip if already approved or if it's an internal approval event
-        // We cast event to any to access the meta property which is used for internal state tracking
-        const meta = event.meta;
-        if (meta?.approved ||
-            event.type === "action:approve" ||
-            event.type === "action:deny" ||
-            event.type === "ui" ||
-            event.type.endsWith(":status")) {
+        const type = String(event.type ?? "");
+        const meta = event.meta ?? {};
+        // Never intercept internal approval events or already-approved replays.
+        if (type === "action:approve" || type === "action:deny" || meta.approved === true) {
             return;
         }
-        const rule = rules.find(r => event.type.startsWith(r.action));
+        const rule = rules.find((r) => type.startsWith(r.action));
         if (!rule)
             return;
         const approvalId = `approve_${generateId()}`;
-        if (!state.pendingApprovals) {
-            state.pendingApprovals = {};
-        }
-        state.pendingApprovals[approvalId] = event;
-        // Use suspend(event) to emit the UI and halt execution of any handlers for this event.
-        // This effectively "pauses" the run for user input.
-        const approvalData = buildApprovalData(event, rule);
-        suspend(ui.event(widgets.approvalCard("Approval Required", approvalData, {
-            type: "action:approve",
-            data: { id: approvalId }
-        }, {
-            type: "action:deny",
-            data: { id: approvalId }
-        })));
+        state.pendingApprovals ?? (state.pendingApprovals = {});
+        state.pendingApprovals[approvalId] = {
+            originalEvent: event,
+            createdAt: Date.now(),
+        };
+        suspend({
+            type: "suspend",
+            data: {
+                reason: "approval",
+                id: approvalId,
+                event: uiEvent(widgets.approvalCard("Approval Required", buildApprovalData(event, rule), { type: "action:approve", data: { id: approvalId } }, { type: "action:deny", data: { id: approvalId } }, { placement: "attention", id: approvalId })),
+            },
+        });
     });
-    // Handle Approval response from user
     builder.on("action:approve", async function* (event, { state }) {
-        const { id } = event.data;
-        const originalEvent = state.pendingApprovals?.[id];
-        if (originalEvent) {
-            delete state.pendingApprovals[id];
-            yield ui.event(widgets.status("Action approved", "success"));
-            // Re-emit the original event with approved: true.
-            // The interceptor will see it, but bypass because of meta.approved.
-            // Then the appropriate handlers for the event will finally run.
-            yield {
-                ...originalEvent,
-                meta: {
-                    ...originalEvent.meta,
-                    approved: true,
-                },
-            };
+        const id = event?.data?.id;
+        if (!id)
+            return;
+        const pending = state.pendingApprovals?.[id];
+        if (!pending) {
+            return;
         }
+        delete state.pendingApprovals[id];
+        // Re-emit the original action with approval marker.
+        yield {
+            ...pending.originalEvent,
+            meta: {
+                ...(pending.originalEvent?.meta ?? {}),
+                approved: true,
+            },
+        };
     });
-    // Handle Denial response from user
     builder.on("action:deny", async function* (event, { state }) {
-        const { id } = event.data;
-        const originalEvent = state.pendingApprovals?.[id];
-        if (originalEvent) {
-            delete state.pendingApprovals[id];
-            yield ui.event(widgets.status("Action denied", "error"));
-            // If it was a tool call (action:*), return a result error so the LLM knows it failed
-            if (originalEvent.data?.toolCallId) {
-                yield {
-                    type: "action:result",
-                    data: {
-                        action: originalEvent.type.replace("action:", ""),
-                        toolCallId: originalEvent.data.toolCallId,
-                        result: { error: "Action denied by user" },
-                        success: false,
-                    },
-                };
-            }
+        const id = event?.data?.id;
+        if (!id)
+            return;
+        const pending = state.pendingApprovals?.[id];
+        if (!pending) {
+            return;
+        }
+        delete state.pendingApprovals[id];
+        yield uiEvent(widgets.status("Action denied", "error", { placement: "attention", id }));
+        const originalEvent = pending.originalEvent;
+        const toolCallId = originalEvent?.data?.toolCallId;
+        if (toolCallId) {
+            yield {
+                type: "action:result",
+                data: {
+                    action: getActionName(String(originalEvent.type ?? "")),
+                    toolCallId,
+                    result: { error: "Action denied by user", denied: true },
+                    success: false,
+                    halt: true,
+                },
+            };
         }
     });
 };

package/dist/plugins/file-system/index.js

@@ -1,4 +1,4 @@
-import { ui } from "@melony/ui-kit/server";
+import { uiEvent } from "../../ui/block.js";
 import { z } from "zod";
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
@@ -166,6 +166,6 @@ export const fileSystemPlugin = (options = {}) => (builder) => {
         }
     });
     builder.on("file-system:status", async function* (event) {
-        yield ui.event(statusWidget(event.data.message, event.data.severity));
+        yield uiEvent(statusWidget(event.data.message, event.data.severity));
     });
 };

package/dist/plugins/llm/context-budget.js

@@ -0,0 +1,139 @@
+function truncateMiddle(value, maxChars) {
+    if (value.length <= maxChars)
+        return value;
+    const half = Math.floor(maxChars / 2);
+    const removed = value.length - maxChars;
+    return `${value.slice(0, half)}\n\n[... ${removed} characters truncated ...]\n\n${value.slice(-half)}`;
+}
+function estimateStringTokens(value) {
+    if (!value)
+        return 0;
+    return Math.max(1, Math.ceil(value.length / 4));
+}
+function estimateMessageTokens(message) {
+    if (typeof message.content === "string") {
+        return estimateStringTokens(message.content);
+    }
+    if (Array.isArray(message.content)) {
+        let total = 0;
+        for (const part of message.content) {
+            if (typeof part === "string") {
+                total += estimateStringTokens(part);
+            }
+            else if (part && typeof part === "object") {
+                total += estimateStringTokens(JSON.stringify(part));
+            }
+        }
+        return total;
+    }
+    return 0;
+}
+function cloneWithTrimmedSystemMessages(messages, maxSystemChars) {
+    return messages.map((message) => {
+        if (message.role !== "system" || typeof message.content !== "string") {
+            return message;
+        }
+        return {
+            ...message,
+            content: truncateMiddle(message.content, maxSystemChars),
+        };
+    });
+}
+function getLeadingSystemCount(messages) {
+    let count = 0;
+    for (const message of messages) {
+        if (message.role !== "system")
+            break;
+        count++;
+    }
+    return count;
+}
+function collectAssistantToolPairings(messages) {
+    const toolCallToAssistant = new Map();
+    const assistantToToolResults = new Map();
+    messages.forEach((message, index) => {
+        if (message.role !== "assistant" || !Array.isArray(message.content))
+            return;
+        for (const part of message.content) {
+            if (part
+                && typeof part === "object"
+                && part.type === "tool-call"
+                && typeof part.toolCallId === "string") {
+                toolCallToAssistant.set(part.toolCallId, index);
+            }
+        }
+    });
+    messages.forEach((message, index) => {
+        if (message.role !== "tool" || !Array.isArray(message.content))
+            return;
+        for (const part of message.content) {
+            if (!part || typeof part !== "object")
+                continue;
+            const toolCallId = part.toolCallId;
+            if (typeof toolCallId !== "string")
+                continue;
+            const assistantIndex = toolCallToAssistant.get(toolCallId);
+            if (typeof assistantIndex !== "number")
+                continue;
+            const bucket = assistantToToolResults.get(assistantIndex) ?? new Set();
+            bucket.add(index);
+            assistantToToolResults.set(assistantIndex, bucket);
+        }
+    });
+    return assistantToToolResults;
+}
+function buildAtomicGroupIndices(index, messages, assistantToToolResults) {
+    const message = messages[index];
+    if (!message)
+        return [];
+    if (message.role === "assistant") {
+        const results = assistantToToolResults.get(index);
+        if (!results || results.size === 0)
+            return [index];
+        return [index, ...Array.from(results)].sort((a, b) => a - b);
+    }
+    if (message.role === "tool") {
+        for (const [assistantIndex, resultIndices] of assistantToToolResults.entries()) {
+            if (resultIndices.has(index)) {
+                return [assistantIndex, ...Array.from(resultIndices)].sort((a, b) => a - b);
+            }
+        }
+    }
+    return [index];
+}
+export function buildBudgetedMessages(messages, options) {
+    if (messages.length === 0)
+        return messages;
+    const normalized = cloneWithTrimmedSystemMessages(messages, options.maxSystemChars);
+    const systemCount = getLeadingSystemCount(normalized);
+    const leadingSystem = normalized.slice(0, systemCount);
+    const history = normalized.slice(systemCount);
+    const usableBudget = Math.max(1, options.maxContextTokens - options.reserveOutputTokens);
+    let used = leadingSystem.reduce((sum, message) => sum + estimateMessageTokens(message), 0);
+    const requiredIndices = new Set();
+    for (let i = history.length - 1; i >= 0; i--) {
+        if (history[i].role === "user") {
+            requiredIndices.add(i);
+            break;
+        }
+    }
+    const assistantToToolResults = collectAssistantToolPairings(history);
+    const included = new Set();
+    const visited = new Set();
+    for (let i = history.length - 1; i >= 0; i--) {
+        if (visited.has(i))
+            continue;
+        const group = buildAtomicGroupIndices(i, history, assistantToToolResults);
+        for (const idx of group)
+            visited.add(idx);
+        const groupTokens = group.reduce((sum, idx) => sum + estimateMessageTokens(history[idx]), 0);
+        const isRequired = group.some((idx) => requiredIndices.has(idx));
+        if (isRequired || used + groupTokens <= usableBudget) {
+            for (const idx of group)
+                included.add(idx);
+            used += groupTokens;
+        }
+    }
+    const selectedHistory = history.filter((_, index) => included.has(index));
+    return [...leadingSystem, ...selectedHistory];
+}

package/dist/plugins/llm/index.js

@@ -1,6 +1,15 @@
-import { streamText } from "ai";
+import { streamText, Output } from "ai";
+async function toInlineDataUrl(url, mimeType) {
+    const response = await fetch(url);
+    if (!response.ok) {
+        throw new Error(`Failed to fetch image: ${response.status} ${response.statusText}`);
+    }
+    const arrayBuffer = await response.arrayBuffer();
+    const base64 = Buffer.from(arrayBuffer).toString("base64");
+    return `data:${mimeType};base64,${base64}`;
+}
 async function toModelMessages(messages) {
-    return messages.map((message) => {
+    return Promise.all(messages.map(async (message) => {
         if (message.role === "tool") {
             return {
                 role: "tool",
@@ -50,23 +59,33 @@ async function toModelMessages(messages) {
         }
         if (message.role === "user") {
             if (message.attachments && message.attachments.length > 0) {
+                const attachmentParts = await Promise.all(message.attachments.map(async (a) => {
+                    if (a.mimeType.startsWith("image/")) {
+                        try {
+                            return {
+                                type: "image",
+                                image: await toInlineDataUrl(a.url, a.mimeType),
+                            };
+                        }
+                        catch (error) {
+                            console.warn(`Failed to inline image attachment (${a.name}). Falling back to URL: ${a.url}`, error);
+                            return {
+                                type: "image",
+                                image: a.url,
+                            };
+                        }
+                    }
+                    return {
+                        type: "file",
+                        data: a.url,
+                        mimeType: a.mimeType,
+                    };
+                }));
                 return {
                     role: "user",
                     content: [
                         { type: "text", text: message.content },
-                        ...message.attachments.map((a) => {
-                            if (a.mimeType.startsWith("image/")) {
-                                return {
-                                    type: "image",
-                                    image: a.url,
-                                };
-                            }
-                            return {
-                                type: "file",
-                                data: a.url,
-                                mimeType: a.mimeType,
-                            };
-                        }),
+                        ...attachmentParts,
                     ],
                 };
             }
@@ -79,7 +98,7 @@ async function toModelMessages(messages) {
             role: message.role,
             content: message.content,
         };
-    });
+    }));
 }
 // Helper to find pending tool calls in history
 function getPendingToolCalls(messages) {
@@ -156,7 +175,7 @@ function insertToolResult(messages, toolResultMsg) {
  * It can also automatically trigger events based on tool calls.
  */
 export const llmPlugin = (options) => (builder) => {
-    const { model, system, toolDefinitions = {}, actionEventPrefix = "action:", promptInputType = "agent:input", actionResultInputType = "action:result", completionEventType = "agent:output", usageEventType = "usage:update", usageScope = "default", modelId, } = options;
+    const { model, system, toolDefinitions = {}, actionEventPrefix = "action:", promptInputType = "agent:input", actionResultInputType = "action:result", completionEventType = "agent:output", usageEventType = "usage:update", usageScope = "default", modelId, outputSchema, } = options;
     async function* routeToLLM(newMessage, context, silent = false) {
         const state = context.state;
         if (!state.messages) {
@@ -217,19 +236,40 @@ export const llmPlugin = (options) => (builder) => {
             system: systemPrompt,
             messages: modelMessages,
             tools: toolDefinitions,
+            output: outputSchema ? Output.object({ schema: outputSchema }) : undefined,
             onError: (error) => {
                 console.error("streamText error:::::", JSON.stringify(error, null, 2));
             },
         });
-        for await (const delta of result.textStream) {
-            assistantMessage.content += delta;
-            if (!silent) {
+        if (outputSchema) {
+            for await (const delta of result.partialOutputStream) {
+                if (!silent) {
+                    yield {
+                        type: "agent:output-delta",
+                        data: { delta: "", content: JSON.stringify(delta) },
+                    };
+                }
+            }
+            const finalObject = await result.output;
+            assistantMessage.content = JSON.stringify(finalObject);
+            if (completionEventType && !silent) {
                 yield {
-                    type: "agent:output-delta",
-                    data: { delta, content: assistantMessage.content },
+                    type: completionEventType,
+                    data: finalObject,
                 };
             }
         }
+        else {
+            for await (const delta of result.textStream) {
+                assistantMessage.content += delta;
+                if (!silent) {
+                    yield {
+                        type: "agent:output-delta",
+                        data: { delta, content: assistantMessage.content },
+                    };
+                }
+            }
+        }
         const assistantText = assistantMessage.content;
         // Wait for tool calls to complete
         const toolCalls = await result.toolCalls;
@@ -252,10 +292,13 @@ export const llmPlugin = (options) => (builder) => {
         }
         else {
             if (completionEventType && !silent) {
-                yield {
-                    type: completionEventType,
-                    data: { content: assistantText },
-                };
+                // If it's structured output, we already yielded the final object
+                if (!outputSchema) {
+                    yield {
+                        type: completionEventType,
+                        data: { content: assistantText },
+                    };
+                }
             }
         }
         const usage = await result.usage;
@@ -307,20 +350,31 @@ export const llmPlugin = (options) => (builder) => {
     });
     // Feed action results back as system-role feedback to the model.
     builder.on(actionResultInputType, async function* (event, context) {
-        const { action, result, toolCallId } = event.data;
+        const { action, result, toolCallId, halt } = event.data;
         const normalizedAction = typeof action === "string" ? action : "unknown";
         const summary = typeof result === "string" ? result : JSON.stringify(result);
-        yield* routeToLLM({
+        const toolResultMessage = {
             role: "tool",
             content: [{
-                    type: 'tool-result',
+                    type: "tool-result",
                     toolCallId,
                     toolName: normalizedAction,
                     output: {
-                        type: 'text',
+                        type: "text",
                         value: summary,
                     },
                 }],
-        }, context);
+        };
+        // Hard-stop mode: record tool result to unblock pending calls, but do not
+        // continue the autonomous tool loop in this turn.
+        if (halt === true) {
+            const state = context.state;
+            if (!state.messages) {
+                state.messages = [];
+            }
+            insertToolResult(state.messages, toolResultMessage);
+            return;
+        }
+        yield* routeToLLM(toolResultMessage, context);
     });
 };

package/dist/plugins/memory/index.js

@@ -1,4 +1,4 @@
-import { ui } from "@melony/ui-kit/server";
+import { uiEvent } from "../../ui/block.js";
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
 import { createMemoryModule } from "./memory.js";
@@ -214,7 +214,7 @@ export const memoryPlugin = (options) => (builder) => {
         }
     });
     builder.on("memory:status", async function* (event) {
-        yield ui.event(statusWidget(event.data.message, event.data.severity));
+        yield uiEvent(statusWidget(event.data.message, event.data.severity));
     });
 };
 export default memoryPlugin;

package/dist/plugins/shell/index.js

@@ -1,4 +1,4 @@
-import { ui } from "@melony/ui-kit/server";
+import { uiEvent } from "../../ui/block.js";
 import { z } from "zod";
 import { exec } from "node:child_process";
 import { promisify } from "node:util";
@@ -95,6 +95,6 @@ export const shellPlugin = (options = {}) => (builder) => {
         }
     });
     builder.on("shell:status", async function* (event) {
-        yield ui.event(statusWidget(event.data.message, event.data.severity));
+        yield uiEvent(statusWidget(event.data.message, event.data.severity));
     });
 };

package/dist/plugins/skills/index.js

@@ -1,4 +1,4 @@
-import { ui } from "@melony/ui-kit/server";
+import { uiEvent, block } from "../../ui/block.js";
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
 import matter from "gray-matter";
@@ -275,11 +275,11 @@ export const skillsPlugin = (options) => (builder) => {
         }
     });
     builder.on("skills:status", async function* (event) {
-        yield ui.event(statusWidget(event.data.message, event.data.severity));
+        yield uiEvent(statusWidget(event.data.message, event.data.severity));
     });
     builder.on("skills:loaded", async function* (event) {
-        yield ui.event(resourceCardWidget(event.data.title, "", [
-            ui.text(event.data.instructions),
+        yield uiEvent(resourceCardWidget(event.data.title, "", [
+            block('text', { value: event.data.instructions }),
         ]));
     });
 };