@intent-systems/nexus 2026.1.5-3

package/dist/agents/tools/browser-tool.js

@@ -0,0 +1,295 @@
+import { Type } from "@sinclair/typebox";
+import { browserCloseTab, browserFocusTab, browserOpenTab, browserSnapshot, browserStart, browserStatus, browserStop, browserTabs, } from "../../browser/client.js";
+import { browserAct, browserArmDialog, browserArmFileChooser, browserConsoleMessages, browserNavigate, browserPdfSave, browserScreenshotAction, } from "../../browser/client-actions.js";
+import { DEFAULT_AI_SNAPSHOT_MAX_CHARS } from "../../browser/constants.js";
+import { resolveBrowserConfig } from "../../browser/config.js";
+import { loadConfig } from "../../config/config.js";
+import { imageResultFromFile, jsonResult, readStringParam, } from "./common.js";
+const BrowserActSchema = Type.Union([
+    Type.Object({
+        kind: Type.Literal("click"),
+        ref: Type.String(),
+        targetId: Type.Optional(Type.String()),
+        doubleClick: Type.Optional(Type.Boolean()),
+        button: Type.Optional(Type.String()),
+        modifiers: Type.Optional(Type.Array(Type.String())),
+    }),
+    Type.Object({
+        kind: Type.Literal("type"),
+        ref: Type.String(),
+        text: Type.String(),
+        targetId: Type.Optional(Type.String()),
+        submit: Type.Optional(Type.Boolean()),
+        slowly: Type.Optional(Type.Boolean()),
+    }),
+    Type.Object({
+        kind: Type.Literal("press"),
+        key: Type.String(),
+        targetId: Type.Optional(Type.String()),
+    }),
+    Type.Object({
+        kind: Type.Literal("hover"),
+        ref: Type.String(),
+        targetId: Type.Optional(Type.String()),
+    }),
+    Type.Object({
+        kind: Type.Literal("drag"),
+        startRef: Type.String(),
+        endRef: Type.String(),
+        targetId: Type.Optional(Type.String()),
+    }),
+    Type.Object({
+        kind: Type.Literal("select"),
+        ref: Type.String(),
+        values: Type.Array(Type.String()),
+        targetId: Type.Optional(Type.String()),
+    }),
+    Type.Object({
+        kind: Type.Literal("fill"),
+        fields: Type.Array(Type.Record(Type.String(), Type.Unknown())),
+        targetId: Type.Optional(Type.String()),
+    }),
+    Type.Object({
+        kind: Type.Literal("resize"),
+        width: Type.Number(),
+        height: Type.Number(),
+        targetId: Type.Optional(Type.String()),
+    }),
+    Type.Object({
+        kind: Type.Literal("wait"),
+        timeMs: Type.Optional(Type.Number()),
+        text: Type.Optional(Type.String()),
+        textGone: Type.Optional(Type.String()),
+        targetId: Type.Optional(Type.String()),
+    }),
+    Type.Object({
+        kind: Type.Literal("evaluate"),
+        fn: Type.String(),
+        ref: Type.Optional(Type.String()),
+        targetId: Type.Optional(Type.String()),
+    }),
+    Type.Object({
+        kind: Type.Literal("close"),
+        targetId: Type.Optional(Type.String()),
+    }),
+]);
+// IMPORTANT: OpenAI function tool schemas must have a top-level `type: "object"`.
+// A root-level `Type.Union([...])` compiles to `{ anyOf: [...] }` (no `type`),
+// which OpenAI rejects ("Invalid schema ... type: None"). Keep this schema an object.
+const BrowserToolSchema = Type.Object({
+    action: Type.Union([
+        Type.Literal("status"),
+        Type.Literal("start"),
+        Type.Literal("stop"),
+        Type.Literal("tabs"),
+        Type.Literal("open"),
+        Type.Literal("focus"),
+        Type.Literal("close"),
+        Type.Literal("snapshot"),
+        Type.Literal("screenshot"),
+        Type.Literal("navigate"),
+        Type.Literal("console"),
+        Type.Literal("pdf"),
+        Type.Literal("upload"),
+        Type.Literal("dialog"),
+        Type.Literal("act"),
+    ]),
+    controlUrl: Type.Optional(Type.String()),
+    targetUrl: Type.Optional(Type.String()),
+    targetId: Type.Optional(Type.String()),
+    limit: Type.Optional(Type.Number()),
+    format: Type.Optional(Type.Union([Type.Literal("aria"), Type.Literal("ai")])),
+    fullPage: Type.Optional(Type.Boolean()),
+    ref: Type.Optional(Type.String()),
+    element: Type.Optional(Type.String()),
+    type: Type.Optional(Type.Union([Type.Literal("png"), Type.Literal("jpeg")])),
+    level: Type.Optional(Type.String()),
+    paths: Type.Optional(Type.Array(Type.String())),
+    inputRef: Type.Optional(Type.String()),
+    timeoutMs: Type.Optional(Type.Number()),
+    accept: Type.Optional(Type.Boolean()),
+    promptText: Type.Optional(Type.String()),
+    request: Type.Optional(BrowserActSchema),
+});
+function resolveBrowserBaseUrl(controlUrl) {
+    const cfg = loadConfig();
+    const resolved = resolveBrowserConfig(cfg.browser);
+    if (!resolved.enabled && !controlUrl?.trim()) {
+        throw new Error("Browser control is disabled. Set browser.enabled=true in ~/.nexus/nexus.json.");
+    }
+    const url = controlUrl?.trim() ? controlUrl.trim() : resolved.controlUrl;
+    return url.replace(/\/$/, "");
+}
+export function createBrowserTool(opts) {
+    return {
+        label: "Browser",
+        name: "browser",
+        description: "Control nexus's dedicated browser (status/start/stop/tabs/open/snapshot/screenshot/actions). Use snapshot+act for UI automation. Avoid act:wait by default; use only in exceptional cases when no reliable UI state exists.",
+        parameters: BrowserToolSchema,
+        execute: async (_toolCallId, args) => {
+            const params = args;
+            const action = readStringParam(params, "action", { required: true });
+            const controlUrl = readStringParam(params, "controlUrl");
+            const baseUrl = resolveBrowserBaseUrl(controlUrl ?? opts?.defaultControlUrl);
+            switch (action) {
+                case "status":
+                    return jsonResult(await browserStatus(baseUrl));
+                case "start":
+                    await browserStart(baseUrl);
+                    return jsonResult(await browserStatus(baseUrl));
+                case "stop":
+                    await browserStop(baseUrl);
+                    return jsonResult(await browserStatus(baseUrl));
+                case "tabs":
+                    return jsonResult({ tabs: await browserTabs(baseUrl) });
+                case "open": {
+                    const targetUrl = readStringParam(params, "targetUrl", {
+                        required: true,
+                    });
+                    return jsonResult(await browserOpenTab(baseUrl, targetUrl));
+                }
+                case "focus": {
+                    const targetId = readStringParam(params, "targetId", {
+                        required: true,
+                    });
+                    await browserFocusTab(baseUrl, targetId);
+                    return jsonResult({ ok: true });
+                }
+                case "close": {
+                    const targetId = readStringParam(params, "targetId");
+                    if (targetId)
+                        await browserCloseTab(baseUrl, targetId);
+                    else
+                        await browserAct(baseUrl, { kind: "close" });
+                    return jsonResult({ ok: true });
+                }
+                case "snapshot": {
+                    const format = params.format === "ai" || params.format === "aria"
+                        ? params.format
+                        : "ai";
+                    const targetId = typeof params.targetId === "string"
+                        ? params.targetId.trim()
+                        : undefined;
+                    const limit = typeof params.limit === "number" && Number.isFinite(params.limit)
+                        ? params.limit
+                        : undefined;
+                    const maxChars = typeof params.maxChars === "number" &&
+                        Number.isFinite(params.maxChars) &&
+                        params.maxChars > 0
+                        ? Math.floor(params.maxChars)
+                        : undefined;
+                    const resolvedMaxChars = format === "ai" ? maxChars ?? DEFAULT_AI_SNAPSHOT_MAX_CHARS : undefined;
+                    const snapshot = await browserSnapshot(baseUrl, {
+                        format,
+                        targetId,
+                        limit,
+                    });
+                    if (snapshot.format === "ai") {
+                        return {
+                            content: [{ type: "text", text: snapshot.snapshot }],
+                            details: snapshot,
+                        };
+                    }
+                    return jsonResult(snapshot);
+                }
+                case "screenshot": {
+                    const targetId = readStringParam(params, "targetId");
+                    const fullPage = Boolean(params.fullPage);
+                    const ref = readStringParam(params, "ref");
+                    const element = readStringParam(params, "element");
+                    const type = params.type === "jpeg" ? "jpeg" : "png";
+                    const result = await browserScreenshotAction(baseUrl, {
+                        targetId,
+                        fullPage,
+                        ref,
+                        element,
+                        type,
+                    });
+                    return await imageResultFromFile({
+                        label: "browser:screenshot",
+                        path: result.path,
+                        details: result,
+                    });
+                }
+                case "navigate": {
+                    const targetUrl = readStringParam(params, "targetUrl", {
+                        required: true,
+                    });
+                    const targetId = readStringParam(params, "targetId");
+                    return jsonResult(await browserNavigate(baseUrl, { url: targetUrl, targetId }));
+                }
+                case "console": {
+                    const level = typeof params.level === "string" ? params.level.trim() : undefined;
+                    const targetId = typeof params.targetId === "string"
+                        ? params.targetId.trim()
+                        : undefined;
+                    return jsonResult(await browserConsoleMessages(baseUrl, { level, targetId }));
+                }
+                case "pdf": {
+                    const targetId = typeof params.targetId === "string"
+                        ? params.targetId.trim()
+                        : undefined;
+                    const result = await browserPdfSave(baseUrl, { targetId });
+                    return {
+                        content: [{ type: "text", text: `FILE:${result.path}` }],
+                        details: result,
+                    };
+                }
+                case "upload": {
+                    const paths = Array.isArray(params.paths)
+                        ? params.paths.map((p) => String(p))
+                        : [];
+                    if (paths.length === 0)
+                        throw new Error("paths required");
+                    const ref = readStringParam(params, "ref");
+                    const inputRef = readStringParam(params, "inputRef");
+                    const element = readStringParam(params, "element");
+                    const targetId = typeof params.targetId === "string"
+                        ? params.targetId.trim()
+                        : undefined;
+                    const timeoutMs = typeof params.timeoutMs === "number" &&
+                        Number.isFinite(params.timeoutMs)
+                        ? params.timeoutMs
+                        : undefined;
+                    return jsonResult(await browserArmFileChooser(baseUrl, {
+                        paths,
+                        ref,
+                        inputRef,
+                        element,
+                        targetId,
+                        timeoutMs,
+                    }));
+                }
+                case "dialog": {
+                    const accept = Boolean(params.accept);
+                    const promptText = typeof params.promptText === "string"
+                        ? params.promptText
+                        : undefined;
+                    const targetId = typeof params.targetId === "string"
+                        ? params.targetId.trim()
+                        : undefined;
+                    const timeoutMs = typeof params.timeoutMs === "number" &&
+                        Number.isFinite(params.timeoutMs)
+                        ? params.timeoutMs
+                        : undefined;
+                    return jsonResult(await browserArmDialog(baseUrl, {
+                        accept,
+                        promptText,
+                        targetId,
+                        timeoutMs,
+                    }));
+                }
+                case "act": {
+                    const request = params.request;
+                    if (!request || typeof request !== "object") {
+                        throw new Error("request required");
+                    }
+                    const result = await browserAct(baseUrl, request);
+                    return jsonResult(result);
+                }
+                default:
+                    throw new Error(`Unknown action: ${action}`);
+            }
+        },
+    };
+}

package/dist/agents/tools/canvas-tool.js

@@ -0,0 +1,193 @@
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import { Type } from "@sinclair/typebox";
+import { writeBase64ToFile } from "../../cli/nodes-camera.js";
+import { canvasSnapshotTempPath, parseCanvasSnapshotPayload, } from "../../cli/nodes-canvas.js";
+import { imageMimeFromFormat } from "../../media/mime.js";
+import { imageResult, jsonResult, readStringParam, } from "./common.js";
+import { callGatewayTool } from "./gateway.js";
+import { resolveNodeId } from "./nodes-utils.js";
+const CanvasToolSchema = Type.Union([
+    Type.Object({
+        action: Type.Literal("present"),
+        gatewayUrl: Type.Optional(Type.String()),
+        gatewayToken: Type.Optional(Type.String()),
+        timeoutMs: Type.Optional(Type.Number()),
+        node: Type.Optional(Type.String()),
+        target: Type.Optional(Type.String()),
+        x: Type.Optional(Type.Number()),
+        y: Type.Optional(Type.Number()),
+        width: Type.Optional(Type.Number()),
+        height: Type.Optional(Type.Number()),
+    }),
+    Type.Object({
+        action: Type.Literal("hide"),
+        gatewayUrl: Type.Optional(Type.String()),
+        gatewayToken: Type.Optional(Type.String()),
+        timeoutMs: Type.Optional(Type.Number()),
+        node: Type.Optional(Type.String()),
+    }),
+    Type.Object({
+        action: Type.Literal("navigate"),
+        gatewayUrl: Type.Optional(Type.String()),
+        gatewayToken: Type.Optional(Type.String()),
+        timeoutMs: Type.Optional(Type.Number()),
+        node: Type.Optional(Type.String()),
+        url: Type.String(),
+    }),
+    Type.Object({
+        action: Type.Literal("eval"),
+        gatewayUrl: Type.Optional(Type.String()),
+        gatewayToken: Type.Optional(Type.String()),
+        timeoutMs: Type.Optional(Type.Number()),
+        node: Type.Optional(Type.String()),
+        javaScript: Type.String(),
+    }),
+    Type.Object({
+        action: Type.Literal("snapshot"),
+        gatewayUrl: Type.Optional(Type.String()),
+        gatewayToken: Type.Optional(Type.String()),
+        timeoutMs: Type.Optional(Type.Number()),
+        node: Type.Optional(Type.String()),
+        format: Type.Optional(Type.Union([
+            Type.Literal("png"),
+            Type.Literal("jpg"),
+            Type.Literal("jpeg"),
+        ])),
+        maxWidth: Type.Optional(Type.Number()),
+        quality: Type.Optional(Type.Number()),
+        delayMs: Type.Optional(Type.Number()),
+    }),
+    Type.Object({
+        action: Type.Literal("a2ui_push"),
+        gatewayUrl: Type.Optional(Type.String()),
+        gatewayToken: Type.Optional(Type.String()),
+        timeoutMs: Type.Optional(Type.Number()),
+        node: Type.Optional(Type.String()),
+        jsonl: Type.Optional(Type.String()),
+        jsonlPath: Type.Optional(Type.String()),
+    }),
+    Type.Object({
+        action: Type.Literal("a2ui_reset"),
+        gatewayUrl: Type.Optional(Type.String()),
+        gatewayToken: Type.Optional(Type.String()),
+        timeoutMs: Type.Optional(Type.Number()),
+        node: Type.Optional(Type.String()),
+    }),
+]);
+export function createCanvasTool() {
+    return {
+        label: "Canvas",
+        name: "canvas",
+        description: "Control node canvases (present/hide/navigate/eval/snapshot/A2UI). Use snapshot to capture the rendered UI.",
+        parameters: CanvasToolSchema,
+        execute: async (_toolCallId, args) => {
+            const params = args;
+            const action = readStringParam(params, "action", { required: true });
+            const gatewayOpts = {
+                gatewayUrl: readStringParam(params, "gatewayUrl", { trim: false }),
+                gatewayToken: readStringParam(params, "gatewayToken", { trim: false }),
+                timeoutMs: typeof params.timeoutMs === "number" ? params.timeoutMs : undefined,
+            };
+            const nodeId = await resolveNodeId(gatewayOpts, readStringParam(params, "node", { trim: true }), true);
+            const invoke = async (command, invokeParams) => await callGatewayTool("node.invoke", gatewayOpts, {
+                nodeId,
+                command,
+                params: invokeParams,
+                idempotencyKey: crypto.randomUUID(),
+            });
+            switch (action) {
+                case "present": {
+                    const placement = {
+                        x: typeof params.x === "number" ? params.x : undefined,
+                        y: typeof params.y === "number" ? params.y : undefined,
+                        width: typeof params.width === "number" ? params.width : undefined,
+                        height: typeof params.height === "number" ? params.height : undefined,
+                    };
+                    const invokeParams = {};
+                    if (typeof params.target === "string" && params.target.trim()) {
+                        invokeParams.url = params.target.trim();
+                    }
+                    if (Number.isFinite(placement.x) ||
+                        Number.isFinite(placement.y) ||
+                        Number.isFinite(placement.width) ||
+                        Number.isFinite(placement.height)) {
+                        invokeParams.placement = placement;
+                    }
+                    await invoke("canvas.present", invokeParams);
+                    return jsonResult({ ok: true });
+                }
+                case "hide":
+                    await invoke("canvas.hide", undefined);
+                    return jsonResult({ ok: true });
+                case "navigate": {
+                    const url = readStringParam(params, "url", { required: true });
+                    await invoke("canvas.navigate", { url });
+                    return jsonResult({ ok: true });
+                }
+                case "eval": {
+                    const javaScript = readStringParam(params, "javaScript", {
+                        required: true,
+                    });
+                    const raw = (await invoke("canvas.eval", { javaScript }));
+                    const result = raw?.payload?.result;
+                    if (result) {
+                        return {
+                            content: [{ type: "text", text: result }],
+                            details: { result },
+                        };
+                    }
+                    return jsonResult({ ok: true });
+                }
+                case "snapshot": {
+                    const formatRaw = typeof params.format === "string"
+                        ? params.format.toLowerCase()
+                        : "png";
+                    const format = formatRaw === "jpg" || formatRaw === "jpeg" ? "jpeg" : "png";
+                    const maxWidth = typeof params.maxWidth === "number" &&
+                        Number.isFinite(params.maxWidth)
+                        ? params.maxWidth
+                        : undefined;
+                    const quality = typeof params.quality === "number" &&
+                        Number.isFinite(params.quality)
+                        ? params.quality
+                        : undefined;
+                    const raw = (await invoke("canvas.snapshot", {
+                        format,
+                        maxWidth,
+                        quality,
+                    }));
+                    const payload = parseCanvasSnapshotPayload(raw?.payload);
+                    const filePath = canvasSnapshotTempPath({
+                        ext: payload.format === "jpeg" ? "jpg" : payload.format,
+                    });
+                    await writeBase64ToFile(filePath, payload.base64);
+                    const mimeType = imageMimeFromFormat(payload.format) ?? "image/png";
+                    return await imageResult({
+                        label: "canvas:snapshot",
+                        path: filePath,
+                        base64: payload.base64,
+                        mimeType,
+                        details: { format: payload.format },
+                    });
+                }
+                case "a2ui_push": {
+                    const jsonl = typeof params.jsonl === "string" && params.jsonl.trim()
+                        ? params.jsonl
+                        : typeof params.jsonlPath === "string" && params.jsonlPath.trim()
+                            ? await fs.readFile(params.jsonlPath.trim(), "utf8")
+                            : "";
+                    if (!jsonl.trim())
+                        throw new Error("jsonl or jsonlPath required");
+                    await invoke("canvas.a2ui.pushJSONL", { jsonl });
+                    return jsonResult({ ok: true });
+                }
+                case "a2ui_reset":
+                    await invoke("canvas.a2ui.reset", undefined);
+                    return jsonResult({ ok: true });
+                default:
+                    throw new Error(`Unknown action: ${action}`);
+            }
+        },
+    };
+}

package/dist/agents/tools/common.js

@@ -0,0 +1,88 @@
+import fs from "node:fs/promises";
+import { detectMime } from "../../media/mime.js";
+import { sanitizeToolResultImages } from "../tool-images.js";
+export function readStringParam(params, key, options = {}) {
+    const { required = false, trim = true, label = key } = options;
+    const raw = params[key];
+    if (typeof raw !== "string") {
+        if (required)
+            throw new Error(`${label} required`);
+        return undefined;
+    }
+    const value = trim ? raw.trim() : raw;
+    if (!value) {
+        if (required)
+            throw new Error(`${label} required`);
+        return undefined;
+    }
+    return value;
+}
+export function readStringArrayParam(params, key, options = {}) {
+    const { required = false, label = key } = options;
+    const raw = params[key];
+    if (Array.isArray(raw)) {
+        const values = raw
+            .filter((entry) => typeof entry === "string")
+            .map((entry) => entry.trim())
+            .filter(Boolean);
+        if (values.length === 0) {
+            if (required)
+                throw new Error(`${label} required`);
+            return undefined;
+        }
+        return values;
+    }
+    if (typeof raw === "string") {
+        const value = raw.trim();
+        if (!value) {
+            if (required)
+                throw new Error(`${label} required`);
+            return undefined;
+        }
+        return [value];
+    }
+    if (required)
+        throw new Error(`${label} required`);
+    return undefined;
+}
+export function jsonResult(payload) {
+    return {
+        content: [
+            {
+                type: "text",
+                text: JSON.stringify(payload, null, 2),
+            },
+        ],
+        details: payload,
+    };
+}
+export async function imageResult(params) {
+    const content = [
+        {
+            type: "text",
+            text: params.extraText ?? `MEDIA:${params.path}`,
+        },
+        {
+            type: "image",
+            data: params.base64,
+            mimeType: params.mimeType,
+        },
+    ];
+    const result = {
+        content,
+        details: { path: params.path, ...params.details },
+    };
+    return await sanitizeToolResultImages(result, params.label);
+}
+export async function imageResultFromFile(params) {
+    const buf = await fs.readFile(params.path);
+    const mimeType = (await detectMime({ buffer: buf.slice(0, 256) })) ?? "image/png";
+    return await imageResult({
+        label: params.label,
+        path: params.path,
+        base64: buf.toString("base64"),
+        mimeType,
+        extraText: params.extraText,
+        details: params.details,
+    });
+}