@next-open-ai/openbot 0.1.1

package/dist/tools/browser-tool.js

@@ -0,0 +1,362 @@
+import { Type } from "@sinclair/typebox";
+import { BrowserManager } from "agent-browser/dist/browser.js";
+/**
+ * Browser tool schema - defines all possible browser actions
+ */
+const BrowserToolSchema = Type.Object({
+    action: Type.Union([
+        Type.Literal("navigate"),
+        Type.Literal("snapshot"),
+        Type.Literal("screenshot"),
+        Type.Literal("click"),
+        Type.Literal("type"),
+        Type.Literal("fill"),
+        Type.Literal("scroll"),
+        Type.Literal("extract"),
+        Type.Literal("wait"),
+        Type.Literal("back"),
+        Type.Literal("forward"),
+        Type.Literal("download"),
+        Type.Literal("close"),
+    ], { description: "Browser action to perform" }),
+    url: Type.Optional(Type.String({ description: "URL to navigate to (for navigate action) or download from (for download action)" })),
+    selector: Type.Optional(Type.String({ description: "CSS selector or ref (e.g., @e1) to target element" })),
+    text: Type.Optional(Type.String({ description: "Text to type or fill (for type/fill actions)" })),
+    direction: Type.Optional(Type.Union([
+        Type.Literal("up"),
+        Type.Literal("down"),
+        Type.Literal("left"),
+        Type.Literal("right"),
+    ], { description: "Scroll direction" })),
+    pixels: Type.Optional(Type.Number({ description: "Number of pixels to scroll" })),
+    interactive: Type.Optional(Type.Boolean({ description: "Get only interactive elements in snapshot" })),
+    fullPage: Type.Optional(Type.Boolean({ description: "Take full page screenshot" })),
+    timeout: Type.Optional(Type.Number({ description: "Timeout in milliseconds for wait action" })),
+    downloadPath: Type.Optional(Type.String({ description: "Local path to save downloaded file (for download action)" })),
+});
+/**
+ * Shared browser manager instance
+ */
+let browserManager = null;
+/**
+ * Get or create browser manager instance
+ */
+async function getBrowserManager() {
+    if (!browserManager) {
+        browserManager = new BrowserManager();
+    }
+    // Launch browser if not already launched
+    if (!browserManager.isLaunched()) {
+        await browserManager.launch({
+            action: "launch",
+            id: "default",
+            headless: true,
+            viewport: { width: 1280, height: 720 },
+        });
+    }
+    return browserManager;
+}
+/**
+ * Create browser automation tool
+ * @param workspaceDir Optional workspace directory for downloads
+ */
+export function createBrowserTool(workspaceDir) {
+    return {
+        name: "browser",
+        label: "Browser",
+        description: [
+            "Control a headless browser for web automation.",
+            "Actions: navigate (go to URL), snapshot (get page content with refs), screenshot (capture image),",
+            "click (click element), type (type text), fill (clear and fill input), scroll (scroll page),",
+            "extract (get text from element), wait (wait for element), download (download file from URL or by clicking),",
+            "back/forward (navigation), close (close browser).",
+            "Use refs from snapshot (e.g., @e1, @e2) to interact with elements reliably.",
+        ].join(" "),
+        parameters: BrowserToolSchema,
+        execute: async (_toolCallId, args) => {
+            const params = args;
+            console.log(`[Browser Tool] Executing action: ${params.action}`);
+            try {
+                // Ensure browser is ready
+                const browser = await getBrowserManager();
+                console.log(`[Browser Tool] Browser manager ready`);
+                switch (params.action) {
+                    case "navigate": {
+                        if (!params.url) {
+                            throw new Error("URL is required for navigate action");
+                        }
+                        const page = browser.getPage();
+                        await page.goto(params.url, { waitUntil: "domcontentloaded" });
+                        const url = page.url();
+                        const title = await page.title();
+                        return {
+                            content: [{
+                                    type: "text",
+                                    text: `Navigated to: ${title}\nURL: ${url}`
+                                }],
+                            details: { url, title },
+                        };
+                    }
+                    case "snapshot": {
+                        const snapshot = await browser.getSnapshot({
+                            interactive: params.interactive ?? true,
+                            compact: true,
+                        });
+                        const page = browser.getPage();
+                        const url = page.url();
+                        const title = await page.title();
+                        return {
+                            content: [{
+                                    type: "text",
+                                    text: `Page: ${title}\nURL: ${url}\n\n${snapshot.tree}`
+                                }],
+                            details: {
+                                url,
+                                title,
+                                snapshot: snapshot.tree,
+                                refs: snapshot.refs
+                            },
+                        };
+                    }
+                    case "screenshot": {
+                        const page = browser.getPage();
+                        const screenshot = await page.screenshot({
+                            fullPage: params.fullPage ?? false,
+                            type: "png",
+                        });
+                        const base64 = screenshot.toString("base64");
+                        return {
+                            content: [
+                                { type: "text", text: "Screenshot captured" },
+                                {
+                                    type: "image",
+                                    data: base64,
+                                    mimeType: "image/png"
+                                },
+                            ],
+                            details: { fullPage: params.fullPage ?? false },
+                        };
+                    }
+                    case "click": {
+                        if (!params.selector) {
+                            throw new Error("Selector is required for click action");
+                        }
+                        const locator = browser.getLocator(params.selector);
+                        await locator.click();
+                        return {
+                            content: [{
+                                    type: "text",
+                                    text: `Clicked element: ${params.selector}`
+                                }],
+                            details: { selector: params.selector },
+                        };
+                    }
+                    case "type": {
+                        if (!params.selector || !params.text) {
+                            throw new Error("Selector and text are required for type action");
+                        }
+                        const locator = browser.getLocator(params.selector);
+                        await locator.pressSequentially(params.text);
+                        return {
+                            content: [{
+                                    type: "text",
+                                    text: `Typed text into: ${params.selector}`
+                                }],
+                            details: { selector: params.selector, text: params.text },
+                        };
+                    }
+                    case "fill": {
+                        if (!params.selector || !params.text) {
+                            throw new Error("Selector and text are required for fill action");
+                        }
+                        const locator = browser.getLocator(params.selector);
+                        await locator.fill(params.text);
+                        return {
+                            content: [{
+                                    type: "text",
+                                    text: `Filled text into: ${params.selector}`
+                                }],
+                            details: { selector: params.selector, text: params.text },
+                        };
+                    }
+                    case "scroll": {
+                        const page = browser.getPage();
+                        const direction = params.direction ?? "down";
+                        const pixels = params.pixels ?? 500;
+                        const scrollMap = {
+                            down: { x: 0, y: pixels },
+                            up: { x: 0, y: -pixels },
+                            right: { x: pixels, y: 0 },
+                            left: { x: -pixels, y: 0 },
+                        };
+                        const { x, y } = scrollMap[direction];
+                        await page.evaluate(({ x, y }) => {
+                            window.scrollBy(x, y);
+                        }, { x, y });
+                        return {
+                            content: [{
+                                    type: "text",
+                                    text: `Scrolled ${direction} by ${pixels}px`
+                                }],
+                            details: { direction, pixels },
+                        };
+                    }
+                    case "extract": {
+                        if (!params.selector) {
+                            throw new Error("Selector is required for extract action");
+                        }
+                        const locator = browser.getLocator(params.selector);
+                        const text = await locator.textContent();
+                        return {
+                            content: [{
+                                    type: "text",
+                                    text: `Extracted text: ${text || "(empty)"}`
+                                }],
+                            details: { selector: params.selector, text },
+                        };
+                    }
+                    case "wait": {
+                        if (!params.selector) {
+                            throw new Error("Selector is required for wait action");
+                        }
+                        const locator = browser.getLocator(params.selector);
+                        await locator.waitFor({
+                            state: "visible",
+                            timeout: params.timeout ?? 30000
+                        });
+                        return {
+                            content: [{
+                                    type: "text",
+                                    text: `Element is now visible: ${params.selector}`
+                                }],
+                            details: { selector: params.selector },
+                        };
+                    }
+                    case "back": {
+                        const page = browser.getPage();
+                        await page.goBack();
+                        const url = page.url();
+                        return {
+                            content: [{
+                                    type: "text",
+                                    text: `Navigated back to: ${url}`
+                                }],
+                            details: { url },
+                        };
+                    }
+                    case "forward": {
+                        const page = browser.getPage();
+                        await page.goForward();
+                        const url = page.url();
+                        return {
+                            content: [{
+                                    type: "text",
+                                    text: `Navigated forward to: ${url}`
+                                }],
+                            details: { url },
+                        };
+                    }
+                    case "download": {
+                        const fs = await import("fs/promises");
+                        const path = await import("path");
+                        // Create downloads directory if it doesn't exist
+                        const downloadsDir = workspaceDir ? path.join(workspaceDir, "downloads") : path.join(process.cwd(), "downloads");
+                        try {
+                            await fs.mkdir(downloadsDir, { recursive: true });
+                        }
+                        catch (err) {
+                            // Directory might already exist, ignore
+                        }
+                        // Hybrid approach: URL download or click-triggered download
+                        if (params.url) {
+                            // Direct URL download
+                            const response = await fetch(params.url);
+                            if (!response.ok) {
+                                throw new Error(`Download failed: ${response.statusText}`);
+                            }
+                            const buffer = await response.arrayBuffer();
+                            const filename = params.downloadPath || path.basename(params.url) || `download_${Date.now()}`;
+                            const fullPath = path.isAbsolute(filename)
+                                ? filename
+                                : path.join(downloadsDir, filename);
+                            await fs.writeFile(fullPath, Buffer.from(buffer));
+                            return {
+                                content: [{
+                                        type: "text",
+                                        text: `Downloaded file to: ${fullPath}\nSize: ${buffer.byteLength} bytes`
+                                    }],
+                                details: {
+                                    path: fullPath,
+                                    size: buffer.byteLength,
+                                    url: params.url
+                                },
+                            };
+                        }
+                        else if (params.selector) {
+                            // Click-triggered download using Playwright
+                            const page = browser.getPage();
+                            const downloadPromise = page.waitForEvent('download', { timeout: 30000 });
+                            const locator = browser.getLocator(params.selector);
+                            await locator.click();
+                            const download = await downloadPromise;
+                            const suggestedFilename = download.suggestedFilename();
+                            const filename = params.downloadPath || suggestedFilename || `download_${Date.now()}`;
+                            const fullPath = path.isAbsolute(filename)
+                                ? filename
+                                : path.join(downloadsDir, filename);
+                            await download.saveAs(fullPath);
+                            return {
+                                content: [{
+                                        type: "text",
+                                        text: `Downloaded file to: ${fullPath}\nOriginal filename: ${suggestedFilename}`
+                                    }],
+                                details: {
+                                    path: fullPath,
+                                    originalFilename: suggestedFilename,
+                                    selector: params.selector
+                                },
+                            };
+                        }
+                        else {
+                            throw new Error("Download action requires either url or selector parameter");
+                        }
+                    }
+                    case "close": {
+                        if (browserManager) {
+                            await browserManager.close();
+                            browserManager = null;
+                        }
+                        return {
+                            content: [{
+                                    type: "text",
+                                    text: "Browser closed"
+                                }],
+                            details: { closed: true },
+                        };
+                    }
+                    default:
+                        throw new Error(`Unknown action: ${params.action}`);
+                }
+            }
+            catch (error) {
+                const errorMessage = error instanceof Error ? error.message : String(error);
+                return {
+                    content: [{
+                            type: "text",
+                            text: `Browser action failed: ${errorMessage}`
+                        }],
+                    details: { error: errorMessage },
+                };
+            }
+        },
+    };
+}
+/**
+ * Cleanup function to close browser on process exit
+ */
+export async function closeBrowser() {
+    if (browserManager) {
+        await browserManager.close();
+        browserManager = null;
+    }
+}

package/dist/tools/index.d.ts

@@ -0,0 +1,3 @@
+export { createBrowserTool, closeBrowser } from "./browser-tool.js";
+export { createSaveExperienceTool } from "./save-experience-tool.js";
+export { createInstallSkillTool } from "./install-skill-tool.js";

package/dist/tools/index.js

@@ -0,0 +1,3 @@
+export { createBrowserTool, closeBrowser } from "./browser-tool.js";
+export { createSaveExperienceTool } from "./save-experience-tool.js";
+export { createInstallSkillTool } from "./install-skill-tool.js";

package/dist/tools/install-skill-tool.d.ts

@@ -0,0 +1,9 @@
+import type { ToolDefinition } from "@mariozechner/pi-coding-agent";
+/**
+ * 创建 install_skill 工具：通过后端 API 安装技能。
+ * 创建时绑定 targetAgentId，安装目标由此决定：
+ * - 具体 agentId → 安装到该 agent 的 workspace/skills
+ * - "global"|"all" → 安装到全局 OPENBOT_AGENT_DIR/skills
+ * - 未传则后端按默认/全局处理
+ */
+export declare function createInstallSkillTool(targetAgentId: string | undefined): ToolDefinition;

package/dist/tools/install-skill-tool.js

@@ -0,0 +1,77 @@
+import { Type } from "@sinclair/typebox";
+import { getBackendBaseUrl } from "../gateway/backend-url.js";
+const InstallSkillSchema = Type.Object({
+    url: Type.String({
+        description: "Skill 安装地址，如 GitHub 简写 owner/repo、完整 URL 或 owner/repo@skillName",
+    }),
+});
+/**
+ * 创建 install_skill 工具：通过后端 API 安装技能。
+ * 创建时绑定 targetAgentId，安装目标由此决定：
+ * - 具体 agentId → 安装到该 agent 的 workspace/skills
+ * - "global"|"all" → 安装到全局 OPENBOT_AGENT_DIR/skills
+ * - 未传则后端按默认/全局处理
+ */
+export function createInstallSkillTool(targetAgentId) {
+    return {
+        name: "install_skill",
+        label: "Install Skill",
+        description: "将指定地址的技能安装到当前智能体工作区目录。在 OpenBot 中为用户安装技能时，" +
+            "应使用本工具而非 npx skills add，以保证安装到当前智能体对应的技能目录。",
+        parameters: InstallSkillSchema,
+        execute: async (_toolCallId, params, _signal, _onUpdate, _ctx) => {
+            const url = (params.url ?? "").trim();
+            if (!url) {
+                return {
+                    content: [{ type: "text", text: "请提供技能安装地址（如 owner/repo 或 owner/repo@skillName）。" }],
+                    details: undefined,
+                };
+            }
+            const base = getBackendBaseUrl();
+            if (!base) {
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: "当前环境未连接后端，无法通过工具安装。请让用户在应用内使用「手动安装」并填入该地址：" + url,
+                        },
+                    ],
+                    details: undefined,
+                };
+            }
+            const installUrl = `${base.replace(/\/$/, "")}/server-api/skills/install`;
+            try {
+                const res = await fetch(installUrl, {
+                    method: "POST",
+                    headers: { "Content-Type": "application/json" },
+                    body: JSON.stringify({ url, targetAgentId: targetAgentId }),
+                });
+                const data = (await res.json().catch(() => ({})));
+                if (!res.ok) {
+                    const msg = data?.message ?? data?.data?.stderr ?? res.statusText;
+                    return {
+                        content: [{ type: "text", text: `安装请求失败: ${msg}` }],
+                        details: undefined,
+                    };
+                }
+                const out = data?.data?.stdout ?? "";
+                const err = data?.data?.stderr ?? "";
+                const installDir = data?.data?.installDir;
+                const dirLine = installDir ? `\n安装目录：${installDir}` : "";
+                const baseText = out.trim() || (err.trim() ? `安装完成。\n${err}` : "技能已安装完成。");
+                const text = dirLine ? `${baseText}${dirLine}` : baseText;
+                return {
+                    content: [{ type: "text", text }],
+                    details: data?.data,
+                };
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                return {
+                    content: [{ type: "text", text: `安装请求异常: ${msg}` }],
+                    details: undefined,
+                };
+            }
+        },
+    };
+}

package/dist/tools/save-experience-tool.d.ts

@@ -0,0 +1,5 @@
+import type { ToolDefinition } from "@mariozechner/pi-coding-agent";
+/**
+ * 创建 save_experience 工具：对话结束时由 Agent 调用，将经验总结存入向量库（infotype: experience）
+ */
+export declare function createSaveExperienceTool(sessionId: string): ToolDefinition;

package/dist/tools/save-experience-tool.js

@@ -0,0 +1,54 @@
+import { Type } from "@sinclair/typebox";
+import { addMemory } from "../memory/index.js";
+const SaveExperienceSchema = Type.Object({
+    summary: Type.String({
+        description: "本轮对话中总结出的经验、要点或可复用的结论，用于存入长期记忆",
+    }),
+});
+/**
+ * 创建 save_experience 工具：对话结束时由 Agent 调用，将经验总结存入向量库（infotype: experience）
+ */
+export function createSaveExperienceTool(sessionId) {
+    return {
+        name: "save_experience",
+        label: "Save Experience",
+        description: "在对话结束时调用，将本轮对话中总结出的经验、要点或可复用的结论存入长期记忆，便于后续会话检索。",
+        parameters: SaveExperienceSchema,
+        execute: async (_toolCallId, params, _signal, _onUpdate, _ctx) => {
+            const text = (params.summary ?? "").trim();
+            if (!text) {
+                return {
+                    content: [{ type: "text", text: "未提供总结内容，未写入记忆。" }],
+                    details: undefined,
+                };
+            }
+            try {
+                const id = await addMemory(text, {
+                    infotype: "experience",
+                    sessionId,
+                });
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `经验已存入长期记忆（id: ${id}）。`,
+                        },
+                    ],
+                    details: { id },
+                };
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `存入记忆失败: ${msg}`,
+                        },
+                    ],
+                    details: undefined,
+                };
+            }
+        },
+    };
+}

package/package.json

@@ -0,0 +1,80 @@
+{
+    "name": "@next-open-ai/openbot",
+    "publishConfig": {
+        "access": "public"
+    },
+    "version": "0.1.1",
+    "description": "CLI and library to run prompts with skill paths (Agent Skills style). Use as npm package or openbot CLI.",
+    "type": "module",
+    "main": "dist/index.js",
+    "types": "dist/index.d.ts",
+    "exports": {
+        ".": {
+            "types": "./dist/index.d.ts",
+            "import": "./dist/index.js",
+            "default": "./dist/index.js"
+        }
+    },
+    "files": [
+        "dist",
+        "skills",
+        "README.md"
+    ],
+    "bin": {
+        "openbot": "./dist/cli.js"
+    },
+"dependencies": {
+    "better-sqlite3": "^11.6.0",
+    "croner": "^9.1.0",
+    "@mariozechner/pi-ai": "^0.51.2",
+        "@mariozechner/pi-coding-agent": "^0.51.2",
+        "@nestjs/common": "^10.3.0",
+        "@nestjs/core": "^10.3.0",
+        "@nestjs/platform-express": "^10.3.0",
+        "@nestjs/platform-socket.io": "^10.3.0",
+        "@nestjs/websockets": "^10.3.0",
+        "@sinclair/typebox": "^0.34.41",
+        "@xenova/transformers": "^2.17.2",
+        "agent-browser": "^0.8.5",
+        "commander": "^14.0.2",
+        "reflect-metadata": "^0.2.1",
+        "rxjs": "^7.8.1",
+        "vectra": "^0.12.3",
+        "ws": "^8.19.0"
+    },
+"devDependencies": {
+    "@types/better-sqlite3": "^7.6.11",
+    "@types/express": "^4.17.21",
+        "@types/node": "^22.0.0",
+        "@types/jest": "^29.5.14",
+        "@types/ws": "^8.18.1",
+        "@nestjs/testing": "^10.3.0",
+        "jest": "^29.7.0",
+        "ts-jest": "^29.2.5",
+        "typescript": "^5.7.0"
+    },
+    "scripts": {
+        "clean": "rm -rf dist",
+        "build": "npm run clean && tsc",
+        "start": "node dist/cli.js",
+        "run": "node dist/cli.js",
+        "dev": "npm run build && node dist/cli.js",
+        "desktop:dev": "cd desktop && npm run dev",
+        "desktop:build": "cd desktop && npm run build",
+        "desktop:install": "cd desktop && npm install",
+        "test": "jest --config test/jest.config.cjs",
+        "test:memory": "node test/run-memory.mjs",
+        "test:e2e": "OPENBOT_DB_PATH=:memory: jest --config test/jest.config.cjs --testPathPattern=e2e"
+    },
+    "engines": {
+        "node": ">=20.0.0"
+    },
+    "keywords": [
+        "skill",
+        "agent",
+        "cli",
+        "llm",
+        "openbot"
+    ],
+    "license": "MIT"
+}