@modelrelay/sdk 1.28.0 → 1.29.0

package/dist/node.js

@@ -1,8 +1,9 @@
 import {
   PathEscapeError,
   ToolArgumentError,
-  ToolRegistry
-} from "./chunk-G5H7EY4F.js";
+  ToolRegistry,
+  ToolTypes
+} from "./chunk-CV3DTA6P.js";
 
 // src/tools_local_fs.ts
 import { promises as fs } from "fs";
@@ -601,11 +602,589 @@ function createLocalFSToolPack(options) {
 function createLocalFSTools(options) {
   return createLocalFSToolPack(options).toRegistry();
 }
+
+// src/tools_browser.ts
+var BrowserToolNames = {
+  /** Navigate to a URL and return accessibility tree */
+  NAVIGATE: "browser.navigate",
+  /** Click an element by accessible name/role */
+  CLICK: "browser.click",
+  /** Type text into an input field */
+  TYPE: "browser.type",
+  /** Get current accessibility tree */
+  SNAPSHOT: "browser.snapshot",
+  /** Scroll the page */
+  SCROLL: "browser.scroll",
+  /** Capture a screenshot */
+  SCREENSHOT: "browser.screenshot",
+  /** Extract data using CSS selectors */
+  EXTRACT: "browser.extract"
+};
+var BrowserDefaults = {
+  /** Navigation timeout in milliseconds */
+  NAVIGATION_TIMEOUT_MS: 3e4,
+  /** Action timeout in milliseconds */
+  ACTION_TIMEOUT_MS: 5e3,
+  /** Maximum nodes to include in accessibility tree output */
+  MAX_SNAPSHOT_NODES: 500,
+  /** Maximum screenshot size in bytes */
+  MAX_SCREENSHOT_BYTES: 5e6
+};
+var BrowserToolPack = class {
+  constructor(options = {}) {
+    this.browser = null;
+    this.context = null;
+    this.page = null;
+    this.cdpSession = null;
+    this.ownsBrowser = false;
+    this.ownsContext = false;
+    this.cfg = {
+      allowedDomains: options.allowedDomains ?? [],
+      blockedDomains: options.blockedDomains ?? [],
+      navigationTimeoutMs: options.navigationTimeoutMs ?? BrowserDefaults.NAVIGATION_TIMEOUT_MS,
+      actionTimeoutMs: options.actionTimeoutMs ?? BrowserDefaults.ACTION_TIMEOUT_MS,
+      maxSnapshotNodes: options.maxSnapshotNodes ?? BrowserDefaults.MAX_SNAPSHOT_NODES,
+      headless: options.headless ?? true
+    };
+    if (options.browser) {
+      this.browser = options.browser;
+      this.ownsBrowser = false;
+    }
+    if (options.context) {
+      this.context = options.context;
+      this.ownsContext = false;
+    }
+  }
+  /**
+   * Initialize the browser. Must be called before using any tools.
+   */
+  async initialize() {
+    if (this.page) {
+      return;
+    }
+    const { chromium } = await import("playwright");
+    if (!this.browser) {
+      this.browser = await chromium.launch({
+        headless: this.cfg.headless
+      });
+      this.ownsBrowser = true;
+    }
+    if (!this.context) {
+      this.context = await this.browser.newContext();
+      this.ownsContext = true;
+    }
+    this.page = await this.context.newPage();
+  }
+  /**
+   * Close the browser and clean up resources.
+   */
+  async close() {
+    if (this.cdpSession) {
+      await this.cdpSession.detach().catch(() => {
+      });
+      this.cdpSession = null;
+    }
+    if (this.page) {
+      await this.page.close().catch(() => {
+      });
+      this.page = null;
+    }
+    if (this.ownsContext && this.context) {
+      await this.context.close().catch(() => {
+      });
+      this.context = null;
+    }
+    if (this.ownsBrowser && this.browser) {
+      await this.browser.close().catch(() => {
+      });
+      this.browser = null;
+    }
+  }
+  /**
+   * Get tool definitions for use with LLM APIs.
+   */
+  getToolDefinitions() {
+    return [
+      {
+        type: ToolTypes.Function,
+        function: {
+          name: BrowserToolNames.NAVIGATE,
+          description: "Navigate to a URL and return the page's accessibility tree. The tree shows interactive elements (buttons, links, inputs) with their accessible names.",
+          parameters: {
+            type: "object",
+            properties: {
+              url: {
+                type: "string",
+                description: "The URL to navigate to (must be http/https)"
+              },
+              waitUntil: {
+                type: "string",
+                enum: ["load", "domcontentloaded", "networkidle"],
+                description: "When to consider navigation complete. Default: domcontentloaded"
+              }
+            },
+            required: ["url"]
+          }
+        }
+      },
+      {
+        type: ToolTypes.Function,
+        function: {
+          name: BrowserToolNames.CLICK,
+          description: "Click an element by its accessible name. Returns updated accessibility tree.",
+          parameters: {
+            type: "object",
+            properties: {
+              name: {
+                type: "string",
+                description: "The accessible name of the element (from button text, aria-label, etc.)"
+              },
+              role: {
+                type: "string",
+                enum: [
+                  "button",
+                  "link",
+                  "menuitem",
+                  "checkbox",
+                  "radio",
+                  "tab"
+                ],
+                description: "ARIA role to match. If omitted, searches buttons, links, and menuitems."
+              }
+            },
+            required: ["name"]
+          }
+        }
+      },
+      {
+        type: ToolTypes.Function,
+        function: {
+          name: BrowserToolNames.TYPE,
+          description: "Type text into an input field identified by accessible name.",
+          parameters: {
+            type: "object",
+            properties: {
+              name: {
+                type: "string",
+                description: "The accessible name of the input (from label, aria-label, placeholder)"
+              },
+              text: {
+                type: "string",
+                description: "The text to type"
+              },
+              role: {
+                type: "string",
+                enum: ["textbox", "searchbox", "combobox"],
+                description: "ARIA role. Default: textbox"
+              }
+            },
+            required: ["name", "text"]
+          }
+        }
+      },
+      {
+        type: ToolTypes.Function,
+        function: {
+          name: BrowserToolNames.SNAPSHOT,
+          description: "Get the current page's accessibility tree without navigating.",
+          parameters: {
+            type: "object",
+            properties: {}
+          }
+        }
+      },
+      {
+        type: ToolTypes.Function,
+        function: {
+          name: BrowserToolNames.SCROLL,
+          description: "Scroll the page in a given direction.",
+          parameters: {
+            type: "object",
+            properties: {
+              direction: {
+                type: "string",
+                enum: ["up", "down"],
+                description: "Scroll direction"
+              },
+              amount: {
+                type: "string",
+                enum: ["page", "half", "toTop", "toBottom"],
+                description: "How much to scroll. Default: page"
+              }
+            },
+            required: ["direction"]
+          }
+        }
+      },
+      {
+        type: ToolTypes.Function,
+        function: {
+          name: BrowserToolNames.SCREENSHOT,
+          description: "Capture a PNG screenshot of the current page. Use sparingly - prefer accessibility tree for decisions.",
+          parameters: {
+            type: "object",
+            properties: {
+              fullPage: {
+                type: "boolean",
+                description: "Capture full scrollable page. Default: false (viewport only)"
+              }
+            }
+          }
+        }
+      },
+      {
+        type: ToolTypes.Function,
+        function: {
+          name: BrowserToolNames.EXTRACT,
+          description: "Extract structured data from the page using CSS selectors.",
+          parameters: {
+            type: "object",
+            properties: {
+              selector: {
+                type: "string",
+                description: "CSS selector for elements to extract"
+              },
+              attribute: {
+                type: "string",
+                description: "Attribute to extract (textContent, href, src, etc.). Default: textContent"
+              },
+              multiple: {
+                type: "boolean",
+                description: "Return all matches as JSON array. Default: false (first match only)"
+              }
+            },
+            required: ["selector"]
+          }
+        }
+      }
+    ];
+  }
+  /**
+   * Register tool handlers into an existing registry.
+   */
+  registerInto(registry) {
+    registry.register(BrowserToolNames.NAVIGATE, this.navigate.bind(this));
+    registry.register(BrowserToolNames.CLICK, this.click.bind(this));
+    registry.register(BrowserToolNames.TYPE, this.type.bind(this));
+    registry.register(BrowserToolNames.SNAPSHOT, this.snapshot.bind(this));
+    registry.register(BrowserToolNames.SCROLL, this.scroll.bind(this));
+    registry.register(BrowserToolNames.SCREENSHOT, this.screenshot.bind(this));
+    registry.register(BrowserToolNames.EXTRACT, this.extract.bind(this));
+    return registry;
+  }
+  /**
+   * Create a new registry with just this pack's tools.
+   */
+  toRegistry() {
+    return this.registerInto(new ToolRegistry());
+  }
+  // ========================================================================
+  // Private: Helpers
+  // ========================================================================
+  ensureInitialized() {
+    if (!this.page) {
+      throw new Error(
+        "BrowserToolPack not initialized. Call initialize() first."
+      );
+    }
+  }
+  parseArgs(call, required) {
+    const func = call.function;
+    if (!func) {
+      throw new ToolArgumentError({
+        message: "tool call missing function",
+        toolCallId: call.id,
+        toolName: "",
+        rawArguments: ""
+      });
+    }
+    const rawArgs = func.arguments || "{}";
+    let parsed;
+    try {
+      parsed = JSON.parse(rawArgs);
+    } catch (err) {
+      throw new ToolArgumentError({
+        message: `invalid JSON arguments: ${err.message}`,
+        toolCallId: call.id,
+        toolName: func.name,
+        rawArguments: rawArgs
+      });
+    }
+    if (typeof parsed !== "object" || parsed === null) {
+      throw new ToolArgumentError({
+        message: "arguments must be an object",
+        toolCallId: call.id,
+        toolName: func.name,
+        rawArguments: rawArgs
+      });
+    }
+    const args = parsed;
+    for (const key of required) {
+      const value = args[key];
+      if (value === void 0 || value === null || value === "") {
+        throw new ToolArgumentError({
+          message: `${key} is required`,
+          toolCallId: call.id,
+          toolName: func.name,
+          rawArguments: rawArgs
+        });
+      }
+    }
+    return args;
+  }
+  validateUrl(url, call) {
+    let parsed;
+    try {
+      parsed = new URL(url);
+    } catch {
+      throw new ToolArgumentError({
+        message: `Invalid URL: ${url}`,
+        toolCallId: call.id,
+        toolName: call.function?.name ?? "",
+        rawArguments: call.function?.arguments ?? ""
+      });
+    }
+    if (!["http:", "https:"].includes(parsed.protocol)) {
+      throw new ToolArgumentError({
+        message: `Invalid protocol: ${parsed.protocol}. Only http/https allowed.`,
+        toolCallId: call.id,
+        toolName: call.function?.name ?? "",
+        rawArguments: call.function?.arguments ?? ""
+      });
+    }
+    const domain = parsed.hostname;
+    if (this.cfg.blockedDomains.some((d) => domain.endsWith(d))) {
+      throw new ToolArgumentError({
+        message: `Domain blocked: ${domain}`,
+        toolCallId: call.id,
+        toolName: call.function?.name ?? "",
+        rawArguments: call.function?.arguments ?? ""
+      });
+    }
+    if (this.cfg.allowedDomains.length > 0) {
+      if (!this.cfg.allowedDomains.some((d) => domain.endsWith(d))) {
+        throw new ToolArgumentError({
+          message: `Domain not in allowlist: ${domain}`,
+          toolCallId: call.id,
+          toolName: call.function?.name ?? "",
+          rawArguments: call.function?.arguments ?? ""
+        });
+      }
+    }
+  }
+  /**
+   * Validates the current page URL against allowlist/blocklist.
+   * Called after navigation and before any action to catch redirects
+   * and in-session navigation to blocked domains.
+   */
+  ensureCurrentUrlAllowed() {
+    if (!this.page) return;
+    const currentUrl = this.page.url();
+    if (currentUrl === "about:blank") return;
+    let parsed;
+    try {
+      parsed = new URL(currentUrl);
+    } catch {
+      throw new Error(`Current page has invalid URL: ${currentUrl}`);
+    }
+    if (!["http:", "https:"].includes(parsed.protocol)) {
+      throw new Error(
+        `Current page protocol not allowed: ${parsed.protocol}. Only http/https allowed.`
+      );
+    }
+    const domain = parsed.hostname;
+    if (this.cfg.blockedDomains.some((d) => domain.endsWith(d))) {
+      throw new Error(`Current page domain is blocked: ${domain}`);
+    }
+    if (this.cfg.allowedDomains.length > 0) {
+      if (!this.cfg.allowedDomains.some((d) => domain.endsWith(d))) {
+        throw new Error(`Current page domain not in allowlist: ${domain}`);
+      }
+    }
+  }
+  async getAccessibilityTree() {
+    this.ensureInitialized();
+    if (!this.cdpSession) {
+      this.cdpSession = await this.page.context().newCDPSession(this.page);
+      await this.cdpSession.send("Accessibility.enable");
+    }
+    const response = await this.cdpSession.send(
+      "Accessibility.getFullAXTree"
+    );
+    return response.nodes;
+  }
+  formatAXTree(nodes) {
+    const lines = [];
+    let count = 0;
+    for (const node of nodes) {
+      if (count >= this.cfg.maxSnapshotNodes) {
+        lines.push(`[truncated at ${this.cfg.maxSnapshotNodes} nodes]`);
+        break;
+      }
+      if (node.ignored) {
+        continue;
+      }
+      const role = node.role?.value || "unknown";
+      const name = node.name?.value || "";
+      if (!name && ["generic", "none", "text"].includes(role)) {
+        continue;
+      }
+      const states = [];
+      if (node.properties) {
+        for (const prop of node.properties) {
+          if (prop.value?.value === true) {
+            const stateName = prop.name;
+            if (["focused", "checked", "disabled", "expanded", "selected"].includes(
+              stateName
+            )) {
+              states.push(stateName);
+            }
+          }
+        }
+      }
+      const stateStr = states.length ? " " + states.join(" ") : "";
+      const nameStr = name ? ` "${name}"` : "";
+      lines.push(`[${role}${nameStr}${stateStr}]`);
+      count++;
+    }
+    return lines.join("\n");
+  }
+  // ========================================================================
+  // Private: Tool Handlers
+  // ========================================================================
+  async navigate(_args, call) {
+    const args = this.parseArgs(call, ["url"]);
+    this.validateUrl(args.url, call);
+    this.ensureInitialized();
+    const waitUntil = args.waitUntil ?? "domcontentloaded";
+    await this.page.goto(args.url, {
+      timeout: this.cfg.navigationTimeoutMs,
+      waitUntil
+    });
+    this.ensureCurrentUrlAllowed();
+    const tree = await this.getAccessibilityTree();
+    return this.formatAXTree(tree);
+  }
+  async click(_args, call) {
+    const args = this.parseArgs(call, ["name"]);
+    this.ensureInitialized();
+    this.ensureCurrentUrlAllowed();
+    let locator;
+    if (args.role) {
+      locator = this.page.getByRole(args.role, {
+        name: args.name
+      });
+    } else {
+      locator = this.page.getByRole("button", { name: args.name }).or(this.page.getByRole("link", { name: args.name })).or(this.page.getByRole("menuitem", { name: args.name }));
+    }
+    await locator.click({ timeout: this.cfg.actionTimeoutMs });
+    const tree = await this.getAccessibilityTree();
+    return this.formatAXTree(tree);
+  }
+  async type(_args, call) {
+    const args = this.parseArgs(call, ["name", "text"]);
+    this.ensureInitialized();
+    this.ensureCurrentUrlAllowed();
+    const role = args.role ?? "textbox";
+    const locator = this.page.getByRole(role, { name: args.name });
+    await locator.fill(args.text, { timeout: this.cfg.actionTimeoutMs });
+    return `Typed "${args.text}" into ${role} "${args.name}"`;
+  }
+  async snapshot(_args, _call) {
+    this.ensureInitialized();
+    this.ensureCurrentUrlAllowed();
+    const tree = await this.getAccessibilityTree();
+    return this.formatAXTree(tree);
+  }
+  async scroll(_args, call) {
+    const args = this.parseArgs(call, ["direction"]);
+    this.ensureInitialized();
+    this.ensureCurrentUrlAllowed();
+    const amount = args.amount ?? "page";
+    if (amount === "toTop") {
+      await this.page.evaluate(() => window.scrollTo(0, 0));
+    } else if (amount === "toBottom") {
+      await this.page.evaluate(
+        () => window.scrollTo(0, document.body.scrollHeight)
+      );
+    } else {
+      const viewport = this.page.viewportSize();
+      const height = viewport?.height ?? 800;
+      const scrollAmount = amount === "half" ? height / 2 : height;
+      const delta = args.direction === "down" ? scrollAmount : -scrollAmount;
+      await this.page.evaluate((d) => window.scrollBy(0, d), delta);
+    }
+    const tree = await this.getAccessibilityTree();
+    return this.formatAXTree(tree);
+  }
+  async screenshot(_args, call) {
+    const args = this.parseArgs(call, []);
+    this.ensureInitialized();
+    this.ensureCurrentUrlAllowed();
+    const buffer = await this.page.screenshot({
+      fullPage: args.fullPage ?? false,
+      type: "png"
+    });
+    if (buffer.length > BrowserDefaults.MAX_SCREENSHOT_BYTES) {
+      throw new Error(
+        `Screenshot size (${buffer.length} bytes) exceeds maximum allowed (${BrowserDefaults.MAX_SCREENSHOT_BYTES} bytes). Try capturing viewport only.`
+      );
+    }
+    const base64 = buffer.toString("base64");
+    return `data:image/png;base64,${base64}`;
+  }
+  async extract(_args, call) {
+    const args = this.parseArgs(call, ["selector"]);
+    this.ensureInitialized();
+    this.ensureCurrentUrlAllowed();
+    const attribute = args.attribute ?? "textContent";
+    const multiple = args.multiple ?? false;
+    if (multiple) {
+      const elements = this.page.locator(args.selector);
+      const count = await elements.count();
+      const results = [];
+      for (let i = 0; i < count; i++) {
+        const el = elements.nth(i);
+        let value;
+        if (attribute === "textContent") {
+          value = await el.textContent();
+        } else {
+          value = await el.getAttribute(attribute);
+        }
+        if (value !== null) {
+          results.push(value.trim());
+        }
+      }
+      return JSON.stringify(results);
+    } else {
+      const el = this.page.locator(args.selector).first();
+      let value;
+      if (attribute === "textContent") {
+        value = await el.textContent();
+      } else {
+        value = await el.getAttribute(attribute);
+      }
+      return value?.trim() ?? "";
+    }
+  }
+};
+function createBrowserToolPack(options = {}) {
+  return new BrowserToolPack(options);
+}
+function createBrowserTools(options = {}) {
+  const pack = new BrowserToolPack(options);
+  return { pack, registry: pack.toRegistry() };
+}
 export {
+  BrowserDefaults,
+  BrowserToolNames,
+  BrowserToolPack,
   DEFAULT_IGNORE_DIRS,
   FSDefaults,
   ToolNames as FSToolNames,
   LocalFSToolPack,
+  createBrowserToolPack,
+  createBrowserTools,
   createLocalFSToolPack,
   createLocalFSTools
 };

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@modelrelay/sdk",
-	"version": "1.28.0",
+	"version": "1.29.0",
 	"description": "TypeScript SDK for the ModelRelay API",
 	"type": "module",
 	"main": "dist/index.cjs",