genkitx-playwright 0.3.0

package/src/tools.ts

@@ -0,0 +1,759 @@
+import { MessageData, Part, z } from 'genkit';
+import { tool } from 'genkit/beta';
+import type { Locator, Page } from 'playwright';
+import { BrowserSession } from './browser.js';
+
+
+/** Names of all tools this middleware can inject. */
+export const ALL_TOOL_NAMES = [
+  // navigation
+  'browser_navigate',
+  'browser_navigate_back',
+  'browser_navigate_forward',
+  // observation
+  'browser_snapshot',
+  'browser_screenshot',
+  'browser_get_console_logs',
+  'browser_get_network_requests',
+  // interaction
+  'browser_click',
+  'browser_type',
+  'browser_fill',
+  'browser_press_key',
+  'browser_hover',
+  'browser_select_option',
+  'browser_scroll',
+  'browser_drag',
+  'browser_file_upload',
+  // synchronization
+  'browser_wait_for',
+  // tabs
+  'browser_tab_list',
+  'browser_tab_new',
+  'browser_tab_select',
+  'browser_tab_close',
+  // dialogs
+  'browser_handle_dialog',
+  // escape hatch
+  'browser_evaluate',
+] as const;
+
+export type PlaywrightToolName = (typeof ALL_TOOL_NAMES)[number];
+
+/** Tools that mutate page state. Disabled in read-only mode. */
+const WRITE_TOOLS: PlaywrightToolName[] = [
+  'browser_click',
+  'browser_type',
+  'browser_fill',
+  'browser_press_key',
+  'browser_hover',
+  'browser_select_option',
+  'browser_scroll',
+  'browser_drag',
+  'browser_file_upload',
+  'browser_handle_dialog',
+];
+
+export interface ToolFactoryOptions {
+  /** The active browser session. */
+  session: BrowserSession;
+  /**
+   * Queue of messages to inject into the conversation history. Tools that
+   * produce media (e.g. screenshots) push `user` messages here; the middleware
+   * drains the queue into the request on the next turn so the content enters
+   * the model's context as proper multimodal parts instead of a JSON blob.
+   */
+  messageQueue?: MessageData[];
+  /** Prefix to prepend to every tool name. */
+  toolNamePrefix?: string;
+
+  /** If true, only observe-only (non-mutating) tools are created. */
+  readOnly?: boolean;
+  /** If true, the `browser_evaluate` JS escape-hatch tool is created. */
+  allowEvaluate?: boolean;
+  /** Optional allow-list of (unprefixed) tool names to create. */
+  tools?: PlaywrightToolName[];
+  /** Restrict navigation to these hostnames (suffix match). */
+  allowedDomains?: string[];
+  /** Block navigation to these hostnames (suffix match). */
+  blockedDomains?: string[];
+}
+
+function hostnameAllowed(
+  url: string,
+  allowed?: string[],
+  blocked?: string[]
+): boolean {
+  let host: string;
+  try {
+    host = new URL(url).hostname;
+  } catch {
+    // Relative or invalid URLs are allowed; Playwright will surface errors.
+    return true;
+  }
+  const matches = (domain: string) =>
+    host === domain || host.endsWith(`.${domain}`);
+  if (blocked?.some(matches)) return false;
+  if (allowed && allowed.length > 0) return allowed.some(matches);
+  return true;
+}
+
+/**
+ * Resolves a snapshot ref (e.g. "e12") to a Playwright locator using the
+ * `aria-ref` selector engine populated by `browser_snapshot`.
+ */
+function locatorForRef(page: Page, ref: string): Locator {
+  return page.locator(`aria-ref=${ref}`);
+}
+
+/**
+ * Produces an accessibility snapshot annotated with stable `ref` ids for each
+ * interactive element. Refs can be passed to interaction tools.
+ */
+async function snapshotForAI(page: Page): Promise<string> {
+  // The "ai" mode of ariaSnapshot yields an ARIA tree where each element is
+  // tagged with a stable [ref=eN] id. Those refs resolve via the built-in
+  // `aria-ref=` selector engine (see locatorForRef). This is the same
+  // mechanism the official Playwright MCP server relies on.
+  try {
+    return await (
+      page.ariaSnapshot as (opts: { mode: 'ai' }) => Promise<string>
+    )({ mode: 'ai' });
+  } catch {
+    // Fallback: plain ARIA snapshot (no refs). Interaction tools won't be able
+    // to resolve refs, but the model can still read the page structure.
+    return page.locator('body').ariaSnapshot();
+  }
+}
+
+/**
+ * Creates the set of Playwright browser tools for a single generation.
+ */
+
+export function createPlaywrightTools(opts: ToolFactoryOptions) {
+  const { session } = opts;
+  const prefix = opts.toolNamePrefix ?? '';
+  const name = (n: PlaywrightToolName) => `${prefix}${n}`;
+
+  // Determine which tools are enabled.
+  const enabled = new Set<PlaywrightToolName>(opts.tools ?? ALL_TOOL_NAMES);
+  if (opts.readOnly) {
+    for (const t of WRITE_TOOLS) enabled.delete(t);
+  }
+  if (!opts.allowEvaluate) {
+    enabled.delete('browser_evaluate');
+  }
+
+  const refSchema = z
+    .string()
+    .describe('Element ref from the latest browser_snapshot, e.g. "e12".');
+
+  /**
+   * Enqueues parts as a `user` message for the middleware to inject into the
+   * conversation history. Appends to the trailing queued `user` message when
+   * possible so consecutive media parts coalesce into one message.
+   */
+  const enqueueParts = (parts: Part[], toolName: string) => {
+    const queue = opts.messageQueue;
+    if (!queue) return;
+    const last = queue[queue.length - 1];
+    if (last && last.role === 'user') {
+      last.content.push(...parts);
+    } else {
+      queue.push({
+        role: 'user',
+        content: parts,
+        metadata: { playwrightMiddlewareTool: toolName },
+      });
+    }
+  };
+
+  const tools: any[] = [];
+
+  const add = (toolName: PlaywrightToolName, factory: () => any) => {
+    if (enabled.has(toolName)) tools.push(factory());
+  };
+
+  // ---- Navigation ----------------------------------------------------------
+
+  add('browser_navigate', () =>
+    tool(
+      {
+        name: name('browser_navigate'),
+        description: 'Navigate the current page to a URL.',
+        inputSchema: z.object({
+          url: z.string().describe('Absolute URL to navigate to.'),
+        }),
+        outputSchema: z.string(),
+      },
+      async ({ url }) => {
+        if (!hostnameAllowed(url, opts.allowedDomains, opts.blockedDomains)) {
+          throw new Error(`Navigation to "${url}" is not allowed by policy.`);
+        }
+        const page = await session.getPage();
+        await page.goto(url, { waitUntil: 'domcontentloaded' });
+        return `Navigated to ${page.url()} (title: "${await page.title()}").`;
+      }
+    )
+  );
+
+  add('browser_navigate_back', () =>
+    tool(
+      {
+        name: name('browser_navigate_back'),
+        description: 'Go back to the previous page in history.',
+        inputSchema: z.object({}),
+        outputSchema: z.string(),
+      },
+      async () => {
+        const page = await session.getPage();
+        await page.goBack({ waitUntil: 'domcontentloaded' });
+        return `Went back to ${page.url()}.`;
+      }
+    )
+  );
+
+  add('browser_navigate_forward', () =>
+    tool(
+      {
+        name: name('browser_navigate_forward'),
+        description: 'Go forward to the next page in history.',
+        inputSchema: z.object({}),
+        outputSchema: z.string(),
+      },
+      async () => {
+        const page = await session.getPage();
+        await page.goForward({ waitUntil: 'domcontentloaded' });
+        return `Went forward to ${page.url()}.`;
+      }
+    )
+  );
+
+  // ---- Observation ---------------------------------------------------------
+
+  add('browser_snapshot', () =>
+    tool(
+      {
+        name: name('browser_snapshot'),
+        description:
+          'Capture an accessibility snapshot of the current page. Each ' +
+          'interactive element is tagged with a [ref=eN] id you can pass to ' +
+          'interaction tools (click, type, etc.). Prefer this over screenshots ' +
+          'for deciding what to interact with.',
+        inputSchema: z.object({}),
+        outputSchema: z.object({
+          url: z.string(),
+          title: z.string(),
+          snapshot: z.string(),
+        }),
+      },
+      async () => {
+        const page = await session.getPage();
+        return {
+          url: page.url(),
+          title: await page.title(),
+          snapshot: await snapshotForAI(page),
+        };
+      }
+    )
+  );
+
+  add('browser_screenshot', () =>
+    tool(
+      {
+        name: name('browser_screenshot'),
+        description:
+          'Take a screenshot of the current page. The image is added to the ' +
+          'conversation so you can view it on your next turn.',
+        inputSchema: z.object({
+          fullPage: z
+            .boolean()
+            .optional()
+            .describe('Capture the full scrollable page. Defaults to false.'),
+        }),
+        outputSchema: z.string(),
+      },
+      async ({ fullPage }) => {
+        const page = await session.getPage();
+        const buffer = await page.screenshot({
+          fullPage: fullPage ?? false,
+          type: 'png',
+        });
+        const dataUri = `data:image/png;base64,${buffer.toString('base64')}`;
+
+        // Enqueue the image as a real multimodal `user` message part so the
+        // model actually "sees" it, instead of receiving a JSON blob in the
+        // tool response. The middleware drains this queue into the request.
+        enqueueParts(
+          [
+            { text: `\n\nbrowser_screenshot result image/png (${page.url()})` },
+            { media: { url: dataUri, contentType: 'image/png' } },
+          ],
+          name('browser_screenshot')
+        );
+
+        return 'Screenshot captured. The image is shown below.';
+      }
+    )
+  );
+
+
+  add('browser_get_console_logs', () =>
+    tool(
+      {
+        name: name('browser_get_console_logs'),
+        description: 'Return console messages collected from the page.',
+        inputSchema: z.object({}),
+        outputSchema: z.array(
+          z.object({ type: z.string(), text: z.string() })
+        ),
+      },
+      async () => session.consoleLogs
+    )
+  );
+
+  add('browser_get_network_requests', () =>
+    tool(
+      {
+        name: name('browser_get_network_requests'),
+        description:
+          'List network requests made by the current page (URL, method, ' +
+          'and response status when available).',
+        inputSchema: z.object({}),
+        outputSchema: z.array(
+          z.object({
+            url: z.string(),
+            method: z.string(),
+            status: z.number().optional(),
+          })
+        ),
+      },
+      async () => {
+        const page = await session.getPage();
+        // Best-effort: use the page's performance entries for a snapshot of
+        // resources. Live request interception is intentionally avoided to keep
+        // the session lightweight.
+        const entries = await page.evaluate(() => {
+          const list = performance.getEntriesByType(
+            'resource'
+          ) as PerformanceResourceTiming[];
+          return list.map((e) => ({
+            url: e.name,
+            method: 'GET',
+            status: undefined as number | undefined,
+          }));
+        });
+        return entries;
+      }
+    )
+  );
+
+  // ---- Interaction ---------------------------------------------------------
+
+  add('browser_click', () =>
+    tool(
+      {
+        name: name('browser_click'),
+        description: 'Click an element identified by its snapshot ref.',
+        inputSchema: z.object({
+          ref: refSchema,
+          doubleClick: z
+            .boolean()
+            .optional()
+            .describe('Perform a double click instead of a single click.'),
+        }),
+        outputSchema: z.string(),
+      },
+      async ({ ref, doubleClick }) => {
+        const page = await session.getPage();
+        const locator = locatorForRef(page, ref);
+        if (doubleClick) {
+          await locator.dblclick();
+        } else {
+          await locator.click();
+        }
+        return `Clicked element ${ref}.`;
+      }
+    )
+  );
+
+  add('browser_type', () =>
+    tool(
+      {
+        name: name('browser_type'),
+        description:
+          'Type text into an element by ref, character by character ' +
+          '(does not clear existing content).',
+        inputSchema: z.object({
+          ref: refSchema,
+          text: z.string().describe('Text to type.'),
+          submit: z
+            .boolean()
+            .optional()
+            .describe('Press Enter after typing.'),
+        }),
+        outputSchema: z.string(),
+      },
+      async ({ ref, text, submit }) => {
+        const page = await session.getPage();
+        const locator = locatorForRef(page, ref);
+        await locator.pressSequentially(text);
+        if (submit) await locator.press('Enter');
+        return `Typed into element ${ref}.`;
+      }
+    )
+  );
+
+  add('browser_fill', () =>
+    tool(
+      {
+        name: name('browser_fill'),
+        description:
+          'Set the value of an input/textarea by ref, clearing it first.',
+        inputSchema: z.object({
+          ref: refSchema,
+          value: z.string().describe('Value to set.'),
+        }),
+        outputSchema: z.string(),
+      },
+      async ({ ref, value }) => {
+        const page = await session.getPage();
+        await locatorForRef(page, ref).fill(value);
+        return `Filled element ${ref}.`;
+      }
+    )
+  );
+
+  add('browser_press_key', () =>
+    tool(
+      {
+        name: name('browser_press_key'),
+        description:
+          'Press a keyboard key (e.g. "Enter", "Tab", "ArrowDown", "a"). ' +
+          'Optionally targets an element by ref.',
+        inputSchema: z.object({
+          key: z.string().describe('Key to press.'),
+          ref: refSchema.optional(),
+        }),
+        outputSchema: z.string(),
+      },
+      async ({ key, ref }) => {
+        const page = await session.getPage();
+        if (ref) {
+          await locatorForRef(page, ref).press(key);
+        } else {
+          await page.keyboard.press(key);
+        }
+        return `Pressed key "${key}".`;
+      }
+    )
+  );
+
+  add('browser_hover', () =>
+    tool(
+      {
+        name: name('browser_hover'),
+        description: 'Hover the mouse over an element by ref.',
+        inputSchema: z.object({ ref: refSchema }),
+        outputSchema: z.string(),
+      },
+      async ({ ref }) => {
+        const page = await session.getPage();
+        await locatorForRef(page, ref).hover();
+        return `Hovered element ${ref}.`;
+      }
+    )
+  );
+
+  add('browser_select_option', () =>
+    tool(
+      {
+        name: name('browser_select_option'),
+        description: 'Select option(s) in a <select> element by ref.',
+        inputSchema: z.object({
+          ref: refSchema,
+          values: z
+            .array(z.string())
+            .describe('Option values or labels to select.'),
+        }),
+        outputSchema: z.string(),
+      },
+      async ({ ref, values }) => {
+        const page = await session.getPage();
+        await locatorForRef(page, ref).selectOption(values);
+        return `Selected ${values.length} option(s) in ${ref}.`;
+      }
+    )
+  );
+
+  add('browser_scroll', () =>
+    tool(
+      {
+        name: name('browser_scroll'),
+        description:
+          'Scroll the page up or down, or scroll a specific element into view.',
+        inputSchema: z.object({
+          direction: z
+            .enum(['up', 'down'])
+            .optional()
+            .describe('Direction to scroll the window. Defaults to "down".'),
+          ref: refSchema
+            .optional()
+            .describe('If provided, scroll this element into view instead.'),
+        }),
+        outputSchema: z.string(),
+      },
+      async ({ direction, ref }) => {
+        const page = await session.getPage();
+        if (ref) {
+          await locatorForRef(page, ref).scrollIntoViewIfNeeded();
+          return `Scrolled element ${ref} into view.`;
+        }
+        const dir = direction ?? 'down';
+        await page.evaluate((d) => {
+          window.scrollBy(0, d === 'down' ? window.innerHeight : -window.innerHeight);
+        }, dir);
+        return `Scrolled ${dir}.`;
+      }
+    )
+  );
+
+  add('browser_drag', () =>
+    tool(
+      {
+        name: name('browser_drag'),
+        description: 'Drag one element onto another, by their refs.',
+        inputSchema: z.object({
+          startRef: refSchema.describe('Ref of the element to drag.'),
+          endRef: refSchema.describe('Ref of the drop target.'),
+        }),
+        outputSchema: z.string(),
+      },
+      async ({ startRef, endRef }) => {
+        const page = await session.getPage();
+        await locatorForRef(page, startRef).dragTo(locatorForRef(page, endRef));
+        return `Dragged ${startRef} onto ${endRef}.`;
+      }
+    )
+  );
+
+  add('browser_file_upload', () =>
+    tool(
+      {
+        name: name('browser_file_upload'),
+        description:
+          'Set files on a file input element by ref (provide local file paths).',
+        inputSchema: z.object({
+          ref: refSchema,
+          paths: z.array(z.string()).describe('Local file paths to upload.'),
+        }),
+        outputSchema: z.string(),
+      },
+      async ({ ref, paths }) => {
+        const page = await session.getPage();
+        await locatorForRef(page, ref).setInputFiles(paths);
+        return `Uploaded ${paths.length} file(s) to ${ref}.`;
+      }
+    )
+  );
+
+  // ---- Synchronization -----------------------------------------------------
+
+  add('browser_wait_for', () =>
+    tool(
+      {
+        name: name('browser_wait_for'),
+        description:
+          'Wait for text to appear/disappear on the page, or for a fixed time.',
+        inputSchema: z.object({
+          text: z.string().optional().describe('Text to wait to appear.'),
+          textGone: z
+            .string()
+            .optional()
+            .describe('Text to wait to disappear.'),
+          time: z
+            .number()
+            .optional()
+            .describe('Seconds to wait. Capped at 30.'),
+        }),
+        outputSchema: z.string(),
+      },
+      async ({ text, textGone, time }) => {
+        const page = await session.getPage();
+        if (text) {
+          await page.getByText(text).first().waitFor({ state: 'visible' });
+          return `Text "${text}" appeared.`;
+        }
+        if (textGone) {
+          await page.getByText(textGone).first().waitFor({ state: 'hidden' });
+          return `Text "${textGone}" disappeared.`;
+        }
+        const seconds = Math.min(time ?? 1, 30);
+        await page.waitForTimeout(seconds * 1000);
+        return `Waited ${seconds}s.`;
+      }
+    )
+  );
+
+  // ---- Tabs ----------------------------------------------------------------
+
+  add('browser_tab_list', () =>
+    tool(
+      {
+        name: name('browser_tab_list'),
+        description: 'List open tabs with their index, URL, and title.',
+        inputSchema: z.object({}),
+        outputSchema: z.array(
+          z.object({
+            index: z.number(),
+            url: z.string(),
+            title: z.string(),
+          })
+        ),
+      },
+      async () => {
+        const pages = await session.getPages();
+        return Promise.all(
+          pages.map(async (p, index) => ({
+            index,
+            url: p.url(),
+            title: await p.title(),
+          }))
+        );
+      }
+    )
+  );
+
+  add('browser_tab_new', () =>
+    tool(
+      {
+        name: name('browser_tab_new'),
+        description: 'Open a new tab, optionally navigating it to a URL.',
+        inputSchema: z.object({
+          url: z.string().optional().describe('URL to open in the new tab.'),
+        }),
+        outputSchema: z.string(),
+      },
+      async ({ url }) => {
+        const page = await session.newPage();
+        if (url) {
+          if (!hostnameAllowed(url, opts.allowedDomains, opts.blockedDomains)) {
+            throw new Error(`Navigation to "${url}" is not allowed by policy.`);
+          }
+          await page.goto(url, { waitUntil: 'domcontentloaded' });
+        }
+        const pages = await session.getPages();
+        return `Opened tab ${pages.length - 1}${url ? ` at ${url}` : ''}.`;
+      }
+    )
+  );
+
+  add('browser_tab_select', () =>
+    tool(
+      {
+        name: name('browser_tab_select'),
+        description: 'Bring a tab to the front by its index.',
+        inputSchema: z.object({
+          index: z.number().describe('Tab index from browser_tab_list.'),
+        }),
+        outputSchema: z.string(),
+      },
+      async ({ index }) => {
+        const pages = await session.getPages();
+        const page = pages[index];
+        if (!page) throw new Error(`No tab at index ${index}.`);
+        await page.bringToFront();
+        return `Selected tab ${index}.`;
+      }
+    )
+  );
+
+  add('browser_tab_close', () =>
+    tool(
+      {
+        name: name('browser_tab_close'),
+        description:
+          'Close a tab by index (defaults to the current/last tab).',
+        inputSchema: z.object({
+          index: z.number().optional().describe('Tab index to close.'),
+        }),
+        outputSchema: z.string(),
+      },
+      async ({ index }) => {
+        const pages = await session.getPages();
+        const page = index === undefined ? pages[pages.length - 1] : pages[index];
+        if (!page) throw new Error(`No tab at index ${index}.`);
+        await page.close();
+        return `Closed tab ${index ?? pages.length - 1}.`;
+      }
+    )
+  );
+
+  // ---- Dialogs -------------------------------------------------------------
+
+  add('browser_handle_dialog', () =>
+    tool(
+      {
+        name: name('browser_handle_dialog'),
+        description:
+          'Accept or dismiss a pending JavaScript dialog (alert/confirm/prompt).',
+        inputSchema: z.object({
+          accept: z.boolean().describe('Accept (true) or dismiss (false).'),
+          promptText: z
+            .string()
+            .optional()
+            .describe('Text to enter for prompt dialogs.'),
+        }),
+        outputSchema: z.string(),
+      },
+      async ({ accept, promptText }) => {
+        const dialog = session.activeDialog;
+        if (!dialog) return 'No pending dialog to handle.';
+        if (accept) {
+          await dialog.accept(promptText);
+        } else {
+          await dialog.dismiss();
+        }
+        session.activeDialog = undefined;
+        session.pendingDialog = undefined;
+        return `Dialog ${accept ? 'accepted' : 'dismissed'}.`;
+      }
+    )
+  );
+
+  // ---- Escape hatch --------------------------------------------------------
+
+  add('browser_evaluate', () =>
+    tool(
+      {
+        name: name('browser_evaluate'),
+        description:
+          'Evaluate arbitrary JavaScript in the page context and return the ' +
+          'JSON-serializable result. Use only when other tools are insufficient.',
+        inputSchema: z.object({
+          script: z
+            .string()
+            .describe(
+              'A JS expression or function body, e.g. "document.title".'
+            ),
+        }),
+        outputSchema: z.any(),
+      },
+      async ({ script }) => {
+        const page = await session.getPage();
+        // Wrap so both expressions and statements work.
+        const result = await page.evaluate(
+          (code) => {
+            // eslint-disable-next-line no-new-func
+            const fn = new Function(`return (async () => { ${code} })()`);
+            return fn();
+          },
+          /(\breturn\b|;)/.test(script) ? script : `return (${script});`
+        );
+        return result ?? null;
+      }
+    )
+  );
+
+  return tools;
+}