@handstage/agent 1.0.0

package/README.md

@@ -0,0 +1,3 @@
+# @handstage/agent
+
+AI agent tools and schemas for driving Handstage. Provides structured inputs/outputs for browser interactions (navigation, clicking, typing, page info) tailored for use with LLMs and AI agents.

package/dist/definitions.js

@@ -0,0 +1,209 @@
+import { tool } from "ai";
+import { 
+// ClickInputSchema,
+ClickOnIdInputSchema, 
+// BringToFrontInputSchema,
+// BringToFrontOutputSchema,
+ClosePageInputSchema, ClosePageOutputSchema, 
+// ClickOnInputSchema,
+ElementActionOutputSchema, FillOnIdInputSchema, 
+// FillOnInputSchema,
+GoBackInputSchema, GoForwardInputSchema, GotoInputSchema, GotoOutputSchema, HistoryNavOutputSchema, 
+// HoverInputSchema,
+HoverOnIdInputSchema, 
+// HoverOnInputSchema,
+NewPageInputSchema, NewPageOutputSchema, 
+// PageInfoInputSchema,
+// PageInfoOutputSchema,
+PagesInputSchema, PagesOutputSchema, 
+// PointerOutputSchema,
+ReloadInputSchema, ReloadOutputSchema, 
+// ScrollInputSchema,
+SnapshotDomInputSchema, SnapshotDomOutputSchema, 
+// TypeInputSchema,
+TypeOnIdInputSchema, } from "./schemas";
+function pagesToXml(pages) {
+    return `<pages>\n${pages
+        .map((page) => `<page>\n<pageId>${page.pageId}</pageId>\n<url>${page.url}</url>\n<title>${page.title}</title>\n</page>`)
+        .join("\n")}\n</pages>`;
+}
+/**
+ * Same object as {@link handstageAgentTools}; kept for callers that only need a `ToolSet`.
+ */
+export function createHandstageAgentToolDefinitions(handler) {
+    const tools = {
+        pages: tool({
+            description: "List opend pages in current browser context. Each entry has pageId, url, and title. Pages are returned in the order they were opened; no tab is implicitly 'active' — pass the pageId you want to act on with every tool call.",
+            inputSchema: PagesInputSchema,
+            outputSchema: PagesOutputSchema,
+            toModelOutput: ({ output }) => {
+                return {
+                    type: "text",
+                    value: pagesToXml(output.pages),
+                };
+            },
+            execute: async (input) => {
+                return await handler.pages(input);
+            },
+        }),
+        newPage: tool({
+            description: "Open a new browser page. Returns the new page's pageId. Optional starting URL (defaults to about:blank).",
+            inputSchema: NewPageInputSchema,
+            outputSchema: NewPageOutputSchema,
+            execute: async (input) => {
+                return await handler.newPage(input);
+            },
+        }),
+        closePage: tool({
+            description: "Close a browser page by pageId.",
+            inputSchema: ClosePageInputSchema,
+            outputSchema: ClosePageOutputSchema,
+            execute: async (input) => {
+                return await handler.closePage(input);
+            },
+        }),
+        // bringToFront: tool({
+        // 	description:
+        // 		"Foreground a tab by pageId (Target.activateTarget). Necessary before input events can land on that tab in headful Chrome.",
+        // 	inputSchema: BringToFrontInputSchema,
+        // 	outputSchema: BringToFrontOutputSchema,
+        // }),
+        goto: tool({
+            description: "Navigate a page to a URL.",
+            inputSchema: GotoInputSchema,
+            outputSchema: GotoOutputSchema,
+            execute: async (input) => {
+                return await handler.goto(input);
+            },
+        }),
+        reload: tool({
+            description: "Reload the current document in a page.",
+            inputSchema: ReloadInputSchema,
+            outputSchema: ReloadOutputSchema,
+            execute: async (input) => {
+                return await handler.reload(input);
+            },
+        }),
+        goBack: tool({
+            description: "Go back in history for a page, if possible.",
+            inputSchema: GoBackInputSchema,
+            outputSchema: HistoryNavOutputSchema,
+            execute: async (input) => {
+                return await handler.goBack(input);
+            },
+        }),
+        goForward: tool({
+            description: "Go forward in history for a page, if possible.",
+            inputSchema: GoForwardInputSchema,
+            outputSchema: HistoryNavOutputSchema,
+            execute: async (input) => {
+                return await handler.goForward(input);
+            },
+        }),
+        snapshot_dom: tool({
+            description: "Accessibility tree for a page (pageId). Multiline outline with encoded node ids in brackets (e.g. [1-42]); use those ids with click_on_id, fill_on_id, type_on_id, or hover_on_id.",
+            inputSchema: SnapshotDomInputSchema,
+            outputSchema: SnapshotDomOutputSchema,
+            toModelOutput: ({ output }) => {
+                if (output.ok) {
+                    return {
+                        type: "text",
+                        value: output.tree,
+                    };
+                }
+                return {
+                    type: "text",
+                    value: output.error,
+                };
+            },
+            execute: async (input) => {
+                return await handler.snapshot_dom(input);
+            },
+        }),
+        // pageInfo: tool({
+        // 	description: "Current URL and document title for a page.",
+        // 	inputSchema: PageInfoInputSchema,
+        // 	outputSchema: PageInfoOutputSchema,
+        // }),
+        // click: tool({
+        // 	description:
+        // 		"Click at viewport coordinates (CSS pixels). Does not scroll; ensure the target is visible.",
+        // 	inputSchema: ClickInputSchema,
+        // 	outputSchema: PointerOutputSchema,
+        // }),
+        // hover: tool({
+        // 	description: "Move the pointer to viewport coordinates (CSS pixels).",
+        // 	inputSchema: HoverInputSchema,
+        // 	outputSchema: PointerOutputSchema,
+        // }),
+        // scroll: tool({
+        // 	description:
+        // 		"Dispatch a mouse wheel at viewport coordinates (deltaX/deltaY in pixels).",
+        // 	inputSchema: ScrollInputSchema,
+        // 	outputSchema: PointerOutputSchema,
+        // }),
+        // type: tool({
+        // 	description:
+        // 		"Type text using key events at the current focus. Focus an input first (e.g. click_on_id) or tab to it.",
+        // 	inputSchema: TypeInputSchema,
+        // 	outputSchema: TypeOutputSchema,
+        // }),
+        // click_on: tool({
+        // 	description:
+        // 		"Click the first element matching a CSS or XPath selector in the page's main frame.",
+        // 	inputSchema: ClickOnInputSchema,
+        // 	outputSchema: ElementActionOutputSchema,
+        // }),
+        // fill_on: tool({
+        // 	description:
+        // 		"Clear and fill an input element matched by a CSS or XPath selector (main frame).",
+        // 	inputSchema: FillOnInputSchema,
+        // 	outputSchema: ElementActionOutputSchema,
+        // }),
+        // type_on: tool({
+        // 	description:
+        // 		"Type into an element matched by a CSS or XPath selector (focuses the element first).",
+        // 	inputSchema: TypeOnInputSchema,
+        // 	outputSchema: ElementActionOutputSchema,
+        // }),
+        // hover_on: tool({
+        // 	description:
+        // 		"Hover the first element matching a CSS or XPath selector in the page's main frame.",
+        // 	inputSchema: HoverOnInputSchema,
+        // 	outputSchema: ElementActionOutputSchema,
+        // }),
+        click_on_id: tool({
+            description: "Click the element for an encoded accessibility tree node id from snapshot_dom.",
+            inputSchema: ClickOnIdInputSchema,
+            outputSchema: ElementActionOutputSchema,
+            execute: async (input) => {
+                return await handler.click_on_id(input);
+            },
+        }),
+        fill_on_id: tool({
+            description: "Clear and fill an input for an encoded accessibility tree node id from snapshot_dom.",
+            inputSchema: FillOnIdInputSchema,
+            outputSchema: ElementActionOutputSchema,
+            execute: async (input) => {
+                return await handler.fill_on_id(input);
+            },
+        }),
+        type_on_id: tool({
+            description: "Type into an element for an encoded accessibility tree node id from snapshot_dom.",
+            inputSchema: TypeOnIdInputSchema,
+            outputSchema: ElementActionOutputSchema,
+            execute: async (input) => {
+                return await handler.type_on_id(input);
+            },
+        }),
+        hover_on_id: tool({
+            description: "Hover the element for an encoded accessibility tree node id from snapshot_dom.",
+            inputSchema: HoverOnIdInputSchema,
+            outputSchema: ElementActionOutputSchema,
+            execute: async (input) => {
+                return await handler.hover_on_id(input);
+            },
+        }),
+    };
+    return tools;
+}

package/dist/handlers.js

@@ -0,0 +1,104 @@
+import { errResult, tryAgentResult, withPage } from "./result";
+export class HandstageContextAgentToolHandlers {
+    ctx;
+    constructor(ctx) {
+        this.ctx = ctx;
+    }
+    async pages(_input) {
+        const pages = await Promise.all(this.ctx.pages().map(async (page) => ({
+            pageId: page.pageId,
+            url: page.url(),
+            title: await page.title(),
+        })));
+        return { pages };
+    }
+    async newPage(input) {
+        const page = await this.ctx.newPage(input.url ?? "about:blank");
+        return { pageId: page.pageId };
+    }
+    async closePage(input) {
+        const page = this.ctx.resolvePageByTargetId(input.pageId);
+        if (!page)
+            return errResult(`Unknown pageId: ${input.pageId}`);
+        return tryAgentResult(async () => {
+            await page.close();
+            return {};
+        });
+    }
+    async goto(input) {
+        return withPage(this.ctx, input.pageId, async (page) => {
+            await page.goto(input.url, {
+                waitUntil: input.waitUntil,
+                timeoutMs: input.timeoutMs,
+            });
+            return { url: page.url() };
+        });
+    }
+    async reload(input) {
+        return withPage(this.ctx, input.pageId, async (page) => {
+            await page.reload({
+                waitUntil: input.waitUntil,
+                timeoutMs: input.timeoutMs,
+                ignoreCache: input.ignoreCache,
+            });
+            return { url: page.url() };
+        });
+    }
+    async goBack(input) {
+        return withPage(this.ctx, input.pageId, async (page) => {
+            const response = await page.goBack({
+                waitUntil: input.waitUntil,
+                timeoutMs: input.timeoutMs,
+            });
+            return { navigated: response !== null, url: page.url() };
+        });
+    }
+    async goForward(input) {
+        return withPage(this.ctx, input.pageId, async (page) => {
+            const response = await page.goForward({
+                waitUntil: input.waitUntil,
+                timeoutMs: input.timeoutMs,
+            });
+            return { navigated: response !== null, url: page.url() };
+        });
+    }
+    async snapshot_dom(input) {
+        return withPage(this.ctx, input.pageId, async (page) => {
+            const snapshot = await page.snapshot({
+                includeIframes: input.includeIframes,
+            });
+            return {
+                tree: snapshot.formattedTree,
+                xpathMap: snapshot.xpathMap,
+                urlMap: snapshot.urlMap,
+            };
+        });
+    }
+    async click_on_id(input) {
+        return this.actOnEncodedId(input.pageId, input.id, (locator) => locator.click());
+    }
+    async fill_on_id(input) {
+        return this.actOnEncodedId(input.pageId, input.id, (locator) => locator.fill(input.value));
+    }
+    async type_on_id(input) {
+        return this.actOnEncodedId(input.pageId, input.id, (locator) => locator.type(input.text, { delay: input.delay }));
+    }
+    async hover_on_id(input) {
+        return this.actOnEncodedId(input.pageId, input.id, (locator) => locator.hover());
+    }
+    async actOnEncodedId(pageId, encodedId, action) {
+        return withPage(this.ctx, pageId, async (page) => {
+            const { xpathMap } = await page.snapshot();
+            const xpath = xpathMap[encodedId];
+            if (!xpath) {
+                throw new Error(`Unknown encoded id: ${encodedId}`);
+            }
+            await action(page.deepLocator(xpath));
+            await page.waitForLoadState("networkidle", 5000);
+            return {};
+        });
+    }
+}
+export function createHandstageContextAgentToolHandlers(ctx) {
+    return new HandstageContextAgentToolHandlers(ctx);
+}

package/dist/index.js

@@ -0,0 +1,4 @@
+export { createHandstageAgentToolDefinitions, } from "./definitions";
+export { createHandstageContextAgentToolHandlers, HandstageContextAgentToolHandlers, } from "./handlers";
+export * from "./handlerTypes";
+export * from "./schemas";

package/dist/result.js

@@ -0,0 +1,21 @@
+export function formatError(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+export function errResult(error) {
+    return { ok: false, error };
+}
+export async function tryAgentResult(fn) {
+    try {
+        const data = await fn();
+        return { ok: true, ...data };
+    }
+    catch (error) {
+        return errResult(formatError(error));
+    }
+}
+export async function withPage(ctx, pageId, fn) {
+    const page = ctx.resolvePageByTargetId(pageId);
+    if (!page)
+        return errResult(`Unknown pageId: ${pageId}`);
+    return tryAgentResult(() => fn(page));
+}

package/dist/schemas.js

@@ -0,0 +1,168 @@
+import { z } from "zod";
+export const LoadStateSchema = z
+    .enum(["load", "domcontentloaded", "networkidle"])
+    .optional()
+    .describe("Wait until this lifecycle event (navigation / reload / history)");
+export const PageIdSchema = z
+    .string()
+    .describe("Target id of the page tab (from pages or newPage)");
+export const PagesInputSchema = z.object({});
+export const NewPageInputSchema = z.object({
+    url: z.string().optional().describe('Initial URL (default "about:blank")'),
+});
+export const BringToFrontInputSchema = z.object({ pageId: PageIdSchema });
+export const ClosePageInputSchema = z.object({ pageId: PageIdSchema });
+export const GotoInputSchema = z.object({
+    pageId: PageIdSchema,
+    url: z.string().min(1),
+    waitUntil: LoadStateSchema,
+    timeoutMs: z.number().positive().optional(),
+});
+export const ReloadInputSchema = z.object({
+    pageId: PageIdSchema,
+    waitUntil: LoadStateSchema,
+    timeoutMs: z.number().positive().optional(),
+    ignoreCache: z.boolean().optional(),
+});
+export const GoBackInputSchema = z.object({
+    pageId: PageIdSchema,
+    waitUntil: LoadStateSchema,
+    timeoutMs: z.number().positive().optional(),
+});
+export const GoForwardInputSchema = z.object({
+    pageId: PageIdSchema,
+    waitUntil: LoadStateSchema,
+    timeoutMs: z.number().positive().optional(),
+});
+export const SnapshotDomInputSchema = z.object({
+    pageId: PageIdSchema,
+    includeIframes: z.boolean().optional(),
+});
+export const PageInfoInputSchema = z.object({ pageId: PageIdSchema });
+export const ClickInputSchema = z.object({
+    pageId: PageIdSchema,
+    x: z.number(),
+    y: z.number(),
+    button: z.enum(["left", "right", "middle"]).optional(),
+    clickCount: z.number().int().positive().optional(),
+});
+export const HoverInputSchema = z.object({
+    pageId: PageIdSchema,
+    x: z.number(),
+    y: z.number(),
+});
+export const ScrollInputSchema = z.object({
+    pageId: PageIdSchema,
+    x: z.number(),
+    y: z.number(),
+    deltaX: z.number(),
+    deltaY: z.number(),
+});
+export const TypeInputSchema = z.object({
+    pageId: PageIdSchema,
+    text: z.string(),
+    delay: z.number().nonnegative().optional(),
+    withMistakes: z.boolean().optional(),
+});
+export const ClickOnInputSchema = z.object({
+    pageId: PageIdSchema,
+    select: z
+        .string()
+        .min(1)
+        .describe("CSS selector or XPath (e.g. //button[@id='x'])"),
+});
+export const FillOnInputSchema = z.object({
+    pageId: PageIdSchema,
+    select: z.string().min(1).describe("CSS selector or XPath"),
+    value: z.string(),
+});
+export const TypeOnInputSchema = z.object({
+    pageId: PageIdSchema,
+    select: z.string().min(1).describe("CSS selector or XPath"),
+    text: z.string(),
+    delay: z.number().nonnegative().optional(),
+});
+export const HoverOnInputSchema = z.object({
+    pageId: PageIdSchema,
+    select: z.string().min(1).describe("CSS selector or XPath"),
+});
+/** Encoded node id from snapshot_dom (e.g. `1-42` — frameOrdinal-backendNodeId). */
+export const A11yEncodedIdSchema = z
+    .string()
+    .min(1)
+    .regex(/^\d+-\d+$/, "Expected frameOrdinal-backendNodeId from snapshot_dom")
+    .describe("Encoded node id from snapshot_dom (bracketed id in the a11y tree)");
+export const ClickOnIdInputSchema = z.object({
+    pageId: PageIdSchema,
+    id: A11yEncodedIdSchema,
+});
+export const FillOnIdInputSchema = z.object({
+    pageId: PageIdSchema,
+    id: A11yEncodedIdSchema,
+    value: z.string(),
+});
+export const TypeOnIdInputSchema = z.object({
+    pageId: PageIdSchema,
+    id: A11yEncodedIdSchema,
+    text: z.string(),
+    delay: z.number().nonnegative().optional(),
+});
+export const HoverOnIdInputSchema = z.object({
+    pageId: PageIdSchema,
+    id: A11yEncodedIdSchema,
+});
+/** Shared `{ ok: true } | { ok: false; error }` tool result shape */
+export const HandstageAgentOkOrErrOutputSchema = z.discriminatedUnion("ok", [
+    z.object({ ok: z.literal(true) }),
+    z.object({ ok: z.literal(false), error: z.string() }),
+]);
+export const HandstageAgentPageEntrySchema = z.object({
+    pageId: z.string(),
+    url: z.string(),
+    title: z.string(),
+});
+export const PagesOutputSchema = z.object({
+    pages: z.array(HandstageAgentPageEntrySchema),
+});
+export const NewPageOutputSchema = z.object({ pageId: z.string() });
+export const BringToFrontOutputSchema = HandstageAgentOkOrErrOutputSchema;
+export const ClosePageOutputSchema = HandstageAgentOkOrErrOutputSchema;
+export const GotoOutputSchema = z.discriminatedUnion("ok", [
+    z.object({ ok: z.literal(true), url: z.string() }),
+    z.object({ ok: z.literal(false), error: z.string() }),
+]);
+export const ReloadOutputSchema = GotoOutputSchema;
+export const HistoryNavOutputSchema = z.discriminatedUnion("ok", [
+    z.object({
+        ok: z.literal(true),
+        navigated: z.boolean(),
+        url: z.string(),
+    }),
+    z.object({ ok: z.literal(false), error: z.string() }),
+]);
+export const SnapshotDomOutputSchema = z.discriminatedUnion("ok", [
+    z.object({
+        ok: z.literal(true),
+        tree: z.string(),
+        xpathMap: z.record(z.string(), z.string()),
+        urlMap: z.record(z.string(), z.string()),
+    }),
+    z.object({ ok: z.literal(false), error: z.string() }),
+]);
+export const PageInfoOutputSchema = z.discriminatedUnion("ok", [
+    z.object({
+        ok: z.literal(true),
+        url: z.string(),
+        title: z.string(),
+    }),
+    z.object({ ok: z.literal(false), error: z.string() }),
+]);
+export const PointerOutputSchema = z.discriminatedUnion("ok", [
+    z.object({
+        ok: z.literal(true),
+        xpathAtPoint: z.string().optional(),
+    }),
+    z.object({ ok: z.literal(false), error: z.string() }),
+]);
+export const TypeOutputSchema = HandstageAgentOkOrErrOutputSchema;
+export const ElementActionOutputSchema = HandstageAgentOkOrErrOutputSchema;

package/package.json

@@ -0,0 +1,32 @@
+{
+	"name": "@handstage/agent",
+	"version": "1.0.0",
+	"type": "module",
+	"exports": {
+		".": {
+			"types": "./src/index.ts",
+			"import": "./dist/index.js"
+		}
+	},
+	"files": [
+		"dist",
+		"src"
+	],
+	"scripts": {
+		"build": "rm -rf dist && tsc"
+	},
+	"devDependencies": {
+		"@ai-sdk/devtools": "^0.0.18",
+		"@ai-sdk/openai-compatible": "^2.0.48",
+		"@types/bun": "1.3.14",
+		"@types/node": "^25.9.1"
+	},
+	"peerDependencies": {
+		"typescript": "^6.0.3"
+	},
+	"dependencies": {
+		"@handstage/core": "1.0.0",
+		"ai": "^6.0.193",
+		"zod": "^4.4.3"
+	}
+}