@handstage/agent 1.0.0

package/src/definitions.ts

@@ -0,0 +1,263 @@
+import { type ToolSet, tool } from "ai"
+import type { HandstageAgentToolHandlers } from "./handlerTypes"
+import {
+	// ClickInputSchema,
+	ClickOnIdInputSchema,
+	// BringToFrontInputSchema,
+	// BringToFrontOutputSchema,
+	ClosePageInputSchema,
+	ClosePageOutputSchema,
+	// ClickOnInputSchema,
+	ElementActionOutputSchema,
+	FillOnIdInputSchema,
+	// FillOnInputSchema,
+	GoBackInputSchema,
+	GoForwardInputSchema,
+	GotoInputSchema,
+	GotoOutputSchema,
+	HistoryNavOutputSchema,
+	// HoverInputSchema,
+	HoverOnIdInputSchema,
+	// HoverOnInputSchema,
+	NewPageInputSchema,
+	NewPageOutputSchema,
+	// PageInfoInputSchema,
+	// PageInfoOutputSchema,
+	PagesInputSchema,
+	PagesOutputSchema,
+	// PointerOutputSchema,
+	ReloadInputSchema,
+	ReloadOutputSchema,
+	// ScrollInputSchema,
+	SnapshotDomInputSchema,
+	SnapshotDomOutputSchema,
+	// TypeInputSchema,
+	TypeOnIdInputSchema,
+} from "./schemas"
+
+export type HandstageAgentToolSet = ReturnType<
+	typeof createHandstageAgentToolDefinitions
+>
+
+function pagesToXml(
+	pages: Array<{ pageId: string; url: string; title: string }>,
+): string {
+	return `<pages>\n${pages
+		.map(
+			(page) =>
+				`<page>\n<pageId>${page.pageId}</pageId>\n<url>${page.url}</url>\n<title>${page.title}</title>\n</page>`,
+		)
+		.join("\n")}\n</pages>`
+}
+
+/**
+ * Same object as {@link handstageAgentTools}; kept for callers that only need a `ToolSet`.
+ */
+export function createHandstageAgentToolDefinitions(
+	handler: HandstageAgentToolHandlers,
+) {
+	const tools = {
+		pages: tool({
+			description:
+				"List opend pages in current browser context. Each entry has pageId, url, and title. Pages are returned in the order they were opened; no tab is implicitly 'active' — pass the pageId you want to act on with every tool call.",
+			inputSchema: PagesInputSchema,
+			outputSchema: PagesOutputSchema,
+			toModelOutput: ({ output }) => {
+				return {
+					type: "text",
+					value: pagesToXml(output.pages),
+				}
+			},
+			execute: async (input) => {
+				return await handler.pages(input)
+			},
+		}),
+
+		newPage: tool({
+			description:
+				"Open a new browser page. Returns the new page's pageId. Optional starting URL (defaults to about:blank).",
+			inputSchema: NewPageInputSchema,
+			outputSchema: NewPageOutputSchema,
+			execute: async (input) => {
+				return await handler.newPage(input)
+			},
+		}),
+
+		closePage: tool({
+			description: "Close a browser page by pageId.",
+			inputSchema: ClosePageInputSchema,
+			outputSchema: ClosePageOutputSchema,
+			execute: async (input) => {
+				return await handler.closePage(input)
+			},
+		}),
+
+		// bringToFront: tool({
+		// 	description:
+		// 		"Foreground a tab by pageId (Target.activateTarget). Necessary before input events can land on that tab in headful Chrome.",
+		// 	inputSchema: BringToFrontInputSchema,
+		// 	outputSchema: BringToFrontOutputSchema,
+		// }),
+
+		goto: tool({
+			description: "Navigate a page to a URL.",
+			inputSchema: GotoInputSchema,
+			outputSchema: GotoOutputSchema,
+			execute: async (input) => {
+				return await handler.goto(input)
+			},
+		}),
+
+		reload: tool({
+			description: "Reload the current document in a page.",
+			inputSchema: ReloadInputSchema,
+			outputSchema: ReloadOutputSchema,
+			execute: async (input) => {
+				return await handler.reload(input)
+			},
+		}),
+
+		goBack: tool({
+			description: "Go back in history for a page, if possible.",
+			inputSchema: GoBackInputSchema,
+			outputSchema: HistoryNavOutputSchema,
+			execute: async (input) => {
+				return await handler.goBack(input)
+			},
+		}),
+
+		goForward: tool({
+			description: "Go forward in history for a page, if possible.",
+			inputSchema: GoForwardInputSchema,
+			outputSchema: HistoryNavOutputSchema,
+			execute: async (input) => {
+				return await handler.goForward(input)
+			},
+		}),
+
+		snapshot_dom: tool({
+			description:
+				"Accessibility tree for a page (pageId). Multiline outline with encoded node ids in brackets (e.g. [1-42]); use those ids with click_on_id, fill_on_id, type_on_id, or hover_on_id.",
+			inputSchema: SnapshotDomInputSchema,
+			outputSchema: SnapshotDomOutputSchema,
+			toModelOutput: ({ output }) => {
+				if (output.ok) {
+					return {
+						type: "text",
+						value: output.tree,
+					}
+				}
+				return {
+					type: "text",
+					value: output.error,
+				}
+			},
+			execute: async (input) => {
+				return await handler.snapshot_dom(input)
+			},
+		}),
+
+		// pageInfo: tool({
+		// 	description: "Current URL and document title for a page.",
+		// 	inputSchema: PageInfoInputSchema,
+		// 	outputSchema: PageInfoOutputSchema,
+		// }),
+
+		// click: tool({
+		// 	description:
+		// 		"Click at viewport coordinates (CSS pixels). Does not scroll; ensure the target is visible.",
+		// 	inputSchema: ClickInputSchema,
+		// 	outputSchema: PointerOutputSchema,
+		// }),
+
+		// hover: tool({
+		// 	description: "Move the pointer to viewport coordinates (CSS pixels).",
+		// 	inputSchema: HoverInputSchema,
+		// 	outputSchema: PointerOutputSchema,
+		// }),
+
+		// scroll: tool({
+		// 	description:
+		// 		"Dispatch a mouse wheel at viewport coordinates (deltaX/deltaY in pixels).",
+		// 	inputSchema: ScrollInputSchema,
+		// 	outputSchema: PointerOutputSchema,
+		// }),
+
+		// type: tool({
+		// 	description:
+		// 		"Type text using key events at the current focus. Focus an input first (e.g. click_on_id) or tab to it.",
+		// 	inputSchema: TypeInputSchema,
+		// 	outputSchema: TypeOutputSchema,
+		// }),
+
+		// click_on: tool({
+		// 	description:
+		// 		"Click the first element matching a CSS or XPath selector in the page's main frame.",
+		// 	inputSchema: ClickOnInputSchema,
+		// 	outputSchema: ElementActionOutputSchema,
+		// }),
+
+		// fill_on: tool({
+		// 	description:
+		// 		"Clear and fill an input element matched by a CSS or XPath selector (main frame).",
+		// 	inputSchema: FillOnInputSchema,
+		// 	outputSchema: ElementActionOutputSchema,
+		// }),
+
+		// type_on: tool({
+		// 	description:
+		// 		"Type into an element matched by a CSS or XPath selector (focuses the element first).",
+		// 	inputSchema: TypeOnInputSchema,
+		// 	outputSchema: ElementActionOutputSchema,
+		// }),
+
+		// hover_on: tool({
+		// 	description:
+		// 		"Hover the first element matching a CSS or XPath selector in the page's main frame.",
+		// 	inputSchema: HoverOnInputSchema,
+		// 	outputSchema: ElementActionOutputSchema,
+		// }),
+
+		click_on_id: tool({
+			description:
+				"Click the element for an encoded accessibility tree node id from snapshot_dom.",
+			inputSchema: ClickOnIdInputSchema,
+			outputSchema: ElementActionOutputSchema,
+			execute: async (input) => {
+				return await handler.click_on_id(input)
+			},
+		}),
+
+		fill_on_id: tool({
+			description:
+				"Clear and fill an input for an encoded accessibility tree node id from snapshot_dom.",
+			inputSchema: FillOnIdInputSchema,
+			outputSchema: ElementActionOutputSchema,
+			execute: async (input) => {
+				return await handler.fill_on_id(input)
+			},
+		}),
+
+		type_on_id: tool({
+			description:
+				"Type into an element for an encoded accessibility tree node id from snapshot_dom.",
+			inputSchema: TypeOnIdInputSchema,
+			outputSchema: ElementActionOutputSchema,
+			execute: async (input) => {
+				return await handler.type_on_id(input)
+			},
+		}),
+
+		hover_on_id: tool({
+			description:
+				"Hover the element for an encoded accessibility tree node id from snapshot_dom.",
+			inputSchema: HoverOnIdInputSchema,
+			outputSchema: ElementActionOutputSchema,
+			execute: async (input) => {
+				return await handler.hover_on_id(input)
+			},
+		}),
+	} as const satisfies ToolSet
+
+	return tools
+}

package/src/handlerTypes.ts

@@ -0,0 +1,83 @@
+import type { Page } from "@handstage/core"
+import type {
+	// ClickInput,
+	ClickOnIdInput,
+	ClickOnIdOutput,
+	// BringToFrontInput,
+	// BringToFrontOutput,
+	ClosePageInput,
+	ClosePageOutput,
+	// ClickOutput,
+	FillOnIdInput,
+	FillOnIdOutput,
+	GoBackInput,
+	GoBackOutput,
+	GoForwardInput,
+	GoForwardOutput,
+	GotoInput,
+	GotoOutput,
+	// HoverInput,
+	HoverOnIdInput,
+	HoverOnIdOutput,
+	// HoverOutput,
+	NewPageInput,
+	NewPageOutput,
+	// PageInfoInput,
+	// PageInfoOutput,
+	PagesInput,
+	PagesOutput,
+	ReloadInput,
+	ReloadOutput,
+	// ScrollInput,
+	// ScrollOutput,
+	SnapshotDomInput,
+	SnapshotDomOutput,
+	// TypeInput,
+	TypeOnIdInput,
+	TypeOnIdOutput,
+	// TypeOutput,
+} from "./types"
+
+/**
+ * Browser context exposed by Handstage (`Handstage.defaultBrowserContext()`). Implementations
+ * of {@link HandstageAgentToolHandlers} typically hold this.
+ *
+ * There is no implicit "active page" — callers track Page references they
+ * received from `newPage()` or `pages()` explicitly, and pass `pageId` on
+ * every tool call.
+ */
+export interface HandstageAgentContext {
+	pages(): Page[]
+	newPage(url?: string): Promise<Page>
+}
+
+/**
+ * Implementations perform Handstage actions for each tool. Inputs and outputs are
+ * inferred from Zod schemas in {@link ./schemas}.
+ */
+export interface HandstageAgentToolHandlers {
+	pages(input: PagesInput): Promise<PagesOutput>
+	newPage(input: NewPageInput): Promise<NewPageOutput>
+	closePage(input: ClosePageInput): Promise<ClosePageOutput>
+	// bringToFront(
+	// 	input: BringToFrontInput,
+	// ): Promise<BringToFrontOutput>
+	goto(input: GotoInput): Promise<GotoOutput>
+	reload(input: ReloadInput): Promise<ReloadOutput>
+	goBack(input: GoBackInput): Promise<GoBackOutput>
+	goForward(input: GoForwardInput): Promise<GoForwardOutput>
+	snapshot_dom(input: SnapshotDomInput): Promise<SnapshotDomOutput>
+	// pageInfo(input: PageInfoInput): Promise<PageInfoOutput>
+	// click(input: ClickInput): Promise<ClickOutput>
+	// hover(input: HoverInput): Promise<HoverOutput>
+	// scroll(input: ScrollInput): Promise<ScrollOutput>
+	// type(input: TypeInput): Promise<TypeOutput>
+	// click_on(input: ClickOnInput): Promise<ClickOnOutput>
+	// fill_on(input: FillOnInput): Promise<FillOnOutput>
+	// type_on(input: TypeOnInput): Promise<TypeOnOutput>
+	// hover_on(input: HoverOnInput): Promise<HoverOnOutput>
+	click_on_id(input: ClickOnIdInput): Promise<ClickOnIdOutput>
+	fill_on_id(input: FillOnIdInput): Promise<FillOnIdOutput>
+	type_on_id(input: TypeOnIdInput): Promise<TypeOnIdOutput>
+	hover_on_id(input: HoverOnIdInput): Promise<HoverOnIdOutput>
+}

package/src/handlers.ts

@@ -0,0 +1,163 @@
+import type { Context, Page } from "@handstage/core"
+import type { HandstageAgentToolHandlers } from "./handlerTypes"
+import { errResult, tryAgentResult, withPage } from "./result"
+import type {
+	ClickOnIdInput,
+	ClickOnIdOutput,
+	ClosePageInput,
+	ClosePageOutput,
+	FillOnIdInput,
+	FillOnIdOutput,
+	GoBackInput,
+	GoBackOutput,
+	GoForwardInput,
+	GoForwardOutput,
+	GotoInput,
+	GotoOutput,
+	HoverOnIdInput,
+	HoverOnIdOutput,
+	NewPageInput,
+	NewPageOutput,
+	PagesInput,
+	PagesOutput,
+	ReloadInput,
+	ReloadOutput,
+	SnapshotDomInput,
+	SnapshotDomOutput,
+	TypeOnIdInput,
+	TypeOnIdOutput,
+} from "./types"
+
+type DeepLocator = ReturnType<Page["deepLocator"]>
+
+export class HandstageContextAgentToolHandlers
+	implements HandstageAgentToolHandlers
+{
+	constructor(private readonly ctx: Context) {}
+
+	async pages(_input: PagesInput): Promise<PagesOutput> {
+		const pages = await Promise.all(
+			this.ctx.pages().map(async (page) => ({
+				pageId: page.pageId,
+				url: page.url(),
+				title: await page.title(),
+			})),
+		)
+		return { pages }
+	}
+
+	async newPage(input: NewPageInput): Promise<NewPageOutput> {
+		const page = await this.ctx.newPage(input.url ?? "about:blank")
+		return { pageId: page.pageId }
+	}
+
+	async closePage(input: ClosePageInput): Promise<ClosePageOutput> {
+		const page = this.ctx.resolvePageByTargetId(input.pageId)
+		if (!page) return errResult(`Unknown pageId: ${input.pageId}`)
+		return tryAgentResult(async () => {
+			await page.close()
+			return {}
+		})
+	}
+
+	async goto(input: GotoInput): Promise<GotoOutput> {
+		return withPage(this.ctx, input.pageId, async (page) => {
+			await page.goto(input.url, {
+				waitUntil: input.waitUntil,
+				timeoutMs: input.timeoutMs,
+			})
+			return { url: page.url() }
+		})
+	}
+
+	async reload(input: ReloadInput): Promise<ReloadOutput> {
+		return withPage(this.ctx, input.pageId, async (page) => {
+			await page.reload({
+				waitUntil: input.waitUntil,
+				timeoutMs: input.timeoutMs,
+				ignoreCache: input.ignoreCache,
+			})
+			return { url: page.url() }
+		})
+	}
+
+	async goBack(input: GoBackInput): Promise<GoBackOutput> {
+		return withPage(this.ctx, input.pageId, async (page) => {
+			const response = await page.goBack({
+				waitUntil: input.waitUntil,
+				timeoutMs: input.timeoutMs,
+			})
+			return { navigated: response !== null, url: page.url() }
+		})
+	}
+
+	async goForward(input: GoForwardInput): Promise<GoForwardOutput> {
+		return withPage(this.ctx, input.pageId, async (page) => {
+			const response = await page.goForward({
+				waitUntil: input.waitUntil,
+				timeoutMs: input.timeoutMs,
+			})
+			return { navigated: response !== null, url: page.url() }
+		})
+	}
+
+	async snapshot_dom(input: SnapshotDomInput): Promise<SnapshotDomOutput> {
+		return withPage(this.ctx, input.pageId, async (page) => {
+			const snapshot = await page.snapshot({
+				includeIframes: input.includeIframes,
+			})
+			return {
+				tree: snapshot.formattedTree,
+				xpathMap: snapshot.xpathMap,
+				urlMap: snapshot.urlMap,
+			}
+		})
+	}
+
+	async click_on_id(input: ClickOnIdInput): Promise<ClickOnIdOutput> {
+		return this.actOnEncodedId(input.pageId, input.id, (locator) =>
+			locator.click(),
+		)
+	}
+
+	async fill_on_id(input: FillOnIdInput): Promise<FillOnIdOutput> {
+		return this.actOnEncodedId(input.pageId, input.id, (locator) =>
+			locator.fill(input.value),
+		)
+	}
+
+	async type_on_id(input: TypeOnIdInput): Promise<TypeOnIdOutput> {
+		return this.actOnEncodedId(input.pageId, input.id, (locator) =>
+			locator.type(input.text, { delay: input.delay }),
+		)
+	}
+
+	async hover_on_id(input: HoverOnIdInput): Promise<HoverOnIdOutput> {
+		return this.actOnEncodedId(input.pageId, input.id, (locator) =>
+			locator.hover(),
+		)
+	}
+
+	private async actOnEncodedId(
+		pageId: string,
+		encodedId: string,
+		action: (locator: DeepLocator) => Promise<void>,
+	): Promise<ClickOnIdOutput> {
+		return withPage(this.ctx, pageId, async (page) => {
+			const { xpathMap } = await page.snapshot()
+			const xpath = xpathMap[encodedId]
+			if (!xpath) {
+				throw new Error(`Unknown encoded id: ${encodedId}`)
+			}
+			await action(page.deepLocator(xpath))
+			await page.waitForLoadState("networkidle", 5000)
+			return {}
+		})
+	}
+}
+
+export function createHandstageContextAgentToolHandlers(
+	ctx: Context,
+): HandstageAgentToolHandlers {
+	return new HandstageContextAgentToolHandlers(ctx)
+}

package/src/index.ts

@@ -0,0 +1,10 @@
+export {
+	createHandstageAgentToolDefinitions,
+	type HandstageAgentToolSet,
+} from "./definitions"
+export {
+	createHandstageContextAgentToolHandlers,
+	HandstageContextAgentToolHandlers,
+} from "./handlers"
+export * from "./handlerTypes"
+export * from "./schemas"

package/src/result.ts

@@ -0,0 +1,31 @@
+import type { Context, Page } from "@handstage/core"
+import type { ErrResult } from "./types"
+
+export function formatError(error: unknown): string {
+	return error instanceof Error ? error.message : String(error)
+}
+
+export function errResult(error: string): ErrResult {
+	return { ok: false, error }
+}
+
+export async function tryAgentResult<T extends Record<string, unknown>>(
+	fn: () => Promise<T>,
+): Promise<({ ok: true } & T) | ErrResult> {
+	try {
+		const data = await fn()
+		return { ok: true, ...data }
+	} catch (error) {
+		return errResult(formatError(error))
+	}
+}
+
+export async function withPage<T extends Record<string, unknown>>(
+	ctx: Context,
+	pageId: string,
+	fn: (page: Page) => Promise<T>,
+): Promise<({ ok: true } & T) | ErrResult> {
+	const page = ctx.resolvePageByTargetId(pageId)
+	if (!page) return errResult(`Unknown pageId: ${pageId}`)
+	return tryAgentResult(() => fn(page))
+}