gsd-pi 2.22.0 → 2.24.0

package/src/resources/extensions/browser-tools/tools/codegen.ts

@@ -0,0 +1,274 @@
+import type { ExtensionAPI } from "@gsd/pi-coding-agent";
+import { Type } from "@sinclair/typebox";
+import type { ToolDeps } from "../state.js";
+import { getActionTimeline } from "../state.js";
+
+/**
+ * Test code generation — transform recorded browser session into a Playwright test script.
+ */
+
+export function registerCodegenTools(pi: ExtensionAPI, deps: ToolDeps): void {
+	pi.registerTool({
+		name: "browser_generate_test",
+		label: "Browser Generate Test",
+		description:
+			"Generate a runnable Playwright test script from the recorded action timeline. " +
+			"Transforms navigation, click, type, and assertion actions into standard Playwright test syntax. " +
+			"Uses stable selectors (role-based preferred). Writes the test file to a configurable path.",
+		parameters: Type.Object({
+			name: Type.Optional(
+				Type.String({ description: "Test name (used for describe/test block and filename). Default: 'recorded-session'." }),
+			),
+			outputPath: Type.Optional(
+				Type.String({
+					description:
+						"Output file path for the generated test. Default: writes to session artifacts directory. " +
+						"Use a path ending in .spec.ts for standard Playwright test convention.",
+				}),
+			),
+			includeAssertions: Type.Optional(
+				Type.Boolean({ description: "Include assertion steps from the timeline (default: true)." }),
+			),
+		}),
+
+		async execute(_toolCallId, params, _signal, _onUpdate, _ctx) {
+			try {
+				await deps.ensureBrowser();
+				const timeline = getActionTimeline();
+
+				if (timeline.entries.length === 0) {
+					return {
+						content: [{ type: "text", text: "No actions recorded in the current session. Interact with pages first, then generate a test." }],
+						details: { error: "no_actions" },
+						isError: true,
+					};
+				}
+
+				const testName = params.name ?? "recorded-session";
+				const includeAssertions = params.includeAssertions ?? true;
+
+				// Transform timeline entries into Playwright test code
+				const testLines: string[] = [];
+				const imports = new Set<string>();
+				imports.add("test");
+				imports.add("expect");
+
+				testLines.push(`test.describe('${escapeString(testName)}', () => {`);
+				testLines.push(`  test('recorded session', async ({ page }) => {`);
+
+				let lastUrl = "";
+				let actionCount = 0;
+
+				for (const entry of timeline.entries) {
+					if (entry.status === "error" && entry.tool !== "browser_assert") continue;
+
+					const params = parseParamsSummary(entry.paramsSummary);
+
+					switch (entry.tool) {
+						case "browser_navigate": {
+							const url = params.url;
+							if (url && url !== lastUrl) {
+								testLines.push(`    await page.goto(${quote(url)});`);
+								lastUrl = url;
+								actionCount++;
+							}
+							break;
+						}
+
+						case "browser_click": {
+							const selector = params.selector;
+							if (selector) {
+								testLines.push(`    await page.locator(${quote(selector)}).click();`);
+								actionCount++;
+							}
+							break;
+						}
+
+						case "browser_click_ref": {
+							// Refs are session-specific — add comment
+							testLines.push(`    // browser_click_ref: ${entry.paramsSummary} — replace with stable selector`);
+							actionCount++;
+							break;
+						}
+
+						case "browser_type": {
+							const selector = params.selector;
+							const text = params.text;
+							if (selector && text) {
+								testLines.push(`    await page.locator(${quote(selector)}).fill(${quote(text)});`);
+								actionCount++;
+							}
+							break;
+						}
+
+						case "browser_fill_ref": {
+							testLines.push(`    // browser_fill_ref: ${entry.paramsSummary} — replace with stable selector`);
+							actionCount++;
+							break;
+						}
+
+						case "browser_key_press": {
+							const key = params.key;
+							if (key) {
+								testLines.push(`    await page.keyboard.press(${quote(key)});`);
+								actionCount++;
+							}
+							break;
+						}
+
+						case "browser_select_option": {
+							const selector = params.selector;
+							const option = params.option;
+							if (selector && option) {
+								testLines.push(`    await page.locator(${quote(selector)}).selectOption(${quote(option)});`);
+								actionCount++;
+							}
+							break;
+						}
+
+						case "browser_set_checked": {
+							const selector = params.selector;
+							const checked = params.checked;
+							if (selector) {
+								testLines.push(`    await page.locator(${quote(selector)}).setChecked(${checked === "true"});`);
+								actionCount++;
+							}
+							break;
+						}
+
+						case "browser_hover": {
+							const selector = params.selector;
+							if (selector) {
+								testLines.push(`    await page.locator(${quote(selector)}).hover();`);
+								actionCount++;
+							}
+							break;
+						}
+
+						case "browser_wait_for": {
+							const condition = params.condition;
+							const value = params.value;
+							if (condition === "selector_visible" && value) {
+								testLines.push(`    await expect(page.locator(${quote(value)})).toBeVisible();`);
+								actionCount++;
+							} else if (condition === "text_visible" && value) {
+								testLines.push(`    await expect(page.locator('body')).toContainText(${quote(value)});`);
+								actionCount++;
+							} else if (condition === "url_contains" && value) {
+								testLines.push(`    await page.waitForURL(${quote(`**/*${value}*`)});`);
+								actionCount++;
+							} else if (condition === "network_idle") {
+								testLines.push(`    await page.waitForLoadState('networkidle');`);
+								actionCount++;
+							} else if (condition === "delay" && value) {
+								testLines.push(`    await page.waitForTimeout(${value});`);
+								actionCount++;
+							}
+							break;
+						}
+
+						case "browser_assert": {
+							if (!includeAssertions) break;
+							// The assertion details are in verificationSummary
+							if (entry.verificationSummary) {
+								testLines.push(`    // Assertion: ${entry.verificationSummary}`);
+							}
+							actionCount++;
+							break;
+						}
+
+						case "browser_scroll": {
+							const direction = params.direction;
+							const amount = params.amount ?? "300";
+							const delta = direction === "up" ? `-${amount}` : amount;
+							testLines.push(`    await page.mouse.wheel(0, ${delta});`);
+							actionCount++;
+							break;
+						}
+
+						case "browser_set_viewport": {
+							const width = params.width;
+							const height = params.height;
+							if (width && height) {
+								testLines.push(`    await page.setViewportSize({ width: ${width}, height: ${height} });`);
+								actionCount++;
+							}
+							break;
+						}
+
+						default:
+							// Skip tools that don't map to Playwright test actions
+							break;
+					}
+				}
+
+				testLines.push(`  });`);
+				testLines.push(`});`);
+
+				const importLine = `import { ${[...imports].join(", ")} } from '@playwright/test';`;
+				const fullTest = `${importLine}\n\n${testLines.join("\n")}\n`;
+
+				// Write to file
+				let outputPath: string;
+				if (params.outputPath) {
+					outputPath = params.outputPath;
+				} else {
+					const safeName = deps.sanitizeArtifactName(testName, "recorded-session");
+					outputPath = deps.buildSessionArtifactPath(`${safeName}.spec.ts`);
+				}
+
+				await deps.ensureSessionArtifactDir();
+				const { path: writtenPath, bytes } = await deps.writeArtifactFile(outputPath, fullTest);
+
+				return {
+					content: [{
+						type: "text",
+						text: `Test generated: ${writtenPath}\nActions: ${actionCount}\nTimeline entries processed: ${timeline.entries.length}\n\n${fullTest}`,
+					}],
+					details: {
+						path: writtenPath,
+						bytes,
+						actionCount,
+						timelineEntries: timeline.entries.length,
+						testCode: fullTest,
+					},
+				};
+			} catch (err: any) {
+				return {
+					content: [{ type: "text", text: `Test generation failed: ${err.message}` }],
+					details: { error: err.message },
+					isError: true,
+				};
+			}
+		},
+	});
+}
+
+function escapeString(s: string): string {
+	return s.replace(/'/g, "\\'").replace(/\\/g, "\\\\");
+}
+
+function quote(s: string): string {
+	// Use single quotes for simple strings, backtick for those with quotes
+	if (!s.includes("'")) return `'${s}'`;
+	if (!s.includes("`")) return `\`${s}\``;
+	return `'${s.replace(/'/g, "\\'")}'`;
+}
+
+/**
+ * Parse the paramsSummary string back into key-value pairs.
+ * Format: key="value", key=value, key=[N], key={...}
+ */
+function parseParamsSummary(summary: string): Record<string, string> {
+	const result: Record<string, string> = {};
+	if (!summary) return result;
+
+	const regex = /(\w+)=(?:"([^"]*(?:\\"[^"]*)*)"|([^,\s]+))/g;
+	let match;
+	while ((match = regex.exec(summary)) !== null) {
+		const key = match[1];
+		const value = match[2] ?? match[3];
+		result[key] = value;
+	}
+	return result;
+}

package/src/resources/extensions/browser-tools/tools/device.ts

@@ -0,0 +1,183 @@
+import type { ExtensionAPI } from "@gsd/pi-coding-agent";
+import { Type } from "@sinclair/typebox";
+import type { ToolDeps } from "../state.js";
+
+/**
+ * Device emulation tool — full device simulation using Playwright's built-in device descriptors.
+ */
+
+export function registerDeviceTools(pi: ExtensionAPI, deps: ToolDeps): void {
+	pi.registerTool({
+		name: "browser_emulate_device",
+		label: "Browser Emulate Device",
+		description:
+			"Simulate a specific device by setting viewport, user agent, device scale factor, touch, and mobile flag. " +
+			"Uses Playwright's built-in device descriptors (~143 devices). Accepts fuzzy matching on device name. " +
+			"Note: Full emulation (user agent, isMobile) requires a context restart — the current page state will be lost. " +
+			"The tool recreates the context with the device profile applied.",
+		parameters: Type.Object({
+			device: Type.String({
+				description:
+					"Device name (e.g., 'iPhone 15', 'Pixel 7', 'iPad Pro 11'). " +
+					"Case-insensitive fuzzy matching. Use 'list' to see all available devices.",
+			}),
+		}),
+
+		async execute(_toolCallId, params, _signal, _onUpdate, _ctx) {
+			try {
+				const { chromium, devices } = await import("playwright");
+				const allDeviceNames = Object.keys(devices);
+
+				// Handle 'list' request
+				if (params.device.toLowerCase() === "list") {
+					// Group by base device name (remove landscape variants for cleaner display)
+					const baseNames = allDeviceNames.filter((n) => !n.endsWith(" landscape"));
+					return {
+						content: [{
+							type: "text",
+							text: `Available devices (${allDeviceNames.length} total, ${baseNames.length} base):\n${baseNames.join("\n")}`,
+						}],
+						details: { devices: baseNames, total: allDeviceNames.length },
+					};
+				}
+
+				// Fuzzy match device name
+				const needle = params.device.toLowerCase();
+				let exactMatch = allDeviceNames.find((n) => n.toLowerCase() === needle);
+				if (!exactMatch) {
+					// Try contains match
+					const containsMatches = allDeviceNames.filter((n) => n.toLowerCase().includes(needle));
+					if (containsMatches.length === 1) {
+						exactMatch = containsMatches[0];
+					} else if (containsMatches.length > 1) {
+						// Pick the shortest match (most specific)
+						containsMatches.sort((a, b) => a.length - b.length);
+						exactMatch = containsMatches[0];
+						const suggestions = containsMatches.slice(0, 5).join(", ");
+						// Continue with best match but mention alternatives
+					} else {
+						// No match at all — suggest closest
+						const suggestions = allDeviceNames
+							.map((n) => ({ name: n, score: fuzzyScore(needle, n.toLowerCase()) }))
+							.sort((a, b) => b.score - a.score)
+							.slice(0, 5)
+							.map((s) => s.name);
+
+						return {
+							content: [{
+								type: "text",
+								text: `No device matching "${params.device}". Did you mean:\n${suggestions.map((s) => `  - ${s}`).join("\n")}`,
+							}],
+							details: { error: "no_match", suggestions },
+							isError: true,
+						};
+					}
+				}
+
+				const deviceDescriptor = devices[exactMatch!];
+				if (!deviceDescriptor) {
+					return {
+						content: [{ type: "text", text: `Device descriptor not found for "${exactMatch}"` }],
+						details: { error: "descriptor_not_found" },
+						isError: true,
+					};
+				}
+
+				// Context restart required for full emulation.
+				// Save current URL to navigate back after restart.
+				const { page: currentPage, context: currentCtx } = await deps.ensureBrowser();
+				const currentUrl = currentPage.url();
+
+				// Close existing browser and relaunch with device profile
+				await deps.closeBrowser();
+
+				// Re-launch — ensureBrowser doesn't accept device params, so we do it manually.
+				// This is a one-off context creation with device emulation.
+				const needsHeadless = process.platform === "linux" && !process.env.DISPLAY;
+				const launchOptions: Record<string, unknown> = {
+					headless: needsHeadless || process.env.FORCE_HEADLESS === "true",
+				};
+				const customPath = process.env.BROWSER_PATH;
+				if (customPath) launchOptions.executablePath = customPath;
+
+				const browser = await chromium.launch(launchOptions);
+				const context = await browser.newContext({
+					...deviceDescriptor,
+				});
+
+				// Inject evaluate helpers
+				const { EVALUATE_HELPERS_SOURCE } = await import("../evaluate-helpers.js");
+				await context.addInitScript(EVALUATE_HELPERS_SOURCE);
+
+				// Wire up state
+				const {
+					setBrowser, setContext, pageRegistry, setSessionStartedAt,
+					setSessionArtifactDir, resetAllState,
+				} = await import("../state.js");
+				const { registryAddPage, registrySetActive } = await import("../core.js");
+
+				// Reset state for new session
+				resetAllState();
+				setBrowser(browser);
+				setContext(context);
+				setSessionStartedAt(Date.now());
+
+				const page = await context.newPage();
+				const entry = registryAddPage(pageRegistry, {
+					page,
+					title: "",
+					url: "about:blank",
+					opener: null,
+				});
+				registrySetActive(pageRegistry, entry.id);
+				deps.attachPageListeners(page, entry.id);
+
+				// Navigate back to previous URL if it wasn't about:blank
+				if (currentUrl && currentUrl !== "about:blank") {
+					await page.goto(currentUrl, { waitUntil: "domcontentloaded", timeout: 15000 }).catch(() => {});
+				}
+
+				const viewport = deviceDescriptor.viewport;
+				const vpText = viewport ? `${viewport.width}x${viewport.height}` : "unknown";
+
+				return {
+					content: [{
+						type: "text",
+						text: `Device emulation active: ${exactMatch}\nViewport: ${vpText}\nUser Agent: ${deviceDescriptor.userAgent?.slice(0, 80) ?? "default"}...\nMobile: ${deviceDescriptor.isMobile ?? false}\nTouch: ${deviceDescriptor.hasTouch ?? false}\nScale Factor: ${deviceDescriptor.deviceScaleFactor ?? 1}\n\nContext was restarted for full emulation. Page state was reset.`,
+					}],
+					details: {
+						device: exactMatch,
+						viewport: vpText,
+						isMobile: deviceDescriptor.isMobile ?? false,
+						hasTouch: deviceDescriptor.hasTouch ?? false,
+						deviceScaleFactor: deviceDescriptor.deviceScaleFactor ?? 1,
+						userAgent: deviceDescriptor.userAgent,
+						restoredUrl: currentUrl,
+					},
+				};
+			} catch (err: any) {
+				return {
+					content: [{ type: "text", text: `Device emulation failed: ${err.message}` }],
+					details: { error: err.message },
+					isError: true,
+				};
+			}
+		},
+	});
+}
+
+/**
+ * Simple fuzzy scoring — counts matching characters in order.
+ */
+function fuzzyScore(needle: string, haystack: string): number {
+	let score = 0;
+	let hi = 0;
+	for (let ni = 0; ni < needle.length && hi < haystack.length; ni++) {
+		const idx = haystack.indexOf(needle[ni], hi);
+		if (idx >= 0) {
+			score++;
+			hi = idx + 1;
+		}
+	}
+	return score / Math.max(needle.length, 1);
+}

package/src/resources/extensions/browser-tools/tools/extract.ts

@@ -0,0 +1,229 @@
+import type { ExtensionAPI } from "@gsd/pi-coding-agent";
+import { Type } from "@sinclair/typebox";
+import type { ToolDeps } from "../state.js";
+
+/**
+ * Structured data extraction with JSON Schema validation.
+ */
+
+export function registerExtractTools(pi: ExtensionAPI, deps: ToolDeps): void {
+	pi.registerTool({
+		name: "browser_extract",
+		label: "Browser Extract",
+		description:
+			"Extract structured data from the current page using CSS selectors and validate against a JSON Schema. " +
+			"Provide a schema describing the shape of data you want. The tool extracts data by evaluating " +
+			"CSS selectors in the page context, then validates the result against your schema. " +
+			"Supports extracting single objects or arrays of items. Waits for network idle before extraction.",
+		parameters: Type.Object({
+			schema: Type.Record(Type.String(), Type.Unknown(), {
+				description:
+					"JSON Schema describing the data shape to extract. Properties should include " +
+					"'_selector' (CSS selector) and '_attribute' (attribute to read, default: 'textContent') hints. " +
+					"Example: { type: 'object', properties: { title: { _selector: 'h1', _attribute: 'textContent' }, price: { _selector: '.price', _attribute: 'textContent' } } }",
+			}),
+			selector: Type.Optional(
+				Type.String({ description: "CSS selector to scope extraction to a specific container element." }),
+			),
+			multiple: Type.Optional(
+				Type.Boolean({
+					description:
+						"If true, extract an array of items. The 'selector' parameter becomes the item container selector, " +
+						"and schema properties are extracted relative to each matched container.",
+				}),
+			),
+		}),
+
+		async execute(_toolCallId, params, _signal, _onUpdate, _ctx) {
+			try {
+				const { page: p } = await deps.ensureBrowser();
+
+				// Wait for network idle before extraction
+				await p.waitForLoadState("networkidle", { timeout: 10000 }).catch(() => {});
+
+				const schema = params.schema as any;
+				const scopeSelector = params.selector;
+				const multiple = params.multiple ?? false;
+
+				// Build extraction plan from schema
+				const extractionPlan = buildExtractionPlan(schema);
+
+				// Execute extraction in page context
+				const rawData = await p.evaluate(
+					({ plan, scope, multi }: { plan: ExtractionField[]; scope: string | undefined; multi: boolean }) => {
+						function extractFromContainer(container: Element, fields: typeof plan): Record<string, unknown> {
+							const result: Record<string, unknown> = {};
+							for (const field of fields) {
+								const el = container.querySelector(field.selector);
+								if (!el) {
+									result[field.name] = null;
+									continue;
+								}
+								let value: unknown;
+								switch (field.attribute) {
+									case "textContent":
+										value = (el.textContent ?? "").trim();
+										break;
+									case "innerText":
+										value = ((el as HTMLElement).innerText ?? "").trim();
+										break;
+									case "innerHTML":
+										value = el.innerHTML;
+										break;
+									case "href":
+										value = (el as HTMLAnchorElement).href ?? el.getAttribute("href");
+										break;
+									case "src":
+										value = (el as HTMLImageElement).src ?? el.getAttribute("src");
+										break;
+									case "value":
+										value = (el as HTMLInputElement).value;
+										break;
+									default:
+										value = el.getAttribute(field.attribute) ?? (el.textContent ?? "").trim();
+								}
+								// Type coercion
+								if (field.type === "number" && typeof value === "string") {
+									const num = parseFloat(value.replace(/[^0-9.-]/g, ""));
+									value = isNaN(num) ? value : num;
+								} else if (field.type === "boolean" && typeof value === "string") {
+									value = value.toLowerCase() === "true" || value === "1";
+								}
+								result[field.name] = value;
+							}
+							return result;
+						}
+
+						const root = scope ? document.querySelector(scope) : document.body;
+						if (!root) return { data: null, error: `Scope selector "${scope}" not found` };
+
+						if (multi) {
+							// For multiple items, scope is the item selector
+							const containers = scope
+								? document.querySelectorAll(scope)
+								: [document.body];
+							const items = Array.from(containers).map((container) =>
+								extractFromContainer(container, plan),
+							);
+							return { data: items, error: null };
+						} else {
+							return { data: extractFromContainer(root, plan), error: null };
+						}
+					},
+					{ plan: extractionPlan, scope: scopeSelector, multi: multiple },
+				);
+
+				if (rawData.error) {
+					return {
+						content: [{ type: "text", text: `Extraction failed: ${rawData.error}` }],
+						details: { error: rawData.error },
+						isError: true,
+					};
+				}
+
+				// Validate against schema using ajv
+				const validationErrors = await validateData(rawData.data, schema, multiple);
+
+				const resultText = JSON.stringify(rawData.data, null, 2);
+				const truncated = resultText.length > 4000 ? resultText.slice(0, 4000) + "\n...(truncated)" : resultText;
+
+				return {
+					content: [{
+						type: "text",
+						text: validationErrors.length > 0
+							? `Extracted data (with ${validationErrors.length} validation warning(s)):\n${truncated}\n\nValidation warnings:\n${validationErrors.join("\n")}`
+							: `Extracted data:\n${truncated}`,
+					}],
+					details: {
+						data: rawData.data,
+						validationErrors: validationErrors.length > 0 ? validationErrors : undefined,
+						fieldCount: extractionPlan.length,
+						itemCount: multiple ? (rawData.data as any[])?.length ?? 0 : 1,
+					},
+				};
+			} catch (err: any) {
+				return {
+					content: [{ type: "text", text: `Extraction failed: ${err.message}` }],
+					details: { error: err.message },
+					isError: true,
+				};
+			}
+		},
+	});
+}
+
+interface ExtractionField {
+	name: string;
+	selector: string;
+	attribute: string;
+	type: string;
+}
+
+function buildExtractionPlan(schema: any): ExtractionField[] {
+	const fields: ExtractionField[] = [];
+
+	if (!schema || typeof schema !== "object") return fields;
+
+	const properties = schema.properties ?? schema;
+
+	for (const [name, propSchema] of Object.entries(properties)) {
+		const prop = propSchema as any;
+		if (!prop || typeof prop !== "object") continue;
+
+		// Skip meta fields
+		if (name === "type" || name === "required" || name === "properties" || name === "$schema") continue;
+
+		const selector = prop._selector ?? prop.selector ?? `[data-field="${name}"], .${name}, #${name}`;
+		const attribute = prop._attribute ?? prop.attribute ?? "textContent";
+		const type = prop.type ?? "string";
+
+		fields.push({ name, selector, attribute, type });
+	}
+
+	return fields;
+}
+
+async function validateData(data: unknown, schema: any, isArray: boolean): Promise<string[]> {
+	const errors: string[] = [];
+
+	try {
+		const ajvModule = await import("ajv");
+		const Ajv = ajvModule.default ?? ajvModule;
+		const ajv = new (Ajv as any)({ allErrors: true, strict: false });
+
+		// Clean schema — remove our custom _selector/_attribute hints before validation
+		const cleanSchema = cleanSchemaForValidation(schema);
+
+		// Wrap in array schema if multiple
+		const validationSchema = isArray
+			? { type: "array", items: cleanSchema }
+			: cleanSchema;
+
+		const validate = ajv.compile(validationSchema);
+		const valid = validate(data);
+
+		if (!valid && validate.errors) {
+			for (const err of validate.errors) {
+				errors.push(`${err.instancePath || "/"}: ${err.message}`);
+			}
+		}
+	} catch (err: any) {
+		errors.push(`Schema validation setup failed: ${err.message}`);
+	}
+
+	return errors;
+}
+
+function cleanSchemaForValidation(schema: any): any {
+	if (!schema || typeof schema !== "object") return schema;
+	if (Array.isArray(schema)) return schema.map(cleanSchemaForValidation);
+
+	const cleaned: any = {};
+	for (const [key, value] of Object.entries(schema)) {
+		if (key.startsWith("_")) continue; // Remove our custom hints
+		if (key === "selector" && typeof value === "string") continue; // Also remove plain 'selector'
+		if (key === "attribute" && typeof value === "string") continue; // Also remove plain 'attribute'
+		cleaned[key] = cleanSchemaForValidation(value);
+	}
+	return cleaned;
+}