gsd-pi 2.7.1 → 2.8.0

package/src/resources/extensions/browser-tools/lifecycle.ts

@@ -0,0 +1,265 @@
+/**
+ * browser-tools — browser lifecycle management
+ *
+ * Manages the shared Browser + BrowserContext + Page singleton.
+ * Injects EVALUATE_HELPERS_SOURCE via context.addInitScript() so that
+ * page.evaluate() callbacks can reference window.__pi.* utilities.
+ */
+
+import type { Browser, BrowserContext, Frame, Page } from "playwright";
+import path from "node:path";
+import {
+	registryAddPage,
+	registryGetActive,
+	registryRemovePage,
+	registrySetActive,
+} from "./core.js";
+import {
+	getBrowser,
+	setBrowser,
+	getContext,
+	setContext,
+	pageRegistry,
+	getActiveFrame,
+	setActiveFrame,
+	logPusher,
+	getConsoleLogs,
+	getNetworkLogs,
+	getDialogLogs,
+	getPendingCriticalRequestsByPage,
+	setHarState,
+	resetAllState,
+	HAR_FILENAME,
+	type ConsoleEntry,
+	type NetworkEntry,
+} from "./state.js";
+import {
+	isCriticalResourceType,
+	updatePendingCriticalRequests,
+	ensureSessionStartedAt,
+	ensureSessionArtifactDir,
+} from "./utils.js";
+import { EVALUATE_HELPERS_SOURCE } from "./evaluate-helpers.js";
+
+// ---------------------------------------------------------------------------
+// Page event wiring
+// ---------------------------------------------------------------------------
+
+/** Attach all event listeners to a page. Called on initial page and new tabs. */
+export function attachPageListeners(p: Page, pageId: number): void {
+	const pendingMap = getPendingCriticalRequestsByPage();
+	pendingMap.set(p, 0);
+
+	const consoleLogs = getConsoleLogs();
+	const networkLogs = getNetworkLogs();
+	const dialogLogs = getDialogLogs();
+
+	// Console messages
+	p.on("console", (msg) => {
+		logPusher(consoleLogs, {
+			type: msg.type(),
+			text: msg.text(),
+			timestamp: Date.now(),
+			url: p.url(),
+			pageId,
+		});
+	});
+
+	// Uncaught JS errors
+	p.on("pageerror", (err) => {
+		logPusher(consoleLogs, {
+			type: "pageerror",
+			text: err.message,
+			timestamp: Date.now(),
+			url: p.url(),
+			pageId,
+		});
+	});
+
+	// Network requests — start/completed/failed
+	p.on("request", (request) => {
+		if (isCriticalResourceType(request.resourceType())) {
+			updatePendingCriticalRequests(p, 1);
+		}
+	});
+
+	p.on("requestfinished", async (request) => {
+		if (isCriticalResourceType(request.resourceType())) {
+			updatePendingCriticalRequests(p, -1);
+		}
+		try {
+			const response = await request.response();
+			const status = response?.status() ?? null;
+			const entry: NetworkEntry = {
+				method: request.method(),
+				url: request.url(),
+				status,
+				resourceType: request.resourceType(),
+				timestamp: Date.now(),
+				failed: false,
+				pageId,
+			};
+			if (response && status !== null && status >= 400) {
+				try {
+					const body = await response.text();
+					entry.responseBody = body.slice(0, 2000);
+				} catch {}
+			}
+			logPusher(networkLogs, entry);
+		} catch {}
+	});
+
+	p.on("requestfailed", (request) => {
+		if (isCriticalResourceType(request.resourceType())) {
+			updatePendingCriticalRequests(p, -1);
+		}
+		logPusher(networkLogs, {
+			method: request.method(),
+			url: request.url(),
+			status: null,
+			resourceType: request.resourceType(),
+			timestamp: Date.now(),
+			failed: true,
+			failureText: request.failure()?.errorText ?? "Unknown failure",
+			pageId,
+		});
+	});
+
+	// Auto-handle JS dialogs (alert, confirm, prompt, beforeunload)
+	p.on("dialog", async (dialog) => {
+		logPusher(dialogLogs, {
+			type: dialog.type(),
+			message: dialog.message(),
+			timestamp: Date.now(),
+			url: p.url(),
+			defaultValue: dialog.defaultValue() || undefined,
+			accepted: true,
+			pageId,
+		});
+		// Auto-accept all dialogs to prevent page freezes
+		await dialog.accept().catch(() => {});
+	});
+
+	// Frame detach handler — clears activeFrame if the selected frame detaches
+	p.on("framedetached", (frame) => {
+		if (getActiveFrame() === frame) setActiveFrame(null);
+	});
+
+	// Page close handler — removes page from registry and handles active fallback
+	p.on("close", () => {
+		try {
+			registryRemovePage(pageRegistry, pageId);
+		} catch {
+			// Page already removed (e.g. during closeBrowser)
+		}
+	});
+}
+
+// ---------------------------------------------------------------------------
+// Browser lifecycle
+// ---------------------------------------------------------------------------
+
+export async function ensureBrowser(): Promise<{ browser: Browser; context: BrowserContext; page: Page }> {
+	const existingBrowser = getBrowser();
+	const existingContext = getContext();
+	if (existingBrowser && existingContext) {
+		return { browser: existingBrowser, context: existingContext, page: getActivePage() };
+	}
+
+	const startedAt = ensureSessionStartedAt();
+	const artifactDir = await ensureSessionArtifactDir();
+	const sessionHarPath = path.join(artifactDir, HAR_FILENAME);
+	setHarState({
+		enabled: true,
+		configuredAtContextCreation: true,
+		path: sessionHarPath,
+		exportCount: 0,
+		lastExportedPath: null,
+		lastExportedAt: null,
+	});
+
+	// Lazy import so playwright is only loaded when actually needed
+	const { chromium } = await import("playwright");
+
+	const launchOptions: Record<string, unknown> = { headless: false };
+	const customPath = process.env.BROWSER_PATH;
+	if (customPath) launchOptions.executablePath = customPath;
+	const browser = await chromium.launch(launchOptions);
+	const context = await browser.newContext({
+		deviceScaleFactor: 2,
+		viewport: { width: 1280, height: 800 },
+		recordHar: {
+			path: sessionHarPath,
+			mode: "minimal",
+			content: "omit",
+		},
+	});
+
+	// Inject shared browser-side utilities into every new page/frame
+	await context.addInitScript(EVALUATE_HELPERS_SOURCE);
+
+	setBrowser(browser);
+	setContext(context);
+
+	const initialPage = await context.newPage();
+	const pageEntry = registryAddPage(pageRegistry, {
+		page: initialPage,
+		title: await initialPage.title().catch(() => ""),
+		url: initialPage.url(),
+		opener: null,
+	});
+	registrySetActive(pageRegistry, pageEntry.id);
+	attachPageListeners(initialPage, pageEntry.id);
+
+	// Register new pages (popups, target="_blank", window.open) but do NOT auto-switch
+	context.on("page", (newPage) => {
+		// Determine opener page ID — find which registry page opened this one
+		const openerPage = newPage.opener();
+		let openerId: number | null = null;
+		if (openerPage) {
+			const openerEntry = pageRegistry.pages.find((e: any) => e.page === openerPage);
+			if (openerEntry) openerId = openerEntry.id;
+		}
+		const entry = registryAddPage(pageRegistry, {
+			page: newPage,
+			title: "",
+			url: newPage.url(),
+			opener: openerId,
+		});
+		attachPageListeners(newPage, entry.id);
+		// Update title once loaded
+		newPage.waitForLoadState("domcontentloaded", { timeout: 5000 })
+			.then(() => newPage.title())
+			.then((title) => { entry.title = title; })
+			.catch(() => {});
+	});
+
+	return { browser, context, page: getActivePage() };
+}
+
+/** Get the currently active page from the registry. */
+export function getActivePage(): Page {
+	return registryGetActive(pageRegistry).page;
+}
+
+/** Get the active target — returns the selected frame if one is active, otherwise the active page. */
+export function getActiveTarget(): Page | Frame {
+	return getActiveFrame() ?? getActivePage();
+}
+
+/** Safe accessor for error handling — returns the active page or null if unavailable. */
+export function getActivePageOrNull(): Page | null {
+	try {
+		return getActivePage();
+	} catch {
+		return null;
+	}
+}
+
+export async function closeBrowser(): Promise<void> {
+	const browser = getBrowser();
+	if (browser) {
+		await browser.close().catch(() => {});
+	}
+	resetAllState();
+}

package/src/resources/extensions/browser-tools/package.json

@@ -10,11 +10,15 @@
     "extensions": ["./index.ts"]
   },
   "peerDependencies": {
-    "playwright": ">=1.40.0"
+    "playwright": ">=1.40.0",
+    "sharp": ">=0.33.0"
   },
   "peerDependenciesMeta": {
     "playwright": {
       "optional": true
+    },
+    "sharp": {
+      "optional": true
     }
   }
 }

package/src/resources/extensions/browser-tools/refs.ts

@@ -0,0 +1,264 @@
+/**
+ * browser-tools — ref snapshot and resolution
+ *
+ * Builds deterministic element snapshots and resolves ref targets.
+ * Uses window.__pi.* utilities injected via addInitScript (from
+ * evaluate-helpers.ts) instead of redeclaring functions inline.
+ *
+ * Functions kept inline (not shared/duplicated):
+ *   - matchesMode, computeNearestHeading, computeFormOwnership
+ */
+
+import type { Frame, Page } from "playwright";
+import type { RefNode } from "./state.js";
+import { getSnapshotModeConfig } from "./core.js";
+
+// ---------------------------------------------------------------------------
+// buildRefSnapshot
+// ---------------------------------------------------------------------------
+
+export async function buildRefSnapshot(
+	target: Page | Frame,
+	options: { selector?: string; interactiveOnly: boolean; limit: number; mode?: string },
+): Promise<Array<Omit<RefNode, "ref">>> {
+	// Resolve mode config in Node context and serialize it as plain data for the evaluate callback
+	const modeConfig = options.mode ? getSnapshotModeConfig(options.mode) : null;
+	return await target.evaluate(({ selector, interactiveOnly, limit, modeConfig: mc }) => {
+		const root = selector ? document.querySelector(selector) : document.body;
+		if (!root) {
+			throw new Error(`Selector scope not found: ${selector}`);
+		}
+
+		// Use injected window.__pi utilities
+		const pi = (window as any).__pi;
+		const simpleHash = pi.simpleHash;
+		const isVisible = pi.isVisible;
+		const isEnabled = pi.isEnabled;
+		const inferRole = pi.inferRole;
+		const accessibleName = pi.accessibleName;
+		const isInteractiveEl = pi.isInteractiveEl;
+		const cssPath = pi.cssPath;
+		const domPath = pi.domPath;
+		const selectorHints = pi.selectorHints;
+
+		// Mode-based element matching — used when a snapshot mode config is provided
+		const matchesMode = (el: Element, cfg: { tags: string[]; roles: string[]; selectors: string[]; ariaAttributes: string[] }): boolean => {
+			const tag = el.tagName.toLowerCase();
+			if (cfg.tags.length > 0 && cfg.tags.includes(tag)) return true;
+			const role = inferRole(el);
+			if (cfg.roles.length > 0 && cfg.roles.includes(role)) return true;
+			for (const sel of cfg.selectors) {
+				try { if (el.matches(sel)) return true; } catch { /* invalid selector, skip */ }
+			}
+			for (const attr of cfg.ariaAttributes) {
+				if (el.hasAttribute(attr)) return true;
+			}
+			return false;
+		};
+
+		let elements = Array.from(root.querySelectorAll("*"));
+
+		if (mc) {
+			// Mode takes precedence over interactiveOnly
+			if (mc.visibleOnly) {
+				// visible_only mode: include all elements that are visible
+				elements = elements.filter((el) => isVisible(el));
+			} else if (mc.useInteractiveFilter) {
+				// interactive mode: reuse existing isInteractiveEl
+				elements = elements.filter((el) => isInteractiveEl(el));
+			} else if (mc.containerExpand) {
+				// Container-expanding modes (dialog, errors): match containers, then include
+				// all interactive children of those containers, plus the containers themselves
+				const containers: Element[] = [];
+				const directMatches: Element[] = [];
+				for (const el of elements) {
+					if (matchesMode(el, mc)) {
+						// Check if this is a container element (has children)
+						const childEls = el.querySelectorAll("*");
+						if (childEls.length > 0) {
+							containers.push(el);
+						} else {
+							directMatches.push(el);
+						}
+					}
+				}
+				// Collect container elements + all interactive children inside containers
+				const result = new Set<Element>(directMatches);
+				for (const container of containers) {
+					result.add(container);
+					const children = Array.from(container.querySelectorAll("*"));
+					for (const child of children) {
+						if (isInteractiveEl(child)) result.add(child);
+					}
+				}
+				elements = Array.from(result);
+			} else {
+				// Standard mode filtering by tag/role/selector/ariaAttribute
+				elements = elements.filter((el) => matchesMode(el, mc));
+			}
+		} else if (!interactiveOnly) {
+			if (root instanceof Element) elements.unshift(root);
+		} else {
+			elements = elements.filter((el) => isInteractiveEl(el));
+		}
+
+		const seen = new Set<Element>();
+		const unique = elements.filter((el) => {
+			if (seen.has(el)) return false;
+			seen.add(el);
+			return true;
+		});
+
+		// Fingerprint helpers — computed for each element in the snapshot
+		const computeNearestHeading = (el: Element): string => {
+			const headingTags = new Set(["H1", "H2", "H3", "H4", "H5", "H6"]);
+			// Walk up ancestors looking for heading or preceding-sibling heading
+			let current: Element | null = el;
+			while (current && current !== document.body) {
+				// Check preceding siblings of current
+				let sib: Element | null = current.previousElementSibling;
+				while (sib) {
+					if (headingTags.has(sib.tagName) || sib.getAttribute("role") === "heading") {
+						return (sib.textContent || "").trim().replace(/\s+/g, " ").slice(0, 80);
+					}
+					sib = sib.previousElementSibling;
+				}
+				// Check if the parent itself is a heading (unlikely but possible)
+				const parent = current.parentElement;
+				if (parent && (headingTags.has(parent.tagName) || parent.getAttribute("role") === "heading")) {
+					return (parent.textContent || "").trim().replace(/\s+/g, " ").slice(0, 80);
+				}
+				current = parent;
+			}
+			return "";
+		};
+
+		const computeFormOwnership = (el: Element): string => {
+			// Check form attribute (explicit form association)
+			const formAttr = el.getAttribute("form");
+			if (formAttr) return formAttr;
+			// Walk up ancestors looking for <form>
+			let current: Element | null = el.parentElement;
+			while (current && current !== document.body) {
+				if (current.tagName === "FORM") {
+					return (current as HTMLFormElement).id || (current as HTMLFormElement).name || "form";
+				}
+				current = current.parentElement;
+			}
+			return "";
+		};
+
+		return unique.slice(0, limit).map((el) => {
+			const tag = el.tagName.toLowerCase();
+			const role = inferRole(el);
+			const textContent = (el.textContent || "").trim().replace(/\s+/g, " ").slice(0, 200);
+			const childTags = Array.from(el.children).map((c) => c.tagName.toLowerCase());
+
+			return {
+				tag,
+				role,
+				name: accessibleName(el),
+				selectorHints: selectorHints(el),
+				isVisible: isVisible(el),
+				isEnabled: isEnabled(el),
+				xpathOrPath: cssPath(el),
+				href: el.getAttribute("href") || undefined,
+				type: el.getAttribute("type") || undefined,
+				path: domPath(el),
+				contentHash: simpleHash(textContent),
+				structuralSignature: simpleHash(`${tag}|${role}|${childTags.join(",")}`),
+				nearestHeading: computeNearestHeading(el),
+				formOwnership: computeFormOwnership(el),
+			};
+		});
+	}, { ...options, modeConfig });
+}
+
+// ---------------------------------------------------------------------------
+// resolveRefTarget
+// ---------------------------------------------------------------------------
+
+export async function resolveRefTarget(
+	target: Page | Frame,
+	node: RefNode,
+): Promise<{ ok: true; selector: string } | { ok: false; reason: string }> {
+	return await target.evaluate((refNode) => {
+		// Use injected window.__pi utilities
+		const pi = (window as any).__pi;
+		const cssPath = pi.cssPath;
+		const simpleHash = pi.simpleHash;
+
+		const byPath = (): Element | null => {
+			let current: Element | null = document.documentElement;
+			for (const idx of refNode.path || []) {
+				if (!current || idx < 0 || idx >= current.children.length) return null;
+				current = current.children[idx] as Element;
+			}
+			return current;
+		};
+
+		const nodeName = (el: Element): string => {
+			return (
+				el.getAttribute("aria-label")?.trim() ||
+				(el as HTMLInputElement).value?.trim() ||
+				el.getAttribute("placeholder")?.trim() ||
+				(el.textContent || "").trim().replace(/\s+/g, " ").slice(0, 80)
+			);
+		};
+
+		// Tier 1: path-based resolution
+		const pathEl = byPath();
+		if (pathEl && pathEl.tagName.toLowerCase() === refNode.tag) {
+			return { ok: true as const, selector: cssPath(pathEl) };
+		}
+
+		// Tier 2: selector hints
+		for (const hint of refNode.selectorHints || []) {
+			try {
+				const el = document.querySelector(hint);
+				if (!el) continue;
+				if (el.tagName.toLowerCase() !== refNode.tag) continue;
+				return { ok: true as const, selector: cssPath(el) };
+			} catch {
+				// ignore malformed selector hint
+			}
+		}
+
+		// Tier 3: role + name match
+		const candidates = Array.from(document.querySelectorAll(refNode.tag));
+		const matchTarget = candidates.find((el) => {
+			const role = el.getAttribute("role") || "";
+			const name = nodeName(el);
+			const roleMatch = !refNode.role || role === refNode.role;
+			const nameMatch = !!refNode.name && name.toLowerCase() === refNode.name.toLowerCase();
+			return roleMatch && nameMatch;
+		});
+		if (matchTarget) {
+			return { ok: true as const, selector: cssPath(matchTarget) };
+		}
+
+		// Tier 4: structural signature + content hash fingerprint matching
+		if (refNode.contentHash && refNode.structuralSignature) {
+			const fpMatches: Element[] = [];
+			for (const candidate of candidates) {
+				const tag = candidate.tagName.toLowerCase();
+				const role = candidate.getAttribute("role") || "";
+				const textContent = (candidate.textContent || "").trim().replace(/\s+/g, " ").slice(0, 200);
+				const childTags = Array.from(candidate.children).map((c) => c.tagName.toLowerCase());
+				const candidateContentHash = simpleHash(textContent);
+				const candidateStructSig = simpleHash(`${tag}|${role}|${childTags.join(",")}`);
+				if (candidateContentHash === refNode.contentHash && candidateStructSig === refNode.structuralSignature) {
+					fpMatches.push(candidate);
+				}
+			}
+			if (fpMatches.length === 1) {
+				return { ok: true as const, selector: cssPath(fpMatches[0]) };
+			}
+			if (fpMatches.length > 1) {
+				return { ok: false as const, reason: "multiple fingerprint matches — ambiguous" };
+			}
+		}
+
+		return { ok: false as const, reason: "element not found in current DOM" };
+	}, node);
+}