gsd-pi 2.10.12 → 2.12.0

package/dist/resources/extensions/browser-tools/evaluate-helpers.ts

@@ -0,0 +1,184 @@
+/**
+ * browser-tools — browser-side evaluate helpers
+ *
+ * Exports a single string constant `EVALUATE_HELPERS_SOURCE` containing an IIFE
+ * that attaches utility functions to `window.__pi`.  This is injected into every
+ * new BrowserContext via `context.addInitScript()` so that `page.evaluate()`
+ * callbacks can reference `window.__pi.cssPath(el)` etc. instead of redeclaring
+ * the same functions inline.
+ *
+ * The `simpleHash` function uses the djb2 algorithm identical to
+ * `computeContentHash` / `computeStructuralSignature` in `core.js`.
+ *
+ * Functions provided (9):
+ *   cssPath, simpleHash, isVisible, isEnabled, inferRole,
+ *   accessibleName, isInteractiveEl, domPath, selectorHints
+ */
+
+export const EVALUATE_HELPERS_SOURCE = `(function() {
+  var pi = window.__pi = window.__pi || {};
+
+  // -----------------------------------------------------------------------
+  // 1. simpleHash — djb2 hash matching core.js computeContentHash
+  // -----------------------------------------------------------------------
+  pi.simpleHash = function simpleHash(str) {
+    if (!str) return "0";
+    var h = 5381;
+    for (var i = 0; i < str.length; i++) {
+      h = ((h << 5) - h + str.charCodeAt(i)) | 0;
+    }
+    return (h >>> 0).toString(16);
+  };
+
+  // -----------------------------------------------------------------------
+  // 2. isVisible
+  // -----------------------------------------------------------------------
+  pi.isVisible = function isVisible(el) {
+    var style = window.getComputedStyle(el);
+    if (style.display === "none" || style.visibility === "hidden") return false;
+    var rect = el.getBoundingClientRect();
+    return rect.width > 0 && rect.height > 0;
+  };
+
+  // -----------------------------------------------------------------------
+  // 3. isEnabled
+  // -----------------------------------------------------------------------
+  pi.isEnabled = function isEnabled(el) {
+    var disabledAttr = el.getAttribute("disabled") !== null;
+    var ariaDisabled = (el.getAttribute("aria-disabled") || "").toLowerCase() === "true";
+    return !disabledAttr && !ariaDisabled;
+  };
+
+  // -----------------------------------------------------------------------
+  // 4. inferRole
+  // -----------------------------------------------------------------------
+  pi.inferRole = function inferRole(el) {
+    var explicit = (el.getAttribute("role") || "").trim();
+    if (explicit) return explicit;
+    var tag = el.tagName.toLowerCase();
+    if (tag === "a" && el.getAttribute("href")) return "link";
+    if (tag === "button") return "button";
+    if (tag === "select") return "combobox";
+    if (tag === "textarea") return "textbox";
+    if (tag === "input") {
+      var type = (el.getAttribute("type") || "text").toLowerCase();
+      if (["button", "submit", "reset"].indexOf(type) !== -1) return "button";
+      if (type === "checkbox") return "checkbox";
+      if (type === "radio") return "radio";
+      if (type === "search") return "searchbox";
+      return "textbox";
+    }
+    return "";
+  };
+
+  // -----------------------------------------------------------------------
+  // 5. accessibleName
+  // -----------------------------------------------------------------------
+  pi.accessibleName = function accessibleName(el) {
+    var ariaLabel = el.getAttribute("aria-label");
+    if (ariaLabel && ariaLabel.trim()) return ariaLabel.trim();
+    var labelledBy = el.getAttribute("aria-labelledby");
+    if (labelledBy && labelledBy.trim()) {
+      var text = labelledBy.trim().split(/\\s+/).map(function(id) {
+        var ref = document.getElementById(id);
+        return ref ? (ref.textContent || "").trim() : "";
+      }).join(" ").trim();
+      if (text) return text;
+    }
+    var placeholder = el.getAttribute("placeholder");
+    if (placeholder && placeholder.trim()) return placeholder.trim();
+    var alt = el.getAttribute("alt");
+    if (alt && alt.trim()) return alt.trim();
+    var value = el.value;
+    if (value && typeof value === "string" && value.trim()) return value.trim().slice(0, 80);
+    return (el.textContent || "").trim().replace(/\\s+/g, " ").slice(0, 80);
+  };
+
+  // -----------------------------------------------------------------------
+  // 6. isInteractiveEl
+  // -----------------------------------------------------------------------
+  var interactiveRoles = {
+    button: 1, link: 1, textbox: 1, searchbox: 1, combobox: 1,
+    checkbox: 1, radio: 1, "switch": 1, menuitem: 1,
+    menuitemcheckbox: 1, menuitemradio: 1, tab: 1, option: 1,
+    slider: 1, spinbutton: 1
+  };
+  pi.isInteractiveEl = function isInteractiveEl(el) {
+    var tag = el.tagName.toLowerCase();
+    var role = pi.inferRole(el);
+    if (["button", "input", "select", "textarea", "summary", "option"].indexOf(tag) !== -1) return true;
+    if (tag === "a" && !!el.getAttribute("href")) return true;
+    if (interactiveRoles[role]) return true;
+    if (el.tabIndex >= 0) return true;
+    if (el.isContentEditable) return true;
+    return false;
+  };
+
+  // -----------------------------------------------------------------------
+  // 7. cssPath
+  // -----------------------------------------------------------------------
+  pi.cssPath = function cssPath(el) {
+    if (el.id) return "#" + CSS.escape(el.id);
+    var parts = [];
+    var current = el;
+    while (current && current.nodeType === Node.ELEMENT_NODE && current !== document.body) {
+      var tag = current.tagName.toLowerCase();
+      var part = tag;
+      var parent = current.parentElement;
+      if (parent) {
+        var siblings = Array.from(parent.children).filter(function(c) {
+          return c.tagName === current.tagName;
+        });
+        if (siblings.length > 1) {
+          var idx = siblings.indexOf(current) + 1;
+          part += ":nth-of-type(" + idx + ")";
+        }
+      }
+      parts.unshift(part);
+      current = current.parentElement;
+    }
+    return "body > " + parts.join(" > ");
+  };
+
+  // -----------------------------------------------------------------------
+  // 8. domPath
+  // -----------------------------------------------------------------------
+  pi.domPath = function domPath(el) {
+    var path = [];
+    var current = el;
+    while (current && current !== document.documentElement) {
+      var parent = current.parentElement;
+      if (!parent) break;
+      var idx = Array.from(parent.children).indexOf(current);
+      path.unshift(idx);
+      current = parent;
+    }
+    return path;
+  };
+
+  // -----------------------------------------------------------------------
+  // 9. selectorHints
+  // -----------------------------------------------------------------------
+  pi.selectorHints = function selectorHints(el) {
+    var hints = [];
+    if (el.id) hints.push("#" + CSS.escape(el.id));
+    var nameAttr = el.getAttribute("name");
+    if (nameAttr) hints.push(el.tagName.toLowerCase() + '[name="' + CSS.escape(nameAttr) + '"]');
+    var aria = el.getAttribute("aria-label");
+    if (aria) hints.push(el.tagName.toLowerCase() + '[aria-label="' + CSS.escape(aria) + '"]');
+    var placeholder = el.getAttribute("placeholder");
+    if (placeholder) hints.push(el.tagName.toLowerCase() + '[placeholder="' + CSS.escape(placeholder) + '"]');
+    var cls = Array.from(el.classList).slice(0, 2);
+    if (cls.length > 0) hints.push(el.tagName.toLowerCase() + "." + cls.map(function(c) { return CSS.escape(c); }).join("."));
+    hints.push(pi.cssPath(el));
+    var seen = {};
+    var unique = [];
+    for (var i = 0; i < hints.length; i++) {
+      if (!seen[hints[i]]) {
+        seen[hints[i]] = true;
+        unique.push(hints[i]);
+      }
+    }
+    return unique.slice(0, 6);
+  };
+})();`;

package/dist/resources/extensions/browser-tools/index.ts

@@ -0,0 +1,51 @@
+/** browser-tools — pi extension: full browser interaction via Playwright. */
+import type { ExtensionAPI } from "@gsd/pi-coding-agent";
+import type { ToolDeps } from "./state.js";
+import { ensureBrowser, closeBrowser, getActivePage, getActiveTarget, getActivePageOrNull, attachPageListeners } from "./lifecycle.js";
+import { captureCompactPageState, postActionSummary, constrainScreenshot, captureErrorScreenshot } from "./capture.js";
+import { settleAfterActionAdaptive, ensureMutationCounter } from "./settle.js";
+import { buildRefSnapshot, resolveRefTarget } from "./refs.js";
+import * as u from "./utils.js";
+import { registerNavigationTools } from "./tools/navigation.js";
+import { registerScreenshotTools } from "./tools/screenshot.js";
+import { registerInteractionTools } from "./tools/interaction.js";
+import { registerInspectionTools } from "./tools/inspection.js";
+import { registerSessionTools } from "./tools/session.js";
+import { registerAssertionTools } from "./tools/assertions.js";
+import { registerRefTools } from "./tools/refs.js";
+import { registerWaitTools } from "./tools/wait.js";
+import { registerPageTools } from "./tools/pages.js";
+import { registerFormTools } from "./tools/forms.js";
+import { registerIntentTools } from "./tools/intent.js";
+
+export default function (pi: ExtensionAPI) {
+	pi.on("session_shutdown", async () => { await closeBrowser(); });
+	const deps: ToolDeps = {
+		ensureBrowser, closeBrowser, getActivePage, getActiveTarget, getActivePageOrNull, attachPageListeners,
+		captureCompactPageState, postActionSummary, constrainScreenshot, captureErrorScreenshot,
+		formatCompactStateSummary: u.formatCompactStateSummary, getRecentErrors: u.getRecentErrors,
+		settleAfterActionAdaptive, ensureMutationCounter, buildRefSnapshot, resolveRefTarget,
+		parseRef: u.parseRef, formatVersionedRef: u.formatVersionedRef, staleRefGuidance: u.staleRefGuidance,
+		beginTrackedAction: u.beginTrackedAction, finishTrackedAction: u.finishTrackedAction,
+		truncateText: u.truncateText, verificationFromChecks: u.verificationFromChecks,
+		verificationLine: u.verificationLine, collectAssertionState: (p, checks, target?) => u.collectAssertionState(p, checks, captureCompactPageState, target),
+		formatAssertionText: u.formatAssertionText, formatDiffText: u.formatDiffText,
+		getUrlHash: u.getUrlHash,
+		captureClickTargetState: u.captureClickTargetState, readInputLikeValue: u.readInputLikeValue,
+		firstErrorLine: u.firstErrorLine, captureAccessibilityMarkdown: (selector?) => u.captureAccessibilityMarkdown(getActiveTarget(), selector),
+		resolveAccessibilityScope: u.resolveAccessibilityScope,
+		getLivePagesSnapshot: u.createGetLivePagesSnapshot(ensureBrowser),
+		getSinceTimestamp: u.getSinceTimestamp, getConsoleEntriesSince: u.getConsoleEntriesSince,
+		getNetworkEntriesSince: u.getNetworkEntriesSince, writeArtifactFile: u.writeArtifactFile,
+		copyArtifactFile: u.copyArtifactFile, ensureSessionArtifactDir: u.ensureSessionArtifactDir,
+		buildSessionArtifactPath: u.buildSessionArtifactPath, getSessionArtifactMetadata: u.getSessionArtifactMetadata,
+		sanitizeArtifactName: u.sanitizeArtifactName, formatArtifactTimestamp: u.formatArtifactTimestamp,
+	};
+	registerNavigationTools(pi, deps);  registerScreenshotTools(pi, deps);
+	registerInteractionTools(pi, deps); registerInspectionTools(pi, deps);
+	registerSessionTools(pi, deps);     registerAssertionTools(pi, deps);
+	registerRefTools(pi, deps);         registerWaitTools(pi, deps);
+	registerPageTools(pi, deps);
+	registerFormTools(pi, deps);
+	registerIntentTools(pi, deps);
+}

package/dist/resources/extensions/browser-tools/lifecycle.ts

@@ -0,0 +1,270 @@
+/**
+ * browser-tools — browser lifecycle management
+ *
+ * Manages the shared Browser + BrowserContext + Page singleton.
+ * Injects EVALUATE_HELPERS_SOURCE via context.addInitScript() so that
+ * page.evaluate() callbacks can reference window.__pi.* utilities.
+ */
+
+import type { Browser, BrowserContext, Frame, Page } from "playwright";
+import path from "node:path";
+import {
+	registryAddPage,
+	registryGetActive,
+	registryRemovePage,
+	registrySetActive,
+} from "./core.js";
+import {
+	getBrowser,
+	setBrowser,
+	getContext,
+	setContext,
+	pageRegistry,
+	getActiveFrame,
+	setActiveFrame,
+	logPusher,
+	getConsoleLogs,
+	getNetworkLogs,
+	getDialogLogs,
+	getPendingCriticalRequestsByPage,
+	setHarState,
+	resetAllState,
+	HAR_FILENAME,
+	type ConsoleEntry,
+	type NetworkEntry,
+} from "./state.js";
+import {
+	isCriticalResourceType,
+	updatePendingCriticalRequests,
+	ensureSessionStartedAt,
+	ensureSessionArtifactDir,
+} from "./utils.js";
+import { EVALUATE_HELPERS_SOURCE } from "./evaluate-helpers.js";
+
+// ---------------------------------------------------------------------------
+// Page event wiring
+// ---------------------------------------------------------------------------
+
+/** Attach all event listeners to a page. Called on initial page and new tabs. */
+export function attachPageListeners(p: Page, pageId: number): void {
+	const pendingMap = getPendingCriticalRequestsByPage();
+	pendingMap.set(p, 0);
+
+	const consoleLogs = getConsoleLogs();
+	const networkLogs = getNetworkLogs();
+	const dialogLogs = getDialogLogs();
+
+	// Console messages
+	p.on("console", (msg) => {
+		logPusher(consoleLogs, {
+			type: msg.type(),
+			text: msg.text(),
+			timestamp: Date.now(),
+			url: p.url(),
+			pageId,
+		});
+	});
+
+	// Uncaught JS errors
+	p.on("pageerror", (err) => {
+		logPusher(consoleLogs, {
+			type: "pageerror",
+			text: err.message,
+			timestamp: Date.now(),
+			url: p.url(),
+			pageId,
+		});
+	});
+
+	// Network requests — start/completed/failed
+	p.on("request", (request) => {
+		if (isCriticalResourceType(request.resourceType())) {
+			updatePendingCriticalRequests(p, 1);
+		}
+	});
+
+	p.on("requestfinished", async (request) => {
+		if (isCriticalResourceType(request.resourceType())) {
+			updatePendingCriticalRequests(p, -1);
+		}
+		try {
+			const response = await request.response();
+			const status = response?.status() ?? null;
+			const entry: NetworkEntry = {
+				method: request.method(),
+				url: request.url(),
+				status,
+				resourceType: request.resourceType(),
+				timestamp: Date.now(),
+				failed: false,
+				pageId,
+			};
+			if (response && status !== null && status >= 400) {
+				try {
+					const body = await response.text();
+					entry.responseBody = body.slice(0, 2000);
+				} catch {}
+			}
+			logPusher(networkLogs, entry);
+		} catch {}
+	});
+
+	p.on("requestfailed", (request) => {
+		if (isCriticalResourceType(request.resourceType())) {
+			updatePendingCriticalRequests(p, -1);
+		}
+		logPusher(networkLogs, {
+			method: request.method(),
+			url: request.url(),
+			status: null,
+			resourceType: request.resourceType(),
+			timestamp: Date.now(),
+			failed: true,
+			failureText: request.failure()?.errorText ?? "Unknown failure",
+			pageId,
+		});
+	});
+
+	// Auto-handle JS dialogs (alert, confirm, prompt, beforeunload)
+	p.on("dialog", async (dialog) => {
+		logPusher(dialogLogs, {
+			type: dialog.type(),
+			message: dialog.message(),
+			timestamp: Date.now(),
+			url: p.url(),
+			defaultValue: dialog.defaultValue() || undefined,
+			accepted: true,
+			pageId,
+		});
+		// Auto-accept all dialogs to prevent page freezes
+		await dialog.accept().catch(() => {});
+	});
+
+	// Frame detach handler — clears activeFrame if the selected frame detaches
+	p.on("framedetached", (frame) => {
+		if (getActiveFrame() === frame) setActiveFrame(null);
+	});
+
+	// Page close handler — removes page from registry and handles active fallback
+	p.on("close", () => {
+		try {
+			registryRemovePage(pageRegistry, pageId);
+		} catch {
+			// Page already removed (e.g. during closeBrowser)
+		}
+	});
+}
+
+// ---------------------------------------------------------------------------
+// Browser lifecycle
+// ---------------------------------------------------------------------------
+
+export async function ensureBrowser(): Promise<{ browser: Browser; context: BrowserContext; page: Page }> {
+	const existingBrowser = getBrowser();
+	const existingContext = getContext();
+	if (existingBrowser && existingContext) {
+		return { browser: existingBrowser, context: existingContext, page: getActivePage() };
+	}
+
+	const startedAt = ensureSessionStartedAt();
+	const artifactDir = await ensureSessionArtifactDir();
+	const sessionHarPath = path.join(artifactDir, HAR_FILENAME);
+	setHarState({
+		enabled: true,
+		configuredAtContextCreation: true,
+		path: sessionHarPath,
+		exportCount: 0,
+		lastExportedPath: null,
+		lastExportedAt: null,
+	});
+
+	// Lazy import so playwright is only loaded when actually needed
+	const { chromium } = await import("playwright");
+
+	// Auto-detect headless environments: Linux without $DISPLAY has no GUI.
+	// All browser tool operations (navigation, screenshots, DOM) work in headless mode.
+	const needsHeadless = process.platform === "linux" && !process.env.DISPLAY;
+	const launchOptions: Record<string, unknown> = {
+		headless: needsHeadless || process.env.FORCE_HEADLESS === "true",
+	};
+	const customPath = process.env.BROWSER_PATH;
+	if (customPath) launchOptions.executablePath = customPath;
+	const browser = await chromium.launch(launchOptions);
+	const context = await browser.newContext({
+		deviceScaleFactor: 2,
+		viewport: { width: 1280, height: 800 },
+		recordHar: {
+			path: sessionHarPath,
+			mode: "minimal",
+			content: "omit",
+		},
+	});
+
+	// Inject shared browser-side utilities into every new page/frame
+	await context.addInitScript(EVALUATE_HELPERS_SOURCE);
+
+	setBrowser(browser);
+	setContext(context);
+
+	const initialPage = await context.newPage();
+	const pageEntry = registryAddPage(pageRegistry, {
+		page: initialPage,
+		title: await initialPage.title().catch(() => ""),
+		url: initialPage.url(),
+		opener: null,
+	});
+	registrySetActive(pageRegistry, pageEntry.id);
+	attachPageListeners(initialPage, pageEntry.id);
+
+	// Register new pages (popups, target="_blank", window.open) but do NOT auto-switch
+	context.on("page", (newPage) => {
+		// Determine opener page ID — find which registry page opened this one
+		const openerPage = newPage.opener();
+		let openerId: number | null = null;
+		if (openerPage) {
+			const openerEntry = pageRegistry.pages.find((e: any) => e.page === openerPage);
+			if (openerEntry) openerId = openerEntry.id;
+		}
+		const entry = registryAddPage(pageRegistry, {
+			page: newPage,
+			title: "",
+			url: newPage.url(),
+			opener: openerId,
+		});
+		attachPageListeners(newPage, entry.id);
+		// Update title once loaded
+		newPage.waitForLoadState("domcontentloaded", { timeout: 5000 })
+			.then(() => newPage.title())
+			.then((title) => { entry.title = title; })
+			.catch(() => {});
+	});
+
+	return { browser, context, page: getActivePage() };
+}
+
+/** Get the currently active page from the registry. */
+export function getActivePage(): Page {
+	return registryGetActive(pageRegistry).page;
+}
+
+/** Get the active target — returns the selected frame if one is active, otherwise the active page. */
+export function getActiveTarget(): Page | Frame {
+	return getActiveFrame() ?? getActivePage();
+}
+
+/** Safe accessor for error handling — returns the active page or null if unavailable. */
+export function getActivePageOrNull(): Page | null {
+	try {
+		return getActivePage();
+	} catch {
+		return null;
+	}
+}
+
+export async function closeBrowser(): Promise<void> {
+	const browser = getBrowser();
+	if (browser) {
+		await browser.close().catch(() => {});
+	}
+	resetAllState();
+}

package/dist/resources/extensions/browser-tools/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "pi-browser-tools",
+  "private": true,
+  "version": "1.0.0",
+  "type": "module",
+  "scripts": {
+    "test": "node --test tests/*.test.mjs"
+  },
+  "pi": {
+    "extensions": ["./index.ts"]
+  },
+  "peerDependencies": {
+    "playwright": ">=1.40.0",
+    "sharp": ">=0.33.0"
+  },
+  "peerDependenciesMeta": {
+    "playwright": {
+      "optional": true
+    },
+    "sharp": {
+      "optional": true
+    }
+  }
+}