gsd-pi 2.7.1 → 2.8.0

package/packages/pi-coding-agent/src/modes/interactive/interactive-mode.ts

@@ -176,7 +176,7 @@ export class InteractiveMode {
 	private pendingTools = new Map<string, ToolExecutionComponent>();
 
 	// Tool output expansion state
-	private toolOutputExpanded = false;
+	private toolOutputExpanded = true;
 
 	// Thinking block visibility state
 	private hideThinkingBlock = false;

package/src/resources/extensions/browser-tools/capture.ts

@@ -0,0 +1,165 @@
+/**
+ * browser-tools — page state capture
+ *
+ * Functions for capturing compact page state, screenshots, and summaries.
+ * Used by tool implementations for post-action feedback.
+ */
+
+import type { Frame, Page } from "playwright";
+import sharp from "sharp";
+import type { CompactPageState, CompactSelectorState } from "./state.js";
+import { formatCompactStateSummary } from "./utils.js";
+
+// Anthropic API rejects images > 2000px in multi-image requests.
+// Cap at 1568px (recommended optimal size) to stay well within limits.
+const MAX_SCREENSHOT_DIM = 1568;
+
+// ---------------------------------------------------------------------------
+// Compact page state capture
+// ---------------------------------------------------------------------------
+
+export async function captureCompactPageState(
+	p: Page,
+	options: { selectors?: string[]; includeBodyText?: boolean; target?: Page | Frame } = {},
+): Promise<CompactPageState> {
+	const selectors = Array.from(new Set((options.selectors ?? []).filter(Boolean)));
+	const target = options.target ?? p;
+	const domState = await target.evaluate(({ selectors, includeBodyText }) => {
+		const selectorStates: Record<string, {
+			exists: boolean;
+			visible: boolean;
+			value: string;
+			checked: boolean | null;
+			text: string;
+		}> = {};
+		for (const selector of selectors) {
+			let el: Element | null = null;
+			try {
+				el = document.querySelector(selector);
+			} catch {
+				el = null;
+			}
+			if (!el) {
+				selectorStates[selector] = {
+					exists: false,
+					visible: false,
+					value: "",
+					checked: null,
+					text: "",
+				};
+				continue;
+			}
+			const htmlEl = el as HTMLElement;
+			const style = window.getComputedStyle(htmlEl);
+			const rect = htmlEl.getBoundingClientRect();
+			const visible = style.display !== "none" && style.visibility !== "hidden" && rect.width > 0 && rect.height > 0;
+			const input = el as HTMLInputElement;
+			selectorStates[selector] = {
+				exists: true,
+				visible,
+				value:
+					el instanceof HTMLInputElement ||
+					el instanceof HTMLTextAreaElement ||
+					el instanceof HTMLSelectElement
+						? el.value
+						: htmlEl.getAttribute("value") || "",
+				checked: el instanceof HTMLInputElement && ["checkbox", "radio"].includes(input.type) ? input.checked : null,
+				text: (htmlEl.innerText || htmlEl.textContent || "").trim().replace(/\s+/g, " ").slice(0, 160),
+			};
+		}
+
+		const focused = document.activeElement as HTMLElement | null;
+		const focusedDesc = focused && focused !== document.body && focused !== document.documentElement
+			? `${focused.tagName.toLowerCase()}${focused.id ? '#' + focused.id : ''}${focused.getAttribute('aria-label') ? ' "' + focused.getAttribute('aria-label') + '"' : ''}`
+			: "";
+		const headings = Array.from(document.querySelectorAll('h1,h2,h3')).slice(0, 5).map((h) => (h.textContent || '').trim().replace(/\s+/g, ' ').slice(0, 80));
+		const dialog = document.querySelector('[role="dialog"]:not([hidden]),dialog[open]');
+		const dialogTitle = dialog?.querySelector('[role="heading"],[aria-label]')?.textContent?.trim().slice(0, 80) ?? "";
+		const bodyText = includeBodyText
+			? (document.body?.innerText || document.body?.textContent || "").trim().replace(/\s+/g, ' ').slice(0, 4000)
+			: "";
+		return {
+			url: window.location.href,
+			title: document.title,
+			focus: focusedDesc,
+			headings,
+			bodyText,
+			counts: {
+				landmarks: document.querySelectorAll('[role="main"],[role="banner"],[role="navigation"],[role="contentinfo"],[role="complementary"],[role="search"],[role="form"],[role="dialog"],[role="alert"],main,header,nav,footer,aside,section,form,dialog').length,
+				buttons: document.querySelectorAll('button,[role="button"]').length,
+				links: document.querySelectorAll('a[href]').length,
+				inputs: document.querySelectorAll('input,textarea,select').length,
+			},
+			dialog: {
+				count: document.querySelectorAll('[role="dialog"]:not([hidden]),dialog[open]').length,
+				title: dialogTitle,
+			},
+			selectorStates,
+		};
+	}, { selectors, includeBodyText: options.includeBodyText === true });
+	// URL and title always come from the Page, not the frame
+	return { ...domState, url: p.url(), title: await p.title() };
+}
+
+// ---------------------------------------------------------------------------
+// Post-action summary
+// ---------------------------------------------------------------------------
+
+/** Lightweight page summary after an action. Returns ~50-150 tokens instead of full tree. */
+export async function postActionSummary(p: Page, target?: Page | Frame): Promise<string> {
+	try {
+		const state = await captureCompactPageState(p, { target });
+		return formatCompactStateSummary(state);
+	} catch {
+		return "[summary unavailable]";
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Screenshot helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * If either dimension of the image buffer exceeds MAX_SCREENSHOT_DIM,
+ * downscale proportionally using sharp. Returns the original buffer
+ * unchanged if already within limits.
+ *
+ * `page` parameter is retained for ToolDeps signature stability (D008)
+ * but is no longer used — all processing is server-side via sharp.
+ */
+export async function constrainScreenshot(
+	_page: Page,
+	buffer: Buffer,
+	mimeType: string,
+	quality: number,
+): Promise<Buffer> {
+	const { width, height } = await sharp(buffer).metadata();
+
+	if (
+		width !== undefined &&
+		height !== undefined &&
+		width <= MAX_SCREENSHOT_DIM &&
+		height <= MAX_SCREENSHOT_DIM
+	) {
+		return buffer;
+	}
+
+	const resizer = sharp(buffer).resize(MAX_SCREENSHOT_DIM, MAX_SCREENSHOT_DIM, { fit: "inside" });
+
+	if (mimeType === "image/png") {
+		return Buffer.from(await resizer.png().toBuffer());
+	}
+	return Buffer.from(await resizer.jpeg({ quality }).toBuffer());
+}
+
+/** Capture a JPEG screenshot for error debugging. Returns base64 or null. */
+export async function captureErrorScreenshot(p: Page | null): Promise<{ data: string; mimeType: string } | null> {
+	if (!p) return null;
+	try {
+		let buf = await p.screenshot({ type: "jpeg", quality: 60, scale: "css" });
+		buf = await constrainScreenshot(p, buf, "image/jpeg", 60);
+		return { data: buf.toString("base64"), mimeType: "image/jpeg" };
+	} catch {
+		return null;
+	}
+}

package/src/resources/extensions/browser-tools/evaluate-helpers.ts

@@ -0,0 +1,184 @@
+/**
+ * browser-tools — browser-side evaluate helpers
+ *
+ * Exports a single string constant `EVALUATE_HELPERS_SOURCE` containing an IIFE
+ * that attaches utility functions to `window.__pi`.  This is injected into every
+ * new BrowserContext via `context.addInitScript()` so that `page.evaluate()`
+ * callbacks can reference `window.__pi.cssPath(el)` etc. instead of redeclaring
+ * the same functions inline.
+ *
+ * The `simpleHash` function uses the djb2 algorithm identical to
+ * `computeContentHash` / `computeStructuralSignature` in `core.js`.
+ *
+ * Functions provided (9):
+ *   cssPath, simpleHash, isVisible, isEnabled, inferRole,
+ *   accessibleName, isInteractiveEl, domPath, selectorHints
+ */
+
+export const EVALUATE_HELPERS_SOURCE = `(function() {
+  var pi = window.__pi = window.__pi || {};
+
+  // -----------------------------------------------------------------------
+  // 1. simpleHash — djb2 hash matching core.js computeContentHash
+  // -----------------------------------------------------------------------
+  pi.simpleHash = function simpleHash(str) {
+    if (!str) return "0";
+    var h = 5381;
+    for (var i = 0; i < str.length; i++) {
+      h = ((h << 5) - h + str.charCodeAt(i)) | 0;
+    }
+    return (h >>> 0).toString(16);
+  };
+
+  // -----------------------------------------------------------------------
+  // 2. isVisible
+  // -----------------------------------------------------------------------
+  pi.isVisible = function isVisible(el) {
+    var style = window.getComputedStyle(el);
+    if (style.display === "none" || style.visibility === "hidden") return false;
+    var rect = el.getBoundingClientRect();
+    return rect.width > 0 && rect.height > 0;
+  };
+
+  // -----------------------------------------------------------------------
+  // 3. isEnabled
+  // -----------------------------------------------------------------------
+  pi.isEnabled = function isEnabled(el) {
+    var disabledAttr = el.getAttribute("disabled") !== null;
+    var ariaDisabled = (el.getAttribute("aria-disabled") || "").toLowerCase() === "true";
+    return !disabledAttr && !ariaDisabled;
+  };
+
+  // -----------------------------------------------------------------------
+  // 4. inferRole
+  // -----------------------------------------------------------------------
+  pi.inferRole = function inferRole(el) {
+    var explicit = (el.getAttribute("role") || "").trim();
+    if (explicit) return explicit;
+    var tag = el.tagName.toLowerCase();
+    if (tag === "a" && el.getAttribute("href")) return "link";
+    if (tag === "button") return "button";
+    if (tag === "select") return "combobox";
+    if (tag === "textarea") return "textbox";
+    if (tag === "input") {
+      var type = (el.getAttribute("type") || "text").toLowerCase();
+      if (["button", "submit", "reset"].indexOf(type) !== -1) return "button";
+      if (type === "checkbox") return "checkbox";
+      if (type === "radio") return "radio";
+      if (type === "search") return "searchbox";
+      return "textbox";
+    }
+    return "";
+  };
+
+  // -----------------------------------------------------------------------
+  // 5. accessibleName
+  // -----------------------------------------------------------------------
+  pi.accessibleName = function accessibleName(el) {
+    var ariaLabel = el.getAttribute("aria-label");
+    if (ariaLabel && ariaLabel.trim()) return ariaLabel.trim();
+    var labelledBy = el.getAttribute("aria-labelledby");
+    if (labelledBy && labelledBy.trim()) {
+      var text = labelledBy.trim().split(/\\s+/).map(function(id) {
+        var ref = document.getElementById(id);
+        return ref ? (ref.textContent || "").trim() : "";
+      }).join(" ").trim();
+      if (text) return text;
+    }
+    var placeholder = el.getAttribute("placeholder");
+    if (placeholder && placeholder.trim()) return placeholder.trim();
+    var alt = el.getAttribute("alt");
+    if (alt && alt.trim()) return alt.trim();
+    var value = el.value;
+    if (value && typeof value === "string" && value.trim()) return value.trim().slice(0, 80);
+    return (el.textContent || "").trim().replace(/\\s+/g, " ").slice(0, 80);
+  };
+
+  // -----------------------------------------------------------------------
+  // 6. isInteractiveEl
+  // -----------------------------------------------------------------------
+  var interactiveRoles = {
+    button: 1, link: 1, textbox: 1, searchbox: 1, combobox: 1,
+    checkbox: 1, radio: 1, "switch": 1, menuitem: 1,
+    menuitemcheckbox: 1, menuitemradio: 1, tab: 1, option: 1,
+    slider: 1, spinbutton: 1
+  };
+  pi.isInteractiveEl = function isInteractiveEl(el) {
+    var tag = el.tagName.toLowerCase();
+    var role = pi.inferRole(el);
+    if (["button", "input", "select", "textarea", "summary", "option"].indexOf(tag) !== -1) return true;
+    if (tag === "a" && !!el.getAttribute("href")) return true;
+    if (interactiveRoles[role]) return true;
+    if (el.tabIndex >= 0) return true;
+    if (el.isContentEditable) return true;
+    return false;
+  };
+
+  // -----------------------------------------------------------------------
+  // 7. cssPath
+  // -----------------------------------------------------------------------
+  pi.cssPath = function cssPath(el) {
+    if (el.id) return "#" + CSS.escape(el.id);
+    var parts = [];
+    var current = el;
+    while (current && current.nodeType === Node.ELEMENT_NODE && current !== document.body) {
+      var tag = current.tagName.toLowerCase();
+      var part = tag;
+      var parent = current.parentElement;
+      if (parent) {
+        var siblings = Array.from(parent.children).filter(function(c) {
+          return c.tagName === current.tagName;
+        });
+        if (siblings.length > 1) {
+          var idx = siblings.indexOf(current) + 1;
+          part += ":nth-of-type(" + idx + ")";
+        }
+      }
+      parts.unshift(part);
+      current = current.parentElement;
+    }
+    return "body > " + parts.join(" > ");
+  };
+
+  // -----------------------------------------------------------------------
+  // 8. domPath
+  // -----------------------------------------------------------------------
+  pi.domPath = function domPath(el) {
+    var path = [];
+    var current = el;
+    while (current && current !== document.documentElement) {
+      var parent = current.parentElement;
+      if (!parent) break;
+      var idx = Array.from(parent.children).indexOf(current);
+      path.unshift(idx);
+      current = parent;
+    }
+    return path;
+  };
+
+  // -----------------------------------------------------------------------
+  // 9. selectorHints
+  // -----------------------------------------------------------------------
+  pi.selectorHints = function selectorHints(el) {
+    var hints = [];
+    if (el.id) hints.push("#" + CSS.escape(el.id));
+    var nameAttr = el.getAttribute("name");
+    if (nameAttr) hints.push(el.tagName.toLowerCase() + '[name="' + CSS.escape(nameAttr) + '"]');
+    var aria = el.getAttribute("aria-label");
+    if (aria) hints.push(el.tagName.toLowerCase() + '[aria-label="' + CSS.escape(aria) + '"]');
+    var placeholder = el.getAttribute("placeholder");
+    if (placeholder) hints.push(el.tagName.toLowerCase() + '[placeholder="' + CSS.escape(placeholder) + '"]');
+    var cls = Array.from(el.classList).slice(0, 2);
+    if (cls.length > 0) hints.push(el.tagName.toLowerCase() + "." + cls.map(function(c) { return CSS.escape(c); }).join("."));
+    hints.push(pi.cssPath(el));
+    var seen = {};
+    var unique = [];
+    for (var i = 0; i < hints.length; i++) {
+      if (!seen[hints[i]]) {
+        seen[hints[i]] = true;
+        unique.push(hints[i]);
+      }
+    }
+    return unique.slice(0, 6);
+  };
+})();`;