gsd-pi 2.33.1-dev.235d83a → 2.33.1-dev.29a8268

package/dist/resources/extensions/browser-tools/tools/intent.js

@@ -1,556 +0,0 @@
-import { Type } from "@sinclair/typebox";
-import { StringEnum } from "@gsd/pi-ai";
-import { diffCompactStates } from "../core.js";
-import { setLastActionBeforeState, setLastActionAfterState, } from "../state.js";
-// ---------------------------------------------------------------------------
-// Intent definitions
-// ---------------------------------------------------------------------------
-const INTENTS = [
-    "submit_form",
-    "close_dialog",
-    "primary_cta",
-    "search_field",
-    "next_step",
-    "dismiss",
-    "auth_action",
-    "back_navigation",
-];
-// ---------------------------------------------------------------------------
-// Scoring evaluate script — runs entirely in-browser via page.evaluate()
-// ---------------------------------------------------------------------------
-/**
- * Builds a self-contained IIFE string that scores candidate elements for a
- * given intent. Returns top 5 candidates sorted by score descending, each
- * with { score, selector, tag, role, name, text, reason }.
- *
- * Uses window.__pi utilities (injected via addInitScript) for element
- * metadata — no inline redeclarations.
- */
-function buildIntentScoringScript(intent, scope) {
-    const scopeSelector = JSON.stringify(scope ?? null);
-    return `(() => {
-	var pi = window.__pi;
-	if (!pi) return { error: "window.__pi not available — browser helpers not injected" };
-
-	var intentRaw = ${JSON.stringify(intent)};
-	var normalized = intentRaw.toLowerCase().replace(/[\\s_\\-]+/g, "");
-	var scopeSel = ${scopeSelector};
-	var root = scopeSel ? document.querySelector(scopeSel) : document.body;
-	if (!root) return { error: "Scope selector not found: " + scopeSel };
-
-	// --- Shared helpers ---
-	function textOf(el) {
-		return (el.textContent || "").trim().replace(/\\s+/g, " ").slice(0, 120).toLowerCase();
-	}
-
-	function clamp01(v) { return Math.max(0, Math.min(1, v)); }
-
-	function makeCandidate(el, score, reason) {
-		return {
-			score: Math.round(clamp01(score) * 100) / 100,
-			selector: pi.cssPath(el),
-			tag: el.tagName.toLowerCase(),
-			role: pi.inferRole(el) || "",
-			name: pi.accessibleName(el) || "",
-			text: textOf(el).slice(0, 80),
-			reason: reason,
-		};
-	}
-
-	function qsa(sel) { return Array.from(root.querySelectorAll(sel)); }
-
-	function visibleEnabled(el) {
-		return pi.isVisible(el) && pi.isEnabled(el);
-	}
-
-	function textMatches(el, patterns) {
-		var t = textOf(el);
-		var n = (pi.accessibleName(el) || "").toLowerCase();
-		var combined = t + " " + n;
-		for (var i = 0; i < patterns.length; i++) {
-			if (combined.indexOf(patterns[i]) !== -1) return true;
-		}
-		return false;
-	}
-
-	function textMatchStrength(el, patterns) {
-		var t = textOf(el);
-		var n = (pi.accessibleName(el) || "").toLowerCase();
-		var combined = t + " " + n;
-		var count = 0;
-		for (var i = 0; i < patterns.length; i++) {
-			if (combined.indexOf(patterns[i]) !== -1) count++;
-		}
-		return Math.min(count / Math.max(patterns.length, 1), 1);
-	}
-
-	// --- Intent-specific scoring ---
-	var candidates = [];
-
-	if (normalized === "submitform") {
-		var els = qsa('button[type="submit"], input[type="submit"], button:not([type]), button[type="button"]');
-		for (var i = 0; i < els.length; i++) {
-			var el = els[i];
-			if (!visibleEnabled(el)) continue;
-			var d1 = el.type === "submit" || el.getAttribute("type") === "submit" ? 0.35 : 0;
-			var d2 = el.closest("form") ? 0.3 : 0;
-			var d3 = textMatches(el, ["submit", "send", "save", "create", "add", "post", "confirm", "ok", "done", "register", "sign up", "log in"]) ? 0.2 : 0;
-			var d4 = 0.15;
-			var score = d1 + d2 + d3 + d4;
-			var reasons = [];
-			if (d1 > 0) reasons.push("submit-type");
-			if (d2 > 0) reasons.push("inside-form");
-			if (d3 > 0) reasons.push("text-suggests-submit");
-			reasons.push("visible+enabled");
-			candidates.push(makeCandidate(el, score, reasons.join(", ")));
-		}
-	}
-
-	else if (normalized === "closedialog") {
-		var containers = qsa('[role="dialog"], dialog, [aria-modal="true"], [role="alertdialog"]');
-		for (var ci = 0; ci < containers.length; ci++) {
-			var btns = containers[ci].querySelectorAll("button, a, [role='button']");
-			for (var bi = 0; bi < btns.length; bi++) {
-				var el = btns[bi];
-				if (!visibleEnabled(el)) continue;
-				var d1 = textMatches(el, ["close", "cancel", "dismiss", "×", "✕", "x", "got it", "ok", "done"]) ? 0.35 : 0;
-				var ariaLbl = (el.getAttribute("aria-label") || "").toLowerCase();
-				var d2 = (ariaLbl.indexOf("close") !== -1 || ariaLbl.indexOf("dismiss") !== -1) ? 0.25 : 0;
-				var d3 = 0.2;
-				var rect = el.getBoundingClientRect();
-				var parentRect = containers[ci].getBoundingClientRect();
-				var isTopRight = rect.top - parentRect.top < 60 && parentRect.right - rect.right < 60;
-				var d4 = isTopRight ? 0.2 : 0;
-				var score = d1 + d2 + d3 + d4;
-				var reasons = [];
-				if (d1 > 0) reasons.push("text-matches-close");
-				if (d2 > 0) reasons.push("aria-label-close");
-				reasons.push("inside-dialog");
-				if (d4 > 0) reasons.push("top-right-position");
-				candidates.push(makeCandidate(el, score, reasons.join(", ")));
-			}
-		}
-	}
-
-	else if (normalized === "primarycta") {
-		var els = qsa("button, a, [role='button'], input[type='submit'], input[type='button']");
-		for (var i = 0; i < els.length; i++) {
-			var el = els[i];
-			if (!visibleEnabled(el)) continue;
-			var rect = el.getBoundingClientRect();
-			var area = rect.width * rect.height;
-			var d1 = clamp01(area / 12000);
-			var role = pi.inferRole(el);
-			var d2 = role === "button" ? 0.25 : (role === "link" ? 0.1 : 0.15);
-			var isNegative = textMatches(el, ["cancel", "dismiss", "close", "skip", "no thanks", "no, thanks", "maybe later"]);
-			var d3 = isNegative ? 0 : 0.2;
-			var inMain = !!el.closest("main, [role='main'], article, section, .hero, .content");
-			var d4 = inMain ? 0.15 : 0;
-			var score = d1 + d2 + d3 + d4;
-			var reasons = [];
-			reasons.push("size:" + Math.round(area));
-			if (d2 >= 0.25) reasons.push("button-role");
-			if (d3 > 0) reasons.push("non-dismissive");
-			if (d4 > 0) reasons.push("in-main-content");
-			candidates.push(makeCandidate(el, score, reasons.join(", ")));
-		}
-	}
-
-	else if (normalized === "searchfield") {
-		var els = qsa("input, textarea, [role='searchbox'], [role='combobox'], [contenteditable='true']");
-		for (var i = 0; i < els.length; i++) {
-			var el = els[i];
-			if (!pi.isVisible(el)) continue;
-			var type = (el.getAttribute("type") || "text").toLowerCase();
-			if (["hidden", "submit", "button", "reset", "image", "checkbox", "radio", "file"].indexOf(type) !== -1 && el.tagName.toLowerCase() === "input") continue;
-			var d1 = type === "search" || pi.inferRole(el) === "searchbox" ? 0.4 : 0;
-			var ph = (el.getAttribute("placeholder") || "").toLowerCase();
-			var nm = (el.getAttribute("name") || "").toLowerCase();
-			var ariaLbl = (el.getAttribute("aria-label") || "").toLowerCase();
-			var combined = ph + " " + nm + " " + ariaLbl;
-			var d2 = combined.indexOf("search") !== -1 || combined.indexOf("query") !== -1 || combined.indexOf("find") !== -1 ? 0.3 : 0;
-			var d3 = pi.isEnabled(el) ? 0.15 : 0;
-			var inHeader = !!el.closest("header, nav, [role='banner'], [role='navigation'], [role='search']");
-			var d4 = inHeader ? 0.15 : 0;
-			var score = d1 + d2 + d3 + d4;
-			if (score < 0.1) continue;
-			var reasons = [];
-			if (d1 > 0) reasons.push("search-type/role");
-			if (d2 > 0) reasons.push("name/placeholder-match");
-			if (d3 > 0) reasons.push("enabled");
-			if (d4 > 0) reasons.push("in-header/nav");
-			candidates.push(makeCandidate(el, score, reasons.join(", ")));
-		}
-	}
-
-	else if (normalized === "nextstep") {
-		var els = qsa("button, a, [role='button'], input[type='submit'], input[type='button']");
-		var patterns = ["next", "continue", "proceed", "forward", "go", "step"];
-		for (var i = 0; i < els.length; i++) {
-			var el = els[i];
-			if (!visibleEnabled(el)) continue;
-			var d1 = textMatchStrength(el, patterns) * 0.4;
-			if (d1 === 0) continue;
-			var role = pi.inferRole(el);
-			var d2 = role === "button" ? 0.25 : 0.1;
-			var d3 = 0.2;
-			var isDisabled = !pi.isEnabled(el);
-			var d4 = isDisabled ? 0 : 0.15;
-			var score = d1 + d2 + d3 + d4;
-			var reasons = [];
-			reasons.push("text-match");
-			if (d2 >= 0.25) reasons.push("button-role");
-			reasons.push("visible");
-			if (d4 > 0) reasons.push("enabled");
-			candidates.push(makeCandidate(el, score, reasons.join(", ")));
-		}
-	}
-
-	else if (normalized === "dismiss") {
-		var els = qsa("button, a, [role='button'], [role='link']");
-		var patterns = ["close", "cancel", "dismiss", "skip", "no thanks", "no, thanks", "maybe later", "not now", "×", "✕"];
-		for (var i = 0; i < els.length; i++) {
-			var el = els[i];
-			if (!visibleEnabled(el)) continue;
-			var d1 = textMatchStrength(el, patterns) * 0.35;
-			if (d1 === 0) continue;
-			var inOverlay = !!el.closest('[role="dialog"], dialog, [aria-modal="true"], [role="alertdialog"], .modal, .overlay, .popup, .popover, .toast, .banner');
-			var d2 = inOverlay ? 0.3 : 0.05;
-			var rect = el.getBoundingClientRect();
-			var isEdge = rect.top < 80 || rect.right > window.innerWidth - 80;
-			var d3 = isEdge ? 0.15 : 0;
-			var d4 = 0.15;
-			var score = d1 + d2 + d3 + d4;
-			var reasons = [];
-			reasons.push("text-match");
-			if (d2 >= 0.3) reasons.push("inside-overlay");
-			if (d3 > 0) reasons.push("edge-position");
-			reasons.push("visible+enabled");
-			candidates.push(makeCandidate(el, score, reasons.join(", ")));
-		}
-	}
-
-	else if (normalized === "authaction") {
-		var els = qsa("button, a, [role='button'], [role='link'], input[type='submit']");
-		var patterns = ["log in", "login", "sign in", "signin", "sign up", "signup", "register", "create account", "join", "get started"];
-		for (var i = 0; i < els.length; i++) {
-			var el = els[i];
-			if (!visibleEnabled(el)) continue;
-			var d1 = textMatchStrength(el, patterns) * 0.4;
-			if (d1 === 0) continue;
-			var role = pi.inferRole(el);
-			var d2 = (role === "button" || role === "link") ? 0.25 : 0.1;
-			var rect = el.getBoundingClientRect();
-			var inHeader = !!el.closest("header, nav, [role='banner'], [role='navigation']");
-			var isProminent = inHeader || rect.top < 200;
-			var d3 = isProminent ? 0.2 : 0.05;
-			var d4 = 0.15;
-			var score = d1 + d2 + d3 + d4;
-			var reasons = [];
-			reasons.push("text-match");
-			if (d2 >= 0.25) reasons.push("button-or-link");
-			if (d3 >= 0.2) reasons.push("prominent-position");
-			reasons.push("visible+enabled");
-			candidates.push(makeCandidate(el, score, reasons.join(", ")));
-		}
-	}
-
-	else if (normalized === "backnavigation") {
-		var els = qsa("button, a, [role='button'], [role='link']");
-		var patterns = ["back", "previous", "prev", "return", "go back"];
-		for (var i = 0; i < els.length; i++) {
-			var el = els[i];
-			if (!visibleEnabled(el)) continue;
-			var d1 = textMatchStrength(el, patterns) * 0.35;
-			if (d1 === 0) continue;
-			var innerHtml = el.innerHTML.toLowerCase();
-			var hasArrow = innerHtml.indexOf("←") !== -1 || innerHtml.indexOf("&larr") !== -1 || innerHtml.indexOf("arrow") !== -1 || innerHtml.indexOf("chevron-left") !== -1 || innerHtml.indexOf("back") !== -1;
-			var d2 = hasArrow ? 0.25 : 0;
-			var inNav = !!el.closest("header, nav, [role='banner'], [role='navigation'], .breadcrumb, .toolbar");
-			var d3 = inNav ? 0.25 : 0.05;
-			var d4 = 0.15;
-			var score = d1 + d2 + d3 + d4;
-			var reasons = [];
-			reasons.push("text-match");
-			if (d2 > 0) reasons.push("has-back-arrow/icon");
-			if (d3 >= 0.25) reasons.push("in-nav/header");
-			reasons.push("visible+enabled");
-			candidates.push(makeCandidate(el, score, reasons.join(", ")));
-		}
-	}
-
-	else {
-		return { error: "Unknown intent: " + intentRaw + ". Valid: submit_form, close_dialog, primary_cta, search_field, next_step, dismiss, auth_action, back_navigation" };
-	}
-
-	// Sort by score descending, cap at 5
-	candidates.sort(function(a, b) { return b.score - a.score; });
-	candidates = candidates.slice(0, 5);
-
-	return { intent: intentRaw, normalized: normalized, count: candidates.length, candidates: candidates };
-})()`;
-}
-// ---------------------------------------------------------------------------
-// Registration
-// ---------------------------------------------------------------------------
-export function registerIntentTools(pi, deps) {
-    // -----------------------------------------------------------------------
-    // browser_find_best
-    // -----------------------------------------------------------------------
-    pi.registerTool({
-        name: "browser_find_best",
-        label: "Find Best",
-        description: "Find the best-matching element for a semantic intent. Returns up to 5 scored candidates (0-1) ranked by structural position, role, text signals, and visibility. Use this to discover which element the agent should interact with for a given goal — e.g. intent=\"submit_form\" finds submit buttons, intent=\"close_dialog\" finds close/dismiss buttons inside dialogs. Each candidate includes a CSS selector usable with browser_click.",
-        parameters: Type.Object({
-            intent: StringEnum(INTENTS, {
-                description: "Semantic intent: submit_form, close_dialog, primary_cta, search_field, next_step, dismiss, auth_action, back_navigation",
-            }),
-            scope: Type.Optional(Type.String({
-                description: "CSS selector to narrow the search area. If omitted, searches the full page.",
-            })),
-        }),
-        async execute(_toolCallId, params, _signal, _onUpdate, _ctx) {
-            let actionId = null;
-            let beforeState = null;
-            try {
-                const { page: p } = await deps.ensureBrowser();
-                const target = deps.getActiveTarget();
-                beforeState = await deps.captureCompactPageState(p, {
-                    selectors: params.scope ? [params.scope] : [],
-                    includeBodyText: false,
-                    target,
-                });
-                actionId = deps.beginTrackedAction("browser_find_best", params, beforeState.url).id;
-                const script = buildIntentScoringScript(params.intent, params.scope);
-                const result = await target.evaluate(script);
-                if (result.error) {
-                    deps.finishTrackedAction(actionId, {
-                        status: "error",
-                        error: result.error,
-                        beforeState,
-                    });
-                    return {
-                        content: [{ type: "text", text: result.error }],
-                        details: {},
-                        isError: true,
-                    };
-                }
-                const afterState = await deps.captureCompactPageState(p, {
-                    selectors: params.scope ? [params.scope] : [],
-                    includeBodyText: false,
-                    target,
-                });
-                setLastActionBeforeState(beforeState);
-                setLastActionAfterState(afterState);
-                deps.finishTrackedAction(actionId, {
-                    status: "success",
-                    afterUrl: afterState.url,
-                    beforeState,
-                    afterState,
-                });
-                // Format output
-                const lines = [];
-                lines.push(`Intent: ${params.intent} → ${result.count} candidate(s)`);
-                if (params.scope)
-                    lines.push(`Scope: ${params.scope}`);
-                lines.push("");
-                if (result.candidates.length === 0) {
-                    lines.push("No candidates found for this intent on the current page.");
-                }
-                else {
-                    for (let i = 0; i < result.candidates.length; i++) {
-                        const c = result.candidates[i];
-                        lines.push(`${i + 1}. **${c.score}** \`${c.selector}\``);
-                        lines.push(`   ${c.tag}${c.role ? ` [${c.role}]` : ""} — "${c.name || c.text}"`);
-                        lines.push(`   Reason: ${c.reason}`);
-                    }
-                }
-                return {
-                    content: [{ type: "text", text: lines.join("\n") }],
-                    details: { intentResult: result },
-                };
-            }
-            catch (err) {
-                const screenshot = await deps.captureErrorScreenshot((() => { try {
-                    return deps.getActivePage();
-                }
-                catch {
-                    return null;
-                } })());
-                const errMsg = deps.firstErrorLine(err);
-                if (actionId !== null) {
-                    deps.finishTrackedAction(actionId, {
-                        status: "error",
-                        error: errMsg,
-                        beforeState: beforeState ?? undefined,
-                    });
-                }
-                const content = [
-                    { type: "text", text: `browser_find_best failed: ${errMsg}` },
-                ];
-                if (screenshot) {
-                    content.push({ type: "image", data: screenshot.data, mimeType: screenshot.mimeType });
-                }
-                return { content, details: {}, isError: true };
-            }
-        },
-    });
-    // -----------------------------------------------------------------------
-    // browser_act
-    // -----------------------------------------------------------------------
-    pi.registerTool({
-        name: "browser_act",
-        label: "Browser Act",
-        description: "Execute a semantic action in one call. Resolves the top candidate for the given intent (same scoring as browser_find_best), performs the action (click for buttons/links, focus for search fields), settles the page, and returns a before/after diff. Use when you know what you want to accomplish semantically — e.g. intent=\"submit_form\" finds and clicks the submit button, intent=\"close_dialog\" dismisses the dialog.",
-        parameters: Type.Object({
-            intent: StringEnum(INTENTS, {
-                description: "Semantic intent: submit_form, close_dialog, primary_cta, search_field, next_step, dismiss, auth_action, back_navigation",
-            }),
-            scope: Type.Optional(Type.String({
-                description: "CSS selector to narrow the search area. If omitted, searches the full page.",
-            })),
-        }),
-        async execute(_toolCallId, params, _signal, _onUpdate, _ctx) {
-            let actionId = null;
-            let beforeState = null;
-            try {
-                const { page: p } = await deps.ensureBrowser();
-                const target = deps.getActiveTarget();
-                beforeState = await deps.captureCompactPageState(p, {
-                    selectors: params.scope ? [params.scope] : [],
-                    includeBodyText: true,
-                    target,
-                });
-                actionId = deps.beginTrackedAction("browser_act", params, beforeState.url).id;
-                // Score candidates
-                const script = buildIntentScoringScript(params.intent, params.scope);
-                const result = await target.evaluate(script);
-                if (result.error) {
-                    deps.finishTrackedAction(actionId, {
-                        status: "error",
-                        error: result.error,
-                        beforeState,
-                    });
-                    return {
-                        content: [{ type: "text", text: `browser_act failed: ${result.error}` }],
-                        details: {},
-                        isError: true,
-                    };
-                }
-                if (result.candidates.length === 0) {
-                    deps.finishTrackedAction(actionId, {
-                        status: "error",
-                        error: `No candidates found for intent "${params.intent}"`,
-                        beforeState,
-                    });
-                    return {
-                        content: [{
-                                type: "text",
-                                text: `browser_act: No candidates found for intent "${params.intent}" on the current page. The page may not have the expected elements (e.g. no dialog for close_dialog, no form for submit_form).`,
-                            }],
-                        details: { intentResult: result },
-                        isError: true,
-                    };
-                }
-                // Take top candidate and execute action
-                const top = result.candidates[0];
-                const normalizedIntent = params.intent.toLowerCase().replace(/[\s_-]+/g, "");
-                if (normalizedIntent === "searchfield") {
-                    // Focus instead of click for search fields
-                    try {
-                        await target.locator(top.selector).first().focus({ timeout: 5000 });
-                    }
-                    catch {
-                        // Fallback: click to focus
-                        await target.locator(top.selector).first().click({ timeout: 5000 });
-                    }
-                }
-                else {
-                    // Click via Playwright locator (D021)
-                    try {
-                        await target.locator(top.selector).first().click({ timeout: 5000 });
-                    }
-                    catch {
-                        // getByRole fallback from interaction.ts pattern
-                        const nameMatch = top.selector.match(/\[(?:aria-label|name|placeholder)="([^"]+)"\]/i);
-                        const roleName = nameMatch?.[1];
-                        let clicked = false;
-                        for (const role of ["button", "link", "combobox", "textbox"]) {
-                            try {
-                                const loc = roleName
-                                    ? target.getByRole(role, { name: new RegExp(roleName, "i") })
-                                    : target.getByRole(role, { name: new RegExp(top.name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i") });
-                                await loc.first().click({ timeout: 3000 });
-                                clicked = true;
-                                break;
-                            }
-                            catch { /* try next role */ }
-                        }
-                        if (!clicked) {
-                            throw new Error(`Could not click top candidate "${top.selector}" for intent "${params.intent}"`);
-                        }
-                    }
-                }
-                // Settle after action
-                await deps.settleAfterActionAdaptive(p);
-                // Capture after state and diff
-                const afterState = await deps.captureCompactPageState(p, {
-                    selectors: params.scope ? [params.scope] : [],
-                    includeBodyText: true,
-                    target,
-                });
-                const diff = diffCompactStates(beforeState, afterState);
-                const summary = deps.formatCompactStateSummary(afterState);
-                const jsErrors = deps.getRecentErrors(p.url());
-                setLastActionBeforeState(beforeState);
-                setLastActionAfterState(afterState);
-                deps.finishTrackedAction(actionId, {
-                    status: "success",
-                    afterUrl: afterState.url,
-                    diffSummary: diff.summary,
-                    beforeState,
-                    afterState,
-                });
-                // Format output
-                const lines = [];
-                lines.push(`Intent: ${params.intent}`);
-                lines.push(`Action: ${normalizedIntent === "searchfield" ? "focused" : "clicked"} top candidate (score: ${top.score})`);
-                lines.push(`Target: \`${top.selector}\` — "${top.name || top.text}"`);
-                lines.push(`Reason: ${top.reason}`);
-                lines.push("");
-                lines.push(`Diff:\n${deps.formatDiffText(diff)}`);
-                if (jsErrors.trim()) {
-                    lines.push(`\nJS Errors:\n${jsErrors}`);
-                }
-                lines.push(`\nPage summary:\n${summary}`);
-                return {
-                    content: [{ type: "text", text: lines.join("\n") }],
-                    details: { intentResult: result, topCandidate: top, diff },
-                };
-            }
-            catch (err) {
-                const screenshot = await deps.captureErrorScreenshot((() => { try {
-                    return deps.getActivePage();
-                }
-                catch {
-                    return null;
-                } })());
-                const errMsg = deps.firstErrorLine(err);
-                if (actionId !== null) {
-                    deps.finishTrackedAction(actionId, {
-                        status: "error",
-                        error: errMsg,
-                        beforeState: beforeState ?? undefined,
-                    });
-                }
-                const content = [
-                    { type: "text", text: `browser_act failed: ${errMsg}` },
-                ];
-                if (screenshot) {
-                    content.push({ type: "image", data: screenshot.data, mimeType: screenshot.mimeType });
-                }
-                return { content, details: {}, isError: true };
-            }
-        },
-    });
-}