@elizaos/agent 2.0.0-alpha.173 → 2.0.0-alpha.192

package/packages/agent/src/services/browser-workspace-elements.d.ts

@@ -0,0 +1,42 @@
+import type { JSDOM } from "jsdom";
+import type { BrowserWorkspaceCommand, BrowserWorkspaceDomElementSummary, BrowserWorkspaceRuntimeState } from "./browser-workspace-types.js";
+export declare function buildBrowserWorkspaceElementSelector(element: Element): string;
+export declare function createBrowserWorkspaceElementSummary(element: Element): BrowserWorkspaceDomElementSummary;
+export declare function collectBrowserWorkspaceInspectElements(document: Document): BrowserWorkspaceDomElementSummary[];
+export declare function resolveBrowserWorkspaceIframeDocument(runtime: BrowserWorkspaceRuntimeState, frameElement: Element | null, baseUrl: string): Document | null;
+export declare function resolveWebBrowserWorkspaceCommandDocument(tab: {
+    id: string;
+    url: string;
+}, dom: JSDOM, runtimeState: import("./browser-workspace-types.js").BrowserWorkspaceRuntimeState): {
+    document: Document;
+    frameSelector: string | null;
+};
+export declare function getBrowserWorkspaceElementSearchTexts(element: Element): string[];
+export declare function browserWorkspaceTextMatches(candidate: string, wanted: string, exact?: boolean): boolean;
+export declare function isBrowserWorkspaceElementVisible(element: Element): boolean;
+export declare function findBrowserWorkspaceElementByLabel(document: Document, labelText: string, exact?: boolean): Element | null;
+export declare function getBrowserWorkspaceNativeRole(element: Element): string | null;
+export declare function findBrowserWorkspaceElementByRole(document: Document, role: string, name?: string, exact?: boolean): Element | null;
+export declare function trimBrowserWorkspaceQuotedValue(value: string): string;
+export declare function normalizeBrowserWorkspaceSelectorSyntax(selector: string): string;
+export declare function parseBrowserWorkspaceSemanticSelector(selector: string): Pick<BrowserWorkspaceCommand, "findBy" | "name" | "role" | "selector" | "text"> | null;
+export declare function mergeBrowserWorkspaceSelectorCommand(command: BrowserWorkspaceCommand | undefined, selector: string): BrowserWorkspaceCommand | null;
+export declare function queryBrowserWorkspaceSelector(root: Document | Element, selector: string): Element | null;
+export declare function queryAllBrowserWorkspaceSelector(root: Document | Element, selector: string): Element[];
+export declare function findBrowserWorkspaceElementByText(document: Document, needle: string): Element | null;
+export declare function resolveBrowserWorkspaceFindElement(document: Document, command: BrowserWorkspaceCommand): Element | null;
+export declare function resolveBrowserWorkspaceElement(document: Document, selector?: string, text?: string, command?: BrowserWorkspaceCommand): Element | null;
+export declare function getBrowserWorkspaceElementBox(element: Element): {
+    bottom: number;
+    height: number;
+    left: number;
+    right: number;
+    top: number;
+    width: number;
+    x: number;
+    y: number;
+};
+export declare function getBrowserWorkspaceElementValue(element: Element): string | boolean | null;
+export declare function getBrowserWorkspaceElementStyles(element: Element, window: Window): Record<string, string | null>;
+export declare function findClosestBrowserWorkspaceForm(element: Element | null): HTMLFormElement | null;
+//# sourceMappingURL=browser-workspace-elements.d.ts.map

package/packages/agent/src/services/browser-workspace-elements.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-workspace-elements.d.ts","sourceRoot":"","sources":["../../../../../src/services/browser-workspace-elements.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAMnC,OAAO,KAAK,EACV,uBAAuB,EACvB,iCAAiC,EACjC,4BAA4B,EAC7B,MAAM,8BAA8B,CAAC;AAEtC,wBAAgB,oCAAoC,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,CA+C7E;AAED,wBAAgB,oCAAoC,CAClD,OAAO,EAAE,OAAO,GACf,iCAAiC,CAsBnC;AAED,wBAAgB,sCAAsC,CACpD,QAAQ,EAAE,QAAQ,GACjB,iCAAiC,EAAE,CAsBrC;AAED,wBAAgB,qCAAqC,CACnD,OAAO,EAAE,4BAA4B,EACrC,YAAY,EAAE,OAAO,GAAG,IAAI,EAC5B,OAAO,EAAE,MAAM,GACd,QAAQ,GAAG,IAAI,CAiCjB;AAED,wBAAgB,yCAAyC,CACvD,GAAG,EAAE;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EAChC,GAAG,EAAE,KAAK,EACV,YAAY,EAAE,OAAO,8BAA8B,EAAE,4BAA4B,GAChF;IAAE,QAAQ,EAAE,QAAQ,CAAC;IAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAqBtD;AAED,wBAAgB,qCAAqC,CACnD,OAAO,EAAE,OAAO,GACf,MAAM,EAAE,CAuBV;AAED,wBAAgB,2BAA2B,CACzC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,KAAK,UAAQ,GACZ,OAAO,CAUT;AAED,wBAAgB,gCAAgC,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAgB1E;AAED,wBAAgB,kCAAkC,CAChD,QAAQ,EAAE,QAAQ,EAClB,SAAS,EAAE,MAAM,EACjB,KAAK,UAAQ,GACZ,OAAO,GAAG,IAAI,CAuBhB;AAED,wBAAgB,6BAA6B,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAyB7E;AAED,wBAAgB,iCAAiC,CAC/C,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE,MAAM,EACb,KAAK,UAAQ,GACZ,OAAO,GAAG,IAAI,CA0BhB;AAED,wBAAgB,+BAA+B,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAarE;AAED,wBAAgB,uCAAuC,CACrD,QAAQ,EAAE,MAAM,GACf,MAAM,CAWR;AAED,wBAAgB,qCAAqC,CACnD,QAAQ,EAAE,MAAM,GACf,IAAI,CACL,uBAAuB,EACvB,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,GAAG,MAAM,CACjD,GAAG,IAAI,CA8DP;AAED,wBAAgB,oCAAoC,CAClD,OAAO,EAAE,uBAAuB,GAAG,SAAS,EAC5C,QAAQ,EAAE,MAAM,GACf,uBAAuB,GAAG,IAAI,CAWhC;AAED,wBAAgB,6BAA6B,CAC3C,IAAI,EAAE,QAAQ,GAAG,OAAO,EACxB,QAAQ,EAAE,MAAM,GACf,OAAO,GAAG,IAAI,CAMhB;AAED,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,QAAQ,GAAG,OAAO,EACxB,QAAQ,EAAE,MAAM,GACf,OAAO,EAAE,CAMX;AAED,wBAAgB,iCAAiC,CAC/C,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,MAAM,GACb,OAAO,GAAG,IAAI,CAgChB;AAED,wBAAgB,kCAAkC,CAChD,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,uBAAuB,GAC/B,OAAO,GAAG,IAAI,CAuFhB;AAED,wBAAgB,8BAA8B,CAC5C,QAAQ,EAAE,QAAQ,EAClB,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,uBAAuB,GAChC,OAAO,GAAG,IAAI,CA2DhB;AAED,wBAAgB,6BAA6B,CAAC,OAAO,EAAE,OAAO,GAAG;IAC/D,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;CACX,CAwBA;AAED,wBAAgB,+BAA+B,CAC7C,OAAO,EAAE,OAAO,GACf,MAAM,GAAG,OAAO,GAAG,IAAI,CAoBzB;AAED,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,MAAM,GACb,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,CAO/B;AAED,wBAAgB,+BAA+B,CAC7C,OAAO,EAAE,OAAO,GAAG,IAAI,GACtB,eAAe,GAAG,IAAI,CAOxB"}

package/packages/agent/src/services/browser-workspace-elements.js

@@ -0,0 +1,508 @@
+import { buildBrowserWorkspaceCssStringLiteral, normalizeBrowserWorkspaceText, } from "./browser-workspace-helpers.js";
+import { getJSDOMClass } from "./browser-workspace-jsdom.js";
+export function buildBrowserWorkspaceElementSelector(element) {
+    const escapedId = typeof globalThis.CSS
+        ?.escape === "function"
+        ? globalThis.CSS.escape(element.id)
+        : element.id.replace(/[^a-zA-Z0-9_-]/g, "\\$&");
+    if (element.id) {
+        return `#${escapedId}`;
+    }
+    const testId = element.getAttribute("data-testid")?.trim();
+    if (testId) {
+        return `[data-testid=${buildBrowserWorkspaceCssStringLiteral(testId)}]`;
+    }
+    const name = element.getAttribute("name")?.trim();
+    if (name) {
+        return `${element.tagName.toLowerCase()}[name=${buildBrowserWorkspaceCssStringLiteral(name)}]`;
+    }
+    const type = element.getAttribute("type")?.trim();
+    if (type) {
+        return `${element.tagName.toLowerCase()}[type=${buildBrowserWorkspaceCssStringLiteral(type)}]`;
+    }
+    const parent = element.parentElement;
+    if (!parent) {
+        return element.tagName.toLowerCase();
+    }
+    const siblings = parent.children;
+    let index = 1;
+    for (let cursor = 0; cursor < siblings.length; cursor += 1) {
+        const sibling = siblings.item(cursor);
+        if (!sibling || sibling.tagName !== element.tagName) {
+            continue;
+        }
+        if (sibling === element) {
+            break;
+        }
+        index += 1;
+    }
+    return `${element.tagName.toLowerCase()}:nth-of-type(${index})`;
+}
+export function createBrowserWorkspaceElementSummary(element) {
+    const inputLike = element.tagName === "INPUT" ||
+        element.tagName === "TEXTAREA" ||
+        element.tagName === "SELECT";
+    const elementValue = inputLike
+        ? (element
+            .value ?? null)
+        : null;
+    return {
+        selector: buildBrowserWorkspaceElementSelector(element),
+        tag: element.tagName.toLowerCase(),
+        text: normalizeBrowserWorkspaceText(inputLike ? elementValue : element.textContent),
+        type: element.getAttribute("type"),
+        name: element.getAttribute("name"),
+        href: element.getAttribute("href"),
+        value: typeof elementValue === "string" ? elementValue : null,
+    };
+}
+export function collectBrowserWorkspaceInspectElements(document) {
+    const elements = Array.from(document.querySelectorAll("a, button, input, textarea, select, form, [role='button'], [data-testid]"));
+    const summaries = [];
+    const seenSelectors = new Set();
+    for (const element of elements) {
+        const summary = createBrowserWorkspaceElementSummary(element);
+        if (seenSelectors.has(summary.selector)) {
+            continue;
+        }
+        seenSelectors.add(summary.selector);
+        summaries.push(summary);
+        if (summaries.length >= 40) {
+            break;
+        }
+    }
+    return summaries;
+}
+export function resolveBrowserWorkspaceIframeDocument(runtime, frameElement, baseUrl) {
+    if (!frameElement || frameElement.tagName !== "IFRAME") {
+        return null;
+    }
+    const iframe = frameElement;
+    const srcdoc = iframe.getAttribute("srcdoc");
+    if (srcdoc?.trim()) {
+        const selector = buildBrowserWorkspaceElementSelector(frameElement);
+        const cached = runtime.frameDoms.get(selector);
+        if (cached) {
+            return cached.window.document;
+        }
+        if (iframe.contentDocument &&
+            normalizeBrowserWorkspaceText(iframe.contentDocument.body?.textContent)
+                .length > 0) {
+            return iframe.contentDocument;
+        }
+        const parsed = new (getJSDOMClass())(srcdoc, {
+            pretendToBeVisual: true,
+            url: baseUrl,
+        });
+        runtime.frameDoms.set(selector, parsed);
+        return parsed.window.document;
+    }
+    if (iframe.contentDocument) {
+        return iframe.contentDocument;
+    }
+    return null;
+}
+export function resolveWebBrowserWorkspaceCommandDocument(tab, dom, runtimeState) {
+    const state = runtimeState;
+    const frameSelector = state.currentFrame?.trim() || null;
+    if (!frameSelector) {
+        return { document: dom.window.document, frameSelector: null };
+    }
+    const frameElement = resolveBrowserWorkspaceElement(dom.window.document, frameSelector);
+    const frameDocument = resolveBrowserWorkspaceIframeDocument(state, frameElement, tab.url);
+    if (!frameDocument) {
+        return { document: dom.window.document, frameSelector: null };
+    }
+    return { document: frameDocument, frameSelector };
+}
+export function getBrowserWorkspaceElementSearchTexts(element) {
+    const labelText = element.id && element.ownerDocument
+        ? Array.from(element.ownerDocument.querySelectorAll(`label[for="${element.id}"]`))
+            .map((label) => label.textContent)
+            .join(" ")
+        : "";
+    return [
+        element.textContent,
+        element.getAttribute("aria-label"),
+        element.getAttribute("placeholder"),
+        element.getAttribute("title"),
+        element.getAttribute("name"),
+        element.getAttribute("alt"),
+        element.getAttribute("data-testid"),
+        labelText,
+        element
+            .value,
+    ]
+        .map((value) => normalizeBrowserWorkspaceText(value))
+        .filter(Boolean);
+}
+export function browserWorkspaceTextMatches(candidate, wanted, exact = false) {
+    const normalizedCandidate = normalizeBrowserWorkspaceText(candidate).toLowerCase();
+    const normalizedWanted = normalizeBrowserWorkspaceText(wanted).toLowerCase();
+    if (!normalizedCandidate || !normalizedWanted) {
+        return false;
+    }
+    return exact
+        ? normalizedCandidate === normalizedWanted
+        : normalizedCandidate.includes(normalizedWanted);
+}
+export function isBrowserWorkspaceElementVisible(element) {
+    if (element.hasAttribute("hidden") ||
+        element.getAttribute("aria-hidden") === "true") {
+        return false;
+    }
+    const htmlElement = element;
+    const inlineDisplay = htmlElement.style?.display?.trim().toLowerCase();
+    const inlineVisibility = htmlElement.style?.visibility?.trim().toLowerCase();
+    if (inlineDisplay === "none" || inlineVisibility === "hidden") {
+        return false;
+    }
+    return true;
+}
+export function findBrowserWorkspaceElementByLabel(document, labelText, exact = false) {
+    const labels = Array.from(document.querySelectorAll("label"));
+    for (const label of labels) {
+        if (!browserWorkspaceTextMatches(label.textContent ?? "", labelText, exact)) {
+            continue;
+        }
+        const forId = label.getAttribute("for")?.trim();
+        if (forId) {
+            const explicit = document.getElementById(forId);
+            if (explicit) {
+                return explicit;
+            }
+        }
+        const nested = label.querySelector("input, textarea, select, button");
+        if (nested) {
+            return nested;
+        }
+    }
+    return null;
+}
+export function getBrowserWorkspaceNativeRole(element) {
+    const explicitRole = element.getAttribute("role")?.trim().toLowerCase();
+    if (explicitRole) {
+        return explicitRole;
+    }
+    const tag = element.tagName.toLowerCase();
+    if (tag === "a" && element.getAttribute("href"))
+        return "link";
+    if (tag === "button")
+        return "button";
+    if (tag === "select")
+        return "combobox";
+    if (tag === "option")
+        return "option";
+    if (tag === "textarea")
+        return "textbox";
+    if (tag === "form")
+        return "form";
+    if (/^h[1-6]$/.test(tag))
+        return "heading";
+    if (tag === "input") {
+        const input = element;
+        const type = (input.type || "text").toLowerCase();
+        if (type === "checkbox")
+            return "checkbox";
+        if (type === "radio")
+            return "radio";
+        if (["button", "submit", "reset", "image"].includes(type)) {
+            return "button";
+        }
+        return "textbox";
+    }
+    return null;
+}
+export function findBrowserWorkspaceElementByRole(document, role, name, exact = false) {
+    const wantedRole = role.trim().toLowerCase();
+    if (!wantedRole) {
+        return null;
+    }
+    const candidates = Array.from(document.querySelectorAll("a, button, input, textarea, select, option, form, h1, h2, h3, h4, h5, h6, [role], [data-testid]"));
+    for (const candidate of candidates) {
+        if (getBrowserWorkspaceNativeRole(candidate) !== wantedRole) {
+            continue;
+        }
+        if (!name?.trim()) {
+            return candidate;
+        }
+        const haystacks = getBrowserWorkspaceElementSearchTexts(candidate);
+        if (haystacks.some((value) => browserWorkspaceTextMatches(value, name, exact))) {
+            return candidate;
+        }
+    }
+    return null;
+}
+export function trimBrowserWorkspaceQuotedValue(value) {
+    const trimmed = value.trim();
+    const hasTextMatch = trimmed.match(/^has-text\((['"])([\s\S]*?)\1\)$/i);
+    if (hasTextMatch?.[2]) {
+        return hasTextMatch[2].trim();
+    }
+    if ((trimmed.startsWith('"') && trimmed.endsWith('"')) ||
+        (trimmed.startsWith("'") && trimmed.endsWith("'"))) {
+        return trimmed.slice(1, -1).trim();
+    }
+    return trimmed;
+}
+export function normalizeBrowserWorkspaceSelectorSyntax(selector) {
+    let normalized = selector.trim();
+    normalized = normalized.replace(/^role\s*[:=]\s*([a-z0-9_-]+)\s+name\s*[:=]\s*(.+)$/i, "role=$1[name=$2]");
+    normalized = normalized.replace(/^((?:label|text|placeholder|alt|title|testid|data-testid)\s*[:=]\s*(?:has-text\((['"])[\s\S]*?\2\)|"[^"]+"|'[^']+'|[^>]+?))\s+((?:input|textarea|select)[\s\S]*)$/i, "$1 >> $3");
+    return normalized;
+}
+export function parseBrowserWorkspaceSemanticSelector(selector) {
+    const trimmed = normalizeBrowserWorkspaceSelectorSyntax(selector);
+    const match = trimmed.match(/^([a-z-]+)\s*[:=]\s*(.+)$/i);
+    if (!match) {
+        return null;
+    }
+    const kind = match[1]?.trim().toLowerCase();
+    const rawValue = match[2]?.trim() ?? "";
+    if (!kind || !rawValue) {
+        return null;
+    }
+    switch (kind) {
+        case "alt":
+            return { findBy: "alt", text: trimBrowserWorkspaceQuotedValue(rawValue) };
+        case "css":
+            return { selector: trimBrowserWorkspaceQuotedValue(rawValue) };
+        case "data-testid":
+        case "testid":
+            return {
+                findBy: "testid",
+                text: trimBrowserWorkspaceQuotedValue(rawValue),
+            };
+        case "label":
+            return {
+                findBy: "label",
+                text: trimBrowserWorkspaceQuotedValue(rawValue),
+            };
+        case "placeholder":
+            return {
+                findBy: "placeholder",
+                text: trimBrowserWorkspaceQuotedValue(rawValue),
+            };
+        case "role": {
+            const roleMatch = rawValue.match(/^([a-z0-9_-]+)(?:\s*\[\s*name\s*[:=]\s*(.+?)\s*\])?$/i);
+            if (!roleMatch?.[1]) {
+                return null;
+            }
+            return {
+                findBy: "role",
+                name: roleMatch[2]
+                    ? trimBrowserWorkspaceQuotedValue(roleMatch[2])
+                    : undefined,
+                role: roleMatch[1].trim().toLowerCase(),
+            };
+        }
+        case "text":
+            return {
+                findBy: "text",
+                text: trimBrowserWorkspaceQuotedValue(rawValue),
+            };
+        case "title":
+            return {
+                findBy: "title",
+                text: trimBrowserWorkspaceQuotedValue(rawValue),
+            };
+        default:
+            return null;
+    }
+}
+export function mergeBrowserWorkspaceSelectorCommand(command, selector) {
+    const parsed = parseBrowserWorkspaceSemanticSelector(selector);
+    if (!parsed) {
+        return null;
+    }
+    return {
+        ...command,
+        ...parsed,
+        selector: parsed.selector,
+    };
+}
+export function queryBrowserWorkspaceSelector(root, selector) {
+    try {
+        return root.querySelector(selector);
+    }
+    catch {
+        throw new Error(`Invalid selector ${selector}`);
+    }
+}
+export function queryAllBrowserWorkspaceSelector(root, selector) {
+    try {
+        return Array.from(root.querySelectorAll(selector));
+    }
+    catch {
+        throw new Error(`Invalid selector ${selector}`);
+    }
+}
+export function findBrowserWorkspaceElementByText(document, needle) {
+    const wanted = normalizeBrowserWorkspaceText(needle).toLowerCase();
+    if (!wanted) {
+        return null;
+    }
+    const candidates = Array.from(document.querySelectorAll("a, button, input, textarea, select, option, label, h1, h2, h3, [role='button'], [data-testid]"));
+    for (const element of candidates) {
+        const haystacks = [
+            element.textContent,
+            element.getAttribute("aria-label"),
+            element.getAttribute("placeholder"),
+            element.getAttribute("title"),
+            element.getAttribute("name"),
+            element
+                .value,
+        ]
+            .map((value) => normalizeBrowserWorkspaceText(value))
+            .filter(Boolean)
+            .map((value) => value.toLowerCase());
+        if (haystacks.some((value) => value.includes(wanted))) {
+            return element;
+        }
+    }
+    return null;
+}
+export function resolveBrowserWorkspaceFindElement(document, command) {
+    switch (command.findBy) {
+        case "alt":
+            return (Array.from(document.querySelectorAll("[alt]")).find((element) => browserWorkspaceTextMatches(element.getAttribute("alt") ?? "", command.text ?? "", command.exact)) ?? null);
+        case "first":
+            return command.selector?.trim()
+                ? queryBrowserWorkspaceSelector(document, command.selector)
+                : null;
+        case "label":
+            return command.text?.trim()
+                ? findBrowserWorkspaceElementByLabel(document, command.text, command.exact)
+                : null;
+        case "last":
+            return command.selector?.trim()
+                ? (queryAllBrowserWorkspaceSelector(document, command.selector).at(-1) ?? null)
+                : null;
+        case "nth":
+            if (!command.selector?.trim()) {
+                return null;
+            }
+            if (typeof command.index !== "number" ||
+                !Number.isInteger(command.index)) {
+                return null;
+            }
+            return (queryAllBrowserWorkspaceSelector(document, command.selector).at(command.index) ?? null);
+        case "placeholder":
+            return (Array.from(document.querySelectorAll("[placeholder]")).find((element) => browserWorkspaceTextMatches(element.getAttribute("placeholder") ?? "", command.text ?? "", command.exact)) ?? null);
+        case "role":
+            return command.role?.trim()
+                ? findBrowserWorkspaceElementByRole(document, command.role, command.name, command.exact)
+                : null;
+        case "testid":
+            return command.text?.trim()
+                ? document.querySelector(`[data-testid=${buildBrowserWorkspaceCssStringLiteral(command.text)}]`)
+                : null;
+        case "text":
+            return command.text?.trim()
+                ? findBrowserWorkspaceElementByText(document, command.text)
+                : null;
+        case "title":
+            return (Array.from(document.querySelectorAll("[title]")).find((element) => browserWorkspaceTextMatches(element.getAttribute("title") ?? "", command.text ?? "", command.exact)) ?? null);
+        default:
+            return null;
+    }
+}
+export function resolveBrowserWorkspaceElement(document, selector, text, command) {
+    const normalizedSelector = selector
+        ? normalizeBrowserWorkspaceSelectorSyntax(selector)
+        : undefined;
+    if (normalizedSelector) {
+        const selectorChain = normalizedSelector
+            .split(/\s*>>\s*/)
+            .map((segment) => segment.trim())
+            .filter(Boolean);
+        if (selectorChain.length > 1) {
+            let current = resolveBrowserWorkspaceElement(document, selectorChain[0], undefined, command);
+            for (let index = 1; current && index < selectorChain.length; index += 1) {
+                const segment = selectorChain[index];
+                if (!segment) {
+                    continue;
+                }
+                if (typeof current.matches === "function" &&
+                    current.matches(segment)) {
+                    continue;
+                }
+                if (/^(input|textarea|select)(?:\[[^\]]+\])?$/i.test(segment) &&
+                    (current.tagName === "INPUT" ||
+                        current.tagName === "TEXTAREA" ||
+                        current.tagName === "SELECT")) {
+                    continue;
+                }
+                current = queryBrowserWorkspaceSelector(current, segment);
+            }
+            return current;
+        }
+        const semanticCommand = mergeBrowserWorkspaceSelectorCommand(command, normalizedSelector);
+        if (semanticCommand) {
+            return resolveBrowserWorkspaceFindElement(document, semanticCommand);
+        }
+        return queryBrowserWorkspaceSelector(document, normalizedSelector);
+    }
+    if (command?.findBy) {
+        return resolveBrowserWorkspaceFindElement(document, command);
+    }
+    const normalizedText = text?.trim();
+    if (normalizedText) {
+        return findBrowserWorkspaceElementByText(document, normalizedText);
+    }
+    return null;
+}
+export function getBrowserWorkspaceElementBox(element) {
+    const box = typeof element.getBoundingClientRect === "function"
+        ? element.getBoundingClientRect()
+        : {
+            bottom: 0,
+            height: 0,
+            left: 0,
+            right: 0,
+            top: 0,
+            width: 0,
+            x: 0,
+            y: 0,
+        };
+    return {
+        bottom: box.bottom,
+        height: box.height,
+        left: box.left,
+        right: box.right,
+        top: box.top,
+        width: box.width,
+        x: box.x,
+        y: box.y,
+    };
+}
+export function getBrowserWorkspaceElementValue(element) {
+    if (element.tagName === "INPUT" ||
+        element.tagName === "TEXTAREA" ||
+        element.tagName === "SELECT") {
+        const control = element;
+        if (element.tagName === "INPUT") {
+            const input = control;
+            const type = input.type.trim().toLowerCase();
+            if (type === "checkbox" || type === "radio") {
+                return input.checked;
+            }
+        }
+        return control.value;
+    }
+    return null;
+}
+export function getBrowserWorkspaceElementStyles(element, window) {
+    const computed = window.getComputedStyle(element);
+    return {
+        display: computed.display || null,
+        visibility: computed.visibility || null,
+        opacity: computed.opacity || null,
+    };
+}
+export function findClosestBrowserWorkspaceForm(element) {
+    if (!element) {
+        return null;
+    }
+    return (element.tagName === "FORM" ? element : element.closest("form"));
+}

package/packages/agent/src/services/browser-workspace-forms.d.ts

@@ -0,0 +1,19 @@
+import type { JSDOM } from "jsdom";
+import type { BrowserWorkspaceCommandResult, BrowserWorkspaceScrollDirection, WebBrowserWorkspaceTabState } from "./browser-workspace-types.js";
+export declare function ensureBrowserWorkspaceFormControlElement(element: Element, subaction: "clipboard" | "fill" | "keyboardinserttext" | "keyboardtype" | "select" | "type"): HTMLInputElement | HTMLTextAreaElement | HTMLSelectElement;
+export declare function ensureBrowserWorkspaceCheckboxElement(element: Element, subaction: "check" | "uncheck"): HTMLInputElement;
+export declare function setBrowserWorkspaceControlValue(control: HTMLInputElement | HTMLTextAreaElement | HTMLSelectElement, nextValue: string): void;
+export declare function activateWebBrowserWorkspaceElement(tab: WebBrowserWorkspaceTabState, element: Element, subaction: "click" | "dblclick"): Promise<BrowserWorkspaceCommandResult>;
+export declare function scrollWebBrowserWorkspaceTarget(dom: JSDOM, element: Element | null, direction: BrowserWorkspaceScrollDirection, pixels: number): {
+    axis: "x" | "y";
+    selector: string | null;
+    value: number;
+};
+export declare function submitWebBrowserWorkspaceForm(tab: WebBrowserWorkspaceTabState, form: HTMLFormElement): Promise<void>;
+import type { BrowserWorkspaceTab } from "./browser-workspace-types.js";
+export declare function clearWebBrowserWorkspaceTabElementRefs(tabId: string): void;
+export declare function cloneWebBrowserWorkspaceTabState(tab: WebBrowserWorkspaceTabState): BrowserWorkspaceTab;
+export declare function pushWebBrowserWorkspaceHistory(tab: WebBrowserWorkspaceTabState, nextUrl: string): void;
+export declare function loadWebBrowserWorkspaceTabDocument(tab: WebBrowserWorkspaceTabState): Promise<void>;
+export declare function ensureLoadedWebBrowserWorkspaceTabDocument(tab: WebBrowserWorkspaceTabState): Promise<JSDOM>;
+//# sourceMappingURL=browser-workspace-forms.d.ts.map

package/packages/agent/src/services/browser-workspace-forms.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-workspace-forms.d.ts","sourceRoot":"","sources":["../../../../../src/services/browser-workspace-forms.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAsBnC,OAAO,KAAK,EACV,6BAA6B,EAC7B,+BAA+B,EAC/B,2BAA2B,EAC5B,MAAM,8BAA8B,CAAC;AAEtC,wBAAgB,wCAAwC,CACtD,OAAO,EAAE,OAAO,EAChB,SAAS,EACL,WAAW,GACX,MAAM,GACN,oBAAoB,GACpB,cAAc,GACd,QAAQ,GACR,MAAM,GACT,gBAAgB,GAAG,mBAAmB,GAAG,iBAAiB,CAe5D;AAED,wBAAgB,qCAAqC,CACnD,OAAO,EAAE,OAAO,EAChB,SAAS,EAAE,OAAO,GAAG,SAAS,GAC7B,gBAAgB,CAYlB;AAED,wBAAgB,+BAA+B,CAC7C,OAAO,EAAE,gBAAgB,GAAG,mBAAmB,GAAG,iBAAiB,EACnE,SAAS,EAAE,MAAM,GAChB,IAAI,CAMN;AAED,wBAAsB,kCAAkC,CACtD,GAAG,EAAE,2BAA2B,EAChC,OAAO,EAAE,OAAO,EAChB,SAAS,EAAE,OAAO,GAAG,UAAU,GAC9B,OAAO,CAAC,6BAA6B,CAAC,CAwExC;AAED,wBAAgB,+BAA+B,CAC7C,GAAG,EAAE,KAAK,EACV,OAAO,EAAE,OAAO,GAAG,IAAI,EACvB,SAAS,EAAE,+BAA+B,EAC1C,MAAM,EAAE,MAAM,GACb;IACD,IAAI,EAAE,GAAG,GAAG,GAAG,CAAC;IAChB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;CACf,CAsCA;AAED,wBAAsB,6BAA6B,CACjD,GAAG,EAAE,2BAA2B,EAChC,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC,IAAI,CAAC,CA8Df;AAKD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AAExE,wBAAgB,sCAAsC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAE1E;AAED,wBAAgB,gCAAgC,CAC9C,GAAG,EAAE,2BAA2B,GAC/B,mBAAmB,CAWrB;AAED,wBAAgB,8BAA8B,CAC5C,GAAG,EAAE,2BAA2B,EAChC,OAAO,EAAE,MAAM,GACd,IAAI,CAKN;AAED,wBAAsB,kCAAkC,CACtD,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC,CA2Cf;AAED,wBAAsB,0CAA0C,CAC9D,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,KAAK,CAAC,CAKhB"}