@timetotest/cli 0.1.10 → 0.2.0

package/dist/src/lib/local-tools/file-tools.js

@@ -0,0 +1,45 @@
+/**
+ * File Tools - Artifact and configuration file management
+ */
+import * as fs from "fs";
+import * as path from "path";
+export async function readFile(filePath) {
+    try {
+        if (!fs.existsSync(filePath)) {
+            return { error: "File not found" };
+        }
+        const content = fs.readFileSync(filePath, "utf-8");
+        return { success: true, content, path: filePath };
+    }
+    catch (error) {
+        return { error: error.message };
+    }
+}
+export async function writeFile(filePath, content) {
+    try {
+        const dir = path.dirname(filePath);
+        if (!fs.existsSync(dir)) {
+            fs.mkdirSync(dir, { recursive: true });
+        }
+        fs.writeFileSync(filePath, content, "utf-8");
+        return { success: true, path: filePath, bytes: content.length };
+    }
+    catch (error) {
+        return { error: error.message };
+    }
+}
+export async function saveArtifact(name, data, sessionPath) {
+    try {
+        const artifactPath = path.join(sessionPath, "artifacts", `${name}.json`);
+        const dir = path.dirname(artifactPath);
+        if (!fs.existsSync(dir)) {
+            fs.mkdirSync(dir, { recursive: true });
+        }
+        fs.writeFileSync(artifactPath, JSON.stringify(data, null, 2), "utf-8");
+        return { success: true, path: artifactPath };
+    }
+    catch (error) {
+        return { error: error.message };
+    }
+}
+//# sourceMappingURL=file-tools.js.map

package/dist/src/lib/local-tools/file-tools.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-tools.js","sourceRoot":"","sources":["../../../../src/lib/local-tools/file-tools.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,QAAgB;IAC7C,IAAI,CAAC;QACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC7B,OAAO,EAAC,KAAK,EAAE,gBAAgB,EAAC,CAAC;QACnC,CAAC;QACD,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACnD,OAAO,EAAC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAC,CAAC;IAClD,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QACpB,OAAO,EAAC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,QAAgB,EAChB,OAAe;IAEf,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAC,SAAS,EAAE,IAAI,EAAC,CAAC,CAAC;QACvC,CAAC;QACD,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC7C,OAAO,EAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAC,CAAC;IAChE,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QACpB,OAAO,EAAC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAY,EACZ,IAAyB,EACzB,WAAmB;IAEnB,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,GAAG,IAAI,OAAO,CAAC,CAAC;QACzE,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QACvC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAC,SAAS,EAAE,IAAI,EAAC,CAAC,CAAC;QACvC,CAAC;QACD,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QACvE,OAAO,EAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QACpB,OAAO,EAAC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAC,CAAC;IAChC,CAAC;AACH,CAAC"}

package/dist/src/lib/local-tools/general/discover-local-services.js

@@ -0,0 +1,95 @@
+import axios from "axios";
+import { Socket } from "net";
+const COMMON_PORTS = [3000, 3001, 4000, 5000, 5001, 8000, 8080, 8888, 9000];
+export async function discoverLocalServices() {
+    const services = [];
+    for (const port of COMMON_PORTS) {
+        if (await isPortOpen(port)) {
+            const baseUrl = `http://localhost:${port}`;
+            const type = await detectServiceType(baseUrl);
+            const info = { port, type, baseUrl };
+            if (type === "api") {
+                info.endpoints = await probeApiEndpoints(baseUrl);
+            }
+            services.push(info);
+        }
+    }
+    return { success: true, services };
+}
+async function isPortOpen(port) {
+    return new Promise((resolve) => {
+        const socket = new Socket();
+        const onError = () => {
+            socket.destroy();
+            resolve(false);
+        };
+        socket
+            .setTimeout(1_000)
+            .once("connect", () => {
+            socket.end();
+            resolve(true);
+        })
+            .once("timeout", onError)
+            .once("error", onError)
+            .connect(port, "127.0.0.1");
+    });
+}
+async function detectServiceType(baseUrl) {
+    if (await postJson(`${baseUrl}/graphql`, { query: "{ __typename }" })) {
+        return "graphql";
+    }
+    const apiCandidates = ["/api", "/v1", "/docs", "/swagger", "/openapi.json"];
+    for (const suffix of apiCandidates) {
+        if (await getUrl(`${baseUrl}${suffix}`)) {
+            return "api";
+        }
+    }
+    const uiResponse = await getUrl(baseUrl);
+    if (uiResponse?.headers["content-type"]?.includes("text/html")) {
+        return "ui";
+    }
+    return "unknown";
+}
+async function probeApiEndpoints(baseUrl) {
+    const endpoints = [];
+    const candidates = [
+        "/api",
+        "/api/v1",
+        "/api/health",
+        "/health",
+        "/status",
+        "/swagger",
+        "/openapi.json",
+    ];
+    for (const path of candidates) {
+        if (await getUrl(`${baseUrl}${path}`)) {
+            endpoints.push(path);
+        }
+    }
+    return endpoints;
+}
+async function getUrl(url) {
+    try {
+        const response = await axios.get(url, { timeout: 2_000 });
+        if (response.status < 400) {
+            return response;
+        }
+    }
+    catch {
+        /* noop */
+    }
+    return null;
+}
+async function postJson(url, body) {
+    try {
+        const response = await axios.post(url, body, { timeout: 2_000 });
+        if (response.status < 400) {
+            return response;
+        }
+    }
+    catch {
+        /* noop */
+    }
+    return null;
+}
+//# sourceMappingURL=discover-local-services.js.map

package/dist/src/lib/local-tools/general/discover-local-services.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"discover-local-services.js","sourceRoot":"","sources":["../../../../../src/lib/local-tools/general/discover-local-services.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAmB,MAAM,EAAC,MAAM,KAAK,CAAC;AAE7C,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAS5E,MAAM,CAAC,KAAK,UAAU,qBAAqB;IACzC,MAAM,QAAQ,GAAuB,EAAE,CAAC;IACxC,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,IAAI,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,oBAAoB,IAAI,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,OAAO,CAAC,CAAC;YAC9C,MAAM,IAAI,GAAqB,EAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAC,CAAC;YACrD,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;gBACnB,IAAI,CAAC,SAAS,GAAG,MAAM,iBAAiB,CAAC,OAAO,CAAC,CAAC;YACpD,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IACD,OAAO,EAAC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAC,CAAC;AACnC,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY;IACpC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO,CAAC,KAAK,CAAC,CAAC;QACjB,CAAC,CAAC;QACF,MAAM;aACH,UAAU,CAAC,KAAK,CAAC;aACjB,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE;YACpB,MAAM,CAAC,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC,CAAC;QAChB,CAAC,CAAC;aACD,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC;aACxB,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC;aACtB,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,OAAe;IAC9C,IAAI,MAAM,QAAQ,CAAC,GAAG,OAAO,UAAU,EAAE,EAAC,KAAK,EAAE,gBAAgB,EAAC,CAAC,EAAE,CAAC;QACpE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,aAAa,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,eAAe,CAAC,CAAC;IAC5E,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;QACnC,IAAI,MAAM,MAAM,CAAC,GAAG,OAAO,GAAG,MAAM,EAAE,CAAC,EAAE,CAAC;YACxC,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,UAAU,EAAE,OAAO,CAAC,cAAc,CAAC,EAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAC/D,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,OAAe;IAC9C,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,UAAU,GAAG;QACjB,MAAM;QACN,SAAS;QACT,aAAa;QACb,SAAS;QACT,SAAS;QACT,UAAU;QACV,eAAe;KAChB,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,MAAM,MAAM,CAAC,GAAG,OAAO,GAAG,IAAI,EAAE,CAAC,EAAE,CAAC;YACtC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,MAAM,CAAC,GAAW;IAC/B,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAC,OAAO,EAAE,KAAK,EAAC,CAAC,CAAC;QACxD,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC1B,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,UAAU;IACZ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,GAAW,EAAE,IAAyB;IAC5D,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,EAAC,OAAO,EAAE,KAAK,EAAC,CAAC,CAAC;QAC/D,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC1B,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,UAAU;IACZ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/src/lib/local-tools/general/index.js

@@ -0,0 +1,2 @@
+export { discoverLocalServices } from "./discover-local-services.js";
+//# sourceMappingURL=index.js.map

package/dist/src/lib/local-tools/general/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/lib/local-tools/general/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,qBAAqB,EAAC,MAAM,8BAA8B,CAAC"}

package/dist/src/lib/local-tools/ui/click-element.js

@@ -0,0 +1,105 @@
+import { domRagSearch, ensureDomRagIndex } from "./dom-rag.js";
+import { resolveSelectorWithAgent } from "./selector-resolver.js";
+export async function clickElement(ctx, goal, selector, query, requireUnique = true) {
+    const page = ctx.page;
+    if (!page) {
+        return { error: "Browser page not initialized" };
+    }
+    try {
+        let resolvedSelector = selector?.trim();
+        // If no explicit selector, use local DOM RAG to find best candidate
+        let summaryCandidates = [];
+        let resolverChosenCoordinates;
+        if (!resolvedSelector && query?.length) {
+            const searchRes = await domRagSearch(page, query);
+            if (!searchRes.success) {
+                return { error: searchRes.error || "No elements matched search queries" };
+            }
+            const results = searchRes.data?.results || [];
+            summaryCandidates = Array.isArray(results) ? results : [];
+            const raw = searchRes.data?.raw_results || [];
+            const rawCandidates = Array.isArray(raw)
+                ? raw
+                : [];
+            if (!rawCandidates.length) {
+                return { error: "No selector candidates found" };
+            }
+            const resolver = await resolveSelectorWithAgent(page, {
+                goal,
+                questions: query,
+                candidates: rawCandidates,
+            });
+            if (!resolver.success || !resolver.selector) {
+                return {
+                    error: resolver.error || "Selector resolver failed",
+                };
+            }
+            resolverChosenCoordinates = resolver.coordinates;
+            if (Array.isArray(resolver.selector)) {
+                resolvedSelector = resolver.selector.find((sel) => typeof sel === "string" && sel.trim());
+            }
+            else if (typeof resolver.selector === "string") {
+                resolvedSelector = resolver.selector.trim();
+            }
+            if (resolvedSelector && resolverChosenCoordinates) {
+                summaryCandidates = summaryCandidates.map((cand) => cand.selector === resolvedSelector
+                    ? { ...cand, coordinates: resolverChosenCoordinates }
+                    : cand);
+            }
+        }
+        if (!resolvedSelector) {
+            return { error: "Could not determine element selector" };
+        }
+        if (resolvedSelector &&
+            !summaryCandidates.some((cand) => cand.selector === resolvedSelector)) {
+            summaryCandidates = [
+                ...summaryCandidates,
+                resolverChosenCoordinates
+                    ? { selector: resolvedSelector, coordinates: resolverChosenCoordinates }
+                    : { selector: resolvedSelector },
+            ];
+        }
+        if (resolvedSelector &&
+            summaryCandidates.length > 0 &&
+            !summaryCandidates.some((cand) => cand.selector === resolvedSelector)) {
+            summaryCandidates = [
+                ...summaryCandidates,
+                resolverChosenCoordinates
+                    ? { selector: resolvedSelector, coordinates: resolverChosenCoordinates }
+                    : { selector: resolvedSelector },
+            ];
+        }
+        // Try clicking the resolved selector
+        await page.click(resolvedSelector, { timeout: 5_000 });
+        void (async () => {
+            try {
+                await page.waitForTimeout(250);
+                await ensureDomRagIndex(page);
+            }
+            catch {
+                /* best-effort */
+            }
+        })();
+        const resultPayload = {
+            success: true,
+            selector: resolvedSelector,
+            goal,
+            candidates: summaryCandidates.length
+                ? summaryCandidates
+                : [{ selector: resolvedSelector }],
+        };
+        // Attach coordinates if resolver provided them
+        const matchingCandidate = summaryCandidates.find((cand) => cand.selector === resolvedSelector);
+        if (matchingCandidate?.coordinates) {
+            resultPayload.coordinates = matchingCandidate.coordinates;
+        }
+        else if (resolverChosenCoordinates) {
+            resultPayload.coordinates = resolverChosenCoordinates;
+        }
+        return resultPayload;
+    }
+    catch (error) {
+        return { error: error?.message ?? String(error) };
+    }
+}
+//# sourceMappingURL=click-element.js.map

package/dist/src/lib/local-tools/ui/click-element.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"click-element.js","sourceRoot":"","sources":["../../../../../src/lib/local-tools/ui/click-element.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,YAAY,EAAE,iBAAiB,EAAC,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAC,wBAAwB,EAAC,MAAM,wBAAwB,CAAC;AAEhE,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAwB,EACxB,IAAY,EACZ,QAAiB,EACjB,KAAgB,EAChB,gBAAyB,IAAI;IAE7B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,EAAC,KAAK,EAAE,8BAA8B,EAAC,CAAC;IACjD,CAAC;IAED,IAAI,CAAC;QACH,IAAI,gBAAgB,GAAG,QAAQ,EAAE,IAAI,EAAE,CAAC;QAExC,oEAAoE;QACpE,IAAI,iBAAiB,GAA+B,EAAE,CAAC;QACvD,IAAI,yBAA8B,CAAC;QACnC,IAAI,CAAC,gBAAgB,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;YACvC,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAClD,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;gBACvB,OAAO,EAAC,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,oCAAoC,EAAC,CAAC;YAC1E,CAAC;YAED,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC;YAC9C,iBAAiB,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1D,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC;YAC9C,MAAM,aAAa,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;gBACtC,CAAC,CAAE,GAKE;gBACL,CAAC,CAAC,EAAE,CAAC;YAEP,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;gBAC1B,OAAO,EAAC,KAAK,EAAE,8BAA8B,EAAC,CAAC;YACjD,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE;gBACpD,IAAI;gBACJ,SAAS,EAAE,KAAK;gBAChB,UAAU,EAAE,aAAa;aAC1B,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;gBAC5C,OAAO;oBACL,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,0BAA0B;iBACpD,CAAC;YACJ,CAAC;YAED,yBAAyB,GAAG,QAAQ,CAAC,WAAW,CAAC;YAEjD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACrC,gBAAgB,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CACvC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,CAC/C,CAAC;YACJ,CAAC;iBAAM,IAAI,OAAO,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBACjD,gBAAgB,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC9C,CAAC;YAED,IAAI,gBAAgB,IAAI,yBAAyB,EAAE,CAAC;gBAClD,iBAAiB,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACjD,IAAI,CAAC,QAAQ,KAAK,gBAAgB;oBAChC,CAAC,CAAC,EAAC,GAAG,IAAI,EAAE,WAAW,EAAE,yBAAyB,EAAC;oBACnD,CAAC,CAAC,IAAI,CACT,CAAC;YACJ,CAAC;QACH,CAAC;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,OAAO,EAAC,KAAK,EAAE,sCAAsC,EAAC,CAAC;QACzD,CAAC;QAED,IACE,gBAAgB;YAChB,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,gBAAgB,CAAC,EACrE,CAAC;YACD,iBAAiB,GAAG;gBAClB,GAAG,iBAAiB;gBACpB,yBAAyB;oBACvB,CAAC,CAAC,EAAC,QAAQ,EAAE,gBAAgB,EAAE,WAAW,EAAE,yBAAyB,EAAC;oBACtE,CAAC,CAAC,EAAC,QAAQ,EAAE,gBAAgB,EAAC;aACjC,CAAC;QACJ,CAAC;QAED,IACE,gBAAgB;YAChB,iBAAiB,CAAC,MAAM,GAAG,CAAC;YAC5B,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,gBAAgB,CAAC,EACrE,CAAC;YACD,iBAAiB,GAAG;gBAClB,GAAG,iBAAiB;gBACpB,yBAAyB;oBACvB,CAAC,CAAC,EAAC,QAAQ,EAAE,gBAAgB,EAAE,WAAW,EAAE,yBAAyB,EAAC;oBACtE,CAAC,CAAC,EAAC,QAAQ,EAAE,gBAAgB,EAAC;aACjC,CAAC;QACJ,CAAC;QAED,qCAAqC;QACrC,MAAM,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,EAAC,OAAO,EAAE,KAAK,EAAC,CAAC,CAAC;QACrD,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;gBAC/B,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,iBAAiB;YACnB,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;QACL,MAAM,aAAa,GAAwB;YACzC,OAAO,EAAE,IAAI;YACb,QAAQ,EAAE,gBAAgB;YAC1B,IAAI;YACJ,UAAU,EAAE,iBAAiB,CAAC,MAAM;gBAClC,CAAC,CAAC,iBAAiB;gBACnB,CAAC,CAAC,CAAC,EAAC,QAAQ,EAAE,gBAAgB,EAAC,CAAC;SACnC,CAAC;QACF,+CAA+C;QAC/C,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,IAAI,CAC9C,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,gBAAgB,CAC7C,CAAC;QACF,IAAI,iBAAiB,EAAE,WAAW,EAAE,CAAC;YACnC,aAAa,CAAC,WAAW,GAAG,iBAAiB,CAAC,WAAW,CAAC;QAC5D,CAAC;aAAM,IAAI,yBAAyB,EAAE,CAAC;YACrC,aAAa,CAAC,WAAW,GAAG,yBAAyB,CAAC;QACxD,CAAC;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QACpB,OAAO,EAAC,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAC,CAAC;IAClD,CAAC;AACH,CAAC"}

package/dist/src/lib/local-tools/ui/dom-rag.js

@@ -0,0 +1,201 @@
+import crypto from "node:crypto";
+// Simple in-memory index keyed by page URL
+const domIndexByUrl = new Map();
+const INTERACTIVE_SELECTORS = [
+    "button",
+    "a",
+    "input",
+    "textarea",
+    "select",
+    '[role="button"]',
+    '[role="link"]',
+    '[role="menuitem"]',
+    '[role="tab"]',
+    '[role="checkbox"]',
+    '[role="radio"]',
+    '[role="switch"]',
+    '[contenteditable]'
+].join(",");
+function md5(text) {
+    return crypto.createHash("md5").update(text).digest("hex");
+}
+function now() {
+    return Date.now();
+}
+function buildCombinedText(sel, text, attrs) {
+    const attrPairs = [];
+    for (const [k, v] of Object.entries(attrs || {})) {
+        if (v == null)
+            continue;
+        const sv = String(v);
+        if (!sv)
+            continue;
+        // keep only small-ish values
+        if (sv.length <= 256)
+            attrPairs.push(`${k}='${sv}'`);
+    }
+    return [`Selector: ${sel}`, `Text: ${text || ""}`, `Attrs: ${attrPairs.join(" ")}`]
+        .filter(Boolean)
+        .join("\n");
+}
+export async function ensureDomRagIndex(page) {
+    const url = page.url();
+    if (!url)
+        return false;
+    const html = await page.content();
+    const hash = md5(html);
+    const existing = domIndexByUrl.get(url);
+    const cooldownMs = 2000; // 2s cooldown similar to backend
+    if (existing && existing.domHash === hash && now() - existing.lastIndexedAt < cooldownMs) {
+        return false; // unchanged & within cooldown
+    }
+    // Extract interactive elements within the page context to avoid extra deps
+    const elements = await page.evaluate((selector) => {
+        function simpleSelector(el) {
+            const id = el.id;
+            if (id)
+                return `${el.tagName.toLowerCase()}#${id}`;
+            const dataTestId = el.getAttribute("data-testid");
+            if (dataTestId)
+                return `${el.tagName.toLowerCase()}[data-testid="${dataTestId}"]`;
+            const dataCy = el.getAttribute("data-cy");
+            if (dataCy)
+                return `${el.tagName.toLowerCase()}[data-cy="${dataCy}"]`;
+            const className = el.className || "";
+            const classPart = className
+                ? "." + String(className).trim().split(/\s+/).filter(Boolean).slice(0, 3).join(".")
+                : "";
+            return `${el.tagName.toLowerCase()}${classPart}`;
+        }
+        function pickAttrs(el) {
+            const attrs = {};
+            const src = el;
+            const keys = [
+                "id",
+                "name",
+                "type",
+                "value",
+                "placeholder",
+                "aria-label",
+                "aria-labelledby",
+                "role",
+                "tabindex",
+                "title",
+                "alt",
+                "href",
+                "target",
+                "data-testid",
+                "data-cy",
+            ];
+            for (const k of keys) {
+                const v = el.getAttribute?.(k) ?? src[k];
+                if (v != null && v !== "")
+                    attrs[k] = v;
+            }
+            attrs["tag"] = el.tagName.toLowerCase();
+            return attrs;
+        }
+        const out = [];
+        const nodeList = Array.from(document.querySelectorAll(selector));
+        // cap to a reasonable count to avoid huge payloads
+        const max = 600;
+        for (const el of nodeList.slice(0, max)) {
+            const tag = el.tagName.toLowerCase();
+            const text = (el.textContent || "").trim();
+            const sel = simpleSelector(el);
+            const attributes = pickAttrs(el);
+            const combinedText = [sel, text, JSON.stringify(attributes)].join("\n");
+            out.push({ selector: sel, tag, text, attributes, combinedText });
+        }
+        return out;
+    }, INTERACTIVE_SELECTORS);
+    // Skip indexing if page appears mostly empty (e.g., loading)
+    if (elements.length === 0) {
+        return false;
+    }
+    domIndexByUrl.set(page.url(), {
+        domHash: hash,
+        elements: elements.map((e) => ({
+            ...e,
+            combinedText: buildCombinedText(e.selector, e.text, e.attributes),
+        })),
+        lastIndexedAt: now(),
+    });
+    return true;
+}
+function summarizeResults(results) {
+    const out = [];
+    const seen = new Set();
+    for (const r of results) {
+        if (out.length >= 8)
+            break;
+        if (seen.has(r.selector))
+            continue;
+        seen.add(r.selector);
+        const attrs = r.attributes || {};
+        const summary = {
+            selector: r.selector,
+        };
+        if (attrs.placeholder)
+            summary.placeholder = attrs.placeholder;
+        if (attrs.role)
+            summary.role = attrs.role;
+        if (attrs.type)
+            summary.type = attrs.type;
+        out.push(summary);
+    }
+    return out;
+}
+export async function domRagSearch(page, queries) {
+    if (!queries || !Array.isArray(queries)) {
+        return { success: false, error: "No search queries provided (query array is required)" };
+    }
+    // Keep only the first non-empty query (parity with backend enforcement)
+    const normalizedQueries = queries
+        .map((s) => (s || "").trim())
+        .filter(Boolean);
+    if (!normalizedQueries.length) {
+        return { success: false, error: "No valid query provided" };
+    }
+    await ensureDomRagIndex(page);
+    const url = page.url();
+    const index = domIndexByUrl.get(url);
+    if (!index || !index.elements?.length) {
+        return { success: false, error: "DOM index is empty" };
+    }
+    const terms = Array.from(new Set(normalizedQueries.flatMap((query) => query
+        .toLowerCase()
+        .split(/\s+/)
+        .map((term) => term.trim())
+        .filter(Boolean))));
+    const matches = index.elements.filter((rec) => {
+        if (!terms.length)
+            return true;
+        const haystackParts = [];
+        haystackParts.push(rec.selector.toLowerCase());
+        if (rec.text)
+            haystackParts.push(rec.text.toLowerCase());
+        if (rec.attributes) {
+            for (const value of Object.values(rec.attributes)) {
+                if (typeof value === "string") {
+                    haystackParts.push(value.toLowerCase());
+                }
+            }
+        }
+        const haystack = haystackParts.join(" ");
+        return terms.every((term) => haystack.includes(term));
+    });
+    if (!matches.length) {
+        return { success: false, error: "No DOM elements matched the search queries" };
+    }
+    const limited = matches.slice(0, 50);
+    const rawResults = limited.slice(0, 20).map((rec) => ({
+        selector: rec.selector,
+        tag: rec.tag,
+        text: rec.text,
+        attributes: rec.attributes,
+    }));
+    const results = summarizeResults(limited);
+    return { success: true, data: { results, raw_results: rawResults, rag: true } };
+}
+//# sourceMappingURL=dom-rag.js.map

package/dist/src/lib/local-tools/ui/dom-rag.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dom-rag.js","sourceRoot":"","sources":["../../../../../src/lib/local-tools/ui/dom-rag.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,aAAa,CAAC;AAgBjC,2CAA2C;AAC3C,MAAM,aAAa,GAAG,IAAI,GAAG,EAAqB,CAAC;AAEnD,MAAM,qBAAqB,GAAG;IAC5B,QAAQ;IACR,GAAG;IACH,OAAO;IACP,UAAU;IACV,QAAQ;IACR,iBAAiB;IACjB,eAAe;IACf,mBAAmB;IACnB,cAAc;IACd,mBAAmB;IACnB,gBAAgB;IAChB,iBAAiB;IACjB,mBAAmB;CACpB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEZ,SAAS,GAAG,CAAC,IAAY;IACvB,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC7D,CAAC;AAED,SAAS,GAAG;IACV,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC;AACpB,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAW,EAAE,IAAY,EAAE,KAA0B;IAC9E,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,CAAC;QACjD,IAAI,CAAC,IAAI,IAAI;YAAE,SAAS;QACxB,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACrB,IAAI,CAAC,EAAE;YAAE,SAAS;QAClB,6BAA6B;QAC7B,IAAI,EAAE,CAAC,MAAM,IAAI,GAAG;YAAE,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,CAAC,aAAa,GAAG,EAAE,EAAE,SAAS,IAAI,IAAI,EAAE,EAAE,EAAE,UAAU,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;SAChF,MAAM,CAAC,OAAO,CAAC;SACf,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAAU;IAChD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,IAAI,CAAC,GAAG;QAAE,OAAO,KAAK,CAAC;IACvB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;IACvB,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACxC,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,iCAAiC;IAE1D,IAAI,QAAQ,IAAI,QAAQ,CAAC,OAAO,KAAK,IAAI,IAAI,GAAG,EAAE,GAAG,QAAQ,CAAC,aAAa,GAAG,UAAU,EAAE,CAAC;QACzF,OAAO,KAAK,CAAC,CAAC,8BAA8B;IAC9C,CAAC;IAED,2EAA2E;IAC3E,MAAM,QAAQ,GAAoB,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,QAAgB,EAAE,EAAE;QACzE,SAAS,cAAc,CAAC,EAAW;YACjC,MAAM,EAAE,GAAI,EAAkB,CAAC,EAAE,CAAC;YAClC,IAAI,EAAE;gBAAE,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,EAAE,CAAC;YACnD,MAAM,UAAU,GAAI,EAAkB,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC;YACnE,IAAI,UAAU;gBAAE,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,iBAAiB,UAAU,IAAI,CAAC;YAClF,MAAM,MAAM,GAAI,EAAkB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;YAC3D,IAAI,MAAM;gBAAE,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,aAAa,MAAM,IAAI,CAAC;YACtE,MAAM,SAAS,GAAI,EAAkB,CAAC,SAAS,IAAI,EAAE,CAAC;YACtD,MAAM,SAAS,GAAG,SAAS;gBACzB,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;gBACnF,CAAC,CAAC,EAAE,CAAC;YACP,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,SAAS,EAAE,CAAC;QACnD,CAAC;QAED,SAAS,SAAS,CAAC,EAAW;YAC5B,MAAM,KAAK,GAAwB,EAAE,CAAC;YACtC,MAAM,GAAG,GAAG,EAAwB,CAAC;YACrC,MAAM,IAAI,GAAG;gBACX,IAAI;gBACJ,MAAM;gBACN,MAAM;gBACN,OAAO;gBACP,aAAa;gBACb,YAAY;gBACZ,iBAAiB;gBACjB,MAAM;gBACN,UAAU;gBACV,OAAO;gBACP,KAAK;gBACL,MAAM;gBACN,QAAQ;gBACR,aAAa;gBACb,SAAS;aACV,CAAC;YACF,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,CAAC,GAAI,EAAkB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;gBAC1D,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE;oBAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAC1C,CAAC;YACD,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YACxC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,GAAG,GAAoB,EAAS,CAAC;QACvC,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAc,CAAC;QAC9E,mDAAmD;QACnD,MAAM,GAAG,GAAG,GAAG,CAAC;QAChB,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YACxC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3C,MAAM,GAAG,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC;YAC/B,MAAM,UAAU,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;YACjC,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACvE,GAAW,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC,CAAC;QAC5E,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,qBAAqB,CAAC,CAAC;IAE1B,6DAA6D;IAC7D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE;QAC5B,OAAO,EAAE,IAAI;QACb,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7B,GAAG,CAAC;YACJ,YAAY,EAAE,iBAAiB,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC;SAClE,CAAC,CAAC;QACH,aAAa,EAAE,GAAG,EAAE;KACrB,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAwB;IAChD,MAAM,GAAG,GAA+B,EAAE,CAAC;IAC3C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC;YAAE,MAAM;QAC3B,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;YAAE,SAAS;QACnC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACrB,MAAM,KAAK,GAAG,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC;QACjC,MAAM,OAAO,GAAwB;YACnC,QAAQ,EAAE,CAAC,CAAC,QAAQ;SACrB,CAAC;QACF,IAAI,KAAK,CAAC,WAAW;YAAE,OAAO,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC;QAC/D,IAAI,KAAK,CAAC,IAAI;YAAE,OAAO,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;QAC1C,IAAI,KAAK,CAAC,IAAI;YAAE,OAAO,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;QAC1C,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAU,EAAE,OAAiB;IAW9D,IAAI,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,sDAAsD,EAAE,CAAC;IAC3F,CAAC;IACD,wEAAwE;IACxE,MAAM,iBAAiB,GAAG,OAAO;SAC9B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;SAC5B,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;QAC9B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,yBAAyB,EAAE,CAAC;IAC9D,CAAC;IAED,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACrC,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;QACtC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,oBAAoB,EAAE,CAAC;IACzD,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CACtB,IAAI,GAAG,CACL,iBAAiB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAClC,KAAK;SACF,WAAW,EAAE;SACb,KAAK,CAAC,KAAK,CAAC;SACZ,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,OAAO,CAAC,CACnB,CACF,CACF,CAAC;IAEF,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;QAC5C,IAAI,CAAC,KAAK,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAC/B,MAAM,aAAa,GAAa,EAAE,CAAC;QACnC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QAC/C,IAAI,GAAG,CAAC,IAAI;YAAE,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACzD,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;YACnB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBAClD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;oBAC9B,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,4CAA4C,EAAE,CAAC;IACjF,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAErC,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACpD,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,UAAU,EAAE,GAAG,CAAC,UAAU;KAC3B,CAAC,CAAC,CAAC;IAEJ,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC1C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC;AAClF,CAAC"}

package/dist/src/lib/local-tools/ui/find-element.js

@@ -0,0 +1,31 @@
+export async function findElementByQuery(page, query, requireUnique) {
+    const selectors = buildSelectorCandidates(query);
+    for (const selector of selectors) {
+        try {
+            const locator = page.locator(selector);
+            const count = await locator.count();
+            if (count === 1 || (!requireUnique && count > 0)) {
+                return selector;
+            }
+        }
+        catch {
+            continue;
+        }
+    }
+    return null;
+}
+function buildSelectorCandidates(query) {
+    const [first = ""] = query;
+    const text = first.trim();
+    if (!text) {
+        return [];
+    }
+    return [
+        `button:has-text("${text}")`,
+        `a:has-text("${text}")`,
+        `input[placeholder*="${text}"]`,
+        `label:has-text("${text}")`,
+        `[role="button"]:has-text("${text}")`,
+    ];
+}
+//# sourceMappingURL=find-element.js.map

package/dist/src/lib/local-tools/ui/find-element.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"find-element.js","sourceRoot":"","sources":["../../../../../src/lib/local-tools/ui/find-element.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAU,EACV,KAAe,EACf,aAAsB;IAEtB,MAAM,SAAS,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;IAEjD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACvC,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACpC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,aAAa,IAAI,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;gBACjD,OAAO,QAAQ,CAAC;YAClB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAe;IAC9C,MAAM,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,KAAK,CAAC;IAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO;QACL,oBAAoB,IAAI,IAAI;QAC5B,eAAe,IAAI,IAAI;QACvB,uBAAuB,IAAI,IAAI;QAC/B,mBAAmB,IAAI,IAAI;QAC3B,6BAA6B,IAAI,IAAI;KACtC,CAAC;AACJ,CAAC"}

package/dist/src/lib/local-tools/ui/hover-element.js

@@ -0,0 +1,94 @@
+import { domRagSearch, ensureDomRagIndex } from "./dom-rag.js";
+import { resolveSelectorWithAgent } from "./selector-resolver.js";
+export async function hoverElement(ctx, goal, selector, query, requireUnique = true) {
+    const page = ctx.page;
+    if (!page) {
+        return { error: "Browser page not initialized" };
+    }
+    try {
+        let resolvedSelector = selector?.trim();
+        // If no explicit selector, use local DOM RAG to find best candidate
+        let summaryCandidates = [];
+        let resolverChosenCoordinates;
+        if (!resolvedSelector && query?.length) {
+            const searchRes = await domRagSearch(page, query);
+            if (!searchRes.success) {
+                return { error: searchRes.error || "No elements matched search queries" };
+            }
+            const results = searchRes.data?.results || [];
+            summaryCandidates = Array.isArray(results) ? results : [];
+            const raw = searchRes.data?.raw_results || [];
+            const rawCandidates = Array.isArray(raw)
+                ? raw
+                : [];
+            if (!rawCandidates.length) {
+                return { error: "No selector candidates found" };
+            }
+            const resolver = await resolveSelectorWithAgent(page, {
+                goal,
+                questions: query,
+                candidates: rawCandidates,
+            });
+            if (!resolver.success || !resolver.selector) {
+                return {
+                    error: resolver.error || "Selector resolver failed",
+                };
+            }
+            resolverChosenCoordinates = resolver.coordinates;
+            if (Array.isArray(resolver.selector)) {
+                resolvedSelector = resolver.selector.find((sel) => typeof sel === "string" && sel.trim());
+            }
+            else if (typeof resolver.selector === "string") {
+                resolvedSelector = resolver.selector.trim();
+            }
+            if (resolvedSelector && resolverChosenCoordinates) {
+                summaryCandidates = summaryCandidates.map((cand) => cand.selector === resolvedSelector
+                    ? { ...cand, coordinates: resolverChosenCoordinates }
+                    : cand);
+            }
+        }
+        if (!resolvedSelector) {
+            return { error: "Could not determine element selector" };
+        }
+        if (resolvedSelector &&
+            !summaryCandidates.some((cand) => cand.selector === resolvedSelector)) {
+            summaryCandidates = [
+                ...summaryCandidates,
+                resolverChosenCoordinates
+                    ? { selector: resolvedSelector, coordinates: resolverChosenCoordinates }
+                    : { selector: resolvedSelector },
+            ];
+        }
+        // Try hover on resolved selector
+        await page.hover(resolvedSelector, { timeout: 5_000 });
+        void (async () => {
+            try {
+                await page.waitForTimeout(250);
+                await ensureDomRagIndex(page);
+            }
+            catch {
+                /* best-effort */
+            }
+        })();
+        const resultPayload = {
+            success: true,
+            selector: resolvedSelector,
+            goal,
+            candidates: summaryCandidates.length
+                ? summaryCandidates
+                : [{ selector: resolvedSelector }],
+        };
+        const matchingCandidate = summaryCandidates.find((cand) => cand.selector === resolvedSelector);
+        if (matchingCandidate?.coordinates) {
+            resultPayload.coordinates = matchingCandidate.coordinates;
+        }
+        else if (resolverChosenCoordinates) {
+            resultPayload.coordinates = resolverChosenCoordinates;
+        }
+        return resultPayload;
+    }
+    catch (error) {
+        return { error: error?.message ?? String(error) };
+    }
+}
+//# sourceMappingURL=hover-element.js.map

package/dist/src/lib/local-tools/ui/hover-element.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hover-element.js","sourceRoot":"","sources":["../../../../../src/lib/local-tools/ui/hover-element.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,YAAY,EAAE,iBAAiB,EAAC,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAC,wBAAwB,EAAC,MAAM,wBAAwB,CAAC;AAEhE,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAwB,EACxB,IAAY,EACZ,QAAiB,EACjB,KAAgB,EAChB,gBAAyB,IAAI;IAE7B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,EAAC,KAAK,EAAE,8BAA8B,EAAC,CAAC;IACjD,CAAC;IACD,IAAI,CAAC;QACH,IAAI,gBAAgB,GAAG,QAAQ,EAAE,IAAI,EAAE,CAAC;QAExC,oEAAoE;QACpE,IAAI,iBAAiB,GAA+B,EAAE,CAAC;QACvD,IAAI,yBAA8B,CAAC;QACnC,IAAI,CAAC,gBAAgB,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;YACvC,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAClD,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;gBACvB,OAAO,EAAC,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,oCAAoC,EAAC,CAAC;YAC1E,CAAC;YAED,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC;YAC9C,iBAAiB,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1D,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC;YAC9C,MAAM,aAAa,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;gBACtC,CAAC,CAAE,GAKE;gBACL,CAAC,CAAC,EAAE,CAAC;YAEP,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;gBAC1B,OAAO,EAAC,KAAK,EAAE,8BAA8B,EAAC,CAAC;YACjD,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE;gBACpD,IAAI;gBACJ,SAAS,EAAE,KAAK;gBAChB,UAAU,EAAE,aAAa;aAC1B,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;gBAC5C,OAAO;oBACL,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,0BAA0B;iBACpD,CAAC;YACJ,CAAC;YAED,yBAAyB,GAAG,QAAQ,CAAC,WAAW,CAAC;YAEjD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACrC,gBAAgB,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CACvC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,CAC/C,CAAC;YACJ,CAAC;iBAAM,IAAI,OAAO,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBACjD,gBAAgB,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC9C,CAAC;YAED,IAAI,gBAAgB,IAAI,yBAAyB,EAAE,CAAC;gBAClD,iBAAiB,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACjD,IAAI,CAAC,QAAQ,KAAK,gBAAgB;oBAChC,CAAC,CAAC,EAAC,GAAG,IAAI,EAAE,WAAW,EAAE,yBAAyB,EAAC;oBACnD,CAAC,CAAC,IAAI,CACT,CAAC;YACJ,CAAC;QACH,CAAC;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,OAAO,EAAC,KAAK,EAAE,sCAAsC,EAAC,CAAC;QACzD,CAAC;QAED,IACE,gBAAgB;YAChB,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,gBAAgB,CAAC,EACrE,CAAC;YACD,iBAAiB,GAAG;gBAClB,GAAG,iBAAiB;gBACpB,yBAAyB;oBACvB,CAAC,CAAC,EAAC,QAAQ,EAAE,gBAAgB,EAAE,WAAW,EAAE,yBAAyB,EAAC;oBACtE,CAAC,CAAC,EAAC,QAAQ,EAAE,gBAAgB,EAAC;aACjC,CAAC;QACJ,CAAC;QAED,iCAAiC;QACjC,MAAM,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,EAAC,OAAO,EAAE,KAAK,EAAC,CAAC,CAAC;QACrD,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;gBAC/B,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,iBAAiB;YACnB,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;QACL,MAAM,aAAa,GAAwB;YACzC,OAAO,EAAE,IAAI;YACb,QAAQ,EAAE,gBAAgB;YAC1B,IAAI;YACJ,UAAU,EAAE,iBAAiB,CAAC,MAAM;gBAClC,CAAC,CAAC,iBAAiB;gBACnB,CAAC,CAAC,CAAC,EAAC,QAAQ,EAAE,gBAAgB,EAAC,CAAC;SACnC,CAAC;QACF,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,IAAI,CAC9C,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,gBAAgB,CAC7C,CAAC;QACF,IAAI,iBAAiB,EAAE,WAAW,EAAE,CAAC;YACnC,aAAa,CAAC,WAAW,GAAG,iBAAiB,CAAC,WAAW,CAAC;QAC5D,CAAC;aAAM,IAAI,yBAAyB,EAAE,CAAC;YACrC,aAAa,CAAC,WAAW,GAAG,yBAAyB,CAAC;QACxD,CAAC;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QACpB,OAAO,EAAC,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAC,CAAC;IAClD,CAAC;AACH,CAAC"}

package/dist/src/lib/local-tools/ui/index.js

@@ -0,0 +1,3 @@
+export { ensureMcpConnection, closeMcpConnection, callMcpTool, parseSnapshot, parseCallResult, } from "./playwright-mcp.js";
+export { snapshotQuery } from "./snapshot-query.js";
+//# sourceMappingURL=index.js.map

package/dist/src/lib/local-tools/ui/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/lib/local-tools/ui/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,WAAW,EACX,aAAa,EACb,eAAe,GAChB,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC"}