donobu 5.60.8 → 5.61.0

package/dist/tools/ReplayableInteraction.js

@@ -5,6 +5,7 @@ const v4_1 = require("zod/v4");
 const PageClosedException_1 = require("../exceptions/PageClosedException");
 const ElementSelector_1 = require("../models/ElementSelector");
 const ToolSchema_1 = require("../models/ToolSchema");
+const AdvancedSelectorGenerator_1 = require("../utils/AdvancedSelectorGenerator");
 const Logger_1 = require("../utils/Logger");
 const PlaywrightUtils_1 = require("../utils/PlaywrightUtils");
 const TargetUtils_1 = require("../utils/TargetUtils");
@@ -412,53 +413,87 @@ class ReplayableInteraction extends Tool_1.Tool {
                 },
             };
         }
-        return this.callCore(context, parameters, locators, parameters.selector);
-    }
-    /**
-     * SUPERVISED-mode preview: resolve the element this interaction *would*
-     * target (from either an annotation- or selector-based proposal) and move the
-     * on-screen cursor to it, without performing the interaction. Best-effort —
-     * an unresolvable element simply leaves the cursor where it is.
-     */
-    async previewInteraction(context, parameters) {
-        const page = (0, TargetUtils_1.webPage)(context);
-        const locator = await this.resolvePreviewLocator(context, page, parameters);
-        if (!locator) {
-            return;
+        // Try each candidate locator in priority order, and within a candidate each
+        // matched element, returning on the first successful interaction and
+        // otherwise summarizing what was tried.
+        const coreParameters = this.coreSchema.parse(parameters);
+        const selectorAttempts = [];
+        for (const selectorLocator of locators) {
+            const attemptSummary = {
+                selector: selectorLocator.selector,
+                matchCount: null,
+                errors: [],
+            };
+            selectorAttempts.push(attemptSummary);
+            try {
+                const count = await selectorLocator.locator.count();
+                attemptSummary.matchCount = count;
+                if (count === 0) {
+                    attemptSummary.errors.push('Locator resolved to 0 elements before interaction.');
+                }
+                for (let i = 0; i < count; ++i) {
+                    try {
+                        const forLlm = await this.interactWithLocator(context, coreParameters, selectorLocator.locator.nth(i), page);
+                        // Be careful to not report back the internal, ephemeral, Donobu selector
+                        // used for autonomous runs. If we are in autonomous mode, just report
+                        // back the top-resolved selector.
+                        const reportedSelector = !selectorLocator.selector.includes((0, TargetUtils_1.webInspector)(context).interactableElementAttribute)
+                            ? selectorLocator.selector
+                            : parameters.selector.element[0];
+                        return {
+                            isSuccessful: true,
+                            forLlm: forLlm,
+                            metadata: {
+                                ...parameters.selector,
+                                usedSelector: reportedSelector,
+                            },
+                        };
+                    }
+                    catch (elementError) {
+                        if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(elementError)) {
+                            throw elementError;
+                        }
+                        Logger_1.appLogger.error(`Failed to interact with element '${i}' for selector '${selectorLocator.selector}' due to exception, will fail over to remaining elements (if any)`, elementError);
+                        attemptSummary.errors.push(`element ${i}: ${ReplayableInteraction.describeError(elementError)}`);
+                    }
+                }
+            }
+            catch (locatorError) {
+                if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(locatorError)) {
+                    throw locatorError;
+                }
+                Logger_1.appLogger.error(`Failed to interact with selector '${selectorLocator.selector}' due to exception, will fail over to remaining selectors (if any)`, locatorError);
+                attemptSummary.errors.push(`selector error: ${ReplayableInteraction.describeError(locatorError)}`);
+            }
         }
-        // Point at the same visible target (or its label) the real interaction
-        // would, so the preview matches what approval will touch.
-        const pointTarget = await ReplayableInteraction.getLocatorOrItsLabel(locator.first());
-        // Only reveal the cursor now that we have a real target to point at, so a
-        // non-interactive proposal never pops a stationary cursor.
-        await (0, TargetUtils_1.webInspector)(context).showInteractionCursor();
-        await context.interactionVisualizer.pointAt(page, pointTarget.first(), ReplayableInteraction.PREVIEW_CURSOR_DURATION_MILLIS);
+        return {
+            isSuccessful: false,
+            forLlm: `FAILED! Unable to apply operation. ${ReplayableInteraction.summarizeAttemptsForLlm(selectorAttempts)}`,
+            metadata: {
+                selectorForReplay: parameters.selector,
+                selectorAttempts: selectorAttempts,
+            },
+        };
     }
     async callFromGpt(context, parameters) {
         const page = (0, TargetUtils_1.webPage)(context);
         const elementSelector = `[${(0, TargetUtils_1.webInspector)(context).interactableElementAttribute}="${parameters.annotation}"]`;
-        let locatorData = null;
+        // Find the annotated element in whichever frame holds it.
+        let locator = null;
         for (const frame of page.frames()) {
             if (frame.isDetached()) {
                 continue;
             }
-            const locator = frame.locator(elementSelector);
-            if ((await locator.count()) > 0) {
-                const selectorCandidates = (await PlaywrightUtils_1.PlaywrightUtils.generateSelectors(locator)).slice(0, ReplayableInteraction.MAX_SELECTOR_FAILOVERS);
-                const frameSelector = frame.parentFrame() === null
-                    ? null
-                    : (await PlaywrightUtils_1.PlaywrightUtils.generateSelectors(await frame.frameElement()))[0];
-                locatorData = {
-                    locators: [locator],
-                    selectorForReplay: {
-                        element: selectorCandidates,
-                        frame: frameSelector,
-                    },
-                };
+            const candidate = frame.locator(elementSelector);
+            if ((await candidate.count()) > 0) {
+                locator = candidate;
                 break;
             }
         }
-        if (!locatorData?.locators.length) {
+        const targetHandle = locator
+            ? await locator.elementHandle({ timeout: 1000 }).catch(() => null)
+            : null;
+        if (!locator || !targetHandle) {
             return {
                 isSuccessful: false,
                 forLlm: `FAILED! Unable to resolve HTML element for annotation '${parameters.annotation}'.`,
@@ -468,12 +503,60 @@ class ReplayableInteraction extends Tool_1.Tool {
                 },
             };
         }
-        return this.callCore(context, parameters, locatorData.locators.map((locator) => {
+        // Turn the live element into the best replayable selector — generated from
+        // the DOM and, for env-parameterized actions, re-parameterized + pruned so it
+        // reflects intent rather than incidental position. This runs BEFORE the
+        // interaction: the generation reads the pre-interaction DOM.
+        const selectorForReplay = await (0, AdvancedSelectorGenerator_1.generateAdvancedSelectors)(targetHandle, context);
+        // Resolve the annotation to its single element and act on it directly — no
+        // failover loop, since the annotation selector is unique.
+        const coreParameters = this.coreSchema.parse(parameters);
+        try {
+            const forLlm = await this.interactWithLocator(context, coreParameters, locator, page);
+            return {
+                isSuccessful: true,
+                forLlm,
+                // Report the top generated selector (the annotation attribute is
+                // internal/ephemeral).
+                metadata: {
+                    ...selectorForReplay,
+                    usedSelector: selectorForReplay.element[0],
+                },
+            };
+        }
+        catch (error) {
+            if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
+                throw error;
+            }
+            Logger_1.appLogger.error(`Failed to interact with annotation '${parameters.annotation}'`, error);
             return {
-                selector: elementSelector,
-                locator: locator,
+                isSuccessful: false,
+                forLlm: `FAILED! Unable to apply operation. ${ReplayableInteraction.describeError(error)}`,
+                metadata: {
+                    selectorForReplay,
+                },
             };
-        }), locatorData.selectorForReplay);
+        }
+    }
+    /**
+     * SUPERVISED-mode preview: resolve the element this interaction *would*
+     * target (from either an annotation- or selector-based proposal) and move the
+     * on-screen cursor to it, without performing the interaction. Best-effort —
+     * an unresolvable element simply leaves the cursor where it is.
+     */
+    async previewInteraction(context, parameters) {
+        const page = (0, TargetUtils_1.webPage)(context);
+        const locator = await this.resolvePreviewLocator(context, page, parameters);
+        if (!locator) {
+            return;
+        }
+        // Point at the same visible target (or its label) the real interaction
+        // would, so the preview matches what approval will touch.
+        const pointTarget = await ReplayableInteraction.getLocatorOrItsLabel(locator.first());
+        // Only reveal the cursor now that we have a real target to point at, so a
+        // non-interactive proposal never pops a stationary cursor.
+        await (0, TargetUtils_1.webInspector)(context).showInteractionCursor();
+        await context.interactionVisualizer.pointAt(page, pointTarget.first(), ReplayableInteraction.PREVIEW_CURSOR_DURATION_MILLIS);
     }
     /**
      * Transform a historical {@link ToolCall} into a replay-ready
@@ -608,105 +691,52 @@ class ReplayableInteraction extends Tool_1.Tool {
         }
         return null;
     }
-    async callCore(context, parameters, selectorLocators, selectorForReplay) {
-        // Extract core parameters using the stored schema
-        const coreParameters = this.coreSchema.parse(parameters);
+    /**
+     * Resolve a single, already-located element and perform the interaction on it:
+     * grab the target handle (and its label, if any), verify it is still attached,
+     * capture an HTML snippet, run the concrete {@link invoke}, and wait for the
+     * page to settle. Returns the LLM-facing string (action result + snippet) and
+     * throws on any failure so callers can fail over or record it.
+     *
+     * `locator` must already point at the intended element (callers narrow with
+     * `.nth(i)` or a unique annotation selector); only its first match is used.
+     */
+    async interactWithLocator(context, coreParameters, locator, page) {
         const timeoutOpt = {
             timeout: 1000, // Millis
         };
-        const page = (0, TargetUtils_1.webPage)(context);
-        const selectorAttempts = [];
-        for (const selectorLocator of selectorLocators) {
-            const attemptSummary = {
-                selector: selectorLocator.selector,
-                matchCount: null,
-                errors: [],
-            };
-            selectorAttempts.push(attemptSummary);
-            try {
-                const count = await selectorLocator.locator.count();
-                attemptSummary.matchCount = count;
-                if (count === 0) {
-                    attemptSummary.errors.push('Locator resolved to 0 elements before interaction.');
-                }
-                for (let i = 0; i < count; ++i) {
-                    try {
-                        const originalLocator = selectorLocator.locator.nth(i);
-                        const targetHandle = await originalLocator
-                            .first()
-                            .elementHandle(timeoutOpt);
-                        if (!targetHandle) {
-                            throw new Error(`Failed to resolve element handle for selector '${selectorLocator.selector}'`);
-                        }
-                        const labelLocator = await ReplayableInteraction.getLocatorOrItsLabel(originalLocator);
-                        let labelHandle;
-                        try {
-                            labelHandle = await labelLocator
-                                .first()
-                                .elementHandle(timeoutOpt);
-                        }
-                        catch {
-                            labelHandle = null;
-                        }
-                        const chosenHandle = labelHandle ?? targetHandle;
-                        // Check if element is still attached before attempting to do anything meaningful.
-                        const isAttached = await chosenHandle.evaluate((el) => el.isConnected);
-                        if (!isAttached) {
-                            throw new Error(`Element '${i}' for selector '${selectorLocator.selector}' does not exist in the DOM`);
-                        }
-                        let htmlSnippet = '';
-                        try {
-                            htmlSnippet +=
-                                await (0, TargetUtils_1.webInspector)(context).pageInspector.getHtmlSnippet(chosenHandle);
-                        }
-                        catch (error) {
-                            Logger_1.appLogger.warn('Failed to get HTML snippet', error);
-                        }
-                        const forLlm = await this.invoke(context, coreParameters, {
-                            target: targetHandle,
-                            label: labelHandle ?? undefined,
-                        });
-                        await PlaywrightUtils_1.PlaywrightUtils.waitForPageStability(page);
-                        // Be careful to not report back the internal, ephemeral, Donobu selector
-                        // used for autonomous runs. If we are in autonomous mode, just report
-                        // back the top-resolved selector.
-                        const reportedSelector = !selectorLocator.selector.includes((0, TargetUtils_1.webInspector)(context).interactableElementAttribute)
-                            ? selectorLocator.selector
-                            : selectorForReplay.element[0];
-                        return {
-                            isSuccessful: true,
-                            forLlm: forLlm + htmlSnippet,
-                            metadata: {
-                                ...selectorForReplay,
-                                usedSelector: reportedSelector,
-                            },
-                        };
-                    }
-                    catch (elementError) {
-                        if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(elementError)) {
-                            throw elementError;
-                        }
-                        Logger_1.appLogger.error(`Failed to interact with element '${i}' for selector '${selectorLocator.selector}' due to exception, will fail over to remaining elements (if any)`, elementError);
-                        attemptSummary.errors.push(`element ${i}: ${ReplayableInteraction.describeError(elementError)}`);
-                    }
-                }
-            }
-            catch (locatorError) {
-                if (PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(locatorError)) {
-                    throw locatorError;
-                }
-                Logger_1.appLogger.error(`Failed to interact with selector '${selectorLocator.selector}' due to exception, will fail over to remaining selectors (if any)`, locatorError);
-                attemptSummary.errors.push(`selector error: ${ReplayableInteraction.describeError(locatorError)}`);
-            }
+        const targetHandle = await locator.first().elementHandle(timeoutOpt);
+        if (!targetHandle) {
+            throw new Error('Failed to resolve an element handle for the locator.');
         }
-        return {
-            isSuccessful: false,
-            forLlm: `FAILED! Unable to apply operation. ${ReplayableInteraction.summarizeAttemptsForLlm(selectorAttempts)}`,
-            metadata: {
-                selectorForReplay: selectorForReplay,
-                selectorAttempts: selectorAttempts,
-            },
-        };
+        const labelLocator = await ReplayableInteraction.getLocatorOrItsLabel(locator);
+        let labelHandle;
+        try {
+            labelHandle = await labelLocator.first().elementHandle(timeoutOpt);
+        }
+        catch {
+            labelHandle = null;
+        }
+        const chosenHandle = labelHandle ?? targetHandle;
+        // Check if element is still attached before attempting to do anything meaningful.
+        const isAttached = await chosenHandle.evaluate((el) => el.isConnected);
+        if (!isAttached) {
+            throw new Error('The resolved element is no longer attached to the DOM.');
+        }
+        let htmlSnippet = '';
+        try {
+            htmlSnippet +=
+                await (0, TargetUtils_1.webInspector)(context).pageInspector.getHtmlSnippet(chosenHandle);
+        }
+        catch (error) {
+            Logger_1.appLogger.warn('Failed to get HTML snippet', error);
+        }
+        const forLlm = await this.invoke(context, coreParameters, {
+            target: targetHandle,
+            label: labelHandle ?? undefined,
+        });
+        await PlaywrightUtils_1.PlaywrightUtils.waitForPageStability(page);
+        return forLlm + htmlSnippet;
     }
 }
 exports.ReplayableInteraction = ReplayableInteraction;

package/dist/utils/AdvancedSelectorGenerator.d.ts

@@ -0,0 +1,106 @@
+import type { ElementHandle } from 'playwright';
+import { z } from 'zod/v4';
+import type { ElementSelector } from '../models/ElementSelector';
+import type { GptMessage } from '../models/GptMessage';
+import type { ToolCallContext } from '../models/ToolCallContext';
+/**
+ * The model's decision for advancing a generated selector: the SAME candidate
+ * list, in the same order and count, each rewritten so env-derived substrings are
+ * expressed as `{{$.env.NAME}}` (optionally with a `| filter`) placeholders, or
+ * returned unchanged. The deterministic core ({@link advanceSelectorCandidates})
+ * validates and applies it.
+ */
+export declare const AdvancedSelectorDecisionSchema: z.ZodObject<{
+    element: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+export type AdvancedSelectorDecision = z.infer<typeof AdvancedSelectorDecisionSchema>;
+/**
+ * The model's synthesized selector: one that locates the target by anchoring on
+ * nearby env-derived text (e.g. a card's title), or null if it can't. Validated
+ * against the live page before being accepted.
+ */
+export declare const SynthesizedSelectorSchema: z.ZodObject<{
+    selector: z.ZodNullable<z.ZodString>;
+}, z.core.$strip>;
+/**
+ * The slice of {@link ToolCallContext} this routine needs: the env data and the
+ * action's references (to know what was baked in), plus the GPT client and flow
+ * metadata to run + account for the one model call. Callers pass their
+ * `ToolCallContext` directly.
+ */
+export type AdvancedSelectorContext = Pick<ToolCallContext, 'gptClient' | 'metadata' | 'envData' | 'rawParameters'>;
+/** A referenced env var and the forms of its value that appear in the candidates. */
+type BakedEnvForm = {
+    name: string;
+    value: string;
+    forms: Array<{
+        filter: string | null;
+        form: string;
+    }>;
+};
+/**
+ * Turn the element the agent acted on into the best *replayable* selector:
+ * generate raw candidates from the DOM, then (only for env-parameterized actions
+ * whose value got baked in) re-parameterize and prune them so the cached selector
+ * reflects the action's semantic intent rather than incidental DOM position.
+ *
+ * The DOM read and the model call are the only impurities and live here; the
+ * decision logic is delegated to the pure helpers below. Never returns null — it
+ * falls back to the raw generated selector whenever there is nothing to advance
+ * or the model's rewrite can't be validated.
+ */
+export declare function generateAdvancedSelectors(target: ElementHandle<HTMLElement | SVGElement>, context: AdvancedSelectorContext): Promise<ElementSelector>;
+/**
+ * For each referenced env var, the distinct forms of its value (verbatim or via a
+ * transform — e.g. "contact" / "Contact" / "/contact") that actually appear in
+ * the generated candidates. PURE.
+ */
+export declare function computeBakedEnvForms(candidates: string[], envData: Record<string, string>, referencedEnvVars: string[]): BakedEnvForm[];
+/**
+ * Build the model prompt: deterministic `form -> placeholder` hints (with the
+ * `| filter` to use for each transformed form) plus the candidate list. PURE.
+ */
+export declare function buildAdvancedSelectorRequest(candidates: string[], baked: BakedEnvForm[]): GptMessage[];
+/**
+ * Build the synthesis prompt: the target's neighborhood HTML (with the target
+ * marked) and the env values in play. PURE.
+ */
+export declare function buildSynthesisRequest(neighborhoodHtml: string, envMapping: string, markerAttribute: string): GptMessage[];
+/**
+ * Apply the model's decision and prune the result. PURE:
+ *
+ * 1. **Validate + templatize.** A rewrite is accepted only if interpolating it
+ *    with the current env values reproduces the exact original candidate — so the
+ *    model is bound to placeholder substitution; any other edit is rejected and
+ *    that candidate keeps its literal. A wrong-shaped decision is ignored wholesale.
+ * 2. **Elide.** Once a candidate tracks an env value, drop any purely-positional
+ *    candidate that does not: for a data-driven action, position is a coincidence
+ *    of the recorded data and would mislocate when the value (or order) changes.
+ *
+ * Never returns empty — falls back to the raw candidates if pruning would remove
+ * everything (and elision never fires unless a semantic, env-bearing candidate
+ * survives to take over).
+ */
+export declare function advanceSelectorCandidates(rawCandidates: string[], envData: Record<string, string>, decision: AdvancedSelectorDecision): string[];
+/**
+ * Once a candidate tracks an env value, drop purely-positional candidates that
+ * don't — for a data-driven action, position is a coincidence of the recorded
+ * data and would mislocate when the value (or order) changes. Returns the input
+ * unchanged when nothing is env-bearing (no semantic anchor to take over) or when
+ * pruning would remove everything. PURE.
+ */
+export declare function pruneCoincidentalPositional(candidates: string[]): string[];
+/** A selector that carries an env reference (so it tracks the value at replay). */
+export declare function isEnvBearingSelector(selector: string): boolean;
+/**
+ * Heuristic for a selector that locates an element *purely by DOM position* — an
+ * `:nth-*` pseudo, an XPath numeric index `[N]`, or a bare combinator chain — with
+ * no meaningful anchor (id, class, attribute, or text predicate). PURE.
+ *
+ * The "meaningful anchor" guard is what keeps a text/attribute selector that also
+ * happens to carry an index (e.g. `(.//a[normalize-space(.)='X'])[2]`) from being
+ * treated as positional.
+ */
+export declare function isPurelyPositionalSelector(selector: string): boolean;
+export {};
+//# sourceMappingURL=AdvancedSelectorGenerator.d.ts.map