@cutleryapp/agent 1.0.36 → 1.0.38

package/dist/mcp-executor.js

@@ -206,8 +206,129 @@ class TestExecutor {
                             handled = true;
                         }
                     }
+                    // 5b. Press key — keyboard actions
+                    if (!handled && (lower.startsWith("press ") || lower.startsWith("hit "))) {
+                        const keyMatch = raw.match(/(?:press|hit)\s+(.+)/i);
+                        if (keyMatch) {
+                            const keyName = keyMatch[1].trim();
+                            const keyMap = {
+                                enter: 'Enter', return: 'Enter', tab: 'Tab', escape: 'Escape', esc: 'Escape',
+                                space: 'Space', backspace: 'Backspace', delete: 'Delete', del: 'Delete',
+                                up: 'ArrowUp', down: 'ArrowDown', left: 'ArrowLeft', right: 'ArrowRight',
+                            };
+                            const key = keyMap[keyName.toLowerCase()] || keyName;
+                            await page.keyboard.press(key);
+                            handled = true;
+                        }
+                    }
+                    // 5c. Hover
+                    if (!handled && lower.startsWith("hover")) {
+                        const hoverMatch = raw.match(/hover\s+(?:over\s+)?(?:on\s+)?(?:the\s+)?"?([^"]+?)"?(?:\s+(?:button|link|element|icon))?$/i);
+                        if (hoverMatch) {
+                            const target = hoverMatch[1].trim();
+                            const nameRe = new RegExp(escapeRegex(target), 'i');
+                            for (const fn of [
+                                () => page.getByRole('button', { name: nameRe }).first().hover({ timeout: 1500 }),
+                                () => page.getByRole('link', { name: nameRe }).first().hover({ timeout: 1500 }),
+                                () => page.getByText(nameRe).first().hover({ timeout: 1500 }),
+                                () => page.locator(`[aria-label="${target}" i],[title="${target}" i]`).first().hover({ timeout: 1500 }),
+                            ]) {
+                                try {
+                                    await fn();
+                                    handled = true;
+                                    break;
+                                }
+                                catch { /* next */ }
+                            }
+                        }
+                    }
+                    // 5d. Scroll
+                    if (!handled && lower.includes("scroll")) {
+                        if (/scroll\s+(?:to\s+)?(?:the\s+)?(?:top|beginning)/i.test(raw)) {
+                            await page.evaluate(() => window.scrollTo(0, 0));
+                            handled = true;
+                        }
+                        else if (/scroll\s+(?:to\s+)?(?:the\s+)?(?:bottom|end)/i.test(raw)) {
+                            await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+                            handled = true;
+                        }
+                        else if (/scroll\s+down/i.test(raw)) {
+                            const pxMatch = raw.match(/(\d+)/);
+                            await page.evaluate((px) => window.scrollBy(0, px), pxMatch ? parseInt(pxMatch[1]) : 400);
+                            handled = true;
+                        }
+                        else if (/scroll\s+up/i.test(raw)) {
+                            const pxMatch = raw.match(/(\d+)/);
+                            await page.evaluate((px) => window.scrollBy(0, -px), pxMatch ? parseInt(pxMatch[1]) : 400);
+                            handled = true;
+                        }
+                        else {
+                            const scrollToMatch = raw.match(/scroll\s+(?:to\s+)?(?:the\s+)?"?([^"]+?)"?\s*(?:element|section|field)?$/i);
+                            if (scrollToMatch) {
+                                const target = scrollToMatch[1].trim();
+                                try {
+                                    await page.getByText(new RegExp(escapeRegex(target), 'i')).first().scrollIntoViewIfNeeded({ timeout: 2000 });
+                                    handled = true;
+                                }
+                                catch { /* fall through */ }
+                            }
+                        }
+                    }
+                    // 5e. Double click
+                    if (!handled && lower.startsWith("double click")) {
+                        const dcMatch = raw.match(/double\s+click\s+(?:on\s+)?(?:the\s+)?"?([^"]+?)"?$/i);
+                        if (dcMatch) {
+                            const target = dcMatch[1].trim();
+                            const nameRe = new RegExp(escapeRegex(target), 'i');
+                            for (const fn of [
+                                () => page.getByRole('button', { name: nameRe }).first().dblclick({ timeout: 1500 }),
+                                () => page.getByText(nameRe).first().dblclick({ timeout: 1500 }),
+                                () => page.locator(`[aria-label="${target}" i]`).first().dblclick({ timeout: 1500 }),
+                            ]) {
+                                try {
+                                    await fn();
+                                    handled = true;
+                                    break;
+                                }
+                                catch { /* next */ }
+                            }
+                        }
+                    }
+                    // 5f. Clear field
+                    if (!handled && (lower.startsWith("clear ") || lower.includes(" clear the "))) {
+                        const clearMatch = raw.match(/clear\s+(?:the\s+)?"?([^"]+?)"?\s*(?:field|input|box)?$/i);
+                        if (clearMatch) {
+                            const fieldLabel = clearMatch[1].trim();
+                            try {
+                                await tryFill(page, fieldLabel, '');
+                                handled = true;
+                            }
+                            catch { /* fall through */ }
+                        }
+                    }
+                    // 5g. Upload file — "upload X to Y" / "choose X in Y field" where X looks like a path
+                    if (!handled && (lower.includes("upload") || (lower.includes("choose") && /\.(jpg|jpeg|png|gif|pdf|csv|xlsx?|docx?|zip|txt)/i.test(raw)))) {
+                        const uploadMatch = raw.match(/(?:upload|choose|attach)\s+"?([^"]+?)"?\s+(?:to|in|into)\s+"?([^"]+?)"?\s*(?:field|input)?$/i) ||
+                            raw.match(/(?:upload|choose|attach)\s+"?([^"]+\.\w+)"?\s+(?:to|in|into)?\s*"?([^"]+?)"?$/i);
+                        if (uploadMatch) {
+                            const filePath = uploadMatch[1].trim();
+                            const fieldLabel = uploadMatch[2].trim();
+                            const fileLocators = [
+                                page.getByLabel(new RegExp(escapeRegex(fieldLabel), 'i')),
+                                page.locator(`input[type="file"]`),
+                            ];
+                            for (const loc of fileLocators) {
+                                try {
+                                    await loc.first().setInputFiles(filePath, { timeout: 3000 });
+                                    handled = true;
+                                    break;
+                                }
+                                catch { /* try next */ }
+                            }
+                        }
+                    }
                     // 6. Select — native dropdown, then React-select/autocomplete fallback
-                    if (!handled && (lower.includes("select") || lower.includes("choose"))) {
+                    if (!handled && (lower.includes("select") || (lower.includes("choose") && !/\.(jpg|jpeg|png|gif|pdf|csv|xlsx?|docx?|zip|txt)/i.test(raw)))) {
                         const selMatch = raw.match(/(?:select|choose)\s+"?([^"]+?)"?\s+(?:from|in)\s+"?([^"]+?)"?\s*(?:dropdown|select|field)?$/i);
                         if (selMatch) {
                             const optionValue = selMatch[1].trim();
@@ -270,29 +391,29 @@ class TestExecutor {
                             }
                         }
                     }
-                    // 7. AI — single-shot for deterministic steps, full loop for intent steps
+                    // 7. Generic click fallback — try any element containing the step text before AI
                     if (!handled) {
-                        const isDeterministic = /^(click|fill|enter|type|verify|check|assert|select|choose|wait|hover|scroll)/i.test(lower.trim());
-                        if (isDeterministic) {
-                            console.log(`  🤖 Quick AI selector lookup for: "${raw}"`);
-                            await aiSingleShot(page, raw);
-                        }
-                        else {
-                            console.log(`  🤖 AI intent loop for: "${raw}"`);
-                            await aiStepFallback(page, raw, null);
+                        // Extract the most meaningful noun phrase from the step
+                        const nounMatch = raw.match(/(?:click|press|tap|submit|open|close|expand|collapse|toggle|activate|dismiss|confirm|cancel|accept|reject|approve|deny|enable|disable|show|hide)\s+(?:on\s+|the\s+)?(?:the\s+)?"?([^"]+?)"?(?:\s+(?:button|link|tab|icon|menu|modal|dialog|popup|dropdown|option))?$/i);
+                        if (nounMatch) {
+                            const target = nounMatch[1].trim();
+                            const nameRe = new RegExp(escapeRegex(target), 'i');
+                            const clicked = await tryClick(page, nameRe, target);
+                            if (clicked)
+                                handled = true;
                         }
                     }
-                }
-                catch (err) {
-                    // MCP execution failed — single-shot AI recovery, no loop
-                    console.log(`  ⚠️  MCP step failed (${err.message.split('\n')[0]}), trying AI...`);
-                    try {
+                    // 8. AI — last resort only, single-shot for deterministic steps
+                    if (!handled) {
+                        console.log(`  🤖 AI fallback for: "${raw}"`);
                         await aiSingleShot(page, raw);
                     }
-                    catch (aiErr) {
-                        stepError = err.message;
-                        result.success = false;
-                    }
+                }
+                catch (err) {
+                    // Log the error — do NOT call AI here, it's too slow and usually can't recover either
+                    console.log(`  ⚠️  Step failed: ${err.message.split('\n')[0]}`);
+                    stepError = err.message;
+                    result.success = false;
                 }
                 // Screenshot after each step
                 let screenshotB64 = "";
@@ -765,9 +886,7 @@ async function tryAIClick(page, selector) {
 }
 /** Select an option from a React-select / autocomplete / combobox / dropdown */
 async function tryAutocomplete(page, fieldLabel, value) {
-    const esc = (s) => s.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
-    const labelRe = new RegExp(esc(fieldLabel), 'i');
-    // Click an option from whatever dropdown is currently open on the page
+    // Click the matching option in whatever dropdown is currently open — returns true only on success
     async function clickOpenOption() {
         const optionSelectors = [
             `[role="option"]:has-text("${value}")`,
@@ -789,38 +908,57 @@ async function tryAutocomplete(page, fieldLabel, value) {
         }
         return false;
     }
-    // Strategy 1: React-select — click the control div, type into its hidden input
-    const reactControlSelectors = [
-        `[class*="react-select__control"]`,
-        `[class*="select__control"]`,
-        `[class*="Select__control"]`,
-    ];
-    for (const ctrlSel of reactControlSelectors) {
-        try {
-            // Find a React-select control near the label text
-            // Find label element, then look for the control near it via DOM traversal
-            const labelEl = page.getByText(labelRe, { exact: false }).first();
-            if (!await labelEl.isVisible({ timeout: 500 }).catch(() => false))
-                continue;
-            // Try sibling/parent-scoped control
-            const ctrl = page.locator(ctrlSel);
-            if (await ctrl.count() === 0)
-                continue;
-            await ctrl.first().click({ timeout: 2000 });
-            // The hidden input inside the control is now active
-            const innerInput = page.locator(`[class*="react-select__input"] input, [class*="select__input"] input`).first();
-            await innerInput.type(value, { delay: 60 });
-            await page.waitForTimeout(500);
+    // Strategy 1: React-select — find the control that BELONGS to this label via DOM traversal
+    // so we don't accidentally open the wrong dropdown on a page with multiple selects
+    try {
+        const opened = await page.evaluate((lbl) => {
+            const allEls = Array.from(document.querySelectorAll('label, legend, [class*="label"]'));
+            const labelEl = allEls.find(el => (el.textContent || '').trim().toLowerCase().includes(lbl.toLowerCase()));
+            if (!labelEl)
+                return false;
+            // Walk up ancestors looking for a container that holds a react-select control
+            let ancestor = labelEl.parentElement;
+            for (let i = 0; i < 6 && ancestor; i++) {
+                const ctrl = ancestor.querySelector('[class*="react-select__control"],[class*="select__control"],[class*="Select__control"]');
+                if (ctrl) {
+                    ctrl.click();
+                    return true;
+                }
+                ancestor = ancestor.parentElement;
+            }
+            // Try label[for] → wrapper containing a react-select
+            if (labelEl.tagName === 'LABEL') {
+                const forId = labelEl.htmlFor;
+                const wrapper = forId ? document.getElementById(forId) : null;
+                const ctrl = wrapper?.querySelector('[class*="control"]');
+                if (ctrl) {
+                    ctrl.click();
+                    return true;
+                }
+            }
+            return false;
+        }, fieldLabel);
+        if (opened) {
+            await page.waitForTimeout(300);
+            // Type into the now-visible input inside the react-select
+            const innerInput = page.locator('[class*="react-select__input"] input,[class*="select__input"] input').first();
+            try {
+                await innerInput.waitFor({ state: 'visible', timeout: 800 });
+                await innerInput.type(value, { delay: 60 });
+            }
+            catch {
+                await page.keyboard.type(value, { delay: 60 });
+            }
+            await page.waitForTimeout(600);
             if (await clickOpenOption())
                 return true;
-            // If specific option not found, press Enter
-            await page.keyboard.press('Enter');
-            await page.waitForTimeout(200);
-            return true;
+            // No confirmed click → don't claim success, fall through to next strategy
         }
-        catch { /* try next */ }
     }
-    // Strategy 2: combobox / input by label or placeholder — type and pick option
+    catch { /* DOM eval failed, try next */ }
+    // Strategy 2: combobox / text input by label or placeholder — type + pick option
+    const esc = (s) => s.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
+    const labelRe = new RegExp(esc(fieldLabel), 'i');
     const inputLocators = [
         page.getByRole('combobox', { name: labelRe }),
         page.getByLabel(labelRe),
@@ -829,8 +967,8 @@ async function tryAutocomplete(page, fieldLabel, value) {
     for (const loc of inputLocators) {
         try {
             const input = loc.first();
+            // Skip wrapper divs — getByLabel can return the React-select container div
             const tag = await input.evaluate((el) => el.tagName.toLowerCase()).catch(() => '');
-            // Skip non-input elements (e.g. wrapper divs returned by getByLabel for React-select)
             if (tag && !['input', 'textarea'].includes(tag))
                 continue;
             await input.waitFor({ state: 'visible', timeout: 1500 });
@@ -840,9 +978,7 @@ async function tryAutocomplete(page, fieldLabel, value) {
             await page.waitForTimeout(500);
             if (await clickOpenOption())
                 return true;
-            await page.keyboard.press('Enter');
-            await page.waitForTimeout(200);
-            return true;
+            // Only count success when the option was actually clicked
         }
         catch { /* try next */ }
     }
@@ -995,7 +1131,7 @@ async function tryClickScoped(page, nameRe, target, scope) {
     return false;
 }
 async function tryFill(page, label, value) {
-    const FAST = 800;
+    const FAST = 500;
     const labelRe = new RegExp(escapeRegex(label), "i");
     const variants = labelVariants(label);
     const attrContains = (attr) => variants
@@ -1047,14 +1183,6 @@ async function tryFill(page, label, value) {
             errors.push(e?.message?.split("\n")[0] || String(e));
         }
     }
-    // Autocomplete fallback — type + wait for dropdown + click option
-    const acSuccess = await tryAutocomplete(page, label, value);
-    if (acSuccess)
-        return;
-    // AI vision fallback
-    const aiSuccess = await aiFillFallback(page, label, value);
-    if (aiSuccess)
-        return;
     throw new Error(`Could not find input field: "${label}". Tried ${strategies.length} strategies.`);
 }
 /** Token-aware variant generation matching executor.ts/labelVariants. */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@cutleryapp/agent",
-    "version": "1.0.36",
+    "version": "1.0.38",
     "description": "Local agent that connects your machine to the Cutlery QA platform and runs UI tests via Playwright",
     "main": "dist/cli.js",
     "bin": {