@cutleryapp/agent 1.0.37 → 1.0.39

package/dist/mcp-executor.js

@@ -206,8 +206,129 @@ class TestExecutor {
                             handled = true;
                         }
                     }
+                    // 5b. Press key — keyboard actions
+                    if (!handled && (lower.startsWith("press ") || lower.startsWith("hit "))) {
+                        const keyMatch = raw.match(/(?:press|hit)\s+(.+)/i);
+                        if (keyMatch) {
+                            const keyName = keyMatch[1].trim();
+                            const keyMap = {
+                                enter: 'Enter', return: 'Enter', tab: 'Tab', escape: 'Escape', esc: 'Escape',
+                                space: 'Space', backspace: 'Backspace', delete: 'Delete', del: 'Delete',
+                                up: 'ArrowUp', down: 'ArrowDown', left: 'ArrowLeft', right: 'ArrowRight',
+                            };
+                            const key = keyMap[keyName.toLowerCase()] || keyName;
+                            await page.keyboard.press(key);
+                            handled = true;
+                        }
+                    }
+                    // 5c. Hover
+                    if (!handled && lower.startsWith("hover")) {
+                        const hoverMatch = raw.match(/hover\s+(?:over\s+)?(?:on\s+)?(?:the\s+)?"?([^"]+?)"?(?:\s+(?:button|link|element|icon))?$/i);
+                        if (hoverMatch) {
+                            const target = hoverMatch[1].trim();
+                            const nameRe = new RegExp(escapeRegex(target), 'i');
+                            for (const fn of [
+                                () => page.getByRole('button', { name: nameRe }).first().hover({ timeout: 1500 }),
+                                () => page.getByRole('link', { name: nameRe }).first().hover({ timeout: 1500 }),
+                                () => page.getByText(nameRe).first().hover({ timeout: 1500 }),
+                                () => page.locator(`[aria-label="${target}" i],[title="${target}" i]`).first().hover({ timeout: 1500 }),
+                            ]) {
+                                try {
+                                    await fn();
+                                    handled = true;
+                                    break;
+                                }
+                                catch { /* next */ }
+                            }
+                        }
+                    }
+                    // 5d. Scroll
+                    if (!handled && lower.includes("scroll")) {
+                        if (/scroll\s+(?:to\s+)?(?:the\s+)?(?:top|beginning)/i.test(raw)) {
+                            await page.evaluate(() => window.scrollTo(0, 0));
+                            handled = true;
+                        }
+                        else if (/scroll\s+(?:to\s+)?(?:the\s+)?(?:bottom|end)/i.test(raw)) {
+                            await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+                            handled = true;
+                        }
+                        else if (/scroll\s+down/i.test(raw)) {
+                            const pxMatch = raw.match(/(\d+)/);
+                            await page.evaluate((px) => window.scrollBy(0, px), pxMatch ? parseInt(pxMatch[1]) : 400);
+                            handled = true;
+                        }
+                        else if (/scroll\s+up/i.test(raw)) {
+                            const pxMatch = raw.match(/(\d+)/);
+                            await page.evaluate((px) => window.scrollBy(0, -px), pxMatch ? parseInt(pxMatch[1]) : 400);
+                            handled = true;
+                        }
+                        else {
+                            const scrollToMatch = raw.match(/scroll\s+(?:to\s+)?(?:the\s+)?"?([^"]+?)"?\s*(?:element|section|field)?$/i);
+                            if (scrollToMatch) {
+                                const target = scrollToMatch[1].trim();
+                                try {
+                                    await page.getByText(new RegExp(escapeRegex(target), 'i')).first().scrollIntoViewIfNeeded({ timeout: 2000 });
+                                    handled = true;
+                                }
+                                catch { /* fall through */ }
+                            }
+                        }
+                    }
+                    // 5e. Double click
+                    if (!handled && lower.startsWith("double click")) {
+                        const dcMatch = raw.match(/double\s+click\s+(?:on\s+)?(?:the\s+)?"?([^"]+?)"?$/i);
+                        if (dcMatch) {
+                            const target = dcMatch[1].trim();
+                            const nameRe = new RegExp(escapeRegex(target), 'i');
+                            for (const fn of [
+                                () => page.getByRole('button', { name: nameRe }).first().dblclick({ timeout: 1500 }),
+                                () => page.getByText(nameRe).first().dblclick({ timeout: 1500 }),
+                                () => page.locator(`[aria-label="${target}" i]`).first().dblclick({ timeout: 1500 }),
+                            ]) {
+                                try {
+                                    await fn();
+                                    handled = true;
+                                    break;
+                                }
+                                catch { /* next */ }
+                            }
+                        }
+                    }
+                    // 5f. Clear field
+                    if (!handled && (lower.startsWith("clear ") || lower.includes(" clear the "))) {
+                        const clearMatch = raw.match(/clear\s+(?:the\s+)?"?([^"]+?)"?\s*(?:field|input|box)?$/i);
+                        if (clearMatch) {
+                            const fieldLabel = clearMatch[1].trim();
+                            try {
+                                await tryFill(page, fieldLabel, '');
+                                handled = true;
+                            }
+                            catch { /* fall through */ }
+                        }
+                    }
+                    // 5g. Upload file — "upload X to Y" / "choose X in Y field" where X looks like a path
+                    if (!handled && (lower.includes("upload") || (lower.includes("choose") && /\.(jpg|jpeg|png|gif|pdf|csv|xlsx?|docx?|zip|txt)/i.test(raw)))) {
+                        const uploadMatch = raw.match(/(?:upload|choose|attach)\s+"?([^"]+?)"?\s+(?:to|in|into)\s+"?([^"]+?)"?\s*(?:field|input)?$/i) ||
+                            raw.match(/(?:upload|choose|attach)\s+"?([^"]+\.\w+)"?\s+(?:to|in|into)?\s*"?([^"]+?)"?$/i);
+                        if (uploadMatch) {
+                            const filePath = uploadMatch[1].trim();
+                            const fieldLabel = uploadMatch[2].trim();
+                            const fileLocators = [
+                                page.getByLabel(new RegExp(escapeRegex(fieldLabel), 'i')),
+                                page.locator(`input[type="file"]`),
+                            ];
+                            for (const loc of fileLocators) {
+                                try {
+                                    await loc.first().setInputFiles(filePath, { timeout: 3000 });
+                                    handled = true;
+                                    break;
+                                }
+                                catch { /* try next */ }
+                            }
+                        }
+                    }
                     // 6. Select — native dropdown, then React-select/autocomplete fallback
-                    if (!handled && (lower.includes("select") || lower.includes("choose"))) {
+                    if (!handled && (lower.includes("select") || (lower.includes("choose") && !/\.(jpg|jpeg|png|gif|pdf|csv|xlsx?|docx?|zip|txt)/i.test(raw)))) {
                         const selMatch = raw.match(/(?:select|choose)\s+"?([^"]+?)"?\s+(?:from|in)\s+"?([^"]+?)"?\s*(?:dropdown|select|field)?$/i);
                         if (selMatch) {
                             const optionValue = selMatch[1].trim();
@@ -216,7 +337,7 @@ class TestExecutor {
                             // Try native <select>
                             try {
                                 const fieldLoc = page.getByLabel(new RegExp(fieldLabel, 'i')).first();
-                                await fieldLoc.selectOption({ label: optionValue }, { timeout: 3000 });
+                                await fieldLoc.selectOption({ label: optionValue }, { timeout: 800 });
                                 selHandled = true;
                             }
                             catch { /* not a native select */ }
@@ -270,38 +391,41 @@ class TestExecutor {
                             }
                         }
                     }
-                    // 7. AI — single-shot for deterministic steps, full loop for intent steps
+                    // 7. Generic click fallback — try any element containing the step text before AI
                     if (!handled) {
-                        const isDeterministic = /^(click|fill|enter|type|verify|check|assert|select|choose|wait|hover|scroll)/i.test(lower.trim());
-                        if (isDeterministic) {
-                            console.log(`  🤖 Quick AI selector lookup for: "${raw}"`);
-                            await aiSingleShot(page, raw);
-                        }
-                        else {
-                            console.log(`  🤖 AI intent loop for: "${raw}"`);
-                            await aiStepFallback(page, raw, null);
+                        // Extract the most meaningful noun phrase from the step
+                        const nounMatch = raw.match(/(?:click|press|tap|submit|open|close|expand|collapse|toggle|activate|dismiss|confirm|cancel|accept|reject|approve|deny|enable|disable|show|hide)\s+(?:on\s+|the\s+)?(?:the\s+)?"?([^"]+?)"?(?:\s+(?:button|link|tab|icon|menu|modal|dialog|popup|dropdown|option))?$/i);
+                        if (nounMatch) {
+                            const target = nounMatch[1].trim();
+                            const nameRe = new RegExp(escapeRegex(target), 'i');
+                            const clicked = await tryClick(page, nameRe, target);
+                            if (clicked)
+                                handled = true;
                         }
                     }
-                }
-                catch (err) {
-                    // MCP execution failed — single-shot AI recovery, no loop
-                    console.log(`  ⚠️  MCP step failed (${err.message.split('\n')[0]}), trying AI...`);
-                    try {
+                    // 8. AI — last resort only, single-shot for deterministic steps
+                    if (!handled) {
+                        console.log(`  🤖 AI fallback for: "${raw}"`);
                         await aiSingleShot(page, raw);
                     }
-                    catch (aiErr) {
-                        stepError = err.message;
-                        result.success = false;
-                    }
                 }
-                // Screenshot after each step
+                catch (err) {
+                    // Log the error — do NOT call AI here, it's too slow and usually can't recover either
+                    console.log(`  ⚠️  Step failed: ${err.message.split('\n')[0]}`);
+                    stepError = err.message;
+                    result.success = false;
+                }
+                // Screenshot on failure, on the last step, or every 5 steps — not every step
                 let screenshotB64 = "";
-                try {
-                    const buf = await page.screenshot({ fullPage: false });
-                    screenshotB64 = buf.toString("base64");
-                    result.screenshots.push(screenshotB64);
+                const isLastStep = i === steps.length - 1;
+                if (stepError || isLastStep || i % 5 === 0) {
+                    try {
+                        const buf = await page.screenshot({ fullPage: false });
+                        screenshotB64 = buf.toString("base64");
+                        result.screenshots.push(screenshotB64);
+                    }
+                    catch { /* ignore screenshot errors */ }
                 }
-                catch { /* ignore screenshot errors */ }
                 result.steps.push({
                     step: raw,
                     action: raw,
@@ -347,24 +471,32 @@ function extractSelector(step, pattern) {
 }
 // Fast probe: try each locator strategy with a short timeout so fallbacks don't stall
 async function tryClick(page, nameRe, label) {
-    const FAST = 800;
-    const strategies = [
-        () => page.getByRole('button', { name: nameRe }).first().click({ timeout: FAST }),
-        () => page.getByRole('link', { name: nameRe }).first().click({ timeout: FAST }),
-        () => page.getByText(nameRe).first().click({ timeout: FAST }),
-        () => page.locator(`[value="${label}"], [aria-label="${label}"], [title="${label}"]`).first().click({ timeout: FAST }),
-        // data-* attributes (common in test automation)
-        () => page.locator(`[data-test*="${label}" i], [data-testid*="${label}" i], [id*="${label}" i]`).first().click({ timeout: FAST }),
+    // Race all common role/text strategies simultaneously — first one wins
+    const T = 1000;
+    const raceStrategies = [
+        page.getByRole('button', { name: nameRe }).first().click({ timeout: T }),
+        page.getByRole('link', { name: nameRe }).first().click({ timeout: T }),
+        page.getByRole('tab', { name: nameRe }).first().click({ timeout: T }),
+        page.getByText(nameRe, { exact: false }).first().click({ timeout: T }),
+        page.locator(`[aria-label="${label}" i],[title="${label}" i],[data-test*="${label}" i],[data-testid*="${label}" i]`).first().click({ timeout: T }),
     ];
-    for (const fn of strategies) {
+    try {
+        await Promise.any(raceStrategies);
+        return true;
+    }
+    catch { /* all failed, try sequential fallbacks */ }
+    // Sequential fallbacks for id/value/role selectors
+    for (const fn of [
+        () => page.locator(`#${label.replace(/\s+/g, '-')}, #${label.replace(/\s+/g, '_')}, [id*="${label}" i]`).first().click({ timeout: 500 }),
+        () => page.locator(`input[value="${label}" i], button[value="${label}" i]`).first().click({ timeout: 500 }),
+    ]) {
         try {
             await fn();
             return true;
         }
-        catch { /* try next */ }
+        catch { /* next */ }
     }
-    // AI vision fallback
-    return await aiClickFallback(page, label);
+    return false;
 }
 function buildAgentPrompt(stepText, round, hasAttachment = false) {
     const attachmentSection = hasAttachment ? `
@@ -823,12 +955,12 @@ async function tryAutocomplete(page, fieldLabel, value) {
             const innerInput = page.locator('[class*="react-select__input"] input,[class*="select__input"] input').first();
             try {
                 await innerInput.waitFor({ state: 'visible', timeout: 800 });
-                await innerInput.type(value, { delay: 60 });
+                await innerInput.type(value, { delay: 30 });
             }
             catch {
-                await page.keyboard.type(value, { delay: 60 });
+                await page.keyboard.type(value, { delay: 30 });
             }
-            await page.waitForTimeout(600);
+            await page.waitForTimeout(400);
             if (await clickOpenOption())
                 return true;
             // No confirmed click → don't claim success, fall through to next strategy
@@ -853,11 +985,10 @@ async function tryAutocomplete(page, fieldLabel, value) {
             await input.waitFor({ state: 'visible', timeout: 1500 });
             await input.click({ timeout: 1500 });
             await input.fill('');
-            await input.type(value, { delay: 60 });
-            await page.waitForTimeout(500);
+            await input.type(value, { delay: 30 });
+            await page.waitForTimeout(350);
             if (await clickOpenOption())
                 return true;
-            // Only count success when the option was actually clicked
         }
         catch { /* try next */ }
     }
@@ -956,7 +1087,7 @@ async function aiFillFallback(page, label, value) {
     }
 }
 async function tryClickScoped(page, nameRe, target, scope) {
-    const FAST = 3000;
+    const FAST = 1000;
     // Strip trailing generic nouns that won't appear verbatim on the page
     const cleanScope = scope.replace(/\s+(?:product|item|section|card|row|container|element|button|link|area|panel|block)$/i, '').trim();
     // Use card/item container selectors — these are tight enough to contain the button
@@ -1010,67 +1141,44 @@ async function tryClickScoped(page, nameRe, target, scope) {
     return false;
 }
 async function tryFill(page, label, value) {
-    const FAST = 800;
     const labelRe = new RegExp(escapeRegex(label), "i");
     const variants = labelVariants(label);
-    const attrContains = (attr) => variants
-        .map((v) => `input[${attr}*="${cssEscape(v)}" i], textarea[${attr}*="${cssEscape(v)}" i], [contenteditable="true"][${attr}*="${cssEscape(v)}" i]`)
-        .join(", ");
-    // Ordered most-likely-to-resolve first so the success path matches the old
-    // 4-strategy implementation in latency. The extra strategies only run when
-    // the cheaper ones miss (i.e. the field genuinely needs fuzzier matching).
-    const strategies = [
-        // 1–2. Same as before — covers labelled inputs and placeholder-only inputs.
+    const attrContains = (attr) => variants.map(v => `input[${attr}*="${cssEscape(v)}" i], textarea[${attr}*="${cssEscape(v)}" i], [contenteditable="true"][${attr}*="${cssEscape(v)}" i]`).join(", ");
+    function withTimeout(fn, ms) {
+        return Promise.race([fn(), new Promise((_, r) => setTimeout(() => r(new Error('timeout')), ms))]);
+    }
+    // Phase 1: race the 3 highest-coverage strategies simultaneously (500ms cap)
+    // These cover labelled inputs, placeholder inputs, and role-textbox — 95%+ of forms
+    const phase1 = [
         () => page.getByLabel(labelRe).first().fill(value),
         () => page.getByPlaceholder(labelRe).first().fill(value),
-        // 3. Accessible role match.
         () => page.getByRole("textbox", { name: labelRe }).first().fill(value),
-        // 4. Common automation hooks.
+    ];
+    try {
+        await Promise.any(phase1.map(fn => withTimeout(fn, 500)));
+        return;
+    }
+    catch { /* all 3 failed, try phase 2 */ }
+    // Phase 2: attribute-based selectors covering name/id/data-test variants (300ms each)
+    const exactAttrs = variants.flatMap(v => [
+        `input[name="${cssEscape(v)}" i]`, `input[id="${cssEscape(v)}" i]`,
+        `textarea[name="${cssEscape(v)}" i]`, `textarea[id="${cssEscape(v)}" i]`,
+    ]).join(", ");
+    const phase2 = [
+        () => page.locator(exactAttrs).first().fill(value),
+        () => page.locator(`${attrContains("name")}, ${attrContains("id")}`).first().fill(value),
         () => page.locator(attrContains("data-test")).first().fill(value),
-        () => page.locator(attrContains("data-testid")).first().fill(value),
-        // 5. Native attributes — exact across all variants (kebab/snake/camel/etc.).
-        () => page
-            .locator(variants
-            .flatMap((v) => [
-            `input[name="${cssEscape(v)}" i]`,
-            `input[id="${cssEscape(v)}" i]`,
-            `textarea[name="${cssEscape(v)}" i]`,
-            `textarea[id="${cssEscape(v)}" i]`,
-        ])
-            .join(", "))
-            .first()
-            .fill(value),
-        // 6. Native attributes — contains across all variants.
-        () => page
-            .locator(`${attrContains("name")}, ${attrContains("id")}`)
-            .first()
-            .fill(value),
-        // 7. ARIA / placeholder fallbacks.
         () => page.locator(attrContains("aria-label")).first().fill(value),
         () => page.locator(attrContains("placeholder")).first().fill(value),
     ];
-    const errors = [];
-    for (const fn of strategies) {
+    for (const fn of phase2) {
         try {
-            await Promise.race([
-                fn(),
-                new Promise((_, r) => setTimeout(() => r(new Error("timeout")), FAST)),
-            ]);
+            await withTimeout(fn, 300);
             return;
         }
-        catch (e) {
-            errors.push(e?.message?.split("\n")[0] || String(e));
-        }
+        catch { /* next */ }
     }
-    // Autocomplete fallback — type + wait for dropdown + click option
-    const acSuccess = await tryAutocomplete(page, label, value);
-    if (acSuccess)
-        return;
-    // AI vision fallback
-    const aiSuccess = await aiFillFallback(page, label, value);
-    if (aiSuccess)
-        return;
-    throw new Error(`Could not find input field: "${label}". Tried ${strategies.length} strategies.`);
+    throw new Error(`Could not find input field: "${label}"`);
 }
 /** Token-aware variant generation matching executor.ts/labelVariants. */
 function labelVariants(label) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@cutleryapp/agent",
-    "version": "1.0.37",
+    "version": "1.0.39",
     "description": "Local agent that connects your machine to the Cutlery QA platform and runs UI tests via Playwright",
     "main": "dist/cli.js",
     "bin": {