@cutleryapp/agent 1.0.20 → 1.0.22

package/dist/mcp-executor.js

@@ -60,29 +60,109 @@ class TestExecutor {
                 });
                 let stepError;
                 try {
-                    // Navigate is handled directly — URL extraction doesn't need vision
+                    let handled = false;
+                    // 1. Navigate — direct URL goto, no selector needed
                     if (lower.includes("navigate to") || lower.includes("go to")) {
                         const urlMatch = raw.match(/(?:navigate\s+to|go\s+to)\s+(https?:\/\/\S+|\/\S*|\S+\.\S+)/i);
                         if (urlMatch) {
                             let url = urlMatch[1].trim();
-                            if (url.startsWith("/") && this.options.baseUrl) {
+                            if (url.startsWith("/") && this.options.baseUrl)
                                 url = this.options.baseUrl.replace(/\/$/, "") + url;
-                            }
                             await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
+                            handled = true;
                         }
                         else if (this.options.baseUrl) {
                             await page.goto(this.options.baseUrl, { waitUntil: "domcontentloaded", timeout: 30000 });
+                            handled = true;
                         }
                     }
-                    else {
-                        // ALL other steps: AI reads the screen and performs the action
-                        console.log(`  🤖 AI executing: "${raw}"`);
+                    // 2. Click — smart selector strategies via MCP/Playwright
+                    if (!handled && lower.includes("click")) {
+                        const labelMatch = raw.match(/click\s+(?:on\s+)?(?:the\s+)?"?([^"]+?)"?(?:\s+(?:button|link|tab))?$/i);
+                        const label = labelMatch?.[1]?.trim();
+                        if (label) {
+                            const scopeMatch = label.match(/^(.+?)\s+(?:under|inside|within|in the|in)\s+(.+)$/i);
+                            const target = scopeMatch ? scopeMatch[1].trim() : label;
+                            const scope = scopeMatch ? scopeMatch[2].trim() : null;
+                            const nameRe = new RegExp(escapeRegex(target), 'i');
+                            const clicked = scope
+                                ? await tryClickScoped(page, nameRe, target, scope)
+                                : await tryClick(page, nameRe, target);
+                            if (clicked)
+                                handled = true;
+                        }
+                    }
+                    // 3. Fill — smart selector strategies via MCP/Playwright
+                    if (!handled && (lower.includes("fill") || lower.includes("type") || lower.includes("enter"))) {
+                        const match = raw.match(/(?:enter|fill|type)\s+"([^"]+)"\s+(?:in|into)\s+(?:the\s+)?"?([^"]+?)"?\s*(?:field|input|box|area)?\s*$/i) ||
+                            raw.match(/(?:enter|fill|type)\s+(\S+)\s+(?:in|into)\s+(?:the\s+)?(.+?)\s*(?:field|input|box|area)?\s*$/i);
+                        if (match) {
+                            const value = match[1].trim();
+                            const fieldLabel = match[2].trim();
+                            const looksLikeCss = (s) => /[#.\[\]:>]/.test(s);
+                            if (looksLikeCss(fieldLabel)) {
+                                await page.waitForSelector(fieldLabel, { state: "visible", timeout: 5000 });
+                                await page.fill(fieldLabel, value);
+                            }
+                            else {
+                                await tryFill(page, fieldLabel, value);
+                            }
+                            handled = true;
+                        }
+                    }
+                    // 4. Verify — check page text
+                    if (!handled && (lower.includes("verify") || lower.includes("assert") || lower.includes("check") || lower.includes("should"))) {
+                        const isNegative = /not\s+(?:displayed|visible|present)/i.test(raw);
+                        const textMatch = raw.match(/"([^"]+)"/) ||
+                            raw.match(/(?:verify|check|assert)\s+(?:i\s+see\s+(?:text\s+)?|text\s+)?(.+?)(?:\s+is\s+(?:not\s+)?(?:displayed|visible|present))?$/i);
+                        if (textMatch) {
+                            const expected = textMatch[1].trim();
+                            if (isNegative) {
+                                const content = await page.textContent('body') || '';
+                                if (content.includes(expected))
+                                    throw new Error(`Text "${expected}" should NOT be visible`);
+                            }
+                            else {
+                                await page.waitForFunction((t) => document.body.innerText.includes(t), expected, { timeout: 10000 });
+                            }
+                            handled = true;
+                        }
+                    }
+                    // 5. Wait — simple timeout
+                    if (!handled && lower.includes("wait")) {
+                        const ms = raw.match(/wait\s+(\d+)\s*(?:second|ms|millisecond)/i);
+                        if (ms) {
+                            await page.waitForTimeout(parseInt(ms[1]) * (raw.toLowerCase().includes('ms') ? 1 : 1000));
+                            handled = true;
+                        }
+                    }
+                    // 6. Select — dropdown
+                    if (!handled && (lower.includes("select") || lower.includes("choose"))) {
+                        const selMatch = raw.match(/select\s+"?([^"]+?)"?\s+(?:from|in)\s+"?([^"]+?)"?\s*(?:dropdown|select|field)?$/i);
+                        if (selMatch) {
+                            try {
+                                await page.selectOption(selMatch[2].trim(), { label: selMatch[1].trim() });
+                                handled = true;
+                            }
+                            catch { /* fall to AI */ }
+                        }
+                    }
+                    // 7. AI fallback — for anything not handled or ambiguous
+                    if (!handled) {
+                        console.log(`  🤖 MCP could not handle step, using AI: "${raw}"`);
                         await aiStepFallback(page, raw);
                     }
                 }
                 catch (err) {
-                    stepError = err.message;
-                    result.success = false;
+                    // MCP execution failed — let AI try to recover
+                    console.log(`  ⚠️  MCP step failed (${err.message}), trying AI...`);
+                    try {
+                        await aiStepFallback(page, raw);
+                    }
+                    catch (aiErr) {
+                        stepError = err.message;
+                        result.success = false;
+                    }
                 }
                 // Screenshot after each step
                 let screenshotB64 = "";
@@ -213,14 +293,77 @@ Set "done": true with empty "actions" array when the goal is fully accomplished.
  * and returns a SEQUENCE of actions to accomplish it — then executes them one by one.
  * After each action it re-screenshots so the AI can verify progress and adapt.
  */
+/** Extract real interactive elements from the DOM for AI selector accuracy */
+async function extractDomElements(page) {
+    try {
+        const elements = await page.evaluate(() => {
+            const selectors = [
+                'a', 'button', 'input', 'select', 'textarea',
+                '[role="button"]', '[role="link"]', '[role="tab"]',
+                '[role="menuitem"]', '[role="option"]', '[onclick]',
+                '[data-test]', '[data-testid]', '[data-cy]',
+            ];
+            const seen = new Set();
+            const result = [];
+            document.querySelectorAll(selectors.join(',')).forEach((el) => {
+                const rect = el.getBoundingClientRect();
+                if (rect.width === 0 || rect.height === 0)
+                    return;
+                const style = window.getComputedStyle(el);
+                if (style.display === 'none' || style.visibility === 'hidden')
+                    return;
+                const tag = el.tagName.toLowerCase();
+                const id = el.id ? `#${el.id}` : '';
+                const cls = Array.from(el.classList).slice(0, 3).map(c => `.${c}`).join('');
+                const text = (el.textContent || '').trim().slice(0, 40);
+                const ariaLabel = el.getAttribute('aria-label') || '';
+                const dataTest = el.getAttribute('data-test') || el.getAttribute('data-testid') || el.getAttribute('data-cy') || '';
+                const placeholder = el.getAttribute('placeholder') || '';
+                const name = el.getAttribute('name') || '';
+                const type = el.getAttribute('type') || '';
+                const href = el.getAttribute('href') || '';
+                const key = `${tag}${id}${text}`;
+                if (seen.has(key))
+                    return;
+                seen.add(key);
+                const parts = [tag];
+                if (id)
+                    parts.push(`id="${el.id}"`);
+                if (name)
+                    parts.push(`name="${name}"`);
+                if (type)
+                    parts.push(`type="${type}"`);
+                if (ariaLabel)
+                    parts.push(`aria-label="${ariaLabel}"`);
+                if (dataTest)
+                    parts.push(`data-test="${dataTest}"`);
+                if (placeholder)
+                    parts.push(`placeholder="${placeholder}"`);
+                if (href && href !== '#')
+                    parts.push(`href="${href.slice(0, 50)}"`);
+                if (text)
+                    parts.push(`text="${text}"`);
+                result.push(parts.join(' | '));
+            });
+            return result.slice(0, 60).join('\n');
+        });
+        return elements || '(no elements found)';
+    }
+    catch {
+        return '(could not extract elements)';
+    }
+}
 async function aiStepFallback(page, stepText) {
     const openaiKey = process.env.OPENAI_API_KEY;
     if (!openaiKey)
         throw new Error(`No OPENAI_API_KEY — cannot use AI fallback for: "${stepText}"`);
     const { default: OpenAI } = await import('openai');
     const openai = new OpenAI({ apiKey: openaiKey });
-    const MAX_ROUNDS = 6; // prevent infinite loops
+    const MAX_ROUNDS = 6;
+    let consecutiveFailures = 0;
     for (let round = 0; round < MAX_ROUNDS; round++) {
+        // Extract real DOM elements so AI uses accurate selectors
+        const domElements = await extractDomElements(page);
         const screenshotBuffer = await page.screenshot({ type: 'png' });
         const base64 = screenshotBuffer.toString('base64');
         const response = await openai.chat.completions.create({
@@ -229,16 +372,20 @@ async function aiStepFallback(page, stepText) {
             messages: [{
                     role: 'user',
                     content: [
-                        { type: 'text', text: buildAgentPrompt(stepText, round) },
+                        {
+                            type: 'text',
+                            text: buildAgentPrompt(stepText, round) + `
+
+## REAL DOM ELEMENTS ON THIS PAGE
+Use these actual elements — prefer id, data-test, name, aria-label over guessing:
+${domElements}`
+                        },
                         { type: 'image_url', image_url: { url: `data:image/png;base64,${base64}` } }
                     ]
                 }]
         });
         const raw = (response.choices[0]?.message?.content || '')
-            .trim()
-            .replace(/```json\n?/gi, '')
-            .replace(/```/g, '')
-            .trim();
+            .trim().replace(/```json\n?/gi, '').replace(/```/g, '').trim();
         if (!raw)
             throw new Error(`AI returned empty response for: "${stepText}"`);
         const plan = JSON.parse(raw);
@@ -247,16 +394,19 @@ async function aiStepFallback(page, stepText) {
             console.log(`  ✅ AI agent completed: "${stepText}"`);
             return;
         }
-        // Execute each action in this round
+        let anySucceeded = false;
         for (const act of plan.actions) {
             console.log(`  🤖 Executing: ${JSON.stringify(act)}`);
             try {
                 if (act.action === 'click') {
-                    await page.locator(act.selector).first().click({ timeout: 10000 });
-                    await page.waitForTimeout(500); // brief settle
+                    // Try multiple selector variants derived from the DOM
+                    const tried = await tryAIClick(page, act.selector);
+                    if (!tried)
+                        throw new Error(`Could not find element: ${act.selector}`);
+                    await page.waitForTimeout(500);
                 }
                 else if (act.action === 'fill') {
-                    await page.locator(act.selector).first().fill(act.value || '');
+                    await tryAIFill(page, act.selector, act.value || '');
                 }
                 else if (act.action === 'select') {
                     await page.locator(act.selector).first().selectOption({ label: act.value });
@@ -274,23 +424,77 @@ async function aiStepFallback(page, stepText) {
                 }
                 else if (act.action === 'scroll') {
                     if (act.selector) {
-                        await page.locator(act.selector).first().scrollIntoViewIfNeeded();
+                        await page.locator(act.selector).first().scrollIntoViewIfNeeded().catch(() => { });
                     }
                     else {
                         await page.evaluate(() => window.scrollBy(0, 400));
                     }
                 }
+                anySucceeded = true;
+                consecutiveFailures = 0;
             }
             catch (e) {
-                console.log(`  ⚠️ Action failed (${e.message}), continuing to next round...`);
+                console.log(`  ⚠️ Action failed (${e.message.split('\n')[0]})`);
+                consecutiveFailures++;
+            }
+        }
+        // If 2 consecutive rounds had zero successes, bail early
+        if (!anySucceeded) {
+            if (consecutiveFailures >= plan.actions.length * 2) {
+                throw new Error(`AI agent stuck — no actions succeeded after multiple rounds for: "${stepText}"`);
             }
         }
-        // Wait for page to settle before next round
         await page.waitForLoadState('domcontentloaded').catch(() => { });
         await page.waitForTimeout(300);
     }
     throw new Error(`AI agent could not complete "${stepText}" within ${MAX_ROUNDS} rounds`);
 }
+/** Try clicking with multiple selector strategies derived from AI suggestion */
+async function tryAIClick(page, selector) {
+    const TIMEOUT = 5000;
+    // Build fallback variants: the AI selector + text-based alternatives
+    const textMatch = selector.match(/:has-text\("([^"]+)"\)|:text\("([^"]+)"\)/);
+    const text = textMatch ? (textMatch[1] || textMatch[2]) : null;
+    const strategies = [
+        () => page.locator(selector).first().click({ timeout: TIMEOUT }),
+        ...(text ? [
+            () => page.getByRole('button', { name: new RegExp(text, 'i') }).first().click({ timeout: TIMEOUT }),
+            () => page.getByRole('link', { name: new RegExp(text, 'i') }).first().click({ timeout: TIMEOUT }),
+            () => page.getByText(new RegExp(text, 'i')).first().click({ timeout: TIMEOUT }),
+            () => page.locator(`[class*="${text.toLowerCase().replace(/\s+/g, '_')}"]`).first().click({ timeout: TIMEOUT }),
+        ] : []),
+    ];
+    for (const fn of strategies) {
+        try {
+            await fn();
+            return true;
+        }
+        catch { /* next */ }
+    }
+    return false;
+}
+/** Try filling with multiple selector strategies */
+async function tryAIFill(page, selector, value) {
+    const TIMEOUT = 5000;
+    const placeholderMatch = selector.match(/\[placeholder="([^"]+)"\]/);
+    const nameMatch = selector.match(/\[name="([^"]+)"\]|#([\w-]+)/);
+    const label = placeholderMatch?.[1] || nameMatch?.[1] || nameMatch?.[2] || '';
+    const strategies = [
+        () => page.locator(selector).first().fill(value),
+        ...(label ? [
+            () => page.getByPlaceholder(new RegExp(label, 'i')).first().fill(value),
+            () => page.getByLabel(new RegExp(label, 'i')).first().fill(value),
+        ] : []),
+    ];
+    for (const fn of strategies) {
+        try {
+            await Promise.race([fn(), new Promise((_, r) => setTimeout(() => r(new Error('timeout')), TIMEOUT))]);
+            return;
+        }
+        catch { /* next */ }
+    }
+    throw new Error(`Could not fill element: ${selector}`);
+}
 /** Use OpenAI vision to identify the element and generate a selector, then click it */
 async function aiClickFallback(page, description) {
     const openaiKey = process.env.OPENAI_API_KEY;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@cutleryapp/agent",
-    "version": "1.0.20",
+    "version": "1.0.22",
     "description": "Local agent that connects your machine to the Cutlery QA platform and runs UI tests via Playwright",
     "main": "dist/cli.js",
     "bin": {