@cutleryapp/agent 1.0.21 → 1.0.23

package/dist/mcp-executor.js

@@ -147,17 +147,24 @@ class TestExecutor {
                             catch { /* fall to AI */ }
                         }
                     }
-                    // 7. AI fallback — for anything not handled or ambiguous
+                    // 7. AI — single-shot for deterministic steps, full loop for intent steps
                     if (!handled) {
-                        console.log(`  🤖 MCP could not handle step, using AI: "${raw}"`);
-                        await aiStepFallback(page, raw);
+                        const isDeterministic = /^(click|fill|enter|type|verify|check|assert|select|choose|wait|hover|scroll)/i.test(lower.trim());
+                        if (isDeterministic) {
+                            console.log(`  🤖 Quick AI selector lookup for: "${raw}"`);
+                            await aiSingleShot(page, raw);
+                        }
+                        else {
+                            console.log(`  🤖 AI intent loop for: "${raw}"`);
+                            await aiStepFallback(page, raw);
+                        }
                     }
                 }
                 catch (err) {
-                    // MCP execution failed — let AI try to recover
-                    console.log(`  ⚠️  MCP step failed (${err.message}), trying AI...`);
+                    // MCP execution failed — single-shot AI recovery, no loop
+                    console.log(`  ⚠️  MCP step failed (${err.message.split('\n')[0]}), trying AI...`);
                     try {
-                        await aiStepFallback(page, raw);
+                        await aiSingleShot(page, raw);
                     }
                     catch (aiErr) {
                         stepError = err.message;
@@ -293,14 +300,147 @@ Set "done": true with empty "actions" array when the goal is fully accomplished.
  * and returns a SEQUENCE of actions to accomplish it — then executes them one by one.
  * After each action it re-screenshots so the AI can verify progress and adapt.
  */
+/** Single-shot AI: one DOM extract + screenshot → one action → done. No looping. */
+async function aiSingleShot(page, stepText) {
+    const openaiKey = process.env.OPENAI_API_KEY;
+    if (!openaiKey)
+        throw new Error(`No OPENAI_API_KEY for: "${stepText}"`);
+    const { default: OpenAI } = await import('openai');
+    const openai = new OpenAI({ apiKey: openaiKey });
+    const domElements = await extractDomElements(page);
+    const screenshotBuffer = await page.screenshot({ type: 'png' });
+    const base64 = screenshotBuffer.toString('base64');
+    const response = await openai.chat.completions.create({
+        model: 'gpt-4o',
+        max_tokens: 300,
+        messages: [{
+                role: 'user',
+                content: [
+                    {
+                        type: 'text',
+                        text: `You are a Playwright selector expert. Given this test step and the current page, return a single JSON action.
+
+Step: "${stepText}"
+
+## REAL PAGE ELEMENTS (use these — do NOT guess selectors):
+${domElements}
+
+Return ONLY valid JSON, one of:
+{"action":"click","selector":"EXACT_SELECTOR"}
+{"action":"fill","selector":"EXACT_SELECTOR","value":"VALUE"}
+{"action":"verify","text":"TEXT_TO_CHECK","not":false}
+{"action":"select","selector":"EXACT_SELECTOR","value":"OPTION"}
+{"action":"wait","ms":1000}
+
+Rules:
+- Pick selector from the DOM list above using id, name, data-test, aria-label, class exactly as shown
+- For "icon" steps: find element whose class/id/data-test contains the icon keyword
+- For verify: check if text appears in page body`
+                    },
+                    { type: 'image_url', image_url: { url: `data:image/png;base64,${base64}` } }
+                ]
+            }]
+    });
+    const raw2 = (response.choices[0]?.message?.content || '')
+        .trim().replace(/```json\n?/gi, '').replace(/```/g, '').trim();
+    if (!raw2 || raw2 === 'NOT_FOUND')
+        throw new Error(`AI could not find element for: "${stepText}"`);
+    const act = JSON.parse(raw2);
+    console.log(`  🤖 AI action: ${JSON.stringify(act)}`);
+    if (act.action === 'click') {
+        const ok = await tryAIClick(page, act.selector);
+        if (!ok)
+            throw new Error(`AI click failed: ${act.selector}`);
+    }
+    else if (act.action === 'fill') {
+        await tryAIFill(page, act.selector, act.value || '');
+    }
+    else if (act.action === 'verify') {
+        const content = await page.textContent('body') || '';
+        const found = content.includes(act.text);
+        if (act.not && found)
+            throw new Error(`Text "${act.text}" should NOT be visible`);
+        if (!act.not && !found)
+            throw new Error(`Expected text not found: "${act.text}"`);
+    }
+    else if (act.action === 'select') {
+        await page.locator(act.selector).first().selectOption({ label: act.value });
+    }
+    else if (act.action === 'wait') {
+        await page.waitForTimeout(act.ms || 1000);
+    }
+}
+/** Extract real interactive elements from the DOM for AI selector accuracy */
+async function extractDomElements(page) {
+    try {
+        const elements = await page.evaluate(() => {
+            const selectors = [
+                'a', 'button', 'input', 'select', 'textarea',
+                '[role="button"]', '[role="link"]', '[role="tab"]',
+                '[role="menuitem"]', '[role="option"]', '[onclick]',
+                '[data-test]', '[data-testid]', '[data-cy]',
+            ];
+            const seen = new Set();
+            const result = [];
+            document.querySelectorAll(selectors.join(',')).forEach((el) => {
+                const rect = el.getBoundingClientRect();
+                if (rect.width === 0 || rect.height === 0)
+                    return;
+                const style = window.getComputedStyle(el);
+                if (style.display === 'none' || style.visibility === 'hidden')
+                    return;
+                const tag = el.tagName.toLowerCase();
+                const id = el.id ? `#${el.id}` : '';
+                const cls = Array.from(el.classList).slice(0, 3).map(c => `.${c}`).join('');
+                const text = (el.textContent || '').trim().slice(0, 40);
+                const ariaLabel = el.getAttribute('aria-label') || '';
+                const dataTest = el.getAttribute('data-test') || el.getAttribute('data-testid') || el.getAttribute('data-cy') || '';
+                const placeholder = el.getAttribute('placeholder') || '';
+                const name = el.getAttribute('name') || '';
+                const type = el.getAttribute('type') || '';
+                const href = el.getAttribute('href') || '';
+                const key = `${tag}${id}${text}`;
+                if (seen.has(key))
+                    return;
+                seen.add(key);
+                const parts = [tag];
+                if (id)
+                    parts.push(`id="${el.id}"`);
+                if (name)
+                    parts.push(`name="${name}"`);
+                if (type)
+                    parts.push(`type="${type}"`);
+                if (ariaLabel)
+                    parts.push(`aria-label="${ariaLabel}"`);
+                if (dataTest)
+                    parts.push(`data-test="${dataTest}"`);
+                if (placeholder)
+                    parts.push(`placeholder="${placeholder}"`);
+                if (href && href !== '#')
+                    parts.push(`href="${href.slice(0, 50)}"`);
+                if (text)
+                    parts.push(`text="${text}"`);
+                result.push(parts.join(' | '));
+            });
+            return result.slice(0, 60).join('\n');
+        });
+        return elements || '(no elements found)';
+    }
+    catch {
+        return '(could not extract elements)';
+    }
+}
 async function aiStepFallback(page, stepText) {
     const openaiKey = process.env.OPENAI_API_KEY;
     if (!openaiKey)
         throw new Error(`No OPENAI_API_KEY — cannot use AI fallback for: "${stepText}"`);
     const { default: OpenAI } = await import('openai');
     const openai = new OpenAI({ apiKey: openaiKey });
-    const MAX_ROUNDS = 6; // prevent infinite loops
+    const MAX_ROUNDS = 6;
+    let consecutiveFailures = 0;
     for (let round = 0; round < MAX_ROUNDS; round++) {
+        // Extract real DOM elements so AI uses accurate selectors
+        const domElements = await extractDomElements(page);
         const screenshotBuffer = await page.screenshot({ type: 'png' });
         const base64 = screenshotBuffer.toString('base64');
         const response = await openai.chat.completions.create({
@@ -309,16 +449,20 @@ async function aiStepFallback(page, stepText) {
             messages: [{
                     role: 'user',
                     content: [
-                        { type: 'text', text: buildAgentPrompt(stepText, round) },
+                        {
+                            type: 'text',
+                            text: buildAgentPrompt(stepText, round) + `
+
+## REAL DOM ELEMENTS ON THIS PAGE
+Use these actual elements — prefer id, data-test, name, aria-label over guessing:
+${domElements}`
+                        },
                         { type: 'image_url', image_url: { url: `data:image/png;base64,${base64}` } }
                     ]
                 }]
         });
         const raw = (response.choices[0]?.message?.content || '')
-            .trim()
-            .replace(/```json\n?/gi, '')
-            .replace(/```/g, '')
-            .trim();
+            .trim().replace(/```json\n?/gi, '').replace(/```/g, '').trim();
         if (!raw)
             throw new Error(`AI returned empty response for: "${stepText}"`);
         const plan = JSON.parse(raw);
@@ -327,16 +471,19 @@ async function aiStepFallback(page, stepText) {
             console.log(`  ✅ AI agent completed: "${stepText}"`);
             return;
         }
-        // Execute each action in this round
+        let anySucceeded = false;
         for (const act of plan.actions) {
             console.log(`  🤖 Executing: ${JSON.stringify(act)}`);
             try {
                 if (act.action === 'click') {
-                    await page.locator(act.selector).first().click({ timeout: 10000 });
-                    await page.waitForTimeout(500); // brief settle
+                    // Try multiple selector variants derived from the DOM
+                    const tried = await tryAIClick(page, act.selector);
+                    if (!tried)
+                        throw new Error(`Could not find element: ${act.selector}`);
+                    await page.waitForTimeout(500);
                 }
                 else if (act.action === 'fill') {
-                    await page.locator(act.selector).first().fill(act.value || '');
+                    await tryAIFill(page, act.selector, act.value || '');
                 }
                 else if (act.action === 'select') {
                     await page.locator(act.selector).first().selectOption({ label: act.value });
@@ -354,23 +501,77 @@ async function aiStepFallback(page, stepText) {
                 }
                 else if (act.action === 'scroll') {
                     if (act.selector) {
-                        await page.locator(act.selector).first().scrollIntoViewIfNeeded();
+                        await page.locator(act.selector).first().scrollIntoViewIfNeeded().catch(() => { });
                     }
                     else {
                         await page.evaluate(() => window.scrollBy(0, 400));
                     }
                 }
+                anySucceeded = true;
+                consecutiveFailures = 0;
             }
             catch (e) {
-                console.log(`  ⚠️ Action failed (${e.message}), continuing to next round...`);
+                console.log(`  ⚠️ Action failed (${e.message.split('\n')[0]})`);
+                consecutiveFailures++;
+            }
+        }
+        // If 2 consecutive rounds had zero successes, bail early
+        if (!anySucceeded) {
+            if (consecutiveFailures >= plan.actions.length * 2) {
+                throw new Error(`AI agent stuck — no actions succeeded after multiple rounds for: "${stepText}"`);
             }
         }
-        // Wait for page to settle before next round
         await page.waitForLoadState('domcontentloaded').catch(() => { });
         await page.waitForTimeout(300);
     }
     throw new Error(`AI agent could not complete "${stepText}" within ${MAX_ROUNDS} rounds`);
 }
+/** Try clicking with multiple selector strategies derived from AI suggestion */
+async function tryAIClick(page, selector) {
+    const TIMEOUT = 5000;
+    // Build fallback variants: the AI selector + text-based alternatives
+    const textMatch = selector.match(/:has-text\("([^"]+)"\)|:text\("([^"]+)"\)/);
+    const text = textMatch ? (textMatch[1] || textMatch[2]) : null;
+    const strategies = [
+        () => page.locator(selector).first().click({ timeout: TIMEOUT }),
+        ...(text ? [
+            () => page.getByRole('button', { name: new RegExp(text, 'i') }).first().click({ timeout: TIMEOUT }),
+            () => page.getByRole('link', { name: new RegExp(text, 'i') }).first().click({ timeout: TIMEOUT }),
+            () => page.getByText(new RegExp(text, 'i')).first().click({ timeout: TIMEOUT }),
+            () => page.locator(`[class*="${text.toLowerCase().replace(/\s+/g, '_')}"]`).first().click({ timeout: TIMEOUT }),
+        ] : []),
+    ];
+    for (const fn of strategies) {
+        try {
+            await fn();
+            return true;
+        }
+        catch { /* next */ }
+    }
+    return false;
+}
+/** Try filling with multiple selector strategies */
+async function tryAIFill(page, selector, value) {
+    const TIMEOUT = 5000;
+    const placeholderMatch = selector.match(/\[placeholder="([^"]+)"\]/);
+    const nameMatch = selector.match(/\[name="([^"]+)"\]|#([\w-]+)/);
+    const label = placeholderMatch?.[1] || nameMatch?.[1] || nameMatch?.[2] || '';
+    const strategies = [
+        () => page.locator(selector).first().fill(value),
+        ...(label ? [
+            () => page.getByPlaceholder(new RegExp(label, 'i')).first().fill(value),
+            () => page.getByLabel(new RegExp(label, 'i')).first().fill(value),
+        ] : []),
+    ];
+    for (const fn of strategies) {
+        try {
+            await Promise.race([fn(), new Promise((_, r) => setTimeout(() => r(new Error('timeout')), TIMEOUT))]);
+            return;
+        }
+        catch { /* next */ }
+    }
+    throw new Error(`Could not fill element: ${selector}`);
+}
 /** Use OpenAI vision to identify the element and generate a selector, then click it */
 async function aiClickFallback(page, description) {
     const openaiKey = process.env.OPENAI_API_KEY;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@cutleryapp/agent",
-    "version": "1.0.21",
+    "version": "1.0.23",
     "description": "Local agent that connects your machine to the Cutlery QA platform and runs UI tests via Playwright",
     "main": "dist/cli.js",
     "bin": {