@cutleryapp/agent 1.0.21 → 1.0.22

package/dist/mcp-executor.js

@@ -293,14 +293,77 @@ Set "done": true with empty "actions" array when the goal is fully accomplished.
  * and returns a SEQUENCE of actions to accomplish it — then executes them one by one.
  * After each action it re-screenshots so the AI can verify progress and adapt.
  */
+/** Extract real interactive elements from the DOM for AI selector accuracy */
+async function extractDomElements(page) {
+    try {
+        const elements = await page.evaluate(() => {
+            const selectors = [
+                'a', 'button', 'input', 'select', 'textarea',
+                '[role="button"]', '[role="link"]', '[role="tab"]',
+                '[role="menuitem"]', '[role="option"]', '[onclick]',
+                '[data-test]', '[data-testid]', '[data-cy]',
+            ];
+            const seen = new Set();
+            const result = [];
+            document.querySelectorAll(selectors.join(',')).forEach((el) => {
+                const rect = el.getBoundingClientRect();
+                if (rect.width === 0 || rect.height === 0)
+                    return;
+                const style = window.getComputedStyle(el);
+                if (style.display === 'none' || style.visibility === 'hidden')
+                    return;
+                const tag = el.tagName.toLowerCase();
+                const id = el.id ? `#${el.id}` : '';
+                const cls = Array.from(el.classList).slice(0, 3).map(c => `.${c}`).join('');
+                const text = (el.textContent || '').trim().slice(0, 40);
+                const ariaLabel = el.getAttribute('aria-label') || '';
+                const dataTest = el.getAttribute('data-test') || el.getAttribute('data-testid') || el.getAttribute('data-cy') || '';
+                const placeholder = el.getAttribute('placeholder') || '';
+                const name = el.getAttribute('name') || '';
+                const type = el.getAttribute('type') || '';
+                const href = el.getAttribute('href') || '';
+                const key = `${tag}${id}${text}`;
+                if (seen.has(key))
+                    return;
+                seen.add(key);
+                const parts = [tag];
+                if (id)
+                    parts.push(`id="${el.id}"`);
+                if (name)
+                    parts.push(`name="${name}"`);
+                if (type)
+                    parts.push(`type="${type}"`);
+                if (ariaLabel)
+                    parts.push(`aria-label="${ariaLabel}"`);
+                if (dataTest)
+                    parts.push(`data-test="${dataTest}"`);
+                if (placeholder)
+                    parts.push(`placeholder="${placeholder}"`);
+                if (href && href !== '#')
+                    parts.push(`href="${href.slice(0, 50)}"`);
+                if (text)
+                    parts.push(`text="${text}"`);
+                result.push(parts.join(' | '));
+            });
+            return result.slice(0, 60).join('\n');
+        });
+        return elements || '(no elements found)';
+    }
+    catch {
+        return '(could not extract elements)';
+    }
+}
 async function aiStepFallback(page, stepText) {
     const openaiKey = process.env.OPENAI_API_KEY;
     if (!openaiKey)
         throw new Error(`No OPENAI_API_KEY — cannot use AI fallback for: "${stepText}"`);
     const { default: OpenAI } = await import('openai');
     const openai = new OpenAI({ apiKey: openaiKey });
-    const MAX_ROUNDS = 6; // prevent infinite loops
+    const MAX_ROUNDS = 6;
+    let consecutiveFailures = 0;
     for (let round = 0; round < MAX_ROUNDS; round++) {
+        // Extract real DOM elements so AI uses accurate selectors
+        const domElements = await extractDomElements(page);
         const screenshotBuffer = await page.screenshot({ type: 'png' });
         const base64 = screenshotBuffer.toString('base64');
         const response = await openai.chat.completions.create({
@@ -309,16 +372,20 @@ async function aiStepFallback(page, stepText) {
             messages: [{
                     role: 'user',
                     content: [
-                        { type: 'text', text: buildAgentPrompt(stepText, round) },
+                        {
+                            type: 'text',
+                            text: buildAgentPrompt(stepText, round) + `
+
+## REAL DOM ELEMENTS ON THIS PAGE
+Use these actual elements — prefer id, data-test, name, aria-label over guessing:
+${domElements}`
+                        },
                         { type: 'image_url', image_url: { url: `data:image/png;base64,${base64}` } }
                     ]
                 }]
         });
         const raw = (response.choices[0]?.message?.content || '')
-            .trim()
-            .replace(/```json\n?/gi, '')
-            .replace(/```/g, '')
-            .trim();
+            .trim().replace(/```json\n?/gi, '').replace(/```/g, '').trim();
         if (!raw)
             throw new Error(`AI returned empty response for: "${stepText}"`);
         const plan = JSON.parse(raw);
@@ -327,16 +394,19 @@ async function aiStepFallback(page, stepText) {
             console.log(`  ✅ AI agent completed: "${stepText}"`);
             return;
         }
-        // Execute each action in this round
+        let anySucceeded = false;
         for (const act of plan.actions) {
             console.log(`  🤖 Executing: ${JSON.stringify(act)}`);
             try {
                 if (act.action === 'click') {
-                    await page.locator(act.selector).first().click({ timeout: 10000 });
-                    await page.waitForTimeout(500); // brief settle
+                    // Try multiple selector variants derived from the DOM
+                    const tried = await tryAIClick(page, act.selector);
+                    if (!tried)
+                        throw new Error(`Could not find element: ${act.selector}`);
+                    await page.waitForTimeout(500);
                 }
                 else if (act.action === 'fill') {
-                    await page.locator(act.selector).first().fill(act.value || '');
+                    await tryAIFill(page, act.selector, act.value || '');
                 }
                 else if (act.action === 'select') {
                     await page.locator(act.selector).first().selectOption({ label: act.value });
@@ -354,23 +424,77 @@ async function aiStepFallback(page, stepText) {
                 }
                 else if (act.action === 'scroll') {
                     if (act.selector) {
-                        await page.locator(act.selector).first().scrollIntoViewIfNeeded();
+                        await page.locator(act.selector).first().scrollIntoViewIfNeeded().catch(() => { });
                     }
                     else {
                         await page.evaluate(() => window.scrollBy(0, 400));
                     }
                 }
+                anySucceeded = true;
+                consecutiveFailures = 0;
             }
             catch (e) {
-                console.log(`  ⚠️ Action failed (${e.message}), continuing to next round...`);
+                console.log(`  ⚠️ Action failed (${e.message.split('\n')[0]})`);
+                consecutiveFailures++;
+            }
+        }
+        // If 2 consecutive rounds had zero successes, bail early
+        if (!anySucceeded) {
+            if (consecutiveFailures >= plan.actions.length * 2) {
+                throw new Error(`AI agent stuck — no actions succeeded after multiple rounds for: "${stepText}"`);
             }
         }
-        // Wait for page to settle before next round
         await page.waitForLoadState('domcontentloaded').catch(() => { });
         await page.waitForTimeout(300);
     }
     throw new Error(`AI agent could not complete "${stepText}" within ${MAX_ROUNDS} rounds`);
 }
+/** Try clicking with multiple selector strategies derived from AI suggestion */
+async function tryAIClick(page, selector) {
+    const TIMEOUT = 5000;
+    // Build fallback variants: the AI selector + text-based alternatives
+    const textMatch = selector.match(/:has-text\("([^"]+)"\)|:text\("([^"]+)"\)/);
+    const text = textMatch ? (textMatch[1] || textMatch[2]) : null;
+    const strategies = [
+        () => page.locator(selector).first().click({ timeout: TIMEOUT }),
+        ...(text ? [
+            () => page.getByRole('button', { name: new RegExp(text, 'i') }).first().click({ timeout: TIMEOUT }),
+            () => page.getByRole('link', { name: new RegExp(text, 'i') }).first().click({ timeout: TIMEOUT }),
+            () => page.getByText(new RegExp(text, 'i')).first().click({ timeout: TIMEOUT }),
+            () => page.locator(`[class*="${text.toLowerCase().replace(/\s+/g, '_')}"]`).first().click({ timeout: TIMEOUT }),
+        ] : []),
+    ];
+    for (const fn of strategies) {
+        try {
+            await fn();
+            return true;
+        }
+        catch { /* next */ }
+    }
+    return false;
+}
+/** Try filling with multiple selector strategies */
+async function tryAIFill(page, selector, value) {
+    const TIMEOUT = 5000;
+    const placeholderMatch = selector.match(/\[placeholder="([^"]+)"\]/);
+    const nameMatch = selector.match(/\[name="([^"]+)"\]|#([\w-]+)/);
+    const label = placeholderMatch?.[1] || nameMatch?.[1] || nameMatch?.[2] || '';
+    const strategies = [
+        () => page.locator(selector).first().fill(value),
+        ...(label ? [
+            () => page.getByPlaceholder(new RegExp(label, 'i')).first().fill(value),
+            () => page.getByLabel(new RegExp(label, 'i')).first().fill(value),
+        ] : []),
+    ];
+    for (const fn of strategies) {
+        try {
+            await Promise.race([fn(), new Promise((_, r) => setTimeout(() => r(new Error('timeout')), TIMEOUT))]);
+            return;
+        }
+        catch { /* next */ }
+    }
+    throw new Error(`Could not fill element: ${selector}`);
+}
 /** Use OpenAI vision to identify the element and generate a selector, then click it */
 async function aiClickFallback(page, description) {
     const openaiKey = process.env.OPENAI_API_KEY;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@cutleryapp/agent",
-    "version": "1.0.21",
+    "version": "1.0.22",
     "description": "Local agent that connects your machine to the Cutlery QA platform and runs UI tests via Playwright",
     "main": "dist/cli.js",
     "bin": {