@cutleryapp/agent 1.0.22 → 1.0.23

package/dist/mcp-executor.js

@@ -147,17 +147,24 @@ class TestExecutor {
                             catch { /* fall to AI */ }
                         }
                     }
-                    // 7. AI fallback — for anything not handled or ambiguous
+                    // 7. AI — single-shot for deterministic steps, full loop for intent steps
                     if (!handled) {
-                        console.log(`  🤖 MCP could not handle step, using AI: "${raw}"`);
-                        await aiStepFallback(page, raw);
+                        const isDeterministic = /^(click|fill|enter|type|verify|check|assert|select|choose|wait|hover|scroll)/i.test(lower.trim());
+                        if (isDeterministic) {
+                            console.log(`  🤖 Quick AI selector lookup for: "${raw}"`);
+                            await aiSingleShot(page, raw);
+                        }
+                        else {
+                            console.log(`  🤖 AI intent loop for: "${raw}"`);
+                            await aiStepFallback(page, raw);
+                        }
                     }
                 }
                 catch (err) {
-                    // MCP execution failed — let AI try to recover
-                    console.log(`  ⚠️  MCP step failed (${err.message}), trying AI...`);
+                    // MCP execution failed — single-shot AI recovery, no loop
+                    console.log(`  ⚠️  MCP step failed (${err.message.split('\n')[0]}), trying AI...`);
                     try {
-                        await aiStepFallback(page, raw);
+                        await aiSingleShot(page, raw);
                     }
                     catch (aiErr) {
                         stepError = err.message;
@@ -293,6 +300,76 @@ Set "done": true with empty "actions" array when the goal is fully accomplished.
  * and returns a SEQUENCE of actions to accomplish it — then executes them one by one.
  * After each action it re-screenshots so the AI can verify progress and adapt.
  */
+/** Single-shot AI: one DOM extract + screenshot → one action → done. No looping. */
+async function aiSingleShot(page, stepText) {
+    const openaiKey = process.env.OPENAI_API_KEY;
+    if (!openaiKey)
+        throw new Error(`No OPENAI_API_KEY for: "${stepText}"`);
+    const { default: OpenAI } = await import('openai');
+    const openai = new OpenAI({ apiKey: openaiKey });
+    const domElements = await extractDomElements(page);
+    const screenshotBuffer = await page.screenshot({ type: 'png' });
+    const base64 = screenshotBuffer.toString('base64');
+    const response = await openai.chat.completions.create({
+        model: 'gpt-4o',
+        max_tokens: 300,
+        messages: [{
+                role: 'user',
+                content: [
+                    {
+                        type: 'text',
+                        text: `You are a Playwright selector expert. Given this test step and the current page, return a single JSON action.
+
+Step: "${stepText}"
+
+## REAL PAGE ELEMENTS (use these — do NOT guess selectors):
+${domElements}
+
+Return ONLY valid JSON, one of:
+{"action":"click","selector":"EXACT_SELECTOR"}
+{"action":"fill","selector":"EXACT_SELECTOR","value":"VALUE"}
+{"action":"verify","text":"TEXT_TO_CHECK","not":false}
+{"action":"select","selector":"EXACT_SELECTOR","value":"OPTION"}
+{"action":"wait","ms":1000}
+
+Rules:
+- Pick selector from the DOM list above using id, name, data-test, aria-label, class exactly as shown
+- For "icon" steps: find element whose class/id/data-test contains the icon keyword
+- For verify: check if text appears in page body`
+                    },
+                    { type: 'image_url', image_url: { url: `data:image/png;base64,${base64}` } }
+                ]
+            }]
+    });
+    const raw2 = (response.choices[0]?.message?.content || '')
+        .trim().replace(/```json\n?/gi, '').replace(/```/g, '').trim();
+    if (!raw2 || raw2 === 'NOT_FOUND')
+        throw new Error(`AI could not find element for: "${stepText}"`);
+    const act = JSON.parse(raw2);
+    console.log(`  🤖 AI action: ${JSON.stringify(act)}`);
+    if (act.action === 'click') {
+        const ok = await tryAIClick(page, act.selector);
+        if (!ok)
+            throw new Error(`AI click failed: ${act.selector}`);
+    }
+    else if (act.action === 'fill') {
+        await tryAIFill(page, act.selector, act.value || '');
+    }
+    else if (act.action === 'verify') {
+        const content = await page.textContent('body') || '';
+        const found = content.includes(act.text);
+        if (act.not && found)
+            throw new Error(`Text "${act.text}" should NOT be visible`);
+        if (!act.not && !found)
+            throw new Error(`Expected text not found: "${act.text}"`);
+    }
+    else if (act.action === 'select') {
+        await page.locator(act.selector).first().selectOption({ label: act.value });
+    }
+    else if (act.action === 'wait') {
+        await page.waitForTimeout(act.ms || 1000);
+    }
+}
 /** Extract real interactive elements from the DOM for AI selector accuracy */
 async function extractDomElements(page) {
     try {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@cutleryapp/agent",
-    "version": "1.0.22",
+    "version": "1.0.23",
     "description": "Local agent that connects your machine to the Cutlery QA platform and runs UI tests via Playwright",
     "main": "dist/cli.js",
     "bin": {