@cutleryapp/agent 1.0.16 → 1.0.18

package/dist/mcp-executor.js

@@ -122,31 +122,59 @@ class TestExecutor {
                             await page.waitForSelector(sel, { state: "visible", timeout: 15000 });
                     }
                     else if (lower.includes("verify") || lower.includes("check") || lower.includes("assert") || lower.includes("should")) {
-                        // Extract the quoted text to verify it's present on the page
-                        const textMatch = raw.match(/"([^"]+)"/);
+                        // Support: Verify "text", Verify I see text Foo, Verify text Foo is not displayed
+                        const isNegative = /not\s+(?:displayed|visible|present)/i.test(raw);
+                        const textMatch = raw.match(/"([^"]+)"/) ||
+                            raw.match(/(?:verify|check|assert)\s+(?:i\s+see\s+(?:text\s+)?|text\s+)?(.+?)(?:\s+is\s+(?:not\s+)?(?:displayed|visible|present))?$/i);
                         if (textMatch) {
-                            const expected = textMatch[1];
-                            try {
-                                await page.waitForFunction((text) => document.body.innerText.includes(text), expected, { timeout: 10000 });
+                            const expected = textMatch[1].trim();
+                            if (isNegative) {
+                                const content = await page.textContent('body') || '';
+                                if (content.includes(expected))
+                                    throw new Error(`Text "${expected}" should NOT be visible but was found`);
                             }
-                            catch {
-                                throw new Error(`Expected text not found: "${expected}"`);
+                            else {
+                                try {
+                                    await page.waitForFunction((text) => document.body.innerText.includes(text), expected, { timeout: 10000 });
+                                }
+                                catch {
+                                    throw new Error(`Expected text not found: "${expected}"`);
+                                }
                             }
                         }
                     }
                     else if (lower.includes("select") || lower.includes("choose")) {
-                        const selMatch = raw.match(/select\s+"([^"]+)"\s+(?:from|in)\s+"([^"]+)"/i);
+                        const selMatch = raw.match(/select\s+"?([^"]+?)"?\s+(?:from|in)\s+"?([^"]+?)"?\s*(?:dropdown|select|field)?$/i);
                         if (selMatch) {
-                            await page.selectOption(selMatch[2], { label: selMatch[1] });
+                            try {
+                                await page.selectOption(selMatch[2].trim(), { label: selMatch[1].trim() });
+                            }
+                            catch {
+                                // fallback: click the dropdown then click the option
+                                await tryClick(page, new RegExp(escapeRegex(selMatch[2].trim()), 'i'), selMatch[2].trim());
+                                await tryClick(page, new RegExp(escapeRegex(selMatch[1].trim()), 'i'), selMatch[1].trim());
+                            }
+                        }
+                        else {
+                            await aiStepFallback(page, raw);
                         }
                     }
                     else {
-                        console.log(`  ⚠️  Unrecognised step (skipped): ${raw}`);
+                        // Unknown step — let AI interpret and execute it
+                        await aiStepFallback(page, raw);
                     }
                 }
                 catch (err) {
-                    stepError = err.message;
-                    result.success = false;
+                    // If a recognised handler threw, try AI fallback before marking as failed
+                    console.log(`  ⚠️  Step failed (${err.message}), trying AI fallback...`);
+                    try {
+                        await aiStepFallback(page, raw);
+                        stepError = undefined; // AI recovered it
+                    }
+                    catch (aiErr) {
+                        stepError = err.message; // Report original error
+                        result.success = false;
+                    }
                 }
                 // Screenshot after each step
                 let screenshotB64 = "";
@@ -220,6 +248,112 @@ async function tryClick(page, nameRe, label) {
     // AI vision fallback
     return await aiClickFallback(page, label);
 }
+/**
+ * Universal AI agentic fallback.
+ * GPT-4o sees the current page screenshot, understands the GOAL of the step,
+ * and returns a SEQUENCE of actions to accomplish it — then executes them one by one.
+ * After each action it re-screenshots so the AI can verify progress and adapt.
+ */
+async function aiStepFallback(page, stepText) {
+    const openaiKey = process.env.OPENAI_API_KEY;
+    if (!openaiKey)
+        throw new Error(`No OPENAI_API_KEY — cannot use AI fallback for: "${stepText}"`);
+    const { default: OpenAI } = await import('openai');
+    const openai = new OpenAI({ apiKey: openaiKey });
+    const MAX_ROUNDS = 6; // prevent infinite loops
+    for (let round = 0; round < MAX_ROUNDS; round++) {
+        const screenshotBuffer = await page.screenshot({ type: 'png' });
+        const base64 = screenshotBuffer.toString('base64');
+        const response = await openai.chat.completions.create({
+            model: 'gpt-4o',
+            max_tokens: 600,
+            messages: [{
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'text',
+                            text: `You are an autonomous browser test agent. Your goal is to accomplish this test step:
+
+"${stepText}"
+
+Look at the current screenshot and decide what actions are needed RIGHT NOW to make progress toward the goal.
+
+Return ONLY valid JSON (no markdown) with this structure:
+{
+  "reasoning": "brief explanation of what you see and what needs to happen",
+  "done": false,
+  "actions": [
+    {"action": "click",  "selector": "CSS_OR_TEXT_SELECTOR"},
+    {"action": "fill",   "selector": "CSS_OR_TEXT_SELECTOR", "value": "TEXT"},
+    {"action": "select", "selector": "CSS_OR_TEXT_SELECTOR", "value": "OPTION_TEXT"},
+    {"action": "verify", "text": "EXPECTED_TEXT", "not": false},
+    {"action": "wait",   "ms": 500}
+  ]
+}
+
+Set "done": true and empty "actions" array when the goal has been fully accomplished.
+
+Rules:
+- Return only the actions needed for THIS screenshot — after executing them you will see the next state
+- Use text-based selectors when possible: button:has-text("Checkout"), [data-test="..."], #id
+- For "checkout" goals: look for cart/checkout buttons and click them in sequence
+- For "login" goals: fill username, fill password, click login
+- For "add to cart" goals: find and click the Add to cart button
+- Maximum 3 actions per round to stay precise`
+                        },
+                        { type: 'image_url', image_url: { url: `data:image/png;base64,${base64}` } }
+                    ]
+                }]
+        });
+        const raw = (response.choices[0]?.message?.content || '')
+            .trim()
+            .replace(/```json\n?/gi, '')
+            .replace(/```/g, '')
+            .trim();
+        if (!raw)
+            throw new Error(`AI returned empty response for: "${stepText}"`);
+        const plan = JSON.parse(raw);
+        console.log(`  🤖 AI round ${round + 1} — ${plan.reasoning}`);
+        if (plan.done || !plan.actions?.length) {
+            console.log(`  ✅ AI agent completed: "${stepText}"`);
+            return;
+        }
+        // Execute each action in this round
+        for (const act of plan.actions) {
+            console.log(`  🤖 Executing: ${JSON.stringify(act)}`);
+            try {
+                if (act.action === 'click') {
+                    await page.locator(act.selector).first().click({ timeout: 10000 });
+                    await page.waitForTimeout(500); // brief settle
+                }
+                else if (act.action === 'fill') {
+                    await page.locator(act.selector).first().fill(act.value || '');
+                }
+                else if (act.action === 'select') {
+                    await page.locator(act.selector).first().selectOption({ label: act.value });
+                }
+                else if (act.action === 'verify') {
+                    const content = await page.textContent('body') || '';
+                    const found = content.includes(act.text);
+                    if (act.not && found)
+                        throw new Error(`Text "${act.text}" should NOT be visible`);
+                    if (!act.not && !found)
+                        throw new Error(`Expected text not found: "${act.text}"`);
+                }
+                else if (act.action === 'wait') {
+                    await page.waitForTimeout(act.ms || 1000);
+                }
+            }
+            catch (e) {
+                console.log(`  ⚠️ Action failed (${e.message}), continuing to next round...`);
+            }
+        }
+        // Wait for page to settle before next round
+        await page.waitForLoadState('domcontentloaded').catch(() => { });
+        await page.waitForTimeout(300);
+    }
+    throw new Error(`AI agent could not complete "${stepText}" within ${MAX_ROUNDS} rounds`);
+}
 /** Use OpenAI vision to identify the element and generate a selector, then click it */
 async function aiClickFallback(page, description) {
     const openaiKey = process.env.OPENAI_API_KEY;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@cutleryapp/agent",
-    "version": "1.0.16",
+    "version": "1.0.18",
     "description": "Local agent that connects your machine to the Cutlery QA platform and runs UI tests via Playwright",
     "main": "dist/cli.js",
     "bin": {