@cutleryapp/agent 1.0.15 → 1.0.17

package/dist/mcp-executor.js

@@ -76,7 +76,7 @@ class TestExecutor {
                         }
                     }
                     else if (lower.includes("click")) {
-                        const labelMatch = raw.match(/click\s+(?:on\s+)?(?:the\s+)?"?([^"]+?)"?\s*(?:button|link|tab)?$/i);
+                        const labelMatch = raw.match(/click\s+(?:on\s+)?(?:the\s+)?"?([^"]+?)"?(?:\s+(?:button|link|tab))?$/i);
                         let label = labelMatch?.[1]?.trim();
                         if (label) {
                             // Split "Add to cart under Sauce Labs Bike Light product" into target + scope
@@ -122,31 +122,59 @@ class TestExecutor {
                             await page.waitForSelector(sel, { state: "visible", timeout: 15000 });
                     }
                     else if (lower.includes("verify") || lower.includes("check") || lower.includes("assert") || lower.includes("should")) {
-                        // Extract the quoted text to verify it's present on the page
-                        const textMatch = raw.match(/"([^"]+)"/);
+                        // Support: Verify "text", Verify I see text Foo, Verify text Foo is not displayed
+                        const isNegative = /not\s+(?:displayed|visible|present)/i.test(raw);
+                        const textMatch = raw.match(/"([^"]+)"/) ||
+                            raw.match(/(?:verify|check|assert)\s+(?:i\s+see\s+(?:text\s+)?|text\s+)?(.+?)(?:\s+is\s+(?:not\s+)?(?:displayed|visible|present))?$/i);
                         if (textMatch) {
-                            const expected = textMatch[1];
-                            try {
-                                await page.waitForFunction((text) => document.body.innerText.includes(text), expected, { timeout: 10000 });
+                            const expected = textMatch[1].trim();
+                            if (isNegative) {
+                                const content = await page.textContent('body') || '';
+                                if (content.includes(expected))
+                                    throw new Error(`Text "${expected}" should NOT be visible but was found`);
                             }
-                            catch {
-                                throw new Error(`Expected text not found: "${expected}"`);
+                            else {
+                                try {
+                                    await page.waitForFunction((text) => document.body.innerText.includes(text), expected, { timeout: 10000 });
+                                }
+                                catch {
+                                    throw new Error(`Expected text not found: "${expected}"`);
+                                }
                             }
                         }
                     }
                     else if (lower.includes("select") || lower.includes("choose")) {
-                        const selMatch = raw.match(/select\s+"([^"]+)"\s+(?:from|in)\s+"([^"]+)"/i);
+                        const selMatch = raw.match(/select\s+"?([^"]+?)"?\s+(?:from|in)\s+"?([^"]+?)"?\s*(?:dropdown|select|field)?$/i);
                         if (selMatch) {
-                            await page.selectOption(selMatch[2], { label: selMatch[1] });
+                            try {
+                                await page.selectOption(selMatch[2].trim(), { label: selMatch[1].trim() });
+                            }
+                            catch {
+                                // fallback: click the dropdown then click the option
+                                await tryClick(page, new RegExp(escapeRegex(selMatch[2].trim()), 'i'), selMatch[2].trim());
+                                await tryClick(page, new RegExp(escapeRegex(selMatch[1].trim()), 'i'), selMatch[1].trim());
+                            }
+                        }
+                        else {
+                            await aiStepFallback(page, raw);
                         }
                     }
                     else {
-                        console.log(`  ⚠️  Unrecognised step (skipped): ${raw}`);
+                        // Unknown step — let AI interpret and execute it
+                        await aiStepFallback(page, raw);
                     }
                 }
                 catch (err) {
-                    stepError = err.message;
-                    result.success = false;
+                    // If a recognised handler threw, try AI fallback before marking as failed
+                    console.log(`  ⚠️  Step failed (${err.message}), trying AI fallback...`);
+                    try {
+                        await aiStepFallback(page, raw);
+                        stepError = undefined; // AI recovered it
+                    }
+                    catch (aiErr) {
+                        stepError = err.message; // Report original error
+                        result.success = false;
+                    }
                 }
                 // Screenshot after each step
                 let screenshotB64 = "";
@@ -207,6 +235,8 @@ async function tryClick(page, nameRe, label) {
         () => page.getByRole('link', { name: nameRe }).first().click({ timeout: FAST }),
         () => page.getByText(nameRe).first().click({ timeout: FAST }),
         () => page.locator(`[value="${label}"], [aria-label="${label}"], [title="${label}"]`).first().click({ timeout: FAST }),
+        // data-* attributes (common in test automation)
+        () => page.locator(`[data-test*="${label}" i], [data-testid*="${label}" i], [id*="${label}" i]`).first().click({ timeout: FAST }),
     ];
     for (const fn of strategies) {
         try {
@@ -215,7 +245,144 @@ async function tryClick(page, nameRe, label) {
         }
         catch { /* try next */ }
     }
-    return false;
+    // AI vision fallback
+    return await aiClickFallback(page, label);
+}
+/**
+ * Universal AI fallback — takes a screenshot + the raw step text and asks GPT-4o
+ * what to do (click, fill, verify, select, etc.) and returns a JSON action to execute.
+ */
+async function aiStepFallback(page, stepText) {
+    const openaiKey = process.env.OPENAI_API_KEY;
+    if (!openaiKey)
+        throw new Error(`No OPENAI_API_KEY — cannot use AI fallback for: "${stepText}"`);
+    const { default: OpenAI } = await import('openai');
+    const openai = new OpenAI({ apiKey: openaiKey });
+    const screenshotBuffer = await page.screenshot({ type: 'png' });
+    const base64 = screenshotBuffer.toString('base64');
+    const response = await openai.chat.completions.create({
+        model: 'gpt-4o',
+        max_tokens: 200,
+        messages: [{
+                role: 'user',
+                content: [
+                    {
+                        type: 'text',
+                        text: `You are a Playwright test automation AI. Analyse this screenshot and the test step below, then return a JSON action to execute.
+
+Test step: "${stepText}"
+
+Return ONLY valid JSON (no markdown, no explanation) in one of these formats:
+- Click:   {"action":"click","selector":"CSS_SELECTOR"}
+- Fill:    {"action":"fill","selector":"CSS_SELECTOR","value":"TEXT"}
+- Select:  {"action":"select","selector":"CSS_SELECTOR","value":"OPTION"}
+- Verify:  {"action":"verify","text":"EXPECTED_TEXT","not":false}
+- Wait:    {"action":"wait","ms":1000}
+
+Rules:
+- Use the most specific selector you can see (data-testid, id, aria-label, class, text)
+- For verify steps, set "not":true if the step says "not displayed/visible"
+- Return NOT_FOUND if you cannot determine the action`
+                    },
+                    { type: 'image_url', image_url: { url: `data:image/png;base64,${base64}` } }
+                ]
+            }]
+    });
+    const raw2 = (response.choices[0]?.message?.content || '').trim().replace(/```json\n?/gi, '').replace(/```/g, '').trim();
+    if (!raw2 || raw2 === 'NOT_FOUND')
+        throw new Error(`AI could not determine action for: "${stepText}"`);
+    const action = JSON.parse(raw2);
+    console.log(`  🤖 AI action: ${JSON.stringify(action)}`);
+    if (action.action === 'click') {
+        await page.locator(action.selector).first().click({ timeout: 10000 });
+    }
+    else if (action.action === 'fill') {
+        await page.locator(action.selector).first().fill(action.value);
+    }
+    else if (action.action === 'select') {
+        await page.locator(action.selector).first().selectOption({ label: action.value });
+    }
+    else if (action.action === 'verify') {
+        const content = await page.textContent('body') || '';
+        const found = content.includes(action.text);
+        if (action.not && found)
+            throw new Error(`Text "${action.text}" should NOT be visible`);
+        if (!action.not && !found)
+            throw new Error(`Expected text not found: "${action.text}"`);
+    }
+    else if (action.action === 'wait') {
+        await page.waitForTimeout(action.ms || 1000);
+    }
+}
+/** Use OpenAI vision to identify the element and generate a selector, then click it */
+async function aiClickFallback(page, description) {
+    const openaiKey = process.env.OPENAI_API_KEY;
+    if (!openaiKey)
+        return false;
+    try {
+        const { default: OpenAI } = await import('openai');
+        const openai = new OpenAI({ apiKey: openaiKey });
+        const screenshotBuffer = await page.screenshot({ type: 'png' });
+        const base64 = screenshotBuffer.toString('base64');
+        const response = await openai.chat.completions.create({
+            model: 'gpt-4o',
+            max_tokens: 100,
+            messages: [{
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'text',
+                            text: `I need to click on: "${description}"\n\nLook at this screenshot and return a single CSS selector that identifies the element to click.\nRules:\n- Return ONLY the CSS selector, nothing else\n- No markdown, no explanation\n- If not found, return: NOT_FOUND\n- Use data-testid, id, aria-label, or text-based selectors\n- Prefer: [data-test="..."], #id, [aria-label="..."], button:has-text("...")`
+                        },
+                        { type: 'image_url', image_url: { url: `data:image/png;base64,${base64}` } }
+                    ]
+                }]
+        });
+        const selector = response.choices[0]?.message?.content?.trim() || '';
+        if (!selector || selector === 'NOT_FOUND')
+            return false;
+        const clean = selector.replace(/```[a-z]*\n?/gi, '').replace(/```/g, '').trim();
+        await page.locator(clean).first().click({ timeout: 5000 });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/** Use OpenAI vision to identify a field and fill it */
+async function aiFillFallback(page, label, value) {
+    const openaiKey = process.env.OPENAI_API_KEY;
+    if (!openaiKey)
+        return false;
+    try {
+        const { default: OpenAI } = await import('openai');
+        const openai = new OpenAI({ apiKey: openaiKey });
+        const screenshotBuffer = await page.screenshot({ type: 'png' });
+        const base64 = screenshotBuffer.toString('base64');
+        const response = await openai.chat.completions.create({
+            model: 'gpt-4o',
+            max_tokens: 100,
+            messages: [{
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'text',
+                            text: `I need to fill the "${label}" input field with the value "${value}".\n\nLook at this screenshot and return a single CSS selector for the input field.\nRules:\n- Return ONLY the CSS selector, nothing else\n- No markdown, no explanation\n- If not found, return: NOT_FOUND\n- Prefer: input[name="..."], input[id="..."], input[placeholder="..."], [aria-label="..."]`
+                        },
+                        { type: 'image_url', image_url: { url: `data:image/png;base64,${base64}` } }
+                    ]
+                }]
+        });
+        const selector = response.choices[0]?.message?.content?.trim() || '';
+        if (!selector || selector === 'NOT_FOUND')
+            return false;
+        const clean = selector.replace(/```[a-z]*\n?/gi, '').replace(/```/g, '').trim();
+        await page.locator(clean).first().fill(value);
+        return true;
+    }
+    catch {
+        return false;
+    }
 }
 async function tryClickScoped(page, nameRe, target, scope) {
     const FAST = 3000;
@@ -324,6 +491,10 @@ async function tryFill(page, label, value) {
             errors.push(e?.message?.split("\n")[0] || String(e));
         }
     }
+    // AI vision fallback
+    const aiSuccess = await aiFillFallback(page, label, value);
+    if (aiSuccess)
+        return;
     throw new Error(`Could not find input field: "${label}". Tried ${strategies.length} strategies.`);
 }
 /** Token-aware variant generation matching executor.ts/labelVariants. */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@cutleryapp/agent",
-    "version": "1.0.15",
+    "version": "1.0.17",
     "description": "Local agent that connects your machine to the Cutlery QA platform and runs UI tests via Playwright",
     "main": "dist/cli.js",
     "bin": {