@cutleryapp/agent 1.0.18 → 1.0.20

package/dist/mcp-executor.js

@@ -60,9 +60,9 @@ class TestExecutor {
                 });
                 let stepError;
                 try {
+                    // Navigate is handled directly — URL extraction doesn't need vision
                     if (lower.includes("navigate to") || lower.includes("go to")) {
-                        // Extract URL wherever it appears in the step (handles emoji/symbol prefixes)
-                        const urlMatch = raw.match(/(?:navigate to|go to)\s+(https?:\/\/\S+|\/\S*|\S+\.\S+)/i);
+                        const urlMatch = raw.match(/(?:navigate\s+to|go\s+to)\s+(https?:\/\/\S+|\/\S*|\S+\.\S+)/i);
                         if (urlMatch) {
                             let url = urlMatch[1].trim();
                             if (url.startsWith("/") && this.options.baseUrl) {
@@ -71,110 +71,18 @@ class TestExecutor {
                             await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
                         }
                         else if (this.options.baseUrl) {
-                            // Step says "navigate to" but no URL found — go to baseUrl
                             await page.goto(this.options.baseUrl, { waitUntil: "domcontentloaded", timeout: 30000 });
                         }
                     }
-                    else if (lower.includes("click")) {
-                        const labelMatch = raw.match(/click\s+(?:on\s+)?(?:the\s+)?"?([^"]+?)"?(?:\s+(?:button|link|tab))?$/i);
-                        let label = labelMatch?.[1]?.trim();
-                        if (label) {
-                            // Split "Add to cart under Sauce Labs Bike Light product" into target + scope
-                            const scopeMatch = label.match(/^(.+?)\s+(?:under|inside|within|in the|in)\s+(.+)$/i);
-                            const target = scopeMatch ? scopeMatch[1].trim() : label;
-                            const scope = scopeMatch ? scopeMatch[2].trim() : null;
-                            const nameRe = new RegExp(escapeRegex(target), 'i');
-                            const clicked = scope
-                                ? await tryClickScoped(page, nameRe, target, scope)
-                                : await tryClick(page, nameRe, target);
-                            if (!clicked)
-                                throw new Error(`Could not find clickable element: "${label}"`);
-                        }
-                    }
-                    else if (lower.includes("fill") || lower.includes("type") || lower.includes("enter")) {
-                        // Support both quoted and unquoted formats:
-                        //   Fill "standard_user" in "Username"
-                        //   Fill standard_user in Username field
-                        // Split on first " in " / " into " to separate value from field
-                        const match = raw.match(/(?:enter|fill|type)\s+"([^"]+)"\s+(?:in|into)\s+(?:the\s+)?"?([^"]+?)"?\s*(?:field|input|box|area)?\s*$/i) ||
-                            raw.match(/(?:enter|fill|type)\s+(\S+)\s+(?:in|into)\s+(?:the\s+)?(.+?)\s*(?:field|input|box|area)?\s*$/i);
-                        if (match) {
-                            const value = match[1].trim();
-                            const fieldLabel = match[2].trim();
-                            const looksLikeCss = (s) => /[#.\[\]:>]/.test(s);
-                            if (looksLikeCss(fieldLabel)) {
-                                await page.waitForSelector(fieldLabel, { state: "visible", timeout: 5000 });
-                                await page.fill(fieldLabel, value);
-                            }
-                            else {
-                                await tryFill(page, fieldLabel, value);
-                            }
-                        }
-                    }
-                    else if (lower.includes("wait") && lower.includes("second")) {
-                        const ms = raw.match(/wait\s+(\d+)\s*sec/i);
-                        if (ms)
-                            await page.waitForTimeout(parseInt(ms[1]) * 1000);
-                    }
-                    else if (lower.includes("wait for") && !lower.includes("second")) {
-                        const sel = extractSelector(raw, /wait for\s+"?([^"]+)"?\s+to be/i);
-                        if (sel)
-                            await page.waitForSelector(sel, { state: "visible", timeout: 15000 });
-                    }
-                    else if (lower.includes("verify") || lower.includes("check") || lower.includes("assert") || lower.includes("should")) {
-                        // Support: Verify "text", Verify I see text Foo, Verify text Foo is not displayed
-                        const isNegative = /not\s+(?:displayed|visible|present)/i.test(raw);
-                        const textMatch = raw.match(/"([^"]+)"/) ||
-                            raw.match(/(?:verify|check|assert)\s+(?:i\s+see\s+(?:text\s+)?|text\s+)?(.+?)(?:\s+is\s+(?:not\s+)?(?:displayed|visible|present))?$/i);
-                        if (textMatch) {
-                            const expected = textMatch[1].trim();
-                            if (isNegative) {
-                                const content = await page.textContent('body') || '';
-                                if (content.includes(expected))
-                                    throw new Error(`Text "${expected}" should NOT be visible but was found`);
-                            }
-                            else {
-                                try {
-                                    await page.waitForFunction((text) => document.body.innerText.includes(text), expected, { timeout: 10000 });
-                                }
-                                catch {
-                                    throw new Error(`Expected text not found: "${expected}"`);
-                                }
-                            }
-                        }
-                    }
-                    else if (lower.includes("select") || lower.includes("choose")) {
-                        const selMatch = raw.match(/select\s+"?([^"]+?)"?\s+(?:from|in)\s+"?([^"]+?)"?\s*(?:dropdown|select|field)?$/i);
-                        if (selMatch) {
-                            try {
-                                await page.selectOption(selMatch[2].trim(), { label: selMatch[1].trim() });
-                            }
-                            catch {
-                                // fallback: click the dropdown then click the option
-                                await tryClick(page, new RegExp(escapeRegex(selMatch[2].trim()), 'i'), selMatch[2].trim());
-                                await tryClick(page, new RegExp(escapeRegex(selMatch[1].trim()), 'i'), selMatch[1].trim());
-                            }
-                        }
-                        else {
-                            await aiStepFallback(page, raw);
-                        }
-                    }
                     else {
-                        // Unknown step — let AI interpret and execute it
+                        // ALL other steps: AI reads the screen and performs the action
+                        console.log(`  🤖 AI executing: "${raw}"`);
                         await aiStepFallback(page, raw);
                     }
                 }
                 catch (err) {
-                    // If a recognised handler threw, try AI fallback before marking as failed
-                    console.log(`  ⚠️  Step failed (${err.message}), trying AI fallback...`);
-                    try {
-                        await aiStepFallback(page, raw);
-                        stepError = undefined; // AI recovered it
-                    }
-                    catch (aiErr) {
-                        stepError = err.message; // Report original error
-                        result.success = false;
-                    }
+                    stepError = err.message;
+                    result.success = false;
                 }
                 // Screenshot after each step
                 let screenshotB64 = "";
@@ -248,6 +156,57 @@ async function tryClick(page, nameRe, label) {
     // AI vision fallback
     return await aiClickFallback(page, label);
 }
+function buildAgentPrompt(stepText, round) {
+    return `You are an intelligent browser test agent with vision. Your job is to look at the current screen, understand the test step intent, and decide what actions to perform.
+
+GOAL: "${stepText}"
+ROUND: ${round + 1}
+
+## YOUR CAPABILITIES
+1. SCREEN UNDERSTANDING — Identify every visible UI element, form field, button, link, and label on screen.
+2. INTENT MAPPING — Understand what the test step MEANS even if the wording is vague or high-level (e.g. "checkout the product" = navigate to cart → fill shipping info → complete purchase).
+3. DATA SIMULATION — If a form needs data that is not specified, INVENT realistic test data:
+   - Names: "John Smith" or "Test User"
+   - Email: "testuser@example.com"
+   - Phone: "555-0100"
+   - Address: "123 Test St, Springfield"
+   - Postal/ZIP: "12345"
+   - Credit card: "4111111111111111", expiry "12/25", CVV "123"
+   - Password: "Test@1234"
+   - Any other field: invent plausible data based on the field label
+
+## RETURN FORMAT
+Return ONLY valid JSON — no markdown, no explanation:
+{
+  "reasoning": "What I see on screen and what I plan to do",
+  "done": false,
+  "actions": [
+    {"action": "click",  "selector": "SELECTOR"},
+    {"action": "fill",   "selector": "SELECTOR", "value": "VALUE"},
+    {"action": "select", "selector": "SELECTOR", "value": "OPTION_LABEL"},
+    {"action": "verify", "text": "TEXT_TO_CHECK", "not": false},
+    {"action": "wait",   "ms": 500}
+  ]
+}
+
+Set "done": true with empty "actions" array when the goal is fully accomplished.
+
+## SELECTOR RULES
+- Prefer visible text: button:has-text("Checkout"), a:has-text("Login")
+- Use id when visible: #first-name, #postal-code
+- Use data attributes: [data-test="checkout-button"], [data-testid="..."]
+- Use placeholder for inputs: input[placeholder="First Name"]
+- Use name attribute: input[name="firstName"]
+- NEVER use position or coordinates
+
+## SMART RULES
+- Look at ALL visible form fields and fill them ALL in one round
+- If you see a multi-step form, complete this step fully then click continue/next/submit
+- If the goal is "checkout": cart → fill info → continue → finish
+- If the goal is "register" or "sign up": fill all fields + submit
+- If the goal is already accomplished (correct page shown), set done: true immediately
+- Maximum 5 actions per round`;
+}
 /**
  * Universal AI agentic fallback.
  * GPT-4o sees the current page screenshot, understands the GOAL of the step,
@@ -266,41 +225,11 @@ async function aiStepFallback(page, stepText) {
         const base64 = screenshotBuffer.toString('base64');
         const response = await openai.chat.completions.create({
             model: 'gpt-4o',
-            max_tokens: 600,
+            max_tokens: 800,
             messages: [{
                     role: 'user',
                     content: [
-                        {
-                            type: 'text',
-                            text: `You are an autonomous browser test agent. Your goal is to accomplish this test step:
-
-"${stepText}"
-
-Look at the current screenshot and decide what actions are needed RIGHT NOW to make progress toward the goal.
-
-Return ONLY valid JSON (no markdown) with this structure:
-{
-  "reasoning": "brief explanation of what you see and what needs to happen",
-  "done": false,
-  "actions": [
-    {"action": "click",  "selector": "CSS_OR_TEXT_SELECTOR"},
-    {"action": "fill",   "selector": "CSS_OR_TEXT_SELECTOR", "value": "TEXT"},
-    {"action": "select", "selector": "CSS_OR_TEXT_SELECTOR", "value": "OPTION_TEXT"},
-    {"action": "verify", "text": "EXPECTED_TEXT", "not": false},
-    {"action": "wait",   "ms": 500}
-  ]
-}
-
-Set "done": true and empty "actions" array when the goal has been fully accomplished.
-
-Rules:
-- Return only the actions needed for THIS screenshot — after executing them you will see the next state
-- Use text-based selectors when possible: button:has-text("Checkout"), [data-test="..."], #id
-- For "checkout" goals: look for cart/checkout buttons and click them in sequence
-- For "login" goals: fill username, fill password, click login
-- For "add to cart" goals: find and click the Add to cart button
-- Maximum 3 actions per round to stay precise`
-                        },
+                        { type: 'text', text: buildAgentPrompt(stepText, round) },
                         { type: 'image_url', image_url: { url: `data:image/png;base64,${base64}` } }
                     ]
                 }]
@@ -343,6 +272,14 @@ Rules:
                 else if (act.action === 'wait') {
                     await page.waitForTimeout(act.ms || 1000);
                 }
+                else if (act.action === 'scroll') {
+                    if (act.selector) {
+                        await page.locator(act.selector).first().scrollIntoViewIfNeeded();
+                    }
+                    else {
+                        await page.evaluate(() => window.scrollBy(0, 400));
+                    }
+                }
             }
             catch (e) {
                 console.log(`  ⚠️ Action failed (${e.message}), continuing to next round...`);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@cutleryapp/agent",
-    "version": "1.0.18",
+    "version": "1.0.20",
     "description": "Local agent that connects your machine to the Cutlery QA platform and runs UI tests via Playwright",
     "main": "dist/cli.js",
     "bin": {