@cutleryapp/agent 1.0.17 → 1.0.19

package/dist/mcp-executor.js

@@ -248,9 +248,62 @@ async function tryClick(page, nameRe, label) {
     // AI vision fallback
     return await aiClickFallback(page, label);
 }
+function buildAgentPrompt(stepText, round) {
+    return `You are an intelligent browser test agent with vision. Your job is to look at the current screen, understand the test step intent, and decide what actions to perform.
+
+GOAL: "${stepText}"
+ROUND: ${round + 1}
+
+## YOUR CAPABILITIES
+1. SCREEN UNDERSTANDING — Identify every visible UI element, form field, button, link, and label on screen.
+2. INTENT MAPPING — Understand what the test step MEANS even if the wording is vague or high-level (e.g. "checkout the product" = navigate to cart → fill shipping info → complete purchase).
+3. DATA SIMULATION — If a form needs data that is not specified, INVENT realistic test data:
+   - Names: "John Smith" or "Test User"
+   - Email: "testuser@example.com"
+   - Phone: "555-0100"
+   - Address: "123 Test St, Springfield"
+   - Postal/ZIP: "12345"
+   - Credit card: "4111111111111111", expiry "12/25", CVV "123"
+   - Password: "Test@1234"
+   - Any other field: invent plausible data based on the field label
+
+## RETURN FORMAT
+Return ONLY valid JSON — no markdown, no explanation:
+{
+  "reasoning": "What I see on screen and what I plan to do",
+  "done": false,
+  "actions": [
+    {"action": "click",  "selector": "SELECTOR"},
+    {"action": "fill",   "selector": "SELECTOR", "value": "VALUE"},
+    {"action": "select", "selector": "SELECTOR", "value": "OPTION_LABEL"},
+    {"action": "verify", "text": "TEXT_TO_CHECK", "not": false},
+    {"action": "wait",   "ms": 500}
+  ]
+}
+
+Set "done": true with empty "actions" array when the goal is fully accomplished.
+
+## SELECTOR RULES
+- Prefer visible text: button:has-text("Checkout"), a:has-text("Login")
+- Use id when visible: #first-name, #postal-code
+- Use data attributes: [data-test="checkout-button"], [data-testid="..."]
+- Use placeholder for inputs: input[placeholder="First Name"]
+- Use name attribute: input[name="firstName"]
+- NEVER use position or coordinates
+
+## SMART RULES
+- Look at ALL visible form fields and fill them ALL in one round
+- If you see a multi-step form, complete this step fully then click continue/next/submit
+- If the goal is "checkout": cart → fill info → continue → finish
+- If the goal is "register" or "sign up": fill all fields + submit
+- If the goal is already accomplished (correct page shown), set done: true immediately
+- Maximum 5 actions per round`;
+}
 /**
- * Universal AI fallback — takes a screenshot + the raw step text and asks GPT-4o
- * what to do (click, fill, verify, select, etc.) and returns a JSON action to execute.
+ * Universal AI agentic fallback.
+ * GPT-4o sees the current page screenshot, understands the GOAL of the step,
+ * and returns a SEQUENCE of actions to accomplish it — then executes them one by one.
+ * After each action it re-screenshots so the AI can verify progress and adapt.
  */
 async function aiStepFallback(page, stepText) {
     const openaiKey = process.env.OPENAI_API_KEY;
@@ -258,61 +311,77 @@ async function aiStepFallback(page, stepText) {
         throw new Error(`No OPENAI_API_KEY — cannot use AI fallback for: "${stepText}"`);
     const { default: OpenAI } = await import('openai');
     const openai = new OpenAI({ apiKey: openaiKey });
-    const screenshotBuffer = await page.screenshot({ type: 'png' });
-    const base64 = screenshotBuffer.toString('base64');
-    const response = await openai.chat.completions.create({
-        model: 'gpt-4o',
-        max_tokens: 200,
-        messages: [{
-                role: 'user',
-                content: [
-                    {
-                        type: 'text',
-                        text: `You are a Playwright test automation AI. Analyse this screenshot and the test step below, then return a JSON action to execute.
-
-Test step: "${stepText}"
-
-Return ONLY valid JSON (no markdown, no explanation) in one of these formats:
-- Click:   {"action":"click","selector":"CSS_SELECTOR"}
-- Fill:    {"action":"fill","selector":"CSS_SELECTOR","value":"TEXT"}
-- Select:  {"action":"select","selector":"CSS_SELECTOR","value":"OPTION"}
-- Verify:  {"action":"verify","text":"EXPECTED_TEXT","not":false}
-- Wait:    {"action":"wait","ms":1000}
-
-Rules:
-- Use the most specific selector you can see (data-testid, id, aria-label, class, text)
-- For verify steps, set "not":true if the step says "not displayed/visible"
-- Return NOT_FOUND if you cannot determine the action`
-                    },
-                    { type: 'image_url', image_url: { url: `data:image/png;base64,${base64}` } }
-                ]
-            }]
-    });
-    const raw2 = (response.choices[0]?.message?.content || '').trim().replace(/```json\n?/gi, '').replace(/```/g, '').trim();
-    if (!raw2 || raw2 === 'NOT_FOUND')
-        throw new Error(`AI could not determine action for: "${stepText}"`);
-    const action = JSON.parse(raw2);
-    console.log(`  🤖 AI action: ${JSON.stringify(action)}`);
-    if (action.action === 'click') {
-        await page.locator(action.selector).first().click({ timeout: 10000 });
-    }
-    else if (action.action === 'fill') {
-        await page.locator(action.selector).first().fill(action.value);
-    }
-    else if (action.action === 'select') {
-        await page.locator(action.selector).first().selectOption({ label: action.value });
-    }
-    else if (action.action === 'verify') {
-        const content = await page.textContent('body') || '';
-        const found = content.includes(action.text);
-        if (action.not && found)
-            throw new Error(`Text "${action.text}" should NOT be visible`);
-        if (!action.not && !found)
-            throw new Error(`Expected text not found: "${action.text}"`);
-    }
-    else if (action.action === 'wait') {
-        await page.waitForTimeout(action.ms || 1000);
+    const MAX_ROUNDS = 6; // prevent infinite loops
+    for (let round = 0; round < MAX_ROUNDS; round++) {
+        const screenshotBuffer = await page.screenshot({ type: 'png' });
+        const base64 = screenshotBuffer.toString('base64');
+        const response = await openai.chat.completions.create({
+            model: 'gpt-4o',
+            max_tokens: 800,
+            messages: [{
+                    role: 'user',
+                    content: [
+                        { type: 'text', text: buildAgentPrompt(stepText, round) },
+                        { type: 'image_url', image_url: { url: `data:image/png;base64,${base64}` } }
+                    ]
+                }]
+        });
+        const raw = (response.choices[0]?.message?.content || '')
+            .trim()
+            .replace(/```json\n?/gi, '')
+            .replace(/```/g, '')
+            .trim();
+        if (!raw)
+            throw new Error(`AI returned empty response for: "${stepText}"`);
+        const plan = JSON.parse(raw);
+        console.log(`  🤖 AI round ${round + 1} — ${plan.reasoning}`);
+        if (plan.done || !plan.actions?.length) {
+            console.log(`  ✅ AI agent completed: "${stepText}"`);
+            return;
+        }
+        // Execute each action in this round
+        for (const act of plan.actions) {
+            console.log(`  🤖 Executing: ${JSON.stringify(act)}`);
+            try {
+                if (act.action === 'click') {
+                    await page.locator(act.selector).first().click({ timeout: 10000 });
+                    await page.waitForTimeout(500); // brief settle
+                }
+                else if (act.action === 'fill') {
+                    await page.locator(act.selector).first().fill(act.value || '');
+                }
+                else if (act.action === 'select') {
+                    await page.locator(act.selector).first().selectOption({ label: act.value });
+                }
+                else if (act.action === 'verify') {
+                    const content = await page.textContent('body') || '';
+                    const found = content.includes(act.text);
+                    if (act.not && found)
+                        throw new Error(`Text "${act.text}" should NOT be visible`);
+                    if (!act.not && !found)
+                        throw new Error(`Expected text not found: "${act.text}"`);
+                }
+                else if (act.action === 'wait') {
+                    await page.waitForTimeout(act.ms || 1000);
+                }
+                else if (act.action === 'scroll') {
+                    if (act.selector) {
+                        await page.locator(act.selector).first().scrollIntoViewIfNeeded();
+                    }
+                    else {
+                        await page.evaluate(() => window.scrollBy(0, 400));
+                    }
+                }
+            }
+            catch (e) {
+                console.log(`  ⚠️ Action failed (${e.message}), continuing to next round...`);
+            }
+        }
+        // Wait for page to settle before next round
+        await page.waitForLoadState('domcontentloaded').catch(() => { });
+        await page.waitForTimeout(300);
     }
+    throw new Error(`AI agent could not complete "${stepText}" within ${MAX_ROUNDS} rounds`);
 }
 /** Use OpenAI vision to identify the element and generate a selector, then click it */
 async function aiClickFallback(page, description) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@cutleryapp/agent",
-    "version": "1.0.17",
+    "version": "1.0.19",
     "description": "Local agent that connects your machine to the Cutlery QA platform and runs UI tests via Playwright",
     "main": "dist/cli.js",
     "bin": {