@tangle-network/browser-agent-driver 0.22.0 → 0.24.0

package/dist/brain/index.js

@@ -26,6 +26,7 @@ ACTIONS:
 - {"action": "wait", "ms": 1000}
 - {"action": "evaluate", "criteria": "Is the layout professional? Are colors consistent?"}
 - {"action": "runScript", "script": "document.querySelector('.count').textContent"} — run JS in page context and get the result. Use for reading content not in the a11y tree (canvas, computed styles, hidden state).
+- {"action": "extractWithIndex", "query": "p, span, dd, code", "contains": "downloads"} — return a NUMBERED list of every visible element matching \`query\`, with each element's tag, full textContent, key attributes, and a stable selector. PREFER THIS OVER runScript when you need to find data inside the page but don't know the exact selector. The wide query (e.g. \`'p, span, strong'\`) finds candidates and the response shows the actual text so you can pick by content match. Optional \`contains\` filters matches to those whose text contains a substring (case-insensitive). After this action, your next turn can complete with the picked element's text or click its selector.
 - {"action": "verifyPreview"} — after the app builds, inspect the preview iframe. Returns URL, title, a11y tree, and errors. Use this AFTER you see a preview iframe on the page.
 - {"action": "fill", "fields": {"@t1": "Jordan", "@t2": "Rivera"}, "selects": {"@s1": "WA"}, "checks": ["@c1", "@c2"]} — BATCH fill multiple form fields, dropdowns, and checkboxes in ONE turn. Use this whenever you can see 2+ form fields you need to fill — it's dramatically faster than per-field type/click. fields/selects/checks are all optional but at least one must be non-empty.
 - {"action": "clickSequence", "refs": ["@r1", "@r2", "@r3"]} — click a known sequence of refs in order. Use for multi-step UI navigation chains where the click order is obvious from the page structure.
@@ -72,17 +73,59 @@ const SEARCH_RULES = `
 15. SEARCH FORMS: Always interact with the form (type in search box, then click Search or press Enter). Do NOT navigate to a URL with search query parameters — many sites require form submission to trigger filtering. If a search yields no results, try the page's own search box rather than the site-wide search
 16. CONTENT DISCOVERY: If the ELEMENTS list doesn't show the link/content you need (e.g., the page has many links but the a11y tree is truncated), use runScript to find it: document.querySelectorAll('a[href]') filtered by keyword. Navigate to the discovered URL directly instead of clicking blindly through menus
 17. EXTERNAL SEARCH REDIRECTS: If a site's search form redirects to an external search engine (e.g., search.usa.gov for .gov sites), the results still link back to the original site. Click a relevant search result link — it will take you to the target domain. Do NOT abandon search results to navigate the target site manually`;
-/** Data extraction rules (18, 21-23): injected when goal involves extracting data */
+/** Data extraction rules (18, 21-23, 25): injected when goal involves extracting data */
 const DATA_EXTRACTION_RULES = `
-18. DATA EXTRACTION: When the goal asks for specific data (prices, ratings, counts, names) from a list or search results page, use runScript to extract all needed data at once: e.g., document.querySelectorAll('.product-card').forEach(...). Do NOT click into each individual item when the data is visible on the list page. Extract first, then complete with the extracted data
-21. EFFICIENT COMPLETION: When you have enough data to answer the goal, complete immediately. Do not navigate to additional pages for "confirmation" if the data was already extracted via runScript or is visible in the current a11y tree. Include all extracted data in the completion result
-22. EXTRACT BEFORE NAVIGATING: On search results, directory listings, or any page showing multiple items, ALWAYS extract ALL needed data via runScript BEFORE clicking into individual items. This includes names, phone numbers, addresses, ratings, prices — anything visible on list cards. Use: document.querySelectorAll('.result-card, .listing, [class*="card"]') to grab everything at once. Many sites use anti-bot protection on detail pages but leave listing pages accessible. If you can answer the goal from list-level data, do so without navigating deeper. NEVER click into 3+ individual items when the data is on the list page
-23. FILTER vs SEARCH: When a goal asks to filter results (e.g., "under $50", "4+ stars"), look for filter controls (sliders, dropdowns, checkboxes in a sidebar or toolbar) rather than typing filter values into the search box. Search boxes are for keyword queries, not numeric filters. After applying a filter: (1) wait 2-3 seconds for results to update, (2) verify the filter took effect by checking the updated results, (3) extract the filtered data via runScript. Do NOT keep searching for more filter controls after one is applied — extract and complete`;
+18. DATA EXTRACTION: When the goal asks for specific data (prices, ratings, counts, names) from a list or search results page, prefer extractWithIndex with a wide query (e.g. \`'p, span, dd, code, strong'\`) over runScript when you don't already see the value in the snapshot. extractWithIndex returns the actual textContent of every match so you can pick the right one by content. Use runScript only when you need a transformation the LLM can't do from text alone.
+21. EFFICIENT COMPLETION: When you have enough data to answer the goal, complete immediately. Do not navigate to additional pages for "confirmation" if the data was already extracted or is visible in the current a11y tree. Include all extracted data in the completion result
+22. EXTRACT BEFORE NAVIGATING: On search results, directory listings, or any page showing multiple items, ALWAYS extract ALL needed data BEFORE clicking into individual items. Use extractWithIndex with a wide query for unknown structure, or runScript with document.querySelectorAll('.result-card') if the structure is well-known. Many sites use anti-bot protection on detail pages but leave listing pages accessible. If you can answer the goal from list-level data, do so without navigating deeper.
+23. FILTER vs SEARCH: When a goal asks to filter results (e.g., "under $50", "4+ stars"), look for filter controls (sliders, dropdowns, checkboxes in a sidebar or toolbar) rather than typing filter values into the search box. Search boxes are for keyword queries, not numeric filters. After applying a filter: (1) wait 2-3 seconds for results to update, (2) verify the filter took effect by checking the updated results, (3) extract the filtered data. Do NOT keep searching for more filter controls after one is applied — extract and complete
+25. EXTRACTWITHINDEX RECOVERY: If a previous runScript returned null/empty/{x:null} on an extraction task, the selector was wrong. DO NOT retry the same runScript or guess a similar selector — the LLM cannot guess CSS class names that aren't visible in the snapshot. Switch to extractWithIndex with a WIDE query: \`'p, span, dd, code, strong, em'\` plus a \`contains\` filter naming the expected text fragment (e.g. contains: "downloads" for npm download counts, contains: "callbackFn" for MDN method signatures). The response shows you the actual text per element so you can pick by content match. This is the Gen 10 recovery pattern — pick-by-content beats pick-by-selector on every page where the planner couldn't see the data at plan time.`;
 /** Heavy page rules (19-20, 24): injected when snapshot is large or turn count is high */
 const HEAVY_PAGE_RULES = `
 19. FORM FIELD TARGETING: Before typing, verify you are targeting the correct input field using its @ref from the ELEMENTS list. If multiple inputs are visible (e.g., search box + price filter), ensure you select the right one by checking its label or placeholder text in the a11y tree. Never assume focus — always specify the exact @ref
 20. SECTION NAVIGATION: When you need to find a specific section (e.g., rugby, sports, travel) and the nav links aren't in the truncated a11y tree, use runScript to discover navigation: JSON.stringify(Array.from(document.querySelectorAll('nav a, header a, [role="navigation"] a, .nav a')).slice(0, 30).map(a => ({text: a.textContent.trim(), href: a.href}))). Then navigate directly to the matching section URL
 24. HEAVY PAGE RECOVERY: If a page takes very long to load or seems stuck, do NOT wait — use runScript to check document.readyState and extract whatever content is already in the DOM. Partial data is better than a timeout. If the page is completely blank, try navigating to a simpler version (mobile site, search page) instead of waiting`;
+/** Gen 24: URL-first navigation — GENERAL PURPOSE, not site-specific.
+ * Teaches the agent to construct search/results URLs from goal text
+ * instead of fighting form UIs. Works on any site with URL parameters. */
+const URL_FIRST_RULES = `
+URL-FIRST NAVIGATION: When a search form is complex (date pickers, multi-step dropdowns, dynamic widgets), try constructing a results URL directly instead of interacting with the form.
+
+STRATEGY:
+1. Look at the current URL structure. Most search sites encode parameters: ?q=query, ?checkin=date, ?dest=city, etc.
+2. Construct a URL with the goal's parameters filled in. Use the site's own URL pattern.
+3. Navigate directly to that URL — skip the form entirely.
+4. If the URL doesn't work (wrong page, error), fall back to form interaction.
+
+HOW TO DISCOVER URL PATTERNS:
+- If you're on a search results page, the URL already shows the pattern. Modify the parameters for your goal.
+- Most sites accept ?q= or ?search= for keyword queries.
+- Travel sites typically use: checkin/checkout dates, destination/origin, adults count.
+- Use runScript to read window.location.href if the URL isn't visible in the snapshot.
+- ENCODED PARAMETERS: If a URL contains encoded parameters (base64, protobuf), you may be able to replicate them from a previous successful URL, but do NOT invent new encodings. If a navigate with encoded parameters lands on the wrong page, do not retry with a different encoding — it will waste turns.
+
+IMPORTANT EXCEPTIONS — some sites BLOCK direct URL navigation:
+- If a direct URL navigate lands on the homepage or an error page instead of results, the site blocks URL manipulation. STOP trying URLs and use the site's form/search UI instead.
+- After ONE failed URL attempt, switch to form interaction immediately. Do NOT retry different URL patterns — you will waste turns.
+
+SEARCH ENGINE FALLBACK: If BOTH URL construction AND form interaction have stalled (3+ turns on the same form with no progress), use DuckDuckGo as a proxy:
+1. Navigate to https://duckduckgo.com/?q={goal rephrased as a search query}. Example: goal "Find cheapest one-way flight from NYC to London on Jan 15" → https://duckduckgo.com/?q=cheapest+one+way+flight+NYC+to+London+January+15+2026
+2. Read the search results to find the answer, OR click through to a comparison site (Kayak, Skyscanner, Google Flights via result link) that has the data.
+3. IMPORTANT: Do NOT use google.com/search — it blocks automated browsers with CAPTCHA pages. Always use duckduckgo.com instead.
+4. This is a LAST RESORT after URL construction and direct form interaction have both failed. Do not jump to search on the first turn.
+
+FORM RESET DETECTION: Some sites (especially SPAs) silently reset form fields after filling. After batch-filling a form:
+1. Use runScript to verify values stuck: document.querySelector('[aria-label="From"]')?.value or similar.
+2. If fields reset to defaults (wrong city, blank dates), do NOT re-fill with the same approach — it will reset again.
+3. Instead, switch to keyboard-only interaction: click the field, type the value character by character, wait for autocomplete dropdown, press Enter to confirm. Then Tab to the next field.
+4. If the form keeps resetting after 3 fill attempts, use the search engine fallback immediately.
+
+DATE PICKER STRATEGY: Calendar widgets often ignore programmatic fill/type. When a date field opens a calendar popup that blocks further input:
+1. Try typing the date directly into the field in the site's format (e.g., "Jan 25, 2026" or "01/25/2026"). Press Escape first if the calendar covers the input.
+2. If typing doesn't stick, use runScript to find clickable date elements: document.querySelectorAll('[data-iso],[aria-label*="January"],[aria-label*="25"]') and click the matching element.
+3. NEVER spend more than 4 turns on a single date field. If it's not set after 4 attempts, switch to the search engine fallback above.
+
+WHY: Complex forms with date pickers, calendar widgets, and multi-step dropdowns consume many turns and often time out. A single "navigate" action replaces 5-10 form interaction turns. But only use this on sites that support it.`;
 /** Reasoning framework and examples (always appended after rules) */
 const REASONING_SUFFIX = `
 
@@ -103,6 +146,134 @@ EXAMPLE 3 — Batch fill a multi-field form (one turn instead of ten):
 {"plan":["Fill all visible Personal Info fields","Click Next","Fill Contact step","Submit"],"currentStep":0,"action":{"action":"fill","fields":{"@firstname":"Jordan","@lastname":"Rivera","@dob":"1990-04-15"},"selects":{"@gender":"other"}},"reasoning":"Step 1 of the form has 3 text fields and 1 select all visible at once. Filling them in a single batch action saves 7 turns vs typing each individually.","expectedEffect":"All four Step 1 fields populated with the supplied values"}`;
 /** Full static prompt (all rules) — used as default when config.systemPrompt is not set */
 const SYSTEM_PROMPT = CORE_RULES + SEARCH_RULES + DATA_EXTRACTION_RULES + HEAVY_PAGE_RULES + REASONING_SUFFIX;
+// Gen 13: Vision-first system prompt (pure coordinate actions).
+const VISION_FIRST_PROMPT = `You are a browser automation agent. You operate by looking at screenshots and clicking on elements using pixel coordinates.
+
+The screenshot shows the current page state at 1024×768 resolution. You identify elements visually and specify where to click using (x, y) coordinates in this coordinate space.
+
+ACTIONS:
+- {"action": "clickAt", "x": 512, "y": 384} — click at pixel coordinates (x, y) in 1024×768 space
+- {"action": "typeAt", "x": 300, "y": 200, "text": "search query"} — click at coordinates then type text
+- {"action": "scroll", "direction": "up" | "down", "amount": 500}
+- {"action": "navigate", "url": "https://..."}
+- {"action": "wait", "ms": 1000}
+- {"action": "complete", "result": "description of what was accomplished"}
+- {"action": "abort", "reason": "why you cannot continue"}
+- {"action": "runScript", "script": "document.querySelector('.count').textContent"} — run JS in page context for data the screenshot can't show
+- {"action": "extractWithIndex", "query": "p, span, dd", "contains": "keyword"} — find text in the DOM by content match
+
+COORDINATE SYSTEM:
+- (0, 0) is the top-left corner of the viewport
+- (1024, 768) is the bottom-right corner
+- Click the CENTER of the target element, not its edge
+- For text inputs, click the middle of the input field
+- For buttons, click the center of the button text or icon
+
+RESPONSE FORMAT — respond with ONLY a JSON object:
+{
+  "plan": ["step 1", "step 2", ...],
+  "currentStep": 0,
+  "action": { "action": "clickAt", "x": 512, "y": 384 },
+  "reasoning": "I see [element description] at approximately (x, y). Clicking it to [purpose].",
+  "expectedEffect": "What should change after this action"
+}
+
+RULES:
+1. Respond with ONLY valid JSON, no markdown or extra text
+2. LOOK at the screenshot carefully — it is your primary information source
+3. Include plan, currentStep, reasoning, and expectedEffect in every response
+4. When the goal is achieved, use "complete" with a detailed result description
+5. If stuck after multiple attempts, use "abort" — don't loop forever
+6. If an action failed, try a DIFFERENT approach (different location, different strategy)
+7. For search: click the search box, type your query, then press Enter
+8. For navigation: click visible links or use the "navigate" action for direct URLs
+9. BLOCKER-FIRST: if a modal, cookie banner, or error dialog blocks progress, dismiss it first
+10. Use runScript or extractWithIndex when you need to extract data that isn't clearly visible in the screenshot
+11. BATCH: when filling forms, you can type in one field, then immediately use clickAt on the next field. Plan multiple actions per turn when they are sequential and obvious.
+12. VERIFY BEFORE COMPLETING: Before using "complete", re-read the GOAL and check: does your result ACTUALLY answer what was asked? If the goal asks for "5 beauty salons with ratings > 4.8" and you only found 3, do NOT complete — keep searching. If the goal asks for a specific date/price/name and your result doesn't contain it, do NOT complete. Premature completion with wrong data is worse than using another turn.
+13. DATE PICKER BYPASS: If you encounter a complex date picker widget (calendar popup, date spinner) that is hard to interact with, DO NOT spend multiple turns clicking through calendar months. Instead, use "navigate" to construct a URL with the date parameters encoded. For Google Flights: navigate to google.com/travel/flights with search params. For Booking: navigate to booking.com/searchresults with checkin/checkout params. URL-based date setting is faster and more reliable than fighting date picker UIs.
+
+REASONING FRAMEWORK:
+1. What do I see in the screenshot? Describe the visual layout.
+2. Where is the element I need to interact with? Estimate its (x, y) coordinates.
+3. What is the smallest action that makes progress toward the goal?
+4. If my last action failed, WHY did it fail? Try a different location or strategy.`;
+// Gen 15: Unified vision+DOM prompt. The model sees BOTH the screenshot AND
+// the ARIA snapshot with @refs. It can use EITHER coordinate actions (clickAt/
+// typeAt for visual targets) OR ref actions (click/type/fill for DOM elements).
+// This lets it pick the best tool per interaction: vision for visual layout,
+// DOM for precise form interaction.
+const UNIFIED_VISION_DOM_PROMPT = `You are a browser automation agent with TWO input modalities: a screenshot showing the visual page state, and a structured ELEMENTS list with interactive element refs.
+
+Use BOTH together:
+- The screenshot shows layout, visual state, images, icons — what a human sees
+- The ELEMENTS list shows interactive elements with @ref IDs for precise targeting
+
+ACTIONS — pick the best tool for each interaction:
+
+LABEL ACTIONS (PREFERRED — use the [N] numbered labels visible on the screenshot):
+- {"action": "clickLabel", "label": 3} — click element labeled [3] in the screenshot
+- {"action": "typeLabel", "label": 5, "text": "query"} — click [5] then type text
+The screenshot has numbered red badges on interactive elements. Use these labels — they're MORE ACCURATE than coordinate guessing.
+
+REF ACTIONS (use for form fields, buttons, links with clear @refs from ELEMENTS):
+- {"action": "click", "selector": "@REF"}
+- {"action": "type", "selector": "@REF", "text": "text"}
+- {"action": "press", "selector": "@REF", "key": "Enter"}
+- {"action": "select", "selector": "@REF", "value": "option"}
+- {"action": "fill", "fields": {"@REF1": "val1", "@REF2": "val2"}} — batch fill multiple form fields
+
+COORDINATE ACTIONS (fallback when no label or ref is available):
+- {"action": "clickAt", "x": 512, "y": 384} — click at pixel (x, y) in 1024×768 space
+- {"action": "typeAt", "x": 300, "y": 200, "text": "query"} — click + type
+
+SHARED ACTIONS:
+- {"action": "scroll", "direction": "up" | "down", "amount": 500}
+- {"action": "navigate", "url": "https://..."}
+- {"action": "wait", "ms": 1000}
+- {"action": "runScript", "script": "..."} — run JS in page context
+- {"action": "extractWithIndex", "query": "p, span", "contains": "keyword"}
+- {"action": "complete", "result": "description"}
+- {"action": "abort", "reason": "why"}
+
+WHEN TO USE WHICH (priority order):
+1. Element has a [N] label in the screenshot → use clickLabel/typeLabel (most accurate)
+2. Element has an @ref in ELEMENTS → use click/type/fill (fast and precise)
+3. Element is visible but has no label or ref → use clickAt/typeAt (coordinate fallback)
+- Date pickers, dropdown items rendered dynamically → use clickLabel if labeled, else clickAt
+
+RESPONSE FORMAT — respond with ONLY a JSON object:
+{
+  "plan": ["step 1", "step 2", ...],
+  "currentStep": 0,
+  "action": { "action": "click", "selector": "@REF" },
+  "nextActions": [{ "action": "type", "selector": "@REF2", "text": "query" }],
+  "reasoning": "Why I chose this action",
+  "expectedEffect": "What should change"
+}
+
+NOTE: "nextActions" is optional — include up to 3 safe follow-up actions (click, type, press, clickAt, typeAt, scroll) that are DETERMINISTIC given the current state. For example: click a search box THEN type a query. This saves turns.
+
+RULES:
+1. Respond with ONLY valid JSON
+2. Use @ref selectors from ELEMENTS when available — they are stable and precise
+3. Fall back to clickAt coordinates when the target has no ref or is visual-only
+4. LOOK at the screenshot — it shows visual state the ELEMENTS list may miss
+5. When the goal is achieved, use "complete" with a detailed result
+6. BLOCKER-FIRST: dismiss modals, cookie banners, login walls before continuing
+7. BATCH FILL: when 2+ form fields are visible with refs, use a single "fill" action
+8. If stuck after multiple attempts, use "abort"
+9. VERIFY BEFORE COMPLETING: Before using "complete", re-read the GOAL and check: does your result ACTUALLY answer what was asked? If the goal asks for specific data (prices, names, ratings, counts) and your result doesn't contain ALL of them, keep going. Premature completion with partial data is worse than using another turn.
+10. FORM RESET DETECTION: After batch-filling a form, verify values stuck via runScript. If fields reset to defaults, switch to keyboard-only: click field → type value → wait for autocomplete → press Enter → Tab to next. Do NOT re-fill with the same approach if it reset once.
+11. DATE PICKER STRATEGY: When a calendar popup opens over a date field:
+  a. Press Escape to dismiss, then type the date directly (e.g., "Jan 25, 2026").
+  b. If typing doesn't stick, use runScript to find clickable dates: document.querySelectorAll('[data-iso],[aria-label*="25"]').
+  c. NEVER spend more than 4 turns on a single date field. Switch to search fallback after 4 attempts.
+12. SEARCH ENGINE FALLBACK: When BOTH form interaction AND URL construction have failed (3+ turns stuck):
+  a. Navigate to https://duckduckgo.com/?q={goal rephrased as a search query}
+  b. Read search results or click through to comparison sites (Kayak, Skyscanner) for the data.
+  c. IMPORTANT: Do NOT use google.com/search — it blocks automated browsers with CAPTCHAs. Always use duckduckgo.com.
+  d. This is a LAST RESORT — do not use it as a first approach.`;
 /** Pattern for detecting data-extraction keywords in goal text */
 const DATA_EXTRACTION_PATTERN = /\b(extract|list|find|data|price|pric|names?|rating|cost|count)\b/i;
 /** Pattern for detecting search-related roles in snapshot text */
@@ -199,6 +370,7 @@ export class Brain {
     maxHistoryTurns;
     visionEnabled;
     visionStrategy;
+    observationMode;
     llmTimeoutMs;
     compactFirstTurn;
     lastDecisionUrl;
@@ -206,6 +378,13 @@ export class Brain {
     scoutModelName;
     scoutProvider;
     scoutUseVision;
+    // Gen 28: per-role model overrides
+    plannerModel;
+    plannerProvider;
+    verifierModel;
+    verifierProvider;
+    supervisorModel;
+    supervisorProvider;
     sandboxBackendType;
     sandboxBackendProfile;
     sandboxBackendProvider;
@@ -228,6 +407,7 @@ export class Brain {
         this.maxHistoryTurns = config.maxHistoryTurns || 10;
         this.visionEnabled = config.vision !== false;
         this.visionStrategy = config.visionStrategy ?? (this.visionEnabled ? 'always' : 'never');
+        this.observationMode = config.observationMode ?? 'dom';
         this.compactFirstTurn = config.compactFirstTurn === true;
         this.sandboxBackendType = config.sandboxBackendType;
         this.sandboxBackendProfile = config.sandboxBackendProfile;
@@ -235,6 +415,19 @@ export class Brain {
         this.scoutModelName = config.scout?.model;
         this.scoutProvider = config.scout?.provider;
         this.scoutUseVision = config.scout?.useVision === true;
+        // Gen 28: per-role model overrides
+        this.plannerModel = config.models?.planner?.model;
+        this.plannerProvider = config.models?.planner?.provider;
+        this.verifierModel = config.models?.verifier?.model;
+        this.verifierProvider = config.models?.verifier?.provider;
+        this.supervisorModel = config.models?.supervisor?.model;
+        this.supervisorProvider = config.models?.supervisor?.provider;
+        // executor uses navModel (already wired) — config.models.executor overrides it
+        if (config.models?.executor) {
+            this.navModelName = config.models.executor.model;
+            this.navProvider = config.models.executor.provider || this.navProvider;
+            this.adaptiveModelRouting = true; // enable routing when executor model is set
+        }
     }
     resolveModelName(provider, requestedModel) {
         return resolveProviderModelName(provider, requestedModel, {
@@ -517,10 +710,18 @@ export class Brain {
      * Current strategy: nav model is ONLY used for verification (see verifyGoalCompletion),
      * not for decide(). The flag is kept for future experiments with better routing signals.
      */
-    shouldUseNavigationModel(_state, _extraContext, _turnInfo) {
-        // Disabled for decide() — primary model is more cost-effective overall.
-        // Verification still routes to nav model (separate code path).
-        return false;
+    shouldUseNavigationModel(state, extraContext, turnInfo) {
+        if (!this.adaptiveModelRouting || !this.navModelName)
+            return false;
+        // Gen 22: use cheap model for DOM-only same-page turns (form filling,
+        // clicking known elements). Keep expensive model for first turn, new
+        // pages, and error recovery where reasoning matters.
+        const isFirstTurn = !turnInfo || turnInfo.current <= 1;
+        const samePageAsPrevious = this.lastDecisionUrl === state.url;
+        const hasError = extraContext?.includes('REJECTED') || extraContext?.includes('ERROR');
+        if (isFirstTurn || !samePageAsPrevious || hasError)
+            return false;
+        return true;
     }
     /**
      * Build the system prompt dynamically, injecting conditional rule groups
@@ -850,6 +1051,10 @@ export class Brain {
         });
     }
     async decide(goal, state, extraContext, turnInfo, options) {
+        // Gen 13: vision-first or hybrid mode — delegate to the vision path
+        if (this.observationMode === 'vision' || this.observationMode === 'hybrid') {
+            return this.decideVision(goal, state, extraContext, turnInfo);
+        }
         const useCompactFirstTurn = this.compactFirstTurn && turnInfo?.current === 1;
         const samePageAsPrevious = this.lastDecisionUrl === state.url;
         const isFirstTurn = !turnInfo || turnInfo.current <= 1;
@@ -865,8 +1070,11 @@ export class Brain {
             && diffChanges > 0
             && diffTotal > 0
             && diffChanges / diffTotal < 0.3;
-        // Tighter snapshot budget on same-page turns — agent already saw the full page
-        const snapshotBudget = samePageAsPrevious ? 8_000 : 16_000;
+        // Tighter snapshot budget on same-page turns — agent already saw the full page.
+        // Gen 10C: raised new-page budget from 16k to 24k so extraction tasks (MDN/Python
+        // docs/W3C spec) get the full <dl>/<code>/<pre> content the LLM needs to write
+        // a working runScript on the first try.
+        const snapshotBudget = samePageAsPrevious ? 8_000 : 24_000;
         let visibleSnapshot;
         let elementsHeader;
         if (useDiffOnly) {
@@ -1013,6 +1221,135 @@ ${visibleSnapshot}`;
             modelUsed: effectiveModel,
         };
     }
+    /**
+     * Gen 13: Vision-first decision path. The screenshot is the primary
+     * observation; DOM snapshot is minimal context (URL, title only in pure
+     * vision mode, or compact DOM in hybrid mode). The LLM outputs
+     * coordinate-based actions (clickAt, typeAt) in 1024×768 virtual space.
+     */
+    async decideVision(goal, state, extraContext, turnInfo) {
+        this.lastDecisionUrl = state.url;
+        // Gen 18: adaptive observation — diff-focused on same-page turns.
+        // When the page changed slightly after an action (modal opened, dropdown
+        // expanded, content loaded), the DIFF is the signal. Send only what
+        // changed instead of the full snapshot. Saves 3-5k tokens per turn.
+        const isHybrid = this.observationMode === 'hybrid';
+        const samePageAsPrevious = this.lastDecisionUrl === state.url;
+        const isFirstTurn = !turnInfo || turnInfo.current <= 1;
+        const rawDiff = state.snapshotDiffRaw;
+        const diffChanges = rawDiff ? rawDiff.added.length + rawDiff.removed.length + rawDiff.changed.length : 0;
+        const diffTotal = rawDiff ? diffChanges + rawDiff.unchangedCount : 0;
+        const useDiffOnly = isHybrid && samePageAsPrevious && !isFirstTurn
+            && rawDiff !== undefined && diffChanges > 0 && diffTotal > 0
+            && diffChanges / diffTotal < 0.4;
+        let textContent = `GOAL: ${goal}
+
+CURRENT PAGE:
+URL: ${state.url}
+Title: ${state.title}`;
+        if (isHybrid && state.snapshot) {
+            if (useDiffOnly) {
+                // Diff-focused: only what changed since last turn
+                const lines = [];
+                if (rawDiff.added.length)
+                    lines.push('ADDED:', ...rawDiff.added);
+                if (rawDiff.changed.length)
+                    lines.push('CHANGED:', ...rawDiff.changed);
+                if (rawDiff.removed.length)
+                    lines.push('REMOVED:', ...rawDiff.removed);
+                lines.push(`(${rawDiff.unchangedCount} elements unchanged — refs from previous turn still valid)`);
+                textContent += `\n\nPAGE CHANGES (what changed after your last action — this is the important part):\n${lines.join('\n')}`;
+            }
+            else {
+                // Progressive budget reduction: more turns on same page = less snapshot
+                // needed (agent has already seen the full page, rely on screenshot + diff).
+                const sameTurnCount = samePageAsPrevious ? (turnInfo?.current || 0) : 0;
+                const snapshotBudget = samePageAsPrevious
+                    ? (sameTurnCount >= 8 ? 2_500 : 4_000) // aggressive after 8+ same-page turns
+                    : 6_000;
+                const snap = budgetSnapshot(state.snapshot, snapshotBudget);
+                textContent += `\n\nELEMENTS:\n${snap}`;
+            }
+        }
+        if (turnInfo) {
+            const remaining = turnInfo.max - turnInfo.current;
+            textContent += `\n\nTURN: ${turnInfo.current}/${turnInfo.max} (${remaining} remaining)`;
+            if (remaining === 1) {
+                textContent += ` — FINAL TURN: return a terminal action only (complete or abort)`;
+            }
+            else if (remaining <= 3) {
+                textContent += ` — RUNNING LOW, prioritize completing the goal or aborting`;
+            }
+        }
+        if (extraContext) {
+            textContent += `\n\n${extraContext}`;
+        }
+        textContent += '\n\nLook at the screenshot. What action should you take?';
+        // Screenshot is required for vision-first mode
+        if (!state.screenshot) {
+            return {
+                action: { action: 'wait', ms: 500 },
+                reasoning: 'No screenshot available for vision-first mode — waiting for page to render',
+                raw: '{"action":{"action":"wait","ms":500}}',
+            };
+        }
+        const userContent = [
+            { type: 'text', text: textContent },
+            { type: 'image', image: state.screenshot, mediaType: 'image/jpeg' },
+        ];
+        // Gen 14: strip ALL screenshots from history. The current turn's
+        // screenshot is the only image the model needs — old screenshots are
+        // dead weight that pushes cumulative token count past the cost cap.
+        // This alone fixes 3/5 Gen 13 failures (cost_cap at 101-106k).
+        const compacted = this.compactHistory().map((msg) => {
+            if (msg.role !== 'user' || !Array.isArray(msg.content))
+                return msg;
+            const textOnly = msg.content
+                .filter((part) => part.type === 'text');
+            if (textOnly.length === msg.content.length)
+                return msg;
+            return { ...msg, content: textOnly };
+        });
+        const messages = [
+            ...compacted,
+            { role: 'user', content: userContent },
+        ];
+        // Gen 27: model cascade for vision turns. Same-page non-error turns
+        // use the cheap nav model (gpt-4.1-mini) since they're just following
+        // instructions, not reasoning about new pages.
+        const useNavModel = this.shouldUseNavigationModel(state, extraContext, turnInfo);
+        const effectiveModel = useNavModel ? (this.navModelName || this.modelName) : this.modelName;
+        const modelOpts = { provider: this.provider, model: effectiveModel };
+        const nearingEnd = turnInfo && turnInfo.current >= turnInfo.max - 3;
+        const maxTokens = nearingEnd ? 1200 : 600;
+        // Gen 15: hybrid uses the unified prompt with both action vocabularies
+        const systemPrompt = isHybrid ? UNIFIED_VISION_DOM_PROMPT : VISION_FIRST_PROMPT;
+        const result = await this.generate(systemPrompt, messages, modelOpts, maxTokens);
+        const raw = result.text;
+        if (!raw) {
+            throw new Error('Brain.decideVision: LLM returned empty response');
+        }
+        if (this.debug) {
+            console.log('[Brain/Vision] Response:', raw.slice(0, 300));
+        }
+        const parsed = this.parse(raw);
+        this.history.push({ role: 'user', content: userContent });
+        this.history.push({ role: 'assistant', content: raw });
+        const maxMessages = this.maxHistoryTurns * 2;
+        if (this.history.length > maxMessages) {
+            this.history = this.history.slice(-maxMessages);
+        }
+        return {
+            ...parsed,
+            raw,
+            tokensUsed: result.tokensUsed,
+            inputTokens: result.inputTokens,
+            outputTokens: result.outputTokens,
+            cacheReadInputTokens: result.cacheReadInputTokens,
+            cacheCreationInputTokens: result.cacheCreationInputTokens,
+            modelUsed: this.modelName,
+        };
+    }
     /**
      * Gen 7: ONE LLM call generates a structured plan for the entire task.
      *
@@ -1034,10 +1371,12 @@ ${visibleSnapshot}`;
         const startedAt = Date.now();
         const maxSteps = options?.maxSteps ?? 12;
         const extraContext = options?.extraContext;
-        // Snapshot budget for the planner: keep it conservative — the planner
-        // prompt is itself substantial, and we want the LLM to focus on the
-        // STRUCTURE of the page rather than every leaf element.
-        const snapshot = budgetSnapshot(state.snapshot, 12_000);
+        // Snapshot budget for the planner: Gen 10C raised from 12k to 24k. The
+        // planner is the most important caller for extraction tasks because it
+        // writes the runScript that runs on the first observation. Without enough
+        // snapshot context (especially `<dl>/<dt>/<code>/<pre>` content lines that
+        // Gen 10C now preserves), the planner emits selectors that don't exist.
+        const snapshot = budgetSnapshot(state.snapshot, 24_000);
         const planSystemPrompt = `You are a planning engine for a browser automation agent.
 
 Given a user goal and the current page state, your job is to generate a complete, ordered plan of actions that the agent will execute deterministically without re-entering you between steps. After each step, the runner verifies your stated post-condition. If verification fails the runner falls back to a per-action loop, so your job is to write a plan that requires the FEWEST steps and where every step's post-condition is reliably observable.
@@ -1061,19 +1400,24 @@ KEY PRINCIPLES:
 7. EXTRACTION TASKS: when the goal asks you to READ, EXTRACT, REPORT, or RETURN values from the page (numbers, text, lists, structured data), the LAST step of your plan MUST be \`runScript\`. Do NOT emit a \`complete\` step after the runScript with literal values in \`result\`, because at planning time you cannot know what runScript will return — any values you write would be fabricated. The runner has a deterministic substitution path: it will use the runScript output as the final result, OR fall through to per-action mode where the LLM can see the script output. Either way is fine. The wrong move is to put placeholder JSON like \`{"x":null,"y":null}\` or \`"<from prior step>"\` in the complete result; the runner detects and replaces those, but it's cleaner if you simply omit the complete step. RIGHT: \`[{action:runScript, script:"..."}]\`. WRONG: \`[{action:runScript,...}, {action:complete, result:"{x:null}"}]\`.
 
 ACTION VERBS (same as the per-action prompt):
-- {"action": "click", "selector": "@REF"}
-- {"action": "type", "selector": "@REF", "text": "..."}
+- {"action": "click", "selector": "@REF"} — use when the element has a ref in ELEMENTS
+- {"action": "type", "selector": "@REF", "text": "..."} — type into a ref element
 - {"action": "press", "selector": "@REF", "key": "Enter"}
 - {"action": "select", "selector": "@REF", "value": "..."}
+- {"action": "clickAt", "x": 512, "y": 384} — click at pixel coordinates (use when you can see the element in the screenshot but it has no ref)
+- {"action": "typeAt", "x": 300, "y": 200, "text": "..."} — click at coordinates then type
 - {"action": "scroll", "direction": "up"|"down", "amount": 500}
 - {"action": "navigate", "url": "..."}
 - {"action": "wait", "ms": 1000}
 - {"action": "fill", "fields": {"@a": "v1", "@b": "v2"}, "selects": {"@c": "v3"}, "checks": ["@d", "@e"]}
 - {"action": "clickSequence", "refs": ["@a", "@b", "@c"]}
 - {"action": "runScript", "script": "document.querySelector('.x').textContent"}
+- {"action": "extractWithIndex", "query": "p, span, dd, code", "contains": "downloads"} — return a NUMBERED list of visible elements matching the query with their full textContent. Use this for extraction tasks where the data lives in obscurely-classed wrappers (npm download counts, MDN \`<dl>/<dt>/<dd>\` content, Python docs \`<code>\` blocks, W3C spec content) and the planner cannot guarantee a precise selector. The next step (in plan or per-action mode) reads the result and picks the right index. STRONGLY PREFER THIS OVER runScript on ANY extraction task where the snapshot doesn't already show the value verbatim.
 - {"action": "complete", "result": "..."}
 - {"action": "abort", "reason": "..."}
 
+${URL_FIRST_RULES}
+
 RESPONSE FORMAT — respond with ONLY this JSON:
 {
   "reasoning": "1-2 sentence strategy summary",
@@ -1102,7 +1446,22 @@ ELEMENTS:
 ${snapshot}
 ${extraContext ? `\n${extraContext}\n` : ''}
 What is the complete plan?`;
-        const result = await this.generate(planSystemPrompt, [{ role: 'user', content: userText }], { provider: this.provider, model: this.modelName }, 
+        // Gen 20: vision-aware planner. When hybrid mode is active AND a
+        // screenshot is available, send the screenshot alongside the DOM so
+        // the planner can see the visual layout. This helps on sites like
+        // Google Flights where the DOM doesn't convey form structure well.
+        const isVisionPlanner = (this.observationMode === 'hybrid' || this.observationMode === 'vision') && !!state.screenshot;
+        const userContent = isVisionPlanner
+            ? [
+                { type: 'text', text: userText },
+                { type: 'image', image: state.screenshot, mediaType: 'image/jpeg' },
+            ]
+            : userText;
+        // Gen 28: planner can use a different model (e.g., Claude Opus for reasoning)
+        const planModelOpts = this.plannerModel
+            ? { provider: (this.plannerProvider || this.provider), model: this.plannerModel }
+            : { provider: this.provider, model: this.modelName };
+        const result = await this.generate(planSystemPrompt, [{ role: 'user', content: userContent }], planModelOpts, 
         // Plans need more output tokens than decide() — a 10-step plan with
         // batch fills + rationale per step is comfortably over 1000 tokens.
         2_500).catch((err) => ({
@@ -1358,13 +1717,12 @@ ${budgetSnapshot(state.snapshot)}${siteBoundaryNote ? `\n\n${siteBoundaryNote}`
 
 Was the goal actually achieved? Analyze the current page state carefully.`;
         const userContent = this.buildUserContent(textContent, state.screenshot, true);
-        // Verification is a structured yes/no task — use nav model if available
-        const verifyProvider = this.adaptiveModelRouting && this.navModelName
-            ? (this.navProvider || this.provider)
-            : undefined;
-        const verifyModel = this.adaptiveModelRouting && this.navModelName
-            ? this.navModelName
-            : undefined;
+        // Gen 28: verifier can use its own model, falls back to nav model, then main
+        const verifyProvider = this.verifierProvider
+            ? this.verifierProvider
+            : (this.adaptiveModelRouting && this.navModelName ? (this.navProvider || this.provider) : undefined);
+        const verifyModel = this.verifierModel
+            || (this.adaptiveModelRouting && this.navModelName ? this.navModelName : undefined);
         const result = await this.generate(`Verify whether the browser agent achieved its goal. Respond with ONLY JSON:
 {"achieved":true,"confidence":0.9,"evidence":["observation"],"missing":[]}
 
@@ -1546,8 +1904,11 @@ Only include facts that are genuinely useful. Quality over quantity. Max 10 fact
         const VALID_ACTIONS = new Set([
             'click', 'type', 'press', 'hover', 'select',
             'scroll', 'navigate', 'wait', 'evaluate', 'runScript',
+            'extractWithIndex',
             'verifyPreview', 'complete', 'abort',
             'fill', 'clickSequence',
+            'clickAt', 'typeAt',
+            'clickLabel', 'typeLabel',
         ]);
         try {
             const parsed = JSON.parse(text);
@@ -1602,6 +1963,26 @@ function deduplicateSnapshot(snapshot) {
     // Strip trailing numbers/ordinals to get a "name stem" for grouping.
     // "Page 1" and "Page 20" → "Page ", "Item #3" and "Item #42" → "Item #"
     const nameStem = (name) => name.replace(/\d+/g, '#');
+    // Structural fingerprint for a block of lines (element + its children).
+    // Used for card-level dedup: two hotel cards have different names but the
+    // same structure (listitem > link + img + text + text + button).
+    const structuralFingerprint = (startIdx, baseIndent) => {
+        const roles = [];
+        let j = startIdx;
+        while (j < lines.length) {
+            const p = parseLine(lines[j]);
+            if (!p) {
+                j++;
+                continue;
+            }
+            // Stop when we hit an element at the same or shallower indent (sibling or parent)
+            if (j > startIdx && p.indent.length <= baseIndent.length)
+                break;
+            roles.push(p.role);
+            j++;
+        }
+        return { fp: roles.join(','), endIdx: j };
+    };
     let i = 0;
     while (i < lines.length) {
         const parsed = parseLine(lines[i]);
@@ -1611,7 +1992,41 @@ function deduplicateSnapshot(snapshot) {
             i++;
             continue;
         }
-        // Collect a consecutive run of same (indent, role) with similar name stems
+        // Try block-level dedup first: look for consecutive sibling blocks
+        // with the same structural fingerprint (same child-role sequence).
+        // This catches card patterns like Booking hotel results, Allrecipes cards.
+        if (/\b(?:listitem|article|group|region)\b/i.test(parsed.role)) {
+            const { fp: firstFp, endIdx: firstEnd } = structuralFingerprint(i, parsed.indent);
+            if (firstEnd > i + 2 && firstFp.includes(',')) { // non-trivial block
+                const blocks = [
+                    { start: i, end: firstEnd, firstLine: lines[i] },
+                ];
+                let scanIdx = firstEnd;
+                while (scanIdx < lines.length) {
+                    const nextParsed = parseLine(lines[scanIdx]);
+                    if (!nextParsed || nextParsed.indent !== parsed.indent || nextParsed.role !== parsed.role)
+                        break;
+                    const { fp: nextFp, endIdx: nextEnd } = structuralFingerprint(scanIdx, nextParsed.indent);
+                    if (nextFp !== firstFp)
+                        break;
+                    blocks.push({ start: scanIdx, end: nextEnd, firstLine: lines[scanIdx] });
+                    scanIdx = nextEnd;
+                }
+                if (blocks.length >= 3) {
+                    // Emit first 2 blocks fully, summarize the rest
+                    for (let b = 0; b < Math.min(2, blocks.length); b++) {
+                        for (let k = blocks[b].start; k < blocks[b].end; k++) {
+                            out.push(lines[k]);
+                        }
+                    }
+                    const remaining = blocks.length - 2;
+                    out.push(`${parsed.indent}... [${remaining} more similar ${parsed.role} blocks with same structure]`);
+                    i = blocks[blocks.length - 1].end;
+                    continue;
+                }
+            }
+        }
+        // Line-level dedup: consecutive runs of same (indent, role, name stem)
         const group = [parsed];
         const stem = nameStem(parsed.name);
         let j = i + 1;
@@ -1640,39 +2055,53 @@ function deduplicateSnapshot(snapshot) {
     return out.join('\n');
 }
 /**
- * Cap snapshot size for non-first turns to control token cost on large pages.
- * Keeps the full snapshot when it fits within budget; otherwise truncates
- * non-interactive decorative lines first, then hard-caps with a notice.
+ * Cap snapshot size to control token cost on large pages.
+ * Keeps the full snapshot when it fits within budget; otherwise preserves:
+ *   1. Interactive elements with refs (buttons, inputs, links — for action targets)
+ *   2. Content lines: term/definition/code/pre/paragraph (for extraction tasks
+ *      like MDN, Python docs, W3C spec where the value the agent needs lives in
+ *      a `<dl>/<code>/<pre>` block, not in an interactive element)
+ *
+ * Gen 10C raised the default from 16k to 24k to give planners on extraction
+ * pages more room before truncation kicks in. The 8k same-page budget remains
+ * unchanged because by then the LLM has already seen the full snapshot once.
  */
-function budgetSnapshot(snapshot, maxChars = 16_000) {
+export function budgetSnapshot(snapshot, maxChars = 24_000) {
     // Skip dedup on small snapshots — not enough repetition to justify the O(n) scan
     if (snapshot.length > 6_000) {
         snapshot = deduplicateSnapshot(snapshot);
     }
     if (snapshot.length <= maxChars)
         return snapshot;
-    // First pass: drop non-interactive lines (images, paragraphs, decorative text)
-    // to keep interactive elements (buttons, links, textboxes, headings).
+    // First pass: separate keep-set (interactive + content lines) from decorative.
+    // Content roles (term, definition, code, pre, paragraph) carry text the LLM
+    // needs for extraction tasks. They have no [ref=] but the text is the data.
     const lines = snapshot.split('\n');
     const interactive = [];
+    const content = [];
     const decorative = [];
     for (const line of lines) {
         if (/\b(?:button|link|textbox|combobox|menuitem|checkbox|radio|select|heading|dialog|alertdialog)\b/i.test(line) && /\[ref=/.test(line)) {
             interactive.push(line);
         }
+        else if (/^\s*-\s+(?:term|definition|code|pre|paragraph)\b/i.test(line)) {
+            content.push(line);
+        }
         else {
             decorative.push(line);
         }
     }
-    // If interactive-only fits, use it with a truncation note
-    const interactiveText = interactive.join('\n');
-    if (interactiveText.length <= maxChars) {
-        return interactiveText + `\n... [${decorative.length} decorative elements omitted for brevity]`;
+    // If interactive + content fits, use both with a truncation note
+    const keepSet = interactive.concat(content);
+    const keepText = keepSet.join('\n');
+    if (keepText.length <= maxChars) {
+        return keepText + `\n... [${decorative.length} decorative elements omitted for brevity]`;
     }
-    // Second pass: when interactive elements still exceed budget, prioritize:
-    // 1. searchbox/textbox/combobox (inputs — essential for form tasks)
-    // 2. headings (structural navigation)
-    // 3. links/buttons (main content — keep all, trim from end as last resort)
+    // Second pass: when interactive + content still exceed budget, prioritize:
+    // 1. inputs (searchbox/textbox/combobox) — essential for form tasks
+    // 2. headings + dialogs — structural navigation
+    // 3. content lines (term/definition/code/pre) — extraction data
+    // 4. bulk links/buttons — main content
     const priority = [];
     const bulk = [];
     for (const line of interactive) {
@@ -1683,7 +2112,10 @@ function budgetSnapshot(snapshot, maxChars = 16_000) {
             bulk.push(line);
         }
     }
-    const priorityText = priority.join('\n');
+    // Content lines come right after priority interactive (they're the extraction
+    // data) and before bulk links/buttons.
+    const priorityWithContent = priority.concat(content);
+    const priorityText = priorityWithContent.join('\n');
     const remaining = maxChars - priorityText.length - 80; // reserve space for note
     if (remaining > 0) {
         const bulkText = bulk.join('\n');
@@ -1768,6 +2200,15 @@ function validateAction(actionType, data) {
             return { action: 'evaluate', criteria: optStr('criteria') };
         case 'runScript':
             return { action: 'runScript', script: requireStr('script') };
+        case 'extractWithIndex': {
+            const query = requireStr('query');
+            const contains = typeof data.contains === 'string' ? data.contains : '';
+            return {
+                action: 'extractWithIndex',
+                query,
+                ...(contains ? { contains } : {}),
+            };
+        }
         case 'verifyPreview':
             return { action: 'verifyPreview' };
         case 'complete':
@@ -1807,6 +2248,16 @@ function validateAction(actionType, data) {
                 ...(typeof data.intervalMs === 'number' ? { intervalMs: data.intervalMs } : {}),
             };
         }
+        // Gen 13: Vision-first coordinate actions
+        case 'clickAt':
+            return { action: 'clickAt', x: num(data.x, 0), y: num(data.y, 0) };
+        case 'typeAt':
+            return { action: 'typeAt', x: num(data.x, 0), y: num(data.y, 0), text: optStr('text') };
+        // Gen 23: SoM label-based actions
+        case 'clickLabel':
+            return { action: 'clickLabel', label: num(data.label, 0) };
+        case 'typeLabel':
+            return { action: 'typeLabel', label: num(data.label, 0), text: optStr('text') };
         default:
             throw new Error(`Unknown action type: ${actionType}`);
     }