browserclaw-agent 0.1.0 → 0.2.0

package/dist/agent-loop.d.ts

@@ -1,6 +1,7 @@
 import type { CrawlPage, BrowserClaw } from 'browserclaw';
 import type { UserMessage } from './types.js';
-import type { AgentAction, AgentLoopResult, CatalogSkill } from './types.js';
+import type { AgentAction, AgentLoopResult, CatalogSkill, TaskLesson } from './types.js';
+import type { AgentConfig } from './config.js';
 /**
  * Wrap drained user interjections in a nonce-tagged block so the LLM can
  * distinguish user data from system instructions. Sanitizes the text to
@@ -26,24 +27,31 @@ export declare function shouldCheckTermination(step: number): boolean;
 export interface PageHolder {
     page: CrawlPage;
 }
+/**
+ * Extension points for consumers of the agent loop.
+ *
+ * - `systemPrompt`   — Replace or extend the main system prompt.
+ * - `customActions`  — Handle action types not built into the agent.
+ *                      Return `{ outcome }` to override the step outcome text.
+ * - `buildTask`      — Pre-process the user prompt before the planner sees it.
+ *                      Useful for adding context or domain constraints.
+ * - `getLesson`      — Inject a lesson provider (replaces the default S3-backed store).
+ *                      Return `null` to skip lesson loading.
+ */
+export interface AgentLoopOptions {
+    systemPrompt?: string | ((defaultPrompt: string) => string);
+    customActions?: Record<string, (action: AgentAction, page: CrawlPage, browser?: BrowserClaw) => Promise<{
+        outcome?: string;
+    } | undefined>>;
+    buildTask?: (prompt: string) => string | Promise<string>;
+    getLesson?: (prompt: string) => Promise<TaskLesson | null>;
+    config?: Partial<AgentConfig>;
+}
 export interface UserChatHooks {
     /** Drain all pending user interjections. Called at the top of each step. */
     drainMessages: () => UserMessage[];
     /** Per-session random token used to wrap user-interjection content so the LLM can't confuse user data for system instructions. */
     nonce: string;
 }
-export interface AgentLoopOptions {
-    waitForUser?: () => Promise<string>;
-    browser?: BrowserClaw;
-    domainSkill?: CatalogSkill | null;
-    maxSteps?: number;
-    userChat?: UserChatHooks;
-    /** Custom action handlers keyed by action name. Called before built-in actions. */
-    customActions?: Record<string, (action: AgentAction, page: CrawlPage) => Promise<void>>;
-    /** Override the default agent system prompt. */
-    systemPrompt?: string;
-    /** Transform the prompt before it reaches the planner and agent loop. */
-    buildTask?: (prompt: string) => string;
-}
-export declare function runAgentLoop(prompt: string, pageOrHolder: CrawlPage | PageHolder, emit: (event: string, data: unknown) => void, signal: AbortSignal, options?: AgentLoopOptions): Promise<AgentLoopResult>;
+export declare function runAgentLoop(prompt: string, pageOrHolder: CrawlPage | PageHolder, emit: (event: string, data: unknown) => void, signal: AbortSignal, waitForUser?: () => Promise<string>, browser?: BrowserClaw, domainSkill?: CatalogSkill | null, maxSteps?: number, userChat?: UserChatHooks, options?: AgentLoopOptions): Promise<AgentLoopResult>;
 //# sourceMappingURL=agent-loop.d.ts.map

package/dist/agent-loop.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agent-loop.d.ts","sourceRoot":"","sources":["../src/agent-loop.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAU1D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAI9C,OAAO,KAAK,EAAE,WAAW,EAAa,eAAe,EAAiB,YAAY,EAAc,MAAM,YAAY,CAAC;AAanH;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAiC5F;AA8kBD,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAsB1D;AAMD,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAIvE;AA0ND,MAAM,MAAM,mBAAmB,GAAG;IAAE,KAAK,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,KAAK,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAEtG;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAG5D;AA2CD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,SAAS,CAAC;CACjB;AAED,MAAM,WAAW,aAAa;IAC5B,4EAA4E;IAC5E,aAAa,EAAE,MAAM,WAAW,EAAE,CAAC;IACnC,kIAAkI;IAClI,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,gBAAgB;IAC/B,WAAW,CAAC,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACpC,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,WAAW,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,mFAAmF;IACnF,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IACxF,gDAAgD;IAChD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,yEAAyE;IACzE,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;CACxC;AAED,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,SAAS,GAAG,UAAU,EACpC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,KAAK,IAAI,EAC5C,MAAM,EAAE,WAAW,EACnB,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,CAAC,CAm1B1B"}
+{"version":3,"file":"agent-loop.d.ts","sourceRoot":"","sources":["../src/agent-loop.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAW1D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAG9C,OAAO,KAAK,EAAE,WAAW,EAAa,eAAe,EAAiB,YAAY,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACnH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAY/C;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAiC5F;AA4lBD,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAsB1D;AAMD,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAIvE;AAoND,MAAM,MAAM,mBAAmB,GAAG;IAAE,KAAK,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,KAAK,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAEtG;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAG5D;AA2CD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,SAAS,CAAC;CACjB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,gBAAgB;IAC/B,YAAY,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,aAAa,EAAE,MAAM,KAAK,MAAM,CAAC,CAAC;IAC5D,aAAa,CAAC,EAAE,MAAM,CACpB,MAAM,EACN,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,CAC3G,CAAC;IACF,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACzD,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IAC3D,MAAM,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,4EAA4E;IAC5E,aAAa,EAAE,MAAM,WAAW,EAAE,CAAC;IACnC,kIAAkI;IAClI,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,SAAS,GAAG,UAAU,EACpC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,KAAK,IAAI,EAC5C,MAAM,EAAE,WAAW,EACnB,WAAW,CAAC,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,EACnC,OAAO,CAAC,EAAE,WAAW,EACrB,WAAW,CAAC,EAAE,YAAY,GAAG,IAAI,EACjC,QAAQ,CAAC,EAAE,MAAM,EACjB,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,CAAC,CAw4B1B"}

package/dist/agent-loop.js

@@ -6,12 +6,21 @@ import { detectPageState, shouldBlockDone } from './skills/page-state.js';
 import { diagnoseStuckAgent, formatRecovery } from './skills/recovery.js';
 import { TabManager } from './skills/tab-manager.js';
 import { getCdpBaseUrl, activateCdpTarget } from './skills/cdp-utils.js';
+import { extractItems } from './skills/extract-items.js';
 import { webSearch } from './web-search.js';
 import { llmJson, llmVision, sanitizeErrorText, getTokenUsage } from './llm.js';
-import { getLesson, formatLessonForPrompt } from './lesson-store.js';
 import { LlmParseError } from './types.js';
+import { defaultAgentConfig, INTERJECTION_INJECTION_MAX_CHARS } from './config.js';
 import { logger } from './logger.js';
-import { WAIT_AFTER_TYPE_MS, WAIT_AFTER_CLICK_MS, WAIT_AFTER_OTHER_MS, WAIT_ACTION_MS, SCROLL_PIXELS, LLM_MAX_TOKENS, MAX_STEPS, INTERJECTION_INJECTION_MAX_CHARS, } from './config.js';
+function formatLessonForPrompt(lesson) {
+    const worked = lesson.domains.filter((d) => d.status === 'worked');
+    if (worked.length === 0)
+        return '';
+    const lines = ['Sites that completed this type of task before (use if they still fit):'];
+    for (const d of worked)
+        lines.push(`  ${d.domain} (${d.reason})`);
+    return lines.join('\n');
+}
 /**
  * Wrap drained user interjections in a nonce-tagged block so the LLM can
  * distinguish user data from system instructions. Sanitizes the text to
@@ -105,8 +114,13 @@ Rules:
 - "back" to go back in browser history. Use this instead of manually tracking URLs when you need to return to the previous page.
 - "press_and_hold" solves press-and-hold human-verification challenges. Include "hold_ms" (milliseconds) to override the default duration when a prior attempt was too short or the UI specifies a time.
 - "click_cloudflare" solves Cloudflare "Verify you are human" checkbox challenges. The system locates and clicks the checkbox.
-- "extract" when the accessibility snapshot is missing data you need — prices, descriptions, table values, form field values, counts, or any text that's visually on the page but absent from the snapshot. Provide a JavaScript expression; the result appears in the next step. Examples: 'document.querySelector(".price")?.textContent', 'Array.from(document.querySelectorAll("td")).map(el=>el.textContent.trim())'. Use this like a human would: when you can see there's content but can't read it from the snapshot.
+- "extract" when the accessibility snapshot is missing data you need — prices, descriptions, table values, form field values, counts, or any text that's visually on the page but absent from the snapshot.
+  - With an "expression" (JavaScript string): runs that expression and returns the value. Examples: 'document.querySelector(".price")?.textContent', 'Array.from(document.querySelectorAll("td")).map(el=>el.textContent.trim())'.
+  - Without an "expression": runs a built-in structured-items extractor that tries __NEXT_DATA__ → Apollo state → __INITIAL_STATE__ → JSON-LD → DOM card heuristics, in that order, and returns an array of records with fields like price, address, bedrooms, url. Prefer this on list/results pages — it is more reliable than hand-written selector code.
 - On modern JS/SPA sites (Next.js, React apps), listing and detail data is usually embedded in structured state — try these first before hand-rolling selectors: JSON.parse(document.getElementById('__NEXT_DATA__').textContent), window.__APOLLO_STATE__, window.__INITIAL_STATE__, or JSON-LD via document.querySelectorAll('script[type="application/ld+json"]'). One well-aimed extract from structured state beats five DOM-selector attempts.
+- On any filtered search-results page where you need structured item data (prices, addresses, names, URLs across multiple cards), your FIRST extract must use the built-in extractor: call "extract" with NO "expression" field at all. It auto-tries __NEXT_DATA__ / Apollo / __INITIAL_STATE__ / JSON-LD / DOM cards and returns an array of records. Do not write your own selector code for list pages — it fails more often than it succeeds. Only if the built-in extractor's "source" comes back "none" should you attempt a custom "expression".
+- If the built-in extract returned items whose URLs are present but some named per-item fields are missing (e.g. JSON-LD often carries only the URL), do not give up. Navigate to each item URL in turn and run another "extract" with NO "expression" — the built-in extractor also handles single-item detail pages and will return the missing fields. Collect enough complete items to satisfy the task's count before calling done.
+- On cards with nested links (photo, title, container), the outer link often routes to a promo or profile rather than the listing. When similar links stack within a card, extract hrefs first to disambiguate — click the link whose URL path matches the target, not one guessed from the visible label.
 - "web_search" to search the web when you don't know which site to use, need to find a specific service, or want to compare options across sources. Provide "query"; returns top results with titles, URLs, and snippets. Use this instead of guessing URLs.
 - "switch_tab" to switch to a different open tab. Use the tab_id from the tab list shown in the context. Use this when a link opened a new tab and you want to return to a previous one, or when the information you need is in a different tab.
 - "close_tab" to close a tab by tab_id. Use only when a tab is no longer needed.
@@ -192,9 +206,12 @@ Before giving up:
 - Only "fail" after you've genuinely exhausted your options.
 
 When to call "done":
-- The moment your "memory" contains an answer to the user's core question, "done" is your next action. Missing a sub-detail is not a reason to keep gathering.
-- A partial, honest answer delivered now beats a complete answer you never deliver. State uncertainty explicitly — "couldn't verify X" is a valid part of a done answer.
-- Do not extract, scroll, or click "just to be sure" once you can answer. That is how runs time out.
+- First, re-read the original prompt. Every explicit constraint (filters like "dog-friendly", "under $4,200", "in Chelsea"; counts like "5 listings"; date/time bounds) must be satisfied by an action you took and confirmed on the page — not merely mentioned in memory. If a constraint hasn't been applied and verified, apply it now; do not call done yet.
+- Once all constraints are satisfied and your "memory" contains the answer, "done" is your next action. Missing a sub-detail (one nice-to-have field) is not a reason to keep gathering.
+- When the task asks for a list of items with named per-item fields, every named field is required for every item you report. If an item is missing any named field, do a focused follow-up extract for that field, or drop the item and get another that has all fields. Never call done with items that have "not recovered" or "unknown" in a named field.
+- If an item you considered fails one of the task's explicit constraints (price over cap, wrong location, wrong date, etc.), drop it and keep hunting — open the next candidate from the list, scroll for more, or paginate. Do not call done with fewer qualifying items than the task asked for just because the first few you inspected didn't qualify.
+- A partial, honest answer delivered now beats a complete answer you never deliver. State uncertainty explicitly — "couldn't verify X" is a valid part of a done answer, but only for optional fields, never for explicit constraints.
+- Do not extract, scroll, or click "just to be sure" once every constraint is satisfied. That is how runs time out.
 - Exception — transactional tasks (book, buy, submit, send): verify the action actually completed. A click on "Submit" is not the same as a successful submission; look for confirmation text, a reference number, or a state change before calling done.
 
 Data grounding:
@@ -223,16 +240,20 @@ function isPageReady(snapshot) {
 }
 const PAGE_READY_RETRIES = 2;
 const PAGE_READY_WAIT_MS = 2000;
+const SNAPSHOT_TIMEOUT_MS = 10000;
 async function safeSnapshot(page) {
     let snapshot;
+    const t0 = Date.now();
+    logger.info('snapshot: starting');
     try {
-        snapshot = (await page.snapshot({ interactive: true, compact: true })).snapshot;
+        snapshot = (await page.snapshot({ interactive: true, compact: true, timeoutMs: SNAPSHOT_TIMEOUT_MS })).snapshot;
+        logger.info({ ms: Date.now() - t0 }, 'snapshot: complete');
     }
     catch (firstErr) {
         logger.warn({ error: sanitizeErrorText(firstErr instanceof Error ? firstErr.message : 'unknown') }, 'Snapshot failed — retrying');
         await page.waitFor({ timeMs: PAGE_READY_WAIT_MS });
         try {
-            snapshot = (await page.snapshot({ interactive: true, compact: true })).snapshot;
+            snapshot = (await page.snapshot({ interactive: true, compact: true, timeoutMs: SNAPSHOT_TIMEOUT_MS })).snapshot;
         }
         catch (err) {
             logger.error({ error: sanitizeErrorText(err instanceof Error ? err.message : 'unknown') }, 'Snapshot failed after retry');
@@ -246,7 +267,7 @@ async function safeSnapshot(page) {
         for (let i = 0; i < PAGE_READY_RETRIES; i++) {
             await page.waitFor({ timeMs: PAGE_READY_WAIT_MS });
             try {
-                snapshot = (await page.snapshot({ interactive: true, compact: true })).snapshot;
+                snapshot = (await page.snapshot({ interactive: true, compact: true, timeoutMs: SNAPSHOT_TIMEOUT_MS })).snapshot;
                 if (isPageReady(snapshot) === 'ready')
                     break;
             }
@@ -537,6 +558,7 @@ function parseActions(parsed) {
             options: item.options,
             direction: item.direction,
             expression: item.expression,
+            query: item.query,
             tab_id: item.tab_id,
         };
     });
@@ -571,11 +593,7 @@ export function assertExtractExpressionAllowed(expression) {
         throw new Error('extract: expression references disallowed API (cookies, storage, network, or eval)');
     }
 }
-async function executeAction(action, page, customActions) {
-    if (customActions?.[action.action] !== undefined) {
-        await customActions[action.action](action, page);
-        return;
-    }
+async function executeAction(action, page, config) {
     switch (action.action) {
         case 'click':
             if (action.ref === undefined || action.ref === '')
@@ -593,7 +611,9 @@ async function executeAction(action, page, customActions) {
             if (action.url === undefined || action.url === '')
                 throw new Error('navigate action requires url');
             assertNavigateUrlAllowed(action.url);
+            logger.info({ url: action.url }, 'navigate: starting');
             await page.goto(action.url);
+            logger.info({ url: action.url }, 'navigate: complete');
             break;
         case 'back':
             await page.evaluate('window.history.back()');
@@ -612,11 +632,11 @@ async function executeAction(action, page, customActions) {
             break;
         case 'scroll':
             await page.evaluate(action.direction === 'up'
-                ? `window.scrollBy(0, -${String(SCROLL_PIXELS)})`
-                : `window.scrollBy(0, ${String(SCROLL_PIXELS)})`);
+                ? `window.scrollBy(0, -${String(config.scrollPixels)})`
+                : `window.scrollBy(0, ${String(config.scrollPixels)})`);
             break;
         case 'wait':
-            await page.waitFor({ timeMs: WAIT_ACTION_MS });
+            await page.waitFor({ timeMs: config.waitActionMs });
             break;
         case 'done':
         case 'fail':
@@ -630,12 +650,12 @@ async function executeAction(action, page, customActions) {
             break;
     }
 }
-function getWaitMs(action) {
+function getWaitMs(action, config) {
     switch (action) {
         case 'type':
-            return WAIT_AFTER_TYPE_MS;
+            return config.waitAfterTypeMs;
         case 'click':
-            return WAIT_AFTER_CLICK_MS;
+            return config.waitAfterClickMs;
         case 'navigate':
         case 'back':
         case 'select':
@@ -651,7 +671,7 @@ function getWaitMs(action) {
         case 'done':
         case 'fail':
         case 'ask_user':
-            return WAIT_AFTER_OTHER_MS;
+            return config.waitAfterOtherMs;
     }
 }
 function describeActionError(action, err) {
@@ -808,8 +828,9 @@ Use ONLY data that appears in the memory or steps below — never invent values.
         return { ready: false, missing: 'judgment unavailable' };
     }
 }
-export async function runAgentLoop(prompt, pageOrHolder, emit, signal, options) {
-    const { waitForUser, browser, domainSkill, maxSteps = MAX_STEPS, userChat, customActions, systemPrompt: customSystemPrompt, buildTask, } = options ?? {};
+export async function runAgentLoop(prompt, pageOrHolder, emit, signal, waitForUser, browser, domainSkill, maxSteps, userChat, options) {
+    const cfg = defaultAgentConfig(options?.config);
+    maxSteps ??= cfg.maxSteps;
     // Accept either a bare CrawlPage or a PageHolder. When a PageHolder is
     // provided the caller's reference is updated on tab switches.
     const holder = 'page' in pageOrHolder && typeof pageOrHolder.page === 'object'
@@ -831,6 +852,9 @@ export async function runAgentLoop(prompt, pageOrHolder, emit, signal, options)
     let pendingSiteSwitch = null;
     let duplicateMemoryCount = 0;
     let consecutiveNotReadyJudgments = 0;
+    let lastJudgmentMemoryLength = 0;
+    const bannedRefs = new Map();
+    const BAN_THRESHOLD = 2;
     const forceComplete = async (reason, precomputedAnswer) => {
         logger.warn({ step, reason }, 'Force-completing run');
         const answer = precomputedAnswer ?? (await getFinalSummary(refinedPrompt, history));
@@ -856,27 +880,29 @@ export async function runAgentLoop(prompt, pageOrHolder, emit, signal, options)
             final_url: currentUrl,
         };
     };
-    const effectivePrompt = buildTask !== undefined ? buildTask(prompt) : prompt;
     // ── Load task lessons ──────────────────────────────────────────────────────
     let taskLesson = null;
-    try {
-        taskLesson = await getLesson(effectivePrompt);
-        if (taskLesson !== null) {
-            const blocked = taskLesson.domains.filter((d) => d.status === 'blocked').length;
-            const worked = taskLesson.domains.filter((d) => d.status === 'worked').length;
-            logger.info({ blocked, worked, hash: taskLesson.task_hash }, 'Loaded task lessons');
+    if (options?.getLesson !== undefined) {
+        try {
+            taskLesson = await options.getLesson(prompt);
+            if (taskLesson !== null) {
+                const blocked = taskLesson.domains.filter((d) => d.status === 'blocked').length;
+                const worked = taskLesson.domains.filter((d) => d.status === 'worked').length;
+                logger.info({ blocked, worked, hash: taskLesson.task_hash }, 'Loaded task lessons');
+            }
+        }
+        catch (err) {
+            logger.error({ err }, 'Failed to load task lessons');
         }
-    }
-    catch (err) {
-        logger.error({ err }, 'Failed to load task lessons');
     }
     // ── Planning + goal refinement (single LLM call) ──────────────────────────
     // If the prompt is vague (e.g. "find apartments in Chelsea"), the planner
     // also produces a SMART task with clear scope and stopping criteria.
-    let refinedPrompt = effectivePrompt;
+    let refinedPrompt = prompt;
     let planText = null;
     try {
-        let planMessage = `User prompt: ${effectivePrompt}`;
+        const taskInput = options?.buildTask !== undefined ? await options.buildTask(prompt) : prompt;
+        let planMessage = `User prompt: ${taskInput}`;
         if (domainSkill !== undefined && domainSkill !== null) {
             planMessage += `\n\nWe have a proven skill for this site: "${domainSkill.skill.title}" — ${domainSkill.skill.description}`;
             planMessage += '\nLeverage it — no need to rediscover what already works.';
@@ -884,7 +910,7 @@ export async function runAgentLoop(prompt, pageOrHolder, emit, signal, options)
         if (taskLesson !== null) {
             const lessonText = formatLessonForPrompt(taskLesson);
             if (lessonText !== '') {
-                planMessage += `\n\n${lessonText}\nDo NOT navigate to blocked domains. Use alternatives that worked before.`;
+                planMessage += `\n\n${lessonText}`;
             }
         }
         const plan = await llmJson({
@@ -1090,7 +1116,14 @@ Respond with JSON: {"plan": "your revised plan here"}`,
                     logger.info({ step }, 'Judge: answer ready — force-completing');
                     return await forceComplete('Judge ruled answer ready', judgment.answer);
                 }
-                consecutiveNotReadyJudgments++;
+                const memoryGrew = mem.length > lastJudgmentMemoryLength + 50;
+                lastJudgmentMemoryLength = mem.length;
+                if (memoryGrew) {
+                    consecutiveNotReadyJudgments = 0;
+                }
+                else {
+                    consecutiveNotReadyJudgments++;
+                }
                 if (consecutiveNotReadyJudgments >= JUDGE_FATIGUE_LIMIT) {
                     logger.warn({ step, nudges: consecutiveNotReadyJudgments, stillMissing: judgment.missing }, 'Judge fatigue — force-completing with partial answer');
                     return await forceComplete(`Fatigue: ${String(consecutiveNotReadyJudgments)} consecutive not-ready judgments; still missing "${judgment.missing}"`);
@@ -1165,13 +1198,18 @@ Respond with JSON: {"plan": "your revised plan here"}`,
         const userMessage = interjectionBlock !== null ? `${interjectionBlock}\n\n---\n\n${baseUserMessage}` : baseUserMessage;
         // On the last step, force the agent to produce a final answer
         const isLastStep = stepsRemaining === 0;
-        const systemPrompt = isLastStep ? LAST_STEP_PROMPT : (customSystemPrompt ?? SYSTEM_PROMPT);
+        const resolvedSystemPrompt = options?.systemPrompt === undefined
+            ? SYSTEM_PROMPT
+            : typeof options.systemPrompt === 'function'
+                ? options.systemPrompt(SYSTEM_PROMPT)
+                : options.systemPrompt;
+        const systemPrompt = isLastStep ? LAST_STEP_PROMPT : resolvedSystemPrompt;
         let actions;
         try {
             const parsed = await llmJson({
                 system: systemPrompt,
                 message: userMessage,
-                maxTokens: LLM_MAX_TOKENS,
+                maxTokens: cfg.llmMaxTokens,
             });
             try {
                 actions = parseActions(parsed);
@@ -1179,6 +1217,10 @@ Respond with JSON: {"plan": "your revised plan here"}`,
             catch (parseErr) {
                 throw new LlmParseError(parseErr instanceof Error ? parseErr.message : 'Invalid action structure', JSON.stringify(parsed).slice(0, 200));
             }
+            const banned = actions[0]?.ref !== undefined ? bannedRefs.get(actions[0].ref) : undefined;
+            if (banned !== undefined && banned.failures >= BAN_THRESHOLD && banned.action === actions[0].action) {
+                throw new LlmParseError(`Ref "${actions[0].ref ?? ''}" is blocked after ${String(banned.failures)} consecutive "${actions[0].action}" failures on it. Pick a different ref from the current snapshot, or try a different approach (URL parameters, keyboard, different element). Do NOT use "${actions[0].ref ?? ''}" again.`, JSON.stringify(parsed).slice(0, 200));
+            }
             consecutiveParseFailures = 0;
             crossSiteParseFailures = 0;
             consecutiveApiFailures = 0;
@@ -1235,7 +1277,12 @@ Respond with JSON: {"plan": "your revised plan here"}`,
                 // LLM responded but not with valid JSON — burn a step, the call was made
                 consecutiveParseFailures++;
                 crossSiteParseFailures++;
-                logger.warn({ step, consecutive: consecutiveParseFailures, crossSite: crossSiteParseFailures }, 'LLM returned non-JSON response');
+                logger.warn({
+                    step,
+                    consecutive: consecutiveParseFailures,
+                    crossSite: crossSiteParseFailures,
+                    rawSnippet: err.responseSnippet,
+                }, 'LLM returned non-JSON response');
                 emit('step_error', {
                     step,
                     error: 'LLM response was not valid JSON',
@@ -1268,7 +1315,12 @@ Respond with JSON: {"plan": "your revised plan here"}`,
             }
             // API/network error — don't burn a step, the agent never got to act
             consecutiveApiFailures++;
-            logger.error({ step, attempt: consecutiveApiFailures, maxAttempts: MAX_API_FAILURES }, 'LLM API error');
+            logger.error({
+                step,
+                attempt: consecutiveApiFailures,
+                maxAttempts: MAX_API_FAILURES,
+                errorMessage: err instanceof Error ? sanitizeErrorText(err.message).slice(0, 300) : 'unknown',
+            }, 'LLM API error');
             emit('step_error', { step, error: 'AI service temporarily unavailable', type: 'api_error' });
             if (consecutiveApiFailures >= MAX_API_FAILURES) {
                 return {
@@ -1370,7 +1422,6 @@ Respond with JSON: {"plan": "your revised plan here"}`,
                 step++;
                 break;
             }
-            // #1 Evaluate fallback: LLM-requested JS extraction
             if (action.action === 'extract') {
                 if (action.expression !== undefined && action.expression !== '') {
                     try {
@@ -1387,7 +1438,18 @@ Respond with JSON: {"plan": "your revised plan here"}`,
                     }
                 }
                 else {
-                    agentStep.action.extract_result = 'Error: no expression provided';
+                    const items = await extractItems(holder.page);
+                    const payload = JSON.stringify({
+                        source: items.source,
+                        count: items.count,
+                        truncated: items.truncated,
+                        records: items.records,
+                    });
+                    agentStep.action.extract_result = payload.slice(0, EXTRACT_RESULT_MAX_CHARS);
+                    if (payload.length > EXTRACT_RESULT_MAX_CHARS) {
+                        agentStep.action.extract_result += '\n…(truncated)';
+                    }
+                    logger.info({ step, source: items.source, count: items.count }, 'extract items auto');
                 }
                 logger.info({ step, result: agentStep.action.extract_result.slice(0, 100) }, 'Extract action');
                 step++;
@@ -1458,12 +1520,20 @@ Respond with JSON: {"plan": "your revised plan here"}`,
             // --- Normal actions ---
             const preActionUrl = await holder.page.url();
             try {
-                await executeAction(action, holder.page, customActions);
+                const customHandler = options?.customActions?.[action.action];
+                if (customHandler !== undefined) {
+                    const result = await customHandler(action, holder.page, browser);
+                    if (result?.outcome !== undefined)
+                        agentStep.outcome = result.outcome;
+                }
+                else {
+                    await executeAction(action, holder.page, cfg);
+                }
                 // After typing, detect autocomplete/combobox fields
                 if (action.action === 'type') {
                     await holder.page.waitFor({ timeMs: 400 });
                     try {
-                        const postTypeSnapshot = (await holder.page.snapshot({ interactive: true, compact: true })).snapshot;
+                        const postTypeSnapshot = (await holder.page.snapshot({ interactive: true, compact: true, timeoutMs: SNAPSHOT_TIMEOUT_MS })).snapshot;
                         if (/combobox|listbox|aria-autocomplete|suggestion|dropdown/i.test(postTypeSnapshot)) {
                             agentStep.action.error_feedback =
                                 'AUTOCOMPLETE DETECTED: A dropdown/suggestion list appeared after typing. Wait for it to fully load, then click the correct suggestion. Do NOT press Enter.';
@@ -1479,7 +1549,7 @@ Respond with JSON: {"plan": "your revised plan here"}`,
                 let postActionUrl = preActionUrl;
                 try {
                     postActionUrl = await holder.page.url();
-                    const postSnapshot = (await holder.page.snapshot({ interactive: true, compact: true })).snapshot;
+                    const postSnapshot = (await holder.page.snapshot({ interactive: true, compact: true, timeoutMs: SNAPSHOT_TIMEOUT_MS })).snapshot;
                     const outcome = validateAction(action, preActionUrl, postActionUrl, snapshot.length, postSnapshot.length);
                     if (outcome !== undefined) {
                         agentStep.outcome = outcome;
@@ -1505,6 +1575,12 @@ Respond with JSON: {"plan": "your revised plan here"}`,
                 emit('step_error', { step, action: action.action, error: rawMessage });
                 agentStep.action.error_feedback = feedback;
                 recentFailureCount++;
+                if (action.ref !== undefined && action.ref !== '' && /not found or not visible/i.test(rawMessage)) {
+                    const entry = bannedRefs.get(action.ref) ?? { action: action.action, failures: 0 };
+                    entry.failures += 1;
+                    entry.action = action.action;
+                    bannedRefs.set(action.ref, entry);
+                }
                 await holder.page.waitFor({ timeMs: 1000 });
                 if (await detectPopup(holder.page)) {
                     await dismissPopup(holder.page);
@@ -1540,7 +1616,7 @@ Respond with JSON: {"plan": "your revised plan here"}`,
                     break; // New tab opened — need new snapshot
                 }
             }
-            const waitMs = getWaitMs(action.action);
+            const waitMs = getWaitMs(action.action, cfg);
             await holder.page.waitFor({ timeMs: waitMs });
             step++;
         }