explorbot 0.4.9 → 0.5.0

package/dist/src/ai/judge-tool.js

@@ -0,0 +1,33 @@
+import { tool } from 'ai';
+import dedent from 'dedent';
+import { z } from 'zod';
+import { failedToolResult, successToolResult } from "./tools.js";
+export function createJudgeTool(deps, buildState) {
+    const judge = deps.judge;
+    if (!judge?.toolEnabled)
+        return {};
+    return {
+        judge: tool({
+            description: dedent `
+        Settle one judgement about the current page instead of guessing. Phrase it literally and concretely.
+      `,
+            inputSchema: z.object({
+                question: z.string().describe('The statement to confirm, or the question the options answer'),
+                options: z.array(z.string()).optional().describe('Possible answers. Omit to confirm a statement'),
+                context: z.string().optional().describe('Anything the page observation does not already carry'),
+            }),
+            execute: async ({ question, options, context }) => {
+                const state = await buildState();
+                if (context)
+                    state.context = context;
+                const decision = await judge.consult(question, options ?? null, state);
+                if (decision.rejected) {
+                    return failedToolResult('judge', `Not confirmed: ${question}`, {
+                        suggestion: 'The page does not settle this. Gather more context or take another route; do not assume either answer.',
+                    });
+                }
+                return successToolResult('judge', { question, answer: decision.value, confidence: decision.confidence });
+            },
+        }),
+    };
+}

package/dist/src/ai/judge.d.ts

@@ -0,0 +1,28 @@
+import type { AIConfig } from '../config.js';
+import { JudgeProvider } from './judge-provider.js';
+export declare const UNDECIDED = "undecided";
+export declare const JUDGE_PAGE_CAP = 12000;
+export declare class Decision {
+    readonly value: string | null;
+    readonly confidence: number;
+    constructor(value: string | null, confidence: number);
+    get approved(): boolean;
+    get rejected(): boolean;
+}
+export declare class Judge {
+    provider: JudgeProvider;
+    enabled: {
+        tool: boolean;
+        direct: boolean;
+    };
+    constructor(provider: JudgeProvider, enabled: {
+        tool: boolean;
+        direct: boolean;
+    });
+    static fromConfig(config: AIConfig['decisionModel']): Judge | null;
+    get toolEnabled(): boolean;
+    decide(question: string, options: string[] | boolean | null, state: unknown): Promise<Decision>;
+    consult(question: string, options: string[] | boolean | null, state: unknown): Promise<Decision>;
+    request(question: string, options: string[] | boolean | null, state: unknown): Promise<Decision>;
+    recordFailure(error: unknown): null;
+}

package/dist/src/ai/judge.js

@@ -0,0 +1,71 @@
+import { clearActivity, setActivity } from "../activity.js";
+import { Observability } from "../observability.js";
+import { createDebug } from "../utils/logger.js";
+import { JudgeProvider } from "./judge-provider.js";
+const debugLog = createDebug('explorbot:judge');
+const APPROVAL_THRESHOLD = 0.7;
+export const UNDECIDED = 'undecided';
+export const JUDGE_PAGE_CAP = 12000;
+export class Decision {
+    value;
+    confidence;
+    constructor(value, confidence) {
+        this.value = value;
+        this.confidence = confidence;
+    }
+    get approved() {
+        return this.value !== null;
+    }
+    get rejected() {
+        return this.value === null;
+    }
+}
+export class Judge {
+    provider;
+    enabled;
+    constructor(provider, enabled) {
+        this.provider = provider;
+        this.enabled = enabled;
+    }
+    static fromConfig(config) {
+        if (!config)
+            return null;
+        return new Judge(new JudgeProvider(config.provider, config.model), { tool: config.tool !== false, direct: config.direct !== false });
+    }
+    get toolEnabled() {
+        return this.enabled.tool;
+    }
+    async decide(question, options, state) {
+        if (!this.enabled.direct)
+            return new Decision(null, 0);
+        return this.consult(question, options, state);
+    }
+    async consult(question, options, state) {
+        if (Array.isArray(options) && options.length < 2)
+            return new Decision(null, 0);
+        return Observability.run('judge.decide', { tags: ['judge'] }, async () => {
+            setActivity('⚖️ Asking judge...', 'ai');
+            const decision = await this.request(question, options, state).finally(() => clearActivity());
+            Observability.getSpan()?.setAttribute('ai.telemetry.metadata.judgeDecision', JSON.stringify({ question, value: decision.value, confidence: decision.confidence }));
+            return decision;
+        });
+    }
+    async request(question, options, state) {
+        let list;
+        if (Array.isArray(options))
+            list = options;
+        const answer = await this.provider.decide(state, question, list).catch((error) => this.recordFailure(error));
+        if (!answer)
+            return new Decision(null, 0);
+        if (answer.probability <= APPROVAL_THRESHOLD)
+            return new Decision(null, answer.probability);
+        if (answer.value === UNDECIDED)
+            return new Decision(null, answer.probability);
+        return new Decision(answer.value, answer.probability);
+    }
+    recordFailure(error) {
+        debugLog('judge declined: %s', error);
+        Observability.getSpan()?.setAttribute('ai.telemetry.metadata.judgeError', String(error));
+        return null;
+    }
+}

package/dist/src/ai/navigator.d.ts

@@ -8,6 +8,7 @@ import type { KnowledgeTracker } from '../knowledge-tracker.js';
 import { type StateManager } from '../state-manager.js';
 import { HooksRunner } from '../utils/hooks-runner.js';
 import type { Agent, AgentDeps } from './agent.js';
+import { type Decision, type Judge } from './judge.js';
 import type { Provider } from './provider.js';
 declare class Navigator implements Agent {
     emoji: string;
@@ -23,10 +24,10 @@ declare class Navigator implements Agent {
     explorer: Explorer;
     config: ExplorbotConfig;
     stateManager: StateManager;
+    judge?: Judge;
     constructor(deps: AgentDeps);
     get verifyAttempts(): number;
     get verifyTimeout(): number;
-    getBaseOrigin(): string | null;
     getComparableCurrentUrl(stateManager: any, expectedUrl: string): string;
     comparableUrl(state: {
         url?: string;
@@ -83,6 +84,7 @@ declare class Navigator implements Agent {
             args: any[];
         }>;
         totalAttempted: number;
+        judged?: Decision;
     }>;
     checkAlreadyVerified(aiResponse: string, actionResult: ActionResult): boolean;
 }

package/dist/src/ai/navigator.js

@@ -12,7 +12,8 @@ import { createDebug, pluralize, tag } from '../utils/logger.js';
 import { loop, pause } from '../utils/loop.js';
 import { RulesLoader } from "../utils/rules-loader.js";
 import { normalizeInlineText } from "../utils/strings.js";
-import { extractStatePath, matchesNavigationUrl } from '../utils/url-matcher.js';
+import { extractStatePath, isSameHostFamily, matchesNavigationUrl } from '../utils/url-matcher.js';
+import { JUDGE_PAGE_CAP, UNDECIDED } from "./judge.js";
 import { Researcher } from "./researcher.js";
 import { actionRule, locatorRule, unexpectedPopupRule } from './rules.js';
 import { isInteractive } from './task-agent.js';
@@ -67,6 +68,7 @@ class Navigator {
     explorer;
     config;
     stateManager;
+    judge;
     constructor(deps) {
         this.provider = deps.ai;
         this.explorer = deps.explorer;
@@ -75,6 +77,7 @@ class Navigator {
         this.knowledgeTracker = deps.knowledgeTracker;
         this.experienceTracker = deps.stateManager.getExperienceTracker();
         this.hooksRunner = new HooksRunner(deps.explorer, deps.config);
+        this.judge = deps.judge;
     }
     get verifyAttempts() {
         return this.config.ai?.agents?.navigator?.verifyAttempts ?? 3;
@@ -82,15 +85,6 @@ class Navigator {
     get verifyTimeout() {
         return this.config.ai?.agents?.navigator?.verifyTimeout ?? 1500;
     }
-    getBaseOrigin() {
-        const baseUrl = this.config.playwright.url;
-        try {
-            return new URL(baseUrl).origin;
-        }
-        catch {
-            return null;
-        }
-    }
     getComparableCurrentUrl(stateManager, expectedUrl) {
         const currentState = stateManager.getCurrentState();
         if (!currentState)
@@ -109,19 +103,14 @@ class Navigator {
         const currentFullUrl = currentState.fullUrl || currentState.url || '';
         if (!currentFullUrl)
             return false;
-        try {
-            const currentOrigin = new URL(currentFullUrl).origin;
-            if (/^https?:\/\//i.test(expectedUrl)) {
-                return currentOrigin === new URL(expectedUrl).origin;
-            }
-            const baseOrigin = this.getBaseOrigin();
-            if (!baseOrigin)
-                return true;
-            return currentOrigin === baseOrigin;
-        }
-        catch {
+        if (!/^https?:\/\//i.test(currentFullUrl))
             return !/^https?:\/\//i.test(expectedUrl);
-        }
+        if (/^https?:\/\//i.test(expectedUrl))
+            return isSameHostFamily(currentFullUrl, expectedUrl);
+        const baseUrl = this.config.playwright.url;
+        if (!baseUrl)
+            return true;
+        return isSameHostFamily(currentFullUrl, baseUrl);
     }
     isOnExpectedPage(expectedUrl, stateManager) {
         if (!this.isSameExpectedOrigin(expectedUrl, stateManager)) {
@@ -289,8 +278,9 @@ class Navigator {
                         tag('warning').log(`Page state did not change at ${check.freshState.url}`);
                     }
                     else {
-                        lastFailure = `Reached ${check.freshState.url}, expected ${expectedUrl}`;
-                        tag('warning').log(`URL verification failed: expected ${expectedUrl}, got ${check.freshState.url}`);
+                        const reachedUrl = check.freshState.fullUrl || check.freshState.url;
+                        lastFailure = `Reached ${reachedUrl}, expected ${expectedUrl}`;
+                        tag('warning').log(`URL verification failed: expected ${expectedUrl}, got ${reachedUrl}`);
                     }
                     batchFailures.push({
                         code: codeBlock,
@@ -530,8 +520,7 @@ class Navigator {
             if (!value)
                 return;
             const normalized = normalizeUrl(value);
-            if (normalized)
-                visitCounts.set(normalized, (visitCounts.get(normalized) || 0) + 1);
+            visitCounts.set(normalized, (visitCounts.get(normalized) || 0) + 1);
         };
         for (const transition of history) {
             countVisit(transition.toState.url);
@@ -540,7 +529,7 @@ class Navigator {
         if (opts?.visitedUrls) {
             for (const url of opts.visitedUrls) {
                 const normalized = normalizeUrl(url);
-                if (normalized && !visitCounts.has(normalized)) {
+                if (!visitCounts.has(normalized)) {
                     visitCounts.set(normalized, 1);
                 }
             }
@@ -632,6 +621,12 @@ class Navigator {
             tag('operation').log(`Reusing cached verification: ${cachedVerification ? 'PASS' : 'FAIL'}`);
             return { verified: cachedVerification, inexpressible: false, results: [], successfulCodes: [], assertionSteps: [], totalAttempted: 0 };
         }
+        const verifiedClaims = Object.keys(actionResult.verifications ?? {}).filter((claim) => actionResult.getVerification(claim) === true);
+        const same = await this.judge?.decide('Which already verified claim means the same as the claim under consideration?', [...verifiedClaims, UNDECIDED], { claim: message });
+        if (same?.approved) {
+            tag('operation').log(`Judge matched claim to an already verified one: "${same.value}"`);
+            return { verified: true, inexpressible: false, results: [], successfulCodes: [], assertionSteps: [], totalAttempted: 0 };
+        }
         const knowledge = this.knowledgeTracker.renderRelevantContext(actionResult);
         let experience = '';
         if (!actionResult.isInsideIframe) {
@@ -754,7 +749,8 @@ class Navigator {
         const inexpressible = !alreadyVerified && totalAttempted === 0;
         if (inexpressible) {
             tag('warning').log('No assertion could express this claim');
-            return { verified: false, inexpressible, results, successfulCodes, assertionSteps, totalAttempted };
+            const judged = await this.judge?.decide('The page shows that this claim is true.', null, { claim: message, page: actionResult.getCompactARIA().slice(0, JUDGE_PAGE_CAP) });
+            return { verified: false, inexpressible, results, successfulCodes, assertionSteps, totalAttempted, judged };
         }
         actionResult.addVerification(message, verified);
         this.stateManager.updateState(actionResult);

package/dist/src/ai/pilot.d.ts

@@ -7,6 +7,7 @@ import { type Test } from '../test-plan.js';
 import type { Agent, AgentDeps } from './agent.js';
 import type { Conversation } from './conversation.js';
 import type { Fisherman } from './fisherman.js';
+import { type Judge } from './judge.js';
 import type { Navigator } from './navigator.js';
 import type { Provider } from './provider.js';
 import type { Researcher } from './researcher.js';
@@ -21,6 +22,7 @@ export declare class Pilot implements Agent {
     requestStore: RequestStore;
     playwrightRecorder: PlaywrightRecorder;
     fisherman: Fisherman | null;
+    judge?: Judge;
     constructor(deps: AgentDeps, agentTools: any, researcher: Researcher);
     setFisherman(fisherman: Fisherman): void;
     get stepsToReview(): number;
@@ -41,12 +43,14 @@ export declare class Pilot implements Agent {
     reviewNewPage(task: Test, currentState: ActionResult, testerConversation: Conversation): Promise<string>;
     analyzeProgress(task: Test, currentState: ActionResult, testerConversation: Conversation): Promise<string>;
     settleExpectations(task: Test, finalState?: ActionResult): Promise<SettledExpectation[]>;
+    settleByJudge(task: Test, expectations: string[]): Promise<Map<string, SettledStatus>>;
     formatExpectations(task: Test): string;
     sendToPilot(userText: string, functionId: string, opts: {
         tools?: boolean;
         maxToolRoundtrips?: number;
         task: Test;
     }): Promise<string>;
+    announcePreparedData(text: string, task: Test, preparedCount: number): string;
     getExperienceToc(): string;
     pickPlanningTools(): Record<string, unknown>;
     fishermanStatus(): string;

package/dist/src/ai/pilot.js

@@ -4,13 +4,14 @@ import { z } from 'zod';
 import { ActionResult } from "../action-result.js";
 import { ConfigParser } from "../config.js";
 import { Stats } from "../stats.js";
-import { TestResult } from "../test-plan.js";
+import { TestResult, TestStatus } from "../test-plan.js";
 import { collectInteractiveNodes } from "../utils/aria.js";
 import { ErrorPageError } from "../utils/error-page.js";
 import { createDebug, tag } from "../utils/logger.js";
 const debugLog = createDebug('explorbot:pilot');
 import { truncateJson } from "../utils/strings.js";
 import { createAskApiTool } from "./fisherman/tools.js";
+import { UNDECIDED } from "./judge.js";
 import { capabilityGroundingRule, dataProtectionRules } from "./rules.js";
 import { isInteractive } from "./task-agent.js";
 import { withdrawVisionTools } from "./tools.js";
@@ -21,6 +22,10 @@ const PILOT_REASONING_LIMIT = 500;
 const PILOT_MESSAGE_LIMIT = 2;
 const PILOT_MESSAGE_MAX_LENGTH = 160;
 const PILOT_REQUEST_LIMIT = 5;
+const OUTCOME_STATUS = {
+    'The run shows this outcome happened.': 'passed',
+    'The run shows this outcome did not happen.': 'failed',
+};
 export class Pilot {
     emoji = '🧭';
     provider;
@@ -32,6 +37,7 @@ export class Pilot {
     requestStore;
     playwrightRecorder;
     fisherman = null;
+    judge;
     constructor(deps, agentTools, researcher) {
         this.provider = deps.ai;
         this.agentTools = agentTools;
@@ -40,6 +46,7 @@ export class Pilot {
         this.stateManager = deps.stateManager;
         this.requestStore = deps.requestStore;
         this.playwrightRecorder = deps.playwrightRecorder;
+        this.judge = deps.judge;
     }
     setFisherman(fisherman) {
         this.fisherman = fisherman;
@@ -95,7 +102,9 @@ export class Pilot {
             }
         }
         const schema = z.object({
-            decision: z.enum(['pass', 'fail', 'continue', 'skipped']).describe('pass = test succeeded, fail = test failed, continue = tester should keep going, skipped = scenario is irrelevant OR systematic execution failures prevented testing'),
+            decision: z
+                .enum(['pass', 'fail', 'continue', 'skipped'])
+                .describe('pass = scenario goal accomplished, fail = the app misbehaved, continue = tester should keep going, skipped = the scenario cannot be judged against this app (its premise does not hold, it is irrelevant, or systematic execution failures prevented testing)'),
             reason: z.string().describe('Concise user-facing reason, maximum 1 short sentence and 120 characters. Do NOT repeat the decision status; explain only the evidence. For continue: explain why rejected and suggest alternatives.'),
             guidance: z.string().nullable().describe('Required for "continue": specific actionable instruction for the tester — what exactly to verify, retry differently, or complete next. Be concrete.'),
             requestVerification: z
@@ -358,9 +367,13 @@ export class Pilot {
         DOM assertion can't be made.
         Do not pass when Tester achieved only a related navigation/filter/tab/status outcome instead of the
         requested action, workflow, or entity detail goal.
-      - "fail": goal not achieved and no further step toward it is available on the current page.
-      - "skipped": scenario is irrelevant to the app, OR systematic infrastructure failures (LLM errors,
-        crashes) prevented testing. NOT for "test failed to interact" — that's "fail" or "continue".
+      - "fail": the app misbehaved — the scenario's action ran against the right target and the app
+        produced a wrong, broken, or missing outcome. Not reaching the goal is not by itself a fail.
+      - "skipped": the scenario cannot be judged against this app — the page shows its premise does not
+        hold (the assumed constraint, field, or behaviour is designed differently), the target entity or
+        feature is not the one here, the scenario is irrelevant, OR systematic infrastructure failures
+        (LLM errors, crashes) prevented testing. NOT for "test failed to interact" — that's "fail" or
+        "continue".
       - "continue": goal incomplete but the control for the NEXT step is present on the current page, or a
         concrete missing check would change your verdict. Guidance must name that step.
         If a verify() asserted a state that was ALREADY TRUE before the test, it proves nothing — reject.
@@ -474,6 +487,9 @@ export class Pilot {
         const toolCalls = testerConversation.getToolExecutions().slice(-this.stepsToReview);
         const actionsContext = this.formatActions(toolCalls);
         const stateContext = this.buildStateContext(currentState);
+        const healthy = await this.judge?.decide('The run is moving toward the goal and can continue without a supervisor reviewing it now.', null, { scenario: task.scenario, state: stateContext, recentActions: actionsContext });
+        if (healthy?.approved)
+            return '';
         const hasFailures = toolCalls.length === 0 || toolCalls.some((t) => !t.wasSuccessful);
         const text = await this.sendToPilot(dedent `
         START URL: ${task.startUrl}
@@ -510,8 +526,12 @@ export class Pilot {
         let undecided = task.expected.filter((text) => !task.getCheckedExpectations().includes(text));
         if (image)
             undecided = task.expected;
+        let settledByJudge = new Map();
+        if (!image)
+            settledByJudge = await this.settleByJudge(task, undecided);
+        undecided = undecided.filter((text) => !settledByJudge.has(text));
         if (!undecided.length)
-            return task.expected.map((text) => ({ text, status: decided(text) }));
+            return task.expected.map((text) => ({ text, status: settledByJudge.get(text) || decided(text) }));
         const schema = z.object({
             outcomes: z.array(z.object({
                 expectation: z.string().describe('The expected outcome, repeated exactly as it was given'),
@@ -575,6 +595,9 @@ export class Pilot {
             response = await settle(userContent, this.provider.getAgenticModel('pilot'));
         const judged = new Map((response?.object?.outcomes || []).map((outcome) => [outcome.expectation, outcome]));
         return task.expected.map((text) => {
+            const byJudge = settledByJudge.get(text);
+            if (byJudge)
+                return { text, status: byJudge };
             if (!undecided.includes(text))
                 return { text, status: decided(text) };
             const outcome = judged.get(text);
@@ -583,6 +606,20 @@ export class Pilot {
             return { text, status: outcome.status || 'unverified', evidence: outcome.evidence };
         });
     }
+    async settleByJudge(task, expectations) {
+        const settled = new Map();
+        const judge = this.judge;
+        if (!judge)
+            return settled;
+        const state = { scenario: task.scenario, runLog: task.notesToString() || 'No steps recorded.' };
+        await Promise.all(expectations.map(async (text) => {
+            const decision = await judge.decide(`What did this run establish about the expected outcome: ${text}`, [...Object.keys(OUTCOME_STATUS), UNDECIDED], state);
+            const status = OUTCOME_STATUS[decision.value ?? ''];
+            if (status)
+                settled.set(text, status);
+        }));
+        return settled;
+    }
     formatExpectations(task) {
         const checked = task.getCheckedExpectations();
         const remaining = task.getRemainingExpectations();
@@ -599,6 +636,7 @@ export class Pilot {
         }
         this.conversation.addUserText(finalUserText);
         const tools = { ...this.pickPlanningTools(), ...this.buildFishermanTools(opts.task) };
+        const preparedCount = opts.task.preparedData.length;
         const result = await this.provider.invokeConversation(this.conversation, tools, {
             maxToolRoundtrips: opts.maxToolRoundtrips ?? 0,
             toolChoice: opts.tools ? 'auto' : 'none',
@@ -606,7 +644,7 @@ export class Pilot {
             stopWhen: () => opts.task.hasFinished,
             telemetry: { functionId },
         });
-        const text = result?.response?.text || '';
+        const text = this.announcePreparedData(result?.response?.text || '', opts.task, preparedCount);
         const learned = (result?.toolExecutions || []).filter((e) => e.toolName === 'learnExperience' && e.output?.content).map((e) => ({ url: e.output.url, content: e.output.content }));
         if (learned.length === 0)
             return text;
@@ -622,6 +660,23 @@ export class Pilot {
       </applied_experience>
     `;
     }
+    announcePreparedData(text, task, preparedCount) {
+        const prepared = task.preparedData.slice(preparedCount);
+        if (prepared.length === 0)
+            return text;
+        let refresh = '';
+        if (task.status === TestStatus.IN_PROGRESS)
+            refresh = 'It was created after the page loaded, so the page does not show it yet. Run I.refreshPage() through form() before looking for it.';
+        return dedent `
+      ${text}
+
+      <prepared_data>
+      Pilot created this data through the API for this test. Use it instead of creating the same data through the UI:
+      ${prepared.map((item) => `- ${item}`).join('\n')}
+      ${refresh}
+      </prepared_data>
+    `;
+    }
     getExperienceToc() {
         const state = this.stateManager.getCurrentState();
         if (!state)
@@ -629,7 +684,7 @@ export class Pilot {
         return this.stateManager.getExperienceTracker().renderExperienceTocFor(ActionResult.fromState(state));
     }
     pickPlanningTools() {
-        const { see, context, verify, research, getVisitedStates, xpathCheck, learnExperience, askUser } = this.agentTools ?? {};
+        const { see, context, verify, research, getVisitedStates, xpathCheck, learnExperience, askUser, judge } = this.agentTools ?? {};
         const planning = {};
         if (see)
             planning.see = see;
@@ -647,6 +702,8 @@ export class Pilot {
             planning.learnExperience = learnExperience;
         if (askUser)
             planning.askUser = askUser;
+        if (judge)
+            planning.judge = judge;
         withdrawVisionTools(planning);
         return planning;
     }
@@ -694,6 +751,7 @@ export class Pilot {
                     });
                     const stepText = `Precondition: created ${items.join(', ')}`;
                     task.addStep(stepText);
+                    task.preparedData.push(...items);
                     tag('success').log(stepText);
                     return { noted: true, prepared: true, created: result.created };
                 },
@@ -1047,7 +1105,7 @@ export class Pilot {
       Tester tools: click, pressKey, form, see, verify, interact, context, research, xpathCheck,
       visualClick, back, getVisitedStates, reset, stop, finish, record.
       Use tool names exactly as listed. Do not invent combined names or aliases.
-      Reloading is not a tool: to re-read a page from the server, instruct Tester to run I.reloadPage() through form.
+      Reloading is not a tool: to re-read a page from the server, instruct Tester to run I.refreshPage() through form.
 
       ${capabilityGroundingRule}
 

package/dist/src/ai/planner.js

@@ -263,13 +263,14 @@ export class Planner extends PlannerBase {
                 .replaceEach((section) => {
                 const heading = section.query('heading').text().trim();
                 const withoutHeadings = mdq(section.text()).query('heading').replace('');
-                const body = mdq(withoutHeadings).query('hr').replace('').trim();
+                const body = mdq(withoutHeadings).query('hr').replace('').toString().trim();
                 if (body && !seenTitles.has(heading)) {
                     seenTitles.add(heading);
                     return section.text();
                 }
                 return '';
-            });
+            })
+                .toString();
         }
         const trimmedTitles = new Set();
         for (const selector of ['section2', 'section3']) {
@@ -282,10 +283,11 @@ export class Planner extends PlannerBase {
                 const count = section.query('blockquote').count();
                 if (count <= 10)
                     return section.text();
-                const kept = mdq(section.text()).query('blockquote[10:]').replace('');
+                const kept = mdq(section.text()).query('blockquote[10:]').replace('').toString();
                 trimmedTitles.add(heading);
                 return `${kept.trimEnd()}\n> ... and ${count - 10} more discoveries\n`;
-            });
+            })
+                .toString();
         }
         return result.trim() || null;
     }
@@ -360,7 +362,7 @@ export class Planner extends PlannerBase {
         if (this.scout && this.docsWeight > 0) {
             docsPromise = this.scout.collectDocs({ url: state.url, title: state.title, feature, excludeUrls: this.knowledgeTracker.applicationSpecUrls(state) });
         }
-        let plannerResearch = mdq(research).query('code').replace('');
+        let plannerResearch = mdq(research).query('code').replace('').toString();
         plannerResearch = mdq(plannerResearch)
             .query('table')
             .replaceEach((table) => {
@@ -372,7 +374,8 @@ export class Planner extends PlannerBase {
                 Type: r.Type || '',
             }));
             return jsonToTable(elementWithType, ['Element', 'Type']);
-        });
+        })
+            .toString();
         const hasFocusedOverlay = hasFocusedSection(plannerResearch);
         const focusNote = hasFocusedOverlay ? "IMPORTANT: One section is marked as **Focused** — this is the user's current focus area. Concentrate testing on the Focused section FIRST — test all interactions inside it before planning tests for the rest of the page." : '';
         const featureFilter = feature ? `FOCUS FILTER: Only propose scenarios using elements relevant to "${feature}". Ignore all other elements.` : '';

package/dist/src/ai/provider.d.ts

@@ -32,8 +32,11 @@ export declare class Provider {
     finalizeConfig(config: Record<string, any>, options: any, telemetry: any): void;
     buildGenerateConfig(defaults: Record<string, any>, overrides: Record<string, any>, options: any): Record<string, any>;
     recordUsage(agentName: string, modelName: string, usage: any): void;
-    raceWithIdleTimeout<T>(fn: (signal: AbortSignal) => Promise<T>, timeoutMs: number): Promise<T>;
+    raceWithIdleTimeout<T>(fn: (signal: AbortSignal) => Promise<T>, timeoutMs: number, busy?: {
+        tools: number;
+    }): Promise<T>;
     recoverFromContextLength(error: any, messages: ModelMessage[], options: any, retry: (messages: ModelMessage[], options: any) => Promise<any>): Promise<any>;
+    recoverWithPlainJson(messages: ModelMessage[], schema: any, model: any, options: any): Promise<any>;
     initLangfuse(): void;
     getTelemetry(options: any): any;
     startConversation(systemMessage: string, agentName?: string, model?: any): Conversation;