explorbot 0.2.4 → 0.3.0

package/dist/src/action.js

@@ -17,6 +17,8 @@ const debugLog = createDebug('explorbot:action');
 const CAPTURE_NAVIGATION_TRANSITION_ATTEMPTS = 3;
 const DEFAULT_ACTION_TIMEOUT = 3000;
 const DEFAULT_PAGE_TIMEOUT = 3000;
+const MAX_NETWORK_CALLS = 10;
+const IMPORTANT_LOG_LEVELS = new Set(['info', 'error', 'warning', 'warn']);
 class Action {
     actor;
     stateManager;
@@ -32,6 +34,8 @@ class Action {
     recorder;
     recovery;
     mainDocumentStatus = undefined;
+    networkRequests = [];
+    baseOrigin;
     constructor(actor, stateManager, recorder, recovery) {
         this.actor = actor;
         this.stateManager = stateManager;
@@ -39,6 +43,7 @@ class Action {
         this.playwrightHelper = container.helpers('Playwright');
         this.recorder = recorder;
         this.recovery = recovery || ((fn) => fn());
+        this.baseOrigin = URL.parse(this.config.playwright?.url || '')?.origin || '';
     }
     async saveScreenshot() {
         const currentState = this.stateManager.getCurrentState();
@@ -50,6 +55,7 @@ class Action {
             await this.actor.saveScreenshot(filename);
             if (currentState)
                 currentState.screenshotFile = filename;
+            tag('data').log('screenshot', { path: outputPath('states', filename) });
             return filename;
         }
         catch (err) {
@@ -97,7 +103,10 @@ class Action {
                 const screenshotPath = join(statesDir, filename);
                 screenshotFile = await page
                     ?.screenshot({ path: screenshotPath, fullPage: true })
-                    .then(() => filename)
+                    .then(() => {
+                    tag('data').log('screenshot', { path: screenshotPath });
+                    return filename;
+                })
                     .catch((err) => {
                     debugLog('Screenshot failed, continuing without it:', err);
                     return undefined;
@@ -113,9 +122,7 @@ class Action {
             const logPath = join(statesDir, logFile);
             const formattedLogs = browserLogs.map((log) => {
                 const logTimestamp = new Date().toISOString();
-                const level = (log.type || log.level || 'LOG').toUpperCase();
-                const message = log.text || log.message || String(log);
-                return `[${logTimestamp}] ${level}: ${message}`;
+                return `[${logTimestamp}] ${log.type.toUpperCase()}: ${log.text}`;
             });
             fs.writeFileSync(logPath, `${formattedLogs.join('\n')}\n`, 'utf8');
             debugLog('Page:', { url, title, size: html.length, html: html.substring(0, 100) });
@@ -138,12 +145,15 @@ class Action {
                 fs.writeFileSync(ariaPath, ariaSnapshot, 'utf8');
                 ariaSnapshotFile = ariaFileName;
             }
+            const networkRequests = this.networkRequests;
+            this.networkRequests = [];
             const result = new ActionResult({
                 html,
                 title,
                 httpStatus: await this.captureMainDocumentStatus(),
                 url,
                 browserLogs,
+                networkRequests,
                 htmlFile,
                 logFile,
                 screenshotFile,
@@ -188,27 +198,52 @@ class Action {
             return undefined;
         }
     }
-    captureMainDocumentResponse() {
+    captureResponses() {
         const page = this.playwrightHelper.page;
         if (!page?.on || !page?.off)
             return () => { };
         this.mainDocumentStatus = undefined;
+        this.networkRequests = [];
         const handler = (response) => {
             const request = response.request();
-            if (request.resourceType() !== 'document')
-                return;
-            if (response.frame() !== page.mainFrame())
-                return;
             const status = response.status();
             if (typeof status !== 'number')
                 return;
             if (status <= 0)
                 return;
-            this.mainDocumentStatus = status;
+            if (request.resourceType() === 'document') {
+                if (response.frame() !== page.mainFrame())
+                    return;
+                this.mainDocumentStatus = status;
+                return;
+            }
+            this.recordNetworkCall(request, status);
         };
         page.on('response', handler);
         return () => page.off('response', handler);
     }
+    recordNetworkCall(request, status) {
+        const resourceType = request.resourceType();
+        if (resourceType !== 'xhr' && resourceType !== 'fetch')
+            return;
+        const url = URL.parse(request.url());
+        if (!url)
+            return;
+        if (url.origin !== this.baseOrigin)
+            return;
+        const call = { method: request.method(), path: url.pathname, status };
+        if (this.networkRequests.some((r) => r.method === call.method && r.path === call.path && r.status === call.status))
+            return;
+        if (this.networkRequests.length >= MAX_NETWORK_CALLS) {
+            if (status < 400)
+                return;
+            const succeeded = this.networkRequests.findIndex((r) => r.status < 400);
+            if (succeeded === -1)
+                return;
+            this.networkRequests.splice(succeeded, 1);
+        }
+        this.networkRequests.push(call);
+    }
     /**
      * Capture HTML snapshots of all iframes on the page
      */
@@ -245,12 +280,7 @@ class Action {
     async captureBrowserLogs() {
         try {
             const logs = await this.actor.grabBrowserLogs();
-            // Filter for important logs (info, error, warning)
-            const importantLogs = logs.filter((log) => {
-                const level = log.type || log.level;
-                return ['info', 'error', 'warning', 'warn'].includes(level);
-            });
-            return importantLogs;
+            return logs.map(toBrowserLog).filter((log) => IMPORTANT_LOG_LEVELS.has(log.type));
         }
         catch (error) {
             debugLog('Failed to capture browser logs:', error);
@@ -266,7 +296,7 @@ class Action {
         const stepListener = attachStepLogger(executedSteps, assertionSteps);
         const groupId = this.recorder ? await this.recorder.beginAction(codeString) : null;
         this.playwrightGroupId = groupId;
-        const detachMainDocumentResponse = this.captureMainDocumentResponse();
+        const detachResponses = this.captureResponses();
         const activeSpan = Observability.getSpan();
         const tracer = trace.getTracer('ai');
         const stepSpan = activeSpan ? tracer.startSpan('codeceptjs.step', undefined, trace.setSpan(context.active(), activeSpan)) : null;
@@ -310,7 +340,7 @@ class Action {
         }
         finally {
             this.restorePageTimeout();
-            detachMainDocumentResponse();
+            detachResponses();
             if (groupId)
                 await this.recorder.endAction();
             detachStepLogger(stepListener);
@@ -389,6 +419,11 @@ async function captureHtml(page, frame, actor) {
         return actor.grabSource();
     throw new Error('Playwright page is unavailable for HTML capture');
 }
+function toBrowserLog(log) {
+    const type = typeof log.type === 'function' ? log.type() : log.type || log.level || 'log';
+    const text = typeof log.text === 'function' ? log.text() : log.text || log.message || String(log);
+    return { type, text: text.replace(/\s+/g, ' ').trim() };
+}
 async function captureTitle(page, actor) {
     if (page?.title)
         return page.title();

package/dist/src/ai/captain/web-mode.js

@@ -15,7 +15,7 @@ export function WithWebMode(Base) {
                 researcher: ctx.explorBot.agentResearcher(),
                 navigator: ctx.explorBot.agentNavigator(),
             });
-            const { see, context, visualClick, learnExperience } = agentTools;
+            const { see, context, visualClick } = agentTools;
             const tools = {
                 navigate: tool({
                     description: 'Navigate to a URL or page description using AI-powered navigation.',
@@ -115,7 +115,6 @@ export function WithWebMode(Base) {
                 }),
                 ...codeceptTools,
                 context,
-                learnExperience,
             };
             if (see)
                 tools.see = see;

package/dist/src/ai/captain.d.ts

@@ -59,6 +59,16 @@ export declare class Captain extends CaptainBase implements Agent {
     planSummary(): string;
     reinjectContextIfNeeded(conversation: Conversation, currentState: WebPageState): Promise<void>;
     coreTools(task: Task, onDone: (summary: string) => void): {
+        learnExperience: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
+            fileTag: any;
+            sectionIndex: any;
+        }, {
+            title: string;
+            url: string;
+            content: string;
+        } | {
+            error: string;
+        }, import("@ai-sdk/provider-utils").Context>>;
         done: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
             summary: string;
             details?: string;
@@ -91,6 +101,16 @@ export declare class Captain extends CaptainBase implements Agent {
         }, import("@ai-sdk/provider-utils").Context>>;
     };
     tools(task: Task, onDone: (summary: string) => void): Promise<{
+        learnExperience: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
+            fileTag: any;
+            sectionIndex: any;
+        }, {
+            title: string;
+            url: string;
+            content: string;
+        } | {
+            error: string;
+        }, import("@ai-sdk/provider-utils").Context>>;
         done: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
             summary: string;
             details?: string;

package/dist/src/ai/captain.js

@@ -15,7 +15,7 @@ import { WithWebMode } from "./captain/web-mode.js";
 import { toolExecutionLabel } from './conversation.js';
 import { Researcher } from "./researcher.js";
 import { TaskAgent } from "./task-agent.js";
-import { withdrawVisionTools } from "./tools.js";
+import { createLearnExperienceTool, withdrawVisionTools } from "./tools.js";
 const MAX_STEPS = 15;
 const CaptainBase = WithTestMode(WithWebMode(WithIdleMode(TaskAgent)));
 export class Captain extends CaptainBase {
@@ -216,6 +216,15 @@ export class Captain extends CaptainBase {
     }
     coreTools(task, onDone) {
         return {
+            learnExperience: createLearnExperienceTool({
+                getExperienceTracker: () => this.getExperienceTracker(),
+                getState: () => {
+                    const state = this.explorBot.stateManager().getCurrentState();
+                    if (!state)
+                        return null;
+                    return ActionResult.fromState(state);
+                },
+            }),
             done: tool({
                 description: 'Call when the user request is fulfilled.',
                 inputSchema: z.object({

package/dist/src/ai/driller.js

@@ -14,7 +14,7 @@ import { eidxInContainer } from "../utils/web-eidx.js";
 import { WebElement } from "../utils/web-element.js";
 import { drillLocatorRule } from "./rules.js";
 import { TaskAgent, isInteractive } from "./task-agent.js";
-import { createCodeceptJSTools } from "./tools.js";
+import { createCodeceptJSTools, createLearnExperienceTool } from "./tools.js";
 const debugLog = createDebug('explorbot:driller');
 export class Driller extends TaskAgent {
     ACTION_TOOLS = ['click', 'pressKey', 'form'];
@@ -234,7 +234,11 @@ export class Driller extends TaskAgent {
         conversation.addUserText(await this.buildComponentPrompt(originalState, component));
         let finished = false;
         const actionTools = this.createVerifiedActionTools(createCodeceptJSTools(this.toolDeps, test), component);
-        const tools = { ...actionTools, ...this.createDrillFlowTools(originalState, test, interactive) };
+        const learnExperience = createLearnExperienceTool({
+            getExperienceTracker: () => this.getExperienceTracker(),
+            getState: () => ActionResult.fromState(this.stateManager.getCurrentState() || originalState),
+        });
+        const tools = { ...actionTools, learnExperience, ...this.createDrillFlowTools(originalState, test, interactive) };
         await loop(async ({ stop, iteration }) => {
             debugLog(`Drilling component ${component.name}, iteration ${iteration}`);
             setActivity(`${this.emoji} Drilling ${component.name}...`, 'action');

package/dist/src/ai/fisherman-tools.d.ts

@@ -9,18 +9,49 @@ export declare function createFishermanTools(apiClient: ApiClient, requestStore:
             method: any;
             path: any;
         }, {
+            source: string;
+            method: any;
+            path: any;
+            definition: string;
+            rejectedCapture: {
+                status: number;
+                requestBody: any;
+            };
+            usable?: undefined;
+            rejectedRequestBody?: undefined;
+            status?: undefined;
+            requestBody?: undefined;
+            error?: undefined;
+        } | {
+            source: string;
+            method: any;
+            path: any;
+            usable: boolean;
+            rejectedRequestBody: any;
+            status: number;
+            definition?: undefined;
+            rejectedCapture?: undefined;
+            requestBody?: undefined;
+            error?: undefined;
+        } | {
             source: string;
             method: string;
             path: string;
             status: number;
             requestBody: any;
             definition?: undefined;
+            rejectedCapture?: undefined;
+            usable?: undefined;
+            rejectedRequestBody?: undefined;
             error?: undefined;
         } | {
             source: string;
             definition: string;
             method?: undefined;
             path?: undefined;
+            rejectedCapture?: undefined;
+            usable?: undefined;
+            rejectedRequestBody?: undefined;
             status?: undefined;
             requestBody?: undefined;
             error?: undefined;
@@ -29,20 +60,25 @@ export declare function createFishermanTools(apiClient: ApiClient, requestStore:
             error: any;
             method?: undefined;
             path?: undefined;
+            definition?: undefined;
+            rejectedCapture?: undefined;
+            usable?: undefined;
+            rejectedRequestBody?: undefined;
             status?: undefined;
             requestBody?: undefined;
-            definition?: undefined;
         }, import("@ai-sdk/provider-utils").Context>>;
         request: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<any, {
             success: boolean;
             error: string;
             status?: undefined;
             statusText?: undefined;
+            category?: undefined;
             errorPreview?: undefined;
         } | {
             success: boolean;
             status: number;
             statusText: string;
+            category: ResponseCategory;
             errorPreview: string;
             error?: undefined;
         } | {
@@ -50,6 +86,7 @@ export declare function createFishermanTools(apiClient: ApiClient, requestStore:
             status: number;
             error?: undefined;
             statusText?: undefined;
+            category?: undefined;
             errorPreview?: undefined;
         }, import("@ai-sdk/provider-utils").Context>>;
         finish: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
@@ -88,3 +125,5 @@ export interface FishermanResult {
         reason: string;
     }>;
 }
+type ResponseCategory = 'validation' | 'authorization' | 'not_found' | 'conflict' | 'temporary' | 'server' | 'client';
+export {};

package/dist/src/ai/fisherman-tools.js

@@ -23,6 +23,29 @@ export function createFishermanTools(apiClient, requestStore, opts) {
                 tag('step').log(`Fisherman: spec lookup ${method} ${path}`);
                 const captured = requestStore.findCapturedRequest(method, path);
                 if (captured) {
+                    if (captured.status >= 400) {
+                        const rejectedCapture = {
+                            status: captured.status,
+                            requestBody: captured.requestBody || 'no body',
+                        };
+                        if (opts.spec) {
+                            try {
+                                const definition = extractEndpointDefinition(opts.spec, path, opts.baseEndpoint);
+                                return { source: 'spec', method, path, definition, rejectedCapture };
+                            }
+                            catch {
+                                return { source: 'captured', method, path, usable: false, rejectedRequestBody: captured.requestBody || 'no body', status: captured.status };
+                            }
+                        }
+                        return {
+                            source: 'captured',
+                            method: captured.method,
+                            path: captured.path,
+                            status: captured.status,
+                            usable: false,
+                            rejectedRequestBody: captured.requestBody || 'no body',
+                        };
+                    }
                     return {
                         source: 'captured',
                         method: captured.method,
@@ -74,6 +97,7 @@ export function createFishermanTools(apiClient, requestStore, opts) {
                         success: false,
                         status: reqResult.status,
                         statusText: reqResult.statusText,
+                        category: responseCategory(reqResult.status),
                         errorPreview: reqResult.rawResponseBody.substring(0, 300),
                     };
                 }
@@ -127,6 +151,21 @@ export function createFishermanTools(apiClient, requestStore, opts) {
     };
     return { tools, getResult, isFinished };
 }
+function responseCategory(status) {
+    if (status === 400 || status === 422)
+        return 'validation';
+    if (status === 401 || status === 403)
+        return 'authorization';
+    if (status === 404)
+        return 'not_found';
+    if (status === 409)
+        return 'conflict';
+    if (status === 408 || status === 425 || status === 429)
+        return 'temporary';
+    if (status >= 500)
+        return 'server';
+    return 'client';
+}
 function extractKeyFields(body, result = {}, depth = 0) {
     if (!body || typeof body !== 'object' || depth > 5)
         return result;

package/dist/src/ai/fisherman.js

@@ -170,7 +170,8 @@ export class Fisherman {
       RULES:
       - Always call getEndpointSpec before your first request to an unfamiliar endpoint
       - Chain requests logically — create parent resources before children
-      - If a request fails, try once more with adjusted data before reporting failure
+      - Use the response category and error text to decide what failed: validation requires corrected data, authorization requires valid access, not_found requires a valid path or parent, and conflict requires resolving the conflicting state
+      - Retry temporary or server failures once. Retry other failures only when the specification or error text gives a concrete correction
       - Use realistic but unique data for each item (vary names, titles)
 
       ${dataProtectionRules}

package/dist/src/ai/navigator.d.ts

@@ -15,6 +15,7 @@ declare class Navigator implements Agent {
     experienceTracker: ExperienceTracker;
     hooksRunner: HooksRunner;
     MAX_ATTEMPTS: number;
+    lastFailureReason: string | null;
     systemPrompt: string;
     freeSailSystemPrompt: string;
     explorer: Explorer;
@@ -25,18 +26,39 @@ declare class Navigator implements Agent {
     get verifyTimeout(): number;
     getBaseOrigin(): string | null;
     getComparableCurrentUrl(stateManager: any, expectedUrl: string): string;
+    comparableUrl(state: {
+        url?: string;
+        fullUrl?: string;
+    }, expectedUrl: string): string;
     isSameExpectedOrigin(expectedUrl: string, stateManager: any): boolean;
     isOnExpectedPage(expectedUrl: string, stateManager: any): boolean;
     visit(url: string): Promise<void>;
     visitOnce(url: string): Promise<void>;
+    navigationError(url: string, fallback: string): Error;
+    failureReason(stopReason: string | null, knowledge: string, url: string): string | null;
     resolveState(message: string, actionResult: ActionResult, opts?: {
         action?: Action;
         expectedUrl?: string;
+        experience?: string;
         onAttempt?: (attempt: {
             code: string;
             error?: string;
         }) => void;
     }): Promise<boolean>;
+    buildResolutionPrompt(message: string, actionResult: ActionResult, injectedExperience?: string): Promise<string>;
+    buildRetryFeedback(failures: BatchFailure[], includeHtml: boolean, actionResult: ActionResult): Promise<string>;
+    executeAttempt(action: Action, codeBlock: string, message: string): Promise<{
+        ok: boolean;
+        error?: string;
+    }>;
+    verifyNavigation(action: Action, expectedUrl: string): Promise<{
+        freshState: ActionResult;
+        urlMatches: boolean;
+    }>;
+    ariaDiff(freshState: ActionResult, previous: ActionResult): Promise<string | null>;
+    saveFlow(message: string, expectedUrl: string | undefined, actionResult: ActionResult, progressBlocks: string[]): void;
+    rescueDelayedRedirect(action: Action, expectedUrl: string): Promise<boolean>;
+    askUserToResolve(action: Action, message: string, expectedUrl: string | undefined, stopReason: string | null): Promise<boolean>;
     buildExperienceTools(): {
         learnExperience: unknown;
     } | undefined;
@@ -61,6 +83,12 @@ declare class Navigator implements Agent {
     }>;
     checkAlreadyVerified(aiResponse: string, actionResult: ActionResult): boolean;
 }
+type BatchFailure = {
+    code: string;
+    error: string;
+    ariaChanges?: string | null;
+    urlAfter?: string;
+};
 export type AssertionResult = {
     code: string;
     passed: boolean;