explorbot 0.2.4 → 0.3.0

package/src/action-result.ts

@@ -4,7 +4,7 @@ import { ConfigParser, type HtmlConfig, outputPath } from './config.ts';
 import type { Link, WebPageState } from './state-manager.ts';
 import { LARGE_ARIA_CHANGE_THRESHOLD, compactAriaSnapshot, diffAriaSnapshots } from './utils/aria.ts';
 import { TTLCache } from './utils/cache.ts';
-import { type HtmlDiffPart, type HtmlDiffResult, htmlDiff } from './utils/html-diff.ts';
+import { type HtmlDiffPart, type HtmlDiffResult, htmlDiff, liveRegionMessages } from './utils/html-diff.ts';
 import { extractHeadings, extractLinks, extractTargetedHtml, htmlCombinedSnapshot, htmlMinimalUISnapshot, htmlTextSnapshot, minifyHtml } from './utils/html.ts';
 import { createDebug } from './utils/logger.ts';
 import { slugify } from './utils/strings.ts';
@@ -27,6 +27,7 @@ interface ActionResultData extends WebPageState {
   h3?: string | undefined;
   h4?: string | undefined;
   browserLogs?: any[];
+  networkRequests?: NetworkCall[];
   iframeSnapshots?: Array<{ src: string; html: string; id?: string }>;
   ariaSnapshot?: string | null;
   ariaSnapshotFile?: string;
@@ -41,6 +42,9 @@ export interface PageDiff {
   currentUrl: string;
   ariaChanges?: string | null;
   ariaChangeCount?: number;
+  messages?: string[];
+  requests?: NetworkCall[];
+  consoleErrors?: string[];
   htmlParts?: HtmlDiffPart[];
   iframes?: string;
 }
@@ -66,6 +70,7 @@ export class ActionResult implements ActionResultData {
   public url = '';
   public fullUrl: string | undefined = undefined;
   public browserLogs: any[] = [];
+  public networkRequests: NetworkCall[] = [];
   public iframeSnapshots: Array<{ src: string; html: string; id?: string }> = [];
   public iframeURL: string | undefined = undefined;
   readonly screenshotFile: string | undefined = undefined;
@@ -91,6 +96,7 @@ export class ActionResult implements ActionResultData {
     this.httpStatus = data.httpStatus;
     this.error = data.error ?? null;
     this.browserLogs = data.browserLogs ?? [];
+    this.networkRequests = data.networkRequests ?? [];
     this.iframeSnapshots = data.iframeSnapshots ?? [];
     this.iframeURL = data.iframeURL;
     this.notes = data.notes ?? [];
@@ -508,23 +514,30 @@ export class ActionResult implements ActionResultData {
       return result;
     }
 
-    const urlChanged = previousState ? !this.isSameUrl({ url: previousState.url }) : true;
+    const pageDiff: PageDiff = {
+      urlChanged: previousState ? !this.isSameUrl({ url: previousState.url }) : true,
+      currentUrl: this.url,
+    };
+    result.pageDiff = pageDiff;
 
-    if (!previousState) {
-      result.pageDiff = {
-        urlChanged: true,
-        currentUrl: this.url,
-      };
-      return result;
+    if (this.networkRequests.length > 0) {
+      pageDiff.requests = this.networkRequests;
+    }
+
+    const consoleErrors = this.consoleErrors();
+    if (consoleErrors.length > 0) {
+      pageDiff.consoleErrors = consoleErrors;
     }
 
+    if (!previousState) return result;
+
+    pageDiff.previousUrl = previousState.url;
+
     const diff = await this.diff(previousState);
 
-    const pageDiff: PageDiff = {
-      urlChanged,
-      previousUrl: previousState.url,
-      currentUrl: this.url,
-    };
+    if (diff.messages.length > 0) {
+      pageDiff.messages = diff.messages;
+    }
 
     if (diff.ariaChanged) {
       pageDiff.ariaChanges = diff.ariaChanged;
@@ -552,11 +565,28 @@ export class ActionResult implements ActionResultData {
       }
     }
 
-    result.pageDiff = pageDiff;
     return result;
   }
+
+  private consoleErrors(): string[] {
+    const errors: string[] = [];
+
+    for (const log of this.browserLogs) {
+      if ((log.type || log.level) !== 'error') continue;
+      const text = String(log.text || log.message || log).trim();
+      if (!text) continue;
+      if (errors.includes(text)) continue;
+      errors.push(text.slice(0, CONSOLE_ERROR_MAX_LENGTH));
+      if (errors.length === CONSOLE_ERROR_LIMIT) break;
+    }
+
+    return errors;
+  }
 }
 
+const CONSOLE_ERROR_MAX_LENGTH = 300;
+const CONSOLE_ERROR_LIMIT = 3;
+
 const HTML_PARTS_TOTAL_BUDGET = 8000;
 const HTML_PARTS_COUNT_LIMIT = 8;
 const HTML_PART_SUBTREE_BUDGET = 2000;
@@ -584,6 +614,7 @@ function collapseHtmlParts(parts: HtmlDiffPart[]): HtmlDiffPart[] {
 
 export class Diff {
   private _htmlDiffResult: HtmlDiffResult | null = null;
+  private _messages: string[] = [];
   private _ariaDiffResult: string | null = null;
   private _ariaChangeCount = 0;
   private _isSameUrl: boolean;
@@ -636,19 +667,33 @@ export class Diff {
     return this._htmlDiffResult;
   }
 
+  get messages(): string[] {
+    return this._messages;
+  }
+
   async calculate(): Promise<void> {
     if (!this.previous) return;
 
-    if (this._isSameUrl) {
-      this._htmlDiffResult = await htmlDiff(this.previous.html, this.current.html, ConfigParser.getInstance().getConfig().html);
+    if (!this._isSameUrl) {
+      this._messages = liveRegionMessages(this.previous.html, this.current.html);
+      return;
     }
 
+    this._htmlDiffResult = await htmlDiff(this.previous.html, this.current.html, ConfigParser.getInstance().getConfig().html);
+    this._messages = this._htmlDiffResult.messages;
+
     const ariaDiff = diffAriaSnapshots(this.previous.ariaSnapshot, this.current.ariaSnapshot);
     this._ariaDiffResult = ariaDiff.text;
     this._ariaChangeCount = ariaDiff.count;
   }
 }
 
+export interface NetworkCall {
+  method: string;
+  path: string;
+  status: number;
+}
+
 export interface FocusedElement {
   role: string;
   name: string;

package/src/action.ts

@@ -3,7 +3,7 @@ import { join } from 'node:path';
 import { context, trace } from '@opentelemetry/api';
 import { container, recorder } from 'codeceptjs';
 import * as codeceptjs from 'codeceptjs';
-import { ActionResult, type FocusedElement } from './action-result.js';
+import { ActionResult, type FocusedElement, type NetworkCall } from './action-result.js';
 import { clearActivity, setActivity } from './activity.ts';
 import { ConfigParser, outputPath } from './config.js';
 import type { ExplorbotConfig } from './config.js';
@@ -21,6 +21,8 @@ const debugLog = createDebug('explorbot:action');
 const CAPTURE_NAVIGATION_TRANSITION_ATTEMPTS = 3;
 const DEFAULT_ACTION_TIMEOUT = 3000;
 const DEFAULT_PAGE_TIMEOUT = 3000;
+const MAX_NETWORK_CALLS = 10;
+const IMPORTANT_LOG_LEVELS = new Set(['info', 'error', 'warning', 'warn']);
 
 class Action {
   private actor: CodeceptJS.I;
@@ -38,6 +40,8 @@ class Action {
   private recorder?: PlaywrightRecorder;
   private recovery: RecoveryRunner;
   private mainDocumentStatus: number | undefined = undefined;
+  private networkRequests: NetworkCall[] = [];
+  private baseOrigin: string;
 
   constructor(actor: CodeceptJS.I, stateManager: StateManager, recorder?: PlaywrightRecorder, recovery?: RecoveryRunner) {
     this.actor = actor;
@@ -46,6 +50,7 @@ class Action {
     this.playwrightHelper = container.helpers('Playwright');
     this.recorder = recorder;
     this.recovery = recovery || ((fn) => fn());
+    this.baseOrigin = URL.parse(this.config.playwright?.url || '')?.origin || '';
   }
 
   async saveScreenshot(): Promise<string | undefined> {
@@ -57,6 +62,7 @@ class Action {
     try {
       await (this.actor as any).saveScreenshot(filename);
       if (currentState) currentState.screenshotFile = filename;
+      tag('data').log('screenshot', { path: outputPath('states', filename) });
       return filename;
     } catch (err) {
       debugLog('Screenshot failed:', err);
@@ -105,7 +111,10 @@ class Action {
         const screenshotPath = join(statesDir, filename);
         screenshotFile = await page
           ?.screenshot({ path: screenshotPath, fullPage: true })
-          .then(() => filename)
+          .then(() => {
+            tag('data').log('screenshot', { path: screenshotPath });
+            return filename;
+          })
           .catch((err: Error) => {
             debugLog('Screenshot failed, continuing without it:', err);
             return undefined;
@@ -123,9 +132,7 @@ class Action {
       const logPath = join(statesDir, logFile);
       const formattedLogs = browserLogs.map((log: any) => {
         const logTimestamp = new Date().toISOString();
-        const level = (log.type || log.level || 'LOG').toUpperCase();
-        const message = log.text || log.message || String(log);
-        return `[${logTimestamp}] ${level}: ${message}`;
+        return `[${logTimestamp}] ${log.type.toUpperCase()}: ${log.text}`;
       });
       fs.writeFileSync(logPath, `${formattedLogs.join('\n')}\n`, 'utf8');
 
@@ -153,12 +160,16 @@ class Action {
         ariaSnapshotFile = ariaFileName;
       }
 
+      const networkRequests = this.networkRequests;
+      this.networkRequests = [];
+
       const result = new ActionResult({
         html,
         title,
         httpStatus: await this.captureMainDocumentStatus(),
         url,
         browserLogs,
+        networkRequests,
         htmlFile,
         logFile,
         screenshotFile,
@@ -198,26 +209,53 @@ class Action {
     }
   }
 
-  private captureMainDocumentResponse(): () => void {
+  private captureResponses(): () => void {
     const page = this.playwrightHelper.page;
     if (!page?.on || !page?.off) return () => {};
 
     this.mainDocumentStatus = undefined;
+    this.networkRequests = [];
 
     const handler = (response: any) => {
       const request = response.request();
-      if (request.resourceType() !== 'document') return;
-      if (response.frame() !== page.mainFrame()) return;
       const status = response.status();
       if (typeof status !== 'number') return;
       if (status <= 0) return;
-      this.mainDocumentStatus = status;
+
+      if (request.resourceType() === 'document') {
+        if (response.frame() !== page.mainFrame()) return;
+        this.mainDocumentStatus = status;
+        return;
+      }
+
+      this.recordNetworkCall(request, status);
     };
 
     page.on('response', handler);
     return () => page.off('response', handler);
   }
 
+  private recordNetworkCall(request: any, status: number): void {
+    const resourceType = request.resourceType();
+    if (resourceType !== 'xhr' && resourceType !== 'fetch') return;
+
+    const url = URL.parse(request.url());
+    if (!url) return;
+    if (url.origin !== this.baseOrigin) return;
+
+    const call: NetworkCall = { method: request.method(), path: url.pathname, status };
+    if (this.networkRequests.some((r) => r.method === call.method && r.path === call.path && r.status === call.status)) return;
+
+    if (this.networkRequests.length >= MAX_NETWORK_CALLS) {
+      if (status < 400) return;
+      const succeeded = this.networkRequests.findIndex((r) => r.status < 400);
+      if (succeeded === -1) return;
+      this.networkRequests.splice(succeeded, 1);
+    }
+
+    this.networkRequests.push(call);
+  }
+
   /**
    * Capture HTML snapshots of all iframes on the page
    */
@@ -263,13 +301,7 @@ class Action {
     try {
       const logs = await (this.actor as any).grabBrowserLogs();
 
-      // Filter for important logs (info, error, warning)
-      const importantLogs = logs.filter((log: any) => {
-        const level = log.type || log.level;
-        return ['info', 'error', 'warning', 'warn'].includes(level);
-      });
-
-      return importantLogs;
+      return logs.map(toBrowserLog).filter((log: any) => IMPORTANT_LOG_LEVELS.has(log.type));
     } catch (error) {
       debugLog('Failed to capture browser logs:', error);
       return [];
@@ -288,7 +320,7 @@ class Action {
     const stepListener = attachStepLogger(executedSteps, assertionSteps);
     const groupId = this.recorder ? await this.recorder.beginAction(codeString) : null;
     this.playwrightGroupId = groupId;
-    const detachMainDocumentResponse = this.captureMainDocumentResponse();
+    const detachResponses = this.captureResponses();
     const activeSpan = Observability.getSpan();
     const tracer = trace.getTracer('ai');
     const stepSpan = activeSpan ? tracer.startSpan('codeceptjs.step', undefined, trace.setSpan(context.active(), activeSpan)) : null;
@@ -337,7 +369,7 @@ class Action {
       throw err;
     } finally {
       this.restorePageTimeout();
-      detachMainDocumentResponse();
+      detachResponses();
       if (groupId) await this.recorder!.endAction();
       detachStepLogger(stepListener);
       if (stepSpan) {
@@ -425,6 +457,12 @@ async function captureHtml(page: any, frame: any, actor: any): Promise<string> {
   throw new Error('Playwright page is unavailable for HTML capture');
 }
 
+function toBrowserLog(log: any): { type: string; text: string } {
+  const type = typeof log.type === 'function' ? log.type() : log.type || log.level || 'log';
+  const text = typeof log.text === 'function' ? log.text() : log.text || log.message || String(log);
+  return { type, text: text.replace(/\s+/g, ' ').trim() };
+}
+
 async function captureTitle(page: any, actor: any): Promise<string> {
   if (page?.title) return page.title();
   if (actor?.grabTitle) return actor.grabTitle();

package/src/ai/captain/web-mode.ts

@@ -16,7 +16,7 @@ export function WithWebMode<T extends Constructor>(Base: T) {
         researcher: ctx.explorBot.agentResearcher(),
         navigator: ctx.explorBot.agentNavigator(),
       });
-      const { see, context, visualClick, learnExperience } = agentTools;
+      const { see, context, visualClick } = agentTools;
 
       const tools: Record<string, any> = {
         navigate: tool({
@@ -124,7 +124,6 @@ export function WithWebMode<T extends Constructor>(Base: T) {
 
         ...codeceptTools,
         context,
-        learnExperience,
       };
 
       if (see) tools.see = see;

package/src/ai/captain.ts

@@ -21,7 +21,7 @@ import type { Navigator } from './navigator.ts';
 import type { Provider } from './provider.ts';
 import { Researcher } from './researcher.ts';
 import { TaskAgent } from './task-agent.ts';
-import { withdrawVisionTools } from './tools.ts';
+import { createLearnExperienceTool, withdrawVisionTools } from './tools.ts';
 
 const MAX_STEPS = 15;
 
@@ -241,6 +241,14 @@ export class Captain extends CaptainBase implements Agent {
 
   private coreTools(task: Task, onDone: (summary: string) => void) {
     return {
+      learnExperience: createLearnExperienceTool({
+        getExperienceTracker: () => this.getExperienceTracker(),
+        getState: () => {
+          const state = this.explorBot.stateManager().getCurrentState();
+          if (!state) return null;
+          return ActionResult.fromState(state);
+        },
+      }),
       done: tool({
         description: 'Call when the user request is fulfilled.',
         inputSchema: z.object({

package/src/ai/driller.ts

@@ -30,7 +30,7 @@ import type { Navigator } from './navigator.ts';
 import type { Provider } from './provider.ts';
 import { drillLocatorRule } from './rules.ts';
 import { TaskAgent, isInteractive } from './task-agent.ts';
-import { createCodeceptJSTools } from './tools.ts';
+import { createCodeceptJSTools, createLearnExperienceTool } from './tools.ts';
 
 const debugLog = createDebug('explorbot:driller');
 
@@ -306,7 +306,11 @@ export class Driller extends TaskAgent implements Agent {
 
     let finished = false;
     const actionTools = this.createVerifiedActionTools(createCodeceptJSTools(this.toolDeps, test), component);
-    const tools = { ...actionTools, ...this.createDrillFlowTools(originalState, test, interactive) };
+    const learnExperience = createLearnExperienceTool({
+      getExperienceTracker: () => this.getExperienceTracker(),
+      getState: () => ActionResult.fromState(this.stateManager.getCurrentState() || originalState),
+    });
+    const tools = { ...actionTools, learnExperience, ...this.createDrillFlowTools(originalState, test, interactive) };
 
     await loop(
       async ({ stop, iteration }) => {

package/src/ai/fisherman-tools.ts

@@ -29,6 +29,28 @@ export function createFishermanTools(apiClient: ApiClient, requestStore: Request
 
         const captured = requestStore.findCapturedRequest(method, path);
         if (captured) {
+          if (captured.status >= 400) {
+            const rejectedCapture = {
+              status: captured.status,
+              requestBody: captured.requestBody || 'no body',
+            };
+            if (opts.spec) {
+              try {
+                const definition = extractEndpointDefinition(opts.spec, path, opts.baseEndpoint);
+                return { source: 'spec', method, path, definition, rejectedCapture };
+              } catch {
+                return { source: 'captured', method, path, usable: false, rejectedRequestBody: captured.requestBody || 'no body', status: captured.status };
+              }
+            }
+            return {
+              source: 'captured',
+              method: captured.method,
+              path: captured.path,
+              status: captured.status,
+              usable: false,
+              rejectedRequestBody: captured.requestBody || 'no body',
+            };
+          }
           return {
             source: 'captured',
             method: captured.method,
@@ -87,6 +109,7 @@ export function createFishermanTools(apiClient: ApiClient, requestStore: Request
             success: false,
             status: reqResult.status,
             statusText: reqResult.statusText,
+            category: responseCategory(reqResult.status),
             errorPreview: reqResult.rawResponseBody.substring(0, 300),
           };
         }
@@ -149,6 +172,16 @@ export function createFishermanTools(apiClient: ApiClient, requestStore: Request
   return { tools, getResult, isFinished };
 }
 
+function responseCategory(status: number): ResponseCategory {
+  if (status === 400 || status === 422) return 'validation';
+  if (status === 401 || status === 403) return 'authorization';
+  if (status === 404) return 'not_found';
+  if (status === 409) return 'conflict';
+  if (status === 408 || status === 425 || status === 429) return 'temporary';
+  if (status >= 500) return 'server';
+  return 'client';
+}
+
 function extractKeyFields(body: any, result: Record<string, any> = {}, depth = 0): Record<string, any> {
   if (!body || typeof body !== 'object' || depth > 5) return result;
 
@@ -179,3 +212,5 @@ export interface FishermanResult {
   created: Array<{ type: string; id?: string | number; title?: string }>;
   failed: Array<{ type: string; reason: string }>;
 }
+
+type ResponseCategory = 'validation' | 'authorization' | 'not_found' | 'conflict' | 'temporary' | 'server' | 'client';

package/src/ai/fisherman.ts

@@ -208,7 +208,8 @@ export class Fisherman implements Agent {
       RULES:
       - Always call getEndpointSpec before your first request to an unfamiliar endpoint
       - Chain requests logically — create parent resources before children
-      - If a request fails, try once more with adjusted data before reporting failure
+      - Use the response category and error text to decide what failed: validation requires corrected data, authorization requires valid access, not_found requires a valid path or parent, and conflict requires resolving the conflicting state
+      - Retry temporary or server failures once. Retry other failures only when the specification or error text gives a concrete correction
       - Use realistic but unique data for each item (vary names, titles)
 
       ${dataProtectionRules}