explorbot 0.1.32-beta.1 → 0.2.0

package/dist/src/utils/web-element.d.ts

@@ -1,4 +1,4 @@
-import { type RawElementData } from './html.js';
+import { EXPLORBOT_ATTRS, type RawElementData } from './html.js';
 import { type XPathMatch } from './xpath.js';
 export { extractElementData } from './html.js';
 export declare class WebElement {
@@ -26,6 +26,7 @@ export declare class WebElement {
     get keyAttrs(): string;
     get coordinates(): string;
     get eidx(): string | null;
+    ourAttr(name: keyof typeof EXPLORBOT_ATTRS): string | undefined;
     get isNavigationLink(): boolean;
     get filteredClasses(): string[];
     get areaHints(): string[];

package/dist/src/utils/web-element.js

@@ -39,6 +39,9 @@ export class WebElement {
     get eidx() {
         return this.attrs[EXPLORBOT_ATTRS.eidx] || this.attrs.eidx || null;
     }
+    ourAttr(name) {
+        return this.attrs[EXPLORBOT_ATTRS[name]];
+    }
     get isNavigationLink() {
         if (this.tag !== 'a')
             return false;

package/dist/src/utils/web-sandbox.d.ts

@@ -0,0 +1,4 @@
+export declare function sanitizeCodeBlock(code: string): string;
+export declare function hasPlaywrightCommands(code: string): boolean;
+export declare function playwrightSandbox(page: any, code: string): Promise<any>;
+export declare function codeceptJSSandbox(actor: any, codeOrFn: string | ((...args: any[]) => void)): void;

package/dist/src/utils/web-sandbox.js

@@ -0,0 +1,37 @@
+import { faker } from '@faker-js/faker';
+import { hopeThat, retryTo, tryTo, within } from 'codeceptjs/lib/effects';
+import step from 'codeceptjs/steps';
+const SHADOWED_GLOBALS = ['process', 'global', 'globalThis', 'fetch', 'Bun', 'require', 'module', 'exports'];
+const ALLOWED_COMMAND_HEADS = ['I.', 'page.', 'await page.', 'await I.'];
+const PLAYWRIGHT_COMMAND_HEADS = ['page.', 'await page.'];
+const PLAYWRIGHT_ARG_NAMES = ['page'];
+const CODECEPT_ARG_NAMES = ['I', 'tryTo', 'retryTo', 'within', 'hopeThat', 'step', 'faker'];
+export function sanitizeCodeBlock(code) {
+    return code
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => ALLOWED_COMMAND_HEADS.some((head) => line.startsWith(head)))
+        .join('\n');
+}
+export function hasPlaywrightCommands(code) {
+    return code.split('\n').some((line) => {
+        const trimmed = line.trim();
+        return PLAYWRIGHT_COMMAND_HEADS.some((head) => trimmed.startsWith(head));
+    });
+}
+export function playwrightSandbox(page, code) {
+    const run = createSandbox(PLAYWRIGHT_ARG_NAMES, `return (async () => { ${code} })()`);
+    return run(page);
+}
+export function codeceptJSSandbox(actor, codeOrFn) {
+    if (typeof codeOrFn === 'function') {
+        codeOrFn(actor, tryTo, retryTo, within, hopeThat, step, faker);
+        return;
+    }
+    const run = createSandbox(CODECEPT_ARG_NAMES, codeOrFn);
+    run(actor, tryTo, retryTo, within, hopeThat, step, faker);
+}
+function createSandbox(argNames, body) {
+    const fn = new Function(...argNames, ...SHADOWED_GLOBALS, `'use strict';\n${body}`);
+    return (...args) => fn(...args);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.1.32-beta.1",
+  "version": "0.2.0",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",
@@ -16,7 +16,18 @@
   "bin": {
     "explorbot": "./dist/bin/explorbot-cli.js"
   },
-  "files": ["dist/", "src/**/*.ts", "src/**/*.tsx", "bin/**/*.ts", "boat/api-tester/src/**/*.ts", "boat/doc-collector/src/**/*.ts", "boat/doc-collector/bin/**/*.ts", "boat/doc-collector/package.json", "rules/", "assets/sample-files/"],
+  "files": [
+    "dist/",
+    "src/**/*.ts",
+    "src/**/*.tsx",
+    "bin/**/*.ts",
+    "boat/api-tester/src/**/*.ts",
+    "boat/doc-collector/src/**/*.ts",
+    "boat/doc-collector/bin/**/*.ts",
+    "boat/doc-collector/package.json",
+    "rules/",
+    "assets/sample-files/"
+  ],
   "scripts": {
     "build": "bun run build:bin",
     "build:bin": "bun build bin/explorbot-cli.ts --outdir bin --target node --external commander --format esm",
@@ -43,7 +54,13 @@
     "check:fix": "biome check --write .",
     "langfuse:export": "bun run .claude/skills/explorbot-debug/langfuse-export.ts"
   },
-  "keywords": ["cli", "react", "ink", "codeceptjs", "playwright"],
+  "keywords": [
+    "cli",
+    "react",
+    "ink",
+    "codeceptjs",
+    "playwright"
+  ],
   "repository": {
     "type": "git",
     "url": "https://github.com/testomatio/explorbot"

package/src/action-result.ts

@@ -2,10 +2,12 @@ import fs from 'node:fs';
 import { join } from 'node:path';
 import { ConfigParser, type HtmlConfig, outputPath } from './config.ts';
 import type { Link, WebPageState } from './state-manager.ts';
-import { compactAriaSnapshot, diffAriaSnapshots } from './utils/aria.ts';
+import { LARGE_ARIA_CHANGE_THRESHOLD, compactAriaSnapshot, diffAriaSnapshots } from './utils/aria.ts';
+import { TTLCache } from './utils/cache.ts';
 import { type HtmlDiffPart, type HtmlDiffResult, htmlDiff } from './utils/html-diff.ts';
 import { extractHeadings, extractLinks, extractTargetedHtml, htmlCombinedSnapshot, htmlMinimalUISnapshot, htmlTextSnapshot, minifyHtml } from './utils/html.ts';
 import { createDebug } from './utils/logger.ts';
+import { slugify } from './utils/strings.ts';
 import { extractStatePath, matchesUrl } from './utils/url-matcher.ts';
 
 const debugLog = createDebug('explorbot:state');
@@ -37,6 +39,7 @@ export interface PageDiff {
   previousUrl?: string;
   currentUrl: string;
   ariaChanges?: string | null;
+  ariaChangeCount?: number;
   htmlParts?: HtmlDiffPart[];
   iframes?: string;
 }
@@ -68,13 +71,12 @@ export class ActionResult implements ActionResultData {
   private _screenshot: Buffer | undefined = undefined;
   readonly htmlFile: string | undefined = undefined;
   private _html: string | undefined = undefined;
-  private snapshotCache = new Map<string, string>();
+  private snapshotCache = new TTLCache<string>();
   readonly logFile: string | undefined = undefined;
-  private _browserLogs: any[] | undefined = undefined;
   readonly ariaSnapshotFile: string | undefined = undefined;
   private _ariaSnapshot: string | null | undefined = undefined;
   private _lastExtractedHtml: string | undefined = undefined;
-  notes: any;
+  notes: string[] = [];
   public links: Link[] = [];
   public verifications?: Record<string, boolean>;
 
@@ -106,14 +108,11 @@ export class ActionResult implements ActionResultData {
       this.ariaSnapshotFile = data.ariaSnapshotFile;
     }
 
-    // Store HTML and browser logs in private properties if provided
+    // Store HTML in a private property if provided
     if (data.html !== undefined) {
       this._html = data.html;
     }
 
-    if (data.browserLogs !== undefined) {
-      this._browserLogs = data.browserLogs;
-    }
     if (data.screenshot !== undefined) {
       this._screenshot = data.screenshot;
     }
@@ -121,7 +120,7 @@ export class ActionResult implements ActionResultData {
       this._ariaSnapshot = data.ariaSnapshot;
     }
 
-    if (!this.fullUrl && this.url && this.url !== '') {
+    if (!this.fullUrl && this.url) {
       this.fullUrl = this.url;
     }
 
@@ -133,7 +132,7 @@ export class ActionResult implements ActionResultData {
       this.links = extractLinks(this._html);
     }
 
-    if (this.url && this.url !== '') {
+    if (this.url) {
       this.url = extractStatePath(this.url);
     }
   }
@@ -165,16 +164,6 @@ export class ActionResult implements ActionResultData {
     return undefined;
   }
 
-  get browserLogsContent(): any[] {
-    if (this._browserLogs !== undefined) {
-      return this._browserLogs;
-    }
-    if (this.logFile) {
-      return ActionResult.loadBrowserLogsFromFile(this.logFile);
-    }
-    return [];
-  }
-
   get ariaSnapshot(): string | null {
     if (this._ariaSnapshot !== undefined) {
       return this._ariaSnapshot;
@@ -223,14 +212,14 @@ export class ActionResult implements ActionResultData {
   }
 
   isSameUrl(state: WebPageState): boolean {
-    if (!this.url || this.url === '') {
+    if (!this.url) {
       return false;
     }
     return extractStatePath(state.url) === extractStatePath(this.url);
   }
 
   isMatchedBy(state: WebPageState): boolean {
-    if (!this.url || this.url === '') {
+    if (!this.url) {
       return false;
     }
 
@@ -266,13 +255,13 @@ export class ActionResult implements ActionResultData {
   async simplifiedHtml(htmlConfig?: HtmlConfig): Promise<string> {
     const normalizedConfig = this.normalizeHtmlConfig(htmlConfig);
     const cacheKey = `simplified:${JSON.stringify(normalizedConfig?.minimal ?? null)}`;
-    return this.cachedSnapshot(cacheKey, () => minifyHtml(htmlMinimalUISnapshot(this.html ?? '', normalizedConfig?.minimal)));
+    return this.snapshotCache.fetch(cacheKey, () => minifyHtml(htmlMinimalUISnapshot(this.html ?? '', normalizedConfig?.minimal)));
   }
 
   async combinedHtml(htmlConfig?: HtmlConfig & { keepPositions?: boolean }): Promise<string> {
     const normalizedConfig = this.normalizeHtmlConfig(htmlConfig);
     const cacheKey = `combined:${!!htmlConfig?.keepPositions}:${JSON.stringify(normalizedConfig?.combined ?? null)}`;
-    return this.cachedSnapshot(cacheKey, async () => {
+    return this.snapshotCache.fetch(cacheKey, async () => {
       const combinedHtml = await minifyHtml(htmlCombinedSnapshot(this.html ?? '', normalizedConfig?.combined, { keepPositions: htmlConfig?.keepPositions }));
       debugLog(`----${this.url}----`);
       debugLog(`Combined HTML: \n${combinedHtml}`);
@@ -284,7 +273,7 @@ export class ActionResult implements ActionResultData {
   async textHtml(htmlConfig?: HtmlConfig): Promise<string> {
     const normalizedConfig = this.normalizeHtmlConfig(htmlConfig);
     const cacheKey = `text:${JSON.stringify(normalizedConfig?.text ?? null)}`;
-    return this.cachedSnapshot(cacheKey, () => minifyHtml(htmlTextSnapshot(this.html ?? '', normalizedConfig?.text)));
+    return this.snapshotCache.fetch(cacheKey, () => minifyHtml(htmlTextSnapshot(this.html ?? '', normalizedConfig?.text)));
   }
 
   getInteractiveARIA(): string {
@@ -303,14 +292,6 @@ export class ActionResult implements ActionResultData {
     return parser.getConfig().html;
   }
 
-  private async cachedSnapshot(key: string, compute: () => Promise<string>): Promise<string> {
-    const cached = this.snapshotCache.get(key);
-    if (cached !== undefined) return cached;
-    const value = await compute();
-    this.snapshotCache.set(key, value);
-    return value;
-  }
-
   static fromState(state: WebPageState): ActionResult {
     let html: string | undefined = undefined;
     let screenshot: Buffer | undefined = undefined;
@@ -356,26 +337,26 @@ export class ActionResult implements ActionResultData {
     return new ActionResult(actionResultData);
   }
 
-  private static loadHtmlFromFile(htmlFile: string): string | null {
+  private static loadStateFile(file: string): string | null {
     try {
-      const filePath = outputPath('states', htmlFile);
-      if (fs.existsSync(filePath)) {
-        return fs.readFileSync(filePath, 'utf8');
-      }
-      return null;
+      const filePath = outputPath('states', file);
+      if (!fs.existsSync(filePath)) return null;
+      return fs.readFileSync(filePath, 'utf8');
     } catch (error) {
-      console.error('Failed to load HTML from file:', error);
+      console.error('Failed to load state file:', error);
       return null;
     }
   }
 
+  private static loadHtmlFromFile(htmlFile: string): string | null {
+    return ActionResult.loadStateFile(htmlFile);
+  }
+
   private static loadScreenshotFromFile(screenshotFile: string): Buffer | undefined {
     try {
       const filePath = outputPath('states', screenshotFile);
-      if (fs.existsSync(filePath)) {
-        return fs.readFileSync(filePath);
-      }
-      return undefined;
+      if (!fs.existsSync(filePath)) return undefined;
+      return fs.readFileSync(filePath);
     } catch (error) {
       console.error('Failed to load screenshot from file:', error);
       return undefined;
@@ -383,58 +364,38 @@ export class ActionResult implements ActionResultData {
   }
 
   private static loadBrowserLogsFromFile(logFile: string): any[] {
-    try {
-      const filePath = outputPath('states', logFile);
-      if (fs.existsSync(filePath)) {
-        const logContent = fs.readFileSync(filePath, 'utf8');
-        return logContent
-          .split('\n')
-          .filter((line) => line.trim())
-          .map((line) => {
-            const match = line.match(/\[([^\]]+)\] (\w+): (.+)/);
-            if (match) {
-              return {
-                timestamp: match[1],
-                type: match[2].toLowerCase(),
-                text: match[3],
-                level: match[2].toLowerCase(),
-                message: match[3],
-              };
-            }
-            return { text: line, type: 'log', level: 'log', message: line };
-          });
-      }
-      return [];
-    } catch (error) {
-      console.error('Failed to load browser logs from file:', error);
-      return [];
-    }
+    const logContent = ActionResult.loadStateFile(logFile);
+    if (!logContent) return [];
+    return logContent
+      .split('\n')
+      .filter((line) => line.trim())
+      .map((line) => {
+        const match = line.match(/\[([^\]]+)\] (\w+): (.+)/);
+        if (match) {
+          return {
+            timestamp: match[1],
+            type: match[2].toLowerCase(),
+            text: match[3],
+            level: match[2].toLowerCase(),
+            message: match[3],
+          };
+        }
+        return { text: line, type: 'log', level: 'log', message: line };
+      });
   }
 
   private static loadAriaSnapshotFromFile(ariaSnapshotFile: string): string | null {
-    try {
-      const filePath = outputPath('states', ariaSnapshotFile);
-      if (!fs.existsSync(filePath)) {
-        return null;
-      }
-
-      const content = fs.readFileSync(filePath, 'utf8');
-      const trimmed = content.trim();
-      if (!trimmed) {
-        return null;
-      }
-
-      return trimmed.endsWith('\n') ? trimmed : `${trimmed}\n`;
-    } catch (error) {
-      console.error('Failed to load aria snapshot from file:', error);
-      return null;
-    }
+    const content = ActionResult.loadStateFile(ariaSnapshotFile);
+    if (!content) return null;
+    const trimmed = content.trim();
+    if (!trimmed) return null;
+    return trimmed.endsWith('\n') ? trimmed : `${trimmed}\n`;
   }
 
   toAiContext(): string {
     const parts: string[] = [];
 
-    if (this.url && this.url !== '') {
+    if (this.url) {
       parts.push(`<url>${this.url}</url>`);
     }
 
@@ -480,7 +441,7 @@ export class ActionResult implements ActionResultData {
   }
 
   get relativeUrl(): string | null {
-    if (!this.url || this.url === '') return null;
+    if (!this.url) return null;
 
     try {
       const urlObj = new URL(this.url);
@@ -505,22 +466,10 @@ export class ActionResult implements ActionResultData {
 
     this.extractHeadings(this.html);
 
-    const headings = ['h1', 'h2'];
+    if (this.h1) parts.push(`h1_${this.h1}`);
+    if (this.h2) parts.push(`h2_${this.h2}`);
 
-    for (const heading of headings) {
-      const value = this[heading as keyof this] as string;
-      if (value) {
-        parts.push(`${heading}_${value}`);
-      }
-    }
-
-    let stateString = parts
-      .map((part) => part.substring(0, 100))
-      .join('_')
-      .replace(/[^a-zA-Z0-9_]/g, '_')
-      .replace(/_+/g, '_')
-      .replace(/^_|_$/g, '')
-      .toLowerCase();
+    let stateString = slugify(parts.map((part) => part.substring(0, 100)).join('_'));
 
     if (stateString.length > 200) {
       stateString = stateString.substring(0, 200);
@@ -532,60 +481,8 @@ export class ActionResult implements ActionResultData {
     return stateString;
   }
 
-  private saveBrowserLogs(): void {
-    const logs = this.browserLogsContent;
-    if (!logs || logs.length === 0) return;
-
-    try {
-      const outputDir = 'output';
-      const stateHash = this.getStateHash();
-      const filename = `${stateHash}.log`;
-      const filePath = join(outputDir, filename);
-
-      // Ensure output directory exists
-      if (!fs.existsSync(outputDir)) {
-        fs.mkdirSync(outputDir, { recursive: true });
-      }
-
-      // Format logs for saving
-      const formattedLogs = logs.map((log: any) => {
-        const timestamp = new Date().toISOString();
-        const level = (log.type || log.level || 'LOG').toUpperCase();
-        const message = log.text || log.message || String(log);
-        return `[${timestamp}] ${level}: ${message}`;
-      });
-
-      // Save log content to file
-      const logContent = `${formattedLogs.join('\n')}\n`;
-      fs.writeFileSync(filePath, logContent, 'utf8');
-    } catch (error) {
-      // Silently fail to avoid breaking the main flow
-      console.error('Failed to save browser logs:', error);
-    }
-  }
-
-  private saveHtmlOutput(): void {
-    try {
-      const outputDir = 'output';
-      const stateHash = this.getStateHash();
-      const filename = `${stateHash}.html`;
-      const filePath = join(outputDir, filename);
-
-      // Ensure output directory exists
-      if (!fs.existsSync(outputDir)) {
-        fs.mkdirSync(outputDir, { recursive: true });
-      }
-
-      // Save HTML content to file
-      fs.writeFileSync(filePath, this.html, 'utf8');
-    } catch (error) {
-      // Silently fail to avoid breaking the main flow
-      console.error('Failed to save HTML output:', error);
-    }
-  }
-
   async diff(previousState: ActionResult | null): Promise<Diff> {
-    return new Diff(this, previousState);
+    return Diff.create(this, previousState);
   }
 
   async toToolResult(previousState: ActionResult | null, locator: string): Promise<ToolResultMetadata> {
@@ -617,7 +514,6 @@ export class ActionResult implements ActionResultData {
     }
 
     const diff = await this.diff(previousState);
-    await diff.calculate();
 
     const pageDiff: PageDiff = {
       urlChanged,
@@ -627,6 +523,7 @@ export class ActionResult implements ActionResultData {
 
     if (diff.ariaChanged) {
       pageDiff.ariaChanges = diff.ariaChanged;
+      pageDiff.ariaChangeCount = diff.ariaChangeCount;
     }
 
     if (diff.htmlParts.length > 0) {
@@ -645,10 +542,7 @@ export class ActionResult implements ActionResultData {
     }
 
     if (pageDiff.ariaChanges && this.iframeSnapshots.length > 0) {
-      const addedCount = (pageDiff.ariaChanges.match(/\n {4}- /g) || []).length;
-      const removedMatch = pageDiff.ariaChanges.match(/removed: (\d+) interactive/);
-      const removedCount = removedMatch ? Number.parseInt(removedMatch[1]) : 0;
-      if (addedCount + removedCount >= 50) {
+      if (diff.ariaChangeCount >= LARGE_ARIA_CHANGE_THRESHOLD) {
         pageDiff.iframes = this.iframeSnapshots.map((snap) => `iframe src="${snap.src}":\n${snap.html}`).join('\n\n');
       }
     }
@@ -686,20 +580,25 @@ function collapseHtmlParts(parts: HtmlDiffPart[]): HtmlDiffPart[] {
 export class Diff {
   private _htmlDiffResult: HtmlDiffResult | null = null;
   private _ariaDiffResult: string | null = null;
+  private _ariaChangeCount = 0;
   private _isSameUrl: boolean;
-  private _urlChanged: boolean;
 
   constructor(
     private current: ActionResult,
     private previous: ActionResult | null
   ) {
     this._isSameUrl = previous ? current.isSameUrl({ url: previous.url }) : false;
-    this._urlChanged = !this._isSameUrl;
+  }
+
+  static async create(current: ActionResult, previous: ActionResult | null): Promise<Diff> {
+    const diff = new Diff(current, previous);
+    await diff.calculate();
+    return diff;
   }
 
   hasChanges(): boolean {
     if (!this.previous) return false;
-    if (this._urlChanged) return true;
+    if (!this._isSameUrl) return true;
 
     const hasHtmlChanges = this._htmlDiffResult && (this._htmlDiffResult.parts.length > 0 || this._htmlDiffResult.added.length > 0 || this._htmlDiffResult.removed.length > 0);
     const hasAriaChanges = this._ariaDiffResult !== null;
@@ -712,7 +611,7 @@ export class Diff {
   }
 
   urlHasChanged(): boolean {
-    return this._urlChanged;
+    return !this._isSameUrl;
   }
 
   get htmlParts(): HtmlDiffPart[] {
@@ -724,18 +623,14 @@ export class Diff {
     return this._ariaDiffResult;
   }
 
-  get ariaRemoved(): string | null {
-    return this._ariaDiffResult;
+  get ariaChangeCount(): number {
+    return this._ariaChangeCount;
   }
 
   get htmlDiff(): HtmlDiffResult | null {
     return this._htmlDiffResult;
   }
 
-  get ariaDiff(): string | null {
-    return this._ariaDiffResult;
-  }
-
   async calculate(): Promise<void> {
     if (!this.previous) return;
 
@@ -743,6 +638,8 @@ export class Diff {
       this._htmlDiffResult = await htmlDiff(this.previous.html, this.current.html, ConfigParser.getInstance().getConfig().html);
     }
 
-    this._ariaDiffResult = diffAriaSnapshots(this.previous.ariaSnapshot, this.current.ariaSnapshot);
+    const ariaDiff = diffAriaSnapshots(this.previous.ariaSnapshot, this.current.ariaSnapshot);
+    this._ariaDiffResult = ariaDiff.text;
+    this._ariaChangeCount = ariaDiff.count;
   }
 }