explorbot 0.1.32-beta.1 → 0.2.0

package/dist/src/explorer.d.ts

@@ -5,7 +5,7 @@ import { AIProvider } from './ai/provider.js';
 import { RequestStore } from './api/request-store.js';
 import { XhrCapture } from './api/xhr-capture.js';
 import type { ExplorbotConfig } from './config.js';
-import type { UserResolveFunction } from './explorbot.js';
+import type { ExperienceTracker } from './experience-tracker.js';
 import { KnowledgeTracker } from './knowledge-tracker.js';
 import { PlaywrightRecorder } from './playwright-recorder.js';
 import { Reporter } from './reporter.js';
@@ -42,7 +42,6 @@ declare class Explorer {
     stateManager: StateManager;
     knowledgeTracker: KnowledgeTracker;
     config: ExplorbotConfig;
-    userResolveFn: UserResolveFunction | null;
     options?: {
         show?: boolean;
         headless?: boolean;
@@ -59,19 +58,17 @@ declare class Explorer {
     testPageErrorHandler: ((error: Error) => void) | null;
     testConsoleHandler: ((message: any) => void) | null;
     testDialogHandler: ((dialog: any) => void) | null;
-    constructor(config: ExplorbotConfig, aiProvider: AIProvider, options?: {
+    constructor(config: ExplorbotConfig, aiProvider: AIProvider, options: {
         show?: boolean;
         headless?: boolean;
         incognito?: boolean;
         session?: string;
-    });
+    } | undefined, experienceTracker: ExperienceTracker, knowledgeTracker: KnowledgeTracker);
     initializeContainer(): void;
     convertToCodeceptConfig(config: ExplorbotConfig): any;
     getConfig(): ExplorbotConfig;
-    getConfigPath(): string | null;
     getAIProvider(): AIProvider;
     getStateManager(): StateManager;
-    getCurrentUrl(): string;
     getKnowledgeTracker(): KnowledgeTracker;
     getReporter(): Reporter;
     getRequestStore(): RequestStore | null;
@@ -87,7 +84,7 @@ declare class Explorer {
     }): Promise<ActionResult>;
     capturePageWithScreenshot(): Promise<ActionResult>;
     executeAction(code: string): Promise<Action>;
-    attemptAction(code: string, originalMessage?: string, experience?: boolean): Promise<boolean>;
+    attemptAction(code: string, originalMessage?: string): Promise<boolean>;
     getPlaywrightRecorder(): PlaywrightRecorder;
     visit(url: string): Promise<Action>;
     visitOnce(url: string): Promise<Action>;
@@ -100,7 +97,6 @@ declare class Explorer {
     }): Promise<number>;
     getEidxInContainer(containerCss: string | null): Promise<string[]>;
     getEidxByLocator(locator: string, container?: string | null): Promise<string | null>;
-    reload(): Promise<void>;
     resolveBrowserUrl(url?: string): string | null;
     isFatalBrowserError(error: unknown): boolean;
     recoverFromBrowserError(): Promise<boolean>;
@@ -112,22 +108,15 @@ declare class Explorer {
     hasOtherTabs(): boolean;
     getOtherTabsInfo(): TabInfo[];
     clearOtherTabsInfo(): void;
-    setUserResolve(userResolveFn: UserResolveFunction): void;
     listenToStateChanged(): void;
     stop(): Promise<void>;
     get activeTest(): Test | null;
     startTest(test: Test): Promise<boolean>;
     ensurePageAvailable(): Promise<boolean>;
-    ensureActiveTestPageAvailable(): Promise<boolean>;
     handleExecutionError(error: unknown): Promise<BrowserExecutionErrorResult>;
     watchActiveTestPage(page?: any): void;
     stopTest(test: Test, meta?: Record<string, string>): Promise<void>;
     unwatchActiveTestPages(): void;
-    hasPlaywrightLocator(locatorFn: (page: any) => any, opts?: {
-        multiple?: boolean;
-        contents?: boolean;
-        success?: (locator: any) => Promise<void> | void;
-    }): Promise<boolean>;
     playwrightLocatorCount(locatorFn: (page: any) => any): Promise<number>;
     bindFrameNavigated(page: any): void;
     openFreshTab(): Promise<void>;

package/dist/src/explorer.js

@@ -12,15 +12,14 @@ import { visuallyAnnotateContainers } from "./ai/researcher/coordinates.js";
 import { RequestStore } from "./api/request-store.js";
 import { XhrCapture } from "./api/xhr-capture.js";
 import { ConfigParser } from './config.js';
-import { KnowledgeTracker } from './knowledge-tracker.js';
 import { PlaywrightRecorder } from "./playwright-recorder.js";
 import { Reporter } from "./reporter.js";
 import { StateManager } from './state-manager.js';
 import { TestResult } from "./test-plan.js";
-import { BrowserRecoveryError, isFatalBrowserError, isNavigationTransitionError } from "./utils/browser-errors.js";
+import { BrowserRecoveryError, browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from "./utils/browser-errors.js";
 import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from "./utils/html.js";
 import { createDebug, log, tag } from './utils/logger.js';
-import { waitForPageReadiness } from "./utils/page-readiness.js";
+import { sleep, waitForPageReadiness } from "./utils/page-readiness.js";
 import { WebElement } from "./utils/web-element.js";
 const debugLog = createDebug('explorbot:explorer');
 const RECOVERABLE_NAVIGATION_ERRORS = /net::ERR_ABORTED|page\.screenshot.*Timeout|waiting for fonts to load/i;
@@ -33,7 +32,6 @@ class Explorer {
     stateManager;
     knowledgeTracker;
     config;
-    userResolveFn = null;
     options;
     reporter;
     otherTabs = [];
@@ -45,21 +43,20 @@ class Explorer {
     testPageErrorHandler = null;
     testConsoleHandler = null;
     testDialogHandler = null;
-    constructor(config, aiProvider, options) {
+    constructor(config, aiProvider, options, experienceTracker, knowledgeTracker) {
         this.config = config;
         this.aiProvider = aiProvider;
         this.options = options;
         this.initializeContainer();
-        this.stateManager = new StateManager({ incognito: this.options?.incognito });
-        this.knowledgeTracker = new KnowledgeTracker();
+        this.knowledgeTracker = knowledgeTracker;
+        this.stateManager = new StateManager(experienceTracker, knowledgeTracker);
         this.reporter = new Reporter(config.reporter, this.stateManager);
     }
     initializeContainer() {
         try {
             // Use project root for output directory, not current working directory
             const configParser = ConfigParser.getInstance();
-            const configPath = configParser.getConfigPath();
-            const projectRoot = configPath ? path.dirname(configPath) : process.cwd();
+            const projectRoot = configParser.getProjectRoot();
             global.output_dir = path.join(projectRoot, 'output', 'states');
             global.codecept_dir = projectRoot;
             configParser.validateConfig(this.config);
@@ -113,31 +110,20 @@ class Explorer {
             },
         };
         if (this.config.stepsFile) {
-            const configPath = ConfigParser.getInstance().getConfigPath();
-            const projectRoot = configPath ? path.dirname(configPath) : process.cwd();
+            const projectRoot = ConfigParser.getInstance().getProjectRoot();
             codeceptConfig.include = { I: path.resolve(projectRoot, this.config.stepsFile) };
         }
         return codeceptConfig;
     }
     getConfig() {
-        if (!this.config) {
-            throw new Error('Configuration not loaded. Call run() first.');
-        }
         return this.config;
     }
-    getConfigPath() {
-        const configParser = ConfigParser.getInstance();
-        return configParser.getConfigPath();
-    }
     getAIProvider() {
         return this.aiProvider;
     }
     getStateManager() {
         return this.stateManager;
     }
-    getCurrentUrl() {
-        return this.stateManager.getCurrentState().url || '?';
-    }
     getKnowledgeTracker() {
         return this.knowledgeTracker;
     }
@@ -175,9 +161,6 @@ class Explorer {
         if (this.isStarted) {
             return;
         }
-        if (!this.config) {
-            await this.initializeContainer();
-        }
         await codeceptjs.recorder.start();
         await codeceptjs.container.started(null);
         storeListener();
@@ -294,8 +277,8 @@ class Explorer {
     async executeAction(code) {
         return this.runWithBrowserRecovery('executeAction', () => this.createAction().execute(code));
     }
-    async attemptAction(code, originalMessage, experience = true) {
-        return this.runWithBrowserRecovery('attemptAction', () => this.createAction().attempt(code, originalMessage, experience));
+    async attemptAction(code, originalMessage) {
+        return this.runWithBrowserRecovery('attemptAction', () => this.createAction().attempt(code, originalMessage));
     }
     getPlaywrightRecorder() {
         return this.playwrightRecorder;
@@ -318,7 +301,7 @@ class Explorer {
                 await action.execute(`I.amOnPage(${serializedUrl})`);
             }
             catch (err) {
-                const msg = err instanceof Error ? err.message : String(err);
+                const msg = browserErrorMessage(err);
                 if (!RECOVERABLE_NAVIGATION_ERRORS.test(msg))
                     throw err;
                 tag('warning').log(`Navigation warning (continuing after load): ${msg.split('\n')[0]}`);
@@ -363,7 +346,7 @@ class Explorer {
         }
         catch (error) {
             if (this.isFatalBrowserError(error)) {
-                tag('warning').log(`getEidxInContainer: ${error instanceof Error ? error.message : error}`);
+                tag('warning').log(`getEidxInContainer: ${browserErrorMessage(error)}`);
                 await this.recoverFromBrowserError();
             }
             return [];
@@ -378,16 +361,12 @@ class Explorer {
         }
         catch (error) {
             if (this.isFatalBrowserError(error)) {
-                tag('warning').log(`getEidxByLocator: ${error instanceof Error ? error.message : error}`);
+                tag('warning').log(`getEidxByLocator: ${browserErrorMessage(error)}`);
                 await this.recoverFromBrowserError();
             }
             return null;
         }
     }
-    async reload() {
-        await this.closeOtherTabs();
-        await this.playwrightHelper.page.reload();
-    }
     resolveBrowserUrl(url) {
         if (!url)
             return null;
@@ -433,7 +412,7 @@ class Explorer {
             return this.waitForPageReadiness();
         }
         catch (err) {
-            tag('error').log(`Browser recovery failed: ${err instanceof Error ? err.message : err}`);
+            tag('error').log(`Browser recovery failed: ${browserErrorMessage(err)}`);
             return false;
         }
     }
@@ -471,7 +450,7 @@ class Explorer {
             return true;
         }
         catch (err) {
-            tag('error').log(`Browser restart failed: ${err instanceof Error ? err.message : err}`);
+            tag('error').log(`Browser restart failed: ${browserErrorMessage(err)}`);
             return false;
         }
     }
@@ -502,7 +481,7 @@ class Explorer {
         }
         catch (error) {
             if (this.isFatalBrowserError(error)) {
-                tag('warning').log(`isInsideIframe: ${error instanceof Error ? error.message : error}`);
+                tag('warning').log(`isInsideIframe: ${browserErrorMessage(error)}`);
                 await this.recoverFromBrowserError();
             }
             return false;
@@ -522,9 +501,6 @@ class Explorer {
     clearOtherTabsInfo() {
         this.otherTabs = [];
     }
-    setUserResolve(userResolveFn) {
-        this.userResolveFn = userResolveFn;
-    }
     listenToStateChanged() {
         if (!this.playwrightHelper) {
             debugLog('Playwright helper not available for state monitoring');
@@ -658,11 +634,8 @@ class Explorer {
         this.watchActiveTestPage();
         return true;
     }
-    async ensureActiveTestPageAvailable() {
-        return this.ensurePageAvailable();
-    }
     async handleExecutionError(error) {
-        const message = error instanceof Error ? error.message : String(error);
+        const message = browserErrorMessage(error);
         tag('error').log(`Browser execution error: ${message}`);
         if (error instanceof Error && error.name === 'AbortError') {
             return {
@@ -766,29 +739,6 @@ class Explorer {
         }
         this.observedTestPages.clear();
     }
-    async hasPlaywrightLocator(locatorFn, opts = {}) {
-        try {
-            const pwLocator = locatorFn(this.playwrightHelper.page);
-            const count = await pwLocator.count();
-            if (opts.multiple ? count === 0 : count !== 1)
-                return false;
-            if (opts.contents) {
-                const html = await pwLocator.first().innerHTML();
-                if (!html?.trim())
-                    return false;
-            }
-            if (opts.success)
-                await opts.success(pwLocator);
-            return true;
-        }
-        catch (error) {
-            if (this.isFatalBrowserError(error)) {
-                tag('warning').log(`hasPlaywrightLocator: ${error instanceof Error ? error.message : error}`);
-                await this.recoverFromBrowserError();
-            }
-            return false;
-        }
-    }
     async playwrightLocatorCount(locatorFn) {
         try {
             const pwLocator = locatorFn(this.playwrightHelper.page);
@@ -796,7 +746,7 @@ class Explorer {
         }
         catch (error) {
             if (this.isFatalBrowserError(error)) {
-                tag('warning').log(`playwrightLocatorCount: ${error instanceof Error ? error.message : error}`);
+                tag('warning').log(`playwrightLocatorCount: ${browserErrorMessage(error)}`);
                 await this.recoverFromBrowserError();
             }
             throw error;
@@ -815,7 +765,7 @@ class Explorer {
                 debugLog('Failed to get page title:', error);
             }
             this.stateManager.updateStateFromBasic(newUrl, newTitle, 'navigation');
-            await new Promise((resolve) => setTimeout(resolve, 500));
+            await sleep(500);
         });
     }
     async openFreshTab() {

package/dist/src/knowledge-tracker.d.ts

@@ -1,5 +1,5 @@
 import { ActionResult } from './action-result.js';
-interface Knowledge {
+export interface Knowledge {
     filePath: string;
     url: string;
     content: string;
@@ -12,6 +12,7 @@ export declare class KnowledgeTracker {
     constructor();
     loadKnowledgeFiles(): void;
     getRelevantKnowledge(state: ActionResult): Knowledge[];
+    renderRelevantKnowledge(state: ActionResult): string;
     addKnowledge(urlPattern: string, description: string): {
         filename: string;
         filePath: string;
@@ -30,4 +31,3 @@ export declare class KnowledgeTracker {
     normalizeUrl(url: string): string;
     getStateParameters(state: ActionResult, keys: string[]): Record<string, any>;
 }
-export {};

package/dist/src/knowledge-tracker.js

@@ -1,11 +1,15 @@
-import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
-import { dirname, join } from 'node:path';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import dedent from 'dedent';
 import matter from 'gray-matter';
 import { ActionResult } from './action-result.js';
 import { ConfigParser } from './config.js';
 import { getCliName } from "./utils/cli-name.js";
-import { createDebug } from './utils/logger.js';
+import { createDebug, pluralize, tag } from './utils/logger.js';
+import { loadMarkdownFiles } from './utils/markdown-files.js';
+import { mdq } from './utils/markdown-query.js';
 import { isSecretName, registerSecret } from './utils/secrets.js';
+import { slugify } from './utils/strings.js';
 const debugLog = createDebug('explorbot:knowledge-tracker');
 export class KnowledgeTracker {
     knowledgeDir;
@@ -14,14 +18,7 @@ export class KnowledgeTracker {
     constructor() {
         const configParser = ConfigParser.getInstance();
         const config = configParser.getConfig();
-        const configPath = configParser.getConfigPath();
-        if (configPath) {
-            const projectRoot = dirname(configPath);
-            this.knowledgeDir = join(projectRoot, config.dirs?.knowledge || 'knowledge');
-        }
-        else {
-            this.knowledgeDir = config.dirs?.knowledge || 'knowledge';
-        }
+        this.knowledgeDir = configParser.resolveProjectDir(config.dirs?.knowledge || 'knowledge');
         if (!existsSync(this.knowledgeDir)) {
             mkdirSync(this.knowledgeDir, { recursive: true });
         }
@@ -30,27 +27,13 @@ export class KnowledgeTracker {
         if (this.isLoaded)
             return;
         this.knowledgeFiles = [];
-        if (!existsSync(this.knowledgeDir)) {
-            return;
-        }
-        const files = readdirSync(this.knowledgeDir)
-            .filter((file) => file.endsWith('.md'))
-            .map((file) => join(this.knowledgeDir, file));
-        for (const filePath of files) {
-            try {
-                const fileContent = readFileSync(filePath, 'utf8');
-                const parsed = matter(fileContent);
-                const urlPattern = parsed.data.url || parsed.data.path || '*';
-                this.knowledgeFiles.push({
-                    filePath,
-                    url: urlPattern,
-                    content: this.interpolateVars(parsed.content),
-                    ...parsed.data,
-                });
-            }
-            catch (error) {
-                // Skip invalid files
-            }
+        for (const entry of loadMarkdownFiles(this.knowledgeDir, { recursive: true })) {
+            this.knowledgeFiles.push({
+                filePath: entry.filePath,
+                url: entry.data.url || entry.data.path || '*',
+                content: this.interpolateVars(entry.content),
+                ...entry.data,
+            });
         }
         this.isLoaded = true;
     }
@@ -60,21 +43,35 @@ export class KnowledgeTracker {
             return state.isMatchedBy(knowledge);
         });
     }
+    renderRelevantKnowledge(state) {
+        const knowledgeFiles = this.getRelevantKnowledge(state);
+        if (knowledgeFiles.length === 0)
+            return '';
+        const knowledgeContent = knowledgeFiles
+            .map((k) => k.content)
+            .filter((k) => !!k)
+            .join('\n\n');
+        tag('operation').log(`Found ${knowledgeFiles.length} relevant knowledge ${pluralize(knowledgeFiles.length, 'file')}`);
+        return dedent `
+      <knowledge>
+      Here is relevant knowledge for this page:
+
+      ${knowledgeContent}
+      </knowledge>
+    `;
+    }
     addKnowledge(urlPattern, description) {
         const configParser = ConfigParser.getInstance();
-        const config = configParser.getConfig();
         const configPath = configParser.getConfigPath();
         if (!configPath) {
             throw new Error(`No explorbot configuration found. Please run "${getCliName()} init" first.`);
         }
-        const projectRoot = dirname(configPath);
-        const knowledgeDir = join(projectRoot, config.dirs?.knowledge || 'knowledge');
-        if (!existsSync(knowledgeDir)) {
-            mkdirSync(knowledgeDir, { recursive: true });
+        if (!existsSync(this.knowledgeDir)) {
+            mkdirSync(this.knowledgeDir, { recursive: true });
         }
         const normalizedUrl = this.normalizeUrl(urlPattern);
         const filename = this.generateFilename(normalizedUrl);
-        const filePath = join(knowledgeDir, filename);
+        const filePath = join(this.knowledgeDir, filename);
         const isNewFile = !existsSync(filePath);
         if (isNewFile) {
             const frontmatter = {
@@ -101,6 +98,7 @@ export class KnowledgeTracker {
             const fileContent = matter.stringify(newContent, frontmatter);
             writeFileSync(filePath, fileContent, 'utf8');
         }
+        this.isLoaded = false;
         return { filename, filePath, isNewFile };
     }
     interpolateVars(content) {
@@ -129,12 +127,7 @@ export class KnowledgeTracker {
         });
     }
     generateFilename(url) {
-        let filename = url
-            .replace(/https?:\/\//g, '')
-            .replace(/[^a-zA-Z0-9_]/g, '_')
-            .replace(/_+/g, '_')
-            .replace(/^_|_$/g, '')
-            .toLowerCase();
+        let filename = slugify(url.replace(/https?:\/\//g, ''));
         if (!filename || filename === '*') {
             filename = 'general';
         }
@@ -156,7 +149,7 @@ export class KnowledgeTracker {
         this.loadKnowledgeFiles();
         return this.knowledgeFiles.map((knowledge) => {
             const content = knowledge.content.trim();
-            const firstLine = content.split('\n')[0]?.trim() || '';
+            const firstLine = mdq(content).meta()[0]?.text.split('\n')[0]?.trim() || '';
             return {
                 url: knowledge.url,
                 firstLine,

package/dist/src/state-manager.d.ts

@@ -1,5 +1,6 @@
 import { ActionResult } from './action-result.js';
-import { ExperienceTracker } from './experience-tracker.js';
+import type { ExperienceTracker } from './experience-tracker.js';
+import type { KnowledgeTracker, Knowledge } from './knowledge-tracker.js';
 export interface Link {
     title: string;
     url: string;
@@ -54,24 +55,18 @@ export interface StateTransition {
     trigger: 'manual' | 'navigation' | 'automatic';
 }
 export type StateChangeListener = (event: StateTransition) => void;
-export interface Knowledge extends WebPageState {
-    /** File path */
-    filePath: string;
-    /** Markdown content */
-    content: string;
-}
+export type { Knowledge };
 export declare class StateManager {
     currentState: WebPageState | null;
     stateHistory: StateTransition[];
     allVisitedUrls: Set<string>;
-    knowledgeCache: Knowledge[];
-    lastKnowledgeScan: Date | null;
     stateChangeListeners: StateChangeListener[];
     experienceTracker: ExperienceTracker;
-    knowledgeDir: string;
+    knowledgeTracker: KnowledgeTracker;
     nextStateId: number;
-    constructor();
+    constructor(experienceTracker: ExperienceTracker, knowledgeTracker: KnowledgeTracker);
     getExperienceTracker(): ExperienceTracker;
+    getKnowledgeTracker(): KnowledgeTracker;
     /**
      * Subscribe to state change events
      */
@@ -80,11 +75,6 @@ export declare class StateManager {
      * Emit state change event to all listeners
      */
     emitStateChange(event: StateTransition): void;
-    /**
-     * Extract state path from full URL
-     * Removes domain, port, protocol
-     * Keeps path, query, and hash: /path/to/page?tab=users#section
-     */
     /**
      * Update current state from ActionResult and record transition if state changed
      */
@@ -119,10 +109,6 @@ export declare class StateManager {
      * Get the last N transitions
      */
     getRecentTransitions(count?: number): StateTransition[];
-    /**
-     * Scan knowledge directory for .md files and cache them
-     */
-    scanKnowledgeFiles(): void;
     /**
      * Get relevant knowledge files for current state
      */
@@ -134,12 +120,6 @@ export declare class StateManager {
     /**
      * Get all context for current state (knowledge + experience)
      */
-    getCurrentContext(): {
-        state: WebPageState;
-        knowledge: Knowledge[];
-        experience: string[];
-        recentTransitions: StateTransition[];
-    };
     /**
      * Check if we've been in this state before
      */
@@ -158,11 +138,6 @@ export declare class StateManager {
      * If the last transition changed URL, returns the fromState (for URL change detection).
      * Otherwise returns the most recent toState with content (for diffing).
      */
-    getPreviousState(): WebPageState | null;
-    /**
-     * Load HTML content from file
-     */
-    loadHtmlFromFile(htmlFile: string): string | null;
     /**
      * Clear state history (useful for testing or reset)
      * Note: This preserves the current state, only clears navigation history