explorbot 0.1.31 → 0.2.0

package/src/explorer.ts

@@ -14,17 +14,17 @@ import { visuallyAnnotateContainers } from './ai/researcher/coordinates.ts';
 import { RequestStore } from './api/request-store.ts';
 import { XhrCapture } from './api/xhr-capture.ts';
 import type { ExplorbotConfig } from './config.js';
-import { ConfigParser, outputPath } from './config.js';
-import type { UserResolveFunction } from './explorbot.js';
+import { ConfigParser } from './config.js';
+import type { ExperienceTracker } from './experience-tracker.js';
 import { KnowledgeTracker } from './knowledge-tracker.js';
 import { PlaywrightRecorder } from './playwright-recorder.ts';
 import { Reporter } from './reporter.ts';
 import { StateManager } from './state-manager.js';
 import { Test, TestResult } from './test-plan.ts';
-import { BrowserRecoveryError, isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts';
+import { BrowserRecoveryError, browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts';
 import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from './utils/html.ts';
 import { createDebug, log, tag } from './utils/logger.js';
-import { waitForPageReadiness } from './utils/page-readiness.ts';
+import { sleep, waitForPageReadiness } from './utils/page-readiness.ts';
 import { WebElement } from './utils/web-element.ts';
 
 declare global {
@@ -64,7 +64,6 @@ class Explorer {
   private stateManager!: StateManager;
   private knowledgeTracker!: KnowledgeTracker;
   config: ExplorbotConfig;
-  private userResolveFn: UserResolveFunction | null = null;
   private options?: { show?: boolean; headless?: boolean; incognito?: boolean; session?: string };
   private reporter!: Reporter;
   private otherTabs: TabInfo[] = [];
@@ -77,13 +76,13 @@ class Explorer {
   private testConsoleHandler: ((message: any) => void) | null = null;
   private testDialogHandler: ((dialog: any) => void) | null = null;
 
-  constructor(config: ExplorbotConfig, aiProvider: AIProvider, options?: { show?: boolean; headless?: boolean; incognito?: boolean; session?: string }) {
+  constructor(config: ExplorbotConfig, aiProvider: AIProvider, options: { show?: boolean; headless?: boolean; incognito?: boolean; session?: string } | undefined, experienceTracker: ExperienceTracker, knowledgeTracker: KnowledgeTracker) {
     this.config = config;
     this.aiProvider = aiProvider;
     this.options = options;
     this.initializeContainer();
-    this.stateManager = new StateManager({ incognito: this.options?.incognito });
-    this.knowledgeTracker = new KnowledgeTracker();
+    this.knowledgeTracker = knowledgeTracker;
+    this.stateManager = new StateManager(experienceTracker, knowledgeTracker);
     this.reporter = new Reporter(config.reporter, this.stateManager);
   }
 
@@ -91,8 +90,7 @@ class Explorer {
     try {
       // Use project root for output directory, not current working directory
       const configParser = ConfigParser.getInstance();
-      const configPath = configParser.getConfigPath();
-      const projectRoot = configPath ? path.dirname(configPath) : process.cwd();
+      const projectRoot = configParser.getProjectRoot();
       (global as any).output_dir = path.join(projectRoot, 'output', 'states');
       (global as any).codecept_dir = projectRoot;
 
@@ -155,8 +153,7 @@ class Explorer {
     };
 
     if (this.config.stepsFile) {
-      const configPath = ConfigParser.getInstance().getConfigPath();
-      const projectRoot = configPath ? path.dirname(configPath) : process.cwd();
+      const projectRoot = ConfigParser.getInstance().getProjectRoot();
       codeceptConfig.include = { I: path.resolve(projectRoot, this.config.stepsFile) };
     }
 
@@ -164,17 +161,9 @@ class Explorer {
   }
 
   public getConfig(): ExplorbotConfig {
-    if (!this.config) {
-      throw new Error('Configuration not loaded. Call run() first.');
-    }
     return this.config;
   }
 
-  public getConfigPath(): string | null {
-    const configParser = ConfigParser.getInstance();
-    return configParser.getConfigPath();
-  }
-
   public getAIProvider(): AIProvider {
     return this.aiProvider;
   }
@@ -183,10 +172,6 @@ class Explorer {
     return this.stateManager;
   }
 
-  public getCurrentUrl(): string {
-    return this.stateManager.getCurrentState()!.url || '?';
-  }
-
   public getKnowledgeTracker(): KnowledgeTracker {
     return this.knowledgeTracker;
   }
@@ -227,10 +212,6 @@ class Explorer {
       return;
     }
 
-    if (!this.config) {
-      await this.initializeContainer();
-    }
-
     await codeceptjs.recorder.start();
     await codeceptjs.container.started(null);
     storeListener();
@@ -359,8 +340,8 @@ class Explorer {
     return this.runWithBrowserRecovery('executeAction', () => this.createAction().execute(code));
   }
 
-  async attemptAction(code: string, originalMessage?: string, experience = true): Promise<boolean> {
-    return this.runWithBrowserRecovery('attemptAction', () => this.createAction().attempt(code, originalMessage, experience));
+  async attemptAction(code: string, originalMessage?: string): Promise<boolean> {
+    return this.runWithBrowserRecovery('attemptAction', () => this.createAction().attempt(code, originalMessage));
   }
 
   getPlaywrightRecorder(): PlaywrightRecorder {
@@ -388,7 +369,7 @@ class Explorer {
       try {
         await action.execute(`I.amOnPage(${serializedUrl})`);
       } catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
+        const msg = browserErrorMessage(err);
         if (!RECOVERABLE_NAVIGATION_ERRORS.test(msg)) throw err;
         tag('warning').log(`Navigation warning (continuing after load): ${msg.split('\n')[0]}`);
         await this.waitForPageReadiness();
@@ -437,7 +418,7 @@ class Explorer {
       return result;
     } catch (error) {
       if (this.isFatalBrowserError(error)) {
-        tag('warning').log(`getEidxInContainer: ${error instanceof Error ? error.message : error}`);
+        tag('warning').log(`getEidxInContainer: ${browserErrorMessage(error)}`);
         await this.recoverFromBrowserError();
       }
       return [];
@@ -452,18 +433,13 @@ class Explorer {
       return await el.first().getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx);
     } catch (error) {
       if (this.isFatalBrowserError(error)) {
-        tag('warning').log(`getEidxByLocator: ${error instanceof Error ? error.message : error}`);
+        tag('warning').log(`getEidxByLocator: ${browserErrorMessage(error)}`);
         await this.recoverFromBrowserError();
       }
       return null;
     }
   }
 
-  async reload() {
-    await this.closeOtherTabs();
-    await this.playwrightHelper.page.reload();
-  }
-
   private resolveBrowserUrl(url?: string): string | null {
     if (!url) return null;
     try {
@@ -508,7 +484,7 @@ class Explorer {
       await this.playwrightHelper.page.reload({ waitUntil: 'domcontentloaded', timeout: 10000 });
       return this.waitForPageReadiness();
     } catch (err) {
-      tag('error').log(`Browser recovery failed: ${err instanceof Error ? err.message : err}`);
+      tag('error').log(`Browser recovery failed: ${browserErrorMessage(err)}`);
       return false;
     }
   }
@@ -552,7 +528,7 @@ class Explorer {
       tag('success').log('Browser restarted');
       return true;
     } catch (err) {
-      tag('error').log(`Browser restart failed: ${err instanceof Error ? err.message : err}`);
+      tag('error').log(`Browser restart failed: ${browserErrorMessage(err)}`);
       return false;
     }
   }
@@ -584,7 +560,7 @@ class Explorer {
       return await page.evaluate(() => window.top !== window.self);
     } catch (error) {
       if (this.isFatalBrowserError(error)) {
-        tag('warning').log(`isInsideIframe: ${error instanceof Error ? error.message : error}`);
+        tag('warning').log(`isInsideIframe: ${browserErrorMessage(error)}`);
         await this.recoverFromBrowserError();
       }
       return false;
@@ -608,10 +584,6 @@ class Explorer {
     this.otherTabs = [];
   }
 
-  setUserResolve(userResolveFn: UserResolveFunction): void {
-    this.userResolveFn = userResolveFn;
-  }
-
   private listenToStateChanged(): void {
     if (!this.playwrightHelper) {
       debugLog('Playwright helper not available for state monitoring');
@@ -727,17 +699,22 @@ class Explorer {
 
     this.watchActiveTestPage();
 
-    codeceptjs.event.dispatcher.emit('test.before', codeceptjsTest);
-    codeceptjs.event.dispatcher.emit('test.start', codeceptjsTest);
-    codeceptjs.event.dispatcher.on('step.passed', (step: any) => stepHandler(step, 'passed'));
-    codeceptjs.event.dispatcher.on('step.failed', (step: any, error: any) => {
+    const onStepPassed = (step: any) => stepHandler(step, 'passed');
+    const onStepFailed = (step: any, error: any) => {
       stepHandler(step, 'failed', error?.message || String(error), error?.stack);
-    });
-    codeceptjs.event.dispatcher.on('test.after', () => {
-      codeceptjs.event.dispatcher.off('step.passed', stepHandler);
-      codeceptjs.event.dispatcher.off('step.failed', stepHandler);
+    };
+    const onTestAfter = () => {
+      codeceptjs.event.dispatcher.off('step.passed', onStepPassed);
+      codeceptjs.event.dispatcher.off('step.failed', onStepFailed);
+      codeceptjs.event.dispatcher.off('test.after', onTestAfter);
       this.unwatchActiveTestPages();
-    });
+    };
+
+    codeceptjs.event.dispatcher.emit('test.before', codeceptjsTest);
+    codeceptjs.event.dispatcher.emit('test.start', codeceptjsTest);
+    codeceptjs.event.dispatcher.on('step.passed', onStepPassed);
+    codeceptjs.event.dispatcher.on('step.failed', onStepFailed);
+    codeceptjs.event.dispatcher.on('test.after', onTestAfter);
 
     return true;
   }
@@ -755,12 +732,8 @@ class Explorer {
     return true;
   }
 
-  async ensureActiveTestPageAvailable(): Promise<boolean> {
-    return this.ensurePageAvailable();
-  }
-
   async handleExecutionError(error: unknown): Promise<BrowserExecutionErrorResult> {
-    const message = error instanceof Error ? error.message : String(error);
+    const message = browserErrorMessage(error);
     tag('error').log(`Browser execution error: ${message}`);
 
     if (error instanceof Error && error.name === 'AbortError') {
@@ -868,33 +841,13 @@ class Explorer {
     this.observedTestPages.clear();
   }
 
-  async hasPlaywrightLocator(locatorFn: (page: any) => any, opts: { multiple?: boolean; contents?: boolean; success?: (locator: any) => Promise<void> | void } = {}): Promise<boolean> {
-    try {
-      const pwLocator = locatorFn(this.playwrightHelper.page);
-      const count = await pwLocator.count();
-      if (opts.multiple ? count === 0 : count !== 1) return false;
-      if (opts.contents) {
-        const html = await pwLocator.first().innerHTML();
-        if (!html?.trim()) return false;
-      }
-      if (opts.success) await opts.success(pwLocator);
-      return true;
-    } catch (error) {
-      if (this.isFatalBrowserError(error)) {
-        tag('warning').log(`hasPlaywrightLocator: ${error instanceof Error ? error.message : error}`);
-        await this.recoverFromBrowserError();
-      }
-      return false;
-    }
-  }
-
   async playwrightLocatorCount(locatorFn: (page: any) => any): Promise<number> {
     try {
       const pwLocator = locatorFn(this.playwrightHelper.page);
       return await pwLocator.count();
     } catch (error) {
       if (this.isFatalBrowserError(error)) {
-        tag('warning').log(`playwrightLocatorCount: ${error instanceof Error ? error.message : error}`);
+        tag('warning').log(`playwrightLocatorCount: ${browserErrorMessage(error)}`);
         await this.recoverFromBrowserError();
       }
       throw error;
@@ -916,7 +869,7 @@ class Explorer {
 
       this.stateManager.updateStateFromBasic(newUrl, newTitle, 'navigation');
 
-      await new Promise((resolve) => setTimeout(resolve, 500));
+      await sleep(500);
     });
   }
 

package/src/index.ts

@@ -0,0 +1,9 @@
+export { ExplorBot } from './explorbot.ts';
+export type { ExplorBotOptions, UserResolveFunction } from './explorbot.ts';
+
+export { Plan, Test, TestResult, TestStatus } from './test-plan.ts';
+export type { Note, StepData, TestResultType, TestStatusType } from './test-plan.ts';
+
+export type { Link, StateTransition, WebPageState } from './state-manager.ts';
+
+export type { AIConfig, ExplorbotConfig, PlaywrightConfig } from './config.ts';

package/src/knowledge-tracker.ts

@@ -1,14 +1,19 @@
-import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
-import { dirname, join, resolve } from 'node:path';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import dedent from 'dedent';
 import matter from 'gray-matter';
 import { ActionResult } from './action-result.js';
 import { ConfigParser } from './config.js';
 import { getCliName } from './utils/cli-name.ts';
-import { createDebug } from './utils/logger.js';
+import { createDebug, pluralize, tag } from './utils/logger.js';
+import { loadMarkdownFiles } from './utils/markdown-files.js';
+import { mdq } from './utils/markdown-query.js';
+import { isSecretName, registerSecret } from './utils/secrets.js';
+import { slugify } from './utils/strings.js';
 
 const debugLog = createDebug('explorbot:knowledge-tracker');
 
-interface Knowledge {
+export interface Knowledge {
   filePath: string;
   url: string;
   content: string;
@@ -23,14 +28,7 @@ export class KnowledgeTracker {
   constructor() {
     const configParser = ConfigParser.getInstance();
     const config = configParser.getConfig();
-    const configPath = configParser.getConfigPath();
-
-    if (configPath) {
-      const projectRoot = dirname(configPath);
-      this.knowledgeDir = join(projectRoot, config.dirs?.knowledge || 'knowledge');
-    } else {
-      this.knowledgeDir = config.dirs?.knowledge || 'knowledge';
-    }
+    this.knowledgeDir = configParser.resolveProjectDir(config.dirs?.knowledge || 'knowledge');
 
     if (!existsSync(this.knowledgeDir)) {
       mkdirSync(this.knowledgeDir, { recursive: true });
@@ -42,29 +40,13 @@ export class KnowledgeTracker {
 
     this.knowledgeFiles = [];
 
-    if (!existsSync(this.knowledgeDir)) {
-      return;
-    }
-
-    const files = readdirSync(this.knowledgeDir)
-      .filter((file) => file.endsWith('.md'))
-      .map((file) => join(this.knowledgeDir, file));
-
-    for (const filePath of files) {
-      try {
-        const fileContent = readFileSync(filePath, 'utf8');
-        const parsed = matter(fileContent);
-        const urlPattern = parsed.data.url || parsed.data.path || '*';
-
-        this.knowledgeFiles.push({
-          filePath,
-          url: urlPattern,
-          content: this.interpolateVars(parsed.content),
-          ...parsed.data,
-        });
-      } catch (error) {
-        // Skip invalid files
-      }
+    for (const entry of loadMarkdownFiles(this.knowledgeDir, { recursive: true })) {
+      this.knowledgeFiles.push({
+        filePath: entry.filePath,
+        url: entry.data.url || entry.data.path || '*',
+        content: this.interpolateVars(entry.content),
+        ...entry.data,
+      });
     }
 
     this.isLoaded = true;
@@ -78,25 +60,40 @@ export class KnowledgeTracker {
     });
   }
 
+  renderRelevantKnowledge(state: ActionResult): string {
+    const knowledgeFiles = this.getRelevantKnowledge(state);
+    if (knowledgeFiles.length === 0) return '';
+
+    const knowledgeContent = knowledgeFiles
+      .map((k) => k.content)
+      .filter((k) => !!k)
+      .join('\n\n');
+
+    tag('operation').log(`Found ${knowledgeFiles.length} relevant knowledge ${pluralize(knowledgeFiles.length, 'file')}`);
+    return dedent`
+      <knowledge>
+      Here is relevant knowledge for this page:
+
+      ${knowledgeContent}
+      </knowledge>
+    `;
+  }
+
   addKnowledge(urlPattern: string, description: string): { filename: string; filePath: string; isNewFile: boolean } {
     const configParser = ConfigParser.getInstance();
-    const config = configParser.getConfig();
     const configPath = configParser.getConfigPath();
 
     if (!configPath) {
       throw new Error(`No explorbot configuration found. Please run "${getCliName()} init" first.`);
     }
 
-    const projectRoot = dirname(configPath);
-    const knowledgeDir = join(projectRoot, config.dirs?.knowledge || 'knowledge');
-
-    if (!existsSync(knowledgeDir)) {
-      mkdirSync(knowledgeDir, { recursive: true });
+    if (!existsSync(this.knowledgeDir)) {
+      mkdirSync(this.knowledgeDir, { recursive: true });
     }
 
     const normalizedUrl = this.normalizeUrl(urlPattern);
     const filename = this.generateFilename(normalizedUrl);
-    const filePath = join(knowledgeDir, filename);
+    const filePath = join(this.knowledgeDir, filename);
 
     const isNewFile = !existsSync(filePath);
 
@@ -127,6 +124,8 @@ export class KnowledgeTracker {
       writeFileSync(filePath, fileContent, 'utf8');
     }
 
+    this.isLoaded = false;
+
     return { filename, filePath, isNewFile };
   }
 
@@ -138,9 +137,14 @@ export class KnowledgeTracker {
       const namespace = expr.slice(0, dotIndex);
       const key = expr.slice(dotIndex + 1);
 
-      if (namespace === 'env') return process.env[key] ?? '';
+      if (namespace === 'env') {
+        const value = process.env[key] ?? '';
+        if (isSecretName(key)) registerSecret(value);
+        return value;
+      }
 
       if (namespace === 'config') {
+        if (isSecretName(key)) return '';
         const config = ConfigParser.getInstance().getConfig();
         const value = key.split('.').reduce((obj: any, k) => obj?.[k], config);
         if (value !== undefined && typeof value !== 'object') return String(value);
@@ -152,12 +156,7 @@ export class KnowledgeTracker {
   }
 
   private generateFilename(url: string): string {
-    let filename = url
-      .replace(/https?:\/\//g, '')
-      .replace(/[^a-zA-Z0-9_]/g, '_')
-      .replace(/_+/g, '_')
-      .replace(/^_|_$/g, '')
-      .toLowerCase();
+    let filename = slugify(url.replace(/https?:\/\//g, ''));
 
     if (!filename || filename === '*') {
       filename = 'general';
@@ -188,7 +187,7 @@ export class KnowledgeTracker {
 
     return this.knowledgeFiles.map((knowledge) => {
       const content = knowledge.content.trim();
-      const firstLine = content.split('\n')[0]?.trim() || '';
+      const firstLine = mdq(content).meta()[0]?.text.split('\n')[0]?.trim() || '';
       return {
         url: knowledge.url,
         firstLine,

package/src/observability.ts

@@ -1,4 +1,5 @@
 import { type Span, trace } from '@opentelemetry/api';
+import { redactSecrets } from './utils/secrets.js';
 
 type TelemetryMetadata = Record<string, unknown>;
 
@@ -84,7 +85,7 @@ function buildRootSpanAttributes(name: string, metadata: TelemetryMetadata): Rec
     attributes['langfuse.trace.tags'] = metadata.tags as string[];
   }
   if (metadata.input !== undefined) {
-    attributes['langfuse.trace.input'] = JSON.stringify(metadata.input);
+    attributes['langfuse.trace.input'] = redactSecrets(JSON.stringify(metadata.input));
   }
 
   return attributes;