explorbot 0.1.17 → 0.1.18

package/dist/src/explorer.js

@@ -189,7 +189,20 @@ class Explorer {
         }
         await this.connectOrLaunchBrowser();
         const hasSession = this.options?.session && existsSync(this.options.session);
-        const contextOptions = hasSession ? { storageState: this.options.session } : undefined;
+        const helperOptions = this.playwrightHelper.options || {};
+        // CodeceptJS skips _createContextPage when sessions/storageState are involved, so we
+        // build contextOptions ourselves. Most keys share a name with Playwright's
+        // BrowserContextOptions and are copied as-is; `emulate` must be flattened, `basicAuth`
+        // renamed to `httpCredentials`, and `storageState` comes from the --session flag.
+        const contextOptions = {
+            ...helperOptions,
+        };
+        if (helperOptions.emulate)
+            Object.assign(contextOptions, helperOptions.emulate);
+        if (helperOptions.basicAuth)
+            contextOptions.httpCredentials = helperOptions.basicAuth;
+        if (hasSession)
+            contextOptions.storageState = this.options.session;
         await this.playwrightHelper._createContextPage(contextOptions);
         await this.playwrightRecorder.start(this.playwrightHelper.browserContext);
         this.setupXhrCapture();

package/dist/src/state-manager.js

@@ -416,6 +416,9 @@ export class StateManager {
     }
 }
 export function normalizeUrl(url) {
+    if (url.startsWith('/')) {
+        return url.replace(/^\/+/, '').replace(/\/+$/g, '');
+    }
     try {
         const parsed = new URL(url, 'http://localhost');
         const path = parsed.pathname.replace(/^\/+|\/+$/g, '');

package/dist/src/utils/url-matcher.js

@@ -90,11 +90,13 @@ export function matchesUrl(pattern, path) {
     }
 }
 export function extractStatePath(url) {
-    if (url.startsWith('/'))
-        return url;
+    if (url.startsWith('/')) {
+        return `/${url.replace(/^\/+/, '')}`;
+    }
     try {
         const urlObj = new URL(url);
-        return `${urlObj.pathname}${urlObj.search}${urlObj.hash}`;
+        const normalizedPathname = `/${urlObj.pathname.replace(/^\/+/, '')}`;
+        return `${normalizedPathname}${urlObj.search}${urlObj.hash}`;
     }
     catch {
         return url;

package/dist/src/utils/web-element.js

@@ -109,7 +109,8 @@ export class WebElement {
         return WebElement.fromPlaywrightLocator(page.locator(`[${EXPLORBOT_ATTRS.eidx}="${eidx}"]`));
     }
     static async fromEidxList(page, eidxList) {
-        if (eidxList.length === 0)
+        const validEidxList = eidxList.filter((eidx) => /^e\d+$/i.test(eidx));
+        if (validEidxList.length === 0)
             return [];
         const rawList = await page.evaluate(([list, extractFnStr, config]) => {
             const extract = new Function(`return ${extractFnStr}`)();
@@ -123,7 +124,7 @@ export class WebElement {
                     results.push(data);
             }
             return results;
-        }, [eidxList, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG]);
+        }, [validEidxList, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG]);
         return rawList.map((d) => WebElement.fromRawData(d));
     }
     static async findByXPath(html, xpath) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.1.17",
+  "version": "0.1.18",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",
@@ -20,6 +20,9 @@
     "src/**/*.tsx",
     "bin/**/*.ts",
     "boat/api-tester/src/**/*.ts",
+    "boat/doc-collector/src/**/*.ts",
+    "boat/doc-collector/bin/**/*.ts",
+    "boat/doc-collector/package.json",
     "rules/",
     "assets/sample-files/"
   ],

package/src/ai/researcher/coordinates.ts

@@ -198,7 +198,7 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
       const eidxWithoutCoords: string[] = [];
       for (const section of sections) {
         for (const el of section.elements) {
-          if (el.eidx && !el.coordinates) eidxWithoutCoords.push(el.eidx);
+          if (el.eidx && /^e\d+$/i.test(el.eidx) && !el.coordinates) eidxWithoutCoords.push(el.eidx);
         }
       }
       if (eidxWithoutCoords.length === 0) return;

package/src/ai/researcher/parser.ts

@@ -64,6 +64,9 @@ export function mapRowToElement(row: Record<string, string>): ResearchElement |
 
   let eidxRaw = (colMap.eidx || '').trim();
   if (eidxRaw && /^\d+$/.test(eidxRaw)) eidxRaw = `e${eidxRaw}`;
+  if (eidxRaw && !/^e\d+$/i.test(eidxRaw)) {
+    eidxRaw = '';
+  }
 
   const aria = parseAriaLocator(colMap.aria || '-');
 

package/src/ai/researcher.ts

@@ -121,7 +121,8 @@ export class Researcher extends ResearcherBase implements Agent {
 
     const sessionName = `researcher: ${state.url}`;
     return Observability.run(sessionName, { tags: ['researcher'], sessionId: stateHash }, async () => {
-      tag('info').log(`Researching ${state.url} to understand the context...`);
+      const displayUrl = state.fullUrl || state.url;
+      tag('info').log(`Researching ${displayUrl} to understand the context...`);
       setActivity(`${this.emoji} Researching...`, 'action');
 
       await this.ensureNavigated(state.url, screenshot && this.provider.hasVision());

package/src/config.ts

@@ -266,6 +266,7 @@ export class ConfigParser {
   private static instance: ConfigParser;
   private config: ExplorbotConfig | null = null;
   private configPath: string | null = null;
+  private runtimeBaseUrlOverride: string | null = null;
 
   private constructor() {}
 
@@ -285,8 +286,9 @@ export class ConfigParser {
   public async loadConfig(options?: {
     config?: string;
     path?: string;
+    baseUrl?: string;
   }): Promise<ExplorbotConfig> {
-    if (this.config && !options?.config && !options?.path) {
+    if (this.config && !options?.config && !options?.path && this.runtimeBaseUrlOverride === (options?.baseUrl || null)) {
       return this.config;
     }
 
@@ -317,7 +319,8 @@ export class ConfigParser {
         throw new Error('Configuration file is empty or invalid');
       }
 
-      this.config = this.resolveConfig(loadedConfig as ExplorbotConfig);
+      this.config = this.resolveConfig(loadedConfig as ExplorbotConfig, options);
+      this.runtimeBaseUrlOverride = options?.baseUrl || null;
       this.configPath = resolvedPath;
 
       log(`Configuration loaded from: ${resolvedPath}`);
@@ -372,6 +375,7 @@ export class ConfigParser {
     if (ConfigParser.instance) {
       ConfigParser.instance.config = null;
       ConfigParser.instance.configPath = null;
+      ConfigParser.instance.runtimeBaseUrlOverride = null;
     }
   }
 
@@ -455,11 +459,17 @@ export class ConfigParser {
     }
   }
 
-  private resolveConfig(config: ExplorbotConfig): ExplorbotConfig {
+  private resolveConfig(config: ExplorbotConfig, options?: { baseUrl?: string }): ExplorbotConfig {
     if (config.web?.url && !config.playwright?.url) {
       config.playwright = config.playwright || { browser: 'chromium', url: '' };
       config.playwright.url = config.web.url;
     }
+
+    if (options?.baseUrl) {
+      config.playwright = config.playwright || { browser: 'chromium', url: '' };
+      config.playwright.url = options.baseUrl;
+    }
+
     return config;
   }
 

package/src/explorbot.ts

@@ -34,6 +34,7 @@ import { sanitizeFilename } from './utils/strings.ts';
 
 export interface ExplorBotOptions {
   from?: string;
+  baseUrl?: string;
   verbose?: boolean;
   config?: string;
   path?: string;

package/src/explorer.ts

@@ -8,6 +8,7 @@ import { createTest } from 'codeceptjs/lib/mocha/test';
 import { ActionResult } from './action-result.ts';
 import Action from './action.js';
 import { AIProvider } from './ai/provider.js';
+import type { BrowserContextOptions } from 'playwright';
 import { visuallyAnnotateContainers } from './ai/researcher/coordinates.ts';
 import { RequestStore } from './api/request-store.ts';
 import { XhrCapture } from './api/xhr-capture.ts';
@@ -238,7 +239,17 @@ class Explorer {
     }
     await this.connectOrLaunchBrowser();
     const hasSession = this.options?.session && existsSync(this.options.session);
-    const contextOptions = hasSession ? { storageState: this.options!.session } : undefined;
+    const helperOptions = this.playwrightHelper.options || {};
+    // CodeceptJS skips _createContextPage when sessions/storageState are involved, so we
+    // build contextOptions ourselves. Most keys share a name with Playwright's
+    // BrowserContextOptions and are copied as-is; `emulate` must be flattened, `basicAuth`
+    // renamed to `httpCredentials`, and `storageState` comes from the --session flag.
+    const contextOptions: BrowserContextOptions = {
+      ...helperOptions,
+    };
+    if (helperOptions.emulate) Object.assign(contextOptions, helperOptions.emulate);
+    if (helperOptions.basicAuth) contextOptions.httpCredentials = helperOptions.basicAuth;
+    if (hasSession) contextOptions.storageState = this.options!.session;
     await this.playwrightHelper._createContextPage(contextOptions);
     await this.playwrightRecorder.start(this.playwrightHelper.browserContext);
     this.setupXhrCapture();

package/src/state-manager.ts

@@ -547,6 +547,10 @@ export class StateManager {
 }
 
 export function normalizeUrl(url: string): string {
+  if (url.startsWith('/')) {
+    return url.replace(/^\/+/, '').replace(/\/+$/g, '');
+  }
+
   try {
     const parsed = new URL(url, 'http://localhost');
     const path = parsed.pathname.replace(/^\/+|\/+$/g, '');

package/src/utils/url-matcher.ts

@@ -82,10 +82,13 @@ export function matchesUrl(pattern: string, path: string): boolean {
 }
 
 export function extractStatePath(url: string): string {
-  if (url.startsWith('/')) return url;
+  if (url.startsWith('/')) {
+    return `/${url.replace(/^\/+/, '')}`;
+  }
   try {
     const urlObj = new URL(url);
-    return `${urlObj.pathname}${urlObj.search}${urlObj.hash}`;
+    const normalizedPathname = `/${urlObj.pathname.replace(/^\/+/, '')}`;
+    return `${normalizedPathname}${urlObj.search}${urlObj.hash}`;
   } catch {
     return url;
   }

package/src/utils/web-element.ts

@@ -122,7 +122,8 @@ export class WebElement {
   }
 
   static async fromEidxList(page: any, eidxList: string[]): Promise<WebElement[]> {
-    if (eidxList.length === 0) return [];
+    const validEidxList = eidxList.filter((eidx) => /^e\d+$/i.test(eidx));
+    if (validEidxList.length === 0) return [];
 
     const rawList: RawElementData[] = await page.evaluate(
       ([list, extractFnStr, config]: [string[], string, ElementExtractionConfig]) => {
@@ -136,7 +137,7 @@ export class WebElement {
         }
         return results;
       },
-      [eidxList, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG] as [string[], string, ElementExtractionConfig]
+      [validEidxList, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG] as [string[], string, ElementExtractionConfig]
     );
 
     return rawList.map((d) => WebElement.fromRawData(d));