explorbot 0.1.32-beta.1 → 0.2.1

package/src/utils/markdown-files.ts

@@ -0,0 +1,30 @@
+import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import matter from 'gray-matter';
+
+export function loadMarkdownFiles(dir: string, options: { recursive?: boolean; onError?: (filePath: string, error: unknown) => void } = {}): MarkdownFile[] {
+  if (!existsSync(dir)) return [];
+
+  const results: MarkdownFile[] = [];
+  const files = readdirSync(dir, { recursive: options.recursive === true })
+    .filter((file): file is string => typeof file === 'string' && file.endsWith('.md'))
+    .map((file) => join(dir, file));
+
+  for (const filePath of files) {
+    try {
+      const parsed = matter(readFileSync(filePath, 'utf8'));
+      results.push({ filePath, content: parsed.content, data: parsed.data, mtime: statSync(filePath).mtime });
+    } catch (error) {
+      options.onError?.(filePath, error);
+    }
+  }
+
+  return results;
+}
+
+export interface MarkdownFile {
+  filePath: string;
+  content: string;
+  data: Record<string, any>;
+  mtime: Date;
+}

package/src/utils/markdown-query.ts

@@ -408,6 +408,10 @@ export class MarkdownQuery {
   }
 
   replace(content: string): string {
+    return this.replaceEach(() => content);
+  }
+
+  replaceEach(replacer: (match: MarkdownQuery, index: number) => string): string {
     const sorted = [...this.matches].sort((a, b) => a.start - b.start);
 
     const kept: MatchedRange[] = [];
@@ -418,10 +422,11 @@ export class MarkdownQuery {
       lastEnd = range.start + range.length;
     }
 
+    const replacements = kept.map((range, index) => replacer(new MarkdownQuery(this.source, [range]), index));
     let result = this.source;
     for (let i = kept.length - 1; i >= 0; i--) {
       const range = kept[i];
-      result = result.slice(0, range.start) + content + result.slice(range.start + range.length);
+      result = result.slice(0, range.start) + replacements[i] + result.slice(range.start + range.length);
     }
 
     return result;
@@ -459,6 +464,15 @@ export class MarkdownQuery {
   each(): MarkdownQuery[] {
     return this.matches.map((m) => new MarkdownQuery(this.source, [m]));
   }
+
+  meta(): Array<{ type: string; depth: number | null; text: string }> {
+    return this.matches.map((range) => {
+      const token = range.token as any;
+      let depth: number | null = null;
+      if (token.type === 'heading') depth = token.depth;
+      return { type: token.type, depth, text: getTokenText(range.token) };
+    });
+  }
 }
 
 export function mdq(source: string): MarkdownQuery {

package/src/utils/page-readiness.ts

@@ -49,7 +49,7 @@ function waitForPageBodyContent(page: any, timeout: number): Promise<void> {
     .catch(() => {});
 }
 
-function sleep(ms: number): Promise<void> {
+export function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 

package/src/utils/secrets.ts

@@ -1,4 +1,5 @@
 const MIN_SECRET_LENGTH = 4;
+const SECRET_NAME_TOKENS = ['password', 'passwd', 'secret', 'token', 'apikey', 'api_key', 'credential', 'private_key', 'privatekey', 'access_key', 'jwt', 'auth', 'bearer', 'authorization', 'cert', 'certificate'];
 const secretValues = new Set<string>();
 
 export function registerSecret(value: string): void {
@@ -10,7 +11,7 @@ export function registerSecret(value: string): void {
 export function redactSecrets(text: string): string {
   if (!text) return text;
   let result = text;
-  for (const value of secretValues) {
+  for (const value of [...secretValues].sort((a, b) => b.length - a.length)) {
     if (!result.includes(value)) continue;
     result = result.split(value).join('***REDACTED***');
   }
@@ -21,8 +22,6 @@ export function clearRegisteredSecrets(): void {
   secretValues.clear();
 }
 
-const SECRET_NAME_TOKENS = ['password', 'passwd', 'secret', 'token', 'apikey', 'api_key', 'credential', 'private_key', 'privatekey', 'access_key'];
-
 export function isSecretName(name: string): boolean {
   const lower = name.toLowerCase();
   return SECRET_NAME_TOKENS.some((token) => lower.includes(token));

package/src/utils/strings.ts

@@ -6,6 +6,18 @@ export function truncateJson(input: any): string {
   return str.length <= 80 ? str : `${str.slice(0, 77)}...`;
 }
 
+export function slugify(text: string): string {
+  return text
+    .replace(/[^a-zA-Z0-9_]/g, '_')
+    .replace(/_+/g, '_')
+    .replace(/^_|_$/g, '')
+    .toLowerCase();
+}
+
+export function normalizeInlineText(text: string): string {
+  return text.normalize('NFKC').replace(/\s+/g, ' ').trim();
+}
+
 export function sanitizeFilename(name: string): string {
   return name
     .toLowerCase()

package/src/utils/test-files.ts

@@ -76,7 +76,7 @@ export async function dryRunTestFile(filePath: string): Promise<void> {
     },
   };
 
-  (global as any).output_dir = path.join(projectRoot, 'output', 'states');
+  (global as any).output_dir = ConfigParser.getInstance().getStatesDir();
   (global as any).codecept_dir = projectRoot;
 
   codeceptjs.container.create(codeceptConfig, {});

package/src/utils/web-annotate.ts

@@ -0,0 +1,64 @@
+import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from './html.ts';
+import { createDebug } from './logger.js';
+import { WebElement } from './web-element.ts';
+
+const debugLog = createDebug('explorbot:web-annotate');
+
+const REF_LINE_PATTERN = /^(\s*)-\s+(\w+)\s*(?:"([^"]*)")?.*?\[ref=(e\d+)\]/;
+
+const ANNOTATABLE_ROLES = new Set(['button', 'link', 'textbox', 'searchbox', 'checkbox', 'radio', 'switch', 'combobox', 'tab', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'slider', 'spinbutton', 'treeitem']);
+
+function parseAriaRefs(ariaSnapshot: string): Array<{ role: string; name: string; ref: string }> {
+  const entries: Array<{ role: string; name: string; ref: string }> = [];
+  for (const line of ariaSnapshot.split('\n')) {
+    const match = line.match(REF_LINE_PATTERN);
+    if (!match) continue;
+    if (!ANNOTATABLE_ROLES.has(match[2])) continue;
+    entries.push({ role: match[2], name: match[3] || '', ref: match[4] });
+  }
+  return entries;
+}
+
+export async function annotatePageElements(page: any): Promise<{ ariaSnapshot: string; elements: WebElement[] }> {
+  const ariaSnapshot: string = await page.locator('body').ariaSnapshot({ mode: 'ai' });
+  const refEntries = parseAriaRefs(ariaSnapshot);
+
+  const byRole = new Map<string, Array<{ name: string; ref: string }>>();
+  for (const { role, name, ref } of refEntries) {
+    let list = byRole.get(role);
+    if (!list) {
+      list = [];
+      byRole.set(role, list);
+    }
+    list.push({ name, ref });
+  }
+
+  const elements: WebElement[] = [];
+  for (const [role, entries] of byRole) {
+    try {
+      const rawList = await page.getByRole(role).evaluateAll(
+        (domElements: Element[], [data, extractFnStr, config]: [Array<{ name: string; ref: string }>, string, typeof ELEMENT_EXTRACTION_CONFIG]) => {
+          const extract = new Function(`return ${extractFnStr}`)() as (el: Element) => any;
+          const results: any[] = [];
+          let ariaIdx = 0;
+          for (const el of domElements) {
+            if (ariaIdx >= data.length) break;
+            el.setAttribute(config.attrs.eidx, data[ariaIdx].ref);
+            const elData = extract(el, config);
+            if (elData) results.push(elData);
+            ariaIdx++;
+          }
+          return results;
+        },
+        [entries, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG]
+      );
+      for (const raw of rawList) {
+        elements.push(WebElement.fromRawData(raw, role));
+      }
+    } catch {
+      debugLog(`Failed to annotate role=${role}`);
+    }
+  }
+
+  return { ariaSnapshot, elements };
+}

package/src/utils/web-eidx.ts

@@ -0,0 +1,21 @@
+import { ELEMENT_EXTRACTION_CONFIG } from './html.ts';
+
+export async function eidxInContainer(page: any, containerCss: string | null): Promise<string[]> {
+  const selector = containerCss ? `${containerCss} [${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]` : `[${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]`;
+  const elements = await page.locator(selector).all();
+  const result: string[] = [];
+  for (const el of elements) {
+    const attr = await el.getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx).catch(() => null);
+    if (attr) result.push(attr);
+  }
+  return result;
+}
+
+export async function eidxByLocator(page: any, locator: string, container?: string | null): Promise<string | null> {
+  const base = container ? page.locator(container) : page;
+  const el = locator.startsWith('//') ? base.locator(`xpath=${locator}`) : base.locator(locator);
+  return el
+    .first()
+    .getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx)
+    .catch(() => null);
+}

package/src/utils/web-element.ts

@@ -47,6 +47,10 @@ export class WebElement {
     return this.attrs[EXPLORBOT_ATTRS.eidx] || this.attrs.eidx || null;
   }
 
+  ourAttr(name: keyof typeof EXPLORBOT_ATTRS): string | undefined {
+    return this.attrs[EXPLORBOT_ATTRS[name]];
+  }
+
   get isNavigationLink(): boolean {
     if (this.tag !== 'a') return false;
     const href = this.attrs.href || '';
@@ -143,6 +147,36 @@ export class WebElement {
     return rawList.map((d) => WebElement.fromRawData(d));
   }
 
+  static async commonAncestor(page: any, eidxList: string[]): Promise<WebElement | null> {
+    const validEidxList = eidxList.filter((eidx) => /^e\d+$/i.test(eidx));
+    if (validEidxList.length < 2) return null;
+
+    const raw: RawElementData | null = await page.evaluate(
+      ([list, extractFnStr, config]: [string[], string, ElementExtractionConfig]) => {
+        const extract = new Function(`return ${extractFnStr}`)() as (el: Element, cfg: ElementExtractionConfig) => any;
+        const els: Element[] = [];
+        for (const eidx of list) {
+          const el = document.querySelector(`[${config.attrs.eidx}="${eidx}"]`);
+          if (el && !els.includes(el)) els.push(el);
+        }
+        if (els.length < 2) return null;
+
+        const containsAll = (node: Element) => els.every((el) => node.contains(el));
+        let ancestor: Element | null = els[0].parentElement;
+        while (ancestor && !containsAll(ancestor)) ancestor = ancestor.parentElement;
+        if (!ancestor) return null;
+
+        const tag = ancestor.tagName.toLowerCase();
+        if (tag === 'body' || tag === 'html') return null;
+        return extract(ancestor, config);
+      },
+      [validEidxList, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG] as [string[], string, ElementExtractionConfig]
+    );
+
+    if (!raw) return null;
+    return WebElement.fromRawData(raw);
+  }
+
   static async findByXPath(html: string, xpath: string): Promise<{ totalFound: number; elements: WebElement[]; error?: string }> {
     const result = await evaluateXPath(html, xpath);
     if (result.error) return { totalFound: 0, elements: [], error: result.error };

package/src/utils/web-sandbox.ts

@@ -0,0 +1,43 @@
+import { faker } from '@faker-js/faker';
+import { hopeThat, retryTo, tryTo, within } from 'codeceptjs/lib/effects';
+import step from 'codeceptjs/steps';
+
+const SHADOWED_GLOBALS = ['process', 'global', 'globalThis', 'fetch', 'Bun', 'require', 'module', 'exports'];
+const ALLOWED_COMMAND_HEADS = ['I.', 'page.', 'await page.', 'await I.'];
+const PLAYWRIGHT_COMMAND_HEADS = ['page.', 'await page.'];
+const PLAYWRIGHT_ARG_NAMES = ['page'];
+const CODECEPT_ARG_NAMES = ['I', 'tryTo', 'retryTo', 'within', 'hopeThat', 'step', 'faker'];
+
+export function sanitizeCodeBlock(code: string): string {
+  return code
+    .split('\n')
+    .map((line) => line.trim())
+    .filter((line) => ALLOWED_COMMAND_HEADS.some((head) => line.startsWith(head)))
+    .join('\n');
+}
+
+export function hasPlaywrightCommands(code: string): boolean {
+  return code.split('\n').some((line) => {
+    const trimmed = line.trim();
+    return PLAYWRIGHT_COMMAND_HEADS.some((head) => trimmed.startsWith(head));
+  });
+}
+
+export function playwrightSandbox(page: any, code: string): Promise<any> {
+  const run = createSandbox(PLAYWRIGHT_ARG_NAMES, `return (async () => { ${code} })()`);
+  return run(page);
+}
+
+export function codeceptJSSandbox(actor: any, codeOrFn: string | ((...args: any[]) => void)): void {
+  if (typeof codeOrFn === 'function') {
+    codeOrFn(actor, tryTo, retryTo, within, hopeThat, step, faker);
+    return;
+  }
+  const run = createSandbox(CODECEPT_ARG_NAMES, codeOrFn);
+  run(actor, tryTo, retryTo, within, hopeThat, step, faker);
+}
+
+function createSandbox(argNames: string[], body: string): (...args: any[]) => any {
+  const fn = new Function(...argNames, ...SHADOWED_GLOBALS, `'use strict';\n${body}`);
+  return (...args: any[]) => fn(...args);
+}

package/dist/boat/api-tester/example/apibot.config.js

@@ -1,30 +0,0 @@
-import { createOpenRouter } from '@openrouter/ai-sdk-provider';
-const openrouter = createOpenRouter({
-    apiKey: process.env.OPENROUTER_API_KEY,
-});
-export default {
-    ai: {
-        model: openrouter('mistralai/devstral-small:nitro') //   openrouter('x-ai/grok-4-fast') // openrouter('moonshotai/kimi-k2.5:nitro'), // openrouter('x-ai/grok-4-fast') // minimax/minimax-m2.5:nitro
-    },
-    api: {
-        baseEndpoint: 'http://localhost:3000/api/v2/amr_testing',
-        headers: {
-            'Authorization': `Bearer ${process.env.TESTOMATIO_API_KEY}`,
-            'Content-Type': 'application/json',
-        },
-        spec: [
-            'http://127.0.0.1:3000/api/v2/openapi'
-        ],
-        // bootstrap: async ({ headers, baseEndpoint }) => {
-        //   // Run before tests — e.g. obtain auth token
-        //   // Return headers to merge: { Authorization: 'Bearer ...' }
-        // },
-        // teardown: async ({ headers, baseEndpoint }) => {
-        //   // Run after tests — e.g. cleanup test data
-        // },
-    },
-    dirs: {
-        output: 'output',
-        knowledge: 'knowledge',
-    },
-};