explorbot 0.1.32-beta.1 → 0.2.0

package/src/utils/cache.ts

@@ -0,0 +1,40 @@
+export class TTLCache<T> {
+  private store = new Map<string, CacheEntry<T>>();
+  private ttlMs: number;
+
+  constructor(ttlMs = Number.POSITIVE_INFINITY) {
+    this.ttlMs = ttlMs;
+  }
+
+  get(key: string): T | undefined {
+    const entry = this.store.get(key);
+    if (!entry) return undefined;
+    if (Date.now() - entry.at > this.ttlMs) {
+      this.store.delete(key);
+      return undefined;
+    }
+    return entry.value;
+  }
+
+  set(key: string, value: T): void {
+    this.store.set(key, { value, at: Date.now() });
+  }
+
+  async fetch(key: string, compute: () => Promise<T>): Promise<T> {
+    const cached = this.get(key);
+    if (cached !== undefined) return cached;
+    const value = await compute();
+    this.set(key, value);
+    return value;
+  }
+
+  delete(key: string): void {
+    this.store.delete(key);
+  }
+
+  clear(): void {
+    this.store.clear();
+  }
+}
+
+type CacheEntry<T> = { value: T; at: number };

package/src/utils/context-formatter.ts

@@ -29,6 +29,15 @@ function indent(text: string): string {
     .join('\n');
 }
 
+export function formatHeadings(headings: { h1?: string; h2?: string; h3?: string; h4?: string }): string[] {
+  const lines: string[] = [];
+  if (headings.h1) lines.push(`H1: ${headings.h1}`);
+  if (headings.h2) lines.push(`H2: ${headings.h2}`);
+  if (headings.h3) lines.push(`H3: ${headings.h3}`);
+  if (headings.h4) lines.push(`H4: ${headings.h4}`);
+  return lines;
+}
+
 export type ContextMode = 'attached' | 'compact' | 'full';
 
 export function formatContextSummary(data: ContextData, mode: ContextMode = 'compact'): string {
@@ -51,11 +60,7 @@ function formatContextAttached(data: ContextData): string {
   }
   lines.push('');
 
-  const headingLines: string[] = [];
-  if (data.headings.h1) headingLines.push(`H1: ${data.headings.h1}`);
-  if (data.headings.h2) headingLines.push(`H2: ${data.headings.h2}`);
-  if (data.headings.h3) headingLines.push(`H3: ${data.headings.h3}`);
-  if (data.headings.h4) headingLines.push(`H4: ${data.headings.h4}`);
+  const headingLines = formatHeadings(data.headings);
 
   if (headingLines.length > 0) {
     lines.push(section('HEADINGS') + chalk.gray(' (Auto-attached)'));
@@ -116,11 +121,7 @@ function formatContextCompact(data: ContextData): string {
   }
   lines.push('');
 
-  const headingLines: string[] = [];
-  if (data.headings.h1) headingLines.push(`H1: ${data.headings.h1}`);
-  if (data.headings.h2) headingLines.push(`H2: ${data.headings.h2}`);
-  if (data.headings.h3) headingLines.push(`H3: ${data.headings.h3}`);
-  if (data.headings.h4) headingLines.push(`H4: ${data.headings.h4}`);
+  const headingLines = formatHeadings(data.headings);
 
   if (headingLines.length > 0) {
     lines.push(section('HEADINGS') + chalk.gray(' (Auto-attached)'));
@@ -177,11 +178,7 @@ function formatContextFull(data: ContextData): string {
   }
   lines.push('');
 
-  const headingLines: string[] = [];
-  if (data.headings.h1) headingLines.push(`H1: ${data.headings.h1}`);
-  if (data.headings.h2) headingLines.push(`H2: ${data.headings.h2}`);
-  if (data.headings.h3) headingLines.push(`H3: ${data.headings.h3}`);
-  if (data.headings.h4) headingLines.push(`H4: ${data.headings.h4}`);
+  const headingLines = formatHeadings(data.headings);
 
   if (headingLines.length > 0) {
     lines.push(section('HEADINGS') + chalk.gray(' (Auto-attached)'));

package/src/utils/html.ts

@@ -85,8 +85,10 @@ const HIDDEN_CLASSES = new Set(['hidden', 'invisible', 'd-none', 'hide', 'dn', '
 
 export const EXPLORBOT_ATTRS = {
   area: 'data-explorbot-area',
+  coveredBy: 'data-explorbot-covered-by',
   context: 'data-explorbot-context',
   eidx: 'data-explorbot-eidx',
+  hit: 'data-explorbot-hit',
   variant: 'data-explorbot-variant',
 } as const;
 
@@ -169,14 +171,49 @@ export type ComponentScopeExtractionConfig = {
   limits: typeof HTML_EXTRACTION_LIMITS;
 };
 
+export function captureHtmlForSnapshot(): string {
+  const clone = document.documentElement.cloneNode(true) as HTMLElement;
+  const liveControls = Array.from(document.querySelectorAll('input, textarea, select')) as Array<HTMLInputElement | HTMLTextAreaElement | HTMLSelectElement>;
+  const clonedControls = Array.from(clone.querySelectorAll('input, textarea, select')) as HTMLElement[];
+
+  for (let i = 0; i < liveControls.length; i++) {
+    const source = liveControls[i];
+    const target = clonedControls[i];
+    if (!target) continue;
+
+    if (source instanceof HTMLInputElement || source instanceof HTMLTextAreaElement) {
+      target.setAttribute('data-explorbot-value', source.value);
+    }
+
+    if (source instanceof HTMLInputElement && (source.type === 'checkbox' || source.type === 'radio')) {
+      let checked = 'false';
+      if (source.checked) checked = 'true';
+      target.setAttribute('data-explorbot-checked', checked);
+    }
+
+    if (source instanceof HTMLSelectElement) {
+      target.setAttribute(
+        'data-explorbot-value',
+        Array.from(source.selectedOptions)
+          .map((option) => option.value)
+          .join(',')
+      );
+    }
+  }
+
+  return clone.outerHTML;
+}
+
 export function extractElementData(el: Element, config?: ElementExtractionConfig) {
   const cfg =
     config ||
     ({
       attrs: {
         area: 'data-explorbot-area',
+        coveredBy: 'data-explorbot-covered-by',
         context: 'data-explorbot-context',
         eidx: 'data-explorbot-eidx',
+        hit: 'data-explorbot-hit',
         variant: 'data-explorbot-variant',
       },
       codeEditorMarkers: ['monaco', 'codemirror', 'ace', 'ace_editor', 'code'],
@@ -246,6 +283,45 @@ export function extractElementData(el: Element, config?: ElementExtractionConfig
     return Array.from(tokens).slice(0, 8);
   }
 
+  function describeOverlappingElement(target: Element): string {
+    const parts = [target.tagName.toLowerCase()];
+    const role = target.getAttribute('role');
+    const ariaLabel = target.getAttribute('aria-label');
+    const id = target.getAttribute('id');
+    const className = (target.getAttribute('class') || '').split(/\s+/).filter(Boolean).slice(0, 3).join('.');
+
+    if (id) parts.push(`#${id.slice(0, 40)}`);
+    if (className) parts.push(`.${className.slice(0, 80)}`);
+    if (role) parts.push(`[role="${role.slice(0, 40)}"]`);
+    if (ariaLabel) parts.push(`[aria-label="${ariaLabel.slice(0, 80)}"]`);
+
+    return parts.join('');
+  }
+
+  function inspectHitTarget(target: Element, rect: DOMRect): { hit: string; coveredBy?: string } {
+    const viewportPoints = [
+      { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 },
+      { x: rect.left + Math.min(rect.width - 1, Math.max(1, rect.width * 0.25)), y: rect.top + Math.min(rect.height - 1, Math.max(1, rect.height * 0.25)) },
+      { x: rect.left + Math.min(rect.width - 1, Math.max(1, rect.width * 0.75)), y: rect.top + Math.min(rect.height - 1, Math.max(1, rect.height * 0.75)) },
+    ].filter((point) => point.x >= 0 && point.y >= 0 && point.x <= window.innerWidth && point.y <= window.innerHeight);
+
+    if (viewportPoints.length === 0) {
+      return { hit: 'offscreen' };
+    }
+
+    for (const point of viewportPoints) {
+      const top = document.elementFromPoint(point.x, point.y);
+      if (!top) continue;
+      if (top === target || target.contains(top)) {
+        return { hit: 'target' };
+      }
+    }
+
+    const centerPoint = viewportPoints[0];
+    const top = document.elementFromPoint(centerPoint.x, centerPoint.y);
+    return { hit: 'covered', coveredBy: top ? describeOverlappingElement(top) : undefined };
+  }
+
   function isEmbeddedCodeEditorFrame(target: Element): boolean {
     const src = (target.getAttribute('src') || '').toLowerCase();
     const markerSelector = cfg.codeEditorMarkers.map((marker) => `[class*="${marker}"]`).join(', ');
@@ -354,6 +430,9 @@ export function extractElementData(el: Element, config?: ElementExtractionConfig
   allAttrs[cfg.attrs.area] = areaHints.join('|');
   allAttrs[cfg.attrs.context] = findContextLabel(el);
   allAttrs[cfg.attrs.variant] = collectVariantHints(el).join('|');
+  const hitTarget = inspectHitTarget(el, rect);
+  allAttrs[cfg.attrs.hit] = hitTarget.hit;
+  if (hitTarget.coveredBy) allAttrs[cfg.attrs.coveredBy] = hitTarget.coveredBy;
 
   return {
     tag: el.tagName.toLowerCase(),

package/src/utils/markdown-files.ts

@@ -0,0 +1,30 @@
+import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import matter from 'gray-matter';
+
+export function loadMarkdownFiles(dir: string, options: { recursive?: boolean; onError?: (filePath: string, error: unknown) => void } = {}): MarkdownFile[] {
+  if (!existsSync(dir)) return [];
+
+  const results: MarkdownFile[] = [];
+  const files = readdirSync(dir, { recursive: options.recursive === true })
+    .filter((file): file is string => typeof file === 'string' && file.endsWith('.md'))
+    .map((file) => join(dir, file));
+
+  for (const filePath of files) {
+    try {
+      const parsed = matter(readFileSync(filePath, 'utf8'));
+      results.push({ filePath, content: parsed.content, data: parsed.data, mtime: statSync(filePath).mtime });
+    } catch (error) {
+      options.onError?.(filePath, error);
+    }
+  }
+
+  return results;
+}
+
+export interface MarkdownFile {
+  filePath: string;
+  content: string;
+  data: Record<string, any>;
+  mtime: Date;
+}

package/src/utils/markdown-query.ts

@@ -408,6 +408,10 @@ export class MarkdownQuery {
   }
 
   replace(content: string): string {
+    return this.replaceEach(() => content);
+  }
+
+  replaceEach(replacer: (match: MarkdownQuery, index: number) => string): string {
     const sorted = [...this.matches].sort((a, b) => a.start - b.start);
 
     const kept: MatchedRange[] = [];
@@ -418,10 +422,11 @@ export class MarkdownQuery {
       lastEnd = range.start + range.length;
     }
 
+    const replacements = kept.map((range, index) => replacer(new MarkdownQuery(this.source, [range]), index));
     let result = this.source;
     for (let i = kept.length - 1; i >= 0; i--) {
       const range = kept[i];
-      result = result.slice(0, range.start) + content + result.slice(range.start + range.length);
+      result = result.slice(0, range.start) + replacements[i] + result.slice(range.start + range.length);
     }
 
     return result;
@@ -459,6 +464,15 @@ export class MarkdownQuery {
   each(): MarkdownQuery[] {
     return this.matches.map((m) => new MarkdownQuery(this.source, [m]));
   }
+
+  meta(): Array<{ type: string; depth: number | null; text: string }> {
+    return this.matches.map((range) => {
+      const token = range.token as any;
+      let depth: number | null = null;
+      if (token.type === 'heading') depth = token.depth;
+      return { type: token.type, depth, text: getTokenText(range.token) };
+    });
+  }
 }
 
 export function mdq(source: string): MarkdownQuery {

package/src/utils/page-readiness.ts

@@ -49,7 +49,7 @@ function waitForPageBodyContent(page: any, timeout: number): Promise<void> {
     .catch(() => {});
 }
 
-function sleep(ms: number): Promise<void> {
+export function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 

package/src/utils/secrets.ts

@@ -1,4 +1,5 @@
 const MIN_SECRET_LENGTH = 4;
+const SECRET_NAME_TOKENS = ['password', 'passwd', 'secret', 'token', 'apikey', 'api_key', 'credential', 'private_key', 'privatekey', 'access_key', 'jwt', 'auth', 'bearer', 'authorization', 'cert', 'certificate'];
 const secretValues = new Set<string>();
 
 export function registerSecret(value: string): void {
@@ -10,7 +11,7 @@ export function registerSecret(value: string): void {
 export function redactSecrets(text: string): string {
   if (!text) return text;
   let result = text;
-  for (const value of secretValues) {
+  for (const value of [...secretValues].sort((a, b) => b.length - a.length)) {
     if (!result.includes(value)) continue;
     result = result.split(value).join('***REDACTED***');
   }
@@ -21,8 +22,6 @@ export function clearRegisteredSecrets(): void {
   secretValues.clear();
 }
 
-const SECRET_NAME_TOKENS = ['password', 'passwd', 'secret', 'token', 'apikey', 'api_key', 'credential', 'private_key', 'privatekey', 'access_key'];
-
 export function isSecretName(name: string): boolean {
   const lower = name.toLowerCase();
   return SECRET_NAME_TOKENS.some((token) => lower.includes(token));

package/src/utils/strings.ts

@@ -6,6 +6,18 @@ export function truncateJson(input: any): string {
   return str.length <= 80 ? str : `${str.slice(0, 77)}...`;
 }
 
+export function slugify(text: string): string {
+  return text
+    .replace(/[^a-zA-Z0-9_]/g, '_')
+    .replace(/_+/g, '_')
+    .replace(/^_|_$/g, '')
+    .toLowerCase();
+}
+
+export function normalizeInlineText(text: string): string {
+  return text.normalize('NFKC').replace(/\s+/g, ' ').trim();
+}
+
 export function sanitizeFilename(name: string): string {
   return name
     .toLowerCase()

package/src/utils/web-element.ts

@@ -47,6 +47,10 @@ export class WebElement {
     return this.attrs[EXPLORBOT_ATTRS.eidx] || this.attrs.eidx || null;
   }
 
+  ourAttr(name: keyof typeof EXPLORBOT_ATTRS): string | undefined {
+    return this.attrs[EXPLORBOT_ATTRS[name]];
+  }
+
   get isNavigationLink(): boolean {
     if (this.tag !== 'a') return false;
     const href = this.attrs.href || '';

package/src/utils/web-sandbox.ts

@@ -0,0 +1,43 @@
+import { faker } from '@faker-js/faker';
+import { hopeThat, retryTo, tryTo, within } from 'codeceptjs/lib/effects';
+import step from 'codeceptjs/steps';
+
+const SHADOWED_GLOBALS = ['process', 'global', 'globalThis', 'fetch', 'Bun', 'require', 'module', 'exports'];
+const ALLOWED_COMMAND_HEADS = ['I.', 'page.', 'await page.', 'await I.'];
+const PLAYWRIGHT_COMMAND_HEADS = ['page.', 'await page.'];
+const PLAYWRIGHT_ARG_NAMES = ['page'];
+const CODECEPT_ARG_NAMES = ['I', 'tryTo', 'retryTo', 'within', 'hopeThat', 'step', 'faker'];
+
+export function sanitizeCodeBlock(code: string): string {
+  return code
+    .split('\n')
+    .map((line) => line.trim())
+    .filter((line) => ALLOWED_COMMAND_HEADS.some((head) => line.startsWith(head)))
+    .join('\n');
+}
+
+export function hasPlaywrightCommands(code: string): boolean {
+  return code.split('\n').some((line) => {
+    const trimmed = line.trim();
+    return PLAYWRIGHT_COMMAND_HEADS.some((head) => trimmed.startsWith(head));
+  });
+}
+
+export function playwrightSandbox(page: any, code: string): Promise<any> {
+  const run = createSandbox(PLAYWRIGHT_ARG_NAMES, `return (async () => { ${code} })()`);
+  return run(page);
+}
+
+export function codeceptJSSandbox(actor: any, codeOrFn: string | ((...args: any[]) => void)): void {
+  if (typeof codeOrFn === 'function') {
+    codeOrFn(actor, tryTo, retryTo, within, hopeThat, step, faker);
+    return;
+  }
+  const run = createSandbox(CODECEPT_ARG_NAMES, codeOrFn);
+  run(actor, tryTo, retryTo, within, hopeThat, step, faker);
+}
+
+function createSandbox(argNames: string[], body: string): (...args: any[]) => any {
+  const fn = new Function(...argNames, ...SHADOWED_GLOBALS, `'use strict';\n${body}`);
+  return (...args: any[]) => fn(...args);
+}

package/dist/boat/api-tester/example/apibot.config.js

@@ -1,30 +0,0 @@
-import { createOpenRouter } from '@openrouter/ai-sdk-provider';
-const openrouter = createOpenRouter({
-    apiKey: process.env.OPENROUTER_API_KEY,
-});
-export default {
-    ai: {
-        model: openrouter('mistralai/devstral-small:nitro') //   openrouter('x-ai/grok-4-fast') // openrouter('moonshotai/kimi-k2.5:nitro'), // openrouter('x-ai/grok-4-fast') // minimax/minimax-m2.5:nitro
-    },
-    api: {
-        baseEndpoint: 'http://localhost:3000/api/v2/amr_testing',
-        headers: {
-            'Authorization': `Bearer ${process.env.TESTOMATIO_API_KEY}`,
-            'Content-Type': 'application/json',
-        },
-        spec: [
-            'http://127.0.0.1:3000/api/v2/openapi'
-        ],
-        // bootstrap: async ({ headers, baseEndpoint }) => {
-        //   // Run before tests — e.g. obtain auth token
-        //   // Return headers to merge: { Authorization: 'Bearer ...' }
-        // },
-        // teardown: async ({ headers, baseEndpoint }) => {
-        //   // Run after tests — e.g. cleanup test data
-        // },
-    },
-    dirs: {
-        output: 'output',
-        knowledge: 'knowledge',
-    },
-};