explorbot 0.1.30 → 0.1.32-beta.1

package/src/action-result.ts

@@ -68,6 +68,7 @@ export class ActionResult implements ActionResultData {
   private _screenshot: Buffer | undefined = undefined;
   readonly htmlFile: string | undefined = undefined;
   private _html: string | undefined = undefined;
+  private snapshotCache = new Map<string, string>();
   readonly logFile: string | undefined = undefined;
   private _browserLogs: any[] | undefined = undefined;
   readonly ariaSnapshotFile: string | undefined = undefined;
@@ -151,6 +152,7 @@ export class ActionResult implements ActionResultData {
 
   set html(value: string) {
     this._html = value;
+    this.snapshotCache.clear();
   }
 
   get screenshot(): Buffer | undefined {
@@ -263,21 +265,26 @@ export class ActionResult implements ActionResultData {
 
   async simplifiedHtml(htmlConfig?: HtmlConfig): Promise<string> {
     const normalizedConfig = this.normalizeHtmlConfig(htmlConfig);
-    return await minifyHtml(htmlMinimalUISnapshot(this.html ?? '', normalizedConfig?.minimal));
+    const cacheKey = `simplified:${JSON.stringify(normalizedConfig?.minimal ?? null)}`;
+    return this.cachedSnapshot(cacheKey, () => minifyHtml(htmlMinimalUISnapshot(this.html ?? '', normalizedConfig?.minimal)));
   }
 
   async combinedHtml(htmlConfig?: HtmlConfig & { keepPositions?: boolean }): Promise<string> {
     const normalizedConfig = this.normalizeHtmlConfig(htmlConfig);
-    const combinedHtml = await minifyHtml(htmlCombinedSnapshot(this.html ?? '', normalizedConfig?.combined, { keepPositions: htmlConfig?.keepPositions }));
-    debugLog(`----${this.url}----`);
-    debugLog(`Combined HTML: \n${combinedHtml}`);
-    debugLog('----');
-    return combinedHtml;
+    const cacheKey = `combined:${!!htmlConfig?.keepPositions}:${JSON.stringify(normalizedConfig?.combined ?? null)}`;
+    return this.cachedSnapshot(cacheKey, async () => {
+      const combinedHtml = await minifyHtml(htmlCombinedSnapshot(this.html ?? '', normalizedConfig?.combined, { keepPositions: htmlConfig?.keepPositions }));
+      debugLog(`----${this.url}----`);
+      debugLog(`Combined HTML: \n${combinedHtml}`);
+      debugLog('----');
+      return combinedHtml;
+    });
   }
 
   async textHtml(htmlConfig?: HtmlConfig): Promise<string> {
     const normalizedConfig = this.normalizeHtmlConfig(htmlConfig);
-    return await minifyHtml(htmlTextSnapshot(this.html ?? '', normalizedConfig?.text));
+    const cacheKey = `text:${JSON.stringify(normalizedConfig?.text ?? null)}`;
+    return this.cachedSnapshot(cacheKey, () => minifyHtml(htmlTextSnapshot(this.html ?? '', normalizedConfig?.text)));
   }
 
   getInteractiveARIA(): string {
@@ -296,6 +303,14 @@ export class ActionResult implements ActionResultData {
     return parser.getConfig().html;
   }
 
+  private async cachedSnapshot(key: string, compute: () => Promise<string>): Promise<string> {
+    const cached = this.snapshotCache.get(key);
+    if (cached !== undefined) return cached;
+    const value = await compute();
+    this.snapshotCache.set(key, value);
+    return value;
+  }
+
   static fromState(state: WebPageState): ActionResult {
     let html: string | undefined = undefined;
     let screenshot: Buffer | undefined = undefined;

package/src/action.ts

@@ -303,11 +303,11 @@ class Action {
 
       if (isPlaywright) {
         const page = this.playwrightHelper.page;
-        const asyncFn = new Function('page', `return (async () => { ${sanitizedCode} })()`);
+        const asyncFn = createSandboxedFunction(['page'], `return (async () => { ${sanitizedCode} })()`);
         await asyncFn(page);
         await sleep(this.config.action?.delay || 500);
       } else {
-        const codeFunction = new Function('I', 'tryTo', 'retryTo', 'within', 'hopeThat', 'step', 'faker', sanitizedCode);
+        const codeFunction = createSandboxedFunction(['I', 'tryTo', 'retryTo', 'within', 'hopeThat', 'step', 'faker'], sanitizedCode);
         codeFunction(this.actor, tryTo, retryTo, within, hopeThat, step, faker);
         await recorder.add(() => sleep(this.config.action?.delay || 500));
         await recorder.promise();
@@ -366,7 +366,7 @@ class Action {
         if (!sanitizedCode) {
           throw new Error('No valid I.* commands found in code block');
         }
-        codeFunction = new Function('I', 'tryTo', 'retryTo', 'within', 'hopeThat', 'step', sanitizedCode);
+        codeFunction = createSandboxedFunction(['I', 'tryTo', 'retryTo', 'within', 'hopeThat', 'step'], sanitizedCode);
       }
       codeFunction(this.actor, tryTo, retryTo, within, hopeThat, step);
       await recorder.promise();
@@ -480,14 +480,21 @@ async function captureTitle(page: any, actor: any): Promise<string> {
   return '';
 }
 
-function sanitizeCodeBlock(code: string): string {
+const SHADOWED_GLOBALS = ['process', 'global', 'globalThis', 'fetch', 'Bun', 'require', 'module', 'exports'];
+
+export function sanitizeCodeBlock(code: string): string {
   return code
     .split('\n')
     .map((line) => line.trim())
-    .filter((line) => line.startsWith('I.') || line.startsWith('page.') || line.startsWith('await '))
+    .filter((line) => line.startsWith('I.') || line.startsWith('page.') || line.startsWith('await page.') || line.startsWith('await I.'))
     .join('\n');
 }
 
+export function createSandboxedFunction(argNames: string[], body: string): (...args: any[]) => any {
+  const fn = new Function(...argNames, ...SHADOWED_GLOBALS, `'use strict';\n${body}`);
+  return (...args: any[]) => fn(...args);
+}
+
 function hasPlaywrightCommands(code: string): boolean {
   return code.split('\n').some((line) => {
     const trimmed = line.trim();

package/src/ai/conversation.ts

@@ -51,7 +51,7 @@ export class Conversation {
 
     this.messages.push({
       role: 'user',
-      content: [{ type: 'image', image: imageData }],
+      content: [{ type: 'file', mediaType: 'image/png', data: imageData }],
     });
   }
 

package/src/ai/provider.ts

@@ -31,7 +31,7 @@ function extractCachedTokens(usage: any): number {
   return typeof fromRaw === 'number' ? fromRaw : 0;
 }
 
-function rejectAfterIdle(ms: number, signal: { cancelled: boolean }): Promise<never> {
+function rejectAfterIdle(ms: number, signal: { cancelled: boolean }, controller: AbortController): Promise<never> {
   return new Promise((_, reject) => {
     const tick = () => {
       if (signal.cancelled) return;
@@ -39,6 +39,7 @@ function rejectAfterIdle(ms: number, signal: { cancelled: boolean }): Promise<ne
         setTimeout(tick, ms);
         return;
       }
+      controller.abort();
       reject(new Error('AI request timeout'));
     };
     setTimeout(tick, ms);
@@ -367,13 +368,16 @@ export class Provider {
       const response = await withRetry(async () => {
         const timeout = config.timeout || 30000;
         const cancel = { cancelled: false };
+        const controller = new AbortController();
+        const combinedSignal = AbortSignal.any([controller.signal, executionController.getAbortSignal()].filter(Boolean) as AbortSignal[]);
         try {
           const result = (await Promise.race([
             generateText({
               messages,
               ...config,
+              abortSignal: combinedSignal,
             }),
-            rejectAfterIdle(timeout, cancel),
+            rejectAfterIdle(timeout, cancel, controller),
           ])) as any;
           const hasToolCall = (result.toolCalls?.length || 0) > 0;
           if (!result.text && !hasToolCall && result.finishReason === 'length') {
@@ -455,13 +459,16 @@ export class Provider {
       const response = await withRetry(async () => {
         const timeout = config.timeout || 30000;
         const cancel = { cancelled: false };
+        const controller = new AbortController();
+        const combinedSignal = AbortSignal.any([controller.signal, executionController.getAbortSignal()].filter(Boolean) as AbortSignal[]);
         try {
           return (await Promise.race([
             generateObject({
               messages,
               ...config,
+              abortSignal: combinedSignal,
             }),
-            rejectAfterIdle(timeout, cancel),
+            rejectAfterIdle(timeout, cancel, controller),
           ])) as any;
         } finally {
           cancel.cancelled = true;
@@ -641,8 +648,9 @@ export class Provider {
             text: prompt,
           },
           {
-            type: 'image',
-            image: imageData,
+            type: 'file',
+            mediaType: 'image/png',
+            data: imageData,
           },
         ],
       },

package/src/experience-tracker.ts

@@ -8,6 +8,7 @@ import { KnowledgeTracker } from './knowledge-tracker.js';
 import type { WebPageState } from './state-manager.js';
 import { createDebug, tag } from './utils/logger.js';
 import { mdq } from './utils/markdown-query.js';
+import { redactSecrets } from './utils/secrets.js';
 import { isNonReusableCode } from './utils/step-analyzer.ts';
 import { extractStatePath } from './utils/url-matcher.js';
 
@@ -109,7 +110,7 @@ export class ExperienceTracker {
       return;
     }
     const filePath = this.getExperienceFilePath(stateHash);
-    const fileContent = matter.stringify(content, frontmatter || {});
+    const fileContent = matter.stringify(redactSecrets(content), frontmatter || {});
     writeFileSync(filePath, fileContent, 'utf8');
   }
 

package/src/explorer.ts

@@ -727,17 +727,22 @@ class Explorer {
 
     this.watchActiveTestPage();
 
-    codeceptjs.event.dispatcher.emit('test.before', codeceptjsTest);
-    codeceptjs.event.dispatcher.emit('test.start', codeceptjsTest);
-    codeceptjs.event.dispatcher.on('step.passed', (step: any) => stepHandler(step, 'passed'));
-    codeceptjs.event.dispatcher.on('step.failed', (step: any, error: any) => {
+    const onStepPassed = (step: any) => stepHandler(step, 'passed');
+    const onStepFailed = (step: any, error: any) => {
       stepHandler(step, 'failed', error?.message || String(error), error?.stack);
-    });
-    codeceptjs.event.dispatcher.on('test.after', () => {
-      codeceptjs.event.dispatcher.off('step.passed', stepHandler);
-      codeceptjs.event.dispatcher.off('step.failed', stepHandler);
+    };
+    const onTestAfter = () => {
+      codeceptjs.event.dispatcher.off('step.passed', onStepPassed);
+      codeceptjs.event.dispatcher.off('step.failed', onStepFailed);
+      codeceptjs.event.dispatcher.off('test.after', onTestAfter);
       this.unwatchActiveTestPages();
-    });
+    };
+
+    codeceptjs.event.dispatcher.emit('test.before', codeceptjsTest);
+    codeceptjs.event.dispatcher.emit('test.start', codeceptjsTest);
+    codeceptjs.event.dispatcher.on('step.passed', onStepPassed);
+    codeceptjs.event.dispatcher.on('step.failed', onStepFailed);
+    codeceptjs.event.dispatcher.on('test.after', onTestAfter);
 
     return true;
   }

package/src/index.ts

@@ -0,0 +1,9 @@
+export { ExplorBot } from './explorbot.ts';
+export type { ExplorBotOptions, UserResolveFunction } from './explorbot.ts';
+
+export { Plan, Test, TestResult, TestStatus } from './test-plan.ts';
+export type { Note, StepData, TestResultType, TestStatusType } from './test-plan.ts';
+
+export type { Link, StateTransition, WebPageState } from './state-manager.ts';
+
+export type { AIConfig, ExplorbotConfig, PlaywrightConfig } from './config.ts';

package/src/knowledge-tracker.ts

@@ -5,6 +5,7 @@ import { ActionResult } from './action-result.js';
 import { ConfigParser } from './config.js';
 import { getCliName } from './utils/cli-name.ts';
 import { createDebug } from './utils/logger.js';
+import { isSecretName, registerSecret } from './utils/secrets.js';
 
 const debugLog = createDebug('explorbot:knowledge-tracker');
 
@@ -138,9 +139,14 @@ export class KnowledgeTracker {
       const namespace = expr.slice(0, dotIndex);
       const key = expr.slice(dotIndex + 1);
 
-      if (namespace === 'env') return process.env[key] ?? '';
+      if (namespace === 'env') {
+        const value = process.env[key] ?? '';
+        if (isSecretName(key)) registerSecret(value);
+        return value;
+      }
 
       if (namespace === 'config') {
+        if (isSecretName(key)) return '';
         const config = ConfigParser.getInstance().getConfig();
         const value = key.split('.').reduce((obj: any, k) => obj?.[k], config);
         if (value !== undefined && typeof value !== 'object') return String(value);

package/src/observability.ts

@@ -1,4 +1,5 @@
 import { type Span, trace } from '@opentelemetry/api';
+import { redactSecrets } from './utils/secrets.js';
 
 type TelemetryMetadata = Record<string, unknown>;
 
@@ -84,7 +85,7 @@ function buildRootSpanAttributes(name: string, metadata: TelemetryMetadata): Rec
     attributes['langfuse.trace.tags'] = metadata.tags as string[];
   }
   if (metadata.input !== undefined) {
-    attributes['langfuse.trace.input'] = JSON.stringify(metadata.input);
+    attributes['langfuse.trace.input'] = redactSecrets(JSON.stringify(metadata.input));
   }
 
   return attributes;

package/src/state-manager.ts

@@ -5,7 +5,6 @@ import { ActionResult } from './action-result.js';
 import { ConfigParser, outputPath } from './config.js';
 import { ExperienceTracker } from './experience-tracker.js';
 import { detectFocusArea } from './utils/aria.js';
-import { htmlTextSnapshot } from './utils/html.js';
 import { createDebug, tag } from './utils/logger.js';
 import { extractStatePath } from './utils/url-matcher.js';
 
@@ -126,13 +125,6 @@ export class StateManager {
    * Emit state change event to all listeners
    */
   private emitStateChange(event: StateTransition): void {
-    // Log HTML content when state changes
-    if (event.toState.html && event.toState.html !== event.fromState?.html) {
-      let htmlContent = event?.toState?.html ?? '';
-      htmlContent = htmlTextSnapshot(htmlContent);
-      // tag('html').log(`Page HTML for ${event.toState.url}:\n${htmlContent}`);
-    }
-
     this.stateChangeListeners.forEach((listener) => {
       try {
         listener(event);

package/src/utils/secrets.ts

@@ -0,0 +1,29 @@
+const MIN_SECRET_LENGTH = 4;
+const secretValues = new Set<string>();
+
+export function registerSecret(value: string): void {
+  if (!value) return;
+  if (value.length < MIN_SECRET_LENGTH) return;
+  secretValues.add(value);
+}
+
+export function redactSecrets(text: string): string {
+  if (!text) return text;
+  let result = text;
+  for (const value of secretValues) {
+    if (!result.includes(value)) continue;
+    result = result.split(value).join('***REDACTED***');
+  }
+  return result;
+}
+
+export function clearRegisteredSecrets(): void {
+  secretValues.clear();
+}
+
+const SECRET_NAME_TOKENS = ['password', 'passwd', 'secret', 'token', 'apikey', 'api_key', 'credential', 'private_key', 'privatekey', 'access_key'];
+
+export function isSecretName(name: string): boolean {
+  const lower = name.toLowerCase();
+  return SECRET_NAME_TOKENS.some((token) => lower.includes(token));
+}

package/src/index.tsx

@@ -1,62 +0,0 @@
-#!/usr/bin/env node
-import { Command } from 'commander';
-import { addKnowledgeCommand } from './commands/add-knowledge.js';
-import { cleanCommand } from './commands/clean.js';
-import { exploreCommand } from './commands/explore.js';
-import { initCommand } from './commands/init.js';
-
-const program = new Command();
-
-program
-  .name('explorbot')
-  .description('AI-powered web exploration tool')
-  .option('-f, --from <url>', 'Start exploration from a specific URL')
-  .option('-v, --verbose', 'Enable verbose logging')
-  .option('--debug', 'Enable debug logging (same as --verbose)')
-  .option('-c, --config <path>', 'Path to configuration file')
-  .option('-p, --path <path>', 'Working directory path')
-  .helpOption('-h, --help', 'Show this help message');
-
-program
-  .command('clean')
-  .description('Clean up artifacts or experience folders')
-  .option('-t, --type <type>', 'Type of cleanup (artifacts|experience|all)', 'artifacts')
-  .option('-p, --path <path>', 'Custom path to clean')
-  .action(async (options, command) => {
-    const globalOptions = command.parent.opts();
-    const allOptions = { ...globalOptions, ...options };
-    await cleanCommand(allOptions);
-  });
-
-program
-  .command('init')
-  .description('Initialize a new project with configuration')
-  .option('-c, --config-path <path>', 'Path for the config file', './explorbot.config.js')
-  .option('-f, --force', 'Overwrite existing config file', false)
-  .option('-p, --path <path>', 'Working directory for initialization')
-  .action(async (options, command) => {
-    const globalOptions = command.parent.opts();
-    const allOptions = { ...globalOptions, ...options };
-    await initCommand(allOptions);
-  });
-
-program
-  .command('add-knowledge')
-  .alias('learn')
-  .description('Add knowledge for specific URLs')
-  .option('-p, --path <path>', 'Knowledge directory path')
-  .action(async (options) => {
-    await addKnowledgeCommand(options);
-  });
-
-program.parse();
-
-const options = program.opts();
-const command = program.args[0];
-
-if (command === 'clean' || command === 'init' || command === 'add-knowledge' || command === 'learn') {
-  // These commands are handled by their respective actions
-} else {
-  // Default to explore command
-  exploreCommand(options);
-}