explorbot 0.0.1 → 0.1.0

package/src/config.ts

@@ -0,0 +1,491 @@
+import { existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs';
+import path, { dirname, join, resolve } from 'node:path';
+import { parseEnv } from 'node:util';
+import { log } from './utils/logger.js';
+
+interface PlaywrightConfig {
+  browser: 'chromium' | 'firefox' | 'webkit';
+  url: string;
+  show?: boolean;
+  windowSize?: string;
+  slowMo?: number;
+  chromium?: {
+    args?: string[];
+  };
+  firefox?: {
+    args?: string[];
+  };
+  webkit?: {
+    args?: string[];
+  };
+  timeout?: number;
+  waitForAction?: number;
+  waitForNavigation?: 'load' | 'domcontentloaded' | 'networkidle';
+  waitForTimeout?: number;
+  ignoreHTTPSErrors?: boolean;
+  userAgent?: string;
+  viewport?: {
+    width: number;
+    height: number;
+  };
+  args?: string[];
+}
+
+type PlaywrightHookFn = (ctx: { page: any; url: string }) => Promise<void> | void;
+type CodeceptJSHookFn = (ctx: { I: any; url: string }) => Promise<void> | void;
+
+interface PlaywrightHook {
+  type: 'playwright';
+  hook: PlaywrightHookFn;
+}
+
+interface CodeceptJSHook {
+  type: 'codeceptjs';
+  hook: CodeceptJSHookFn;
+}
+
+type Hook = PlaywrightHook | CodeceptJSHook;
+type HookPatternMap = Record<string, Hook>;
+type HookConfig = Hook | HookPatternMap;
+
+interface HooksConfig {
+  beforeHook?: HookConfig;
+  afterHook?: HookConfig;
+}
+
+interface AgentConfig extends HooksConfig {
+  model?: any;
+  enabled?: boolean;
+  systemPrompt?: string;
+  rules?: RuleEntry[];
+  providerOptions?: Record<string, any>;
+}
+
+interface ResearcherAgentConfig extends AgentConfig {
+  excludeSelectors?: string[];
+  includeSelectors?: string[];
+  stopWords?: string[];
+  maxElementsToExplore?: number;
+  maxExpandableClicks?: number;
+  retries?: number;
+  sections?: string[];
+  errorPageTimeout?: number;
+}
+
+interface TesterAgentConfig extends AgentConfig {
+  progressCheckInterval?: number;
+}
+
+interface PilotAgentConfig extends AgentConfig {
+  stepsToReview?: number;
+}
+
+interface NavigatorAgentConfig extends AgentConfig {
+  addHtmlOnTry?: number;
+  maxAttempts?: number;
+}
+
+interface PlannerAgentConfig extends AgentConfig {
+  styles?: string[];
+  stylesDir?: string;
+}
+
+interface AgentsConfig {
+  tester?: TesterAgentConfig;
+  navigator?: NavigatorAgentConfig;
+  researcher?: ResearcherAgentConfig;
+  planner?: PlannerAgentConfig;
+  pilot?: PilotAgentConfig;
+  'experience-compactor'?: AgentConfig;
+  captain?: AgentConfig;
+  quartermaster?: AgentConfig;
+  historian?: AgentConfig;
+  fisherman?: AgentConfig;
+  chief?: AgentConfig;
+  curler?: AgentConfig;
+}
+
+interface AIConfig {
+  model: any;
+  apiKey?: string;
+  config?: Record<string, any>;
+  langfuse?: {
+    enabled?: boolean;
+    publicKey?: string;
+    secretKey?: string;
+    baseUrl?: string;
+  };
+  tools?: {
+    enabled: boolean;
+    maxConcurrency: number;
+    timeout: number;
+  };
+  vision?: boolean;
+  visionModel?: any;
+  agenticModel?: any;
+  maxAttempts?: number;
+  retryDelay?: number;
+  agents?: AgentsConfig;
+}
+
+interface HtmlConfig {
+  minimal?: {
+    include?: string[];
+    exclude?: string[];
+  };
+  combined?: {
+    include?: string[];
+    exclude?: string[];
+  };
+  text?: {
+    include?: string[];
+    exclude?: string[];
+  };
+}
+
+interface ActionConfig {
+  delay?: number;
+  retries?: number;
+}
+
+interface ReporterConfig {
+  enabled?: boolean;
+  html?: boolean;
+}
+
+type ApiHookFn = (ctx: { headers: Record<string, string>; baseEndpoint: string }) => Promise<Record<string, string> | undefined> | Record<string, string> | undefined;
+
+interface ApiConfig {
+  baseEndpoint?: string;
+  spec?: string[];
+  headers?: Record<string, string>;
+  bootstrap?: ApiHookFn;
+  teardown?: ApiHookFn;
+}
+
+interface WebConfig {
+  url: string;
+}
+
+interface ExplorbotConfig {
+  web?: WebConfig;
+  playwright: PlaywrightConfig;
+  ai: AIConfig;
+  html?: HtmlConfig;
+  action?: ActionConfig;
+  dirs?: {
+    knowledge: string;
+    experience: string;
+    output: string;
+  };
+  experience?: {
+    maxReadLines?: number;
+  };
+  reporter?: ReporterConfig;
+  api?: ApiConfig;
+  stepsFile?: string;
+  files?: Record<string, string>;
+  dynamicPageRegex?: string;
+}
+
+const config: ExplorbotConfig = {
+  playwright: {
+    browser: 'chromium',
+    url: 'http://localhost:3000',
+  },
+
+  ai: {
+    model: null as any,
+  },
+};
+
+type RuleEntry = string | Record<string, string>;
+
+export const EXPLORBOT_CONFIG_PATHS = ['explorbot.config.js', 'explorbot.config.mjs', 'explorbot.config.ts'];
+
+export type {
+  ExplorbotConfig,
+  PlaywrightConfig,
+  AIConfig,
+  HtmlConfig,
+  ActionConfig,
+  AgentConfig,
+  AgentsConfig,
+  ResearcherAgentConfig,
+  NavigatorAgentConfig,
+  PlannerAgentConfig,
+  Hook,
+  HookConfig,
+  HooksConfig,
+  PlaywrightHook,
+  CodeceptJSHook,
+  HookPatternMap,
+  RuleEntry,
+  ReporterConfig,
+  ApiConfig,
+  WebConfig,
+  ApiHookFn,
+};
+
+export class ConfigParser {
+  private static instance: ConfigParser;
+  private config: ExplorbotConfig | null = null;
+  private configPath: string | null = null;
+
+  private constructor() {}
+
+  public static loadEnv(filePath: string): void {
+    const resolved = resolve(filePath);
+    if (!existsSync(resolved)) return;
+    Object.assign(process.env, parseEnv(readFileSync(resolved, 'utf8')));
+  }
+
+  public static getInstance(): ConfigParser {
+    if (!ConfigParser.instance) {
+      ConfigParser.instance = new ConfigParser();
+    }
+    return ConfigParser.instance;
+  }
+
+  public async loadConfig(options?: {
+    config?: string;
+    path?: string;
+  }): Promise<ExplorbotConfig> {
+    if (this.config && !options?.config && !options?.path) {
+      return this.config;
+    }
+
+    // Store the initial working directory for reference
+    if (!process.env.INITIAL_CWD) {
+      process.env.INITIAL_CWD = process.cwd();
+    }
+
+    const originalCwd = process.cwd();
+    if (options?.path) {
+      const resolvedWorkingPath = resolve(options.path);
+      process.chdir(resolvedWorkingPath);
+    }
+
+    ConfigParser.loadEnv('.env');
+
+    try {
+      const resolvedPath = options?.config || this.findConfigFile();
+
+      if (!resolvedPath) {
+        throw new Error('No configuration file found. Please create explorbot.config.js or explorbot.config.ts');
+      }
+
+      const configModule = await this.loadConfigModule(resolvedPath);
+      const loadedConfig = configModule.default || configModule;
+
+      if (!loadedConfig) {
+        throw new Error('Configuration file is empty or invalid');
+      }
+
+      this.config = this.resolveConfig(loadedConfig as ExplorbotConfig);
+      this.configPath = resolvedPath;
+
+      log(`Configuration loaded from: ${resolvedPath}`);
+
+      // Restore original directory after successful config load
+      if (options?.path && originalCwd !== process.cwd()) {
+        process.chdir(originalCwd);
+      }
+
+      return this.config;
+    } catch (error) {
+      // Restore original directory on error
+      if (options?.path && originalCwd !== process.cwd()) {
+        process.chdir(originalCwd);
+      }
+      throw new Error(`Failed to load configuration: ${error}`);
+    }
+  }
+
+  public getConfig(): ExplorbotConfig {
+    if (!this.config) {
+      throw new Error('Configuration not loaded. Call loadConfig() first.');
+    }
+    return this.config;
+  }
+
+  public getConfigPath(): string | null {
+    return this.configPath;
+  }
+
+  public getOutputDir(): string {
+    const config = this.getConfig();
+    const configPath = this.getConfigPath();
+    if (!configPath) throw new Error('Config path not found');
+    return path.join(path.dirname(configPath), config.dirs?.output || 'output');
+  }
+
+  public getStatesDir(): string {
+    return outputPath('states');
+  }
+
+  public getPlansDir(): string {
+    return outputPath('plans');
+  }
+
+  // For testing purposes only
+  public static resetForTesting(): void {
+    if (ConfigParser.instance) {
+      ConfigParser.instance.config = null;
+      ConfigParser.instance.configPath = null;
+    }
+  }
+
+  // For testing purposes only - sets up minimal default config
+  public static setupTestConfig(): void {
+    const instance = ConfigParser.getInstance();
+    // Create unique directory names for this test run to ensure isolation
+    const testId = `${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
+    const testBaseDir = join(process.cwd(), 'test-dirs', testId);
+
+    instance.config = {
+      playwright: {
+        url: 'https://example.com',
+        browser: 'chromium',
+        show: false,
+      },
+      ai: {
+        model: { modelId: 'test-model', provider: 'test' },
+        config: {},
+        vision: false,
+      },
+      dirs: {
+        knowledge: join(testBaseDir, 'knowledge'),
+        experience: join(testBaseDir, 'experience'),
+        output: join(testBaseDir, 'output'),
+      },
+    };
+    instance.configPath = join(testBaseDir, 'test-config');
+  }
+
+  // For testing purposes only - get test directories for cleanup
+  public static getTestDirectories(): string[] {
+    const instance = ConfigParser.getInstance();
+    if (!instance.config?.dirs) return [];
+
+    return [instance.config.dirs.knowledge, instance.config.dirs.experience, instance.config.dirs.output, dirname(instance.configPath || '')].filter((dir) => dir?.includes('test-dirs'));
+  }
+
+  // For testing purposes only - clean up all test directories
+  public static cleanupAllTestDirectories(): void {
+    try {
+      const testDirsBase = join(process.cwd(), 'test-dirs');
+      if (existsSync(testDirsBase)) {
+        rmSync(testDirsBase, { recursive: true, force: true });
+      }
+    } catch (error) {
+      // Ignore cleanup errors
+    }
+  }
+
+  private findConfigFile(): string | null {
+    const possiblePaths = [...EXPLORBOT_CONFIG_PATHS, 'config/explorbot.config.js', 'config/explorbot.config.mjs', 'config/explorbot.config.ts', 'src/config/explorbot.config.js', 'src/config/explorbot.config.mjs', 'src/config/explorbot.config.ts'];
+
+    for (const path of possiblePaths) {
+      const fullPath = resolve(process.cwd(), path);
+      if (existsSync(fullPath)) {
+        return fullPath;
+      }
+    }
+
+    return null;
+  }
+
+  private async loadConfigModule(configPath: string): Promise<any> {
+    const ext = configPath.split('.').pop();
+
+    if (ext === 'ts') {
+      try {
+        const module = await import(configPath);
+        return module;
+      } catch (error) {
+        const require = (await import('node:module')).createRequire(import.meta.url);
+        return require(configPath);
+      }
+    } else if (ext === 'js' || ext === 'mjs') {
+      const module = await import(configPath);
+      return module;
+    } else {
+      const content = readFileSync(configPath, 'utf8');
+      return JSON.parse(content);
+    }
+  }
+
+  private resolveConfig(config: ExplorbotConfig): ExplorbotConfig {
+    if (config.web?.url && !config.playwright?.url) {
+      config.playwright = config.playwright || { browser: 'chromium', url: '' };
+      config.playwright.url = config.web.url;
+    }
+    return config;
+  }
+
+  public validateConfig(config: ExplorbotConfig): void {
+    if (!config.ai?.model) {
+      throw new Error('Missing required configuration field: ai.model');
+    }
+
+    const url = config.playwright?.url || config.web?.url;
+    if (!url) {
+      throw new Error('Missing required configuration: web.url or playwright.url');
+    }
+
+    try {
+      new URL(url);
+    } catch {
+      throw new Error(`Invalid URL in configuration: ${url}`);
+    }
+  }
+
+  private getNestedValue(obj: any, path: string): any {
+    return path.split('.').reduce((current, key) => current?.[key], obj);
+  }
+
+  public mergeWithDefaults(config: Partial<ExplorbotConfig>): ExplorbotConfig {
+    const defaults = {
+      playwright: {
+        browser: 'chromium',
+        show: false, // we need headless
+      },
+      action: {
+        delay: 1000,
+        retries: 3,
+      },
+      dirs: {
+        knowledge: 'knowledge',
+        experience: 'experience',
+        output: 'output',
+      },
+    };
+
+    return this.deepMerge(defaults, config);
+  }
+
+  private deepMerge(target: any, source: any): any {
+    const result = { ...target };
+
+    for (const key in source) {
+      if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key]) && source[key].constructor === Object) {
+        result[key] = this.deepMerge(result[key] || {}, source[key]);
+      } else {
+        result[key] = source[key];
+      }
+    }
+
+    return result;
+  }
+
+  public ensureDirectory(path: string): void {
+    if (!existsSync(path)) {
+      mkdirSync(path, { recursive: true });
+    }
+  }
+}
+
+export function outputPath(...segments: string[]): string {
+  return path.join(ConfigParser.getInstance().getOutputDir(), ...segments);
+}

package/src/execution-controller.ts

@@ -0,0 +1,109 @@
+import { EventEmitter } from 'node:events';
+import * as readline from 'node:readline';
+import { clearActivity } from './activity.ts';
+
+export type InputCallback = (prompt: string) => Promise<string | null>;
+
+export class ExecutionController extends EventEmitter {
+  private static instance: ExecutionController;
+  private interrupted = false;
+  private inputCallback: InputCallback | null = null;
+  private interruptResolvers: Array<() => void> = [];
+  private abortController: AbortController | null = null;
+
+  private constructor() {
+    super();
+  }
+
+  static getInstance(): ExecutionController {
+    if (!ExecutionController.instance) {
+      ExecutionController.instance = new ExecutionController();
+    }
+    return ExecutionController.instance;
+  }
+
+  setInputCallback(callback: InputCallback): void {
+    this.inputCallback = callback;
+  }
+
+  startExecution(): void {
+    this.interrupted = false;
+    this.abortController = new AbortController();
+  }
+
+  getAbortSignal(): AbortSignal | undefined {
+    return this.abortController?.signal;
+  }
+
+  interrupt(): void {
+    clearActivity();
+    if (this.interrupted) return;
+    this.interrupted = true;
+    this.abortController?.abort();
+    this.emit('interrupt');
+    for (const resolve of this.interruptResolvers) {
+      resolve();
+    }
+    this.interruptResolvers = [];
+    this.emit('idle');
+  }
+
+  isInterrupted(): boolean {
+    return this.interrupted;
+  }
+
+  waitForInterrupt(): Promise<void> {
+    if (this.interrupted) {
+      return Promise.resolve();
+    }
+    return new Promise<void>((resolve) => {
+      this.interruptResolvers.push(resolve);
+    });
+  }
+
+  async checkInterrupt(): Promise<string | null> {
+    if (!this.interrupted) return null;
+
+    const userInput = await this.requestInput('Execution interrupted. What should we do instead?');
+    this.interrupted = false;
+    return userInput;
+  }
+
+  async handleInterrupt(prompt?: string): Promise<string | null> {
+    const message = prompt || 'Execution interrupted. Enter new instruction (or "stop"/"exit" to cancel):';
+    const userInput = await this.requestInput(message);
+    this.interrupted = false;
+    return userInput;
+  }
+
+  async requestInput(prompt: string): Promise<string | null> {
+    if (this.inputCallback) {
+      return await this.inputCallback(prompt);
+    }
+
+    return await this.readlineInput(prompt);
+  }
+
+  private async readlineInput(prompt: string): Promise<string | null> {
+    const rl = readline.createInterface({
+      input: process.stdin,
+      output: process.stdout,
+    });
+
+    return new Promise<string | null>((resolve) => {
+      rl.question(`${prompt}\n> `, (answer) => {
+        rl.close();
+        const trimmed = answer.trim();
+        resolve(trimmed || null);
+      });
+    });
+  }
+
+  reset(): void {
+    this.interrupted = false;
+    this.interruptResolvers = [];
+    this.abortController = null;
+  }
+}
+
+export const executionController = ExecutionController.getInstance();