explorbot 0.1.16 → 0.1.18

package/bin/explorbot-cli.ts

@@ -122,7 +122,15 @@ addCommonOptions(program.command('start [path]').description('Start web explorat
   await startTUI(explorBot);
 });
 
-addCommonOptions(program.command('explore <path>').description('Explore a page autonomously and run invented scenarios').option('--max-tests <count>', 'Maximum number of tests to run').option('--focus <feature>', 'Focus area for exploration')).action(async (explorePath, options) => {
+addCommonOptions(
+  program
+    .command('explore <path>')
+    .description('Explore a page autonomously and run invented scenarios')
+    .option('--max-tests <count>', 'Maximum number of tests to run')
+    .option('--focus <feature>', 'Focus area for exploration')
+    .option('--configure <spec>', 'Reuse spec: keys new|from|style|subpages|pick_by|priority, e.g. "new:25%;pick_by=random;priority=critical,high"')
+    .option('--dry-run', 'Mark picked tests as skipped without executing or generating new ones')
+).action(async (explorePath, options) => {
   try {
     const explorBot = new ExplorBot(buildExplorBotOptions(explorePath, options));
     await explorBot.start();
@@ -130,8 +138,11 @@ addCommonOptions(program.command('explore <path>').description('Explore a page a
     const { ExploreCommand } = await import('../src/commands/explore-command.js');
     const cmd = new ExploreCommand(explorBot);
     if (options.maxTests) cmd.maxTests = Number.parseInt(options.maxTests, 10);
+    if (options.dryRun) cmd.dryRun = true;
     const execArgs: string[] = [];
     if (options.focus) execArgs.push('--focus', `"${options.focus}"`);
+    if (options.configure) execArgs.push('--configure', `"${options.configure}"`);
+    if (options.dryRun) execArgs.push('--dry-run');
     await cmd.execute(execArgs.join(' '));
     await explorBot.stop();
     await showStatsAndExit(0);
@@ -812,6 +823,8 @@ program
   });
 
 import { createApiCommands } from '../boat/api-tester/src/cli.ts';
+import { createDocsCommands } from '../boat/doc-collector/src/cli.ts';
 program.addCommand(createApiCommands('api'));
+program.addCommand(createDocsCommands('docs'));
 
 program.parse();

package/boat/doc-collector/bin/doc-collector-cli.ts

@@ -0,0 +1,5 @@
+#!/usr/bin/env bun
+import { createDocsCommands } from '../src/cli.ts';
+
+const program = createDocsCommands('doc-collector');
+program.parse();

package/boat/doc-collector/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "doc-collector",
+  "version": "1.0.0",
+  "description": "AI-powered website documentation collector",
+  "type": "module",
+  "bin": {
+    "doc-collector": "./bin/doc-collector-cli.ts"
+  },
+  "scripts": {
+    "format": "biome format --write .",
+    "lint:fix": "biome lint --write .",
+    "check:fix": "biome check --write ."
+  },
+  "dependencies": {
+    "ai": "^6.0.6",
+    "commander": "^14.0.1",
+    "dedent": "^1.6.0",
+    "zod": "^4.1.8"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "^1.5.3",
+    "typescript": "^5.0.0"
+  }
+}

package/boat/doc-collector/src/ai/documentarian.ts

@@ -0,0 +1,184 @@
+import dedent from 'dedent';
+import { z } from 'zod';
+import type { AIProvider } from '../../../../src/ai/provider.ts';
+import type { WebPageState } from '../../../../src/state-manager.ts';
+import type { DocbotConfig } from '../config.ts';
+
+class Documentarian {
+  private provider: AIProvider;
+  private config: DocbotConfig;
+
+  constructor(provider: AIProvider, config: DocbotConfig = {}) {
+    this.provider = provider;
+    this.config = config;
+  }
+
+  async document(state: WebPageState, research: string): Promise<PageDocumentation> {
+    try {
+      return await this.generateDocumentation(state, research);
+    } catch (error) {
+      if (!this.shouldRetryWithSanitizedResearch(error)) {
+        throw error;
+      }
+
+      return this.generateDocumentation(state, this.sanitizeResearch(research), true);
+    }
+  }
+
+  private getSystemPrompt(): string {
+    const customPrompt = this.config.docs?.prompt;
+    let promptSuffix = '';
+    if (customPrompt) {
+      promptSuffix = customPrompt;
+    }
+
+    return dedent`
+    <role>
+    You are a product analyst preparing functional website documentation from UI research.
+    </role>
+
+    <task>
+    Convert exploratory UI research into a precise spec of what users can do on the current page.
+    Distinguish proven capabilities from assumptions.
+    Prefer accuracy over coverage.
+    </task>
+
+    <rules>
+    Only list capabilities that are grounded in the provided page research.
+    Put actions into "can" only when there is direct evidence in the page context.
+    Put actions into "might" only when the UI strongly suggests a capability but proof is incomplete.
+    Describe each action from the end-user perspective.
+    Be explicit about scope:
+    - one item
+    - list of items
+    - bulk operations
+    - all items
+    - page-level
+    Avoid implementation details, selectors, and QA wording.
+    Avoid duplicate actions with different phrasing.
+    </rules>
+
+    ${promptSuffix}
+    `;
+  }
+
+  private buildPrompt(state: WebPageState, research: string, simplified = false): string {
+    const headings = [state.h1, state.h2, state.h3, state.h4].filter(Boolean).join(' | ');
+    const links = (state.links || [])
+      .slice(0, 50)
+      .map((link) => `- ${link.title}: ${link.url}`)
+      .join('\n');
+    const simplificationNote = simplified
+      ? dedent`
+        <fallback_mode>
+        The research text was simplified because the original formatting was noisy.
+        Ignore malformed table syntax and rely only on clear, repeated signals.
+        Prefer fewer actions over speculative coverage.
+        </fallback_mode>
+        `
+      : '';
+
+    return dedent`
+    <page>
+    URL: ${state.url}
+    Title: ${state.title || ''}
+    Headings: ${headings}
+    </page>
+
+    <navigation_links>
+    ${links}
+    </navigation_links>
+
+    <research>
+    ${research}
+    </research>
+
+    ${simplificationNote}
+
+    <output_requirements>
+    Return structured data.
+    summary: short page purpose statement.
+    can: actions you are 100% sure are available on page.
+    might: actions that look possible but are not fully proven.
+    For each action provide:
+    - action: concise user-facing capability phrased as "user can ..."
+    - scope: one of one item, list of items, bulk operations, all items, page-level
+    - evidence: short reason based on visible UI or research
+    </output_requirements>
+    `;
+  }
+
+  private async generateDocumentation(state: WebPageState, research: string, simplified = false): Promise<PageDocumentation> {
+    const messages = [
+      {
+        role: 'system' as const,
+        content: this.getSystemPrompt(),
+      },
+      {
+        role: 'user' as const,
+        content: this.buildPrompt(state, research, simplified),
+      },
+    ];
+
+    const response = await this.provider.generateObject(messages, pageDocumentationSchema, undefined, {
+      agentName: 'documentarian',
+    });
+
+    return response.object as PageDocumentation;
+  }
+
+  private shouldRetryWithSanitizedResearch(error: unknown): boolean {
+    const message = error instanceof Error ? error.message : String(error);
+    return message.includes('Failed to generate JSON') || message.includes('failed_generation');
+  }
+
+  private sanitizeResearch(research: string): string {
+    const lines = research.split('\n');
+    const sanitized: string[] = [];
+
+    for (const line of lines) {
+      if (!line.trim()) {
+        sanitized.push(line);
+        continue;
+      }
+
+      if (!line.includes('|')) {
+        sanitized.push(line);
+        continue;
+      }
+
+      const pipeCount = (line.match(/\|/g) || []).length;
+      if (pipeCount < 2) {
+        continue;
+      }
+
+      if (line.includes('|------')) {
+        sanitized.push(line);
+        continue;
+      }
+
+      if (line.trim().startsWith('|') && pipeCount >= 4) {
+        sanitized.push(line);
+      }
+    }
+
+    return sanitized.join('\n');
+  }
+}
+
+const capabilitySchema = z.object({
+  action: z.string(),
+  scope: z.enum(['one item', 'list of items', 'bulk operations', 'all items', 'page-level']),
+  evidence: z.string(),
+});
+
+const pageDocumentationSchema = z.object({
+  summary: z.string(),
+  can: z.array(capabilitySchema),
+  might: z.array(capabilitySchema),
+});
+
+type PageDocumentation = z.infer<typeof pageDocumentationSchema>;
+
+export { Documentarian };
+export type { PageDocumentation };

package/boat/doc-collector/src/cli.ts

@@ -0,0 +1,119 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { Command } from 'commander';
+import { setPreserveConsoleLogs } from '../../../src/utils/logger.ts';
+import { DocBot, type DocbotOptions } from './docbot.ts';
+
+function buildOptions(options: any): DocbotOptions {
+  let session = options.session;
+  if (options.session === true) {
+    session = 'output/session.json';
+  }
+
+  return {
+    verbose: options.verbose || options.debug,
+    config: options.config,
+    path: options.path,
+    show: options.show,
+    headless: options.headless,
+    incognito: options.incognito,
+    session,
+    docsConfig: options.docsConfig,
+  };
+}
+
+function addCommonOptions(cmd: Command): Command {
+  return cmd
+    .option('-v, --verbose', 'Enable verbose logging')
+    .option('--debug', 'Enable debug logging')
+    .option('-c, --config <path>', 'Path to explorbot configuration file')
+    .option('--docs-config <path>', 'Path to doc collector configuration file')
+    .option('-p, --path <path>', 'Working directory path')
+    .option('-s, --show', 'Show browser window')
+    .option('--headless', 'Run browser in headless mode')
+    .option('--incognito', 'Run without recording experiences')
+    .option('--session [file]', 'Save/restore browser session from file');
+}
+
+export function createDocsCommands(name = 'docs'): Command {
+  const cmd = new Command(name);
+  cmd.description('AI-powered website documentation collector');
+
+  addCommonOptions(cmd.command('collect <path>').description('Crawl pages and generate documentation spec').option('--max-pages <count>', 'Maximum number of pages to document')).action(async (startPath, options) => {
+    setPreserveConsoleLogs(true);
+
+    try {
+      const bot = new DocBot({
+        ...buildOptions(options),
+        startUrl: startPath,
+      });
+      await bot.start();
+
+      let maxPages: number | undefined;
+      if (options.maxPages) {
+        maxPages = Number.parseInt(options.maxPages, 10);
+      }
+
+      const result = await bot.collect(startPath, { maxPages });
+
+      console.log(`\nDocumented ${result.pages.length} page(s)`);
+      console.log(`Skipped ${result.skipped.length} page(s)`);
+      console.log(`Spec index: ${result.indexPath}`);
+      console.log(`Pages dir: ${path.join(result.outputDir, 'pages')}`);
+
+      await bot.stop();
+      process.exit(0);
+    } catch (error) {
+      console.error('Failed:', error instanceof Error ? error.message : 'Unknown error');
+      process.exit(1);
+    }
+  });
+
+  cmd
+    .command('init')
+    .description('Initialize doc collector configuration')
+    .option('-f, --force', 'Overwrite existing config file')
+    .option('-p, --path <path>', 'Working directory for initialization')
+    .action(async (options) => {
+      const originalCwd = process.cwd();
+      if (options.path) {
+        const resolvedPath = path.resolve(options.path);
+        fs.mkdirSync(resolvedPath, { recursive: true });
+        process.chdir(resolvedPath);
+        console.log(`Working in: ${resolvedPath}`);
+      }
+
+      const configPath = path.resolve('docbot.config.ts');
+      if (fs.existsSync(configPath) && !options.force) {
+        console.log(`Config file already exists: ${configPath}`);
+        console.log('Use --force to overwrite.');
+        process.exit(1);
+      }
+
+      const configContent = `export default {
+          docs: {
+            maxPages: 100,
+            output: 'docs',
+            screenshot: true,
+            collapseDynamicPages: true,
+            scope: 'site',
+            includePaths: [],
+            excludePaths: [],
+            deniedPathSegments: ['callback', 'callbacks', 'logout', 'signout', 'sign_out', 'destroy', 'delete', 'remove'],
+            minCanActions: 1,
+            minInteractiveElements: 3,
+            // prompt: 'Add domain-specific documentation guidance here',
+          },
+        };
+      `;
+
+      fs.writeFileSync(configPath, configContent, 'utf8');
+      console.log(`Created: ${configPath}`);
+
+      if (process.cwd() !== originalCwd) {
+        process.chdir(originalCwd);
+      }
+    });
+
+  return cmd;
+}

package/boat/doc-collector/src/config.ts

@@ -0,0 +1,162 @@
+import { existsSync, readFileSync } from 'node:fs';
+import path, { resolve } from 'node:path';
+import { parseEnv } from 'node:util';
+import { ConfigParser } from '../../../src/config.ts';
+
+class DocbotConfigParser {
+  private static instance: DocbotConfigParser;
+  private config: DocbotConfig | null = null;
+  private configPath: string | null = null;
+
+  private constructor() {}
+
+  static getInstance(): DocbotConfigParser {
+    if (!DocbotConfigParser.instance) {
+      DocbotConfigParser.instance = new DocbotConfigParser();
+    }
+    return DocbotConfigParser.instance;
+  }
+
+  static loadEnv(filePath: string): void {
+    const resolved = resolve(filePath);
+    if (!existsSync(resolved)) return;
+    Object.assign(process.env, parseEnv(readFileSync(resolved, 'utf8')));
+  }
+
+  async loadConfig(options?: { config?: string; path?: string }): Promise<DocbotConfig> {
+    if (this.config && !options?.config && !options?.path) {
+      return this.config;
+    }
+
+    const originalCwd = process.cwd();
+    if (options?.path) {
+      process.chdir(resolve(options.path));
+    }
+
+    DocbotConfigParser.loadEnv('.env');
+
+    try {
+      const resolvedPath = options?.config || this.findConfigFile();
+      if (!resolvedPath) {
+        this.config = this.mergeWithDefaults({});
+        this.configPath = null;
+        return this.config;
+      }
+
+      const configModule = await this.loadConfigModule(resolvedPath);
+      const loadedConfig = configModule.default || configModule;
+      this.config = this.mergeWithDefaults(loadedConfig || {});
+      this.configPath = resolvedPath;
+      return this.config;
+    } finally {
+      if (options?.path && originalCwd !== process.cwd()) {
+        process.chdir(originalCwd);
+      }
+    }
+  }
+
+  getConfig(): DocbotConfig {
+    if (this.config) {
+      return this.config;
+    }
+    return this.mergeWithDefaults({});
+  }
+
+  getConfigPath(): string | null {
+    return this.configPath;
+  }
+
+  getOutputDir(): string {
+    const outputDir = ConfigParser.getInstance().getOutputDir();
+    const docsOutput = this.getConfig().docs?.output;
+    if (!docsOutput) {
+      return path.join(outputDir, 'docs');
+    }
+    return path.join(outputDir, docsOutput);
+  }
+
+  private findConfigFile(): string | null {
+    const possiblePaths = ['docbot.config.js', 'docbot.config.mjs', 'docbot.config.ts'];
+
+    for (const candidate of possiblePaths) {
+      const fullPath = resolve(process.cwd(), candidate);
+      if (existsSync(fullPath)) {
+        return fullPath;
+      }
+    }
+
+    return null;
+  }
+
+  private async loadConfigModule(configPath: string): Promise<any> {
+    const ext = configPath.split('.').pop();
+
+    if (ext === 'ts') {
+      try {
+        return await import(configPath);
+      } catch {
+        const require = (await import('node:module')).createRequire(import.meta.url);
+        return require(configPath);
+      }
+    }
+
+    if (ext === 'js' || ext === 'mjs') {
+      return await import(configPath);
+    }
+
+    return JSON.parse(readFileSync(configPath, 'utf8'));
+  }
+
+  private mergeWithDefaults(config: Partial<DocbotConfig>): DocbotConfig {
+    return this.deepMerge(
+      {
+        docs: {
+          maxPages: 100,
+          output: 'docs',
+          screenshot: true,
+          collapseDynamicPages: true,
+          scope: 'site',
+          includePaths: [],
+          excludePaths: [],
+          deniedPathSegments: ['callback', 'callbacks', 'logout', 'signout', 'sign_out', 'destroy', 'delete', 'remove'],
+          minCanActions: 1,
+          minInteractiveElements: 3,
+        },
+      },
+      config
+    );
+  }
+
+  private deepMerge(target: any, source: any): any {
+    const result = { ...target };
+
+    for (const key in source) {
+      if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key]) && source[key].constructor === Object) {
+        result[key] = this.deepMerge(result[key] || {}, source[key]);
+        continue;
+      }
+      result[key] = source[key];
+    }
+
+    return result;
+  }
+}
+
+interface DocbotConfig {
+  docs?: {
+    maxPages?: number;
+    output?: string;
+    screenshot?: boolean;
+    prompt?: string;
+    collapseDynamicPages?: boolean;
+    scope?: 'site' | 'section' | 'subtree';
+    includePaths?: string[];
+    excludePaths?: string[];
+    deniedPathSegments?: string[];
+    minCanActions?: number;
+    minInteractiveElements?: number;
+  };
+}
+
+export { DocbotConfigParser };
+export type { DocbotConfig };