explorbot 0.1.17 → 0.1.18

package/dist/bin/explorbot-cli.js

@@ -748,5 +748,7 @@ program
     });
 });
 import { createApiCommands } from "../boat/api-tester/src/cli.js";
+import { createDocsCommands } from "../boat/doc-collector/src/cli.js";
 program.addCommand(createApiCommands('api'));
+program.addCommand(createDocsCommands('docs'));
 program.parse();

package/dist/boat/doc-collector/bin/doc-collector-cli.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env bun
+import { createDocsCommands } from "../src/cli.js";
+const program = createDocsCommands('doc-collector');
+program.parse();

package/dist/boat/doc-collector/src/ai/documentarian.js

@@ -0,0 +1,157 @@
+import dedent from 'dedent';
+import { z } from 'zod';
+class Documentarian {
+    provider;
+    config;
+    constructor(provider, config = {}) {
+        this.provider = provider;
+        this.config = config;
+    }
+    async document(state, research) {
+        try {
+            return await this.generateDocumentation(state, research);
+        }
+        catch (error) {
+            if (!this.shouldRetryWithSanitizedResearch(error)) {
+                throw error;
+            }
+            return this.generateDocumentation(state, this.sanitizeResearch(research), true);
+        }
+    }
+    getSystemPrompt() {
+        const customPrompt = this.config.docs?.prompt;
+        let promptSuffix = '';
+        if (customPrompt) {
+            promptSuffix = customPrompt;
+        }
+        return dedent `
+    <role>
+    You are a product analyst preparing functional website documentation from UI research.
+    </role>
+
+    <task>
+    Convert exploratory UI research into a precise spec of what users can do on the current page.
+    Distinguish proven capabilities from assumptions.
+    Prefer accuracy over coverage.
+    </task>
+
+    <rules>
+    Only list capabilities that are grounded in the provided page research.
+    Put actions into "can" only when there is direct evidence in the page context.
+    Put actions into "might" only when the UI strongly suggests a capability but proof is incomplete.
+    Describe each action from the end-user perspective.
+    Be explicit about scope:
+    - one item
+    - list of items
+    - bulk operations
+    - all items
+    - page-level
+    Avoid implementation details, selectors, and QA wording.
+    Avoid duplicate actions with different phrasing.
+    </rules>
+
+    ${promptSuffix}
+    `;
+    }
+    buildPrompt(state, research, simplified = false) {
+        const headings = [state.h1, state.h2, state.h3, state.h4].filter(Boolean).join(' | ');
+        const links = (state.links || [])
+            .slice(0, 50)
+            .map((link) => `- ${link.title}: ${link.url}`)
+            .join('\n');
+        const simplificationNote = simplified
+            ? dedent `
+        <fallback_mode>
+        The research text was simplified because the original formatting was noisy.
+        Ignore malformed table syntax and rely only on clear, repeated signals.
+        Prefer fewer actions over speculative coverage.
+        </fallback_mode>
+        `
+            : '';
+        return dedent `
+    <page>
+    URL: ${state.url}
+    Title: ${state.title || ''}
+    Headings: ${headings}
+    </page>
+
+    <navigation_links>
+    ${links}
+    </navigation_links>
+
+    <research>
+    ${research}
+    </research>
+
+    ${simplificationNote}
+
+    <output_requirements>
+    Return structured data.
+    summary: short page purpose statement.
+    can: actions you are 100% sure are available on page.
+    might: actions that look possible but are not fully proven.
+    For each action provide:
+    - action: concise user-facing capability phrased as "user can ..."
+    - scope: one of one item, list of items, bulk operations, all items, page-level
+    - evidence: short reason based on visible UI or research
+    </output_requirements>
+    `;
+    }
+    async generateDocumentation(state, research, simplified = false) {
+        const messages = [
+            {
+                role: 'system',
+                content: this.getSystemPrompt(),
+            },
+            {
+                role: 'user',
+                content: this.buildPrompt(state, research, simplified),
+            },
+        ];
+        const response = await this.provider.generateObject(messages, pageDocumentationSchema, undefined, {
+            agentName: 'documentarian',
+        });
+        return response.object;
+    }
+    shouldRetryWithSanitizedResearch(error) {
+        const message = error instanceof Error ? error.message : String(error);
+        return message.includes('Failed to generate JSON') || message.includes('failed_generation');
+    }
+    sanitizeResearch(research) {
+        const lines = research.split('\n');
+        const sanitized = [];
+        for (const line of lines) {
+            if (!line.trim()) {
+                sanitized.push(line);
+                continue;
+            }
+            if (!line.includes('|')) {
+                sanitized.push(line);
+                continue;
+            }
+            const pipeCount = (line.match(/\|/g) || []).length;
+            if (pipeCount < 2) {
+                continue;
+            }
+            if (line.includes('|------')) {
+                sanitized.push(line);
+                continue;
+            }
+            if (line.trim().startsWith('|') && pipeCount >= 4) {
+                sanitized.push(line);
+            }
+        }
+        return sanitized.join('\n');
+    }
+}
+const capabilitySchema = z.object({
+    action: z.string(),
+    scope: z.enum(['one item', 'list of items', 'bulk operations', 'all items', 'page-level']),
+    evidence: z.string(),
+});
+const pageDocumentationSchema = z.object({
+    summary: z.string(),
+    can: z.array(capabilitySchema),
+    might: z.array(capabilitySchema),
+});
+export { Documentarian };

package/dist/boat/doc-collector/src/cli.js

@@ -0,0 +1,104 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { Command } from 'commander';
+import { setPreserveConsoleLogs } from "../../../src/utils/logger.js";
+import { DocBot } from "./docbot.js";
+function buildOptions(options) {
+    let session = options.session;
+    if (options.session === true) {
+        session = 'output/session.json';
+    }
+    return {
+        verbose: options.verbose || options.debug,
+        config: options.config,
+        path: options.path,
+        show: options.show,
+        headless: options.headless,
+        incognito: options.incognito,
+        session,
+        docsConfig: options.docsConfig,
+    };
+}
+function addCommonOptions(cmd) {
+    return cmd
+        .option('-v, --verbose', 'Enable verbose logging')
+        .option('--debug', 'Enable debug logging')
+        .option('-c, --config <path>', 'Path to explorbot configuration file')
+        .option('--docs-config <path>', 'Path to doc collector configuration file')
+        .option('-p, --path <path>', 'Working directory path')
+        .option('-s, --show', 'Show browser window')
+        .option('--headless', 'Run browser in headless mode')
+        .option('--incognito', 'Run without recording experiences')
+        .option('--session [file]', 'Save/restore browser session from file');
+}
+export function createDocsCommands(name = 'docs') {
+    const cmd = new Command(name);
+    cmd.description('AI-powered website documentation collector');
+    addCommonOptions(cmd.command('collect <path>').description('Crawl pages and generate documentation spec').option('--max-pages <count>', 'Maximum number of pages to document')).action(async (startPath, options) => {
+        setPreserveConsoleLogs(true);
+        try {
+            const bot = new DocBot({
+                ...buildOptions(options),
+                startUrl: startPath,
+            });
+            await bot.start();
+            let maxPages;
+            if (options.maxPages) {
+                maxPages = Number.parseInt(options.maxPages, 10);
+            }
+            const result = await bot.collect(startPath, { maxPages });
+            console.log(`\nDocumented ${result.pages.length} page(s)`);
+            console.log(`Skipped ${result.skipped.length} page(s)`);
+            console.log(`Spec index: ${result.indexPath}`);
+            console.log(`Pages dir: ${path.join(result.outputDir, 'pages')}`);
+            await bot.stop();
+            process.exit(0);
+        }
+        catch (error) {
+            console.error('Failed:', error instanceof Error ? error.message : 'Unknown error');
+            process.exit(1);
+        }
+    });
+    cmd
+        .command('init')
+        .description('Initialize doc collector configuration')
+        .option('-f, --force', 'Overwrite existing config file')
+        .option('-p, --path <path>', 'Working directory for initialization')
+        .action(async (options) => {
+        const originalCwd = process.cwd();
+        if (options.path) {
+            const resolvedPath = path.resolve(options.path);
+            fs.mkdirSync(resolvedPath, { recursive: true });
+            process.chdir(resolvedPath);
+            console.log(`Working in: ${resolvedPath}`);
+        }
+        const configPath = path.resolve('docbot.config.ts');
+        if (fs.existsSync(configPath) && !options.force) {
+            console.log(`Config file already exists: ${configPath}`);
+            console.log('Use --force to overwrite.');
+            process.exit(1);
+        }
+        const configContent = `export default {
+          docs: {
+            maxPages: 100,
+            output: 'docs',
+            screenshot: true,
+            collapseDynamicPages: true,
+            scope: 'site',
+            includePaths: [],
+            excludePaths: [],
+            deniedPathSegments: ['callback', 'callbacks', 'logout', 'signout', 'sign_out', 'destroy', 'delete', 'remove'],
+            minCanActions: 1,
+            minInteractiveElements: 3,
+            // prompt: 'Add domain-specific documentation guidance here',
+          },
+        };
+      `;
+        fs.writeFileSync(configPath, configContent, 'utf8');
+        console.log(`Created: ${configPath}`);
+        if (process.cwd() !== originalCwd) {
+            process.chdir(originalCwd);
+        }
+    });
+    return cmd;
+}

package/dist/boat/doc-collector/src/config.js

@@ -0,0 +1,129 @@
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+import { existsSync, readFileSync } from 'node:fs';
+import path, { resolve } from 'node:path';
+import { parseEnv } from 'node:util';
+import { ConfigParser } from "../../../src/config.js";
+class DocbotConfigParser {
+    static instance;
+    config = null;
+    configPath = null;
+    constructor() { }
+    static getInstance() {
+        if (!DocbotConfigParser.instance) {
+            DocbotConfigParser.instance = new DocbotConfigParser();
+        }
+        return DocbotConfigParser.instance;
+    }
+    static loadEnv(filePath) {
+        const resolved = resolve(filePath);
+        if (!existsSync(resolved))
+            return;
+        Object.assign(process.env, parseEnv(readFileSync(resolved, 'utf8')));
+    }
+    async loadConfig(options) {
+        if (this.config && !options?.config && !options?.path) {
+            return this.config;
+        }
+        const originalCwd = process.cwd();
+        if (options?.path) {
+            process.chdir(resolve(options.path));
+        }
+        DocbotConfigParser.loadEnv('.env');
+        try {
+            const resolvedPath = options?.config || this.findConfigFile();
+            if (!resolvedPath) {
+                this.config = this.mergeWithDefaults({});
+                this.configPath = null;
+                return this.config;
+            }
+            const configModule = await this.loadConfigModule(resolvedPath);
+            const loadedConfig = configModule.default || configModule;
+            this.config = this.mergeWithDefaults(loadedConfig || {});
+            this.configPath = resolvedPath;
+            return this.config;
+        }
+        finally {
+            if (options?.path && originalCwd !== process.cwd()) {
+                process.chdir(originalCwd);
+            }
+        }
+    }
+    getConfig() {
+        if (this.config) {
+            return this.config;
+        }
+        return this.mergeWithDefaults({});
+    }
+    getConfigPath() {
+        return this.configPath;
+    }
+    getOutputDir() {
+        const outputDir = ConfigParser.getInstance().getOutputDir();
+        const docsOutput = this.getConfig().docs?.output;
+        if (!docsOutput) {
+            return path.join(outputDir, 'docs');
+        }
+        return path.join(outputDir, docsOutput);
+    }
+    findConfigFile() {
+        const possiblePaths = ['docbot.config.js', 'docbot.config.mjs', 'docbot.config.ts'];
+        for (const candidate of possiblePaths) {
+            const fullPath = resolve(process.cwd(), candidate);
+            if (existsSync(fullPath)) {
+                return fullPath;
+            }
+        }
+        return null;
+    }
+    async loadConfigModule(configPath) {
+        const ext = configPath.split('.').pop();
+        if (ext === 'ts') {
+            try {
+                return await import(__rewriteRelativeImportExtension(configPath));
+            }
+            catch {
+                const require = (await import('node:module')).createRequire(import.meta.url);
+                return require(configPath);
+            }
+        }
+        if (ext === 'js' || ext === 'mjs') {
+            return await import(__rewriteRelativeImportExtension(configPath));
+        }
+        return JSON.parse(readFileSync(configPath, 'utf8'));
+    }
+    mergeWithDefaults(config) {
+        return this.deepMerge({
+            docs: {
+                maxPages: 100,
+                output: 'docs',
+                screenshot: true,
+                collapseDynamicPages: true,
+                scope: 'site',
+                includePaths: [],
+                excludePaths: [],
+                deniedPathSegments: ['callback', 'callbacks', 'logout', 'signout', 'sign_out', 'destroy', 'delete', 'remove'],
+                minCanActions: 1,
+                minInteractiveElements: 3,
+            },
+        }, config);
+    }
+    deepMerge(target, source) {
+        const result = { ...target };
+        for (const key in source) {
+            if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key]) && source[key].constructor === Object) {
+                result[key] = this.deepMerge(result[key] || {}, source[key]);
+                continue;
+            }
+            result[key] = source[key];
+        }
+        return result;
+    }
+}
+export { DocbotConfigParser };