explorbot 0.1.17 → 0.1.19

package/dist/boat/doc-collector/src/docbot.js

@@ -0,0 +1,326 @@
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { ExplorBot } from "../../../src/explorbot.js";
+import { normalizeUrl } from "../../../src/state-manager.js";
+import { sanitizeFilename } from "../../../src/utils/strings.js";
+import { tag } from "../../../src/utils/logger.js";
+import { Documentarian } from "./ai/documentarian.js";
+import { DocbotConfigParser } from "./config.js";
+import { renderPageDocumentation, renderSpecIndex } from "./docs-renderer.js";
+import { getDocPageKey, shouldCrawlDocPath } from "./path-filter.js";
+import { extractResearchNavigationTargets } from "./research-navigation.js";
+class DocBot {
+    explorBot;
+    configParser;
+    config = {};
+    documentarian;
+    options;
+    scopeRoot = '/';
+    constructor(options = {}) {
+        this.options = options;
+        const baseUrl = this.extractAbsoluteBaseUrl(options.startUrl || '/');
+        this.explorBot = new ExplorBot({
+            baseUrl,
+            verbose: options.verbose,
+            config: options.config,
+            path: options.path,
+            show: options.show,
+            headless: options.headless,
+            incognito: options.incognito,
+            session: options.session,
+        });
+        this.configParser = DocbotConfigParser.getInstance();
+    }
+    async start() {
+        await this.explorBot.start();
+        this.config = await this.configParser.loadConfig({
+            config: this.options.docsConfig,
+            path: this.options.path,
+        });
+        this.documentarian = new Documentarian(this.explorBot.getProvider(), this.config);
+        this.ensureDirectory(this.configParser.getOutputDir());
+        this.ensureDirectory(this.getPagesDir());
+    }
+    async stop() {
+        await this.explorBot.stop();
+    }
+    async collect(startPath, opts = {}) {
+        const effectiveStartPath = this.normalizeStartPath(startPath);
+        this.scopeRoot = this.getScopeRoot(effectiveStartPath);
+        const effectiveMaxPages = this.getMaxPages(opts.maxPages);
+        const queue = [];
+        const queued = new Set();
+        const documented = new Set();
+        const pages = [];
+        const skipped = [];
+        const baseUrl = this.explorBot.getConfig().playwright.url;
+        this.enqueuePath(effectiveStartPath, queue, queued);
+        while (queue.length > 0 && pages.length < effectiveMaxPages) {
+            const target = queue.shift();
+            if (!target) {
+                continue;
+            }
+            const targetKey = this.getPageKey(target);
+            if (documented.has(targetKey)) {
+                continue;
+            }
+            const stateManager = this.explorBot.getExplorer().getStateManager();
+            if (stateManager.hasVisitedState(target)) {
+                continue;
+            }
+            try {
+                tag('info').log(`Collecting docs for ${this.toDisplayUrl(target, baseUrl)}`);
+                await this.explorBot.visit(target);
+                if (stateManager.isInDeadLoop()) {
+                    tag('warning').log('Dead loop detected during docs crawl, stopping collection');
+                    skipped.push({
+                        url: target,
+                        reason: 'dead loop detected during crawl',
+                    });
+                    break;
+                }
+                const state = this.explorBot.getCurrentState();
+                if (!state) {
+                    skipped.push({
+                        url: target,
+                        reason: 'page state was not captured after navigation',
+                    });
+                    continue;
+                }
+                const pageKey = this.getPageKey(state.url || target);
+                if (documented.has(pageKey)) {
+                    continue;
+                }
+                const research = await this.explorBot.agentResearcher().research(state, {
+                    screenshot: this.shouldUseScreenshots(),
+                    force: true,
+                });
+                const documentation = await this.documentarian.document(state, research);
+                const lowSignalReason = this.getLowSignalReason(documentation, research);
+                if (lowSignalReason) {
+                    skipped.push({
+                        url: state.url,
+                        reason: lowSignalReason,
+                    });
+                    documented.add(pageKey);
+                    continue;
+                }
+                const filePath = this.savePageDocumentation(state, documentation);
+                pages.push({
+                    url: state.url,
+                    title: state.title || '',
+                    summary: documentation.summary,
+                    canCount: documentation.can.length,
+                    mightCount: documentation.might.length,
+                    canActions: documentation.can.map((item) => item.action),
+                    mightActions: documentation.might.map((item) => item.action),
+                    filePath,
+                });
+                documented.add(pageKey);
+                const nextPaths = this.extractNextPaths(state, baseUrl, research);
+                for (const nextPath of nextPaths) {
+                    if (documented.has(this.getPageKey(nextPath))) {
+                        continue;
+                    }
+                    if (stateManager.hasVisitedState(nextPath)) {
+                        continue;
+                    }
+                    this.enqueuePath(nextPath, queue, queued);
+                }
+            }
+            catch (error) {
+                const reason = error instanceof Error ? error.message : String(error);
+                tag('warning').log(`Skipping ${target}: ${reason}`);
+                skipped.push({
+                    url: target,
+                    reason,
+                });
+            }
+        }
+        const indexPath = this.saveIndex(effectiveStartPath, pages, skipped, effectiveMaxPages);
+        return {
+            pages,
+            skipped,
+            indexPath,
+            outputDir: this.configParser.getOutputDir(),
+        };
+    }
+    getMaxPages(override) {
+        if (override && override > 0) {
+            return override;
+        }
+        const configured = this.config.docs?.maxPages;
+        if (configured && configured > 0) {
+            return configured;
+        }
+        return 100;
+    }
+    shouldUseScreenshots() {
+        const screenshot = this.config.docs?.screenshot;
+        if (screenshot === false) {
+            return false;
+        }
+        return true;
+    }
+    extractNextPaths(state, baseUrl, research) {
+        const paths = [];
+        const seen = new Set();
+        for (const link of state.links || []) {
+            const nextPath = this.resolveLink(link, baseUrl);
+            if (!nextPath) {
+                continue;
+            }
+            if (!shouldCrawlDocPath(nextPath, this.config)) {
+                continue;
+            }
+            if (!this.isInScope(nextPath)) {
+                continue;
+            }
+            if (seen.has(nextPath)) {
+                continue;
+            }
+            seen.add(nextPath);
+            paths.push(nextPath);
+        }
+        for (const target of extractResearchNavigationTargets(state, research)) {
+            if (!shouldCrawlDocPath(target, this.config)) {
+                continue;
+            }
+            if (!this.isInScope(target)) {
+                continue;
+            }
+            if (seen.has(target)) {
+                continue;
+            }
+            seen.add(target);
+            paths.push(target);
+        }
+        return paths;
+    }
+    resolveLink(link, baseUrl) {
+        let resolved;
+        try {
+            resolved = new URL(link.url, baseUrl);
+        }
+        catch {
+            return null;
+        }
+        const base = new URL(baseUrl);
+        if (resolved.origin !== base.origin) {
+            return null;
+        }
+        const pathName = resolved.pathname || '/';
+        return `${pathName}${resolved.search}${resolved.hash}`;
+    }
+    toDisplayUrl(target, baseUrl) {
+        try {
+            return new URL(target, baseUrl).toString();
+        }
+        catch {
+            return target;
+        }
+    }
+    enqueuePath(inputPath, queue, queued) {
+        const normalized = normalizeUrl(inputPath);
+        const pageKey = this.getPageKey(inputPath);
+        if (queued.has(pageKey)) {
+            return;
+        }
+        queued.add(pageKey);
+        if (!inputPath.startsWith('/')) {
+            queue.push(`/${normalized}`);
+            return;
+        }
+        queue.push(inputPath);
+    }
+    getPageKey(pageUrl) {
+        return getDocPageKey(pageUrl, this.config);
+    }
+    normalizeStartPath(startPath) {
+        try {
+            const parsed = new URL(startPath);
+            return `${parsed.pathname || '/'}${parsed.search}${parsed.hash}`;
+        }
+        catch {
+            return startPath;
+        }
+    }
+    extractAbsoluteBaseUrl(startPath) {
+        try {
+            const parsed = new URL(startPath);
+            return parsed.origin;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    isInScope(target) {
+        const normalized = this.normalizeStartPath(target);
+        const scope = this.config.docs?.scope || 'site';
+        if (scope === 'site') {
+            return true;
+        }
+        if (scope === 'subtree') {
+            return normalized === this.scopeRoot || normalized.startsWith(`${this.scopeRoot}/`);
+        }
+        if (scope === 'section') {
+            return normalized === this.scopeRoot || normalized.startsWith(`${this.scopeRoot}/`) || normalized.startsWith(`${this.scopeRoot}-`);
+        }
+        return true;
+    }
+    getScopeRoot(startPath) {
+        const normalized = this.normalizeStartPath(startPath);
+        const parts = normalized.split('/').filter(Boolean);
+        if (parts.length === 0) {
+            return '/';
+        }
+        if (parts.length >= 4) {
+            return `/${parts.slice(0, 4).join('/')}`;
+        }
+        return `/${parts.join('/')}`;
+    }
+    getLowSignalReason(documentation, research) {
+        const minCanActions = this.config.docs?.minCanActions ?? 1;
+        const minInteractiveElements = this.config.docs?.minInteractiveElements ?? 3;
+        if (documentation.can.length >= minCanActions) {
+            return null;
+        }
+        const interactiveCount = this.countInteractiveElements(research);
+        if (interactiveCount >= minInteractiveElements) {
+            return null;
+        }
+        return `low-signal page: only ${documentation.can.length} proven actions and ${interactiveCount} interactive elements`;
+    }
+    countInteractiveElements(research) {
+        const matches = [...research.matchAll(/\((\d+) elements?\)/g)];
+        return matches.reduce((sum, match) => sum + Number.parseInt(match[1], 10), 0);
+    }
+    savePageDocumentation(state, documentation) {
+        const pagePath = this.getPageFilePath(state.url);
+        writeFileSync(pagePath, renderPageDocumentation(state, documentation), 'utf8');
+        return pagePath;
+    }
+    saveIndex(startPath, pages, skipped, maxPages) {
+        const indexPath = path.join(this.configParser.getOutputDir(), 'spec.md');
+        writeFileSync(indexPath, renderSpecIndex(this.configParser.getOutputDir(), startPath, pages, skipped, maxPages), 'utf8');
+        return indexPath;
+    }
+    getPagesDir() {
+        return path.join(this.configParser.getOutputDir(), 'pages');
+    }
+    getPageFilePath(pageUrl) {
+        const normalized = normalizeUrl(pageUrl || '/');
+        const baseName = sanitizeFilename(normalized || 'root');
+        if (baseName) {
+            return path.join(this.getPagesDir(), `${baseName}.md`);
+        }
+        return path.join(this.getPagesDir(), 'root.md');
+    }
+    ensureDirectory(dirPath) {
+        if (existsSync(dirPath)) {
+            return;
+        }
+        mkdirSync(dirPath, { recursive: true });
+    }
+}
+export { DocBot };

package/dist/boat/doc-collector/src/docs-renderer.js

@@ -0,0 +1,141 @@
+import path from 'node:path';
+function renderPageDocumentation(state, documentation) {
+    const lines = [];
+    lines.push(`# ${state.url}`);
+    lines.push('');
+    if (state.title) {
+        lines.push(`Title: ${normalizeInlineText(state.title)}`);
+        lines.push('');
+    }
+    lines.push('## Purpose');
+    lines.push('');
+    lines.push(ensureSentence(documentation.summary));
+    lines.push('');
+    lines.push('## User Can');
+    lines.push('');
+    if (documentation.can.length === 0) {
+        lines.push('- No proven actions were identified from the collected research.');
+        lines.push('');
+    }
+    for (const item of documentation.can) {
+        lines.push(`- ${normalizeAction(item.action)} -> ${item.scope}`);
+        lines.push(`  Proof: ${ensureSentence(item.evidence)}`);
+    }
+    if (documentation.can.length > 0) {
+        lines.push('');
+    }
+    lines.push('## User Might');
+    lines.push('');
+    if (documentation.might.length === 0) {
+        lines.push('- No assumption-based actions were identified.');
+        lines.push('');
+    }
+    for (const item of documentation.might) {
+        lines.push(`- ${normalizeAction(item.action, 'might')} -> ${item.scope}`);
+        lines.push(`  Signal: ${ensureSentence(item.evidence)}`);
+    }
+    if (documentation.might.length > 0) {
+        lines.push('');
+    }
+    return `${lines.join('\n').trimEnd()}\n`;
+}
+function renderSpecIndex(outputDir, startPath, pages, skipped, maxPages) {
+    const lines = [];
+    lines.push('# Website Spec');
+    lines.push('');
+    lines.push('## Overview');
+    lines.push('');
+    lines.push(`Start page: ${startPath}`);
+    lines.push(`Pages documented: ${pages.length}`);
+    lines.push(`Pages skipped: ${skipped.length}`);
+    lines.push(`Max pages: ${maxPages}`);
+    lines.push('');
+    lines.push('## Pages');
+    lines.push('');
+    if (pages.length === 0) {
+        lines.push('- No pages were documented.');
+        lines.push('');
+    }
+    for (const page of pages) {
+        const relativeFile = path.relative(outputDir, page.filePath).replaceAll('\\', '/');
+        lines.push(`### [${page.url}](${relativeFile})`);
+        lines.push('');
+        lines.push(`Purpose: ${ensureSentence(page.summary)}`);
+        lines.push(`Proven actions: ${page.canCount}`);
+        lines.push(`Possible actions: ${page.mightCount}`);
+        if (page.title) {
+            lines.push(`Title: ${normalizeInlineText(page.title)}`);
+        }
+        lines.push('');
+        if (page.canActions.length > 0) {
+            lines.push('User Can:');
+            for (const action of page.canActions) {
+                lines.push(`- ${normalizeAction(action, 'can')}`);
+            }
+            lines.push('');
+        }
+        if (page.mightActions.length > 0) {
+            lines.push('User Might:');
+            for (const action of page.mightActions) {
+                lines.push(`- ${normalizeAction(action, 'might')}`);
+            }
+            lines.push('');
+        }
+    }
+    if (skipped.length > 0) {
+        lines.push('## Skipped');
+        lines.push('');
+        for (const page of skipped) {
+            lines.push(`- ${page.url}. Reason: ${ensureSentence(page.reason)}`);
+        }
+        lines.push('');
+    }
+    return `${lines.join('\n').trimEnd()}\n`;
+}
+function normalizeAction(action, kind = 'can') {
+    const trimmed = normalizeInlineText(action);
+    if (!trimmed) {
+        return 'user can interact with this page';
+    }
+    const normalized = ensureSentence(trimmed).slice(0, -1);
+    const lower = normalized.toLowerCase();
+    if (kind === 'can') {
+        if (lower.startsWith('user can ')) {
+            return normalized;
+        }
+        if (lower.startsWith('can ')) {
+            return `user can ${normalized.slice(4)}`;
+        }
+        if (lower.startsWith('user might ')) {
+            return `user can ${normalized.slice(11)}`;
+        }
+        return `user can ${normalized}`;
+    }
+    if (lower.startsWith('user might ')) {
+        return normalized;
+    }
+    if (lower.startsWith('might ')) {
+        return `user might ${normalized.slice(6)}`;
+    }
+    if (lower.startsWith('user can ')) {
+        return `user might ${normalized.slice(9)}`;
+    }
+    if (lower.startsWith('can ')) {
+        return `user might ${normalized.slice(4)}`;
+    }
+    return `user might ${normalized}`;
+}
+function ensureSentence(text) {
+    const trimmed = normalizeInlineText(text);
+    if (!trimmed) {
+        return '';
+    }
+    if (/[.!?]$/.test(trimmed)) {
+        return trimmed;
+    }
+    return `${trimmed}.`;
+}
+function normalizeInlineText(text) {
+    return text.normalize('NFKC').replace(/\s+/g, ' ').trim();
+}
+export { renderPageDocumentation, renderSpecIndex, ensureSentence, normalizeAction };

package/dist/boat/doc-collector/src/path-filter.js

@@ -0,0 +1,35 @@
+import { normalizeUrl } from "../../../src/state-manager.js";
+import { matchesUrl, generalizeUrl } from "../../../src/utils/url-matcher.js";
+const DEFAULT_DENIED_PATH_SEGMENTS = ['callback', 'callbacks', 'logout', 'signout', 'sign_out', 'destroy', 'delete', 'remove'];
+export function shouldCrawlDocPath(nextPath, config = {}) {
+    const parsed = new URL(nextPath, 'http://localhost');
+    const segments = parsed.pathname
+        .split('/')
+        .map((segment) => segment.trim().toLowerCase())
+        .filter(Boolean);
+    const normalizedPath = parsed.pathname || '/';
+    const includePaths = config.docs?.includePaths || [];
+    if (includePaths.length > 0) {
+        return includePaths.some((pattern) => matchesUrl(pattern, normalizedPath));
+    }
+    const excludePaths = config.docs?.excludePaths || [];
+    if (excludePaths.some((pattern) => matchesUrl(pattern, normalizedPath))) {
+        return false;
+    }
+    if (segments.length === 0) {
+        return true;
+    }
+    const terminalActions = new Set((config.docs?.deniedPathSegments || DEFAULT_DENIED_PATH_SEGMENTS).map((segment) => segment.trim().toLowerCase()).filter(Boolean));
+    if (segments.some((segment) => terminalActions.has(segment))) {
+        return false;
+    }
+    return true;
+}
+export function getDocPageKey(pageUrl, config = {}) {
+    const normalized = normalizeUrl(pageUrl || '/');
+    const path = normalized.startsWith('/') ? normalized : `/${normalized}`;
+    if (config.docs?.collapseDynamicPages === false) {
+        return normalizeUrl(path);
+    }
+    return normalizeUrl(generalizeUrl(path));
+}

package/dist/boat/doc-collector/src/research-navigation.js

@@ -0,0 +1,71 @@
+import { parseResearchSections } from "../../../src/ai/researcher/parser.js";
+const OPEN_API_TAG_SELECTOR_PATTERN = /api-\d+\/tag\/([a-z0-9-]+)(?:["'#/\]\s]|$)/i;
+const OPEN_API_NAVIGATION_SECTION_KEYWORDS = ['navigation', 'menu'];
+export function extractResearchNavigationTargets(state, research) {
+    const currentUrl = state.url || '/';
+    const sections = parseResearchSections(research);
+    const targets = [];
+    const seen = new Set();
+    for (const section of sections) {
+        const sectionName = section.name.toLowerCase();
+        if (!OPEN_API_NAVIGATION_SECTION_KEYWORDS.some((keyword) => sectionName.includes(keyword))) {
+            continue;
+        }
+        for (const element of section.elements) {
+            const target = extractNavigationTarget(currentUrl, element);
+            if (!target || seen.has(target)) {
+                continue;
+            }
+            seen.add(target);
+            targets.push(target);
+        }
+    }
+    return targets;
+}
+function extractNavigationTarget(currentUrl, element) {
+    const openApiTagFromCss = extractOpenApiTagHashFromCss(element.css);
+    if (openApiTagFromCss) {
+        return buildSamePageHashTarget(currentUrl, openApiTagFromCss);
+    }
+    if (!currentUrl.includes('#tag/')) {
+        return null;
+    }
+    const inferredOpenApiTag = inferOpenApiTagSlugFromLabel(element.name);
+    if (!inferredOpenApiTag) {
+        return null;
+    }
+    return buildSamePageHashTarget(currentUrl, `tag/${inferredOpenApiTag}`);
+}
+function extractOpenApiTagHashFromCss(css) {
+    if (!css) {
+        return null;
+    }
+    const normalizedSelector = css.replaceAll('\\/', '/');
+    const match = normalizedSelector.match(OPEN_API_TAG_SELECTOR_PATTERN);
+    if (!match?.[1]) {
+        return null;
+    }
+    return `tag/${match[1].toLowerCase()}`;
+}
+function inferOpenApiTagSlugFromLabel(name) {
+    const cleanedLabel = name
+        .replace(/^'+|'+$/g, '')
+        .replace(/\(expanded\)|\(collapsed\)|open group|close group|show more/gi, '')
+        .trim();
+    if (!cleanedLabel.includes('/')) {
+        return null;
+    }
+    const slug = cleanedLabel
+        .split('/')
+        .map((part) => part.trim().toLowerCase())
+        .filter(Boolean)
+        .join('-')
+        .replace(/[^a-z0-9-]+/g, '-')
+        .replace(/-+/g, '-')
+        .replace(/^-|-$/g, '');
+    return slug || null;
+}
+function buildSamePageHashTarget(currentUrl, hashPath) {
+    const [baseWithSearch] = currentUrl.split('#');
+    return `${baseWithSearch}#${hashPath}`;
+}

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.1.17",
+  "version": "0.1.19",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",
@@ -20,6 +20,9 @@
     "src/**/*.tsx",
     "bin/**/*.ts",
     "boat/api-tester/src/**/*.ts",
+    "boat/doc-collector/src/**/*.ts",
+    "boat/doc-collector/bin/**/*.ts",
+    "boat/doc-collector/package.json",
     "rules/",
     "assets/sample-files/"
   ],

package/dist/src/action.js

@@ -2,7 +2,6 @@ import fs from 'node:fs';
 import { join } from 'node:path';
 import { faker } from '@faker-js/faker';
 import { context, trace } from '@opentelemetry/api';
-import { highlight } from 'cli-highlight';
 import { container, recorder } from 'codeceptjs';
 import * as codeceptjs from 'codeceptjs';
 import { hopeThat, retryTo, tryTo, within } from 'codeceptjs/lib/effects';
@@ -12,7 +11,7 @@ import { clearActivity, setActivity } from "./activity.js";
 import { ConfigParser, outputPath } from './config.js';
 import { Observability } from "./observability.js";
 import { htmlCombinedSnapshot, minifyHtml } from './utils/html.js';
-import { createDebug, log, setStepSpanParent, tag } from './utils/logger.js';
+import { createDebug, setStepSpanParent, tag } from './utils/logger.js';
 import { safeFilename } from "./utils/strings.js";
 const debugLog = createDebug('explorbot:action');
 const FATAL_BROWSER_ERRORS = /Frame was detached|Target closed|Execution context was destroyed|Protocol error|Session closed/i;
@@ -257,7 +256,13 @@ class Action {
     async expect(codeOrFunction) {
         const codeString = typeof codeOrFunction === 'string' ? codeOrFunction : codeOrFunction.toString();
         this.expectation = codeString.toString();
-        log('Expecting', highlight(codeString, { language: 'javascript' }));
+        const expectationPreview = sanitizeCodeBlock(codeString)
+            .split('\n')
+            .map((line) => line.trim())
+            .filter(Boolean)
+            .slice(0, 2)
+            .join(' ');
+        tag('step').log(`Expecting: ${expectationPreview || 'assertion'}`);
         try {
             debugLog('Executing expectation:', codeString);
             let codeFunction;

package/dist/src/ai/driller.js

@@ -96,7 +96,7 @@ export class Driller extends TaskAgent {
         const sessionName = `driller_${Date.now().toString(36)}`;
         this.allResults = [];
         return Observability.run(`driller: ${currentState.url}`, { tags: ['driller'], sessionId: sessionName }, async () => {
-            tag('info').log(`Driller starting on ${currentState.url}`);
+            tag('step').log(`Drilling page: ${currentState.url}`);
             await this.hooksRunner.runBeforeHook('driller', currentState.url);
             const originalState = await this.captureAnnotatedState();
             const components = await this.collectComponents(originalState, maxComponents);