explorbot 0.1.17 → 0.1.18

package/boat/doc-collector/src/docbot.ts

@@ -0,0 +1,391 @@
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { ExplorBot, type ExplorBotOptions } from '../../../src/explorbot.ts';
+import type { Link, WebPageState } from '../../../src/state-manager.ts';
+import { normalizeUrl } from '../../../src/state-manager.ts';
+import { sanitizeFilename } from '../../../src/utils/strings.ts';
+import { tag } from '../../../src/utils/logger.ts';
+import { Documentarian, type PageDocumentation } from './ai/documentarian.ts';
+import { type DocbotConfig, DocbotConfigParser } from './config.ts';
+import { type DocumentedPage, renderPageDocumentation, renderSpecIndex, type SkippedPage } from './docs-renderer.ts';
+import { getDocPageKey, shouldCrawlDocPath } from './path-filter.ts';
+import { extractResearchNavigationTargets } from './research-navigation.ts';
+
+class DocBot {
+  private explorBot: ExplorBot;
+  private configParser: DocbotConfigParser;
+  private config: DocbotConfig = {};
+  private documentarian!: Documentarian;
+  private options: DocbotOptions;
+  private scopeRoot = '/';
+
+  constructor(options: DocbotOptions = {}) {
+    this.options = options;
+    const baseUrl = this.extractAbsoluteBaseUrl(options.startUrl || '/');
+    this.explorBot = new ExplorBot({
+      baseUrl,
+      verbose: options.verbose,
+      config: options.config,
+      path: options.path,
+      show: options.show,
+      headless: options.headless,
+      incognito: options.incognito,
+      session: options.session,
+    });
+    this.configParser = DocbotConfigParser.getInstance();
+  }
+
+  async start(): Promise<void> {
+    await this.explorBot.start();
+    this.config = await this.configParser.loadConfig({
+      config: this.options.docsConfig,
+      path: this.options.path,
+    });
+    this.documentarian = new Documentarian(this.explorBot.getProvider(), this.config);
+    this.ensureDirectory(this.configParser.getOutputDir());
+    this.ensureDirectory(this.getPagesDir());
+  }
+
+  async stop(): Promise<void> {
+    await this.explorBot.stop();
+  }
+
+  async collect(startPath: string, opts: CollectOptions = {}): Promise<CollectionResult> {
+    const effectiveStartPath = this.normalizeStartPath(startPath);
+    this.scopeRoot = this.getScopeRoot(effectiveStartPath);
+    const effectiveMaxPages = this.getMaxPages(opts.maxPages);
+    const queue: string[] = [];
+    const queued = new Set<string>();
+    const documented = new Set<string>();
+    const pages: DocumentedPage[] = [];
+    const skipped: SkippedPage[] = [];
+    const baseUrl = this.explorBot.getConfig().playwright.url;
+
+    this.enqueuePath(effectiveStartPath, queue, queued);
+
+    while (queue.length > 0 && pages.length < effectiveMaxPages) {
+      const target = queue.shift();
+      if (!target) {
+        continue;
+      }
+
+      const targetKey = this.getPageKey(target);
+      if (documented.has(targetKey)) {
+        continue;
+      }
+
+      const stateManager = this.explorBot.getExplorer().getStateManager();
+      if (stateManager.hasVisitedState(target)) {
+        continue;
+      }
+
+      try {
+        tag('info').log(`Collecting docs for ${this.toDisplayUrl(target, baseUrl)}`);
+        await this.explorBot.visit(target);
+
+        if (stateManager.isInDeadLoop()) {
+          tag('warning').log('Dead loop detected during docs crawl, stopping collection');
+          skipped.push({
+            url: target,
+            reason: 'dead loop detected during crawl',
+          });
+          break;
+        }
+
+        const state = this.explorBot.getCurrentState();
+        if (!state) {
+          skipped.push({
+            url: target,
+            reason: 'page state was not captured after navigation',
+          });
+          continue;
+        }
+
+        const pageKey = this.getPageKey(state.url || target);
+        if (documented.has(pageKey)) {
+          continue;
+        }
+
+        const research = await this.explorBot.agentResearcher().research(state, {
+          screenshot: this.shouldUseScreenshots(),
+          force: true,
+        });
+        const documentation = await this.documentarian.document(state, research);
+        const lowSignalReason = this.getLowSignalReason(documentation, research);
+        if (lowSignalReason) {
+          skipped.push({
+            url: state.url,
+            reason: lowSignalReason,
+          });
+          documented.add(pageKey);
+          continue;
+        }
+        const filePath = this.savePageDocumentation(state, documentation);
+
+        pages.push({
+          url: state.url,
+          title: state.title || '',
+          summary: documentation.summary,
+          canCount: documentation.can.length,
+          mightCount: documentation.might.length,
+          canActions: documentation.can.map((item) => item.action),
+          mightActions: documentation.might.map((item) => item.action),
+          filePath,
+        });
+        documented.add(pageKey);
+
+        const nextPaths = this.extractNextPaths(state, baseUrl, research);
+        for (const nextPath of nextPaths) {
+          if (documented.has(this.getPageKey(nextPath))) {
+            continue;
+          }
+          if (stateManager.hasVisitedState(nextPath)) {
+            continue;
+          }
+          this.enqueuePath(nextPath, queue, queued);
+        }
+      } catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        tag('warning').log(`Skipping ${target}: ${reason}`);
+        skipped.push({
+          url: target,
+          reason,
+        });
+      }
+    }
+
+    const indexPath = this.saveIndex(effectiveStartPath, pages, skipped, effectiveMaxPages);
+
+    return {
+      pages,
+      skipped,
+      indexPath,
+      outputDir: this.configParser.getOutputDir(),
+    };
+  }
+
+  private getMaxPages(override?: number): number {
+    if (override && override > 0) {
+      return override;
+    }
+
+    const configured = this.config.docs?.maxPages;
+    if (configured && configured > 0) {
+      return configured;
+    }
+
+    return 100;
+  }
+
+  private shouldUseScreenshots(): boolean {
+    const screenshot = this.config.docs?.screenshot;
+    if (screenshot === false) {
+      return false;
+    }
+    return true;
+  }
+
+  private extractNextPaths(state: WebPageState, baseUrl: string, research: string): string[] {
+    const paths: string[] = [];
+    const seen = new Set<string>();
+
+    for (const link of state.links || []) {
+      const nextPath = this.resolveLink(link, baseUrl);
+      if (!nextPath) {
+        continue;
+      }
+      if (!shouldCrawlDocPath(nextPath, this.config)) {
+        continue;
+      }
+      if (!this.isInScope(nextPath)) {
+        continue;
+      }
+      if (seen.has(nextPath)) {
+        continue;
+      }
+      seen.add(nextPath);
+      paths.push(nextPath);
+    }
+
+    for (const target of extractResearchNavigationTargets(state, research)) {
+      if (!shouldCrawlDocPath(target, this.config)) {
+        continue;
+      }
+      if (!this.isInScope(target)) {
+        continue;
+      }
+      if (seen.has(target)) {
+        continue;
+      }
+      seen.add(target);
+      paths.push(target);
+    }
+
+    return paths;
+  }
+
+  private resolveLink(link: Link, baseUrl: string): string | null {
+    let resolved: URL;
+
+    try {
+      resolved = new URL(link.url, baseUrl);
+    } catch {
+      return null;
+    }
+
+    const base = new URL(baseUrl);
+    if (resolved.origin !== base.origin) {
+      return null;
+    }
+
+    const pathName = resolved.pathname || '/';
+    return `${pathName}${resolved.search}${resolved.hash}`;
+  }
+
+  private toDisplayUrl(target: string, baseUrl: string): string {
+    try {
+      return new URL(target, baseUrl).toString();
+    } catch {
+      return target;
+    }
+  }
+
+  private enqueuePath(inputPath: string, queue: string[], queued: Set<string>): void {
+    const normalized = normalizeUrl(inputPath);
+    const pageKey = this.getPageKey(inputPath);
+    if (queued.has(pageKey)) {
+      return;
+    }
+    queued.add(pageKey);
+    if (!inputPath.startsWith('/')) {
+      queue.push(`/${normalized}`);
+      return;
+    }
+    queue.push(inputPath);
+  }
+
+  private getPageKey(pageUrl: string): string {
+    return getDocPageKey(pageUrl, this.config);
+  }
+
+  private normalizeStartPath(startPath: string): string {
+    try {
+      const parsed = new URL(startPath);
+      return `${parsed.pathname || '/'}${parsed.search}${parsed.hash}`;
+    } catch {
+      return startPath;
+    }
+  }
+
+  private extractAbsoluteBaseUrl(startPath: string): string | undefined {
+    try {
+      const parsed = new URL(startPath);
+      return parsed.origin;
+    } catch {
+      return undefined;
+    }
+  }
+
+  private isInScope(target: string): boolean {
+    const normalized = this.normalizeStartPath(target);
+    const scope = this.config.docs?.scope || 'site';
+
+    if (scope === 'site') {
+      return true;
+    }
+
+    if (scope === 'subtree') {
+      return normalized === this.scopeRoot || normalized.startsWith(`${this.scopeRoot}/`);
+    }
+
+    if (scope === 'section') {
+      return normalized === this.scopeRoot || normalized.startsWith(`${this.scopeRoot}/`) || normalized.startsWith(`${this.scopeRoot}-`);
+    }
+
+    return true;
+  }
+
+  private getScopeRoot(startPath: string): string {
+    const normalized = this.normalizeStartPath(startPath);
+    const parts = normalized.split('/').filter(Boolean);
+    if (parts.length === 0) {
+      return '/';
+    }
+    if (parts.length >= 4) {
+      return `/${parts.slice(0, 4).join('/')}`;
+    }
+    return `/${parts.join('/')}`;
+  }
+
+  private getLowSignalReason(documentation: PageDocumentation, research: string): string | null {
+    const minCanActions = this.config.docs?.minCanActions ?? 1;
+    const minInteractiveElements = this.config.docs?.minInteractiveElements ?? 3;
+
+    if (documentation.can.length >= minCanActions) {
+      return null;
+    }
+
+    const interactiveCount = this.countInteractiveElements(research);
+    if (interactiveCount >= minInteractiveElements) {
+      return null;
+    }
+
+    return `low-signal page: only ${documentation.can.length} proven actions and ${interactiveCount} interactive elements`;
+  }
+
+  private countInteractiveElements(research: string): number {
+    const matches = [...research.matchAll(/\((\d+) elements?\)/g)];
+    return matches.reduce((sum, match) => sum + Number.parseInt(match[1], 10), 0);
+  }
+
+  private savePageDocumentation(state: WebPageState, documentation: PageDocumentation): string {
+    const pagePath = this.getPageFilePath(state.url);
+    writeFileSync(pagePath, renderPageDocumentation(state, documentation), 'utf8');
+    return pagePath;
+  }
+
+  private saveIndex(startPath: string, pages: DocumentedPage[], skipped: SkippedPage[], maxPages: number): string {
+    const indexPath = path.join(this.configParser.getOutputDir(), 'spec.md');
+    writeFileSync(indexPath, renderSpecIndex(this.configParser.getOutputDir(), startPath, pages, skipped, maxPages), 'utf8');
+    return indexPath;
+  }
+
+  private getPagesDir(): string {
+    return path.join(this.configParser.getOutputDir(), 'pages');
+  }
+
+  private getPageFilePath(pageUrl: string): string {
+    const normalized = normalizeUrl(pageUrl || '/');
+    const baseName = sanitizeFilename(normalized || 'root');
+    if (baseName) {
+      return path.join(this.getPagesDir(), `${baseName}.md`);
+    }
+    return path.join(this.getPagesDir(), 'root.md');
+  }
+
+  private ensureDirectory(dirPath: string): void {
+    if (existsSync(dirPath)) {
+      return;
+    }
+    mkdirSync(dirPath, { recursive: true });
+  }
+}
+
+interface DocbotOptions extends ExplorBotOptions {
+  docsConfig?: string;
+  startUrl?: string;
+}
+
+interface CollectOptions {
+  maxPages?: number;
+}
+
+interface CollectionResult {
+  pages: DocumentedPage[];
+  skipped: SkippedPage[];
+  indexPath: string;
+  outputDir: string;
+}
+
+export { DocBot };
+export type { DocbotOptions, CollectOptions, CollectionResult, DocumentedPage, SkippedPage };

package/boat/doc-collector/src/docs-renderer.ts

@@ -0,0 +1,187 @@
+import path from 'node:path';
+import type { WebPageState } from '../../../src/state-manager.ts';
+import type { PageDocumentation } from './ai/documentarian.ts';
+
+function renderPageDocumentation(state: WebPageState, documentation: PageDocumentation): string {
+  const lines: string[] = [];
+  lines.push(`# ${state.url}`);
+  lines.push('');
+
+  if (state.title) {
+    lines.push(`Title: ${normalizeInlineText(state.title)}`);
+    lines.push('');
+  }
+
+  lines.push('## Purpose');
+  lines.push('');
+  lines.push(ensureSentence(documentation.summary));
+  lines.push('');
+  lines.push('## User Can');
+  lines.push('');
+
+  if (documentation.can.length === 0) {
+    lines.push('- No proven actions were identified from the collected research.');
+    lines.push('');
+  }
+
+  for (const item of documentation.can) {
+    lines.push(`- ${normalizeAction(item.action)} -> ${item.scope}`);
+    lines.push(`  Proof: ${ensureSentence(item.evidence)}`);
+  }
+
+  if (documentation.can.length > 0) {
+    lines.push('');
+  }
+
+  lines.push('## User Might');
+  lines.push('');
+
+  if (documentation.might.length === 0) {
+    lines.push('- No assumption-based actions were identified.');
+    lines.push('');
+  }
+
+  for (const item of documentation.might) {
+    lines.push(`- ${normalizeAction(item.action, 'might')} -> ${item.scope}`);
+    lines.push(`  Signal: ${ensureSentence(item.evidence)}`);
+  }
+
+  if (documentation.might.length > 0) {
+    lines.push('');
+  }
+
+  return `${lines.join('\n').trimEnd()}\n`;
+}
+
+function renderSpecIndex(outputDir: string, startPath: string, pages: DocumentedPage[], skipped: SkippedPage[], maxPages: number): string {
+  const lines: string[] = [];
+  lines.push('# Website Spec');
+  lines.push('');
+  lines.push('## Overview');
+  lines.push('');
+  lines.push(`Start page: ${startPath}`);
+  lines.push(`Pages documented: ${pages.length}`);
+  lines.push(`Pages skipped: ${skipped.length}`);
+  lines.push(`Max pages: ${maxPages}`);
+  lines.push('');
+  lines.push('## Pages');
+  lines.push('');
+
+  if (pages.length === 0) {
+    lines.push('- No pages were documented.');
+    lines.push('');
+  }
+
+  for (const page of pages) {
+    const relativeFile = path.relative(outputDir, page.filePath).replaceAll('\\', '/');
+    lines.push(`### [${page.url}](${relativeFile})`);
+    lines.push('');
+    lines.push(`Purpose: ${ensureSentence(page.summary)}`);
+    lines.push(`Proven actions: ${page.canCount}`);
+    lines.push(`Possible actions: ${page.mightCount}`);
+    if (page.title) {
+      lines.push(`Title: ${normalizeInlineText(page.title)}`);
+    }
+    lines.push('');
+
+    if (page.canActions.length > 0) {
+      lines.push('User Can:');
+      for (const action of page.canActions) {
+        lines.push(`- ${normalizeAction(action, 'can')}`);
+      }
+      lines.push('');
+    }
+
+    if (page.mightActions.length > 0) {
+      lines.push('User Might:');
+      for (const action of page.mightActions) {
+        lines.push(`- ${normalizeAction(action, 'might')}`);
+      }
+      lines.push('');
+    }
+  }
+
+  if (skipped.length > 0) {
+    lines.push('## Skipped');
+    lines.push('');
+
+    for (const page of skipped) {
+      lines.push(`- ${page.url}. Reason: ${ensureSentence(page.reason)}`);
+    }
+
+    lines.push('');
+  }
+
+  return `${lines.join('\n').trimEnd()}\n`;
+}
+
+function normalizeAction(action: string, kind: 'can' | 'might' = 'can'): string {
+  const trimmed = normalizeInlineText(action);
+  if (!trimmed) {
+    return 'user can interact with this page';
+  }
+
+  const normalized = ensureSentence(trimmed).slice(0, -1);
+  const lower = normalized.toLowerCase();
+
+  if (kind === 'can') {
+    if (lower.startsWith('user can ')) {
+      return normalized;
+    }
+    if (lower.startsWith('can ')) {
+      return `user can ${normalized.slice(4)}`;
+    }
+    if (lower.startsWith('user might ')) {
+      return `user can ${normalized.slice(11)}`;
+    }
+    return `user can ${normalized}`;
+  }
+
+  if (lower.startsWith('user might ')) {
+    return normalized;
+  }
+  if (lower.startsWith('might ')) {
+    return `user might ${normalized.slice(6)}`;
+  }
+  if (lower.startsWith('user can ')) {
+    return `user might ${normalized.slice(9)}`;
+  }
+  if (lower.startsWith('can ')) {
+    return `user might ${normalized.slice(4)}`;
+  }
+  return `user might ${normalized}`;
+}
+
+function ensureSentence(text: string): string {
+  const trimmed = normalizeInlineText(text);
+  if (!trimmed) {
+    return '';
+  }
+  if (/[.!?]$/.test(trimmed)) {
+    return trimmed;
+  }
+  return `${trimmed}.`;
+}
+
+function normalizeInlineText(text: string): string {
+  return text.normalize('NFKC').replace(/\s+/g, ' ').trim();
+}
+
+interface DocumentedPage {
+  url: string;
+  title: string;
+  summary: string;
+  canCount: number;
+  mightCount: number;
+  canActions: string[];
+  mightActions: string[];
+  filePath: string;
+}
+
+interface SkippedPage {
+  url: string;
+  reason: string;
+}
+
+export { renderPageDocumentation, renderSpecIndex, ensureSentence, normalizeAction };
+export type { DocumentedPage, SkippedPage };

package/boat/doc-collector/src/path-filter.ts

@@ -0,0 +1,46 @@
+import { normalizeUrl } from '../../../src/state-manager.ts';
+import { matchesUrl, generalizeUrl } from '../../../src/utils/url-matcher.ts';
+import type { DocbotConfig } from './config.ts';
+
+const DEFAULT_DENIED_PATH_SEGMENTS = ['callback', 'callbacks', 'logout', 'signout', 'sign_out', 'destroy', 'delete', 'remove'];
+
+export function shouldCrawlDocPath(nextPath: string, config: DocbotConfig = {}): boolean {
+  const parsed = new URL(nextPath, 'http://localhost');
+  const segments = parsed.pathname
+    .split('/')
+    .map((segment) => segment.trim().toLowerCase())
+    .filter(Boolean);
+  const normalizedPath = parsed.pathname || '/';
+
+  const includePaths = config.docs?.includePaths || [];
+  if (includePaths.length > 0) {
+    return includePaths.some((pattern) => matchesUrl(pattern, normalizedPath));
+  }
+
+  const excludePaths = config.docs?.excludePaths || [];
+  if (excludePaths.some((pattern) => matchesUrl(pattern, normalizedPath))) {
+    return false;
+  }
+
+  if (segments.length === 0) {
+    return true;
+  }
+
+  const terminalActions = new Set((config.docs?.deniedPathSegments || DEFAULT_DENIED_PATH_SEGMENTS).map((segment) => segment.trim().toLowerCase()).filter(Boolean));
+  if (segments.some((segment) => terminalActions.has(segment))) {
+    return false;
+  }
+
+  return true;
+}
+
+export function getDocPageKey(pageUrl: string, config: DocbotConfig = {}): string {
+  const normalized = normalizeUrl(pageUrl || '/');
+  const path = normalized.startsWith('/') ? normalized : `/${normalized}`;
+
+  if (config.docs?.collapseDynamicPages === false) {
+    return normalizeUrl(path);
+  }
+
+  return normalizeUrl(generalizeUrl(path));
+}

package/boat/doc-collector/src/research-navigation.ts

@@ -0,0 +1,90 @@
+import type { WebPageState } from '../../../src/state-manager.ts';
+import { parseResearchSections, type ResearchElement } from '../../../src/ai/researcher/parser.ts';
+
+const OPEN_API_TAG_SELECTOR_PATTERN = /api-\d+\/tag\/([a-z0-9-]+)(?:["'#/\]\s]|$)/i;
+const OPEN_API_NAVIGATION_SECTION_KEYWORDS = ['navigation', 'menu'];
+
+export function extractResearchNavigationTargets(state: WebPageState, research: string): string[] {
+  const currentUrl = state.url || '/';
+  const sections = parseResearchSections(research);
+  const targets: string[] = [];
+  const seen = new Set<string>();
+
+  for (const section of sections) {
+    const sectionName = section.name.toLowerCase();
+    if (!OPEN_API_NAVIGATION_SECTION_KEYWORDS.some((keyword) => sectionName.includes(keyword))) {
+      continue;
+    }
+
+    for (const element of section.elements) {
+      const target = extractNavigationTarget(currentUrl, element);
+      if (!target || seen.has(target)) {
+        continue;
+      }
+
+      seen.add(target);
+      targets.push(target);
+    }
+  }
+
+  return targets;
+}
+
+function extractNavigationTarget(currentUrl: string, element: ResearchElement): string | null {
+  const openApiTagFromCss = extractOpenApiTagHashFromCss(element.css);
+  if (openApiTagFromCss) {
+    return buildSamePageHashTarget(currentUrl, openApiTagFromCss);
+  }
+
+  if (!currentUrl.includes('#tag/')) {
+    return null;
+  }
+
+  const inferredOpenApiTag = inferOpenApiTagSlugFromLabel(element.name);
+  if (!inferredOpenApiTag) {
+    return null;
+  }
+
+  return buildSamePageHashTarget(currentUrl, `tag/${inferredOpenApiTag}`);
+}
+
+function extractOpenApiTagHashFromCss(css: string | null): string | null {
+  if (!css) {
+    return null;
+  }
+
+  const normalizedSelector = css.replaceAll('\\/', '/');
+  const match = normalizedSelector.match(OPEN_API_TAG_SELECTOR_PATTERN);
+  if (!match?.[1]) {
+    return null;
+  }
+
+  return `tag/${match[1].toLowerCase()}`;
+}
+
+function inferOpenApiTagSlugFromLabel(name: string): string | null {
+  const cleanedLabel = name
+    .replace(/^'+|'+$/g, '')
+    .replace(/\(expanded\)|\(collapsed\)|open group|close group|show more/gi, '')
+    .trim();
+
+  if (!cleanedLabel.includes('/')) {
+    return null;
+  }
+
+  const slug = cleanedLabel
+    .split('/')
+    .map((part) => part.trim().toLowerCase())
+    .filter(Boolean)
+    .join('-')
+    .replace(/[^a-z0-9-]+/g, '-')
+    .replace(/-+/g, '-')
+    .replace(/^-|-$/g, '');
+
+  return slug || null;
+}
+
+function buildSamePageHashTarget(currentUrl: string, hashPath: string): string {
+  const [baseWithSearch] = currentUrl.split('#');
+  return `${baseWithSearch}#${hashPath}`;
+}