explorbot 0.4.1 → 0.4.3

package/boat/doc-collector/src/screenshots.ts

@@ -1,14 +1,19 @@
 import { mkdirSync } from 'node:fs';
 import path from 'node:path';
+import { PNG } from 'pngjs';
+import { removeVisualAnnotations } from '../../../src/ai/researcher/coordinates.ts';
 import { parseResearchSections } from '../../../src/ai/researcher/parser.ts';
+import { parseAriaLocator } from '../../../src/utils/aria.ts';
 import type Explorer from '../../../src/explorer.ts';
-import type { WebPageState } from '../../../src/state-manager.ts';
+import { type WebPageState, normalizeUrl } from '../../../src/state-manager.ts';
 import { safeFilename, sanitizeFilename } from '../../../src/utils/strings.ts';
+import type { PageDocumentation } from './ai/documentarian.ts';
 import type { DocStateTransition } from './ai/tools.ts';
 import type { DocbotConfig } from './config.ts';
-import { captureInteractionAfter, captureInteractionBefore } from './interaction-screenshots.ts';
+import { addPadding, captureInteractionAfter, captureInteractionBefore, regionAround, saveRegion } from './interaction-screenshots.ts';
 
 const DEFAULT_MAX_SECTION_SCREENSHOTS = 8;
+const EVIDENCE_PADDING = 250;
 
 export async function captureDocumentationScreenshots(explorer: Explorer, state: WebPageState, research: string, options: DocumentationScreenshotOptions): Promise<DocumentationScreenshot[]> {
   const page = explorer.page;
@@ -16,6 +21,8 @@ export async function captureDocumentationScreenshots(explorer: Explorer, state:
     return [];
   }
 
+  await dismissTransientOverlay(page);
+  await removeVisualAnnotations(page);
   mkdirSync(options.screenshotsDir, { recursive: true });
 
   const screenshots: DocumentationScreenshot[] = [];
@@ -37,6 +44,101 @@ export async function captureDocumentationScreenshots(explorer: Explorer, state:
   return screenshots;
 }
 
+export async function captureEvidenceScreenshots(explorer: Explorer, state: WebPageState, documentation: PageDocumentation, options: DocumentationScreenshotOptions): Promise<Array<DocumentationScreenshot | null>> {
+  const page = explorer.page;
+  if (!page) {
+    return documentation.can.map(() => null);
+  }
+
+  await dismissTransientOverlay(page);
+  await removeVisualAnnotations(page);
+  if (!isOnDocumentedPage(page, state)) {
+    return documentation.can.map(() => null);
+  }
+  mkdirSync(options.screenshotsDir, { recursive: true });
+  const pageName = sanitizeFilename(state.url || 'page') || 'page';
+  const shots: Array<DocumentationScreenshot | null> = [];
+  for (const [index, item] of documentation.can.entries()) {
+    shots.push(await captureEvidenceScreenshot(page, pageName, index, item, options));
+  }
+  return shots;
+}
+
+async function captureEvidenceScreenshot(page: any, pageName: string, index: number, item: { action: string; element?: string | null }, options: DocumentationScreenshotOptions): Promise<DocumentationScreenshot | null> {
+  if (!item.element) return null;
+  const element = resolveResearchLocator(item.element, options.research);
+  if (!element) return null;
+  const locatorInfo = parseAriaLocator(element);
+  try {
+    const locator = resolveEvidenceLocator(page, element, locatorInfo);
+    if ((await locator.count()) !== 1) return null;
+    await locator.scrollIntoViewIfNeeded({ timeout: 2000 });
+    const box = await locator.boundingBox();
+    if (!box || box.width <= 0 || box.height <= 0) return null;
+    const image = await page.screenshot();
+    const png = PNG.sync.read(image);
+    const region = regionAround(png, box, await page.viewportSize(), EVIDENCE_PADDING);
+    const filePath = path.join(options.screenshotsDir, safeFilename(`${pageName}_can_${index + 1}`, '.png'));
+    saveRegion(png, region, filePath);
+    return {
+      title: item.action,
+      path: filePath,
+      relativePath: toMarkdownPath(options.pageFilePath, filePath),
+      kind: 'evidence',
+    };
+  } catch {
+    return null;
+  }
+}
+
+function resolveResearchLocator(requested: string, research?: string): string | null {
+  if (!research) return requested;
+  const requestedAria = parseAriaLocator(requested);
+  const requestedCss = unwrapLocator(requested);
+  for (const section of parseResearchSections(research)) {
+    for (const element of section.elements) {
+      if (requestedAria && element.aria?.role === requestedAria.role && element.aria.text === requestedAria.text) {
+        return `{ role: '${element.aria.role}', text: '${element.aria.text}' }`;
+      }
+      if (element.css && requestedCss === unwrapLocator(element.css)) return element.css;
+    }
+  }
+  return null;
+}
+
+function unwrapLocator(locator: string): string {
+  const trimmed = locator.trim();
+  const first = trimmed.at(0);
+  const last = trimmed.at(-1);
+  if (first === last && (first === '`' || first === '"' || first === "'")) {
+    return trimmed.slice(1, -1).trim();
+  }
+  return trimmed;
+}
+
+function isOnDocumentedPage(page: any, state: WebPageState): boolean {
+  try {
+    const current = new URL(page.url());
+    const expected = new URL(state.url || '', current.origin);
+    return current.origin === expected.origin && normalizeUrl(current.href) === normalizeUrl(expected.href);
+  } catch {
+    return false;
+  }
+}
+
+function resolveEvidenceLocator(page: any, element: string, locatorInfo: { role: string; text: string } | null): any {
+  if (locatorInfo?.text) {
+    return page.getByRole(locatorInfo.role, { name: locatorInfo.text, exact: true });
+  }
+  return page.locator(element);
+}
+
+async function dismissTransientOverlay(page: any): Promise<void> {
+  try {
+    await page.keyboard.press('Escape');
+  } catch {}
+}
+
 export function getScreenshotSections(research: string): ScreenshotSection[] {
   const sections: ScreenshotSection[] = [];
   const seen = new Set<string>();
@@ -150,7 +252,7 @@ export interface DocumentationScreenshot {
   title: string;
   path: string;
   relativePath: string;
-  kind: 'page' | 'section' | 'state';
+  kind: 'page' | 'section' | 'state' | 'evidence';
   selector?: string;
 }
 
@@ -158,6 +260,7 @@ interface DocumentationScreenshotOptions {
   pageFilePath: string;
   screenshotsDir: string;
   config: DocbotConfig;
+  research?: string;
 }
 
 interface ScreenshotSection {

package/boat/doc-collector/src/template-dedup.ts

@@ -0,0 +1,31 @@
+import { ariaTemplateSignature } from '../../../src/utils/aria.ts';
+import { diceSimilarity } from '../../../src/utils/similarity.ts';
+
+const TEMPLATE_MIN_SIGNATURE_SIZE = 8;
+const TEMPLATE_SIMILARITY_THRESHOLD = 90;
+
+export function buildTemplateRecord(url: string, snapshot: string | null): TemplateRecord | null {
+  const signature = ariaTemplateSignature(snapshot);
+  if (signature.size < TEMPLATE_MIN_SIGNATURE_SIZE) return null;
+  return { url, signature };
+}
+
+export function findTemplateMatch(snapshot: string | null, known: TemplateRecord[], threshold: number = TEMPLATE_SIMILARITY_THRESHOLD): string | null {
+  const signature = ariaTemplateSignature(snapshot);
+  if (signature.size < TEMPLATE_MIN_SIGNATURE_SIZE) return null;
+  let bestUrl: string | null = null;
+  let bestScore = 0;
+  for (const record of known) {
+    const score = diceSimilarity(signature, record.signature);
+    if (score < bestScore) continue;
+    bestScore = score;
+    bestUrl = record.url;
+  }
+  if (bestScore < threshold) return null;
+  return bestUrl;
+}
+
+export interface TemplateRecord {
+  url: string;
+  signature: Set<string>;
+}

package/boat/prima/src/cli.ts

@@ -1,6 +1,7 @@
 import { Command } from 'commander';
 import dedent from 'dedent';
 import { keepServerRunning } from '../../../src/browser-server.ts';
+import { RecommendedModelsCommand } from '../../../src/commands/recommended-models-command.ts';
 import { browserErrorMessage } from '../../../src/utils/browser-errors.ts';
 import { isVerboseMode, setQuietMode } from '../../../src/utils/logger.ts';
 import { clearActivityLine, trackActivityLine } from './activity-line.ts';
@@ -220,6 +221,8 @@ export function createPrimaCommands(name = 'prima'): Command {
       process.exit(0);
     });
 
+  RecommendedModelsCommand.register(cmd);
+
   addCommonOptions(cmd.command('status <hash>').description('Show the artifacts and page detail recorded for an earlier command'))
     .addHelpText('after', `\n${statusHelp}`)
     .action(async (hash, options) => {

package/boat/prima/src/prima.ts

@@ -15,7 +15,7 @@ import { getAliveEndpoint, launchServer, listInstances, stopServer } from '../..
 import { ConfigCommand } from '../../../src/commands/config-command.ts';
 import { ConfigMissingError, ConfigParser, EXPLORBOT_ENV_VARS, type ExplorbotConfig, outputPath } from '../../../src/config.ts';
 import { ExplorBot } from '../../../src/explorbot.ts';
-import { listSites } from '../../../src/global-config.ts';
+import { findSiteWith, listSites } from '../../../src/global-config.ts';
 import { Reporter } from '../../../src/reporter.ts';
 import type { WebPageState } from '../../../src/state-manager.ts';
 import { Stats } from '../../../src/stats.ts';
@@ -1067,8 +1067,7 @@ export class Prima {
 
   async status(hash: string): Promise<EnvelopeData> {
     if (!this.artifactsDir) {
-      const sites = listSites();
-      const site = sites.find((candidate) => existsSync(path.join(candidate.dir, 'output', 'prima', hash))) || sites[0];
+      const site = findSiteWith(path.join('output', 'prima', hash)) || listSites()[0];
       if (site && !this.configBaseUrl()) this.sessionUrl = site.url;
       await this.loadConfig();
     }
@@ -1113,7 +1112,7 @@ export class Prima {
       aria: result.ariaSnapshot,
       html: await result.combinedHtml(),
       screenshot: result.screenshot,
-      requests: this.bot.requestStore().getRequests(),
+      requests: this.bot.requestStore().getMadeRequests(),
     });
   }
 

package/dist/bin/explorbot-cli.js

@@ -8,6 +8,8 @@ import figureSet from 'figures';
 import { render } from 'ink';
 import React from 'react';
 import { flushTelemetry } from '../src/ai/provider.js';
+import { HelpJsonCommand } from '../src/commands/help-json-command.js';
+import { RecommendedModelsCommand } from '../src/commands/recommended-models-command.js';
 import { App } from '../src/components/App.js';
 import { StatusPane } from '../src/components/StatusPane.js';
 import { knowledgeOption, wsOption } from '../src/commands/options/index.js';
@@ -36,7 +38,8 @@ process.on('uncaughtException', async (error) => {
 process.on('unhandledRejection', (reason) => {
     tag('error').log(`Unhandled rejection: ${reason instanceof Error ? `${reason.message}\n${reason.stack}` : String(reason)}`);
 });
-if (!process.env.EXPLORBOT_NO_BANNER && !process.argv.includes('prima')) {
+const printsJson = process.argv.includes('--json') || process.argv.includes('help-json');
+if (!process.env.EXPLORBOT_NO_BANNER && !process.argv.includes('prima') && !printsJson) {
     console.log(`⛵ ${chalk.yellow.bold(`Explorbot v${pkgVersion}`)} ${chalk.dim('Autonomous Testing Agent')}`);
 }
 function buildExplorBotOptions(from, options) {
@@ -296,10 +299,14 @@ addCommonOptions(program.command('test <planfile> [index]').description('Execute
             planfileArg = options.fromPlan;
             indexArg = planfile;
         }
-        const planTarget = Plan.loadFromFile(planfileArg)?.startUrl;
-        const explorBot = new ExplorBot(buildExplorBotOptions(planTarget, options));
+        const peeked = Plan.loadFromFile(planfileArg);
+        const explorBot = new ExplorBot(buildExplorBotOptions(peeked?.startUrl, options));
         await explorBot.start();
-        const plan = explorBot.loadPlan(planfileArg);
+        let plan = peeked;
+        if (plan)
+            explorBot.setCurrentPlan(plan);
+        if (!plan)
+            plan = explorBot.loadPlan(planfileArg);
         const pending = plan.getPendingTests();
         log(`Plan loaded: "${plan.title}" (${plan.tests.length} tests, ${pending.length} pending)`);
         const startUrl = plan.startUrl;
@@ -420,6 +427,7 @@ program
         process.exit(1);
     }
 });
+RecommendedModelsCommand.register(program);
 program
     .command('init')
     .description('Initialize configuration for a project or for this machine')
@@ -861,5 +869,7 @@ ${rows}
       ${cli} explore /login --max-tests 3
 `;
 };
+program.addHelpText('after', `\nFor agents and tools:\n  ${cli} help-json [command...]  the same definitions as JSON — commands, arguments, options, defaults\n`);
 program.addHelpText('after', envHelp);
+HelpJsonCommand.register(program);
 program.parse();

package/dist/boat/api-tester/src/ai/curler-tools.js

@@ -61,7 +61,7 @@ export function createCurlerTools(apiClient, requestState, test, searchSpec) {
                     body: input.body,
                     queryParams: input.queryParams,
                 });
-                requestState.addRequest(result);
+                requestState.addMadeRequest(result);
                 if (result.error) {
                     tag('error').log(`${input.method} ${input.path} > Network error: ${result.error}`);
                     activeNote.commit(TestResult.FAILED);

package/dist/boat/api-tester/src/cli.js

@@ -2,6 +2,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { Command } from 'commander';
 import { ConfigCommand } from "../../../src/commands/config-command.js";
+import { RecommendedModelsCommand } from "../../../src/commands/recommended-models-command.js";
 import { listSites } from "../../../src/global-config.js";
 import { setPreserveConsoleLogs } from "../../../src/utils/logger.js";
 import { getStyles } from "./ai/chief/styles.js";
@@ -95,6 +96,7 @@ export function createApiCommands(name = 'api') {
             process.exit(1);
         }
     });
+    RecommendedModelsCommand.register(cmd);
     addCommonOptions(cmd.command('test <planfile> [index]').description('Execute tests from a plan file. Index: 1, 1-3, *')).action(async (planfile, index, options) => {
         setPreserveConsoleLogs(true);
         try {

package/dist/boat/doc-collector/src/ai/documentarian.js

@@ -1,6 +1,8 @@
 import dedent from 'dedent';
 import { z } from 'zod';
+import { parseAriaLocator } from "../../../../src/utils/aria.js";
 import { tag } from "../../../../src/utils/logger.js";
+import { parseResearchSections } from "../../../../src/ai/researcher/parser.js";
 import { collectDocInteractions } from "./tools.js";
 class Documentarian {
     provider;
@@ -147,6 +149,9 @@ class Documentarian {
       Only list capabilities that are grounded in the provided page research.
       Put actions into "can" only when there is direct evidence in the page context.
       Put actions into "might" only when the UI strongly suggests a capability but proof is incomplete.
+      Going to another page is navigation, not a capability. "can" and "might" list only actions the user performs on this page; links and menus are documented in a separate navigation section, never here.
+      Raw interaction observations that change the URL are navigation evidence only. Never turn them into "can" or "might" actions.
+      Exclude capabilities supplied by unrelated embedded support, marketing, consent, or feedback widgets. Include an embedded interface only when it is part of the current page's documented purpose.
       Describe each action from the end-user perspective.
       Be explicit about scope:
       - one item
@@ -154,7 +159,7 @@ class Documentarian {
       - bulk operations
       - all items
       - page-level
-      Avoid implementation details, selectors, and QA wording.
+      Avoid implementation details, selectors, and QA wording — except the machine-consumed element field, which is a locator, not prose.
       Avoid duplicate actions with different phrasing.
       </rules>
 
@@ -203,6 +208,8 @@ class Documentarian {
       - action: concise user-facing capability phrased as "user can ..."
       - scope: one of one item, list of items, bulk operations, all items, page-level
       - evidence: short reason based on visible UI or research
+      - element: required for every "can" and "might" action. Return the proving or suggesting control's locator copied verbatim from the page research table — the value from the ARIA column, or from the CSS column when the ARIA column is empty. Return null only when no single element supports the action. It is machine metadata used for validation and never shown as prose.
+      Never copy the human-readable Element description from interaction_observations into element. It is not a locator.
       </output_requirements>
     `;
     }
@@ -233,8 +240,37 @@ class Documentarian {
         const message = error instanceof Error ? error.message : String(error);
         return message.includes('Failed to generate JSON') || message.includes('Failed to validate JSON') || message.includes('failed_generation') || message.includes('No object generated') || message.includes('response did not match schema');
     }
-    normalizeDocumentation(documentation, _state, _research) {
+    normalizeDocumentation(documentation, _state, research) {
         const normalized = { ...documentation };
+        const navigationLocators = new Set();
+        const researchLocators = new Set();
+        for (const section of parseResearchSections(research)) {
+            for (const element of section.elements) {
+                const aria = element.aria ? `{ role: '${element.aria.role}', text: '${element.aria.text}' }` : null;
+                if (aria)
+                    researchLocators.add(this.locatorKey(aria));
+                if (element.css)
+                    researchLocators.add(this.locatorKey(element.css));
+                if (element.aria?.role !== 'link')
+                    continue;
+                if (aria)
+                    navigationLocators.add(this.locatorKey(aria));
+                if (element.css)
+                    navigationLocators.add(this.locatorKey(element.css));
+            }
+        }
+        normalized.can = normalized.can.filter((capability) => {
+            if (!capability.element)
+                return true;
+            const key = this.locatorKey(capability.element);
+            return researchLocators.has(key) && !navigationLocators.has(key);
+        });
+        normalized.might = normalized.might.filter((capability) => {
+            if (!capability.element)
+                return false;
+            const key = this.locatorKey(capability.element);
+            return researchLocators.has(key) && !navigationLocators.has(key);
+        });
         if (!normalized.interactions) {
             normalized.interactions = undefined;
         }
@@ -244,6 +280,18 @@ class Documentarian {
             qualityNotes,
         });
     }
+    locatorKey(locator) {
+        const aria = parseAriaLocator(locator);
+        if (aria)
+            return `aria:${aria.role}:${aria.text}`;
+        const trimmed = locator.trim();
+        const first = trimmed.at(0);
+        const last = trimmed.at(-1);
+        if (first === last && (first === '`' || first === '"' || first === "'")) {
+            return `css:${trimmed.slice(1, -1).trim()}`;
+        }
+        return `css:${trimmed}`;
+    }
     evaluateDocumentationQuality(documentation) {
         const notes = [];
         if ((documentation.interactions || []).length === 0 && this.config.docs?.interactive) {
@@ -300,6 +348,13 @@ const capabilitySchema = z.object({
     action: z.string(),
     scope: z.enum(['one item', 'list of items', 'bulk operations', 'all items', 'page-level']),
     evidence: z.string(),
+    element: z.string().nullable().optional(),
+});
+const generatedCapabilitySchema = z.object({
+    action: z.string(),
+    scope: z.enum(['one item', 'list of items', 'bulk operations', 'all items', 'page-level']),
+    evidence: z.string(),
+    element: z.string().nullable(),
 });
 const stateTransitionSchema = z.object({
     action: z.string(),
@@ -339,11 +394,14 @@ const stateTransitionSchema = z.object({
         .optional(),
 });
 const generatedPageDocumentationSchema = z.object({
+    summary: z.string(),
+    can: z.array(generatedCapabilitySchema),
+    might: z.array(generatedCapabilitySchema),
+});
+const pageDocumentationSchema = z.object({
     summary: z.string(),
     can: z.array(capabilitySchema),
     might: z.array(capabilitySchema),
-});
-const pageDocumentationSchema = generatedPageDocumentationSchema.extend({
     interactions: z.array(stateTransitionSchema).optional(),
     qualityNotes: z.array(z.string()).optional(),
 });

package/dist/boat/doc-collector/src/ai/tools.js

@@ -132,7 +132,7 @@ function buildTransition(candidate, beforeState, afterState, changes) {
 function describeTargetState(beforeState, afterState, candidate) {
     const beforeFocus = detectFocusArea(beforeState.ariaSnapshot || null);
     const afterFocus = detectFocusArea(afterState.ariaSnapshot || null);
-    if (afterFocus.detected && (!beforeFocus.detected || beforeFocus.name !== afterFocus.name)) {
+    if (afterFocus.type && (!beforeFocus.type || beforeFocus.name !== afterFocus.name)) {
         return {
             kind: afterFocus.type || 'dialog',
             label: afterFocus.name || candidate.element.name.trim(),

package/dist/boat/doc-collector/src/cli.js

@@ -2,6 +2,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { Command } from 'commander';
 import { ConfigCommand } from "../../../src/commands/config-command.js";
+import { RecommendedModelsCommand } from "../../../src/commands/recommended-models-command.js";
 import { remote } from "../../../src/remote.js";
 import { isVerboseMode, setPreserveConsoleLogs, setQuietMode } from "../../../src/utils/logger.js";
 import { DocBot } from "./docbot.js";
@@ -34,7 +35,12 @@ function addCommonOptions(cmd) {
 export function createDocsCommands(name = 'docs') {
     const cmd = new Command(name);
     cmd.description('AI-powered website documentation collector');
-    addCommonOptions(cmd.command('collect <path>').description('Crawl pages and generate documentation spec').option('--max-pages <count>', 'Maximum number of pages to document')).action(async (startPath, options) => {
+    addCommonOptions(cmd
+        .command('collect <path>')
+        .description('Crawl pages and generate documentation spec')
+        .option('--max-pages <count>', 'Maximum number of pages to document')
+        .option('--no-collapse-template-pages', 'Visit every page even when its layout matches a documented page')
+        .option('--template-similarity <percent>', 'Structural similarity percent that counts pages as the same layout (default 90)')).action(async (startPath, options) => {
         setPreserveConsoleLogs(true);
         try {
             const bot = new DocBot({
@@ -46,7 +52,11 @@ export function createDocsCommands(name = 'docs') {
             if (options.maxPages) {
                 maxPages = Number.parseInt(options.maxPages, 10);
             }
-            const result = await bot.collect(startPath, { maxPages });
+            let templateSimilarity;
+            if (options.templateSimilarity) {
+                templateSimilarity = Number.parseInt(options.templateSimilarity, 10);
+            }
+            const result = await bot.collect(startPath, { maxPages, collapseTemplatePages: options.collapseTemplatePages, templateSimilarity });
             console.log(`\nDocumented ${result.pages.length} page(s)`);
             console.log(`Skipped ${result.skipped.length} page(s)`);
             console.log(`Spec index: ${result.indexPath}`);
@@ -74,6 +84,7 @@ export function createDocsCommands(name = 'docs') {
             process.exit(1);
         }
     });
+    RecommendedModelsCommand.register(cmd);
     cmd
         .command('init')
         .description('Initialize doc collector configuration')
@@ -101,6 +112,8 @@ export function createDocsCommands(name = 'docs') {
             interactive: false,
             ignoreErrors: true,
             collapseDynamicPages: true,
+            collapseTemplatePages: true,
+            templateSimilarity: 90,
             scope: 'site',
             includePaths: [],
             excludePaths: [],

package/dist/boat/doc-collector/src/config.js

@@ -109,6 +109,7 @@ class DocbotConfigParser {
                 interactive: false,
                 ignoreErrors: true,
                 collapseDynamicPages: true,
+                collapseTemplatePages: true,
                 scope: 'site',
                 includePaths: [],
                 excludePaths: [],

package/dist/boat/doc-collector/src/docbot.js

@@ -9,8 +9,9 @@ import { DocbotConfigParser } from "./config.js";
 import { renderPageDocumentation, renderSpecIndex } from "./docs-renderer.js";
 import { getDocPageKey, shouldCrawlDocPath } from "./path-filter.js";
 import { extractResearchNavigationTargets } from "./research-navigation.js";
-import { captureBeforeInteraction, captureDocumentationScreenshots, captureInteractionScreenshot } from "./screenshots.js";
+import { captureBeforeInteraction, captureDocumentationScreenshots, captureEvidenceScreenshots, captureInteractionScreenshot } from "./screenshots.js";
 import { renderMermaidBody } from "./state-diagram.js";
+import { buildTemplateRecord, findTemplateMatch } from "./template-dedup.js";
 class DocBot {
     explorBot;
     configParser;
@@ -53,6 +54,7 @@ class DocBot {
         const queue = [];
         const queued = new Set();
         const documented = new Set();
+        const templates = [];
         const pages = [];
         const skipped = [];
         const baseUrl = this.explorBot.getConfig().playwright.url;
@@ -73,14 +75,6 @@ class DocBot {
             try {
                 tag('info').log(`Collecting docs for ${this.toDisplayUrl(target, baseUrl)}`);
                 await this.explorBot.visit(target);
-                if (stateManager.isInDeadLoop()) {
-                    tag('warning').log('Dead loop detected during docs crawl, stopping collection');
-                    skipped.push({
-                        url: target,
-                        reason: 'dead loop detected during crawl',
-                    });
-                    break;
-                }
                 const state = this.explorBot.getCurrentState();
                 if (!state) {
                     skipped.push({
@@ -93,6 +87,22 @@ class DocBot {
                 if (documented.has(pageKey)) {
                     continue;
                 }
+                if (this.shouldCollapseTemplates(opts.collapseTemplatePages)) {
+                    const templateUrl = findTemplateMatch(state.ariaSnapshot ?? null, templates, this.getTemplateSimilarity(opts.templateSimilarity));
+                    if (templateUrl) {
+                        skipped.push({
+                            url: state.url,
+                            reason: `same layout as ${templateUrl} (only content differs)`,
+                        });
+                        documented.add(pageKey);
+                        for (const nextPath of this.extractNextPaths(state, baseUrl, '')) {
+                            if (documented.has(this.getPageKey(nextPath)) || stateManager.hasVisitedState(nextPath))
+                                continue;
+                            this.enqueuePath(nextPath, queue, queued);
+                        }
+                        continue;
+                    }
+                }
                 const research = await this.explorBot.agentResearcher().research(state, {
                     screenshot: this.shouldUseScreenshots(),
                     force: true,
@@ -134,6 +144,9 @@ class DocBot {
                     filePath,
                 });
                 documented.add(pageKey);
+                const templateRecord = buildTemplateRecord(state.url, state.ariaSnapshot ?? null);
+                if (templateRecord)
+                    templates.push(templateRecord);
                 const nextPaths = this.extractNextPaths(state, baseUrl, research, documentation);
                 const interactionPriorityPaths = new Set(this.extractInteractionPaths(baseUrl, documentation));
                 for (const nextPath of nextPaths) {
@@ -184,6 +197,23 @@ class DocBot {
         }
         return true;
     }
+    shouldCollapseTemplates(override) {
+        if (override === false) {
+            return false;
+        }
+        if (this.config.docs?.collapseTemplatePages === false) {
+            return false;
+        }
+        return true;
+    }
+    getTemplateSimilarity(override) {
+        const value = override ?? this.config.docs?.templateSimilarity;
+        if (value == null)
+            return undefined;
+        if (!Number.isFinite(value) || value <= 0 || value > 100)
+            return undefined;
+        return value;
+    }
     extractNextPaths(state, baseUrl, research, documentation) {
         const paths = [];
         const seen = new Set();
@@ -393,7 +423,8 @@ class DocBot {
     async savePageDocumentation(state, documentation, research) {
         const pagePath = this.getPageFilePath(state.url);
         const screenshots = await this.captureScreenshots(state, research, pagePath);
-        writeFileSync(pagePath, renderPageDocumentation(state, documentation, screenshots), 'utf8');
+        const evidence = await this.captureEvidence(state, documentation, research, pagePath);
+        writeFileSync(pagePath, renderPageDocumentation(state, documentation, screenshots, evidence), 'utf8');
         return pagePath;
     }
     async captureScreenshots(state, research, pagePath) {
@@ -406,6 +437,17 @@ class DocBot {
             config: this.config,
         });
     }
+    async captureEvidence(state, documentation, research, pagePath) {
+        if (!this.shouldUseScreenshots()) {
+            return documentation.can.map(() => null);
+        }
+        return captureEvidenceScreenshots(this.explorBot.getExplorer(), state, documentation, {
+            pageFilePath: pagePath,
+            screenshotsDir: this.getScreenshotsDir(),
+            config: this.config,
+            research,
+        });
+    }
     saveIndex(startPath, pages, skipped, maxPages) {
         const outputDir = this.configParser.getOutputDir();
         const indexPath = path.join(outputDir, 'index.md');