explorbot 0.4.0 → 0.4.2

package/dist/bin/explorbot-cli.js

@@ -8,8 +8,10 @@ import figureSet from 'figures';
 import { render } from 'ink';
 import React from 'react';
 import { flushTelemetry } from '../src/ai/provider.js';
+import { RecommendedModelsCommand } from '../src/commands/recommended-models-command.js';
 import { App } from '../src/components/App.js';
 import { StatusPane } from '../src/components/StatusPane.js';
+import { knowledgeOption, wsOption } from '../src/commands/options/index.js';
 import { ConfigParser, EXPLORBOT_ENV_VARS, PROVIDERS } from '../src/config.js';
 import { ExplorBot } from '../src/explorbot.js';
 import { remote } from '../src/remote.js';
@@ -25,7 +27,8 @@ const cli = getCliName();
 const pkgPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../package.json');
 const pkgVersion = JSON.parse(fs.readFileSync(pkgPath, 'utf-8')).version;
 program.name(cli).description('AI-powered web exploration tool').version(pkgVersion, '-V, --version');
-remote.registerOption(program);
+wsOption.register(program);
+knowledgeOption.register(program);
 process.on('uncaughtException', async (error) => {
     tag('error').log(`Uncaught exception: ${error instanceof Error ? `${error.message}\n${error.stack}` : String(error)}`);
     await flushTelemetry();
@@ -294,10 +297,14 @@ addCommonOptions(program.command('test <planfile> [index]').description('Execute
             planfileArg = options.fromPlan;
             indexArg = planfile;
         }
-        const planTarget = Plan.loadFromFile(planfileArg)?.startUrl;
-        const explorBot = new ExplorBot(buildExplorBotOptions(planTarget, options));
+        const peeked = Plan.loadFromFile(planfileArg);
+        const explorBot = new ExplorBot(buildExplorBotOptions(peeked?.startUrl, options));
         await explorBot.start();
-        const plan = explorBot.loadPlan(planfileArg);
+        let plan = peeked;
+        if (plan)
+            explorBot.setCurrentPlan(plan);
+        if (!plan)
+            plan = explorBot.loadPlan(planfileArg);
         const pending = plan.getPendingTests();
         log(`Plan loaded: "${plan.title}" (${plan.tests.length} tests, ${pending.length} pending)`);
         const startUrl = plan.startUrl;
@@ -418,6 +425,7 @@ program
         process.exit(1);
     }
 });
+RecommendedModelsCommand.register(program);
 program
     .command('init')
     .description('Initialize configuration for a project or for this machine')
@@ -644,13 +652,13 @@ addCommonOptions(program.command('navigate <url>').description('Navigate to a UR
         await showStatsAndExit(1);
     }
 });
-addCommonOptions(program.command('drill <url>').alias('driller').description('Drill all components on a page to learn interactions').option('--knowledge <path>', 'Save learned interactions to knowledge file at this URL path').option('--max-components <count>', 'Maximum number of components to drill')).action(async (url, options) => {
+addCommonOptions(program.command('drill <url>').alias('driller').description('Drill all components on a page to learn interactions').option('--save-knowledge <path>', 'Save learned interactions to knowledge file at this URL path').option('--max-components <count>', 'Maximum number of components to drill')).action(async (url, options) => {
     try {
         const explorBot = new ExplorBot(buildExplorBotOptions(url, options));
         await explorBot.start();
         await explorBot.visit(url);
         const plan = await explorBot.agentDriller().drill({
-            knowledgePath: options.knowledge,
+            knowledgePath: options.saveKnowledge,
             maxComponents: Number.parseInt(options.maxComponents || '30', 10),
             interactive: false,
         });

package/dist/boat/api-tester/bin/apibot-cli.js

@@ -1,6 +1,7 @@
 #!/usr/bin/env bun
-import { remote } from "../../../src/remote.js";
+import { knowledgeOption, wsOption } from "../../../src/commands/options/index.js";
 import { createApiCommands } from "../src/cli.js";
 const program = createApiCommands('apibot');
-remote.registerOption(program);
+wsOption.register(program);
+knowledgeOption.register(program);
 program.parse();

package/dist/boat/api-tester/src/ai/chief.js

@@ -24,15 +24,17 @@ export class Chief extends ChiefBase {
     provider;
     config;
     apiClient;
+    knowledgeTracker;
     currentPlan = null;
     lastStyleName = '';
     MIN_TASKS = 3;
     MAX_TASKS = 10;
-    constructor(provider, config, apiClient) {
+    constructor(provider, config, apiClient, knowledgeTracker) {
         super();
         this.provider = provider;
         this.config = config;
         this.apiClient = apiClient || null;
+        this.knowledgeTracker = knowledgeTracker;
     }
     async plan(endpoint, opts) {
         tag('info').log(`Planning API tests for ${endpoint}`);
@@ -42,6 +44,9 @@ export class Chief extends ChiefBase {
         await Observability.run(`chief: ${endpoint}`, { tags: ['chief'], sessionId: endpoint }, async () => {
             const sampleData = await this.collectSampleData(endpoint);
             const conversation = this.buildConversation(endpoint, opts?.style, sampleData);
+            const knowledge = this.knowledgeTracker?.renderEndpointKnowledge(endpoint);
+            if (knowledge)
+                conversation.addUserText(knowledge);
             if (opts?.specDefinition) {
                 conversation.addUserText(dedent `
           <api_spec>

package/dist/boat/api-tester/src/ai/curler-tools.js

@@ -61,7 +61,7 @@ export function createCurlerTools(apiClient, requestState, test, searchSpec) {
                     body: input.body,
                     queryParams: input.queryParams,
                 });
-                requestState.addRequest(result);
+                requestState.addMadeRequest(result);
                 if (result.error) {
                     tag('error').log(`${input.method} ${input.path} > Network error: ${result.error}`);
                     activeNote.commit(TestResult.FAILED);

package/dist/boat/api-tester/src/ai/curler.js

@@ -11,11 +11,13 @@ export class Curler {
     apiClient;
     requestState;
     reporter;
-    constructor(provider, apiClient, requestState, reporter) {
+    knowledgeTracker;
+    constructor(provider, apiClient, requestState, reporter, knowledgeTracker) {
         this.provider = provider;
         this.apiClient = apiClient;
         this.requestState = requestState;
         this.reporter = reporter;
+        this.knowledgeTracker = knowledgeTracker;
     }
     async test(test, opts) {
         tag('info').log(`Testing: ${test.scenario}`);
@@ -25,6 +27,9 @@ export class Curler {
         await this.reporter.reportTestStart(test);
         const conversation = this.provider.startConversation(this.buildSystemPrompt(), 'curler', this.provider.getAgenticModel('curler'));
         const tools = createCurlerTools(this.apiClient, this.requestState, test, opts?.searchSpec);
+        const knowledge = test.startUrl && this.knowledgeTracker?.renderEndpointKnowledge(test.startUrl);
+        if (knowledge)
+            conversation.addUserText(knowledge);
         const initialPrompt = this.buildTestPrompt(test, opts?.specDefinition, opts?.baseEndpoint);
         conversation.addUserText(initialPrompt);
         await loop(async ({ stop, iteration }) => {

package/dist/boat/api-tester/src/apibot.js

@@ -3,6 +3,7 @@ import path from 'node:path';
 import { AIProvider } from "../../../src/ai/provider.js";
 import { RequestStore } from "../../../src/api/request-store.js";
 import { extractEndpointDefinition, loadSpec, searchEndpoints, validateSpecs } from "../../../src/api/spec-reader.js";
+import { KnowledgeTracker } from "../../../src/knowledge-tracker.js";
 import { Reporter } from "../../../src/reporter.js";
 import { Plan } from "../../../src/test-plan.js";
 import { setVerboseMode, tag } from "../../../src/utils/logger.js";
@@ -19,6 +20,7 @@ export class ApiBot {
     apiClient;
     requestState;
     reporter;
+    knowledgeTracker;
     options;
     apiSpec;
     constructor(options = {}) {
@@ -30,7 +32,7 @@ export class ApiBot {
         }
     }
     async start() {
-        this.config = await this.configParser.loadConfig({ config: this.options.config, path: this.options.path, endpoint: this.options.endpoint });
+        this.config = await this.configParser.loadConfig(this.options);
         this.provider = new AIProvider(this.config.ai);
         await this.provider.validateConnection();
         this.apiClient = new ApiClient(this.config.api.baseEndpoint, this.config.api.headers || {}, {
@@ -42,6 +44,7 @@ export class ApiBot {
         this.configParser.ensureDirectory(outputDir);
         this.requestState = new RequestStore(outputDir);
         this.reporter = new Reporter(this.config.reporter);
+        this.knowledgeTracker = new KnowledgeTracker({ knowledgeDir: this.configParser.getKnowledgeDir() });
         validateSpecs(this.config.api.spec);
         this.apiSpec = await loadSpec(this.config.api.spec, outputDir);
         tag('info').log('OpenAPI spec loaded');
@@ -78,13 +81,14 @@ export class ApiBot {
             config: this.config,
             apiClient: this.apiClient,
             requestState: this.requestState,
+            knowledge: this.knowledgeTracker,
         });
     }
     agentChief() {
-        return (this.agents.chief ||= this.createAgent(({ ai, config, apiClient }) => new Chief(ai, config, apiClient)));
+        return (this.agents.chief ||= this.createAgent(({ ai, config, apiClient, knowledge }) => new Chief(ai, config, apiClient, knowledge)));
     }
     agentCurler() {
-        return (this.agents.curler ||= this.createAgent(({ ai, apiClient, requestState }) => new Curler(ai, apiClient, requestState, this.reporter)));
+        return (this.agents.curler ||= this.createAgent(({ ai, apiClient, requestState, knowledge }) => new Curler(ai, apiClient, requestState, this.reporter, knowledge)));
     }
     async plan(target, opts = {}) {
         if (opts.fresh) {

package/dist/boat/api-tester/src/cli.js

@@ -2,6 +2,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { Command } from 'commander';
 import { ConfigCommand } from "../../../src/commands/config-command.js";
+import { RecommendedModelsCommand } from "../../../src/commands/recommended-models-command.js";
 import { listSites } from "../../../src/global-config.js";
 import { setPreserveConsoleLogs } from "../../../src/utils/logger.js";
 import { getStyles } from "./ai/chief/styles.js";
@@ -12,10 +13,18 @@ function buildOptions(options) {
         verbose: options.verbose || options.debug,
         config: options.config,
         path: options.path,
+        baseEndpoint: options.endpoint,
+        spec: options.spec,
     };
 }
 function addCommonOptions(cmd) {
-    return cmd.option('-v, --verbose', 'Enable verbose logging').option('--debug', 'Enable debug logging').option('-c, --config <path>', 'Path to configuration file').option('-p, --path <path>', 'Working directory path');
+    return cmd
+        .option('-v, --verbose', 'Enable verbose logging')
+        .option('--debug', 'Enable debug logging')
+        .option('-c, --config <path>', 'Path to configuration file')
+        .option('-p, --path <path>', 'Working directory path')
+        .option('--endpoint <url>', 'Base API endpoint to test (env: EXPLORBOT_URL)')
+        .option('--spec <path>', 'OpenAPI spec file or URL (env: EXPLORBOT_API_SPEC)');
 }
 function selectTests(tests, index) {
     if (!index || index === '*' || index === 'all') {
@@ -76,8 +85,10 @@ export function createApiCommands(name = 'api') {
         .action(async (endpoint, options) => {
         const parser = ApibotConfigParser.getInstance();
         const [site] = listSites();
+        const runOptions = buildOptions(options);
+        runOptions.endpoint = endpoint || site?.url;
         try {
-            const config = await parser.loadConfig({ config: options.config, path: options.path, endpoint: endpoint || site?.url });
+            const config = await parser.loadConfig(runOptions);
             console.log(ConfigCommand.render(config, { configPath: parser.getConfigPath(), root: parser.getProjectRoot(), json: options.json }));
         }
         catch (error) {
@@ -85,6 +96,7 @@ export function createApiCommands(name = 'api') {
             process.exit(1);
         }
     });
+    RecommendedModelsCommand.register(cmd);
     addCommonOptions(cmd.command('test <planfile> [index]').description('Execute tests from a plan file. Index: 1, 1-3, *')).action(async (planfile, index, options) => {
         setPreserveConsoleLogs(true);
         try {

package/dist/boat/api-tester/src/config.js

@@ -39,6 +39,7 @@ export class ApibotConfigParser {
         }
         ApibotConfigParser.loadEnv(globalEnvPath());
         ApibotConfigParser.loadEnv('.env');
+        this.applyRunOptions(options);
         const resolvedPath = options?.config || this.findConfigFile();
         if (!resolvedPath) {
             try {
@@ -63,6 +64,9 @@ export class ApibotConfigParser {
                 };
             }
             this.config = this.mergeWithDefaults(loadedConfig);
+            this.applyEnvSpec(this.config.api);
+            if (options?.baseEndpoint)
+                this.config.api.baseEndpoint = options.baseEndpoint.replace(/\/$/, '');
             await resolveConfigModels(this.config.ai);
             this.configPath = resolvedPath;
             this.site = null;
@@ -104,6 +108,13 @@ export class ApibotConfigParser {
         const resolved = resolveSiteTarget(endpoint, this.site.url);
         if (resolved.baseUrl !== this.site.url)
             return endpoint;
+        const basePath = new URL(this.getConfig().api.baseEndpoint).pathname.replace(/\/$/, '');
+        if (!basePath)
+            return resolved.path;
+        if (resolved.path === basePath)
+            return '/';
+        if (resolved.path.startsWith(`${basePath}/`))
+            return resolved.path.slice(basePath.length);
         return resolved.path;
     }
     getPlansDir() {
@@ -121,13 +132,27 @@ export class ApibotConfigParser {
             mkdirSync(dirPath, { recursive: true });
         }
     }
+    applyRunOptions(options) {
+        if (options?.baseEndpoint)
+            process.env.EXPLORBOT_URL = options.baseEndpoint;
+        if (options?.spec)
+            process.env.EXPLORBOT_API_SPEC = options.spec;
+    }
+    applyEnvSpec(api) {
+        if (!process.env.EXPLORBOT_API_SPEC)
+            return;
+        api.spec = [process.env.EXPLORBOT_API_SPEC];
+    }
     enterGlobalMode(config, endpoint) {
         const site = resolveSiteTarget(endpoint);
         this.site = registerSite(site.baseUrl);
+        let baseEndpoint = site.baseUrl;
+        const envUrl = process.env.EXPLORBOT_URL;
+        if (envUrl && URL.parse(envUrl)?.origin === site.baseUrl)
+            baseEndpoint = envUrl.replace(/\/$/, '');
         config.dirs = { output: 'output', knowledge: 'knowledge' };
-        config.api = { ...config.api, baseEndpoint: site.baseUrl };
-        if (process.env.EXPLORBOT_API_SPEC)
-            config.api.spec = [process.env.EXPLORBOT_API_SPEC];
+        config.api = { ...config.api, baseEndpoint };
+        materializeKnowledge(this.site.dir);
     }
     async loadEnvConfig() {
         const provider = process.env.EXPLORBOT_AI_PROVIDER;
@@ -140,14 +165,12 @@ export class ApibotConfigParser {
         }
         const baseEndpoint = process.env.EXPLORBOT_URL;
         if (!baseEndpoint) {
-            throw new Error('No API endpoint to test. Set EXPLORBOT_URL to the API base endpoint');
+            throw new Error('No API endpoint to test. Pass --endpoint or set EXPLORBOT_URL to the API base endpoint');
         }
         const outputRoot = resolveOutputRoot();
         materializeKnowledge(outputRoot);
         const api = { baseEndpoint };
-        if (process.env.EXPLORBOT_API_SPEC) {
-            api.spec = [process.env.EXPLORBOT_API_SPEC];
-        }
+        this.applyEnvSpec(api);
         let model;
         if (provider && modelSpec)
             model = await createModel(provider, modelSpec);
@@ -210,7 +233,7 @@ export class ApibotConfigParser {
         }
     }
     mergeWithDefaults(config) {
-        return this.deepMerge({ dirs: { output: 'output' } }, config);
+        return this.deepMerge({ dirs: { output: 'output' }, api: {} }, config);
     }
     deepMerge(target, source) {
         const result = { ...target };

package/dist/boat/doc-collector/bin/doc-collector-cli.js

@@ -1,6 +1,7 @@
 #!/usr/bin/env bun
-import { remote } from "../../../src/remote.js";
+import { knowledgeOption, wsOption } from "../../../src/commands/options/index.js";
 import { createDocsCommands } from "../src/cli.js";
 const program = createDocsCommands('doc-collector');
-remote.registerOption(program);
+wsOption.register(program);
+knowledgeOption.register(program);
 program.parse();

package/dist/boat/doc-collector/src/ai/documentarian.js

@@ -1,6 +1,8 @@
 import dedent from 'dedent';
 import { z } from 'zod';
+import { parseAriaLocator } from "../../../../src/utils/aria.js";
 import { tag } from "../../../../src/utils/logger.js";
+import { parseResearchSections } from "../../../../src/ai/researcher/parser.js";
 import { collectDocInteractions } from "./tools.js";
 class Documentarian {
     provider;
@@ -147,6 +149,9 @@ class Documentarian {
       Only list capabilities that are grounded in the provided page research.
       Put actions into "can" only when there is direct evidence in the page context.
       Put actions into "might" only when the UI strongly suggests a capability but proof is incomplete.
+      Going to another page is navigation, not a capability. "can" and "might" list only actions the user performs on this page; links and menus are documented in a separate navigation section, never here.
+      Raw interaction observations that change the URL are navigation evidence only. Never turn them into "can" or "might" actions.
+      Exclude capabilities supplied by unrelated embedded support, marketing, consent, or feedback widgets. Include an embedded interface only when it is part of the current page's documented purpose.
       Describe each action from the end-user perspective.
       Be explicit about scope:
       - one item
@@ -154,7 +159,7 @@ class Documentarian {
       - bulk operations
       - all items
       - page-level
-      Avoid implementation details, selectors, and QA wording.
+      Avoid implementation details, selectors, and QA wording — except the machine-consumed element field, which is a locator, not prose.
       Avoid duplicate actions with different phrasing.
       </rules>
 
@@ -203,6 +208,8 @@ class Documentarian {
       - action: concise user-facing capability phrased as "user can ..."
       - scope: one of one item, list of items, bulk operations, all items, page-level
       - evidence: short reason based on visible UI or research
+      - element: required for every "can" and "might" action. Return the proving or suggesting control's locator copied verbatim from the page research table — the value from the ARIA column, or from the CSS column when the ARIA column is empty. Return null only when no single element supports the action. It is machine metadata used for validation and never shown as prose.
+      Never copy the human-readable Element description from interaction_observations into element. It is not a locator.
       </output_requirements>
     `;
     }
@@ -233,8 +240,37 @@ class Documentarian {
         const message = error instanceof Error ? error.message : String(error);
         return message.includes('Failed to generate JSON') || message.includes('Failed to validate JSON') || message.includes('failed_generation') || message.includes('No object generated') || message.includes('response did not match schema');
     }
-    normalizeDocumentation(documentation, _state, _research) {
+    normalizeDocumentation(documentation, _state, research) {
         const normalized = { ...documentation };
+        const navigationLocators = new Set();
+        const researchLocators = new Set();
+        for (const section of parseResearchSections(research)) {
+            for (const element of section.elements) {
+                const aria = element.aria ? `{ role: '${element.aria.role}', text: '${element.aria.text}' }` : null;
+                if (aria)
+                    researchLocators.add(this.locatorKey(aria));
+                if (element.css)
+                    researchLocators.add(this.locatorKey(element.css));
+                if (element.aria?.role !== 'link')
+                    continue;
+                if (aria)
+                    navigationLocators.add(this.locatorKey(aria));
+                if (element.css)
+                    navigationLocators.add(this.locatorKey(element.css));
+            }
+        }
+        normalized.can = normalized.can.filter((capability) => {
+            if (!capability.element)
+                return true;
+            const key = this.locatorKey(capability.element);
+            return researchLocators.has(key) && !navigationLocators.has(key);
+        });
+        normalized.might = normalized.might.filter((capability) => {
+            if (!capability.element)
+                return false;
+            const key = this.locatorKey(capability.element);
+            return researchLocators.has(key) && !navigationLocators.has(key);
+        });
         if (!normalized.interactions) {
             normalized.interactions = undefined;
         }
@@ -244,6 +280,18 @@ class Documentarian {
             qualityNotes,
         });
     }
+    locatorKey(locator) {
+        const aria = parseAriaLocator(locator);
+        if (aria)
+            return `aria:${aria.role}:${aria.text}`;
+        const trimmed = locator.trim();
+        const first = trimmed.at(0);
+        const last = trimmed.at(-1);
+        if (first === last && (first === '`' || first === '"' || first === "'")) {
+            return `css:${trimmed.slice(1, -1).trim()}`;
+        }
+        return `css:${trimmed}`;
+    }
     evaluateDocumentationQuality(documentation) {
         const notes = [];
         if ((documentation.interactions || []).length === 0 && this.config.docs?.interactive) {
@@ -300,6 +348,13 @@ const capabilitySchema = z.object({
     action: z.string(),
     scope: z.enum(['one item', 'list of items', 'bulk operations', 'all items', 'page-level']),
     evidence: z.string(),
+    element: z.string().nullable().optional(),
+});
+const generatedCapabilitySchema = z.object({
+    action: z.string(),
+    scope: z.enum(['one item', 'list of items', 'bulk operations', 'all items', 'page-level']),
+    evidence: z.string(),
+    element: z.string().nullable(),
 });
 const stateTransitionSchema = z.object({
     action: z.string(),
@@ -339,11 +394,14 @@ const stateTransitionSchema = z.object({
         .optional(),
 });
 const generatedPageDocumentationSchema = z.object({
+    summary: z.string(),
+    can: z.array(generatedCapabilitySchema),
+    might: z.array(generatedCapabilitySchema),
+});
+const pageDocumentationSchema = z.object({
     summary: z.string(),
     can: z.array(capabilitySchema),
     might: z.array(capabilitySchema),
-});
-const pageDocumentationSchema = generatedPageDocumentationSchema.extend({
     interactions: z.array(stateTransitionSchema).optional(),
     qualityNotes: z.array(z.string()).optional(),
 });

package/dist/boat/doc-collector/src/ai/tools.js

@@ -132,7 +132,7 @@ function buildTransition(candidate, beforeState, afterState, changes) {
 function describeTargetState(beforeState, afterState, candidate) {
     const beforeFocus = detectFocusArea(beforeState.ariaSnapshot || null);
     const afterFocus = detectFocusArea(afterState.ariaSnapshot || null);
-    if (afterFocus.detected && (!beforeFocus.detected || beforeFocus.name !== afterFocus.name)) {
+    if (afterFocus.type && (!beforeFocus.type || beforeFocus.name !== afterFocus.name)) {
         return {
             kind: afterFocus.type || 'dialog',
             label: afterFocus.name || candidate.element.name.trim(),

package/dist/boat/doc-collector/src/cli.js

@@ -2,6 +2,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { Command } from 'commander';
 import { ConfigCommand } from "../../../src/commands/config-command.js";
+import { RecommendedModelsCommand } from "../../../src/commands/recommended-models-command.js";
 import { remote } from "../../../src/remote.js";
 import { isVerboseMode, setPreserveConsoleLogs, setQuietMode } from "../../../src/utils/logger.js";
 import { DocBot } from "./docbot.js";
@@ -15,6 +16,7 @@ function buildOptions(options) {
         incognito: options.incognito,
         session: options.session,
         docsConfig: options.docsConfig,
+        baseUrl: options.url,
     };
 }
 function addCommonOptions(cmd) {
@@ -24,6 +26,7 @@ function addCommonOptions(cmd) {
         .option('-c, --config <path>', 'Path to explorbot configuration file')
         .option('--docs-config <path>', 'Path to doc collector configuration file')
         .option('-p, --path <path>', 'Working directory path')
+        .option('--url <url>', 'Base URL of the site, when the path argument is relative (env: EXPLORBOT_URL)')
         .option('-s, --show', 'Show browser window')
         .option('--headless', 'Run browser in headless mode')
         .option('--incognito', 'Run without recording experiences')
@@ -32,7 +35,12 @@ function addCommonOptions(cmd) {
 export function createDocsCommands(name = 'docs') {
     const cmd = new Command(name);
     cmd.description('AI-powered website documentation collector');
-    addCommonOptions(cmd.command('collect <path>').description('Crawl pages and generate documentation spec').option('--max-pages <count>', 'Maximum number of pages to document')).action(async (startPath, options) => {
+    addCommonOptions(cmd
+        .command('collect <path>')
+        .description('Crawl pages and generate documentation spec')
+        .option('--max-pages <count>', 'Maximum number of pages to document')
+        .option('--no-collapse-template-pages', 'Visit every page even when its layout matches a documented page')
+        .option('--template-similarity <percent>', 'Structural similarity percent that counts pages as the same layout (default 90)')).action(async (startPath, options) => {
         setPreserveConsoleLogs(true);
         try {
             const bot = new DocBot({
@@ -44,7 +52,11 @@ export function createDocsCommands(name = 'docs') {
             if (options.maxPages) {
                 maxPages = Number.parseInt(options.maxPages, 10);
             }
-            const result = await bot.collect(startPath, { maxPages });
+            let templateSimilarity;
+            if (options.templateSimilarity) {
+                templateSimilarity = Number.parseInt(options.templateSimilarity, 10);
+            }
+            const result = await bot.collect(startPath, { maxPages, collapseTemplatePages: options.collapseTemplatePages, templateSimilarity });
             console.log(`\nDocumented ${result.pages.length} page(s)`);
             console.log(`Skipped ${result.skipped.length} page(s)`);
             console.log(`Spec index: ${result.indexPath}`);
@@ -65,13 +77,14 @@ export function createDocsCommands(name = 'docs') {
         .action(async (url, options) => {
         setQuietMode(!isVerboseMode());
         try {
-            console.log(await ConfigCommand.summary({ config: options.config, path: options.path, url, json: options.json }));
+            console.log(await ConfigCommand.summary({ config: options.config, path: options.path, url: url || options.url, json: options.json }));
         }
         catch (error) {
             console.error(error instanceof Error ? error.message : 'Unknown error');
             process.exit(1);
         }
     });
+    RecommendedModelsCommand.register(cmd);
     cmd
         .command('init')
         .description('Initialize doc collector configuration')
@@ -99,6 +112,8 @@ export function createDocsCommands(name = 'docs') {
             interactive: false,
             ignoreErrors: true,
             collapseDynamicPages: true,
+            collapseTemplatePages: true,
+            templateSimilarity: 90,
             scope: 'site',
             includePaths: [],
             excludePaths: [],

package/dist/boat/doc-collector/src/config.js

@@ -109,6 +109,7 @@ class DocbotConfigParser {
                 interactive: false,
                 ignoreErrors: true,
                 collapseDynamicPages: true,
+                collapseTemplatePages: true,
                 scope: 'site',
                 includePaths: [],
                 excludePaths: [],