explorbot 0.4.0 → 0.4.2

package/bin/explorbot-cli.ts

@@ -8,8 +8,10 @@ import figureSet from 'figures';
 import { render } from 'ink';
 import React from 'react';
 import { flushTelemetry } from '../src/ai/provider.js';
+import { RecommendedModelsCommand } from '../src/commands/recommended-models-command.js';
 import { App } from '../src/components/App.js';
 import { StatusPane } from '../src/components/StatusPane.js';
+import { knowledgeOption, wsOption } from '../src/commands/options/index.js';
 import { ConfigParser, EXPLORBOT_ENV_VARS, PROVIDERS } from '../src/config.js';
 import { ExplorBot, type ExplorBotOptions } from '../src/explorbot.js';
 import { remote } from '../src/remote.js';
@@ -28,7 +30,8 @@ const pkgPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../p
 const pkgVersion = JSON.parse(fs.readFileSync(pkgPath, 'utf-8')).version as string;
 
 program.name(cli).description('AI-powered web exploration tool').version(pkgVersion, '-V, --version');
-remote.registerOption(program);
+wsOption.register(program);
+knowledgeOption.register(program);
 
 process.on('uncaughtException', async (error) => {
   tag('error').log(`Uncaught exception: ${error instanceof Error ? `${error.message}\n${error.stack}` : String(error)}`);
@@ -334,12 +337,14 @@ addCommonOptions(program.command('test <planfile> [index]').description('Execute
         indexArg = planfile;
       }
 
-      const planTarget = Plan.loadFromFile(planfileArg)?.startUrl;
+      const peeked = Plan.loadFromFile(planfileArg);
 
-      const explorBot = new ExplorBot(buildExplorBotOptions(planTarget, options));
+      const explorBot = new ExplorBot(buildExplorBotOptions(peeked?.startUrl, options));
       await explorBot.start();
 
-      const plan = explorBot.loadPlan(planfileArg);
+      let plan = peeked;
+      if (plan) explorBot.setCurrentPlan(plan);
+      if (!plan) plan = explorBot.loadPlan(planfileArg);
       const pending = plan.getPendingTests();
       log(`Plan loaded: "${plan.title}" (${plan.tests.length} tests, ${pending.length} pending)`);
 
@@ -468,6 +473,8 @@ program
     }
   });
 
+RecommendedModelsCommand.register(program);
+
 program
   .command('init')
   .description('Initialize configuration for a project or for this machine')
@@ -706,7 +713,7 @@ addCommonOptions(program.command('navigate <url>').description('Navigate to a UR
 });
 
 addCommonOptions(
-  program.command('drill <url>').alias('driller').description('Drill all components on a page to learn interactions').option('--knowledge <path>', 'Save learned interactions to knowledge file at this URL path').option('--max-components <count>', 'Maximum number of components to drill')
+  program.command('drill <url>').alias('driller').description('Drill all components on a page to learn interactions').option('--save-knowledge <path>', 'Save learned interactions to knowledge file at this URL path').option('--max-components <count>', 'Maximum number of components to drill')
 ).action(async (url, options) => {
   try {
     const explorBot = new ExplorBot(buildExplorBotOptions(url, options));
@@ -715,7 +722,7 @@ addCommonOptions(
     await explorBot.visit(url);
 
     const plan = await explorBot.agentDriller().drill({
-      knowledgePath: options.knowledge,
+      knowledgePath: options.saveKnowledge,
       maxComponents: Number.parseInt(options.maxComponents || '30', 10),
       interactive: false,
     });

package/boat/api-tester/src/ai/chief.ts

@@ -3,6 +3,7 @@ import { z } from 'zod';
 import { Conversation } from '../../../../src/ai/conversation.ts';
 import { WithSessionDedup } from '../../../../src/ai/planner/session-dedup.ts';
 import type { AIProvider } from '../../../../src/ai/provider.ts';
+import type { KnowledgeTracker } from '../../../../src/knowledge-tracker.ts';
 import { Observability } from '../../../../src/observability.ts';
 import { Plan, Test } from '../../../../src/test-plan.ts';
 import { createDebug, tag } from '../../../../src/utils/logger.ts';
@@ -33,17 +34,19 @@ export class Chief extends ChiefBase {
   private provider: AIProvider;
   private config: ApibotConfig;
   private apiClient: ApiClient | null;
+  private knowledgeTracker?: KnowledgeTracker;
   currentPlan: Plan | null = null;
   private lastStyleName = '';
 
   MIN_TASKS = 3;
   MAX_TASKS = 10;
 
-  constructor(provider: AIProvider, config: ApibotConfig, apiClient?: ApiClient | null) {
+  constructor(provider: AIProvider, config: ApibotConfig, apiClient?: ApiClient | null, knowledgeTracker?: KnowledgeTracker) {
     super();
     this.provider = provider;
     this.config = config;
     this.apiClient = apiClient || null;
+    this.knowledgeTracker = knowledgeTracker;
   }
 
   async plan(endpoint: string, opts?: { style?: string; specDefinition?: string }): Promise<Plan> {
@@ -56,6 +59,9 @@ export class Chief extends ChiefBase {
       const sampleData = await this.collectSampleData(endpoint);
       const conversation = this.buildConversation(endpoint, opts?.style, sampleData);
 
+      const knowledge = this.knowledgeTracker?.renderEndpointKnowledge(endpoint);
+      if (knowledge) conversation.addUserText(knowledge);
+
       if (opts?.specDefinition) {
         conversation.addUserText(dedent`
           <api_spec>

package/boat/api-tester/src/ai/curler-tools.ts

@@ -66,7 +66,7 @@ export function createCurlerTools(apiClient: ApiClient, requestState: RequestSto
           queryParams: input.queryParams,
         });
 
-        requestState.addRequest(result);
+        requestState.addMadeRequest(result);
 
         if (result.error) {
           tag('error').log(`${input.method} ${input.path} > Network error: ${result.error}`);

package/boat/api-tester/src/ai/curler.ts

@@ -2,6 +2,7 @@ import dedent from 'dedent';
 import { z } from 'zod';
 import type { AIProvider } from '../../../../src/ai/provider.ts';
 import type { RequestStore } from '../../../../src/api/request-store.ts';
+import type { KnowledgeTracker } from '../../../../src/knowledge-tracker.ts';
 import type { Reporter } from '../../../../src/reporter.ts';
 import { type Test, TestResult } from '../../../../src/test-plan.ts';
 import { createDebug, tag } from '../../../../src/utils/logger.ts';
@@ -18,12 +19,14 @@ export class Curler {
   private apiClient: ApiClient;
   private requestState: RequestStore;
   private reporter: Reporter;
+  private knowledgeTracker?: KnowledgeTracker;
 
-  constructor(provider: AIProvider, apiClient: ApiClient, requestState: RequestStore, reporter: Reporter) {
+  constructor(provider: AIProvider, apiClient: ApiClient, requestState: RequestStore, reporter: Reporter, knowledgeTracker?: KnowledgeTracker) {
     this.provider = provider;
     this.apiClient = apiClient;
     this.requestState = requestState;
     this.reporter = reporter;
+    this.knowledgeTracker = knowledgeTracker;
   }
 
   async test(test: Test, opts?: { specDefinition?: string; baseEndpoint?: string; searchSpec?: (query: string) => string }): Promise<{ success: boolean }> {
@@ -37,6 +40,9 @@ export class Curler {
     const conversation = this.provider.startConversation(this.buildSystemPrompt(), 'curler', this.provider.getAgenticModel('curler'));
     const tools = createCurlerTools(this.apiClient, this.requestState, test, opts?.searchSpec);
 
+    const knowledge = test.startUrl && this.knowledgeTracker?.renderEndpointKnowledge(test.startUrl);
+    if (knowledge) conversation.addUserText(knowledge);
+
     const initialPrompt = this.buildTestPrompt(test, opts?.specDefinition, opts?.baseEndpoint);
     conversation.addUserText(initialPrompt);
 

package/boat/api-tester/src/apibot.ts

@@ -3,6 +3,7 @@ import path from 'node:path';
 import { AIProvider } from '../../../src/ai/provider.ts';
 import { RequestStore } from '../../../src/api/request-store.ts';
 import { extractEndpointDefinition, loadSpec, searchEndpoints, validateSpecs } from '../../../src/api/spec-reader.ts';
+import { KnowledgeTracker } from '../../../src/knowledge-tracker.ts';
 import { Reporter } from '../../../src/reporter.ts';
 import { Plan } from '../../../src/test-plan.ts';
 import { setVerboseMode, tag } from '../../../src/utils/logger.ts';
@@ -20,6 +21,7 @@ export class ApiBot {
   private apiClient!: ApiClient;
   private requestState!: RequestStore;
   private reporter!: Reporter;
+  private knowledgeTracker!: KnowledgeTracker;
   private options: ApibotOptions;
   private apiSpec: any;
 
@@ -33,7 +35,7 @@ export class ApiBot {
   }
 
   async start(): Promise<void> {
-    this.config = await this.configParser.loadConfig({ config: this.options.config, path: this.options.path, endpoint: this.options.endpoint });
+    this.config = await this.configParser.loadConfig(this.options);
     this.provider = new AIProvider(this.config.ai);
     await this.provider.validateConnection();
 
@@ -47,6 +49,7 @@ export class ApiBot {
     this.configParser.ensureDirectory(outputDir);
     this.requestState = new RequestStore(outputDir);
     this.reporter = new Reporter(this.config.reporter);
+    this.knowledgeTracker = new KnowledgeTracker({ knowledgeDir: this.configParser.getKnowledgeDir() });
 
     validateSpecs(this.config.api.spec);
     this.apiSpec = await loadSpec(this.config.api.spec!, outputDir);
@@ -84,21 +87,22 @@ export class ApiBot {
     await this.apiClient?.teardown();
   }
 
-  createAgent<T>(factory: (deps: { ai: AIProvider; config: ApibotConfig; apiClient: ApiClient; requestState: RequestStore }) => T): T {
+  createAgent<T>(factory: (deps: { ai: AIProvider; config: ApibotConfig; apiClient: ApiClient; requestState: RequestStore; knowledge: KnowledgeTracker }) => T): T {
     return factory({
       ai: this.provider,
       config: this.config,
       apiClient: this.apiClient,
       requestState: this.requestState,
+      knowledge: this.knowledgeTracker,
     });
   }
 
   agentChief(): Chief {
-    return (this.agents.chief ||= this.createAgent(({ ai, config, apiClient }) => new Chief(ai, config, apiClient)));
+    return (this.agents.chief ||= this.createAgent(({ ai, config, apiClient, knowledge }) => new Chief(ai, config, apiClient, knowledge)));
   }
 
   agentCurler(): Curler {
-    return (this.agents.curler ||= this.createAgent(({ ai, apiClient, requestState }) => new Curler(ai, apiClient, requestState, this.reporter)));
+    return (this.agents.curler ||= this.createAgent(({ ai, apiClient, requestState, knowledge }) => new Curler(ai, apiClient, requestState, this.reporter, knowledge)));
   }
 
   async plan(target: string, opts: { style?: string; fresh?: boolean } = {}): Promise<Plan> {
@@ -200,6 +204,8 @@ interface ApibotOptions {
   config?: string;
   path?: string;
   endpoint?: string;
+  baseEndpoint?: string;
+  spec?: string;
 }
 
 export type { ApibotOptions };

package/boat/api-tester/src/cli.ts

@@ -2,6 +2,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { Command } from 'commander';
 import { ConfigCommand } from '../../../src/commands/config-command.ts';
+import { RecommendedModelsCommand } from '../../../src/commands/recommended-models-command.ts';
 import { listSites } from '../../../src/global-config.ts';
 import { setPreserveConsoleLogs } from '../../../src/utils/logger.ts';
 import { getStyles } from './ai/chief/styles.ts';
@@ -13,11 +14,19 @@ function buildOptions(options: any): ApibotOptions {
     verbose: options.verbose || options.debug,
     config: options.config,
     path: options.path,
+    baseEndpoint: options.endpoint,
+    spec: options.spec,
   };
 }
 
 function addCommonOptions(cmd: Command): Command {
-  return cmd.option('-v, --verbose', 'Enable verbose logging').option('--debug', 'Enable debug logging').option('-c, --config <path>', 'Path to configuration file').option('-p, --path <path>', 'Working directory path');
+  return cmd
+    .option('-v, --verbose', 'Enable verbose logging')
+    .option('--debug', 'Enable debug logging')
+    .option('-c, --config <path>', 'Path to configuration file')
+    .option('-p, --path <path>', 'Working directory path')
+    .option('--endpoint <url>', 'Base API endpoint to test (env: EXPLORBOT_URL)')
+    .option('--spec <path>', 'OpenAPI spec file or URL (env: EXPLORBOT_API_SPEC)');
 }
 
 function selectTests(tests: any[], index?: string): any[] {
@@ -90,8 +99,10 @@ export function createApiCommands(name = 'api'): Command {
     .action(async (endpoint, options) => {
       const parser = ApibotConfigParser.getInstance();
       const [site] = listSites();
+      const runOptions = buildOptions(options);
+      runOptions.endpoint = endpoint || site?.url;
       try {
-        const config = await parser.loadConfig({ config: options.config, path: options.path, endpoint: endpoint || site?.url });
+        const config = await parser.loadConfig(runOptions);
         console.log(ConfigCommand.render(config, { configPath: parser.getConfigPath(), root: parser.getProjectRoot(), json: options.json }));
       } catch (error) {
         console.error(error instanceof Error ? error.message : 'Unknown error');
@@ -99,6 +110,8 @@ export function createApiCommands(name = 'api'): Command {
       }
     });
 
+  RecommendedModelsCommand.register(cmd);
+
   addCommonOptions(cmd.command('test <planfile> [index]').description('Execute tests from a plan file. Index: 1, 1-3, *')).action(async (planfile, index, options) => {
     setPreserveConsoleLogs(true);
     try {

package/boat/api-tester/src/config.ts

@@ -45,7 +45,7 @@ export class ApibotConfigParser {
     Object.assign(process.env, parseEnv(readFileSync(resolved, 'utf8')));
   }
 
-  async loadConfig(options?: { config?: string; path?: string; endpoint?: string }): Promise<ApibotConfig> {
+  async loadConfig(options?: { config?: string; path?: string; endpoint?: string; baseEndpoint?: string; spec?: string }): Promise<ApibotConfig> {
     if (this.config && !options?.config && !options?.path) return this.config;
 
     const originalCwd = process.cwd();
@@ -55,6 +55,7 @@ export class ApibotConfigParser {
 
     ApibotConfigParser.loadEnv(globalEnvPath());
     ApibotConfigParser.loadEnv('.env');
+    this.applyRunOptions(options);
 
     const resolvedPath = options?.config || this.findConfigFile();
     if (!resolvedPath) {
@@ -82,6 +83,8 @@ export class ApibotConfigParser {
       }
 
       this.config = this.mergeWithDefaults(loadedConfig);
+      this.applyEnvSpec(this.config.api);
+      if (options?.baseEndpoint) this.config.api.baseEndpoint = options.baseEndpoint.replace(/\/$/, '');
       await resolveConfigModels(this.config.ai);
       this.configPath = resolvedPath;
       this.site = null;
@@ -126,6 +129,11 @@ export class ApibotConfigParser {
 
     const resolved = resolveSiteTarget(endpoint, this.site.url);
     if (resolved.baseUrl !== this.site.url) return endpoint;
+
+    const basePath = new URL(this.getConfig().api.baseEndpoint).pathname.replace(/\/$/, '');
+    if (!basePath) return resolved.path;
+    if (resolved.path === basePath) return '/';
+    if (resolved.path.startsWith(`${basePath}/`)) return resolved.path.slice(basePath.length);
     return resolved.path;
   }
 
@@ -148,13 +156,27 @@ export class ApibotConfigParser {
     }
   }
 
+  private applyRunOptions(options?: { baseEndpoint?: string; spec?: string }): void {
+    if (options?.baseEndpoint) process.env.EXPLORBOT_URL = options.baseEndpoint;
+    if (options?.spec) process.env.EXPLORBOT_API_SPEC = options.spec;
+  }
+
+  private applyEnvSpec(api: ApiConfig): void {
+    if (!process.env.EXPLORBOT_API_SPEC) return;
+    api.spec = [process.env.EXPLORBOT_API_SPEC];
+  }
+
   private enterGlobalMode(config: ApibotConfig, endpoint?: string): void {
     const site = resolveSiteTarget(endpoint);
     this.site = registerSite(site.baseUrl);
 
+    let baseEndpoint = site.baseUrl;
+    const envUrl = process.env.EXPLORBOT_URL;
+    if (envUrl && URL.parse(envUrl)?.origin === site.baseUrl) baseEndpoint = envUrl.replace(/\/$/, '');
+
     config.dirs = { output: 'output', knowledge: 'knowledge' };
-    config.api = { ...config.api, baseEndpoint: site.baseUrl };
-    if (process.env.EXPLORBOT_API_SPEC) config.api.spec = [process.env.EXPLORBOT_API_SPEC];
+    config.api = { ...config.api, baseEndpoint };
+    materializeKnowledge(this.site.dir);
   }
 
   private async loadEnvConfig(): Promise<ApibotConfig> {
@@ -169,16 +191,14 @@ export class ApibotConfigParser {
 
     const baseEndpoint = process.env.EXPLORBOT_URL;
     if (!baseEndpoint) {
-      throw new Error('No API endpoint to test. Set EXPLORBOT_URL to the API base endpoint');
+      throw new Error('No API endpoint to test. Pass --endpoint or set EXPLORBOT_URL to the API base endpoint');
     }
 
     const outputRoot = resolveOutputRoot();
     materializeKnowledge(outputRoot);
 
     const api: ApiConfig = { baseEndpoint };
-    if (process.env.EXPLORBOT_API_SPEC) {
-      api.spec = [process.env.EXPLORBOT_API_SPEC];
-    }
+    this.applyEnvSpec(api);
 
     let model: any;
     if (provider && modelSpec) model = await createModel(provider, modelSpec);
@@ -246,7 +266,7 @@ export class ApibotConfigParser {
   }
 
   private mergeWithDefaults(config: Partial<ApibotConfig>): ApibotConfig {
-    return this.deepMerge({ dirs: { output: 'output' } }, config);
+    return this.deepMerge({ dirs: { output: 'output' }, api: {} }, config);
   }
 
   private deepMerge(target: any, source: any): any {

package/boat/doc-collector/bin/doc-collector-cli.ts

@@ -1,7 +1,8 @@
 #!/usr/bin/env bun
-import { remote } from '../../../src/remote.ts';
+import { knowledgeOption, wsOption } from '../../../src/commands/options/index.ts';
 import { createDocsCommands } from '../src/cli.ts';
 
 const program = createDocsCommands('doc-collector');
-remote.registerOption(program);
+wsOption.register(program);
+knowledgeOption.register(program);
 program.parse();

package/boat/doc-collector/src/ai/documentarian.ts

@@ -4,7 +4,9 @@ import type { AIProvider } from '../../../../src/ai/provider.ts';
 import type Explorer from '../../../../src/explorer.ts';
 import type { StateManager } from '../../../../src/state-manager.ts';
 import type { WebPageState } from '../../../../src/state-manager.ts';
+import { parseAriaLocator } from '../../../../src/utils/aria.ts';
 import { tag } from '../../../../src/utils/logger.ts';
+import { parseResearchSections } from '../../../../src/ai/researcher/parser.ts';
 import type { DocbotConfig } from '../config.ts';
 import { type CaptureInteractionState, type DocStateTransition, collectDocInteractions } from './tools.ts';
 
@@ -182,6 +184,9 @@ class Documentarian {
       Only list capabilities that are grounded in the provided page research.
       Put actions into "can" only when there is direct evidence in the page context.
       Put actions into "might" only when the UI strongly suggests a capability but proof is incomplete.
+      Going to another page is navigation, not a capability. "can" and "might" list only actions the user performs on this page; links and menus are documented in a separate navigation section, never here.
+      Raw interaction observations that change the URL are navigation evidence only. Never turn them into "can" or "might" actions.
+      Exclude capabilities supplied by unrelated embedded support, marketing, consent, or feedback widgets. Include an embedded interface only when it is part of the current page's documented purpose.
       Describe each action from the end-user perspective.
       Be explicit about scope:
       - one item
@@ -189,7 +194,7 @@ class Documentarian {
       - bulk operations
       - all items
       - page-level
-      Avoid implementation details, selectors, and QA wording.
+      Avoid implementation details, selectors, and QA wording — except the machine-consumed element field, which is a locator, not prose.
       Avoid duplicate actions with different phrasing.
       </rules>
 
@@ -241,6 +246,8 @@ class Documentarian {
       - action: concise user-facing capability phrased as "user can ..."
       - scope: one of one item, list of items, bulk operations, all items, page-level
       - evidence: short reason based on visible UI or research
+      - element: required for every "can" and "might" action. Return the proving or suggesting control's locator copied verbatim from the page research table — the value from the ARIA column, or from the CSS column when the ARIA column is empty. Return null only when no single element supports the action. It is machine metadata used for validation and never shown as prose.
+      Never copy the human-readable Element description from interaction_observations into element. It is not a locator.
       </output_requirements>
     `;
   }
@@ -274,8 +281,30 @@ class Documentarian {
     return message.includes('Failed to generate JSON') || message.includes('Failed to validate JSON') || message.includes('failed_generation') || message.includes('No object generated') || message.includes('response did not match schema');
   }
 
-  private normalizeDocumentation(documentation: GeneratedPageDocumentation & Partial<Pick<PageDocumentation, 'interactions'>>, _state: WebPageState, _research: string): PageDocumentation {
+  private normalizeDocumentation(documentation: GeneratedPageDocumentation & Partial<Pick<PageDocumentation, 'interactions'>>, _state: WebPageState, research: string): PageDocumentation {
     const normalized = { ...documentation };
+    const navigationLocators = new Set<string>();
+    const researchLocators = new Set<string>();
+    for (const section of parseResearchSections(research)) {
+      for (const element of section.elements) {
+        const aria = element.aria ? `{ role: '${element.aria.role}', text: '${element.aria.text}' }` : null;
+        if (aria) researchLocators.add(this.locatorKey(aria));
+        if (element.css) researchLocators.add(this.locatorKey(element.css));
+        if (element.aria?.role !== 'link') continue;
+        if (aria) navigationLocators.add(this.locatorKey(aria));
+        if (element.css) navigationLocators.add(this.locatorKey(element.css));
+      }
+    }
+    normalized.can = normalized.can.filter((capability) => {
+      if (!capability.element) return true;
+      const key = this.locatorKey(capability.element);
+      return researchLocators.has(key) && !navigationLocators.has(key);
+    });
+    normalized.might = normalized.might.filter((capability) => {
+      if (!capability.element) return false;
+      const key = this.locatorKey(capability.element);
+      return researchLocators.has(key) && !navigationLocators.has(key);
+    });
     if (!normalized.interactions) {
       normalized.interactions = undefined;
     }
@@ -288,6 +317,18 @@ class Documentarian {
     });
   }
 
+  private locatorKey(locator: string): string {
+    const aria = parseAriaLocator(locator);
+    if (aria) return `aria:${aria.role}:${aria.text}`;
+    const trimmed = locator.trim();
+    const first = trimmed.at(0);
+    const last = trimmed.at(-1);
+    if (first === last && (first === '`' || first === '"' || first === "'")) {
+      return `css:${trimmed.slice(1, -1).trim()}`;
+    }
+    return `css:${trimmed}`;
+  }
+
   private evaluateDocumentationQuality(documentation: PageDocumentation): string[] {
     const notes: string[] = [];
 
@@ -355,6 +396,14 @@ const capabilitySchema = z.object({
   action: z.string(),
   scope: z.enum(['one item', 'list of items', 'bulk operations', 'all items', 'page-level']),
   evidence: z.string(),
+  element: z.string().nullable().optional(),
+});
+
+const generatedCapabilitySchema = z.object({
+  action: z.string(),
+  scope: z.enum(['one item', 'list of items', 'bulk operations', 'all items', 'page-level']),
+  evidence: z.string(),
+  element: z.string().nullable(),
 });
 
 const stateTransitionSchema = z.object({
@@ -397,11 +446,14 @@ const stateTransitionSchema = z.object({
 
 const generatedPageDocumentationSchema = z.object({
   summary: z.string(),
-  can: z.array(capabilitySchema),
-  might: z.array(capabilitySchema),
+  can: z.array(generatedCapabilitySchema),
+  might: z.array(generatedCapabilitySchema),
 });
 
-const pageDocumentationSchema = generatedPageDocumentationSchema.extend({
+const pageDocumentationSchema = z.object({
+  summary: z.string(),
+  can: z.array(capabilitySchema),
+  might: z.array(capabilitySchema),
   interactions: z.array(stateTransitionSchema).optional(),
   qualityNotes: z.array(z.string()).optional(),
 });

package/boat/doc-collector/src/ai/tools.ts

@@ -225,7 +225,7 @@ function buildTransition(candidate: InteractionCandidate, beforeState: WebPageSt
 function describeTargetState(beforeState: WebPageState, afterState: WebPageState, candidate: InteractionCandidate): InteractionState {
   const beforeFocus = detectFocusArea(beforeState.ariaSnapshot || null);
   const afterFocus = detectFocusArea(afterState.ariaSnapshot || null);
-  if (afterFocus.detected && (!beforeFocus.detected || beforeFocus.name !== afterFocus.name)) {
+  if (afterFocus.type && (!beforeFocus.type || beforeFocus.name !== afterFocus.name)) {
     return {
       kind: afterFocus.type || 'dialog',
       label: afterFocus.name || candidate.element.name.trim(),

package/boat/doc-collector/src/cli.ts

@@ -2,6 +2,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { Command } from 'commander';
 import { ConfigCommand } from '../../../src/commands/config-command.ts';
+import { RecommendedModelsCommand } from '../../../src/commands/recommended-models-command.ts';
 import { remote } from '../../../src/remote.ts';
 import { isVerboseMode, setPreserveConsoleLogs, setQuietMode } from '../../../src/utils/logger.ts';
 import { DocBot, type DocbotOptions } from './docbot.ts';
@@ -16,6 +17,7 @@ function buildOptions(options: any): DocbotOptions {
     incognito: options.incognito,
     session: options.session,
     docsConfig: options.docsConfig,
+    baseUrl: options.url,
   };
 }
 
@@ -26,6 +28,7 @@ function addCommonOptions(cmd: Command): Command {
     .option('-c, --config <path>', 'Path to explorbot configuration file')
     .option('--docs-config <path>', 'Path to doc collector configuration file')
     .option('-p, --path <path>', 'Working directory path')
+    .option('--url <url>', 'Base URL of the site, when the path argument is relative (env: EXPLORBOT_URL)')
     .option('-s, --show', 'Show browser window')
     .option('--headless', 'Run browser in headless mode')
     .option('--incognito', 'Run without recording experiences')
@@ -36,7 +39,14 @@ export function createDocsCommands(name = 'docs'): Command {
   const cmd = new Command(name);
   cmd.description('AI-powered website documentation collector');
 
-  addCommonOptions(cmd.command('collect <path>').description('Crawl pages and generate documentation spec').option('--max-pages <count>', 'Maximum number of pages to document')).action(async (startPath, options) => {
+  addCommonOptions(
+    cmd
+      .command('collect <path>')
+      .description('Crawl pages and generate documentation spec')
+      .option('--max-pages <count>', 'Maximum number of pages to document')
+      .option('--no-collapse-template-pages', 'Visit every page even when its layout matches a documented page')
+      .option('--template-similarity <percent>', 'Structural similarity percent that counts pages as the same layout (default 90)')
+  ).action(async (startPath, options) => {
     setPreserveConsoleLogs(true);
 
     try {
@@ -50,8 +60,12 @@ export function createDocsCommands(name = 'docs'): Command {
       if (options.maxPages) {
         maxPages = Number.parseInt(options.maxPages, 10);
       }
+      let templateSimilarity: number | undefined;
+      if (options.templateSimilarity) {
+        templateSimilarity = Number.parseInt(options.templateSimilarity, 10);
+      }
 
-      const result = await bot.collect(startPath, { maxPages });
+      const result = await bot.collect(startPath, { maxPages, collapseTemplatePages: options.collapseTemplatePages, templateSimilarity });
 
       console.log(`\nDocumented ${result.pages.length} page(s)`);
       console.log(`Skipped ${result.skipped.length} page(s)`);
@@ -74,13 +88,15 @@ export function createDocsCommands(name = 'docs'): Command {
     .action(async (url, options) => {
       setQuietMode(!isVerboseMode());
       try {
-        console.log(await ConfigCommand.summary({ config: options.config, path: options.path, url, json: options.json }));
+        console.log(await ConfigCommand.summary({ config: options.config, path: options.path, url: url || options.url, json: options.json }));
       } catch (error) {
         console.error(error instanceof Error ? error.message : 'Unknown error');
         process.exit(1);
       }
     });
 
+  RecommendedModelsCommand.register(cmd);
+
   cmd
     .command('init')
     .description('Initialize doc collector configuration')
@@ -110,6 +126,8 @@ export function createDocsCommands(name = 'docs'): Command {
             interactive: false,
             ignoreErrors: true,
             collapseDynamicPages: true,
+            collapseTemplatePages: true,
+            templateSimilarity: 90,
             scope: 'site',
             includePaths: [],
             excludePaths: [],

package/boat/doc-collector/src/config.ts

@@ -119,6 +119,7 @@ class DocbotConfigParser {
           interactive: false,
           ignoreErrors: true,
           collapseDynamicPages: true,
+          collapseTemplatePages: true,
           scope: 'site',
           includePaths: [],
           excludePaths: [],
@@ -157,6 +158,8 @@ interface DocbotConfig {
     ignoreErrors?: boolean | string[];
     prompt?: string;
     collapseDynamicPages?: boolean;
+    collapseTemplatePages?: boolean;
+    templateSimilarity?: number;
     scope?: 'site' | 'section' | 'subtree';
     includePaths?: string[];
     excludePaths?: string[];