explorbot 0.5.6 → 0.5.8

package/bin/explorbot-cli.ts

@@ -23,6 +23,8 @@ import { isVerboseMode, log, setPreserveConsoleLogs, setQuietMode, tag } from '.
 import { jsonToTable } from '../src/utils/markdown-parser.js';
 import { parseMarkdownToTerminal } from '../src/utils/markdown-terminal.js';
 import { type NextStepSection, printNextSteps, relativeToCwd } from '../src/utils/next-steps.ts';
+import { readCliSitemap } from '../src/utils/sitemap.ts';
+import { UrlFilter } from '../src/utils/url-filter.ts';
 
 const program = new Command();
 const cli = getCliName();
@@ -153,8 +155,10 @@ addCommonOptions(program.command('start [path]').description('Start web explorat
 
 addCommonOptions(
   program
-    .command('explore <path>')
+    .command('explore [path]')
     .description('Explore a page autonomously and run invented scenarios')
+    .option('--sitemap <source>', 'Explore every page listed in a sitemap: file path, URL, or - for stdin (a piped stdin is read automatically)')
+    .option('--url-filter <spec>', 'Pick pages to explore: keys filter|exclude|limit, plus sort:priority|lastmod for a sitemap, e.g. "filter:/admin/*;exclude:/admin/logs/*;limit:20"')
     .option('--max-tests <count>', 'Maximum number of tests to run')
     .option('--max-duration <minutes>', 'Wall-clock budget in minutes for the whole run; wraps up before the limit is hit')
     .option('--focus <feature>', 'Focus area for exploration')
@@ -162,11 +166,15 @@ addCommonOptions(
     .option('--dry-run', 'Mark picked tests as skipped without executing or generating new ones')
 ).action(async (explorePath, options) => {
   try {
-    const explorBot = new ExplorBot(buildExplorBotOptions(explorePath, options));
+    const urlFilter = new UrlFilter(options.urlFilter);
+    const sitemapUrls = await readCliSitemap(explorePath, options.sitemap, urlFilter);
+    const explorBot = new ExplorBot(buildExplorBotOptions(explorePath || sitemapUrls?.[0], options));
     await explorBot.start();
-    await explorBot.visit(explorePath);
+    if (explorePath) await explorBot.visit(explorePath);
     const { ExploreCommand } = await import('../src/commands/explore-command.js');
     const cmd = new ExploreCommand(explorBot);
+    cmd.sitemapUrls = sitemapUrls;
+    cmd.urlFilter = urlFilter;
     if (options.maxTests) cmd.maxTests = Number.parseInt(options.maxTests, 10);
     if (options.maxDuration) cmd.maxDurationMinutes = Number.parseInt(options.maxDuration, 10);
     else if (process.env.EXPLORBOT_MAX_DURATION) cmd.maxDurationMinutes = Number.parseInt(process.env.EXPLORBOT_MAX_DURATION, 10);
@@ -454,10 +462,13 @@ addCommonOptions(
     .option('--scope <prefix>', 'Restrict navigation to URL prefix')
     .option('--max-tests <count>', 'Maximum number of tests to run')
     .option('--max-duration <minutes>', 'Wall-clock budget in minutes for the whole run')
+    .option('--url-filter <spec>', 'Pick pages to explore: keys filter|exclude|limit, e.g. "filter:/admin/*;exclude:/admin/logs/*;limit:20"')
 ).action(async (startUrl, options) => {
   const explorBot = new ExplorBot(buildExplorBotOptions(startUrl || '/', options));
   await explorBot.start();
-  const args = [options.deep && '--deep', options.shallow && '--shallow', options.scope && `--scope ${options.scope}`, options.maxTests && `--max-tests ${options.maxTests}`, options.maxDuration && `--max-duration ${options.maxDuration}`].filter(Boolean).join(' ');
+  const args = [options.deep && '--deep', options.shallow && '--shallow', options.scope && `--scope ${options.scope}`, options.maxTests && `--max-tests ${options.maxTests}`, options.maxDuration && `--max-duration ${options.maxDuration}`, options.urlFilter && `--url-filter "${options.urlFilter}"`]
+    .filter(Boolean)
+    .join(' ');
   const { FreesailCommand } = await import('../src/commands/freesail-command.js');
   const cmd = new FreesailCommand(explorBot);
   await cmd.execute(args);
@@ -491,6 +502,7 @@ program
   .option('-g, --global', 'Configure explorbot in ~/.explorbot to run from anywhere')
   .option('--provider <name>', `AI provider for the global config: ${Object.keys(PROVIDERS).join(', ')}`)
   .option('--api-key <key>', 'API key stored in ~/.explorbot/.env')
+  .option('--url <url>', 'Web application URL written to web.url in the local config')
   .action(async (options) => {
     try {
       const { runInit } = await import('../src/commands/init-command.js');
@@ -501,6 +513,7 @@ program
         global: options.global,
         provider: options.provider,
         apiKey: options.apiKey,
+        url: options.url,
       });
     } catch (error) {
       console.error('Failed:', error instanceof Error ? error.message : 'Unknown error');

package/boat/doc-collector/src/cli.ts

@@ -5,6 +5,8 @@ import { ConfigCommand } from '../../../src/commands/config-command.ts';
 import { RecommendedModelsCommand } from '../../../src/commands/recommended-models-command.ts';
 import { remote } from '../../../src/remote.ts';
 import { isVerboseMode, setPreserveConsoleLogs, setQuietMode } from '../../../src/utils/logger.ts';
+import { readCliSitemap } from '../../../src/utils/sitemap.ts';
+import { UrlFilter } from '../../../src/utils/url-filter.ts';
 import { DocBot, type DocbotOptions } from './docbot.ts';
 
 function buildOptions(options: any): DocbotOptions {
@@ -41,15 +43,19 @@ export function createDocsCommands(name = 'docs'): Command {
 
   addCommonOptions(
     cmd
-      .command('collect <path>')
+      .command('collect [path]')
       .description('Crawl pages and generate documentation spec')
+      .option('--sitemap <source>', 'Document only the pages listed in a sitemap: file path, URL, or - for stdin (a piped stdin is read automatically)')
+      .option('--url-filter <spec>', 'Pick sitemap pages: keys filter|exclude|sort|limit, e.g. "filter:/admin/*;exclude:/admin/logs/*;sort:priority;limit:20"')
       .option('--max-pages <count>', 'Maximum number of pages to document')
       .option('--no-collapse-template-pages', 'Visit every page even when its layout matches a documented page')
       .option('--template-similarity <percent>', 'Structural similarity percent that counts pages as the same layout (default 90)')
-  ).action(async (startPath, options) => {
+  ).action(async (pathArg, options) => {
     setPreserveConsoleLogs(true);
 
     try {
+      const sitemapUrls = await readCliSitemap(pathArg, options.sitemap, new UrlFilter(options.urlFilter));
+      const startPath = pathArg || sitemapUrls![0];
       const bot = new DocBot({
         ...buildOptions(options),
         startUrl: startPath,
@@ -65,7 +71,7 @@ export function createDocsCommands(name = 'docs'): Command {
         templateSimilarity = Number.parseInt(options.templateSimilarity, 10);
       }
 
-      const result = await bot.collect(startPath, { maxPages, collapseTemplatePages: options.collapseTemplatePages, templateSimilarity });
+      const result = await bot.collect(startPath, { maxPages, collapseTemplatePages: options.collapseTemplatePages, templateSimilarity, sitemapUrls });
 
       console.log(`\nDocumented ${result.pages.length} page(s)`);
       console.log(`Skipped ${result.skipped.length} page(s)`);

package/boat/doc-collector/src/docbot.ts

@@ -5,6 +5,7 @@ import type { Link, WebPageState } from '../../../src/state-manager.ts';
 import { normalizeUrl } from '../../../src/state-manager.ts';
 import { tag } from '../../../src/utils/logger.ts';
 import { sanitizeFilename } from '../../../src/utils/strings.ts';
+import { keepSiteUrls } from '../../../src/utils/sitemap.ts';
 import { Documentarian, type PageDocumentation } from './ai/documentarian.ts';
 import type { DocStateTransition } from './ai/tools.ts';
 import { type DocbotConfig, DocbotConfigParser } from './config.ts';
@@ -66,7 +67,12 @@ class DocBot {
     const skipped: SkippedPage[] = [];
     const baseUrl = this.explorBot.getConfig().playwright.url;
 
-    this.enqueuePath(effectiveStartPath, queue, queued);
+    const followLinks = !opts.sitemapUrls;
+    let seeds = [effectiveStartPath];
+    if (opts.sitemapUrls) seeds = keepSiteUrls(opts.sitemapUrls, baseUrl).filter((url) => shouldCrawlDocPath(url, this.config));
+    for (const seed of seeds) {
+      this.enqueuePath(seed, queue, queued);
+    }
 
     while (queue.length > 0 && pages.length < effectiveMaxPages) {
       const target = queue.shift();
@@ -80,7 +86,7 @@ class DocBot {
       }
 
       const stateManager = this.explorBot.stateManager();
-      if (stateManager.hasVisitedState(target)) {
+      if (followLinks && stateManager.hasVisitedState(target)) {
         continue;
       }
 
@@ -110,6 +116,7 @@ class DocBot {
               reason: `same layout as ${templateUrl} (only content differs)`,
             });
             documented.add(pageKey);
+            if (!followLinks) continue;
             for (const nextPath of this.extractNextPaths(state, baseUrl, '')) {
               if (documented.has(this.getPageKey(nextPath)) || stateManager.hasVisitedState(nextPath)) continue;
               this.enqueuePath(nextPath, queue, queued);
@@ -166,6 +173,7 @@ class DocBot {
 
         const templateRecord = buildTemplateRecord(state.url, state.ariaSnapshot ?? null);
         if (templateRecord) templates.push(templateRecord);
+        if (!followLinks) continue;
 
         const nextPaths = this.extractNextPaths(state, baseUrl, research, documentation);
         const interactionPriorityPaths = new Set(this.extractInteractionPaths(baseUrl, documentation));
@@ -559,6 +567,7 @@ interface CollectOptions {
   maxPages?: number;
   collapseTemplatePages?: boolean;
   templateSimilarity?: number;
+  sitemapUrls?: string[];
 }
 
 interface CollectionResult {

package/dist/bin/explorbot-cli.js

@@ -23,6 +23,8 @@ import { isVerboseMode, log, setPreserveConsoleLogs, setQuietMode, tag } from '.
 import { jsonToTable } from '../src/utils/markdown-parser.js';
 import { parseMarkdownToTerminal } from '../src/utils/markdown-terminal.js';
 import { printNextSteps, relativeToCwd } from "../src/utils/next-steps.js";
+import { readCliSitemap } from "../src/utils/sitemap.js";
+import { UrlFilter } from "../src/utils/url-filter.js";
 const program = new Command();
 const cli = getCliName();
 const pkgPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../package.json');
@@ -119,19 +121,26 @@ addCommonOptions(program.command('start [path]').description('Start web explorat
     await startTUI(explorBot);
 });
 addCommonOptions(program
-    .command('explore <path>')
+    .command('explore [path]')
     .description('Explore a page autonomously and run invented scenarios')
+    .option('--sitemap <source>', 'Explore every page listed in a sitemap: file path, URL, or - for stdin (a piped stdin is read automatically)')
+    .option('--url-filter <spec>', 'Pick pages to explore: keys filter|exclude|limit, plus sort:priority|lastmod for a sitemap, e.g. "filter:/admin/*;exclude:/admin/logs/*;limit:20"')
     .option('--max-tests <count>', 'Maximum number of tests to run')
     .option('--max-duration <minutes>', 'Wall-clock budget in minutes for the whole run; wraps up before the limit is hit')
     .option('--focus <feature>', 'Focus area for exploration')
     .option('--configure <spec>', 'Reuse spec: keys new|from|style|subpages|pick_by|priority, e.g. "new:25%;pick_by=random;priority=critical,high"')
     .option('--dry-run', 'Mark picked tests as skipped without executing or generating new ones')).action(async (explorePath, options) => {
     try {
-        const explorBot = new ExplorBot(buildExplorBotOptions(explorePath, options));
+        const urlFilter = new UrlFilter(options.urlFilter);
+        const sitemapUrls = await readCliSitemap(explorePath, options.sitemap, urlFilter);
+        const explorBot = new ExplorBot(buildExplorBotOptions(explorePath || sitemapUrls?.[0], options));
         await explorBot.start();
-        await explorBot.visit(explorePath);
+        if (explorePath)
+            await explorBot.visit(explorePath);
         const { ExploreCommand } = await import('../src/commands/explore-command.js');
         const cmd = new ExploreCommand(explorBot);
+        cmd.sitemapUrls = sitemapUrls;
+        cmd.urlFilter = urlFilter;
         if (options.maxTests)
             cmd.maxTests = Number.parseInt(options.maxTests, 10);
         if (options.maxDuration)
@@ -405,10 +414,13 @@ addCommonOptions(program
     .option('--shallow', 'Breadth-first: pick globally least-visited page')
     .option('--scope <prefix>', 'Restrict navigation to URL prefix')
     .option('--max-tests <count>', 'Maximum number of tests to run')
-    .option('--max-duration <minutes>', 'Wall-clock budget in minutes for the whole run')).action(async (startUrl, options) => {
+    .option('--max-duration <minutes>', 'Wall-clock budget in minutes for the whole run')
+    .option('--url-filter <spec>', 'Pick pages to explore: keys filter|exclude|limit, e.g. "filter:/admin/*;exclude:/admin/logs/*;limit:20"')).action(async (startUrl, options) => {
     const explorBot = new ExplorBot(buildExplorBotOptions(startUrl || '/', options));
     await explorBot.start();
-    const args = [options.deep && '--deep', options.shallow && '--shallow', options.scope && `--scope ${options.scope}`, options.maxTests && `--max-tests ${options.maxTests}`, options.maxDuration && `--max-duration ${options.maxDuration}`].filter(Boolean).join(' ');
+    const args = [options.deep && '--deep', options.shallow && '--shallow', options.scope && `--scope ${options.scope}`, options.maxTests && `--max-tests ${options.maxTests}`, options.maxDuration && `--max-duration ${options.maxDuration}`, options.urlFilter && `--url-filter "${options.urlFilter}"`]
+        .filter(Boolean)
+        .join(' ');
     const { FreesailCommand } = await import('../src/commands/freesail-command.js');
     const cmd = new FreesailCommand(explorBot);
     await cmd.execute(args);
@@ -440,6 +452,7 @@ program
     .option('-g, --global', 'Configure explorbot in ~/.explorbot to run from anywhere')
     .option('--provider <name>', `AI provider for the global config: ${Object.keys(PROVIDERS).join(', ')}`)
     .option('--api-key <key>', 'API key stored in ~/.explorbot/.env')
+    .option('--url <url>', 'Web application URL written to web.url in the local config')
     .action(async (options) => {
     try {
         const { runInit } = await import('../src/commands/init-command.js');
@@ -450,6 +463,7 @@ program
             global: options.global,
             provider: options.provider,
             apiKey: options.apiKey,
+            url: options.url,
         });
     }
     catch (error) {

package/dist/boat/doc-collector/src/cli.js

@@ -5,6 +5,8 @@ import { ConfigCommand } from "../../../src/commands/config-command.js";
 import { RecommendedModelsCommand } from "../../../src/commands/recommended-models-command.js";
 import { remote } from "../../../src/remote.js";
 import { isVerboseMode, setPreserveConsoleLogs, setQuietMode } from "../../../src/utils/logger.js";
+import { readCliSitemap } from "../../../src/utils/sitemap.js";
+import { UrlFilter } from "../../../src/utils/url-filter.js";
 import { DocBot } from "./docbot.js";
 function buildOptions(options) {
     return {
@@ -36,13 +38,17 @@ export function createDocsCommands(name = 'docs') {
     const cmd = new Command(name);
     cmd.description('AI-powered website documentation collector');
     addCommonOptions(cmd
-        .command('collect <path>')
+        .command('collect [path]')
         .description('Crawl pages and generate documentation spec')
+        .option('--sitemap <source>', 'Document only the pages listed in a sitemap: file path, URL, or - for stdin (a piped stdin is read automatically)')
+        .option('--url-filter <spec>', 'Pick sitemap pages: keys filter|exclude|sort|limit, e.g. "filter:/admin/*;exclude:/admin/logs/*;sort:priority;limit:20"')
         .option('--max-pages <count>', 'Maximum number of pages to document')
         .option('--no-collapse-template-pages', 'Visit every page even when its layout matches a documented page')
-        .option('--template-similarity <percent>', 'Structural similarity percent that counts pages as the same layout (default 90)')).action(async (startPath, options) => {
+        .option('--template-similarity <percent>', 'Structural similarity percent that counts pages as the same layout (default 90)')).action(async (pathArg, options) => {
         setPreserveConsoleLogs(true);
         try {
+            const sitemapUrls = await readCliSitemap(pathArg, options.sitemap, new UrlFilter(options.urlFilter));
+            const startPath = pathArg || sitemapUrls[0];
             const bot = new DocBot({
                 ...buildOptions(options),
                 startUrl: startPath,
@@ -56,7 +62,7 @@ export function createDocsCommands(name = 'docs') {
             if (options.templateSimilarity) {
                 templateSimilarity = Number.parseInt(options.templateSimilarity, 10);
             }
-            const result = await bot.collect(startPath, { maxPages, collapseTemplatePages: options.collapseTemplatePages, templateSimilarity });
+            const result = await bot.collect(startPath, { maxPages, collapseTemplatePages: options.collapseTemplatePages, templateSimilarity, sitemapUrls });
             console.log(`\nDocumented ${result.pages.length} page(s)`);
             console.log(`Skipped ${result.skipped.length} page(s)`);
             console.log(`Spec index: ${result.indexPath}`);

package/dist/boat/doc-collector/src/docbot.js

@@ -4,6 +4,7 @@ import { ExplorBot } from "../../../src/explorbot.js";
 import { normalizeUrl } from "../../../src/state-manager.js";
 import { tag } from "../../../src/utils/logger.js";
 import { sanitizeFilename } from "../../../src/utils/strings.js";
+import { keepSiteUrls } from "../../../src/utils/sitemap.js";
 import { Documentarian } from "./ai/documentarian.js";
 import { DocbotConfigParser } from "./config.js";
 import { renderPageDocumentation, renderSpecIndex } from "./docs-renderer.js";
@@ -58,7 +59,13 @@ class DocBot {
         const pages = [];
         const skipped = [];
         const baseUrl = this.explorBot.getConfig().playwright.url;
-        this.enqueuePath(effectiveStartPath, queue, queued);
+        const followLinks = !opts.sitemapUrls;
+        let seeds = [effectiveStartPath];
+        if (opts.sitemapUrls)
+            seeds = keepSiteUrls(opts.sitemapUrls, baseUrl).filter((url) => shouldCrawlDocPath(url, this.config));
+        for (const seed of seeds) {
+            this.enqueuePath(seed, queue, queued);
+        }
         while (queue.length > 0 && pages.length < effectiveMaxPages) {
             const target = queue.shift();
             if (!target) {
@@ -69,7 +76,7 @@ class DocBot {
                 continue;
             }
             const stateManager = this.explorBot.stateManager();
-            if (stateManager.hasVisitedState(target)) {
+            if (followLinks && stateManager.hasVisitedState(target)) {
                 continue;
             }
             try {
@@ -95,6 +102,8 @@ class DocBot {
                             reason: `same layout as ${templateUrl} (only content differs)`,
                         });
                         documented.add(pageKey);
+                        if (!followLinks)
+                            continue;
                         for (const nextPath of this.extractNextPaths(state, baseUrl, '')) {
                             if (documented.has(this.getPageKey(nextPath)) || stateManager.hasVisitedState(nextPath))
                                 continue;
@@ -149,6 +158,8 @@ class DocBot {
                 const templateRecord = buildTemplateRecord(state.url, state.ariaSnapshot ?? null);
                 if (templateRecord)
                     templates.push(templateRecord);
+                if (!followLinks)
+                    continue;
                 const nextPaths = this.extractNextPaths(state, baseUrl, research, documentation);
                 const interactionPriorityPaths = new Set(this.extractInteractionPaths(baseUrl, documentation));
                 for (const nextPath of nextPaths) {

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.5.6",
+  "version": "0.5.8",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",

package/dist/src/ai/navigator.d.ts

@@ -7,6 +7,7 @@ import Explorer from '../explorer.js';
 import type { KnowledgeTracker } from '../knowledge-tracker.js';
 import { type StateManager } from '../state-manager.js';
 import { HooksRunner } from '../utils/hooks-runner.js';
+import type { UrlFilter } from '../utils/url-filter.js';
 import type { Agent, AgentDeps } from './agent.js';
 import { type Decision, type Judge } from './judge.js';
 import type { Provider } from './provider.js';
@@ -72,6 +73,7 @@ declare class Navigator implements Agent {
         strategy?: 'deep' | 'shallow';
         scope?: string;
         visitedUrls?: Set<string>;
+        urlFilter?: UrlFilter;
     }, actionResult?: ActionResult): Promise<{
         target: string;
         reason: string;

package/dist/src/ai/navigator.js

@@ -555,6 +555,12 @@ class Navigator {
         if (opts?.scope) {
             scopeInstruction = `IMPORTANT: Only suggest URLs that start with "${opts.scope}". Do not suggest URLs outside this scope.`;
         }
+        if (opts?.urlFilter?.include.length) {
+            scopeInstruction += `\nOnly suggest URLs matching one of these patterns: ${opts.urlFilter.include.join(', ')}`;
+        }
+        if (opts?.urlFilter?.exclude.length) {
+            scopeInstruction += `\nNever suggest URLs matching these patterns: ${opts.urlFilter.exclude.join(', ')}`;
+        }
         const prompt = dedent `
       <research>
       ${research || 'No cached research available'}
@@ -599,6 +605,10 @@ class Navigator {
                 conversation.addUserText(`"${target}" is outside scope "${opts.scope}". Suggest a URL within scope.`);
                 return;
             }
+            if (opts?.urlFilter && !opts.urlFilter.allows(target)) {
+                conversation.addUserText(`"${target}" does not pass the URL filter. Suggest a URL that does.`);
+                return;
+            }
             if (!target.startsWith('/') && !target.startsWith('http')) {
                 conversation.addUserText(`"${target}" is not a valid URL path. Suggest a URL path starting with "/".`);
                 return;

package/dist/src/ai/pilot.js

@@ -537,7 +537,7 @@ export class Pilot {
         if (finalState?.screenshot && this.provider.hasVision())
             image = finalState.screenshot;
         const decided = (text) => {
-            if (task.hasAchievedAny() && !task.getRemainingExpectations().includes(text))
+            if (task.hasAchieved(text))
                 return 'passed';
             return 'failed';
         };
@@ -553,7 +553,7 @@ export class Pilot {
         const schema = z.object({
             outcomes: z.array(z.object({
                 expectation: z.string().describe('The expected outcome, repeated exactly as it was given'),
-                status: z.enum(['passed', 'failed', 'unverified', 'contradiction']).describe('passed = the evidence shows it happened, failed = the evidence shows it did not, unverified = the run never established either way, contradiction = the picture and the run disagree'),
+                status: z.enum(['passed', 'failed', 'unverified', 'contradiction']).describe('passed = the evidence shows it happened to the thing it names, failed = the evidence shows it did not, unverified = the run never established either way, contradiction = the picture and the run disagree'),
                 evidence: z.string().nullable().describe('What settled it. For a contradiction, what each side shows. Null when there is nothing to add'),
             })),
         });
@@ -576,6 +576,10 @@ export class Pilot {
         const userContent = dedent `
       A test run has finished. Decide, for each expected outcome, what the run established about it.
 
+      <scenario>
+      ${task.scenario}
+      </scenario>
+
       <expected_outcomes>
       ${undecided.map((text) => `- ${text}`).join('\n')}
       </expected_outcomes>
@@ -588,6 +592,9 @@ export class Pilot {
 
       The log is written in the tester's own words, so an outcome can be satisfied by a step that describes it
       differently. Judge by what the steps show happened, not by whether the wording matches.
+      Wording may differ, the subject may not. An outcome refers to what the scenario names or chooses; it holds
+      only when that same thing appears. A different route or a different thing reaching a similar state is "failed",
+      and so is an outcome about something the run never managed to choose or create.
       Choose "unverified" only when the evidence neither shows the outcome happening nor shows it failing —
       that is a statement about the run, not about the application.
     `;

package/dist/src/commands/explore-command.d.ts

@@ -1,4 +1,5 @@
 import { type Plan, type Test } from '../test-plan.js';
+import { UrlFilter } from '../utils/url-filter.js';
 import { type ArgumentCompletion, BaseCommand, type Suggestion } from './base-command.js';
 export declare const DEADLINE_RESERVE_MS: number;
 export declare const DEADLINE_TEST_ALLOWANCE_MS: number;
@@ -14,6 +15,8 @@ export declare class ExploreCommand extends BaseCommand {
     maxDurationMinutes?: number;
     hardDeadlineAt?: number;
     dryRun: boolean;
+    sitemapUrls?: string[];
+    urlFilter: UrlFilter;
     testsRun: number;
     deadlineLogged: boolean;
     completedPlans: Plan[];
@@ -26,8 +29,10 @@ export declare class ExploreCommand extends BaseCommand {
     originLabel(test: Test): string;
     printPreview(label: string, tests: Test[]): void;
     runFreshMode(mainUrl: string | undefined, feature: string | undefined, styles?: string[]): Promise<void>;
+    runSitemapMode(sitemapUrls: string[], feature: string | undefined, styles?: string[]): Promise<void>;
     runReuseMode(mainUrl: string | undefined, feature: string | undefined, cfg: ConfigureSpec): Promise<void>;
     discoverNewSubPages(mainPlan: Plan, mainUrl: string | undefined, styles: string[] | undefined, knownUrls: Set<string>): Promise<void>;
+    explorePage(url: string, feature: string | undefined, parentPlan: Plan | undefined, styles?: string[]): Promise<boolean>;
     replanAndRun(pageUrl: string | undefined, feature: string | undefined, existingPlan: Plan, styles?: string[]): Promise<void>;
     runAllStyles(pageUrl?: string, feature?: string, parentPlan?: Plan, completedPlans?: Plan[], styles?: string[]): Promise<void>;
     rememberCurrentPlan(): void;

package/dist/src/commands/explore-command.js

@@ -9,7 +9,10 @@ import { getCliName } from "../utils/cli-name.js";
 import { ErrorPageError, getStateErrorPageError } from "../utils/error-page.js";
 import { tag } from '../utils/logger.js';
 import { printNextSteps, relativeToCwd } from "../utils/next-steps.js";
+import { keepSiteUrls, readSitemap } from "../utils/sitemap.js";
+import { parseSpecPairs } from "../utils/spec.js";
 import { safeFilename } from "../utils/strings.js";
+import { UrlFilter } from "../utils/url-filter.js";
 import { BaseCommand } from './base-command.js';
 const MAX_SUB_PAGE_ATTEMPTS = 30;
 const PRIORITY_ORDER = { critical: 0, important: 1, high: 2, normal: 3, low: 4 };
@@ -24,6 +27,8 @@ export class ExploreCommand extends BaseCommand {
         { flags: '--focus <feature>', description: 'Focus area for exploration' },
         { flags: '--configure <spec>', description: 'Reuse spec: keys new|from|style|subpages|pick_by|priority, e.g. "new:25%;pick_by=random;priority=critical,high"' },
         { flags: '--dry-run', description: 'Mark picked tests as skipped without executing or generating new ones' },
+        { flags: '--sitemap <source>', description: 'Explore every page listed in a sitemap (file path, URL, or - for stdin) instead of discovering sub-pages' },
+        { flags: '--url-filter <spec>', description: 'Pick pages to explore: keys filter|exclude|limit, plus sort:priority|lastmod for a sitemap, e.g. "filter:/admin/*;exclude:/admin/logs/*;limit:20"' },
     ];
     suggestions = [
         { command: 'navigate <page>', hint: 'go to another page' },
@@ -34,6 +39,8 @@ export class ExploreCommand extends BaseCommand {
     maxDurationMinutes;
     hardDeadlineAt;
     dryRun = false;
+    sitemapUrls;
+    urlFilter = new UrlFilter();
     testsRun = 0;
     deadlineLogged = false;
     completedPlans = [];
@@ -52,6 +59,10 @@ export class ExploreCommand extends BaseCommand {
         if (opts.maxDuration) {
             this.maxDurationMinutes = Number.parseInt(opts.maxDuration, 10);
         }
+        if (opts.urlFilter)
+            this.urlFilter = new UrlFilter(opts.urlFilter);
+        if (opts.sitemap)
+            this.sitemapUrls = await readSitemap(opts.sitemap, this.urlFilter);
         if (this.hardDeadlineAt == null && this.maxDurationMinutes != null) {
             this.hardDeadlineAt = Date.now() + this.maxDurationMinutes * 60_000 - DEADLINE_RESERVE_MS;
         }
@@ -72,12 +83,15 @@ export class ExploreCommand extends BaseCommand {
         Stats.focus ??= feature;
         const mainUrl = this.getCurrentPageUrl();
         const error = getStateErrorPageError(this.explorBot.stateManager().getCurrentState());
-        if (error) {
+        if (error && !this.sitemapUrls) {
             tag('warning').log(error.message);
             return;
         }
         try {
-            if (cfg.enabled) {
+            if (this.sitemapUrls) {
+                await this.runSitemapMode(this.sitemapUrls, feature, cfg.styles);
+            }
+            else if (cfg.enabled) {
                 await this.runReuseMode(mainUrl, feature, cfg);
             }
             else {
@@ -130,6 +144,21 @@ export class ExploreCommand extends BaseCommand {
             return;
         await this.discoverNewSubPages(mainPlan, mainUrl, styles, new Set());
     }
+    async runSitemapMode(sitemapUrls, feature, styles) {
+        const urls = keepSiteUrls(sitemapUrls, this.explorBot.getConfig().playwright.url);
+        tag('info').log(`Sitemap: ${urls.length} page(s) to explore`);
+        for (const url of urls) {
+            if (this.isLimitReached())
+                break;
+            tag('info').log(`Exploring sitemap page: ${url}`);
+            try {
+                await this.explorePage(url, feature, undefined, styles);
+            }
+            catch (err) {
+                tag('warning').log(`Sitemap page exploration failed: ${err instanceof Error ? err.message : err}`);
+            }
+        }
+    }
     async runReuseMode(mainUrl, feature, cfg) {
         const filename = cfg.fromPath || this.explorBot.generatePlanFilename(feature);
         let loadedPlans = [];
@@ -226,6 +255,8 @@ export class ExploreCommand extends BaseCommand {
                     break;
                 if (!subPlan.url)
                     continue;
+                if (!this.urlFilter.admit(subPlan.url))
+                    continue;
                 try {
                     if (!this.dryRun)
                         await this.explorBot.visit(subPlan.url);
@@ -257,27 +288,22 @@ export class ExploreCommand extends BaseCommand {
                 break;
             const candidates = planner.collectSubPageCandidates(mainPlan, mainUrl || '/').filter((c) => {
                 const norm = normalizeUrl(c.url);
-                return !this.failedSubPages.has(norm) && !knownUrls.has(norm);
+                return !this.failedSubPages.has(norm) && !knownUrls.has(norm) && this.urlFilter.allows(c.url);
             });
             if (candidates.length === 0)
                 break;
             const pick = await planner.pickNextSubPage(candidates);
             if (!pick)
                 break;
+            if (!this.urlFilter.admit(pick.url))
+                break;
             tag('info').log(`Exploring sub-page: ${pick.url} (${pick.reason})`);
             try {
-                await this.explorBot.visit(pick.url);
-                const errorPage = getStateErrorPageError(this.explorBot.stateManager().getCurrentState());
-                if (errorPage) {
-                    tag('warning').log(`Skipping sub-page: ${errorPage.message}`);
+                const explored = await this.explorePage(pick.url, undefined, mainPlan, styles);
+                if (!explored) {
                     this.failedSubPages.add(normalizeUrl(pick.url));
                     continue;
                 }
-                await this.runAllStyles(pick.url, undefined, mainPlan, this.completedPlans, styles);
-                const subPlan = this.explorBot.getCurrentPlan();
-                if (subPlan?.tests.length && !this.completedPlans.includes(subPlan)) {
-                    this.completedPlans.push(subPlan);
-                }
                 knownUrls.add(normalizeUrl(pick.url));
             }
             catch (err) {
@@ -286,6 +312,20 @@ export class ExploreCommand extends BaseCommand {
             }
         }
     }
+    async explorePage(url, feature, parentPlan, styles) {
+        await this.explorBot.visit(url);
+        const errorPage = getStateErrorPageError(this.explorBot.stateManager().getCurrentState());
+        if (errorPage) {
+            tag('warning').log(`Skipping page: ${errorPage.message}`);
+            return false;
+        }
+        await this.runAllStyles(url, feature, parentPlan, this.completedPlans, styles);
+        const plan = this.explorBot.getCurrentPlan();
+        if (plan?.tests.length && !this.completedPlans.includes(plan)) {
+            this.completedPlans.push(plan);
+        }
+        return true;
+    }
     async replanAndRun(pageUrl, feature, existingPlan, styles) {
         const styleList = styles ?? Object.keys(getStyles());
         for (const style of styleList) {
@@ -366,17 +406,7 @@ export class ExploreCommand extends BaseCommand {
         const allStyles = Object.keys(getStyles());
         const validSubpages = new Set(['none', 'same', 'new', 'both']);
         let hasReuseSignal = false;
-        for (const pair of raw.split(';')) {
-            const trimmed = pair.trim();
-            if (!trimmed)
-                continue;
-            const sepMatch = trimmed.match(/^([^:=]+)\s*[:=]\s*(.*)$/);
-            if (!sepMatch) {
-                tag('warning').log(`Ignoring malformed configure pair: ${trimmed}`);
-                continue;
-            }
-            const key = sepMatch[1].trim().toLowerCase();
-            const value = sepMatch[2].trim();
+        for (const { key, value } of parseSpecPairs(raw)) {
             if (key === 'new') {
                 const ratio = parseRatio(value);
                 if (ratio == null) {
@@ -546,6 +576,13 @@ export class ExploreCommand extends BaseCommand {
                 commands,
             });
         }
+        sections.push({
+            label: 'Explore more',
+            commands: [
+                { label: 'Whole site', command: `${cli} explore /` },
+                { label: 'Docs', command: 'https://testomat.ai/docs/explorbot' },
+            ],
+        });
         printNextSteps(sections);
     }
     isLimitReached() {