explorbot 0.5.6 → 0.5.7

package/dist/src/test-plan.js

@@ -272,10 +272,11 @@ export class Test extends Task {
             return Object.values(this.notes).some((note) => note.message === expectation && !!note.status);
         });
     }
+    hasAchieved(expectation) {
+        return Object.values(this.notes).some((note) => note.message === expectation && note.status === TestResult.PASSED);
+    }
     hasAchievedAny() {
-        return this.expected.some((expectation) => {
-            return Object.values(this.notes).some((note) => note.message === expectation && note.status === TestResult.PASSED);
-        });
+        return this.expected.some((expectation) => this.hasAchieved(expectation));
     }
     getRunResult() {
         if (this.isSuccessful)
@@ -285,9 +286,7 @@ export class Test extends Task {
         return super.getRunResult();
     }
     hasAchievedAll() {
-        return this.expected.every((expectation) => {
-            return Object.values(this.notes).some((note) => note.message === expectation && note.status === TestResult.PASSED);
-        });
+        return this.expected.every((expectation) => this.hasAchieved(expectation));
     }
     isComplete() {
         return this.expected.every((expectation) => {

package/dist/src/utils/sitemap.d.ts

@@ -0,0 +1,11 @@
+import type { UrlFilter } from './url-filter.js';
+export declare function readSitemap(source: string, filter?: UrlFilter): Promise<string[]>;
+export declare function readCliSitemap(path: string | undefined, source: string | undefined, filter?: UrlFilter): Promise<string[] | undefined>;
+export declare function keepSiteUrls(urls: string[], siteUrl: string): string[];
+export declare function extractEntries(content: string): SitemapEntry[];
+interface SitemapEntry {
+    url: string;
+    priority: number;
+    lastmod?: number;
+}
+export {};

package/dist/src/utils/sitemap.js

@@ -0,0 +1,106 @@
+import { readFileSync } from 'node:fs';
+import { tag } from './logger.js';
+const ENTRY_PATTERN = /<(url|sitemap)\b[^>]*>(.*?)<\/\1>/gis;
+const XML_ENTITIES = { '&amp;': '&', '&lt;': '<', '&gt;': '>', '&quot;': '"', '&apos;': "'" };
+const MAX_NESTED_SITEMAPS = 50;
+const DEFAULT_PRIORITY = 0.5;
+const STDIN_SOURCE = '-';
+export async function readSitemap(source, filter) {
+    const entries = [];
+    const seen = new Set();
+    const pending = [source];
+    let fetched = 0;
+    while (pending.length > 0) {
+        if (fetched >= MAX_NESTED_SITEMAPS) {
+            tag('warning').log(`Sitemap index lists more than ${MAX_NESTED_SITEMAPS} sitemaps, the rest are ignored`);
+            break;
+        }
+        const next = pending.shift();
+        fetched++;
+        const content = await readSource(next);
+        const isIndex = content.includes('<sitemapindex');
+        for (const entry of extractEntries(content)) {
+            if (seen.has(entry.url))
+                continue;
+            seen.add(entry.url);
+            if (isIndex) {
+                pending.push(entry.url);
+                continue;
+            }
+            entries.push(entry);
+        }
+    }
+    return applyFilter(entries, filter).map((entry) => entry.url);
+}
+export async function readCliSitemap(path, source, filter) {
+    const usage = 'Pass a page path, --sitemap <source>, or pipe a sitemap to stdin';
+    let resolved = source;
+    if (!resolved && !path && !process.stdin.isTTY)
+        resolved = STDIN_SOURCE;
+    if (!resolved && !path)
+        throw new Error(usage);
+    if (!resolved)
+        return undefined;
+    const urls = await readSitemap(resolved, filter);
+    if (urls.length === 0 && !source)
+        throw new Error(usage);
+    if (urls.length === 0)
+        throw new Error(`No URLs left in sitemap ${resolved}`);
+    return urls;
+}
+export function keepSiteUrls(urls, siteUrl) {
+    const siteOrigin = URL.parse(siteUrl)?.origin;
+    const kept = urls.filter((url) => url.startsWith('/') || URL.parse(url)?.origin === siteOrigin);
+    if (kept.length < urls.length)
+        tag('warning').log(`Skipped ${urls.length - kept.length} sitemap URL(s) outside ${siteOrigin}`);
+    return kept;
+}
+export function extractEntries(content) {
+    if (!content.trimStart().startsWith('<')) {
+        return content
+            .split('\n')
+            .map((line) => line.trim())
+            .filter((line) => line.startsWith('http://') || line.startsWith('https://') || line.startsWith('/'))
+            .map((url) => ({ url, priority: DEFAULT_PRIORITY }));
+    }
+    const entries = [];
+    for (const match of content.matchAll(ENTRY_PATTERN)) {
+        const url = tagValue(match[2], 'loc');
+        if (!url)
+            continue;
+        const entry = { url, priority: DEFAULT_PRIORITY };
+        const priority = Number.parseFloat(tagValue(match[2], 'priority') || '');
+        if (!Number.isNaN(priority))
+            entry.priority = priority;
+        const lastmod = Date.parse(tagValue(match[2], 'lastmod') || '');
+        if (!Number.isNaN(lastmod))
+            entry.lastmod = lastmod;
+        entries.push(entry);
+    }
+    return entries;
+}
+function applyFilter(entries, filter) {
+    if (!filter)
+        return entries;
+    let result = entries.filter((entry) => filter.allows(entry.url));
+    if (filter.sort === 'priority')
+        result = result.toSorted((a, b) => b.priority - a.priority);
+    if (filter.sort === 'lastmod')
+        result = result.toSorted((a, b) => (b.lastmod ?? 0) - (a.lastmod ?? 0));
+    return result.slice(0, filter.limit);
+}
+function tagValue(block, name) {
+    const match = block.match(new RegExp(`<${name}>\\s*(?:<!\\[CDATA\\[)?\\s*(.*?)\\s*(?:\\]\\]>)?\\s*</${name}>`, 'is'));
+    return match?.[1].replace(/&(amp|lt|gt|quot|apos);/g, (entity) => XML_ENTITIES[entity]) || undefined;
+}
+async function readSource(source) {
+    if (source === STDIN_SOURCE)
+        return readFileSync(0, 'utf8');
+    if (source.startsWith('http://') || source.startsWith('https://')) {
+        const response = await fetch(source);
+        if (!response.ok)
+            throw new Error(`Sitemap ${source} returned HTTP ${response.status}`);
+        return await response.text();
+    }
+    return readFileSync(source, 'utf8');
+}

package/dist/src/utils/spec.d.ts

@@ -0,0 +1,5 @@
+export declare function parseSpecPairs(raw: string | undefined): SpecPair[];
+export interface SpecPair {
+    key: string;
+    value: string;
+}

package/dist/src/utils/spec.js

@@ -0,0 +1,16 @@
+import { tag } from './logger.js';
+export function parseSpecPairs(raw) {
+    const pairs = [];
+    for (const pair of (raw || '').split(';')) {
+        const trimmed = pair.trim();
+        if (!trimmed)
+            continue;
+        const sepMatch = trimmed.match(/^([^:=]+)\s*[:=]\s*(.*)$/);
+        if (!sepMatch) {
+            tag('warning').log(`Ignoring malformed spec pair: ${trimmed}`);
+            continue;
+        }
+        pairs.push({ key: sepMatch[1].trim().toLowerCase(), value: sepMatch[2].trim() });
+    }
+    return pairs;
+}

package/dist/src/utils/url-filter.d.ts

@@ -0,0 +1,10 @@
+export declare class UrlFilter {
+    readonly include: string[];
+    readonly exclude: string[];
+    sort?: 'priority' | 'lastmod';
+    limit: number;
+    admitted: Set<string>;
+    constructor(spec?: string);
+    allows(url: string): boolean;
+    admit(url: string): boolean;
+}

package/dist/src/utils/url-filter.js

@@ -0,0 +1,52 @@
+import { tag } from './logger.js';
+import { parseSpecPairs } from "./spec.js";
+import { matchesUrl } from "./url-matcher.js";
+export class UrlFilter {
+    include = [];
+    exclude = [];
+    sort;
+    limit = Number.POSITIVE_INFINITY;
+    admitted = new Set();
+    constructor(spec) {
+        for (const { key, value } of parseSpecPairs(spec)) {
+            if (key === 'filter') {
+                this.include.push(value);
+                continue;
+            }
+            if (key === 'exclude') {
+                this.exclude.push(value);
+                continue;
+            }
+            if (key === 'sort' && (value === 'priority' || value === 'lastmod')) {
+                this.sort = value;
+                continue;
+            }
+            if (key === 'limit' && Number.parseInt(value, 10) > 0) {
+                this.limit = Number.parseInt(value, 10);
+                continue;
+            }
+            tag('warning').log(`Ignoring URL filter pair ${key}:${value} (use filter|exclude:<url pattern>, sort:priority|lastmod, limit:<n>)`);
+        }
+    }
+    allows(url) {
+        const path = urlPath(url);
+        if (this.include.length > 0 && !this.include.some((pattern) => matchesUrl(pattern, path)))
+            return false;
+        return !this.exclude.some((pattern) => matchesUrl(pattern, path));
+    }
+    admit(url) {
+        if (!this.allows(url))
+            return false;
+        const path = urlPath(url);
+        if (this.admitted.has(path))
+            return true;
+        if (this.admitted.size >= this.limit)
+            return false;
+        this.admitted.add(path);
+        return true;
+    }
+}
+function urlPath(url) {
+    const parsed = new URL(url, 'http://localhost');
+    return `${parsed.pathname}${parsed.search}`;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.5.6",
+  "version": "0.5.7",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",

package/src/ai/navigator.ts

@@ -20,6 +20,7 @@ import { loop, pause } from '../utils/loop.js';
 import { RulesLoader } from '../utils/rules-loader.ts';
 import { normalizeInlineText } from '../utils/strings.ts';
 import { extractStatePath, isSameHostFamily, matchesNavigationUrl } from '../utils/url-matcher.js';
+import type { UrlFilter } from '../utils/url-filter.ts';
 import type { Agent, AgentDeps } from './agent.js';
 import type { Conversation } from './conversation.js';
 import { type Decision, JUDGE_PAGE_CAP, type Judge, UNDECIDED } from './judge.ts';
@@ -564,7 +565,7 @@ class Navigator implements Agent {
     return { learnExperience: createLearnExperienceTool({ getExperienceTracker: () => this.experienceTracker, getState }) };
   }
 
-  async freeSail(opts?: { strategy?: 'deep' | 'shallow'; scope?: string; visitedUrls?: Set<string> }, actionResult?: ActionResult): Promise<{ target: string; reason: string } | null> {
+  async freeSail(opts?: { strategy?: 'deep' | 'shallow'; scope?: string; visitedUrls?: Set<string>; urlFilter?: UrlFilter }, actionResult?: ActionResult): Promise<{ target: string; reason: string } | null> {
     const stateManager = this.stateManager;
     const state = stateManager.getCurrentState();
     if (!state) {
@@ -612,6 +613,12 @@ class Navigator implements Agent {
     if (opts?.scope) {
       scopeInstruction = `IMPORTANT: Only suggest URLs that start with "${opts.scope}". Do not suggest URLs outside this scope.`;
     }
+    if (opts?.urlFilter?.include.length) {
+      scopeInstruction += `\nOnly suggest URLs matching one of these patterns: ${opts.urlFilter.include.join(', ')}`;
+    }
+    if (opts?.urlFilter?.exclude.length) {
+      scopeInstruction += `\nNever suggest URLs matching these patterns: ${opts.urlFilter.exclude.join(', ')}`;
+    }
 
     const prompt = dedent`
       <research>
@@ -665,6 +672,11 @@ class Navigator implements Agent {
           return;
         }
 
+        if (opts?.urlFilter && !opts.urlFilter.allows(target)) {
+          conversation.addUserText(`"${target}" does not pass the URL filter. Suggest a URL that does.`);
+          return;
+        }
+
         if (!target.startsWith('/') && !target.startsWith('http')) {
           conversation.addUserText(`"${target}" is not a valid URL path. Suggest a URL path starting with "/".`);
           return;

package/src/ai/pilot.ts

@@ -615,7 +615,7 @@ export class Pilot implements Agent {
     if (finalState?.screenshot && this.provider.hasVision()) image = finalState.screenshot;
 
     const decided = (text: string): 'passed' | 'failed' => {
-      if (task.hasAchievedAny() && !task.getRemainingExpectations().includes(text)) return 'passed';
+      if (task.hasAchieved(text)) return 'passed';
       return 'failed';
     };
 
@@ -631,7 +631,7 @@ export class Pilot implements Agent {
       outcomes: z.array(
         z.object({
           expectation: z.string().describe('The expected outcome, repeated exactly as it was given'),
-          status: z.enum(['passed', 'failed', 'unverified', 'contradiction']).describe('passed = the evidence shows it happened, failed = the evidence shows it did not, unverified = the run never established either way, contradiction = the picture and the run disagree'),
+          status: z.enum(['passed', 'failed', 'unverified', 'contradiction']).describe('passed = the evidence shows it happened to the thing it names, failed = the evidence shows it did not, unverified = the run never established either way, contradiction = the picture and the run disagree'),
           evidence: z.string().nullable().describe('What settled it. For a contradiction, what each side shows. Null when there is nothing to add'),
         })
       ),
@@ -657,6 +657,10 @@ export class Pilot implements Agent {
     const userContent = dedent`
       A test run has finished. Decide, for each expected outcome, what the run established about it.
 
+      <scenario>
+      ${task.scenario}
+      </scenario>
+
       <expected_outcomes>
       ${undecided.map((text) => `- ${text}`).join('\n')}
       </expected_outcomes>
@@ -669,6 +673,9 @@ export class Pilot implements Agent {
 
       The log is written in the tester's own words, so an outcome can be satisfied by a step that describes it
       differently. Judge by what the steps show happened, not by whether the wording matches.
+      Wording may differ, the subject may not. An outcome refers to what the scenario names or chooses; it holds
+      only when that same thing appears. A different route or a different thing reaching a similar state is "failed",
+      and so is an outcome about something the run never managed to choose or create.
       Choose "unverified" only when the evidence neither shows the outcome happening nor shows it failing —
       that is a statement about the run, not about the application.
     `;

package/src/commands/explore-command.ts

@@ -9,7 +9,10 @@ import { getCliName } from '../utils/cli-name.ts';
 import { ErrorPageError, getStateErrorPageError } from '../utils/error-page.ts';
 import { tag } from '../utils/logger.js';
 import { type NextStepSection, printNextSteps, relativeToCwd } from '../utils/next-steps.ts';
+import { keepSiteUrls, readSitemap } from '../utils/sitemap.ts';
+import { parseSpecPairs } from '../utils/spec.ts';
 import { safeFilename } from '../utils/strings.ts';
+import { UrlFilter } from '../utils/url-filter.ts';
 import { type ArgumentCompletion, BaseCommand, type Suggestion } from './base-command.js';
 
 const MAX_SUB_PAGE_ATTEMPTS = 30;
@@ -26,6 +29,8 @@ export class ExploreCommand extends BaseCommand {
     { flags: '--focus <feature>', description: 'Focus area for exploration' },
     { flags: '--configure <spec>', description: 'Reuse spec: keys new|from|style|subpages|pick_by|priority, e.g. "new:25%;pick_by=random;priority=critical,high"' },
     { flags: '--dry-run', description: 'Mark picked tests as skipped without executing or generating new ones' },
+    { flags: '--sitemap <source>', description: 'Explore every page listed in a sitemap (file path, URL, or - for stdin) instead of discovering sub-pages' },
+    { flags: '--url-filter <spec>', description: 'Pick pages to explore: keys filter|exclude|limit, plus sort:priority|lastmod for a sitemap, e.g. "filter:/admin/*;exclude:/admin/logs/*;limit:20"' },
   ];
   suggestions: Suggestion[] = [
     { command: 'navigate <page>', hint: 'go to another page' },
@@ -37,6 +42,8 @@ export class ExploreCommand extends BaseCommand {
   maxDurationMinutes?: number;
   hardDeadlineAt?: number;
   dryRun = false;
+  sitemapUrls?: string[];
+  urlFilter = new UrlFilter();
   private testsRun = 0;
   private deadlineLogged = false;
   private completedPlans: Plan[] = [];
@@ -57,6 +64,8 @@ export class ExploreCommand extends BaseCommand {
     if (opts.maxDuration) {
       this.maxDurationMinutes = Number.parseInt(opts.maxDuration as string, 10);
     }
+    if (opts.urlFilter) this.urlFilter = new UrlFilter(opts.urlFilter as string);
+    if (opts.sitemap) this.sitemapUrls = await readSitemap(opts.sitemap as string, this.urlFilter);
     if (this.hardDeadlineAt == null && this.maxDurationMinutes != null) {
       this.hardDeadlineAt = Date.now() + this.maxDurationMinutes * 60_000 - DEADLINE_RESERVE_MS;
     }
@@ -76,13 +85,15 @@ export class ExploreCommand extends BaseCommand {
     Stats.focus ??= feature;
     const mainUrl = this.getCurrentPageUrl();
     const error = getStateErrorPageError(this.explorBot.stateManager().getCurrentState());
-    if (error) {
+    if (error && !this.sitemapUrls) {
       tag('warning').log(error.message);
       return;
     }
 
     try {
-      if (cfg.enabled) {
+      if (this.sitemapUrls) {
+        await this.runSitemapMode(this.sitemapUrls, feature, cfg.styles);
+      } else if (cfg.enabled) {
         await this.runReuseMode(mainUrl, feature, cfg);
       } else {
         await this.runFreshMode(mainUrl, feature, cfg.styles);
@@ -135,6 +146,20 @@ export class ExploreCommand extends BaseCommand {
     await this.discoverNewSubPages(mainPlan, mainUrl, styles, new Set());
   }
 
+  private async runSitemapMode(sitemapUrls: string[], feature: string | undefined, styles?: string[]): Promise<void> {
+    const urls = keepSiteUrls(sitemapUrls, this.explorBot.getConfig().playwright.url);
+    tag('info').log(`Sitemap: ${urls.length} page(s) to explore`);
+    for (const url of urls) {
+      if (this.isLimitReached()) break;
+      tag('info').log(`Exploring sitemap page: ${url}`);
+      try {
+        await this.explorePage(url, feature, undefined, styles);
+      } catch (err) {
+        tag('warning').log(`Sitemap page exploration failed: ${err instanceof Error ? err.message : err}`);
+      }
+    }
+  }
+
   private async runReuseMode(mainUrl: string | undefined, feature: string | undefined, cfg: ConfigureSpec): Promise<void> {
     const filename = cfg.fromPath || this.explorBot.generatePlanFilename(feature);
 
@@ -234,6 +259,7 @@ export class ExploreCommand extends BaseCommand {
       for (const subPlan of subPlans) {
         if (this.isLimitReached()) break;
         if (!subPlan.url) continue;
+        if (!this.urlFilter.admit(subPlan.url)) continue;
         try {
           if (!this.dryRun) await this.explorBot.visit(subPlan.url);
           await this.replanAndRun(subPlan.url, undefined, subPlan, cfg.styles);
@@ -264,27 +290,21 @@ export class ExploreCommand extends BaseCommand {
 
       const candidates = planner.collectSubPageCandidates(mainPlan, mainUrl || '/').filter((c) => {
         const norm = normalizeUrl(c.url);
-        return !this.failedSubPages.has(norm) && !knownUrls.has(norm);
+        return !this.failedSubPages.has(norm) && !knownUrls.has(norm) && this.urlFilter.allows(c.url);
       });
       if (candidates.length === 0) break;
 
       const pick = await planner.pickNextSubPage(candidates);
       if (!pick) break;
+      if (!this.urlFilter.admit(pick.url)) break;
 
       tag('info').log(`Exploring sub-page: ${pick.url} (${pick.reason})`);
       try {
-        await this.explorBot.visit(pick.url);
-        const errorPage = getStateErrorPageError(this.explorBot.stateManager().getCurrentState());
-        if (errorPage) {
-          tag('warning').log(`Skipping sub-page: ${errorPage.message}`);
+        const explored = await this.explorePage(pick.url, undefined, mainPlan, styles);
+        if (!explored) {
           this.failedSubPages.add(normalizeUrl(pick.url));
           continue;
         }
-        await this.runAllStyles(pick.url, undefined, mainPlan, this.completedPlans, styles);
-        const subPlan = this.explorBot.getCurrentPlan();
-        if (subPlan?.tests.length && !this.completedPlans.includes(subPlan)) {
-          this.completedPlans.push(subPlan);
-        }
         knownUrls.add(normalizeUrl(pick.url));
       } catch (err) {
         this.failedSubPages.add(normalizeUrl(pick.url));
@@ -293,6 +313,21 @@ export class ExploreCommand extends BaseCommand {
     }
   }
 
+  private async explorePage(url: string, feature: string | undefined, parentPlan: Plan | undefined, styles?: string[]): Promise<boolean> {
+    await this.explorBot.visit(url);
+    const errorPage = getStateErrorPageError(this.explorBot.stateManager().getCurrentState());
+    if (errorPage) {
+      tag('warning').log(`Skipping page: ${errorPage.message}`);
+      return false;
+    }
+    await this.runAllStyles(url, feature, parentPlan, this.completedPlans, styles);
+    const plan = this.explorBot.getCurrentPlan();
+    if (plan?.tests.length && !this.completedPlans.includes(plan)) {
+      this.completedPlans.push(plan);
+    }
+    return true;
+  }
+
   private async replanAndRun(pageUrl: string | undefined, feature: string | undefined, existingPlan: Plan, styles?: string[]): Promise<void> {
     const styleList = styles ?? Object.keys(getStyles());
     for (const style of styleList) {
@@ -369,17 +404,7 @@ export class ExploreCommand extends BaseCommand {
     const validSubpages = new Set(['none', 'same', 'new', 'both']);
     let hasReuseSignal = false;
 
-    for (const pair of raw.split(';')) {
-      const trimmed = pair.trim();
-      if (!trimmed) continue;
-      const sepMatch = trimmed.match(/^([^:=]+)\s*[:=]\s*(.*)$/);
-      if (!sepMatch) {
-        tag('warning').log(`Ignoring malformed configure pair: ${trimmed}`);
-        continue;
-      }
-      const key = sepMatch[1].trim().toLowerCase();
-      const value = sepMatch[2].trim();
-
+    for (const { key, value } of parseSpecPairs(raw)) {
       if (key === 'new') {
         const ratio = parseRatio(value);
         if (ratio == null) {

package/src/commands/freesail-command.ts

@@ -4,6 +4,7 @@ import { normalizeUrl } from '../state-manager.js';
 import { Stats } from '../stats.js';
 import { tag } from '../utils/logger.js';
 import { loop } from '../utils/loop.js';
+import { UrlFilter } from '../utils/url-filter.ts';
 import { BaseCommand } from './base-command.js';
 import { DEADLINE_RESERVE_MS, DEADLINE_TEST_ALLOWANCE_MS, ExploreCommand } from './explore-command.js';
 
@@ -18,6 +19,7 @@ export class FreesailCommand extends BaseCommand {
     { flags: '--scope <url>', description: 'Limit navigation to URLs starting with this prefix' },
     { flags: '--max-tests <number>', description: 'Maximum number of tests to run' },
     { flags: '--max-duration <number>', description: 'Wall-clock budget in minutes for the whole run' },
+    { flags: '--url-filter <spec>', description: 'Pick pages to explore: keys filter|exclude|limit, e.g. "filter:/admin/*;exclude:/admin/logs/*;limit:20"' },
   ];
 
   async execute(args: string): Promise<void> {
@@ -27,6 +29,7 @@ export class FreesailCommand extends BaseCommand {
     if (opts.deep) strategy = 'deep';
     if (opts.shallow) strategy = 'shallow';
     const scope = opts.scope as string | undefined;
+    const urlFilter = new UrlFilter(opts.urlFilter as string | undefined);
     const maxTests = opts.maxTests ? Number.parseInt(opts.maxTests as string, 10) : undefined;
     const maxDuration = opts.maxDuration ? Number.parseInt(opts.maxDuration as string, 10) : undefined;
     let hardDeadlineAt: number | undefined;
@@ -55,6 +58,7 @@ export class FreesailCommand extends BaseCommand {
           const exploreCmd = new ExploreCommand(this.explorBot);
           if (maxTests != null) exploreCmd.maxTests = maxTests - testsRun;
           if (hardDeadlineAt != null) exploreCmd.hardDeadlineAt = hardDeadlineAt;
+          exploreCmd.urlFilter = urlFilter;
           await exploreCmd.execute('');
 
           const plan = this.explorBot.getCurrentPlan();
@@ -65,7 +69,7 @@ export class FreesailCommand extends BaseCommand {
 
         const navigator = this.explorBot.agentNavigator();
         const visitedUrls = stateManager.getAllVisitedUrls();
-        const suggestion = await navigator.freeSail({ strategy, scope, visitedUrls });
+        const suggestion = await navigator.freeSail({ strategy, scope, visitedUrls, urlFilter });
         if (!suggestion) {
           tag('info').log('No navigation suggestion available');
           ctx.stop();
@@ -76,6 +80,11 @@ export class FreesailCommand extends BaseCommand {
           ctx.stop();
         }
 
+        if (!urlFilter.admit(suggestion.target)) {
+          tag('info').log(`URL filter allows no more pages (${suggestion.target} not admitted); stopping exploration`);
+          ctx.stop();
+        }
+
         const currentUrl = stateManager.getCurrentState()?.url;
         if (currentUrl && normalizeUrl(suggestion.target) === normalizeUrl(currentUrl)) {
           tag('info').log(`No new navigation target available after ${currentUrl}; stopping exploration`);

package/src/test-plan.ts

@@ -331,10 +331,12 @@ export class Test extends Task {
     });
   }
 
+  hasAchieved(expectation: string): boolean {
+    return Object.values(this.notes).some((note) => note.message === expectation && note.status === TestResult.PASSED);
+  }
+
   hasAchievedAny(): boolean {
-    return this.expected.some((expectation) => {
-      return Object.values(this.notes).some((note) => note.message === expectation && note.status === TestResult.PASSED);
-    });
+    return this.expected.some((expectation) => this.hasAchieved(expectation));
   }
 
   getRunResult(): 'success' | 'partial' | 'failed' {
@@ -344,9 +346,7 @@ export class Test extends Task {
   }
 
   hasAchievedAll(): boolean {
-    return this.expected.every((expectation) => {
-      return Object.values(this.notes).some((note) => note.message === expectation && note.status === TestResult.PASSED);
-    });
+    return this.expected.every((expectation) => this.hasAchieved(expectation));
   }
 
   isComplete(): boolean {

package/src/utils/sitemap.ts

@@ -0,0 +1,108 @@
+import { readFileSync } from 'node:fs';
+import { tag } from './logger.js';
+import type { UrlFilter } from './url-filter.ts';
+
+const ENTRY_PATTERN = /<(url|sitemap)\b[^>]*>(.*?)<\/\1>/gis;
+const XML_ENTITIES: Record<string, string> = { '&amp;': '&', '&lt;': '<', '&gt;': '>', '&quot;': '"', '&apos;': "'" };
+const MAX_NESTED_SITEMAPS = 50;
+const DEFAULT_PRIORITY = 0.5;
+const STDIN_SOURCE = '-';
+
+export async function readSitemap(source: string, filter?: UrlFilter): Promise<string[]> {
+  const entries: SitemapEntry[] = [];
+  const seen = new Set<string>();
+  const pending = [source];
+  let fetched = 0;
+
+  while (pending.length > 0) {
+    if (fetched >= MAX_NESTED_SITEMAPS) {
+      tag('warning').log(`Sitemap index lists more than ${MAX_NESTED_SITEMAPS} sitemaps, the rest are ignored`);
+      break;
+    }
+    const next = pending.shift()!;
+    fetched++;
+    const content = await readSource(next);
+    const isIndex = content.includes('<sitemapindex');
+    for (const entry of extractEntries(content)) {
+      if (seen.has(entry.url)) continue;
+      seen.add(entry.url);
+      if (isIndex) {
+        pending.push(entry.url);
+        continue;
+      }
+      entries.push(entry);
+    }
+  }
+
+  return applyFilter(entries, filter).map((entry) => entry.url);
+}
+
+export async function readCliSitemap(path: string | undefined, source: string | undefined, filter?: UrlFilter): Promise<string[] | undefined> {
+  const usage = 'Pass a page path, --sitemap <source>, or pipe a sitemap to stdin';
+  let resolved = source;
+  if (!resolved && !path && !process.stdin.isTTY) resolved = STDIN_SOURCE;
+  if (!resolved && !path) throw new Error(usage);
+  if (!resolved) return undefined;
+  const urls = await readSitemap(resolved, filter);
+  if (urls.length === 0 && !source) throw new Error(usage);
+  if (urls.length === 0) throw new Error(`No URLs left in sitemap ${resolved}`);
+  return urls;
+}
+
+export function keepSiteUrls(urls: string[], siteUrl: string): string[] {
+  const siteOrigin = URL.parse(siteUrl)?.origin;
+  const kept = urls.filter((url) => url.startsWith('/') || URL.parse(url)?.origin === siteOrigin);
+  if (kept.length < urls.length) tag('warning').log(`Skipped ${urls.length - kept.length} sitemap URL(s) outside ${siteOrigin}`);
+  return kept;
+}
+
+export function extractEntries(content: string): SitemapEntry[] {
+  if (!content.trimStart().startsWith('<')) {
+    return content
+      .split('\n')
+      .map((line) => line.trim())
+      .filter((line) => line.startsWith('http://') || line.startsWith('https://') || line.startsWith('/'))
+      .map((url) => ({ url, priority: DEFAULT_PRIORITY }));
+  }
+  const entries: SitemapEntry[] = [];
+  for (const match of content.matchAll(ENTRY_PATTERN)) {
+    const url = tagValue(match[2], 'loc');
+    if (!url) continue;
+    const entry: SitemapEntry = { url, priority: DEFAULT_PRIORITY };
+    const priority = Number.parseFloat(tagValue(match[2], 'priority') || '');
+    if (!Number.isNaN(priority)) entry.priority = priority;
+    const lastmod = Date.parse(tagValue(match[2], 'lastmod') || '');
+    if (!Number.isNaN(lastmod)) entry.lastmod = lastmod;
+    entries.push(entry);
+  }
+  return entries;
+}
+
+function applyFilter(entries: SitemapEntry[], filter: UrlFilter | undefined): SitemapEntry[] {
+  if (!filter) return entries;
+  let result = entries.filter((entry) => filter.allows(entry.url));
+  if (filter.sort === 'priority') result = result.toSorted((a, b) => b.priority - a.priority);
+  if (filter.sort === 'lastmod') result = result.toSorted((a, b) => (b.lastmod ?? 0) - (a.lastmod ?? 0));
+  return result.slice(0, filter.limit);
+}
+
+function tagValue(block: string, name: string): string | undefined {
+  const match = block.match(new RegExp(`<${name}>\\s*(?:<!\\[CDATA\\[)?\\s*(.*?)\\s*(?:\\]\\]>)?\\s*</${name}>`, 'is'));
+  return match?.[1].replace(/&(amp|lt|gt|quot|apos);/g, (entity) => XML_ENTITIES[entity]) || undefined;
+}
+
+async function readSource(source: string): Promise<string> {
+  if (source === STDIN_SOURCE) return readFileSync(0, 'utf8');
+  if (source.startsWith('http://') || source.startsWith('https://')) {
+    const response = await fetch(source);
+    if (!response.ok) throw new Error(`Sitemap ${source} returned HTTP ${response.status}`);
+    return await response.text();
+  }
+  return readFileSync(source, 'utf8');
+}
+
+interface SitemapEntry {
+  url: string;
+  priority: number;
+  lastmod?: number;
+}