explorbot 0.5.5 → 0.5.7

package/src/commands/explore-command.ts

@@ -9,7 +9,10 @@ import { getCliName } from '../utils/cli-name.ts';
 import { ErrorPageError, getStateErrorPageError } from '../utils/error-page.ts';
 import { tag } from '../utils/logger.js';
 import { type NextStepSection, printNextSteps, relativeToCwd } from '../utils/next-steps.ts';
+import { keepSiteUrls, readSitemap } from '../utils/sitemap.ts';
+import { parseSpecPairs } from '../utils/spec.ts';
 import { safeFilename } from '../utils/strings.ts';
+import { UrlFilter } from '../utils/url-filter.ts';
 import { type ArgumentCompletion, BaseCommand, type Suggestion } from './base-command.js';
 
 const MAX_SUB_PAGE_ATTEMPTS = 30;
@@ -26,6 +29,8 @@ export class ExploreCommand extends BaseCommand {
     { flags: '--focus <feature>', description: 'Focus area for exploration' },
     { flags: '--configure <spec>', description: 'Reuse spec: keys new|from|style|subpages|pick_by|priority, e.g. "new:25%;pick_by=random;priority=critical,high"' },
     { flags: '--dry-run', description: 'Mark picked tests as skipped without executing or generating new ones' },
+    { flags: '--sitemap <source>', description: 'Explore every page listed in a sitemap (file path, URL, or - for stdin) instead of discovering sub-pages' },
+    { flags: '--url-filter <spec>', description: 'Pick pages to explore: keys filter|exclude|limit, plus sort:priority|lastmod for a sitemap, e.g. "filter:/admin/*;exclude:/admin/logs/*;limit:20"' },
   ];
   suggestions: Suggestion[] = [
     { command: 'navigate <page>', hint: 'go to another page' },
@@ -37,6 +42,8 @@ export class ExploreCommand extends BaseCommand {
   maxDurationMinutes?: number;
   hardDeadlineAt?: number;
   dryRun = false;
+  sitemapUrls?: string[];
+  urlFilter = new UrlFilter();
   private testsRun = 0;
   private deadlineLogged = false;
   private completedPlans: Plan[] = [];
@@ -57,6 +64,8 @@ export class ExploreCommand extends BaseCommand {
     if (opts.maxDuration) {
       this.maxDurationMinutes = Number.parseInt(opts.maxDuration as string, 10);
     }
+    if (opts.urlFilter) this.urlFilter = new UrlFilter(opts.urlFilter as string);
+    if (opts.sitemap) this.sitemapUrls = await readSitemap(opts.sitemap as string, this.urlFilter);
     if (this.hardDeadlineAt == null && this.maxDurationMinutes != null) {
       this.hardDeadlineAt = Date.now() + this.maxDurationMinutes * 60_000 - DEADLINE_RESERVE_MS;
     }
@@ -76,13 +85,15 @@ export class ExploreCommand extends BaseCommand {
     Stats.focus ??= feature;
     const mainUrl = this.getCurrentPageUrl();
     const error = getStateErrorPageError(this.explorBot.stateManager().getCurrentState());
-    if (error) {
+    if (error && !this.sitemapUrls) {
       tag('warning').log(error.message);
       return;
     }
 
     try {
-      if (cfg.enabled) {
+      if (this.sitemapUrls) {
+        await this.runSitemapMode(this.sitemapUrls, feature, cfg.styles);
+      } else if (cfg.enabled) {
         await this.runReuseMode(mainUrl, feature, cfg);
       } else {
         await this.runFreshMode(mainUrl, feature, cfg.styles);
@@ -135,6 +146,20 @@ export class ExploreCommand extends BaseCommand {
     await this.discoverNewSubPages(mainPlan, mainUrl, styles, new Set());
   }
 
+  private async runSitemapMode(sitemapUrls: string[], feature: string | undefined, styles?: string[]): Promise<void> {
+    const urls = keepSiteUrls(sitemapUrls, this.explorBot.getConfig().playwright.url);
+    tag('info').log(`Sitemap: ${urls.length} page(s) to explore`);
+    for (const url of urls) {
+      if (this.isLimitReached()) break;
+      tag('info').log(`Exploring sitemap page: ${url}`);
+      try {
+        await this.explorePage(url, feature, undefined, styles);
+      } catch (err) {
+        tag('warning').log(`Sitemap page exploration failed: ${err instanceof Error ? err.message : err}`);
+      }
+    }
+  }
+
   private async runReuseMode(mainUrl: string | undefined, feature: string | undefined, cfg: ConfigureSpec): Promise<void> {
     const filename = cfg.fromPath || this.explorBot.generatePlanFilename(feature);
 
@@ -234,6 +259,7 @@ export class ExploreCommand extends BaseCommand {
       for (const subPlan of subPlans) {
         if (this.isLimitReached()) break;
         if (!subPlan.url) continue;
+        if (!this.urlFilter.admit(subPlan.url)) continue;
         try {
           if (!this.dryRun) await this.explorBot.visit(subPlan.url);
           await this.replanAndRun(subPlan.url, undefined, subPlan, cfg.styles);
@@ -264,27 +290,21 @@ export class ExploreCommand extends BaseCommand {
 
       const candidates = planner.collectSubPageCandidates(mainPlan, mainUrl || '/').filter((c) => {
         const norm = normalizeUrl(c.url);
-        return !this.failedSubPages.has(norm) && !knownUrls.has(norm);
+        return !this.failedSubPages.has(norm) && !knownUrls.has(norm) && this.urlFilter.allows(c.url);
       });
       if (candidates.length === 0) break;
 
       const pick = await planner.pickNextSubPage(candidates);
       if (!pick) break;
+      if (!this.urlFilter.admit(pick.url)) break;
 
       tag('info').log(`Exploring sub-page: ${pick.url} (${pick.reason})`);
       try {
-        await this.explorBot.visit(pick.url);
-        const errorPage = getStateErrorPageError(this.explorBot.stateManager().getCurrentState());
-        if (errorPage) {
-          tag('warning').log(`Skipping sub-page: ${errorPage.message}`);
+        const explored = await this.explorePage(pick.url, undefined, mainPlan, styles);
+        if (!explored) {
           this.failedSubPages.add(normalizeUrl(pick.url));
           continue;
         }
-        await this.runAllStyles(pick.url, undefined, mainPlan, this.completedPlans, styles);
-        const subPlan = this.explorBot.getCurrentPlan();
-        if (subPlan?.tests.length && !this.completedPlans.includes(subPlan)) {
-          this.completedPlans.push(subPlan);
-        }
         knownUrls.add(normalizeUrl(pick.url));
       } catch (err) {
         this.failedSubPages.add(normalizeUrl(pick.url));
@@ -293,6 +313,21 @@ export class ExploreCommand extends BaseCommand {
     }
   }
 
+  private async explorePage(url: string, feature: string | undefined, parentPlan: Plan | undefined, styles?: string[]): Promise<boolean> {
+    await this.explorBot.visit(url);
+    const errorPage = getStateErrorPageError(this.explorBot.stateManager().getCurrentState());
+    if (errorPage) {
+      tag('warning').log(`Skipping page: ${errorPage.message}`);
+      return false;
+    }
+    await this.runAllStyles(url, feature, parentPlan, this.completedPlans, styles);
+    const plan = this.explorBot.getCurrentPlan();
+    if (plan?.tests.length && !this.completedPlans.includes(plan)) {
+      this.completedPlans.push(plan);
+    }
+    return true;
+  }
+
   private async replanAndRun(pageUrl: string | undefined, feature: string | undefined, existingPlan: Plan, styles?: string[]): Promise<void> {
     const styleList = styles ?? Object.keys(getStyles());
     for (const style of styleList) {
@@ -369,17 +404,7 @@ export class ExploreCommand extends BaseCommand {
     const validSubpages = new Set(['none', 'same', 'new', 'both']);
     let hasReuseSignal = false;
 
-    for (const pair of raw.split(';')) {
-      const trimmed = pair.trim();
-      if (!trimmed) continue;
-      const sepMatch = trimmed.match(/^([^:=]+)\s*[:=]\s*(.*)$/);
-      if (!sepMatch) {
-        tag('warning').log(`Ignoring malformed configure pair: ${trimmed}`);
-        continue;
-      }
-      const key = sepMatch[1].trim().toLowerCase();
-      const value = sepMatch[2].trim();
-
+    for (const { key, value } of parseSpecPairs(raw)) {
       if (key === 'new') {
         const ratio = parseRatio(value);
         if (ratio == null) {

package/src/commands/freesail-command.ts

@@ -4,6 +4,7 @@ import { normalizeUrl } from '../state-manager.js';
 import { Stats } from '../stats.js';
 import { tag } from '../utils/logger.js';
 import { loop } from '../utils/loop.js';
+import { UrlFilter } from '../utils/url-filter.ts';
 import { BaseCommand } from './base-command.js';
 import { DEADLINE_RESERVE_MS, DEADLINE_TEST_ALLOWANCE_MS, ExploreCommand } from './explore-command.js';
 
@@ -18,6 +19,7 @@ export class FreesailCommand extends BaseCommand {
     { flags: '--scope <url>', description: 'Limit navigation to URLs starting with this prefix' },
     { flags: '--max-tests <number>', description: 'Maximum number of tests to run' },
     { flags: '--max-duration <number>', description: 'Wall-clock budget in minutes for the whole run' },
+    { flags: '--url-filter <spec>', description: 'Pick pages to explore: keys filter|exclude|limit, e.g. "filter:/admin/*;exclude:/admin/logs/*;limit:20"' },
   ];
 
   async execute(args: string): Promise<void> {
@@ -27,6 +29,7 @@ export class FreesailCommand extends BaseCommand {
     if (opts.deep) strategy = 'deep';
     if (opts.shallow) strategy = 'shallow';
     const scope = opts.scope as string | undefined;
+    const urlFilter = new UrlFilter(opts.urlFilter as string | undefined);
     const maxTests = opts.maxTests ? Number.parseInt(opts.maxTests as string, 10) : undefined;
     const maxDuration = opts.maxDuration ? Number.parseInt(opts.maxDuration as string, 10) : undefined;
     let hardDeadlineAt: number | undefined;
@@ -55,6 +58,7 @@ export class FreesailCommand extends BaseCommand {
           const exploreCmd = new ExploreCommand(this.explorBot);
           if (maxTests != null) exploreCmd.maxTests = maxTests - testsRun;
           if (hardDeadlineAt != null) exploreCmd.hardDeadlineAt = hardDeadlineAt;
+          exploreCmd.urlFilter = urlFilter;
           await exploreCmd.execute('');
 
           const plan = this.explorBot.getCurrentPlan();
@@ -65,7 +69,7 @@ export class FreesailCommand extends BaseCommand {
 
         const navigator = this.explorBot.agentNavigator();
         const visitedUrls = stateManager.getAllVisitedUrls();
-        const suggestion = await navigator.freeSail({ strategy, scope, visitedUrls });
+        const suggestion = await navigator.freeSail({ strategy, scope, visitedUrls, urlFilter });
         if (!suggestion) {
           tag('info').log('No navigation suggestion available');
           ctx.stop();
@@ -76,6 +80,11 @@ export class FreesailCommand extends BaseCommand {
           ctx.stop();
         }
 
+        if (!urlFilter.admit(suggestion.target)) {
+          tag('info').log(`URL filter allows no more pages (${suggestion.target} not admitted); stopping exploration`);
+          ctx.stop();
+        }
+
         const currentUrl = stateManager.getCurrentState()?.url;
         if (currentUrl && normalizeUrl(suggestion.target) === normalizeUrl(currentUrl)) {
           tag('info').log(`No new navigation target available after ${currentUrl}; stopping exploration`);

package/src/commands/init-command.ts

@@ -260,7 +260,9 @@ export function modelLines(provider: string, only?: ModelRole[]): string {
   let selected = roles;
   if (only) selected = roles.filter(([role]) => only.includes(role));
 
-  return selected.map(([role, comment]) => `    // ${comment}\n    ${role}: '${provider}/${recommended[role] || '<model-id>'}',`).join('\n');
+  const lines = selected.map(([role, comment]) => `    // ${comment}\n    ${role}: '${provider}/${recommended[role] || '<model-id>'}',`);
+  if (!only && recommended.decisionModel) lines.push(`    // decision model judging ambiguous outcomes\n    decisionModel: { provider: '${provider}', model: '${recommended.decisionModel}' },`);
+  return lines.join('\n');
 }
 
 function globalConfigTemplate(provider: string): string {

package/src/commands/recommended-models-command.ts

@@ -52,6 +52,7 @@ export class RecommendedModelsCommand extends BaseCommand {
         if (roles[role]) models.push([label, roles[role]]);
         if (!roles[role]) models.push([label, chalk.dim('not served, pair with another provider')]);
       }
+      if (roles.decisionModel) models.push(['decisionModel (judge)', roles.decisionModel]);
       lines.push(...renderSection('models', models, 2));
 
       const envKey = PROVIDERS[provider]?.envKey;
@@ -70,6 +71,7 @@ export class RecommendedModelsCommand extends BaseCommand {
         env.push([label, `${ROLES[role].env}=${provider}/${roles[role]}`]);
         label = '';
       }
+      if (roles.decisionModel) env.push(['decision', `EXPLORBOT_DECISION_MODEL=${provider}/${roles.decisionModel}`]);
       lines.push(...renderSection('env variables', env, 2));
     }
 

package/src/config.ts

@@ -420,6 +420,11 @@ export class ConfigParser {
         log(`Configuration built from EXPLORBOT_* environment variables. Output: ${outputRoot}`);
       }
 
+      const configuredUrl = loadedConfig?.playwright?.url || loadedConfig?.web?.url || '';
+      if (!this.runBase && resolvedPath && !isGlobalConfigPath(resolvedPath) && URL.canParse(configuredUrl)) {
+        this.runBase = parseRunBase(configuredUrl);
+      }
+
       let config = this.resolveConfig(loadedConfig as ExplorbotConfig);
       await resolveConfigModels(config.ai);
       this.site = null;
@@ -675,6 +680,8 @@ export class ConfigParser {
     if (agenticSpec) ai.agenticModel = await resolveModel(agenticSpec, 'agenticModel');
     if (!agenticSpec && recommended.agenticModel) ai.agenticModel = await resolveModel(provider!, 'agenticModel');
 
+    if (recommended.decisionModel) ai.decisionModel = { provider: provider!, model: recommended.decisionModel };
+
     const dirs = { knowledge: 'knowledge', experience: 'experience', output: 'output' };
     if (process.env.EXPLORBOT_OUTPUT) dirs.output = '.';
 

package/src/test-plan.ts

@@ -331,10 +331,12 @@ export class Test extends Task {
     });
   }
 
+  hasAchieved(expectation: string): boolean {
+    return Object.values(this.notes).some((note) => note.message === expectation && note.status === TestResult.PASSED);
+  }
+
   hasAchievedAny(): boolean {
-    return this.expected.some((expectation) => {
-      return Object.values(this.notes).some((note) => note.message === expectation && note.status === TestResult.PASSED);
-    });
+    return this.expected.some((expectation) => this.hasAchieved(expectation));
   }
 
   getRunResult(): 'success' | 'partial' | 'failed' {
@@ -344,9 +346,7 @@ export class Test extends Task {
   }
 
   hasAchievedAll(): boolean {
-    return this.expected.every((expectation) => {
-      return Object.values(this.notes).some((note) => note.message === expectation && note.status === TestResult.PASSED);
-    });
+    return this.expected.every((expectation) => this.hasAchieved(expectation));
   }
 
   isComplete(): boolean {

package/src/utils/sitemap.ts

@@ -0,0 +1,108 @@
+import { readFileSync } from 'node:fs';
+import { tag } from './logger.js';
+import type { UrlFilter } from './url-filter.ts';
+
+const ENTRY_PATTERN = /<(url|sitemap)\b[^>]*>(.*?)<\/\1>/gis;
+const XML_ENTITIES: Record<string, string> = { '&amp;': '&', '&lt;': '<', '&gt;': '>', '&quot;': '"', '&apos;': "'" };
+const MAX_NESTED_SITEMAPS = 50;
+const DEFAULT_PRIORITY = 0.5;
+const STDIN_SOURCE = '-';
+
+export async function readSitemap(source: string, filter?: UrlFilter): Promise<string[]> {
+  const entries: SitemapEntry[] = [];
+  const seen = new Set<string>();
+  const pending = [source];
+  let fetched = 0;
+
+  while (pending.length > 0) {
+    if (fetched >= MAX_NESTED_SITEMAPS) {
+      tag('warning').log(`Sitemap index lists more than ${MAX_NESTED_SITEMAPS} sitemaps, the rest are ignored`);
+      break;
+    }
+    const next = pending.shift()!;
+    fetched++;
+    const content = await readSource(next);
+    const isIndex = content.includes('<sitemapindex');
+    for (const entry of extractEntries(content)) {
+      if (seen.has(entry.url)) continue;
+      seen.add(entry.url);
+      if (isIndex) {
+        pending.push(entry.url);
+        continue;
+      }
+      entries.push(entry);
+    }
+  }
+
+  return applyFilter(entries, filter).map((entry) => entry.url);
+}
+
+export async function readCliSitemap(path: string | undefined, source: string | undefined, filter?: UrlFilter): Promise<string[] | undefined> {
+  const usage = 'Pass a page path, --sitemap <source>, or pipe a sitemap to stdin';
+  let resolved = source;
+  if (!resolved && !path && !process.stdin.isTTY) resolved = STDIN_SOURCE;
+  if (!resolved && !path) throw new Error(usage);
+  if (!resolved) return undefined;
+  const urls = await readSitemap(resolved, filter);
+  if (urls.length === 0 && !source) throw new Error(usage);
+  if (urls.length === 0) throw new Error(`No URLs left in sitemap ${resolved}`);
+  return urls;
+}
+
+export function keepSiteUrls(urls: string[], siteUrl: string): string[] {
+  const siteOrigin = URL.parse(siteUrl)?.origin;
+  const kept = urls.filter((url) => url.startsWith('/') || URL.parse(url)?.origin === siteOrigin);
+  if (kept.length < urls.length) tag('warning').log(`Skipped ${urls.length - kept.length} sitemap URL(s) outside ${siteOrigin}`);
+  return kept;
+}
+
+export function extractEntries(content: string): SitemapEntry[] {
+  if (!content.trimStart().startsWith('<')) {
+    return content
+      .split('\n')
+      .map((line) => line.trim())
+      .filter((line) => line.startsWith('http://') || line.startsWith('https://') || line.startsWith('/'))
+      .map((url) => ({ url, priority: DEFAULT_PRIORITY }));
+  }
+  const entries: SitemapEntry[] = [];
+  for (const match of content.matchAll(ENTRY_PATTERN)) {
+    const url = tagValue(match[2], 'loc');
+    if (!url) continue;
+    const entry: SitemapEntry = { url, priority: DEFAULT_PRIORITY };
+    const priority = Number.parseFloat(tagValue(match[2], 'priority') || '');
+    if (!Number.isNaN(priority)) entry.priority = priority;
+    const lastmod = Date.parse(tagValue(match[2], 'lastmod') || '');
+    if (!Number.isNaN(lastmod)) entry.lastmod = lastmod;
+    entries.push(entry);
+  }
+  return entries;
+}
+
+function applyFilter(entries: SitemapEntry[], filter: UrlFilter | undefined): SitemapEntry[] {
+  if (!filter) return entries;
+  let result = entries.filter((entry) => filter.allows(entry.url));
+  if (filter.sort === 'priority') result = result.toSorted((a, b) => b.priority - a.priority);
+  if (filter.sort === 'lastmod') result = result.toSorted((a, b) => (b.lastmod ?? 0) - (a.lastmod ?? 0));
+  return result.slice(0, filter.limit);
+}
+
+function tagValue(block: string, name: string): string | undefined {
+  const match = block.match(new RegExp(`<${name}>\\s*(?:<!\\[CDATA\\[)?\\s*(.*?)\\s*(?:\\]\\]>)?\\s*</${name}>`, 'is'));
+  return match?.[1].replace(/&(amp|lt|gt|quot|apos);/g, (entity) => XML_ENTITIES[entity]) || undefined;
+}
+
+async function readSource(source: string): Promise<string> {
+  if (source === STDIN_SOURCE) return readFileSync(0, 'utf8');
+  if (source.startsWith('http://') || source.startsWith('https://')) {
+    const response = await fetch(source);
+    if (!response.ok) throw new Error(`Sitemap ${source} returned HTTP ${response.status}`);
+    return await response.text();
+  }
+  return readFileSync(source, 'utf8');
+}
+
+interface SitemapEntry {
+  url: string;
+  priority: number;
+  lastmod?: number;
+}

package/src/utils/spec.ts

@@ -0,0 +1,21 @@
+import { tag } from './logger.js';
+
+export function parseSpecPairs(raw: string | undefined): SpecPair[] {
+  const pairs: SpecPair[] = [];
+  for (const pair of (raw || '').split(';')) {
+    const trimmed = pair.trim();
+    if (!trimmed) continue;
+    const sepMatch = trimmed.match(/^([^:=]+)\s*[:=]\s*(.*)$/);
+    if (!sepMatch) {
+      tag('warning').log(`Ignoring malformed spec pair: ${trimmed}`);
+      continue;
+    }
+    pairs.push({ key: sepMatch[1].trim().toLowerCase(), value: sepMatch[2].trim() });
+  }
+  return pairs;
+}
+
+export interface SpecPair {
+  key: string;
+  value: string;
+}

package/src/utils/url-filter.ts

@@ -0,0 +1,53 @@
+import { tag } from './logger.js';
+import { parseSpecPairs } from './spec.ts';
+import { matchesUrl } from './url-matcher.ts';
+
+export class UrlFilter {
+  readonly include: string[] = [];
+  readonly exclude: string[] = [];
+  sort?: 'priority' | 'lastmod';
+  limit = Number.POSITIVE_INFINITY;
+  private admitted = new Set<string>();
+
+  constructor(spec?: string) {
+    for (const { key, value } of parseSpecPairs(spec)) {
+      if (key === 'filter') {
+        this.include.push(value);
+        continue;
+      }
+      if (key === 'exclude') {
+        this.exclude.push(value);
+        continue;
+      }
+      if (key === 'sort' && (value === 'priority' || value === 'lastmod')) {
+        this.sort = value;
+        continue;
+      }
+      if (key === 'limit' && Number.parseInt(value, 10) > 0) {
+        this.limit = Number.parseInt(value, 10);
+        continue;
+      }
+      tag('warning').log(`Ignoring URL filter pair ${key}:${value} (use filter|exclude:<url pattern>, sort:priority|lastmod, limit:<n>)`);
+    }
+  }
+
+  allows(url: string): boolean {
+    const path = urlPath(url);
+    if (this.include.length > 0 && !this.include.some((pattern) => matchesUrl(pattern, path))) return false;
+    return !this.exclude.some((pattern) => matchesUrl(pattern, path));
+  }
+
+  admit(url: string): boolean {
+    if (!this.allows(url)) return false;
+    const path = urlPath(url);
+    if (this.admitted.has(path)) return true;
+    if (this.admitted.size >= this.limit) return false;
+    this.admitted.add(path);
+    return true;
+  }
+}
+
+function urlPath(url: string): string {
+  const parsed = new URL(url, 'http://localhost');
+  return `${parsed.pathname}${parsed.search}`;
+}