explorbot 0.5.6 → 0.5.8

package/src/commands/init-command.ts

@@ -1,25 +1,30 @@
 import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
 import { dirname, extname, join, resolve } from 'node:path';
 import chalk from 'chalk';
+import figureSet from 'figures';
 import { ConfigParser, type ModelRole, PROVIDERS, missingModelRoles } from '../config.ts';
 import { findGlobalConfig, globalConfigPath, globalDir, globalEnvPath } from '../global-config.ts';
 import { getCliName } from '../utils/cli-name.ts';
 import { log, tag } from '../utils/logger.js';
-import { relativeToCwd } from '../utils/next-steps.ts';
+import { type NextStepCommand, type NextStepSection, printNextSteps, relativeToCwd } from '../utils/next-steps.ts';
 
-function defaultConfigTemplate(provider: string): string {
+function defaultConfigTemplate(provider: string, url?: string): string {
+  const appUrl = url || 'http://<your-app-host-here>';
   return `// 'provider/model-id' uses a bundled provider.
 // It is also possible to import provider as a module from Vercel AI SDK.
-// https://github.com/testomatio/explorbot/blob/main/docs/basics/providers.md
+// https://testomat.ai/docs/explorbot
 
 const config = {
   web: {
     // use application host without path prefix (e.g., http://localhost:3000)
-    url: 'http://<your-app-host-here>',
+    url: '${appUrl}',
   },
 
   ai: {
-${modelLines(provider)}
+    // every role (model, visionModel, agenticModel) takes the recommended ${provider} models bundled with this version
+    // override a single role as 'provider/model-id':
+    // model: '${provider}/<model-id>',
+    recommendedModels: '${provider}',
   },
 
   reporter: {
@@ -37,8 +42,9 @@ export default config;
 }
 
 export function envTemplate(provider: string): string {
+  const { keyUrl } = PROVIDERS[provider];
   const keyLines = Object.entries(PROVIDERS).map(([name, { envKey }]) => {
-    if (name === provider) return `${envKey}=`;
+    if (name === provider) return `# Create a key at ${keyUrl} and paste it after '='\n${envKey}=`;
     return `# ${envKey}=`;
   });
 
@@ -69,10 +75,10 @@ export async function runInit(options: InitCommandOptions): Promise<void> {
   const choice = await renderInitWizard('choose');
   if (choice !== 'local') return;
 
-  const provider = await renderLocalProviderWizard();
-  if (!provider) return;
+  const setup = await renderLocalSetupWizard();
+  if (!setup) return;
 
-  runInitCommand({ ...options, provider });
+  runInitCommand({ ...options, ...setup });
 }
 
 export function writeGlobalConfig(provider: string, apiKey?: string): void {
@@ -82,24 +88,28 @@ export function writeGlobalConfig(provider: string, apiKey?: string): void {
 
   mkdirSync(globalDir(), { recursive: true });
   writeFileSync(globalConfigPath(), globalConfigTemplate(provider), 'utf8');
-  log(`Created global config: ${globalConfigPath()}`);
+  tag('success').log(`${figureSet.tick} Created global config: ${globalConfigPath()}`);
 
   const envKey = PROVIDERS[provider].envKey;
   writeEnvKey(envKey, apiKey || '');
-  log(`Stored ${envKey} in ${globalEnvPath()}`);
+  tag('success').log(`${figureSet.tick} Stored ${envKey} in ${globalEnvPath()}`);
 
   const missing = missingModelRoles(provider);
   if (missing.length) {
     tag('warning').log(`No recommended ${missing.join(' and ')} for ${provider} — set the model ids in ${globalConfigPath()}`);
   }
   if (!apiKey && !process.env[envKey]) {
-    tag('warning').log(`Add your API key to ${globalEnvPath()}`);
+    printApiKeySteps(provider, globalEnvPath());
   }
 
   log('');
-  log('Explorbot now runs from any directory:');
-  tag('substep').log(chalk.yellow(`${getCliName()} explore https://your-app.example.com`));
-  tag('substep').log(chalk.yellow(`${getCliName()} sites`));
+  log(`${figureSet.star} What's next:`);
+  printInitNextSteps({
+    siteCommands: [
+      { label: 'Any site', command: `${getCliName()} explore https://your-app.example.com` },
+      { label: 'Saved sites', command: `${getCliName()} sites` },
+    ],
+  });
 }
 
 export function runInitCommand(options: InitCommandOptions): void {
@@ -141,38 +151,42 @@ export function runInitCommand(options: InitCommandOptions): void {
       process.exit(1);
     }
 
-    writeFileSync(outPath, defaultConfigTemplate(provider), 'utf8');
-    log(`Created config file: ${relativeToCwd(outPath)}`);
+    writeFileSync(outPath, defaultConfigTemplate(provider, options.url), 'utf8');
+    tag('success').log(`${figureSet.tick} Created config file: ${relativeToCwd(outPath)}`);
 
     const envPath = resolve(process.cwd(), '.env');
     if (!existsSync(envPath)) {
       writeFileSync(envPath, `${envTemplate(provider)}\n`, 'utf8');
-      log(`Created env file: ${relativeToCwd(envPath)}`);
+      tag('success').log(`${figureSet.tick} Created env file: ${relativeToCwd(envPath)}`);
     } else {
       log(`Env file already exists: ${relativeToCwd(envPath)}`);
     }
 
+    const envKey = PROVIDERS[provider].envKey;
+    if (options.apiKey) {
+      writeEnvKey(envKey, options.apiKey, envPath);
+      tag('success').log(`${figureSet.tick} Stored ${envKey} in ${relativeToCwd(envPath)}`);
+    }
+    if (!options.apiKey && !process.env[envKey]) {
+      printApiKeySteps(provider, relativeToCwd(envPath));
+    }
+    if (!options.url) {
+      tag('warning').log(`Set your application URL in web.url: ${relativeToCwd(outPath)}`);
+    }
+
     const missing = missingModelRoles(provider);
     if (missing.length) {
       tag('warning').log(`No recommended ${missing.join(' and ')} for ${provider} — set the model ids in ${relativeToCwd(outPath)}`);
     }
 
-    log('');
-    log('Next steps:');
-    log('1. Configure AI provider in .env');
-    log('2. Set AI models config file');
-    log('3. Set web application URL in the config file');
-    log('4. Add initial knowledge (how to authorize to the application, etc.)');
-    tag('substep').log(chalk.yellow(`${getCliName()} learn '*' 'to authorize use these credentials: admin@example.com / secret123'`));
-    tag('substep').log('You can use ${env.LOGIN} and ${env.PASSWORD} to reference environment variables.');
-
-    log('5. Launch application on a relative URL');
-    tag('substep').log(chalk.yellow(`${getCliName()} start /dashboard`));
-
     if (!existsSync('./output')) {
       mkdirSync('./output', { recursive: true });
-      log('Created directory: output');
+      tag('success').log(`${figureSet.tick} Created directory: output`);
     }
+
+    log('');
+    log(`${figureSet.star} What's next:`);
+    printInitNextSteps({ url: options.url });
   } catch (error) {
     log('Failed to create config file:', error);
     process.exit(1);
@@ -227,13 +241,13 @@ async function renderInitWizard(mode: 'choose' | 'global'): Promise<'local' | 'g
   });
 }
 
-async function renderLocalProviderWizard(): Promise<string | null> {
+async function renderLocalSetupWizard(): Promise<LocalSetup | null> {
   const [{ render }, React, InitWizard] = await Promise.all([import('ink'), import('react'), import('../components/InitWizard.js').then((m) => m.default)]);
 
   return new Promise((resolve) => {
-    const finish = (provider: string | null) => {
+    const finish = (setup: LocalSetup | null) => {
       unmount();
-      resolve(provider);
+      resolve(setup);
     };
     const { unmount } = render(
       React.createElement(InitWizard, {
@@ -242,7 +256,7 @@ async function renderLocalProviderWizard(): Promise<string | null> {
         onLocal: () => finish(null),
         onComplete: () => finish(null),
         onCancel: () => finish(null),
-        onLocalProvider: (provider: string) => finish(provider),
+        onLocalSetup: (setup) => finish(setup),
       }),
       { exitOnCtrlC: false, patchConsole: false }
     );
@@ -274,7 +288,7 @@ function globalConfigTemplate(provider: string): string {
 // Models are written as 'provider/model-id' so they resolve without a local node_modules.
 // The key is read from ${envKey} in ~/.explorbot/.env
 // Model ids are snapshotted from the recommendations of this Explorbot version.
-// https://github.com/testomatio/explorbot/blob/main/docs/basics/providers.md
+// https://testomat.ai/docs/explorbot
 const config = {
   ai: {
 ${modelLines(provider)}
@@ -292,8 +306,47 @@ export default config;
 `;
 }
 
-function writeEnvKey(key: string, value: string): void {
-  const envPath = globalEnvPath();
+function printInitNextSteps(options: { url?: string; siteCommands?: NextStepCommand[] }): void {
+  const cli = getCliName();
+  const sections: NextStepSection[] = [
+    {
+      label: 'Try the demo on a sample app',
+      commands: [{ command: `${cli} explore https://todomvc.com/examples/react/dist/` }],
+    },
+  ];
+
+  if (options.url) {
+    sections.push({
+      label: 'Explore your app',
+      commands: [
+        { label: 'Whole site', command: `${cli} explore ${options.url}` },
+        { label: 'One page', command: `${cli} explore /dashboard` },
+      ],
+    });
+  }
+
+  if (options.siteCommands) sections.push({ label: 'Explore any site', commands: options.siteCommands });
+
+  sections.push(
+    {
+      label: 'Run it from VS Code',
+      commands: [{ command: 'https://testomat.ai/explorbot' }],
+    },
+    {
+      label: 'Docs',
+      commands: [{ command: 'https://testomat.ai/docs/explorbot' }],
+    }
+  );
+
+  printNextSteps(sections);
+
+  log('');
+  log('Add knowledge (logins, credentials) — created on first use in ./knowledge:');
+  tag('substep').log(chalk.yellow(`${cli} learn '*' 'to authorize use these credentials: admin@example.com / secret123'`));
+  tag('substep').log('You can use ${env.LOGIN} and ${env.PASSWORD} to reference environment variables.');
+}
+
+function writeEnvKey(key: string, value: string, envPath = globalEnvPath()): void {
   let content = '# AI provider API keys';
   if (existsSync(envPath)) content = readFileSync(envPath, 'utf8').trimEnd();
 
@@ -306,6 +359,19 @@ function writeEnvKey(key: string, value: string): void {
   writeFileSync(envPath, `${lines.join('\n').trimEnd()}\n`, 'utf8');
 }
 
+function printApiKeySteps(provider: string, envPath: string): void {
+  const { envKey, keyUrl } = PROVIDERS[provider];
+  tag('warning').log(`No API key yet — explorbot needs one to connect to ${provider}`);
+  tag('substep').log(`1. Create a key at ${keyUrl}`);
+  tag('substep').log(`2. Paste it into ${envPath}: ${envKey}=<your-key>`);
+}
+
+type LocalSetup = {
+  provider: string;
+  apiKey: string;
+  url: string;
+};
+
 type InitCommandOptions = {
   configPath?: string;
   force?: boolean;
@@ -313,4 +379,5 @@ type InitCommandOptions = {
   global?: boolean;
   provider?: string;
   apiKey?: string;
+  url?: string;
 };

package/src/components/InitWizard.tsx

@@ -15,14 +15,16 @@ interface InitWizardProps {
   onLocal: () => void;
   onComplete: () => void;
   onCancel: () => void;
-  onLocalProvider?: (provider: string) => void;
+  onLocalSetup?: (setup: { provider: string; apiKey: string; url: string }) => void;
 }
 
-const InitWizard: React.FC<InitWizardProps> = ({ mode, globalConfigExists, onLocal, onComplete, onCancel, onLocalProvider }) => {
-  const [step, setStep] = useState<'target' | 'provider' | 'key' | 'validate'>(mode === 'choose' ? 'target' : 'provider');
+const InitWizard: React.FC<InitWizardProps> = ({ mode, globalConfigExists, onLocal, onComplete, onCancel, onLocalSetup }) => {
+  const [step, setStep] = useState<'target' | 'provider' | 'key' | 'validate' | 'url'>(mode === 'choose' ? 'target' : 'provider');
   const [targetIndex, setTargetIndex] = useState(0);
   const [providerIndex, setProviderIndex] = useState(0);
   const [apiKey, setApiKey] = useState('');
+  const [url, setUrl] = useState('');
+  const [urlError, setUrlError] = useState('');
   const [status, setStatus] = useState('');
   const [error, setError] = useState('');
 
@@ -30,6 +32,11 @@ const InitWizard: React.FC<InitWizardProps> = ({ mode, globalConfigExists, onLoc
   const envKey = PROVIDERS[provider].envKey;
 
   const save = () => {
+    if (mode === 'local') {
+      setStatus('');
+      setStep('url');
+      return;
+    }
     writeGlobalConfig(provider, apiKey.trim());
     onComplete();
   };
@@ -61,7 +68,7 @@ const InitWizard: React.FC<InitWizardProps> = ({ mode, globalConfigExists, onLoc
       return;
     }
 
-    if (step === 'key') return;
+    if (step === 'key' || step === 'url') return;
 
     if (key.escape) {
       onCancel();
@@ -79,10 +86,7 @@ const InitWizard: React.FC<InitWizardProps> = ({ mode, globalConfigExists, onLoc
     if (step === 'provider') {
       if (key.upArrow) setProviderIndex((index) => Math.max(0, index - 1));
       if (key.downArrow) setProviderIndex((index) => Math.min(PROVIDER_NAMES.length - 1, index + 1));
-      if (key.return) {
-        if (mode === 'local') onLocalProvider?.(provider);
-        else setStep('key');
-      }
+      if (key.return) setStep('key');
       return;
     }
 
@@ -132,7 +136,10 @@ const InitWizard: React.FC<InitWizardProps> = ({ mode, globalConfigExists, onLoc
       {step === 'key' && (
         <Box flexDirection="column">
           <Text>
-            Enter the API key for {provider} <Text dimColor>(stored as {envKey})</Text>
+            Enter the API key for {provider}{' '}
+            <Text dimColor>
+              (stored as {envKey}, create one at {PROVIDERS[provider].keyUrl})
+            </Text>
           </Text>
           <Box borderStyle="single" borderColor="blue" paddingX={1}>
             <InputReadline value={apiKey} onChange={setApiKey} onSubmit={() => setStep('validate')} placeholder={process.env[envKey] ? 'leave empty to keep the key from your environment' : 'paste the key'} isActive showPrompt={false} mask />
@@ -147,15 +154,40 @@ const InitWizard: React.FC<InitWizardProps> = ({ mode, globalConfigExists, onLoc
           {error && (
             <Box flexDirection="column">
               <Text color="red">{error}</Text>
-              <Text dimColor>r: re-enter the key | n: save anyway</Text>
+              <Text dimColor>r: re-enter the key | n: continue anyway</Text>
             </Box>
           )}
         </Box>
       )}
 
+      {step === 'url' && (
+        <Box flexDirection="column">
+          <Text>
+            What is the URL of the web application to test? <Text dimColor>(written to web.url in the config)</Text>
+          </Text>
+          <Box borderStyle="single" borderColor="blue" paddingX={1}>
+            <InputReadline
+              value={url}
+              onChange={setUrl}
+              onSubmit={() => {
+                if (!URL.canParse(url.trim())) {
+                  setUrlError('Enter a full URL, e.g. http://localhost:3000');
+                  return;
+                }
+                onLocalSetup?.({ provider, apiKey: apiKey.trim(), url: url.trim() });
+              }}
+              placeholder="http://localhost:3000"
+              isActive
+              showPrompt={false}
+            />
+          </Box>
+          {urlError && <Text color="red">{urlError}</Text>}
+        </Box>
+      )}
+
       <Box marginTop={1}>
         <Text dimColor>
-          Config goes to {mode === 'local' ? 'the current directory' : globalDir()} | {step === 'key' ? 'Enter: continue' : '↑↓: select | Enter: confirm'} | Ctrl+C: exit
+          Config goes to {mode === 'local' ? 'the current directory' : globalDir()} | {step === 'key' || step === 'url' ? 'Enter: continue' : '↑↓: select | Enter: confirm'} | Ctrl+C: exit
         </Text>
       </Box>
     </Box>

package/src/config.ts

@@ -11,13 +11,13 @@ import { deepMerge } from './utils/merge.js';
 import { log, tag } from './utils/logger.js';
 
 export const PROVIDERS: Record<string, ProviderInfo> = {
-  openai: { envKey: 'OPENAI_API_KEY', load: async () => (await import('@ai-sdk/openai')).createOpenAI() },
-  anthropic: { envKey: 'ANTHROPIC_API_KEY', load: async () => (await import('@ai-sdk/anthropic')).createAnthropic() },
-  google: { envKey: 'GOOGLE_GENERATIVE_AI_API_KEY', load: async () => (await import('@ai-sdk/google')).createGoogleGenerativeAI() },
-  groq: { envKey: 'GROQ_API_KEY', load: async () => (await import('@ai-sdk/groq')).createGroq() },
-  mistral: { envKey: 'MISTRAL_API_KEY', load: async () => (await import('@ai-sdk/mistral')).createMistral() },
-  openrouter: { envKey: 'OPENROUTER_API_KEY', load: async () => (await import('@openrouter/ai-sdk-provider')).createOpenRouter() },
-  sambanova: { envKey: 'SAMBANOVA_API_KEY', load: async () => (await import('sambanova-ai-provider')).createSambaNova() },
+  openai: { envKey: 'OPENAI_API_KEY', keyUrl: 'https://platform.openai.com/api-keys', load: async () => (await import('@ai-sdk/openai')).createOpenAI() },
+  anthropic: { envKey: 'ANTHROPIC_API_KEY', keyUrl: 'https://console.anthropic.com/settings/keys', load: async () => (await import('@ai-sdk/anthropic')).createAnthropic() },
+  google: { envKey: 'GOOGLE_GENERATIVE_AI_API_KEY', keyUrl: 'https://aistudio.google.com/apikey', load: async () => (await import('@ai-sdk/google')).createGoogleGenerativeAI() },
+  groq: { envKey: 'GROQ_API_KEY', keyUrl: 'https://console.groq.com/keys', load: async () => (await import('@ai-sdk/groq')).createGroq() },
+  mistral: { envKey: 'MISTRAL_API_KEY', keyUrl: 'https://console.mistral.ai/api-keys', load: async () => (await import('@ai-sdk/mistral')).createMistral() },
+  openrouter: { envKey: 'OPENROUTER_API_KEY', keyUrl: 'https://openrouter.ai/keys', load: async () => (await import('@openrouter/ai-sdk-provider')).createOpenRouter() },
+  sambanova: { envKey: 'SAMBANOVA_API_KEY', keyUrl: 'https://cloud.sambanova.ai/apis', load: async () => (await import('sambanova-ai-provider')).createSambaNova() },
 };
 
 export const MODEL_ROLES: ModelRole[] = ['model', 'visionModel', 'agenticModel'];
@@ -169,6 +169,7 @@ interface AgentsConfig {
 
 interface AIConfig {
   model: any;
+  recommendedModels?: string;
   apiKey?: string;
   config?: Record<string, any>;
   langfuse?: {
@@ -890,11 +891,34 @@ export async function resolveConfigModels(ai?: AIConfig): Promise<void> {
     if (typeof ai[role] === 'string') ai[role] = await resolveModel(ai[role], role);
   }
 
+  await fillRecommendedModels(ai);
+
   for (const agent of Object.values(ai.agents || {})) {
     if (typeof agent?.model === 'string') agent.model = await resolveModel(agent.model);
   }
 }
 
+async function fillRecommendedModels(ai: AIConfig): Promise<void> {
+  const provider = ai.recommendedModels;
+  if (!provider) return;
+
+  const recommended = ConfigParser.recommendedModels()[provider];
+  if (!recommended) {
+    throw new Error(`No recommended models for "${provider}". Use a provider with recommendations: ${Object.keys(ConfigParser.recommendedModels()).join(', ')}`);
+  }
+
+  for (const role of MODEL_ROLES) {
+    if (ai[role]) continue;
+    if (!recommended[role]) continue;
+    ai[role] = await resolveModel(provider, role);
+  }
+
+  if (!ai.decisionModel && recommended.decisionModel) ai.decisionModel = { provider, model: recommended.decisionModel };
+
+  const missing = missingModelRoles(provider);
+  if (missing.length) tag('warning').log(`No recommended ${missing.join(' and ')} for ${provider} — set them explicitly as "provider/model-id"`);
+}
+
 export function resolveLangfuse(ai?: AIConfig): void {
   if (!ai) return;
 
@@ -994,6 +1018,7 @@ interface ConfiguredModel {
 
 interface ProviderInfo {
   envKey: string;
+  keyUrl: string;
   load: () => Promise<(modelId: string) => any>;
 }
 

package/src/test-plan.ts

@@ -331,10 +331,12 @@ export class Test extends Task {
     });
   }
 
+  hasAchieved(expectation: string): boolean {
+    return Object.values(this.notes).some((note) => note.message === expectation && note.status === TestResult.PASSED);
+  }
+
   hasAchievedAny(): boolean {
-    return this.expected.some((expectation) => {
-      return Object.values(this.notes).some((note) => note.message === expectation && note.status === TestResult.PASSED);
-    });
+    return this.expected.some((expectation) => this.hasAchieved(expectation));
   }
 
   getRunResult(): 'success' | 'partial' | 'failed' {
@@ -344,9 +346,7 @@ export class Test extends Task {
   }
 
   hasAchievedAll(): boolean {
-    return this.expected.every((expectation) => {
-      return Object.values(this.notes).some((note) => note.message === expectation && note.status === TestResult.PASSED);
-    });
+    return this.expected.every((expectation) => this.hasAchieved(expectation));
   }
 
   isComplete(): boolean {

package/src/utils/sitemap.ts

@@ -0,0 +1,108 @@
+import { readFileSync } from 'node:fs';
+import { tag } from './logger.js';
+import type { UrlFilter } from './url-filter.ts';
+
+const ENTRY_PATTERN = /<(url|sitemap)\b[^>]*>(.*?)<\/\1>/gis;
+const XML_ENTITIES: Record<string, string> = { '&amp;': '&', '&lt;': '<', '&gt;': '>', '&quot;': '"', '&apos;': "'" };
+const MAX_NESTED_SITEMAPS = 50;
+const DEFAULT_PRIORITY = 0.5;
+const STDIN_SOURCE = '-';
+
+export async function readSitemap(source: string, filter?: UrlFilter): Promise<string[]> {
+  const entries: SitemapEntry[] = [];
+  const seen = new Set<string>();
+  const pending = [source];
+  let fetched = 0;
+
+  while (pending.length > 0) {
+    if (fetched >= MAX_NESTED_SITEMAPS) {
+      tag('warning').log(`Sitemap index lists more than ${MAX_NESTED_SITEMAPS} sitemaps, the rest are ignored`);
+      break;
+    }
+    const next = pending.shift()!;
+    fetched++;
+    const content = await readSource(next);
+    const isIndex = content.includes('<sitemapindex');
+    for (const entry of extractEntries(content)) {
+      if (seen.has(entry.url)) continue;
+      seen.add(entry.url);
+      if (isIndex) {
+        pending.push(entry.url);
+        continue;
+      }
+      entries.push(entry);
+    }
+  }
+
+  return applyFilter(entries, filter).map((entry) => entry.url);
+}
+
+export async function readCliSitemap(path: string | undefined, source: string | undefined, filter?: UrlFilter): Promise<string[] | undefined> {
+  const usage = 'Pass a page path, --sitemap <source>, or pipe a sitemap to stdin';
+  let resolved = source;
+  if (!resolved && !path && !process.stdin.isTTY) resolved = STDIN_SOURCE;
+  if (!resolved && !path) throw new Error(usage);
+  if (!resolved) return undefined;
+  const urls = await readSitemap(resolved, filter);
+  if (urls.length === 0 && !source) throw new Error(usage);
+  if (urls.length === 0) throw new Error(`No URLs left in sitemap ${resolved}`);
+  return urls;
+}
+
+export function keepSiteUrls(urls: string[], siteUrl: string): string[] {
+  const siteOrigin = URL.parse(siteUrl)?.origin;
+  const kept = urls.filter((url) => url.startsWith('/') || URL.parse(url)?.origin === siteOrigin);
+  if (kept.length < urls.length) tag('warning').log(`Skipped ${urls.length - kept.length} sitemap URL(s) outside ${siteOrigin}`);
+  return kept;
+}
+
+export function extractEntries(content: string): SitemapEntry[] {
+  if (!content.trimStart().startsWith('<')) {
+    return content
+      .split('\n')
+      .map((line) => line.trim())
+      .filter((line) => line.startsWith('http://') || line.startsWith('https://') || line.startsWith('/'))
+      .map((url) => ({ url, priority: DEFAULT_PRIORITY }));
+  }
+  const entries: SitemapEntry[] = [];
+  for (const match of content.matchAll(ENTRY_PATTERN)) {
+    const url = tagValue(match[2], 'loc');
+    if (!url) continue;
+    const entry: SitemapEntry = { url, priority: DEFAULT_PRIORITY };
+    const priority = Number.parseFloat(tagValue(match[2], 'priority') || '');
+    if (!Number.isNaN(priority)) entry.priority = priority;
+    const lastmod = Date.parse(tagValue(match[2], 'lastmod') || '');
+    if (!Number.isNaN(lastmod)) entry.lastmod = lastmod;
+    entries.push(entry);
+  }
+  return entries;
+}
+
+function applyFilter(entries: SitemapEntry[], filter: UrlFilter | undefined): SitemapEntry[] {
+  if (!filter) return entries;
+  let result = entries.filter((entry) => filter.allows(entry.url));
+  if (filter.sort === 'priority') result = result.toSorted((a, b) => b.priority - a.priority);
+  if (filter.sort === 'lastmod') result = result.toSorted((a, b) => (b.lastmod ?? 0) - (a.lastmod ?? 0));
+  return result.slice(0, filter.limit);
+}
+
+function tagValue(block: string, name: string): string | undefined {
+  const match = block.match(new RegExp(`<${name}>\\s*(?:<!\\[CDATA\\[)?\\s*(.*?)\\s*(?:\\]\\]>)?\\s*</${name}>`, 'is'));
+  return match?.[1].replace(/&(amp|lt|gt|quot|apos);/g, (entity) => XML_ENTITIES[entity]) || undefined;
+}
+
+async function readSource(source: string): Promise<string> {
+  if (source === STDIN_SOURCE) return readFileSync(0, 'utf8');
+  if (source.startsWith('http://') || source.startsWith('https://')) {
+    const response = await fetch(source);
+    if (!response.ok) throw new Error(`Sitemap ${source} returned HTTP ${response.status}`);
+    return await response.text();
+  }
+  return readFileSync(source, 'utf8');
+}
+
+interface SitemapEntry {
+  url: string;
+  priority: number;
+  lastmod?: number;
+}

package/src/utils/spec.ts

@@ -0,0 +1,21 @@
+import { tag } from './logger.js';
+
+export function parseSpecPairs(raw: string | undefined): SpecPair[] {
+  const pairs: SpecPair[] = [];
+  for (const pair of (raw || '').split(';')) {
+    const trimmed = pair.trim();
+    if (!trimmed) continue;
+    const sepMatch = trimmed.match(/^([^:=]+)\s*[:=]\s*(.*)$/);
+    if (!sepMatch) {
+      tag('warning').log(`Ignoring malformed spec pair: ${trimmed}`);
+      continue;
+    }
+    pairs.push({ key: sepMatch[1].trim().toLowerCase(), value: sepMatch[2].trim() });
+  }
+  return pairs;
+}
+
+export interface SpecPair {
+  key: string;
+  value: string;
+}

package/src/utils/url-filter.ts

@@ -0,0 +1,53 @@
+import { tag } from './logger.js';
+import { parseSpecPairs } from './spec.ts';
+import { matchesUrl } from './url-matcher.ts';
+
+export class UrlFilter {
+  readonly include: string[] = [];
+  readonly exclude: string[] = [];
+  sort?: 'priority' | 'lastmod';
+  limit = Number.POSITIVE_INFINITY;
+  private admitted = new Set<string>();
+
+  constructor(spec?: string) {
+    for (const { key, value } of parseSpecPairs(spec)) {
+      if (key === 'filter') {
+        this.include.push(value);
+        continue;
+      }
+      if (key === 'exclude') {
+        this.exclude.push(value);
+        continue;
+      }
+      if (key === 'sort' && (value === 'priority' || value === 'lastmod')) {
+        this.sort = value;
+        continue;
+      }
+      if (key === 'limit' && Number.parseInt(value, 10) > 0) {
+        this.limit = Number.parseInt(value, 10);
+        continue;
+      }
+      tag('warning').log(`Ignoring URL filter pair ${key}:${value} (use filter|exclude:<url pattern>, sort:priority|lastmod, limit:<n>)`);
+    }
+  }
+
+  allows(url: string): boolean {
+    const path = urlPath(url);
+    if (this.include.length > 0 && !this.include.some((pattern) => matchesUrl(pattern, path))) return false;
+    return !this.exclude.some((pattern) => matchesUrl(pattern, path));
+  }
+
+  admit(url: string): boolean {
+    if (!this.allows(url)) return false;
+    const path = urlPath(url);
+    if (this.admitted.has(path)) return true;
+    if (this.admitted.size >= this.limit) return false;
+    this.admitted.add(path);
+    return true;
+  }
+}
+
+function urlPath(url: string): string {
+  const parsed = new URL(url, 'http://localhost');
+  return `${parsed.pathname}${parsed.search}`;
+}