@detiq/crawler 0.1.3

package/src/session-export.test.ts

@@ -0,0 +1,75 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { toCypressSnippet, toSeleniumPythonDict, toWebdriverIOSnippet, exportSessionAs, type StorageState } from './session-export.js';
+
+const sampleState: StorageState = {
+  cookies: [
+    { name: 'session_id', value: 'abc123', domain: '.example.com', path: '/', secure: true, httpOnly: true, sameSite: 'Lax', expires: 1893456000 },
+  ],
+  origins: [
+    { origin: 'https://example.com', localStorage: [{ name: 'authToken', value: 'xyz789' }] },
+  ],
+};
+
+test('toCypressSnippet: emits cy.setCookie for each cookie and cy.window().localStorage for each origin', () => {
+  const out = toCypressSnippet(sampleState);
+  assert.match(out, /cy\.setCookie\("session_id", "abc123"/);
+  assert.match(out, /domain: "\.example\.com"/);
+  assert.match(out, /secure: true/);
+  assert.match(out, /httpOnly: true/);
+  assert.match(out, /cy\.visit\("https:\/\/example\.com"\)/);
+  assert.match(out, /localStorage\.setItem\("authToken", "xyz789"\)/);
+  assert.match(out, /Cypress\.Commands\.add\('restoreZetaSession'/);
+});
+
+test('toSeleniumPythonDict: navigates to each cookie domain before add_cookie, and to each origin for localStorage', () => {
+  const out = toSeleniumPythonDict(sampleState);
+  assert.match(out, /driver\.get\("https:\/\/example\.com"\)/);
+  assert.match(out, /driver\.add_cookie\(/);
+  assert.match(out, /"name": "session_id"/);
+  assert.match(out, /"value": "abc123"/);
+  assert.match(out, /driver\.execute_script\(/);
+});
+
+test('toSeleniumPythonDict: emits Python True/False, not JSON true/false, for boolean fields', () => {
+  const out = toSeleniumPythonDict(sampleState);
+  // sampleState's cookie has secure: true — must render as Python's True.
+  assert.match(out, /"secure": True/);
+  assert.doesNotMatch(out, /:\s*true\b/);
+  assert.doesNotMatch(out, /:\s*false\b/);
+});
+
+test('toWebdriverIOSnippet: browser.setCookies() array shape close to Playwright cookies, plus localStorage via execute()', () => {
+  const out = toWebdriverIOSnippet(sampleState);
+  assert.match(out, /browser\.setCookies\(/);
+  assert.match(out, /"name": "session_id"/);
+  assert.match(out, /"domain": "\.example\.com"/);
+  assert.match(out, /browser\.url\("https:\/\/example\.com"\)/);
+  assert.match(out, /localStorage\.setItem\("authToken", "xyz789"\)/);
+  assert.match(out, /module\.exports = \{ restoreZetaSession \}/);
+});
+
+test('exportSessionAs: returns correct filename/mimeType per format', () => {
+  const cy = exportSessionAs('cypress', sampleState);
+  assert.equal(cy.filename, 'zeta-session.cy-commands.js');
+  assert.equal(cy.mimeType, 'application/javascript');
+
+  const py = exportSessionAs('selenium-python', sampleState);
+  assert.equal(py.filename, 'zeta_session.py');
+  assert.equal(py.mimeType, 'text/x-python');
+
+  const wdio = exportSessionAs('webdriverio', sampleState);
+  assert.equal(wdio.filename, 'zeta-session.wdio.js');
+  assert.equal(wdio.mimeType, 'application/javascript');
+});
+
+test('exportSessionAs: throws on an unknown format', () => {
+  assert.throws(() => exportSessionAs('unknown' as any, sampleState), /Unknown session export format/);
+});
+
+test('handles empty cookies/origins gracefully (no crash, minimal output)', () => {
+  const empty: StorageState = { cookies: [], origins: [] };
+  assert.doesNotThrow(() => toCypressSnippet(empty));
+  assert.doesNotThrow(() => toSeleniumPythonDict(empty));
+  assert.doesNotThrow(() => toWebdriverIOSnippet(empty));
+});

package/src/session-export.ts

@@ -0,0 +1,172 @@
+/**
+ * Converts a Playwright storageState (the format ZeTa already captures via
+ * session-capture.ts/live-view.ts and stores per-project) into snippets for
+ * other automation frameworks. Playwright's storageState is the de-facto
+ * standard for browser session export — see docs/UNIVERSAL-SESSION-CAPTURE-
+ * STRATEGY.md — so every other framework's import mechanism is a small,
+ * mechanical conversion away, not a new capture flow.
+ *
+ * Pure, dependency-free functions — no I/O, no network, fully unit-testable.
+ */
+
+export interface StorageStateCookie {
+  name: string;
+  value: string;
+  domain: string;
+  path: string;
+  expires?: number;
+  httpOnly?: boolean;
+  secure?: boolean;
+  sameSite?: 'Strict' | 'Lax' | 'None';
+}
+
+export interface StorageStateOrigin {
+  origin: string;
+  localStorage: Array<{ name: string; value: string }>;
+}
+
+export interface StorageState {
+  cookies: StorageStateCookie[];
+  origins: StorageStateOrigin[];
+}
+
+export type SessionExportFormat = 'cypress' | 'selenium-python' | 'webdriverio';
+
+function jsStringLiteral(s: string): string {
+  return JSON.stringify(s);
+}
+
+/** Python dict-literal repr — JSON.stringify's `true`/`false`/`null` aren't
+ *  valid Python (Python needs `True`/`False`/`None`), so a plain
+ *  JSON.stringify would emit syntactically invalid Python for any cookie
+ *  with a boolean field. */
+function pythonDictLiteral(obj: Record<string, unknown>): string {
+  const parts = Object.entries(obj).map(([k, v]) => {
+    let repr: string;
+    if (typeof v === 'boolean') repr = v ? 'True' : 'False';
+    else if (v === null || v === undefined) repr = 'None';
+    else if (typeof v === 'number') repr = String(v);
+    else repr = JSON.stringify(v);
+    return `${JSON.stringify(k)}: ${repr}`;
+  });
+  return `{${parts.join(', ')}}`;
+}
+
+/** cy.setCookie() + localStorage.setItem() inside a reusable custom command,
+ *  following Cypress's documented cy.session() pattern (see references in
+ *  docs/UNIVERSAL-SESSION-CAPTURE-STRATEGY.md). */
+export function toCypressSnippet(state: StorageState): string {
+  const cookieLines = state.cookies.map((c) => {
+    const opts: string[] = [`domain: ${jsStringLiteral(c.domain)}`, `path: ${jsStringLiteral(c.path)}`];
+    if (c.secure) opts.push('secure: true');
+    if (c.httpOnly) opts.push('httpOnly: true');
+    if (c.sameSite) opts.push(`sameSite: ${jsStringLiteral(c.sameSite.toLowerCase())}`);
+    return `  cy.setCookie(${jsStringLiteral(c.name)}, ${jsStringLiteral(c.value)}, { ${opts.join(', ')} });`;
+  });
+
+  const localStorageBlocks = state.origins
+    .filter((o) => o.localStorage.length > 0)
+    .map((o) => {
+      const sets = o.localStorage
+        .map((kv) => `      window.localStorage.setItem(${jsStringLiteral(kv.name)}, ${jsStringLiteral(kv.value)});`)
+        .join('\n');
+      return `  cy.visit(${jsStringLiteral(o.origin)});\n  cy.window().then((window) => {\n${sets}\n  });`;
+    });
+
+  return [
+    '// Generated by ZeTa-AI — restores a captured browser session in Cypress.',
+    '// Usage: import this file and call restoreZetaSession() inside cy.session(), e.g.:',
+    "//   cy.session('zeta-session', restoreZetaSession)",
+    "Cypress.Commands.add('restoreZetaSession', () => {",
+    ...cookieLines,
+    ...localStorageBlocks,
+    '});',
+  ].join('\n');
+}
+
+/** driver.add_cookie() (Selenium requires the browser already be on the
+ *  cookie's domain before adding it) + execute_script() for localStorage,
+ *  per Selenium's documented Cookie API. */
+export function toSeleniumPythonDict(state: StorageState): string {
+  const byDomain = new Map<string, StorageStateCookie[]>();
+  for (const c of state.cookies) {
+    const list = byDomain.get(c.domain) ?? [];
+    list.push(c);
+    byDomain.set(c.domain, list);
+  }
+
+  const lines: string[] = [
+    '# Generated by ZeTa-AI — restores a captured browser session in Selenium.',
+    '# Selenium requires the driver to already be on a matching domain before',
+    '# add_cookie() will accept a cookie for it — navigate first, then restore.',
+    'def restore_zeta_session(driver):',
+  ];
+
+  for (const [domain, cookies] of byDomain) {
+    lines.push(`    driver.get(${JSON.stringify(`https://${domain.replace(/^\./, '')}`)})`);
+    for (const c of cookies) {
+      const cookieDict: Record<string, unknown> = { name: c.name, value: c.value, path: c.path };
+      if (c.expires && c.expires > 0) cookieDict.expiry = Math.floor(c.expires);
+      if (c.secure) cookieDict.secure = true;
+      lines.push(`    driver.add_cookie(${pythonDictLiteral(cookieDict)})`);
+    }
+  }
+
+  for (const o of state.origins) {
+    if (o.localStorage.length === 0) continue;
+    lines.push(`    driver.get(${JSON.stringify(o.origin)})`);
+    for (const kv of o.localStorage) {
+      lines.push(
+        `    driver.execute_script(${JSON.stringify(`window.localStorage.setItem(arguments[0], arguments[1]);`)}, ${JSON.stringify(kv.name)}, ${JSON.stringify(kv.value)})`,
+      );
+    }
+  }
+
+  return lines.join('\n');
+}
+
+/** browser.setCookies() accepts an array shape very close to Playwright's own
+ *  cookie objects (per WebdriverIO's documented API) — closest 1:1 mapping
+ *  of the three target formats. */
+export function toWebdriverIOSnippet(state: StorageState): string {
+  const cookies = state.cookies.map((c) => ({
+    name: c.name,
+    value: c.value,
+    domain: c.domain,
+    path: c.path,
+    secure: !!c.secure,
+    httpOnly: !!c.httpOnly,
+    ...(c.expires && c.expires > 0 ? { expiry: Math.floor(c.expires) } : {}),
+  }));
+
+  const localStorageSets = state.origins
+    .filter((o) => o.localStorage.length > 0)
+    .map((o) => {
+      const sets = o.localStorage
+        .map((kv) => `    localStorage.setItem(${jsStringLiteral(kv.name)}, ${jsStringLiteral(kv.value)});`)
+        .join('\n');
+      return `  await browser.url(${jsStringLiteral(o.origin)});\n  await browser.execute(() => {\n${sets}\n  });`;
+    });
+
+  return [
+    '// Generated by ZeTa-AI — restores a captured browser session in WebdriverIO.',
+    'async function restoreZetaSession(browser) {',
+    `  await browser.setCookies(${JSON.stringify(cookies, null, 2).split('\n').join('\n  ')});`,
+    ...localStorageSets,
+    '}',
+    'module.exports = { restoreZetaSession };',
+  ].join('\n');
+}
+
+export function exportSessionAs(format: SessionExportFormat, state: StorageState): { content: string; filename: string; mimeType: string } {
+  switch (format) {
+    case 'cypress':
+      return { content: toCypressSnippet(state), filename: 'zeta-session.cy-commands.js', mimeType: 'application/javascript' };
+    case 'selenium-python':
+      return { content: toSeleniumPythonDict(state), filename: 'zeta_session.py', mimeType: 'text/x-python' };
+    case 'webdriverio':
+      return { content: toWebdriverIOSnippet(state), filename: 'zeta-session.wdio.js', mimeType: 'application/javascript' };
+    default:
+      throw new Error(`Unknown session export format: ${format}`);
+  }
+}

package/src/sitemap-writer.ts

@@ -0,0 +1,65 @@
+/**
+ * Sitemap writer — generate XML sitemap from crawled screens.
+ *
+ * Compliant with sitemaps.org protocol (sitemap index + individual sitemap files).
+ * Splits into 50k-URL files per the spec when needed.
+ * Used for: SEO audit export, future crawl seed, and crawler/robots.txt comparison.
+ */
+
+import * as fs from 'fs/promises';
+import * as path from 'path';
+
+export interface SitemapUrl {
+  loc: string;
+  lastmod?: string;
+  changefreq?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
+  priority?: number; // 0.0–1.0
+}
+
+const XML_HEADER = '<?xml version="1.0" encoding="UTF-8"?>\n';
+const SITEMAP_NS = 'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"';
+
+function escape(s: string): string {
+  return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;').replace(/'/g, '&apos;');
+}
+
+function buildSitemapXml(urls: SitemapUrl[]): string {
+  const entries = urls.map(u => [
+    '  <url>',
+    `    <loc>${escape(u.loc)}</loc>`,
+    u.lastmod ? `    <lastmod>${u.lastmod}</lastmod>` : '',
+    u.changefreq ? `    <changefreq>${u.changefreq}</changefreq>` : '',
+    u.priority !== undefined ? `    <priority>${u.priority.toFixed(1)}</priority>` : '',
+    '  </url>',
+  ].filter(Boolean).join('\n')).join('\n');
+
+  return `${XML_HEADER}<urlset ${SITEMAP_NS}>\n${entries}\n</urlset>`;
+}
+
+export async function writeSitemap(
+  urls: SitemapUrl[],
+  outputPath: string,
+): Promise<void> {
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+  await fs.writeFile(outputPath, buildSitemapXml(urls), 'utf8');
+}
+
+/** Convert crawled screen records to SitemapUrl objects */
+export function screensToSitemapUrls(
+  screens: Array<{ url: string; updatedAt?: Date; depth?: number }>,
+): SitemapUrl[] {
+  return screens
+    .filter(s => {
+      try { return new URL(s.url).protocol.startsWith('http'); } catch { return false; }
+    })
+    .map(s => ({
+      loc: s.url,
+      lastmod: s.updatedAt ? s.updatedAt.toISOString().split('T')[0] : undefined,
+      changefreq: 'weekly' as const,
+      priority: s.depth === 0 ? 1.0 : s.depth === 1 ? 0.8 : 0.6,
+    }));
+}
+
+export function buildSitemapString(urls: SitemapUrl[]): string {
+  return buildSitemapXml(urls);
+}

package/src/sitemap.test.ts

@@ -0,0 +1,131 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import http from 'node:http';
+import os from 'node:os';
+import { XMLParser } from 'fast-xml-parser';
+import { extractUrlsFromParsedSitemap, fetchSitemapUrls, discoverSitemapUrls } from './sitemap.js';
+
+const parser = new XMLParser();
+
+test('extractUrlsFromParsedSitemap: parses a plain <urlset> sitemap', () => {
+  const xml = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url><loc>https://example.com/</loc></url>
+  <url><loc>https://example.com/pricing</loc></url>
+  <url><loc>https://example.com/about</loc></url>
+</urlset>`;
+  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parser.parse(xml), 'https://example.com');
+  assert.deepEqual(pageUrls.sort(), ['https://example.com/', 'https://example.com/about', 'https://example.com/pricing'].sort());
+  assert.deepEqual(subSitemapUrls, []);
+});
+
+test('extractUrlsFromParsedSitemap: parses a <sitemapindex> into sub-sitemap URLs', () => {
+  const xml = `<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <sitemap><loc>https://example.com/sitemap-pages.xml</loc></sitemap>
+  <sitemap><loc>https://example.com/sitemap-blog.xml</loc></sitemap>
+</sitemapindex>`;
+  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parser.parse(xml), 'https://example.com');
+  assert.deepEqual(pageUrls, []);
+  assert.deepEqual(subSitemapUrls, ['https://example.com/sitemap-pages.xml', 'https://example.com/sitemap-blog.xml']);
+});
+
+test('extractUrlsFromParsedSitemap: filters out cross-origin URLs', () => {
+  const xml = `<urlset>
+  <url><loc>https://example.com/page</loc></url>
+  <url><loc>https://attacker.example/evil</loc></url>
+</urlset>`;
+  const { pageUrls } = extractUrlsFromParsedSitemap(parser.parse(xml), 'https://example.com');
+  assert.deepEqual(pageUrls, ['https://example.com/page']);
+});
+
+test('extractUrlsFromParsedSitemap: dedupes repeated URLs', () => {
+  const xml = `<urlset>
+  <url><loc>https://example.com/a</loc></url>
+  <url><loc>https://example.com/a</loc></url>
+</urlset>`;
+  const { pageUrls } = extractUrlsFromParsedSitemap(parser.parse(xml), 'https://example.com');
+  assert.deepEqual(pageUrls, ['https://example.com/a']);
+});
+
+test('extractUrlsFromParsedSitemap: a single <url> entry is not an array (fast-xml-parser quirk) — still handled', () => {
+  const xml = `<urlset><url><loc>https://example.com/only</loc></url></urlset>`;
+  const { pageUrls } = extractUrlsFromParsedSitemap(parser.parse(xml), 'https://example.com');
+  assert.deepEqual(pageUrls, ['https://example.com/only']);
+});
+
+test('extractUrlsFromParsedSitemap: handles empty/malformed input without throwing', () => {
+  assert.deepEqual(extractUrlsFromParsedSitemap({}, 'https://example.com'), { pageUrls: [], subSitemapUrls: [] });
+  assert.deepEqual(extractUrlsFromParsedSitemap(null, 'https://example.com'), { pageUrls: [], subSitemapUrls: [] });
+  assert.deepEqual(extractUrlsFromParsedSitemap({ urlset: { url: [{ notLoc: 'x' }] } }, 'https://example.com'), { pageUrls: [], subSitemapUrls: [] });
+});
+
+// --- Live end-to-end: a real local HTTP server, real fetch, real XML parse ---
+// Bound to the sandbox's actual (non-loopback) address — assertCrawlUrlSafe
+// deliberately blocks literal localhost/127.0.0.1 (meaningless as a crawl
+// target, would just hit the crawler container itself) but allows other
+// addresses, matching how a real deployment reaches a tenant's internal app.
+function getNonLoopbackAddress(): string {
+  for (const ifaces of Object.values(os.networkInterfaces())) {
+    for (const iface of ifaces as any[]) {
+      if (iface.family === 'IPv4' && !iface.internal) return iface.address;
+    }
+  }
+  throw new Error('No non-loopback IPv4 address found on this host — cannot run the live sitemap test');
+}
+
+test('fetchSitemapUrls: live end-to-end against a real HTTP server serving a sitemap index', async () => {
+  const host = getNonLoopbackAddress();
+  const indexXml = `<?xml version="1.0"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <sitemap><loc>http://${host}:PORT/sub1.xml</loc></sitemap>
+  <sitemap><loc>http://${host}:PORT/sub2.xml</loc></sitemap>
+</sitemapindex>`;
+  const sub1Xml = `<urlset><url><loc>http://${host}:PORT/page1</loc></url><url><loc>http://${host}:PORT/page2</loc></url></urlset>`;
+  const sub2Xml = `<urlset><url><loc>http://${host}:PORT/page3</loc></url></urlset>`;
+
+  const server = http.createServer((req, res) => {
+    res.writeHead(200, { 'Content-Type': 'application/xml' });
+    if (req.url === '/sitemap.xml') res.end(indexXml.replaceAll('PORT', String(port)));
+    else if (req.url === '/sub1.xml') res.end(sub1Xml.replaceAll('PORT', String(port)));
+    else if (req.url === '/sub2.xml') res.end(sub2Xml.replaceAll('PORT', String(port)));
+    else { res.writeHead(404); res.end(); }
+  });
+  await new Promise<void>((resolve) => server.listen(0, host, resolve));
+  const port = (server.address() as any).port;
+
+  try {
+    const origin = `http://${host}:${port}`;
+    const urls = await fetchSitemapUrls(`${origin}/sitemap.xml`, origin);
+    assert.deepEqual(
+      urls.sort(),
+      [`${origin}/page1`, `${origin}/page2`, `${origin}/page3`].sort(),
+    );
+  } finally {
+    server.close();
+  }
+});
+
+test('discoverSitemapUrls: defaults to /sitemap.xml at the app origin and returns [] gracefully on 404', async () => {
+  const host = getNonLoopbackAddress();
+  const server = http.createServer((_req, res) => { res.writeHead(404); res.end(); });
+  await new Promise<void>((resolve) => server.listen(0, host, resolve));
+  const port = (server.address() as any).port;
+
+  try {
+    const urls = await discoverSitemapUrls(`http://${host}:${port}/app`);
+    assert.deepEqual(urls, []);
+  } finally {
+    server.close();
+  }
+});
+
+test('discoverSitemapUrls: never throws for an invalid appUrl', async () => {
+  const urls = await discoverSitemapUrls('not-a-valid-url');
+  assert.deepEqual(urls, []);
+});
+
+test('discoverSitemapUrls: is blocked (returns []) for a disallowed target like localhost — assertCrawlUrlSafe still applies', async () => {
+  const urls = await discoverSitemapUrls('http://localhost:1/app');
+  assert.deepEqual(urls, []);
+});

package/src/sitemap.ts

@@ -0,0 +1,224 @@
+/**
+ * Sitemap.xml discovery — seeds the crawl queue from a project's sitemap
+ * instead of (or alongside) plain link-following, matching crawl4AI's
+ * sitemap-seeding capability. Best-effort throughout: most sites don't have
+ * a sitemap, or it 404s, or it's malformed — none of that should ever fail
+ * a crawl, it just means sitemap seeding contributes nothing this run.
+ */
+import { XMLParser } from 'fast-xml-parser';
+import { ProxyAgent } from 'undici';
+import { assertCrawlUrlSafe } from '@detiq/core';
+import { logger } from './logger.js';
+
+const FETCH_TIMEOUT_MS = 10_000;
+const MAX_SITEMAP_INDEX_DEPTH = 3; // sitemap index -> sub-sitemaps -> (no deeper nesting expected)
+const MAX_SUB_SITEMAPS = 50; // bound fan-out from a single sitemap index
+const MAX_URLS = 5000; // sane upper bound so a huge/malicious sitemap can't blow up memory
+
+function toArray<T>(x: T | T[] | undefined | null): T[] {
+  if (x == null) return [];
+  return Array.isArray(x) ? x : [x];
+}
+
+/** Registrable-host key: lowercased hostname with a leading `www.` stripped. */
+function hostKey(hostname: string): string {
+  return hostname.replace(/^www\./i, '').toLowerCase();
+}
+
+/**
+ * Same-site check that tolerates www/non-www and http/https differences — a sitemap on
+ * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).
+ */
+export function sameSite(candidateUrl: string, siteOrigin: string): boolean {
+  try {
+    const originHost = /^https?:\/\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;
+    return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap
+ * document (either a <urlset> or a <sitemapindex>), filtered to same-origin
+ * and deduped. Pure function — no I/O — for direct unit testing without a
+ * real HTTP fetch.
+ */
+export function extractUrlsFromParsedSitemap(parsed: any, origin: string): { pageUrls: string[]; subSitemapUrls: string[] } {
+  const rawPageUrls: string[] = [];
+  let subSitemapUrls: string[] = [];
+
+  if (parsed?.sitemapindex) {
+    subSitemapUrls = toArray(parsed.sitemapindex.sitemap)
+      .map((s: any) => s?.loc)
+      .filter((loc: any): loc is string => typeof loc === 'string')
+      .slice(0, MAX_SUB_SITEMAPS);
+  }
+
+  if (parsed?.urlset) {
+    rawPageUrls.push(
+      ...toArray(parsed.urlset.url)
+        .map((u: any) => u?.loc)
+        .filter((loc: any): loc is string => typeof loc === 'string'),
+    );
+  }
+
+  // Same-site only (www/protocol-tolerant) — a sitemap listing external URLs shouldn't seed this project's queue.
+  const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));
+
+  return { pageUrls: [...new Set(sameOrigin)].slice(0, MAX_URLS), subSitemapUrls };
+}
+
+async function fetchAndParseXml(url: string, dispatcher?: any): Promise<any | null> {
+  // Same SSRF posture as the crawl itself (assertCrawlUrlSafe, not the
+  // stricter webhook-oriented assertResolvesToPublicAddress) — a sitemap
+  // living on a tenant's internal/VPN-only staging server is a legitimate
+  // crawl target, not an attack; only loopback/cloud-metadata are blocked.
+  try {
+    assertCrawlUrlSafe(url);
+  } catch {
+    return null;
+  }
+  try {
+    // Redirects followed normally here (unlike the webhook-dispatch.ts
+    // 'manual' pattern) — an http→https or bare→www redirect on the
+    // sitemap is completely ordinary, and Playwright's actual page
+    // navigation later in the crawl already follows redirects to whatever
+    // this same site sends it to; a sitemap fetch isn't a more sensitive
+    // operation than that.
+    const res = await fetch(url, {
+      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+sitemap-discovery)' },
+      ...(dispatcher ? { dispatcher } : {}),
+    });
+    if (!res.ok) return null;
+    const text = await res.text();
+    return new XMLParser().parse(text);
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Fetches and parses a sitemap (or sitemap index, recursed up to a small
+ * depth) into a flat, deduped list of same-origin URLs.
+ */
+export async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth = 0, dispatcher?: any): Promise<string[]> {
+  if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];
+  const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);
+  if (!parsed) return [];
+
+  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);
+  const collected = [...pageUrls];
+
+  for (const subUrl of subSitemapUrls) {
+    if (collected.length >= MAX_URLS) break;
+    collected.push(...(await fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher)));
+  }
+
+  return [...new Set(collected)].slice(0, MAX_URLS);
+}
+
+/**
+ * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an
+ * explicit override) and returns the URLs it lists. Never throws.
+ */
+export async function discoverSitemapUrls(appUrl: string, sitemapUrl?: string): Promise<string[]> {
+  let origin: string;
+  try {
+    origin = new URL(appUrl).origin;
+  } catch {
+    return [];
+  }
+  const target = sitemapUrl || `${origin}/sitemap.xml`;
+  try {
+    return await fetchSitemapUrls(target, origin);
+  } catch (err) {
+    logger.warn({ err: String(err), target }, '[sitemap] discovery failed, continuing without it');
+    return [];
+  }
+}
+
+/**
+ * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because
+ * DataImpulse-style usernames contain commas (country codes) that break new URL().
+ * Returns undefined for no/auto proxy — the caller then fetches directly.
+ */
+export function buildProxyDispatcher(proxyUrl?: string): ProxyAgent | undefined {
+  if (!proxyUrl || proxyUrl === 'auto') return undefined;
+  try {
+    const m = proxyUrl.match(/^(https?):\/\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);
+    if (!m) return new ProxyAgent(proxyUrl);
+    const [, scheme, user, pass, hostPort] = m;
+    const uri = `${scheme}://${hostPort}`;
+    if (user) {
+      const token = 'Basic ' + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? '')}`).toString('base64');
+      return new ProxyAgent({ uri, token });
+    }
+    return new ProxyAgent({ uri });
+  } catch {
+    return undefined;
+  }
+}
+
+/** Extract same-origin `Sitemap:` directives from a robots.txt body. */
+export function parseRobotsSitemaps(robotsTxt: string, origin: string): string[] {
+  const out: string[] = [];
+  for (const line of robotsTxt.split(/\r?\n/)) {
+    const m = line.match(/^\s*sitemap:\s*(\S+)/i);
+    if (m && sameSite(m[1], origin)) out.push(m[1]);
+  }
+  return [...new Set(out)];
+}
+
+export interface UrlCandidate {
+  url: string;
+  source: 'sitemap' | 'robots' | 'link';
+}
+
+/**
+ * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap
+ * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped
+ * candidate list the user can review and seed before launching the Playwright crawl.
+ * Never throws — bot-protected sites (Akamai/Cloudflare) simply yield an empty list.
+ */
+export async function discoverProjectUrls(
+  appUrl: string,
+  opts?: { sitemapUrl?: string; proxyUrl?: string },
+): Promise<{ candidates: UrlCandidate[]; sitemapFound: boolean }> {
+  let origin: string;
+  try { origin = new URL(appUrl).origin; } catch { return { candidates: [], sitemapFound: false }; }
+
+  const dispatcher = buildProxyDispatcher(opts?.proxyUrl);
+
+  // 1. robots.txt → declared sitemaps
+  let robotsSitemaps: string[] = [];
+  try {
+    assertCrawlUrlSafe(`${origin}/robots.txt`);
+    const res = await fetch(`${origin}/robots.txt`, {
+      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+url-discovery)' },
+      ...(dispatcher ? { dispatcher } : {}),
+    });
+    if (res.ok) robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);
+  } catch { /* best-effort */ }
+
+  // 2. sitemap targets: explicit override + robots-declared + conventional default
+  const robotsSet = new Set(robotsSitemaps);
+  const sitemapTargets = [...new Set([
+    ...(opts?.sitemapUrl ? [opts.sitemapUrl] : []),
+    ...robotsSitemaps,
+    `${origin}/sitemap.xml`,
+  ])];
+
+  const seen = new Map<string, UrlCandidate>();
+  let sitemapFound = false;
+  for (const sm of sitemapTargets) {
+    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher);
+    if (urls.length) sitemapFound = true;
+    const source: UrlCandidate['source'] = robotsSet.has(sm) ? 'robots' : 'sitemap';
+    for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });
+  }
+
+  return { candidates: [...seen.values()].slice(0, MAX_URLS), sitemapFound };
+}