@taodigital/crawler 0.0.0-stage → 1.2.1

package/src/session-export.ts

@@ -0,0 +1,172 @@
+/**
+ * Converts a Playwright storageState (the format ZeTa already captures via
+ * session-capture.ts/live-view.ts and stores per-project) into snippets for
+ * other automation frameworks. Playwright's storageState is the de-facto
+ * standard for browser session export — see docs/UNIVERSAL-SESSION-CAPTURE-
+ * STRATEGY.md — so every other framework's import mechanism is a small,
+ * mechanical conversion away, not a new capture flow.
+ *
+ * Pure, dependency-free functions — no I/O, no network, fully unit-testable.
+ */
+
+export interface StorageStateCookie {
+  name: string;
+  value: string;
+  domain: string;
+  path: string;
+  expires?: number;
+  httpOnly?: boolean;
+  secure?: boolean;
+  sameSite?: 'Strict' | 'Lax' | 'None';
+}
+
+export interface StorageStateOrigin {
+  origin: string;
+  localStorage: Array<{ name: string; value: string }>;
+}
+
+export interface StorageState {
+  cookies: StorageStateCookie[];
+  origins: StorageStateOrigin[];
+}
+
+export type SessionExportFormat = 'cypress' | 'selenium-python' | 'webdriverio';
+
+function jsStringLiteral(s: string): string {
+  return JSON.stringify(s);
+}
+
+/** Python dict-literal repr — JSON.stringify's `true`/`false`/`null` aren't
+ *  valid Python (Python needs `True`/`False`/`None`), so a plain
+ *  JSON.stringify would emit syntactically invalid Python for any cookie
+ *  with a boolean field. */
+function pythonDictLiteral(obj: Record<string, unknown>): string {
+  const parts = Object.entries(obj).map(([k, v]) => {
+    let repr: string;
+    if (typeof v === 'boolean') repr = v ? 'True' : 'False';
+    else if (v === null || v === undefined) repr = 'None';
+    else if (typeof v === 'number') repr = String(v);
+    else repr = JSON.stringify(v);
+    return `${JSON.stringify(k)}: ${repr}`;
+  });
+  return `{${parts.join(', ')}}`;
+}
+
+/** cy.setCookie() + localStorage.setItem() inside a reusable custom command,
+ *  following Cypress's documented cy.session() pattern (see references in
+ *  docs/UNIVERSAL-SESSION-CAPTURE-STRATEGY.md). */
+export function toCypressSnippet(state: StorageState): string {
+  const cookieLines = state.cookies.map((c) => {
+    const opts: string[] = [`domain: ${jsStringLiteral(c.domain)}`, `path: ${jsStringLiteral(c.path)}`];
+    if (c.secure) opts.push('secure: true');
+    if (c.httpOnly) opts.push('httpOnly: true');
+    if (c.sameSite) opts.push(`sameSite: ${jsStringLiteral(c.sameSite.toLowerCase())}`);
+    return `  cy.setCookie(${jsStringLiteral(c.name)}, ${jsStringLiteral(c.value)}, { ${opts.join(', ')} });`;
+  });
+
+  const localStorageBlocks = state.origins
+    .filter((o) => o.localStorage.length > 0)
+    .map((o) => {
+      const sets = o.localStorage
+        .map((kv) => `      window.localStorage.setItem(${jsStringLiteral(kv.name)}, ${jsStringLiteral(kv.value)});`)
+        .join('\n');
+      return `  cy.visit(${jsStringLiteral(o.origin)});\n  cy.window().then((window) => {\n${sets}\n  });`;
+    });
+
+  return [
+    '// Generated by ZeTa-AI — restores a captured browser session in Cypress.',
+    '// Usage: import this file and call restoreZetaSession() inside cy.session(), e.g.:',
+    "//   cy.session('zeta-session', restoreZetaSession)",
+    "Cypress.Commands.add('restoreZetaSession', () => {",
+    ...cookieLines,
+    ...localStorageBlocks,
+    '});',
+  ].join('\n');
+}
+
+/** driver.add_cookie() (Selenium requires the browser already be on the
+ *  cookie's domain before adding it) + execute_script() for localStorage,
+ *  per Selenium's documented Cookie API. */
+export function toSeleniumPythonDict(state: StorageState): string {
+  const byDomain = new Map<string, StorageStateCookie[]>();
+  for (const c of state.cookies) {
+    const list = byDomain.get(c.domain) ?? [];
+    list.push(c);
+    byDomain.set(c.domain, list);
+  }
+
+  const lines: string[] = [
+    '# Generated by ZeTa-AI — restores a captured browser session in Selenium.',
+    '# Selenium requires the driver to already be on a matching domain before',
+    '# add_cookie() will accept a cookie for it — navigate first, then restore.',
+    'def restore_zeta_session(driver):',
+  ];
+
+  for (const [domain, cookies] of byDomain) {
+    lines.push(`    driver.get(${JSON.stringify(`https://${domain.replace(/^\./, '')}`)})`);
+    for (const c of cookies) {
+      const cookieDict: Record<string, unknown> = { name: c.name, value: c.value, path: c.path };
+      if (c.expires && c.expires > 0) cookieDict.expiry = Math.floor(c.expires);
+      if (c.secure) cookieDict.secure = true;
+      lines.push(`    driver.add_cookie(${pythonDictLiteral(cookieDict)})`);
+    }
+  }
+
+  for (const o of state.origins) {
+    if (o.localStorage.length === 0) continue;
+    lines.push(`    driver.get(${JSON.stringify(o.origin)})`);
+    for (const kv of o.localStorage) {
+      lines.push(
+        `    driver.execute_script(${JSON.stringify(`window.localStorage.setItem(arguments[0], arguments[1]);`)}, ${JSON.stringify(kv.name)}, ${JSON.stringify(kv.value)})`,
+      );
+    }
+  }
+
+  return lines.join('\n');
+}
+
+/** browser.setCookies() accepts an array shape very close to Playwright's own
+ *  cookie objects (per WebdriverIO's documented API) — closest 1:1 mapping
+ *  of the three target formats. */
+export function toWebdriverIOSnippet(state: StorageState): string {
+  const cookies = state.cookies.map((c) => ({
+    name: c.name,
+    value: c.value,
+    domain: c.domain,
+    path: c.path,
+    secure: !!c.secure,
+    httpOnly: !!c.httpOnly,
+    ...(c.expires && c.expires > 0 ? { expiry: Math.floor(c.expires) } : {}),
+  }));
+
+  const localStorageSets = state.origins
+    .filter((o) => o.localStorage.length > 0)
+    .map((o) => {
+      const sets = o.localStorage
+        .map((kv) => `    localStorage.setItem(${jsStringLiteral(kv.name)}, ${jsStringLiteral(kv.value)});`)
+        .join('\n');
+      return `  await browser.url(${jsStringLiteral(o.origin)});\n  await browser.execute(() => {\n${sets}\n  });`;
+    });
+
+  return [
+    '// Generated by ZeTa-AI — restores a captured browser session in WebdriverIO.',
+    'async function restoreZetaSession(browser) {',
+    `  await browser.setCookies(${JSON.stringify(cookies, null, 2).split('\n').join('\n  ')});`,
+    ...localStorageSets,
+    '}',
+    'module.exports = { restoreZetaSession };',
+  ].join('\n');
+}
+
+export function exportSessionAs(format: SessionExportFormat, state: StorageState): { content: string; filename: string; mimeType: string } {
+  switch (format) {
+    case 'cypress':
+      return { content: toCypressSnippet(state), filename: 'zeta-session.cy-commands.js', mimeType: 'application/javascript' };
+    case 'selenium-python':
+      return { content: toSeleniumPythonDict(state), filename: 'zeta_session.py', mimeType: 'text/x-python' };
+    case 'webdriverio':
+      return { content: toWebdriverIOSnippet(state), filename: 'zeta-session.wdio.js', mimeType: 'application/javascript' };
+    default:
+      throw new Error(`Unknown session export format: ${format}`);
+  }
+}

package/src/shadow-dom-crawler.ts

@@ -0,0 +1,198 @@
+/**
+ * Shadow DOM crawler — enumerate shadow hosts and detect web component libraries.
+ *
+ * Walks the live DOM recursively (up to depth 8) to find all open shadow roots
+ * and custom elements with closed shadow roots, then classifies the component
+ * framework in use. Results feed ZeTa's test generator so it can account for
+ * elements that are invisible to standard Playwright locators.
+ */
+
+import type { Page } from 'playwright';
+
+export interface ShadowInteractiveEl {
+  role: string;
+  label: string;
+  type: string;
+  tagName: string;
+}
+
+export interface ShadowHostInfo {
+  tagName: string;
+  hostId: string;
+  shadowMode: 'open' | 'closed';
+  depth: number;
+  interactiveCount: number;
+  interactiveElements: ShadowInteractiveEl[];
+  slotNames: string[];
+  childShadowHosts: number;
+}
+
+export interface ShadowDomCrawlResult {
+  shadowHostCount: number;
+  openShadowCount: number;
+  closedShadowCount: number;
+  maxNestingDepth: number;
+  hosts: ShadowHostInfo[];
+  totalInteractiveInShadow: number;
+  detectedLibraries: string[];
+  blindSpots: string[];
+}
+
+interface RawShadowHost {
+  tagName: string;
+  hostId: string;
+  shadowMode: 'open' | 'closed';
+  depth: number;
+  interactiveCount: number;
+  interactiveDetails: Array<{ role: string; label: string; type: string; tagName: string }>;
+  slotNames: string[];
+  childShadowHosts: number;
+}
+
+async function enumerateShadowHosts(page: any): Promise<ShadowHostInfo[]> {
+  const raw: RawShadowHost[] = await page.evaluate(() => {
+    // esbuild injects __name() for named function declarations; polyfill prevents ReferenceError in browser.
+    if (typeof (globalThis as any).__name === 'undefined') (globalThis as any).__name = (fn: any) => fn;
+    function collectHosts(root: Document | ShadowRoot, depth: number): RawShadowHost[] {
+      if (depth > 8) return [];
+      const hosts: RawShadowHost[] = [];
+      const els = root.querySelectorAll('*');
+      for (const el of Array.from(els)) {
+        const htmlEl = el as HTMLElement & { shadowRoot: ShadowRoot | null };
+        if (htmlEl.shadowRoot) {
+          const interactive = htmlEl.shadowRoot.querySelectorAll(
+            'button,input,select,textarea,a[href],[role="button"],[role="link"],[role="checkbox"],[role="menuitem"],[role="tab"],[role="switch"]'
+          );
+          const slots = Array.from(htmlEl.shadowRoot.querySelectorAll('slot')).map(
+            (s) => (s as HTMLSlotElement).name || 'default'
+          );
+          const nestedHosts = Array.from(htmlEl.shadowRoot.querySelectorAll('*')).filter(
+            (c) => (c as HTMLElement & { shadowRoot: ShadowRoot | null }).shadowRoot
+          ).length;
+          hosts.push({
+            tagName: htmlEl.tagName.toLowerCase(),
+            hostId:
+              htmlEl.id ||
+              (htmlEl.className?.toString()?.split(' ')[0]) ||
+              '',
+            shadowMode: 'open',
+            depth,
+            interactiveCount: interactive.length,
+            interactiveDetails: Array.from(interactive)
+              .slice(0, 8)
+              .map((i) => {
+                const iEl = i as HTMLElement;
+                return {
+                  role: iEl.getAttribute('role') || iEl.tagName.toLowerCase(),
+                  label: (
+                    iEl.getAttribute('aria-label') ||
+                    iEl.getAttribute('placeholder') ||
+                    iEl.textContent?.trim() ||
+                    ''
+                  ).slice(0, 40),
+                  type: iEl.getAttribute('type') || '',
+                  tagName: iEl.tagName.toLowerCase(),
+                };
+              }),
+            slotNames: slots,
+            childShadowHosts: nestedHosts,
+          });
+          const nested = collectHosts(htmlEl.shadowRoot, depth + 1);
+          hosts.push(...nested);
+        } else if (el.tagName.includes('-') && !(el as HTMLElement & { shadowRoot: ShadowRoot | null }).shadowRoot) {
+          hosts.push({
+            tagName: el.tagName.toLowerCase(),
+            hostId: (el as HTMLElement).id || '',
+            shadowMode: 'closed',
+            depth,
+            interactiveCount: 0,
+            interactiveDetails: [],
+            slotNames: [],
+            childShadowHosts: 0,
+          });
+        }
+      }
+      return hosts;
+    }
+    return collectHosts(document, 0);
+  });
+
+  return raw.slice(0, 100).map((h) => ({
+    tagName: h.tagName,
+    hostId: h.hostId,
+    shadowMode: h.shadowMode,
+    depth: h.depth,
+    interactiveCount: h.interactiveCount,
+    interactiveElements: h.interactiveDetails,
+    slotNames: h.slotNames,
+    childShadowHosts: h.childShadowHosts,
+  }));
+}
+
+async function detectWebComponentLibrary(page: any): Promise<string[]> {
+  return page.evaluate(() => {
+    const checks: [string, string][] = [
+      ['sl-button', 'Shoelace'],
+      ['fast-button', 'Microsoft FAST'],
+      ['mwc-button', 'Material Web Components'],
+      ['ion-button', 'Ionic'],
+      ['vaadin-button', 'Vaadin'],
+      ['fluent-button', 'Fluent UI Web Components'],
+      ['hy-icon', 'Hydra'],
+    ];
+    const detected: string[] = [];
+    for (const [tag, name] of checks) {
+      if (customElements.get(tag)) detected.push(name);
+    }
+    const allTags = Array.from(document.querySelectorAll('*'))
+      .map((e) => e.tagName.toLowerCase())
+      .filter((t) => t.includes('-'));
+    const uniqueTags = Array.from(new Set(allTags));
+    if (uniqueTags.some((t) => t.startsWith('sl-')) && !detected.includes('Shoelace'))
+      detected.push('Shoelace');
+    if (uniqueTags.some((t) => t.startsWith('fast-')) && !detected.includes('Microsoft FAST'))
+      detected.push('Microsoft FAST');
+    if (uniqueTags.some((t) => t.startsWith('ion-')) && !detected.includes('Ionic'))
+      detected.push('Ionic');
+    return detected;
+  });
+}
+
+export async function crawlShadowDom(page: any): Promise<ShadowDomCrawlResult> {
+  const empty: ShadowDomCrawlResult = {
+    shadowHostCount: 0,
+    openShadowCount: 0,
+    closedShadowCount: 0,
+    maxNestingDepth: 0,
+    hosts: [],
+    totalInteractiveInShadow: 0,
+    detectedLibraries: [],
+    blindSpots: [],
+  };
+
+  try {
+    const [hosts, detectedLibraries] = await Promise.all([
+      enumerateShadowHosts(page),
+      detectWebComponentLibrary(page),
+    ]);
+
+    const openHosts = hosts.filter((h) => h.shadowMode === 'open');
+    const closedHosts = hosts.filter((h) => h.shadowMode === 'closed');
+    const maxNestingDepth = hosts.reduce((max, h) => Math.max(max, h.depth), 0);
+    const totalInteractiveInShadow = openHosts.reduce((sum, h) => sum + h.interactiveCount, 0);
+    const blindSpots = [...new Set(closedHosts.map((h) => h.tagName))];
+
+    return {
+      shadowHostCount: hosts.length,
+      openShadowCount: openHosts.length,
+      closedShadowCount: closedHosts.length,
+      maxNestingDepth,
+      hosts,
+      totalInteractiveInShadow,
+      detectedLibraries,
+      blindSpots,
+    };
+  } catch {
+    return empty;
+  }
+}

package/src/sitemap-writer.ts

@@ -0,0 +1,65 @@
+/**
+ * Sitemap writer — generate XML sitemap from crawled screens.
+ *
+ * Compliant with sitemaps.org protocol (sitemap index + individual sitemap files).
+ * Splits into 50k-URL files per the spec when needed.
+ * Used for: SEO audit export, future crawl seed, and crawler/robots.txt comparison.
+ */
+
+import * as fs from 'fs/promises';
+import * as path from 'path';
+
+export interface SitemapUrl {
+  loc: string;
+  lastmod?: string;
+  changefreq?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
+  priority?: number; // 0.0–1.0
+}
+
+const XML_HEADER = '<?xml version="1.0" encoding="UTF-8"?>\n';
+const SITEMAP_NS = 'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"';
+
+function escape(s: string): string {
+  return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;').replace(/'/g, '&apos;');
+}
+
+function buildSitemapXml(urls: SitemapUrl[]): string {
+  const entries = urls.map(u => [
+    '  <url>',
+    `    <loc>${escape(u.loc)}</loc>`,
+    u.lastmod ? `    <lastmod>${u.lastmod}</lastmod>` : '',
+    u.changefreq ? `    <changefreq>${u.changefreq}</changefreq>` : '',
+    u.priority !== undefined ? `    <priority>${u.priority.toFixed(1)}</priority>` : '',
+    '  </url>',
+  ].filter(Boolean).join('\n')).join('\n');
+
+  return `${XML_HEADER}<urlset ${SITEMAP_NS}>\n${entries}\n</urlset>`;
+}
+
+export async function writeSitemap(
+  urls: SitemapUrl[],
+  outputPath: string,
+): Promise<void> {
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+  await fs.writeFile(outputPath, buildSitemapXml(urls), 'utf8');
+}
+
+/** Convert crawled screen records to SitemapUrl objects */
+export function screensToSitemapUrls(
+  screens: Array<{ url: string; updatedAt?: Date; depth?: number }>,
+): SitemapUrl[] {
+  return screens
+    .filter(s => {
+      try { return new URL(s.url).protocol.startsWith('http'); } catch { return false; }
+    })
+    .map(s => ({
+      loc: s.url,
+      lastmod: s.updatedAt ? s.updatedAt.toISOString().split('T')[0] : undefined,
+      changefreq: 'weekly' as const,
+      priority: s.depth === 0 ? 1.0 : s.depth === 1 ? 0.8 : 0.6,
+    }));
+}
+
+export function buildSitemapString(urls: SitemapUrl[]): string {
+  return buildSitemapXml(urls);
+}

package/src/sitemap.test.ts

@@ -0,0 +1,127 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import http from 'node:http';
+import os from 'node:os';
+import { MockAgent } from 'undici';
+import { XMLParser } from 'fast-xml-parser';
+import { extractUrlsFromParsedSitemap, fetchSitemapUrls, discoverSitemapUrls } from './sitemap.js';
+
+const parser = new XMLParser();
+
+test('extractUrlsFromParsedSitemap: parses a plain <urlset> sitemap', () => {
+  const xml = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url><loc>https://example.com/</loc></url>
+  <url><loc>https://example.com/pricing</loc></url>
+  <url><loc>https://example.com/about</loc></url>
+</urlset>`;
+  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parser.parse(xml), 'https://example.com');
+  assert.deepEqual(pageUrls.sort(), ['https://example.com/', 'https://example.com/about', 'https://example.com/pricing'].sort());
+  assert.deepEqual(subSitemapUrls, []);
+});
+
+test('extractUrlsFromParsedSitemap: parses a <sitemapindex> into sub-sitemap URLs', () => {
+  const xml = `<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <sitemap><loc>https://example.com/sitemap-pages.xml</loc></sitemap>
+  <sitemap><loc>https://example.com/sitemap-blog.xml</loc></sitemap>
+</sitemapindex>`;
+  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parser.parse(xml), 'https://example.com');
+  assert.deepEqual(pageUrls, []);
+  assert.deepEqual(subSitemapUrls, ['https://example.com/sitemap-pages.xml', 'https://example.com/sitemap-blog.xml']);
+});
+
+test('extractUrlsFromParsedSitemap: filters out cross-origin URLs', () => {
+  const xml = `<urlset>
+  <url><loc>https://example.com/page</loc></url>
+  <url><loc>https://attacker.example/evil</loc></url>
+</urlset>`;
+  const { pageUrls } = extractUrlsFromParsedSitemap(parser.parse(xml), 'https://example.com');
+  assert.deepEqual(pageUrls, ['https://example.com/page']);
+});
+
+test('extractUrlsFromParsedSitemap: dedupes repeated URLs', () => {
+  const xml = `<urlset>
+  <url><loc>https://example.com/a</loc></url>
+  <url><loc>https://example.com/a</loc></url>
+</urlset>`;
+  const { pageUrls } = extractUrlsFromParsedSitemap(parser.parse(xml), 'https://example.com');
+  assert.deepEqual(pageUrls, ['https://example.com/a']);
+});
+
+test('extractUrlsFromParsedSitemap: a single <url> entry is not an array (fast-xml-parser quirk) — still handled', () => {
+  const xml = `<urlset><url><loc>https://example.com/only</loc></url></urlset>`;
+  const { pageUrls } = extractUrlsFromParsedSitemap(parser.parse(xml), 'https://example.com');
+  assert.deepEqual(pageUrls, ['https://example.com/only']);
+});
+
+test('extractUrlsFromParsedSitemap: handles empty/malformed input without throwing', () => {
+  assert.deepEqual(extractUrlsFromParsedSitemap({}, 'https://example.com'), { pageUrls: [], subSitemapUrls: [] });
+  assert.deepEqual(extractUrlsFromParsedSitemap(null, 'https://example.com'), { pageUrls: [], subSitemapUrls: [] });
+  assert.deepEqual(extractUrlsFromParsedSitemap({ urlset: { url: [{ notLoc: 'x' }] } }, 'https://example.com'), { pageUrls: [], subSitemapUrls: [] });
+});
+
+// --- End-to-end fetch path: real fetch, real XML parse, mocked socket layer ---
+// Bound to the sandbox's actual (non-loopback) address — assertCrawlUrlSafe
+// deliberately blocks literal localhost/127.0.0.1 (meaningless as a crawl
+// target, would just hit the crawler container itself) but allows other
+// addresses, matching how a real deployment reaches a tenant's internal app.
+function getNonLoopbackAddress(): string {
+  for (const ifaces of Object.values(os.networkInterfaces())) {
+    for (const iface of ifaces as any[]) {
+      if (iface.family === 'IPv4' && !iface.internal) return iface.address;
+    }
+  }
+  throw new Error('No non-loopback IPv4 address found on this host — cannot run the live sitemap test');
+}
+
+test('fetchSitemapUrls: end-to-end against a sitemap index response', async () => {
+  const origin = 'http://example.test';
+  const indexXml = `<?xml version="1.0"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <sitemap><loc>${origin}/sub1.xml</loc></sitemap>
+  <sitemap><loc>${origin}/sub2.xml</loc></sitemap>
+</sitemapindex>`;
+  const sub1Xml = `<urlset><url><loc>${origin}/page1</loc></url><url><loc>${origin}/page2</loc></url></urlset>`;
+  const sub2Xml = `<urlset><url><loc>${origin}/page3</loc></url></urlset>`;
+  const dispatcher = new MockAgent();
+  dispatcher.disableNetConnect();
+  const pool = dispatcher.get(origin);
+  pool.intercept({ path: '/sitemap.xml', method: 'GET' }).reply(200, indexXml);
+  pool.intercept({ path: '/sub1.xml', method: 'GET' }).reply(200, sub1Xml);
+  pool.intercept({ path: '/sub2.xml', method: 'GET' }).reply(200, sub2Xml);
+
+  try {
+    const urls = await fetchSitemapUrls(`${origin}/sitemap.xml`, origin, 0, dispatcher);
+    assert.deepEqual(
+      urls.sort(),
+      [`${origin}/page1`, `${origin}/page2`, `${origin}/page3`].sort(),
+    );
+  } finally {
+    await dispatcher.close();
+  }
+});
+
+test('discoverSitemapUrls: defaults to /sitemap.xml at the app origin and returns [] gracefully on 404', async () => {
+  const host = getNonLoopbackAddress();
+  const server = http.createServer((_req, res) => { res.writeHead(404); res.end(); });
+  await new Promise<void>((resolve) => server.listen(0, host, resolve));
+  const port = (server.address() as any).port;
+
+  try {
+    const urls = await discoverSitemapUrls(`http://${host}:${port}/app`);
+    assert.deepEqual(urls, []);
+  } finally {
+    server.close();
+  }
+});
+
+test('discoverSitemapUrls: never throws for an invalid appUrl', async () => {
+  const urls = await discoverSitemapUrls('not-a-valid-url');
+  assert.deepEqual(urls, []);
+});
+
+test('discoverSitemapUrls: is blocked (returns []) for a disallowed target like localhost — assertCrawlUrlSafe still applies', async () => {
+  const urls = await discoverSitemapUrls('http://localhost:1/app');
+  assert.deepEqual(urls, []);
+});