bfsg-scanner 0.1.0

package/dist/discovery/crawl.js

@@ -0,0 +1,79 @@
+import { fetchWithUserAgent } from '../shared/user-agent.js';
+import { extractLinks, isCrawlableUrl, matchesExcludePattern } from './links.js';
+import { fetchRobotsRules, isAllowedByRobots } from './robots.js';
+/**
+ * Breadth-first fallback crawl, used when a site has no sitemap. Starts at
+ * baseUrl, follows same-origin links in document order, and stops once
+ * maxPages pages have been collected. robots.txt, the configured
+ * excludePaths, and non-page assets (mailto:, PDFs, images, ...) are all
+ * applied identically to the seed URL and to every discovered link, so a
+ * disallowed or excluded home page yields zero results rather than being
+ * scanned anyway.
+ *
+ * Deterministic: links are visited in a plain FIFO queue in the order they
+ * appear in each page's HTML, so the same input always produces the same
+ * page list in the same order.
+ */
+export async function crawlSite(options) {
+    const { baseUrl, maxPages, excludePaths = [], fetchFn = fetchWithUserAgent } = options;
+    const origin = new URL(baseUrl).origin;
+    const robotsRules = await fetchRobotsRules(origin, fetchFn);
+    const isPermitted = (url) => url.origin === origin &&
+        isCrawlableUrl(url) &&
+        !matchesExcludePattern(url.pathname, excludePaths) &&
+        isAllowedByRobots(url.pathname + url.search, robotsRules);
+    const startUrl = new URL(baseUrl);
+    startUrl.hash = '';
+    const queue = [];
+    const seen = new Set();
+    const collected = [];
+    if (isPermitted(startUrl)) {
+        const startHref = startUrl.toString();
+        queue.push(startHref);
+        seen.add(startHref);
+    }
+    while (queue.length > 0 && collected.length < maxPages) {
+        const currentUrl = queue.shift();
+        if (currentUrl === undefined) {
+            break;
+        }
+        const html = await fetchPage(fetchFn, currentUrl);
+        if (html === null) {
+            continue;
+        }
+        collected.push(currentUrl);
+        if (collected.length >= maxPages) {
+            break;
+        }
+        enqueueLinks(extractLinks(html, currentUrl), isPermitted, seen, queue);
+    }
+    return collected;
+}
+async function fetchPage(fetchFn, url) {
+    try {
+        const response = await fetchFn(url);
+        return response.ok ? await response.text() : null;
+    }
+    catch {
+        return null;
+    }
+}
+function enqueueLinks(hrefs, isPermitted, seen, queue) {
+    for (const href of hrefs) {
+        let url;
+        try {
+            url = new URL(href);
+        }
+        catch {
+            continue;
+        }
+        url.hash = '';
+        const normalized = url.toString();
+        if (seen.has(normalized) || !isPermitted(url)) {
+            continue;
+        }
+        seen.add(normalized);
+        queue.push(normalized);
+    }
+}
+//# sourceMappingURL=crawl.js.map

package/dist/discovery/crawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/discovery/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AACjF,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAUlE;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAqB;IACnD,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,GAAG,EAAE,EAAE,OAAO,GAAG,kBAAkB,EAAE,GAAG,OAAO,CAAC;IAEvF,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IACvC,MAAM,WAAW,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAE5D,MAAM,WAAW,GAAG,CAAC,GAAQ,EAAW,EAAE,CACxC,GAAG,CAAC,MAAM,KAAK,MAAM;QACrB,cAAc,CAAC,GAAG,CAAC;QACnB,CAAC,qBAAqB,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC;QAClD,iBAAiB,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAE5D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAClC,QAAQ,CAAC,IAAI,GAAG,EAAE,CAAC;IAEnB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,SAAS,GAAa,EAAE,CAAC;IAE/B,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,SAAS,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACtB,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;QACvD,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;QACjC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM;QACR,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAClD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,SAAS;QACX,CAAC;QAED,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC3B,IAAI,SAAS,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;YACjC,MAAM;QACR,CAAC;QAED,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;IACzE,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,OAAqB,EAAE,GAAW;IACzD,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,OAAO,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACpD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CACnB,KAAwB,EACxB,WAAkC,EAClC,IAAiB,EACjB,KAAe;IAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,GAAQ,CAAC;QACb,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;QACd,MAAM,UAAU,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;QAElC,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9C,SAAS;QACX,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACrB,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACzB,CAAC;AACH,CAAC"}

package/dist/discovery/discover-site.d.ts

@@ -0,0 +1,26 @@
+export interface DiscoverSiteOptions {
+    readonly baseUrl: string;
+    readonly maxPages: number;
+    /** Glob patterns matched against a URL pathname; matches are skipped. Defaults to none. */
+    readonly excludePaths?: readonly string[];
+    /** Injectable for tests; defaults to the global fetch (with User-Agent). */
+    readonly fetchFn?: typeof fetch;
+}
+/**
+ * Resolves the list of page URLs to scan for a site.
+ *
+ * Strategy: prefer the site's sitemap.xml — it is the owner's own
+ * declaration of which pages exist. If the root sitemap is missing or
+ * unparseable, fall back to a breadth-first crawl from baseUrl.
+ *
+ * Both strategies apply excludePaths and the non-page-asset filter
+ * identically. They differ on robots.txt: the crawl path obeys it (a
+ * disallowed page is never fetched, and a disallowed home page yields an
+ * empty list), while the sitemap path does not — a URL the owner put in
+ * their own sitemap is taken as intended for scanning.
+ *
+ * On the sitemap path, baseUrl is prepended when the sitemap did not list
+ * it, so the home page is always covered; the crawl path already starts
+ * from baseUrl. The result never exceeds maxPages.
+ */
+export declare function discoverSite(options: DiscoverSiteOptions): Promise<string[]>;

package/dist/discovery/discover-site.js

@@ -0,0 +1,50 @@
+import { crawlSite } from './crawl.js';
+import { discoverFromSitemap } from './discover.js';
+import { SitemapError } from './sitemap.js';
+/**
+ * Resolves the list of page URLs to scan for a site.
+ *
+ * Strategy: prefer the site's sitemap.xml — it is the owner's own
+ * declaration of which pages exist. If the root sitemap is missing or
+ * unparseable, fall back to a breadth-first crawl from baseUrl.
+ *
+ * Both strategies apply excludePaths and the non-page-asset filter
+ * identically. They differ on robots.txt: the crawl path obeys it (a
+ * disallowed page is never fetched, and a disallowed home page yields an
+ * empty list), while the sitemap path does not — a URL the owner put in
+ * their own sitemap is taken as intended for scanning.
+ *
+ * On the sitemap path, baseUrl is prepended when the sitemap did not list
+ * it, so the home page is always covered; the crawl path already starts
+ * from baseUrl. The result never exceeds maxPages.
+ */
+export async function discoverSite(options) {
+    const { baseUrl, maxPages, excludePaths = [], fetchFn } = options;
+    const shared = { baseUrl, maxPages, excludePaths, ...(fetchFn ? { fetchFn } : {}) };
+    try {
+        const pages = await discoverFromSitemap(shared);
+        return withBaseUrl(pages, baseUrl, maxPages);
+    }
+    catch (error) {
+        if (!(error instanceof SitemapError)) {
+            throw error;
+        }
+    }
+    return crawlSite(shared);
+}
+function withBaseUrl(pages, baseUrl, maxPages) {
+    const normalized = new URL(baseUrl).toString();
+    const alreadyListed = pages.some((page) => {
+        try {
+            return new URL(page).toString() === normalized;
+        }
+        catch {
+            return false;
+        }
+    });
+    if (alreadyListed) {
+        return [...pages];
+    }
+    return [normalized, ...pages].slice(0, maxPages);
+}
+//# sourceMappingURL=discover-site.js.map

package/dist/discovery/discover-site.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"discover-site.js","sourceRoot":"","sources":["../../src/discovery/discover-site.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAW5C;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC7D,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,GAAG,EAAE,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IAClE,MAAM,MAAM,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IAEpF,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAChD,OAAO,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,CAAC,KAAK,YAAY,YAAY,CAAC,EAAE,CAAC;YACrC,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC;AAC3B,CAAC;AAED,SAAS,WAAW,CAAC,KAAwB,EAAE,OAAe,EAAE,QAAgB;IAC9E,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC/C,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;QACxC,IAAI,CAAC;YACH,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,UAAU,CAAC;QACjD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,aAAa,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,CAAC,UAAU,EAAE,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;AACnD,CAAC"}

package/dist/discovery/discover.d.ts

@@ -0,0 +1,25 @@
+export interface DiscoverFromSitemapOptions {
+    readonly baseUrl: string;
+    readonly maxPages: number;
+    /** Path to the root sitemap, relative to baseUrl. Defaults to "/sitemap.xml". */
+    readonly sitemapPath?: string;
+    /** Glob patterns matched against a URL pathname; matches are skipped. Defaults to none. */
+    readonly excludePaths?: readonly string[];
+    /** Injectable for tests; defaults to the global fetch. */
+    readonly fetchFn?: typeof fetch;
+    /** Guards against pathological or cyclic sitemap indexes. Defaults to 3. */
+    readonly maxSitemapDepth?: number;
+}
+/**
+ * Discovers page URLs from a site's sitemap, following a sitemap index to
+ * its sub-sitemaps as needed. Off-origin entries are dropped (a sitemap
+ * should only ever list its own site, but nothing enforces that), non-page
+ * assets and excludePaths matches are filtered out (identically to
+ * crawlSite), duplicate URLs are collapsed, and discovery stops as soon as
+ * maxPages is reached.
+ *
+ * A failure to load the root sitemap is fatal (the caller should fall back
+ * to crawling). A failure on a sub-sitemap referenced by an index is not:
+ * that one sub-sitemap is skipped and discovery continues with the rest.
+ */
+export declare function discoverFromSitemap(options: DiscoverFromSitemapOptions): Promise<string[]>;

package/dist/discovery/discover.js

@@ -0,0 +1,81 @@
+import { fetchWithUserAgent } from '../shared/user-agent.js';
+import { isCrawlableUrl, matchesExcludePattern } from './links.js';
+import { parseSitemapXml, SitemapError } from './sitemap.js';
+/**
+ * Discovers page URLs from a site's sitemap, following a sitemap index to
+ * its sub-sitemaps as needed. Off-origin entries are dropped (a sitemap
+ * should only ever list its own site, but nothing enforces that), non-page
+ * assets and excludePaths matches are filtered out (identically to
+ * crawlSite), duplicate URLs are collapsed, and discovery stops as soon as
+ * maxPages is reached.
+ *
+ * A failure to load the root sitemap is fatal (the caller should fall back
+ * to crawling). A failure on a sub-sitemap referenced by an index is not:
+ * that one sub-sitemap is skipped and discovery continues with the rest.
+ */
+export async function discoverFromSitemap(options) {
+    const { baseUrl, maxPages, sitemapPath = '/sitemap.xml', excludePaths = [], fetchFn = fetchWithUserAgent, maxSitemapDepth = 3, } = options;
+    const origin = new URL(baseUrl).origin;
+    const rootSitemapUrl = new URL(sitemapPath, baseUrl).toString();
+    const collected = [];
+    const seenPages = new Set();
+    const visitedSitemaps = new Set();
+    await visitSitemap(rootSitemapUrl, 0);
+    return collected.slice(0, maxPages);
+    async function visitSitemap(sitemapUrl, depth) {
+        if (collected.length >= maxPages) {
+            return;
+        }
+        if (visitedSitemaps.has(sitemapUrl) || depth > maxSitemapDepth) {
+            return;
+        }
+        visitedSitemaps.add(sitemapUrl);
+        let xml;
+        try {
+            const response = await fetchFn(sitemapUrl);
+            if (!response.ok) {
+                throw new Error(`HTTP ${response.status}`);
+            }
+            xml = await response.text();
+        }
+        catch (cause) {
+            if (depth === 0) {
+                throw new SitemapError(`Cannot load sitemap at "${sitemapUrl}": ${cause instanceof Error ? cause.message : String(cause)}`);
+            }
+            return;
+        }
+        const doc = parseSitemapXml(xml);
+        if (doc.kind === 'sitemapindex') {
+            for (const childUrl of doc.sitemaps) {
+                if (collected.length >= maxPages) {
+                    return;
+                }
+                await visitSitemap(childUrl, depth + 1);
+            }
+            return;
+        }
+        for (const pageUrl of doc.urls) {
+            if (collected.length >= maxPages) {
+                return;
+            }
+            if (seenPages.has(pageUrl)) {
+                continue;
+            }
+            seenPages.add(pageUrl);
+            let parsed;
+            try {
+                parsed = new URL(pageUrl);
+            }
+            catch {
+                continue;
+            }
+            if (parsed.origin !== origin ||
+                !isCrawlableUrl(parsed) ||
+                matchesExcludePattern(parsed.pathname, excludePaths)) {
+                continue;
+            }
+            collected.push(pageUrl);
+        }
+    }
+}
+//# sourceMappingURL=discover.js.map

package/dist/discovery/discover.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"discover.js","sourceRoot":"","sources":["../../src/discovery/discover.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAe7D;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAAmC;IAC3E,MAAM,EACJ,OAAO,EACP,QAAQ,EACR,WAAW,GAAG,cAAc,EAC5B,YAAY,GAAG,EAAE,EACjB,OAAO,GAAG,kBAAkB,EAC5B,eAAe,GAAG,CAAC,GACpB,GAAG,OAAO,CAAC;IAEZ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IACvC,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;IAEhE,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAC;IAE1C,MAAM,YAAY,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IAEtC,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAEpC,KAAK,UAAU,YAAY,CAAC,UAAkB,EAAE,KAAa;QAC3D,IAAI,SAAS,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;YACjC,OAAO;QACT,CAAC;QACD,IAAI,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,KAAK,GAAG,eAAe,EAAE,CAAC;YAC/D,OAAO;QACT,CAAC;QACD,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEhC,IAAI,GAAW,CAAC;QAChB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;YAC3C,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAC7C,CAAC;YACD,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC9B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBAChB,MAAM,IAAI,YAAY,CACpB,2BAA2B,UAAU,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACpG,CAAC;YACJ,CAAC;YACD,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QAEjC,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAChC,KAAK,MAAM,QAAQ,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;gBACpC,IAAI,SAAS,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;oBACjC,OAAO;gBACT,CAAC;gBACD,MAAM,YAAY,CAAC,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YAC1C,CAAC;YACD,OAAO;QACT,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YAC/B,IAAI,SAAS,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;gBACjC,OAAO;YACT,CAAC;YACD,IAAI,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3B,SAAS;YACX,CAAC;YACD,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAEvB,IAAI,MAAW,CAAC;YAChB,IAAI,CAAC;gBACH,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;YAC5B,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YACD,IACE,MAAM,CAAC,MAAM,KAAK,MAAM;gBACxB,CAAC,cAAc,CAAC,MAAM,CAAC;gBACvB,qBAAqB,CAAC,MAAM,CAAC,QAAQ,EAAE,YAAY,CAAC,EACpD,CAAC;gBACD,SAAS;YACX,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/discovery/index.d.ts

@@ -0,0 +1,11 @@
+export type { CrawlOptions } from './crawl.js';
+export { crawlSite } from './crawl.js';
+export type { DiscoverFromSitemapOptions } from './discover.js';
+export { discoverFromSitemap } from './discover.js';
+export type { DiscoverSiteOptions } from './discover-site.js';
+export { discoverSite } from './discover-site.js';
+export { extractLinks, isCrawlableUrl, matchesExcludePattern } from './links.js';
+export type { RobotsRule } from './robots.js';
+export { fetchRobotsRules, isAllowedByRobots, parseRobotsTxt } from './robots.js';
+export type { SitemapDocument } from './sitemap.js';
+export { parseSitemapXml, SitemapError } from './sitemap.js';

package/dist/discovery/index.js

@@ -0,0 +1,7 @@
+export { crawlSite } from './crawl.js';
+export { discoverFromSitemap } from './discover.js';
+export { discoverSite } from './discover-site.js';
+export { extractLinks, isCrawlableUrl, matchesExcludePattern } from './links.js';
+export { fetchRobotsRules, isAllowedByRobots, parseRobotsTxt } from './robots.js';
+export { parseSitemapXml, SitemapError } from './sitemap.js';
+//# sourceMappingURL=index.js.map

package/dist/discovery/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/discovery/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAEpD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAEjF,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElF,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC"}

package/dist/discovery/links.d.ts

@@ -0,0 +1,11 @@
+/** Extracts every <a href> from raw HTML, resolved against pageUrl. Href
+ * values that fail to resolve (empty, malformed) are silently skipped —
+ * this is a best-effort fallback crawler, not an HTML validator. */
+export declare function extractLinks(html: string, pageUrl: string): string[];
+/** True for http(s) links that don't point at a known non-page asset.
+ * Rejects mailto:/tel:/javascript: via the protocol check. */
+export declare function isCrawlableUrl(url: URL): boolean;
+/** Simple glob matching against a URL pathname: `*` matches any sequence of
+ * characters, and a pattern with no trailing wildcard also matches
+ * everything under it (e.g. "/impressum" matches "/impressum/legal"). */
+export declare function matchesExcludePattern(pathname: string, patterns: readonly string[]): boolean;

package/dist/discovery/links.js

@@ -0,0 +1,69 @@
+const NON_PAGE_EXTENSIONS = new Set([
+    '.pdf',
+    '.jpg',
+    '.jpeg',
+    '.png',
+    '.gif',
+    '.svg',
+    '.webp',
+    '.ico',
+    '.css',
+    '.js',
+    '.mjs',
+    '.json',
+    '.xml',
+    '.zip',
+    '.mp4',
+    '.mp3',
+    '.woff',
+    '.woff2',
+    '.ttf',
+    '.eot',
+    '.doc',
+    '.docx',
+    '.xls',
+    '.xlsx',
+]);
+const HREF_PATTERN = /<a\s[^>]*?href\s*=\s*("([^"]*)"|'([^']*)')/gi;
+/** Extracts every <a href> from raw HTML, resolved against pageUrl. Href
+ * values that fail to resolve (empty, malformed) are silently skipped —
+ * this is a best-effort fallback crawler, not an HTML validator. */
+export function extractLinks(html, pageUrl) {
+    const links = [];
+    for (const match of html.matchAll(HREF_PATTERN)) {
+        const raw = match[2] ?? match[3] ?? '';
+        if (raw.length === 0) {
+            continue;
+        }
+        try {
+            links.push(new URL(raw, pageUrl).toString());
+        }
+        catch {
+            // ignore unparsable hrefs
+        }
+    }
+    return links;
+}
+/** True for http(s) links that don't point at a known non-page asset.
+ * Rejects mailto:/tel:/javascript: via the protocol check. */
+export function isCrawlableUrl(url) {
+    if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+        return false;
+    }
+    const lastSegment = url.pathname.slice(url.pathname.lastIndexOf('/') + 1);
+    const lastDot = lastSegment.lastIndexOf('.');
+    if (lastDot === -1) {
+        return true;
+    }
+    return !NON_PAGE_EXTENSIONS.has(lastSegment.slice(lastDot).toLowerCase());
+}
+/** Simple glob matching against a URL pathname: `*` matches any sequence of
+ * characters, and a pattern with no trailing wildcard also matches
+ * everything under it (e.g. "/impressum" matches "/impressum/legal"). */
+export function matchesExcludePattern(pathname, patterns) {
+    return patterns.some((pattern) => {
+        const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*');
+        return new RegExp(`^${escaped}(?:/.*)?$`).test(pathname);
+    });
+}
+//# sourceMappingURL=links.js.map

package/dist/discovery/links.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"links.js","sourceRoot":"","sources":["../../src/discovery/links.ts"],"names":[],"mappings":"AAAA,MAAM,mBAAmB,GAAwB,IAAI,GAAG,CAAC;IACvD,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,KAAK;IACL,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,QAAQ;IACR,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,OAAO;CACR,CAAC,CAAC;AAEH,MAAM,YAAY,GAAG,8CAA8C,CAAC;AAEpE;;oEAEoE;AACpE,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,OAAe;IACxD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;QAChD,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACvC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrB,SAAS;QACX,CAAC;QACD,IAAI,CAAC;YACH,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,0BAA0B;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;8DAC8D;AAC9D,MAAM,UAAU,cAAc,CAAC,GAAQ;IACrC,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,WAAW,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1E,MAAM,OAAO,GAAG,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC7C,IAAI,OAAO,KAAK,CAAC,CAAC,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,CAAC,mBAAmB,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;AAC5E,CAAC;AAED;;yEAEyE;AACzE,MAAM,UAAU,qBAAqB,CAAC,QAAgB,EAAE,QAA2B;IACjF,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;QAC/B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACnF,OAAO,IAAI,MAAM,CAAC,IAAI,OAAO,WAAW,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/discovery/robots.d.ts

@@ -0,0 +1,18 @@
+export interface RobotsRule {
+    readonly pattern: string;
+    readonly allow: boolean;
+}
+/**
+ * Parses the rule group matching `userAgent` (default "*") out of a
+ * robots.txt file. Only Allow/Disallow/User-agent are recognised — Sitemap,
+ * Crawl-delay, and anything else are ignored, since this parser exists only
+ * to decide whether a URL may be crawled.
+ */
+export declare function parseRobotsTxt(text: string, userAgent?: string): readonly RobotsRule[];
+/**
+ * Standard robots.txt precedence: the most specific (longest) matching
+ * pattern wins; an Allow breaks a tie with a Disallow of equal length.
+ * With no matching rule, the path is allowed by default.
+ */
+export declare function isAllowedByRobots(pathAndQuery: string, rules: readonly RobotsRule[]): boolean;
+export declare function fetchRobotsRules(origin: string, fetchFn?: typeof fetch): Promise<readonly RobotsRule[]>;

package/dist/discovery/robots.js

@@ -0,0 +1,91 @@
+import { fetchWithUserAgent } from '../shared/user-agent.js';
+/**
+ * Parses the rule group matching `userAgent` (default "*") out of a
+ * robots.txt file. Only Allow/Disallow/User-agent are recognised — Sitemap,
+ * Crawl-delay, and anything else are ignored, since this parser exists only
+ * to decide whether a URL may be crawled.
+ */
+export function parseRobotsTxt(text, userAgent = '*') {
+    const lines = stripBom(text).split(/\r?\n/);
+    const groups = [];
+    let current = null;
+    for (const rawLine of lines) {
+        const line = stripComment(rawLine).trim();
+        if (line.length === 0) {
+            continue;
+        }
+        const separatorIndex = line.indexOf(':');
+        if (separatorIndex === -1) {
+            continue;
+        }
+        const directive = line.slice(0, separatorIndex).trim().toLowerCase();
+        const value = line.slice(separatorIndex + 1).trim();
+        if (directive === 'user-agent') {
+            // A fresh User-agent line after rules have already been recorded
+            // starts a new group; consecutive User-agent lines with no rules yet
+            // belong to the same group (robots.txt allows grouping agents).
+            if (current === null || current.rules.length > 0) {
+                current = { agents: [], rules: [] };
+                groups.push(current);
+            }
+            current.agents.push(value.toLowerCase());
+        }
+        else if (directive === 'allow' || directive === 'disallow') {
+            if (current === null || (directive === 'disallow' && value.length === 0)) {
+                continue;
+            }
+            current.rules.push({ pattern: value, allow: directive === 'allow' });
+        }
+    }
+    const group = groups.find((g) => g.agents.includes(userAgent.toLowerCase()));
+    return group?.rules ?? [];
+}
+/**
+ * Standard robots.txt precedence: the most specific (longest) matching
+ * pattern wins; an Allow breaks a tie with a Disallow of equal length.
+ * With no matching rule, the path is allowed by default.
+ */
+export function isAllowedByRobots(pathAndQuery, rules) {
+    let best = null;
+    for (const rule of rules) {
+        if (!matchesRobotsPattern(pathAndQuery, rule.pattern)) {
+            continue;
+        }
+        if (best === null ||
+            rule.pattern.length > best.pattern.length ||
+            (rule.pattern.length === best.pattern.length && rule.allow && !best.allow)) {
+            best = rule;
+        }
+    }
+    return best === null || best.allow;
+}
+export async function fetchRobotsRules(origin, fetchFn = fetchWithUserAgent) {
+    try {
+        const response = await fetchFn(new URL('/robots.txt', origin).toString());
+        if (!response.ok) {
+            return [];
+        }
+        return parseRobotsTxt(await response.text());
+    }
+    catch {
+        return [];
+    }
+}
+function matchesRobotsPattern(pathAndQuery, pattern) {
+    if (pattern.length === 0) {
+        return false;
+    }
+    const endsWithDollar = pattern.endsWith('$');
+    const body = endsWithDollar ? pattern.slice(0, -1) : pattern;
+    const escaped = body.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*');
+    const regex = new RegExp(`^${escaped}${endsWithDollar ? '$' : ''}`);
+    return regex.test(pathAndQuery);
+}
+function stripBom(text) {
+    return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
+}
+function stripComment(line) {
+    const hashIndex = line.indexOf('#');
+    return hashIndex === -1 ? line : line.slice(0, hashIndex);
+}
+//# sourceMappingURL=robots.js.map

package/dist/discovery/robots.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../src/discovery/robots.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAO7D;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,SAAS,GAAG,GAAG;IAC1D,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAgD,EAAE,CAAC;IAC/D,IAAI,OAAO,GAAqD,IAAI,CAAC;IAErE,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,SAAS;QACX,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACzC,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;YAC1B,SAAS;QACX,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAEpD,IAAI,SAAS,KAAK,YAAY,EAAE,CAAC;YAC/B,iEAAiE;YACjE,qEAAqE;YACrE,gEAAgE;YAChE,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjD,OAAO,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;YACD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QAC3C,CAAC;aAAM,IAAI,SAAS,KAAK,OAAO,IAAI,SAAS,KAAK,UAAU,EAAE,CAAC;YAC7D,IAAI,OAAO,KAAK,IAAI,IAAI,CAAC,SAAS,KAAK,UAAU,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;gBACzE,SAAS;YACX,CAAC;YACD,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,KAAK,OAAO,EAAE,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAC7E,OAAO,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC;AAC5B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,YAAoB,EAAE,KAA4B;IAClF,IAAI,IAAI,GAAsB,IAAI,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,oBAAoB,CAAC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACtD,SAAS;QACX,CAAC;QACD,IACE,IAAI,KAAK,IAAI;YACb,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM;YACzC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAC1E,CAAC;YACD,IAAI,GAAG,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC;AACrC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAc,EACd,UAAwB,kBAAkB;IAE1C,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC1E,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,OAAO,cAAc,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,YAAoB,EAAE,OAAe;IACjE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAC7D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,IAAI,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACpE,OAAO,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;AAClC,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9D,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACpC,OAAO,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;AAC5D,CAAC"}

package/dist/discovery/sitemap.d.ts

@@ -0,0 +1,18 @@
+export declare class SitemapError extends Error {
+    constructor(message: string);
+}
+export type SitemapDocument = {
+    readonly kind: 'urlset';
+    readonly urls: readonly string[];
+} | {
+    readonly kind: 'sitemapindex';
+    readonly sitemaps: readonly string[];
+};
+/**
+ * Parses a sitemaps.org XML document (urlset or sitemapindex), extracting
+ * only each entry's own <loc>. Extensions nested under a <url> — most
+ * commonly <image:image><image:loc>...</image:loc></image:image> in
+ * WordPress-generated sitemaps — are deliberately ignored: they share the
+ * tag name "loc" but are not page URLs.
+ */
+export declare function parseSitemapXml(xml: string): SitemapDocument;

package/dist/discovery/sitemap.js

@@ -0,0 +1,36 @@
+import { XMLParser } from 'fast-xml-parser';
+export class SitemapError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'SitemapError';
+    }
+}
+const parser = new XMLParser({ ignoreAttributes: true });
+/**
+ * Parses a sitemaps.org XML document (urlset or sitemapindex), extracting
+ * only each entry's own <loc>. Extensions nested under a <url> — most
+ * commonly <image:image><image:loc>...</image:loc></image:image> in
+ * WordPress-generated sitemaps — are deliberately ignored: they share the
+ * tag name "loc" but are not page URLs.
+ */
+export function parseSitemapXml(xml) {
+    const doc = parser.parse(xml);
+    if (isRecord(doc.urlset)) {
+        return { kind: 'urlset', urls: extractLocs(doc.urlset.url) };
+    }
+    if (isRecord(doc.sitemapindex)) {
+        return { kind: 'sitemapindex', sitemaps: extractLocs(doc.sitemapindex.sitemap) };
+    }
+    throw new SitemapError('Expected a <urlset> or <sitemapindex> root element');
+}
+function extractLocs(entries) {
+    const list = Array.isArray(entries) ? entries : entries === undefined ? [] : [entries];
+    return list
+        .filter(isRecord)
+        .map((entry) => entry.loc)
+        .filter((loc) => typeof loc === 'string');
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null;
+}
+//# sourceMappingURL=sitemap.js.map

package/dist/discovery/sitemap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../src/discovery/sitemap.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,MAAM,OAAO,YAAa,SAAQ,KAAK;IACrC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAMD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;AAEzD;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC;IAEzD,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IAC/D,CAAC;IAED,IAAI,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,WAAW,CAAC,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC;IACnF,CAAC;IAED,MAAM,IAAI,YAAY,CAAC,oDAAoD,CAAC,CAAC;AAC/E,CAAC;AAED,SAAS,WAAW,CAAC,OAAgB;IACnC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACvF,OAAO,IAAI;SACR,MAAM,CAAC,QAAQ,CAAC;SAChB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC;SACzB,MAAM,CAAC,CAAC,GAAG,EAAiB,EAAE,CAAC,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC;AAC7D,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC;AACrD,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1 @@
+export declare const SCHEMA_VERSION = 1;

package/dist/index.js

@@ -0,0 +1,2 @@
+export const SCHEMA_VERSION = 1;
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC"}

package/dist/report/build.d.ts

@@ -0,0 +1,14 @@
+import type { ScanResult } from '../scan/index.js';
+import type { Report, ReportTarget, ReportToolInfo } from './types.js';
+export interface BuildReportOptions {
+    /** Overrides the generated-at timestamp; defaults to now. Injectable for tests. */
+    readonly generatedAt?: Date;
+    /** Overrides tool name/version; defaults to package.json. Injectable for tests. */
+    readonly tool?: ReportToolInfo;
+}
+/**
+ * Assembles a versioned, self-describing report from a scan result. Pure
+ * apart from the default clock and the one-time package.json read, both of
+ * which can be overridden via options.
+ */
+export declare function buildReport(scanResult: ScanResult, target: ReportTarget, options?: BuildReportOptions): Report;