@eightmedia/flightcheck 1.0.1 → 1.2.0

package/dist/crawler/defaults.js

@@ -1,4 +1,5 @@
-export const DEFAULT_MAX_PAGES = 1000;
+/** Default crawl budget; smart selection prefers root/list pages within this cap. */
+export const DEFAULT_MAX_PAGES = 100;
 export const DEFAULT_DELAY_MS = 500;
 export const DEFAULT_TIMEOUT_MS = 10000;
 export const DEFAULT_CONCURRENCY = 1;

package/dist/crawler/runCrawler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runCrawler.d.ts","sourceRoot":"","sources":["../../src/crawler/runCrawler.ts"],"names":[],"mappings":"AAUA,OAAO,EAAgB,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAOjE,OAAO,KAAK,EAAiB,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAKrE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,YAAY,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IAC/B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,wBAAsB,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,kBAAkB,EAAE,CAAC;IAAC,WAAW,EAAE,OAAO,CAAA;CAAE,CAAC,CAqK7H"}
+{"version":3,"file":"runCrawler.d.ts","sourceRoot":"","sources":["../../src/crawler/runCrawler.ts"],"names":[],"mappings":"AAUA,OAAO,EAAgB,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAQjE,OAAO,KAAK,EAAiB,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAMrE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,YAAY,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IAC/B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,wBAAsB,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,kBAAkB,EAAE,CAAC;IAAC,WAAW,EAAE,OAAO,CAAA;CAAE,CAAC,CAkM7H"}

package/dist/crawler/runCrawler.js

@@ -1,13 +1,15 @@
-import { CheerioCrawler, RequestQueue, log, LogLevel } from "crawlee";
+import { CheerioCrawler, Configuration, MemoryStorage, RequestQueue, log, LogLevel } from "crawlee";
 import pino from "pino";
 import { DEFAULT_BURST_DELAY_MS, DEFAULT_CONCURRENCY, DEFAULT_DELAY_MS, DEFAULT_MAX_PAGES, DEFAULT_TIMEOUT_MS, DEFAULT_USER_AGENT, } from "./defaults.js";
 import { CHECK_GROUPS } from "./checkGroups.js";
+import { cleanupRunStorageDir, createRunStoragePath } from "./crawleeStorage.js";
 import { extractInternalLinks, extractVerificationRecord } from "./extractors.js";
 import { detectStackFromHtml } from "./stack.js";
 import { extractSameOriginScripts } from "./jsAssets.js";
 import { loadRobotsPolicy } from "./robots.js";
 import { writeReportToFile } from "./reporter.js";
 import { scanText } from "./secrets.js";
+import { createDiscoveryGate } from "./smartPageSelection.js";
 import { matchesExcludePattern, normalizeUrl, sleep } from "./urlUtils.js";
 const MAX_JS_ASSET_BYTES = 5_000_000; // 5 MB — skip larger bundles to bound memory
 export async function runCrawler(options) {
@@ -81,92 +83,118 @@ export async function runCrawler(options) {
         return { secrets, scannedAssets };
     };
     const robotsPolicy = respectRobotsTxt ? await loadRobotsPolicy(origin, timeoutMs) : null;
-    const requestQueue = await RequestQueue.open();
-    const records = [];
-    const seen = new Set();
-    for (const seed of seedUrls) {
-        if (!matchesExcludePattern(seed, excludePatterns)) {
-            await requestQueue.addRequest({ url: seed, uniqueKey: normalizeUrl(seed) });
-            seen.add(normalizeUrl(seed));
-        }
-    }
-    let processedSinceBurst = 0;
-    let interrupted = false;
-    const persist = () => {
-        writeReportToFile(records, options.outputPath);
-        return records;
-    };
-    process.on("SIGINT", async () => {
-        interrupted = true;
-        logger.info("[crawl] SIGINT ontvangen, graceful stop gestart");
+    // Per-run Crawlee storage so concurrent crawls do not share/purge each other,
+    // then remove this run's subdirectory when finished.
+    const { runDir, runId } = createRunStoragePath({ seed: startUrl });
+    const configuration = new Configuration({
+        storageClient: new MemoryStorage({ localDataDirectory: runDir }),
+        defaultRequestQueueId: runId,
+        purgeOnStart: true,
     });
-    logger.info(`[crawl] start url=${startUrl} output=${options.outputPath} maxPages=${maxPages} concurrency=${concurrency} delayMs=${delayMs} burstDelayMs=${burstDelayMs} timeoutMs=${timeoutMs} seeds=${seedUrls.length} respectRobotsTxt=${respectRobotsTxt}`);
-    const crawler = new CheerioCrawler({
-        requestQueue,
-        maxRequestsPerCrawl: maxPages,
-        maxConcurrency: concurrency,
-        // +60 s headroom when security scanning is active: the handler does sequential JS fetches
-        // per page; the failedAssets/jsScanCache ensures dead/cached assets aren't re-fetched.
-        requestHandlerTimeoutSecs: Math.ceil(timeoutMs / 1000) + (activeGroups.has("security") ? 60 : 0),
-        requestHandler: async ({ request, body, contentType, response }) => {
-            if (interrupted)
-                return;
-            if (delayMs > 0) {
-                await sleep(delayMs);
+    try {
+        const requestQueue = await RequestQueue.open(runId, { config: configuration });
+        const records = [];
+        const seen = new Set();
+        const discovery = createDiscoveryGate(seedUrls, maxPages);
+        // Seeds are already smart-ordered (home/root/list before details); FIFO preserves that.
+        for (const seed of seedUrls) {
+            if (!matchesExcludePattern(seed, excludePatterns)) {
+                const normalized = normalizeUrl(seed);
+                await requestQueue.addRequest({ url: seed, uniqueKey: normalized });
+                seen.add(normalized);
+                discovery.remember(normalized);
             }
-            if (burstDelayMs > 0 && processedSinceBurst > 0 && processedSinceBurst % concurrency === 0) {
-                await sleep(burstDelayMs);
-            }
-            const currentUrl = normalizeUrl(request.loadedUrl ?? request.url);
-            if (robotsPolicy && !robotsPolicy.isAllowed(currentUrl)) {
-                const skipped = extractVerificationRecord("", currentUrl, null, "");
-                skipped.note = "Skipped by robots.txt policy";
-                records.push(skipped);
+        }
+        let processedSinceBurst = 0;
+        let interrupted = false;
+        const persist = () => {
+            writeReportToFile(records, options.outputPath);
+            return records;
+        };
+        process.on("SIGINT", async () => {
+            interrupted = true;
+            logger.info("[crawl] SIGINT ontvangen, graceful stop gestart");
+        });
+        logger.info(`[crawl] start url=${startUrl} output=${options.outputPath} maxPages=${maxPages} concurrency=${concurrency} delayMs=${delayMs} burstDelayMs=${burstDelayMs} timeoutMs=${timeoutMs} seeds=${seedUrls.length} respectRobotsTxt=${respectRobotsTxt} storage=${runDir}`);
+        const crawler = new CheerioCrawler({
+            requestQueue,
+            maxRequestsPerCrawl: maxPages,
+            maxConcurrency: concurrency,
+            // +60 s headroom when security scanning is active: the handler does sequential JS fetches
+            // per page; the failedAssets/jsScanCache ensures dead/cached assets aren't re-fetched.
+            requestHandlerTimeoutSecs: Math.ceil(timeoutMs / 1000) + (activeGroups.has("security") ? 60 : 0),
+            requestHandler: async ({ request, body, contentType, response }) => {
+                if (interrupted)
+                    return;
+                if (delayMs > 0) {
+                    await sleep(delayMs);
+                }
+                if (burstDelayMs > 0 && processedSinceBurst > 0 && processedSinceBurst % concurrency === 0) {
+                    await sleep(burstDelayMs);
+                }
+                const currentUrl = normalizeUrl(request.loadedUrl ?? request.url);
+                if (robotsPolicy && !robotsPolicy.isAllowed(currentUrl)) {
+                    const skipped = extractVerificationRecord("", currentUrl, null, "");
+                    skipped.note = "Skipped by robots.txt policy";
+                    records.push(skipped);
+                    const currentRecords = persist();
+                    processedSinceBurst += 1;
+                    logger.info(`[crawl] processed=${currentRecords.length} url=${currentUrl} status=robots-blocked`);
+                    return;
+                }
+                const html = typeof body === "string" ? body : "";
+                const status = response?.statusCode ?? null;
+                const effectiveContentType = contentType?.type ?? "";
+                const record = extractVerificationRecord(html, currentUrl, status, effectiveContentType, activeGroups);
+                if (activeGroups.has("stack") && effectiveContentType.includes("text/html") && html) {
+                    record.stack = { detections: detectStackFromHtml(html, currentUrl) };
+                }
+                if (activeGroups.has("security") && effectiveContentType.includes("text/html") && html) {
+                    record.security = await scanSecurity(html, currentUrl);
+                }
+                records.push(record);
                 const currentRecords = persist();
                 processedSinceBurst += 1;
-                logger.info(`[crawl] processed=${currentRecords.length} url=${currentUrl} status=robots-blocked`);
-                return;
-            }
-            const html = typeof body === "string" ? body : "";
-            const status = response?.statusCode ?? null;
-            const effectiveContentType = contentType?.type ?? "";
-            const record = extractVerificationRecord(html, currentUrl, status, effectiveContentType, activeGroups);
-            if (activeGroups.has("stack") && effectiveContentType.includes("text/html") && html) {
-                record.stack = { detections: detectStackFromHtml(html, currentUrl) };
-            }
-            if (activeGroups.has("security") && effectiveContentType.includes("text/html") && html) {
-                record.security = await scanSecurity(html, currentUrl);
-            }
-            records.push(record);
-            const currentRecords = persist();
-            processedSinceBurst += 1;
-            logger.info(`[crawl] processed=${currentRecords.length} url=${currentUrl} status=${status ?? "error"}`);
-            if (effectiveContentType.includes("text/html")) {
-                const links = extractInternalLinks(html, currentUrl, origin);
-                for (const link of links) {
-                    const normalized = normalizeUrl(link);
-                    if (seen.has(normalized))
-                        continue;
-                    if (matchesExcludePattern(normalized, excludePatterns))
-                        continue;
-                    if (robotsPolicy && !robotsPolicy.isAllowed(normalized))
-                        continue;
-                    seen.add(normalized);
-                    await requestQueue.addRequest({ url: normalized, uniqueKey: normalized });
+                logger.info(`[crawl] processed=${currentRecords.length} url=${currentUrl} status=${status ?? "error"}`);
+                if (effectiveContentType.includes("text/html")) {
+                    const links = extractInternalLinks(html, currentUrl, origin);
+                    for (const link of links) {
+                        const normalized = normalizeUrl(link);
+                        if (seen.has(normalized))
+                            continue;
+                        if (matchesExcludePattern(normalized, excludePatterns))
+                            continue;
+                        if (robotsPolicy && !robotsPolicy.isAllowed(normalized))
+                            continue;
+                        if (!discovery.allowEnqueue(normalized))
+                            continue;
+                        seen.add(normalized);
+                        discovery.remember(normalized);
+                        const forefront = discovery.isStructural(normalized);
+                        await requestQueue.addRequest({ url: normalized, uniqueKey: normalized }, { forefront });
+                    }
                 }
-            }
-        },
-        failedRequestHandler: async ({ request }, error) => {
-            const currentUrl = normalizeUrl(request.loadedUrl ?? request.url);
-            const record = extractVerificationRecord("", currentUrl, null, "");
-            record.error = error instanceof Error ? error.message : "Unknown error";
-            records.push(record);
-            const currentRecords = persist();
-            logger.info(`[crawl] processed=${currentRecords.length} url=${currentUrl} status=error`);
-        },
-    });
-    await crawler.run();
-    const finalRecords = persist();
-    logger.info(`[crawl] klaar pages=${finalRecords.length} interrupted=${interrupted}`);
-    return { records: finalRecords, interrupted };
+            },
+            failedRequestHandler: async ({ request }, error) => {
+                const currentUrl = normalizeUrl(request.loadedUrl ?? request.url);
+                const record = extractVerificationRecord("", currentUrl, null, "");
+                record.error = error instanceof Error ? error.message : "Unknown error";
+                records.push(record);
+                const currentRecords = persist();
+                logger.info(`[crawl] processed=${currentRecords.length} url=${currentUrl} status=error`);
+            },
+        }, configuration);
+        await crawler.run();
+        const finalRecords = persist();
+        logger.info(`[crawl] klaar pages=${finalRecords.length} interrupted=${interrupted}`);
+        return { records: finalRecords, interrupted };
+    }
+    finally {
+        try {
+            await cleanupRunStorageDir(runDir);
+        }
+        catch (error) {
+            logger.warn(`[crawl] kon Crawlee storage niet opruimen path=${runDir} error=${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
 }

package/dist/crawler/smartPageSelection.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Smart crawl URL selection based on URL-tree shape (not slug names).
+ *
+ * A "list" parent is a path with many direct children that are mostly leaves
+ * (no further subpages) — typical for archives like news/events/blog, whatever
+ * they are named. Within --max-pages we keep structural pages and sample
+ * leaves under each list parent.
+ */
+export type UrlKind = "home" | "root" | "list" | "detail" | "other";
+export type ClassifiedUrl = {
+    url: string;
+    kind: UrlKind;
+    /** Pathname of the list parent when kind is list/detail, e.g. "/stories". */
+    section: string | null;
+    /** Sort priority (higher = sooner). */
+    priority: number;
+};
+export type UrlTreeAnalysis = {
+    /** Pathnames that look like list indexes (many leaf children). */
+    listParents: ReadonlySet<string>;
+    /** Origin used to rebuild URLs from pathnames when needed. */
+    origin: string;
+};
+/** Minimum direct leaf children before a path is treated as a list parent. */
+export declare const DEFAULT_LIST_MIN_CHILDREN = 8;
+/** Share of direct children that must be leaves to count as a list. */
+export declare const DEFAULT_LIST_LEAF_RATIO = 0.7;
+export declare function pathSegments(urlOrPath: string): string[];
+export declare function pathnameOf(urlString: string): string;
+/** Parent pathname: `/a/b` → `/a`, `/a` → `/`, `/` → null. */
+export declare function parentPathname(pathname: string): string | null;
+/**
+ * Build parent → direct children maps from a URL set and detect list parents:
+ * paths with many children that are mostly leaves (no further children in the set).
+ */
+export declare function analyzeUrlTree(urls: string[], options?: {
+    startUrl?: string;
+    listMinChildren?: number;
+    listLeafRatio?: number;
+}): UrlTreeAnalysis;
+/** Closest list-parent pathname for this path, or null. */
+export declare function listParentForPathname(pathname: string, listParents: ReadonlySet<string>): string | null;
+export declare function classifyCrawlUrl(urlString: string, tree: UrlTreeAnalysis): ClassifiedUrl;
+export declare function crawlRequestPriority(urlString: string, tree: UrlTreeAnalysis): number;
+/**
+ * Soft per-list-parent cap for detail URLs discovered during a crawl.
+ * Structural pages (home/root/list) are never capped this way.
+ */
+export declare function detailCapPerSection(maxPages: number, listParentCount?: number): number;
+/**
+ * Pick up to `maxPages` URLs using URL-tree shape: keep home/root/list pages,
+ * then round-robin a sample of leaf details under each detected list parent.
+ */
+export declare function selectSmartCrawlUrls(urls: string[], maxPages: number, options?: {
+    startUrl?: string;
+    listMinChildren?: number;
+    listLeafRatio?: number;
+}): {
+    urls: string[];
+    tree: UrlTreeAnalysis;
+};
+/** Sort URLs by crawl priority (desc), stable for equal priority. */
+export declare function sortByCrawlPriority(urls: string[], tree: UrlTreeAnalysis): string[];
+/**
+ * Tracks list parents during a live crawl: uses seed analysis up front, and
+ * promotes parents to "list" when enough leaf-like children are observed.
+ */
+export declare function createDiscoveryGate(seedUrls: string[], maxPages: number, options?: {
+    listMinChildren?: number;
+    listLeafRatio?: number;
+}): {
+    tree: UrlTreeAnalysis;
+    allowEnqueue: (url: string) => boolean;
+    remember: (url: string) => void;
+    isStructural: (url: string) => boolean;
+};
+//# sourceMappingURL=smartPageSelection.d.ts.map

package/dist/crawler/smartPageSelection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"smartPageSelection.d.ts","sourceRoot":"","sources":["../../src/crawler/smartPageSelection.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,MAAM,MAAM,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;AAEpE,MAAM,MAAM,aAAa,GAAG;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,OAAO,CAAC;IACd,6EAA6E;IAC7E,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,uCAAuC;IACvC,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,kEAAkE;IAClE,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACjC,8DAA8D;IAC9D,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,8EAA8E;AAC9E,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAE3C,uEAAuE;AACvE,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAU3C,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAOxD;AAED,wBAAgB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAQpD;AAED,8DAA8D;AAC9D,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAK9D;AA2BD;;;GAGG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,CAAC,EAAE;IACR,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GACA,eAAe,CA2CjB;AAED,2DAA2D;AAC3D,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,GAC/B,MAAM,GAAG,IAAI,CAOf;AAED,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,GAAG,aAAa,CAsBxF;AAED,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,GAAG,MAAM,CAErF;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,eAAe,SAAI,GAAG,MAAM,CAIjF;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,MAAM,EAAE,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;IACR,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GACA;IAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAAC,IAAI,EAAE,eAAe,CAAA;CAAE,CAmE3C;AAED,qEAAqE;AACrE,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,eAAe,GAAG,MAAM,EAAE,CAKnF;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,MAAM,EAAE,EAClB,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;IAAE,eAAe,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAAE,GAC7D;IACD,IAAI,EAAE,eAAe,CAAC;IACtB,YAAY,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;IACvC,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,YAAY,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;CACxC,CAqEA"}

package/dist/crawler/smartPageSelection.js

@@ -0,0 +1,308 @@
+/**
+ * Smart crawl URL selection based on URL-tree shape (not slug names).
+ *
+ * A "list" parent is a path with many direct children that are mostly leaves
+ * (no further subpages) — typical for archives like news/events/blog, whatever
+ * they are named. Within --max-pages we keep structural pages and sample
+ * leaves under each list parent.
+ */
+/** Minimum direct leaf children before a path is treated as a list parent. */
+export const DEFAULT_LIST_MIN_CHILDREN = 8;
+/** Share of direct children that must be leaves to count as a list. */
+export const DEFAULT_LIST_LEAF_RATIO = 0.7;
+const PRIORITY = {
+    home: 100,
+    root: 80,
+    list: 60,
+    detail: 40,
+    other: 20,
+};
+export function pathSegments(urlOrPath) {
+    try {
+        const pathname = urlOrPath.includes("://") ? new URL(urlOrPath).pathname : urlOrPath;
+        return pathname.split("/").filter(Boolean);
+    }
+    catch {
+        return [];
+    }
+}
+export function pathnameOf(urlString) {
+    try {
+        const pathname = new URL(urlString).pathname;
+        if (!pathname || pathname === "")
+            return "/";
+        return pathname.length > 1 && pathname.endsWith("/") ? pathname.slice(0, -1) : pathname;
+    }
+    catch {
+        return "/";
+    }
+}
+/** Parent pathname: `/a/b` → `/a`, `/a` → `/`, `/` → null. */
+export function parentPathname(pathname) {
+    if (pathname === "/" || pathname === "")
+        return null;
+    const segments = pathname.split("/").filter(Boolean);
+    if (segments.length <= 1)
+        return "/";
+    return `/${segments.slice(0, -1).join("/")}`;
+}
+function originOf(urls, startUrl) {
+    const candidate = startUrl ?? urls[0];
+    if (!candidate)
+        return "https://example.com";
+    try {
+        return new URL(candidate).origin;
+    }
+    catch {
+        return "https://example.com";
+    }
+}
+function uniqueUrls(urls, startUrl) {
+    const seen = new Set();
+    const unique = [];
+    if (startUrl) {
+        unique.push(startUrl);
+        seen.add(startUrl);
+    }
+    for (const url of urls) {
+        if (seen.has(url))
+            continue;
+        seen.add(url);
+        unique.push(url);
+    }
+    return unique;
+}
+/**
+ * Build parent → direct children maps from a URL set and detect list parents:
+ * paths with many children that are mostly leaves (no further children in the set).
+ */
+export function analyzeUrlTree(urls, options) {
+    const minChildren = options?.listMinChildren ?? DEFAULT_LIST_MIN_CHILDREN;
+    const leafRatio = options?.listLeafRatio ?? DEFAULT_LIST_LEAF_RATIO;
+    const unique = uniqueUrls(urls, options?.startUrl);
+    const origin = originOf(unique, options?.startUrl);
+    const childrenOf = new Map();
+    const pathnames = new Set();
+    for (const url of unique) {
+        const pathname = pathnameOf(url);
+        pathnames.add(pathname);
+        let child = pathname;
+        let parent = parentPathname(child);
+        while (parent !== null) {
+            let bucket = childrenOf.get(parent);
+            if (!bucket) {
+                bucket = new Set();
+                childrenOf.set(parent, bucket);
+            }
+            bucket.add(child);
+            child = parent;
+            parent = parentPathname(child);
+        }
+    }
+    const isLeaf = (pathname) => {
+        const kids = childrenOf.get(pathname);
+        return !kids || kids.size === 0;
+    };
+    const listParents = new Set();
+    for (const [parent, children] of childrenOf) {
+        if (parent === "/")
+            continue; // home fan-out is structural, not an archive list
+        const childList = [...children];
+        if (childList.length < minChildren)
+            continue;
+        const leafCount = childList.filter((child) => isLeaf(child)).length;
+        if (leafCount >= minChildren && leafCount / childList.length >= leafRatio) {
+            listParents.add(parent);
+        }
+    }
+    return { listParents, origin };
+}
+/** Closest list-parent pathname for this path, or null. */
+export function listParentForPathname(pathname, listParents) {
+    let current = parentPathname(pathname);
+    while (current !== null) {
+        if (listParents.has(current))
+            return current;
+        current = parentPathname(current);
+    }
+    return null;
+}
+export function classifyCrawlUrl(urlString, tree) {
+    const pathname = pathnameOf(urlString);
+    if (pathname === "/") {
+        return { url: urlString, kind: "home", section: null, priority: PRIORITY.home };
+    }
+    if (tree.listParents.has(pathname)) {
+        return { url: urlString, kind: "list", section: pathname, priority: PRIORITY.list };
+    }
+    const listParent = listParentForPathname(pathname, tree.listParents);
+    if (listParent) {
+        return { url: urlString, kind: "detail", section: listParent, priority: PRIORITY.detail };
+    }
+    const depth = pathSegments(pathname).length;
+    if (depth <= 2) {
+        return { url: urlString, kind: "root", section: null, priority: PRIORITY.root - (depth - 1) * 5 };
+    }
+    return { url: urlString, kind: "other", section: null, priority: PRIORITY.other };
+}
+export function crawlRequestPriority(urlString, tree) {
+    return classifyCrawlUrl(urlString, tree).priority;
+}
+/**
+ * Soft per-list-parent cap for detail URLs discovered during a crawl.
+ * Structural pages (home/root/list) are never capped this way.
+ */
+export function detailCapPerSection(maxPages, listParentCount = 8) {
+    const detailBudget = Math.max(8, Math.floor(maxPages * 0.5));
+    const sections = Math.max(1, listParentCount);
+    return Math.max(2, Math.floor(detailBudget / Math.min(sections, 8)));
+}
+/**
+ * Pick up to `maxPages` URLs using URL-tree shape: keep home/root/list pages,
+ * then round-robin a sample of leaf details under each detected list parent.
+ */
+export function selectSmartCrawlUrls(urls, maxPages, options) {
+    if (!Number.isFinite(maxPages) || maxPages <= 0) {
+        return { urls: [], tree: analyzeUrlTree([], options) };
+    }
+    const limit = Math.floor(maxPages);
+    const unique = uniqueUrls(urls, options?.startUrl);
+    const tree = analyzeUrlTree(unique, options);
+    if (unique.length <= limit) {
+        return { urls: sortByCrawlPriority(unique, tree), tree };
+    }
+    const classified = unique.map((url) => classifyCrawlUrl(url, tree));
+    const home = [];
+    const roots = [];
+    const lists = [];
+    const others = [];
+    const detailsBySection = new Map();
+    for (const item of classified) {
+        if (item.kind === "home")
+            home.push(item);
+        else if (item.kind === "root")
+            roots.push(item);
+        else if (item.kind === "list")
+            lists.push(item);
+        else if (item.kind === "detail") {
+            const key = item.section ?? "_";
+            const bucket = detailsBySection.get(key) ?? [];
+            bucket.push(item);
+            detailsBySection.set(key, bucket);
+        }
+        else
+            others.push(item);
+    }
+    const selected = [];
+    const selectedSet = new Set();
+    const take = (items) => {
+        for (const item of items) {
+            if (selected.length >= limit)
+                return;
+            if (selectedSet.has(item.url))
+                continue;
+            selected.push(item.url);
+            selectedSet.add(item.url);
+        }
+    };
+    take(home);
+    take(roots);
+    take(lists);
+    const sectionQueues = [...detailsBySection.values()].map((bucket) => [...bucket]);
+    let progress = true;
+    while (selected.length < limit && progress) {
+        progress = false;
+        for (const queue of sectionQueues) {
+            if (selected.length >= limit)
+                break;
+            while (queue.length > 0) {
+                const next = queue.shift();
+                if (selectedSet.has(next.url))
+                    continue;
+                selected.push(next.url);
+                selectedSet.add(next.url);
+                progress = true;
+                break;
+            }
+        }
+    }
+    take(others);
+    return { urls: selected, tree };
+}
+/** Sort URLs by crawl priority (desc), stable for equal priority. */
+export function sortByCrawlPriority(urls, tree) {
+    return urls
+        .map((url, index) => ({ url, index, priority: crawlRequestPriority(url, tree) }))
+        .sort((a, b) => b.priority - a.priority || a.index - b.index)
+        .map((row) => row.url);
+}
+/**
+ * Tracks list parents during a live crawl: uses seed analysis up front, and
+ * promotes parents to "list" when enough leaf-like children are observed.
+ */
+export function createDiscoveryGate(seedUrls, maxPages, options) {
+    const minChildren = options?.listMinChildren ?? DEFAULT_LIST_MIN_CHILDREN;
+    const leafRatio = options?.listLeafRatio ?? DEFAULT_LIST_LEAF_RATIO;
+    const tree = analyzeUrlTree(seedUrls, options);
+    const listParents = new Set(tree.listParents);
+    const childrenOf = new Map();
+    const known = new Set();
+    const detailCounts = new Map();
+    const register = (url) => {
+        const pathname = pathnameOf(url);
+        if (known.has(pathname))
+            return;
+        known.add(pathname);
+        let child = pathname;
+        let parent = parentPathname(child);
+        while (parent !== null) {
+            let bucket = childrenOf.get(parent);
+            if (!bucket) {
+                bucket = new Set();
+                childrenOf.set(parent, bucket);
+            }
+            bucket.add(child);
+            if (parent !== "/") {
+                const childList = [...bucket];
+                if (childList.length >= minChildren) {
+                    const leafCount = childList.filter((c) => {
+                        const grand = childrenOf.get(c);
+                        return !grand || grand.size === 0;
+                    }).length;
+                    if (leafCount >= minChildren && leafCount / childList.length >= leafRatio) {
+                        listParents.add(parent);
+                    }
+                }
+            }
+            child = parent;
+            parent = parentPathname(child);
+        }
+    };
+    for (const seed of seedUrls)
+        register(seed);
+    const liveTree = () => ({ listParents, origin: tree.origin });
+    const capFor = () => detailCapPerSection(maxPages, Math.max(1, listParents.size));
+    return {
+        get tree() {
+            return liveTree();
+        },
+        allowEnqueue(url) {
+            const pathname = pathnameOf(url);
+            const parent = listParentForPathname(pathname, listParents);
+            if (!parent)
+                return true;
+            return (detailCounts.get(parent) ?? 0) < capFor();
+        },
+        remember(url) {
+            register(url);
+            const classified = classifyCrawlUrl(url, liveTree());
+            if (classified.kind !== "detail" || !classified.section)
+                return;
+            detailCounts.set(classified.section, (detailCounts.get(classified.section) ?? 0) + 1);
+        },
+        isStructural(url) {
+            const kind = classifyCrawlUrl(url, liveTree()).kind;
+            return kind === "home" || kind === "root" || kind === "list";
+        },
+    };
+}

package/docs/agent-ready.md

@@ -25,8 +25,8 @@ robots.txt existence and sitemaps remain in the existing `robots-txt` and `sitem
 - Network runner: `runAgentReadyCheck()` — parallel fetches of robots, llms.txt, homepage
   (HTML + markdown Accept), and well-known endpoints
 - Wave 1 (parallel with crawl / DNS / headers / robots)
-- Report: issues land in `findings[]` (`category: "infra"`) in `*-report.json`; the
-  Markdown report lists them under **Findings**
+- Report: issues land in `findings[]` (`category: "infra"`) in `*-report.json` with a
+  `fix` suggestion; the Markdown report lists them under **Findings** with a Fix line
 
 ## CLI