@eightmedia/flightcheck 1.0.0 → 1.1.0

package/dist/checks/sitemap.js

@@ -16,12 +16,12 @@ export function analyzeSitemapCompare(input) {
     const issues = [];
     for (const source of input.sources) {
         if (source.status === null || source.status >= 400) {
-            issues.push(issue("sitemap-unreachable", "error", `Sitemap not reachable: ${source.url} (${source.status ?? "unreachable"}).`));
+            issues.push(issue("sitemap-unreachable", "error", `Sitemap not reachable: ${source.url} (${source.status ?? "unreachable"}).`, `Ensure ${source.url} returns HTTP 200 with a valid XML sitemap or sitemap index.`));
         }
     }
     const successfulSources = input.sources.filter((source) => source.status !== null && source.status >= 200 && source.status < 300);
     if (successfulSources.length === 0) {
-        issues.push(issue("sitemap-missing", "error", "No sitemap found."));
+        issues.push(issue("sitemap-missing", "error", "No sitemap found.", "Publish /sitemap.xml (or reference it from robots.txt) and make sure it returns HTTP 200."));
         return {
             sources: input.sources,
             sitemapPageUrls: [],
@@ -35,7 +35,7 @@ export function analyzeSitemapCompare(input) {
     const sitemapPageUrls = filterComparableUrls(successfulSources.flatMap((source) => source.pageUrls), excludePatterns);
     const crawledUrls = filterComparableUrls(input.crawledUrls, excludePatterns);
     if (sitemapPageUrls.length === 0) {
-        issues.push(issue("sitemap-empty", "error", "Sitemap was reachable but contained no page URLs (check sitemap index nesting and host aliases such as apex vs www)."));
+        issues.push(issue("sitemap-empty", "error", "Sitemap was reachable but contained no page URLs (check sitemap index nesting and host aliases such as apex vs www).", "Populate the sitemap with page `<loc>` entries, or fix nested sitemap indexes and apex/www host mismatches."));
         return {
             sources: input.sources,
             sitemapPageUrls,
@@ -51,17 +51,17 @@ export function analyzeSitemapCompare(input) {
     const inSitemapNotCrawled = sitemapPageUrls.filter((url) => !crawledSet.has(url));
     const crawledNotInSitemap = crawledUrls.filter((url) => !sitemapSet.has(url));
     if (inSitemapNotCrawled.length > 0) {
-        issues.push(issue("sitemap-url-not-crawled", "critical", `${inSitemapNotCrawled.length} sitemap URL(s) were not crawled. ${SITEMAP_URL_NOT_CRAWLED_TIP}`));
+        issues.push(issue("sitemap-url-not-crawled", "critical", `${inSitemapNotCrawled.length} sitemap URL(s) were not crawled. ${SITEMAP_URL_NOT_CRAWLED_TIP}`, "Remove stale sitemap entries, fix robots blocks, or raise --max-pages so listed URLs can be crawled."));
     }
     if (crawledNotInSitemap.length > 0) {
-        issues.push(issue("sitemap-missing-crawled-url", "critical", `${crawledNotInSitemap.length} crawled URL(s) are missing from the sitemap.`));
+        issues.push(issue("sitemap-missing-crawled-url", "critical", `${crawledNotInSitemap.length} crawled URL(s) are missing from the sitemap.`, "Add the crawled URLs to the sitemap (or regenerate the sitemap from the live route set)."));
     }
     const total = new Set([...sitemapPageUrls, ...crawledUrls]).size;
     const mismatchCount = new Set([...inSitemapNotCrawled, ...crawledNotInSitemap]).size;
     const mismatchPercent = total === 0 ? 0 : (mismatchCount / total) * 100;
     if (mismatchPercent > threshold) {
         issues.push(issue("sitemap-mismatch-threshold", "critical", `Sitemap mismatch is ${mismatchPercent.toFixed(1)}% (threshold ${threshold}%). ` +
-            "Sitemap URLs must match crawled URLs exactly, including hostname."));
+            "Sitemap URLs must match crawled URLs exactly, including hostname.", "Align sitemap hostnames and paths with the live site (apex vs www) and regenerate the sitemap."));
     }
     return {
         sources: input.sources,

package/dist/cli.js

@@ -5,15 +5,19 @@ import { DEFAULT_CHECK_IDS } from "./checks/types.js";
 import { DEFAULT_BURST_DELAY_MS, DEFAULT_CONCURRENCY, DEFAULT_DELAY_MS, DEFAULT_MAX_PAGES, DEFAULT_TIMEOUT_MS, } from "./crawler/defaults.js";
 import { buildDefaultOutputPaths } from "./crawler/outputPaths.js";
 import { isHttpUrl, normalizeUrl, parseExcludePatterns } from "./crawler/urlUtils.js";
+import { cliFlagsFromArgv, getPackageVersion } from "./version.js";
 async function main() {
     const sanitizedArgv = [...process.argv];
     const separatorIndex = sanitizedArgv.indexOf("--");
     if (separatorIndex >= 2) {
         sanitizedArgv.splice(separatorIndex, 1);
     }
+    const runFlags = cliFlagsFromArgv(sanitizedArgv);
+    const flightcheckVersion = getPackageVersion();
     const program = new Command();
     program
         .name("flightcheck")
+        .version(flightcheckVersion)
         .description("Crawl a website and generate crawl JSON plus structured report JSON/Markdown.")
         .argument("<startUrl>", "Start URL for the crawl")
         .option("--output <file>", "Output crawl JSON file path")
@@ -22,7 +26,7 @@ async function main() {
         .option("--errors-only", "Include only critical/error findings in the Markdown report", false)
         .option("--parallel", "Run independent checks in parallel", false)
         .option("--checks <ids>", "Comma-separated checks to run (crawl,analyze,lighthouse,security-headers,dns,robots-txt,sitemap,agent-ready)")
-        .option("--max-pages <n>", "Maximum number of pages", String(DEFAULT_MAX_PAGES))
+        .option("--max-pages <n>", "Maximum pages to crawl (default 100; prefers structural pages, samples leaf URLs under large list branches)", String(DEFAULT_MAX_PAGES))
         .option("--delay-ms <n>", "Delay between requests in ms", String(DEFAULT_DELAY_MS))
         .option("--timeout-ms <n>", "Request timeout in ms", String(DEFAULT_TIMEOUT_MS))
         .option("--concurrency <n>", "Maximum concurrent requests", String(DEFAULT_CONCURRENCY))
@@ -79,6 +83,8 @@ async function main() {
                 reportJsonPath: defaultPaths.reportJson,
                 errorsOnly,
                 siteUrl: normalizedStartUrl,
+                flightcheckVersion,
+                flags: runFlags,
             },
         });
         const crawlResult = outputs.crawl;

package/dist/crawler/defaults.d.ts

@@ -1,4 +1,5 @@
-export declare const DEFAULT_MAX_PAGES = 1000;
+/** Default crawl budget; smart selection prefers root/list pages within this cap. */
+export declare const DEFAULT_MAX_PAGES = 100;
 export declare const DEFAULT_DELAY_MS = 500;
 export declare const DEFAULT_TIMEOUT_MS = 10000;
 export declare const DEFAULT_CONCURRENCY = 1;

package/dist/crawler/defaults.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"defaults.d.ts","sourceRoot":"","sources":["../../src/crawler/defaults.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,iBAAiB,OAAO,CAAC;AACtC,eAAO,MAAM,gBAAgB,MAAM,CAAC;AACpC,eAAO,MAAM,kBAAkB,QAAQ,CAAC;AACxC,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC,eAAO,MAAM,sBAAsB,IAAI,CAAC;AACxC,eAAO,MAAM,kBAAkB,kBAAkB,CAAC;AAClD,eAAO,MAAM,0BAA0B,iBAAiB,CAAC;AACzD,eAAO,MAAM,oBAAoB,KAAK,CAAC;AACvC,eAAO,MAAM,wBAAwB,QAAQ,CAAC;AAC9C,eAAO,MAAM,kBAAkB,2CAA2C,CAAC"}
+{"version":3,"file":"defaults.d.ts","sourceRoot":"","sources":["../../src/crawler/defaults.ts"],"names":[],"mappings":"AAAA,qFAAqF;AACrF,eAAO,MAAM,iBAAiB,MAAM,CAAC;AACrC,eAAO,MAAM,gBAAgB,MAAM,CAAC;AACpC,eAAO,MAAM,kBAAkB,QAAQ,CAAC;AACxC,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC,eAAO,MAAM,sBAAsB,IAAI,CAAC;AACxC,eAAO,MAAM,kBAAkB,kBAAkB,CAAC;AAClD,eAAO,MAAM,0BAA0B,iBAAiB,CAAC;AACzD,eAAO,MAAM,oBAAoB,KAAK,CAAC;AACvC,eAAO,MAAM,wBAAwB,QAAQ,CAAC;AAC9C,eAAO,MAAM,kBAAkB,2CAA2C,CAAC"}

package/dist/crawler/defaults.js

@@ -1,4 +1,5 @@
-export const DEFAULT_MAX_PAGES = 1000;
+/** Default crawl budget; smart selection prefers root/list pages within this cap. */
+export const DEFAULT_MAX_PAGES = 100;
 export const DEFAULT_DELAY_MS = 500;
 export const DEFAULT_TIMEOUT_MS = 10000;
 export const DEFAULT_CONCURRENCY = 1;

package/dist/crawler/runCrawler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runCrawler.d.ts","sourceRoot":"","sources":["../../src/crawler/runCrawler.ts"],"names":[],"mappings":"AAUA,OAAO,EAAgB,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAOjE,OAAO,KAAK,EAAiB,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAKrE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,YAAY,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IAC/B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,wBAAsB,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,kBAAkB,EAAE,CAAC;IAAC,WAAW,EAAE,OAAO,CAAA;CAAE,CAAC,CAqK7H"}
+{"version":3,"file":"runCrawler.d.ts","sourceRoot":"","sources":["../../src/crawler/runCrawler.ts"],"names":[],"mappings":"AAUA,OAAO,EAAgB,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAOjE,OAAO,KAAK,EAAiB,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAMrE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,YAAY,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IAC/B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,wBAAsB,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,kBAAkB,EAAE,CAAC;IAAC,WAAW,EAAE,OAAO,CAAA;CAAE,CAAC,CA4K7H"}

package/dist/crawler/runCrawler.js

@@ -8,6 +8,7 @@ import { extractSameOriginScripts } from "./jsAssets.js";
 import { loadRobotsPolicy } from "./robots.js";
 import { writeReportToFile } from "./reporter.js";
 import { scanText } from "./secrets.js";
+import { createDiscoveryGate } from "./smartPageSelection.js";
 import { matchesExcludePattern, normalizeUrl, sleep } from "./urlUtils.js";
 const MAX_JS_ASSET_BYTES = 5_000_000; // 5 MB — skip larger bundles to bound memory
 export async function runCrawler(options) {
@@ -84,10 +85,14 @@ export async function runCrawler(options) {
     const requestQueue = await RequestQueue.open();
     const records = [];
     const seen = new Set();
+    const discovery = createDiscoveryGate(seedUrls, maxPages);
+    // Seeds are already smart-ordered (home/root/list before details); FIFO preserves that.
     for (const seed of seedUrls) {
         if (!matchesExcludePattern(seed, excludePatterns)) {
-            await requestQueue.addRequest({ url: seed, uniqueKey: normalizeUrl(seed) });
-            seen.add(normalizeUrl(seed));
+            const normalized = normalizeUrl(seed);
+            await requestQueue.addRequest({ url: seed, uniqueKey: normalized });
+            seen.add(normalized);
+            discovery.remember(normalized);
         }
     }
     let processedSinceBurst = 0;
@@ -151,8 +156,12 @@ export async function runCrawler(options) {
                         continue;
                     if (robotsPolicy && !robotsPolicy.isAllowed(normalized))
                         continue;
+                    if (!discovery.allowEnqueue(normalized))
+                        continue;
                     seen.add(normalized);
-                    await requestQueue.addRequest({ url: normalized, uniqueKey: normalized });
+                    discovery.remember(normalized);
+                    const forefront = discovery.isStructural(normalized);
+                    await requestQueue.addRequest({ url: normalized, uniqueKey: normalized }, { forefront });
                 }
             }
         },

package/dist/crawler/smartPageSelection.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Smart crawl URL selection based on URL-tree shape (not slug names).
+ *
+ * A "list" parent is a path with many direct children that are mostly leaves
+ * (no further subpages) — typical for archives like news/events/blog, whatever
+ * they are named. Within --max-pages we keep structural pages and sample
+ * leaves under each list parent.
+ */
+export type UrlKind = "home" | "root" | "list" | "detail" | "other";
+export type ClassifiedUrl = {
+    url: string;
+    kind: UrlKind;
+    /** Pathname of the list parent when kind is list/detail, e.g. "/stories". */
+    section: string | null;
+    /** Sort priority (higher = sooner). */
+    priority: number;
+};
+export type UrlTreeAnalysis = {
+    /** Pathnames that look like list indexes (many leaf children). */
+    listParents: ReadonlySet<string>;
+    /** Origin used to rebuild URLs from pathnames when needed. */
+    origin: string;
+};
+/** Minimum direct leaf children before a path is treated as a list parent. */
+export declare const DEFAULT_LIST_MIN_CHILDREN = 8;
+/** Share of direct children that must be leaves to count as a list. */
+export declare const DEFAULT_LIST_LEAF_RATIO = 0.7;
+export declare function pathSegments(urlOrPath: string): string[];
+export declare function pathnameOf(urlString: string): string;
+/** Parent pathname: `/a/b` → `/a`, `/a` → `/`, `/` → null. */
+export declare function parentPathname(pathname: string): string | null;
+/**
+ * Build parent → direct children maps from a URL set and detect list parents:
+ * paths with many children that are mostly leaves (no further children in the set).
+ */
+export declare function analyzeUrlTree(urls: string[], options?: {
+    startUrl?: string;
+    listMinChildren?: number;
+    listLeafRatio?: number;
+}): UrlTreeAnalysis;
+/** Closest list-parent pathname for this path, or null. */
+export declare function listParentForPathname(pathname: string, listParents: ReadonlySet<string>): string | null;
+export declare function classifyCrawlUrl(urlString: string, tree: UrlTreeAnalysis): ClassifiedUrl;
+export declare function crawlRequestPriority(urlString: string, tree: UrlTreeAnalysis): number;
+/**
+ * Soft per-list-parent cap for detail URLs discovered during a crawl.
+ * Structural pages (home/root/list) are never capped this way.
+ */
+export declare function detailCapPerSection(maxPages: number, listParentCount?: number): number;
+/**
+ * Pick up to `maxPages` URLs using URL-tree shape: keep home/root/list pages,
+ * then round-robin a sample of leaf details under each detected list parent.
+ */
+export declare function selectSmartCrawlUrls(urls: string[], maxPages: number, options?: {
+    startUrl?: string;
+    listMinChildren?: number;
+    listLeafRatio?: number;
+}): {
+    urls: string[];
+    tree: UrlTreeAnalysis;
+};
+/** Sort URLs by crawl priority (desc), stable for equal priority. */
+export declare function sortByCrawlPriority(urls: string[], tree: UrlTreeAnalysis): string[];
+/**
+ * Tracks list parents during a live crawl: uses seed analysis up front, and
+ * promotes parents to "list" when enough leaf-like children are observed.
+ */
+export declare function createDiscoveryGate(seedUrls: string[], maxPages: number, options?: {
+    listMinChildren?: number;
+    listLeafRatio?: number;
+}): {
+    tree: UrlTreeAnalysis;
+    allowEnqueue: (url: string) => boolean;
+    remember: (url: string) => void;
+    isStructural: (url: string) => boolean;
+};
+//# sourceMappingURL=smartPageSelection.d.ts.map

package/dist/crawler/smartPageSelection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"smartPageSelection.d.ts","sourceRoot":"","sources":["../../src/crawler/smartPageSelection.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,MAAM,MAAM,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;AAEpE,MAAM,MAAM,aAAa,GAAG;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,OAAO,CAAC;IACd,6EAA6E;IAC7E,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,uCAAuC;IACvC,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,kEAAkE;IAClE,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACjC,8DAA8D;IAC9D,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,8EAA8E;AAC9E,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAE3C,uEAAuE;AACvE,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAU3C,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAOxD;AAED,wBAAgB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAQpD;AAED,8DAA8D;AAC9D,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAK9D;AA2BD;;;GAGG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,CAAC,EAAE;IACR,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GACA,eAAe,CA2CjB;AAED,2DAA2D;AAC3D,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,GAC/B,MAAM,GAAG,IAAI,CAOf;AAED,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,GAAG,aAAa,CAsBxF;AAED,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,GAAG,MAAM,CAErF;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,eAAe,SAAI,GAAG,MAAM,CAIjF;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,MAAM,EAAE,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;IACR,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GACA;IAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAAC,IAAI,EAAE,eAAe,CAAA;CAAE,CAmE3C;AAED,qEAAqE;AACrE,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,eAAe,GAAG,MAAM,EAAE,CAKnF;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,MAAM,EAAE,EAClB,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;IAAE,eAAe,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAAE,GAC7D;IACD,IAAI,EAAE,eAAe,CAAC;IACtB,YAAY,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;IACvC,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,YAAY,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;CACxC,CAqEA"}

package/dist/crawler/smartPageSelection.js

@@ -0,0 +1,308 @@
+/**
+ * Smart crawl URL selection based on URL-tree shape (not slug names).
+ *
+ * A "list" parent is a path with many direct children that are mostly leaves
+ * (no further subpages) — typical for archives like news/events/blog, whatever
+ * they are named. Within --max-pages we keep structural pages and sample
+ * leaves under each list parent.
+ */
+/** Minimum direct leaf children before a path is treated as a list parent. */
+export const DEFAULT_LIST_MIN_CHILDREN = 8;
+/** Share of direct children that must be leaves to count as a list. */
+export const DEFAULT_LIST_LEAF_RATIO = 0.7;
+const PRIORITY = {
+    home: 100,
+    root: 80,
+    list: 60,
+    detail: 40,
+    other: 20,
+};
+export function pathSegments(urlOrPath) {
+    try {
+        const pathname = urlOrPath.includes("://") ? new URL(urlOrPath).pathname : urlOrPath;
+        return pathname.split("/").filter(Boolean);
+    }
+    catch {
+        return [];
+    }
+}
+export function pathnameOf(urlString) {
+    try {
+        const pathname = new URL(urlString).pathname;
+        if (!pathname || pathname === "")
+            return "/";
+        return pathname.length > 1 && pathname.endsWith("/") ? pathname.slice(0, -1) : pathname;
+    }
+    catch {
+        return "/";
+    }
+}
+/** Parent pathname: `/a/b` → `/a`, `/a` → `/`, `/` → null. */
+export function parentPathname(pathname) {
+    if (pathname === "/" || pathname === "")
+        return null;
+    const segments = pathname.split("/").filter(Boolean);
+    if (segments.length <= 1)
+        return "/";
+    return `/${segments.slice(0, -1).join("/")}`;
+}
+function originOf(urls, startUrl) {
+    const candidate = startUrl ?? urls[0];
+    if (!candidate)
+        return "https://example.com";
+    try {
+        return new URL(candidate).origin;
+    }
+    catch {
+        return "https://example.com";
+    }
+}
+function uniqueUrls(urls, startUrl) {
+    const seen = new Set();
+    const unique = [];
+    if (startUrl) {
+        unique.push(startUrl);
+        seen.add(startUrl);
+    }
+    for (const url of urls) {
+        if (seen.has(url))
+            continue;
+        seen.add(url);
+        unique.push(url);
+    }
+    return unique;
+}
+/**
+ * Build parent → direct children maps from a URL set and detect list parents:
+ * paths with many children that are mostly leaves (no further children in the set).
+ */
+export function analyzeUrlTree(urls, options) {
+    const minChildren = options?.listMinChildren ?? DEFAULT_LIST_MIN_CHILDREN;
+    const leafRatio = options?.listLeafRatio ?? DEFAULT_LIST_LEAF_RATIO;
+    const unique = uniqueUrls(urls, options?.startUrl);
+    const origin = originOf(unique, options?.startUrl);
+    const childrenOf = new Map();
+    const pathnames = new Set();
+    for (const url of unique) {
+        const pathname = pathnameOf(url);
+        pathnames.add(pathname);
+        let child = pathname;
+        let parent = parentPathname(child);
+        while (parent !== null) {
+            let bucket = childrenOf.get(parent);
+            if (!bucket) {
+                bucket = new Set();
+                childrenOf.set(parent, bucket);
+            }
+            bucket.add(child);
+            child = parent;
+            parent = parentPathname(child);
+        }
+    }
+    const isLeaf = (pathname) => {
+        const kids = childrenOf.get(pathname);
+        return !kids || kids.size === 0;
+    };
+    const listParents = new Set();
+    for (const [parent, children] of childrenOf) {
+        if (parent === "/")
+            continue; // home fan-out is structural, not an archive list
+        const childList = [...children];
+        if (childList.length < minChildren)
+            continue;
+        const leafCount = childList.filter((child) => isLeaf(child)).length;
+        if (leafCount >= minChildren && leafCount / childList.length >= leafRatio) {
+            listParents.add(parent);
+        }
+    }
+    return { listParents, origin };
+}
+/** Closest list-parent pathname for this path, or null. */
+export function listParentForPathname(pathname, listParents) {
+    let current = parentPathname(pathname);
+    while (current !== null) {
+        if (listParents.has(current))
+            return current;
+        current = parentPathname(current);
+    }
+    return null;
+}
+export function classifyCrawlUrl(urlString, tree) {
+    const pathname = pathnameOf(urlString);
+    if (pathname === "/") {
+        return { url: urlString, kind: "home", section: null, priority: PRIORITY.home };
+    }
+    if (tree.listParents.has(pathname)) {
+        return { url: urlString, kind: "list", section: pathname, priority: PRIORITY.list };
+    }
+    const listParent = listParentForPathname(pathname, tree.listParents);
+    if (listParent) {
+        return { url: urlString, kind: "detail", section: listParent, priority: PRIORITY.detail };
+    }
+    const depth = pathSegments(pathname).length;
+    if (depth <= 2) {
+        return { url: urlString, kind: "root", section: null, priority: PRIORITY.root - (depth - 1) * 5 };
+    }
+    return { url: urlString, kind: "other", section: null, priority: PRIORITY.other };
+}
+export function crawlRequestPriority(urlString, tree) {
+    return classifyCrawlUrl(urlString, tree).priority;
+}
+/**
+ * Soft per-list-parent cap for detail URLs discovered during a crawl.
+ * Structural pages (home/root/list) are never capped this way.
+ */
+export function detailCapPerSection(maxPages, listParentCount = 8) {
+    const detailBudget = Math.max(8, Math.floor(maxPages * 0.5));
+    const sections = Math.max(1, listParentCount);
+    return Math.max(2, Math.floor(detailBudget / Math.min(sections, 8)));
+}
+/**
+ * Pick up to `maxPages` URLs using URL-tree shape: keep home/root/list pages,
+ * then round-robin a sample of leaf details under each detected list parent.
+ */
+export function selectSmartCrawlUrls(urls, maxPages, options) {
+    if (!Number.isFinite(maxPages) || maxPages <= 0) {
+        return { urls: [], tree: analyzeUrlTree([], options) };
+    }
+    const limit = Math.floor(maxPages);
+    const unique = uniqueUrls(urls, options?.startUrl);
+    const tree = analyzeUrlTree(unique, options);
+    if (unique.length <= limit) {
+        return { urls: sortByCrawlPriority(unique, tree), tree };
+    }
+    const classified = unique.map((url) => classifyCrawlUrl(url, tree));
+    const home = [];
+    const roots = [];
+    const lists = [];
+    const others = [];
+    const detailsBySection = new Map();
+    for (const item of classified) {
+        if (item.kind === "home")
+            home.push(item);
+        else if (item.kind === "root")
+            roots.push(item);
+        else if (item.kind === "list")
+            lists.push(item);
+        else if (item.kind === "detail") {
+            const key = item.section ?? "_";
+            const bucket = detailsBySection.get(key) ?? [];
+            bucket.push(item);
+            detailsBySection.set(key, bucket);
+        }
+        else
+            others.push(item);
+    }
+    const selected = [];
+    const selectedSet = new Set();
+    const take = (items) => {
+        for (const item of items) {
+            if (selected.length >= limit)
+                return;
+            if (selectedSet.has(item.url))
+                continue;
+            selected.push(item.url);
+            selectedSet.add(item.url);
+        }
+    };
+    take(home);
+    take(roots);
+    take(lists);
+    const sectionQueues = [...detailsBySection.values()].map((bucket) => [...bucket]);
+    let progress = true;
+    while (selected.length < limit && progress) {
+        progress = false;
+        for (const queue of sectionQueues) {
+            if (selected.length >= limit)
+                break;
+            while (queue.length > 0) {
+                const next = queue.shift();
+                if (selectedSet.has(next.url))
+                    continue;
+                selected.push(next.url);
+                selectedSet.add(next.url);
+                progress = true;
+                break;
+            }
+        }
+    }
+    take(others);
+    return { urls: selected, tree };
+}
+/** Sort URLs by crawl priority (desc), stable for equal priority. */
+export function sortByCrawlPriority(urls, tree) {
+    return urls
+        .map((url, index) => ({ url, index, priority: crawlRequestPriority(url, tree) }))
+        .sort((a, b) => b.priority - a.priority || a.index - b.index)
+        .map((row) => row.url);
+}
+/**
+ * Tracks list parents during a live crawl: uses seed analysis up front, and
+ * promotes parents to "list" when enough leaf-like children are observed.
+ */
+export function createDiscoveryGate(seedUrls, maxPages, options) {
+    const minChildren = options?.listMinChildren ?? DEFAULT_LIST_MIN_CHILDREN;
+    const leafRatio = options?.listLeafRatio ?? DEFAULT_LIST_LEAF_RATIO;
+    const tree = analyzeUrlTree(seedUrls, options);
+    const listParents = new Set(tree.listParents);
+    const childrenOf = new Map();
+    const known = new Set();
+    const detailCounts = new Map();
+    const register = (url) => {
+        const pathname = pathnameOf(url);
+        if (known.has(pathname))
+            return;
+        known.add(pathname);
+        let child = pathname;
+        let parent = parentPathname(child);
+        while (parent !== null) {
+            let bucket = childrenOf.get(parent);
+            if (!bucket) {
+                bucket = new Set();
+                childrenOf.set(parent, bucket);
+            }
+            bucket.add(child);
+            if (parent !== "/") {
+                const childList = [...bucket];
+                if (childList.length >= minChildren) {
+                    const leafCount = childList.filter((c) => {
+                        const grand = childrenOf.get(c);
+                        return !grand || grand.size === 0;
+                    }).length;
+                    if (leafCount >= minChildren && leafCount / childList.length >= leafRatio) {
+                        listParents.add(parent);
+                    }
+                }
+            }
+            child = parent;
+            parent = parentPathname(child);
+        }
+    };
+    for (const seed of seedUrls)
+        register(seed);
+    const liveTree = () => ({ listParents, origin: tree.origin });
+    const capFor = () => detailCapPerSection(maxPages, Math.max(1, listParents.size));
+    return {
+        get tree() {
+            return liveTree();
+        },
+        allowEnqueue(url) {
+            const pathname = pathnameOf(url);
+            const parent = listParentForPathname(pathname, listParents);
+            if (!parent)
+                return true;
+            return (detailCounts.get(parent) ?? 0) < capFor();
+        },
+        remember(url) {
+            register(url);
+            const classified = classifyCrawlUrl(url, liveTree());
+            if (classified.kind !== "detail" || !classified.section)
+                return;
+            detailCounts.set(classified.section, (detailCounts.get(classified.section) ?? 0) + 1);
+        },
+        isStructural(url) {
+            const kind = classifyCrawlUrl(url, liveTree()).kind;
+            return kind === "home" || kind === "root" || kind === "list";
+        },
+    };
+}

package/dist/version.d.ts

@@ -0,0 +1,9 @@
+/** Installed package version from package.json (works from src/ and dist/). */
+export declare function getPackageVersion(): string;
+/**
+ * CLI flags as passed on the command line (everything after node + script path).
+ * Example: `["https://example.com", "--quiet", "--max-pages", "50"]`
+ */
+export declare function cliFlagsFromArgv(argv?: string[]): string[];
+export declare function formatCliFlags(flags: string[]): string;
+//# sourceMappingURL=version.d.ts.map

package/dist/version.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"version.d.ts","sourceRoot":"","sources":["../src/version.ts"],"names":[],"mappings":"AAIA,+EAA+E;AAC/E,wBAAgB,iBAAiB,IAAI,MAAM,CAO1C;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,GAAE,MAAM,EAAiB,GAAG,MAAM,EAAE,CAExE;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAGtD"}

package/dist/version.js

@@ -0,0 +1,24 @@
+import { createRequire } from "node:module";
+const require = createRequire(import.meta.url);
+/** Installed package version from package.json (works from src/ and dist/). */
+export function getPackageVersion() {
+    try {
+        const pkg = require("../package.json");
+        return pkg.version ?? "unknown";
+    }
+    catch {
+        return "unknown";
+    }
+}
+/**
+ * CLI flags as passed on the command line (everything after node + script path).
+ * Example: `["https://example.com", "--quiet", "--max-pages", "50"]`
+ */
+export function cliFlagsFromArgv(argv = process.argv) {
+    return argv.slice(2);
+}
+export function formatCliFlags(flags) {
+    if (flags.length === 0)
+        return "(defaults)";
+    return flags.map((flag) => (/\s/.test(flag) ? JSON.stringify(flag) : flag)).join(" ");
+}

package/docs/agent-fixes.md

@@ -3,15 +3,13 @@
 **Status:** deprecated. Analyze no longer writes `*-fix-manifest.json` or
 `*-agent-jobs/`.
 
-Use the structured report instead:
-
-- [Analyze reports](./analyze.md) — `*-report.json` + `*-report.md`
-- [Design](./superpowers/specs/2026-10-27-structured-report-design.md)
+Use the structured report instead: [Analyze reports](./analyze.md)
+(`*-report.json` + `*-report.md`).
 
 The previous agent-jobs / fix-manifest workflow (work packages, `prompt.md` per
-package, `flightcheck-orchestrate`) remains in the repo for transitional imports
-but is not part of the default output. Prefer building any new agent automation
-on `findings[]` in `*-report.json`.
+package, orchestrate helpers) may still exist in the source tree for reference, but
+it is not part of the published package or default CLI output. Prefer building any
+new agent automation on `findings[]` in `*-report.json`.
 
 Stack detection still feeds the report `analysis.stack` block. See
 [stack.md](./stack.md).

package/docs/agent-ready.md

@@ -25,8 +25,8 @@ robots.txt existence and sitemaps remain in the existing `robots-txt` and `sitem
 - Network runner: `runAgentReadyCheck()` — parallel fetches of robots, llms.txt, homepage
   (HTML + markdown Accept), and well-known endpoints
 - Wave 1 (parallel with crawl / DNS / headers / robots)
-- Report section: `## Agent readiness` via `buildAgentReadySection()` in `reportTier1.ts`
-- Fix package: `agent-ready` → `prompts/fix/agent-ready.md`
+- Report: issues land in `findings[]` (`category: "infra"`) in `*-report.json` with a
+  `fix` suggestion; the Markdown report lists them under **Findings** with a Fix line
 
 ## CLI