@goflag/cli 0.2.2 → 0.2.4

package/dist/{chunk-6WOWKLXY.js → chunk-TVJVVSLV.js}

@@ -1925,10 +1925,10 @@ function dedupeBy(items, key) {
   return out;
 }
 function splitRoute(pathname) {
-  const segments = pathname.split("/").filter(Boolean);
-  if (segments.length > 0 && looksLikeLocaleSegment(segments[0])) {
-    const locale = localeIdentity(segments[0]);
-    const rest = segments.slice(1).join("/");
+  const segments2 = pathname.split("/").filter(Boolean);
+  if (segments2.length > 0 && looksLikeLocaleSegment(segments2[0])) {
+    const locale = localeIdentity(segments2[0]);
+    const rest = segments2.slice(1).join("/");
     return { route: rest ? `/${rest}` : "/", locale };
   }
   return { route: pathname || "/", locale: "x-default" };
@@ -2289,7 +2289,12 @@ function parseSitemap(body) {
 
 // src/lib/core/sitemap/discover.ts
 var DEFAULTS = {
-  timeoutMs: 8e3,
+  // A sitemap is one document and it can be large: this repository's own
+  // openfinanceguide serves 3.5 MB across 4008 URLs, and that took 1.2 s on a
+  // good connection. Eight seconds was tight enough that the fetch failed
+  // intermittently, and every failure silently dropped the crawl to link
+  // following. Twenty is generous for a document fetched once per run.
+  timeoutMs: 2e4,
   maxUrls: 5e3,
   maxSitemaps: 50,
   crawlDepth: 1,
@@ -2332,6 +2337,10 @@ async function discoverSitemap(baseUrl, options = {}) {
     const doc = await fetchDoc(candidate.url, options);
     if (!doc.body) {
       if (diagnostics.status === 0) diagnostics.status = doc.status;
+      if (doc.error) {
+        diagnostics.unreachable ??= `${candidate.url}: ${doc.error}`;
+        diagnostics.warnings.push(`Sitemap unreachable at ${candidate.url} \u2014 ${doc.error}`);
+      }
       continue;
     }
     const parsed = parseSitemap(doc.body);
@@ -2472,8 +2481,9 @@ async function fetchDoc(url, options) {
       }
     }
     return { status: res.status, body: await res.text() };
-  } catch {
-    return { status: 0 };
+  } catch (err) {
+    const reason = err instanceof Error ? err.message : String(err);
+    return { status: 0, error: reason || "network error" };
   } finally {
     cleanup();
     restoreTls();
@@ -2489,6 +2499,90 @@ function relaxTlsIfRequested2(enabled) {
   };
 }
 
+// src/lib/core/coverage.ts
+var DEFAULT_PER_FAMILY = 3;
+var DEFAULT_THRESHOLD = 8;
+function segments(url) {
+  try {
+    return new URL(url).pathname.split("/").filter(Boolean);
+  } catch {
+    return [];
+  }
+}
+function shapeOf(parts, locales) {
+  return parts.map((part) => locales.has(part.toLowerCase()) ? "{locale}" : part);
+}
+function patternOf(parts, varDepths) {
+  const out = [];
+  for (let i = 0; i < parts.length; i += 1) {
+    const prefix = parts.slice(0, i).join("/");
+    out.push(varDepths.has(`${i}:${prefix}`) ? `{${i}}` : parts[i]);
+  }
+  return `/${out.join("/")}`;
+}
+function representatives(sorted2, k) {
+  if (sorted2.length <= k) return [...sorted2];
+  if (k <= 1) return [sorted2[0]];
+  const picked = [];
+  for (let i = 0; i < k; i += 1) {
+    picked.push(sorted2[Math.round(i * (sorted2.length - 1) / (k - 1))]);
+  }
+  return [...new Set(picked)];
+}
+function selectByStructure(urls, options = {}) {
+  const perFamily = options.perFamily ?? DEFAULT_PER_FAMILY;
+  const threshold = options.threshold ?? DEFAULT_THRESHOLD;
+  const locales = new Set((options.locales ?? []).map((l) => l.toLowerCase()));
+  const parsed = urls.map((url) => ({ url, parts: shapeOf(segments(url), locales) }));
+  const childrenByPrefix = /* @__PURE__ */ new Map();
+  for (const { parts } of parsed) {
+    for (let i = 0; i < parts.length; i += 1) {
+      const key = `${i}:${parts.slice(0, i).join("/")}`;
+      const set = childrenByPrefix.get(key) ?? /* @__PURE__ */ new Set();
+      set.add(parts[i]);
+      childrenByPrefix.set(key, set);
+    }
+  }
+  const varDepths = /* @__PURE__ */ new Set();
+  for (const [key, children] of childrenByPrefix) {
+    const colon = key.indexOf(":");
+    const depth = Number(key.slice(0, colon));
+    const prefix = key.slice(colon + 1);
+    const isFirstSection = depth === 0 || depth === 1 && prefix === "{locale}";
+    if (isFirstSection) continue;
+    if (children.size >= threshold) varDepths.add(key);
+  }
+  const groups = /* @__PURE__ */ new Map();
+  for (const { url, parts } of parsed) {
+    const pattern = patternOf(parts, varDepths);
+    const localePart = segments(url).find((p) => locales.has(p.toLowerCase())) ?? "";
+    const key = `${localePart}|${pattern}`;
+    const group = groups.get(key) ?? { pattern, urls: [] };
+    group.urls.push(url);
+    groups.set(key, group);
+  }
+  const keep = /* @__PURE__ */ new Set();
+  const families = /* @__PURE__ */ new Map();
+  for (const { pattern, urls: groupUrls } of groups.values()) {
+    const sorted2 = [...groupUrls].sort();
+    const chosen = sorted2.length < threshold ? sorted2 : representatives(sorted2, perFamily);
+    for (const url of chosen) keep.add(url);
+    if (sorted2.length >= threshold) {
+      const seen = families.get(pattern);
+      families.set(pattern, {
+        pattern,
+        size: (seen?.size ?? 0) + sorted2.length,
+        sampled: (seen?.sampled ?? 0) + chosen.length
+      });
+    }
+  }
+  return {
+    urls: urls.filter((url) => keep.has(url)),
+    families: [...families.values()].sort((a, b) => b.size - a.size),
+    total: urls.length
+  };
+}
+
 // src/lib/rules/advisory.ts
 function resolveEvidencePath(extraction, path) {
   let current = extraction;
@@ -3402,7 +3496,11 @@ async function runAudit(inputUrl, options = {}) {
     allowInsecureTls: options.allowInsecureTls,
     crawlFallback: false
   }).catch(() => void 0);
-  const sitemapUrls = (discovery?.urls ?? []).map((u) => u.loc);
+  const allSitemapUrls = (discovery?.urls ?? []).map((u) => u.loc);
+  const coverageMode = options.coverage ?? (allSitemapUrls.length > 0 ? "structural" : "all");
+  const selection = coverageMode === "structural" && allSitemapUrls.length > 0 ? selectByStructure(allSitemapUrls, { locales: options.locales }) : void 0;
+  const sitemapUrls = selection?.urls ?? allSitemapUrls;
+  const effectiveMaxPages = selection ? Math.max(options.maxPages ?? maxPages, selection.urls.length + 5) : maxPages;
   const robots = await probeRobots(origin, {
     signal: options.signal,
     timeoutMs: options.timeoutMs
@@ -3411,7 +3509,7 @@ async function runAudit(inputUrl, options = {}) {
     entryUrl: entry,
     seedUrls: sitemapUrls,
     depth,
-    maxPages,
+    maxPages: effectiveMaxPages,
     include: options.include,
     exclude: options.exclude,
     inspectOptions: {
@@ -3434,11 +3532,18 @@ async function runAudit(inputUrl, options = {}) {
   const okPages = pages.filter((p) => isOkStatus(p.fetch.status));
   const documents = okPages.filter(isHtmlPage);
   const { kept: htmlPages, dropped: duplicatePages } = dropCanonicalDuplicates(documents);
-  const unreachablePages = pages.filter((p) => !isOkStatus(p.fetch.status)).map((p) => ({
-    id: fingerprint("page", routeKey(p.fetch.finalUrl)),
-    url: p.fetch.finalUrl,
-    status: p.fetch.status
-  }));
+  const unreachablePages = [
+    ...pages.filter((p) => !isOkStatus(p.fetch.status)).map((p) => ({
+      id: fingerprint("page", routeKey(p.fetch.finalUrl)),
+      url: p.fetch.finalUrl,
+      status: p.fetch.status
+    })),
+    ...crawlResult.errors.map((e) => ({
+      id: fingerprint("page", routeKey(e.url)),
+      url: e.url,
+      status: 0
+    }))
+  ];
   const seoIssues = [];
   const conformanceRows = [];
   const advisories = [];
@@ -3531,7 +3636,7 @@ async function runAudit(inputUrl, options = {}) {
     allowInsecureTls: options.allowInsecureTls,
     checkExternal,
     timeoutMs: options.timeoutMs,
-    maxPages,
+    maxPages: effectiveMaxPages,
     signal: options.signal,
     onProgress: (p) => options.onProgress?.({
       phase: p.phase === "scan" ? "scan" : "links",
@@ -3621,14 +3726,26 @@ async function runAudit(inputUrl, options = {}) {
       pagesFailed: linkReport.diagnostics.pagesFailed,
       truncated: crawlResult.truncated || linkReport.truncated,
       warnings,
+      // What the run actually looked at. A sampled audit that does not say so
+      // is a partial audit wearing the face of a complete one, which is worse
+      // than the cap it replaces.
+      coverage: {
+        mode: coverageMode,
+        ...selection ? {
+          considered: selection.total,
+          selected: selection.urls.length,
+          families: selection.families
+        } : {}
+      },
       ...ignoredHoles > 0 ? { ignoredHoles } : {},
       ...duplicatePages > 0 ? { duplicatePages } : {},
       ...discovery ? {
         sitemap: {
           found: discovery.diagnostics.found,
           sitemapUrl: discovery.diagnostics.sitemapUrl,
-          urlCount: sitemapUrls.length,
-          uncrawled: countUncrawled(sitemapUrls, pages)
+          urlCount: allSitemapUrls.length,
+          uncrawled: countUncrawled(sitemapUrls, pages),
+          ...discovery.diagnostics.unreachable ? { unreachable: discovery.diagnostics.unreachable } : {}
         }
       } : {}
     }
@@ -3699,6 +3816,24 @@ function renderTerminal(report, options = {}) {
       (report.profile === DEFAULT_PROFILE ? "" : `, profile ${report.profile}`)
     )
   );
+  const coverage = report.diagnostics.coverage;
+  if (coverage?.families?.length && coverage.considered && coverage.selected) {
+    const biggest = coverage.families[0];
+    const audited = report.diagnostics.pagesCrawled;
+    const capped = audited < coverage.selected ? `, capped by --max-pages` : "";
+    lines.push(
+      c(
+        ANSI.dim,
+        `COVERAGE  ${audited} of ${coverage.considered} pages audited${capped} \xB7 ${coverage.families.length} families sampled, largest ${biggest.sampled}/${biggest.size} ${biggest.pattern}`
+      )
+    );
+    lines.push(
+      c(
+        ANSI.dim,
+        `          Template rules are conclusive. Copy rules \u2014 title.length, description.length \u2014 are sampled.`
+      )
+    );
+  }
   lines.push("");
   lines.push(
     [
@@ -4275,4 +4410,4 @@ export {
   summarize,
   Logger
 };
-//# sourceMappingURL=chunk-6WOWKLXY.js.map
+//# sourceMappingURL=chunk-TVJVVSLV.js.map