afdocs 0.4.0 → 0.5.0

package/dist/helpers/get-page-urls.js

@@ -1,5 +1,6 @@
 import { extractMarkdownLinks } from '../checks/llms-txt/llms-txt-valid.js';
 import { MAX_SITEMAP_URLS } from '../constants.js';
+import { isNonPageUrl } from './to-md-urls.js';
 /**
  * Extract `<loc>` URLs from a sitemap XML string.
  * Also returns any `<sitemap><loc>` entries from sitemap index files.
@@ -30,10 +31,11 @@ export function parseSitemapUrls(xml) {
     }
     return { urls, sitemapIndexUrls };
 }
-function getUrlsFromCachedLlmsTxt(ctx) {
+export async function getUrlsFromCachedLlmsTxt(ctx) {
     const existsResult = ctx.previousResults.get('llms-txt-exists');
     const discovered = (existsResult?.details?.discoveredFiles ?? []);
-    return extractLinksFromLlmsTxtFiles(discovered);
+    const urls = extractLinksFromLlmsTxtFiles(discovered);
+    return walkAggregateLinks(ctx, urls);
 }
 function extractLinksFromLlmsTxtFiles(files) {
     const urls = new Set();
@@ -43,10 +45,92 @@ function extractLinksFromLlmsTxtFiles(files) {
             if (link.url.startsWith('http://') || link.url.startsWith('https://')) {
                 urls.add(link.url);
             }
+            else if (link.url.startsWith('/')) {
+                // Resolve root-relative URLs against the source file's origin
+                try {
+                    const base = new URL(file.url);
+                    urls.add(new URL(link.url, base.origin).toString());
+                }
+                catch {
+                    // Skip malformed URLs
+                }
+            }
         }
     }
     return Array.from(urls);
 }
+/**
+ * Identify .txt links that are likely aggregate/index files (progressive
+ * disclosure pattern) and walk them one level deep to find page URLs.
+ *
+ * A link is considered walkable when it ends in .txt and is on the same
+ * origin as the site being tested. This covers both sub-product llms.txt
+ * files (Cloudflare) and aggregate content files (Supabase).
+ */
+async function walkAggregateLinks(ctx, urls) {
+    const pageUrls = [];
+    const aggregateUrls = [];
+    for (const url of urls) {
+        try {
+            const parsed = new URL(url);
+            if (/\.txt$/i.test(parsed.pathname)) {
+                // .txt files are either aggregate indexes to walk (same origin)
+                // or external resources to skip — never page URLs themselves
+                if (parsed.origin === ctx.origin) {
+                    aggregateUrls.push(url);
+                }
+            }
+            else {
+                pageUrls.push(url);
+            }
+        }
+        catch {
+            pageUrls.push(url);
+        }
+    }
+    if (aggregateUrls.length === 0)
+        return pageUrls;
+    // Fetch aggregate files and extract their links
+    for (const aggUrl of aggregateUrls) {
+        try {
+            const response = await ctx.http.fetch(aggUrl);
+            if (!response.ok)
+                continue;
+            const contentType = response.headers.get('content-type') ?? '';
+            if (!contentType.includes('text/'))
+                continue;
+            const content = await response.text();
+            const trimmed = content.trimStart().toLowerCase();
+            if (trimmed.startsWith('<!') || trimmed.startsWith('<html'))
+                continue;
+            if (content.trim().length === 0)
+                continue;
+            const subFile = {
+                url: aggUrl,
+                content,
+                status: response.status,
+                redirected: response.redirected,
+            };
+            const subUrls = extractLinksFromLlmsTxtFiles([subFile]);
+            for (const subUrl of subUrls) {
+                // Only keep same-origin page URLs (skip further .txt nesting)
+                try {
+                    const parsed = new URL(subUrl);
+                    if (parsed.origin === ctx.origin && !isNonPageUrl(subUrl)) {
+                        pageUrls.push(subUrl);
+                    }
+                }
+                catch {
+                    // Skip malformed URLs
+                }
+            }
+        }
+        catch {
+            // Skip failed fetches
+        }
+    }
+    return pageUrls;
+}
 /**
  * Directly fetch llms.txt candidate URLs and extract links.
  * Used when `llms-txt-exists` hasn't run (e.g. standalone check mode).
@@ -82,7 +166,8 @@ async function fetchLlmsTxtUrls(ctx) {
             // Skip failed fetches
         }
     }
-    return extractLinksFromLlmsTxtFiles(discovered);
+    const urls = extractLinksFromLlmsTxtFiles(discovered);
+    return walkAggregateLinks(ctx, urls);
 }
 /**
  * Parse `Sitemap:` directives from a robots.txt body.
@@ -101,8 +186,8 @@ export function parseSitemapDirectives(robotsTxt) {
 /**
  * Discover sitemap URLs by checking robots.txt first, then falling back to /sitemap.xml.
  */
-async function discoverSitemapUrls(ctx) {
-    // Try robots.txt for Sitemap directives
+async function discoverSitemapUrls(ctx, originOverride) {
+    // Try robots.txt for Sitemap directives at the original origin first
     try {
         const robotsResponse = await ctx.http.fetch(`${ctx.origin}/robots.txt`);
         if (robotsResponse.ok) {
@@ -115,8 +200,23 @@ async function discoverSitemapUrls(ctx) {
     catch {
         // robots.txt fetch failed; fall through
     }
-    // Default to /sitemap.xml
-    return [`${ctx.origin}/sitemap.xml`];
+    // If there's an effective origin (cross-host redirect), try its robots.txt too
+    if (originOverride && originOverride !== ctx.origin) {
+        try {
+            const robotsResponse = await ctx.http.fetch(`${originOverride}/robots.txt`);
+            if (robotsResponse.ok) {
+                const body = await robotsResponse.text();
+                const directives = parseSitemapDirectives(body);
+                if (directives.length > 0)
+                    return directives;
+            }
+        }
+        catch {
+            // fall through
+        }
+    }
+    // Default to /sitemap.xml (prefer effective origin if available)
+    return [`${originOverride ?? ctx.origin}/sitemap.xml`];
 }
 function isGzipped(url) {
     return /\.gz($|\?)/i.test(url);
@@ -137,20 +237,21 @@ async function fetchSitemap(ctx, sitemapUrl, warnings) {
         return { urls: [], sitemapIndexUrls: [] };
     }
 }
-async function getUrlsFromSitemap(ctx, warnings) {
-    const sitemapUrls = await discoverSitemapUrls(ctx);
+export async function getUrlsFromSitemap(ctx, warnings, maxUrls = MAX_SITEMAP_URLS, originOverride) {
+    const sitemapUrls = await discoverSitemapUrls(ctx, originOverride);
     const urls = [];
+    const matchOrigin = originOverride ?? ctx.origin;
     for (const sitemapUrl of sitemapUrls) {
-        if (urls.length >= MAX_SITEMAP_URLS)
+        if (urls.length >= maxUrls)
             break;
         const parsed = await fetchSitemap(ctx, sitemapUrl, warnings);
         // Add direct URLs (filtered to same origin)
         for (const url of parsed.urls) {
-            if (urls.length >= MAX_SITEMAP_URLS)
+            if (urls.length >= maxUrls)
                 break;
             try {
                 const u = new URL(url);
-                if (u.origin === ctx.origin) {
+                if (u.origin === matchOrigin) {
                     urls.push(url);
                 }
             }
@@ -159,17 +260,17 @@ async function getUrlsFromSitemap(ctx, warnings) {
             }
         }
         // Follow one level of sitemap index
-        if (parsed.sitemapIndexUrls.length > 0 && urls.length < MAX_SITEMAP_URLS) {
+        if (parsed.sitemapIndexUrls.length > 0 && urls.length < maxUrls) {
             for (const subSitemapUrl of parsed.sitemapIndexUrls) {
-                if (urls.length >= MAX_SITEMAP_URLS)
+                if (urls.length >= maxUrls)
                     break;
                 const subParsed = await fetchSitemap(ctx, subSitemapUrl, warnings);
                 for (const url of subParsed.urls) {
-                    if (urls.length >= MAX_SITEMAP_URLS)
+                    if (urls.length >= maxUrls)
                         break;
                     try {
                         const u = new URL(url);
-                        if (u.origin === ctx.origin) {
+                        if (u.origin === matchOrigin) {
                             urls.push(url);
                         }
                     }
@@ -193,7 +294,7 @@ async function getUrlsFromSitemap(ctx, warnings) {
 export async function getPageUrls(ctx) {
     const warnings = [];
     // 1. Try llms.txt links from cached results (if llms-txt-exists ran)
-    const cachedUrls = getUrlsFromCachedLlmsTxt(ctx);
+    const cachedUrls = await getUrlsFromCachedLlmsTxt(ctx);
     if (cachedUrls.length > 0)
         return { urls: cachedUrls, warnings };
     // 2. Try fetching llms.txt directly (standalone mode, llms-txt-exists didn't run)
@@ -215,20 +316,51 @@ export async function getPageUrls(ctx) {
  *
  * The result is cached on ctx so that all checks within a single run
  * share the same sampled page list, avoiding inconsistent results.
+ *
+ * Sampling strategies:
+ * - `random`: Fisher-Yates shuffle, then take the first maxLinksToTest. (Default.)
+ * - `deterministic`: Sort URLs lexicographically, then pick every Nth URL
+ *   so that the result is reproducible across runs (as long as the discovered
+ *   URL set is stable).
+ * - `none`: Skip discovery entirely; return only the baseUrl.
  */
 export async function discoverAndSamplePages(ctx) {
     if (ctx._sampledPages)
         return ctx._sampledPages;
+    const strategy = ctx.options.samplingStrategy;
+    // "none" skips discovery and uses only the URL the user provided.
+    if (strategy === 'none') {
+        ctx._sampledPages = {
+            urls: [ctx.baseUrl],
+            totalPages: 1,
+            sampled: false,
+            warnings: [],
+        };
+        return ctx._sampledPages;
+    }
     const discovery = await getPageUrls(ctx);
     let urls = discovery.urls;
     const totalPages = urls.length;
     const sampled = totalPages > ctx.options.maxLinksToTest;
     if (sampled) {
-        for (let i = urls.length - 1; i > 0; i--) {
-            const j = Math.floor(Math.random() * (i + 1));
-            [urls[i], urls[j]] = [urls[j], urls[i]];
+        if (strategy === 'deterministic') {
+            // Sort lexicographically for a stable ordering, then pick evenly-spaced URLs.
+            urls.sort();
+            const stride = urls.length / ctx.options.maxLinksToTest;
+            const picked = [];
+            for (let i = 0; i < ctx.options.maxLinksToTest; i++) {
+                picked.push(urls[Math.floor(i * stride)]);
+            }
+            urls = picked;
+        }
+        else {
+            // "random" — Fisher-Yates shuffle
+            for (let i = urls.length - 1; i > 0; i--) {
+                const j = Math.floor(Math.random() * (i + 1));
+                [urls[i], urls[j]] = [urls[j], urls[i]];
+            }
+            urls = urls.slice(0, ctx.options.maxLinksToTest);
         }
-        urls = urls.slice(0, ctx.options.maxLinksToTest);
     }
     ctx._sampledPages = { urls, totalPages, sampled, warnings: discovery.warnings };
     return ctx._sampledPages;

package/dist/helpers/get-page-urls.js.map

@@ -1 +1 @@
-{"version":3,"file":"get-page-urls.js","sourceRoot":"","sources":["../../src/helpers/get-page-urls.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AAC5E,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAGnD;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC1C,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,gBAAgB,GAAa,EAAE,CAAC;IAEtC,oDAAoD;IACpD,MAAM,cAAc,GAAG,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAElD,IAAI,cAAc,EAAE,CAAC;QACnB,wCAAwC;QACxC,MAAM,iBAAiB,GAAG,oCAAoC,CAAC;QAC/D,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACtD,MAAM,QAAQ,GAAG,0BAA0B,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3D,IAAI,QAAQ,EAAE,CAAC;gBACb,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,6CAA6C;QAC7C,MAAM,QAAQ,GAAG,2BAA2B,CAAC;QAC7C,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC;AACpC,CAAC;AAED,SAAS,wBAAwB,CAAC,GAAiB;IACjD,MAAM,YAAY,GAAG,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IAChE,MAAM,UAAU,GAAG,CAAC,YAAY,EAAE,OAAO,EAAE,eAAe,IAAI,EAAE,CAAqB,CAAC;IAEtF,OAAO,4BAA4B,CAAC,UAAU,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,4BAA4B,CAAC,KAAuB;IAC3D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBACtE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,gBAAgB,CAAC,GAAiB;IAC/C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,UAAU,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC;IAC1C,UAAU,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,WAAW,CAAC,CAAC;IACzC,UAAU,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,gBAAgB,CAAC,CAAC;IAE9C,MAAM,UAAU,GAAqB,EAAE,CAAC;IACxC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAAE,SAAS;YAC3B,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC/D,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,SAAS;YAC7C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,WAAW,EAAE,CAAC;YAClD,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC;gBAAE,SAAS;YACtE,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAC1C,UAAU,CAAC,IAAI,CAAC;gBACd,GAAG;gBACH,OAAO;gBACP,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;aAChC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,sBAAsB;QACxB,CAAC;IACH,CAAC;IAED,OAAO,4BAA4B,CAAC,UAAU,CAAC,CAAC;AAClD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,SAAiB;IACtD,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,mBAAmB,CAAC,GAAiB;IAClD,wCAAwC;IACxC,IAAI,CAAC;QACH,MAAM,cAAc,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,aAAa,CAAC,CAAC;QACxE,IAAI,cAAc,CAAC,EAAE,EAAE,CAAC;YACtB,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;YACzC,MAAM,UAAU,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAChD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,UAAU,CAAC;QAC/C,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,wCAAwC;IAC1C,CAAC;IAED,0BAA0B;IAC1B,OAAO,CAAC,GAAG,GAAG,CAAC,MAAM,cAAc,CAAC,CAAC;AACvC,CAAC;AAOD,SAAS,SAAS,CAAC,GAAW;IAC5B,OAAO,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACjC,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,GAAiB,EACjB,UAAkB,EAClB,QAAkB;IAElB,IAAI,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC;QAC1B,QAAQ,CAAC,IAAI,CAAC,4CAA4C,UAAU,EAAE,CAAC,CAAC;QACxE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,gBAAgB,EAAE,EAAE,EAAE,CAAC;IAC5C,CAAC;IAED,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAClD,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,gBAAgB,EAAE,EAAE,EAAE,CAAC;QAC5D,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,gBAAgB,CAAC,GAAG,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,gBAAgB,EAAE,EAAE,EAAE,CAAC;IAC5C,CAAC;AACH,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,GAAiB,EAAE,QAAkB;IACrE,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAAC,GAAG,CAAC,CAAC;IACnD,MAAM,IAAI,GAAa,EAAE,CAAC;IAE1B,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,MAAM,IAAI,gBAAgB;YAAE,MAAM;QAE3C,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;QAE7D,4CAA4C;QAC5C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,MAAM,IAAI,gBAAgB;gBAAE,MAAM;YAC3C,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;gBACvB,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC;oBAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACjB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,sBAAsB;YACxB,CAAC;QACH,CAAC;QAED,oCAAoC;QACpC,IAAI,MAAM,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,gBAAgB,EAAE,CAAC;YACzE,KAAK,MAAM,aAAa,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBACpD,IAAI,IAAI,CAAC,MAAM,IAAI,gBAAgB;oBAAE,MAAM;gBAE3C,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;gBAEnE,KAAK,MAAM,GAAG,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC;oBACjC,IAAI,IAAI,CAAC,MAAM,IAAI,gBAAgB;wBAAE,MAAM;oBAC3C,IAAI,CAAC;wBACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;wBACvB,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC;4BAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACjB,CAAC;oBACH,CAAC;oBAAC,MAAM,CAAC;wBACP,sBAAsB;oBACxB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAiB;IACjD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,qEAAqE;IACrE,MAAM,UAAU,GAAG,wBAAwB,CAAC,GAAG,CAAC,CAAC;IACjD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;IAEjE,kFAAkF;IAClF,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAChD,MAAM,WAAW,GAAG,MAAM,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAChD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;IACrE,CAAC;IAED,iBAAiB;IACjB,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC5D,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;IAEnE,cAAc;IACd,OAAO,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;AAC3C,CAAC;AASD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,GAAiB;IAC5D,IAAI,GAAG,CAAC,aAAa;QAAE,OAAO,GAAG,CAAC,aAAa,CAAC;IAEhD,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC;IAC1B,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC;IAE/B,MAAM,OAAO,GAAG,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC;IACxD,IAAI,OAAO,EAAE,CAAC;QACZ,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9C,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1C,CAAC;QACD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IACnD,CAAC;IAED,GAAG,CAAC,aAAa,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,CAAC;IAChF,OAAO,GAAG,CAAC,aAAa,CAAC;AAC3B,CAAC"}
+{"version":3,"file":"get-page-urls.js","sourceRoot":"","sources":["../../src/helpers/get-page-urls.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AAC5E,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAG/C;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC1C,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,gBAAgB,GAAa,EAAE,CAAC;IAEtC,oDAAoD;IACpD,MAAM,cAAc,GAAG,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAElD,IAAI,cAAc,EAAE,CAAC;QACnB,wCAAwC;QACxC,MAAM,iBAAiB,GAAG,oCAAoC,CAAC;QAC/D,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACtD,MAAM,QAAQ,GAAG,0BAA0B,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3D,IAAI,QAAQ,EAAE,CAAC;gBACb,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,6CAA6C;QAC7C,MAAM,QAAQ,GAAG,2BAA2B,CAAC;QAC7C,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,GAAiB;IAC9D,MAAM,YAAY,GAAG,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IAChE,MAAM,UAAU,GAAG,CAAC,YAAY,EAAE,OAAO,EAAE,eAAe,IAAI,EAAE,CAAqB,CAAC;IAEtF,MAAM,IAAI,GAAG,4BAA4B,CAAC,UAAU,CAAC,CAAC;IACtD,OAAO,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,4BAA4B,CAAC,KAAuB;IAC3D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBACtE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,CAAC;iBAAM,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACpC,8DAA8D;gBAC9D,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAC/B,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACtD,CAAC;gBAAC,MAAM,CAAC;oBACP,sBAAsB;gBACxB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,kBAAkB,CAAC,GAAiB,EAAE,IAAc;IACjE,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,aAAa,GAAa,EAAE,CAAC;IAEnC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACpC,gEAAgE;gBAChE,6DAA6D;gBAC7D,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC;oBACjC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAEhD,gDAAgD;IAChD,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC9C,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAAE,SAAS;YAC3B,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC/D,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,SAAS;YAC7C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,WAAW,EAAE,CAAC;YAClD,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC;gBAAE,SAAS;YACtE,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAE1C,MAAM,OAAO,GAAmB;gBAC9B,GAAG,EAAE,MAAM;gBACX,OAAO;gBACP,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;aAChC,CAAC;YACF,MAAM,OAAO,GAAG,4BAA4B,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YAExD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,8DAA8D;gBAC9D,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;oBAC/B,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;wBAC1D,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACxB,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,sBAAsB;gBACxB,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,sBAAsB;QACxB,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,gBAAgB,CAAC,GAAiB;IAC/C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,UAAU,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC;IAC1C,UAAU,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,WAAW,CAAC,CAAC;IACzC,UAAU,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,gBAAgB,CAAC,CAAC;IAE9C,MAAM,UAAU,GAAqB,EAAE,CAAC;IACxC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAAE,SAAS;YAC3B,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC/D,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,SAAS;YAC7C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,WAAW,EAAE,CAAC;YAClD,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC;gBAAE,SAAS;YACtE,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAC1C,UAAU,CAAC,IAAI,CAAC;gBACd,GAAG;gBACH,OAAO;gBACP,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;aAChC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,sBAAsB;QACxB,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,4BAA4B,CAAC,UAAU,CAAC,CAAC;IACtD,OAAO,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AACvC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,SAAiB;IACtD,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,mBAAmB,CAAC,GAAiB,EAAE,cAAuB;IAC3E,qEAAqE;IACrE,IAAI,CAAC;QACH,MAAM,cAAc,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,aAAa,CAAC,CAAC;QACxE,IAAI,cAAc,CAAC,EAAE,EAAE,CAAC;YACtB,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;YACzC,MAAM,UAAU,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAChD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,UAAU,CAAC;QAC/C,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,wCAAwC;IAC1C,CAAC;IAED,+EAA+E;IAC/E,IAAI,cAAc,IAAI,cAAc,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,cAAc,aAAa,CAAC,CAAC;YAC5E,IAAI,cAAc,CAAC,EAAE,EAAE,CAAC;gBACtB,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;gBACzC,MAAM,UAAU,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;gBAChD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;oBAAE,OAAO,UAAU,CAAC;YAC/C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;IACH,CAAC;IAED,iEAAiE;IACjE,OAAO,CAAC,GAAG,cAAc,IAAI,GAAG,CAAC,MAAM,cAAc,CAAC,CAAC;AACzD,CAAC;AAOD,SAAS,SAAS,CAAC,GAAW;IAC5B,OAAO,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACjC,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,GAAiB,EACjB,UAAkB,EAClB,QAAkB;IAElB,IAAI,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC;QAC1B,QAAQ,CAAC,IAAI,CAAC,4CAA4C,UAAU,EAAE,CAAC,CAAC;QACxE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,gBAAgB,EAAE,EAAE,EAAE,CAAC;IAC5C,CAAC;IAED,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAClD,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,gBAAgB,EAAE,EAAE,EAAE,CAAC;QAC5D,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,gBAAgB,CAAC,GAAG,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,gBAAgB,EAAE,EAAE,EAAE,CAAC;IAC5C,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,GAAiB,EACjB,QAAkB,EAClB,UAAkB,gBAAgB,EAClC,cAAuB;IAEvB,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;IACnE,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,WAAW,GAAG,cAAc,IAAI,GAAG,CAAC,MAAM,CAAC;IAEjD,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO;YAAE,MAAM;QAElC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;QAE7D,4CAA4C;QAC5C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO;gBAAE,MAAM;YAClC,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;gBACvB,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;oBAC7B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACjB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,sBAAsB;YACxB,CAAC;QACH,CAAC;QAED,oCAAoC;QACpC,IAAI,MAAM,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,CAAC;YAChE,KAAK,MAAM,aAAa,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBACpD,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO;oBAAE,MAAM;gBAElC,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;gBAEnE,KAAK,MAAM,GAAG,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC;oBACjC,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO;wBAAE,MAAM;oBAClC,IAAI,CAAC;wBACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;wBACvB,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;4BAC7B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACjB,CAAC;oBACH,CAAC;oBAAC,MAAM,CAAC;wBACP,sBAAsB;oBACxB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAiB;IACjD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,qEAAqE;IACrE,MAAM,UAAU,GAAG,MAAM,wBAAwB,CAAC,GAAG,CAAC,CAAC;IACvD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;IAEjE,kFAAkF;IAClF,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAChD,MAAM,WAAW,GAAG,MAAM,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAChD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;IACrE,CAAC;IAED,iBAAiB;IACjB,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC5D,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;IAEnE,cAAc;IACd,OAAO,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;AAC3C,CAAC;AASD;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,GAAiB;IAC5D,IAAI,GAAG,CAAC,aAAa;QAAE,OAAO,GAAG,CAAC,aAAa,CAAC;IAEhD,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC;IAE9C,kEAAkE;IAClE,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;QACxB,GAAG,CAAC,aAAa,GAAG;YAClB,IAAI,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC;YACnB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,KAAK;YACd,QAAQ,EAAE,EAAE;SACb,CAAC;QACF,OAAO,GAAG,CAAC,aAAa,CAAC;IAC3B,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC;IAC1B,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC;IAE/B,MAAM,OAAO,GAAG,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC;IACxD,IAAI,OAAO,EAAE,CAAC;QACZ,IAAI,QAAQ,KAAK,eAAe,EAAE,CAAC;YACjC,8EAA8E;YAC9E,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC;YACxD,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YAC5C,CAAC;YACD,IAAI,GAAG,MAAM,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,kCAAkC;YAClC,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAC9C,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1C,CAAC;YACD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED,GAAG,CAAC,aAAa,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,CAAC;IAChF,OAAO,GAAG,CAAC,aAAa,CAAC;AAC3B,CAAC"}

package/dist/helpers/index.d.ts

@@ -5,4 +5,9 @@ export { getPageUrls, discoverAndSamplePages, parseSitemapUrls, parseSitemapDire
 export type { PageUrlResult, SampledPages } from './get-page-urls.js';
 export { toMdUrls, isNonPageUrl } from './to-md-urls.js';
 export { htmlToMarkdown } from './html-to-markdown.js';
+export { fetchPage } from './fetch-page.js';
+export { detectTabGroups } from './detect-tabs.js';
+export type { DetectedTabGroup, TabPanel } from './detect-tabs.js';
+export { analyzeRendering } from './detect-rendering.js';
+export type { RenderingAnalysis } from './detect-rendering.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/helpers/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/helpers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EACL,WAAW,EACX,sBAAsB,EACtB,gBAAgB,EAChB,sBAAsB,GACvB,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/helpers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EACL,WAAW,EACX,sBAAsB,EACtB,gBAAgB,EAChB,sBAAsB,GACvB,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,YAAY,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,YAAY,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC"}

package/dist/helpers/index.js

@@ -4,4 +4,7 @@ export { looksLikeMarkdown, looksLikeHtml } from './detect-markdown.js';
 export { getPageUrls, discoverAndSamplePages, parseSitemapUrls, parseSitemapDirectives, } from './get-page-urls.js';
 export { toMdUrls, isNonPageUrl } from './to-md-urls.js';
 export { htmlToMarkdown } from './html-to-markdown.js';
+export { fetchPage } from './fetch-page.js';
+export { detectTabGroups } from './detect-tabs.js';
+export { analyzeRendering } from './detect-rendering.js';
 //# sourceMappingURL=index.js.map

package/dist/helpers/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/helpers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EACL,WAAW,EACX,sBAAsB,EACtB,gBAAgB,EAChB,sBAAsB,GACvB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/helpers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EACL,WAAW,EACX,sBAAsB,EACtB,gBAAgB,EAChB,sBAAsB,GACvB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC"}

package/dist/helpers/to-md-urls.d.ts

@@ -1,3 +1,7 @@
+/**
+ * Returns true if the two URLs have different hosts (i.e. a cross-host redirect).
+ */
+export declare function isCrossHostRedirect(originalUrl: string, finalUrl: string): boolean;
 /**
  * Returns true if the URL points to a non-page file type (e.g. .json, .xml, .txt)
  * where we would not expect a markdown equivalent.

package/dist/helpers/to-md-urls.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"to-md-urls.d.ts","sourceRoot":"","sources":["../../src/helpers/to-md-urls.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAUjD;AAED;;;;GAIG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CA2B9C"}
+{"version":3,"file":"to-md-urls.d.ts","sourceRoot":"","sources":["../../src/helpers/to-md-urls.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAQlF;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAUjD;AAED;;;;GAIG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CA2B9C"}

package/dist/helpers/to-md-urls.js

@@ -1,3 +1,16 @@
+/**
+ * Returns true if the two URLs have different hosts (i.e. a cross-host redirect).
+ */
+export function isCrossHostRedirect(originalUrl, finalUrl) {
+    try {
+        const original = new URL(originalUrl);
+        const final_ = new URL(finalUrl);
+        return original.host !== final_.host;
+    }
+    catch {
+        return false;
+    }
+}
 /**
  * Returns true if the URL points to a non-page file type (e.g. .json, .xml, .txt)
  * where we would not expect a markdown equivalent.

package/dist/helpers/to-md-urls.js.map

@@ -1 +1 @@
-{"version":3,"file":"to-md-urls.js","sourceRoot":"","sources":["../../src/helpers/to-md-urls.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;IAC3D,sDAAsD;IACtD,OAAO,CACL,eAAe,CAAC,IAAI,CAAC,WAAW,CAAC;QACjC,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC;QAC9B,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC;QAC5B,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,CAC9B,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAE5B,6DAA6D;IAC7D,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACxE,OAAO,CAAC,GAAG,CAAC,CAAC;IACf,CAAC;IAED,uEAAuE;IACvE,IAAI,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;IAC1D,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,oDAAoD;IACpD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC5C,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC;IAC7D,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;IAErC,uBAAuB;IACvB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC3C,OAAO,CAAC,QAAQ,GAAG,QAAQ,GAAG,WAAW,CAAC;IAC1C,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IAEpC,OAAO,UAAU,CAAC;AACpB,CAAC"}
+{"version":3,"file":"to-md-urls.js","sourceRoot":"","sources":["../../src/helpers/to-md-urls.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,WAAmB,EAAE,QAAgB;IACvE,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjC,OAAO,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;IAC3D,sDAAsD;IACtD,OAAO,CACL,eAAe,CAAC,IAAI,CAAC,WAAW,CAAC;QACjC,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC;QAC9B,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC;QAC5B,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,CAC9B,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAE5B,6DAA6D;IAC7D,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACxE,OAAO,CAAC,GAAG,CAAC,CAAC;IACf,CAAC;IAED,uEAAuE;IACvE,IAAI,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;IAC1D,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,oDAAoD;IACpD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC5C,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC;IAC7D,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;IAErC,uBAAuB;IACvB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC3C,OAAO,CAAC,QAAQ,GAAG,QAAQ,GAAG,WAAW,CAAC;IAC1C,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IAEpC,OAAO,UAAU,CAAC;AACpB,CAAC"}

package/dist/runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAe,aAAa,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAmCzF,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,GAAG,YAAY,CAgB7F;AAED,wBAAsB,SAAS,CAC7B,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,GAC/B,OAAO,CAAC,YAAY,CAAC,CAkEvB"}
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAe,aAAa,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAmCzF,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,GAAG,YAAY,CAiB7F;AAED,wBAAsB,SAAS,CAC7B,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,GAC/B,OAAO,CAAC,YAAY,CAAC,CAkEvB"}

package/dist/runner.js

@@ -42,6 +42,7 @@ export function createContext(baseUrl, options) {
         }),
         options: merged,
         pageCache: new Map(),
+        htmlCache: new Map(),
     };
 }
 export async function runChecks(baseUrl, options) {

package/dist/runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvD;;;;GAIG;AACH,SAAS,aAAa,CAAC,IAA2B;IAChD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,IAAI,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;QAChC,sDAAsD;QACtD,OAAQ,IAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,IAAkB,CAAC;AAC5B,CAAC;AAED,SAAS,oBAAoB,CAC3B,IAA2B,EAC3B,eAAyC;IAEzC,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACvC,KAAK,MAAM,OAAO,IAAI,UAAU,EAAE,CAAC;QACjC,kEAAkE;QAClE,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE;YACpC,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACvC,OAAO,MAAM,EAAE,MAAM,KAAK,MAAM,IAAI,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;QAChE,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,SAAS;YAAE,OAAO,KAAK,CAAC;IAC/B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAe,EAAE,OAAgC;IAC7E,MAAM,MAAM,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAC;IAClD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAE7B,OAAO;QACL,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,eAAe,EAAE,IAAI,GAAG,EAAE;QAC1B,IAAI,EAAE,gBAAgB,CAAC;YACrB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,cAAc,EAAE,MAAM,CAAC,cAAc;SACtC,CAAC;QACF,OAAO,EAAE,MAAM;QACf,SAAS,EAAE,IAAI,GAAG,EAAE;KACrB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,OAAe,EACf,OAAgC;IAEhC,MAAM,GAAG,GAAG,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;IACpC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;IAEnC,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;QAC9B,4CAA4C;QAC5C,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC;YACpE,SAAS;QACX,CAAC;QAED,0FAA0F;QAC1F,uFAAuF;QACvF,0BAA0B;QAC1B,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YAClD,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAClD,CAAC;YACF,IAAI,SAAS,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC;gBAC7E,MAAM,MAAM,GAAgB;oBAC1B,EAAE,EAAE,KAAK,CAAC,EAAE;oBACZ,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,wCAAwC;oBACjD,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE;iBAC7B,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrB,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;gBAC1C,SAAS;YACX,CAAC;QACH,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,MAAM,GAAgB;gBAC1B,EAAE,EAAE,KAAK,CAAC,EAAE;gBACZ,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,MAAM,EAAE,OAAO;gBACf,OAAO,EAAE,gBAAgB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;aAC5E,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG;QACd,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;QACvD,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;QACvD,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;QACvD,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;QACvD,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM;KAC1D,CAAC;IAEF,OAAO;QACL,GAAG,EAAE,OAAO;QACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,OAAO;QACP,OAAO;KACR,CAAC;AACJ,CAAC"}
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvD;;;;GAIG;AACH,SAAS,aAAa,CAAC,IAA2B;IAChD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,IAAI,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;QAChC,sDAAsD;QACtD,OAAQ,IAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,IAAkB,CAAC;AAC5B,CAAC;AAED,SAAS,oBAAoB,CAC3B,IAA2B,EAC3B,eAAyC;IAEzC,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACvC,KAAK,MAAM,OAAO,IAAI,UAAU,EAAE,CAAC;QACjC,kEAAkE;QAClE,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE;YACpC,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACvC,OAAO,MAAM,EAAE,MAAM,KAAK,MAAM,IAAI,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;QAChE,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,SAAS;YAAE,OAAO,KAAK,CAAC;IAC/B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAe,EAAE,OAAgC;IAC7E,MAAM,MAAM,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAC;IAClD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAE7B,OAAO;QACL,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,eAAe,EAAE,IAAI,GAAG,EAAE;QAC1B,IAAI,EAAE,gBAAgB,CAAC;YACrB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,cAAc,EAAE,MAAM,CAAC,cAAc;SACtC,CAAC;QACF,OAAO,EAAE,MAAM;QACf,SAAS,EAAE,IAAI,GAAG,EAAE;QACpB,SAAS,EAAE,IAAI,GAAG,EAAE;KACrB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,OAAe,EACf,OAAgC;IAEhC,MAAM,GAAG,GAAG,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;IACpC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;IAEnC,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;QAC9B,4CAA4C;QAC5C,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC;YACpE,SAAS;QACX,CAAC;QAED,0FAA0F;QAC1F,uFAAuF;QACvF,0BAA0B;QAC1B,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YAClD,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAClD,CAAC;YACF,IAAI,SAAS,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC;gBAC7E,MAAM,MAAM,GAAgB;oBAC1B,EAAE,EAAE,KAAK,CAAC,EAAE;oBACZ,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,wCAAwC;oBACjD,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE;iBAC7B,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrB,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;gBAC1C,SAAS;YACX,CAAC;QACH,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,MAAM,GAAgB;gBAC1B,EAAE,EAAE,KAAK,CAAC,EAAE;gBACZ,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,MAAM,EAAE,OAAO;gBACf,OAAO,EAAE,gBAAgB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;aAC5E,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG;QACd,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;QACvD,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;QACvD,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;QACvD,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;QACvD,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM;KAC1D,CAAC;IAEF,OAAO;QACL,GAAG,EAAE,OAAO;QACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,OAAO;QACP,OAAO;KACR,CAAC;AACJ,CAAC"}

package/dist/types.d.ts

@@ -15,11 +15,25 @@ export interface CachedPage {
         source: 'md-url' | 'content-negotiation';
     };
 }
+export interface FetchedPage {
+    url: string;
+    status: number;
+    body: string;
+    contentType: string;
+    isHtml: boolean;
+}
 export interface CheckContext {
     /** The base URL being checked (as provided by the user). */
     baseUrl: string;
     /** The origin (scheme + host) derived from baseUrl. */
     origin: string;
+    /**
+     * The actual origin where content lives, when the baseUrl origin redirects
+     * cross-host. Set by llms-txt-exists when it detects a cross-host redirect.
+     * Checks that need ground-truth data (e.g. sitemap for freshness) should
+     * use this over `origin`; checks that test agent experience should use `origin`.
+     */
+    effectiveOrigin?: string;
     /** Results from previously-run checks, keyed by check ID. */
     previousResults: Map<string, CheckResult>;
     /** HTTP client with rate limiting. */
@@ -28,9 +42,12 @@ export interface CheckContext {
     options: CheckOptions;
     /** Cached page content, keyed by original page URL. */
     pageCache: Map<string, CachedPage>;
+    /** Cached raw HTML fetches, keyed by URL. Shared across checks within a single run. */
+    htmlCache: Map<string, FetchedPage>;
     /** Cached sampled pages result, shared across checks within a single run. */
     _sampledPages?: SampledPages;
 }
+export type SamplingStrategy = 'random' | 'deterministic' | 'none';
 export interface CheckOptions {
     /** Maximum concurrent HTTP requests within a single check. */
     maxConcurrency: number;
@@ -40,6 +57,8 @@ export interface CheckOptions {
     requestTimeout: number;
     /** Maximum number of links to test in link-resolution checks. */
     maxLinksToTest: number;
+    /** URL sampling strategy: random (default), deterministic, or none. */
+    samplingStrategy: SamplingStrategy;
     /** Size thresholds. */
     thresholds: SizeThresholds;
 }

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE/D,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AAEtE,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,WAAW,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,CAAC,EAAE;QACT,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,QAAQ,GAAG,qBAAqB,CAAC;KAC1C,CAAC;CACH;AAED,MAAM,WAAW,YAAY;IAC3B,4DAA4D;IAC5D,OAAO,EAAE,MAAM,CAAC;IAChB,uDAAuD;IACvD,MAAM,EAAE,MAAM,CAAC;IACf,6DAA6D;IAC7D,eAAe,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAC1C,sCAAsC;IACtC,IAAI,EAAE,UAAU,CAAC;IACjB,uBAAuB;IACvB,OAAO,EAAE,YAAY,CAAC;IACtB,uDAAuD;IACvD,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACnC,6EAA6E;IAC7E,aAAa,CAAC,EAAE,YAAY,CAAC;CAC9B;AAED,MAAM,WAAW,YAAY;IAC3B,8DAA8D;IAC9D,cAAc,EAAE,MAAM,CAAC;IACvB,yCAAyC;IACzC,YAAY,EAAE,MAAM,CAAC;IACrB,kDAAkD;IAClD,cAAc,EAAE,MAAM,CAAC;IACvB,iEAAiE;IACjE,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB;IACvB,UAAU,EAAE,cAAc,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,8DAA8D;IAC9D,IAAI,EAAE,MAAM,CAAC;IACb,8DAA8D;IAC9D,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,MAAM,aAAa,GAAG,CAAC,GAAG,EAAE,YAAY,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC;AAExE,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,8FAA8F;IAC9F,SAAS,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,CAAC;IACjC,GAAG,EAAE,aAAa,CAAC;CACpB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC/B,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,OAAO,CAAC;IACpB,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,OAAO,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,aAAc,SAAQ,YAAY;IACjD,6DAA6D;IAC7D,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;CACjC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE/D,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AAEtE,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,WAAW,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,CAAC,EAAE;QACT,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,QAAQ,GAAG,qBAAqB,CAAC;KAC1C,CAAC;CACH;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B,4DAA4D;IAC5D,OAAO,EAAE,MAAM,CAAC;IAChB,uDAAuD;IACvD,MAAM,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,6DAA6D;IAC7D,eAAe,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAC1C,sCAAsC;IACtC,IAAI,EAAE,UAAU,CAAC;IACjB,uBAAuB;IACvB,OAAO,EAAE,YAAY,CAAC;IACtB,uDAAuD;IACvD,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACnC,uFAAuF;IACvF,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACpC,6EAA6E;IAC7E,aAAa,CAAC,EAAE,YAAY,CAAC;CAC9B;AAED,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,eAAe,GAAG,MAAM,CAAC;AAEnE,MAAM,WAAW,YAAY;IAC3B,8DAA8D;IAC9D,cAAc,EAAE,MAAM,CAAC;IACvB,yCAAyC;IACzC,YAAY,EAAE,MAAM,CAAC;IACrB,kDAAkD;IAClD,cAAc,EAAE,MAAM,CAAC;IACvB,iEAAiE;IACjE,cAAc,EAAE,MAAM,CAAC;IACvB,uEAAuE;IACvE,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,uBAAuB;IACvB,UAAU,EAAE,cAAc,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,8DAA8D;IAC9D,IAAI,EAAE,MAAM,CAAC;IACb,8DAA8D;IAC9D,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,MAAM,aAAa,GAAG,CAAC,GAAG,EAAE,YAAY,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC;AAExE,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,8FAA8F;IAC9F,SAAS,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,CAAC;IACjC,GAAG,EAAE,aAAa,CAAC;CACpB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC/B,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,OAAO,CAAC;IACpB,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,OAAO,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,aAAc,SAAQ,YAAY;IACjD,6DAA6D;IAC7D,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;CACjC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "afdocs",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "Test your documentation site against the Agent-Friendly Documentation Spec",
   "type": "module",
   "main": "dist/index.js",
@@ -60,6 +60,7 @@
   "dependencies": {
     "chalk": "^5.4.1",
     "commander": "^13.1.0",
+    "node-html-parser": "^7.1.0",
     "turndown": "^7.2.2",
     "yaml": "^2.7.0"
   },