afdocs 0.1.3 → 0.2.0

package/dist/checks/markdown-availability/content-negotiation.js

@@ -1,10 +1,106 @@
 import { registerCheck } from '../registry.js';
-async function check(_ctx) {
+import { looksLikeMarkdown } from '../../helpers/detect-markdown.js';
+import { discoverAndSamplePages } from '../../helpers/get-page-urls.js';
+async function check(ctx) {
+    const id = 'content-negotiation';
+    const category = 'markdown-availability';
+    const { urls: pageUrls, totalPages, sampled: wasSampled, warnings, } = await discoverAndSamplePages(ctx);
+    const results = [];
+    const concurrency = ctx.options.maxConcurrency;
+    for (let i = 0; i < pageUrls.length; i += concurrency) {
+        const batch = pageUrls.slice(i, i + concurrency);
+        const batchResults = await Promise.all(batch.map(async (url) => {
+            try {
+                const response = await ctx.http.fetch(url, {
+                    headers: { Accept: 'text/markdown' },
+                });
+                const body = await response.text();
+                const contentType = response.headers.get('content-type') ?? '';
+                const isMarkdownType = contentType.includes('text/markdown');
+                const isMarkdownBody = looksLikeMarkdown(body);
+                let classification;
+                if (isMarkdownType && isMarkdownBody) {
+                    classification = 'markdown-with-correct-type';
+                    // Cache the markdown content (only if not already cached by md-url check)
+                    if (!ctx.pageCache.has(url)) {
+                        ctx.pageCache.set(url, {
+                            url,
+                            markdown: { content: body, source: 'content-negotiation' },
+                        });
+                    }
+                }
+                else if (isMarkdownBody) {
+                    classification = 'markdown-with-wrong-type';
+                    if (!ctx.pageCache.has(url)) {
+                        ctx.pageCache.set(url, {
+                            url,
+                            markdown: { content: body, source: 'content-negotiation' },
+                        });
+                    }
+                }
+                else {
+                    classification = 'html';
+                }
+                return { url, classification, contentType, status: response.status };
+            }
+            catch (err) {
+                return {
+                    url,
+                    classification: 'html',
+                    contentType: '',
+                    status: 0,
+                    error: err instanceof Error ? err.message : String(err),
+                };
+            }
+        }));
+        results.push(...batchResults);
+    }
+    const markdownWithCorrectType = results.filter((r) => r.classification === 'markdown-with-correct-type').length;
+    const markdownWithWrongType = results.filter((r) => r.classification === 'markdown-with-wrong-type').length;
+    const htmlOnly = results.filter((r) => r.classification === 'html').length;
+    const negotiationRate = Math.round((markdownWithCorrectType / results.length) * 100);
+    const fetchErrors = results.filter((r) => r.error).length;
+    const rateLimited = results.filter((r) => r.status === 429).length;
+    const pageLabel = wasSampled ? 'sampled pages' : 'pages';
+    const suffix = (fetchErrors > 0 ? `; ${fetchErrors} failed to fetch` : '') +
+        (rateLimited > 0 ? `; ${rateLimited} rate-limited (HTTP 429)` : '');
+    const details = {
+        totalPages,
+        testedPages: results.length,
+        sampled: wasSampled,
+        markdownWithCorrectType,
+        markdownWithWrongType,
+        htmlOnly,
+        negotiationRate,
+        fetchErrors,
+        rateLimited,
+        pageResults: results,
+        discoveryWarnings: warnings,
+    };
+    if (negotiationRate >= 90) {
+        return {
+            id,
+            category,
+            status: 'pass',
+            message: `${markdownWithCorrectType}/${results.length} ${pageLabel} support content negotiation (${negotiationRate}%)${suffix}`,
+            details,
+        };
+    }
+    if (markdownWithCorrectType > 0 || markdownWithWrongType > 0) {
+        return {
+            id,
+            category,
+            status: 'warn',
+            message: `Content negotiation partially supported: ${markdownWithCorrectType} correct type, ${markdownWithWrongType} wrong type, ${htmlOnly} HTML only${suffix}`,
+            details,
+        };
+    }
     return {
-        id: 'content-negotiation',
-        category: 'markdown-availability',
-        status: 'skip',
-        message: 'Not yet implemented',
+        id,
+        category,
+        status: 'fail',
+        message: `Server ignores Accept: text/markdown header (0/${results.length} ${pageLabel} return markdown)${suffix}`,
+        details,
     };
 }
 registerCheck({

package/dist/checks/markdown-availability/content-negotiation.js.map

@@ -1 +1 @@
-{"version":3,"file":"content-negotiation.js","sourceRoot":"","sources":["../../../src/checks/markdown-availability/content-negotiation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C,KAAK,UAAU,KAAK,CAAC,IAAkB;IACrC,OAAO;QACL,EAAE,EAAE,qBAAqB;QACzB,QAAQ,EAAE,uBAAuB;QACjC,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,qBAAqB;KAC/B,CAAC;AACJ,CAAC;AAED,aAAa,CAAC;IACZ,EAAE,EAAE,qBAAqB;IACzB,QAAQ,EAAE,uBAAuB;IACjC,WAAW,EAAE,sDAAsD;IACnE,SAAS,EAAE,EAAE;IACb,GAAG,EAAE,KAAK;CACX,CAAC,CAAC"}
+{"version":3,"file":"content-negotiation.js","sourceRoot":"","sources":["../../../src/checks/markdown-availability/content-negotiation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAaxE,KAAK,UAAU,KAAK,CAAC,GAAiB;IACpC,MAAM,EAAE,GAAG,qBAAqB,CAAC;IACjC,MAAM,QAAQ,GAAG,uBAAuB,CAAC;IAEzC,MAAM,EACJ,IAAI,EAAE,QAAQ,EACd,UAAU,EACV,OAAO,EAAE,UAAU,EACnB,QAAQ,GACT,GAAG,MAAM,sBAAsB,CAAC,GAAG,CAAC,CAAC;IAEtC,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC;IAE/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAuB,EAAE;YAC3C,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE;oBACzC,OAAO,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE;iBACrC,CAAC,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;gBAC/D,MAAM,cAAc,GAAG,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;gBAC7D,MAAM,cAAc,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;gBAE/C,IAAI,cAA8B,CAAC;gBACnC,IAAI,cAAc,IAAI,cAAc,EAAE,CAAC;oBACrC,cAAc,GAAG,4BAA4B,CAAC;oBAC9C,0EAA0E;oBAC1E,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC5B,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;4BACrB,GAAG;4BACH,QAAQ,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,qBAAqB,EAAE;yBAC3D,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;qBAAM,IAAI,cAAc,EAAE,CAAC;oBAC1B,cAAc,GAAG,0BAA0B,CAAC;oBAC5C,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC5B,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;4BACrB,GAAG;4BACH,QAAQ,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,qBAAqB,EAAE;yBAC3D,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,cAAc,GAAG,MAAM,CAAC;gBAC1B,CAAC;gBAED,OAAO,EAAE,GAAG,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;YACvE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO;oBACL,GAAG;oBACH,cAAc,EAAE,MAAM;oBACtB,WAAW,EAAE,EAAE;oBACf,MAAM,EAAE,CAAC;oBACT,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBACxD,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,uBAAuB,GAAG,OAAO,CAAC,MAAM,CAC5C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,4BAA4B,CACzD,CAAC,MAAM,CAAC;IACT,MAAM,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAC1C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,0BAA0B,CACvD,CAAC,MAAM,CAAC;IACT,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3E,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,uBAAuB,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;IACrF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;IAC1D,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC;IAEnE,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC;IACzD,MAAM,MAAM,GACV,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,0BAA0B,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAEtE,MAAM,OAAO,GAA4B;QACvC,UAAU;QACV,WAAW,EAAE,OAAO,CAAC,MAAM;QAC3B,OAAO,EAAE,UAAU;QACnB,uBAAuB;QACvB,qBAAqB;QACrB,QAAQ;QACR,eAAe;QACf,WAAW;QACX,WAAW;QACX,WAAW,EAAE,OAAO;QACpB,iBAAiB,EAAE,QAAQ;KAC5B,CAAC;IAEF,IAAI,eAAe,IAAI,EAAE,EAAE,CAAC;QAC1B,OAAO;YACL,EAAE;YACF,QAAQ;YACR,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,uBAAuB,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,iCAAiC,eAAe,KAAK,MAAM,EAAE;YAC/H,OAAO;SACR,CAAC;IACJ,CAAC;IAED,IAAI,uBAAuB,GAAG,CAAC,IAAI,qBAAqB,GAAG,CAAC,EAAE,CAAC;QAC7D,OAAO;YACL,EAAE;YACF,QAAQ;YACR,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,4CAA4C,uBAAuB,kBAAkB,qBAAqB,gBAAgB,QAAQ,aAAa,MAAM,EAAE;YAChK,OAAO;SACR,CAAC;IACJ,CAAC;IAED,OAAO;QACL,EAAE;QACF,QAAQ;QACR,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,kDAAkD,OAAO,CAAC,MAAM,IAAI,SAAS,oBAAoB,MAAM,EAAE;QAClH,OAAO;KACR,CAAC;AACJ,CAAC;AAED,aAAa,CAAC;IACZ,EAAE,EAAE,qBAAqB;IACzB,QAAQ,EAAE,uBAAuB;IACjC,WAAW,EAAE,sDAAsD;IACnE,SAAS,EAAE,EAAE;IACb,GAAG,EAAE,KAAK;CACX,CAAC,CAAC"}

package/dist/checks/markdown-availability/markdown-url-support.js

@@ -1,10 +1,87 @@
 import { registerCheck } from '../registry.js';
-async function check(_ctx) {
+import { looksLikeMarkdown } from '../../helpers/detect-markdown.js';
+import { discoverAndSamplePages } from '../../helpers/get-page-urls.js';
+import { toMdUrls } from '../../helpers/to-md-urls.js';
+async function check(ctx) {
+    const id = 'markdown-url-support';
+    const category = 'markdown-availability';
+    const { urls: pageUrls, totalPages, sampled: wasSampled, warnings, } = await discoverAndSamplePages(ctx);
+    const results = [];
+    const concurrency = ctx.options.maxConcurrency;
+    for (let i = 0; i < pageUrls.length; i += concurrency) {
+        const batch = pageUrls.slice(i, i + concurrency);
+        const batchResults = await Promise.all(batch.map(async (url) => {
+            const candidates = toMdUrls(url);
+            let lastError;
+            for (const mdUrl of candidates) {
+                try {
+                    const response = await ctx.http.fetch(mdUrl);
+                    const body = await response.text();
+                    const contentType = response.headers.get('content-type') ?? '';
+                    const isMarkdownType = contentType.includes('text/markdown');
+                    const isMarkdownBody = looksLikeMarkdown(body);
+                    const supported = response.ok && (isMarkdownType || isMarkdownBody);
+                    if (supported) {
+                        ctx.pageCache.set(url, {
+                            url,
+                            markdown: { content: body, source: 'md-url' },
+                        });
+                        return { url, mdUrl, supported: true, status: response.status };
+                    }
+                    lastError = undefined; // Got a response, not a fetch error
+                }
+                catch (err) {
+                    lastError = err instanceof Error ? err.message : String(err);
+                }
+            }
+            return { url, mdUrl: candidates[0], supported: false, status: 0, error: lastError };
+        }));
+        results.push(...batchResults);
+    }
+    const mdSupported = results.filter((r) => r.supported).length;
+    const mdUnsupported = results.length - mdSupported;
+    const supportRate = Math.round((mdSupported / results.length) * 100);
+    const fetchErrors = results.filter((r) => r.error).length;
+    const rateLimited = results.filter((r) => r.status === 429).length;
+    const pageLabel = wasSampled ? 'sampled pages' : 'pages';
+    const suffix = (fetchErrors > 0 ? `; ${fetchErrors} failed to fetch` : '') +
+        (rateLimited > 0 ? `; ${rateLimited} rate-limited (HTTP 429)` : '');
+    const details = {
+        totalPages,
+        testedPages: results.length,
+        sampled: wasSampled,
+        mdSupported,
+        mdUnsupported,
+        supportRate,
+        fetchErrors,
+        rateLimited,
+        pageResults: results,
+        discoveryWarnings: warnings,
+    };
+    if (supportRate >= 90) {
+        return {
+            id,
+            category,
+            status: 'pass',
+            message: `${mdSupported}/${results.length} ${pageLabel} support .md URLs (${supportRate}%)${suffix}`,
+            details,
+        };
+    }
+    if (mdSupported > 0) {
+        return {
+            id,
+            category,
+            status: 'warn',
+            message: `${mdSupported}/${results.length} ${pageLabel} support .md URLs (${supportRate}%); inconsistent support${suffix}`,
+            details,
+        };
+    }
     return {
-        id: 'markdown-url-support',
-        category: 'markdown-availability',
-        status: 'skip',
-        message: 'Not yet implemented',
+        id,
+        category,
+        status: 'fail',
+        message: `No ${pageLabel} support .md URLs (0/${results.length} tested)${suffix}`,
+        details,
     };
 }
 registerCheck({

package/dist/checks/markdown-availability/markdown-url-support.js.map

@@ -1 +1 @@
-{"version":3,"file":"markdown-url-support.js","sourceRoot":"","sources":["../../../src/checks/markdown-availability/markdown-url-support.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C,KAAK,UAAU,KAAK,CAAC,IAAkB;IACrC,OAAO;QACL,EAAE,EAAE,sBAAsB;QAC1B,QAAQ,EAAE,uBAAuB;QACjC,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,qBAAqB;KAC/B,CAAC;AACJ,CAAC;AAED,aAAa,CAAC;IACZ,EAAE,EAAE,sBAAsB;IAC1B,QAAQ,EAAE,uBAAuB;IACjC,WAAW,EAAE,2DAA2D;IACxE,SAAS,EAAE,EAAE;IACb,GAAG,EAAE,KAAK;CACX,CAAC,CAAC"}
+{"version":3,"file":"markdown-url-support.js","sourceRoot":"","sources":["../../../src/checks/markdown-availability/markdown-url-support.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAWvD,KAAK,UAAU,KAAK,CAAC,GAAiB;IACpC,MAAM,EAAE,GAAG,sBAAsB,CAAC;IAClC,MAAM,QAAQ,GAAG,uBAAuB,CAAC;IAEzC,MAAM,EACJ,IAAI,EAAE,QAAQ,EACd,UAAU,EACV,OAAO,EAAE,UAAU,EACnB,QAAQ,GACT,GAAG,MAAM,sBAAsB,CAAC,GAAG,CAAC,CAAC;IAEtC,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC;IAE/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAuB,EAAE;YAC3C,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,SAA6B,CAAC;YAClC,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;gBAC/B,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAC7C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;oBACnC,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;oBAC/D,MAAM,cAAc,GAAG,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;oBAC7D,MAAM,cAAc,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;oBAC/C,MAAM,SAAS,GAAG,QAAQ,CAAC,EAAE,IAAI,CAAC,cAAc,IAAI,cAAc,CAAC,CAAC;oBAEpE,IAAI,SAAS,EAAE,CAAC;wBACd,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;4BACrB,GAAG;4BACH,QAAQ,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE;yBAC9C,CAAC,CAAC;wBACH,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;oBAClE,CAAC;oBACD,SAAS,GAAG,SAAS,CAAC,CAAC,oCAAoC;gBAC7D,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,SAAS,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC/D,CAAC;YACH,CAAC;YACD,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;QACtF,CAAC,CAAC,CACH,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC;IAC9D,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,GAAG,WAAW,CAAC;IACnD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;IACrE,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;IAC1D,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC;IAEnE,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC;IACzD,MAAM,MAAM,GACV,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,0BAA0B,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAEtE,MAAM,OAAO,GAA4B;QACvC,UAAU;QACV,WAAW,EAAE,OAAO,CAAC,MAAM;QAC3B,OAAO,EAAE,UAAU;QACnB,WAAW;QACX,aAAa;QACb,WAAW;QACX,WAAW;QACX,WAAW;QACX,WAAW,EAAE,OAAO;QACpB,iBAAiB,EAAE,QAAQ;KAC5B,CAAC;IAEF,IAAI,WAAW,IAAI,EAAE,EAAE,CAAC;QACtB,OAAO;YACL,EAAE;YACF,QAAQ;YACR,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,WAAW,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,sBAAsB,WAAW,KAAK,MAAM,EAAE;YACpG,OAAO;SACR,CAAC;IACJ,CAAC;IAED,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACpB,OAAO;YACL,EAAE;YACF,QAAQ;YACR,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,WAAW,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,sBAAsB,WAAW,2BAA2B,MAAM,EAAE;YAC1H,OAAO;SACR,CAAC;IACJ,CAAC;IAED,OAAO;QACL,EAAE;QACF,QAAQ;QACR,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,MAAM,SAAS,wBAAwB,OAAO,CAAC,MAAM,WAAW,MAAM,EAAE;QACjF,OAAO;KACR,CAAC;AACJ,CAAC;AAED,aAAa,CAAC;IACZ,EAAE,EAAE,sBAAsB;IAC1B,QAAQ,EAAE,uBAAuB;IACjC,WAAW,EAAE,2DAA2D;IACxE,SAAS,EAAE,EAAE;IACb,GAAG,EAAE,KAAK;CACX,CAAC,CAAC"}

package/dist/checks/page-size/content-start-position.js

@@ -1,10 +1,166 @@
 import { registerCheck } from '../registry.js';
-async function check(_ctx) {
+import { looksLikeHtml } from '../../helpers/detect-markdown.js';
+import { discoverAndSamplePages } from '../../helpers/get-page-urls.js';
+import { htmlToMarkdown } from '../../helpers/html-to-markdown.js';
+const CSS_PATTERN = /[{}\s]*[a-z-]+\s*:\s*[^;]+;/;
+const JS_PATTERNS = [/^\s*(function|var|const|let|import|export)\b/, /^\s*\/\//, /[{};]\s*$/];
+const NAV_MAX_LENGTH = 40;
+/**
+ * Find the character position where meaningful content begins in converted markdown.
+ * Meaningful content is a heading or a prose paragraph (not CSS, JS, or short nav text).
+ */
+function findContentStart(markdown) {
+    const lines = markdown.split('\n');
+    let charPos = 0;
+    for (let idx = 0; idx < lines.length; idx++) {
+        const line = lines[idx];
+        const trimmed = line.trim();
+        if (trimmed.length === 0) {
+            charPos += line.length + 1;
+            continue;
+        }
+        // ATX heading: starts with # at beginning of line
+        if (/^#{1,6}\s+\S/.test(trimmed)) {
+            return charPos;
+        }
+        // Setext heading: current line is text, next line is === or ---
+        const nextLine = idx + 1 < lines.length ? lines[idx + 1].trim() : '';
+        if (/^[=-]+$/.test(nextLine) && nextLine.length >= 2 && trimmed.length > 0) {
+            return charPos;
+        }
+        // Skip CSS-like lines
+        if (CSS_PATTERN.test(trimmed)) {
+            charPos += line.length + 1;
+            continue;
+        }
+        // Skip JS-like lines
+        if (JS_PATTERNS.some((p) => p.test(trimmed))) {
+            charPos += line.length + 1;
+            continue;
+        }
+        // Skip very short nav-like tokens (e.g., "Home", "Docs", "API")
+        if (trimmed.length <= NAV_MAX_LENGTH && !/[.!?]/.test(trimmed) && !trimmed.includes(' ')) {
+            charPos += line.length + 1;
+            continue;
+        }
+        // Prose-like paragraph: contains spaces (multiple words) and is reasonably long
+        if (trimmed.length > NAV_MAX_LENGTH || (trimmed.includes(' ') && trimmed.length > 20)) {
+            return charPos;
+        }
+        charPos += line.length + 1;
+    }
+    // If nothing matched, content starts at the end (all boilerplate)
+    return charPos;
+}
+function positionStatus(percent) {
+    if (percent <= 10)
+        return 'pass';
+    if (percent <= 50)
+        return 'warn';
+    return 'fail';
+}
+function worstStatus(statuses) {
+    if (statuses.includes('fail'))
+        return 'fail';
+    if (statuses.includes('warn'))
+        return 'warn';
+    return 'pass';
+}
+async function check(ctx) {
+    const id = 'content-start-position';
+    const category = 'page-size';
+    const { urls: pageUrls, totalPages, sampled: wasSampled, warnings, } = await discoverAndSamplePages(ctx);
+    const results = [];
+    const concurrency = ctx.options.maxConcurrency;
+    for (let i = 0; i < pageUrls.length; i += concurrency) {
+        const batch = pageUrls.slice(i, i + concurrency);
+        const batchResults = await Promise.all(batch.map(async (url) => {
+            try {
+                const response = await ctx.http.fetch(url);
+                const body = await response.text();
+                const contentType = response.headers.get('content-type') ?? '';
+                const isMarkdownType = contentType.includes('text/markdown') || contentType.includes('text/plain');
+                const isHtml = !isMarkdownType && (contentType.includes('text/html') || looksLikeHtml(body));
+                const markdown = isHtml ? htmlToMarkdown(body) : body;
+                const totalChars = markdown.length;
+                const contentStartChar = findContentStart(markdown);
+                const contentStartPercent = totalChars > 0 ? Math.round((contentStartChar / totalChars) * 100) : 0;
+                return {
+                    url,
+                    contentStartChar,
+                    totalChars,
+                    contentStartPercent,
+                    status: positionStatus(contentStartPercent),
+                };
+            }
+            catch (err) {
+                return {
+                    url,
+                    contentStartChar: 0,
+                    totalChars: 0,
+                    contentStartPercent: 100,
+                    status: 'fail',
+                    error: err instanceof Error ? err.message : String(err),
+                };
+            }
+        }));
+        results.push(...batchResults);
+    }
+    const successful = results.filter((r) => !r.error);
+    const fetchErrors = results.filter((r) => r.error).length;
+    if (successful.length === 0) {
+        return {
+            id,
+            category,
+            status: 'fail',
+            message: `Could not fetch any pages to analyze${fetchErrors > 0 ? `; ${fetchErrors} failed to fetch` : ''}`,
+            details: {
+                totalPages,
+                testedPages: results.length,
+                sampled: wasSampled,
+                fetchErrors,
+                pageResults: results,
+                discoveryWarnings: warnings,
+            },
+        };
+    }
+    const percents = successful.map((r) => r.contentStartPercent);
+    const medianPercent = percents.sort((a, b) => a - b)[Math.floor(percents.length / 2)];
+    const maxPercent = Math.max(...percents);
+    const overallStatus = worstStatus(successful.map((r) => r.status));
+    const pageLabel = wasSampled ? 'sampled pages' : 'pages';
+    const passBucket = successful.filter((r) => r.status === 'pass').length;
+    const warnBucket = successful.filter((r) => r.status === 'warn').length;
+    const failBucket = successful.filter((r) => r.status === 'fail').length;
+    const suffix = fetchErrors > 0 ? `; ${fetchErrors} failed to fetch` : '';
+    let message;
+    if (overallStatus === 'pass') {
+        message = `Content starts within first 10% on all ${successful.length} ${pageLabel} (median ${medianPercent}%)${suffix}`;
+    }
+    else if (overallStatus === 'warn') {
+        message = `${warnBucket} of ${successful.length} ${pageLabel} have content starting at 10–50% (worst ${maxPercent}%)${suffix}`;
+    }
+    else {
+        message = `${failBucket} of ${successful.length} ${pageLabel} have content starting past 50% (worst ${maxPercent}%)${suffix}`;
+    }
     return {
-        id: 'content-start-position',
-        category: 'page-size',
-        status: 'skip',
-        message: 'Not yet implemented',
+        id,
+        category,
+        status: overallStatus,
+        message,
+        details: {
+            totalPages,
+            testedPages: results.length,
+            sampled: wasSampled,
+            medianPercent,
+            maxPercent,
+            passBucket,
+            warnBucket,
+            failBucket,
+            fetchErrors,
+            pageResults: results,
+            discoveryWarnings: warnings,
+        },
     };
 }
 registerCheck({

package/dist/checks/page-size/content-start-position.js.map

@@ -1 +1 @@
-{"version":3,"file":"content-start-position.js","sourceRoot":"","sources":["../../../src/checks/page-size/content-start-position.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C,KAAK,UAAU,KAAK,CAAC,IAAkB;IACrC,OAAO;QACL,EAAE,EAAE,wBAAwB;QAC5B,QAAQ,EAAE,WAAW;QACrB,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,qBAAqB;KAC/B,CAAC;AACJ,CAAC;AAED,aAAa,CAAC;IACZ,EAAE,EAAE,wBAAwB;IAC5B,QAAQ,EAAE,WAAW;IACrB,WAAW,EAAE,sDAAsD;IACnE,SAAS,EAAE,EAAE;IACb,GAAG,EAAE,KAAK;CACX,CAAC,CAAC"}
+{"version":3,"file":"content-start-position.js","sourceRoot":"","sources":["../../../src/checks/page-size/content-start-position.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,kCAAkC,CAAC;AACjE,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AAYnE,MAAM,WAAW,GAAG,6BAA6B,CAAC;AAClD,MAAM,WAAW,GAAG,CAAC,8CAA8C,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;AAC9F,MAAM,cAAc,GAAG,EAAE,CAAC;AAE1B;;;GAGG;AACH,SAAS,gBAAgB,CAAC,QAAgB;IACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QACxB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAE5B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAC3B,SAAS;QACX,CAAC;QAED,kDAAkD;QAClD,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,gEAAgE;QAChE,MAAM,QAAQ,GAAG,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrE,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3E,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,sBAAsB;QACtB,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAC3B,SAAS;QACX,CAAC;QAED,qBAAqB;QACrB,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;YAC7C,OAAO,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAC3B,SAAS;QACX,CAAC;QAED,gEAAgE;QAChE,IAAI,OAAO,CAAC,MAAM,IAAI,cAAc,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACzF,OAAO,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAC3B,SAAS;QACX,CAAC;QAED,gFAAgF;QAChF,IAAI,OAAO,CAAC,MAAM,GAAG,cAAc,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,CAAC;YACtF,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,OAAO,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,kEAAkE;IAClE,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,cAAc,CAAC,OAAe;IACrC,IAAI,OAAO,IAAI,EAAE;QAAE,OAAO,MAAM,CAAC;IACjC,IAAI,OAAO,IAAI,EAAE;QAAE,OAAO,MAAM,CAAC;IACjC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,QAAuB;IAC1C,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,MAAM,CAAC;IAC7C,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,MAAM,CAAC;IAC7C,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,GAAiB;IACpC,MAAM,EAAE,GAAG,wBAAwB,CAAC;IACpC,MAAM,QAAQ,GAAG,WAAW,CAAC;IAE7B,MAAM,EACJ,IAAI,EAAE,QAAQ,EACd,UAAU,EACV,OAAO,EAAE,UAAU,EACnB,QAAQ,GACT,GAAG,MAAM,sBAAsB,CAAC,GAAG,CAAC,CAAC;IAEtC,MAAM,OAAO,GAAyB,EAAE,CAAC;IACzC,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC;IAE/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAA+B,EAAE;YACnD,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC3C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;gBAC/D,MAAM,cAAc,GAClB,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;gBAC9E,MAAM,MAAM,GACV,CAAC,cAAc,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;gBAChF,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBACtD,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC;gBACnC,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;gBACpD,MAAM,mBAAmB,GACvB,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,gBAAgB,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAEzE,OAAO;oBACL,GAAG;oBACH,gBAAgB;oBAChB,UAAU;oBACV,mBAAmB;oBACnB,MAAM,EAAE,cAAc,CAAC,mBAAmB,CAAC;iBAC5C,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO;oBACL,GAAG;oBACH,gBAAgB,EAAE,CAAC;oBACnB,UAAU,EAAE,CAAC;oBACb,mBAAmB,EAAE,GAAG;oBACxB,MAAM,EAAE,MAAM;oBACd,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBACxD,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACnD,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;IAE1D,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO;YACL,EAAE;YACF,QAAQ;YACR,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,uCAAuC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,kBAAkB,CAAC,CAAC,CAAC,EAAE,EAAE;YAC3G,OAAO,EAAE;gBACP,UAAU;gBACV,WAAW,EAAE,OAAO,CAAC,MAAM;gBAC3B,OAAO,EAAE,UAAU;gBACnB,WAAW;gBACX,WAAW,EAAE,OAAO;gBACpB,iBAAiB,EAAE,QAAQ;aAC5B;SACF,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC;IAC9D,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IACtF,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;IAEzC,MAAM,aAAa,GAAG,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACnE,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC;IAEzD,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAExE,MAAM,MAAM,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;IAEzE,IAAI,OAAe,CAAC;IACpB,IAAI,aAAa,KAAK,MAAM,EAAE,CAAC;QAC7B,OAAO,GAAG,0CAA0C,UAAU,CAAC,MAAM,IAAI,SAAS,YAAY,aAAa,KAAK,MAAM,EAAE,CAAC;IAC3H,CAAC;SAAM,IAAI,aAAa,KAAK,MAAM,EAAE,CAAC;QACpC,OAAO,GAAG,GAAG,UAAU,OAAO,UAAU,CAAC,MAAM,IAAI,SAAS,2CAA2C,UAAU,KAAK,MAAM,EAAE,CAAC;IACjI,CAAC;SAAM,CAAC;QACN,OAAO,GAAG,GAAG,UAAU,OAAO,UAAU,CAAC,MAAM,IAAI,SAAS,0CAA0C,UAAU,KAAK,MAAM,EAAE,CAAC;IAChI,CAAC;IAED,OAAO;QACL,EAAE;QACF,QAAQ;QACR,MAAM,EAAE,aAAa;QACrB,OAAO;QACP,OAAO,EAAE;YACP,UAAU;YACV,WAAW,EAAE,OAAO,CAAC,MAAM;YAC3B,OAAO,EAAE,UAAU;YACnB,aAAa;YACb,UAAU;YACV,UAAU;YACV,UAAU;YACV,UAAU;YACV,WAAW;YACX,WAAW,EAAE,OAAO;YACpB,iBAAiB,EAAE,QAAQ;SAC5B;KACF,CAAC;AACJ,CAAC;AAED,aAAa,CAAC;IACZ,EAAE,EAAE,wBAAwB;IAC5B,QAAQ,EAAE,WAAW;IACrB,WAAW,EAAE,sDAAsD;IACnE,SAAS,EAAE,EAAE;IACb,GAAG,EAAE,KAAK;CACX,CAAC,CAAC"}

package/dist/checks/page-size/page-size-html.js

@@ -1,10 +1,132 @@
 import { registerCheck } from '../registry.js';
-async function check(_ctx) {
+import { looksLikeHtml } from '../../helpers/detect-markdown.js';
+import { discoverAndSamplePages } from '../../helpers/get-page-urls.js';
+import { htmlToMarkdown } from '../../helpers/html-to-markdown.js';
+function sizeStatus(chars, pass, fail) {
+    if (chars <= pass)
+        return 'pass';
+    if (chars <= fail)
+        return 'warn';
+    return 'fail';
+}
+function worstStatus(statuses) {
+    if (statuses.includes('fail'))
+        return 'fail';
+    if (statuses.includes('warn'))
+        return 'warn';
+    return 'pass';
+}
+function formatSize(chars) {
+    if (chars >= 1000)
+        return `${Math.round(chars / 1000)}K`;
+    return String(chars);
+}
+async function check(ctx) {
+    const id = 'page-size-html';
+    const category = 'page-size';
+    const { pass: passThreshold, fail: failThreshold } = ctx.options.thresholds;
+    const { urls: pageUrls, totalPages, sampled: wasSampled, warnings, } = await discoverAndSamplePages(ctx);
+    const results = [];
+    const concurrency = ctx.options.maxConcurrency;
+    for (let i = 0; i < pageUrls.length; i += concurrency) {
+        const batch = pageUrls.slice(i, i + concurrency);
+        const batchResults = await Promise.all(batch.map(async (url) => {
+            try {
+                const response = await ctx.http.fetch(url);
+                const body = await response.text();
+                const contentType = response.headers.get('content-type') ?? '';
+                const isMarkdownType = contentType.includes('text/markdown') || contentType.includes('text/plain');
+                const isHtml = !isMarkdownType && (contentType.includes('text/html') || looksLikeHtml(body));
+                // Skip conversion if the response is already markdown
+                const html = isHtml ? body : '';
+                const htmlChars = html.length;
+                const converted = isHtml ? htmlToMarkdown(body) : body;
+                const convertedChars = converted.length;
+                const ratio = htmlChars > 0 ? Math.round((1 - convertedChars / htmlChars) * 100) : 0;
+                return {
+                    url,
+                    htmlCharacters: htmlChars,
+                    convertedCharacters: convertedChars,
+                    conversionRatio: ratio,
+                    status: sizeStatus(convertedChars, passThreshold, failThreshold),
+                };
+            }
+            catch (err) {
+                return {
+                    url,
+                    htmlCharacters: 0,
+                    convertedCharacters: 0,
+                    conversionRatio: 0,
+                    status: 'fail',
+                    error: err instanceof Error ? err.message : String(err),
+                };
+            }
+        }));
+        results.push(...batchResults);
+    }
+    const successful = results.filter((r) => !r.error);
+    const fetchErrors = results.filter((r) => r.error).length;
+    const rateLimited = results.filter((r) => r.error && r.error.includes('429')).length;
+    if (successful.length === 0) {
+        const suffix = fetchErrors > 0 ? `; ${fetchErrors} failed to fetch` : '';
+        return {
+            id,
+            category,
+            status: 'fail',
+            message: `Could not fetch any pages to measure${suffix}`,
+            details: {
+                totalPages,
+                testedPages: results.length,
+                sampled: wasSampled,
+                fetchErrors,
+                rateLimited,
+                pageResults: results,
+                discoveryWarnings: warnings,
+            },
+        };
+    }
+    const convertedSizes = successful.map((r) => r.convertedCharacters).sort((a, b) => a - b);
+    const median = convertedSizes[Math.floor(convertedSizes.length / 2)];
+    const max = convertedSizes[convertedSizes.length - 1];
+    const avgRatio = Math.round(successful.reduce((sum, r) => sum + r.conversionRatio, 0) / successful.length);
+    const overallStatus = worstStatus(successful.map((r) => r.status));
+    const pageLabel = wasSampled ? 'sampled pages' : 'pages';
+    const passBucket = successful.filter((r) => r.status === 'pass').length;
+    const warnBucket = successful.filter((r) => r.status === 'warn').length;
+    const failBucket = successful.filter((r) => r.status === 'fail').length;
+    const suffix = (fetchErrors > 0 ? `; ${fetchErrors} failed to fetch` : '') +
+        (rateLimited > 0 ? `; ${rateLimited} rate-limited (HTTP 429)` : '');
+    let message;
+    if (overallStatus === 'pass') {
+        message = `All ${successful.length} ${pageLabel} convert under ${formatSize(passThreshold)} chars (median ${formatSize(median)}, ${avgRatio}% boilerplate)${suffix}`;
+    }
+    else if (overallStatus === 'warn') {
+        message = `${warnBucket} of ${successful.length} ${pageLabel} convert to ${formatSize(passThreshold)}–${formatSize(failThreshold)} chars (max ${formatSize(max)}, ${avgRatio}% boilerplate)${suffix}`;
+    }
+    else {
+        message = `${failBucket} of ${successful.length} ${pageLabel} convert to over ${formatSize(failThreshold)} chars (max ${formatSize(max)}, ${avgRatio}% boilerplate)${suffix}`;
+    }
     return {
-        id: 'page-size-html',
-        category: 'page-size',
-        status: 'skip',
-        message: 'Not yet implemented',
+        id,
+        category,
+        status: overallStatus,
+        message,
+        details: {
+            totalPages,
+            testedPages: results.length,
+            sampled: wasSampled,
+            median,
+            max,
+            avgBoilerplatePercent: avgRatio,
+            passBucket,
+            warnBucket,
+            failBucket,
+            fetchErrors,
+            rateLimited,
+            thresholds: { pass: passThreshold, fail: failThreshold },
+            pageResults: results,
+            discoveryWarnings: warnings,
+        },
     };
 }
 registerCheck({

package/dist/checks/page-size/page-size-html.js.map

@@ -1 +1 @@
-{"version":3,"file":"page-size-html.js","sourceRoot":"","sources":["../../../src/checks/page-size/page-size-html.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C,KAAK,UAAU,KAAK,CAAC,IAAkB;IACrC,OAAO;QACL,EAAE,EAAE,gBAAgB;QACpB,QAAQ,EAAE,WAAW;QACrB,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,qBAAqB;KAC/B,CAAC;AACJ,CAAC;AAED,aAAa,CAAC;IACZ,EAAE,EAAE,gBAAgB;IACpB,QAAQ,EAAE,WAAW;IACrB,WAAW,EAAE,2DAA2D;IACxE,SAAS,EAAE,EAAE;IACb,GAAG,EAAE,KAAK;CACX,CAAC,CAAC"}
+{"version":3,"file":"page-size-html.js","sourceRoot":"","sources":["../../../src/checks/page-size/page-size-html.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,kCAAkC,CAAC;AACjE,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AAYnE,SAAS,UAAU,CAAC,KAAa,EAAE,IAAY,EAAE,IAAY;IAC3D,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,MAAM,CAAC;IACjC,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,MAAM,CAAC;IACjC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,QAAuB;IAC1C,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,MAAM,CAAC;IAC7C,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,MAAM,CAAC;IAC7C,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC;IACzD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AACvB,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,GAAiB;IACpC,MAAM,EAAE,GAAG,gBAAgB,CAAC;IAC5B,MAAM,QAAQ,GAAG,WAAW,CAAC;IAC7B,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC;IAE5E,MAAM,EACJ,IAAI,EAAE,QAAQ,EACd,UAAU,EACV,OAAO,EAAE,UAAU,EACnB,QAAQ,GACT,GAAG,MAAM,sBAAsB,CAAC,GAAG,CAAC,CAAC;IAEtC,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC;IAE/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAA2B,EAAE;YAC/C,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC3C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;gBAC/D,MAAM,cAAc,GAClB,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;gBAC9E,MAAM,MAAM,GACV,CAAC,cAAc,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;gBAEhF,sDAAsD;gBACtD,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC;gBAC9B,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBACvD,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,CAAC;gBACxC,MAAM,KAAK,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,cAAc,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAErF,OAAO;oBACL,GAAG;oBACH,cAAc,EAAE,SAAS;oBACzB,mBAAmB,EAAE,cAAc;oBACnC,eAAe,EAAE,KAAK;oBACtB,MAAM,EAAE,UAAU,CAAC,cAAc,EAAE,aAAa,EAAE,aAAa,CAAC;iBACjE,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO;oBACL,GAAG;oBACH,cAAc,EAAE,CAAC;oBACjB,mBAAmB,EAAE,CAAC;oBACtB,eAAe,EAAE,CAAC;oBAClB,MAAM,EAAE,MAAM;oBACd,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBACxD,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACnD,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;IAC1D,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;IAErF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,OAAO;YACL,EAAE;YACF,QAAQ;YACR,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,uCAAuC,MAAM,EAAE;YACxD,OAAO,EAAE;gBACP,UAAU;gBACV,WAAW,EAAE,OAAO,CAAC,MAAM;gBAC3B,OAAO,EAAE,UAAU;gBACnB,WAAW;gBACX,WAAW;gBACX,WAAW,EAAE,OAAO;gBACpB,iBAAiB,EAAE,QAAQ;aAC5B;SACF,CAAC;IACJ,CAAC;IAED,MAAM,cAAc,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1F,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IACrE,MAAM,GAAG,GAAG,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CACzB,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,CAC9E,CAAC;IAEF,MAAM,aAAa,GAAG,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACnE,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC;IAEzD,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAExE,MAAM,MAAM,GACV,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,0BAA0B,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAEtE,IAAI,OAAe,CAAC;IACpB,IAAI,aAAa,KAAK,MAAM,EAAE,CAAC;QAC7B,OAAO,GAAG,OAAO,UAAU,CAAC,MAAM,IAAI,SAAS,kBAAkB,UAAU,CAAC,aAAa,CAAC,kBAAkB,UAAU,CAAC,MAAM,CAAC,KAAK,QAAQ,iBAAiB,MAAM,EAAE,CAAC;IACvK,CAAC;SAAM,IAAI,aAAa,KAAK,MAAM,EAAE,CAAC;QACpC,OAAO,GAAG,GAAG,UAAU,OAAO,UAAU,CAAC,MAAM,IAAI,SAAS,eAAe,UAAU,CAAC,aAAa,CAAC,IAAI,UAAU,CAAC,aAAa,CAAC,eAAe,UAAU,CAAC,GAAG,CAAC,KAAK,QAAQ,iBAAiB,MAAM,EAAE,CAAC;IACxM,CAAC;SAAM,CAAC;QACN,OAAO,GAAG,GAAG,UAAU,OAAO,UAAU,CAAC,MAAM,IAAI,SAAS,oBAAoB,UAAU,CAAC,aAAa,CAAC,eAAe,UAAU,CAAC,GAAG,CAAC,KAAK,QAAQ,iBAAiB,MAAM,EAAE,CAAC;IAChL,CAAC;IAED,OAAO;QACL,EAAE;QACF,QAAQ;QACR,MAAM,EAAE,aAAa;QACrB,OAAO;QACP,OAAO,EAAE;YACP,UAAU;YACV,WAAW,EAAE,OAAO,CAAC,MAAM;YAC3B,OAAO,EAAE,UAAU;YACnB,MAAM;YACN,GAAG;YACH,qBAAqB,EAAE,QAAQ;YAC/B,UAAU;YACV,UAAU;YACV,UAAU;YACV,WAAW;YACX,WAAW;YACX,UAAU,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,aAAa,EAAE;YACxD,WAAW,EAAE,OAAO;YACpB,iBAAiB,EAAE,QAAQ;SAC5B;KACF,CAAC;AACJ,CAAC;AAED,aAAa,CAAC;IACZ,EAAE,EAAE,gBAAgB;IACpB,QAAQ,EAAE,WAAW;IACrB,WAAW,EAAE,2DAA2D;IACxE,SAAS,EAAE,EAAE;IACb,GAAG,EAAE,KAAK;CACX,CAAC,CAAC"}