@rivi-software/agentic-seo-check 0.6.1 → 0.7.0

package/dist/core/fetch.js

@@ -13,6 +13,26 @@ const KEPT_HEADERS = [
     "vary",
     "content-security-policy",
 ];
+/**
+ * A request the gate refused, thrown out of `fetchPage` so that no caller takes it for an answer:
+ * a link that was not requested is not checked, never broken.
+ */
+export class RequestRefusedError extends Error {
+    scope;
+    /** The request that was refused: the URL asked for, or a redirect it led to. */
+    url;
+    constructor(refusal, url) {
+        super(refusal.message);
+        this.name = "RequestRefusedError";
+        this.scope = refusal.scope;
+        this.url = url;
+    }
+    get host() {
+        return new URL(this.url).hostname;
+    }
+}
+/** The most of a body an audit reads unless `maxBodyBytes` says otherwise. */
+export const MAX_BODY_BYTES = 5_000_000;
 const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
 const pickHeaders = (headers) => {
     const out = {};
@@ -55,8 +75,10 @@ const readLimited = async (res, maxBytes) => {
             break;
         }
         if (value) {
-            chunks.push(value);
-            total += value.byteLength;
+            // A chunk can be large: keep only what fits, so the cap holds to the byte.
+            const room = maxBytes - total;
+            chunks.push(value.byteLength > room ? value.subarray(0, room) : value);
+            total += Math.min(value.byteLength, room);
             if (total >= maxBytes) {
                 truncated = true;
                 break;
@@ -79,8 +101,49 @@ const decodeBody = (bytes, contentType) => {
         return new TextDecoder("utf-8").decode(bytes);
     }
 };
+/** Asks the gate before a request, and throws its refusal. */
+const askGate = async (gate, current) => {
+    // A URL that does not parse fails in fetch(), with an answer that says so.
+    if (!gate || !URL.canParse(current)) {
+        return;
+    }
+    const refusal = await gate.beforeRequest(new URL(current));
+    if (refusal) {
+        throw new RequestRefusedError(refusal, current);
+    }
+};
+/** Shows the gate an answer, and throws its refusal once the body is let go. */
+const showGate = async (gate, current, res) => {
+    const refusal = gate?.afterResponse?.(new URL(current), res);
+    if (refusal) {
+        await res.body?.cancel().catch(() => null);
+        throw new RequestRefusedError(refusal, current);
+    }
+};
+/** The answer with its body read, up to `maxBodyBytes`. */
+const readAnswer = async (walk, current, res, headers) => {
+    const { url, options, started, chain } = walk;
+    const read = await readLimited(res, options.maxBodyBytes ?? MAX_BODY_BYTES);
+    const done = result(url, current, res.status, headers, 
+    // The caller reads `bytes`; decoding them too would hold the body a second time.
+    options.keepBytes
+        ? ""
+        : decodeBody(read.bytes, res.headers.get("content-type")), chain, started);
+    if (read.truncated) {
+        done.truncated = true;
+    }
+    if (options.keepBytes) {
+        done.bytes = read.bytes;
+    }
+    const refusal = options.gate?.afterBody?.(new URL(current), done);
+    if (refusal) {
+        throw new RequestRefusedError(refusal, current);
+    }
+    return done;
+};
 const fetchHop = async (walk, current) => {
     const { url, options, readBody, timeoutMs, started, chain } = walk;
+    await askGate(options.gate, current);
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), timeoutMs);
     try {
@@ -95,6 +158,7 @@ const fetchHop = async (walk, current) => {
                 ...options.headers,
             },
         });
+        await showGate(options.gate, current, res);
         const headers = pickHeaders(res.headers);
         if (REDIRECT_STATUSES.has(res.status) && headers["location"]) {
             chain.push({ url: current, status: res.status });
@@ -121,21 +185,13 @@ const fetchHop = async (walk, current) => {
                 done: result(url, current, res.status, headers, null, chain, started),
             };
         }
-        const read = await readLimited(res, options.maxBodyBytes ?? 5_000_000);
-        const done = result(url, current, res.status, headers, 
-        // The caller reads `bytes`; decoding them too would hold the body a second time.
-        options.keepBytes
-            ? ""
-            : decodeBody(read.bytes, res.headers.get("content-type")), chain, started);
-        if (read.truncated) {
-            done.truncated = true;
-        }
-        if (options.keepBytes) {
-            done.bytes = read.bytes;
-        }
-        return { done };
+        return { done: await readAnswer(walk, current, res, headers) };
     }
     catch (error) {
+        // A refusal is no failure of the request: it goes to the caller as it is.
+        if (error instanceof RequestRefusedError) {
+            throw error;
+        }
         let message = String(error);
         if (error instanceof Error) {
             message =
@@ -187,6 +243,9 @@ export const toResponseInfo = (r) => {
     if (r.error) {
         info.error = r.error;
     }
+    if (r.truncated) {
+        info.truncated = true;
+    }
     return info;
 };
 export const isHtmlResponse = (r) => {

package/dist/core/fetch.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/core/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,MAAM,CAAC,MAAM,kBAAkB,GAAG,8CAA8C,OAAO,wDAAwD,CAAC;AAEhJ,MAAM,YAAY,GAAG;IACnB,cAAc;IACd,kBAAkB;IAClB,cAAc;IACd,MAAM;IACN,kBAAkB;IAClB,eAAe;IACf,eAAe;IACf,UAAU;IACV,gBAAgB;IAChB,MAAM;IACN,yBAAyB;CAC1B,CAAC;AAgCF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE7D,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,EAAE;IACvC,MAAM,GAAG,GAA2B,EAAE,CAAC;IAEvC,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEhC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CACb,YAAoB,EACpB,QAAgB,EAChB,MAAqB,EACrB,OAA+B,EAC/B,IAAmB,EACnB,KAAoB,EACpB,OAAe,EACf,KAAc,EACD,EAAE;IACf,MAAM,CAAC,GAAgB;QACrB,YAAY;QACZ,QAAQ;QACR,MAAM;QACN,OAAO;QACP,IAAI;QACJ,aAAa,EAAE,KAAK;QACpB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QAC/C,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;KAC7C,CAAC;IAEF,IAAI,KAAK,EAAE,CAAC;QACV,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAOF,MAAM,WAAW,GAAG,KAAK,EACvB,GAAa,EACb,QAAgB,EACG,EAAE;IACrB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACd,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACtD,CAAC;IAED,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;IACpC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,SAAS,CAAC;QACR,uGAAuG;QACvG,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAE5C,IAAI,IAAI,EAAE,CAAC;YACT,MAAM;QACR,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;YAE1B,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;gBACtB,SAAS,GAAG,IAAI,CAAC;gBACjB,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;AACrD,CAAC,CAAC;AAEF,wFAAwF;AACxF,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,WAA0B,EAAU,EAAE;IACvE,MAAM,OAAO,GAAG,8BAA8B,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,MAAM;QAC5E,EAAE,OAAO,CAAC;IAEZ,IAAI,CAAC;QACH,OAAO,IAAI,WAAW,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC;AACH,CAAC,CAAC;AAeF,MAAM,QAAQ,GAAG,KAAK,EACpB,IAAkB,EAClB,OAAe,EACM,EAAE;IACvB,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IACnE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YAC/B,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE;gBACP,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB;gBACrD,MAAM,EACJ,iEAAiE;gBACnE,iBAAiB,EAAE,YAAY;gBAC/B,GAAG,OAAO,CAAC,OAAO;aACnB;SACF,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7D,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YACjD,IAAI,OAAe,CAAC;YAEpB,IAAI,CAAC;gBACH,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO;oBACL,IAAI,EAAE,MAAM,CACV,GAAG,EACH,OAAO,EACP,GAAG,CAAC,MAAM,EACV,OAAO,EACP,IAAI,EACJ,KAAK,EACL,OAAO,EACP,4BAA4B,OAAO,CAAC,UAAU,CAAC,EAAE,CAClD;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;YAE3C,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;gBAChE,OAAO;oBACL,IAAI,EAAE,MAAM,CACV,GAAG,EACH,OAAO,EACP,GAAG,CAAC,MAAM,EACV,OAAO,EACP,IAAI,EACJ,KAAK,EACL,OAAO,EACP,eAAe,CAChB;iBACF,CAAC;YACJ,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;YAE3C,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC;aACtE,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC,CAAC;QAEvE,MAAM,IAAI,GAAG,MAAM,CACjB,GAAG,EACH,OAAO,EACP,GAAG,CAAC,MAAM,EACV,OAAO;QACP,iFAAiF;QACjF,OAAO,CAAC,SAAS;YACf,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,EAC3D,KAAK,EACL,OAAO,CACR,CAAC;QAEF,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QAC1B,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAE5B,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO;gBACL,KAAK,CAAC,IAAI,KAAK,YAAY;oBACzB,CAAC,CAAC,mBAAmB,SAAS,KAAK;oBACnC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;QACtB,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC;SACpE,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC,CAAC;AAEF,wEAAwE;AACxE,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,GAAW,EACX,OAAO,GAAiB,EAAE,EAC1B,QAAQ,GAAG,IAAI,EACO,EAAE;IACxB,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC;IAEhD,MAAM,IAAI,GAAiB;QACzB,GAAG;QACH,OAAO;QACP,QAAQ;QACR,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM;QACtC,OAAO,EAAE,WAAW,CAAC,GAAG,EAAE;QAC1B,KAAK,EAAE,EAAE;KACV,CAAC;IAEF,IAAI,OAAO,GAAG,GAAG,CAAC;IAElB,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,YAAY,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QAChD,2GAA2G;QAC3G,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAE9C,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;YACtB,OAAO,OAAO,CAAC,IAAI,CAAC;QACtB,CAAC;QAED,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,OAAO,MAAM,CACX,GAAG,EACH,OAAO,EACP,IAAI,EACJ,EAAE,EACF,IAAI,EACJ,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,EACZ,aAAa,YAAY,YAAY,CACtC,CAAC;AACJ,CAAC,CAAC;AAEF,0DAA0D;AAC1D,MAAM,CAAC,MAAM,QAAQ,GAAG,CACtB,GAAW,EACX,OAAO,GAAiB,EAAE,EACJ,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;AAE1D,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAc,EAAgB,EAAE;IAC7D,MAAM,IAAI,GAAiB;QACzB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC;IAEF,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;QACZ,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;IACvB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAc,EAAW,EAAE;IACxD,MAAM,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAE9C,IAAI,EAAE,EAAE,CAAC;QACP,OAAO,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IACtE,CAAC;IAED,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACZ,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AACzD,CAAC,CAAC"}
+{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/core/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,MAAM,CAAC,MAAM,kBAAkB,GAAG,8CAA8C,OAAO,wDAAwD,CAAC;AAEhJ,MAAM,YAAY,GAAG;IACnB,cAAc;IACd,kBAAkB;IAClB,cAAc;IACd,MAAM;IACN,kBAAkB;IAClB,eAAe;IACf,eAAe;IACf,UAAU;IACV,gBAAgB;IAChB,MAAM;IACN,yBAAyB;CAC1B,CAAC;AAaF;;;GAGG;AACH,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IACnC,KAAK,CAAmB;IACjC,gFAAgF;IACvE,GAAG,CAAS;IAErB,YAAY,OAAgB,EAAE,GAAW;QACvC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACvB,IAAI,CAAC,IAAI,GAAG,qBAAqB,CAAC;QAClC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IACpC,CAAC;CACF;AAoCD,8EAA8E;AAC9E,MAAM,CAAC,MAAM,cAAc,GAAG,SAAS,CAAC;AAkBxC,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE7D,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,EAAE;IACvC,MAAM,GAAG,GAA2B,EAAE,CAAC;IAEvC,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEhC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CACb,YAAoB,EACpB,QAAgB,EAChB,MAAqB,EACrB,OAA+B,EAC/B,IAAmB,EACnB,KAAoB,EACpB,OAAe,EACf,KAAc,EACD,EAAE;IACf,MAAM,CAAC,GAAgB;QACrB,YAAY;QACZ,QAAQ;QACR,MAAM;QACN,OAAO;QACP,IAAI;QACJ,aAAa,EAAE,KAAK;QACpB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QAC/C,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;KAC7C,CAAC;IAEF,IAAI,KAAK,EAAE,CAAC;QACV,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAOF,MAAM,WAAW,GAAG,KAAK,EACvB,GAAa,EACb,QAAgB,EACG,EAAE;IACrB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACd,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACtD,CAAC;IAED,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;IACpC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,SAAS,CAAC;QACR,uGAAuG;QACvG,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAE5C,IAAI,IAAI,EAAE,CAAC;YACT,MAAM;QACR,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACV,2EAA2E;YAC3E,MAAM,IAAI,GAAG,QAAQ,GAAG,KAAK,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACvE,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;YAE1C,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;gBACtB,SAAS,GAAG,IAAI,CAAC;gBACjB,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;AACrD,CAAC,CAAC;AAEF,wFAAwF;AACxF,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,WAA0B,EAAU,EAAE;IACvE,MAAM,OAAO,GAAG,8BAA8B,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,MAAM;QAC5E,EAAE,OAAO,CAAC;IAEZ,IAAI,CAAC;QACH,OAAO,IAAI,WAAW,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC;AACH,CAAC,CAAC;AAeF,8DAA8D;AAC9D,MAAM,OAAO,GAAG,KAAK,EACnB,IAA6B,EAC7B,OAAe,EACA,EAAE;IACjB,2EAA2E;IAC3E,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACpC,OAAO;IACT,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;IAE3D,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,IAAI,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAClD,CAAC;AACH,CAAC,CAAC;AAEF,gFAAgF;AAChF,MAAM,QAAQ,GAAG,KAAK,EACpB,IAA6B,EAC7B,OAAe,EACf,GAAa,EACE,EAAE;IACjB,MAAM,OAAO,GAAG,IAAI,EAAE,aAAa,EAAE,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,EAAE,GAAG,CAAC,CAAC;IAE7D,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAE3C,MAAM,IAAI,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAClD,CAAC;AACH,CAAC,CAAC;AAEF,2DAA2D;AAC3D,MAAM,UAAU,GAAG,KAAK,EACtB,IAAkB,EAClB,OAAe,EACf,GAAa,EACb,OAA+B,EACT,EAAE;IACxB,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IAC9C,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,YAAY,IAAI,cAAc,CAAC,CAAC;IAE5E,MAAM,IAAI,GAAG,MAAM,CACjB,GAAG,EACH,OAAO,EACP,GAAG,CAAC,MAAM,EACV,OAAO;IACP,iFAAiF;IACjF,OAAO,CAAC,SAAS;QACf,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,EAC3D,KAAK,EACL,OAAO,CACR,CAAC;IAEF,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;QACnB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IAC1B,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;IAElE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,IAAI,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,KAAK,EACpB,IAAkB,EAClB,OAAe,EACM,EAAE;IACvB,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IACnE,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YAC/B,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE;gBACP,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB;gBACrD,MAAM,EACJ,iEAAiE;gBACnE,iBAAiB,EAAE,YAAY;gBAC/B,GAAG,OAAO,CAAC,OAAO;aACnB;SACF,CAAC,CAAC;QAEH,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;QAC3C,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7D,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YACjD,IAAI,OAAe,CAAC;YAEpB,IAAI,CAAC;gBACH,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO;oBACL,IAAI,EAAE,MAAM,CACV,GAAG,EACH,OAAO,EACP,GAAG,CAAC,MAAM,EACV,OAAO,EACP,IAAI,EACJ,KAAK,EACL,OAAO,EACP,4BAA4B,OAAO,CAAC,UAAU,CAAC,EAAE,CAClD;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;YAE3C,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;gBAChE,OAAO;oBACL,IAAI,EAAE,MAAM,CACV,GAAG,EACH,OAAO,EACP,GAAG,CAAC,MAAM,EACV,OAAO,EACP,IAAI,EACJ,KAAK,EACL,OAAO,EACP,eAAe,CAChB;iBACF,CAAC;YACJ,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;YAE3C,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC;aACtE,CAAC;QACJ,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;IACjE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,0EAA0E;QAC1E,IAAI,KAAK,YAAY,mBAAmB,EAAE,CAAC;YACzC,MAAM,KAAK,CAAC;QACd,CAAC;QAED,IAAI,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAE5B,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO;gBACL,KAAK,CAAC,IAAI,KAAK,YAAY;oBACzB,CAAC,CAAC,mBAAmB,SAAS,KAAK;oBACnC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;QACtB,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC;SACpE,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC,CAAC;AAEF,wEAAwE;AACxE,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,GAAW,EACX,OAAO,GAAiB,EAAE,EAC1B,QAAQ,GAAG,IAAI,EACO,EAAE;IACxB,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC;IAEhD,MAAM,IAAI,GAAiB;QACzB,GAAG;QACH,OAAO;QACP,QAAQ;QACR,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM;QACtC,OAAO,EAAE,WAAW,CAAC,GAAG,EAAE;QAC1B,KAAK,EAAE,EAAE;KACV,CAAC;IAEF,IAAI,OAAO,GAAG,GAAG,CAAC;IAElB,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,YAAY,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QAChD,2GAA2G;QAC3G,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAE9C,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;YACtB,OAAO,OAAO,CAAC,IAAI,CAAC;QACtB,CAAC;QAED,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,OAAO,MAAM,CACX,GAAG,EACH,OAAO,EACP,IAAI,EACJ,EAAE,EACF,IAAI,EACJ,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,EACZ,aAAa,YAAY,YAAY,CACtC,CAAC;AACJ,CAAC,CAAC;AAEF,0DAA0D;AAC1D,MAAM,CAAC,MAAM,QAAQ,GAAG,CACtB,GAAW,EACX,OAAO,GAAiB,EAAE,EACJ,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;AAE1D,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAc,EAAgB,EAAE;IAC7D,MAAM,IAAI,GAAiB;QACzB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC;IAEF,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;QACZ,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;IACvB,CAAC;IAED,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC;QAChB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAc,EAAW,EAAE;IACxD,MAAM,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAE9C,IAAI,EAAE,EAAE,CAAC;QACP,OAAO,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IACtE,CAAC;IAED,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACZ,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AACzD,CAAC,CAAC"}

package/dist/core/hosted.d.ts

@@ -0,0 +1,54 @@
+import type { Refusal, RequestGate } from "./fetch.js";
+/** The local server, which none of these limits apply to: the way out of every refusal. */
+export declare const LOCAL_SERVER = "npx -y @rivi-software/agentic-seo-check mcp";
+/**
+ * What the hosted server reads and requests at most. A Worker isolate has 128 MB of memory: one
+ * 2 MB page takes about 80 MB of it to audit, a 5 MB sitemap about 64 MB to parse.
+ */
+export declare const HOSTED_LIMITS: {
+    /** Bytes of a page (5 MB on the local server). Above document.html-too-large's 1.5 MB. */
+    readonly maxBodyBytes: 2000000;
+    /** Characters of inline `html`. */
+    readonly maxHtmlChars: 2000000;
+    /**
+     * Bytes of a request to the server, checked before it is parsed. 2,000,000 characters of HTML
+     * are up to 6 MB in UTF-8, and JSON escaping adds to that.
+     */
+    readonly maxRequestBytes: 8000000;
+    readonly maxPages: 25;
+    readonly maxDepth: 3;
+    /** Requests at a time; a Worker has at most 6 connections waiting for headers. */
+    readonly concurrency: 4;
+    /** Internal links, sitemap URLs and images a crawl checks, and as many external links. */
+    readonly maxLinkChecks: 50;
+    readonly timeoutMs: 15000;
+    readonly maxSitemaps: 10;
+    readonly maxSitemapBytes: 5000000;
+    /** `includePatterns` and `excludePatterns` are the caller's regular expressions, run here. */
+    readonly maxPatterns: 20;
+    readonly maxPatternChars: 200;
+    /** `rules` and `disableRules`. */
+    readonly maxRuleIds: 500;
+    readonly maxRuleIdChars: 100;
+    /** `baseline`: the finding ids of an earlier audit. */
+    readonly maxBaselineIds: 5000;
+    readonly maxBaselineIdChars: 300;
+    readonly maxKeywordChars: 200;
+};
+/**
+ * Why the hosted server will not request `url`, or undefined when it may: it requests http and
+ * https URLs on public hostnames only. A Worker reaches no private network anyway; the rules give
+ * a clear answer instead of a failed request.
+ */
+export declare const hostedUrlProblem: (url: URL) => string | undefined;
+/**
+ * Why an answer is the site refusing the hosted server rather than its page: a challenge, a rate
+ * limit or a 403. A 429 or a challenge holds for every later request to the host; a 403 for the
+ * URL alone, since a site may forbid one path to everyone. Undefined for a page.
+ */
+export declare const siteRefusal: (url: URL, response: Pick<Response, "status" | "headers">) => Refusal | undefined;
+/**
+ * The gate of a hosted tool call: the URL rules and the site refusals, then the Worker's own gate
+ * (its request limits), which only sees requests the rules allow.
+ */
+export declare const hostedGate: (limits?: RequestGate) => RequestGate;

package/dist/core/hosted.js

@@ -0,0 +1,134 @@
+/**
+ * The hosted server's rules (hosted/, issue #35): what it reads at most, which URLs it requests,
+ * and which answers are a site turning it away rather than the site's page. The Worker adds its
+ * limits on top, through the gate it passes to `hostedGate`.
+ */
+import { challengeTitle } from "./aeo/probe.js";
+/** The local server, which none of these limits apply to: the way out of every refusal. */
+export const LOCAL_SERVER = "npx -y @rivi-software/agentic-seo-check mcp";
+const USE_LOCAL = `Use the local server for it, which sends the requests from your machine: ${LOCAL_SERVER}`;
+/**
+ * What the hosted server reads and requests at most. A Worker isolate has 128 MB of memory: one
+ * 2 MB page takes about 80 MB of it to audit, a 5 MB sitemap about 64 MB to parse.
+ */
+export const HOSTED_LIMITS = {
+    /** Bytes of a page (5 MB on the local server). Above document.html-too-large's 1.5 MB. */
+    maxBodyBytes: 2_000_000,
+    /** Characters of inline `html`. */
+    maxHtmlChars: 2_000_000,
+    /**
+     * Bytes of a request to the server, checked before it is parsed. 2,000,000 characters of HTML
+     * are up to 6 MB in UTF-8, and JSON escaping adds to that.
+     */
+    maxRequestBytes: 8_000_000,
+    maxPages: 25,
+    maxDepth: 3,
+    /** Requests at a time; a Worker has at most 6 connections waiting for headers. */
+    concurrency: 4,
+    /** Internal links, sitemap URLs and images a crawl checks, and as many external links. */
+    maxLinkChecks: 50,
+    timeoutMs: 15_000,
+    maxSitemaps: 10,
+    maxSitemapBytes: 5_000_000,
+    /** `includePatterns` and `excludePatterns` are the caller's regular expressions, run here. */
+    maxPatterns: 20,
+    maxPatternChars: 200,
+    /** `rules` and `disableRules`. */
+    maxRuleIds: 500,
+    maxRuleIdChars: 100,
+    /** `baseline`: the finding ids of an earlier audit. */
+    maxBaselineIds: 5000,
+    maxBaselineIdChars: 300,
+    maxKeywordChars: 200,
+};
+/** Names that only resolve on a local machine or network, never on the public internet. */
+const LOCAL_SUFFIXES = [".localhost", ".local", ".internal"];
+/** An IPv4 address as the URL parser writes it, which it does for every numeric form. */
+const IPV4 = /^\d{1,3}(?:\.\d{1,3}){3}$/u;
+/**
+ * Why the hosted server will not request `url`, or undefined when it may: it requests http and
+ * https URLs on public hostnames only. A Worker reaches no private network anyway; the rules give
+ * a clear answer instead of a failed request.
+ */
+export const hostedUrlProblem = (url) => {
+    if (url.protocol !== "http:" && url.protocol !== "https:") {
+        return `The hosted server requests http and https URLs only, not ${url.protocol} ones (${url.href}).`;
+    }
+    if (url.username || url.password) {
+        return `The hosted server does not send credentials in a URL (${url.host}). A site behind a login is one for the local server, which can send your headers: ${LOCAL_SERVER}`;
+    }
+    const host = url.hostname.replace(/\.$/u, "");
+    if (IPV4.test(host) || host.startsWith("[")) {
+        return `The hosted server requests hostnames only, not IP addresses (${host}). ${USE_LOCAL}`;
+    }
+    if (host === "localhost" ||
+        !host.includes(".") ||
+        LOCAL_SUFFIXES.some((suffix) => host.endsWith(suffix))) {
+        return `The hosted server cannot reach ${host}: it runs on Cloudflare's network, not on your machine or your network. ${USE_LOCAL}`;
+    }
+    return undefined;
+};
+const REFUSED_BY_SITE = `Many sites turn away requests from Cloudflare Workers, which the hosted server runs on, while they serve the page to everyone else. ${USE_LOCAL}`;
+/**
+ * Why an answer is the site refusing the hosted server rather than its page: a challenge, a rate
+ * limit or a 403. A 429 or a challenge holds for every later request to the host; a 403 for the
+ * URL alone, since a site may forbid one path to everyone. Undefined for a page.
+ */
+export const siteRefusal = (url, response) => {
+    // Cloudflare marks every challenge page so (developers.cloudflare.com, "Detect a challenge page").
+    if (response.headers.get("cf-mitigated")?.toLowerCase() === "challenge") {
+        return {
+            scope: "host",
+            message: `${url.hostname} answered the hosted server with a challenge page (cf-mitigated: challenge). ${REFUSED_BY_SITE}`,
+        };
+    }
+    if (response.status === 429) {
+        return {
+            scope: "host",
+            message: `${url.hostname} answered the hosted server with HTTP 429 (Too Many Requests). A site that counts requests per address counts every Cloudflare Worker as one. Try again later, or use the local server, which sends the requests from your machine: ${LOCAL_SERVER}`,
+        };
+    }
+    if (response.status === 403) {
+        return {
+            scope: "url",
+            message: `${url.href} answered the hosted server with HTTP 403 (Forbidden). ${REFUSED_BY_SITE}`,
+        };
+    }
+    return undefined;
+};
+/**
+ * The statuses a challenge page comes with besides 403 and 429, which `siteRefusal` answers from
+ * the status alone: 503 (an "under attack" check) and 405 (AWS WAF's CAPTCHA). A 401 or 404 page
+ * titled "Access denied" is the site's own.
+ */
+const CHALLENGE_STATUSES = new Set([405, 503]);
+/** An error page that is a CDN's or bot protection's interstitial rather than the site's own. */
+const challengeRefusal = (url, result) => {
+    if (result.status === null ||
+        !CHALLENGE_STATUSES.has(result.status) ||
+        !result.body) {
+        return undefined;
+    }
+    const title = challengeTitle(result.body.slice(0, 10_000));
+    return title === null
+        ? undefined
+        : {
+            scope: "host",
+            message: `${url.hostname} answered the hosted server with a challenge page ("${title}", HTTP ${result.status}). ${REFUSED_BY_SITE}`,
+        };
+};
+/**
+ * The gate of a hosted tool call: the URL rules and the site refusals, then the Worker's own gate
+ * (its request limits), which only sees requests the rules allow.
+ */
+export const hostedGate = (limits) => ({
+    beforeRequest: (url) => {
+        const problem = hostedUrlProblem(url);
+        return problem
+            ? { scope: "url", message: problem }
+            : limits?.beforeRequest(url);
+    },
+    afterResponse: (url, response) => siteRefusal(url, response) ?? limits?.afterResponse?.(url, response),
+    afterBody: (url, result) => challengeRefusal(url, result) ?? limits?.afterBody?.(url, result),
+});
+//# sourceMappingURL=hosted.js.map

package/dist/core/hosted.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hosted.js","sourceRoot":"","sources":["../../src/core/hosted.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAGhD,2FAA2F;AAC3F,MAAM,CAAC,MAAM,YAAY,GAAG,6CAA6C,CAAC;AAE1E,MAAM,SAAS,GAAG,4EAA4E,YAAY,EAAE,CAAC;AAE7G;;;GAGG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,0FAA0F;IAC1F,YAAY,EAAE,SAAS;IACvB,mCAAmC;IACnC,YAAY,EAAE,SAAS;IACvB;;;OAGG;IACH,eAAe,EAAE,SAAS;IAC1B,QAAQ,EAAE,EAAE;IACZ,QAAQ,EAAE,CAAC;IACX,kFAAkF;IAClF,WAAW,EAAE,CAAC;IACd,0FAA0F;IAC1F,aAAa,EAAE,EAAE;IACjB,SAAS,EAAE,MAAM;IACjB,WAAW,EAAE,EAAE;IACf,eAAe,EAAE,SAAS;IAC1B,8FAA8F;IAC9F,WAAW,EAAE,EAAE;IACf,eAAe,EAAE,GAAG;IACpB,kCAAkC;IAClC,UAAU,EAAE,GAAG;IACf,cAAc,EAAE,GAAG;IACnB,uDAAuD;IACvD,cAAc,EAAE,IAAI;IACpB,kBAAkB,EAAE,GAAG;IACvB,eAAe,EAAE,GAAG;CACZ,CAAC;AAEX,2FAA2F;AAC3F,MAAM,cAAc,GAAG,CAAC,YAAY,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;AAE7D,yFAAyF;AACzF,MAAM,IAAI,GAAG,4BAA4B,CAAC;AAE1C;;;;GAIG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,GAAQ,EAAsB,EAAE;IAC/D,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,OAAO,4DAA4D,GAAG,CAAC,QAAQ,UAAU,GAAG,CAAC,IAAI,IAAI,CAAC;IACxG,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACjC,OAAO,yDAAyD,GAAG,CAAC,IAAI,sFAAsF,YAAY,EAAE,CAAC;IAC/K,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAE9C,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5C,OAAO,gEAAgE,IAAI,MAAM,SAAS,EAAE,CAAC;IAC/F,CAAC;IAED,IACE,IAAI,KAAK,WAAW;QACpB,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QACnB,cAAc,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EACtD,CAAC;QACD,OAAO,kCAAkC,IAAI,2EAA2E,SAAS,EAAE,CAAC;IACtI,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,uIAAuI,SAAS,EAAE,CAAC;AAE3K;;;;GAIG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,GAAQ,EACR,QAA8C,EACzB,EAAE;IACvB,mGAAmG;IACnG,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,WAAW,EAAE,KAAK,WAAW,EAAE,CAAC;QACxE,OAAO;YACL,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,GAAG,GAAG,CAAC,QAAQ,gFAAgF,eAAe,EAAE;SAC1H,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC5B,OAAO;YACL,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,GAAG,GAAG,CAAC,QAAQ,uOAAuO,YAAY,EAAE;SAC9Q,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC5B,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,OAAO,EAAE,GAAG,GAAG,CAAC,IAAI,0DAA0D,eAAe,EAAE;SAChG,CAAC;IACJ,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE/C,iGAAiG;AACjG,MAAM,gBAAgB,GAAG,CACvB,GAAQ,EACR,MAAmB,EACE,EAAE;IACvB,IACE,MAAM,CAAC,MAAM,KAAK,IAAI;QACtB,CAAC,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;QACtC,CAAC,MAAM,CAAC,IAAI,EACZ,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;IAE3D,OAAO,KAAK,KAAK,IAAI;QACnB,CAAC,CAAC,SAAS;QACX,CAAC,CAAC;YACE,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,GAAG,GAAG,CAAC,QAAQ,uDAAuD,KAAK,WAAW,MAAM,CAAC,MAAM,MAAM,eAAe,EAAE;SACpI,CAAC;AACR,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,MAAoB,EAAe,EAAE,CAAC,CAAC;IAChE,aAAa,EAAE,CAAC,GAAG,EAAE,EAAE;QACrB,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAEtC,OAAO,OAAO;YACZ,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE;YACpC,CAAC,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC;IACD,aAAa,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAC/B,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,MAAM,EAAE,aAAa,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC;IACtE,SAAS,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CACzB,gBAAgB,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,MAAM,EAAE,SAAS,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC;CACpE,CAAC,CAAC"}

package/dist/core/refusals.d.ts

@@ -0,0 +1,27 @@
+import { RequestRefusedError } from "./fetch.js";
+import type { FetchOptions } from "./fetch.js";
+/**
+ * The requests of one audit that a gate refused (the hosted server's), and what each rules out:
+ * the URL alone, its host for the rest of the audit, or every later request. A request that was
+ * not sent says nothing about its URL, so callers leave it unchecked, never broken.
+ */
+export interface Refusals {
+    /**
+     * Runs `request`, which fetches `url`, unless a refusal already rules it out. Null when it is
+     * refused, now or before; any other error goes to the caller.
+     */
+    run: <T>(url: string, request: () => Promise<T>) => Promise<T | null>;
+    /** The refusal that rules out `url`, if one does. */
+    of: (url: string) => RequestRefusedError | undefined;
+    /**
+     * The refusal that ended the requests to a host `isSite` accepts, or to every host: the reason a
+     * crawl of that site stopped early.
+     */
+    stopped: (isSite: (host: string) => boolean) => RequestRefusedError | undefined;
+    /**
+     * `options` with a gate that also refuses the hosts already refused, so a redirect to one of them
+     * is not sent again. Without a gate nothing is ever refused, and `options` stays as it is.
+     */
+    guard: (options: FetchOptions) => FetchOptions;
+}
+export declare const trackRefusals: () => Refusals;

package/dist/core/refusals.js

@@ -0,0 +1,56 @@
+import { RequestRefusedError } from "./fetch.js";
+import { hostOf } from "./url.js";
+export const trackRefusals = () => {
+    const urls = new Map();
+    const hosts = new Map();
+    let all;
+    const of = (url) => all ?? urls.get(url) ?? hosts.get(hostOf(url) ?? "");
+    const record = (url, error) => {
+        // The URL asked for, refused itself or through a redirect it led to.
+        urls.set(url, error);
+        if (error.scope === "all") {
+            all ??= error;
+        }
+        else if (error.scope === "host" && !hosts.has(error.host)) {
+            hosts.set(error.host, error);
+        }
+    };
+    return {
+        of,
+        stopped: (isSite) => all ?? [...hosts].find(([host]) => isSite(host))?.[1],
+        guard: (options) => {
+            const { gate } = options;
+            if (!gate) {
+                return options;
+            }
+            return {
+                ...options,
+                gate: {
+                    ...gate,
+                    beforeRequest: (url) => {
+                        const known = all ?? hosts.get(url.hostname);
+                        return known
+                            ? { scope: known.scope, message: known.message }
+                            : gate.beforeRequest(url);
+                    },
+                },
+            };
+        },
+        run: async (url, request) => {
+            if (of(url)) {
+                return null;
+            }
+            try {
+                return await request();
+            }
+            catch (error) {
+                if (!(error instanceof RequestRefusedError)) {
+                    throw error;
+                }
+                record(url, error);
+                return null;
+            }
+        },
+    };
+};
+//# sourceMappingURL=refusals.js.map

package/dist/core/refusals.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"refusals.js","sourceRoot":"","sources":["../../src/core/refusals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAEjD,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AA6BlC,MAAM,CAAC,MAAM,aAAa,GAAG,GAAa,EAAE;IAC1C,MAAM,IAAI,GAAG,IAAI,GAAG,EAA+B,CAAC;IACpD,MAAM,KAAK,GAAG,IAAI,GAAG,EAA+B,CAAC;IACrD,IAAI,GAAoC,CAAC;IAEzC,MAAM,EAAE,GAAG,CAAC,GAAW,EAAmC,EAAE,CAC1D,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;IAEvD,MAAM,MAAM,GAAG,CAAC,GAAW,EAAE,KAA0B,EAAQ,EAAE;QAC/D,qEAAqE;QACrE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAErB,IAAI,KAAK,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;YAC1B,GAAG,KAAK,KAAK,CAAC;QAChB,CAAC;aAAM,IAAI,KAAK,CAAC,KAAK,KAAK,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5D,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC,CAAC;IAEF,OAAO;QACL,EAAE;QACF,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC1E,KAAK,EAAE,CAAC,OAAO,EAAE,EAAE;YACjB,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC;YAEzB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO,OAAO,CAAC;YACjB,CAAC;YAED,OAAO;gBACL,GAAG,OAAO;gBACV,IAAI,EAAE;oBACJ,GAAG,IAAI;oBACP,aAAa,EAAE,CAAC,GAAG,EAAE,EAAE;wBACrB,MAAM,KAAK,GAAG,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;wBAE7C,OAAO,KAAK;4BACV,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE;4BAChD,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;oBAC9B,CAAC;iBACF;aACF,CAAC;QACJ,CAAC;QACD,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE;YAC1B,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;gBACZ,OAAO,IAAI,CAAC;YACd,CAAC;YAED,IAAI,CAAC;gBACH,OAAO,MAAM,OAAO,EAAE,CAAC;YACzB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,CAAC,KAAK,YAAY,mBAAmB,CAAC,EAAE,CAAC;oBAC5C,MAAM,KAAK,CAAC;gBACd,CAAC;gBAED,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAEnB,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC,CAAC"}

package/dist/core/robots.d.ts

@@ -2,11 +2,22 @@ import type { FetchOptions } from "./fetch.js";
 import type { RobotsTxtInfo } from "./types.js";
 export interface RobotsResult {
     info: RobotsTxtInfo;
-    /** Returns true when crawling the URL is allowed (unknown/missing robots.txt allows everything). */
+    /**
+     * Returns true when robots.txt allows the URL for the audit's User-Agent as robots-parser reads
+     * it (unknown/missing robots.txt allows everything). The site rules judge blocked pages by this.
+     */
     allows: (url: string) => boolean;
+    /**
+     * Returns true when this crawler may request the URL: the group for its product token
+     * (`agentic-seo-check`), or `*` when there is none (RFC 9309, section 2.2.1). A site can keep
+     * the audit out without blocking anyone else.
+     */
+    crawlerAllows: (url: string) => boolean;
     /** The robots.txt text up to the size crawlers must read (RFC 9309: 500 KiB), or null when there is none. */
     text: string | null;
 }
+/** Short bot token used for robots.txt matching, derived from the full UA string. */
+export declare const robotsUserAgentToken: (userAgent: string) => string;
 /**
  * Whether robots.txt at `url` lets the crawler with product token `agent` fetch `target`. A URL
  * on another origin is allowed: this robots.txt has no say over it.
@@ -19,5 +30,3 @@ export declare const agentMatcher: (url: string, text: string) => ((target: stri
 export declare const ruleLineMatcher: (url: string, text: string) => ((target: string, agent: string) => number);
 export declare const buildRobots: (url: string, status: number | null, error: string | undefined, text: string | null, userAgent: string) => RobotsResult;
 export declare const loadRobotsTxt: (origin: string, userAgent: string, options?: FetchOptions) => Promise<RobotsResult>;
-/** Short bot token used for robots.txt matching, derived from the full UA string. */
-export declare const robotsUserAgentToken: (userAgent: string) => string;

package/dist/core/robots.js

@@ -1,11 +1,20 @@
-import { createRequire } from "node:module";
+import robotsParserModule from "robots-parser";
 import { fetchPage } from "./fetch.js";
-// robots-parser is CommonJS; require() avoids ESM default-export interop mismatches.
-// SAFETY: require() returns the package's module.exports, which is the parser function that its
-// index.d.ts declares as the default export.
-const robotsParser = createRequire(import.meta.url)("robots-parser");
+/**
+ * robots-parser is CommonJS, imported statically so a bundler (the hosted server's) takes it in.
+ * Node and bundlers hand a default import its module.exports, the parser function itself; its
+ * index.d.ts declares the parser as the module's `default`, so TypeScript's Node resolution types
+ * the import as the module and its bundler resolution as the parser. Either way, this is the parser.
+ */
+const parserOf = (imported) => ("default" in imported ? imported.default : imported);
+const robotsParser = parserOf(robotsParserModule);
 /** The part of a robots.txt file a crawler must read at least (RFC 9309 section 2.5). */
 const MAX_ROBOTS_BYTES = 512_000;
+/** Short bot token used for robots.txt matching, derived from the full UA string. */
+export const robotsUserAgentToken = (userAgent) => {
+    const match = /agentic-seo-check|Googlebot|bingbot/iu.exec(userAgent);
+    return match ? match[0] : userAgent;
+};
 /**
  * Whether robots.txt at `url` lets the crawler with product token `agent` fetch `target`. A URL
  * on another origin is allowed: this robots.txt has no say over it.
@@ -35,17 +44,18 @@ export const buildRobots = (url, status, error, text, userAgent) => {
         info.error = error;
     }
     if (text === null) {
-        return { info, allows: () => true, text: null };
+        return { info, allows: () => true, crawlerAllows: () => true, text: null };
     }
     const robot = robotsParser(url, text);
     info.sitemaps = robot.getSitemaps();
+    const token = robotsUserAgentToken(userAgent);
     return {
         info,
         text: text.slice(0, MAX_ROBOTS_BYTES),
-        allows: (target) => {
-            const allowed = robot.isAllowed(target, userAgent);
-            return allowed === undefined ? true : allowed;
-        },
+        allows: (target) => robot.isAllowed(target, userAgent) ?? true,
+        // robots-parser keeps what comes before the first "/" of a User-Agent ("Mozilla" for the
+        // audit's own), so the product token is passed on its own.
+        crawlerAllows: (target) => robot.isAllowed(target, token) ?? true,
     };
 };
 export const loadRobotsTxt = async (origin, userAgent, options = {}) => {
@@ -60,9 +70,4 @@ export const loadRobotsTxt = async (origin, userAgent, options = {}) => {
         !/<html/iu.test(res.body.slice(0, 500));
     return buildRobots(url, res.status, res.error, found ? res.body : null, userAgent);
 };
-/** Short bot token used for robots.txt matching, derived from the full UA string. */
-export const robotsUserAgentToken = (userAgent) => {
-    const match = /agentic-seo-check|Googlebot|bingbot/iu.exec(userAgent);
-    return match ? match[0] : userAgent;
-};
 //# sourceMappingURL=robots.js.map

package/dist/core/robots.js.map

@@ -1 +1 @@
-{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../src/core/robots.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAI5C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAIvC,qFAAqF;AACrF,gGAAgG;AAChG,6CAA6C;AAC7C,MAAM,YAAY,GAAG,aAAa,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CACjD,eAAe,CAC0B,CAAC;AAU5C,yFAAyF;AACzF,MAAM,gBAAgB,GAAG,OAAO,CAAC;AAEjC;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,GAAW,EACX,IAAY,EACkC,EAAE;IAChD,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAEjE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC;AACnE,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAC7B,GAAW,EACX,IAAY,EACiC,EAAE;IAC/C,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAEjE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AACvE,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,GAAW,EACX,MAAqB,EACrB,KAAyB,EACzB,IAAmB,EACnB,SAAiB,EACH,EAAE;IAChB,MAAM,IAAI,GAAkB;QAC1B,GAAG;QACH,MAAM;QACN,KAAK,EAAE,IAAI,KAAK,IAAI;QACpB,QAAQ,EAAE,EAAE;QACZ,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI;QACzC,KAAK,EAAE,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC;KACtD,CAAC;IAEF,IAAI,KAAK,EAAE,CAAC;QACV,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAClD,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACtC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IAEpC,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC;QACrC,MAAM,EAAE,CAAC,MAAc,EAAE,EAAE;YACzB,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAEnD,OAAO,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC;QAChD,CAAC;KACF,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAChC,MAAc,EACd,SAAiB,EACjB,OAAO,GAAiB,EAAE,EACH,EAAE;IACzB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;IAEhD,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE;QAC/B,GAAG,OAAO;QACV,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,MAAM,EAAE,MAAM,CAAC;KACzD,CAAC,CAAC;IAEH,MAAM,KAAK,GACT,GAAG,CAAC,MAAM,KAAK,GAAG;QAClB,GAAG,CAAC,IAAI,KAAK,IAAI;QACjB,CAAC,GAAG,CAAC,KAAK;QACV,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAE1C,OAAO,WAAW,CAChB,GAAG,EACH,GAAG,CAAC,MAAM,EACV,GAAG,CAAC,KAAK,EACT,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EACvB,SAAS,CACV,CAAC;AACJ,CAAC,CAAC;AAEF,qFAAqF;AACrF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,SAAiB,EAAU,EAAE;IAChE,MAAM,KAAK,GAAG,uCAAuC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAEtE,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACtC,CAAC,CAAC"}
+{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../src/core/robots.ts"],"names":[],"mappings":"AAAA,OAAO,kBAAkB,MAAM,eAAe,CAAC;AAE/C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAavC;;;;;GAKG;AACH,MAAM,QAAQ,GAAG,CACf,QAAkD,EACpC,EAAE,CAAC,CAAC,SAAS,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;AAEzE,MAAM,YAAY,GAAG,QAAQ,CAAC,kBAAkB,CAAC,CAAC;AAmBlD,yFAAyF;AACzF,MAAM,gBAAgB,GAAG,OAAO,CAAC;AAEjC,qFAAqF;AACrF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,SAAiB,EAAU,EAAE;IAChE,MAAM,KAAK,GAAG,uCAAuC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAEtE,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACtC,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,GAAW,EACX,IAAY,EACkC,EAAE;IAChD,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAEjE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC;AACnE,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAC7B,GAAW,EACX,IAAY,EACiC,EAAE;IAC/C,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAEjE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AACvE,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,GAAW,EACX,MAAqB,EACrB,KAAyB,EACzB,IAAmB,EACnB,SAAiB,EACH,EAAE;IAChB,MAAM,IAAI,GAAkB;QAC1B,GAAG;QACH,MAAM;QACN,KAAK,EAAE,IAAI,KAAK,IAAI;QACpB,QAAQ,EAAE,EAAE;QACZ,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI;QACzC,KAAK,EAAE,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC;KACtD,CAAC;IAEF,IAAI,KAAK,EAAE,CAAC;QACV,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC7E,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACtC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,oBAAoB,CAAC,SAAS,CAAC,CAAC;IAE9C,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC;QACrC,MAAM,EAAE,CAAC,MAAc,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,IAAI;QACtE,yFAAyF;QACzF,2DAA2D;QAC3D,aAAa,EAAE,CAAC,MAAc,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,IAAI;KAC1E,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAChC,MAAc,EACd,SAAiB,EACjB,OAAO,GAAiB,EAAE,EACH,EAAE;IACzB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;IAEhD,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE;QAC/B,GAAG,OAAO;QACV,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,MAAM,EAAE,MAAM,CAAC;KACzD,CAAC,CAAC;IAEH,MAAM,KAAK,GACT,GAAG,CAAC,MAAM,KAAK,GAAG;QAClB,GAAG,CAAC,IAAI,KAAK,IAAI;QACjB,CAAC,GAAG,CAAC,KAAK;QACV,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAE1C,OAAO,WAAW,CAChB,GAAG,EACH,GAAG,CAAC,MAAM,EACV,GAAG,CAAC,KAAK,EACT,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EACvB,SAAS,CACV,CAAC;AACJ,CAAC,CAAC"}

package/dist/core/rules/aeo.js

@@ -275,6 +275,17 @@ export const aeoRules = [
             })
             : null;
     }),
+    defineRule({
+        id: "aeo.page-truncated",
+        category: "aeo",
+        severity: "info",
+        description: "The page is larger than the audit reads (5 MB; 2 MB on the hosted server), so only its first part was audited.",
+        recommendation: "Findings about the rest of the page are missing. HTML this large is slow for crawlers too: move inline scripts, data and SVG into files the page loads. On the hosted server, the local server reads up to 5 MB.",
+    }, ({ page, response }, make) => response?.truncated
+        ? make(`The audit read the first ${(page.htmlBytes / 1_000_000).toFixed(1)} MB of the page and stopped there.`, 
+        // About the audit, not the page.
+        { fix: [], evidence: { bytesRead: page.htmlBytes } })
+        : null),
     defineRule({
         id: "aeo.noindex",
         category: "aeo",