ax-audit 3.1.0 → 3.6.0

package/dist/checks/crawl-efficiency.js

@@ -0,0 +1,122 @@
+import { guideUrl } from '../guide-urls.js';
+import { buildResult } from './utils.js';
+export const meta = {
+    id: 'crawl-efficiency',
+    name: 'Crawl Efficiency',
+    description: 'Checks compression, conditional GET (ETag / Last-Modified), and response size',
+    weight: 0, // Informational in 3.x — will gain weight in v4.0 (score-affecting changes are treated as breaking).
+};
+/** Page sizes (decompressed HTML bytes) above this earn a warning. */
+const LARGE_PAGE_BYTES = 2_000_000;
+const ACCEPTABLE_PAGE_BYTES = 500_000;
+export default async function check(ctx) {
+    const start = performance.now();
+    const findings = [];
+    let score = 100;
+    // Request explicitly advertising modern compression. The runtime's fetch
+    // transparently decompresses the body, but the Content-Encoding header still
+    // reflects what the server sent on the wire.
+    const res = await ctx.fetch(ctx.url, { headers: { 'Accept-Encoding': 'br, gzip, deflate' } });
+    if (!res.ok) {
+        findings.push({
+            status: 'fail',
+            message: 'Homepage request failed — cannot assess crawl efficiency',
+            detail: `HTTP ${res.status || 'network error'}`,
+            learnMoreUrl: guideUrl(meta.id, 'fetch-failed'),
+        });
+        return buildResult(meta, 0, findings, start);
+    }
+    /* ── Compression ──────────────────────────────────────────────────── */
+    const encoding = (res.headers['content-encoding'] ?? '').toLowerCase().trim();
+    if (encoding === 'br') {
+        findings.push({ status: 'pass', message: 'Response compressed with Brotli (br)' });
+    }
+    else if (encoding === 'gzip' || encoding === 'deflate' || encoding === 'zstd') {
+        findings.push({
+            status: 'pass',
+            message: `Response compressed with ${encoding}`,
+            detail: 'Brotli (br) typically compresses text 10–20% smaller — consider enabling it.',
+        });
+    }
+    else {
+        findings.push({
+            status: 'warn',
+            message: 'Response is not compressed',
+            detail: encoding ? `Content-Encoding: ${encoding}` : 'No Content-Encoding header',
+            hint: 'Enable Brotli or gzip for text responses. Compression cuts bytes transferred by ~70–80% and ' +
+                'reduces crawl cost for every agent and bot that fetches your pages.',
+            learnMoreUrl: guideUrl(meta.id, 'no-compression'),
+        });
+        score -= 30;
+    }
+    /* ── Conditional GET (ETag / Last-Modified → 304) ─────────────────── */
+    const etag = res.headers['etag'];
+    const lastModified = res.headers['last-modified'];
+    if (!etag && !lastModified) {
+        findings.push({
+            status: 'warn',
+            message: 'No ETag or Last-Modified header — conditional requests unsupported',
+            hint: 'Send an ETag or Last-Modified header so crawlers can revalidate with If-None-Match / ' +
+                'If-Modified-Since and receive a cheap 304 Not Modified instead of the full body.',
+            learnMoreUrl: guideUrl(meta.id, 'no-validators'),
+        });
+        score -= 30;
+    }
+    else {
+        const validator = etag ? 'ETag' : 'Last-Modified';
+        findings.push({ status: 'pass', message: `Cache validator present (${validator})` });
+        const conditionalHeaders = {};
+        if (etag)
+            conditionalHeaders['If-None-Match'] = etag;
+        if (lastModified)
+            conditionalHeaders['If-Modified-Since'] = lastModified;
+        const conditional = await ctx.fetch(ctx.url, { headers: conditionalHeaders });
+        if (conditional.status === 304) {
+            findings.push({ status: 'pass', message: 'Conditional request returns 304 Not Modified' });
+        }
+        else {
+            findings.push({
+                status: 'warn',
+                message: `Conditional request returned ${conditional.status} instead of 304 Not Modified`,
+                detail: `Re-requested with ${Object.keys(conditionalHeaders).join(' + ')}`,
+                hint: 'The server advertises a cache validator but does not honor If-None-Match / If-Modified-Since. ' +
+                    'Configure it to return 304 when the validator matches, so crawlers avoid re-downloading unchanged pages.',
+                learnMoreUrl: guideUrl(meta.id, 'no-304'),
+            });
+            score -= 15;
+        }
+    }
+    /* ── Response size ────────────────────────────────────────────────── */
+    const bytes = Buffer.byteLength(res.body, 'utf8');
+    if (bytes > LARGE_PAGE_BYTES) {
+        findings.push({
+            status: 'warn',
+            message: `Homepage is very large (${formatBytes(bytes)} decompressed)`,
+            hint: 'Large documents inflate crawl cost and token usage. Trim inlined data, split content, ' +
+                'or serve a Markdown representation to agents (see the content-negotiation check).',
+            learnMoreUrl: guideUrl(meta.id, 'large-page'),
+        });
+        score -= 10;
+    }
+    else if (bytes > ACCEPTABLE_PAGE_BYTES) {
+        findings.push({
+            status: 'warn',
+            message: `Homepage is on the large side (${formatBytes(bytes)} decompressed)`,
+            hint: 'Consider trimming inlined payloads to reduce crawl cost.',
+            learnMoreUrl: guideUrl(meta.id, 'large-page'),
+        });
+        score -= 5;
+    }
+    else {
+        findings.push({ status: 'pass', message: `Homepage size is reasonable (${formatBytes(bytes)} decompressed)` });
+    }
+    return buildResult(meta, score, findings, start);
+}
+function formatBytes(bytes) {
+    if (bytes < 1024)
+        return `${bytes} B`;
+    if (bytes < 1024 * 1024)
+        return `${(bytes / 1024).toFixed(1)} KB`;
+    return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
+//# sourceMappingURL=crawl-efficiency.js.map

package/dist/checks/crawl-efficiency.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl-efficiency.js","sourceRoot":"","sources":["../../src/checks/crawl-efficiency.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAEzC,MAAM,CAAC,MAAM,IAAI,GAAc;IAC7B,EAAE,EAAE,kBAAkB;IACtB,IAAI,EAAE,kBAAkB;IACxB,WAAW,EAAE,+EAA+E;IAC5F,MAAM,EAAE,CAAC,EAAE,qGAAqG;CACjH,CAAC;AAEF,sEAAsE;AACtE,MAAM,gBAAgB,GAAG,SAAS,CAAC;AACnC,MAAM,qBAAqB,GAAG,OAAO,CAAC;AAEtC,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,GAAiB;IACnD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,KAAK,GAAG,GAAG,CAAC;IAEhB,yEAAyE;IACzE,6EAA6E;IAC7E,6CAA6C;IAC7C,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,EAAE,CAAC,CAAC;IAE9F,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,0DAA0D;YACnE,MAAM,EAAE,QAAQ,GAAG,CAAC,MAAM,IAAI,eAAe,EAAE;YAC/C,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,CAAC;SAChD,CAAC,CAAC;QACH,OAAO,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAED,yEAAyE;IAEzE,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IAC9E,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,sCAAsC,EAAE,CAAC,CAAC;IACrF,CAAC;SAAM,IAAI,QAAQ,KAAK,MAAM,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;QAChF,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,4BAA4B,QAAQ,EAAE;YAC/C,MAAM,EAAE,8EAA8E;SACvF,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,4BAA4B;YACrC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC,CAAC,4BAA4B;YACjF,IAAI,EACF,8FAA8F;gBAC9F,qEAAqE;YACvE,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,gBAAgB,CAAC;SAClD,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;IAED,yEAAyE;IAEzE,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACjC,MAAM,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IAElD,IAAI,CAAC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QAC3B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,oEAAoE;YAC7E,IAAI,EACF,uFAAuF;gBACvF,kFAAkF;YACpF,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,eAAe,CAAC;SACjD,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;SAAM,CAAC;QACN,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC;QAClD,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,4BAA4B,SAAS,GAAG,EAAE,CAAC,CAAC;QAErF,MAAM,kBAAkB,GAA2B,EAAE,CAAC;QACtD,IAAI,IAAI;YAAE,kBAAkB,CAAC,eAAe,CAAC,GAAG,IAAI,CAAC;QACrD,IAAI,YAAY;YAAE,kBAAkB,CAAC,mBAAmB,CAAC,GAAG,YAAY,CAAC;QAEzE,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,kBAAkB,EAAE,CAAC,CAAC;QAC9E,IAAI,WAAW,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC/B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,8CAA8C,EAAE,CAAC,CAAC;QAC7F,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,gCAAgC,WAAW,CAAC,MAAM,8BAA8B;gBACzF,MAAM,EAAE,qBAAqB,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;gBAC1E,IAAI,EACF,gGAAgG;oBAChG,0GAA0G;gBAC5G,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC;aAC1C,CAAC,CAAC;YACH,KAAK,IAAI,EAAE,CAAC;QACd,CAAC;IACH,CAAC;IAED,yEAAyE;IAEzE,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAClD,IAAI,KAAK,GAAG,gBAAgB,EAAE,CAAC;QAC7B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,2BAA2B,WAAW,CAAC,KAAK,CAAC,gBAAgB;YACtE,IAAI,EACF,wFAAwF;gBACxF,mFAAmF;YACrF,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC;SAC9C,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;SAAM,IAAI,KAAK,GAAG,qBAAqB,EAAE,CAAC;QACzC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,kCAAkC,WAAW,CAAC,KAAK,CAAC,gBAAgB;YAC7E,IAAI,EAAE,0DAA0D;YAChE,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC;SAC9C,CAAC,CAAC;QACH,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,gCAAgC,WAAW,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC;IACjH,CAAC;IAED,OAAO,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IAChC,IAAI,KAAK,GAAG,IAAI;QAAE,OAAO,GAAG,KAAK,IAAI,CAAC;IACtC,IAAI,KAAK,GAAG,IAAI,GAAG,IAAI;QAAE,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAClE,OAAO,GAAG,CAAC,KAAK,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;AACpD,CAAC"}

package/dist/checks/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/checks/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAkB/C,eAAO,MAAM,MAAM,EAAE,WAAW,EAgB/B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/checks/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAqB/C,eAAO,MAAM,MAAM,EAAE,WAAW,EAmB/B,CAAC"}

package/dist/checks/index.js

@@ -13,6 +13,9 @@ import seoBasics, { meta as seoBasicsMeta } from './seo-basics.js';
 import tlsHttps, { meta as tlsHttpsMeta } from './tls-https.js';
 import wellKnownAi, { meta as wellKnownAiMeta } from './well-known-ai.js';
 import contentNegotiation, { meta as contentNegotiationMeta } from './content-negotiation.js';
+import rsl, { meta as rslMeta } from './rsl.js';
+import agentAccess, { meta as agentAccessMeta } from './agent-access.js';
+import crawlEfficiency, { meta as crawlEfficiencyMeta } from './crawl-efficiency.js';
 export const checks = [
     { run: llmsTxt, meta: llmsTxtMeta },
     { run: robotsTxt, meta: robotsTxtMeta },
@@ -29,5 +32,8 @@ export const checks = [
     { run: tlsHttps, meta: tlsHttpsMeta },
     { run: wellKnownAi, meta: wellKnownAiMeta },
     { run: contentNegotiation, meta: contentNegotiationMeta },
+    { run: rsl, meta: rslMeta },
+    { run: agentAccess, meta: agentAccessMeta },
+    { run: crawlEfficiency, meta: crawlEfficiencyMeta },
 ];
 //# sourceMappingURL=index.js.map

package/dist/checks/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/checks/index.ts"],"names":[],"mappings":"AAEA,OAAO,OAAO,EAAE,EAAE,IAAI,IAAI,WAAW,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,SAAS,EAAE,EAAE,IAAI,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,SAAS,EAAE,EAAE,IAAI,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,WAAW,EAAE,EAAE,IAAI,IAAI,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,cAAc,EAAE,EAAE,IAAI,IAAI,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAClF,OAAO,QAAQ,EAAE,EAAE,IAAI,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,OAAO,EAAE,EAAE,IAAI,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,WAAW,EAAE,EAAE,IAAI,IAAI,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,GAAG,EAAE,EAAE,IAAI,IAAI,OAAO,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,aAAa,EAAE,EAAE,IAAI,IAAI,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,OAAO,EAAE,EAAE,IAAI,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,SAAS,EAAE,EAAE,IAAI,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,QAAQ,EAAE,EAAE,IAAI,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,WAAW,EAAE,EAAE,IAAI,IAAI,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1E,OAAO,kBAAkB,EAAE,EAAE,IAAI,IAAI,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAE9F,MAAM,CAAC,MAAM,MAAM,GAAkB;IACnC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE;IACnC,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE;IACvC,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE;IACvC,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE;IAC3C,EAAE,GAAG,EAAE,cAAc,EAAE,IAAI,EAAE,kBAAkB,EAAE;IACjD,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE;IACrC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE;IACnC,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE;IAC3C,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE;IAC3B,EAAE,GAAG,EAAE,aAAa,EAAE,IAAI,EAAE,iBAAiB,EAAE;IAC/C,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE;IACnC,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE;IACvC,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE;IACrC,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE;IAC3C,EAAE,GAAG,EAAE,kBAAkB,EAAE,IAAI,EAAE,sBAAsB,EAAE;CAC1D,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/checks/index.ts"],"names":[],"mappings":"AAEA,OAAO,OAAO,EAAE,EAAE,IAAI,IAAI,WAAW,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,SAAS,EAAE,EAAE,IAAI,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,SAAS,EAAE,EAAE,IAAI,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,WAAW,EAAE,EAAE,IAAI,IAAI,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,cAAc,EAAE,EAAE,IAAI,IAAI,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAClF,OAAO,QAAQ,EAAE,EAAE,IAAI,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,OAAO,EAAE,EAAE,IAAI,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,WAAW,EAAE,EAAE,IAAI,IAAI,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,GAAG,EAAE,EAAE,IAAI,IAAI,OAAO,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,aAAa,EAAE,EAAE,IAAI,IAAI,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,OAAO,EAAE,EAAE,IAAI,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,SAAS,EAAE,EAAE,IAAI,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,QAAQ,EAAE,EAAE,IAAI,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,WAAW,EAAE,EAAE,IAAI,IAAI,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1E,OAAO,kBAAkB,EAAE,EAAE,IAAI,IAAI,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAC9F,OAAO,GAAG,EAAE,EAAE,IAAI,IAAI,OAAO,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,WAAW,EAAE,EAAE,IAAI,IAAI,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,eAAe,EAAE,EAAE,IAAI,IAAI,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAErF,MAAM,CAAC,MAAM,MAAM,GAAkB;IACnC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE;IACnC,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE;IACvC,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE;IACvC,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE;IAC3C,EAAE,GAAG,EAAE,cAAc,EAAE,IAAI,EAAE,kBAAkB,EAAE;IACjD,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE;IACrC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE;IACnC,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE;IAC3C,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE;IAC3B,EAAE,GAAG,EAAE,aAAa,EAAE,IAAI,EAAE,iBAAiB,EAAE;IAC/C,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE;IACnC,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE;IACvC,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE;IACrC,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE;IAC3C,EAAE,GAAG,EAAE,kBAAkB,EAAE,IAAI,EAAE,sBAAsB,EAAE;IACzD,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE;IAC3B,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,eAAe,EAAE;IAC3C,EAAE,GAAG,EAAE,eAAe,EAAE,IAAI,EAAE,mBAAmB,EAAE;CACpD,CAAC"}

package/dist/checks/robots-txt.d.ts

@@ -1,4 +1,24 @@
 import type { CheckContext, CheckResult, CheckMeta } from '../types.js';
 export declare const meta: CheckMeta;
+export interface BotEntry {
+    name: string;
+    disallowed: boolean;
+    hasRestrictions: boolean;
+    hasAllow: boolean;
+}
 export default function check(ctx: CheckContext): Promise<CheckResult>;
+export declare function parseUserAgents(text: string): BotEntry[];
+interface ContentSignalDecl {
+    /** User-agent names of the group this declaration belongs to (empty = outside any group). */
+    userAgents: string[];
+    /** Raw directive value, e.g. "search=yes, ai-train=no". */
+    raw: string;
+}
+/**
+ * Extract `Content-Signal:` declarations and the User-agent group each belongs to.
+ * Mirrors the grouping rules of `parseUserAgents`: consecutive User-agent lines form
+ * one group; any directive line closes the group.
+ */
+export declare function parseContentSignalDecls(text: string): ContentSignalDecl[];
+export {};
 //# sourceMappingURL=robots-txt.d.ts.map

package/dist/checks/robots-txt.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"robots-txt.d.ts","sourceRoot":"","sources":["../../src/checks/robots-txt.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAW,MAAM,aAAa,CAAC;AAGjF,eAAO,MAAM,IAAI,EAAE,SAKlB,CAAC;AASF,wBAA8B,KAAK,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CA+H3E"}
+{"version":3,"file":"robots-txt.d.ts","sourceRoot":"","sources":["../../src/checks/robots-txt.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAW,MAAM,aAAa,CAAC;AAGjF,eAAO,MAAM,IAAI,EAAE,SAKlB,CAAC;AAEF,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,OAAO,CAAC;IACpB,eAAe,EAAE,OAAO,CAAC;IACzB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,wBAA8B,KAAK,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAkI3E;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,EAAE,CAwCxD;AAID,UAAU,iBAAiB;IACzB,6FAA6F;IAC7F,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,2DAA2D;IAC3D,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE,CAgCzE"}

package/dist/checks/robots-txt.js

@@ -1,4 +1,4 @@
-import { ALL_AI_CRAWLERS, CORE_AI_CRAWLERS } from '../constants.js';
+import { ALL_AI_CRAWLERS, CONTENT_SIGNALS, CORE_AI_CRAWLERS } from '../constants.js';
 import { guideUrl } from '../guide-urls.js';
 import { buildResult } from './utils.js';
 export const meta = {
@@ -101,6 +101,8 @@ export default async function check(ctx) {
         });
         score -= 5;
     }
+    // Content Signals Policy (contentsignals.org) — informational in 3.x: findings only, no score impact.
+    addContentSignalFindings(text, findings);
     const totalConfigured = ALL_AI_CRAWLERS.filter((bot) => configuredBots.some((b) => b.name.toLowerCase() === bot.toLowerCase()));
     findings.push({
         status: totalConfigured.length >= 10 ? 'pass' : 'warn',
@@ -114,7 +116,7 @@ export default async function check(ctx) {
     });
     return buildResult(meta, score, findings, start);
 }
-function parseUserAgents(text) {
+export function parseUserAgents(text) {
     const entries = [];
     let currentGroup = [];
     let inDirectives = false;
@@ -151,10 +153,116 @@ function parseUserAgents(text) {
                 entry.hasAllow = true;
             }
         }
-        else if (/^(Sitemap|Crawl-delay|Host):/i.test(trimmed)) {
+        else if (/^(Sitemap|Crawl-delay|Host|Content-Signal):/i.test(trimmed)) {
             inDirectives = true;
         }
     }
     return entries;
 }
+/**
+ * Extract `Content-Signal:` declarations and the User-agent group each belongs to.
+ * Mirrors the grouping rules of `parseUserAgents`: consecutive User-agent lines form
+ * one group; any directive line closes the group.
+ */
+export function parseContentSignalDecls(text) {
+    const decls = [];
+    let currentUAs = [];
+    let inDirectives = false;
+    for (const line of text.split('\n')) {
+        const trimmed = line.trim();
+        if (!trimmed || trimmed.startsWith('#'))
+            continue;
+        const uaMatch = trimmed.match(/^User-agent:\s*(.+)/i);
+        if (uaMatch) {
+            if (inDirectives) {
+                currentUAs = [];
+                inDirectives = false;
+            }
+            currentUAs.push(uaMatch[1].trim());
+            continue;
+        }
+        const csMatch = trimmed.match(/^Content-Signal:\s*(.+)/i);
+        if (csMatch) {
+            inDirectives = true;
+            decls.push({ userAgents: [...currentUAs], raw: csMatch[1].trim() });
+            continue;
+        }
+        if (/^(Disallow|Allow|Sitemap|Crawl-delay|Host):/i.test(trimmed)) {
+            inDirectives = true;
+        }
+    }
+    return decls;
+}
+function parseSignalPairs(raw) {
+    const result = { valid: [], malformed: [], unknown: [] };
+    for (const segment of raw.split(',')) {
+        const s = segment.trim();
+        if (!s)
+            continue;
+        const m = s.match(/^([a-z][a-z-]*)\s*=\s*(yes|no)$/i);
+        if (!m) {
+            result.malformed.push(s);
+            continue;
+        }
+        const name = m[1].toLowerCase();
+        if (!CONTENT_SIGNALS.includes(name)) {
+            result.unknown.push(s);
+            continue;
+        }
+        result.valid.push(`${name}=${m[2].toLowerCase()}`);
+    }
+    return result;
+}
+function addContentSignalFindings(text, findings) {
+    const decls = parseContentSignalDecls(text);
+    if (decls.length === 0) {
+        findings.push({
+            status: 'warn',
+            message: 'No Content-Signal directive found (optional)',
+            hint: 'Declare how crawlers may use your content after access with the Content Signals Policy, ' +
+                'e.g.: Content-Signal: search=yes, ai-train=no. Known signals: ' +
+                `${CONTENT_SIGNALS.join(', ')}. Generate yours at contentsignals.org.`,
+            learnMoreUrl: guideUrl(meta.id, 'missing-content-signals'),
+        });
+        return;
+    }
+    for (const decl of decls) {
+        const group = decl.userAgents.length > 0 ? decl.userAgents.join(', ') : null;
+        const { valid, malformed, unknown } = parseSignalPairs(decl.raw);
+        if (group === null) {
+            findings.push({
+                status: 'warn',
+                message: 'Content-Signal directive outside a User-agent group',
+                detail: `Content-Signal: ${decl.raw}`,
+                hint: 'Place Content-Signal inside a User-agent group, after the User-agent line it applies to.',
+                learnMoreUrl: guideUrl(meta.id, 'invalid-content-signal'),
+            });
+            continue;
+        }
+        if (valid.length > 0) {
+            findings.push({
+                status: 'pass',
+                message: `Content signals declared for User-agent: ${group} — ${valid.join(', ')}`,
+            });
+        }
+        if (malformed.length > 0) {
+            findings.push({
+                status: 'warn',
+                message: `Malformed content signal segment(s) for User-agent: ${group}`,
+                detail: malformed.join(', '),
+                hint: 'Use comma-delimited signal=yes|no pairs, e.g.: Content-Signal: search=yes, ai-train=no.',
+                learnMoreUrl: guideUrl(meta.id, 'invalid-content-signal'),
+            });
+        }
+        if (unknown.length > 0) {
+            findings.push({
+                status: 'warn',
+                message: `Unknown content signal name(s) for User-agent: ${group}`,
+                detail: unknown.join(', '),
+                hint: `The Content Signals Policy defines: ${CONTENT_SIGNALS.join(', ')}. Other names are ignored by crawlers.`,
+                learnMoreUrl: guideUrl(meta.id, 'unknown-content-signal'),
+            });
+        }
+    }
+}
 //# sourceMappingURL=robots-txt.js.map

package/dist/checks/robots-txt.js.map

@@ -1 +1 @@
-{"version":3,"file":"robots-txt.js","sourceRoot":"","sources":["../../src/checks/robots-txt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAEzC,MAAM,CAAC,MAAM,IAAI,GAAc;IAC7B,EAAE,EAAE,YAAY;IAChB,IAAI,EAAE,YAAY;IAClB,WAAW,EAAE,+CAA+C;IAC5D,MAAM,EAAE,EAAE;CACX,CAAC;AASF,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,GAAiB;IACnD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,KAAK,GAAG,GAAG,CAAC;IAEhB,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;IAErD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,uBAAuB;YAChC,IAAI,EAAE,8IAA8I;YACpJ,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC;SAC7C,CAAC,CAAC;QACH,OAAO,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEjE,MAAM,cAAc,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CACrD,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC,CACvE,CAAC;IACF,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,CACzC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC,CACjF,CAAC;IAEF,IAAI,cAAc,CAAC,MAAM,KAAK,gBAAgB,CAAC,MAAM,EAAE,CAAC;QACtD,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,gBAAgB,CAAC,MAAM,yCAAyC,EAAE,CAAC,CAAC;IACtH,CAAC;SAAM,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,cAAc,CAAC,MAAM,IAAI,gBAAgB,CAAC,MAAM,8BAA8B;YAC1F,MAAM,EAAE,YAAY,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAC5C,IAAI,EAAE,sFAAsF;YAC5F,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,kBAAkB,CAAC;SACpD,CAAC,CAAC;QACH,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IAC3E,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,2CAA2C;YACpD,MAAM,EAAE,aAAa,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAClD,IAAI,EAAE,kJAAkJ;YACxJ,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,kBAAkB,CAAC;SACpD,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;IAED,mDAAmD;IACnD,IAAI,aAAa,EAAE,UAAU,EAAE,CAAC;QAC9B,MAAM,iBAAiB,GAAG,gBAAgB,CAAC,MAAM,CAC/C,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC,CACjF,CAAC;QACF,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,GAAG,iBAAiB,CAAC,MAAM,wDAAwD;gBAC5F,MAAM,EAAE,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC;gBACpC,IAAI,EAAE,sJAAsJ;gBAC5J,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,qBAAqB,CAAC;aACvD,CAAC,CAAC;YACH,KAAK,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CACvC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,CAAC,UAAU,CACjH,CAAC;IACF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,WAAW,CAAC,MAAM,mCAAmC;YACjE,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACjD,IAAI,EAAE,oIAAoI;YAC1I,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,oBAAoB,CAAC;SACtD,CAAC,CAAC;QACH,KAAK,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;IAClC,CAAC;IAED,0EAA0E;IAC1E,MAAM,cAAc,GAAG,cAAc,CAAC,MAAM,CAC1C,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,IAAI,KAAK,GAAG;QACd,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;QACvE,CAAC,CAAC,eAAe;QACjB,CAAC,CAAC,CAAC,UAAU,CAChB,CAAC;IACF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,cAAc,CAAC,MAAM,+CAA+C;YAChF,MAAM,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACpD,IAAI,EAAE,4JAA4J;YAClK,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,sBAAsB,CAAC;SACxD,CAAC,CAAC;IACL,CAAC;IAED,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,2BAA2B,EAAE,CAAC,CAAC;IAC1E,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,4BAA4B;YACrC,IAAI,EAAE,wFAAwF;YAC9F,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,iBAAiB,CAAC;SACnD,CAAC,CAAC;QACH,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IAED,MAAM,eAAe,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CACrD,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC,CACvE,CAAC;IACF,QAAQ,CAAC,IAAI,CAAC;QACZ,MAAM,EAAE,eAAe,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;QACtD,OAAO,EAAE,GAAG,eAAe,CAAC,MAAM,IAAI,eAAe,CAAC,MAAM,wCAAwC;QACpG,GAAG,CAAC,eAAe,CAAC,MAAM,GAAG,EAAE;YAC7B,CAAC,CAAC;gBACE,IAAI,EAAE,mFAAmF;gBACzF,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,CAAC;aAChD;YACH,CAAC,CAAC,EAAE,CAAC;KACR,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,IAAI,YAAY,GAAe,EAAE,CAAC;IAClC,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAElD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACtD,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,YAAY,EAAE,CAAC;gBACjB,YAAY,GAAG,EAAE,CAAC;gBAClB,YAAY,GAAG,KAAK,CAAC;YACvB,CAAC;YACD,MAAM,KAAK,GAAa,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YAChH,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;aAAM,IAAI,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAChD,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBACjC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC;gBACxB,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC;YAC/B,CAAC;QACH,CAAC;aAAM,IAAI,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9C,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBACjC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC;YAC/B,CAAC;QACH,CAAC;aAAM,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACpC,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBACjC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;YACxB,CAAC;QACH,CAAC;aAAM,IAAI,+BAA+B,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACzD,YAAY,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
+{"version":3,"file":"robots-txt.js","sourceRoot":"","sources":["../../src/checks/robots-txt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACrF,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAEzC,MAAM,CAAC,MAAM,IAAI,GAAc;IAC7B,EAAE,EAAE,YAAY;IAChB,IAAI,EAAE,YAAY;IAClB,WAAW,EAAE,+CAA+C;IAC5D,MAAM,EAAE,EAAE;CACX,CAAC;AASF,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,GAAiB;IACnD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,KAAK,GAAG,GAAG,CAAC;IAEhB,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;IAErD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,uBAAuB;YAChC,IAAI,EAAE,8IAA8I;YACpJ,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC;SAC7C,CAAC,CAAC;QACH,OAAO,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEjE,MAAM,cAAc,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CACrD,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC,CACvE,CAAC;IACF,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,CACzC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC,CACjF,CAAC;IAEF,IAAI,cAAc,CAAC,MAAM,KAAK,gBAAgB,CAAC,MAAM,EAAE,CAAC;QACtD,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,gBAAgB,CAAC,MAAM,yCAAyC,EAAE,CAAC,CAAC;IACtH,CAAC;SAAM,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,cAAc,CAAC,MAAM,IAAI,gBAAgB,CAAC,MAAM,8BAA8B;YAC1F,MAAM,EAAE,YAAY,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAC5C,IAAI,EAAE,sFAAsF;YAC5F,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,kBAAkB,CAAC;SACpD,CAAC,CAAC;QACH,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IAC3E,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,2CAA2C;YACpD,MAAM,EAAE,aAAa,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAClD,IAAI,EAAE,kJAAkJ;YACxJ,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,kBAAkB,CAAC;SACpD,CAAC,CAAC;QACH,KAAK,IAAI,EAAE,CAAC;IACd,CAAC;IAED,mDAAmD;IACnD,IAAI,aAAa,EAAE,UAAU,EAAE,CAAC;QAC9B,MAAM,iBAAiB,GAAG,gBAAgB,CAAC,MAAM,CAC/C,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC,CACjF,CAAC;QACF,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,GAAG,iBAAiB,CAAC,MAAM,wDAAwD;gBAC5F,MAAM,EAAE,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC;gBACpC,IAAI,EAAE,sJAAsJ;gBAC5J,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,qBAAqB,CAAC;aACvD,CAAC,CAAC;YACH,KAAK,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CACvC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,CAAC,UAAU,CACjH,CAAC;IACF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,WAAW,CAAC,MAAM,mCAAmC;YACjE,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACjD,IAAI,EAAE,oIAAoI;YAC1I,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,oBAAoB,CAAC;SACtD,CAAC,CAAC;QACH,KAAK,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;IAClC,CAAC;IAED,0EAA0E;IAC1E,MAAM,cAAc,GAAG,cAAc,CAAC,MAAM,CAC1C,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,IAAI,KAAK,GAAG;QACd,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;QACvE,CAAC,CAAC,eAAe;QACjB,CAAC,CAAC,CAAC,UAAU,CAChB,CAAC;IACF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,cAAc,CAAC,MAAM,+CAA+C;YAChF,MAAM,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACpD,IAAI,EAAE,4JAA4J;YAClK,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,sBAAsB,CAAC;SACxD,CAAC,CAAC;IACL,CAAC;IAED,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,2BAA2B,EAAE,CAAC,CAAC;IAC1E,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,4BAA4B;YACrC,IAAI,EAAE,wFAAwF;YAC9F,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,iBAAiB,CAAC;SACnD,CAAC,CAAC;QACH,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IAED,sGAAsG;IACtG,wBAAwB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAEzC,MAAM,eAAe,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CACrD,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC,CACvE,CAAC;IACF,QAAQ,CAAC,IAAI,CAAC;QACZ,MAAM,EAAE,eAAe,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;QACtD,OAAO,EAAE,GAAG,eAAe,CAAC,MAAM,IAAI,eAAe,CAAC,MAAM,wCAAwC;QACpG,GAAG,CAAC,eAAe,CAAC,MAAM,GAAG,EAAE;YAC7B,CAAC,CAAC;gBACE,IAAI,EAAE,mFAAmF;gBACzF,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,CAAC;aAChD;YACH,CAAC,CAAC,EAAE,CAAC;KACR,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,IAAI,YAAY,GAAe,EAAE,CAAC;IAClC,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAElD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACtD,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,YAAY,EAAE,CAAC;gBACjB,YAAY,GAAG,EAAE,CAAC;gBAClB,YAAY,GAAG,KAAK,CAAC;YACvB,CAAC;YACD,MAAM,KAAK,GAAa,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YAChH,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;aAAM,IAAI,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAChD,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBACjC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC;gBACxB,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC;YAC/B,CAAC;QACH,CAAC;aAAM,IAAI,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9C,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBACjC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC;YAC/B,CAAC;QACH,CAAC;aAAM,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACpC,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBACjC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;YACxB,CAAC;QACH,CAAC;aAAM,IAAI,8CAA8C,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACxE,YAAY,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAWD;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAAY;IAClD,MAAM,KAAK,GAAwB,EAAE,CAAC;IACtC,IAAI,UAAU,GAAa,EAAE,CAAC;IAC9B,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAElD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACtD,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,YAAY,EAAE,CAAC;gBACjB,UAAU,GAAG,EAAE,CAAC;gBAChB,YAAY,GAAG,KAAK,CAAC;YACvB,CAAC;YACD,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACnC,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC1D,IAAI,OAAO,EAAE,CAAC;YACZ,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,CAAC,GAAG,UAAU,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACpE,SAAS;QACX,CAAC;QAED,IAAI,8CAA8C,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACjE,YAAY,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAQD,SAAS,gBAAgB,CAAC,GAAW;IACnC,MAAM,MAAM,GAAkB,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACxE,KAAK,MAAM,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACtD,IAAI,CAAC,CAAC,EAAE,CAAC;YACP,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACzB,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;QAChC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACpC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACvB,SAAS;QACX,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAY,EAAE,QAAmB;IACjE,MAAM,KAAK,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAE5C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,8CAA8C;YACvD,IAAI,EACF,0FAA0F;gBAC1F,gEAAgE;gBAChE,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,yCAAyC;YACxE,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,yBAAyB,CAAC;SAC3D,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7E,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,qDAAqD;gBAC9D,MAAM,EAAE,mBAAmB,IAAI,CAAC,GAAG,EAAE;gBACrC,IAAI,EAAE,0FAA0F;gBAChG,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,wBAAwB,CAAC;aAC1D,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,4CAA4C,KAAK,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;aACnF,CAAC,CAAC;QACL,CAAC;QACD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,uDAAuD,KAAK,EAAE;gBACvE,MAAM,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC5B,IAAI,EAAE,yFAAyF;gBAC/F,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,wBAAwB,CAAC;aAC1D,CAAC,CAAC;QACL,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,kDAAkD,KAAK,EAAE;gBAClE,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC1B,IAAI,EAAE,uCAAuC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,wCAAwC;gBAC/G,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,wBAAwB,CAAC;aAC1D,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/checks/rsl.d.ts

@@ -0,0 +1,6 @@
+import type { CheckContext, CheckResult, CheckMeta } from '../types.js';
+export declare const meta: CheckMeta;
+/** Extract `License:` directive values from robots.txt (RSL 1.0 §4.4.1). */
+export declare function parseRobotsLicenseDirectives(text: string): string[];
+export default function check(ctx: CheckContext): Promise<CheckResult>;
+//# sourceMappingURL=rsl.d.ts.map

package/dist/checks/rsl.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rsl.d.ts","sourceRoot":"","sources":["../../src/checks/rsl.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAW,MAAM,aAAa,CAAC;AAKjF,eAAO,MAAM,IAAI,EAAE,SAKlB,CAAC;AAUF,4EAA4E;AAC5E,wBAAgB,4BAA4B,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAOnE;AAWD,wBAA8B,KAAK,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CA+F3E"}

package/dist/checks/rsl.js

@@ -0,0 +1,252 @@
+import { RSL_MIME, RSL_NAMESPACE, RSL_PAYMENT_TYPES, RSL_PERMIT_TYPES, RSL_USAGE_TOKENS, RSL_USER_TOKENS, } from '../constants.js';
+import { guideUrl } from '../guide-urls.js';
+import { buildResult, checkContentType } from './utils.js';
+import { findLinkTags, getAttribute, getTagAttribute } from './html-utils.js';
+import { parseLinkHeader } from './http-headers.js';
+export const meta = {
+    id: 'rsl',
+    name: 'RSL License',
+    description: 'Checks Really Simple Licensing (RSL) discovery and license document validity',
+    weight: 0, // Informational in 3.x — will gain weight in v4.0 (score-affecting changes are treated as breaking).
+};
+/** Score when discovery exists but the referenced license document cannot be fetched. */
+const UNREACHABLE_DOC_SCORE = 25;
+/** Extract `License:` directive values from robots.txt (RSL 1.0 §4.4.1). */
+export function parseRobotsLicenseDirectives(text) {
+    const values = [];
+    for (const line of text.split('\n')) {
+        const m = line.trim().match(/^License:\s*(.+)$/i);
+        if (m)
+            values.push(m[1].trim());
+    }
+    return values;
+}
+function isAbsoluteUrl(value) {
+    try {
+        new URL(value);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export default async function check(ctx) {
+    const start = performance.now();
+    const findings = [];
+    let score = 100;
+    /* ── Discovery (RSL 1.0 §4.4–4.6) ─────────────────────────────────── */
+    const discoveries = [];
+    const robotsRes = await ctx.fetch(`${ctx.url}/robots.txt`);
+    const directives = robotsRes.ok ? parseRobotsLicenseDirectives(robotsRes.body) : [];
+    for (const value of directives) {
+        discoveries.push({ mechanism: 'robots.txt License directive', url: value });
+    }
+    const linkEntries = parseLinkHeader(ctx.headers['link'] ?? '');
+    for (const entry of linkEntries) {
+        if (entry.params['rel'] === 'license' && (entry.params['type'] ?? '').includes(RSL_MIME)) {
+            discoveries.push({ mechanism: 'Link header', url: entry.url });
+        }
+    }
+    for (const tag of findLinkTags(ctx.html, 'license')) {
+        const type = getAttribute(tag, 'type');
+        const href = getAttribute(tag, 'href');
+        if (type !== null && type.toLowerCase().includes(RSL_MIME) && href !== null) {
+            discoveries.push({ mechanism: 'HTML <link rel="license">', url: href });
+        }
+    }
+    if (discoveries.length === 0) {
+        findings.push({
+            status: 'fail',
+            message: 'No RSL license discovery found',
+            detail: 'Checked robots.txt License directive, Link header, and <link rel="license" type="application/rsl+xml">',
+            hint: 'Declare machine-readable licensing terms for your content with Really Simple Licensing. ' +
+                'Add to robots.txt: License: https://your-site.com/license.xml — then publish the RSL document. ' +
+                'See https://rslstandard.org.',
+            learnMoreUrl: guideUrl(meta.id, 'not-found'),
+        });
+        return buildResult(meta, 0, findings, start);
+    }
+    const mechanisms = [...new Set(discoveries.map((d) => d.mechanism))];
+    findings.push({
+        status: 'pass',
+        message: `RSL license discovered via ${mechanisms.join(' + ')} (${discoveries.length} reference(s))`,
+    });
+    // The robots.txt directive value MUST be an absolute URI (RSL 1.0 §4.4.1).
+    const relativeDirectives = directives.filter((d) => !isAbsoluteUrl(d));
+    if (relativeDirectives.length > 0) {
+        findings.push({
+            status: 'warn',
+            message: 'robots.txt License directive must be an absolute URI',
+            detail: relativeDirectives.join(', '),
+            hint: 'Use a fully qualified URL: License: https://your-site.com/license.xml',
+            learnMoreUrl: guideUrl(meta.id, 'relative-license-url'),
+        });
+        score -= 10;
+    }
+    /* ── Fetch and validate the license document ──────────────────────── */
+    const target = discoveries[0];
+    const docUrl = isAbsoluteUrl(target.url) ? target.url : new URL(target.url, `${ctx.url}/`).toString();
+    const docRes = await ctx.fetch(docUrl);
+    if (!docRes.ok) {
+        findings.push({
+            status: 'fail',
+            message: `RSL license document could not be fetched (${docUrl})`,
+            detail: `HTTP ${docRes.status || 'network error'}`,
+            hint: 'The discovery reference points to a missing document. Publish the RSL XML at the referenced URL.',
+            learnMoreUrl: guideUrl(meta.id, 'fetch-failed'),
+        });
+        return buildResult(meta, Math.min(score, UNREACHABLE_DOC_SCORE), findings, start);
+    }
+    const ctFinding = checkContentType(docRes, [RSL_MIME], {
+        checkId: meta.id,
+        resourceLabel: 'RSL license document',
+        anchor: 'wrong-content-type',
+    });
+    if (ctFinding) {
+        findings.push(ctFinding);
+        score -= 5;
+    }
+    else {
+        findings.push({ status: 'pass', message: `RSL document Content-Type OK (${RSL_MIME})` });
+    }
+    score = validateRslDocument(docRes.body, findings, score);
+    return buildResult(meta, score, findings, start);
+}
+/**
+ * Regex-based structural validation of an RSL 1.0 document. Not a full XML parse —
+ * consistent with the dependency-free approach used across all checks — but covers
+ * the conformance points of §2.2 and §3: root element + namespace, <content url>,
+ * <license> presence, permits/prohibits vocabulary, and payment types.
+ */
+function validateRslDocument(rawBody, findings, score) {
+    const body = rawBody.replace(/<!--[\s\S]*?-->/g, '');
+    if (!/<rsl[\s>]/i.test(body)) {
+        findings.push({
+            status: 'fail',
+            message: 'Root <rsl> element not found in license document',
+            hint: 'An RSL document must have <rsl xmlns="https://rslstandard.org/rsl"> as its root element.',
+            learnMoreUrl: guideUrl(meta.id, 'invalid-root'),
+        });
+        return score - 40;
+    }
+    const xmlns = getTagAttribute(body, 'rsl', 'xmlns');
+    if (xmlns === RSL_NAMESPACE) {
+        findings.push({ status: 'pass', message: 'Root <rsl> element with correct namespace' });
+    }
+    else {
+        findings.push({
+            status: 'warn',
+            message: xmlns === null ? 'Missing xmlns on <rsl> root element' : `Wrong RSL namespace: "${xmlns}"`,
+            hint: `Declare the default namespace: <rsl xmlns="${RSL_NAMESPACE}">`,
+            learnMoreUrl: guideUrl(meta.id, 'wrong-namespace'),
+        });
+        score -= 15;
+    }
+    const contentTags = [...body.matchAll(/<content\b([^>]*?)\/?>/gi)].map((m) => m[1]);
+    if (contentTags.length === 0) {
+        findings.push({
+            status: 'warn',
+            message: 'No <content> elements found in license document',
+            hint: 'Declare at least one <content url="/"> element wrapping a <license>.',
+            learnMoreUrl: guideUrl(meta.id, 'no-content-elements'),
+        });
+        return score - 20;
+    }
+    // `url` is required on every <content>; an empty value is legal for HTML pages (§3.3).
+    const missingUrl = contentTags.filter((attrs) => getAttribute(attrs, 'url') === null).length;
+    if (missingUrl > 0) {
+        findings.push({
+            status: 'warn',
+            message: `${missingUrl}/${contentTags.length} <content> element(s) missing the required url attribute`,
+            hint: 'Every <content> element must carry a url attribute identifying the licensed asset (url="" is allowed for pages linked via <link rel="license">).',
+            learnMoreUrl: guideUrl(meta.id, 'missing-content-url'),
+        });
+        score -= 10;
+    }
+    else {
+        findings.push({ status: 'pass', message: `${contentTags.length} <content> element(s), all with url attribute` });
+    }
+    if (!/<license[\s>]/i.test(body)) {
+        findings.push({
+            status: 'warn',
+            message: 'No <license> elements found in license document',
+            hint: 'Each <content> element must contain at least one <license> element with its terms.',
+            learnMoreUrl: guideUrl(meta.id, 'no-license-elements'),
+        });
+        score -= 15;
+    }
+    score = validatePermits(body, findings, score);
+    score = validatePayments(body, findings, score);
+    return score;
+}
+function validatePermits(body, findings, score) {
+    const invalidTypes = [];
+    const unknownTokens = [];
+    const validSummaries = [];
+    for (const m of body.matchAll(/<(permits|prohibits)\b([^>]*)>([^<]*)</gi)) {
+        const element = m[1].toLowerCase();
+        const type = (getAttribute(m[2], 'type') ?? '').toLowerCase();
+        const tokens = m[3].trim().split(/\s+/).filter(Boolean);
+        if (!RSL_PERMIT_TYPES.includes(type)) {
+            invalidTypes.push(`<${element} type="${type || '(none)'}">`);
+            continue;
+        }
+        const vocabulary = type === 'usage' ? RSL_USAGE_TOKENS : type === 'user' ? RSL_USER_TOKENS : null;
+        const bad = vocabulary !== null
+            ? tokens.filter((t) => !vocabulary.includes(t.toLowerCase()))
+            : tokens.filter((t) => !/^[A-Z]{2}$/.test(t)); // geo: ISO 3166-1 alpha-2
+        if (bad.length > 0) {
+            unknownTokens.push(`<${element} type="${type}">: ${bad.join(' ')}`);
+        }
+        else if (tokens.length > 0) {
+            validSummaries.push(`${element}[${type}]: ${tokens.join(' ')}`);
+        }
+    }
+    if (validSummaries.length > 0) {
+        findings.push({ status: 'pass', message: `License terms declared — ${validSummaries.join('; ')}` });
+    }
+    if (invalidTypes.length > 0) {
+        findings.push({
+            status: 'warn',
+            message: 'permits/prohibits with invalid type attribute',
+            detail: invalidTypes.join(', '),
+            hint: `Valid types are: ${RSL_PERMIT_TYPES.join(', ')}.`,
+            learnMoreUrl: guideUrl(meta.id, 'invalid-permits'),
+        });
+        score -= 5;
+    }
+    if (unknownTokens.length > 0) {
+        findings.push({
+            status: 'warn',
+            message: 'permits/prohibits with tokens outside the RSL vocabulary',
+            detail: unknownTokens.join('; '),
+            hint: `Usage tokens: ${RSL_USAGE_TOKENS.join(', ')}. User tokens: ${RSL_USER_TOKENS.join(', ')}. ` +
+                'Geo tokens: ISO 3166-1 alpha-2 codes. Pre-1.0 draft tokens (train-ai, train-genai, ai-use, ai-summarize) ' +
+                'were replaced in RSL 1.0 by ai-train, ai-input, and ai-index.',
+            learnMoreUrl: guideUrl(meta.id, 'invalid-permits'),
+        });
+        score -= 5;
+    }
+    return score;
+}
+function validatePayments(body, findings, score) {
+    const invalid = [];
+    for (const m of body.matchAll(/<payment\b([^>]*?)\/?>/gi)) {
+        const type = getAttribute(m[1], 'type');
+        if (type !== null && !RSL_PAYMENT_TYPES.includes(type.toLowerCase())) {
+            invalid.push(type);
+        }
+    }
+    if (invalid.length > 0) {
+        findings.push({
+            status: 'warn',
+            message: 'payment element(s) with invalid type attribute',
+            detail: invalid.join(', '),
+            hint: `Valid payment types are: ${RSL_PAYMENT_TYPES.join(', ')}. Omitting type means free.`,
+            learnMoreUrl: guideUrl(meta.id, 'invalid-payment-type'),
+        });
+        score -= 5;
+    }
+    return score;
+}
+//# sourceMappingURL=rsl.js.map