ax-audit 3.6.0 → 4.0.0

package/dist/checks/robots-parser.d.ts

@@ -0,0 +1,110 @@
+/** A `User-agent` group and the rules attached to it. */
+export interface RobotsGroup {
+    /** User-agent tokens sharing this group, as written. */
+    userAgents: string[];
+    allow: string[];
+    disallow: string[];
+    /** Raw `Content-Signal:` values declared in this group. */
+    contentSignals: string[];
+    /** Raw `Content-Usage:` values declared in this group. */
+    contentUsage: string[];
+    /** Group-scoped RSL `License:` values (override the global ones per RSL §4.4.1). */
+    licenses: string[];
+    crawlDelay?: string;
+}
+/** Everything ax-audit reads out of a robots.txt, parsed once. */
+export interface RobotsTxt {
+    groups: RobotsGroup[];
+    sitemaps: string[];
+    /** Global (out-of-group) RSL `License:` directives. */
+    licenses: string[];
+    /** `Agentmap:` directives pointing at an ai-catalog / ARD document. */
+    agentmaps: string[];
+    /** `Content-Signal:` / `Content-Usage:` lines that appeared outside any group. */
+    orphanContentSignals: string[];
+    orphanContentUsage: string[];
+    /** True when Cloudflare's managed-content block markers are present. */
+    cloudflareManaged: boolean;
+    /** Total non-comment, non-blank lines — used to distinguish empty from absent. */
+    ruleLineCount: number;
+}
+/** Backwards-compatible per-bot view derived from the parsed groups. */
+export interface BotEntry {
+    name: string;
+    /** A full `Disallow: /` applies to this bot. */
+    disallowed: boolean;
+    /** Any `Disallow:` rule applies to this bot (full or path-scoped). */
+    hasRestrictions: boolean;
+    hasAllow: boolean;
+}
+/**
+ * Parse a robots.txt document. Never throws: unrecognised lines are ignored,
+ * which is exactly what RFC 9309 §2.2.4 requires of conforming parsers.
+ */
+export declare function parseRobotsTxt(text: string): RobotsTxt;
+/**
+ * Flatten parsed groups into one entry per User-agent token, preserving the
+ * shape the 3.x checks were written against.
+ *
+ * A bot named in several groups is merged: `disallowed` is true when any group
+ * fully blocks it. RFC 9309 says a crawler picks the most specific matching
+ * group, but robots.txt files that name the same token twice are almost always
+ * a mistake, so merging surfaces the restriction rather than hiding it.
+ */
+export declare function toBotEntries(robots: RobotsTxt): BotEntry[];
+/** Parse text straight into bot entries. Convenience for checks that need nothing else. */
+export declare function parseUserAgents(text: string): BotEntry[];
+/** Every RSL `License:` value in the document, group-scoped ones included. */
+export declare function collectLicenses(robots: RobotsTxt): string[];
+/**
+ * Whether robots.txt expresses the intent to block a crawler: an explicit full
+ * `Disallow: /` for it, or a wildcard full block with no explicit entry.
+ */
+export declare function intentBlocked(entries: BotEntry[], crawler: string): boolean;
+export interface ParsedSignals {
+    /** Normalised `name=value` pairs with known names. */
+    valid: string[];
+    /** Segments not matching `name=value`. */
+    malformed: string[];
+    /** Well-formed pairs whose name is outside the policy vocabulary. */
+    unknown: string[];
+    /** Well-formed pairs with a known name but an out-of-vocabulary value. */
+    invalidValue: string[];
+}
+/**
+ * Parse one `Content-Signal:` value.
+ *
+ * Accepts the three boolean signals (`search`, `ai-input`, `ai-train` with
+ * `yes`/`no`) and the `use` field Cloudflare added on 2026-07-01, whose values
+ * are `immediate`, `reference` (the stated default) and `full`. Cloudflare's
+ * managed robots.txt writes `Content-signal:` with a lowercase `s` and a space
+ * after each comma, while the launch blog uses `Content-Signal:` without
+ * spaces; both parse identically here.
+ */
+export declare function parseContentSignalValue(raw: string): ParsedSignals;
+export interface ParsedUsage {
+    /** Path pattern the preference is scoped to, or `null` for the whole group. */
+    path: string | null;
+    /** Normalised `token=value` pairs over the AIPREF vocabulary. */
+    valid: string[];
+    /** Dictionary members outside the AIPREF vocabulary. */
+    unknown: string[];
+    /** Known tokens with values other than `y`/`n`. */
+    invalidValue: string[];
+    /** Segments the Structured Fields parser rejected. */
+    malformed: string[];
+    /**
+     * Content Signals / RSL tokens used under `Content-Usage`, e.g. `ai-train=no`
+     * where AIPREF expects `train-ai=n`. Reported separately because the two
+     * vocabularies are near-mirror images and confusing them is silent breakage.
+     */
+    crossVocabulary: string[];
+}
+/**
+ * Parse one `Content-Usage:` value per draft-ietf-aipref-attach-05 §3:
+ * an optional path pattern, whitespace, then a Structured Fields dictionary.
+ */
+export declare function parseContentUsageValue(raw: string): ParsedUsage;
+/** Extract every `License:` directive value (RSL 1.0 §4.4.1). */
+export declare function parseRobotsLicenseDirectives(text: string): string[];
+//# sourceMappingURL=robots-parser.d.ts.map

package/dist/checks/robots-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots-parser.d.ts","sourceRoot":"","sources":["../../src/checks/robots-parser.ts"],"names":[],"mappings":"AAgCA,yDAAyD;AACzD,MAAM,WAAW,WAAW;IAC1B,wDAAwD;IACxD,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,2DAA2D;IAC3D,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,0DAA0D;IAC1D,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,oFAAoF;IACpF,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,kEAAkE;AAClE,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,uDAAuD;IACvD,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,uEAAuE;IACvE,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,kFAAkF;IAClF,oBAAoB,EAAE,MAAM,EAAE,CAAC;IAC/B,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,wEAAwE;IACxE,iBAAiB,EAAE,OAAO,CAAC;IAC3B,kFAAkF;IAClF,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,wEAAwE;AACxE,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,UAAU,EAAE,OAAO,CAAC;IACpB,sEAAsE;IACtE,eAAe,EAAE,OAAO,CAAC;IACzB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAID;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CA+FtD;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,SAAS,GAAG,QAAQ,EAAE,CA0B1D;AAED,2FAA2F;AAC3F,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,EAAE,CAExD;AAED,8EAA8E;AAC9E,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,EAAE,CAE3D;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAI3E;AAID,MAAM,WAAW,aAAa;IAC5B,sDAAsD;IACtD,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,0CAA0C;IAC1C,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,qEAAqE;IACrE,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,0EAA0E;IAC1E,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED;;;;;;;;;GASG;AACH,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,CAoClE;AAID,MAAM,WAAW,WAAW;IAC1B,+EAA+E;IAC/E,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,iEAAiE;IACjE,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,wDAAwD;IACxD,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,mDAAmD;IACnD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,sDAAsD;IACtD,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB;;;;OAIG;IACH,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B;AAKD;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,MAAM,GAAG,WAAW,CA0C/D;AAID,iEAAiE;AACjE,wBAAgB,4BAA4B,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAEnE"}

package/dist/checks/robots-parser.js

@@ -0,0 +1,277 @@
+/**
+ * One robots.txt parser for every check that reads the file.
+ *
+ * Before 3.7 the group-splitting logic lived in three places (`robots-txt`
+ * counted User-agent groups, `rsl` scanned for `License:`, `agent-access`
+ * re-derived block intent) and each had its own idea of what closes a group.
+ * They now share this module, so a fix to the grouping rules fixes every check.
+ *
+ * Grouping follows RFC 9309 §2.2.1: consecutive `User-agent` lines accumulate
+ * into one group; the first rule line closes the accumulation, and the next
+ * `User-agent` starts a fresh group. Directives that are not part of a group
+ * (`Sitemap`) are collected globally.
+ *
+ * Beyond RFC 9309 this parser understands four extensions, all optional and all
+ * reported rather than assumed:
+ *
+ * - `Content-Signal:` — Content Signals Policy (contentsignals.org, CC0), served
+ *   by Cloudflare's managed robots.txt on millions of zones. Values are
+ *   `name=yes|no` pairs over `search`, `ai-input`, `ai-train`, plus the `use=`
+ *   field Cloudflare added on 2026-07-01 (`immediate` | `reference` | `full`).
+ * - `Content-Usage:` — IETF AIPREF (draft-ietf-aipref-attach-05, 2026-08-19).
+ *   An optional path pattern followed by a Structured Fields dictionary over
+ *   `train-ai` and `search` with `y`/`n` values. Note the token inversion
+ *   against Content Signals: AIPREF says `train-ai`, Content Signals says
+ *   `ai-train`. Mixing them up is a real-world bug this parser can surface.
+ * - `License:` — Really Simple Licensing 1.0 §4.4.1, global or group-scoped.
+ * - `Agentmap:` — the ARD / ai-catalog discovery hint Lighthouse's `ard-schema`
+ *   audit reads, analogous to `Sitemap:`.
+ */
+import { CONTENT_SIGNALS, CONTENT_SIGNAL_USE_VALUES, AIPREF_TOKENS, AIPREF_VALUES } from '../constants.js';
+import { parseDictionary } from './structured-fields.js';
+const GROUP_DIRECTIVES = /^(Disallow|Allow|Crawl-delay|Content-Signal|Content-Usage|License|Noindex|Request-rate):/i;
+/**
+ * Parse a robots.txt document. Never throws: unrecognised lines are ignored,
+ * which is exactly what RFC 9309 §2.2.4 requires of conforming parsers.
+ */
+export function parseRobotsTxt(text) {
+    const result = {
+        groups: [],
+        sitemaps: [],
+        licenses: [],
+        agentmaps: [],
+        orphanContentSignals: [],
+        orphanContentUsage: [],
+        cloudflareManaged: /#\s*(BEGIN|END)\s+Cloudflare\s+Managed\s+[Cc]ontent/i.test(text),
+        ruleLineCount: 0,
+    };
+    let current = null;
+    // True once a rule line has been seen in the current group: the next
+    // `User-agent` line then starts a new group instead of joining this one.
+    let groupClosed = false;
+    for (const rawLine of text.split('\n')) {
+        // A comment may follow a value on the same line; strip from the first `#`
+        // that is not inside the value we care about. robots.txt has no quoting,
+        // so a plain split is correct.
+        const line = rawLine.split('#')[0].trim();
+        if (!line)
+            continue;
+        result.ruleLineCount++;
+        const ua = line.match(/^User-agent:\s*(.*)$/i);
+        if (ua) {
+            const name = ua[1].trim();
+            if (!name)
+                continue;
+            if (current === null || groupClosed) {
+                current = {
+                    userAgents: [],
+                    allow: [],
+                    disallow: [],
+                    contentSignals: [],
+                    contentUsage: [],
+                    licenses: [],
+                };
+                result.groups.push(current);
+                groupClosed = false;
+            }
+            current.userAgents.push(name);
+            continue;
+        }
+        const sitemap = line.match(/^Sitemap:\s*(.+)$/i);
+        if (sitemap) {
+            result.sitemaps.push(sitemap[1].trim());
+            continue;
+        }
+        const agentmap = line.match(/^Agentmap:\s*(.+)$/i);
+        if (agentmap) {
+            result.agentmaps.push(agentmap[1].trim());
+            continue;
+        }
+        if (!GROUP_DIRECTIVES.test(line))
+            continue;
+        const [name, ...rest] = line.split(':');
+        const directive = name.trim().toLowerCase();
+        const value = rest.join(':').trim();
+        if (current !== null)
+            groupClosed = true;
+        switch (directive) {
+            case 'disallow':
+                current?.disallow.push(value);
+                break;
+            case 'allow':
+                current?.allow.push(value);
+                break;
+            case 'crawl-delay':
+                if (current)
+                    current.crawlDelay = value;
+                break;
+            case 'content-signal':
+                if (current)
+                    current.contentSignals.push(value);
+                else
+                    result.orphanContentSignals.push(value);
+                break;
+            case 'content-usage':
+                if (current)
+                    current.contentUsage.push(value);
+                else
+                    result.orphanContentUsage.push(value);
+                break;
+            case 'license':
+                // RSL §4.4.1: a License inside a group scopes to that group; before any
+                // group it applies site-wide.
+                if (current)
+                    current.licenses.push(value);
+                else
+                    result.licenses.push(value);
+                break;
+            default:
+                break;
+        }
+    }
+    return result;
+}
+/**
+ * Flatten parsed groups into one entry per User-agent token, preserving the
+ * shape the 3.x checks were written against.
+ *
+ * A bot named in several groups is merged: `disallowed` is true when any group
+ * fully blocks it. RFC 9309 says a crawler picks the most specific matching
+ * group, but robots.txt files that name the same token twice are almost always
+ * a mistake, so merging surfaces the restriction rather than hiding it.
+ */
+export function toBotEntries(robots) {
+    const byName = new Map();
+    for (const group of robots.groups) {
+        const fullBlock = group.disallow.some((p) => p === '/');
+        const anyDisallow = group.disallow.some((p) => p !== '');
+        const anyAllow = group.allow.length > 0;
+        for (const name of group.userAgents) {
+            const existing = byName.get(name);
+            if (existing) {
+                existing.disallowed ||= fullBlock;
+                existing.hasRestrictions ||= anyDisallow;
+                existing.hasAllow ||= anyAllow;
+                continue;
+            }
+            byName.set(name, {
+                name,
+                disallowed: fullBlock,
+                hasRestrictions: anyDisallow,
+                hasAllow: anyAllow,
+            });
+        }
+    }
+    return [...byName.values()];
+}
+/** Parse text straight into bot entries. Convenience for checks that need nothing else. */
+export function parseUserAgents(text) {
+    return toBotEntries(parseRobotsTxt(text));
+}
+/** Every RSL `License:` value in the document, group-scoped ones included. */
+export function collectLicenses(robots) {
+    return [...robots.licenses, ...robots.groups.flatMap((g) => g.licenses)];
+}
+/**
+ * Whether robots.txt expresses the intent to block a crawler: an explicit full
+ * `Disallow: /` for it, or a wildcard full block with no explicit entry.
+ */
+export function intentBlocked(entries, crawler) {
+    const explicit = entries.find((e) => e.name.toLowerCase() === crawler.toLowerCase());
+    if (explicit)
+        return explicit.disallowed;
+    return entries.find((e) => e.name === '*')?.disallowed ?? false;
+}
+/**
+ * Parse one `Content-Signal:` value.
+ *
+ * Accepts the three boolean signals (`search`, `ai-input`, `ai-train` with
+ * `yes`/`no`) and the `use` field Cloudflare added on 2026-07-01, whose values
+ * are `immediate`, `reference` (the stated default) and `full`. Cloudflare's
+ * managed robots.txt writes `Content-signal:` with a lowercase `s` and a space
+ * after each comma, while the launch blog uses `Content-Signal:` without
+ * spaces; both parse identically here.
+ */
+export function parseContentSignalValue(raw) {
+    const result = { valid: [], malformed: [], unknown: [], invalidValue: [] };
+    for (const segment of raw.split(',')) {
+        const s = segment.trim();
+        if (!s)
+            continue;
+        const m = s.match(/^([a-z][a-z-]*)\s*=\s*([a-z-]+)$/i);
+        if (!m) {
+            result.malformed.push(s);
+            continue;
+        }
+        const name = m[1].toLowerCase();
+        const value = m[2].toLowerCase();
+        if (name === 'use') {
+            if (CONTENT_SIGNAL_USE_VALUES.includes(value))
+                result.valid.push(`use=${value}`);
+            else
+                result.invalidValue.push(s);
+            continue;
+        }
+        if (!CONTENT_SIGNALS.includes(name)) {
+            result.unknown.push(s);
+            continue;
+        }
+        if (value !== 'yes' && value !== 'no') {
+            result.invalidValue.push(s);
+            continue;
+        }
+        result.valid.push(`${name}=${value}`);
+    }
+    return result;
+}
+/** Tokens that belong to a *different* usage vocabulary, flagged when seen under Content-Usage. */
+const FOREIGN_TOKENS = new Set(['ai-train', 'ai-input', 'ai-index', 'ai-all', 'use']);
+/**
+ * Parse one `Content-Usage:` value per draft-ietf-aipref-attach-05 §3:
+ * an optional path pattern, whitespace, then a Structured Fields dictionary.
+ */
+export function parseContentUsageValue(raw) {
+    const result = {
+        path: null,
+        valid: [],
+        unknown: [],
+        invalidValue: [],
+        malformed: [],
+        crossVocabulary: [],
+    };
+    let body = raw.trim();
+    // A leading path pattern is present when the first whitespace-delimited token
+    // starts with `/` (or `*`, for a bare wildcard pattern).
+    const firstSpace = body.search(/\s/);
+    if (firstSpace > 0) {
+        const head = body.slice(0, firstSpace);
+        if (head.startsWith('/') || head.startsWith('*')) {
+            result.path = head;
+            body = body.slice(firstSpace).trim();
+        }
+    }
+    const dict = parseDictionary(body);
+    result.malformed.push(...dict.malformed);
+    for (const member of dict.members) {
+        if (FOREIGN_TOKENS.has(member.key)) {
+            result.crossVocabulary.push(member.raw);
+            continue;
+        }
+        if (!AIPREF_TOKENS.includes(member.key)) {
+            result.unknown.push(member.raw);
+            continue;
+        }
+        if (!AIPREF_VALUES.includes(member.value.toLowerCase())) {
+            result.invalidValue.push(member.raw);
+            continue;
+        }
+        result.valid.push(`${member.key}=${member.value.toLowerCase()}`);
+    }
+    return result;
+}
+/* ── RSL License directives ─────────────────────────────────────────────── */
+/** Extract every `License:` directive value (RSL 1.0 §4.4.1). */
+export function parseRobotsLicenseDirectives(text) {
+    return collectLicenses(parseRobotsTxt(text));
+}
+//# sourceMappingURL=robots-parser.js.map

package/dist/checks/robots-parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots-parser.js","sourceRoot":"","sources":["../../src/checks/robots-parser.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,OAAO,EAAE,eAAe,EAAE,yBAAyB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC3G,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AA4CzD,MAAM,gBAAgB,GAAG,2FAA2F,CAAC;AAErH;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,MAAM,GAAc;QACxB,MAAM,EAAE,EAAE;QACV,QAAQ,EAAE,EAAE;QACZ,QAAQ,EAAE,EAAE;QACZ,SAAS,EAAE,EAAE;QACb,oBAAoB,EAAE,EAAE;QACxB,kBAAkB,EAAE,EAAE;QACtB,iBAAiB,EAAE,sDAAsD,CAAC,IAAI,CAAC,IAAI,CAAC;QACpF,aAAa,EAAE,CAAC;KACjB,CAAC;IAEF,IAAI,OAAO,GAAuB,IAAI,CAAC;IACvC,qEAAqE;IACrE,yEAAyE;IACzE,IAAI,WAAW,GAAG,KAAK,CAAC;IAExB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACvC,0EAA0E;QAC1E,yEAAyE;QACzE,+BAA+B;QAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,CAAC,aAAa,EAAE,CAAC;QAEvB,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC/C,IAAI,EAAE,EAAE,CAAC;YACP,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC,IAAI;gBAAE,SAAS;YACpB,IAAI,OAAO,KAAK,IAAI,IAAI,WAAW,EAAE,CAAC;gBACpC,OAAO,GAAG;oBACR,UAAU,EAAE,EAAE;oBACd,KAAK,EAAE,EAAE;oBACT,QAAQ,EAAE,EAAE;oBACZ,cAAc,EAAE,EAAE;oBAClB,YAAY,EAAE,EAAE;oBAChB,QAAQ,EAAE,EAAE;iBACb,CAAC;gBACF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC5B,WAAW,GAAG,KAAK,CAAC;YACtB,CAAC;YACD,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACjD,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACxC,SAAS;QACX,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACnD,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1C,SAAS;QACX,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QAE3C,MAAM,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAEpC,IAAI,OAAO,KAAK,IAAI;YAAE,WAAW,GAAG,IAAI,CAAC;QAEzC,QAAQ,SAAS,EAAE,CAAC;YAClB,KAAK,UAAU;gBACb,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC9B,MAAM;YACR,KAAK,OAAO;gBACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC3B,MAAM;YACR,KAAK,aAAa;gBAChB,IAAI,OAAO;oBAAE,OAAO,CAAC,UAAU,GAAG,KAAK,CAAC;gBACxC,MAAM;YACR,KAAK,gBAAgB;gBACnB,IAAI,OAAO;oBAAE,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;;oBAC3C,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC7C,MAAM;YACR,KAAK,eAAe;gBAClB,IAAI,OAAO;oBAAE,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;;oBACzC,MAAM,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC3C,MAAM;YACR,KAAK,SAAS;gBACZ,wEAAwE;gBACxE,8BAA8B;gBAC9B,IAAI,OAAO;oBAAE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;;oBACrC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACjC,MAAM;YACR;gBACE,MAAM;QACV,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAAC,MAAiB;IAC5C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;IAE3C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;QACxD,MAAM,WAAW,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QAExC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACpC,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClC,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,CAAC,UAAU,KAAK,SAAS,CAAC;gBAClC,QAAQ,CAAC,eAAe,KAAK,WAAW,CAAC;gBACzC,QAAQ,CAAC,QAAQ,KAAK,QAAQ,CAAC;gBAC/B,SAAS;YACX,CAAC;YACD,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE;gBACf,IAAI;gBACJ,UAAU,EAAE,SAAS;gBACrB,eAAe,EAAE,WAAW;gBAC5B,QAAQ,EAAE,QAAQ;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;AAC9B,CAAC;AAED,2FAA2F;AAC3F,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,OAAO,YAAY,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;AAC5C,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,eAAe,CAAC,MAAiB;IAC/C,OAAO,CAAC,GAAG,MAAM,CAAC,QAAQ,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,OAAmB,EAAE,OAAe;IAChE,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IACrF,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC,UAAU,CAAC;IACzC,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,UAAU,IAAI,KAAK,CAAC;AAClE,CAAC;AAeD;;;;;;;;;GASG;AACH,MAAM,UAAU,uBAAuB,CAAC,GAAW;IACjD,MAAM,MAAM,GAAkB,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC;IAE1F,KAAK,MAAM,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC;YAAE,SAAS;QAEjB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,IAAI,CAAC,CAAC,EAAE,CAAC;YACP,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACzB,SAAS;QACX,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;QAEjC,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;YACnB,IAAI,yBAAyB,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,EAAE,CAAC,CAAC;;gBAC5E,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjC,SAAS;QACX,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACpC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACvB,SAAS;QACX,CAAC;QAED,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACtC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5B,SAAS;QACX,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAuBD,mGAAmG;AACnG,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;AAEtF;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,GAAW;IAChD,MAAM,MAAM,GAAgB;QAC1B,IAAI,EAAE,IAAI;QACV,KAAK,EAAE,EAAE;QACT,OAAO,EAAE,EAAE;QACX,YAAY,EAAE,EAAE;QAChB,SAAS,EAAE,EAAE;QACb,eAAe,EAAE,EAAE;KACpB,CAAC;IAEF,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IACtB,8EAA8E;IAC9E,yDAAyD;IACzD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QACvC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACjD,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC;YACnB,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;QACvC,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;IAEzC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,IAAI,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACxC,SAAS;QACX,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACxC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAChC,SAAS;QACX,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YACxD,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrC,SAAS;QACX,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,+EAA+E;AAE/E,iEAAiE;AACjE,MAAM,UAAU,4BAA4B,CAAC,IAAY;IACvD,OAAO,eAAe,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;AAC/C,CAAC"}

package/dist/checks/robots-txt.d.ts

@@ -1,24 +1,6 @@
 import type { CheckContext, CheckResult, CheckMeta } from '../types.js';
 export declare const meta: CheckMeta;
-export interface BotEntry {
-    name: string;
-    disallowed: boolean;
-    hasRestrictions: boolean;
-    hasAllow: boolean;
-}
+export { parseUserAgents, intentBlocked } from './robots-parser.js';
+export type { BotEntry } from './robots-parser.js';
 export default function check(ctx: CheckContext): Promise<CheckResult>;
-export declare function parseUserAgents(text: string): BotEntry[];
-interface ContentSignalDecl {
-    /** User-agent names of the group this declaration belongs to (empty = outside any group). */
-    userAgents: string[];
-    /** Raw directive value, e.g. "search=yes, ai-train=no". */
-    raw: string;
-}
-/**
- * Extract `Content-Signal:` declarations and the User-agent group each belongs to.
- * Mirrors the grouping rules of `parseUserAgents`: consecutive User-agent lines form
- * one group; any directive line closes the group.
- */
-export declare function parseContentSignalDecls(text: string): ContentSignalDecl[];
-export {};
 //# sourceMappingURL=robots-txt.d.ts.map

package/dist/checks/robots-txt.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"robots-txt.d.ts","sourceRoot":"","sources":["../../src/checks/robots-txt.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAW,MAAM,aAAa,CAAC;AAGjF,eAAO,MAAM,IAAI,EAAE,SAKlB,CAAC;AAEF,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,OAAO,CAAC;IACpB,eAAe,EAAE,OAAO,CAAC;IACzB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,wBAA8B,KAAK,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAkI3E;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,EAAE,CAwCxD;AAID,UAAU,iBAAiB;IACzB,6FAA6F;IAC7F,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,2DAA2D;IAC3D,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE,CAgCzE"}
+{"version":3,"file":"robots-txt.d.ts","sourceRoot":"","sources":["../../src/checks/robots-txt.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAW,MAAM,aAAa,CAAC;AAUjF,eAAO,MAAM,IAAI,EAAE,SAIlB,CAAC;AAIF,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACpE,YAAY,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAEnD,wBAA8B,KAAK,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAqJ3E"}