@rivi-software/agentic-seo-check 0.1.1 → 0.2.0

package/dist/core/aeo/audit.js

@@ -0,0 +1,310 @@
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import { DEFAULT_CRAWL, readStdin, resolveTarget, rollupIssues, trimFinding, } from "../audit.js";
+import { crawlSite } from "../crawl.js";
+import { DEFAULT_USER_AGENT, fetchPage, toResponseInfo } from "../fetch.js";
+import { extractPage } from "../html/extract.js";
+import { loadRobotsTxt } from "../robots.js";
+import { AEO_PAGE_RULES, runPageRules, sortFindings } from "../rules/index.js";
+import { pageScore, siteScore } from "../score.js";
+import { scanDirectory } from "../static.js";
+import { countFindings, DEFAULT_THRESHOLDS } from "../types.js";
+import { AI_BOTS, crawlerAccess } from "./bots.js";
+import { loadLlmsTxt, readLlmsTxt } from "./llms-txt.js";
+import { aeoPageSummary } from "./page.js";
+import { probeBots } from "./probe.js";
+import { aeoSiteFindings } from "./site-rules.js";
+const userAgentOf = (options) => options.fetch?.userAgent ?? DEFAULT_USER_AGENT;
+/** robots.txt and llms.txt of the origin a page is served from. */
+const loadOrigin = async (pageUrl, options) => {
+    const { origin } = new URL(pageUrl);
+    options.onProgress?.({
+        phase: "robots",
+        message: `Fetching ${origin}/robots.txt and ${origin}/llms.txt`,
+    });
+    const [robots, llmsTxt] = await Promise.all([
+        loadRobotsTxt(origin, userAgentOf(options), options.fetch),
+        loadLlmsTxt(origin, options.fetch),
+    ]);
+    return { robots: robots.info, robotsText: robots.text, llmsTxt };
+};
+const collectUrl = async (url, options) => {
+    const res = await fetchPage(url, options.fetch ?? {});
+    const page = extractPage(res.body ?? "", {
+        url: res.finalUrl,
+        includeSubdomains: options.includeSubdomains,
+    });
+    // A page that never answered says nothing about the site; its robots.txt would fail the same way.
+    const site = res.status === null ? null : await loadOrigin(res.finalUrl, options);
+    return {
+        target: url,
+        url: res.finalUrl,
+        source: "url",
+        pageSource: "url",
+        records: [
+            {
+                url: res.finalUrl,
+                requestedUrl: url,
+                page,
+                response: toResponseInfo(res),
+                depth: 0,
+            },
+        ],
+        robots: site?.robots ?? null,
+        robotsText: site?.robotsText ?? null,
+        llmsTxt: site?.llmsTxt ?? null,
+        live: true,
+    };
+};
+const collectSite = async (url, options) => {
+    const result = await crawlSite(url, {
+        maxPages: options.maxPages ?? DEFAULT_CRAWL.maxPages,
+        maxDepth: options.maxDepth ?? DEFAULT_CRAWL.maxDepth,
+        concurrency: options.concurrency ?? DEFAULT_CRAWL.concurrency,
+        delayMs: options.delayMs ?? DEFAULT_CRAWL.delayMs,
+        respectRobots: options.respectRobots ?? true,
+        includeSubdomains: options.includeSubdomains ?? false,
+        checkExternalLinks: false,
+        // The AEO audit reads pages; broken links, sitemaps and redirect probes are the SEO crawl's job.
+        maxLinkChecks: 0,
+        siteChecks: false,
+        // oxlint-disable-next-line eslint/require-unicode-regexp -- the user's own patterns, in JavaScript's default syntax; the u flag would reject some of them
+        includePatterns: (options.includePatterns ?? []).map((p) => new RegExp(p)),
+        // oxlint-disable-next-line eslint/require-unicode-regexp -- the user's own patterns, in JavaScript's default syntax; the u flag would reject some of them
+        excludePatterns: (options.excludePatterns ?? []).map((p) => new RegExp(p)),
+        fetch: { ...options.fetch, userAgent: userAgentOf(options) },
+        onProgress: options.onProgress,
+    });
+    options.onProgress?.({ phase: "llms-txt", message: "Fetching llms.txt" });
+    return {
+        target: url,
+        url: result.startUrl,
+        source: "crawl",
+        pageSource: "url",
+        records: result.records,
+        robots: result.robots,
+        robotsText: result.robotsText ?? null,
+        llmsTxt: await loadLlmsTxt(new URL(result.startUrl).origin, options.fetch),
+        live: true,
+    };
+};
+const collectDirectory = async (dir, options) => {
+    const baseUrl = options.baseUrl ?? "http://localhost/";
+    const result = await scanDirectory(dir, {
+        baseUrl,
+        includeSubdomains: options.includeSubdomains ?? false,
+        maxPages: options.maxPages ?? 500,
+        userAgent: options.fetch?.userAgent,
+    });
+    return {
+        target: dir,
+        url: dir,
+        source: "directory",
+        pageSource: "file",
+        records: result.records,
+        robots: result.robots,
+        robotsText: result.robotsText ?? null,
+        llmsTxt: await readLlmsTxt(dir, baseUrl),
+        live: false,
+    };
+};
+/** One page from HTML in hand: no site around it, so no robots.txt or llms.txt. */
+const collectHtml = (html, url, target, source, options) => ({
+    target,
+    url,
+    source,
+    pageSource: source,
+    records: [
+        {
+            url,
+            requestedUrl: url,
+            page: extractPage(html, {
+                url,
+                includeSubdomains: options.includeSubdomains,
+            }),
+            depth: 0,
+        },
+    ],
+    robots: null,
+    robotsText: null,
+    llmsTxt: null,
+    live: false,
+});
+const evaluatePage = (record, source, options) => {
+    let findings = runPageRules({
+        page: record.page,
+        source,
+        response: record.response,
+        thresholds: DEFAULT_THRESHOLDS,
+        depth: record.depth,
+    }, { disable: new Set(options.disableRules), rules: AEO_PAGE_RULES });
+    if (options.includeInfo === false) {
+        findings = findings.filter((f) => f.severity !== "info");
+    }
+    const counts = countFindings(findings);
+    return { record, findings, counts, score: pageScore(counts) };
+};
+/**
+ * Requests the start page as each AI bot and records the answers on `crawlers`. Returns why the
+ * probe could not run, when it could not.
+ */
+const runProbe = async (c, start, crawlers, options) => {
+    const response = start?.record.response;
+    if (!c.live || !response) {
+        return "it needs a live URL, and this target is HTML, a file or a directory";
+    }
+    if (response.error || response.status === null || response.status >= 400) {
+        return `the audit's own request got ${response.status === null ? "no answer" : `HTTP ${response.status}`}, so a refusal could not be told apart from a page that fails for everyone`;
+    }
+    if (!crawlers) {
+        return "robots.txt could not be read";
+    }
+    options.onProgress?.({
+        phase: "ai-bots",
+        message: `Requesting ${response.finalUrl} with AI bot User-Agents`,
+    });
+    const probes = await probeBots(response.finalUrl, AI_BOTS, options.fetch);
+    crawlers.probeBaseline = {
+        userAgent: userAgentOf(options),
+        status: response.status,
+    };
+    for (const bot of crawlers.bots) {
+        const probe = probes.get(bot.name);
+        if (probe) {
+            bot.probe = probe;
+        }
+    }
+    return undefined;
+};
+const blockedBots = (crawlers, purpose) => crawlers
+    ? crawlers.bots
+        .filter((b) => b.purpose === purpose && (!b.allowed || b.blockedPages > 0))
+        .map((b) => b.name)
+    : [];
+const pageEntry = (p) => {
+    // url and depth lead the entry, the way a reader scans a list of pages.
+    const head = {
+        url: p.record.file ?? p.record.url,
+    };
+    if (p.record.depth >= 0) {
+        head.depth = p.record.depth;
+    }
+    return {
+        ...head,
+        score: p.score,
+        counts: p.counts,
+        summary: aeoPageSummary(p.record.page, p.record.response),
+    };
+};
+const buildAudit = async (c, options) => {
+    const isSite = c.source === "crawl" || c.source === "directory";
+    const pages = c.records.map((r) => evaluatePage(r, c.pageSource, options));
+    const start = pages.find((p) => p.record.depth === 0) ?? pages[0];
+    const crawlers = c.robots
+        ? crawlerAccess(c.robots, c.robotsText, start?.record.url ?? c.url, c.records.map((r) => r.url))
+        : null;
+    const probeSkipped = options.aiBots
+        ? await runProbe(c, start, crawlers, options)
+        : undefined;
+    const disabled = new Set(options.disableRules);
+    let siteFindings = aeoSiteFindings({
+        crawlers,
+        llmsTxt: c.llmsTxt,
+        pageCount: pages.length,
+        probeSkipped,
+    }).filter((f) => !disabled.has(f.rule));
+    if (options.includeInfo === false) {
+        siteFindings = siteFindings.filter((f) => f.severity !== "info");
+    }
+    const pageFindings = pages.flatMap((p) => p.findings);
+    const audit = {
+        schemaVersion: 1,
+        kind: "aeo",
+        target: c.target,
+        url: c.url,
+        source: c.source,
+        score: siteScore(pages, siteFindings),
+        counts: countFindings([...siteFindings, ...pageFindings]),
+        auditedAt: new Date().toISOString(),
+        summary: {
+            pages: pages.length,
+            robotsTxt: crawlers?.status ?? null,
+            searchCrawlersBlocked: blockedBots(crawlers, "search"),
+            answerBotsBlocked: blockedBots(crawlers, "answer"),
+            trainingBotsBlocked: blockedBots(crawlers, "training"),
+            trainingPreference: crawlers?.trainingPreference ?? null,
+            blockedByServer: crawlers?.probeBaseline
+                ? crawlers.bots.filter((b) => b.probe?.blocked).map((b) => b.name)
+                : null,
+            llmsTxt: c.llmsTxt?.status ?? null,
+            page: start
+                ? aeoPageSummary(start.record.page, start.record.response)
+                : null,
+        },
+        findings: sortFindings(isSite && options.pageDetail === "summary"
+            ? siteFindings
+            : [...siteFindings, ...pageFindings]),
+        crawlers,
+        llmsTxt: c.llmsTxt,
+    };
+    if (isSite) {
+        audit.pages = pages
+            .toSorted((a, b) => a.record.depth - b.record.depth || a.score - b.score)
+            .map(pageEntry);
+        audit.issues = rollupIssues(pages.map((p) => ({
+            url: p.record.file ?? p.record.url,
+            findings: p.findings,
+        })));
+    }
+    return audit;
+};
+/** AEO audit of an HTML string. `url` is where the HTML would be served, for links and display. */
+// oxlint-disable-next-line eslint/require-await -- public API: it returns a promise like the other audit functions, so a throw reaches callers as a rejection
+export const auditAeoHtml = async (html, input = {}, options = {}) => {
+    const url = input.url ?? options.baseUrl ?? "http://localhost/";
+    return buildAudit(collectHtml(html, url, input.target ?? url, "html", options), options);
+};
+/** AEO audit of one live page, with its site's robots.txt and llms.txt. */
+export const auditAeoUrl = async (url, options = {}) => buildAudit(await collectUrl(url, options), options);
+/** AEO audit of a site, crawled from `startUrl`. */
+export const auditAeoSite = async (startUrl, options = {}) => buildAudit(await collectSite(startUrl, options), options);
+/** AEO audit of a directory of built HTML, with the robots.txt and llms.txt in it. */
+export const auditAeoDirectory = async (dir, options = {}) => buildAudit(await collectDirectory(dir, options), options);
+/** AEO audit of a URL (a crawl with `crawl`), a directory, an .html file, or "-" for stdin. */
+export const auditAeoTarget = async (target, options = {}) => {
+    if (target === "-" || options.stdinHtml !== undefined) {
+        const html = options.stdinHtml ?? (await readStdin());
+        return auditAeoHtml(html, { url: options.baseUrl, target: "stdin" }, options);
+    }
+    const resolved = await resolveTarget(target);
+    switch (resolved.kind) {
+        case "url": {
+            return options.crawl
+                ? auditAeoSite(resolved.url, options)
+                : auditAeoUrl(resolved.url, options);
+        }
+        case "directory": {
+            return auditAeoDirectory(resolved.path, options);
+        }
+        default: {
+            const base = options.baseUrl ?? "http://localhost/";
+            const url = new URL(`/${path.basename(target)}`, base.endsWith("/") ? base : `${base}/`).href;
+            return buildAudit(collectHtml(await fs.readFile(target, "utf-8"), url, target, "file", options), options);
+        }
+    }
+};
+/** Drops bulky detail to save tokens when an LLM reads the result. */
+export const compactAeoAudit = (audit) => ({
+    ...audit,
+    findings: audit.findings.map(trimFinding),
+    crawlers: audit.crawlers
+        ? {
+            ...audit.crawlers,
+            contentSignals: audit.crawlers.contentSignals.slice(0, 3),
+        }
+        : null,
+    llmsTxt: audit.llmsTxt
+        ? { ...audit.llmsTxt, sections: audit.llmsTxt.sections.slice(0, 5) }
+        : null,
+});
+//# sourceMappingURL=audit.js.map

package/dist/core/aeo/audit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"audit.js","sourceRoot":"","sources":["../../../src/core/aeo/audit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EACL,aAAa,EACb,SAAS,EACT,aAAa,EACb,YAAY,EACZ,WAAW,GACZ,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE5E,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAYhE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAgDlD,MAAM,WAAW,GAAG,CAAC,OAAmB,EAAU,EAAE,CAClD,OAAO,CAAC,KAAK,EAAE,SAAS,IAAI,kBAAkB,CAAC;AAEjD,mEAAmE;AACnE,MAAM,UAAU,GAAG,KAAK,EAAE,OAAe,EAAE,OAAmB,EAAE,EAAE;IAChE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAEpC,OAAO,CAAC,UAAU,EAAE,CAAC;QACnB,KAAK,EAAE,QAAQ;QACf,OAAO,EAAE,YAAY,MAAM,mBAAmB,MAAM,WAAW;KAChE,CAAC,CAAC;IAEH,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC1C,aAAa,CAAC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC;QAC1D,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC;KACnC,CAAC,CAAC;IAEH,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;AACnE,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,KAAK,EACtB,GAAW,EACX,OAAmB,EACC,EAAE;IACtB,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAEtD,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE;QACvC,GAAG,EAAE,GAAG,CAAC,QAAQ;QACjB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;KAC7C,CAAC,CAAC;IAEH,kGAAkG;IAClG,MAAM,IAAI,GACR,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAEvE,OAAO;QACL,MAAM,EAAE,GAAG;QACX,GAAG,EAAE,GAAG,CAAC,QAAQ;QACjB,MAAM,EAAE,KAAK;QACb,UAAU,EAAE,KAAK;QACjB,OAAO,EAAE;YACP;gBACE,GAAG,EAAE,GAAG,CAAC,QAAQ;gBACjB,YAAY,EAAE,GAAG;gBACjB,IAAI;gBACJ,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC;gBAC7B,KAAK,EAAE,CAAC;aACT;SACF;QACD,MAAM,EAAE,IAAI,EAAE,MAAM,IAAI,IAAI;QAC5B,UAAU,EAAE,IAAI,EAAE,UAAU,IAAI,IAAI;QACpC,OAAO,EAAE,IAAI,EAAE,OAAO,IAAI,IAAI;QAC9B,IAAI,EAAE,IAAI;KACX,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,KAAK,EACvB,GAAW,EACX,OAAmB,EACC,EAAE;IACtB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE;QAClC,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,aAAa,CAAC,QAAQ;QACpD,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,aAAa,CAAC,QAAQ;QACpD,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,aAAa,CAAC,WAAW;QAC7D,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,aAAa,CAAC,OAAO;QACjD,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,IAAI;QAC5C,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,KAAK;QACrD,kBAAkB,EAAE,KAAK;QACzB,iGAAiG;QACjG,aAAa,EAAE,CAAC;QAChB,UAAU,EAAE,KAAK;QACjB,0JAA0J;QAC1J,eAAe,EAAE,CAAC,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1E,0JAA0J;QAC1J,eAAe,EAAE,CAAC,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1E,KAAK,EAAE,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,SAAS,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE;QAC5D,UAAU,EAAE,OAAO,CAAC,UAAU;KAC/B,CAAC,CAAC;IAEH,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC,CAAC;IAE1E,OAAO;QACL,MAAM,EAAE,GAAG;QACX,GAAG,EAAE,MAAM,CAAC,QAAQ;QACpB,MAAM,EAAE,OAAO;QACf,UAAU,EAAE,KAAK;QACjB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,IAAI;QACrC,OAAO,EAAE,MAAM,WAAW,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC;QAC1E,IAAI,EAAE,IAAI;KACX,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,KAAK,EAC5B,GAAW,EACX,OAAmB,EACC,EAAE;IACtB,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,mBAAmB,CAAC;IAEvD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE;QACtC,OAAO;QACP,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,KAAK;QACrD,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,GAAG;QACjC,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,SAAS;KACpC,CAAC,CAAC;IAEH,OAAO;QACL,MAAM,EAAE,GAAG;QACX,GAAG,EAAE,GAAG;QACR,MAAM,EAAE,WAAW;QACnB,UAAU,EAAE,MAAM;QAClB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,IAAI;QACrC,OAAO,EAAE,MAAM,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC;QACxC,IAAI,EAAE,KAAK;KACZ,CAAC;AACJ,CAAC,CAAC;AAEF,mFAAmF;AACnF,MAAM,WAAW,GAAG,CAClB,IAAY,EACZ,GAAW,EACX,MAAc,EACd,MAAuB,EACvB,OAAmB,EACR,EAAE,CAAC,CAAC;IACf,MAAM;IACN,GAAG;IACH,MAAM;IACN,UAAU,EAAE,MAAM;IAClB,OAAO,EAAE;QACP;YACE,GAAG;YACH,YAAY,EAAE,GAAG;YACjB,IAAI,EAAE,WAAW,CAAC,IAAI,EAAE;gBACtB,GAAG;gBACH,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;aAC7C,CAAC;YACF,KAAK,EAAE,CAAC;SACT;KACF;IACD,MAAM,EAAE,IAAI;IACZ,UAAU,EAAE,IAAI;IAChB,OAAO,EAAE,IAAI;IACb,IAAI,EAAE,KAAK;CACZ,CAAC,CAAC;AAEH,MAAM,YAAY,GAAG,CACnB,MAAkB,EAClB,MAAkB,EAClB,OAAmB,EACJ,EAAE;IACjB,IAAI,QAAQ,GAAG,YAAY,CACzB;QACE,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,MAAM;QACN,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,UAAU,EAAE,kBAAkB;QAC9B,KAAK,EAAE,MAAM,CAAC,KAAK;KACpB,EACD,EAAE,OAAO,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAClE,CAAC;IAEF,IAAI,OAAO,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;QAClC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,MAAM,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;IAEvC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;AAChE,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,QAAQ,GAAG,KAAK,EACpB,CAAY,EACZ,KAAgC,EAChC,QAAgC,EAChC,OAAmB,EACU,EAAE;IAC/B,MAAM,QAAQ,GAAG,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC;IAExC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACzB,OAAO,qEAAqE,CAAC;IAC/E,CAAC;IAED,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,KAAK,IAAI,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACzE,OAAO,+BAA+B,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,QAAQ,CAAC,MAAM,EAAE,4EAA4E,CAAC;IACvL,CAAC;IAED,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,8BAA8B,CAAC;IACxC,CAAC;IAED,OAAO,CAAC,UAAU,EAAE,CAAC;QACnB,KAAK,EAAE,SAAS;QAChB,OAAO,EAAE,cAAc,QAAQ,CAAC,QAAQ,0BAA0B;KACnE,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1E,QAAQ,CAAC,aAAa,GAAG;QACvB,SAAS,EAAE,WAAW,CAAC,OAAO,CAAC;QAC/B,MAAM,EAAE,QAAQ,CAAC,MAAM;KACxB,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEnC,IAAI,KAAK,EAAE,CAAC;YACV,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAClB,QAAgC,EAChC,OAAqB,EACX,EAAE,CACZ,QAAQ;IACN,CAAC,CAAC,QAAQ,CAAC,IAAI;SACV,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,CACnE;SACA,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACvB,CAAC,CAAC,EAAE,CAAC;AAET,MAAM,SAAS,GAAG,CAAC,CAAgB,EAAW,EAAE;IAC9C,wEAAwE;IACxE,MAAM,IAAI,GAAmC;QAC3C,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,GAAG;KACnC,CAAC;IAEF,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC;QACxB,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;IAC9B,CAAC;IAED,OAAO;QACL,GAAG,IAAI;QACP,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;KAC1D,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,KAAK,EACtB,CAAY,EACZ,OAAmB,EACA,EAAE;IACrB,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,KAAK,OAAO,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC;IAChE,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;IAC3E,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;IAElE,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM;QACvB,CAAC,CAAC,aAAa,CACX,CAAC,CAAC,MAAM,EACR,CAAC,CAAC,UAAU,EACZ,KAAK,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,EAC1B,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAC5B;QACH,CAAC,CAAC,IAAI,CAAC;IAET,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM;QACjC,CAAC,CAAC,MAAM,QAAQ,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC;QAC7C,CAAC,CAAC,SAAS,CAAC;IAEd,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAE/C,IAAI,YAAY,GAAG,eAAe,CAAC;QACjC,QAAQ;QACR,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,SAAS,EAAE,KAAK,CAAC,MAAM;QACvB,YAAY;KACb,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAExC,IAAI,OAAO,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;QAClC,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC;IACnE,CAAC;IAED,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAEtD,MAAM,KAAK,GAAa;QACtB,aAAa,EAAE,CAAC;QAChB,IAAI,EAAE,KAAK;QACX,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,YAAY,CAAC;QACrC,MAAM,EAAE,aAAa,CAAC,CAAC,GAAG,YAAY,EAAE,GAAG,YAAY,CAAC,CAAC;QACzD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,OAAO,EAAE;YACP,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,SAAS,EAAE,QAAQ,EAAE,MAAM,IAAI,IAAI;YACnC,qBAAqB,EAAE,WAAW,CAAC,QAAQ,EAAE,QAAQ,CAAC;YACtD,iBAAiB,EAAE,WAAW,CAAC,QAAQ,EAAE,QAAQ,CAAC;YAClD,mBAAmB,EAAE,WAAW,CAAC,QAAQ,EAAE,UAAU,CAAC;YACtD,kBAAkB,EAAE,QAAQ,EAAE,kBAAkB,IAAI,IAAI;YACxD,eAAe,EAAE,QAAQ,EAAE,aAAa;gBACtC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;gBAClE,CAAC,CAAC,IAAI;YACR,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,IAAI,IAAI;YAClC,IAAI,EAAE,KAAK;gBACT,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAC1D,CAAC,CAAC,IAAI;SACT;QACD,QAAQ,EAAE,YAAY,CACpB,MAAM,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS;YACxC,CAAC,CAAC,YAAY;YACd,CAAC,CAAC,CAAC,GAAG,YAAY,EAAE,GAAG,YAAY,CAAC,CACvC;QACD,QAAQ;QACR,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC;IAEF,IAAI,MAAM,EAAE,CAAC;QACX,KAAK,CAAC,KAAK,GAAG,KAAK;aAChB,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;aACxE,GAAG,CAAC,SAAS,CAAC,CAAC;QAClB,KAAK,CAAC,MAAM,GAAG,YAAY,CACzB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAChB,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,GAAG;YAClC,QAAQ,EAAE,CAAC,CAAC,QAAQ;SACrB,CAAC,CAAC,CACJ,CAAC;IACJ,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,mGAAmG;AACnG,8JAA8J;AAC9J,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,EAC/B,IAAY,EACZ,KAAK,GAAsC,EAAE,EAC7C,OAAO,GAAe,EAAE,EACL,EAAE;IACrB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,IAAI,OAAO,CAAC,OAAO,IAAI,mBAAmB,CAAC;IAEhE,OAAO,UAAU,CACf,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,MAAM,IAAI,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAC5D,OAAO,CACR,CAAC;AACJ,CAAC,CAAC;AAEF,2EAA2E;AAC3E,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,GAAW,EACX,OAAO,GAAe,EAAE,EACL,EAAE,CAAC,UAAU,CAAC,MAAM,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;AAE5E,oDAAoD;AACpD,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,EAC/B,QAAgB,EAChB,OAAO,GAAe,EAAE,EACL,EAAE,CACrB,UAAU,CAAC,MAAM,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;AAE5D,sFAAsF;AACtF,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,EACpC,GAAW,EACX,OAAO,GAAe,EAAE,EACL,EAAE,CACrB,UAAU,CAAC,MAAM,gBAAgB,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;AAE5D,+FAA+F;AAC/F,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,EACjC,MAAc,EACd,OAAO,GAAwC,EAAE,EAC9B,EAAE;IACrB,IAAI,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACtD,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,IAAI,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC;QAEtD,OAAO,YAAY,CACjB,IAAI,EACJ,EAAE,GAAG,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,EACzC,OAAO,CACR,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;IAE7C,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,KAAK,KAAK,EAAE,CAAC;YACX,OAAO,OAAO,CAAC,KAAK;gBAClB,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC;gBACrC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACzC,CAAC;QAED,KAAK,WAAW,EAAE,CAAC;YACjB,OAAO,iBAAiB,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACnD,CAAC;QAED,SAAS,CAAC;YACR,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,mBAAmB,CAAC;YAEpD,MAAM,GAAG,GAAG,IAAI,GAAG,CACjB,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAC3B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CACvC,CAAC,IAAI,CAAC;YAEP,OAAO,UAAU,CACf,WAAW,CACT,MAAM,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,EAClC,GAAG,EACH,MAAM,EACN,MAAM,EACN,OAAO,CACR,EACD,OAAO,CACR,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC,CAAC;AAEF,sEAAsE;AACtE,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,KAAe,EAAY,EAAE,CAAC,CAAC;IAC7D,GAAG,KAAK;IACR,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC;IACzC,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACtB,CAAC,CAAC;YACE,GAAG,KAAK,CAAC,QAAQ;YACjB,cAAc,EAAE,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SAC1D;QACH,CAAC,CAAC,IAAI;IACR,OAAO,EAAE,KAAK,CAAC,OAAO;QACpB,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QACpE,CAAC,CAAC,IAAI;CACT,CAAC,CAAC"}

package/dist/core/aeo/bots.d.ts

@@ -0,0 +1,30 @@
+import type { AiBotPurpose, AiCrawlerReport, ContentSignal, RobotsTxtInfo } from "../types.js";
+export interface AiBot {
+    /** The robots.txt product token. */
+    name: string;
+    operator: string;
+    purpose: AiBotPurpose;
+    usedFor: string;
+    docs: string;
+    /**
+     * The User-Agent the probe sends. Absent for robots.txt-only tokens (Google-Extended), and for
+     * search crawlers, which CDNs often block when the request does not come from the crawler's IPs.
+     */
+    userAgent?: string;
+}
+/**
+ * The crawlers that decide whether a site can show up in AI answers, from the operators' own
+ * documentation. Order: search crawlers, answer bots, training bots.
+ */
+export declare const AI_BOTS: AiBot[];
+/** The policy token of the shared AI training preference (draft-kelley-aipref-training-preference-signal). */
+export declare const TRAINING_PREFERENCE_TOKEN = "__usage_ai-training";
+/**
+ * Content-Signal lines: `Content-Signal: search=yes, ai-input=no` or, for one path,
+ * `Content-Signal: /blog/ ai-train=no`.
+ */
+export declare const contentSignals: (text: string) => ContentSignal[];
+/** Signals in a Content-Signal line that the policy does not define, or values other than yes and no. */
+export declare const invalidSignals: (signal: ContentSignal) => string[];
+/** What robots.txt lets each AI bot do on the audited pages; `startUrl` is the page the summary describes. */
+export declare const crawlerAccess: (robots: RobotsTxtInfo, text: string | null, startUrl: string, pageUrls: string[]) => AiCrawlerReport;

package/dist/core/aeo/bots.js

@@ -0,0 +1,299 @@
+import { agentMatcher } from "../robots.js";
+const OPENAI_DOCS = "https://developers.openai.com/api/docs/bots";
+const ANTHROPIC_DOCS = "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler";
+const PERPLEXITY_DOCS = "https://docs.perplexity.ai/guides/bots";
+const GOOGLE_CRAWLERS_DOCS = "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers";
+/**
+ * The crawlers that decide whether a site can show up in AI answers, from the operators' own
+ * documentation. Order: search crawlers, answer bots, training bots.
+ */
+export const AI_BOTS = [
+    {
+        name: "Googlebot",
+        operator: "Google",
+        purpose: "search",
+        usedFor: "Google Search, AI Overviews and AI Mode",
+        docs: "https://developers.google.com/search/docs/appearance/ai-features",
+    },
+    {
+        name: "Bingbot",
+        operator: "Microsoft",
+        purpose: "search",
+        usedFor: "Bing search and Microsoft Copilot",
+        docs: "https://www.bing.com/webmasters/help/webmaster-guidelines-30fba23a",
+    },
+    {
+        name: "Applebot",
+        operator: "Apple",
+        purpose: "search",
+        usedFor: "Siri, Spotlight and Safari search",
+        docs: "https://support.apple.com/en-us/119829",
+    },
+    {
+        name: "OAI-SearchBot",
+        operator: "OpenAI",
+        purpose: "answer",
+        usedFor: "ChatGPT search results",
+        docs: OPENAI_DOCS,
+        userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot",
+    },
+    {
+        name: "ChatGPT-User",
+        operator: "OpenAI",
+        purpose: "answer",
+        usedFor: "pages ChatGPT fetches for a user; robots.txt may not apply",
+        docs: OPENAI_DOCS,
+        userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",
+    },
+    {
+        name: "Claude-SearchBot",
+        operator: "Anthropic",
+        purpose: "answer",
+        usedFor: "Claude's search results",
+        docs: ANTHROPIC_DOCS,
+        userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-SearchBot/1.0; +https://www.anthropic.com)",
+    },
+    {
+        name: "Claude-User",
+        operator: "Anthropic",
+        purpose: "answer",
+        usedFor: "pages Claude fetches for a user's question",
+        docs: ANTHROPIC_DOCS,
+        userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +https://www.anthropic.com)",
+    },
+    {
+        name: "PerplexityBot",
+        operator: "Perplexity",
+        purpose: "answer",
+        usedFor: "Perplexity search results",
+        docs: PERPLEXITY_DOCS,
+        userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
+    },
+    {
+        name: "Perplexity-User",
+        operator: "Perplexity",
+        purpose: "answer",
+        usedFor: "pages Perplexity fetches for a user",
+        docs: PERPLEXITY_DOCS,
+        userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)",
+    },
+    {
+        name: "Google-Extended",
+        operator: "Google",
+        purpose: "answer",
+        usedFor: "Gemini Apps grounding and Gemini training, as a robots.txt token; not AI Overviews",
+        docs: GOOGLE_CRAWLERS_DOCS,
+    },
+    {
+        name: "DuckAssistBot",
+        operator: "DuckDuckGo",
+        purpose: "answer",
+        usedFor: "DuckDuckGo's AI-assisted answers",
+        docs: "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
+        userAgent: "DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)",
+    },
+    {
+        name: "MistralAI-User",
+        operator: "Mistral AI",
+        purpose: "answer",
+        usedFor: "pages Le Chat fetches for a user",
+        docs: "https://docs.mistral.ai/robots",
+        userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)",
+    },
+    {
+        name: "GPTBot",
+        operator: "OpenAI",
+        purpose: "training",
+        usedFor: "training OpenAI's models",
+        docs: OPENAI_DOCS,
+        userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot",
+    },
+    {
+        name: "ClaudeBot",
+        operator: "Anthropic",
+        purpose: "training",
+        usedFor: "training Anthropic's models",
+        docs: ANTHROPIC_DOCS,
+        userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
+    },
+    {
+        name: "Applebot-Extended",
+        operator: "Apple",
+        purpose: "training",
+        usedFor: "training Apple's models, as a robots.txt token",
+        docs: "https://support.apple.com/en-us/119829",
+    },
+    {
+        name: "CCBot",
+        operator: "Common Crawl",
+        purpose: "training",
+        usedFor: "Common Crawl's archive, which many models train on",
+        docs: "https://commoncrawl.org/ccbot",
+        userAgent: "CCBot/2.0 (https://commoncrawl.org/faq/)",
+    },
+    {
+        name: "meta-externalagent",
+        operator: "Meta",
+        purpose: "training",
+        usedFor: "training Meta's AI models and indexing for Meta products",
+        docs: "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
+        userAgent: "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
+    },
+];
+/** The policy token of the shared AI training preference (draft-kelley-aipref-training-preference-signal). */
+export const TRAINING_PREFERENCE_TOKEN = "__usage_ai-training";
+/** Signals the Content Signals Policy defines, plus Cloudflare's experimental `use`. */
+const KNOWN_SIGNALS = new Set(["search", "ai-input", "ai-train", "use"]);
+/** How robots.txt groups match product tokens: lower-cased, without a version. */
+const productToken = (agent) => agent.split("/")[0]?.trim().toLowerCase() ?? "";
+/** The groups of a robots.txt file: every user agent it names, and each rule line with its group. */
+const readGroups = (text) => {
+    const named = new Set();
+    const lines = [];
+    let agents = [];
+    let inAgentRun = false;
+    for (const raw of text.split(/\r\n|\r|\n/u)) {
+        const hash = raw.indexOf("#");
+        const line = (hash === -1 ? raw : raw.slice(0, hash)).trim();
+        const colon = line.indexOf(":");
+        if (colon === -1) {
+            continue;
+        }
+        const key = line.slice(0, colon).trim().toLowerCase();
+        const value = line.slice(colon + 1).trim();
+        if (key === "user-agent") {
+            // Consecutive user-agent lines share one group; a rule line in between starts a new one.
+            if (!inAgentRun) {
+                agents = [];
+            }
+            if (value) {
+                agents.push(value);
+                named.add(productToken(value));
+            }
+            inAgentRun = true;
+            continue;
+        }
+        inAgentRun = false;
+        lines.push({ agents, key, value });
+    }
+    return { named, lines };
+};
+/**
+ * Content-Signal lines: `Content-Signal: search=yes, ai-input=no` or, for one path,
+ * `Content-Signal: /blog/ ai-train=no`.
+ */
+export const contentSignals = (text) => readGroups(text)
+    .lines.filter((l) => l.key === "content-signal" && l.value)
+    .slice(0, 50)
+    .map((l) => {
+    let rest = l.value;
+    let path = null;
+    if (rest.startsWith("/")) {
+        const space = rest.search(/\s/u);
+        path = space === -1 ? rest : rest.slice(0, space);
+        rest = space === -1 ? "" : rest.slice(space + 1);
+    }
+    const signals = {};
+    for (const pair of rest.split(",")) {
+        const eq = pair.indexOf("=");
+        const name = (eq === -1 ? pair : pair.slice(0, eq))
+            .trim()
+            .toLowerCase();
+        if (name) {
+            signals[name] =
+                eq === -1
+                    ? ""
+                    : pair
+                        .slice(eq + 1)
+                        .trim()
+                        .toLowerCase();
+        }
+    }
+    return { userAgents: l.agents, path, signals };
+});
+/** Signals in a Content-Signal line that the policy does not define, or values other than yes and no. */
+export const invalidSignals = (signal) => Object.entries(signal.signals).flatMap(([name, value]) => {
+    if (!KNOWN_SIGNALS.has(name)) {
+        return [`${name}=${value}`];
+    }
+    return name !== "use" && value !== "yes" && value !== "no"
+        ? [`${name}=${value}`]
+        : [];
+});
+/**
+ * robots.txt as RFC 9309 has crawlers read it: a file is "found" with status 200, "missing" with a
+ * 4xx (crawl everything), "unreachable" with a 5xx or no answer (crawl nothing).
+ */
+const robotsStatus = (robots) => {
+    if (robots.found) {
+        return "found";
+    }
+    if (robots.status === null || robots.status >= 500) {
+        return "unreachable";
+    }
+    return "missing";
+};
+/** What robots.txt lets each AI bot do on the audited pages; `startUrl` is the page the summary describes. */
+export const crawlerAccess = (robots, text, startUrl, pageUrls) => {
+    const status = robotsStatus(robots);
+    const found = status === "found" && text !== null;
+    const groups = found ? readGroups(text) : null;
+    const allows = found ? agentMatcher(robots.url, text) : null;
+    const urls = pageUrls.length ? pageUrls : [startUrl];
+    const bots = AI_BOTS.map((bot) => {
+        const token = productToken(bot.name);
+        let robotsGroup = "none";
+        let agent = bot.name;
+        if (groups?.named.has(token)) {
+            robotsGroup = "named";
+        }
+        else if (token === "applebot" && groups?.named.has("googlebot")) {
+            // Apple: without an Applebot group, Applebot follows the Googlebot group.
+            robotsGroup = "googlebot";
+            agent = "Googlebot";
+        }
+        else if (groups?.named.has("*")) {
+            robotsGroup = "wildcard";
+        }
+        if (status === "unreachable") {
+            return {
+                name: bot.name,
+                operator: bot.operator,
+                purpose: bot.purpose,
+                usedFor: bot.usedFor,
+                robotsGroup,
+                allowed: false,
+                blockedPages: urls.length,
+            };
+        }
+        return {
+            name: bot.name,
+            operator: bot.operator,
+            purpose: bot.purpose,
+            usedFor: bot.usedFor,
+            robotsGroup,
+            allowed: allows ? allows(startUrl, agent) : true,
+            // A path `User-agent: *` keeps every crawler from is a site-wide choice, which the SEO
+            // audit reports; only a page this bot may not fetch while other crawlers may counts here.
+            blockedPages: allows
+                ? urls.filter((u) => !allows(u, agent) && allows(u, "*")).length
+                : 0,
+        };
+    });
+    let trainingPreference = "none";
+    // `User-agent: *` never speaks for the policy token, so only a group that names it counts.
+    if (allows && groups?.named.has(TRAINING_PREFERENCE_TOKEN)) {
+        trainingPreference = allows(startUrl, TRAINING_PREFERENCE_TOKEN)
+            ? "allowed"
+            : "disallowed";
+    }
+    return {
+        robotsTxt: robots.url,
+        status,
+        httpStatus: robots.status,
+        trainingPreference,
+        contentSignals: found ? contentSignals(text) : [],
+        bots,
+    };
+};
+//# sourceMappingURL=bots.js.map