@rivi-software/agentic-seo-check 0.1.0 → 0.2.0

package/dist/core/aeo/bots.js

@@ -0,0 +1,299 @@
+import { agentMatcher } from "../robots.js";
+const OPENAI_DOCS = "https://developers.openai.com/api/docs/bots";
+const ANTHROPIC_DOCS = "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler";
+const PERPLEXITY_DOCS = "https://docs.perplexity.ai/guides/bots";
+const GOOGLE_CRAWLERS_DOCS = "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers";
+/**
+ * The crawlers that decide whether a site can show up in AI answers, from the operators' own
+ * documentation. Order: search crawlers, answer bots, training bots.
+ */
+export const AI_BOTS = [
+    {
+        name: "Googlebot",
+        operator: "Google",
+        purpose: "search",
+        usedFor: "Google Search, AI Overviews and AI Mode",
+        docs: "https://developers.google.com/search/docs/appearance/ai-features",
+    },
+    {
+        name: "Bingbot",
+        operator: "Microsoft",
+        purpose: "search",
+        usedFor: "Bing search and Microsoft Copilot",
+        docs: "https://www.bing.com/webmasters/help/webmaster-guidelines-30fba23a",
+    },
+    {
+        name: "Applebot",
+        operator: "Apple",
+        purpose: "search",
+        usedFor: "Siri, Spotlight and Safari search",
+        docs: "https://support.apple.com/en-us/119829",
+    },
+    {
+        name: "OAI-SearchBot",
+        operator: "OpenAI",
+        purpose: "answer",
+        usedFor: "ChatGPT search results",
+        docs: OPENAI_DOCS,
+        userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot",
+    },
+    {
+        name: "ChatGPT-User",
+        operator: "OpenAI",
+        purpose: "answer",
+        usedFor: "pages ChatGPT fetches for a user; robots.txt may not apply",
+        docs: OPENAI_DOCS,
+        userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",
+    },
+    {
+        name: "Claude-SearchBot",
+        operator: "Anthropic",
+        purpose: "answer",
+        usedFor: "Claude's search results",
+        docs: ANTHROPIC_DOCS,
+        userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-SearchBot/1.0; +https://www.anthropic.com)",
+    },
+    {
+        name: "Claude-User",
+        operator: "Anthropic",
+        purpose: "answer",
+        usedFor: "pages Claude fetches for a user's question",
+        docs: ANTHROPIC_DOCS,
+        userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +https://www.anthropic.com)",
+    },
+    {
+        name: "PerplexityBot",
+        operator: "Perplexity",
+        purpose: "answer",
+        usedFor: "Perplexity search results",
+        docs: PERPLEXITY_DOCS,
+        userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
+    },
+    {
+        name: "Perplexity-User",
+        operator: "Perplexity",
+        purpose: "answer",
+        usedFor: "pages Perplexity fetches for a user",
+        docs: PERPLEXITY_DOCS,
+        userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)",
+    },
+    {
+        name: "Google-Extended",
+        operator: "Google",
+        purpose: "answer",
+        usedFor: "Gemini Apps grounding and Gemini training, as a robots.txt token; not AI Overviews",
+        docs: GOOGLE_CRAWLERS_DOCS,
+    },
+    {
+        name: "DuckAssistBot",
+        operator: "DuckDuckGo",
+        purpose: "answer",
+        usedFor: "DuckDuckGo's AI-assisted answers",
+        docs: "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
+        userAgent: "DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)",
+    },
+    {
+        name: "MistralAI-User",
+        operator: "Mistral AI",
+        purpose: "answer",
+        usedFor: "pages Le Chat fetches for a user",
+        docs: "https://docs.mistral.ai/robots",
+        userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)",
+    },
+    {
+        name: "GPTBot",
+        operator: "OpenAI",
+        purpose: "training",
+        usedFor: "training OpenAI's models",
+        docs: OPENAI_DOCS,
+        userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot",
+    },
+    {
+        name: "ClaudeBot",
+        operator: "Anthropic",
+        purpose: "training",
+        usedFor: "training Anthropic's models",
+        docs: ANTHROPIC_DOCS,
+        userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
+    },
+    {
+        name: "Applebot-Extended",
+        operator: "Apple",
+        purpose: "training",
+        usedFor: "training Apple's models, as a robots.txt token",
+        docs: "https://support.apple.com/en-us/119829",
+    },
+    {
+        name: "CCBot",
+        operator: "Common Crawl",
+        purpose: "training",
+        usedFor: "Common Crawl's archive, which many models train on",
+        docs: "https://commoncrawl.org/ccbot",
+        userAgent: "CCBot/2.0 (https://commoncrawl.org/faq/)",
+    },
+    {
+        name: "meta-externalagent",
+        operator: "Meta",
+        purpose: "training",
+        usedFor: "training Meta's AI models and indexing for Meta products",
+        docs: "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
+        userAgent: "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
+    },
+];
+/** The policy token of the shared AI training preference (draft-kelley-aipref-training-preference-signal). */
+export const TRAINING_PREFERENCE_TOKEN = "__usage_ai-training";
+/** Signals the Content Signals Policy defines, plus Cloudflare's experimental `use`. */
+const KNOWN_SIGNALS = new Set(["search", "ai-input", "ai-train", "use"]);
+/** How robots.txt groups match product tokens: lower-cased, without a version. */
+const productToken = (agent) => agent.split("/")[0]?.trim().toLowerCase() ?? "";
+/** The groups of a robots.txt file: every user agent it names, and each rule line with its group. */
+const readGroups = (text) => {
+    const named = new Set();
+    const lines = [];
+    let agents = [];
+    let inAgentRun = false;
+    for (const raw of text.split(/\r\n|\r|\n/u)) {
+        const hash = raw.indexOf("#");
+        const line = (hash === -1 ? raw : raw.slice(0, hash)).trim();
+        const colon = line.indexOf(":");
+        if (colon === -1) {
+            continue;
+        }
+        const key = line.slice(0, colon).trim().toLowerCase();
+        const value = line.slice(colon + 1).trim();
+        if (key === "user-agent") {
+            // Consecutive user-agent lines share one group; a rule line in between starts a new one.
+            if (!inAgentRun) {
+                agents = [];
+            }
+            if (value) {
+                agents.push(value);
+                named.add(productToken(value));
+            }
+            inAgentRun = true;
+            continue;
+        }
+        inAgentRun = false;
+        lines.push({ agents, key, value });
+    }
+    return { named, lines };
+};
+/**
+ * Content-Signal lines: `Content-Signal: search=yes, ai-input=no` or, for one path,
+ * `Content-Signal: /blog/ ai-train=no`.
+ */
+export const contentSignals = (text) => readGroups(text)
+    .lines.filter((l) => l.key === "content-signal" && l.value)
+    .slice(0, 50)
+    .map((l) => {
+    let rest = l.value;
+    let path = null;
+    if (rest.startsWith("/")) {
+        const space = rest.search(/\s/u);
+        path = space === -1 ? rest : rest.slice(0, space);
+        rest = space === -1 ? "" : rest.slice(space + 1);
+    }
+    const signals = {};
+    for (const pair of rest.split(",")) {
+        const eq = pair.indexOf("=");
+        const name = (eq === -1 ? pair : pair.slice(0, eq))
+            .trim()
+            .toLowerCase();
+        if (name) {
+            signals[name] =
+                eq === -1
+                    ? ""
+                    : pair
+                        .slice(eq + 1)
+                        .trim()
+                        .toLowerCase();
+        }
+    }
+    return { userAgents: l.agents, path, signals };
+});
+/** Signals in a Content-Signal line that the policy does not define, or values other than yes and no. */
+export const invalidSignals = (signal) => Object.entries(signal.signals).flatMap(([name, value]) => {
+    if (!KNOWN_SIGNALS.has(name)) {
+        return [`${name}=${value}`];
+    }
+    return name !== "use" && value !== "yes" && value !== "no"
+        ? [`${name}=${value}`]
+        : [];
+});
+/**
+ * robots.txt as RFC 9309 has crawlers read it: a file is "found" with status 200, "missing" with a
+ * 4xx (crawl everything), "unreachable" with a 5xx or no answer (crawl nothing).
+ */
+const robotsStatus = (robots) => {
+    if (robots.found) {
+        return "found";
+    }
+    if (robots.status === null || robots.status >= 500) {
+        return "unreachable";
+    }
+    return "missing";
+};
+/** What robots.txt lets each AI bot do on the audited pages; `startUrl` is the page the summary describes. */
+export const crawlerAccess = (robots, text, startUrl, pageUrls) => {
+    const status = robotsStatus(robots);
+    const found = status === "found" && text !== null;
+    const groups = found ? readGroups(text) : null;
+    const allows = found ? agentMatcher(robots.url, text) : null;
+    const urls = pageUrls.length ? pageUrls : [startUrl];
+    const bots = AI_BOTS.map((bot) => {
+        const token = productToken(bot.name);
+        let robotsGroup = "none";
+        let agent = bot.name;
+        if (groups?.named.has(token)) {
+            robotsGroup = "named";
+        }
+        else if (token === "applebot" && groups?.named.has("googlebot")) {
+            // Apple: without an Applebot group, Applebot follows the Googlebot group.
+            robotsGroup = "googlebot";
+            agent = "Googlebot";
+        }
+        else if (groups?.named.has("*")) {
+            robotsGroup = "wildcard";
+        }
+        if (status === "unreachable") {
+            return {
+                name: bot.name,
+                operator: bot.operator,
+                purpose: bot.purpose,
+                usedFor: bot.usedFor,
+                robotsGroup,
+                allowed: false,
+                blockedPages: urls.length,
+            };
+        }
+        return {
+            name: bot.name,
+            operator: bot.operator,
+            purpose: bot.purpose,
+            usedFor: bot.usedFor,
+            robotsGroup,
+            allowed: allows ? allows(startUrl, agent) : true,
+            // A path `User-agent: *` keeps every crawler from is a site-wide choice, which the SEO
+            // audit reports; only a page this bot may not fetch while other crawlers may counts here.
+            blockedPages: allows
+                ? urls.filter((u) => !allows(u, agent) && allows(u, "*")).length
+                : 0,
+        };
+    });
+    let trainingPreference = "none";
+    // `User-agent: *` never speaks for the policy token, so only a group that names it counts.
+    if (allows && groups?.named.has(TRAINING_PREFERENCE_TOKEN)) {
+        trainingPreference = allows(startUrl, TRAINING_PREFERENCE_TOKEN)
+            ? "allowed"
+            : "disallowed";
+    }
+    return {
+        robotsTxt: robots.url,
+        status,
+        httpStatus: robots.status,
+        trainingPreference,
+        contentSignals: found ? contentSignals(text) : [],
+        bots,
+    };
+};
+//# sourceMappingURL=bots.js.map

package/dist/core/aeo/bots.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bots.js","sourceRoot":"","sources":["../../../src/core/aeo/bots.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAuB5C,MAAM,WAAW,GAAG,6CAA6C,CAAC;AAElE,MAAM,cAAc,GAClB,iIAAiI,CAAC;AAEpI,MAAM,eAAe,GAAG,wCAAwC,CAAC;AAEjE,MAAM,oBAAoB,GACxB,sFAAsF,CAAC;AAEzF;;;GAGG;AACH,MAAM,CAAC,MAAM,OAAO,GAAY;IAC9B;QACE,IAAI,EAAE,WAAW;QACjB,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,yCAAyC;QAClD,IAAI,EAAE,kEAAkE;KACzE;IACD;QACE,IAAI,EAAE,SAAS;QACf,QAAQ,EAAE,WAAW;QACrB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,mCAAmC;QAC5C,IAAI,EAAE,oEAAoE;KAC3E;IACD;QACE,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,mCAAmC;QAC5C,IAAI,EAAE,wCAAwC;KAC/C;IACD;QACE,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,wBAAwB;QACjC,IAAI,EAAE,WAAW;QACjB,SAAS,EACP,kHAAkH;KACrH;IACD;QACE,IAAI,EAAE,cAAc;QACpB,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,4DAA4D;QACrE,IAAI,EAAE,WAAW;QACjB,SAAS,EACP,2GAA2G;KAC9G;IACD;QACE,IAAI,EAAE,kBAAkB;QACxB,QAAQ,EAAE,WAAW;QACrB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,yBAAyB;QAClC,IAAI,EAAE,cAAc;QACpB,SAAS,EACP,kHAAkH;KACrH;IACD;QACE,IAAI,EAAE,aAAa;QACnB,QAAQ,EAAE,WAAW;QACrB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,4CAA4C;QACrD,IAAI,EAAE,cAAc;QACpB,SAAS,EACP,6GAA6G;KAChH;IACD;QACE,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,2BAA2B;QACpC,IAAI,EAAE,eAAe;QACrB,SAAS,EACP,yHAAyH;KAC5H;IACD;QACE,IAAI,EAAE,iBAAiB;QACvB,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,qCAAqC;QAC9C,IAAI,EAAE,eAAe;QACrB,SAAS,EACP,6HAA6H;KAChI;IACD;QACE,IAAI,EAAE,iBAAiB;QACvB,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,QAAQ;QACjB,OAAO,EACL,oFAAoF;QACtF,IAAI,EAAE,oBAAoB;KAC3B;IACD;QACE,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,kCAAkC;QAC3C,IAAI,EAAE,oEAAoE;QAC1E,SAAS,EAAE,gEAAgE;KAC5E;IACD;QACE,IAAI,EAAE,gBAAgB;QACtB,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,QAAQ;QACjB,OAAO,EAAE,kCAAkC;QAC3C,IAAI,EAAE,gCAAgC;QACtC,SAAS,EACP,qHAAqH;KACxH;IACD;QACE,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,UAAU;QACnB,OAAO,EAAE,0BAA0B;QACnC,IAAI,EAAE,WAAW;QACjB,SAAS,EACP,wGAAwG;KAC3G;IACD;QACE,IAAI,EAAE,WAAW;QACjB,QAAQ,EAAE,WAAW;QACrB,OAAO,EAAE,UAAU;QACnB,OAAO,EAAE,6BAA6B;QACtC,IAAI,EAAE,cAAc;QACpB,SAAS,EACP,yGAAyG;KAC5G;IACD;QACE,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,UAAU;QACnB,OAAO,EAAE,gDAAgD;QACzD,IAAI,EAAE,wCAAwC;KAC/C;IACD;QACE,IAAI,EAAE,OAAO;QACb,QAAQ,EAAE,cAAc;QACxB,OAAO,EAAE,UAAU;QACnB,OAAO,EAAE,oDAAoD;QAC7D,IAAI,EAAE,+BAA+B;QACrC,SAAS,EAAE,0CAA0C;KACtD;IACD;QACE,IAAI,EAAE,oBAAoB;QAC1B,QAAQ,EAAE,MAAM;QAChB,OAAO,EAAE,UAAU;QACnB,OAAO,EAAE,0DAA0D;QACnE,IAAI,EAAE,sEAAsE;QAC5E,SAAS,EACP,2FAA2F;KAC9F;CACF,CAAC;AAEF,8GAA8G;AAC9G,MAAM,CAAC,MAAM,yBAAyB,GAAG,qBAAqB,CAAC;AAE/D,wFAAwF;AACxF,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;AAEzE,kFAAkF;AAClF,MAAM,YAAY,GAAG,CAAC,KAAa,EAAU,EAAE,CAC7C,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;AASlD,qGAAqG;AACrG,MAAM,UAAU,GAAG,CAAC,IAAY,EAAE,EAAE;IAClC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,IAAI,MAAM,GAAa,EAAE,CAAC;IAC1B,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAEhC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;YACjB,SAAS;QACX,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAE3C,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;YACzB,yFAAyF;YACzF,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,MAAM,GAAG,EAAE,CAAC;YACd,CAAC;YAED,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;YACjC,CAAC;YAED,UAAU,GAAG,IAAI,CAAC;YAClB,SAAS;QACX,CAAC;QAED,UAAU,GAAG,KAAK,CAAC;QACnB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IACrC,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AAC1B,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,IAAY,EAAmB,EAAE,CAC9D,UAAU,CAAC,IAAI,CAAC;KACb,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,gBAAgB,IAAI,CAAC,CAAC,KAAK,CAAC;KAC1D,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;KACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;IACT,IAAI,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC;IACnB,IAAI,IAAI,GAAkB,IAAI,CAAC;IAE/B,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAClD,IAAI,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,OAAO,GAA2B,EAAE,CAAC;IAE3C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAE7B,MAAM,IAAI,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;aAChD,IAAI,EAAE;aACN,WAAW,EAAE,CAAC;QAEjB,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,CAAC,IAAI,CAAC;gBACX,EAAE,KAAK,CAAC,CAAC;oBACP,CAAC,CAAC,EAAE;oBACJ,CAAC,CAAC,IAAI;yBACD,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC;yBACb,IAAI,EAAE;yBACN,WAAW,EAAE,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AACjD,CAAC,CAAC,CAAC;AAEP,yGAAyG;AACzG,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAqB,EAAY,EAAE,CAChE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE;IACvD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC;IAC9B,CAAC;IAED,OAAO,IAAI,KAAK,KAAK,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,IAAI;QACxD,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,KAAK,EAAE,CAAC;QACtB,CAAC,CAAC,EAAE,CAAC;AACT,CAAC,CAAC,CAAC;AAEL;;;GAGG;AACH,MAAM,YAAY,GAAG,CAAC,MAAqB,EAA6B,EAAE;IACxE,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACnD,OAAO,aAAa,CAAC;IACvB,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF,8GAA8G;AAC9G,MAAM,CAAC,MAAM,aAAa,GAAG,CAC3B,MAAqB,EACrB,IAAmB,EACnB,QAAgB,EAChB,QAAkB,EACD,EAAE;IACnB,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG,MAAM,KAAK,OAAO,IAAI,IAAI,KAAK,IAAI,CAAC;IAClD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/C,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7D,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAErD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAe,EAAE;QAC5C,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,WAAW,GAA+B,MAAM,CAAC;QACrD,IAAI,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC;QAErB,IAAI,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7B,WAAW,GAAG,OAAO,CAAC;QACxB,CAAC;aAAM,IAAI,KAAK,KAAK,UAAU,IAAI,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YAClE,0EAA0E;YAC1E,WAAW,GAAG,WAAW,CAAC;YAC1B,KAAK,GAAG,WAAW,CAAC;QACtB,CAAC;aAAM,IAAI,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAClC,WAAW,GAAG,UAAU,CAAC;QAC3B,CAAC;QAED,IAAI,MAAM,KAAK,aAAa,EAAE,CAAC;YAC7B,OAAO;gBACL,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,WAAW;gBACX,OAAO,EAAE,KAAK;gBACd,YAAY,EAAE,IAAI,CAAC,MAAM;aAC1B,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,WAAW;YACX,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;YAChD,uFAAuF;YACvF,0FAA0F;YAC1F,YAAY,EAAE,MAAM;gBAClB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM;gBAChE,CAAC,CAAC,CAAC;SACN,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,IAAI,kBAAkB,GAA0C,MAAM,CAAC;IAEvE,2FAA2F;IAC3F,IAAI,MAAM,IAAI,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,yBAAyB,CAAC,EAAE,CAAC;QAC3D,kBAAkB,GAAG,MAAM,CAAC,QAAQ,EAAE,yBAAyB,CAAC;YAC9D,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,YAAY,CAAC;IACnB,CAAC;IAED,OAAO;QACL,SAAS,EAAE,MAAM,CAAC,GAAG;QACrB,MAAM;QACN,UAAU,EAAE,MAAM,CAAC,MAAM;QACzB,kBAAkB;QAClB,cAAc,EAAE,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;QACjD,IAAI;KACL,CAAC;AACJ,CAAC,CAAC"}

package/dist/core/aeo/llms-txt.d.ts

@@ -0,0 +1,8 @@
+import type { FetchOptions } from "../fetch.js";
+import type { LlmsTxtInfo } from "../types.js";
+/** Reads an llms.txt file against the format at llmstxt.org: an H1 title is the one required part. */
+export declare const parseLlmsTxt: (url: string, httpStatus: number | null, raw: string) => LlmsTxtInfo;
+/** Fetches /llms.txt from an origin. An HTML answer is a single-page app's fallback route: no file. */
+export declare const loadLlmsTxt: (origin: string, options?: FetchOptions) => Promise<LlmsTxtInfo>;
+/** Reads llms.txt from the root of a build output directory. */
+export declare const readLlmsTxt: (dir: string, baseUrl: string) => Promise<LlmsTxtInfo>;

package/dist/core/aeo/llms-txt.js

@@ -0,0 +1,88 @@
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import { fetchPage } from "../fetch.js";
+/** An llms.txt is an index of links; a file this large is not one, and is read no further. */
+const MAX_LLMS_TXT_BYTES = 1_000_000;
+/**
+ * `[name](url)`. Each negated class also stops at the bracket that would start the next attempt,
+ * so a line of "[[[[" or "](](](" costs one pass. A URL with parentheses in it is not counted.
+ */
+const MARKDOWN_LINK = /\[[^[\]\n]*\]\([^()\s]+\)/gu;
+const missing = (url, httpStatus) => ({
+    url,
+    status: "missing",
+    httpStatus,
+    title: null,
+    summary: null,
+    sections: [],
+    links: 0,
+    bytes: 0,
+    problems: [],
+});
+/** Reads an llms.txt file against the format at llmstxt.org: an H1 title is the one required part. */
+export const parseLlmsTxt = (url, httpStatus, raw) => {
+    const text = raw.replace(/^\uFEFF/u, "").slice(0, MAX_LLMS_TXT_BYTES);
+    const lines = text.split(/\r\n|\r|\n/u).map((l) => l.trim());
+    const firstIndex = lines.findIndex(Boolean);
+    const first = firstIndex === -1 ? "" : (lines[firstIndex] ?? "");
+    const title = /^#\s+\S/u.test(first) ? first.replace(/^#\s+/u, "") : null;
+    const afterTitle = lines.slice(firstIndex + 1).find(Boolean) ?? "";
+    const problems = [];
+    if (!first) {
+        problems.push("the file is empty");
+    }
+    else if (!title) {
+        problems.push(`it does not start with an H1 title ("# Site name"); the first line is "${first.slice(0, 80)}"`);
+    }
+    return {
+        url,
+        status: problems.length ? "invalid" : "found",
+        httpStatus,
+        title: title ? title.slice(0, 200) : null,
+        summary: title && afterTitle.startsWith(">")
+            ? afterTitle.replace(/^>\s*/u, "").slice(0, 300)
+            : null,
+        sections: lines
+            .filter((l) => /^##\s+\S/u.test(l))
+            .slice(0, 20)
+            .map((l) => l.replace(/^##\s+/u, "").slice(0, 120)),
+        links: [...text.matchAll(MARKDOWN_LINK)].length,
+        bytes: Buffer.byteLength(text, "utf-8"),
+        problems,
+    };
+};
+/** Fetches /llms.txt from an origin. An HTML answer is a single-page app's fallback route: no file. */
+export const loadLlmsTxt = async (origin, options = {}) => {
+    const url = new URL("/llms.txt", origin).href;
+    const res = await fetchPage(url, {
+        ...options,
+        timeoutMs: Math.min(options.timeoutMs ?? 10_000, 10_000),
+        maxBodyBytes: MAX_LLMS_TXT_BYTES,
+    });
+    const contentType = res.contentType?.toLowerCase() ?? "";
+    const body = res.body ?? "";
+    if (res.status !== 200 ||
+        res.error ||
+        contentType.includes("text/html") ||
+        /^\s*</u.test(body.slice(0, 200))) {
+        return missing(url, res.status);
+    }
+    return parseLlmsTxt(url, res.status, body);
+};
+/** Reads llms.txt from the root of a build output directory. */
+export const readLlmsTxt = async (dir, baseUrl) => {
+    const url = new URL("/llms.txt", baseUrl).href;
+    const file = await fs.open(path.join(dir, "llms.txt")).catch(() => null);
+    if (!file) {
+        return missing(url, 404);
+    }
+    try {
+        const buffer = Buffer.alloc(MAX_LLMS_TXT_BYTES);
+        const { bytesRead } = await file.read(buffer, 0, MAX_LLMS_TXT_BYTES, 0);
+        return parseLlmsTxt(url, 200, buffer.subarray(0, bytesRead).toString("utf-8"));
+    }
+    finally {
+        await file.close();
+    }
+};
+//# sourceMappingURL=llms-txt.js.map

package/dist/core/aeo/llms-txt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llms-txt.js","sourceRoot":"","sources":["../../../src/core/aeo/llms-txt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAIxC,8FAA8F;AAC9F,MAAM,kBAAkB,GAAG,SAAS,CAAC;AAErC;;;GAGG;AACH,MAAM,aAAa,GAAG,6BAA6B,CAAC;AAEpD,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,UAAyB,EAAe,EAAE,CAAC,CAAC;IACxE,GAAG;IACH,MAAM,EAAE,SAAS;IACjB,UAAU;IACV,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,IAAI;IACb,QAAQ,EAAE,EAAE;IACZ,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;IACR,QAAQ,EAAE,EAAE;CACb,CAAC,CAAC;AAEH,sGAAsG;AACtG,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,GAAW,EACX,UAAyB,EACzB,GAAW,EACE,EAAE;IACf,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;IACtE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1E,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACnE,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACrC,CAAC;SAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,QAAQ,CAAC,IAAI,CACX,0EAA0E,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAChG,CAAC;IACJ,CAAC;IAED,OAAO;QACL,GAAG;QACH,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO;QAC7C,UAAU;QACV,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;QACzC,OAAO,EACL,KAAK,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC;YACjC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YAChD,CAAC,CAAC,IAAI;QACV,QAAQ,EAAE,KAAK;aACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aAClC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;aACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACrD,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM;QAC/C,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC;QACvC,QAAQ;KACT,CAAC;AACJ,CAAC,CAAC;AAEF,uGAAuG;AACvG,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,MAAc,EACd,OAAO,GAAiB,EAAE,EACJ,EAAE;IACxB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;IAE9C,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE;QAC/B,GAAG,OAAO;QACV,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,MAAM,EAAE,MAAM,CAAC;QACxD,YAAY,EAAE,kBAAkB;KACjC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,GAAG,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IACzD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;IAE5B,IACE,GAAG,CAAC,MAAM,KAAK,GAAG;QAClB,GAAG,CAAC,KAAK;QACT,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC;QACjC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EACjC,CAAC;QACD,OAAO,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF,gEAAgE;AAChE,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,GAAW,EACX,OAAe,EACO,EAAE;IACxB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IAC/C,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAEzE,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAChD,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC;QAExE,OAAO,YAAY,CACjB,GAAG,EACH,GAAG,EACH,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAChD,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;AACH,CAAC,CAAC"}

package/dist/core/aeo/page.d.ts

@@ -0,0 +1,53 @@
+import type { JsonObject, JsonValue } from "../html/structured-data.js";
+import type { AeoPageSummary, ExtractedPage, ResponseInfo, StructuredDataItem } from "../types.js";
+export type Engine = "google" | "bing";
+export interface ScopedDirective {
+    /** The user agent the directive is for, or null when it is for every crawler. */
+    agent: string | null;
+    directive: string;
+}
+/**
+ * X-Robots-Tag values. "googlebot: nosnippet, noarchive" scopes every directive after the prefix
+ * to googlebot, up to the next prefix.
+ */
+export declare const parseRobotsHeader: (value: string) => ScopedDirective[];
+/** The robots directives one engine applies to the page, from its meta tags and X-Robots-Tag. */
+export declare const directivesFor: (engine: Engine, page: ExtractedPage, response?: ResponseInfo) => string[];
+/** The `max-snippet` limit, or null when none is set (-1 means no limit). */
+export declare const maxSnippet: (directives: string[]) => number | null;
+export declare const hasNoindex: (directives: string[]) => boolean;
+export declare const hasNosnippet: (directives: string[]) => boolean;
+export interface TypedNode {
+    types: string[];
+    node: JsonObject;
+    /** True for the item itself, false for an object nested in it (publisher, author, mainEntity). */
+    topLevel: boolean;
+}
+/** Every typed object in the page's structured data, nested ones included. */
+export declare const typedNodes: (items: StructuredDataItem[]) => TypedNode[];
+export declare const isArticleType: (type: string) => boolean;
+/**
+ * Whether a typed object describes an organization or a person. LocalBusiness alone has over a
+ * hundred subtypes (Dentist, Bakery, ...), so an object with a name and a logo, address or phone
+ * number counts too.
+ */
+export declare const isEntity: ({ types, node }: TypedNode) => boolean;
+/** The text of a name-like value: a string, or the `name` of an object, or of the first in a list. */
+export declare const nameOf: (value: JsonValue | undefined) => string | null;
+export declare const articles: (page: ExtractedPage) => TypedNode[];
+/** Lower-cased letters and digits only, so punctuation and spacing differences do not count. */
+export declare const comparable: (value: string) => string;
+/**
+ * Structured-data text may hold HTML: tags, and entities such as &amp;. `[^<>]` stops each attempt
+ * at the next bracket, so a run of "<" without ">" costs one pass, not one pass per bracket.
+ */
+export declare const stripTags: (value: string) => string;
+export interface MarkedUpText {
+    type: "Question" | "Answer" | "HowToStep";
+    text: string;
+}
+/** The questions, answers and steps FAQPage, QAPage and HowTo markup claims the page shows. */
+export declare const markedUpTexts: (page: ExtractedPage) => MarkedUpText[];
+/** Whether the URL is a site's home page (also /index.html in build output). */
+export declare const isHomePage: (url: string) => boolean;
+export declare const aeoPageSummary: (page: ExtractedPage, response?: ResponseInfo) => AeoPageSummary;