@pseolint/core 0.2.1 → 0.3.0

package/dist/auditor.js

@@ -26,7 +26,15 @@ import { hreflangConsistencyRule } from "./rules/tech/hreflang-consistency.js";
 import { ogCompletenessRule } from "./rules/tech/og-completeness.js";
 import { robotsNoindexConflictRule } from "./rules/tech/robots-noindex-conflict.js";
 import { sitemapCompletenessRule } from "./rules/tech/sitemap-completeness.js";
-import { robotsComplianceRule } from "./rules/tech/robots-sitemap-presence.js";
+import { robotsComplianceRule, parseDisallowPatterns, isBlockedByPattern, parseCrawlDelaySeconds } from "./rules/tech/robots-sitemap-presence.js";
+import { llmsTxtRule } from "./rules/aeo/llms-txt.js";
+import { crawlerAccessRule } from "./rules/aeo/crawler-access.js";
+import { freshnessSignalsRule } from "./rules/aeo/freshness-signals.js";
+import { faqCoverageRule } from "./rules/aeo/faq-coverage.js";
+import { answerFirstRule } from "./rules/aeo/answer-first.js";
+import { citableFactsRule } from "./rules/aeo/citable-facts.js";
+import { nonReplicableValueRule } from "./rules/aeo/non-replicable-value.js";
+import { contentModularityRule } from "./rules/aeo/content-modularity.js";
 import { redirectChainRule } from "./rules/tech/redirect-chain.js";
 import { soft404Rule } from "./rules/tech/soft-404.js";
 import { jsonLdValidRule } from "./rules/schema/json-ld-valid.js";
@@ -38,6 +46,7 @@ import { urlPatternRule } from "./rules/cannibal/url-pattern.js";
 import { templateCoverageRule } from "./rules/spam/template-coverage.js";
 import { dataBindingRule, dataIdenticalRule } from "./rules/data/data-binding.js";
 import { classifyPages, isRuleEnabled } from "./page-classifier.js";
+import { isRuleAllowedInDiff } from "./rules/scope.js";
 import { RULE_REFERENCES } from "./rule-references.js";
 import { enrichFindings } from "./enrich-findings.js";
 import { triageFindings } from "./ai/triage.js";
@@ -61,13 +70,21 @@ const DEFAULTS = {
     hubPagesMaxSiblings: 50,
     titleOverlapThreshold: 0.8,
     keywordCollisionMinShared: 6,
-    templateCoverageMinPages: 5
+    templateCoverageMinPages: 5,
+    answerFirstMaxWords: 100,
+    citableFactsMin: 3,
+    citableFactsTarget: 8,
+    freshnessMaxStaleDays: 180,
+    modularityMaxParagraphWords: 200,
+    modularityMinSelfContainedRatio: 0.7,
+    faqMinQuestionHeadings: 2
 };
 const CATEGORY_WEIGHTS = {
-    spam: 0.4,
-    content: 0.25,
-    links: 0.15,
-    tech: 0.1,
+    spam: 0.35,
+    content: 0.2,
+    aeo: 0.15,
+    links: 0.12,
+    tech: 0.08,
     schema: 0.05,
     cannibal: 0.05,
     /** Dedup / crawl hygiene; does not affect composite score. */
@@ -93,8 +110,9 @@ function resolveGroupRules(baseRules, overrides) {
     }
     return result;
 }
-function runRulesOnPages(pages, resolvedRules, isEnabled, groupName, knownUrls, adjacency, inbound, rootUrl, normalizeUrlOptions, source, entityPatterns, overrides) {
+function runRulesOnPages(pages, resolvedRules, isEnabled, groupName, knownUrls, adjacency, inbound, rootUrl, normalizeUrlOptions, source, entityPatterns, overrides, mode = "full") {
     const findings = [];
+    const modeOk = (ruleId) => mode !== "diff" || isRuleAllowedInDiff(ruleId);
     const tag = (results) => results.map((r) => {
         const override = overrides?.[r.ruleId];
         return {
@@ -106,106 +124,137 @@ function runRulesOnPages(pages, resolvedRules, isEnabled, groupName, knownUrls,
     });
     // Spam rules — always compute cross-page data, only push findings if enabled
     const nearDuplicate = nearDuplicateRule(pages, resolvedRules.nearDuplicateThreshold);
-    if (isEnabled("spam/near-duplicate")) {
+    if (isEnabled("spam/near-duplicate") && modeOk("spam/near-duplicate")) {
         findings.push(...tag(nearDuplicate.findings));
     }
     const entitySwap = entitySwapRule(pages, entityPatterns, resolvedRules.entitySwapThreshold);
-    if (isEnabled("spam/entity-swap")) {
+    if (isEnabled("spam/entity-swap") && modeOk("spam/entity-swap")) {
         findings.push(...tag(entitySwap.findings));
     }
     const thinContent = thinContentRule(pages, resolvedRules.thinContentMinWords);
-    if (isEnabled("spam/thin-content")) {
+    if (isEnabled("spam/thin-content") && modeOk("spam/thin-content")) {
         findings.push(...tag(thinContent.findings));
     }
-    if (isEnabled("spam/doorway-pattern")) {
+    if (isEnabled("spam/doorway-pattern") && modeOk("spam/doorway-pattern")) {
         findings.push(...tag(doorwayPatternRule(nearDuplicate.pairs, entitySwap.pairs, thinContent.thinContentUrls, pages)));
     }
-    if (isEnabled("spam/publication-velocity")) {
+    if (isEnabled("spam/publication-velocity") && modeOk("spam/publication-velocity")) {
         findings.push(...tag(publicationVelocityRule(pages, resolvedRules.publicationVelocityMaxPerDay)));
     }
-    if (isEnabled("spam/boilerplate-ratio")) {
+    if (isEnabled("spam/boilerplate-ratio") && modeOk("spam/boilerplate-ratio")) {
         findings.push(...tag(boilerplateRatioRule(pages, resolvedRules.boilerplateMaxRatio)));
     }
-    if (isEnabled("spam/template-diversity")) {
+    if (isEnabled("spam/template-diversity") && modeOk("spam/template-diversity")) {
         findings.push(...tag(templateDiversityRule(pages, resolvedRules.templateDiversityMinUniqueRatio)));
     }
-    if (isEnabled("spam/template-coverage")) {
+    if (isEnabled("spam/template-coverage") && modeOk("spam/template-coverage")) {
         findings.push(...tag(templateCoverageRule(pages, entityPatterns, resolvedRules.templateCoverageMinPages)));
     }
     // Content rules
-    if (isEnabled("content/unique-value")) {
+    if (isEnabled("content/unique-value") && modeOk("content/unique-value")) {
         findings.push(...tag(uniqueValueRule(pages, resolvedRules.uniqueValueMinWords)));
     }
-    if (isEnabled("content/heading-uniqueness")) {
+    if (isEnabled("content/heading-uniqueness") && modeOk("content/heading-uniqueness")) {
         findings.push(...tag(headingUniquenessRule(pages, entityPatterns)));
     }
-    if (isEnabled("content/meta-uniqueness")) {
+    if (isEnabled("content/meta-uniqueness") && modeOk("content/meta-uniqueness")) {
         findings.push(...tag(metaUniquenessRule(pages, entityPatterns, resolvedRules.metaUniquenessMinJaccard)));
     }
-    if (isEnabled("content/missing-author")) {
+    if (isEnabled("content/missing-author") && modeOk("content/missing-author")) {
         findings.push(...tag(missingAuthorRule(pages)));
     }
-    if (isEnabled("content/eeat-signals")) {
+    if (isEnabled("content/eeat-signals") && modeOk("content/eeat-signals")) {
         findings.push(...tag(eeatSignalsRule(pages)));
     }
     // Link rules — use the global link graph
-    if (isEnabled("links/orphan-pages")) {
+    if (isEnabled("links/orphan-pages") && modeOk("links/orphan-pages")) {
         findings.push(...tag(orphanPagesRule(pages, inbound, rootUrl)));
     }
-    if (isEnabled("links/dead-ends")) {
+    if (isEnabled("links/dead-ends") && modeOk("links/dead-ends")) {
         findings.push(...tag(deadEndsRule(pages, knownUrls, rootUrl)));
     }
-    if (isEnabled("links/link-depth")) {
+    if (isEnabled("links/link-depth") && modeOk("links/link-depth")) {
         if (rootUrl) {
             findings.push(...tag(linkDepthRule(pages, adjacency, rootUrl, resolvedRules.linkDepthMaxClicks, inbound)));
         }
     }
-    if (isEnabled("links/cluster-connectivity")) {
+    if (isEnabled("links/cluster-connectivity") && modeOk("links/cluster-connectivity")) {
         findings.push(...tag(clusterConnectivityRule(pages, knownUrls)));
     }
-    if (isEnabled("links/hub-pages")) {
+    if (isEnabled("links/hub-pages") && modeOk("links/hub-pages")) {
         findings.push(...tag(hubPagesRule(pages, knownUrls, resolvedRules.hubPagesMinSiblings, resolvedRules.hubPagesMaxSiblings)));
     }
     // Tech rules
-    if (isEnabled("tech/canonical-consistency")) {
+    if (isEnabled("tech/canonical-consistency") && modeOk("tech/canonical-consistency")) {
         findings.push(...tag(canonicalConsistencyRule(pages, knownUrls, normalizeUrlOptions)));
     }
-    if (isEnabled("tech/canonical-noindex-conflict")) {
+    if (isEnabled("tech/canonical-noindex-conflict") && modeOk("tech/canonical-noindex-conflict")) {
         findings.push(...tag(canonicalNoindexConflictRule(pages, normalizeUrlOptions)));
     }
-    if (isEnabled("tech/robots-noindex-conflict")) {
+    if (isEnabled("tech/robots-noindex-conflict") && modeOk("tech/robots-noindex-conflict")) {
         findings.push(...tag(robotsNoindexConflictRule(pages, inbound)));
     }
-    if (isEnabled("tech/redirect-chain")) {
+    if (isEnabled("tech/redirect-chain") && modeOk("tech/redirect-chain")) {
         findings.push(...tag(redirectChainRule(pages)));
     }
-    if (isEnabled("tech/soft-404")) {
+    if (isEnabled("tech/soft-404") && modeOk("tech/soft-404")) {
         findings.push(...tag(soft404Rule(pages)));
     }
-    if (isEnabled("tech/og-completeness")) {
+    if (isEnabled("tech/og-completeness") && modeOk("tech/og-completeness")) {
         findings.push(...tag(ogCompletenessRule(pages)));
     }
-    if (isEnabled("tech/hreflang-consistency")) {
+    if (isEnabled("tech/hreflang-consistency") && modeOk("tech/hreflang-consistency")) {
         findings.push(...tag(hreflangConsistencyRule(pages, normalizeUrlOptions)));
     }
     // Schema rules
-    if (isEnabled("schema/json-ld-valid")) {
+    if (isEnabled("schema/json-ld-valid") && modeOk("schema/json-ld-valid")) {
         findings.push(...tag(jsonLdValidRule(pages)));
     }
-    if (isEnabled("schema/required-fields")) {
+    if (isEnabled("schema/required-fields") && modeOk("schema/required-fields")) {
         findings.push(...tag(requiredFieldsRule(pages)));
     }
-    if (isEnabled("schema/consistency")) {
+    if (isEnabled("schema/consistency") && modeOk("schema/consistency")) {
         findings.push(...tag(schemaConsistencyRule(pages)));
     }
+    // AEO rules
+    if (isEnabled("aeo/freshness-signals")) {
+        findings.push(...tag(freshnessSignalsRule(pages, {
+            maxStaleDays: resolvedRules.freshnessMaxStaleDays,
+        })));
+    }
+    if (isEnabled("aeo/faq-coverage")) {
+        findings.push(...tag(faqCoverageRule(pages, {
+            minQuestionHeadings: resolvedRules.faqMinQuestionHeadings,
+        })));
+    }
+    if (isEnabled("aeo/answer-first")) {
+        findings.push(...tag(answerFirstRule(pages, entityPatterns, {
+            maxFirstParagraphWords: resolvedRules.answerFirstMaxWords,
+        })));
+    }
+    if (isEnabled("aeo/citable-facts")) {
+        findings.push(...tag(citableFactsRule(pages, entityPatterns, {
+            minFactsPerPage: resolvedRules.citableFactsMin,
+            targetFactsPerPage: resolvedRules.citableFactsTarget,
+        })));
+    }
+    if (isEnabled("aeo/non-replicable-value")) {
+        findings.push(...tag(nonReplicableValueRule(pages)));
+    }
+    if (isEnabled("aeo/content-modularity")) {
+        findings.push(...tag(contentModularityRule(pages, {
+            maxParagraphWords: resolvedRules.modularityMaxParagraphWords,
+            minSelfContainedRatio: resolvedRules.modularityMinSelfContainedRatio,
+        })));
+    }
     // Cannibal rules
-    if (isEnabled("cannibal/title-overlap")) {
+    if (isEnabled("cannibal/title-overlap") && modeOk("cannibal/title-overlap")) {
         findings.push(...tag(titleOverlapRule(pages, entityPatterns, resolvedRules.titleOverlapThreshold)));
     }
-    if (isEnabled("cannibal/keyword-collision")) {
+    if (isEnabled("cannibal/keyword-collision") && modeOk("cannibal/keyword-collision")) {
         findings.push(...tag(keywordCollisionRule(pages, resolvedRules.keywordCollisionMinShared)));
     }
-    if (isEnabled("cannibal/url-pattern")) {
+    if (isEnabled("cannibal/url-pattern") && modeOk("cannibal/url-pattern")) {
         findings.push(...tag(urlPatternRule(pages)));
     }
     return findings;
@@ -223,6 +272,7 @@ function scoreFromFindings(findings) {
     const raw = {
         spam: 0,
         content: 0,
+        aeo: 0,
         links: 0,
         tech: 0,
         schema: 0,
@@ -238,6 +288,7 @@ function scoreFromFindings(findings) {
     }
     const weighted = raw.spam * CATEGORY_WEIGHTS.spam +
         raw.content * CATEGORY_WEIGHTS.content +
+        raw.aeo * CATEGORY_WEIGHTS.aeo +
         raw.links * CATEGORY_WEIGHTS.links +
         raw.tech * CATEGORY_WEIGHTS.tech +
         raw.schema * CATEGORY_WEIGHTS.schema +
@@ -248,6 +299,7 @@ function scoreFromFindings(findings) {
         categoryScores: {
             spam: raw.spam,
             content: raw.content,
+            aeo: raw.aeo,
             links: raw.links,
             tech: raw.tech,
             schema: raw.schema,
@@ -430,7 +482,35 @@ async function collectUrlsFromSitemap(sitemapText, sitemapUrl, visited, timeoutM
     }
     return allUrls;
 }
-async function loadPagesFromSource(source, concurrency, timeoutMs, crawlDiscovery, discoveryBudget, cache, stats) {
+async function fetchRobotsMeta(origin, timeoutMs, cache, stats) {
+    if (!origin)
+        return { disallow: [], crawlDelaySec: 0 };
+    try {
+        const robotsUrl = `${origin}/robots.txt`;
+        const fetched = await fetchTextStrict(robotsUrl, timeoutMs, cache, stats);
+        return {
+            disallow: parseDisallowPatterns(fetched.text),
+            crawlDelaySec: parseCrawlDelaySeconds(fetched.text),
+        };
+    }
+    catch {
+        return { disallow: [], crawlDelaySec: 0 };
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function isDisallowedByRobots(urlPath, patterns) {
+    for (const pat of patterns) {
+        if (isBlockedByPattern(urlPath, pat))
+            return true;
+    }
+    return false;
+}
+function budgetExceeded(b) {
+    return b.cap > 0 && b.used >= b.cap;
+}
+async function loadPagesFromSource(source, concurrency, timeoutMs, crawlDiscovery, discoveryBudget, cache, stats, fillBudgetViaLinkDiscovery = false, byteBudget = { used: 0, cap: 0 }) {
     if (/^https?:\/\//i.test(source)) {
         let text;
         let contentType;
@@ -467,23 +547,38 @@ async function loadPagesFromSource(source, concurrency, timeoutMs, crawlDiscover
                 ? fisherYatesSample(allSitemapUrls, discoveryBudget)
                 : allSitemapUrls;
             const pages = [];
-            await runWithConcurrency(urlsToFetch, concurrency, async (url) => {
+            // Fetch robots.txt once for the origin — reused for Crawl-Delay pacing and Disallow checks.
+            const sourceOrigin = (() => { try {
+                return new URL(source).origin;
+            }
+            catch {
+                return "";
+            } })();
+            const robots = await fetchRobotsMeta(sourceOrigin, timeoutMs, cache, stats);
+            const effectiveConcurrency = robots.crawlDelaySec > 0 ? 1 : concurrency;
+            const delayMs = robots.crawlDelaySec * 1000;
+            await runWithConcurrency(urlsToFetch, effectiveConcurrency, async (url) => {
+                if (budgetExceeded(byteBudget))
+                    return;
                 const result = await fetchPageWithMeta(url, timeoutMs, cache, stats);
                 if (result) {
+                    byteBudget.used += result.html.length;
                     pages.push(result);
                 }
+                if (delayMs > 0)
+                    await sleep(delayMs);
             });
-            // Skip additional crawl discovery when budget is active — sitemap is authoritative
-            if (crawlDiscovery && discoveryBudget === 0) {
+            // Link discovery fills the sample.
+            // Legacy behavior: no budget set + crawlDiscovery true → fill from links (unchanged).
+            // New behavior: budget set + crawlDiscovery true + opt-in flag → top up to budget.
+            const budgetUnderfilled = discoveryBudget > 0 && pages.length < discoveryBudget;
+            const legacyBudgetless = discoveryBudget === 0;
+            const shouldFill = crawlDiscovery && (legacyBudgetless || (budgetUnderfilled && fillBudgetViaLinkDiscovery));
+            if (shouldFill) {
                 const sitemapUrlSet = new Set(allSitemapUrls);
                 const discoveredUrls = new Set();
-                let sourceOrigin;
-                try {
-                    sourceOrigin = new URL(source).origin;
-                }
-                catch {
-                    sourceOrigin = "";
-                }
+                // robots already fetched above; reuse its Disallow patterns here.
+                const disallowPatterns = robots.disallow;
                 for (const page of pages) {
                     const linkMatches = Array.from(page.html.matchAll(/href=["']([^"']+)["']/gi));
                     for (const match of linkMatches) {
@@ -500,9 +595,11 @@ async function loadPagesFromSource(source, concurrency, timeoutMs, crawlDiscover
                             resolvedUrl.search = "";
                             resolvedUrl.hash = "";
                             const normalized = resolvedUrl.href;
-                            if (!sitemapUrlSet.has(normalized) && !discoveredUrls.has(normalized)) {
-                                discoveredUrls.add(normalized);
-                            }
+                            if (sitemapUrlSet.has(normalized) || discoveredUrls.has(normalized))
+                                continue;
+                            if (isDisallowedByRobots(resolvedUrl.pathname, disallowPatterns))
+                                continue;
+                            discoveredUrls.add(normalized);
                         }
                         catch {
                             continue;
@@ -510,11 +607,21 @@ async function loadPagesFromSource(source, concurrency, timeoutMs, crawlDiscover
                     }
                 }
                 if (discoveredUrls.size > 0) {
-                    await runWithConcurrency(Array.from(discoveredUrls), concurrency, async (url) => {
+                    const candidates = Array.from(discoveredUrls);
+                    // Fisher-Yates shuffle so we don't bias toward the first-discovered links (nav/footer).
+                    const shuffled = fisherYatesSample(candidates, candidates.length);
+                    const remaining = discoveryBudget === 0 ? Infinity : discoveryBudget - pages.length;
+                    const toFetch = remaining === Infinity ? shuffled : shuffled.slice(0, remaining);
+                    await runWithConcurrency(toFetch, effectiveConcurrency, async (url) => {
+                        if (budgetExceeded(byteBudget))
+                            return;
                         const result = await fetchPageWithMeta(url, timeoutMs, cache, stats);
                         if (result && result.httpMeta && result.httpMeta.statusCode >= 200 && result.httpMeta.statusCode < 300) {
+                            byteBudget.used += result.html.length;
                             pages.push(result);
                         }
+                        if (delayMs > 0)
+                            await sleep(delayMs);
                     });
                 }
             }
@@ -647,7 +754,14 @@ export async function auditSource(source, options) {
         hubPagesMaxSiblings: options?.rules?.hubPagesMaxSiblings ?? DEFAULTS.hubPagesMaxSiblings,
         titleOverlapThreshold: options?.rules?.titleOverlapThreshold ?? DEFAULTS.titleOverlapThreshold,
         keywordCollisionMinShared: options?.rules?.keywordCollisionMinShared ?? DEFAULTS.keywordCollisionMinShared,
-        templateCoverageMinPages: options?.rules?.templateCoverageMinPages ?? DEFAULTS.templateCoverageMinPages
+        templateCoverageMinPages: options?.rules?.templateCoverageMinPages ?? DEFAULTS.templateCoverageMinPages,
+        answerFirstMaxWords: options?.rules?.answerFirstMaxWords ?? DEFAULTS.answerFirstMaxWords,
+        citableFactsMin: options?.rules?.citableFactsMin ?? DEFAULTS.citableFactsMin,
+        citableFactsTarget: options?.rules?.citableFactsTarget ?? DEFAULTS.citableFactsTarget,
+        freshnessMaxStaleDays: options?.rules?.freshnessMaxStaleDays ?? DEFAULTS.freshnessMaxStaleDays,
+        modularityMaxParagraphWords: options?.rules?.modularityMaxParagraphWords ?? DEFAULTS.modularityMaxParagraphWords,
+        modularityMinSelfContainedRatio: options?.rules?.modularityMinSelfContainedRatio ?? DEFAULTS.modularityMinSelfContainedRatio,
+        faqMinQuestionHeadings: options?.rules?.faqMinQuestionHeadings ?? DEFAULTS.faqMinQuestionHeadings
     };
     const normalizeUrlOptions = mergeNormalizeUrlOptions({
         stripQuery: options?.rules?.stripUrlQuery ?? true,
@@ -668,7 +782,10 @@ export async function auditSource(source, options) {
             ttlMs: options.cache.ttlMs ?? 7 * 24 * 60 * 60 * 1000,
         }
         : null;
-    const { pages: loadedPagesRaw, sitemapUrls: sitemapUrlSet, discoveredUrlCount } = await loadPagesFromSource(source, concurrency, timeoutMs, crawlDiscovery, discoveryBudget, cacheConfig, cacheStats);
+    const fillBudgetViaLinkDiscovery = options?.fillBudgetViaLinkDiscovery ?? false;
+    const maxFetchBytes = options?.maxFetchBytes ?? 52_428_800;
+    const fetchByteBudget = { used: 0, cap: maxFetchBytes };
+    const { pages: loadedPagesRaw, sitemapUrls: sitemapUrlSet, discoveredUrlCount } = await loadPagesFromSource(source, concurrency, timeoutMs, crawlDiscovery, discoveryBudget, cacheConfig, cacheStats, fillBudgetViaLinkDiscovery, fetchByteBudget);
     const loadedPages = [...loadedPagesRaw];
     if (discoveredUrlCount && discoveredUrlCount > loadedPages.length) {
         console.error(`Discovered ${discoveredUrlCount} pages, fetched ${loadedPages.length} for audit. Use --sample-size 0 for full crawl.`);
@@ -778,9 +895,14 @@ export async function auditSource(source, options) {
                     throw new Error(`Invalid regex flags "${rawFlags}" in entityPatterns for placeholder "${p.placeholder}". ` +
                         `Only the flags g, i, m, s, u, y are permitted.`);
                 }
+                // Entity patterns are used with String.replace to mask every occurrence, which
+                // requires the `g` flag. Add it if the user forgot — a silently broken "only first
+                // match masked" regex would make template-detection rules (answer-first,
+                // citable-facts) miss shared openers.
+                const normalizedFlags = rawFlags.includes("g") ? rawFlags : `${rawFlags}g`;
                 try {
                     // Flags validated against SAFE_FLAGS_RE above; pattern is from trusted local config, not HTTP input.
-                    return { placeholder: p.placeholder, pattern: new RegExp(p.pattern, rawFlags) }; // nosemgrep
+                    return { placeholder: p.placeholder, pattern: new RegExp(p.pattern, normalizedFlags) }; // nosemgrep
                 }
                 catch (err) {
                     throw new Error(`Invalid regex pattern for placeholder "${p.placeholder}": ${err.message}`);
@@ -793,8 +915,9 @@ export async function auditSource(source, options) {
     const allFindings = [...duplicateUrlFindings];
     const groupScores = {};
     const groupPageCounts = {};
+    const auditMode = options?.mode ?? "full";
     // Site-wide rules (run once, outside group loop)
-    if (sitemapUrlSet && sitemapUrlSet.size > 0) {
+    if (sitemapUrlSet && sitemapUrlSet.size > 0 && auditMode !== "diff") {
         const sitemapFindings = sitemapCompletenessRule(parsedPages, sitemapUrlSet);
         allFindings.push(...sitemapFindings.map((f) => ({ ...f, ref: f.ref ?? RULE_REFERENCES[f.ruleId] })));
         if (robotsTxtContent) {
@@ -802,13 +925,24 @@ export async function auditSource(source, options) {
             allFindings.push(...robotsFindings.map((f) => ({ ...f, ref: f.ref ?? RULE_REFERENCES[f.ruleId] })));
         }
     }
+    // AEO site-wide rules. These run unconditionally (consistent with sitemap-completeness
+    // and robots-compliance); page-group rule lists govern per-page AEO rules only.
+    const llmsFindings = await llmsTxtRule(source, { timeoutMs });
+    allFindings.push(...llmsFindings.map((f) => ({ ...f, ref: f.ref ?? RULE_REFERENCES[f.ruleId] })));
+    if (robotsTxtContent) {
+        const crawlerFindings = crawlerAccessRule(robotsTxtContent);
+        allFindings.push(...crawlerFindings.map((f) => ({ ...f, ref: f.ref ?? RULE_REFERENCES[f.ruleId] })));
+    }
     // Data source comparison rules
     if (options?.dataSource?.records && options.dataSource.records.length > 0) {
-        const dataFindings = [
-            ...dataBindingRule(parsedPages, options.dataSource.records),
-            ...dataIdenticalRule(parsedPages, options.dataSource.records),
-        ];
-        allFindings.push(...dataFindings.map((f) => ({ ...f, ref: f.ref ?? RULE_REFERENCES[f.ruleId] })));
+        if (auditMode !== "diff" || isRuleAllowedInDiff("data/missing-binding")) {
+            const dataBindingFindings = dataBindingRule(parsedPages, options.dataSource.records);
+            allFindings.push(...dataBindingFindings.map((f) => ({ ...f, ref: f.ref ?? RULE_REFERENCES[f.ruleId] })));
+        }
+        if (auditMode !== "diff" || isRuleAllowedInDiff("data/identical-across-pages")) {
+            const dataIdenticalFindings = dataIdenticalRule(parsedPages, options.dataSource.records);
+            allFindings.push(...dataIdenticalFindings.map((f) => ({ ...f, ref: f.ref ?? RULE_REFERENCES[f.ruleId] })));
+        }
     }
     for (const [groupName, groupPages] of classified) {
         if (groupPages.length === 0)
@@ -818,7 +952,7 @@ export async function auditSource(source, options) {
             continue;
         const groupRules = resolveGroupRules(resolvedRules, groupConfig?.overrides);
         const enabledCheck = (ruleId) => isRuleEnabled(ruleId, groupConfig?.rules);
-        const findings = runRulesOnPages(groupPages, groupRules, enabledCheck, groupName, knownUrls, adjacency, inbound, rootUrl, normalizeUrlOptions, source, DEFAULT_ENTITY_PATTERNS, groupConfig?.overrides);
+        const findings = runRulesOnPages(groupPages, groupRules, enabledCheck, groupName, knownUrls, adjacency, inbound, rootUrl, normalizeUrlOptions, source, DEFAULT_ENTITY_PATTERNS, groupConfig?.overrides, options?.mode ?? "full");
         allFindings.push(...findings);
         groupPageCounts[groupName] = groupPages.length;
         const { score } = scoreFromFindings(findings);