@nebulacomponents/citable 1.15.1 → 1.17.0

package/src/crawler/ipRanges.js

@@ -0,0 +1,161 @@
+import net from 'node:net';
+import { sha256 } from '../shared/io.js';
+
+/**
+ * Official published IP ranges for search and AI crawlers.
+ * Built-in CIDRs provide deterministic offline verification,
+ * while fetchProviderIpRanges supports dynamic feed refreshing.
+ */
+export const OFFICIAL_PROVIDER_RANGES = {
+  google: {
+    name: 'Google',
+    url: 'https://developers.google.com/search/apis/ipranges/googlebot.json',
+    patterns: [/Googlebot/i, /Google-InspectionTool/i, /Storebot-Google/i],
+    cidrs: [
+      '66.249.64.0/19',
+      '64.233.160.0/19',
+      '72.14.192.0/18',
+      '209.85.128.0/17',
+      '216.239.32.0/19',
+      '2001:4860:4801::/48',
+      '2607:f8b0:4000::/36',
+    ],
+  },
+  bing: {
+    name: 'Bing',
+    url: 'https://www.bing.com/toolbox/bingbot.json',
+    patterns: [/bingbot/i, /BingPreview/i, /adidxbot/i],
+    cidrs: [
+      '157.55.39.0/24',
+      '207.46.13.0/24',
+      '40.77.167.0/24',
+      '13.66.139.0/24',
+      '13.66.144.0/24',
+      '20.105.244.0/24',
+      '2603:1030::/32',
+    ],
+  },
+  openai: {
+    name: 'OpenAI',
+    url: 'https://openai.com/gptbot.json',
+    patterns: [/GPTBot/i, /OAI-SearchBot/i, /ChatGPT-User/i],
+    cidrs: [
+      '20.15.240.64/28',
+      '20.15.240.80/28',
+      '20.15.240.96/28',
+      '20.15.240.112/28',
+      '52.230.152.0/24',
+      '52.230.153.0/24',
+    ],
+  },
+  perplexity: {
+    name: 'Perplexity',
+    url: 'https://www.perplexity.ai/perplexitybot.json',
+    patterns: [/PerplexityBot/i],
+    cidrs: [
+      '199.59.148.0/22',
+      '199.16.156.0/22',
+    ],
+  },
+  anthropic: {
+    name: 'Anthropic',
+    url: 'https://anthropic.com/robots.txt',
+    patterns: [/ClaudeBot/i, /Claude-Web/i],
+    cidrs: [
+      '160.79.104.0/23',
+      '160.79.106.0/23',
+    ],
+  },
+};
+
+export function identifyCrawlerProvider(userAgent) {
+  if (!userAgent) return null;
+  for (const [key, provider] of Object.entries(OFFICIAL_PROVIDER_RANGES)) {
+    if (provider.patterns.some((re) => re.test(userAgent))) {
+      return key;
+    }
+  }
+  return null;
+}
+
+export function matchIpInCidrs(sourceIp, cidrs) {
+  if (!sourceIp || !Array.isArray(cidrs) || cidrs.length === 0) return { matched: false, matchedCidr: null };
+  const family = net.isIP(sourceIp);
+  if (!family) return { matched: false, matchedCidr: null };
+
+  const ipType = family === 4 ? 'ipv4' : 'ipv6';
+  for (const cidr of cidrs) {
+    const [network, prefixStr] = String(cidr).split('/');
+    const prefix = Number(prefixStr);
+    const rangeFamily = net.isIP(network);
+    if (rangeFamily !== family || !Number.isInteger(prefix)) continue;
+
+    const block = new net.BlockList();
+    block.addSubnet(network, prefix, ipType);
+    if (block.check(sourceIp, ipType)) {
+      return { matched: true, matchedCidr: cidr };
+    }
+  }
+  return { matched: false, matchedCidr: null };
+}
+
+export function verifyCrawlerIp(sourceIp, userAgent, options = {}) {
+  const providerKey = identifyCrawlerProvider(userAgent);
+  if (!providerKey) {
+    return {
+      knownProvider: false,
+      provider: null,
+      matched: false,
+      matchedCidr: null,
+      source: 'unrecognized_user_agent',
+    };
+  }
+
+  const providerDef = OFFICIAL_PROVIDER_RANGES[providerKey];
+  const customCidrs = options.ranges?.[userAgent] || options.ranges?.[providerDef.name] || options.ranges?.[providerKey];
+  const cidrs = customCidrs || providerDef.cidrs;
+
+  const result = matchIpInCidrs(sourceIp, cidrs);
+  return {
+    knownProvider: true,
+    provider: providerDef.name,
+    providerKey,
+    matched: result.matched,
+    matchedCidr: result.matchedCidr,
+    rangeSource: customCidrs ? 'custom_supplied' : 'official_published',
+    sourceUrl: providerDef.url,
+  };
+}
+
+export async function fetchProviderIpRanges(providerKey, { fetchImpl = globalThis.fetch, timeoutMs = 5000 } = {}) {
+  const provider = OFFICIAL_PROVIDER_RANGES[providerKey];
+  if (!provider || !provider.url) {
+    throw new Error(`unknown or unconfigured IP range provider: ${providerKey}`);
+  }
+
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+
+  try {
+    const res = await fetchImpl(provider.url, { signal: controller.signal });
+    clearTimeout(timer);
+    if (!res.ok) {
+      throw new Error(`HTTP ${res.status} fetching IP ranges for ${providerKey}`);
+    }
+
+    const text = await res.text();
+    const data = JSON.parse(text);
+    const prefixes = (data.prefixes || []).map((p) => p.ipv4Prefix || p.ipv6Prefix || p.cidr || p).filter(Boolean);
+
+    return {
+      provider: provider.name,
+      url: provider.url,
+      checksum: sha256(text),
+      cidrs: prefixes.length > 0 ? prefixes : provider.cidrs,
+      retrieved_at: new Date().toISOString(),
+    };
+  } catch (err) {
+    clearTimeout(timer);
+    throw err;
+  }
+}

package/src/detectors/agent.js

@@ -11,6 +11,8 @@
  *   - Agentic Commerce (x402, MPP, UCP, ACP)
  */
 
+import fs from 'node:fs';
+import path from 'node:path';
 import { defineDetector } from './framework.js';
 
 // ---------------------------------------------------------------------------
@@ -481,6 +483,130 @@ export const AGENT_010 = defineDetector({
   },
 });
 
+// ---------------------------------------------------------------------------
+// AGENT-011: llms.txt structure and link integrity
+// ---------------------------------------------------------------------------
+
+export const AGENT_011 = defineDetector({
+  id: 'AGENT-011',
+  name: 'llms.txt structure or link integrity broken',
+  namespace: 'AGENT',
+  discipline: ['agent-readiness', 'geo'],
+  severity: 'medium',
+  deterministic: true,
+  description:
+    'An /llms.txt or /llms-full.txt file exists on the site but violates the llmstxt.org specification ' +
+    '(missing H1 title, missing blockquote description, malformed Markdown links) or contains broken links.',
+  applicable_requirement: 'llmstxt.org specification; GEO §3 discoverability; agent-readiness §content-accessibility',
+  remediation:
+    'Format /llms.txt with an opening H1 title, a blockquote summary, and valid Markdown links pointing to active HTTP 200 URLs. Run `citable generate llms-txt --write` to generate a conforming specification.',
+  verification: 'Fetch /llms.txt and verify H1 title, blockquote description, and resolvable link URLs.',
+  check(ctx) {
+    let raw = null;
+    let identifier = '/llms.txt';
+
+    if (ctx.site?.llmsTxt?.raw) {
+      raw = ctx.site.llmsTxt.raw;
+    } else {
+      const page = ctx.site?.pages?.find((p) => p.path === '/llms.txt' || p.url?.endsWith('/llms.txt'));
+      if (page) {
+        raw = page.rawHtml || page.text || null;
+        identifier = page.url || '/llms.txt';
+      } else if (ctx.site?.location) {
+        const localPath = path.join(ctx.site.location, 'llms.txt');
+        if (fs.existsSync(localPath)) {
+          raw = fs.readFileSync(localPath, 'utf8');
+        }
+      }
+    }
+
+    if (!raw || raw.trim().length === 0) return [];
+
+    const findings = [];
+    const problems = [];
+
+    if (!/^#\s+.+/m.test(raw)) {
+      problems.push('Missing H1 project/site title heading (# Title)');
+    }
+
+    if (!/^>\s+.+/m.test(raw)) {
+      problems.push('Missing blockquote description/summary (> Summary)');
+    }
+
+    const linkMatches = [...raw.matchAll(/\[([^\]]+)\]\(([^)]+)\)/g)];
+    const sitePages = ctx.site?.pages || [];
+    const brokenLinks = [];
+
+    for (const match of linkMatches) {
+      const linkText = match[1];
+      const linkUrl = match[2].trim();
+
+      let normalizedPath = null;
+      try {
+        if (linkUrl.startsWith('http://') || linkUrl.startsWith('https://')) {
+          const parsed = new URL(linkUrl);
+          const baseOrigin = ctx.config?.site?.base_url
+            ? new URL(ctx.config.site.base_url).origin
+            : (sitePages[0] ? new URL(sitePages[0].url).origin : null);
+          if (baseOrigin && parsed.origin === baseOrigin) {
+            normalizedPath = parsed.pathname;
+          }
+        } else if (linkUrl.startsWith('/')) {
+          normalizedPath = linkUrl.split('?')[0].split('#')[0];
+        }
+      } catch {
+        problems.push(`Malformed URL in markdown link: [${linkText}](${linkUrl})`);
+        continue;
+      }
+
+      if (normalizedPath && sitePages.length > 0) {
+        const existsInPages = sitePages.some((p) => {
+          try {
+            const pagePath = new URL(p.url).pathname;
+            return (
+              pagePath === normalizedPath ||
+              pagePath === normalizedPath + '/' ||
+              normalizedPath === pagePath + '/' ||
+              pagePath.replace(/\/index\.html$/, '/') === normalizedPath
+            );
+          } catch {
+            return false;
+          }
+        });
+
+        if (!existsInPages && (normalizedPath === '/llms-full.txt' || normalizedPath.endsWith('/llms-full.txt'))) {
+          const fullExists = Boolean(
+            ctx.site?.llmsFullTxt?.raw ||
+            ctx.site?.pages?.some((p) => p.path === '/llms-full.txt' || p.url?.endsWith('/llms-full.txt')) ||
+            (ctx.site?.location && fs.existsSync(path.join(ctx.site.location, 'llms-full.txt')))
+          );
+          if (!fullExists) {
+            brokenLinks.push(`[${linkText}](${linkUrl}) -> referenced /llms-full.txt not found`);
+          }
+        } else if (!existsInPages) {
+          brokenLinks.push(`[${linkText}](${linkUrl}) -> URL not found in crawled site pages`);
+        }
+      }
+    }
+
+    if (brokenLinks.length > 0) {
+      problems.push(`${brokenLinks.length} broken/unresolved internal link(s): ${brokenLinks.slice(0, 3).join(', ')}`);
+    }
+
+    if (problems.length > 0) {
+      findings.push({
+        subject: { type: 'file', identifier },
+        summary: `/llms.txt has ${problems.length} specification / link integrity problem(s)`,
+        evidence: problems,
+        captured: problems.join('; '),
+        expected: 'Valid llmstxt.org format with H1 title, blockquote summary, and resolvable link URLs',
+      });
+    }
+
+    return findings;
+  },
+});
+
 export const AGENT_DETECTORS = [
   AGENT_001,
   AGENT_002,
@@ -492,4 +618,5 @@ export const AGENT_DETECTORS = [
   AGENT_008,
   AGENT_009,
   AGENT_010,
+  AGENT_011,
 ];

package/src/detectors/ans.js

@@ -1,4 +1,5 @@
 import { defineDetector, indexTargets, registryPageFor, pageSubject } from './framework.js';
+import { parse } from 'node-html-parser';
 
 const D = [];
 
@@ -259,4 +260,285 @@ D.push(defineDetector({
   },
 }));
 
+D.push(defineDetector({
+  id: 'ANS-011', name: 'Over-diluted answer passage (missing direct answer lead)', namespace: 'ANS',
+  description: 'A section targeting an informational query or question heading lacks a concise, direct answer passage in its opening sentences, burying key factual answers behind excessive preamble or rambling prose.',
+  discipline: ['aeo', 'geo'], severity: 'medium', deterministic: false, requires: ['site'],
+  impact: { citation: 'medium', representation: 'low' },
+  applicable_requirement: 'AEO §4 direct answer block in first 50–100 words; AEO §5 concise declarative answer passages',
+  false_positive_conditions: ['narrative essays or creative fiction where question headings are rhetorical devices'],
+  remediation: 'Provide a concise 30–70 word direct declarative answer immediately following the question heading before elaborating with background details.',
+  verification: 'Re-inspect question sections to verify the first paragraph delivers a direct factual answer.',
+  check(ctx) {
+    const hits = [];
+    const RAMBLE_RX = /^(to understand|before (exploring|answering|diving|we)|in order to understand|it is (important|essential) to (note|understand|first)|there are many (factors|aspects)|the answer to this question depends on)/i;
+
+    for (const p of indexTargets(ctx)) {
+      if (p.status !== 200 || !p.rawHtml) continue;
+      const doc = parse(p.rawHtml);
+      const headings = doc.querySelectorAll('h2, h3, h4');
+
+      for (const h of headings) {
+        const text = h.text.trim();
+        const isQuestion = /\?$/.test(text) || /^(what|how|why|when|where|who|is|are|can|does|do|which)\b/i.test(text);
+        if (!isQuestion) continue;
+
+        let curr = h.nextElementSibling;
+        let firstP = null;
+        while (curr && !/^h[1-6]$/i.test(curr.tagName)) {
+          if (curr.tagName.toLowerCase() === 'p') {
+            const pt = curr.text.trim();
+            if (pt.length > 0) {
+              firstP = pt;
+              break;
+            }
+          }
+          curr = curr.nextElementSibling;
+        }
+
+        if (firstP) {
+          const words = firstP.split(/\s+/).filter(Boolean);
+          const isRambling = RAMBLE_RX.test(firstP);
+          const isOverlyLongWithoutDirectness = words.length > 130;
+          if (isRambling || isOverlyLongWithoutDirectness) {
+            hits.push({
+              subject: pageSubject(p),
+              summary: `Question section "${text.slice(0, 60)}" lacks a concise direct answer lead (${isRambling ? 'rambling preamble' : `opening paragraph is ${words.length} words`})`,
+              evidence: [
+                `heading: "${text}"`,
+                `opening: "${firstP.slice(0, 150)}..."`,
+                `word_count: ${words.length}`,
+              ],
+              confidence: 'medium',
+            });
+          }
+        }
+      }
+    }
+    return hits;
+  },
+}));
+
+D.push(defineDetector({
+  id: 'ANS-012', name: 'Ungrounded quantitative metric in answer passage', namespace: 'ANS',
+  description: 'An answer-bearing passage asserts high-stakes quantitative metrics, percentages, or multiples without adjacent evidence citation, benchmark reference, or registered evidence link.',
+  discipline: ['aeo', 'geo'], severity: 'medium', deterministic: false, requires: ['site'],
+  impact: { citation: 'medium', legal: 'low' },
+  applicable_requirement: 'AEO §8 version evidence; GEO §5 substantiated factual claims; premise 3.5: quantitative outcomes require grounding',
+  false_positive_conditions: ['introductory marketing pages where detailed case studies are linked elsewhere in the navigation'],
+  remediation: 'Attach an adjacent citation, benchmark link, methodology note, or registered evidence ID near the metric.',
+  verification: 'Confirm every statistical or multiple claim has adjacent attribution or registered evidence.',
+  check(ctx) {
+    const hits = [];
+    const STAT_RX = /\b(\d+(?:\.\d+)?%\s*(?:reduction|increase|faster|decrease|improvement|growth|boost|drop|savings?)|(?:\d+(?:\.\d+)?x)\s*(?:faster|more|performance|throughput|speed|improvement)|\$\d+(?:\.\d+)?[MBK]\b)/i;
+    const ATTRIBUTION_RX = /\[\d+\]|\((?:source|benchmark|study|audit|test|report|see)\b|according to|citation|evidence|EVD-|CLAIM-|\bhttps?:\/\/|href=/i;
+
+    for (const p of indexTargets(ctx)) {
+      if (p.status !== 200 || !p.paragraphs) continue;
+
+      for (const para of p.paragraphs) {
+        const statMatch = STAT_RX.exec(para);
+        if (statMatch) {
+          const hasAttribution = ATTRIBUTION_RX.test(para);
+          if (!hasAttribution) {
+            hits.push({
+              subject: pageSubject(p),
+              summary: `Ungrounded quantitative metric "${statMatch[1]}" in answer prose lacks citation or evidence attribution`,
+              evidence: [
+                `metric: "${statMatch[1]}"`,
+                `passage: "${para.slice(0, 150)}..."`,
+              ],
+              confidence: 'medium',
+            });
+          }
+        }
+      }
+    }
+    return hits;
+  },
+}));
+
+D.push(defineDetector({
+  id: 'ANS-013', name: 'Comparative or procedural section lacking structured table or ordered sequence', namespace: 'ANS',
+  description: 'A section covering comparative evaluation (vs, alternatives, comparison) or multi-step execution (how to, steps) lacks structured <table>, <ol>, or <ul> markup, diminishing answer engine extraction fidelity.',
+  discipline: ['aeo', 'seo'], severity: 'medium', deterministic: true, requires: ['site'],
+  impact: { representation: 'medium', citation: 'low' },
+  applicable_requirement: 'AEO §5 structured answer surfaces; GEO §4 extractability; premise 3.3 information gain',
+  remediation: 'Represent comparisons with <table> markup and sequential instructions with <ol> step lists.',
+  verification: 'Confirm comparative/procedural sections contain <table> or <ol> structured elements.',
+  check(ctx) {
+    const hits = [];
+    const COMPARATIVE_RX = /\b(vs\.?|versus|comparison|alternatives|pros and cons|differences between)\b/i;
+    const PROCEDURAL_RX = /\b(how to|steps to|installation steps|quickstart guide|configuration steps|deployment guide)\b/i;
+
+    for (const p of indexTargets(ctx)) {
+      if (p.status !== 200 || !p.rawHtml) continue;
+      const doc = parse(p.rawHtml);
+      const headings = doc.querySelectorAll('h2, h3');
+
+      for (const h of headings) {
+        const text = h.text.trim();
+        const isComp = COMPARATIVE_RX.test(text);
+        const isProc = PROCEDURAL_RX.test(text);
+        if (!isComp && !isProc) continue;
+
+        let curr = h.nextElementSibling;
+        let hasTable = false;
+        let hasList = false;
+        let textWordCount = 0;
+
+        while (curr && !/^h[1-3]$/i.test(curr.tagName)) {
+          if (curr.tagName.toLowerCase() === 'table' || curr.querySelector('table')) hasTable = true;
+          if (/^(ol|ul)$/i.test(curr.tagName) || curr.querySelector('ol, ul')) hasList = true;
+          if (curr.tagName.toLowerCase() === 'p') {
+            textWordCount += curr.text.split(/\s+/).filter(Boolean).length;
+          }
+          curr = curr.nextElementSibling;
+        }
+
+        if (textWordCount > 50) {
+          if (isComp && !hasTable) {
+            hits.push({
+              subject: pageSubject(p),
+              summary: `Comparative section "${text.slice(0, 60)}" lacks a structured <table> comparison surface`,
+              evidence: [`heading: "${text}"`, `section word count: ${textWordCount}`, 'no <table> detected before next heading'],
+              confidence: 'high',
+            });
+          } else if (isProc && !hasList) {
+            hits.push({
+              subject: pageSubject(p),
+              summary: `Procedural section "${text.slice(0, 60)}" lacks structured <ol> step or list markup`,
+              evidence: [`heading: "${text}"`, `section word count: ${textWordCount}`, 'no <ol>/<ul> detected before next heading'],
+              confidence: 'high',
+            });
+          }
+        }
+      }
+    }
+    return hits;
+  },
+}));
+
+D.push(defineDetector({
+  id: 'ANS-014', name: 'Low information-gain fluff ratio in answer prose', namespace: 'ANS',
+  description: 'An answer passage contains excessive rhetorical filler, clichés, or conversational padding, reducing factual density for generative engine extraction.',
+  discipline: ['aeo', 'geo'], severity: 'low', deterministic: true, requires: ['site'],
+  impact: { representation: 'low', citation: 'low' },
+  applicable_requirement: 'AEO §8 concise factual grounding; GEO §4 information gain; premise 3.3 substantive answer density',
+  remediation: 'Remove conversational clichés, tautologies, and promotional fluff; state verifiable facts, properties, and constraints directly.',
+  verification: 'Re-audit answer passages to ensure cliché and filler phrases are removed.',
+  check(ctx) {
+    const hits = [];
+    const FLUFF_PATTERNS = [
+      /\b(?:as we all know|it goes without saying|needless to say)\b/i,
+      /\bat the end of the day\b/i,
+      /\b(?:in today's rapidly evolving|in this day and age)\b/i,
+      /\b(?:game[- ]changer|revolutionary new|paradigm shift)\b/i,
+      /\b(?:delve into|dive deep into|unlock the power of|leverage the power of)\b/i,
+      /\b(?:seamlessly integrate|cutting[- ]edge technology)\b/i,
+    ];
+
+    for (const p of indexTargets(ctx)) {
+      if (p.status !== 200 || !p.paragraphs) continue;
+
+      for (const para of p.paragraphs) {
+        if (para.length < 60) continue;
+        const matchedFluff = [];
+        for (const rx of FLUFF_PATTERNS) {
+          const m = rx.exec(para);
+          if (m) matchedFluff.push(m[0]);
+        }
+
+        if (matchedFluff.length >= 2) {
+          hits.push({
+            subject: pageSubject(p),
+            summary: `Answer passage contains low information-gain fluff clichés (${matchedFluff.join(', ')})`,
+            evidence: [
+              `clichés detected: ${matchedFluff.join(', ')}`,
+              `passage: "${para.slice(0, 150)}..."`,
+            ],
+            confidence: 'medium',
+          });
+        }
+      }
+    }
+    return hits;
+  },
+}));
+
+D.push(defineDetector({
+  id: 'ANS-015', name: 'Definitional page lacks direct copular definition', namespace: 'ANS',
+  description: 'A page with definitional intent (page_type=definition, /definition/, /glossary/, or "What is" heading) does not provide a direct copular definition ("X is a Y that Z") in the opening paragraph.',
+  discipline: ['aeo', 'geo'], severity: 'medium', deterministic: true, requires: ['site'],
+  impact: { retrieval: 'medium', citation: 'high' },
+  applicable_requirement: 'AEO §4 direct copular answer block; search engine featured snippet & LLM definition synthesis requirements',
+  remediation: 'Formulate the opening sentence with a direct copular definition stating what the concept is, its primary class, and its defining capability.',
+  verification: 'Confirm the opening paragraph defines the entity using a copular verb ("is", "are", "refers to") within the first 140 characters.',
+  check(ctx) {
+    const hits = [];
+    const COPULA_RX = /\b(?:is|are|refers\s+to|denotes|represents|means)\b/i;
+    for (const p of indexTargets(ctx)) {
+      if (p.status !== 200 || !p.paragraphs || p.paragraphs.length === 0) continue;
+      const reg = registryPageFor(ctx, p);
+      const isDefType = reg?.page_type === 'definition' || /\/(definition|glossary)\b/i.test(p.url);
+      const firstH1 = (p.headings || []).find((h) => h.level === 1)?.text || '';
+      const isWhatIsHeading = /^(what\s+is|definition\s+of|meaning\s+of)\b/i.test(firstH1);
+
+      if (isDefType || isWhatIsHeading) {
+        const firstP = p.paragraphs[0] || '';
+        const openingSlice = firstP.slice(0, 140);
+        if (!COPULA_RX.test(openingSlice)) {
+          hits.push({
+            subject: pageSubject(p),
+            summary: `Definitional page opening does not provide a direct copular definition ("is", "are", "refers to")`,
+            evidence: [
+              `opening passage: "${firstP.slice(0, 120)}..."`,
+              'definitional query extractors require immediate copular formulation in lead sentence',
+            ],
+            captured: firstP.slice(0, 80),
+            expected: 'direct copular sentence (e.g. "X is a Y that Z") in first 140 characters',
+          });
+        }
+      }
+    }
+    return hits;
+  },
+}));
+
+D.push(defineDetector({
+  id: 'ANS-016', name: 'Comprehensive long-form guide lacks executive summary or key takeaways', namespace: 'ANS',
+  description: 'An in-depth article or guide exceeding 1,200 words does not provide an executive summary, key takeaways, or TL;DR block, increasing RAG chunk fragmentation and extraction ambiguity.',
+  discipline: ['aeo', 'geo'], severity: 'low', deterministic: true, requires: ['site'],
+  impact: { representation: 'medium', citation: 'low' },
+  applicable_requirement: 'AEO §4 structural answer density; GEO §4 top-level synthesis grounding for RAG embedding retrieval',
+  remediation: 'Add an executive summary or bulleted key takeaways block near the top of the article before detailed section walkthroughs.',
+  verification: 'Re-audit page to verify a summary, key takeaways, or TL;DR section precedes long-form content.',
+  check(ctx) {
+    const hits = [];
+    const SUMMARY_RX = /\b(?:executive\s+summary|key\s+takeaways?|summary|overview|at\s+a\s+glance|tl;?dr|quick\s+summary|highlights?)\b/i;
+    for (const p of indexTargets(ctx)) {
+      if (p.status !== 200) continue;
+      const wordCount = p.rawVisibleWordCount || p.wordCount || 0;
+      if (wordCount < 1200) continue;
+
+      const hasSummaryHeading = (p.headings || []).some((h) => SUMMARY_RX.test(h.text));
+      if (!hasSummaryHeading) {
+        hits.push({
+          subject: pageSubject(p),
+          summary: `Long-form guide (${wordCount} words) lacks an executive summary, key takeaways, or TL;DR section`,
+          evidence: [
+            `word count: ${wordCount} words`,
+            'no heading matches executive summary, key takeaways, or TL;DR pattern',
+            'sprawling unsummarized documents experience lower RAG retrieval relevance in generative answer engines',
+          ],
+          captured: { wordCount, hasSummaryHeading: false },
+          expected: 'executive summary or key takeaways section for documents > 1,200 words',
+        });
+      }
+    }
+    return hits;
+  },
+}));
+
 export default D;
+

package/src/detectors/crawl.js

@@ -1,6 +1,7 @@
 import { defineDetector, entrySubject } from './framework.js';
 import { isAllowed } from '../crawler/robots.js';
 import { isPastDate } from '../shared/io.js';
+import { verifyCrawlerIp } from '../crawler/ipRanges.js';
 
 const D = [];
 
@@ -140,4 +141,49 @@ D.push(defineDetector({
   },
 }));
 
+D.push(defineDetector({
+  id: 'CRAWL-007', name: 'Spoofed crawler identity in access logs', namespace: 'CRAWL',
+  description: 'An observed crawler log row claims an authoritative bot user-agent (Googlebot, Bingbot, GPTBot, ClaudeBot, etc.) but its source IP is outside published provider ranges or reverse-DNS verification failed.',
+  discipline: ['seo', 'aeo', 'geo'], severity: 'high', deterministic: true, requires: ['site'],
+  impact: { retrieval: 'high', legal: 'medium' },
+  applicable_requirement: 'GEO §2 crawler verification; staged identity contract and provider-published IP boundaries',
+  remediation: 'Verify edge/WAF logs against official provider IP ranges or reverse-DNS, and block or throttle unverified impersonating crawlers.',
+  verification: 'Confirm observed crawler requests originate within official published CIDR ranges and pass rDNS.',
+  check(ctx) {
+    if (!ctx.observations) return [];
+    const hits = [];
+    for (const obs of ctx.observations) {
+      if (obs.kind !== 'crawler_log' || !obs.data) continue;
+      const d = obs.data;
+      const ua = d.user_agent;
+      const ip = d.source_ip;
+      const identity = d.crawler_identity;
+      if (!ua) continue;
+
+      const verification = verifyCrawlerIp(ip, ua);
+      const isContradictory = identity?.verification_status === 'contradictory'
+        || identity?.cidr_membership === 'not_matched'
+        || (verification.knownProvider && !verification.matched);
+
+      if (isContradictory) {
+        hits.push({
+          subject: { type: 'crawler', identifier: ua },
+          summary: `Spoofed or unverified crawler traffic detected for ${ua} from IP ${ip || 'unknown'}`,
+          evidence: [
+            `user_agent: ${ua}`,
+            `source_ip: ${ip || 'none'}`,
+            `provider: ${verification.provider || identity?.provider || 'unknown'}`,
+            `verification_status: ${identity?.verification_status || 'contradictory'}`,
+            `cidr_membership: ${identity?.cidr_membership || (verification.matched ? 'matched' : 'not_matched')}`,
+            `range_source: ${verification.sourceUrl || 'provider_published'}`,
+          ],
+          captured: { user_agent: ua, source_ip: ip, verification_status: identity?.verification_status || 'contradictory' },
+          expected: 'Crawler requests must originate from official provider-published IP ranges',
+        });
+      }
+    }
+    return hits;
+  },
+}));
+
 export default D;