@nebulacomponents/citable 1.18.2 → 1.20.0

package/src/crawler/sitemapCollector.js

@@ -0,0 +1,364 @@
+import { gunzipSync } from 'node:zlib';
+import { fetchUrl } from './fetch.js';
+import { parseSitemap } from './sitemap.js';
+
+const DEFAULT_MAX_DEPTH = 4;
+const DEFAULT_MAX_DOCUMENTS = 1000;
+const DEFAULT_MAX_UNCOMPRESSED_BYTES = 20 * 1024 * 1024;
+const DEFAULT_MAX_TOTAL_UNCOMPRESSED_BYTES = 100 * 1024 * 1024;
+const DEFAULT_MAX_DISCOVERED_URLS = 50_000;
+const DEFAULT_MAX_QUEUED_DOCUMENTS = 1000;
+const DEFAULT_MAX_RAW_ENTRIES = 100_000;
+const DEFAULT_MAX_XML_TOKENS = 500_000;
+const DEFAULT_SITEMAP_MAX_BYTES = 5 * 1024 * 1024;
+
+/** Collect a same-origin sitemap graph with deterministic breadth-first traversal. */
+export async function collectSitemapTopology(entryUrls, {
+  fetcher = fetchUrl,
+  maxDepth = DEFAULT_MAX_DEPTH,
+  maxDocuments = DEFAULT_MAX_DOCUMENTS,
+  maxUncompressedBytes = DEFAULT_MAX_UNCOMPRESSED_BYTES,
+  maxTotalUncompressedBytes = DEFAULT_MAX_TOTAL_UNCOMPRESSED_BYTES,
+  maxDiscoveredUrls = DEFAULT_MAX_DISCOVERED_URLS,
+  maxQueuedDocuments = DEFAULT_MAX_QUEUED_DOCUMENTS,
+  maxRawEntries = DEFAULT_MAX_RAW_ENTRIES,
+  maxXmlTokens = DEFAULT_MAX_XML_TOKENS,
+  sitemapMaxBytes = DEFAULT_SITEMAP_MAX_BYTES,
+  origin,
+  userAgent,
+  shouldStop = () => false,
+} = {}) {
+  assertInteger(maxDepth, 'maxDepth', 0);
+  assertInteger(maxDocuments, 'maxDocuments', 1);
+  assertInteger(maxUncompressedBytes, 'maxUncompressedBytes', 1);
+  assertInteger(maxTotalUncompressedBytes, 'maxTotalUncompressedBytes', 1);
+  assertInteger(maxDiscoveredUrls, 'maxDiscoveredUrls', 1);
+  assertInteger(maxQueuedDocuments, 'maxQueuedDocuments', 1);
+  assertInteger(maxRawEntries, 'maxRawEntries', 1);
+  assertInteger(maxXmlTokens, 'maxXmlTokens', 1);
+  assertInteger(sitemapMaxBytes, 'sitemapMaxBytes', 1);
+  const entries = Array.isArray(entryUrls) ? entryUrls : [entryUrls];
+  const auditedOrigin = new URL(origin ?? entries.find(Boolean)).origin;
+  const queue = [];
+  const queued = new Set();
+  const documents = [];
+  const urls = [];
+  const seenUrls = new Set();
+  const exclusions = [];
+  const errors = [];
+  const limitations = [];
+  const stopReasons = [];
+  let totalUncompressedBytes = 0;
+  let queuedDocumentsPeak = 0;
+
+  const addStop = (reason, limitation) => {
+    if (!stopReasons.includes(reason)) stopReasons.push(reason);
+    if (limitation && !limitations.includes(limitation)) limitations.push(limitation);
+  };
+  const enqueue = (rawUrl, depth, parentUrl, kind) => {
+    let parsed;
+    try {
+      parsed = new URL(rawUrl, parentUrl ?? auditedOrigin);
+      parsed.hash = '';
+    } catch {
+      exclusions.push({ url: String(rawUrl ?? ''), kind, source_url: parentUrl, reason: 'malformed_url' });
+      errors.push({ url: String(rawUrl ?? ''), reason: 'malformed_url' });
+      return 'malformed_url';
+    }
+    if (parsed.origin !== auditedOrigin) {
+      exclusions.push({ url: parsed.href, kind, source_url: parentUrl, reason: 'out_of_origin' });
+      errors.push({ url: parsed.href, reason: 'out_of_origin' });
+      return 'out_of_origin';
+    }
+    if (queued.has(parsed.href)) return 'duplicate';
+    if (queue.length >= maxQueuedDocuments) {
+      addStop(
+        'max_queued_documents_exceeded',
+        `Sitemap topology collection reached the ${maxQueuedDocuments}-document queued-frontier limit.`,
+      );
+      return 'max_queued_documents_exceeded';
+    }
+    queued.add(parsed.href);
+    queue.push({ url: parsed.href, depth, parentUrl });
+    queuedDocumentsPeak = Math.max(queuedDocumentsPeak, queue.length);
+    return null;
+  };
+
+  for (let index = 0; index < entries.length; index += 1) {
+    if (index >= maxRawEntries) {
+      addStop(
+        'max_raw_entries_exceeded',
+        `Sitemap topology collection reached the ${maxRawEntries}-entry raw-input safety limit.`,
+      );
+      break;
+    }
+    enqueue(entries[index], 0, null, 'entry_sitemap');
+  }
+
+  collection: while (queue.length) {
+    if (shouldStop()) {
+      addStop('time_budget_exhausted', 'Sitemap topology collection stopped when the whole-run time budget expired.');
+      break;
+    }
+    if (documents.length >= maxDocuments) {
+      addStop('max_documents_exceeded', `Sitemap topology collection reached the ${maxDocuments}-document limit.`);
+      break;
+    }
+    const item = queue.shift();
+    if (item.depth > maxDepth) {
+      addStop('max_depth_exceeded', `Sitemap topology collection reached the maximum depth of ${maxDepth}.`);
+      continue;
+    }
+
+    const record = {
+      requested_url: item.url,
+      effective_url: item.url,
+      depth: item.depth,
+      parent_url: item.parentUrl,
+      http_status: null,
+      compression: 'none',
+      compression_transport_decoded: false,
+      parse_errors: [],
+      url_count: 0,
+      child_count: 0,
+      raw_entry_count: 0,
+      status: 'failed',
+      failure_reason: null,
+      failure_stage: null,
+      parsed: null,
+    };
+    documents.push(record);
+    try {
+      const response = await fetcher(item.url, {
+        userAgent,
+        maxBodyBytes: sitemapMaxBytes,
+        responseType: 'buffer',
+      });
+      record.effective_url = response.url ?? item.url;
+      record.http_status = response.status ?? null;
+      if (new URL(record.effective_url).origin !== auditedOrigin) {
+        record.failure_reason = 'effective_url_out_of_origin';
+        record.failure_stage = 'policy';
+        errors.push({ url: item.url, reason: record.failure_reason });
+        continue;
+      }
+      if (response.status !== 200) {
+        record.failure_reason = `http_status_${response.status}`;
+        record.failure_stage = 'transport';
+        errors.push({ url: item.url, reason: record.failure_reason });
+        continue;
+      }
+
+      record.compression = detectCompression(response.body, response.headers, record.effective_url);
+      const decoded = decodeBody(
+        response.body, response.headers, record.effective_url, maxUncompressedBytes, record.compression,
+      );
+      record.compression_transport_decoded = decoded.transportDecoded;
+      record.uncompressed_bytes = decoded.uncompressedBytes;
+      if (totalUncompressedBytes + decoded.uncompressedBytes > maxTotalUncompressedBytes) {
+        markTruncated(record, 'max_total_uncompressed_bytes_exceeded');
+        addStop(
+          'max_total_uncompressed_bytes_exceeded',
+          `Sitemap topology collection exceeded the ${maxTotalUncompressedBytes}-byte cumulative uncompressed limit.`,
+        );
+        break collection;
+      }
+      totalUncompressedBytes += decoded.uncompressedBytes;
+      const parsed = parseSitemap(decoded.text, {
+        maxUrls: maxRawEntries,
+        maxChildren: maxRawEntries,
+        maxEntries: maxRawEntries,
+        maxTokens: maxXmlTokens,
+      });
+      record.parsed = parsed;
+      record.parse_errors = [...parsed.errors];
+      record.url_count = parsed.urlCount;
+      record.child_count = parsed.childCount;
+      record.raw_entry_count = parsed.rawEntryCount;
+      record.status = parsed.errors.length ? 'malformed' : 'fetched';
+      if (parsed.errors.length) {
+        record.failure_reason = 'sitemap_parse_error';
+        record.failure_stage = 'parse';
+        errors.push({ url: item.url, reason: record.failure_reason, details: [...parsed.errors] });
+      }
+      if (parsed.truncated) {
+        const [reason, limitation] = parsed.truncationReason === 'max_tokens_exceeded'
+          ? ['max_xml_tokens_exceeded', `Sitemap parsing reached the ${maxXmlTokens}-token structural limit.`]
+          : ['max_raw_entries_exceeded', `Sitemap parsing reached the ${maxRawEntries}-entry raw-input safety limit.`];
+        markTruncated(record, reason);
+        addStop(reason, limitation);
+      }
+
+      for (const child of parsed.rootValid ? parsed.children : []) {
+        const exclusionReason = enqueue(child.loc, item.depth + 1, record.effective_url, 'child_sitemap');
+        if (exclusionReason === 'malformed_url') markMalformedLoc(record, child.loc);
+        if (exclusionReason === 'max_queued_documents_exceeded') {
+          markTruncated(record, exclusionReason);
+          break;
+        }
+      }
+      for (const entry of parsed.rootValid ? parsed.urls : []) {
+        let page;
+        try {
+          page = new URL(entry.loc, record.effective_url);
+          page.hash = '';
+        } catch {
+          markMalformedLoc(record, entry.loc);
+          exclusions.push({ url: entry.loc, kind: 'page_url', source_url: record.effective_url, reason: 'malformed_url' });
+          errors.push({ url: entry.loc, reason: 'malformed_url' });
+          continue;
+        }
+        if (page.origin !== auditedOrigin) {
+          exclusions.push({ url: page.href, kind: 'page_url', source_url: record.effective_url, reason: 'out_of_origin' });
+          errors.push({ url: page.href, reason: 'out_of_origin' });
+          continue;
+        }
+        if (seenUrls.has(page.href)) continue;
+        if (urls.length >= maxDiscoveredUrls) {
+          markTruncated(record, 'max_discovered_urls_exceeded');
+          addStop(
+            'max_discovered_urls_exceeded',
+            `Sitemap topology collection reached the ${maxDiscoveredUrls}-URL discovery limit.`,
+          );
+          break;
+        }
+        seenUrls.add(page.href);
+        urls.push({ url: page.href, sitemap_url: record.effective_url, requested_sitemap_url: item.url, sitemap_depth: item.depth });
+      }
+      if (stopReasons.includes('max_discovered_urls_exceeded')) break collection;
+    } catch (error) {
+      record.failure_reason = classifyFailure(error);
+      record.failure_stage = failureStage(record.failure_reason);
+      errors.push({ url: item.url, reason: record.failure_reason, message: error.message });
+    }
+  }
+
+  if (shouldStop()) {
+    addStop('time_budget_exhausted', 'Sitemap topology collection stopped when the whole-run time budget expired.');
+  }
+
+  const truncated = stopReasons.length > 0;
+  const indeterminate = documents.some((document) => ['failed', 'malformed'].includes(document.status))
+    || errors.some((error) => ['malformed_url', 'out_of_origin'].includes(error.reason));
+  return {
+    documents,
+    urls,
+    status: truncated ? 'truncated' : indeterminate ? 'indeterminate' : 'complete',
+    limitations,
+    errors,
+    exclusions,
+    stop_reasons: stopReasons,
+    bounds: {
+      max_depth: maxDepth,
+      max_documents: maxDocuments,
+      max_uncompressed_bytes: maxUncompressedBytes,
+      max_total_uncompressed_bytes: maxTotalUncompressedBytes,
+      max_discovered_urls: maxDiscoveredUrls,
+      max_queued_documents: maxQueuedDocuments,
+      max_raw_entries: maxRawEntries,
+      max_xml_tokens: maxXmlTokens,
+      sitemap_max_bytes: sitemapMaxBytes,
+    },
+    totals: {
+      documents: documents.length,
+      urls: urls.length,
+      uncompressed_bytes: totalUncompressedBytes,
+      queued_documents_peak: queuedDocumentsPeak,
+    },
+  };
+}
+
+function detectCompression(body, headers = {}, url) {
+  const bytes = Buffer.isBuffer(body)
+    ? body
+    : body instanceof Uint8Array
+      ? Buffer.from(body.buffer, body.byteOffset, body.byteLength)
+      : Buffer.from(String(body ?? ''), 'utf8');
+  const contentEncoding = String(headerValue(headers, 'content-encoding') ?? '').toLowerCase();
+  const contentType = String(headerValue(headers, 'content-type') ?? '').toLowerCase();
+  const magicGzip = bytes.length >= 2 && bytes[0] === 0x1f && bytes[1] === 0x8b;
+  const hintedGzip = magicGzip || /(?:^|\W)gzip(?:\W|$)/.test(contentEncoding)
+    || /(?:application|text)\/(?:x-)?gzip/.test(contentType) || new URL(url).pathname.toLowerCase().endsWith('.gz');
+  return hintedGzip ? 'gzip' : 'none';
+}
+
+function decodeBody(body, headers, url, maxUncompressedBytes, compression = detectCompression(body, headers, url)) {
+  const bytes = Buffer.isBuffer(body)
+    ? body
+    : body instanceof Uint8Array
+      ? Buffer.from(body.buffer, body.byteOffset, body.byteLength)
+      : Buffer.from(String(body ?? ''), 'utf8');
+  const magicGzip = bytes.length >= 2 && bytes[0] === 0x1f && bytes[1] === 0x8b;
+  let decoded = bytes;
+  let transportDecoded = false;
+  if (compression === 'gzip') {
+    if (magicGzip) {
+      decoded = gunzipSync(bytes, { maxOutputLength: maxUncompressedBytes });
+    } else if (looksLikeDecodedXml(bytes)) {
+      transportDecoded = true;
+    } else {
+      const error = new Error('gzip-indicated sitemap body is neither gzip bytes nor transport-decoded XML');
+      error.code = 'SITEMAP_DECOMPRESSION_FAILED';
+      throw error;
+    }
+  }
+  if (decoded.byteLength > maxUncompressedBytes) {
+    const error = new Error(`sitemap uncompressed body exceeds ${maxUncompressedBytes} bytes`);
+    error.code = 'SITEMAP_UNCOMPRESSED_LIMIT';
+    throw error;
+  }
+  return { text: decoded.toString('utf8'), compression, transportDecoded, uncompressedBytes: decoded.byteLength };
+}
+
+function looksLikeDecodedXml(bytes) {
+  return /^\s*</u.test(bytes.toString('utf8').replace(/^\uFEFF/u, ''));
+}
+
+function headerValue(headers, name) {
+  if (headers && typeof headers.get === 'function') return headers.get(name);
+  const key = Object.keys(headers ?? {}).find((candidate) => candidate.toLowerCase() === name);
+  return key ? headers[key] : undefined;
+}
+
+function classifyFailure(error) {
+  if (error?.code === 'FETCH_BODY_LIMIT' || /^response body exceeds \d+ bytes$/i.test(error?.message ?? '')) {
+    return 'sitemap_transport_bytes_exceeded';
+  }
+  if (error?.code === 'SITEMAP_UNCOMPRESSED_LIMIT' || error?.code === 'ERR_BUFFER_TOO_LARGE'
+    || /maxoutputlength|larger than/i.test(error?.message ?? '')) {
+    return 'max_uncompressed_bytes_exceeded';
+  }
+  if (error?.code === 'SITEMAP_DECOMPRESSION_FAILED'
+    || /gzip|incorrect header|invalid distance|unexpected end/i.test(error?.message ?? '')) return 'decompression_failed';
+  return 'fetch_failed';
+}
+
+function markMalformedLoc(record, loc) {
+  const message = `entry has malformed <loc>: ${loc}`;
+  if (!record.parse_errors.includes(message)) record.parse_errors.push(message);
+  record.status = 'malformed';
+  record.failure_reason = 'sitemap_parse_error';
+  record.failure_stage = 'parse';
+}
+
+function markTruncated(record, reason) {
+  record.status = 'truncated';
+  record.failure_reason = reason;
+  record.failure_stage = 'collection';
+}
+
+function failureStage(reason) {
+  if (reason === 'sitemap_transport_bytes_exceeded' || reason === 'fetch_failed' || reason.startsWith('http_status_')) {
+    return 'transport';
+  }
+  if (reason === 'max_uncompressed_bytes_exceeded' || reason === 'decompression_failed') return 'decompression';
+  if (reason === 'sitemap_parse_error') return 'parse';
+  if (reason.includes('out_of_origin')) return 'policy';
+  return 'collection';
+}
+
+function assertInteger(value, name, minimum) {
+  if (!Number.isInteger(value) || value < minimum) {
+    throw new TypeError(`${name} must be an integer greater than or equal to ${minimum}`);
+  }
+}

package/src/crawler/urlIdentity.js

@@ -0,0 +1,97 @@
+import { sha256 } from '../shared/io.js';
+
+export const URL_IDENTITY_NORMALIZATION_VERSION = 'url-identity-v1';
+
+/**
+ * URL identity v1 deliberately performs only two transformations: remove the
+ * fragment and remove an explicit default HTTP(S) port. Query ordering,
+ * trailing slashes, casing, percent encoding, and host spelling are evidence
+ * and are therefore preserved.
+ */
+export function normalizeUrlIdentity(url) {
+  if (typeof url !== 'string' || !url) throw new TypeError('URL identity requires a non-empty URL string');
+  const fragmentAt = url.indexOf('#');
+  const withoutFragment = fragmentAt === -1 ? url : url.slice(0, fragmentAt);
+  const match = withoutFragment.match(/^([A-Za-z][A-Za-z\d+.-]*):\/\/([^/?#]*)([\s\S]*)$/);
+  if (!match) throw new TypeError(`URL identity requires an absolute URL: ${url}`);
+  const [, rawScheme, rawAuthority, suffix] = match;
+  const scheme = rawScheme.toLowerCase();
+  if (scheme !== 'http' && scheme !== 'https') throw new TypeError(`unsupported URL identity protocol: ${rawScheme}:`);
+  if (rawAuthority.includes('@')) throw new TypeError('URL credentials are not permitted');
+
+  // Parse for validity and policy comparisons, but never serialize with URL:
+  // WHATWG serialization would lowercase evidence-bearing scheme/host text.
+  const parsed = new URL(withoutFragment);
+  if (parsed.username || parsed.password) throw new TypeError('URL credentials are not permitted');
+  let authority = rawAuthority;
+  const defaultPort = scheme === 'https' ? '443' : '80';
+  const portPattern = new RegExp(`:${defaultPort}$`);
+  if (portPattern.test(authority)) authority = authority.replace(portPattern, '');
+  return `${rawScheme}://${authority}${suffix}`;
+}
+
+function assertSameOrigin(candidate, allowedOrigin, label) {
+  if (new URL(candidate).origin !== allowedOrigin) {
+    throw new Error(`${label} origin differs from requested URL origin`);
+  }
+}
+
+function identityPart(url, baseUrl) {
+  if (url == null || url === '') return null;
+  const raw = String(url);
+  const absolute = baseUrl == null || /^[A-Za-z][A-Za-z\d+.-]*:\/\//.test(raw)
+    ? raw
+    : new URL(raw, baseUrl).href;
+  return { url: absolute, normalized_url: normalizeUrlIdentity(absolute) };
+}
+
+function optionalIdentityPart(url, baseUrl) {
+  if (url == null || url === '') return null;
+  try {
+    return identityPart(url, baseUrl);
+  } catch {
+    // Preserve an invalid declaration for downstream detectors instead of
+    // turning URL metadata enrichment into a page extraction failure.
+    return { url: String(url), normalized_url: null };
+  }
+}
+
+/** Build a provenance-preserving identity without treating canonical as fetch identity. */
+export function createUrlIdentity({
+  requestedUrl,
+  effectiveUrl,
+  redirectChain = [],
+  declaredCanonicalUrl = null,
+} = {}) {
+  const requested = identityPart(requestedUrl);
+  const effective = identityPart(effectiveUrl);
+  if (!requested || !effective) throw new TypeError('requestedUrl and effectiveUrl are required');
+  const allowedOrigin = new URL(requested.url).origin;
+  assertSameOrigin(effective.url, allowedOrigin, 'effective URL');
+
+  const redirect = (redirectChain || []).map((step) => {
+    const source = identityPart(step.url);
+    if (!source) throw new TypeError('redirect URL is required');
+    assertSameOrigin(source.url, allowedOrigin, 'redirect URL');
+    const destination = step.location == null ? null : identityPart(step.location, source.url);
+    if (destination) assertSameOrigin(destination.url, allowedOrigin, 'redirect destination');
+    return {
+      url: source.url,
+      normalized_url: source.normalized_url,
+      status: step.status,
+      location: step.location ?? null,
+      destination_url: destination?.url ?? null,
+      normalized_destination_url: destination?.normalized_url ?? null,
+    };
+  });
+  const canonical = optionalIdentityPart(declaredCanonicalUrl, effective.url);
+
+  return {
+    normalization_version: URL_IDENTITY_NORMALIZATION_VERSION,
+    requested,
+    effective,
+    redirect,
+    canonical,
+    resource_id: `RESOURCE-${sha256(effective.normalized_url).slice(0, 24).toUpperCase()}`,
+  };
+}

package/src/detectors/agent.js

@@ -50,6 +50,7 @@ function siteUrl(ctx) {
 // ---------------------------------------------------------------------------
 
 export const AGENT_001 = defineDetector({
+  coverage_requirement: 'exhaustive_requested_scope',
   id: 'AGENT-001',
   name: 'AI bot rules absent from robots.txt',
   namespace: 'AGENT',
@@ -94,6 +95,7 @@ export const AGENT_001 = defineDetector({
 // ---------------------------------------------------------------------------
 
 export const AGENT_002 = defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'AGENT-002',
   name: 'Link response headers absent',
   namespace: 'AGENT',
@@ -131,6 +133,7 @@ export const AGENT_002 = defineDetector({
 // ---------------------------------------------------------------------------
 
 export const AGENT_003 = defineDetector({
+  coverage_requirement: 'exhaustive_requested_scope',
   id: 'AGENT-003',
   name: 'llms.txt missing',
   namespace: 'AGENT',
@@ -169,6 +172,7 @@ export const AGENT_003 = defineDetector({
 // ---------------------------------------------------------------------------
 
 export const AGENT_004 = defineDetector({
+  coverage_requirement: 'exhaustive_requested_scope',
   id: 'AGENT-004',
   name: 'MCP Server Card not discoverable',
   namespace: 'AGENT',
@@ -215,6 +219,7 @@ export const AGENT_004 = defineDetector({
 // ---------------------------------------------------------------------------
 
 export const AGENT_005 = defineDetector({
+  coverage_requirement: 'exhaustive_requested_scope',
   id: 'AGENT-005',
   name: 'A2A Agent Card not discoverable',
   namespace: 'AGENT',
@@ -258,6 +263,7 @@ export const AGENT_005 = defineDetector({
 // ---------------------------------------------------------------------------
 
 export const AGENT_006 = defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'AGENT-006',
   name: 'Markdown content negotiation not supported',
   namespace: 'AGENT',
@@ -302,6 +308,7 @@ export const AGENT_006 = defineDetector({
 // ---------------------------------------------------------------------------
 
 export const AGENT_007 = defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'AGENT-007',
   name: 'Web Bot Auth not declared',
   namespace: 'AGENT',
@@ -345,6 +352,7 @@ export const AGENT_007 = defineDetector({
 // ---------------------------------------------------------------------------
 
 export const AGENT_008 = defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'AGENT-008',
   name: 'Content-Signals header absent',
   namespace: 'AGENT',
@@ -386,6 +394,7 @@ export const AGENT_008 = defineDetector({
 // ---------------------------------------------------------------------------
 
 export const AGENT_009 = defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'AGENT-009',
   name: 'auth.md not present',
   namespace: 'AGENT',
@@ -431,6 +440,7 @@ export const AGENT_009 = defineDetector({
 // ---------------------------------------------------------------------------
 
 export const AGENT_010 = defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'AGENT-010',
   name: 'Agentic commerce protocols not declared',
   namespace: 'AGENT',
@@ -488,6 +498,7 @@ export const AGENT_010 = defineDetector({
 // ---------------------------------------------------------------------------
 
 export const AGENT_011 = defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'AGENT-011',
   name: 'llms.txt structure or link integrity broken',
   namespace: 'AGENT',

package/src/detectors/ans.js

@@ -6,6 +6,7 @@ const D = [];
 const PREAMBLE_RX = /\b(in today'?s (rapidly |ever[- ])?(evolving|changing)|in the (modern|current|digital) (era|age|landscape|world)|now more than ever|as (technology|ai) continues to (evolve|advance)|in an increasingly)\b/i;
 
 D.push(defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'ANS-001', name: 'Generic preamble before answer', namespace: 'ANS',
   description: 'The opening paragraph uses generic scene-setting boilerplate instead of answering the page’s question.',
   discipline: ['aeo', 'geo'], severity: 'medium', deterministic: false, requires: ['site'],
@@ -26,6 +27,7 @@ D.push(defineDetector({
 }));
 
 D.push(defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'ANS-002', name: 'Question heading without direct answer', namespace: 'ANS',
   description: 'A heading phrased as a question is not followed by prose before the next heading.',
   discipline: ['aeo'], severity: 'medium', deterministic: true, requires: ['site'],
@@ -53,6 +55,7 @@ D.push(defineDetector({
 }));
 
 D.push(defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'ANS-003', name: 'Circular definition', namespace: 'ANS',
   description: 'A definitional sentence defines a term using the term itself as the definition head.',
   discipline: ['aeo', 'geo'], severity: 'medium', deterministic: false, requires: ['site'],
@@ -85,6 +88,7 @@ D.push(defineDetector({
 }));
 
 D.push(defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'ANS-004', name: 'Deictic dependency in answer text', namespace: 'ANS',
   description: 'Answer-bearing prose depends on preceding visual context ("as shown above", "in the diagram below"), which breaks when extracted.',
   discipline: ['aeo'], severity: 'low', deterministic: true, requires: ['site'],
@@ -108,6 +112,7 @@ D.push(defineDetector({
 }));
 
 D.push(defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'ANS-005', name: 'Relative quantity without baseline or timeframe', namespace: 'ANS',
   description: 'Text asserts a relative numeric improvement ("3x faster", "50% reduction") without a nearby baseline, unit, or timeframe.',
   discipline: ['aeo', 'geo'], severity: 'medium', deterministic: false, requires: ['site'],
@@ -138,6 +143,7 @@ D.push(defineDetector({
 }));
 
 D.push(defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'ANS-006', name: 'Procedure content without ordered steps', namespace: 'ANS',
   description: 'A page registered as implementation/how-to content contains no ordered list or numbered steps.',
   discipline: ['aeo'], severity: 'low', deterministic: true, requires: ['site', 'registries'],
@@ -164,6 +170,7 @@ D.push(defineDetector({
 }));
 
 D.push(defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'ANS-007', name: 'Comparison without explicit basis', namespace: 'ANS',
   description: 'A comparison page has neither a comparison table nor prose stating the comparison criteria.',
   discipline: ['aeo', 'geo'], severity: 'medium', deterministic: true, requires: ['site', 'registries'],
@@ -190,6 +197,7 @@ D.push(defineDetector({
 }));
 
 D.push(defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'ANS-008', name: 'Missing scope or limitations on answer page', namespace: 'ANS',
   description: 'A definition/comparison/recommendation page never states scope boundaries, exclusions, or limitations.',
   discipline: ['aeo', 'geo'], severity: 'medium', deterministic: false, requires: ['site', 'registries'],
@@ -218,6 +226,7 @@ D.push(defineDetector({
 }));
 
 D.push(defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'ANS-009', name: 'Answer-target page has no prompt coverage', namespace: 'ANS',
   description: 'An answer-bearing page is intended for discovery but is not mapped to any prompt in the governed prompt corpus.',
   discipline: ['aeo', 'geo'], severity: 'medium', deterministic: true, requires: ['site', 'registries'],
@@ -240,6 +249,7 @@ D.push(defineDetector({
 }));
 
 D.push(defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'ANS-010', name: 'Published claims lack page evidence mapping', namespace: 'ANS',
   description: 'A page publishes governed claims but records no page-level evidence references, preventing evidence-adjacency review.',
   discipline: ['aeo', 'geo'], severity: 'high', deterministic: true, requires: ['site', 'registries'],
@@ -261,6 +271,7 @@ D.push(defineDetector({
 }));
 
 D.push(defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'ANS-011', name: 'Over-diluted answer passage (missing direct answer lead)', namespace: 'ANS',
   description: 'A section targeting an informational query or question heading lacks a concise, direct answer passage in its opening sentences, burying key factual answers behind excessive preamble or rambling prose.',
   discipline: ['aeo', 'geo'], severity: 'medium', deterministic: false, requires: ['site'],
@@ -320,6 +331,7 @@ D.push(defineDetector({
 }));
 
 D.push(defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'ANS-012', name: 'Ungrounded quantitative metric in answer passage', namespace: 'ANS',
   description: 'An answer-bearing passage asserts high-stakes quantitative metrics, percentages, or multiples without adjacent evidence citation, benchmark reference, or registered evidence link.',
   discipline: ['aeo', 'geo'], severity: 'medium', deterministic: false, requires: ['site'],
@@ -359,6 +371,7 @@ D.push(defineDetector({
 }));
 
 D.push(defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'ANS-013', name: 'Comparative or procedural section lacking structured table or ordered sequence', namespace: 'ANS',
   description: 'A section covering comparative evaluation (vs, alternatives, comparison) or multi-step execution (how to, steps) lacks structured <table>, <ol>, or <ul> markup, diminishing answer engine extraction fidelity.',
   discipline: ['aeo', 'seo'], severity: 'medium', deterministic: true, requires: ['site'],
@@ -420,6 +433,7 @@ D.push(defineDetector({
 }));
 
 D.push(defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'ANS-014', name: 'Low information-gain fluff ratio in answer prose', namespace: 'ANS',
   description: 'An answer passage contains excessive rhetorical filler, clichés, or conversational padding, reducing factual density for generative engine extraction.',
   discipline: ['aeo', 'geo'], severity: 'low', deterministic: true, requires: ['site'],
@@ -467,6 +481,7 @@ D.push(defineDetector({
 }));
 
 D.push(defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'ANS-015', name: 'Definitional page lacks direct copular definition', namespace: 'ANS',
   description: 'A page with definitional intent (page_type=definition, /definition/, /glossary/, or "What is" heading) does not provide a direct copular definition ("X is a Y that Z") in the opening paragraph.',
   discipline: ['aeo', 'geo'], severity: 'medium', deterministic: true, requires: ['site'],
@@ -506,6 +521,7 @@ D.push(defineDetector({
 }));
 
 D.push(defineDetector({
+  coverage_requirement: 'evaluated_subset',
   id: 'ANS-016', name: 'Comprehensive long-form guide lacks executive summary or key takeaways', namespace: 'ANS',
   description: 'An in-depth article or guide exceeding 1,200 words does not provide an executive summary, key takeaways, or TL;DR block, increasing RAG chunk fragmentation and extraction ambiguity.',
   discipline: ['aeo', 'geo'], severity: 'low', deterministic: true, requires: ['site'],
@@ -541,4 +557,3 @@ D.push(defineDetector({
 }));
 
 export default D;
-