@nebulacomponents/citable 1.19.0 → 1.20.0

package/src/connectors/wordpress.js

@@ -1,5 +1,6 @@
 import { providerRequest } from './http.js';
 import { sha256, nowIso } from '../shared/io.js';
+import { collectionResult, errorMessage, paginationBoundary } from './collectionResult.js';
 
 const METRICS = {
   published_posts: { unit: 'count', value_type: 'integer' },
@@ -25,6 +26,53 @@ function computeCmsPayloadHash(payload) {
   return sha256(JSON.stringify(canonical));
 }
 
+async function collectWordPressEndpoint(base, endpoint, context) {
+  const maxPages = paginationBoundary(context);
+  const items = [];
+  const limitations = [];
+  const errors = [];
+  let providerTotal = null;
+  let totalPages = null;
+  let page = 1;
+  let pagesRequested = 0;
+  let pagesRetrieved = 0;
+  let continuationState = null;
+  let providerCompleteness = 'declared';
+  let connectorError = null;
+
+  while (page <= maxPages) {
+    pagesRequested += 1;
+    const suffix = page === 1 ? '?per_page=100&status=publish' : `?per_page=100&status=publish&page=${page}`;
+    try {
+      const response = await providerRequest(`${base}/wp-json/wp/v2/${endpoint}${suffix}`, { ...context, rawResponse: true });
+      const pageItems = await response.json();
+      if (!Array.isArray(pageItems)) throw new Error(`WordPress ${endpoint} page ${page} did not return an array`);
+      pagesRetrieved += 1;
+      items.push(...pageItems.map((item) => ({ ...item, _collection_endpoint: endpoint })));
+      const headerTotal = Number(response.headers.get('x-wp-total'));
+      const headerPages = Number(response.headers.get('x-wp-totalpages'));
+      if (Number.isInteger(headerTotal) && headerTotal >= 0) providerTotal = headerTotal;
+      if (Number.isInteger(headerPages) && headerPages >= 1) totalPages = headerPages;
+      if (!totalPages) providerCompleteness = 'unknown';
+      if (pageItems.length === 0 || (totalPages && page >= totalPages)) break;
+      if (!totalPages && pageItems.length >= 100) {
+        continuationState = { endpoint, page: page + 1 };
+        limitations.push('WordPress did not report total pages for a full page of results.');
+        break;
+      }
+      page += 1;
+    } catch (error) {
+      connectorError ||= error;
+      errors.push(`${endpoint} page ${page}: ${errorMessage(error)}`);
+      continuationState = { endpoint, page };
+      break;
+    }
+  }
+  if (!continuationState && totalPages && page < totalPages) continuationState = { endpoint, page };
+  if (continuationState && page > maxPages) limitations.push(`WordPress ${endpoint} collection reached the ${maxPages}-page boundary.`);
+  return { items, providerTotal, providerCompleteness, connectorError, pagesRequested, pagesRetrieved, continuationState, limitations, errors };
+}
+
 export const wordpressConnector = {
   provider: 'wordpress',
   defaultCredentialEnv: 'WORDPRESS_AUTH_TOKEN',
@@ -78,35 +126,71 @@ export const wordpressConnector = {
     const rows = [];
     const metricNames = new Set(metrics.map((m) => m.external_name));
 
+    const collections = [];
+    const collectionErrors = [];
+    const collectionLimitations = [];
+    let pagesRequested = 0;
+    let pagesRetrieved = 0;
+    let continuationState = null;
+    let providerTotal = 0;
+    let providerTotalKnown = true;
+    let connectorError = null;
+
     if (metricNames.has('published_posts')) {
-      const posts = await providerRequest(`${base}/wp-json/wp/v2/posts?per_page=100&status=publish`, context);
+      const postsResult = await collectWordPressEndpoint(base, 'posts', context);
+      collections.push(...postsResult.items);
+      collectionErrors.push(...postsResult.errors);
+      collectionLimitations.push(...postsResult.limitations);
+      pagesRequested += postsResult.pagesRequested;
+      pagesRetrieved += postsResult.pagesRetrieved;
+      continuationState ||= postsResult.continuationState;
+      connectorError ||= postsResult.connectorError;
+      if (postsResult.providerCompleteness === 'unknown') providerTotalKnown = false;
+      if (postsResult.providerTotal === null) providerTotalKnown = false; else providerTotal += postsResult.providerTotal;
       const postMetric = metrics.find((m) => m.external_name === 'published_posts');
       rows.push({
         metric: postMetric,
-        value: Array.isArray(posts) ? posts.length : 0,
+        value: postsResult.items.length,
         dimensions: { date: endDate },
         observed_at: `${endDate}T00:00:00.000Z`,
       });
     }
 
     if (metricNames.has('published_pages')) {
-      const pages = await providerRequest(`${base}/wp-json/wp/v2/pages?per_page=100&status=publish`, context);
+      const pagesResult = await collectWordPressEndpoint(base, 'pages', context);
+      collections.push(...pagesResult.items);
+      collectionErrors.push(...pagesResult.errors);
+      collectionLimitations.push(...pagesResult.limitations);
+      pagesRequested += pagesResult.pagesRequested;
+      pagesRetrieved += pagesResult.pagesRetrieved;
+      continuationState ||= pagesResult.continuationState;
+      connectorError ||= pagesResult.connectorError;
+      if (pagesResult.providerCompleteness === 'unknown') providerTotalKnown = false;
+      if (pagesResult.providerTotal === null) providerTotalKnown = false; else providerTotal += pagesResult.providerTotal;
       const pageMetric = metrics.find((m) => m.external_name === 'published_pages');
       rows.push({
         metric: pageMetric,
-        value: Array.isArray(pages) ? pages.length : 0,
+        value: pagesResult.items.length,
         dimensions: { date: endDate },
         observed_at: `${endDate}T00:00:00.000Z`,
       });
     }
 
+    const collection = collectionResult({
+      items: collections,
+      paginationState: { pages_requested: pagesRequested, pages_retrieved: pagesRetrieved, boundary: { max_pages: paginationBoundary(context) } },
+      providerReportedTotal: providerTotalKnown ? providerTotal : null,
+      continuationState,
+      providerCompleteness: providerTotalKnown ? 'declared' : 'unknown',
+      limitations: [...collectionLimitations, 'WordPress REST API collection is bounded to accessible posts and pages.', 'Custom post types outside standard posts and pages require dedicated endpoint parameters.'],
+      errors: collectionErrors,
+    });
     return {
       rows,
       cursor: endDate,
-      limitations: [
-        'WordPress REST API collection is bounded to accessible posts and pages.',
-        'Custom post types outside standard posts and pages require dedicated endpoint parameters.',
-      ],
+      limitations: collection.limitations,
+      collection,
+      ...(connectorError ? { connectorError } : {}),
     };
   },
 

package/src/crawler/fetch.js

@@ -1,7 +1,12 @@
 import dns from 'node:dns';
+import http from 'node:http';
+import https from 'node:https';
 import net from 'node:net';
+import { Readable } from 'node:stream';
+import { createBrotliDecompress, createGunzip, createInflate } from 'node:zlib';
 
 const blockedAddresses = new net.BlockList();
+const GUARDED_LOOKUP_TRANSPORT = Symbol('citable.guardedLookupTransport');
 for (const [network, prefix] of [
   ['0.0.0.0', 8], ['10.0.0.0', 8], ['100.64.0.0', 10], ['127.0.0.0', 8],
   ['169.254.0.0', 16], ['172.16.0.0', 12], ['192.0.0.0', 24], ['192.0.2.0', 24],
@@ -22,18 +27,89 @@ function isBlockedAddress(address) {
 }
 
 export async function validatePublicUrl(url, { lookup = dns.promises.lookup } = {}) {
+  const parsed = parseNetworkUrl(url);
+  await resolvePublicAddresses(parsed.hostname, lookup);
+  return parsed;
+}
+
+function parseNetworkUrl(url) {
   const parsed = new URL(url);
   if (!['http:', 'https:'].includes(parsed.protocol)) throw new Error(`unsupported URL protocol: ${parsed.protocol}`);
   if (parsed.username || parsed.password) throw new Error('URL credentials are not permitted');
-  const hostname = parsed.hostname.replace(/^\[|\]$/g, '');
+  return parsed;
+}
+
+async function resolvePublicAddresses(rawHostname, lookup, options = {}) {
+  const hostname = rawHostname.replace(/^\[|\]$/g, '');
   const literalFamily = net.isIP(hostname);
   const addresses = literalFamily
     ? [{ address: hostname, family: literalFamily }]
-    : await lookup(hostname, { all: true, verbatim: true });
+    : await lookup(hostname, { ...options, all: true, verbatim: true });
   if (!addresses.length || addresses.some(({ address }) => isBlockedAddress(address))) {
     throw new Error(`refusing private, loopback, or non-public destination: ${hostname}`);
   }
-  return parsed;
+  return addresses;
+}
+
+function createGuardedLookup(lookup) {
+  return (hostname, options, callback) => {
+    const normalizedOptions = typeof options === 'number' ? { family: options } : (options ?? {});
+    resolvePublicAddresses(hostname, lookup, { family: normalizedOptions.family || 0 })
+      .then((addresses) => {
+        if (normalizedOptions.all) callback(null, addresses);
+        else callback(null, addresses[0].address, addresses[0].family);
+      })
+      .catch((error) => callback(error));
+  };
+}
+
+function fetchWithGuardedLookup(url, { headers, signal, lookup }) {
+  const parsed = new URL(url);
+  const transport = parsed.protocol === 'https:' ? https : http;
+  return new Promise((resolve, reject) => {
+    const request = transport.request(parsed, {
+      method: 'GET',
+      headers,
+      lookup,
+      servername: parsed.protocol === 'https:' ? parsed.hostname : undefined,
+      signal,
+    }, (response) => {
+      const responseHeaders = new Headers();
+      for (const [name, value] of Object.entries(response.headers)) {
+        if (Array.isArray(value)) for (const item of value) responseHeaders.append(name, item);
+        else if (value !== undefined) responseHeaders.set(name, value);
+      }
+      const encoding = String(response.headers['content-encoding'] ?? '').toLowerCase();
+      let body = response;
+      if (encoding === 'gzip' || encoding === 'x-gzip') body = response.pipe(createGunzip());
+      else if (encoding === 'deflate') body = response.pipe(createInflate());
+      else if (encoding === 'br') body = response.pipe(createBrotliDecompress());
+      const noBody = response.statusCode === 204 || response.statusCode === 205 || response.statusCode === 304;
+      resolve(new Response(noBody ? null : Readable.toWeb(body), {
+        status: response.statusCode,
+        statusText: response.statusMessage,
+        headers: responseHeaders,
+      }));
+    });
+    request.on('error', reject);
+    request.end();
+  });
+}
+
+/**
+ * Declare that a custom transport uses the `lookup` callback passed by fetchUrl
+ * for its actual socket connection. Undeclared transports are rejected before
+ * execution because URL preflight alone does not prevent DNS rebinding.
+ */
+export function declareGuardedLookupTransport(fetchImpl) {
+  if (typeof fetchImpl !== 'function') throw new TypeError('custom transport must be a function');
+  Object.defineProperty(fetchImpl, GUARDED_LOOKUP_TRANSPORT, {
+    value: true,
+    enumerable: false,
+    configurable: false,
+    writable: false,
+  });
+  return fetchImpl;
 }
 
 /** Enforce the public-network policy for each Playwright request. */
@@ -62,12 +138,12 @@ export async function handlePublicBrowserRoute(route, { lookup = dns.promises.lo
   }
 }
 
-async function readBodyLimited(res, maxBodyBytes) {
+async function readBodyLimited(res, maxBodyBytes, responseType = 'text') {
   const declared = Number(res.headers.get('content-length'));
   if (Number.isFinite(declared) && declared > maxBodyBytes) {
-    throw new Error(`response body exceeds ${maxBodyBytes} bytes`);
+    throw bodyLimitError(maxBodyBytes);
   }
-  if (!res.body) return '';
+  if (!res.body) return { body: responseType === 'buffer' ? Buffer.alloc(0) : '', bodyBytes: Buffer.alloc(0) };
   const reader = res.body.getReader();
   const chunks = [];
   let total = 0;
@@ -76,7 +152,7 @@ async function readBodyLimited(res, maxBodyBytes) {
       const { done, value } = await reader.read();
       if (done) break;
       total += value.byteLength;
-      if (total > maxBodyBytes) throw new Error(`response body exceeds ${maxBodyBytes} bytes`);
+      if (total > maxBodyBytes) throw bodyLimitError(maxBodyBytes);
       chunks.push(value);
     }
   } finally {
@@ -85,42 +161,127 @@ async function readBodyLimited(res, maxBodyBytes) {
   const bytes = new Uint8Array(total);
   let offset = 0;
   for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; }
-  return new TextDecoder().decode(bytes);
+  const bodyBytes = Buffer.from(bytes);
+  return { body: responseType === 'buffer' ? bodyBytes : new TextDecoder().decode(bytes), bodyBytes };
+}
+
+function bodyLimitError(maxBodyBytes) {
+  const error = new Error(`response body exceeds ${maxBodyBytes} bytes`);
+  error.code = 'FETCH_BODY_LIMIT';
+  return error;
+}
+
+function timeoutError(timeoutMs) {
+  const error = new Error(`request timeout after ${timeoutMs}ms`);
+  error.code = 'FETCH_TIMEOUT';
+  return error;
+}
+
+function errorCode(error) {
+  const code = typeof error?.code === 'string' && error.code ? error.code : null;
+  if (code && /^[A-Z][A-Z0-9_]{0,63}$/.test(code)) return code;
+  if (error?.name === 'AbortError') return 'FETCH_ABORTED';
+  return 'FETCH_ERROR';
+}
+
+function attemptUrl(url) {
+  const parsed = new URL(url);
+  parsed.hash = '';
+  if (parsed.search) parsed.search = '?[REDACTED]';
+  return parsed.href;
+}
+
+function isRetryableResponseStatus(status) {
+  return status === 500 || status === 502 || status === 503 || status === 504;
+}
+
+function codedFetchError(message, code) {
+  const error = new Error(message);
+  error.code = code;
+  return error;
+}
+
+function attachAttempts(error, attempts) {
+  const target = error instanceof Error ? error : new Error('request failed');
+  Object.defineProperty(target, 'attempts', {
+    value: attempts.map((attempt) => ({ ...attempt })),
+    enumerable: true,
+    configurable: true,
+  });
+  return target;
 }
 
 /** Fetch a public URL with bounded retries, redirects, time, and response size. */
 export async function fetchUrl(url, {
   userAgent = 'CitableAudit/0.1', maxRedirects = 10, timeoutMs = 20000,
   maxRetries = 3, retryDelayMs = 1000, maxBodyBytes = 5 * 1024 * 1024,
-  fetchImpl = globalThis.fetch, lookup = dns.promises.lookup,
+  responseType = 'text',
+  fetchImpl, lookup = dns.promises.lookup,
+  allowUnsafeCustomTransportForTest = false,
 } = {}) {
-  const requested = await validatePublicUrl(url, { lookup });
+  if (!['text', 'buffer'].includes(responseType)) throw new TypeError('responseType must be text or buffer');
+  const requested = parseNetworkUrl(url);
   const allowedOrigin = requested.origin;
   const chain = [];
+  const attempts = [];
+  let attemptNumber = 0;
   let current = requested.href;
+  const customFetch = fetchImpl && fetchImpl !== globalThis.fetch;
+  if (customFetch && allowUnsafeCustomTransportForTest !== true && fetchImpl[GUARDED_LOOKUP_TRANSPORT] !== true) {
+    throw new TypeError('custom transport must declare and implement the guarded lookup contract');
+  }
+  const effectiveFetch = customFetch ? fetchImpl : fetchWithGuardedLookup;
 
   for (let redirects = 0; redirects <= maxRedirects; redirects += 1) {
-    await validatePublicUrl(current, { lookup });
     let res;
     let lastError;
+    let responseStartedAtMs = null;
     for (let attempt = 0; attempt < Math.max(1, maxRetries); attempt += 1) {
+      attemptNumber += 1;
+      const startedAtMs = performance.now();
+      try {
+        await validatePublicUrl(current, { lookup });
+      } catch (error) {
+        const endedAtMs = performance.now();
+        attempts.push({
+          attempt: attemptNumber, url: attemptUrl(current), startedAtMs, endedAtMs,
+          elapsedMs: Math.max(0, endedAtMs - startedAtMs), outcome: 'error', status: null,
+          errorCode: errorCode(error), retryDecision: 'stop',
+        });
+        throw attachAttempts(error, attempts);
+      }
       const controller = new AbortController();
-      const timer = setTimeout(() => controller.abort(new Error(`request timeout after ${timeoutMs}ms`)), timeoutMs);
+      const timer = setTimeout(() => controller.abort(timeoutError(timeoutMs)), timeoutMs);
       try {
-        res = await fetchImpl(current, {
+        res = await effectiveFetch(current, {
           redirect: 'manual',
           headers: { 'user-agent': userAgent },
           signal: controller.signal,
+          lookup: createGuardedLookup(lookup),
         });
-        if (res.status < 500 || attempt === Math.max(1, maxRetries) - 1) {
+        const retry = isRetryableResponseStatus(res.status) && attempt < Math.max(1, maxRetries) - 1;
+        if (!retry) {
           lastError = null;
+          responseStartedAtMs = startedAtMs;
           break;
         }
+        const endedAtMs = performance.now();
+        attempts.push({
+          attempt: attemptNumber, url: attemptUrl(current), startedAtMs, endedAtMs,
+          elapsedMs: Math.max(0, endedAtMs - startedAtMs), outcome: 'response',
+          status: res.status, errorCode: null, retryDecision: 'retry',
+        });
         await res.body?.cancel();
         lastError = new Error(`server returned ${res.status}`);
       } catch (err) {
-        lastError = err;
-        if (controller.signal.aborted) throw controller.signal.reason ?? err;
+        lastError = controller.signal.aborted ? (controller.signal.reason ?? err) : err;
+        const retry = attempt < Math.max(1, maxRetries) - 1;
+        const endedAtMs = performance.now();
+        attempts.push({
+          attempt: attemptNumber, url: attemptUrl(current), startedAtMs, endedAtMs,
+          elapsedMs: Math.max(0, endedAtMs - startedAtMs), outcome: 'error', status: null,
+          errorCode: errorCode(lastError), retryDecision: retry ? 'retry' : 'stop',
+        });
       } finally {
         clearTimeout(timer);
       }
@@ -129,21 +290,56 @@ export async function fetchUrl(url, {
         await new Promise((resolve) => setTimeout(resolve, delay));
       }
     }
-    if (lastError) throw lastError;
+    if (lastError) throw attachAttempts(lastError, attempts);
 
     const headers = Object.fromEntries(res.headers.entries());
     if (res.status >= 300 && res.status < 400 && headers.location) {
+      const endedAtMs = performance.now();
+      attempts.push({
+        attempt: attemptNumber, url: attemptUrl(current), startedAtMs: responseStartedAtMs, endedAtMs,
+        elapsedMs: Math.max(0, endedAtMs - responseStartedAtMs), outcome: 'response',
+        status: res.status, errorCode: null,
+        retryDecision: redirects === maxRedirects ? 'stop' : 'redirect',
+      });
       if (redirects === maxRedirects) break;
-      const next = new URL(headers.location, current);
-      if (next.origin !== allowedOrigin) throw new Error(`redirect leaves audited origin: ${next.href}`);
-      await validatePublicUrl(next.href, { lookup });
-      chain.push({ url: current, status: res.status, location: headers.location });
+      let next;
+      try {
+        next = new URL(headers.location, current);
+      } catch {
+        throw attachAttempts(codedFetchError('invalid redirect location', 'FETCH_REDIRECT_INVALID'), attempts);
+      }
+      if (next.origin !== allowedOrigin) {
+        throw attachAttempts(codedFetchError('redirect leaves audited origin', 'FETCH_REDIRECT_ORIGIN'), attempts);
+      }
+      chain.push({ url: attemptUrl(current), status: res.status, location: attemptUrl(next.href) });
       current = next.href;
       await res.body?.cancel();
       continue;
     }
-    const body = await readBodyLimited(res, maxBodyBytes);
-    return { url: current, requestedUrl: url, status: res.status, headers, body, redirectChain: chain };
+    try {
+      const { body, bodyBytes } = await readBodyLimited(res, maxBodyBytes, responseType);
+      const endedAtMs = performance.now();
+      attempts.push({
+        attempt: attemptNumber, url: attemptUrl(current), startedAtMs: responseStartedAtMs, endedAtMs,
+        elapsedMs: Math.max(0, endedAtMs - responseStartedAtMs), outcome: 'response',
+        status: res.status, errorCode: null, retryDecision: res.status >= 500 ? 'stop' : 'complete',
+      });
+      return {
+        url: current, requestedUrl: url, status: res.status, headers, body, bodyBytes,
+        bodyComplete: true, redirectChain: chain, attempts,
+      };
+    } catch (error) {
+      const endedAtMs = performance.now();
+      attempts.push({
+        attempt: attemptNumber, url: attemptUrl(current), startedAtMs: responseStartedAtMs, endedAtMs,
+        elapsedMs: Math.max(0, endedAtMs - responseStartedAtMs), outcome: 'error', status: res.status,
+        errorCode: errorCode(error), retryDecision: 'stop',
+      });
+      throw attachAttempts(error, attempts);
+    }
   }
-  throw new Error(`redirect chain exceeded ${maxRedirects} hops for ${url}`);
+  throw attachAttempts(codedFetchError(
+    `redirect chain exceeded ${maxRedirects} hops`,
+    'FETCH_REDIRECT_LIMIT',
+  ), attempts);
 }

package/src/crawler/resourceValidity.js

@@ -0,0 +1,119 @@
+const PATTERN_SCAN_MAX_CHARS = 256 * 1024;
+const MIN_SUBSTANTIVE_TEXT_CHARS = 40;
+
+function headerValue(headers, name) {
+  if (headers instanceof Headers) return headers.get(name) || '';
+  const target = name.toLowerCase();
+  for (const [key, value] of Object.entries(headers || {})) {
+    if (key.toLowerCase() === target) return Array.isArray(value) ? value.join(', ') : String(value ?? '');
+  }
+  return '';
+}
+
+function bodyString(body) {
+  if (body == null) return '';
+  if (Buffer.isBuffer(body) || body instanceof Uint8Array) return Buffer.from(body).toString('utf8');
+  return String(body);
+}
+
+function bodyByteLength(body, converted) {
+  if (Buffer.isBuffer(body) || body instanceof Uint8Array) return body.byteLength;
+  return Buffer.byteLength(converted);
+}
+
+function visibleText(html) {
+  const bodyMatch = html.match(/<body\b[^>]*>([\s\S]*?)<\/body\s*>/i);
+  return (bodyMatch?.[1] ?? html)
+    .replace(/<script\b[^>]*>[\s\S]*?(?:<\/script\s*>|$)/gi, ' ')
+    .replace(/<style\b[^>]*>[\s\S]*?(?:<\/style\s*>|$)/gi, ' ')
+    .replace(/<!--[^]*?-->/g, ' ')
+    .replace(/<[^>]*>/g, ' ')
+    .replace(/&(?:nbsp|#160);/gi, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function hasUnclosedRawTextElement(sample) {
+  const lower = sample.toLowerCase();
+  return ['script', 'style'].some((tag) => lower.lastIndexOf(`<${tag}`) > lower.lastIndexOf(`</${tag}`));
+}
+
+function scanWallSignals(sample) {
+  // These patterns intentionally require bounded, multi-word phrases or a
+  // title/heading context. A bare number (including 404), "login", or
+  // "cookies" is not evidence that the retrieved resource is a wall.
+  const challenge = /(?:<\s*(?:title|h1)[^>]*>[^<]{0,80}(?:just a moment|attention required|security check)|\b(?:verify (?:that )?you are human|complete the security check|checking your browser)\b)/i.test(sample);
+  const login = /(?:<\s*(?:title|h1)[^>]*>[^<]{0,80}(?:sign in required|login required)|\b(?:please )?(?:sign in|log in) to continue\b)/i.test(sample);
+  const consent = /(?:<\s*(?:title|h1)[^>]*>[^<]{0,80}before you continue|\b(?:accept|manage) (?:our )?(?:cookies|consent) to continue\b)/i.test(sample);
+  const soft404 = /<\s*(?:title|h1)[^>]*>\s*(?:(?:error\s*)?404(?:\s|<|[-:])|page (?:not found|does not exist)|not found\s*(?:<|$))/i.test(sample)
+    || /\b(?:the page|page you (?:requested|are looking for)) (?:does not exist|could not be found|is no longer available)\b/i.test(sample);
+  return { challenge, login, consent, soft404 };
+}
+
+/**
+ * Conservatively classify whether a response is usable by the HTML page
+ * evaluator. This does not classify sitemap documents or prove that a page is
+ * indexable. Wall/soft-404 heuristics only scan the first 256 Ki characters and
+ * result in `indeterminate`, never a claim that the page is definitively absent.
+ */
+export function classifyResource({
+  status,
+  headers = {},
+  body,
+  bodyComplete = true,
+  requestedUrl = null,
+  effectiveUrl = null,
+} = {}) {
+  const rawBody = bodyString(body);
+  const contentTypeHeader = headerValue(headers, 'content-type');
+  const contentType = contentTypeHeader.split(';', 1)[0].trim().toLowerCase() || null;
+  const sample = rawBody.slice(0, PATTERN_SCAN_MAX_CHARS);
+  const walls = scanWallSignals(sample);
+  const text = visibleText(sample);
+  const patternScanTruncated = rawBody.length > PATTERN_SCAN_MAX_CHARS;
+  const rawTextElementTruncated = patternScanTruncated && hasUnclosedRawTextElement(sample);
+  const signals = {
+    http_status: Number.isInteger(status) ? status : null,
+    content_type: contentType,
+    html_mime: contentType === 'text/html' || contentType === 'application/xhtml+xml',
+    body_complete: bodyComplete === true,
+    body_bytes: bodyByteLength(body, rawBody),
+    substantive_text_chars: text.length,
+    pattern_scan_max_chars: PATTERN_SCAN_MAX_CHARS,
+    pattern_scan_truncated: patternScanTruncated,
+    raw_text_element_truncated: rawTextElementTruncated,
+    challenge_wall: walls.challenge,
+    login_wall: walls.login,
+    consent_wall: walls.consent,
+    soft_404: walls.soft404,
+    requested_url: requestedUrl ?? null,
+    effective_url: effectiveUrl ?? null,
+  };
+
+  if (!Number.isInteger(status)) return { state: 'indeterminate', reason_codes: ['http_status_missing'], signals };
+  if (status < 200 || status >= 400) return { state: 'invalid_resource', reason_codes: [`http_status_${status}`], signals };
+  if (status >= 300) return { state: 'indeterminate', reason_codes: ['redirect_response'], signals };
+  if (bodyComplete !== true) return { state: 'indeterminate', reason_codes: ['body_truncated'], signals };
+  if (contentType && !signals.html_mime) return { state: 'invalid_resource', reason_codes: ['unexpected_mime'], signals };
+  if (!rawBody.trim()) return { state: 'indeterminate', reason_codes: ['empty_body'], signals };
+  if (!contentType) return { state: 'indeterminate', reason_codes: ['content_type_missing'], signals };
+  if (rawTextElementTruncated) {
+    return { state: 'indeterminate', reason_codes: ['pattern_scan_incomplete_markup'], signals };
+  }
+
+  const wallReasons = [];
+  if (walls.challenge) wallReasons.push('challenge_wall_signal');
+  if (walls.login) wallReasons.push('login_wall_signal');
+  if (walls.consent) wallReasons.push('consent_wall_signal');
+  if (walls.soft404) wallReasons.push('soft_404_signal');
+  if (wallReasons.length) return { state: 'indeterminate', reason_codes: wallReasons, signals };
+  if (text.length < MIN_SUBSTANTIVE_TEXT_CHARS) {
+    return { state: 'indeterminate', reason_codes: ['insubstantial_html'], signals };
+  }
+  return { state: 'valid_resource', reason_codes: ['substantive_html'], signals };
+}
+
+export const RESOURCE_VALIDITY_LIMITS = Object.freeze({
+  pattern_scan_max_chars: PATTERN_SCAN_MAX_CHARS,
+  minimum_substantive_text_chars: MIN_SUBSTANTIVE_TEXT_CHARS,
+});