@nebulacomponents/citable 1.18.2 → 1.20.0

package/src/extractor/site.js

@@ -3,7 +3,106 @@ import path from 'node:path';
 import { extractPage } from './page.js';
 import { parseRobots } from '../crawler/robots.js';
 import { parseSitemap } from '../crawler/sitemap.js';
+import { collectSitemapTopology } from '../crawler/sitemapCollector.js';
 import { fetchUrl } from '../crawler/fetch.js';
+import { classifyResource } from '../crawler/resourceValidity.js';
+import { createUrlIdentity } from '../crawler/urlIdentity.js';
+import { createCoverageLedger } from '../evidence/coverage.js';
+import { hashPageArtifact } from '../evidence/hashes.js';
+
+const MAX_PAGE_FETCH_ATTEMPTS = 100;
+const SAFE_FETCH_ERROR_CODES = new Set([
+  'FETCH_ABORTED', 'FETCH_BODY_LIMIT', 'FETCH_ERROR', 'FETCH_REDIRECT_INVALID',
+  'FETCH_REDIRECT_LIMIT', 'FETCH_REDIRECT_ORIGIN', 'FETCH_TIMEOUT',
+  'EAI_AGAIN', 'ECONNREFUSED', 'ECONNRESET', 'EHOSTUNREACH', 'ENETUNREACH',
+  'ENOTFOUND', 'EPIPE', 'ETIMEDOUT', 'UND_ERR_CONNECT_TIMEOUT',
+]);
+
+function sanitizeEvidenceUrl(rawUrl) {
+  try {
+    const url = new URL(String(rawUrl));
+    url.hash = '';
+    if (url.search) url.search = '?[REDACTED]';
+    return url.href;
+  } catch {
+    return '[invalid-url]';
+  }
+}
+
+function safeErrorCode(value, fallback = 'FETCH_ERROR') {
+  return typeof value === 'string' && SAFE_FETCH_ERROR_CODES.has(value) ? value : fallback;
+}
+
+function finiteNonNegative(value) {
+  return Number.isFinite(value) && value >= 0 ? value : null;
+}
+
+function sanitizeFetchAttempts(attempts) {
+  if (!Array.isArray(attempts)) return { attempts: [], truncated: false };
+  const bounded = attempts.slice(0, MAX_PAGE_FETCH_ATTEMPTS).map((rawAttempt, index) => {
+    const attempt = rawAttempt && typeof rawAttempt === 'object' ? rawAttempt : {};
+    const outcome = attempt.outcome === 'response' ? 'response' : 'error';
+    return {
+      attempt: Number.isInteger(attempt.attempt) && attempt.attempt > 0 ? attempt.attempt : index + 1,
+      url: sanitizeEvidenceUrl(attempt.url),
+      startedAtMs: finiteNonNegative(attempt.startedAtMs),
+      endedAtMs: finiteNonNegative(attempt.endedAtMs),
+      elapsedMs: finiteNonNegative(attempt.elapsedMs),
+      outcome,
+      status: Number.isInteger(attempt.status) && attempt.status >= 100 && attempt.status <= 599
+        ? attempt.status
+        : null,
+      errorCode: outcome === 'error' ? safeErrorCode(attempt.errorCode) : null,
+      retryDecision: ['retry', 'stop', 'redirect', 'complete'].includes(attempt.retryDecision)
+        ? attempt.retryDecision
+        : 'stop',
+    };
+  });
+  return { attempts: bounded, truncated: attempts.length > MAX_PAGE_FETCH_ATTEMPTS };
+}
+
+function sanitizedFetchError(rawUrl, error) {
+  const attempts = sanitizeFetchAttempts(error?.attempts).attempts;
+  const reason = safeErrorCode(error?.code, attempts.at(-1)?.errorCode ?? 'FETCH_ERROR');
+  return `${sanitizeEvidenceUrl(rawUrl)}: ${reason}`;
+}
+
+function sanitizedSitemapError(error) {
+  const reason = typeof error?.reason === 'string' && /^[a-z0-9_]{1,80}$/.test(error.reason)
+    ? error.reason
+    : 'sitemap_error';
+  return `${sanitizeEvidenceUrl(error?.url)}: ${reason}`;
+}
+
+function attachResourceMetadata(page, {
+  requestedUrl,
+  bodyComplete = true,
+  fetchAttempts = [],
+} = {}) {
+  const effectiveUrl = page.url;
+  const declaredCanonicalUrl = page.canonicals[0] ?? null;
+  page.requestedUrl = requestedUrl ?? effectiveUrl;
+  page.bodyComplete = bodyComplete === true;
+  const sanitizedAttempts = sanitizeFetchAttempts(fetchAttempts);
+  page.fetchAttempts = sanitizedAttempts.attempts;
+  page.fetchAttemptsTruncated = sanitizedAttempts.truncated;
+  page.resourceValidity = classifyResource({
+    status: page.status,
+    headers: page.headers,
+    body: page.rawHtml,
+    bodyComplete: page.bodyComplete,
+    requestedUrl: page.requestedUrl,
+    effectiveUrl,
+  });
+  page.urlIdentity = createUrlIdentity({
+    requestedUrl: page.requestedUrl,
+    effectiveUrl,
+    redirectChain: page.redirectChain,
+    declaredCanonicalUrl,
+  });
+  page.artifact_hash = hashPageArtifact(page);
+  return page;
+}
 
 /**
  * Build a SiteModel from a directory of built/static HTML output.
@@ -27,16 +126,17 @@ export function buildSiteFromDir(dir, { baseUrl = 'https://example.test' } = {})
         let urlPath = '/' + rel;
         if (urlPath.endsWith('/index.html')) urlPath = urlPath.slice(0, -'index.html'.length);
         const t = transport['/' + rel] || transport[urlPath] || {};
-        pages.push(
-          extractPage({
-            url: new URL(urlPath, baseUrl).href,
-            html: fs.readFileSync(p, 'utf8'),
-            status: t.status ?? 200,
-            headers: { 'content-type': 'text/html', ...(t.headers || {}) },
-            sourceFile: p,
-            redirectChain: [],
-          })
-        );
+        const responseBody = fs.readFileSync(p);
+        const page = extractPage({
+          url: new URL(urlPath, baseUrl).href,
+          html: responseBody.toString('utf8'),
+          responseBody,
+          status: t.status ?? 200,
+          headers: { 'content-type': 'text/html', ...(t.headers || {}) },
+          sourceFile: p,
+          redirectChain: [],
+        });
+        pages.push(attachResourceMetadata(page, { requestedUrl: page.url }));
       }
     }
   };
@@ -62,68 +162,216 @@ export function buildSiteFromDir(dir, { baseUrl = 'https://example.test' } = {})
 
 /** Build a SiteModel by fetching a deployed URL set (target URL + same-origin discovery, bounded). */
 export async function buildSiteFromUrl(startUrl, {
-  maxPages = 50, userAgent, pageMaxBytes = 5 * 1024 * 1024,
+  maxPages = 500, timeBudgetSeconds = 1800, userAgent, pageMaxBytes = 5 * 1024 * 1024,
+  // Collection is intentionally sequential. Reject unsupported values rather
+  // than silently changing reproducibility or budget semantics.
+  concurrency = 1,
   robotsMaxBytes = 512 * 1024, sitemapMaxBytes = 5 * 1024 * 1024,
+  sitemapMaxDepth = 4, sitemapMaxDocuments = 1000,
+  sitemapMaxUncompressedBytes = 20 * 1024 * 1024,
+  sitemapMaxTotalUncompressedBytes = 100 * 1024 * 1024,
+  sitemapMaxDiscoveredUrls = 50_000, sitemapMaxQueuedDocuments = 1000,
+  sitemapMaxRawEntries = 100_000, sitemapMaxXmlTokens = 500_000,
   fetcher = fetchUrl,
+  now = () => performance.now(),
 } = {}) {
+  if (concurrency !== 1) {
+    throw new RangeError(`URL collection concurrency is fixed at 1; received ${String(concurrency)}`);
+  }
   const origin = new URL(startUrl).origin;
-  const seen = new Set();
-  const queue = [startUrl];
+  const coverageLedger = createCoverageLedger({ startUrl, maxPages, timeBudgetSeconds });
+  const startedAt = now();
+  const timeExpired = () => now() - startedAt >= timeBudgetSeconds * 1000;
   const pages = [];
   const errors = [];
-  while (queue.length && pages.length < maxPages) {
-    const url = queue.shift();
-    const key = url.replace(/#.*$/, '');
-    if (seen.has(key)) continue;
-    seen.add(key);
+  let stopReason = 'frontier_exhausted';
+  const timeBudgetStopped = () => {
+    if (!timeExpired()) return false;
+    if (stopReason === 'frontier_exhausted') stopReason = 'time_budget_exhausted';
+    return true;
+  };
+
+  // Robots and sitemap topology are collected first so sitemap-only pages share
+  // the same bounded, deduplicated frontier as pages found through HTML links.
+  let robotsText = null;
+  if (!timeBudgetStopped()) {
     try {
-      const res = await fetcher(url, { userAgent, maxBodyBytes: pageMaxBytes });
+      const robotsUrl = new URL('/robots.txt', origin).href;
+      const response = await fetcher(robotsUrl, { userAgent, maxBodyBytes: robotsMaxBytes });
+      if (response.status === 200) robotsText = response.body;
+    } catch {
+      // Missing robots remains represented by robotsText=null for existing detectors.
+    }
+    timeBudgetStopped();
+  }
+
+  const declaredSitemaps = robotsText != null ? parseRobots(robotsText).sitemaps : [];
+  const sitemapEntries = declaredSitemaps.length
+    ? declaredSitemaps
+    : [new URL('/sitemap.xml', origin).href];
+  const sitemapTopology = await collectSitemapTopology(sitemapEntries, {
+    fetcher,
+    maxDepth: sitemapMaxDepth,
+    maxDocuments: sitemapMaxDocuments,
+    maxUncompressedBytes: sitemapMaxUncompressedBytes,
+    maxTotalUncompressedBytes: sitemapMaxTotalUncompressedBytes,
+    maxDiscoveredUrls: sitemapMaxDiscoveredUrls,
+    maxQueuedDocuments: sitemapMaxQueuedDocuments,
+    maxRawEntries: sitemapMaxRawEntries,
+    maxXmlTokens: sitemapMaxXmlTokens,
+    sitemapMaxBytes,
+    origin,
+    userAgent,
+    shouldStop: timeExpired,
+  });
+  if (sitemapTopology.stop_reasons.includes('time_budget_exhausted')) timeBudgetStopped();
+  for (const error of sitemapTopology.errors) {
+    errors.push(sanitizedSitemapError(error));
+  }
+  const sitemaps = sitemapTopology.documents
+    .filter((document) => document.http_status === 200 && document.parsed?.rootValid)
+    .map((document) => ({
+      source: document.effective_url,
+      parsed: document.parsed,
+      status: document.status,
+      failure_reason: document.failure_reason,
+    }));
+
+  const seen = new Set();
+  const queued = new Set();
+  const queue = [];
+  const enqueuePage = (rawUrl, source, base = origin) => {
+    let url;
+    let requestedUrl;
+    try {
+      const raw = String(rawUrl).trim();
+      url = new URL(raw, base);
+      url.hash = '';
+      const fragmentAt = raw.indexOf('#');
+      const withoutFragment = fragmentAt === -1 ? raw : raw.slice(0, fragmentAt);
+      if (/^[A-Za-z][A-Za-z\d+.-]*:\/\//.test(withoutFragment)) requestedUrl = withoutFragment;
+      else if (withoutFragment.startsWith('//')) requestedUrl = `${new URL(base).protocol}${withoutFragment}`;
+      else requestedUrl = url.href;
+    } catch {
+      return;
+    }
+    if (url.origin !== origin || isProviderUtilityPath(url.pathname) || seen.has(url.href) || queued.has(url.href)) return;
+    coverageLedger.discover(url.href, source, { requested_url: requestedUrl });
+    queued.add(url.href);
+    queue.push({ url: url.href, requestedUrl, source });
+  };
+  enqueuePage(startUrl, 'start');
+  for (const entry of sitemapTopology.urls) enqueuePage(entry.url, 'sitemap');
+
+  while (queue.length && stopReason !== 'time_budget_exhausted') {
+    if (timeBudgetStopped()) break;
+    if (seen.size >= maxPages) {
+      stopReason = 'page_budget_exhausted';
+      break;
+    }
+    const next = queue.shift();
+    queued.delete(next.url);
+    seen.add(next.url);
+    coverageLedger.attempt(next.url, { attempt: 1, requested_url: next.requestedUrl });
+    try {
+      const response = await fetcher(next.url, { userAgent, maxBodyBytes: pageMaxBytes });
+      coverageLedger.retrieve(next.url, {
+        status: response.status,
+        effective_url: response.url,
+        body_complete: response.bodyComplete ?? true,
+      });
       const page = extractPage({
-        url: res.url, html: res.body, status: res.status, headers: res.headers, redirectChain: res.redirectChain,
+        url: response.url, html: response.body, responseBody: response.bodyBytes ?? response.body, status: response.status,
+        headers: response.headers, redirectChain: response.redirectChain,
       });
-      page.requestedUrl = url;
+      attachResourceMetadata(page, {
+        requestedUrl: next.requestedUrl,
+        bodyComplete: response.bodyComplete ?? true,
+        fetchAttempts: response.attempts ?? [],
+      });
+      page.crawlUrl = next.url;
+      if (page.resourceValidity?.state === 'valid_resource') {
+        coverageLedger.classify(next.url, {
+          state: 'valid_resource',
+          resource_id: page.urlIdentity?.resource_id,
+          requested_url: page.urlIdentity?.requested?.url ?? page.requestedUrl ?? null,
+          effective_url: page.urlIdentity?.effective?.url ?? page.url ?? null,
+          canonical_url: page.urlIdentity?.canonical?.url ?? null,
+          url_identity: page.urlIdentity,
+          response_body_hash: page.response_body_hash,
+          extracted_text_hash: page.extracted_text_hash,
+          structured_data_hash: page.structured_data_hash,
+          evidence_hash: page.evidence_hash,
+          artifact_hash: page.artifact_hash,
+        });
+      } else {
+        coverageLedger.markIndeterminate(next.url, {
+          reason_codes: page.resourceValidity?.reason_codes || ['resource_invalid'],
+          resource_validity: page.resourceValidity?.state || 'indeterminate',
+        });
+      }
+      page.discoverySource = next.source;
       pages.push(page);
-      if (String(res.headers['content-type'] || '').includes('text/html')) {
-        for (const l of page.links) {
-          try {
-            const u = new URL(l.href, res.url);
-            u.hash = '';
-            if (u.origin === origin && !isProviderUtilityPath(u.pathname) && !seen.has(u.href)) queue.push(u.href);
-          } catch { /* unresolvable href — surfaced by LINK detectors */ }
+      if (timeBudgetStopped()) break;
+      if (String(response.headers['content-type'] || '').includes('text/html')) {
+        for (const link of page.links) {
+          if (timeBudgetStopped()) break;
+          enqueuePage(link.href, 'link', response.url);
         }
       }
-    } catch (err) {
-      errors.push(`${url}: ${err.message}`);
+      if (stopReason === 'time_budget_exhausted') break;
+    } catch (error) {
+      coverageLedger.fail(next.url, { reason: error?.code || 'FETCH_ERROR' });
+      errors.push(sanitizedFetchError(next.url, error));
     }
   }
-  let robotsText = null;
-  const sitemaps = [];
-  try {
-    const r = await fetcher(new URL('/robots.txt', origin).href, { userAgent, maxBodyBytes: robotsMaxBytes });
-    if (r.status === 200) robotsText = r.body;
-  } catch { /* recorded as missing robots */ }
-  const smUrls = robotsText ? parseRobots(robotsText).sitemaps : [new URL('/sitemap.xml', origin).href];
-  for (const sm of smUrls) {
-    try {
-      const sitemapUrl = new URL(sm, origin);
-      if (sitemapUrl.origin !== origin) {
-        errors.push(`${sitemapUrl.href}: sitemap URL leaves audited origin`);
-        continue;
-      }
-      const r = await fetcher(sitemapUrl.href, { userAgent, maxBodyBytes: sitemapMaxBytes });
-      if (r.status === 200) sitemaps.push({ source: sm, parsed: parseSitemap(r.body) });
-    } catch { /* absence handled by TECH detectors */ }
-  }
-  const pendingUrls = [...new Set(queue.map((url) => url.replace(/#.*$/, '')).filter((url) => !seen.has(url)))];
+  if (stopReason === 'frontier_exhausted') timeBudgetStopped();
+  const pendingUrls = queue.map((entry) => entry.url).filter((url) => !seen.has(url));
   const crawl = {
     maxPages,
+    timeBudgetSeconds,
+    stopReason,
     pagesFetched: pages.length,
-    truncated: pages.length >= maxPages && pendingUrls.length > 0,
+    truncated: stopReason === 'page_budget_exhausted' || stopReason === 'time_budget_exhausted',
     pendingUrlCount: pendingUrls.length,
     pendingUrls,
+    sitemapStopReasons: [...sitemapTopology.stop_reasons],
+    sitemapTopology: {
+      status: sitemapTopology.status,
+      stopReasons: [...sitemapTopology.stop_reasons],
+      limitations: [...sitemapTopology.limitations],
+    },
   };
   const site = assembleSite({ baseUrl: origin, pages, robotsText, sitemaps, transport: {}, mode: 'url', location: startUrl, crawl });
+  site.coverageLedger = coverageLedger;
+  site.coverageOptions = {
+    discoveryStatus: sitemapTopology.status,
+    discoveryMethods: ['start', 'link', 'sitemap'],
+    discoveryLimitations: [...sitemapTopology.limitations],
+    limitations: [...sitemapTopology.limitations],
+  };
+  site.previewCoverage = () => coverageLedger.preview(stopReason, site.coverageOptions);
+  site.finalizeCoverage = ({ evaluated = true } = {}) => {
+    if (evaluated) {
+      for (const page of pages) {
+        if (page.resourceValidity?.state === 'valid_resource') {
+          coverageLedger.evaluate(page.crawlUrl || page.url, {
+            evaluator: 'audit-detectors',
+            detector_scope: 'selected',
+          });
+        }
+      }
+    } else {
+      for (const page of pages) {
+        if (page.resourceValidity?.state === 'valid_resource') {
+          coverageLedger.markValidButUnevaluated(page.crawlUrl || page.url, { reason: 'not_evaluated' });
+        }
+      }
+    }
+    return coverageLedger.finalize(stopReason, site.coverageOptions);
+  };
   site.fetchErrors = errors;
+  site.sitemapTopology = sitemapTopology;
   return site;
 }
 

package/src/observations/browser/correlation.js

@@ -0,0 +1,83 @@
+import { nowIso } from '../../shared/io.js';
+
+export class CorrelationTracker {
+  constructor() {
+    this._sequence = 0;
+    this._startTime = performance.now();
+    this._currentStep = null; // { step_id, step_sequence, checkpoint_id }
+    this._phase = 'initial'; // 'initial' | 'during_step' | 'between_steps' | 'post_journey'
+  }
+
+  nextSequence() {
+    this._sequence += 1;
+    return this._sequence;
+  }
+
+  timing() {
+    const elapsed = Math.round((performance.now() - this._startTime) * 100) / 100;
+    return {
+      monotonic_offset_ms: elapsed,
+      timestamp: nowIso(),
+    };
+  }
+
+  startStep(step, stepSequence) {
+    this._currentStep = {
+      step_id: step.step_id,
+      step_sequence: stepSequence,
+      checkpoint_id: step.checkpoint_id || null,
+    };
+    this._phase = 'during_step';
+  }
+
+  endStep(step, stepSequence) {
+    this._phase = 'between_steps';
+  }
+
+  finishJourney() {
+    this._phase = 'post_journey';
+  }
+
+  /**
+   * Get correlation context for a newly initiated event/request.
+   */
+  getCorrelation(checkpointId = null) {
+    const stepId = this._currentStep ? this._currentStep.step_id : null;
+    return {
+      step_id: stepId,
+      step_sequence: this._currentStep ? this._currentStep.step_sequence : null,
+      checkpoint_id: checkpointId || (this._currentStep ? this._currentStep.checkpoint_id : null),
+      attribution_phase: this._phase,
+      initiated_step_id: stepId,
+      completed_step_id: null,
+      ambiguous_async: false,
+    };
+  }
+
+  /**
+   * Correlate completion of an asynchronous transaction (e.g. HTTP response).
+   * If the initiating step differs from the completing step/phase, mark as ambiguous_async.
+   */
+  correlateCompletion(initiatedCorrelation) {
+    const currentCorrelation = this.getCorrelation();
+    const isSameStep = initiatedCorrelation.step_id === currentCorrelation.step_id;
+    const isSamePhase = initiatedCorrelation.attribution_phase === currentCorrelation.attribution_phase;
+
+    let phase = initiatedCorrelation.attribution_phase;
+    let ambiguousAsync = false;
+    if (!isSameStep || (!isSamePhase && currentCorrelation.attribution_phase !== 'during_step')) {
+      phase = 'ambiguous_async';
+      ambiguousAsync = true;
+    }
+
+    return {
+      step_id: initiatedCorrelation.step_id,
+      step_sequence: initiatedCorrelation.step_sequence,
+      checkpoint_id: initiatedCorrelation.checkpoint_id,
+      attribution_phase: phase,
+      initiated_step_id: initiatedCorrelation.step_id,
+      completed_step_id: currentCorrelation.step_id,
+      ambiguous_async: ambiguousAsync,
+    };
+  }
+}