agentlinkops 0.5.0

package/src/verifier/index.js

@@ -0,0 +1,311 @@
+import { validatePublicUrl, matchesTarget } from './url.js';
+import { parseDocument, pageDirectives } from './parser.js';
+import { publicFetcher, VerificationError } from './fetch.js';
+import { assessReadiness } from './readiness.js';
+import { nativeRenderEvidence } from './render-evidence.js';
+
+export { validatePublicUrl } from './url.js';
+export { parseLinks } from './parser.js';
+export { transitionState } from './transition.js';
+export { checkObservationKind, normalizeDeclaredSite, withinDeclaredSite } from './observation-kind.js';
+
+export const CHECKER_VERSION = '4';
+const sharedDocument = Symbol('sharedDocument');
+const suppliedDocument = Symbol('suppliedDocument');
+
+const RENDER_REASONS = new Set(['possible_render_required', 'navigation_requires_browser']);
+
+/** Optional adaptive lane. The default remains one bounded HTTP observation. */
+export async function verifyLink(input, options = {}) {
+  const initial = await verifyHttpLink(input, options);
+  const eligible = RENDER_REASONS.has(initial.reason)
+    || (options.compareRendered === true && ['absent', 'present'].includes(initial.state));
+  if (!eligible) return initial;
+  initial.evidence.renderEligibility = RENDER_REASONS.has(initial.reason) ? 'required' : 'comparison';
+  if (!options.renderPage) return initial;
+  if (options.renderPage.publicFetchSafe !== true || options.renderPage.requiresNavigationAuthorization !== true) {
+    initial.evidence.renderAttempt = { attempted: false, reason: 'render_public_fetch_not_configured' };
+    return initial;
+  }
+  // Grouping shares only acquisition. Target scope, expectations and comparison remain independent.
+  const shared = options[sharedDocument];
+  const render = async () => {
+    const controller = new AbortController();
+    const timeout = Math.min(Math.max(options.renderTimeoutMs ?? 20_000, 1), 20_000);
+    let timer;
+    try {
+      const deadline = new Promise((_, reject) => {
+        timer = setTimeout(() => { controller.abort(); reject(new VerificationError('render_timeout')); }, timeout);
+      });
+      return { value: await Promise.race([
+        Promise.resolve().then(() => options.renderPage({ sourceUrl: initial.sourceUrl }, { signal: controller.signal })),
+        deadline,
+      ]) };
+    } catch (error) {
+      // Provider exception messages can contain credentials or remote page text.
+      return { error: controller.signal.aborted ? 'render_timeout' : 'render_failed',
+        ...(Number.isFinite(error?.retryAfterSeconds) && error.retryAfterSeconds >= 0 ? {retryAfterSeconds:Math.min(86400,Math.ceil(error.retryAfterSeconds))} : {}) };
+    } finally { clearTimeout(timer); controller.abort(); }
+  };
+  const acquired = shared ? await (shared.render ??= render()) : await render();
+  const fail = (reason) => {
+    initial.evidence.renderAttempt = { attempted: true, reason };
+    // A failed comparison cannot overrule an already observed anchor. Static absence,
+    // however, cannot be promoted when the requested browser comparison did not finish.
+    if (initial.state === 'absent') {
+      initial.state = 'unknown'; initial.reason = reason; initial.evidence.complete = false;
+      delete initial.linkSignature;
+    }
+    return initial;
+  };
+  if (acquired.error) {
+    if (acquired.retryAfterSeconds !== undefined) initial.retryAfterSeconds = acquired.retryAfterSeconds;
+    return fail(acquired.error);
+  }
+  const doc = acquired.value;
+  if (!doc || typeof doc.html !== 'string' || !validatePublicUrl(doc.finalUrl).valid
+    || !Number.isInteger(doc.httpStatus) || !Array.isArray(doc.redirects) || doc.redirects.length > 5
+    || doc.bodyTruncated !== false || typeof doc.ready !== 'boolean') return fail('invalid_render_evidence');
+  let cursor = initial.sourceUrl;
+  const redirects = [];
+  for (const hop of doc.redirects) {
+    if (!hop || hop.from !== cursor || !validatePublicUrl(hop.to).valid
+      || ![301, 302, 303, 307, 308].includes(hop.status)) return fail('invalid_render_redirects');
+    cursor = hop.to;
+    redirects.push({ from: hop.from, to: hop.to, status: hop.status });
+  }
+  if (cursor !== doc.finalUrl) return fail('invalid_render_redirects');
+  let nativeEvidence;
+  if (options.renderPage.strictEvidenceVersion !== undefined) {
+    const checked = nativeRenderEvidence(doc, options.renderPage.strictEvidenceVersion);
+    if (checked.reason) return fail(checked.reason);
+    nativeEvidence = checked.evidence;
+  }
+  if (doc.httpStatus !== 200) return fail(`render_http_${doc.httpStatus}`);
+  if (doc.xRobotsTag !== undefined && (typeof doc.xRobotsTag !== 'string' || doc.xRobotsTag.length > 4096
+    || /[^\t\x20-\x7e\x80-\xff]/u.test(doc.xRobotsTag))) return fail('invalid_render_directives');
+  if (doc.html.length > 2 * 1024 * 1024) return fail('render_body_too_large');
+  const data = new TextEncoder().encode(doc.html);
+  if (data.length > Math.min(options.maxBytes ?? 2 * 1024 * 1024, 2 * 1024 * 1024)) return fail('render_body_too_large');
+  const rendered = await verifyHttpLink(input, {
+    ...options, [sharedDocument]: undefined,
+    [suppliedDocument]: {
+      metadata: { finalUrl: doc.finalUrl, httpStatus: doc.httpStatus, redirects },
+      document: {
+        response: new Response(null, { status: 200, headers: { 'content-type': 'text/html; charset=utf-8', ...(doc.xRobotsTag ? { 'x-robots-tag': doc.xRobotsTag } : {}) } }),
+        body: { text: doc.html, bytes: data.length, data },
+      },
+    },
+  });
+  rendered.evidence.method = 'browser_html';
+  rendered.evidence.rendered = true;
+  rendered.evidence.ready = doc.ready;
+  if (nativeEvidence) {
+    rendered.evidence.browser = nativeEvidence;
+    if (nativeEvidence.retryAfterSeconds !== undefined && nativeEvidence.retryAfterSeconds !== null) rendered.retryAfterSeconds = nativeEvidence.retryAfterSeconds;
+  }
+  rendered.evidence.renderAttempt = { attempted: true, reason: 'render_completed' };
+  rendered.evidence.staticObservation = {
+    state: initial.state, reason: initial.reason, checkedAt: initial.checkedAt,
+    sha256: initial.evidence.sha256 ?? null, method: 'http_html',
+    ...(initial.robotsHistory ? { robotsHistory: structuredClone(initial.robotsHistory) } : {}),
+    ...(initial.sourceResponse ? { sourceResponse: structuredClone(initial.sourceResponse) } : {}),
+  };
+  if (rendered.state === 'absent' && options.renderPage.positiveOnly === true) {
+    rendered.state = 'unknown'; rendered.reason = 'render_no_positive_evidence';
+    rendered.evidence.complete = false; delete rendered.linkSignature;
+  } else if (rendered.state === 'absent' && !doc.ready) {
+    rendered.state = 'unknown'; rendered.reason = 'render_readiness_unconfirmed';
+    rendered.evidence.complete = false; delete rendered.linkSignature;
+  } else if (rendered.state === 'absent') rendered.reason = 'no_matching_link_in_ready_dom';
+  for (const occurrence of rendered.occurrences) occurrence.visibility = 'rendered_dom_not_visual';
+  rendered.warnings = ['Rendered DOM observation; visual visibility and search-engine indexing were not checked.'];
+  if (initial.state === 'present' && rendered.state !== 'present') {
+    initial.evidence.renderAttempt = { attempted: true, reason: 'render_disagreement', state: rendered.state };
+    return initial;
+  }
+  return rendered;
+}
+
+// The cache lives only for this bounded invocation, never across tenants or jobs.
+// Evaluation stays per target, including redirect-to-target and expectation rules.
+export async function verifyLinks(inputs, options = {}) {
+  if (!Array.isArray(inputs) || inputs.length > 20 || new Set(inputs.map(input => input.sourceUrl)).size > 1) {
+    throw new Error('A verification group requires at most 20 identical source URLs.');
+  }
+  const shared = {};
+  options = { ...options, now: timestamp(options.now) };
+  const results = [];
+  for (const input of inputs) results.push(await verifyLink(input, { ...options, [sharedDocument]: shared }));
+  return results;
+}
+
+async function sha256(bytes) {
+  return [...new Uint8Array(await crypto.subtle.digest('SHA-256', bytes))]
+    .map((value) => value.toString(16).padStart(2, '0')).join('');
+}
+
+function timestamp(now) {
+  return new Date(typeof now === 'function' ? now() : now ?? Date.now()).toISOString();
+}
+
+function expectations(occurrences, expectedAnchor, expectedRel) {
+  const anchor = expectedAnchor === undefined || expectedAnchor === null ? null : String(expectedAnchor).replace(/\s+/gu, ' ').trim();
+  const rel = expectedRel === undefined || expectedRel === null ? null
+    : [...new Set((Array.isArray(expectedRel) ? expectedRel : String(expectedRel).split(/\s+/u)).map((item) => item.toLowerCase()).filter(Boolean))].sort();
+  return {
+    expectedAnchor: anchor,
+    expectedRel: rel,
+    anchorMatches: anchor === null ? null : occurrences.some((item) => item.anchor === anchor),
+    relMatches: rel === null ? null : occurrences.some((item) => [...item.rel].sort().join(' ') === rel.join(' ')),
+    satisfied: occurrences.some((item) => (anchor === null || item.anchor === anchor)
+      && (rel === null || [...item.rel].sort().join(' ') === rel.join(' '))),
+  };
+}
+
+/**
+ * Verify a public page. publicFetchSafe asserts a transport-enforced public-only
+ * destination boundary; input screening alone cannot protect against DNS rebinding.
+ * The source page and robots.txt are the only fetched documents. No link is clicked.
+ */
+async function verifyHttpLink({ sourceUrl, targetUrl, targetScope = 'exact', expectedAnchor, expectedRel }, options = {}) {
+  const checkedAt = timestamp(options.now);
+  const result = {
+    state: 'unknown', reason: 'not_checked', sourceUrl, finalUrl: sourceUrl, targetUrl,
+    targetScope, httpStatus: null, checkedAt, occurrences: [], directives: pageDirectives([], ''),
+    redirects: [], evidence: { method: 'http_html', checkerVersion: CHECKER_VERSION, complete: false, fetchedAt: null },
+  };
+  const source = validatePublicUrl(sourceUrl);
+  const target = validatePublicUrl(targetUrl);
+  if (!source.valid || !target.valid) {
+    result.reason = `invalid_${!source.valid ? 'source' : 'target'}:${!source.valid ? source.reason : target.reason}`;
+    return result;
+  }
+  if (!['exact', 'domain', 'subdomain', 'path'].includes(targetScope)) {
+    result.reason = 'unsupported_target_scope';
+    return result;
+  }
+  result.sourceUrl = source.url;
+  result.finalUrl = source.url;
+  result.targetUrl = target.url;
+  if (options.publicFetchSafe !== true) {
+    result.reason = 'public_fetch_not_configured';
+    return result;
+  }
+  const controller = new AbortController();
+  const timeoutMs = Math.min(Math.max(options.timeoutMs ?? 20_000, 1), 20_000);
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const shared = options[sharedDocument];
+    const fetchSource = async () => {
+      try {
+        const supplied = options[suppliedDocument];
+        if (supplied) Object.assign(result, structuredClone(supplied.metadata));
+        const document = supplied?.document ?? await publicFetcher(options, controller.signal).source(source.url, result);
+        return { document, metadata: structuredClone({ finalUrl: result.finalUrl, httpStatus: result.httpStatus, redirects: result.redirects, ...(result.robots ? { robots: result.robots } : {}), ...(result.robotsHistory ? { robotsHistory: result.robotsHistory } : {}), ...(result.sourceResponse ? { sourceResponse: result.sourceResponse } : {}) }) };
+      } catch (error) {
+        return { error: controller.signal.aborted ? new VerificationError('timeout') : error, metadata: structuredClone({ finalUrl: result.finalUrl, httpStatus: result.httpStatus, redirects: result.redirects, ...(result.robots ? { robots: result.robots } : {}), ...(result.robotsHistory ? { robotsHistory: result.robotsHistory } : {}), ...(result.sourceResponse ? { sourceResponse: result.sourceResponse } : {}) }) };
+      }
+    };
+    const fetched = shared ? await (shared.fetch ??= fetchSource()) : await fetchSource();
+    Object.assign(result, structuredClone(fetched.metadata));
+    if (fetched.error) throw fetched.error;
+    const { response, body } = fetched.document;
+    result.evidence.fetchedAt = checkedAt;
+    if ([404, 410].includes(response.status)) {
+      result.state = 'source_unavailable';
+      result.reason = `source_http_${response.status}`;
+      result.evidence.complete = true;
+      return result;
+    }
+    result.evidence.bytes = body.bytes;
+    result.evidence.contentType = response.headers.get('content-type');
+    result.evidence.etag = response.headers.get('etag');
+    result.evidence.lastModified = response.headers.get('last-modified');
+    // Preserve a replayable private artifact even when classification remains unknown.
+    result.evidence.sha256 = await sha256(body.data);
+    if (options.includeHtml !== false) result.evidence.html = body.text;
+    if (!body.text.trim()) throw new VerificationError('empty_document');
+    let parsed;
+    try { parsed = shared ? (shared.parsed ??= parseDocument(body.text, result.finalUrl)) : parseDocument(body.text, result.finalUrl); }
+    catch (error) { throw new VerificationError(error.reason ?? 'html_parse_error'); }
+    const robotsHeader = response.headers.get('x-robots-tag');
+    if (robotsHeader?.length > 4096) throw new VerificationError('page_directive_too_long');
+    result.directives = pageDirectives(parsed.meta, robotsHeader);
+    if (parsed.incomplete) throw new VerificationError('incomplete_html', { parseErrors: parsed.parseErrors });
+    // Strong, recognizable access-wall evidence. Generic references to captchas in
+    // an article are deliberately insufficient to classify a document as blocked.
+    if (/^(?:just a moment(?:\.\.\.)?|access denied|attention required!?\s*(?:\|\s*cloudflare)?|verify (?:you are|you're) human)$/iu.test(parsed.title)
+      || parsed.hasChallengeScript) {
+      throw new VerificationError('access_challenge');
+    }
+    // A URL that REDIRECTS into the placement's scope is not a page that links to the target:
+    // following it lands on the destination, and parsing that document counts the destination's
+    // own navigation as placements. Measured on real data: two shortened links each reported
+    // `present` with **120 occurrences**, every one of them an internal link on the target's own
+    // homepage, anchored "Skip to main content". The one legitimate same-scope redirect is the
+    // source page canonicalizing ITSELF — same path (trailing slash, stripped tracking query)
+    // on the same host or a parent/child host of it (apex/www, blog to apex) — because then the
+    // destination's links ARE the source page's links. A redirect onto a DIFFERENT page inside
+    // the scope is not that, even when the whole site is the customer's own: DP-0020's
+    // `internal` kind made that case reachable (a blog post whose URL now bounces to the guide
+    // must not verify as a placement carried by the guide's own nav), so the carve-out keys on
+    // same-page canonicalization rather than on where the source started. One rule set for
+    // internal and external; the moved-page case reports `unknown` naming the redirect, and the
+    // customer points the ledger at the new URL.
+    const canonicalPage = (url) => {
+      const { hostname, pathname } = new URL(url);
+      return { host: hostname.toLowerCase().replace(/\.$/, ''),
+        path: pathname.length > 1 && pathname.endsWith('/') ? pathname.slice(0, -1) : pathname };
+    };
+    const sameSite = (a, b) => a === b || a.endsWith(`.${b}`) || b.endsWith(`.${a}`);
+    const from = canonicalPage(source.url);
+    const to = canonicalPage(result.finalUrl);
+    const samePage = from.path === to.path && sameSite(from.host, to.host);
+    if (result.redirects.length && !samePage && matchesTarget(result.finalUrl, target.url, targetScope)) {
+      throw new VerificationError('source_redirects_to_target');
+    }
+    result.occurrences = parsed.links.filter((link) => matchesTarget(link.targetUrl, target.url, targetScope));
+    if (new TextEncoder().encode(JSON.stringify(result.occurrences)).byteLength > 96 * 1024) throw new VerificationError('evidence_too_large');
+    result.expectations = expectations(result.occurrences, expectedAnchor, expectedRel);
+    // Meta refresh and a password form are access/navigation evidence, not proof of
+    // absence. A server-rendered occurrence can still be positively observed.
+    if (!result.occurrences.length && parsed.meta.some((item) => item.name === 'refresh')) {
+      throw new VerificationError('navigation_requires_browser');
+    }
+    const readiness = assessReadiness({ html: body.text, parsed, targetUrl: target.url });
+    result.evidence.readiness = readiness;
+    if (!result.occurrences.length && readiness.possibleLoginWall) {
+      throw new VerificationError('possible_login_wall');
+    }
+    if (!result.occurrences.length && (readiness.requiresRender
+      || (parsed.hasScript && !parsed.links.length && parsed.visibleText.length < 80))) {
+      throw new VerificationError('possible_render_required');
+    }
+    result.state = result.occurrences.length ? 'present' : 'absent';
+    result.reason = result.state === 'present' ? 'link_found' : 'no_matching_link_in_complete_html';
+    result.evidence.complete = true;
+    result.evidence.baseUrl = parsed.baseUrl;
+    result.evidence.rendered = false;
+    result.linkSignature = await sha256(new TextEncoder().encode(JSON.stringify({
+      source: result.finalUrl,
+      occurrences: result.occurrences.map(({ targetUrl: destination, anchor, rel }) => ({ targetUrl: destination, anchor, rel: [...rel].sort() })),
+      noindex: result.directives.noindex,
+      nofollow: result.directives.nofollow,
+    })));
+    result.warnings = ['Static HTML observation; JavaScript execution and visual visibility were not checked.'];
+    return result;
+  } catch (error) {
+    result.state = 'unknown';
+    result.occurrences = [];
+    result.reason = error instanceof VerificationError ? error.reason : controller.signal.aborted ? 'timeout' : 'verification_error';
+    // The same floor a destination check keeps. A publisher rate-limiting the SOURCE page is the
+    // more common case of the two, since a source is fetched on a cadence and a destination is not.
+    if (Number.isFinite(error?.retryAfterSeconds)) result.retryAfterSeconds = error.retryAfterSeconds;
+    if (error.parseErrors) result.evidence.parseErrors = error.parseErrors;
+    return result;
+  } finally {
+    clearTimeout(timer);
+    controller.abort();
+  }
+}

package/src/verifier/observation-kind.js

@@ -0,0 +1,91 @@
+// The write-time check behind DP-0020's `internal` / `external` observation kinds.
+//
+// A receipt names a placement in exactly the ledger's terms, plus one new field: `kind`, the
+// observation type. `internal` means both source and target are pages of the project's own
+// declared site; `external` means the source belongs to someone else. The distinction is
+// mechanical, not editorial — an actor does not get to choose its observation type, because
+// mislabeling an external placement as internal would claim ownership the customer does not
+// have and hide the placement from publisher-delay semantics.
+//
+// This module decides that question and nothing else. It never fetches: it runs at receipt
+// WRITE time, so a receipt that could never be checked honestly (a staging host, a non-public
+// host) is refused once, up front, instead of being checked into an eternal `unknown`.
+import { validatePublicUrl } from './url.js';
+
+// Same host grammar the fetch boundary enforces, so a declared site cannot smuggle in a
+// spelling the verifier would later refuse. A bare label ("intranet") is not a site.
+const HOST_PATTERN = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+$/u;
+
+/**
+ * Normalizes project identity's declared site into a host set. Accepts a single origin/host or
+ * a list of them ("https://customer.com", "customer.com"). Both apex and www forms must be
+ * declared explicitly: membership is exact-host, because ownership is DECLARED, not inferred —
+ * this module no more infers a registrable domain than `matchesTarget` does a public suffix.
+ * That is also what refuses a staging subdomain the customer never declared.
+ */
+export function normalizeDeclaredSite(site) {
+  // `null`/`undefined`/blank is "no declared site" — a distinct answer from junk, because it is
+  // the state of every project that has not declared one yet, and the error must say that.
+  const entries = site === null || site === undefined || (typeof site === 'string' && !site.trim())
+    ? [] : Array.isArray(site) ? site : [site];
+  const hosts = new Set();
+  for (const entry of entries) {
+    if (typeof entry !== 'string' || !entry.trim()) return { valid: false, reason: 'invalid_declared_site_entry' };
+    const text = entry.trim().toLowerCase();
+    let host = text;
+    if (/^https?:\/\//u.test(text)) {
+      let parsed;
+      try { parsed = new URL(text); } catch { return { valid: false, reason: 'invalid_declared_site_entry' }; }
+      // A credentials-bearing origin in project identity is a config mistake, and silently
+      // stripping it would bless a spelling the fetch boundary itself refuses everywhere.
+      if (parsed.username || parsed.password) return { valid: false, reason: 'invalid_declared_site_entry' };
+      host = parsed.hostname;
+    }
+    host = host.replace(/\.$/, '');
+    if (/^\d+\.\d+\.\d+\.\d+$/u.test(host)) return { valid: false, reason: 'invalid_declared_site_entry' };
+    if (!HOST_PATTERN.test(host) || host.length > 253) return { valid: false, reason: 'invalid_declared_site_entry' };
+    hosts.add(host);
+  }
+  if (!hosts.size) return { valid: false, reason: 'no_declared_site' };
+  return { valid: true, hosts: [...hosts].sort() };
+}
+
+/** Exact, case-insensitive host membership in the declared site. No suffix inference. */
+export function withinDeclaredSite(url, hosts) {
+  try { return hosts.includes(new URL(url).hostname.toLowerCase().replace(/\.$/, '')); }
+  catch { return false; }
+}
+
+/**
+ * Parse-level check of a receipt's observation kind against the declared site, per the
+ * DP-0020-T01 contract. Returns `{valid:true, kind, sourceInSite, targetInSite, declaredSite}`
+ * or `{valid:false, reason}` where the reason names the mismatch (the offending host, or the
+ * boundary's own refusal). Callers refuse the write on `valid:false`; the check is free and
+ * meters nothing.
+ *
+ * Refusals, all fixed by the contract:
+ * - `internal` whose source or target falls outside the declared site;
+ * - `external` whose source IS the declared site;
+ * - either kind naming a host the public fetch boundary would refuse (staging spellings on
+ *   reserved suffixes, localhost, IPs) — refused here rather than checked into an unknown;
+ * - either kind without a declared site: with no declared site there is no fact of ownership,
+ *   and the actor's label is the only thing left, which is exactly what must not decide it.
+ */
+export function checkObservationKind({ kind, sourceUrl, targetUrl }, site) {
+  const declared = normalizeDeclaredSite(site);
+  if (!declared.valid) return { valid: false, reason: declared.reason };
+  if (kind !== 'internal' && kind !== 'external') return { valid: false, reason: `unknown_kind:${String(kind)}` };
+  const source = validatePublicUrl(sourceUrl);
+  if (!source.valid) return { valid: false, reason: `invalid_source:${source.reason}` };
+  const target = validatePublicUrl(targetUrl);
+  if (!target.valid) return { valid: false, reason: `invalid_target:${target.reason}` };
+  const sourceInSite = withinDeclaredSite(source.url, declared.hosts);
+  const targetInSite = withinDeclaredSite(target.url, declared.hosts);
+  if (kind === 'internal') {
+    if (!sourceInSite) return { valid: false, reason: `internal_source_outside_declared_site:${source.hostname}` };
+    if (!targetInSite) return { valid: false, reason: `internal_target_outside_declared_site:${target.hostname}` };
+  } else if (sourceInSite) {
+    return { valid: false, reason: `external_source_on_declared_site:${source.hostname}` };
+  }
+  return { valid: true, kind, sourceInSite, targetInSite, declaredSite: declared.hosts };
+}

package/src/verifier/parser.js

@@ -0,0 +1,241 @@
+import { parse, defaultTreeAdapter } from 'parse5';
+
+const INERT = new Set(['script', 'style', 'template', 'noscript']);
+const BLOCKS = new Set(['p', 'li', 'article', 'section', 'div', 'td', 'dd', 'dt', 'blockquote']);
+// Where a browser puts a visual break. Used to decide whether two runs of text are separate words.
+// Everything NOT in here is treated as phrasing content — `span`, `b`, `a`, `em` — because a
+// browser renders `<span>Read</span><span>more</span>` as one word and so must we.
+const TEXT_BLOCKS = new Set([...BLOCKS, 'body', 'main', 'header', 'footer', 'nav', 'aside',
+  'figure', 'figcaption', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'ul', 'ol', 'dl', 'table', 'tr',
+  'th', 'thead', 'tbody', 'tfoot', 'caption', 'pre', 'form', 'fieldset', 'details', 'summary',
+  'address', 'hr', 'option', 'legend']);
+const EOF_ERRORS = new Set([
+  'eof-before-tag-name', 'eof-in-tag', 'eof-in-comment', 'eof-in-doctype',
+  'eof-in-element-that-can-contain-only-text', 'eof-in-script-html-comment-like-text',
+]);
+const attr = (node, name) => node.attrs?.find((value) => value.name === name)?.value;
+const normalize = (text) => text.replace(/\s+/gu, ' ').trim();
+
+function* walk(root, includeInert = false) {
+  const stack = [root];
+  while (stack.length) {
+    const node = stack.pop();
+    if (node.nodeName === '#comment' || (!includeInert && INERT.has(node.tagName))) continue;
+    yield node;
+    const children = node.childNodes ?? [];
+    for (let i = children.length - 1; i >= 0; i--) stack.push(children[i]);
+  }
+}
+
+/** The nearest ancestor a browser would break a line at, stopping at the subtree we were given. */
+function blockAncestor(node, root) {
+  let cursor = node.parentNode;
+  while (cursor && cursor !== root) {
+    if (cursor.tagName && TEXT_BLOCKS.has(cursor.tagName)) return cursor;
+    cursor = cursor.parentNode;
+  }
+  return root;
+}
+
+/**
+ * Text as a reader sees it, with a word boundary exactly where a browser draws one.
+ *
+ * **Two runs of text are separate words when they sit in different BLOCKS, not merely in different
+ * elements.** Without any separator, `<p>See settledestate.com</p><a>Guide</a>` read as
+ * `settledestate.comGuide`, which put a joined string into every occurrence context a customer
+ * sees. With a separator at every element boundary, `<a><span>Read</span><span>more</span></a>`
+ * read as `Read more` — and a browser renders that as one word, so anchor text and therefore the
+ * link signature would have disagreed with the page and emitted spurious change events.
+ *
+ * `normalize` collapses the runs this adds, so over-separating within a block is free and
+ * under-separating across one is not.
+ */
+function textContent(node, limit = 2048) {
+  let text = '';
+  let lastBlock = null;
+  const emit = (value, item) => {
+    const block = blockAncestor(item, node);
+    if (lastBlock !== null && block !== lastBlock) text += ' ';
+    text += value;
+    lastBlock = block;
+  };
+  for (const item of walk(node)) {
+    // A line break is a word boundary wherever it appears, block or not.
+    if (item.tagName === 'br') { text += ' '; lastBlock = null; }
+    if (item.nodeName === '#text') emit(item.value, item);
+    if (item.tagName === 'img') { const alt = attr(item, 'alt'); if (alt) emit(alt, item); }
+    if (text.length > limit * 2) break;
+  }
+  return normalize(text).slice(0, limit);
+}
+
+function contextFor(node) {
+  let context = node.parentNode ?? node;
+  for (let depth = 0; context.parentNode && depth < 4; depth++) {
+    if (BLOCKS.has(context.tagName) || context.tagName === 'body') break;
+    context = context.parentNode;
+  }
+  return textContent(context, 640);
+}
+
+export function parseDocument(html, pageUrl) {
+  if (typeof html !== 'string') throw new TypeError('HTML must be a string');
+  const errors = [];
+  let elements = 0;
+  let anchorElements = 0;
+  const parserStart = performance.now();
+  const treeAdapter = {
+    ...defaultTreeAdapter,
+    createElement(tagName, namespaceURI, attrs) {
+      // Bound parser expansion and evidence output independently of body bytes.
+      if (++elements > 25_000 || attrs.length > 256 || performance.now() - parserStart > 500) {
+        const error = new Error('document_too_complex');
+        error.reason = 'document_too_complex';
+        throw error;
+      }
+      if ((tagName === 'a' || tagName === 'area') && ++anchorElements > 1000) {
+        const error = new Error('too_many_link_occurrences');
+        error.reason = 'too_many_link_occurrences';
+        throw error;
+      }
+      return defaultTreeAdapter.createElement(tagName, namespaceURI, attrs);
+    },
+  };
+  const document = parse(html, {
+    treeAdapter,
+    scriptingEnabled: true,
+    sourceCodeLocationInfo: true,
+    onParseError: ({ code }) => { if (EOF_ERRORS.has(code) && errors.length < 8) errors.push(code); },
+  });
+  let baseUrl = pageUrl;
+  let baseFound = false;
+  let canonical = null;
+  const links = [];
+  const meta = [];
+  let title = '';
+  let hasScript = false;
+  let hasPasswordInput = false;
+  let hasChallengeScript = false;
+  let visibleText = '';
+  const nodes = [...walk(document)];
+  for (const node of walk(document, true)) {
+    if (node.tagName === 'script') {
+      hasScript = true;
+      if ((attr(node, 'src') ?? '').includes('/cdn-cgi/challenge-platform/')) hasChallengeScript = true;
+    }
+  }
+  for (const node of nodes) {
+    if (node.tagName === 'body') visibleText = textContent(node, 160);
+    if (node.tagName === 'input' && (attr(node, 'type') ?? '').toLowerCase() === 'password') hasPasswordInput = true;
+    if (node.tagName === 'base' && !baseFound && attr(node, 'href') !== undefined) {
+      // Browsers use only the first base element with an href, even if invalid.
+      baseFound = true;
+      if (attr(node, 'href').length > 8192) {
+        const error = new Error('base_url_too_long');
+        error.reason = 'base_url_too_long';
+        throw error;
+      }
+      try {
+        const base = new URL(attr(node, 'href'), pageUrl);
+        if (['http:', 'https:'].includes(base.protocol)) baseUrl = base.href;
+      } catch { /* Browser-compatible fallback to the document URL. */ }
+    }
+    if (node.tagName === 'meta') {
+      const name = (attr(node, 'name') ?? '').toLowerCase();
+      if ((['robots', 'googlebot', 'bingbot', 'linktrailbot'].includes(name)
+        || (attr(node, 'http-equiv') ?? '').toLowerCase() === 'refresh')
+        && (attr(node, 'content') ?? '').length > 2048) {
+        const error = new Error('page_directive_too_long');
+        error.reason = 'page_directive_too_long';
+        throw error;
+      }
+      if (['robots', 'googlebot', 'bingbot', 'linktrailbot'].includes(name)) {
+        meta.push({ name, content: (attr(node, 'content') ?? '').slice(0, 2048) });
+      }
+      if ((attr(node, 'http-equiv') ?? '').toLowerCase() === 'refresh') {
+        meta.push({ name: 'refresh', content: (attr(node, 'content') ?? '').slice(0, 2048) });
+      }
+      if (meta.length > 16) {
+        const error = new Error('too_many_page_directives');
+        error.reason = 'too_many_page_directives';
+        throw error;
+      }
+    }
+    if (node.tagName === 'title' && !title) title = textContent(node, 256);
+    // Browsers honour the first canonical link element only.
+    if (node.tagName === 'link' && canonical === null
+      && (attr(node, 'rel') ?? '').toLowerCase().split(/\s+/u).includes('canonical')) {
+      const href = attr(node, 'href') ?? '';
+      if (href && href.length <= 8192) {
+        try {
+          const resolved = new URL(href, baseUrl);
+          if (['http:', 'https:'].includes(resolved.protocol)) canonical = resolved.href;
+        } catch { /* An unparseable canonical is no canonical, exactly as a browser treats it. */ }
+      }
+    }
+  }
+  for (const node of nodes) {
+    if (!['a', 'area'].includes(node.tagName)) continue;
+    const href = attr(node, 'href');
+    if (href === undefined) continue;
+    if (href.length > 8192) {
+      const error = new Error('link_url_too_long');
+      error.reason = 'link_url_too_long';
+      throw error;
+    }
+    let target;
+    try { target = new URL(href, baseUrl); } catch { continue; }
+    if (!['http:', 'https:'].includes(target.protocol) || target.username || target.password) continue;
+    const location = node.sourceCodeLocation;
+    links.push({
+      href,
+      targetUrl: target.href,
+      anchor: node.tagName === 'area' ? normalize(attr(node, 'alt') ?? '') : textContent(node, 1024),
+      rel: [...new Set((attr(node, 'rel') ?? '').toLowerCase().split(/\s+/u).filter(Boolean))],
+      context: contextFor(node),
+      locator: location ? { line: location.startLine, column: location.startCol, offset: location.startOffset } : null,
+      visibility: 'not_rendered',
+    });
+  }
+  return { links, meta, title, baseUrl, canonical, hasScript, hasPasswordInput, hasChallengeScript, visibleText, incomplete: errors.length > 0, parseErrors: errors };
+}
+
+/**
+ * The page's text as a READER sees it, reusing the same walk the parser already uses.
+ *
+ * `walk` already skips `script`, `style`, `template`, `noscript` and comments, which is the
+ * property that matters: a brand name inside a JSON payload or an analytics config is not a
+ * mention anyone read. `parseDocument` caps its own copy at 160 characters because it only needs
+ * to know whether a page is an empty shell; a mention scan needs the whole document, so it asks
+ * for it explicitly rather than raising a cap every check would pay for.
+ *
+ * Takes no base URL, deliberately: text is not resolved against anything, and a parameter that
+ * looks like it matters and does not is how a caller comes to believe the output is absolute.
+ */
+export function pageText(html, limit = 512 * 1024) {
+  const document = parse(String(html), { sourceCodeLocationInfo: false, treeAdapter: defaultTreeAdapter });
+  for (const node of walk(document)) {
+    if (node.tagName === 'body') return textContent(node, limit);
+  }
+  return textContent(document, limit);
+}
+
+/** All HTTP(S) anchor/area occurrences in static HTML; never claims rendered visibility. */
+export function parseLinks(html, baseUrl) {
+  return parseDocument(html, new URL(baseUrl).href).links;
+}
+
+export function pageDirectives(meta, header) {
+  const xRobotsTag = header ? [header] : [];
+  // Preserve targeted header directives without incorrectly applying them to all bots.
+  const genericHeader = header && !/(?:^|,)\s*[\w-]+\s*:/u.test(header) ? header : '';
+  const generic = [genericHeader, ...meta.filter((item) => item.name === 'robots').map((item) => item.content)]
+    .join(',').toLowerCase().split(/[\s,]+/u);
+  return {
+    meta,
+    xRobotsTag,
+    noindex: generic.includes('noindex') || generic.includes('none'),
+    nofollow: generic.includes('nofollow') || generic.includes('none'),
+    indexingStatus: 'not_checked',
+  };
+}