agentlinkops 0.6.13 → 0.6.16

package/src/verifier/index.js

@@ -1,8 +1,11 @@
-import { validatePublicUrl, matchesTarget } from './url.js';
+import { validatePublicUrl, matchesTarget, hasRedirectWrapperToTarget } from './url.js';
 import { parseDocument, pageDirectives } from './parser.js';
 import { publicFetcher, VerificationError } from './fetch.js';
-import { assessReadiness } from './readiness.js';
+import {withAbort} from '../page-acquisition/bytes.js';
+import { analyzeReadiness, isAccessChallenge } from './readiness.js';
 import { nativeRenderEvidence } from './render-evidence.js';
+import {capturedDocument} from '../page-acquisition/capture.js';
+import {readRenderFailure} from '../page-acquisition/render-failure.js';
 
 export { validatePublicUrl } from './url.js';
 export { parseLinks } from './parser.js';
@@ -10,19 +13,32 @@ export { transitionState } from './transition.js';
 export { checkObservationKind, normalizeDeclaredSite, withinDeclaredSite } from './observation-kind.js';
 
 export const CHECKER_VERSION = '4';
+export function unknownLinkObservation({sourceUrl,targetUrl,targetScope='exact'},{now,reason='timeout'}={}){
+  return {state:'unknown',reason,sourceUrl,finalUrl:sourceUrl,targetUrl,targetScope,httpStatus:null,checkedAt:timestamp(now),occurrences:[],
+    directives:pageDirectives([],''),redirects:[],evidence:{method:'http_html',checkerVersion:CHECKER_VERSION,complete:false,fetchedAt:null}};
+}
 const sharedDocument = Symbol('sharedDocument');
 const suppliedDocument = Symbol('suppliedDocument');
+const suppliedInitial = Symbol('suppliedInitial');
+const forceRender = Symbol('forceRender');
 
 const RENDER_REASONS = new Set(['possible_render_required', 'navigation_requires_browser']);
 
 /** Optional adaptive lane. The default remains one bounded HTTP observation. */
 export async function verifyLink(input, options = {}) {
-  const initial = await verifyHttpLink(input, options);
-  const eligible = RENDER_REASONS.has(initial.reason)
+  const initial = options[suppliedInitial] ?? await verifyHttpLink(input, options);
+  const eligible = options[forceRender] === true || RENDER_REASONS.has(initial.reason)
     || (options.compareRendered === true && ['absent', 'present'].includes(initial.state));
   if (!eligible) return initial;
   initial.evidence.renderEligibility = RENDER_REASONS.has(initial.reason) ? 'required' : 'comparison';
-  if (!options.renderPage) return initial;
+  if (initial.robotsHistory?.some(row=>row?.overrideUsed===true)) {
+    initial.evidence.renderAttempt = { attempted:false, reason:'monitor_robots_override_fetch_limit' };
+    return initial;
+  }
+  if (!options.renderPage) {
+    initial.evidence.renderAttempt={attempted:false,reason:'render_not_configured'};
+    return initial;
+  }
   if (options.renderPage.publicFetchSafe !== true || options.renderPage.requiresNavigationAuthorization !== true) {
     initial.evidence.renderAttempt = { attempted: false, reason: 'render_public_fetch_not_configured' };
     return initial;
@@ -30,26 +46,32 @@ export async function verifyLink(input, options = {}) {
   // Grouping shares only acquisition. Target scope, expectations and comparison remain independent.
   const shared = options[sharedDocument];
   const render = async () => {
+    if(options.signal?.aborted||(Number.isFinite(options.deadlineAt)&&options.deadlineAt<=Date.now()))return {error:'render_timeout'};
     const controller = new AbortController();
-    const timeout = Math.min(Math.max(options.renderTimeoutMs ?? 20_000, 1), 20_000);
+    const ownerAbort=()=>controller.abort();
+    options.signal?.addEventListener('abort',ownerAbort,{once:true});
+    const timeout = Math.min(Math.max(options.renderTimeoutMs ?? 20_000, 1), 20_000,
+      options.deadlineAt ? Math.max(1,options.deadlineAt-Date.now()) : 20_000);
     let timer;
     try {
       const deadline = new Promise((_, reject) => {
         timer = setTimeout(() => { controller.abort(); reject(new VerificationError('render_timeout')); }, timeout);
       });
       return { value: await Promise.race([
-        Promise.resolve().then(() => options.renderPage({ sourceUrl: initial.sourceUrl }, { signal: controller.signal })),
+        withAbort(Promise.resolve().then(() => options.renderPage({ sourceUrl: initial.sourceUrl }, { signal: controller.signal,deadlineAt:options.deadlineAt })),controller.signal),
         deadline,
       ]) };
     } catch (error) {
       // Provider exception messages can contain credentials or remote page text.
+      const failure = readRenderFailure(error);
       return { error: controller.signal.aborted ? 'render_timeout' : 'render_failed',
+        ...(failure ? {failure} : {}),
         ...(Number.isFinite(error?.retryAfterSeconds) && error.retryAfterSeconds >= 0 ? {retryAfterSeconds:Math.min(86400,Math.ceil(error.retryAfterSeconds))} : {}) };
-    } finally { clearTimeout(timer); controller.abort(); }
+    } finally { clearTimeout(timer); controller.abort(); options.signal?.removeEventListener('abort', ownerAbort); }
   };
   const acquired = shared ? await (shared.render ??= render()) : await render();
-  const fail = (reason) => {
-    initial.evidence.renderAttempt = { attempted: true, reason };
+  const fail = (reason, failure) => {
+    initial.evidence.renderAttempt = { attempted: true, reason, ...(failure ? {failure} : {}) };
     // A failed comparison cannot overrule an already observed anchor. Static absence,
     // however, cannot be promoted when the requested browser comparison did not finish.
     if (initial.state === 'absent') {
@@ -60,7 +82,7 @@ export async function verifyLink(input, options = {}) {
   };
   if (acquired.error) {
     if (acquired.retryAfterSeconds !== undefined) initial.retryAfterSeconds = acquired.retryAfterSeconds;
-    return fail(acquired.error);
+    return fail(acquired.error,acquired.failure);
   }
   const doc = acquired.value;
   if (!doc || typeof doc.html !== 'string' || !validatePublicUrl(doc.finalUrl).valid
@@ -88,7 +110,7 @@ export async function verifyLink(input, options = {}) {
   const data = new TextEncoder().encode(doc.html);
   if (data.length > Math.min(options.maxBytes ?? 2 * 1024 * 1024, 2 * 1024 * 1024)) return fail('render_body_too_large');
   const rendered = await verifyHttpLink(input, {
-    ...options, [sharedDocument]: undefined,
+    ...options, [sharedDocument]: shared ? (shared.renderedDocument ??= {}) : undefined,
     [suppliedDocument]: {
       metadata: { finalUrl: doc.finalUrl, httpStatus: doc.httpStatus, redirects },
       document: {
@@ -100,6 +122,9 @@ export async function verifyLink(input, options = {}) {
   rendered.evidence.method = 'browser_html';
   rendered.evidence.rendered = true;
   rendered.evidence.ready = doc.ready;
+  rendered.evidence.capture=capturedDocument(rendered,{requestedUrl:initial.sourceUrl,representation:'rendered_dom',
+    transportId:options.renderPage.transportId??'qualified_browser',transportVersion:options.renderPage.transportVersion??'1',
+    qualified:options.renderPage.publicFetchSafe===true&&!!nativeEvidence,charset:'utf-8'});
   if (nativeEvidence) {
     rendered.evidence.browser = nativeEvidence;
     if (nativeEvidence.retryAfterSeconds !== undefined && nativeEvidence.retryAfterSeconds !== null) rendered.retryAfterSeconds = nativeEvidence.retryAfterSeconds;
@@ -140,6 +165,15 @@ export async function verifyLinks(inputs, options = {}) {
   return results;
 }
 
+/** Evaluate one admitted browser acquisition without fetching the static page again. */
+export async function verifyRenderedLinks(inputs, initial, options = {}) {
+  if(!Array.isArray(initial)||initial.length!==inputs.length)throw new TypeError('Initial observations must bind the render inputs.');
+  const shared={};const results=[];
+  for(const [i,input] of inputs.entries())results.push(await verifyLink(input,{...options,
+    [sharedDocument]:shared,[suppliedInitial]:structuredClone(initial[i]),[forceRender]:true}));
+  return results;
+}
+
 async function sha256(bytes) {
   return [...new Uint8Array(await crypto.subtle.digest('SHA-256', bytes))]
     .map((value) => value.toString(16).padStart(2, '0')).join('');
@@ -170,11 +204,7 @@ function expectations(occurrences, expectedAnchor, expectedRel) {
  */
 async function verifyHttpLink({ sourceUrl, targetUrl, targetScope = 'exact', expectedAnchor, expectedRel }, options = {}) {
   const checkedAt = timestamp(options.now);
-  const result = {
-    state: 'unknown', reason: 'not_checked', sourceUrl, finalUrl: sourceUrl, targetUrl,
-    targetScope, httpStatus: null, checkedAt, occurrences: [], directives: pageDirectives([], ''),
-    redirects: [], evidence: { method: 'http_html', checkerVersion: CHECKER_VERSION, complete: false, fetchedAt: null },
-  };
+  const result = unknownLinkObservation({sourceUrl,targetUrl,targetScope},{now:checkedAt,reason:'not_checked'});
   const source = validatePublicUrl(sourceUrl);
   const target = validatePublicUrl(targetUrl);
   if (!source.valid || !target.valid) {
@@ -192,8 +222,14 @@ async function verifyHttpLink({ sourceUrl, targetUrl, targetScope = 'exact', exp
     result.reason = 'public_fetch_not_configured';
     return result;
   }
+  if (options.signal?.aborted || (Number.isFinite(options.deadlineAt) && options.deadlineAt <= Date.now())) {
+    result.reason = 'timeout'; return result;
+  }
   const controller = new AbortController();
-  const timeoutMs = Math.min(Math.max(options.timeoutMs ?? 20_000, 1), 20_000);
+  const ownerAbort = () => controller.abort();
+  options.signal?.addEventListener('abort', ownerAbort, { once: true });
+  const timeoutMs = Math.min(Math.max(Number.isFinite(options.timeoutMs) ? options.timeoutMs : 20_000, 1), 20_000,
+    options.deadlineAt ? Math.max(1,options.deadlineAt-Date.now()) : 20_000);
   const timer = setTimeout(() => controller.abort(), timeoutMs);
   try {
     const shared = options[sharedDocument];
@@ -211,6 +247,7 @@ async function verifyHttpLink({ sourceUrl, targetUrl, targetScope = 'exact', exp
     Object.assign(result, structuredClone(fetched.metadata));
     if (fetched.error) throw fetched.error;
     const { response, body } = fetched.document;
+    if (controller.signal.aborted || (Number.isFinite(options.deadlineAt) && Date.now() >= options.deadlineAt)) throw new VerificationError('timeout');
     result.evidence.fetchedAt = checkedAt;
     if ([404, 410].includes(response.status)) {
       result.state = 'source_unavailable';
@@ -224,6 +261,10 @@ async function verifyHttpLink({ sourceUrl, targetUrl, targetScope = 'exact', exp
     result.evidence.lastModified = response.headers.get('last-modified');
     // Preserve a replayable private artifact even when classification remains unknown.
     result.evidence.sha256 = await sha256(body.data);
+    result.evidence.decodedSha256 = await sha256(new TextEncoder().encode(body.text));
+    result.evidence.capture=capturedDocument(result,{requestedUrl:source.url,transportId:options.captureTransport?.id??'public_http',
+      transportVersion:options.captureTransport?.version??'1',qualified:options.publicFetchSafe===true,
+      charset:/charset\s*=\s*["']?([^\s;"']+)/iu.exec(response.headers.get('content-type')??'')?.[1]??'utf-8'});
     if (options.includeHtml !== false) result.evidence.html = body.text;
     if (!body.text.trim()) throw new VerificationError('empty_document');
     let parsed;
@@ -233,10 +274,11 @@ async function verifyHttpLink({ sourceUrl, targetUrl, targetScope = 'exact', exp
     if (robotsHeader?.length > 4096) throw new VerificationError('page_directive_too_long');
     result.directives = pageDirectives(parsed.meta, robotsHeader);
     if (parsed.incomplete) throw new VerificationError('incomplete_html', { parseErrors: parsed.parseErrors });
+    const { readiness, managedClientChallenge } = analyzeReadiness({ html: body.text, parsed, targetUrl: target.url });
+    result.evidence.readiness = readiness;
     // Strong, recognizable access-wall evidence. Generic references to captchas in
     // an article are deliberately insufficient to classify a document as blocked.
-    if (/^(?:just a moment(?:\.\.\.)?|access denied|attention required!?\s*(?:\|\s*cloudflare)?|verify (?:you are|you're) human)$/iu.test(parsed.title)
-      || parsed.hasChallengeScript) {
+    if (isAccessChallenge(parsed, { managedClientChallenge })) {
       throw new VerificationError('access_challenge');
     }
     // A URL that REDIRECTS into the placement's scope is not a page that links to the target:
@@ -262,19 +304,31 @@ async function verifyHttpLink({ sourceUrl, targetUrl, targetScope = 'exact', exp
     const from = canonicalPage(source.url);
     const to = canonicalPage(result.finalUrl);
     const samePage = from.path === to.path && sameSite(from.host, to.host);
-    if (result.redirects.length && !samePage && matchesTarget(result.finalUrl, target.url, targetScope)) {
+    // Landing on the target PAGE is the same failure even when the URL is not an exact scope
+    // match: a tracking query, trailing slash, scheme or www hop puts the reader on the
+    // destination, whose self-links then read as placements (DP-0004-T24 corpus v2).
+    // Host identity here is exact up to a leading `www.`: a parent/child host with the same
+    // path (blog.example.com/guide versus example.com/guide) is a different page, not the target.
+    const wanted = canonicalPage(target.url);
+    const bareHost = (host) => host.replace(/^www\./, '');
+    const landsOnTargetPage = to.path === wanted.path && bareHost(to.host) === bareHost(wanted.host);
+    if (result.redirects.length && !samePage
+      && (landsOnTargetPage || matchesTarget(result.finalUrl, target.url, targetScope))) {
       throw new VerificationError('source_redirects_to_target');
     }
     result.occurrences = parsed.links.filter((link) => matchesTarget(link.targetUrl, target.url, targetScope));
     if (new TextEncoder().encode(JSON.stringify(result.occurrences)).byteLength > 96 * 1024) throw new VerificationError('evidence_too_large');
     result.expectations = expectations(result.occurrences, expectedAnchor, expectedRel);
+    // A physical tracking link can name the target without proving its eventual
+    // destination. Preserve uncertainty until redirect identity is resolved.
+    if (!result.occurrences.length && hasRedirectWrapperToTarget(parsed.links, target.url, targetScope)) {
+      throw new VerificationError('target_in_redirect_parameter');
+    }
     // Meta refresh and a password form are access/navigation evidence, not proof of
     // absence. A server-rendered occurrence can still be positively observed.
     if (!result.occurrences.length && parsed.meta.some((item) => item.name === 'refresh')) {
       throw new VerificationError('navigation_requires_browser');
     }
-    const readiness = assessReadiness({ html: body.text, parsed, targetUrl: target.url });
-    result.evidence.readiness = readiness;
     if (!result.occurrences.length && readiness.possibleLoginWall) {
       throw new VerificationError('possible_login_wall');
     }
@@ -293,11 +347,14 @@ async function verifyHttpLink({ sourceUrl, targetUrl, targetScope = 'exact', exp
       noindex: result.directives.noindex,
       nofollow: result.directives.nofollow,
     })));
+    if (controller.signal.aborted || (Number.isFinite(options.deadlineAt) && Date.now() >= options.deadlineAt)) throw new VerificationError('timeout');
     result.warnings = ['Static HTML observation; JavaScript execution and visual visibility were not checked.'];
     return result;
   } catch (error) {
     result.state = 'unknown';
     result.occurrences = [];
+    result.evidence.complete = false;
+    delete result.linkSignature;
     result.reason = error instanceof VerificationError ? error.reason : controller.signal.aborted ? 'timeout' : 'verification_error';
     // The same floor a destination check keeps. A publisher rate-limiting the SOURCE page is the
     // more common case of the two, since a source is fetched on a cadence and a destination is not.
@@ -307,5 +364,6 @@ async function verifyHttpLink({ sourceUrl, targetUrl, targetScope = 'exact', exp
   } finally {
     clearTimeout(timer);
     controller.abort();
+    options.signal?.removeEventListener('abort', ownerAbort);
   }
 }

package/src/verifier/page-quality.js

@@ -0,0 +1,14 @@
+import { isAccessChallenge } from './readiness.js';
+
+// A caller holding HTML can supply private readiness analysis without expanding
+// any public evidence DTO. Existing parsed-document-only callers keep their contract.
+export function pageQualityReason(parsed, analysis = null) {
+  if (parsed.incomplete) return 'incomplete_html';
+  if (isAccessChallenge(parsed, analysis ?? {})) return 'access_challenge';
+  if (!parsed.links.length && parsed.meta.some(item => item.name === 'refresh')) return 'navigation_requires_browser';
+  if (!parsed.links.length && (analysis ? analysis.readiness.possibleLoginWall : parsed.hasPasswordInput)) return 'possible_login_wall';
+  if (analysis?.readiness.requiresRender) return 'possible_render_required';
+  if (!parsed.links.length && parsed.hasScript && parsed.visibleText.length < 80) return 'possible_render_required';
+
+  return null;
+}

package/src/verifier/parser.js

@@ -1,241 +1,2 @@
-import { parse, defaultTreeAdapter } from 'parse5';
-
-const INERT = new Set(['script', 'style', 'template', 'noscript']);
-const BLOCKS = new Set(['p', 'li', 'article', 'section', 'div', 'td', 'dd', 'dt', 'blockquote']);
-// Where a browser puts a visual break. Used to decide whether two runs of text are separate words.
-// Everything NOT in here is treated as phrasing content — `span`, `b`, `a`, `em` — because a
-// browser renders `<span>Read</span><span>more</span>` as one word and so must we.
-const TEXT_BLOCKS = new Set([...BLOCKS, 'body', 'main', 'header', 'footer', 'nav', 'aside',
-  'figure', 'figcaption', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'ul', 'ol', 'dl', 'table', 'tr',
-  'th', 'thead', 'tbody', 'tfoot', 'caption', 'pre', 'form', 'fieldset', 'details', 'summary',
-  'address', 'hr', 'option', 'legend']);
-const EOF_ERRORS = new Set([
-  'eof-before-tag-name', 'eof-in-tag', 'eof-in-comment', 'eof-in-doctype',
-  'eof-in-element-that-can-contain-only-text', 'eof-in-script-html-comment-like-text',
-]);
-const attr = (node, name) => node.attrs?.find((value) => value.name === name)?.value;
-const normalize = (text) => text.replace(/\s+/gu, ' ').trim();
-
-function* walk(root, includeInert = false) {
-  const stack = [root];
-  while (stack.length) {
-    const node = stack.pop();
-    if (node.nodeName === '#comment' || (!includeInert && INERT.has(node.tagName))) continue;
-    yield node;
-    const children = node.childNodes ?? [];
-    for (let i = children.length - 1; i >= 0; i--) stack.push(children[i]);
-  }
-}
-
-/** The nearest ancestor a browser would break a line at, stopping at the subtree we were given. */
-function blockAncestor(node, root) {
-  let cursor = node.parentNode;
-  while (cursor && cursor !== root) {
-    if (cursor.tagName && TEXT_BLOCKS.has(cursor.tagName)) return cursor;
-    cursor = cursor.parentNode;
-  }
-  return root;
-}
-
-/**
- * Text as a reader sees it, with a word boundary exactly where a browser draws one.
- *
- * **Two runs of text are separate words when they sit in different BLOCKS, not merely in different
- * elements.** Without any separator, `<p>See settledestate.com</p><a>Guide</a>` read as
- * `settledestate.comGuide`, which put a joined string into every occurrence context a customer
- * sees. With a separator at every element boundary, `<a><span>Read</span><span>more</span></a>`
- * read as `Read more` — and a browser renders that as one word, so anchor text and therefore the
- * link signature would have disagreed with the page and emitted spurious change events.
- *
- * `normalize` collapses the runs this adds, so over-separating within a block is free and
- * under-separating across one is not.
- */
-function textContent(node, limit = 2048) {
-  let text = '';
-  let lastBlock = null;
-  const emit = (value, item) => {
-    const block = blockAncestor(item, node);
-    if (lastBlock !== null && block !== lastBlock) text += ' ';
-    text += value;
-    lastBlock = block;
-  };
-  for (const item of walk(node)) {
-    // A line break is a word boundary wherever it appears, block or not.
-    if (item.tagName === 'br') { text += ' '; lastBlock = null; }
-    if (item.nodeName === '#text') emit(item.value, item);
-    if (item.tagName === 'img') { const alt = attr(item, 'alt'); if (alt) emit(alt, item); }
-    if (text.length > limit * 2) break;
-  }
-  return normalize(text).slice(0, limit);
-}
-
-function contextFor(node) {
-  let context = node.parentNode ?? node;
-  for (let depth = 0; context.parentNode && depth < 4; depth++) {
-    if (BLOCKS.has(context.tagName) || context.tagName === 'body') break;
-    context = context.parentNode;
-  }
-  return textContent(context, 640);
-}
-
-export function parseDocument(html, pageUrl) {
-  if (typeof html !== 'string') throw new TypeError('HTML must be a string');
-  const errors = [];
-  let elements = 0;
-  let anchorElements = 0;
-  const parserStart = performance.now();
-  const treeAdapter = {
-    ...defaultTreeAdapter,
-    createElement(tagName, namespaceURI, attrs) {
-      // Bound parser expansion and evidence output independently of body bytes.
-      if (++elements > 25_000 || attrs.length > 256 || performance.now() - parserStart > 500) {
-        const error = new Error('document_too_complex');
-        error.reason = 'document_too_complex';
-        throw error;
-      }
-      if ((tagName === 'a' || tagName === 'area') && ++anchorElements > 1000) {
-        const error = new Error('too_many_link_occurrences');
-        error.reason = 'too_many_link_occurrences';
-        throw error;
-      }
-      return defaultTreeAdapter.createElement(tagName, namespaceURI, attrs);
-    },
-  };
-  const document = parse(html, {
-    treeAdapter,
-    scriptingEnabled: true,
-    sourceCodeLocationInfo: true,
-    onParseError: ({ code }) => { if (EOF_ERRORS.has(code) && errors.length < 8) errors.push(code); },
-  });
-  let baseUrl = pageUrl;
-  let baseFound = false;
-  let canonical = null;
-  const links = [];
-  const meta = [];
-  let title = '';
-  let hasScript = false;
-  let hasPasswordInput = false;
-  let hasChallengeScript = false;
-  let visibleText = '';
-  const nodes = [...walk(document)];
-  for (const node of walk(document, true)) {
-    if (node.tagName === 'script') {
-      hasScript = true;
-      if ((attr(node, 'src') ?? '').includes('/cdn-cgi/challenge-platform/')) hasChallengeScript = true;
-    }
-  }
-  for (const node of nodes) {
-    if (node.tagName === 'body') visibleText = textContent(node, 160);
-    if (node.tagName === 'input' && (attr(node, 'type') ?? '').toLowerCase() === 'password') hasPasswordInput = true;
-    if (node.tagName === 'base' && !baseFound && attr(node, 'href') !== undefined) {
-      // Browsers use only the first base element with an href, even if invalid.
-      baseFound = true;
-      if (attr(node, 'href').length > 8192) {
-        const error = new Error('base_url_too_long');
-        error.reason = 'base_url_too_long';
-        throw error;
-      }
-      try {
-        const base = new URL(attr(node, 'href'), pageUrl);
-        if (['http:', 'https:'].includes(base.protocol)) baseUrl = base.href;
-      } catch { /* Browser-compatible fallback to the document URL. */ }
-    }
-    if (node.tagName === 'meta') {
-      const name = (attr(node, 'name') ?? '').toLowerCase();
-      if ((['robots', 'googlebot', 'bingbot', 'linktrailbot'].includes(name)
-        || (attr(node, 'http-equiv') ?? '').toLowerCase() === 'refresh')
-        && (attr(node, 'content') ?? '').length > 2048) {
-        const error = new Error('page_directive_too_long');
-        error.reason = 'page_directive_too_long';
-        throw error;
-      }
-      if (['robots', 'googlebot', 'bingbot', 'linktrailbot'].includes(name)) {
-        meta.push({ name, content: (attr(node, 'content') ?? '').slice(0, 2048) });
-      }
-      if ((attr(node, 'http-equiv') ?? '').toLowerCase() === 'refresh') {
-        meta.push({ name: 'refresh', content: (attr(node, 'content') ?? '').slice(0, 2048) });
-      }
-      if (meta.length > 16) {
-        const error = new Error('too_many_page_directives');
-        error.reason = 'too_many_page_directives';
-        throw error;
-      }
-    }
-    if (node.tagName === 'title' && !title) title = textContent(node, 256);
-    // Browsers honour the first canonical link element only.
-    if (node.tagName === 'link' && canonical === null
-      && (attr(node, 'rel') ?? '').toLowerCase().split(/\s+/u).includes('canonical')) {
-      const href = attr(node, 'href') ?? '';
-      if (href && href.length <= 8192) {
-        try {
-          const resolved = new URL(href, baseUrl);
-          if (['http:', 'https:'].includes(resolved.protocol)) canonical = resolved.href;
-        } catch { /* An unparseable canonical is no canonical, exactly as a browser treats it. */ }
-      }
-    }
-  }
-  for (const node of nodes) {
-    if (!['a', 'area'].includes(node.tagName)) continue;
-    const href = attr(node, 'href');
-    if (href === undefined) continue;
-    if (href.length > 8192) {
-      const error = new Error('link_url_too_long');
-      error.reason = 'link_url_too_long';
-      throw error;
-    }
-    let target;
-    try { target = new URL(href, baseUrl); } catch { continue; }
-    if (!['http:', 'https:'].includes(target.protocol) || target.username || target.password) continue;
-    const location = node.sourceCodeLocation;
-    links.push({
-      href,
-      targetUrl: target.href,
-      anchor: node.tagName === 'area' ? normalize(attr(node, 'alt') ?? '') : textContent(node, 1024),
-      rel: [...new Set((attr(node, 'rel') ?? '').toLowerCase().split(/\s+/u).filter(Boolean))],
-      context: contextFor(node),
-      locator: location ? { line: location.startLine, column: location.startCol, offset: location.startOffset } : null,
-      visibility: 'not_rendered',
-    });
-  }
-  return { links, meta, title, baseUrl, canonical, hasScript, hasPasswordInput, hasChallengeScript, visibleText, incomplete: errors.length > 0, parseErrors: errors };
-}
-
-/**
- * The page's text as a READER sees it, reusing the same walk the parser already uses.
- *
- * `walk` already skips `script`, `style`, `template`, `noscript` and comments, which is the
- * property that matters: a brand name inside a JSON payload or an analytics config is not a
- * mention anyone read. `parseDocument` caps its own copy at 160 characters because it only needs
- * to know whether a page is an empty shell; a mention scan needs the whole document, so it asks
- * for it explicitly rather than raising a cap every check would pay for.
- *
- * Takes no base URL, deliberately: text is not resolved against anything, and a parameter that
- * looks like it matters and does not is how a caller comes to believe the output is absolute.
- */
-export function pageText(html, limit = 512 * 1024) {
-  const document = parse(String(html), { sourceCodeLocationInfo: false, treeAdapter: defaultTreeAdapter });
-  for (const node of walk(document)) {
-    if (node.tagName === 'body') return textContent(node, limit);
-  }
-  return textContent(document, limit);
-}
-
-/** All HTTP(S) anchor/area occurrences in static HTML; never claims rendered visibility. */
-export function parseLinks(html, baseUrl) {
-  return parseDocument(html, new URL(baseUrl).href).links;
-}
-
-export function pageDirectives(meta, header) {
-  const xRobotsTag = header ? [header] : [];
-  // Preserve targeted header directives without incorrectly applying them to all bots.
-  const genericHeader = header && !/(?:^|,)\s*[\w-]+\s*:/u.test(header) ? header : '';
-  const generic = [genericHeader, ...meta.filter((item) => item.name === 'robots').map((item) => item.content)]
-    .join(',').toLowerCase().split(/[\s,]+/u);
-  return {
-    meta,
-    xRobotsTag,
-    noindex: generic.includes('noindex') || generic.includes('none'),
-    nofollow: generic.includes('nofollow') || generic.includes('none'),
-    indexingStatus: 'not_checked',
-  };
-}
+// Compatibility facade: acquisition consumers share one bounded static-HTML parser.
+export { parseDocument, parseLinks, pageText, pageDirectives } from '../page-extraction/html.js';

package/src/verifier/readiness.js

@@ -7,11 +7,22 @@ const attr = (node, name) => node.attrs?.find((item) => item.name === name)?.val
 const hidden = (node) => attr(node, 'hidden') !== undefined
   || (node.tagName === 'dialog' && attr(node, 'open') === undefined)
   || /(?:^|;)\s*(?:display\s*:\s*none|visibility\s*:\s*(?:hidden|collapse))\s*(?:!important\s*)?(?:;|$)/iu.test(attr(node, 'style') ?? '');
+const text = (value) => value.replace(/[\u2018\u2019]/gu, "'").replace(/\s+/gu, ' ').trim();
+const unfinishedNotice = (value) => /^(?:still loading|we (?:had|are having) trouble loading(?: [^.!?]{1,80})?|(?:this|your) browser is (?:not|no longer) supported)(?:\.{1,3}|…|!)?$/iu.test(text(value));
+const editorialArticle = (node) => node.tagName === 'article'
+  && (!attr(node, 'itemtype') || /(?:^|\s)https?:\/\/schema\.org\/(?:[a-z]*Article|BlogPosting)(?:\s|$)/iu.test(attr(node, 'itemtype')));
 
-/** Static evidence only: this does not claim computed CSS visibility or browser readiness. */
-export function assessReadiness({ html, parsed, targetUrl }) {
+/** A managed challenge needs actual challenge markup, not prose about bot checks. */
+export function isAccessChallenge(parsed, { managedClientChallenge = false } = {}) {
+  return /^(?:just a moment(?:\.\.\.)?|access denied|attention required!?\s*(?:\|\s*cloudflare)?|verify (?:you are|you're) human)$/iu.test(parsed?.title ?? '')
+    || parsed?.hasChallengeScript === true
+    || (/^client challenge$/iu.test(parsed?.title ?? '') && managedClientChallenge);
+}
+
+/** Internal analysis keeps challenge evidence outside the portable readiness DTO. */
+export function analyzeReadiness({ html, parsed, targetUrl }) {
   if (typeof html !== 'string') throw new TypeError('HTML must be a string');
-  const bounded = () => ({ requiresRender: true, reason: 'readiness_limit_exceeded', possibleLoginWall: false });
+  const bounded = () => ({ readiness: { requiresRender: true, reason: 'readiness_limit_exceeded', possibleLoginWall: false }, managedClientChallenge: false });
   if (html.length > 2 * 1024 * 1024) return bounded();
   let elements = 0;
   const started = performance.now();
@@ -32,6 +43,11 @@ export function assessReadiness({ html, parsed, targetUrl }) {
   }
   let hasScript = false;
   let targetInScript = false;
+  let targetInHydrationData = false;
+  let targetInVisibleText = false;
+  let hasUnfinishedNotice = false;
+  let hasManagedChallengeScript = false;
+  let hasChallengeAlert = false;
   let hasVisiblePassword = false;
   let contentText = '';
   let articleText = '';
@@ -42,27 +58,50 @@ export function assessReadiness({ html, parsed, targetUrl }) {
   while (stack.length) {
     const state = stack.pop();
     const { node } = state;
-    if (hidden(node)) continue;
+    if (!INERT.has(node.tagName) && attr(node, 'id') === 'loading-error' && attr(node, 'role') === 'alert') hasChallengeAlert = true;
+    const nodeHidden = state.hidden || hidden(node);
     if (node.tagName === 'script') {
       hasScript = true;
-      // Script data is a rendering hint, never a link occurrence. Keep this exact and narrow.
-      if (targetUrl && node.childNodes?.some((child) => child.nodeName === '#text'
-        && child.value.replace(/\\\//gu, '/').includes(targetUrl))) targetInScript = true;
+      const value = (node.childNodes ?? []).filter((child) => child.nodeName === '#text').map((child) => child.value).join('');
+      const scriptContainsTarget = targetUrl && (value.replace(/\\\//gu, '/').includes(targetUrl)
+        || value.includes(encodeURIComponent(targetUrl)));
+      // Framework payloads may contain a missing link, but are never anchor evidence.
+      if (scriptContainsTarget) {
+        targetInScript = true;
+        if (attr(node, 'type') === 'application/json' || attr(node, 'data-relay-completed-request') === 'true'
+          || /\b(?:window|self)\.[\w$]*(?:hydration|state|data)[\w$]*\s*=/iu.test(value)
+          || /\b__sveltekit[\w$]*\b/u.test(value)) targetInHydrationData = true;
+      }
+      if (/\/_fs-ch-[a-z0-9_-]+\/(?:errors|script)\.js(?:[?'"\s]|$)/iu.test(`${attr(node, 'src') ?? ''}\n${value}`)) hasManagedChallengeScript = true;
     }
     if (INERT.has(node.tagName)) continue;
+    // Hidden containers suppress their visible content, but do not make an
+    // executable script inert. Hydration chunks can live in such containers.
+    if (nodeHidden) {
+      for (const child of node.childNodes ?? []) stack.push({ ...state, node: child, hidden: true });
+      continue;
+    }
     const chrome = state.chrome || CHROME.has(node.tagName);
     const form = state.form || node.tagName === 'form';
-    const article = state.article || node.tagName === 'article';
+    const article = state.article || editorialArticle(node);
     let root = state.root;
     const id = (attr(node, 'id') ?? '').toLowerCase();
-    if (ROOT_IDS.has(id) || ['profile', 'profile-root', 'profile-container'].includes(id)) {
-      root = { app: ROOT_IDS.has(id), content: false, parent: root };
+    const appRoot = ROOT_IDS.has(id) || attr(node, 'data-reactcontainer') === 'true'
+      || attr(node, 'data-reactroot') !== undefined || (node.tagName === 'body' && attr(node, 'data-sveltekit-preload-data') !== undefined);
+    if (appRoot || ['profile', 'profile-root', 'profile-container'].includes(id)) {
+      root = { app: appRoot, content: false, busy: attr(node, 'aria-busy') === 'true', unfinished: false, parent: root };
       roots.push(root);
     }
     if (node.tagName === 'input' && (attr(node, 'type') ?? '').toLowerCase() === 'password') hasVisiblePassword = true;
     if (!chrome && node.nodeName === '#text' && node.value.trim()) {
       for (let ancestor = root; ancestor && !ancestor.content; ancestor = ancestor.parent) ancestor.content = true;
       if (!form) {
+        const unfinished = !article && unfinishedNotice(node.value);
+        if (unfinished) {
+          hasUnfinishedNotice = true;
+          for (let ancestor = root; ancestor && !ancestor.unfinished; ancestor = ancestor.parent) ancestor.unfinished = true;
+        }
+        if (targetUrl && node.value.includes(targetUrl)) targetInVisibleText = true;
         contentText += node.value.slice(0, 256 - contentText.length);
         if (article) articleText += node.value.slice(0, 256 - articleText.length);
       }
@@ -78,13 +117,25 @@ export function assessReadiness({ html, parsed, targetUrl }) {
   }
   const substantiveContent = contentText.replace(/\s+/gu, ' ').trim().length >= 80
     || articleText.replace(/\s+/gu, ' ').trim().length >= 40;
+  const substantiveArticle = text(articleText).length >= 40;
   const possibleLoginWall = hasVisiblePassword && !substantiveContent;
   let reason = null;
-  if (hasScript && !substantiveContent) {
+  if (hasScript && !substantiveArticle && (roots.some((root) => root.app && (root.busy || root.unfinished))
+    || (emptyAppRoot && hasUnfinishedNotice))) {
+    reason = 'application_content_unfinished';
+  } else if (hasScript && targetInHydrationData && roots.length && !substantiveArticle && !targetInVisibleText) {
+    reason = 'application_target_in_hydration_only';
+  } else if (hasScript && !substantiveContent) {
     if (emptyAppRoot) reason = 'empty_application_root';
     else if (emptyProfileRoot && targetInScript) reason = 'profile_target_in_script_only';
     else if (!(parsed?.links?.length) && !contentText.trim()) reason = 'empty_script_shell';
   }
-  return { requiresRender: reason !== null, reason, possibleLoginWall,
-    hasVisiblePassword, targetInScript, emptyAppRoot, emptyProfileRoot };
+  return { readiness: { requiresRender: reason !== null, reason, possibleLoginWall,
+    hasVisiblePassword, targetInScript, emptyAppRoot, emptyProfileRoot },
+    managedClientChallenge: hasManagedChallengeScript && hasChallengeAlert };
+}
+
+/** Static evidence only; preserves the existing portable readiness schema. */
+export function assessReadiness(input) {
+  return analyzeReadiness(input).readiness;
 }