agentlinkops 0.6.13 → 0.6.16

package/src/page-acquisition/recovery.js

@@ -0,0 +1,164 @@
+import {ACQUISITION_LIMITS,PageAcquisitionError,boundedAcquisitionLimit} from './contracts.js';
+import {withAbort} from './bytes.js';
+import {recoveryEligibility,acquisitionState as stateOf,completedAcquisition as complete} from './policy.js';
+export {recoveryEligibility} from './policy.js';
+
+export const RECOVERY_METHODS = Object.freeze(['workers_http','node_http','node_browser','proxy_http']);
+export const RECOVERY_REASONS = Object.freeze(['disabled','not_enrolled','unsupported','not_configured','unqualified','deadline','rate_limit','robots_override','custody_hold','busy','budget_exhausted','scope_revoked','complete','not_needed']);
+const METHODS=new Set(RECOVERY_METHODS),REASONS=new Set(RECOVERY_REASONS);
+const safeReason=value=>typeof value==='string'&&/^[a-z0-9_:.-]{1,100}$/u.test(value)?value:'acquisition_failed';
+const STOP_PRIORITY={scope_revoked:10,deadline:9,rate_limit:8,robots_override:8,custody_hold:7,busy:6,budget_exhausted:6,unqualified:5,not_configured:4,not_enrolled:3,disabled:2,unsupported:1,not_needed:0};
+const stop=(trace,reason)=>{if(reason!=='complete'&&(STOP_PRIORITY[reason]??0)>(STOP_PRIORITY[trace.terminationReason]??-1))trace.terminationReason=reason;};
+const bodySummary=value=>value?.version===1&&Number.isSafeInteger(value.bytes)&&value.bytes>=0&&value.bytes<=65536&&typeof value.complete==='boolean'
+  &&typeof value.truncated==='boolean'&&/^[a-f0-9]{64}$/u.test(value.sha256??'')&&typeof value.looksLikeHtml==='boolean'
+  ?{version:1,bytes:value.bytes,complete:value.complete,truncated:value.truncated,sha256:value.sha256,looksLikeHtml:value.looksLikeHtml}:undefined;
+function diagnostics(item){
+  const robots=Array.isArray(item.robotsHistory)?item.robotsHistory.slice(0,6).map(row=>({
+    httpStatus:Number.isInteger(row?.httpStatus)&&row.httpStatus>=100&&row.httpStatus<=599?row.httpStatus:null,
+    allowed:typeof row?.allowed==='boolean'?row.allowed:null,reason:row?.reason?safeReason(row.reason):null,
+    ...(row?.overrideUsed===true?{overrideUsed:true}:{}),...(bodySummary(row?.bodyDiagnostic)?{bodyDiagnostic:bodySummary(row.bodyDiagnostic)}:{})})):undefined;
+  const source=item.sourceResponse;
+  return {...(robots?{robotsHistory:robots}:{}),...(source?{sourceResponse:{
+    httpStatus:Number.isInteger(source.httpStatus)&&source.httpStatus>=100&&source.httpStatus<=599?source.httpStatus:null,
+    contentType:typeof source.contentType==='string'&&/^[a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+$/iu.test(source.contentType)?source.contentType.slice(0,128):null,
+    retryAfterSeconds:Number.isFinite(source.retryAfterSeconds)&&source.retryAfterSeconds>=0?Math.min(86400,Math.ceil(source.retryAfterSeconds)):null,
+    challenge:source.challenge===true,...(bodySummary(source.bodyDiagnostic)?{bodyDiagnostic:bodySummary(source.bodyDiagnostic)}:{})}}:{})};
+}
+
+export function acquisitionBudget({signal,timeoutMs,deadlineAt,clock=Date.now}={}){
+  const start=clock();
+  if(!Number.isFinite(start)||(deadlineAt!=null&&!Number.isFinite(deadlineAt)))throw new PageAcquisitionError('acquisition_limits_invalid');
+  const deadline=Math.min(start+boundedAcquisitionLimit(timeoutMs,ACQUISITION_LIMITS.timeoutMs,ACQUISITION_LIMITS.timeoutMs),deadlineAt??Infinity);
+  const controller=new AbortController(),abort=()=>controller.abort();
+  if(signal?.aborted||deadline<=start)abort();else signal?.addEventListener('abort',abort,{once:true});
+  const timer=setTimeout(abort,Math.max(0,deadline-start));
+  return {signal:controller.signal,deadline,remaining:()=>Math.max(0,deadline-clock()),
+    assert(){if(controller.signal.aborted||clock()>=deadline)throw new PageAcquisitionError('timeout');},
+    close(){clearTimeout(timer);controller.abort();signal?.removeEventListener('abort',abort);}};
+}
+
+/** Compact, credential-free diagnostics. Old version-one observations remain readable. */
+export function recoverySummary(value){
+  if(![1,2].includes(value?.version)||!Array.isArray(value.attempts)||value.attempts.length<1||value.attempts.length>(value.version===1?2:4)||!METHODS.has(value.selectedMethod))return undefined;
+  const states=new Set(['present','changed','absent','unknown','source_unavailable','healthy','unavailable','blocked','deferred']);
+  const attempts=[];
+  for(const item of value.attempts){
+    if(!item||!METHODS.has(item.method)||!states.has(item.state)||typeof item.complete!=='boolean'||!Number.isFinite(item.durationMs)||item.durationMs<0||item.durationMs>120000)return undefined;
+    attempts.push({method:item.method,state:item.state,complete:item.complete,durationMs:Math.floor(item.durationMs),
+      ...(typeof item.reason==='string'&&/^[a-z0-9_:.-]{1,100}$/u.test(item.reason)?{reason:item.reason}:{}),
+      ...(value.version===2?diagnostics(item):{})});
+  }
+  if(value.version===1)return {version:1,selectedMethod:value.selectedMethod,attempts};
+  if(!['customer_monitor','discovery','target','research','document','free'].includes(value.purpose)||!Array.isArray(value.routes)||value.routes.length>4
+    ||!value.routes.every(r=>r&&METHODS.has(r.method)&&typeof r.available==='boolean'&&typeof r.admitted==='boolean'&&(r.reason===null||REASONS.has(r.reason)))
+    ||!(value.terminationReason===null||REASONS.has(value.terminationReason)))return undefined;
+  return {version:2,purpose:value.purpose,selectedMethod:value.selectedMethod,attempts,routes:value.routes.map(({method,available,admitted,reason})=>({method,available,admitted,reason})),terminationReason:value.terminationReason};
+}
+
+const attempt=(result,method,durationMs)=>({method,durationMs,state:stateOf(result),...(result?.reason?{reason:safeReason(result.reason)}:{}),complete:complete(result),
+  ...(result?.robotsHistory?{robotsHistory:structuredClone(result.robotsHistory)}:{}),
+  ...(result?.sourceResponse?{sourceResponse:structuredClone(result.sourceResponse)}:{})});
+function admission(value,fallback){
+  if(value?.available===true)return {available:true,reason:null};
+  return {available:false,reason:REASONS.has(value?.reason)?value.reason:fallback};
+}
+
+/**
+ * Owns the whole recovery ladder, never a deadline per route. Executors preserve their
+ * producer's leases, quotas and actual capture receipts; this router grants none of them.
+ * A shared source can recover unknown targets without overwriting already observed links.
+ */
+export async function runRecovery({executeDirect,directFailure,directMethod='workers_http',routes={},authorize=async()=>true,purpose='customer_monitor',options={},clock=Date.now}={}){
+  if(typeof executeDirect!=='function')throw new TypeError('A direct acquisition executor is required.');
+  if(!['workers_http','node_http'].includes(directMethod))throw new TypeError('A direct HTTP method is required.');
+  const owner=acquisitionBudget({...options,clock}),traces=[];let selected,pending=null,directStarted=false;
+  const settings=()=>({...options,signal:owner.signal,deadlineAt:owner.deadline,timeoutMs:Math.max(1,owner.remaining())});
+  const finish=()=>{
+    for(const [i,result] of selected.entries()){
+      result.evidence??={};result.evidence.acquisition=traces[i];
+      const receipts=Object.values(routes).flatMap(route=>Array.isArray(route?.receipts)?route.receipts:[]);
+      if(receipts.length)result.evidence.acquisition.transportReceipts=structuredClone(receipts.slice(0,36));
+    }
+    return selected;
+  };
+  try{
+    owner.assert();if(!await withAbort(Promise.resolve().then(()=>authorize({beforeDispatch:true})),owner.signal))throw new PageAcquisitionError('acquisition_scope_revoked');
+    const started=clock();directStarted=true;selected=await withAbort(Promise.resolve().then(()=>executeDirect(settings())),owner.signal);
+    if(!Array.isArray(selected))throw new PageAcquisitionError('acquisition_result_invalid');
+    for(const result of selected)traces.push({version:2,purpose,selectedMethod:directMethod,attempts:[attempt(result,directMethod,clock()-started)],
+      routes:[{method:directMethod,available:true,admitted:true,reason:null}],terminationReason:null});
+    owner.assert();
+    if(!await withAbort(Promise.resolve().then(()=>authorize({beforeDispatch:false})),owner.signal))throw new PageAcquisitionError('acquisition_scope_revoked');
+    for(const method of ['node_http','node_browser','proxy_http']){
+      if(method===directMethod)continue;
+      const route=routes[method],cap=admission(route?.capability,typeof route?.execute==='function'?'not_configured':'disabled');
+      let status=cap;
+      if(cap.available&&typeof route?.execute!=='function')status={available:false,reason:'not_configured'};
+      if(status.available&&typeof route?.admit==='function'&&selected.some(value=>recoveryEligibility(value,method).allowed)){
+        if(owner.remaining()<=250)status={available:false,reason:'deadline'};
+        else status=admission(await withAbort(Promise.resolve().then(()=>route.admit(settings())),owner.signal),'not_enrolled');
+      }
+      const indices=[];
+      for(const [i,result] of selected.entries()){
+        const eligibility=recoveryEligibility(result,method);
+        const reason=!eligibility.allowed?eligibility.reason:!status.available?status.reason:owner.remaining()<=250?'deadline':null;
+        traces[i].routes.push({method,available:status.available,admitted:reason===null,reason});
+        if(reason===null)indices.push(i);
+        else stop(traces[i],reason);
+      }
+      if(!indices.length)continue;
+      if(!await withAbort(Promise.resolve().then(()=>authorize({beforeDispatch:true})),owner.signal))throw new PageAcquisitionError('acquisition_scope_revoked');
+      owner.assert();
+      if(owner.remaining()<=250){for(const i of indices){Object.assign(traces[i].routes.at(-1),{admitted:false,reason:'deadline'});stop(traces[i],'deadline');}continue;}
+      const at=clock();pending={method,indices,at};
+      let next,routeOwner;
+      try{
+        let routeSettings=settings();
+        if(method==='node_browser'&&routes.proxy_http?.capability?.available===true&&typeof routes.proxy_http.execute==='function'){
+          const proxyAdmission=typeof routes.proxy_http.admit==='function'?admission(await withAbort(Promise.resolve().then(()=>routes.proxy_http.admit(settings())),owner.signal),'not_enrolled'):{available:true};
+          if(proxyAdmission.available){
+            const duration=Math.min(10000,owner.remaining()-3000);
+            if(duration<=250){for(const i of indices){Object.assign(traces[i].routes.at(-1),{admitted:false,reason:'deadline'});stop(traces[i],'deadline');}pending=null;continue;}
+            routeOwner=acquisitionBudget({signal:owner.signal,deadlineAt:Math.min(owner.deadline,clock()+duration),timeoutMs:duration,clock});
+            routeSettings={...routeSettings,signal:routeOwner.signal,deadlineAt:routeOwner.deadline,timeoutMs:duration,renderTimeoutMs:duration};
+          }
+        }
+        next=await withAbort(Promise.resolve().then(()=>route.execute(routeSettings,selected,indices)),routeOwner?.signal??owner.signal);
+      }catch(error){
+        if(error?.reason!=='timeout'||!routeOwner||clock()<routeOwner.deadline||owner.signal.aborted||options.signal?.aborted)throw error;
+        next=selected.map((value,i)=>indices.includes(i)?{...value,...(value.state?{state:'unknown'}:{outcome:'unavailable',html:null}),reason:'route_timeout',occurrences:[],
+          evidence:{...value.evidence,complete:false,...(method==='node_browser'?{renderAttempt:{attempted:true,reason:'render_timeout'}}:{})}}:value);
+      }finally{routeOwner?.close();}
+      owner.assert();
+      if(!Array.isArray(next)||next.length!==selected.length)throw new PageAcquisitionError('acquisition_result_invalid');
+      if(!await withAbort(Promise.resolve().then(()=>authorize({beforeDispatch:false})),owner.signal))throw new PageAcquisitionError('acquisition_scope_revoked');
+      owner.assert();
+      for(const i of indices){selected[i]=next[i];traces[i].attempts.push(attempt(next[i],method,clock()-at));traces[i].selectedMethod=method;traces[i].terminationReason=null;}
+      pending=null;
+    }
+    return finish();
+  }catch(error){
+    // A route deadline is an observed inability to finish, never absence. Owner
+    // cancellation and producer revocation still reject without publishing a page.
+    if(error?.reason!=='timeout'||options.signal?.aborted||clock()<owner.deadline)throw error;
+    if(!selected&&typeof directFailure==='function'){
+      selected=directFailure('timeout');
+      if(!Array.isArray(selected))throw error;
+      for(const result of selected)traces.push({version:2,purpose,selectedMethod:directMethod,attempts:[attempt(result,directMethod,0)],
+        routes:[{method:directMethod,available:true,admitted:directStarted,reason:'deadline'}],terminationReason:'deadline'});
+    }
+    if(!selected||traces.length!==selected.length)throw error;
+    if(pending)for(const i of pending.indices){
+      selected[i]={...selected[i],...(selected[i].state?{state:'unknown'}:{outcome:'unavailable',html:null}),reason:'timeout',occurrences:[],
+        evidence:{...selected[i].evidence,complete:false}};
+      traces[i].attempts.push(attempt(selected[i],pending.method,clock()-pending.at));traces[i].selectedMethod=pending.method;
+    }
+    for(const [i,trace] of traces.entries()){
+      const completed=complete(selected[i])||['present','absent','healthy','source_unavailable'].includes(stateOf(selected[i]));
+      if(!completed)stop(trace,'deadline');
+      for(const method of RECOVERY_METHODS.slice(1))if(!trace.routes.some(row=>row.method===method))
+        trace.routes.push({method,available:routes[method]?.capability?.available===true,admitted:false,reason:completed?'complete':'deadline'});
+    }
+    return finish();
+  }finally{owner.close();}
+}

package/src/page-acquisition/render-failure.js

@@ -0,0 +1,82 @@
+// Error messages, page text and provider diagnostics never enter this contract.
+// Only the signed client records provenance; copying fields onto an Error cannot
+// manufacture an authenticated renderer failure for the verifier.
+const failures = new WeakMap();
+const executorReasons = new Set([
+  'executor_failed','executor_busy','mode_not_enabled_on_endpoint','browser_not_enabled',
+  'browser_failed','browser_unit_failed','browser_namespace_failed','browser_render_failed',
+  'browser_request_failed','browser_controller_failed','browser_controller_not_configured',
+  'browser_controller_response_unverified','browser_controller_response_too_large',
+  'browser_controller_url_not_qualified','browser_cancelled','browser_cancel_unverified',
+  'browser_authority_denied','browser_authority_expired','browser_authority_not_configured',
+  'browser_authority_unavailable','browser_authority_unverified','browser_cleanup_unverified',
+  'browser_insufficient_deadline','browser_resource_admission_failed','browser_job_setup_failed',
+  'browser_native_result_unverified','browser_result_too_large','browser_receipt_too_large',
+  'browser_uid_wrapper_unqualified','browser_executable_unqualified','browser_egress_unqualified',
+  'browser_namespace_escape_probe_failed','browser_namespace_unqualified','browser_not_configured',
+  'browser_access_challenge','browser_html_too_large','browser_navigation_changed',
+  'browser_redirect_limit','browser_request_limit','browser_resource_bytes_exceeded',
+  'browser_resource_concurrency_exceeded','browser_unsafe_redirect','browser_unsafe_navigation',
+  'browser_unsupported_content_type','browser_websocket_interception_unavailable',
+  'robots_access_challenge','robots_crawl_delay_requires_scheduler','robots_invalid_encoding',
+  'robots_not_plain_text','robots_redirect_limit','robots_redirect_loop','robots_unsafe_redirect',
+  'robots_disallowed','robots_fetch_failed','robots_unavailable','robots_too_complex',
+  'deadline_exceeded','body_too_large','incomplete_body','content_encoding_unsupported',
+  'dns_destination_not_public','dns_resolution_failed','response_headers_too_large',
+  'origin_connection_failed','socket_destination_changed','socket_destination_unverified',
+  'socket_hostname_changed','byte_meter_failed','ipc_body_too_large','ipc_tunnel_unverified',
+  'ipc_authentication_failed','ipc_body_unverified','ipc_destination_unsafe',
+  'ipc_invalid_request','ipc_request_too_large','ipc_response_unverified','ipc_route_not_found',
+  'expired_or_invalid_request','request_replayed','replay_store_full','replay_journal_unavailable',
+  'invalid_request','invalid_request_json','invalid_request_identity','invalid_binding',
+  'invalid_browser_binding','invalid_limits','invalid_mode','invalid_request_header',
+  'unrecognized_user_agent','unsupported_request_headers','unsupported_accept',
+  'native_admission_closed','native_admission_epoch_retired','native_admission_state_invalid',
+  'native_custody_journal_full','native_custody_journal_unavailable','native_custody_journal_invalid',
+  'native_custody_identity_missing','native_custody_identity_changed','native_custody_dispatch_changed','native_custody_session_invalid',
+]);
+const urlReasons = '(?:invalid_url|unsupported_protocol|url_credentials|unsupported_port|ip_literal_not_supported|non_public_hostname)';
+const dynamicReason = new RegExp(`^(?:unsafe_url|browser_unsafe_resource):${urlReasons}$`, 'u');
+const localReasons = new Set(['timeout','acquisition_scope_revoked','acquisition_not_configured',
+  'acquisition_failed','acquisition_request_invalid','acquisition_receipt_invalid',
+  'acquisition_receipt_limit','body_too_large','incomplete_body','unsupported_or_invalid_encoding']);
+const token = value => typeof value === 'string' && /^[A-Za-z0-9_-]{1,128}$/u.test(value);
+const session = value => typeof value === 'string' && /^alo-page-browser-job-[a-f0-9]{24}\.service$/u.test(value);
+export function safeExecutorFailureReason(value) {
+  return executorReasons.has(value) || typeof value === 'string' && (dynamicReason.test(value) || /^robots_http_[1-5][0-9]{2}$/u.test(value)) ? value : 'executor_failed';
+}
+function safeReason(value) {
+  if(localReasons.has(value))return value;
+  if(typeof value === 'string' && value.startsWith('acquisition_')
+    && safeExecutorFailureReason(value.slice(12)) === value.slice(12))return value;
+  return 'acquisition_failed';
+}
+function custody(closure) {
+  if(closure?.browserClosed !== true)return {state:'unknown'};
+  if(['signed_executor','signed_executor_lookup'].includes(closure.cleanupSource) && session(closure.sessionId)
+    && closure.cleanup?.unitStopped === true && closure.cleanup.cgroupEmpty === true
+    && closure.cleanup.namespaceDestroyed === true)return {state:'closed',source:closure.cleanupSource,sessionId:closure.sessionId,
+      cleanup:{unitStopped:true,cgroupEmpty:true,namespaceDestroyed:true}};
+  if(closure.browserDispatched === false && ['signed_no_launch','signed_no_launch_lookup','local_no_dispatch'].includes(closure.cleanupSource)
+    && closure.sessionId === undefined && closure.cleanup === undefined)return {state:'not_launched',source:closure.cleanupSource};
+  return {state:'unknown'};
+}
+export function recordRenderFailure(error, {reason,stage,durationMs,closure,receipt,reservationId} = {}) {
+  if(!(error instanceof Error))return;
+  const trace = {version:1,method:'node_browser',reason:safeReason(reason),
+    stage:['admission','transport','response_validation','executor','capture_validation','postflight'].includes(stage)?stage:'transport',
+    durationMs:Number.isFinite(durationMs)?Math.max(0,Math.min(60000,Math.ceil(durationMs))):null,custody:custody(closure)};
+  if(token(reservationId))trace.reservationId=reservationId;
+  if(receipt && token(receipt.requestId) && /^[a-f0-9]{64}$/u.test(receipt.sha256 ?? '')
+    && Number.isInteger(receipt.httpStatus) && receipt.httpStatus >= 200 && receipt.httpStatus <= 599) {
+    trace.receipt={requestId:receipt.requestId,sha256:receipt.sha256,httpStatus:receipt.httpStatus,keyId:'version1',authenticated:true};
+    if(receipt.executorReason !== undefined)trace.executorReason=safeExecutorFailureReason(receipt.executorReason);
+    if(['admission','execution'].includes(receipt.executorStage))trace.executorStage=receipt.executorStage;
+    if(typeof receipt.retryable === 'boolean')trace.retryable=receipt.retryable;
+  }
+  failures.set(error, trace);
+}
+export function readRenderFailure(error) {
+  const value = error instanceof Error ? failures.get(error) : null;
+  return value ? structuredClone(value) : null;
+}

package/src/page-extraction/html.js

@@ -0,0 +1,359 @@
+import { parse, defaultTreeAdapter } from 'parse5';
+
+const INERT = new Set(['script', 'style', 'template', 'noscript']);
+const BLOCKS = new Set(['p', 'li', 'article', 'section', 'div', 'td', 'dd', 'dt', 'blockquote']);
+// Where a browser puts a visual break. Used to decide whether two runs of text are separate words.
+// Everything NOT in here is treated as phrasing content — `span`, `b`, `a`, `em` — because a
+// browser renders `<span>Read</span><span>more</span>` as one word and so must we.
+const TEXT_BLOCKS = new Set([...BLOCKS, 'body', 'main', 'header', 'footer', 'nav', 'aside',
+  'figure', 'figcaption', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'ul', 'ol', 'dl', 'table', 'tr',
+  'th', 'thead', 'tbody', 'tfoot', 'caption', 'pre', 'form', 'fieldset', 'details', 'summary',
+  'address', 'hr', 'option', 'legend']);
+const EOF_ERRORS = new Set([
+  'eof-before-tag-name', 'eof-in-tag', 'eof-in-comment', 'eof-in-doctype',
+  'eof-in-element-that-can-contain-only-text', 'eof-in-script-html-comment-like-text',
+]);
+const attr = (node, name) => node.attrs?.find((value) => value.name === name)?.value;
+const normalize = (text) => text.replace(/\s+/gu, ' ').trim();
+
+function* walk(root, includeInert = false) {
+  const stack = [root];
+  while (stack.length) {
+    const node = stack.pop();
+    if (node.nodeName === '#comment' || (!includeInert && INERT.has(node.tagName))) continue;
+    yield node;
+    const children = node.childNodes ?? [];
+    for (let i = children.length - 1; i >= 0; i--) stack.push(children[i]);
+  }
+}
+
+/** The nearest ancestor a browser would break a line at, stopping at the subtree we were given. */
+function blockAncestor(node, root) {
+  let cursor = node.parentNode;
+  while (cursor && cursor !== root) {
+    if (cursor.tagName && TEXT_BLOCKS.has(cursor.tagName)) return cursor;
+    cursor = cursor.parentNode;
+  }
+  return root;
+}
+
+/**
+ * Text as a reader sees it, with a word boundary exactly where a browser draws one.
+ *
+ * **Two runs of text are separate words when they sit in different BLOCKS, not merely in different
+ * elements.** Without any separator, `<p>See settledestate.com</p><a>Guide</a>` read as
+ * `settledestate.comGuide`, which put a joined string into every occurrence context a customer
+ * sees. With a separator at every element boundary, `<a><span>Read</span><span>more</span></a>`
+ * read as `Read more` — and a browser renders that as one word, so anchor text and therefore the
+ * link signature would have disagreed with the page and emitted spurious change events.
+ *
+ * `normalize` collapses the runs this adds, so over-separating within a block is free and
+ * under-separating across one is not.
+ */
+function textContentResult(node, limit = 2048) {
+  let text = '';
+  let lastBlock = null;
+  let stopped = false;
+  const emit = (value, item) => {
+    const block = blockAncestor(item, node);
+    if (lastBlock !== null && block !== lastBlock) text += ' ';
+    text += value;
+    lastBlock = block;
+  };
+  for (const item of walk(node)) {
+    // A line break is a word boundary wherever it appears, block or not.
+    if (item.tagName === 'br') { text += ' '; lastBlock = null; }
+    if (item.nodeName === '#text') emit(item.value, item);
+    if (item.tagName === 'img') { const alt = attr(item, 'alt'); if (alt) emit(alt, item); }
+    if (text.length > limit * 2) { stopped = true; break; }
+  }
+  const normalized = normalize(text);
+  return { text: normalized.slice(0, limit), truncated: stopped || normalized.length > limit, limit };
+}
+
+function textContent(node, limit = 2048) {
+  return textContentResult(node, limit).text;
+}
+
+function contextFor(node) {
+  let context = node.parentNode ?? node;
+  for (let depth = 0; context.parentNode && depth < 4; depth++) {
+    if (BLOCKS.has(context.tagName) || context.tagName === 'body') break;
+    context = context.parentNode;
+  }
+  return textContent(context, 640);
+}
+
+// Landmark region of a link from its own ancestors in the one parse tree: the
+// nearest main/article/nav/footer/header/aside (or ARIA role), where a template
+// ancestor (navigation, footer, header, aside) wins over a nested main/article.
+// Links inside script/style/template/noscript have no region.
+const LANDMARK_TAGS = new Map([['main', 'main'], ['article', 'main'], ['nav', 'navigation'], ['footer', 'footer'], ['header', 'header'], ['aside', 'aside']]);
+const LANDMARK_ROLES = new Map([['main', 'main'], ['navigation', 'navigation'], ['contentinfo', 'footer'], ['banner', 'header'], ['complementary', 'aside']]);
+const TEMPLATE_REGIONS = new Set(['navigation', 'footer', 'header', 'aside']);
+const NO_REGION = new Set(['script', 'style', 'template', 'noscript']);
+function landmarkRegion(node) {
+  const chain = [];
+  for (let current = node; current; current = current.parentNode) {
+    if (NO_REGION.has(current.tagName)) return 'unknown';
+    chain.push(current);
+  }
+  let region = 'unknown';
+  for (let index = chain.length - 1; index >= 0; index--) {
+    const current = chain[index];
+    if (TEMPLATE_REGIONS.has(region)) break;
+    const role = current.attrs?.find(item => item.name === 'role')?.value;
+    region = LANDMARK_TAGS.get(current.tagName) ?? LANDMARK_ROLES.get(role) ?? region;
+  }
+  return region;
+}
+
+// Parser bounds. The defaults keep a single checked page cheap; a site crawl
+// raises them and keeps the first anchors of a long page instead of failing it
+// (truncateAnchors), reporting linksTruncated so the caller can mark coverage.
+export const PARSE_BOUNDS = Object.freeze({ elements: 25_000, anchors: 1000, parseMs: 500, truncateAnchors: false });
+
+// Main-content paragraphs for mention matching: the outermost paragraph-like
+// blocks in the page's main landmark (or outside every template landmark when
+// the page has none), with the text of links inside them wrapped in U+0002 and
+// U+0003 so a reader-visible passage keeps its words while a mention that is
+// already a link can be told apart.
+const PARAGRAPHS = new Set(['p', 'li', 'blockquote', 'td', 'dd', 'dt', 'figcaption', 'h2', 'h3', 'h4', 'h5', 'h6']);
+export const CONTENT_BLOCK_BOUNDS = Object.freeze({ blocks: 300, blockChars: 1500, totalChars: 24_000, minChars: 30 });
+export const LINK_TEXT_START = '\u0002', LINK_TEXT_END = '\u0003';
+function markedText(block, limit) {
+  let text = '', lastBlock = null, inLink = false;
+  for (const item of walk(block)) {
+    if (item.tagName === 'br') { text += ' '; lastBlock = null; }
+    if (item.nodeName !== '#text' && item.tagName !== 'img') continue;
+    const value = item.nodeName === '#text' ? item.value : attr(item, 'alt');
+    if (!value) continue;
+    let link = false;
+    for (let cursor = item.parentNode; cursor && cursor !== block.parentNode; cursor = cursor.parentNode) if (cursor.tagName === 'a') { link = true; break; }
+    const inner = blockAncestor(item, block);
+    if (lastBlock !== null && inner !== lastBlock) text += ' ';
+    if (link !== inLink) { text += link ? LINK_TEXT_START : LINK_TEXT_END; inLink = link; }
+    text += value; lastBlock = inner;
+    if (text.length > limit * 2) break;
+  }
+  if (inLink) text += LINK_TEXT_END;
+  return normalize(text).slice(0, limit);
+}
+function contentBlocks(nodes) {
+  const bounds = CONTENT_BLOCK_BOUNDS;
+  const hasMain = nodes.some(node => LANDMARK_TAGS.get(node.tagName) === 'main' || LANDMARK_ROLES.get(attr(node, 'role')) === 'main');
+  const blocks = [];
+  let total = 0, truncated = false;
+  for (const node of nodes) {
+    if (!PARAGRAPHS.has(node.tagName)) continue;
+    // A block inside another block, or inside a link (a card whose whole body is
+    // one anchor), is skipped: any words in a linked block are already link text.
+    let nested = false;
+    for (let cursor = node.parentNode; cursor; cursor = cursor.parentNode) if (PARAGRAPHS.has(cursor.tagName) || cursor.tagName === 'a') { nested = true; break; }
+    if (nested) continue;
+    const region = landmarkRegion(node);
+    if (hasMain ? region !== 'main' : region !== 'unknown') continue;
+    const text = markedText(node, bounds.blockChars);
+    if (text.replace(/[\u0002\u0003]/gu, '').length < bounds.minChars) continue;
+    if (blocks.length >= bounds.blocks || total + text.length > bounds.totalChars) { truncated = true; break; }
+    blocks.push(text); total += text.length;
+  }
+  return { blocks, truncated };
+}
+
+export function parseDocument(html, pageUrl, { includeMetadata = false, textLimit = null, landmarkRegions = false, content = false, bounds = PARSE_BOUNDS } = {}) {
+  const limits = { ...PARSE_BOUNDS, ...bounds };
+  if (typeof html !== 'string') throw new TypeError('HTML must be a string');
+  const errors = [];
+  let elements = 0;
+  let anchorElements = 0;
+  const parserStart = performance.now();
+  const treeAdapter = {
+    ...defaultTreeAdapter,
+    createElement(tagName, namespaceURI, attrs) {
+      // Bound parser expansion and evidence output independently of body bytes.
+      if (++elements > limits.elements || attrs.length > 256 || performance.now() - parserStart > limits.parseMs) {
+        const error = new Error('document_too_complex');
+        error.reason = 'document_too_complex';
+        throw error;
+      }
+      if ((tagName === 'a' || tagName === 'area') && ++anchorElements > limits.anchors && !limits.truncateAnchors) {
+        const error = new Error('too_many_link_occurrences');
+        error.reason = 'too_many_link_occurrences';
+        throw error;
+      }
+      return defaultTreeAdapter.createElement(tagName, namespaceURI, attrs);
+    },
+  };
+  const document = parse(html, {
+    treeAdapter,
+    scriptingEnabled: true,
+    sourceCodeLocationInfo: true,
+    onParseError: ({ code }) => { if (EOF_ERRORS.has(code) && errors.length < 8) errors.push(code); },
+  });
+  let baseUrl = pageUrl;
+  let baseFound = false;
+  let canonical = null;
+  const links = [];
+  const meta = [];
+  let title = '';
+  let hasScript = false;
+  let hasPasswordInput = false;
+  let hasChallengeScript = false;
+  let visibleText = '';
+  const nodes = [...walk(document)];
+  for (const node of walk(document, true)) {
+    if (node.tagName === 'script') {
+      hasScript = true;
+      if ((attr(node, 'src') ?? '').includes('/cdn-cgi/challenge-platform/')) hasChallengeScript = true;
+    }
+  }
+  for (const node of nodes) {
+    if (node.tagName === 'body') visibleText = textContent(node, 160);
+    if (node.tagName === 'input' && (attr(node, 'type') ?? '').toLowerCase() === 'password') hasPasswordInput = true;
+    if (node.tagName === 'base' && !baseFound && attr(node, 'href') !== undefined) {
+      // Browsers use only the first base element with an href, even if invalid.
+      baseFound = true;
+      if (attr(node, 'href').length > 8192) {
+        const error = new Error('base_url_too_long');
+        error.reason = 'base_url_too_long';
+        throw error;
+      }
+      try {
+        const base = new URL(attr(node, 'href'), pageUrl);
+        if (['http:', 'https:'].includes(base.protocol)) baseUrl = base.href;
+      } catch { /* Browser-compatible fallback to the document URL. */ }
+    }
+    if (node.tagName === 'meta') {
+      const name = (attr(node, 'name') ?? '').toLowerCase();
+      if ((['robots', 'googlebot', 'bingbot', 'linktrailbot'].includes(name)
+        || (attr(node, 'http-equiv') ?? '').toLowerCase() === 'refresh')
+        && (attr(node, 'content') ?? '').length > 2048) {
+        const error = new Error('page_directive_too_long');
+        error.reason = 'page_directive_too_long';
+        throw error;
+      }
+      if (['robots', 'googlebot', 'bingbot', 'linktrailbot'].includes(name)) {
+        meta.push({ name, content: (attr(node, 'content') ?? '').slice(0, 2048) });
+      }
+      if ((attr(node, 'http-equiv') ?? '').toLowerCase() === 'refresh') {
+        meta.push({ name: 'refresh', content: (attr(node, 'content') ?? '').slice(0, 2048) });
+      }
+      if (meta.length > 16) {
+        const error = new Error('too_many_page_directives');
+        error.reason = 'too_many_page_directives';
+        throw error;
+      }
+    }
+    if (node.tagName === 'title' && !title) title = textContent(node, 256);
+    // Browsers honour the first canonical link element only.
+    if (node.tagName === 'link' && canonical === null
+      && (attr(node, 'rel') ?? '').toLowerCase().split(/\s+/u).includes('canonical')) {
+      const href = attr(node, 'href') ?? '';
+      if (href && href.length <= 8192) {
+        try {
+          const resolved = new URL(href, baseUrl);
+          if (['http:', 'https:'].includes(resolved.protocol)) canonical = resolved.href;
+        } catch { /* An unparseable canonical is no canonical, exactly as a browser treats it. */ }
+      }
+    }
+  }
+  let anchorsSeen = 0, linksTruncated = false;
+  for (const node of nodes) {
+    if (!['a', 'area'].includes(node.tagName)) continue;
+    if (++anchorsSeen > limits.anchors) { linksTruncated = true; break; }
+    const href = attr(node, 'href');
+    if (href === undefined) continue;
+    if (href.length > 8192) {
+      const error = new Error('link_url_too_long');
+      error.reason = 'link_url_too_long';
+      throw error;
+    }
+    let target;
+    try { target = new URL(href, baseUrl); } catch { continue; }
+    if (!['http:', 'https:'].includes(target.protocol) || target.username || target.password) continue;
+    const location = node.sourceCodeLocation;
+    links.push({
+      href,
+      targetUrl: target.href,
+      anchor: node.tagName === 'area' ? normalize(attr(node, 'alt') ?? '') : textContent(node, 1024),
+      rel: [...new Set((attr(node, 'rel') ?? '').toLowerCase().split(/\s+/u).filter(Boolean))],
+      context: contextFor(node),
+      locator: location ? { line: location.startLine, column: location.startCol, offset: location.startOffset } : null,
+      visibility: 'not_rendered',
+      ...(landmarkRegions ? { region: landmarkRegion(node) } : {}),
+    });
+  }
+  const result = { links, linksTruncated, meta, title, baseUrl, canonical, hasScript, hasPasswordInput, hasChallengeScript, visibleText, incomplete: errors.length > 0, parseErrors: errors };
+  if (content) result.content = contentBlocks(nodes);
+  if (includeMetadata) {
+    const description = nodes.find(node => node.tagName === 'meta' && (attr(node, 'name') ?? '').toLowerCase() === 'description');
+    result.metadata = {
+      title: title || null,
+      h1s: nodes.filter(node => node.tagName === 'h1').slice(0, 5).map(node => textContent(node, 500)),
+      meta_description: description ? normalize(attr(description, 'content') ?? '').slice(0, 500) || null : null,
+      canonical,
+      link_counts: { total: links.length },
+      asset_candidate_reasons: [],
+    };
+  }
+  if (textLimit !== null) {
+    if (!Number.isInteger(textLimit) || textLimit < 1 || textLimit > 512 * 1024) throw new RangeError('Invalid static text limit');
+    const body = nodes.find(node => node.tagName === 'body') ?? document;
+    result.staticText = textContentResult(body, textLimit);
+  }
+  return result;
+}
+
+/**
+ * The page's text as a READER sees it, reusing the same walk the parser already uses.
+ *
+ * `walk` already skips `script`, `style`, `template`, `noscript` and comments, which is the
+ * property that matters: a brand name inside a JSON payload or an analytics config is not a
+ * mention anyone read. `parseDocument` caps its own copy at 160 characters because it only needs
+ * to know whether a page is an empty shell; a mention scan needs the whole document, so it asks
+ * for it explicitly rather than raising a cap every check would pay for.
+ *
+ * Takes no base URL, deliberately: text is not resolved against anything, and a parameter that
+ * looks like it matters and does not is how a caller comes to believe the output is absolute.
+ */
+export function pageText(html, limit = 512 * 1024) {
+  return parseDocument(String(html), 'https://page-text.invalid/', { textLimit: limit }).staticText.text;
+}
+
+/** All HTTP(S) anchor/area occurrences in static HTML; never claims rendered visibility. */
+export function parseLinks(html, baseUrl) {
+  return parseDocument(html, new URL(baseUrl).href).links;
+}
+
+function googleDirectiveTokens(meta, header) {
+  const values = meta.filter(item => item.name === 'robots' || item.name === 'googlebot').map(item => item.content);
+  let agent = '*';
+  // A targeted X-Robots-Tag applies to the following comma-separated tokens
+  // until another agent prefix. Directive arguments themselves are not agents.
+  const argumentDirectives = new Set(['unavailable_after', 'max-snippet', 'max-image-preview', 'max-video-preview']);
+  for (let part of String(header ?? '').toLowerCase().split(',')) {
+    const prefix = /^\s*([\w-]+)\s*:\s*(.*)$/u.exec(part);
+    if (prefix && !argumentDirectives.has(prefix[1])) { agent = prefix[1]; part = prefix[2]; }
+    if (agent === '*' || agent === 'googlebot') values.push(part);
+  }
+  return values.join(',').toLowerCase().split(/[\s,]+/u);
+}
+
+export function pageDirectives(meta, header) {
+  const xRobotsTag = header ? [header] : [];
+  // Preserve targeted header directives without incorrectly applying them to all bots.
+  const genericHeader = header && !/(?:^|,)\s*[\w-]+\s*:/u.test(header) ? header : '';
+  const generic = [genericHeader, ...meta.filter((item) => item.name === 'robots').map((item) => item.content)]
+    .join(',').toLowerCase().split(/[\s,]+/u);
+  const google = googleDirectiveTokens(meta, header);
+  return {
+    meta,
+    xRobotsTag,
+    noindex: generic.includes('noindex') || generic.includes('none'),
+    nofollow: generic.includes('nofollow') || generic.includes('none'),
+    googlebot: { noindex: google.includes('noindex') || google.includes('none'),
+      nofollow: google.includes('nofollow') || google.includes('none') },
+    indexingStatus: 'not_checked',
+  };
+}

package/src/page-extraction/index.js

@@ -0,0 +1,30 @@
+import { parseDocument, pageDirectives } from './html.js';
+import { validateCaptureReceipt, PageAcquisitionError } from '../page-acquisition/contracts.js';
+
+export { parseDocument, parseLinks, pageText, pageDirectives } from './html.js';
+export { extractPageMetadata } from './metadata.js';
+export const STATIC_TEXT_LIMIT = 512 * 1024;
+
+/**
+ * Extract from an already acquired document. A receipt is checked when supplied;
+ * legacy HTML remains caller-supplied evidence, without invented fetch provenance.
+ * Static text is not a claim about CSS visibility, iframes or shadow DOM.
+ */
+export function extractPage(html, { baseUrl, receipt = null, textLimit = STATIC_TEXT_LIMIT, xRobotsTag = null, landmarkRegions = false, content = false, bounds } = {}) {
+  const capture = receipt === null ? null : validateCaptureReceipt(receipt, { finalUrl: baseUrl });
+  if (capture && (!['raw_html', 'rendered_dom'].includes(capture.representation) || capture.httpStatus < 200 || capture.httpStatus >= 300)) throw new PageAcquisitionError('capture_invalid');
+  const parsed = parseDocument(html, baseUrl, { includeMetadata: true, textLimit, landmarkRegions, content, ...(bounds ? { bounds } : {}) });
+  return {
+    ...parsed,
+    directives: pageDirectives(parsed.meta, xRobotsTag),
+    coverage: {
+      documentComplete: !parsed.incomplete,
+      textComplete: !parsed.staticText.truncated,
+      bodyComplete: capture?.body.complete ?? null,
+      transportQualified: capture?.transport.qualified ?? false,
+      mainDocument: true, iframes: false, shadowRoots: false,
+    },
+    provenance: capture ? { kind: 'capture_receipt', receipt: capture }
+      : { kind: 'caller_supplied_html', acquiredAt: null, httpStatus: null },
+  };
+}