agentlinkops 0.6.11 → 0.6.16

package/src/page-acquisition/contracts.js

@@ -0,0 +1,67 @@
+import {validatePublicUrl} from '../verifier/url.js';
+import {recoveryEligibility} from './policy.js';
+
+export const CAPTURE_VERSION = 1;
+export const ACQUISITION_LIMITS = Object.freeze({maxBytes:2*1024*1024,documentBytes:10*1024*1024,robotsBytes:512*1024,maxRedirects:5,timeoutMs:20_000,maxAttempts:4});
+export const PURPOSE_PROFILES = Object.freeze({
+  placement:Object.freeze({representation:'raw_html',allowAlternate:true,allowRender:true}),
+  target:Object.freeze({representation:'raw_html',allowAlternate:true,allowRender:false}),
+  research:Object.freeze({representation:'raw_html',allowAlternate:false,allowRender:false}),
+  document:Object.freeze({representation:'document',allowAlternate:false,allowRender:false}),
+  free:Object.freeze({representation:'raw_html',allowAlternate:false,allowRender:false}),
+});
+export class PageAcquisitionError extends Error {
+  constructor(reason){super(reason);this.name='PageAcquisitionError';this.reason=reason;}
+}
+const fail=reason=>{throw new PageAcquisitionError(reason);};
+const hashPattern=/^[a-f0-9]{64}$/u;
+export function boundedAcquisitionLimit(value,fallback,cap){
+  if(value===undefined||value===null)return fallback;
+  if(typeof value!=='number'||!Number.isFinite(value)||value<1)fail('acquisition_limits_invalid');
+  return Math.min(Math.floor(value),cap);
+}
+const pin=value=>typeof value==='string'&&value.length>0&&value.length<=128&&value.trim()===value&&!/[\x00-\x20\x7f]/u.test(value);
+const captureByteLimit=representation=>representation==='document'?ACQUISITION_LIMITS.documentBytes:ACQUISITION_LIMITS.maxBytes;
+export async function sha256Bytes(bytes){return [...new Uint8Array(await crypto.subtle.digest('SHA-256',bytes))].map(v=>v.toString(16).padStart(2,'0')).join('');}
+export async function decodedDigest(html){return sha256Bytes(new TextEncoder().encode(html));}
+
+export function validateCaptureReceipt(receipt,{finalUrl,allowUnqualified=false}={}){
+  if(!receipt||receipt.version!==CAPTURE_VERSION||!['raw_html','rendered_dom','document'].includes(receipt.representation)
+    ||!validatePublicUrl(receipt.requestedUrl).valid||!validatePublicUrl(receipt.finalUrl).valid
+    ||!Number.isInteger(receipt.httpStatus)||receipt.httpStatus<100||receipt.httpStatus>599
+    ||typeof receipt.acquiredAt!=='string'||!Number.isFinite(Date.parse(receipt.acquiredAt))
+    ||!receipt.body||typeof receipt.body.complete!=='boolean'||typeof receipt.body.truncated!=='boolean'
+    ||!Number.isSafeInteger(receipt.body.bytes)||receipt.body.bytes<0||receipt.body.bytes>captureByteLimit(receipt.representation)||!hashPattern.test(receipt.body.sha256??'')
+    ||!hashPattern.test(receipt.body.decodedSha256??'')||!receipt.transport||!pin(receipt.transport.id)
+    ||!pin(receipt.transport.version)||typeof receipt.transport.qualified!=='boolean'
+    ||(receipt.body.charset!==undefined&&receipt.body.charset!==null&&(typeof receipt.body.charset!=='string'||receipt.body.charset.length>64))
+    ||!receipt.coverage||receipt.coverage.mainDocument!==true||typeof receipt.coverage.iframes!=='boolean'
+    ||typeof receipt.coverage.shadowRoots!=='boolean')fail('capture_invalid');
+  if(finalUrl&&finalUrl!==receipt.finalUrl)fail('capture_source_mismatch');
+  if(!receipt.body.complete||receipt.body.truncated)fail('capture_incomplete');
+  if(!allowUnqualified&&!receipt.transport.qualified)fail('capture_unqualified');
+  if(receipt.body.charset!=null){try{new TextDecoder(receipt.body.charset,{fatal:true});}catch{fail('capture_invalid_charset');}}
+  return structuredClone(receipt);
+}
+export async function validateDecodedCapture(receipt,html){
+  const valid=validateCaptureReceipt(receipt);
+  if(typeof html!=='string')fail('capture_digest_mismatch');
+  const limit=captureByteLimit(valid.representation);
+  // Supported legacy decoders cannot produce more UTF-16 units than consumed bytes;
+  // their UTF-8 re-encoding can use up to three bytes per unit (e.g. Windows-1252).
+  if(html.length>limit)fail('capture_decoded_too_large');
+  const bytes=new TextEncoder().encode(html);
+  if(bytes.byteLength>limit*3)fail('capture_decoded_too_large');
+  if(await sha256Bytes(bytes)!==valid.body.decodedSha256)fail('capture_digest_mismatch');
+  return valid;
+}
+
+export function acquisitionRouteEligible(results){
+  return Array.isArray(results)&&results.length>0&&results.every(r=>recoveryEligibility(r,'node_http').allowed);
+}
+
+export function captureCacheKey({workspaceId,projectId,url,representation='raw_html',profileVersion='1',transportVersion,locale='en-US',viewport='none'}){
+  if(![workspaceId,projectId,transportVersion].every(x=>typeof x==='string'&&x.length>0)||!validatePublicUrl(url).valid)fail('capture_cache_scope_invalid');
+  // Tenant scope and representation are inseparable from artifact identity.
+  return JSON.stringify([workspaceId,projectId,url,representation,profileVersion,transportVersion,locale,viewport]);
+}

package/src/page-acquisition/documents.js

@@ -0,0 +1,166 @@
+import { validatePublicUrl } from '../verifier/url.js';
+import { PRODUCT_TOKEN } from './http.js';
+import { VerificationError, cancelBody, withAbort, readBoundedBytes, decodeBytes, inflateGzipBounded } from './bytes.js';
+
+const MAX_DOCUMENT_BYTES = 10 * 1024 * 1024;
+const MAX_ROBOTS_BYTES = 512 * 1024;
+const MAX_TIMEOUT_MS = 20_000;
+const REDIRECTS = new Set([301, 302, 303, 307, 308]);
+
+function bounded(value, fallback, cap, minimum = 0) {
+  return Number.isFinite(value) ? Math.min(Math.max(Math.floor(value), minimum), cap) : fallback;
+}
+
+function abortableWait(ms, { signal } = {}) {
+  return new Promise((resolve, reject) => {
+    if (signal?.aborted) { reject(new VerificationError('timeout')); return; }
+    const onAbort = () => { clearTimeout(timer); reject(new VerificationError('timeout')); };
+    const timer = setTimeout(() => { signal?.removeEventListener('abort', onAbort); resolve(); }, ms);
+    signal?.addEventListener('abort', onAbort, { once: true });
+  });
+}
+
+/**
+ * Bounded bootstrap documents: robots.txt must not recursively authorize itself.
+ * The caller owns transport egress qualification; lexical URL checks are not DNS proof.
+ * Fetching an XML document here does not enroll a crawl or establish link presence.
+ *
+ * One absolute budget covers pacing, every redirect, headers, body and inflation.
+ * `signal` and `deadlineAt` can shorten that budget, never extend it.
+ */
+export function createDocumentTransport({
+  fetchImpl = globalThis.fetch, timeoutMs = MAX_TIMEOUT_MS, maxBytes = MAX_DOCUMENT_BYTES,
+  maxRedirects = 5, hostDelayMs = 2_000, now = () => Date.now(), wait = abortableWait,
+  remainingBytes = null, retainKnownAbsentRobots = false,
+} = {}) {
+  const byteCap = bounded(maxBytes, MAX_DOCUMENT_BYTES, MAX_DOCUMENT_BYTES, 1);
+  const timeout = bounded(timeoutMs, MAX_TIMEOUT_MS, MAX_TIMEOUT_MS, 1);
+  const redirectCap = bounded(maxRedirects, 5, 5);
+  const nextAllowed = new Map();
+
+  async function pace(host, signal) {
+    const delay = (nextAllowed.get(host) ?? 0) - now();
+    if (delay > 0) await withAbort(Promise.resolve().then(() => wait(delay, { signal })), signal);
+    if (signal.aborted) throw new VerificationError('timeout');
+    nextAllowed.set(host, now() + Math.max(0, hostDelayMs));
+  }
+
+  async function fetchText(rawUrl, {
+    accept = 'text/plain,application/xml,text/xml', signal = null, deadlineAt = null,
+    purpose = 'document',
+  } = {}) {
+    const checked = validatePublicUrl(rawUrl);
+    if (!checked.valid) return { ok: false, url: rawUrl, reason: `unsafe_url:${checked.reason}` };
+    let url = checked.url;
+    const controller = new AbortController();
+    const ownerAbort = () => controller.abort();
+    signal?.addEventListener('abort', ownerAbort, { once: true });
+    if (signal?.aborted) controller.abort();
+    const remaining = Math.min(timeout, Number.isFinite(deadlineAt) ? deadlineAt - Date.now() : timeout);
+    const absoluteDeadline = Date.now() + remaining;
+    if (remaining <= 0) controller.abort();
+    const timer = setTimeout(() => controller.abort(), Math.max(0, remaining));
+    const limit = purpose === 'robots' ? Math.min(byteCap, MAX_ROBOTS_BYTES) : byteCap;
+    const seen = new Set();
+    let status;
+    let phase = 'request';
+    try {
+      for (let redirects = 0; redirects <= redirectCap; redirects++) {
+        status = undefined;
+        if (controller.signal.aborted) throw new VerificationError('timeout');
+        if (seen.has(url)) return { ok: false, url, reason: 'redirect_loop' };
+        seen.add(url);
+        phase = 'request';
+        await pace(new URL(url).hostname, controller.signal);
+        const pending = Promise.resolve().then(() => {
+          if (controller.signal.aborted) throw new VerificationError('timeout');
+          return fetchImpl(url, { method: 'GET', redirect: 'manual', credentials: 'omit',
+            signal: controller.signal, headers: { 'User-Agent': `${PRODUCT_TOKEN}/0.1`, Accept: accept } });
+        });
+        // A custom transport may ignore abort. Dispose of its late body as soon as it appears.
+        pending.then(response => { if (controller.signal.aborted) cancelBody(response); }, () => {});
+        const response = await withAbort(pending, controller.signal);
+        status = response.status;
+        let followed;
+        try { followed = response.redirected || (response.url && new URL(response.url).href !== url); }
+        catch { followed = true; }
+        if (followed) {
+          cancelBody(response);
+          return { ok: false, url, status, reason: 'transport_followed_redirect' };
+        }
+        if (response.headers.get('cf-mitigated') === 'challenge') {
+          cancelBody(response);
+          return { ok: false, url, status, reason: 'access_challenge' };
+        }
+        if (REDIRECTS.has(status)) {
+          cancelBody(response);
+          const location = response.headers.get('location');
+          let next;
+          try { next = location ? validatePublicUrl(new URL(location, url).href) : null; }
+          catch { next = null; }
+          if (!next?.valid) return { ok: false, url, reason: 'unsafe_or_missing_redirect' };
+          url = next.url;
+          continue;
+        }
+        if (status === 404 || status === 410) {
+          // Private durable discovery may retain truthful robots404 evidence on
+          // this SAME response. Ordinary callers keep their exact old result.
+          if (retainKnownAbsentRobots === true && purpose === 'robots' && status === 404) {
+            phase = 'body';
+            const remaining = remainingBytes ? remainingBytes() : limit;
+            const readLimit = Number.isFinite(remaining) ? Math.min(limit, Math.max(0, Math.floor(remaining))) : 0;
+            if (!readLimit) { cancelBody(response); throw new VerificationError('body_too_large'); }
+            const body = await readBoundedBytes(response, readLimit, controller.signal);
+            let data = body.data;
+            const gzipped = data.length >= 2 && data[0] === 0x1f && data[1] === 0x8b;
+            if (gzipped) { phase = 'inflate'; data = (await inflateGzipBounded(data, readLimit, controller.signal)).data; }
+            phase = 'decode'; const text = decodeBytes(data);
+            if (controller.signal.aborted || Date.now() >= absoluteDeadline) throw new VerificationError('timeout');
+            return { ok: false, url, final_url: url, status, reason: 'not_found', text, bytes: body.bytes, gzipped };
+          }
+          cancelBody(response);
+          return { ok: false, url, status, reason: 'not_found' };
+        }
+        if (!response.ok) {
+          cancelBody(response);
+          return { ok: false, url, status, reason: `http_${status}` };
+        }
+        phase = 'body';
+        // Authorization hooks may read robots for a redirected origin. Take the
+        // shared remaining allowance after those hooks, immediately before body use.
+        const remaining = remainingBytes ? remainingBytes() : limit;
+        const readLimit = Number.isFinite(remaining) ? Math.min(limit, Math.max(0, Math.floor(remaining))) : 0;
+        if (!readLimit) { cancelBody(response); throw new VerificationError('body_too_large'); }
+        const body = await readBoundedBytes(response, readLimit, controller.signal);
+        let data = body.data;
+        const gzipped = data.length >= 2 && data[0] === 0x1f && data[1] === 0x8b;
+        if (gzipped) {
+          phase = 'inflate';
+          data = (await inflateGzipBounded(data, readLimit, controller.signal)).data;
+        }
+        if (controller.signal.aborted) throw new VerificationError('timeout');
+        phase = 'decode';
+        const text = decodeBytes(data);
+        if (controller.signal.aborted || Date.now() >= absoluteDeadline) throw new VerificationError('timeout');
+        if (purpose === 'robots' && (/text\/html/iu.test(response.headers.get('content-type') ?? '')
+          || /<(?:!doctype|html|body)\b/iu.test(text))) {
+          return { ok: false, url, status, reason: 'robots_not_plain_text' };
+        }
+        return { ok: true, url, final_url: url, status, text, bytes: body.bytes, gzipped };
+      }
+      return { ok: false, url, reason: 'redirect_limit' };
+    } catch (error) {
+      let reason = error?.reason ?? (phase === 'request' ? 'timeout_or_network_error' : 'read_failed');
+      if (phase === 'inflate' && reason !== 'timeout') {
+        reason = reason === 'body_too_large' ? 'sitemap_too_large' : 'sitemap_gunzip_failed';
+      }
+      return { ok: false, url, ...(status === undefined ? {} : { status }), reason };
+    } finally {
+      clearTimeout(timer);
+      signal?.removeEventListener('abort', ownerAbort);
+      controller.abort();
+    }
+  }
+
+  return { fetchText };
+}

package/src/page-acquisition/http.js

@@ -0,0 +1,307 @@
+import { validatePublicUrl } from '../verifier/url.js';
+import { robotsDecision } from '../verifier/robots.js';
+import { VerificationError, readBounded, readBodyDiagnostic, withAbort, cancelBody } from './bytes.js';
+
+export { VerificationError, readBounded } from './bytes.js';
+
+/** One shared budget for robots, redirects, hooks and source-body reads. */
+export const PRODUCT_TOKEN = 'LinktrailBot';
+
+// A day. Anything longer is a publisher saying "not today", and a scheduler that honoured a
+// month-long Retry-After would silently retire the check instead of retrying it.
+export const MAX_RETRY_AFTER_SECONDS = 86_400;
+
+/**
+ * RFC 9110 `Retry-After`: delta-seconds or an HTTP-date. Returns null for anything else.
+ *
+ * Both forms appear in the wild and a parser that reads only the integer form silently discards
+ * the date form, which is the one a CDN tends to send.
+ */
+export function retryAfterSeconds(value, now = Date.now()) {
+  if (!value) return null;
+  const text = String(value).trim();
+  if (/^\d+$/u.test(text)) {
+    const seconds = Number(text);
+    return Number.isFinite(seconds) ? Math.min(seconds, MAX_RETRY_AFTER_SECONDS) : null;
+  }
+  // `Date.parse` accepts far more than an HTTP-date — it reads "-5" as a year and returns a real
+  // timestamp, which turned a malformed header into a wait of zero. An IMF-fixdate always carries
+  // a month and day name, so requiring a letter is enough to keep the loose parser off junk.
+  if (!/[A-Za-z]/u.test(text)) return null;
+  const at = Date.parse(text);
+  if (!Number.isFinite(at)) return null;
+  // A date already in the past means "now", never a negative wait.
+  return Math.min(Math.max(0, Math.ceil((at - now) / 1000)), MAX_RETRY_AFTER_SECONDS);
+}
+
+export function publicFetcher(options, signal) {
+  const fetchImpl = options.fetchImpl ?? globalThis.fetch;
+  // A caller may share one robots record per origin across consecutive captures
+  // (a bounded scan step); by default every fetcher reads robots itself.
+  const robotsByOrigin = options.robotsCache instanceof Map ? options.robotsCache : new Map();
+  // The product token is what robots matches on and never changes. A contact URL may be appended
+  // so a publisher who sees us in their logs can find out who we are — the convention every
+  // reputable crawler follows, and the thing most likely to move a reflexive 403 or 429.
+  //
+  // It defaults to ABSENT on purpose. It was written before AgentLinkOps had a public domain, and a bot
+  // advertising a URL that does not resolve is worse than one advertising nothing: it reads as
+  // impersonation rather than identification. Set it when a real domain exists.
+  const userAgent = options.userAgentContact
+    ? `${PRODUCT_TOKEN}/0.1 (+${String(options.userAgentContact).slice(0, 200)})`
+    : `${PRODUCT_TOKEN}/0.1`;
+  const maxRedirects = Math.min(options.maxRedirects ?? 5, 5);
+  const maxBytes = Math.min(options.maxBytes ?? 2 * 1024 * 1024, 2 * 1024 * 1024);
+  // Feeds and sitemaps may ask for up to 10 MiB; pages stay at the page bound.
+  const documentMaxBytes = Math.min(options.maxBytes ?? 2 * 1024 * 1024, 10 * 1024 * 1024);
+  let monitorOverrideConsumed = false;
+
+  async function request(url, context = {}) {
+    const checked = validatePublicUrl(url);
+    if (!checked.valid) throw new VerificationError(`unsafe_url:${checked.reason}`);
+    if (options.beforeFetch) await withAbort(Promise.resolve().then(() => options.beforeFetch(checked.url, { ...context, signal })), signal);
+    let response;
+    try {
+      const pending = Promise.resolve().then(() => {
+        if (signal.aborted) throw new VerificationError('timeout');
+        // Conditional headers ride only on the request that earned them. A validator belongs
+        // to one resource; across a redirect hop it would be re-sent to a different one, so
+        // the caller drops validators after the first hop and the redirect loop revalidates
+        // the destination from scratch rather than trusting a moved resource's old validators.
+        const conditional = {};
+        if (context.kind === 'source' && context.validators?.etag) conditional['If-None-Match'] = context.validators.etag;
+        if (context.kind === 'source' && context.validators?.lastModified) conditional['If-Modified-Since'] = context.validators.lastModified;
+        return fetchImpl(checked.url, {
+          method: 'GET',
+          redirect: 'manual',
+          credentials: 'omit',
+          signal,
+          headers: {
+            'User-Agent': userAgent,
+            Accept: context.kind === 'robots' ? 'text/plain' : context.accept ?? 'text/html',
+            ...conditional,
+          },
+        });
+      });
+      // An injected transport may ignore cancellation. Its eventual body still
+      // belongs to this aborted attempt and must never remain open or be read.
+      pending.then(late => { if (signal.aborted) cancelBody(late); }, () => {});
+      response = await withAbort(pending, signal);
+    } catch (error) {
+      if (error instanceof VerificationError) throw error;
+      throw new VerificationError(signal.aborted ? 'timeout' : 'network_error');
+    }
+    // Custom transports must never silently follow redirects or bypass destination checks.
+    if (response.redirected || (response.url && new URL(response.url).href !== checked.url)) {
+      cancelBody(response);
+      throw new VerificationError('transport_followed_redirect');
+    }
+    return response;
+  }
+
+  async function fetchRobots(origin, diagnostic) {
+    let url = `${origin}/robots.txt`;
+    const seen = new Set();
+    for (let redirects = 0; redirects <= maxRedirects; redirects++) {
+      diagnostic.robotsUrl = url;
+      diagnostic.httpStatus = null;
+      if (seen.has(url)) throw new VerificationError('robots_redirect_loop');
+      seen.add(url);
+      const response = await request(url, { kind: 'robots' });
+      diagnostic.httpStatus = response.status;
+      // A challenge replaces the requested resource, even when its status would
+      // otherwise mean missing robots or a redirect. Never infer policy from it.
+      if (response.headers.get('cf-mitigated') === 'challenge') {
+        cancelBody(response);
+        const retry = retryAfterSeconds(response.headers.get('retry-after'));
+        if (retry !== null) diagnostic.retryAfterSeconds = retry;
+        throw new VerificationError('robots_access_challenge', retry === null ? {} : { retryAfterSeconds: retry });
+      }
+      if ([301, 302, 303, 307, 308].includes(response.status)) {
+        cancelBody(response);
+        const location = response.headers.get('location');
+        if (!location) throw new VerificationError('robots_redirect_missing_location');
+        const next = validatePublicUrl(new URL(location, url).href);
+        if (!next.valid) throw new VerificationError(`unsafe_robots_redirect:${next.reason}`);
+        if (diagnostic.redirects.length < 5) diagnostic.redirects.push({ from: url, to: next.url, status: response.status });
+        url = next.url;
+        continue;
+      }
+      // RFC 9309 §2.3.1.3 and Google's implementation agree: a 4xx on robots.txt means the file is
+      // UNAVAILABLE, and an unavailable robots.txt imposes no restrictions. Only 404 and 410 were
+      // honoured that way here, so a 401 or 403 — a WAF answering before the file is even looked
+      // for — was read as "this publisher forbids us" and refused the whole fetch. Measured on 800
+      // real destinations, that was 39 refusals, about 5% of the run, attributed to publishers who
+      // had said nothing.
+      //
+      // Proceeding is also BETTER REPORTING even when the page then fails: `target_http_403` says
+      // the publisher refused the page, where `robots_http_403` said we never asked.
+      if (response.status >= 400 && response.status < 500 && response.status !== 429) {
+        diagnostic.bodyDiagnostic = await readBodyDiagnostic(response, signal);
+        return { text: '', reason: response.status === 404 || response.status === 410 ? 'robots_not_found' : `robots_unavailable_${response.status}` };
+      }
+      // 429 and 5xx are the other half of the same rule and stay a refusal: they mean TEMPORARILY
+      // unavailable, and the spec's answer to that is to stay away rather than to assume consent.
+      if (!response.ok) {
+        diagnostic.bodyDiagnostic = await readBodyDiagnostic(response, signal);
+        const retry = retryAfterSeconds(response.headers.get('retry-after'));
+        if (retry !== null) diagnostic.retryAfterSeconds = retry;
+        throw new VerificationError(`robots_http_${response.status}`, retry === null ? {} : { retryAfterSeconds: retry });
+      }
+      const body = await readBounded(response, 512 * 1024, signal);
+      if (/text\/html/iu.test(response.headers.get('content-type') ?? '') || /<(?:!doctype|html|body)\b/iu.test(body.text)) {
+        diagnostic.bodyDiagnostic = await readBodyDiagnostic(new Response(body.data,{headers:{'content-length':String(body.bytes)}}),signal);
+        throw new VerificationError('robots_not_plain_text');
+      }
+      return { text: body.text, reason: 'robots_fetched' };
+    }
+    throw new VerificationError('robots_redirect_limit');
+  }
+
+  async function checkRobots(url, trace, permitMonitorPolicy = false) {
+    const origin = new URL(url).origin;
+    const diagnostic = { sourceUrl: url, robotsUrl: `${origin}/robots.txt`, httpStatus: null,
+      productToken: PRODUCT_TOKEN, allowed: null, reason: null, matchedRule: null,
+      crawlDelaySeconds: null, fetched: null, redirects: [],
+      googlebot: { productToken: 'Googlebot', allowed: null, reason: null, matchedRule: null, crawlDelaySeconds: null } };
+    // Write before fetching/evaluating so refusals keep the failing hop, not the
+    // last allowed origin. Both histories are capped by our redirect budget.
+    trace.robots = diagnostic;
+    trace.robotsHistory ??= [];
+    if (trace.robotsHistory.length < 6) trace.robotsHistory.push(diagnostic);
+    try {
+      if (!robotsByOrigin.has(origin)) {
+        const record = await fetchRobots(origin, diagnostic);
+        robotsByOrigin.set(origin, { ...record, diagnostic: structuredClone(diagnostic) });
+      }
+      const record = robotsByOrigin.get(origin);
+      Object.assign(diagnostic, structuredClone(record.diagnostic), { sourceUrl: url });
+      const decision = robotsDecision(record.text, url);
+      // Google crawl permission is a separate publisher fact, never inferred from
+      // our crawler's permission or from this check's monitoring authorization.
+      diagnostic.googlebot = { productToken: 'Googlebot', matchedRule: null, crawlDelaySeconds: null,
+        ...robotsDecision(record.text, url, 'Googlebot') };
+      Object.assign(diagnostic, decision, { decisionReason: decision.reason, fetched: record.reason });
+      const monitorPolicy = permitMonitorPolicy && options.robotsPolicy === 'customer_supplied_monitor';
+      const override = monitorPolicy && !monitorOverrideConsumed && decision.allowed === false && decision.reason === 'robots_disallowed';
+      if (monitorPolicy) Object.assign(diagnostic, { policy: 'customer_supplied_monitor',
+        fetchAuthorized: decision.allowed === true || override, overrideUsed: override });
+      options.onRobots?.(url, decision);
+      if (decision.allowed !== true && !override) throw new VerificationError(decision.reason);
+      if (decision.crawlDelaySeconds > 0 && !options.beforeFetch) {
+        throw new VerificationError('robots_crawl_delay_requires_scheduler');
+      }
+      if (override) monitorOverrideConsumed = true;
+      return diagnostic;
+    } catch (error) {
+      diagnostic.reason = error instanceof VerificationError ? error.reason : 'verification_error';
+      if (diagnostic.googlebot.allowed === null && diagnostic.googlebot.reason === null) diagnostic.googlebot.reason = diagnostic.reason;
+      throw error;
+    }
+  }
+
+  return {
+    // Authorize a native browser document without refetching its body through HTTP.
+    // The caller still needs an independently verified browser egress boundary.
+    async authorizeNavigation(url, trace = {}) {
+      const checked = validatePublicUrl(url);
+      if (!checked.valid) throw new VerificationError('unsafe_render_navigation');
+      const robots = await checkRobots(checked.url, trace);
+      if (signal.aborted) throw new VerificationError('timeout');
+      if (options.beforeFetch) await withAbort(Promise.resolve().then(() => options.beforeFetch(checked.url,
+        { kind: 'source', crawlDelaySeconds: robots.crawlDelaySeconds, signal })), signal);
+      if (signal.aborted) throw new VerificationError('timeout');
+      return robots;
+    },
+    /**
+     * Fetches one source document. `{ validators }` turns this into a conditional GET: the
+     * caller passes the ETag/Last-Modified a previous fetch earned, and a 304 comes back as
+     * `{ notModified: true }` rather than an error. A 304 we did not ask for stays an error —
+     * the server revalidating nobody's cache is a defect, not evidence of anything.
+     *
+     * `{ document: true }` fetches a feed or sitemap instead of a page: the Accept header and
+     * the content-type gate admit the XML family rather than HTML. Everything else — robots per
+     * hop, public-destination screening, the redirect budget — is identical, because a feed is
+     * fetched from the same publishers under the same politeness rules as a page.
+     */
+    async source(initialUrl, trace, { validators = null, document = false } = {}) {
+      const accept = document
+        ? 'application/rss+xml, application/atom+xml, application/xml;q=0.9, text/xml;q=0.9, text/plain;q=0.8'
+        : 'text/html';
+      let url = initialUrl;
+      let revalidators = validators;
+      const seen = new Set();
+      for (let redirects = 0; redirects <= maxRedirects; redirects++) {
+        trace.finalUrl = url;
+        if (seen.has(url)) throw new VerificationError('redirect_loop');
+        seen.add(url);
+        // The exception belongs only to the customer's supplied page. It never
+        // grants a redirect destination, feed, sitemap or another denied request.
+        const robots = await checkRobots(url, trace, !document && redirects === 0);
+        trace.robots = robots;
+        const response = await request(url, { kind: 'source', crawlDelaySeconds: robots.crawlDelaySeconds, validators: revalidators, accept });
+        trace.httpStatus = response.status;
+        trace.sourceResponse = { url, httpStatus: response.status,
+          contentType: /^[a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+/iu.exec(response.headers.get('content-type') ?? '')?.[0].slice(0, 128) ?? null,
+          retryAfterSeconds: retryAfterSeconds(response.headers.get('retry-after')),
+          challenge: response.headers.get('cf-mitigated') === 'challenge' };
+        if (trace.sourceResponse.challenge) {
+          cancelBody(response);
+          const retry = trace.sourceResponse.retryAfterSeconds;
+          throw new VerificationError('access_challenge', retry === null ? {} : { retryAfterSeconds: retry });
+        }
+        if ([301, 302, 303, 307, 308].includes(response.status)) {
+          cancelBody(response);
+          if (robots.overrideUsed === true) throw new VerificationError('monitor_robots_override_fetch_limit');
+          const location = response.headers.get('location');
+          if (!location) throw new VerificationError('redirect_missing_location');
+          let next;
+          try { next = validatePublicUrl(new URL(location, url).href); }
+          catch { throw new VerificationError('invalid_redirect'); }
+          if (!next.valid) throw new VerificationError(`unsafe_redirect:${next.reason}`);
+          trace.redirects.push({ from: url, to: next.url, status: response.status });
+          url = next.url;
+          // Validators were earned for the pre-redirect resource; the destination revalidates
+          // from scratch (see the conditional-header note in `request`).
+          revalidators = null;
+          continue;
+        }
+        if ([404, 410].includes(response.status)) {
+          trace.sourceResponse.bodyDiagnostic = await readBodyDiagnostic(response, signal);
+          return { response, body: null };
+        }
+        if (response.status === 304) {
+          cancelBody(response);
+          if (!revalidators) throw new VerificationError('not_modified_without_baseline');
+          return { response, body: null, notModified: true,
+            validators: { etag: response.headers.get('etag') ?? revalidators.etag ?? null,
+              lastModified: response.headers.get('last-modified') ?? revalidators.lastModified ?? null } };
+        }
+        if (response.status !== 200) {
+          trace.sourceResponse.bodyDiagnostic = await readBodyDiagnostic(response, signal);
+          // A 429 or a 503 may state WHEN to come back. We keep it and never act on it here:
+          // sleeping inside a request holds an origin lease and a worker for someone else's
+          // backoff, and this system already decided that **the cron is the sleep**. Surfacing it
+          // lets a scheduler re-queue at the stated time, the same way a robots crawl delay is
+          // handed to the scheduler rather than slept on.
+          throw new VerificationError(`source_http_${response.status}`,
+            retryAfterSeconds(response.headers.get('retry-after')) === null ? {}
+              : { retryAfterSeconds: retryAfterSeconds(response.headers.get('retry-after')) });
+        }
+        const contentType = response.headers.get('content-type') ?? '';
+        // The document gate admits the XML family (`application/xml`, `text/xml`,
+        // `application/rss+xml`, `application/atom+xml`) and plain text (the sitemap protocol's
+        // text format). It is still a gate: a feed URL serving HTML is a page wearing a feed's
+        // name, and parsing it as a feed would read markup as entries.
+        const expected = document ? /(?:[+/]xml|text\/plain)(?:\s*;|$)/iu : /^text\/html(?:;|$)/iu;
+        if (!expected.test(contentType)) {
+          trace.sourceResponse.bodyDiagnostic = await readBodyDiagnostic(response, signal);
+          throw new VerificationError('unsupported_content_type');
+        }
+        const body = await readBounded(response, document ? documentMaxBytes : maxBytes, signal);
+        return { response, body, notModified: false,
+          validators: { etag: response.headers.get('etag'), lastModified: response.headers.get('last-modified') } };
+      }
+      throw new VerificationError('redirect_limit');
+    },
+  };
+}

package/src/page-acquisition/policy.js

@@ -0,0 +1,20 @@
+const HTTP_REASONS=new Set(['access_challenge','source_http_403','target_http_403','network_error','route_timeout','robots_access_challenge','robots_fetch_failed','robots_not_plain_text']);
+const BROWSER_REASONS=new Set([...HTTP_REASONS,'possible_render_required','navigation_requires_browser']);
+const PROXY_REASONS=new Set([...HTTP_REASONS,'render_failed','render_timeout','render_not_admitted',
+  'acquisition_origin_connection_failed','acquisition_dns_resolution_failed','acquisition_executor_failed','acquisition_executor_busy']);
+export const acquisitionState=value=>value?.state??(value?.outcome==='fetched'&&value.reason?'unknown':({fetched:'healthy',not_found:'unavailable',blocked:'unknown',unavailable:'unknown',not_configured:'unknown',deferred:'deferred',unchanged:'healthy'}[value?.outcome]??'unknown'));
+export const completedAcquisition=value=>value?.evidence?.complete===true||value?.outcome==='fetched'&&!value.reason&&value.body_truncated!==true||['unchanged','not_found'].includes(value?.outcome);
+
+/** Recovery never waives access policy, earned-monitor limits or publisher backoff. */
+export function recoveryEligibility(value,method){
+  const history=value?.robotsHistory??(value?.robots?[value.robots]:[]);
+  if(history.some(row=>row?.overrideUsed===true))return {allowed:false,reason:'robots_override'};
+  if(Number.isFinite(value?.retryAfterSeconds)||Number.isFinite(value?.sourceResponse?.retryAfterSeconds)
+    ||/^((?:source|target|robots)_http_429|origin_busy|robots_crawl_delay_)/u.test(value?.reason??''))return {allowed:false,reason:'rate_limit'};
+  if(completedAcquisition(value))return {allowed:false,reason:'complete'};
+  if(['present','absent','healthy','source_unavailable'].includes(acquisitionState(value)))return {allowed:false,reason:'complete'};
+  if(!['unknown','deferred'].includes(acquisitionState(value)))return {allowed:false,reason:'unsupported'};
+  if(value?.occurrences?.length)return {allowed:false,reason:'complete'};
+  const allowed=(method==='node_browser'?BROWSER_REASONS:method==='proxy_http'?PROXY_REASONS:HTTP_REASONS).has(value?.reason);
+  return {allowed,reason:allowed?null:'unsupported'};
+}