agentlinkops 0.6.11 → 0.6.16

package/src/competitors/contract.js

@@ -22,7 +22,23 @@ export const TargetScope = z.strictObject({ target_kind: z.enum(['domain', 'exac
   if (!DiscoveryQuery.safeParse({ ...scope, ...policy }).success)
     ctx.addIssue({ code: 'custom', message: 'Expected an explicit canonical discovery target scope.' });
 });
-const memberSchema = z.strictObject({ id: identifier, role: z.enum(['customer', 'competitor']), scope: TargetScope });
+// Customer-authored selection context, not a provider verdict or evidence of a link.
+// Keep this optional with no defaults: parsing legacy revisions must not add bytes
+// to the set hash embedded in their immutable inventory snapshots.
+export const CompetitorSelectionRationale = z.strictObject({
+  relationship: z.enum(['business', 'search', 'shared_backlink_neighbor', 'unknown']),
+  audience: z.string().trim().min(1).max(500).nullable(),
+  topic: z.string().trim().min(1).max(500).nullable(),
+  evidence: z.array(z.strictObject({
+    source_url: z.string().min(1).max(2048).refine(value => {
+      const checked = validatePublicUrl(value);
+      return checked.valid && checked.url === value;
+    }, 'Expected a canonical public HTTP(S) source URL.'),
+    note: z.string().trim().min(1).max(1000),
+  })).max(5),
+});
+const memberSchema = z.strictObject({ id: identifier, role: z.enum(['customer', 'competitor']), scope: TargetScope,
+  selection_rationale: CompetitorSelectionRationale.optional() });
 export const ApprovedCompetitorSet = z.strictObject({
   v: z.literal(COMPETITOR_VERSION), id: identifier, revision: z.number().int().min(1).max(Number.MAX_SAFE_INTEGER),
   workspace_id: identifier, project_id: identifier, approved_by: identifier, approved_at: timestamp,

package/src/context/site-profile.js

@@ -29,7 +29,8 @@
 import { SEED_LIMITS, parseSitemap, sitemapDirectives, withinSitemapScope } from '../corpus/seeds.js';
 import { createAcquisition } from '../corpus/acquire.js';
 import { validatePublicUrl } from '../verifier/url.js';
-import { gunzipSync } from 'node:zlib';
+import { createDocumentTransport } from '../page-acquisition/documents.js';
+import { extractPageMetadata } from '../page-extraction/metadata.js';
 import { readJsonl, appendJsonl, sha256Hex } from './util.js';
 import { readManual } from './gsc.js';
 
@@ -52,93 +53,10 @@ export class ProfileError extends Error {
 // decompression (the acquisition layer's job per seeds.js), and per-host courtesy pacing.
 // ---------------------------------------------------------------------------
 
-/**
- * Reads a body to raw bytes under the same bound the verifier's reader enforces, WITHOUT the
- * UTF-8 decode: a gzipped sitemap is binary on the wire, and the decode must wait until after
- * the gunzip decision. Content-length is pre-checked, the stream is capped, the reader cancelled.
- */
-async function readBoundedBytes(response, maxBytes, signal) {
-  const length = response.headers.get('content-length');
-  const encoded = response.headers.get('content-encoding');
-  if (!encoded && length && Number(length) > maxBytes) {
-    response.body?.cancel().catch(() => {});
-    throw new ProfileError('body_too_large');
-  }
-  const reader = response.body?.getReader();
-  if (!reader) return { data: new Uint8Array(0), bytes: 0 };
-  const chunks = [];
-  let bytes = 0;
-  try {
-    for (;;) {
-      const { done, value } = await reader.read();
-      if (done) break;
-      bytes += value.byteLength;
-      if (bytes > maxBytes) throw new ProfileError('body_too_large');
-      chunks.push(value);
-    }
-    const combined = new Uint8Array(bytes);
-    let offset = 0;
-    for (const chunk of chunks) { combined.set(chunk, offset); offset += chunk.byteLength; }
-    return { data: combined, bytes };
-  } finally {
-    reader.cancel().catch(() => {});
-    reader.releaseLock();
-  }
-}
-
-export function createSiteTransport({ fetchImpl = globalThis.fetch, timeoutMs = 20_000, maxBytes = SEED_LIMITS.documentBytes, maxRedirects = 5, hostDelayMs = PROFILE_LIMITS.hostDelayMs, now = () => Date.now(), wait = ms => new Promise(r => setTimeout(r, ms)) } = {}) {
-  const nextAllowed = new Map();
-  async function pace(host) {
-    const allowed = nextAllowed.get(host) ?? 0;
-    const delay = allowed - now();
-    if (delay > 0) await wait(delay);
-    nextAllowed.set(host, now() + hostDelayMs);
-  }
-
-  async function fetchText(rawUrl, { accept = 'text/plain,application/xml,text/xml' } = {}) {
-    const checked = validatePublicUrl(rawUrl);
-    if (!checked.valid) return { ok: false, url: rawUrl, reason: `unsafe_url:${checked.reason}` };
-    let url = checked.url;
-    const seen = new Set();
-    for (let redirects = 0; redirects <= maxRedirects; redirects++) {
-      if (seen.has(url)) return { ok: false, url, reason: 'redirect_loop' };
-      seen.add(url);
-      await pace(new URL(url).hostname);
-      const controller = new AbortController();
-      const timer = setTimeout(() => controller.abort(), timeoutMs);
-      let response;
-      try {
-        response = await fetchImpl(url, { method: 'GET', redirect: 'manual', credentials: 'omit', signal: controller.signal, headers: { 'User-Agent': 'LinktrailBot/0.1', Accept: accept } });
-      } catch { return { ok: false, url, reason: 'timeout_or_network_error' }; }
-      finally { clearTimeout(timer); }
-      if ([301, 302, 303, 307, 308].includes(response.status)) {
-        response.body?.cancel().catch(() => {});
-        const location = response.headers.get('location');
-        const next = location ? validatePublicUrl(new URL(location, url).href) : null;
-        if (!next?.valid) return { ok: false, url, reason: 'unsafe_or_missing_redirect' };
-        url = next.url;
-        continue;
-      }
-      if (response.status === 404 || response.status === 410) { response.body?.cancel().catch(() => {}); return { ok: false, url, status: response.status, reason: 'not_found' }; }
-      if (!response.ok) { response.body?.cancel().catch(() => {}); return { ok: false, url, status: response.status, reason: `http_${response.status}` }; }
-      let body;
-      try { body = await readBoundedBytes(response, maxBytes, controller.signal); }
-      catch (error) { return { ok: false, url, status: response.status, reason: error?.reason ?? 'read_failed' }; }
-      let data = body.data, gzipped = false;
-      if (data?.length >= 2 && data[0] === 0x1f && data[1] === 0x8b) {
-        gzipped = true;
-        try { data = gunzipSync(data); } catch { return { ok: false, url, status: response.status, reason: 'sitemap_gunzip_failed' }; }
-        if (data.byteLength > maxBytes) return { ok: false, url, status: response.status, reason: 'sitemap_too_large' };
-      }
-      let text;
-      try { text = new TextDecoder('utf-8', { fatal: true }).decode(data); }
-      catch { return { ok: false, url, status: response.status, reason: 'unsupported_or_invalid_encoding' }; }
-      return { ok: true, url, final_url: url, status: response.status, text, bytes: body.bytes, gzipped };
-    }
-    return { ok: false, url, reason: 'redirect_limit' };
-  }
-
-  return { fetchText };
+/** Compatibility entrypoint; byte limits and abortable document reads live in one core. */
+export function createSiteTransport(options = {}) {
+  return createDocumentTransport({ maxBytes: SEED_LIMITS.documentBytes,
+    hostDelayMs: PROFILE_LIMITS.hostDelayMs, ...options });
 }
 
 // ---------------------------------------------------------------------------
@@ -146,17 +64,6 @@ export function createSiteTransport({ fetchImpl = globalThis.fetch, timeoutMs =
 // fetch date, matching the verifier's rule that evidence HTML never lives in a payload.
 // ---------------------------------------------------------------------------
 
-const firstMatch = (pattern, html, group = 1) => pattern.exec(html)?.[group]?.replace(/\s+/gu, ' ').trim() ?? null;
-
-function boundedMatches(pattern, html, limit) {
-  const found = [];
-  for (const match of html.matchAll(pattern)) {
-    found.push(match[1].replace(/\s+/gu, ' ').trim());
-    if (found.length >= limit) break;
-  }
-  return found;
-}
-
 /** Asset candidates are NOMINATIONS for judgment, never findings: path conventions the publisher chose plus names the manual declared. */
 const ASSET_PATH_PATTERNS = Object.freeze([
   [/\/(tools?|calculators?|templates?|checklists?|guides?)\//iu, 'asset_path_convention'],
@@ -164,19 +71,8 @@ const ASSET_PATH_PATTERNS = Object.freeze([
   [/\.(pdf|xlsx?|csv)$/iu, 'downloadable_file'],
 ]);
 
-export function derivePageFields(html) {
-  if (typeof html !== 'string' || !html) return { title: null, h1s: [], meta_description: null, canonical: null, link_counts: null, asset_candidate_reasons: null };
-  const links = [...html.matchAll(/<a\b[^>]*href\s*=\s*["']([^"']+)["'][^>]*>/giu)];
-  return {
-    title: firstMatch(/<title[^>]*>([\s\S]*?)<\/title>/iu, html),
-    h1s: boundedMatches(/<h1\b[^>]*>([\s\S]*?)<\/h1>/giu, html, 5),
-    meta_description: (firstMatch(/<meta\b[^>]*name\s*=\s*["']description["'][^>]*content\s*=\s*["']([^"']*)["']/iu, html)
-      ?? firstMatch(/<meta\b[^>]*content\s*=\s*["']([^"']*)["'][^>]*name\s*=\s*["']description["']/iu, html))?.slice(0, 500) ?? null,
-    canonical: firstMatch(/<link\b[^>]*rel\s*=\s*["']canonical["'][^>]*href\s*=\s*["']([^"']+)["']/iu, html)
-      ?? firstMatch(/<link\b[^>]*href\s*=\s*["']([^"']+)["'][^>]*rel\s*=\s*["']canonical["']/iu, html),
-    link_counts: { total: links.length },
-    asset_candidate_reasons: [], // filled by the caller, who knows the URL and the manual assets
-  };
+export function derivePageFields(html, { baseUrl } = {}) {
+  return extractPageMetadata(html, { baseUrl });
 }
 
 function assetCandidateReasons(url, derived, manualAssets) {
@@ -290,6 +186,7 @@ function classifySelection(url, manualTargets, homepage, sitemapUrls) {
  */
 export async function buildSiteProfile({
   site = null, paths, manual = null, transport = null, acquisition = null,
+  recovery = null,
   extraPages = [], now = () => new Date().toISOString(), limits = {},
 }) {
   const cap = { ...PROFILE_LIMITS, ...limits };
@@ -312,7 +209,7 @@ export async function buildSiteProfile({
   //    refused rather than guessing at /sitemap.xml, and the page lane proceeds under the
   //    verifier's own current robots posture.
   const robotsUrl = `${origin}/robots.txt`;
-  const robotsFetch = await docFetch.fetchText(robotsUrl);
+  const robotsFetch = await docFetch.fetchText(robotsUrl, { purpose: 'robots' });
   const robotsText = robotsFetch.ok ? robotsFetch.text : '';
   const directives = robotsFetch.ok ? sitemapDirectives(robotsText, { baseUrl: robotsUrl }) : [];
   const robotsFact = await siteFactRow('site.robots', robotsUrl, {
@@ -383,24 +280,39 @@ export async function buildSiteProfile({
   const selectedFinal = selected.slice(0, cap.selectedPages);
 
   // 4. Page fetches on the verifier's public fetch boundary, with the CLI's courtesy pacing.
-  const pageAcquisition = acquisition ?? createAcquisition({ publicFetchSafe: true });
+  // This local producer owns no Worker job or native reservation. Explicit caller
+  // adapters may recover a page; omitted privileged routes remain unsupported.
+  const pageAcquisition = acquisition ?? createAcquisition({ publicFetchSafe: true,purpose:'research',...(recovery?{recovery}:{}) });
   const pacer = createPacer({ hostDelayMs: cap.hostDelayMs });
   const pageRows = [];
   for (const url of selectedFinal) {
     const fetchAt = now();
-    const result = await pageAcquisition.fetchPage(url, { kind: 'direct', beforeFetch: pacer.beforeFetch });
-    const fetched = result.outcome === 'fetched';
-    const derived = fetched ? derivePageFields(result.html) : { title: null, h1s: [], meta_description: null, canonical: null, link_counts: null };
-    derived.asset_candidate_reasons = fetched ? assetCandidateReasons(url, derived, manualInputs.assets) : null; // unknown, never "no assets"
+    const result = await pageAcquisition.fetchPage(url, { kind: 'direct',purpose:'research', beforeFetch: pacer.beforeFetch,...(recovery?{recovery}:{}) });
+    const acquired = result.outcome === 'fetched';
+    let outcome = result.outcome, reason = result.reason ?? null;
+    let derived = { title: null, h1s: [], meta_description: null, canonical: null, link_counts: null };
+    if (acquired) {
+      try {
+        if (result.body_truncated === true) throw new ProfileError('body_truncated');
+        derived = derivePageFields(result.html, { baseUrl: result.final_url ?? url });
+      }
+      catch (error) {
+        // A refused/incomplete parse is an explicit unknown, never an empty page or "no assets".
+        outcome = 'unavailable'; reason = error?.reason ?? 'unreadable_document';
+      }
+    }
+    const fetched = outcome === 'fetched';
+    derived.asset_candidate_reasons = fetched ? assetCandidateReasons(url, derived, manualInputs.assets) : null;
     const fact = await siteFactRow('site.page', url, {
-      outcome: result.outcome, reason: result.reason ?? null, http_status: result.http_status ?? null,
+      outcome, reason, http_status: result.http_status ?? null,
       final_url: result.final_url ?? url, robots: result.robots?.reason ?? null,
       title: derived.title, h1s: derived.h1s, meta_description: derived.meta_description,
       canonical: derived.canonical, link_counts: derived.link_counts,
       asset_candidate_reasons: derived.asset_candidate_reasons,
       in_sitemap: sitemapUrls.includes(url),
       selection_reasons: classifySelection(url, manualInputs.target_pages ?? [], homepage, sitemapUrls),
-      bytes: result.bytes ?? null, content_sha256: fetched ? await sha256Hex(result.html ?? '') : null,
+      bytes: result.bytes ?? null, content_sha256: acquired ? await sha256Hex(result.html ?? '') : null,
+      ...(result.acquisition?{acquisition:result.acquisition}:{}),
       fetched_at: fetchAt,
     });
     facts.push(fact); pageRows.push(fact);
@@ -417,7 +329,8 @@ export async function buildSiteProfile({
       capped: sitemapCapped, directives_seen: directives.length,
       failures: sitemapFailures, skipped: sitemapSkipped,
     },
-    pages: pageRows.map(row => ({ url: row.url, outcome: row.outcome, reason: row.reason, title: row.title, in_sitemap: row.in_sitemap, selection_reasons: row.selection_reasons, asset_candidate_reasons: row.asset_candidate_reasons })),
+    pages: pageRows.map(row => ({ url: row.url, outcome: row.outcome, reason: row.reason, title: row.title, in_sitemap: row.in_sitemap, selection_reasons: row.selection_reasons, asset_candidate_reasons: row.asset_candidate_reasons,
+      ...(row.acquisition?{acquisition:row.acquisition}:{}) })),
     limits: { sitemap_documents: cap.sitemapDocuments, selected_pages: cap.selectedPages, pages_capped: pagesCapped, pages_selected: pageRows.length },
     rows_written: facts.length,
     notes: [

package/src/corpus/acquire.js

@@ -1,5 +1,7 @@
-import { publicFetcher, VerificationError } from '../verifier/fetch.js';
+import { createPageAcquisition } from '../page-acquisition/acquire.js';
 import { validatePublicUrl } from '../verifier/url.js';
+import {runRecovery,recoverySummary} from '../page-acquisition/recovery.js';
+import {extractEdges,ExtractionError} from './extract.js';
 
 // One interface over every way we acquire a page, so a runtime can be added without
 // rebuilding the corpus. Direct fetching is implemented here on the verifier's public
@@ -51,24 +53,22 @@ export function createAcquisition(options = {}) {
   const runtimes = options.runtimes ?? {};
   const clock = options.now ?? (() => new Date().toISOString());
 
-  async function direct(url, beforeFetch, { validators = null, document = false } = {}) {
-    // The public boundary is a property of the transport, not of this module. Refuse rather
-    // than fetch on an unasserted one; a lexically public hostname can still resolve inside.
+  async function direct(url, beforeFetch, { validators = null, document = false, alternate = null, method = 'workers_http',
+    signal = options.signal, deadlineAt = options.deadlineAt } = {}) {
+    // Preserve the established compatibility refusal. New shared capture metadata does
+    // not establish public egress qualification or activate the stopped corpus machinery.
     if (options.publicFetchSafe !== true) throw new AcquisitionError('public_fetch_boundary_not_asserted');
-    const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), timeoutMs);
-    const trace = { finalUrl: url, redirects: [], robots: null, httpStatus: null };
-    try {
-      const fetcher = publicFetcher({ ...options, beforeFetch: beforeFetch ?? options.beforeFetch, maxBytes, maxRedirects: ACQUISITION_LIMITS.maxRedirects }, controller.signal);
-      const { response, body, notModified, validators: earned } = await fetcher.source(url, trace, { validators, document });
-      if (notModified) return { outcome: 'unchanged', trace, html: null, validators: earned };
-      if (NOT_FOUND.has(response.status)) return { outcome: 'not_found', trace, html: null };
-      return { outcome: 'fetched', trace, html: body?.text ?? '', bytes: body?.bytes ?? 0,
-        linkHeader: response.headers.get('link'),
-        validators: { etag: response.headers.get('etag'), lastModified: response.headers.get('last-modified') } };
-    } finally {
-      clearTimeout(timer);
-    }
+    const acquisition = createPageAcquisition({ ...options, beforeFetch: beforeFetch ?? options.beforeFetch,
+      ...(alternate?{fetchImpl:(target,settings)=>alternate.fetch(target,{...settings,deadlineAt})}:{}),
+      transportId: alternate?method:options.captureTransport?.id ?? 'public_http',
+      transportVersion: alternate?'alo-public-node-v1':options.captureTransport?.version ?? '1' });
+    const { response, body, notModified, validators: earned, trace, capture } = await acquisition.acquire(url,
+      { maxBytes, timeoutMs, validators, document, signal, deadlineAt });
+    if (notModified) return { outcome: 'unchanged', trace, html: null, validators: earned, capture: null };
+    if (NOT_FOUND.has(response.status)) return { outcome: 'not_found', trace, html: null, capture: null };
+    return { outcome: 'fetched', trace, html: body?.text ?? '', bytes: body?.bytes ?? 0, capture,
+      linkHeader: response.headers.get('link'),
+      validators: { etag: response.headers.get('etag'), lastModified: response.headers.get('last-modified') } };
   }
 
   /**
@@ -85,6 +85,8 @@ export function createAcquisition(options = {}) {
   }
 
   return {
+    recoveryManaged:true,
+    maxFetches:1+(options.recovery?.alternate?1:0)+(runtimes.rendered?.publicFetchSafe===true?1:0)+(runtimes.proxied?.publicFetchSafe===true?1:0),
     /**
      * Whether a fetch kind can actually be served. The crawl loop asks before escalating a
      * page to a browser, because queueing work no runtime can serve would leave the frontier
@@ -104,7 +106,7 @@ export function createAcquisition(options = {}) {
      * previous fetch of this URL earned, and an unchanged page comes back as outcome
      * `unchanged` with `http_status` 304 and no body — a dated observation, not an absence.
      */
-    async fetchPage(rawUrl, { kind = 'direct', beforeFetch = null, validators = null } = {}) {
+    async fetchPage(rawUrl, { kind = 'direct', beforeFetch = null, validators = null, recovery = options.recovery??{},purpose = options.purpose??'discovery', signal = options.signal, deadlineAt = options.deadlineAt } = {}) {
       const at = clock();
       const base = { url: rawUrl, final_url: null, kind, http_status: null, robots: null,
         html: null, body_truncated: false, fetched_at: at, reason: null, etag: null, last_modified: null };
@@ -112,21 +114,30 @@ export function createAcquisition(options = {}) {
       const checked = validatePublicUrl(rawUrl);
       if (!checked.valid) return { ...base, outcome: 'unavailable', reason: `unsafe_url:${checked.reason}` };
 
-      try {
-        if (kind === 'direct') {
-          const { outcome, trace, html, validators: earned, linkHeader } = await direct(checked.url, beforeFetch, { validators });
-          return { ...base, outcome, final_url: trace.finalUrl, http_status: trace.httpStatus,
-            robots: trace.robots, redirects: trace.redirects, html,
+      const consumed=[];
+      const acquireOne=async(routeKind,settings,provider=null,method='workers_http')=>{
+       try {
+        if (routeKind === 'direct') {
+          const { outcome, trace, html, validators: earned, linkHeader, bytes, capture } = await direct(checked.url, beforeFetch, { validators,...settings,alternate:provider,method });
+          const result={ ...base, outcome, final_url: trace.finalUrl, http_status: trace.httpStatus,
+            robots: trace.robots, redirects: trace.redirects, html, bytes, capture,
+            robotsHistory:trace.robotsHistory,sourceResponse:trace.sourceResponse,
             etag: earned?.etag ?? null, last_modified: earned?.lastModified ?? null,
             link_header: linkHeader ?? null,
             // The verifier's reader refuses an oversized body outright, so a returned body was
             // read to the end. Truncation reaches extraction from runtimes with their own caps.
             body_truncated: false, render_ms: 0, proxy_bytes: 0 };
+          // Acquisition and parsing failures enter the same router. No fetched shell
+          // can silently become an empty publisher in context or the owned corpus.
+          if(result.outcome==='fetched')try{extractEdges(html,{pageUrl:trace.finalUrl,fetchKind:'direct'});}
+          catch(error){if(!(error instanceof ExtractionError))throw error;return {...result,outcome:classifyFailure(error.reason),reason:error.reason,html:null};}
+          return result;
         }
-        const runtime = assertRuntime(kind);
-        const result = await runtime.fetchPage(checked.url, { maxBytes, timeoutMs });
-        return { ...base, outcome: result.outcome ?? 'fetched', final_url: result.finalUrl ?? checked.url,
+        const runtime = assertRuntime(routeKind);
+        const result = await runtime.fetchPage(checked.url, { maxBytes,...settings,beforeFetch });
+        const observed={ ...base,kind:routeKind,outcome: result.outcome ?? 'fetched', final_url: result.finalUrl ?? checked.url,
           http_status: result.httpStatus ?? null, html: result.html ?? null,
+          positive_only:result.positiveOnly===true,robots:result.robots??null,robotsHistory:result.robotsHistory,sourceResponse:result.sourceResponse,
           body_truncated: result.bodyTruncated === true, reason: result.reason ?? null,
           // The units a runtime actually consumed. Rendering and proxy traffic are what make
           // these paths expensive, so dropping them here would leave every metered run
@@ -134,13 +145,38 @@ export function createAcquisition(options = {}) {
           // one. A direct fetch reports neither, which is correct: it spends neither.
           render_ms: Number.isInteger(result.renderMs) ? result.renderMs : 0,
           proxy_bytes: Number.isInteger(result.proxyBytes) ? result.proxyBytes : 0 };
-      } catch (error) {
+        consumed.push(observed);
+        if(observed.outcome==='fetched')try{extractEdges(observed.html??'',{pageUrl:observed.final_url,fetchKind:routeKind,bodyTruncated:observed.body_truncated||observed.positive_only});}
+        catch(error){if(!(error instanceof ExtractionError))throw error;return {...observed,outcome:classifyFailure(error.reason),reason:error.reason,html:null};}
+        return observed;
+       } catch (error) {
         // Any error carrying its own reason keeps it, including the pacing hook's, so a
         // slot we could not take is classified as deferrable rather than as a failure.
         const reason = typeof error?.reason === 'string' ? error.reason : 'acquisition_error';
         const outcome = error instanceof AcquisitionError && /not_configured|not_asserted|_invalid$/.test(reason)
           ? 'not_configured' : classifyFailure(reason);
-        return { ...base, outcome, reason };
+        const trace=error.acquisitionTrace??{};
+        return { ...base,kind:routeKind,outcome,reason,robots:trace.robots,robotsHistory:trace.robotsHistory,sourceResponse:trace.sourceResponse,
+          http_status:trace.httpStatus??null,final_url:trace.finalUrl??null};
+       }
+      };
+      try{
+        if(kind!=='direct')return await acquireOne(kind,{signal,deadlineAt,timeoutMs});
+        const provider=recovery.alternate;
+        const runtimeRoute=(runtimeKind,method)=>({capability:recovery.capabilities?.[method]??{available:runtimes[runtimeKind]?.publicFetchSafe===true&&typeof runtimes[runtimeKind]?.fetchPage==='function',reason:'unsupported'},
+          admit:recovery[`${runtimeKind}Admission`],execute:async settings=>[await acquireOne(runtimeKind,settings)]});
+        const [result]=await runRecovery({purpose,directMethod:options.directMethod??'workers_http',authorize:recovery.authorize,options:{signal,deadlineAt,timeoutMs},
+          directFailure:reason=>[{...base,outcome:'unavailable',reason}],
+          executeDirect:async settings=>[await acquireOne('direct',settings)],routes:{
+            node_http:{capability:recovery.capabilities?.node_http??{available:!!provider,reason:'unsupported'},receipts:provider?.receipts,
+              execute:provider?async settings=>[await acquireOne('direct',settings,provider,'node_http')]:undefined},
+            node_browser:runtimeRoute('rendered','node_browser'),proxy_http:runtimeRoute('proxied','proxy_http')}});
+        result.acquisition=recoverySummary(result.evidence?.acquisition);
+        result.fetches=result.acquisition?.attempts.length??1;
+        result.render_ms=consumed.reduce((sum,row)=>sum+(row.render_ms??0),0);result.proxy_bytes=consumed.reduce((sum,row)=>sum+(row.proxy_bytes??0),0);
+        return result;
+      }catch(error){
+        return {...base,outcome:classifyFailure(error.reason??'acquisition_error'),reason:error.reason??'acquisition_error'};
       }
     },
 
@@ -153,16 +189,16 @@ export function createAcquisition(options = {}) {
      * The crawl loop never calls this; it exists for the freshness lanes (sitemap diffing,
      * feed polling) that read publisher declarations rather than pages.
      */
-    async fetchDocument(rawUrl, { beforeFetch = null, validators = null } = {}) {
+    async fetchDocument(rawUrl, { beforeFetch = null, validators = null, signal = options.signal, deadlineAt = options.deadlineAt } = {}) {
       const at = clock();
       const base = { url: rawUrl, final_url: null, kind: 'direct', http_status: null,
         body: null, fetched_at: at, reason: null, etag: null, last_modified: null };
       const checked = validatePublicUrl(rawUrl);
       if (!checked.valid) return { ...base, outcome: 'unavailable', reason: `unsafe_url:${checked.reason}` };
       try {
-        const { outcome, trace, html, validators: earned, linkHeader } = await direct(checked.url, beforeFetch, { validators, document: true });
+        const { outcome, trace, html, validators: earned, linkHeader, bytes, capture } = await direct(checked.url, beforeFetch, { validators, document: true, signal, deadlineAt });
         return { ...base, outcome, final_url: trace.finalUrl, http_status: trace.httpStatus,
-          body: outcome === 'fetched' ? html : null,
+          body: outcome === 'fetched' ? html : null, bytes, capture,
           etag: earned?.etag ?? null, last_modified: earned?.lastModified ?? null,
           link_header: linkHeader ?? null };
       } catch (error) {

package/src/corpus/extract.js

@@ -0,0 +1,147 @@
+import { parseDocument, pageDirectives } from '../verifier/parser.js';
+import { validatePublicUrl } from '../verifier/url.js';
+import { frontierKey } from './frontier.js';
+import { analyzeReadiness, isAccessChallenge } from '../verifier/readiness.js';
+
+// Turns one fetched public HTML page into the outbound edges our corpus stores.
+// Pure: no fetch, no persistence, no scheduling. The caller owns acquisition.
+//
+// This deliberately reuses the verifier's parser rather than adopting an audit crawler's.
+// That parser already keeps every anchor/area occurrence instead of collapsing them by
+// destination, keeps raw rel tokens rather than a nofollow boolean, and THROWS on its
+// complexity limits instead of returning a short list. The last property is the one that
+// matters most here: a page we could not fully read must never be recorded as a page with
+// few links, because absence is what the whole product must not fake.
+export const EXTRACTION_VERSION = 1;
+
+// A link carrying any of these is a hint, not an endorsement, so it does not pass equity.
+const UNFOLLOWED_REL = new Set(['nofollow', 'sponsored', 'ugc']);
+
+export class ExtractionError extends Error {
+  constructor(reason) { super(reason); this.name = 'ExtractionError'; this.reason = reason; }
+}
+
+/** Same host, or one host is a subdomain of the other. */
+function relatedHost(host, other) {
+  return host === other || host.endsWith(`.${other}`) || other.endsWith(`.${host}`);
+}
+
+/**
+ * @param html   Decoded response body. The caller must say if it was cut short.
+ * @param pageUrl Canonical URL of the page the body came from, after redirects.
+ * @param options.fetchKind      How the body was acquired: direct, rendered or proxied.
+ * @param options.bodyTruncated  True when the byte budget stopped the read.
+ * @param options.xRobotsTag     Raw X-Robots-Tag response header, if any.
+ * @param options.internalHosts  Hosts to treat as the source's own. Defaults to the source
+ *                               host and its subdomains. Pass the site's real host list
+ *                               when siblings such as blog. and www. must count as internal;
+ *                               without a public suffix list this cannot be inferred.
+ * @throws ExtractionError with `.reason` when the page could not be read completely enough
+ *         to enumerate its links. Callers record an unknown page, never an empty one.
+ */
+export function extractEdges(html, { pageUrl, fetchKind, bodyTruncated = false, xRobotsTag = null, internalHosts } = {}) {
+  if (!['direct', 'rendered', 'proxied'].includes(fetchKind)) throw new ExtractionError('unsupported_fetch_kind');
+  const page = validatePublicUrl(pageUrl);
+  if (!page.valid) throw new ExtractionError(`source_${page.reason}`);
+
+  let document;
+  // Complexity limits reject the page rather than returning part of it. Re-thrown as an
+  // ExtractionError so a caller cannot mistake a refusal for a result.
+  try { document = parseDocument(html, page.url); }
+  catch (error) { throw new ExtractionError(error.reason ?? 'unreadable_document'); }
+
+  // A page whose links we could not have seen must not be stored as a page with few links.
+  // These are the verifier's own "cannot conclude absence" rules, which matter even more
+  // here: a plain-fetch crawler that records a JavaScript-rendered publisher as link-free
+  // poisons the corpus silently, and that is the single most common false-negative in this
+  // category. The rules are asymmetric on purpose — a real article that happens to carry a
+  // login form still extracts, because it produced links.
+  const { readiness, managedClientChallenge } = analyzeReadiness({ html, parsed: document });
+  if (isAccessChallenge(document, { managedClientChallenge })) throw new ExtractionError('access_challenge');
+  if (!document.links.length) {
+    if (document.meta.some(item => item.name === 'refresh')) throw new ExtractionError('navigation_requires_browser');
+    if (readiness.possibleLoginWall) throw new ExtractionError('possible_login_wall');
+    if (document.hasScript && document.visibleText.length < 80) throw new ExtractionError('possible_render_required');
+  }
+  // Navigation links around an empty application root do not make its content complete.
+  if(readiness.requiresRender)throw new ExtractionError('possible_render_required');
+
+  const directives = pageDirectives(document.meta, xRobotsTag);
+  const owned = internalHosts?.length
+    ? internalHosts.map(host => host.toLowerCase())
+    : [page.hostname];
+
+  // Group occurrences by canonical destination. Discovery reports that an edge exists and
+  // how often; per-occurrence anchor, rel and evidence belong to a verification check,
+  // which the verifier already produces on demand for a selected candidate.
+  const byTarget = new Map();
+  let outlinks = 0, externalOutlinks = 0;
+  for (const link of document.links) {
+    const target = validatePublicUrl(link.targetUrl);
+    if (!target.valid) continue;
+    outlinks++;
+    const internal = owned.some(host => relatedHost(target.hostname, host));
+    if (!internal) externalOutlinks++;
+    const existing = byTarget.get(target.url);
+    const followed = !directives.nofollow && !link.rel.some(token => UNFOLLOWED_REL.has(token));
+    if (existing) {
+      existing.links_count++;
+      // Any followed occurrence makes the edge followed: a nofollow copy in a comment does
+      // not undo a followed link in the body. `rel` stays the first occurrence's tokens,
+      // so `dofollow` is NOT a function of the `rel` reported beside it.
+      existing.dofollow ||= followed;
+      continue;
+    }
+    byTarget.set(target.url, {
+      target_url: target.url, target_hostname: target.hostname, internal,
+      anchor: link.anchor || null, rel: link.rel.length ? link.rel : null, dofollow: followed,
+      links_count: 1, context: link.context || null, locator: link.locator,
+    });
+  }
+
+  return {
+    v: EXTRACTION_VERSION,
+    source_url: page.url, source_hostname: page.hostname, fetch_kind: fetchKind,
+    // A body the byte budget cut short, or a document with unfinished markup, was not read
+    // to the end. The contract refuses `complete_for_query` coverage on such a page.
+    extraction_complete: !bodyTruncated && !document.incomplete,
+    page_nofollow: directives.nofollow, page_noindex: directives.noindex,
+    // Non-null when the page declares a DIFFERENT URL as canonical, i.e. it is a duplicate
+    // of that one. Recorded and surfaced, never used to silently drop the page: cross-domain
+    // and plain wrong canonicals are common, and discarding them would lose real links.
+    // Compared through the frontier key, so a canonical differing only by a trailing slash,
+    // an index filename or a tracking parameter is the SAME page, not a duplicate of another.
+    // Measured against real publishers, raw string comparison flagged 11 of 14 pages as
+    // duplicates when almost all of them were declaring themselves.
+    canonical_url: document.canonical && frontierKey(document.canonical) !== frontierKey(page.url)
+      ? document.canonical : null,
+    source_outlink_count: outlinks, source_external_outlink_count: externalOutlinks,
+    title: document.title || null,
+    edges: [...byTarget.values()],
+  };
+}
+
+/**
+ * Shapes one extracted edge as a discovery candidate for the owned corpus. The caller
+ * supplies run lineage and the crawl clocks; nothing here invents a date or a check.
+ */
+export function edgeToCandidate(edge, page, { runId, workspaceId, projectId, candidateId, taskId, retrievedAt, firstSeen = null, prevSeen = null, lastSeen = null }) {
+  return {
+    v: 1, id: candidateId, workspace_id: workspaceId, project_id: projectId, discovery_run_id: runId,
+    source_url: page.source_url, target_url: edge.target_url,
+    provider: 'linktrail_corpus', data_mode: 'owned_corpus',
+    provider_task_id: taskId, provider_retrieved_at: retrievedAt,
+    provider_first_seen: firstSeen, provider_prev_seen: prevSeen, provider_last_seen: lastSeen,
+    // A crawl that saw the link says nothing about loss or breakage, and our own crawl
+    // clocks are still upstream data rather than a check against a customer expectation.
+    provider_status: { is_lost: null, is_broken: null, is_new: null },
+    anchor: edge.anchor, rel: edge.rel, dofollow: edge.dofollow, link_type: 'anchor',
+    source_http_status: null, target_http_status: null, links_count: edge.links_count,
+    provider_metrics: { linktrail_corpus: {
+      source_outlink_count: page.source_outlink_count,
+      source_external_outlink_count: page.source_external_outlink_count,
+      fetch_kind: page.fetch_kind, extraction_complete: page.extraction_complete,
+    } },
+    verification_status: 'not_checked', verified_at: null, observation_id: null, evidence_id: null,
+  };
+}