agentlinkops 0.6.13 → 0.6.16

package/src/corpus/frontier.js

@@ -0,0 +1,151 @@
+import { validatePublicUrl } from '../verifier/url.js';
+
+// Decides what the crawler is allowed to queue, and in what order. Pure: no fetch, no
+// persistence, no clock of its own. Persistence and leasing belong to the store; this
+// module owns the judgement that keeps a crawl finite and on-topic.
+export const FRONTIER_VERSION = 1;
+
+export const FRONTIER_LIMITS = Object.freeze({
+  pathDepth: 12,          // deeper than any editorial page we want; archives spiral past this
+  querySize: 4,           // faceted navigation multiplies past this
+  repeatedSegment: 3,     // /a/b/a/b/a/b is a cycle, not a hierarchy
+  urlLength: 2048,
+  hostPages: 500,         // per host, per run
+});
+
+// Stripped before deduplication, never a reason to reject. These change nothing about the
+// page served, so keeping them would fetch one page once per inbound campaign link.
+const TRACKING_PARAMS = [/^utm_/i, /^(gclid|fbclid|msclkid|mc_[ce]id|igshid|ref|ref_src|source|_ga|yclid|dclid|s_kwcid|vero_id|oly_enc_id)$/i];
+// Parameters whose whole purpose is to enumerate. Presence is a trap signal, not a strip.
+const ENUMERATING_PARAMS = /^(sid|sessionid|jsessionid|phpsessid|calendar|cal_date|date|from_date|to_date|start_date|end_date|month|year|day|week|filter|facet|orderby|order_by|sort_by|price_min|price_max|replytocom)$/i;
+// Endpoints that generate results rather than publish pages.
+const GENERATED_PATHS = /(^|\/)(search|cart|checkout|login|signin|signup|register|logout|account|wp-admin|wp-login\.php|xmlrpc\.php|feed|rss|atom|print|amp|tag|author|comment-page-\d+|page\/\d{3,})(\/|$)/i;
+// Files that are not pages we can extract editorial links from.
+const NON_PAGE_EXTENSION = /\.(jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|mp4|webm|mov|avi|mp3|wav|ogg|flac|pdf|zip|gz|tar|rar|7z|dmg|exe|msi|pkg|deb|rpm|css|js|mjs|json|xml|txt|csv|xlsx?|docx?|pptx?|woff2?|ttf|otf|eot)$/i;
+
+export class FrontierRejection extends Error {
+  constructor(reason) { super(reason); this.name = 'FrontierRejection'; this.reason = reason; }
+}
+
+/**
+ * Canonical deduplication key. Two URLs sharing a key serve the same page for our purposes,
+ * so the frontier fetches one of them. Returns null when the URL is not crawlable at all.
+ */
+export function frontierKey(rawUrl) {
+  const checked = validatePublicUrl(rawUrl);
+  if (!checked.valid) return null;
+  const url = new URL(checked.url);
+  // Sorting alone is not enough. Scrapy's canonicalize_url sorts query keys and keeps
+  // duplicates, so `?a=1&a=1` and `?a=1` fetch the same page twice; collapse identical pairs.
+  const seen = new Set();
+  const kept = [...url.searchParams.entries()]
+    .filter(([name]) => !TRACKING_PARAMS.some(pattern => pattern.test(name)))
+    .filter(([name, value]) => { const pair = `${name}=${value}`; if (seen.has(pair)) return false; seen.add(pair); return true; })
+    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+  url.search = '';
+  for (const [name, value] of kept) url.searchParams.append(name, value);
+  // A trailing slash and an explicit directory index address the same document far more
+  // often than not, and treating them as two pages doubles every crawl.
+  url.pathname = url.pathname.replace(/\/index\.(html?|php|aspx?)$/i, '/').replace(/(.)\/$/, '$1');
+  return url.href;
+}
+
+/** Longest run of a repeated path segment, which marks a cycle rather than a hierarchy. */
+function repeatedSegmentRun(segments) {
+  const seen = new Map();
+  let worst = 0;
+  for (const segment of segments) {
+    const count = (seen.get(segment) ?? 0) + 1;
+    seen.set(segment, count);
+    if (count > worst) worst = count;
+  }
+  return worst;
+}
+
+/**
+ * Decides whether a discovered URL may enter the frontier.
+ *
+ * @param rawUrl        Absolute URL found on a crawled page.
+ * @param context.reason        How it was found: seed, sitemap, outlink or refresh.
+ * @param context.seenKeys      Keys already queued or crawled. A Set, read only.
+ * @param context.hostPages     Pages already queued or crawled for this host, this run.
+ * @param context.allowedHosts  When present, only these hosts (and their subdomains) admit.
+ * @param context.robotsAllows  Callback (pathname) => boolean. Defaults to allowing.
+ * @returns {{admit: boolean, reason: string, key: string|null, hostname: string|null, priority: number}}
+ */
+export function admitUrl(rawUrl, context = {}) {
+  const { reason = 'outlink', seenKeys, hostPages = 0, allowedHosts, robotsAllows } = context;
+  const reject = why => ({ admit: false, reason: why, key: null, hostname: null, priority: 0 });
+
+  if (typeof rawUrl !== 'string' || rawUrl.length > FRONTIER_LIMITS.urlLength) return reject('url_too_long');
+  const checked = validatePublicUrl(rawUrl);
+  if (!checked.valid) return reject(checked.reason);
+  const key = frontierKey(rawUrl);
+  if (!key) return reject('invalid_url');
+  const url = new URL(key), hostname = checked.hostname;
+
+  if (allowedHosts?.length && !allowedHosts.some(host => hostname === host || hostname.endsWith(`.${host}`)))
+    return reject('host_not_in_scope');
+  if (seenKeys?.has(key)) return reject('already_seen');
+  // The budget is per host so one large publisher cannot consume a whole run.
+  if (hostPages >= FRONTIER_LIMITS.hostPages) return reject('host_budget_exhausted');
+  if (NON_PAGE_EXTENSION.test(url.pathname)) return reject('not_a_page');
+  if (GENERATED_PATHS.test(url.pathname)) return reject('generated_endpoint');
+
+  const segments = url.pathname.split('/').filter(Boolean);
+  if (segments.length > FRONTIER_LIMITS.pathDepth) return reject('path_too_deep');
+  if (repeatedSegmentRun(segments) >= FRONTIER_LIMITS.repeatedSegment) return reject('repeated_path_segment');
+
+  const params = [...url.searchParams.keys()];
+  if (params.length > FRONTIER_LIMITS.querySize) return reject('query_too_large');
+  if (params.some(name => ENUMERATING_PARAMS.test(name))) return reject('enumerating_parameter');
+  // robots is consulted last so a rejected URL is described by its own defect first.
+  if (robotsAllows && !robotsAllows(url.pathname + url.search)) return reject('robots_disallowed');
+
+  return { admit: true, reason, key, hostname, priority: priorityFor(reason, segments.length) };
+}
+
+/**
+ * Lower sorts first. Seeds and sitemaps outrank pages we merely stumbled onto, and a
+ * shallow page outranks a deep one, because editorial resource pages sit near the root.
+ */
+export function priorityFor(reason, depth) {
+  const base = { seed: 0, sitemap: 100, refresh: 200, outlink: 300 }[reason] ?? 400;
+  return base + Math.min(depth, FRONTIER_LIMITS.pathDepth);
+}
+
+/**
+ * Selects the next batch to fetch: cheapest priority first, at most one URL per host per
+ * batch so a batch never stacks requests on one origin, and nothing whose host is still
+ * inside its crawl delay.
+ *
+ * @param entries  [{ key, hostname, priority, nextEligibleAt }]
+ * @param now      ISO timestamp.
+ * @param batchSize Maximum URLs to return.
+ */
+export function selectBatch(entries, { now, batchSize = 10 } = {}) {
+  const at = Date.parse(now);
+  if (!Number.isFinite(at)) throw new FrontierRejection('invalid_clock');
+  const ready = entries
+    .filter(entry => !entry.nextEligibleAt || Date.parse(entry.nextEligibleAt) <= at)
+    .sort((a, b) => a.priority - b.priority || (a.key < b.key ? -1 : 1));
+  const hosts = new Set(), batch = [];
+  for (const entry of ready) {
+    if (hosts.has(entry.hostname)) continue;
+    hosts.add(entry.hostname);
+    batch.push(entry);
+    if (batch.length >= batchSize) break;
+  }
+  return batch;
+}
+
+/**
+ * When a host may next be fetched. A robots crawl delay is honoured as stated; otherwise a
+ * courtesy floor applies. Never returns a time earlier than the delay a publisher asked for.
+ */
+export function nextEligibleAt(now, { crawlDelaySeconds = null, courtesySeconds = 5 } = {}) {
+  const at = Date.parse(now);
+  if (!Number.isFinite(at)) throw new FrontierRejection('invalid_clock');
+  const seconds = Math.max(courtesySeconds, crawlDelaySeconds ?? 0);
+  return new Date(at + seconds * 1000).toISOString();
+}

package/src/discovery/contract.js

@@ -17,6 +17,7 @@ export const DISCOVERY_DATA_MODES = Object.freeze(['synthetic', 'owned_corpus',
 export const IMPORT_SUPPLIERS = Object.freeze([
   'ahrefs', 'google_search_console', 'bing_webmaster_tools', 'semrush', 'majestic', 'moz', 'dataforseo', 'linkody', 'csv',
 ]);
+export const ImportTargetBasis = z.enum(['supplier_observed', 'user_supplied_target_expectation']);
 
 // A data mode belongs to exactly one source, except fixtures, which any source may model.
 const SOURCE_DATA_MODES = Object.freeze({ linktrail_corpus: 'owned_corpus', dataforseo: 'provider_index', imported: 'imported' });
@@ -92,12 +93,16 @@ export const DiscoveryCandidate = z.strictObject({
   provider_metrics: z.union([
     z.strictObject({ dataforseo: z.strictObject({
       rank: count.max(1000).nullable(), page_from_rank: count.max(1000).nullable(),
+      domain_from: z.string().min(1).max(253).optional(),
       domain_from_rank: count.max(1000).nullable(), backlink_spam_score: count.max(100).nullable(),
       rank_scale: z.literal('one_thousand'),
     }) }),
     z.strictObject({ imported: z.strictObject({
       supplier: z.enum(IMPORT_SUPPLIERS),
       supplier_row_id: nullableText(256),
+      // Caller-authored destination provenance stays separate from opaque supplier columns.
+      // Older imports omit it and keep their exact destination behavior.
+      target_basis: ImportTargetBasis.optional(),
       // What the EXPORT says about its own age, which is often nothing. `provider_retrieved_at`
       // on the row above is when WE read the file; these are different facts and conflating
       // them would let an import of a two-year-old export read as fresh data.
@@ -108,6 +113,9 @@ export const DiscoveryCandidate = z.strictObject({
       // the scale is the supplier's name itself.
       supplier_metrics: z.record(z.string().min(1).max(64), z.union([z.number(), z.string().max(256), z.null()]))
         .refine(value => Object.keys(value).length <= 32, 'At most 32 supplier metrics.').nullable(),
+    }).superRefine((imported, ctx) => {
+      if (imported.target_basis === 'user_supplied_target_expectation' && imported.supplier !== 'google_search_console')
+        ctx.addIssue({ code: 'custom', message: 'A target expectation is supported only for Google Search Console imports.', path: ['target_basis'] });
     }) }),
     z.strictObject({ linktrail_corpus: z.strictObject({
       // Real crawl facts. Outlink counts qualify a page; they are not an authority metric.

package/src/discovery/import.js

@@ -19,7 +19,7 @@
 //   nothing here turns them into an AgentLinkOps score.
 import { z } from 'zod';
 import { validatePublicUrl, matchesTarget } from '../verifier/url.js';
-import { DISCOVERY_VERSION, DISCOVERY_LIMITS, IMPORT_SUPPLIERS, DiscoveryCandidate, candidateId, sourceQueryTerms } from './contract.js';
+import { DISCOVERY_VERSION, DISCOVERY_LIMITS, IMPORT_SUPPLIERS, ImportTargetBasis, DiscoveryCandidate, candidateId, sourceQueryTerms } from './contract.js';
 
 export class ImportError extends Error {
   constructor(code, status = 400) { super(code); this.name = 'ImportError'; this.code = code; this.status = status; }
@@ -30,6 +30,7 @@ const fail = (code, status) => { throw new ImportError(code, status); };
 export const ImportRow = z.object({
   source_url: z.string().max(4096),
   target_url: z.string().max(4096),
+  target_basis: ImportTargetBasis.optional(),
   anchor: z.string().max(4096).nullish(),
   rel: z.union([z.string().max(1024), z.array(z.string().max(128)).max(32)]).nullish(),
   dofollow: z.boolean().nullish(),
@@ -71,6 +72,16 @@ export function normalizeRow(row, { runId, workspaceId, projectId, supplier, tas
   // A row about someone else's site is not a row about this import's target. Accepting it
   // would quietly widen the run's scope past what its coverage statement describes.
   if (!matchesTarget(destination.url, target, scope)) return { error: 'target_url: outside the scope of this import' };
+  if (value.target_basis === 'user_supplied_target_expectation') {
+    if (supplier !== 'google_search_console')
+      return { error: 'target_basis: a target expectation is supported only for Google Search Console imports' };
+    const expected = validatePublicUrl(target);
+    const siteScope = scope === 'domain' || scope === 'subdomain';
+    const rootExpectation = expected.valid && new URL(expected.url).pathname === '/' && !new URL(expected.url).search;
+    if (!expected.valid || expected.url !== target || (!siteScope && scope !== 'exact')
+      || (siteScope && !rootExpectation) || destination.url !== target)
+      return { error: 'target_basis: the target expectation must match the canonical import target' };
+  }
 
   const dates = {};
   for (const field of ['first_seen', 'last_seen']) {
@@ -101,6 +112,7 @@ export function normalizeRow(row, { runId, workspaceId, projectId, supplier, tas
       source_http_status: null, target_http_status: null, links_count: null,
       provider_metrics: { imported: {
         supplier, supplier_row_id: value.supplier_row_id ?? null,
+        ...(supplier === 'google_search_console' && value.target_basis !== undefined ? { target_basis: value.target_basis } : {}),
         supplier_generated_at: generatedAt, supplier_metrics: value.supplier_metrics ?? null,
       } },
       verification_status: 'not_checked', verified_at: null, observation_id: null, evidence_id: null,

package/src/mentions.js

@@ -0,0 +1,193 @@
+import {pageQualityReason} from './verifier/page-quality.js';
+import { analyzeReadiness } from './verifier/readiness.js';
+// Unlinked mentions: a page that names the customer and does not link to them (DP-0009-T03).
+//
+// The [scope rule](../docs/PRODUCT-DIRECTION-2026-09-10.md) admits anything that fits declare,
+// verify, watch, diff, and names reclamation first. This fits exactly, and it is the cheapest new
+// observation available because both hard parts already exist: the verifier parses a page into
+// per-occurrence links with raw rel tokens, and the corpus holds real pages to test against.
+//
+// **Everything interesting here is a refusal.** Finding the string is trivial; the work is in the
+// four cases where finding it means nothing.
+import { extractPage, STATIC_TEXT_LIMIT } from './page-extraction/index.js';
+import { matchesTarget, validatePublicUrl, hasRedirectWrapperToTarget } from './verifier/url.js';
+
+export const MENTION_LIMITS = Object.freeze({
+  occurrences: 25,
+  contextChars: 200,
+  brands: 5,
+  textChars: STATIC_TEXT_LIMIT,
+});
+
+export class MentionError extends Error {
+  constructor(code) { super(code); this.name = 'MentionError'; this.code = code; }
+}
+
+/** Escapes a literal for use inside a regular expression. */
+const literal = value => String(value).replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
+
+/**
+ * Where a needle appears in text, with bounded context and a character offset.
+ *
+ * Matching is word-bounded on both sides so `customer.com` does not match inside
+ * `notcustomer.community` — the same near-miss the URL layer refuses for links, applied to text.
+ */
+export function findOccurrences(text, needle, { limit = MENTION_LIMITS.occurrences, contextChars = MENTION_LIMITS.contextChars } = {}) {
+  const found = [];
+  // Counted past the cap on purpose. `found.length` is how many we KEPT, and reporting that as
+  // the number of mentions would turn a storage limit into a measurement — a page naming a brand
+  // eighty times would read as naming it twenty-five, and nothing in the output would say so.
+  let total = 0;
+    // The lookahead refuses a following label too, so `customer.com.evil.com` is not
+  // `customer.com` — the same near-miss the URL layer refuses for links. A trailing full stop at
+  // the end of a sentence still matches, because `\.?` consumes it and a space follows.
+  const pattern = new RegExp(`(?<![\\p{L}\\p{N}@._-])${literal(needle)}(?!\\.?[\\p{L}\\p{N}-])`, 'giu');
+  for (const match of String(text).matchAll(pattern)) {
+    total += 1;
+    if (found.length >= limit) continue;
+    const start = Math.max(0, match.index - Math.floor(contextChars / 2));
+    found.push({
+      offset: match.index,
+      text: match[0],
+      context: String(text).slice(start, start + contextChars).trim(),
+    });
+  }
+  found.total = total;
+  found.truncated = total > found.length;
+  return found;
+}
+
+const hostOf = url => { try { return new URL(url).hostname.replace(/^www\./u, ''); } catch { return null; } };
+
+/**
+ * Scans one fetched page for mentions of a target, and decides whether they are reclaimable.
+ *
+ * Takes HTML that has already been fetched — by the verifier, the crawler, or a caller holding a
+ * corpus page — because a mention scan should cost nothing when we already hold the bytes.
+ *
+ * @returns {{state: 'mentioned_not_linked'|'mentioned_and_linked'|'not_mentioned'|'unknown'}}
+ */
+export function scanMentions(html, pageUrl, { targetUrl, targetScope = 'domain', brands = [], receipt = null, bodyTruncated = false, textLimit = MENTION_LIMITS.textChars } = {}) {
+  const target = validatePublicUrl(targetUrl);
+  if (!target.valid) throw new MentionError(`invalid_target:${target.reason}`);
+  const page = validatePublicUrl(pageUrl);
+  if (!page.valid) throw new MentionError(`invalid_page:${page.reason}`);
+
+  const result = {
+    v: 1, state: 'unknown', reason: 'not_scanned', page_url: page.url, target_url: target.url,
+    target_scope: targetScope, occurrences: [], occurrence_count: 0, links_to_target: 0, complete: false,
+    provenance: { kind: receipt === null ? 'caller_supplied_html' : 'capture_receipt', qualified: false },
+    coverage: { documentComplete: false, textComplete: false, bodyComplete: receipt === null ? null : false },
+  };
+
+  if (typeof html !== 'string') { result.reason = 'html_required'; return result; }
+  if (bodyTruncated === true) { result.reason = 'incomplete_body'; return result; }
+  let parsed, analysis;
+  try {
+    parsed = extractPage(html, { baseUrl: page.url, receipt, textLimit });
+    analysis = analyzeReadiness({ html, parsed, targetUrl: target.url });
+  }
+  catch (error) { result.reason = error?.reason ?? 'html_parse_error'; return result; }
+  result.coverage = parsed.coverage;
+  result.provenance.qualified = parsed.coverage.transportQualified;
+  if (receipt !== null) result.provenance.representation = parsed.provenance.receipt.representation;
+
+  // The verifier's "cannot conclude absence" rules carry over WHOLE, because the conclusion this
+  // tool draws is an absence: no link on this page. On a page we could not read completely, that
+  // is not a conclusion, and a reclamation tool that forgot it would generate outreach about
+  // links that are already there.
+  const qualityReason = pageQualityReason(parsed, analysis);
+  if (qualityReason) { result.reason = qualityReason; return result; }
+  // The complete link index alone cannot prove an absent mention when text hit its cap.
+  if (!parsed.coverage.textComplete) { result.reason = 'text_extraction_truncated'; return result; }
+
+  const linked = parsed.links.filter(link => matchesTarget(link.targetUrl, target.url, targetScope));
+  // A physical wrapper may already lead to this target. Complete captured bytes
+  // do not resolve that destination; never turn this hint into reclamation.
+  // Evaluate each target separately, and retain genuine direct-link precedence.
+  if (!linked.length && hasRedirectWrapperToTarget(parsed.links, target.url, targetScope)) {
+    result.reason = 'target_in_redirect_parameter';
+    return result;
+  }
+  result.complete = true;
+  result.links_to_target = linked.length;
+
+  // Reader-visible text only: a brand name inside a script, a comment or a JSON payload is not a
+  // mention anyone read.
+  const text = parsed.staticText.text;
+  const host = hostOf(target.url);
+  const needles = [];
+  if (host) needles.push({ kind: 'domain', value: host });
+  for (const brand of brands.slice(0, MENTION_LIMITS.brands)) {
+    const value = String(typeof brand === 'string' ? brand : brand?.value ?? '').trim();
+    const forced = typeof brand === 'object' && brand?.force === true;
+    if (!value) continue;
+    // **Length cannot tell a brand from an ordinary word.** "Settled" is seven characters,
+    // "Notion" and "Stripe" are six, and a customer called any of them cannot be matched on the
+    // word without a dictionary we do not have. So the rule is structural rather than a magic
+    // number: a SINGLE-TOKEN brand needs the customer to say explicitly that they mean it, and a
+    // multi-word one is distinctive enough on its own. The refusal is reported, so a customer
+    // whose brand produced nothing can see why.
+    if (!forced && !/\s/u.test(value)) {
+      result.rejected_brands = [...(result.rejected_brands ?? []),
+        { brand: value, reason: 'single_word_brand_needs_force', note: 'A one-word brand cannot be told apart from ordinary use. Pass { value, force: true } to match it anyway.' }];
+      continue;
+    }
+    needles.push({ kind: 'brand', value });
+  }
+  let total = 0;
+  for (const needle of needles) {
+    const found = findOccurrences(text, needle.value);
+    total += found.total;
+    for (const occurrence of found) result.occurrences.push({ kind: needle.kind, matched: needle.value, ...occurrence });
+  }
+  const kept = result.occurrences.slice(0, MENTION_LIMITS.occurrences);
+  // Three separate numbers, because they answer three different questions: how many times the
+  // page names the target, how many of those we kept context for, and whether anything was cut.
+  result.occurrence_count = total;
+  result.occurrences_kept = kept.length;
+  result.occurrences_truncated = total > kept.length;
+  result.occurrences = kept;
+
+  if (!total) { result.state = 'not_mentioned'; result.reason = 'no_mention_in_complete_page'; return result; }
+  // A mention that is already linked is not an opportunity, it is a placement — and telling a
+  // customer to go ask for a link they already have is the fastest way to lose a publisher.
+  if (linked.length) {
+    result.state = 'mentioned_and_linked';
+    result.reason = 'mention_and_link_present';
+    result.anchors = linked.slice(0, 5).map(link => ({ anchor: link.anchor, rel: link.rel, href: link.href }));
+    return result;
+  }
+  result.state = 'mentioned_not_linked';
+  result.reason = 'mention_without_link_in_complete_page';
+  return result;
+}
+
+/**
+ * Scans many already-fetched pages and reports the reclaimable ones.
+ *
+ * Deliberately takes pages rather than fetching: **a scan over bytes we already hold costs
+ * nothing and is not billed**, which is the pricing contract's fourth principle, and a corpus of
+ * two thousand pages is the obvious first place to point this.
+ */
+export function scanPages(pages, { targetUrl, targetScope = 'domain', brands = [] } = {}) {
+  const results = [], counts = {};
+  for (const page of pages) {
+    let scanned;
+    try { scanned = scanMentions(page.html, page.url, { targetUrl, targetScope, brands,
+      receipt: page.receipt ?? page.capture ?? null, bodyTruncated: page.body_truncated === true || page.bodyTruncated === true }); }
+    catch (error) { scanned = { state: 'unknown', reason: error?.code ?? 'scan_failed', page_url: page.url }; }
+    counts[scanned.state] = (counts[scanned.state] ?? 0) + 1;
+    if (scanned.state !== 'not_mentioned') results.push(scanned);
+  }
+  const readable = (counts.mentioned_not_linked ?? 0) + (counts.mentioned_and_linked ?? 0) + (counts.not_mentioned ?? 0);
+  return {
+    v: 1, target_url: targetUrl, target_scope: targetScope,
+    pages_scanned: pages.length, counts,
+    // Stated because the denominator is the whole question: 12 reclaimable out of 20 readable is
+    // a different fact from 12 out of 2,000, and an unknown is neither a yes nor a no.
+    coverage: { readable, unreadable: pages.length - readable },
+    reclaimable: results.filter(row => row.state === 'mentioned_not_linked'),
+    already_linked: results.filter(row => row.state === 'mentioned_and_linked').length,
+  };
+}

package/src/page-acquisition/acquire.js

@@ -0,0 +1,50 @@
+import {publicFetcher} from '../verifier/fetch.js';
+import {ACQUISITION_LIMITS,CAPTURE_VERSION,PageAcquisitionError,sha256Bytes,decodedDigest,boundedAcquisitionLimit,validateCaptureReceipt} from './contracts.js';
+import {validatePublicUrl} from '../verifier/url.js';
+import {acquisitionBudget,runRecovery} from './recovery.js';
+import {verifyRenderedLinks,unknownLinkObservation} from '../verifier/index.js';
+import {unknownTargetObservation} from '../verifier/target.js';
+
+const budget=acquisitionBudget;
+
+/** A bounded capture primitive. Owner jobs retain storage, entitlement and verdict authority. */
+export function createPageAcquisition({fetchImpl=globalThis.fetch,publicFetchSafe=false,beforeFetch,onRobots,robotsCache=null,clock=Date.now,userAgentContact='https://agentlinkops.com',transportId='workers_http',transportVersion='1'}={}){
+  return {async acquire(url,{signal,timeoutMs=ACQUISITION_LIMITS.timeoutMs,maxBytes=ACQUISITION_LIMITS.maxBytes,deadlineAt=null,document=false,validators=null}={}){
+    if(publicFetchSafe!==true)throw new PageAcquisitionError('public_fetch_not_configured');
+    const checked=validatePublicUrl(url);if(!checked.valid)throw new PageAcquisitionError(`unsafe_url:${checked.reason}`);
+    // Documents (sitemaps) may ask for up to the document bound; pages keep the page bound.
+    const byteCap=boundedAcquisitionLimit(maxBytes,ACQUISITION_LIMITS.maxBytes,document?ACQUISITION_LIMITS.documentBytes:ACQUISITION_LIMITS.maxBytes);
+    const owner=budget({signal,timeoutMs,deadlineAt,clock});
+    const trace={finalUrl:checked.url,httpStatus:null,redirects:[]};
+    try{
+      owner.assert();
+      const result=await publicFetcher({fetchImpl,publicFetchSafe,beforeFetch,onRobots,robotsCache,maxBytes:byteCap,userAgentContact},owner.signal).source(checked.url,trace,{document,validators});
+      owner.assert();
+      if(result.notModified||!result.body)return {...result,trace,capture:null};
+      const {body}=result;
+      const capture={version:CAPTURE_VERSION,representation:document?'document':'raw_html',requestedUrl:checked.url,finalUrl:trace.finalUrl,httpStatus:trace.httpStatus,
+        acquiredAt:new Date(clock()).toISOString(),body:{complete:true,truncated:false,bytes:body.bytes,sha256:await sha256Bytes(body.data??new Uint8Array(0)),decodedSha256:await decodedDigest(body.text),charset:body.charset??null},
+        transport:{id:transportId,version:transportVersion,qualified:true},coverage:{mainDocument:true,iframes:false,shadowRoots:false}};
+      owner.assert();
+      return {...result,trace,capture:validateCaptureReceipt(capture)};
+    }catch(error){error.acquisitionTrace=trace;throw error;}finally{owner.close();}
+  }};
+}
+
+/** One logical observation, with a shared absolute budget across direct and alternate routes. */
+export async function verifyWithAcquisition(inputs,options={},{verifyGroup,alternate=null,proxy=null,authorize=async()=>true,recoveryCapabilities={},renderAdmission,proxyAdmission,clock=Date.now}={}){
+  const raw=settings=>({...settings,renderPage:undefined,compareRendered:false});
+  const transport=(provider,method)=>({capability:recoveryCapabilities[method]??{available:!!provider,reason:'disabled'},receipts:provider?.receipts,
+    execute:settings=>verifyGroup(inputs,{...raw(settings),fetchImpl:(url,requestOptions)=>provider.fetch(url,{...requestOptions,deadlineAt:settings.deadlineAt}),
+      captureTransport:{id:method,version:'alo-public-node-v1'}})});
+  return runRecovery({purpose:options.acquisitionPurpose??'customer_monitor',options,authorize,clock,
+    directFailure:reason=>inputs.map(input=>input.sourceUrl===undefined?unknownTargetObservation(input,{now:options.now,reason}):unknownLinkObservation(input,{now:options.now,reason})),
+    executeDirect:settings=>verifyGroup(inputs,raw(settings)),routes:{
+      node_http:transport(alternate,'node_http'),proxy_http:{...transport(proxy,'proxy_http'),admit:proxyAdmission},
+      node_browser:{capability:recoveryCapabilities.node_browser??{available:!!options.renderPage,reason:'disabled'},admit:renderAdmission,
+        receipts:options.renderPage?.receipts,execute:async(settings,previous,indices)=>{
+          const observed=await verifyRenderedLinks(indices.map(i=>inputs[i]),indices.map(i=>previous[i]),settings);
+          const next=[...previous];indices.forEach((index,i)=>{next[index]=observed[i];});return next;
+        }},
+    }});
+}

package/src/page-acquisition/bytes.js

@@ -0,0 +1,138 @@
+/** Shared error identity, including callers of the legacy verifier/fetch.js facade. */
+export class VerificationError extends Error {
+  constructor(reason, details = {}) {
+    super(reason);
+    this.name = 'VerificationError';
+    this.reason = reason;
+    Object.assign(this, details);
+  }
+}
+
+/** Cancellation is best effort and must not extend the owner's deadline. */
+export function cancelBody(response) {
+  try { response.body?.cancel().catch(() => {}); } catch { /* Already locked/closed. */ }
+}
+
+/** Race every blocking phase against the same caller-owned signal. */
+export function withAbort(promise, signal) {
+  if (!signal) return Promise.resolve(promise);
+  if (signal.aborted) return Promise.reject(new VerificationError('timeout'));
+  return new Promise((resolve, reject) => {
+    const cleanup = () => signal.removeEventListener('abort', onAbort);
+    const onAbort = () => { cleanup(); reject(new VerificationError('timeout')); };
+    signal.addEventListener('abort', onAbort, { once: true });
+    Promise.resolve(promise).then(
+      value => { cleanup(); resolve(value); },
+      error => { cleanup(); reject(error); },
+    );
+  });
+}
+
+/**
+ * Consume a body without decoding it. A transport may expose decoded response bytes
+ * even with Content-Encoding present, so that header disables wire-length equality.
+ * The returned bytes describe exactly the representation the reader consumed.
+ */
+export async function readBoundedBytes(response, maxBytes, signal) {
+  const length = response.headers.get('content-length');
+  const encoded = response.headers.get('content-encoding');
+  if (!encoded && length && Number(length) > maxBytes) {
+    cancelBody(response);
+    throw new VerificationError('body_too_large');
+  }
+  const reader = response.body?.getReader();
+  if (!reader) return { data: new Uint8Array(0), bytes: 0 };
+  const chunks = [];
+  let bytes = 0;
+  try {
+    while (true) {
+      const { done, value } = await withAbort(reader.read(), signal);
+      if (done) break;
+      bytes += value.byteLength;
+      if (bytes > maxBytes) throw new VerificationError('body_too_large');
+      chunks.push(value);
+    }
+    if (!encoded && length && Number(length) !== bytes) throw new VerificationError('incomplete_body');
+    const data = new Uint8Array(bytes);
+    let offset = 0;
+    for (const chunk of chunks) { data.set(chunk, offset); offset += chunk.byteLength; }
+    return { data, bytes };
+  } catch (error) {
+    reader.cancel().catch(() => {});
+    throw error;
+  } finally {
+    reader.releaseLock();
+  }
+}
+
+/** Decode only a complete bounded representation; invalid input is never empty HTML. */
+export function decodeBytes(data, charset = 'utf-8') {
+  try { return new TextDecoder(charset, { fatal: true }).decode(data); }
+  catch { throw new VerificationError('unsupported_or_invalid_encoding'); }
+}
+
+/** Existing verifier API, including its no-body return shape and charset behavior. */
+export async function readBounded(response, maxBytes, signal) {
+  const body = await readBoundedBytes(response, maxBytes, signal);
+  if (!response.body) return { text: '', bytes: 0 };
+  const contentType = response.headers.get('content-type') ?? '';
+  const charset = /charset\s*=\s*["']?([^\s;"']+)/iu.exec(contentType)?.[1] ?? 'utf-8';
+  return { text: decodeBytes(body.data, charset), bytes: body.bytes, data: body.data };
+}
+
+/** A refusal may retain a small diagnostic prefix; it is never a complete page capture. */
+export async function readBodyDiagnostic(response, signal, maxBytes = 64 * 1024) {
+  const reader=response.body?.getReader(),chunks=[];
+  const controller=new AbortController(),abort=()=>controller.abort();
+  signal?.addEventListener('abort',abort,{once:true});
+  if(signal?.aborted)abort();
+  // Diagnostics must not consume the recovery window or wait for a publisher
+  // to finish an error stream. The caller's earlier deadline still wins.
+  const timer=setTimeout(abort,250);
+  let bytes=0,complete=!reader,truncated=false;
+  try {
+    if(reader)while(true){
+      const {done,value}=await withAbort(reader.read(),controller.signal);
+      if(done){complete=true;break;}
+      const keep=Math.min(value.byteLength,maxBytes-bytes);
+      if(keep){chunks.push(value.slice(0,keep));bytes+=keep;}
+      if(keep<value.byteLength||bytes===maxBytes){truncated=true;reader.cancel().catch(()=>{});break;}
+    }
+  }catch(error){
+    reader?.cancel().catch(()=>{});
+    if(signal?.aborted||!controller.signal.aborted)throw error;
+    truncated=true;
+  }finally{clearTimeout(timer);signal?.removeEventListener('abort',abort);reader?.releaseLock();}
+  const data=new Uint8Array(bytes);let offset=0;
+  for(const chunk of chunks){data.set(chunk,offset);offset+=chunk.byteLength;}
+  const length=response.headers.get('content-length');
+  if(!response.headers.get('content-encoding')&&length&&Number(length)!==bytes)complete=false;
+  const sha256=Array.from(new Uint8Array(await crypto.subtle.digest('SHA-256',data)),b=>b.toString(16).padStart(2,'0')).join('');
+  const text=new TextDecoder('utf-8').decode(data);
+  // Saved diagnostics carry facts and a digest, never publisher/error text or cookies.
+  return {version:1,bytes,complete,truncated,sha256,
+    looksLikeHtml:/<(?:!doctype\s+html|html|body|head)\b/iu.test(text)};
+}
+
+/**
+ * Inflate incrementally and apply the cap to output chunks before retaining them.
+ * Unlike synchronous gunzip, this never materializes an unbounded expanded document.
+ * Web streams keep the implementation usable in both Workers and Node runtimes.
+ */
+export async function inflateGzipBounded(data, maxBytes, signal) {
+  if (signal?.aborted) throw new VerificationError('timeout');
+  // Feed small slices: a transform must not be handed an entire compressed bomb in
+  // one write before its output consumer has an opportunity to enforce the bound.
+  let offset = 0;
+  const compressed = new ReadableStream({
+    pull(controller) {
+      if (signal?.aborted) { controller.error(new VerificationError('timeout')); return; }
+      if (offset >= data.length) { controller.close(); return; }
+      const next = Math.min(offset + 1024, data.length);
+      controller.enqueue(data.subarray(offset, next));
+      offset = next;
+    },
+  });
+  const expanded = compressed.pipeThrough(new DecompressionStream('gzip'));
+  return readBoundedBytes(new Response(expanded), maxBytes, signal);
+}

package/src/page-acquisition/capture.js

@@ -0,0 +1,11 @@
+import {CAPTURE_VERSION} from './contracts.js';
+
+/** Constructed only after the bounded reader has finished and both digests exist. */
+export function capturedDocument(result,{requestedUrl,representation='raw_html',transportId='public_http',transportVersion='1',qualified=false,charset=null}={}){
+  const evidence=result.evidence;
+  if(!evidence||!Number.isSafeInteger(evidence.bytes)||!evidence.sha256||!evidence.decodedSha256)return null;
+  return {version:CAPTURE_VERSION,representation,requestedUrl,finalUrl:result.finalUrl,httpStatus:result.httpStatus,
+    acquiredAt:evidence.fetchedAt??result.checkedAt,
+    body:{complete:true,truncated:false,bytes:evidence.bytes,sha256:evidence.sha256,decodedSha256:evidence.decodedSha256,charset},
+    transport:{id:transportId,version:transportVersion,qualified},coverage:{mainDocument:true,iframes:false,shadowRoots:false}};
+}