agentlinkops 0.5.0

package/src/discovery/import.js

@@ -0,0 +1,156 @@
+// Imported discovery rows: the customer's own supplier export, as a first-class source.
+//
+// This is the lane that makes "we do not make you depend on our data" true while our corpus is
+// near empty. The customer keeps paying Ahrefs, or exports Search Console, and AgentLinkOps
+// verifies and watches what they already own.
+//
+// Three rules shape it, and each is a refusal rather than a feature.
+//
+//   **An import never claims completeness.** Nothing in an export says whether it is page one
+//   of forty or a filtered view, so coverage is `partial` and says so. It is the one claim in
+//   the discovery contract we would have no way at all to check.
+//
+//   **A malformed row is reported, with its position and what failed.** Never dropped. An
+//   import that silently discards forty rows of a paid export produces a number that is wrong
+//   in a way the customer finds months later.
+//
+//   **The supplier's numbers stay the supplier's.** Ahrefs DR, Moz DA and Majestic Trust Flow
+//   measure different things on different scales. They are carried named and unnormalised, and
+//   nothing here turns them into an AgentLinkOps score.
+import { z } from 'zod';
+import { validatePublicUrl, matchesTarget } from '../verifier/url.js';
+import { DISCOVERY_VERSION, DISCOVERY_LIMITS, IMPORT_SUPPLIERS, DiscoveryCandidate, candidateId, sourceQueryTerms } from './contract.js';
+
+export class ImportError extends Error {
+  constructor(code, status = 400) { super(code); this.name = 'ImportError'; this.code = code; this.status = status; }
+}
+const fail = (code, status) => { throw new ImportError(code, status); };
+
+/** One row as a caller may supply it. Deliberately loose on the supplier's own fields. */
+export const ImportRow = z.object({
+  source_url: z.string().max(4096),
+  target_url: z.string().max(4096),
+  anchor: z.string().max(4096).nullish(),
+  rel: z.union([z.string().max(1024), z.array(z.string().max(128)).max(32)]).nullish(),
+  dofollow: z.boolean().nullish(),
+  link_type: z.string().max(64).nullish(),
+  first_seen: z.string().max(64).nullish(),
+  last_seen: z.string().max(64).nullish(),
+  is_lost: z.boolean().nullish(),
+  supplier_row_id: z.string().max(256).nullish(),
+  supplier_metrics: z.record(z.string().min(1).max(64), z.union([z.number(), z.string().max(256), z.null()])).nullish(),
+});
+
+const REL_TOKEN = /^[a-zA-Z0-9_-]{1,128}$/u;
+const toIso = value => {
+  if (value === null || value === undefined || value === '') return null;
+  const time = Date.parse(value);
+  return Number.isFinite(time) ? new Date(time).toISOString() : undefined;
+};
+
+/**
+ * Normalizes one supplier row into the candidate shape, or explains why it cannot be.
+ *
+ * A supplier date that cannot be parsed is an ERROR rather than a null, because a null would
+ * read as "the supplier did not say" when in fact it said something we failed to understand.
+ */
+export function normalizeRow(row, { runId, workspaceId, projectId, supplier, taskId, retrievedAt, target, scope, generatedAt = null }) {
+  const parsed = ImportRow.safeParse(row);
+  if (!parsed.success) return { error: parsed.error.issues.map(issue => `${issue.path.join('.') || 'row'}: ${issue.message}`).join('; ') };
+  const value = parsed.data;
+  // An EMPTY cell and a malformed URL are different problems with different fixes — a column
+  // mapped to the wrong field, versus rows that need cleaning — and `invalid_url` for both
+  // sends the reader to the wrong one.
+  for (const field of ['source_url', 'target_url']) {
+    if (!String(value[field] ?? '').trim()) return { error: `${field}: empty` };
+  }
+  const source = validatePublicUrl(value.source_url);
+  if (!source.valid) return { error: `source_url: ${source.reason}` };
+  const destination = validatePublicUrl(value.target_url);
+  if (!destination.valid) return { error: `target_url: ${destination.reason}` };
+  // A row about someone else's site is not a row about this import's target. Accepting it
+  // would quietly widen the run's scope past what its coverage statement describes.
+  if (!matchesTarget(destination.url, target, scope)) return { error: 'target_url: outside the scope of this import' };
+
+  const dates = {};
+  for (const field of ['first_seen', 'last_seen']) {
+    const iso = toIso(value[field]);
+    if (iso === undefined) return { error: `${field}: not a date we can read (${String(value[field]).slice(0, 40)})` };
+    dates[field] = iso;
+  }
+  let rel = null;
+  if (value.rel !== null && value.rel !== undefined) {
+    const tokens = (Array.isArray(value.rel) ? value.rel : String(value.rel).split(/[\s,]+/u)).filter(Boolean);
+    if (tokens.some(token => !REL_TOKEN.test(token))) return { error: 'rel: not a list of relation tokens' };
+    rel = [...new Set(tokens.map(token => token.toLowerCase()))].sort();
+  }
+  return {
+    candidate: {
+      v: DISCOVERY_VERSION, id: null,
+      workspace_id: workspaceId, project_id: projectId, discovery_run_id: runId,
+      source_url: source.url, target_url: destination.url,
+      provider: 'imported', data_mode: 'imported',
+      provider_task_id: taskId, provider_retrieved_at: retrievedAt,
+      provider_first_seen: dates.first_seen, provider_prev_seen: null, provider_last_seen: dates.last_seen,
+      provider_status: { is_lost: value.is_lost ?? null, is_broken: null, is_new: null },
+      anchor: value.anchor ?? null, rel,
+      // A supplier that states `rel` and no follow flag has still told us the answer; one that
+      // states neither has not, and a guess would become a fact the moment it was stored.
+      dofollow: value.dofollow ?? (rel ? !rel.includes('nofollow') : null),
+      link_type: value.link_type ?? null,
+      source_http_status: null, target_http_status: null, links_count: null,
+      provider_metrics: { imported: {
+        supplier, supplier_row_id: value.supplier_row_id ?? null,
+        supplier_generated_at: generatedAt, supplier_metrics: value.supplier_metrics ?? null,
+      } },
+      verification_status: 'not_checked', verified_at: null, observation_id: null, evidence_id: null,
+    },
+  };
+}
+
+/**
+ * Turns a supplier export into a run's worth of candidates.
+ *
+ * Returns accepted candidates, per-row rejections and per-row duplicates. The caller decides
+ * what to do with the rejections; this function's only obligation is never to hide them.
+ */
+export async function prepareImport({ runId, workspaceId, projectId, supplier, rows, target, scope, taskId, retrievedAt, generatedAt = null, seen = new Map(), rowOffset = 0 }) {
+  if (!IMPORT_SUPPLIERS.includes(supplier)) fail('UNKNOWN_IMPORT_SUPPLIER');
+  if (!Array.isArray(rows)) fail('INVALID_IMPORT_ROWS');
+  if (rows.length > DISCOVERY_LIMITS.rows) {
+    // A bare code tells the customer nothing about a file they are looking at right now.
+    const error = new ImportError('IMPORT_TOO_LARGE', 413);
+    error.message = `IMPORT_TOO_LARGE: ${rows.length} rows, and one import holds at most ${DISCOVERY_LIMITS.rows}`;
+    throw error;
+  }
+  const accepted = [], rejected = [], duplicates = [];
+  // `seen` may be shared across calls so a file larger than one import can still detect a
+  // placement repeated in two different chunks of itself.
+  for (const [position, row] of rows.entries()) {
+    const index = position + rowOffset;
+    const result = normalizeRow(row, { runId, workspaceId, projectId, supplier, taskId, retrievedAt, target, scope, generatedAt });
+    if (result.error) { rejected.push({ row: index + 1, reason: result.error }); continue; }
+    const candidate = result.candidate;
+    candidate.id = await candidateId(runId, candidate.source_url, candidate.target_url);
+    // The same placement twice in one export is the supplier's duplicate, not a second link.
+    if (seen.has(candidate.id)) { duplicates.push({ row: index + 1, first_seen_on_row: seen.get(candidate.id) }); continue; }
+    seen.set(candidate.id, index + 1);
+    // Parsed rather than trusted: a row that cannot satisfy the candidate contract is a
+    // rejection with a reason, even when it came from our own normalizer.
+    const checked = DiscoveryCandidate.safeParse(candidate);
+    if (!checked.success) { rejected.push({ row: index + 1, reason: checked.error.issues.map(issue => `${issue.path.join('.')}: ${issue.message}`).join('; ') }); continue; }
+    accepted.push(checked.data);
+  }
+  return { accepted, rejected, duplicates, returned_rows: rows.length };
+}
+
+/** The query an import describes: this target, as supplied, no ranking claimed. */
+export function importQuery({ targetKind, target, includeSubdomains = false, statusType = 'all', rowLimit = DISCOVERY_LIMITS.rows }) {
+  return {
+    target_kind: targetKind, target, include_subdomains: includeSubdomains,
+    backlinks_status_type: statusType, exclude_internal_backlinks: true,
+    mode: 'as_is', filters: null,
+    ...sourceQueryTerms('imported'),
+    page_limit: rowLimit, row_limit: rowLimit,
+  };
+}

package/src/tool-views.js

@@ -0,0 +1,117 @@
+// Toolsets, tiers, the card-catalog manifest, the meta tools and the exposure views, derived
+// from registry metadata. Pure: no cloud, filesystem or service imports, so the measurement
+// script, the MCP router, the CLI index and the generators all read the same definitions.
+// Rules: docs/technical/AGENT-SURFACE-CONTRACT.md (DP-0036).
+
+export const TOOLSETS = ['monitoring','evidence','discovery','library','competitors','reports','workspace','notifications','webhooks'];
+
+// One line per toolset: what the group is for. Printed by the site brief, the REST index and the
+// catalog snapshot so the same wording reaches every client; a toolset without a summary is a
+// gate failure in the measurement script.
+export const TOOLSET_SUMMARIES = {
+  monitoring: 'Watch earned links and destination URLs: create, list, update, import, export, recheck.',
+  evidence: 'What a check observed: histories, snapshots, change feeds, check jobs, published contacts.',
+  discovery: 'Import candidate rows, read stored runs, verify selected candidates, enroll them.',
+  library: 'Read and export the coverage-stated opportunity library.',
+  competitors: 'Competitor sets, dated inventories, scheduled refresh, gap and domain-mix reports.',
+  reports: 'Profile, anchor and report summaries over the tracked dataset.',
+  workspace: 'Workspace, projects, usage, members, invitations, scratch-resource cleanup.',
+  notifications: 'Email notification preferences, previews, tests and deliveries.',
+  webhooks: 'Webhook endpoints, state, secrets and delivery records.',
+};
+
+// Provisional core tier until DP-0036-T13 usage evidence replaces it. At most six.
+export const CORE_DEFAULT = ['get_workspace','list_projects','list_link_watches','monitor_link','list_events'];
+
+// Fallback toolset by documentation category and by name, used only for a binding that does not
+// carry an explicit `toolset`. DP-0036-T02 makes every binding explicit; the fallback stays so a
+// new command without metadata is still measured instead of silently uncounted.
+const CATEGORY_TOOLSET = { monitoring:'monitoring', destinations:'monitoring', imports:'discovery', verification:'discovery', competitors:'competitors', reports:'reports', workspace:'workspace', webhooks:'webhooks' };
+const NAME_TOOLSET = {
+  get_evidence:'evidence', locate_link:'evidence', get_history:'evidence',
+  get_public_contacts:'evidence', list_events:'evidence', list_check_jobs:'evidence', get_check_job:'evidence',
+  send_invitation_email:'notifications', preview_notification_email:'notifications', get_notification_preferences:'notifications', save_notification_preferences:'notifications', list_notification_deliveries:'notifications', test_notification_email:'notifications',
+  export_link_watches:'monitoring',
+  list_opportunities:'library', get_opportunity:'library', export_opportunities:'library',
+};
+
+export function toolsetOf(name, binding = {}, category = null) {
+  return binding.toolset ?? NAME_TOOLSET[name] ?? CATEGORY_TOOLSET[category] ?? 'workspace';
+}
+export function tierOf(name, binding = {}) {
+  return binding.tier ?? (CORE_DEFAULT.includes(name) ? 'core' : 'deferred');
+}
+
+// The first sentence of a description, cut to `max` characters on a word boundary.
+export function oneLiner(description, max = 60) {
+  const sentence = (description ?? '').split(/(?<=\.)\s/)[0].trim();
+  if (sentence.length <= max) return sentence;
+  const cut = sentence.slice(0, max + 1);
+  return `${cut.slice(0, Math.max(cut.lastIndexOf(' '), 20)).replace(/[,;:]$/, '')}…`;
+}
+
+// entries: [{ name, description, toolset, tier, readOnly }]. Grouped, one line per command.
+export function manifestText(entries, { max = 60 } = {}) {
+  const groups = new Map(TOOLSETS.map(t => [t, []]));
+  for (const entry of entries) (groups.get(entry.toolset) ?? groups.set(entry.toolset, []).get(entry.toolset)).push(entry);
+  const lines = ['Commands by toolset (name: what it does). Use search_tools or describe_tools for the full schema before calling.'];
+  for (const [toolset, members] of groups) {
+    if (!members.length) continue;
+    lines.push(`\n${toolset}:`);
+    for (const entry of members.sort((a, b) => a.name.localeCompare(b.name))) lines.push(`- ${entry.name}${entry.tier === 'core' ? ' (core)' : ''}${entry.readOnly ? '' : ' [write]'}: ${oneLiner(entry.description, max)}`);
+  }
+  return lines.join('\n');
+}
+
+// The four meta tools of the default view. Their schemas live here so the measurement script
+// counts the real definitions the router will register.
+const argumentsSchema = { type:'object', description:'Arguments for the named command, exactly as its input schema describes them.', additionalProperties:true };
+export const META_TOOLS = [
+  { name:'search_tools', description:`Find AgentLinkOps commands by what you want to do. Searches names, descriptions, parameter names and keywords across the toolsets: ${TOOLSETS.join(', ')}. Returns names with one-line summaries; call describe_tools for a schema.`, readOnly:true,
+    inputSchema:{ type:'object', properties:{ query:{ type:'string', minLength:1, maxLength:500, description:'Natural-language task or keywords.' }, toolset:{ type:'string', enum:TOOLSETS, description:'Restrict results to one toolset.' }, limit:{ type:'integer', minimum:1, maximum:25, description:'Maximum results; default 5.' } }, required:['query'], additionalProperties:false } },
+  { name:'describe_tools', description:'Read the full input schema, examples and scopes for one or more commands by name. Include the output schema on request. Use before run_read_command or run_write_command.', readOnly:true,
+    inputSchema:{ type:'object', properties:{ names:{ type:'array', items:{ type:'string', minLength:1, maxLength:64 }, minItems:1, maxItems:10, description:'Command names from search_tools or the toolset listing.' }, include:{ type:'string', enum:['input','input+output','examples'], description:'What to include; default input.' } }, required:['names'], additionalProperties:false } },
+  { name:'run_read_command', description:'Run a read-only AgentLinkOps command by name with its arguments. Reads never change workspace data. Arguments are validated against the command\'s schema; describe_tools shows it.', readOnly:true,
+    inputSchema:{ type:'object', properties:{ name:{ type:'string', minLength:1, maxLength:64, description:'A read-only command name.' }, arguments:argumentsSchema }, required:['name'], additionalProperties:false } },
+  { name:'run_write_command', description:'Run an AgentLinkOps command that creates, changes or removes workspace data, by name with its arguments. Confirm irreversible changes with the person first. Arguments are validated against the command\'s schema; describe_tools shows it.', readOnly:false,
+    inputSchema:{ type:'object', properties:{ name:{ type:'string', minLength:1, maxLength:64, description:'A write command name.' }, arguments:argumentsSchema }, required:['name'], additionalProperties:false } },
+];
+
+// DP-0036-T12: the three tools of the code view (/mcp/code). `execute` runs the model's
+// JavaScript in a Dynamic Worker isolate with no network; the only way out is
+// `agentlinkops.<command>(args)`, which dispatches to the registry under the caller's scopes.
+export const CODE_TOOLS = [
+  { name:'search', description:'Find AgentLinkOps commands by what you want to do; returns names, toolsets and one-line summaries. Then describe for the exact signature.', readOnly:true,
+    inputSchema:{ type:'object', properties:{ query:{ type:'string', minLength:1, maxLength:500, description:'Natural-language task or keywords.' }, toolset:{ type:'string', enum:TOOLSETS }, limit:{ type:'integer', minimum:1, maximum:25, description:'Maximum results; default 5.' } }, required:['query'], additionalProperties:false } },
+  { name:'describe', description:'Read the TypeScript signature of one or more commands as they are callable from execute: agentlinkops.<name>(args) with the exact argument and result shapes, scopes and examples.', readOnly:true,
+    inputSchema:{ type:'object', properties:{ names:{ type:'array', items:{ type:'string', minLength:1, maxLength:64 }, minItems:1, maxItems:10 } }, required:['names'], additionalProperties:false } },
+  { name:'execute', description:'Run JavaScript against AgentLinkOps in an isolated sandbox. Write the body of an async function; `return` the value you want back; call commands as `await agentlinkops.<name>({...})` (any command, with this connection\'s scopes; retired names resolve). No network, no filesystem, no env: fetch() throws. Results over about 6,000 tokens are truncated with a marker, so filter and aggregate in code. Up to 200 command calls and 30 seconds per execution.', readOnly:false,
+    inputSchema:{ type:'object', properties:{ code:{ type:'string', minLength:1, maxLength:64000, description:'JavaScript: the body of an async function, or an async arrow function.' }, readOnly:{ type:'boolean', description:'When true, any write command call fails instead of running. Default false.' } }, required:['code'], additionalProperties:false } },
+];
+export const CODE_LIMITS = { outputChars:24000, timeoutMs:30000, maxCalls:200 };
+
+// Views. `entries` carry name, toolset, tier, readOnly and (from tool-exposure.js) unavailable.
+// An unavailable command stays a member of its views: it is listed with its marker, never
+// hidden, so a client that calls it gets the stable configuration error rather than not-found.
+export const VIEW_BUDGETS = {
+  default: { definitions:12, tokens:6000 },
+  toolset: { definitions:22, tokens:8000 },
+  code: { definitions:3, tokens:2000 },
+  manifest: { tokens:2500 },
+};
+export function viewMembers(entries, view) {
+  const visible = entries;
+  const core = visible.filter(e => e.tier === 'core');
+  if (view === 'all') return visible;
+  if (view === 'readonly') return visible.filter(e => e.readOnly);
+  if (view === 'default') return core;
+  if (view === 'code') return [];
+  const m = /^toolset:([a-z]+)(:readonly)?$/.exec(view);
+  if (!m) throw new Error(`Unknown view ${view}`);
+  const members = visible.filter(e => e.toolset === m[1] || e.tier === 'core');
+  return m[2] ? members.filter(e => e.readOnly) : members;
+}
+export function viewNames(entries) {
+  const toolsets = [...new Set(entries.map(e => e.toolset))].sort();
+  return ['default','all','readonly','code', ...toolsets.flatMap(t => [`toolset:${t}`, `toolset:${t}:readonly`])];
+}

package/src/verifier/fetch.js

@@ -0,0 +1,303 @@
+import { validatePublicUrl } from './url.js';
+import { robotsDecision } from './robots.js';
+
+export class VerificationError extends Error {
+  constructor(reason, details = {}) {
+    super(reason);
+    this.name = 'VerificationError';
+    this.reason = reason;
+    Object.assign(this, details);
+  }
+}
+
+function cancelBody(response) {
+  // Cancellation must not extend the request deadline on a misbehaving transport.
+  response.body?.cancel().catch(() => {});
+}
+
+export async function readBounded(response, maxBytes, signal) {
+  const length = response.headers.get('content-length');
+  const encoded = response.headers.get('content-encoding');
+  if (!encoded && length && Number(length) > maxBytes) {
+    cancelBody(response);
+    throw new VerificationError('body_too_large');
+  }
+  const reader = response.body?.getReader();
+  if (!reader) return { text: '', bytes: 0 };
+  const chunks = [];
+  let bytes = 0;
+  try {
+    while (true) {
+      const { done, value } = await withAbort(reader.read(), signal);
+      if (done) break;
+      bytes += value.byteLength;
+      if (bytes > maxBytes) throw new VerificationError('body_too_large');
+      chunks.push(value);
+    }
+    if (!encoded && length && Number(length) !== bytes) throw new VerificationError('incomplete_body');
+    const combined = new Uint8Array(bytes);
+    let offset = 0;
+    for (const chunk of chunks) { combined.set(chunk, offset); offset += chunk.byteLength; }
+    const contentType = response.headers.get('content-type') ?? '';
+    const charset = /charset\s*=\s*["']?([^\s;"']+)/iu.exec(contentType)?.[1] ?? 'utf-8';
+    let text;
+    try { text = new TextDecoder(charset, { fatal: true }).decode(combined); }
+    catch { throw new VerificationError('unsupported_or_invalid_encoding'); }
+    return { text, bytes, data: combined };
+  } catch (error) {
+    reader.cancel().catch(() => {});
+    throw error;
+  } finally {
+    reader.releaseLock();
+  }
+}
+
+function withAbort(promise, signal) {
+  if (signal.aborted) return Promise.reject(new VerificationError('timeout'));
+  return new Promise((resolve, reject) => {
+    const onAbort = () => reject(new VerificationError('timeout'));
+    signal.addEventListener('abort', onAbort, { once: true });
+    promise.then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort));
+  });
+}
+
+/** One shared budget for robots, redirects, hooks and source-body reads. */
+export const PRODUCT_TOKEN = 'LinktrailBot';
+
+// A day. Anything longer is a publisher saying "not today", and a scheduler that honoured a
+// month-long Retry-After would silently retire the check instead of retrying it.
+export const MAX_RETRY_AFTER_SECONDS = 86_400;
+
+/**
+ * RFC 9110 `Retry-After`: delta-seconds or an HTTP-date. Returns null for anything else.
+ *
+ * Both forms appear in the wild and a parser that reads only the integer form silently discards
+ * the date form, which is the one a CDN tends to send.
+ */
+export function retryAfterSeconds(value, now = Date.now()) {
+  if (!value) return null;
+  const text = String(value).trim();
+  if (/^\d+$/u.test(text)) {
+    const seconds = Number(text);
+    return Number.isFinite(seconds) ? Math.min(seconds, MAX_RETRY_AFTER_SECONDS) : null;
+  }
+  // `Date.parse` accepts far more than an HTTP-date — it reads "-5" as a year and returns a real
+  // timestamp, which turned a malformed header into a wait of zero. An IMF-fixdate always carries
+  // a month and day name, so requiring a letter is enough to keep the loose parser off junk.
+  if (!/[A-Za-z]/u.test(text)) return null;
+  const at = Date.parse(text);
+  if (!Number.isFinite(at)) return null;
+  // A date already in the past means "now", never a negative wait.
+  return Math.min(Math.max(0, Math.ceil((at - now) / 1000)), MAX_RETRY_AFTER_SECONDS);
+}
+
+export function publicFetcher(options, signal) {
+  const fetchImpl = options.fetchImpl ?? globalThis.fetch;
+  const robotsByOrigin = new Map();
+  // The product token is what robots matches on and never changes. A contact URL may be appended
+  // so a publisher who sees us in their logs can find out who we are — the convention every
+  // reputable crawler follows, and the thing most likely to move a reflexive 403 or 429.
+  //
+  // It defaults to ABSENT on purpose. It was written before AgentLinkOps had a public domain, and a bot
+  // advertising a URL that does not resolve is worse than one advertising nothing: it reads as
+  // impersonation rather than identification. Set it when a real domain exists.
+  const userAgent = options.userAgentContact
+    ? `${PRODUCT_TOKEN}/0.1 (+${String(options.userAgentContact).slice(0, 200)})`
+    : `${PRODUCT_TOKEN}/0.1`;
+  const maxRedirects = Math.min(options.maxRedirects ?? 5, 5);
+  const maxBytes = Math.min(options.maxBytes ?? 2 * 1024 * 1024, 2 * 1024 * 1024);
+
+  async function request(url, context = {}) {
+    const checked = validatePublicUrl(url);
+    if (!checked.valid) throw new VerificationError(`unsafe_url:${checked.reason}`);
+    if (options.beforeFetch) await withAbort(Promise.resolve().then(() => options.beforeFetch(checked.url, { ...context, signal })), signal);
+    let response;
+    try {
+      response = await withAbort(Promise.resolve().then(() => {
+        if (signal.aborted) throw new VerificationError('timeout');
+        return fetchImpl(checked.url, {
+          method: 'GET',
+          redirect: 'manual',
+          credentials: 'omit',
+          signal,
+          headers: {
+            'User-Agent': userAgent,
+            Accept: context.kind === 'robots' ? 'text/plain' : 'text/html',
+          },
+        });
+      }), signal);
+    } catch (error) {
+      if (error instanceof VerificationError) throw error;
+      throw new VerificationError(signal.aborted ? 'timeout' : 'network_error');
+    }
+    // Custom transports must never silently follow redirects or bypass destination checks.
+    if (response.redirected || (response.url && new URL(response.url).href !== checked.url)) {
+      cancelBody(response);
+      throw new VerificationError('transport_followed_redirect');
+    }
+    return response;
+  }
+
+  async function fetchRobots(origin, diagnostic) {
+    let url = `${origin}/robots.txt`;
+    const seen = new Set();
+    for (let redirects = 0; redirects <= maxRedirects; redirects++) {
+      diagnostic.robotsUrl = url;
+      diagnostic.httpStatus = null;
+      if (seen.has(url)) throw new VerificationError('robots_redirect_loop');
+      seen.add(url);
+      const response = await request(url, { kind: 'robots' });
+      diagnostic.httpStatus = response.status;
+      // A challenge replaces the requested resource, even when its status would
+      // otherwise mean missing robots or a redirect. Never infer policy from it.
+      if (response.headers.get('cf-mitigated') === 'challenge') {
+        cancelBody(response);
+        const retry = retryAfterSeconds(response.headers.get('retry-after'));
+        if (retry !== null) diagnostic.retryAfterSeconds = retry;
+        throw new VerificationError('robots_access_challenge', retry === null ? {} : { retryAfterSeconds: retry });
+      }
+      if ([301, 302, 303, 307, 308].includes(response.status)) {
+        cancelBody(response);
+        const location = response.headers.get('location');
+        if (!location) throw new VerificationError('robots_redirect_missing_location');
+        const next = validatePublicUrl(new URL(location, url).href);
+        if (!next.valid) throw new VerificationError(`unsafe_robots_redirect:${next.reason}`);
+        if (diagnostic.redirects.length < 5) diagnostic.redirects.push({ from: url, to: next.url, status: response.status });
+        url = next.url;
+        continue;
+      }
+      // RFC 9309 §2.3.1.3 and Google's implementation agree: a 4xx on robots.txt means the file is
+      // UNAVAILABLE, and an unavailable robots.txt imposes no restrictions. Only 404 and 410 were
+      // honoured that way here, so a 401 or 403 — a WAF answering before the file is even looked
+      // for — was read as "this publisher forbids us" and refused the whole fetch. Measured on 800
+      // real destinations, that was 39 refusals, about 5% of the run, attributed to publishers who
+      // had said nothing.
+      //
+      // Proceeding is also BETTER REPORTING even when the page then fails: `target_http_403` says
+      // the publisher refused the page, where `robots_http_403` said we never asked.
+      if (response.status >= 400 && response.status < 500 && response.status !== 429) {
+        cancelBody(response);
+        return { text: '', reason: response.status === 404 || response.status === 410 ? 'robots_not_found' : `robots_unavailable_${response.status}` };
+      }
+      // 429 and 5xx are the other half of the same rule and stay a refusal: they mean TEMPORARILY
+      // unavailable, and the spec's answer to that is to stay away rather than to assume consent.
+      if (!response.ok) {
+        cancelBody(response);
+        const retry = retryAfterSeconds(response.headers.get('retry-after'));
+        if (retry !== null) diagnostic.retryAfterSeconds = retry;
+        throw new VerificationError(`robots_http_${response.status}`, retry === null ? {} : { retryAfterSeconds: retry });
+      }
+      const body = await readBounded(response, 512 * 1024, signal);
+      if (/text\/html/iu.test(response.headers.get('content-type') ?? '') || /<(?:!doctype|html|body)\b/iu.test(body.text)) {
+        throw new VerificationError('robots_not_plain_text');
+      }
+      return { text: body.text, reason: 'robots_fetched' };
+    }
+    throw new VerificationError('robots_redirect_limit');
+  }
+
+  async function checkRobots(url, trace) {
+    const origin = new URL(url).origin;
+    const diagnostic = { sourceUrl: url, robotsUrl: `${origin}/robots.txt`, httpStatus: null,
+      productToken: PRODUCT_TOKEN, allowed: null, reason: null, matchedRule: null,
+      crawlDelaySeconds: null, fetched: null, redirects: [] };
+    // Write before fetching/evaluating so refusals keep the failing hop, not the
+    // last allowed origin. Both histories are capped by our redirect budget.
+    trace.robots = diagnostic;
+    trace.robotsHistory ??= [];
+    if (trace.robotsHistory.length < 6) trace.robotsHistory.push(diagnostic);
+    try {
+      if (!robotsByOrigin.has(origin)) {
+        const record = await fetchRobots(origin, diagnostic);
+        robotsByOrigin.set(origin, { ...record, diagnostic: structuredClone(diagnostic) });
+      }
+      const record = robotsByOrigin.get(origin);
+      Object.assign(diagnostic, structuredClone(record.diagnostic), { sourceUrl: url });
+      const decision = robotsDecision(record.text, url);
+      Object.assign(diagnostic, decision, { fetched: record.reason });
+      options.onRobots?.(url, decision);
+      if (decision.allowed !== true) throw new VerificationError(decision.reason);
+      if (decision.crawlDelaySeconds > 0 && !options.beforeFetch) {
+        throw new VerificationError('robots_crawl_delay_requires_scheduler');
+      }
+      return diagnostic;
+    } catch (error) {
+      diagnostic.reason = error instanceof VerificationError ? error.reason : 'verification_error';
+      throw error;
+    }
+  }
+
+  return {
+    // Authorize a native browser document without refetching its body through HTTP.
+    // The caller still needs an independently verified browser egress boundary.
+    async authorizeNavigation(url, trace = {}) {
+      const checked = validatePublicUrl(url);
+      if (!checked.valid) throw new VerificationError('unsafe_render_navigation');
+      const robots = await checkRobots(checked.url, trace);
+      if (signal.aborted) throw new VerificationError('timeout');
+      await options.beforeFetch?.(checked.url, { kind: 'source', crawlDelaySeconds: robots.crawlDelaySeconds, signal });
+      if (signal.aborted) throw new VerificationError('timeout');
+      return robots;
+    },
+    async source(initialUrl, trace) {
+      let url = initialUrl;
+      const seen = new Set();
+      for (let redirects = 0; redirects <= maxRedirects; redirects++) {
+        trace.finalUrl = url;
+        if (seen.has(url)) throw new VerificationError('redirect_loop');
+        seen.add(url);
+        const robots = await checkRobots(url, trace);
+        trace.robots = robots;
+        const response = await request(url, { kind: 'source', crawlDelaySeconds: robots.crawlDelaySeconds });
+        trace.httpStatus = response.status;
+        trace.sourceResponse = { url, httpStatus: response.status,
+          contentType: /^[a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+/iu.exec(response.headers.get('content-type') ?? '')?.[0].slice(0, 128) ?? null,
+          retryAfterSeconds: retryAfterSeconds(response.headers.get('retry-after')),
+          challenge: response.headers.get('cf-mitigated') === 'challenge' };
+        if (trace.sourceResponse.challenge) {
+          cancelBody(response);
+          const retry = trace.sourceResponse.retryAfterSeconds;
+          throw new VerificationError('access_challenge', retry === null ? {} : { retryAfterSeconds: retry });
+        }
+        if ([301, 302, 303, 307, 308].includes(response.status)) {
+          cancelBody(response);
+          const location = response.headers.get('location');
+          if (!location) throw new VerificationError('redirect_missing_location');
+          let next;
+          try { next = validatePublicUrl(new URL(location, url).href); }
+          catch { throw new VerificationError('invalid_redirect'); }
+          if (!next.valid) throw new VerificationError(`unsafe_redirect:${next.reason}`);
+          trace.redirects.push({ from: url, to: next.url, status: response.status });
+          url = next.url;
+          continue;
+        }
+        if ([404, 410].includes(response.status)) {
+          cancelBody(response);
+          return { response, body: null };
+        }
+        if (response.status === 304) {
+          cancelBody(response);
+          throw new VerificationError('not_modified_without_baseline');
+        }
+        if (response.status !== 200) {
+          cancelBody(response);
+          // A 429 or a 503 may state WHEN to come back. We keep it and never act on it here:
+          // sleeping inside a request holds an origin lease and a worker for someone else's
+          // backoff, and this system already decided that **the cron is the sleep**. Surfacing it
+          // lets a scheduler re-queue at the stated time, the same way a robots crawl delay is
+          // handed to the scheduler rather than slept on.
+          throw new VerificationError(`source_http_${response.status}`,
+            retryAfterSeconds(response.headers.get('retry-after')) === null ? {}
+              : { retryAfterSeconds: retryAfterSeconds(response.headers.get('retry-after')) });
+        }
+        if (!/^text\/html(?:;|$)/iu.test(response.headers.get('content-type') ?? '')) {
+          cancelBody(response);
+          throw new VerificationError('unsupported_content_type');
+        }
+        const body = await readBounded(response, maxBytes, signal);
+        return { response, body };
+      }
+      throw new VerificationError('redirect_limit');
+    },
+  };
+}