agentlinkops 0.5.0

package/src/corpus/acquire.js

@@ -0,0 +1,135 @@
+import { publicFetcher, VerificationError } from '../verifier/fetch.js';
+import { validatePublicUrl } from '../verifier/url.js';
+
+// One interface over every way we acquire a page, so a runtime can be added without
+// rebuilding the corpus. Direct fetching is implemented here on the verifier's public
+// fetcher, which already validates every redirect hop, applies robots per hop, bounds the
+// body and refuses a transport that follows redirects on its own.
+//
+// Rendered and proxied acquisition are declared and NOT configured. They return an explicit
+// `not_configured` outcome, the same stance the discovery adapter takes towards a supplier
+// it has no credentials for: an unconfigured path yields an error, never fabricated data.
+export const FETCH_KINDS = Object.freeze(['direct', 'rendered', 'proxied']);
+export const ACQUISITION_LIMITS = Object.freeze({ maxBytes: 2 * 1024 * 1024, timeoutMs: 20_000, maxRedirects: 5 });
+
+// Every way a page attempt can end. The frontier records the outcome; none of them is ever
+// allowed to reach the corpus as "this page has no links".
+export const OUTCOMES = Object.freeze(['fetched', 'not_found', 'blocked', 'unavailable', 'not_configured', 'deferred']);
+
+const BLOCKED_REASONS = /^(robots_disallowed|robots_|access_challenge|source_http_(401|403|429))/;
+// A slot we could not take is not the publisher's fault and not a defect: retry it later.
+const DEFERRABLE = new Set(['origin_busy']);
+const NOT_FOUND = new Set([404, 410]);
+
+export class AcquisitionError extends Error {
+  constructor(reason) { super(reason); this.name = 'AcquisitionError'; this.reason = reason; }
+}
+
+/** Sorts a fetch failure into an outcome. Everything unrecognised is unavailable, never empty. */
+export function classifyFailure(reason) {
+  if (DEFERRABLE.has(reason)) return 'deferred';
+  if (reason === 'robots_crawl_delay_requires_scheduler') return 'blocked';
+  if (BLOCKED_REASONS.test(reason)) return 'blocked';
+  if (/^source_http_(404|410)$/.test(reason)) return 'not_found';
+  return 'unavailable';
+}
+
+/**
+ * @param options.publicFetchSafe  Asserts the transport enforces a public-only destination
+ *   boundary, which lexical URL screening cannot. Required for direct fetching.
+ * @param options.runtimes  { rendered, proxied } acquisition runtimes. Each must assert its
+ *   OWN boundary; see the safety note on `assertRuntime`.
+ * @param options.beforeFetch  Pacing hook, called before robots and page requests. Required
+ *   whenever a publisher asks for a crawl delay, which is why the frontier owns scheduling.
+ */
+export function createAcquisition(options = {}) {
+  const maxBytes = Math.min(options.maxBytes ?? ACQUISITION_LIMITS.maxBytes, ACQUISITION_LIMITS.maxBytes);
+  const timeoutMs = Math.min(Math.max(options.timeoutMs ?? ACQUISITION_LIMITS.timeoutMs, 1), ACQUISITION_LIMITS.timeoutMs);
+  const runtimes = options.runtimes ?? {};
+  const clock = options.now ?? (() => new Date().toISOString());
+
+  async function direct(url, beforeFetch) {
+    // The public boundary is a property of the transport, not of this module. Refuse rather
+    // than fetch on an unasserted one; a lexically public hostname can still resolve inside.
+    if (options.publicFetchSafe !== true) throw new AcquisitionError('public_fetch_boundary_not_asserted');
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    const trace = { finalUrl: url, redirects: [], robots: null, httpStatus: null };
+    try {
+      const fetcher = publicFetcher({ ...options, beforeFetch: beforeFetch ?? options.beforeFetch, maxBytes, maxRedirects: ACQUISITION_LIMITS.maxRedirects }, controller.signal);
+      const { response, body } = await fetcher.source(url, trace);
+      if (NOT_FOUND.has(response.status)) return { outcome: 'not_found', trace, html: null };
+      return { outcome: 'fetched', trace, html: body?.text ?? '', bytes: body?.bytes ?? 0 };
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  /**
+   * A runtime that is not the platform's own public fetch does NOT inherit its guarantees.
+   * A proxy or headless browser resolves and connects on its own, so the boundary that makes
+   * direct fetching safe says nothing about it. Each must assert its own, explicitly.
+   */
+  function assertRuntime(kind) {
+    const runtime = runtimes[kind];
+    if (!runtime) throw new AcquisitionError(`${kind}_runtime_not_configured`);
+    if (runtime.publicFetchSafe !== true) throw new AcquisitionError(`${kind}_runtime_boundary_not_asserted`);
+    if (typeof runtime.fetchPage !== 'function') throw new AcquisitionError(`${kind}_runtime_invalid`);
+    return runtime;
+  }
+
+  return {
+    /**
+     * Whether a fetch kind can actually be served. The crawl loop asks before escalating a
+     * page to a browser, because queueing work no runtime can serve would leave the frontier
+     * permanently non-empty and the run never finished.
+     */
+    supports(kind) {
+      if (kind === 'direct') return options.publicFetchSafe === true;
+      const runtime = runtimes[kind];
+      return Boolean(runtime) && runtime.publicFetchSafe === true && typeof runtime.fetchPage === 'function';
+    },
+
+    /**
+     * Acquires one page. Always resolves: an attempt that failed is a recorded outcome, not
+     * a thrown error, because the corpus must keep attempted outcomes and not only successes.
+     */
+    async fetchPage(rawUrl, { kind = 'direct', beforeFetch = null } = {}) {
+      const at = clock();
+      const base = { url: rawUrl, final_url: null, kind, http_status: null, robots: null,
+        html: null, body_truncated: false, fetched_at: at, reason: null };
+      if (!FETCH_KINDS.includes(kind)) return { ...base, outcome: 'unavailable', reason: 'unsupported_fetch_kind' };
+      const checked = validatePublicUrl(rawUrl);
+      if (!checked.valid) return { ...base, outcome: 'unavailable', reason: `unsafe_url:${checked.reason}` };
+
+      try {
+        if (kind === 'direct') {
+          const { outcome, trace, html } = await direct(checked.url, beforeFetch);
+          return { ...base, outcome, final_url: trace.finalUrl, http_status: trace.httpStatus,
+            robots: trace.robots, redirects: trace.redirects, html,
+            // The verifier's reader refuses an oversized body outright, so a returned body was
+            // read to the end. Truncation reaches extraction from runtimes with their own caps.
+            body_truncated: false, render_ms: 0, proxy_bytes: 0 };
+        }
+        const runtime = assertRuntime(kind);
+        const result = await runtime.fetchPage(checked.url, { maxBytes, timeoutMs });
+        return { ...base, outcome: result.outcome ?? 'fetched', final_url: result.finalUrl ?? checked.url,
+          http_status: result.httpStatus ?? null, html: result.html ?? null,
+          body_truncated: result.bodyTruncated === true, reason: result.reason ?? null,
+          // The units a runtime actually consumed. Rendering and proxy traffic are what make
+          // these paths expensive, so dropping them here would leave every metered run
+          // reporting zero and make the ceiling unenforceable for exactly the kinds that need
+          // one. A direct fetch reports neither, which is correct: it spends neither.
+          render_ms: Number.isInteger(result.renderMs) ? result.renderMs : 0,
+          proxy_bytes: Number.isInteger(result.proxyBytes) ? result.proxyBytes : 0 };
+      } catch (error) {
+        // Any error carrying its own reason keeps it, including the pacing hook's, so a
+        // slot we could not take is classified as deferrable rather than as a failure.
+        const reason = typeof error?.reason === 'string' ? error.reason : 'acquisition_error';
+        const outcome = error instanceof AcquisitionError && /not_configured|not_asserted|_invalid$/.test(reason)
+          ? 'not_configured' : classifyFailure(reason);
+        return { ...base, outcome, reason };
+      }
+    },
+  };
+}

package/src/corpus/seeds.js

@@ -0,0 +1,182 @@
+import { validatePublicUrl } from '../verifier/url.js';
+
+// Turns a publisher's own declarations into crawl seeds. Pure: parsing only, no fetching.
+//
+// Sitemap discovery lives here rather than in the verifier's robots parser because it is a
+// crawl concern. `robotsDecision` answers "may I fetch this URL"; a Sitemap directive is a
+// non-group line that answers a different question and must not change that verdict.
+export const SEED_LIMITS = Object.freeze({
+  documentBytes: 10 * 1024 * 1024, // sitemap protocol caps an uncompressed file at 50 MB; we read far less
+  urlsPerDocument: 50_000,         // the protocol's own per-file URL cap
+  sitemapDirectives: 100,
+  indexDepth: 2,                   // an index of indexes is legal; deeper is a loop or a mistake
+});
+
+const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" };
+
+export class SeedError extends Error {
+  constructor(reason) { super(reason); this.name = 'SeedError'; this.reason = reason; }
+}
+
+function decodeXml(value) {
+  return value.replace(/&(?:#(\d{1,7})|#x([0-9a-f]{1,6})|([a-z]+));/giu, (whole, dec, hex, name) => {
+    if (dec || hex) {
+      const code = Number.parseInt(dec ?? hex, dec ? 10 : 16);
+      return Number.isSafeInteger(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : whole;
+    }
+    return ENTITIES[name.toLowerCase()] ?? whole;
+  });
+}
+
+/**
+ * Absolute, public sitemap URLs declared in a robots.txt body. A `Sitemap:` line is
+ * position-independent and belongs to no user-agent group, so it is read from the whole file.
+ */
+export function sitemapDirectives(robotsText, { baseUrl } = {}) {
+  if (typeof robotsText !== 'string') return [];
+  const found = [];
+  for (const line of robotsText.replace(/^/u, '').split(/\r?\n/u)) {
+    const stripped = line.split('#', 1)[0].trim();
+    const colon = stripped.indexOf(':');
+    if (colon < 0 || stripped.slice(0, colon).trim().toLowerCase() !== 'sitemap') continue;
+    const value = stripped.slice(colon + 1).trim();
+    if (!value) continue;
+    let absolute;
+    try { absolute = baseUrl ? new URL(value, baseUrl).href : value; } catch { continue; }
+    const checked = validatePublicUrl(absolute);
+    // A sitemap on another host is legal but is a scope decision for the caller, not a
+    // silent expansion here, so it is returned with its host intact and judged upstream.
+    if (checked.valid && !found.includes(checked.url)) found.push(checked.url);
+    if (found.length >= SEED_LIMITS.sitemapDirectives) break;
+  }
+  return found;
+}
+
+/**
+ * A sitemap may only declare URLs at or below its own path. This is the "cross-submit" rule
+ * from the sitemap protocol, and it is a scope control, not a formality: without it a sitemap
+ * at `anyone.example/sitemap.xml` could inject URLs for a host we never chose to crawl, and a
+ * sitemap at `/blog/sitemap.xml` could claim the whole site. The reference implementation
+ * (crawler-commons) enforces exactly this, with a lenient mode; we have no lenient mode.
+ */
+export function withinSitemapScope(candidate, sitemapUrl, scope = 'path') {
+  let url, base;
+  try { url = new URL(candidate); base = new URL(sitemapUrl); } catch { return false; }
+  if (url.hostname !== base.hostname) return false;
+  // A sitemap NAMED IN robots.txt is authorised for the whole host, because robots.txt sits
+  // at the root and is itself the authorisation. Only a sitemap found some other way is
+  // confined to its own directory. Without this, a perfectly ordinary site that keeps its
+  // sitemap at /sitemaps/example.com/sitemap.xml has every one of its URLs rejected —
+  // measured against nolo.com, which is exactly that shape.
+  if (scope === 'host') return true;
+  const directory = base.pathname.slice(0, base.pathname.lastIndexOf('/') + 1);
+  return url.pathname.startsWith(directory);
+}
+
+/**
+ * Reads a sitemap or sitemap index.
+ *
+ * Deliberately a bounded scan for `<loc>` values rather than a full XML parse: we consume
+ * exactly one element type from a document written by a third party, and a scanner cannot be
+ * driven into entity expansion or deep-nesting attacks by hostile markup. The tradeoff is
+ * recorded rather than hidden — this does not validate the document, and a `<loc>` inside an
+ * unexpected element would still be read.
+ *
+ * @throws SeedError when the document is too large or is not a sitemap at all.
+ */
+export function parseSitemap(input, { baseUrl, enforceScope = true, scope = 'path' } = {}) {
+  if (typeof input !== 'string') throw new SeedError('sitemap_not_text');
+  if (input.length > SEED_LIMITS.documentBytes) throw new SeedError('sitemap_too_large');
+  // A byte-order mark before the root element defeats every subsequent match.
+  const xml = input.replace(/^\uFEFF/u, '').replace(/^\s+/u, '');
+  // Gzipped sitemaps are extremely common and are the acquisition layer's job to decompress.
+  // Say so rather than reporting a valid .xml.gz as "not a sitemap".
+  if (xml.charCodeAt(0) === 0x1f && xml.charCodeAt(1) === 0x8b) throw new SeedError('sitemap_is_gzipped');
+  const indexAt = xml.search(/<\s*sitemapindex[\s>]/iu);
+  const setAt = xml.search(/<\s*urlset[\s>]/iu);
+  // A plain-text sitemap is one URL per line and is part of the protocol.
+  if (indexAt < 0 && setAt < 0) return parseTextSitemap(xml, { baseUrl, enforceScope, scope });
+  const kind = indexAt >= 0 && (setAt < 0 || indexAt < setAt) ? 'index' : 'urlset';
+
+  const urls = [];
+  const skipped = {};
+  let truncated = false;
+  const pattern = /<\s*loc\s*>([\s\S]*?)<\s*\/\s*loc\s*>/giu;
+  for (const match of xml.matchAll(pattern)) {
+    if (urls.length >= SEED_LIMITS.urlsPerDocument) { truncated = true; break; }
+    let raw = decodeXml(match[1].replace(/<!\[CDATA\[([\s\S]*?)\]\]>/gu, '$1')).trim();
+    if (!raw) continue;
+    // The protocol requires an absolute URL here. Resolving a relative value against the
+    // sitemap would turn any stray text into a crawlable path on the publisher's host.
+    const checked = absoluteEntry(raw);
+    if (!checked.valid) { skipped[checked.reason] = (skipped[checked.reason] ?? 0) + 1; continue; }
+    if (enforceScope && baseUrl && !withinSitemapScope(checked.url, baseUrl, scope)) {
+      skipped.out_of_sitemap_scope = (skipped.out_of_sitemap_scope ?? 0) + 1;
+      continue;
+    }
+    if (!urls.includes(checked.url)) urls.push(checked.url);
+  }
+  return { kind, urls, truncated, skipped };
+}
+
+/** A sitemap entry must stand on its own as a public absolute URL. */
+function absoluteEntry(value) {
+  try { new URL(value); } catch { return { valid: false, reason: 'not_absolute' }; }
+  return validatePublicUrl(value);
+}
+
+/** The protocol's plain-text format: one absolute URL per line, no markup. */
+function parseTextSitemap(text, { baseUrl, enforceScope, scope = 'path' }) {
+  const urls = [], skipped = {};
+  let truncated = false;
+  for (const line of text.split(/\r?\n/u)) {
+    const value = line.trim();
+    if (!value || value.startsWith('#')) continue;
+    // Anything with markup is not a text sitemap; do not half-read an unknown document.
+    if (value.startsWith('<')) throw new SeedError('not_a_sitemap');
+    if (urls.length >= SEED_LIMITS.urlsPerDocument) { truncated = true; break; }
+    const checked = absoluteEntry(value);
+    if (!checked.valid) { skipped[checked.reason] = (skipped[checked.reason] ?? 0) + 1; continue; }
+    if (enforceScope && baseUrl && !withinSitemapScope(checked.url, baseUrl, scope)) {
+      skipped.out_of_sitemap_scope = (skipped.out_of_sitemap_scope ?? 0) + 1;
+      continue;
+    }
+    if (!urls.includes(checked.url)) urls.push(checked.url);
+  }
+  if (!urls.length && !Object.keys(skipped).length) throw new SeedError('not_a_sitemap');
+  return { kind: 'urlset', urls, truncated, skipped };
+}
+
+/**
+ * Walks a sitemap index to its leaf URLs, using a caller-supplied fetch so this module still
+ * performs no I/O. `fetchDocument(url)` returns the body text or null when it could not be read.
+ *
+ * A document we could not read is counted, never treated as a sitemap with no URLs — the same
+ * rule extraction follows, for the same reason.
+ */
+export async function expandSitemaps(roots, fetchDocument, { maxDepth = SEED_LIMITS.indexDepth, maxDocuments = SEED_LIMITS.sitemapDirectives, scope = 'path' } = {}) {
+  const urls = [], visited = new Set(), failures = {};
+  let documents = 0, truncated = false;
+  const queue = roots.map(url => ({ url, depth: 0 }));
+  while (queue.length) {
+    const { url, depth } = queue.shift();
+    if (visited.has(url)) continue;
+    visited.add(url);
+    if (documents >= maxDocuments) { truncated = true; break; }
+    documents++;
+    const body = await fetchDocument(url);
+    if (body === null || body === undefined) { failures.unreadable = (failures.unreadable ?? 0) + 1; continue; }
+    let parsed;
+    try { parsed = parseSitemap(body, { baseUrl: url, scope }); }
+    catch (error) { failures[error.reason] = (failures[error.reason] ?? 0) + 1; continue; }
+    if (parsed.truncated) truncated = true;
+    if (parsed.kind === 'index') {
+      // An index deeper than the limit is a loop or a mistake; stop rather than follow it.
+      if (depth >= maxDepth) { failures.index_too_deep = (failures.index_too_deep ?? 0) + 1; continue; }
+      for (const child of parsed.urls) queue.push({ url: child, depth: depth + 1 });
+      continue;
+    }
+    for (const found of parsed.urls) if (!urls.includes(found)) urls.push(found);
+  }
+  return { urls, documents, truncated, failures };
+}

package/src/discovery/contract.js

@@ -0,0 +1,237 @@
+import { z } from 'zod';
+import { validatePublicUrl } from '../verifier/url.js';
+
+// Internal v1 contract only. No route, queue or source is activated by this module.
+export const DISCOVERY_VERSION = 1;
+export const DISCOVERY_LIMITS = Object.freeze({ rows: 1000, bodyBytes: 2 * 1024 * 1024, timeoutMs: 15000, cursorBytes: 8192 });
+
+// Where an edge came from. `linktrail_corpus` is our own crawl; a purchased index is one
+// alternative, never the assumed default. The `provider_*` prefix is deliberately kept for
+// owned rows: it marks upstream data that has NOT been checked against the customer's
+// expectation, whoever produced it. Only the verification_* fields carry a real check.
+export const DISCOVERY_SOURCES = Object.freeze(['linktrail_corpus', 'dataforseo', 'imported']);
+export const DISCOVERY_DATA_MODES = Object.freeze(['synthetic', 'owned_corpus', 'provider_index', 'imported']);
+
+// Suppliers whose exports we know how to read. The name is recorded so a row can always say
+// where it came from; it is NOT a claim that we queried them or that they endorse anything.
+export const IMPORT_SUPPLIERS = Object.freeze([
+  'ahrefs', 'google_search_console', 'bing_webmaster_tools', 'semrush', 'majestic', 'moz', 'dataforseo', 'linkody', 'csv',
+]);
+
+// A data mode belongs to exactly one source, except fixtures, which any source may model.
+const SOURCE_DATA_MODES = Object.freeze({ linktrail_corpus: 'owned_corpus', dataforseo: 'provider_index', imported: 'imported' });
+// A source orders and scores rows in its own terms. An owned corpus has no authority metric,
+// so it must not borrow a supplier's rank scale to look like one — and an IMPORT has no
+// ordering at all. A customer's export arrives in whatever order their tool wrote it, filtered
+// however they filtered it, so `as_supplied` is the only honest ordering to claim.
+const SOURCE_ORDER_BY = Object.freeze({ linktrail_corpus: 'first_seen,desc', dataforseo: 'rank,desc', imported: 'as_supplied' });
+const SOURCE_RANK_SCALE = Object.freeze({ linktrail_corpus: null, dataforseo: 'one_thousand', imported: null });
+const discoverySource = z.enum(DISCOVERY_SOURCES);
+const dataMode = z.enum(DISCOVERY_DATA_MODES);
+
+// Reject a source/mode pair that cannot exist, and hold each source to its own ordering terms.
+function refineSourceShape(row, ctx) {
+  if (row.data_mode !== 'synthetic' && SOURCE_DATA_MODES[row.provider] !== row.data_mode)
+    ctx.addIssue({ code: 'custom', message: `Data mode ${row.data_mode} does not belong to source ${row.provider}.`, path: ['data_mode'] });
+}
+function refineQueryForSource(provider, query, ctx) {
+  if (query.order_by[0] !== SOURCE_ORDER_BY[provider])
+    ctx.addIssue({ code: 'custom', message: `Source ${provider} cannot order by ${query.order_by[0]}.`, path: ['query', 'order_by'] });
+  if (query.rank_scale !== SOURCE_RANK_SCALE[provider])
+    ctx.addIssue({ code: 'custom', message: `Source ${provider} does not publish that rank scale.`, path: ['query', 'rank_scale'] });
+}
+const count = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER);
+const timestamp = z.iso.datetime({ precision: 3 });
+const identifier = z.string().min(1).max(128).regex(/^[a-zA-Z0-9_-]+$/);
+const nullableText = max => z.string().max(max).nullable();
+const publicUrl = z.string().max(4096).refine(value => {
+  const checked = validatePublicUrl(value);
+  return checked.valid && checked.url === value;
+}, 'Expected a canonical public HTTP(S) URL.');
+const hash = z.string().regex(/^[a-f0-9]{64}$/);
+
+export const DiscoveryQuery = z.strictObject({
+  target_kind: z.enum(['domain', 'exact_url']),
+  target: z.string().min(1).max(4096),
+  include_subdomains: z.boolean(),
+  backlinks_status_type: z.enum(['live', 'lost', 'all']),
+  exclude_internal_backlinks: z.boolean(),
+  mode: z.literal('as_is'),
+  filters: z.null(),
+  order_by: z.tuple([z.enum(Object.values(SOURCE_ORDER_BY))]),
+  rank_scale: z.enum(['one_thousand']).nullable(),
+  page_limit: count.min(1).max(DISCOVERY_LIMITS.rows),
+  row_limit: count.min(1).max(DISCOVERY_LIMITS.rows),
+}).superRefine((q, ctx) => {
+  const url = validatePublicUrl(q.target_kind === 'domain' ? `https://${q.target}/` : q.target);
+  if (!url.valid || (q.target_kind === 'domain' ? url.hostname !== q.target || q.target.startsWith('www.') : url.url !== q.target))
+    ctx.addIssue({ code: 'custom', message: 'Invalid or noncanonical discovery target.', path: ['target'] });
+  if (q.target_kind === 'exact_url' && q.include_subdomains)
+    ctx.addIssue({ code: 'custom', message: 'Exact URL scope cannot include subdomains.', path: ['include_subdomains'] });
+  if (q.page_limit > q.row_limit)
+    ctx.addIssue({ code: 'custom', message: 'Page limit exceeds run row limit.', path: ['page_limit'] });
+});
+
+export const DiscoveryCandidate = z.strictObject({
+  v: z.literal(DISCOVERY_VERSION),
+  id: z.string().regex(/^dc_[a-f0-9]{64}$/),
+  workspace_id: identifier, project_id: identifier, discovery_run_id: identifier,
+  source_url: publicUrl, target_url: publicUrl,
+  provider: discoverySource, data_mode: dataMode,
+  provider_task_id: identifier,
+  provider_retrieved_at: timestamp,
+  provider_first_seen: timestamp.nullable(), provider_prev_seen: timestamp.nullable(), provider_last_seen: timestamp.nullable(),
+  provider_status: z.strictObject({ is_lost: z.boolean().nullable(), is_broken: z.boolean().nullable(), is_new: z.boolean().nullable() }),
+  anchor: nullableText(4096), rel: z.array(z.string().min(1).max(128)).max(32).nullable(),
+  dofollow: z.boolean().nullable(), link_type: nullableText(64),
+  source_http_status: z.number().int().min(100).max(599).nullable(),
+  target_http_status: z.number().int().min(100).max(599).nullable(),
+  links_count: count.nullable(),
+  // Exactly one source's metric block, keyed by that source. No shape is mandatory across
+  // sources: an owned crawl publishes what it actually observed and no authority score.
+  provider_metrics: z.union([
+    z.strictObject({ dataforseo: z.strictObject({
+      rank: count.max(1000).nullable(), page_from_rank: count.max(1000).nullable(),
+      domain_from_rank: count.max(1000).nullable(), backlink_spam_score: count.max(100).nullable(),
+      rank_scale: z.literal('one_thousand'),
+    }) }),
+    z.strictObject({ imported: z.strictObject({
+      supplier: z.enum(IMPORT_SUPPLIERS),
+      supplier_row_id: nullableText(256),
+      // What the EXPORT says about its own age, which is often nothing. `provider_retrieved_at`
+      // on the row above is when WE read the file; these are different facts and conflating
+      // them would let an import of a two-year-old export read as fresh data.
+      supplier_generated_at: timestamp.nullable(),
+      // The supplier's own numbers, on the supplier's own scale, never normalised. Ahrefs DR,
+      // Moz DA and Majestic Trust Flow measure different things on different scales; a single
+      // "authority" column would be nonsense dressed as a metric. Each keeps its own name, and
+      // the scale is the supplier's name itself.
+      supplier_metrics: z.record(z.string().min(1).max(64), z.union([z.number(), z.string().max(256), z.null()]))
+        .refine(value => Object.keys(value).length <= 32, 'At most 32 supplier metrics.').nullable(),
+    }) }),
+    z.strictObject({ linktrail_corpus: z.strictObject({
+      // Real crawl facts. Outlink counts qualify a page; they are not an authority metric.
+      source_outlink_count: count.nullable(), source_external_outlink_count: count.nullable(),
+      fetch_kind: z.enum(['direct', 'rendered', 'proxied']),
+      // False when the body was truncated or parsing stopped early. Carries the truncation
+      // rule into the data: an incomplete read can confirm a link it saw, never absence.
+      extraction_complete: z.boolean(),
+    }) }),
+  ]),
+  // Provider flags, dates and scores may never fill these independent check fields.
+  verification_status: z.enum(['not_checked', 'present', 'absent', 'unknown', 'source_unavailable']),
+  verified_at: timestamp.nullable(), observation_id: identifier.nullable(), evidence_id: z.string().min(1).max(1024).nullable(),
+}).superRefine((c, ctx) => {
+  const unverified = c.verification_status === 'not_checked';
+  if ([c.verified_at, c.observation_id, c.evidence_id].some(value => (value === null) !== unverified))
+    ctx.addIssue({ code: 'custom', message: 'A check result requires its own timestamp, observation and evidence; unverified candidates have none.' });
+  refineSourceShape(c, ctx);
+  if (!Object.hasOwn(c.provider_metrics, c.provider))
+    ctx.addIssue({ code: 'custom', message: `Metrics are not keyed by the source ${c.provider} that produced this row.`, path: ['provider_metrics'] });
+});
+
+export const ProviderCheckpoint = z.strictObject({
+  query_hash: hash,
+  page_index: count,
+  offset: count.max(20000),
+  search_after_token: z.string().min(1).max(DISCOVERY_LIMITS.cursorBytes).nullable(),
+  // Preserve the offset of the request that started a token chain, if any.
+  request_offset: count.max(20000).nullable().default(null),
+});
+
+export const ProviderPage = z.strictObject({
+  v: z.literal(DISCOVERY_VERSION), provider: discoverySource, data_mode: dataMode,
+  provider_task_id: identifier, provider_retrieved_at: timestamp,
+  provider_total_count: count.nullable(), provider_reported_cost_microusd: count.nullable(),
+  returned_rows: count.max(DISCOVERY_LIMITS.rows), rejected_rows: count, duplicate_rows: count,
+  candidates: z.array(DiscoveryCandidate).max(DISCOVERY_LIMITS.rows),
+  coverage: z.enum(['complete_for_query', 'capped', 'partial']),
+  coverage_reason: nullableText(128), next_checkpoint: ProviderCheckpoint.nullable(),
+}).superRefine((page, ctx) => {
+  if (page.candidates.length + page.rejected_rows + page.duplicate_rows !== page.returned_rows)
+    ctx.addIssue({ code: 'custom', message: 'Provider page row accounting does not balance.' });
+  if (page.coverage === 'complete_for_query' && (page.rejected_rows > 0 || page.next_checkpoint !== null || page.provider_total_count === null))
+    ctx.addIssue({ code: 'custom', message: 'Provider page cannot claim complete query coverage.' });
+  // An import can never claim completeness. We are reading a file a customer exported, and
+  // nothing in it tells us whether they took page one of forty or filtered it first. Saying
+  // "complete" about someone else's export would be the one claim in this contract we have no
+  // way at all to check.
+  if (page.provider === 'imported' && page.coverage === 'complete_for_query')
+    ctx.addIssue({ code: 'custom', message: 'An import cannot claim complete query coverage: its scope is the export, which is unknown.', path: ['coverage'] });
+  refineSourceShape(page, ctx);
+  if (page.candidates.some(c => c.provider !== page.provider || c.data_mode !== page.data_mode))
+    ctx.addIssue({ code: 'custom', message: 'A page cannot carry rows from another source or data mode.', path: ['candidates'] });
+  // A truncated read cannot have seen every link on the page, so it cannot close the query.
+  if (page.coverage === 'complete_for_query'
+    && page.candidates.some(c => c.provider_metrics.linktrail_corpus?.extraction_complete === false))
+    ctx.addIssue({ code: 'custom', message: 'An incomplete extraction cannot claim complete query coverage.', path: ['coverage'] });
+});
+
+export const DiscoveryRun = z.strictObject({
+  v: z.literal(DISCOVERY_VERSION), id: identifier, workspace_id: identifier, project_id: identifier,
+  provider: discoverySource, data_mode: dataMode,
+  query: DiscoveryQuery, query_hash: hash,
+  status: z.enum(['queued', 'running', 'succeeded', 'partial', 'failed', 'reconciliation_required']),
+  created_at: timestamp, started_at: timestamp.nullable(), finished_at: timestamp.nullable(),
+  coverage: z.enum(['pending', 'complete_for_query', 'capped', 'partial', 'unknown']),
+  coverage_reason: nullableText(128),
+  provider_total_count: count.nullable(), checkpoint: ProviderCheckpoint.nullable(),
+  usage: z.strictObject({
+    unit: z.literal('discovery'), currency: z.literal('USD'), quote_id: identifier,
+    max_cost_microusd: count, reserved_cost_microusd: count,
+    provider_reported_cost_microusd: count.nullable(),
+    request_count: count, returned_rows: count, accepted_candidates: count, rejected_rows: count, duplicate_rows: count,
+  }),
+}).superRefine((run, ctx) => {
+  const u = run.usage;
+  if (u.returned_rows > run.query.row_limit || u.accepted_candidates + u.rejected_rows + u.duplicate_rows !== u.returned_rows)
+    ctx.addIssue({ code: 'custom', message: 'Discovery row accounting does not balance within the cap.' });
+  if (u.reserved_cost_microusd > u.max_cost_microusd)
+    ctx.addIssue({ code: 'custom', message: 'Reservation exceeds the authorized ceiling.' });
+  if (run.checkpoint && run.checkpoint.query_hash !== run.query_hash)
+    ctx.addIssue({ code: 'custom', message: 'Checkpoint belongs to another query.' });
+  if (run.coverage === 'complete_for_query' && (run.status !== 'succeeded' || u.rejected_rows > 0 || run.checkpoint !== null || run.provider_total_count === null))
+    ctx.addIssue({ code: 'custom', message: 'Incomplete or rejected data cannot claim complete query coverage.' });
+  if (run.provider === 'imported' && run.coverage === 'complete_for_query')
+    ctx.addIssue({ code: 'custom', message: 'An import cannot claim complete query coverage: its scope is the export, which is unknown.', path: ['coverage'] });
+  // An import is a file, not a metered request. A ceiling above zero would mean we intended to
+  // spend on it, and a receipt for a purchase nobody made is worse than no receipt.
+  if (run.provider === 'imported' && (u.max_cost_microusd !== 0 || u.reserved_cost_microusd !== 0 || (u.provider_reported_cost_microusd ?? 0) !== 0))
+    ctx.addIssue({ code: 'custom', message: 'An import costs nothing and must not carry a spend.', path: ['usage'] });
+  refineSourceShape(run, ctx);
+  refineQueryForSource(run.provider, run.query, ctx);
+});
+
+export async function discoveryHash(value) {
+  const bytes = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(JSON.stringify(value)));
+  return Array.from(new Uint8Array(bytes), byte => byte.toString(16).padStart(2, '0')).join('');
+}
+
+// Parsing fixes key order before hashing; transport pagination is not query identity.
+export async function queryHash(query) { return discoveryHash(DiscoveryQuery.parse(query)); }
+// Identity is (run, source page, target page), which is exactly the uniqueness the schema
+// enforces. A run already fixes its source, so naming one here would only make the ID
+// unstable if a source were ever relabelled.
+export async function candidateId(runId, sourceUrl, targetUrl) {
+  identifier.parse(runId); publicUrl.parse(sourceUrl); publicUrl.parse(targetUrl);
+  return `dc_${await discoveryHash([DISCOVERY_VERSION, runId, sourceUrl, targetUrl])}`;
+}
+
+// The ordering terms a source accepts, for a caller assembling a query it can run.
+export function sourceQueryTerms(provider) {
+  discoverySource.parse(provider);
+  return { order_by: [SOURCE_ORDER_BY[provider]], rank_scale: SOURCE_RANK_SCALE[provider] };
+}
+
+export async function createDiscoveryRun({ id, workspaceId, projectId, provider, query, dataMode, quoteId, maxCostMicrousd, now }) {
+  const normalized = DiscoveryQuery.parse(query);
+  return DiscoveryRun.parse({
+    v: DISCOVERY_VERSION, id, workspace_id: workspaceId, project_id: projectId,
+    provider: discoverySource.parse(provider), data_mode: dataMode, query: normalized, query_hash: await queryHash(normalized),
+    status: 'queued', created_at: now, started_at: null, finished_at: null,
+    coverage: 'pending', coverage_reason: null, provider_total_count: null, checkpoint: null,
+    usage: { unit: 'discovery', currency: 'USD', quote_id: quoteId, max_cost_microusd: maxCostMicrousd,
+      reserved_cost_microusd: 0, provider_reported_cost_microusd: null, request_count: 0,
+      returned_rows: 0, accepted_candidates: 0, rejected_rows: 0, duplicate_rows: 0 },
+  });
+}