crawldna 0.1.0

package/index.d.ts

@@ -0,0 +1,376 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// Type definitions for crawldna — the public API of `src/index.mjs`.
+// Hand-written (the source is plain ESM JavaScript). Kept in sync with
+// DEFAULT_OPTIONS and the `crawlDocs` result shape documented in README.md.
+
+export type Provider = 'ollama' | 'openai';
+export type BrowserMode = 'never' | 'auto' | 'always';
+/**
+ * What to extract — an explicit switch, never inferred from the task wording.
+ * `"complete"` (default): everything reachable (completeness shortcuts tried first,
+ * pages kept whole, zero AI link-gate/scoping calls). `"targeted"`: only what the
+ * task asks (AI link gate + section scoping — requires AI, incompatible with `noAi`).
+ * `"auto"`: legacy behaviour — a multilingual regex on the task picks the docs path;
+ * never the default (#23), kept only for saved runs and callers that name it.
+ */
+export type CrawlMode = 'auto' | 'complete' | 'targeted';
+
+/** One unit of work: a URL plus the natural-language task describing what to extract. */
+export interface Target {
+  url: string;
+  /** What to extract. Falls back to `options.task` when omitted. */
+  task?: string;
+}
+
+/** Accepted shapes for the first argument of {@link crawlDocs}. */
+export type Targets = string | string[] | Target | Target[];
+
+export interface CrawlOptions {
+  /** Shared/default task when a target doesn't carry its own. */
+  task?: string;
+  /**
+   * Model id. REQUIRED — there is no default. For Ollama use a local model id
+   * (e.g. `"qwen3-coder:30b"`); for an OpenAI-compatible API use that API's id
+   * (e.g. `"gpt-4o-mini"`).
+   */
+  model?: string;
+  /** `"ollama"` (local, default) or `"openai"` (any OpenAI-compatible API). */
+  provider?: Provider;
+  /**
+   * Optional embedding model id (#22), from the same provider as `model` (e.g.
+   * `"nomic-embed-text"` on Ollama, `"text-embedding-3-small"` on OpenAI). Makes
+   * task→link relevance SEMANTIC — multilingual and synonym-aware — feeding the
+   * best-first frontier ordering, the route budget, the opt-in `minRelevance`
+   * pruning and the reshape context retrieval. Embeddings only ORDER; they never
+   * drop links by themselves. Unset (default) = the lexical scorer; unreachable
+   * backend = one loud warning, then the lexical floor. Ignored with `noAi`
+   * (zero calls to ANY model, embeddings included).
+   */
+  embedModel?: string;
+  /**
+   * Crawl with ZERO model calls (no model needed): reveal runs on DOM heuristics,
+   * pages are kept whole, every in-scope link is followed. Zero tokens; the output
+   * is not task-filtered and big sites can take longer — contain with `maxPages` /
+   * `include` / `exclude`. #23 — the task speaks only to the AI, so here it has NO
+   * role: an explicit `task` (or `minRelevance`, which reads it) is refused loudly,
+   * and output files are named from the site. Incompatible with `mode: "targeted"`
+   * (task-filtering IS the model's job) — that combination is refused loudly.
+   * Default `false`.
+   */
+  noAi?: boolean;
+  /** What to extract (see {@link CrawlMode}). Default `"complete"`. */
+  mode?: CrawlMode;
+  /** API base URL for `provider: "openai"` (e.g. `https://api.openai.com/v1`). */
+  baseUrl?: string;
+  /** API key for `provider: "openai"`. Falls back to `CRAWLDNA_API_KEY` / `OPENAI_API_KEY`. */
+  apiKey?: string;
+  /** Override the Ollama server URL (default `http://127.0.0.1:11434`). */
+  ollamaHost?: string;
+  /** Whether to render pages in a real browser. Lazy-loads Playwright. Default `"auto"`. */
+  browser?: BrowserMode;
+  /** Parallel page renders. Default `4`. */
+  concurrency?: number;
+  /** Per-scan page cap; `0` = unlimited. Default `0`. */
+  maxPages?: number;
+  /** Per-page reveal action budget. Default `40`. */
+  maxActions?: number;
+  /** Only crawl URLs matching this pattern. */
+  include?: string | RegExp;
+  /** Skip URLs matching this pattern. */
+  exclude?: string | RegExp;
+  /**
+   * Politeness, opt-in (#14): minimum milliseconds between requests to the SAME
+   * host (concurrent workers queue behind each other per host, never across
+   * hosts). `0` (default) = off.
+   */
+  delay?: number;
+  /**
+   * Politeness, opt-in (#14): read each origin's robots.txt — disallowed URLs
+   * are SKIPPED with a `robots` warning (never silently) and `Crawl-delay` is
+   * honoured (the larger of it and `delay` wins). Default `false`: the tool is
+   * user-directed, like wget. Note the anti-bot CHALLENGE guard is separate and
+   * always on: a bot-defense interstitial is never kept as content (one retry
+   * with backoff, then a declared skip with an `anti-bot` warning).
+   */
+  respectRobots?: boolean;
+  /**
+   * Cap on the speculative JS-mined route candidates (paths dug out of script/JSON
+   * blobs, up to 800 per page) sent to the AI link gate, top-ranked by task
+   * relevance. `0` = unlimited. Default `200`. Conservative: only cuts when the
+   * relevance scores discriminate among the routes — a generic task cuts nothing —
+   * and real DOM links are never capped.
+   */
+  maxRoutes?: number;
+  /**
+   * Focused mode (opt-in): prune links whose task-relevance score (`0..1`) falls below
+   * this threshold BEFORE the AI link gate. `0` (default) = off — relevance then only
+   * orders links best-first and never drops any. Trades some recall for speed/scope,
+   * and only applies when the task actually discriminates among a page's links.
+   */
+  minRelevance?: number;
+  /**
+   * Persist the run to the cache. Library default `false` — the result is returned
+   * in memory only. The CLI and Web UI set this to `true`. Passing `cacheDir` also
+   * turns saving on.
+   */
+  save?: boolean;
+  /** Where to save when saving is on. Default `<cwd>/.crawldna/runs`. */
+  cacheDir?: string;
+  /**
+   * Also package one identifiable Markdown document per page (with metadata + a stable
+   * id), plus an `index.md` and a `documents.jsonl`, alongside the consolidated `.md`.
+   * Off by default. Pure repackaging — content stays verbatim. See {@link Scan.documents}.
+   */
+  perDocument?: boolean;
+  /**
+   * Collapse near-duplicate pages ACROSS different paths whose 64-bit SimHash is within
+   * this Hamming distance of ANY already-kept page. `0` (default) = off. **Opt-in**
+   * because content similarity alone cannot tell a duplicate from a sibling — measured
+   * on a real run, genuinely distinct templated API pages sat at distance ≤3 — so this
+   * aggressive tier can drop real content and stays a deliberate user choice.
+   */
+  nearDupHamming?: number;
+  /**
+   * Collapse mirror/variant re-servings of a kept page (default `8`, on). A page is
+   * dropped only when BOTH signals agree: its URL is a SIBLING of a kept page's (same
+   * path once a leading locale segment is stripped — mirror hosts like dev./v2., UI-state
+   * query variants like `?panel=settings`, locale twins `/en/x` vs `/x`) AND its content
+   * SimHash is within this Hamming distance. Sibling-shaped pages with real content
+   * differences (`?version=A` vs `?version=B`) measure far apart and are kept. Links found
+   * on a dropped duplicate are not followed, so mirror cascades stop at the first page.
+   * `0` = off.
+   */
+  mirrorHamming?: number;
+  /**
+   * Incremental re-crawl (opt-in). On a re-crawl of the same target, reuse pages whose
+   * sitemap `<lastmod>` — or an HTTP `304` on a static-safe page — is unchanged since the
+   * last `incremental` run, skipping render + reveal, and re-crawl only what changed.
+   * Conservative: any doubt re-crawls, so a change is never missed. Implies saving; the
+   * first run establishes the baseline. Default `false`.
+   */
+  incremental?: boolean;
+  /** Called for every event, in addition to the async iterator. */
+  onEvent?: (event: CrawlEvent) => void;
+}
+
+export type EventType =
+  | 'site'
+  | 'strategy'
+  | 'discover'
+  | 'page'
+  | 'action'
+  | 'extracted'
+  | 'dedup'
+  | 'incremental'
+  | 'resume'
+  | 'progress'
+  | 'warn'
+  | 'error'
+  | 'saved'
+  | 'done';
+
+/**
+ * A streamed progress event. `type` is one of {@link EventType}; the remaining
+ * fields depend on the type (e.g. `extracted` carries `url`/`title`/`bytes`,
+ * `progress` carries `done`/`total`, `warn` carries `reason`/`message`). Every
+ * event is also stamped with the active `scanId`/`scanIndex`.
+ */
+export interface CrawlEvent {
+  type: EventType;
+  scanId?: string;
+  scanIndex?: number;
+  [key: string]: unknown;
+}
+
+export interface PageMeta {
+  strategy: string;
+  framework?: string;
+  fetchedAt?: string;
+  bytes?: number;
+  /**
+   * The reveal exit audit (#21): characters of text still hidden in the page's main
+   * content when the reveal loop ended. `0` = the page was measurably drained.
+   * Present on browser-engine pages (`strategy: "agent"`); absent on static paths.
+   */
+  revealResidualChars?: number;
+}
+
+/** One captured reveal state, whole and verbatim — the faithful per-state record.
+ *  The consolidated `markdown` is compact (shared frame once); this keeps each
+ *  state's full co-occurrence recoverable (e.g. a tab/view swap, a partial change). */
+export interface RevealState {
+  /** The state's variant marker (e.g. a tab/view label), or null for the base state. */
+  label: string | null;
+  /** How the state was surfaced: `"baseline"`/`"tab:…"`/`"expander:…"`/`"dropdown:…"`/`"loadmore"`. */
+  provenance: string;
+  /** The revealing control's vertical position (#27), used to order states. */
+  order: number;
+  /** The whole state's Markdown, verbatim. */
+  markdown: string;
+}
+
+export interface Page {
+  url: string;
+  task: string;
+  title: string;
+  markdown: string;
+  /** The faithful per-state reveal record — present only when a page's reveal
+   *  captured MORE THAN ONE state (else the single state IS `markdown`). Also
+   *  written to disk under `states/<page>.md` when the run is saved. */
+  states?: RevealState[];
+  meta: PageMeta;
+}
+
+/** One consolidated, verbatim Markdown file produced for a scan. */
+export interface OutputFile {
+  filename: string;
+  title: string;
+  /** The file's full Markdown — available in memory even when nothing is saved. */
+  markdown: string;
+  bytes: number;
+  /** Source page URLs that contributed to this file. */
+  pages: string[];
+}
+
+/** AI token usage counters. `byKind` splits the same totals by call type
+ *  (`"reveal"`, `"scope"`, `"links"`, `"nav-plan"`, `"embed"`, `"health"`, …) so cost
+ *  can be attributed to the judgment that spent it. `cachedInputTokens` is the slice of
+ *  `inputTokens` a remote provider served from its prompt cache (~10× cheaper);
+ *  `0` for providers that don't report it (e.g. local Ollama). */
+export interface TokenUsage {
+  calls: number;
+  inputTokens: number;
+  outputTokens: number;
+  cachedInputTokens: number;
+  byKind: Record<
+    string,
+    { calls: number; inputTokens: number; outputTokens: number; cachedInputTokens: number }
+  >;
+}
+
+export interface Stats {
+  pages: number;
+  durationMs: number;
+  strategyCounts: Record<string, number>;
+  /** AI token usage for the run/scan; input and output are billed differently. */
+  tokens: TokenUsage;
+  /** Pages dropped as duplicates, by tier: `exact` (sha1), `mirror` (URL-sibling +
+   *  SimHash, default on), `near` (cross-path SimHash, opt-in). */
+  deduped?: { exact: number; mirror: number; near: number };
+  /** Reveal exit audit (#21): how many kept pages ended with text still hidden, and the
+   *  total hidden characters. `{ pages: 0, chars: 0 }` = every page measurably drained. */
+  revealResidual?: { pages: number; chars: number };
+}
+
+export interface Warning {
+  url?: string;
+  reason?: string;
+  message: string;
+}
+
+/** One page packaged as an identifiable document (opt-in `perDocument`). The body is
+ *  the page's verbatim Markdown; `id` is stable (derived from the URL). */
+export interface Document {
+  id: string;
+  url: string;
+  title: string;
+  fetchedAt: string;
+  bytes: number;
+  /** The verbatim page Markdown (no header). */
+  markdown: string;
+  /** An H1–H3 outline of the page, for section paths / chunking. */
+  headings: Array<{ level: number; text: string }>;
+  /** The per-document filename written under `documents/` when the run is saved. */
+  file: string;
+}
+
+/** One submitted link's crawl: its own pages, output files, dedup and stats. */
+export interface Scan {
+  scanId: string;
+  index: number;
+  url: string;
+  task: string;
+  title: string;
+  pages: Page[];
+  files: OutputFile[];
+  /** Per-page documents, populated only when `perDocument` is enabled; else empty. */
+  documents: Document[];
+  stats: Stats;
+  warnings: Warning[];
+}
+
+/** Present on the result only when the run was saved (otherwise `run` is `null`). */
+export interface SavedRun {
+  id: string;
+  dir: string;
+  scans: unknown[];
+}
+
+export interface Result {
+  scans: Scan[];
+  stats: Stats;
+  warnings: Warning[];
+  /** Where the run was saved, or `null` when saving was off (the library default). */
+  run: SavedRun | null;
+}
+
+/**
+ * A running crawl. Async-iterate it for live {@link CrawlEvent}s, await
+ * {@link Run.result} for the final {@link Result}, or call {@link Run.stop} to
+ * end early (the result still resolves with whatever was gathered).
+ */
+export interface Run extends AsyncIterable<CrawlEvent> {
+  result: Promise<Result>;
+  stop(): void;
+}
+
+/** The default option values; spread under any options you pass to {@link crawlDocs}. */
+export declare const DEFAULT_OPTIONS: Required<
+  Pick<
+    CrawlOptions,
+    | 'task'
+    | 'model'
+    | 'provider'
+    | 'noAi'
+    | 'mode'
+    | 'browser'
+    | 'delay'
+    | 'respectRobots'
+    | 'concurrency'
+    | 'maxPages'
+    | 'maxActions'
+    | 'maxRoutes'
+    | 'minRelevance'
+    | 'nearDupHamming'
+    | 'mirrorHamming'
+    | 'save'
+    | 'perDocument'
+  >
+> &
+  CrawlOptions;
+
+/** Normalise the various accepted `targets` shapes into `{ url, task }[]`. */
+export declare function normalizeTargets(targets: Targets, defaultTask?: string): Target[];
+
+/**
+ * Crawl one or more links, each with a natural-language task, and return a
+ * {@link Run}. The crawl renders pages in a real browser, reveals interaction-
+ * hidden content, and extracts task-relevant Markdown verbatim. Nothing is written
+ * to disk unless `save: true` or a `cacheDir` is given. When saving is on, every
+ * kept page is also journaled to disk as it is captured, so an interrupted run
+ * can be completed later with {@link resumeCrawl}.
+ */
+export declare function crawlDocs(targets: Targets, options?: CrawlOptions): Run;
+
+/**
+ * Resume an interrupted saved run: pages already extracted are restored verbatim
+ * from the run's incremental journal (never re-rendered), their recorded links
+ * re-seed the frontier, and the crawl completes into the same run folder.
+ * `overrides` are merged over the run's saved options. An API key is never
+ * persisted — for `provider: "openai"` pass `apiKey` again (or set the env var).
+ * Rejects if the run is already complete.
+ */
+export declare function resumeCrawl(runId: string, overrides?: CrawlOptions): Promise<Run>;
+
+export default crawlDocs;

package/package.json

@@ -0,0 +1,64 @@
+{
+  "name": "crawldna",
+  "version": "0.1.0",
+  "description": "General, task-driven web crawler that reveals interaction-hidden content and outputs clean Markdown.",
+  "type": "module",
+  "license": "AGPL-3.0-only",
+  "author": "BogdanVasaiu",
+  "homepage": "https://crawldna.com",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/BogdanVasaiu/crawlDNA.git"
+  },
+  "bugs": {
+    "url": "https://github.com/BogdanVasaiu/crawlDNA/issues"
+  },
+  "keywords": [
+    "crawler",
+    "scraper",
+    "markdown",
+    "ai",
+    "llm",
+    "playwright",
+    "documentation",
+    "extraction"
+  ],
+  "main": "./src/index.mjs",
+  "types": "./index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./index.d.ts",
+      "default": "./src/index.mjs"
+    }
+  },
+  "bin": {
+    "crawldna": "bin/cli.mjs"
+  },
+  "files": [
+    "src",
+    "bin",
+    "index.d.ts",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "start": "node bin/cli.mjs",
+    "serve": "node bin/cli.mjs serve",
+    "test": "node --test \"test/*.test.mjs\"",
+    "eval": "node scripts/eval.mjs",
+    "prepublishOnly": "npm test"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "dependencies": {
+    "fast-xml-parser": "^4.5.0",
+    "node-html-parser": "^7.0.1",
+    "ollama": "^0.5.11",
+    "turndown": "^7.2.0",
+    "turndown-plugin-gfm": "^1.0.2"
+  },
+  "optionalDependencies": {
+    "playwright": "^1.49.0"
+  }
+}

package/src/engine/actions.mjs

@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// Low-level page actuation via Playwright. Elements are targeted by the
+// `data-crawldna-id` attribute that perceive() stamped on the live DOM.
+// The "wait until the click's effects landed" signal lives in lib/settle.mjs
+// (#15) — shared with the initial render and the browser-escalation fetch.
+
+import { settle } from '../lib/settle.mjs';
+
+const SETTLE_MS = 350;
+
+/**
+ * Activate a revealer by id. Guards against navigation: if the click took us to
+ * a new URL (the control was really a link), we record that URL and go back so
+ * revealing continues on the original page.
+ *
+ * @returns {Promise<{ ok: boolean, navigatedTo?: string, note?: string }>}
+ */
+export async function clickRevealer(page, id) {
+  const before = page.url();
+  const loc = page.locator(`[data-crawldna-id="${id}"]`).first();
+  try {
+    await loc.scrollIntoViewIfNeeded({ timeout: 2500 }).catch(() => {});
+    await loc.click({ timeout: 4000 });
+  } catch (err) {
+    return { ok: false, note: String(err && err.message).slice(0, 120) };
+  }
+  // Wait for the click's async content (AJAX panels + any first-touch lazy-load
+  // cascade) to finish before the caller captures. This is what reliably gets a
+  // slow first reveal — e.g. the day-2 slot grid — that a single immediate
+  // snapshot, or networkidle alone, can miss.
+  await settle(page);
+
+  const after = page.url();
+  if (after !== before) {
+    await page.goBack({ waitUntil: 'domcontentloaded', timeout: 15000 }).catch(() => {});
+    await page.waitForTimeout(SETTLE_MS);
+    return { ok: true, navigatedTo: after };
+  }
+  return { ok: true };
+}
+
+/**
+ * Scroll down one viewport to trigger lazy/infinite content.
+ * @returns {Promise<boolean>} whether the document grew (more content loaded).
+ */
+export async function scrollStep(page) {
+  try {
+    const before = await page.evaluate(() => document.body.scrollHeight);
+    await page.evaluate(() => window.scrollBy(0, Math.round(window.innerHeight * 0.9)));
+    await page.waitForTimeout(500);
+    const after = await page.evaluate(() => document.body.scrollHeight);
+    return after > before + 4;
+  } catch {
+    return false;
+  }
+}

package/src/engine/consent.mjs

@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// #21a — consent/overlay dismissal: the DECISION, kept pure and testable.
+//
+// perceive() MEASURES (a visible button-like element inside a true overlay —
+// fixed/sticky/dialog/aria-modal — with its label, geometry and the overlay's
+// text); THIS module DECIDES which of those buttons to click. The split is the
+// whole point: the in-page code stays a thin sensor, the policy is unit-tested
+// offline with multilingual fixtures.
+//
+// Universality argument (why a lexicon is allowed here at all): we never read
+// the SITE — we read the ARTEFACT OF THE DEFENSE, the cookie/consent banner,
+// which is the same everywhere because the law that mandates it is. The words
+// on its buttons come from a tiny, closed vocabulary per language ("accept
+// all" / "reject all" / "close"), so a micro-lexicon of ~40 stems covers the
+// world's banners the way `documenta` covers the word "documentation" — it is
+// the deterministic-backstop rule (#4), not a per-site rule.
+//
+// Policy, in order:
+//   1. Overlay ABOUT consent (its text mentions cookie/consent/GDPR/privacy):
+//      prefer a REJECT button (closes the banner just the same, and is the more
+//      respectful default), then ACCEPT, then a plain dismiss (ok/close/…),
+//      then — banners with exotic wording — the PRIMARY button by geometry
+//      (largest visible button with a short label).
+//   2. Any OTHER overlay (newsletter modal, app interstitial): only a plain
+//      dismiss/close is ever clicked. Never accept/reject words out of context,
+//      and NEVER geometry — the biggest button of an arbitrary dialog is
+//      "Subscribe"/"Log in", exactly what must not be pressed.
+
+/** Does the overlay's own text say it is a consent/cookie banner? These four
+ * stems are quasi-universal: "cookie" is used verbatim in virtually every
+ * language, "consent/GDPR/privacy" cover the legal phrasing. */
+export const CONSENT_TOPIC_RE = /cookie|consent|gdpr|privacy|rgpd|dsgvo/i;
+
+// The micro-lexicon. Stems, not full words, so inflections match; each entry
+// reads the BANNER's button, never the site's content.
+const REJECT_RE = new RegExp(
+  [
+    // en / it / de / fr / es / pt / nl / scandinavian / fi / pl / cs / hu / ro / ru / zh / ja / ko
+    'reject', 'decline', 'refuse', 'deny', 'necessary only', 'only necessary',
+    'rifiut', 'solo necessari',
+    'ablehn', 'nur notwendige',
+    'refuser', 'tout refuser',
+    'rechaz', 'solo necesarias',
+    'recus', 'rejeit',
+    'weiger', 'alleen noodzakelijk',
+    'avvisa', 'avslå', 'afvis', 'kun nødvendige',
+    'hylkää', 'odrzuć', 'odmítn', 'elutasít', 'resping',
+    'отклон', 'отказ',
+    '拒绝', '全部拒绝', '拒否', '거부',
+  ].join('|'),
+  'i',
+);
+const ACCEPT_RE = new RegExp(
+  [
+    'accept', 'agree', 'allow', 'got it', 'i understand', 'understood',
+    'accett', 'consent[io]', 'ho capito',
+    'akzeptier', 'zustimm', 'einverstanden', 'verstanden',
+    'accepter', "j'accepte", 'tout accepter',
+    'acept', 'aceit', 'de acuerdo',
+    'accepteer', 'akkoord',
+    'acceptera', 'godkänn', 'godta', 'accepter alle', 'hyväksy',
+    'akceptuj', 'zgadzam', 'přijm', 'souhlas', 'elfogad', 'accept toate',
+    'принять', 'принима', 'соглас',
+    '同意', '接受', '同意する', '승인', '동의',
+  ].join('|'),
+  'i',
+);
+const DISMISS_RE = new RegExp(
+  [
+    '^ok$', '^okay$', 'close', 'dismiss', 'continue', 'no,? thanks', 'not now', 'maybe later',
+    'chiudi', 'schließen', 'weiter', 'fermer', 'continuer', 'cerrar',
+    'continuar', 'fechar', 'sluiten', 'luk', 'stäng', 'lukk', 'sulje',
+    'zamknij', 'zavří', 'bezár', 'închide', 'закрыть', 'продолжить',
+    '关闭', '閉じる', '닫기', '^[×✕x]$',
+  ].join('|'),
+  'i',
+);
+
+/**
+ * Pick which overlay buttons to dismiss, one per overlay.
+ *
+ * @param {Array<{ id:number, label:string, area:number, overlayText:string, href?:string }>} candidates
+ *   measured by perceive(): every visible short-labelled button inside a true overlay.
+ * @returns {Array<{ id:number, label:string }>} the buttons to click, in order.
+ */
+export function pickConsent(candidates = []) {
+  // Group by overlay (candidates carry their overlay's text sample).
+  const overlays = new Map();
+  for (const c of candidates) {
+    if (!c || !c.label) continue;
+    const key = c.overlayText || '';
+    if (!overlays.has(key)) overlays.set(key, []);
+    overlays.get(key).push(c);
+  }
+
+  const picks = [];
+  for (const [overlayText, group] of overlays) {
+    const isConsent = CONSENT_TOPIC_RE.test(overlayText);
+    // A policy/settings LINK ("Cookie policy", "Learn more") is never the way to
+    // close a banner — buttons only. (An <a> used AS the button has no href.)
+    const buttons = group.filter((c) => !c.href);
+    const byLexicon = (re) => buttons.find((c) => re.test(c.label));
+
+    let pick = null;
+    if (isConsent) {
+      pick =
+        byLexicon(REJECT_RE) ||
+        byLexicon(ACCEPT_RE) ||
+        byLexicon(DISMISS_RE) ||
+        // Exotic wording: the PRIMARY button by geometry — banners paint their
+        // one-click exit as the biggest button. Short label only (a long label
+        // is body text, not a button caption).
+        [...buttons].filter((c) => c.label.length < 40).sort((a, b) => (b.area || 0) - (a.area || 0))[0] ||
+        null;
+    } else {
+      // Not (visibly) a consent banner: only ever a plain dismiss. No accept/
+      // reject out of context, no geometry — never press an arbitrary modal's
+      // primary action.
+      pick = byLexicon(DISMISS_RE) || null;
+    }
+    if (pick) picks.push({ id: pick.id, label: pick.label });
+  }
+  return picks;
+}