@d-zero/page-cluster 0.2.0 → 0.3.0

package/dist/pass0-blocking.js

@@ -0,0 +1,87 @@
+import { filterFirstPartyStylesheetHrefs } from './filter-first-party-stylesheet-hrefs.js';
+import { reassignOrphanBlockKeys } from './reassign-orphan-block-keys.js';
+import { resolveBlockingGroupKeys } from './resolve-blocking-group-keys.js';
+/**
+ * Splits `resolvePageClusterKeys` into a size-flat first pass so the driver
+ * can decide per-block memory strategy before loading any page HTML. Runs the
+ * three corpus-wide, HTML-free stages of blocking in the same order the
+ * in-memory driver already uses — first-party stylesheet filtering, blocking-
+ * key derivation, orphan reassignment — and returns one final block key per
+ * input page in input order.
+ *
+ * ## Why extract this from resolvePageClusterKeys?
+ *
+ * The in-memory driver holds every page's `html`, `remainderHtml`,
+ * `landmarks[]`, and pre-Stage-A prepared HTML at once. At 176k pages × ~57
+ * KB average, that alone breaks a 17 GB RAM machine well before Stage A
+ * starts (measured: OS SIGKILL at ~15,000 pages, before the resolve phase
+ * even began). All three blocking stages, in contrast, depend only on
+ * `paths` / `stylesheetHrefs` / `host` — a few hundred bytes per page. Running
+ * them first, HTML-free, lets the downstream per-block clustering hold HTML
+ * for only one block's pages at a time.
+ *
+ * ## Preserves in-memory driver semantics exactly
+ *
+ * The output of this function is byte-identical to the block-key portion of
+ * the current `resolvePageClusterKeys` for the same input, because it reuses
+ * the same three underlying functions in the same order with the same
+ * defaults. That guarantee is load-bearing: the size-threshold-gated Pass 1
+ * that follows this function runs the current in-memory implementation
+ * unchanged for small blocks, and any drift in block-key computation between
+ * Pass 0 and Pass 1 would silently misroute pages between the two paths.
+ * @param pages
+ * @param options
+ * @example
+ * ```ts
+ * const blockKeys = resolveBlockKeys([
+ *   { paths: ['news', '1'], stylesheetHrefs: ['/a.css'], host: 'example.com' },
+ *   { paths: ['news', '2'], stylesheetHrefs: ['/a.css'], host: 'example.com' },
+ *   { paths: ['about'],     stylesheetHrefs: [],        host: 'example.com' },
+ * ]);
+ * // ['css:<hash>', 'css:<hash>', 'path:about']
+ * ```
+ */
+export function resolveBlockKeys(pages, options) {
+    const restrictStylesheetsToFirstParty = options?.restrictStylesheetsToFirstParty ?? true;
+    const blockingPages = restrictStylesheetsToFirstParty
+        ? filterFirstPartyStylesheetHrefs(pages)
+        : pages;
+    const rawBlockKeys = resolveBlockingGroupKeys(blockingPages, options);
+    const reassignOrphans = options?.reassignOrphans ?? true;
+    if (!reassignOrphans)
+        return rawBlockKeys;
+    // Orphan reassignment always uses a numeric `pathDepth`. When the caller
+    // asked for `'auto'`, fall back to the historical default 1 here — a
+    // future PR that wires the auto-cut depth through can compute it once
+    // and pass it as a number to both `resolveBlockingGroupKeys` and this
+    // call to keep them consistent.
+    const numericPathDepth = typeof options?.pathDepth === 'number' ? options.pathDepth : undefined;
+    return reassignOrphanBlockKeys(blockingPages, rawBlockKeys, numericPathDepth);
+}
+/**
+ * Groups pages by their block key while preserving each block's members in
+ * input order. Returned as a `Map` so the caller can iterate blocks in
+ * insertion order (first-seen block first) — matching the order
+ * `resolvePageClusterKeys`'s own per-block loop already uses so cluster IDs
+ * assigned per block stay deterministic across in-memory and streaming
+ * paths.
+ * @param blockKeys
+ * @example
+ * ```ts
+ * const indices = groupIndicesByBlockKey(['a', 'b', 'a', 'c', 'a']);
+ * // Map { 'a' => [0, 2, 4], 'b' => [1], 'c' => [3] }
+ * ```
+ */
+export function groupIndicesByBlockKey(blockKeys) {
+    const groups = new Map();
+    for (const [index, blockKey] of blockKeys.entries()) {
+        const list = groups.get(blockKey);
+        if (list) {
+            list.push(index);
+        }
+        else {
+            groups.set(blockKey, [index]);
+        }
+    }
+    return groups;
+}

package/dist/per-page-landmark-signatures.d.ts

@@ -0,0 +1,48 @@
+import type { ExtractLandmarksResult, LandmarkType } from './extract-landmarks.js';
+import type { TokenizeOptions } from './types.js';
+/**
+ * Every landmark type extractLandmarks may populate, iterated in a fixed
+ * order so downstream signature vectors are byte-stable and every consumer
+ * agrees on the same enumeration.
+ */
+export declare const ALL_LANDMARK_TYPES: readonly LandmarkType[];
+/**
+ * One landmark instance's tokenized identity: the token set produced by
+ * tokenizing the instance's raw HTML wrapped in a `<body>` shell, plus that
+ * token set's canonical signature string (via
+ * {@link ./canonicalize-token-set.js | canonicalizeTokenSet}). Signatures
+ * are reused across consumers so two callers see the same "same instance"
+ * verdict without independently re-canonicalizing.
+ */
+export type PerPageLandmarkInstance = {
+    readonly type: LandmarkType;
+    readonly tokens: ReadonlySet<string>;
+    readonly signature: string;
+};
+/**
+ * Tokenizes every landmark instance across every page once, keyed by page
+ * index. Shared by:
+ * - {@link ./merge-cross-block-clusters.js | shellQuorum} — for cross-block
+ *   shell corroboration
+ * - {@link ./resolve-page-cluster-keys.js | computeLocalLandmarkPseudoTokens}
+ *   — for injecting local-chrome pseudo-tokens into Stage A block token
+ *   sets
+ * - {@link ./resolve-landmark-variant-keys.js | resolveLandmarkVariantKeys}
+ *   — for picking each page's canonical instance for variant clustering
+ *
+ * Every consumer previously reimplemented this loop, which was both a
+ * duplication risk (a tokenization fix landing in one but not the others)
+ * and a real performance cost — every landmark region on the page was
+ * tokenized 2 or 3 times per pipeline invocation. This single pass replaces
+ * all of that.
+ *
+ * Within each page, instances that tokenize to the same signature are
+ * deduped (kept once): a CMS glitch that duplicates the site footer, or a
+ * `<header role="navigation">` matching both `header` and `nav` at
+ * identical spans, must not count twice against the corpus histogram.
+ * Across pages there is no dedupe — each page contributes its own instance
+ * list.
+ * @param landmarks
+ * @param tokenizeOptions
+ */
+export declare function computePerPageLandmarkInstances(landmarks: readonly ExtractLandmarksResult[], tokenizeOptions?: TokenizeOptions): readonly (readonly PerPageLandmarkInstance[])[];

package/dist/per-page-landmark-signatures.js

@@ -0,0 +1,62 @@
+import { canonicalizeTokenSet } from './canonicalize-token-set.js';
+import { tokenize } from './tokenize.js';
+/**
+ * Every landmark type extractLandmarks may populate, iterated in a fixed
+ * order so downstream signature vectors are byte-stable and every consumer
+ * agrees on the same enumeration.
+ */
+export const ALL_LANDMARK_TYPES = [
+    'header',
+    'footer',
+    'nav',
+    'aside',
+    'form',
+    'search',
+];
+/**
+ * Tokenizes every landmark instance across every page once, keyed by page
+ * index. Shared by:
+ * - {@link ./merge-cross-block-clusters.js | shellQuorum} — for cross-block
+ *   shell corroboration
+ * - {@link ./resolve-page-cluster-keys.js | computeLocalLandmarkPseudoTokens}
+ *   — for injecting local-chrome pseudo-tokens into Stage A block token
+ *   sets
+ * - {@link ./resolve-landmark-variant-keys.js | resolveLandmarkVariantKeys}
+ *   — for picking each page's canonical instance for variant clustering
+ *
+ * Every consumer previously reimplemented this loop, which was both a
+ * duplication risk (a tokenization fix landing in one but not the others)
+ * and a real performance cost — every landmark region on the page was
+ * tokenized 2 or 3 times per pipeline invocation. This single pass replaces
+ * all of that.
+ *
+ * Within each page, instances that tokenize to the same signature are
+ * deduped (kept once): a CMS glitch that duplicates the site footer, or a
+ * `<header role="navigation">` matching both `header` and `nav` at
+ * identical spans, must not count twice against the corpus histogram.
+ * Across pages there is no dedupe — each page contributes its own instance
+ * list.
+ * @param landmarks
+ * @param tokenizeOptions
+ */
+export function computePerPageLandmarkInstances(landmarks, tokenizeOptions) {
+    return landmarks.map((entry) => {
+        const seenSignatures = new Set();
+        const out = [];
+        for (const type of ALL_LANDMARK_TYPES) {
+            for (const instanceHtml of entry[type]) {
+                if (!instanceHtml)
+                    continue;
+                const tokens = new Set(tokenize(`<body>${instanceHtml}</body>`, tokenizeOptions).tokens);
+                if (tokens.size === 0)
+                    continue;
+                const signature = canonicalizeTokenSet(tokens);
+                if (seenSignatures.has(signature))
+                    continue;
+                seenSignatures.add(signature);
+                out.push({ type, tokens, signature });
+            }
+        }
+        return out;
+    });
+}

package/dist/reservoir-sample.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Runs Algorithm R (Vitter, 1985) — the standard one-pass reservoir sampling
+ * algorithm — to pick `sampleSize` items from `items`, deterministically for
+ * a given `seed`. Returns them in input order.
+ *
+ * ## Why deterministic
+ *
+ * `resolvePageClusterKeys`'s cluster keys must be reproducible: given the
+ * same input pages in the same order, subsequent runs must produce the same
+ * cluster keys, or downstream code that stores/compares them by value
+ * silently drifts across runs. `Math.random()` violates that outright.
+ * Reservoir sampling on top of a seedable PRNG (see {@link ./reservoir-sample.js | mulberry32})
+ * preserves determinism while retaining reservoir sampling's O(1)-space,
+ * one-pass memory profile — the whole point of using it in the first place
+ * (a block too large for full in-memory processing must not require
+ * per-page auxiliary state for sampling either).
+ *
+ * ## Seed handling
+ *
+ * A `number` seed is used as-is; a `string` seed is hashed via
+ * {@link ./reservoir-sample.js | fnv1a32} first so the caller can pass a
+ * stable identifier (e.g. a block key like `"orphan-merge:news"`) without
+ * having to compute a numeric hash itself. Different blocks get different
+ * samples by passing each block's key as the seed.
+ *
+ * ## Edge cases
+ *
+ * - `sampleSize <= 0` — returns `[]`.
+ * - `sampleSize >= items.length` — returns all of `items` in input order,
+ *   without invoking the PRNG. This matters for regression: a block small
+ *   enough to fit its whole membership in the sample must not have items
+ *   reordered by any sampling logic, so downstream cluster labels stay in
+ *   the same first-seen order as the in-memory path.
+ * @param items
+ * @param sampleSize
+ * @param seed
+ * @example
+ * ```ts
+ * const sample = reservoirSample([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 3, 'block-a');
+ * // deterministic 3-item subset, always the same for seed 'block-a'
+ * ```
+ */
+export declare function reservoirSample<T>(items: readonly T[], sampleSize: number, seed?: number | string): T[];

package/dist/reservoir-sample.js

@@ -0,0 +1,98 @@
+/**
+ * Mulberry32 — a tiny, well-known 32-bit PRNG. Chosen because it fits in a
+ * closure, seeds from a single 32-bit integer (so the caller can derive it
+ * deterministically from something stable like a block key), and produces a
+ * uniform enough sequence for reservoir sampling. Not cryptographic; that is
+ * not what this file needs.
+ * @param seed
+ */
+function mulberry32(seed) {
+    let state = seed >>> 0;
+    return () => {
+        state = (state + 0x6d_2b_79_f5) >>> 0;
+        let t = state;
+        t = Math.imul(t ^ (t >>> 15), t | 1);
+        t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+        return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296;
+    };
+}
+/**
+ * FNV-1a 32-bit hash of a string, used as the default seed source for
+ * {@link ./reservoir-sample.js | reservoirSample} when the caller passes a
+ * `string` seed. Small, dependency-free, and deterministic across runs and
+ * machines — the whole point of using it as a seed is that a block key like
+ * `"orphan-merge:news"` always samples the same subset of members.
+ * @param input
+ */
+function fnv1a32(input) {
+    let hash = 0x81_1c_9d_c5;
+    for (let i = 0; i < input.length; i++) {
+        hash ^= input.codePointAt(i) ?? 0;
+        hash = Math.imul(hash, 0x01_00_01_93);
+    }
+    return hash >>> 0;
+}
+/**
+ * Runs Algorithm R (Vitter, 1985) — the standard one-pass reservoir sampling
+ * algorithm — to pick `sampleSize` items from `items`, deterministically for
+ * a given `seed`. Returns them in input order.
+ *
+ * ## Why deterministic
+ *
+ * `resolvePageClusterKeys`'s cluster keys must be reproducible: given the
+ * same input pages in the same order, subsequent runs must produce the same
+ * cluster keys, or downstream code that stores/compares them by value
+ * silently drifts across runs. `Math.random()` violates that outright.
+ * Reservoir sampling on top of a seedable PRNG (see {@link ./reservoir-sample.js | mulberry32})
+ * preserves determinism while retaining reservoir sampling's O(1)-space,
+ * one-pass memory profile — the whole point of using it in the first place
+ * (a block too large for full in-memory processing must not require
+ * per-page auxiliary state for sampling either).
+ *
+ * ## Seed handling
+ *
+ * A `number` seed is used as-is; a `string` seed is hashed via
+ * {@link ./reservoir-sample.js | fnv1a32} first so the caller can pass a
+ * stable identifier (e.g. a block key like `"orphan-merge:news"`) without
+ * having to compute a numeric hash itself. Different blocks get different
+ * samples by passing each block's key as the seed.
+ *
+ * ## Edge cases
+ *
+ * - `sampleSize <= 0` — returns `[]`.
+ * - `sampleSize >= items.length` — returns all of `items` in input order,
+ *   without invoking the PRNG. This matters for regression: a block small
+ *   enough to fit its whole membership in the sample must not have items
+ *   reordered by any sampling logic, so downstream cluster labels stay in
+ *   the same first-seen order as the in-memory path.
+ * @param items
+ * @param sampleSize
+ * @param seed
+ * @example
+ * ```ts
+ * const sample = reservoirSample([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 3, 'block-a');
+ * // deterministic 3-item subset, always the same for seed 'block-a'
+ * ```
+ */
+export function reservoirSample(items, sampleSize, seed = 0) {
+    if (!(Number.isInteger(sampleSize) && sampleSize >= 0)) {
+        throw new RangeError(`reservoirSample: sampleSize must be a non-negative integer, got ${sampleSize}`);
+    }
+    if (sampleSize === 0 || items.length === 0)
+        return [];
+    if (sampleSize >= items.length)
+        return [...items];
+    const numericSeed = typeof seed === 'string' ? fnv1a32(seed) : seed >>> 0;
+    const rand = mulberry32(numericSeed);
+    // Reservoir holds the picked positions (indices into `items`); we return
+    // items sorted by those positions to preserve input order in the result.
+    const reservoirIndices = Array.from({ length: sampleSize }, (_, i) => i);
+    for (let i = sampleSize; i < items.length; i++) {
+        const j = Math.floor(rand() * (i + 1));
+        if (j < sampleSize) {
+            reservoirIndices[j] = i;
+        }
+    }
+    reservoirIndices.sort((a, b) => a - b);
+    return reservoirIndices.map((position) => items[position]);
+}

package/dist/resolve-blocking-group-keys.d.ts

@@ -12,8 +12,14 @@ export type PageBlockingSignals = {
  * @see resolveBlockingGroupKeys
  */
 export type ResolveBlockingGroupKeysOptions = {
-    /** Forwarded to `derivePathGroupKey` as-is. */
-    pathDepth?: number;
+    /**
+     * Forwarded to `derivePathGroupKey` when a number, which is the historical
+     * default. Set to `'auto'` to instead run
+     * {@link ./derive-path-cluster-keys.js | derivePathClusterKeys} on the
+     * corpus and let it pick the depth data-driven — see that function's
+     * JSDoc for the algorithm and its opt-in staging rationale.
+     */
+    pathDepth?: number | 'auto';
     /**
      * Minimum number of pages that must share a stylesheet-derived key before
      * it's trusted as real evidence, rather than a coincidence. Must be at

package/dist/resolve-blocking-group-keys.js

@@ -1,4 +1,5 @@
 import { computeDocumentFrequency } from './compute-document-frequency.js';
+import { derivePathClusterKeys } from './derive-path-cluster-keys.js';
 import { derivePathGroupKey } from './derive-path-group-key.js';
 import { deriveStylesheetGroupKey } from './derive-stylesheet-group-key.js';
 import { splitTokensByFrequency } from './split-tokens-by-frequency.js';
@@ -87,7 +88,7 @@ const DEFAULT_MIN_CSS_GROUP_SIZE = 2;
  * ```
  */
 export function resolveBlockingGroupKeys(pages, options) {
-    const pathDepth = options?.pathDepth;
+    const pathDepthOption = options?.pathDepth;
     const minCssGroupSize = options?.minCssGroupSize ?? DEFAULT_MIN_CSS_GROUP_SIZE;
     const hrefCommonThreshold = options?.hrefCommonThreshold;
     if (!(Number.isInteger(minCssGroupSize) && minCssGroupSize >= 2)) {
@@ -95,9 +96,19 @@ export function resolveBlockingGroupKeys(pages, options) {
     }
     // Eagerly delegate pathDepth/hrefCommonThreshold validation to the
     // functions that own it, instead of only discovering an invalid option
-    // once some page's data happens to reach that branch below.
-    derivePathGroupKey([], pathDepth);
+    // once some page's data happens to reach that branch below. `'auto'`
+    // skips validation here because it doesn't reach derivePathGroupKey's
+    // number-only signature until per-page fallback below.
+    if (pathDepthOption !== 'auto') {
+        derivePathGroupKey([], pathDepthOption);
+    }
     splitTokensByFrequency(new Set(), { documentFrequency: new Map(), pageCount: 0 }, hrefCommonThreshold);
+    // Resolve `pathDepth: 'auto'` to a data-driven per-page key list once,
+    // before the per-page loop below, so the auto-cut sweep is amortized
+    // over the whole call rather than repeated per page.
+    const perPagePathKeys = pathDepthOption === 'auto'
+        ? derivePathClusterKeys(pages.map((page) => page.paths)).keys
+        : null;
     const hrefSets = pages.map((page) => new Set(page.stylesheetHrefs));
     // Pages with no stylesheets at all must not count toward the denominator:
     // see the JSDoc note above on document-frequency dilution.
@@ -115,6 +126,9 @@ export function resolveBlockingGroupKeys(pages, options) {
         if (cssKey !== undefined && (cssKeyCounts.get(cssKey) ?? 0) >= minCssGroupSize) {
             return `css:${cssKey}`;
         }
-        return `path:${derivePathGroupKey(page.paths, pathDepth)}`;
+        const pathKey = perPagePathKeys === null
+            ? derivePathGroupKey(page.paths, pathDepthOption)
+            : (perPagePathKeys[index] ?? '');
+        return `path:${pathKey}`;
     });
 }

package/dist/resolve-landmark-variant-keys.d.ts

@@ -15,27 +15,42 @@ export type ResolveLandmarkVariantKeysOptions = TokenizeOptions & ResolveStructu
  * `resolvePageClusterKeys` and combine the results themselves; this function
  * does not know about, or merge with, the other one's output.
  *
- * Each call re-runs {@link ./extract-landmarks.js | extractLandmarks} over
- * the entire `htmlList`, keeping only the one field matching `landmarkType`
- * and discarding the other three it also computed. Calling this once per
- * `landmarkType` (as the paragraph above suggests, for a caller that wants
- * more than one) therefore re-parses every page once per type requested. A
- * caller for whom that cost is material should call `extractLandmarks`
- * itself once per page, read all four fields off the single result, and feed
- * each field's token sets to
- * {@link ./resolve-structural-cluster-keys.js | resolveStructuralClusterKeys}
- * directly (with the same empty-set sentinel for a missing field) instead of
- * calling this function multiple times.
+ * ## Per-page canonical instance selection
+ *
+ * A page can now have any number of landmark instances of the requested
+ * type (see {@link ./extract-landmarks.js | extractLandmarks} for why the
+ * old shallowest-wins rule was removed). For "which variant does this page
+ * have", the canonical instance is the one whose token-set signature is
+ * most common across the corpus, ties broken by document order. This picks
+ * the site-wide chrome instance automatically — the shared site header
+ * dominates the corpus histogram — while article-specific `<header>`s that
+ * vary per page carry frequency 1 and are never selected. Choosing this way
+ * is the data-driven analogue of the old shallowest-wins rule, without
+ * fragmenting variant keys into singletons (a real regression risk of a
+ * naive "union every instance's tokens" approach: 10 pages that each carry
+ * both a site header and a per-article header would every one produce a
+ * distinct union token set, collapsing every page into its own variant key).
+ *
+ * A page with no matching landmark compares as an empty token set.
+ * `jaccardSimilarity`'s documented treatment of two empty sets as
+ * similarity `1` (see its JSDoc) means every landmark-less page lands in
+ * the same "has no such landmark" group with no extra branching needed
+ * here, and unambiguously in a different group from every page that does
+ * have one (`jaccardSimilarity(∅, nonEmpty)` is always `0`). A landmark
+ * that exists but tokenizes to an empty set never collides with this
+ * sentinel because `tokenize` skips it (see `computePerPageLandmarkInstances`).
  *
- * A page with no match for `landmarkType` (per
- * {@link ./extract-landmarks.js | extractLandmarks}) compares as an empty
- * token set. `jaccardSimilarity`'s documented treatment of two empty sets as
- * similarity `1` (see its JSDoc) means every landmark-less page lands in the
- * same "has no such landmark" group with no extra branching needed here, and
- * unambiguously in a different group from every page that does have one
- * (`jaccardSimilarity(∅, nonEmpty)` is always `0`). A landmark that exists
- * but is empty (e.g. `<header></header>`) never collides with this sentinel:
- * `tokenize` still emits at least the element's own segment for it.
+ * ## Cost
+ *
+ * Each call re-runs `extractLandmarks` over the entire `htmlList`, keeping
+ * only the one field matching `landmarkType`. Calling this once per
+ * `landmarkType` (as callers wanting more than one design signal do)
+ * re-parses every page once per type requested. A caller for whom that cost
+ * is material should call `extractLandmarks` itself once per page, read all
+ * fields off the single result, and feed each field's canonical-instance
+ * tokens to
+ * {@link ./resolve-structural-cluster-keys.js | resolveStructuralClusterKeys}
+ * directly instead.
  *
  * Does not block by URL path or stylesheet first (unlike
  * `resolvePageClusterKeys`): the same header design is normally reused
@@ -50,6 +65,7 @@ export type ResolveLandmarkVariantKeysOptions = TokenizeOptions & ResolveStructu
  * @param options
  * @example
  * ```ts
+ * // Header variants
  * resolveLandmarkVariantKeys(
  * 	[
  * 		'<body><header><nav>A</nav></header></body>',
@@ -61,6 +77,11 @@ export type ResolveLandmarkVariantKeysOptions = TokenizeOptions & ResolveStructu
  * // pages 0 and 1 (structurally identical header) share a key; page 2 (a
  * // different header structure) gets its own — text content alone (e.g.
  * // the "A" vs "B" text) would not, since tokenize() discards visible text.
+ *
+ * // Same idea works for the other landmark types:
+ * resolveLandmarkVariantKeys(htmlList, 'nav');
+ * resolveLandmarkVariantKeys(htmlList, 'footer');
+ * resolveLandmarkVariantKeys(htmlList, 'aside');
  * ```
  */
 export declare function resolveLandmarkVariantKeys(htmlList: readonly string[], landmarkType: LandmarkType, options?: ResolveLandmarkVariantKeysOptions): string[];

package/dist/resolve-landmark-variant-keys.js

@@ -1,6 +1,6 @@
 import { extractLandmarks } from './extract-landmarks.js';
+import { computePerPageLandmarkInstances } from './per-page-landmark-signatures.js';
 import { resolveStructuralClusterKeys } from './resolve-structural-cluster-keys.js';
-import { tokenize } from './tokenize.js';
 /**
  * Classifies which *variant* of a single landmark type (e.g. "which header
  * design") each page has, independently of
@@ -11,27 +11,42 @@ import { tokenize } from './tokenize.js';
  * `resolvePageClusterKeys` and combine the results themselves; this function
  * does not know about, or merge with, the other one's output.
  *
- * Each call re-runs {@link ./extract-landmarks.js | extractLandmarks} over
- * the entire `htmlList`, keeping only the one field matching `landmarkType`
- * and discarding the other three it also computed. Calling this once per
- * `landmarkType` (as the paragraph above suggests, for a caller that wants
- * more than one) therefore re-parses every page once per type requested. A
- * caller for whom that cost is material should call `extractLandmarks`
- * itself once per page, read all four fields off the single result, and feed
- * each field's token sets to
- * {@link ./resolve-structural-cluster-keys.js | resolveStructuralClusterKeys}
- * directly (with the same empty-set sentinel for a missing field) instead of
- * calling this function multiple times.
+ * ## Per-page canonical instance selection
+ *
+ * A page can now have any number of landmark instances of the requested
+ * type (see {@link ./extract-landmarks.js | extractLandmarks} for why the
+ * old shallowest-wins rule was removed). For "which variant does this page
+ * have", the canonical instance is the one whose token-set signature is
+ * most common across the corpus, ties broken by document order. This picks
+ * the site-wide chrome instance automatically — the shared site header
+ * dominates the corpus histogram — while article-specific `<header>`s that
+ * vary per page carry frequency 1 and are never selected. Choosing this way
+ * is the data-driven analogue of the old shallowest-wins rule, without
+ * fragmenting variant keys into singletons (a real regression risk of a
+ * naive "union every instance's tokens" approach: 10 pages that each carry
+ * both a site header and a per-article header would every one produce a
+ * distinct union token set, collapsing every page into its own variant key).
  *
- * A page with no match for `landmarkType` (per
- * {@link ./extract-landmarks.js | extractLandmarks}) compares as an empty
- * token set. `jaccardSimilarity`'s documented treatment of two empty sets as
- * similarity `1` (see its JSDoc) means every landmark-less page lands in the
- * same "has no such landmark" group with no extra branching needed here, and
- * unambiguously in a different group from every page that does have one
- * (`jaccardSimilarity(∅, nonEmpty)` is always `0`). A landmark that exists
- * but is empty (e.g. `<header></header>`) never collides with this sentinel:
- * `tokenize` still emits at least the element's own segment for it.
+ * A page with no matching landmark compares as an empty token set.
+ * `jaccardSimilarity`'s documented treatment of two empty sets as
+ * similarity `1` (see its JSDoc) means every landmark-less page lands in
+ * the same "has no such landmark" group with no extra branching needed
+ * here, and unambiguously in a different group from every page that does
+ * have one (`jaccardSimilarity(∅, nonEmpty)` is always `0`). A landmark
+ * that exists but tokenizes to an empty set never collides with this
+ * sentinel because `tokenize` skips it (see `computePerPageLandmarkInstances`).
+ *
+ * ## Cost
+ *
+ * Each call re-runs `extractLandmarks` over the entire `htmlList`, keeping
+ * only the one field matching `landmarkType`. Calling this once per
+ * `landmarkType` (as callers wanting more than one design signal do)
+ * re-parses every page once per type requested. A caller for whom that cost
+ * is material should call `extractLandmarks` itself once per page, read all
+ * fields off the single result, and feed each field's canonical-instance
+ * tokens to
+ * {@link ./resolve-structural-cluster-keys.js | resolveStructuralClusterKeys}
+ * directly instead.
  *
  * Does not block by URL path or stylesheet first (unlike
  * `resolvePageClusterKeys`): the same header design is normally reused
@@ -46,6 +61,7 @@ import { tokenize } from './tokenize.js';
  * @param options
  * @example
  * ```ts
+ * // Header variants
  * resolveLandmarkVariantKeys(
  * 	[
  * 		'<body><header><nav>A</nav></header></body>',
@@ -57,15 +73,42 @@ import { tokenize } from './tokenize.js';
  * // pages 0 and 1 (structurally identical header) share a key; page 2 (a
  * // different header structure) gets its own — text content alone (e.g.
  * // the "A" vs "B" text) would not, since tokenize() discards visible text.
+ *
+ * // Same idea works for the other landmark types:
+ * resolveLandmarkVariantKeys(htmlList, 'nav');
+ * resolveLandmarkVariantKeys(htmlList, 'footer');
+ * resolveLandmarkVariantKeys(htmlList, 'aside');
  * ```
  */
 export function resolveLandmarkVariantKeys(htmlList, landmarkType, options) {
-    const tokenSets = htmlList.map((html) => {
-        const region = extractLandmarks(html)[landmarkType];
-        if (region === undefined) {
-            return new Set();
+    const landmarks = htmlList.map((html) => extractLandmarks(html));
+    const perPageInstances = computePerPageLandmarkInstances(landmarks, options);
+    // Corpus-wide instance-signature histogram (restricted to the requested
+    // landmark type). Used to pick each page's canonical instance —
+    // see this function's JSDoc for why "most common" beats "union of all".
+    const corpusInstanceCount = new Map();
+    for (const instances of perPageInstances) {
+        const seenTypedSignatures = new Set();
+        for (const inst of instances) {
+            if (inst.type !== landmarkType)
+                continue;
+            if (seenTypedSignatures.has(inst.signature))
+                continue;
+            seenTypedSignatures.add(inst.signature);
+            corpusInstanceCount.set(inst.signature, (corpusInstanceCount.get(inst.signature) ?? 0) + 1);
+        }
+    }
+    const tokenSets = perPageInstances.map((instances) => {
+        let best;
+        for (const inst of instances) {
+            if (inst.type !== landmarkType)
+                continue;
+            const count = corpusInstanceCount.get(inst.signature) ?? 0;
+            if (best === undefined || count > best.count) {
+                best = { tokens: inst.tokens, count };
+            }
         }
-        return new Set(tokenize(`<body>${region}</body>`, options).tokens);
+        return best?.tokens ?? new Set();
     });
     return resolveStructuralClusterKeys(tokenSets, options);
 }