@d-zero/page-cluster 0.2.0

package/dist/merge-landmark-affined-clusters.js

@@ -0,0 +1,544 @@
+import { jaccardSimilarity } from './jaccard-similarity.js';
+import { tokenize } from './tokenize.js';
+/**
+ * The four landmark types checked, in the fixed order every signature string
+ * built below iterates them in — order must be stable so two pages with the
+ * same set of matching-and-rare types always produce byte-identical
+ * signature strings.
+ */
+const LANDMARK_TYPES = ['header', 'footer', 'nav', 'aside'];
+const DEFAULT_SIMILARITY_THRESHOLD = 0.8;
+const DEFAULT_LANDMARK_RARITY_THRESHOLD = 0.05;
+const DEFAULT_LANDMARK_GATE_SIMILARITY_THRESHOLD = 0.6;
+/**
+ * Same technique and value as `BOUNDARY_EPSILON` in
+ * `resolve-structural-cluster-keys.ts` and `split-tokens-by-frequency.ts`,
+ * kept as an independent per-file copy by this package's convention (see
+ * `resolve-structural-cluster-keys.ts`'s own `BOUNDARY_EPSILON` JSDoc).
+ */
+const BOUNDARY_EPSILON = 1e-9;
+/**
+ * Prefix distinguishing a landmark-gated merge's key from every other key
+ * family this package produces (`css:`/`path:`/`orphan-merge:`/the
+ * `[blockKey, "cluster:N"]` JSON pairs `resolvePageClusterKeys` itself
+ * emits), so the families can never collide. Mirrors
+ * `reassign-orphan-block-keys.ts`'s `REASSIGNED_KEY_PREFIX`.
+ */
+const MERGED_KEY_PREFIX = 'landmark-merge:';
+/**
+ * Reads `values[index]`, throwing instead of returning `undefined`. Every
+ * call site here indexes with a position this function generated itself, so
+ * the thrown branch is unreachable in practice; it exists to satisfy
+ * `noUncheckedIndexedAccess` without a non-null assertion. Independent copy
+ * by this package's established convention — see
+ * `resolve-page-cluster-keys.ts`'s own `requireIndex` JSDoc.
+ * @param values
+ * @param index
+ */
+function requireIndex(values, index) {
+    const value = values[index];
+    if (value === undefined) {
+        throw new Error('mergeLandmarkAffinedClusters: index out of bounds');
+    }
+    return value;
+}
+/**
+ * Reads `map.get(key)`, throwing instead of returning `undefined`. Every
+ * call site here looks up a key this function (or its caller, in the same
+ * pass) just inserted, so the thrown branch is unreachable in practice — the
+ * `Map` analogue of `requireIndex` above.
+ * @param map
+ * @param key
+ */
+function requireMapValue(map, key) {
+    const value = map.get(key);
+    if (value === undefined) {
+        throw new Error('mergeLandmarkAffinedClusters: expected map entry missing');
+    }
+    return value;
+}
+/**
+ * Validates that `value` (one of this file's three `[0, 1]`-range options,
+ * `name` being its option name for the thrown message) is in range,
+ * throwing `RangeError` otherwise. Shared by
+ * `validateMergeLandmarkAffinedClustersOptions`'s three checks so their
+ * range and message format can never drift apart from each other.
+ * @param value
+ * @param name
+ */
+function requireThreshold(value, name) {
+    if (!(value >= 0 && value <= 1)) {
+        throw new RangeError(`mergeLandmarkAffinedClusters: ${name} must be between 0 and 1, got ${value}`);
+    }
+    return value;
+}
+/**
+ * Validates `similarityThreshold`/`landmarkRarityThreshold`/
+ * `landmarkGateSimilarityThreshold` without running
+ * `mergeLandmarkAffinedClusters` itself — exported so
+ * `resolvePageClusterKeys` can fail fast on bad options even when `pages` is
+ * empty (its own per-block loop never reaches this function at all in that
+ * case). Mirrors `detect-content-depth-cap.ts`'s
+ * `validateDetectContentDepthCapOptions` exact rationale and shape.
+ * @param options
+ * @example
+ * ```ts
+ * // Fails fast on a bad option even though nothing here would otherwise
+ * // call mergeLandmarkAffinedClusters yet (e.g. cluster keys haven't been
+ * // computed).
+ * validateMergeLandmarkAffinedClustersOptions({ landmarkRarityThreshold: -1 }); // throws RangeError
+ * ```
+ */
+export function validateMergeLandmarkAffinedClustersOptions(options) {
+    requireThreshold(options?.similarityThreshold ?? DEFAULT_SIMILARITY_THRESHOLD, 'similarityThreshold');
+    requireThreshold(options?.landmarkRarityThreshold ?? DEFAULT_LANDMARK_RARITY_THRESHOLD, 'landmarkRarityThreshold');
+    requireThreshold(options?.landmarkGateSimilarityThreshold ??
+        DEFAULT_LANDMARK_GATE_SIMILARITY_THRESHOLD, 'landmarkGateSimilarityThreshold');
+}
+/**
+ * Canonicalizes a token set into a string that is identical for two sets iff
+ * their members are identical — sorted so the (undefined) `Set` iteration
+ * order can never make two structurally-equal sets hash differently. Used
+ * only to bucket byte-for-byte-identical landmark token sets before
+ * clustering (see `computeLandmarkStatus`'s JSDoc); not a general-purpose
+ * set-hashing utility.
+ *
+ * `JSON.stringify` of the sorted array, not a plain joined string: a token
+ * itself can contain a space (`format-bracket.ts` splices an element's raw
+ * `role`/`type` attribute value in verbatim, e.g. `role="a b"` produces the
+ * literal token `header[role=a b]`), so a delimiter-joined string would let
+ * two genuinely different sets — e.g. `{"a b", "c"}` and `{"a", "b", "c"}` —
+ * collide on the identical joined string `"a b c"`. `JSON.stringify` escapes
+ * each array element as its own quoted string, so no element's content can
+ * ever be mistaken for the array's own structural delimiters. Matches this
+ * file's own `landmark-merge:` key construction (`JSON.stringify(sortedKeys)`),
+ * which relies on the same collision-safety for the same reason.
+ * @param tokens
+ */
+function canonicalizeTokenSet(tokens) {
+    return JSON.stringify([...tokens].toSorted());
+}
+/**
+ * Finds the representative (root) of `index`'s set, compressing every
+ * traversed link. Independent copy of
+ * `resolve-structural-cluster-keys.ts`'s own `find`, by the same convention
+ * as `requireIndex` above.
+ * @param parent
+ * @param index
+ */
+function find(parent, index) {
+    let root = index;
+    while (requireIndex(parent, root) !== root) {
+        root = requireIndex(parent, root);
+    }
+    let current = index;
+    while (current !== root) {
+        const next = requireIndex(parent, current);
+        parent[current] = root;
+        current = next;
+    }
+    return root;
+}
+/**
+ * Complete-linkage merge of a small number of nodes (`nodeCount`) given a
+ * precomputed, symmetric `nodeCount`x`nodeCount` similarity matrix. Returns
+ * the resulting partition as groups of node indices.
+ *
+ * Deliberately not a shared import of `resolve-structural-cluster-keys.ts`'s
+ * NN-chain implementation, which is hard-wired to compute Jaccard similarity
+ * from token sets internally rather than accepting a precomputed matrix.
+ * Both call sites in this file (`computeLandmarkStatus`'s deduplicated
+ * landmark-variant buckets, and `mergeLandmarkAffinedClusters`'s
+ * rare-signature groups' distinct cluster keys) feed this function a
+ * `nodeCount` that is self-limited to a small size by construction — see
+ * `computeLandmarkStatus`'s own JSDoc for the cost analysis — so a
+ * brute-force repeated-best-pair merge (same reference shape as
+ * `resolve-structural-cluster-keys.spec.ts`'s differential-test helper) is
+ * used directly rather than re-implementing NN-chain a second time for a
+ * negligible input size.
+ * @param nodeCount
+ * @param similarity
+ * @param threshold
+ */
+function mergeSmallClustersByCompleteLinkage(nodeCount, similarity, threshold) {
+    let groups = Array.from({ length: nodeCount }, (_, index) => [index]);
+    for (;;) {
+        let bestPair;
+        let bestScore = Number.NEGATIVE_INFINITY;
+        for (let i = 0; i < groups.length; i++) {
+            for (let j = i + 1; j < groups.length; j++) {
+                let minSimilarity = Number.POSITIVE_INFINITY;
+                for (const a of requireIndex(groups, i)) {
+                    for (const b of requireIndex(groups, j)) {
+                        minSimilarity = Math.min(minSimilarity, requireIndex(similarity, a * nodeCount + b));
+                    }
+                }
+                if (minSimilarity > bestScore) {
+                    bestScore = minSimilarity;
+                    bestPair = [i, j];
+                }
+            }
+        }
+        if (!bestPair || bestScore < threshold - BOUNDARY_EPSILON) {
+            break;
+        }
+        const [i, j] = bestPair;
+        groups[i] = [...requireIndex(groups, i), ...requireIndex(groups, j)];
+        groups = groups.filter((_, index) => index !== j);
+    }
+    return groups;
+}
+/**
+ * For one landmark type, determines each page's corpus-wide variant identity
+ * and whether that variant is rare (see `mergeLandmarkAffinedClusters`'s
+ * JSDoc for what "rare" gates). A page missing this landmark type reports
+ * `{ exists: false }` unconditionally — it never counts toward any variant's
+ * frequency and is never itself "rare" or "common".
+ *
+ * Does not call `resolveStructuralClusterKeys` (unlike
+ * `resolve-landmark-variant-keys.ts`'s own landmark-variant classification).
+ * That function unconditionally narrows its input via
+ * `deriveComparisonSets` once given 10+ items — stripping out tokens shared
+ * by 90%+ of the compared sets as "chrome" and comparing only what's left.
+ * That is the right behavior for comparing *whole pages* (shared chrome
+ * should not inflate similarity between otherwise-different content), but
+ * it is backwards for comparing *landmark fragments to each other*: the
+ * stable, shared bulk of a header's markup is exactly the signal that two
+ * pages have "the same header design", and stripping it out would leave
+ * only incidental per-page differences (e.g. a "current page" nav-highlight
+ * class) to compare on. This function therefore uses raw `jaccardSimilarity`
+ * directly instead.
+ *
+ * It also skips the O(n²) all-pairs comparison `resolveStructuralClusterKeys`
+ * would otherwise run across the *entire, unblocked* corpus (a real cost:
+ * estimated ~19s and ~640MB for a single such call over an 8,936-page corpus,
+ * extrapolated from `detectContentDepthCap`'s own measured ~4s/call over a
+ * 4,085-page block — four landmark types would multiply that to ~76s). Real
+ * sites near-universally reuse byte-identical (post-tokenization) landmark
+ * markup across pages of the same template, so pages are first bucketed by
+ * exact token-set equality (`canonicalizeTokenSet`, O(n)) before any
+ * similarity is computed at all; only the resulting *distinct* buckets
+ * (expected in the tens at most, even for a large real corpus) are compared
+ * pairwise and complete-linkage-merged. This is not an approximation:
+ * deduplicating identical items before a Jaccard-based complete-linkage
+ * clustering step, then broadcasting each surviving cluster's label back to
+ * every item in the buckets it absorbed, produces the same partition a full
+ * item-by-item comparison would (`jaccardSimilarity` of two identical sets is
+ * always `1`, so duplicates always land in the same cluster; a duplicate's
+ * similarity to every other item is by definition identical to its
+ * representative's). If a corpus instead has near-zero landmark reuse (every
+ * page's markup for this type is unique), bucket count approaches page
+ * count and this degrades toward the O(n²) cost it otherwise avoids — but
+ * that scenario also means there is no shared, rare landmark for this
+ * mechanism to find evidence in regardless, so the degenerate cost case and
+ * the case where this feature has nothing to contribute coincide.
+ * @param type
+ * @param landmarks
+ * @param similarityThreshold
+ * @param landmarkRarityThreshold
+ * @param options
+ */
+function computeLandmarkStatus(type, landmarks, similarityThreshold, landmarkRarityThreshold, options) {
+    const pageCount = landmarks.length;
+    const tokenSets = landmarks.map((entry) => {
+        const region = entry[type];
+        return region === undefined
+            ? undefined
+            : new Set(tokenize(`<body>${region}</body>`, options).tokens);
+    });
+    const bucketIndexByKey = new Map();
+    const bucketRepresentatives = [];
+    const bucketIndexOfPage = tokenSets.map((tokens) => {
+        if (tokens === undefined) {
+            return;
+        }
+        const key = canonicalizeTokenSet(tokens);
+        const existing = bucketIndexByKey.get(key);
+        if (existing !== undefined) {
+            return existing;
+        }
+        const index = bucketRepresentatives.length;
+        bucketRepresentatives.push(tokens);
+        bucketIndexByKey.set(key, index);
+        return index;
+    });
+    const bucketCount = bucketRepresentatives.length;
+    const similarity = new Float64Array(bucketCount * bucketCount);
+    for (let i = 0; i < bucketCount; i++) {
+        for (let j = i + 1; j < bucketCount; j++) {
+            const score = jaccardSimilarity(requireIndex(bucketRepresentatives, i), requireIndex(bucketRepresentatives, j));
+            similarity[i * bucketCount + j] = score;
+            similarity[j * bucketCount + i] = score;
+        }
+    }
+    const groups = mergeSmallClustersByCompleteLinkage(bucketCount, similarity, similarityThreshold);
+    const groupLabelByBucketIndex = new Map();
+    for (const [groupIndex, bucketIndices] of groups.entries()) {
+        for (const bucketIndex of bucketIndices) {
+            groupLabelByBucketIndex.set(bucketIndex, `variant:${groupIndex}`);
+        }
+    }
+    const variantLabelOfPage = bucketIndexOfPage.map((bucketIndex) => bucketIndex === undefined
+        ? undefined
+        : requireMapValue(groupLabelByBucketIndex, bucketIndex));
+    const countByLabel = new Map();
+    for (const label of variantLabelOfPage) {
+        if (label !== undefined) {
+            countByLabel.set(label, (countByLabel.get(label) ?? 0) + 1);
+        }
+    }
+    return variantLabelOfPage.map((label) => {
+        if (label === undefined) {
+            return { exists: false };
+        }
+        const ratio = requireMapValue(countByLabel, label) / pageCount;
+        return { exists: true, rare: ratio < landmarkRarityThreshold, variantLabel: label };
+    });
+}
+/**
+ * Re-keys the pages of two or more distinct
+ * {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys} clusters
+ * onto one shared key when every landmark type present on their pages is
+ * both *identical* and *rare* corpus-wide, and their actual content clears a
+ * secondary, looser similarity threshold.
+ *
+ * Reimplements a mechanism previously prototyped under this same name and
+ * withdrawn (no trace survives in commit history — this JSDoc is the only
+ * record). The withdrawn version merged clusters whenever their
+ * header/footer/nav/aside matched, full stop. Validated against two real
+ * crawl corpora (302 and 8,936 pages), that produced runaway over-merging:
+ * header/footer/nav were present on 99%+ of pages and typically reused
+ * site-wide unchanged (see `extractLandmarks`'s own JSDoc for that figure),
+ * so "landmarks match" was true for nearly every page pair and carried no
+ * discriminative power at all. This reimplementation only ever treats a
+ * landmark match as merge evidence when that specific landmark *variant* is
+ * itself uncommon corpus-wide (`landmarkRarityThreshold`) — the condition
+ * the withdrawn attempt lacked.
+ *
+ * The match requirement is deliberately the most conservative option
+ * considered: *every* landmark type actually present on a page must both
+ * match its counterpart's variant and be rare — a page with even one common
+ * ("everybody has this exact header") present type contributes no evidence
+ * at all, rather than partially qualifying. A looser rule (e.g. "at least
+ * one shared rare type is enough") was rejected because it reintroduces a
+ * version of the original failure mode: a page could ride a single
+ * incidentally-rare landmark into a merge despite otherwise-ordinary,
+ * ubiquitous chrome elsewhere on the same page.
+ *
+ * Frequency is counted corpus-wide, not per-block: a `resolveStructuralClusterKeys`
+ * cluster label (`cluster:N`) is only unique within the block it was computed
+ * in, but rarity here needs one consistent count across the whole input, the
+ * same reason `resolvePageClusterKeys` itself composes `[blockKey,
+ * localLabel]` via `JSON.stringify` rather than reusing bare labels across
+ * blocks.
+ *
+ * A page with none of the four landmark types present is excluded from
+ * consideration entirely (`existingCount === 0` below) — without this, every
+ * landmark-less page across the whole corpus would share one large,
+ * unbounded "no landmarks" group, defeating the self-limiting cost bound
+ * `landmarkRarityThreshold` is otherwise supposed to guarantee (see
+ * `computeLandmarkStatus`'s JSDoc for the cost analysis this depends on).
+ *
+ * Once pages are grouped by matching-and-rare landmark signature, only
+ * signature groups spanning two or more distinct existing cluster keys do
+ * any further work. Within such a group, the *content* token sets of the
+ * group's distinct cluster keys are complete-linkage-merged at
+ * `landmarkGateSimilarityThreshold` — looser than
+ * `resolveStructuralClusterKeys`'s own `similarityThreshold`, since the
+ * whole point of this mechanism is to bridge clusters whose *content*
+ * similarity alone fell just short of the primary threshold. Complete-linkage
+ * (not single-linkage) is used for the same reason
+ * `resolveStructuralClusterKeys` itself uses it: single-linkage's chaining
+ * would let one loosely-matching pair bridge two genuinely-unrelated
+ * clusters transitively.
+ *
+ * The resulting merge is applied at *page* granularity, not by blanket-
+ * reassigning every page of the involved cluster keys: only the specific
+ * pages that were actually pooled into the qualifying signature group (and,
+ * transitively, any other page unioned with them via a different signature
+ * group) move onto the shared key. A cluster's pages that never carried the
+ * rare landmark evidence keep their original key untouched, even if some
+ * other page sharing that same cluster key did qualify and merge elsewhere.
+ * This is deliberate, not an incidental restriction: applying a merge
+ * decision to *every* page of the involved cluster keys — evidenced by only
+ * a small subset of them — would extrapolate a coincidental pairing (e.g.
+ * one outlier page in each of two otherwise-unrelated clusters happening to
+ * share a rare seasonal-campaign header) into force-merging the clusters'
+ * entire, otherwise-dissimilar membership. That is the withdrawn prototype's
+ * over-merging failure mode reappearing through a different mechanism
+ * (whole-cluster application of a single-pair signal) rather than the
+ * landmark-commonality mechanism this file was reimplemented to fix — see
+ * this function's own regression test for a worked example.
+ *
+ * Merged pages are re-keyed to `landmark-merge:${JSON.stringify(sortedKeys)}`
+ * (`sortedKeys` being the *original* cluster keys the merged pages came
+ * from) — a fresh prefix that cannot collide with `css:`/`path:`/
+ * `orphan-merge:` or `resolvePageClusterKeys`'s own `[blockKey, "cluster:N"]`
+ * pairs (mirrors `reassign-orphan-block-keys.ts`'s `orphan-merge:` prefix).
+ * @param clusterKeys - one existing final key per page, same order/length as `landmarks`/`contentTokenSets`
+ * @param landmarks - `extractLandmarks(page.html)`'s full result per page (all four fields, not just `remainderHtml`)
+ * @param contentTokenSets - per-page content token sets to use for the secondary similarity gate. Should be independent of whichever landmark markup qualified the page as evidence (e.g. always landmark-excised), so this gate is a genuine second signal rather than re-counting the same landmark tokens already used to select the page — see `resolvePageClusterKeys`'s own call site for how it builds these
+ * @param options
+ * @example
+ * ```ts
+ * // tokenize() discards visible text (see its own JSDoc), so the two
+ * // header variants below must differ structurally (child element/class),
+ * // not merely in text, to compare as different landmark variants.
+ * mergeLandmarkAffinedClusters(
+ * 	['["css:a", "cluster:0"]', '["css:b", "cluster:0"]', 'path:other'],
+ * 	[
+ * 		{ header: '<header><i class="mark-a"></i></header>', remainderHtml: '' },
+ * 		{ header: '<header><i class="mark-a"></i></header>', remainderHtml: '' },
+ * 		{ header: '<header><b class="mark-b"></b></header>', remainderHtml: '' },
+ * 	],
+ * 	[new Set(['a', 'b']), new Set(['a', 'c']), new Set(['z'])],
+ * 	{ landmarkRarityThreshold: 0.7, landmarkGateSimilarityThreshold: 0.3 },
+ * );
+ * // pages 0 and 1 share an identical header used by only 2 of the 3 pages
+ * // (a 2/3 ≈ 0.667 corpus frequency, rare at threshold 0.7) and their
+ * // content clears 0.3, so they merge onto one landmark-merge: key; page 2
+ * // (a structurally different header) is left untouched
+ * ```
+ */
+export function mergeLandmarkAffinedClusters(clusterKeys, landmarks, contentTokenSets, options) {
+    // Self-validates, unlike relying solely on a caller's separate
+    // validateMergeLandmarkAffinedClustersOptions call: mirrors
+    // detectContentDepthCap's own first line, and matters here because this
+    // function is directly importable via this package's
+    // `./merge-landmark-affined-clusters` subpath export, not only reachable
+    // through resolvePageClusterKeys's own eager pre-validation. Without
+    // this, an out-of-range or NaN threshold would silently defeat
+    // mergeSmallClustersByCompleteLinkage's stop condition
+    // (`bestScore < threshold - BOUNDARY_EPSILON` never becomes true against
+    // a negative or NaN threshold) and force-merge every candidate group
+    // into one, with no error raised.
+    validateMergeLandmarkAffinedClustersOptions(options);
+    if (clusterKeys.length === 0) {
+        return [];
+    }
+    const similarityThreshold = options?.similarityThreshold ?? DEFAULT_SIMILARITY_THRESHOLD;
+    const landmarkRarityThreshold = options?.landmarkRarityThreshold ?? DEFAULT_LANDMARK_RARITY_THRESHOLD;
+    const landmarkGateSimilarityThreshold = options?.landmarkGateSimilarityThreshold ??
+        DEFAULT_LANDMARK_GATE_SIMILARITY_THRESHOLD;
+    const statusByType = new Map(LANDMARK_TYPES.map((type) => [
+        type,
+        computeLandmarkStatus(type, landmarks, similarityThreshold, landmarkRarityThreshold, options),
+    ]));
+    const signatures = clusterKeys.map((_, pageIndex) => {
+        let existingCount = 0;
+        let allRare = true;
+        const parts = [];
+        for (const type of LANDMARK_TYPES) {
+            const status = requireIndex(requireMapValue(statusByType, type), pageIndex);
+            if (!status.exists) {
+                continue;
+            }
+            existingCount++;
+            if (!status.rare) {
+                allRare = false;
+            }
+            parts.push(`${type}:${status.variantLabel}`);
+        }
+        return existingCount > 0 && allRare ? parts.join('|') : undefined;
+    });
+    const pageIndicesBySignature = new Map();
+    for (const [pageIndex, signature] of signatures.entries()) {
+        if (signature === undefined) {
+            continue;
+        }
+        const indices = pageIndicesBySignature.get(signature);
+        if (indices) {
+            indices.push(pageIndex);
+        }
+        else {
+            pageIndicesBySignature.set(signature, [pageIndex]);
+        }
+    }
+    // Page-level union-find, not cluster-key-level: a merge decision drawn
+    // from a signature group's (necessarily partial — only the pages that
+    // happened to land in that group) evidence must bind only the specific
+    // pages that supplied it. Unioning by cluster key instead would apply a
+    // single qualifying pair's evidence to every page sharing either
+    // cluster key, including pages with no evidence at all — see this
+    // function's own JSDoc "page granularity" section for why that
+    // reintroduces the withdrawn prototype's over-merging failure.
+    const parent = Int32Array.from({ length: clusterKeys.length }, (_, index) => index);
+    for (const pageIndices of pageIndicesBySignature.values()) {
+        const memberIndicesByClusterKey = new Map();
+        for (const pageIndex of pageIndices) {
+            const clusterKey = requireIndex(clusterKeys, pageIndex);
+            const members = memberIndicesByClusterKey.get(clusterKey);
+            if (members) {
+                members.push(pageIndex);
+            }
+            else {
+                memberIndicesByClusterKey.set(clusterKey, [pageIndex]);
+            }
+        }
+        const groupClusterKeys = [...memberIndicesByClusterKey.keys()];
+        if (groupClusterKeys.length <= 1) {
+            continue;
+        }
+        const groupMembers = groupClusterKeys.map((key) => requireMapValue(memberIndicesByClusterKey, key));
+        const k = groupClusterKeys.length;
+        const similarity = new Float64Array(k * k);
+        for (let i = 0; i < k; i++) {
+            for (let j = i + 1; j < k; j++) {
+                let minSimilarity = Number.POSITIVE_INFINITY;
+                for (const p of requireIndex(groupMembers, i)) {
+                    for (const q of requireIndex(groupMembers, j)) {
+                        const score = jaccardSimilarity(requireIndex(contentTokenSets, p), requireIndex(contentTokenSets, q));
+                        minSimilarity = Math.min(minSimilarity, score);
+                    }
+                }
+                similarity[i * k + j] = minSimilarity;
+                similarity[j * k + i] = minSimilarity;
+            }
+        }
+        const mergedGroups = mergeSmallClustersByCompleteLinkage(k, similarity, landmarkGateSimilarityThreshold);
+        for (const group of mergedGroups) {
+            if (group.length <= 1) {
+                continue;
+            }
+            // Union only the specific evidence pages behind the cluster keys
+            // in this merged group — not every page that happens to share
+            // those keys elsewhere in the corpus (see this function's JSDoc).
+            const evidencePages = group.flatMap((clusterIndex) => requireIndex(groupMembers, clusterIndex));
+            const firstPage = requireIndex(evidencePages, 0);
+            for (let i = 1; i < evidencePages.length; i++) {
+                const rootA = find(parent, firstPage);
+                const rootB = find(parent, requireIndex(evidencePages, i));
+                if (rootA !== rootB) {
+                    parent[rootB] = rootA;
+                }
+            }
+        }
+    }
+    const pageIndicesByRoot = new Map();
+    for (let pageIndex = 0; pageIndex < clusterKeys.length; pageIndex++) {
+        const root = find(parent, pageIndex);
+        const indices = pageIndicesByRoot.get(root);
+        if (indices) {
+            indices.push(pageIndex);
+        }
+        else {
+            pageIndicesByRoot.set(root, [pageIndex]);
+        }
+    }
+    const remap = new Map();
+    for (const indices of pageIndicesByRoot.values()) {
+        const originalKeys = new Set(indices.map((index) => requireIndex(clusterKeys, index)));
+        // A component every one of whose pages already shares one original
+        // cluster key never actually crossed a cluster boundary (the common
+        // case: most pages never entered any signature group at all, so
+        // their root is just themselves) — nothing to re-key.
+        if (originalKeys.size <= 1) {
+            continue;
+        }
+        const mergedKey = `${MERGED_KEY_PREFIX}${JSON.stringify([...originalKeys].toSorted())}`;
+        for (const index of indices) {
+            remap.set(index, mergedKey);
+        }
+    }
+    return clusterKeys.map((key, index) => remap.get(index) ?? key);
+}

package/dist/merge-spans.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Merges a set of (possibly overlapping or nested) `[start, end)` spans into
+ * the smallest equivalent set of disjoint spans, sorted by start offset.
+ * Matched spans commonly nest in real markup (e.g. a site nav living inside
+ * the header, `<header><nav>...</nav></header>`) — merging first means the
+ * later excision pass never has to reason about overlap.
+ * @param spans
+ */
+export declare function mergeSpans(spans: readonly {
+    start: number;
+    end: number;
+}[]): {
+    start: number;
+    end: number;
+}[];

package/dist/merge-spans.js

@@ -0,0 +1,22 @@
+/**
+ * Merges a set of (possibly overlapping or nested) `[start, end)` spans into
+ * the smallest equivalent set of disjoint spans, sorted by start offset.
+ * Matched spans commonly nest in real markup (e.g. a site nav living inside
+ * the header, `<header><nav>...</nav></header>`) — merging first means the
+ * later excision pass never has to reason about overlap.
+ * @param spans
+ */
+export function mergeSpans(spans) {
+    const sorted = [...spans].toSorted((a, b) => a.start - b.start);
+    const merged = [];
+    for (const span of sorted) {
+        const last = merged.at(-1);
+        if (last && span.start <= last.end) {
+            last.end = Math.max(last.end, span.end);
+        }
+        else {
+            merged.push({ ...span });
+        }
+    }
+    return merged;
+}

package/dist/noise-class-patterns.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Heuristics for auto-generated class names that change on every build even
+ * when the underlying template is unchanged (CSS Modules, styled-components,
+ * emotion, bundler content-hash suffixes). Left in place, these would make
+ * identical templates look structurally different across builds/deploys,
+ * defeating near-duplicate detection. The generic alphanumeric-hash pattern requires
+ * both a letter and a digit so real words (e.g. BEM modifiers like
+ * `card--active`) are not caught by accident; it is still the least precise
+ * entry here, which is why `filterNoiseClasses` can be turned off.
+ *
+ * The `sc-`/`css-`/generic-hex patterns below all follow the same
+ * "require a digit or uppercase letter" idiom to rule out real English words
+ * that happen to fit the hash's character-set shape (`sc-header`,
+ * `css-editor`, `section-facade`, ...). This trades a small, accepted
+ * false-negative rate for hash generators that occasionally produce an
+ * all-lowercase, all-letter run (same trade-off already made for the
+ * double-underscore pattern below) against eliminating false positives on
+ * ordinary class names, which is the more common and more disruptive
+ * failure for this package's purpose.
+ */
+export declare const DEFAULT_NOISE_CLASS_PATTERNS: readonly RegExp[];