@d-zero/page-cluster 0.5.5 → 0.5.7

package/dist/find-cross-cluster-duplicates.d.ts

@@ -0,0 +1,94 @@
+import type { MirrorAxis } from './detect-mirror-axis.js';
+/**
+ * A page as {@link findCrossClusterDuplicates} and
+ * {@link ./validate-cluster-partition.js | validateClusterPartition} need
+ * it: already tokenized (no re-tokenization here — a caller validating a
+ * result from {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}
+ * already has these token sets on hand) and carrying the blocking-relevant
+ * signals ({@link ./resolve-page-cluster-keys.js | PageClusterSignals}'s
+ * `paths`/`stylesheetHrefs`) needed to corroborate a match against a
+ * {@link MirrorAxis}.
+ */
+export type ClusteredPage = {
+    readonly clusterKey: string;
+    readonly tokens: ReadonlySet<string>;
+    readonly paths: readonly string[];
+    readonly stylesheetHrefs: readonly string[];
+};
+/**
+ * Options for {@link findCrossClusterDuplicates}.
+ */
+export type FindCrossClusterDuplicatesOptions = {
+    /**
+     * A `MirrorAxis` to corroborate near-duplicate (similarity `< 1`) pairs
+     * against. Omitting it (or passing `null`) still finds exact-duplicate
+     * pairs — see {@link findCrossClusterDuplicates}'s JSDoc — just without
+     * the near-duplicate pass, and every result's `corroboratedByMirrorAxis`
+     * is `false` (corroboration was never attempted, not disproven).
+     */
+    readonly mirrorAxis?: MirrorAxis | null;
+    /** @see DEFAULT_CORROBORATED_SIMILARITY_THRESHOLD */
+    readonly corroboratedSimilarityThreshold?: number;
+};
+/**
+ * One pair of clusters found to plausibly be the same template, split by
+ * the clustering run under review.
+ */
+export type CrossClusterDuplicate = {
+    /** The two cluster keys, ordered so `clusterKeyA < clusterKeyB` (stable regardless of input order). */
+    readonly clusterKeyA: string;
+    readonly clusterKeyB: string;
+    /** The highest pairwise token-set Jaccard similarity found between the two clusters' members. */
+    readonly similarity: number;
+    /**
+     * Whether the best-similarity pair was also corroborated by the mirror
+     * axis (same axis-normalized path shape and axis-normalized stylesheet
+     * href set). Always `false` when no axis was supplied. `similarity === 1`
+     * pairs are reported regardless of this flag — see this function's JSDoc.
+     */
+    readonly corroboratedByMirrorAxis: boolean;
+};
+/**
+ * Finds pairs of clusters whose members are plausibly the same template,
+ * split apart by the clustering run under review — the signal a caller acts
+ * on to merge them back (see {@link ./merge-validated-clusters.js |
+ * mergeValidatedClusters}).
+ *
+ * Two passes, both bounded well below the full `O(pageCount²)` cross
+ * product:
+ *
+ * 1. **Exact match** — pages are grouped by an exact hash of their token
+ *    set (`O(pageCount)`). Any hash group spanning more than one cluster key
+ *    is an instant duplicate at `similarity: 1`: two pages whose structural
+ *    tokens are byte-identical being in different clusters is a partition
+ *    error regardless of *why* — no corroboration is required or checked
+ *    for acceptance (an axis check still runs on the pair, if one was
+ *    supplied, purely to populate `corroboratedByMirrorAxis` informationally).
+ *    Pages with an empty token set are excluded — no structural evidence to
+ *    match on.
+ * 2. **Axis-corroborated near match** (only when `options.mirrorAxis` is
+ *    given) — pages are grouped by
+ *    {@link ./normalize-path-by-mirror-axis.js | normalizePathByMirrorAxis}
+ *    shape, which is naturally small per group (bounded by how many mirror
+ *    values recur under that shape, not by corpus size). Within a shape
+ *    group spanning more than one cluster key, every cross-cluster pair is
+ *    compared directly; a pair clearing `corroboratedSimilarityThreshold`
+ *    **and** matching under
+ *    {@link ./normalize-href-by-mirror-axis.js | normalizeHrefByMirrorAxis}
+ *    is accepted. Both signals are required here — same path shape alone
+ *    can recur by coincidence (two different templates that happen to sit at
+ *    the same depth), and same stylesheet shape alone doesn't imply the same
+ *    DOM structure.
+ *
+ * Results are deduplicated to one entry per cluster-key pair, keeping the
+ * highest similarity found and OR-ing the corroboration flag across every
+ * qualifying page pair for that cluster pair.
+ * @param pages
+ * @param options
+ * @example
+ * ```ts
+ * const duplicates = findCrossClusterDuplicates(pages, { mirrorAxis: axis });
+ * const safeToMerge = duplicates.filter((d) => d.similarity === 1 || d.corroboratedByMirrorAxis);
+ * ```
+ */
+export declare function findCrossClusterDuplicates(pages: readonly ClusteredPage[], options?: FindCrossClusterDuplicatesOptions): readonly CrossClusterDuplicate[];

package/dist/find-cross-cluster-duplicates.js

@@ -0,0 +1,164 @@
+import { jaccardSimilarity } from './jaccard-similarity.js';
+import { normalizeHrefByMirrorAxis } from './normalize-href-by-mirror-axis.js';
+import { normalizePathByMirrorAxis } from './normalize-path-by-mirror-axis.js';
+/**
+ * Similarity required for a page pair below exact match to be accepted —
+ * only once axis+href corroboration also holds (see
+ * {@link findCrossClusterDuplicates}'s JSDoc). Matches the 80% floor already
+ * used throughout this package's Stage B for corroborating a secondary
+ * signal ({@link ./merge-cross-block-clusters.js | CROSS_BLOCK_THRESHOLD},
+ * `SHELL_CORROBORATION_THRESHOLD`) rather than carrying a decision alone.
+ */
+const DEFAULT_CORROBORATED_SIMILARITY_THRESHOLD = 0.8;
+/**
+ * @param tokens
+ */
+function exactHashKey(tokens) {
+    return [...tokens].toSorted().join(' ');
+}
+/**
+ * @param a
+ * @param b
+ * @param axis
+ */
+function hrefsMatchUnderAxis(a, b, axis) {
+    const normalize = (hrefs) => [...new Set(hrefs.map((h) => normalizeHrefByMirrorAxis(h, axis)))]
+        .toSorted()
+        .join(' ');
+    return normalize(a.stylesheetHrefs) === normalize(b.stylesheetHrefs);
+}
+/**
+ * @param best
+ * @param a
+ * @param b
+ * @param similarity
+ * @param axis
+ */
+function recordCandidate(best, a, b, similarity, axis) {
+    if (a.clusterKey === b.clusterKey)
+        return;
+    const [keyA, keyB] = a.clusterKey < b.clusterKey
+        ? [a.clusterKey, b.clusterKey]
+        : [b.clusterKey, a.clusterKey];
+    const corroborated = axis !== null && hrefsMatchUnderAxis(a, b, axis);
+    let inner = best.get(keyA);
+    if (!inner) {
+        inner = new Map();
+        best.set(keyA, inner);
+    }
+    const existing = inner.get(keyB);
+    if (!existing || similarity > existing.similarity) {
+        inner.set(keyB, { similarity, corroboratedByMirrorAxis: corroborated });
+    }
+    else if (similarity === existing.similarity &&
+        corroborated &&
+        !existing.corroboratedByMirrorAxis) {
+        inner.set(keyB, { similarity, corroboratedByMirrorAxis: true });
+    }
+}
+/**
+ * Finds pairs of clusters whose members are plausibly the same template,
+ * split apart by the clustering run under review — the signal a caller acts
+ * on to merge them back (see {@link ./merge-validated-clusters.js |
+ * mergeValidatedClusters}).
+ *
+ * Two passes, both bounded well below the full `O(pageCount²)` cross
+ * product:
+ *
+ * 1. **Exact match** — pages are grouped by an exact hash of their token
+ *    set (`O(pageCount)`). Any hash group spanning more than one cluster key
+ *    is an instant duplicate at `similarity: 1`: two pages whose structural
+ *    tokens are byte-identical being in different clusters is a partition
+ *    error regardless of *why* — no corroboration is required or checked
+ *    for acceptance (an axis check still runs on the pair, if one was
+ *    supplied, purely to populate `corroboratedByMirrorAxis` informationally).
+ *    Pages with an empty token set are excluded — no structural evidence to
+ *    match on.
+ * 2. **Axis-corroborated near match** (only when `options.mirrorAxis` is
+ *    given) — pages are grouped by
+ *    {@link ./normalize-path-by-mirror-axis.js | normalizePathByMirrorAxis}
+ *    shape, which is naturally small per group (bounded by how many mirror
+ *    values recur under that shape, not by corpus size). Within a shape
+ *    group spanning more than one cluster key, every cross-cluster pair is
+ *    compared directly; a pair clearing `corroboratedSimilarityThreshold`
+ *    **and** matching under
+ *    {@link ./normalize-href-by-mirror-axis.js | normalizeHrefByMirrorAxis}
+ *    is accepted. Both signals are required here — same path shape alone
+ *    can recur by coincidence (two different templates that happen to sit at
+ *    the same depth), and same stylesheet shape alone doesn't imply the same
+ *    DOM structure.
+ *
+ * Results are deduplicated to one entry per cluster-key pair, keeping the
+ * highest similarity found and OR-ing the corroboration flag across every
+ * qualifying page pair for that cluster pair.
+ * @param pages
+ * @param options
+ * @example
+ * ```ts
+ * const duplicates = findCrossClusterDuplicates(pages, { mirrorAxis: axis });
+ * const safeToMerge = duplicates.filter((d) => d.similarity === 1 || d.corroboratedByMirrorAxis);
+ * ```
+ */
+export function findCrossClusterDuplicates(pages, options) {
+    const axis = options?.mirrorAxis ?? null;
+    const threshold = options?.corroboratedSimilarityThreshold ?? DEFAULT_CORROBORATED_SIMILARITY_THRESHOLD;
+    const best = new Map();
+    // Pass 1: exact match via hash grouping.
+    const byExactHash = new Map();
+    for (const page of pages) {
+        if (page.tokens.size === 0)
+            continue;
+        const key = exactHashKey(page.tokens);
+        const group = byExactHash.get(key);
+        if (group)
+            group.push(page);
+        else
+            byExactHash.set(key, [page]);
+    }
+    for (const group of byExactHash.values()) {
+        if (new Set(group.map((p) => p.clusterKey)).size < 2)
+            continue;
+        for (let a = 0; a < group.length; a++) {
+            for (let b = a + 1; b < group.length; b++) {
+                recordCandidate(best, group[a], group[b], 1, axis);
+            }
+        }
+    }
+    // Pass 2: axis-corroborated near match, scoped to same axis-normalized shape.
+    if (axis !== null) {
+        const byShape = new Map();
+        for (const page of pages) {
+            const shape = normalizePathByMirrorAxis(page.paths, axis);
+            const group = byShape.get(shape);
+            if (group)
+                group.push(page);
+            else
+                byShape.set(shape, [page]);
+        }
+        for (const group of byShape.values()) {
+            if (new Set(group.map((p) => p.clusterKey)).size < 2)
+                continue;
+            for (let a = 0; a < group.length; a++) {
+                for (let b = a + 1; b < group.length; b++) {
+                    const pageA = group[a];
+                    const pageB = group[b];
+                    if (pageA.clusterKey === pageB.clusterKey)
+                        continue;
+                    const similarity = jaccardSimilarity(pageA.tokens, pageB.tokens);
+                    if (similarity < threshold)
+                        continue;
+                    if (!hrefsMatchUnderAxis(pageA, pageB, axis))
+                        continue;
+                    recordCandidate(best, pageA, pageB, similarity, axis);
+                }
+            }
+        }
+    }
+    const results = [];
+    for (const [clusterKeyA, inner] of best) {
+        for (const [clusterKeyB, { similarity, corroboratedByMirrorAxis }] of inner) {
+            results.push({ clusterKeyA, clusterKeyB, similarity, corroboratedByMirrorAxis });
+        }
+    }
+    return results;
+}

package/dist/merge-cross-block-clusters.d.ts

@@ -17,6 +17,18 @@ export type CrossBlockUnit = {
     readonly key: string;
     readonly memberTokenSets: readonly ReadonlySet<string>[];
     readonly memberLandmarkInstances: readonly (readonly PerPageLandmarkInstance[])[];
+    /**
+     * Original corpus index of each member, parallel to `memberTokenSets` —
+     * lets a caller that pools `finalGroupsByRoot`'s output back into
+     * page-level records (e.g. to build a
+     * {@link ./find-cross-cluster-duplicates.js | ClusteredPage}) recover
+     * which page each merged token set came from, without a second
+     * corpus-wide pass. Optional because most callers (every hand-built test
+     * fixture in this file included) only need `mergeCrossBlockClusters` for
+     * its cluster-key decisions and never read this back; omitted entries are
+     * normalized to `-1` internally rather than left misaligned.
+     */
+    readonly memberPageIndices?: readonly number[];
 };
 /**
  * Frequency-based token core of a group of member pages: a token must be
@@ -42,6 +54,12 @@ export declare function computeQuorumCore(memberDistinctiveTokens: readonly Read
 export type FinalGroupMembers = {
     readonly tokenSets: readonly ReadonlySet<string>[];
     readonly landmarkInstances: readonly (readonly PerPageLandmarkInstance[])[];
+    /**
+     * Parallel to `tokenSets` — see {@link CrossBlockUnit}'s own
+     * `memberPageIndices`. An entry is `-1` for any member whose originating
+     * unit omitted `memberPageIndices`.
+     */
+    readonly pageIndices: readonly number[];
 };
 /**
  * `mergeCrossBlockClusters`'s result: the root-key mapping every caller

package/dist/merge-cross-block-clusters.js

@@ -118,6 +118,154 @@ export function computeQuorumCore(memberDistinctiveTokens) {
     }
     return union;
 }
+/**
+ * Minimum ratio a proposed merge's post-merge quorum-core size must retain,
+ * relative to the strongest lineage anchor on either side (see
+ * `anchorByRoot` in {@link filterMergesByCohesion}), or the merge is
+ * discarded.
+ *
+ * Chosen against `merge-cross-block-clusters.spec.ts`'s own fixtures, which
+ * bound both edges this value has to sit between:
+ * - "a hub unit does not absorb several mutually-unrelated units via
+ *   containment" needs a ratio *above* ~`0.3` to reject each absorption (the
+ *   hub's own 10-token core collapses to the 3 tokens the absorbed unit
+ *   happens to also carry).
+ * - the bundled `buildMirroredTemplateFixture` fixture (see
+ *   `resolve-page-cluster-keys.spec.ts`) needs a ratio *at or below* ~`0.8`
+ *   to keep merging every one of its genuine per-mirror units (the same
+ *   template, once per axis value) — above that, some legitimate mirrors
+ *   stop merging because a handful of per-page module drops (this fixture's
+ *   stand-in for optional-section variation) dip just far enough below a
+ *   stricter bar.
+ *
+ * `0.7` sits with margin inside `(0.3, 0.8]` rather than against either
+ * edge. This is a per-step ratio, not an absolute floor — see
+ * `anchorByRoot`'s own JSDoc for why an anchor was needed at all, and for
+ * the residual limitation neither the ratio nor the anchor fixes: many
+ * originally-small-core units chained together one step at a time can each
+ * individually clear this ratio against the previous step's *already-small*
+ * anchor, so a long enough chain of naturally low-information pages can
+ * still end up merged even though no single step looks anomalous. Longer
+ * chains are exactly what {@link ./build-cluster-reason.js | ClusterReason}'s
+ * `blocking` array length and
+ * {@link ./compute-cluster-cohesion.js | computeClusterCohesion}'s
+ * `suspicious` flag are for — this guard reduces how often that happens and
+ * how far it goes, it does not claim to make it impossible.
+ */
+const MIN_COHESION_RATIO = 0.7;
+/**
+ * `computeQuorumCore` without its full-union fallback for empty cores. The
+ * fallback exists so a final `ClusterReason.structuralCoreTokens` is never
+ * empty for a genuinely tiny unit — but it makes core *size* useless as a
+ * cohesion signal: a merge that destroys every token's 80% quorum would
+ * silently read as "core size is now the size of the union", i.e. bigger,
+ * not smaller. {@link filterMergesByCohesion} needs "zero tokens survive
+ * quorum" to mean zero.
+ * @param memberDistinctiveTokens
+ */
+function strictQuorumCoreSize(memberDistinctiveTokens) {
+    const n = memberDistinctiveTokens.length;
+    if (n === 0)
+        return 0;
+    const minCount = Math.ceil(QUORUM_FRACTION * n);
+    const tokenCount = new Map();
+    for (const tokens of memberDistinctiveTokens) {
+        for (const token of tokens) {
+            tokenCount.set(token, (tokenCount.get(token) ?? 0) + 1);
+        }
+    }
+    let coreSize = 0;
+    for (const count of tokenCount.values()) {
+        if (count >= minCount)
+            coreSize++;
+    }
+    return coreSize;
+}
+/**
+ * Filters a round's proposed `[absorbed, root]` merges, rejecting any merge
+ * whose post-merge quorum core would collapse relative to the best core any
+ * single original unit now pooled into either side ever had — the guard
+ * against Stage B's fine/L2 stages successively absorbing unrelated units
+ * into a "catch-all" whose core shrinks toward shell-only tokens with every
+ * additional merge (each individual merge can look locally justified — the
+ * pair's *pre-merge* cores still overlap enough to clear
+ * `CROSS_BLOCK_THRESHOLD`/containment/L2 — while the *post-merge* core keeps
+ * shrinking, which none of those pre-merge checks observe).
+ *
+ * ## Why an anchor, not just the immediately preceding step
+ *
+ * An earlier version compared each merge only against the pool as it stood
+ * after the *previous* accepted merge for that root. That still lets a long
+ * chain erode a core to nothing, one acceptable-looking step at a time: if
+ * each step's ratio is checked only against the *result of the previous
+ * step*, and each step dilutes the pool a little, the reference the ratio is
+ * measured against keeps shrinking right along with the pool being measured
+ * — nothing ever compares the current state back to where the lineage
+ * started, so a chain of many individually-small erosions can compound into
+ * a total collapse no single step's check would have allowed on its own.
+ * `anchorByRoot` fixes this: every original unit's *own*, pre-any-merge core
+ * size (`anchorCoreSizeByKey`, computed once before the round loop) is
+ * carried forward — via `Math.max`, never re-derived from the current pool —
+ * as units merge into a root, so every later merge attempt is still measured
+ * against the strongest evidence its lineage ever had, not against
+ * whatever the lineage has been diluted to by the time of the attempt.
+ *
+ * Merges proposed for the same root are applied incrementally, in the order
+ * given, checking each one against the pool as it stood *after* the
+ * previously accepted merges for that root, so a chain of merges within a
+ * single call cannot each pass by being compared to a `pooled` state that
+ * never reflects the merges already accepted earlier in the same call.
+ *
+ * Two things intentionally do not gate rejection alone:
+ * - `strictQuorumCoreSize` is used instead of `computeQuorumCore`'s size —
+ *   see that function's own JSDoc for why the fallback would invert the
+ *   signal for exactly the merges this guard exists to catch.
+ * - A merge whose post-merge core size is `0` is always rejected, even when
+ *   the anchor was also `0` (which would make the ratio check
+ *   `0 >= ratio * 0` vacuously pass) — otherwise a unit that already lost
+ *   its own core would become a sink that absorbs anything with no further
+ *   resistance.
+ * @param proposedMerges `[absorbedKey, rootKey]` pairs, as produced by the
+ *   fine or L2 stage's own union-find pass.
+ * @param groupDistinctive This round's per-group distinctive token sets,
+ *   keyed by group key. Callers pass the class-name-stripped
+ *   `groupDistinctiveShaped` projection (see
+ *   {@link mergeCrossBlockClusters}'s own body) rather than raw
+ *   `groupDistinctive` — the fine stage's shape-Jaccard merges pair units
+ *   with disjoint raw tokens by construction, and a cohesion check against
+ *   raw tokens would reject every one of those merges outright.
+ * @param anchorByRoot Every current root's strongest lineage core size (see
+ *   above). Mutated in place: an accepted merge's root inherits
+ *   `Math.max(root's anchor, absorbed's anchor)`.
+ */
+function filterMergesByCohesion(proposedMerges, groupDistinctive, anchorByRoot) {
+    const byRoot = new Map();
+    for (const [absorbed, root] of proposedMerges) {
+        const list = byRoot.get(root);
+        if (list)
+            list.push(absorbed);
+        else
+            byRoot.set(root, [absorbed]);
+    }
+    const accepted = [];
+    for (const [root, absorbedKeys] of byRoot) {
+        let pooled = [...(groupDistinctive.get(root) ?? [])];
+        for (const absorbed of absorbedKeys) {
+            const absorbedTokens = groupDistinctive.get(absorbed) ?? [];
+            const candidatePool = [...pooled, ...absorbedTokens];
+            const candidateCoreSize = strictQuorumCoreSize(candidatePool);
+            const referenceCoreSize = Math.max(anchorByRoot.get(root) ?? strictQuorumCoreSize(pooled), anchorByRoot.get(absorbed) ?? strictQuorumCoreSize(absorbedTokens));
+            if (candidateCoreSize > 0 &&
+                candidateCoreSize >= MIN_COHESION_RATIO * referenceCoreSize) {
+                pooled = candidatePool;
+                anchorByRoot.set(root, referenceCoreSize);
+                anchorByRoot.delete(absorbed);
+                accepted.push([absorbed, root]);
+            }
+        }
+    }
+    return accepted;
+}
 /**
  *
  * @param core
@@ -164,6 +312,62 @@ function l2Contained(xSig, ySig) {
     }
     return true;
 }
+/**
+ * Canonical id for an `l2Signature`'s *shape* — its key set, ignoring the
+ * per-key counts — so {@link hasDiscriminatingL2Signatures} can tell whether
+ * two units reduced to the same vocabulary of `main`-anchored shapes,
+ * independent of how many pages contributed to each count.
+ * @param signature
+ */
+function l2SignatureShapeId(signature) {
+    return [...signature.keys()].toSorted().join('');
+}
+/**
+ * Minimum number of units an L2-degeneracy check requires before it will
+ * reject the whole comparison — below this, "every unit shares one shape"
+ * is unremarkable (there is nothing to discriminate between yet), not
+ * evidence the signature itself lacks resolving power.
+ */
+const MIN_L2_PARTICIPANTS_FOR_DEGENERACY_CHECK = 3;
+/**
+ * Whether this round's L2 signatures carry any discriminating power at all,
+ * checked once per round *before* running the `O(l2n²)` containment
+ * comparison rather than discovering it empirically pair by pair.
+ *
+ * `l2Signature` truncates to `main` plus up to 2 shape-stripped levels (see
+ * its own JSDoc); a corpus where the actual template content sits under a
+ * shared `main > article > <wrapper>` chain collapses every unit's
+ * signature to the exact same handful of keys (`main>article>*`, in the
+ * bundled `buildMirroredTemplateFixture` fixture's own case — see
+ * `merge-cross-block-clusters.spec.ts`), at which point `l2Contained`'s
+ * multiset containment degenerates into a plain count comparison with no
+ * structural meaning left. Rather than let that degenerate comparison run
+ * (and rely solely on {@link filterMergesByCohesion} to catch whatever it
+ * proposes), this is checked up front: if every participating unit reduces
+ * to the *same* shape, the signature has already lost all resolving power
+ * for this round, and comparing pairs is wasted work.
+ *
+ * Only total collapse (all participants share one shape) is detected —
+ * partial collapse (e.g. 8 units reducing to 2 shapes that don't line up
+ * with their true 8 templates) is not, and still relies on
+ * {@link filterMergesByCohesion} downstream.
+ * @param l2Keys
+ * @param getL2Sig
+ */
+function hasDiscriminatingL2Signatures(l2Keys, getL2Sig) {
+    const shapeIds = new Set();
+    let participantCount = 0;
+    for (const key of l2Keys) {
+        const sig = getL2Sig(key);
+        if (!sig)
+            continue;
+        participantCount++;
+        shapeIds.add(l2SignatureShapeId(sig));
+        if (shapeIds.size > 1)
+            return true;
+    }
+    return participantCount < MIN_L2_PARTICIPANTS_FOR_DEGENERACY_CHECK;
+}
 /**
  * Merges cross-block clusters (Stage B) via recursive quorum-core comparison.
  *
@@ -198,7 +402,11 @@ export function mergeCrossBlockClusters(units, options) {
             rootByKey: new Map(units.map((u) => [u.key, u.key])),
             finalGroupsByRoot: new Map(units.map((u) => [
                 u.key,
-                { tokenSets: u.memberTokenSets, landmarkInstances: u.memberLandmarkInstances },
+                {
+                    tokenSets: u.memberTokenSets,
+                    landmarkInstances: u.memberLandmarkInstances,
+                    pageIndices: u.memberPageIndices ?? u.memberTokenSets.map(() => -1),
+                },
             ])),
         };
     }
@@ -209,10 +417,26 @@ export function mergeCrossBlockClusters(units, options) {
         groups.set(unit.key, {
             tokenSets: [...unit.memberTokenSets],
             landmarkInstances: [...unit.memberLandmarkInstances],
+            pageIndices: [...(unit.memberPageIndices ?? unit.memberTokenSets.map(() => -1))],
         });
     }
     // Maps every original key to its current root (updated on each merge)
     const keyToRoot = new Map(units.map((u) => [u.key, u.key]));
+    // Each unit's own pre-any-merge core size, carried forward by
+    // `filterMergesByCohesion` as units merge — see that function's own
+    // JSDoc for why an anchor is needed at all. Computed the same way round
+    // 1's own `groupDistinctiveShaped` would (document frequency over the
+    // full initial unit set, then class-name-stripped), so a solo unit's
+    // anchor matches what the very first round would already compute for it.
+    const initialFrequency = computeDocumentFrequency(units.flatMap((u) => u.memberTokenSets));
+    const anchorByRoot = new Map(units.map((u) => {
+        const distinctiveShaped = u.memberTokenSets.map((tokens) => {
+            const { contentTokens } = splitTokensByFrequency(tokens, initialFrequency);
+            const distinctive = contentTokens.size > 0 ? contentTokens : tokens;
+            return new Set([...distinctive].map((t) => shapeToken(t)));
+        });
+        return [u.key, strictQuorumCoreSize(distinctiveShaped)];
+    }));
     /**
      * Applies a list of [absorbed, root] merges to `groups` and `keyToRoot`.
      * All absorbed groups' members are folded into their respective roots.
@@ -229,18 +453,21 @@ export function mergeCrossBlockClusters(units, options) {
                 ...rootG.landmarkInstances,
                 ...absorbedG.landmarkInstances,
             ];
+            const mergedPageIndices = [...rootG.pageIndices, ...absorbedG.pageIndices];
             // Only down-sample when the caller explicitly opts in (streaming
-            // path). Same-index sampling keeps memberTokenSets[i] and
-            // landmarkInstances[i] parallel.
+            // path). Same-index sampling keeps memberTokenSets[i],
+            // landmarkInstances[i], and pageIndices[i] parallel.
             if (capMembers !== undefined && mergedTokenSets.length > capMembers) {
                 const indices = mergedTokenSets.map((_, i) => i);
                 const kept = reservoirSample(indices, capMembers, root);
                 rootG.tokenSets = kept.map((i) => mergedTokenSets[i]);
                 rootG.landmarkInstances = kept.map((i) => mergedLandmarkInstances[i]);
+                rootG.pageIndices = kept.map((i) => mergedPageIndices[i]);
             }
             else {
                 rootG.tokenSets = mergedTokenSets;
                 rootG.landmarkInstances = mergedLandmarkInstances;
+                rootG.pageIndices = mergedPageIndices;
             }
             groups.delete(absorbed);
             for (const [origKey, cur] of keyToRoot) {
@@ -270,6 +497,18 @@ export function mergeCrossBlockClusters(units, options) {
             }
             groupDistinctive.set(key, dist);
         }
+        // Class-name-stripped projection of `groupDistinctive`, for
+        // {@link filterMergesByCohesion} only: the fine stage's own
+        // shape-Jaccard step (below) merges units whose *raw* tokens are
+        // disjoint by construction (same skeleton, different BEM class
+        // names — see `SHAPE_JACCARD_THRESHOLD`'s JSDoc), so a cohesion check
+        // against raw tokens would reject every shape-Jaccard merge outright.
+        // Shaping first lets the guard see that 'section.c-reports' and
+        // 'section.c-projects' both contribute to a shared 'section' token.
+        const groupDistinctiveShaped = new Map();
+        for (const [key, dist] of groupDistinctive) {
+            groupDistinctiveShaped.set(key, dist.map((tokens) => new Set([...tokens].map((t) => shapeToken(t)))));
+        }
         // Quorum core per group
         const cores = new Map();
         for (const key of groupKeys) {
@@ -367,8 +606,9 @@ export function mergeCrossBlockClusters(units, options) {
                 fineMerges.push([gk, rootKey]);
             }
         }
-        if (fineMerges.length > 0) {
-            applyMerges(fineMerges);
+        const acceptedFineMerges = filterMergesByCohesion(fineMerges, groupDistinctiveShaped, anchorByRoot);
+        if (acceptedFineMerges.length > 0) {
+            applyMerges(acceptedFineMerges);
             continue; // next round
         }
         // ---------------------------------------------------------------
@@ -393,6 +633,8 @@ export function mergeCrossBlockClusters(units, options) {
             }
             return shellCache.get(key) ?? new Set();
         };
+        if (!hasDiscriminatingL2Signatures(l2Keys, getL2Sig))
+            break;
         // Collect valid L2 containment pairs and apply via union-find
         // Direction: x is contained in y → x is absorbed by y
         // Multiple pairs can apply in one round if they form consistent groups
@@ -455,13 +697,18 @@ export function mergeCrossBlockClusters(units, options) {
                 l2Merges.push([gk, rootKey]);
             }
         }
-        if (l2Merges.length === 0)
-            break; // fully converged
-        applyMerges(l2Merges);
+        const acceptedL2Merges = filterMergesByCohesion(l2Merges, groupDistinctiveShaped, anchorByRoot);
+        if (acceptedL2Merges.length === 0)
+            break; // fully converged (or every proposal was rejected)
+        applyMerges(acceptedL2Merges);
     }
     const finalGroupsByRoot = new Map([...groups.entries()].map(([root, g]) => [
         root,
-        { tokenSets: g.tokenSets, landmarkInstances: g.landmarkInstances },
+        {
+            tokenSets: g.tokenSets,
+            landmarkInstances: g.landmarkInstances,
+            pageIndices: g.pageIndices,
+        },
     ]));
     return { rootByKey: keyToRoot, finalGroupsByRoot };
 }

package/dist/merge-validated-clusters.d.ts

@@ -0,0 +1,27 @@
+import type { CrossClusterDuplicate } from './find-cross-cluster-duplicates.js';
+/**
+ * Applies a set of confirmed cluster-pair merges to a `clusterKey` array,
+ * via union-find over the distinct cluster keys. Every `duplicates` entry is
+ * merged unconditionally — deciding *which* {@link CrossClusterDuplicate}s
+ * are trustworthy enough to act on (e.g. `similarity === 1` or
+ * `corroboratedByMirrorAxis`) is the caller's job, same separation of
+ * detection from action as
+ * {@link ./find-cross-cluster-duplicates.js | findCrossClusterDuplicates}
+ * itself.
+ *
+ * The surviving key for a merged group is its alphabetically smallest
+ * member — arbitrary but deterministic, so repeated calls on the same input
+ * produce the same output (mirrors the "lower index wins" rule
+ * {@link ./merge-cross-block-clusters.js | mergeCrossBlockClusters}'s own
+ * union-find already uses).
+ * @param clusterKeys Every page's current cluster key, in input order.
+ * @param duplicates Cluster-pair merges to apply.
+ * @example
+ * ```ts
+ * const merged = mergeValidatedClusters(
+ *   clusterKeys,
+ *   duplicates.filter((d) => d.similarity === 1 || d.corroboratedByMirrorAxis),
+ * );
+ * ```
+ */
+export declare function mergeValidatedClusters(clusterKeys: readonly string[], duplicates: readonly CrossClusterDuplicate[]): string[];