@d-zero/page-cluster 0.5.6 → 0.6.0

package/dist/compute-cluster-cohesion.js

@@ -0,0 +1,107 @@
+import { jaccardSimilarity } from './jaccard-similarity.js';
+import { reservoirSample } from './reservoir-sample.js';
+/**
+ * Maximum members sampled per cluster before computing pairwise similarity.
+ * Bounds the cost to `O(maxSampleSize²)` per cluster regardless of how large
+ * the cluster actually is — 40² / 2 = 780 comparisons, cheap even across
+ * many clusters.
+ */
+const DEFAULT_MAX_SAMPLE_SIZE = 40;
+/**
+ * Below this median pairwise-similarity, a cluster is flagged `suspicious`.
+ *
+ * Chosen against the bundled `buildMirroredTemplateFixture` fixture (see
+ * `compute-cluster-cohesion.spec.ts`): every single-template cluster there
+ * has a median of `1.0` (structural tokens are identical across pages that
+ * differ only in text content and per-mirror stylesheet href), while every
+ * cluster built by mixing two *different* templates' members has a median
+ * at or below `0.70`. `0.75` sits with margin on both sides of that gap.
+ * Real crawl data can have single-template clusters with genuinely lower
+ * cohesion than this synthetic fixture models (legitimate content-driven
+ * structural variation the fixture doesn't produce) — `suspiciousMedianBelow`
+ * exists so a caller who has measured their own corpus can override it.
+ */
+const DEFAULT_SUSPICIOUS_MEDIAN_BELOW = 0.75;
+/**
+ * @param sortedAscending
+ */
+function median(sortedAscending) {
+    const mid = Math.floor(sortedAscending.length / 2);
+    if (sortedAscending.length % 2 === 1)
+        return sortedAscending[mid];
+    return (sortedAscending[mid - 1] + sortedAscending[mid]) / 2;
+}
+/**
+ * @param sortedAscending
+ */
+function p10(sortedAscending) {
+    const index = Math.floor(0.1 * (sortedAscending.length - 1));
+    return sortedAscending[index];
+}
+/**
+ * Reports, per cluster, how similar its members' structural token sets
+ * actually are to each other — not just which tokens they share (that's
+ * {@link ./merge-cross-block-clusters.js | computeQuorumCore}'s job), but
+ * whether the members hang together at all. A cluster built by merging
+ * unrelated templates has member pairs that mostly disagree even though a
+ * small frequency core still exists among them; this surfaces that
+ * disagreement directly, as a distribution rather than a single score, since
+ * a single "average similarity" would be pulled toward the middle by exactly
+ * the kind of partial-overlap noise this is meant to catch.
+ *
+ * Sampling uses {@link ./reservoir-sample.js | reservoirSample} seeded by
+ * each cluster's own key, so repeated calls on the same partition sample the
+ * same members and produce the same report.
+ *
+ * Deliberately returns only structured numbers, no verdict text — same
+ * design as {@link ./build-cluster-reason.js | ClusterReason} — so a caller
+ * decides what "suspicious" should mean for their own use (an interactive
+ * review queue vs. an automated gate might want different behavior for the
+ * same numbers).
+ * @param membersByKey Every cluster's member token sets, keyed by cluster key.
+ * @param options
+ * @example
+ * ```ts
+ * const report = computeClusterCohesion(membersByKey);
+ * const worstFirst = [...report].filter((r) => r.suspicious)
+ *   .toSorted((a, b) => (a.medianPairSimilarity ?? 0) - (b.medianPairSimilarity ?? 0));
+ * ```
+ */
+export function computeClusterCohesion(membersByKey, options) {
+    const maxSampleSize = options?.maxSampleSize ?? DEFAULT_MAX_SAMPLE_SIZE;
+    const suspiciousMedianBelow = options?.suspiciousMedianBelow ?? DEFAULT_SUSPICIOUS_MEDIAN_BELOW;
+    const results = [];
+    for (const [clusterKey, members] of membersByKey) {
+        const sample = reservoirSample(members, maxSampleSize, clusterKey);
+        if (sample.length < 2) {
+            results.push({
+                clusterKey,
+                memberCount: members.length,
+                sampledMemberCount: sample.length,
+                medianPairSimilarity: null,
+                p10PairSimilarity: null,
+                minPairSimilarity: null,
+                suspicious: false,
+            });
+            continue;
+        }
+        const similarities = [];
+        for (let a = 0; a < sample.length; a++) {
+            for (let b = a + 1; b < sample.length; b++) {
+                similarities.push(jaccardSimilarity(sample[a], sample[b]));
+            }
+        }
+        similarities.sort((x, y) => x - y);
+        const medianSimilarity = median(similarities);
+        results.push({
+            clusterKey,
+            memberCount: members.length,
+            sampledMemberCount: sample.length,
+            medianPairSimilarity: medianSimilarity,
+            p10PairSimilarity: p10(similarities),
+            minPairSimilarity: similarities[0],
+            suspicious: medianSimilarity < suspiciousMedianBelow,
+        });
+    }
+    return results;
+}

package/dist/detect-mirror-axis.d.ts

@@ -0,0 +1,74 @@
+/**
+ * A URL path segment position at which a whole site section is mirrored
+ * under a fixed set of alternative values — most commonly a language
+ * directory (`/en/`, `/zh/`, `/ko/`, `/th/`), but the same shape also arises
+ * from staging/production mirrors, device-variant subsites (`/sp/`), or
+ * versioned documentation trees. Detected by {@link detectMirrorAxis} from
+ * `paths` alone, with no built-in list of language codes or other
+ * site-specific vocabulary.
+ */
+export type MirrorAxis = {
+    /** Index into a page's `paths` array where the mirrored value sits. */
+    readonly position: number;
+    /** The alternative values observed at `position` across the mirror. */
+    readonly values: ReadonlySet<string>;
+};
+/**
+ * Options for {@link detectMirrorAxis}.
+ */
+export type DetectMirrorAxisOptions = {
+    /**
+     * How many leading path segments to scan for a mirror axis. Bounds the
+     * cost to `O(maxPosition × pageCount)`; a mirror axis deep enough to need
+     * a larger value is unusual (language/environment/version directories
+     * are conventionally near the root).
+     */
+    readonly maxPosition?: number;
+    /**
+     * Minimum number of distinct path skeletons that must share the exact
+     * same value set at a position before it is even considered a candidate
+     * axis. A single skeleton with multiple values at some position (e.g. a
+     * `/faq/{01,02,03}` set of sibling pages) is not a mirror — nothing about
+     * it repeats — so admitting `skeletonCount === 1` candidates would
+     * misidentify ordinary sibling pages as a site-wide axis. 3 is the
+     * smallest count for which "this exact value set recurs" stops being
+     * describable as coincidence.
+     */
+    readonly minSkeletonCount?: number;
+};
+/**
+ * Finds a mirror axis in a page corpus's URL paths, with no built-in
+ * knowledge of language codes or any other site-specific vocabulary — the
+ * axis is inferred purely from how often the exact same set of alternative
+ * values recurs across otherwise-identical path skeletons.
+ *
+ * For each scanned segment position, every page's path is reduced to a
+ * "skeleton" (that position blanked to `*`); skeletons that recur with
+ * `≥ 2` distinct values at that position are candidates, keyed by their
+ * exact value set. Candidates are ranked by how many distinct skeletons
+ * share that value set — the more independent path shapes that recur under
+ * the same alternative values, the more likely that position is a genuine
+ * site-wide mirror rather than a coincidence — and the top-ranked position's
+ * value set is accepted unless {@link autoCutThreshold}'s max-gap cut (using
+ * the top candidate's own count as `upperBound`, so the cut never selects a
+ * value the top candidate doesn't already clear) finds the next-best
+ * candidate too close behind to call decisively.
+ *
+ * Returns `null` when no position has any candidate clearing
+ * `minSkeletonCount` — including a single-language/single-mirror corpus
+ * (nothing recurs under alternative values at all) and a corpus whose only
+ * repeated-value-set position is a single skeleton's sibling pages (e.g.
+ * `/faq/{01,02,03}` — excluded by `minSkeletonCount`, see its own JSDoc).
+ * @param pagePaths Every page's URL path segments (e.g. `ExURL.paths`).
+ * @param options
+ * @example
+ * ```ts
+ * const axis = detectMirrorAxis([
+ *   ['en', 'faq', 'index.html'], ['zh', 'faq', 'index.html'],
+ *   ['en', 'access', 'index.html'], ['zh', 'access', 'index.html'],
+ *   ['en', 'gallery', 'index.html'], ['zh', 'gallery', 'index.html'],
+ * ]);
+ * // { position: 0, values: Set(['en', 'zh']) }
+ * ```
+ */
+export declare function detectMirrorAxis(pagePaths: readonly (readonly string[])[], options?: DetectMirrorAxisOptions): MirrorAxis | null;

package/dist/find-cross-cluster-duplicates.d.ts

@@ -0,0 +1,94 @@
+import type { MirrorAxis } from './detect-mirror-axis.js';
+/**
+ * A page as {@link findCrossClusterDuplicates} and
+ * {@link ./validate-cluster-partition.js | validateClusterPartition} need
+ * it: already tokenized (no re-tokenization here — a caller validating a
+ * result from {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}
+ * already has these token sets on hand) and carrying the blocking-relevant
+ * signals ({@link ./resolve-page-cluster-keys.js | PageClusterSignals}'s
+ * `paths`/`stylesheetHrefs`) needed to corroborate a match against a
+ * {@link MirrorAxis}.
+ */
+export type ClusteredPage = {
+    readonly clusterKey: string;
+    readonly tokens: ReadonlySet<string>;
+    readonly paths: readonly string[];
+    readonly stylesheetHrefs: readonly string[];
+};
+/**
+ * Options for {@link findCrossClusterDuplicates}.
+ */
+export type FindCrossClusterDuplicatesOptions = {
+    /**
+     * A `MirrorAxis` to corroborate near-duplicate (similarity `< 1`) pairs
+     * against. Omitting it (or passing `null`) still finds exact-duplicate
+     * pairs — see {@link findCrossClusterDuplicates}'s JSDoc — just without
+     * the near-duplicate pass, and every result's `corroboratedByMirrorAxis`
+     * is `false` (corroboration was never attempted, not disproven).
+     */
+    readonly mirrorAxis?: MirrorAxis | null;
+    /** @see DEFAULT_CORROBORATED_SIMILARITY_THRESHOLD */
+    readonly corroboratedSimilarityThreshold?: number;
+};
+/**
+ * One pair of clusters found to plausibly be the same template, split by
+ * the clustering run under review.
+ */
+export type CrossClusterDuplicate = {
+    /** The two cluster keys, ordered so `clusterKeyA < clusterKeyB` (stable regardless of input order). */
+    readonly clusterKeyA: string;
+    readonly clusterKeyB: string;
+    /** The highest pairwise token-set Jaccard similarity found between the two clusters' members. */
+    readonly similarity: number;
+    /**
+     * Whether the best-similarity pair was also corroborated by the mirror
+     * axis (same axis-normalized path shape and axis-normalized stylesheet
+     * href set). Always `false` when no axis was supplied. `similarity === 1`
+     * pairs are reported regardless of this flag — see this function's JSDoc.
+     */
+    readonly corroboratedByMirrorAxis: boolean;
+};
+/**
+ * Finds pairs of clusters whose members are plausibly the same template,
+ * split apart by the clustering run under review — the signal a caller acts
+ * on to merge them back (see {@link ./merge-validated-clusters.js |
+ * mergeValidatedClusters}).
+ *
+ * Two passes, both bounded well below the full `O(pageCount²)` cross
+ * product:
+ *
+ * 1. **Exact match** — pages are grouped by an exact hash of their token
+ *    set (`O(pageCount)`). Any hash group spanning more than one cluster key
+ *    is an instant duplicate at `similarity: 1`: two pages whose structural
+ *    tokens are byte-identical being in different clusters is a partition
+ *    error regardless of *why* — no corroboration is required or checked
+ *    for acceptance (an axis check still runs on the pair, if one was
+ *    supplied, purely to populate `corroboratedByMirrorAxis` informationally).
+ *    Pages with an empty token set are excluded — no structural evidence to
+ *    match on.
+ * 2. **Axis-corroborated near match** (only when `options.mirrorAxis` is
+ *    given) — pages are grouped by
+ *    {@link ./normalize-path-by-mirror-axis.js | normalizePathByMirrorAxis}
+ *    shape, which is naturally small per group (bounded by how many mirror
+ *    values recur under that shape, not by corpus size). Within a shape
+ *    group spanning more than one cluster key, every cross-cluster pair is
+ *    compared directly; a pair clearing `corroboratedSimilarityThreshold`
+ *    **and** matching under
+ *    {@link ./normalize-href-by-mirror-axis.js | normalizeHrefByMirrorAxis}
+ *    is accepted. Both signals are required here — same path shape alone
+ *    can recur by coincidence (two different templates that happen to sit at
+ *    the same depth), and same stylesheet shape alone doesn't imply the same
+ *    DOM structure.
+ *
+ * Results are deduplicated to one entry per cluster-key pair, keeping the
+ * highest similarity found and OR-ing the corroboration flag across every
+ * qualifying page pair for that cluster pair.
+ * @param pages
+ * @param options
+ * @example
+ * ```ts
+ * const duplicates = findCrossClusterDuplicates(pages, { mirrorAxis: axis });
+ * const safeToMerge = duplicates.filter((d) => d.similarity === 1 || d.corroboratedByMirrorAxis);
+ * ```
+ */
+export declare function findCrossClusterDuplicates(pages: readonly ClusteredPage[], options?: FindCrossClusterDuplicatesOptions): readonly CrossClusterDuplicate[];

package/dist/find-cross-cluster-duplicates.js

@@ -0,0 +1,164 @@
+import { jaccardSimilarity } from './jaccard-similarity.js';
+import { normalizeHrefByMirrorAxis } from './normalize-href-by-mirror-axis.js';
+import { normalizePathByMirrorAxis } from './normalize-path-by-mirror-axis.js';
+/**
+ * Similarity required for a page pair below exact match to be accepted —
+ * only once axis+href corroboration also holds (see
+ * {@link findCrossClusterDuplicates}'s JSDoc). Matches the 80% floor already
+ * used throughout this package's Stage B for corroborating a secondary
+ * signal ({@link ./merge-cross-block-clusters.js | CROSS_BLOCK_THRESHOLD},
+ * `SHELL_CORROBORATION_THRESHOLD`) rather than carrying a decision alone.
+ */
+const DEFAULT_CORROBORATED_SIMILARITY_THRESHOLD = 0.8;
+/**
+ * @param tokens
+ */
+function exactHashKey(tokens) {
+    return [...tokens].toSorted().join(' ');
+}
+/**
+ * @param a
+ * @param b
+ * @param axis
+ */
+function hrefsMatchUnderAxis(a, b, axis) {
+    const normalize = (hrefs) => [...new Set(hrefs.map((h) => normalizeHrefByMirrorAxis(h, axis)))]
+        .toSorted()
+        .join(' ');
+    return normalize(a.stylesheetHrefs) === normalize(b.stylesheetHrefs);
+}
+/**
+ * @param best
+ * @param a
+ * @param b
+ * @param similarity
+ * @param axis
+ */
+function recordCandidate(best, a, b, similarity, axis) {
+    if (a.clusterKey === b.clusterKey)
+        return;
+    const [keyA, keyB] = a.clusterKey < b.clusterKey
+        ? [a.clusterKey, b.clusterKey]
+        : [b.clusterKey, a.clusterKey];
+    const corroborated = axis !== null && hrefsMatchUnderAxis(a, b, axis);
+    let inner = best.get(keyA);
+    if (!inner) {
+        inner = new Map();
+        best.set(keyA, inner);
+    }
+    const existing = inner.get(keyB);
+    if (!existing || similarity > existing.similarity) {
+        inner.set(keyB, { similarity, corroboratedByMirrorAxis: corroborated });
+    }
+    else if (similarity === existing.similarity &&
+        corroborated &&
+        !existing.corroboratedByMirrorAxis) {
+        inner.set(keyB, { similarity, corroboratedByMirrorAxis: true });
+    }
+}
+/**
+ * Finds pairs of clusters whose members are plausibly the same template,
+ * split apart by the clustering run under review — the signal a caller acts
+ * on to merge them back (see {@link ./merge-validated-clusters.js |
+ * mergeValidatedClusters}).
+ *
+ * Two passes, both bounded well below the full `O(pageCount²)` cross
+ * product:
+ *
+ * 1. **Exact match** — pages are grouped by an exact hash of their token
+ *    set (`O(pageCount)`). Any hash group spanning more than one cluster key
+ *    is an instant duplicate at `similarity: 1`: two pages whose structural
+ *    tokens are byte-identical being in different clusters is a partition
+ *    error regardless of *why* — no corroboration is required or checked
+ *    for acceptance (an axis check still runs on the pair, if one was
+ *    supplied, purely to populate `corroboratedByMirrorAxis` informationally).
+ *    Pages with an empty token set are excluded — no structural evidence to
+ *    match on.
+ * 2. **Axis-corroborated near match** (only when `options.mirrorAxis` is
+ *    given) — pages are grouped by
+ *    {@link ./normalize-path-by-mirror-axis.js | normalizePathByMirrorAxis}
+ *    shape, which is naturally small per group (bounded by how many mirror
+ *    values recur under that shape, not by corpus size). Within a shape
+ *    group spanning more than one cluster key, every cross-cluster pair is
+ *    compared directly; a pair clearing `corroboratedSimilarityThreshold`
+ *    **and** matching under
+ *    {@link ./normalize-href-by-mirror-axis.js | normalizeHrefByMirrorAxis}
+ *    is accepted. Both signals are required here — same path shape alone
+ *    can recur by coincidence (two different templates that happen to sit at
+ *    the same depth), and same stylesheet shape alone doesn't imply the same
+ *    DOM structure.
+ *
+ * Results are deduplicated to one entry per cluster-key pair, keeping the
+ * highest similarity found and OR-ing the corroboration flag across every
+ * qualifying page pair for that cluster pair.
+ * @param pages
+ * @param options
+ * @example
+ * ```ts
+ * const duplicates = findCrossClusterDuplicates(pages, { mirrorAxis: axis });
+ * const safeToMerge = duplicates.filter((d) => d.similarity === 1 || d.corroboratedByMirrorAxis);
+ * ```
+ */
+export function findCrossClusterDuplicates(pages, options) {
+    const axis = options?.mirrorAxis ?? null;
+    const threshold = options?.corroboratedSimilarityThreshold ?? DEFAULT_CORROBORATED_SIMILARITY_THRESHOLD;
+    const best = new Map();
+    // Pass 1: exact match via hash grouping.
+    const byExactHash = new Map();
+    for (const page of pages) {
+        if (page.tokens.size === 0)
+            continue;
+        const key = exactHashKey(page.tokens);
+        const group = byExactHash.get(key);
+        if (group)
+            group.push(page);
+        else
+            byExactHash.set(key, [page]);
+    }
+    for (const group of byExactHash.values()) {
+        if (new Set(group.map((p) => p.clusterKey)).size < 2)
+            continue;
+        for (let a = 0; a < group.length; a++) {
+            for (let b = a + 1; b < group.length; b++) {
+                recordCandidate(best, group[a], group[b], 1, axis);
+            }
+        }
+    }
+    // Pass 2: axis-corroborated near match, scoped to same axis-normalized shape.
+    if (axis !== null) {
+        const byShape = new Map();
+        for (const page of pages) {
+            const shape = normalizePathByMirrorAxis(page.paths, axis);
+            const group = byShape.get(shape);
+            if (group)
+                group.push(page);
+            else
+                byShape.set(shape, [page]);
+        }
+        for (const group of byShape.values()) {
+            if (new Set(group.map((p) => p.clusterKey)).size < 2)
+                continue;
+            for (let a = 0; a < group.length; a++) {
+                for (let b = a + 1; b < group.length; b++) {
+                    const pageA = group[a];
+                    const pageB = group[b];
+                    if (pageA.clusterKey === pageB.clusterKey)
+                        continue;
+                    const similarity = jaccardSimilarity(pageA.tokens, pageB.tokens);
+                    if (similarity < threshold)
+                        continue;
+                    if (!hrefsMatchUnderAxis(pageA, pageB, axis))
+                        continue;
+                    recordCandidate(best, pageA, pageB, similarity, axis);
+                }
+            }
+        }
+    }
+    const results = [];
+    for (const [clusterKeyA, inner] of best) {
+        for (const [clusterKeyB, { similarity, corroboratedByMirrorAxis }] of inner) {
+            results.push({ clusterKeyA, clusterKeyB, similarity, corroboratedByMirrorAxis });
+        }
+    }
+    return results;
+}

package/dist/merge-cross-block-clusters.d.ts

@@ -17,6 +17,18 @@ export type CrossBlockUnit = {
     readonly key: string;
     readonly memberTokenSets: readonly ReadonlySet<string>[];
     readonly memberLandmarkInstances: readonly (readonly PerPageLandmarkInstance[])[];
+    /**
+     * Original corpus index of each member, parallel to `memberTokenSets` —
+     * lets a caller that pools `finalGroupsByRoot`'s output back into
+     * page-level records (e.g. to build a
+     * {@link ./find-cross-cluster-duplicates.js | ClusteredPage}) recover
+     * which page each merged token set came from, without a second
+     * corpus-wide pass. Optional because most callers (every hand-built test
+     * fixture in this file included) only need `mergeCrossBlockClusters` for
+     * its cluster-key decisions and never read this back; omitted entries are
+     * normalized to `-1` internally rather than left misaligned.
+     */
+    readonly memberPageIndices?: readonly number[];
 };
 /**
  * Frequency-based token core of a group of member pages: a token must be
@@ -42,6 +54,12 @@ export declare function computeQuorumCore(memberDistinctiveTokens: readonly Read
 export type FinalGroupMembers = {
     readonly tokenSets: readonly ReadonlySet<string>[];
     readonly landmarkInstances: readonly (readonly PerPageLandmarkInstance[])[];
+    /**
+     * Parallel to `tokenSets` — see {@link CrossBlockUnit}'s own
+     * `memberPageIndices`. An entry is `-1` for any member whose originating
+     * unit omitted `memberPageIndices`.
+     */
+    readonly pageIndices: readonly number[];
 };
 /**
  * `mergeCrossBlockClusters`'s result: the root-key mapping every caller