@d-zero/page-cluster 0.5.6 → 0.6.0

package/dist/resolve-page-cluster-keys.js

@@ -6,11 +6,13 @@ import { extractLandmarks } from './extract-landmarks.js';
 import { filterFirstPartyStylesheetHrefs } from './filter-first-party-stylesheet-hrefs.js';
 import { jaccardSimilarity } from './jaccard-similarity.js';
 import { mergeCrossBlockClusters } from './merge-cross-block-clusters.js';
+import { mergeValidatedClusters } from './merge-validated-clusters.js';
 import { groupIndicesByBlockKey, resolveBlockKeys } from './pass0-blocking.js';
 import { computePerPageLandmarkInstances } from './per-page-landmark-signatures.js';
 import { removeContentBlocks } from './remove-content-blocks.js';
 import { stageAPerBlock } from './stage-a-per-block.js';
 import { tokenize } from './tokenize.js';
+import { validateClusterPartition } from './validate-cluster-partition.js';
 /**
  * FNV-1a 32-bit hash of a string, used to seed the per-block PRNG so
  * reservoir sampling on the streaming path is deterministic for a given
@@ -257,6 +259,92 @@ function resolveBlockKeysForClustering(blockingPages, options, needReasons) {
     const result = resolveBlockKeys(blockingPages, { ...options, includeReasons: true });
     return { blockKeys: result.blockKeys, reasonsByBlockKey: result.reasonsByBlockKey };
 }
+/**
+ * Runs {@link ./validate-cluster-partition.js | validateClusterPartition}
+ * over Stage B's finished grouping and applies the built-in auto-merge
+ * policy (see `onPartitionReport`'s own JSDoc) directly to `finalKeys`,
+ * mutating it in place — the same in-place style the driver's own
+ * `rootByKey` rewrite loop already uses just before calling this.
+ *
+ * No-ops (returning `rootByKey`/`finalGroupsByRoot` unchanged) when
+ * `onPartitionReport` is `undefined` — the single gate every driver defers
+ * to, mirroring `emitClusterReasons`'s own gate for `onClusterReason`.
+ *
+ * When a merge is applied, `rootByKey` and `finalGroupsByRoot` are folded
+ * according to the same rename so `emitClusterReasons` (called with this
+ * function's return value, not the pre-merge originals) builds each
+ * survivor's `ClusterReason` from its *actual* post-merge membership —
+ * without this, a merged-away key's members would report `finalKeys[i]`
+ * pointing at the survivor while its own stale `ClusterReason.memberCount`
+ * still reflected only its pre-merge share.
+ * @param finalKeys Every page's current final cluster key, in input order.
+ *   Mutated in place when a merge is applied.
+ * @param rootByKey Stage B's own result (see
+ *   {@link ./merge-cross-block-clusters.js | MergeCrossBlockClustersResult}).
+ * @param finalGroupsByRoot Stage B's own result.
+ * @param getPageSignals Given an original page index, returns that page's
+ *   `paths`/`stylesheetHrefs` — `undefined` for an index the caller has no
+ *   record of (defensive; should not occur for indices Stage A/B actually
+ *   retained).
+ * @param onPartitionReport
+ */
+function validateAndMergePartition(finalKeys, rootByKey, finalGroupsByRoot, getPageSignals, onPartitionReport) {
+    if (!onPartitionReport)
+        return { rootByKey, finalGroupsByRoot };
+    const clusteredPages = [];
+    for (const [finalKey, group] of finalGroupsByRoot) {
+        for (const [i, tokens] of group.tokenSets.entries()) {
+            const pageIndex = group.pageIndices[i] ?? -1;
+            if (pageIndex === -1)
+                continue;
+            const signals = getPageSignals(pageIndex);
+            if (!signals)
+                continue;
+            clusteredPages.push({
+                clusterKey: finalKey,
+                tokens,
+                paths: signals.paths,
+                stylesheetHrefs: signals.stylesheetHrefs,
+            });
+        }
+    }
+    const report = validateClusterPartition(clusteredPages);
+    onPartitionReport(report);
+    // `findCrossClusterDuplicates` only ever returns entries that already
+    // satisfy this condition (its exact-match pass sets `similarity: 1`
+    // unconditionally; its near-match pass requires `corroboratedByMirrorAxis`
+    // to accept a pair at all) — this filter is a defensive invariant check,
+    // not something that currently narrows the result, kept so this call site
+    // keeps working correctly if that policy ever loosens.
+    const safeToMerge = report.crossClusterDuplicates.filter((d) => d.similarity === 1 || d.corroboratedByMirrorAxis);
+    if (safeToMerge.length === 0)
+        return { rootByKey, finalGroupsByRoot };
+    const mergedKeys = mergeValidatedClusters(finalKeys, safeToMerge);
+    const renameMap = new Map();
+    for (const [i, mergedKey] of mergedKeys.entries()) {
+        if (finalKeys[i] !== mergedKey)
+            renameMap.set(finalKeys[i], mergedKey);
+        finalKeys[i] = mergedKey;
+    }
+    const mergedFinalGroupsByRoot = new Map();
+    for (const [key, group] of finalGroupsByRoot) {
+        const target = renameMap.get(key) ?? key;
+        const existing = mergedFinalGroupsByRoot.get(target);
+        mergedFinalGroupsByRoot.set(target, {
+            tokenSets: [...(existing?.tokenSets ?? []), ...group.tokenSets],
+            landmarkInstances: [
+                ...(existing?.landmarkInstances ?? []),
+                ...group.landmarkInstances,
+            ],
+            pageIndices: [...(existing?.pageIndices ?? []), ...group.pageIndices],
+        });
+    }
+    const mergedRootByKey = new Map();
+    for (const [unitKey, stageBRoot] of rootByKey) {
+        mergedRootByKey.set(unitKey, renameMap.get(stageBRoot) ?? stageBRoot);
+    }
+    return { rootByKey: mergedRootByKey, finalGroupsByRoot: mergedFinalGroupsByRoot };
+}
 /**
  * Builds and emits one {@link ClusterReason} per final cluster via
  * `onClusterReason`, from data Stage A/B already computed for clustering
@@ -459,7 +547,8 @@ export function resolvePageClusterKeysInMemory(pages, options) {
             finalKeys[i] = rootKey;
         }
     }
-    emitClusterReasons(crossBlockUnits, rootByKey, finalGroupsByRoot, reasonsByBlockKey, siblingUnitKeysByBlock, onClusterReason);
+    const validated = validateAndMergePartition(finalKeys, rootByKey, finalGroupsByRoot, (i) => pages[i], options?.onPartitionReport);
+    emitClusterReasons(crossBlockUnits, validated.rootByKey, validated.finalGroupsByRoot, reasonsByBlockKey, siblingUnitKeysByBlock, onClusterReason);
     return finalKeys;
 }
 /**
@@ -557,7 +646,8 @@ async function resolveSmallCorpusWithProgress(pages, onProgress, options) {
             finalKeys[i] = rootKey;
         }
     }
-    emitClusterReasons(crossBlockUnits, rootByKey, finalGroupsByRoot, reasonsByBlockKey, siblingUnitKeysByBlock, onClusterReason);
+    const validated = validateAndMergePartition(finalKeys, rootByKey, finalGroupsByRoot, (i) => pages[i], options?.onPartitionReport);
+    emitClusterReasons(crossBlockUnits, validated.rootByKey, validated.finalGroupsByRoot, reasonsByBlockKey, siblingUnitKeysByBlock, onClusterReason);
     return finalKeys;
 }
 /**
@@ -844,7 +934,13 @@ export async function resolvePageClusterKeys(pages, options) {
             finalKeys[i] = rootKey;
         }
     }
-    emitClusterReasons(crossBlockUnits, rootByKey, finalGroupsByRoot, reasonsByBlockKey, siblingUnitKeysByBlock, onClusterReason);
+    // `blockingSignals` (built during Pass 0, held for the whole call) has an
+    // entry for every page in the corpus, sampled or not — only the pages
+    // `finalGroupsByRoot` actually kept a token set for (reservoir-sampled
+    // block representatives) end up in the report, per `onPartitionReport`'s
+    // own JSDoc.
+    const validated = validateAndMergePartition(finalKeys, rootByKey, finalGroupsByRoot, (i) => blockingSignals[i], options?.onPartitionReport);
+    emitClusterReasons(crossBlockUnits, validated.rootByKey, validated.finalGroupsByRoot, reasonsByBlockKey, siblingUnitKeysByBlock, onClusterReason);
     return finalKeys;
 }
 /**

package/dist/stage-a-per-block.js

@@ -172,6 +172,7 @@ export function stageAPerBlock(input, options) {
             key: unitKey,
             memberTokenSets: sampledPositions.map((pos) => blockTokenSets[pos]),
             memberLandmarkInstances: sampledPositions.map((pos) => memberLandmarkInstancesByPage[pos]),
+            memberPageIndices: sampledPositions.map((pos) => memberIndices[pos]),
         });
     }
     return { pageKeys, crossBlockUnits };

package/dist/validate-cluster-partition.d.ts

@@ -0,0 +1,70 @@
+import type { ClusterCohesion, ClusterCohesionOptions } from './compute-cluster-cohesion.js';
+import type { DetectMirrorAxisOptions, MirrorAxis } from './detect-mirror-axis.js';
+import type { ClusteredPage, CrossClusterDuplicate } from './find-cross-cluster-duplicates.js';
+/**
+ * Options for {@link validateClusterPartition}, forwarded to the three
+ * checks it composes.
+ */
+export type ValidateClusterPartitionOptions = {
+    readonly mirrorAxis?: DetectMirrorAxisOptions;
+    readonly cohesion?: ClusterCohesionOptions;
+    /**
+     * Only `corroboratedSimilarityThreshold` — `mirrorAxis` itself is always
+     * the axis this function just detected, not independently settable here.
+     */
+    readonly crossClusterDuplicates?: {
+        readonly corroboratedSimilarityThreshold?: number;
+    };
+};
+/**
+ * Structured evidence that a clustering run's partition may need
+ * correcting: which pages are plausibly the same template despite ending up
+ * in different clusters, and which clusters plausibly mix unrelated
+ * templates together. Carries no verdict on whether to act — same design as
+ * {@link ./build-cluster-reason.js | ClusterReason} — a caller decides what
+ * to do with `crossClusterDuplicates` (e.g. via
+ * {@link ./merge-validated-clusters.js | mergeValidatedClusters}) and with
+ * `cohesion`'s `suspicious` flags.
+ */
+export type ClusterPartitionReport = {
+    /**
+     * The mirror axis detected across every page's `paths`, or `null` if
+     * none was found — see
+     * {@link ./detect-mirror-axis.js | detectMirrorAxis}.
+     */
+    readonly mirrorAxis: MirrorAxis | null;
+    /** One entry per distinct cluster key present in `pages`. */
+    readonly cohesion: readonly ClusterCohesion[];
+    /** See {@link ./find-cross-cluster-duplicates.js | findCrossClusterDuplicates}. */
+    readonly crossClusterDuplicates: readonly CrossClusterDuplicate[];
+};
+/**
+ * Validates a finished clustering partition by checking it against itself,
+ * rather than trying to get the clustering right the first time: whether
+ * pages that ended up in different clusters are nonetheless structurally
+ * identical or near-identical (a likely over-split, see
+ * {@link ./find-cross-cluster-duplicates.js | findCrossClusterDuplicates}),
+ * and whether any single cluster's members actually agree with each other
+ * (a likely over-merge, see
+ * {@link ./compute-cluster-cohesion.js | computeClusterCohesion}). Neither
+ * check depends on *how* the partition was produced — this is deliberately
+ * decoupled from `resolvePageClusterKeys`'s own Stage A/B internals so it
+ * can validate a partition regardless of its origin, including a stored one
+ * loaded back from an archive.
+ *
+ * Takes already-tokenized pages rather than raw HTML — a caller with a
+ * `resolvePageClusterKeys` result already has token sets on hand (or can
+ * derive them via {@link ./tokenize.js | tokenize}), and re-tokenizing here
+ * would cost a second full corpus pass for no benefit.
+ * @param pages
+ * @param options
+ * @example
+ * ```ts
+ * const report = validateClusterPartition(pages);
+ * const safeToMerge = report.crossClusterDuplicates.filter(
+ *   (d) => d.similarity === 1 || d.corroboratedByMirrorAxis,
+ * );
+ * const mergedKeys = mergeValidatedClusters(clusterKeys, safeToMerge);
+ * ```
+ */
+export declare function validateClusterPartition(pages: readonly ClusteredPage[], options?: ValidateClusterPartitionOptions): ClusterPartitionReport;

package/dist/validate-cluster-partition.js

@@ -0,0 +1,49 @@
+import { computeClusterCohesion } from './compute-cluster-cohesion.js';
+import { detectMirrorAxis } from './detect-mirror-axis.js';
+import { findCrossClusterDuplicates } from './find-cross-cluster-duplicates.js';
+/**
+ * Validates a finished clustering partition by checking it against itself,
+ * rather than trying to get the clustering right the first time: whether
+ * pages that ended up in different clusters are nonetheless structurally
+ * identical or near-identical (a likely over-split, see
+ * {@link ./find-cross-cluster-duplicates.js | findCrossClusterDuplicates}),
+ * and whether any single cluster's members actually agree with each other
+ * (a likely over-merge, see
+ * {@link ./compute-cluster-cohesion.js | computeClusterCohesion}). Neither
+ * check depends on *how* the partition was produced — this is deliberately
+ * decoupled from `resolvePageClusterKeys`'s own Stage A/B internals so it
+ * can validate a partition regardless of its origin, including a stored one
+ * loaded back from an archive.
+ *
+ * Takes already-tokenized pages rather than raw HTML — a caller with a
+ * `resolvePageClusterKeys` result already has token sets on hand (or can
+ * derive them via {@link ./tokenize.js | tokenize}), and re-tokenizing here
+ * would cost a second full corpus pass for no benefit.
+ * @param pages
+ * @param options
+ * @example
+ * ```ts
+ * const report = validateClusterPartition(pages);
+ * const safeToMerge = report.crossClusterDuplicates.filter(
+ *   (d) => d.similarity === 1 || d.corroboratedByMirrorAxis,
+ * );
+ * const mergedKeys = mergeValidatedClusters(clusterKeys, safeToMerge);
+ * ```
+ */
+export function validateClusterPartition(pages, options) {
+    const mirrorAxis = detectMirrorAxis(pages.map((p) => p.paths), options?.mirrorAxis);
+    const membersByKey = new Map();
+    for (const page of pages) {
+        const members = membersByKey.get(page.clusterKey);
+        if (members)
+            members.push(page.tokens);
+        else
+            membersByKey.set(page.clusterKey, [page.tokens]);
+    }
+    const cohesion = computeClusterCohesion(membersByKey, options?.cohesion);
+    const crossClusterDuplicates = findCrossClusterDuplicates(pages, {
+        mirrorAxis,
+        corroboratedSimilarityThreshold: options?.crossClusterDuplicates?.corroboratedSimilarityThreshold,
+    });
+    return { mirrorAxis, cohesion, crossClusterDuplicates };
+}

package/package.json

@@ -1,12 +1,15 @@
 {
 	"name": "@d-zero/page-cluster",
-	"version": "0.5.6",
+	"version": "0.6.0",
 	"description": "Clusters crawled HTML pages by DOM-structure similarity — assigns the same key to pages sharing a template, ignoring text content. CLI-first, with library APIs.",
 	"author": "D-ZERO",
 	"license": "MIT",
 	"publishConfig": {
 		"access": "public"
 	},
+	"engines": {
+		"node": ">=24.11.0"
+	},
 	"type": "module",
 	"exports": {
 		".": {
@@ -36,6 +39,34 @@
 		"./build-cluster-reason": {
 			"import": "./dist/build-cluster-reason.js",
 			"types": "./dist/build-cluster-reason.d.ts"
+		},
+		"./detect-mirror-axis": {
+			"import": "./dist/detect-mirror-axis.js",
+			"types": "./dist/detect-mirror-axis.d.ts"
+		},
+		"./normalize-path-by-mirror-axis": {
+			"import": "./dist/normalize-path-by-mirror-axis.js",
+			"types": "./dist/normalize-path-by-mirror-axis.d.ts"
+		},
+		"./normalize-href-by-mirror-axis": {
+			"import": "./dist/normalize-href-by-mirror-axis.js",
+			"types": "./dist/normalize-href-by-mirror-axis.d.ts"
+		},
+		"./compute-cluster-cohesion": {
+			"import": "./dist/compute-cluster-cohesion.js",
+			"types": "./dist/compute-cluster-cohesion.d.ts"
+		},
+		"./find-cross-cluster-duplicates": {
+			"import": "./dist/find-cross-cluster-duplicates.js",
+			"types": "./dist/find-cross-cluster-duplicates.d.ts"
+		},
+		"./validate-cluster-partition": {
+			"import": "./dist/validate-cluster-partition.js",
+			"types": "./dist/validate-cluster-partition.d.ts"
+		},
+		"./merge-validated-clusters": {
+			"import": "./dist/merge-validated-clusters.js",
+			"types": "./dist/merge-validated-clusters.d.ts"
 		}
 	},
 	"bin": "dist/cli.js",
@@ -48,8 +79,9 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/dealer": "1.10.4",
-		"@d-zero/shared": "0.22.5",
+		"@d-zero/cli-core": "1.4.0",
+		"@d-zero/dealer": "1.11.0",
+		"@d-zero/shared": "0.23.0",
 		"htmlparser2": "12.0.0"
 	},
 	"repository": {
@@ -57,5 +89,5 @@
 		"url": "https://github.com/d-zero-dev/tools.git",
 		"directory": "packages/@d-zero/page-cluster"
 	},
-	"gitHead": "b06568f6812c4f560b4d8c3a1d280cd42ec264a5"
+	"gitHead": "d48a3ed9c01d32f0ca1a5b00e4abc36c376ce880"
 }