@d-zero/page-cluster 0.5.1 → 0.5.3

package/dist/resolve-page-cluster-keys.d.ts

@@ -165,10 +165,9 @@ export type ResolvePageClusterKeysOptions = TokenizeOptions & ResolveBlockingGro
      * sync helper to running a per-block async loop that emits
      * `pass1-block-complete` and `stage-b-start`. Omitting `onProgress`
      * keeps the small-corpus branch on the pre-refactor sync path with
-     * zero yield overhead. Ignored when `onClusterReason` is set — that
-     * option always routes the small-corpus branch through the sync
-     * helper (see `onClusterReason`'s own JSDoc), so no progress events
-     * fire in that combination.
+     * zero yield overhead. Independent of `onClusterReason` — both hooks
+     * can be set together and each fires on its own schedule (see
+     * `onClusterReason`'s own JSDoc).
      */
     onProgress?: (event: ProgressEvent) => void;
     /**
@@ -188,10 +187,11 @@ export type ResolvePageClusterKeysOptions = TokenizeOptions & ResolveBlockingGro
      * discovery. Omitting `onClusterReason` skips that bookkeeping
      * entirely, so existing callers pay nothing for this option.
      *
-     * On the async factory-based `resolvePageClusterKeys`, setting this
-     * forces the small-corpus branch through the sync
-     * `resolvePageClusterKeysInMemory` helper regardless of `onProgress`
-     * — see `onProgress`'s own JSDoc.
+     * Independent of `onProgress`: on the async factory-based
+     * `resolvePageClusterKeys`, the small-corpus branch still chooses
+     * between the sync and progress-emitting helpers based on `onProgress`
+     * alone, and both helpers derive `ClusterReason`s the same way when
+     * this option is set.
      * @example
      * ```ts
      * const reasons = new Map<string, ClusterReason>();

package/dist/resolve-page-cluster-keys.js

@@ -236,6 +236,27 @@ export function computeLocalChromeArtifacts(landmarks, tokenizeOptions) {
 export function computeLocalLandmarkTokens(landmarks, tokenizeOptions) {
     return [...computeLocalChromeArtifacts(landmarks, tokenizeOptions).localTokensByPage];
 }
+/**
+ * Resolves block keys for a clustering pass, deriving each block key's
+ * `BlockingReason` in the same call (via `resolveBlockKeys`'s
+ * `includeReasons: true` overload) whenever a caller wants `ClusterReason`s.
+ * All three clustering drivers below (in-memory, small-corpus-with-progress,
+ * streaming) share this exact branch so the "does this caller need reasons"
+ * decision can't drift between them.
+ * @param blockingPages
+ * @param options
+ * @param needReasons
+ */
+function resolveBlockKeysForClustering(blockingPages, options, needReasons) {
+    if (!needReasons) {
+        return {
+            blockKeys: resolveBlockKeys(blockingPages, options),
+            reasonsByBlockKey: undefined,
+        };
+    }
+    const result = resolveBlockKeys(blockingPages, { ...options, includeReasons: true });
+    return { blockKeys: result.blockKeys, reasonsByBlockKey: result.reasonsByBlockKey };
+}
 /**
  * Builds and emits one {@link ClusterReason} per final cluster via
  * `onClusterReason`, from data Stage A/B already computed for clustering
@@ -245,6 +266,12 @@ export function computeLocalLandmarkTokens(landmarks, tokenizeOptions) {
  * derived, and the per-block sibling-unit-key lists the driver accumulated
  * alongside its Stage A loop. No re-tokenization and no extra corpus pass —
  * this only re-groups references the driver already held.
+ *
+ * No-ops when any of `reasonsByBlockKey` / `siblingUnitKeysByBlock` /
+ * `onClusterReason` is `undefined` — the single gate for "was
+ * `onClusterReason` requested" that all three clustering drivers below defer
+ * to, instead of each repeating the same three-way null check before calling
+ * this function.
  * @param crossBlockUnits
  * @param rootByKey
  * @param finalGroupsByRoot
@@ -253,6 +280,8 @@ export function computeLocalLandmarkTokens(landmarks, tokenizeOptions) {
  * @param onClusterReason
  */
 function emitClusterReasons(crossBlockUnits, rootByKey, finalGroupsByRoot, reasonsByBlockKey, siblingUnitKeysByBlock, onClusterReason) {
+    if (!onClusterReason || !reasonsByBlockKey || !siblingUnitKeysByBlock)
+        return;
     const unitKeysByRoot = new Map();
     for (const unit of crossBlockUnits) {
         const root = rootByKey.get(unit.key) ?? unit.key;
@@ -395,16 +424,7 @@ export function resolvePageClusterKeysInMemory(pages, options) {
     // actually asked for `onClusterReason` — see that option's own JSDoc for
     // why this is the only place ClusterReason bookkeeping is opt-in.
     const onClusterReason = options?.onClusterReason;
-    let blockKeys;
-    let reasonsByBlockKey;
-    if (onClusterReason) {
-        const result = resolveBlockKeys(blockingPages, { ...options, includeReasons: true });
-        blockKeys = result.blockKeys;
-        reasonsByBlockKey = result.reasonsByBlockKey;
-    }
-    else {
-        blockKeys = resolveBlockKeys(blockingPages, options);
-    }
+    const { blockKeys, reasonsByBlockKey } = resolveBlockKeysForClustering(blockingPages, options, onClusterReason !== undefined);
     const indicesByBlockKey = groupIndicesByBlockKey(blockKeys);
     // Validated here, eagerly, because it's otherwise only reached from
     // inside the per-block loop below — which never runs at all for an empty
@@ -439,9 +459,7 @@ export function resolvePageClusterKeysInMemory(pages, options) {
             finalKeys[i] = rootKey;
         }
     }
-    if (onClusterReason && reasonsByBlockKey && siblingUnitKeysByBlock) {
-        emitClusterReasons(crossBlockUnits, rootByKey, finalGroupsByRoot, reasonsByBlockKey, siblingUnitKeysByBlock, onClusterReason);
-    }
+    emitClusterReasons(crossBlockUnits, rootByKey, finalGroupsByRoot, reasonsByBlockKey, siblingUnitKeysByBlock, onClusterReason);
     return finalKeys;
 }
 /**
@@ -457,7 +475,9 @@ export function resolvePageClusterKeysInMemory(pages, options) {
  * un-capped Stage B across the entire crossBlockUnits array. `finalKeys`
  * returned here must be byte-for-byte identical to what the sync path
  * would have produced for the same `pages` input — spec-enforced by
- * `resolve-page-cluster-keys-streaming.spec.ts`.
+ * `resolve-page-cluster-keys-streaming.spec.ts`. When `options.onClusterReason`
+ * is set, it derives and emits `ClusterReason`s the same way the sync path
+ * does, so the two hooks compose freely.
  *
  * The sync `resolvePageClusterKeysInMemory` is deliberately left in place
  * as its own implementation rather than being folded into a shared helper.
@@ -491,11 +511,17 @@ async function resolveSmallCorpusWithProgress(pages, onProgress, options) {
     const blockingPages = restrictStylesheetsToFirstParty
         ? filterFirstPartyStylesheetHrefs(pages)
         : pages;
-    const blockKeys = resolveBlockKeys(blockingPages, options);
+    // Reasons (blocking evidence) are only worth deriving when a caller
+    // actually asked for `onClusterReason` — same gate as the in-memory path.
+    const onClusterReason = options?.onClusterReason;
+    const { blockKeys, reasonsByBlockKey } = resolveBlockKeysForClustering(blockingPages, options, onClusterReason !== undefined);
     const indicesByBlockKey = groupIndicesByBlockKey(blockKeys);
     validateDetectContentDepthCapOptions(options);
     const finalKeys = Array.from({ length: pages.length });
     const crossBlockUnits = [];
+    const siblingUnitKeysByBlock = onClusterReason
+        ? new Map()
+        : undefined;
     const totalBlocks = indicesByBlockKey.size;
     let blocksProcessed = 0;
     for (const [blockKey, indices] of indicesByBlockKey) {
@@ -510,6 +536,7 @@ async function resolveSmallCorpusWithProgress(pages, onProgress, options) {
             finalKeys[pageIndex] = key;
         }
         crossBlockUnits.push(...result.crossBlockUnits);
+        siblingUnitKeysByBlock?.set(blockKey, result.crossBlockUnits.map((u) => u.key));
         blocksProcessed++;
         onProgress({
             phase: 'pass1-block-complete',
@@ -522,7 +549,7 @@ async function resolveSmallCorpusWithProgress(pages, onProgress, options) {
         await new Promise((resolve) => setImmediate(resolve));
     }
     onProgress({ phase: 'stage-b-start', unitCount: crossBlockUnits.length });
-    const { rootByKey } = mergeCrossBlockClusters(crossBlockUnits, options);
+    const { rootByKey, finalGroupsByRoot } = mergeCrossBlockClusters(crossBlockUnits, options);
     for (let i = 0; i < finalKeys.length; i++) {
         const currentKey = finalKeys[i];
         const rootKey = rootByKey.get(currentKey);
@@ -530,6 +557,7 @@ async function resolveSmallCorpusWithProgress(pages, onProgress, options) {
             finalKeys[i] = rootKey;
         }
     }
+    emitClusterReasons(crossBlockUnits, rootByKey, finalGroupsByRoot, reasonsByBlockKey, siblingUnitKeysByBlock, onClusterReason);
     return finalKeys;
 }
 /**
@@ -609,12 +637,11 @@ export async function resolvePageClusterKeys(pages, options) {
         // into per-block progress, so delegate to the untouched sync path —
         // keeping behavior byte-for-byte identical (and yield-overhead-free)
         // to how library-only consumers experienced this before the CLI
-        // progress work landed. `onClusterReason` always routes here too (see
-        // its own JSDoc): `resolveSmallCorpusWithProgress` has no
-        // cluster-reason support, and duplicating that logic into the
-        // progress-emitting path for a reporting feature that has nothing to
-        // do with progress observability isn't worth the added surface.
-        if (onProgress === undefined || options?.onClusterReason) {
+        // progress work landed. `onClusterReason` is independent of this
+        // choice: both the sync and progress-emitting paths derive
+        // ClusterReasons the same way, so passing it doesn't change which
+        // path handles a small corpus.
+        if (onProgress === undefined) {
             return resolvePageClusterKeysInMemory(fullPages, options);
         }
         return resolveSmallCorpusWithProgress(fullPages, onProgress, options);
@@ -634,19 +661,7 @@ export async function resolvePageClusterKeys(pages, options) {
     // keyed by distinct block key, not by page — but are only derived when a
     // caller actually asked for `onClusterReason`.
     const onClusterReason = options?.onClusterReason;
-    let blockKeys;
-    let reasonsByBlockKey;
-    if (onClusterReason) {
-        const result = resolveBlockKeys(blockingPagesForKeys, {
-            ...options,
-            includeReasons: true,
-        });
-        blockKeys = result.blockKeys;
-        reasonsByBlockKey = result.reasonsByBlockKey;
-    }
-    else {
-        blockKeys = resolveBlockKeys(blockingPagesForKeys, options);
-    }
+    const { blockKeys, reasonsByBlockKey } = resolveBlockKeysForClustering(blockingPagesForKeys, options, onClusterReason !== undefined);
     const indicesByBlockKey = groupIndicesByBlockKey(blockKeys);
     const finalKeys = Array.from({ length: blockingSignals.length });
     const crossBlockUnits = [];
@@ -829,9 +844,7 @@ export async function resolvePageClusterKeys(pages, options) {
             finalKeys[i] = rootKey;
         }
     }
-    if (onClusterReason && reasonsByBlockKey && siblingUnitKeysByBlock) {
-        emitClusterReasons(crossBlockUnits, rootByKey, finalGroupsByRoot, reasonsByBlockKey, siblingUnitKeysByBlock, onClusterReason);
-    }
+    emitClusterReasons(crossBlockUnits, rootByKey, finalGroupsByRoot, reasonsByBlockKey, siblingUnitKeysByBlock, onClusterReason);
     return finalKeys;
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@d-zero/page-cluster",
-	"version": "0.5.1",
+	"version": "0.5.3",
 	"description": "Clusters crawled HTML pages by DOM-structure similarity — assigns the same key to pages sharing a template, ignoring text content. CLI-first, with library APIs.",
 	"author": "D-ZERO",
 	"license": "MIT",
@@ -32,6 +32,10 @@
 		"./jaccard-similarity": {
 			"import": "./dist/jaccard-similarity.js",
 			"types": "./dist/jaccard-similarity.d.ts"
+		},
+		"./build-cluster-reason": {
+			"import": "./dist/build-cluster-reason.js",
+			"types": "./dist/build-cluster-reason.d.ts"
 		}
 	},
 	"bin": "./dist/cli.js",
@@ -44,8 +48,8 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/dealer": "1.10.1",
-		"@d-zero/shared": "0.22.2",
+		"@d-zero/dealer": "1.10.2",
+		"@d-zero/shared": "0.22.3",
 		"htmlparser2": "12.0.0"
 	},
 	"repository": {
@@ -53,5 +57,5 @@
 		"url": "https://github.com/d-zero-dev/tools.git",
 		"directory": "packages/@d-zero/page-cluster"
 	},
-	"gitHead": "f3dac7605cb4f94b640770f7ca2513de9a556e50"
+	"gitHead": "7be4609bd7a2ca80a34d87cde2ade18191c952f9"
 }