npm - @d-zero/page-cluster - Versions diffs - 0.3.1 → 0.5.0 - Mend

@d-zero/page-cluster 0.3.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (28) hide show

package/README.md +45 -8
package/dist/build-cluster-reason.d.ts +119 -0
package/dist/build-cluster-reason.js +63 -0
package/dist/cli.d.ts +1 -0
package/dist/cli.js +64 -6
package/dist/derive-blocking-reason.d.ts +32 -0
package/dist/derive-blocking-reason.js +1 -0
package/dist/extract-landmarks.d.ts +75 -14
package/dist/extract-landmarks.js +64 -4
package/dist/is-chrome-landmark-instance.d.ts +41 -0
package/dist/is-chrome-landmark-instance.js +50 -0
package/dist/merge-cross-block-clusters.d.ts +39 -3
package/dist/merge-cross-block-clusters.js +28 -107
package/dist/offset-to-line-column.d.ts +38 -0
package/dist/offset-to-line-column.js +49 -0
package/dist/pass0-blocking.d.ts +10 -0
package/dist/pass0-blocking.js +26 -46
package/dist/per-page-landmark-signatures.d.ts +9 -1
package/dist/per-page-landmark-signatures.js +16 -4
package/dist/reassign-orphan-block-keys.d.ts +9 -0
package/dist/reassign-orphan-block-keys.js +5 -2
package/dist/resolve-blocking-group-keys.d.ts +10 -0
package/dist/resolve-blocking-group-keys.js +16 -86
package/dist/resolve-page-cluster-keys.d.ts +37 -1
package/dist/resolve-page-cluster-keys.js +123 -17
package/dist/shell-quorum.d.ts +70 -0
package/dist/shell-quorum.js +110 -0
package/package.json +10 -2

package/dist/per-page-landmark-signatures.d.ts CHANGED Viewed

@@ -1,4 +1,4 @@
-import type { ExtractLandmarksResult, LandmarkType } from './extract-landmarks.js';
+import type { ExtractLandmarksResult, LandmarkPosition, LandmarkType } from './extract-landmarks.js';
 import type { TokenizeOptions } from './types.js';
 /**
  * Every landmark type extractLandmarks may populate, iterated in a fixed
@@ -13,11 +13,19 @@ export declare const ALL_LANDMARK_TYPES: readonly LandmarkType[];
  * {@link ./canonicalize-token-set.js | canonicalizeTokenSet}). Signatures
  * are reused across consumers so two callers see the same "same instance"
  * verdict without independently re-canonicalizing.
+ *
+ * `position` is the instance's location in the page it came from, computed
+ * once by {@link ./extract-landmarks.js | extractLandmarks} and carried here
+ * unchanged — a back-reference for callers that need to report where a
+ * chrome-classified instance actually sits, not just that it exists. It
+ * plays no part in `tokens`/`signature` computation or in the corpus-
+ * frequency logic that consumes this type.
  */
 export type PerPageLandmarkInstance = {
     readonly type: LandmarkType;
     readonly tokens: ReadonlySet<string>;
     readonly signature: string;
+    readonly position: LandmarkPosition;
 };
 /**
  * Tokenizes every landmark instance across every page once, keyed by page

package/dist/per-page-landmark-signatures.js CHANGED Viewed

@@ -44,17 +44,29 @@ export function computePerPageLandmarkInstances(landmarks, tokenizeOptions) {
         const seenSignatures = new Set();
         const out = [];
         for (const type of ALL_LANDMARK_TYPES) {
-            for (const instanceHtml of entry[type]) {
-                if (!instanceHtml)
+            for (const instance of entry[type]) {
+                if (!instance.html)
                     continue;
-                const tokens = new Set(tokenize(`<body>${instanceHtml}</body>`, tokenizeOptions).tokens);
+                const tokens = new Set(tokenize(`<body>${instance.html}</body>`, tokenizeOptions).tokens);
                 if (tokens.size === 0)
                     continue;
                 const signature = canonicalizeTokenSet(tokens);
                 if (seenSignatures.has(signature))
                     continue;
                 seenSignatures.add(signature);
-                out.push({ type, tokens, signature });
+                out.push({
+                    type,
+                    tokens,
+                    signature,
+                    position: {
+                        startOffset: instance.startOffset,
+                        endOffset: instance.endOffset,
+                        startLine: instance.startLine,
+                        startColumn: instance.startColumn,
+                        endLine: instance.endLine,
+                        endColumn: instance.endColumn,
+                    },
+                });
             }
         }
         return out;

package/dist/reassign-orphan-block-keys.d.ts CHANGED Viewed

@@ -1,4 +1,13 @@
 import type { PageBlockingSignals } from './resolve-blocking-group-keys.js';
+/**
+ * Prefix distinguishing a reassigned key from the `css:`/`path:` keys
+ * {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys} itself
+ * produces, so the two families can never collide. Exported so
+ * {@link ./pass0-blocking.js | resolveBlockKeys} can recover the confined
+ * path key back out of a reassigned block key when building `BlockingReason`s,
+ * without duplicating this literal.
+ */
+export declare const REASSIGNED_KEY_PREFIX = "orphan-merge:";
 /**
  * Rewrites the `path:`-fallback key of an "orphan" page — one with no
  * stylesheet references recorded at all — to match a same-URL-section `css:`

package/dist/reassign-orphan-block-keys.js CHANGED Viewed

@@ -2,9 +2,12 @@ import { derivePathGroupKey } from './derive-path-group-key.js';
 /**
  * Prefix distinguishing a reassigned key from the `css:`/`path:` keys
  * {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys} itself
- * produces, so the two families can never collide.
+ * produces, so the two families can never collide. Exported so
+ * {@link ./pass0-blocking.js | resolveBlockKeys} can recover the confined
+ * path key back out of a reassigned block key when building `BlockingReason`s,
+ * without duplicating this literal.
  */
-const REASSIGNED_KEY_PREFIX = 'orphan-merge:';
+export const REASSIGNED_KEY_PREFIX = 'orphan-merge:';
 /**
  * Reads `values[index]`, throwing instead of returning `undefined`. Every
  * call site here indexes `pages`/`pathKeys`/`blockKeys` with a position

package/dist/resolve-blocking-group-keys.d.ts CHANGED Viewed

@@ -1,3 +1,4 @@
+import type { BlockingReason } from './derive-blocking-reason.js';
 /**
  * The two blocking signals {@link ./derive-path-group-key.js | derivePathGroupKey}
  * and {@link ./derive-stylesheet-group-key.js | deriveStylesheetGroupKey} need,
@@ -36,6 +37,12 @@ export type ResolveBlockingGroupKeysOptions = {
     /** Forwarded to `splitTokensByFrequency` as-is. */
     hrefCommonThreshold?: number;
 };
+/** Return shape when `includeReasons: true` is passed to `resolveBlockingGroupKeys`. */
+export type BlockingGroupKeysWithReasons = {
+    readonly keys: string[];
+    /** One entry per distinct blocking key produced, keyed by that key. */
+    readonly reasonsByKey: ReadonlyMap<string, BlockingReason>;
+};
 /**
  * Resolves, per page, which of the two independent blocking signals — the
  * exact stylesheet set or the URL path — to actually use as that page's
@@ -119,4 +126,7 @@ export type ResolveBlockingGroupKeysOptions = {
  * // common.css is loaded by all 3 pages and is filtered out as non-discriminative chrome.
  * ```
  */
+export declare function resolveBlockingGroupKeys(pages: readonly PageBlockingSignals[], options: ResolveBlockingGroupKeysOptions & {
+    includeReasons: true;
+}): BlockingGroupKeysWithReasons;
 export declare function resolveBlockingGroupKeys(pages: readonly PageBlockingSignals[], options?: ResolveBlockingGroupKeysOptions): string[];

package/dist/resolve-blocking-group-keys.js CHANGED Viewed

@@ -4,89 +4,6 @@ import { derivePathGroupKey } from './derive-path-group-key.js';
 import { deriveStylesheetGroupKey } from './derive-stylesheet-group-key.js';
 import { splitTokensByFrequency } from './split-tokens-by-frequency.js';
 const DEFAULT_MIN_CSS_GROUP_SIZE = 2;
-/**
- * Resolves, per page, which of the two independent blocking signals — the
- * exact stylesheet set or the URL path — to actually use as that page's
- * grouping key. Returns one key per page, in the same order as `pages`.
- *
- * Literature on entity-resolution blocking (Michelson & Knoblock's DNF
- * scheme, canopy clustering, ensemble blocking) combines independent
- * blocking predicates with OR to generate *candidate pairs* for a later
- * similarity/classification pass. This function instead commits each page to
- * exactly one final key: `resolve-page-cluster-keys.js`'s
- * `resolvePageClusterKeys` *does* run a later refinement step
- * (`resolveStructuralClusterKeys`) on top of whichever key a page lands on,
- * but only within that one key's candidate pool — it has no way to pull in
- * a page that this function routed to a different key. So this function's
- * per-page choice is still effectively final for blocking purposes: a page
- * assigned to the wrong key here never re-enters the correct key's pool
- * downstream. A true OR-merge (letting a page carry both the stylesheet and
- * path candidates, deferring to the refinement step to reconcile overlapping
- * results across them) would close that gap, but is deliberately deferred —
- * it needs the same literature-plus-real-data validation cycle this
- * package's linkage-criterion and NN-chain choices already went through, not
- * a change bundled in alongside unrelated fixes. Until then, a
- * priority-with-fallback decision — try the strong signal, fall back to the
- * weak one — is the applicable pattern here, not OR-merge: a union of
- * equivalence relations can only ever coarsen a partition, never split it,
- * but the whole point of preferring the stylesheet signal is that it *splits*
- * pages a URL-path-only grouping would otherwise lump together (confirmed
- * against real crawl data: a single page embedded under an otherwise-uniform
- * URL section, but loading a completely different stylesheet set, is exactly
- * the case a path-only key misses and a stylesheet key catches).
- *
- * Before comparing stylesheet sets, this reuses
- * {@link ./compute-document-frequency.js | computeDocumentFrequency} and
- * {@link ./split-tokens-by-frequency.js | splitTokensByFrequency} — originally
- * built to separate a page's site-wide chrome from its page-specific HTML
- * structure — to strip stylesheet hrefs that recur across most of `pages`
- * (e.g. a shared reset/font stylesheet) before hashing. Without this, two
- * pages from otherwise-unrelated sections that happen to load only that one
- * shared stylesheet would satisfy `minCssGroupSize` and be wrongly treated as
- * the same template family: the problem there isn't too few pages sharing
- * the key (raising `minCssGroupSize` doesn't fix it), it's that the key
- * itself carries no discriminative information. A page whose stylesheet set
- * is empty, or becomes empty after this filtering, always falls back to the
- * path key — loading no distinctive stylesheet is an absence of evidence,
- * not evidence of a shared template, so it must never itself become a
- * matching signal.
- *
- * Document frequency is computed only over pages that load at least one
- * stylesheet: including stylesheet-less pages in the denominator would dilute
- * every href's frequency ratio (e.g. a stylesheet loaded by 100% of the pages
- * that load *any* stylesheet would read as a low, "distinctive" frequency if
- * most pages in the batch load none), letting a genuinely non-discriminative,
- * site-wide stylesheet slip through the common-href filter.
- *
- * Like `computeDocumentFrequency` itself, this expects `pages` to be a
- * roughly homogeneous batch (one site, or one section of a large
- * multi-template site) — see that function's JSDoc for why a federation of
- * independently-templated sub-sections defeats frequency-based filtering.
- * Splitting a heterogeneous crawl into sections before calling this function
- * is the caller's responsibility.
- *
- * This filtering needs enough stylesheet-bearing pages to tell "loaded by
- * every page that has any stylesheet" apart from "coincidentally the only
- * stylesheet two pages happen to load": with only two stylesheet-bearing
- * pages in the whole batch and nothing else to contrast against, any
- * stylesheet they share reads as 100% common and gets filtered out,
- * producing a path-key fallback even when the two pages are a genuine
- * template match. A third, differently-styled page (as in the example below)
- * is what gives the shared stylesheet a frequency below the common-href
- * cutoff.
- * @param pages
- * @param options
- * @example
- * ```ts
- * resolveBlockingGroupKeys([
- * 	{ paths: ['dept-a', 'news', '1'], stylesheetHrefs: ['https://example.com/a.css', 'https://example.com/common.css'] },
- * 	{ paths: ['dept-a', 'news', '2'], stylesheetHrefs: ['https://example.com/a.css', 'https://example.com/common.css'] },
- * 	{ paths: ['dept-b', 'about'], stylesheetHrefs: ['https://example.com/common.css'] },
- * ]);
- * // ['css:<hash of a.css>', 'css:<hash of a.css>', 'path:dept-b']
- * // common.css is loaded by all 3 pages and is filtered out as non-discriminative chrome.
- * ```
- */
 export function resolveBlockingGroupKeys(pages, options) {
     const pathDepthOption = options?.pathDepth;
     const minCssGroupSize = options?.minCssGroupSize ?? DEFAULT_MIN_CSS_GROUP_SIZE;
@@ -121,14 +38,27 @@ export function resolveBlockingGroupKeys(pages, options) {
             cssKeyCounts.set(cssKey, (cssKeyCounts.get(cssKey) ?? 0) + 1);
         }
     }
-    return pages.map((page, index) => {
+    const reasonsByKey = new Map();
+    const keys = pages.map((page, index) => {
         const cssKey = cssKeys[index];
         if (cssKey !== undefined && (cssKeyCounts.get(cssKey) ?? 0) >= minCssGroupSize) {
-            return `css:${cssKey}`;
+            const key = `css:${cssKey}`;
+            if (!reasonsByKey.has(key)) {
+                reasonsByKey.set(key, {
+                    kind: 'css',
+                    distinctiveStylesheetHrefs: [...(distinctiveHrefs[index] ?? [])].toSorted(),
+                });
+            }
+            return key;
         }
         const pathKey = perPagePathKeys === null
             ? derivePathGroupKey(page.paths, pathDepthOption)
             : (perPagePathKeys[index] ?? '');
-        return `path:${pathKey}`;
+        const key = `path:${pathKey}`;
+        if (!reasonsByKey.has(key)) {
+            reasonsByKey.set(key, { kind: 'path', pathKey });
+        }
+        return key;
     });
+    return options?.includeReasons ? { keys, reasonsByKey } : keys;
 }

package/dist/resolve-page-cluster-keys.d.ts CHANGED Viewed

@@ -1,4 +1,6 @@
+import type { ClusterReason } from './build-cluster-reason.js';
 import type { ExtractLandmarksResult } from './extract-landmarks.js';
+import type { PerPageLandmarkInstance } from './per-page-landmark-signatures.js';
 import type { ResolveBlockingGroupKeysOptions } from './resolve-blocking-group-keys.js';
 import type { ResolveStructuralClusterKeysOptions } from './resolve-structural-cluster-keys.js';
 import type { TokenizeOptions } from './types.js';
@@ -65,6 +67,7 @@ import type { TokenizeOptions } from './types.js';
 export declare function computeLocalChromeArtifacts(landmarks: readonly ExtractLandmarksResult[], tokenizeOptions: TokenizeOptions | undefined): {
     readonly localSignatures: ReadonlySet<string>;
     readonly localTokensByPage: readonly ReadonlySet<string>[];
+    readonly perPageInstances: readonly (readonly PerPageLandmarkInstance[])[];
 };
 /**
  * Reinjects each page's *local* (non-corpus-wide) landmark-instance tokens
@@ -162,9 +165,42 @@ export type ResolvePageClusterKeysOptions = TokenizeOptions & ResolveBlockingGro
      * sync helper to running a per-block async loop that emits
      * `pass1-block-complete` and `stage-b-start`. Omitting `onProgress`
      * keeps the small-corpus branch on the pre-refactor sync path with
-     * zero yield overhead.
+     * zero yield overhead. Ignored when `onClusterReason` is set — that
+     * option always routes the small-corpus branch through the sync
+     * helper (see `onClusterReason`'s own JSDoc), so no progress events
+     * fire in that combination.
      */
     onProgress?: (event: ProgressEvent) => void;
+    /**
+     * Optional observability hook invoked once per **final cluster** (not
+     * per page) with that cluster's {@link ClusterReason} — the blocking
+     * signal that grouped it, its DOM-structural token core, its
+     * per-landmark-type commonality, and the sibling cluster keys it was
+     * split from within the same Pass-0 block. Unlike `onProgress`, this
+     * fires on every path — small-corpus and streaming alike — because a
+     * `ClusterReason` is sized by cluster count, not page count, so it
+     * carries no streaming-path memory risk the way a per-page report
+     * would.
+     *
+     * Building the reasons re-uses Stage A/B's own intermediate state (the
+     * quorum core, the per-unit landmark instances, the blocking
+     * evidence) — it does not re-tokenize pages or re-run corpus-wide
+     * discovery. Omitting `onClusterReason` skips that bookkeeping
+     * entirely, so existing callers pay nothing for this option.
+     *
+     * On the async factory-based `resolvePageClusterKeys`, setting this
+     * forces the small-corpus branch through the sync
+     * `resolvePageClusterKeysInMemory` helper regardless of `onProgress`
+     * — see `onProgress`'s own JSDoc.
+     * @example
+     * ```ts
+     * const reasons = new Map<string, ClusterReason>();
+     * const keys = await resolvePageClusterKeys(pages, {
+     *   onClusterReason: (key, reason) => reasons.set(key, reason),
+     * });
+     * ```
+     */
+    onClusterReason?: (clusterKey: string, reason: ClusterReason) => void;
 };
 /**
  * Corpus size at or below which the async factory-based

package/dist/resolve-page-cluster-keys.js CHANGED Viewed

@@ -1,4 +1,5 @@
 import { autoCutThreshold } from './auto-cut-threshold.js';
+import { buildClusterReason } from './build-cluster-reason.js';
 import { capContentDepth } from './cap-content-depth.js';
 import { detectContentDepthCap, validateDetectContentDepthCapOptions, } from './detect-content-depth-cap.js';
 import { extractLandmarks } from './extract-landmarks.js';
@@ -162,8 +163,9 @@ function assignPageToNearestCluster(html, assignment, excludeLandmarks, contentB
  */
 export function computeLocalChromeArtifacts(landmarks, tokenizeOptions) {
     const pageCount = landmarks.length;
-    if (pageCount === 0)
-        return { localSignatures: new Set(), localTokensByPage: [] };
+    if (pageCount === 0) {
+        return { localSignatures: new Set(), localTokensByPage: [], perPageInstances: [] };
+    }
     const perPageInstances = computePerPageLandmarkInstances(landmarks, tokenizeOptions);
     // Corpus-wide histogram: signature → { count, tokens }. tokens is the
     // token set of any one occurrence of the signature (all occurrences are
@@ -184,6 +186,7 @@ export function computeLocalChromeArtifacts(landmarks, tokenizeOptions) {
         return {
             localSignatures: new Set(),
             localTokensByPage: landmarks.map(() => new Set()),
+            perPageInstances,
         };
     }
     const frequencies = [];
@@ -202,6 +205,7 @@ export function computeLocalChromeArtifacts(landmarks, tokenizeOptions) {
         return {
             localSignatures: new Set(),
             localTokensByPage: landmarks.map(() => new Set()),
+            perPageInstances,
         };
     }
     const localTokensByPage = perPageInstances.map((instances) => {
@@ -214,7 +218,7 @@ export function computeLocalChromeArtifacts(landmarks, tokenizeOptions) {
         }
         return out;
     });
-    return { localSignatures, localTokensByPage };
+    return { localSignatures, localTokensByPage, perPageInstances };
 }
 /**
  * Reinjects each page's *local* (non-corpus-wide) landmark-instance tokens
@@ -232,6 +236,63 @@ export function computeLocalChromeArtifacts(landmarks, tokenizeOptions) {
 export function computeLocalLandmarkTokens(landmarks, tokenizeOptions) {
     return [...computeLocalChromeArtifacts(landmarks, tokenizeOptions).localTokensByPage];
 }
+/**
+ * Builds and emits one {@link ClusterReason} per final cluster via
+ * `onClusterReason`, from data Stage A/B already computed for clustering
+ * itself: `crossBlockUnits` (Stage A's pre-merge units, each carrying its
+ * originating block key inside `JSON.parse(unit.key)[0]`), Stage B's
+ * `rootByKey`/`finalGroupsByRoot`, the per-block-key `BlockingReason`s Pass 0
+ * derived, and the per-block sibling-unit-key lists the driver accumulated
+ * alongside its Stage A loop. No re-tokenization and no extra corpus pass —
+ * this only re-groups references the driver already held.
+ * @param crossBlockUnits
+ * @param rootByKey
+ * @param finalGroupsByRoot
+ * @param reasonsByBlockKey
+ * @param siblingUnitKeysByBlock
+ * @param onClusterReason
+ */
+function emitClusterReasons(crossBlockUnits, rootByKey, finalGroupsByRoot, reasonsByBlockKey, siblingUnitKeysByBlock, onClusterReason) {
+    const unitKeysByRoot = new Map();
+    for (const unit of crossBlockUnits) {
+        const root = rootByKey.get(unit.key) ?? unit.key;
+        const list = unitKeysByRoot.get(root);
+        if (list) {
+            list.push(unit.key);
+        }
+        else {
+            unitKeysByRoot.set(root, [unit.key]);
+        }
+    }
+    for (const [rootKey, unitKeys] of unitKeysByRoot) {
+        const finalGroup = finalGroupsByRoot.get(rootKey);
+        if (!finalGroup)
+            continue;
+        const seenBlockKeys = new Set();
+        const blocking = [];
+        const siblingRoots = new Set();
+        for (const unitKey of unitKeys) {
+            const blockKey = JSON.parse(unitKey)[0];
+            if (!seenBlockKeys.has(blockKey)) {
+                seenBlockKeys.add(blockKey);
+                const reason = reasonsByBlockKey.get(blockKey);
+                if (reason)
+                    blocking.push({ blockKey, reason });
+            }
+            for (const siblingUnitKey of siblingUnitKeysByBlock.get(blockKey) ?? []) {
+                const siblingRoot = rootByKey.get(siblingUnitKey) ?? siblingUnitKey;
+                if (siblingRoot !== rootKey)
+                    siblingRoots.add(siblingRoot);
+            }
+        }
+        onClusterReason(rootKey, buildClusterReason({
+            tokenSets: finalGroup.tokenSets,
+            landmarkInstances: finalGroup.landmarkInstances,
+            blocking,
+            siblingClusterKeys: [...siblingRoots].toSorted(),
+        }));
+    }
+}
 /**
  * Corpus size at or below which the async factory-based
  * `resolvePageClusterKeys` reads the entire input into an array and delegates
@@ -313,11 +374,9 @@ export function resolvePageClusterKeysInMemory(pages, options) {
         throw new RangeError(`resolvePageClusterKeys: similarityThreshold must be between 0 and 1, got ${similarityThreshold}`);
     }
     // Always computed: landmark fields are needed by Stage B's shell
-    // corroboration regardless of `excludeLandmarks`, and `remainderHtml` is
-    // needed whenever `excludeLandmarks` is true.
+    // corroboration regardless of `excludeLandmarks`.
     const landmarks = pages.map((page) => extractLandmarks(page.html));
-    // Corpus-level chrome discovery
-    const localLandmarkTokensByPage = computeLocalLandmarkTokens(landmarks, options);
+    const { localTokensByPage: localLandmarkTokensByPage } = computeLocalChromeArtifacts(landmarks, options);
     const contentBlockAttribute = options?.contentBlockAttribute;
     const preparedHtml = pages.map((page, index) => {
         const landmarksExcised = excludeLandmarks
@@ -332,7 +391,20 @@ export function resolvePageClusterKeysInMemory(pages, options) {
     const blockingPages = restrictStylesheetsToFirstParty
         ? filterFirstPartyStylesheetHrefs(pages)
         : pages;
-    const blockKeys = resolveBlockKeys(blockingPages, options);
+    // Reasons (blocking evidence) are only worth deriving when a caller
+    // actually asked for `onClusterReason` — see that option's own JSDoc for
+    // why this is the only place ClusterReason bookkeeping is opt-in.
+    const onClusterReason = options?.onClusterReason;
+    let blockKeys;
+    let reasonsByBlockKey;
+    if (onClusterReason) {
+        const result = resolveBlockKeys(blockingPages, { ...options, includeReasons: true });
+        blockKeys = result.blockKeys;
+        reasonsByBlockKey = result.reasonsByBlockKey;
+    }
+    else {
+        blockKeys = resolveBlockKeys(blockingPages, options);
+    }
     const indicesByBlockKey = groupIndicesByBlockKey(blockKeys);
     // Validated here, eagerly, because it's otherwise only reached from
     // inside the per-block loop below — which never runs at all for an empty
@@ -341,6 +413,9 @@ export function resolvePageClusterKeysInMemory(pages, options) {
     validateDetectContentDepthCapOptions(options);
     const finalKeys = Array.from({ length: pages.length });
     const crossBlockUnits = [];
+    const siblingUnitKeysByBlock = onClusterReason
+        ? new Map()
+        : undefined;
     for (const [blockKey, indices] of indicesByBlockKey) {
         const result = stageAPerBlock({
             blockKey,
@@ -353,16 +428,20 @@ export function resolvePageClusterKeysInMemory(pages, options) {
             finalKeys[pageIndex] = key;
         }
         crossBlockUnits.push(...result.crossBlockUnits);
+        siblingUnitKeysByBlock?.set(blockKey, result.crossBlockUnits.map((u) => u.key));
     }
     // Stage B: cross-block merge — always runs regardless of options
-    const stageBResult = mergeCrossBlockClusters(crossBlockUnits, options);
+    const { rootByKey, finalGroupsByRoot } = mergeCrossBlockClusters(crossBlockUnits, options);
     for (let i = 0; i < finalKeys.length; i++) {
         const currentKey = finalKeys[i];
-        const rootKey = stageBResult.get(currentKey);
+        const rootKey = rootByKey.get(currentKey);
         if (rootKey !== undefined && rootKey !== currentKey) {
             finalKeys[i] = rootKey;
         }
     }
+    if (onClusterReason && reasonsByBlockKey && siblingUnitKeysByBlock) {
+        emitClusterReasons(crossBlockUnits, rootByKey, finalGroupsByRoot, reasonsByBlockKey, siblingUnitKeysByBlock, onClusterReason);
+    }
     return finalKeys;
 }
 /**
@@ -443,10 +522,10 @@ async function resolveSmallCorpusWithProgress(pages, onProgress, options) {
         await new Promise((resolve) => setImmediate(resolve));
     }
     onProgress({ phase: 'stage-b-start', unitCount: crossBlockUnits.length });
-    const stageBResult = mergeCrossBlockClusters(crossBlockUnits, options);
+    const { rootByKey } = mergeCrossBlockClusters(crossBlockUnits, options);
     for (let i = 0; i < finalKeys.length; i++) {
         const currentKey = finalKeys[i];
-        const rootKey = stageBResult.get(currentKey);
+        const rootKey = rootByKey.get(currentKey);
         if (rootKey !== undefined && rootKey !== currentKey) {
             finalKeys[i] = rootKey;
         }
@@ -530,8 +609,12 @@ export async function resolvePageClusterKeys(pages, options) {
         // into per-block progress, so delegate to the untouched sync path —
         // keeping behavior byte-for-byte identical (and yield-overhead-free)
         // to how library-only consumers experienced this before the CLI
-        // progress work landed.
-        if (onProgress === undefined) {
+        // progress work landed. `onClusterReason` always routes here too (see
+        // its own JSDoc): `resolveSmallCorpusWithProgress` has no
+        // cluster-reason support, and duplicating that logic into the
+        // progress-emitting path for a reporting feature that has nothing to
+        // do with progress observability isn't worth the added surface.
+        if (onProgress === undefined || options?.onClusterReason) {
             return resolvePageClusterKeysInMemory(fullPages, options);
         }
         return resolveSmallCorpusWithProgress(fullPages, onProgress, options);
@@ -547,11 +630,30 @@ export async function resolvePageClusterKeys(pages, options) {
     const blockingPagesForKeys = restrictStylesheetsToFirstParty
         ? filterFirstPartyStylesheetHrefs(blockingSignals)
         : blockingSignals;
-    const blockKeys = resolveBlockKeys(blockingPagesForKeys, options);
+    // Reasons (blocking evidence) cost nothing beyond bookkeeping — a Map
+    // keyed by distinct block key, not by page — but are only derived when a
+    // caller actually asked for `onClusterReason`.
+    const onClusterReason = options?.onClusterReason;
+    let blockKeys;
+    let reasonsByBlockKey;
+    if (onClusterReason) {
+        const result = resolveBlockKeys(blockingPagesForKeys, {
+            ...options,
+            includeReasons: true,
+        });
+        blockKeys = result.blockKeys;
+        reasonsByBlockKey = result.reasonsByBlockKey;
+    }
+    else {
+        blockKeys = resolveBlockKeys(blockingPagesForKeys, options);
+    }
     const indicesByBlockKey = groupIndicesByBlockKey(blockKeys);
     const finalKeys = Array.from({ length: blockingSignals.length });
     const crossBlockUnits = [];
     const contentBlockAttribute = options?.contentBlockAttribute;
+    const siblingUnitKeysByBlock = onClusterReason
+        ? new Map()
+        : undefined;
     /** Non-sample page indices that need Pass 1b Jaccard-based assignment. */
     const pendingAssignmentBlockKeyByIndex = new Map();
     /** Block-level artifacts saved after Stage A runs on the sample. */
@@ -590,6 +692,7 @@ export async function resolvePageClusterKeys(pages, options) {
             finalKeys[idx] = key;
         }
         crossBlockUnits.push(...result.crossBlockUnits);
+        siblingUnitKeysByBlock?.set(bucket.blockKey, result.crossBlockUnits.map((u) => u.key));
         if (bucket.seenCount > bucket.reservoirIndices.length) {
             // Save assignment artifacts for Pass 1b.
             const maxMainDepth = bucket.reservoirPreparedHtml.length > 1
@@ -715,17 +818,20 @@ export async function resolvePageClusterKeys(pages, options) {
     if (onProgress) {
         onProgress({ phase: 'stage-b-start', unitCount: crossBlockUnits.length });
     }
-    const stageBResult = mergeCrossBlockClusters(crossBlockUnits, {
+    const { rootByKey, finalGroupsByRoot } = mergeCrossBlockClusters(crossBlockUnits, {
         ...options,
         capMembers: BLOCK_SAMPLE_SIZE,
     });
     for (let i = 0; i < finalKeys.length; i++) {
         const currentKey = finalKeys[i];
-        const rootKey = stageBResult.get(currentKey);
+        const rootKey = rootByKey.get(currentKey);
         if (rootKey !== undefined && rootKey !== currentKey) {
             finalKeys[i] = rootKey;
         }
     }
+    if (onClusterReason && reasonsByBlockKey && siblingUnitKeysByBlock) {
+        emitClusterReasons(crossBlockUnits, rootByKey, finalGroupsByRoot, reasonsByBlockKey, siblingUnitKeysByBlock, onClusterReason);
+    }
     return finalKeys;
 }
 /**

package/dist/shell-quorum.d.ts ADDED Viewed

@@ -0,0 +1,70 @@
+import type { PerPageLandmarkInstance } from './per-page-landmark-signatures.js';
+/**
+ * Discovers a unit's shell tokens by auto-cutting the per-*token* page-
+ * frequency histogram of every landmark instance's tokens. This is the same
+ * max-gap primitive used for Stage A merge-height cutoffs, applied
+ * recursively at the landmark-token layer.
+ *
+ * ## Why per-token and not per-signature
+ *
+ * An earlier iteration ran the histogram at the level of full landmark-
+ * instance signatures (canonicalized token sets). That failed on a real,
+ * common pattern: a shared site chrome whose markup carries a per-page
+ * distinguishing element (a breadcrumb, a page-title element with a page-
+ * specific class, a "current" state). All pages have most of the same
+ * tokens, but every page's full signature is distinct because tokens embed
+ * class names. Per-signature counting saw 5 signatures at freq 0.2 each,
+ * autoCutThreshold on the flat distribution returned the clamp, and the
+ * shell collapsed to empty even though every page shared the core header
+ * skeleton. Per-token counting handles the same case correctly — the shared
+ * skeleton tokens each hit freq 1.0.
+ *
+ * ## The histogram
+ *
+ * For every member page, all its landmark instances are tokenized and
+ * unioned into a single per-page token set (order-agnostic, deduped: a
+ * token appearing in two of the page's landmarks still counts once for
+ * that page). The corpus histogram is then "how many pages contain each
+ * token". Tokens that appear on nearly every page are the unit's chrome;
+ * tokens that appear on only a handful are page-specific content that
+ * happens to be tagged as a landmark.
+ *
+ * ## Why auto-cut instead of a hard-coded quorum
+ *
+ * A fixed 80% quorum (this file's earlier implementation) baked one
+ * threshold in for every unit. Real corpora don't obey a universal cutoff:
+ * a section-local landmark token that appears on 60% of a unit's pages is
+ * the section's chrome under any reasonable reading, but 80% quorum
+ * discards it. Auto-cut looks at the *shape* of the frequency distribution
+ * and picks the widest gap between adjacent frequencies — if the
+ * distribution is `{1.00, 1.00, 0.65, 0.03, 0.02}`, the gap between 0.65
+ * and 0.03 (0.62) dwarfs everything else and the cut lands mid-gap around
+ * 0.34, correctly grouping the 0.65 tokens with the site-wide 1.00 ones as
+ * "chrome for this unit". If instead the distribution is flat, the clamp
+ * to {@link SHELL_QUORUM_FALLBACK_FRACTION} keeps the threshold from being
+ * tighter than the fallback default.
+ *
+ * ## Fallbacks
+ *
+ * A single distinct token (`heights.length < 2`) or a perfectly flat
+ * distribution (`maxGap === 0`) returns the
+ * {@link SHELL_QUORUM_FALLBACK_FRACTION} clamp verbatim — exactly the same
+ * 80%-quorum behavior as before. So degenerate cases degrade to the old
+ * contract; only richer distributions get the auto-cut benefit.
+ *
+ * A page with no landmarks contributes an empty set, deliberately, so the
+ * shell-corroboration jaccard between two landmark-less pages is 0 rather
+ * than 1 (which it would be if we handed back a `<body></body>`-derived
+ * `{body}` fallback set to both sides).
+ *
+ * ## Reuse
+ *
+ * Originally private to {@link ./merge-cross-block-clusters.js | mergeCrossBlockClusters}'s
+ * Stage B L2 corroboration; exported from its own module so
+ * {@link ./build-cluster-reason.js | buildClusterReason} can run it once per
+ * final cluster, per landmark type, to classify individual landmark
+ * instances as chrome (see {@link ./is-chrome-landmark-instance.js |
+ * isChromeLandmarkInstance}).
+ * @param perPageInstances
+ */
+export declare function shellQuorum(perPageInstances: readonly (readonly PerPageLandmarkInstance[])[]): ReadonlySet<string>;