@d-zero/page-cluster 0.4.0 → 0.5.0

package/dist/pass0-blocking.d.ts

@@ -1,3 +1,4 @@
+import type { BlockingReason } from './derive-blocking-reason.js';
 import type { ResolveBlockingGroupKeysOptions } from './resolve-blocking-group-keys.js';
 /**
  * The corpus-wide, HTML-free inputs needed by
@@ -31,6 +32,12 @@ export type ResolveBlockKeysOptions = ResolveBlockingGroupKeysOptions & {
      */
     readonly restrictStylesheetsToFirstParty?: boolean;
 };
+/** Return shape when `includeReasons: true` is passed to `resolveBlockKeys`. */
+export type BlockKeysWithReasons = {
+    readonly blockKeys: string[];
+    /** One entry per distinct final block key produced, keyed by that key. */
+    readonly reasonsByBlockKey: ReadonlyMap<string, BlockingReason>;
+};
 /**
  * Splits `resolvePageClusterKeys` into a size-flat first pass so the driver
  * can decide per-block memory strategy before loading any page HTML. Runs the
@@ -71,6 +78,9 @@ export type ResolveBlockKeysOptions = ResolveBlockingGroupKeysOptions & {
  * // ['css:<hash>', 'css:<hash>', 'path:about']
  * ```
  */
+export declare function resolveBlockKeys(pages: readonly Pass0PageSignals[], options: ResolveBlockKeysOptions & {
+    includeReasons: true;
+}): BlockKeysWithReasons;
 export declare function resolveBlockKeys(pages: readonly Pass0PageSignals[], options?: ResolveBlockKeysOptions): string[];
 /**
  * Groups pages by their block key while preserving each block's members in

package/dist/pass0-blocking.js

@@ -1,62 +1,42 @@
 import { filterFirstPartyStylesheetHrefs } from './filter-first-party-stylesheet-hrefs.js';
-import { reassignOrphanBlockKeys } from './reassign-orphan-block-keys.js';
+import { REASSIGNED_KEY_PREFIX, reassignOrphanBlockKeys, } from './reassign-orphan-block-keys.js';
 import { resolveBlockingGroupKeys } from './resolve-blocking-group-keys.js';
-/**
- * Splits `resolvePageClusterKeys` into a size-flat first pass so the driver
- * can decide per-block memory strategy before loading any page HTML. Runs the
- * three corpus-wide, HTML-free stages of blocking in the same order the
- * in-memory driver already uses — first-party stylesheet filtering, blocking-
- * key derivation, orphan reassignment — and returns one final block key per
- * input page in input order.
- *
- * ## Why extract this from resolvePageClusterKeys?
- *
- * The in-memory driver holds every page's `html`, `remainderHtml`,
- * `landmarks[]`, and pre-Stage-A prepared HTML at once. At 176k pages × ~57
- * KB average, that alone breaks a 17 GB RAM machine well before Stage A
- * starts (measured: OS SIGKILL at ~15,000 pages, before the resolve phase
- * even began). All three blocking stages, in contrast, depend only on
- * `paths` / `stylesheetHrefs` / `host` — a few hundred bytes per page. Running
- * them first, HTML-free, lets the downstream per-block clustering hold HTML
- * for only one block's pages at a time.
- *
- * ## Preserves in-memory driver semantics exactly
- *
- * The output of this function is byte-identical to the block-key portion of
- * the current `resolvePageClusterKeys` for the same input, because it reuses
- * the same three underlying functions in the same order with the same
- * defaults. That guarantee is load-bearing: the size-threshold-gated Pass 1
- * that follows this function runs the current in-memory implementation
- * unchanged for small blocks, and any drift in block-key computation between
- * Pass 0 and Pass 1 would silently misroute pages between the two paths.
- * @param pages
- * @param options
- * @example
- * ```ts
- * const blockKeys = resolveBlockKeys([
- *   { paths: ['news', '1'], stylesheetHrefs: ['/a.css'], host: 'example.com' },
- *   { paths: ['news', '2'], stylesheetHrefs: ['/a.css'], host: 'example.com' },
- *   { paths: ['about'],     stylesheetHrefs: [],        host: 'example.com' },
- * ]);
- * // ['css:<hash>', 'css:<hash>', 'path:about']
- * ```
- */
 export function resolveBlockKeys(pages, options) {
     const restrictStylesheetsToFirstParty = options?.restrictStylesheetsToFirstParty ?? true;
     const blockingPages = restrictStylesheetsToFirstParty
         ? filterFirstPartyStylesheetHrefs(pages)
         : pages;
-    const rawBlockKeys = resolveBlockingGroupKeys(blockingPages, options);
-    const reassignOrphans = options?.reassignOrphans ?? true;
-    if (!reassignOrphans)
-        return rawBlockKeys;
     // Orphan reassignment always uses a numeric `pathDepth`. When the caller
     // asked for `'auto'`, fall back to the historical default 1 here — a
     // future PR that wires the auto-cut depth through can compute it once
     // and pass it as a number to both `resolveBlockingGroupKeys` and this
     // call to keep them consistent.
     const numericPathDepth = typeof options?.pathDepth === 'number' ? options.pathDepth : undefined;
-    return reassignOrphanBlockKeys(blockingPages, rawBlockKeys, numericPathDepth);
+    const reassignOrphans = options?.reassignOrphans ?? true;
+    if (!options?.includeReasons) {
+        const rawBlockKeys = resolveBlockingGroupKeys(blockingPages, options);
+        if (!reassignOrphans)
+            return rawBlockKeys;
+        return reassignOrphanBlockKeys(blockingPages, rawBlockKeys, numericPathDepth);
+    }
+    const { keys: rawBlockKeys, reasonsByKey } = resolveBlockingGroupKeys(blockingPages, {
+        ...options,
+        includeReasons: true,
+    });
+    if (!reassignOrphans) {
+        return { blockKeys: rawBlockKeys, reasonsByBlockKey: reasonsByKey };
+    }
+    const finalBlockKeys = reassignOrphanBlockKeys(blockingPages, rawBlockKeys, numericPathDepth);
+    const reasonsByBlockKey = new Map(reasonsByKey);
+    for (const blockKey of finalBlockKeys) {
+        if (reasonsByBlockKey.has(blockKey))
+            continue;
+        reasonsByBlockKey.set(blockKey, {
+            kind: 'orphanMerge',
+            pathKey: blockKey.slice(REASSIGNED_KEY_PREFIX.length),
+        });
+    }
+    return { blockKeys: finalBlockKeys, reasonsByBlockKey };
 }
 /**
  * Groups pages by their block key while preserving each block's members in

package/dist/per-page-landmark-signatures.d.ts

@@ -16,11 +16,10 @@ export declare const ALL_LANDMARK_TYPES: readonly LandmarkType[];
  *
  * `position` is the instance's location in the page it came from, computed
  * once by {@link ./extract-landmarks.js | extractLandmarks} and carried here
- * unchanged — a back-reference for callers (e.g.
- * {@link ./build-page-landmark-report.js | buildPageLandmarkReport}) that
- * need to report where a chrome-classified instance actually sits, not just
- * that it exists. It plays no part in `tokens`/`signature` computation or in
- * the corpus-frequency logic that consumes this type.
+ * unchanged — a back-reference for callers that need to report where a
+ * chrome-classified instance actually sits, not just that it exists. It
+ * plays no part in `tokens`/`signature` computation or in the corpus-
+ * frequency logic that consumes this type.
  */
 export type PerPageLandmarkInstance = {
     readonly type: LandmarkType;

package/dist/reassign-orphan-block-keys.d.ts

@@ -1,4 +1,13 @@
 import type { PageBlockingSignals } from './resolve-blocking-group-keys.js';
+/**
+ * Prefix distinguishing a reassigned key from the `css:`/`path:` keys
+ * {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys} itself
+ * produces, so the two families can never collide. Exported so
+ * {@link ./pass0-blocking.js | resolveBlockKeys} can recover the confined
+ * path key back out of a reassigned block key when building `BlockingReason`s,
+ * without duplicating this literal.
+ */
+export declare const REASSIGNED_KEY_PREFIX = "orphan-merge:";
 /**
  * Rewrites the `path:`-fallback key of an "orphan" page — one with no
  * stylesheet references recorded at all — to match a same-URL-section `css:`

package/dist/reassign-orphan-block-keys.js

@@ -2,9 +2,12 @@ import { derivePathGroupKey } from './derive-path-group-key.js';
 /**
  * Prefix distinguishing a reassigned key from the `css:`/`path:` keys
  * {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys} itself
- * produces, so the two families can never collide.
+ * produces, so the two families can never collide. Exported so
+ * {@link ./pass0-blocking.js | resolveBlockKeys} can recover the confined
+ * path key back out of a reassigned block key when building `BlockingReason`s,
+ * without duplicating this literal.
  */
-const REASSIGNED_KEY_PREFIX = 'orphan-merge:';
+export const REASSIGNED_KEY_PREFIX = 'orphan-merge:';
 /**
  * Reads `values[index]`, throwing instead of returning `undefined`. Every
  * call site here indexes `pages`/`pathKeys`/`blockKeys` with a position

package/dist/resolve-blocking-group-keys.d.ts

@@ -1,3 +1,4 @@
+import type { BlockingReason } from './derive-blocking-reason.js';
 /**
  * The two blocking signals {@link ./derive-path-group-key.js | derivePathGroupKey}
  * and {@link ./derive-stylesheet-group-key.js | deriveStylesheetGroupKey} need,
@@ -36,6 +37,12 @@ export type ResolveBlockingGroupKeysOptions = {
     /** Forwarded to `splitTokensByFrequency` as-is. */
     hrefCommonThreshold?: number;
 };
+/** Return shape when `includeReasons: true` is passed to `resolveBlockingGroupKeys`. */
+export type BlockingGroupKeysWithReasons = {
+    readonly keys: string[];
+    /** One entry per distinct blocking key produced, keyed by that key. */
+    readonly reasonsByKey: ReadonlyMap<string, BlockingReason>;
+};
 /**
  * Resolves, per page, which of the two independent blocking signals — the
  * exact stylesheet set or the URL path — to actually use as that page's
@@ -119,4 +126,7 @@ export type ResolveBlockingGroupKeysOptions = {
  * // common.css is loaded by all 3 pages and is filtered out as non-discriminative chrome.
  * ```
  */
+export declare function resolveBlockingGroupKeys(pages: readonly PageBlockingSignals[], options: ResolveBlockingGroupKeysOptions & {
+    includeReasons: true;
+}): BlockingGroupKeysWithReasons;
 export declare function resolveBlockingGroupKeys(pages: readonly PageBlockingSignals[], options?: ResolveBlockingGroupKeysOptions): string[];

package/dist/resolve-blocking-group-keys.js

@@ -4,89 +4,6 @@ import { derivePathGroupKey } from './derive-path-group-key.js';
 import { deriveStylesheetGroupKey } from './derive-stylesheet-group-key.js';
 import { splitTokensByFrequency } from './split-tokens-by-frequency.js';
 const DEFAULT_MIN_CSS_GROUP_SIZE = 2;
-/**
- * Resolves, per page, which of the two independent blocking signals — the
- * exact stylesheet set or the URL path — to actually use as that page's
- * grouping key. Returns one key per page, in the same order as `pages`.
- *
- * Literature on entity-resolution blocking (Michelson & Knoblock's DNF
- * scheme, canopy clustering, ensemble blocking) combines independent
- * blocking predicates with OR to generate *candidate pairs* for a later
- * similarity/classification pass. This function instead commits each page to
- * exactly one final key: `resolve-page-cluster-keys.js`'s
- * `resolvePageClusterKeys` *does* run a later refinement step
- * (`resolveStructuralClusterKeys`) on top of whichever key a page lands on,
- * but only within that one key's candidate pool — it has no way to pull in
- * a page that this function routed to a different key. So this function's
- * per-page choice is still effectively final for blocking purposes: a page
- * assigned to the wrong key here never re-enters the correct key's pool
- * downstream. A true OR-merge (letting a page carry both the stylesheet and
- * path candidates, deferring to the refinement step to reconcile overlapping
- * results across them) would close that gap, but is deliberately deferred —
- * it needs the same literature-plus-real-data validation cycle this
- * package's linkage-criterion and NN-chain choices already went through, not
- * a change bundled in alongside unrelated fixes. Until then, a
- * priority-with-fallback decision — try the strong signal, fall back to the
- * weak one — is the applicable pattern here, not OR-merge: a union of
- * equivalence relations can only ever coarsen a partition, never split it,
- * but the whole point of preferring the stylesheet signal is that it *splits*
- * pages a URL-path-only grouping would otherwise lump together (confirmed
- * against real crawl data: a single page embedded under an otherwise-uniform
- * URL section, but loading a completely different stylesheet set, is exactly
- * the case a path-only key misses and a stylesheet key catches).
- *
- * Before comparing stylesheet sets, this reuses
- * {@link ./compute-document-frequency.js | computeDocumentFrequency} and
- * {@link ./split-tokens-by-frequency.js | splitTokensByFrequency} — originally
- * built to separate a page's site-wide chrome from its page-specific HTML
- * structure — to strip stylesheet hrefs that recur across most of `pages`
- * (e.g. a shared reset/font stylesheet) before hashing. Without this, two
- * pages from otherwise-unrelated sections that happen to load only that one
- * shared stylesheet would satisfy `minCssGroupSize` and be wrongly treated as
- * the same template family: the problem there isn't too few pages sharing
- * the key (raising `minCssGroupSize` doesn't fix it), it's that the key
- * itself carries no discriminative information. A page whose stylesheet set
- * is empty, or becomes empty after this filtering, always falls back to the
- * path key — loading no distinctive stylesheet is an absence of evidence,
- * not evidence of a shared template, so it must never itself become a
- * matching signal.
- *
- * Document frequency is computed only over pages that load at least one
- * stylesheet: including stylesheet-less pages in the denominator would dilute
- * every href's frequency ratio (e.g. a stylesheet loaded by 100% of the pages
- * that load *any* stylesheet would read as a low, "distinctive" frequency if
- * most pages in the batch load none), letting a genuinely non-discriminative,
- * site-wide stylesheet slip through the common-href filter.
- *
- * Like `computeDocumentFrequency` itself, this expects `pages` to be a
- * roughly homogeneous batch (one site, or one section of a large
- * multi-template site) — see that function's JSDoc for why a federation of
- * independently-templated sub-sections defeats frequency-based filtering.
- * Splitting a heterogeneous crawl into sections before calling this function
- * is the caller's responsibility.
- *
- * This filtering needs enough stylesheet-bearing pages to tell "loaded by
- * every page that has any stylesheet" apart from "coincidentally the only
- * stylesheet two pages happen to load": with only two stylesheet-bearing
- * pages in the whole batch and nothing else to contrast against, any
- * stylesheet they share reads as 100% common and gets filtered out,
- * producing a path-key fallback even when the two pages are a genuine
- * template match. A third, differently-styled page (as in the example below)
- * is what gives the shared stylesheet a frequency below the common-href
- * cutoff.
- * @param pages
- * @param options
- * @example
- * ```ts
- * resolveBlockingGroupKeys([
- * 	{ paths: ['dept-a', 'news', '1'], stylesheetHrefs: ['https://example.com/a.css', 'https://example.com/common.css'] },
- * 	{ paths: ['dept-a', 'news', '2'], stylesheetHrefs: ['https://example.com/a.css', 'https://example.com/common.css'] },
- * 	{ paths: ['dept-b', 'about'], stylesheetHrefs: ['https://example.com/common.css'] },
- * ]);
- * // ['css:<hash of a.css>', 'css:<hash of a.css>', 'path:dept-b']
- * // common.css is loaded by all 3 pages and is filtered out as non-discriminative chrome.
- * ```
- */
 export function resolveBlockingGroupKeys(pages, options) {
     const pathDepthOption = options?.pathDepth;
     const minCssGroupSize = options?.minCssGroupSize ?? DEFAULT_MIN_CSS_GROUP_SIZE;
@@ -121,14 +38,27 @@ export function resolveBlockingGroupKeys(pages, options) {
             cssKeyCounts.set(cssKey, (cssKeyCounts.get(cssKey) ?? 0) + 1);
         }
     }
-    return pages.map((page, index) => {
+    const reasonsByKey = new Map();
+    const keys = pages.map((page, index) => {
         const cssKey = cssKeys[index];
         if (cssKey !== undefined && (cssKeyCounts.get(cssKey) ?? 0) >= minCssGroupSize) {
-            return `css:${cssKey}`;
+            const key = `css:${cssKey}`;
+            if (!reasonsByKey.has(key)) {
+                reasonsByKey.set(key, {
+                    kind: 'css',
+                    distinctiveStylesheetHrefs: [...(distinctiveHrefs[index] ?? [])].toSorted(),
+                });
+            }
+            return key;
         }
         const pathKey = perPagePathKeys === null
             ? derivePathGroupKey(page.paths, pathDepthOption)
             : (perPagePathKeys[index] ?? '');
-        return `path:${pathKey}`;
+        const key = `path:${pathKey}`;
+        if (!reasonsByKey.has(key)) {
+            reasonsByKey.set(key, { kind: 'path', pathKey });
+        }
+        return key;
     });
+    return options?.includeReasons ? { keys, reasonsByKey } : keys;
 }

package/dist/resolve-page-cluster-keys.d.ts

@@ -1,9 +1,9 @@
+import type { ClusterReason } from './build-cluster-reason.js';
 import type { ExtractLandmarksResult } from './extract-landmarks.js';
 import type { PerPageLandmarkInstance } from './per-page-landmark-signatures.js';
 import type { ResolveBlockingGroupKeysOptions } from './resolve-blocking-group-keys.js';
 import type { ResolveStructuralClusterKeysOptions } from './resolve-structural-cluster-keys.js';
 import type { TokenizeOptions } from './types.js';
-import { type PageLandmarkReport } from './build-page-landmark-report.js';
 /**
  * Reinjects each page's *local* (non-corpus-wide) landmark-instance tokens
  * into its block token set for Stage A clustering, restoring exactly the
@@ -165,47 +165,42 @@ export type ResolvePageClusterKeysOptions = TokenizeOptions & ResolveBlockingGro
      * sync helper to running a per-block async loop that emits
      * `pass1-block-complete` and `stage-b-start`. Omitting `onProgress`
      * keeps the small-corpus branch on the pre-refactor sync path with
-     * zero yield overhead. Ignored when `includeLandmarkPositions` is
-     * `true` — that option always routes the small-corpus branch through
-     * the sync helper (see `includeLandmarkPositions`'s own JSDoc), so no
-     * progress events fire in that combination.
+     * zero yield overhead. Ignored when `onClusterReason` is set — that
+     * option always routes the small-corpus branch through the sync
+     * helper (see `onClusterReason`'s own JSDoc), so no progress events
+     * fire in that combination.
      */
     onProgress?: (event: ProgressEvent) => void;
     /**
-     * When `true`, every result entry additionally carries a
-     * {@link PageLandmarkReport} — each landmark instance's position, plus
-     * a chrome/content verdict for the six excisable types (see
-     * {@link ./build-page-landmark-report.js | buildPageLandmarkReport}).
-     * Changes the return shape from `string[]` to
-     * `{@link PageClusterKeyResult}[]` (see the overloads on
-     * `resolvePageClusterKeysInMemory`/`resolvePageClusterKeys`/
-     * `resolvePageClusterKeysFromArray`). Defaults to `false`, in which
-     * case every existing caller's behavior and return type are
-     * unchanged.
+     * Optional observability hook invoked once per **final cluster** (not
+     * per page) with that cluster's {@link ClusterReason} — the blocking
+     * signal that grouped it, its DOM-structural token core, its
+     * per-landmark-type commonality, and the sibling cluster keys it was
+     * split from within the same Pass-0 block. Unlike `onProgress`, this
+     * fires on every path — small-corpus and streaming alike — because a
+     * `ClusterReason` is sized by cluster count, not page count, so it
+     * carries no streaming-path memory risk the way a per-page report
+     * would.
      *
-     * Not supported on the streaming path
-     * (`pageCount > {@link CORPUS_INLINE_THRESHOLD}`): reservoir sampling
-     * and Jaccard-based non-sample assignment there have no notion of
-     * "this page's shell tokens" to classify chrome against, and
-     * retrofitting one is out of scope. `resolvePageClusterKeys` throws a
-     * `RangeError` up front if both apply to the same call, rather than
-     * silently degrading semantics.
+     * Building the reasons re-uses Stage A/B's own intermediate state (the
+     * quorum core, the per-unit landmark instances, the blocking
+     * evidence) — it does not re-tokenize pages or re-run corpus-wide
+     * discovery. Omitting `onClusterReason` skips that bookkeeping
+     * entirely, so existing callers pay nothing for this option.
      *
-     * On the async factory-based `resolvePageClusterKeys`, this option
+     * On the async factory-based `resolvePageClusterKeys`, setting this
      * forces the small-corpus branch through the sync
      * `resolvePageClusterKeysInMemory` helper regardless of `onProgress`
      * — see `onProgress`'s own JSDoc.
+     * @example
+     * ```ts
+     * const reasons = new Map<string, ClusterReason>();
+     * const keys = await resolvePageClusterKeys(pages, {
+     *   onClusterReason: (key, reason) => reasons.set(key, reason),
+     * });
+     * ```
      */
-    includeLandmarkPositions?: boolean;
-};
-/**
- * One page's clustering result when `includeLandmarkPositions` is `true`:
- * the same `clusterKey` every caller already gets, plus that page's
- * {@link PageLandmarkReport}.
- */
-export type PageClusterKeyResult = {
-    readonly clusterKey: string;
-    readonly landmarks: PageLandmarkReport;
+    onClusterReason?: (clusterKey: string, reason: ClusterReason) => void;
 };
 /**
  * Corpus size at or below which the async factory-based
@@ -229,17 +224,6 @@ export type PageClusterKeyResult = {
  * anything above 20,000 is routed to streaming.
  */
 export declare const CORPUS_INLINE_THRESHOLD = 20000;
-/**
- * Throws when `includeLandmarkPositions` is combined with a corpus over
- * `threshold` pages (the streaming path — see `includeLandmarkPositions`'s
- * own JSDoc for why it has no sample-based equivalent there). Split out from
- * its call site so tests can exercise the boundary with a small injected
- * `threshold` instead of constructing a 20,001-page fixture to cross the
- * real {@link CORPUS_INLINE_THRESHOLD}.
- * @param pageCount
- * @param threshold
- */
-export declare function assertLandmarkPositionsSupportedForPageCount(pageCount: number, threshold: number): void;
 /**
  * Reservoir-sample size per block on the streaming path. Blocks larger than
  * this have Stage A run on a random sample of `BLOCK_SAMPLE_SIZE` pages,
@@ -292,9 +276,6 @@ export declare const BLOCK_SAMPLE_SIZE = 100;
  * @param pages
  * @param options
  */
-export declare function resolvePageClusterKeysInMemory(pages: readonly PageClusterSignals[], options: ResolvePageClusterKeysOptions & {
-    includeLandmarkPositions: true;
-}): PageClusterKeyResult[];
 export declare function resolvePageClusterKeysInMemory(pages: readonly PageClusterSignals[], options?: ResolvePageClusterKeysOptions): string[];
 /**
  * Factory function returning an iterator over pages. Called once per streaming
@@ -358,9 +339,6 @@ export type PageFactory = () => Iterable<PageClusterSignals> | AsyncIterable<Pag
  * });
  * ```
  */
-export declare function resolvePageClusterKeys(pages: PageFactory, options: ResolvePageClusterKeysOptions & {
-    includeLandmarkPositions: true;
-}): Promise<PageClusterKeyResult[]>;
 export declare function resolvePageClusterKeys(pages: PageFactory, options?: ResolvePageClusterKeysOptions): Promise<string[]>;
 /**
  * Convenience wrapper that runs {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}
@@ -378,7 +356,4 @@ export declare function resolvePageClusterKeys(pages: PageFactory, options?: Res
  * ]);
  * ```
  */
-export declare function resolvePageClusterKeysFromArray(pages: readonly PageClusterSignals[], options: ResolvePageClusterKeysOptions & {
-    includeLandmarkPositions: true;
-}): Promise<PageClusterKeyResult[]>;
 export declare function resolvePageClusterKeysFromArray(pages: readonly PageClusterSignals[], options?: ResolvePageClusterKeysOptions): Promise<string[]>;