@d-zero/page-cluster 0.2.0 → 0.3.1

package/dist/resolve-structural-cluster-keys.d.ts

@@ -35,6 +35,15 @@ export type ResolveStructuralClusterKeysOptions = {
  * `tokenSets.length` is large enough for that to be statistically meaningful
  * — below that floor, chrome dilution is accepted as the lesser failure and
  * comparison falls back to the raw sets.
+ *
+ * The clustering algorithm is NN-chain complete-linkage (Murtagh, F., 1983,
+ * "A Survey of Recent Advances in Hierarchical Clustering Algorithms," The
+ * Computer Journal 26(4)), implemented in `completeLinkageDendrogram`. The
+ * dendrogram is cut at `similarityThreshold` by `labelsAtThreshold`, using
+ * Lance-Williams monotonicity (Lance, G. N. & Williams, W. T., 1967, "A
+ * General Theory of Classificatory Sorting Strategies," The Computer Journal
+ * 9(4)) to guarantee that threshold cuts are safe regardless of the order
+ * in which NN-chain discovered the merges.
  * @param tokenSets
  * @param options
  * @example

package/dist/resolve-structural-cluster-keys.js

@@ -1,235 +1,6 @@
-import { computeDocumentFrequency } from './compute-document-frequency.js';
-import { jaccardSimilarity } from './jaccard-similarity.js';
-import { splitTokensByFrequency } from './split-tokens-by-frequency.js';
+import { completeLinkageDendrogram, labelsAtThreshold, } from './complete-linkage-dendrogram.js';
+import { deriveComparisonSets } from './derive-comparison-sets.js';
 const DEFAULT_SIMILARITY_THRESHOLD = 0.8;
-/**
- * `jaccardSimilarity()` returns `intersectionSize / unionSize`, a
- * floating-point division that can land a hair below the caller's intended
- * threshold even when the two are mathematically equal (e.g. a threshold
- * assembled from arithmetic like `0.1 + 0.2` is `0.30000000000000004`, not
- * `0.3`), which would otherwise make a pair at the documented inclusive
- * boundary fail the `>=` check it should pass. Subtracting this epsilon
- * before comparing absorbs that rounding noise (same technique and value as
- * `BOUNDARY_EPSILON` in `split-tokens-by-frequency.ts`).
- */
-const BOUNDARY_EPSILON = 1e-9;
-/**
- * Below this many pages, `computeDocumentFrequency`/`splitTokensByFrequency`
- * (the default 90% cutoff) degenerate rather than usefully separate chrome
- * from content — see `deriveComparisonSets` for the failure mode. Derived
- * from `splitTokensByFrequency`'s own cutoff: a token missing from exactly
- * one page out of `n` still counts as chrome only if
- * `(n - 1) / n >= 0.9`, i.e. `n >= 10`. Below that, this function falls back
- * to comparing `tokenSets` directly, unfiltered.
- */
-const MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT = 10;
-/**
- * Narrows each page's token set to its page-specific content before
- * clustering, so pages that only share site-wide chrome (header/nav/footer)
- * don't read as more similar than they structurally are, and so genuine
- * layout matches aren't swamped by chrome noise at loose thresholds — see
- * `splitTokensByFrequency`'s JSDoc for the two failure modes this fixes.
- * Confirmed on real crawl data (a corporate site using a freeform CMS block
- * editor for its content area): without this, two pages built from the same
- * article template but a different mix of content blocks could score *lower*
- * on raw Jaccard than two pages built from genuinely different templates
- * that happen to share more chrome relative to their (smaller) content area.
- *
- * Skipped entirely below `MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT` pages: at the
- * default 90% cutoff, `splitTokensByFrequency` requires a token to appear on
- * literally every page to count as chrome once `n < 10` (see that constant's
- * JSDoc for the derivation). At `n = 2` this is a total degenerate case, not
- * just an imprecise one — `content(A) = A \ B` and `content(B) = B \ A` are
- * disjoint *by construction* for any two sets, so
- * `jaccardSimilarity(content(A), content(B))` is always `0` unless `A` and
- * `B` are identical, regardless of how similar they actually are (confirmed:
- * two pages sharing 999 of 1000 tokens, differing in exactly one each, go
- * from a raw similarity of `0.998` to a content-only similarity of `0`).
- * Falling back to unfiltered `tokenSets` below the floor accepts chrome
- * dilution for small blocks rather than this much sharper failure.
- *
- * A page whose *entire* token set narrows away (every one of its tokens
- * clears the chrome cutoff) falls back to its own raw tokens rather than the
- * empty result: `jaccardSimilarity` treats two empty sets as similarity `1`
- * (by design, for two genuinely-empty `<body>`s — see its JSDoc), but two
- * different* all-chrome pages narrowing to empty for unrelated reasons
- * (e.g. one page is only a header+footer, another is only a nav) would
- * otherwise be forced into the same cluster by that shortcut regardless of
- * whether their actual structure matches. Falling back only when narrowing
- * collapsed a page to nothing — not for every page — keeps the normal case
- * (a page with at least one page-specific token) unaffected.
- * @param tokenSets
- */
-function deriveComparisonSets(tokenSets) {
-    if (tokenSets.length < MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT) {
-        return tokenSets;
-    }
-    const corpusFrequency = computeDocumentFrequency(tokenSets);
-    return tokenSets.map((tokens) => {
-        const { contentTokens } = splitTokensByFrequency(tokens, corpusFrequency);
-        return contentTokens.size === 0 && tokens.size > 0 ? tokens : contentTokens;
-    });
-}
-/**
- * Reads `values[index]`, throwing instead of returning `undefined`. Every
- * call site here indexes within bounds it just established itself (loop
- * ranges, or an index freshly returned by the same array's own scan), so the
- * thrown branch is unreachable in practice; it exists to satisfy
- * `noUncheckedIndexedAccess` without a non-null assertion (same rationale as
- * `readDpValue` in `array-edit-distance.ts`, generalized to any array-like).
- * Deliberately not exported and shared with `resolve-page-cluster-keys.ts`'s
- * own copy: this file's `export`s are its intended public API surface (the
- * main function and its options type), and every other internal helper here
- * (`find`, `clusterByCompleteLinkage`, `deriveComparisonSets`) is likewise
- * kept unexported — sharing just this one helper across files would carve an
- * exception into that boundary for a ~7-line generic utility.
- * @param values
- * @param index
- */
-function requireIndex(values, index) {
-    const value = values[index];
-    if (value === undefined) {
-        throw new Error('resolveStructuralClusterKeys: index out of bounds');
-    }
-    return value;
-}
-/**
- * Finds the representative (root) of `index`'s set, compressing every
- * traversed link so future lookups on the same path are near-constant time.
- * @param parent
- * @param index
- */
-function find(parent, index) {
-    let root = index;
-    while (requireIndex(parent, root) !== root) {
-        root = requireIndex(parent, root);
-    }
-    let current = index;
-    while (current !== root) {
-        const next = requireIndex(parent, current);
-        parent[current] = root;
-        current = next;
-    }
-    return root;
-}
-/**
- * Complete-linkage hierarchical clustering of `tokenSets` (whatever sets the
- * caller wants compared — `resolveStructuralClusterKeys` passes
- * `deriveComparisonSets`'s output, not necessarily the raw per-page token
- * sets), cut at `threshold`, computed via the NN-chain algorithm (Murtagh,
- * F., 1983, "A
- * Survey of Recent Advances in Hierarchical Clustering Algorithms," The
- * Computer Journal 26(4)). NN-chain produces the exact same dendrogram as
- * naively re-scanning every live cluster pair for the best merge at each
- * step, but in O(n²) time instead of O(n³): each cluster follows a chain of
- * mutually-improving nearest neighbors until it lands on a pair that are
- * each other's nearest neighbor (a "reciprocal nearest neighbor", RNN); that
- * pair's merge is provably a valid next step in the correct dendrogram. This
- * is a genuine algorithmic speedup, not an approximation — see
- * `resolveStructuralClusterKeys`'s JSDoc for why an approximation was
- * rejected.
- *
- * Complete-linkage was chosen over single-linkage (connected components of
- * the threshold graph) because single-linkage's "chaining" lets one
- * unrepresentative page transitively merge two otherwise-unrelated
- * templates — the opposite of what template detection needs. Complete-
- * linkage requires *every* pair across two clusters to clear the threshold
- * before merging them, which rules that out. Cluster-to-cluster similarity
- * is maintained via the Lance-Williams update for complete-linkage:
- * `similarity(merged, Z) = min(similarity(X, Z), similarity(Y, Z))`.
- *
- * The algorithm always runs every one of the `size - 1` possible merges to
- * completion (down to a single root), never stopping early at `threshold`.
- * This looks wasteful but isn't optional: Lance-Williams monotonicity
- * (Lance, G. N. & Williams, W. T., 1967, "A General Theory of Classificatory
- * Sorting Strategies," The Computer Journal 9(4)) guarantees no height
- * inversions inside the dendrogram itself (a merge's similarity is always ≥
- * the similarity of every merge nested inside it), but says nothing about
- * the chronological order in which independent, not-yet-connected
- * chains happen to resolve their own RNN pairs — one chain can easily
- * stumble onto a low-similarity RNN pair before a different, still-unvisited
- * chain uncovers a high-similarity one elsewhere. Stopping the whole
- * algorithm at the first below-threshold merge would therefore discard
- * later, still-valid above-threshold merges (confirmed by this file's
- * differential test against a naive reference — an earlier version of this
- * function that broke early on the first below-threshold RNN pair failed it
- * for exactly this reason). Instead, every merge is always folded into the
- * `active`/`similarity` bookkeeping so the algorithm can keep discovering
- * the rest of the true dendrogram, but only merges scoring `>= threshold`
- * are recorded in `parent` (the union-find used for final membership).
- * Monotonicity guarantees this is safe: any merge scoring `>= threshold` was
- * necessarily built out of children merges that scored at least as high, so
- * restricting the union-find to threshold-clearing merges — regardless of
- * the chronological order they were discovered in — reconstructs exactly
- * the correct threshold cut.
- * @param tokenSets
- * @param threshold
- */
-function clusterByCompleteLinkage(tokenSets, threshold) {
-    const size = tokenSets.length;
-    const parent = Int32Array.from({ length: size }, (_, index) => index);
-    const similarity = new Float64Array(size * size);
-    for (let i = 0; i < size; i++) {
-        for (let j = i + 1; j < size; j++) {
-            const score = jaccardSimilarity(requireIndex(tokenSets, i), requireIndex(tokenSets, j));
-            similarity[i * size + j] = score;
-            similarity[j * size + i] = score;
-        }
-    }
-    const active = new Uint8Array(size).fill(1);
-    const chain = [];
-    const findFreshStart = () => {
-        for (let index = 0; index < size; index++) {
-            if (requireIndex(active, index) === 1) {
-                return index;
-            }
-        }
-        throw new Error('resolveStructuralClusterKeys: no active cluster left to resume from');
-    };
-    let activeCount = size;
-    while (activeCount > 1) {
-        if (chain.length === 0) {
-            chain.push(findFreshStart());
-        }
-        const top = requireIndex(chain, chain.length - 1);
-        let best = -1;
-        let bestScore = Number.NEGATIVE_INFINITY;
-        for (let candidate = 0; candidate < size; candidate++) {
-            if (candidate !== top && requireIndex(active, candidate) === 1) {
-                const score = requireIndex(similarity, top * size + candidate);
-                if (score > bestScore) {
-                    bestScore = score;
-                    best = candidate;
-                }
-            }
-        }
-        const secondFromTop = chain.length >= 2 ? chain.at(-2) : undefined;
-        if (best === secondFromTop) {
-            chain.pop();
-            chain.pop();
-            const survivor = Math.min(top, best);
-            const dead = Math.max(top, best);
-            for (let candidate = 0; candidate < size; candidate++) {
-                if (candidate !== top &&
-                    candidate !== best &&
-                    requireIndex(active, candidate) === 1) {
-                    const merged = Math.min(requireIndex(similarity, top * size + candidate), requireIndex(similarity, best * size + candidate));
-                    similarity[survivor * size + candidate] = merged;
-                    similarity[candidate * size + survivor] = merged;
-                }
-            }
-            active[dead] = 0;
-            if (bestScore >= threshold - BOUNDARY_EPSILON) {
-                parent[find(parent, dead)] = find(parent, survivor);
-            }
-            activeCount--;
-        }
-        else {
-            chain.push(best);
-        }
-    }
-    return Array.from({ length: size }, (_, index) => find(parent, index));
-}
 /**
  * Resolves, within a single already-blocked group of pages (e.g. one key
  * from {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys}),
@@ -253,6 +24,15 @@ function clusterByCompleteLinkage(tokenSets, threshold) {
  * `tokenSets.length` is large enough for that to be statistically meaningful
  * — below that floor, chrome dilution is accepted as the lesser failure and
  * comparison falls back to the raw sets.
+ *
+ * The clustering algorithm is NN-chain complete-linkage (Murtagh, F., 1983,
+ * "A Survey of Recent Advances in Hierarchical Clustering Algorithms," The
+ * Computer Journal 26(4)), implemented in `completeLinkageDendrogram`. The
+ * dendrogram is cut at `similarityThreshold` by `labelsAtThreshold`, using
+ * Lance-Williams monotonicity (Lance, G. N. & Williams, W. T., 1967, "A
+ * General Theory of Classificatory Sorting Strategies," The Computer Journal
+ * 9(4)) to guarantee that threshold cuts are safe regardless of the order
+ * in which NN-chain discovered the merges.
  * @param tokenSets
  * @param options
  * @example
@@ -274,7 +54,9 @@ export function resolveStructuralClusterKeys(tokenSets, options) {
         return [];
     }
     const comparisonSets = deriveComparisonSets(tokenSets);
-    const roots = clusterByCompleteLinkage(comparisonSets, similarityThreshold);
+    const size = comparisonSets.length;
+    const merges = completeLinkageDendrogram(comparisonSets);
+    const roots = labelsAtThreshold(size, merges, similarityThreshold);
     const rootToLabel = new Map();
     return roots.map((root) => {
         let label = rootToLabel.get(root);

package/dist/shape-token.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Stripping classes while keeping the bracket reveals the tag-level (or
+ * wrapper-level) shape: `section.c-page-sub__reports` and
+ * `section.c-page-sub__projects` both shape to `section`, exposing that they
+ * are the same structural element with different BEM class names. Used in
+ * cross-block clustering to merge blocks whose class names differ but whose
+ * skeletons match (confirmed on real crawl data: reports/projects/news list
+ * pages had class-name Jaccard 0.000 and shape Jaccard 1.000).
+ * @param token
+ */
+export declare function shapeToken(token: string): string;

package/dist/shape-token.js

@@ -0,0 +1,38 @@
+/**
+ * @param segment
+ */
+function shapeSegment(segment) {
+    const bracketIndex = segment.indexOf('[');
+    const bracket = bracketIndex === -1 ? '' : segment.slice(bracketIndex);
+    const base = bracketIndex === -1 ? segment : segment.slice(0, bracketIndex);
+    let shapedBase;
+    if (base.startsWith('.')) {
+        // Foldable tag (div/span) with classes: .class1.class2 → *
+        shapedBase = '*';
+    }
+    else {
+        const dotIndex = base.indexOf('.');
+        if (dotIndex > 0) {
+            // Non-foldable tag with classes: tag.class1 → tag
+            shapedBase = base.slice(0, dotIndex);
+        }
+        else {
+            // No classes (tag name only, or empty for bracket-only segments)
+            shapedBase = base;
+        }
+    }
+    return `${shapedBase}${bracket}`;
+}
+/**
+ * Stripping classes while keeping the bracket reveals the tag-level (or
+ * wrapper-level) shape: `section.c-page-sub__reports` and
+ * `section.c-page-sub__projects` both shape to `section`, exposing that they
+ * are the same structural element with different BEM class names. Used in
+ * cross-block clustering to merge blocks whose class names differ but whose
+ * skeletons match (confirmed on real crawl data: reports/projects/news list
+ * pages had class-name Jaccard 0.000 and shape Jaccard 1.000).
+ * @param token
+ */
+export function shapeToken(token) {
+    return token.split('>').map(shapeSegment).join('>');
+}

package/dist/stage-a-per-block.d.ts

@@ -0,0 +1,133 @@
+import type { ExtractLandmarksResult } from './extract-landmarks.js';
+import type { CrossBlockUnit } from './merge-cross-block-clusters.js';
+import type { TokenizeOptions } from './types.js';
+/**
+ * Maximum number of member pages retained per `CrossBlockUnit` for Stage B.
+ * Units larger than this are down-sampled deterministically (URL-independent
+ * — seeded from the unit key) via {@link ./reservoir-sample.js | reservoirSample}.
+ *
+ * ## Why cap
+ *
+ * `CrossBlockUnit.memberTokenSets` is `readonly ReadonlySet<string>[]`.
+ * V8's Set carries substantial per-entry overhead (hash slot + string
+ * reference + backing array padding), so a 200-token set weighs ~20 KB in
+ * practice — an order of magnitude more than the raw byte sum of its
+ * strings. Without a cap, the streaming path's batch-by-batch accumulation
+ * of `CrossBlockUnit`s across a 176k-page corpus reached OOM at ~100k pages
+ * on an 8 GB heap, entirely from Set overhead of retained members.
+ *
+ * ## Why the value
+ *
+ * Stage B's per-round computations (`computeDocumentFrequency`,
+ * `quorumCore`, `shellQuorum`) are frequency-based, so a representative
+ * sample gives statistically similar cores and shells to the full
+ * membership. 100 members is enough for the 80 % quorum threshold to
+ * discriminate signal from noise (needs ≥ 80 of 100 = 80 % vs. the
+ * corpus-wide 80 %), and keeps a per-unit memory footprint of ~2.5 MB
+ * (100 × ~25 KB per member incl. tokens + landmark instances). For a
+ * corpus with ~500 units that's ~1.25 GB — well within an 8 GB heap.
+ *
+ * Applied at unit *creation* here in {@link ./stage-a-per-block.js | stageAPerBlock},
+ * and re-applied after each merge in
+ * {@link ./merge-cross-block-clusters.js | mergeCrossBlockClusters}'s
+ * `applyMerges`, so a merged group can never balloon past this cap either.
+ *
+ * ## In-memory-path preservation
+ *
+ * A block with `≤ MAX_MEMBERS_PER_UNIT` members hits the deterministic
+ * "return the full input unchanged" branch of {@link ./reservoir-sample.js | reservoirSample},
+ * so every previously validated corpus (302 / 1,416 / 8,936 / 89 pages)
+ * keeps every member of every unit — the cap only kicks in for cluster
+ * sizes that could not run under the pre-refactor implementation anyway.
+ */
+export declare const MAX_MEMBERS_PER_UNIT = 100;
+/**
+ * @see stageAPerBlock
+ */
+export type StageAPerBlockOptions = TokenizeOptions & {
+    readonly similarityThreshold?: number;
+    readonly minKneeRatio?: number;
+    readonly maxCandidateDepth?: number;
+    /**
+     * When set, any Stage-A cluster with more than this many members has its
+     * `memberTokenSets` / `memberLandmarkInstances` down-sampled via
+     * {@link ./reservoir-sample.js | reservoirSample} (seed = unit key,
+     * deterministic). Callers on the in-memory path omit this so validated
+     * corpora keep full membership; the streaming path sets it to
+     * {@link MAX_MEMBERS_PER_UNIT} to bound accumulated Stage B state.
+     */
+    readonly capMembers?: number;
+};
+/**
+ * @see stageAPerBlock
+ */
+export type StageAPerBlockInput = {
+    /** The block's block key (as produced by pass 0). */
+    readonly blockKey: string;
+    /**
+     * Original input indices of this block's members, in input order. Returned
+     * unchanged in the mapping so the caller can write pages' cluster keys
+     * back into a global keys array. Same length as `preparedHtml` /
+     * `landmarks` / `localLandmarkTokensByPage`.
+     */
+    readonly memberIndices: readonly number[];
+    /**
+     * The block's per-page landmark-excised (and optionally
+     * `removeContentBlocks`-stripped) HTML — what the in-memory driver calls
+     * `preparedHtml`. Provided by the caller to keep this helper stateless
+     * about `excludeLandmarks` / `contentBlockAttribute` decisions.
+     */
+    readonly preparedHtml: readonly string[];
+    /** The block's per-page {@link ./extract-landmarks.js | ExtractLandmarksResult}. */
+    readonly landmarks: readonly ExtractLandmarksResult[];
+    /**
+     * The block's per-page local-landmark reinjection token sets. Same-shape
+     * output of {@link ./resolve-page-cluster-keys.js | computeLocalLandmarkTokens},
+     * pre-computed by the caller so this helper stays agnostic to whether
+     * chrome discovery ran corpus-wide (small-corpus in-memory path) or
+     * per-block (large-corpus streaming path).
+     */
+    readonly localLandmarkTokensByPage: readonly ReadonlySet<string>[];
+};
+/**
+ * @see stageAPerBlock
+ */
+export type StageAPerBlockResult = {
+    /**
+     * Map from `memberIndices[i]` (original input index) to that page's
+     * post-Stage-A cluster key. Keys are of the form
+     * `JSON.stringify([blockKey, "cluster:N"])` (unchanged from the in-memory
+     * driver's per-block loop) — Stage B later rewrites them into merged
+     * groups when it finds cross-block matches.
+     */
+    readonly pageKeys: ReadonlyMap<number, string>;
+    /**
+     * One {@link ./merge-cross-block-clusters.js | CrossBlockUnit} per
+     * post-Stage-A cluster, in first-seen order.
+     */
+    readonly crossBlockUnits: readonly CrossBlockUnit[];
+};
+/**
+ * Runs Stage A (dendrogram + auto-cut + containment assignment) on one
+ * block's pages, and returns both the per-page cluster keys and the
+ * {@link ./merge-cross-block-clusters.js | CrossBlockUnit} rows that Stage B
+ * needs afterward.
+ *
+ * ## Why extract this from `resolvePageClusterKeys`?
+ *
+ * The in-memory driver holds every page's HTML/landmarks/preparedHtml in
+ * arrays before the per-block loop begins, and any dataset large enough to
+ * break memory does so before Stage A even starts. For streaming mode this
+ * inner per-block loop needs to run for one block at a time: read that
+ * block's HTML, run Stage A, emit its crossBlockUnit rows, free the block's
+ * memory, move to the next. Splitting the Stage A body into a standalone
+ * function is what makes that per-block iteration possible without
+ * duplicating the Stage A logic between the two drivers.
+ *
+ * Preserves the in-memory driver's per-block behavior exactly for the same
+ * inputs — same `preparedHtml`, same `landmarks`, same
+ * `localLandmarkTokensByPage`, same `options` → same output.
+ * @param input
+ * @param options
+ */
+export declare function stageAPerBlock(input: StageAPerBlockInput, options?: StageAPerBlockOptions): StageAPerBlockResult;

package/dist/stage-a-per-block.js

@@ -0,0 +1,178 @@
+import { assignContainedClusters } from './assign-contained-clusters.js';
+import { autoCutThreshold } from './auto-cut-threshold.js';
+import { capContentDepth } from './cap-content-depth.js';
+import { collapseAnonymousDivs } from './collapse-anonymous-divs.js';
+import { completeLinkageDendrogram, labelsAtThreshold, } from './complete-linkage-dendrogram.js';
+import { deriveComparisonSets, MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT, } from './derive-comparison-sets.js';
+import { detectContentDepthCap } from './detect-content-depth-cap.js';
+import { computePerPageLandmarkInstances } from './per-page-landmark-signatures.js';
+import { reservoirSample } from './reservoir-sample.js';
+import { tokenize } from './tokenize.js';
+/**
+ * Maximum number of member pages retained per `CrossBlockUnit` for Stage B.
+ * Units larger than this are down-sampled deterministically (URL-independent
+ * — seeded from the unit key) via {@link ./reservoir-sample.js | reservoirSample}.
+ *
+ * ## Why cap
+ *
+ * `CrossBlockUnit.memberTokenSets` is `readonly ReadonlySet<string>[]`.
+ * V8's Set carries substantial per-entry overhead (hash slot + string
+ * reference + backing array padding), so a 200-token set weighs ~20 KB in
+ * practice — an order of magnitude more than the raw byte sum of its
+ * strings. Without a cap, the streaming path's batch-by-batch accumulation
+ * of `CrossBlockUnit`s across a 176k-page corpus reached OOM at ~100k pages
+ * on an 8 GB heap, entirely from Set overhead of retained members.
+ *
+ * ## Why the value
+ *
+ * Stage B's per-round computations (`computeDocumentFrequency`,
+ * `quorumCore`, `shellQuorum`) are frequency-based, so a representative
+ * sample gives statistically similar cores and shells to the full
+ * membership. 100 members is enough for the 80 % quorum threshold to
+ * discriminate signal from noise (needs ≥ 80 of 100 = 80 % vs. the
+ * corpus-wide 80 %), and keeps a per-unit memory footprint of ~2.5 MB
+ * (100 × ~25 KB per member incl. tokens + landmark instances). For a
+ * corpus with ~500 units that's ~1.25 GB — well within an 8 GB heap.
+ *
+ * Applied at unit *creation* here in {@link ./stage-a-per-block.js | stageAPerBlock},
+ * and re-applied after each merge in
+ * {@link ./merge-cross-block-clusters.js | mergeCrossBlockClusters}'s
+ * `applyMerges`, so a merged group can never balloon past this cap either.
+ *
+ * ## In-memory-path preservation
+ *
+ * A block with `≤ MAX_MEMBERS_PER_UNIT` members hits the deterministic
+ * "return the full input unchanged" branch of {@link ./reservoir-sample.js | reservoirSample},
+ * so every previously validated corpus (302 / 1,416 / 8,936 / 89 pages)
+ * keeps every member of every unit — the cap only kicks in for cluster
+ * sizes that could not run under the pre-refactor implementation anyway.
+ */
+export const MAX_MEMBERS_PER_UNIT = 100;
+/**
+ * Runs Stage A (dendrogram + auto-cut + containment assignment) on one
+ * block's pages, and returns both the per-page cluster keys and the
+ * {@link ./merge-cross-block-clusters.js | CrossBlockUnit} rows that Stage B
+ * needs afterward.
+ *
+ * ## Why extract this from `resolvePageClusterKeys`?
+ *
+ * The in-memory driver holds every page's HTML/landmarks/preparedHtml in
+ * arrays before the per-block loop begins, and any dataset large enough to
+ * break memory does so before Stage A even starts. For streaming mode this
+ * inner per-block loop needs to run for one block at a time: read that
+ * block's HTML, run Stage A, emit its crossBlockUnit rows, free the block's
+ * memory, move to the next. Splitting the Stage A body into a standalone
+ * function is what makes that per-block iteration possible without
+ * duplicating the Stage A logic between the two drivers.
+ *
+ * Preserves the in-memory driver's per-block behavior exactly for the same
+ * inputs — same `preparedHtml`, same `landmarks`, same
+ * `localLandmarkTokensByPage`, same `options` → same output.
+ * @param input
+ * @param options
+ */
+export function stageAPerBlock(input, options) {
+    const { blockKey, memberIndices, preparedHtml, landmarks, localLandmarkTokensByPage } = input;
+    const similarityThreshold = options?.similarityThreshold ?? 0.8;
+    // A block of 1 can never produce more than one cluster regardless of how
+    // it's tokenized — nothing to compare it against — so detecting a knee
+    // and capping for it would only spend a full multi-depth sweep to arrive
+    // back at the same single-cluster result. Skipped rather than swept.
+    const maxMainDepth = preparedHtml.length > 1 ? detectContentDepthCap(preparedHtml, options) : undefined;
+    const blockTokenSets = preparedHtml.map((html, position) => {
+        const capped = maxMainDepth === undefined
+            ? html
+            : capContentDepth(html, { landmark: 'main', maxDepth: maxMainDepth })
+                .remainderHtml;
+        const tokens = new Set(tokenize(capped, options).tokens);
+        // Reinject each page's local-landmark tokens (see
+        // resolve-page-cluster-keys.ts's computeLocalLandmarkTokens JSDoc).
+        const localTokens = localLandmarkTokensByPage[position];
+        if (localTokens !== undefined) {
+            for (const token of localTokens)
+                tokens.add(token);
+        }
+        return tokens;
+    });
+    // Stage A: dendrogram + auto-cut + optional containment assignment
+    const blockSize = blockTokenSets.length;
+    const comparisonSets = deriveComparisonSets(blockTokenSets);
+    const merges = completeLinkageDendrogram(comparisonSets);
+    const cut = autoCutThreshold(merges.map((m) => m.height), similarityThreshold);
+    let roots = labelsAtThreshold(blockSize, merges, cut);
+    // Containment assignment only for blocks large enough to have had
+    // frequency-based comparison sets (same MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT
+    // threshold as deriveComparisonSets).
+    if (blockSize >= MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT) {
+        const clusterTokens = new Map();
+        const clusterPageCount = new Map();
+        for (let i = 0; i < blockSize; i++) {
+            const r = roots[i];
+            let tokens = clusterTokens.get(r);
+            if (!tokens) {
+                tokens = new Set();
+                clusterTokens.set(r, tokens);
+            }
+            for (const t of comparisonSets[i])
+                tokens.add(collapseAnonymousDivs(t));
+            clusterPageCount.set(r, (clusterPageCount.get(r) ?? 0) + 1);
+        }
+        const entries = [...clusterTokens.entries()].map(([id, tokens]) => ({
+            id,
+            tokens: tokens,
+            pageCount: clusterPageCount.get(id) ?? 0,
+        }));
+        const contResult = assignContainedClusters(entries);
+        roots = roots.map((r) => contResult.get(r) ?? r);
+    }
+    // Assign string cluster labels in first-seen order
+    const rootToLabel = new Map();
+    const localLabels = roots.map((root) => {
+        let label = rootToLabel.get(root);
+        if (label === undefined) {
+            label = `cluster:${rootToLabel.size}`;
+            rootToLabel.set(root, label);
+        }
+        return label;
+    });
+    const pageKeys = new Map();
+    const unitKeyToPositions = new Map();
+    for (const [position, memberIndex] of memberIndices.entries()) {
+        const unitKey = JSON.stringify([blockKey, localLabels[position]]);
+        pageKeys.set(memberIndex, unitKey);
+        let positions = unitKeyToPositions.get(unitKey);
+        if (!positions) {
+            positions = [];
+            unitKeyToPositions.set(unitKey, positions);
+        }
+        positions.push(position);
+    }
+    // Pre-tokenize each page's landmark instances once now, and hand Stage B
+    // those instance lists directly instead of the ~10×-larger raw
+    // `ExtractLandmarksResult` objects. Stage B's only consumer of landmark
+    // data (`shellQuorum`) previously re-ran `computePerPageLandmarkInstances`
+    // on every call — the new signature accepts pre-tokenized instances, so
+    // we compute them once per page here and skip the repeat work as a
+    // side benefit. The memory reduction is the primary reason: 176k pages ×
+    // ~1 KB PerPageLandmarkInstance is ~200 MB, versus ~2–3 GB when we kept
+    // full ExtractLandmarksResult objects.
+    const memberLandmarkInstancesByPage = computePerPageLandmarkInstances(landmarks, options);
+    const crossBlockUnits = [];
+    for (const [unitKey, positions] of unitKeyToPositions) {
+        // Down-sample any unit that exceeds `capMembers` (opt-in — the
+        // streaming path passes MAX_MEMBERS_PER_UNIT; the in-memory path
+        // omits the option to preserve full-membership Stage B semantics
+        // unchanged for validated corpora). Reservoir seed is the unit key
+        // so the sampling is deterministic across runs for the same input.
+        const cap = options?.capMembers;
+        const sampledPositions = cap !== undefined && positions.length > cap
+            ? reservoirSample(positions, cap, unitKey)
+            : positions;
+        crossBlockUnits.push({
+            key: unitKey,
+            memberTokenSets: sampledPositions.map((pos) => blockTokenSets[pos]),
+            memberLandmarkInstances: sampledPositions.map((pos) => memberLandmarkInstancesByPage[pos]),
+        });
+    }
+    return { pageKeys, crossBlockUnits };
+}

package/dist/tokenize.d.ts

@@ -53,6 +53,12 @@ export type { TokenizeOptions, TokenizeResult } from './types.js';
  * ```ts
  * tokenize('<body><div class="card"><ul><li>A</li><li>B</li></ul></div></body>');
  * // { tokens: ["body>.card>ul>li", "body>.card>ul>li"], bodyClassList: [] }
+ *
+ * // Disable the built-in hash-noise class filter to keep every class name.
+ * tokenize('<body><div class="c-abc123"><p>x</p></div></body>', {
+ *   filterNoiseClasses: false,
+ * });
+ * // { tokens: ["body>.c-abc123>p"], bodyClassList: [] }
  * ```
  */
 export declare function tokenize(html: string, options?: TokenizeOptions): TokenizeResult;

package/dist/tokenize.js

@@ -53,6 +53,12 @@ import { runTokenizer } from './run-tokenizer.js';
  * ```ts
  * tokenize('<body><div class="card"><ul><li>A</li><li>B</li></ul></div></body>');
  * // { tokens: ["body>.card>ul>li", "body>.card>ul>li"], bodyClassList: [] }
+ *
+ * // Disable the built-in hash-noise class filter to keep every class name.
+ * tokenize('<body><div class="c-abc123"><p>x</p></div></body>', {
+ *   filterNoiseClasses: false,
+ * });
+ * // { tokens: ["body>.c-abc123>p"], bodyClassList: [] }
  * ```
  */
 export function tokenize(html, options) {