@d-zero/page-cluster 0.2.0 → 0.3.0

package/dist/resolve-page-cluster-keys.js

@@ -1,124 +1,327 @@
+import { autoCutThreshold } from './auto-cut-threshold.js';
 import { capContentDepth } from './cap-content-depth.js';
 import { detectContentDepthCap, validateDetectContentDepthCapOptions, } from './detect-content-depth-cap.js';
 import { extractLandmarks } from './extract-landmarks.js';
 import { filterFirstPartyStylesheetHrefs } from './filter-first-party-stylesheet-hrefs.js';
-import { mergeLandmarkAffinedClusters, validateMergeLandmarkAffinedClustersOptions, } from './merge-landmark-affined-clusters.js';
-import { reassignOrphanBlockKeys } from './reassign-orphan-block-keys.js';
+import { jaccardSimilarity } from './jaccard-similarity.js';
+import { mergeCrossBlockClusters } from './merge-cross-block-clusters.js';
+import { groupIndicesByBlockKey, resolveBlockKeys } from './pass0-blocking.js';
+import { computePerPageLandmarkInstances } from './per-page-landmark-signatures.js';
 import { removeContentBlocks } from './remove-content-blocks.js';
-import { resolveBlockingGroupKeys } from './resolve-blocking-group-keys.js';
-import { resolveStructuralClusterKeys } from './resolve-structural-cluster-keys.js';
+import { stageAPerBlock } from './stage-a-per-block.js';
 import { tokenize } from './tokenize.js';
 /**
- * Reads `values[index]`, throwing instead of returning `undefined`. Every
- * call site here indexes with a position this function generated itself
- * (an entry from `Map#entries()`/`Array#entries()` over an array it just
- * built), so the thrown branch is unreachable in practice; it exists to
- * satisfy `noUncheckedIndexedAccess` without a non-null assertion. Not
- * imported from `resolve-structural-cluster-keys.ts`'s own copy: that file's
- * `export`s are its intended public API surface, and this ~7-line generic
- * helper isn't worth carving an exception into that boundary for (same
- * rationale as `readDpValue` in `array-edit-distance.ts` being its own
- * independent copy rather than a shared import).
- * @param values
- * @param index
+ * FNV-1a 32-bit hash of a string, used to seed the per-block PRNG so
+ * reservoir sampling on the streaming path is deterministic for a given
+ * corpus (same input order → same sampled indices → same cluster keys).
+ * @param input
  */
-function requireIndex(values, index) {
-    const value = values[index];
-    if (value === undefined) {
-        throw new Error('resolvePageClusterKeys: index out of bounds');
+function fnv1a32(input) {
+    let hash = 0x81_1c_9d_c5;
+    for (let i = 0; i < input.length; i++) {
+        hash ^= input.codePointAt(i) ?? 0;
+        hash = Math.imul(hash, 0x01_00_01_93);
     }
-    return value;
+    return hash >>> 0;
 }
 /**
- * Narrows `landmarks` from `readonly ExtractLandmarksResult[] | undefined` to
- * defined, throwing instead of asserting non-null. Always defined in
- * practice at every call site below: both call sites are only reached when
- * `excludeLandmarks || mergeRareLandmarkClusters` is true, exactly the
- * condition under which `landmarks` is populated. Exists only to satisfy the
- * type checker without a non-null assertion (same rationale as
- * `requireIndex` above).
+ * Mulberry32 — small, well-known 32-bit PRNG. Kept independent per block
+ * (each block seeds from its own block key via {@link ./resolve-page-cluster-keys.js | fnv1a32})
+ * so different blocks sample independently.
+ * @param seed
+ */
+function makeSeededPrng(seed) {
+    let state = (typeof seed === 'string' ? fnv1a32(seed) : seed) >>> 0;
+    return () => {
+        state = (state + 0x6d_2b_79_f5) >>> 0;
+        let t = state;
+        t = Math.imul(t ^ (t >>> 15), t | 1);
+        t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+        return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296;
+    };
+}
+/**
+ * Tokenizes a non-sample page using the block's learned parameters
+ * (`maxMainDepth` and local-signature reinjection set) and returns the
+ * key of the sample-derived cluster whose members it most closely matches
+ * by Jaccard similarity. Ties break in first-seen order (JS `Map`
+ * iteration). Falls back to a block-scoped singleton key when the block
+ * has no clusters at all (edge case: an empty sample, which shouldn't
+ * happen for a non-empty block but is defended against here).
+ * @param html
+ * @param assignment
+ * @param assignment.maxMainDepth
+ * @param assignment.localSignatures
+ * @param assignment.clustersByUnitKey
+ * @param excludeLandmarks
+ * @param contentBlockAttribute
+ * @param tokenizeOptions
+ * @param blockKey
+ */
+function assignPageToNearestCluster(html, assignment, excludeLandmarks, contentBlockAttribute, tokenizeOptions, blockKey) {
+    const landmarkResult = extractLandmarks(html);
+    const landmarksExcised = excludeLandmarks ? landmarkResult.remainderHtml : html;
+    let prepared = contentBlockAttribute === undefined
+        ? landmarksExcised
+        : removeContentBlocks(landmarksExcised, { blockAttribute: contentBlockAttribute })
+            .remainderHtml;
+    if (assignment.maxMainDepth !== undefined) {
+        prepared = capContentDepth(prepared, {
+            landmark: 'main',
+            maxDepth: assignment.maxMainDepth,
+        }).remainderHtml;
+    }
+    const pageTokens = new Set(tokenize(prepared, tokenizeOptions).tokens);
+    // Reinject tokens for landmark instances whose signature matches the
+    // block's learned local-signature set (same rule the sample-side Stage
+    // A applied via computeLocalChromeArtifacts).
+    if (assignment.localSignatures.size > 0) {
+        const instances = computePerPageLandmarkInstances([landmarkResult], tokenizeOptions)[0];
+        for (const inst of instances) {
+            if (!assignment.localSignatures.has(inst.signature))
+                continue;
+            for (const t of inst.tokens)
+                pageTokens.add(t);
+        }
+    }
+    let bestKey;
+    let bestScore = -1;
+    for (const [unitKey, memberTokenSets] of assignment.clustersByUnitKey) {
+        let clusterBest = 0;
+        for (const memberTokens of memberTokenSets) {
+            const score = jaccardSimilarity(pageTokens, memberTokens);
+            if (score > clusterBest)
+                clusterBest = score;
+        }
+        if (clusterBest > bestScore) {
+            bestScore = clusterBest;
+            bestKey = unitKey;
+        }
+    }
+    return bestKey ?? JSON.stringify([blockKey, 'cluster:unassigned']);
+}
+/**
+ * Reinjects each page's *local* (non-corpus-wide) landmark-instance tokens
+ * into its block token set for Stage A clustering, restoring exactly the
+ * structural signal that landmark excision removed for those pages while
+ * keeping global chrome removed (the whole point of `excludeLandmarks`).
+ *
+ * ## Why token-level reinjection instead of one opaque pseudo-token
+ *
+ * An earlier iteration returned a single opaque token per local signature.
+ * That failed on real data: adding one distinctive token to a 100+-token
+ * page's set produces jaccard ~0.99 between "with-local-landmark" and
+ * "without-local-landmark" siblings, so Stage A's 0.8-clamped auto-cut
+ * silently merged them anyway. Reinjecting the landmark's actual tokens
+ * (typically 4–20 per landmark) restores the full structural weight of
+ * the distinction. A real mid-sized crawl corpus's section subtree with
+ * a shared section-local `<nav>` now splits correctly from siblings
+ * without one, since the reinjected local-nav tokens push jaccard below
+ * the cut.
+ *
+ * ## The corpus-level auto-cut
+ *
+ * Every page's landmark instances are canonicalized to a signature (via
+ * {@link ./canonicalize-token-set.js | canonicalizeTokenSet}); the corpus-
+ * wide histogram of "how many pages carry this signature" is fed to
+ * {@link ./auto-cut-threshold.js | autoCutThreshold} — the same primitive
+ * used at every other layer of this pipeline for merge-height cutoffs. The
+ * clamp caps the auto-cut at 0.8 so it never picks a threshold *above* the
+ * conservative default. A signature at or above the cut is global chrome —
+ * appears on effectively every page, so its tokens carry no discriminatory
+ * signal and are left excised. A signature below the cut is local chrome
+ * for the pages that carry it, and its tokens are reinjected into those
+ * pages' block token sets. Same technique as the per-unit shellQuorum in
+ * {@link ./merge-cross-block-clusters.js | mergeCrossBlockClusters}, one
+ * layer up.
+ *
+ * ## The `count >= 2` gate
+ *
+ * A signature present on exactly one page is per-page variation, not
+ * shared local chrome — no "these pages have the same local chrome, those
+ * pages don't" grouping can be built from a singleton, and admitting
+ * singleton signatures would reinject each per-page-unique landmark into
+ * exactly one page's token set, causing spurious per-page cluster
+ * fragmentation across the corpus (confirmed against a 2-page fixture
+ * where two pages carry byte-different `<header>`s produced identical
+ * clusters as expected; without the gate, each would carry its own
+ * reinjected tokens and split).
+ * @param landmarks
+ * @param tokenizeOptions
+ */
+/**
+ * Companion to {@link ./resolve-page-cluster-keys.js | computeLocalLandmarkTokens}
+ * that also returns the local-signature *set* the streaming path needs to
+ * reuse when tokenizing non-sample pages during Pass 1b. The in-memory path
+ * only cares about the per-page token sets (which pages carry which
+ * chrome-below-the-cut tokens); the streaming path additionally needs to
+ * apply the *same* "which signatures are local" verdict to pages that were
+ * not part of the sample the verdict was learned from.
  * @param landmarks
+ * @param tokenizeOptions
  */
-function requireLandmarks(landmarks) {
-    if (landmarks === undefined) {
-        throw new Error('resolvePageClusterKeys: landmarks were not computed');
+export function computeLocalChromeArtifacts(landmarks, tokenizeOptions) {
+    const pageCount = landmarks.length;
+    if (pageCount === 0)
+        return { localSignatures: new Set(), localTokensByPage: [] };
+    const perPageInstances = computePerPageLandmarkInstances(landmarks, tokenizeOptions);
+    // Corpus-wide histogram: signature → { count, tokens }. tokens is the
+    // token set of any one occurrence of the signature (all occurrences are
+    // equal by construction).
+    const corpusHistogram = new Map();
+    for (const instances of perPageInstances) {
+        for (const inst of instances) {
+            const entry = corpusHistogram.get(inst.signature);
+            if (entry) {
+                entry.count++;
+            }
+            else {
+                corpusHistogram.set(inst.signature, { count: 1, tokens: inst.tokens });
+            }
+        }
+    }
+    if (corpusHistogram.size === 0) {
+        return {
+            localSignatures: new Set(),
+            localTokensByPage: landmarks.map(() => new Set()),
+        };
+    }
+    const frequencies = [];
+    for (const entry of corpusHistogram.values()) {
+        frequencies.push(entry.count / pageCount);
+    }
+    const cut = autoCutThreshold(frequencies, 0.8);
+    // Signatures whose tokens we'll reinject: below cut, non-singleton.
+    const localSignatures = new Set();
+    for (const [sig, entry] of corpusHistogram) {
+        if (entry.count >= 2 && entry.count / pageCount < cut) {
+            localSignatures.add(sig);
+        }
+    }
+    if (localSignatures.size === 0) {
+        return {
+            localSignatures: new Set(),
+            localTokensByPage: landmarks.map(() => new Set()),
+        };
     }
-    return landmarks;
+    const localTokensByPage = perPageInstances.map((instances) => {
+        const out = new Set();
+        for (const inst of instances) {
+            if (!localSignatures.has(inst.signature))
+                continue;
+            for (const token of inst.tokens)
+                out.add(token);
+        }
+        return out;
+    });
+    return { localSignatures, localTokensByPage };
+}
+/**
+ * Reinjects each page's *local* (non-corpus-wide) landmark-instance tokens
+ * into its block token set for Stage A clustering, restoring exactly the
+ * structural signal that landmark excision removed for those pages while
+ * keeping global chrome removed (the whole point of `excludeLandmarks`).
+ *
+ * See {@link ./resolve-page-cluster-keys.js | computeLocalChromeArtifacts}
+ * for the underlying algorithm — this function is a thin wrapper that
+ * discards the local-signature set, exposed for callers that only need the
+ * per-page tokens (the in-memory driver's use case).
+ * @param landmarks
+ * @param tokenizeOptions
+ */
+export function computeLocalLandmarkTokens(landmarks, tokenizeOptions) {
+    return [...computeLocalChromeArtifacts(landmarks, tokenizeOptions).localTokensByPage];
 }
 /**
- * Connects the two stages this package otherwise leaves for the caller to
- * wire together: {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys}
- * (coarse blocking by URL path or stylesheet set) and
- * {@link ./resolve-structural-cluster-keys.js | resolveStructuralClusterKeys}
- * (exact structural clustering *within* one block). Returns one final key
- * per page, in the same order as `pages`, unique across the whole input —
- * not just within a block.
- *
- * `resolveStructuralClusterKeys` numbers its clusters `cluster:0`,
- * `cluster:1`, ... independently every time it's called, so two different
- * blocks' `cluster:0` are unrelated but identically named. Composing the
- * block key and the per-block cluster label via `JSON.stringify` (rather
- * than plain string concatenation, e.g. a `::` separator) rules out
- * collisions regardless of what either half happens to contain, without
- * depending on `resolveStructuralClusterKeys`'s label format never changing.
- *
- * `excludeLandmarks` and `similarityThreshold` interact: removing shared
- * chrome makes every remaining comparison stricter (there's no more
- * chrome-driven baseline similarity propping scores up), so a threshold
- * tuned against raw, chrome-included tokens can become too strict once
- * landmarks are excluded. Confirmed on real crawl data: a 4-page block where
- * 3 same-template pages merged at the default `similarityThreshold` (0.8)
- * using raw tokens split one of the 3 into its own singleton once landmarks
- * were excluded, and re-merged correctly at `similarityThreshold: 0.6` — re-
- * tune per site after switching this on, the same as `similarityThreshold`
- * itself already needs.
- *
- * `reassignOrphans` only ever pools a `path:`-fallback orphan alongside a
- * same-section `css:` block for `resolveStructuralClusterKeys` to compare —
- * it never forces a merge itself. An orphan that turns out not to match
- * anything in that pool (confirmed on real crawl data) correctly surfaces as
- * its own singleton, the same as it would have without this option.
- *
- * `restrictStylesheetsToFirstParty` runs before `reassignOrphans`: a page
- * whose only stylesheet reference was third-party becomes an orphan (no
- * first-party stylesheet left) *because of* the filtering, and is then
- * itself eligible for orphan reassignment — this is intentional, not an
- * ordering accident, since the underlying reason both options exist is the
- * same (a page's blocking key should reflect its template, not incidental
- * third-party embeds or missing crawl data).
+ * Corpus size at or below which the async factory-based
+ * `resolvePageClusterKeys` reads the entire input into an array and delegates
+ * to {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeysInMemory}
+ * unchanged — preserving corpus-wide semantics (chrome discovery, Stage B
+ * across all pages) exactly.
+ *
+ * Above this threshold, the streaming path takes over: block dispatch during
+ * a second factory read, per-block chrome discovery (semantic drift from
+ * corpus-wide, unavoidable when the whole corpus does not fit in memory),
+ * and Stage B fed with the incrementally-accumulated cross-block units.
+ *
+ * Chosen from Phase 0 spike measurements: a ~9,000-page real crawl (biggest
+ * block ~3,900) completed in ~108s / 1.25 GB heap on the in-memory path.
+ * Doubling that headroom to 20,000 keeps every corpus previously validated
+ * (302, 1,416, 8,936, 89 pages) on the exact code path they were validated
+ * against, so their gate values (9 / 21 / 63 / 3 clusters respectively)
+ * remain byte-reproducible. A ~176,000-page real crawl OOM'd on the in-memory
+ * path well below this threshold worth of pages ever being materialized, so
+ * anything above 20,000 is routed to streaming.
+ */
+export const CORPUS_INLINE_THRESHOLD = 20_000;
+/**
+ * Reservoir-sample size per block on the streaming path. Blocks larger than
+ * this have Stage A run on a random sample of `BLOCK_SAMPLE_SIZE` pages,
+ * with the remaining non-sample pages assigned via Jaccard similarity to
+ * the sample-derived clusters during Pass 1b. Blocks at or below this size
+ * still work — the sampling degenerates to "keep every input page unchanged"
+ * per the `reservoirSample` contract, so small blocks behave identically to
+ * the in-memory path.
+ *
+ * Chosen to bound accumulated Stage-B state: units × sample_size × per-
+ * member memory ≈ 200 units × 100 members × 25 KB ≈ 500 MB, well within an
+ * 8 GB Node heap even on macOS where jetsam (the kernel OOM killer) reacts
+ * to RSS pressure before V8's own heap limit trips.
+ *
+ * ## Semantic differences from the in-memory path
+ *
+ * - **Chrome discovery is sample-based per block.** Landmark signatures that
+ *   are rare in the sample get treated as global chrome; only signatures
+ *   that show up on ≥ 2 sample members and below the sample-derived
+ *   auto-cut are reinjected. Full-block chrome discovery would see rare
+ *   signatures too — the sample-based decision approximates it.
+ * - **Non-sample pages are assigned by max-Jaccard against sample member
+ *   token sets.** A page whose closest sample member is genuinely dissimilar
+ *   still gets slotted into the least-bad cluster; this is a pragmatic
+ *   trade for a bounded assignment cost (no unbounded "outlier" cluster
+ *   growth).
+ * - **Stage B sees the sample-based `CrossBlockUnit`s only.** Non-sample
+ *   pages carry the final key that Stage B produces for their assigned
+ *   sample cluster, without contributing to Stage B's own DF / quorum-core
+ *   / shell-quorum computations.
+ *
+ * Preserves the in-memory path unchanged for corpora at or below
+ * {@link CORPUS_INLINE_THRESHOLD} — sampling is streaming-mode only.
+ */
+export const BLOCK_SAMPLE_SIZE = 100;
+/**
+ * Preserves the previous synchronous, array-in / array-out API of
+ * `resolvePageClusterKeys` under a new name so the factory-based async
+ * export can take the primary name while callers that already had a
+ * materialized page array (spec tests, the in-repo dogfood harness,
+ * downstream code that hasn't switched to streaming yet) retain the
+ * exact same behavior.
+ *
+ * Semantics: identical to the pre-refactor `resolvePageClusterKeys`.
+ * Corpus-wide chrome discovery, Stage B across every page, no memory
+ * bound — meant to be called on inputs already known to fit in memory.
+ * The async factory-based export delegates here whenever
+ * `pages.length ≤ CORPUS_INLINE_THRESHOLD`, guaranteeing existing corpora
+ * hit exactly this code path.
  * @param pages
  * @param options
- * @example
- * ```ts
- * resolvePageClusterKeys([
- * 	{ paths: ['news', '1'], stylesheetHrefs: [], html: '<body><article>one</article></body>' },
- * 	{ paths: ['news', '2'], stylesheetHrefs: [], html: '<body><article>two</article></body>' },
- * 	{ paths: ['about'], stylesheetHrefs: [], html: '<body><section>about</section></body>' },
- * ]);
- * // pages 0 and 1 (same block, same structure) share a key; page 2 (different block) gets its own
- * ```
  */
-export function resolvePageClusterKeys(pages, options) {
+export function resolvePageClusterKeysInMemory(pages, options) {
     const excludeLandmarks = options?.excludeLandmarks ?? true;
-    const mergeRareLandmarkClusters = options?.mergeRareLandmarkClusters ?? false;
-    if (mergeRareLandmarkClusters) {
-        // Eager, same rationale as autoCapMainDepth's own eager validation
-        // below: this option's own validation is otherwise only reached from
-        // mergeLandmarkAffinedClusters's call at the very end of this
-        // function, which never runs at all for an empty `pages`.
-        validateMergeLandmarkAffinedClustersOptions(options);
-    }
-    // extractLandmarks parses the whole page once; excludeLandmarks needs
-    // remainderHtml and mergeRareLandmarkClusters needs the four landmark
-    // fields, so this computes it once up front whenever either is needed,
-    // rather than each option branch parsing independently.
-    const landmarks = excludeLandmarks || mergeRareLandmarkClusters
-        ? pages.map((page) => extractLandmarks(page.html))
-        : undefined;
+    const similarityThreshold = options?.similarityThreshold ?? 0.8;
+    if (!(similarityThreshold >= 0 && similarityThreshold <= 1)) {
+        throw new RangeError(`resolvePageClusterKeys: similarityThreshold must be between 0 and 1, got ${similarityThreshold}`);
+    }
+    // Always computed: landmark fields are needed by Stage B's shell
+    // corroboration regardless of `excludeLandmarks`, and `remainderHtml` is
+    // needed whenever `excludeLandmarks` is true.
+    const landmarks = pages.map((page) => extractLandmarks(page.html));
+    // Corpus-level chrome discovery
+    const localLandmarkTokensByPage = computeLocalLandmarkTokens(landmarks, options);
     const contentBlockAttribute = options?.contentBlockAttribute;
     const preparedHtml = pages.map((page, index) => {
         const landmarksExcised = excludeLandmarks
-            ? requireIndex(requireLandmarks(landmarks), index).remainderHtml
+            ? landmarks[index].remainderHtml
             : page.html;
         return contentBlockAttribute === undefined
             ? landmarksExcised
@@ -129,70 +332,418 @@ export function resolvePageClusterKeys(pages, options) {
     const blockingPages = restrictStylesheetsToFirstParty
         ? filterFirstPartyStylesheetHrefs(pages)
         : pages;
-    const reassignOrphans = options?.reassignOrphans ?? true;
-    const rawBlockKeys = resolveBlockingGroupKeys(blockingPages, options);
-    const blockKeys = reassignOrphans
-        ? reassignOrphanBlockKeys(blockingPages, rawBlockKeys, options?.pathDepth)
-        : rawBlockKeys;
-    const indicesByBlockKey = new Map();
-    for (const [index, blockKey] of blockKeys.entries()) {
-        const indices = indicesByBlockKey.get(blockKey);
-        if (indices) {
-            indices.push(index);
+    const blockKeys = resolveBlockKeys(blockingPages, options);
+    const indicesByBlockKey = groupIndicesByBlockKey(blockKeys);
+    // Validated here, eagerly, because it's otherwise only reached from
+    // inside the per-block loop below — which never runs at all for an empty
+    // `pages` (no blocks), silently skipping a bad option instead of failing
+    // fast the way a direct `detectContentDepthCap` call always does.
+    validateDetectContentDepthCapOptions(options);
+    const finalKeys = Array.from({ length: pages.length });
+    const crossBlockUnits = [];
+    for (const [blockKey, indices] of indicesByBlockKey) {
+        const result = stageAPerBlock({
+            blockKey,
+            memberIndices: indices,
+            preparedHtml: indices.map((i) => preparedHtml[i]),
+            landmarks: indices.map((i) => landmarks[i]),
+            localLandmarkTokensByPage: indices.map((i) => localLandmarkTokensByPage[i]),
+        }, options);
+        for (const [pageIndex, key] of result.pageKeys) {
+            finalKeys[pageIndex] = key;
         }
-        else {
-            indicesByBlockKey.set(blockKey, [index]);
+        crossBlockUnits.push(...result.crossBlockUnits);
+    }
+    // Stage B: cross-block merge — always runs regardless of options
+    const stageBResult = mergeCrossBlockClusters(crossBlockUnits, options);
+    for (let i = 0; i < finalKeys.length; i++) {
+        const currentKey = finalKeys[i];
+        const rootKey = stageBResult.get(currentKey);
+        if (rootKey !== undefined && rootKey !== currentKey) {
+            finalKeys[i] = rootKey;
         }
     }
-    const autoCapMainDepth = options?.autoCapMainDepth ?? false;
-    if (autoCapMainDepth) {
-        // Validated here, eagerly, because it's otherwise only reached from
-        // inside the per-block loop below — which never runs at all for an
-        // empty `pages` (no blocks), silently skipping a bad option instead
-        // of failing fast the way a direct `detectContentDepthCap` call
-        // always does.
-        validateDetectContentDepthCapOptions(options);
+    return finalKeys;
+}
+/**
+ * Async twin of {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeysInMemory}
+ * that emits `pass1-block-complete` (per block) and `stage-b-start`
+ * `ProgressEvent`s and yields control back to the event loop between
+ * blocks with `setImmediate`, so the async factory-based
+ * `resolvePageClusterKeys` can expose live progress on small corpora
+ * (`≤ CORPUS_INLINE_THRESHOLD`) without blocking the caller's UI thread.
+ *
+ * Semantic equivalence with `resolvePageClusterKeysInMemory` is preserved
+ * exactly: same corpus-wide chrome discovery, same per-block Stage A, same
+ * un-capped Stage B across the entire crossBlockUnits array. `finalKeys`
+ * returned here must be byte-for-byte identical to what the sync path
+ * would have produced for the same `pages` input — spec-enforced by
+ * `resolve-page-cluster-keys-streaming.spec.ts`.
+ *
+ * The sync `resolvePageClusterKeysInMemory` is deliberately left in place
+ * as its own implementation rather than being folded into a shared helper.
+ * The intentional duplication guarantees that library callers who pass no
+ * `onProgress` incur zero behavioral difference from the pre-refactor code
+ * (see the `onProgress === undefined` short-circuit in
+ * {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}).
+ * @param pages
+ * @param onProgress
+ * @param options
+ */
+async function resolveSmallCorpusWithProgress(pages, onProgress, options) {
+    const excludeLandmarks = options?.excludeLandmarks ?? true;
+    const similarityThreshold = options?.similarityThreshold ?? 0.8;
+    if (!(similarityThreshold >= 0 && similarityThreshold <= 1)) {
+        throw new RangeError(`resolvePageClusterKeys: similarityThreshold must be between 0 and 1, got ${similarityThreshold}`);
     }
+    const landmarks = pages.map((page) => extractLandmarks(page.html));
+    const localLandmarkTokensByPage = computeLocalLandmarkTokens(landmarks, options);
+    const contentBlockAttribute = options?.contentBlockAttribute;
+    const preparedHtml = pages.map((page, index) => {
+        const landmarksExcised = excludeLandmarks
+            ? landmarks[index].remainderHtml
+            : page.html;
+        return contentBlockAttribute === undefined
+            ? landmarksExcised
+            : removeContentBlocks(landmarksExcised, { blockAttribute: contentBlockAttribute })
+                .remainderHtml;
+    });
+    const restrictStylesheetsToFirstParty = options?.restrictStylesheetsToFirstParty ?? true;
+    const blockingPages = restrictStylesheetsToFirstParty
+        ? filterFirstPartyStylesheetHrefs(pages)
+        : pages;
+    const blockKeys = resolveBlockKeys(blockingPages, options);
+    const indicesByBlockKey = groupIndicesByBlockKey(blockKeys);
+    validateDetectContentDepthCapOptions(options);
     const finalKeys = Array.from({ length: pages.length });
+    const crossBlockUnits = [];
+    const totalBlocks = indicesByBlockKey.size;
+    let blocksProcessed = 0;
     for (const [blockKey, indices] of indicesByBlockKey) {
-        const blockPreparedHtml = indices.map((index) => requireIndex(preparedHtml, index));
-        // A block of 1 can never produce more than one cluster regardless of
-        // how it's tokenized — nothing to compare it against — so detecting a
-        // knee and capping for it would only spend a full multi-depth sweep
-        // (see detectContentDepthCap's own cost notes) to arrive back at the
-        // same single-cluster result. Skipped rather than swept.
-        const maxMainDepth = autoCapMainDepth && blockPreparedHtml.length > 1
-            ? detectContentDepthCap(blockPreparedHtml, options)
-            : undefined;
-        const blockTokenSets = blockPreparedHtml.map((html) => {
-            const capped = maxMainDepth === undefined
-                ? html
-                : capContentDepth(html, { landmark: 'main', maxDepth: maxMainDepth })
-                    .remainderHtml;
-            return new Set(tokenize(capped, options).tokens);
+        const result = stageAPerBlock({
+            blockKey,
+            memberIndices: indices,
+            preparedHtml: indices.map((i) => preparedHtml[i]),
+            landmarks: indices.map((i) => landmarks[i]),
+            localLandmarkTokensByPage: indices.map((i) => localLandmarkTokensByPage[i]),
+        }, options);
+        for (const [pageIndex, key] of result.pageKeys) {
+            finalKeys[pageIndex] = key;
+        }
+        crossBlockUnits.push(...result.crossBlockUnits);
+        blocksProcessed++;
+        onProgress({
+            phase: 'pass1-block-complete',
+            blockKey,
+            blocksProcessed,
+            totalBlocks,
         });
-        const localLabels = resolveStructuralClusterKeys(blockTokenSets, options);
-        for (const [position, index] of indices.entries()) {
-            finalKeys[index] = JSON.stringify([blockKey, requireIndex(localLabels, position)]);
-        }
-    }
-    if (!mergeRareLandmarkClusters) {
-        return finalKeys;
-    }
-    // Deliberately not a reuse of blockTokenSets (this function's own
-    // primary-clustering token sets): those follow excludeLandmarks (raw
-    // HTML, landmarks included, when false) and resolveStructuralClusterKeys
-    // narrows them further via its internal deriveComparisonSets once a
-    // block reaches 10+ pages. mergeLandmarkAffinedClusters's secondary
-    // content-similarity gate is meant to be independent corroboration
-    // alongside the landmark match already used to select these pages — if
-    // it reused landmark-inclusive tokens, a bulky shared rare header's own
-    // tokens could inflate two otherwise-unrelated pages' similarity past
-    // the gate, and if it reused the frequency-narrowed set, the gate would
-    // silently test different content than its own JSDoc describes. Always
-    // tokenizing each page's landmark-excised remainderHtml here keeps the
-    // gate's evidence independent of both.
-    const resolvedLandmarks = requireLandmarks(landmarks);
-    const mergeGateContentTokenSets = resolvedLandmarks.map((entry) => new Set(tokenize(entry.remainderHtml, options).tokens));
-    return mergeLandmarkAffinedClusters(finalKeys, resolvedLandmarks, mergeGateContentTokenSets, options);
+        // Yield to the event loop so Lanes' setTimeout frame can paint the
+        // updated header before the next block starts.
+        await new Promise((resolve) => setImmediate(resolve));
+    }
+    onProgress({ phase: 'stage-b-start', unitCount: crossBlockUnits.length });
+    const stageBResult = mergeCrossBlockClusters(crossBlockUnits, options);
+    for (let i = 0; i < finalKeys.length; i++) {
+        const currentKey = finalKeys[i];
+        const rootKey = stageBResult.get(currentKey);
+        if (rootKey !== undefined && rootKey !== currentKey) {
+            finalKeys[i] = rootKey;
+        }
+    }
+    return finalKeys;
+}
+/**
+ * Streaming, memory-bounded version of `resolvePageClusterKeysInMemory`.
+ *
+ * ## Behavior gate
+ *
+ * - `pageCount ≤ CORPUS_INLINE_THRESHOLD` — reads the whole factory into an
+ *   array, delegates to `resolvePageClusterKeysInMemory`. Same corpus-wide
+ *   chrome discovery, same Stage B across every page. All previously
+ *   validated corpora (302 / 1,416 / 8,936 / 89 pages) hit this path.
+ * - `pageCount > CORPUS_INLINE_THRESHOLD` — streaming path: reads the
+ *   factory twice (once for blocking signals, once for HTML processing),
+ *   dispatches HTML per block, runs Stage A per block, accumulates
+ *   cross-block units, then runs Stage B across all accumulated units. Peak
+ *   memory ≈ largest single block, not the whole corpus.
+ *
+ * ## Semantic differences in streaming mode
+ *
+ * - **Chrome discovery is per-block, not corpus-wide.** In the in-memory
+ *   path, {@link ./resolve-page-cluster-keys.js | computeLocalLandmarkTokens}
+ *   runs on all pages at once. In streaming mode the entire corpus cannot
+ *   be held at once, so chrome discovery runs per block. A landmark
+ *   signature that is rare corpus-wide but common within one block will
+ *   be treated as global chrome in streaming mode, whereas the in-memory
+ *   mode would treat it as local. This trade-off is why the threshold
+ *   above is set generously — every real corpus historically validated
+ *   here stays on the in-memory path.
+ * @param pages
+ * @param options
+ * @example
+ * ```ts
+ * // JSONL file source — factory can be re-invoked to re-open the file.
+ * import { createReadStream } from 'node:fs';
+ * import readline from 'node:readline';
+ *
+ * const keys = await resolvePageClusterKeys(() => {
+ *   const lines = readline.createInterface({ input: createReadStream('pages.jsonl') });
+ *   return (async function* () {
+ *     for await (const line of lines) yield JSON.parse(line);
+ *   })();
+ * });
+ * ```
+ */
+export async function resolvePageClusterKeys(pages, options) {
+    const onProgress = options?.onProgress;
+    // Pass 0: HTML-free — collect blocking signals (paths, stylesheetHrefs,
+    // host) into an array. This is the only per-page state we keep across
+    // the whole corpus in streaming mode.
+    const blockingSignals = [];
+    for await (const page of pages()) {
+        if (onProgress && blockingSignals.length > 0 && blockingSignals.length % 1000 === 0) {
+            onProgress({ phase: 'pass0-signals', pagesSeen: blockingSignals.length });
+        }
+        blockingSignals.push({
+            paths: page.paths,
+            stylesheetHrefs: page.stylesheetHrefs,
+            host: page.host,
+        });
+    }
+    if (blockingSignals.length === 0)
+        return [];
+    // Small corpus: read the whole factory again with HTML this time, then
+    // delegate to the in-memory path. Preserves every corpus-wide semantic
+    // (chrome, Stage B) for corpora within the threshold.
+    if (blockingSignals.length <= CORPUS_INLINE_THRESHOLD) {
+        const fullPages = [];
+        for await (const page of pages()) {
+            fullPages.push({
+                paths: page.paths,
+                stylesheetHrefs: page.stylesheetHrefs,
+                html: page.html,
+                host: page.host,
+            });
+        }
+        // Without an onProgress callback the caller does not need visibility
+        // into per-block progress, so delegate to the untouched sync path —
+        // keeping behavior byte-for-byte identical (and yield-overhead-free)
+        // to how library-only consumers experienced this before the CLI
+        // progress work landed.
+        if (onProgress === undefined) {
+            return resolvePageClusterKeysInMemory(fullPages, options);
+        }
+        return resolveSmallCorpusWithProgress(fullPages, onProgress, options);
+    }
+    // Large corpus: streaming path.
+    const excludeLandmarks = options?.excludeLandmarks ?? true;
+    const similarityThreshold = options?.similarityThreshold ?? 0.8;
+    if (!(similarityThreshold >= 0 && similarityThreshold <= 1)) {
+        throw new RangeError(`resolvePageClusterKeys: similarityThreshold must be between 0 and 1, got ${similarityThreshold}`);
+    }
+    validateDetectContentDepthCapOptions(options);
+    const restrictStylesheetsToFirstParty = options?.restrictStylesheetsToFirstParty ?? true;
+    const blockingPagesForKeys = restrictStylesheetsToFirstParty
+        ? filterFirstPartyStylesheetHrefs(blockingSignals)
+        : blockingSignals;
+    const blockKeys = resolveBlockKeys(blockingPagesForKeys, options);
+    const indicesByBlockKey = groupIndicesByBlockKey(blockKeys);
+    const finalKeys = Array.from({ length: blockingSignals.length });
+    const crossBlockUnits = [];
+    const contentBlockAttribute = options?.contentBlockAttribute;
+    /** Non-sample page indices that need Pass 1b Jaccard-based assignment. */
+    const pendingAssignmentBlockKeyByIndex = new Map();
+    /** Block-level artifacts saved after Stage A runs on the sample. */
+    const blockAssignments = new Map();
+    const buckets = new Map();
+    for (const [blockKey, indices] of indicesByBlockKey) {
+        buckets.set(blockKey, {
+            blockKey,
+            targetSize: indices.length,
+            reservoirIndices: [],
+            reservoirPreparedHtml: [],
+            reservoirLandmarks: [],
+            seenCount: 0,
+            prng: makeSeededPrng(blockKey),
+        });
+    }
+    /**
+     * Runs Stage A on the block's reservoir sample, records sample-member
+     * cluster keys into `finalKeys`, saves per-cluster member token sets so
+     * Pass 1b can Jaccard-assign non-sample pages, and appends the produced
+     * `CrossBlockUnit`s to the Stage-B input.
+     * @param bucket
+     */
+    function flushBlock(bucket) {
+        const { localSignatures, localTokensByPage } = computeLocalChromeArtifacts(bucket.reservoirLandmarks, options);
+        const result = stageAPerBlock({
+            blockKey: bucket.blockKey,
+            memberIndices: bucket.reservoirIndices,
+            preparedHtml: bucket.reservoirPreparedHtml,
+            landmarks: bucket.reservoirLandmarks,
+            localLandmarkTokensByPage: localTokensByPage,
+        }, 
+        // No capMembers — the reservoir already bounds `sampleSize`.
+        options);
+        for (const [idx, key] of result.pageKeys) {
+            finalKeys[idx] = key;
+        }
+        crossBlockUnits.push(...result.crossBlockUnits);
+        if (bucket.seenCount > bucket.reservoirIndices.length) {
+            // Save assignment artifacts for Pass 1b.
+            const maxMainDepth = bucket.reservoirPreparedHtml.length > 1
+                ? detectContentDepthCap(bucket.reservoirPreparedHtml, options)
+                : undefined;
+            const clustersByUnitKey = new Map();
+            for (const unit of result.crossBlockUnits) {
+                clustersByUnitKey.set(unit.key, [...unit.memberTokenSets]);
+            }
+            blockAssignments.set(bucket.blockKey, {
+                maxMainDepth,
+                localSignatures,
+                clustersByUnitKey,
+            });
+        }
+        // Encourage V8 to reclaim the reservoir's transient allocations.
+        const maybeGc = globalThis.gc;
+        if (maybeGc !== undefined)
+            maybeGc();
+    }
+    // Pass 1: stream HTML pages, reservoir-sample each block, run Stage A
+    // on the sample the moment the block is fully seen.
+    let pageIndex = 0;
+    for await (const page of pages()) {
+        const blockKey = blockKeys[pageIndex];
+        const bucket = buckets.get(blockKey);
+        if (bucket === undefined) {
+            throw new Error(`resolvePageClusterKeys: block "${blockKey}" is missing from the bucket registry`);
+        }
+        // Reservoir sampling (Algorithm R): keep the first BLOCK_SAMPLE_SIZE
+        // pages, then for each subsequent one replace a random reservoir slot
+        // with decreasing probability.
+        if (bucket.reservoirIndices.length < BLOCK_SAMPLE_SIZE) {
+            const landmarkResult = extractLandmarks(page.html);
+            const landmarksExcised = excludeLandmarks
+                ? landmarkResult.remainderHtml
+                : page.html;
+            const prepared = contentBlockAttribute === undefined
+                ? landmarksExcised
+                : removeContentBlocks(landmarksExcised, {
+                    blockAttribute: contentBlockAttribute,
+                }).remainderHtml;
+            const strippedLandmark = { ...landmarkResult, remainderHtml: '' };
+            bucket.reservoirIndices.push(pageIndex);
+            bucket.reservoirPreparedHtml.push(prepared);
+            bucket.reservoirLandmarks.push(strippedLandmark);
+        }
+        else {
+            const j = Math.floor(bucket.prng() * (bucket.seenCount + 1));
+            if (j < BLOCK_SAMPLE_SIZE) {
+                const landmarkResult = extractLandmarks(page.html);
+                const landmarksExcised = excludeLandmarks
+                    ? landmarkResult.remainderHtml
+                    : page.html;
+                const prepared = contentBlockAttribute === undefined
+                    ? landmarksExcised
+                    : removeContentBlocks(landmarksExcised, {
+                        blockAttribute: contentBlockAttribute,
+                    }).remainderHtml;
+                const strippedLandmark = { ...landmarkResult, remainderHtml: '' };
+                const evicted = bucket.reservoirIndices[j];
+                pendingAssignmentBlockKeyByIndex.set(evicted, bucket.blockKey);
+                bucket.reservoirIndices[j] = pageIndex;
+                bucket.reservoirPreparedHtml[j] = prepared;
+                bucket.reservoirLandmarks[j] = strippedLandmark;
+            }
+            else {
+                pendingAssignmentBlockKeyByIndex.set(pageIndex, bucket.blockKey);
+            }
+        }
+        bucket.seenCount++;
+        if (bucket.seenCount === bucket.targetSize) {
+            flushBlock(bucket);
+            buckets.delete(bucket.blockKey);
+            if (onProgress) {
+                onProgress({
+                    phase: 'pass1-block-complete',
+                    blockKey: bucket.blockKey,
+                    blocksProcessed: indicesByBlockKey.size - buckets.size,
+                    totalBlocks: indicesByBlockKey.size,
+                });
+            }
+        }
+        pageIndex++;
+    }
+    if (pageIndex !== blockingSignals.length) {
+        throw new Error(`resolvePageClusterKeys: streaming input yielded ${pageIndex} pages but Pass 0 saw ${blockingSignals.length} — factory must produce the same pages in the same order on repeated invocations`);
+    }
+    if (buckets.size > 0) {
+        throw new Error(`resolvePageClusterKeys: ${buckets.size} block(s) never reached their target size — this should not happen if the factory produced identical pages across the two passes`);
+    }
+    // Pass 1b: for every non-sample page (evicted from a block's reservoir
+    // or never selected), re-stream its HTML and Jaccard-assign it to the
+    // nearest sample-derived cluster in its block. Nothing is added to
+    // crossBlockUnits here — the assignment writes directly into finalKeys.
+    if (pendingAssignmentBlockKeyByIndex.size > 0) {
+        const totalToAssign = pendingAssignmentBlockKeyByIndex.size;
+        let assignedCount = 0;
+        let assignPageIndex = 0;
+        for await (const page of pages()) {
+            const targetBlockKey = pendingAssignmentBlockKeyByIndex.get(assignPageIndex);
+            if (targetBlockKey !== undefined) {
+                const assignment = blockAssignments.get(targetBlockKey);
+                if (assignment !== undefined) {
+                    finalKeys[assignPageIndex] = assignPageToNearestCluster(page.html, assignment, excludeLandmarks, contentBlockAttribute, options, targetBlockKey);
+                }
+                assignedCount++;
+                if (onProgress && assignedCount % 1000 === 0) {
+                    onProgress({
+                        phase: 'pass1b-assign',
+                        pagesAssigned: assignedCount,
+                        pagesToAssign: totalToAssign,
+                    });
+                }
+            }
+            assignPageIndex++;
+        }
+    }
+    // Stage B: cross-block merge over the accumulated (sample-based) units.
+    // Each unit already has at most BLOCK_SAMPLE_SIZE members from the
+    // reservoir sampling above, so no additional cap is needed here — the
+    // per-merge cost stays bounded across rounds.
+    if (onProgress) {
+        onProgress({ phase: 'stage-b-start', unitCount: crossBlockUnits.length });
+    }
+    const stageBResult = mergeCrossBlockClusters(crossBlockUnits, {
+        ...options,
+        capMembers: BLOCK_SAMPLE_SIZE,
+    });
+    for (let i = 0; i < finalKeys.length; i++) {
+        const currentKey = finalKeys[i];
+        const rootKey = stageBResult.get(currentKey);
+        if (rootKey !== undefined && rootKey !== currentKey) {
+            finalKeys[i] = rootKey;
+        }
+    }
+    return finalKeys;
+}
+/**
+ * Convenience wrapper that runs {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}
+ * on a materialized array. Preserves the pre-refactor sync API for callers
+ * that already have all pages in memory, while flowing through the same
+ * async driver so behavior stays consistent across the two entry points.
+ * @param pages
+ * @param options
+ * @example
+ * ```ts
+ * const keys = await resolvePageClusterKeysFromArray([
+ *   { paths: ['news', '1'], stylesheetHrefs: [], html: '<body><article>one</article></body>' },
+ *   { paths: ['news', '2'], stylesheetHrefs: [], html: '<body><article>two</article></body>' },
+ *   { paths: ['about'], stylesheetHrefs: [], html: '<body><section>about</section></body>' },
+ * ]);
+ * ```
+ */
+export function resolvePageClusterKeysFromArray(pages, options) {
+    return resolvePageClusterKeys(() => pages, options);
 }