@d-zero/page-cluster 0.2.0 → 0.3.0

package/dist/resolve-page-cluster-keys.d.ts

@@ -1,222 +1,323 @@
+import type { ExtractLandmarksResult } from './extract-landmarks.js';
 import type { ResolveBlockingGroupKeysOptions } from './resolve-blocking-group-keys.js';
 import type { ResolveStructuralClusterKeysOptions } from './resolve-structural-cluster-keys.js';
 import type { TokenizeOptions } from './types.js';
+/**
+ * Reinjects each page's *local* (non-corpus-wide) landmark-instance tokens
+ * into its block token set for Stage A clustering, restoring exactly the
+ * structural signal that landmark excision removed for those pages while
+ * keeping global chrome removed (the whole point of `excludeLandmarks`).
+ *
+ * ## Why token-level reinjection instead of one opaque pseudo-token
+ *
+ * An earlier iteration returned a single opaque token per local signature.
+ * That failed on real data: adding one distinctive token to a 100+-token
+ * page's set produces jaccard ~0.99 between "with-local-landmark" and
+ * "without-local-landmark" siblings, so Stage A's 0.8-clamped auto-cut
+ * silently merged them anyway. Reinjecting the landmark's actual tokens
+ * (typically 4–20 per landmark) restores the full structural weight of
+ * the distinction. A real mid-sized crawl corpus's section subtree with
+ * a shared section-local `<nav>` now splits correctly from siblings
+ * without one, since the reinjected local-nav tokens push jaccard below
+ * the cut.
+ *
+ * ## The corpus-level auto-cut
+ *
+ * Every page's landmark instances are canonicalized to a signature (via
+ * {@link ./canonicalize-token-set.js | canonicalizeTokenSet}); the corpus-
+ * wide histogram of "how many pages carry this signature" is fed to
+ * {@link ./auto-cut-threshold.js | autoCutThreshold} — the same primitive
+ * used at every other layer of this pipeline for merge-height cutoffs. The
+ * clamp caps the auto-cut at 0.8 so it never picks a threshold *above* the
+ * conservative default. A signature at or above the cut is global chrome —
+ * appears on effectively every page, so its tokens carry no discriminatory
+ * signal and are left excised. A signature below the cut is local chrome
+ * for the pages that carry it, and its tokens are reinjected into those
+ * pages' block token sets. Same technique as the per-unit shellQuorum in
+ * {@link ./merge-cross-block-clusters.js | mergeCrossBlockClusters}, one
+ * layer up.
+ *
+ * ## The `count >= 2` gate
+ *
+ * A signature present on exactly one page is per-page variation, not
+ * shared local chrome — no "these pages have the same local chrome, those
+ * pages don't" grouping can be built from a singleton, and admitting
+ * singleton signatures would reinject each per-page-unique landmark into
+ * exactly one page's token set, causing spurious per-page cluster
+ * fragmentation across the corpus (confirmed against a 2-page fixture
+ * where two pages carry byte-different `<header>`s produced identical
+ * clusters as expected; without the gate, each would carry its own
+ * reinjected tokens and split).
+ * @param landmarks
+ * @param tokenizeOptions
+ */
+/**
+ * Companion to {@link ./resolve-page-cluster-keys.js | computeLocalLandmarkTokens}
+ * that also returns the local-signature *set* the streaming path needs to
+ * reuse when tokenizing non-sample pages during Pass 1b. The in-memory path
+ * only cares about the per-page token sets (which pages carry which
+ * chrome-below-the-cut tokens); the streaming path additionally needs to
+ * apply the *same* "which signatures are local" verdict to pages that were
+ * not part of the sample the verdict was learned from.
+ * @param landmarks
+ * @param tokenizeOptions
+ */
+export declare function computeLocalChromeArtifacts(landmarks: readonly ExtractLandmarksResult[], tokenizeOptions: TokenizeOptions | undefined): {
+    readonly localSignatures: ReadonlySet<string>;
+    readonly localTokensByPage: readonly ReadonlySet<string>[];
+};
+/**
+ * Reinjects each page's *local* (non-corpus-wide) landmark-instance tokens
+ * into its block token set for Stage A clustering, restoring exactly the
+ * structural signal that landmark excision removed for those pages while
+ * keeping global chrome removed (the whole point of `excludeLandmarks`).
+ *
+ * See {@link ./resolve-page-cluster-keys.js | computeLocalChromeArtifacts}
+ * for the underlying algorithm — this function is a thin wrapper that
+ * discards the local-signature set, exposed for callers that only need the
+ * per-page tokens (the in-memory driver's use case).
+ * @param landmarks
+ * @param tokenizeOptions
+ */
+export declare function computeLocalLandmarkTokens(landmarks: readonly ExtractLandmarksResult[], tokenizeOptions: TokenizeOptions | undefined): ReadonlySet<string>[];
 /**
  * Per-page input to {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}:
  * the blocking signals {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys}
- * needs, plus the page's raw HTML. Raw HTML (rather than a pre-tokenized
- * `Set`, as earlier versions of this type required) because
- * `resolvePageClusterKeys` now needs to decide *how* to tokenize each page
- * (see `excludeLandmarks` below) — a decision a caller handed a bare
- * `Set<string>` could no longer make correctly on its own.
+ * needs, plus the page's raw HTML.
  */
 export type PageClusterSignals = {
     paths: readonly string[];
     stylesheetHrefs: readonly string[];
     html: string;
+    /**
+     * This page's own URL host (hostname, optionally `:port` — same shape as
+     * `new URL(pageUrl).host`), forwarded to
+     * {@link ./filter-first-party-stylesheet-hrefs.js | filterFirstPartyStylesheetHrefs}
+     * so it can judge that page's `stylesheetHrefs` by direct comparison
+     * instead of inferring a batch-wide dominant host.
+     */
+    host?: string;
+};
+/**
+ * Progress event emitted by the async factory-based
+ * `resolvePageClusterKeys` when an `onProgress` callback is provided. Meant
+ * as a lightweight, opt-in observability hook for callers who need visible
+ * progress on multi-minute jobs — the CLI wires this straight to stderr.
+ *
+ * The event is a discriminated union on `phase`:
+ * - `pass0-signals`: streaming path only. Reading blocking signals (paths /
+ *   stylesheetHrefs / host) from the factory. Fires every ~1,000 pages
+ *   during Pass 0.
+ * - `pass1-block-complete`: one block's Stage A finished. Fires on **both**
+ *   the small-corpus path (`≤ CORPUS_INLINE_THRESHOLD`, once per block in
+ *   `indicesByBlockKey` iteration order) and the streaming path (once per
+ *   block as its reservoir fills). The event carries the block's key and a
+ *   running count of how many blocks have completed so far.
+ * - `pass1b-assign`: streaming path only. Streaming assignment of
+ *   non-sample pages is in progress. Fires every ~1,000 pages of Pass 1b.
+ *   Conceptually absent on the small-corpus path — every page is a "sample"
+ *   there.
+ * - `stage-b-start`: cross-block merge has begun. Fires once per call on
+ *   both paths.
+ *
+ * Stage B does not currently emit per-round events — a future extension
+ * that passes a callback down into `mergeCrossBlockClusters` can add them
+ * without breaking the existing shape.
+ *
+ * The **sync** `resolvePageClusterKeysInMemory` never emits any progress
+ * (it has no way to yield to a caller mid-block anyway). Only the async
+ * factory-based entry point participates in `onProgress`.
+ */
+export type ProgressEvent = {
+    readonly phase: 'pass0-signals';
+    readonly pagesSeen: number;
+} | {
+    readonly phase: 'pass1-block-complete';
+    readonly blockKey: string;
+    readonly blocksProcessed: number;
+    readonly totalBlocks: number;
+} | {
+    readonly phase: 'pass1b-assign';
+    readonly pagesAssigned: number;
+    readonly pagesToAssign: number;
+} | {
+    readonly phase: 'stage-b-start';
+    readonly unitCount: number;
 };
 /**
  * @see resolvePageClusterKeys
  */
 export type ResolvePageClusterKeysOptions = TokenizeOptions & ResolveBlockingGroupKeysOptions & ResolveStructuralClusterKeysOptions & {
-    /**
-     * Tokenize each page's `<header>`/`<footer>`/`<nav>`/`<aside>`-excised
-     * remainder ({@link ./extract-landmarks.js | extractLandmarks}'s
-     * `remainderHtml`) instead of its raw HTML, so shared site chrome never
-     * reaches the structural-similarity comparison. Defaults to `true`.
-     * Set to `false` to fall back to tokenizing the untouched page (the
-     * pre-landmark-extraction behavior) — this is a large behavioral
-     * change not yet validated across many sites beyond the two real
-     * corpora checked so far, so the escape hatch is kept available.
-     *
-     * Leaving this `true` means every page's HTML is parsed twice (once by
-     * `extractLandmarks`, once by `tokenize` on its `remainderHtml`)
-     * instead of once. Measured on a real crawl corpus (8,936 pages),
-     * this is still net faster overall than the single-parse `false`
-     * path (17,557ms vs 25,997ms end-to-end): `remainderHtml` is
-     * substantially shorter than the original page once landmarks are
-     * excised, and the resulting smaller `tokenize` pass costs less than
-     * the extra `extractLandmarks` pass adds.
-     */
     excludeLandmarks?: boolean;
-    /**
-     * Apply {@link ./reassign-orphan-block-keys.js | reassignOrphanBlockKeys}
-     * to the blocking keys before clustering, so a page with no recorded
-     * stylesheets ("orphan" — often a crawl-completeness gap, not evidence
-     * the page is actually template-less) can rejoin a same-URL-section
-     * `css:` block instead of being stranded on its weaker `path:` fallback.
-     * Defaults to `true`. Set to `false` to fall back to the raw
-     * {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys}
-     * output — kept available both because this is not yet broadly-
-     * validated beyond the two real crawls checked so far, and because it
-     * has a known trade-off documented on
-     * {@link ./reassign-orphan-block-keys.js | reassignOrphanBlockKeys}
-     * itself: pooling pages for comparison can change unrelated pages'
-     * cluster outcomes too, not just the orphan's.
-     */
     reassignOrphans?: boolean;
-    /**
-     * Apply {@link ./remove-content-blocks.js | removeContentBlocks} to each
-     * page's landmark-excised remainder before tokenizing, so a freeform
-     * block-editor content area's page-to-page variation (which specific
-     * mix of blocks an author used) never reaches the structural-similarity
-     * comparison. No default — unlike `excludeLandmarks`/`reassignOrphans`,
-     * this needs the caller's own block-editor attribute name (see
-     * `removeContentBlocks`'s `blockAttribute` option), which this package
-     * cannot guess. Omit to skip this step entirely.
-     */
     contentBlockAttribute?: string;
-    /**
-     * Apply {@link ./filter-first-party-stylesheet-hrefs.js |
-     * filterFirstPartyStylesheetHrefs} to `pages` before computing blocking
-     * keys, so a page's incidental third-party embeds (e.g. a video
-     * player's own stylesheet, extra web-font requests pulled in by a
-     * widget) never get mistaken for a template-identifying signal by
-     * {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys}.
-     * Defaults to `true`. Set to `false` to block on every page's full,
-     * unfiltered `stylesheetHrefs` — kept available for the same reason
-     * `excludeLandmarks`'s escape hatch is: a real but not yet broadly-
-     * validated behavioral change (confirmed so far on one real crawl).
-     *
-     * Inherits `filterFirstPartyStylesheetHrefs`'s "roughly homogeneous
-     * batch" precondition (see that function's own JSDoc): `pages` should
-     * be one site or one section, the same expectation
-     * `resolveBlockingGroupKeys` already places on its own input.
-     */
     restrictStylesheetsToFirstParty?: boolean;
     /**
-     * Apply {@link ./detect-content-depth-cap.js | detectContentDepthCap}
-     * separately *within each block* (after `excludeLandmarks`/
-     * `contentBlockAttribute`, after blocking, before that block's own
-     * tokenizing) to find how many levels of nesting inside
-     * `<main>`/`role="main"` to keep, then
-     * {@link ./cap-content-depth.js | capContentDepth} each of that
-     * block's pages at that depth — a `contentBlockAttribute`-style fix
-     * for freeform-content noise that needs no site-specific attribute
-     * name, since `<main>` is an HTML5/ARIA standard. Defaults to
-     * `false`: unlike `contentBlockAttribute` (which does nothing unless
-     * a matching attribute is actually present), this discards real
-     * content whenever a page has a `<main>`/`role="main"` at all, so
-     * it's opt-in until validated on more than the two real corpora
-     * checked so far.
-     *
-     * Per-block rather than once across the whole corpus: different
-     * blocks (different templates/sections) can have genuinely different
-     * "skeleton depths." Confirmed on real crawl data: an 814-page block
-     * whose own knee sits at depth 2 stayed at 189 clusters (barely moved
-     * from 224 uncapped) when capped at depth 3 — the knee derived from
-     * the *whole* 8,936-page corpus, dominated by two much larger blocks
-     * whose own knee is 3. Re-deriving the knee for that block alone
-     * brings it down to 46. A block too small for its knee-detection
-     * sweep to find a reliable jump just falls through to
-     * `detectContentDepthCap`'s own no-knee fallback (the largest
-     * candidate depth, effectively "don't cap") — the same safe default
-     * it already has for any input, now reached per-block instead of
-     * corpus-wide. Skipped entirely (no cap) for a block of exactly 1
-     * page — nothing to compare it against, so a knee sweep there could
-     * only ever confirm what's already true.
-     *
-     * Trade-off of going per-block: a corpus-wide sweep's cluster-count
-     * ratios are diluted by thousands of ordinary pages, so one
-     * incidental outlier (e.g. a single page with an extra wrapper `div`
-     * from a stray widget) barely moves them. A small block's sweep has
-     * no such dilution — a similar outlier among only a handful of pages
-     * can itself clear `minKneeRatio` and produce a too-shallow cap for
-     * that block. Not yet observed on either real corpus checked so far
-     * (both corpora's small blocks happened to be uniform enough that
-     * this didn't come up), so no size-based guard is added speculatively;
-     * revisit if real data surfaces it.
-     *
-     * Composes with `contentBlockAttribute` rather than replacing it: both
-     * can be set at once — `removeContentBlocks` runs first, then
-     * `capContentDepth` on what's left — for a site whose CMS marks *some*
-     * blocks with a known attribute but still has other, unmarked
-     * freeform depth the attribute alone doesn't catch.
-     *
-     * Confirmed on real crawl data this can outperform
-     * `contentBlockAttribute` on its own, not just stand in for it when the
-     * attribute is unknown: on a 302-page corpus, `autoCapMainDepth` alone
-     * produced 20 final clusters versus 27 for
-     * `contentBlockAttribute: 'data-bgb'` together with
-     * `restrictStylesheetsToFirstParty` — the site's known CMS attribute
-     * doesn't mark every source of freeform depth, but the `<main>`
-     * boundary catches all of it uniformly. See `detectContentDepthCap`'s
-     * JSDoc for the real cost/accuracy numbers this per-block sweep
-     * measures on the same two corpora.
-     */
-    autoCapMainDepth?: boolean;
-    /**
-     * Re-key two or more otherwise-distinct clusters onto one shared key
-     * when every landmark type present on their pages is both identical
-     * and rare corpus-wide — see
-     * {@link ./merge-landmark-affined-clusters.js | mergeLandmarkAffinedClusters}'s
-     * JSDoc for the exact rule, the withdrawn earlier prototype this
-     * reimplements, and why "rare" (not merely "identical") is required.
-     * Defaults to `false`.
-     *
-     * Kept `false` by default: unlike `autoCapMainDepth`/
-     * `restrictStylesheetsToFirstParty`, this has not been run against
-     * real crawl data at all as of this change — only synthetic-fixture
-     * unit/regression tests. See `mergeLandmarkAffinedClusters`'s JSDoc
-     * for its cost profile before enabling this on a large corpus.
+     * Optional observability hook — invoked at every progress event
+     * documented on {@link ./resolve-page-cluster-keys.js | ProgressEvent}.
+     * Fires only on the async factory-based `resolvePageClusterKeys`;
+     * the sync `resolvePageClusterKeysInMemory` never emits events.
+     * On the async path, passing this option promotes the small-corpus
+     * branch (`≤ CORPUS_INLINE_THRESHOLD`) from delegating to the
+     * sync helper to running a per-block async loop that emits
+     * `pass1-block-complete` and `stage-b-start`. Omitting `onProgress`
+     * keeps the small-corpus branch on the pre-refactor sync path with
+     * zero yield overhead.
      */
-    mergeRareLandmarkClusters?: boolean;
-    /** Forwarded to {@link ./merge-landmark-affined-clusters.js | mergeLandmarkAffinedClusters} as-is. */
-    landmarkRarityThreshold?: number;
-    /** Forwarded to {@link ./merge-landmark-affined-clusters.js | mergeLandmarkAffinedClusters} as-is. */
-    landmarkGateSimilarityThreshold?: number;
+    onProgress?: (event: ProgressEvent) => void;
 };
 /**
- * Connects the two stages this package otherwise leaves for the caller to
- * wire together: {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys}
- * (coarse blocking by URL path or stylesheet set) and
- * {@link ./resolve-structural-cluster-keys.js | resolveStructuralClusterKeys}
- * (exact structural clustering *within* one block). Returns one final key
- * per page, in the same order as `pages`, unique across the whole input —
- * not just within a block.
- *
- * `resolveStructuralClusterKeys` numbers its clusters `cluster:0`,
- * `cluster:1`, ... independently every time it's called, so two different
- * blocks' `cluster:0` are unrelated but identically named. Composing the
- * block key and the per-block cluster label via `JSON.stringify` (rather
- * than plain string concatenation, e.g. a `::` separator) rules out
- * collisions regardless of what either half happens to contain, without
- * depending on `resolveStructuralClusterKeys`'s label format never changing.
- *
- * `excludeLandmarks` and `similarityThreshold` interact: removing shared
- * chrome makes every remaining comparison stricter (there's no more
- * chrome-driven baseline similarity propping scores up), so a threshold
- * tuned against raw, chrome-included tokens can become too strict once
- * landmarks are excluded. Confirmed on real crawl data: a 4-page block where
- * 3 same-template pages merged at the default `similarityThreshold` (0.8)
- * using raw tokens split one of the 3 into its own singleton once landmarks
- * were excluded, and re-merged correctly at `similarityThreshold: 0.6` — re-
- * tune per site after switching this on, the same as `similarityThreshold`
- * itself already needs.
- *
- * `reassignOrphans` only ever pools a `path:`-fallback orphan alongside a
- * same-section `css:` block for `resolveStructuralClusterKeys` to compare —
- * it never forces a merge itself. An orphan that turns out not to match
- * anything in that pool (confirmed on real crawl data) correctly surfaces as
- * its own singleton, the same as it would have without this option.
- *
- * `restrictStylesheetsToFirstParty` runs before `reassignOrphans`: a page
- * whose only stylesheet reference was third-party becomes an orphan (no
- * first-party stylesheet left) *because of* the filtering, and is then
- * itself eligible for orphan reassignment — this is intentional, not an
- * ordering accident, since the underlying reason both options exist is the
- * same (a page's blocking key should reflect its template, not incidental
- * third-party embeds or missing crawl data).
+ * Corpus size at or below which the async factory-based
+ * `resolvePageClusterKeys` reads the entire input into an array and delegates
+ * to {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeysInMemory}
+ * unchanged — preserving corpus-wide semantics (chrome discovery, Stage B
+ * across all pages) exactly.
+ *
+ * Above this threshold, the streaming path takes over: block dispatch during
+ * a second factory read, per-block chrome discovery (semantic drift from
+ * corpus-wide, unavoidable when the whole corpus does not fit in memory),
+ * and Stage B fed with the incrementally-accumulated cross-block units.
+ *
+ * Chosen from Phase 0 spike measurements: a ~9,000-page real crawl (biggest
+ * block ~3,900) completed in ~108s / 1.25 GB heap on the in-memory path.
+ * Doubling that headroom to 20,000 keeps every corpus previously validated
+ * (302, 1,416, 8,936, 89 pages) on the exact code path they were validated
+ * against, so their gate values (9 / 21 / 63 / 3 clusters respectively)
+ * remain byte-reproducible. A ~176,000-page real crawl OOM'd on the in-memory
+ * path well below this threshold worth of pages ever being materialized, so
+ * anything above 20,000 is routed to streaming.
+ */
+export declare const CORPUS_INLINE_THRESHOLD = 20000;
+/**
+ * Reservoir-sample size per block on the streaming path. Blocks larger than
+ * this have Stage A run on a random sample of `BLOCK_SAMPLE_SIZE` pages,
+ * with the remaining non-sample pages assigned via Jaccard similarity to
+ * the sample-derived clusters during Pass 1b. Blocks at or below this size
+ * still work — the sampling degenerates to "keep every input page unchanged"
+ * per the `reservoirSample` contract, so small blocks behave identically to
+ * the in-memory path.
+ *
+ * Chosen to bound accumulated Stage-B state: units × sample_size × per-
+ * member memory ≈ 200 units × 100 members × 25 KB ≈ 500 MB, well within an
+ * 8 GB Node heap even on macOS where jetsam (the kernel OOM killer) reacts
+ * to RSS pressure before V8's own heap limit trips.
+ *
+ * ## Semantic differences from the in-memory path
+ *
+ * - **Chrome discovery is sample-based per block.** Landmark signatures that
+ *   are rare in the sample get treated as global chrome; only signatures
+ *   that show up on ≥ 2 sample members and below the sample-derived
+ *   auto-cut are reinjected. Full-block chrome discovery would see rare
+ *   signatures too — the sample-based decision approximates it.
+ * - **Non-sample pages are assigned by max-Jaccard against sample member
+ *   token sets.** A page whose closest sample member is genuinely dissimilar
+ *   still gets slotted into the least-bad cluster; this is a pragmatic
+ *   trade for a bounded assignment cost (no unbounded "outlier" cluster
+ *   growth).
+ * - **Stage B sees the sample-based `CrossBlockUnit`s only.** Non-sample
+ *   pages carry the final key that Stage B produces for their assigned
+ *   sample cluster, without contributing to Stage B's own DF / quorum-core
+ *   / shell-quorum computations.
+ *
+ * Preserves the in-memory path unchanged for corpora at or below
+ * {@link CORPUS_INLINE_THRESHOLD} — sampling is streaming-mode only.
+ */
+export declare const BLOCK_SAMPLE_SIZE = 100;
+/**
+ * Preserves the previous synchronous, array-in / array-out API of
+ * `resolvePageClusterKeys` under a new name so the factory-based async
+ * export can take the primary name while callers that already had a
+ * materialized page array (spec tests, the in-repo dogfood harness,
+ * downstream code that hasn't switched to streaming yet) retain the
+ * exact same behavior.
+ *
+ * Semantics: identical to the pre-refactor `resolvePageClusterKeys`.
+ * Corpus-wide chrome discovery, Stage B across every page, no memory
+ * bound — meant to be called on inputs already known to fit in memory.
+ * The async factory-based export delegates here whenever
+ * `pages.length ≤ CORPUS_INLINE_THRESHOLD`, guaranteeing existing corpora
+ * hit exactly this code path.
+ * @param pages
+ * @param options
+ */
+export declare function resolvePageClusterKeysInMemory(pages: readonly PageClusterSignals[], options?: ResolvePageClusterKeysOptions): string[];
+/**
+ * Factory function returning an iterator over pages. Called once per streaming
+ * pass — the driver may invoke it multiple times to re-read the same corpus
+ * (once HTML-free for blocking, once again per block for HTML processing).
+ * Callers with a materialized array can wrap it as
+ * `() => pagesArray[Symbol.iterator]()`, or use
+ * {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeysFromArray}.
+ *
+ * ## Why a factory rather than an `AsyncIterable`
+ *
+ * An `AsyncIterable` returned once cannot be traversed a second time (the
+ * iterator is spent after the first `for await`). The streaming driver must
+ * read the corpus at least twice — once with HTML dropped to compute
+ * blocking keys, and once again per block to process HTML. A factory
+ * function lets the caller build a fresh iterator each pass (typically by
+ * re-opening a JSONL file or re-issuing an archive query), so per-corpus
+ * memory stays proportional to the largest single block rather than to the
+ * full corpus.
+ */
+export type PageFactory = () => Iterable<PageClusterSignals> | AsyncIterable<PageClusterSignals>;
+/**
+ * Streaming, memory-bounded version of `resolvePageClusterKeysInMemory`.
+ *
+ * ## Behavior gate
+ *
+ * - `pageCount ≤ CORPUS_INLINE_THRESHOLD` — reads the whole factory into an
+ *   array, delegates to `resolvePageClusterKeysInMemory`. Same corpus-wide
+ *   chrome discovery, same Stage B across every page. All previously
+ *   validated corpora (302 / 1,416 / 8,936 / 89 pages) hit this path.
+ * - `pageCount > CORPUS_INLINE_THRESHOLD` — streaming path: reads the
+ *   factory twice (once for blocking signals, once for HTML processing),
+ *   dispatches HTML per block, runs Stage A per block, accumulates
+ *   cross-block units, then runs Stage B across all accumulated units. Peak
+ *   memory ≈ largest single block, not the whole corpus.
+ *
+ * ## Semantic differences in streaming mode
+ *
+ * - **Chrome discovery is per-block, not corpus-wide.** In the in-memory
+ *   path, {@link ./resolve-page-cluster-keys.js | computeLocalLandmarkTokens}
+ *   runs on all pages at once. In streaming mode the entire corpus cannot
+ *   be held at once, so chrome discovery runs per block. A landmark
+ *   signature that is rare corpus-wide but common within one block will
+ *   be treated as global chrome in streaming mode, whereas the in-memory
+ *   mode would treat it as local. This trade-off is why the threshold
+ *   above is set generously — every real corpus historically validated
+ *   here stays on the in-memory path.
+ * @param pages
+ * @param options
+ * @example
+ * ```ts
+ * // JSONL file source — factory can be re-invoked to re-open the file.
+ * import { createReadStream } from 'node:fs';
+ * import readline from 'node:readline';
+ *
+ * const keys = await resolvePageClusterKeys(() => {
+ *   const lines = readline.createInterface({ input: createReadStream('pages.jsonl') });
+ *   return (async function* () {
+ *     for await (const line of lines) yield JSON.parse(line);
+ *   })();
+ * });
+ * ```
+ */
+export declare function resolvePageClusterKeys(pages: PageFactory, options?: ResolvePageClusterKeysOptions): Promise<string[]>;
+/**
+ * Convenience wrapper that runs {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}
+ * on a materialized array. Preserves the pre-refactor sync API for callers
+ * that already have all pages in memory, while flowing through the same
+ * async driver so behavior stays consistent across the two entry points.
  * @param pages
  * @param options
  * @example
  * ```ts
- * resolvePageClusterKeys([
- * 	{ paths: ['news', '1'], stylesheetHrefs: [], html: '<body><article>one</article></body>' },
- * 	{ paths: ['news', '2'], stylesheetHrefs: [], html: '<body><article>two</article></body>' },
- * 	{ paths: ['about'], stylesheetHrefs: [], html: '<body><section>about</section></body>' },
+ * const keys = await resolvePageClusterKeysFromArray([
+ *   { paths: ['news', '1'], stylesheetHrefs: [], html: '<body><article>one</article></body>' },
+ *   { paths: ['news', '2'], stylesheetHrefs: [], html: '<body><article>two</article></body>' },
+ *   { paths: ['about'], stylesheetHrefs: [], html: '<body><section>about</section></body>' },
  * ]);
- * // pages 0 and 1 (same block, same structure) share a key; page 2 (different block) gets its own
  * ```
  */
-export declare function resolvePageClusterKeys(pages: readonly PageClusterSignals[], options?: ResolvePageClusterKeysOptions): string[];
+export declare function resolvePageClusterKeysFromArray(pages: readonly PageClusterSignals[], options?: ResolvePageClusterKeysOptions): Promise<string[]>;