@d-zero/page-cluster 0.3.1 → 0.4.0

package/dist/merge-cross-block-clusters.js

@@ -1,11 +1,11 @@
 import { assignContainedClusters } from './assign-contained-clusters.js';
-import { autoCutThreshold } from './auto-cut-threshold.js';
 import { collapseAnonymousDivs } from './collapse-anonymous-divs.js';
 import { completeLinkageDendrogram, labelsAtThreshold, } from './complete-linkage-dendrogram.js';
 import { computeDocumentFrequency } from './compute-document-frequency.js';
 import { jaccardSimilarity } from './jaccard-similarity.js';
 import { reservoirSample } from './reservoir-sample.js';
 import { shapeToken } from './shape-token.js';
+import { shellQuorum } from './shell-quorum.js';
 import { splitTokensByFrequency } from './split-tokens-by-frequency.js';
 /**
  * Fixed complete-linkage threshold for the cross-block fine stage.
@@ -26,11 +26,10 @@ const CROSS_BLOCK_THRESHOLD = 0.8;
  * on real crawl data. Full union is shell-dominated: 298 pages collapsed into
  * 4 clusters — also confirmed. 80% quorum avoids both failure modes.
  *
- * Also reused as the fallback clamp for {@link ./auto-cut-threshold.js |
- * autoCutThreshold} when running on the per-landmark-instance frequency
- * distribution in {@link ./merge-cross-block-clusters.js | shellQuorum}. The
- * clamp only ever *loosens* the cut relative to this floor (never tightens),
- * and only fires in the degenerate cases the JSDoc there describes.
+ * Not shared with {@link ./shell-quorum.js | shellQuorum}'s own fallback
+ * clamp (`SHELL_QUORUM_FALLBACK_FRACTION`) — the two happen to be the same
+ * value today because both were validated against the same real crawl
+ * corpora, but they are independently tunable.
  */
 const QUORUM_FRACTION = 0.8;
 /**
@@ -159,97 +158,6 @@ function l2Contained(xSig, ySig) {
     }
     return true;
 }
-/**
- * Discovers a unit's shell tokens by auto-cutting the per-*token* page-
- * frequency histogram of every landmark instance's tokens. This is the same
- * max-gap primitive used for Stage A merge-height cutoffs, applied
- * recursively at the landmark-token layer.
- *
- * ## Why per-token and not per-signature
- *
- * An earlier iteration ran the histogram at the level of full landmark-
- * instance signatures (canonicalized token sets). That failed on a real,
- * common pattern: a shared site chrome whose markup carries a per-page
- * distinguishing element (a breadcrumb, a page-title element with a page-
- * specific class, a "current" state). All pages have most of the same
- * tokens, but every page's full signature is distinct because tokens embed
- * class names. Per-signature counting saw 5 signatures at freq 0.2 each,
- * autoCutThreshold on the flat distribution returned the clamp, and the
- * shell collapsed to empty even though every page shared the core header
- * skeleton. Per-token counting handles the same case correctly — the shared
- * skeleton tokens each hit freq 1.0.
- *
- * ## The histogram
- *
- * For every member page, all its landmark instances are tokenized and
- * unioned into a single per-page token set (order-agnostic, deduped: a
- * token appearing in two of the page's landmarks still counts once for
- * that page). The corpus histogram is then "how many pages contain each
- * token". Tokens that appear on nearly every page are the unit's chrome;
- * tokens that appear on only a handful are page-specific content that
- * happens to be tagged as a landmark.
- *
- * ## Why auto-cut instead of a hard-coded quorum
- *
- * A fixed 80% quorum (this file's earlier implementation) baked one
- * threshold in for every unit. Real corpora don't obey a universal cutoff:
- * a section-local landmark token that appears on 60% of a unit's pages is
- * the section's chrome under any reasonable reading, but 80% quorum
- * discards it. Auto-cut looks at the *shape* of the frequency distribution
- * and picks the widest gap between adjacent frequencies — if the
- * distribution is `{1.00, 1.00, 0.65, 0.03, 0.02}`, the gap between 0.65
- * and 0.03 (0.62) dwarfs everything else and the cut lands mid-gap around
- * 0.34, correctly grouping the 0.65 tokens with the site-wide 1.00 ones as
- * "chrome for this unit". If instead the distribution is flat, the clamp
- * to {@link QUORUM_FRACTION} keeps the threshold from being tighter than
- * the fallback default.
- *
- * ## Fallbacks
- *
- * A single distinct token (`heights.length < 2`) or a perfectly flat
- * distribution (`maxGap === 0`) returns the {@link QUORUM_FRACTION} clamp
- * verbatim — exactly the same 80%-quorum behavior as before. So degenerate
- * cases degrade to the old contract; only richer distributions get the
- * auto-cut benefit.
- *
- * A page with no landmarks contributes an empty set, deliberately, so the
- * shell-corroboration jaccard between two landmark-less pages is 0 rather
- * than 1 (which it would be if we handed back a `<body></body>`-derived
- * `{body}` fallback set to both sides).
- * @param perPageInstances
- */
-function shellQuorum(perPageInstances) {
-    const pageCount = perPageInstances.length;
-    if (pageCount === 0)
-        return new Set();
-    // Union all instance token sets per page (dedupe within page: a token
-    // present on both header and footer of the same page still counts once
-    // for that page's contribution).
-    const tokenPageCount = new Map();
-    for (const instances of perPageInstances) {
-        const perPageUnion = new Set();
-        for (const inst of instances) {
-            for (const token of inst.tokens)
-                perPageUnion.add(token);
-        }
-        for (const token of perPageUnion) {
-            tokenPageCount.set(token, (tokenPageCount.get(token) ?? 0) + 1);
-        }
-    }
-    if (tokenPageCount.size === 0)
-        return new Set();
-    const frequencies = [];
-    for (const count of tokenPageCount.values()) {
-        frequencies.push(count / pageCount);
-    }
-    const cut = autoCutThreshold(frequencies, QUORUM_FRACTION);
-    const shell = new Set();
-    for (const [token, count] of tokenPageCount) {
-        if (count / pageCount >= cut)
-            shell.add(token);
-    }
-    return shell;
-}
 // ---------------------------------------------------------------------------
 // Main function
 // ---------------------------------------------------------------------------

package/dist/offset-to-line-column.d.ts

@@ -0,0 +1,38 @@
+/**
+ * A 1-based line/column position within an HTML string.
+ */
+export type LineColumn = {
+    readonly line: number;
+    readonly column: number;
+};
+/**
+ * Precomputed lookup structure for {@link ./offset-to-line-column.js | offsetToLineColumn}:
+ * every newline's string-index offset, ascending.
+ */
+export type LineColumnIndex = {
+    readonly newlineOffsets: readonly number[];
+};
+/**
+ * Scans `html` once and records every `\n` offset, so repeated
+ * {@link ./offset-to-line-column.js | offsetToLineColumn} calls against the
+ * same HTML string can binary-search instead of re-scanning from the start
+ * each time. Built once per page and reused across every landmark instance's
+ * start/end offset — a page with dozens of landmark instances would
+ * otherwise pay for a fresh O(n) scan per offset instead of one O(n) scan
+ * total.
+ * @param html
+ */
+export declare function buildLineColumnIndex(html: string): LineColumnIndex;
+/**
+ * Converts a string-index `offset` (the same unit as `htmlparser2`'s
+ * `startIndex`/`endIndex`, i.e. UTF-16 code units) into a 1-based
+ * `{line, column}` position, using an index built by
+ * {@link ./offset-to-line-column.js | buildLineColumnIndex}.
+ *
+ * `\r\n` line endings are handled without special-casing: the `\r` is
+ * counted as the last column of its own line, matching how most editors
+ * report position for CRLF files.
+ * @param index
+ * @param offset
+ */
+export declare function offsetToLineColumn(index: LineColumnIndex, offset: number): LineColumn;

package/dist/offset-to-line-column.js

@@ -0,0 +1,49 @@
+/**
+ * Scans `html` once and records every `\n` offset, so repeated
+ * {@link ./offset-to-line-column.js | offsetToLineColumn} calls against the
+ * same HTML string can binary-search instead of re-scanning from the start
+ * each time. Built once per page and reused across every landmark instance's
+ * start/end offset — a page with dozens of landmark instances would
+ * otherwise pay for a fresh O(n) scan per offset instead of one O(n) scan
+ * total.
+ * @param html
+ */
+export function buildLineColumnIndex(html) {
+    const newlineOffsets = [];
+    for (let i = 0; i < html.length; i++) {
+        if (html.codePointAt(i) === 10)
+            newlineOffsets.push(i);
+    }
+    return { newlineOffsets };
+}
+/**
+ * Converts a string-index `offset` (the same unit as `htmlparser2`'s
+ * `startIndex`/`endIndex`, i.e. UTF-16 code units) into a 1-based
+ * `{line, column}` position, using an index built by
+ * {@link ./offset-to-line-column.js | buildLineColumnIndex}.
+ *
+ * `\r\n` line endings are handled without special-casing: the `\r` is
+ * counted as the last column of its own line, matching how most editors
+ * report position for CRLF files.
+ * @param index
+ * @param offset
+ */
+export function offsetToLineColumn(index, offset) {
+    const { newlineOffsets } = index;
+    let low = 0;
+    let high = newlineOffsets.length;
+    while (low < high) {
+        const mid = (low + high) >>> 1;
+        if (newlineOffsets[mid] < offset) {
+            low = mid + 1;
+        }
+        else {
+            high = mid;
+        }
+    }
+    // `low` is the count of newlines strictly before `offset`, i.e. the
+    // number of completed lines — so `low` newlines completed means we're on
+    // line `low + 1`.
+    const lineStart = low === 0 ? 0 : newlineOffsets[low - 1] + 1;
+    return { line: low + 1, column: offset - lineStart + 1 };
+}

package/dist/per-page-landmark-signatures.d.ts

@@ -1,4 +1,4 @@
-import type { ExtractLandmarksResult, LandmarkType } from './extract-landmarks.js';
+import type { ExtractLandmarksResult, LandmarkPosition, LandmarkType } from './extract-landmarks.js';
 import type { TokenizeOptions } from './types.js';
 /**
  * Every landmark type extractLandmarks may populate, iterated in a fixed
@@ -13,11 +13,20 @@ export declare const ALL_LANDMARK_TYPES: readonly LandmarkType[];
  * {@link ./canonicalize-token-set.js | canonicalizeTokenSet}). Signatures
  * are reused across consumers so two callers see the same "same instance"
  * verdict without independently re-canonicalizing.
+ *
+ * `position` is the instance's location in the page it came from, computed
+ * once by {@link ./extract-landmarks.js | extractLandmarks} and carried here
+ * unchanged — a back-reference for callers (e.g.
+ * {@link ./build-page-landmark-report.js | buildPageLandmarkReport}) that
+ * need to report where a chrome-classified instance actually sits, not just
+ * that it exists. It plays no part in `tokens`/`signature` computation or in
+ * the corpus-frequency logic that consumes this type.
  */
 export type PerPageLandmarkInstance = {
     readonly type: LandmarkType;
     readonly tokens: ReadonlySet<string>;
     readonly signature: string;
+    readonly position: LandmarkPosition;
 };
 /**
  * Tokenizes every landmark instance across every page once, keyed by page

package/dist/per-page-landmark-signatures.js

@@ -44,17 +44,29 @@ export function computePerPageLandmarkInstances(landmarks, tokenizeOptions) {
         const seenSignatures = new Set();
         const out = [];
         for (const type of ALL_LANDMARK_TYPES) {
-            for (const instanceHtml of entry[type]) {
-                if (!instanceHtml)
+            for (const instance of entry[type]) {
+                if (!instance.html)
                     continue;
-                const tokens = new Set(tokenize(`<body>${instanceHtml}</body>`, tokenizeOptions).tokens);
+                const tokens = new Set(tokenize(`<body>${instance.html}</body>`, tokenizeOptions).tokens);
                 if (tokens.size === 0)
                     continue;
                 const signature = canonicalizeTokenSet(tokens);
                 if (seenSignatures.has(signature))
                     continue;
                 seenSignatures.add(signature);
-                out.push({ type, tokens, signature });
+                out.push({
+                    type,
+                    tokens,
+                    signature,
+                    position: {
+                        startOffset: instance.startOffset,
+                        endOffset: instance.endOffset,
+                        startLine: instance.startLine,
+                        startColumn: instance.startColumn,
+                        endLine: instance.endLine,
+                        endColumn: instance.endColumn,
+                    },
+                });
             }
         }
         return out;

package/dist/resolve-page-cluster-keys.d.ts

@@ -1,7 +1,9 @@
 import type { ExtractLandmarksResult } from './extract-landmarks.js';
+import type { PerPageLandmarkInstance } from './per-page-landmark-signatures.js';
 import type { ResolveBlockingGroupKeysOptions } from './resolve-blocking-group-keys.js';
 import type { ResolveStructuralClusterKeysOptions } from './resolve-structural-cluster-keys.js';
 import type { TokenizeOptions } from './types.js';
+import { type PageLandmarkReport } from './build-page-landmark-report.js';
 /**
  * Reinjects each page's *local* (non-corpus-wide) landmark-instance tokens
  * into its block token set for Stage A clustering, restoring exactly the
@@ -65,6 +67,7 @@ import type { TokenizeOptions } from './types.js';
 export declare function computeLocalChromeArtifacts(landmarks: readonly ExtractLandmarksResult[], tokenizeOptions: TokenizeOptions | undefined): {
     readonly localSignatures: ReadonlySet<string>;
     readonly localTokensByPage: readonly ReadonlySet<string>[];
+    readonly perPageInstances: readonly (readonly PerPageLandmarkInstance[])[];
 };
 /**
  * Reinjects each page's *local* (non-corpus-wide) landmark-instance tokens
@@ -162,9 +165,47 @@ export type ResolvePageClusterKeysOptions = TokenizeOptions & ResolveBlockingGro
      * sync helper to running a per-block async loop that emits
      * `pass1-block-complete` and `stage-b-start`. Omitting `onProgress`
      * keeps the small-corpus branch on the pre-refactor sync path with
-     * zero yield overhead.
+     * zero yield overhead. Ignored when `includeLandmarkPositions` is
+     * `true` — that option always routes the small-corpus branch through
+     * the sync helper (see `includeLandmarkPositions`'s own JSDoc), so no
+     * progress events fire in that combination.
      */
     onProgress?: (event: ProgressEvent) => void;
+    /**
+     * When `true`, every result entry additionally carries a
+     * {@link PageLandmarkReport} — each landmark instance's position, plus
+     * a chrome/content verdict for the six excisable types (see
+     * {@link ./build-page-landmark-report.js | buildPageLandmarkReport}).
+     * Changes the return shape from `string[]` to
+     * `{@link PageClusterKeyResult}[]` (see the overloads on
+     * `resolvePageClusterKeysInMemory`/`resolvePageClusterKeys`/
+     * `resolvePageClusterKeysFromArray`). Defaults to `false`, in which
+     * case every existing caller's behavior and return type are
+     * unchanged.
+     *
+     * Not supported on the streaming path
+     * (`pageCount > {@link CORPUS_INLINE_THRESHOLD}`): reservoir sampling
+     * and Jaccard-based non-sample assignment there have no notion of
+     * "this page's shell tokens" to classify chrome against, and
+     * retrofitting one is out of scope. `resolvePageClusterKeys` throws a
+     * `RangeError` up front if both apply to the same call, rather than
+     * silently degrading semantics.
+     *
+     * On the async factory-based `resolvePageClusterKeys`, this option
+     * forces the small-corpus branch through the sync
+     * `resolvePageClusterKeysInMemory` helper regardless of `onProgress`
+     * — see `onProgress`'s own JSDoc.
+     */
+    includeLandmarkPositions?: boolean;
+};
+/**
+ * One page's clustering result when `includeLandmarkPositions` is `true`:
+ * the same `clusterKey` every caller already gets, plus that page's
+ * {@link PageLandmarkReport}.
+ */
+export type PageClusterKeyResult = {
+    readonly clusterKey: string;
+    readonly landmarks: PageLandmarkReport;
 };
 /**
  * Corpus size at or below which the async factory-based
@@ -188,6 +229,17 @@ export type ResolvePageClusterKeysOptions = TokenizeOptions & ResolveBlockingGro
  * anything above 20,000 is routed to streaming.
  */
 export declare const CORPUS_INLINE_THRESHOLD = 20000;
+/**
+ * Throws when `includeLandmarkPositions` is combined with a corpus over
+ * `threshold` pages (the streaming path — see `includeLandmarkPositions`'s
+ * own JSDoc for why it has no sample-based equivalent there). Split out from
+ * its call site so tests can exercise the boundary with a small injected
+ * `threshold` instead of constructing a 20,001-page fixture to cross the
+ * real {@link CORPUS_INLINE_THRESHOLD}.
+ * @param pageCount
+ * @param threshold
+ */
+export declare function assertLandmarkPositionsSupportedForPageCount(pageCount: number, threshold: number): void;
 /**
  * Reservoir-sample size per block on the streaming path. Blocks larger than
  * this have Stage A run on a random sample of `BLOCK_SAMPLE_SIZE` pages,
@@ -240,6 +292,9 @@ export declare const BLOCK_SAMPLE_SIZE = 100;
  * @param pages
  * @param options
  */
+export declare function resolvePageClusterKeysInMemory(pages: readonly PageClusterSignals[], options: ResolvePageClusterKeysOptions & {
+    includeLandmarkPositions: true;
+}): PageClusterKeyResult[];
 export declare function resolvePageClusterKeysInMemory(pages: readonly PageClusterSignals[], options?: ResolvePageClusterKeysOptions): string[];
 /**
  * Factory function returning an iterator over pages. Called once per streaming
@@ -303,6 +358,9 @@ export type PageFactory = () => Iterable<PageClusterSignals> | AsyncIterable<Pag
  * });
  * ```
  */
+export declare function resolvePageClusterKeys(pages: PageFactory, options: ResolvePageClusterKeysOptions & {
+    includeLandmarkPositions: true;
+}): Promise<PageClusterKeyResult[]>;
 export declare function resolvePageClusterKeys(pages: PageFactory, options?: ResolvePageClusterKeysOptions): Promise<string[]>;
 /**
  * Convenience wrapper that runs {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}
@@ -320,4 +378,7 @@ export declare function resolvePageClusterKeys(pages: PageFactory, options?: Res
  * ]);
  * ```
  */
+export declare function resolvePageClusterKeysFromArray(pages: readonly PageClusterSignals[], options: ResolvePageClusterKeysOptions & {
+    includeLandmarkPositions: true;
+}): Promise<PageClusterKeyResult[]>;
 export declare function resolvePageClusterKeysFromArray(pages: readonly PageClusterSignals[], options?: ResolvePageClusterKeysOptions): Promise<string[]>;

package/dist/resolve-page-cluster-keys.js

@@ -1,4 +1,5 @@
 import { autoCutThreshold } from './auto-cut-threshold.js';
+import { buildPageLandmarkReport, } from './build-page-landmark-report.js';
 import { capContentDepth } from './cap-content-depth.js';
 import { detectContentDepthCap, validateDetectContentDepthCapOptions, } from './detect-content-depth-cap.js';
 import { extractLandmarks } from './extract-landmarks.js';
@@ -8,6 +9,7 @@ import { mergeCrossBlockClusters } from './merge-cross-block-clusters.js';
 import { groupIndicesByBlockKey, resolveBlockKeys } from './pass0-blocking.js';
 import { computePerPageLandmarkInstances } from './per-page-landmark-signatures.js';
 import { removeContentBlocks } from './remove-content-blocks.js';
+import { shellQuorum } from './shell-quorum.js';
 import { stageAPerBlock } from './stage-a-per-block.js';
 import { tokenize } from './tokenize.js';
 /**
@@ -162,8 +164,9 @@ function assignPageToNearestCluster(html, assignment, excludeLandmarks, contentB
  */
 export function computeLocalChromeArtifacts(landmarks, tokenizeOptions) {
     const pageCount = landmarks.length;
-    if (pageCount === 0)
-        return { localSignatures: new Set(), localTokensByPage: [] };
+    if (pageCount === 0) {
+        return { localSignatures: new Set(), localTokensByPage: [], perPageInstances: [] };
+    }
     const perPageInstances = computePerPageLandmarkInstances(landmarks, tokenizeOptions);
     // Corpus-wide histogram: signature → { count, tokens }. tokens is the
     // token set of any one occurrence of the signature (all occurrences are
@@ -184,6 +187,7 @@ export function computeLocalChromeArtifacts(landmarks, tokenizeOptions) {
         return {
             localSignatures: new Set(),
             localTokensByPage: landmarks.map(() => new Set()),
+            perPageInstances,
         };
     }
     const frequencies = [];
@@ -202,6 +206,7 @@ export function computeLocalChromeArtifacts(landmarks, tokenizeOptions) {
         return {
             localSignatures: new Set(),
             localTokensByPage: landmarks.map(() => new Set()),
+            perPageInstances,
         };
     }
     const localTokensByPage = perPageInstances.map((instances) => {
@@ -214,7 +219,7 @@ export function computeLocalChromeArtifacts(landmarks, tokenizeOptions) {
         }
         return out;
     });
-    return { localSignatures, localTokensByPage };
+    return { localSignatures, localTokensByPage, perPageInstances };
 }
 /**
  * Reinjects each page's *local* (non-corpus-wide) landmark-instance tokens
@@ -232,6 +237,39 @@ export function computeLocalChromeArtifacts(landmarks, tokenizeOptions) {
 export function computeLocalLandmarkTokens(landmarks, tokenizeOptions) {
     return [...computeLocalChromeArtifacts(landmarks, tokenizeOptions).localTokensByPage];
 }
+/**
+ * Builds every page's {@link PageLandmarkReport} for the `includeLandmarkPositions`
+ * result path. Groups pages by their *final* cluster key (post–Stage-B), runs
+ * {@link ./shell-quorum.js | shellQuorum} once per cluster over the pooled
+ * member `PerPageLandmarkInstance`s (the same "shell tokens" concept Stage B
+ * itself uses for L2 shell corroboration, just recomputed at the final-
+ * cluster granularity rather than the pre-merge unit granularity), then
+ * classifies each member page's own landmark instances against that
+ * cluster-level shell.
+ * @param finalKeys
+ * @param landmarks
+ * @param perPageInstances
+ */
+function buildLandmarkReportsByCluster(finalKeys, landmarks, perPageInstances) {
+    const indicesByKey = new Map();
+    for (const [i, key] of finalKeys.entries()) {
+        const indices = indicesByKey.get(key);
+        if (indices) {
+            indices.push(i);
+        }
+        else {
+            indicesByKey.set(key, [i]);
+        }
+    }
+    const reports = Array.from({ length: finalKeys.length });
+    for (const indices of indicesByKey.values()) {
+        const shellTokens = shellQuorum(indices.map((i) => perPageInstances[i]));
+        for (const i of indices) {
+            reports[i] = buildPageLandmarkReport(landmarks[i], shellTokens);
+        }
+    }
+    return reports;
+}
 /**
  * Corpus size at or below which the async factory-based
  * `resolvePageClusterKeys` reads the entire input into an array and delegates
@@ -254,6 +292,21 @@ export function computeLocalLandmarkTokens(landmarks, tokenizeOptions) {
  * anything above 20,000 is routed to streaming.
  */
 export const CORPUS_INLINE_THRESHOLD = 20_000;
+/**
+ * Throws when `includeLandmarkPositions` is combined with a corpus over
+ * `threshold` pages (the streaming path — see `includeLandmarkPositions`'s
+ * own JSDoc for why it has no sample-based equivalent there). Split out from
+ * its call site so tests can exercise the boundary with a small injected
+ * `threshold` instead of constructing a 20,001-page fixture to cross the
+ * real {@link CORPUS_INLINE_THRESHOLD}.
+ * @param pageCount
+ * @param threshold
+ */
+export function assertLandmarkPositionsSupportedForPageCount(pageCount, threshold) {
+    if (pageCount > threshold) {
+        throw new RangeError(`resolvePageClusterKeys: includeLandmarkPositions is not supported for corpora larger than ${threshold} pages (the streaming path) — this corpus has ${pageCount}`);
+    }
+}
 /**
  * Reservoir-sample size per block on the streaming path. Blocks larger than
  * this have Stage A run on a random sample of `BLOCK_SAMPLE_SIZE` pages,
@@ -289,23 +342,6 @@ export const CORPUS_INLINE_THRESHOLD = 20_000;
  * {@link CORPUS_INLINE_THRESHOLD} — sampling is streaming-mode only.
  */
 export const BLOCK_SAMPLE_SIZE = 100;
-/**
- * Preserves the previous synchronous, array-in / array-out API of
- * `resolvePageClusterKeys` under a new name so the factory-based async
- * export can take the primary name while callers that already had a
- * materialized page array (spec tests, the in-repo dogfood harness,
- * downstream code that hasn't switched to streaming yet) retain the
- * exact same behavior.
- *
- * Semantics: identical to the pre-refactor `resolvePageClusterKeys`.
- * Corpus-wide chrome discovery, Stage B across every page, no memory
- * bound — meant to be called on inputs already known to fit in memory.
- * The async factory-based export delegates here whenever
- * `pages.length ≤ CORPUS_INLINE_THRESHOLD`, guaranteeing existing corpora
- * hit exactly this code path.
- * @param pages
- * @param options
- */
 export function resolvePageClusterKeysInMemory(pages, options) {
     const excludeLandmarks = options?.excludeLandmarks ?? true;
     const similarityThreshold = options?.similarityThreshold ?? 0.8;
@@ -316,8 +352,11 @@ export function resolvePageClusterKeysInMemory(pages, options) {
     // corroboration regardless of `excludeLandmarks`, and `remainderHtml` is
     // needed whenever `excludeLandmarks` is true.
     const landmarks = pages.map((page) => extractLandmarks(page.html));
-    // Corpus-level chrome discovery
-    const localLandmarkTokensByPage = computeLocalLandmarkTokens(landmarks, options);
+    // Corpus-level chrome discovery. `perPageInstances` is only consumed
+    // below when `includeLandmarkPositions` is set — computed unconditionally
+    // anyway since `computeLocalChromeArtifacts` already builds it internally
+    // for chrome discovery, so exposing it here costs nothing extra.
+    const { localTokensByPage: localLandmarkTokensByPage, perPageInstances } = computeLocalChromeArtifacts(landmarks, options);
     const contentBlockAttribute = options?.contentBlockAttribute;
     const preparedHtml = pages.map((page, index) => {
         const landmarksExcised = excludeLandmarks
@@ -363,7 +402,10 @@ export function resolvePageClusterKeysInMemory(pages, options) {
             finalKeys[i] = rootKey;
         }
     }
-    return finalKeys;
+    if (!options?.includeLandmarkPositions)
+        return finalKeys;
+    const reports = buildLandmarkReportsByCluster(finalKeys, landmarks, perPageInstances);
+    return finalKeys.map((clusterKey, i) => ({ clusterKey, landmarks: reports[i] }));
 }
 /**
  * Async twin of {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeysInMemory}
@@ -453,48 +495,6 @@ async function resolveSmallCorpusWithProgress(pages, onProgress, options) {
     }
     return finalKeys;
 }
-/**
- * Streaming, memory-bounded version of `resolvePageClusterKeysInMemory`.
- *
- * ## Behavior gate
- *
- * - `pageCount ≤ CORPUS_INLINE_THRESHOLD` — reads the whole factory into an
- *   array, delegates to `resolvePageClusterKeysInMemory`. Same corpus-wide
- *   chrome discovery, same Stage B across every page. All previously
- *   validated corpora (302 / 1,416 / 8,936 / 89 pages) hit this path.
- * - `pageCount > CORPUS_INLINE_THRESHOLD` — streaming path: reads the
- *   factory twice (once for blocking signals, once for HTML processing),
- *   dispatches HTML per block, runs Stage A per block, accumulates
- *   cross-block units, then runs Stage B across all accumulated units. Peak
- *   memory ≈ largest single block, not the whole corpus.
- *
- * ## Semantic differences in streaming mode
- *
- * - **Chrome discovery is per-block, not corpus-wide.** In the in-memory
- *   path, {@link ./resolve-page-cluster-keys.js | computeLocalLandmarkTokens}
- *   runs on all pages at once. In streaming mode the entire corpus cannot
- *   be held at once, so chrome discovery runs per block. A landmark
- *   signature that is rare corpus-wide but common within one block will
- *   be treated as global chrome in streaming mode, whereas the in-memory
- *   mode would treat it as local. This trade-off is why the threshold
- *   above is set generously — every real corpus historically validated
- *   here stays on the in-memory path.
- * @param pages
- * @param options
- * @example
- * ```ts
- * // JSONL file source — factory can be re-invoked to re-open the file.
- * import { createReadStream } from 'node:fs';
- * import readline from 'node:readline';
- *
- * const keys = await resolvePageClusterKeys(() => {
- *   const lines = readline.createInterface({ input: createReadStream('pages.jsonl') });
- *   return (async function* () {
- *     for await (const line of lines) yield JSON.parse(line);
- *   })();
- * });
- * ```
- */
 export async function resolvePageClusterKeys(pages, options) {
     const onProgress = options?.onProgress;
     // Pass 0: HTML-free — collect blocking signals (paths, stylesheetHrefs,
@@ -530,13 +530,22 @@ export async function resolvePageClusterKeys(pages, options) {
         // into per-block progress, so delegate to the untouched sync path —
         // keeping behavior byte-for-byte identical (and yield-overhead-free)
         // to how library-only consumers experienced this before the CLI
-        // progress work landed.
-        if (onProgress === undefined) {
+        // progress work landed. `includeLandmarkPositions` always routes here
+        // too (see its own JSDoc): `resolveSmallCorpusWithProgress` has no
+        // landmark-report support, and duplicating that logic into the
+        // progress-emitting path for a reporting feature that has nothing to
+        // do with progress observability isn't worth the added surface.
+        if (onProgress === undefined || options?.includeLandmarkPositions) {
             return resolvePageClusterKeysInMemory(fullPages, options);
         }
         return resolveSmallCorpusWithProgress(fullPages, onProgress, options);
     }
-    // Large corpus: streaming path.
+    // Large corpus: streaming path. includeLandmarkPositions has no sample-
+    // based equivalent (see its own JSDoc) — fail fast rather than silently
+    // ignoring the option or returning a semantically-wrong report.
+    if (options?.includeLandmarkPositions) {
+        assertLandmarkPositionsSupportedForPageCount(blockingSignals.length, CORPUS_INLINE_THRESHOLD);
+    }
     const excludeLandmarks = options?.excludeLandmarks ?? true;
     const similarityThreshold = options?.similarityThreshold ?? 0.8;
     if (!(similarityThreshold >= 0 && similarityThreshold <= 1)) {
@@ -728,22 +737,6 @@ export async function resolvePageClusterKeys(pages, options) {
     }
     return finalKeys;
 }
-/**
- * Convenience wrapper that runs {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}
- * on a materialized array. Preserves the pre-refactor sync API for callers
- * that already have all pages in memory, while flowing through the same
- * async driver so behavior stays consistent across the two entry points.
- * @param pages
- * @param options
- * @example
- * ```ts
- * const keys = await resolvePageClusterKeysFromArray([
- *   { paths: ['news', '1'], stylesheetHrefs: [], html: '<body><article>one</article></body>' },
- *   { paths: ['news', '2'], stylesheetHrefs: [], html: '<body><article>two</article></body>' },
- *   { paths: ['about'], stylesheetHrefs: [], html: '<body><section>about</section></body>' },
- * ]);
- * ```
- */
 export function resolvePageClusterKeysFromArray(pages, options) {
     return resolvePageClusterKeys(() => pages, options);
 }