npm - @d-zero/page-cluster - Versions diffs - 0.3.1 → 0.5.0 - Mend

@d-zero/page-cluster 0.3.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (28) hide show

package/README.md +45 -8
package/dist/build-cluster-reason.d.ts +119 -0
package/dist/build-cluster-reason.js +63 -0
package/dist/cli.d.ts +1 -0
package/dist/cli.js +64 -6
package/dist/derive-blocking-reason.d.ts +32 -0
package/dist/derive-blocking-reason.js +1 -0
package/dist/extract-landmarks.d.ts +75 -14
package/dist/extract-landmarks.js +64 -4
package/dist/is-chrome-landmark-instance.d.ts +41 -0
package/dist/is-chrome-landmark-instance.js +50 -0
package/dist/merge-cross-block-clusters.d.ts +39 -3
package/dist/merge-cross-block-clusters.js +28 -107
package/dist/offset-to-line-column.d.ts +38 -0
package/dist/offset-to-line-column.js +49 -0
package/dist/pass0-blocking.d.ts +10 -0
package/dist/pass0-blocking.js +26 -46
package/dist/per-page-landmark-signatures.d.ts +9 -1
package/dist/per-page-landmark-signatures.js +16 -4
package/dist/reassign-orphan-block-keys.d.ts +9 -0
package/dist/reassign-orphan-block-keys.js +5 -2
package/dist/resolve-blocking-group-keys.d.ts +10 -0
package/dist/resolve-blocking-group-keys.js +16 -86
package/dist/resolve-page-cluster-keys.d.ts +37 -1
package/dist/resolve-page-cluster-keys.js +123 -17
package/dist/shell-quorum.d.ts +70 -0
package/dist/shell-quorum.js +110 -0
package/package.json +10 -2

package/dist/extract-landmarks.js CHANGED Viewed

@@ -1,11 +1,13 @@
 import { excise } from './excise.js';
 import { findMatchingElements, } from './find-shallowest-elements.js';
+import { buildLineColumnIndex, offsetToLineColumn } from './offset-to-line-column.js';
 const TAG_TO_TYPE = {
     header: 'header',
     footer: 'footer',
     nav: 'nav',
     aside: 'aside',
     search: 'search',
+    main: 'main',
 };
 const ROLE_TO_TYPE = {
     banner: 'header',
@@ -14,6 +16,7 @@ const ROLE_TO_TYPE = {
     complementary: 'aside',
     form: 'form',
     search: 'search',
+    main: 'main',
 };
 /**
  * Determines which landmark type(s) an element matches by tag name or
@@ -40,6 +43,16 @@ function matchLandmarkTypes(tagName, role) {
     }
     return types;
 }
+/**
+ * Type guard splitting `findMatchingElements`' combined match list into the
+ * excisable six landmark types vs `main`. `main` is content, not chrome, and
+ * must never be mixed into the same `keepOutermost` sweep as the other six
+ * — see `extractLandmarks`'s "main handling" note for why.
+ * @param match
+ */
+function isMainMatch(match) {
+    return match.type === 'main';
+}
 /**
  * Filters out any match whose whole-element span is strictly contained by
  * another match's span, keeping only outermost instances. Runs across all
@@ -151,21 +164,50 @@ function keepOutermost(matches) {
  * segment then disappears from the surviving paths, shortening them by one
  * level. This is inherent to "delete the matched span, use whatever's
  * left" and is not treated as a bug.
+ *
+ * ## Main handling
+ *
+ * `main` (the `<main>` tag or `role="main"`) is collected the same way as
+ * the other six types — one entry per genuinely-closed instance, in
+ * document order — but is kept out of every mechanism the other six feed:
+ *
+ * - It is **never excised**: its span is never added to `remainderHtml`'s
+ *   excise list, because `main` is the page's actual content, not chrome.
+ *   Removing it would gut `remainderHtml` down to whatever sits outside
+ *   `<main>` (nothing, on most real pages).
+ * - Its `keepOutermost` nesting sweep runs **separately** from the other six
+ *   types'. If it shared the sweep, a `<main>` that wraps most of the page
+ *   (as it typically does) would make every `header`/`nav`/`aside` nested
+ *   inside it look "contained by main" and get dropped — destroying the
+ *   section-local chrome detection this module exists to enable (see "Why
+ *   collect every instance" above). Only nested `<main>`s (an edge case —
+ *   HTML discourages more than one) are deduplicated against each other.
+ * - It never contributes to chrome/shell-frequency analysis (`main` is
+ *   absent from `computePerPageLandmarkInstances`'s `ALL_LANDMARK_TYPES`):
+ *   its instances are reported for position purposes only, never treated as
+ *   candidate chrome.
  * @param html
  * @example
  * ```ts
  * extractLandmarks('<body><header>H</header><main>M</main><footer>F</footer></body>');
  * // {
- * //   header: ['<header>H</header>'],
- * //   footer: ['<footer>F</footer>'],
+ * //   header: [{ html: '<header>H</header>', startOffset: 6, endOffset: 24,
+ * //              startLine: 1, startColumn: 7, endLine: 1, endColumn: 25 }],
+ * //   footer: [{ html: '<footer>F</footer>', startOffset: 38, endOffset: 56,
+ * //              startLine: 1, startColumn: 39, endLine: 1, endColumn: 57 }],
  * //   nav: [], aside: [], form: [], search: [],
+ * //   main: [{ html: '<main>M</main>', startOffset: 24, endOffset: 38,
+ * //            startLine: 1, startColumn: 25, endLine: 1, endColumn: 39 }],
  * //   remainderHtml: '<body><main>M</main></body>',
  * // }
  * ```
  */
 export function extractLandmarks(html) {
     const allMatches = findMatchingElements(html, matchLandmarkTypes);
-    const outermost = keepOutermost(allMatches);
+    const mainMatches = allMatches.filter(isMainMatch);
+    const excisableMatches = allMatches.filter((match) => !isMainMatch(match));
+    const outermost = keepOutermost(excisableMatches);
+    const outermostMain = keepOutermost(mainMatches);
     const result = {
         header: [],
         footer: [],
@@ -173,13 +215,31 @@ export function extractLandmarks(html) {
         aside: [],
         form: [],
         search: [],
+        main: [],
         remainderHtml: html,
     };
     const spans = [];
+    const lineColumnIndex = buildLineColumnIndex(html);
+    const toInstance = (match) => {
+        const start = offsetToLineColumn(lineColumnIndex, match.startOffset);
+        const end = offsetToLineColumn(lineColumnIndex, match.endOffset);
+        return {
+            html: html.slice(match.startOffset, match.endOffset),
+            startOffset: match.startOffset,
+            endOffset: match.endOffset,
+            startLine: start.line,
+            startColumn: start.column,
+            endLine: end.line,
+            endColumn: end.column,
+        };
+    };
     for (const match of outermost) {
-        result[match.type].push(html.slice(match.startOffset, match.endOffset));
+        result[match.type].push(toInstance(match));
         spans.push({ start: match.startOffset, end: match.endOffset });
     }
+    for (const match of outermostMain) {
+        result.main.push(toInstance(match));
+    }
     result.remainderHtml = excise(html, spans);
     return result;
 }

package/dist/is-chrome-landmark-instance.d.ts ADDED Viewed

@@ -0,0 +1,41 @@
+/**
+ * Default containment threshold for {@link ./is-chrome-landmark-instance.js | isChromeLandmarkInstance}.
+ * Chosen to match the quorum/shell fractions used elsewhere in this pipeline
+ * ({@link ./merge-cross-block-clusters.js | mergeCrossBlockClusters}'s
+ * `QUORUM_FRACTION`, {@link ./shell-quorum.js | SHELL_QUORUM_FALLBACK_FRACTION}) —
+ * unlike those, this exact value has not been validated against real crawl
+ * data; it is a starting point carried over by convention, adjustable via
+ * this function's `threshold` parameter.
+ */
+export declare const DEFAULT_CHROME_OVERLAP_THRESHOLD = 0.8;
+/**
+ * Classifies a single landmark instance as chrome (shared site/section
+ * furniture) or content, given the instance's own tokens and the shell
+ * token set {@link ./shell-quorum.js | shellQuorum} discovered for its unit.
+ *
+ * ## Why containment (`|instance ∩ shell| / |instance|`) and not Jaccard
+ *
+ * `shellTokens` is the union of chrome tokens across an entire unit's
+ * landmark instances (header + nav + footer + …, corpus-wide), so it is
+ * usually far larger than any single instance's own token set. Jaccard's
+ * denominator is the *union* of both sets, which stays shell-sized even when
+ * the instance is 100% shell tokens — driving the score down regardless of
+ * how purely "shell" the instance is. Containment instead asks "of this
+ * instance's own tokens, how many are shell tokens", which is the question
+ * that actually matters for classifying one instance.
+ *
+ * An instance with zero tokens is never chrome (there is nothing to
+ * corroborate) — matches {@link ./per-page-landmark-signatures.js | computePerPageLandmarkInstances}'s
+ * own choice to drop zero-token instances before they ever reach a
+ * `PerPageLandmarkInstance`, kept here as a defensive default rather than an
+ * assumption about every caller.
+ * @param instanceTokens
+ * @param shellTokens
+ * @param threshold
+ * @example
+ * ```ts
+ * const shellTokens = shellQuorum(unitPerPageInstances);
+ * const isChrome = isChromeLandmarkInstance(instance.tokens, shellTokens);
+ * ```
+ */
+export declare function isChromeLandmarkInstance(instanceTokens: ReadonlySet<string>, shellTokens: ReadonlySet<string>, threshold?: number): boolean;

package/dist/is-chrome-landmark-instance.js ADDED Viewed

@@ -0,0 +1,50 @@
+/**
+ * Default containment threshold for {@link ./is-chrome-landmark-instance.js | isChromeLandmarkInstance}.
+ * Chosen to match the quorum/shell fractions used elsewhere in this pipeline
+ * ({@link ./merge-cross-block-clusters.js | mergeCrossBlockClusters}'s
+ * `QUORUM_FRACTION`, {@link ./shell-quorum.js | SHELL_QUORUM_FALLBACK_FRACTION}) —
+ * unlike those, this exact value has not been validated against real crawl
+ * data; it is a starting point carried over by convention, adjustable via
+ * this function's `threshold` parameter.
+ */
+export const DEFAULT_CHROME_OVERLAP_THRESHOLD = 0.8;
+/**
+ * Classifies a single landmark instance as chrome (shared site/section
+ * furniture) or content, given the instance's own tokens and the shell
+ * token set {@link ./shell-quorum.js | shellQuorum} discovered for its unit.
+ *
+ * ## Why containment (`|instance ∩ shell| / |instance|`) and not Jaccard
+ *
+ * `shellTokens` is the union of chrome tokens across an entire unit's
+ * landmark instances (header + nav + footer + …, corpus-wide), so it is
+ * usually far larger than any single instance's own token set. Jaccard's
+ * denominator is the *union* of both sets, which stays shell-sized even when
+ * the instance is 100% shell tokens — driving the score down regardless of
+ * how purely "shell" the instance is. Containment instead asks "of this
+ * instance's own tokens, how many are shell tokens", which is the question
+ * that actually matters for classifying one instance.
+ *
+ * An instance with zero tokens is never chrome (there is nothing to
+ * corroborate) — matches {@link ./per-page-landmark-signatures.js | computePerPageLandmarkInstances}'s
+ * own choice to drop zero-token instances before they ever reach a
+ * `PerPageLandmarkInstance`, kept here as a defensive default rather than an
+ * assumption about every caller.
+ * @param instanceTokens
+ * @param shellTokens
+ * @param threshold
+ * @example
+ * ```ts
+ * const shellTokens = shellQuorum(unitPerPageInstances);
+ * const isChrome = isChromeLandmarkInstance(instance.tokens, shellTokens);
+ * ```
+ */
+export function isChromeLandmarkInstance(instanceTokens, shellTokens, threshold = DEFAULT_CHROME_OVERLAP_THRESHOLD) {
+    if (instanceTokens.size === 0)
+        return false;
+    let hit = 0;
+    for (const token of instanceTokens) {
+        if (shellTokens.has(token))
+            hit++;
+    }
+    return hit / instanceTokens.size >= threshold;
+}

package/dist/merge-cross-block-clusters.d.ts CHANGED Viewed

@@ -18,11 +18,47 @@ export type CrossBlockUnit = {
     readonly memberTokenSets: readonly ReadonlySet<string>[];
     readonly memberLandmarkInstances: readonly (readonly PerPageLandmarkInstance[])[];
 };
+/**
+ * Frequency-based token core of a group of member pages: a token must be
+ * present in at least `QUORUM_FRACTION` of members to enter the core, with a
+ * full-union fallback when no token clears that bar (see
+ * {@link mergeCrossBlockClusters}'s JSDoc for why quorum beats strict
+ * intersection or full union). Exported so callers building a `ClusterReason`
+ * (`build-cluster-reason.ts`) can re-derive a final group's structural core
+ * from `finalGroupsByRoot` without duplicating this logic.
+ * @param memberDistinctiveTokens
+ */
+export declare function computeQuorumCore(memberDistinctiveTokens: readonly ReadonlySet<string>[]): ReadonlySet<string>;
+/**
+ * One root key's pooled member state after Stage B converges: every
+ * `tokenSets`/`landmarkInstances` entry folded in from every unit merged into
+ * this root (down-sampled to `capMembers` when the caller opts in, same as
+ * during merging). This is the exact state `mergeCrossBlockClusters` already
+ * builds internally to run quorum-core/shell comparisons each round — it was
+ * discarded once `keyToRoot` was returned. Exposing it lets a `ClusterReason`
+ * (`build-cluster-reason.ts`) be built from data Stage B already computed,
+ * with no extra pass over the corpus.
+ */
+export type FinalGroupMembers = {
+    readonly tokenSets: readonly ReadonlySet<string>[];
+    readonly landmarkInstances: readonly (readonly PerPageLandmarkInstance[])[];
+};
+/**
+ * `mergeCrossBlockClusters`'s result: the root-key mapping every caller
+ * needs for `clusterKey` resolution, plus each surviving root's final pooled
+ * member state for callers that also want to explain *why* (`ClusterReason`).
+ */
+export type MergeCrossBlockClustersResult = {
+    /** Every input unit's `key` mapped to its final root key. Units not absorbed into any other unit map to themselves. */
+    readonly rootByKey: ReadonlyMap<string, string>;
+    /** Every surviving root key's final pooled member state. */
+    readonly finalGroupsByRoot: ReadonlyMap<string, FinalGroupMembers>;
+};
 /**
  * Merges cross-block clusters (Stage B) via recursive quorum-core comparison.
  *
- * Returns a `Map` from each input unit's `key` to its final root key. Units
- * not absorbed into any other unit map to themselves.
+ * Returns the root-key mapping (see {@link MergeCrossBlockClustersResult}).
+ * Units not absorbed into any other unit map to themselves.
  *
  * Three merge mechanisms run per round, in order:
  * 1. **Fine stage** — complete-linkage at `CROSS_BLOCK_THRESHOLD` on quorum
@@ -58,4 +94,4 @@ export declare function mergeCrossBlockClusters(units: readonly CrossBlockUnit[]
      * full-membership merge behavior unchanged.
      */
     capMembers?: number;
-}): Map<string, string>;
+}): MergeCrossBlockClustersResult;

package/dist/merge-cross-block-clusters.js CHANGED Viewed

@@ -1,11 +1,11 @@
 import { assignContainedClusters } from './assign-contained-clusters.js';
-import { autoCutThreshold } from './auto-cut-threshold.js';
 import { collapseAnonymousDivs } from './collapse-anonymous-divs.js';
 import { completeLinkageDendrogram, labelsAtThreshold, } from './complete-linkage-dendrogram.js';
 import { computeDocumentFrequency } from './compute-document-frequency.js';
 import { jaccardSimilarity } from './jaccard-similarity.js';
 import { reservoirSample } from './reservoir-sample.js';
 import { shapeToken } from './shape-token.js';
+import { shellQuorum } from './shell-quorum.js';
 import { splitTokensByFrequency } from './split-tokens-by-frequency.js';
 /**
  * Fixed complete-linkage threshold for the cross-block fine stage.
@@ -26,11 +26,10 @@ const CROSS_BLOCK_THRESHOLD = 0.8;
  * on real crawl data. Full union is shell-dominated: 298 pages collapsed into
  * 4 clusters — also confirmed. 80% quorum avoids both failure modes.
  *
- * Also reused as the fallback clamp for {@link ./auto-cut-threshold.js |
- * autoCutThreshold} when running on the per-landmark-instance frequency
- * distribution in {@link ./merge-cross-block-clusters.js | shellQuorum}. The
- * clamp only ever *loosens* the cut relative to this floor (never tightens),
- * and only fires in the degenerate cases the JSDoc there describes.
+ * Not shared with {@link ./shell-quorum.js | shellQuorum}'s own fallback
+ * clamp (`SHELL_QUORUM_FALLBACK_FRACTION`) — the two happen to be the same
+ * value today because both were validated against the same real crawl
+ * corpora, but they are independently tunable.
  */
 const QUORUM_FRACTION = 0.8;
 /**
@@ -84,10 +83,16 @@ const GENERIC_SEGMENTS = new Set([
 // Internal helpers
 // ---------------------------------------------------------------------------
 /**
- *
+ * Frequency-based token core of a group of member pages: a token must be
+ * present in at least `QUORUM_FRACTION` of members to enter the core, with a
+ * full-union fallback when no token clears that bar (see
+ * {@link mergeCrossBlockClusters}'s JSDoc for why quorum beats strict
+ * intersection or full union). Exported so callers building a `ClusterReason`
+ * (`build-cluster-reason.ts`) can re-derive a final group's structural core
+ * from `finalGroupsByRoot` without duplicating this logic.
  * @param memberDistinctiveTokens
  */
-function quorumCore(memberDistinctiveTokens) {
+export function computeQuorumCore(memberDistinctiveTokens) {
     const n = memberDistinctiveTokens.length;
     if (n === 0)
         return new Set();
@@ -159,105 +164,11 @@ function l2Contained(xSig, ySig) {
     }
     return true;
 }
-/**
- * Discovers a unit's shell tokens by auto-cutting the per-*token* page-
- * frequency histogram of every landmark instance's tokens. This is the same
- * max-gap primitive used for Stage A merge-height cutoffs, applied
- * recursively at the landmark-token layer.
- *
- * ## Why per-token and not per-signature
- *
- * An earlier iteration ran the histogram at the level of full landmark-
- * instance signatures (canonicalized token sets). That failed on a real,
- * common pattern: a shared site chrome whose markup carries a per-page
- * distinguishing element (a breadcrumb, a page-title element with a page-
- * specific class, a "current" state). All pages have most of the same
- * tokens, but every page's full signature is distinct because tokens embed
- * class names. Per-signature counting saw 5 signatures at freq 0.2 each,
- * autoCutThreshold on the flat distribution returned the clamp, and the
- * shell collapsed to empty even though every page shared the core header
- * skeleton. Per-token counting handles the same case correctly — the shared
- * skeleton tokens each hit freq 1.0.
- *
- * ## The histogram
- *
- * For every member page, all its landmark instances are tokenized and
- * unioned into a single per-page token set (order-agnostic, deduped: a
- * token appearing in two of the page's landmarks still counts once for
- * that page). The corpus histogram is then "how many pages contain each
- * token". Tokens that appear on nearly every page are the unit's chrome;
- * tokens that appear on only a handful are page-specific content that
- * happens to be tagged as a landmark.
- *
- * ## Why auto-cut instead of a hard-coded quorum
- *
- * A fixed 80% quorum (this file's earlier implementation) baked one
- * threshold in for every unit. Real corpora don't obey a universal cutoff:
- * a section-local landmark token that appears on 60% of a unit's pages is
- * the section's chrome under any reasonable reading, but 80% quorum
- * discards it. Auto-cut looks at the *shape* of the frequency distribution
- * and picks the widest gap between adjacent frequencies — if the
- * distribution is `{1.00, 1.00, 0.65, 0.03, 0.02}`, the gap between 0.65
- * and 0.03 (0.62) dwarfs everything else and the cut lands mid-gap around
- * 0.34, correctly grouping the 0.65 tokens with the site-wide 1.00 ones as
- * "chrome for this unit". If instead the distribution is flat, the clamp
- * to {@link QUORUM_FRACTION} keeps the threshold from being tighter than
- * the fallback default.
- *
- * ## Fallbacks
- *
- * A single distinct token (`heights.length < 2`) or a perfectly flat
- * distribution (`maxGap === 0`) returns the {@link QUORUM_FRACTION} clamp
- * verbatim — exactly the same 80%-quorum behavior as before. So degenerate
- * cases degrade to the old contract; only richer distributions get the
- * auto-cut benefit.
- *
- * A page with no landmarks contributes an empty set, deliberately, so the
- * shell-corroboration jaccard between two landmark-less pages is 0 rather
- * than 1 (which it would be if we handed back a `<body></body>`-derived
- * `{body}` fallback set to both sides).
- * @param perPageInstances
- */
-function shellQuorum(perPageInstances) {
-    const pageCount = perPageInstances.length;
-    if (pageCount === 0)
-        return new Set();
-    // Union all instance token sets per page (dedupe within page: a token
-    // present on both header and footer of the same page still counts once
-    // for that page's contribution).
-    const tokenPageCount = new Map();
-    for (const instances of perPageInstances) {
-        const perPageUnion = new Set();
-        for (const inst of instances) {
-            for (const token of inst.tokens)
-                perPageUnion.add(token);
-        }
-        for (const token of perPageUnion) {
-            tokenPageCount.set(token, (tokenPageCount.get(token) ?? 0) + 1);
-        }
-    }
-    if (tokenPageCount.size === 0)
-        return new Set();
-    const frequencies = [];
-    for (const count of tokenPageCount.values()) {
-        frequencies.push(count / pageCount);
-    }
-    const cut = autoCutThreshold(frequencies, QUORUM_FRACTION);
-    const shell = new Set();
-    for (const [token, count] of tokenPageCount) {
-        if (count / pageCount >= cut)
-            shell.add(token);
-    }
-    return shell;
-}
-// ---------------------------------------------------------------------------
-// Main function
-// ---------------------------------------------------------------------------
 /**
  * Merges cross-block clusters (Stage B) via recursive quorum-core comparison.
  *
- * Returns a `Map` from each input unit's `key` to its final root key. Units
- * not absorbed into any other unit map to themselves.
+ * Returns the root-key mapping (see {@link MergeCrossBlockClustersResult}).
+ * Units not absorbed into any other unit map to themselves.
  *
  * Three merge mechanisms run per round, in order:
  * 1. **Fine stage** — complete-linkage at `CROSS_BLOCK_THRESHOLD` on quorum
@@ -283,7 +194,13 @@ function shellQuorum(perPageInstances) {
  */
 export function mergeCrossBlockClusters(units, options) {
     if (units.length <= 1) {
-        return new Map(units.map((u) => [u.key, u.key]));
+        return {
+            rootByKey: new Map(units.map((u) => [u.key, u.key])),
+            finalGroupsByRoot: new Map(units.map((u) => [
+                u.key,
+                { tokenSets: u.memberTokenSets, landmarkInstances: u.memberLandmarkInstances },
+            ])),
+        };
     }
     const threshold = options?.similarityThreshold ?? CROSS_BLOCK_THRESHOLD;
     const capMembers = options?.capMembers;
@@ -356,7 +273,7 @@ export function mergeCrossBlockClusters(units, options) {
         // Quorum core per group
         const cores = new Map();
         for (const key of groupKeys) {
-            cores.set(key, quorumCore(groupDistinctive.get(key) ?? []));
+            cores.set(key, computeQuorumCore(groupDistinctive.get(key) ?? []));
         }
         // ---------------------------------------------------------------
         // Fine stage: union-find over group indices
@@ -542,5 +459,9 @@ export function mergeCrossBlockClusters(units, options) {
             break; // fully converged
         applyMerges(l2Merges);
     }
-    return keyToRoot;
+    const finalGroupsByRoot = new Map([...groups.entries()].map(([root, g]) => [
+        root,
+        { tokenSets: g.tokenSets, landmarkInstances: g.landmarkInstances },
+    ]));
+    return { rootByKey: keyToRoot, finalGroupsByRoot };
 }

package/dist/offset-to-line-column.d.ts ADDED Viewed

@@ -0,0 +1,38 @@
+/**
+ * A 1-based line/column position within an HTML string.
+ */
+export type LineColumn = {
+    readonly line: number;
+    readonly column: number;
+};
+/**
+ * Precomputed lookup structure for {@link ./offset-to-line-column.js | offsetToLineColumn}:
+ * every newline's string-index offset, ascending.
+ */
+export type LineColumnIndex = {
+    readonly newlineOffsets: readonly number[];
+};
+/**
+ * Scans `html` once and records every `\n` offset, so repeated
+ * {@link ./offset-to-line-column.js | offsetToLineColumn} calls against the
+ * same HTML string can binary-search instead of re-scanning from the start
+ * each time. Built once per page and reused across every landmark instance's
+ * start/end offset — a page with dozens of landmark instances would
+ * otherwise pay for a fresh O(n) scan per offset instead of one O(n) scan
+ * total.
+ * @param html
+ */
+export declare function buildLineColumnIndex(html: string): LineColumnIndex;
+/**
+ * Converts a string-index `offset` (the same unit as `htmlparser2`'s
+ * `startIndex`/`endIndex`, i.e. UTF-16 code units) into a 1-based
+ * `{line, column}` position, using an index built by
+ * {@link ./offset-to-line-column.js | buildLineColumnIndex}.
+ *
+ * `\r\n` line endings are handled without special-casing: the `\r` is
+ * counted as the last column of its own line, matching how most editors
+ * report position for CRLF files.
+ * @param index
+ * @param offset
+ */
+export declare function offsetToLineColumn(index: LineColumnIndex, offset: number): LineColumn;

package/dist/offset-to-line-column.js ADDED Viewed

@@ -0,0 +1,49 @@
+/**
+ * Scans `html` once and records every `\n` offset, so repeated
+ * {@link ./offset-to-line-column.js | offsetToLineColumn} calls against the
+ * same HTML string can binary-search instead of re-scanning from the start
+ * each time. Built once per page and reused across every landmark instance's
+ * start/end offset — a page with dozens of landmark instances would
+ * otherwise pay for a fresh O(n) scan per offset instead of one O(n) scan
+ * total.
+ * @param html
+ */
+export function buildLineColumnIndex(html) {
+    const newlineOffsets = [];
+    for (let i = 0; i < html.length; i++) {
+        if (html.codePointAt(i) === 10)
+            newlineOffsets.push(i);
+    }
+    return { newlineOffsets };
+}
+/**
+ * Converts a string-index `offset` (the same unit as `htmlparser2`'s
+ * `startIndex`/`endIndex`, i.e. UTF-16 code units) into a 1-based
+ * `{line, column}` position, using an index built by
+ * {@link ./offset-to-line-column.js | buildLineColumnIndex}.
+ *
+ * `\r\n` line endings are handled without special-casing: the `\r` is
+ * counted as the last column of its own line, matching how most editors
+ * report position for CRLF files.
+ * @param index
+ * @param offset
+ */
+export function offsetToLineColumn(index, offset) {
+    const { newlineOffsets } = index;
+    let low = 0;
+    let high = newlineOffsets.length;
+    while (low < high) {
+        const mid = (low + high) >>> 1;
+        if (newlineOffsets[mid] < offset) {
+            low = mid + 1;
+        }
+        else {
+            high = mid;
+        }
+    }
+    // `low` is the count of newlines strictly before `offset`, i.e. the
+    // number of completed lines — so `low` newlines completed means we're on
+    // line `low + 1`.
+    const lineStart = low === 0 ? 0 : newlineOffsets[low - 1] + 1;
+    return { line: low + 1, column: offset - lineStart + 1 };
+}

package/dist/pass0-blocking.d.ts CHANGED Viewed

@@ -1,3 +1,4 @@
+import type { BlockingReason } from './derive-blocking-reason.js';
 import type { ResolveBlockingGroupKeysOptions } from './resolve-blocking-group-keys.js';
 /**
  * The corpus-wide, HTML-free inputs needed by
@@ -31,6 +32,12 @@ export type ResolveBlockKeysOptions = ResolveBlockingGroupKeysOptions & {
      */
     readonly restrictStylesheetsToFirstParty?: boolean;
 };
+/** Return shape when `includeReasons: true` is passed to `resolveBlockKeys`. */
+export type BlockKeysWithReasons = {
+    readonly blockKeys: string[];
+    /** One entry per distinct final block key produced, keyed by that key. */
+    readonly reasonsByBlockKey: ReadonlyMap<string, BlockingReason>;
+};
 /**
  * Splits `resolvePageClusterKeys` into a size-flat first pass so the driver
  * can decide per-block memory strategy before loading any page HTML. Runs the
@@ -71,6 +78,9 @@ export type ResolveBlockKeysOptions = ResolveBlockingGroupKeysOptions & {
  * // ['css:<hash>', 'css:<hash>', 'path:about']
  * ```
  */
+export declare function resolveBlockKeys(pages: readonly Pass0PageSignals[], options: ResolveBlockKeysOptions & {
+    includeReasons: true;
+}): BlockKeysWithReasons;
 export declare function resolveBlockKeys(pages: readonly Pass0PageSignals[], options?: ResolveBlockKeysOptions): string[];
 /**
  * Groups pages by their block key while preserving each block's members in

package/dist/pass0-blocking.js CHANGED Viewed

@@ -1,62 +1,42 @@
 import { filterFirstPartyStylesheetHrefs } from './filter-first-party-stylesheet-hrefs.js';
-import { reassignOrphanBlockKeys } from './reassign-orphan-block-keys.js';
+import { REASSIGNED_KEY_PREFIX, reassignOrphanBlockKeys, } from './reassign-orphan-block-keys.js';
 import { resolveBlockingGroupKeys } from './resolve-blocking-group-keys.js';
-/**
- * Splits `resolvePageClusterKeys` into a size-flat first pass so the driver
- * can decide per-block memory strategy before loading any page HTML. Runs the
- * three corpus-wide, HTML-free stages of blocking in the same order the
- * in-memory driver already uses — first-party stylesheet filtering, blocking-
- * key derivation, orphan reassignment — and returns one final block key per
- * input page in input order.
- *
- * ## Why extract this from resolvePageClusterKeys?
- *
- * The in-memory driver holds every page's `html`, `remainderHtml`,
- * `landmarks[]`, and pre-Stage-A prepared HTML at once. At 176k pages × ~57
- * KB average, that alone breaks a 17 GB RAM machine well before Stage A
- * starts (measured: OS SIGKILL at ~15,000 pages, before the resolve phase
- * even began). All three blocking stages, in contrast, depend only on
- * `paths` / `stylesheetHrefs` / `host` — a few hundred bytes per page. Running
- * them first, HTML-free, lets the downstream per-block clustering hold HTML
- * for only one block's pages at a time.
- *
- * ## Preserves in-memory driver semantics exactly
- *
- * The output of this function is byte-identical to the block-key portion of
- * the current `resolvePageClusterKeys` for the same input, because it reuses
- * the same three underlying functions in the same order with the same
- * defaults. That guarantee is load-bearing: the size-threshold-gated Pass 1
- * that follows this function runs the current in-memory implementation
- * unchanged for small blocks, and any drift in block-key computation between
- * Pass 0 and Pass 1 would silently misroute pages between the two paths.
- * @param pages
- * @param options
- * @example
- * ```ts
- * const blockKeys = resolveBlockKeys([
- *   { paths: ['news', '1'], stylesheetHrefs: ['/a.css'], host: 'example.com' },
- *   { paths: ['news', '2'], stylesheetHrefs: ['/a.css'], host: 'example.com' },
- *   { paths: ['about'],     stylesheetHrefs: [],        host: 'example.com' },
- * ]);
- * // ['css:<hash>', 'css:<hash>', 'path:about']
- * ```
- */
 export function resolveBlockKeys(pages, options) {
     const restrictStylesheetsToFirstParty = options?.restrictStylesheetsToFirstParty ?? true;
     const blockingPages = restrictStylesheetsToFirstParty
         ? filterFirstPartyStylesheetHrefs(pages)
         : pages;
-    const rawBlockKeys = resolveBlockingGroupKeys(blockingPages, options);
-    const reassignOrphans = options?.reassignOrphans ?? true;
-    if (!reassignOrphans)
-        return rawBlockKeys;
     // Orphan reassignment always uses a numeric `pathDepth`. When the caller
     // asked for `'auto'`, fall back to the historical default 1 here — a
     // future PR that wires the auto-cut depth through can compute it once
     // and pass it as a number to both `resolveBlockingGroupKeys` and this
     // call to keep them consistent.
     const numericPathDepth = typeof options?.pathDepth === 'number' ? options.pathDepth : undefined;
-    return reassignOrphanBlockKeys(blockingPages, rawBlockKeys, numericPathDepth);
+    const reassignOrphans = options?.reassignOrphans ?? true;
+    if (!options?.includeReasons) {
+        const rawBlockKeys = resolveBlockingGroupKeys(blockingPages, options);
+        if (!reassignOrphans)
+            return rawBlockKeys;
+        return reassignOrphanBlockKeys(blockingPages, rawBlockKeys, numericPathDepth);
+    }
+    const { keys: rawBlockKeys, reasonsByKey } = resolveBlockingGroupKeys(blockingPages, {
+        ...options,
+        includeReasons: true,
+    });
+    if (!reassignOrphans) {
+        return { blockKeys: rawBlockKeys, reasonsByBlockKey: reasonsByKey };
+    }
+    const finalBlockKeys = reassignOrphanBlockKeys(blockingPages, rawBlockKeys, numericPathDepth);
+    const reasonsByBlockKey = new Map(reasonsByKey);
+    for (const blockKey of finalBlockKeys) {
+        if (reasonsByBlockKey.has(blockKey))
+            continue;
+        reasonsByBlockKey.set(blockKey, {
+            kind: 'orphanMerge',
+            pathKey: blockKey.slice(REASSIGNED_KEY_PREFIX.length),
+        });
+    }
+    return { blockKeys: finalBlockKeys, reasonsByBlockKey };
 }
 /**
  * Groups pages by their block key while preserving each block's members in