@d-zero/page-cluster 0.2.0

package/dist/derive-stylesheet-group-key.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Derives a coarse grouping key from the set of stylesheet URLs a page
+ * loads. This is a *blocking key* in the record-linkage sense (see
+ * {@link ./derive-path-group-key.js | derivePathGroupKey}): pages sharing
+ * the exact same stylesheet set are near-certainly the same template
+ * family, making this a strong but sparse signal — many pages load few or
+ * no stylesheets, so this key is meant to be used alongside, not instead
+ * of, weaker-but-always-present signals like a URL-path-based key.
+ *
+ * `stylesheetHrefs` must already be resolved to a form that is comparable
+ * across the whole corpus (e.g. absolute URLs). This function only compares
+ * the strings it is given: two pages that both reference the same
+ * unresolved relative href text (e.g. both link `href="style.css"`) but
+ * from different directories, and would therefore load different physical
+ * files, produce the same key here unless the caller has already resolved
+ * each href against its page's URL before calling.
+ *
+ * Input order does not affect the result: the arrangement of `<link>` tags
+ * in a document has no bearing on template identity, so hrefs are sorted
+ * (and deduplicated, since a repeated href contributes no extra information
+ * about what the page loads) before hashing. The sorted list is
+ * JSON-serialized before hashing rather than joined with a plain delimiter
+ * (e.g. `"\n"`) so that no character sequence inside one href can be
+ * mistaken for a boundary between two hrefs. Hashing (via SHA-256, reusing
+ * {@link ./hash-content.js | HASH_LENGTH} for the same truncation length the
+ * package's other hashed keys use) keeps the key a fixed size regardless of
+ * how many stylesheets a page loads or how long their URLs are.
+ * @param stylesheetHrefs
+ * @example
+ * ```ts
+ * deriveStylesheetGroupKey(['https://example.com/assets/site.css', 'https://example.com/assets/theme.css']);
+ * deriveStylesheetGroupKey(['https://example.com/assets/theme.css', 'https://example.com/assets/site.css']);
+ * // same result for both calls above — order-independent
+ * ```
+ */
+export declare function deriveStylesheetGroupKey(stylesheetHrefs: readonly string[]): string;

package/dist/derive-stylesheet-group-key.js

@@ -0,0 +1,41 @@
+import { hash } from '@d-zero/shared/hash';
+import { HASH_LENGTH } from './hash-content.js';
+/**
+ * Derives a coarse grouping key from the set of stylesheet URLs a page
+ * loads. This is a *blocking key* in the record-linkage sense (see
+ * {@link ./derive-path-group-key.js | derivePathGroupKey}): pages sharing
+ * the exact same stylesheet set are near-certainly the same template
+ * family, making this a strong but sparse signal — many pages load few or
+ * no stylesheets, so this key is meant to be used alongside, not instead
+ * of, weaker-but-always-present signals like a URL-path-based key.
+ *
+ * `stylesheetHrefs` must already be resolved to a form that is comparable
+ * across the whole corpus (e.g. absolute URLs). This function only compares
+ * the strings it is given: two pages that both reference the same
+ * unresolved relative href text (e.g. both link `href="style.css"`) but
+ * from different directories, and would therefore load different physical
+ * files, produce the same key here unless the caller has already resolved
+ * each href against its page's URL before calling.
+ *
+ * Input order does not affect the result: the arrangement of `<link>` tags
+ * in a document has no bearing on template identity, so hrefs are sorted
+ * (and deduplicated, since a repeated href contributes no extra information
+ * about what the page loads) before hashing. The sorted list is
+ * JSON-serialized before hashing rather than joined with a plain delimiter
+ * (e.g. `"\n"`) so that no character sequence inside one href can be
+ * mistaken for a boundary between two hrefs. Hashing (via SHA-256, reusing
+ * {@link ./hash-content.js | HASH_LENGTH} for the same truncation length the
+ * package's other hashed keys use) keeps the key a fixed size regardless of
+ * how many stylesheets a page loads or how long their URLs are.
+ * @param stylesheetHrefs
+ * @example
+ * ```ts
+ * deriveStylesheetGroupKey(['https://example.com/assets/site.css', 'https://example.com/assets/theme.css']);
+ * deriveStylesheetGroupKey(['https://example.com/assets/theme.css', 'https://example.com/assets/site.css']);
+ * // same result for both calls above — order-independent
+ * ```
+ */
+export function deriveStylesheetGroupKey(stylesheetHrefs) {
+    const sorted = [...new Set(stylesheetHrefs)].toSorted();
+    return hash(JSON.stringify(sorted)).slice(0, HASH_LENGTH);
+}

package/dist/detect-content-depth-cap.d.ts

@@ -0,0 +1,114 @@
+import type { ContentDepthLandmark } from './cap-content-depth.js';
+import type { ResolveStructuralClusterKeysOptions } from './resolve-structural-cluster-keys.js';
+import type { TokenizeOptions } from './types.js';
+/**
+ * @see detectContentDepthCap
+ */
+export type DetectContentDepthCapOptions = TokenizeOptions & ResolveStructuralClusterKeysOptions & {
+    /** Forwarded to {@link ./cap-content-depth.js | capContentDepth}. Defaults to `'main'`. */
+    landmark?: ContentDepthLandmark;
+    /**
+     * Depths to try, in strictly ascending order (`RangeError` otherwise —
+     * the knee-detection loop below assumes each depth is deeper than the
+     * last). Defaults to `[1, 2, 3, 4, 5, 6, 8, 10]` — chosen to cover the
+     * range confirmed on real crawl data (the knee landed at 3 on both
+     * corpora checked) with a few extra steps past it to confirm the
+     * explosion is sustained, without trying every single depth up to an
+     * arbitrary ceiling.
+     */
+    candidateDepths?: readonly number[];
+    /**
+     * The minimum cluster-count ratio between two consecutive candidate
+     * depths (`clusterCount[i] / clusterCount[i-1]`) required to call that
+     * jump "the knee." Must be a finite number greater than 1 (`RangeError`
+     * otherwise — a ratio at or below 1 means "no growth," which can never
+     * meaningfully gate a knee). Defaults to `1.5` (a 50% jump). Below this,
+     * growth is treated as gradual/expected rather than evidence of a
+     * freeform-content boundary, and no cap is recommended.
+     */
+    minKneeRatio?: number;
+};
+/**
+ * Validates the `candidateDepths`/`minKneeRatio` parts of
+ * {@link DetectContentDepthCapOptions} without running the sweep itself.
+ * {@link detectContentDepthCap} always calls this on its own, so a direct
+ * caller never needs to; it's exported only so a caller that invokes
+ * `detectContentDepthCap` conditionally (e.g. once per block, skipped
+ * entirely for blocks too small to matter — see
+ * {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}'s
+ * `autoCapMainDepth`) can still fail fast on a bad option even when that
+ * per-call skip means the sweep itself might never run for a given input
+ * (e.g. an empty page list has no blocks at all).
+ * @param options
+ * @example
+ * ```ts
+ * // Fails fast on a bad option even though nothing here would otherwise
+ * // call detectContentDepthCap yet (e.g. blocks haven't been computed).
+ * validateDetectContentDepthCapOptions({ minKneeRatio: 1 }); // throws RangeError
+ * ```
+ */
+export declare function validateDetectContentDepthCapOptions(options?: DetectContentDepthCapOptions): void;
+/**
+ * Finds the depth just before {@link ./cap-content-depth.js | capContentDepth}
+ * ("`maxDepth`") would start throwing away real structural signal, by trying
+ * each of `options.candidateDepths` in turn and looking for the first big
+ * jump in resulting cluster count.
+ *
+ * Confirmed on two unrelated real crawls (302 and ~4,100 pages, sharing no
+ * code or template lineage): the number of distinct
+ * {@link ./resolve-structural-cluster-keys.js | resolveStructuralClusterKeys}
+ * clusters stays roughly flat (or grows gently) as `maxDepth` increases,
+ * then jumps sharply (14x and 9x respectively) at one specific depth — the
+ * point past which comparisons start seeing freeform, page-to-page-varying
+ * editorial content instead of shared template structure. That depth landed
+ * at 3 on both corpora, but this function doesn't hardcode that: it
+ * re-derives it per corpus, so a differently-nested template doesn't get
+ * the wrong number silently baked in.
+ *
+ * Returns the *last* candidate depth before the biggest qualifying jump
+ * (`options.minKneeRatio` or steeper) — i.e. the depth to actually cap
+ * at, already chosen so the jump lands past it. If no jump in
+ * `candidateDepths` clears `minKneeRatio` (growth looks gradual, or
+ * `htmlList` is too small/uniform to tell), the *largest* candidate depth is
+ * returned — deliberately not capping rather than guessing.
+ *
+ * Forwards `options`' `TokenizeOptions`/`ResolveStructuralClusterKeysOptions`
+ * fields (e.g. `filterNoiseClasses`, `similarityThreshold`) to every sweep's
+ * `tokenize`/`resolveStructuralClusterKeys` call, so the knee is detected
+ * against the same tokenization and clustering configuration
+ * {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys} actually
+ * clusters with afterward — passing a different configuration here than
+ * downstream would pick a cap tuned for a comparison that's never actually
+ * performed.
+ *
+ * This calls {@link ./resolve-structural-cluster-keys.js |
+ * resolveStructuralClusterKeys} once per candidate depth (each an O(n²)
+ * comparison over `htmlList`), so cost scales with both `htmlList.length`
+ * and `candidateDepths.length`. Measured standalone on a real 4,085-page
+ * single-block corpus: ~4s per candidate depth, ~30s total for the default 8
+ * depths. {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}'s
+ * `autoCapMainDepth` option calls this once *per block* rather than once
+ * globally (different blocks can have different knees — see that option's
+ * own JSDoc for why this matters, not just for cost) — measured end to end on
+ * a real 8,936-page whole-site corpus (32 blocks, largest ~4,082 pages):
+ * ~119s total with `autoCapMainDepth` versus ~18s without it, cutting that
+ * corpus's final cluster count from 1,972 to 134. Partitioning the O(n²) cost
+ * across blocks rather than paying it once over the whole corpus is itself
+ * why this got *cheaper* than an earlier global-sweep design that measured
+ * ~5m50s for the same corpus (the sum of each block's `memberCount²` is far
+ * below `htmlList.length²` once a corpus splits into more than a couple of
+ * blocks). Sampling a single block's `htmlList` down before calling this
+ * (accepting a less precise knee estimate) is the natural next step if one
+ * particular block's cost becomes a problem, but isn't implemented here
+ * without real evidence it's needed.
+ * @param htmlList
+ * @param options
+ * @example
+ * ```ts
+ * const maxDepth = detectContentDepthCap(pages.map((p) => p.html));
+ * const tokenSets = pages.map(
+ * 	(p) => new Set(tokenize(capContentDepth(p.html, { landmark: 'main', maxDepth }).remainderHtml).tokens),
+ * );
+ * ```
+ */
+export declare function detectContentDepthCap(htmlList: readonly string[], options?: DetectContentDepthCapOptions): number;

package/dist/detect-content-depth-cap.js

@@ -0,0 +1,137 @@
+import { capContentDepth } from './cap-content-depth.js';
+import { resolveStructuralClusterKeys } from './resolve-structural-cluster-keys.js';
+import { tokenize } from './tokenize.js';
+/**
+ * Validates the `candidateDepths`/`minKneeRatio` parts of
+ * {@link DetectContentDepthCapOptions} without running the sweep itself.
+ * {@link detectContentDepthCap} always calls this on its own, so a direct
+ * caller never needs to; it's exported only so a caller that invokes
+ * `detectContentDepthCap` conditionally (e.g. once per block, skipped
+ * entirely for blocks too small to matter — see
+ * {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}'s
+ * `autoCapMainDepth`) can still fail fast on a bad option even when that
+ * per-call skip means the sweep itself might never run for a given input
+ * (e.g. an empty page list has no blocks at all).
+ * @param options
+ * @example
+ * ```ts
+ * // Fails fast on a bad option even though nothing here would otherwise
+ * // call detectContentDepthCap yet (e.g. blocks haven't been computed).
+ * validateDetectContentDepthCapOptions({ minKneeRatio: 1 }); // throws RangeError
+ * ```
+ */
+export function validateDetectContentDepthCapOptions(options) {
+    const candidateDepths = options?.candidateDepths ?? [1, 2, 3, 4, 5, 6, 8, 10];
+    const minKneeRatio = options?.minKneeRatio ?? 1.5;
+    if (candidateDepths.length === 0) {
+        throw new RangeError('detectContentDepthCap: candidateDepths must not be empty');
+    }
+    let previousDepth = -Infinity;
+    for (const depth of candidateDepths) {
+        if (depth <= previousDepth) {
+            throw new RangeError(`detectContentDepthCap: candidateDepths must be in strictly ascending order, got ${JSON.stringify(candidateDepths)}`);
+        }
+        previousDepth = depth;
+    }
+    if (!(Number.isFinite(minKneeRatio) && minKneeRatio > 1)) {
+        throw new RangeError(`detectContentDepthCap: minKneeRatio must be a finite number greater than 1, got ${minKneeRatio}`);
+    }
+}
+/**
+ * Finds the depth just before {@link ./cap-content-depth.js | capContentDepth}
+ * ("`maxDepth`") would start throwing away real structural signal, by trying
+ * each of `options.candidateDepths` in turn and looking for the first big
+ * jump in resulting cluster count.
+ *
+ * Confirmed on two unrelated real crawls (302 and ~4,100 pages, sharing no
+ * code or template lineage): the number of distinct
+ * {@link ./resolve-structural-cluster-keys.js | resolveStructuralClusterKeys}
+ * clusters stays roughly flat (or grows gently) as `maxDepth` increases,
+ * then jumps sharply (14x and 9x respectively) at one specific depth — the
+ * point past which comparisons start seeing freeform, page-to-page-varying
+ * editorial content instead of shared template structure. That depth landed
+ * at 3 on both corpora, but this function doesn't hardcode that: it
+ * re-derives it per corpus, so a differently-nested template doesn't get
+ * the wrong number silently baked in.
+ *
+ * Returns the *last* candidate depth before the biggest qualifying jump
+ * (`options.minKneeRatio` or steeper) — i.e. the depth to actually cap
+ * at, already chosen so the jump lands past it. If no jump in
+ * `candidateDepths` clears `minKneeRatio` (growth looks gradual, or
+ * `htmlList` is too small/uniform to tell), the *largest* candidate depth is
+ * returned — deliberately not capping rather than guessing.
+ *
+ * Forwards `options`' `TokenizeOptions`/`ResolveStructuralClusterKeysOptions`
+ * fields (e.g. `filterNoiseClasses`, `similarityThreshold`) to every sweep's
+ * `tokenize`/`resolveStructuralClusterKeys` call, so the knee is detected
+ * against the same tokenization and clustering configuration
+ * {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys} actually
+ * clusters with afterward — passing a different configuration here than
+ * downstream would pick a cap tuned for a comparison that's never actually
+ * performed.
+ *
+ * This calls {@link ./resolve-structural-cluster-keys.js |
+ * resolveStructuralClusterKeys} once per candidate depth (each an O(n²)
+ * comparison over `htmlList`), so cost scales with both `htmlList.length`
+ * and `candidateDepths.length`. Measured standalone on a real 4,085-page
+ * single-block corpus: ~4s per candidate depth, ~30s total for the default 8
+ * depths. {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}'s
+ * `autoCapMainDepth` option calls this once *per block* rather than once
+ * globally (different blocks can have different knees — see that option's
+ * own JSDoc for why this matters, not just for cost) — measured end to end on
+ * a real 8,936-page whole-site corpus (32 blocks, largest ~4,082 pages):
+ * ~119s total with `autoCapMainDepth` versus ~18s without it, cutting that
+ * corpus's final cluster count from 1,972 to 134. Partitioning the O(n²) cost
+ * across blocks rather than paying it once over the whole corpus is itself
+ * why this got *cheaper* than an earlier global-sweep design that measured
+ * ~5m50s for the same corpus (the sum of each block's `memberCount²` is far
+ * below `htmlList.length²` once a corpus splits into more than a couple of
+ * blocks). Sampling a single block's `htmlList` down before calling this
+ * (accepting a less precise knee estimate) is the natural next step if one
+ * particular block's cost becomes a problem, but isn't implemented here
+ * without real evidence it's needed.
+ * @param htmlList
+ * @param options
+ * @example
+ * ```ts
+ * const maxDepth = detectContentDepthCap(pages.map((p) => p.html));
+ * const tokenSets = pages.map(
+ * 	(p) => new Set(tokenize(capContentDepth(p.html, { landmark: 'main', maxDepth }).remainderHtml).tokens),
+ * );
+ * ```
+ */
+export function detectContentDepthCap(htmlList, options) {
+    validateDetectContentDepthCapOptions(options);
+    const landmark = options?.landmark ?? 'main';
+    const candidateDepths = options?.candidateDepths ?? [1, 2, 3, 4, 5, 6, 8, 10];
+    const minKneeRatio = options?.minKneeRatio ?? 1.5;
+    const clusterCounts = candidateDepths.map((maxDepth) => {
+        const tokenSets = htmlList.map((html) => {
+            const capped = capContentDepth(html, { landmark, maxDepth }).remainderHtml;
+            return new Set(tokenize(capped, options).tokens);
+        });
+        return new Set(resolveStructuralClusterKeys(tokenSets, options)).size;
+    });
+    // bestRatio starts below any possible ratio (rather than at minKneeRatio
+    // itself) so a jump that exactly *meets* minKneeRatio still qualifies —
+    // this option's own JSDoc only calls growth "gradual" (i.e. rejected)
+    // when it's *below* the threshold, not at or above it.
+    let bestRatio = -Infinity;
+    let kneeIndex = -1;
+    for (let i = 1; i < clusterCounts.length; i++) {
+        const previous = clusterCounts[i - 1];
+        const current = clusterCounts[i];
+        if (previous === undefined || current === undefined || previous === 0) {
+            continue;
+        }
+        const ratio = current / previous;
+        if (ratio >= minKneeRatio && ratio > bestRatio) {
+            bestRatio = ratio;
+            kneeIndex = i;
+        }
+    }
+    if (kneeIndex === -1) {
+        return candidateDepths.at(-1) ?? 0;
+    }
+    return candidateDepths[kneeIndex - 1] ?? 0;
+}

package/dist/escape-reg-exp.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Escapes regex metacharacters in `text` so it can be interpolated into a
+ * `RegExp` literally. Needed because a tag name reaching
+ * {@link ./is-genuine-close.js | isGenuineClose} is not guaranteed to be a
+ * plain HTML tag name: htmlparser2 accepts characters like `(`/`[` inside a
+ * tag name (`<div(foo role="banner">` parses with tag name `"div(foo"`),
+ * which would otherwise either throw (an unbalanced `(` is an invalid regex)
+ * or silently change what the regex matches.
+ * @param text
+ */
+export declare function escapeRegExp(text: string): string;

package/dist/escape-reg-exp.js

@@ -0,0 +1,13 @@
+/**
+ * Escapes regex metacharacters in `text` so it can be interpolated into a
+ * `RegExp` literally. Needed because a tag name reaching
+ * {@link ./is-genuine-close.js | isGenuineClose} is not guaranteed to be a
+ * plain HTML tag name: htmlparser2 accepts characters like `(`/`[` inside a
+ * tag name (`<div(foo role="banner">` parses with tag name `"div(foo"`),
+ * which would otherwise either throw (an unbalanced `(` is an invalid regex)
+ * or silently change what the regex matches.
+ * @param text
+ */
+export function escapeRegExp(text) {
+    return text.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}

package/dist/excise.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Excises `spans` (merged via {@link ./merge-spans.js | mergeSpans}) from
+ * `html`, returning what's left. No placeholder is left in a span's place: a
+ * placeholder string would itself become a token once the remainder is
+ * tokenized, reintroducing exactly the kind of synthetic signal callers of
+ * this function exist to remove.
+ * @param html
+ * @param spans
+ */
+export declare function excise(html: string, spans: readonly {
+    start: number;
+    end: number;
+}[]): string;

package/dist/excise.js

@@ -0,0 +1,24 @@
+import { mergeSpans } from './merge-spans.js';
+/**
+ * Excises `spans` (merged via {@link ./merge-spans.js | mergeSpans}) from
+ * `html`, returning what's left. No placeholder is left in a span's place: a
+ * placeholder string would itself become a token once the remainder is
+ * tokenized, reintroducing exactly the kind of synthetic signal callers of
+ * this function exist to remove.
+ * @param html
+ * @param spans
+ */
+export function excise(html, spans) {
+    if (spans.length === 0) {
+        return html;
+    }
+    const merged = mergeSpans(spans);
+    let remainder = '';
+    let cursor = 0;
+    for (const span of merged) {
+        remainder += html.slice(cursor, span.start);
+        cursor = span.end;
+    }
+    remainder += html.slice(cursor);
+    return remainder;
+}

package/dist/extract-landmarks.d.ts

@@ -0,0 +1,82 @@
+/**
+ * The four structural regions this module knows how to carve out of a page.
+ * Chosen to match both the HTML5 sectioning-element vocabulary and the
+ * corresponding ARIA landmark roles, since real sites use either or both
+ * (confirmed on two real crawl archives, ~9,200 pages combined: `<header>`/
+ * `<footer>`/`<nav>` present on 99%+ of pages; ARIA roles present on ~53% of
+ * one of the two sites, layered on top of the tags rather than replacing
+ * them).
+ */
+export type LandmarkType = 'header' | 'footer' | 'nav' | 'aside';
+/**
+ * Result of {@link ./extract-landmarks.js | extractLandmarks}. Each landmark
+ * field holds the raw HTML of the single chosen instance of that region (see
+ * `extractLandmarks`'s JSDoc for the "shallowest wins" selection rule);
+ * absent if the page has none — or if the only candidate(s) found were
+ * malformed markup `extractLandmarks` declined to trust (see its JSDoc's
+ * note on discarded candidates). `remainderHtml` is the original HTML
+ * with every chosen region's markup excised, meant to be fed straight into
+ * {@link ./tokenize.js | tokenize} as the page's content-only signal.
+ */
+export type ExtractLandmarksResult = {
+    header?: string;
+    footer?: string;
+    nav?: string;
+    aside?: string;
+    remainderHtml: string;
+};
+/**
+ * Finds, for each of the four landmark types, the single best-matching
+ * region in `html` (by tag name or ARIA role — see `matchLandmarkTypes`),
+ * and returns both that region's own HTML and the rest of the page with all
+ * chosen regions removed.
+ *
+ * When a type has more than one candidate (confirmed on real crawl data: one
+ * page had 11 `<header>` elements, most pages have 2-3 `<nav>` elements —
+ * typically a site-wide nav plus in-content ones like a "related articles"
+ * block), the shallowest one wins (fewest ancestors since `<body>`; ties
+ * broken by document order). The rationale: the site-wide chrome instance is
+ * structurally the outermost one — anything nested deeper inside `<main>`/
+ * `<article>` content is, definitionally, part of the page's own content
+ * rather than shared site chrome, even if it happens to reuse the same tag
+ * or role.
+ *
+ * Only the first `<body>` is in scope, matching `tokenize()`'s own contract
+ * (`<head>` and anything outside body is ignored; a duplicated top-level
+ * `<body>` from broken SSR/templating is ignored, same as
+ * `run-tokenizer.ts`).
+ *
+ * `remainderHtml` is built by excising the chosen regions' raw markup
+ * outright — no placeholder is left in their place, since a placeholder
+ * string would itself become a token once `remainderHtml` is tokenized,
+ * reintroducing exactly the kind of synthetic chrome signal this function
+ * exists to remove. One known, accepted side effect of this: if a chosen
+ * landmark and the remaining content share a class-less/role-less `<div>`/
+ * `<span>` wrapper as siblings, removing the landmark can change that
+ * wrapper's child count and flip it from "not fold-eligible" to
+ * "fold-eligible" once `remainderHtml` is tokenized (see
+ * `resolveClosedFrame`'s fold rule) — the wrapper's own segment then
+ * disappears from the surviving paths, shortening them by one level. This
+ * is inherent to "delete the matched span, use whatever's left" and is not
+ * treated as a bug.
+ *
+ * A candidate whose closing tag can't be confirmed as genuine (an unclosed
+ * or self-closed-with-`/>` landmark tag — see `isGenuineClose`) is discarded
+ * rather than trusted: safety against corrupting `remainderHtml` outweighs
+ * completeness of landmark detection for malformed markup. That type then
+ * falls back to another well-formed candidate of the same type if one
+ * exists (regardless of its depth relative to the discarded one), or is
+ * left absent if none do — instead of the page's real content being
+ * silently deleted.
+ * @param html
+ * @example
+ * ```ts
+ * extractLandmarks('<body><header>H</header><main>M</main><footer>F</footer></body>');
+ * // {
+ * //   header: '<header>H</header>',
+ * //   footer: '<footer>F</footer>',
+ * //   remainderHtml: '<body><main>M</main></body>',
+ * // }
+ * ```
+ */
+export declare function extractLandmarks(html: string): ExtractLandmarksResult;

package/dist/extract-landmarks.js

@@ -0,0 +1,104 @@
+import { excise } from './excise.js';
+import { findShallowestElements } from './find-shallowest-elements.js';
+const TAG_TO_TYPE = {
+    header: 'header',
+    footer: 'footer',
+    nav: 'nav',
+    aside: 'aside',
+};
+const ROLE_TO_TYPE = {
+    banner: 'header',
+    contentinfo: 'footer',
+    navigation: 'nav',
+    complementary: 'aside',
+};
+/**
+ * Determines which landmark type(s) an element matches by tag name or
+ * `role`. Deliberately returns every match rather than the first: a
+ * `<header role="navigation">` is simultaneously a `header` candidate (by
+ * tag) and a `nav` candidate (by role) — both are independently correct
+ * answers to "where is this page's header" and "where is this page's nav",
+ * so both must be recorded from the same element.
+ * @param tagName
+ * @param role
+ */
+function matchLandmarkTypes(tagName, role) {
+    const types = [];
+    const byTag = TAG_TO_TYPE[tagName];
+    if (byTag) {
+        types.push(byTag);
+    }
+    // role is matched as a single exact literal, same limitation as
+    // `create-frame.ts`'s own `attribs.role` handling: no whitespace-
+    // separated multi-role splitting, no case normalization.
+    const byRole = role ? ROLE_TO_TYPE[role] : undefined;
+    if (byRole && !types.includes(byRole)) {
+        types.push(byRole);
+    }
+    return types;
+}
+/**
+ * Finds, for each of the four landmark types, the single best-matching
+ * region in `html` (by tag name or ARIA role — see `matchLandmarkTypes`),
+ * and returns both that region's own HTML and the rest of the page with all
+ * chosen regions removed.
+ *
+ * When a type has more than one candidate (confirmed on real crawl data: one
+ * page had 11 `<header>` elements, most pages have 2-3 `<nav>` elements —
+ * typically a site-wide nav plus in-content ones like a "related articles"
+ * block), the shallowest one wins (fewest ancestors since `<body>`; ties
+ * broken by document order). The rationale: the site-wide chrome instance is
+ * structurally the outermost one — anything nested deeper inside `<main>`/
+ * `<article>` content is, definitionally, part of the page's own content
+ * rather than shared site chrome, even if it happens to reuse the same tag
+ * or role.
+ *
+ * Only the first `<body>` is in scope, matching `tokenize()`'s own contract
+ * (`<head>` and anything outside body is ignored; a duplicated top-level
+ * `<body>` from broken SSR/templating is ignored, same as
+ * `run-tokenizer.ts`).
+ *
+ * `remainderHtml` is built by excising the chosen regions' raw markup
+ * outright — no placeholder is left in their place, since a placeholder
+ * string would itself become a token once `remainderHtml` is tokenized,
+ * reintroducing exactly the kind of synthetic chrome signal this function
+ * exists to remove. One known, accepted side effect of this: if a chosen
+ * landmark and the remaining content share a class-less/role-less `<div>`/
+ * `<span>` wrapper as siblings, removing the landmark can change that
+ * wrapper's child count and flip it from "not fold-eligible" to
+ * "fold-eligible" once `remainderHtml` is tokenized (see
+ * `resolveClosedFrame`'s fold rule) — the wrapper's own segment then
+ * disappears from the surviving paths, shortening them by one level. This
+ * is inherent to "delete the matched span, use whatever's left" and is not
+ * treated as a bug.
+ *
+ * A candidate whose closing tag can't be confirmed as genuine (an unclosed
+ * or self-closed-with-`/>` landmark tag — see `isGenuineClose`) is discarded
+ * rather than trusted: safety against corrupting `remainderHtml` outweighs
+ * completeness of landmark detection for malformed markup. That type then
+ * falls back to another well-formed candidate of the same type if one
+ * exists (regardless of its depth relative to the discarded one), or is
+ * left absent if none do — instead of the page's real content being
+ * silently deleted.
+ * @param html
+ * @example
+ * ```ts
+ * extractLandmarks('<body><header>H</header><main>M</main><footer>F</footer></body>');
+ * // {
+ * //   header: '<header>H</header>',
+ * //   footer: '<footer>F</footer>',
+ * //   remainderHtml: '<body><main>M</main></body>',
+ * // }
+ * ```
+ */
+export function extractLandmarks(html) {
+    const matches = findShallowestElements(html, matchLandmarkTypes);
+    const result = { remainderHtml: html };
+    const winnerSpans = [];
+    for (const match of matches) {
+        result[match.type] = html.slice(match.startOffset, match.endOffset);
+        winnerSpans.push({ start: match.startOffset, end: match.endOffset });
+    }
+    result.remainderHtml = excise(html, winnerSpans);
+    return result;
+}

package/dist/filter-first-party-stylesheet-hrefs.d.ts

@@ -0,0 +1,73 @@
+/**
+ * Narrows every page's `stylesheetHrefs` down to just the hrefs whose host
+ * matches the single most common host across the whole batch (the site's own
+ * first-party domain), dropping every other host.
+ *
+ * Confirmed on real crawl data (302 pages): a handful of articles embedding
+ * a YouTube video pulled in `youtube.com`'s own player stylesheet plus a
+ * per-embed tracking URL that resembles a stylesheet reference; other
+ * articles embedding a particular widget pulled in two extra
+ * `fonts.googleapis.com` URLs beyond the site's usual one. Both are
+ * incidental to whatever third-party content a page happens to embed, not
+ * evidence of which template the page uses — but
+ * {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys}'s
+ * document-frequency filtering has no way to tell "rare because it's a
+ * genuinely distinctive template" apart from "rare because almost no other
+ * page happens to embed this same third party," so it let these through as
+ * if they were real template signals, splitting a handful of otherwise-
+ * identical pages (confirmed via direct comparison: 100% token overlap with
+ * their section's main cluster) away from where they belonged. Filtering to
+ * first-party hrefs before blocking removes that false signal at the
+ * source, rather than trying to recognize its effects downstream.
+ *
+ * Determining "first-party" from the batch's own href distribution (rather
+ * than, say, comparing each href's host against each page's own URL) means
+ * this needs no extra per-page input beyond what
+ * {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys} already
+ * takes — but it inherits that same function's "roughly homogeneous batch"
+ * precondition (see `computeDocumentFrequency`'s own JSDoc): a batch that
+ * mixes pages from more than one site in one call has no single genuine
+ * first-party host to find, and this function has no way to detect that
+ * it's been handed one — it will still confidently pick *a* dominant host
+ * (whichever site contributes more stylesheet-bearing pages) and silently
+ * strip every other site's real first-party hrefs. Splitting a
+ * multi-site/section batch into homogeneous groups before calling this is
+ * the caller's responsibility, same as it already is for
+ * `resolveBlockingGroupKeys`.
+ *
+ * The dominant host is picked by how many *pages* reference it at least
+ * once, not by how many stylesheet `<link>` tags reference it — a page
+ * loading one first-party stylesheet plus two third-party font requests
+ * must not let the font host outvote the actual first-party one just for
+ * appearing on more `<link>` tags. Compared by `host` (hostname + port),
+ * not the full origin (which also includes the scheme): the same first-party
+ * site served over both `http:` and `https:` (mid-migration, or a stray
+ * unresolved protocol-relative URL) is still one site, not two competing
+ * "hosts" splitting its own vote.
+ *
+ * The trade-off: a site that legitimately serves its own stylesheets from
+ * more than one first-party host (e.g. a CDN subdomain alongside the main
+ * domain) will have its non-dominant host's hrefs dropped too, same as any
+ * genuinely-third-party host — not yet observed on real data, but a known
+ * limitation of picking a single dominant host rather than a set.
+ *
+ * A batch where no page has any stylesheet href at all (or none of the
+ * hrefs are parseable absolute URLs) has no host to detect; every page's
+ * `stylesheetHrefs` is returned unchanged in that case, matching this
+ * function's job of narrowing signal, not fabricating it.
+ * @param pages
+ * @example
+ * ```ts
+ * filterFirstPartyStylesheetHrefs([
+ * 	{ stylesheetHrefs: ['https://example.com/a.css', 'https://example.com/b.css'] },
+ * 	{ stylesheetHrefs: ['https://example.com/a.css', 'https://fonts.googleapis.com/css?family=x'] },
+ * ]);
+ * // [
+ * // 	{ stylesheetHrefs: ['https://example.com/a.css', 'https://example.com/b.css'] },
+ * // 	{ stylesheetHrefs: ['https://example.com/a.css'] }, // fonts.googleapis.com dropped
+ * // ]
+ * ```
+ */
+export declare function filterFirstPartyStylesheetHrefs<T extends {
+    stylesheetHrefs: readonly string[];
+}>(pages: readonly T[]): T[];