@d-zero/page-cluster 0.2.0

package/dist/filter-first-party-stylesheet-hrefs.js

@@ -0,0 +1,118 @@
+/**
+ * Reads the host (not the full origin — see `filterFirstPartyStylesheetHrefs`'s
+ * JSDoc for why) out of `href`, or `undefined` if it isn't a parseable
+ * absolute URL. `stylesheetHrefs` is already expected to be absolute (see
+ * {@link ./derive-stylesheet-group-key.js | deriveStylesheetGroupKey}'s own
+ * JSDoc) — this is defensive, not a normalization step.
+ * @param href
+ */
+function tryGetHost(href) {
+    try {
+        return new URL(href).host;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Narrows every page's `stylesheetHrefs` down to just the hrefs whose host
+ * matches the single most common host across the whole batch (the site's own
+ * first-party domain), dropping every other host.
+ *
+ * Confirmed on real crawl data (302 pages): a handful of articles embedding
+ * a YouTube video pulled in `youtube.com`'s own player stylesheet plus a
+ * per-embed tracking URL that resembles a stylesheet reference; other
+ * articles embedding a particular widget pulled in two extra
+ * `fonts.googleapis.com` URLs beyond the site's usual one. Both are
+ * incidental to whatever third-party content a page happens to embed, not
+ * evidence of which template the page uses — but
+ * {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys}'s
+ * document-frequency filtering has no way to tell "rare because it's a
+ * genuinely distinctive template" apart from "rare because almost no other
+ * page happens to embed this same third party," so it let these through as
+ * if they were real template signals, splitting a handful of otherwise-
+ * identical pages (confirmed via direct comparison: 100% token overlap with
+ * their section's main cluster) away from where they belonged. Filtering to
+ * first-party hrefs before blocking removes that false signal at the
+ * source, rather than trying to recognize its effects downstream.
+ *
+ * Determining "first-party" from the batch's own href distribution (rather
+ * than, say, comparing each href's host against each page's own URL) means
+ * this needs no extra per-page input beyond what
+ * {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys} already
+ * takes — but it inherits that same function's "roughly homogeneous batch"
+ * precondition (see `computeDocumentFrequency`'s own JSDoc): a batch that
+ * mixes pages from more than one site in one call has no single genuine
+ * first-party host to find, and this function has no way to detect that
+ * it's been handed one — it will still confidently pick *a* dominant host
+ * (whichever site contributes more stylesheet-bearing pages) and silently
+ * strip every other site's real first-party hrefs. Splitting a
+ * multi-site/section batch into homogeneous groups before calling this is
+ * the caller's responsibility, same as it already is for
+ * `resolveBlockingGroupKeys`.
+ *
+ * The dominant host is picked by how many *pages* reference it at least
+ * once, not by how many stylesheet `<link>` tags reference it — a page
+ * loading one first-party stylesheet plus two third-party font requests
+ * must not let the font host outvote the actual first-party one just for
+ * appearing on more `<link>` tags. Compared by `host` (hostname + port),
+ * not the full origin (which also includes the scheme): the same first-party
+ * site served over both `http:` and `https:` (mid-migration, or a stray
+ * unresolved protocol-relative URL) is still one site, not two competing
+ * "hosts" splitting its own vote.
+ *
+ * The trade-off: a site that legitimately serves its own stylesheets from
+ * more than one first-party host (e.g. a CDN subdomain alongside the main
+ * domain) will have its non-dominant host's hrefs dropped too, same as any
+ * genuinely-third-party host — not yet observed on real data, but a known
+ * limitation of picking a single dominant host rather than a set.
+ *
+ * A batch where no page has any stylesheet href at all (or none of the
+ * hrefs are parseable absolute URLs) has no host to detect; every page's
+ * `stylesheetHrefs` is returned unchanged in that case, matching this
+ * function's job of narrowing signal, not fabricating it.
+ * @param pages
+ * @example
+ * ```ts
+ * filterFirstPartyStylesheetHrefs([
+ * 	{ stylesheetHrefs: ['https://example.com/a.css', 'https://example.com/b.css'] },
+ * 	{ stylesheetHrefs: ['https://example.com/a.css', 'https://fonts.googleapis.com/css?family=x'] },
+ * ]);
+ * // [
+ * // 	{ stylesheetHrefs: ['https://example.com/a.css', 'https://example.com/b.css'] },
+ * // 	{ stylesheetHrefs: ['https://example.com/a.css'] }, // fonts.googleapis.com dropped
+ * // ]
+ * ```
+ */
+export function filterFirstPartyStylesheetHrefs(pages) {
+    const pageHrefHosts = pages.map((page) => ({
+        page,
+        hrefHosts: page.stylesheetHrefs.map((href) => ({ href, host: tryGetHost(href) })),
+    }));
+    const hostPageCounts = new Map();
+    for (const { hrefHosts } of pageHrefHosts) {
+        const distinctHosts = new Set(hrefHosts
+            .map(({ host }) => host)
+            .filter((host) => host !== undefined));
+        for (const host of distinctHosts) {
+            hostPageCounts.set(host, (hostPageCounts.get(host) ?? 0) + 1);
+        }
+    }
+    let dominantHost;
+    let dominantCount = 0;
+    for (const [host, count] of hostPageCounts) {
+        if (count > dominantCount) {
+            dominantHost = host;
+            dominantCount = count;
+        }
+    }
+    if (dominantHost === undefined) {
+        return [...pages];
+    }
+    return pageHrefHosts.map(({ page, hrefHosts }) => ({
+        ...page,
+        stylesheetHrefs: hrefHosts
+            .filter(({ host }) => host === dominantHost)
+            .map(({ href }) => href),
+    }));
+}

package/dist/find-shallowest-elements.d.ts

@@ -0,0 +1,39 @@
+/**
+ * One winning element for type `T`: the shallowest (fewest ancestors since
+ * `<body>`) genuinely-closed match, ties broken by document order. Both the
+ * whole-element span (`startOffset`/`endOffset`) and the inner-content span
+ * (`contentStart`/`contentEnd`, excluding the element's own opening/closing
+ * tags) are always computed — {@link ./extract-landmarks.js | extractLandmarks}
+ * only needs the former, {@link ./cap-content-depth.js | capContentDepth}
+ * only needs the latter, and computing both is cheap enough (two
+ * `indexOf`/`lastIndexOf` calls) that carrying the unused half costs nothing
+ * a caller need worry about.
+ */
+export type ShallowestElementMatch<T extends string> = {
+    type: T;
+    startOffset: number;
+    endOffset: number;
+    contentStart: number;
+    contentEnd: number;
+};
+/**
+ * Shared walk behind {@link ./extract-landmarks.js | extractLandmarks} (which
+ * matches four landmark types per element in one pass) and
+ * {@link ./cap-content-depth.js | capContentDepth} (which matches a single
+ * landmark). Both need the identical "shallowest genuinely-closed match
+ * wins" search — same `<body>`-scoping, same opaque-tag skip, same malformed-
+ * markup discard via {@link ./is-genuine-close.js | isGenuineClose} — so a
+ * fix to one (e.g. the body-scoping edge case already fixed once in
+ * `extractLandmarks`) can't silently fail to apply to the other.
+ *
+ * Only the first `<body>` is in scope, and nothing inside an opaque tag
+ * (`script`/`style`/`noscript`/`svg`) is searched — see
+ * `extractLandmarks`/`capContentDepth`'s own JSDoc for why.
+ * @param html
+ * @param matchTypes Given an element's tag name and `role` attribute (already
+ * normalized: an empty/absent `role` arrives as `undefined`), returns every
+ * type `T` that element matches. Returning more than one lets a single
+ * element (e.g. `<header role="navigation">`) win more than one type at
+ * once.
+ */
+export declare function findShallowestElements<T extends string>(html: string, matchTypes: (tagName: string, role: string | undefined) => readonly T[]): ShallowestElementMatch<T>[];

package/dist/find-shallowest-elements.js

@@ -0,0 +1,121 @@
+import { Parser } from 'htmlparser2';
+import { isGenuineClose } from './is-genuine-close.js';
+import { isOpaqueTagName } from './opaque-tags.js';
+/**
+ * Shared walk behind {@link ./extract-landmarks.js | extractLandmarks} (which
+ * matches four landmark types per element in one pass) and
+ * {@link ./cap-content-depth.js | capContentDepth} (which matches a single
+ * landmark). Both need the identical "shallowest genuinely-closed match
+ * wins" search — same `<body>`-scoping, same opaque-tag skip, same malformed-
+ * markup discard via {@link ./is-genuine-close.js | isGenuineClose} — so a
+ * fix to one (e.g. the body-scoping edge case already fixed once in
+ * `extractLandmarks`) can't silently fail to apply to the other.
+ *
+ * Only the first `<body>` is in scope, and nothing inside an opaque tag
+ * (`script`/`style`/`noscript`/`svg`) is searched — see
+ * `extractLandmarks`/`capContentDepth`'s own JSDoc for why.
+ * @param html
+ * @param matchTypes Given an element's tag name and `role` attribute (already
+ * normalized: an empty/absent `role` arrives as `undefined`), returns every
+ * type `T` that element matches. Returning more than one lets a single
+ * element (e.g. `<header role="navigation">`) win more than one type at
+ * once.
+ */
+export function findShallowestElements(html, matchTypes) {
+    const stack = [];
+    const candidates = [];
+    let opaque = null;
+    let bodyDone = false;
+    let ignoredBodyOpens = 0;
+    const parser = new Parser({
+        onopentag(name, attribs) {
+            if (opaque) {
+                if (name === opaque.tagName)
+                    opaque.depth++;
+                return;
+            }
+            if (stack.length === 0) {
+                if (name === 'body' && !bodyDone) {
+                    stack.push({
+                        tagName: name,
+                        matchedTypes: matchTypes(name, attribs.role || undefined),
+                        startOffset: parser.startIndex,
+                    });
+                }
+                return;
+            }
+            if (name === 'body') {
+                ignoredBodyOpens++;
+                return;
+            }
+            if (isOpaqueTagName(name)) {
+                opaque = { tagName: name, depth: 1 };
+                return;
+            }
+            stack.push({
+                tagName: name,
+                matchedTypes: matchTypes(name, attribs.role || undefined),
+                startOffset: parser.startIndex,
+            });
+        },
+        onclosetag(name) {
+            if (opaque) {
+                if (name === opaque.tagName) {
+                    opaque.depth--;
+                    if (opaque.depth === 0)
+                        opaque = null;
+                }
+                return;
+            }
+            if (name === 'body' && ignoredBodyOpens > 0) {
+                ignoredBodyOpens--;
+                return;
+            }
+            if (stack.length === 0)
+                return;
+            const frame = stack.pop();
+            if (!frame)
+                return;
+            const depth = stack.length;
+            const endOffset = parser.endIndex + 1;
+            if (frame.matchedTypes.length > 0 &&
+                isGenuineClose(html, endOffset, frame.tagName)) {
+                const contentStart = html.indexOf('>', frame.startOffset) + 1;
+                const contentEnd = html.lastIndexOf('<', endOffset - 1);
+                for (const type of frame.matchedTypes) {
+                    candidates.push({
+                        type,
+                        depth,
+                        startOffset: frame.startOffset,
+                        endOffset,
+                        contentStart,
+                        contentEnd,
+                    });
+                }
+            }
+            if (stack.length === 0)
+                bodyDone = true;
+        },
+    }, { decodeEntities: false });
+    parser.end(html);
+    const winners = new Map();
+    for (const candidate of candidates) {
+        const current = winners.get(candidate.type);
+        if (current === undefined ||
+            candidate.depth < current.depth ||
+            (candidate.depth === current.depth && candidate.startOffset < current.startOffset)) {
+            winners.set(candidate.type, candidate);
+        }
+    }
+    // `depth` (Candidate's own tie-break field) is deliberately not part of
+    // ShallowestElementMatch: it's an internal selection detail, not
+    // something either caller (extractLandmarks, capContentDepth) uses once
+    // the winner is chosen.
+    return [...winners.values()].map(({ type, startOffset, endOffset, contentStart, contentEnd }) => ({
+        type,
+        startOffset,
+        endOffset,
+        contentStart,
+        contentEnd,
+    }));
+}

package/dist/foldable-tags.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Tags eligible to have their tag name replaced by `.class` in
+ * `build-segment.ts` and to be elided entirely from the path when they have
+ * exactly one element child in `is-fold-candidate.ts`. Kept as a single
+ * source of truth so the two modules can't silently disagree on which tags
+ * are "generic wrappers".
+ */
+export declare const FOLDABLE_TAGS: Set<string>;

package/dist/foldable-tags.js

@@ -0,0 +1,8 @@
+/**
+ * Tags eligible to have their tag name replaced by `.class` in
+ * `build-segment.ts` and to be elided entirely from the path when they have
+ * exactly one element child in `is-fold-candidate.ts`. Kept as a single
+ * source of truth so the two modules can't silently disagree on which tags
+ * are "generic wrappers".
+ */
+export const FOLDABLE_TAGS = new Set(['div', 'span']);

package/dist/format-bracket.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Formats a `[key=value,...]` suffix from named attributes, omitting
+ * `undefined` entries and sorting the `key=value` pairs alphabetically for a
+ * deterministic order. Shared by `build-segment.ts` (role/type on ordinary
+ * elements) and `run-tokenizer.ts` (role/type/sha on opaque
+ * `script`/`style`/`svg`/`noscript` elements), since an opaque element like
+ * `<svg role="img">` still carries a meaningful `role` alongside its content
+ * hash.
+ * @param attrs
+ */
+export declare function formatBracket(attrs: Record<string, string | undefined>): string;

package/dist/format-bracket.js

@@ -0,0 +1,17 @@
+/**
+ * Formats a `[key=value,...]` suffix from named attributes, omitting
+ * `undefined` entries and sorting the `key=value` pairs alphabetically for a
+ * deterministic order. Shared by `build-segment.ts` (role/type on ordinary
+ * elements) and `run-tokenizer.ts` (role/type/sha on opaque
+ * `script`/`style`/`svg`/`noscript` elements), since an opaque element like
+ * `<svg role="img">` still carries a meaningful `role` alongside its content
+ * hash.
+ * @param attrs
+ */
+export function formatBracket(attrs) {
+    const entries = Object.entries(attrs)
+        .filter((entry) => entry[1] !== undefined)
+        .map(([key, value]) => `${key}=${value}`)
+        .toSorted();
+    return entries.length > 0 ? `[${entries.join(',')}]` : '';
+}

package/dist/hash-content.d.ts

@@ -0,0 +1,22 @@
+/**
+ * 16 hex characters (8 bytes) of SHA-256. Full 64-character digests would
+ * make tokens unwieldy for no practical benefit here: this hash only needs
+ * to answer "did this script/style/svg/comment's content change", not
+ * resist deliberate collision attacks, so the reduced collision resistance
+ * of a truncated digest is an acceptable trade-off.
+ *
+ * Exported so other hashed-key producers in this package (e.g.
+ * `deriveStylesheetGroupKey`) use the same truncation length instead of
+ * picking their own.
+ */
+export declare const HASH_LENGTH = 16;
+/**
+ * Hashes `script`/`style`/`svg`/`noscript`/comment content instead of
+ * tokenizing it. The raw content is never retained in the output: keeping it
+ * verbatim would bloat tokens with implementation detail (JS/CSS source,
+ * SVG path data) that carries no structural signal for duplicate-page
+ * detection, and could leak sensitive inline content collected from crawled
+ * pages.
+ * @param raw
+ */
+export declare function hashContent(raw: string): string;

package/dist/hash-content.js

@@ -0,0 +1,26 @@
+import { hash } from '@d-zero/shared/hash';
+import { normalizeForHash } from './normalize-for-hash.js';
+/**
+ * 16 hex characters (8 bytes) of SHA-256. Full 64-character digests would
+ * make tokens unwieldy for no practical benefit here: this hash only needs
+ * to answer "did this script/style/svg/comment's content change", not
+ * resist deliberate collision attacks, so the reduced collision resistance
+ * of a truncated digest is an acceptable trade-off.
+ *
+ * Exported so other hashed-key producers in this package (e.g.
+ * `deriveStylesheetGroupKey`) use the same truncation length instead of
+ * picking their own.
+ */
+export const HASH_LENGTH = 16;
+/**
+ * Hashes `script`/`style`/`svg`/`noscript`/comment content instead of
+ * tokenizing it. The raw content is never retained in the output: keeping it
+ * verbatim would bloat tokens with implementation detail (JS/CSS source,
+ * SVG path data) that carries no structural signal for duplicate-page
+ * detection, and could leak sensitive inline content collected from crawled
+ * pages.
+ * @param raw
+ */
+export function hashContent(raw) {
+    return hash(normalizeForHash(raw)).slice(0, HASH_LENGTH);
+}

package/dist/html-region-utils.d.ts

@@ -0,0 +1,74 @@
+/**
+ * Shared by {@link ./extract-landmarks.js | extractLandmarks} and
+ * {@link ./remove-content-blocks.js | removeContentBlocks}: both walk `html`
+ * with htmlparser2 to find `[start, end)` spans of matched regions, then
+ * excise them from the original string. Not part of either file's own
+ * public API surface (unlike the smaller, ~7-line `requireIndex` copies
+ * duplicated elsewhere in this package), so it's factored out here instead
+ * of duplicated: this logic already had one real bug (see
+ * `escapeRegExp`/`isGenuineClose` below) found and fixed after
+ * `extractLandmarks` shipped, and a second independent copy would be a
+ * second place for that class of bug to reappear.
+ */
+/**
+ * Escapes regex metacharacters in `text` so it can be interpolated into a
+ * `RegExp` literally. Needed because a tag name reaching {@link isGenuineClose}
+ * is not guaranteed to be a plain HTML tag name: htmlparser2 accepts
+ * characters like `(`/`[` inside a tag name (`<div(foo role="banner">`
+ * parses with tag name `"div(foo"`), which would otherwise either throw
+ * (an unbalanced `(` is an invalid regex) or silently change what the regex
+ * matches.
+ * @param text
+ */
+export declare function escapeRegExp(text: string): string;
+/**
+ * Whether `html` actually contains a literal closing tag for `tagName`
+ * ending at `endOffset`. htmlparser2 fires `onclosetag` not only for real
+ * closing tags but also when it force-closes a still-open ancestor to
+ * resolve a mismatch (e.g. `<header>H<main>...</main></body>` with no
+ * `</header>` ever written) — and in that forced case it reports the
+ * force-closed element's `endIndex` as wherever the *other*, unrelated
+ * closing tag that triggered the cascade happens to sit, not any position
+ * derived from the matched element itself (confirmed by direct htmlparser2
+ * event tracing: both the synthetic close and the real `body` close report
+ * the identical `endIndex`, because there is no real closing tag in the
+ * source for htmlparser2 to anchor a distinct position to). Trusting that
+ * offset would slice a candidate spanning all the way to wherever the
+ * unrelated tag ends, silently swallowing real content into the caller's
+ * remainder HTML. Checking that the text immediately preceding `endOffset`
+ * actually spells the expected closing tag catches exactly this: a genuine
+ * close always ends with it; a forced one ends with whatever unrelated tag
+ * forced it instead.
+ * @param html
+ * @param endOffset
+ * @param tagName
+ */
+export declare function isGenuineClose(html: string, endOffset: number, tagName: string): boolean;
+/**
+ * Merges a set of (possibly overlapping or nested) `[start, end)` spans into
+ * the smallest equivalent set of disjoint spans, sorted by start offset.
+ * Matched spans commonly nest in real markup (e.g. a site nav living inside
+ * the header, `<header><nav>...</nav></header>`) — merging first means the
+ * later excision pass never has to reason about overlap.
+ * @param spans
+ */
+export declare function mergeSpans(spans: readonly {
+    start: number;
+    end: number;
+}[]): {
+    start: number;
+    end: number;
+}[];
+/**
+ * Excises `spans` (merged via {@link mergeSpans}) from `html`, returning
+ * what's left. No placeholder is left in a span's place: a placeholder
+ * string would itself become a token once the remainder is tokenized,
+ * reintroducing exactly the kind of synthetic signal callers of this
+ * function exist to remove.
+ * @param html
+ * @param spans
+ */
+export declare function excise(html: string, spans: readonly {
+    start: number;
+    end: number;
+}[]): string;

package/dist/html-region-utils.js

@@ -0,0 +1,96 @@
+/**
+ * Shared by {@link ./extract-landmarks.js | extractLandmarks} and
+ * {@link ./remove-content-blocks.js | removeContentBlocks}: both walk `html`
+ * with htmlparser2 to find `[start, end)` spans of matched regions, then
+ * excise them from the original string. Not part of either file's own
+ * public API surface (unlike the smaller, ~7-line `requireIndex` copies
+ * duplicated elsewhere in this package), so it's factored out here instead
+ * of duplicated: this logic already had one real bug (see
+ * `escapeRegExp`/`isGenuineClose` below) found and fixed after
+ * `extractLandmarks` shipped, and a second independent copy would be a
+ * second place for that class of bug to reappear.
+ */
+/**
+ * Escapes regex metacharacters in `text` so it can be interpolated into a
+ * `RegExp` literally. Needed because a tag name reaching {@link isGenuineClose}
+ * is not guaranteed to be a plain HTML tag name: htmlparser2 accepts
+ * characters like `(`/`[` inside a tag name (`<div(foo role="banner">`
+ * parses with tag name `"div(foo"`), which would otherwise either throw
+ * (an unbalanced `(` is an invalid regex) or silently change what the regex
+ * matches.
+ * @param text
+ */
+export function escapeRegExp(text) {
+    return text.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+/**
+ * Whether `html` actually contains a literal closing tag for `tagName`
+ * ending at `endOffset`. htmlparser2 fires `onclosetag` not only for real
+ * closing tags but also when it force-closes a still-open ancestor to
+ * resolve a mismatch (e.g. `<header>H<main>...</main></body>` with no
+ * `</header>` ever written) — and in that forced case it reports the
+ * force-closed element's `endIndex` as wherever the *other*, unrelated
+ * closing tag that triggered the cascade happens to sit, not any position
+ * derived from the matched element itself (confirmed by direct htmlparser2
+ * event tracing: both the synthetic close and the real `body` close report
+ * the identical `endIndex`, because there is no real closing tag in the
+ * source for htmlparser2 to anchor a distinct position to). Trusting that
+ * offset would slice a candidate spanning all the way to wherever the
+ * unrelated tag ends, silently swallowing real content into the caller's
+ * remainder HTML. Checking that the text immediately preceding `endOffset`
+ * actually spells the expected closing tag catches exactly this: a genuine
+ * close always ends with it; a forced one ends with whatever unrelated tag
+ * forced it instead.
+ * @param html
+ * @param endOffset
+ * @param tagName
+ */
+export function isGenuineClose(html, endOffset, tagName) {
+    const windowStart = Math.max(0, endOffset - tagName.length - 3);
+    return new RegExp(`</\\s*${escapeRegExp(tagName)}\\s*>$`, 'i').test(html.slice(windowStart, endOffset));
+}
+/**
+ * Merges a set of (possibly overlapping or nested) `[start, end)` spans into
+ * the smallest equivalent set of disjoint spans, sorted by start offset.
+ * Matched spans commonly nest in real markup (e.g. a site nav living inside
+ * the header, `<header><nav>...</nav></header>`) — merging first means the
+ * later excision pass never has to reason about overlap.
+ * @param spans
+ */
+export function mergeSpans(spans) {
+    const sorted = [...spans].toSorted((a, b) => a.start - b.start);
+    const merged = [];
+    for (const span of sorted) {
+        const last = merged.at(-1);
+        if (last && span.start <= last.end) {
+            last.end = Math.max(last.end, span.end);
+        }
+        else {
+            merged.push({ ...span });
+        }
+    }
+    return merged;
+}
+/**
+ * Excises `spans` (merged via {@link mergeSpans}) from `html`, returning
+ * what's left. No placeholder is left in a span's place: a placeholder
+ * string would itself become a token once the remainder is tokenized,
+ * reintroducing exactly the kind of synthetic signal callers of this
+ * function exist to remove.
+ * @param html
+ * @param spans
+ */
+export function excise(html, spans) {
+    if (spans.length === 0) {
+        return html;
+    }
+    const merged = mergeSpans(spans);
+    let remainder = '';
+    let cursor = 0;
+    for (const span of merged) {
+        remainder += html.slice(cursor, span.start);
+        cursor = span.end;
+    }
+    remainder += html.slice(cursor);
+    return remainder;
+}

package/dist/is-fold-candidate.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Whether this element is eligible to be elided from the path when it turns
+ * out to have exactly one element child (see `resolve-closed-frame.ts`).
+ *
+ * Only class-less, role-less, type-less `div`/`span` qualify: a `class`,
+ * `role`, or `type` means the element carries structural or semantic
+ * information that would be lost if the element disappeared from the path.
+ * @param tagName
+ * @param classList
+ * @param role
+ * @param type
+ */
+export declare function isFoldCandidate(tagName: string, classList: readonly string[], role?: string, type?: string): boolean;

package/dist/is-fold-candidate.js

@@ -0,0 +1,16 @@
+import { FOLDABLE_TAGS } from './foldable-tags.js';
+/**
+ * Whether this element is eligible to be elided from the path when it turns
+ * out to have exactly one element child (see `resolve-closed-frame.ts`).
+ *
+ * Only class-less, role-less, type-less `div`/`span` qualify: a `class`,
+ * `role`, or `type` means the element carries structural or semantic
+ * information that would be lost if the element disappeared from the path.
+ * @param tagName
+ * @param classList
+ * @param role
+ * @param type
+ */
+export function isFoldCandidate(tagName, classList, role, type) {
+    return FOLDABLE_TAGS.has(tagName) && classList.length === 0 && !role && !type;
+}

package/dist/is-genuine-close.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Whether `html` actually contains a literal closing tag for `tagName`
+ * ending at `endOffset`. htmlparser2 fires `onclosetag` not only for real
+ * closing tags but also when it force-closes a still-open ancestor to
+ * resolve a mismatch (e.g. `<header>H<main>...</main></body>` with no
+ * `</header>` ever written) — and in that forced case it reports the
+ * force-closed element's `endIndex` as wherever the *other*, unrelated
+ * closing tag that triggered the cascade happens to sit, not any position
+ * derived from the matched element itself (confirmed by direct htmlparser2
+ * event tracing: both the synthetic close and the real `body` close report
+ * the identical `endIndex`, because there is no real closing tag in the
+ * source for htmlparser2 to anchor a distinct position to). Trusting that
+ * offset would slice a candidate spanning all the way to wherever the
+ * unrelated tag ends, silently swallowing real content into the caller's
+ * remainder HTML. Checking that the text immediately preceding `endOffset`
+ * actually spells the expected closing tag catches exactly this: a genuine
+ * close always ends with it; a forced one ends with whatever unrelated tag
+ * forced it instead.
+ * @param html
+ * @param endOffset
+ * @param tagName
+ */
+export declare function isGenuineClose(html: string, endOffset: number, tagName: string): boolean;

package/dist/is-genuine-close.js

@@ -0,0 +1,27 @@
+import { escapeRegExp } from './escape-reg-exp.js';
+/**
+ * Whether `html` actually contains a literal closing tag for `tagName`
+ * ending at `endOffset`. htmlparser2 fires `onclosetag` not only for real
+ * closing tags but also when it force-closes a still-open ancestor to
+ * resolve a mismatch (e.g. `<header>H<main>...</main></body>` with no
+ * `</header>` ever written) — and in that forced case it reports the
+ * force-closed element's `endIndex` as wherever the *other*, unrelated
+ * closing tag that triggered the cascade happens to sit, not any position
+ * derived from the matched element itself (confirmed by direct htmlparser2
+ * event tracing: both the synthetic close and the real `body` close report
+ * the identical `endIndex`, because there is no real closing tag in the
+ * source for htmlparser2 to anchor a distinct position to). Trusting that
+ * offset would slice a candidate spanning all the way to wherever the
+ * unrelated tag ends, silently swallowing real content into the caller's
+ * remainder HTML. Checking that the text immediately preceding `endOffset`
+ * actually spells the expected closing tag catches exactly this: a genuine
+ * close always ends with it; a forced one ends with whatever unrelated tag
+ * forced it instead.
+ * @param html
+ * @param endOffset
+ * @param tagName
+ */
+export function isGenuineClose(html, endOffset, tagName) {
+    const windowStart = Math.max(0, endOffset - tagName.length - 3);
+    return new RegExp(`</\\s*${escapeRegExp(tagName)}\\s*>$`, 'i').test(html.slice(windowStart, endOffset));
+}

package/dist/is-noise-class.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Whether `className` matches any of `patterns` (auto-generated/hash-like class names).
+ * @param className
+ * @param patterns
+ */
+export declare function isNoiseClass(className: string, patterns: readonly RegExp[]): boolean;

package/dist/is-noise-class.js

@@ -0,0 +1,8 @@
+/**
+ * Whether `className` matches any of `patterns` (auto-generated/hash-like class names).
+ * @param className
+ * @param patterns
+ */
+export function isNoiseClass(className, patterns) {
+    return patterns.some((pattern) => pattern.test(className));
+}