@d-zero/page-cluster 0.2.0

package/dist/noise-class-patterns.js

@@ -0,0 +1,74 @@
+/**
+ * Heuristics for auto-generated class names that change on every build even
+ * when the underlying template is unchanged (CSS Modules, styled-components,
+ * emotion, bundler content-hash suffixes). Left in place, these would make
+ * identical templates look structurally different across builds/deploys,
+ * defeating near-duplicate detection. The generic alphanumeric-hash pattern requires
+ * both a letter and a digit so real words (e.g. BEM modifiers like
+ * `card--active`) are not caught by accident; it is still the least precise
+ * entry here, which is why `filterNoiseClasses` can be turned off.
+ *
+ * The `sc-`/`css-`/generic-hex patterns below all follow the same
+ * "require a digit or uppercase letter" idiom to rule out real English words
+ * that happen to fit the hash's character-set shape (`sc-header`,
+ * `css-editor`, `section-facade`, ...). This trades a small, accepted
+ * false-negative rate for hash generators that occasionally produce an
+ * all-lowercase, all-letter run (same trade-off already made for the
+ * double-underscore pattern below) against eliminating false positives on
+ * ordinary class names, which is the more common and more disruptive
+ * failure for this package's purpose.
+ */
+export const DEFAULT_NOISE_CLASS_PATTERNS = [
+    // CSS Modules bare hash suffix, e.g. `_a1b2c3`
+    /^_[a-z0-9]{5,}$/i,
+    // styled-components, e.g. `sc-bdVaJa`. Requires an uppercase letter or
+    // digit somewhere after `sc-`: real styled-components hashes are
+    // generated from a mixed-alphabet base, so they reliably contain one,
+    // while genuine author-written classes like `sc-header`/`sc-container`
+    // (all-lowercase English words) do not. Confirmed by running both
+    // against real examples before and after this fix. Length is
+    // deliberately left unbounded (`{5,}`, matching the pre-fix pattern) —
+    // an upper bound here would reject genuine, longer auto-generated hashes
+    // for no benefit, since length was never the discriminator; the
+    // lookahead is.
+    /^sc-(?=[a-zA-Z0-9]*[A-Z\d])[a-zA-Z0-9]{5,}$/,
+    // emotion, e.g. `css-1x2y3z`. Requires a digit somewhere after `css-`,
+    // for the same reason as `sc-` above (emotion's example already has
+    // digits; `css-editor`/`css-toolbar` do not).
+    /^css-(?=[a-z0-9]*\d)[a-z0-9]{5,8}$/i,
+    // bundler content-hash suffix, e.g. `chunk-a3f9c1`. Requires a true
+    // digit (0-9) somewhere in the hex-looking suffix: a hash is far more
+    // likely to contain one than a real English word that happens to be
+    // spelled entirely with a-f (e.g. `facade`, `decade` both previously
+    // misclassified as noise).
+    /^[a-z]+-(?=[a-f0-9]*\d)[a-f0-9]{6,8}$/i,
+    // generic alphanumeric hash, e.g. `k3j9zq2a` (must mix letters and digits).
+    // Deliberately case-sensitive (no `i` flag): auto-generated hash tokens are
+    // conventionally all-lowercase, and real CamelCase/mixed-case class names
+    // that happen to end in a digit (e.g. `Section1`, `Banner99`) must not be
+    // caught by this — the whole point of requiring both a letter and a digit.
+    /^(?=.*[a-z])(?=.*\d)[a-z0-9]{6,10}$/,
+    // webpack CSS Modules' default `[name]_[local]__[hash]` convention, e.g.
+    // `Layout_root__f3k9d`, `Header_title__3xJ9k`. Anchored on the trailing
+    // `__` (exactly two underscores, not one) rather than the whole string,
+    // since here the hash is a suffix of an otherwise-meaningful name.
+    // Deliberately does NOT also match a single underscore: real-world class
+    // names commonly use one underscore as a general-purpose separator
+    // followed by a short alphanumeric variant suffix that is not a hash at
+    // all — e.g. Divi Builder's `et_pb_gutters3` (gutter-width setting 3 of
+    // 8), or hypothetical `grid_col12`/`row_span24`. Matching those against
+    // this pattern with only one underscore required turned them into false
+    // positives (confirmed via the production-scale fixture corpus); double
+    // underscore is a much rarer, more specifically BEM/CSS-Modules-coded
+    // convention, so it's a safer anchor. This does mean single-underscore
+    // hash suffixes (e.g. HubSpot's `hsForm_9f8e7d6c`) are not caught — an
+    // accepted gap, since under-filtering a rare pattern is safer than
+    // over-filtering common ones. Requiring both a letter and a digit in the
+    // suffix (case-insensitive: real generators mix case, e.g. `q7Rp1`)
+    // keeps genuine short BEM-ish element names ending in one digit and
+    // nothing else (`col2`, 4 chars) below the 5-char floor from matching;
+    // residual risk of a real element name coincidentally looking like a
+    // 5-8 char mixed hash right after `__` (e.g. `__col12`) is accepted,
+    // same trade-off as the generic pattern above.
+    /__(?=[a-z0-9]{5,8}$)(?=[a-z0-9]*[a-z])(?=[a-z0-9]*\d)[a-z0-9]{5,8}$/i,
+];

package/dist/normalize-for-hash.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Normalizes raw `script`/`style`/`svg`/`noscript`/comment content before
+ * hashing so that formatting differences (indentation, line breaks, minified
+ * vs. pretty-printed) don't produce different hashes for otherwise-identical
+ * content. Whitespace runs collapse to a single space rather than being
+ * removed outright — removing them entirely would merge adjacent tokens
+ * (e.g. `"var  a=1"` → `"vara=1"`) and change the content's meaning.
+ * @param raw
+ */
+export declare function normalizeForHash(raw: string): string;

package/dist/normalize-for-hash.js

@@ -0,0 +1,12 @@
+/**
+ * Normalizes raw `script`/`style`/`svg`/`noscript`/comment content before
+ * hashing so that formatting differences (indentation, line breaks, minified
+ * vs. pretty-printed) don't produce different hashes for otherwise-identical
+ * content. Whitespace runs collapse to a single space rather than being
+ * removed outright — removing them entirely would merge adjacent tokens
+ * (e.g. `"var  a=1"` → `"vara=1"`) and change the content's meaning.
+ * @param raw
+ */
+export function normalizeForHash(raw) {
+    return raw.trim().replaceAll(/\s+/g, ' ');
+}

package/dist/opaque-tags.d.ts

@@ -0,0 +1,17 @@
+import type { OpaqueTagName } from './types.js';
+/**
+ * Tags whose contents are opaque to structural analysis: `<script>`/`<style>`
+ * content is parsed as raw text by htmlparser2 (no `onopentag`/`onclosetag`
+ * fires inside them at all), so only `<svg>`/`<noscript>` actually need
+ * active self-nesting suppression in callers that track it — kept as a set
+ * of four for symmetry rather than because all four need the same handling.
+ * Shared by `run-tokenizer.ts` and `extract-landmarks.ts` so a future fix to
+ * this list (or to the self-nesting handling built on top of it) can't
+ * silently diverge between the two.
+ */
+export declare const OPAQUE_TAGS: Set<OpaqueTagName>;
+/**
+ *
+ * @param name
+ */
+export declare function isOpaqueTagName(name: string): name is OpaqueTagName;

package/dist/opaque-tags.js

@@ -0,0 +1,18 @@
+/**
+ * Tags whose contents are opaque to structural analysis: `<script>`/`<style>`
+ * content is parsed as raw text by htmlparser2 (no `onopentag`/`onclosetag`
+ * fires inside them at all), so only `<svg>`/`<noscript>` actually need
+ * active self-nesting suppression in callers that track it — kept as a set
+ * of four for symmetry rather than because all four need the same handling.
+ * Shared by `run-tokenizer.ts` and `extract-landmarks.ts` so a future fix to
+ * this list (or to the self-nesting handling built on top of it) can't
+ * silently diverge between the two.
+ */
+export const OPAQUE_TAGS = new Set(['script', 'style', 'noscript', 'svg']);
+/**
+ *
+ * @param name
+ */
+export function isOpaqueTagName(name) {
+    return OPAQUE_TAGS.has(name);
+}

package/dist/parse-class-list.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Splits a `class` attribute value into a deduplicated, optionally
+ * noise-filtered, case-insensitively sorted list.
+ *
+ * Sorting ignores case so `"Beta alpha"` reads as `["alpha", "Beta"]` rather
+ * than the code-point order a plain `.sort()` would give.
+ * @param classAttr
+ * @param filterNoise
+ */
+export declare function parseClassList(classAttr: string | undefined, filterNoise: boolean): string[];

package/dist/parse-class-list.js

@@ -0,0 +1,23 @@
+import { alphabeticalComparator } from '@d-zero/shared/sort/alphabetical';
+import { isNoiseClass } from './is-noise-class.js';
+import { DEFAULT_NOISE_CLASS_PATTERNS } from './noise-class-patterns.js';
+/**
+ * Splits a `class` attribute value into a deduplicated, optionally
+ * noise-filtered, case-insensitively sorted list.
+ *
+ * Sorting ignores case so `"Beta alpha"` reads as `["alpha", "Beta"]` rather
+ * than the code-point order a plain `.sort()` would give.
+ * @param classAttr
+ * @param filterNoise
+ */
+export function parseClassList(classAttr, filterNoise) {
+    if (!classAttr) {
+        return [];
+    }
+    const raw = classAttr.trim().split(/\s+/).filter(Boolean);
+    const deduped = [...new Set(raw)];
+    const filtered = filterNoise
+        ? deduped.filter((name) => !isNoiseClass(name, DEFAULT_NOISE_CLASS_PATTERNS))
+        : deduped;
+    return filtered.toSorted(alphabeticalComparator);
+}

package/dist/reassign-orphan-block-keys.d.ts

@@ -0,0 +1,81 @@
+import type { PageBlockingSignals } from './resolve-blocking-group-keys.js';
+/**
+ * Rewrites the `path:`-fallback key of an "orphan" page — one with no
+ * stylesheet references recorded at all — to match a same-URL-section `css:`
+ * key, when one exists that is itself confined to that same section.
+ *
+ * {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys} commits
+ * every page to exactly one key: a page with zero stylesheet references can
+ * never produce a `css:` candidate, so it always falls back to `path:`, even
+ * when every other page of the same template loaded a distinctive stylesheet
+ * and landed on a shared `css:` key instead. That function's own JSDoc notes
+ * a literal OR-merge (letting a page carry both candidates) is deliberately
+ * deferred — but a literal OR-merge would not even apply here, since an
+ * orphan never had a `css:` candidate to OR with in the first place. This is
+ * a narrower, targeted fix for that specific gap, confirmed against a crawl
+ * (nitpicker) where a subset of same-template pages were missing stylesheet
+ * data entirely (a crawl-completeness gap, not a `resolveBlockingGroupKeys`
+ * logic error) and fragmented away from the rest of their template's `css:`
+ * block as a result.
+ *
+ * A `css:` block is only treated as a merge target when *every* one of its
+ * members shares the orphan's `derivePathGroupKey` value ("confined" to that
+ * section) — a `css:` block spanning multiple sections (e.g. a shared
+ * cross-department template) is left untouched, since there is no single
+ * section to merge it into. Non-orphan pages sharing the orphan's `path:` key
+ * (pages with a stylesheet set that was genuinely non-discriminative, not
+ * merely unrecorded) are deliberately left out of the rewritten key: complete-
+ * linkage clustering's min-linkage aggregation means a third point entering a
+ * comparison pool can change whether two *other* points end up merged (e.g.
+ * two pages at a pairwise similarity just above threshold can fail to merge
+ * once a third, more tightly-matching point joins the pool and "uses up" one
+ * of them first) — see this function's spec for a worked example. Folding in
+ * only the confirmed orphans, not the whole `path:` bucket, keeps this
+ * function from perturbing clustering decisions for pages it has no evidence
+ * about.
+ *
+ * Does not itself compare page content: it only decides which pages should
+ * be pooled together for {@link ./resolve-structural-cluster-keys.js |
+ * resolveStructuralClusterKeys} to adjudicate (via
+ * {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}'s existing
+ * per-key grouping, unchanged) — an orphan folded into a `css:` block's pool
+ * still ends up in its own singleton cluster if its content doesn't actually
+ * match.
+ *
+ * A known, accepted trade-off: "confined" is checked against `pathKey` alone
+ * (the same coarse granularity `pathDepth` already gives `path:` keys), so
+ * multiple genuinely-different-template `css:` blocks that merely happen to
+ * share one broad URL section (e.g. everything under a large sub-site's
+ * top-level segment) get pooled into the *same* `resolveStructuralClusterKeys`
+ * call as each other, not just alongside the orphan that triggered the merge.
+ * Confirmed on real crawl data (8,936 pages): this correctly reunited an
+ * orphan with a 1,008-page same-template cluster it had been split from, and
+ * the corpus's cluster count improved net (1,984 → 1,972) — but as a side
+ * effect, 15 small clusters *unrelated* to any orphan (all inside one very
+ * large, already-confined `css:` block spanning a whole sub-site) came out
+ * differently than they would have without this option, because the extra
+ * pages pooled alongside them changed the NN-chain merge order (the same
+ * mechanism described above, just triggered by the pooling itself rather
+ * than by admitting a non-orphan page). A companion real-site run (302 pages)
+ * showed zero change. Narrowing "confined" to reduce this blast radius (e.g.
+ * preferring the `css:` block closest in size to the orphan count when
+ * several share a `pathKey`) is possible future work, not yet justified
+ * without more real-corpus evidence of it mattering in practice.
+ * @param pages
+ * @param blockKeys
+ * @param pathDepth
+ * @example
+ * ```ts
+ * const pages = [
+ * 	{ paths: ['news', '1'], stylesheetHrefs: ['https://example.com/a.css'] },
+ * 	{ paths: ['news', '2'], stylesheetHrefs: ['https://example.com/a.css'] },
+ * 	{ paths: ['news', '3'], stylesheetHrefs: [] }, // same template, but crawl missed its <link>
+ * 	{ paths: ['about'], stylesheetHrefs: ['https://example.com/b.css'] },
+ * ];
+ * const blockKeys = resolveBlockingGroupKeys(pages);
+ * // ['css:<hash of a.css>', 'css:<hash of a.css>', 'path:news', 'path:about']
+ * reassignOrphanBlockKeys(pages, blockKeys);
+ * // ['orphan-merge:news', 'orphan-merge:news', 'orphan-merge:news', 'path:about']
+ * ```
+ */
+export declare function reassignOrphanBlockKeys(pages: readonly PageBlockingSignals[], blockKeys: readonly string[], pathDepth?: number): string[];

package/dist/reassign-orphan-block-keys.js

@@ -0,0 +1,159 @@
+import { derivePathGroupKey } from './derive-path-group-key.js';
+/**
+ * Prefix distinguishing a reassigned key from the `css:`/`path:` keys
+ * {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys} itself
+ * produces, so the two families can never collide.
+ */
+const REASSIGNED_KEY_PREFIX = 'orphan-merge:';
+/**
+ * Reads `values[index]`, throwing instead of returning `undefined`. Every
+ * call site here indexes `pages`/`pathKeys`/`blockKeys` with a position
+ * derived from one of those same arrays' own `.entries()` or `.map()`, so the
+ * thrown branch is unreachable in practice; it exists to satisfy
+ * `noUncheckedIndexedAccess` without a non-null assertion (same rationale as
+ * `requireIndex` in `resolve-page-cluster-keys.ts` and
+ * `resolve-structural-cluster-keys.ts`, each kept as an independent copy for
+ * the same reason those two are).
+ * @param values
+ * @param index
+ */
+function requireIndex(values, index) {
+    const value = values[index];
+    if (value === undefined) {
+        throw new Error('reassignOrphanBlockKeys: index out of bounds');
+    }
+    return value;
+}
+/**
+ * Rewrites the `path:`-fallback key of an "orphan" page — one with no
+ * stylesheet references recorded at all — to match a same-URL-section `css:`
+ * key, when one exists that is itself confined to that same section.
+ *
+ * {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys} commits
+ * every page to exactly one key: a page with zero stylesheet references can
+ * never produce a `css:` candidate, so it always falls back to `path:`, even
+ * when every other page of the same template loaded a distinctive stylesheet
+ * and landed on a shared `css:` key instead. That function's own JSDoc notes
+ * a literal OR-merge (letting a page carry both candidates) is deliberately
+ * deferred — but a literal OR-merge would not even apply here, since an
+ * orphan never had a `css:` candidate to OR with in the first place. This is
+ * a narrower, targeted fix for that specific gap, confirmed against a crawl
+ * (nitpicker) where a subset of same-template pages were missing stylesheet
+ * data entirely (a crawl-completeness gap, not a `resolveBlockingGroupKeys`
+ * logic error) and fragmented away from the rest of their template's `css:`
+ * block as a result.
+ *
+ * A `css:` block is only treated as a merge target when *every* one of its
+ * members shares the orphan's `derivePathGroupKey` value ("confined" to that
+ * section) — a `css:` block spanning multiple sections (e.g. a shared
+ * cross-department template) is left untouched, since there is no single
+ * section to merge it into. Non-orphan pages sharing the orphan's `path:` key
+ * (pages with a stylesheet set that was genuinely non-discriminative, not
+ * merely unrecorded) are deliberately left out of the rewritten key: complete-
+ * linkage clustering's min-linkage aggregation means a third point entering a
+ * comparison pool can change whether two *other* points end up merged (e.g.
+ * two pages at a pairwise similarity just above threshold can fail to merge
+ * once a third, more tightly-matching point joins the pool and "uses up" one
+ * of them first) — see this function's spec for a worked example. Folding in
+ * only the confirmed orphans, not the whole `path:` bucket, keeps this
+ * function from perturbing clustering decisions for pages it has no evidence
+ * about.
+ *
+ * Does not itself compare page content: it only decides which pages should
+ * be pooled together for {@link ./resolve-structural-cluster-keys.js |
+ * resolveStructuralClusterKeys} to adjudicate (via
+ * {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}'s existing
+ * per-key grouping, unchanged) — an orphan folded into a `css:` block's pool
+ * still ends up in its own singleton cluster if its content doesn't actually
+ * match.
+ *
+ * A known, accepted trade-off: "confined" is checked against `pathKey` alone
+ * (the same coarse granularity `pathDepth` already gives `path:` keys), so
+ * multiple genuinely-different-template `css:` blocks that merely happen to
+ * share one broad URL section (e.g. everything under a large sub-site's
+ * top-level segment) get pooled into the *same* `resolveStructuralClusterKeys`
+ * call as each other, not just alongside the orphan that triggered the merge.
+ * Confirmed on real crawl data (8,936 pages): this correctly reunited an
+ * orphan with a 1,008-page same-template cluster it had been split from, and
+ * the corpus's cluster count improved net (1,984 → 1,972) — but as a side
+ * effect, 15 small clusters *unrelated* to any orphan (all inside one very
+ * large, already-confined `css:` block spanning a whole sub-site) came out
+ * differently than they would have without this option, because the extra
+ * pages pooled alongside them changed the NN-chain merge order (the same
+ * mechanism described above, just triggered by the pooling itself rather
+ * than by admitting a non-orphan page). A companion real-site run (302 pages)
+ * showed zero change. Narrowing "confined" to reduce this blast radius (e.g.
+ * preferring the `css:` block closest in size to the orphan count when
+ * several share a `pathKey`) is possible future work, not yet justified
+ * without more real-corpus evidence of it mattering in practice.
+ * @param pages
+ * @param blockKeys
+ * @param pathDepth
+ * @example
+ * ```ts
+ * const pages = [
+ * 	{ paths: ['news', '1'], stylesheetHrefs: ['https://example.com/a.css'] },
+ * 	{ paths: ['news', '2'], stylesheetHrefs: ['https://example.com/a.css'] },
+ * 	{ paths: ['news', '3'], stylesheetHrefs: [] }, // same template, but crawl missed its <link>
+ * 	{ paths: ['about'], stylesheetHrefs: ['https://example.com/b.css'] },
+ * ];
+ * const blockKeys = resolveBlockingGroupKeys(pages);
+ * // ['css:<hash of a.css>', 'css:<hash of a.css>', 'path:news', 'path:about']
+ * reassignOrphanBlockKeys(pages, blockKeys);
+ * // ['orphan-merge:news', 'orphan-merge:news', 'orphan-merge:news', 'path:about']
+ * ```
+ */
+export function reassignOrphanBlockKeys(pages, blockKeys, pathDepth) {
+    // Eagerly delegate pathDepth validation to derivePathGroupKey, instead of
+    // only discovering an invalid option once some page's data happens to
+    // reach that branch below — mirrors resolveBlockingGroupKeys's own eager
+    // `derivePathGroupKey([], pathDepth)` call, so this function fails fast
+    // on the same invalid input even when `pages` is empty.
+    derivePathGroupKey([], pathDepth);
+    const pathKeys = pages.map((page) => derivePathGroupKey(page.paths, pathDepth));
+    const isOrphan = (index) => requireIndex(blockKeys, index).startsWith('path:') &&
+        requireIndex(pages, index).stylesheetHrefs.length === 0;
+    const cssMemberIndicesByKey = new Map();
+    for (const [index, blockKey] of blockKeys.entries()) {
+        if (blockKey.startsWith('css:')) {
+            const indices = cssMemberIndicesByKey.get(blockKey);
+            if (indices) {
+                indices.push(index);
+            }
+            else {
+                cssMemberIndicesByKey.set(blockKey, [index]);
+            }
+        }
+    }
+    // A css: block is a merge target for pathKey `p` only when every one of
+    // its members shares `p` — collected as a single confined pathKey per
+    // block (or left unset if the block spans more than one).
+    const confinedPathKeyByCssKey = new Map();
+    for (const [cssKey, indices] of cssMemberIndicesByKey) {
+        const candidatePathKey = requireIndex(pathKeys, requireIndex(indices, 0));
+        if (indices.every((index) => requireIndex(pathKeys, index) === candidatePathKey)) {
+            confinedPathKeyByCssKey.set(cssKey, candidatePathKey);
+        }
+    }
+    const pathKeysWithConfinedCssBlock = new Set(confinedPathKeyByCssKey.values());
+    const orphanPathKeys = new Set();
+    for (const index of blockKeys.keys()) {
+        const pathKey = requireIndex(pathKeys, index);
+        if (isOrphan(index) && pathKeysWithConfinedCssBlock.has(pathKey)) {
+            orphanPathKeys.add(pathKey);
+        }
+    }
+    if (orphanPathKeys.size === 0) {
+        return [...blockKeys];
+    }
+    return blockKeys.map((blockKey, index) => {
+        const pathKey = requireIndex(pathKeys, index);
+        if (!orphanPathKeys.has(pathKey)) {
+            return blockKey;
+        }
+        const isConfinedCssMember = confinedPathKeyByCssKey.get(blockKey) === pathKey;
+        return isOrphan(index) || isConfinedCssMember
+            ? `${REASSIGNED_KEY_PREFIX}${pathKey}`
+            : blockKey;
+    });
+}

package/dist/remove-content-blocks.d.ts

@@ -0,0 +1,67 @@
+/**
+ * @see removeContentBlocks
+ */
+export type RemoveContentBlocksOptions = {
+    /**
+     * The HTML attribute name that marks one instance of a freeform,
+     * block-editor-authored content region (e.g. a WYSIWYG CMS's own
+     * per-block marker attribute). No default: unlike
+     * {@link ./extract-landmarks.js | extractLandmarks}'s tag names and ARIA
+     * roles, there is no cross-site standard for this — every CMS/page
+     * builder invents its own convention, so the caller must name theirs.
+     */
+    blockAttribute: string;
+};
+/**
+ * Result of {@link ./remove-content-blocks.js | removeContentBlocks}.
+ */
+export type RemoveContentBlocksResult = {
+    remainderHtml: string;
+};
+/**
+ * Removes every element in `html` carrying `options.blockAttribute`,
+ * including its full subtree, and returns what's left.
+ *
+ * Built for a specific real-crawl finding: two pages built from the same
+ * article template, but authored with a different mix of freeform CMS
+ * content blocks (e.g. one page uses a "wysiwyg" block then two "image"
+ * blocks, another uses a "title" block then one "image" block), tokenize to
+ * almost entirely disjoint leaf paths — each block's own internal markup
+ * differs, and that difference dominates the page's whole token set. This
+ * defeats {@link ./resolve-structural-cluster-keys.js |
+ * resolveStructuralClusterKeys}'s frequency-based chrome/content split
+ * (`deriveComparisonSets`), which only recognizes tokens common to *nearly
+ * every* page in a block as chrome; a block-editor region varies too much
+ * page-to-page to ever clear that bar, yet contributes no genuine
+ * template-identifying signal either. Confirmed on a real corpus (302
+ * pages): removing these regions before tokenizing cut structural-cluster
+ * count from 192 to 32 (with {@link ./extract-landmarks.js | extractLandmarks}
+ * and blocking-key fixes already applied) — the single largest lever found
+ * for that corpus.
+ *
+ * Unlike `extractLandmarks`, every matching element is removed (there is no
+ * "one instance per type" — a page can have any number of content blocks),
+ * and matching is by a single caller-supplied attribute rather than a fixed
+ * tag/role vocabulary — see `RemoveContentBlocksOptions.blockAttribute`'s
+ * JSDoc for why. `remainderHtml` drops each matched region's markup
+ * entirely (no placeholder), for the same reason `extractLandmarks` does:
+ * a placeholder string would itself become a token once tokenized.
+ *
+ * Only the first `<body>` is in scope and opaque tags
+ * (`script`/`style`/`svg`/`noscript`) are not searched inside, matching
+ * `extractLandmarks`'s and `tokenize()`'s own contracts. A candidate whose
+ * closing tag can't be confirmed as genuine (see `isGenuineClose`) is
+ * discarded rather than trusted, for the same safety reason `extractLandmarks`
+ * discards one.
+ * @param html
+ * @param options
+ * @example
+ * ```ts
+ * removeContentBlocks(
+ * 	'<body><main><div data-bgb="wysiwyg">free text</div><div data-bgb="image1">...</div></main></body>',
+ * 	{ blockAttribute: 'data-bgb' },
+ * );
+ * // { remainderHtml: '<body><main></main></body>' }
+ * ```
+ */
+export declare function removeContentBlocks(html: string, options: RemoveContentBlocksOptions): RemoveContentBlocksResult;

package/dist/remove-content-blocks.js

@@ -0,0 +1,150 @@
+import { Parser } from 'htmlparser2';
+import { excise } from './excise.js';
+import { isGenuineClose } from './is-genuine-close.js';
+import { isOpaqueTagName } from './opaque-tags.js';
+/**
+ * Removes every element in `html` carrying `options.blockAttribute`,
+ * including its full subtree, and returns what's left.
+ *
+ * Built for a specific real-crawl finding: two pages built from the same
+ * article template, but authored with a different mix of freeform CMS
+ * content blocks (e.g. one page uses a "wysiwyg" block then two "image"
+ * blocks, another uses a "title" block then one "image" block), tokenize to
+ * almost entirely disjoint leaf paths — each block's own internal markup
+ * differs, and that difference dominates the page's whole token set. This
+ * defeats {@link ./resolve-structural-cluster-keys.js |
+ * resolveStructuralClusterKeys}'s frequency-based chrome/content split
+ * (`deriveComparisonSets`), which only recognizes tokens common to *nearly
+ * every* page in a block as chrome; a block-editor region varies too much
+ * page-to-page to ever clear that bar, yet contributes no genuine
+ * template-identifying signal either. Confirmed on a real corpus (302
+ * pages): removing these regions before tokenizing cut structural-cluster
+ * count from 192 to 32 (with {@link ./extract-landmarks.js | extractLandmarks}
+ * and blocking-key fixes already applied) — the single largest lever found
+ * for that corpus.
+ *
+ * Unlike `extractLandmarks`, every matching element is removed (there is no
+ * "one instance per type" — a page can have any number of content blocks),
+ * and matching is by a single caller-supplied attribute rather than a fixed
+ * tag/role vocabulary — see `RemoveContentBlocksOptions.blockAttribute`'s
+ * JSDoc for why. `remainderHtml` drops each matched region's markup
+ * entirely (no placeholder), for the same reason `extractLandmarks` does:
+ * a placeholder string would itself become a token once tokenized.
+ *
+ * Only the first `<body>` is in scope and opaque tags
+ * (`script`/`style`/`svg`/`noscript`) are not searched inside, matching
+ * `extractLandmarks`'s and `tokenize()`'s own contracts. A candidate whose
+ * closing tag can't be confirmed as genuine (see `isGenuineClose`) is
+ * discarded rather than trusted, for the same safety reason `extractLandmarks`
+ * discards one.
+ * @param html
+ * @param options
+ * @example
+ * ```ts
+ * removeContentBlocks(
+ * 	'<body><main><div data-bgb="wysiwyg">free text</div><div data-bgb="image1">...</div></main></body>',
+ * 	{ blockAttribute: 'data-bgb' },
+ * );
+ * // { remainderHtml: '<body><main></main></body>' }
+ * ```
+ */
+export function removeContentBlocks(html, options) {
+    const { blockAttribute } = options;
+    const stack = [];
+    const spans = [];
+    // Same rationale and shape as extractLandmarks's own `opaque` tracking:
+    // a different opaque tag opening/closing while already inside one must
+    // not perturb this counter. Also tracks whether the opaque tag itself
+    // (its own attributes, not any descendant's) carries `blockAttribute` —
+    // an opaque tag can be a content block's own root (e.g. an inline `<svg>`
+    // chart authored as one block), and diving into its children to look for
+    // the attribute would be pointless (they're never searched for landmarks
+    // or blocks either) but the opaque root itself must still be checked.
+    let opaque = null;
+    let bodyDone = false;
+    let ignoredBodyOpens = 0;
+    const parser = new Parser({
+        onopentag(name, attribs) {
+            if (opaque) {
+                if (name === opaque.tagName) {
+                    opaque.depth++;
+                }
+                return;
+            }
+            if (stack.length === 0) {
+                if (name === 'body' && !bodyDone) {
+                    // The whole page can never itself be "a content block" —
+                    // unlike extractLandmarks's ARIA-role matching (a
+                    // standardized vocabulary under which `<body role="banner">`
+                    // is a legitimate whole-page landmark), `blockAttribute` is
+                    // an arbitrary caller-chosen name that could collide with an
+                    // unrelated attribute a CMS/theme happens to stamp onto
+                    // `<body>` (a page-type marker, a `body_class()`-style hook).
+                    // Treating that as a match would silently empty the page's
+                    // entire remainderHtml.
+                    stack.push({
+                        tagName: name,
+                        hasBlockAttribute: false,
+                        startOffset: parser.startIndex,
+                    });
+                }
+                // Ignore everything else outside <body>, same as
+                // extractLandmarks/run-tokenizer.ts.
+                return;
+            }
+            if (name === 'body') {
+                ignoredBodyOpens++;
+                return;
+            }
+            if (isOpaqueTagName(name)) {
+                opaque = {
+                    tagName: name,
+                    depth: 1,
+                    matched: attribs[blockAttribute] !== undefined,
+                    startOffset: parser.startIndex,
+                };
+                return;
+            }
+            stack.push({
+                tagName: name,
+                hasBlockAttribute: attribs[blockAttribute] !== undefined,
+                startOffset: parser.startIndex,
+            });
+        },
+        onclosetag(name) {
+            if (opaque) {
+                if (name === opaque.tagName) {
+                    opaque.depth--;
+                    if (opaque.depth === 0) {
+                        const endOffset = parser.endIndex + 1;
+                        if (opaque.matched && isGenuineClose(html, endOffset, opaque.tagName)) {
+                            spans.push({ start: opaque.startOffset, end: endOffset });
+                        }
+                        opaque = null;
+                    }
+                }
+                return;
+            }
+            if (name === 'body' && ignoredBodyOpens > 0) {
+                ignoredBodyOpens--;
+                return;
+            }
+            if (stack.length === 0) {
+                return;
+            }
+            const frame = stack.pop();
+            if (!frame) {
+                return;
+            }
+            const endOffset = parser.endIndex + 1;
+            if (frame.hasBlockAttribute && isGenuineClose(html, endOffset, frame.tagName)) {
+                spans.push({ start: frame.startOffset, end: endOffset });
+            }
+            if (stack.length === 0) {
+                bodyDone = true;
+            }
+        },
+    }, { decodeEntities: false });
+    parser.end(html);
+    return { remainderHtml: excise(html, spans) };
+}