npm - @d-zero/page-cluster - Versions diffs - 0.2.0 - Mend

@d-zero/page-cluster 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (80) hide show

package/README.md +68 -0
package/dist/array-edit-distance.d.ts +20 -0
package/dist/array-edit-distance.js +52 -0
package/dist/build-segment.d.ts +18 -0
package/dist/build-segment.js +27 -0
package/dist/cap-content-depth.d.ts +69 -0
package/dist/cap-content-depth.js +161 -0
package/dist/compute-document-frequency.d.ts +33 -0
package/dist/compute-document-frequency.js +40 -0
package/dist/create-frame.d.ts +16 -0
package/dist/create-frame.js +29 -0
package/dist/derive-path-group-key.d.ts +44 -0
package/dist/derive-path-group-key.js +51 -0
package/dist/derive-stylesheet-group-key.d.ts +36 -0
package/dist/derive-stylesheet-group-key.js +41 -0
package/dist/detect-content-depth-cap.d.ts +114 -0
package/dist/detect-content-depth-cap.js +137 -0
package/dist/escape-reg-exp.d.ts +11 -0
package/dist/escape-reg-exp.js +13 -0
package/dist/excise.d.ts +13 -0
package/dist/excise.js +24 -0
package/dist/extract-landmarks.d.ts +82 -0
package/dist/extract-landmarks.js +104 -0
package/dist/filter-first-party-stylesheet-hrefs.d.ts +73 -0
package/dist/filter-first-party-stylesheet-hrefs.js +118 -0
package/dist/find-shallowest-elements.d.ts +39 -0
package/dist/find-shallowest-elements.js +121 -0
package/dist/foldable-tags.d.ts +8 -0
package/dist/foldable-tags.js +8 -0
package/dist/format-bracket.d.ts +11 -0
package/dist/format-bracket.js +17 -0
package/dist/hash-content.d.ts +22 -0
package/dist/hash-content.js +26 -0
package/dist/html-region-utils.d.ts +74 -0
package/dist/html-region-utils.js +96 -0
package/dist/is-fold-candidate.d.ts +13 -0
package/dist/is-fold-candidate.js +16 -0
package/dist/is-genuine-close.d.ts +23 -0
package/dist/is-genuine-close.js +27 -0
package/dist/is-noise-class.d.ts +6 -0
package/dist/is-noise-class.js +8 -0
package/dist/jaccard-similarity.d.ts +23 -0
package/dist/jaccard-similarity.js +36 -0
package/dist/merge-landmark-affined-clusters.d.ts +179 -0
package/dist/merge-landmark-affined-clusters.js +544 -0
package/dist/merge-spans.d.ts +15 -0
package/dist/merge-spans.js +22 -0
package/dist/noise-class-patterns.d.ts +21 -0
package/dist/noise-class-patterns.js +74 -0
package/dist/normalize-for-hash.d.ts +10 -0
package/dist/normalize-for-hash.js +12 -0
package/dist/opaque-tags.d.ts +17 -0
package/dist/opaque-tags.js +18 -0
package/dist/parse-class-list.d.ts +10 -0
package/dist/parse-class-list.js +23 -0
package/dist/reassign-orphan-block-keys.d.ts +81 -0
package/dist/reassign-orphan-block-keys.js +159 -0
package/dist/remove-content-blocks.d.ts +67 -0
package/dist/remove-content-blocks.js +150 -0
package/dist/resolve-blocking-group-keys.d.ts +116 -0
package/dist/resolve-blocking-group-keys.js +120 -0
package/dist/resolve-closed-frame.d.ts +26 -0
package/dist/resolve-closed-frame.js +33 -0
package/dist/resolve-landmark-variant-keys.d.ts +66 -0
package/dist/resolve-landmark-variant-keys.js +71 -0
package/dist/resolve-options.d.ts +6 -0
package/dist/resolve-options.js +10 -0
package/dist/resolve-page-cluster-keys.d.ts +222 -0
package/dist/resolve-page-cluster-keys.js +198 -0
package/dist/resolve-structural-cluster-keys.d.ts +50 -0
package/dist/resolve-structural-cluster-keys.js +287 -0
package/dist/run-tokenizer.d.ts +33 -0
package/dist/run-tokenizer.js +152 -0
package/dist/split-tokens-by-frequency.d.ts +46 -0
package/dist/split-tokens-by-frequency.js +88 -0
package/dist/tokenize.d.ts +58 -0
package/dist/tokenize.js +60 -0
package/dist/types.d.ts +85 -0
package/dist/types.js +1 -0
package/package.json +102 -0

package/dist/resolve-structural-cluster-keys.js ADDED Viewed

@@ -0,0 +1,287 @@
+import { computeDocumentFrequency } from './compute-document-frequency.js';
+import { jaccardSimilarity } from './jaccard-similarity.js';
+import { splitTokensByFrequency } from './split-tokens-by-frequency.js';
+const DEFAULT_SIMILARITY_THRESHOLD = 0.8;
+/**
+ * `jaccardSimilarity()` returns `intersectionSize / unionSize`, a
+ * floating-point division that can land a hair below the caller's intended
+ * threshold even when the two are mathematically equal (e.g. a threshold
+ * assembled from arithmetic like `0.1 + 0.2` is `0.30000000000000004`, not
+ * `0.3`), which would otherwise make a pair at the documented inclusive
+ * boundary fail the `>=` check it should pass. Subtracting this epsilon
+ * before comparing absorbs that rounding noise (same technique and value as
+ * `BOUNDARY_EPSILON` in `split-tokens-by-frequency.ts`).
+ */
+const BOUNDARY_EPSILON = 1e-9;
+/**
+ * Below this many pages, `computeDocumentFrequency`/`splitTokensByFrequency`
+ * (the default 90% cutoff) degenerate rather than usefully separate chrome
+ * from content — see `deriveComparisonSets` for the failure mode. Derived
+ * from `splitTokensByFrequency`'s own cutoff: a token missing from exactly
+ * one page out of `n` still counts as chrome only if
+ * `(n - 1) / n >= 0.9`, i.e. `n >= 10`. Below that, this function falls back
+ * to comparing `tokenSets` directly, unfiltered.
+ */
+const MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT = 10;
+/**
+ * Narrows each page's token set to its page-specific content before
+ * clustering, so pages that only share site-wide chrome (header/nav/footer)
+ * don't read as more similar than they structurally are, and so genuine
+ * layout matches aren't swamped by chrome noise at loose thresholds — see
+ * `splitTokensByFrequency`'s JSDoc for the two failure modes this fixes.
+ * Confirmed on real crawl data (a corporate site using a freeform CMS block
+ * editor for its content area): without this, two pages built from the same
+ * article template but a different mix of content blocks could score *lower*
+ * on raw Jaccard than two pages built from genuinely different templates
+ * that happen to share more chrome relative to their (smaller) content area.
+ *
+ * Skipped entirely below `MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT` pages: at the
+ * default 90% cutoff, `splitTokensByFrequency` requires a token to appear on
+ * literally every page to count as chrome once `n < 10` (see that constant's
+ * JSDoc for the derivation). At `n = 2` this is a total degenerate case, not
+ * just an imprecise one — `content(A) = A \ B` and `content(B) = B \ A` are
+ * disjoint *by construction* for any two sets, so
+ * `jaccardSimilarity(content(A), content(B))` is always `0` unless `A` and
+ * `B` are identical, regardless of how similar they actually are (confirmed:
+ * two pages sharing 999 of 1000 tokens, differing in exactly one each, go
+ * from a raw similarity of `0.998` to a content-only similarity of `0`).
+ * Falling back to unfiltered `tokenSets` below the floor accepts chrome
+ * dilution for small blocks rather than this much sharper failure.
+ *
+ * A page whose *entire* token set narrows away (every one of its tokens
+ * clears the chrome cutoff) falls back to its own raw tokens rather than the
+ * empty result: `jaccardSimilarity` treats two empty sets as similarity `1`
+ * (by design, for two genuinely-empty `<body>`s — see its JSDoc), but two
+ * different* all-chrome pages narrowing to empty for unrelated reasons
+ * (e.g. one page is only a header+footer, another is only a nav) would
+ * otherwise be forced into the same cluster by that shortcut regardless of
+ * whether their actual structure matches. Falling back only when narrowing
+ * collapsed a page to nothing — not for every page — keeps the normal case
+ * (a page with at least one page-specific token) unaffected.
+ * @param tokenSets
+ */
+function deriveComparisonSets(tokenSets) {
+    if (tokenSets.length < MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT) {
+        return tokenSets;
+    }
+    const corpusFrequency = computeDocumentFrequency(tokenSets);
+    return tokenSets.map((tokens) => {
+        const { contentTokens } = splitTokensByFrequency(tokens, corpusFrequency);
+        return contentTokens.size === 0 && tokens.size > 0 ? tokens : contentTokens;
+    });
+}
+/**
+ * Reads `values[index]`, throwing instead of returning `undefined`. Every
+ * call site here indexes within bounds it just established itself (loop
+ * ranges, or an index freshly returned by the same array's own scan), so the
+ * thrown branch is unreachable in practice; it exists to satisfy
+ * `noUncheckedIndexedAccess` without a non-null assertion (same rationale as
+ * `readDpValue` in `array-edit-distance.ts`, generalized to any array-like).
+ * Deliberately not exported and shared with `resolve-page-cluster-keys.ts`'s
+ * own copy: this file's `export`s are its intended public API surface (the
+ * main function and its options type), and every other internal helper here
+ * (`find`, `clusterByCompleteLinkage`, `deriveComparisonSets`) is likewise
+ * kept unexported — sharing just this one helper across files would carve an
+ * exception into that boundary for a ~7-line generic utility.
+ * @param values
+ * @param index
+ */
+function requireIndex(values, index) {
+    const value = values[index];
+    if (value === undefined) {
+        throw new Error('resolveStructuralClusterKeys: index out of bounds');
+    }
+    return value;
+}
+/**
+ * Finds the representative (root) of `index`'s set, compressing every
+ * traversed link so future lookups on the same path are near-constant time.
+ * @param parent
+ * @param index
+ */
+function find(parent, index) {
+    let root = index;
+    while (requireIndex(parent, root) !== root) {
+        root = requireIndex(parent, root);
+    }
+    let current = index;
+    while (current !== root) {
+        const next = requireIndex(parent, current);
+        parent[current] = root;
+        current = next;
+    }
+    return root;
+}
+/**
+ * Complete-linkage hierarchical clustering of `tokenSets` (whatever sets the
+ * caller wants compared — `resolveStructuralClusterKeys` passes
+ * `deriveComparisonSets`'s output, not necessarily the raw per-page token
+ * sets), cut at `threshold`, computed via the NN-chain algorithm (Murtagh,
+ * F., 1983, "A
+ * Survey of Recent Advances in Hierarchical Clustering Algorithms," The
+ * Computer Journal 26(4)). NN-chain produces the exact same dendrogram as
+ * naively re-scanning every live cluster pair for the best merge at each
+ * step, but in O(n²) time instead of O(n³): each cluster follows a chain of
+ * mutually-improving nearest neighbors until it lands on a pair that are
+ * each other's nearest neighbor (a "reciprocal nearest neighbor", RNN); that
+ * pair's merge is provably a valid next step in the correct dendrogram. This
+ * is a genuine algorithmic speedup, not an approximation — see
+ * `resolveStructuralClusterKeys`'s JSDoc for why an approximation was
+ * rejected.
+ *
+ * Complete-linkage was chosen over single-linkage (connected components of
+ * the threshold graph) because single-linkage's "chaining" lets one
+ * unrepresentative page transitively merge two otherwise-unrelated
+ * templates — the opposite of what template detection needs. Complete-
+ * linkage requires *every* pair across two clusters to clear the threshold
+ * before merging them, which rules that out. Cluster-to-cluster similarity
+ * is maintained via the Lance-Williams update for complete-linkage:
+ * `similarity(merged, Z) = min(similarity(X, Z), similarity(Y, Z))`.
+ *
+ * The algorithm always runs every one of the `size - 1` possible merges to
+ * completion (down to a single root), never stopping early at `threshold`.
+ * This looks wasteful but isn't optional: Lance-Williams monotonicity
+ * (Lance, G. N. & Williams, W. T., 1967, "A General Theory of Classificatory
+ * Sorting Strategies," The Computer Journal 9(4)) guarantees no height
+ * inversions inside the dendrogram itself (a merge's similarity is always ≥
+ * the similarity of every merge nested inside it), but says nothing about
+ * the chronological order in which independent, not-yet-connected
+ * chains happen to resolve their own RNN pairs — one chain can easily
+ * stumble onto a low-similarity RNN pair before a different, still-unvisited
+ * chain uncovers a high-similarity one elsewhere. Stopping the whole
+ * algorithm at the first below-threshold merge would therefore discard
+ * later, still-valid above-threshold merges (confirmed by this file's
+ * differential test against a naive reference — an earlier version of this
+ * function that broke early on the first below-threshold RNN pair failed it
+ * for exactly this reason). Instead, every merge is always folded into the
+ * `active`/`similarity` bookkeeping so the algorithm can keep discovering
+ * the rest of the true dendrogram, but only merges scoring `>= threshold`
+ * are recorded in `parent` (the union-find used for final membership).
+ * Monotonicity guarantees this is safe: any merge scoring `>= threshold` was
+ * necessarily built out of children merges that scored at least as high, so
+ * restricting the union-find to threshold-clearing merges — regardless of
+ * the chronological order they were discovered in — reconstructs exactly
+ * the correct threshold cut.
+ * @param tokenSets
+ * @param threshold
+ */
+function clusterByCompleteLinkage(tokenSets, threshold) {
+    const size = tokenSets.length;
+    const parent = Int32Array.from({ length: size }, (_, index) => index);
+    const similarity = new Float64Array(size * size);
+    for (let i = 0; i < size; i++) {
+        for (let j = i + 1; j < size; j++) {
+            const score = jaccardSimilarity(requireIndex(tokenSets, i), requireIndex(tokenSets, j));
+            similarity[i * size + j] = score;
+            similarity[j * size + i] = score;
+        }
+    }
+    const active = new Uint8Array(size).fill(1);
+    const chain = [];
+    const findFreshStart = () => {
+        for (let index = 0; index < size; index++) {
+            if (requireIndex(active, index) === 1) {
+                return index;
+            }
+        }
+        throw new Error('resolveStructuralClusterKeys: no active cluster left to resume from');
+    };
+    let activeCount = size;
+    while (activeCount > 1) {
+        if (chain.length === 0) {
+            chain.push(findFreshStart());
+        }
+        const top = requireIndex(chain, chain.length - 1);
+        let best = -1;
+        let bestScore = Number.NEGATIVE_INFINITY;
+        for (let candidate = 0; candidate < size; candidate++) {
+            if (candidate !== top && requireIndex(active, candidate) === 1) {
+                const score = requireIndex(similarity, top * size + candidate);
+                if (score > bestScore) {
+                    bestScore = score;
+                    best = candidate;
+                }
+            }
+        }
+        const secondFromTop = chain.length >= 2 ? chain.at(-2) : undefined;
+        if (best === secondFromTop) {
+            chain.pop();
+            chain.pop();
+            const survivor = Math.min(top, best);
+            const dead = Math.max(top, best);
+            for (let candidate = 0; candidate < size; candidate++) {
+                if (candidate !== top &&
+                    candidate !== best &&
+                    requireIndex(active, candidate) === 1) {
+                    const merged = Math.min(requireIndex(similarity, top * size + candidate), requireIndex(similarity, best * size + candidate));
+                    similarity[survivor * size + candidate] = merged;
+                    similarity[candidate * size + survivor] = merged;
+                }
+            }
+            active[dead] = 0;
+            if (bestScore >= threshold - BOUNDARY_EPSILON) {
+                parent[find(parent, dead)] = find(parent, survivor);
+            }
+            activeCount--;
+        }
+        else {
+            chain.push(best);
+        }
+    }
+    return Array.from({ length: size }, (_, index) => find(parent, index));
+}
+/**
+ * Resolves, within a single already-blocked group of pages (e.g. one key
+ * from {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys}),
+ * which pages share a structural template. Returns one cluster key per
+ * page, in the same order as `tokenSets`. Does not call
+ * {@link ./tokenize.js | tokenize} itself (callers pass pages already
+ * tokenized and turned into `Set`s, mirroring
+ * {@link ./compute-document-frequency.js | computeDocumentFrequency}'s
+ * contract) and does not orchestrate multiple blocks — a heterogeneous
+ * corpus should be split into blocks by the caller before reaching this
+ * function.
+ *
+ * MinHash/LSH-based approximation and medoid-based refinement of these
+ * clusters are intentionally out of scope: NN-chain already computes the
+ * exact complete-linkage clustering in O(n²), so there is no accuracy being
+ * traded away by not approximating, and no evidence yet that O(n²) is a
+ * real bottleneck at the block sizes this function actually sees.
+ *
+ * Before comparing, each page's token set is narrowed to its page-specific
+ * content via `splitTokensByFrequency` (see `deriveComparisonSets`) once
+ * `tokenSets.length` is large enough for that to be statistically meaningful
+ * — below that floor, chrome dilution is accepted as the lesser failure and
+ * comparison falls back to the raw sets.
+ * @param tokenSets
+ * @param options
+ * @example
+ * ```ts
+ * resolveStructuralClusterKeys([
+ * 	new Set(['body>header', 'body>main>.card', 'body>footer']),
+ * 	new Set(['body>header', 'body>main>.card', 'body>footer']),
+ * 	new Set(['body>nav', 'body>main>form']),
+ * ]);
+ * // ['cluster:0', 'cluster:0', 'cluster:1']
+ * ```
+ */
+export function resolveStructuralClusterKeys(tokenSets, options) {
+    const similarityThreshold = options?.similarityThreshold ?? DEFAULT_SIMILARITY_THRESHOLD;
+    if (!(similarityThreshold >= 0 && similarityThreshold <= 1)) {
+        throw new RangeError(`resolveStructuralClusterKeys: similarityThreshold must be between 0 and 1, got ${similarityThreshold}`);
+    }
+    if (tokenSets.length === 0) {
+        return [];
+    }
+    const comparisonSets = deriveComparisonSets(tokenSets);
+    const roots = clusterByCompleteLinkage(comparisonSets, similarityThreshold);
+    const rootToLabel = new Map();
+    return roots.map((root) => {
+        let label = rootToLabel.get(root);
+        if (label === undefined) {
+            label = `cluster:${rootToLabel.size}`;
+            rootToLabel.set(root, label);
+        }
+        return label;
+    });
+}

package/dist/run-tokenizer.d.ts ADDED Viewed

@@ -0,0 +1,33 @@
+import type { ResolvedOptions, TokenizeResult } from './types.js';
+/**
+ * Drives `htmlparser2` over `html` and returns the ordered leaf paths found
+ * under the first `<body>`. Anything outside that first `<body>` is ignored
+ * (`<head>`, a second top-level `<body>`) — see `tokenize.ts` for why — and so
+ * is any stray `<body>` tag nested *inside* the first one's content, a known
+ * artifact of broken SSR/templating: browsers create no node for it, so its
+ * open/close tags are swallowed while its content still attaches to whatever
+ * real element actually contains it.
+ *
+ * WHY a stack of frames rather than emitting tokens the moment a tag opens:
+ * whether a class-less/role-less/type-less `div`/`span` folds away depends
+ * on its final child-element count, which is only settled once *it* closes
+ * — by which point any leaf inside it has already been visited. Buffering
+ * each open ancestor's not-yet-finalized descendant paths
+ * (`Frame.pendingPaths`) lets that decision be made retroactively without
+ * ever materializing the whole document as a tree: memory stays
+ * proportional to nesting depth and pending output, not document size.
+ *
+ * `htmlparser2`'s `script`/`style` handling never fires `onopentag`/
+ * `onclosetag` for content inside those tags (they're parsed as raw text),
+ * so only `svg`/`noscript` need active suppression of their nested
+ * open/close events; `depth` guards against those two self-nesting
+ * (`<svg><svg>...`).
+ *
+ * The `<body>` tag's own `class` is deliberately excluded when building its
+ * frame's `segment` (always plain `"body"`, never `"body.xxx"`) — see
+ * `TokenizeResult`'s JSDoc for why — and captured separately into
+ * `bodyClassList` instead of being discarded outright.
+ * @param html
+ * @param options
+ */
+export declare function runTokenizer(html: string, options: ResolvedOptions): TokenizeResult;

package/dist/run-tokenizer.js ADDED Viewed

@@ -0,0 +1,152 @@
+import { Parser } from 'htmlparser2';
+import { createFrame } from './create-frame.js';
+import { formatBracket } from './format-bracket.js';
+import { hashContent } from './hash-content.js';
+import { isOpaqueTagName } from './opaque-tags.js';
+import { parseClassList } from './parse-class-list.js';
+import { resolveClosedFrame } from './resolve-closed-frame.js';
+/**
+ * The innermost open frame. Every call site only reaches this after checking
+ * `stack.length > 0`, so the thrown branch is unreachable in practice; it
+ * exists to satisfy `noUncheckedIndexedAccess` without a non-null assertion.
+ * @param stack
+ */
+function topOf(stack) {
+    const frame = stack.at(-1);
+    if (!frame) {
+        throw new Error('runTokenizer: expected an open frame on the stack');
+    }
+    return frame;
+}
+/**
+ * Drives `htmlparser2` over `html` and returns the ordered leaf paths found
+ * under the first `<body>`. Anything outside that first `<body>` is ignored
+ * (`<head>`, a second top-level `<body>`) — see `tokenize.ts` for why — and so
+ * is any stray `<body>` tag nested *inside* the first one's content, a known
+ * artifact of broken SSR/templating: browsers create no node for it, so its
+ * open/close tags are swallowed while its content still attaches to whatever
+ * real element actually contains it.
+ *
+ * WHY a stack of frames rather than emitting tokens the moment a tag opens:
+ * whether a class-less/role-less/type-less `div`/`span` folds away depends
+ * on its final child-element count, which is only settled once *it* closes
+ * — by which point any leaf inside it has already been visited. Buffering
+ * each open ancestor's not-yet-finalized descendant paths
+ * (`Frame.pendingPaths`) lets that decision be made retroactively without
+ * ever materializing the whole document as a tree: memory stays
+ * proportional to nesting depth and pending output, not document size.
+ *
+ * `htmlparser2`'s `script`/`style` handling never fires `onopentag`/
+ * `onclosetag` for content inside those tags (they're parsed as raw text),
+ * so only `svg`/`noscript` need active suppression of their nested
+ * open/close events; `depth` guards against those two self-nesting
+ * (`<svg><svg>...`).
+ *
+ * The `<body>` tag's own `class` is deliberately excluded when building its
+ * frame's `segment` (always plain `"body"`, never `"body.xxx"`) — see
+ * `TokenizeResult`'s JSDoc for why — and captured separately into
+ * `bodyClassList` instead of being discarded outright.
+ * @param html
+ * @param options
+ */
+export function runTokenizer(html, options) {
+    const stack = [];
+    let opaque = null;
+    let bodyDone = false;
+    let result = [];
+    let bodyClassList = [];
+    // Counts <body> open tags ignored because a body was already open (a
+    // stray/duplicated body from broken SSR/templating). Browsers create no
+    // node for these, so neither the open nor its matching close tag should
+    // touch the frame stack; this counter lets onclosetag recognize and
+    // swallow that matching close instead of popping an unrelated frame.
+    let ignoredBodyOpens = 0;
+    const parser = new Parser({
+        onopentag(name, attribs) {
+            if (opaque) {
+                if (name === opaque.tagName) {
+                    opaque.depth++;
+                }
+                return;
+            }
+            if (stack.length === 0) {
+                if (name === 'body' && !bodyDone) {
+                    bodyClassList = parseClassList(attribs.class, options.filterNoiseClasses);
+                    stack.push(createFrame(name, { ...attribs, class: '' }, options));
+                }
+                // Ignore everything else outside <body> (head, a second top-level <html>/<body>, ...).
+                return;
+            }
+            if (name === 'body') {
+                ignoredBodyOpens++;
+                return;
+            }
+            topOf(stack).childElementCount++;
+            if (isOpaqueTagName(name)) {
+                opaque = {
+                    tagName: name,
+                    depth: 1,
+                    contentStart: parser.endIndex + 1,
+                    role: attribs.role || undefined,
+                    type: attribs.type || undefined,
+                };
+                return;
+            }
+            stack.push(createFrame(name, attribs, options));
+        },
+        onclosetag(name) {
+            if (opaque) {
+                if (name === opaque.tagName) {
+                    opaque.depth--;
+                    if (opaque.depth === 0) {
+                        const raw = html.slice(opaque.contentStart, parser.startIndex);
+                        const bracket = formatBracket({
+                            role: opaque.role,
+                            type: opaque.type,
+                            sha: hashContent(raw),
+                        });
+                        topOf(stack).pendingPaths.push(`${name}${bracket}`);
+                        opaque = null;
+                    }
+                }
+                return;
+            }
+            if (name === 'body' && ignoredBodyOpens > 0) {
+                ignoredBodyOpens--;
+                return;
+            }
+            if (stack.length === 0) {
+                // Closing tag outside <body> (or a second top-level </body>): nothing to do.
+                return;
+            }
+            const frame = stack.pop();
+            if (!frame) {
+                return;
+            }
+            const contributed = resolveClosedFrame(frame);
+            if (stack.length === 0) {
+                result = contributed;
+                bodyDone = true;
+            }
+            else {
+                // Not `push(...contributed)`: spreading tens of thousands of
+                // arguments into a single call (a realistic count for a flat,
+                // high-fan-out template like a sitemap/listing page — exactly
+                // the shape this package targets) throws `RangeError: Maximum
+                // call stack size exceeded`. A plain loop has no such limit.
+                const target = topOf(stack).pendingPaths;
+                for (const path of contributed) {
+                    target.push(path);
+                }
+            }
+        },
+        oncomment(data) {
+            if (!options.includeComments || opaque || stack.length === 0) {
+                return;
+            }
+            topOf(stack).pendingPaths.push(`comment[sha=${hashContent(data)}]`);
+        },
+    });
+    parser.parseComplete(html);
+    return { tokens: result, bodyClassList };
+}

package/dist/split-tokens-by-frequency.d.ts ADDED Viewed

@@ -0,0 +1,46 @@
+import type { DocumentFrequency } from './types.js';
+/**
+ * Splits one page's tokens into "template" (site chrome: header/nav/footer,
+ * or any other structure repeated across most of the corpus) and "content"
+ * (page-specific structure), using each token's document frequency from
+ * `computeDocumentFrequency`. Comparing these two groups separately with
+ * `jaccardSimilarity()` — rather than the page's full token set at once —
+ * is what fixes two failures a single flat Jaccard has: common chrome
+ * diluting genuine content differences at loose similarity thresholds, and
+ * page-specific content differences (e.g. a freeform CMS block editor page,
+ * where the exact block mix varies per page) swamping a real *layout*
+ * match. See `computeDocumentFrequency`'s JSDoc for why `corpusFrequency`
+ * must come from a homogeneous page collection for this split to work.
+ *
+ * `corpusFrequency` bundles `documentFrequency` with the `pageCount` it was
+ * computed from (rather than taking `pageCount` as a separate argument) so
+ * the two can never be passed out of sync — e.g. a caller re-slicing or
+ * filtering the page list after computing frequencies but before using
+ * them, which would otherwise silently produce a wrong cutoff with no error
+ * raised anywhere.
+ *
+ * A token absent from `documentFrequency` is treated as frequency 0 (i.e.
+ * content): it never appeared in the corpus the frequency map was built
+ * from, so it cannot be corpus-wide chrome. If `pageCount` is 0 (empty
+ * corpus), every token is classified as content for the same reason: with
+ * no pages to have observed repetition across, nothing can be confirmed as
+ * chrome.
+ *
+ * `threshold` must be a fraction in `(0, 1]`, not a percentage — passing
+ * `90` instead of `0.9` would make the cutoff exceed every possible
+ * frequency and misclassify even universal chrome as content, so this is
+ * validated eagerly rather than left to fail silently downstream.
+ * @param tokens
+ * @param corpusFrequency
+ * @param threshold
+ * @example
+ * ```ts
+ * const corpusFrequency = computeDocumentFrequency(allPagesTokenSets);
+ * splitTokensByFrequency(pageTokens, corpusFrequency);
+ * // { templateTokens: Set(...), contentTokens: Set(...) }
+ * ```
+ */
+export declare function splitTokensByFrequency(tokens: ReadonlySet<string>, corpusFrequency: DocumentFrequency, threshold?: number): {
+    templateTokens: Set<string>;
+    contentTokens: Set<string>;
+};

package/dist/split-tokens-by-frequency.js ADDED Viewed

@@ -0,0 +1,88 @@
+/**
+ * Default document-frequency cutoff: a token present in at least 90% of the
+ * pages passed to `computeDocumentFrequency` is treated as site chrome.
+ * Real-data validation against a small single-layout corporate site (a few
+ * hundred pages) found the corpus's tokens cleanly bimodal — the same set
+ * of chrome tokens was identified whether the cutoff was set anywhere from
+ * 50% to 95% — so the exact value is not sensitive within that range for a
+ * homogeneous corpus. 90% was picked as a value comfortably inside that
+ * stable range rather than at either edge.
+ */
+const DEFAULT_TEMPLATE_FREQUENCY_THRESHOLD = 0.9;
+/**
+ * `threshold * pageCount` is a floating-point product and can overshoot the
+ * intended integer boundary (verified: `0.55 * 100 === 55.00000000000001`
+ * in JS), which would otherwise make a token at the exact documented
+ * inclusive boundary (`frequency === threshold * pageCount`) fail the
+ * `>=` check it should pass. Subtracting this epsilon before comparing
+ * absorbs that rounding noise without being large enough to affect any
+ * genuinely-below-threshold token (frequencies are integers, so the true
+ * gap between "at the boundary" and "one below it" is always >= 1).
+ */
+const BOUNDARY_EPSILON = 1e-9;
+/**
+ * Splits one page's tokens into "template" (site chrome: header/nav/footer,
+ * or any other structure repeated across most of the corpus) and "content"
+ * (page-specific structure), using each token's document frequency from
+ * `computeDocumentFrequency`. Comparing these two groups separately with
+ * `jaccardSimilarity()` — rather than the page's full token set at once —
+ * is what fixes two failures a single flat Jaccard has: common chrome
+ * diluting genuine content differences at loose similarity thresholds, and
+ * page-specific content differences (e.g. a freeform CMS block editor page,
+ * where the exact block mix varies per page) swamping a real *layout*
+ * match. See `computeDocumentFrequency`'s JSDoc for why `corpusFrequency`
+ * must come from a homogeneous page collection for this split to work.
+ *
+ * `corpusFrequency` bundles `documentFrequency` with the `pageCount` it was
+ * computed from (rather than taking `pageCount` as a separate argument) so
+ * the two can never be passed out of sync — e.g. a caller re-slicing or
+ * filtering the page list after computing frequencies but before using
+ * them, which would otherwise silently produce a wrong cutoff with no error
+ * raised anywhere.
+ *
+ * A token absent from `documentFrequency` is treated as frequency 0 (i.e.
+ * content): it never appeared in the corpus the frequency map was built
+ * from, so it cannot be corpus-wide chrome. If `pageCount` is 0 (empty
+ * corpus), every token is classified as content for the same reason: with
+ * no pages to have observed repetition across, nothing can be confirmed as
+ * chrome.
+ *
+ * `threshold` must be a fraction in `(0, 1]`, not a percentage — passing
+ * `90` instead of `0.9` would make the cutoff exceed every possible
+ * frequency and misclassify even universal chrome as content, so this is
+ * validated eagerly rather than left to fail silently downstream.
+ * @param tokens
+ * @param corpusFrequency
+ * @param threshold
+ * @example
+ * ```ts
+ * const corpusFrequency = computeDocumentFrequency(allPagesTokenSets);
+ * splitTokensByFrequency(pageTokens, corpusFrequency);
+ * // { templateTokens: Set(...), contentTokens: Set(...) }
+ * ```
+ */
+export function splitTokensByFrequency(tokens, corpusFrequency, threshold = DEFAULT_TEMPLATE_FREQUENCY_THRESHOLD) {
+    if (!(threshold > 0 && threshold <= 1)) {
+        throw new RangeError(`splitTokensByFrequency: threshold must be a fraction in (0, 1], got ${threshold}`);
+    }
+    const { documentFrequency, pageCount } = corpusFrequency;
+    const templateTokens = new Set();
+    const contentTokens = new Set();
+    if (pageCount === 0) {
+        for (const token of tokens) {
+            contentTokens.add(token);
+        }
+        return { templateTokens, contentTokens };
+    }
+    const cutoff = threshold * pageCount - BOUNDARY_EPSILON;
+    for (const token of tokens) {
+        const frequency = documentFrequency.get(token) ?? 0;
+        if (frequency >= cutoff) {
+            templateTokens.add(token);
+        }
+        else {
+            contentTokens.add(token);
+        }
+    }
+    return { templateTokens, contentTokens };
+}

package/dist/tokenize.d.ts ADDED Viewed

@@ -0,0 +1,58 @@
+import type { TokenizeOptions, TokenizeResult } from './types.js';
+export type { TokenizeOptions, TokenizeResult } from './types.js';
+/**
+ * Tokenizes the structural skeleton of an HTML document's `<body>` for
+ * duplicate/near-duplicate page detection at crawl scale. This is the first
+ * building block of `@d-zero/page-cluster`; clustering on top of these
+ * tokens is layered on by
+ * {@link ./resolve-structural-cluster-keys.js | resolveStructuralClusterKeys}
+ * (exact O(n²) complete-linkage via NN-chain) and orchestrated across blocks
+ * by {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys} —
+ * MinHash/LSH-based approximation was considered and rejected: see
+ * `resolveStructuralClusterKeys`'s JSDoc for why.
+ *
+ * Only `<body>` is tokenized. `<head>` (title/meta/link/OGP/...) is ignored
+ * entirely: `@d-zero/beholder` already extracts it comprehensively, but from
+ * a live `Document` (Puppeteer/jsdom) rather than a raw HTML string, so that
+ * logic can't be reused here without building a DOM — exactly what this
+ * function avoids for speed. Callers who need head metadata should call
+ * `@d-zero/beholder` separately.
+ *
+ * Visible text is discarded entirely: this function measures *structural*
+ * similarity, and including per-page text would make otherwise-identical
+ * templates look unique.
+ *
+ * The returned array intentionally does not deduplicate or
+ * run-length-compress repeated paths — even though the eventual consumer
+ * (a MinHash/LSH classifier) will reduce this array to a `Set` for
+ * comparison, and a `Set` alone already collapses any number of repeated
+ * entries. Compressing here first (e.g. `"li>a*3"`) would embed the
+ * arrangement of neighboring siblings into the token string itself: a
+ * `current`/`active`-style state class on exactly one sibling (its position
+ * varies per page, e.g. which nav item is "current") shifts which runs are
+ * adjacent, so the same template could serialize as `"li>a*2"` on one page
+ * and as two separate `"li>a"` entries (split by the state-bearing sibling)
+ * on another — literally different strings for what should compare equal
+ * once turned into a `Set`. Leaving the array uncompressed sidesteps that
+ * entirely:
+ * `Set(["li>a", "li.current>a", "li>a"])` and
+ * `Set(["li.current>a", "li>a", "li>a"])` are the same two-element set no
+ * matter where the state class lands. If a future consumer needs a shorter
+ * array for e.g. array-edit-distance comparisons on pathologically large
+ * pages, that consumer should apply its own compression tuned to its own
+ * needs, since compression is coupled to how the caller will read counts
+ * back out again — folding that guess into this package's contract can't be
+ * un-shipped later.
+ *
+ * `<body>`'s own `class` is excluded from every leaf path and returned
+ * separately as `bodyClassList` — see {@link ./types.js | TokenizeResult}'s
+ * JSDoc for why.
+ * @param html
+ * @param options
+ * @example
+ * ```ts
+ * tokenize('<body><div class="card"><ul><li>A</li><li>B</li></ul></div></body>');
+ * // { tokens: ["body>.card>ul>li", "body>.card>ul>li"], bodyClassList: [] }
+ * ```
+ */
+export declare function tokenize(html: string, options?: TokenizeOptions): TokenizeResult;