@d-zero/page-cluster 0.2.0 → 0.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@d-zero/page-cluster",
-	"version": "0.2.0",
+	"version": "0.3.0",
 	"description": "Tokenizes an HTML document's body into a structural signature for duplicate/near-duplicate page detection at crawl scale",
 	"author": "D-ZERO",
 	"license": "MIT",
@@ -13,58 +13,10 @@
 			"import": "./dist/tokenize.js",
 			"types": "./dist/tokenize.d.ts"
 		},
-		"./array-edit-distance": {
-			"import": "./dist/array-edit-distance.js",
-			"types": "./dist/array-edit-distance.d.ts"
-		},
-		"./cap-content-depth": {
-			"import": "./dist/cap-content-depth.js",
-			"types": "./dist/cap-content-depth.d.ts"
-		},
-		"./compute-document-frequency": {
-			"import": "./dist/compute-document-frequency.js",
-			"types": "./dist/compute-document-frequency.d.ts"
-		},
-		"./derive-path-group-key": {
-			"import": "./dist/derive-path-group-key.js",
-			"types": "./dist/derive-path-group-key.d.ts"
-		},
-		"./derive-stylesheet-group-key": {
-			"import": "./dist/derive-stylesheet-group-key.js",
-			"types": "./dist/derive-stylesheet-group-key.d.ts"
-		},
-		"./detect-content-depth-cap": {
-			"import": "./dist/detect-content-depth-cap.js",
-			"types": "./dist/detect-content-depth-cap.d.ts"
-		},
 		"./extract-landmarks": {
 			"import": "./dist/extract-landmarks.js",
 			"types": "./dist/extract-landmarks.d.ts"
 		},
-		"./filter-first-party-stylesheet-hrefs": {
-			"import": "./dist/filter-first-party-stylesheet-hrefs.js",
-			"types": "./dist/filter-first-party-stylesheet-hrefs.d.ts"
-		},
-		"./jaccard-similarity": {
-			"import": "./dist/jaccard-similarity.js",
-			"types": "./dist/jaccard-similarity.d.ts"
-		},
-		"./merge-landmark-affined-clusters": {
-			"import": "./dist/merge-landmark-affined-clusters.js",
-			"types": "./dist/merge-landmark-affined-clusters.d.ts"
-		},
-		"./reassign-orphan-block-keys": {
-			"import": "./dist/reassign-orphan-block-keys.js",
-			"types": "./dist/reassign-orphan-block-keys.d.ts"
-		},
-		"./remove-content-blocks": {
-			"import": "./dist/remove-content-blocks.js",
-			"types": "./dist/remove-content-blocks.d.ts"
-		},
-		"./resolve-blocking-group-keys": {
-			"import": "./dist/resolve-blocking-group-keys.js",
-			"types": "./dist/resolve-blocking-group-keys.d.ts"
-		},
 		"./resolve-landmark-variant-keys": {
 			"import": "./dist/resolve-landmark-variant-keys.js",
 			"types": "./dist/resolve-landmark-variant-keys.d.ts"
@@ -72,16 +24,9 @@
 		"./resolve-page-cluster-keys": {
 			"import": "./dist/resolve-page-cluster-keys.js",
 			"types": "./dist/resolve-page-cluster-keys.d.ts"
-		},
-		"./resolve-structural-cluster-keys": {
-			"import": "./dist/resolve-structural-cluster-keys.js",
-			"types": "./dist/resolve-structural-cluster-keys.d.ts"
-		},
-		"./split-tokens-by-frequency": {
-			"import": "./dist/split-tokens-by-frequency.js",
-			"types": "./dist/split-tokens-by-frequency.d.ts"
 		}
 	},
+	"bin": "./dist/cli.js",
 	"files": [
 		"dist"
 	],
@@ -91,6 +36,7 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
+		"@d-zero/dealer": "1.10.0",
 		"@d-zero/shared": "0.22.2",
 		"htmlparser2": "12.0.0"
 	},
@@ -98,5 +44,6 @@
 		"type": "git",
 		"url": "https://github.com/d-zero-dev/tools.git",
 		"directory": "packages/@d-zero/page-cluster"
-	}
+	},
+	"gitHead": "5429d79837cca262ae5b74b32922eae6f7ef2649"
 }

package/dist/html-region-utils.d.ts

@@ -1,74 +0,0 @@
-/**
- * Shared by {@link ./extract-landmarks.js | extractLandmarks} and
- * {@link ./remove-content-blocks.js | removeContentBlocks}: both walk `html`
- * with htmlparser2 to find `[start, end)` spans of matched regions, then
- * excise them from the original string. Not part of either file's own
- * public API surface (unlike the smaller, ~7-line `requireIndex` copies
- * duplicated elsewhere in this package), so it's factored out here instead
- * of duplicated: this logic already had one real bug (see
- * `escapeRegExp`/`isGenuineClose` below) found and fixed after
- * `extractLandmarks` shipped, and a second independent copy would be a
- * second place for that class of bug to reappear.
- */
-/**
- * Escapes regex metacharacters in `text` so it can be interpolated into a
- * `RegExp` literally. Needed because a tag name reaching {@link isGenuineClose}
- * is not guaranteed to be a plain HTML tag name: htmlparser2 accepts
- * characters like `(`/`[` inside a tag name (`<div(foo role="banner">`
- * parses with tag name `"div(foo"`), which would otherwise either throw
- * (an unbalanced `(` is an invalid regex) or silently change what the regex
- * matches.
- * @param text
- */
-export declare function escapeRegExp(text: string): string;
-/**
- * Whether `html` actually contains a literal closing tag for `tagName`
- * ending at `endOffset`. htmlparser2 fires `onclosetag` not only for real
- * closing tags but also when it force-closes a still-open ancestor to
- * resolve a mismatch (e.g. `<header>H<main>...</main></body>` with no
- * `</header>` ever written) — and in that forced case it reports the
- * force-closed element's `endIndex` as wherever the *other*, unrelated
- * closing tag that triggered the cascade happens to sit, not any position
- * derived from the matched element itself (confirmed by direct htmlparser2
- * event tracing: both the synthetic close and the real `body` close report
- * the identical `endIndex`, because there is no real closing tag in the
- * source for htmlparser2 to anchor a distinct position to). Trusting that
- * offset would slice a candidate spanning all the way to wherever the
- * unrelated tag ends, silently swallowing real content into the caller's
- * remainder HTML. Checking that the text immediately preceding `endOffset`
- * actually spells the expected closing tag catches exactly this: a genuine
- * close always ends with it; a forced one ends with whatever unrelated tag
- * forced it instead.
- * @param html
- * @param endOffset
- * @param tagName
- */
-export declare function isGenuineClose(html: string, endOffset: number, tagName: string): boolean;
-/**
- * Merges a set of (possibly overlapping or nested) `[start, end)` spans into
- * the smallest equivalent set of disjoint spans, sorted by start offset.
- * Matched spans commonly nest in real markup (e.g. a site nav living inside
- * the header, `<header><nav>...</nav></header>`) — merging first means the
- * later excision pass never has to reason about overlap.
- * @param spans
- */
-export declare function mergeSpans(spans: readonly {
-    start: number;
-    end: number;
-}[]): {
-    start: number;
-    end: number;
-}[];
-/**
- * Excises `spans` (merged via {@link mergeSpans}) from `html`, returning
- * what's left. No placeholder is left in a span's place: a placeholder
- * string would itself become a token once the remainder is tokenized,
- * reintroducing exactly the kind of synthetic signal callers of this
- * function exist to remove.
- * @param html
- * @param spans
- */
-export declare function excise(html: string, spans: readonly {
-    start: number;
-    end: number;
-}[]): string;

package/dist/html-region-utils.js

@@ -1,96 +0,0 @@
-/**
- * Shared by {@link ./extract-landmarks.js | extractLandmarks} and
- * {@link ./remove-content-blocks.js | removeContentBlocks}: both walk `html`
- * with htmlparser2 to find `[start, end)` spans of matched regions, then
- * excise them from the original string. Not part of either file's own
- * public API surface (unlike the smaller, ~7-line `requireIndex` copies
- * duplicated elsewhere in this package), so it's factored out here instead
- * of duplicated: this logic already had one real bug (see
- * `escapeRegExp`/`isGenuineClose` below) found and fixed after
- * `extractLandmarks` shipped, and a second independent copy would be a
- * second place for that class of bug to reappear.
- */
-/**
- * Escapes regex metacharacters in `text` so it can be interpolated into a
- * `RegExp` literally. Needed because a tag name reaching {@link isGenuineClose}
- * is not guaranteed to be a plain HTML tag name: htmlparser2 accepts
- * characters like `(`/`[` inside a tag name (`<div(foo role="banner">`
- * parses with tag name `"div(foo"`), which would otherwise either throw
- * (an unbalanced `(` is an invalid regex) or silently change what the regex
- * matches.
- * @param text
- */
-export function escapeRegExp(text) {
-    return text.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&');
-}
-/**
- * Whether `html` actually contains a literal closing tag for `tagName`
- * ending at `endOffset`. htmlparser2 fires `onclosetag` not only for real
- * closing tags but also when it force-closes a still-open ancestor to
- * resolve a mismatch (e.g. `<header>H<main>...</main></body>` with no
- * `</header>` ever written) — and in that forced case it reports the
- * force-closed element's `endIndex` as wherever the *other*, unrelated
- * closing tag that triggered the cascade happens to sit, not any position
- * derived from the matched element itself (confirmed by direct htmlparser2
- * event tracing: both the synthetic close and the real `body` close report
- * the identical `endIndex`, because there is no real closing tag in the
- * source for htmlparser2 to anchor a distinct position to). Trusting that
- * offset would slice a candidate spanning all the way to wherever the
- * unrelated tag ends, silently swallowing real content into the caller's
- * remainder HTML. Checking that the text immediately preceding `endOffset`
- * actually spells the expected closing tag catches exactly this: a genuine
- * close always ends with it; a forced one ends with whatever unrelated tag
- * forced it instead.
- * @param html
- * @param endOffset
- * @param tagName
- */
-export function isGenuineClose(html, endOffset, tagName) {
-    const windowStart = Math.max(0, endOffset - tagName.length - 3);
-    return new RegExp(`</\\s*${escapeRegExp(tagName)}\\s*>$`, 'i').test(html.slice(windowStart, endOffset));
-}
-/**
- * Merges a set of (possibly overlapping or nested) `[start, end)` spans into
- * the smallest equivalent set of disjoint spans, sorted by start offset.
- * Matched spans commonly nest in real markup (e.g. a site nav living inside
- * the header, `<header><nav>...</nav></header>`) — merging first means the
- * later excision pass never has to reason about overlap.
- * @param spans
- */
-export function mergeSpans(spans) {
-    const sorted = [...spans].toSorted((a, b) => a.start - b.start);
-    const merged = [];
-    for (const span of sorted) {
-        const last = merged.at(-1);
-        if (last && span.start <= last.end) {
-            last.end = Math.max(last.end, span.end);
-        }
-        else {
-            merged.push({ ...span });
-        }
-    }
-    return merged;
-}
-/**
- * Excises `spans` (merged via {@link mergeSpans}) from `html`, returning
- * what's left. No placeholder is left in a span's place: a placeholder
- * string would itself become a token once the remainder is tokenized,
- * reintroducing exactly the kind of synthetic signal callers of this
- * function exist to remove.
- * @param html
- * @param spans
- */
-export function excise(html, spans) {
-    if (spans.length === 0) {
-        return html;
-    }
-    const merged = mergeSpans(spans);
-    let remainder = '';
-    let cursor = 0;
-    for (const span of merged) {
-        remainder += html.slice(cursor, span.start);
-        cursor = span.end;
-    }
-    remainder += html.slice(cursor);
-    return remainder;
-}

package/dist/merge-landmark-affined-clusters.d.ts

@@ -1,179 +0,0 @@
-import type { ExtractLandmarksResult } from './extract-landmarks.js';
-import type { TokenizeOptions } from './types.js';
-/**
- * @see mergeLandmarkAffinedClusters
- */
-export type MergeLandmarkAffinedClustersOptions = TokenizeOptions & {
-    /**
-     * Reused as the landmark-variant identity threshold inside
-     * `computeLandmarkStatus`. Must be in `[0, 1]` (`RangeError` otherwise).
-     * Defaults to `0.8`, matching `resolveStructuralClusterKeys`'s own
-     * default — this file never calls that function, but the two thresholds
-     * represent the same concept ("how much token overlap counts as the same
-     * design").
-     *
-     * Deliberately not a separate, independent option: when
-     * `resolvePageClusterKeys` forwards its caller's single `options` object
-     * to both `resolveStructuralClusterKeys` (primary content clustering) and
-     * this file (landmark-variant identity), the same `similarityThreshold`
-     * value drives both, so loosening one to re-tune primary clustering (the
-     * documented `excludeLandmarks`/`similarityThreshold` interaction on
-     * `resolvePageClusterKeys`) also loosens landmark-variant matching. This
-     * mirrors `resolve-landmark-variant-keys.ts`'s own precedent
-     * (`resolveLandmarkVariantKeys` likewise forwards its caller's
-     * `options.similarityThreshold` straight into
-     * `resolveStructuralClusterKeys` with no landmark-specific override) and
-     * keeps the option surface to the three fields this file actually adds.
-     * Accepted as a known trade-off rather than split into its own knob until
-     * real-corpus tuning shows the two thresholds genuinely need to diverge —
-     * the same "starting heuristic, not yet corpus-validated" status this
-     * option's own default already carries.
-     */
-    similarityThreshold?: number;
-    /**
-     * Upper bound, as a fraction of the whole corpus (`[0, 1]`), on how many
-     * pages may share a given (landmark type, variant) pair before that
-     * variant is considered too common to serve as evidence of a genuine
-     * template affinity. Strictly `<` this fraction ("rare", not
-     * "rare-or-equal"). `RangeError` outside `[0, 1]`. Defaults to `0.05` —
-     * an unvalidated starting heuristic, the same status as
-     * `similarityThreshold`'s own `0.8` default (see this file's JSDoc for
-     * why real-corpus validation is out of scope for this change).
-     */
-    landmarkRarityThreshold?: number;
-    /**
-     * The secondary, looser complete-linkage content-similarity threshold
-     * applied only to pages whose landmark signature already qualifies (see
-     * `mergeLandmarkAffinedClusters`'s JSDoc). `RangeError` outside `[0, 1]`.
-     * Defaults to `0.6` — the value a withdrawn earlier prototype of this
-     * same mechanism proposed, and also the value
-     * `resolvePageClusterKeys`'s own `excludeLandmarks` JSDoc cites as having
-     * correctly re-merged a real 3-page block once landmarks were excluded
-     * and the raw-token `similarityThreshold` (`0.8`) became too strict.
-     */
-    landmarkGateSimilarityThreshold?: number;
-};
-/**
- * Validates `similarityThreshold`/`landmarkRarityThreshold`/
- * `landmarkGateSimilarityThreshold` without running
- * `mergeLandmarkAffinedClusters` itself — exported so
- * `resolvePageClusterKeys` can fail fast on bad options even when `pages` is
- * empty (its own per-block loop never reaches this function at all in that
- * case). Mirrors `detect-content-depth-cap.ts`'s
- * `validateDetectContentDepthCapOptions` exact rationale and shape.
- * @param options
- * @example
- * ```ts
- * // Fails fast on a bad option even though nothing here would otherwise
- * // call mergeLandmarkAffinedClusters yet (e.g. cluster keys haven't been
- * // computed).
- * validateMergeLandmarkAffinedClustersOptions({ landmarkRarityThreshold: -1 }); // throws RangeError
- * ```
- */
-export declare function validateMergeLandmarkAffinedClustersOptions(options?: MergeLandmarkAffinedClustersOptions): void;
-/**
- * Re-keys the pages of two or more distinct
- * {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys} clusters
- * onto one shared key when every landmark type present on their pages is
- * both *identical* and *rare* corpus-wide, and their actual content clears a
- * secondary, looser similarity threshold.
- *
- * Reimplements a mechanism previously prototyped under this same name and
- * withdrawn (no trace survives in commit history — this JSDoc is the only
- * record). The withdrawn version merged clusters whenever their
- * header/footer/nav/aside matched, full stop. Validated against two real
- * crawl corpora (302 and 8,936 pages), that produced runaway over-merging:
- * header/footer/nav were present on 99%+ of pages and typically reused
- * site-wide unchanged (see `extractLandmarks`'s own JSDoc for that figure),
- * so "landmarks match" was true for nearly every page pair and carried no
- * discriminative power at all. This reimplementation only ever treats a
- * landmark match as merge evidence when that specific landmark *variant* is
- * itself uncommon corpus-wide (`landmarkRarityThreshold`) — the condition
- * the withdrawn attempt lacked.
- *
- * The match requirement is deliberately the most conservative option
- * considered: *every* landmark type actually present on a page must both
- * match its counterpart's variant and be rare — a page with even one common
- * ("everybody has this exact header") present type contributes no evidence
- * at all, rather than partially qualifying. A looser rule (e.g. "at least
- * one shared rare type is enough") was rejected because it reintroduces a
- * version of the original failure mode: a page could ride a single
- * incidentally-rare landmark into a merge despite otherwise-ordinary,
- * ubiquitous chrome elsewhere on the same page.
- *
- * Frequency is counted corpus-wide, not per-block: a `resolveStructuralClusterKeys`
- * cluster label (`cluster:N`) is only unique within the block it was computed
- * in, but rarity here needs one consistent count across the whole input, the
- * same reason `resolvePageClusterKeys` itself composes `[blockKey,
- * localLabel]` via `JSON.stringify` rather than reusing bare labels across
- * blocks.
- *
- * A page with none of the four landmark types present is excluded from
- * consideration entirely (`existingCount === 0` below) — without this, every
- * landmark-less page across the whole corpus would share one large,
- * unbounded "no landmarks" group, defeating the self-limiting cost bound
- * `landmarkRarityThreshold` is otherwise supposed to guarantee (see
- * `computeLandmarkStatus`'s JSDoc for the cost analysis this depends on).
- *
- * Once pages are grouped by matching-and-rare landmark signature, only
- * signature groups spanning two or more distinct existing cluster keys do
- * any further work. Within such a group, the *content* token sets of the
- * group's distinct cluster keys are complete-linkage-merged at
- * `landmarkGateSimilarityThreshold` — looser than
- * `resolveStructuralClusterKeys`'s own `similarityThreshold`, since the
- * whole point of this mechanism is to bridge clusters whose *content*
- * similarity alone fell just short of the primary threshold. Complete-linkage
- * (not single-linkage) is used for the same reason
- * `resolveStructuralClusterKeys` itself uses it: single-linkage's chaining
- * would let one loosely-matching pair bridge two genuinely-unrelated
- * clusters transitively.
- *
- * The resulting merge is applied at *page* granularity, not by blanket-
- * reassigning every page of the involved cluster keys: only the specific
- * pages that were actually pooled into the qualifying signature group (and,
- * transitively, any other page unioned with them via a different signature
- * group) move onto the shared key. A cluster's pages that never carried the
- * rare landmark evidence keep their original key untouched, even if some
- * other page sharing that same cluster key did qualify and merge elsewhere.
- * This is deliberate, not an incidental restriction: applying a merge
- * decision to *every* page of the involved cluster keys — evidenced by only
- * a small subset of them — would extrapolate a coincidental pairing (e.g.
- * one outlier page in each of two otherwise-unrelated clusters happening to
- * share a rare seasonal-campaign header) into force-merging the clusters'
- * entire, otherwise-dissimilar membership. That is the withdrawn prototype's
- * over-merging failure mode reappearing through a different mechanism
- * (whole-cluster application of a single-pair signal) rather than the
- * landmark-commonality mechanism this file was reimplemented to fix — see
- * this function's own regression test for a worked example.
- *
- * Merged pages are re-keyed to `landmark-merge:${JSON.stringify(sortedKeys)}`
- * (`sortedKeys` being the *original* cluster keys the merged pages came
- * from) — a fresh prefix that cannot collide with `css:`/`path:`/
- * `orphan-merge:` or `resolvePageClusterKeys`'s own `[blockKey, "cluster:N"]`
- * pairs (mirrors `reassign-orphan-block-keys.ts`'s `orphan-merge:` prefix).
- * @param clusterKeys - one existing final key per page, same order/length as `landmarks`/`contentTokenSets`
- * @param landmarks - `extractLandmarks(page.html)`'s full result per page (all four fields, not just `remainderHtml`)
- * @param contentTokenSets - per-page content token sets to use for the secondary similarity gate. Should be independent of whichever landmark markup qualified the page as evidence (e.g. always landmark-excised), so this gate is a genuine second signal rather than re-counting the same landmark tokens already used to select the page — see `resolvePageClusterKeys`'s own call site for how it builds these
- * @param options
- * @example
- * ```ts
- * // tokenize() discards visible text (see its own JSDoc), so the two
- * // header variants below must differ structurally (child element/class),
- * // not merely in text, to compare as different landmark variants.
- * mergeLandmarkAffinedClusters(
- * 	['["css:a", "cluster:0"]', '["css:b", "cluster:0"]', 'path:other'],
- * 	[
- * 		{ header: '<header><i class="mark-a"></i></header>', remainderHtml: '' },
- * 		{ header: '<header><i class="mark-a"></i></header>', remainderHtml: '' },
- * 		{ header: '<header><b class="mark-b"></b></header>', remainderHtml: '' },
- * 	],
- * 	[new Set(['a', 'b']), new Set(['a', 'c']), new Set(['z'])],
- * 	{ landmarkRarityThreshold: 0.7, landmarkGateSimilarityThreshold: 0.3 },
- * );
- * // pages 0 and 1 share an identical header used by only 2 of the 3 pages
- * // (a 2/3 ≈ 0.667 corpus frequency, rare at threshold 0.7) and their
- * // content clears 0.3, so they merge onto one landmark-merge: key; page 2
- * // (a structurally different header) is left untouched
- * ```
- */
-export declare function mergeLandmarkAffinedClusters(clusterKeys: readonly string[], landmarks: readonly ExtractLandmarksResult[], contentTokenSets: readonly ReadonlySet<string>[], options?: MergeLandmarkAffinedClustersOptions): string[];