npm - @d-zero/page-cluster - Versions diffs - 0.2.0 - Mend

@d-zero/page-cluster 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (80) hide show

package/README.md +68 -0
package/dist/array-edit-distance.d.ts +20 -0
package/dist/array-edit-distance.js +52 -0
package/dist/build-segment.d.ts +18 -0
package/dist/build-segment.js +27 -0
package/dist/cap-content-depth.d.ts +69 -0
package/dist/cap-content-depth.js +161 -0
package/dist/compute-document-frequency.d.ts +33 -0
package/dist/compute-document-frequency.js +40 -0
package/dist/create-frame.d.ts +16 -0
package/dist/create-frame.js +29 -0
package/dist/derive-path-group-key.d.ts +44 -0
package/dist/derive-path-group-key.js +51 -0
package/dist/derive-stylesheet-group-key.d.ts +36 -0
package/dist/derive-stylesheet-group-key.js +41 -0
package/dist/detect-content-depth-cap.d.ts +114 -0
package/dist/detect-content-depth-cap.js +137 -0
package/dist/escape-reg-exp.d.ts +11 -0
package/dist/escape-reg-exp.js +13 -0
package/dist/excise.d.ts +13 -0
package/dist/excise.js +24 -0
package/dist/extract-landmarks.d.ts +82 -0
package/dist/extract-landmarks.js +104 -0
package/dist/filter-first-party-stylesheet-hrefs.d.ts +73 -0
package/dist/filter-first-party-stylesheet-hrefs.js +118 -0
package/dist/find-shallowest-elements.d.ts +39 -0
package/dist/find-shallowest-elements.js +121 -0
package/dist/foldable-tags.d.ts +8 -0
package/dist/foldable-tags.js +8 -0
package/dist/format-bracket.d.ts +11 -0
package/dist/format-bracket.js +17 -0
package/dist/hash-content.d.ts +22 -0
package/dist/hash-content.js +26 -0
package/dist/html-region-utils.d.ts +74 -0
package/dist/html-region-utils.js +96 -0
package/dist/is-fold-candidate.d.ts +13 -0
package/dist/is-fold-candidate.js +16 -0
package/dist/is-genuine-close.d.ts +23 -0
package/dist/is-genuine-close.js +27 -0
package/dist/is-noise-class.d.ts +6 -0
package/dist/is-noise-class.js +8 -0
package/dist/jaccard-similarity.d.ts +23 -0
package/dist/jaccard-similarity.js +36 -0
package/dist/merge-landmark-affined-clusters.d.ts +179 -0
package/dist/merge-landmark-affined-clusters.js +544 -0
package/dist/merge-spans.d.ts +15 -0
package/dist/merge-spans.js +22 -0
package/dist/noise-class-patterns.d.ts +21 -0
package/dist/noise-class-patterns.js +74 -0
package/dist/normalize-for-hash.d.ts +10 -0
package/dist/normalize-for-hash.js +12 -0
package/dist/opaque-tags.d.ts +17 -0
package/dist/opaque-tags.js +18 -0
package/dist/parse-class-list.d.ts +10 -0
package/dist/parse-class-list.js +23 -0
package/dist/reassign-orphan-block-keys.d.ts +81 -0
package/dist/reassign-orphan-block-keys.js +159 -0
package/dist/remove-content-blocks.d.ts +67 -0
package/dist/remove-content-blocks.js +150 -0
package/dist/resolve-blocking-group-keys.d.ts +116 -0
package/dist/resolve-blocking-group-keys.js +120 -0
package/dist/resolve-closed-frame.d.ts +26 -0
package/dist/resolve-closed-frame.js +33 -0
package/dist/resolve-landmark-variant-keys.d.ts +66 -0
package/dist/resolve-landmark-variant-keys.js +71 -0
package/dist/resolve-options.d.ts +6 -0
package/dist/resolve-options.js +10 -0
package/dist/resolve-page-cluster-keys.d.ts +222 -0
package/dist/resolve-page-cluster-keys.js +198 -0
package/dist/resolve-structural-cluster-keys.d.ts +50 -0
package/dist/resolve-structural-cluster-keys.js +287 -0
package/dist/run-tokenizer.d.ts +33 -0
package/dist/run-tokenizer.js +152 -0
package/dist/split-tokens-by-frequency.d.ts +46 -0
package/dist/split-tokens-by-frequency.js +88 -0
package/dist/tokenize.d.ts +58 -0
package/dist/tokenize.js +60 -0
package/dist/types.d.ts +85 -0
package/dist/types.js +1 -0
package/package.json +102 -0

package/dist/jaccard-similarity.d.ts ADDED Viewed

@@ -0,0 +1,23 @@
+/**
+ * Structural-similarity primitive shared by two downstream stages that are
+ * not yet implemented: base-cluster generation over `tokenize()` output
+ * turned into sets (MinHash/LSH candidate scoring approximates this same
+ * ratio) and the default distance for merging medoids in the eventual
+ * hierarchical clustering step. Both need "how much of these two token sets
+ * overlaps" as a plain, parameter-free calculation, independent of whichever
+ * hashing/banding scheme ends up approximating it at scale.
+ *
+ * Two empty sets return `1`, not `0` or `NaN`: an empty `<body>` compared
+ * against another empty `<body>` has no structural difference to report, so
+ * treating them as identical (rather than "undefined" or "no overlap") keeps
+ * the result usable directly as a similarity score without a caller-side
+ * special case.
+ * @param a
+ * @param b
+ * @example
+ * ```ts
+ * jaccardSimilarity(new Set(['body>ul>li']), new Set(['body>ul>li']));
+ * // 1
+ * ```
+ */
+export declare function jaccardSimilarity(a: ReadonlySet<string>, b: ReadonlySet<string>): number;

package/dist/jaccard-similarity.js ADDED Viewed

@@ -0,0 +1,36 @@
+/**
+ * Structural-similarity primitive shared by two downstream stages that are
+ * not yet implemented: base-cluster generation over `tokenize()` output
+ * turned into sets (MinHash/LSH candidate scoring approximates this same
+ * ratio) and the default distance for merging medoids in the eventual
+ * hierarchical clustering step. Both need "how much of these two token sets
+ * overlaps" as a plain, parameter-free calculation, independent of whichever
+ * hashing/banding scheme ends up approximating it at scale.
+ *
+ * Two empty sets return `1`, not `0` or `NaN`: an empty `<body>` compared
+ * against another empty `<body>` has no structural difference to report, so
+ * treating them as identical (rather than "undefined" or "no overlap") keeps
+ * the result usable directly as a similarity score without a caller-side
+ * special case.
+ * @param a
+ * @param b
+ * @example
+ * ```ts
+ * jaccardSimilarity(new Set(['body>ul>li']), new Set(['body>ul>li']));
+ * // 1
+ * ```
+ */
+export function jaccardSimilarity(a, b) {
+    if (a.size === 0 && b.size === 0) {
+        return 1;
+    }
+    let intersectionSize = 0;
+    const [smaller, larger] = a.size <= b.size ? [a, b] : [b, a];
+    for (const token of smaller) {
+        if (larger.has(token)) {
+            intersectionSize++;
+        }
+    }
+    const unionSize = a.size + b.size - intersectionSize;
+    return intersectionSize / unionSize;
+}

package/dist/merge-landmark-affined-clusters.d.ts ADDED Viewed

@@ -0,0 +1,179 @@
+import type { ExtractLandmarksResult } from './extract-landmarks.js';
+import type { TokenizeOptions } from './types.js';
+/**
+ * @see mergeLandmarkAffinedClusters
+ */
+export type MergeLandmarkAffinedClustersOptions = TokenizeOptions & {
+    /**
+     * Reused as the landmark-variant identity threshold inside
+     * `computeLandmarkStatus`. Must be in `[0, 1]` (`RangeError` otherwise).
+     * Defaults to `0.8`, matching `resolveStructuralClusterKeys`'s own
+     * default — this file never calls that function, but the two thresholds
+     * represent the same concept ("how much token overlap counts as the same
+     * design").
+     *
+     * Deliberately not a separate, independent option: when
+     * `resolvePageClusterKeys` forwards its caller's single `options` object
+     * to both `resolveStructuralClusterKeys` (primary content clustering) and
+     * this file (landmark-variant identity), the same `similarityThreshold`
+     * value drives both, so loosening one to re-tune primary clustering (the
+     * documented `excludeLandmarks`/`similarityThreshold` interaction on
+     * `resolvePageClusterKeys`) also loosens landmark-variant matching. This
+     * mirrors `resolve-landmark-variant-keys.ts`'s own precedent
+     * (`resolveLandmarkVariantKeys` likewise forwards its caller's
+     * `options.similarityThreshold` straight into
+     * `resolveStructuralClusterKeys` with no landmark-specific override) and
+     * keeps the option surface to the three fields this file actually adds.
+     * Accepted as a known trade-off rather than split into its own knob until
+     * real-corpus tuning shows the two thresholds genuinely need to diverge —
+     * the same "starting heuristic, not yet corpus-validated" status this
+     * option's own default already carries.
+     */
+    similarityThreshold?: number;
+    /**
+     * Upper bound, as a fraction of the whole corpus (`[0, 1]`), on how many
+     * pages may share a given (landmark type, variant) pair before that
+     * variant is considered too common to serve as evidence of a genuine
+     * template affinity. Strictly `<` this fraction ("rare", not
+     * "rare-or-equal"). `RangeError` outside `[0, 1]`. Defaults to `0.05` —
+     * an unvalidated starting heuristic, the same status as
+     * `similarityThreshold`'s own `0.8` default (see this file's JSDoc for
+     * why real-corpus validation is out of scope for this change).
+     */
+    landmarkRarityThreshold?: number;
+    /**
+     * The secondary, looser complete-linkage content-similarity threshold
+     * applied only to pages whose landmark signature already qualifies (see
+     * `mergeLandmarkAffinedClusters`'s JSDoc). `RangeError` outside `[0, 1]`.
+     * Defaults to `0.6` — the value a withdrawn earlier prototype of this
+     * same mechanism proposed, and also the value
+     * `resolvePageClusterKeys`'s own `excludeLandmarks` JSDoc cites as having
+     * correctly re-merged a real 3-page block once landmarks were excluded
+     * and the raw-token `similarityThreshold` (`0.8`) became too strict.
+     */
+    landmarkGateSimilarityThreshold?: number;
+};
+/**
+ * Validates `similarityThreshold`/`landmarkRarityThreshold`/
+ * `landmarkGateSimilarityThreshold` without running
+ * `mergeLandmarkAffinedClusters` itself — exported so
+ * `resolvePageClusterKeys` can fail fast on bad options even when `pages` is
+ * empty (its own per-block loop never reaches this function at all in that
+ * case). Mirrors `detect-content-depth-cap.ts`'s
+ * `validateDetectContentDepthCapOptions` exact rationale and shape.
+ * @param options
+ * @example
+ * ```ts
+ * // Fails fast on a bad option even though nothing here would otherwise
+ * // call mergeLandmarkAffinedClusters yet (e.g. cluster keys haven't been
+ * // computed).
+ * validateMergeLandmarkAffinedClustersOptions({ landmarkRarityThreshold: -1 }); // throws RangeError
+ * ```
+ */
+export declare function validateMergeLandmarkAffinedClustersOptions(options?: MergeLandmarkAffinedClustersOptions): void;
+/**
+ * Re-keys the pages of two or more distinct
+ * {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys} clusters
+ * onto one shared key when every landmark type present on their pages is
+ * both *identical* and *rare* corpus-wide, and their actual content clears a
+ * secondary, looser similarity threshold.
+ *
+ * Reimplements a mechanism previously prototyped under this same name and
+ * withdrawn (no trace survives in commit history — this JSDoc is the only
+ * record). The withdrawn version merged clusters whenever their
+ * header/footer/nav/aside matched, full stop. Validated against two real
+ * crawl corpora (302 and 8,936 pages), that produced runaway over-merging:
+ * header/footer/nav were present on 99%+ of pages and typically reused
+ * site-wide unchanged (see `extractLandmarks`'s own JSDoc for that figure),
+ * so "landmarks match" was true for nearly every page pair and carried no
+ * discriminative power at all. This reimplementation only ever treats a
+ * landmark match as merge evidence when that specific landmark *variant* is
+ * itself uncommon corpus-wide (`landmarkRarityThreshold`) — the condition
+ * the withdrawn attempt lacked.
+ *
+ * The match requirement is deliberately the most conservative option
+ * considered: *every* landmark type actually present on a page must both
+ * match its counterpart's variant and be rare — a page with even one common
+ * ("everybody has this exact header") present type contributes no evidence
+ * at all, rather than partially qualifying. A looser rule (e.g. "at least
+ * one shared rare type is enough") was rejected because it reintroduces a
+ * version of the original failure mode: a page could ride a single
+ * incidentally-rare landmark into a merge despite otherwise-ordinary,
+ * ubiquitous chrome elsewhere on the same page.
+ *
+ * Frequency is counted corpus-wide, not per-block: a `resolveStructuralClusterKeys`
+ * cluster label (`cluster:N`) is only unique within the block it was computed
+ * in, but rarity here needs one consistent count across the whole input, the
+ * same reason `resolvePageClusterKeys` itself composes `[blockKey,
+ * localLabel]` via `JSON.stringify` rather than reusing bare labels across
+ * blocks.
+ *
+ * A page with none of the four landmark types present is excluded from
+ * consideration entirely (`existingCount === 0` below) — without this, every
+ * landmark-less page across the whole corpus would share one large,
+ * unbounded "no landmarks" group, defeating the self-limiting cost bound
+ * `landmarkRarityThreshold` is otherwise supposed to guarantee (see
+ * `computeLandmarkStatus`'s JSDoc for the cost analysis this depends on).
+ *
+ * Once pages are grouped by matching-and-rare landmark signature, only
+ * signature groups spanning two or more distinct existing cluster keys do
+ * any further work. Within such a group, the *content* token sets of the
+ * group's distinct cluster keys are complete-linkage-merged at
+ * `landmarkGateSimilarityThreshold` — looser than
+ * `resolveStructuralClusterKeys`'s own `similarityThreshold`, since the
+ * whole point of this mechanism is to bridge clusters whose *content*
+ * similarity alone fell just short of the primary threshold. Complete-linkage
+ * (not single-linkage) is used for the same reason
+ * `resolveStructuralClusterKeys` itself uses it: single-linkage's chaining
+ * would let one loosely-matching pair bridge two genuinely-unrelated
+ * clusters transitively.
+ *
+ * The resulting merge is applied at *page* granularity, not by blanket-
+ * reassigning every page of the involved cluster keys: only the specific
+ * pages that were actually pooled into the qualifying signature group (and,
+ * transitively, any other page unioned with them via a different signature
+ * group) move onto the shared key. A cluster's pages that never carried the
+ * rare landmark evidence keep their original key untouched, even if some
+ * other page sharing that same cluster key did qualify and merge elsewhere.
+ * This is deliberate, not an incidental restriction: applying a merge
+ * decision to *every* page of the involved cluster keys — evidenced by only
+ * a small subset of them — would extrapolate a coincidental pairing (e.g.
+ * one outlier page in each of two otherwise-unrelated clusters happening to
+ * share a rare seasonal-campaign header) into force-merging the clusters'
+ * entire, otherwise-dissimilar membership. That is the withdrawn prototype's
+ * over-merging failure mode reappearing through a different mechanism
+ * (whole-cluster application of a single-pair signal) rather than the
+ * landmark-commonality mechanism this file was reimplemented to fix — see
+ * this function's own regression test for a worked example.
+ *
+ * Merged pages are re-keyed to `landmark-merge:${JSON.stringify(sortedKeys)}`
+ * (`sortedKeys` being the *original* cluster keys the merged pages came
+ * from) — a fresh prefix that cannot collide with `css:`/`path:`/
+ * `orphan-merge:` or `resolvePageClusterKeys`'s own `[blockKey, "cluster:N"]`
+ * pairs (mirrors `reassign-orphan-block-keys.ts`'s `orphan-merge:` prefix).
+ * @param clusterKeys - one existing final key per page, same order/length as `landmarks`/`contentTokenSets`
+ * @param landmarks - `extractLandmarks(page.html)`'s full result per page (all four fields, not just `remainderHtml`)
+ * @param contentTokenSets - per-page content token sets to use for the secondary similarity gate. Should be independent of whichever landmark markup qualified the page as evidence (e.g. always landmark-excised), so this gate is a genuine second signal rather than re-counting the same landmark tokens already used to select the page — see `resolvePageClusterKeys`'s own call site for how it builds these
+ * @param options
+ * @example
+ * ```ts
+ * // tokenize() discards visible text (see its own JSDoc), so the two
+ * // header variants below must differ structurally (child element/class),
+ * // not merely in text, to compare as different landmark variants.
+ * mergeLandmarkAffinedClusters(
+ * 	['["css:a", "cluster:0"]', '["css:b", "cluster:0"]', 'path:other'],
+ * 	[
+ * 		{ header: '<header><i class="mark-a"></i></header>', remainderHtml: '' },
+ * 		{ header: '<header><i class="mark-a"></i></header>', remainderHtml: '' },
+ * 		{ header: '<header><b class="mark-b"></b></header>', remainderHtml: '' },
+ * 	],
+ * 	[new Set(['a', 'b']), new Set(['a', 'c']), new Set(['z'])],
+ * 	{ landmarkRarityThreshold: 0.7, landmarkGateSimilarityThreshold: 0.3 },
+ * );
+ * // pages 0 and 1 share an identical header used by only 2 of the 3 pages
+ * // (a 2/3 ≈ 0.667 corpus frequency, rare at threshold 0.7) and their
+ * // content clears 0.3, so they merge onto one landmark-merge: key; page 2
+ * // (a structurally different header) is left untouched
+ * ```
+ */
+export declare function mergeLandmarkAffinedClusters(clusterKeys: readonly string[], landmarks: readonly ExtractLandmarksResult[], contentTokenSets: readonly ReadonlySet<string>[], options?: MergeLandmarkAffinedClustersOptions): string[];