@d-zero/page-cluster 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +68 -0
  2. package/dist/array-edit-distance.d.ts +20 -0
  3. package/dist/array-edit-distance.js +52 -0
  4. package/dist/build-segment.d.ts +18 -0
  5. package/dist/build-segment.js +27 -0
  6. package/dist/cap-content-depth.d.ts +69 -0
  7. package/dist/cap-content-depth.js +161 -0
  8. package/dist/compute-document-frequency.d.ts +33 -0
  9. package/dist/compute-document-frequency.js +40 -0
  10. package/dist/create-frame.d.ts +16 -0
  11. package/dist/create-frame.js +29 -0
  12. package/dist/derive-path-group-key.d.ts +44 -0
  13. package/dist/derive-path-group-key.js +51 -0
  14. package/dist/derive-stylesheet-group-key.d.ts +36 -0
  15. package/dist/derive-stylesheet-group-key.js +41 -0
  16. package/dist/detect-content-depth-cap.d.ts +114 -0
  17. package/dist/detect-content-depth-cap.js +137 -0
  18. package/dist/escape-reg-exp.d.ts +11 -0
  19. package/dist/escape-reg-exp.js +13 -0
  20. package/dist/excise.d.ts +13 -0
  21. package/dist/excise.js +24 -0
  22. package/dist/extract-landmarks.d.ts +82 -0
  23. package/dist/extract-landmarks.js +104 -0
  24. package/dist/filter-first-party-stylesheet-hrefs.d.ts +73 -0
  25. package/dist/filter-first-party-stylesheet-hrefs.js +118 -0
  26. package/dist/find-shallowest-elements.d.ts +39 -0
  27. package/dist/find-shallowest-elements.js +121 -0
  28. package/dist/foldable-tags.d.ts +8 -0
  29. package/dist/foldable-tags.js +8 -0
  30. package/dist/format-bracket.d.ts +11 -0
  31. package/dist/format-bracket.js +17 -0
  32. package/dist/hash-content.d.ts +22 -0
  33. package/dist/hash-content.js +26 -0
  34. package/dist/html-region-utils.d.ts +74 -0
  35. package/dist/html-region-utils.js +96 -0
  36. package/dist/is-fold-candidate.d.ts +13 -0
  37. package/dist/is-fold-candidate.js +16 -0
  38. package/dist/is-genuine-close.d.ts +23 -0
  39. package/dist/is-genuine-close.js +27 -0
  40. package/dist/is-noise-class.d.ts +6 -0
  41. package/dist/is-noise-class.js +8 -0
  42. package/dist/jaccard-similarity.d.ts +23 -0
  43. package/dist/jaccard-similarity.js +36 -0
  44. package/dist/merge-landmark-affined-clusters.d.ts +179 -0
  45. package/dist/merge-landmark-affined-clusters.js +544 -0
  46. package/dist/merge-spans.d.ts +15 -0
  47. package/dist/merge-spans.js +22 -0
  48. package/dist/noise-class-patterns.d.ts +21 -0
  49. package/dist/noise-class-patterns.js +74 -0
  50. package/dist/normalize-for-hash.d.ts +10 -0
  51. package/dist/normalize-for-hash.js +12 -0
  52. package/dist/opaque-tags.d.ts +17 -0
  53. package/dist/opaque-tags.js +18 -0
  54. package/dist/parse-class-list.d.ts +10 -0
  55. package/dist/parse-class-list.js +23 -0
  56. package/dist/reassign-orphan-block-keys.d.ts +81 -0
  57. package/dist/reassign-orphan-block-keys.js +159 -0
  58. package/dist/remove-content-blocks.d.ts +67 -0
  59. package/dist/remove-content-blocks.js +150 -0
  60. package/dist/resolve-blocking-group-keys.d.ts +116 -0
  61. package/dist/resolve-blocking-group-keys.js +120 -0
  62. package/dist/resolve-closed-frame.d.ts +26 -0
  63. package/dist/resolve-closed-frame.js +33 -0
  64. package/dist/resolve-landmark-variant-keys.d.ts +66 -0
  65. package/dist/resolve-landmark-variant-keys.js +71 -0
  66. package/dist/resolve-options.d.ts +6 -0
  67. package/dist/resolve-options.js +10 -0
  68. package/dist/resolve-page-cluster-keys.d.ts +222 -0
  69. package/dist/resolve-page-cluster-keys.js +198 -0
  70. package/dist/resolve-structural-cluster-keys.d.ts +50 -0
  71. package/dist/resolve-structural-cluster-keys.js +287 -0
  72. package/dist/run-tokenizer.d.ts +33 -0
  73. package/dist/run-tokenizer.js +152 -0
  74. package/dist/split-tokens-by-frequency.d.ts +46 -0
  75. package/dist/split-tokens-by-frequency.js +88 -0
  76. package/dist/tokenize.d.ts +58 -0
  77. package/dist/tokenize.js +60 -0
  78. package/dist/types.d.ts +85 -0
  79. package/dist/types.js +1 -0
  80. package/package.json +102 -0
package/dist/tokenize.js ADDED
@@ -0,0 +1,60 @@
1
+ import { resolveOptions } from './resolve-options.js';
2
+ import { runTokenizer } from './run-tokenizer.js';
3
+ /**
4
+ * Tokenizes the structural skeleton of an HTML document's `<body>` for
5
+ * duplicate/near-duplicate page detection at crawl scale. This is the first
6
+ * building block of `@d-zero/page-cluster`; clustering on top of these
7
+ * tokens is layered on by
8
+ * {@link ./resolve-structural-cluster-keys.js | resolveStructuralClusterKeys}
9
+ * (exact O(n²) complete-linkage via NN-chain) and orchestrated across blocks
10
+ * by {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys} —
11
+ * MinHash/LSH-based approximation was considered and rejected: see
12
+ * `resolveStructuralClusterKeys`'s JSDoc for why.
13
+ *
14
+ * Only `<body>` is tokenized. `<head>` (title/meta/link/OGP/...) is ignored
15
+ * entirely: `@d-zero/beholder` already extracts it comprehensively, but from
16
+ * a live `Document` (Puppeteer/jsdom) rather than a raw HTML string, so that
17
+ * logic can't be reused here without building a DOM — exactly what this
18
+ * function avoids for speed. Callers who need head metadata should call
19
+ * `@d-zero/beholder` separately.
20
+ *
21
+ * Visible text is discarded entirely: this function measures *structural*
22
+ * similarity, and including per-page text would make otherwise-identical
23
+ * templates look unique.
24
+ *
25
+ * The returned array intentionally does not deduplicate or
26
+ * run-length-compress repeated paths — even though the eventual consumer
27
+ * (a MinHash/LSH classifier) will reduce this array to a `Set` for
28
+ * comparison, and a `Set` alone already collapses any number of repeated
29
+ * entries. Compressing here first (e.g. `"li>a*3"`) would embed the
30
+ * arrangement of neighboring siblings into the token string itself: a
31
+ * `current`/`active`-style state class on exactly one sibling (its position
32
+ * varies per page, e.g. which nav item is "current") shifts which runs are
33
+ * adjacent, so the same template could serialize as `"li>a*2"` on one page
34
+ * and as two separate `"li>a"` entries (split by the state-bearing sibling)
35
+ * on another — literally different strings for what should compare equal
36
+ * once turned into a `Set`. Leaving the array uncompressed sidesteps that
37
+ * entirely:
38
+ * `Set(["li>a", "li.current>a", "li>a"])` and
39
+ * `Set(["li.current>a", "li>a", "li>a"])` are the same two-element set no
40
+ * matter where the state class lands. If a future consumer needs a shorter
41
+ * array for e.g. array-edit-distance comparisons on pathologically large
42
+ * pages, that consumer should apply its own compression tuned to its own
43
+ * needs, since compression is coupled to how the caller will read counts
44
+ * back out again — folding that guess into this package's contract can't be
45
+ * un-shipped later.
46
+ *
47
+ * `<body>`'s own `class` is excluded from every leaf path and returned
48
+ * separately as `bodyClassList` — see {@link ./types.js | TokenizeResult}'s
49
+ * JSDoc for why.
50
+ * @param html
51
+ * @param options
52
+ * @example
53
+ * ```ts
54
+ * tokenize('<body><div class="card"><ul><li>A</li><li>B</li></ul></div></body>');
55
+ * // { tokens: ["body>.card>ul>li", "body>.card>ul>li"], bodyClassList: [] }
56
+ * ```
57
+ */
58
+ export function tokenize(html, options) {
59
+ return runTokenizer(html, resolveOptions(options));
60
+ }
package/dist/types.d.ts ADDED
@@ -0,0 +1,85 @@
1
+ /**
2
+ * Result of {@link ../tokenize.js | tokenize}. `bodyClassList` is kept
3
+ * separate from `tokens` rather than folded into the root of every leaf
4
+ * path: CMS `body_class()` conventions (WordPress, Drupal, ...) commonly
5
+ * inject a per-page-category label onto `<body>` (e.g. `law-page`,
6
+ * `humanities-page`), and a caller that mixed it into every path would find
7
+ * that a single such label corrupts *every* token for that page, since it's
8
+ * the common prefix of every root-to-leaf path — collapsing near-duplicate
9
+ * pages to zero similarity even when their descendant structure is
10
+ * otherwise identical. Moving it out preserves the information (a caller
11
+ * that genuinely needs body-class-driven grouping still has it) without
12
+ * letting it poison structural comparison by default.
13
+ */
14
+ export type TokenizeResult = {
15
+ tokens: string[];
16
+ bodyClassList: string[];
17
+ };
18
+ /**
19
+ * Options for {@link ../tokenize.js | tokenize}.
20
+ */
21
+ export type TokenizeOptions = {
22
+ /** Exclude hash-like auto-generated class names (CSS Modules, styled-components, emotion, bundler content-hash suffixes) before building segments. Defaults to `true`. */
23
+ filterNoiseClasses?: boolean;
24
+ /** Emit `comment[sha=...]` tokens for HTML comment nodes. Defaults to `false`. */
25
+ includeComments?: boolean;
26
+ };
27
+ /**
28
+ * `TokenizeOptions` with every field defaulted.
29
+ */
30
+ export type ResolvedOptions = {
31
+ filterNoiseClasses: boolean;
32
+ includeComments: boolean;
33
+ };
34
+ /**
35
+ * One entry in the currently-open ancestor chain.
36
+ *
37
+ * Descendant leaf paths are accumulated in `pendingPaths` *relative to this
38
+ * frame* (i.e. without this frame's own `segment` prefixed yet), because
39
+ * whether this frame folds away or keeps its segment is only known once it
40
+ * closes (see `resolve-closed-frame.ts`). Holding one frame per open ancestor
41
+ * — rather than the whole parsed document — keeps memory proportional to
42
+ * nesting depth, not document size.
43
+ */
44
+ export type Frame = {
45
+ tagName: string;
46
+ /** This element's own path segment (class/role/type already applied). */
47
+ segment: string;
48
+ /** Whether this is a class-less/role-less/type-less `div` or `span`, eligible to be elided when it has exactly one element child. */
49
+ isFoldCandidate: boolean;
50
+ /** Count of direct element children (text and comment nodes are not counted). */
51
+ childElementCount: number;
52
+ /** Finalized descendant leaf paths, relative to this frame, in document order. */
53
+ pendingPaths: string[];
54
+ };
55
+ /**
56
+ * Result of {@link ../compute-document-frequency.js | computeDocumentFrequency}: how many pages (out of `pageCount`) contain each token.
57
+ *
58
+ * `pageCount` travels bundled with `documentFrequency` rather than being a
59
+ * separate argument at call sites that consume it (e.g.
60
+ * `splitTokensByFrequency`), so the two can never be passed out of sync with
61
+ * each other (e.g. a caller re-slicing the page list after computing
62
+ * frequencies but before using them).
63
+ */
64
+ export type DocumentFrequency = {
65
+ documentFrequency: ReadonlyMap<string, number>;
66
+ pageCount: number;
67
+ };
68
+ /**
69
+ * Tags whose contents are hashed instead of being tokenized further.
70
+ */
71
+ export type OpaqueTagName = 'script' | 'style' | 'noscript' | 'svg';
72
+ /**
73
+ * Tracks an open `script`/`style`/`noscript`/`svg` region so its raw source
74
+ * can be sliced out once it closes. `depth` guards against self-nesting
75
+ * (`<svg><svg>...`) closing the region prematurely.
76
+ */
77
+ export type OpaqueRegion = {
78
+ tagName: OpaqueTagName;
79
+ depth: number;
80
+ /** Offset into the original HTML string, just after the opening tag's `>`. */
81
+ contentStart: number;
82
+ /** `role`/`type` of the *outermost* opaque tag, captured when it opens (e.g. `<svg role="img">`). */
83
+ role: string | undefined;
84
+ type: string | undefined;
85
+ };
package/dist/types.js ADDED
@@ -0,0 +1 @@
1
+ export {};
package/package.json ADDED
@@ -0,0 +1,102 @@
1
+ {
2
+ "name": "@d-zero/page-cluster",
3
+ "version": "0.2.0",
4
+ "description": "Tokenizes an HTML document's body into a structural signature for duplicate/near-duplicate page detection at crawl scale",
5
+ "author": "D-ZERO",
6
+ "license": "MIT",
7
+ "publishConfig": {
8
+ "access": "public"
9
+ },
10
+ "type": "module",
11
+ "exports": {
12
+ ".": {
13
+ "import": "./dist/tokenize.js",
14
+ "types": "./dist/tokenize.d.ts"
15
+ },
16
+ "./array-edit-distance": {
17
+ "import": "./dist/array-edit-distance.js",
18
+ "types": "./dist/array-edit-distance.d.ts"
19
+ },
20
+ "./cap-content-depth": {
21
+ "import": "./dist/cap-content-depth.js",
22
+ "types": "./dist/cap-content-depth.d.ts"
23
+ },
24
+ "./compute-document-frequency": {
25
+ "import": "./dist/compute-document-frequency.js",
26
+ "types": "./dist/compute-document-frequency.d.ts"
27
+ },
28
+ "./derive-path-group-key": {
29
+ "import": "./dist/derive-path-group-key.js",
30
+ "types": "./dist/derive-path-group-key.d.ts"
31
+ },
32
+ "./derive-stylesheet-group-key": {
33
+ "import": "./dist/derive-stylesheet-group-key.js",
34
+ "types": "./dist/derive-stylesheet-group-key.d.ts"
35
+ },
36
+ "./detect-content-depth-cap": {
37
+ "import": "./dist/detect-content-depth-cap.js",
38
+ "types": "./dist/detect-content-depth-cap.d.ts"
39
+ },
40
+ "./extract-landmarks": {
41
+ "import": "./dist/extract-landmarks.js",
42
+ "types": "./dist/extract-landmarks.d.ts"
43
+ },
44
+ "./filter-first-party-stylesheet-hrefs": {
45
+ "import": "./dist/filter-first-party-stylesheet-hrefs.js",
46
+ "types": "./dist/filter-first-party-stylesheet-hrefs.d.ts"
47
+ },
48
+ "./jaccard-similarity": {
49
+ "import": "./dist/jaccard-similarity.js",
50
+ "types": "./dist/jaccard-similarity.d.ts"
51
+ },
52
+ "./merge-landmark-affined-clusters": {
53
+ "import": "./dist/merge-landmark-affined-clusters.js",
54
+ "types": "./dist/merge-landmark-affined-clusters.d.ts"
55
+ },
56
+ "./reassign-orphan-block-keys": {
57
+ "import": "./dist/reassign-orphan-block-keys.js",
58
+ "types": "./dist/reassign-orphan-block-keys.d.ts"
59
+ },
60
+ "./remove-content-blocks": {
61
+ "import": "./dist/remove-content-blocks.js",
62
+ "types": "./dist/remove-content-blocks.d.ts"
63
+ },
64
+ "./resolve-blocking-group-keys": {
65
+ "import": "./dist/resolve-blocking-group-keys.js",
66
+ "types": "./dist/resolve-blocking-group-keys.d.ts"
67
+ },
68
+ "./resolve-landmark-variant-keys": {
69
+ "import": "./dist/resolve-landmark-variant-keys.js",
70
+ "types": "./dist/resolve-landmark-variant-keys.d.ts"
71
+ },
72
+ "./resolve-page-cluster-keys": {
73
+ "import": "./dist/resolve-page-cluster-keys.js",
74
+ "types": "./dist/resolve-page-cluster-keys.d.ts"
75
+ },
76
+ "./resolve-structural-cluster-keys": {
77
+ "import": "./dist/resolve-structural-cluster-keys.js",
78
+ "types": "./dist/resolve-structural-cluster-keys.d.ts"
79
+ },
80
+ "./split-tokens-by-frequency": {
81
+ "import": "./dist/split-tokens-by-frequency.js",
82
+ "types": "./dist/split-tokens-by-frequency.d.ts"
83
+ }
84
+ },
85
+ "files": [
86
+ "dist"
87
+ ],
88
+ "scripts": {
89
+ "build": "tsc",
90
+ "watch": "tsc --watch",
91
+ "clean": "tsc --build --clean"
92
+ },
93
+ "dependencies": {
94
+ "@d-zero/shared": "0.22.2",
95
+ "htmlparser2": "12.0.0"
96
+ },
97
+ "repository": {
98
+ "type": "git",
99
+ "url": "https://github.com/d-zero-dev/tools.git",
100
+ "directory": "packages/@d-zero/page-cluster"
101
+ }
102
+ }