@d-zero/page-cluster 0.3.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. package/README.md +45 -8
  2. package/dist/build-cluster-reason.d.ts +119 -0
  3. package/dist/build-cluster-reason.js +63 -0
  4. package/dist/cli.d.ts +1 -0
  5. package/dist/cli.js +64 -6
  6. package/dist/derive-blocking-reason.d.ts +32 -0
  7. package/dist/derive-blocking-reason.js +1 -0
  8. package/dist/extract-landmarks.d.ts +75 -14
  9. package/dist/extract-landmarks.js +64 -4
  10. package/dist/is-chrome-landmark-instance.d.ts +41 -0
  11. package/dist/is-chrome-landmark-instance.js +50 -0
  12. package/dist/merge-cross-block-clusters.d.ts +39 -3
  13. package/dist/merge-cross-block-clusters.js +28 -107
  14. package/dist/offset-to-line-column.d.ts +38 -0
  15. package/dist/offset-to-line-column.js +49 -0
  16. package/dist/pass0-blocking.d.ts +10 -0
  17. package/dist/pass0-blocking.js +26 -46
  18. package/dist/per-page-landmark-signatures.d.ts +9 -1
  19. package/dist/per-page-landmark-signatures.js +16 -4
  20. package/dist/reassign-orphan-block-keys.d.ts +9 -0
  21. package/dist/reassign-orphan-block-keys.js +5 -2
  22. package/dist/resolve-blocking-group-keys.d.ts +10 -0
  23. package/dist/resolve-blocking-group-keys.js +16 -86
  24. package/dist/resolve-page-cluster-keys.d.ts +37 -1
  25. package/dist/resolve-page-cluster-keys.js +123 -17
  26. package/dist/shell-quorum.d.ts +70 -0
  27. package/dist/shell-quorum.js +110 -0
  28. package/package.json +10 -2
package/dist/shell-quorum.js ADDED
@@ -0,0 +1,110 @@
1
+ import { autoCutThreshold } from './auto-cut-threshold.js';
2
+ /**
3
+ * Fallback clamp for {@link ./auto-cut-threshold.js | autoCutThreshold} when
4
+ * run on the per-landmark-instance token-frequency distribution in
5
+ * {@link ./shell-quorum.js | shellQuorum}. Independently tunable from
6
+ * `merge-cross-block-clusters.ts`'s own `QUORUM_FRACTION` (same value today,
7
+ * 0.8, by coincidence of both having been validated against the same real
8
+ * crawl corpora — not because the two are meant to move together).
9
+ */
10
+ const SHELL_QUORUM_FALLBACK_FRACTION = 0.8;
11
+ /**
12
+ * Discovers a unit's shell tokens by auto-cutting the per-*token* page-
13
+ * frequency histogram of every landmark instance's tokens. This is the same
14
+ * max-gap primitive used for Stage A merge-height cutoffs, applied
15
+ * recursively at the landmark-token layer.
16
+ *
17
+ * ## Why per-token and not per-signature
18
+ *
19
+ * An earlier iteration ran the histogram at the level of full landmark-
20
+ * instance signatures (canonicalized token sets). That failed on a real,
21
+ * common pattern: a shared site chrome whose markup carries a per-page
22
+ * distinguishing element (a breadcrumb, a page-title element with a page-
23
+ * specific class, a "current" state). All pages have most of the same
24
+ * tokens, but every page's full signature is distinct because tokens embed
25
+ * class names. Per-signature counting saw 5 signatures at freq 0.2 each,
26
+ * autoCutThreshold on the flat distribution returned the clamp, and the
27
+ * shell collapsed to empty even though every page shared the core header
28
+ * skeleton. Per-token counting handles the same case correctly — the shared
29
+ * skeleton tokens each hit freq 1.0.
30
+ *
31
+ * ## The histogram
32
+ *
33
+ * For every member page, all its landmark instances are tokenized and
34
+ * unioned into a single per-page token set (order-agnostic, deduped: a
35
+ * token appearing in two of the page's landmarks still counts once for
36
+ * that page). The corpus histogram is then "how many pages contain each
37
+ * token". Tokens that appear on nearly every page are the unit's chrome;
38
+ * tokens that appear on only a handful are page-specific content that
39
+ * happens to be tagged as a landmark.
40
+ *
41
+ * ## Why auto-cut instead of a hard-coded quorum
42
+ *
43
+ * A fixed 80% quorum (this file's earlier implementation) baked one
44
+ * threshold in for every unit. Real corpora don't obey a universal cutoff:
45
+ * a section-local landmark token that appears on 60% of a unit's pages is
46
+ * the section's chrome under any reasonable reading, but 80% quorum
47
+ * discards it. Auto-cut looks at the *shape* of the frequency distribution
48
+ * and picks the widest gap between adjacent frequencies — if the
49
+ * distribution is `{1.00, 1.00, 0.65, 0.03, 0.02}`, the gap between 0.65
50
+ * and 0.03 (0.62) dwarfs everything else and the cut lands mid-gap around
51
+ * 0.34, correctly grouping the 0.65 tokens with the site-wide 1.00 ones as
52
+ * "chrome for this unit". If instead the distribution is flat, the clamp
53
+ * to {@link SHELL_QUORUM_FALLBACK_FRACTION} keeps the threshold from being
54
+ * tighter than the fallback default.
55
+ *
56
+ * ## Fallbacks
57
+ *
58
+ * A single distinct token (`heights.length < 2`) or a perfectly flat
59
+ * distribution (`maxGap === 0`) returns the
60
+ * {@link SHELL_QUORUM_FALLBACK_FRACTION} clamp verbatim — exactly the same
61
+ * 80%-quorum behavior as before. So degenerate cases degrade to the old
62
+ * contract; only richer distributions get the auto-cut benefit.
63
+ *
64
+ * A page with no landmarks contributes an empty set, deliberately, so the
65
+ * shell-corroboration jaccard between two landmark-less pages is 0 rather
66
+ * than 1 (which it would be if we handed back a `<body></body>`-derived
67
+ * `{body}` fallback set to both sides).
68
+ *
69
+ * ## Reuse
70
+ *
71
+ * Originally private to {@link ./merge-cross-block-clusters.js | mergeCrossBlockClusters}'s
72
+ * Stage B L2 corroboration; exported from its own module so
73
+ * {@link ./build-cluster-reason.js | buildClusterReason} can run it once per
74
+ * final cluster, per landmark type, to classify individual landmark
75
+ * instances as chrome (see {@link ./is-chrome-landmark-instance.js |
76
+ * isChromeLandmarkInstance}).
77
+ * @param perPageInstances
78
+ */
79
+ export function shellQuorum(perPageInstances) {
80
+ const pageCount = perPageInstances.length;
81
+ if (pageCount === 0)
82
+ return new Set();
83
+ // Union all instance token sets per page (dedupe within page: a token
84
+ // present on both header and footer of the same page still counts once
85
+ // for that page's contribution).
86
+ const tokenPageCount = new Map();
87
+ for (const instances of perPageInstances) {
88
+ const perPageUnion = new Set();
89
+ for (const inst of instances) {
90
+ for (const token of inst.tokens)
91
+ perPageUnion.add(token);
92
+ }
93
+ for (const token of perPageUnion) {
94
+ tokenPageCount.set(token, (tokenPageCount.get(token) ?? 0) + 1);
95
+ }
96
+ }
97
+ if (tokenPageCount.size === 0)
98
+ return new Set();
99
+ const frequencies = [];
100
+ for (const count of tokenPageCount.values()) {
101
+ frequencies.push(count / pageCount);
102
+ }
103
+ const cut = autoCutThreshold(frequencies, SHELL_QUORUM_FALLBACK_FRACTION);
104
+ const shell = new Set();
105
+ for (const [token, count] of tokenPageCount) {
106
+ if (count / pageCount >= cut)
107
+ shell.add(token);
108
+ }
109
+ return shell;
110
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@d-zero/page-cluster",
3
- "version": "0.3.1",
3
+ "version": "0.5.0",
4
4
  "description": "Clusters crawled HTML pages by DOM-structure similarity — assigns the same key to pages sharing a template, ignoring text content. CLI-first, with library APIs.",
5
5
  "author": "D-ZERO",
6
6
  "license": "MIT",
@@ -24,6 +24,14 @@
24
24
  "./resolve-page-cluster-keys": {
25
25
  "import": "./dist/resolve-page-cluster-keys.js",
26
26
  "types": "./dist/resolve-page-cluster-keys.d.ts"
27
+ },
28
+ "./is-chrome-landmark-instance": {
29
+ "import": "./dist/is-chrome-landmark-instance.js",
30
+ "types": "./dist/is-chrome-landmark-instance.d.ts"
31
+ },
32
+ "./jaccard-similarity": {
33
+ "import": "./dist/jaccard-similarity.js",
34
+ "types": "./dist/jaccard-similarity.d.ts"
27
35
  }
28
36
  },
29
37
  "bin": "./dist/cli.js",
@@ -45,5 +53,5 @@
45
53
  "url": "https://github.com/d-zero-dev/tools.git",
46
54
  "directory": "packages/@d-zero/page-cluster"
47
55
  },
48
- "gitHead": "9c1e3da176d39972eae9903a979e942a59515229"
56
+ "gitHead": "956fa3488d875756ad57fb391bc4822704570f6b"
49
57
  }