@d-zero/page-cluster 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +131 -39
- package/dist/assign-contained-clusters.d.ts +42 -0
- package/dist/assign-contained-clusters.js +156 -0
- package/dist/auto-cut-threshold.d.ts +17 -0
- package/dist/auto-cut-threshold.js +36 -0
- package/dist/canonicalize-token-set.d.ts +17 -0
- package/dist/canonicalize-token-set.js +19 -0
- package/dist/cli.d.ts +39 -0
- package/dist/cli.js +381 -0
- package/dist/collapse-anonymous-divs.d.ts +21 -0
- package/dist/collapse-anonymous-divs.js +42 -0
- package/dist/complete-linkage-dendrogram.d.ts +41 -0
- package/dist/complete-linkage-dendrogram.js +140 -0
- package/dist/derive-comparison-sets.d.ts +22 -0
- package/dist/derive-comparison-sets.js +33 -0
- package/dist/derive-path-cluster-keys.d.ts +53 -0
- package/dist/derive-path-cluster-keys.js +109 -0
- package/dist/extract-landmarks.d.ts +91 -45
- package/dist/extract-landmarks.js +122 -41
- package/dist/filter-first-party-stylesheet-hrefs.d.ts +58 -24
- package/dist/filter-first-party-stylesheet-hrefs.js +72 -33
- package/dist/find-shallowest-elements.d.ts +48 -11
- package/dist/find-shallowest-elements.js +41 -21
- package/dist/merge-cross-block-clusters.d.ts +61 -0
- package/dist/merge-cross-block-clusters.js +546 -0
- package/dist/pass0-blocking.d.ts +89 -0
- package/dist/pass0-blocking.js +87 -0
- package/dist/per-page-landmark-signatures.d.ts +48 -0
- package/dist/per-page-landmark-signatures.js +62 -0
- package/dist/reservoir-sample.d.ts +43 -0
- package/dist/reservoir-sample.js +98 -0
- package/dist/resolve-blocking-group-keys.d.ts +8 -2
- package/dist/resolve-blocking-group-keys.js +18 -4
- package/dist/resolve-landmark-variant-keys.d.ts +41 -20
- package/dist/resolve-landmark-variant-keys.js +69 -26
- package/dist/resolve-page-cluster-keys.d.ts +292 -191
- package/dist/resolve-page-cluster-keys.js +708 -157
- package/dist/resolve-structural-cluster-keys.d.ts +9 -0
- package/dist/resolve-structural-cluster-keys.js +14 -232
- package/dist/shape-token.d.ts +11 -0
- package/dist/shape-token.js +38 -0
- package/dist/stage-a-per-block.d.ts +133 -0
- package/dist/stage-a-per-block.js +178 -0
- package/dist/tokenize.d.ts +6 -0
- package/dist/tokenize.js +6 -0
- package/package.json +6 -59
- package/dist/html-region-utils.d.ts +0 -74
- package/dist/html-region-utils.js +0 -96
- package/dist/merge-landmark-affined-clusters.d.ts +0 -179
- package/dist/merge-landmark-affined-clusters.js +0 -544
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 D-ZERO Co., Ltd.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# `@d-zero/page-cluster`
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
大量クロール HTML の重複・類似ページを構造トークンで検出するパッケージ。HTML ページ集合を受け取って、**同一テンプレートと判定できるページ**に同じクラスタキーを振る。テキストは無視して DOM 構造だけを見るので、本文が違っても同じテンプレートを使うページ群は 1 つのクラスタにまとまる。単一サイトで数万〜十数万ページ規模のクロール成果物を、テンプレート単位に畳んで概観したいときに使う。CLI が主、ライブラリ関数群がオマケ。
|
|
4
4
|
|
|
5
5
|
## Installation
|
|
6
6
|
|
|
@@ -8,61 +8,153 @@
|
|
|
8
8
|
yarn add @d-zero/page-cluster
|
|
9
9
|
```
|
|
10
10
|
|
|
11
|
+
インストールすると `page-cluster` コマンドが `node_modules/.bin/` 配下に入る。
|
|
12
|
+
|
|
11
13
|
## Usage
|
|
12
14
|
|
|
13
|
-
|
|
14
|
-
import { tokenize } from '@d-zero/page-cluster';
|
|
15
|
+
### CLI
|
|
15
16
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
);
|
|
19
|
-
// tokens: ["body>.card>ul>li", "body>.card>ul>li"]
|
|
20
|
-
// bodyClassList: ["law-page"]
|
|
17
|
+
```sh
|
|
18
|
+
page-cluster [--content-block-attribute <name>] < pages.jsonl > clusters.jsonl
|
|
21
19
|
```
|
|
22
20
|
|
|
23
|
-
|
|
21
|
+
**入力**: JSONL 1 行 1 ページ。フィールドは以下。`html` 以外はすべて任意（`paths` / `stylesheetHrefs` がないと粗い分類になる）。
|
|
24
22
|
|
|
25
|
-
```
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
23
|
+
```json
|
|
24
|
+
{
|
|
25
|
+
"id": "任意の識別子",
|
|
26
|
+
"html": "<html>...</html>",
|
|
27
|
+
"paths": ["news", "1"],
|
|
28
|
+
"stylesheetHrefs": ["/a.css"],
|
|
29
|
+
"host": "example.com"
|
|
30
|
+
}
|
|
30
31
|
```
|
|
31
32
|
|
|
32
|
-
|
|
33
|
+
**出力**: JSONL 1 行 1 ページ、入力順。
|
|
33
34
|
|
|
34
|
-
|
|
35
|
+
```json
|
|
36
|
+
{ "id": "任意の識別子", "clusterKey": "..." }
|
|
37
|
+
```
|
|
35
38
|
|
|
36
|
-
|
|
39
|
+
クローラ出力が JSON 配列の場合は `jq` で line-delimited に変換して食わせる:
|
|
37
40
|
|
|
38
|
-
|
|
41
|
+
```sh
|
|
42
|
+
jq -c '.[]' crawl-output.json | page-cluster > clusters.jsonl
|
|
43
|
+
```
|
|
39
44
|
|
|
40
|
-
|
|
45
|
+
#### オプション
|
|
46
|
+
|
|
47
|
+
- `--content-block-attribute <name>` — CMS が自由編集コンテンツブロックに付与している属性名（例: `data-bgb`）が分かっている場合に指定する。指定すると比較前にその属性を持つ要素配下を無視するので、同じテンプレートで本文構成だけ違うページを混同しなくなる。唯一の site-specific なオプションで、未指定でも `<main>` / `role="main"` を起点にした自動深さキャップが常時働く（詳細は `resolve-page-cluster-keys.ts` の JSDoc を参照）
|
|
48
|
+
- `--help` / `-h` — ヘルプを表示する
|
|
49
|
+
- `--version` / `-v` — バージョンを表示する
|
|
50
|
+
|
|
51
|
+
#### 進捗表示
|
|
52
|
+
|
|
53
|
+
処理中は stderr に進捗を出す。stdout の JSONL 出力は影響を受けない。
|
|
54
|
+
|
|
55
|
+
**対話端末（TTY）**: アニメーション付きの単一ヘッダー行が in-place に書き換わり、現在のフェーズ・進捗・経過時間を表示する。
|
|
56
|
+
|
|
57
|
+
```
|
|
58
|
+
🌏 page-cluster — clustering 12/47 blocks (elapsed 23s)
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
**非 TTY（パイプ・ファイルリダイレクト・CI）**: `[page-cluster] ...` 形式の行を追記する。`pass0:` / `pass1:` / `pass1b:` / `stage-b:` の phase トークンを含むので `grep` / `awk` 互換。
|
|
41
62
|
|
|
42
|
-
```ts
|
|
43
|
-
import { resolvePageClusterKeys } from '@d-zero/page-cluster/resolve-page-cluster-keys';
|
|
44
|
-
|
|
45
|
-
const keys = resolvePageClusterKeys(
|
|
46
|
-
pages.map((page) => ({
|
|
47
|
-
paths: page.urlPathSegments,
|
|
48
|
-
stylesheetHrefs: page.stylesheetHrefs,
|
|
49
|
-
html: page.html,
|
|
50
|
-
})),
|
|
51
|
-
{ contentBlockAttribute: 'data-bgb' }, // 使っているCMSのブロック属性名に合わせて指定
|
|
52
|
-
);
|
|
53
|
-
// pagesと同じ順序・同じ長さ。同じキーのページが同一テンプレートと判定されたページ群
|
|
54
63
|
```
|
|
64
|
+
[page-cluster] reading input pages...
|
|
65
|
+
[page-cluster] read 10000 pages, clustering...
|
|
66
|
+
[page-cluster] pass0: 10000 pages read
|
|
67
|
+
[page-cluster] pass1: clustered block 12/47
|
|
68
|
+
[page-cluster] pass1b: 30000/70000 pages assigned
|
|
69
|
+
[page-cluster] stage-b: merging 47 units
|
|
70
|
+
[page-cluster] done — 10000 pages in 47 clusters (elapsed 87s)
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
silence したい場合は `2>/dev/null`。ログに残したい場合は `2> progress.log`。
|
|
55
74
|
|
|
56
|
-
###
|
|
75
|
+
### Library
|
|
57
76
|
|
|
58
|
-
|
|
77
|
+
サブパスエクスポート構成。import パスと提供関数の対応は以下。
|
|
78
|
+
|
|
79
|
+
| import パス | 提供関数 |
|
|
80
|
+
| ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
81
|
+
| `@d-zero/page-cluster` | `tokenize` — `<body>` 配下を構造トークン列に変換する低レベルプリミティブ |
|
|
82
|
+
| `@d-zero/page-cluster/resolve-page-cluster-keys` | `resolvePageClusterKeys`（非同期・ファクトリ入力・メモリ有界のメインエントリー）、`resolvePageClusterKeysFromArray`（array 入力ラッパー）、`resolvePageClusterKeysInMemory`（同期・array 入力） |
|
|
83
|
+
| `@d-zero/page-cluster/extract-landmarks` | `extractLandmarks` — 6 種の HTML5 ランドマーク（header / footer / nav / aside / form / search）を抽出 |
|
|
84
|
+
| `@d-zero/page-cluster/resolve-landmark-variant-keys` | `resolveLandmarkVariantKeys` — 特定ランドマークのデザインバリアントでページを分類 |
|
|
59
85
|
|
|
60
86
|
```ts
|
|
61
|
-
import {
|
|
87
|
+
import { resolvePageClusterKeysFromArray } from '@d-zero/page-cluster/resolve-page-cluster-keys';
|
|
88
|
+
|
|
89
|
+
const keys = await resolvePageClusterKeysFromArray([
|
|
90
|
+
{
|
|
91
|
+
paths: ['news', '1'],
|
|
92
|
+
stylesheetHrefs: ['/a.css'],
|
|
93
|
+
html: '<body><article>one</article></body>',
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
paths: ['news', '2'],
|
|
97
|
+
stylesheetHrefs: ['/a.css'],
|
|
98
|
+
html: '<body><article>two</article></body>',
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
paths: ['about'],
|
|
102
|
+
stylesheetHrefs: ['/a.css'],
|
|
103
|
+
html: '<body><section>about</section></body>',
|
|
104
|
+
},
|
|
105
|
+
]);
|
|
106
|
+
// keys[0] === keys[1]（同一テンプレート）、keys[2] は別クラスタ
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
オプション・型・設計判断の WHY はすべて各関数の JSDoc に記載している。CLI 経由で十分な場合は読み飛ばして OK。
|
|
110
|
+
|
|
111
|
+
## アルゴリズム概観
|
|
112
|
+
|
|
113
|
+
`clusterKey` がどう決まるかを知っておくと、出力の解釈（なぜこの 2 ページが同じキーなのか）とオプションの選択がしやすくなる。実装詳細の WHY は各ソースファイルの JSDoc が正。
|
|
114
|
+
|
|
115
|
+
### 全体パイプライン
|
|
62
116
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
117
|
+
```mermaid
|
|
118
|
+
flowchart TD
|
|
119
|
+
IN[入力ページ集合] --> P0["Pass 0: ブロッキング<br>URL パス + first-party CSS 集合 → blockKey<br>（orphan ページの再割当を含む）"]
|
|
120
|
+
P0 --> GATE{"ページ数 ≤ 20,000?"}
|
|
121
|
+
|
|
122
|
+
GATE -- "yes（in-memory）" --> CHROME_ALL["chrome discovery（コーパス全体）<br>ランドマーク署名の度数分布に auto-cut<br>→ グローバル chrome 除外 / ローカル chrome 再注入"]
|
|
123
|
+
CHROME_ALL --> SA_ALL["Stage A × 全ブロック<br>深さキャップ → tokenize →<br>complete-linkage + auto-cut → 包含割当"]
|
|
124
|
+
|
|
125
|
+
GATE -- "no（ストリーミング）" --> RES["ブロックごとにリザーバサンプリング<br>（各ブロック最大 100 ページ、決定的シード）"]
|
|
126
|
+
RES --> SA_SAMPLE["chrome discovery + Stage A<br>（サンプルのみ、ブロック単位で逐次 flush）"]
|
|
127
|
+
SA_SAMPLE --> P1B["Pass 1b: 非サンプルページを<br>max-Jaccard で最寄りクラスタへ割当"]
|
|
128
|
+
|
|
129
|
+
SA_ALL --> SB["Stage B: ブロック越えマージ<br>（不動点ループ、下図）"]
|
|
130
|
+
P1B --> SB
|
|
131
|
+
SB --> OUT["clusterKey を入力順に出力"]
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
- **Pass 0（ブロッキング）** — HTML を読まず、URL パスと first-party stylesheet 集合だけで粗く分割する。高価な構造比較を同一ブロック内に閉じ込め、コーパス全体の比較コストを O(n²) から劇的に減らす。stylesheet を持たない orphan ページは同一セクションの CSS ブロックへ再割当される
|
|
135
|
+
- **chrome discovery** — 全ページのランドマーク署名の度数分布に auto-cut を当て、閾値以上を「グローバル chrome」（サイト共通のヘッダー等）として比較から除外し、閾値未満かつ 2 ページ以上に出現するものを「ローカル chrome」（セクション固有のナビ等）としてトークン再注入する
|
|
136
|
+
- **Stage A（ブロック内クラスタリング）** — ブロックごとに直線的な処理。`<main>` の深さキャップ（候補深度を全走査して knee を探す自動選択）→ tokenize → complete-linkage 階層クラスタリング → max-gap auto-cut でカット高を決定 → 最後に包含関係にあるクラスタを吸収する包含割当（割当チェーンを辿り、循環はメンバー最大のクラスタをルートに選んで解決）
|
|
137
|
+
- **Pass 1b（ストリーミング時のみ）** — 20,000 ページ超では各ブロックをリザーバサンプリング（最大 100 ページ、ブロックキーをシードにした決定的乱数）で代表させ、サンプル外のページは Stage A 完了後に max-Jaccard で最寄りクラスタへ一括割当する。メモリ使用量はコーパス全体ではなくサンプルサイズに比例する
|
|
138
|
+
- **Stage B（ブロック越えマージ）** — ブロック分割はあくまで比較コスト削減のためなので、最後に同一テンプレートがブロックを跨いで分かれていないか再統合する。これが唯一の反復処理（次節）
|
|
139
|
+
|
|
140
|
+
### Stage B: ブロック越え統合の不動点ループ
|
|
141
|
+
|
|
142
|
+
```mermaid
|
|
143
|
+
flowchart TD
|
|
144
|
+
START["ラウンド開始（最大 10 ラウンド）"] --> CORE["現在のプール済みメンバーから再計算:<br>文書頻度 → distinctive tokens → quorum core（80%）"]
|
|
145
|
+
CORE --> FINE["fine stage（単一 union-find 上で 3 経路）:<br>① complete-linkage（固定 0.8）<br>② 包含割当（0.9、チェーン走査 + サイクル解決）<br>③ shape-Jaccard（0.9、複数ページユニットのみ）"]
|
|
146
|
+
FINE --> Q1{"fine でマージ発生?"}
|
|
147
|
+
Q1 -- yes --> APPLY1["マージ適用（メンバー統合）"]
|
|
148
|
+
APPLY1 --> START
|
|
149
|
+
Q1 -- no --> L2["L2 stage:<br>L2 signature 包含 + shell 相互裏付け<br>（shell は auto-cut で自己発見）"]
|
|
150
|
+
L2 --> Q2{"L2 でマージ発生?"}
|
|
151
|
+
Q2 -- yes --> APPLY2["マージ適用"]
|
|
152
|
+
APPLY2 --> START
|
|
153
|
+
Q2 -- no --> DONE["収束 — 全ユニットのキーが不動点に到達"]
|
|
68
154
|
```
|
|
155
|
+
|
|
156
|
+
マージが起きるとユニットのメンバー構成が変わり、文書頻度も quorum core も変わる。そのため毎ラウンド、統合後のプールから全指標を**再計算**してマージを再試行する。fine stage・L2 stage の両方でマージが 1 件も出なくなった時点で不動点に到達したとみなして収束する（安全弁として最大 10 ラウンド。実データでは 7 ラウンド以内に収束）。L2 stage は fine stage が空振りしたラウンドでしか実行されない最後の粗い経路で、誤マージ防止のために shell（ランドマーク由来トークン）の相互裏付けを要求する。
|
|
157
|
+
|
|
158
|
+
### Self-tuning
|
|
159
|
+
|
|
160
|
+
閾値の多くは **max-gap auto-cut**（度数分布の隣接ギャップ最大の中点を境界とする）でデータから自己発見される。① Stage A のカット高、② Stage B の shell 判定、③ chrome discovery のグローバル/ローカル判定、④ Pass 0 の URL パス深さ選択、の 4 箇所で同一プリミティブを再利用しているので、サイトごとにハイパーパラメータをチューニングする必要はない。詳細は `autoCutThreshold` の JSDoc を参照。例外的に Stage B fine stage の complete-linkage だけは固定閾値 0.8 を使う（理由は `merge-cross-block-clusters.ts` の JSDoc を参照）。
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One cluster's entry for containment comparison. `tokens` is the union of
|
|
3
|
+
* comparison-set tokens for all pages in the cluster (Stage A uses
|
|
4
|
+
* frequency-narrowed comparison sets; Stage B uses quorum cores).
|
|
5
|
+
* `pageCount` is used only as a tiebreaker when containment and union size
|
|
6
|
+
* are tied between two candidate targets.
|
|
7
|
+
*/
|
|
8
|
+
export type ContainedClusterEntry = {
|
|
9
|
+
readonly id: number;
|
|
10
|
+
readonly tokens: ReadonlySet<string>;
|
|
11
|
+
readonly pageCount: number;
|
|
12
|
+
};
|
|
13
|
+
/**
|
|
14
|
+
* Assigns each cluster to the "best" cluster that contains it (i.e., whose
|
|
15
|
+
* token set subsumes the cluster's token union at `>= CONTAINMENT_CUTOFF`).
|
|
16
|
+
* Returns a `Map<id, rootId>` — clusters not assigned to anything map to
|
|
17
|
+
* themselves.
|
|
18
|
+
*
|
|
19
|
+
* This is a *directed* assignment (not union-find): X is absorbed by Y but Y
|
|
20
|
+
* is not absorbed by X, unless Y is independently assigned elsewhere too.
|
|
21
|
+
* This prevents hub-chaining: if /help/ is a structural superset of many
|
|
22
|
+
* clusters (because its HTML includes every nav variant), each of those
|
|
23
|
+
* clusters gets assigned to /help/, but they don't get merged with *each
|
|
24
|
+
* other* — only with /help/. Confirmed on real crawl data: a pure union-find
|
|
25
|
+
* approach produced a 9-cluster hub chain through one common-superset page.
|
|
26
|
+
*
|
|
27
|
+
* Principle: conditional rendering only *removes* elements from a template
|
|
28
|
+
* (an empty section, a missing paginator) — it never adds. So a
|
|
29
|
+
* conditionally-shorter page is always a structural subset of the
|
|
30
|
+
* full-featured template. Confirmed on real crawl data: a 43-page works
|
|
31
|
+
* cluster contained a 3-page outlier cluster at containment 1.000.
|
|
32
|
+
*
|
|
33
|
+
* Best target selection: highest containment → largest union size → most
|
|
34
|
+
* pages. Chain resolution and cycle breaking are applied after all raw
|
|
35
|
+
* assignments are computed (see the implementation).
|
|
36
|
+
*
|
|
37
|
+
* Cycles (mutual containment ≥ 0.9) mean the two clusters are practically
|
|
38
|
+
* identical token sets. The one with the larger token set (more pages as
|
|
39
|
+
* tiebreaker) becomes the root of the cycle.
|
|
40
|
+
* @param clusters
|
|
41
|
+
*/
|
|
42
|
+
export declare function assignContainedClusters(clusters: readonly ContainedClusterEntry[]): Map<number, number>;
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimum containment fraction for cluster X to be assigned into cluster Y.
|
|
3
|
+
* Containment is `|X ∩ Y| / |X|` — the fraction of X's tokens also present
|
|
4
|
+
* in Y's token set. At this threshold, X is "almost a subset" of Y.
|
|
5
|
+
*
|
|
6
|
+
* 0.9 was chosen because the observed containment scores for genuine
|
|
7
|
+
* template-subset relationships (e.g. a page missing one optional section)
|
|
8
|
+
* cluster tightly between 1.000 and 0.90–0.92 on real crawl data, while
|
|
9
|
+
* unrelated clusters score well below 0.8. A gap exists between ~0.92 and
|
|
10
|
+
* ~0.80 in practice, so the exact value within that gap is not sensitive.
|
|
11
|
+
*/
|
|
12
|
+
const CONTAINMENT_CUTOFF = 0.9;
|
|
13
|
+
/**
|
|
14
|
+
* Assigns each cluster to the "best" cluster that contains it (i.e., whose
|
|
15
|
+
* token set subsumes the cluster's token union at `>= CONTAINMENT_CUTOFF`).
|
|
16
|
+
* Returns a `Map<id, rootId>` — clusters not assigned to anything map to
|
|
17
|
+
* themselves.
|
|
18
|
+
*
|
|
19
|
+
* This is a *directed* assignment (not union-find): X is absorbed by Y but Y
|
|
20
|
+
* is not absorbed by X, unless Y is independently assigned elsewhere too.
|
|
21
|
+
* This prevents hub-chaining: if /help/ is a structural superset of many
|
|
22
|
+
* clusters (because its HTML includes every nav variant), each of those
|
|
23
|
+
* clusters gets assigned to /help/, but they don't get merged with *each
|
|
24
|
+
* other* — only with /help/. Confirmed on real crawl data: a pure union-find
|
|
25
|
+
* approach produced a 9-cluster hub chain through one common-superset page.
|
|
26
|
+
*
|
|
27
|
+
* Principle: conditional rendering only *removes* elements from a template
|
|
28
|
+
* (an empty section, a missing paginator) — it never adds. So a
|
|
29
|
+
* conditionally-shorter page is always a structural subset of the
|
|
30
|
+
* full-featured template. Confirmed on real crawl data: a 43-page works
|
|
31
|
+
* cluster contained a 3-page outlier cluster at containment 1.000.
|
|
32
|
+
*
|
|
33
|
+
* Best target selection: highest containment → largest union size → most
|
|
34
|
+
* pages. Chain resolution and cycle breaking are applied after all raw
|
|
35
|
+
* assignments are computed (see the implementation).
|
|
36
|
+
*
|
|
37
|
+
* Cycles (mutual containment ≥ 0.9) mean the two clusters are practically
|
|
38
|
+
* identical token sets. The one with the larger token set (more pages as
|
|
39
|
+
* tiebreaker) becomes the root of the cycle.
|
|
40
|
+
* @param clusters
|
|
41
|
+
*/
|
|
42
|
+
export function assignContainedClusters(clusters) {
|
|
43
|
+
// Phase 1: find best raw assignment for each cluster
|
|
44
|
+
const raw = new Map();
|
|
45
|
+
for (const x of clusters) {
|
|
46
|
+
if (x.tokens.size === 0)
|
|
47
|
+
continue;
|
|
48
|
+
let bestTargetId = -1;
|
|
49
|
+
let bestContainment = CONTAINMENT_CUTOFF - 1e-9;
|
|
50
|
+
let bestUnionSize = 0;
|
|
51
|
+
let bestPageCount = 0;
|
|
52
|
+
for (const y of clusters) {
|
|
53
|
+
if (y.id === x.id)
|
|
54
|
+
continue;
|
|
55
|
+
let intersection = 0;
|
|
56
|
+
for (const token of x.tokens) {
|
|
57
|
+
if (y.tokens.has(token))
|
|
58
|
+
intersection++;
|
|
59
|
+
}
|
|
60
|
+
const containment = intersection / x.tokens.size;
|
|
61
|
+
if (containment < CONTAINMENT_CUTOFF)
|
|
62
|
+
continue;
|
|
63
|
+
const unionSize = x.tokens.size + y.tokens.size - intersection;
|
|
64
|
+
if (containment > bestContainment ||
|
|
65
|
+
(containment === bestContainment && unionSize > bestUnionSize) ||
|
|
66
|
+
(containment === bestContainment &&
|
|
67
|
+
unionSize === bestUnionSize &&
|
|
68
|
+
y.pageCount > bestPageCount)) {
|
|
69
|
+
bestContainment = containment;
|
|
70
|
+
bestUnionSize = unionSize;
|
|
71
|
+
bestPageCount = y.pageCount;
|
|
72
|
+
bestTargetId = y.id;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
if (bestTargetId >= 0) {
|
|
76
|
+
raw.set(x.id, bestTargetId);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
// Phase 2: resolve chains and cycles
|
|
80
|
+
// Walk the raw assignment chain from each node; detect cycles by tracking
|
|
81
|
+
// the path walked so far.
|
|
82
|
+
const resolved = new Map();
|
|
83
|
+
const idToEntry = new Map(clusters.map((cl) => [cl.id, cl]));
|
|
84
|
+
for (const c of clusters) {
|
|
85
|
+
if (resolved.has(c.id))
|
|
86
|
+
continue;
|
|
87
|
+
const path = [];
|
|
88
|
+
const pathSet = new Set();
|
|
89
|
+
let current = c.id;
|
|
90
|
+
// Walk until we reach a node with no further assignment or a cycle
|
|
91
|
+
while (!resolved.has(current)) {
|
|
92
|
+
const next = raw.get(current);
|
|
93
|
+
if (next === undefined) {
|
|
94
|
+
// No assignment → current is a root
|
|
95
|
+
resolved.set(current, current);
|
|
96
|
+
break;
|
|
97
|
+
}
|
|
98
|
+
if (pathSet.has(next)) {
|
|
99
|
+
// Cycle detected: find the cycle members and pick the root.
|
|
100
|
+
// Include `current` (the node that closed the back-edge) so it
|
|
101
|
+
// participates in root selection even if it has the largest token set.
|
|
102
|
+
const cycleStart = path.indexOf(next);
|
|
103
|
+
const cycleIds = [...path.slice(cycleStart), current];
|
|
104
|
+
// Root of the cycle: cluster with the largest token set
|
|
105
|
+
// (page count as tiebreaker)
|
|
106
|
+
let cycleRoot = cycleIds[0] ?? current;
|
|
107
|
+
for (const id of cycleIds) {
|
|
108
|
+
const bc = idToEntry.get(cycleRoot);
|
|
109
|
+
const cc = idToEntry.get(id);
|
|
110
|
+
const bcSize = bc?.tokens.size ?? 0;
|
|
111
|
+
const ccSize = cc?.tokens.size ?? 0;
|
|
112
|
+
if (ccSize > bcSize) {
|
|
113
|
+
cycleRoot = id;
|
|
114
|
+
}
|
|
115
|
+
else if (ccSize === bcSize && (cc?.pageCount ?? 0) > (bc?.pageCount ?? 0)) {
|
|
116
|
+
cycleRoot = id;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
for (const id of cycleIds) {
|
|
120
|
+
resolved.set(id, cycleRoot);
|
|
121
|
+
}
|
|
122
|
+
// Everything before the cycle resolves to the cycle root too
|
|
123
|
+
for (const id of path.slice(0, cycleStart)) {
|
|
124
|
+
resolved.set(id, cycleRoot);
|
|
125
|
+
}
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
128
|
+
if (resolved.has(next)) {
|
|
129
|
+
// Already resolved — propagate to everything in path
|
|
130
|
+
const root = resolved.get(next) ?? next;
|
|
131
|
+
resolved.set(current, root);
|
|
132
|
+
for (const id of path) {
|
|
133
|
+
resolved.set(id, root);
|
|
134
|
+
}
|
|
135
|
+
break;
|
|
136
|
+
}
|
|
137
|
+
path.push(current);
|
|
138
|
+
pathSet.add(current);
|
|
139
|
+
current = next;
|
|
140
|
+
}
|
|
141
|
+
// If the path didn't resolve in the loop above, propagate what we know
|
|
142
|
+
if (!resolved.has(c.id) && resolved.has(current)) {
|
|
143
|
+
const root = resolved.get(current) ?? current;
|
|
144
|
+
for (const id of path) {
|
|
145
|
+
resolved.set(id, root);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
// Ensure every cluster id has an entry (fallback to self)
|
|
150
|
+
for (const c of clusters) {
|
|
151
|
+
if (!resolved.has(c.id)) {
|
|
152
|
+
resolved.set(c.id, c.id);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return resolved;
|
|
156
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Finds the largest gap between adjacent merge heights and returns the
|
|
3
|
+
* midpoint of that gap as the cut threshold, clamped to `[0, upperBound]`.
|
|
4
|
+
*
|
|
5
|
+
* The clamp prevents the auto-cut from selecting a value *above* `upperBound`
|
|
6
|
+
* (the caller's intended default): this function only ever *loosens* the
|
|
7
|
+
* threshold relative to the default, never tightens it. Confirmed on real
|
|
8
|
+
* crawl data (8,936-page corpus): without the clamp, an 814-page block's
|
|
9
|
+
* auto-cut selected 0.952 — above the default 0.8 — and turned 46 clusters
|
|
10
|
+
* into 54; with the clamp it stays at 0.8 and the result is unchanged.
|
|
11
|
+
*
|
|
12
|
+
* Falls back to `upperBound` when the heights array has fewer than 2 entries
|
|
13
|
+
* (no gap to measure) or when all heights are equal (no gap exists).
|
|
14
|
+
* @param heights Merge heights from the dendrogram, in any order.
|
|
15
|
+
* @param upperBound Maximum allowed threshold (the caller's default).
|
|
16
|
+
*/
|
|
17
|
+
export declare function autoCutThreshold(heights: readonly number[], upperBound: number): number;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Finds the largest gap between adjacent merge heights and returns the
|
|
3
|
+
* midpoint of that gap as the cut threshold, clamped to `[0, upperBound]`.
|
|
4
|
+
*
|
|
5
|
+
* The clamp prevents the auto-cut from selecting a value *above* `upperBound`
|
|
6
|
+
* (the caller's intended default): this function only ever *loosens* the
|
|
7
|
+
* threshold relative to the default, never tightens it. Confirmed on real
|
|
8
|
+
* crawl data (8,936-page corpus): without the clamp, an 814-page block's
|
|
9
|
+
* auto-cut selected 0.952 — above the default 0.8 — and turned 46 clusters
|
|
10
|
+
* into 54; with the clamp it stays at 0.8 and the result is unchanged.
|
|
11
|
+
*
|
|
12
|
+
* Falls back to `upperBound` when the heights array has fewer than 2 entries
|
|
13
|
+
* (no gap to measure) or when all heights are equal (no gap exists).
|
|
14
|
+
* @param heights Merge heights from the dendrogram, in any order.
|
|
15
|
+
* @param upperBound Maximum allowed threshold (the caller's default).
|
|
16
|
+
*/
|
|
17
|
+
export function autoCutThreshold(heights, upperBound) {
|
|
18
|
+
if (heights.length < 2) {
|
|
19
|
+
return upperBound;
|
|
20
|
+
}
|
|
21
|
+
const sorted = [...heights].toSorted((a, b) => b - a);
|
|
22
|
+
let maxGap = 0;
|
|
23
|
+
let gapIndex = 0;
|
|
24
|
+
for (let i = 0; i < sorted.length - 1; i++) {
|
|
25
|
+
const gap = (sorted[i] ?? 0) - (sorted[i + 1] ?? 0);
|
|
26
|
+
if (gap > maxGap) {
|
|
27
|
+
maxGap = gap;
|
|
28
|
+
gapIndex = i;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
if (maxGap === 0) {
|
|
32
|
+
return upperBound;
|
|
33
|
+
}
|
|
34
|
+
const midpoint = ((sorted[gapIndex] ?? 0) + (sorted[gapIndex + 1] ?? 0)) / 2;
|
|
35
|
+
return Math.min(midpoint, upperBound);
|
|
36
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic identity of a token set.
|
|
3
|
+
*
|
|
4
|
+
* Uses `JSON.stringify` on the sorted tokens rather than a space-joined
|
|
5
|
+
* string so that pathological (but real) tokens containing spaces can't
|
|
6
|
+
* collide with a different set on the identical joined string `"a b c"`.
|
|
7
|
+
* `JSON.stringify` escapes each array element as its own quoted string, so
|
|
8
|
+
* no element's content can ever be mistaken for the array's structural
|
|
9
|
+
* delimiters.
|
|
10
|
+
*
|
|
11
|
+
* Shared by any pipeline stage that needs two token sets with the exact
|
|
12
|
+
* same members to hash to the same key — e.g.
|
|
13
|
+
* {@link ./merge-cross-block-clusters.js | mergeCrossBlockClusters}'s
|
|
14
|
+
* per-landmark-instance frequency histogram construction.
|
|
15
|
+
* @param tokens
|
|
16
|
+
*/
|
|
17
|
+
export declare function canonicalizeTokenSet(tokens: ReadonlySet<string>): string;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic identity of a token set.
|
|
3
|
+
*
|
|
4
|
+
* Uses `JSON.stringify` on the sorted tokens rather than a space-joined
|
|
5
|
+
* string so that pathological (but real) tokens containing spaces can't
|
|
6
|
+
* collide with a different set on the identical joined string `"a b c"`.
|
|
7
|
+
* `JSON.stringify` escapes each array element as its own quoted string, so
|
|
8
|
+
* no element's content can ever be mistaken for the array's structural
|
|
9
|
+
* delimiters.
|
|
10
|
+
*
|
|
11
|
+
* Shared by any pipeline stage that needs two token sets with the exact
|
|
12
|
+
* same members to hash to the same key — e.g.
|
|
13
|
+
* {@link ./merge-cross-block-clusters.js | mergeCrossBlockClusters}'s
|
|
14
|
+
* per-landmark-instance frequency histogram construction.
|
|
15
|
+
* @param tokens
|
|
16
|
+
*/
|
|
17
|
+
export function canonicalizeTokenSet(tokens) {
|
|
18
|
+
return JSON.stringify([...tokens].toSorted());
|
|
19
|
+
}
|
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Parsed CLI shape. Kept as a plain record so tests can build it directly
|
|
4
|
+
* without going through the argv parser.
|
|
5
|
+
*/
|
|
6
|
+
type CliArgs = {
|
|
7
|
+
readonly contentBlockAttribute?: string;
|
|
8
|
+
readonly help?: boolean;
|
|
9
|
+
readonly version?: boolean;
|
|
10
|
+
readonly unknownFlag?: string;
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* Parses `process.argv`-style arguments (already sliced past `node script`)
|
|
14
|
+
* into a `CliArgs`. Deliberately tolerant of an unknown flag so the caller
|
|
15
|
+
* can decide the error message shape, and so tests can assert on the
|
|
16
|
+
* unrecognized flag name directly.
|
|
17
|
+
* @param argv
|
|
18
|
+
*/
|
|
19
|
+
export declare function parseArgs(argv: readonly string[]): CliArgs;
|
|
20
|
+
/**
|
|
21
|
+
* Test-friendly entry point: takes the run's stdin/stdout/stderr streams
|
|
22
|
+
* and the parsed CLI flags rather than reading them out of the process
|
|
23
|
+
* globals. `runCli` returns the exit code, allowing the caller (either the
|
|
24
|
+
* top-level `main` here or a spec test) to decide how to signal it.
|
|
25
|
+
* @param options
|
|
26
|
+
* @param options.stdin
|
|
27
|
+
* @param options.stdout
|
|
28
|
+
* @param options.stderr
|
|
29
|
+
* @param options.argv
|
|
30
|
+
* @param options.version
|
|
31
|
+
*/
|
|
32
|
+
export declare function runCli(options: {
|
|
33
|
+
stdin: NodeJS.ReadableStream;
|
|
34
|
+
stdout: NodeJS.WritableStream;
|
|
35
|
+
stderr: NodeJS.WritableStream;
|
|
36
|
+
argv: readonly string[];
|
|
37
|
+
version: string;
|
|
38
|
+
}): Promise<number>;
|
|
39
|
+
export {};
|