@d-zero/page-cluster 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +95 -41
- package/dist/assign-contained-clusters.d.ts +42 -0
- package/dist/assign-contained-clusters.js +156 -0
- package/dist/auto-cut-threshold.d.ts +17 -0
- package/dist/auto-cut-threshold.js +36 -0
- package/dist/canonicalize-token-set.d.ts +17 -0
- package/dist/canonicalize-token-set.js +19 -0
- package/dist/cli.d.ts +39 -0
- package/dist/cli.js +381 -0
- package/dist/collapse-anonymous-divs.d.ts +21 -0
- package/dist/collapse-anonymous-divs.js +42 -0
- package/dist/complete-linkage-dendrogram.d.ts +41 -0
- package/dist/complete-linkage-dendrogram.js +140 -0
- package/dist/derive-comparison-sets.d.ts +22 -0
- package/dist/derive-comparison-sets.js +33 -0
- package/dist/derive-path-cluster-keys.d.ts +53 -0
- package/dist/derive-path-cluster-keys.js +109 -0
- package/dist/extract-landmarks.d.ts +91 -45
- package/dist/extract-landmarks.js +122 -41
- package/dist/filter-first-party-stylesheet-hrefs.d.ts +58 -24
- package/dist/filter-first-party-stylesheet-hrefs.js +72 -33
- package/dist/find-shallowest-elements.d.ts +48 -11
- package/dist/find-shallowest-elements.js +41 -21
- package/dist/merge-cross-block-clusters.d.ts +61 -0
- package/dist/merge-cross-block-clusters.js +546 -0
- package/dist/pass0-blocking.d.ts +89 -0
- package/dist/pass0-blocking.js +87 -0
- package/dist/per-page-landmark-signatures.d.ts +48 -0
- package/dist/per-page-landmark-signatures.js +62 -0
- package/dist/reservoir-sample.d.ts +43 -0
- package/dist/reservoir-sample.js +98 -0
- package/dist/resolve-blocking-group-keys.d.ts +8 -2
- package/dist/resolve-blocking-group-keys.js +18 -4
- package/dist/resolve-landmark-variant-keys.d.ts +41 -20
- package/dist/resolve-landmark-variant-keys.js +69 -26
- package/dist/resolve-page-cluster-keys.d.ts +292 -191
- package/dist/resolve-page-cluster-keys.js +708 -157
- package/dist/resolve-structural-cluster-keys.d.ts +9 -0
- package/dist/resolve-structural-cluster-keys.js +14 -232
- package/dist/shape-token.d.ts +11 -0
- package/dist/shape-token.js +38 -0
- package/dist/stage-a-per-block.d.ts +133 -0
- package/dist/stage-a-per-block.js +178 -0
- package/dist/tokenize.d.ts +6 -0
- package/dist/tokenize.js +6 -0
- package/package.json +5 -58
- package/dist/html-region-utils.d.ts +0 -74
- package/dist/html-region-utils.js +0 -96
- package/dist/merge-landmark-affined-clusters.d.ts +0 -179
- package/dist/merge-landmark-affined-clusters.js +0 -544
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 D-ZERO Co., Ltd.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,68 +1,122 @@
|
|
|
1
1
|
# `@d-zero/page-cluster`
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
大量クロール HTML の重複・類似ページを構造トークンで検出するパッケージ。CLI が主、ライブラリ関数群がオマケ。
|
|
4
4
|
|
|
5
|
-
##
|
|
5
|
+
## What this does
|
|
6
|
+
|
|
7
|
+
`page-cluster` は HTML ページ集合を受け取って、**同一テンプレートと判定できるページ**に同じキーを振る。テキストは無視して DOM 構造だけを見るので、記事本文が違うが同じテンプレートを使うページ群は 1 つのクラスタにまとまる。単一サイトで数万〜十数万ページ規模のクロール成果物を、テンプレート単位に畳んで概観したいときに使う。
|
|
8
|
+
|
|
9
|
+
## Install
|
|
6
10
|
|
|
7
11
|
```sh
|
|
8
12
|
yarn add @d-zero/page-cluster
|
|
9
13
|
```
|
|
10
14
|
|
|
11
|
-
|
|
15
|
+
インストールすると `page-cluster` コマンドが `node_modules/.bin/` 配下に入る。
|
|
12
16
|
|
|
13
|
-
|
|
14
|
-
import { tokenize } from '@d-zero/page-cluster';
|
|
17
|
+
## Quickstart (CLI)
|
|
15
18
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
**入力**: JSONL 1 行 1 ページ。フィールドは以下。`html` 以外はすべて任意（`paths` / `stylesheetHrefs` がないと粗い分類になる）。
|
|
20
|
+
|
|
21
|
+
```json
|
|
22
|
+
{
|
|
23
|
+
"id": "任意の識別子",
|
|
24
|
+
"html": "<html>...</html>",
|
|
25
|
+
"paths": ["news", "1"],
|
|
26
|
+
"stylesheetHrefs": ["/a.css"],
|
|
27
|
+
"host": "example.com"
|
|
28
|
+
}
|
|
21
29
|
```
|
|
22
30
|
|
|
23
|
-
|
|
31
|
+
**出力**: JSONL 1 行 1 ページ、入力順。
|
|
24
32
|
|
|
25
|
-
```
|
|
26
|
-
|
|
27
|
-
filterNoiseClasses: true, // 既定値。ハッシュ的自動生成class名を除外する
|
|
28
|
-
includeComments: false, // 既定値。コメントノードをトークン化しない
|
|
29
|
-
});
|
|
33
|
+
```json
|
|
34
|
+
{ "id": "任意の識別子", "clusterKey": "..." }
|
|
30
35
|
```
|
|
31
36
|
|
|
32
|
-
###
|
|
37
|
+
### クローラ出力（JSON 配列）を JSONL に変換して食わせる
|
|
33
38
|
|
|
34
|
-
|
|
39
|
+
`jq` のワンライナーで配列を line-delimited にする典型例:
|
|
35
40
|
|
|
36
|
-
|
|
41
|
+
```sh
|
|
42
|
+
jq -c '.[]' crawl-output.json | page-cluster > clusters.jsonl
|
|
43
|
+
```
|
|
37
44
|
|
|
38
|
-
|
|
45
|
+
### `--content-block-attribute`
|
|
39
46
|
|
|
40
|
-
|
|
47
|
+
CMS が自由編集コンテンツブロックに付与している属性名（例: `data-bgb`）が分かっている場合に指定する。指定すると比較前にその属性を持つ要素配下を無視するので、同じテンプレートで本文構成だけ違うページを混同しなくなる。
|
|
41
48
|
|
|
42
|
-
```
|
|
43
|
-
|
|
49
|
+
```sh
|
|
50
|
+
page-cluster --content-block-attribute data-bgb < pages.jsonl > clusters.jsonl
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
### 進捗
|
|
54
|
+
|
|
55
|
+
処理中は stderr に進捗を出す。stdout の JSONL 出力は影響を受けない。
|
|
56
|
+
|
|
57
|
+
**対話端末 (TTY)**: `%earth%` アニメ付きの単一ヘッダー行が in-place に書き換わり、現在のフェーズ・進捗・経過時間を表示する。
|
|
44
58
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
paths: page.urlPathSegments,
|
|
48
|
-
stylesheetHrefs: page.stylesheetHrefs,
|
|
49
|
-
html: page.html,
|
|
50
|
-
})),
|
|
51
|
-
{ contentBlockAttribute: 'data-bgb' }, // 使っているCMSのブロック属性名に合わせて指定
|
|
52
|
-
);
|
|
53
|
-
// pagesと同じ順序・同じ長さ。同じキーのページが同一テンプレートと判定されたページ群
|
|
59
|
+
```
|
|
60
|
+
🌏 page-cluster — clustering 12/47 blocks (elapsed 23s)
|
|
54
61
|
```
|
|
55
62
|
|
|
56
|
-
|
|
63
|
+
**非TTY (パイプ・ファイルリダイレクト・CI)**: `[page-cluster] ...` 形式の行を追記する。`pass0:` / `pass1:` / `pass1b:` / `stage-b:` の phase トークンを含むので `grep` / `awk` 互換。
|
|
57
64
|
|
|
58
|
-
|
|
65
|
+
```
|
|
66
|
+
[page-cluster] reading input pages...
|
|
67
|
+
[page-cluster] read 10000 pages, clustering...
|
|
68
|
+
[page-cluster] pass0: 10000 pages read
|
|
69
|
+
[page-cluster] pass1: clustered block 12/47
|
|
70
|
+
[page-cluster] pass1b: 30000/70000 pages assigned
|
|
71
|
+
[page-cluster] stage-b: merging 47 units
|
|
72
|
+
[page-cluster] done — 10000 pages in 47 clusters (elapsed 87s)
|
|
73
|
+
```
|
|
59
74
|
|
|
60
|
-
|
|
61
|
-
import { resolveLandmarkVariantKeys } from '@d-zero/page-cluster/resolve-landmark-variant-keys';
|
|
75
|
+
silence したい場合は `2>/dev/null`。ログに残したい場合は `2> progress.log`。
|
|
62
76
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
77
|
+
## API (brief)
|
|
78
|
+
|
|
79
|
+
すべての詳細は各関数の JSDoc にある。CLI 経由で十分な場合は読み飛ばして OK。
|
|
80
|
+
|
|
81
|
+
- **`tokenize(html, options?)`** — `<body>` 配下の HTML を構造トークン列に変換する低レベルプリミティブ
|
|
82
|
+
- **`resolvePageClusterKeys(pagesFactory, options?)`** — ページ集合からクラスタキーを返すメインエントリー。ファクトリ関数入力で大規模コーパスに対応
|
|
83
|
+
- **`resolvePageClusterKeysFromArray(pages, options?)`** — メモリに全ページ載る前提の array 入力ラッパー
|
|
84
|
+
- **`resolveLandmarkVariantKeys(htmlList, landmarkType, options?)`** — `header` / `footer` / `nav` / `aside` などのランドマークバリアント分類
|
|
85
|
+
- **`extractLandmarks(html)`** — 1 ページから 6 種の HTML5 ランドマーク（header / footer / nav / aside / form / search）を抽出
|
|
86
|
+
|
|
87
|
+
## Algorithm
|
|
88
|
+
|
|
89
|
+
```
|
|
90
|
+
┌────────────────────────────────────────┐
|
|
91
|
+
│ Blocking (paths / stylesheet 集合) │
|
|
92
|
+
└──────────────────┬─────────────────────┘
|
|
93
|
+
│ 同じテンプレートを共有する候補群
|
|
94
|
+
▼
|
|
95
|
+
┌────────────────────────────────────────┐
|
|
96
|
+
│ Stage A: complete-linkage クラスタリング │
|
|
97
|
+
│ (ブロック内、Jaccard 距離) │
|
|
98
|
+
└──────────────────┬─────────────────────┘
|
|
99
|
+
│ 各ブロックのクラスタ代表
|
|
100
|
+
▼
|
|
101
|
+
┌────────────────────────────────────────┐
|
|
102
|
+
│ Stage B: quorum-core cross-block merge │
|
|
103
|
+
│ (ブロック境界を越えた再統合) │
|
|
104
|
+
└────────────────────────────────────────┘
|
|
68
105
|
```
|
|
106
|
+
|
|
107
|
+
- **Blocking** — URL パスと stylesheet 集合を安価なブロッキング信号として粗く分割。同一ブロック内でだけ高価な構造比較を行うので、コーパス全体に対する比較コストを O(n²) から劇的に減らす
|
|
108
|
+
- **Stage A** — ブロック内で `<main>` 配下のトークン列に対して complete-linkage 階層的クラスタリングを実行し、max-gap detection でカット高を選ぶ
|
|
109
|
+
- **Stage B** — 各クラスタの quorum-core（80% クォーラム）を代表としてブロック境界をまたぐ再統合を反復。complete-linkage、包含、shape-Jaccard、L2 signature の 4 経路で融合を試みて不動点まで回す
|
|
110
|
+
- **大規模自動切替** — 20,000 ページ超で自動的に**ストリーミング経路**に切り替わる。ブロックごとにリザーバサンプルで代表を学ばせ、非サンプルページを Jaccard で最寄りクラスタに割当。メモリ使用量が最大ブロックのサイズに比例するようになる
|
|
111
|
+
|
|
112
|
+
### Self-tuning
|
|
113
|
+
|
|
114
|
+
閾値はすべて **max-gap auto-cut**（度数分布の最大ギャップの中点を境界とする）でデータから自己発見される。Stage A のマージ高さカット、Stage B のシェル判定、コーパス全体の共通クローム判定など、3 階層でこの同一プリミティブを再帰使用しているので、サイトごとにハイパーパラメータをチューニングする必要はない。詳細は `autoCutThreshold` の JSDoc を参照。
|
|
115
|
+
|
|
116
|
+
## Notes
|
|
117
|
+
|
|
118
|
+
### `contentBlockAttribute` の存在意義
|
|
119
|
+
|
|
120
|
+
このパッケージが持つ唯一の site-specific なオプション。CMS の自由編集ブロックに付与される属性名（例: `data-bgb`）は HTML から自動検知できないので外部知識として受け取る形にしている。指定された属性を持つ要素の配下は比較対象から除外され、同じテンプレート上で本文構成だけ違うページの誤分割を防ぐ。
|
|
121
|
+
|
|
122
|
+
未指定でも大半のケースで動くよう、`<main>` / `role="main"` を起点にした自動深さキャップが常時有効になっている。
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One cluster's entry for containment comparison. `tokens` is the union of
|
|
3
|
+
* comparison-set tokens for all pages in the cluster (Stage A uses
|
|
4
|
+
* frequency-narrowed comparison sets; Stage B uses quorum cores).
|
|
5
|
+
* `pageCount` is used only as a tiebreaker when containment and union size
|
|
6
|
+
* are tied between two candidate targets.
|
|
7
|
+
*/
|
|
8
|
+
export type ContainedClusterEntry = {
|
|
9
|
+
readonly id: number;
|
|
10
|
+
readonly tokens: ReadonlySet<string>;
|
|
11
|
+
readonly pageCount: number;
|
|
12
|
+
};
|
|
13
|
+
/**
|
|
14
|
+
* Assigns each cluster to the "best" cluster that contains it (i.e., whose
|
|
15
|
+
* token set subsumes the cluster's token union at `>= CONTAINMENT_CUTOFF`).
|
|
16
|
+
* Returns a `Map<id, rootId>` — clusters not assigned to anything map to
|
|
17
|
+
* themselves.
|
|
18
|
+
*
|
|
19
|
+
* This is a *directed* assignment (not union-find): X is absorbed by Y but Y
|
|
20
|
+
* is not absorbed by X, unless Y is independently assigned elsewhere too.
|
|
21
|
+
* This prevents hub-chaining: if /help/ is a structural superset of many
|
|
22
|
+
* clusters (because its HTML includes every nav variant), each of those
|
|
23
|
+
* clusters gets assigned to /help/, but they don't get merged with *each
|
|
24
|
+
* other* — only with /help/. Confirmed on real crawl data: a pure union-find
|
|
25
|
+
* approach produced a 9-cluster hub chain through one common-superset page.
|
|
26
|
+
*
|
|
27
|
+
* Principle: conditional rendering only *removes* elements from a template
|
|
28
|
+
* (an empty section, a missing paginator) — it never adds. So a
|
|
29
|
+
* conditionally-shorter page is always a structural subset of the
|
|
30
|
+
* full-featured template. Confirmed on real crawl data: a 43-page works
|
|
31
|
+
* cluster contained a 3-page outlier cluster at containment 1.000.
|
|
32
|
+
*
|
|
33
|
+
* Best target selection: highest containment → largest union size → most
|
|
34
|
+
* pages. Chain resolution and cycle breaking are applied after all raw
|
|
35
|
+
* assignments are computed (see the implementation).
|
|
36
|
+
*
|
|
37
|
+
* Cycles (mutual containment ≥ 0.9) mean the two clusters are practically
|
|
38
|
+
* identical token sets. The one with the larger token set (more pages as
|
|
39
|
+
* tiebreaker) becomes the root of the cycle.
|
|
40
|
+
* @param clusters
|
|
41
|
+
*/
|
|
42
|
+
export declare function assignContainedClusters(clusters: readonly ContainedClusterEntry[]): Map<number, number>;
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimum containment fraction for cluster X to be assigned into cluster Y.
|
|
3
|
+
* Containment is `|X ∩ Y| / |X|` — the fraction of X's tokens also present
|
|
4
|
+
* in Y's token set. At this threshold, X is "almost a subset" of Y.
|
|
5
|
+
*
|
|
6
|
+
* 0.9 was chosen because the observed containment scores for genuine
|
|
7
|
+
* template-subset relationships (e.g. a page missing one optional section)
|
|
8
|
+
* cluster tightly between 1.000 and 0.90–0.92 on real crawl data, while
|
|
9
|
+
* unrelated clusters score well below 0.8. A gap exists between ~0.92 and
|
|
10
|
+
* ~0.80 in practice, so the exact value within that gap is not sensitive.
|
|
11
|
+
*/
|
|
12
|
+
const CONTAINMENT_CUTOFF = 0.9;
|
|
13
|
+
/**
|
|
14
|
+
* Assigns each cluster to the "best" cluster that contains it (i.e., whose
|
|
15
|
+
* token set subsumes the cluster's token union at `>= CONTAINMENT_CUTOFF`).
|
|
16
|
+
* Returns a `Map<id, rootId>` — clusters not assigned to anything map to
|
|
17
|
+
* themselves.
|
|
18
|
+
*
|
|
19
|
+
* This is a *directed* assignment (not union-find): X is absorbed by Y but Y
|
|
20
|
+
* is not absorbed by X, unless Y is independently assigned elsewhere too.
|
|
21
|
+
* This prevents hub-chaining: if /help/ is a structural superset of many
|
|
22
|
+
* clusters (because its HTML includes every nav variant), each of those
|
|
23
|
+
* clusters gets assigned to /help/, but they don't get merged with *each
|
|
24
|
+
* other* — only with /help/. Confirmed on real crawl data: a pure union-find
|
|
25
|
+
* approach produced a 9-cluster hub chain through one common-superset page.
|
|
26
|
+
*
|
|
27
|
+
* Principle: conditional rendering only *removes* elements from a template
|
|
28
|
+
* (an empty section, a missing paginator) — it never adds. So a
|
|
29
|
+
* conditionally-shorter page is always a structural subset of the
|
|
30
|
+
* full-featured template. Confirmed on real crawl data: a 43-page works
|
|
31
|
+
* cluster contained a 3-page outlier cluster at containment 1.000.
|
|
32
|
+
*
|
|
33
|
+
* Best target selection: highest containment → largest union size → most
|
|
34
|
+
* pages. Chain resolution and cycle breaking are applied after all raw
|
|
35
|
+
* assignments are computed (see the implementation).
|
|
36
|
+
*
|
|
37
|
+
* Cycles (mutual containment ≥ 0.9) mean the two clusters are practically
|
|
38
|
+
* identical token sets. The one with the larger token set (more pages as
|
|
39
|
+
* tiebreaker) becomes the root of the cycle.
|
|
40
|
+
* @param clusters
|
|
41
|
+
*/
|
|
42
|
+
export function assignContainedClusters(clusters) {
|
|
43
|
+
// Phase 1: find best raw assignment for each cluster
|
|
44
|
+
const raw = new Map();
|
|
45
|
+
for (const x of clusters) {
|
|
46
|
+
if (x.tokens.size === 0)
|
|
47
|
+
continue;
|
|
48
|
+
let bestTargetId = -1;
|
|
49
|
+
let bestContainment = CONTAINMENT_CUTOFF - 1e-9;
|
|
50
|
+
let bestUnionSize = 0;
|
|
51
|
+
let bestPageCount = 0;
|
|
52
|
+
for (const y of clusters) {
|
|
53
|
+
if (y.id === x.id)
|
|
54
|
+
continue;
|
|
55
|
+
let intersection = 0;
|
|
56
|
+
for (const token of x.tokens) {
|
|
57
|
+
if (y.tokens.has(token))
|
|
58
|
+
intersection++;
|
|
59
|
+
}
|
|
60
|
+
const containment = intersection / x.tokens.size;
|
|
61
|
+
if (containment < CONTAINMENT_CUTOFF)
|
|
62
|
+
continue;
|
|
63
|
+
const unionSize = x.tokens.size + y.tokens.size - intersection;
|
|
64
|
+
if (containment > bestContainment ||
|
|
65
|
+
(containment === bestContainment && unionSize > bestUnionSize) ||
|
|
66
|
+
(containment === bestContainment &&
|
|
67
|
+
unionSize === bestUnionSize &&
|
|
68
|
+
y.pageCount > bestPageCount)) {
|
|
69
|
+
bestContainment = containment;
|
|
70
|
+
bestUnionSize = unionSize;
|
|
71
|
+
bestPageCount = y.pageCount;
|
|
72
|
+
bestTargetId = y.id;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
if (bestTargetId >= 0) {
|
|
76
|
+
raw.set(x.id, bestTargetId);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
// Phase 2: resolve chains and cycles
|
|
80
|
+
// Walk the raw assignment chain from each node; detect cycles by tracking
|
|
81
|
+
// the path walked so far.
|
|
82
|
+
const resolved = new Map();
|
|
83
|
+
const idToEntry = new Map(clusters.map((cl) => [cl.id, cl]));
|
|
84
|
+
for (const c of clusters) {
|
|
85
|
+
if (resolved.has(c.id))
|
|
86
|
+
continue;
|
|
87
|
+
const path = [];
|
|
88
|
+
const pathSet = new Set();
|
|
89
|
+
let current = c.id;
|
|
90
|
+
// Walk until we reach a node with no further assignment or a cycle
|
|
91
|
+
while (!resolved.has(current)) {
|
|
92
|
+
const next = raw.get(current);
|
|
93
|
+
if (next === undefined) {
|
|
94
|
+
// No assignment → current is a root
|
|
95
|
+
resolved.set(current, current);
|
|
96
|
+
break;
|
|
97
|
+
}
|
|
98
|
+
if (pathSet.has(next)) {
|
|
99
|
+
// Cycle detected: find the cycle members and pick the root.
|
|
100
|
+
// Include `current` (the node that closed the back-edge) so it
|
|
101
|
+
// participates in root selection even if it has the largest token set.
|
|
102
|
+
const cycleStart = path.indexOf(next);
|
|
103
|
+
const cycleIds = [...path.slice(cycleStart), current];
|
|
104
|
+
// Root of the cycle: cluster with the largest token set
|
|
105
|
+
// (page count as tiebreaker)
|
|
106
|
+
let cycleRoot = cycleIds[0] ?? current;
|
|
107
|
+
for (const id of cycleIds) {
|
|
108
|
+
const bc = idToEntry.get(cycleRoot);
|
|
109
|
+
const cc = idToEntry.get(id);
|
|
110
|
+
const bcSize = bc?.tokens.size ?? 0;
|
|
111
|
+
const ccSize = cc?.tokens.size ?? 0;
|
|
112
|
+
if (ccSize > bcSize) {
|
|
113
|
+
cycleRoot = id;
|
|
114
|
+
}
|
|
115
|
+
else if (ccSize === bcSize && (cc?.pageCount ?? 0) > (bc?.pageCount ?? 0)) {
|
|
116
|
+
cycleRoot = id;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
for (const id of cycleIds) {
|
|
120
|
+
resolved.set(id, cycleRoot);
|
|
121
|
+
}
|
|
122
|
+
// Everything before the cycle resolves to the cycle root too
|
|
123
|
+
for (const id of path.slice(0, cycleStart)) {
|
|
124
|
+
resolved.set(id, cycleRoot);
|
|
125
|
+
}
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
128
|
+
if (resolved.has(next)) {
|
|
129
|
+
// Already resolved — propagate to everything in path
|
|
130
|
+
const root = resolved.get(next) ?? next;
|
|
131
|
+
resolved.set(current, root);
|
|
132
|
+
for (const id of path) {
|
|
133
|
+
resolved.set(id, root);
|
|
134
|
+
}
|
|
135
|
+
break;
|
|
136
|
+
}
|
|
137
|
+
path.push(current);
|
|
138
|
+
pathSet.add(current);
|
|
139
|
+
current = next;
|
|
140
|
+
}
|
|
141
|
+
// If the path didn't resolve in the loop above, propagate what we know
|
|
142
|
+
if (!resolved.has(c.id) && resolved.has(current)) {
|
|
143
|
+
const root = resolved.get(current) ?? current;
|
|
144
|
+
for (const id of path) {
|
|
145
|
+
resolved.set(id, root);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
// Ensure every cluster id has an entry (fallback to self)
|
|
150
|
+
for (const c of clusters) {
|
|
151
|
+
if (!resolved.has(c.id)) {
|
|
152
|
+
resolved.set(c.id, c.id);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return resolved;
|
|
156
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Finds the largest gap between adjacent merge heights and returns the
|
|
3
|
+
* midpoint of that gap as the cut threshold, clamped to `[0, upperBound]`.
|
|
4
|
+
*
|
|
5
|
+
* The clamp prevents the auto-cut from selecting a value *above* `upperBound`
|
|
6
|
+
* (the caller's intended default): this function only ever *loosens* the
|
|
7
|
+
* threshold relative to the default, never tightens it. Confirmed on real
|
|
8
|
+
* crawl data (8,936-page corpus): without the clamp, an 814-page block's
|
|
9
|
+
* auto-cut selected 0.952 — above the default 0.8 — and turned 46 clusters
|
|
10
|
+
* into 54; with the clamp it stays at 0.8 and the result is unchanged.
|
|
11
|
+
*
|
|
12
|
+
* Falls back to `upperBound` when the heights array has fewer than 2 entries
|
|
13
|
+
* (no gap to measure) or when all heights are equal (no gap exists).
|
|
14
|
+
* @param heights Merge heights from the dendrogram, in any order.
|
|
15
|
+
* @param upperBound Maximum allowed threshold (the caller's default).
|
|
16
|
+
*/
|
|
17
|
+
export declare function autoCutThreshold(heights: readonly number[], upperBound: number): number;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Finds the largest gap between adjacent merge heights and returns the
|
|
3
|
+
* midpoint of that gap as the cut threshold, clamped to `[0, upperBound]`.
|
|
4
|
+
*
|
|
5
|
+
* The clamp prevents the auto-cut from selecting a value *above* `upperBound`
|
|
6
|
+
* (the caller's intended default): this function only ever *loosens* the
|
|
7
|
+
* threshold relative to the default, never tightens it. Confirmed on real
|
|
8
|
+
* crawl data (8,936-page corpus): without the clamp, an 814-page block's
|
|
9
|
+
* auto-cut selected 0.952 — above the default 0.8 — and turned 46 clusters
|
|
10
|
+
* into 54; with the clamp it stays at 0.8 and the result is unchanged.
|
|
11
|
+
*
|
|
12
|
+
* Falls back to `upperBound` when the heights array has fewer than 2 entries
|
|
13
|
+
* (no gap to measure) or when all heights are equal (no gap exists).
|
|
14
|
+
* @param heights Merge heights from the dendrogram, in any order.
|
|
15
|
+
* @param upperBound Maximum allowed threshold (the caller's default).
|
|
16
|
+
*/
|
|
17
|
+
export function autoCutThreshold(heights, upperBound) {
|
|
18
|
+
if (heights.length < 2) {
|
|
19
|
+
return upperBound;
|
|
20
|
+
}
|
|
21
|
+
const sorted = [...heights].toSorted((a, b) => b - a);
|
|
22
|
+
let maxGap = 0;
|
|
23
|
+
let gapIndex = 0;
|
|
24
|
+
for (let i = 0; i < sorted.length - 1; i++) {
|
|
25
|
+
const gap = (sorted[i] ?? 0) - (sorted[i + 1] ?? 0);
|
|
26
|
+
if (gap > maxGap) {
|
|
27
|
+
maxGap = gap;
|
|
28
|
+
gapIndex = i;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
if (maxGap === 0) {
|
|
32
|
+
return upperBound;
|
|
33
|
+
}
|
|
34
|
+
const midpoint = ((sorted[gapIndex] ?? 0) + (sorted[gapIndex + 1] ?? 0)) / 2;
|
|
35
|
+
return Math.min(midpoint, upperBound);
|
|
36
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic identity of a token set.
|
|
3
|
+
*
|
|
4
|
+
* Uses `JSON.stringify` on the sorted tokens rather than a space-joined
|
|
5
|
+
* string so that pathological (but real) tokens containing spaces can't
|
|
6
|
+
* collide with a different set on the identical joined string `"a b c"`.
|
|
7
|
+
* `JSON.stringify` escapes each array element as its own quoted string, so
|
|
8
|
+
* no element's content can ever be mistaken for the array's structural
|
|
9
|
+
* delimiters.
|
|
10
|
+
*
|
|
11
|
+
* Shared by any pipeline stage that needs two token sets with the exact
|
|
12
|
+
* same members to hash to the same key — e.g.
|
|
13
|
+
* {@link ./merge-cross-block-clusters.js | mergeCrossBlockClusters}'s
|
|
14
|
+
* per-landmark-instance frequency histogram construction.
|
|
15
|
+
* @param tokens
|
|
16
|
+
*/
|
|
17
|
+
export declare function canonicalizeTokenSet(tokens: ReadonlySet<string>): string;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic identity of a token set.
|
|
3
|
+
*
|
|
4
|
+
* Uses `JSON.stringify` on the sorted tokens rather than a space-joined
|
|
5
|
+
* string so that pathological (but real) tokens containing spaces can't
|
|
6
|
+
* collide with a different set on the identical joined string `"a b c"`.
|
|
7
|
+
* `JSON.stringify` escapes each array element as its own quoted string, so
|
|
8
|
+
* no element's content can ever be mistaken for the array's structural
|
|
9
|
+
* delimiters.
|
|
10
|
+
*
|
|
11
|
+
* Shared by any pipeline stage that needs two token sets with the exact
|
|
12
|
+
* same members to hash to the same key — e.g.
|
|
13
|
+
* {@link ./merge-cross-block-clusters.js | mergeCrossBlockClusters}'s
|
|
14
|
+
* per-landmark-instance frequency histogram construction.
|
|
15
|
+
* @param tokens
|
|
16
|
+
*/
|
|
17
|
+
export function canonicalizeTokenSet(tokens) {
|
|
18
|
+
return JSON.stringify([...tokens].toSorted());
|
|
19
|
+
}
|
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Parsed CLI shape. Kept as a plain record so tests can build it directly
|
|
4
|
+
* without going through the argv parser.
|
|
5
|
+
*/
|
|
6
|
+
type CliArgs = {
|
|
7
|
+
readonly contentBlockAttribute?: string;
|
|
8
|
+
readonly help?: boolean;
|
|
9
|
+
readonly version?: boolean;
|
|
10
|
+
readonly unknownFlag?: string;
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* Parses `process.argv`-style arguments (already sliced past `node script`)
|
|
14
|
+
* into a `CliArgs`. Deliberately tolerant of an unknown flag so the caller
|
|
15
|
+
* can decide the error message shape, and so tests can assert on the
|
|
16
|
+
* unrecognized flag name directly.
|
|
17
|
+
* @param argv
|
|
18
|
+
*/
|
|
19
|
+
export declare function parseArgs(argv: readonly string[]): CliArgs;
|
|
20
|
+
/**
|
|
21
|
+
* Test-friendly entry point: takes the run's stdin/stdout/stderr streams
|
|
22
|
+
* and the parsed CLI flags rather than reading them out of the process
|
|
23
|
+
* globals. `runCli` returns the exit code, allowing the caller (either the
|
|
24
|
+
* top-level `main` here or a spec test) to decide how to signal it.
|
|
25
|
+
* @param options
|
|
26
|
+
* @param options.stdin
|
|
27
|
+
* @param options.stdout
|
|
28
|
+
* @param options.stderr
|
|
29
|
+
* @param options.argv
|
|
30
|
+
* @param options.version
|
|
31
|
+
*/
|
|
32
|
+
export declare function runCli(options: {
|
|
33
|
+
stdin: NodeJS.ReadableStream;
|
|
34
|
+
stdout: NodeJS.WritableStream;
|
|
35
|
+
stderr: NodeJS.WritableStream;
|
|
36
|
+
argv: readonly string[];
|
|
37
|
+
version: string;
|
|
38
|
+
}): Promise<number>;
|
|
39
|
+
export {};
|