@nitpicker/core 0.24.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -35
- package/lib/assert-u32-id.d.ts +15 -0
- package/lib/assert-u32-id.js +22 -0
- package/lib/check-native-executor-config.d.ts +21 -0
- package/lib/check-native-executor-config.js +24 -0
- package/lib/compute-alias-keys.d.ts +28 -0
- package/lib/compute-alias-keys.js +30 -0
- package/lib/compute-body-hash.d.ts +22 -0
- package/lib/compute-body-hash.js +25 -0
- package/lib/compute-url-sort-ranks.d.ts +28 -0
- package/lib/compute-url-sort-ranks.js +49 -0
- package/lib/create-isolated-cluster-run.d.ts +35 -0
- package/lib/create-isolated-cluster-run.js +52 -0
- package/lib/create-template-classification-run.d.ts +49 -0
- package/lib/create-template-classification-run.js +386 -0
- package/lib/group-alias-candidates.d.ts +40 -0
- package/lib/group-alias-candidates.js +104 -0
- package/lib/load-native-binding.d.ts +26 -0
- package/lib/load-native-binding.js +152 -0
- package/lib/match-js-license-comment-signal.d.ts +19 -0
- package/lib/match-js-license-comment-signal.js +21 -0
- package/lib/match-selector-html.d.ts +41 -0
- package/lib/match-selector-html.js +42 -0
- package/lib/match-selector-snapshots.d.ts +40 -0
- package/lib/match-selector-snapshots.js +57 -0
- package/lib/match-structural-technology-signals.d.ts +23 -0
- package/lib/match-structural-technology-signals.js +25 -0
- package/lib/max-decoded-snapshot-bytes.d.ts +22 -0
- package/lib/max-decoded-snapshot-bytes.js +35 -0
- package/lib/native-binding.d.ts +9 -0
- package/lib/native-binding.js +9 -0
- package/lib/pack-stored-snapshots.d.ts +23 -0
- package/lib/pack-stored-snapshots.js +53 -0
- package/lib/run-with-signal.d.ts +27 -0
- package/lib/run-with-signal.js +54 -0
- package/lib/search-html-document.d.ts +32 -0
- package/lib/search-html-document.js +31 -0
- package/lib/search-html-snapshots.d.ts +46 -0
- package/lib/search-html-snapshots.js +54 -0
- package/lib/template-block-sample-size.d.ts +13 -0
- package/lib/template-block-sample-size.js +14 -0
- package/lib/template-exact-page-limit.d.ts +18 -0
- package/lib/template-exact-page-limit.js +19 -0
- package/lib/template-memory-budget.d.ts +16 -0
- package/lib/template-memory-budget.js +17 -0
- package/lib/to-native-selector-plan.d.ts +16 -0
- package/lib/to-native-selector-plan.js +40 -0
- package/lib/to-u32-ids.d.ts +14 -0
- package/lib/to-u32-ids.js +23 -0
- package/lib/types.d.ts +736 -248
- package/lib/validate-search-pattern.d.ts +25 -0
- package/lib/validate-search-pattern.js +27 -0
- package/package.json +83 -16
- package/lib/core.d.ts +0 -4
- package/lib/core.js +0 -4
- package/lib/discover-analyze-plugins.d.ts +0 -14
- package/lib/discover-analyze-plugins.js +0 -32
- package/lib/get-table-cache-root.d.ts +0 -18
- package/lib/get-table-cache-root.js +0 -22
- package/lib/hooks/define-plugin.d.ts +0 -68
- package/lib/hooks/define-plugin.js +0 -69
- package/lib/import-modules.d.ts +0 -24
- package/lib/import-modules.js +0 -38
- package/lib/load-plugin-settings.d.ts +0 -44
- package/lib/load-plugin-settings.js +0 -125
- package/lib/nitpicker.d.ts +0 -150
- package/lib/nitpicker.js +0 -437
- package/lib/page-analysis-worker.d.ts +0 -58
- package/lib/page-analysis-worker.js +0 -68
- package/lib/read-plugin-labels.d.ts +0 -15
- package/lib/read-plugin-labels.js +0 -30
- package/lib/table.d.ts +0 -75
- package/lib/table.js +0 -132
- package/lib/url-event-bus.d.ts +0 -33
- package/lib/url-event-bus.js +0 -21
- package/lib/worker/runner.d.ts +0 -25
- package/lib/worker/runner.js +0 -31
- package/lib/worker/types.d.ts +0 -23
- package/lib/worker/types.js +0 -1
- package/lib/worker/worker-pool.d.ts +0 -82
- package/lib/worker/worker-pool.js +0 -278
- package/lib/worker/worker.d.ts +0 -34
- package/lib/worker/worker.js +0 -108
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { CreateTemplateClassificationRunOptions, TemplateClassificationRun } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Starts a template classification run in the native addon: Pass 0 splits
|
|
4
|
+
* the pages into blocks by URL section and stylesheets, Stage A clusters
|
|
5
|
+
* each block by DOM structure, Stage B merges clusters across blocks and
|
|
6
|
+
* explains each final one.
|
|
7
|
+
*
|
|
8
|
+
* The caller reads the corpus and hands it over: the signals of every page,
|
|
9
|
+
* then the HTML once or twice. Each computing call does its work on the
|
|
10
|
+
* addon's own thread pool (not Node's main thread, not libuv's), so the
|
|
11
|
+
* caller can read the next batch while one is processed. The keys and
|
|
12
|
+
* reasons are the ones stored in existing archives, byte for byte
|
|
13
|
+
* (`@nitpicker/archive`'s golden records pin them).
|
|
14
|
+
*
|
|
15
|
+
* Every block is clustered whole — every page kept until its block is
|
|
16
|
+
* clustered, landmark chrome judged across all of those blocks' pages —
|
|
17
|
+
* unless its Stage A similarity table (`n * (n - 1) / 2 * 8` bytes for `n`
|
|
18
|
+
* pages) would exceed `memoryBudget`, or unless the blocks clustered whole
|
|
19
|
+
* would hold more than `TEMPLATE_EXACT_PAGE_LIMIT` pages together (the
|
|
20
|
+
* largest blocks of more than `TEMPLATE_BLOCK_SAMPLE_SIZE` pages are then
|
|
21
|
+
* sampled until they fit, or none is left). Such a block keeps a deterministic
|
|
22
|
+
* sample of `TEMPLATE_BLOCK_SAMPLE_SIZE` pages instead, chrome is judged
|
|
23
|
+
* within the sample, and the second read assigns every other page of the
|
|
24
|
+
* block to the nearest sampled cluster; `block()` reports each one as a
|
|
25
|
+
* `block-sampled` event and `takeOutput()` lists them in `sampledBlocks`.
|
|
26
|
+
* Every read pushes every page in corpus order; `pageMask()` tells which
|
|
27
|
+
* ones the read uses, so the others can go with empty HTML (and need not be
|
|
28
|
+
* read at all).
|
|
29
|
+
* @param options - See {@link CreateTemplateClassificationRunOptions}.
|
|
30
|
+
* @returns The run.
|
|
31
|
+
* @throws {RangeError} When `memoryBudget` is negative or not a finite number.
|
|
32
|
+
* @example
|
|
33
|
+
* ```ts
|
|
34
|
+
* const run = createTemplateClassificationRun();
|
|
35
|
+
* run.pushSignals([
|
|
36
|
+
* { paths: ['news', '1'], stylesheetHrefs: [] },
|
|
37
|
+
* { paths: ['news', '2'], stylesheetHrefs: [] },
|
|
38
|
+
* ]);
|
|
39
|
+
* await run.block();
|
|
40
|
+
* await run.pushPages([
|
|
41
|
+
* { html: '<body><main><p>a</p></main></body>' },
|
|
42
|
+
* { html: '<body><main><p>b</p></main></body>' },
|
|
43
|
+
* ]);
|
|
44
|
+
* await run.finishPages();
|
|
45
|
+
* const { keys, keyOfPage } = await run.takeOutput();
|
|
46
|
+
* keys[keyOfPage[0]!]; // '["path:news","cluster:0"]'
|
|
47
|
+
* ```
|
|
48
|
+
*/
|
|
49
|
+
export declare function createTemplateClassificationRun(options?: Partial<CreateTemplateClassificationRunOptions>): TemplateClassificationRun;
|
|
@@ -0,0 +1,386 @@
|
|
|
1
|
+
import { nativeBinding } from './native-binding.js';
|
|
2
|
+
/**
|
|
3
|
+
* A field the addon must report, read by its key so the name in the error is
|
|
4
|
+
* the field actually read; throws when it is missing.
|
|
5
|
+
* @param read - The record, the field's key, and what the record is, for the error.
|
|
6
|
+
* @param read.from - As the addon reports it.
|
|
7
|
+
* @param read.field - The field's key.
|
|
8
|
+
* @param read.context - What the record is.
|
|
9
|
+
* @returns The field's value, defined.
|
|
10
|
+
*/
|
|
11
|
+
function required(read) {
|
|
12
|
+
const value = read.from[read.field];
|
|
13
|
+
if (value === undefined) {
|
|
14
|
+
throw new Error(`template classification ${read.context} is missing "${read.field}"`);
|
|
15
|
+
}
|
|
16
|
+
return value;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* The landmark types a reason's `landmarks` may name, as a record so the
|
|
20
|
+
* compiler checks it lists exactly {@link TemplateLandmarkType}'s members.
|
|
21
|
+
*/
|
|
22
|
+
const LANDMARK_TYPES = {
|
|
23
|
+
header: true,
|
|
24
|
+
footer: true,
|
|
25
|
+
nav: true,
|
|
26
|
+
aside: true,
|
|
27
|
+
form: true,
|
|
28
|
+
search: true,
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Whether `type` is one of {@link LANDMARK_TYPES}.
|
|
32
|
+
* @param type - As the addon reports it.
|
|
33
|
+
*/
|
|
34
|
+
function isLandmarkType(type) {
|
|
35
|
+
return Object.hasOwn(LANDMARK_TYPES, type);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Wakes a call waiting for its progress events.
|
|
39
|
+
* @param state - The run's progress state.
|
|
40
|
+
*/
|
|
41
|
+
function releaseWait(state) {
|
|
42
|
+
const resolve = state.onDelivered;
|
|
43
|
+
state.onDelivered = undefined;
|
|
44
|
+
resolve?.();
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* The callback the addon reports progress to. Built here, outside
|
|
48
|
+
* `createTemplateClassificationRun`, so its closure holds only `state` and
|
|
49
|
+
* `onProgress`: V8 gives the closures of one function a shared context, and
|
|
50
|
+
* the addon keeps this callback alive for as long as the run lives, so a
|
|
51
|
+
* callback created next to the run's methods would hold the run strongly
|
|
52
|
+
* and the pair could never be collected.
|
|
53
|
+
* @param state - The run's progress state.
|
|
54
|
+
* @param onProgress - The caller's callback.
|
|
55
|
+
* @returns The callback for the addon.
|
|
56
|
+
*/
|
|
57
|
+
function createProgressCallback(state, onProgress) {
|
|
58
|
+
return (event) => {
|
|
59
|
+
state.delivered++;
|
|
60
|
+
// After a failure the caller has its error; later events of the
|
|
61
|
+
// failed call would only arrive out of turn.
|
|
62
|
+
if (!state.failure) {
|
|
63
|
+
try {
|
|
64
|
+
onProgress(toProgress(event));
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
// Thrown here it would surface as an uncaught exception on the
|
|
68
|
+
// addon's callback, not to whoever awaits the run.
|
|
69
|
+
state.failure = { error };
|
|
70
|
+
state.nativeRef?.deref()?.cancel();
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
if (state.delivered >= state.expected)
|
|
74
|
+
releaseWait(state);
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Narrows a sampling reason the addon reports.
|
|
79
|
+
* @param reason - As the addon reports it.
|
|
80
|
+
* @returns The typed reason.
|
|
81
|
+
* @throws {Error} On a reason the wrapper does not know.
|
|
82
|
+
*/
|
|
83
|
+
function toSampleReason(reason) {
|
|
84
|
+
if (reason === 'table-over-budget' || reason === 'pages-over-limit') {
|
|
85
|
+
return reason;
|
|
86
|
+
}
|
|
87
|
+
throw new Error(`unknown template classification sample reason "${reason}"`);
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Narrows the addon's flat progress record to
|
|
91
|
+
* {@link TemplateClassificationProgress}.
|
|
92
|
+
* @param event - As the addon reports it.
|
|
93
|
+
* @returns The typed event.
|
|
94
|
+
* @throws {Error} On a phase the wrapper does not know or a field the phase
|
|
95
|
+
* needs missing: the addon and the wrapper disagree, which is not for a
|
|
96
|
+
* default to hide.
|
|
97
|
+
*/
|
|
98
|
+
function toProgress(event) {
|
|
99
|
+
const context = `progress phase "${event.phase}"`;
|
|
100
|
+
switch (event.phase) {
|
|
101
|
+
case 'block-sampled': {
|
|
102
|
+
return {
|
|
103
|
+
phase: 'block-sampled',
|
|
104
|
+
blockKey: required({ from: event, field: 'blockKey', context }),
|
|
105
|
+
pageCount: required({ from: event, field: 'pageCount', context }),
|
|
106
|
+
pairBytes: required({ from: event, field: 'pairBytes', context }),
|
|
107
|
+
budget: required({ from: event, field: 'budget', context }),
|
|
108
|
+
reason: toSampleReason(required({ from: event, field: 'reason', context })),
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
case 'pass1b-assign': {
|
|
112
|
+
return {
|
|
113
|
+
phase: 'pass1b-assign',
|
|
114
|
+
pagesAssigned: required({ from: event, field: 'pagesAssigned', context }),
|
|
115
|
+
pagesToAssign: required({ from: event, field: 'pagesToAssign', context }),
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
case 'stage-b-start': {
|
|
119
|
+
return {
|
|
120
|
+
phase: 'stage-b-start',
|
|
121
|
+
unitCount: required({ from: event, field: 'unitCount', context }),
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
case 'pass1-block-complete': {
|
|
125
|
+
return {
|
|
126
|
+
phase: 'pass1-block-complete',
|
|
127
|
+
blockKey: required({ from: event, field: 'blockKey', context }),
|
|
128
|
+
blocksProcessed: required({ from: event, field: 'blocksProcessed', context }),
|
|
129
|
+
totalBlocks: required({ from: event, field: 'totalBlocks', context }),
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
default: {
|
|
133
|
+
throw new Error(`unknown template classification progress phase "${event.phase}"`);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* One blocking reason of the addon's flat record as a
|
|
139
|
+
* {@link TemplateBlockingReason}, keys in the order the stored JSON has them.
|
|
140
|
+
* @param entry - As the addon returns it.
|
|
141
|
+
* @returns The typed reason.
|
|
142
|
+
* @throws {Error} On a kind the wrapper does not know or a field the kind
|
|
143
|
+
* needs missing.
|
|
144
|
+
*/
|
|
145
|
+
function toBlockingReason(entry) {
|
|
146
|
+
const context = `blocking reason of kind "${entry.kind}"`;
|
|
147
|
+
switch (entry.kind) {
|
|
148
|
+
case 'css': {
|
|
149
|
+
return {
|
|
150
|
+
kind: 'css',
|
|
151
|
+
distinctiveStylesheetHrefs: required({
|
|
152
|
+
from: entry,
|
|
153
|
+
field: 'distinctiveStylesheetHrefs',
|
|
154
|
+
context,
|
|
155
|
+
}),
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
case 'path':
|
|
159
|
+
case 'orphanMerge': {
|
|
160
|
+
return {
|
|
161
|
+
kind: entry.kind,
|
|
162
|
+
pathKey: required({ from: entry, field: 'pathKey', context }),
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
default: {
|
|
166
|
+
throw new Error(`unknown template classification blocking reason kind "${entry.kind}"`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* `new URL(href).host`, or `null` where it throws. Taken from the platform
|
|
172
|
+
* so the URL standard's host rules (IDNA included) are applied as browsers
|
|
173
|
+
* apply them.
|
|
174
|
+
* @param href - A stylesheet href.
|
|
175
|
+
* @returns Its host.
|
|
176
|
+
*/
|
|
177
|
+
function hostOf(href) {
|
|
178
|
+
try {
|
|
179
|
+
return new URL(href).host;
|
|
180
|
+
}
|
|
181
|
+
catch {
|
|
182
|
+
return null;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Turns the addon's flattened reasons into {@link TemplateClusterReason}s,
|
|
187
|
+
* building every object with its keys in the order the stored JSON has them.
|
|
188
|
+
* @param reasons - As the addon returns them.
|
|
189
|
+
* @returns By cluster key, in the addon's order.
|
|
190
|
+
* @throws {Error} On a landmark type or blocking kind the wrapper does not
|
|
191
|
+
* know, a landmark type listed twice, or a field a blocking kind needs
|
|
192
|
+
* missing.
|
|
193
|
+
*/
|
|
194
|
+
function toReasons(reasons) {
|
|
195
|
+
const result = new Map();
|
|
196
|
+
for (const reason of reasons) {
|
|
197
|
+
const landmarks = {};
|
|
198
|
+
for (const landmark of reason.landmarks) {
|
|
199
|
+
if (!isLandmarkType(landmark.type)) {
|
|
200
|
+
throw new Error(`unknown template classification landmark type "${landmark.type}"`);
|
|
201
|
+
}
|
|
202
|
+
if (Object.hasOwn(landmarks, landmark.type)) {
|
|
203
|
+
throw new Error(`template classification reason of "${reason.clusterKey}" lists landmark type "${landmark.type}" twice`);
|
|
204
|
+
}
|
|
205
|
+
landmarks[landmark.type] = {
|
|
206
|
+
presenceRate: landmark.presenceRate,
|
|
207
|
+
chromeRate: landmark.chromeRate,
|
|
208
|
+
shellTokens: landmark.shellTokens,
|
|
209
|
+
memberCountWithInstance: landmark.memberCountWithInstance,
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
result.set(reason.clusterKey, {
|
|
213
|
+
memberCount: reason.memberCount,
|
|
214
|
+
blocking: reason.blocking.map((entry) => ({
|
|
215
|
+
blockKey: entry.blockKey,
|
|
216
|
+
reason: toBlockingReason(entry),
|
|
217
|
+
})),
|
|
218
|
+
structuralCoreTokens: reason.structuralCoreTokens,
|
|
219
|
+
landmarks,
|
|
220
|
+
siblingClusterKeys: reason.siblingClusterKeys,
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
return result;
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Starts a template classification run in the native addon: Pass 0 splits
|
|
227
|
+
* the pages into blocks by URL section and stylesheets, Stage A clusters
|
|
228
|
+
* each block by DOM structure, Stage B merges clusters across blocks and
|
|
229
|
+
* explains each final one.
|
|
230
|
+
*
|
|
231
|
+
* The caller reads the corpus and hands it over: the signals of every page,
|
|
232
|
+
* then the HTML once or twice. Each computing call does its work on the
|
|
233
|
+
* addon's own thread pool (not Node's main thread, not libuv's), so the
|
|
234
|
+
* caller can read the next batch while one is processed. The keys and
|
|
235
|
+
* reasons are the ones stored in existing archives, byte for byte
|
|
236
|
+
* (`@nitpicker/archive`'s golden records pin them).
|
|
237
|
+
*
|
|
238
|
+
* Every block is clustered whole — every page kept until its block is
|
|
239
|
+
* clustered, landmark chrome judged across all of those blocks' pages —
|
|
240
|
+
* unless its Stage A similarity table (`n * (n - 1) / 2 * 8` bytes for `n`
|
|
241
|
+
* pages) would exceed `memoryBudget`, or unless the blocks clustered whole
|
|
242
|
+
* would hold more than `TEMPLATE_EXACT_PAGE_LIMIT` pages together (the
|
|
243
|
+
* largest blocks of more than `TEMPLATE_BLOCK_SAMPLE_SIZE` pages are then
|
|
244
|
+
* sampled until they fit, or none is left). Such a block keeps a deterministic
|
|
245
|
+
* sample of `TEMPLATE_BLOCK_SAMPLE_SIZE` pages instead, chrome is judged
|
|
246
|
+
* within the sample, and the second read assigns every other page of the
|
|
247
|
+
* block to the nearest sampled cluster; `block()` reports each one as a
|
|
248
|
+
* `block-sampled` event and `takeOutput()` lists them in `sampledBlocks`.
|
|
249
|
+
* Every read pushes every page in corpus order; `pageMask()` tells which
|
|
250
|
+
* ones the read uses, so the others can go with empty HTML (and need not be
|
|
251
|
+
* read at all).
|
|
252
|
+
* @param options - See {@link CreateTemplateClassificationRunOptions}.
|
|
253
|
+
* @returns The run.
|
|
254
|
+
* @throws {RangeError} When `memoryBudget` is negative or not a finite number.
|
|
255
|
+
* @example
|
|
256
|
+
* ```ts
|
|
257
|
+
* const run = createTemplateClassificationRun();
|
|
258
|
+
* run.pushSignals([
|
|
259
|
+
* { paths: ['news', '1'], stylesheetHrefs: [] },
|
|
260
|
+
* { paths: ['news', '2'], stylesheetHrefs: [] },
|
|
261
|
+
* ]);
|
|
262
|
+
* await run.block();
|
|
263
|
+
* await run.pushPages([
|
|
264
|
+
* { html: '<body><main><p>a</p></main></body>' },
|
|
265
|
+
* { html: '<body><main><p>b</p></main></body>' },
|
|
266
|
+
* ]);
|
|
267
|
+
* await run.finishPages();
|
|
268
|
+
* const { keys, keyOfPage } = await run.takeOutput();
|
|
269
|
+
* keys[keyOfPage[0]!]; // '["path:news","cluster:0"]'
|
|
270
|
+
* ```
|
|
271
|
+
*/
|
|
272
|
+
export function createTemplateClassificationRun(options = {}) {
|
|
273
|
+
const { onProgress, memoryBudget } = options;
|
|
274
|
+
// A NaN or negative budget would reach the addon as 0 and silently
|
|
275
|
+
// sample every block, which stores different keys.
|
|
276
|
+
if (memoryBudget !== undefined &&
|
|
277
|
+
!(Number.isFinite(memoryBudget) && memoryBudget >= 0)) {
|
|
278
|
+
throw new RangeError(`createTemplateClassificationRun: memoryBudget must be a non-negative number of bytes, got ${memoryBudget}`);
|
|
279
|
+
}
|
|
280
|
+
const state = {
|
|
281
|
+
delivered: 0,
|
|
282
|
+
expected: 0,
|
|
283
|
+
onDelivered: undefined,
|
|
284
|
+
failure: undefined,
|
|
285
|
+
nativeRef: undefined,
|
|
286
|
+
};
|
|
287
|
+
// Pages of a site share their stylesheets, so each href is parsed once.
|
|
288
|
+
const hosts = new Map();
|
|
289
|
+
const native = new nativeBinding.TemplateClassificationRun(onProgress ? createProgressCallback(state, onProgress) : undefined, memoryBudget);
|
|
290
|
+
// The callback first runs after the constructor returns.
|
|
291
|
+
state.nativeRef = new WeakRef(native);
|
|
292
|
+
const throwIfFailed = () => {
|
|
293
|
+
if (state.failure)
|
|
294
|
+
throw state.failure.error;
|
|
295
|
+
};
|
|
296
|
+
/**
|
|
297
|
+
* Awaits a computing call, then the progress events it sent, which travel
|
|
298
|
+
* separately from its promise. A rejected call fails the run: the events
|
|
299
|
+
* it sent before failing are not counted, so later calls could no longer
|
|
300
|
+
* tell their own events apart.
|
|
301
|
+
* @param call - Starts the addon's call.
|
|
302
|
+
* @param sentOf - The number of events the call's result says it sent.
|
|
303
|
+
*/
|
|
304
|
+
const settle = async (call, sentOf) => {
|
|
305
|
+
throwIfFailed();
|
|
306
|
+
// Thrown synchronously — the run was busy — the call never started, so
|
|
307
|
+
// the run is unaffected.
|
|
308
|
+
const pending = call();
|
|
309
|
+
let value;
|
|
310
|
+
try {
|
|
311
|
+
value = await pending;
|
|
312
|
+
}
|
|
313
|
+
catch (error) {
|
|
314
|
+
state.failure ??= { error };
|
|
315
|
+
throw state.failure.error;
|
|
316
|
+
}
|
|
317
|
+
state.expected += sentOf(value);
|
|
318
|
+
if (state.delivered < state.expected) {
|
|
319
|
+
// The addon's callback does not keep Node's event loop alive, so
|
|
320
|
+
// this timer does while the events are on their way.
|
|
321
|
+
const keepAlive = setInterval(() => { }, 60_000);
|
|
322
|
+
try {
|
|
323
|
+
await new Promise((resolve) => {
|
|
324
|
+
state.onDelivered = resolve;
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
finally {
|
|
328
|
+
clearInterval(keepAlive);
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
throwIfFailed();
|
|
332
|
+
return value;
|
|
333
|
+
};
|
|
334
|
+
const count = (sent) => sent;
|
|
335
|
+
return {
|
|
336
|
+
pushSignals: (signals) => {
|
|
337
|
+
throwIfFailed();
|
|
338
|
+
native.pushSignals(signals.map((page) => ({
|
|
339
|
+
paths: page.paths,
|
|
340
|
+
stylesheetHrefs: page.stylesheetHrefs,
|
|
341
|
+
stylesheetHosts: page.stylesheetHrefs.map((href) => {
|
|
342
|
+
let host = hosts.get(href);
|
|
343
|
+
if (host === undefined) {
|
|
344
|
+
host = hostOf(href);
|
|
345
|
+
hosts.set(href, host);
|
|
346
|
+
}
|
|
347
|
+
return host;
|
|
348
|
+
}),
|
|
349
|
+
...(page.host === undefined ? {} : { host: page.host }),
|
|
350
|
+
})));
|
|
351
|
+
},
|
|
352
|
+
block: async () => {
|
|
353
|
+
await settle(() => native.block(), count);
|
|
354
|
+
hosts.clear();
|
|
355
|
+
},
|
|
356
|
+
pushPages: async (pages) => {
|
|
357
|
+
await settle(() => native.pushPages(pages), count);
|
|
358
|
+
},
|
|
359
|
+
finishPages: async () => {
|
|
360
|
+
await settle(() => native.finishPages(), count);
|
|
361
|
+
},
|
|
362
|
+
pageMask: () => native.pageMask(),
|
|
363
|
+
needsAssignment: () => native.needsAssignment(),
|
|
364
|
+
pushAssignmentPages: async (pages) => {
|
|
365
|
+
await settle(() => native.pushAssignmentPages(pages), count);
|
|
366
|
+
},
|
|
367
|
+
takeOutput: async () => {
|
|
368
|
+
const { output } = await settle(() => native.takeOutput(), ({ sent }) => sent);
|
|
369
|
+
return {
|
|
370
|
+
keys: output.keys,
|
|
371
|
+
keyOfPage: output.keyOfPage,
|
|
372
|
+
reasons: toReasons(output.reasons),
|
|
373
|
+
sampledBlocks: output.sampledBlocks.map((block) => ({
|
|
374
|
+
...block,
|
|
375
|
+
reason: toSampleReason(block.reason),
|
|
376
|
+
})),
|
|
377
|
+
};
|
|
378
|
+
},
|
|
379
|
+
cancel: () => {
|
|
380
|
+
native.cancel();
|
|
381
|
+
// Events only fail to arrive when Node is tearing the addon down;
|
|
382
|
+
// a cancel must not leave a call waiting on them forever.
|
|
383
|
+
releaseWait(state);
|
|
384
|
+
},
|
|
385
|
+
};
|
|
386
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { AliasCandidate } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Groups the pages that are the same page under another URL, and elects each
|
|
4
|
+
* group's representative.
|
|
5
|
+
*
|
|
6
|
+
* Two candidates are in one group when a chain of the following links them:
|
|
7
|
+
*
|
|
8
|
+
* - **Tier A**: the same title and the same Tier A key (`computeAliasKeys`);
|
|
9
|
+
* - **Tier B**: the same title, the same Tier B key **and** the same body
|
|
10
|
+
* hash (a candidate without a body hash is not grouped this way).
|
|
11
|
+
*
|
|
12
|
+
* A URL that gives no key is not grouped by it. Each tier alone is an
|
|
13
|
+
* equivalence but their union is not, so the groups are the connected
|
|
14
|
+
* components of both.
|
|
15
|
+
*
|
|
16
|
+
* The representative of a group of two or more is elected in turn by: the
|
|
17
|
+
* most `<link rel="canonical">` declarations **within the group** pointing at
|
|
18
|
+
* its URL (when no member is pointed at, every member stays in the running);
|
|
19
|
+
* the shortest URL in UTF-16 code units; the smallest URL by UTF-16 code unit
|
|
20
|
+
* order. URLs are distinct, so the result does not depend on the order of the
|
|
21
|
+
* candidates.
|
|
22
|
+
*
|
|
23
|
+
* Only the columns cross to the addon, and the grouping runs there.
|
|
24
|
+
* @param candidates - The pages. `urlId` is `url_refs.id`: two candidates
|
|
25
|
+
* have the same URL exactly when they have the same id.
|
|
26
|
+
* @returns For each candidate, the index of its group's representative, or
|
|
27
|
+
* `-1` for a representative and for a candidate in a group of one.
|
|
28
|
+
* @throws {RangeError} When an id is not an integer in `0..=4294967294`, or
|
|
29
|
+
* the URLs or hashes are too many bytes for the call to address.
|
|
30
|
+
* @example
|
|
31
|
+
* ```ts
|
|
32
|
+
* const page = { titleTextId: 1, canonicalUrlId: null, bodyHash: null };
|
|
33
|
+
* groupAliasCandidates([
|
|
34
|
+
* { ...page, url: 'https://example.com/about/index.html', urlId: 1 },
|
|
35
|
+
* { ...page, url: 'https://example.com/about/', urlId: 2 },
|
|
36
|
+
* { ...page, url: 'https://example.com/contact/', urlId: 3 },
|
|
37
|
+
* ]); // Int32Array [1, -1, -1]: the shortest URL is the representative
|
|
38
|
+
* ```
|
|
39
|
+
*/
|
|
40
|
+
export declare function groupAliasCandidates(candidates: readonly AliasCandidate[]): Int32Array;
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { assertU32Id } from './assert-u32-id.js';
|
|
2
|
+
import { nativeBinding } from './native-binding.js';
|
|
3
|
+
/** The native call addresses the bytes with `u32` offsets. */
|
|
4
|
+
const MAX_BYTES = 4_294_967_295;
|
|
5
|
+
/** The index of a candidate returned as `-1` must fit an `Int32Array`. */
|
|
6
|
+
const MAX_CANDIDATES = 2_147_483_647;
|
|
7
|
+
/** The addon's "none" for a canonical id. */
|
|
8
|
+
const NO_CANONICAL = 4_294_967_295;
|
|
9
|
+
/**
|
|
10
|
+
* Groups the pages that are the same page under another URL, and elects each
|
|
11
|
+
* group's representative.
|
|
12
|
+
*
|
|
13
|
+
* Two candidates are in one group when a chain of the following links them:
|
|
14
|
+
*
|
|
15
|
+
* - **Tier A**: the same title and the same Tier A key (`computeAliasKeys`);
|
|
16
|
+
* - **Tier B**: the same title, the same Tier B key **and** the same body
|
|
17
|
+
* hash (a candidate without a body hash is not grouped this way).
|
|
18
|
+
*
|
|
19
|
+
* A URL that gives no key is not grouped by it. Each tier alone is an
|
|
20
|
+
* equivalence but their union is not, so the groups are the connected
|
|
21
|
+
* components of both.
|
|
22
|
+
*
|
|
23
|
+
* The representative of a group of two or more is elected in turn by: the
|
|
24
|
+
* most `<link rel="canonical">` declarations **within the group** pointing at
|
|
25
|
+
* its URL (when no member is pointed at, every member stays in the running);
|
|
26
|
+
* the shortest URL in UTF-16 code units; the smallest URL by UTF-16 code unit
|
|
27
|
+
* order. URLs are distinct, so the result does not depend on the order of the
|
|
28
|
+
* candidates.
|
|
29
|
+
*
|
|
30
|
+
* Only the columns cross to the addon, and the grouping runs there.
|
|
31
|
+
* @param candidates - The pages. `urlId` is `url_refs.id`: two candidates
|
|
32
|
+
* have the same URL exactly when they have the same id.
|
|
33
|
+
* @returns For each candidate, the index of its group's representative, or
|
|
34
|
+
* `-1` for a representative and for a candidate in a group of one.
|
|
35
|
+
* @throws {RangeError} When an id is not an integer in `0..=4294967294`, or
|
|
36
|
+
* the URLs or hashes are too many bytes for the call to address.
|
|
37
|
+
* @example
|
|
38
|
+
* ```ts
|
|
39
|
+
* const page = { titleTextId: 1, canonicalUrlId: null, bodyHash: null };
|
|
40
|
+
* groupAliasCandidates([
|
|
41
|
+
* { ...page, url: 'https://example.com/about/index.html', urlId: 1 },
|
|
42
|
+
* { ...page, url: 'https://example.com/about/', urlId: 2 },
|
|
43
|
+
* { ...page, url: 'https://example.com/contact/', urlId: 3 },
|
|
44
|
+
* ]); // Int32Array [1, -1, -1]: the shortest URL is the representative
|
|
45
|
+
* ```
|
|
46
|
+
*/
|
|
47
|
+
export function groupAliasCandidates(candidates) {
|
|
48
|
+
const count = candidates.length;
|
|
49
|
+
if (count > MAX_CANDIDATES) {
|
|
50
|
+
throw new RangeError(`More than ${MAX_CANDIDATES} candidates`);
|
|
51
|
+
}
|
|
52
|
+
// Sized exactly, so the buffers are no bigger than what crosses to the addon.
|
|
53
|
+
let urlBytes = 0;
|
|
54
|
+
let hashBytes = 0;
|
|
55
|
+
for (const candidate of candidates) {
|
|
56
|
+
urlBytes += Buffer.byteLength(candidate.url, 'utf8');
|
|
57
|
+
hashBytes += candidate.bodyHash?.length ?? 0;
|
|
58
|
+
}
|
|
59
|
+
if (urlBytes > MAX_BYTES) {
|
|
60
|
+
throw new RangeError(`The URLs take more than ${MAX_BYTES} bytes`);
|
|
61
|
+
}
|
|
62
|
+
if (hashBytes > MAX_BYTES) {
|
|
63
|
+
throw new RangeError(`The body hashes take more than ${MAX_BYTES} bytes`);
|
|
64
|
+
}
|
|
65
|
+
const urls = Buffer.allocUnsafe(urlBytes);
|
|
66
|
+
const urlOffsets = new Uint32Array(count + 1);
|
|
67
|
+
const bodyHashes = new Uint8Array(hashBytes);
|
|
68
|
+
const hashOffsets = new Uint32Array(count + 1);
|
|
69
|
+
const hashPresent = new Uint8Array(count);
|
|
70
|
+
const urlIds = new Uint32Array(count);
|
|
71
|
+
const titleTextIds = new Uint32Array(count);
|
|
72
|
+
const canonicalUrlIds = new Uint32Array(count);
|
|
73
|
+
let urlTotal = 0;
|
|
74
|
+
let hashTotal = 0;
|
|
75
|
+
for (const [index, candidate] of candidates.entries()) {
|
|
76
|
+
urlTotal += urls.write(candidate.url, urlTotal, 'utf8');
|
|
77
|
+
urlOffsets[index + 1] = urlTotal;
|
|
78
|
+
if (candidate.bodyHash) {
|
|
79
|
+
bodyHashes.set(candidate.bodyHash, hashTotal);
|
|
80
|
+
hashTotal += candidate.bodyHash.length;
|
|
81
|
+
hashPresent[index] = 1;
|
|
82
|
+
}
|
|
83
|
+
hashOffsets[index + 1] = hashTotal;
|
|
84
|
+
urlIds[index] = assertU32Id(candidate.urlId, 'url id');
|
|
85
|
+
titleTextIds[index] = assertU32Id(candidate.titleTextId, 'title text id');
|
|
86
|
+
canonicalUrlIds[index] =
|
|
87
|
+
candidate.canonicalUrlId === null
|
|
88
|
+
? NO_CANONICAL
|
|
89
|
+
: assertU32Id(candidate.canonicalUrlId, 'canonical url id');
|
|
90
|
+
}
|
|
91
|
+
const representatives = nativeBinding.groupAliasCandidates({
|
|
92
|
+
urls,
|
|
93
|
+
urlOffsets,
|
|
94
|
+
urlIds,
|
|
95
|
+
titleTextIds,
|
|
96
|
+
canonicalUrlIds,
|
|
97
|
+
bodyHashes,
|
|
98
|
+
hashOffsets,
|
|
99
|
+
hashPresent,
|
|
100
|
+
});
|
|
101
|
+
// Indexes are below 2**31, so reading the same bytes as signed turns the
|
|
102
|
+
// addon's `4294967295` "none" into `-1` without a copy.
|
|
103
|
+
return new Int32Array(representatives.buffer, representatives.byteOffset, representatives.length);
|
|
104
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { LoadNativeBindingOptions, NativeBinding } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Loads the native addon for the running platform.
|
|
4
|
+
*
|
|
5
|
+
* Looks for a locally built `core.<platform>.node` at the package root
|
|
6
|
+
* first (a monorepo checkout after `yarn build`), then for the prebuilt
|
|
7
|
+
* platform package installed through `optionalDependencies`. Prebuilt
|
|
8
|
+
* binaries exist only for darwin-arm64 and linux-x64 (glibc), and there is
|
|
9
|
+
* no JavaScript fallback, so any other platform fails here with the list of
|
|
10
|
+
* supported ones instead of a bare `MODULE_NOT_FOUND`.
|
|
11
|
+
*
|
|
12
|
+
* Load errors of an addon that does exist (a corrupt or ABI-incompatible
|
|
13
|
+
* file) are rethrown as-is rather than retried against the other location:
|
|
14
|
+
* silently picking a different binary would hide the broken one.
|
|
15
|
+
* @param options - Overrides for the host facts; omitted fields use the
|
|
16
|
+
* running process.
|
|
17
|
+
* @returns The addon's exported functions.
|
|
18
|
+
* @throws {Error} When the platform is unsupported or the platform package is
|
|
19
|
+
* not installed.
|
|
20
|
+
* @example
|
|
21
|
+
* ```ts
|
|
22
|
+
* const binding = loadNativeBinding();
|
|
23
|
+
* binding.computeBodyHash(Buffer.from('<body>x</body>'));
|
|
24
|
+
* ```
|
|
25
|
+
*/
|
|
26
|
+
export declare function loadNativeBinding(options?: Partial<LoadNativeBindingOptions>): NativeBinding;
|