@nitpicker/core 0.24.0 → 0.25.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.md +165 -35
  2. package/lib/assert-u32-id.d.ts +15 -0
  3. package/lib/assert-u32-id.js +22 -0
  4. package/lib/check-native-executor-config.d.ts +21 -0
  5. package/lib/check-native-executor-config.js +24 -0
  6. package/lib/compute-alias-keys.d.ts +28 -0
  7. package/lib/compute-alias-keys.js +30 -0
  8. package/lib/compute-body-hash.d.ts +22 -0
  9. package/lib/compute-body-hash.js +25 -0
  10. package/lib/compute-url-sort-ranks.d.ts +28 -0
  11. package/lib/compute-url-sort-ranks.js +49 -0
  12. package/lib/create-isolated-cluster-run.d.ts +35 -0
  13. package/lib/create-isolated-cluster-run.js +52 -0
  14. package/lib/create-template-classification-run.d.ts +49 -0
  15. package/lib/create-template-classification-run.js +386 -0
  16. package/lib/group-alias-candidates.d.ts +40 -0
  17. package/lib/group-alias-candidates.js +104 -0
  18. package/lib/load-native-binding.d.ts +26 -0
  19. package/lib/load-native-binding.js +152 -0
  20. package/lib/match-js-license-comment-signal.d.ts +19 -0
  21. package/lib/match-js-license-comment-signal.js +21 -0
  22. package/lib/match-selector-html.d.ts +41 -0
  23. package/lib/match-selector-html.js +42 -0
  24. package/lib/match-selector-snapshots.d.ts +40 -0
  25. package/lib/match-selector-snapshots.js +57 -0
  26. package/lib/match-structural-technology-signals.d.ts +23 -0
  27. package/lib/match-structural-technology-signals.js +25 -0
  28. package/lib/max-decoded-snapshot-bytes.d.ts +22 -0
  29. package/lib/max-decoded-snapshot-bytes.js +35 -0
  30. package/lib/native-binding.d.ts +9 -0
  31. package/lib/native-binding.js +9 -0
  32. package/lib/pack-stored-snapshots.d.ts +23 -0
  33. package/lib/pack-stored-snapshots.js +53 -0
  34. package/lib/run-with-signal.d.ts +27 -0
  35. package/lib/run-with-signal.js +54 -0
  36. package/lib/search-html-document.d.ts +32 -0
  37. package/lib/search-html-document.js +31 -0
  38. package/lib/search-html-snapshots.d.ts +46 -0
  39. package/lib/search-html-snapshots.js +54 -0
  40. package/lib/template-block-sample-size.d.ts +13 -0
  41. package/lib/template-block-sample-size.js +14 -0
  42. package/lib/template-exact-page-limit.d.ts +18 -0
  43. package/lib/template-exact-page-limit.js +19 -0
  44. package/lib/template-memory-budget.d.ts +16 -0
  45. package/lib/template-memory-budget.js +17 -0
  46. package/lib/to-native-selector-plan.d.ts +16 -0
  47. package/lib/to-native-selector-plan.js +40 -0
  48. package/lib/to-u32-ids.d.ts +14 -0
  49. package/lib/to-u32-ids.js +22 -0
  50. package/lib/types.d.ts +736 -248
  51. package/lib/validate-search-pattern.d.ts +25 -0
  52. package/lib/validate-search-pattern.js +27 -0
  53. package/package.json +83 -16
  54. package/lib/core.d.ts +0 -4
  55. package/lib/core.js +0 -4
  56. package/lib/discover-analyze-plugins.d.ts +0 -14
  57. package/lib/discover-analyze-plugins.js +0 -32
  58. package/lib/get-table-cache-root.d.ts +0 -18
  59. package/lib/get-table-cache-root.js +0 -22
  60. package/lib/hooks/define-plugin.d.ts +0 -68
  61. package/lib/hooks/define-plugin.js +0 -69
  62. package/lib/import-modules.d.ts +0 -24
  63. package/lib/import-modules.js +0 -38
  64. package/lib/load-plugin-settings.d.ts +0 -44
  65. package/lib/load-plugin-settings.js +0 -125
  66. package/lib/nitpicker.d.ts +0 -150
  67. package/lib/nitpicker.js +0 -437
  68. package/lib/page-analysis-worker.d.ts +0 -58
  69. package/lib/page-analysis-worker.js +0 -68
  70. package/lib/read-plugin-labels.d.ts +0 -15
  71. package/lib/read-plugin-labels.js +0 -30
  72. package/lib/table.d.ts +0 -75
  73. package/lib/table.js +0 -132
  74. package/lib/url-event-bus.d.ts +0 -33
  75. package/lib/url-event-bus.js +0 -21
  76. package/lib/worker/runner.d.ts +0 -25
  77. package/lib/worker/runner.js +0 -31
  78. package/lib/worker/types.d.ts +0 -23
  79. package/lib/worker/types.js +0 -1
  80. package/lib/worker/worker-pool.d.ts +0 -82
  81. package/lib/worker/worker-pool.js +0 -278
  82. package/lib/worker/worker.d.ts +0 -34
  83. package/lib/worker/worker.js +0 -108
@@ -0,0 +1,49 @@
1
+ import type { CreateTemplateClassificationRunOptions, TemplateClassificationRun } from './types.js';
2
+ /**
3
+ * Starts a template classification run in the native addon: Pass 0 splits
4
+ * the pages into blocks by URL section and stylesheets, Stage A clusters
5
+ * each block by DOM structure, Stage B merges clusters across blocks and
6
+ * explains each final one.
7
+ *
8
+ * The caller reads the corpus and hands it over: the signals of every page,
9
+ * then the HTML once or twice. Each computing call does its work on the
10
+ * addon's own thread pool (not Node's main thread, not libuv's), so the
11
+ * caller can read the next batch while one is processed. The keys and
12
+ * reasons are the ones stored in existing archives, byte for byte
13
+ * (`@nitpicker/archive`'s golden records pin them).
14
+ *
15
+ * Every block is clustered whole — every page kept until its block is
16
+ * clustered, landmark chrome judged across all of those blocks' pages —
17
+ * unless its Stage A similarity table (`n * (n - 1) / 2 * 8` bytes for `n`
18
+ * pages) would exceed `memoryBudget`, or unless the blocks clustered whole
19
+ * would hold more than `TEMPLATE_EXACT_PAGE_LIMIT` pages together (the
20
+ * largest blocks of more than `TEMPLATE_BLOCK_SAMPLE_SIZE` pages are then
21
+ * sampled until they fit, or none is left). Such a block keeps a deterministic
22
+ * sample of `TEMPLATE_BLOCK_SAMPLE_SIZE` pages instead, chrome is judged
23
+ * within the sample, and the second read assigns every other page of the
24
+ * block to the nearest sampled cluster; `block()` reports each one as a
25
+ * `block-sampled` event and `takeOutput()` lists them in `sampledBlocks`.
26
+ * Every read pushes every page in corpus order; `pageMask()` tells which
27
+ * ones the read uses, so the others can go with empty HTML (and need not be
28
+ * read at all).
29
+ * @param options - See {@link CreateTemplateClassificationRunOptions}.
30
+ * @returns The run.
31
+ * @throws {RangeError} When `memoryBudget` is negative or not a finite number.
32
+ * @example
33
+ * ```ts
34
+ * const run = createTemplateClassificationRun();
35
+ * run.pushSignals([
36
+ * { paths: ['news', '1'], stylesheetHrefs: [] },
37
+ * { paths: ['news', '2'], stylesheetHrefs: [] },
38
+ * ]);
39
+ * await run.block();
40
+ * await run.pushPages([
41
+ * { html: '<body><main><p>a</p></main></body>' },
42
+ * { html: '<body><main><p>b</p></main></body>' },
43
+ * ]);
44
+ * await run.finishPages();
45
+ * const { keys, keyOfPage } = await run.takeOutput();
46
+ * keys[keyOfPage[0]!]; // '["path:news","cluster:0"]'
47
+ * ```
48
+ */
49
+ export declare function createTemplateClassificationRun(options?: Partial<CreateTemplateClassificationRunOptions>): TemplateClassificationRun;
@@ -0,0 +1,386 @@
1
+ import { nativeBinding } from './native-binding.js';
2
+ /**
3
+ * A field the addon must report, read by its key so the name in the error is
4
+ * the field actually read; throws when it is missing.
5
+ * @param read - The record, the field's key, and what the record is, for the error.
6
+ * @param read.from - As the addon reports it.
7
+ * @param read.field - The field's key.
8
+ * @param read.context - What the record is.
9
+ * @returns The field's value, defined.
10
+ */
11
+ function required(read) {
12
+ const value = read.from[read.field];
13
+ if (value === undefined) {
14
+ throw new Error(`template classification ${read.context} is missing "${read.field}"`);
15
+ }
16
+ return value;
17
+ }
18
+ /**
19
+ * The landmark types a reason's `landmarks` may name, as a record so the
20
+ * compiler checks it lists exactly {@link TemplateLandmarkType}'s members.
21
+ */
22
+ const LANDMARK_TYPES = {
23
+ header: true,
24
+ footer: true,
25
+ nav: true,
26
+ aside: true,
27
+ form: true,
28
+ search: true,
29
+ };
30
+ /**
31
+ * Whether `type` is one of {@link LANDMARK_TYPES}.
32
+ * @param type - As the addon reports it.
33
+ */
34
+ function isLandmarkType(type) {
35
+ return Object.hasOwn(LANDMARK_TYPES, type);
36
+ }
37
+ /**
38
+ * Wakes a call waiting for its progress events.
39
+ * @param state - The run's progress state.
40
+ */
41
+ function releaseWait(state) {
42
+ const resolve = state.onDelivered;
43
+ state.onDelivered = undefined;
44
+ resolve?.();
45
+ }
46
+ /**
47
+ * The callback the addon reports progress to. Built here, outside
48
+ * `createTemplateClassificationRun`, so its closure holds only `state` and
49
+ * `onProgress`: V8 gives the closures of one function a shared context, and
50
+ * the addon keeps this callback alive for as long as the run lives, so a
51
+ * callback created next to the run's methods would hold the run strongly
52
+ * and the pair could never be collected.
53
+ * @param state - The run's progress state.
54
+ * @param onProgress - The caller's callback.
55
+ * @returns The callback for the addon.
56
+ */
57
+ function createProgressCallback(state, onProgress) {
58
+ return (event) => {
59
+ state.delivered++;
60
+ // After a failure the caller has its error; later events of the
61
+ // failed call would only arrive out of turn.
62
+ if (!state.failure) {
63
+ try {
64
+ onProgress(toProgress(event));
65
+ }
66
+ catch (error) {
67
+ // Thrown here it would surface as an uncaught exception on the
68
+ // addon's callback, not to whoever awaits the run.
69
+ state.failure = { error };
70
+ state.nativeRef?.deref()?.cancel();
71
+ }
72
+ }
73
+ if (state.delivered >= state.expected)
74
+ releaseWait(state);
75
+ };
76
+ }
77
+ /**
78
+ * Narrows a sampling reason the addon reports.
79
+ * @param reason - As the addon reports it.
80
+ * @returns The typed reason.
81
+ * @throws {Error} On a reason the wrapper does not know.
82
+ */
83
+ function toSampleReason(reason) {
84
+ if (reason === 'table-over-budget' || reason === 'pages-over-limit') {
85
+ return reason;
86
+ }
87
+ throw new Error(`unknown template classification sample reason "${reason}"`);
88
+ }
89
+ /**
90
+ * Narrows the addon's flat progress record to
91
+ * {@link TemplateClassificationProgress}.
92
+ * @param event - As the addon reports it.
93
+ * @returns The typed event.
94
+ * @throws {Error} On a phase the wrapper does not know or a field the phase
95
+ * needs missing: the addon and the wrapper disagree, which is not for a
96
+ * default to hide.
97
+ */
98
+ function toProgress(event) {
99
+ const context = `progress phase "${event.phase}"`;
100
+ switch (event.phase) {
101
+ case 'block-sampled': {
102
+ return {
103
+ phase: 'block-sampled',
104
+ blockKey: required({ from: event, field: 'blockKey', context }),
105
+ pageCount: required({ from: event, field: 'pageCount', context }),
106
+ pairBytes: required({ from: event, field: 'pairBytes', context }),
107
+ budget: required({ from: event, field: 'budget', context }),
108
+ reason: toSampleReason(required({ from: event, field: 'reason', context })),
109
+ };
110
+ }
111
+ case 'pass1b-assign': {
112
+ return {
113
+ phase: 'pass1b-assign',
114
+ pagesAssigned: required({ from: event, field: 'pagesAssigned', context }),
115
+ pagesToAssign: required({ from: event, field: 'pagesToAssign', context }),
116
+ };
117
+ }
118
+ case 'stage-b-start': {
119
+ return {
120
+ phase: 'stage-b-start',
121
+ unitCount: required({ from: event, field: 'unitCount', context }),
122
+ };
123
+ }
124
+ case 'pass1-block-complete': {
125
+ return {
126
+ phase: 'pass1-block-complete',
127
+ blockKey: required({ from: event, field: 'blockKey', context }),
128
+ blocksProcessed: required({ from: event, field: 'blocksProcessed', context }),
129
+ totalBlocks: required({ from: event, field: 'totalBlocks', context }),
130
+ };
131
+ }
132
+ default: {
133
+ throw new Error(`unknown template classification progress phase "${event.phase}"`);
134
+ }
135
+ }
136
+ }
137
+ /**
138
+ * One blocking reason of the addon's flat record as a
139
+ * {@link TemplateBlockingReason}, keys in the order the stored JSON has them.
140
+ * @param entry - As the addon returns it.
141
+ * @returns The typed reason.
142
+ * @throws {Error} On a kind the wrapper does not know or a field the kind
143
+ * needs missing.
144
+ */
145
+ function toBlockingReason(entry) {
146
+ const context = `blocking reason of kind "${entry.kind}"`;
147
+ switch (entry.kind) {
148
+ case 'css': {
149
+ return {
150
+ kind: 'css',
151
+ distinctiveStylesheetHrefs: required({
152
+ from: entry,
153
+ field: 'distinctiveStylesheetHrefs',
154
+ context,
155
+ }),
156
+ };
157
+ }
158
+ case 'path':
159
+ case 'orphanMerge': {
160
+ return {
161
+ kind: entry.kind,
162
+ pathKey: required({ from: entry, field: 'pathKey', context }),
163
+ };
164
+ }
165
+ default: {
166
+ throw new Error(`unknown template classification blocking reason kind "${entry.kind}"`);
167
+ }
168
+ }
169
+ }
170
+ /**
171
+ * `new URL(href).host`, or `null` where it throws. Taken from the platform
172
+ * so the URL standard's host rules (IDNA included) are applied as browsers
173
+ * apply them.
174
+ * @param href - A stylesheet href.
175
+ * @returns Its host.
176
+ */
177
+ function hostOf(href) {
178
+ try {
179
+ return new URL(href).host;
180
+ }
181
+ catch {
182
+ return null;
183
+ }
184
+ }
185
+ /**
186
+ * Turns the addon's flattened reasons into {@link TemplateClusterReason}s,
187
+ * building every object with its keys in the order the stored JSON has them.
188
+ * @param reasons - As the addon returns them.
189
+ * @returns By cluster key, in the addon's order.
190
+ * @throws {Error} On a landmark type or blocking kind the wrapper does not
191
+ * know, a landmark type listed twice, or a field a blocking kind needs
192
+ * missing.
193
+ */
194
+ function toReasons(reasons) {
195
+ const result = new Map();
196
+ for (const reason of reasons) {
197
+ const landmarks = {};
198
+ for (const landmark of reason.landmarks) {
199
+ if (!isLandmarkType(landmark.type)) {
200
+ throw new Error(`unknown template classification landmark type "${landmark.type}"`);
201
+ }
202
+ if (Object.hasOwn(landmarks, landmark.type)) {
203
+ throw new Error(`template classification reason of "${reason.clusterKey}" lists landmark type "${landmark.type}" twice`);
204
+ }
205
+ landmarks[landmark.type] = {
206
+ presenceRate: landmark.presenceRate,
207
+ chromeRate: landmark.chromeRate,
208
+ shellTokens: landmark.shellTokens,
209
+ memberCountWithInstance: landmark.memberCountWithInstance,
210
+ };
211
+ }
212
+ result.set(reason.clusterKey, {
213
+ memberCount: reason.memberCount,
214
+ blocking: reason.blocking.map((entry) => ({
215
+ blockKey: entry.blockKey,
216
+ reason: toBlockingReason(entry),
217
+ })),
218
+ structuralCoreTokens: reason.structuralCoreTokens,
219
+ landmarks,
220
+ siblingClusterKeys: reason.siblingClusterKeys,
221
+ });
222
+ }
223
+ return result;
224
+ }
225
+ /**
226
+ * Starts a template classification run in the native addon: Pass 0 splits
227
+ * the pages into blocks by URL section and stylesheets, Stage A clusters
228
+ * each block by DOM structure, Stage B merges clusters across blocks and
229
+ * explains each final one.
230
+ *
231
+ * The caller reads the corpus and hands it over: the signals of every page,
232
+ * then the HTML once or twice. Each computing call does its work on the
233
+ * addon's own thread pool (not Node's main thread, not libuv's), so the
234
+ * caller can read the next batch while one is processed. The keys and
235
+ * reasons are the ones stored in existing archives, byte for byte
236
+ * (`@nitpicker/archive`'s golden records pin them).
237
+ *
238
+ * Every block is clustered whole — every page kept until its block is
239
+ * clustered, landmark chrome judged across all of those blocks' pages —
240
+ * unless its Stage A similarity table (`n * (n - 1) / 2 * 8` bytes for `n`
241
+ * pages) would exceed `memoryBudget`, or unless the blocks clustered whole
242
+ * would hold more than `TEMPLATE_EXACT_PAGE_LIMIT` pages together (the
243
+ * largest blocks of more than `TEMPLATE_BLOCK_SAMPLE_SIZE` pages are then
244
+ * sampled until they fit, or none is left). Such a block keeps a deterministic
245
+ * sample of `TEMPLATE_BLOCK_SAMPLE_SIZE` pages instead, chrome is judged
246
+ * within the sample, and the second read assigns every other page of the
247
+ * block to the nearest sampled cluster; `block()` reports each one as a
248
+ * `block-sampled` event and `takeOutput()` lists them in `sampledBlocks`.
249
+ * Every read pushes every page in corpus order; `pageMask()` tells which
250
+ * ones the read uses, so the others can go with empty HTML (and need not be
251
+ * read at all).
252
+ * @param options - See {@link CreateTemplateClassificationRunOptions}.
253
+ * @returns The run.
254
+ * @throws {RangeError} When `memoryBudget` is negative or not a finite number.
255
+ * @example
256
+ * ```ts
257
+ * const run = createTemplateClassificationRun();
258
+ * run.pushSignals([
259
+ * { paths: ['news', '1'], stylesheetHrefs: [] },
260
+ * { paths: ['news', '2'], stylesheetHrefs: [] },
261
+ * ]);
262
+ * await run.block();
263
+ * await run.pushPages([
264
+ * { html: '<body><main><p>a</p></main></body>' },
265
+ * { html: '<body><main><p>b</p></main></body>' },
266
+ * ]);
267
+ * await run.finishPages();
268
+ * const { keys, keyOfPage } = await run.takeOutput();
269
+ * keys[keyOfPage[0]!]; // '["path:news","cluster:0"]'
270
+ * ```
271
+ */
272
+ export function createTemplateClassificationRun(options = {}) {
273
+ const { onProgress, memoryBudget } = options;
274
+ // A NaN or negative budget would reach the addon as 0 and silently
275
+ // sample every block, which stores different keys.
276
+ if (memoryBudget !== undefined &&
277
+ !(Number.isFinite(memoryBudget) && memoryBudget >= 0)) {
278
+ throw new RangeError(`createTemplateClassificationRun: memoryBudget must be a non-negative number of bytes, got ${memoryBudget}`);
279
+ }
280
+ const state = {
281
+ delivered: 0,
282
+ expected: 0,
283
+ onDelivered: undefined,
284
+ failure: undefined,
285
+ nativeRef: undefined,
286
+ };
287
+ // Pages of a site share their stylesheets, so each href is parsed once.
288
+ const hosts = new Map();
289
+ const native = new nativeBinding.TemplateClassificationRun(onProgress ? createProgressCallback(state, onProgress) : undefined, memoryBudget);
290
+ // The callback first runs after the constructor returns.
291
+ state.nativeRef = new WeakRef(native);
292
+ const throwIfFailed = () => {
293
+ if (state.failure)
294
+ throw state.failure.error;
295
+ };
296
+ /**
297
+ * Awaits a computing call, then the progress events it sent, which travel
298
+ * separately from its promise. A rejected call fails the run: the events
299
+ * it sent before failing are not counted, so later calls could no longer
300
+ * tell their own events apart.
301
+ * @param call - Starts the addon's call.
302
+ * @param sentOf - The number of events the call's result says it sent.
303
+ */
304
+ const settle = async (call, sentOf) => {
305
+ throwIfFailed();
306
+ // Thrown synchronously — the run was busy — the call never started, so
307
+ // the run is unaffected.
308
+ const pending = call();
309
+ let value;
310
+ try {
311
+ value = await pending;
312
+ }
313
+ catch (error) {
314
+ state.failure ??= { error };
315
+ throw state.failure.error;
316
+ }
317
+ state.expected += sentOf(value);
318
+ if (state.delivered < state.expected) {
319
+ // The addon's callback does not keep Node's event loop alive, so
320
+ // this timer does while the events are on their way.
321
+ const keepAlive = setInterval(() => { }, 60_000);
322
+ try {
323
+ await new Promise((resolve) => {
324
+ state.onDelivered = resolve;
325
+ });
326
+ }
327
+ finally {
328
+ clearInterval(keepAlive);
329
+ }
330
+ }
331
+ throwIfFailed();
332
+ return value;
333
+ };
334
+ const count = (sent) => sent;
335
+ return {
336
+ pushSignals: (signals) => {
337
+ throwIfFailed();
338
+ native.pushSignals(signals.map((page) => ({
339
+ paths: page.paths,
340
+ stylesheetHrefs: page.stylesheetHrefs,
341
+ stylesheetHosts: page.stylesheetHrefs.map((href) => {
342
+ let host = hosts.get(href);
343
+ if (host === undefined) {
344
+ host = hostOf(href);
345
+ hosts.set(href, host);
346
+ }
347
+ return host;
348
+ }),
349
+ ...(page.host === undefined ? {} : { host: page.host }),
350
+ })));
351
+ },
352
+ block: async () => {
353
+ await settle(() => native.block(), count);
354
+ hosts.clear();
355
+ },
356
+ pushPages: async (pages) => {
357
+ await settle(() => native.pushPages(pages), count);
358
+ },
359
+ finishPages: async () => {
360
+ await settle(() => native.finishPages(), count);
361
+ },
362
+ pageMask: () => native.pageMask(),
363
+ needsAssignment: () => native.needsAssignment(),
364
+ pushAssignmentPages: async (pages) => {
365
+ await settle(() => native.pushAssignmentPages(pages), count);
366
+ },
367
+ takeOutput: async () => {
368
+ const { output } = await settle(() => native.takeOutput(), ({ sent }) => sent);
369
+ return {
370
+ keys: output.keys,
371
+ keyOfPage: output.keyOfPage,
372
+ reasons: toReasons(output.reasons),
373
+ sampledBlocks: output.sampledBlocks.map((block) => ({
374
+ ...block,
375
+ reason: toSampleReason(block.reason),
376
+ })),
377
+ };
378
+ },
379
+ cancel: () => {
380
+ native.cancel();
381
+ // Events only fail to arrive when Node is tearing the addon down;
382
+ // a cancel must not leave a call waiting on them forever.
383
+ releaseWait(state);
384
+ },
385
+ };
386
+ }
@@ -0,0 +1,40 @@
1
+ import type { AliasCandidate } from './types.js';
2
+ /**
3
+ * Groups the pages that are the same page under another URL, and elects each
4
+ * group's representative.
5
+ *
6
+ * Two candidates are in one group when a chain of the following links them:
7
+ *
8
+ * - **Tier A**: the same title and the same Tier A key (`computeAliasKeys`);
9
+ * - **Tier B**: the same title, the same Tier B key **and** the same body
10
+ * hash (a candidate without a body hash is not grouped this way).
11
+ *
12
+ * A URL that gives no key is not grouped by it. Each tier alone is an
13
+ * equivalence but their union is not, so the groups are the connected
14
+ * components of both.
15
+ *
16
+ * The representative of a group of two or more is elected in turn by: the
17
+ * most `<link rel="canonical">` declarations **within the group** pointing at
18
+ * its URL (when no member is pointed at, every member stays in the running);
19
+ * the shortest URL in UTF-16 code units; the smallest URL by UTF-16 code unit
20
+ * order. URLs are distinct, so the result does not depend on the order of the
21
+ * candidates.
22
+ *
23
+ * Only the columns cross to the addon, and the grouping runs there.
24
+ * @param candidates - The pages. `urlId` is `url_refs.id`: two candidates
25
+ * have the same URL exactly when they have the same id.
26
+ * @returns For each candidate, the index of its group's representative, or
27
+ * `-1` for a representative and for a candidate in a group of one.
28
+ * @throws {RangeError} When an id is not an integer in `0..=4294967294`, or
29
+ * the URLs or hashes are too many bytes for the call to address.
30
+ * @example
31
+ * ```ts
32
+ * const page = { titleTextId: 1, canonicalUrlId: null, bodyHash: null };
33
+ * groupAliasCandidates([
34
+ * { ...page, url: 'https://example.com/about/index.html', urlId: 1 },
35
+ * { ...page, url: 'https://example.com/about/', urlId: 2 },
36
+ * { ...page, url: 'https://example.com/contact/', urlId: 3 },
37
+ * ]); // Int32Array [1, -1, -1]: the shortest URL is the representative
38
+ * ```
39
+ */
40
+ export declare function groupAliasCandidates(candidates: readonly AliasCandidate[]): Int32Array;
@@ -0,0 +1,104 @@
1
+ import { assertU32Id } from './assert-u32-id.js';
2
+ import { nativeBinding } from './native-binding.js';
3
+ /** The native call addresses the bytes with `u32` offsets. */
4
+ const MAX_BYTES = 4_294_967_295;
5
+ /** The index of a candidate returned as `-1` must fit an `Int32Array`. */
6
+ const MAX_CANDIDATES = 2_147_483_647;
7
+ /** The addon's "none" for a canonical id. */
8
+ const NO_CANONICAL = 4_294_967_295;
9
+ /**
10
+ * Groups the pages that are the same page under another URL, and elects each
11
+ * group's representative.
12
+ *
13
+ * Two candidates are in one group when a chain of the following links them:
14
+ *
15
+ * - **Tier A**: the same title and the same Tier A key (`computeAliasKeys`);
16
+ * - **Tier B**: the same title, the same Tier B key **and** the same body
17
+ * hash (a candidate without a body hash is not grouped this way).
18
+ *
19
+ * A URL that gives no key is not grouped by it. Each tier alone is an
20
+ * equivalence but their union is not, so the groups are the connected
21
+ * components of both.
22
+ *
23
+ * The representative of a group of two or more is elected in turn by: the
24
+ * most `<link rel="canonical">` declarations **within the group** pointing at
25
+ * its URL (when no member is pointed at, every member stays in the running);
26
+ * the shortest URL in UTF-16 code units; the smallest URL by UTF-16 code unit
27
+ * order. URLs are distinct, so the result does not depend on the order of the
28
+ * candidates.
29
+ *
30
+ * Only the columns cross to the addon, and the grouping runs there.
31
+ * @param candidates - The pages. `urlId` is `url_refs.id`: two candidates
32
+ * have the same URL exactly when they have the same id.
33
+ * @returns For each candidate, the index of its group's representative, or
34
+ * `-1` for a representative and for a candidate in a group of one.
35
+ * @throws {RangeError} When an id is not an integer in `0..=4294967294`, or
36
+ * the URLs or hashes are too many bytes for the call to address.
37
+ * @example
38
+ * ```ts
39
+ * const page = { titleTextId: 1, canonicalUrlId: null, bodyHash: null };
40
+ * groupAliasCandidates([
41
+ * { ...page, url: 'https://example.com/about/index.html', urlId: 1 },
42
+ * { ...page, url: 'https://example.com/about/', urlId: 2 },
43
+ * { ...page, url: 'https://example.com/contact/', urlId: 3 },
44
+ * ]); // Int32Array [1, -1, -1]: the shortest URL is the representative
45
+ * ```
46
+ */
47
+ export function groupAliasCandidates(candidates) {
48
+ const count = candidates.length;
49
+ if (count > MAX_CANDIDATES) {
50
+ throw new RangeError(`More than ${MAX_CANDIDATES} candidates`);
51
+ }
52
+ // Sized exactly, so the buffers are no bigger than what crosses to the addon.
53
+ let urlBytes = 0;
54
+ let hashBytes = 0;
55
+ for (const candidate of candidates) {
56
+ urlBytes += Buffer.byteLength(candidate.url, 'utf8');
57
+ hashBytes += candidate.bodyHash?.length ?? 0;
58
+ }
59
+ if (urlBytes > MAX_BYTES) {
60
+ throw new RangeError(`The URLs take more than ${MAX_BYTES} bytes`);
61
+ }
62
+ if (hashBytes > MAX_BYTES) {
63
+ throw new RangeError(`The body hashes take more than ${MAX_BYTES} bytes`);
64
+ }
65
+ const urls = Buffer.allocUnsafe(urlBytes);
66
+ const urlOffsets = new Uint32Array(count + 1);
67
+ const bodyHashes = new Uint8Array(hashBytes);
68
+ const hashOffsets = new Uint32Array(count + 1);
69
+ const hashPresent = new Uint8Array(count);
70
+ const urlIds = new Uint32Array(count);
71
+ const titleTextIds = new Uint32Array(count);
72
+ const canonicalUrlIds = new Uint32Array(count);
73
+ let urlTotal = 0;
74
+ let hashTotal = 0;
75
+ for (const [index, candidate] of candidates.entries()) {
76
+ urlTotal += urls.write(candidate.url, urlTotal, 'utf8');
77
+ urlOffsets[index + 1] = urlTotal;
78
+ if (candidate.bodyHash) {
79
+ bodyHashes.set(candidate.bodyHash, hashTotal);
80
+ hashTotal += candidate.bodyHash.length;
81
+ hashPresent[index] = 1;
82
+ }
83
+ hashOffsets[index + 1] = hashTotal;
84
+ urlIds[index] = assertU32Id(candidate.urlId, 'url id');
85
+ titleTextIds[index] = assertU32Id(candidate.titleTextId, 'title text id');
86
+ canonicalUrlIds[index] =
87
+ candidate.canonicalUrlId === null
88
+ ? NO_CANONICAL
89
+ : assertU32Id(candidate.canonicalUrlId, 'canonical url id');
90
+ }
91
+ const representatives = nativeBinding.groupAliasCandidates({
92
+ urls,
93
+ urlOffsets,
94
+ urlIds,
95
+ titleTextIds,
96
+ canonicalUrlIds,
97
+ bodyHashes,
98
+ hashOffsets,
99
+ hashPresent,
100
+ });
101
+ // Indexes are below 2**31, so reading the same bytes as signed turns the
102
+ // addon's `4294967295` "none" into `-1` without a copy.
103
+ return new Int32Array(representatives.buffer, representatives.byteOffset, representatives.length);
104
+ }
@@ -0,0 +1,26 @@
1
+ import type { LoadNativeBindingOptions, NativeBinding } from './types.js';
2
+ /**
3
+ * Loads the native addon for the running platform.
4
+ *
5
+ * Looks for a locally built `core.<platform>.node` at the package root
6
+ * first (a monorepo checkout after `yarn build`), then for the prebuilt
7
+ * platform package installed through `optionalDependencies`. Prebuilt
8
+ * binaries exist only for darwin-arm64 and linux-x64 (glibc), and there is
9
+ * no JavaScript fallback, so any other platform fails here with the list of
10
+ * supported ones instead of a bare `MODULE_NOT_FOUND`.
11
+ *
12
+ * Load errors of an addon that does exist (a corrupt or ABI-incompatible
13
+ * file) are rethrown as-is rather than retried against the other location:
14
+ * silently picking a different binary would hide the broken one.
15
+ * @param options - Overrides for the host facts; omitted fields use the
16
+ * running process.
17
+ * @returns The addon's exported functions.
18
+ * @throws {Error} When the platform is unsupported or the platform package is
19
+ * not installed.
20
+ * @example
21
+ * ```ts
22
+ * const binding = loadNativeBinding();
23
+ * binding.computeBodyHash(Buffer.from('<body>x</body>'));
24
+ * ```
25
+ */
26
+ export declare function loadNativeBinding(options?: Partial<LoadNativeBindingOptions>): NativeBinding;