@nitpicker/core 0.24.0 → 0.25.1

package/lib/run-with-signal.js

@@ -0,0 +1,54 @@
+import { nativeBinding } from './native-binding.js';
+/**
+ * Whether a native call rejected because its cancel token was raised.
+ * @param error - What the call rejected with.
+ */
+function isCancelled(error) {
+    return error instanceof Error && error.code === 'Cancelled';
+}
+/**
+ * Runs one asynchronous native call so an `AbortSignal` can stop it.
+ *
+ * With no signal, `run` gets no token and nothing is allocated. With one, a
+ * fresh native token is raised when the signal fires, and the call stops at
+ * its next check (between snapshots, between mebibytes of a snapshot's
+ * decompression, or when it leaves the queue) and rejects
+ * with code `Cancelled`; that rejection, and only that one, is replaced by
+ * the signal's reason, so an abort reads as the caller's own abort. The
+ * addon reports a cancellation over a decode failure of the same batch (the
+ * caller asked to stop); a batch whose failure came before any snapshot saw
+ * the abort still rejects with that failure, which is passed on unchanged.
+ * A signal that is already aborted rejects with its reason without calling
+ * the addon.
+ * @param signal - The caller's signal, if any.
+ * @param run - Starts the native call, passing the token as its last argument.
+ * @returns What `run` resolved with.
+ * @throws {unknown} The signal's reason when the signal aborted before or during the call.
+ * @example
+ * ```ts
+ * const flags = await runWithSignal(signal, (token) =>
+ * 	nativeBinding.matchSelectorSnapshots(plan, bodies, offsets, codecs, maxDecodedBytes, token),
+ * );
+ * ```
+ */
+export async function runWithSignal(signal, run) {
+    if (!signal) {
+        return run();
+    }
+    signal.throwIfAborted();
+    const token = new nativeBinding.NativeCancelToken();
+    const onAbort = () => token.cancel();
+    signal.addEventListener('abort', onAbort, { once: true });
+    try {
+        return await run(token);
+    }
+    catch (error) {
+        if (signal.aborted && isCancelled(error)) {
+            throw signal.reason;
+        }
+        throw error;
+    }
+    finally {
+        signal.removeEventListener('abort', onAbort);
+    }
+}

package/lib/search-html-document.d.ts

@@ -0,0 +1,32 @@
+import type { NativeSearchPattern, SnapshotSearchOutcome } from './types.js';
+/**
+ * Counts the occurrences of a search pattern in one stored HTML document and
+ * cuts a snippet around the first.
+ *
+ * The document is read as UTF-8 (an invalid sequence becomes U+FFFD, as
+ * `Buffer#toString('utf8')` reads it). Positions and lengths are in Unicode
+ * scalar values, so `.` matches an emoji once. `matchCount` counts
+ * non-overlapping matches as Rust's `find_iter` walks them; the snippet is a
+ * window of `snippetLength` characters centred on the first match, with
+ * whitespace collapsed (JavaScript's `\s`) and the ends trimmed.
+ * @param options - The pattern and the document.
+ * @param options.pattern - The pattern and its flags.
+ * @param options.html - The stored markup, as a string or as UTF-8 bytes.
+ * @param options.snippetLength - Window size in characters.
+ * @returns The match count and the snippet (empty when nothing matched).
+ * @throws {Error} With code `InvalidArg` if the pattern cannot be compiled (see `validateSearchPattern`).
+ * @example
+ * ```ts
+ * const { matchCount, snippet } = searchHtmlDocument({
+ * 	pattern: { source: 'font-family', literal: true, ignoreCase: false, multiLine: false, dotAll: false },
+ * 	html: '<style>p { font-family: serif }</style>',
+ * 	snippetLength: 20,
+ * });
+ * matchCount; // 1
+ * ```
+ */
+export declare function searchHtmlDocument(options: {
+    readonly pattern: NativeSearchPattern;
+    readonly html: string | Uint8Array;
+    readonly snippetLength: number;
+}): SnapshotSearchOutcome;

package/lib/search-html-document.js

@@ -0,0 +1,31 @@
+import { nativeBinding } from './native-binding.js';
+/**
+ * Counts the occurrences of a search pattern in one stored HTML document and
+ * cuts a snippet around the first.
+ *
+ * The document is read as UTF-8 (an invalid sequence becomes U+FFFD, as
+ * `Buffer#toString('utf8')` reads it). Positions and lengths are in Unicode
+ * scalar values, so `.` matches an emoji once. `matchCount` counts
+ * non-overlapping matches as Rust's `find_iter` walks them; the snippet is a
+ * window of `snippetLength` characters centred on the first match, with
+ * whitespace collapsed (JavaScript's `\s`) and the ends trimmed.
+ * @param options - The pattern and the document.
+ * @param options.pattern - The pattern and its flags.
+ * @param options.html - The stored markup, as a string or as UTF-8 bytes.
+ * @param options.snippetLength - Window size in characters.
+ * @returns The match count and the snippet (empty when nothing matched).
+ * @throws {Error} With code `InvalidArg` if the pattern cannot be compiled (see `validateSearchPattern`).
+ * @example
+ * ```ts
+ * const { matchCount, snippet } = searchHtmlDocument({
+ * 	pattern: { source: 'font-family', literal: true, ignoreCase: false, multiLine: false, dotAll: false },
+ * 	html: '<style>p { font-family: serif }</style>',
+ * 	snippetLength: 20,
+ * });
+ * matchCount; // 1
+ * ```
+ */
+export function searchHtmlDocument(options) {
+    const { pattern, html, snippetLength } = options;
+    return nativeBinding.searchHtmlDocument(pattern, typeof html === 'string' ? Buffer.from(html, 'utf8') : html, snippetLength);
+}

package/lib/search-html-snapshots.d.ts

@@ -0,0 +1,46 @@
+import type { NativeSearchPattern, SnapshotSearchOutcome, StoredHtmlSnapshot } from './types.js';
+/**
+ * Searches a batch of stored HTML snapshots for a pattern.
+ *
+ * Each snapshot is decompressed, read as UTF-8 and searched as
+ * {@link import('./search-html-document.js').searchHtmlDocument} does, in
+ * parallel on the native thread pool, so no more decompressed documents are
+ * alive at once than there are threads. The pattern is compiled once, not per
+ * snapshot, and the addon reuses the last compiled pattern across batches. The
+ * call waits its turn behind any other asynchronous native call when
+ * `NITPICKER_NATIVE_CONCURRENCY` calls are already running (see
+ * `checkNativeExecutorConfig`). Only the
+ * batch's stored bytes cross the boundary, as one buffer and one offsets
+ * array.
+ *
+ * A snapshot that cannot be decompressed — its zstd data is corrupt, or it
+ * decompresses past `MAX_DECODED_SNAPSHOT_BYTES` — fails the whole call; it is
+ * never reported as "no match".
+ * @param options - The pattern, the snippet size and the batch.
+ * @param options.pattern - The pattern and its flags.
+ * @param options.snippetLength - Window size in characters.
+ * @param options.snapshots - The stored snapshots, in the order the results are returned.
+ * @param options.signal - Stops the call at its next snapshot, or within the next mebibyte of a decompression under way; a call still waiting for its turn stops when the turn comes.
+ * @returns One outcome per snapshot, in input order; `matchCount` is 0 where nothing matched.
+ * @throws {Error} If a codec is not `zstd` or `none`, or the batch takes more than 4 GiB.
+ *   The returned promise rejects with code `InvalidArg` if the pattern cannot be
+ *   compiled or the batch is malformed, and with a plain error naming the snapshot
+ *   (`snapshot <n> could not be decoded`) if its zstd data is corrupt or
+ *   decompresses past the limit, and with the signal's reason if `signal`
+ *   aborted before the call finished.
+ * @example
+ * ```ts
+ * const outcomes = await searchHtmlSnapshots({
+ * 	pattern: { source: 'gtag(', literal: true, ignoreCase: false, multiLine: false, dotAll: false },
+ * 	snippetLength: 160,
+ * 	snapshots: rows.map((row) => ({ body: row.body, codec: row.codec })),
+ * });
+ * const hits = outcomes.filter((outcome) => outcome.matchCount > 0);
+ * ```
+ */
+export declare function searchHtmlSnapshots(options: {
+    readonly pattern: NativeSearchPattern;
+    readonly snippetLength: number;
+    readonly snapshots: readonly StoredHtmlSnapshot[];
+    readonly signal?: AbortSignal;
+}): Promise<SnapshotSearchOutcome[]>;

package/lib/search-html-snapshots.js

@@ -0,0 +1,54 @@
+import { MAX_DECODED_SNAPSHOT_BYTES } from './max-decoded-snapshot-bytes.js';
+import { nativeBinding } from './native-binding.js';
+import { packStoredSnapshots } from './pack-stored-snapshots.js';
+import { runWithSignal } from './run-with-signal.js';
+/**
+ * Searches a batch of stored HTML snapshots for a pattern.
+ *
+ * Each snapshot is decompressed, read as UTF-8 and searched as
+ * {@link import('./search-html-document.js').searchHtmlDocument} does, in
+ * parallel on the native thread pool, so no more decompressed documents are
+ * alive at once than there are threads. The pattern is compiled once, not per
+ * snapshot, and the addon reuses the last compiled pattern across batches. The
+ * call waits its turn behind any other asynchronous native call when
+ * `NITPICKER_NATIVE_CONCURRENCY` calls are already running (see
+ * `checkNativeExecutorConfig`). Only the
+ * batch's stored bytes cross the boundary, as one buffer and one offsets
+ * array.
+ *
+ * A snapshot that cannot be decompressed — its zstd data is corrupt, or it
+ * decompresses past `MAX_DECODED_SNAPSHOT_BYTES` — fails the whole call; it is
+ * never reported as "no match".
+ * @param options - The pattern, the snippet size and the batch.
+ * @param options.pattern - The pattern and its flags.
+ * @param options.snippetLength - Window size in characters.
+ * @param options.snapshots - The stored snapshots, in the order the results are returned.
+ * @param options.signal - Stops the call at its next snapshot, or within the next mebibyte of a decompression under way; a call still waiting for its turn stops when the turn comes.
+ * @returns One outcome per snapshot, in input order; `matchCount` is 0 where nothing matched.
+ * @throws {Error} If a codec is not `zstd` or `none`, or the batch takes more than 4 GiB.
+ *   The returned promise rejects with code `InvalidArg` if the pattern cannot be
+ *   compiled or the batch is malformed, and with a plain error naming the snapshot
+ *   (`snapshot <n> could not be decoded`) if its zstd data is corrupt or
+ *   decompresses past the limit, and with the signal's reason if `signal`
+ *   aborted before the call finished.
+ * @example
+ * ```ts
+ * const outcomes = await searchHtmlSnapshots({
+ * 	pattern: { source: 'gtag(', literal: true, ignoreCase: false, multiLine: false, dotAll: false },
+ * 	snippetLength: 160,
+ * 	snapshots: rows.map((row) => ({ body: row.body, codec: row.codec })),
+ * });
+ * const hits = outcomes.filter((outcome) => outcome.matchCount > 0);
+ * ```
+ */
+export async function searchHtmlSnapshots(options) {
+    const { pattern, snippetLength, snapshots, signal } = options;
+    // Before packing: a request that is already cancelled copies nothing.
+    signal?.throwIfAborted();
+    const { bodies, offsets, codecs } = packStoredSnapshots(snapshots);
+    const { matchCounts, snippets } = await runWithSignal(signal, (cancelToken) => nativeBinding.searchHtmlSnapshots(pattern, snippetLength, bodies, offsets, codecs, MAX_DECODED_SNAPSHOT_BYTES, cancelToken));
+    return matchCounts.map((matchCount, index) => ({
+        matchCount,
+        snippet: snippets[index],
+    }));
+}

package/lib/template-block-sample-size.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Pages per sampled block a template classification run clusters (see
+ * `createTemplateClassificationRun`); every other page of the block is
+ * assigned to the nearest of those clusters, and the merge across blocks
+ * keeps at most this many members per cluster that holds one. Read from the addon,
+ * so a caller sizing its own work by the sample cannot drift from what the
+ * run actually keeps.
+ * @example
+ * ```ts
+ * const sampledPages = Math.min(blockPageCount, TEMPLATE_BLOCK_SAMPLE_SIZE);
+ * ```
+ */
+export declare const TEMPLATE_BLOCK_SAMPLE_SIZE: number;

package/lib/template-block-sample-size.js

@@ -0,0 +1,14 @@
+import { nativeBinding } from './native-binding.js';
+/**
+ * Pages per sampled block a template classification run clusters (see
+ * `createTemplateClassificationRun`); every other page of the block is
+ * assigned to the nearest of those clusters, and the merge across blocks
+ * keeps at most this many members per cluster that holds one. Read from the addon,
+ * so a caller sizing its own work by the sample cannot drift from what the
+ * run actually keeps.
+ * @example
+ * ```ts
+ * const sampledPages = Math.min(blockPageCount, TEMPLATE_BLOCK_SAMPLE_SIZE);
+ * ```
+ */
+export const TEMPLATE_BLOCK_SAMPLE_SIZE = nativeBinding.TEMPLATE_BLOCK_SAMPLE_SIZE;

package/lib/template-exact-page-limit.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Pages of blocks clustered whole a template classification run (see
+ * `createTemplateClassificationRun`) holds at once, at most. Clustering a
+ * block needs every page's parsed markup until its block is done, about
+ * 60 KB a page, so this bounds that memory whatever the corpus size. When
+ * the blocks clustered whole would hold more, the largest blocks of more
+ * than `TEMPLATE_BLOCK_SAMPLE_SIZE` pages are clustered from a sample
+ * instead (most pages first) until the rest fits; a block of at most that
+ * many pages frees nothing by being sampled, so none is left out for this
+ * limit. Fixed, not a setting: which blocks are sampled decides their
+ * stored keys. Read from the addon, so a caller documenting or sizing by
+ * it cannot drift from what the run actually does.
+ * @example
+ * ```ts
+ * const heldPages = Math.min(wholeBlockPageCount, TEMPLATE_EXACT_PAGE_LIMIT);
+ * ```
+ */
+export declare const TEMPLATE_EXACT_PAGE_LIMIT: number;

package/lib/template-exact-page-limit.js

@@ -0,0 +1,19 @@
+import { nativeBinding } from './native-binding.js';
+/**
+ * Pages of blocks clustered whole a template classification run (see
+ * `createTemplateClassificationRun`) holds at once, at most. Clustering a
+ * block needs every page's parsed markup until its block is done, about
+ * 60 KB a page, so this bounds that memory whatever the corpus size. When
+ * the blocks clustered whole would hold more, the largest blocks of more
+ * than `TEMPLATE_BLOCK_SAMPLE_SIZE` pages are clustered from a sample
+ * instead (most pages first) until the rest fits; a block of at most that
+ * many pages frees nothing by being sampled, so none is left out for this
+ * limit. Fixed, not a setting: which blocks are sampled decides their
+ * stored keys. Read from the addon, so a caller documenting or sizing by
+ * it cannot drift from what the run actually does.
+ * @example
+ * ```ts
+ * const heldPages = Math.min(wholeBlockPageCount, TEMPLATE_EXACT_PAGE_LIMIT);
+ * ```
+ */
+export const TEMPLATE_EXACT_PAGE_LIMIT = nativeBinding.TEMPLATE_EXACT_PAGE_LIMIT;

package/lib/template-memory-budget.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Bytes a block's Stage A similarity table may take before a template
+ * classification run (see `createTemplateClassificationRun`) clusters the
+ * block from a sample instead of whole. The table is an upper triangle of
+ * `f64`, `n * (n - 1) / 2 * 8` bytes for `n` pages and the largest
+ * allocation of a run, so this bounds what clustering holds whatever the
+ * corpus size. Which blocks fall over it decides their stored keys. Read
+ * from the addon, so a caller sizing or documenting the budget cannot drift
+ * from what the run actually does.
+ * @example
+ * ```ts
+ * const pairBytes = (pages: number) => (pages * (pages - 1)) / 2 * 8;
+ * const sampled = pairBytes(blockPageCount) > TEMPLATE_MEMORY_BUDGET;
+ * ```
+ */
+export declare const TEMPLATE_MEMORY_BUDGET: number;

package/lib/template-memory-budget.js

@@ -0,0 +1,17 @@
+import { nativeBinding } from './native-binding.js';
+/**
+ * Bytes a block's Stage A similarity table may take before a template
+ * classification run (see `createTemplateClassificationRun`) clusters the
+ * block from a sample instead of whole. The table is an upper triangle of
+ * `f64`, `n * (n - 1) / 2 * 8` bytes for `n` pages and the largest
+ * allocation of a run, so this bounds what clustering holds whatever the
+ * corpus size. Which blocks fall over it decides their stored keys. Read
+ * from the addon, so a caller sizing or documenting the budget cannot drift
+ * from what the run actually does.
+ * @example
+ * ```ts
+ * const pairBytes = (pages: number) => (pages * (pages - 1)) / 2 * 8;
+ * const sampled = pairBytes(blockPageCount) > TEMPLATE_MEMORY_BUDGET;
+ * ```
+ */
+export const TEMPLATE_MEMORY_BUDGET = nativeBinding.TEMPLATE_MEMORY_BUDGET;

package/lib/to-native-selector-plan.d.ts

@@ -0,0 +1,16 @@
+import type { NativeSelectorPlan, SelectorPlan } from './types.js';
+/**
+ * Rewrites a selector plan into the shape the native conversion reads: the
+ * same plan, with every `null` tag and `null` literal list left out.
+ *
+ * The native object conversion reads an optional field only when it is
+ * absent (`undefined`), not when it is `null`: a `null` list fails with
+ * "Failed to get Array length". The plan states "none" as `null`.
+ * @param plan - The selector plan.
+ * @returns A plan the native functions accept.
+ * @example
+ * ```ts
+ * toNativeSelectorPlan(plan).alternatives[0]!.prefilterLiterals; // undefined, not null
+ * ```
+ */
+export declare function toNativeSelectorPlan(plan: SelectorPlan): NativeSelectorPlan;

package/lib/to-native-selector-plan.js

@@ -0,0 +1,40 @@
+/**
+ * The conditions of a compound, with a `null` tag left out.
+ * @param conditions - A compound, or the compound inside `:not(...)`.
+ * @returns The conditions in the shape the native conversion reads.
+ */
+function toNativeConditions(conditions) {
+    const { tag, attributes, nth } = conditions;
+    return tag === null ? { attributes, nth } : { tag, attributes, nth };
+}
+/**
+ * Rewrites a selector plan into the shape the native conversion reads: the
+ * same plan, with every `null` tag and `null` literal list left out.
+ *
+ * The native object conversion reads an optional field only when it is
+ * absent (`undefined`), not when it is `null`: a `null` list fails with
+ * "Failed to get Array length". The plan states "none" as `null`.
+ * @param plan - The selector plan.
+ * @returns A plan the native functions accept.
+ * @example
+ * ```ts
+ * toNativeSelectorPlan(plan).alternatives[0]!.prefilterLiterals; // undefined, not null
+ * ```
+ */
+export function toNativeSelectorPlan(plan) {
+    return {
+        alternatives: plan.alternatives.map((alternative) => ({
+            compounds: alternative.compounds.map((compound) => ({
+                ...toNativeConditions(compound),
+                negations: compound.negations.map((negation) => toNativeConditions(negation)),
+            })),
+            combinators: alternative.combinators,
+            ...(alternative.prefilterLiterals === null
+                ? {}
+                : { prefilterLiterals: alternative.prefilterLiterals }),
+            direct: alternative.direct,
+        })),
+        needsChildIndex: plan.needsChildIndex,
+        needsTypeIndex: plan.needsTypeIndex,
+    };
+}

package/lib/to-u32-ids.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Copies row ids into the `u32` array the addon takes, refusing any value a
+ * `Uint32Array` would silently change (see `assertU32Id`).
+ * @param values - The ids (database row ids): an array or a typed array.
+ * @param what - What the ids are, for the error message.
+ * @returns The ids as a `Uint32Array`.
+ * @throws {RangeError} When a value is not an integer in `0..=4294967294`.
+ * @example
+ * ```ts
+ * toU32Ids([1, 2, 3], 'candidate ids'); // Uint32Array [1, 2, 3]
+ * toU32Ids([2 ** 32], 'candidate ids'); // throws RangeError
+ * ```
+ */
+export declare function toU32Ids(values: ArrayLike<number>, what: string): Uint32Array;

package/lib/to-u32-ids.js

@@ -0,0 +1,22 @@
+import { assertU32Id } from './assert-u32-id.js';
+/**
+ * Copies row ids into the `u32` array the addon takes, refusing any value a
+ * `Uint32Array` would silently change (see `assertU32Id`).
+ * @param values - The ids (database row ids): an array or a typed array.
+ * @param what - What the ids are, for the error message.
+ * @returns The ids as a `Uint32Array`.
+ * @throws {RangeError} When a value is not an integer in `0..=4294967294`.
+ * @example
+ * ```ts
+ * toU32Ids([1, 2, 3], 'candidate ids'); // Uint32Array [1, 2, 3]
+ * toU32Ids([2 ** 32], 'candidate ids'); // throws RangeError
+ * ```
+ */
+export function toU32Ids(values, what) {
+    const ids = new Uint32Array(values.length);
+    // `ArrayLike` has no iterator, so an index loop is the way to read it.
+    for (let index = 0; index < values.length; index++) {
+        ids[index] = assertU32Id(values[index], what);
+    }
+    return ids;
+}