@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/types.js

@@ -0,0 +1 @@
+export {};

package/lib/url-alias/compute-tier-a-alias-key.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Computes the Tier A alias key for `url`: two URLs share a Tier A key when
+ * they differ only by scheme (`http` vs `https` are folded together),
+ * host letter-casing, or a `/index.{ext}` path suffix — differences that
+ * are safe to treat as "the same resource" from the URL string alone, with
+ * no need to compare rendered content.
+ *
+ * This is a canonical-key function, not a pairwise comparator: grouping
+ * rows by exact string equality of this function's output is definitionally
+ * an equivalence relation (reflexive, symmetric, transitive). A pairwise
+ * "roughly equal" comparison would not be: non-transitivity is inherent to
+ * pairwise comparators (see ARCHITECTURE.md's "URL の自然順は、2 件ずつ比べる
+ * 比較関数ではなくバイト列キーの全順序で決める"), and does not apply to a
+ * canonical-key partition like this one.
+ *
+ * The query string is kept in the key as written — not normalized, reordered
+ * or dropped — so two URLs that differ only in their query never share a key:
+ * a query-string difference is not addressed by this feature (out of scope —
+ * see the `body_hash` masking, a separate content-hash-based mechanism, for
+ * detecting meaningless-parameter duplicates). The fragment is left out:
+ * fragments are client-side only and never affect server-rendered content.
+ *
+ * Each call crosses into the addon and parses the URL, and the addon builds
+ * both tiers' keys from that parse: to group many pages, use
+ * `groupAliasCandidates`, which parses each URL once for both keys, rather
+ * than calling this and `computeTierBAliasKey` per URL.
+ *
+ * The key is computed in the native addon (`@nitpicker/core`,
+ * `crates/nitpicker_url_key`). The JavaScript implementation
+ * (`parse-alias-key-parts.ts`, `format-alias-key.ts`) is not called from here
+ * and is excluded from the build: it is the oracle for
+ * `compute-alias-keys.parity.spec.ts`, which pins the addon to it, and so to
+ * the groupings existing archives already store (`content_items.alias_of_id`).
+ * @param url - The URL string to compute a key for.
+ * @returns The Tier A key, or `null` if `url` is not a parseable http(s) URL.
+ * @example
+ * ```ts
+ * computeTierAAliasKey('http://Example.com/about/index.html');
+ * // 'example.com/about/'
+ * computeTierAAliasKey('https://example.com/about/');
+ * // 'example.com/about/' -- same key: scheme and /index.html are folded
+ * ```
+ */
+export declare function computeTierAAliasKey(url: string): string | null;

package/lib/url-alias/compute-tier-a-alias-key.js

@@ -0,0 +1,47 @@
+import { computeAliasKeys } from '@nitpicker/core/compute-alias-keys';
+/**
+ * Computes the Tier A alias key for `url`: two URLs share a Tier A key when
+ * they differ only by scheme (`http` vs `https` are folded together),
+ * host letter-casing, or a `/index.{ext}` path suffix — differences that
+ * are safe to treat as "the same resource" from the URL string alone, with
+ * no need to compare rendered content.
+ *
+ * This is a canonical-key function, not a pairwise comparator: grouping
+ * rows by exact string equality of this function's output is definitionally
+ * an equivalence relation (reflexive, symmetric, transitive). A pairwise
+ * "roughly equal" comparison would not be: non-transitivity is inherent to
+ * pairwise comparators (see ARCHITECTURE.md's "URL の自然順は、2 件ずつ比べる
+ * 比較関数ではなくバイト列キーの全順序で決める"), and does not apply to a
+ * canonical-key partition like this one.
+ *
+ * The query string is kept in the key as written — not normalized, reordered
+ * or dropped — so two URLs that differ only in their query never share a key:
+ * a query-string difference is not addressed by this feature (out of scope —
+ * see the `body_hash` masking, a separate content-hash-based mechanism, for
+ * detecting meaningless-parameter duplicates). The fragment is left out:
+ * fragments are client-side only and never affect server-rendered content.
+ *
+ * Each call crosses into the addon and parses the URL, and the addon builds
+ * both tiers' keys from that parse: to group many pages, use
+ * `groupAliasCandidates`, which parses each URL once for both keys, rather
+ * than calling this and `computeTierBAliasKey` per URL.
+ *
+ * The key is computed in the native addon (`@nitpicker/core`,
+ * `crates/nitpicker_url_key`). The JavaScript implementation
+ * (`parse-alias-key-parts.ts`, `format-alias-key.ts`) is not called from here
+ * and is excluded from the build: it is the oracle for
+ * `compute-alias-keys.parity.spec.ts`, which pins the addon to it, and so to
+ * the groupings existing archives already store (`content_items.alias_of_id`).
+ * @param url - The URL string to compute a key for.
+ * @returns The Tier A key, or `null` if `url` is not a parseable http(s) URL.
+ * @example
+ * ```ts
+ * computeTierAAliasKey('http://Example.com/about/index.html');
+ * // 'example.com/about/'
+ * computeTierAAliasKey('https://example.com/about/');
+ * // 'example.com/about/' -- same key: scheme and /index.html are folded
+ * ```
+ */
+export function computeTierAAliasKey(url) {
+    return computeAliasKeys(url)?.tierA ?? null;
+}

package/lib/url-alias/compute-tier-b-alias-key.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Computes the Tier B alias key for `url`: like `computeTierAAliasKey`,
+ * but additionally strips one trailing `/` from the path (the bare root
+ * path `/` is left alone). Two URLs sharing a Tier B key differ only by
+ * trailing-slash presence — a resource-identity signal considerably weaker
+ * than Tier A's (a web server can legitimately serve different content at
+ * `/foo` and `/foo/`), so callers must additionally require a matching
+ * `page_meta.body_hash` before treating a Tier B match as the same page.
+ *
+ * Computed in the native addon like the Tier A key — one call and one parse
+ * per URL, so to group many pages use `groupAliasCandidates` — see
+ * `computeTierAAliasKey` for what stays as the oracle.
+ * @param url - The URL string to compute a key for.
+ * @returns The Tier B key, or `null` if `url` is not a parseable http(s) URL.
+ * @example
+ * ```ts
+ * computeTierBAliasKey('https://example.com/foo');
+ * computeTierBAliasKey('https://example.com/foo/');
+ * // both: 'example.com/foo' -- same key, but the caller must still confirm
+ * // a matching body_hash before merging these as the same page.
+ * ```
+ */
+export declare function computeTierBAliasKey(url: string): string | null;

package/lib/url-alias/compute-tier-b-alias-key.js

@@ -0,0 +1,26 @@
+import { computeAliasKeys } from '@nitpicker/core/compute-alias-keys';
+/**
+ * Computes the Tier B alias key for `url`: like `computeTierAAliasKey`,
+ * but additionally strips one trailing `/` from the path (the bare root
+ * path `/` is left alone). Two URLs sharing a Tier B key differ only by
+ * trailing-slash presence — a resource-identity signal considerably weaker
+ * than Tier A's (a web server can legitimately serve different content at
+ * `/foo` and `/foo/`), so callers must additionally require a matching
+ * `page_meta.body_hash` before treating a Tier B match as the same page.
+ *
+ * Computed in the native addon like the Tier A key — one call and one parse
+ * per URL, so to group many pages use `groupAliasCandidates` — see
+ * `computeTierAAliasKey` for what stays as the oracle.
+ * @param url - The URL string to compute a key for.
+ * @returns The Tier B key, or `null` if `url` is not a parseable http(s) URL.
+ * @example
+ * ```ts
+ * computeTierBAliasKey('https://example.com/foo');
+ * computeTierBAliasKey('https://example.com/foo/');
+ * // both: 'example.com/foo' -- same key, but the caller must still confirm
+ * // a matching body_hash before merging these as the same page.
+ * ```
+ */
+export function computeTierBAliasKey(url) {
+    return computeAliasKeys(url)?.tierB ?? null;
+}

package/lib/url-alias/group-alias-candidates.d.ts

@@ -0,0 +1,27 @@
+import type { AliasCandidate } from '@nitpicker/core/types';
+/**
+ * Groups the pages that are the same page under another URL
+ * (`content_items.alias_of_id`) and elects each group's representative: the
+ * pages whose Tier A key (`computeTierAAliasKey`) and title match, or whose
+ * Tier B key (`computeTierBAliasKey`), title and body hash match, and the
+ * pages chained to them by either.
+ *
+ * `@nitpicker/query` reaches the native addon through this archive function
+ * (see ARCHITECTURE.md's dependency rule); the full grouping and election rules
+ * are the docs of `groupAliasCandidates` in `@nitpicker/core`, which this
+ * calls as it is. The persisted result is pinned to the JavaScript reference by
+ * `group-alias-candidates.parity.spec.ts` and `scripts/isolated-alias-golden.mjs`.
+ * @param candidates - The pages. `urlId` is `url_refs.id`: two candidates
+ *   have the same URL exactly when they have the same id.
+ * @returns For each candidate, the index of its group's representative, or
+ *   `-1` for a representative and for a candidate in a group of one.
+ * @example
+ * ```ts
+ * const page = { titleTextId: 1, canonicalUrlId: null, bodyHash: null };
+ * groupAliasCandidates([
+ * 	{ ...page, url: 'https://example.com/about/index.html', urlId: 1 },
+ * 	{ ...page, url: 'https://example.com/about/', urlId: 2 },
+ * ]); // Int32Array [1, -1]
+ * ```
+ */
+export declare function groupAliasCandidates(candidates: readonly AliasCandidate[]): Int32Array;

package/lib/url-alias/group-alias-candidates.js

@@ -0,0 +1,29 @@
+import { groupAliasCandidates as groupAliasCandidatesNative } from '@nitpicker/core/group-alias-candidates';
+/**
+ * Groups the pages that are the same page under another URL
+ * (`content_items.alias_of_id`) and elects each group's representative: the
+ * pages whose Tier A key (`computeTierAAliasKey`) and title match, or whose
+ * Tier B key (`computeTierBAliasKey`), title and body hash match, and the
+ * pages chained to them by either.
+ *
+ * `@nitpicker/query` reaches the native addon through this archive function
+ * (see ARCHITECTURE.md's dependency rule); the full grouping and election rules
+ * are the docs of `groupAliasCandidates` in `@nitpicker/core`, which this
+ * calls as it is. The persisted result is pinned to the JavaScript reference by
+ * `group-alias-candidates.parity.spec.ts` and `scripts/isolated-alias-golden.mjs`.
+ * @param candidates - The pages. `urlId` is `url_refs.id`: two candidates
+ *   have the same URL exactly when they have the same id.
+ * @returns For each candidate, the index of its group's representative, or
+ *   `-1` for a representative and for a candidate in a group of one.
+ * @example
+ * ```ts
+ * const page = { titleTextId: 1, canonicalUrlId: null, bodyHash: null };
+ * groupAliasCandidates([
+ * 	{ ...page, url: 'https://example.com/about/index.html', urlId: 1 },
+ * 	{ ...page, url: 'https://example.com/about/', urlId: 2 },
+ * ]); // Int32Array [1, -1]
+ * ```
+ */
+export function groupAliasCandidates(candidates) {
+    return groupAliasCandidatesNative(candidates);
+}

package/lib/url-alias/types.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Components a Tier A / Tier B alias key is built from. Used only by the
+ * JavaScript alias-key implementation (`parse-alias-key-parts.ts`,
+ * `format-alias-key.ts`), which is the parity oracle for the native keys and
+ * is excluded from the build (`tsconfig.json`).
+ */
+export interface AliasKeyParts {
+    host: string;
+    port: string;
+    path: string;
+    search: string;
+}

package/lib/url-alias/types.js

@@ -0,0 +1 @@
+export {};

package/lib/url-pattern/compute-shape-key.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Computes a URL "shape" key: the host plus path/query with every path
+ * segment that contains a digit collapsed to a fixed placeholder, and every
+ * query value (regardless of content) collapsed to a fixed placeholder.
+ *
+ * This absorbs both the "numeric pager" trap shape (`/news/date/2024/` and
+ * `/news/date/1.5e+32/` collapse to the same key) and the "query trap" shape
+ * (`?page=1` / `?page=2` / `?session=ab12cd` all collapse to the same key),
+ * without needing two separate `parentPath` definitions the way issue
+ * #208's original proposal did.
+ *
+ * Uses `./decompose-url.ts` (the pagination-detection one) — NOT
+ * `../populate-ref-tables/decompose-url.ts`, an unrelated same-named
+ * module with a different `DecomposedUrl` shape used for ref-table population.
+ *
+ * The masking rule here is the deliberate inverse of the body-hash mask
+ * (`@nitpicker/core`'s `crates/nitpicker_html_scan/src/mask_dynamic_ids.rs`,
+ * which `../body-hash/compute-body-hash.ts` runs): that mask leaves
+ * pure-digit tokens untouched (they are more likely stable content than a
+ * dynamic id) and only masks mixed alphanumeric runs. A shape key needs the
+ * opposite: ANY digit inside a path segment marks it as "probably a
+ * pagination/date/id token", so the whole segment is collapsed. Do not share
+ * masking logic between the two — they classify the same kind of text for
+ * opposite purposes.
+ *
+ * Query keys are additionally run through `flattenQueryKeys` before being
+ * folded into the key, because `ExURL`'s query normalization (`URLSearchParams`
+ * decode-then-re-encode round trip in `@d-zero/shared/parse-url`) means a
+ * site that double-encodes `&`/`=` inside a query value produces a URL whose
+ * query string still arrives here percent-encoded — `decomposeUrl`'s literal
+ * `&`/`=` split then sees ONE key that is itself a percent-encoded
+ * `key=value&key=value...` blob, with any variable content buried inside
+ * that key name rather than in a value. Without flattening, every such URL
+ * gets a distinct key name and therefore a distinct shape key, which
+ * silently defeats `--dedupe-cap` (issue #351) no matter the threshold.
+ *
+ * This changes the shape key computed for URLs that hit the flattening
+ * path. `dedupe_cap_events.shape_key` (see `archive/create-adjunct-tables.ts`)
+ * is an append-only value frozen at crawl time and never rewritten, so an
+ * archive crawled before this fix that happened to already have a
+ * `dedupe_cap_events` row for such a URL (rare, since the bug largely
+ * prevented capping from firing on it in the first place) will not
+ * re-match on the next `backfillDedupeCapEventId` recompute — the same
+ * self-healing-backfill trade-off any shape-key algorithm change makes.
+ * @param url - A URL string (protocol-agnostic `//host/...` or full
+ *   `https://host/...`), typically `ExURL.withoutHashAndAuth`.
+ * @returns The shape key, or `null` if `url` cannot be decomposed.
+ * @example
+ * ```ts
+ * computeShapeKey('//example.com/news/date/2024/');
+ * // => 'example.com/news/date/{n}/'
+ * computeShapeKey('//example.com/news/date/1.5e+32/');
+ * // => 'example.com/news/date/{n}/' — same shape
+ * computeShapeKey('//example.com/list?page=1');
+ * // => 'example.com/list?page={v}'
+ * ```
+ */
+export declare function computeShapeKey(url: string): string | null;

package/lib/url-pattern/compute-shape-key.js

@@ -0,0 +1,129 @@
+import { decodeURIComponentSafely } from '@d-zero/shared/decode-uri-safely';
+import { decomposeUrl } from './decompose-url.js';
+const DIGIT_CONTAINING_SEGMENT_PATTERN = /\d/;
+const SEGMENT_PLACEHOLDER = '{n}';
+const VALUE_PLACEHOLDER = '{v}';
+/**
+ * Recursively decodes and re-splits a single query key that may itself be a
+ * percent-encoded `key=value&key=value...` blob (see the "query trap" case
+ * in `computeShapeKey`'s doc comment), peeling one layer of encoding at a
+ * time so a multiply-encoded blob (`%2526` → `%26` → `&`) still converges.
+ *
+ * Each call first decodes via `decodeURIComponentSafely` (falls back to the
+ * input unchanged on malformed `%` sequences). If decoding changed
+ * anything, there may be another layer underneath, so it recurses on the
+ * decoded string to peel further. Once decoding is a no-op (fully decoded,
+ * or undecodable), the value is at a fixed point: split on `&`/`=` if
+ * either is now present and recurse on the pieces (each piece may need its
+ * own decode chain); otherwise it's a leaf.
+ *
+ * Every recursive call operates on a string strictly shorter than its
+ * input — a successful decode always shrinks length (a `%XX` triplet never
+ * expands to more than 1 output character), and splitting on a delimiter
+ * that is present at a fixed point always yields a strictly shorter piece
+ * — so this terminates for any finite input without needing an explicit
+ * depth limit.
+ * @param key - A single raw query key, as returned by `decomposeUrl`.
+ * @returns The one or more sub-key names this key flattens to.
+ */
+function flattenQueryKey(key) {
+    const decoded = decodeURIComponentSafely(key);
+    if (decoded !== key)
+        return flattenQueryKey(decoded);
+    if (!decoded.includes('=') && !decoded.includes('&'))
+        return [decoded];
+    return decoded.split('&').flatMap((pair) => flattenQueryKey(pair.split('=')[0] ?? ''));
+}
+/**
+ * Flattens and normalizes a list of query keys so that keys whose variable
+ * content leaked into the key name itself (via nested percent-encoding)
+ * still converge to the same set of sub-key names regardless of the
+ * variable values or the sub-pairs' original order.
+ *
+ * Deliberately does NOT deduplicate the flattened keys: a legitimate
+ * multi-value query (`?tag=a&tag=b`) must stay distinguishable from a
+ * single-value one (`?tag=a`) — collapsing repeated key names would fold
+ * pages with a different facet count into the same shape, which is a
+ * correctness regression unrelated to the encoding bug this function fixes.
+ * @param keys - Raw query keys, as returned by `decomposeUrl`.
+ * @returns The flattened, sorted key names.
+ */
+function flattenQueryKeys(keys) {
+    return keys
+        .flatMap((key) => flattenQueryKey(key))
+        .toSorted((a, b) => a.localeCompare(b));
+}
+/**
+ * Computes a URL "shape" key: the host plus path/query with every path
+ * segment that contains a digit collapsed to a fixed placeholder, and every
+ * query value (regardless of content) collapsed to a fixed placeholder.
+ *
+ * This absorbs both the "numeric pager" trap shape (`/news/date/2024/` and
+ * `/news/date/1.5e+32/` collapse to the same key) and the "query trap" shape
+ * (`?page=1` / `?page=2` / `?session=ab12cd` all collapse to the same key),
+ * without needing two separate `parentPath` definitions the way issue
+ * #208's original proposal did.
+ *
+ * Uses `./decompose-url.ts` (the pagination-detection one) — NOT
+ * `../populate-ref-tables/decompose-url.ts`, an unrelated same-named
+ * module with a different `DecomposedUrl` shape used for ref-table population.
+ *
+ * The masking rule here is the deliberate inverse of the body-hash mask
+ * (`@nitpicker/core`'s `crates/nitpicker_html_scan/src/mask_dynamic_ids.rs`,
+ * which `../body-hash/compute-body-hash.ts` runs): that mask leaves
+ * pure-digit tokens untouched (they are more likely stable content than a
+ * dynamic id) and only masks mixed alphanumeric runs. A shape key needs the
+ * opposite: ANY digit inside a path segment marks it as "probably a
+ * pagination/date/id token", so the whole segment is collapsed. Do not share
+ * masking logic between the two — they classify the same kind of text for
+ * opposite purposes.
+ *
+ * Query keys are additionally run through `flattenQueryKeys` before being
+ * folded into the key, because `ExURL`'s query normalization (`URLSearchParams`
+ * decode-then-re-encode round trip in `@d-zero/shared/parse-url`) means a
+ * site that double-encodes `&`/`=` inside a query value produces a URL whose
+ * query string still arrives here percent-encoded — `decomposeUrl`'s literal
+ * `&`/`=` split then sees ONE key that is itself a percent-encoded
+ * `key=value&key=value...` blob, with any variable content buried inside
+ * that key name rather than in a value. Without flattening, every such URL
+ * gets a distinct key name and therefore a distinct shape key, which
+ * silently defeats `--dedupe-cap` (issue #351) no matter the threshold.
+ *
+ * This changes the shape key computed for URLs that hit the flattening
+ * path. `dedupe_cap_events.shape_key` (see `archive/create-adjunct-tables.ts`)
+ * is an append-only value frozen at crawl time and never rewritten, so an
+ * archive crawled before this fix that happened to already have a
+ * `dedupe_cap_events` row for such a URL (rare, since the bug largely
+ * prevented capping from firing on it in the first place) will not
+ * re-match on the next `backfillDedupeCapEventId` recompute — the same
+ * self-healing-backfill trade-off any shape-key algorithm change makes.
+ * @param url - A URL string (protocol-agnostic `//host/...` or full
+ *   `https://host/...`), typically `ExURL.withoutHashAndAuth`.
+ * @returns The shape key, or `null` if `url` cannot be decomposed.
+ * @example
+ * ```ts
+ * computeShapeKey('//example.com/news/date/2024/');
+ * // => 'example.com/news/date/{n}/'
+ * computeShapeKey('//example.com/news/date/1.5e+32/');
+ * // => 'example.com/news/date/{n}/' — same shape
+ * computeShapeKey('//example.com/list?page=1');
+ * // => 'example.com/list?page={v}'
+ * ```
+ */
+export function computeShapeKey(url) {
+    const decomposed = decomposeUrl(url);
+    if (!decomposed)
+        return null;
+    const { host, pathSegments, queryKeys } = decomposed;
+    const shapedSegments = pathSegments.map((segment) => DIGIT_CONTAINING_SEGMENT_PATTERN.test(segment) ? SEGMENT_PLACEHOLDER : segment);
+    let key = host;
+    if (shapedSegments.length > 0) {
+        key += `/${shapedSegments.join('/')}`;
+    }
+    if (queryKeys.length > 0) {
+        const flattenedKeys = flattenQueryKeys(queryKeys);
+        const pairs = flattenedKeys.map((k) => `${k}=${VALUE_PLACEHOLDER}`);
+        key += `?${pairs.join('&')}`;
+    }
+    return key;
+}

package/lib/url-pattern/decompose-url.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Intermediate representation of a URL split into comparable tokens.
+ * Used by pagination detection to identify which token changed between two URLs.
+ */
+export interface DecomposedUrl {
+    /** Hostname including port (e.g. `"example.com:8080"`). */
+    host: string;
+    /** Path segments split by `/` (e.g. `["page", "2"]` for `/page/2`). */
+    pathSegments: string[];
+    /** Sorted query parameter keys. */
+    queryKeys: string[];
+    /** Query parameter values sorted by their corresponding key. */
+    queryValues: string[];
+    /** Protocol prefix (e.g. `"https:"`) or empty string if protocol-agnostic. */
+    protocol: string;
+}
+/**
+ * Decomposes a URL string into its constituent tokens for comparison.
+ * Handles both full URLs (`https://host/path?q=v`) and protocol-agnostic
+ * URLs (`//host/path?q=v`). Query parameters are sorted by key for
+ * consistent comparison. {@link import('./reconstruct-url.js').reconstructUrl}
+ * is the inverse (with one token replaced); the two are kept side by side so
+ * the `DecomposedUrl` shape cannot drift between them.
+ * @param url - The URL string to decompose
+ * @returns The decomposed URL, or `null` if the format is invalid
+ * @example
+ * ```ts
+ * decomposeUrl('//example.com/page/2?b=1&a=x');
+ * // { host: 'example.com', pathSegments: ['page', '2'],
+ * //   queryKeys: ['a', 'b'], queryValues: ['x', '1'], protocol: '' }
+ * ```
+ */
+export declare function decomposeUrl(url: string): DecomposedUrl | null;

package/lib/url-pattern/decompose-url.js

@@ -0,0 +1,79 @@
+/**
+ * Decomposes a URL string into its constituent tokens for comparison.
+ * Handles both full URLs (`https://host/path?q=v`) and protocol-agnostic
+ * URLs (`//host/path?q=v`). Query parameters are sorted by key for
+ * consistent comparison. {@link import('./reconstruct-url.js').reconstructUrl}
+ * is the inverse (with one token replaced); the two are kept side by side so
+ * the `DecomposedUrl` shape cannot drift between them.
+ * @param url - The URL string to decompose
+ * @returns The decomposed URL, or `null` if the format is invalid
+ * @example
+ * ```ts
+ * decomposeUrl('//example.com/page/2?b=1&a=x');
+ * // { host: 'example.com', pathSegments: ['page', '2'],
+ * //   queryKeys: ['a', 'b'], queryValues: ['x', '1'], protocol: '' }
+ * ```
+ */
+export function decomposeUrl(url) {
+    // URL format: //host/path?query  or  //host?query  (protocol-agnostic)
+    // Also handle protocol://host/path?query
+    let work = url;
+    let protocol = '';
+    // Strip protocol
+    const protoMatch = /^(https?:)?\/\//.exec(work);
+    if (!protoMatch)
+        return null;
+    protocol = protoMatch[1] ?? '';
+    work = work.slice(protoMatch[0].length);
+    // Split host from rest
+    const slashIdx = work.indexOf('/');
+    const qmarkIdx = work.indexOf('?');
+    let host;
+    let pathPart;
+    let queryPart;
+    if (slashIdx === -1 && qmarkIdx === -1) {
+        host = work;
+        pathPart = '';
+        queryPart = '';
+    }
+    else if (slashIdx === -1) {
+        host = work.slice(0, qmarkIdx);
+        pathPart = '';
+        queryPart = work.slice(qmarkIdx + 1);
+    }
+    else {
+        host = work.slice(0, slashIdx);
+        const pathAndQuery = work.slice(slashIdx + 1);
+        const pq = pathAndQuery.indexOf('?');
+        if (pq === -1) {
+            pathPart = pathAndQuery;
+            queryPart = '';
+        }
+        else {
+            pathPart = pathAndQuery.slice(0, pq);
+            queryPart = pathAndQuery.slice(pq + 1);
+        }
+    }
+    const pathSegments = pathPart ? pathPart.split('/') : [];
+    // Parse query into sorted key-value pairs
+    const queryPairs = [];
+    if (queryPart) {
+        for (const pair of queryPart.split('&')) {
+            const eqIdx = pair.indexOf('=');
+            if (eqIdx === -1) {
+                queryPairs.push([pair, '']);
+            }
+            else {
+                queryPairs.push([pair.slice(0, eqIdx), pair.slice(eqIdx + 1)]);
+            }
+        }
+    }
+    queryPairs.sort((a, b) => a[0].localeCompare(b[0]));
+    return {
+        host,
+        pathSegments,
+        queryKeys: queryPairs.map(([k]) => k),
+        queryValues: queryPairs.map(([, v]) => v),
+        protocol,
+    };
+}

package/lib/url-pattern/reconstruct-url.d.ts

@@ -0,0 +1,15 @@
+import type { DecomposedUrl } from './decompose-url.js';
+/**
+ * Reconstructs a URL string from a decomposed representation with one
+ * token replaced at the specified index.
+ * @param decomposed - The decomposed URL to reconstruct
+ * @param tokenIndex - Index in the combined token array (path segments + query values)
+ * @param newValue - The replacement value for the token at `tokenIndex`
+ * @returns The reconstructed URL string
+ * @example
+ * ```ts
+ * const decomposed = decomposeUrl('//example.com/page/3')!;
+ * reconstructUrl(decomposed, 1, '4'); // '//example.com/page/4'
+ * ```
+ */
+export declare function reconstructUrl(decomposed: DecomposedUrl, tokenIndex: number, newValue: string): string;

package/lib/url-pattern/reconstruct-url.js

@@ -0,0 +1,33 @@
+/**
+ * Reconstructs a URL string from a decomposed representation with one
+ * token replaced at the specified index.
+ * @param decomposed - The decomposed URL to reconstruct
+ * @param tokenIndex - Index in the combined token array (path segments + query values)
+ * @param newValue - The replacement value for the token at `tokenIndex`
+ * @returns The reconstructed URL string
+ * @example
+ * ```ts
+ * const decomposed = decomposeUrl('//example.com/page/3')!;
+ * reconstructUrl(decomposed, 1, '4'); // '//example.com/page/4'
+ * ```
+ */
+export function reconstructUrl(decomposed, tokenIndex, newValue) {
+    const { host, pathSegments, queryKeys, queryValues, protocol } = decomposed;
+    const newPathSegments = [...pathSegments];
+    const newQueryValues = [...queryValues];
+    if (tokenIndex < pathSegments.length) {
+        newPathSegments[tokenIndex] = newValue;
+    }
+    else {
+        newQueryValues[tokenIndex - pathSegments.length] = newValue;
+    }
+    let url = `${protocol}//${host}`;
+    if (newPathSegments.length > 0) {
+        url += `/${newPathSegments.join('/')}`;
+    }
+    if (queryKeys.length > 0) {
+        const pairs = queryKeys.map((k, i) => `${encodeURIComponent(k)}=${encodeURIComponent(newQueryValues[i] ?? '')}`);
+        url += `?${pairs.join('&')}`;
+    }
+    return url;
+}

package/lib/url-sort/compute-url-sort-ranks.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Ranks URLs in the natural order the viewer lists pages and resources in:
+ * numbers inside a path compare by value (`page2` before `page10`), letter
+ * case and percent-escapes do not split the order, a directory's own page
+ * comes before its index pages and its children, and the host decides before
+ * the path.
+ *
+ * The order is a total order — the same URLs in any input order get the same
+ * ranking — and it is persisted (`viewer_pages.natural_url_rank`,
+ * `viewer_url_sort_keys`), so a change to its definition needs a read model
+ * schema version bump. The definition lives in the native crate
+ * (`@nitpicker/core`, `crates/nitpicker_url_key`).
+ *
+ * Each URL is parsed once, natively. Memory is proportional to the URLs'
+ * total length — the UTF-8 bytes handed over plus native keys of roughly the
+ * same size again, on top of the caller's own copy of the strings — so a
+ * caller with a very large URL set ranks the distinct URLs only.
+ * @param urls - The URLs to rank; identical URLs rank in input order.
+ * @returns For each input position, the zero-based position that URL takes in
+ *   the sorted list — a permutation of `0..urls.length`.
+ * @example
+ * ```ts
+ * computeUrlSortRanks([
+ * 	'https://example.com/page10',
+ * 	'https://example.com/page2',
+ * 	'https://example.com/',
+ * ]); // Uint32Array [2, 1, 0]
+ * ```
+ */
+export declare function computeUrlSortRanks(urls: readonly string[]): Uint32Array;

package/lib/url-sort/compute-url-sort-ranks.js

@@ -0,0 +1,33 @@
+import { computeUrlSortRanks as computeUrlSortRanksNative } from '@nitpicker/core/compute-url-sort-ranks';
+/**
+ * Ranks URLs in the natural order the viewer lists pages and resources in:
+ * numbers inside a path compare by value (`page2` before `page10`), letter
+ * case and percent-escapes do not split the order, a directory's own page
+ * comes before its index pages and its children, and the host decides before
+ * the path.
+ *
+ * The order is a total order — the same URLs in any input order get the same
+ * ranking — and it is persisted (`viewer_pages.natural_url_rank`,
+ * `viewer_url_sort_keys`), so a change to its definition needs a read model
+ * schema version bump. The definition lives in the native crate
+ * (`@nitpicker/core`, `crates/nitpicker_url_key`).
+ *
+ * Each URL is parsed once, natively. Memory is proportional to the URLs'
+ * total length — the UTF-8 bytes handed over plus native keys of roughly the
+ * same size again, on top of the caller's own copy of the strings — so a
+ * caller with a very large URL set ranks the distinct URLs only.
+ * @param urls - The URLs to rank; identical URLs rank in input order.
+ * @returns For each input position, the zero-based position that URL takes in
+ *   the sorted list — a permutation of `0..urls.length`.
+ * @example
+ * ```ts
+ * computeUrlSortRanks([
+ * 	'https://example.com/page10',
+ * 	'https://example.com/page2',
+ * 	'https://example.com/',
+ * ]); // Uint32Array [2, 1, 0]
+ * ```
+ */
+export function computeUrlSortRanks(urls) {
+    return computeUrlSortRanksNative(urls);
+}

package/lib/utils/array/each-splitted.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Splits an array into chunks of the specified size and executes a callback
+ * on each chunk in parallel using `Promise.all`.
+ * @template T - The element type of the array.
+ * @param a - The array to split into chunks.
+ * @param count - The maximum number of elements per chunk.
+ * @param callback - A function to invoke on each chunk. May be synchronous or asynchronous.
+ * @returns A promise that resolves when all chunk callbacks have completed.
+ */
+export declare function eachSplitted<T>(a: readonly T[], count: number, callback: (items: T[]) => void | Promise<void>): Promise<void>;