@nitpicker/crawler 0.4.1

package/lib/crawler/link-list.js

@@ -0,0 +1,248 @@
+import { isError } from '@d-zero/beholder';
+import { isLowerLayer } from '@d-zero/shared/is-lower-layer';
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+import { protocolAgnosticKey } from './protocol-agnostic-key.js';
+/**
+ * Manages the queue of URLs discovered during crawling.
+ *
+ * Tracks URLs across three states: pending (queued but not started),
+ * in-progress (currently being scraped), and done (scraping completed).
+ * Provides deduplication based on `withoutHashAndAuth` normalization
+ * and tracks page completion counts for progress reporting.
+ */
+export default class LinkList {
+    #completePages = 0;
+    #done = new Set();
+    #metadataOnlyFlag = new Set();
+    #pending = new Set();
+    #predictedFlag = new Set();
+    #progress = new Set();
+    /**
+     * The number of successfully completed internal HTML pages.
+     *
+     * Only counts pages that are internal, in a lower layer, use HTTP(S),
+     * have no error status, and have `text/html` content type.
+     */
+    get completePages() {
+        return this.#completePages;
+    }
+    /**
+     * Add a URL to the pending queue if it has not been seen before.
+     *
+     * Deduplication is based on the URL's `withoutHashAndAuth` representation.
+     * If the URL is already pending, in progress, or done, this is a no-op.
+     * @param linkUrl - The parsed URL to add to the queue.
+     * @param options - Optional flags for the URL.
+     * @param options.metadataOnly - If `true`, marks this URL for title-only scraping
+     *   (metadata extraction without full page processing).
+     * @param options.predicted - If `true`, marks this URL as a predicted pagination guess
+     *   that should be discarded if it returns a 4xx/5xx status.
+     */
+    add(linkUrl, options) {
+        const key = protocolAgnosticKey(linkUrl.withoutHashAndAuth);
+        if (this.#pending.has(key) || this.#progress.has(key) || this.#done.has(key)) {
+            return;
+        }
+        this.#pending.add(key);
+        if (options?.metadataOnly) {
+            this.#metadataOnlyFlag.add(key);
+        }
+        if (options?.predicted) {
+            this.#predictedFlag.add(key);
+        }
+    }
+    /**
+     * Mark a URL as completed and record its scrape result.
+     *
+     * Moves the URL from pending/progress to done, constructs a {@link Link}
+     * object with scope and layer information, and increments the page counter
+     * if the result qualifies as a valid HTML page.
+     * @param url - The URL that has been scraped.
+     * @param scope - The current scope map (hostname to scope URLs).
+     * @param resource - The scrape result containing page data and/or error information.
+     * @param resource.page - The scraped page data, if the scrape succeeded.
+     * @param resource.error - The error object, if the scrape failed.
+     * @param options - URL parsing options (e.g., `disableQueries`).
+     * @returns The constructed {@link Link} object, or `null` if the URL was not in the queue.
+     */
+    done(url, scope, resource, options) {
+        const key = protocolAgnosticKey(url.withoutHashAndAuth);
+        if (!(this.#pending.has(key) || this.#progress.has(key))) {
+            return null;
+        }
+        this.#pending.delete(key);
+        this.#progress.delete(key);
+        const linkUrl = parseUrl(url, options);
+        if (!linkUrl) {
+            return null;
+        }
+        const sameScopes = scope.get(linkUrl.hostname);
+        const link = {
+            url: linkUrl,
+            isLowerLayer: sameScopes
+                ? sameScopes.some((s) => isLowerLayer(linkUrl.href, s.href, options))
+                : false,
+            isExternal: !sameScopes,
+        };
+        const urlList = new Set([key]);
+        if (resource.page) {
+            link.dest = {
+                redirectPaths: resource.page.redirectPaths,
+                status: resource.page.status,
+                statusText: resource.page.statusText,
+                contentType: resource.page.contentType,
+                contentLength: resource.page.contentLength,
+                responseHeaders: resource.page.responseHeaders,
+                title: resource.page.meta.title,
+            };
+            for (const path of resource.page.redirectPaths) {
+                urlList.add(protocolAgnosticKey(path));
+            }
+        }
+        if (resource.error?.message.includes('ERR_NAME_NOT_RESOLVED')) {
+            link.dest = {
+                redirectPaths: [],
+                status: -1,
+                statusText: resource.error.message,
+                contentType: null,
+                contentLength: null,
+                responseHeaders: {},
+            };
+        }
+        const isPageLink = isPage(link);
+        for (const passedUrl of urlList) {
+            this.#done.add(passedUrl);
+            if (isPageLink) {
+                this.#completePages += 1;
+            }
+        }
+        return link;
+    }
+    /**
+     * Get the current pending and in-progress URL lists.
+     * @returns An object containing arrays of pending and in-progress URL strings.
+     */
+    getLinks() {
+        return {
+            /** URLs queued but not yet started. */
+            pending: [...this.#pending.values()],
+            /** URLs currently being scraped. */
+            progress: [...this.#progress.values()],
+        };
+    }
+    /**
+     * Get a summary of crawl progress counts.
+     * @returns An object with total/completed counts for both all links and pages only.
+     */
+    getPageCount() {
+        const { pending, progress } = this.getLinks();
+        const pendingPages = pending;
+        const progressPages = progress;
+        const totalLinks = pending.length + progress.length + this.#done.size;
+        const completedLinks = this.#done.size;
+        const totalPages = pendingPages.length + progressPages.length + this.#completePages;
+        const completedPages = this.#completePages;
+        return {
+            /** Total number of discovered links (pending + progress + done). */
+            totalLinks,
+            /** Number of links that have been fully processed. */
+            completedLinks,
+            /** Total number of discovered pages (pending + progress + completed pages). */
+            totalPages,
+            /** Number of pages that have been successfully scraped. */
+            completedPages,
+        };
+    }
+    /**
+     * Check whether a URL is flagged for title-only scraping.
+     *
+     * Title-only scraping extracts only the page title and basic metadata,
+     * without processing anchors or capturing the full HTML.
+     * @param urlWithoutHashAndAuth - The normalized URL string (without hash and auth) to check.
+     * @returns `true` if the URL should be scraped in title-only mode.
+     */
+    isMetadataOnly(urlWithoutHashAndAuth) {
+        return this.#metadataOnlyFlag.has(protocolAgnosticKey(urlWithoutHashAndAuth));
+    }
+    /**
+     * Check whether a URL was added as a predicted pagination URL.
+     * @param urlWithoutHashAndAuth - The normalized URL string (without hash and auth) to check.
+     * @returns `true` if the URL was added with the predicted flag.
+     */
+    isPredicted(urlWithoutHashAndAuth) {
+        return this.#predictedFlag.has(protocolAgnosticKey(urlWithoutHashAndAuth));
+    }
+    /**
+     * Transition a URL from the pending state to the in-progress state.
+     *
+     * This should be called when scraping of the URL actually begins.
+     * If the URL is not in the pending set, this is a no-op.
+     * @param url - The URL that is now being actively scraped.
+     */
+    progress(url) {
+        const key = protocolAgnosticKey(url.withoutHashAndAuth);
+        if (!this.#pending.has(key)) {
+            return;
+        }
+        this.#pending.delete(key);
+        this.#progress.add(key);
+    }
+    /**
+     * Restore the link list state from a previous crawl session.
+     *
+     * Re-adds pending URLs to the queue and marks previously done URLs
+     * as completed, enabling the crawler to resume from where it left off.
+     * @param pending - URLs that were pending in the previous session.
+     * @param done - URLs that were already completed in the previous session.
+     * @param options - URL parsing options for re-parsing the pending URLs.
+     * @returns The parsed pending URLs that were successfully added to the queue.
+     */
+    resume(pending, done, options) {
+        const parsedPending = [];
+        for (const url of done) {
+            this.#done.add(protocolAgnosticKey(url));
+        }
+        for (const url of pending) {
+            const parsedUrl = parseUrl(url, options);
+            if (!parsedUrl) {
+                continue;
+            }
+            this.add(parsedUrl);
+            parsedPending.push(parsedUrl);
+        }
+        return parsedPending;
+    }
+}
+/**
+ * Determine whether a link represents a valid internal HTML page.
+ *
+ * A link qualifies as a "page" if it is:
+ * - Internal (not external)
+ * - In a lower layer of the scope
+ * - Using HTTP or HTTPS protocol
+ * - Has destination data with a non-error status
+ * - Has `text/html` content type
+ * @param link - The link to evaluate.
+ * @returns `true` if the link represents a valid internal HTML page.
+ */
+function isPage(link) {
+    if (link.isExternal) {
+        return false;
+    }
+    if (!link.isLowerLayer) {
+        return false;
+    }
+    if (!/^https?:$/.test(link.url.protocol)) {
+        return false;
+    }
+    if (!link.dest) {
+        return false;
+    }
+    if (isError(link.dest.status)) {
+        return false;
+    }
+    if (link.dest.contentType === 'text/html') {
+        return true;
+    }
+    return false;
+}

package/lib/crawler/link-to-page-data.d.ts

@@ -0,0 +1,14 @@
+import type { Link, PageData } from '../utils/index.js';
+/**
+ * Convert a {@link Link} object into a {@link PageData} structure.
+ *
+ * Creates a minimal PageData from the link's destination metadata. This is used
+ * when a full scrape is not performed (e.g., for external pages when
+ * `fetchExternal` is disabled, or when a scrape error produces a fallback result).
+ *
+ * Missing destination fields are filled with sensible defaults (e.g., status -1
+ * for unknown, empty arrays for anchors/images, empty string for HTML).
+ * @param link - The link to convert, containing URL and optional destination metadata.
+ * @returns A PageData object populated from the link's available data.
+ */
+export declare function linkToPageData(link: Link): PageData;

package/lib/crawler/link-to-page-data.js

@@ -0,0 +1,32 @@
+/**
+ * Convert a {@link Link} object into a {@link PageData} structure.
+ *
+ * Creates a minimal PageData from the link's destination metadata. This is used
+ * when a full scrape is not performed (e.g., for external pages when
+ * `fetchExternal` is disabled, or when a scrape error produces a fallback result).
+ *
+ * Missing destination fields are filled with sensible defaults (e.g., status -1
+ * for unknown, empty arrays for anchors/images, empty string for HTML).
+ * @param link - The link to convert, containing URL and optional destination metadata.
+ * @returns A PageData object populated from the link's available data.
+ */
+export function linkToPageData(link) {
+    return {
+        url: link.url,
+        redirectPaths: link.dest?.redirectPaths || [],
+        isTarget: !link.isExternal,
+        isExternal: link.isExternal,
+        status: link.dest?.status || -1,
+        statusText: link.dest?.statusText || 'UnknownError',
+        contentType: link.dest?.contentType || null,
+        contentLength: link.dest?.contentLength || null,
+        responseHeaders: link.dest?.responseHeaders || null,
+        meta: {
+            title: link.dest?.title || '',
+        },
+        anchorList: [],
+        imageList: [],
+        html: '',
+        isSkipped: false,
+    };
+}

package/lib/crawler/net-timeout-error.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Error thrown when a network request (typically an HTTP HEAD check)
+ * exceeds the allowed timeout duration. Used by `fetchDestination`
+ * to signal that the destination server did not respond in time.
+ */
+export default class NetTimeoutError extends Error {
+    constructor(url?: string);
+    name: string;
+}

package/lib/crawler/net-timeout-error.js

@@ -0,0 +1,11 @@
+/**
+ * Error thrown when a network request (typically an HTTP HEAD check)
+ * exceeds the allowed timeout duration. Used by `fetchDestination`
+ * to signal that the destination server did not respond in time.
+ */
+export default class NetTimeoutError extends Error {
+    constructor(url) {
+        super(url ? `Timeout: ${url}` : 'Timeout');
+    }
+    name = 'NetTimeoutError';
+}

package/lib/crawler/network.d.ts

@@ -0,0 +1,30 @@
+import type { ExURL } from '@d-zero/shared/parse-url';
+import type { PageData } from '@nitpicker/beholder';
+/**
+ * Clears the in-memory cache of HTTP request results.
+ * Should be called between crawl sessions to prevent memory leaks.
+ */
+export declare function clearDestinationCache(): void;
+/**
+ * Fetches the destination metadata for a URL using an HTTP HEAD request (or GET as fallback).
+ *
+ * Results are cached in memory so that repeated calls for the same URL
+ * (without hash) return immediately. The request races against a 10-second
+ * timeout; if the server does not respond in time, a {@link NetTimeoutError} is thrown.
+ *
+ * If the server returns 405 (Method Not Allowed), 501 (Not Implemented), or 503
+ * (Service Unavailable) for a HEAD request, the function automatically retries with GET.
+ * @param url - The extended URL to fetch.
+ * @param isExternal - Whether the URL is external to the crawl scope.
+ * @param method - The HTTP method to use. Defaults to `"HEAD"`.
+ * @param options - Additional options.
+ * @param options.titleBytesLimit - When set, forces a GET request and reads up to this many
+ *   bytes from the response body to extract an HTML `<title>` tag. The connection is
+ *   destroyed as soon as the limit is reached or a title is found.
+ * @returns The page metadata obtained from the HTTP response.
+ * @throws {NetTimeoutError} If the request exceeds the 10-second timeout.
+ * @throws {Error} If the HTTP request fails for any other reason.
+ */
+export declare function fetchDestination(url: ExURL, isExternal: boolean, method?: string, options?: {
+    titleBytesLimit?: number;
+}): Promise<PageData>;

package/lib/crawler/network.js

@@ -0,0 +1,226 @@
+import { delay } from '@d-zero/shared/delay';
+import redirects from 'follow-redirects';
+import NetTimeoutError from './net-timeout-error.js';
+/**
+ * In-memory cache of HEAD request results keyed by URL (without hash).
+ * Stores either the successful {@link PageData} or the {@link Error} to avoid
+ * repeated requests to the same destination.
+ */
+const cacheMap = new Map();
+/**
+ * Clears the in-memory cache of HTTP request results.
+ * Should be called between crawl sessions to prevent memory leaks.
+ */
+export function clearDestinationCache() {
+    cacheMap.clear();
+}
+/**
+ * Fetches the destination metadata for a URL using an HTTP HEAD request (or GET as fallback).
+ *
+ * Results are cached in memory so that repeated calls for the same URL
+ * (without hash) return immediately. The request races against a 10-second
+ * timeout; if the server does not respond in time, a {@link NetTimeoutError} is thrown.
+ *
+ * If the server returns 405 (Method Not Allowed), 501 (Not Implemented), or 503
+ * (Service Unavailable) for a HEAD request, the function automatically retries with GET.
+ * @param url - The extended URL to fetch.
+ * @param isExternal - Whether the URL is external to the crawl scope.
+ * @param method - The HTTP method to use. Defaults to `"HEAD"`.
+ * @param options - Additional options.
+ * @param options.titleBytesLimit - When set, forces a GET request and reads up to this many
+ *   bytes from the response body to extract an HTML `<title>` tag. The connection is
+ *   destroyed as soon as the limit is reached or a title is found.
+ * @returns The page metadata obtained from the HTTP response.
+ * @throws {NetTimeoutError} If the request exceeds the 10-second timeout.
+ * @throws {Error} If the HTTP request fails for any other reason.
+ */
+export async function fetchDestination(url, isExternal, method = 'HEAD', options) {
+    const titleBytesLimit = options?.titleBytesLimit;
+    const cacheKey = titleBytesLimit == null ? url.withoutHash : `${url.withoutHash}:title`;
+    if (cacheMap.has(cacheKey)) {
+        const cache = cacheMap.get(cacheKey);
+        if (cache instanceof Error) {
+            throw cache;
+        }
+        return cache;
+    }
+    const effectiveMethod = titleBytesLimit == null ? method : 'GET';
+    const result = await Promise.race([
+        _fetchHead(url, isExternal, effectiveMethod, titleBytesLimit).catch((error) => (error instanceof Error ? error : new Error(String(error)))),
+        (async () => {
+            await delay(10 * 1000);
+            return new NetTimeoutError(url.href);
+        })(),
+    ]);
+    cacheMap.set(cacheKey, result);
+    if (result instanceof Error) {
+        throw result;
+    }
+    return result;
+}
+/**
+ * Performs the actual HTTP request to retrieve page metadata.
+ *
+ * Handles both HTTP and HTTPS protocols via `follow-redirects`, tracks redirect chains,
+ * and falls back to GET on certain status codes (405, 501, 503).
+ * @param url - The extended URL to request.
+ * @param isExternal - Whether the URL is external to the crawl scope.
+ * @param method - The HTTP method (`"HEAD"` or `"GET"`).
+ * @param titleBytesLimit - When set, reads up to this many bytes from the response body
+ *   to extract a `<title>` tag, then destroys the connection.
+ * @returns A promise resolving to {@link PageData} with response metadata.
+ */
+async function _fetchHead(url, isExternal, method, titleBytesLimit) {
+    return new Promise((resolve, reject) => {
+        const hostHeader = url.port ? `${url.hostname}:${url.port}` : url.hostname;
+        const request = {
+            protocol: url.protocol,
+            hostname: url.hostname,
+            port: url.port || undefined,
+            path: url.pathname,
+            method,
+            headers: {
+                host: hostHeader,
+                Connection: 'keep-alive',
+                Pragma: 'no-cache',
+                'Cache-Control': 'no-cache',
+                'Upgrade-Insecure-Requests': 1,
+                Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+                'Accept-Encoding': 'gzip, deflate',
+                'Accept-Language': 'ja,en;q=0.9,zh;q=0.8,en-US;q=0.7,pl;q=0.6,de;q=0.5,zh-CN;q=0.4,zh-TW;q=0.3,th;q=0.2,ko;q=0.1,fr;q=0.1',
+                // Range: url.extname?.toLowerCase() === 'pdf' ? 'bytes=0-0' : undefined,
+            },
+        };
+        if (url.username && url.password) {
+            request.auth = `${url.username}:${url.password}`;
+        }
+        let req;
+        let destroyed = false;
+        const response = (res) => {
+            const chunks = [];
+            let totalBytes = 0;
+            let settled = false;
+            const buildPageData = (title) => {
+                const redirectPaths = res.redirects.map((r) => r.url);
+                const _contentLength = Number.parseInt(res.headers['content-length'] || '');
+                const contentLength = Number.isFinite(_contentLength) ? _contentLength : null;
+                return {
+                    url,
+                    isTarget: !isExternal,
+                    isExternal,
+                    redirectPaths,
+                    status: res.statusCode || 0,
+                    statusText: res.statusMessage || '',
+                    contentType: res.headers['content-type']?.split(';')[0] || null,
+                    contentLength,
+                    responseHeaders: res.headers,
+                    meta: { title },
+                    imageList: [],
+                    anchorList: [],
+                    html: '',
+                    isSkipped: false,
+                };
+            };
+            if (titleBytesLimit == null) {
+                res.on('data', () => { });
+                res.on('end', async () => {
+                    let rep = buildPageData('');
+                    if (rep.status === 405) {
+                        if (method === 'GET') {
+                            reject(new Error(`Method Not Allowed: ${url.href} ${rep.statusText}`));
+                            return;
+                        }
+                        try {
+                            rep = await fetchDestination(url, isExternal, 'GET');
+                        }
+                        catch (error) {
+                            reject(error);
+                            return;
+                        }
+                    }
+                    if (rep.status === 501) {
+                        if (method === 'GET') {
+                            reject(new Error(`Method Not Implemented: ${url.href} ${rep.statusText}`));
+                            return;
+                        }
+                        await delay(5 * 1000);
+                        try {
+                            rep = await fetchDestination(url, isExternal, 'GET');
+                        }
+                        catch (error) {
+                            reject(error);
+                            return;
+                        }
+                    }
+                    if (rep.status === 503) {
+                        if (method === 'GET') {
+                            reject(new Error(`Retrying failed: ${url.href} ${rep.statusText}`));
+                            return;
+                        }
+                        await delay(5 * 1000);
+                        try {
+                            rep = await fetchDestination(url, isExternal, 'GET');
+                        }
+                        catch (error) {
+                            reject(error);
+                            return;
+                        }
+                    }
+                    resolve(rep);
+                });
+            }
+            else {
+                res.on('data', (chunk) => {
+                    if (settled)
+                        return;
+                    chunks.push(chunk);
+                    totalBytes += chunk.length;
+                    // Check for title in accumulated data so far
+                    const body = Buffer.concat(chunks).toString('utf8');
+                    const titleMatch = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(body);
+                    if (titleMatch) {
+                        settled = true;
+                        const title = titleMatch[1]?.trim() ?? '';
+                        resolve(buildPageData(title));
+                        destroyed = true;
+                        req.destroy();
+                        return;
+                    }
+                    // Reached byte limit without finding title
+                    if (totalBytes >= titleBytesLimit) {
+                        settled = true;
+                        resolve(buildPageData(''));
+                        destroyed = true;
+                        req.destroy();
+                    }
+                });
+                res.on('end', () => {
+                    if (settled)
+                        return;
+                    settled = true;
+                    // Stream ended before limit — try to extract title from what we have
+                    const body = Buffer.concat(chunks).toString('utf8');
+                    const titleMatch = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(body);
+                    const title = titleMatch?.[1]?.trim() ?? '';
+                    resolve(buildPageData(title));
+                });
+            }
+        };
+        if (url.protocol === 'https:') {
+            req = redirects.https.request({
+                ...request,
+                rejectUnauthorized: false,
+            }, response);
+        }
+        else {
+            req = redirects.http.request(request, response);
+        }
+        req.on('error', (error) => {
+            // Ignore errors caused by intentional req.destroy()
+            if (destroyed)
+                return;
+            reject(error);
+        });
+        req.end();
+    });
+}

package/lib/crawler/protocol-agnostic-key.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Returns a URL string with the protocol prefix (`http:` / `https:`) stripped.
+ *
+ * Used as a deduplication key so that HTTP and HTTPS variants of the
+ * same URL are treated as identical during crawling.
+ * @param url - A URL string (e.g. `"https://example.com/page"`)
+ * @returns The URL without its protocol prefix (e.g. `"//example.com/page"`)
+ */
+export declare function protocolAgnosticKey(url: string): string;

package/lib/crawler/protocol-agnostic-key.js

@@ -0,0 +1,11 @@
+/**
+ * Returns a URL string with the protocol prefix (`http:` / `https:`) stripped.
+ *
+ * Used as a deduplication key so that HTTP and HTTPS variants of the
+ * same URL are treated as identical during crawling.
+ * @param url - A URL string (e.g. `"https://example.com/page"`)
+ * @returns The URL without its protocol prefix (e.g. `"//example.com/page"`)
+ */
+export function protocolAgnosticKey(url) {
+    return url.replace(/^https?:/, '');
+}

package/lib/crawler/reconstruct-url.d.ts

@@ -0,0 +1,10 @@
+import type { DecomposedUrl } from './decompose-url.js';
+/**
+ * Reconstructs a URL string from a decomposed representation with one
+ * token replaced at the specified index.
+ * @param decomposed - The decomposed URL to reconstruct
+ * @param tokenIndex - Index in the combined token array (path segments + query values)
+ * @param newValue - The replacement value for the token at `tokenIndex`
+ * @returns The reconstructed URL string
+ */
+export declare function reconstructUrl(decomposed: DecomposedUrl, tokenIndex: number, newValue: string): string;

package/lib/crawler/reconstruct-url.js

@@ -0,0 +1,28 @@
+/**
+ * Reconstructs a URL string from a decomposed representation with one
+ * token replaced at the specified index.
+ * @param decomposed - The decomposed URL to reconstruct
+ * @param tokenIndex - Index in the combined token array (path segments + query values)
+ * @param newValue - The replacement value for the token at `tokenIndex`
+ * @returns The reconstructed URL string
+ */
+export function reconstructUrl(decomposed, tokenIndex, newValue) {
+    const { host, pathSegments, queryKeys, queryValues, protocol } = decomposed;
+    const newPathSegments = [...pathSegments];
+    const newQueryValues = [...queryValues];
+    if (tokenIndex < pathSegments.length) {
+        newPathSegments[tokenIndex] = newValue;
+    }
+    else {
+        newQueryValues[tokenIndex - pathSegments.length] = newValue;
+    }
+    let url = `${protocol}//${host}`;
+    if (newPathSegments.length > 0) {
+        url += `/${newPathSegments.join('/')}`;
+    }
+    if (queryKeys.length > 0) {
+        const pairs = queryKeys.map((k, i) => `${encodeURIComponent(k)}=${encodeURIComponent(newQueryValues[i] ?? '')}`);
+        url += `?${pairs.join('&')}`;
+    }
+    return url;
+}