@nitpicker/crawler 0.4.1

package/lib/qzilla.js

@@ -0,0 +1,292 @@
+import Archive from './archive/archive.js';
+import { cleanObject, EventEmitter, parseUrl, sortUrl } from './utils/index.js';
+import pkg from '../package.json' with { type: 'json' };
+import { Crawler } from './crawler/index.js';
+import { crawlerLog, log } from './debug.js';
+/**
+ * Default list of external URL prefixes excluded from crawling.
+ * Includes social media sharing endpoints that are commonly linked
+ * but provide no useful crawl data.
+ */
+export const DEFAULT_EXCLUDED_EXTERNAL_URLS = [
+    'https://social-plugins.line.me',
+    'https://access.line.me',
+    'https://lineit.line.me',
+    'https://line.me',
+    'https://plus.google.com',
+    'https://twitter.com',
+    'https://x.com',
+    'https://www.facebook.com/share.php',
+    'https://www.facebook.com/share/',
+    'https://www.facebook.com/sharer/',
+    'https://www.facebook.com/share_channel/',
+    'https://www.google.com',
+];
+/**
+ * The main entry point for Qzilla web crawling and archiving.
+ *
+ * Qzilla orchestrates the full lifecycle of a crawl session: it creates an archive,
+ * configures a {@link Crawler}, processes discovered pages and resources, and
+ * writes the final archive file. It emits events defined by {@link QzillaEvent}.
+ *
+ * Instances are created via the static factory methods {@link Qzilla.crawling}
+ * or {@link Qzilla.resume}; the constructor is private.
+ * @example
+ * ```ts
+ * const qzilla = await Qzilla.crawling(['https://example.com'], { recursive: true });
+ * await qzilla.write();
+ * ```
+ */
+export class Qzilla extends EventEmitter {
+    #archive;
+    #crawler;
+    #fromList;
+    /**
+     * The underlying archive instance used for storing crawl results.
+     */
+    get archive() {
+        return this.#archive;
+    }
+    // eslint-disable-next-line no-restricted-syntax
+    constructor(archive, options) {
+        super();
+        this.#fromList = !!options?.list;
+        this.#archive = archive;
+        this.#archive.on('error', (e) => {
+            this.#crawler.abort();
+            throw e;
+        });
+        this.#crawler = new Crawler({
+            interval: options?.interval || 0,
+            parallels: options?.parallels || 0,
+            isGettingImages: options?.image,
+            executablePath: options?.executablePath || null,
+            fetchExternal: options?.fetchExternal ?? true,
+            recursive: options?.recursive ?? true,
+            scope: options?.scope ?? [],
+            excludes: optMultiParam(options?.excludes),
+            excludeKeywords: optMultiParam(options?.excludeKeywords),
+            excludeUrls: [
+                ...DEFAULT_EXCLUDED_EXTERNAL_URLS,
+                ...optMultiParam(options?.excludeUrls),
+            ],
+            depthOnAvoid: options?.depthOnAvoid || 10,
+            disableQueries: options?.disableQueries,
+            screenshot: Archive.joinPath(archive.tmpDir, 'screenshots'),
+            verbose: options?.verbose ?? false,
+        });
+    }
+    /**
+     * Abort the current crawl and archive operations.
+     *
+     * Delegates to the archive's abort method, which stops all in-progress
+     * database writes and cleans up temporary resources.
+     * @returns The result of the archive abort operation.
+     */
+    abort() {
+        return this.#archive.abort();
+    }
+    /**
+     * Execute the crawl for the given list of URLs.
+     *
+     * Sets up event listeners on the crawler, starts crawling, and resolves
+     * when the crawl completes. Discovered pages, external pages, skipped pages,
+     * and resources are forwarded to the archive for storage.
+     * @param list - The list of parsed URLs to crawl. The first URL is used as the root.
+     * @returns A promise that resolves when crawling is complete.
+     * @throws {Error} If the URL list is empty.
+     */
+    async crawling(list) {
+        const root = list[0];
+        if (!root) {
+            throw new Error('URL is empty');
+        }
+        return new Promise((resolve, reject) => {
+            this.#crawler.on('error', (error) => {
+                crawlerLog('On error: %O', error);
+                void this.#archive.addError(error);
+                void this.emit('error', error);
+            });
+            this.#crawler.on('page', async ({ result }) => {
+                // const pageId =
+                await this.#archive.setPage(result).catch((error) => reject(error));
+                // await this.#crawler.screenshot(pageId, Archive.joinPath(this.#archive.tmpDir, 'screenshots'));
+            });
+            this.#crawler.on('externalPage', ({ result }) => {
+                this.#archive.setExternalPage(result).catch((error) => reject(error));
+            });
+            this.#crawler.on('skip', ({ url, reason, isExternal }) => {
+                this.#archive
+                    .setSkippedPage(url, reason, isExternal)
+                    .catch((error) => reject(error));
+            });
+            this.#crawler.on('response', ({ resource }) => {
+                this.#archive.setResources(resource).catch((error) => reject(error));
+            });
+            this.#crawler.on('responseReferrers', (resource) => {
+                this.#archive.setResourcesReferrers(resource).catch((error) => reject(error));
+            });
+            this.#crawler.on('crawlEnd', () => {
+                resolve();
+            });
+            if (this.#fromList) {
+                this.#crawler.startMultiple(list);
+            }
+            else {
+                this.#crawler.start(root);
+            }
+        });
+    }
+    /**
+     * Kill any zombie Chromium processes that were not properly cleaned up.
+     *
+     * Retrieves the list of undead process IDs from the crawler and sends
+     * a SIGTERM signal to each one. Chromium is intentionally sent SIGTERM
+     * (not SIGKILL) to avoid leaving zombie processes.
+     */
+    garbageCollect() {
+        const pidList = this.getUndeadPid();
+        log('Undead PIDs: %O', pidList);
+        for (const pid of pidList) {
+            try {
+                log('Garbage collect: kill PID:%d', pid);
+                // Chromium becomes a zombie process if SIGKILL signal.
+                process.kill(pid);
+            }
+            catch (error) {
+                log('Garbage collect: Failed killing PID:%d %O', pid, error);
+            }
+        }
+    }
+    /**
+     * Retrieve the list of process IDs for Chromium instances that are
+     * still running after crawling has ended.
+     * @returns An array of process IDs that should be terminated.
+     */
+    getUndeadPid() {
+        return this.#crawler.getUndeadPid();
+    }
+    /**
+     * Write the archive to its configured file path.
+     *
+     * Emits `writeFileStart` before writing and `writeFileEnd` after
+     * the write completes successfully.
+     */
+    async write() {
+        void this.emit('writeFileStart', { filePath: this.#archive.filePath });
+        await this.#archive.write();
+        void this.emit('writeFileEnd', { filePath: this.#archive.filePath });
+    }
+    /**
+     * Create a new Qzilla instance and start crawling the given URLs.
+     *
+     * This is the primary factory method for starting a fresh crawl. It:
+     * 1. Parses and sorts the input URLs
+     * 2. Creates an archive file
+     * 3. Saves the crawl configuration
+     * 4. Runs the optional initialized callback
+     * 5. Executes the crawl
+     * 6. Sorts the archived URLs in natural order
+     * @param url - One or more URL strings to crawl.
+     * @param options - Optional configuration overrides for the crawl session.
+     * @param initializedCallback - Optional callback invoked after initialization but before crawling starts.
+     * @returns A promise that resolves to the Qzilla instance after crawling completes.
+     * @throws {Error} If the URL list is empty or contains no valid URLs.
+     */
+    static async crawling(url, options, initializedCallback) {
+        const list = sortUrl(url, options);
+        const urlParsed = list[0];
+        if (!urlParsed) {
+            throw new Error('URL is empty');
+        }
+        const fileName = `${urlParsed.hostname}-${Archive.timestamp()}`;
+        const cwd = options?.cwd ?? process.cwd();
+        const filePath = Archive.joinPath(cwd, `${fileName}.${Archive.FILE_EXTENSION}`);
+        const disableQueries = options?.disableQueries || false;
+        const archive = await Archive.create({ filePath, cwd, disableQueries });
+        await archive.setConfig({
+            version: pkg.version,
+            name: fileName,
+            baseUrl: urlParsed.withoutHash,
+            recursive: options?.recursive ?? true,
+            fetchExternal: options?.fetchExternal ?? true,
+            image: options?.image ?? true,
+            interval: options?.interval || 0,
+            parallels: options?.parallels || 0,
+            scope: options?.scope ?? [],
+            // @ts-ignore TODO: Fix CLI arguments
+            excludes: optMultiParam(options?.exclude),
+            // @ts-ignore TODO: Fix CLI arguments
+            excludeKeywords: optMultiParam(options?.excludeKeyword),
+            excludeUrls: [
+                ...DEFAULT_EXCLUDED_EXTERNAL_URLS,
+                // @ts-ignore TODO: Fix CLI arguments
+                ...optMultiParam(options?.excludeUrl),
+            ],
+            depthOnAvoid: options?.depthOnAvoid || 10,
+            fromList: !!options?.list,
+            disableQueries,
+        });
+        const qzilla = new Qzilla(archive, options);
+        const config = await archive.getConfig();
+        if (initializedCallback) {
+            await initializedCallback(qzilla, config);
+        }
+        log('Start crawling');
+        log('URL %O', list.map((url) => url.href));
+        log('Config %O', config);
+        await qzilla.crawling(list);
+        log('Crawling complated');
+        log('Set order natural URL sort');
+        await archive.setUrlOrder();
+        log('Sorting done');
+        return qzilla;
+    }
+    /**
+     * Resume a previously interrupted crawl from an existing archive file.
+     *
+     * Restores the crawl state (pending URLs, scraped URLs, and resources)
+     * from the archive, merges any option overrides, and continues crawling
+     * from where it left off.
+     * @param stubPath - Path to the existing archive file to resume from.
+     * @param options - Optional configuration overrides to apply on top of the archived config.
+     * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @returns A promise that resolves to the Qzilla instance after crawling completes.
+     * @throws {Error} If the archived URL is invalid.
+     */
+    static async resume(stubPath, options, initializedCallback) {
+        const archive = await Archive.resume(stubPath);
+        const archivedConfig = await archive.getConfig();
+        const config = {
+            ...archivedConfig,
+            ...cleanObject(options),
+        };
+        const qzilla = new Qzilla(archive, config);
+        const _url = await archive.getUrl();
+        const url = parseUrl(_url, config);
+        if (!url) {
+            throw new Error(`URL (${_url}) is invalid`);
+        }
+        const { scraped, pending } = await archive.getCrawlingState();
+        const resources = await archive.getResourceUrlList();
+        qzilla.#crawler.resume(pending, scraped, resources);
+        if (initializedCallback) {
+            await initializedCallback(qzilla, config);
+        }
+        log('Start resuming');
+        log('Data %s', stubPath);
+        log('URL %s', url.href);
+        log('Config %O', config);
+        await qzilla.crawling([url]);
+        return qzilla;
+    }
+}
+/**
+ * Normalize an optional parameter that may be a single value, an array,
+ * null, or undefined into a guaranteed array.
+ * @param param - The parameter to normalize.
+ * @returns An array containing the parameter value(s), or an empty array if absent.
+ */
+function optMultiParam(param) {
+    return Array.isArray(param) ? param : param ? [param] : [];
+}

package/lib/types.d.ts

@@ -0,0 +1,27 @@
+import type { CrawlerError } from './utils/index.js';
+/**
+ * Event map for the `CrawlerOrchestrator` class.
+ *
+ * Each key represents an event name and its value is the payload type
+ * passed to listeners subscribed via `on()` or `once()`.
+ */
+export interface CrawlEvent {
+    /**
+     * Emitted when the archive file write operation begins.
+     */
+    writeFileStart: {
+        /** Absolute path of the archive file being written. */
+        filePath: string;
+    };
+    /**
+     * Emitted when the archive file write operation completes.
+     */
+    writeFileEnd: {
+        /** Absolute path of the archive file that was written. */
+        filePath: string;
+    };
+    /**
+     * Emitted when an error occurs during crawling or archiving.
+     */
+    error: CrawlerError;
+}

package/lib/types.js

@@ -0,0 +1 @@
+export {};

package/lib/utils/array/each-splitted.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Splits an array into chunks of the specified size and executes a callback
+ * on each chunk in parallel using `Promise.all`.
+ * @template T - The element type of the array.
+ * @param a - The array to split into chunks.
+ * @param count - The maximum number of elements per chunk.
+ * @param callback - A function to invoke on each chunk. May be synchronous or asynchronous.
+ * @returns A promise that resolves when all chunk callbacks have completed.
+ */
+export declare function eachSplitted<T>(a: T[], count: number, callback: (items: T[]) => void | Promise<void>): Promise<void>;

package/lib/utils/array/each-splitted.js

@@ -0,0 +1,14 @@
+import { splitArray } from '@d-zero/shared/split-array';
+/**
+ * Splits an array into chunks of the specified size and executes a callback
+ * on each chunk in parallel using `Promise.all`.
+ * @template T - The element type of the array.
+ * @param a - The array to split into chunks.
+ * @param count - The maximum number of elements per chunk.
+ * @param callback - A function to invoke on each chunk. May be synchronous or asynchronous.
+ * @returns A promise that resolves when all chunk callbacks have completed.
+ */
+export async function eachSplitted(a, count, callback) {
+    const splitted = splitArray(a, count);
+    await Promise.all(splitted.map((items) => callback(items)));
+}

package/lib/utils/array/index.d.ts

@@ -0,0 +1 @@
+export { eachSplitted } from './each-splitted.js';

package/lib/utils/array/index.js

@@ -0,0 +1 @@
+export { eachSplitted } from './each-splitted.js';

package/lib/utils/async/index.d.ts

@@ -0,0 +1 @@
+export { delay } from '@d-zero/shared/delay';

package/lib/utils/async/index.js

@@ -0,0 +1 @@
+export { delay } from '@d-zero/shared/delay';

package/lib/utils/debug.d.ts

@@ -0,0 +1,5 @@
+import debug from 'debug';
+/** Root debug logger for the Nitpicker application. Namespace: `Nitpicker`. */
+export declare const globalLog: debug.Debugger;
+/** Debug logger for the utils package. Namespace: `Nitpicker:Utils`. */
+export declare const log: debug.Debugger;

package/lib/utils/debug.js

@@ -0,0 +1,5 @@
+import debug from 'debug';
+/** Root debug logger for the Nitpicker application. Namespace: `Nitpicker`. */
+export const globalLog = debug('Nitpicker');
+/** Debug logger for the utils package. Namespace: `Nitpicker:Utils`. */
+export const log = globalLog.extend('Utils');

package/lib/utils/error/dom-evaluation-error.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Error thrown when DOM evaluation (e.g., running scripts within a browser page context)
+ * fails. This typically occurs during page scraping when JavaScript execution
+ * in the browser context encounters an error.
+ */
+export declare class DOMEvaluationError extends Error {
+}

package/lib/utils/error/dom-evaluation-error.js

@@ -0,0 +1,7 @@
+/**
+ * Error thrown when DOM evaluation (e.g., running scripts within a browser page context)
+ * fails. This typically occurs during page scraping when JavaScript execution
+ * in the browser context encounters an error.
+ */
+export class DOMEvaluationError extends Error {
+}

package/lib/utils/error/error-emitter.d.ts

@@ -0,0 +1,18 @@
+import type { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
+/**
+ * Event payload type for error events emitted by classes using the {@link ErrorEmitter} decorator.
+ * @template E - The specific error type, defaults to `Error`.
+ */
+export type ErrorEvent<E extends Error = Error> = {
+    /** The error instance that was caught. */
+    error: E;
+};
+/**
+ * A class method decorator factory that wraps the decorated method with error handling.
+ * When the method throws an `Error`, it emits an `'error'` event on the class instance
+ * (which must extend {@link EventEmitter}) with the caught error, then re-throws the error.
+ * @template C - The class type, which must be an EventEmitter capable of emitting error events.
+ * @template E - The error event type, defaults to {@link ErrorEvent}.
+ * @returns A decorator function that wraps the target method with error-emitting behavior.
+ */
+export declare function ErrorEmitter<C extends EventEmitter<E>, E extends ErrorEvent = ErrorEvent>(): (method: Function, context: ClassMethodDecoratorContext) => (this: C, ...args: unknown[]) => Promise<any>;

package/lib/utils/error/error-emitter.js

@@ -0,0 +1,29 @@
+import { log } from '../debug.js';
+const errorLog = log.extend('ErrorEmitter');
+/**
+ * A class method decorator factory that wraps the decorated method with error handling.
+ * When the method throws an `Error`, it emits an `'error'` event on the class instance
+ * (which must extend {@link EventEmitter}) with the caught error, then re-throws the error.
+ * @template C - The class type, which must be an EventEmitter capable of emitting error events.
+ * @template E - The error event type, defaults to {@link ErrorEvent}.
+ * @returns A decorator function that wraps the target method with error-emitting behavior.
+ */
+export function ErrorEmitter() {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
+    return (method, context) => {
+        return async function (...args) {
+            const constructorName = String(this.constructor?.name || this.constructor || this);
+            const methodName = `${constructorName}.${String(context.name)}`;
+            try {
+                return await method.apply(this, args);
+            }
+            catch (error) {
+                if (error instanceof Error) {
+                    errorLog('%s: %O', methodName, error);
+                    void this.emit('error', error);
+                }
+                throw error;
+            }
+        };
+    };
+}

package/lib/utils/error/index.d.ts

@@ -0,0 +1,3 @@
+export { DOMEvaluationError } from './dom-evaluation-error.js';
+export { ErrorEmitter } from './error-emitter.js';
+export type { ErrorEvent } from './error-emitter.js';

package/lib/utils/error/index.js

@@ -0,0 +1,2 @@
+export { DOMEvaluationError } from './dom-evaluation-error.js';
+export { ErrorEmitter } from './error-emitter.js';

package/lib/utils/event-emitter/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * A typed, async-capable event emitter re-exported from `@d-zero/shared`.
+ * Provides type-safe `emit`, `on`, and `off` methods where event names
+ * and their payload types are enforced at compile time.
+ */
+export { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';

package/lib/utils/event-emitter/index.js

@@ -0,0 +1,6 @@
+/**
+ * A typed, async-capable event emitter re-exported from `@d-zero/shared`.
+ * Provides type-safe `emit`, `on`, and `off` methods where event names
+ * and their payload types are enforced at compile time.
+ */
+export { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';

package/lib/utils/index.d.ts

@@ -0,0 +1,5 @@
+export * from './types/index.js';
+export * from './array/index.js';
+export * from './error/index.js';
+export * from './object/index.js';
+export { globalLog as log } from './debug.js';

package/lib/utils/index.js

@@ -0,0 +1,5 @@
+export * from './types/index.js';
+export * from './array/index.js';
+export * from './error/index.js';
+export * from './object/index.js';
+export { globalLog as log } from './debug.js';

package/lib/utils/network/index.d.ts

@@ -0,0 +1 @@
+export { isError } from '@nitpicker/beholder';

package/lib/utils/network/index.js

@@ -0,0 +1 @@
+export { isError } from '@nitpicker/beholder';

package/lib/utils/object/clean-object.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Creates a shallow copy of an object with all `undefined`-valued properties removed.
+ * If the input is falsy (e.g., `undefined` or `null`), returns an empty object.
+ * @template T - The type of the input object.
+ * @param obj - The object to clean. If falsy, an empty `Partial<T>` is returned.
+ * @returns A new object containing only the properties whose values are not `undefined`.
+ */
+export declare function cleanObject<T extends Record<string, unknown>>(obj?: T): Partial<T>;

package/lib/utils/object/clean-object.js

@@ -0,0 +1,13 @@
+/**
+ * Creates a shallow copy of an object with all `undefined`-valued properties removed.
+ * If the input is falsy (e.g., `undefined` or `null`), returns an empty object.
+ * @template T - The type of the input object.
+ * @param obj - The object to clean. If falsy, an empty `Partial<T>` is returned.
+ * @returns A new object containing only the properties whose values are not `undefined`.
+ */
+export function cleanObject(obj) {
+    if (!obj) {
+        return {};
+    }
+    return Object.fromEntries(Object.entries(obj).filter(([, v]) => v !== undefined));
+}

package/lib/utils/object/index.d.ts

@@ -0,0 +1 @@
+export * from './clean-object.js';

package/lib/utils/object/index.js

@@ -0,0 +1 @@
+export * from './clean-object.js';

package/lib/utils/path/index.d.ts

@@ -0,0 +1 @@
+export { safeFilePath } from '@d-zero/shared/safe-filepath';

package/lib/utils/path/index.js

@@ -0,0 +1 @@
+export { safeFilePath } from '@d-zero/shared/safe-filepath';

package/lib/utils/path/safe-filepath.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Sanitizes a file path by decoding URI-encoded characters and replacing
+ * any characters that are unsafe for use in filenames with underscores.
+ * @param filePath - The raw file path string (possibly URI-encoded) to sanitize.
+ * @returns A sanitized file path string that is safe for use as a filename.
+ */
+export declare function safeFilePath(filePath: string): string;

package/lib/utils/path/safe-filepath.js

@@ -0,0 +1,12 @@
+import sanitize from 'sanitize-filename';
+/**
+ * Sanitizes a file path by decoding URI-encoded characters and replacing
+ * any characters that are unsafe for use in filenames with underscores.
+ * @param filePath - The raw file path string (possibly URI-encoded) to sanitize.
+ * @returns A sanitized file path string that is safe for use as a filename.
+ */
+export function safeFilePath(filePath) {
+    return sanitize(decodeURI(filePath), {
+        replacement: '_',
+    });
+}

package/lib/utils/regexp/index.d.ts

@@ -0,0 +1 @@
+export { strToRegex } from '@d-zero/shared/str-to-regex';

package/lib/utils/regexp/index.js

@@ -0,0 +1 @@
+export { strToRegex } from '@d-zero/shared/str-to-regex';

package/lib/utils/retryable/index.d.ts

@@ -0,0 +1,2 @@
+export { retry as retryable } from '@d-zero/shared/retry';
+export type { RetryDecoratorOptions as RetryableDecoratorOptions } from '@d-zero/shared/retry';

package/lib/utils/retryable/index.js

@@ -0,0 +1 @@
+export { retry as retryable } from '@d-zero/shared/retry';

package/lib/utils/sort/index.d.ts

@@ -0,0 +1,14 @@
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
+/**
+ * Compares two URLs using a natural sorting algorithm. The comparison order is:
+ * hostname, path directories, basename (with index files prioritized), file extension,
+ * query string, hash, protocol, and finally the original URL string as a tiebreaker.
+ * Numeric segments within path components are compared numerically rather than
+ * lexicographically.
+ * @param url1 - The first URL string or ExURL to compare.
+ * @param url2 - The second URL string or ExURL to compare.
+ * @param options - Optional URL parsing options.
+ * @returns `0` if the URLs are equal, `-1` if url1 should come before url2,
+ *          or `1` if url1 should come after url2.
+ */
+export declare function naturalURLSort(url1: string | ExURL, url2: string | ExURL, options?: ParseURLOptions): 0 | -1 | 1;

package/lib/utils/sort/index.js

@@ -0,0 +1,61 @@
+import { alphabeticalComparator } from '@d-zero/shared/sort/alphabetical';
+import { dirComparator } from '@d-zero/shared/sort/dir';
+import { numericalComparator } from '@d-zero/shared/sort/numerical';
+import { parseUrl } from '../url/index.js';
+/**
+ * Compares two URLs using a natural sorting algorithm. The comparison order is:
+ * hostname, path directories, basename (with index files prioritized), file extension,
+ * query string, hash, protocol, and finally the original URL string as a tiebreaker.
+ * Numeric segments within path components are compared numerically rather than
+ * lexicographically.
+ * @param url1 - The first URL string or ExURL to compare.
+ * @param url2 - The second URL string or ExURL to compare.
+ * @param options - Optional URL parsing options.
+ * @returns `0` if the URLs are equal, `-1` if url1 should come before url2,
+ *          or `1` if url1 should come after url2.
+ */
+export function naturalURLSort(url1, url2, options) {
+    const u1 = typeof url1 === 'string' ? parseUrl(url1, options) : url1;
+    const u2 = typeof url2 === 'string' ? parseUrl(url2, options) : url2;
+    if (!u1 || !u2) {
+        return 0;
+    }
+    if (u1.href === u2.href) {
+        return alphabeticalComparator(u1._originUrlString, u2._originUrlString);
+    }
+    const rHost = alphabeticalComparator(u1.hostname, u2.hostname);
+    if (rHost) {
+        return rHost;
+    }
+    const rPaths = dirComparator(u1.paths, u2.paths);
+    if (rPaths) {
+        return rPaths;
+    }
+    if (u1.basename !== u2.basename) {
+        if (u1.isIndex)
+            return -1;
+        if (u2.isIndex)
+            return 1;
+        const rBasename = numericalComparator(u1.basename, u2.basename);
+        if (rBasename) {
+            return rBasename;
+        }
+    }
+    const rExtname = numericalComparator(u1.extname, u2.extname);
+    if (rExtname) {
+        return rExtname;
+    }
+    const rSearch = numericalComparator(u1.query, u2.query);
+    if (rSearch) {
+        return rSearch;
+    }
+    const rHash = numericalComparator(u1.hash, u2.hash);
+    if (rHash) {
+        return rHash;
+    }
+    const rProtocol = alphabeticalComparator(u1.protocol, u2.protocol);
+    if (rProtocol) {
+        return rProtocol;
+    }
+    return numericalComparator(u1._originUrlString, u2._originUrlString);
+}

package/lib/utils/sort/remove-matches.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Strips the common leading prefix from two strings (case-insensitive comparison).
+ * Returns a tuple of the remaining suffixes after the shared prefix is removed.
+ * If the strings are identical (ignoring case), returns `['', '']`.
+ * @param t1 - The first string.
+ * @param t2 - The second string.
+ * @returns A tuple of the two strings with their common leading characters removed.
+ */
+export declare function removeMatches(t1: string, t2: string): [string, string];