@nitpicker/crawler 0.4.1

package/lib/utils/sort/remove-matches.js

@@ -0,0 +1,23 @@
+/**
+ * Strips the common leading prefix from two strings (case-insensitive comparison).
+ * Returns a tuple of the remaining suffixes after the shared prefix is removed.
+ * If the strings are identical (ignoring case), returns `['', '']`.
+ * @param t1 - The first string.
+ * @param t2 - The second string.
+ * @returns A tuple of the two strings with their common leading characters removed.
+ */
+export function removeMatches(t1, t2) {
+    let loopCount = Math.max(t1.length, t2.length);
+    t1 = t1.toLowerCase();
+    t2 = t2.toLowerCase();
+    const a1 = [...t1];
+    const a2 = [...t2];
+    while (loopCount--) {
+        if (a1[0] !== a2[0]) {
+            return [a1.join(''), a2.join('')];
+        }
+        a1.shift();
+        a2.shift();
+    }
+    return ['', ''];
+}

package/lib/utils/types/index.d.ts

@@ -0,0 +1 @@
+export * from './types.js';

package/lib/utils/types/index.js

@@ -0,0 +1 @@
+export * from './types.js';

package/lib/utils/types/types.d.ts

@@ -0,0 +1,46 @@
+import type { ExURL } from '@d-zero/shared/parse-url';
+export type { PageData, ImageElement, SkippedPageData, Resource, AnchorData, Meta, NetworkLog, } from '@d-zero/beholder';
+export type { ExURL } from '@d-zero/shared/parse-url';
+export type { CompressType } from '@d-zero/shared/detect-compress';
+export type { CDNType } from '@d-zero/shared/detect-cdn';
+/**
+ * Represents a discovered link during crawling, with its metadata from the HEAD request.
+ */
+export interface Link {
+    /** The parsed URL of the link. */
+    url: ExURL;
+    /** Whether this link points to an external domain. */
+    isExternal: boolean;
+    /** Whether this link is in a lower layer (subdirectory) of a scope URL. */
+    isLowerLayer: boolean;
+    /** Destination data from the HEAD request, present only if the link was fetched. */
+    dest?: {
+        /** Chain of redirect URLs traversed. */
+        redirectPaths: string[];
+        /** HTTP status code of the final response. */
+        status: number;
+        /** HTTP status text of the final response. */
+        statusText: string;
+        /** The Content-Type header value, or `null` if unavailable. */
+        contentType: string | null;
+        /** The Content-Length header value in bytes, or `null` if unavailable. */
+        contentLength: number | null;
+        /** Raw HTTP response headers, or `null` if unavailable. */
+        responseHeaders: Record<string, string | string[] | undefined> | null;
+        /** The page title, if available from a title-only scrape. */
+        title?: string;
+    };
+}
+/**
+ * An error event emitted during crawling or scraping.
+ */
+export interface CrawlerError {
+    /** The process ID where the error occurred. */
+    pid: number;
+    /** Whether the error occurred in the main process (as opposed to a sub-process). */
+    isMainProcess: boolean;
+    /** The URL being processed when the error occurred, or `null` if not applicable. */
+    url: string | null;
+    /** The error object. */
+    error: Error;
+}

package/lib/utils/types/types.js

@@ -0,0 +1 @@
+export {};

package/lib/utils/url/index.d.ts

@@ -0,0 +1,5 @@
+export { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+export { isLowerLayer } from '@d-zero/shared/is-lower-layer';
+export { pathMatch } from '@d-zero/shared/path-match';
+export { urlPartialMatch } from '@d-zero/shared/url-partial-match';
+export { sortUrl } from '@d-zero/shared/sort-url';

package/lib/utils/url/index.js

@@ -0,0 +1,5 @@
+export { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+export { isLowerLayer } from '@d-zero/shared/is-lower-layer';
+export { pathMatch } from '@d-zero/shared/path-match';
+export { urlPartialMatch } from '@d-zero/shared/url-partial-match';
+export { sortUrl } from '@d-zero/shared/sort-url';

package/lib/utils/url/is-lower-layer.d.ts

@@ -0,0 +1,15 @@
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
+/**
+ * Determines whether the target URL is at the same level or a deeper (lower) layer
+ * in the path hierarchy relative to the base URL. Both URLs must share the same hostname.
+ *
+ * For example, if the base is `https://example.com/docs/`, then
+ * `https://example.com/docs/getting-started` is considered a lower layer,
+ * while `https://example.com/about` is not.
+ * @param target - The target URL string or ExURL to check.
+ * @param base - The base URL string or ExURL to compare against.
+ * @param options - Optional URL parsing options.
+ * @returns `true` if the target URL is at the same level or deeper than the base URL
+ *          within the same hostname; `false` otherwise.
+ */
+export declare function isLowerLayer(target: string | ExURL, base: string | ExURL, options?: ParseURLOptions): boolean;

package/lib/utils/url/is-lower-layer.js

@@ -0,0 +1,55 @@
+import { parseUrl } from './parse-url.js';
+/**
+ * Determines whether the target URL is at the same level or a deeper (lower) layer
+ * in the path hierarchy relative to the base URL. Both URLs must share the same hostname.
+ *
+ * For example, if the base is `https://example.com/docs/`, then
+ * `https://example.com/docs/getting-started` is considered a lower layer,
+ * while `https://example.com/about` is not.
+ * @param target - The target URL string or ExURL to check.
+ * @param base - The base URL string or ExURL to compare against.
+ * @param options - Optional URL parsing options.
+ * @returns `true` if the target URL is at the same level or deeper than the base URL
+ *          within the same hostname; `false` otherwise.
+ */
+export function isLowerLayer(target, base, options) {
+    const a = typeof target === 'string' ? parseUrl(target, options) : target;
+    const b = typeof base === 'string' ? parseUrl(base, options) : base;
+    if (!a || !b) {
+        return false;
+    }
+    if (a.href === b.href) {
+        return true;
+    }
+    const aPathIsEmpty = a.paths.length === 1 && a.paths[0] === '';
+    const bPathIsEmpty = b.paths.length === 1 && b.paths[0] === '';
+    if (a.hostname !== b.hostname) {
+        return false;
+    }
+    if (aPathIsEmpty && bPathIsEmpty) {
+        return true;
+    }
+    if (a.paths == null && b.paths == null) {
+        return true;
+    }
+    if (a.paths && b.paths == null) {
+        return true;
+    }
+    if (!a.paths || !b.paths) {
+        return false;
+    }
+    if (a.paths.length < b.paths.length) {
+        return false;
+    }
+    while (Math.max(a.paths.length, b.paths.length)) {
+        const i1 = a.paths.shift();
+        const i2 = b.paths.shift();
+        if (i1 && !i2) {
+            return true;
+        }
+        if (i1 !== i2) {
+            return false;
+        }
+    }
+    return false;
+}

package/lib/utils/url/parse-url.d.ts

@@ -0,0 +1,11 @@
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
+/**
+ * Parses a URL string into an ExURL object, filtering out non-HTTP URLs
+ * that lack a hostname and protocol. If the input is already an ExURL object,
+ * it is returned as-is without re-parsing.
+ * @param url - The URL string to parse, or an already-parsed ExURL object.
+ * @param options - Optional parsing options forwarded to the underlying parser.
+ * @returns The parsed ExURL object, or `null` if the URL is not a valid HTTP URL
+ *          and has no hostname or protocol.
+ */
+export declare function parseUrl(url: string | ExURL, options?: ParseURLOptions): ExURL | null;

package/lib/utils/url/parse-url.js

@@ -0,0 +1,20 @@
+import { parseUrl as sharedParseUrl } from '@d-zero/shared/parse-url';
+/**
+ * Parses a URL string into an ExURL object, filtering out non-HTTP URLs
+ * that lack a hostname and protocol. If the input is already an ExURL object,
+ * it is returned as-is without re-parsing.
+ * @param url - The URL string to parse, or an already-parsed ExURL object.
+ * @param options - Optional parsing options forwarded to the underlying parser.
+ * @returns The parsed ExURL object, or `null` if the URL is not a valid HTTP URL
+ *          and has no hostname or protocol.
+ */
+export function parseUrl(url, options) {
+    if (typeof url !== 'string') {
+        return url;
+    }
+    const result = sharedParseUrl(url, options);
+    if (!result.isHTTP && !result.hostname && !result.protocol) {
+        return null;
+    }
+    return result;
+}

package/lib/utils/url/path-match.d.ts

@@ -0,0 +1,11 @@
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
+/**
+ * Tests whether the pathname portion of a URL matches a given glob pattern.
+ * Uses micromatch for glob matching.
+ * @param targetPath - The URL string or ExURL whose pathname will be tested.
+ * @param pattern - The glob pattern to match against the pathname (e.g., `"/docs/**"`).
+ * @param options - Optional URL parsing options.
+ * @returns `true` if the URL's pathname matches the pattern; `false` otherwise
+ *          (also returns `false` if the URL cannot be parsed).
+ */
+export declare function pathMatch(targetPath: string | ExURL, pattern: string, options?: ParseURLOptions): boolean;

package/lib/utils/url/path-match.js

@@ -0,0 +1,18 @@
+import micromatch from 'micromatch';
+import { parseUrl } from './parse-url.js';
+/**
+ * Tests whether the pathname portion of a URL matches a given glob pattern.
+ * Uses micromatch for glob matching.
+ * @param targetPath - The URL string or ExURL whose pathname will be tested.
+ * @param pattern - The glob pattern to match against the pathname (e.g., `"/docs/**"`).
+ * @param options - Optional URL parsing options.
+ * @returns `true` if the URL's pathname matches the pattern; `false` otherwise
+ *          (also returns `false` if the URL cannot be parsed).
+ */
+export function pathMatch(targetPath, pattern, options) {
+    const url = typeof targetPath === 'string' ? parseUrl(targetPath, options) : targetPath;
+    if (!url) {
+        return false;
+    }
+    return micromatch.isMatch(url.pathname || '', pattern || '/');
+}

package/lib/utils/url/sort-url.d.ts

@@ -0,0 +1,10 @@
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
+/**
+ * Parses, deduplicates, and sorts a list of URL strings using natural URL sorting.
+ * Duplicate URLs (by normalized href) are removed before sorting.
+ * @param list - An array of URL strings to sort.
+ * @param options - Optional URL parsing options.
+ * @returns A sorted array of ExURL objects with duplicates removed, ordered by
+ *          natural URL sort (hostname, path hierarchy, basename, extension, query, hash).
+ */
+export declare function sortUrl(list: string[], options?: ParseURLOptions): ExURL[];

package/lib/utils/url/sort-url.js

@@ -0,0 +1,24 @@
+import { pathComparator } from '@d-zero/shared/sort/path';
+import { parseUrl } from './parse-url.js';
+/**
+ * Parses, deduplicates, and sorts a list of URL strings using natural URL sorting.
+ * Duplicate URLs (by normalized href) are removed before sorting.
+ * @param list - An array of URL strings to sort.
+ * @param options - Optional URL parsing options.
+ * @returns A sorted array of ExURL objects with duplicates removed, ordered by
+ *          natural URL sort (hostname, path hierarchy, basename, extension, query, hash).
+ */
+export function sortUrl(list, options) {
+    const map = new Map();
+    for (const url of list) {
+        if (map.has(url)) {
+            continue;
+        }
+        const parsedUrl = parseUrl(url, options);
+        if (!parsedUrl) {
+            continue;
+        }
+        map.set(parsedUrl.href, parsedUrl);
+    }
+    return [...map.values()].toSorted((a, b) => pathComparator(a.href, b.href));
+}

package/lib/utils/url/url-partial-match.d.ts

@@ -0,0 +1,11 @@
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Checks whether a URL partially matches a pattern URL. The match succeeds if both the
+ * hostname (case-insensitive) and the pathname match. The pathname matches if it is exactly
+ * equal to the pattern's pathname or if it falls under the pattern's pathname as a sub-path.
+ * @param url - The URL string or ExURL to test.
+ * @param pattern - The pattern URL string to match against (must include hostname and path).
+ * @returns `true` if the URL's hostname matches the pattern's hostname and the URL's pathname
+ *          is equal to or nested under the pattern's pathname; `false` otherwise.
+ */
+export declare function urlPartialMatch(url: string | ExURL, pattern: string): boolean;

package/lib/utils/url/url-partial-match.js

@@ -0,0 +1,32 @@
+import { parseUrl } from './parse-url.js';
+import { pathMatch } from './path-match.js';
+/**
+ * Checks whether a URL partially matches a pattern URL. The match succeeds if both the
+ * hostname (case-insensitive) and the pathname match. The pathname matches if it is exactly
+ * equal to the pattern's pathname or if it falls under the pattern's pathname as a sub-path.
+ * @param url - The URL string or ExURL to test.
+ * @param pattern - The pattern URL string to match against (must include hostname and path).
+ * @returns `true` if the URL's hostname matches the pattern's hostname and the URL's pathname
+ *          is equal to or nested under the pattern's pathname; `false` otherwise.
+ */
+export function urlPartialMatch(url, pattern) {
+    const target = parseUrl(url);
+    if (!target) {
+        return false;
+    }
+    const patternUrl = parseUrl(pattern);
+    if (!patternUrl) {
+        return false;
+    }
+    const { hostname, pathname } = patternUrl;
+    if (hostname.toLowerCase() !== target.hostname.toLowerCase()) {
+        return false;
+    }
+    if (pathname === target.pathname) {
+        return true;
+    }
+    if (pathMatch(url, `${pathname}/**/*`)) {
+        return true;
+    }
+    return false;
+}

package/package.json

@@ -0,0 +1,49 @@
+{
+	"name": "@nitpicker/crawler",
+	"version": "0.4.1",
+	"description": "Web crawler engine with headless browser rendering and archive storage",
+	"author": "D-ZERO",
+	"license": "Apache-2.0",
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/d-zero-dev/nitpicker.git",
+		"directory": "packages/@nitpicker/crawler"
+	},
+	"publishConfig": {
+		"access": "public"
+	},
+	"type": "module",
+	"exports": {
+		".": {
+			"import": "./lib/index.js",
+			"types": "./lib/index.d.ts"
+		}
+	},
+	"scripts": {
+		"build": "tsc",
+		"clean": "tsc --build --clean"
+	},
+	"dependencies": {
+		"@d-zero/beholder": "2.0.0",
+		"@d-zero/dealer": "1.6.3",
+		"@d-zero/fs": "0.2.2",
+		"@d-zero/shared": "0.20.0",
+		"ansi-colors": "4.1.3",
+		"debug": "4.4.3",
+		"follow-redirects": "1.15.11",
+		"fs-extra": "11.3.3",
+		"knex": "3.1.0",
+		"puppeteer": "24.37.5",
+		"robots-parser": "3.0.1",
+		"sqlite3": "5.1.7",
+		"tar": "7.5.9"
+	},
+	"devDependencies": {
+		"@types/debug": "4.1.12",
+		"@types/follow-redirects": "1.14.4",
+		"@types/fs-extra": "11.0.4",
+		"@types/tar": "7.0.87",
+		"@types/unzipper": "0.10.11"
+	},
+	"gitHead": "32b83ee38eba7dfd237adb1b41f69e049e8d4ceb"
+}

package/src/archive/__mock__/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!mock.sqlite