@nitpicker/crawler 0.6.3 → 0.7.0

package/lib/crawler/fetch-destination.js

@@ -28,13 +28,19 @@ export async function fetchDestination(params) {
         return cache;
     }
     const effectiveMethod = titleBytesLimit == null ? method : 'GET';
+    // Race the fetch against a 10-second timeout. The losing timer is cleared
+    // explicitly so it never keeps the event loop alive after the race settles
+    // (a plain `delay()` in `Promise.race` would leak the timer until it fires).
+    let timeoutHandle;
     const result = await Promise.race([
         _fetchHead(url, isExternal, effectiveMethod, titleBytesLimit, userAgent).catch((error) => (error instanceof Error ? error : new Error(String(error)))),
-        (async () => {
-            await delay(10 * 1000);
-            return new NetTimeoutError(url.href);
-        })(),
-    ]);
+        new Promise((resolve) => {
+            timeoutHandle = setTimeout(() => resolve(new NetTimeoutError(url.href)), 10 * 1000);
+        }),
+    ]).finally(() => {
+        if (timeoutHandle)
+            clearTimeout(timeoutHandle);
+    });
     destinationCache.set(cacheKey, result);
     if (result instanceof Error) {
         throw result;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.6.3",
+	"version": "0.7.0",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -48,5 +48,5 @@
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "4b58ca83e5f48833b22889ed46d96984938fa1c9"
+	"gitHead": "3b4c33adf8af69a46a85faf8c04eddb5ab563972"
 }

package/lib/archive/common-queries.d.ts

@@ -1,14 +0,0 @@
-import type { Knex } from 'knex';
-/**
- * Returns a Knex subquery builder that selects page IDs with pagination,
- * ordered by the `order` column (nulls last), excluding redirected pages.
- * @param limit - The maximum number of page IDs to return.
- * @param offset - The number of page IDs to skip before returning results.
- */
-export declare function limitedPageIds(limit: number, offset: number): (qb: Knex.QueryBuilder<Record<string, unknown>, unknown>) => Promise<void>;
-/**
- * Returns a Knex subquery builder that joins pages with their redirect destinations.
- * When `includeNull` is true, also includes pages without redirects (self-referencing).
- * @param includeNull - Whether to include non-redirected pages in the result. Defaults to `true`.
- */
-export declare function redirectTable(includeNull?: boolean): (qb: Knex.QueryBuilder<Record<string, unknown>, unknown>) => Promise<void>;

package/lib/archive/common-queries.js

@@ -1,40 +0,0 @@
-/**
- * Returns a Knex subquery builder that selects page IDs with pagination,
- * ordered by the `order` column (nulls last), excluding redirected pages.
- * @param limit - The maximum number of page IDs to return.
- * @param offset - The number of page IDs to skip before returning results.
- */
-export function limitedPageIds(limit, offset) {
-    return async (qb) => {
-        await qb
-            .select('id')
-            .from('pages')
-            .orderByRaw('`order` ASC NULLS LAST')
-            .whereNull('redirectDestId')
-            .limit(limit)
-            .offset(offset);
-    };
-}
-/**
- * Returns a Knex subquery builder that joins pages with their redirect destinations.
- * When `includeNull` is true, also includes pages without redirects (self-referencing).
- * @param includeNull - Whether to include non-redirected pages in the result. Defaults to `true`.
- */
-export function redirectTable(includeNull = true) {
-    return async (qb) => {
-        const list = qb
-            .select('A.id as fromId', 'A.url as from', 'B.url as to', 'B.id as toId')
-            .from('pages as A')
-            .join('pages as B', (j) => {
-            j.on('A.redirectDestId', '=', 'B.id').andOnNotNull('A.redirectDestId');
-        });
-        if (includeNull) {
-            await list.union(async (qb) => {
-                await qb
-                    .select('A.id as fromId', 'A.url as from', 'A.url as to', 'A.id as toId')
-                    .from('pages as A')
-                    .whereNull('A.redirectDestId');
-            });
-        }
-    };
-}

package/lib/archive/filesystem/index.d.ts

@@ -1,17 +0,0 @@
-export { outputJSON } from './output-json.js';
-export { readJSON } from './read-json.js';
-export { outputText } from './output-text.js';
-export { appendText } from './append-text.js';
-export { readText } from './read-text.js';
-export { copyDir } from './copy-dir.js';
-export { copyDirSync } from './copy-dir-sync.js';
-export { isDir } from './is-dir.js';
-export { remove } from './remove.js';
-export { rename } from './rename.js';
-export { getFileList } from './get-file-list.js';
-export { readline } from './readline.js';
-export { mkdir } from './mkdir.js';
-export { exists } from './exists.js';
-export { tar } from './tar.js';
-export { untar } from './untar.js';
-export { zip, unzip, extractZip } from '@d-zero/fs/zip';

package/lib/archive/filesystem/index.js

@@ -1,17 +0,0 @@
-export { outputJSON } from './output-json.js';
-export { readJSON } from './read-json.js';
-export { outputText } from './output-text.js';
-export { appendText } from './append-text.js';
-export { readText } from './read-text.js';
-export { copyDir } from './copy-dir.js';
-export { copyDirSync } from './copy-dir-sync.js';
-export { isDir } from './is-dir.js';
-export { remove } from './remove.js';
-export { rename } from './rename.js';
-export { getFileList } from './get-file-list.js';
-export { readline } from './readline.js';
-export { mkdir } from './mkdir.js';
-export { exists } from './exists.js';
-export { tar } from './tar.js';
-export { untar } from './untar.js';
-export { zip, unzip, extractZip } from '@d-zero/fs/zip';

package/lib/archive/filesystem/utils.d.ts

@@ -1,109 +0,0 @@
-/**
- * Writes data to a JSON file at the specified path.
- *
- * Creates parent directories if they do not exist.
- * The output is formatted with 2-space indentation.
- * @param filePath - The absolute or relative path to the JSON file to write.
- * @param data - The data to serialize as JSON and write to the file.
- */
-export declare function outputJSON(filePath: string, data: unknown): Promise<void>;
-/**
- * Reads and parses a JSON file from the specified path.
- * @template T - The expected type of the parsed JSON content. Defaults to `unknown`.
- * @param filePath - The absolute or relative path to the JSON file to read.
- * @returns The parsed JSON content, cast to the specified generic type.
- */
-export declare function readJSON<T = unknown>(filePath: string): Promise<T>;
-/**
- * Writes text data to a file at the specified path.
- *
- * Creates parent directories if they do not exist.
- * If the file path exceeds the OS limit (ENAMETOOLONG), the file is saved
- * with an auto-generated short name and an accompanying `.meta.txt` file
- * that records the original file path.
- * @param filePath - The absolute or relative path to the text file to write.
- * @param data - The text content to write to the file.
- */
-export declare function outputText(filePath: string, data: string): Promise<void>;
-/**
- * Appends text data to a file at the specified path.
- *
- * Creates parent directories if they do not exist.
- * A newline character is prepended to the data before appending.
- * @param filePath - The absolute or relative path to the file to append to.
- * @param data - The text content to append to the file.
- */
-export declare function appendText(filePath: string, data: string): Promise<void>;
-/**
- * Reads the entire contents of a text file as a UTF-8 string.
- * @param filePath - The absolute or relative path to the text file to read.
- * @returns The text content of the file.
- */
-export declare function readText(filePath: string): Promise<string>;
-/**
- * Recursively copies a directory and its contents from one location to another.
- * @param from - The source directory path to copy from.
- * @param to - The destination directory path to copy to.
- * @returns `true` if the copy succeeded, `false` if an error occurred.
- */
-export declare function copyDir(from: string, to: string): Promise<boolean>;
-/**
- * Synchronously copies a directory and its contents from one location to another.
- * @param from - The source directory path to copy from.
- * @param to - The destination directory path to copy to.
- */
-export declare function copyDirSync(from: string, to: string): void;
-/**
- * Checks whether the given path points to a directory.
- * @param dirPath - The path to check.
- * @returns `true` if the path is a directory, `false` otherwise.
- */
-export declare function isDir(dirPath: string): Promise<boolean>;
-/**
- * Recursively removes a file or directory at the specified path.
- * @param dirPath - The path of the file or directory to remove.
- */
-export declare function remove(dirPath: string): Promise<void>;
-/**
- * Renames (moves) a file or directory from one path to another.
- *
- * If `override` is `true` and the destination already exists,
- * the destination is removed before renaming.
- * @param oldPath - The current path of the file or directory.
- * @param newPath - The new path for the file or directory.
- * @param override - Whether to overwrite the destination if it already exists. Defaults to `false`.
- */
-export declare function rename(oldPath: string, newPath: string, override?: boolean): Promise<void>;
-/**
- * Lists the file names in a directory, optionally filtered by a pattern.
- * @param dirPath - The directory path to list files from.
- * @param filter - An optional RegExp or string pattern to filter file names.
- *   Only file names matching this pattern are included in the result.
- * @returns An array of file names in the directory that match the filter (or all if no filter is provided).
- */
-export declare function getFileList(dirPath: string, filter?: RegExp | string): Promise<string[]>;
-/**
- * Reads a file line by line and invokes the callback for each line.
- *
- * The callback may return a Promise for asynchronous processing.
- * All callback results are collected and awaited via `Promise.all` before returning.
- * @param filePath - The path to the file to read line by line.
- * @param callback - A function invoked for each line of the file.
- *   May return a Promise for asynchronous operations.
- * @returns A promise that resolves when all line callbacks have completed.
- */
-export declare function readline(filePath: string, callback: (line: string) => Promise<void> | void): Promise<void[]>;
-/**
- * Ensures the parent directory of the given file path exists.
- *
- * If the parent directory does not exist, it is created recursively
- * with permissions `0o755`.
- * @param filePath - The file path whose parent directory should be created.
- */
-export declare function mkdir(filePath: string): void;
-/**
- * Checks whether a file or directory exists at the given path.
- * @param filePath - The path to check for existence.
- * @returns `true` if the path exists, `false` otherwise.
- */
-export declare function exists(filePath: string): boolean;

package/lib/archive/filesystem/utils.js

@@ -1,185 +0,0 @@
-import { createReadStream, existsSync, promises as fs, mkdirSync } from 'node:fs';
-import path from 'node:path';
-import Readline from 'node:readline';
-import fsx from 'fs-extra';
-/**
- * Writes data to a JSON file at the specified path.
- *
- * Creates parent directories if they do not exist.
- * The output is formatted with 2-space indentation.
- * @param filePath - The absolute or relative path to the JSON file to write.
- * @param data - The data to serialize as JSON and write to the file.
- */
-export async function outputJSON(filePath, data) {
-    mkdir(filePath);
-    await fs.writeFile(filePath, JSON.stringify(data, null, 2), { encoding: 'utf8' });
-}
-/**
- * Reads and parses a JSON file from the specified path.
- * @template T - The expected type of the parsed JSON content. Defaults to `unknown`.
- * @param filePath - The absolute or relative path to the JSON file to read.
- * @returns The parsed JSON content, cast to the specified generic type.
- */
-export async function readJSON(filePath) {
-    const data = await fs.readFile(filePath, { encoding: 'utf8' });
-    return JSON.parse(data);
-}
-let filePathTooLongCount = 0;
-/**
- * Writes text data to a file at the specified path.
- *
- * Creates parent directories if they do not exist.
- * If the file path exceeds the OS limit (ENAMETOOLONG), the file is saved
- * with an auto-generated short name and an accompanying `.meta.txt` file
- * that records the original file path.
- * @param filePath - The absolute or relative path to the text file to write.
- * @param data - The text content to write to the file.
- */
-export async function outputText(filePath, data) {
-    mkdir(filePath);
-    await fs.writeFile(filePath, data, { encoding: 'utf8' }).catch(async (error) => {
-        if (error instanceof Error && 'code' in error && error.code === 'ENAMETOOLONG') {
-            // eslint-disable-next-line no-console
-            console.error(`File path too long: ${filePath}`);
-            const dir = path.dirname(filePath);
-            const altFileName = `__file_path_too_long_${(filePathTooLongCount++).toString().padStart(4, '0')}`;
-            const ext = path.extname(filePath);
-            const altFilePath = path.resolve(dir, `${altFileName}${ext}`);
-            // eslint-disable-next-line no-console
-            console.error(`Try to save to: ${altFilePath}`);
-            const altMetaFilePath = path.resolve(dir, `${altFileName}.meta.txt`);
-            await outputText(altFilePath, data);
-            await outputText(altMetaFilePath, `Original file path: ${filePath}`);
-        }
-    });
-}
-/**
- * Appends text data to a file at the specified path.
- *
- * Creates parent directories if they do not exist.
- * A newline character is prepended to the data before appending.
- * @param filePath - The absolute or relative path to the file to append to.
- * @param data - The text content to append to the file.
- */
-export async function appendText(filePath, data) {
-    mkdir(filePath);
-    await fs.appendFile(filePath, `\n${data}`, { encoding: 'utf8' });
-}
-/**
- * Reads the entire contents of a text file as a UTF-8 string.
- * @param filePath - The absolute or relative path to the text file to read.
- * @returns The text content of the file.
- */
-export async function readText(filePath) {
-    const data = await fs.readFile(filePath, { encoding: 'utf8' });
-    return data;
-}
-/**
- * Recursively copies a directory and its contents from one location to another.
- * @param from - The source directory path to copy from.
- * @param to - The destination directory path to copy to.
- * @returns `true` if the copy succeeded, `false` if an error occurred.
- */
-export async function copyDir(from, to) {
-    return fsx
-        .copy(from, to)
-        .then(() => true)
-        .catch(() => false);
-}
-/**
- * Synchronously copies a directory and its contents from one location to another.
- * @param from - The source directory path to copy from.
- * @param to - The destination directory path to copy to.
- */
-export function copyDirSync(from, to) {
-    fsx.copySync(from, to);
-}
-/**
- * Checks whether the given path points to a directory.
- * @param dirPath - The path to check.
- * @returns `true` if the path is a directory, `false` otherwise.
- */
-export async function isDir(dirPath) {
-    const stat = await fsx.stat(dirPath);
-    return stat.isDirectory();
-}
-/**
- * Recursively removes a file or directory at the specified path.
- * @param dirPath - The path of the file or directory to remove.
- */
-export async function remove(dirPath) {
-    await fs.rm(dirPath, {
-        recursive: true,
-    });
-}
-/**
- * Renames (moves) a file or directory from one path to another.
- *
- * If `override` is `true` and the destination already exists,
- * the destination is removed before renaming.
- * @param oldPath - The current path of the file or directory.
- * @param newPath - The new path for the file or directory.
- * @param override - Whether to overwrite the destination if it already exists. Defaults to `false`.
- */
-export async function rename(oldPath, newPath, override = false) {
-    if (override && exists(newPath)) {
-        await remove(newPath);
-    }
-    await fs.rename(oldPath, newPath);
-}
-/**
- * Lists the file names in a directory, optionally filtered by a pattern.
- * @param dirPath - The directory path to list files from.
- * @param filter - An optional RegExp or string pattern to filter file names.
- *   Only file names matching this pattern are included in the result.
- * @returns An array of file names in the directory that match the filter (or all if no filter is provided).
- */
-export async function getFileList(dirPath, filter) {
-    const list = await fsx.readdir(dirPath);
-    return filter ? list.filter((fileName) => fileName.match(filter)) : list;
-}
-/**
- * Reads a file line by line and invokes the callback for each line.
- *
- * The callback may return a Promise for asynchronous processing.
- * All callback results are collected and awaited via `Promise.all` before returning.
- * @param filePath - The path to the file to read line by line.
- * @param callback - A function invoked for each line of the file.
- *   May return a Promise for asynchronous operations.
- * @returns A promise that resolves when all line callbacks have completed.
- */
-export async function readline(filePath, callback) {
-    const stream = createReadStream(filePath);
-    const rLine = Readline.createInterface(stream);
-    const promiseBuffer = [];
-    await new Promise((resolve) => {
-        rLine.on('line', (line) => {
-            promiseBuffer.push(callback(line));
-        });
-        rLine.on('close', () => {
-            resolve();
-        });
-    });
-    return Promise.all(promiseBuffer);
-}
-/**
- * Ensures the parent directory of the given file path exists.
- *
- * If the parent directory does not exist, it is created recursively
- * with permissions `0o755`.
- * @param filePath - The file path whose parent directory should be created.
- */
-export function mkdir(filePath) {
-    const { dir } = path.parse(filePath);
-    if (!existsSync(dir)) {
-        mkdirSync(path.resolve(dir), { recursive: true, mode: 0o755 });
-    }
-}
-/**
- * Checks whether a file or directory exists at the given path.
- * @param filePath - The path to check for existence.
- * @returns `true` if the path exists, `false` otherwise.
- */
-export function exists(filePath) {
-    return existsSync(filePath);
-}

package/lib/archive/filesystem/zip.d.ts

@@ -1,29 +0,0 @@
-/**
- * Creates a ZIP archive from a directory and writes it to a file.
- *
- * All files and subdirectories within `targetDir` are added to the archive
- * at the root level (no wrapping directory).
- * @param outputfilePath - The file path where the ZIP archive will be written.
- * @param targetDir - The directory whose contents will be compressed into the ZIP archive.
- * @returns A promise that resolves when the ZIP file has been fully written,
- *   or rejects if the write stream encounters an error.
- */
-export declare function zip(outputfilePath: string, targetDir: string): Promise<void>;
-/**
- * Extracts a ZIP archive to a target directory.
- * @param zipFilePath - The path to the ZIP file to extract.
- * @param targetDir - The directory where the ZIP contents will be extracted to.
- * @returns A promise that resolves when extraction is complete,
- *   or rejects if an error occurs during extraction.
- */
-export declare function unzip(zipFilePath: string, targetDir: string): Promise<void>;
-/**
- * Opens a ZIP file and returns its directory listing without extracting.
- *
- * This is useful for inspecting the contents of a ZIP archive
- * before performing a full extraction.
- * @param zipFilePath - The path to the ZIP file to open and inspect.
- * @returns A directory object representing the ZIP archive contents,
- *   which can be used to list or selectively extract entries.
- */
-export declare function extractZip(zipFilePath: string): Promise<any>;

package/lib/archive/filesystem/zip.js

@@ -1,53 +0,0 @@
-import fs from 'node:fs';
-import archiver from 'archiver';
-import unzipper from 'unzipper';
-/**
- * Creates a ZIP archive from a directory and writes it to a file.
- *
- * All files and subdirectories within `targetDir` are added to the archive
- * at the root level (no wrapping directory).
- * @param outputfilePath - The file path where the ZIP archive will be written.
- * @param targetDir - The directory whose contents will be compressed into the ZIP archive.
- * @returns A promise that resolves when the ZIP file has been fully written,
- *   or rejects if the write stream encounters an error.
- */
-export async function zip(outputfilePath, targetDir) {
-    const output = fs.createWriteStream(outputfilePath);
-    const archive = archiver('zip');
-    archive.pipe(output);
-    archive.directory(targetDir, false);
-    await archive.finalize();
-    return new Promise((resolve, reject) => {
-        output.on('finish', () => resolve());
-        output.on('error', () => reject(`Failed to save file "${outputfilePath}" from "${targetDir}"`));
-    });
-}
-/**
- * Extracts a ZIP archive to a target directory.
- * @param zipFilePath - The path to the ZIP file to extract.
- * @param targetDir - The directory where the ZIP contents will be extracted to.
- * @returns A promise that resolves when extraction is complete,
- *   or rejects if an error occurs during extraction.
- */
-export async function unzip(zipFilePath, targetDir) {
-    const extract = fs.createReadStream(zipFilePath).pipe(unzipper.Extract({
-        path: targetDir,
-    }));
-    return new Promise((resolve, reject) => {
-        extract.on('finish', () => resolve());
-        extract.on('error', (err) => reject(err));
-    });
-}
-/**
- * Opens a ZIP file and returns its directory listing without extracting.
- *
- * This is useful for inspecting the contents of a ZIP archive
- * before performing a full extraction.
- * @param zipFilePath - The path to the ZIP file to open and inspect.
- * @returns A directory object representing the ZIP archive contents,
- *   which can be used to list or selectively extract entries.
- */
-export async function extractZip(zipFilePath) {
-    const directory = await unzipper.Open.file(zipFilePath);
-    return directory;
-}

package/lib/archive/index.d.ts

@@ -1,6 +0,0 @@
-export * from './archive-accessor.js';
-export type { Redirect, Referrer, Anchor, StaticPageData } from './page.js';
-export { default as Page } from './page.js';
-export { default as Resource } from './resource.js';
-export * from './types.js';
-export { default } from './archive.js';

package/lib/archive/index.js

@@ -1,11 +0,0 @@
-// Archive storage and retrieval layer for Nitpicker crawl data.
-//
-// This package provides the `Archive` class for creating, reading, and writing
-// `.nitpicker` archive files that store crawl results in a SQLite database along with
-// optional HTML snapshots. It also exports the `ArchiveAccessor` for read-only
-// access, `Page` and `Resource` model classes, and all related types.
-export * from './archive-accessor.js';
-export { default as Page } from './page.js';
-export { default as Resource } from './resource.js';
-export * from './types.js';
-export { default } from './archive.js';

package/lib/crawler/index.d.ts

@@ -1,2 +0,0 @@
-export { default as Crawler } from './crawler.js';
-export { clearDestinationCache } from './clear-destination-cache.js';

package/lib/crawler/index.js

@@ -1,2 +0,0 @@
-export { default as Crawler } from './crawler.js';
-export { clearDestinationCache } from './clear-destination-cache.js';

package/lib/crawler/network.d.ts

@@ -1,30 +0,0 @@
-import type { ExURL } from '@d-zero/shared/parse-url';
-import type { PageData } from '@nitpicker/beholder';
-/**
- * Clears the in-memory cache of HTTP request results.
- * Should be called between crawl sessions to prevent memory leaks.
- */
-export declare function clearDestinationCache(): void;
-/**
- * Fetches the destination metadata for a URL using an HTTP HEAD request (or GET as fallback).
- *
- * Results are cached in memory so that repeated calls for the same URL
- * (without hash) return immediately. The request races against a 10-second
- * timeout; if the server does not respond in time, a {@link NetTimeoutError} is thrown.
- *
- * If the server returns 405 (Method Not Allowed), 501 (Not Implemented), or 503
- * (Service Unavailable) for a HEAD request, the function automatically retries with GET.
- * @param url - The extended URL to fetch.
- * @param isExternal - Whether the URL is external to the crawl scope.
- * @param method - The HTTP method to use. Defaults to `"HEAD"`.
- * @param options - Additional options.
- * @param options.titleBytesLimit - When set, forces a GET request and reads up to this many
- *   bytes from the response body to extract an HTML `<title>` tag. The connection is
- *   destroyed as soon as the limit is reached or a title is found.
- * @returns The page metadata obtained from the HTTP response.
- * @throws {NetTimeoutError} If the request exceeds the 10-second timeout.
- * @throws {Error} If the HTTP request fails for any other reason.
- */
-export declare function fetchDestination(url: ExURL, isExternal: boolean, method?: string, options?: {
-    titleBytesLimit?: number;
-}): Promise<PageData>;