@nitpicker/crawler 0.4.1

package/lib/archive/filesystem/utils.js

@@ -0,0 +1,185 @@
+import { createReadStream, existsSync, promises as fs, mkdirSync } from 'node:fs';
+import path from 'node:path';
+import Readline from 'node:readline';
+import fsx from 'fs-extra';
+/**
+ * Writes data to a JSON file at the specified path.
+ *
+ * Creates parent directories if they do not exist.
+ * The output is formatted with 2-space indentation.
+ * @param filePath - The absolute or relative path to the JSON file to write.
+ * @param data - The data to serialize as JSON and write to the file.
+ */
+export async function outputJSON(filePath, data) {
+    mkdir(filePath);
+    await fs.writeFile(filePath, JSON.stringify(data, null, 2), { encoding: 'utf8' });
+}
+/**
+ * Reads and parses a JSON file from the specified path.
+ * @template T - The expected type of the parsed JSON content. Defaults to `unknown`.
+ * @param filePath - The absolute or relative path to the JSON file to read.
+ * @returns The parsed JSON content, cast to the specified generic type.
+ */
+export async function readJSON(filePath) {
+    const data = await fs.readFile(filePath, { encoding: 'utf8' });
+    return JSON.parse(data);
+}
+let filePathTooLongCount = 0;
+/**
+ * Writes text data to a file at the specified path.
+ *
+ * Creates parent directories if they do not exist.
+ * If the file path exceeds the OS limit (ENAMETOOLONG), the file is saved
+ * with an auto-generated short name and an accompanying `.meta.txt` file
+ * that records the original file path.
+ * @param filePath - The absolute or relative path to the text file to write.
+ * @param data - The text content to write to the file.
+ */
+export async function outputText(filePath, data) {
+    mkdir(filePath);
+    await fs.writeFile(filePath, data, { encoding: 'utf8' }).catch(async (error) => {
+        if (error instanceof Error && 'code' in error && error.code === 'ENAMETOOLONG') {
+            // eslint-disable-next-line no-console
+            console.error(`File path too long: ${filePath}`);
+            const dir = path.dirname(filePath);
+            const altFileName = `__file_path_too_long_${(filePathTooLongCount++).toString().padStart(4, '0')}`;
+            const ext = path.extname(filePath);
+            const altFilePath = path.resolve(dir, `${altFileName}${ext}`);
+            // eslint-disable-next-line no-console
+            console.error(`Try to save to: ${altFilePath}`);
+            const altMetaFilePath = path.resolve(dir, `${altFileName}.meta.txt`);
+            await outputText(altFilePath, data);
+            await outputText(altMetaFilePath, `Original file path: ${filePath}`);
+        }
+    });
+}
+/**
+ * Appends text data to a file at the specified path.
+ *
+ * Creates parent directories if they do not exist.
+ * A newline character is prepended to the data before appending.
+ * @param filePath - The absolute or relative path to the file to append to.
+ * @param data - The text content to append to the file.
+ */
+export async function appendText(filePath, data) {
+    mkdir(filePath);
+    await fs.appendFile(filePath, `\n${data}`, { encoding: 'utf8' });
+}
+/**
+ * Reads the entire contents of a text file as a UTF-8 string.
+ * @param filePath - The absolute or relative path to the text file to read.
+ * @returns The text content of the file.
+ */
+export async function readText(filePath) {
+    const data = await fs.readFile(filePath, { encoding: 'utf8' });
+    return data;
+}
+/**
+ * Recursively copies a directory and its contents from one location to another.
+ * @param from - The source directory path to copy from.
+ * @param to - The destination directory path to copy to.
+ * @returns `true` if the copy succeeded, `false` if an error occurred.
+ */
+export async function copyDir(from, to) {
+    return fsx
+        .copy(from, to)
+        .then(() => true)
+        .catch(() => false);
+}
+/**
+ * Synchronously copies a directory and its contents from one location to another.
+ * @param from - The source directory path to copy from.
+ * @param to - The destination directory path to copy to.
+ */
+export function copyDirSync(from, to) {
+    fsx.copySync(from, to);
+}
+/**
+ * Checks whether the given path points to a directory.
+ * @param dirPath - The path to check.
+ * @returns `true` if the path is a directory, `false` otherwise.
+ */
+export async function isDir(dirPath) {
+    const stat = await fsx.stat(dirPath);
+    return stat.isDirectory();
+}
+/**
+ * Recursively removes a file or directory at the specified path.
+ * @param dirPath - The path of the file or directory to remove.
+ */
+export async function remove(dirPath) {
+    await fs.rm(dirPath, {
+        recursive: true,
+    });
+}
+/**
+ * Renames (moves) a file or directory from one path to another.
+ *
+ * If `override` is `true` and the destination already exists,
+ * the destination is removed before renaming.
+ * @param oldPath - The current path of the file or directory.
+ * @param newPath - The new path for the file or directory.
+ * @param override - Whether to overwrite the destination if it already exists. Defaults to `false`.
+ */
+export async function rename(oldPath, newPath, override = false) {
+    if (override && exists(newPath)) {
+        await remove(newPath);
+    }
+    await fs.rename(oldPath, newPath);
+}
+/**
+ * Lists the file names in a directory, optionally filtered by a pattern.
+ * @param dirPath - The directory path to list files from.
+ * @param filter - An optional RegExp or string pattern to filter file names.
+ *   Only file names matching this pattern are included in the result.
+ * @returns An array of file names in the directory that match the filter (or all if no filter is provided).
+ */
+export async function getFileList(dirPath, filter) {
+    const list = await fsx.readdir(dirPath);
+    return filter ? list.filter((fileName) => fileName.match(filter)) : list;
+}
+/**
+ * Reads a file line by line and invokes the callback for each line.
+ *
+ * The callback may return a Promise for asynchronous processing.
+ * All callback results are collected and awaited via `Promise.all` before returning.
+ * @param filePath - The path to the file to read line by line.
+ * @param callback - A function invoked for each line of the file.
+ *   May return a Promise for asynchronous operations.
+ * @returns A promise that resolves when all line callbacks have completed.
+ */
+export async function readline(filePath, callback) {
+    const stream = createReadStream(filePath);
+    const rLine = Readline.createInterface(stream);
+    const promiseBuffer = [];
+    await new Promise((resolve) => {
+        rLine.on('line', (line) => {
+            promiseBuffer.push(callback(line));
+        });
+        rLine.on('close', () => {
+            resolve();
+        });
+    });
+    return Promise.all(promiseBuffer);
+}
+/**
+ * Ensures the parent directory of the given file path exists.
+ *
+ * If the parent directory does not exist, it is created recursively
+ * with permissions `0o755`.
+ * @param filePath - The file path whose parent directory should be created.
+ */
+export function mkdir(filePath) {
+    const { dir } = path.parse(filePath);
+    if (!existsSync(dir)) {
+        mkdirSync(path.resolve(dir), { recursive: true, mode: 0o755 });
+    }
+}
+/**
+ * Checks whether a file or directory exists at the given path.
+ * @param filePath - The path to check for existence.
+ * @returns `true` if the path exists, `false` otherwise.
+ */
+export function exists(filePath) {
+    return existsSync(filePath);
+}

package/lib/archive/filesystem/zip.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Creates a ZIP archive from a directory and writes it to a file.
+ *
+ * All files and subdirectories within `targetDir` are added to the archive
+ * at the root level (no wrapping directory).
+ * @param outputfilePath - The file path where the ZIP archive will be written.
+ * @param targetDir - The directory whose contents will be compressed into the ZIP archive.
+ * @returns A promise that resolves when the ZIP file has been fully written,
+ *   or rejects if the write stream encounters an error.
+ */
+export declare function zip(outputfilePath: string, targetDir: string): Promise<void>;
+/**
+ * Extracts a ZIP archive to a target directory.
+ * @param zipFilePath - The path to the ZIP file to extract.
+ * @param targetDir - The directory where the ZIP contents will be extracted to.
+ * @returns A promise that resolves when extraction is complete,
+ *   or rejects if an error occurs during extraction.
+ */
+export declare function unzip(zipFilePath: string, targetDir: string): Promise<void>;
+/**
+ * Opens a ZIP file and returns its directory listing without extracting.
+ *
+ * This is useful for inspecting the contents of a ZIP archive
+ * before performing a full extraction.
+ * @param zipFilePath - The path to the ZIP file to open and inspect.
+ * @returns A directory object representing the ZIP archive contents,
+ *   which can be used to list or selectively extract entries.
+ */
+export declare function extractZip(zipFilePath: string): Promise<any>;

package/lib/archive/filesystem/zip.js

@@ -0,0 +1,53 @@
+import fs from 'node:fs';
+import archiver from 'archiver';
+import unzipper from 'unzipper';
+/**
+ * Creates a ZIP archive from a directory and writes it to a file.
+ *
+ * All files and subdirectories within `targetDir` are added to the archive
+ * at the root level (no wrapping directory).
+ * @param outputfilePath - The file path where the ZIP archive will be written.
+ * @param targetDir - The directory whose contents will be compressed into the ZIP archive.
+ * @returns A promise that resolves when the ZIP file has been fully written,
+ *   or rejects if the write stream encounters an error.
+ */
+export async function zip(outputfilePath, targetDir) {
+    const output = fs.createWriteStream(outputfilePath);
+    const archive = archiver('zip');
+    archive.pipe(output);
+    archive.directory(targetDir, false);
+    await archive.finalize();
+    return new Promise((resolve, reject) => {
+        output.on('finish', () => resolve());
+        output.on('error', () => reject(`Failed to save file "${outputfilePath}" from "${targetDir}"`));
+    });
+}
+/**
+ * Extracts a ZIP archive to a target directory.
+ * @param zipFilePath - The path to the ZIP file to extract.
+ * @param targetDir - The directory where the ZIP contents will be extracted to.
+ * @returns A promise that resolves when extraction is complete,
+ *   or rejects if an error occurs during extraction.
+ */
+export async function unzip(zipFilePath, targetDir) {
+    const extract = fs.createReadStream(zipFilePath).pipe(unzipper.Extract({
+        path: targetDir,
+    }));
+    return new Promise((resolve, reject) => {
+        extract.on('finish', () => resolve());
+        extract.on('error', (err) => reject(err));
+    });
+}
+/**
+ * Opens a ZIP file and returns its directory listing without extracting.
+ *
+ * This is useful for inspecting the contents of a ZIP archive
+ * before performing a full extraction.
+ * @param zipFilePath - The path to the ZIP file to open and inspect.
+ * @returns A directory object representing the ZIP archive contents,
+ *   which can be used to list or selectively extract entries.
+ */
+export async function extractZip(zipFilePath) {
+    const directory = await unzipper.Open.file(zipFilePath);
+    return directory;
+}

package/lib/archive/index.d.ts

@@ -0,0 +1,6 @@
+export * from './archive-accessor.js';
+export type { Redirect, Referrer, Anchor, StaticPageData } from './page.js';
+export { default as Page } from './page.js';
+export { default as Resource } from './resource.js';
+export * from './types.js';
+export { default } from './archive.js';

package/lib/archive/index.js

@@ -0,0 +1,11 @@
+// Archive storage and retrieval layer for Nitpicker crawl data.
+//
+// This package provides the `Archive` class for creating, reading, and writing
+// `.nitpicker` archive files that store crawl results in a SQLite database along with
+// optional HTML snapshots. It also exports the `ArchiveAccessor` for read-only
+// access, `Page` and `Resource` model classes, and all related types.
+export * from './archive-accessor.js';
+export { default as Page } from './page.js';
+export { default as Resource } from './resource.js';
+export * from './types.js';
+export { default } from './archive.js';

package/lib/archive/page.d.ts

@@ -0,0 +1,263 @@
+import type { ArchiveAccessor } from './archive-accessor.js';
+import type { DB_Anchor, DB_Page, DB_Redirect, DB_Referrer } from './types.js';
+/**
+ * Represents a crawled page stored in the archive.
+ *
+ * Provides access to the page's metadata (title, status, SEO tags, etc.),
+ * its relationships (anchors, referrers, redirects), and its HTML snapshot.
+ * Instances are created by {@link ArchiveAccessor.getPages} or
+ * {@link ArchiveAccessor.getPagesWithRefs}.
+ */
+export default class Page {
+    #private;
+    /**
+     * An array of URLs that redirect to this page.
+     * Each entry contains the source URL and its page ID.
+     * Returns an empty array if no redirects exist.
+     */
+    readonly redirectFrom: Redirect[];
+    /**
+     * The alternate URL from the `<link rel="alternate">` tag, or null if not present.
+     */
+    get alternate(): string | null;
+    /**
+     * The canonical URL from the `<link rel="canonical">` tag, or null if not present.
+     */
+    get canonical(): string | null;
+    /**
+     * The content length of the HTTP response in bytes, or null if unknown.
+     */
+    get contentLength(): number | null;
+    /**
+     * The MIME content type of the HTTP response (e.g., `"text/html"`), or null if unknown.
+     */
+    get contentType(): string | null;
+    /**
+     * The meta description content, or null if not present.
+     */
+    get description(): string | null;
+    /**
+     * Whether this page is on an external domain (outside the crawl scope).
+     */
+    get isExternal(): boolean;
+    /**
+     * Whether this page was skipped during crawling.
+     */
+    get isSkipped(): boolean;
+    /**
+     * Whether this page was a crawl target (as opposed to being discovered incidentally).
+     */
+    get isTarget(): boolean;
+    /**
+     * The reason this page was skipped during crawling, or null if it was not skipped.
+     */
+    get skipReason(): string | null;
+    /**
+     * The meta keywords content, or null if not present.
+     */
+    get keywords(): string | null;
+    /**
+     * The `lang` attribute value from the HTML element, or null if not present.
+     */
+    get lang(): string | null;
+    /**
+     * Whether the noarchive robots directive is set.
+     */
+    get noarchive(): boolean;
+    /**
+     * Whether the nofollow robots directive is set.
+     */
+    get nofollow(): boolean;
+    /**
+     * Whether the noindex robots directive is set.
+     */
+    get noindex(): boolean;
+    /**
+     * The Open Graph description (`og:description`), or null if not present.
+     */
+    get og_description(): string | null;
+    /**
+     * The Open Graph image URL (`og:image`), or null if not present.
+     */
+    get og_image(): string | null;
+    /**
+     * The Open Graph site name (`og:site_name`), or null if not present.
+     */
+    get og_site_name(): string | null;
+    /**
+     * The Open Graph title (`og:title`), or null if not present.
+     */
+    get og_title(): string | null;
+    /**
+     * The Open Graph type (`og:type`), or null if not present.
+     */
+    get og_type(): string | null;
+    /**
+     * The Open Graph URL (`og:url`), or null if not present.
+     */
+    get og_url(): string | null;
+    /**
+     * The parsed HTTP response headers as a key-value record.
+     * Returns an empty object if headers cannot be parsed.
+     */
+    get responseHeaders(): Record<string, string>;
+    /**
+     * The HTTP response status code, or null if the page has not been fetched.
+     */
+    get status(): number | null;
+    /**
+     * The HTTP response status text (e.g., `"OK"`, `"Not Found"`), or null if not fetched.
+     */
+    get statusText(): string | null;
+    /**
+     * The page title from the `<title>` element.
+     * Returns an empty string if no title is set.
+     */
+    get title(): string;
+    /**
+     * The Twitter Card type (`twitter:card`), or null if not present.
+     */
+    get twitter_card(): string | null;
+    /**
+     * The parsed URL of this page as an ExURL object.
+     * Respects the `disableQueries` option for query string handling.
+     */
+    get url(): import("@d-zero/shared/parse-url").ExURL;
+    /**
+     * Creates a new Page instance.
+     * @param archive - The ArchiveAccessor used for lazy-loading relationships.
+     * @param raw - The raw database row for this page.
+     * @param rawRedirects - Pre-loaded redirect records, or undefined for lazy loading.
+     * @param rawAnchors - Pre-loaded anchor records, or undefined for lazy loading.
+     * @param rawReferrers - Pre-loaded referrer records, or undefined for lazy loading.
+     * @param disableQueries - Whether to strip query strings from the URL.
+     */
+    constructor(archive: ArchiveAccessor, raw: DB_Page, rawRedirects?: DB_Redirect[], rawAnchors?: DB_Anchor[], rawReferrers?: DB_Referrer[], disableQueries?: boolean);
+    /**
+     * Retrieves the anchors (outgoing links) found on this page.
+     * Uses pre-loaded data if available, otherwise queries the database.
+     * @returns An array of {@link Anchor} objects representing the links on this page.
+     */
+    getAnchors(): Promise<Anchor[]>;
+    /**
+     * Reads the HTML snapshot content of this page from the archive.
+     * @returns The HTML content as a string, or null if no snapshot was saved.
+     */
+    getHtml(): Promise<string | null>;
+    /**
+     * Retrieves the referrers (incoming links) pointing to this page.
+     * Uses pre-loaded data if available, otherwise queries the database.
+     * @returns An array of {@link Referrer} objects representing pages that link to this page.
+     */
+    getReferrers(): Promise<Referrer[]>;
+    /**
+     * Retrieves all request referrers for this page directly from the database.
+     * Unlike {@link getReferrers}, this always queries the database and does not use pre-loaded data.
+     * @returns An array of {@link Referrer} objects.
+     */
+    getRequests(): Promise<Referrer[]>;
+    /**
+     * Checks whether this page is an internal HTML page (not external and has `text/html` content type).
+     * @returns `true` if this is an internal HTML page, `false` otherwise.
+     */
+    isInternalPage(): boolean;
+    /**
+     * Checks whether this entry represents an HTML page (content type is `text/html`).
+     * @returns `true` if the content type is `text/html`, `false` otherwise.
+     */
+    isPage(): boolean;
+    /**
+     * Serializes the page data to a plain JSON object,
+     * including resolved anchors and referrers.
+     * @returns A plain object containing all page metadata and relationships.
+     */
+    toJSON(): Promise<{
+        url: string;
+        title: string;
+        status: number | null;
+        statusText: string | null;
+        contentType: string | null;
+        contentLength: number | null;
+        responseHeaders: Record<string, string>;
+        isExternal: boolean;
+        isSkipped: boolean;
+        skipReason: string | null;
+        isTarget: boolean;
+        lang: string | null;
+        description: string | null;
+        keywords: string | null;
+        noindex: boolean;
+        nofollow: boolean;
+        noarchive: boolean;
+        canonical: string | null;
+        alternate: string | null;
+        twitter_card: string | null;
+        og_site_name: string | null;
+        og_url: string | null;
+        og_title: string | null;
+        og_description: string | null;
+        og_type: string | null;
+        og_image: string | null;
+        redirectFrom: Redirect[];
+        isPage: boolean;
+        isInternalPage: boolean;
+        getAnchors: Anchor[];
+        getReferrers: Referrer[];
+    }>;
+}
+/**
+ * Utility type that extracts the resolved type from a Promise.
+ */
+type PromiseType<T> = T extends PromiseLike<infer U> ? U : T;
+/**
+ * The static (serialized) representation of a Page, as returned by {@link Page.toJSON}.
+ */
+export type StaticPageData = PromiseType<ReturnType<Page['toJSON']>>;
+/**
+ * Represents a page that links to another page (an incoming link).
+ */
+export interface Referrer {
+    /** The URL of the referring page. */
+    url: string;
+    /** The URL through which the referral passes (may differ due to redirects). */
+    through: string;
+    /** The page ID corresponding to the through URL. */
+    throughId: number;
+    /** The URL fragment (hash) of the referring link, or null if not present. */
+    hash: string | null;
+    /** The text content of the referring anchor element. */
+    textContent: string;
+}
+/**
+ * Represents an outgoing link (anchor element) found on a page.
+ */
+export interface Anchor {
+    /** The resolved destination URL of the anchor. */
+    url: string;
+    /** The original href attribute value of the anchor element. */
+    href: string;
+    /** Whether the anchor points to an external domain. */
+    isExternal: boolean;
+    /** The title attribute of the anchor element, or null if not present. */
+    title: string | null;
+    /** The HTTP status code of the linked page, or null if not yet fetched. */
+    status: number | null;
+    /** The HTTP status text of the linked page, or null if not yet fetched. */
+    statusText: string | null;
+    /** The content type of the linked page, or null if not yet fetched. */
+    contentType: string | null;
+    /** The URL fragment (hash) portion of the link, or null if not present. */
+    hash: string | null;
+    /** The text content of the anchor element, or null if empty. */
+    textContent: string | null;
+}
+/**
+ * Represents a page that redirects to this page.
+ */
+export interface Redirect {
+    /** The URL of the redirect source page. */
+    url: string;
+    /** The database ID of the redirect source page. */
+    pageId: number;
+}
+export {};