@nitpicker/crawler 0.4.1

package/lib/archive/page.js

@@ -0,0 +1,316 @@
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+/**
+ * Represents a crawled page stored in the archive.
+ *
+ * Provides access to the page's metadata (title, status, SEO tags, etc.),
+ * its relationships (anchors, referrers, redirects), and its HTML snapshot.
+ * Instances are created by {@link ArchiveAccessor.getPages} or
+ * {@link ArchiveAccessor.getPagesWithRefs}.
+ */
+export default class Page {
+    /**
+     * An array of URLs that redirect to this page.
+     * Each entry contains the source URL and its page ID.
+     * Returns an empty array if no redirects exist.
+     */
+    redirectFrom;
+    #archive;
+    #disableQueries;
+    #raw;
+    #rawAnchors;
+    #rawReferrers;
+    /**
+     * The alternate URL from the `<link rel="alternate">` tag, or null if not present.
+     */
+    get alternate() {
+        return this.#raw.alternate;
+    }
+    /**
+     * The canonical URL from the `<link rel="canonical">` tag, or null if not present.
+     */
+    get canonical() {
+        return this.#raw.canonical;
+    }
+    /**
+     * The content length of the HTTP response in bytes, or null if unknown.
+     */
+    get contentLength() {
+        return this.#raw.contentLength;
+    }
+    /**
+     * The MIME content type of the HTTP response (e.g., `"text/html"`), or null if unknown.
+     */
+    get contentType() {
+        return this.#raw.contentType;
+    }
+    /**
+     * The meta description content, or null if not present.
+     */
+    get description() {
+        return this.#raw.description;
+    }
+    /**
+     * Whether this page is on an external domain (outside the crawl scope).
+     */
+    get isExternal() {
+        return !!this.#raw.isExternal;
+    }
+    /**
+     * Whether this page was skipped during crawling.
+     */
+    get isSkipped() {
+        return !!this.#raw.isSkipped;
+    }
+    /**
+     * Whether this page was a crawl target (as opposed to being discovered incidentally).
+     */
+    get isTarget() {
+        return !!this.#raw.isTarget;
+    }
+    /**
+     * The reason this page was skipped during crawling, or null if it was not skipped.
+     */
+    get skipReason() {
+        return this.#raw.skipReason;
+    }
+    /**
+     * The meta keywords content, or null if not present.
+     */
+    get keywords() {
+        return this.#raw.keywords;
+    }
+    /**
+     * The `lang` attribute value from the HTML element, or null if not present.
+     */
+    get lang() {
+        return this.#raw.lang;
+    }
+    /**
+     * Whether the noarchive robots directive is set.
+     */
+    get noarchive() {
+        return !!this.#raw.noarchive;
+    }
+    /**
+     * Whether the nofollow robots directive is set.
+     */
+    get nofollow() {
+        return !!this.#raw.nofollow;
+    }
+    /**
+     * Whether the noindex robots directive is set.
+     */
+    get noindex() {
+        return !!this.#raw.noindex;
+    }
+    /**
+     * The Open Graph description (`og:description`), or null if not present.
+     */
+    get og_description() {
+        return this.#raw.og_description;
+    }
+    /**
+     * The Open Graph image URL (`og:image`), or null if not present.
+     */
+    get og_image() {
+        return this.#raw.og_image;
+    }
+    /**
+     * The Open Graph site name (`og:site_name`), or null if not present.
+     */
+    get og_site_name() {
+        return this.#raw.og_site_name;
+    }
+    /**
+     * The Open Graph title (`og:title`), or null if not present.
+     */
+    get og_title() {
+        return this.#raw.og_title;
+    }
+    /**
+     * The Open Graph type (`og:type`), or null if not present.
+     */
+    get og_type() {
+        return this.#raw.og_type;
+    }
+    /**
+     * The Open Graph URL (`og:url`), or null if not present.
+     */
+    get og_url() {
+        return this.#raw.og_url;
+    }
+    /**
+     * The parsed HTTP response headers as a key-value record.
+     * Returns an empty object if headers cannot be parsed.
+     */
+    get responseHeaders() {
+        try {
+            return JSON.parse(this.#raw.responseHeaders);
+        }
+        catch {
+            return {};
+        }
+    }
+    /**
+     * The HTTP response status code, or null if the page has not been fetched.
+     */
+    get status() {
+        return this.#raw.status;
+    }
+    /**
+     * The HTTP response status text (e.g., `"OK"`, `"Not Found"`), or null if not fetched.
+     */
+    get statusText() {
+        return this.#raw.statusText;
+    }
+    /**
+     * The page title from the `<title>` element.
+     * Returns an empty string if no title is set.
+     */
+    get title() {
+        return this.#raw.title || '';
+    }
+    /**
+     * The Twitter Card type (`twitter:card`), or null if not present.
+     */
+    get twitter_card() {
+        return this.#raw.twitter_card;
+    }
+    /**
+     * The parsed URL of this page as an ExURL object.
+     * Respects the `disableQueries` option for query string handling.
+     */
+    get url() {
+        return parseUrl(this.#raw.url, {
+            disableQueries: this.#disableQueries,
+        });
+    }
+    /**
+     * Creates a new Page instance.
+     * @param archive - The ArchiveAccessor used for lazy-loading relationships.
+     * @param raw - The raw database row for this page.
+     * @param rawRedirects - Pre-loaded redirect records, or undefined for lazy loading.
+     * @param rawAnchors - Pre-loaded anchor records, or undefined for lazy loading.
+     * @param rawReferrers - Pre-loaded referrer records, or undefined for lazy loading.
+     * @param disableQueries - Whether to strip query strings from the URL.
+     */
+    constructor(archive, raw, rawRedirects, rawAnchors, rawReferrers, disableQueries) {
+        this.#archive = archive;
+        this.#raw = raw;
+        this.redirectFrom = (rawRedirects || []).map((r) => ({
+            url: r.from,
+            pageId: r.fromId,
+        }));
+        this.#rawAnchors = rawAnchors || null;
+        this.#rawReferrers = rawReferrers || null;
+        this.#disableQueries = disableQueries ?? false;
+    }
+    /**
+     * Retrieves the anchors (outgoing links) found on this page.
+     * Uses pre-loaded data if available, otherwise queries the database.
+     * @returns An array of {@link Anchor} objects representing the links on this page.
+     */
+    async getAnchors() {
+        if (this.#rawAnchors) {
+            return this.#rawAnchors.map((a) => ({
+                url: a.url,
+                href: a.href,
+                isExternal: !!a.isExternal,
+                title: a.title,
+                status: a.status,
+                statusText: a.statusText,
+                contentType: a.contentType,
+                hash: a.hash,
+                textContent: a.textContent,
+            }));
+        }
+        return this.#archive.getAnchorsOnPage(this.#raw.id);
+    }
+    /**
+     * Reads the HTML snapshot content of this page from the archive.
+     * @returns The HTML content as a string, or null if no snapshot was saved.
+     */
+    async getHtml() {
+        return this.#archive.getHtmlOfPage(this.#raw.html);
+    }
+    /**
+     * Retrieves the referrers (incoming links) pointing to this page.
+     * Uses pre-loaded data if available, otherwise queries the database.
+     * @returns An array of {@link Referrer} objects representing pages that link to this page.
+     */
+    async getReferrers() {
+        if (this.#rawReferrers) {
+            return this.#rawReferrers.map((r) => ({
+                url: r.url,
+                through: r.through,
+                throughId: r.throughId,
+                hash: r.hash,
+                textContent: r.textContent || '',
+            }));
+        }
+        return this.#archive.getReferrersOfPage(this.#raw.id);
+    }
+    /**
+     * Retrieves all request referrers for this page directly from the database.
+     * Unlike {@link getReferrers}, this always queries the database and does not use pre-loaded data.
+     * @returns An array of {@link Referrer} objects.
+     */
+    async getRequests() {
+        return this.#archive.getReferrersOfPage(this.#raw.id);
+    }
+    /**
+     * Checks whether this page is an internal HTML page (not external and has `text/html` content type).
+     * @returns `true` if this is an internal HTML page, `false` otherwise.
+     */
+    isInternalPage() {
+        return this.isPage() && !this.isExternal;
+    }
+    /**
+     * Checks whether this entry represents an HTML page (content type is `text/html`).
+     * @returns `true` if the content type is `text/html`, `false` otherwise.
+     */
+    isPage() {
+        const type = this.contentType || '';
+        return type.toLowerCase().trim() === 'text/html';
+    }
+    /**
+     * Serializes the page data to a plain JSON object,
+     * including resolved anchors and referrers.
+     * @returns A plain object containing all page metadata and relationships.
+     */
+    async toJSON() {
+        return {
+            url: this.url.href,
+            title: this.title,
+            status: this.status,
+            statusText: this.statusText,
+            contentType: this.contentType,
+            contentLength: this.contentLength,
+            responseHeaders: this.responseHeaders,
+            isExternal: this.isExternal,
+            isSkipped: this.isSkipped,
+            skipReason: this.skipReason,
+            isTarget: this.isTarget,
+            lang: this.lang,
+            description: this.description,
+            keywords: this.keywords,
+            noindex: this.noindex,
+            nofollow: this.nofollow,
+            noarchive: this.noarchive,
+            canonical: this.canonical,
+            alternate: this.alternate,
+            twitter_card: this.twitter_card,
+            og_site_name: this.og_site_name,
+            og_url: this.og_url,
+            og_title: this.og_title,
+            og_description: this.og_description,
+            og_type: this.og_type,
+            og_image: this.og_image,
+            redirectFrom: this.redirectFrom,
+            isPage: this.isPage(),
+            isInternalPage: this.isInternalPage(),
+            getAnchors: await this.getAnchors(),
+            getReferrers: await this.getReferrers(),
+        };
+    }
+}

package/lib/archive/resource.d.ts

@@ -0,0 +1,46 @@
+import type { ArchiveAccessor } from './archive-accessor.js';
+import type { DB_Resource } from './types.js';
+/**
+ * Represents a sub-resource (CSS, JS, image, font, etc.) stored in the archive.
+ *
+ * Provides access to the resource's HTTP metadata and referrer information.
+ * Instances are created by {@link ArchiveAccessor.getResources}.
+ */
+export default class Resource {
+    #private;
+    /**
+     * The content length of the resource in bytes, or null if unknown.
+     */
+    get contentLength(): number | null;
+    /**
+     * The MIME content type of the resource (e.g., `"text/css"`, `"application/javascript"`), or null if unknown.
+     */
+    get contentType(): string | null;
+    /**
+     * Whether this resource is hosted on an external domain.
+     */
+    get isExternal(): boolean;
+    /**
+     * The HTTP response status code, or null if not yet fetched.
+     */
+    get status(): number | null;
+    /**
+     * The HTTP response status text (e.g., `"OK"`, `"Not Found"`), or null if not yet fetched.
+     */
+    get statusText(): string | null;
+    /**
+     * The URL of the resource.
+     */
+    get url(): string;
+    /**
+     * Creates a new Resource instance.
+     * @param archive - The ArchiveAccessor used for querying referrer data.
+     * @param raw - The raw database row for this resource.
+     */
+    constructor(archive: ArchiveAccessor, raw: DB_Resource);
+    /**
+     * Retrieves the page URLs that reference this resource.
+     * @returns An array of page URL strings that include or reference this resource.
+     */
+    getReferrers(): Promise<string[]>;
+}

package/lib/archive/resource.js

@@ -0,0 +1,62 @@
+/**
+ * Represents a sub-resource (CSS, JS, image, font, etc.) stored in the archive.
+ *
+ * Provides access to the resource's HTTP metadata and referrer information.
+ * Instances are created by {@link ArchiveAccessor.getResources}.
+ */
+export default class Resource {
+    #archive;
+    #raw;
+    /**
+     * The content length of the resource in bytes, or null if unknown.
+     */
+    get contentLength() {
+        return this.#raw.contentLength;
+    }
+    /**
+     * The MIME content type of the resource (e.g., `"text/css"`, `"application/javascript"`), or null if unknown.
+     */
+    get contentType() {
+        return this.#raw.contentType;
+    }
+    /**
+     * Whether this resource is hosted on an external domain.
+     */
+    get isExternal() {
+        return !!this.#raw.isExternal;
+    }
+    /**
+     * The HTTP response status code, or null if not yet fetched.
+     */
+    get status() {
+        return this.#raw.status;
+    }
+    /**
+     * The HTTP response status text (e.g., `"OK"`, `"Not Found"`), or null if not yet fetched.
+     */
+    get statusText() {
+        return this.#raw.statusText;
+    }
+    /**
+     * The URL of the resource.
+     */
+    get url() {
+        return this.#raw.url;
+    }
+    /**
+     * Creates a new Resource instance.
+     * @param archive - The ArchiveAccessor used for querying referrer data.
+     * @param raw - The raw database row for this resource.
+     */
+    constructor(archive, raw) {
+        this.#archive = archive;
+        this.#raw = raw;
+    }
+    /**
+     * Retrieves the page URLs that reference this resource.
+     * @returns An array of page URL strings that include or reference this resource.
+     */
+    async getReferrers() {
+        return this.#archive.getReferrersOfResource(this.#raw.id);
+    }
+}

package/lib/archive/safe-path.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Resolves and validates a file path to prevent path traversal attacks.
+ * Ensures the resolved path stays within the specified base directory.
+ * @param base - The base directory that all paths must stay within.
+ * @param segments - Path segments to resolve relative to the base.
+ * @returns The resolved absolute path.
+ * @throws {Error} If the resolved path escapes the base directory.
+ */
+export declare function safePath(base: string, ...segments: string[]): string;

package/lib/archive/safe-path.js

@@ -0,0 +1,17 @@
+import path from 'node:path';
+/**
+ * Resolves and validates a file path to prevent path traversal attacks.
+ * Ensures the resolved path stays within the specified base directory.
+ * @param base - The base directory that all paths must stay within.
+ * @param segments - Path segments to resolve relative to the base.
+ * @returns The resolved absolute path.
+ * @throws {Error} If the resolved path escapes the base directory.
+ */
+export function safePath(base, ...segments) {
+    const resolvedBase = path.resolve(base);
+    const resolved = path.resolve(base, ...segments);
+    if (!resolved.startsWith(resolvedBase + path.sep) && resolved !== resolvedBase) {
+        throw new Error(`Path traversal detected: ${segments.join('/')}`);
+    }
+    return resolved;
+}

package/lib/archive/types.d.ts

@@ -0,0 +1,210 @@
+import type { ParseURLOptions } from '@d-zero/shared/parse-url';
+/**
+ * Event map for database-related events emitted by the Database and ArchiveAccessor classes.
+ */
+export interface DatabaseEvent {
+    /** An error that occurred during a database operation. */
+    error: Error;
+}
+/**
+ * Configuration stored in the archive database's `info` table.
+ * Represents all crawling options that were used for the crawl session.
+ */
+export interface Config extends Required<Pick<ParseURLOptions, 'disableQueries'>> {
+    /** The starting URL for the crawl. */
+    baseUrl: string;
+    /** Maximum directory depth for excluded paths. */
+    maxExcludedDepth: number;
+    /** URL patterns defining the crawl scope. */
+    scope: string[];
+    /** Keywords used to exclude pages from crawling. */
+    excludeKeywords: string[];
+    /** URL patterns to exclude from crawling. */
+    excludes: string[];
+    /** URL prefixes to exclude from crawling. */
+    excludeUrls: string[];
+    /** Whether to fetch external (off-site) pages. */
+    fetchExternal: boolean;
+    /** Whether the crawl was initiated from a URL list rather than recursive discovery. */
+    fromList: boolean;
+    /** Whether to collect image data during crawling. */
+    image: boolean;
+    /** Interval in milliseconds between requests. */
+    interval: number;
+    /** The name identifier for this crawl session. */
+    name: string;
+    /** Number of parallel crawling processes. */
+    parallels: number;
+    /** Whether to recursively follow links. */
+    recursive: boolean;
+    /** Maximum number of retry attempts per URL on scrape failure. */
+    retry: number;
+    /** The version of Nitpicker that created this archive. */
+    version: string;
+    /** User-Agent string used for HTTP requests. */
+    userAgent: string;
+    /** Whether robots.txt restrictions were ignored during crawling. */
+    ignoreRobots: boolean;
+}
+/**
+ * Filter type for querying pages from the database.
+ *
+ * - `'page'` - HTML pages that are crawl targets
+ * - `'page-included-no-target'` - All HTML pages, including non-target pages
+ * - `'external-page'` - HTML pages on external domains
+ * - `'internal-page'` - HTML pages on the crawled domain
+ * - `'no-page'` - Non-HTML resources (e.g., images, PDFs)
+ * - `'external-no-page'` - External non-HTML resources
+ * - `'internal-no-page'` - Internal non-HTML resources
+ */
+export type PageFilter = 'page' | 'page-included-no-target' | 'external-page' | 'internal-page' | 'no-page' | 'external-no-page' | 'internal-no-page';
+/**
+ * Raw database row representing a crawled page in the `pages` table.
+ */
+export interface DB_Page {
+    /** Auto-incremented primary key. */
+    id: number;
+    /** The canonical URL of the page. */
+    url: string;
+    /** Foreign key to the redirect destination page, or null if not redirected. */
+    redirectDestId: number | null;
+    /** Whether the page has been scraped (1) or is still pending (0). */
+    scraped: 0 | 1;
+    /** Whether the page is a crawl target (1) or discovered incidentally (0). */
+    isTarget: 0 | 1;
+    /** Whether the page is on an external domain (1) or internal (0). */
+    isExternal: 0 | 1;
+    /** HTTP response status code, or null if not yet fetched. */
+    status: number | null;
+    /** HTTP response status text (e.g., "OK", "Not Found"), or null if not yet fetched. */
+    statusText: string | null;
+    /** MIME content type of the response (e.g., "text/html"), or null if unknown. */
+    contentType: string | null;
+    /** Content length in bytes, or null if unknown. */
+    contentLength: number | null;
+    /** JSON-serialized HTTP response headers. */
+    responseHeaders: string;
+    /** The `lang` attribute value from the HTML element, or null if not present. */
+    lang: string | null;
+    /** The page title from the `<title>` element, or null if not present. */
+    title: string | null;
+    /** The meta description content, or null if not present. */
+    description: string | null;
+    /** The meta keywords content, or null if not present. */
+    keywords: string | null;
+    /** Whether the noindex robots directive is set (SQLite INTEGER 0/1). */
+    noindex: number | null;
+    /** Whether the nofollow robots directive is set (SQLite INTEGER 0/1). */
+    nofollow: number | null;
+    /** Whether the noarchive robots directive is set (SQLite INTEGER 0/1). */
+    noarchive: number | null;
+    /** The canonical URL from `<link rel="canonical">`, or null if not present. */
+    canonical: string | null;
+    /** The alternate URL from `<link rel="alternate">`, or null if not present. */
+    alternate: string | null;
+    /** The Open Graph type (`og:type`), or null if not present. */
+    og_type: string | null;
+    /** The Open Graph title (`og:title`), or null if not present. */
+    og_title: string | null;
+    /** The Open Graph site name (`og:site_name`), or null if not present. */
+    og_site_name: string | null;
+    /** The Open Graph description (`og:description`), or null if not present. */
+    og_description: string | null;
+    /** The Open Graph URL (`og:url`), or null if not present. */
+    og_url: string | null;
+    /** The Open Graph image URL (`og:image`), or null if not present. */
+    og_image: string | null;
+    /** The Twitter Card type (`twitter:card`), or null if not present. */
+    twitter_card: string | null;
+    /** JSON-serialized network logs captured during scraping, or null if not collected. */
+    networkLogs: string | null;
+    /** Relative file path to the saved HTML snapshot, or null if not saved. */
+    html: string | null;
+    /** Whether the page was skipped during crawling (1) or processed normally (0). */
+    isSkipped: 0 | 1;
+    /** The reason the page was skipped, or null if it was not skipped. */
+    skipReason: string | null;
+    /** The natural URL sort order index, or null if not yet assigned. */
+    order: number | null;
+}
+/**
+ * Raw database row representing a redirect relationship.
+ * Maps a source page to its redirect destination.
+ */
+export interface DB_Redirect {
+    /** The ID of the destination page after redirect. */
+    pageId: number;
+    /** The URL that was redirected from. */
+    from: string;
+    /** The page ID of the source URL that was redirected. */
+    fromId: number;
+}
+/**
+ * Raw database row representing an anchor (link) found on a page.
+ * Combines data from the `anchors` table and the linked `pages` table.
+ */
+export interface DB_Anchor {
+    /** The ID of the page that contains this anchor. */
+    pageId: number;
+    /** The resolved destination URL of the anchor. */
+    url: string;
+    /** The original href attribute value of the anchor element. */
+    href: string;
+    /** Whether the anchor points to an external domain (1) or internal (0). */
+    isExternal: 0 | 1;
+    /** The title attribute of the anchor element, or null if not present. */
+    title: string | null;
+    /** The HTTP status code of the linked page, or null if not yet fetched. */
+    status: number | null;
+    /** The HTTP status text of the linked page, or null if not yet fetched. */
+    statusText: string | null;
+    /** The content type of the linked page, or null if not yet fetched. */
+    contentType: string | null;
+    /** The URL fragment (hash) portion of the link, or null if not present. */
+    hash: string | null;
+    /** The text content of the anchor element, or null if empty. */
+    textContent: string | null;
+}
+/**
+ * Raw database row representing a referrer relationship.
+ * Indicates which page links to which other page, potentially through redirects.
+ */
+export interface DB_Referrer {
+    /** The ID of the page being referred to. */
+    pageId: number;
+    /** The URL of the referring page. */
+    url: string;
+    /** The URL through which the referral passes (may differ from url due to redirects). */
+    through: string;
+    /** The page ID of the through URL. */
+    throughId: number;
+    /** The URL fragment (hash) of the referring link, or null if not present. */
+    hash: string | null;
+    /** The text content of the referring anchor element, or null if empty. */
+    textContent: string | null;
+}
+/**
+ * Raw database row representing a sub-resource (CSS, JS, image, etc.) in the `resources` table.
+ */
+export interface DB_Resource {
+    /** Auto-incremented primary key. */
+    id: number;
+    /** The URL of the resource. */
+    url: string;
+    /** Whether the resource is hosted on an external domain (1) or internal (0). */
+    isExternal: 0 | 1;
+    /** HTTP response status code, or null if not yet fetched. */
+    status: number | null;
+    /** HTTP response status text, or null if not yet fetched. */
+    statusText: string | null;
+    /** MIME content type of the resource, or null if unknown. */
+    contentType: string | null;
+    /** Content length in bytes, or null if unknown. */
+    contentLength: number | null;
+    /** Compression encoding (e.g., "gzip", "br"), or 0 if not compressed. */
+    compress: string | 0;
+    /** CDN provider identifier, or 0 if not served from a CDN. */
+    cdn: string | 0;
+    /** JSON-serialized HTTP response headers, or null if not available. */
+    responseHeaders: string | null;
+}

package/lib/archive/types.js

@@ -0,0 +1 @@
+export {};

package/lib/crawler/clear-destination-cache.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Clears the in-memory cache of HTTP request results.
+ * Should be called between crawl sessions to prevent memory leaks.
+ */
+export declare function clearDestinationCache(): void;

package/lib/crawler/clear-destination-cache.js

@@ -0,0 +1,8 @@
+import { destinationCache } from './destination-cache.js';
+/**
+ * Clears the in-memory cache of HTTP request results.
+ * Should be called between crawl sessions to prevent memory leaks.
+ */
+export function clearDestinationCache() {
+    destinationCache.clear();
+}