@nitpicker/crawler 0.9.0 → 0.12.0

package/lib/archive/archive-accessor.d.ts

@@ -4,15 +4,13 @@ import type { ParseURLOptions } from '@d-zero/shared/parse-url';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import Page from './page.js';
 import Resource from './resource.js';
-/**
- * Provides read-only access to an archive's database and stored data files.
- *
- * This class is the base for the `Archive` class and is also returned
- * by `Archive.connect` for read-only access to an existing archive.
- * It supports querying pages, anchors, referrers, resources, and custom data.
- */
 export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
     #private;
+    /**
+     * Whether this accessor was opened in read-only mode (no filesystem
+     * mutation on `tmpDir`).
+     */
+    get readOnly(): boolean;
     /**
      * The absolute path to the temporary working directory used by this accessor.
      */
@@ -23,8 +21,47 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * @param db - The Database instance for querying the SQLite database.
      * @param namespace - An optional namespace for scoping custom data storage.
      *                    When null, `setData` is not available.
+     * @param options - Construction options.
+     * @param options.readOnly - When `true`, helpers must not mutate the
+     *   filesystem under `tmpDir` (used for live-crawl / stub-mode opens
+     *   where any write would race the crawler).
      */
-    constructor(tmpDir: string, db: Database, namespace?: string | null);
+    constructor(tmpDir: string, db: Database, namespace?: string | null, options?: {
+        readOnly?: boolean;
+    });
+    /**
+     * Closes the underlying database connection.
+     *
+     * This is the **read-only** close path: it releases the SQLite handle and
+     * does nothing else. The temporary working directory is left untouched and
+     * no `.nitpicker` archive is produced. This makes it safe to call from
+     * read-only consumers (e.g. the viewer attached to an in-progress crawl's
+     * tmpDir), where touching the filesystem would race with — or destroy —
+     * the live crawler's working state.
+     *
+     * Subclasses that own the archive's lifecycle (notably {@link Archive})
+     * override this to add write/cleanup steps.
+     *
+     * **Idempotent and concurrent-safe**: the first invocation captures the
+     * close promise; later invocations (from the same caller, a shutdown
+     * signal handler, or a parallel manager teardown) await the same
+     * promise and resolve together. If `db.destroy()` rejects, the
+     * rejection propagates to *all* awaiters and the accessor stays
+     * latched closed — a hung knex pool is not safe to "retry close".
+     *
+     * **Bounded**: when the optional `timeoutMs` (default {@link
+     * DEFAULT_CLOSE_TIMEOUT_MS}) elapses before `db.destroy()` settles, the
+     * call resolves with a warning. This prevents a viewer shutdown from
+     * being held for the underlying pool's 10-minute `acquireTimeoutMillis`
+     * when the live crawler holds the SQLite write lock.
+     * @param options - Close options.
+     * @param options.timeoutMs - Milliseconds to wait for `db.destroy()`
+     *   before giving up. Use `Infinity` to wait indefinitely (only
+     *   advisable in tests and batch jobs that own the DB exclusively).
+     */
+    close(options?: {
+        timeoutMs?: number;
+    }): Promise<void>;
     /**
      * Retrieves anchor (link) data for a specific page by its database ID.
      * @param pageId - The database ID of the page whose anchors to retrieve.
@@ -51,13 +88,42 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      */
     getData(name: string, format?: 'txt' | 'html'): Promise<string>;
     /**
-     * Reads the HTML content of a page snapshot from the archive.
-     * Supports reading from both unzipped directories and zipped snapshot archives.
-     * @param filePath - The relative file path to the HTML snapshot, or null.
-     * @param openZipped - Whether to attempt unzipping the snapshot archive. Defaults to `true`.
-     * @returns The HTML content as a string, or null if the snapshot is not found or filePath is null.
+     * Reads the HTML snapshot of a page from the archive.
+     *
+     * HTML is stored as zstd-compressed BLOBs in `page_html_blobs` (keyed by
+     * SHA-256 of the raw bytes) with `page_html_ref` linking `page_id → hash`.
+     * The read is a straight join + decompress; writer- and read-only (stub)
+     * accessors take the same code path because nothing here touches the
+     * filesystem.
+     *
+     * Returns `null` when the page row exists but has no stored body — for
+     * example a redirect source, a non-HTML resource (PDF), a page that
+     * failed to render, or an external page (whose row is metadata-only).
+     * Distinguishing "no body stored" from "empty body" is preserved: an
+     * empty HTML string returns `''`, not `null`.
+     *
+     * Throws if the page's referenced blob is missing or the codec marker
+     * is unrecognised — both indicate an archive that was truncated or
+     * written by a future tool, neither of which we silently paper over.
+     * @param pageId - The database id of the page to read.
+     * @returns The HTML content as a UTF-8 string, or `null` when no body
+     *   is stored for `pageId`.
+     * @example
+     * const html = await accessor.getHtmlOfPage(pageId);
+     * if (html === null) {
+     *   // page has no stored body — redirect source / non-HTML / failed render
+     * } else {
+     *   processHtml(html);
+     * }
      */
-    getHtmlOfPage(filePath: string | null, openZipped?: boolean): Promise<string | null>;
+    getHtmlOfPage(pageId: number): Promise<string | null>;
+    /**
+     * Retrieves the JSON-LD / SpeculationRules entries for the given page,
+     * parsed back from the `page_jsonld` table.
+     * @param pageId - The database id of the page.
+     * @returns Ordered entries with `kind`, `type`, `raw`, `parsed`, `parseError`.
+     */
+    getJsonLdOfPage(pageId: number): Promise<import("./meta/types.js").JsonLdRow[]>;
     /**
      * Returns the underlying Knex query builder instance for direct SQL access.
      * Enables advanced queries (GROUP BY, HAVING, JOINs) at the database layer
@@ -106,6 +172,13 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * @returns An array of resource URL strings.
      */
     getResourceUrlList(): Promise<any[]>;
+    /**
+     * Retrieves the Wappalyzer tag entries for the given page, parsed back
+     * from the `page_tags` table.
+     * @param pageId - The database id of the page.
+     * @returns Ordered entries with provider, category, externalId, etc.
+     */
+    getTagsOfPage(pageId: number): Promise<import("./meta/types.js").TagRow[]>;
     /**
      * Stores custom data in the archive under the configured namespace.
      * Requires a namespace to be set on this accessor; throws if namespace is null.

package/lib/archive/archive-accessor.js

@@ -1,8 +1,6 @@
 import path from 'node:path';
-import { extractZip, unzip } from '@d-zero/fs/zip';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import { log } from './debug.js';
-import { exists } from './filesystem/exists.js';
 import { outputJSON } from './filesystem/output-json.js';
 import { outputText } from './filesystem/output-text.js';
 import { readJSON } from './filesystem/read-json.js';
@@ -17,13 +15,45 @@ import { safePath } from './safe-path.js';
  * by `Archive.connect` for read-only access to an existing archive.
  * It supports querying pages, anchors, referrers, resources, and custom data.
  */
+/**
+ * Default timeout for {@link ArchiveAccessor.close}'s `db.destroy()` step.
+ *
+ * `knex.destroy()` will otherwise wait the full `acquireTimeoutMillis`
+ * (10 minutes in this repo) for in-flight queries to drain. For a viewer
+ * shut down by Ctrl-C while the live crawler holds a long write lock that
+ * is an unacceptable user experience, so we bound the wait and treat the
+ * accessor as closed after the timeout regardless.
+ */
+const DEFAULT_CLOSE_TIMEOUT_MS = 5000;
 export class ArchiveAccessor extends EventEmitter {
+    /**
+     * Promise tracking an in-progress (or completed) close. `null` means the
+     * accessor is open and idle; a settled promise means we are closed (the
+     * accessor stays "closed" even if `db.destroy()` rejected, because there
+     * is nothing safe to retry — see {@link close}).
+     */
+    #closeOnce = null;
     /** The SQLite database instance for querying archived data. */
     #db;
     /** Namespace prefix for custom data storage (e.g. `"analysis/plugin-name"`). `null` disables `setData`. */
     #namespace = null;
+    /**
+     * Whether this accessor was opened in read-only mode. With HTML stored
+     * as a SQLite BLOB, this no longer toggles any code path — the SELECT
+     * is identical for writer- and reader-mode accessors. Kept on the
+     * accessor so callers like the viewer can still surface "this archive
+     * is being read read-only" UI hints without re-deriving it.
+     */
+    #readOnly;
     /** Absolute path to the temporary working directory containing the database and files. */
     #tmpDir;
+    /**
+     * Whether this accessor was opened in read-only mode (no filesystem
+     * mutation on `tmpDir`).
+     */
+    get readOnly() {
+        return this.#readOnly;
+    }
     /**
      * The absolute path to the temporary working directory used by this accessor.
      */
@@ -36,16 +66,59 @@ export class ArchiveAccessor extends EventEmitter {
      * @param db - The Database instance for querying the SQLite database.
      * @param namespace - An optional namespace for scoping custom data storage.
      *                    When null, `setData` is not available.
+     * @param options - Construction options.
+     * @param options.readOnly - When `true`, helpers must not mutate the
+     *   filesystem under `tmpDir` (used for live-crawl / stub-mode opens
+     *   where any write would race the crawler).
      */
-    constructor(tmpDir, db, namespace = null) {
+    constructor(tmpDir, db, namespace = null, options = {}) {
         super();
         this.#tmpDir = tmpDir;
         this.#db = db;
         this.#namespace = namespace;
+        this.#readOnly = options.readOnly ?? false;
         this.#db.on('error', (e) => {
             void this.emit('error', e);
         });
     }
+    /**
+     * Closes the underlying database connection.
+     *
+     * This is the **read-only** close path: it releases the SQLite handle and
+     * does nothing else. The temporary working directory is left untouched and
+     * no `.nitpicker` archive is produced. This makes it safe to call from
+     * read-only consumers (e.g. the viewer attached to an in-progress crawl's
+     * tmpDir), where touching the filesystem would race with — or destroy —
+     * the live crawler's working state.
+     *
+     * Subclasses that own the archive's lifecycle (notably {@link Archive})
+     * override this to add write/cleanup steps.
+     *
+     * **Idempotent and concurrent-safe**: the first invocation captures the
+     * close promise; later invocations (from the same caller, a shutdown
+     * signal handler, or a parallel manager teardown) await the same
+     * promise and resolve together. If `db.destroy()` rejects, the
+     * rejection propagates to *all* awaiters and the accessor stays
+     * latched closed — a hung knex pool is not safe to "retry close".
+     *
+     * **Bounded**: when the optional `timeoutMs` (default {@link
+     * DEFAULT_CLOSE_TIMEOUT_MS}) elapses before `db.destroy()` settles, the
+     * call resolves with a warning. This prevents a viewer shutdown from
+     * being held for the underlying pool's 10-minute `acquireTimeoutMillis`
+     * when the live crawler holds the SQLite write lock.
+     * @param options - Close options.
+     * @param options.timeoutMs - Milliseconds to wait for `db.destroy()`
+     *   before giving up. Use `Infinity` to wait indefinitely (only
+     *   advisable in tests and batch jobs that own the DB exclusively).
+     */
+    async close(options = {}) {
+        if (this.#closeOnce) {
+            return this.#closeOnce;
+        }
+        const timeoutMs = options.timeoutMs ?? DEFAULT_CLOSE_TIMEOUT_MS;
+        this.#closeOnce = this.#runClose(timeoutMs);
+        return this.#closeOnce;
+    }
     /**
      * Retrieves anchor (link) data for a specific page by its database ID.
      * @param pageId - The database ID of the page whose anchors to retrieve.
@@ -71,43 +144,50 @@ export class ArchiveAccessor extends EventEmitter {
         return await readText(filePath);
     }
     /**
-     * Reads the HTML content of a page snapshot from the archive.
-     * Supports reading from both unzipped directories and zipped snapshot archives.
-     * @param filePath - The relative file path to the HTML snapshot, or null.
-     * @param openZipped - Whether to attempt unzipping the snapshot archive. Defaults to `true`.
-     * @returns The HTML content as a string, or null if the snapshot is not found or filePath is null.
+     * Reads the HTML snapshot of a page from the archive.
+     *
+     * HTML is stored as zstd-compressed BLOBs in `page_html_blobs` (keyed by
+     * SHA-256 of the raw bytes) with `page_html_ref` linking `page_id → hash`.
+     * The read is a straight join + decompress; writer- and read-only (stub)
+     * accessors take the same code path because nothing here touches the
+     * filesystem.
+     *
+     * Returns `null` when the page row exists but has no stored body — for
+     * example a redirect source, a non-HTML resource (PDF), a page that
+     * failed to render, or an external page (whose row is metadata-only).
+     * Distinguishing "no body stored" from "empty body" is preserved: an
+     * empty HTML string returns `''`, not `null`.
+     *
+     * Throws if the page's referenced blob is missing or the codec marker
+     * is unrecognised — both indicate an archive that was truncated or
+     * written by a future tool, neither of which we silently paper over.
+     * @param pageId - The database id of the page to read.
+     * @returns The HTML content as a UTF-8 string, or `null` when no body
+     *   is stored for `pageId`.
+     * @example
+     * const html = await accessor.getHtmlOfPage(pageId);
+     * if (html === null) {
+     *   // page has no stored body — redirect source / non-HTML / failed render
+     * } else {
+     *   processHtml(html);
+     * }
      */
-    async getHtmlOfPage(filePath, openZipped = true) {
-        if (!filePath) {
-            return null;
-        }
-        const snapshotDir = safePath(this.#tmpDir, path.dirname(filePath));
-        const name = path.basename(filePath);
-        if (openZipped) {
-            await unzip(`${snapshotDir}.zip`, snapshotDir);
-        }
-        if (exists(snapshotDir)) {
-            log('Load %s directly because snapshot dir is unzipped', name);
-            const html = await readText(path.resolve(snapshotDir, name)).catch((error) => error);
-            if (typeof html === 'string') {
-                log('Loaded: %s ...', html.split('\n')[0]);
-                return html;
-            }
-            log('Failed Loading: %O', html);
-            return null;
+    async getHtmlOfPage(pageId) {
+        const html = await this.#db.getHtmlOfPageById(pageId);
+        if (html === null) {
+            log('No HTML body stored for page id=%d', pageId);
         }
-        log('Extracts %s from zipped snapshots', name);
-        const zipDir = await extractZip(`${snapshotDir}.zip`);
-        const file = zipDir.files.find((f) => f.type === 'File' && f.path === name);
-        if (!file) {
-            log('Failed: Not found %s from zipped snapshots', name);
-            return null;
-        }
-        const buffer = await file.buffer();
-        const html = buffer.toString('utf8') || null;
-        log('Succeeded: Extracts %s from zipped snapshots', name);
         return html;
     }
+    /**
+     * Retrieves the JSON-LD / SpeculationRules entries for the given page,
+     * parsed back from the `page_jsonld` table.
+     * @param pageId - The database id of the page.
+     * @returns Ordered entries with `kind`, `type`, `raw`, `parsed`, `parseError`.
+     */
+    async getJsonLdOfPage(pageId) {
+        return this.#db.getJsonLdOfPage(pageId);
+    }
     /**
      * Returns the underlying Knex query builder instance for direct SQL access.
      * Enables advanced queries (GROUP BY, HAVING, JOINs) at the database layer
@@ -200,6 +280,15 @@ export class ArchiveAccessor extends EventEmitter {
     async getResourceUrlList() {
         return this.#db.getResourceUrlList();
     }
+    /**
+     * Retrieves the Wappalyzer tag entries for the given page, parsed back
+     * from the `page_tags` table.
+     * @param pageId - The database id of the page.
+     * @returns Ordered entries with provider, category, externalId, etc.
+     */
+    async getTagsOfPage(pageId) {
+        return this.#db.getTagsOfPage(pageId);
+    }
     /**
      * Stores custom data in the archive under the configured namespace.
      * Requires a namespace to be set on this accessor; throws if namespace is null.
@@ -282,4 +371,41 @@ export class ArchiveAccessor extends EventEmitter {
         log('Create Page Data: Done');
         return pPages;
     }
+    /**
+     * Actual close worker — invoked exactly once per accessor via
+     * {@link close}'s shared promise. Races `db.destroy()` against the
+     * caller-supplied timeout; on timeout we log and resolve so the
+     * consumer (typically a process shutting down) is not blocked, even
+     * though the underlying knex pool may still be draining in the
+     * background.
+     * @param timeoutMs - Maximum time to wait for `db.destroy()`.
+     */
+    async #runClose(timeoutMs) {
+        if (!Number.isFinite(timeoutMs)) {
+            await this.#db.destroy();
+            return;
+        }
+        let timer = null;
+        const timeout = new Promise((resolve) => {
+            timer = setTimeout(() => resolve('timeout'), timeoutMs);
+        });
+        // Track destroy() so we can attach an error-suppressing handler if we
+        // give up waiting — otherwise a late rejection becomes an unhandled
+        // promise rejection on the process.
+        const destroy = this.#db.destroy().then(() => 'done');
+        try {
+            const result = await Promise.race([destroy, timeout]);
+            if (result === 'timeout') {
+                log('ArchiveAccessor.close: db.destroy() did not settle within %dms — giving up', timeoutMs);
+                destroy.catch((error) => {
+                    log('ArchiveAccessor.close: late db.destroy() rejection (post-timeout): %O', error);
+                });
+            }
+        }
+        finally {
+            if (timer) {
+                clearTimeout(timer);
+            }
+        }
+    }
 }