@nitpicker/crawler 0.8.0 → 0.11.0

package/lib/archive/archive.js

@@ -1,5 +1,4 @@
 import path from 'node:path';
-import { zip } from '@d-zero/fs/zip';
 import { ArchiveAccessor } from './archive-accessor.js';
 import { acquireArchiveLock } from './archive-lock.js';
 import { Database } from './database.js';
@@ -7,31 +6,40 @@ import { dbLog, log, saveLog } from './debug.js';
 import { appendText } from './filesystem/append-text.js';
 import { exists } from './filesystem/exists.js';
 import { isDir } from './filesystem/is-dir.js';
-import { outputText } from './filesystem/output-text.js';
+import { peekTarTopDir } from './filesystem/peek-tar-top-dir.js';
 import { remove } from './filesystem/remove.js';
 import { rename } from './filesystem/rename.js';
 import { tar } from './filesystem/tar.js';
 import { untar } from './filesystem/untar.js';
 /**
- * Main archive class for creating, opening, resuming, and writing Nitpicker archive files (`.nitpicker`).
+ * Main archive class for creating, opening, resuming, and writing Nitpicker
+ * archive files (`.nitpicker`).
  *
- * An Archive wraps a SQLite database and optional HTML snapshots into a compressed
- * tar archive. It extends {@link ArchiveAccessor} to provide read access to stored data.
+ * An Archive wraps a single SQLite database into a tar archive. HTML
+ * bodies live inside the same DB as zstd-compressed BLOBs (see #75) — the
+ * tar payload is effectively just `db.sqlite`. It extends
+ * {@link ArchiveAccessor} to provide read access to stored data.
  *
  * Use the static factory methods ({@link Archive.create}, {@link Archive.open},
  * {@link Archive.resume}, {@link Archive.connect}) to obtain instances.
  * The constructor is private.
  */
 export default class Archive extends ArchiveAccessor {
+    /**
+     * Promise tracking an in-progress {@link Archive.close} (or
+     * {@link Archive.releaseHandle}). Acts as the override's idempotency
+     * guard so a second call — e.g. from a signal handler racing the
+     * primary teardown — does not re-enter the destructive prologue
+     * (write/remove) on a half-mutated state.
+     */
+    #closeOnce = null;
     /** The SQLite database instance for reading and writing crawl data. */
     #db;
     /** Absolute path to the output `.nitpicker` archive file. */
     #filePath;
     /** Lock release function held while the writer owns the archive. */
     #releaseLock;
-    /** Absolute path to the HTML snapshot directory within the temporary working directory. */
-    #snapshotDir;
-    /** Absolute path to the temporary working directory containing the SQLite DB and snapshots. */
+    /** Absolute path to the temporary working directory containing the SQLite DB. */
     #tmpDir;
     /**
      * The absolute file path of the archive (`.nitpicker` file).
@@ -39,18 +47,26 @@ export default class Archive extends ArchiveAccessor {
     get filePath() {
         return this.#filePath;
     }
+    /**
+     * The intermediate directory `Archive.write()` produces by renaming
+     * `tmpDir` before tarring (`{cwd}/{archiveName}`). Exposed so the
+     * manager can include it in its cleanup-on-failure path: if `tar()`
+     * fails after the rename, this directory is orphaned and would
+     * otherwise be invisible to a `rmSync(tmpDir)` recovery.
+     */
+    get renamedDir() {
+        return path.resolve(path.dirname(this.#filePath), path.basename(this.#filePath, path.extname(this.#filePath)));
+    }
     // eslint-disable-next-line no-restricted-syntax
     constructor(filePath, tmpDir, db, releaseLock) {
         super(tmpDir, db, '');
         this.#filePath = filePath;
         this.#tmpDir = tmpDir;
-        this.#snapshotDir = path.resolve(this.#tmpDir, Archive.SNAPSHOT_HTML_DIR);
         this.#db = db;
         this.#releaseLock = releaseLock;
         log('create instance: %O', {
             filePath,
             tmpDir,
-            snapshotDir: this.#snapshotDir,
         });
         this.#db.on('error', (e) => {
             void this.emit('error', e);
@@ -61,35 +77,34 @@ export default class Archive extends ArchiveAccessor {
      */
     abort() { }
     /**
-     * Appends an error entry to the archive's error log file.
+     * Records a crawler-level error to both the human-readable `error.log` (full
+     * stack, for debugging) and the structured `crawl_errors` table (queryable,
+     * for the `error-kinds` analysis). The cause is not classified here — it is
+     * derived on read. `error.log` keeps the full stack while `crawl_errors`
+     * stores `error.message`; both normally carry the same cause token (e.g.
+     * `ENOTFOUND`), so classification agrees across the two — only an error whose
+     * cause lives solely in deeper stack frames could differ.
      * @param error - The crawler error object containing process and URL information.
      */
     async addError(error) {
         const logFile = path.resolve(this.#tmpDir, 'error.log');
         await appendText(logFile, `[${error.pid}(${error.isMainProcess ? 'main' : 'sub'})] ${error.url} ${error.error.stack ?? error.error}`);
+        await this.#db.insertCrawlError(error.url, error.error.message, error.isExternal);
     }
     /**
-     * Closes the archive. If the archive file does not yet exist on disk,
-     * it writes the archive first. If the temporary directory still exists,
-     * it is removed.
+     * Records a partial scrape failure against the page identified by `url`.
+     *
+     * The corresponding `pages` row is created on demand (or matched if it
+     * already exists), so the call works even if the page's normal data has
+     * not been written yet.
+     * @param url - URL of the affected page.
+     * @param phase - Scrape phase name (typically `'retryExhausted'`).
+     * @param message - Human-readable failure message.
+     * @param isExternal - Whether the URL is external. Defaults to `false`.
      */
-    async close() {
-        log('Closing');
-        try {
-            if (!exists(this.#filePath)) {
-                log("Save the file because it doesn't exist");
-                await this.write();
-            }
-            else if (exists(this.#tmpDir)) {
-                log('Remove temporary dir');
-                await remove(this.#tmpDir);
-            }
-            await this.#db.destroy();
-        }
-        finally {
-            await this.#releaseLock();
-        }
-        log('Closing done');
+    async addPageError(url, phase, message, isExternal = false) {
+        dbLog('Add page error: %s [%s]', url, phase);
+        await this.#db.insertPageError(url, phase, message, isExternal);
     }
     /**
      * Retrieves the current crawling state, including lists of scraped and pending URLs.
@@ -98,6 +113,47 @@ export default class Archive extends ArchiveAccessor {
     async getCrawlingState() {
         return this.#db.getCrawlingState();
     }
+    /**
+     * Return the subset of `urls` that already exist as `pages.url`. Used by
+     * `CrawlerOrchestrator.inventory` to filter the user-supplied URL list
+     * down to "URLs that are NOT yet in the archive" — only those reach the
+     * HEAD / scrape pipeline. Existing URLs are skipped to keep the second
+     * (and N-th) `--inventory` pass non-destructive.
+     * @param urls - Candidate URLs in `withoutHashAndAuth` form.
+     * @returns URLs already present in `pages`.
+     */
+    async getExistingPageUrls(urls) {
+        return this.#db.getExistingPageUrls(urls);
+    }
+    /**
+     * Return the subset of `urls` that already exist as `resources.url`. See
+     * {@link Archive.getExistingPageUrls} — the resource-side counterpart used
+     * by inventory mode to skip URLs that are already tracked as
+     * sub-resources.
+     * @param urls - Candidate URLs.
+     * @returns URLs already present in `resources`.
+     */
+    async getExistingResourceUrls(urls) {
+        return this.#db.getExistingResourceUrls(urls);
+    }
+    /**
+     * Retrieves a single recorded sub-resource by its URL.
+     * @param urls - URL candidates to match against the stored resource URL.
+     * @returns The raw resource row, or `null` if none match.
+     */
+    async getResourceByUrl(urls) {
+        return this.#db.getResourceByUrl(urls);
+    }
+    /**
+     * Counts the number of pages already scraped as crawl targets in the archive.
+     *
+     * Lets the crawler initialize its session-progress counter on resume so the
+     * displayed HTML-page count accounts for previously-rendered pages.
+     * @returns The count of pages with `isTarget = 1` and `scraped = 1`.
+     */
+    async getScrapedHtmlPageCount() {
+        return this.#db.getScrapedHtmlPageCount();
+    }
     /**
      * Retrieves the base URL of the crawl session from the archive database.
      * @returns The base URL string.
@@ -105,6 +161,24 @@ export default class Archive extends ArchiveAccessor {
     async getUrl() {
         return this.#db.getBaseUrl();
     }
+    /**
+     * Releases the SQLite handle and the advisory lock **without** writing
+     * the archive or removing `tmpDir`.
+     *
+     * Use this when you need to detach from a freshly-created `Archive`
+     * without finalising it — fixtures producing a stub state for tests,
+     * tooling that wants to leave the tmpDir alive for `crawl --resume`,
+     * or any non-orchestrator caller that owns the lifecycle externally.
+     * Shares the same idempotency guard as {@link close}, so the two paths
+     * are mutually exclusive (the first one called wins).
+     */
+    async releaseHandle() {
+        if (this.#closeOnce) {
+            return this.#closeOnce;
+        }
+        this.#closeOnce = this.#runReleaseHandle();
+        return this.#closeOnce;
+    }
     /**
      * Promote previously-external pages that now fall under the (possibly extended)
      * scope back to a pending state so that the crawler re-scrapes them as fully
@@ -117,6 +191,17 @@ export default class Archive extends ArchiveAccessor {
         dbLog('Repromote external pages with %d hostnames in scope', scopes.size);
         return this.#db.repromoteExternalPages(scopes, options);
     }
+    /**
+     * Reset previously-failed pages back to pending so a follow-up crawl re-fetches them.
+     *
+     * Delegates to {@link Database.resetFailedPages}. See that method for the
+     * exact failure criteria (missing status / content type, or a 5xx status).
+     * @returns The URLs of the pages that were reset to pending.
+     */
+    async resetFailedPages() {
+        dbLog('Reset failed pages back to pending');
+        return this.#db.resetFailedPages();
+    }
     /**
      * Stores the crawl configuration into the archive database.
      * @param config - The configuration object to store.
@@ -126,48 +211,50 @@ export default class Archive extends ArchiveAccessor {
         return this.#db.setConfig(config);
     }
     /**
-     * Stores an external page's data in the archive database without saving a snapshot.
+     * Stores an external page's data in the archive database without storing
+     * an HTML snapshot. External-page rows carry only metadata (status, title,
+     * content-type), never a rendered body.
      * @param pageInfo - The page data to store.
+     * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
      */
-    async setExternalPage(pageInfo) {
+    async setExternalPage(pageInfo, source) {
         dbLog('Set external page: %s', pageInfo.url.href);
-        await this.#db.updatePage(pageInfo, null, false);
+        await this.#db.updatePage(pageInfo, false, false, source);
     }
     /**
-     * Stores a crawled page's data in the archive database and optionally saves an HTML snapshot.
-     * If the snapshot file write fails, the HTML path in the database is cleared to prevent
-     * referencing a non-existent file, and the error is re-thrown.
+     * Stores a crawled page's data in the archive database, persisting the
+     * rendered HTML body as a zstd-compressed BLOB inside the same SQLite
+     * transaction. Storage is content-addressable: identical bodies across
+     * pages share a single `page_html_blobs` row.
      * @param pageInfo - The page data to store.
+     * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
      * @returns The database ID of the stored page.
-     * @throws {Error} Re-throws any error from the snapshot file write after clearing the HTML path.
      */
-    async setPage(pageInfo) {
+    async setPage(pageInfo, source) {
         dbLog('Set page: %s', pageInfo.url.href);
-        const { html, pageId } = await this.#db.updatePage(pageInfo, this.#snapshotDir, pageInfo.isTarget);
-        if (html) {
-            try {
-                await outputText(html, pageInfo.html);
-            }
-            catch (error) {
-                dbLog('Snapshot write failed for page %d, clearing html path: %s', pageId, html);
-                try {
-                    await this.#db.clearHtmlPath(pageId);
-                }
-                catch (clearError) {
-                    dbLog('Failed to clear html path for page %d: %s', pageId, clearError);
-                }
-                throw error;
-            }
-        }
-        return pageId;
+        return await this.#db.updatePage(pageInfo, true, pageInfo.isTarget, source);
+    }
+    /**
+     * Records a redirect edge without re-storing the destination's content.
+     *
+     * The crawler calls this (instead of {@link setPage}) when a URL redirects to
+     * a destination that has already been rendered (#73): only the source →
+     * destination edge is written, leaving the destination's stored title / meta /
+     * anchors / images untouched.
+     * @param pageInfo - The HEAD-resolved page data carrying the redirect chain.
+     */
+    async setRedirect(pageInfo) {
+        dbLog('Set redirect: %s', pageInfo.url.href);
+        await this.#db.recordRedirect(pageInfo);
     }
     /**
      * Stores a sub-resource (CSS, JS, image, etc.) in the archive database.
      * @param resource - The resource data to store.
+     * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
      */
-    async setResources(resource) {
+    async setResources(resource, source) {
         dbLog('Set resource: %s', resource.url.href);
-        await this.#db.insertResource(resource);
+        await this.#db.insertResource(resource, source);
     }
     /**
      * Stores the referrer relationship between a resource and the page that references it.
@@ -207,52 +294,90 @@ export default class Archive extends ArchiveAccessor {
         await this.#db.updateConfig(patch);
     }
     /**
-     * Writes the archive to disk as a compressed `.nitpicker` file.
+     * Writes the archive to disk as a `.nitpicker` tar file.
      *
-     * This method compresses the HTML snapshot directory into a zip file,
-     * renames the temporary working directory, and creates the final tar archive.
-     * The temporary directory is removed after writing.
+     * Checkpoints the SQLite WAL so the database is self-contained inside
+     * `db.sqlite`, renames the temporary working directory to the archive's
+     * basename, and tars it into the final `.nitpicker`. The tar container
+     * holds a single `db.sqlite` file (the legacy `snapshot-html.zip` is gone
+     * — HTML lives as BLOBs in the DB), so finalisation is effectively a
+     * single-file copy with no per-snapshot syscalls.
      */
     async write() {
         saveLog('Starts: %s', this.#filePath);
-        const snapshotZip = `${this.#snapshotDir}.zip`;
-        if (exists(this.#snapshotDir)) {
-            if (!exists(snapshotZip)) {
-                saveLog('Zips snapshot dir: %s', this.#snapshotDir);
-                await zip(snapshotZip, this.#snapshotDir);
-            }
-            saveLog('Remove snapshot dir: %s', this.#snapshotDir);
-            await remove(this.#snapshotDir);
-        }
         await this.#db.checkpoint();
         const filePathWithoutExt = path.resolve(path.dirname(this.#filePath), path.basename(this.#filePath, path.extname(this.#filePath)));
         saveLog('Rename temporary dir: %s to %s', this.#tmpDir, filePathWithoutExt);
         await rename(this.#tmpDir, filePathWithoutExt, true);
-        saveLog('Zip temporary dir to file: %s to %s', filePathWithoutExt, this.#filePath);
+        saveLog('Tar temporary dir to file: %s to %s', filePathWithoutExt, this.#filePath);
         await tar(filePathWithoutExt, this.#filePath);
         saveLog('Remove temporary dir: %s', filePathWithoutExt);
         await remove(filePathWithoutExt);
         saveLog('Done: %s', this.#filePath);
     }
+    /**
+     * Worker for {@link close}. Performs the destructive prologue
+     * (write or remove), drops the DB handle via the base class, then
+     * releases the lock in a `finally` so the lock never leaks even on
+     * partial failure.
+     */
+    async #runFullClose() {
+        log('Closing');
+        try {
+            if (!exists(this.#filePath)) {
+                log("Save the file because it doesn't exist");
+                await this.write();
+            }
+            else if (exists(this.#tmpDir)) {
+                log('Remove temporary dir');
+                await remove(this.#tmpDir);
+            }
+            await super.close();
+        }
+        finally {
+            await this.#releaseLock();
+        }
+        log('Closing done');
+    }
+    /**
+     * Worker for {@link releaseHandle}. Drops the SQLite handle and the
+     * advisory lock with no filesystem mutation.
+     */
+    async #runReleaseHandle() {
+        log('Releasing handle (no write, no remove)');
+        try {
+            await super.close();
+        }
+        finally {
+            await this.#releaseLock();
+        }
+    }
     /** The file extension for Nitpicker archive files (without the leading dot). */
     static FILE_EXTENSION = 'nitpicker';
-    /** The directory name used for storing HTML snapshots within the archive. */
-    static SNAPSHOT_HTML_DIR = 'snapshot-html';
     /** The filename of the SQLite database within the archive. */
     static SQLITE_DB_FILE_NAME = 'db.sqlite';
     /** The prefix used for temporary working directories during archive operations. */
     static TMP_DIR_PREFIX = '._nitpicker-';
     /**
      * Opens a read-only connection to an existing archive's database.
+     *
      * Returns an {@link ArchiveAccessor} that provides query methods
-     * without the ability to modify or write the archive.
+     * without the ability to modify or write the archive. The DB is opened
+     * in **read-only mode**: no schema migrations run, and the connection
+     * refuses to resurrect a missing parent directory or db file (so a
+     * TOCTOU window between source classification and this call cannot
+     * silently produce an empty phantom tmpDir).
+     *
+     * The returned accessor is also marked read-only so consumer-facing
+     * helpers (e.g. {@link ArchiveAccessor.getHtmlOfPage}) avoid any
+     * filesystem mutation on the user's tmpDir.
      * @param tmpDir - The path to the temporary directory containing the database.
      * @param namespace - An optional namespace for scoping data access within the archive.
      * @returns An ArchiveAccessor instance for querying the archive data.
      */
     static async connect(tmpDir, namespace = null) {
-        const db = await Archive.#connectDB(tmpDir);
-        const archive = new ArchiveAccessor(tmpDir, db, namespace);
+        const db = await Archive.#connectDB(tmpDir, { readOnly: true });
+        const archive = new ArchiveAccessor(tmpDir, db, namespace, { readOnly: true });
         return archive;
     }
     /**
@@ -301,22 +426,30 @@ export default class Archive extends ArchiveAccessor {
             cwd,
             openPluginData,
         });
-        const fileName = path.basename(filePath, path.extname(filePath));
-        const tmpDir = path.resolve(cwd, Archive.TMP_DIR_PREFIX + fileName);
+        // Read the tar's actual top-level directory name instead of deriving
+        // it from the outer file's basename. `.nitpicker` files are plain
+        // tar archives and users routinely rename them (`mv X.nitpicker
+        // Y.nitpicker`) — that operation must not break `open`. The inner
+        // directory keeps whatever name `Archive.write()` baked in at write
+        // time, and `tmpDir` mirrors the OUTER basename (so concurrent
+        // crawls on differently-named copies of the same archive don't
+        // collide on the lockfile).
+        const outerBasename = path.basename(filePath, path.extname(filePath));
+        const innerDirName = await peekTarTopDir(filePath);
+        const tmpDir = path.resolve(cwd, Archive.TMP_DIR_PREFIX + outerBasename);
         const releaseLock = await acquireArchiveLock(tmpDir);
         try {
             const openFiles = [];
             if (!openPluginData) {
-                const relDdPath = path.join(fileName, Archive.SQLITE_DB_FILE_NAME);
-                const relSnapshotPath = path.join(fileName, Archive.SNAPSHOT_HTML_DIR + '.zip');
-                openFiles.push(relDdPath, relSnapshotPath);
+                const relDdPath = path.join(innerDirName, Archive.SQLITE_DB_FILE_NAME);
+                openFiles.push(relDdPath);
             }
             log('Unzip file: %s (%O)', filePath, openFiles);
             await untar(filePath, {
                 cwd,
                 fileList: openFiles.length > 0 ? openFiles : undefined,
             });
-            const extractedDir = path.resolve(cwd, fileName);
+            const extractedDir = path.resolve(cwd, innerDirName);
             log('Move directory: %s to %s', extractedDir, tmpDir);
             await rename(extractedDir, tmpDir, true);
             return await Archive.#init(filePath, tmpDir, releaseLock);
@@ -373,13 +506,18 @@ export default class Archive extends ArchiveAccessor {
     /**
      * Connects to (or creates) the SQLite database in the given directory.
      * @param tmpDir - Directory containing `db.sqlite`
+     * @param options - Optional connection flags forwarded to
+     *   {@link Database.connect}. Used by {@link Archive.connect} to pass
+     *   `readOnly: true` so no migrations run and a missing tmpDir is not
+     *   resurrected.
+     * @param options.readOnly
      */
-    static async #connectDB(tmpDir) {
+    static async #connectDB(tmpDir, options) {
         const dbPath = path.resolve(tmpDir, Archive.SQLITE_DB_FILE_NAME);
-        dbLog('connects database: %s', dbPath);
+        dbLog('connects database: %s (readOnly=%s)', dbPath, options?.readOnly ?? false);
         return await Database.connect({
-            workingDir: tmpDir,
             filename: dbPath,
+            readOnly: options?.readOnly,
         });
     }
     /**
@@ -397,6 +535,34 @@ export default class Archive extends ArchiveAccessor {
         const archive = new Archive(filePath, tmpDir, db, releaseLock);
         return archive;
     }
+    /**
+     * Closes the archive. If the archive file does not yet exist on disk,
+     * it writes the archive first. If the temporary directory still exists,
+     * it is removed. The database connection is then closed via
+     * {@link ArchiveAccessor.close} (the base class owns the SQLite handle),
+     * and finally the archive's advisory lock is released.
+     *
+     * **Idempotent**: the first invocation captures the close promise;
+     * subsequent invocations (signal handlers, parallel teardowns, retried
+     * orchestrator paths) await the same promise instead of re-entering
+     * the destructive prologue on a half-mutated state. If the first
+     * close fails (e.g. ENOSPC during tar), the rejection propagates to
+     * all awaiters and the archive stays latched closed — there is no
+     * safe way to retry `write()` once `tmpDir` has been renamed.
+     *
+     * **Read-only consumers must not reach this override.** Anything that
+     * obtains an archive view via {@link Archive.connect} receives an
+     * {@link ArchiveAccessor} (not an `Archive`), so `close()` resolves to
+     * the safe base implementation — no `write()`, no `remove()`, no lock
+     * release — leaving the tmpDir intact for the live crawler.
+     */
+    async close() {
+        if (this.#closeOnce) {
+            return this.#closeOnce;
+        }
+        this.#closeOnce = this.#runFullClose();
+        return this.#closeOnce;
+    }
     /**
      * Retrieves the crawl configuration stored in the archive database.
      * @returns The configuration object.