@nitpicker/crawler 0.7.0 → 0.8.0

package/lib/archive/archive-lock.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Lock failure surfaced when an archive's tmpDir is already in use by another
+ * live process.
+ *
+ * The lock path and PID of the holder are exposed so callers (CLI / orchestrator)
+ * can render a precise error message and let operators recover manually.
+ */
+export declare class ArchiveLockError extends Error {
+    /** PID of the live process currently holding the lock, or `null` if unknown. */
+    readonly holderPid: number | null;
+    /** The lock directory path that could not be acquired. */
+    readonly lockPath: string;
+    /**
+     * @param lockPath - Absolute path of the lock directory.
+     * @param holderPid - PID stored in the lock's `pid.txt`, or `null` if unreadable.
+     */
+    constructor(lockPath: string, holderPid: number | null);
+}
+/**
+ * Acquire an advisory lock on the given tmpDir.
+ *
+ * Uses `fs.mkdir(lockPath, { recursive: false })` as the atomic primitive so
+ * that two concurrent processes targeting the same archive cannot both believe
+ * they own it. The lock is published as a sibling directory named
+ * `{tmpDir}.lock` containing a `pid.txt` file with the holder's PID.
+ *
+ * On `EEXIST` the function tries once to detect a stale lock: if the recorded
+ * PID is not alive any more (`process.kill(pid, 0)` throws `ESRCH`), the stale
+ * directory is removed and a single retry is issued. If acquisition still
+ * fails — or the holder is alive — an {@link ArchiveLockError} is thrown.
+ *
+ * The returned function releases the lock; it is idempotent so callers can put
+ * it in a `finally` block without worrying about double-release.
+ * @param tmpDir - Absolute path to the archive's temporary working directory.
+ * @returns A release function to be called when the work is done.
+ * @throws {ArchiveLockError} When the lock cannot be acquired even after a stale-lock retry.
+ */
+export declare function acquireArchiveLock(tmpDir: string): Promise<() => Promise<void>>;

package/lib/archive/archive-lock.js

@@ -0,0 +1,147 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+/**
+ * Lock failure surfaced when an archive's tmpDir is already in use by another
+ * live process.
+ *
+ * The lock path and PID of the holder are exposed so callers (CLI / orchestrator)
+ * can render a precise error message and let operators recover manually.
+ */
+export class ArchiveLockError extends Error {
+    /** PID of the live process currently holding the lock, or `null` if unknown. */
+    holderPid;
+    /** The lock directory path that could not be acquired. */
+    lockPath;
+    /**
+     * @param lockPath - Absolute path of the lock directory.
+     * @param holderPid - PID stored in the lock's `pid.txt`, or `null` if unreadable.
+     */
+    constructor(lockPath, holderPid) {
+        const suffix = holderPid === null ? '' : ` (PID ${holderPid})`;
+        super(`Archive is being used by another process${suffix}: ${lockPath}`);
+        this.name = 'ArchiveLockError';
+        this.lockPath = lockPath;
+        this.holderPid = holderPid;
+    }
+}
+/**
+ * Acquire an advisory lock on the given tmpDir.
+ *
+ * Uses `fs.mkdir(lockPath, { recursive: false })` as the atomic primitive so
+ * that two concurrent processes targeting the same archive cannot both believe
+ * they own it. The lock is published as a sibling directory named
+ * `{tmpDir}.lock` containing a `pid.txt` file with the holder's PID.
+ *
+ * On `EEXIST` the function tries once to detect a stale lock: if the recorded
+ * PID is not alive any more (`process.kill(pid, 0)` throws `ESRCH`), the stale
+ * directory is removed and a single retry is issued. If acquisition still
+ * fails — or the holder is alive — an {@link ArchiveLockError} is thrown.
+ *
+ * The returned function releases the lock; it is idempotent so callers can put
+ * it in a `finally` block without worrying about double-release.
+ * @param tmpDir - Absolute path to the archive's temporary working directory.
+ * @returns A release function to be called when the work is done.
+ * @throws {ArchiveLockError} When the lock cannot be acquired even after a stale-lock retry.
+ */
+export async function acquireArchiveLock(tmpDir) {
+    const lockPath = `${tmpDir}.lock`;
+    await tryAcquire(lockPath);
+    let released = false;
+    return async () => {
+        if (released) {
+            return;
+        }
+        released = true;
+        await releaseLock(lockPath);
+    };
+}
+/**
+ * Attempt to create the lock directory and write the holder PID, handling one
+ * round of stale-lock recovery.
+ * @param lockPath - The absolute lock directory path.
+ */
+async function tryAcquire(lockPath) {
+    try {
+        await fs.mkdir(lockPath, { recursive: false });
+        await fs.writeFile(path.join(lockPath, 'pid.txt'), String(process.pid), 'utf8');
+        return;
+    }
+    catch (error) {
+        if (!isEexist(error)) {
+            throw error;
+        }
+    }
+    const holderPid = await readHolderPid(lockPath);
+    if (holderPid !== null && isProcessAlive(holderPid)) {
+        throw new ArchiveLockError(lockPath, holderPid);
+    }
+    // Stale lock — clean up and retry once. A concurrent acquirer may have
+    // already reclaimed the directory, so a second EEXIST is treated as a real
+    // collision and surfaced to the caller.
+    await fs.rm(lockPath, { recursive: true, force: true });
+    try {
+        await fs.mkdir(lockPath, { recursive: false });
+        await fs.writeFile(path.join(lockPath, 'pid.txt'), String(process.pid), 'utf8');
+    }
+    catch (error) {
+        if (isEexist(error)) {
+            const pid = await readHolderPid(lockPath);
+            throw new ArchiveLockError(lockPath, pid);
+        }
+        throw error;
+    }
+}
+/**
+ * Remove the lock directory. `ENOENT` is swallowed so `finally`-style callers
+ * tolerate an externally-cleaned lock, but any other failure (permissions,
+ * disk full, etc.) is propagated so the caller can react.
+ * @param lockPath - The absolute lock directory path.
+ */
+async function releaseLock(lockPath) {
+    try {
+        await fs.rm(lockPath, { recursive: true, force: true });
+    }
+    catch (error) {
+        if (error.code !== 'ENOENT') {
+            throw error;
+        }
+    }
+}
+/**
+ * Read the PID stored in the lock directory's `pid.txt`.
+ * @param lockPath - The absolute lock directory path.
+ * @returns The parsed PID, or `null` if the file is missing or malformed.
+ */
+async function readHolderPid(lockPath) {
+    try {
+        const raw = await fs.readFile(path.join(lockPath, 'pid.txt'), 'utf8');
+        const pid = Number.parseInt(raw.trim(), 10);
+        return Number.isFinite(pid) && pid > 0 ? pid : null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Check whether the given PID is still alive using a signal-0 probe.
+ * @param pid - The process id to probe.
+ * @returns `true` if the process exists (regardless of permission).
+ */
+function isProcessAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        // EPERM means the process exists but is owned by another user — still alive.
+        return error.code === 'EPERM';
+    }
+}
+/**
+ * Type-guard for an `EEXIST` filesystem error.
+ * @param error - The error value to inspect.
+ * @returns `true` when the error is a Node `EEXIST` from a filesystem call.
+ */
+function isEexist(error) {
+    return error?.code === 'EEXIST';
+}

package/lib/archive/archive.d.ts

@@ -1,6 +1,6 @@
 import type { Config } from './types.js';
 import type { PageData, CrawlerError, Resource } from '../utils/types/types.js';
-import type { ParseURLOptions } from '@d-zero/shared/parse-url';
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
 import { ArchiveAccessor } from './archive-accessor.js';
 /**
  * Main archive class for creating, opening, resuming, and writing Nitpicker archive files (`.nitpicker`).
@@ -47,11 +47,20 @@ export default class Archive extends ArchiveAccessor {
      * @returns The base URL string.
      */
     getUrl(): Promise<any>;
+    /**
+     * Promote previously-external pages that now fall under the (possibly extended)
+     * scope back to a pending state so that the crawler re-scrapes them as fully
+     * internal pages on the next pass.
+     * @param scopes - Hostname-indexed scope map representing the new scope.
+     * @param options - URL parsing options forwarded to the scope-entry lookup.
+     * @returns The URLs that were repromoted.
+     */
+    repromoteExternalPages(scopes: ReadonlyMap<string, readonly ExURL[]>, options?: ParseURLOptions): Promise<string[]>;
     /**
      * Stores the crawl configuration into the archive database.
      * @param config - The configuration object to store.
      */
-    setConfig(config: Config): Promise<Config[]>;
+    setConfig(config: Config): Promise<number[]>;
     /**
      * Stores an external page's data in the archive database without saving a snapshot.
      * @param pageInfo - The page data to store.
@@ -93,6 +102,12 @@ export default class Archive extends ArchiveAccessor {
      * that do not yet have an `order` field set.
      */
     setUrlOrder(): Promise<void>;
+    /**
+     * Updates a subset of fields on the archive's `info` row. Used by the append
+     * flow to extend `roots` / `scope` without rewriting the entire config.
+     * @param patch - Partial {@link Config} fields to overwrite. `undefined` values are ignored.
+     */
+    updateConfig(patch: Partial<Config>): Promise<void>;
     /**
      * Writes the archive to disk as a compressed `.nitpicker` file.
      *

package/lib/archive/archive.js

@@ -1,6 +1,7 @@
 import path from 'node:path';
 import { zip } from '@d-zero/fs/zip';
 import { ArchiveAccessor } from './archive-accessor.js';
+import { acquireArchiveLock } from './archive-lock.js';
 import { Database } from './database.js';
 import { dbLog, log, saveLog } from './debug.js';
 import { appendText } from './filesystem/append-text.js';
@@ -26,6 +27,8 @@ export default class Archive extends ArchiveAccessor {
     #db;
     /** Absolute path to the output `.nitpicker` archive file. */
     #filePath;
+    /** Lock release function held while the writer owns the archive. */
+    #releaseLock;
     /** Absolute path to the HTML snapshot directory within the temporary working directory. */
     #snapshotDir;
     /** Absolute path to the temporary working directory containing the SQLite DB and snapshots. */
@@ -37,12 +40,13 @@ export default class Archive extends ArchiveAccessor {
         return this.#filePath;
     }
     // eslint-disable-next-line no-restricted-syntax
-    constructor(filePath, tmpDir, db) {
+    constructor(filePath, tmpDir, db, releaseLock) {
         super(tmpDir, db, '');
         this.#filePath = filePath;
         this.#tmpDir = tmpDir;
         this.#snapshotDir = path.resolve(this.#tmpDir, Archive.SNAPSHOT_HTML_DIR);
         this.#db = db;
+        this.#releaseLock = releaseLock;
         log('create instance: %O', {
             filePath,
             tmpDir,
@@ -71,15 +75,20 @@ export default class Archive extends ArchiveAccessor {
      */
     async close() {
         log('Closing');
-        if (!exists(this.#filePath)) {
-            log("Save the file because it doesn't exist");
-            await this.write();
+        try {
+            if (!exists(this.#filePath)) {
+                log("Save the file because it doesn't exist");
+                await this.write();
+            }
+            else if (exists(this.#tmpDir)) {
+                log('Remove temporary dir');
+                await remove(this.#tmpDir);
+            }
+            await this.#db.destroy();
         }
-        else if (exists(this.#tmpDir)) {
-            log('Remove temporary dir');
-            await remove(this.#tmpDir);
+        finally {
+            await this.#releaseLock();
         }
-        await this.#db.destroy();
         log('Closing done');
     }
     /**
@@ -96,6 +105,18 @@ export default class Archive extends ArchiveAccessor {
     async getUrl() {
         return this.#db.getBaseUrl();
     }
+    /**
+     * Promote previously-external pages that now fall under the (possibly extended)
+     * scope back to a pending state so that the crawler re-scrapes them as fully
+     * internal pages on the next pass.
+     * @param scopes - Hostname-indexed scope map representing the new scope.
+     * @param options - URL parsing options forwarded to the scope-entry lookup.
+     * @returns The URLs that were repromoted.
+     */
+    async repromoteExternalPages(scopes, options) {
+        dbLog('Repromote external pages with %d hostnames in scope', scopes.size);
+        return this.#db.repromoteExternalPages(scopes, options);
+    }
     /**
      * Stores the crawl configuration into the archive database.
      * @param config - The configuration object to store.
@@ -176,6 +197,15 @@ export default class Archive extends ArchiveAccessor {
         dbLog("Pages didn't have `order` field. So set URL order.");
         await this.#db.setUrlOrder();
     }
+    /**
+     * Updates a subset of fields on the archive's `info` row. Used by the append
+     * flow to extend `roots` / `scope` without rewriting the entire config.
+     * @param patch - Partial {@link Config} fields to overwrite. `undefined` values are ignored.
+     */
+    async updateConfig(patch) {
+        dbLog('Update config: %O', patch);
+        await this.#db.updateConfig(patch);
+    }
     /**
      * Writes the archive to disk as a compressed `.nitpicker` file.
      *
@@ -240,7 +270,14 @@ export default class Archive extends ArchiveAccessor {
         });
         const fileName = path.basename(filePath, path.extname(filePath));
         const tmpDir = path.resolve(cwd, Archive.TMP_DIR_PREFIX + fileName);
-        return await Archive.#init(filePath, tmpDir);
+        const releaseLock = await acquireArchiveLock(tmpDir);
+        try {
+            return await Archive.#init(filePath, tmpDir, releaseLock);
+        }
+        catch (error) {
+            await releaseLock();
+            throw error;
+        }
     }
     /**
      * Joins path segments into an absolute path.
@@ -266,21 +303,28 @@ export default class Archive extends ArchiveAccessor {
         });
         const fileName = path.basename(filePath, path.extname(filePath));
         const tmpDir = path.resolve(cwd, Archive.TMP_DIR_PREFIX + fileName);
-        const openFiles = [];
-        if (!openPluginData) {
-            const relDdPath = path.join(fileName, Archive.SQLITE_DB_FILE_NAME);
-            const relSnapshotPath = path.join(fileName, Archive.SNAPSHOT_HTML_DIR + '.zip');
-            openFiles.push(relDdPath, relSnapshotPath);
+        const releaseLock = await acquireArchiveLock(tmpDir);
+        try {
+            const openFiles = [];
+            if (!openPluginData) {
+                const relDdPath = path.join(fileName, Archive.SQLITE_DB_FILE_NAME);
+                const relSnapshotPath = path.join(fileName, Archive.SNAPSHOT_HTML_DIR + '.zip');
+                openFiles.push(relDdPath, relSnapshotPath);
+            }
+            log('Unzip file: %s (%O)', filePath, openFiles);
+            await untar(filePath, {
+                cwd,
+                fileList: openFiles.length > 0 ? openFiles : undefined,
+            });
+            const extractedDir = path.resolve(cwd, fileName);
+            log('Move directory: %s to %s', extractedDir, tmpDir);
+            await rename(extractedDir, tmpDir, true);
+            return await Archive.#init(filePath, tmpDir, releaseLock);
+        }
+        catch (error) {
+            await releaseLock();
+            throw error;
         }
-        log('Unzip file: %s (%O)', filePath, openFiles);
-        await untar(filePath, {
-            cwd,
-            fileList: openFiles.length > 0 ? openFiles : undefined,
-        });
-        const extractedDir = path.resolve(cwd, fileName);
-        log('Move directory: %s to %s', extractedDir, tmpDir);
-        await rename(extractedDir, tmpDir, true);
-        return await Archive.#init(filePath, tmpDir);
     }
     /**
      * Resumes an archive from an existing temporary directory
@@ -293,11 +337,18 @@ export default class Archive extends ArchiveAccessor {
         log('Resume: %s', targetPath);
         if (await isDir(targetPath)) {
             const tmpDir = targetPath;
-            const db = await Archive.#connectDB(tmpDir);
-            const name = (await db.getName()) ||
-                path.basename(targetPath).replace(Archive.TMP_DIR_PREFIX, '');
-            const filePath = path.resolve(process.cwd(), name + '.' + Archive.FILE_EXTENSION);
-            return await Archive.#init(filePath, tmpDir);
+            const releaseLock = await acquireArchiveLock(tmpDir);
+            try {
+                const db = await Archive.#connectDB(tmpDir);
+                const name = (await db.getName()) ||
+                    path.basename(targetPath).replace(Archive.TMP_DIR_PREFIX, '');
+                const filePath = path.resolve(process.cwd(), name + '.' + Archive.FILE_EXTENSION);
+                return new Archive(filePath, tmpDir, db, releaseLock);
+            }
+            catch (error) {
+                await releaseLock();
+                throw error;
+            }
         }
         throw new Error('The specified path is not a directory. Please ensure the path points to a valid directory.');
     }
@@ -327,19 +378,23 @@ export default class Archive extends ArchiveAccessor {
         const dbPath = path.resolve(tmpDir, Archive.SQLITE_DB_FILE_NAME);
         dbLog('connects database: %s', dbPath);
         return await Database.connect({
-            type: 'sqlite3',
             workingDir: tmpDir,
             filename: dbPath,
         });
     }
     /**
      * Initializes an Archive instance by connecting to the database.
+     *
+     * The advisory lock must already be acquired by the caller; this helper just
+     * threads the release function through to the resulting `Archive` so that
+     * {@link Archive.close} can drop the lock when work is done.
      * @param filePath - Output `.nitpicker` file path
      * @param tmpDir - Temporary working directory path
+     * @param releaseLock - Function returned by {@link acquireArchiveLock}.
      */
-    static async #init(filePath, tmpDir) {
+    static async #init(filePath, tmpDir, releaseLock) {
         const db = await Archive.#connectDB(tmpDir);
-        const archive = new Archive(filePath, tmpDir, db);
+        const archive = new Archive(filePath, tmpDir, db, releaseLock);
         return archive;
     }
     /**

package/lib/archive/database.d.ts

@@ -1,5 +1,6 @@
 import type { Config, DatabaseOption, DB_Anchor, DB_Page, DB_Redirect, DB_Referrer, DB_Resource, DatabaseEvent, PageFilter } from './types.js';
 import type { PageData, Resource } from '../utils/types/types.js';
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
 import type { Knex } from 'knex';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 /**
@@ -54,7 +55,7 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
     getBaseUrl(): Promise<any>;
     /**
      * Retrieves the full crawl configuration from the `info` table.
-     * Deserializes JSON-encoded fields (`excludes`, `excludeKeywords`, `scope`).
+     * Deserializes JSON-encoded fields (`roots`, `excludes`, `excludeKeywords`, `excludeUrls`).
      * @returns The parsed {@link Config} object.
      * @throws {Error} If no configuration is found in the database.
      */
@@ -154,12 +155,34 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param pageUrl - The URL of the page that references the resource.
      */
     insertResourceReferrers(src: string, pageUrl: string): Promise<void>;
+    /**
+     * Promote previously-external pages whose URL falls under any of the new scope
+     * entries back to a "needs scraping" state so that the next crawl picks them up
+     * as full internal pages.
+     *
+     * For each matching page:
+     * - clears the scrape metadata (status, headers, snapshot path, etc.),
+     * - flips `isExternal` to `0` and `scraped` to `0`,
+     * - removes stale `anchors`, `images`, and `resources-referrers` rows so that
+     *   the re-scrape can re-insert fresh ones without duplicates.
+     *
+     * The page row itself is kept (id is preserved) so existing referrers via
+     * `anchors.hrefId` remain valid. SELECT and UPDATE/DELETE statements are
+     * chunked to stay below SQLite's `SQLITE_LIMIT_VARIABLE_NUMBER`.
+     * @param scopes - The hostname-indexed scope map after the new roots are merged.
+     * @param options - URL parsing options forwarded to {@link findScopeEntry}.
+     * @returns The URLs of the pages that were promoted.
+     */
+    repromoteExternalPages(scopes: ReadonlyMap<string, readonly ExURL[]>, options?: ParseURLOptions): Promise<string[]>;
     /**
      * Stores the crawl configuration in the `info` table.
-     * Serializes array fields (`excludes`, `excludeKeywords`, `scope`) as JSON strings.
+     * Only fields in {@link INFO_COLUMN_ALLOWLIST} are forwarded — any extra
+     * runtime-only field on the input is silently dropped so callers can splat
+     * a wider config object without producing SQL errors. JSON-array fields
+     * are serialized via `JSON.stringify`.
      * @param config - The {@link Config} object to store.
      */
-    setConfig(config: Config): Promise<Config[]>;
+    setConfig(config: Config): Promise<number[]>;
     /**
      * Marks a page as skipped in the database with the given reason.
      * Creates the page row if it does not already exist.
@@ -173,6 +196,20 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * Pages are sorted using {@link pathComparator} and assigned sequential order numbers.
      */
     setUrlOrder(): Promise<void>;
+    /**
+     * Update the single row in the `info` table with a partial config patch.
+     *
+     * Used by the append flow to extend `roots` (and any other tweakable
+     * field) without replacing the entire row. JSON-array fields are serialized on
+     * the fly; primitive fields are written verbatim. Unspecified fields stay as-is.
+     *
+     * Unknown keys (anything outside the allow-list of `info`-table columns) are
+     * silently dropped instead of being passed to SQL, so callers that splat a
+     * wider runtime config (e.g. `CrawlConfig` with `cwd` / `executablePath`)
+     * cannot accidentally trigger a "no such column" SQL error.
+     * @param patch - Partial {@link Config} fields to overwrite. `undefined` values are skipped.
+     */
+    updateConfig(patch: Partial<Config>): Promise<void>;
     /**
      * Inserts or updates a crawled page in the database, including its redirect chain,
      * anchors, and images. Optionally creates an HTML snapshot file path entry.
@@ -194,7 +231,7 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * Creates and initializes a new Database instance.
      * Creates the parent directory for the database file if needed,
      * establishes the connection, and initializes tables if they do not exist.
-     * @param options - The database connection options specifying the type and file path.
+     * @param options - Database connection options (working directory + SQLite file path).
      * @returns A fully initialized Database instance.
      */
     static connect(options: DatabaseOption): Promise<Database>;