@nitpicker/crawler 0.6.5-alpha.0 → 0.8.0

package/lib/crawler-orchestrator.js

@@ -1,3 +1,4 @@
+import { copyFile, unlink as unlinkFile } from 'node:fs/promises';
 import path from 'node:path';
 import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
 import { sortUrl } from '@d-zero/shared/sort-url';
@@ -81,7 +82,7 @@ export class CrawlerOrchestrator extends EventEmitter {
             executablePath: options?.executablePath || null,
             fetchExternal: options?.fetchExternal ?? true,
             recursive: options?.recursive ?? true,
-            scope: options?.scope ?? [],
+            roots: options?.roots ?? [],
             excludes: normalizeToArray(options?.excludes),
             excludeKeywords: normalizeToArray(options?.excludeKeywords),
             excludeUrls: [
@@ -159,12 +160,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                     .then(() => resolve())
                     .catch((error) => reject(error));
             });
-            if (this.#fromList) {
-                this.#crawler.startMultiple(list);
-            }
-            else {
-                this.#crawler.start(root);
-            }
+            this.#crawler.start(list, { recursive: !this.#fromList });
         });
     }
     /**
@@ -237,16 +233,18 @@ export class CrawlerOrchestrator extends EventEmitter {
         const disableQueries = options?.disableQueries || false;
         const defaultUserAgent = `Nitpicker/${pkg.version}`;
         const archive = await Archive.create({ filePath, cwd, disableQueries });
+        // Each positional URL is both a starting point and a scope entry.
+        const rootHrefs = list.map((u) => u.withoutHash);
         await archive.setConfig({
             version: pkg.version,
             name: fileName,
-            baseUrl: urlParsed.withoutHash,
+            baseUrl: rootHrefs[0],
+            roots: rootHrefs,
             recursive: options?.recursive ?? true,
             fetchExternal: options?.fetchExternal ?? true,
             image: options?.image ?? true,
             interval: options?.interval || 0,
             parallels: options?.parallels || 0,
-            scope: options?.scope ?? [],
             excludes: normalizeToArray(options?.excludes),
             excludeKeywords: normalizeToArray(options?.excludeKeywords),
             excludeUrls: [
@@ -260,7 +258,10 @@ export class CrawlerOrchestrator extends EventEmitter {
             userAgent: options?.userAgent || defaultUserAgent,
             ignoreRobots: options?.ignoreRobots ?? false,
         });
-        const orchestrator = new CrawlerOrchestrator(archive, options);
+        const orchestrator = new CrawlerOrchestrator(archive, {
+            ...options,
+            roots: rootHrefs,
+        });
         const config = await archive.getConfig();
         if (initializedCallback) {
             await initializedCallback(orchestrator, config);
@@ -276,6 +277,110 @@ export class CrawlerOrchestrator extends EventEmitter {
         log('Sorting done');
         return orchestrator;
     }
+    /**
+     * Append a fresh crawl to an existing `.nitpicker` archive.
+     *
+     * The given `newUrls` become additional recursive roots: their `withoutHash`
+     * form is merged into `info.roots` and the crawler picks them up as
+     * starting URLs. Previously-external pages whose URL now falls under
+     * the expanded scope are demoted back to "needs scraping" so the next pass
+     * re-fetches them as full internal pages. A `<archive>.bak` is created
+     * before the crawl and removed on success; if the crawl throws, the backup
+     * is restored to keep the original archive intact.
+     *
+     * List-mode archives (`info.fromList === true`) are rejected because their
+     * pages are all metadata-only and cannot host a recursive append.
+     * @param archivePath - Absolute or relative path to the existing `.nitpicker`.
+     * @param newUrls - New root URLs to add and crawl.
+     * @param options - Optional config overrides applied on top of the archived config.
+     * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @returns The orchestrator instance after the append crawl completes.
+     * @throws {Error} When `newUrls` is empty, the archive is in list mode, or it cannot be parsed.
+     */
+    static async append(archivePath, newUrls, options, initializedCallback) {
+        if (newUrls.length === 0) {
+            throw new Error('append: newUrls is empty');
+        }
+        const cwd = options?.cwd ?? process.cwd();
+        const absFilePath = path.isAbsolute(archivePath)
+            ? archivePath
+            : path.resolve(cwd, archivePath);
+        const archive = await Archive.open({ filePath: absFilePath, cwd });
+        // Any throw between here and the successful return must release the
+        // archive lock and clean up tmpDir; the caller's `close()` only runs on
+        // the happy path. Errors from `close()` itself are intentionally
+        // best-effort: the original error is what matters.
+        try {
+            const archived = await archive.getConfig();
+            if (archived.fromList) {
+                throw new Error('Cannot append to a list-mode archive: this archive was created with --list/--list-file and contains metadata-only pages. Create a fresh archive instead.');
+            }
+            const newParsed = sortUrl(newUrls, archived);
+            if (newParsed.length === 0) {
+                throw new Error('append: no parseable URLs provided');
+            }
+            const newRoots = newParsed.map((u) => u.withoutHash);
+            const mergedRoots = [...new Set([...archived.roots, ...newRoots])];
+            const mergedConfig = {
+                ...archived,
+                ...cleanObject(options),
+                roots: mergedRoots,
+                fromList: false,
+                recursive: true,
+                baseUrl: mergedRoots[0],
+            };
+            const backupPath = absFilePath + '.bak';
+            await copyFile(absFilePath, backupPath);
+            try {
+                await archive.updateConfig(mergedConfig);
+                const scopeMap = new Map();
+                for (const raw of mergedRoots) {
+                    const parsed = parseUrl(raw, archived);
+                    if (!parsed)
+                        continue;
+                    const existing = scopeMap.get(parsed.hostname) ?? [];
+                    scopeMap.set(parsed.hostname, [...existing, parsed]);
+                }
+                await archive.repromoteExternalPages(scopeMap, archived);
+                const orchestrator = new CrawlerOrchestrator(archive, {
+                    ...mergedConfig,
+                    roots: mergedRoots,
+                });
+                const { scraped, pending } = await archive.getCrawlingState();
+                const resources = await archive.getResourceUrlList();
+                orchestrator.#crawler.resume(pending, scraped, resources);
+                if (initializedCallback) {
+                    await initializedCallback(orchestrator, mergedConfig);
+                }
+                log('Start appending');
+                log('Archive %s', absFilePath);
+                log('New roots %O', newRoots);
+                log('Merged roots %O', mergedRoots);
+                await orchestrator.crawling(newParsed);
+                clearDestinationCache();
+                await archive.setUrlOrder();
+                await ignoreEnoent(unlinkFile(backupPath));
+                return orchestrator;
+            }
+            catch (error) {
+                try {
+                    await copyFile(backupPath, absFilePath);
+                    await ignoreEnoent(unlinkFile(backupPath));
+                }
+                catch (restoreError) {
+                    // Restore itself failed — surface both so the operator knows
+                    // the .bak still exists and the original archive may be
+                    // corrupt. The outer `catch` still releases the lock.
+                    throw new AggregateError([error, restoreError], `append failed AND restore from backup failed. Original archive backup is left at: ${backupPath}`);
+                }
+                throw error;
+            }
+        }
+        catch (error) {
+            await archive.close().catch(() => { });
+            throw error;
+        }
+    }
     /**
      * Resume a previously interrupted crawl from an existing archive file.
      *
@@ -315,3 +420,19 @@ export class CrawlerOrchestrator extends EventEmitter {
         return orchestrator;
     }
 }
+/**
+ * Await a filesystem promise but silently swallow only `ENOENT` errors. Any
+ * other failure (permissions, disk full, etc.) propagates so the caller can
+ * react instead of guessing whether the operation worked.
+ * @param promise - Filesystem operation to await.
+ */
+async function ignoreEnoent(promise) {
+    try {
+        await promise;
+    }
+    catch (error) {
+        if (error.code !== 'ENOENT') {
+            throw error;
+        }
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.6.5-alpha.0",
+	"version": "0.8.0",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -36,9 +36,9 @@
 		"follow-redirects": "1.15.11",
 		"fs-extra": "11.3.3",
 		"knex": "3.1.0",
+		"libsql": "0.5.29",
 		"puppeteer": "24.37.5",
 		"robots-parser": "3.0.1",
-		"sqlite3": "5.1.7",
 		"tar": "7.5.9"
 	},
 	"devDependencies": {
@@ -48,5 +48,5 @@
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "e084aba5a0887a80059ff8aa0608f61a58cc9288"
+	"gitHead": "8456d02c3052b2b3438f0d32e096ed7e916c5234"
 }

package/lib/crawler/find-best-matching-scope.d.ts

@@ -1,13 +0,0 @@
-import type { ExURL } from '@d-zero/shared/parse-url';
-/**
- * Find the scope URL with the deepest matching path for a given URL.
- *
- * Among all scope URLs sharing the same hostname, returns the one whose
- * path segments are a prefix of the target URL's path segments and which
- * has the greatest depth. A root scope (paths: `['']`) matches all paths
- * under the same hostname. Returns `null` if no scope URL matches.
- * @param url - The parsed URL to match against scope URLs.
- * @param scopes - The list of scope URLs to search.
- * @returns The best-matching scope URL, or `null` if none match.
- */
-export declare function findBestMatchingScope(url: ExURL, scopes: readonly ExURL[]): ExURL | null;

package/lib/crawler/find-best-matching-scope.js

@@ -1,52 +0,0 @@
-/**
- * Find the scope URL with the deepest matching path for a given URL.
- *
- * Among all scope URLs sharing the same hostname, returns the one whose
- * path segments are a prefix of the target URL's path segments and which
- * has the greatest depth. A root scope (paths: `['']`) matches all paths
- * under the same hostname. Returns `null` if no scope URL matches.
- * @param url - The parsed URL to match against scope URLs.
- * @param scopes - The list of scope URLs to search.
- * @returns The best-matching scope URL, or `null` if none match.
- */
-export function findBestMatchingScope(url, scopes) {
-    let bestMatch = null;
-    let maxDepth = -1;
-    for (const scope of scopes) {
-        if (url.hostname !== scope.hostname) {
-            continue;
-        }
-        const isMatch = isPathMatch(url.paths, scope.paths);
-        if (isMatch && scope.depth > maxDepth) {
-            bestMatch = scope;
-            maxDepth = scope.depth;
-        }
-    }
-    return bestMatch;
-}
-/**
- * Check whether a target path is equal to or is a descendant of a base path.
- *
- * A root base path (`['']`) unconditionally matches any target path.
- * Otherwise, compares path segments element by element — the target path
- * matches if all segments of the base path appear in the same positions
- * at the beginning of the target path.
- * @param targetPaths - The path segments of the URL being checked.
- * @param basePaths - The path segments of the scope URL to match against.
- * @returns `true` if the target path starts with or equals the base path.
- */
-function isPathMatch(targetPaths, basePaths) {
-    // Root scope (paths: ['']) matches all paths under the same hostname
-    if (basePaths.length === 1 && basePaths[0] === '') {
-        return true;
-    }
-    if (targetPaths.length < basePaths.length) {
-        return false;
-    }
-    for (const [i, basePath] of basePaths.entries()) {
-        if (targetPaths[i] !== basePath) {
-            return false;
-        }
-    }
-    return true;
-}

package/lib/crawler/is-in-any-lower-layer.d.ts

@@ -1,13 +0,0 @@
-import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
-/**
- * Check whether a URL is in a lower layer (subdirectory) of any scope URL.
- *
- * Tests the URL against each scope URL using the `isLowerLayer` utility,
- * which checks if the URL's path is at the same level or deeper than
- * the scope URL's path.
- * @param url - The parsed URL to check.
- * @param scopes - The list of scope URLs to test against.
- * @param options - URL parsing options used for layer comparison.
- * @returns `true` if the URL is in a lower layer of at least one scope URL.
- */
-export declare function isInAnyLowerLayer(url: ExURL, scopes: readonly ExURL[], options: ParseURLOptions): boolean;

package/lib/crawler/is-in-any-lower-layer.js

@@ -1,15 +0,0 @@
-import { isLowerLayer } from '@d-zero/shared/is-lower-layer';
-/**
- * Check whether a URL is in a lower layer (subdirectory) of any scope URL.
- *
- * Tests the URL against each scope URL using the `isLowerLayer` utility,
- * which checks if the URL's path is at the same level or deeper than
- * the scope URL's path.
- * @param url - The parsed URL to check.
- * @param scopes - The list of scope URLs to test against.
- * @param options - URL parsing options used for layer comparison.
- * @returns `true` if the URL is in a lower layer of at least one scope URL.
- */
-export function isInAnyLowerLayer(url, scopes, options) {
-    return scopes.some((scope) => isLowerLayer(url.href, scope.href, options));
-}