@nitpicker/crawler 0.7.0 → 0.8.0

package/lib/crawler/crawler.js

@@ -11,6 +11,7 @@ import pkg from '../../package.json' with { type: 'json' };
 import { crawlerLog } from '../debug.js';
 import { detectPaginationPattern } from './detect-pagination-pattern.js';
 import { fetchDestination } from './fetch-destination.js';
+import { findScopeEntry } from './find-scope-entry.js';
 import { formatCrawlProgress } from './format-crawl-progress.js';
 import { generatePredictedUrls } from './generate-predicted-urls.js';
 import { handleIgnoreAndSkip } from './handle-ignore-and-skip.js';
@@ -18,7 +19,6 @@ import { handleResourceResponse } from './handle-resource-response.js';
 import { handleScrapeEnd } from './handle-scrape-end.js';
 import { handleScrapeError } from './handle-scrape-error.js';
 import { injectScopeAuth } from './inject-scope-auth.js';
-import { isExternalUrl } from './is-external-url.js';
 import LinkList from './link-list.js';
 import { linkToPageData } from './link-to-page-data.js';
 import { protocolAgnosticKey } from './protocol-agnostic-key.js';
@@ -77,7 +77,7 @@ class Crawler extends EventEmitter {
             captureImages: options?.captureImages ?? true,
             executablePath: options?.executablePath ?? null,
             fetchExternal: options?.fetchExternal ?? true,
-            scope: options?.scope ?? [],
+            roots: options?.roots ?? [],
             excludes: options?.excludes || [],
             excludeKeywords: options?.excludeKeywords || [],
             excludeUrls: options?.excludeUrls || [],
@@ -89,7 +89,7 @@ class Crawler extends EventEmitter {
             ignoreRobots: options?.ignoreRobots ?? false,
         };
         this.#robotsChecker = new RobotsChecker(this.#options.userAgent, !this.#options.ignoreRobots);
-        for (const urlStr of this.#options.scope) {
+        for (const urlStr of this.#options.roots) {
             const url = parseUrl(urlStr, this.#options);
             if (url) {
                 const existing = this.#scope.get(url.hostname) || [];
@@ -135,21 +135,56 @@ class Crawler extends EventEmitter {
         }
     }
     /**
-     * Start crawling from a single root URL.
+     * Start crawling from one or more root URLs.
      *
-     * Adds the root URL to the scope (if not already present) and the link list,
-     * then begins the deal-based concurrent crawl. Discovered child pages are
-     * automatically added to the queue when recursive mode is enabled.
-     * @param url - The root URL to begin crawling from.
+     * Each URL is registered as a scope entry (if not already present) and added
+     * to the link list. When `opts.recursive` is `false`, recursion is disabled
+     * and the crawler behaves like the former `startMultiple` (list mode);
+     * otherwise discovered child pages within the scope are followed.
+     *
+     * When resume state is present, the resumed pending URLs are merged with the
+     * newly-provided roots. The merge is deduplicated by protocol-agnostic key
+     * before reaching the dealer so a URL that exists in both sources — which
+     * is common in append-mode when a new root coincides with a repromoted
+     * previously-external page — does not race on two parallel slots.
+     * @param urls - The list of root URLs to begin crawling from. Must be non-empty.
+     * @param opts - Optional overrides; currently only `recursive` is honoured.
+     * @param opts.recursive - When `false`, disables recursive discovery and forces list-mode.
+     *   Defaults to the constructor option's `recursive` value.
+     * @throws {Error} If the URL list is empty.
      */
-    start(url) {
-        const existing = this.#scope.get(url.hostname) || [];
-        if (!existing.some((u) => u.href === url.href)) {
-            this.#scope.set(url.hostname, [...existing, url]);
+    start(urls, opts) {
+        const root = urls[0];
+        if (!root) {
+            throw new Error('urls is empty');
+        }
+        for (const url of urls) {
+            const existing = this.#scope.get(url.hostname) || [];
+            if (!existing.some((u) => u.href === url.href)) {
+                this.#scope.set(url.hostname, [...existing, url]);
+            }
+            this.#linkList.add(url);
+        }
+        const recursive = opts?.recursive ?? this.#options.recursive;
+        if (!recursive) {
+            this.#options.recursive = false;
+            this.#options.fromList = true;
         }
-        this.#linkList.add(url);
         const isResuming = this.#resumedScraped.length > 0;
-        const initialUrls = isResuming ? this.#resumedPending : [url];
+        // Dedupe by the same protocol-agnostic key the dealer uses internally.
+        // Append-mode in particular can put the same URL into both
+        // `#resumedPending` (via `repromoteExternalPages`) and `urls` (the
+        // new root); without this dedupe both copies would grab a parallel
+        // slot and race on the same URL.
+        const seenInitial = new Set();
+        const initialUrls = [];
+        for (const url of isResuming ? [...this.#resumedPending, ...urls] : urls) {
+            const key = protocolAgnosticKey(url.withoutHashAndAuth);
+            if (seenInitial.has(key))
+                continue;
+            seenInitial.add(key);
+            initialUrls.push(url);
+        }
         const resumeOffset = this.#resumedScraped.length;
         if (initialUrls.length === 0) {
             crawlerLog('Crawl End (nothing to resume)');
@@ -158,38 +193,7 @@ class Crawler extends EventEmitter {
         }
         void this.#runDeal(initialUrls, resumeOffset).catch((error) => {
             crawlerLog('runDeal error: %O', error);
-            this.#emitDealErrors(error, url.href);
-            void this.emit('crawlEnd', {});
-        });
-    }
-    /**
-     * Start crawling a pre-defined list of URLs in non-recursive mode.
-     *
-     * Each URL in the list is added to the scope and the link list. Recursive
-     * crawling is disabled; only the provided URLs will be scraped.
-     * @param pageList - The list of URLs to crawl. Must contain at least one URL.
-     * @throws {Error} If the page list is empty.
-     */
-    startMultiple(pageList) {
-        if (!pageList[0]) {
-            throw new Error('pageList is empty');
-        }
-        const scopeMap = new Map();
-        for (const pageUrl of pageList) {
-            const existing = this.#scope.get(pageUrl.hostname) || [];
-            const existingHrefs = scopeMap.get(pageUrl.hostname) || new Set(existing.map((u) => u.href));
-            if (!existingHrefs.has(pageUrl.href)) {
-                this.#scope.set(pageUrl.hostname, [...existing, pageUrl]);
-                existingHrefs.add(pageUrl.href);
-            }
-            scopeMap.set(pageUrl.hostname, existingHrefs);
-            this.#linkList.add(pageUrl);
-        }
-        this.#options.recursive = false;
-        this.#options.fromList = true;
-        void this.#runDeal(pageList).catch((error) => {
-            crawlerLog('runDeal error: %O', error);
-            this.#emitDealErrors(error, pageList[0].href);
+            this.#emitDealErrors(error, root.href);
             void this.emit('crawlEnd', {});
         });
     }
@@ -260,7 +264,8 @@ class Crawler extends EventEmitter {
                     if (!paginationState || !concurrency)
                         return;
                     // metadataOnly / external: update tracking but skip pattern detection
-                    if (opts?.metadataOnly || isExternalUrl(newUrl, this.#scope)) {
+                    if (opts?.metadataOnly ||
+                        findScopeEntry(newUrl, this.#scope, this.#options) === null) {
                         paginationState.lastPushedUrl = newUrl.withoutHashAndAuth;
                         paginationState.lastPushedWasPredicted = false;
                         return;
@@ -301,7 +306,7 @@ class Crawler extends EventEmitter {
                 void this.emit('skip', {
                     url: result.ignored.url.href,
                     reason: JSON.stringify(result.ignored),
-                    isExternal: isExternalUrl(result.ignored.url, this.#scope),
+                    isExternal: findScopeEntry(result.ignored.url, this.#scope, this.#options) === null,
                 });
                 break;
             }
@@ -317,7 +322,7 @@ class Crawler extends EventEmitter {
                     shutdown: result.error.shutdown,
                     pid: undefined,
                 }, this.#linkList, this.#scope, this.#options);
-                const isExternal = isExternalUrl(url, this.#scope);
+                const isExternal = findScopeEntry(url, this.#scope, this.#options) === null;
                 if (pageResult) {
                     if (pageResult.isExternal) {
                         void this.emit('externalPage', { result: pageResult });
@@ -436,7 +441,7 @@ class Crawler extends EventEmitter {
         const externalDoneUrls = new Set();
         // 初期 URL を分類（onPush を通らないため）
         for (const url of initialUrls) {
-            if (isExternalUrl(url, this.#scope)) {
+            if (findScopeEntry(url, this.#scope, this.#options) === null) {
                 externalUrls.add(protocolAgnosticKey(url.withoutHashAndAuth));
             }
         }
@@ -449,10 +454,13 @@ class Crawler extends EventEmitter {
             lastPushedWasPredicted: false,
         };
         await deal(initialUrls, (url, update, _index, setLineHeader, push) => {
-            const isExternal = isExternalUrl(url, this.#scope);
+            const matchedScope = findScopeEntry(url, this.#scope, this.#options);
+            const isExternal = matchedScope === null;
             const urlText = isExternal ? c.dim(url.href) : c.cyan(url.href);
             setLineHeader(`%braille% ${urlText}: `);
-            injectScopeAuth(url, this.#scope);
+            if (matchedScope) {
+                injectScopeAuth(url, matchedScope);
+            }
             this.#linkList.add(url);
             this.#linkList.progress(url);
             return async () => {
@@ -551,7 +559,7 @@ class Crawler extends EventEmitter {
                 if (seen.has(key))
                     return false;
                 seen.add(key);
-                if (isExternalUrl(url, this.#scope)) {
+                if (findScopeEntry(url, this.#scope, this.#options) === null) {
                     externalUrls.add(key);
                 }
                 return true;
@@ -576,7 +584,7 @@ class Crawler extends EventEmitter {
      * @returns The scrape result
      */
     async #scrapePage(url, update, metadataOnly, laneIndex) {
-        const isExternal = isExternalUrl(url, this.#scope);
+        const isExternal = findScopeEntry(url, this.#scope, this.#options) === null;
         // Non-HTTP protocols (mailto:, tel:, etc.) — let the scraper handle early return
         if (!url.isHTTP) {
             return this.#launchBrowserAndScrape(url, update, isExternal, metadataOnly);

package/lib/crawler/find-scope-entry.d.ts

@@ -0,0 +1,25 @@
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
+/**
+ * Find the most-specific scope entry that contains the given URL.
+ *
+ * A scope entry is an `(hostname, port, path)` triple. The target URL belongs
+ * to a scope entry when its hostname AND port match and its path is at the
+ * same level or deeper than the scope entry's path. Among all matching entries,
+ * the one with the greatest depth wins (e.g. `/blog/2024/` is preferred over
+ * `/blog/`). Port comparison uses the WHATWG-normalized `port` field, so
+ * default ports (`80` for http, `443` for https) collapse to an empty string
+ * and match each other regardless of whether the user wrote them explicitly.
+ * A non-default port like `:3000` only matches scope entries that also carry
+ * the same port — this prevents credentialed dev sites (`localhost:3000`)
+ * from leaking auth into siblings on the same hostname (`localhost:8080`).
+ *
+ * Returns `null` if no scope entry contains the URL — i.e. the URL is external.
+ * This single function replaces the previous trio of `isExternalUrl`,
+ * `isInAnyLowerLayer`, and `findBestMatchingScope`, performing a single
+ * hostname lookup and a single pass over the scope array.
+ * @param url - The target URL to classify.
+ * @param scope - Hostname-indexed map of scope URLs.
+ * @param options - URL parsing options forwarded to {@link isLowerLayer}.
+ * @returns The deepest matching scope URL, or `null` when the URL is external.
+ */
+export declare function findScopeEntry(url: ExURL, scope: ReadonlyMap<string, readonly ExURL[]>, options?: ParseURLOptions): ExURL | null;

package/lib/crawler/find-scope-entry.js

@@ -0,0 +1,45 @@
+import { isLowerLayer } from '@d-zero/shared/is-lower-layer';
+/**
+ * Find the most-specific scope entry that contains the given URL.
+ *
+ * A scope entry is an `(hostname, port, path)` triple. The target URL belongs
+ * to a scope entry when its hostname AND port match and its path is at the
+ * same level or deeper than the scope entry's path. Among all matching entries,
+ * the one with the greatest depth wins (e.g. `/blog/2024/` is preferred over
+ * `/blog/`). Port comparison uses the WHATWG-normalized `port` field, so
+ * default ports (`80` for http, `443` for https) collapse to an empty string
+ * and match each other regardless of whether the user wrote them explicitly.
+ * A non-default port like `:3000` only matches scope entries that also carry
+ * the same port — this prevents credentialed dev sites (`localhost:3000`)
+ * from leaking auth into siblings on the same hostname (`localhost:8080`).
+ *
+ * Returns `null` if no scope entry contains the URL — i.e. the URL is external.
+ * This single function replaces the previous trio of `isExternalUrl`,
+ * `isInAnyLowerLayer`, and `findBestMatchingScope`, performing a single
+ * hostname lookup and a single pass over the scope array.
+ * @param url - The target URL to classify.
+ * @param scope - Hostname-indexed map of scope URLs.
+ * @param options - URL parsing options forwarded to {@link isLowerLayer}.
+ * @returns The deepest matching scope URL, or `null` when the URL is external.
+ */
+export function findScopeEntry(url, scope, options) {
+    const scopes = scope.get(url.hostname);
+    if (!scopes) {
+        return null;
+    }
+    let bestMatch = null;
+    let maxDepth = -1;
+    for (const entry of scopes) {
+        if (entry.port !== url.port) {
+            continue;
+        }
+        if (!isLowerLayer(url, entry, options)) {
+            continue;
+        }
+        if (entry.depth > maxDepth) {
+            bestMatch = entry;
+            maxDepth = entry.depth;
+        }
+    }
+    return bestMatch;
+}

package/lib/crawler/handle-scrape-end.js

@@ -1,7 +1,6 @@
 import { crawlerLog } from '../debug.js';
+import { findScopeEntry } from './find-scope-entry.js';
 import { injectScopeAuth } from './inject-scope-auth.js';
-import { isExternalUrl } from './is-external-url.js';
-import { isInAnyLowerLayer } from './is-in-any-lower-layer.js';
 /**
  * Process the result of a successful page scrape.
  *
@@ -36,12 +35,14 @@ export function handleScrapeEnd(result, linkList, scope, options, addUrl) {
  * Process anchor elements extracted from a scraped page and enqueue new URLs.
  *
  * For each anchor:
- * 1. Determines if it is external (outside the crawl scope)
- * 2. Injects authentication credentials from matching scope URLs
- * 3. Reconstructs the `withoutHash` URL with injected auth
- * 4. In recursive mode: enqueues internal lower-layer URLs for full scraping,
- *    and external URLs for metadata-only scraping (if `fetchExternal` is enabled)
- * 5. In non-recursive mode: enqueues all URLs for metadata-only scraping
+ * 1. Resolves the matching scope entry via a single {@link findScopeEntry} call.
+ *    If `null`, the anchor is external; otherwise it is internal under the
+ *    deepest matching scope.
+ * 2. For internal anchors without credentials, inherits auth from the matched
+ *    scope and rebuilds `withoutHash` with the injected auth.
+ * 3. In recursive mode: enqueues internal anchors for full scraping, and
+ *    external anchors for metadata-only scraping (when `fetchExternal` is on).
+ * 4. In non-recursive mode: enqueues every anchor for metadata-only scraping.
  * @param anchors - The list of anchor data extracted from the page.
  * @param scope - Map of hostnames to their scope URLs.
  * @param options - Crawler configuration options.
@@ -50,10 +51,11 @@ export function handleScrapeEnd(result, linkList, scope, options, addUrl) {
  */
 function processAnchors(anchors, scope, options, addUrl) {
     for (const anchor of anchors) {
-        const isExternal = isExternalUrl(anchor.href, scope);
+        const matchedScope = findScopeEntry(anchor.href, scope, options);
+        const isExternal = matchedScope === null;
         anchor.isExternal = isExternal;
-        if (!isExternal && (!anchor.href.username || !anchor.href.password)) {
-            injectScopeAuth(anchor.href, scope);
+        if (matchedScope && (!anchor.href.username || !anchor.href.password)) {
+            injectScopeAuth(anchor.href, matchedScope);
             const auth = anchor.href.username && anchor.href.password
                 ? `${anchor.href.username}:${anchor.href.password}@`
                 : '';
@@ -68,11 +70,10 @@ function processAnchors(anchors, scope, options, addUrl) {
             anchor.href.withoutHash = withoutHash;
         }
         if (options.recursive) {
-            const scopes = scope.get(anchor.href.hostname);
-            if (scopes && isInAnyLowerLayer(anchor.href, scopes, options)) {
+            if (matchedScope) {
                 addUrl(anchor.href);
             }
-            else if (isExternal && options.fetchExternal) {
+            else if (options.fetchExternal) {
                 addUrl(anchor.href, { metadataOnly: true });
             }
             continue;

package/lib/crawler/inject-scope-auth.d.ts

@@ -1,11 +1,14 @@
 import type { ExURL } from '@d-zero/shared/parse-url';
 /**
- * Inject authentication credentials from a matching scope URL into the target URL.
+ * Copy `username` / `password` from a matched scope URL into the target URL.
  *
- * Finds the best-matching scope URL (deepest path match) for the given URL's
- * hostname and copies its `username` and `password` properties. This mutates
- * the `url` parameter in place.
- * @param url - The parsed URL to receive authentication credentials (mutated in place).
- * @param scope - Map of hostnames to their scope URLs.
+ * The matched scope is supplied by the caller (typically the result of a single
+ * {@link findScopeEntry} call). This avoids the previous implementation's
+ * redundant hostname lookup and re-search.
+ *
+ * Mutates the `url` parameter in place. Only non-empty credentials overwrite
+ * existing values.
+ * @param url - The parsed URL to receive credentials (mutated in place).
+ * @param matchedScope - The scope URL whose credentials should be inherited.
  */
-export declare function injectScopeAuth(url: ExURL, scope: ReadonlyMap<string, readonly ExURL[]>): void;
+export declare function injectScopeAuth(url: ExURL, matchedScope: ExURL): void;

package/lib/crawler/inject-scope-auth.js

@@ -1,21 +1,20 @@
-import { findBestMatchingScope } from './find-best-matching-scope.js';
 /**
- * Inject authentication credentials from a matching scope URL into the target URL.
+ * Copy `username` / `password` from a matched scope URL into the target URL.
  *
- * Finds the best-matching scope URL (deepest path match) for the given URL's
- * hostname and copies its `username` and `password` properties. This mutates
- * the `url` parameter in place.
- * @param url - The parsed URL to receive authentication credentials (mutated in place).
- * @param scope - Map of hostnames to their scope URLs.
+ * The matched scope is supplied by the caller (typically the result of a single
+ * {@link findScopeEntry} call). This avoids the previous implementation's
+ * redundant hostname lookup and re-search.
+ *
+ * Mutates the `url` parameter in place. Only non-empty credentials overwrite
+ * existing values.
+ * @param url - The parsed URL to receive credentials (mutated in place).
+ * @param matchedScope - The scope URL whose credentials should be inherited.
  */
-export function injectScopeAuth(url, scope) {
-    const scopes = scope.get(url.hostname);
-    if (!scopes) {
-        return;
-    }
-    const matchedScope = findBestMatchingScope(url, scopes);
-    if (matchedScope) {
+export function injectScopeAuth(url, matchedScope) {
+    if (matchedScope.username) {
         url.username = matchedScope.username;
+    }
+    if (matchedScope.password) {
         url.password = matchedScope.password;
     }
 }

package/lib/crawler/is-external-url.d.ts

@@ -1,11 +1,18 @@
-import type { ExURL } from '@d-zero/shared/parse-url';
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
 /**
  * Determine whether a URL is external to the crawl scope.
  *
- * A URL is considered external if its hostname does not appear
- * as a key in the scope map.
+ * A URL is external when no scope entry contains it — i.e. either no scope
+ * entry shares its hostname, or none of the same-hostname scope entries is at
+ * the same level or shallower in the path hierarchy.
+ *
+ * This is a thin wrapper around {@link findScopeEntry}. Callers that already
+ * need the matched scope (for `injectScopeAuth` etc.) should call
+ * `findScopeEntry` directly instead of `isExternalUrl` to avoid a redundant
+ * lookup.
  * @param url - The parsed URL to check.
- * @param scope - Map of hostnames to their scope URLs.
- * @returns `true` if the URL is outside the crawl scope.
+ * @param scope - Hostname-indexed map of scope URLs.
+ * @param options - URL parsing options forwarded to the scope-entry lookup.
+ * @returns `true` if the URL is outside every scope entry.
  */
-export declare function isExternalUrl(url: ExURL, scope: ReadonlyMap<string, readonly ExURL[]>): boolean;
+export declare function isExternalUrl(url: ExURL, scope: ReadonlyMap<string, readonly ExURL[]>, options?: ParseURLOptions): boolean;

package/lib/crawler/is-external-url.js

@@ -1,12 +1,20 @@
+import { findScopeEntry } from './find-scope-entry.js';
 /**
  * Determine whether a URL is external to the crawl scope.
  *
- * A URL is considered external if its hostname does not appear
- * as a key in the scope map.
+ * A URL is external when no scope entry contains it — i.e. either no scope
+ * entry shares its hostname, or none of the same-hostname scope entries is at
+ * the same level or shallower in the path hierarchy.
+ *
+ * This is a thin wrapper around {@link findScopeEntry}. Callers that already
+ * need the matched scope (for `injectScopeAuth` etc.) should call
+ * `findScopeEntry` directly instead of `isExternalUrl` to avoid a redundant
+ * lookup.
  * @param url - The parsed URL to check.
- * @param scope - Map of hostnames to their scope URLs.
- * @returns `true` if the URL is outside the crawl scope.
+ * @param scope - Hostname-indexed map of scope URLs.
+ * @param options - URL parsing options forwarded to the scope-entry lookup.
+ * @returns `true` if the URL is outside every scope entry.
  */
-export function isExternalUrl(url, scope) {
-    return !scope.has(url.hostname);
+export function isExternalUrl(url, scope, options) {
+    return findScopeEntry(url, scope, options) === null;
 }

package/lib/crawler/types.d.ts

@@ -24,8 +24,8 @@ export interface CrawlerOptions extends Required<Pick<ParseURLOptions, 'disableQ
     executablePath: string | null;
     /** Whether to fetch and scrape external (out-of-scope) pages. */
     fetchExternal: boolean;
-    /** List of scope URL strings that define the crawl boundary. */
-    scope: string[];
+    /** Root URL strings that define the crawl boundary. Each root is also a scope entry (a `(hostname, port, path)` triple) — out-of-bound URLs are classified as external. */
+    roots: string[];
     /** Glob patterns for URLs to exclude from crawling. */
     excludes: string[];
     /** Keywords that trigger page exclusion when found in content. */

package/lib/crawler-orchestrator.d.ts

@@ -125,6 +125,27 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * @throws {Error} If the URL list is empty or contains no valid URLs.
      */
     static crawling(url: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback): Promise<CrawlerOrchestrator>;
+    /**
+     * Append a fresh crawl to an existing `.nitpicker` archive.
+     *
+     * The given `newUrls` become additional recursive roots: their `withoutHash`
+     * form is merged into `info.roots` and the crawler picks them up as
+     * starting URLs. Previously-external pages whose URL now falls under
+     * the expanded scope are demoted back to "needs scraping" so the next pass
+     * re-fetches them as full internal pages. A `<archive>.bak` is created
+     * before the crawl and removed on success; if the crawl throws, the backup
+     * is restored to keep the original archive intact.
+     *
+     * List-mode archives (`info.fromList === true`) are rejected because their
+     * pages are all metadata-only and cannot host a recursive append.
+     * @param archivePath - Absolute or relative path to the existing `.nitpicker`.
+     * @param newUrls - New root URLs to add and crawl.
+     * @param options - Optional config overrides applied on top of the archived config.
+     * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @returns The orchestrator instance after the append crawl completes.
+     * @throws {Error} When `newUrls` is empty, the archive is in list mode, or it cannot be parsed.
+     */
+    static append(archivePath: string, newUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback): Promise<CrawlerOrchestrator>;
     /**
      * Resume a previously interrupted crawl from an existing archive file.
      *