@nitpicker/crawler 0.5.0 → 0.6.1

package/lib/archive/database.d.ts

@@ -176,6 +176,11 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
     /**
      * Inserts or updates a crawled page in the database, including its redirect chain,
      * anchors, and images. Optionally creates an HTML snapshot file path entry.
+     *
+     * Self-redirects (where the source URL equals the destination URL after normalization)
+     * are skipped to avoid marking a page as redirected to itself — a situation caused by
+     * authentication challenges (e.g. Basic Auth 302) that would otherwise exclude the page
+     * from reports via the `whereNull('redirectDestId')` filter.
      * @param page - The page data to store.
      * @param snapshotDir - The directory for saving HTML snapshots, or null to skip snapshots.
      * @param isTarget - Whether this page is a crawl target.

package/lib/archive/database.js

@@ -562,10 +562,13 @@ let Database = (() => {
             }
             const [{ id: resourceId }] = selected;
             const pageId = await this.#getIdByUrl(pageUrl);
-            await this.#instance('resources-referrers').insert({
+            await this.#instance('resources-referrers')
+                .insert({
                 resourceId,
                 pageId,
-            });
+            })
+                .onConflict(['resourceId', 'pageId'])
+                .ignore();
         }
         /**
          * Stores the crawl configuration in the `info` table.
@@ -633,6 +636,11 @@ let Database = (() => {
         /**
          * Inserts or updates a crawled page in the database, including its redirect chain,
          * anchors, and images. Optionally creates an HTML snapshot file path entry.
+         *
+         * Self-redirects (where the source URL equals the destination URL after normalization)
+         * are skipped to avoid marking a page as redirected to itself — a situation caused by
+         * authentication challenges (e.g. Basic Auth 302) that would otherwise exclude the page
+         * from reports via the `whereNull('redirectDestId')` filter.
          * @param page - The page data to store.
          * @param snapshotDir - The directory for saving HTML snapshots, or null to skip snapshots.
          * @param isTarget - Whether this page is a crawl target.
@@ -654,7 +662,12 @@ let Database = (() => {
                     ...page,
                     url: destUrlObject,
                 }, isTarget, trx);
+                const destUrlNormalized = destUrlObject.withoutHashAndAuth;
                 for (const redirect of redirectPaths) {
+                    if (redirect === destUrlNormalized) {
+                        dbLog('Skip self-redirect: %s', redirect);
+                        continue;
+                    }
                     dbLog('Set redirected url: %s -> %s', redirect, destUrl);
                     const redirectId = await this.#getIdByUrl(redirect, undefined, trx);
                     await trx('pages')

package/lib/crawler/crawler.js

@@ -369,6 +369,15 @@ class Crawler extends EventEmitter {
             update('Creating page%dots%');
             const page = await browser.newPage();
             await page.setUserAgent(this.#options.userAgent);
+            // Defence-in-depth: beholder sets Authorization via setExtraHTTPHeaders,
+            // but page.authenticate() handles Chromium-level HTTP auth challenges
+            // (401 + WWW-Authenticate) that setExtraHTTPHeaders cannot cover.
+            if (url.username && url.password) {
+                await page.authenticate({
+                    username: url.username,
+                    password: url.password,
+                });
+            }
             const scraper = new Scraper();
             scraper.on('changePhase', (e) => {
                 const msg = formatPhaseLog(e);
@@ -486,7 +495,7 @@ class Crawler extends EventEmitter {
                     const metadataOnly = this.#linkList.isMetadataOnly(url.withoutHash);
                     const isPredicted = this.#linkList.isPredicted(url.withoutHashAndAuth);
                     log('Scraping%dots%');
-                    const result = await this.#scrapePage(url, log, metadataOnly);
+                    const result = await this.#scrapePage(url, log, metadataOnly, _index);
                     // Discard predicted URLs that failed (404, error, etc.)
                     if (isPredicted && shouldDiscardPredicted(result)) {
                         handleIgnoreAndSkip(url, this.#linkList, this.#scope, this.#options);
@@ -563,9 +572,10 @@ class Crawler extends EventEmitter {
      * @param url - Target URL to scrape
      * @param update - Callback for progress messages
      * @param metadataOnly - When true, only extract title metadata without full browser scraping
+     * @param laneIndex - The dealer lane index, used to create unique countdown IDs
      * @returns The scrape result
      */
-    async #scrapePage(url, update, metadataOnly) {
+    async #scrapePage(url, update, metadataOnly, laneIndex) {
         const isExternal = isExternalUrl(url, this.#scope);
         // Non-HTTP protocols (mailto:, tel:, etc.) — let the scraper handle early return
         if (!url.isHTTP) {
@@ -575,7 +585,7 @@ class Crawler extends EventEmitter {
         update('HEAD request%dots%');
         let headCheckResult;
         try {
-            headCheckResult = await this.#sendHeadRequest(url, isExternal, update);
+            headCheckResult = await this.#sendHeadRequest(url, isExternal, update, laneIndex);
         }
         catch (error) {
             // Server unreachable — skip browser launch entirely
@@ -641,14 +651,15 @@ class Crawler extends EventEmitter {
      * @param url - Target URL to check
      * @param isExternal - Whether the URL is external to the crawl scope
      * @param update - Callback for progress messages shown in the dealer display
+     * @param laneIndex - The dealer lane index, used to create unique countdown IDs
      * @returns Lightweight page data from the HEAD response
      */
-    async #sendHeadRequest(url, isExternal, update) {
+    async #sendHeadRequest(url, isExternal, update, laneIndex) {
         return retryCall(() => fetchDestination({ url, isExternal, userAgent: this.#options.userAgent }), {
             retries: this.#options.retry,
             label: 'HEAD request',
             onWait: (determinedInterval, retryCount, label, error) => {
-                update(`${label}: ${error.message} — %countdown(${determinedInterval},fetchHead_${retryCount},s)%s (retry #${retryCount + 1})`);
+                update(`${label}: ${error.message} — %countdown(${determinedInterval},fetchHead_${laneIndex}_${retryCount},s)%s (retry #${retryCount + 1})`);
             },
             onGiveUp: (retryCount, error, label) => {
                 update(c.red(`${label}: gave up after ${retryCount} retries — ${error.message}`));
@@ -700,7 +711,7 @@ function formatPhaseLog(e) {
             return 'HEAD request%dots%';
         }
         case 'openPage': {
-            return e.message;
+            return `Opening page%dots% ${e.message}`;
         }
         case 'loadDOMContent': {
             return c.dim('DOM loaded');

package/lib/crawler/find-best-matching-scope.d.ts

@@ -4,7 +4,8 @@ import type { ExURL } from '@d-zero/shared/parse-url';
  *
  * Among all scope URLs sharing the same hostname, returns the one whose
  * path segments are a prefix of the target URL's path segments and which
- * has the greatest depth. Returns `null` if no scope URL matches.
+ * has the greatest depth. A root scope (paths: `['']`) matches all paths
+ * under the same hostname. Returns `null` if no scope URL matches.
  * @param url - The parsed URL to match against scope URLs.
  * @param scopes - The list of scope URLs to search.
  * @returns The best-matching scope URL, or `null` if none match.

package/lib/crawler/find-best-matching-scope.js

@@ -3,7 +3,8 @@
  *
  * Among all scope URLs sharing the same hostname, returns the one whose
  * path segments are a prefix of the target URL's path segments and which
- * has the greatest depth. Returns `null` if no scope URL matches.
+ * has the greatest depth. A root scope (paths: `['']`) matches all paths
+ * under the same hostname. Returns `null` if no scope URL matches.
  * @param url - The parsed URL to match against scope URLs.
  * @param scopes - The list of scope URLs to search.
  * @returns The best-matching scope URL, or `null` if none match.
@@ -26,14 +27,19 @@ export function findBestMatchingScope(url, scopes) {
 /**
  * Check whether a target path is equal to or is a descendant of a base path.
  *
- * Compares path segments element by element. The target path matches if
- * all segments of the base path appear in the same positions at the
- * beginning of the target path.
+ * A root base path (`['']`) unconditionally matches any target path.
+ * Otherwise, compares path segments element by element — the target path
+ * matches if all segments of the base path appear in the same positions
+ * at the beginning of the target path.
  * @param targetPaths - The path segments of the URL being checked.
  * @param basePaths - The path segments of the scope URL to match against.
  * @returns `true` if the target path starts with or equals the base path.
  */
 function isPathMatch(targetPaths, basePaths) {
+    // Root scope (paths: ['']) matches all paths under the same hostname
+    if (basePaths.length === 1 && basePaths[0] === '') {
+        return true;
+    }
     if (targetPaths.length < basePaths.length) {
         return false;
     }

package/lib/crawler-orchestrator.js

@@ -7,6 +7,7 @@ import Archive from './archive/archive.js';
 import { clearDestinationCache } from './crawler/clear-destination-cache.js';
 import Crawler from './crawler/crawler.js';
 import { crawlerLog, log } from './debug.js';
+import { normalizeToArray } from './normalize-to-array.js';
 import { resolveOutputPath } from './resolve-output-path.js';
 import { cleanObject } from './utils/object/clean-object.js';
 import { WriteQueue } from './write-queue.js';
@@ -314,12 +315,3 @@ export class CrawlerOrchestrator extends EventEmitter {
         return orchestrator;
     }
 }
-/**
- * Normalize an optional parameter that may be a single value, an array,
- * null, or undefined into a guaranteed array.
- * @param param - The parameter to normalize.
- * @returns An array containing the parameter value(s), or an empty array if absent.
- */
-function normalizeToArray(param) {
-    return Array.isArray(param) ? param : param ? [param] : [];
-}

package/lib/normalize-to-array.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Normalize an optional parameter that may be a single value, an array,
+ * null, or undefined into a guaranteed array.
+ * Comma-separated strings are split into individual elements.
+ * Commas inside brace expressions (e.g. `{html,php}`) are preserved.
+ * @param param - The parameter to normalize.
+ * @returns An array containing the parameter value(s), or an empty array if absent.
+ */
+export declare function normalizeToArray(param: string | string[] | null | undefined): string[];

package/lib/normalize-to-array.js

@@ -0,0 +1,49 @@
+/**
+ * Normalize an optional parameter that may be a single value, an array,
+ * null, or undefined into a guaranteed array.
+ * Comma-separated strings are split into individual elements.
+ * Commas inside brace expressions (e.g. `{html,php}`) are preserved.
+ * @param param - The parameter to normalize.
+ * @returns An array containing the parameter value(s), or an empty array if absent.
+ */
+export function normalizeToArray(param) {
+    if (!param)
+        return [];
+    const arr = Array.isArray(param) ? param : [param];
+    return arr.flatMap((item) => splitByTopLevelComma(item));
+}
+/**
+ * Split a string by commas that are not inside brace expressions (`{}`).
+ * @param input - The string to split.
+ * @returns An array of trimmed, non-empty segments.
+ */
+function splitByTopLevelComma(input) {
+    const segments = [];
+    let current = '';
+    let depth = 0;
+    for (const ch of input) {
+        if (ch === '{') {
+            depth++;
+            current += ch;
+        }
+        else if (ch === '}') {
+            depth = Math.max(0, depth - 1);
+            current += ch;
+        }
+        else if (ch === ',' && depth === 0) {
+            const trimmed = current.trim();
+            if (trimmed) {
+                segments.push(trimmed);
+            }
+            current = '';
+        }
+        else {
+            current += ch;
+        }
+    }
+    const trimmed = current.trim();
+    if (trimmed) {
+        segments.push(trimmed);
+    }
+    return segments;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.5.0",
+	"version": "0.6.1",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -48,5 +48,5 @@
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "607d06bd596a0270d7088f32373d7367eb47ea94"
+	"gitHead": "bc3a766e87f91560268097b99048b06341cd59cd"
 }