@govtechsg/oobee 0.10.92 → 0.10.93

package/AGENTS.md

@@ -79,6 +79,7 @@ All crawlers use Crawlee's `PlaywrightCrawler` with:
 - Docker detection (`/.dockerenv`): adds `--disable-gpu`, `--no-sandbox`, `--disable-dev-shm-usage`
 - Proxy support (manual, PAC, or none) via `getProxyInfo()`
 - Channel set from browser name (undefined for chromium = bundled)
+- `--mute-audio` is added by default in both headless and headful modes, but must be disabled for `customFlow` by calling `getPlaywrightLaunchOptions(browser, { includeMuteAudio: false })`
 
 ### User-Agent
 
@@ -134,6 +135,11 @@ The `constants` default export object holds runtime state:
 | `OOBEE_SLOWMO` | Browser slowmo in ms |
 | `OOBEE_FAST_CRAWLER` | Experimental high-concurrency mode |
 | `OOBEE_DISABLE_BROWSER_DOWNLOAD` | Block browser file downloads |
+| `OOBEE_TAGGED_WEBSITE` | Tag to identify the website in Sentry telemetry (overridden by `--websiteTag` CLI flag) |
+| `OOBEE_SCAN_METADATA` | Overrides `entryUrl` tag in Sentry events |
+| `OOBEE_SCAN_PRODUCT` | Adds `scanProduct` tag to Sentry events |
+| `OOBEE_CONSECUTIVE_MAX_RETRIES` | Max consecutive HTTP failures before circuit breaker aborts crawl (default 100) |
+| `OOBEE_VALIDATE_URL` | If set, exit after URL validation without scanning |
 | `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` | Proxy configuration |
 | `NO_PROXY` / `INCLUDE_PROXY` | Proxy bypass/include lists |
 
@@ -215,6 +221,14 @@ docker run oobee node dist/cli.js ...
 
 8. **Crawlee dataset** — Results are stored as numbered JSON files in `{randomToken}/datasets/default/`. Each file is one page's axe results. `generateArtifacts()` reads all of them.
 
+9. **Auth headers and CORS** — Never set `Authorization` in `extraHTTPHeaders` globally on a browser context. Playwright sends `extraHTTPHeaders` to ALL requests (including cross-origin CDNs), which triggers CORS preflight failures. Instead use `splitAuthHeaders()` from `commonCrawlerFunc.ts` to separate auth from non-auth headers:
+    - Non-auth headers → safe to set globally via `extraHTTPHeaders` on context/launch options
+    - Basic auth → set `httpCredentials` on context (Playwright auto-responds to 401 challenges, origin-aware)
+    - Any Authorization header → send only to same-origin requests via `addAuthRouteHandler()` (route interception) or Crawlee's `preNavigationHooks` (navigation-only)
+    - Credentials come from URL-embedded `user:pass@host` or `-m "Authorization Basic ..."` — both produce the same `extraHTTPHeaders.Authorization` value in `prepareData()`
+
+10. **Intermediate JSONL write safety + corruption tolerance** — `ItemsStore.appendPageItems()` requires strict serialization of writes per rule file to prevent interleaved corruption. It also enforces a strict text sanitization regex to filter out literal `\n` and `\r` control characters from website HTML inputs immediately after `JSON.stringify()`. This ensures no single JSON issue accidentally injects illegal implicit newline boundaries when writing to JSONL format. Maintain backward-compatible `fs.appendFile` queues over buffered WriteStreams to guarantee pipeline sync visibility. `ItemsStore.readRuleItems()` tolerates historical malformed lines via fallback skip logic.
+
 ## Testing Considerations
 
 When making changes, validate these areas which have well-established edge cases:

package/README.md

@@ -92,6 +92,10 @@ verapdf --version
 | WARN_LEVEL | Only used in tests. |  |
 | OOBEE_DISABLE_BROWSER_DOWNLOAD | Experimental flag to disable file downloads on Chrome/Chromium/Edge.  Does not affect Local File scan | |
 | OOBEE_SLOWMO | Experimental flag to slow down web browser behaviour by specified duration (in miliseconds) | |
+| OOBEE_TAGGED_WEBSITE | Tag to identify the website in telemetry. Can also be set via `-z, --websiteTag` CLI flag (CLI flag takes precedence). | |
+| OOBEE_SCAN_METADATA | Overrides the `entryUrl` tag sent to telemetry. | |
+| OOBEE_SCAN_PRODUCT | Adds a `scanProduct` tag to telemetry events. | |
+| OOBEE_CONSECUTIVE_MAX_RETRIES | Max consecutive HTTP failures before the circuit breaker aborts the crawl. | `100` |
 | HTTP_PROXY | URL of the proxy server to be used for HTTP requests (e.g. `http://proxy.example.com:8080`). | |
 | HTTPS_PROXY | URL of the proxy server to be used for HTTPS requests (e.g. `https://proxy.example.com:8080`). | |
 | ALL_PROXY | URL of the proxy server to be used for all requests, typically used for SOCKS5 proxies (e.g. `socks5://proxy.example.com:1080`. Note: IPv6 direct connections may still continue even though socks5 proxy is specified due to a known issue with Chrome/Chromium. (Recommended workaround is to turn off IPv6 at host-level). | |
@@ -413,6 +417,21 @@ Examples:
   > [ -d <device> | -w <viewport_width> ]
 
 ```
+
+### Basic Auth
+
+For sites behind HTTP Basic Authentication, you can provide credentials in two ways:
+
+1. **Embed in URL**: `npm run cli -- -u 'https://user:password@example.com' -c 3`
+2. **Use `-m` flag**: `npm run cli -- -u 'https://example.com' -c 3 -m "Authorization Basic dXNlcjpwYXNzd29yZA=="`
+
+Both methods work across all scan types (sitemap, website, custom flow). For multiple headers, separate with `, `:
+```
+-m "Authorization Basic dXNlcjpwYXNz, X-Custom-Header myvalue"
+```
+
+> **Note:** Authorization headers are only sent to same-origin requests to avoid CORS preflight failures on cross-origin resources (e.g., CDN fonts, analytics scripts).
+
 ### Note on Windows PowerShell:
 You need to run the command as `npm run cli -- --` (with the extra set of `--`) as PowerShell interprets arguments differently.
 

package/dist/combine.js

@@ -89,7 +89,7 @@ const combineRun = async (details, deviceToScan) => {
     let durationExceeded = false;
     switch (type) {
         case ScannerTypes.CUSTOM:
-            const res = await runCustom(url, randomToken, browser, userDataDirectory, viewportSettings, blacklistedPatterns, includeScreenshots, customFlowLabel && customFlowLabel !== 'None' ? customFlowLabel : '');
+            const res = await runCustom(url, randomToken, browser, userDataDirectory, viewportSettings, blacklistedPatterns, includeScreenshots, customFlowLabel && customFlowLabel !== 'None' ? customFlowLabel : '', extraHTTPHeaders);
             urlsCrawledObj = res.urlsCrawled;
             uiCustomFlowLabel = res.customFlowLabel;
             break;

package/dist/constants/common.js

@@ -300,9 +300,19 @@ const checkUrlConnectivityWithBrowser = async (url, browserToRun, clonedDataDir,
     const rawDevice = (playwrightDeviceDetailsObject || {});
     const { viewport, isMobile, hasTouch, userAgent: deviceUserAgent, ...restDevice } = rawDevice;
     const launchOptions = getPlaywrightLaunchOptions(browserToRun);
+    const { Authorization, ...nonAuthHeaders } = extraHTTPHeaders || {};
+    let httpCredentials = undefined;
+    if (Authorization?.startsWith('Basic ')) {
+        const decoded = Buffer.from(Authorization.slice(6), 'base64').toString();
+        const colonIdx = decoded.indexOf(':');
+        if (colonIdx > 0) {
+            httpCredentials = { username: decoded.slice(0, colonIdx), password: decoded.slice(colonIdx + 1) };
+        }
+    }
     const contextOptions = {
         ...restDevice,
-        ...(extraHTTPHeaders && { extraHTTPHeaders }),
+        ...(Object.keys(nonAuthHeaders).length > 0 && { extraHTTPHeaders: nonAuthHeaders }),
+        ...(httpCredentials && { httpCredentials }),
         ignoreHTTPSErrors: true,
         ...(process.env.OOBEE_DISABLE_BROWSER_DOWNLOAD && { acceptDownloads: false }),
     };
@@ -342,6 +352,25 @@ const checkUrlConnectivityWithBrowser = async (url, browserToRun, clonedDataDir,
         return res;
     }
     try {
+        // Only enable generic Authorization header routing interception broadly if 
+        // a non-Basic Bearer auth string is heavily relied upon, thereby bypassing 
+        // performance warnings inside the check checkUrl phase for typical public scans
+        if (Authorization && !httpCredentials) {
+            const entryOrigin = new URL(url).origin;
+            await browserContext.route('**/*', async (route, request) => {
+                try {
+                    if (new URL(request.url()).origin === entryOrigin) {
+                        await route.continue({ headers: { ...request.headers(), Authorization } });
+                    }
+                    else {
+                        await route.continue();
+                    }
+                }
+                catch {
+                    await route.continue();
+                }
+            });
+        }
         const page = await browserContext.newPage();
         // Block native Chrome download UI
         try {
@@ -351,15 +380,6 @@ const checkUrlConnectivityWithBrowser = async (url, browserToRun, clonedDataDir,
         catch (e) {
             consoleLogger.info(`Unable to set download deny: ${e.message}`);
         }
-        // OPTIMIZATION: Block heavy visual resources (Images/Fonts/CSS)
-        // This allows the "Connectivity Check" to pass as soon as HTML is ready
-        await page.route('**/*', (route) => {
-            const type = route.request().resourceType();
-            if (['image', 'media', 'font', 'stylesheet'].includes(type)) {
-                return route.abort();
-            }
-            return route.continue();
-        });
         // STEP 2: Navigate (follows server-side redirects)
         page.once('download', () => {
             res.status = constants.urlCheckStatuses.notASupportedDocument.code;

package/dist/crawlers/commonCrawlerFunc.js

@@ -905,6 +905,49 @@ export const preNavigationHooks = (extraHTTPHeaders) => {
         },
     ];
 };
+/**
+ * Splits extraHTTPHeaders into auth and non-auth parts.
+ * Auth headers (Authorization) must only be sent to same-origin requests to avoid CORS preflight failures.
+ * Non-auth headers are safe to set globally on the browser context.
+ */
+export const splitAuthHeaders = (extraHTTPHeaders) => {
+    const { Authorization, ...nonAuthHeaders } = extraHTTPHeaders || {};
+    return {
+        authHeader: Authorization || null,
+        nonAuthHeaders: Object.keys(nonAuthHeaders).length > 0 ? nonAuthHeaders : null,
+        httpCredentials: (() => {
+            if (!Authorization?.startsWith('Basic '))
+                return null;
+            const decoded = Buffer.from(Authorization.slice(6), 'base64').toString();
+            const colonIdx = decoded.indexOf(':');
+            if (colonIdx <= 0)
+                return null;
+            return { username: decoded.slice(0, colonIdx), password: decoded.slice(colonIdx + 1) };
+        })(),
+    };
+};
+/**
+ * Adds a route handler to a BrowserContext that sends the Authorization header
+ * only to same-origin requests, preventing CORS preflight failures on cross-origin CDN resources.
+ */
+export const addAuthRouteHandler = async (context, entryUrl, authHeader) => {
+    if (!authHeader)
+        return;
+    const entryOrigin = new URL(entryUrl).origin;
+    await context.route('**/*', async (route, request) => {
+        try {
+            if (new URL(request.url()).origin === entryOrigin) {
+                await route.continue({ headers: { ...request.headers(), Authorization: authHeader } });
+            }
+            else {
+                await route.continue();
+            }
+        }
+        catch {
+            await route.continue();
+        }
+    });
+};
 export const postNavigationHooks = [
     async (_crawlingContext) => {
         guiInfoLog(guiInfoStatusTypes.COMPLETED, {});

package/dist/crawlers/crawlDomain.js

@@ -1,6 +1,6 @@
 import crawlee from 'crawlee';
 import { CrawlRateController } from './crawlRateController.js';
-import { createCrawleeSubFolders, getPreLaunchHook, runAxeScript, isUrlPdf, shouldSkipClickDueToDisallowedHref, shouldSkipDueToUnsupportedContent, } from './commonCrawlerFunc.js';
+import { createCrawleeSubFolders, getPreLaunchHook, runAxeScript, isUrlPdf, shouldSkipClickDueToDisallowedHref, shouldSkipDueToUnsupportedContent, splitAuthHeaders, } from './commonCrawlerFunc.js';
 import constants, { blackListedFileExtensions, guiInfoStatusTypes, cssQuerySelectors, STATUS_CODE_METADATA, disallowedListOfPatterns, disallowedSelectorPatterns, FileTypes, } from '../constants/constants.js';
 import { getPlaywrightLaunchOptions, isBlacklistedFileExtensions, isSkippedUrl, isDisallowedInRobotsTxt, getUrlsFromRobotsTxt, waitForPageLoaded, } from '../constants/common.js';
 import { areLinksEqual, isFollowStrategy, isSameHostname, normUrl, register } from '../utils.js';
@@ -275,6 +275,7 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
     };
     let isAbortingScanNow = false;
     const rateController = new CrawlRateController(maxRequestsPerCrawl, specifiedMaxConcurrency || constants.maxConcurrency);
+    const { nonAuthHeaders, httpCredentials } = splitAuthHeaders(extraHTTPHeaders);
     const crawler = register(new crawlee.PlaywrightCrawler({
         launchContext: {
             launcher: constants.launcher,
@@ -293,12 +294,20 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                         ...playwrightDeviceDetailsObject,
                         ...(process.env.OOBEE_USER_AGENT && { userAgent: process.env.OOBEE_USER_AGENT }),
                         ...(process.env.OOBEE_DISABLE_BROWSER_DOWNLOAD && { acceptDownloads: false }),
-                        ...(extraHTTPHeaders && { extraHTTPHeaders }),
+                        ...(nonAuthHeaders && { extraHTTPHeaders: nonAuthHeaders }),
+                        ...(httpCredentials && { httpCredentials }),
                     };
                 },
             ],
         },
         requestQueue,
+        preNavigationHooks: [
+            async (crawlingContext) => {
+                if (extraHTTPHeaders) {
+                    crawlingContext.request.headers = extraHTTPHeaders;
+                }
+            },
+        ],
         postNavigationHooks: [
             async (crawlingContext) => {
                 const { page, request } = crawlingContext;

package/dist/crawlers/crawlIntelligentSitemap.js

@@ -1,4 +1,4 @@
-import { createCrawleeSubFolders } from './commonCrawlerFunc.js';
+import { createCrawleeSubFolders, splitAuthHeaders, addAuthRouteHandler } from './commonCrawlerFunc.js';
 import constants, { guiInfoStatusTypes, sitemapPaths } from '../constants/constants.js';
 import { consoleLogger, guiInfoLog } from '../logs.js';
 import crawlDomain from './crawlDomain.js';
@@ -26,26 +26,31 @@ const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings,
         const homeUrl = getHomeUrl(link);
         let sitemapLink = '';
         const launchOptions = getPlaywrightLaunchOptions(browser);
+        const { authHeader, nonAuthHeaders, httpCredentials } = splitAuthHeaders(extraHTTPHeaders);
         let context;
         let browserInstance;
         if (process.env.CRAWLEE_HEADLESS === '1') {
             const effectiveUserDataDirectory = userDataDirectory || '';
             context = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, {
                 ...launchOptions,
-                ...(extraHTTPHeaders && { extraHTTPHeaders }),
+                ...(nonAuthHeaders && { extraHTTPHeaders: nonAuthHeaders }),
+                ...(httpCredentials && { httpCredentials }),
                 ...(process.env.OOBEE_USER_AGENT && { userAgent: process.env.OOBEE_USER_AGENT }),
             });
             register(context);
         }
         else {
-            // In headful mode, avoid launchPersistentContext to prevent "Browser window not found"
             browserInstance = await constants.launcher.launch(launchOptions);
             register(browserInstance);
             context = await browserInstance.newContext({
-                ...(extraHTTPHeaders && { extraHTTPHeaders }),
+                ...(nonAuthHeaders && { extraHTTPHeaders: nonAuthHeaders }),
+                ...(httpCredentials && { httpCredentials }),
                 ...(process.env.OOBEE_USER_AGENT && { userAgent: process.env.OOBEE_USER_AGENT }),
             });
         }
+        if (authHeader) {
+            await addAuthRouteHandler(context, link, authHeader);
+        }
         const page = await context.newPage();
         for (const path of sitemapPaths) {
             sitemapLink = homeUrl + path;

package/dist/crawlers/runCustom.js

@@ -1,5 +1,5 @@
 /* eslint-env browser */
-import { createCrawleeSubFolders } from './commonCrawlerFunc.js';
+import { createCrawleeSubFolders, splitAuthHeaders, addAuthRouteHandler } from './commonCrawlerFunc.js';
 import { cleanUpAndExit, register, registerSoftClose } from '../utils.js';
 import constants, { getIntermediateScreenshotsPath, guiInfoStatusTypes, } from '../constants/constants.js';
 import { initNewPage, log } from './custom/utils.js';
@@ -18,7 +18,7 @@ export class ProcessPageParams {
         this.randomToken = randomToken;
     }
 }
-const runCustom = async (url, randomToken, browserToRun, userDataDirectory, viewportSettings, blacklistedPatterns, includeScreenshots, initialCustomFlowLabel) => {
+const runCustom = async (url, randomToken, browserToRun, userDataDirectory, viewportSettings, blacklistedPatterns, includeScreenshots, initialCustomFlowLabel, extraHTTPHeaders) => {
     // checks and delete datasets path if it already exists
     process.env.CRAWLEE_STORAGE_DIR = randomToken;
     const urlsCrawled = { ...constants.urlsCrawledObj };
@@ -47,6 +47,7 @@ const runCustom = async (url, randomToken, browserToRun, userDataDirectory, view
             ...baseArgs.filter(a => !a.startsWith('--window-size') && a !== '--start-maximized'),
             ...customArgs,
         ];
+        const { authHeader, nonAuthHeaders, httpCredentials } = splitAuthHeaders(extraHTTPHeaders);
         const context = await constants.launcher.launchPersistentContext(userDataDirectory, {
             ...baseLaunchOptions,
             args: mergedArgs,
@@ -56,7 +57,12 @@ const runCustom = async (url, randomToken, browserToRun, userDataDirectory, view
             viewport: null,
             ...(hasCustomViewport ? contextDeviceOptions : {}),
             userAgent: process.env.OOBEE_USER_AGENT || deviceUserAgent,
+            ...(nonAuthHeaders && { extraHTTPHeaders: nonAuthHeaders }),
+            ...(httpCredentials && { httpCredentials }),
         });
+        if (authHeader) {
+            await addAuthRouteHandler(context, url, authHeader);
+        }
         register(context);
         processPageParams.stopAll = async () => {
             try {

package/dist/generateOobeeClientScanner.js

@@ -51,7 +51,7 @@ const SENTRY_NODE_VERSION = (() => {
         return _require('@sentry/node/package.json').version;
     }
     catch {
-        return '9.47.1'; // safe fallback matching currently installed version
+        return '10.58.0'; // safe fallback matching currently installed version
     }
 })();
 // ---------------------------------------------------------------------------

package/dist/mergeAxeResults/itemsStore.js

@@ -1,9 +1,11 @@
 import fs from 'fs-extra';
 import path from 'path';
 import readline from 'readline';
+import { consoleLogger } from '../logs.js';
 export class ItemsStore {
     constructor(storagePath) {
         this.ensuredDirs = new Set();
+        this.fileWriteQueues = new Map();
         this.basePath = path.join(storagePath, 'tmp-items');
     }
     sanitizeRuleId(ruleId) {
@@ -22,8 +24,25 @@ export class ItemsStore {
     async appendPageItems(category, ruleId, entry) {
         await this.ensureDir(category);
         const filePath = this.getRuleFilePath(category, ruleId);
-        const line = JSON.stringify(entry) + '\n';
-        await fs.appendFile(filePath, line, 'utf8');
+        let line = JSON.stringify(entry);
+        // JSON.stringify should never produce literal newlines inside strings, but HTML content
+        // from page evaluation may contain edge-case characters (e.g. unescaped control chars in
+        // non-spec-compliant innerHTML). Strip any embedded \r or \n that would break JSONL format readline parsing.
+        line = line.replace(/[\n\r]/g, (match) => {
+            if (match === '\n')
+                return '\\n';
+            if (match === '\r')
+                return '\\r';
+            return match;
+        });
+        line += '\n';
+        // Serialize writes per rule file to avoid concurrent append interleaving/truncation.
+        const previous = this.fileWriteQueues.get(filePath) ?? Promise.resolve();
+        const next = previous.then(() => fs.appendFile(filePath, line, 'utf8'));
+        this.fileWriteQueues.set(filePath, next.catch(() => {
+            // Keep queue alive for subsequent writes.
+        }));
+        await next;
     }
     async *readRuleItems(category, ruleId) {
         const filePath = this.getRuleFilePath(category, ruleId);
@@ -31,10 +50,19 @@ export class ItemsStore {
             return;
         const fileStream = fs.createReadStream(filePath, { encoding: 'utf8' });
         const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
+        let lineNumber = 0;
         for await (const line of rl) {
-            if (line.trim()) {
+            lineNumber += 1;
+            if (!line.trim())
+                continue;
+            try {
                 yield JSON.parse(line);
             }
+            catch (error) {
+                // Tolerate malformed/truncated JSONL lines (e.g. interrupted append) so report generation can continue.
+                const preview = line.slice(0, 200);
+                consoleLogger.warn(`Skipping malformed itemsStore JSONL line ${lineNumber} in ${filePath}: ${error.message}. Content preview: ${preview}`);
+            }
         }
     }
     async readRuleItemsMap(category, ruleId) {
@@ -46,6 +74,7 @@ export class ItemsStore {
         return map;
     }
     async cleanup() {
+        await Promise.all(this.fileWriteQueues.values());
         await fs.rm(this.basePath, { recursive: true, force: true });
     }
 }

package/oobee-client-scanner.js

@@ -3,9 +3,9 @@
  * DO NOT EDIT MANUALLY. Re-generate with: node dist/generateOobeeClientScanner.js
  *
  * Embedded at generation time:
- *   App version : 0.10.92
+ *   App version : 0.10.93
  *   Sentry DSN  : (from OOBEE_SENTRY_DSN env var or constants.ts default)
- *   Sentry SDK  : @sentry/browser 9.47.1 (loaded from CDN at runtime)
+ *   Sentry SDK  : @sentry/browser 10.58.0 (loaded from CDN at runtime)
  *
  * Usage:
  *   <script src="oobee-client-scanner.js"></script>
@@ -34883,8 +34883,8 @@
   // ── Sentry browser telemetry (Sentry JS SDK, loaded from CDN) ────────────
   
   var _oobeeSentryDsn          = "https://3b8c7ee46b06f33815a1301b6713ebc3@o4509047624761344.ingest.us.sentry.io/4509327783559168";
-  var _oobeeAppVersion         = "0.10.92";
-  var _oobeeSentryVersion      = "9.47.1";
+  var _oobeeAppVersion         = "0.10.93";
+  var _oobeeSentryVersion      = "10.58.0";
   var _oobeeSentryInitialized  = false;
   var _oobeeSentryLoadPromise  = null;
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.92",
+  "version": "0.10.93",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "bin": {
@@ -11,7 +11,7 @@
     "@aws-sdk/client-s3": "^3.1049.0",
     "@json2csv/node": "^7.0.3",
     "@napi-rs/canvas": "^0.1.53",
-    "@sentry/node": "^9.13.0",
+    "@sentry/node": "^10.58.0",
     "@types/aws-sdk": "^0.0.42",
     "axe-core": "^4.11.4",
     "axios": "^1.8.2",

package/src/combine.ts

@@ -154,6 +154,7 @@ const combineRun = async (details: Data, deviceToScan: string) => {
         blacklistedPatterns,
         includeScreenshots,
         customFlowLabel && customFlowLabel !== 'None' ? customFlowLabel : '',
+        extraHTTPHeaders,
       );
 
       urlsCrawledObj = res.urlsCrawled;

package/src/constants/common.ts

@@ -377,9 +377,21 @@ const checkUrlConnectivityWithBrowser = async (
   } = rawDevice;
 
   const launchOptions = getPlaywrightLaunchOptions(browserToRun);
+  
+  const { Authorization, ...nonAuthHeaders } = extraHTTPHeaders || {};
+  let httpCredentials = undefined;
+  if (Authorization?.startsWith('Basic ')) {
+    const decoded = Buffer.from(Authorization.slice(6), 'base64').toString();
+    const colonIdx = decoded.indexOf(':');
+    if (colonIdx > 0) {
+      httpCredentials = { username: decoded.slice(0, colonIdx), password: decoded.slice(colonIdx + 1) };
+    }
+  }
+
   const contextOptions: Record<string, unknown> = {
     ...restDevice,
-    ...(extraHTTPHeaders && { extraHTTPHeaders }),
+    ...(Object.keys(nonAuthHeaders).length > 0 && { extraHTTPHeaders: nonAuthHeaders }),
+    ...(httpCredentials && { httpCredentials }),
     ignoreHTTPSErrors: true,
     ...(process.env.OOBEE_DISABLE_BROWSER_DOWNLOAD && { acceptDownloads: false }),
   };
@@ -421,6 +433,24 @@ const checkUrlConnectivityWithBrowser = async (
   }
 
   try {
+    // Only enable generic Authorization header routing interception broadly if 
+    // a non-Basic Bearer auth string is heavily relied upon, thereby bypassing 
+    // performance warnings inside the check checkUrl phase for typical public scans
+    if (Authorization && !httpCredentials) {
+      const entryOrigin = new URL(url).origin;
+      await browserContext.route('**/*', async (route: any, request: any) => {
+        try {
+          if (new URL(request.url()).origin === entryOrigin) {
+            await route.continue({ headers: { ...request.headers(), Authorization } });
+          } else {
+            await route.continue();
+          }
+        } catch {
+          await route.continue();
+        }
+      });
+    }
+
     const page = await browserContext.newPage();
 
     // Block native Chrome download UI
@@ -431,16 +461,6 @@ const checkUrlConnectivityWithBrowser = async (
       consoleLogger.info(`Unable to set download deny: ${(e as Error).message}`);
     }
 
-    // OPTIMIZATION: Block heavy visual resources (Images/Fonts/CSS)
-    // This allows the "Connectivity Check" to pass as soon as HTML is ready
-    await page.route('**/*', (route) => {
-      const type = route.request().resourceType();
-      if (['image', 'media', 'font', 'stylesheet'].includes(type)) {
-        return route.abort(); 
-      }
-      return route.continue();
-    });
-
     // STEP 2: Navigate (follows server-side redirects)
     page.once('download', () => {
       res.status = constants.urlCheckStatuses.notASupportedDocument.code;

package/src/crawlers/commonCrawlerFunc.ts

@@ -1153,6 +1153,51 @@ export const preNavigationHooks = (extraHTTPHeaders: Record<string, string>) =>
   ];
 };
 
+/**
+ * Splits extraHTTPHeaders into auth and non-auth parts.
+ * Auth headers (Authorization) must only be sent to same-origin requests to avoid CORS preflight failures.
+ * Non-auth headers are safe to set globally on the browser context.
+ */
+export const splitAuthHeaders = (extraHTTPHeaders?: Record<string, string>) => {
+  const { Authorization, ...nonAuthHeaders } = extraHTTPHeaders || {};
+  return {
+    authHeader: Authorization || null,
+    nonAuthHeaders: Object.keys(nonAuthHeaders).length > 0 ? nonAuthHeaders : null,
+    httpCredentials: (() => {
+      if (!Authorization?.startsWith('Basic ')) return null;
+      const decoded = Buffer.from(Authorization.slice(6), 'base64').toString();
+      const colonIdx = decoded.indexOf(':');
+      if (colonIdx <= 0) return null;
+      return { username: decoded.slice(0, colonIdx), password: decoded.slice(colonIdx + 1) };
+    })(),
+  };
+};
+
+/**
+ * Adds a route handler to a BrowserContext that sends the Authorization header
+ * only to same-origin requests, preventing CORS preflight failures on cross-origin CDN resources.
+ */
+export const addAuthRouteHandler = async (
+  context: BrowserContext,
+  entryUrl: string,
+  authHeader: string | null
+) => {
+  if (!authHeader) return;
+
+  const entryOrigin = new URL(entryUrl).origin;
+  await context.route('**/*', async (route, request) => {
+    try {
+      if (new URL(request.url()).origin === entryOrigin) {
+        await route.continue({ headers: { ...request.headers(), Authorization: authHeader } });
+      } else {
+        await route.continue();
+      }
+    } catch {
+      await route.continue();
+    }
+  });
+};
+
 export const postNavigationHooks = [
   async (_crawlingContext: CrawlingContext) => {
     guiInfoLog(guiInfoStatusTypes.COMPLETED, {});

package/src/crawlers/crawlDomain.ts

@@ -9,6 +9,7 @@ import {
   isUrlPdf,
   shouldSkipClickDueToDisallowedHref,
   shouldSkipDueToUnsupportedContent,
+  splitAuthHeaders,
 } from './commonCrawlerFunc.js';
 import constants, {
   UrlsCrawled,
@@ -385,6 +386,8 @@ const crawlDomain = async ({
     specifiedMaxConcurrency || constants.maxConcurrency,
   );
 
+  const { nonAuthHeaders, httpCredentials } = splitAuthHeaders(extraHTTPHeaders);
+
   const crawler = register(
     new crawlee.PlaywrightCrawler({
       launchContext: {
@@ -404,12 +407,20 @@ const crawlDomain = async ({
               ...playwrightDeviceDetailsObject,
               ...(process.env.OOBEE_USER_AGENT && { userAgent: process.env.OOBEE_USER_AGENT }),
               ...(process.env.OOBEE_DISABLE_BROWSER_DOWNLOAD && { acceptDownloads: false }),
-              ...(extraHTTPHeaders && { extraHTTPHeaders }),
+              ...(nonAuthHeaders && { extraHTTPHeaders: nonAuthHeaders }),
+              ...(httpCredentials && { httpCredentials }),
             };
           },
         ],
       },
       requestQueue,
+      preNavigationHooks: [
+        async (crawlingContext) => {
+          if (extraHTTPHeaders) {
+            crawlingContext.request.headers = extraHTTPHeaders;
+          }
+        },
+      ],
       postNavigationHooks: [
         async crawlingContext => {
           const { page, request } = crawlingContext;

package/src/crawlers/crawlIntelligentSitemap.ts

@@ -1,7 +1,7 @@
 import fs from 'fs';
 import { chromium, Page } from 'playwright';
 import { EnqueueStrategy } from 'crawlee';
-import { createCrawleeSubFolders } from './commonCrawlerFunc.js';
+import { createCrawleeSubFolders, splitAuthHeaders, addAuthRouteHandler } from './commonCrawlerFunc.js';
 import constants, { FileTypes, guiInfoStatusTypes, sitemapPaths } from '../constants/constants.js';
 import { consoleLogger, guiInfoLog } from '../logs.js';
 import crawlDomain from './crawlDomain.js';
@@ -58,6 +58,7 @@ const crawlIntelligentSitemap = async (
     let sitemapLink = '';
 
     const launchOptions = getPlaywrightLaunchOptions(browser);
+    const { authHeader, nonAuthHeaders, httpCredentials } = splitAuthHeaders(extraHTTPHeaders);
     let context;
     let browserInstance;
 
@@ -65,20 +66,25 @@ const crawlIntelligentSitemap = async (
       const effectiveUserDataDirectory = userDataDirectory || '';
       context = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, {
         ...launchOptions,
-        ...(extraHTTPHeaders && { extraHTTPHeaders }),
+        ...(nonAuthHeaders && { extraHTTPHeaders: nonAuthHeaders }),
+        ...(httpCredentials && { httpCredentials }),
         ...(process.env.OOBEE_USER_AGENT && { userAgent: process.env.OOBEE_USER_AGENT }),
       });
       register(context);
     } else {
-      // In headful mode, avoid launchPersistentContext to prevent "Browser window not found"
       browserInstance = await constants.launcher.launch(launchOptions);
       register(browserInstance as unknown as { close: () => Promise<void> });
       context = await browserInstance.newContext({
-        ...(extraHTTPHeaders && { extraHTTPHeaders }),
+        ...(nonAuthHeaders && { extraHTTPHeaders: nonAuthHeaders }),
+        ...(httpCredentials && { httpCredentials }),
         ...(process.env.OOBEE_USER_AGENT && { userAgent: process.env.OOBEE_USER_AGENT }),
       });
     }
 
+    if (authHeader) {
+      await addAuthRouteHandler(context, link, authHeader);
+    }
+
     const page = await context.newPage();
 
     for (const path of sitemapPaths) {

package/src/crawlers/runCustom.ts

@@ -1,5 +1,5 @@
 /* eslint-env browser */
-import { createCrawleeSubFolders } from './commonCrawlerFunc.js';
+import { createCrawleeSubFolders, splitAuthHeaders, addAuthRouteHandler } from './commonCrawlerFunc.js';
 import { cleanUpAndExit, register, registerSoftClose } from '../utils.js';
 import constants, {
   getIntermediateScreenshotsPath,
@@ -60,6 +60,7 @@ const runCustom = async (
   blacklistedPatterns: string[] | null,
   includeScreenshots: boolean,
   initialCustomFlowLabel?: string,
+  extraHTTPHeaders?: Record<string, string>,
 ) => {
   // checks and delete datasets path if it already exists
   process.env.CRAWLEE_STORAGE_DIR = randomToken;
@@ -109,6 +110,8 @@ const runCustom = async (
       ...customArgs,
     ];
 
+    const { authHeader, nonAuthHeaders, httpCredentials } = splitAuthHeaders(extraHTTPHeaders);
+
     const context = await constants.launcher.launchPersistentContext(userDataDirectory, {
       ...baseLaunchOptions,
       args: mergedArgs,
@@ -118,8 +121,14 @@ const runCustom = async (
       viewport: null,
       ...(hasCustomViewport ? contextDeviceOptions : {}),
       userAgent: process.env.OOBEE_USER_AGENT || (deviceUserAgent as string | undefined),
+      ...(nonAuthHeaders && { extraHTTPHeaders: nonAuthHeaders }),
+      ...(httpCredentials && { httpCredentials }),
     });
 
+    if (authHeader) {
+      await addAuthRouteHandler(context, url, authHeader);
+    }
+
     register(context);
 
     processPageParams.stopAll = async () => {

package/src/generateOobeeClientScanner.ts

@@ -60,7 +60,7 @@ const SENTRY_NODE_VERSION: string = (() => {
   try {
     return _require('@sentry/node/package.json').version as string;
   } catch {
-    return '9.47.1';   // safe fallback matching currently installed version
+    return '10.58.0';   // safe fallback matching currently installed version
   }
 })();
 

package/src/mergeAxeResults/itemsStore.ts

@@ -1,6 +1,7 @@
 import fs from 'fs-extra';
 import path from 'path';
 import readline from 'readline';
+import { consoleLogger } from '../logs.js';
 import type { ItemsInfo } from './types.js';
 
 export interface ItemsStoreEntry {
@@ -16,6 +17,7 @@ export interface ItemsStoreEntry {
 export class ItemsStore {
   private basePath: string;
   private ensuredDirs = new Set<string>();
+  private fileWriteQueues = new Map<string, Promise<void>>();
 
   constructor(storagePath: string) {
     this.basePath = path.join(storagePath, 'tmp-items');
@@ -40,8 +42,29 @@ export class ItemsStore {
   async appendPageItems(category: string, ruleId: string, entry: ItemsStoreEntry): Promise<void> {
     await this.ensureDir(category);
     const filePath = this.getRuleFilePath(category, ruleId);
-    const line = JSON.stringify(entry) + '\n';
-    await fs.appendFile(filePath, line, 'utf8');
+    let line = JSON.stringify(entry);
+
+    // JSON.stringify should never produce literal newlines inside strings, but HTML content
+    // from page evaluation may contain edge-case characters (e.g. unescaped control chars in
+    // non-spec-compliant innerHTML). Strip any embedded \r or \n that would break JSONL format readline parsing.
+    line = line.replace(/[\n\r]/g, (match) => {
+      if (match === '\n') return '\\n';
+      if (match === '\r') return '\\r';
+      return match;
+    });
+    line += '\n';
+
+    // Serialize writes per rule file to avoid concurrent append interleaving/truncation.
+    const previous = this.fileWriteQueues.get(filePath) ?? Promise.resolve();
+    const next = previous.then(() => fs.appendFile(filePath, line, 'utf8'));
+    this.fileWriteQueues.set(
+      filePath,
+      next.catch(() => {
+        // Keep queue alive for subsequent writes.
+      }),
+    );
+
+    await next;
   }
 
   async *readRuleItems(category: string, ruleId: string): AsyncGenerator<ItemsStoreEntry> {
@@ -51,9 +74,19 @@ export class ItemsStore {
     const fileStream = fs.createReadStream(filePath, { encoding: 'utf8' });
     const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
 
+    let lineNumber = 0;
     for await (const line of rl) {
-      if (line.trim()) {
+      lineNumber += 1;
+      if (!line.trim()) continue;
+
+      try {
         yield JSON.parse(line) as ItemsStoreEntry;
+      } catch (error) {
+        // Tolerate malformed/truncated JSONL lines (e.g. interrupted append) so report generation can continue.
+        const preview = line.slice(0, 200);
+        consoleLogger.warn(
+          `Skipping malformed itemsStore JSONL line ${lineNumber} in ${filePath}: ${(error as Error).message}. Content preview: ${preview}`,
+        );
       }
     }
   }
@@ -68,6 +101,7 @@ export class ItemsStore {
   }
 
   async cleanup(): Promise<void> {
+    await Promise.all(this.fileWriteQueues.values());
     await fs.rm(this.basePath, { recursive: true, force: true });
   }
 }