@govtechsg/oobee 0.11.0 → 0.11.1

package/AGENTS.md

@@ -144,6 +144,8 @@ The `constants` default export object holds runtime state:
 | `OOBEE_SCAN_PRODUCT` | Adds `scanProduct` tag to Sentry events |
 | `OOBEE_CONSECUTIVE_MAX_RETRIES` | Max consecutive HTTP failures before circuit breaker aborts crawl (default 100) |
 | `OOBEE_VALIDATE_URL` | If set, exit after URL validation without scanning |
+| `OOBEE_SAVE_DOM` | `1` or `true` = save full-page DOM HTML to `pageDOMs/` in results directory. Supported scan types: Website, Sitemap, Intelligent, LocalFile, Custom |
+| `OOBEE_SAVE_PAGE_SCREENSHOT` | `1` or `true` = save full-page desktop + mobile viewport screenshots to `pageDOMs/desktopPageScreenshots/` and `pageDOMs/mobilePageScreenshots/`. Mobile viewport uses iPhone 11 width programmatically. Supported scan types: Website, Sitemap, Intelligent, LocalFile, Custom |
 | `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` | Proxy configuration |
 | `NO_PROXY` / `INCLUDE_PROXY` | Proxy bypass/include lists |
 
@@ -352,5 +354,17 @@ When making changes, validate these areas which have well-established edge cases
 ├── scanPagesDetail.json    # Per-page breakdown
 ├── scanPagesSummary.json   # Page-level summary
 ├── sitemap.xml             # Discovered URLs
-└── screenshots/            # Violation screenshots (if enabled)
+├── screenshots/            # Violation screenshots (if enabled)
+└── pageDOMs/              # Page capture (if OOBEE_SAVE_DOM or OOBEE_SAVE_PAGE_SCREENSHOT)
+    ├── {hash}-{truncated_path}.html          # Full DOM (OOBEE_SAVE_DOM)
+    ├── domManifest.json                     # Maps URLs → hash, file paths, errors
+    ├── desktopPageScreenshots/             # Desktop viewport PNGs (OOBEE_SAVE_PAGE_SCREENSHOT)
+    │   └── {hash}-{truncated_path}.png
+    └── mobilePageScreenshots/              # Mobile viewport PNGs (OOBEE_SAVE_PAGE_SCREENSHOT)
+        └── {hash}-{truncated_path}.png
 ```
+
+- `{hash}` is a 7-character SHA-256 of the page URL (consistent per URL, like git short hashes)
+- `{truncated_path}` is the URL path with `/` replaced by `_`, max 80 characters
+- If filename collisions occur, a `-2`, `-3` etc. suffix is added before the extension
+- `domManifest.json` is written when either env var is enabled, containing URL-to-file mappings

package/CRAWL.md

@@ -114,12 +114,12 @@ A representative scan means enough pages have been checked that the issues found
 
 ### Recommended Hardware
 
-Scanning is CPU and memory intensive — each page runs in a real browser with full JavaScript execution.
+Scanning is CPU and memory intensive — each page runs in a real browser with full JavaScript execution.  The time estimates illustrated below for scans that do not include screenshots.
 
 | Scan Size | Hardware | Max Concurrency | Expected Duration |
 |-----------|----------|-----------------|-------------------|
 | 1,000 pages | ECS Fargate, 2 vCPU / 4 GB RAM | 10 (`-t 10`) | 2-3 hours |
-| 5,000 pages | Laptop/desktop, 8 cores / 12 threads, 24 GB RAM | 25 (`-t 25`) | 2-3 hours |
+| 5,000 pages | Recent Core Ultra or Snapdragon X Series Laptop/Desktop, 8 cores / 12 threads, 24 GB RAM | 25 (`-t 25`) | 2-3 hours |
 
 **Oobee Desktop** runs with `OOBEE_FAST_CRAWLER=true`, which means concurrency scales up aggressively to the maximum (25) as fast as possible. This is suitable for desktop machines with adequate CPU and RAM, but may cause stability issues on low-powered devices.
 

package/README.md

@@ -100,6 +100,8 @@ verapdf --version
 | OOBEE_SCAN_METADATA | Overrides the `entryUrl` tag sent to telemetry. | |
 | OOBEE_SCAN_PRODUCT | Adds a `scanProduct` tag to telemetry events. | |
 | OOBEE_CONSECUTIVE_MAX_RETRIES | Max consecutive HTTP failures before the circuit breaker aborts the crawl. | `100` |
+| OOBEE_SAVE_DOM | Set to `1` or `true` to save full-page DOM HTML for each scanned page to the results directory. Works with all scan types (Website, Sitemap, Intelligent, LocalFile, Custom). | |
+| OOBEE_SAVE_PAGE_SCREENSHOT | Set to `1` or `true` to save full-page desktop and mobile viewport screenshots for each scanned page. Mobile viewport width is determined from iPhone 11 device profile. Works with all scan types (Website, Sitemap, Intelligent, LocalFile, Custom). | |
 | HTTP_PROXY | URL of the proxy server to be used for HTTP requests (e.g. `http://proxy.example.com:8080`). | |
 | HTTPS_PROXY | URL of the proxy server to be used for HTTPS requests (e.g. `https://proxy.example.com:8080`). | |
 | ALL_PROXY | URL of the proxy server to be used for all requests, typically used for SOCKS5 proxies (e.g. `socks5://proxy.example.com:1080`. Note: IPv6 direct connections may still continue even though socks5 proxy is specified due to a known issue with Chrome/Chromium. (Recommended workaround is to turn off IPv6 at host-level). | |

package/dist/combine.js

@@ -12,6 +12,7 @@ import { consoleLogger } from './logs.js';
 import runCustom from './crawlers/runCustom.js';
 import { alertMessageOptions } from './constants/cliFunctions.js';
 import { isS3UploadEnabled, getS3MetadataFromEnv, getS3UploadPrefix, uploadFolderToS3, } from './services/s3Uploader.js';
+import { writeManifest, resetCaptureEntries, isPageCaptureEnabled } from './crawlers/pageCapture.js';
 // Class exports
 export class ViewportSettingsClass {
     constructor(deviceChosen, customDevice, viewportWidth, playwrightDeviceDetailsObject) {
@@ -32,6 +33,9 @@ const combineRun = async (details, deviceToScan) => {
     process.env.CRAWLEE_LOG_LEVEL = 'ERROR';
     process.env.CRAWLEE_STORAGE_DIR = randomToken;
     constants.sitemapFetchedLinks = null;
+    if (isPageCaptureEnabled() && !process.env.OOBEE_SCAN_PRODUCT) {
+        process.env.OOBEE_SCAN_PRODUCT = 'U&A';
+    }
     if (process.env.CRAWLEE_SYSTEM_INFO_V2 === undefined) {
         // Set the environment variable to enable system info v2
         // Resolves issue with when wmic is not installed on Windows
@@ -193,6 +197,8 @@ const combineRun = async (details, deviceToScan) => {
     if (scanDetails.urlsCrawled) {
         if (scanDetails.urlsCrawled.scanned.length > 0) {
             await createAndUpdateResultsFolders(randomToken);
+            await writeManifest(randomToken);
+            resetCaptureEntries();
             const pagesNotScanned = [
                 ...urlsCrawledObj.error,
                 ...urlsCrawledObj.invalid,

package/dist/crawlers/crawlDomain.js

@@ -6,6 +6,7 @@ import { getPlaywrightLaunchOptions, isBlacklistedFileExtensions, isSkippedUrl,
 import { areLinksEqual, isFollowStrategy, isSameHostname, normUrl, register } from '../utils.js';
 import { handlePdfDownload, runPdfScan, mapPdfScanResults, doPdfScreenshots, } from './pdfScanFunc.js';
 import { consoleLogger, guiInfoLog } from '../logs.js';
+import { capturePageData } from './pageCapture.js';
 const isBlacklisted = (url, blacklistedPatterns) => {
     if (!blacklistedPatterns) {
         return false;
@@ -284,7 +285,7 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
             launcher: constants.launcher,
             launchOptions: getPlaywrightLaunchOptions(browser),
         },
-        retryOnBlocked: true,
+        retryOnBlocked: false,
         browserPoolOptions: {
             useFingerprints: false,
             retireBrowserAfterPageCount: 500,
@@ -308,7 +309,6 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
         },
         requestQueue,
         maxRequestRetries: 3,
-        maxSessionRotations: 1,
         preNavigationHooks: [
             ...preNavigationHooks(extraHTTPHeaders),
             async ({ request }) => {
@@ -488,6 +488,21 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                         return;
                     }
                     const responseStatus = response?.status();
+                    if (responseStatus === 403) {
+                        rateController.onFailure(responseStatus, activeCrawler.autoscaledPool);
+                        guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                            numScanned: urlsCrawled.scanned.length,
+                            urlScanned: request.url,
+                        });
+                        urlsCrawled.userExcluded.push({
+                            url: request.url,
+                            pageTitle: request.url,
+                            actualUrl,
+                            metadata: STATUS_CODE_METADATA[403] || STATUS_CODE_METADATA[599],
+                            httpStatusCode: 403,
+                        });
+                        return;
+                    }
                     if (responseStatus && responseStatus >= 300) {
                         guiInfoLog(guiInfoStatusTypes.SKIPPED, {
                             numScanned: urlsCrawled.scanned.length,
@@ -503,6 +518,7 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                         return;
                     }
                     const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
+                    await capturePageData(page, actualUrl, randomToken);
                     // Detect JS redirects that fire during/after axe scan.
                     // Listen for navigation, then give a brief window for pending redirects to complete.
                     try {
@@ -674,12 +690,9 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
             }
             const status = response?.status();
             // Re-enqueue rate-limited (403) URLs once for a retry after concurrency recovers.
-            // Without this, URLs that fail during a rate-limit burst are permanently lost
-            // even though the site is accessible at lower concurrency.
-            // Don't call onFailure here — the re-enqueued request will get a fresh attempt.
-            // If it fails again (rateLimitRetried=true), it falls through to the normal
-            // onFailure + circuit breaker path below.
+            // Call onFailure to reduce concurrency immediately on rate-limit detection.
             if (status === 403 && !request.userData?.rateLimitRetried) {
+                rateController.onFailure(status, crawler.autoscaledPool);
                 try {
                     await requestQueue.addRequest({
                         url: request.url,

package/dist/crawlers/crawlLocalFile.js

@@ -8,6 +8,7 @@ import { runPdfScan, mapPdfScanResults, doPdfScreenshots } from './pdfScanFunc.j
 import { guiInfoLog } from '../logs.js';
 import crawlSitemap from './crawlSitemap.js';
 import { getPdfStoragePath, register } from '../utils.js';
+import { capturePageData } from './pageCapture.js';
 export const crawlLocalFile = async ({ url, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, extraHTTPHeaders, scanDuration = 0, ruleset = [], fromCrawlIntelligentSitemap = false, userUrlInputFromIntelligent = null, datasetFromIntelligent = null, urlsCrawledFromIntelligent = null, }) => {
     let dataset;
     let urlsCrawled;
@@ -108,6 +109,7 @@ export const crawlLocalFile = async ({ url, randomToken, host, viewportSettings,
         }
         const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
         const actualUrl = page.url() || request.loadedUrl || url;
+        await capturePageData(page, actualUrl, randomToken);
         guiInfoLog(guiInfoStatusTypes.SCANNED, {
             numScanned: urlsCrawled.scanned.length,
             urlScanned: url,

package/dist/crawlers/crawlSitemap.js

@@ -6,6 +6,7 @@ import { getLinksFromSitemap, getPlaywrightLaunchOptions, isDisallowedInRobotsTx
 import { areLinksEqual, isFollowStrategy, isWhitelistedContentType, normUrl, register } from '../utils.js';
 import { handlePdfDownload, runPdfScan, mapPdfScanResults, doPdfScreenshots, } from './pdfScanFunc.js';
 import { consoleLogger, guiInfoLog } from '../logs.js';
+import { capturePageData } from './pageCapture.js';
 const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, extraHTTPHeaders, strategy = EnqueueStrategy.All, userUrl = '', scanDuration = 0, fromCrawlIntelligentSitemap = false, userUrlInputFromIntelligent = null, datasetFromIntelligent = null, urlsCrawledFromIntelligent = null, crawledFromLocalFile = false, ruleset = [], }) => {
     const crawlStartTime = Date.now();
     let dataset;
@@ -51,7 +52,7 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
             launcher: constants.launcher,
             launchOptions: getPlaywrightLaunchOptions(browser),
         },
-        retryOnBlocked: true,
+        retryOnBlocked: false,
         browserPoolOptions: {
             useFingerprints: false,
             retireBrowserAfterPageCount: 500,
@@ -75,7 +76,6 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
         requestList,
         requestQueue,
         maxRequestRetries: 3,
-        maxSessionRotations: 1,
         postNavigationHooks: [
             async ({ page }) => {
                 try {
@@ -196,6 +196,21 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                 }
                 const contentType = response?.headers?.()['content-type'] || '';
                 const status = response ? response.status() : 0;
+                if (status === 403) {
+                    rateController.onFailure(status, crawler.autoscaledPool);
+                    guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
+                    });
+                    urlsCrawled.userExcluded.push({
+                        url: request.url,
+                        pageTitle: request.url,
+                        actualUrl,
+                        metadata: STATUS_CODE_METADATA[403] || STATUS_CODE_METADATA[599],
+                        httpStatusCode: 403,
+                    });
+                    return;
+                }
                 if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
                     const isRedirected = !areLinksEqual(page.url(), request.url);
                     const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(item => normUrl(item.actualUrl || item.url) === normUrl(page.url()));
@@ -233,6 +248,7 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                         return;
                     }
                     const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
+                    await capturePageData(page, actualUrl, randomToken);
                     // Detect JS redirects that fire during/after axe scan.
                     // Listen for navigation, then give a brief window for pending redirects to complete.
                     try {
@@ -394,9 +410,9 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
             }
             const status = response?.status();
             // Re-enqueue rate-limited (403) URLs once for a retry after concurrency recovers.
-            // Don't call onFailure here — the re-enqueued request gets a fresh attempt.
-            // If it fails again (rateLimitRetried=true), it falls through to the normal path.
+            // Call onFailure to reduce concurrency immediately on rate-limit detection.
             if (status === 403 && !request.userData?.rateLimitRetried) {
+                rateController.onFailure(status, crawler.autoscaledPool);
                 try {
                     await requestQueue.addRequest({
                         url: request.url,

package/dist/crawlers/custom/utils.js

@@ -4,6 +4,7 @@
 import path from 'path';
 import { getDomain } from 'tldts';
 import { runAxeScript } from '../commonCrawlerFunc.js';
+import { capturePageData } from '../pageCapture.js';
 import { consoleLogger, guiInfoLog } from '../../logs.js';
 import { guiInfoStatusTypes } from '../../constants/constants.js';
 import { isSkippedUrl, validateCustomFlowLabel } from '../../constants/common.js';
@@ -109,6 +110,7 @@ export const screenshotFullPage = async (page, screenshotsDir, screenshotIdx) =>
 };
 export const runAxeScan = async (page, includeScreenshots, randomToken, customFlowDetails, dataset, urlsCrawled) => {
     const result = await runAxeScript({ includeScreenshots, page, randomToken, customFlowDetails });
+    await capturePageData(page, page.url(), randomToken);
     await dataset.pushData(result);
     const rawTitle = result.pageTitle ?? '';
     let pageTitleTextOnly = rawTitle; // Note: The original pageTitle contains the index and is being used in top 10 issues

package/dist/crawlers/pageCapture.js

@@ -0,0 +1,134 @@
+import fs from 'fs-extra';
+import path from 'path';
+import crypto from 'crypto';
+import { devices } from 'playwright';
+import { getStoragePath } from '../utils.js';
+const MOBILE_VIEWPORT_WIDTH = devices['iPhone 11'].viewport.width;
+const MOBILE_VIEWPORT_HEIGHT = devices['iPhone 11'].viewport.height;
+const captureEntries = new Map();
+export function getUrlHash(url) {
+    return crypto.createHash('sha256').update(url).digest('hex').slice(0, 7);
+}
+function getTruncatedPath(url) {
+    try {
+        const parsed = new URL(url);
+        let pathStr = parsed.pathname + (parsed.search || '');
+        pathStr = pathStr.replace(/^\//, '').replace(/\//g, '_').replace(/[^a-zA-Z0-9\-_.]/g, '_');
+        if (pathStr.length > 80) {
+            pathStr = pathStr.slice(0, 80);
+        }
+        return pathStr || 'index';
+    }
+    catch {
+        return 'unknown';
+    }
+}
+function getPageDomsDir(randomToken) {
+    const storagePath = getStoragePath(randomToken);
+    return path.join(storagePath, 'pageDOMs');
+}
+async function getUniqueFilePath(dir, baseName, ext) {
+    let candidate = path.join(dir, `${baseName}${ext}`);
+    if (!await fs.pathExists(candidate))
+        return candidate;
+    let counter = 2;
+    while (await fs.pathExists(candidate)) {
+        candidate = path.join(dir, `${baseName}-${counter}${ext}`);
+        counter++;
+    }
+    return candidate;
+}
+function getRelativeName(filePath, baseDir) {
+    return path.relative(baseDir, filePath).replace(/\\/g, '/');
+}
+export function isSaveDomEnabled() {
+    return process.env.OOBEE_SAVE_DOM === '1' || process.env.OOBEE_SAVE_DOM === 'true';
+}
+export function isSavePageScreenshotEnabled() {
+    return (process.env.OOBEE_SAVE_PAGE_SCREENSHOT === '1' ||
+        process.env.OOBEE_SAVE_PAGE_SCREENSHOT === 'true');
+}
+export function isPageCaptureEnabled() {
+    return isSaveDomEnabled() || isSavePageScreenshotEnabled();
+}
+export async function capturePageData(page, url, randomToken) {
+    if (!isPageCaptureEnabled())
+        return;
+    const hash = getUrlHash(url);
+    const truncatedPath = getTruncatedPath(url);
+    const fileName = `${hash}-${truncatedPath}`;
+    const pageDomsDir = getPageDomsDir(randomToken);
+    const entry = {
+        url,
+        hash,
+        errors: [],
+    };
+    if (isSaveDomEnabled()) {
+        try {
+            await fs.ensureDir(pageDomsDir);
+            const domContent = await page.content();
+            const domFilePath = await getUniqueFilePath(pageDomsDir, fileName, '.html');
+            await fs.writeFile(domFilePath, domContent, 'utf-8');
+            entry.domFile = `pageDOMs/${getRelativeName(domFilePath, pageDomsDir)}`;
+        }
+        catch (err) {
+            entry.errors.push(`DOM save failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    if (isSavePageScreenshotEnabled()) {
+        const desktopDir = path.join(pageDomsDir, 'desktopPageScreenshots');
+        const mobileDir = path.join(pageDomsDir, 'mobilePageScreenshots');
+        try {
+            await fs.ensureDir(desktopDir);
+            const desktopPath = await getUniqueFilePath(desktopDir, fileName, '.png');
+            await page.screenshot({ path: desktopPath, fullPage: true });
+            entry.desktopScreenshot = `pageDOMs/desktopPageScreenshots/${getRelativeName(desktopPath, desktopDir)}`;
+        }
+        catch (err) {
+            entry.errors.push(`Desktop screenshot failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        try {
+            await fs.ensureDir(mobileDir);
+            const currentViewport = page.viewportSize();
+            await page.setViewportSize({
+                width: MOBILE_VIEWPORT_WIDTH,
+                height: MOBILE_VIEWPORT_HEIGHT,
+            });
+            await page.waitForTimeout(500);
+            const mobilePath = await getUniqueFilePath(mobileDir, fileName, '.png');
+            await page.screenshot({ path: mobilePath, fullPage: true });
+            entry.mobileScreenshot = `pageDOMs/mobilePageScreenshots/${getRelativeName(mobilePath, mobileDir)}`;
+            if (currentViewport) {
+                await page.setViewportSize(currentViewport);
+            }
+        }
+        catch (err) {
+            entry.errors.push(`Mobile screenshot failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    captureEntries.set(url, entry);
+}
+export async function writeManifest(randomToken) {
+    if (!isPageCaptureEnabled())
+        return;
+    if (captureEntries.size === 0)
+        return;
+    const pageDomsDir = getPageDomsDir(randomToken);
+    await fs.ensureDir(pageDomsDir);
+    const manifest = {
+        generatedAt: new Date().toISOString(),
+        pages: Array.from(captureEntries.values()).map(entry => ({
+            url: entry.url,
+            hash: entry.hash,
+            ...(entry.domFile && { domFile: entry.domFile }),
+            ...(entry.desktopScreenshot && { desktopScreenshot: entry.desktopScreenshot }),
+            ...(entry.mobileScreenshot && { mobileScreenshot: entry.mobileScreenshot }),
+            errors: entry.errors,
+        })),
+    };
+    const manifestPath = path.join(pageDomsDir, 'domManifest.json');
+    await fs.writeFile(manifestPath, JSON.stringify(manifest, null, 2), 'utf-8');
+}
+export function resetCaptureEntries() {
+    captureEntries.clear();
+}

package/oobee-client-scanner.js

@@ -3,7 +3,7 @@
  * DO NOT EDIT MANUALLY. Re-generate with: node dist/generateOobeeClientScanner.js
  *
  * Embedded at generation time:
- *   App version : 0.11.0
+ *   App version : 0.11.1
  *   Sentry DSN  : (from OOBEE_SENTRY_DSN env var or constants.ts default)
  *   Sentry SDK  : @sentry/browser 10.58.0 (loaded from CDN at runtime)
  *
@@ -34887,7 +34887,7 @@
   // ── Sentry browser telemetry (Sentry JS SDK, loaded from CDN) ────────────
   
   var _oobeeSentryDsn          = "https://3b8c7ee46b06f33815a1301b6713ebc3@o4509047624761344.ingest.us.sentry.io/4509327783559168";
-  var _oobeeAppVersion         = "0.11.0";
+  var _oobeeAppVersion         = "0.11.1";
   var _oobeeSentryVersion      = "10.58.0";
   var _oobeeSentryInitialized  = false;
   var _oobeeSentryLoadPromise  = null;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.11.0",
+  "version": "0.11.1",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "bin": {

package/src/combine.ts

@@ -18,6 +18,7 @@ import {
   getS3UploadPrefix,
   uploadFolderToS3,
 } from './services/s3Uploader.js';
+import { writeManifest, resetCaptureEntries, isPageCaptureEnabled } from './crawlers/pageCapture.js';
 
 // Class exports
 export class ViewportSettingsClass {
@@ -75,6 +76,10 @@ const combineRun = async (details: Data, deviceToScan: string) => {
   process.env.CRAWLEE_STORAGE_DIR = randomToken;
   constants.sitemapFetchedLinks = null;
 
+  if (isPageCaptureEnabled() && !process.env.OOBEE_SCAN_PRODUCT) {
+    process.env.OOBEE_SCAN_PRODUCT = 'U&A';
+  }
+
   if (process.env.CRAWLEE_SYSTEM_INFO_V2 === undefined) {
     // Set the environment variable to enable system info v2
     // Resolves issue with when wmic is not installed on Windows
@@ -285,6 +290,8 @@ const combineRun = async (details: Data, deviceToScan: string) => {
   if (scanDetails.urlsCrawled) {
     if (scanDetails.urlsCrawled.scanned.length > 0) {
       await createAndUpdateResultsFolders(randomToken);
+      await writeManifest(randomToken);
+      resetCaptureEntries();
       const pagesNotScanned = [
         ...urlsCrawledObj.error,
         ...urlsCrawledObj.invalid,

package/src/crawlers/crawlDomain.ts

@@ -41,6 +41,7 @@ import {
 } from './pdfScanFunc.js';
 import { consoleLogger, guiInfoLog } from '../logs.js';
 import { ViewportSettingsClass } from '../combine.js';
+import { capturePageData } from './pageCapture.js';
 
 const isBlacklisted = (url: string, blacklistedPatterns: string[]) => {
   if (!blacklistedPatterns) {
@@ -399,7 +400,7 @@ const crawlDomain = async ({
         launcher: constants.launcher,
         launchOptions: getPlaywrightLaunchOptions(browser),
       },
-      retryOnBlocked: true,
+      retryOnBlocked: false,
       browserPoolOptions: {
         useFingerprints: false,
         retireBrowserAfterPageCount: 500,
@@ -423,7 +424,6 @@ const crawlDomain = async ({
       },
       requestQueue,
       maxRequestRetries: 3,
-      maxSessionRotations: 1,
       preNavigationHooks: [
         ...preNavigationHooks(extraHTTPHeaders),
         async ({ request }) => {
@@ -644,6 +644,21 @@ const crawlDomain = async ({
             }
 
             const responseStatus = response?.status();
+            if (responseStatus === 403) {
+              rateController.onFailure(responseStatus, activeCrawler.autoscaledPool);
+              guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                numScanned: urlsCrawled.scanned.length,
+                urlScanned: request.url,
+              });
+              urlsCrawled.userExcluded.push({
+                url: request.url,
+                pageTitle: request.url,
+                actualUrl,
+                metadata: STATUS_CODE_METADATA[403] || STATUS_CODE_METADATA[599],
+                httpStatusCode: 403,
+              });
+              return;
+            }
             if (responseStatus && responseStatus >= 300) {
               guiInfoLog(guiInfoStatusTypes.SKIPPED, {
                 numScanned: urlsCrawled.scanned.length,
@@ -661,6 +676,8 @@ const crawlDomain = async ({
 
             const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
 
+            await capturePageData(page, actualUrl, randomToken);
+
             // Detect JS redirects that fire during/after axe scan.
             // Listen for navigation, then give a brief window for pending redirects to complete.
             try {
@@ -841,12 +858,9 @@ const crawlDomain = async ({
         const status = response?.status();
 
         // Re-enqueue rate-limited (403) URLs once for a retry after concurrency recovers.
-        // Without this, URLs that fail during a rate-limit burst are permanently lost
-        // even though the site is accessible at lower concurrency.
-        // Don't call onFailure here — the re-enqueued request will get a fresh attempt.
-        // If it fails again (rateLimitRetried=true), it falls through to the normal
-        // onFailure + circuit breaker path below.
+        // Call onFailure to reduce concurrency immediately on rate-limit detection.
         if (status === 403 && !request.userData?.rateLimitRetried) {
+          rateController.onFailure(status, crawler.autoscaledPool);
           try {
             await requestQueue.addRequest({
               url: request.url,

package/src/crawlers/crawlLocalFile.ts

@@ -21,6 +21,7 @@ import { runPdfScan, mapPdfScanResults, doPdfScreenshots } from './pdfScanFunc.j
 import { guiInfoLog } from '../logs.js';
 import crawlSitemap from './crawlSitemap.js';
 import { getPdfStoragePath, getStoragePath, register } from '../utils.js';
+import { capturePageData } from './pageCapture.js';
 
 export const crawlLocalFile = async ({
   url,
@@ -186,6 +187,8 @@ export const crawlLocalFile = async ({
 
     const actualUrl = page.url() || request.loadedUrl || url;
 
+    await capturePageData(page, actualUrl, randomToken);
+
     guiInfoLog(guiInfoStatusTypes.SCANNED, {
       numScanned: urlsCrawled.scanned.length,
       urlScanned: url,

package/src/crawlers/crawlSitemap.ts

@@ -38,6 +38,7 @@ import {
 } from './pdfScanFunc.js';
 import { consoleLogger, guiInfoLog } from '../logs.js';
 import { ViewportSettingsClass } from '../combine.js';
+import { capturePageData } from './pageCapture.js';
 
 const crawlSitemap = async ({
   sitemapUrl,
@@ -149,7 +150,7 @@ const crawlSitemap = async ({
         launcher: constants.launcher,
         launchOptions: getPlaywrightLaunchOptions(browser),
       },
-      retryOnBlocked: true,
+      retryOnBlocked: false,
       browserPoolOptions: {
         useFingerprints: false,
         retireBrowserAfterPageCount: 500,
@@ -173,7 +174,6 @@ const crawlSitemap = async ({
       requestList,
       requestQueue,
       maxRequestRetries: 3,
-      maxSessionRotations: 1,
       postNavigationHooks: [
         async ({ page }) => {
           try {
@@ -319,6 +319,22 @@ const crawlSitemap = async ({
           const contentType = response?.headers?.()['content-type'] || '';
           const status = response ? response.status() : 0;
 
+          if (status === 403) {
+            rateController.onFailure(status, crawler.autoscaledPool);
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
+            urlsCrawled.userExcluded.push({
+              url: request.url,
+              pageTitle: request.url,
+              actualUrl,
+              metadata: STATUS_CODE_METADATA[403] || STATUS_CODE_METADATA[599],
+              httpStatusCode: 403,
+            });
+            return;
+          }
+
           if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
             const isRedirected = !areLinksEqual(page.url(), request.url);
             const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(
@@ -364,6 +380,8 @@ const crawlSitemap = async ({
 
             const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
 
+            await capturePageData(page, actualUrl, randomToken);
+
             // Detect JS redirects that fire during/after axe scan.
             // Listen for navigation, then give a brief window for pending redirects to complete.
             try {
@@ -533,9 +551,9 @@ const crawlSitemap = async ({
         const status = response?.status();
 
         // Re-enqueue rate-limited (403) URLs once for a retry after concurrency recovers.
-        // Don't call onFailure here — the re-enqueued request gets a fresh attempt.
-        // If it fails again (rateLimitRetried=true), it falls through to the normal path.
+        // Call onFailure to reduce concurrency immediately on rate-limit detection.
         if (status === 403 && !request.userData?.rateLimitRetried) {
+          rateController.onFailure(status, crawler.autoscaledPool);
           try {
             await requestQueue.addRequest({
               url: request.url,

package/src/crawlers/custom/utils.ts

@@ -4,6 +4,7 @@
 import path from 'path';
 import { getDomain } from 'tldts';
 import { runAxeScript } from '../commonCrawlerFunc.js';
+import { capturePageData } from '../pageCapture.js';
 import { consoleLogger, guiInfoLog, silentLogger } from '../../logs.js';
 import { guiInfoStatusTypes } from '../../constants/constants.js';
 import { isSkippedUrl, validateCustomFlowLabel } from '../../constants/common.js';
@@ -159,6 +160,8 @@ export const runAxeScan = async (
 ) => {
   const result = await runAxeScript({ includeScreenshots, page, randomToken, customFlowDetails });
 
+  await capturePageData(page, page.url(), randomToken);
+
   await dataset.pushData(result);
 
   const rawTitle = result.pageTitle ?? '';

package/src/crawlers/pageCapture.ts

@@ -0,0 +1,172 @@
+import fs from 'fs-extra';
+import path from 'path';
+import crypto from 'crypto';
+import { Page, devices } from 'playwright';
+import { getStoragePath } from '../utils.js';
+
+const MOBILE_VIEWPORT_WIDTH = devices['iPhone 11'].viewport.width;
+const MOBILE_VIEWPORT_HEIGHT = devices['iPhone 11'].viewport.height;
+
+export interface PageCaptureEntry {
+  url: string;
+  hash: string;
+  domFile?: string;
+  desktopScreenshot?: string;
+  mobileScreenshot?: string;
+  errors: string[];
+}
+
+const captureEntries: Map<string, PageCaptureEntry> = new Map();
+
+export function getUrlHash(url: string): string {
+  return crypto.createHash('sha256').update(url).digest('hex').slice(0, 7);
+}
+
+function getTruncatedPath(url: string): string {
+  try {
+    const parsed = new URL(url);
+    let pathStr = parsed.pathname + (parsed.search || '');
+    pathStr = pathStr.replace(/^\//, '').replace(/\//g, '_').replace(/[^a-zA-Z0-9\-_.]/g, '_');
+    if (pathStr.length > 80) {
+      pathStr = pathStr.slice(0, 80);
+    }
+    return pathStr || 'index';
+  } catch {
+    return 'unknown';
+  }
+}
+
+function getPageDomsDir(randomToken: string): string {
+  const storagePath = getStoragePath(randomToken);
+  return path.join(storagePath, 'pageDOMs');
+}
+
+async function getUniqueFilePath(dir: string, baseName: string, ext: string): Promise<string> {
+  let candidate = path.join(dir, `${baseName}${ext}`);
+  if (!await fs.pathExists(candidate)) return candidate;
+
+  let counter = 2;
+  while (await fs.pathExists(candidate)) {
+    candidate = path.join(dir, `${baseName}-${counter}${ext}`);
+    counter++;
+  }
+  return candidate;
+}
+
+function getRelativeName(filePath: string, baseDir: string): string {
+  return path.relative(baseDir, filePath).replace(/\\/g, '/');
+}
+
+export function isSaveDomEnabled(): boolean {
+  return process.env.OOBEE_SAVE_DOM === '1' || process.env.OOBEE_SAVE_DOM === 'true';
+}
+
+export function isSavePageScreenshotEnabled(): boolean {
+  return (
+    process.env.OOBEE_SAVE_PAGE_SCREENSHOT === '1' ||
+    process.env.OOBEE_SAVE_PAGE_SCREENSHOT === 'true'
+  );
+}
+
+export function isPageCaptureEnabled(): boolean {
+  return isSaveDomEnabled() || isSavePageScreenshotEnabled();
+}
+
+export async function capturePageData(
+  page: Page,
+  url: string,
+  randomToken: string,
+): Promise<void> {
+  if (!isPageCaptureEnabled()) return;
+
+  const hash = getUrlHash(url);
+  const truncatedPath = getTruncatedPath(url);
+  const fileName = `${hash}-${truncatedPath}`;
+  const pageDomsDir = getPageDomsDir(randomToken);
+
+  const entry: PageCaptureEntry = {
+    url,
+    hash,
+    errors: [],
+  };
+
+  if (isSaveDomEnabled()) {
+    try {
+      await fs.ensureDir(pageDomsDir);
+      const domContent = await page.content();
+      const domFilePath = await getUniqueFilePath(pageDomsDir, fileName, '.html');
+      await fs.writeFile(domFilePath, domContent, 'utf-8');
+      entry.domFile = `pageDOMs/${getRelativeName(domFilePath, pageDomsDir)}`;
+    } catch (err) {
+      entry.errors.push(`DOM save failed: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+
+  if (isSavePageScreenshotEnabled()) {
+    const desktopDir = path.join(pageDomsDir, 'desktopPageScreenshots');
+    const mobileDir = path.join(pageDomsDir, 'mobilePageScreenshots');
+
+    try {
+      await fs.ensureDir(desktopDir);
+      const desktopPath = await getUniqueFilePath(desktopDir, fileName, '.png');
+      await page.screenshot({ path: desktopPath, fullPage: true });
+      entry.desktopScreenshot = `pageDOMs/desktopPageScreenshots/${getRelativeName(desktopPath, desktopDir)}`;
+    } catch (err) {
+      entry.errors.push(
+        `Desktop screenshot failed: ${err instanceof Error ? err.message : String(err)}`,
+      );
+    }
+
+    try {
+      await fs.ensureDir(mobileDir);
+      const currentViewport = page.viewportSize();
+
+      await page.setViewportSize({
+        width: MOBILE_VIEWPORT_WIDTH,
+        height: MOBILE_VIEWPORT_HEIGHT,
+      });
+      await page.waitForTimeout(500);
+
+      const mobilePath = await getUniqueFilePath(mobileDir, fileName, '.png');
+      await page.screenshot({ path: mobilePath, fullPage: true });
+      entry.mobileScreenshot = `pageDOMs/mobilePageScreenshots/${getRelativeName(mobilePath, mobileDir)}`;
+
+      if (currentViewport) {
+        await page.setViewportSize(currentViewport);
+      }
+    } catch (err) {
+      entry.errors.push(
+        `Mobile screenshot failed: ${err instanceof Error ? err.message : String(err)}`,
+      );
+    }
+  }
+
+  captureEntries.set(url, entry);
+}
+
+export async function writeManifest(randomToken: string): Promise<void> {
+  if (!isPageCaptureEnabled()) return;
+  if (captureEntries.size === 0) return;
+
+  const pageDomsDir = getPageDomsDir(randomToken);
+  await fs.ensureDir(pageDomsDir);
+
+  const manifest = {
+    generatedAt: new Date().toISOString(),
+    pages: Array.from(captureEntries.values()).map(entry => ({
+      url: entry.url,
+      hash: entry.hash,
+      ...(entry.domFile && { domFile: entry.domFile }),
+      ...(entry.desktopScreenshot && { desktopScreenshot: entry.desktopScreenshot }),
+      ...(entry.mobileScreenshot && { mobileScreenshot: entry.mobileScreenshot }),
+      errors: entry.errors,
+    })),
+  };
+
+  const manifestPath = path.join(pageDomsDir, 'domManifest.json');
+  await fs.writeFile(manifestPath, JSON.stringify(manifest, null, 2), 'utf-8');
+}
+
+export function resetCaptureEntries(): void {
+  captureEntries.clear();
+}