@govtechsg/oobee 0.10.98 → 0.11.1

package/dist/crawlers/crawlDomain.js

@@ -1,11 +1,12 @@
 import crawlee from 'crawlee';
 import { CrawlRateController } from './crawlRateController.js';
-import { createCrawleeSubFolders, getPreLaunchHook, preNavigationHooks, runAxeScript, isUrlPdf, shouldSkipClickDueToDisallowedHref, shouldSkipDueToUnsupportedContent, splitAuthHeaders, } from './commonCrawlerFunc.js';
+import { createCrawleeSubFolders, getPreLaunchHook, getPostPageCloseHook, preNavigationHooks, runAxeScript, isUrlPdf, shouldSkipClickDueToDisallowedHref, shouldSkipDueToUnsupportedContent, splitAuthHeaders, } from './commonCrawlerFunc.js';
 import constants, { blackListedFileExtensions, guiInfoStatusTypes, cssQuerySelectors, STATUS_CODE_METADATA, disallowedListOfPatterns, disallowedSelectorPatterns, FileTypes, } from '../constants/constants.js';
 import { getPlaywrightLaunchOptions, isBlacklistedFileExtensions, isSkippedUrl, isDisallowedInRobotsTxt, getUrlsFromRobotsTxt, waitForPageLoaded, } from '../constants/common.js';
 import { areLinksEqual, isFollowStrategy, isSameHostname, normUrl, register } from '../utils.js';
 import { handlePdfDownload, runPdfScan, mapPdfScanResults, doPdfScreenshots, } from './pdfScanFunc.js';
 import { consoleLogger, guiInfoLog } from '../logs.js';
+import { capturePageData } from './pageCapture.js';
 const isBlacklisted = (url, blacklistedPatterns) => {
     if (!blacklistedPatterns) {
         return false;
@@ -284,7 +285,7 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
             launcher: constants.launcher,
             launchOptions: getPlaywrightLaunchOptions(browser),
         },
-        retryOnBlocked: true,
+        retryOnBlocked: false,
         browserPoolOptions: {
             useFingerprints: false,
             retireBrowserAfterPageCount: 500,
@@ -304,12 +305,23 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                     };
                 },
             ],
+            postPageCloseHooks: [getPostPageCloseHook(userDataDirectory)],
         },
         requestQueue,
         maxRequestRetries: 3,
-        maxSessionRotations: 1,
         preNavigationHooks: [
             ...preNavigationHooks(extraHTTPHeaders),
+            async ({ request }) => {
+                const url = request.url.toLowerCase();
+                try {
+                    const pathname = new URL(url).pathname;
+                    const ext = pathname.split('.').pop();
+                    if (ext && blackListedFileExtensions.includes(ext)) {
+                        request.skipNavigation = true;
+                    }
+                }
+                catch { }
+            },
         ],
         postNavigationHooks: [
             async (crawlingContext) => {
@@ -415,20 +427,17 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                 if (shouldSkipDueToUnsupportedContent(response, request.url) ||
                     (request.skipNavigation && actualUrl === 'about:blank')) {
                     if (!isScanPdfs) {
-                        // Don't inform the user it is skipped since web crawler is best-effort.
-                        /*
-                      guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-                        numScanned: urlsCrawled.scanned.length,
-                        urlScanned: request.url,
-                      });
-                      urlsCrawled.userExcluded.push({
-                        url: request.url,
-                        pageTitle: request.url,
-                        actualUrl: request.url, // because about:blank is not useful
-                        metadata: STATUS_CODE_METADATA[1],
-                        httpStatusCode: 0,
-                      });
-                      */
+                        guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                            numScanned: urlsCrawled.scanned.length,
+                            urlScanned: request.url,
+                        });
+                        urlsCrawled.userExcluded.push({
+                            url: request.url,
+                            pageTitle: request.url,
+                            actualUrl: request.url,
+                            metadata: STATUS_CODE_METADATA[1],
+                            httpStatusCode: 1,
+                        });
                         return;
                     }
                     const { pdfFileName, url: downloadedPdfUrl } = handlePdfDownload(randomToken, pdfDownloads, request, sendRequest, urlsCrawled);
@@ -436,20 +445,17 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                     return;
                 }
                 if (isBlacklistedFileExtensions(actualUrl, blackListedFileExtensions)) {
-                    // Don't inform the user it is skipped since web crawler is best-effort.
-                    /*
-                  guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-                    numScanned: urlsCrawled.scanned.length,
-                    urlScanned: request.url,
-                  });
-                  urlsCrawled.userExcluded.push({
-                    url: request.url,
-                    pageTitle: request.url,
-                    actualUrl, // because about:blank is not useful
-                    metadata: STATUS_CODE_METADATA[1],
-                    httpStatusCode: 0,
-                  });
-                  */
+                    guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
+                    });
+                    urlsCrawled.userExcluded.push({
+                        url: request.url,
+                        pageTitle: request.url,
+                        actualUrl,
+                        metadata: STATUS_CODE_METADATA[1],
+                        httpStatusCode: 1,
+                    });
                     return;
                 }
                 if (!isFollowStrategy(url, actualUrl, strategy) &&
@@ -482,6 +488,21 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                         return;
                     }
                     const responseStatus = response?.status();
+                    if (responseStatus === 403) {
+                        rateController.onFailure(responseStatus, activeCrawler.autoscaledPool);
+                        guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                            numScanned: urlsCrawled.scanned.length,
+                            urlScanned: request.url,
+                        });
+                        urlsCrawled.userExcluded.push({
+                            url: request.url,
+                            pageTitle: request.url,
+                            actualUrl,
+                            metadata: STATUS_CODE_METADATA[403] || STATUS_CODE_METADATA[599],
+                            httpStatusCode: 403,
+                        });
+                        return;
+                    }
                     if (responseStatus && responseStatus >= 300) {
                         guiInfoLog(guiInfoStatusTypes.SKIPPED, {
                             numScanned: urlsCrawled.scanned.length,
@@ -497,6 +518,7 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                         return;
                     }
                     const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
+                    await capturePageData(page, actualUrl, randomToken);
                     // Detect JS redirects that fire during/after axe scan.
                     // Listen for navigation, then give a brief window for pending redirects to complete.
                     try {
@@ -634,7 +656,54 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
             if (isAbortingScanNow) {
                 return;
             }
+            // Handle download-triggered navigation errors: Playwright throws
+            // "Download is starting" when page.goto() hits a file download URL.
+            // Crawlee retries 3 times (all fail) then lands here.
+            const isDownloadError = request.errorMessages?.some((msg) => msg.includes('Download is starting'));
+            if (isDownloadError) {
+                if (isScanPdfs) {
+                    // Re-enqueue with skipNavigation so the requestHandler's PDF download path handles it
+                    try {
+                        await requestQueue.addRequest({
+                            url: request.url,
+                            skipNavigation: true,
+                            label: request.url,
+                            uniqueKey: `download_${request.url}`,
+                        });
+                    }
+                    catch { }
+                }
+                else {
+                    guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
+                    });
+                    urlsCrawled.userExcluded.push({
+                        url: request.url,
+                        pageTitle: request.url,
+                        actualUrl: request.url,
+                        metadata: STATUS_CODE_METADATA[1],
+                        httpStatusCode: 1,
+                    });
+                }
+                return;
+            }
             const status = response?.status();
+            // Re-enqueue rate-limited (403) URLs once for a retry after concurrency recovers.
+            // Call onFailure to reduce concurrency immediately on rate-limit detection.
+            if (status === 403 && !request.userData?.rateLimitRetried) {
+                rateController.onFailure(status, crawler.autoscaledPool);
+                try {
+                    await requestQueue.addRequest({
+                        url: request.url,
+                        label: request.url,
+                        uniqueKey: `ratelimit_${request.url}`,
+                        userData: { rateLimitRetried: true },
+                    });
+                }
+                catch { }
+                return;
+            }
             if (rateController.onFailure(status, crawler.autoscaledPool)) {
                 consoleLogger.info(`Aborting crawl: consecutive HTTP failures threshold reached (site may be rate-limiting). Successfully scanned ${urlsCrawled.scanned.length} pages.`);
                 isAbortingScanNow = true;
@@ -669,7 +738,9 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
     await crawler.run();
     // Additional passes: keep re-visiting scanned seed-hostname pages for
     // click-discovery until no new pages are found or limits are reached.
-    if (!safeMode && !isAbortingScanNow && !durationExceeded) {
+    // Skip when called from intelligent sitemap — the domain phase is only meant
+    // to discover new pages via <a> links, not re-click 3000+ already-scanned pages.
+    if (!safeMode && !isAbortingScanNow && !durationExceeded && !fromCrawlIntelligentSitemap) {
         const seedHostname = new URL(url).hostname;
         const clickPassVisited = new Set();
         let prevScannedCount;

package/dist/crawlers/crawlLocalFile.js

@@ -8,6 +8,7 @@ import { runPdfScan, mapPdfScanResults, doPdfScreenshots } from './pdfScanFunc.j
 import { guiInfoLog } from '../logs.js';
 import crawlSitemap from './crawlSitemap.js';
 import { getPdfStoragePath, register } from '../utils.js';
+import { capturePageData } from './pageCapture.js';
 export const crawlLocalFile = async ({ url, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, extraHTTPHeaders, scanDuration = 0, ruleset = [], fromCrawlIntelligentSitemap = false, userUrlInputFromIntelligent = null, datasetFromIntelligent = null, urlsCrawledFromIntelligent = null, }) => {
     let dataset;
     let urlsCrawled;
@@ -108,6 +109,7 @@ export const crawlLocalFile = async ({ url, randomToken, host, viewportSettings,
         }
         const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
         const actualUrl = page.url() || request.loadedUrl || url;
+        await capturePageData(page, actualUrl, randomToken);
         guiInfoLog(guiInfoStatusTypes.SCANNED, {
             numScanned: urlsCrawled.scanned.length,
             urlScanned: url,

package/dist/crawlers/crawlSitemap.js

@@ -1,11 +1,12 @@
 import crawlee, { EnqueueStrategy, RequestList } from 'crawlee';
 import { CrawlRateController } from './crawlRateController.js';
-import { createCrawleeSubFolders, getPreLaunchHook, preNavigationHooks, runAxeScript, splitAuthHeaders, } from './commonCrawlerFunc.js';
-import constants, { STATUS_CODE_METADATA, guiInfoStatusTypes, disallowedListOfPatterns, FileTypes, } from '../constants/constants.js';
-import { getLinksFromSitemap, getPlaywrightLaunchOptions, isSkippedUrl, waitForPageLoaded, isFilePath, } from '../constants/common.js';
+import { createCrawleeSubFolders, getPreLaunchHook, getPostPageCloseHook, preNavigationHooks, runAxeScript, isUrlPdf, splitAuthHeaders, } from './commonCrawlerFunc.js';
+import constants, { STATUS_CODE_METADATA, guiInfoStatusTypes, blackListedFileExtensions, disallowedListOfPatterns, disallowedSelectorPatterns, FileTypes, } from '../constants/constants.js';
+import { getLinksFromSitemap, getPlaywrightLaunchOptions, isDisallowedInRobotsTxt, isSkippedUrl, waitForPageLoaded, isFilePath, } from '../constants/common.js';
 import { areLinksEqual, isFollowStrategy, isWhitelistedContentType, normUrl, register } from '../utils.js';
 import { handlePdfDownload, runPdfScan, mapPdfScanResults, doPdfScreenshots, } from './pdfScanFunc.js';
 import { consoleLogger, guiInfoLog } from '../logs.js';
+import { capturePageData } from './pageCapture.js';
 const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, extraHTTPHeaders, strategy = EnqueueStrategy.All, userUrl = '', scanDuration = 0, fromCrawlIntelligentSitemap = false, userUrlInputFromIntelligent = null, datasetFromIntelligent = null, urlsCrawledFromIntelligent = null, crawledFromLocalFile = false, ruleset = [], }) => {
     const crawlStartTime = Date.now();
     let dataset;
@@ -41,12 +42,17 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
     const requestList = await RequestList.open({
         sources: linksFromSitemap,
     });
+    // Always create a request queue alongside the request list. An empty queue
+    // has zero impact on crawl behavior (Crawlee processes RequestList first).
+    // Having it available enables: download re-enqueue for PDF scanning,
+    // 403 rate-limit retry, and enqueueLinks for intelligent sitemap discovery.
+    const { requestQueue } = await createCrawleeSubFolders(randomToken);
     const crawler = register(new crawlee.PlaywrightCrawler({
         launchContext: {
             launcher: constants.launcher,
             launchOptions: getPlaywrightLaunchOptions(browser),
         },
-        retryOnBlocked: true,
+        retryOnBlocked: false,
         browserPoolOptions: {
             useFingerprints: false,
             retireBrowserAfterPageCount: 500,
@@ -65,10 +71,11 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                     };
                 },
             ],
+            postPageCloseHooks: [getPostPageCloseHook(userDataDirectory)],
         },
         requestList,
+        requestQueue,
         maxRequestRetries: 3,
-        maxSessionRotations: 1,
         postNavigationHooks: [
             async ({ page }) => {
                 try {
@@ -125,14 +132,22 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                 if (isNotSupportedDocument) {
                     request.skipNavigation = true;
                     request.userData.isNotSupportedDocument = true;
-                    // Log for verification (optional, but not required for correctness)
-                    // console.log(`[SKIP] Not supported: ${request.url}`);
                     return;
                 }
+                try {
+                    const pathname = new URL(url).pathname;
+                    const ext = pathname.split('.').pop();
+                    if (ext && blackListedFileExtensions.includes(ext)) {
+                        request.skipNavigation = true;
+                        request.userData.isNotSupportedDocument = true;
+                        return;
+                    }
+                }
+                catch { }
             },
         ],
         requestHandlerTimeoutSecs: 90,
-        requestHandler: async ({ page, request, response, sendRequest }) => {
+        requestHandler: async ({ page, request, response, sendRequest, enqueueLinks }) => {
             // Log documents that are not supported
             if (request.userData?.isNotSupportedDocument) {
                 guiInfoLog(guiInfoStatusTypes.SKIPPED, {
@@ -181,6 +196,21 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                 }
                 const contentType = response?.headers?.()['content-type'] || '';
                 const status = response ? response.status() : 0;
+                if (status === 403) {
+                    rateController.onFailure(status, crawler.autoscaledPool);
+                    guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
+                    });
+                    urlsCrawled.userExcluded.push({
+                        url: request.url,
+                        pageTitle: request.url,
+                        actualUrl,
+                        metadata: STATUS_CODE_METADATA[403] || STATUS_CODE_METADATA[599],
+                        httpStatusCode: 403,
+                    });
+                    return;
+                }
                 if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
                     const isRedirected = !areLinksEqual(page.url(), request.url);
                     const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(item => normUrl(item.actualUrl || item.url) === normUrl(page.url()));
@@ -218,6 +248,7 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                         return;
                     }
                     const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
+                    await capturePageData(page, actualUrl, randomToken);
                     // Detect JS redirects that fire during/after axe scan.
                     // Listen for navigation, then give a brief window for pending redirects to complete.
                     try {
@@ -268,6 +299,34 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                         results.url = request.url;
                         results.actualUrl = actualUrl;
                         await dataset.pushData(results);
+                        // Discover <a> links from this page for the intelligent sitemap flow.
+                        // This eliminates the need for a separate crawlDomain supplement phase
+                        // that would re-visit all these pages just to extract links.
+                        if (fromCrawlIntelligentSitemap) {
+                            try {
+                                await enqueueLinks({
+                                    selector: `a:not(${disallowedSelectorPatterns})`,
+                                    strategy,
+                                    requestQueue,
+                                    transformRequestFunction: (req) => {
+                                        try {
+                                            req.url = req.url.replace(/(?<=&|\?)utm_.*?(&|$)/gim, '');
+                                        }
+                                        catch { }
+                                        if (isDisallowedInRobotsTxt(req.url))
+                                            return null;
+                                        if (isUrlPdf(req.url)) {
+                                            req.skipNavigation = true;
+                                        }
+                                        req.label = req.url;
+                                        return req;
+                                    },
+                                });
+                            }
+                            catch {
+                                // Best-effort link discovery; don't fail the scan
+                            }
+                        }
                     }
                 }
                 else {
@@ -276,18 +335,34 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                         urlScanned: request.url,
                     });
                     if (isScanHtml) {
-                        // carry through the HTTP status metadata
-                        const status = response?.status();
-                        const metadata = typeof status === 'number'
-                            ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
-                            : STATUS_CODE_METADATA[2];
-                        urlsCrawled.invalid.push({
-                            actualUrl,
-                            url: request.url,
-                            pageTitle: request.url,
-                            metadata,
-                            httpStatusCode: typeof status === 'number' ? status : 0,
-                        });
+                        // Non-HTML content types (images, PDFs, binary files) and URLs
+                        // that redirect to non-HTML resources should be classified as
+                        // unsupported documents, not generic page errors.
+                        const isNonHtmlContent = contentType &&
+                            !contentType.startsWith('text/html') &&
+                            !contentType.includes('html');
+                        if (isNonHtmlContent && status !== 0) {
+                            urlsCrawled.userExcluded.push({
+                                actualUrl,
+                                url: request.url,
+                                pageTitle: request.url,
+                                metadata: STATUS_CODE_METADATA[1],
+                                httpStatusCode: 1,
+                            });
+                        }
+                        else {
+                            const httpStatus = response?.status();
+                            const metadata = typeof httpStatus === 'number'
+                                ? STATUS_CODE_METADATA[httpStatus] || STATUS_CODE_METADATA[599]
+                                : STATUS_CODE_METADATA[2];
+                            urlsCrawled.invalid.push({
+                                actualUrl,
+                                url: request.url,
+                                pageTitle: request.url,
+                                metadata,
+                                httpStatusCode: typeof httpStatus === 'number' ? httpStatus : 0,
+                            });
+                        }
                     }
                 }
             }
@@ -302,7 +377,53 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
             if (isAbortingScan) {
                 return;
             }
+            // Handle download-triggered navigation errors: Playwright throws
+            // "Download is starting" when page.goto() hits a file download URL.
+            const isDownloadError = request.errorMessages?.some((msg) => msg.includes('Download is starting'));
+            if (isDownloadError) {
+                if (isScanPdfs) {
+                    // Re-enqueue with skipNavigation so the requestHandler's PDF download path handles it
+                    try {
+                        await requestQueue.addRequest({
+                            url: request.url,
+                            skipNavigation: true,
+                            label: request.url,
+                            uniqueKey: `download_${request.url}`,
+                        });
+                    }
+                    catch { }
+                }
+                else {
+                    guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
+                    });
+                    urlsCrawled.userExcluded.push({
+                        url: request.url,
+                        pageTitle: request.url,
+                        actualUrl: request.url,
+                        metadata: STATUS_CODE_METADATA[1],
+                        httpStatusCode: 1,
+                    });
+                }
+                return;
+            }
             const status = response?.status();
+            // Re-enqueue rate-limited (403) URLs once for a retry after concurrency recovers.
+            // Call onFailure to reduce concurrency immediately on rate-limit detection.
+            if (status === 403 && !request.userData?.rateLimitRetried) {
+                rateController.onFailure(status, crawler.autoscaledPool);
+                try {
+                    await requestQueue.addRequest({
+                        url: request.url,
+                        label: request.url,
+                        uniqueKey: `ratelimit_${request.url}`,
+                        userData: { rateLimitRetried: true },
+                    });
+                }
+                catch { }
+                return;
+            }
             if (rateController.onFailure(status, crawler.autoscaledPool)) {
                 consoleLogger.info(`Aborting crawl: consecutive HTTP failures threshold reached (site may be rate-limiting). Successfully scanned ${urlsCrawled.scanned.length} pages.`);
                 isAbortingScan = true;

package/dist/crawlers/custom/utils.js

@@ -4,6 +4,7 @@
 import path from 'path';
 import { getDomain } from 'tldts';
 import { runAxeScript } from '../commonCrawlerFunc.js';
+import { capturePageData } from '../pageCapture.js';
 import { consoleLogger, guiInfoLog } from '../../logs.js';
 import { guiInfoStatusTypes } from '../../constants/constants.js';
 import { isSkippedUrl, validateCustomFlowLabel } from '../../constants/common.js';
@@ -109,6 +110,7 @@ export const screenshotFullPage = async (page, screenshotsDir, screenshotIdx) =>
 };
 export const runAxeScan = async (page, includeScreenshots, randomToken, customFlowDetails, dataset, urlsCrawled) => {
     const result = await runAxeScript({ includeScreenshots, page, randomToken, customFlowDetails });
+    await capturePageData(page, page.url(), randomToken);
     await dataset.pushData(result);
     const rawTitle = result.pageTitle ?? '';
     let pageTitleTextOnly = rawTitle; // Note: The original pageTitle contains the index and is being used in top 10 issues

package/dist/crawlers/pageCapture.js

@@ -0,0 +1,134 @@
+import fs from 'fs-extra';
+import path from 'path';
+import crypto from 'crypto';
+import { devices } from 'playwright';
+import { getStoragePath } from '../utils.js';
+const MOBILE_VIEWPORT_WIDTH = devices['iPhone 11'].viewport.width;
+const MOBILE_VIEWPORT_HEIGHT = devices['iPhone 11'].viewport.height;
+const captureEntries = new Map();
+export function getUrlHash(url) {
+    return crypto.createHash('sha256').update(url).digest('hex').slice(0, 7);
+}
+function getTruncatedPath(url) {
+    try {
+        const parsed = new URL(url);
+        let pathStr = parsed.pathname + (parsed.search || '');
+        pathStr = pathStr.replace(/^\//, '').replace(/\//g, '_').replace(/[^a-zA-Z0-9\-_.]/g, '_');
+        if (pathStr.length > 80) {
+            pathStr = pathStr.slice(0, 80);
+        }
+        return pathStr || 'index';
+    }
+    catch {
+        return 'unknown';
+    }
+}
+function getPageDomsDir(randomToken) {
+    const storagePath = getStoragePath(randomToken);
+    return path.join(storagePath, 'pageDOMs');
+}
+async function getUniqueFilePath(dir, baseName, ext) {
+    let candidate = path.join(dir, `${baseName}${ext}`);
+    if (!await fs.pathExists(candidate))
+        return candidate;
+    let counter = 2;
+    while (await fs.pathExists(candidate)) {
+        candidate = path.join(dir, `${baseName}-${counter}${ext}`);
+        counter++;
+    }
+    return candidate;
+}
+function getRelativeName(filePath, baseDir) {
+    return path.relative(baseDir, filePath).replace(/\\/g, '/');
+}
+export function isSaveDomEnabled() {
+    return process.env.OOBEE_SAVE_DOM === '1' || process.env.OOBEE_SAVE_DOM === 'true';
+}
+export function isSavePageScreenshotEnabled() {
+    return (process.env.OOBEE_SAVE_PAGE_SCREENSHOT === '1' ||
+        process.env.OOBEE_SAVE_PAGE_SCREENSHOT === 'true');
+}
+export function isPageCaptureEnabled() {
+    return isSaveDomEnabled() || isSavePageScreenshotEnabled();
+}
+export async function capturePageData(page, url, randomToken) {
+    if (!isPageCaptureEnabled())
+        return;
+    const hash = getUrlHash(url);
+    const truncatedPath = getTruncatedPath(url);
+    const fileName = `${hash}-${truncatedPath}`;
+    const pageDomsDir = getPageDomsDir(randomToken);
+    const entry = {
+        url,
+        hash,
+        errors: [],
+    };
+    if (isSaveDomEnabled()) {
+        try {
+            await fs.ensureDir(pageDomsDir);
+            const domContent = await page.content();
+            const domFilePath = await getUniqueFilePath(pageDomsDir, fileName, '.html');
+            await fs.writeFile(domFilePath, domContent, 'utf-8');
+            entry.domFile = `pageDOMs/${getRelativeName(domFilePath, pageDomsDir)}`;
+        }
+        catch (err) {
+            entry.errors.push(`DOM save failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    if (isSavePageScreenshotEnabled()) {
+        const desktopDir = path.join(pageDomsDir, 'desktopPageScreenshots');
+        const mobileDir = path.join(pageDomsDir, 'mobilePageScreenshots');
+        try {
+            await fs.ensureDir(desktopDir);
+            const desktopPath = await getUniqueFilePath(desktopDir, fileName, '.png');
+            await page.screenshot({ path: desktopPath, fullPage: true });
+            entry.desktopScreenshot = `pageDOMs/desktopPageScreenshots/${getRelativeName(desktopPath, desktopDir)}`;
+        }
+        catch (err) {
+            entry.errors.push(`Desktop screenshot failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        try {
+            await fs.ensureDir(mobileDir);
+            const currentViewport = page.viewportSize();
+            await page.setViewportSize({
+                width: MOBILE_VIEWPORT_WIDTH,
+                height: MOBILE_VIEWPORT_HEIGHT,
+            });
+            await page.waitForTimeout(500);
+            const mobilePath = await getUniqueFilePath(mobileDir, fileName, '.png');
+            await page.screenshot({ path: mobilePath, fullPage: true });
+            entry.mobileScreenshot = `pageDOMs/mobilePageScreenshots/${getRelativeName(mobilePath, mobileDir)}`;
+            if (currentViewport) {
+                await page.setViewportSize(currentViewport);
+            }
+        }
+        catch (err) {
+            entry.errors.push(`Mobile screenshot failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    captureEntries.set(url, entry);
+}
+export async function writeManifest(randomToken) {
+    if (!isPageCaptureEnabled())
+        return;
+    if (captureEntries.size === 0)
+        return;
+    const pageDomsDir = getPageDomsDir(randomToken);
+    await fs.ensureDir(pageDomsDir);
+    const manifest = {
+        generatedAt: new Date().toISOString(),
+        pages: Array.from(captureEntries.values()).map(entry => ({
+            url: entry.url,
+            hash: entry.hash,
+            ...(entry.domFile && { domFile: entry.domFile }),
+            ...(entry.desktopScreenshot && { desktopScreenshot: entry.desktopScreenshot }),
+            ...(entry.mobileScreenshot && { mobileScreenshot: entry.mobileScreenshot }),
+            errors: entry.errors,
+        })),
+    };
+    const manifestPath = path.join(pageDomsDir, 'domManifest.json');
+    await fs.writeFile(manifestPath, JSON.stringify(manifest, null, 2), 'utf-8');
+}
+export function resetCaptureEntries() {
+    captureEntries.clear();
+}

package/dist/mergeAxeResults.js

@@ -831,7 +831,7 @@ generateJsonFiles = false, preferredBrowser) => {
         consoleLogger.info(`Dataset drop: ${error.message}`);
     }
     try {
-        const requestQueue = await RequestQueue.open(crawleeDir);
+        const requestQueue = await RequestQueue.open(`${crawleeDir}_rq`);
         await requestQueue.drop();
     }
     catch (error) {
@@ -841,6 +841,7 @@ generateJsonFiles = false, preferredBrowser) => {
     const crawleePath = path.join(storagePath, 'crawlee');
     try {
         await fs.promises.rm(crawleePath, { recursive: true, force: true });
+        await fs.promises.rm(`${crawleePath}_rq`, { recursive: true, force: true });
     }
     catch {
         // Best-effort; storage was already dropped via API