@govtechsg/oobee 0.10.98 → 0.11.1

package/src/crawlers/crawlSitemap.ts

@@ -4,6 +4,7 @@ import fs from 'fs';
 import {
   createCrawleeSubFolders,
   getPreLaunchHook,
+  getPostPageCloseHook,
   preNavigationHooks,
   runAxeScript,
   isUrlPdf,
@@ -14,13 +15,16 @@ import constants, {
   STATUS_CODE_METADATA,
   guiInfoStatusTypes,
   UrlsCrawled,
+  blackListedFileExtensions,
   disallowedListOfPatterns,
+  disallowedSelectorPatterns,
   FileTypes,
   RuleFlags,
 } from '../constants/constants.js';
 import {
   getLinksFromSitemap,
   getPlaywrightLaunchOptions,
+  isDisallowedInRobotsTxt,
   isSkippedUrl,
   waitForPageLoaded,
   isFilePath,
@@ -34,6 +38,7 @@ import {
 } from './pdfScanFunc.js';
 import { consoleLogger, guiInfoLog } from '../logs.js';
 import { ViewportSettingsClass } from '../combine.js';
+import { capturePageData } from './pageCapture.js';
 
 const crawlSitemap = async ({
   sitemapUrl,
@@ -133,13 +138,19 @@ const crawlSitemap = async ({
     sources: linksFromSitemap,
   });
 
+  // Always create a request queue alongside the request list. An empty queue
+  // has zero impact on crawl behavior (Crawlee processes RequestList first).
+  // Having it available enables: download re-enqueue for PDF scanning,
+  // 403 rate-limit retry, and enqueueLinks for intelligent sitemap discovery.
+  const { requestQueue } = await createCrawleeSubFolders(randomToken);
+
   const crawler = register(
     new crawlee.PlaywrightCrawler({
       launchContext: {
         launcher: constants.launcher,
         launchOptions: getPlaywrightLaunchOptions(browser),
       },
-      retryOnBlocked: true,
+      retryOnBlocked: false,
       browserPoolOptions: {
         useFingerprints: false,
         retireBrowserAfterPageCount: 500,
@@ -158,10 +169,11 @@ const crawlSitemap = async ({
             };
           },
         ],
+        postPageCloseHooks: [getPostPageCloseHook(userDataDirectory)],
       },
       requestList,
+      requestQueue,
       maxRequestRetries: 3,
-      maxSessionRotations: 1,
       postNavigationHooks: [
         async ({ page }) => {
           try {
@@ -225,16 +237,22 @@ const crawlSitemap = async ({
           if (isNotSupportedDocument) {
             request.skipNavigation = true;
             request.userData.isNotSupportedDocument = true;
-
-            // Log for verification (optional, but not required for correctness)
-            // console.log(`[SKIP] Not supported: ${request.url}`);
-
             return;
           }
+
+          try {
+            const pathname = new URL(url).pathname;
+            const ext = pathname.split('.').pop();
+            if (ext && blackListedFileExtensions.includes(ext)) {
+              request.skipNavigation = true;
+              request.userData.isNotSupportedDocument = true;
+              return;
+            }
+          } catch {}
         },
       ],
       requestHandlerTimeoutSecs: 90,
-      requestHandler: async ({ page, request, response, sendRequest }) => {
+      requestHandler: async ({ page, request, response, sendRequest, enqueueLinks }) => {
         // Log documents that are not supported
         if (request.userData?.isNotSupportedDocument) {
           guiInfoLog(guiInfoStatusTypes.SKIPPED, {
@@ -301,6 +319,22 @@ const crawlSitemap = async ({
           const contentType = response?.headers?.()['content-type'] || '';
           const status = response ? response.status() : 0;
 
+          if (status === 403) {
+            rateController.onFailure(status, crawler.autoscaledPool);
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
+            urlsCrawled.userExcluded.push({
+              url: request.url,
+              pageTitle: request.url,
+              actualUrl,
+              metadata: STATUS_CODE_METADATA[403] || STATUS_CODE_METADATA[599],
+              httpStatusCode: 403,
+            });
+            return;
+          }
+
           if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
             const isRedirected = !areLinksEqual(page.url(), request.url);
             const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(
@@ -346,6 +380,8 @@ const crawlSitemap = async ({
 
             const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
 
+            await capturePageData(page, actualUrl, randomToken);
+
             // Detect JS redirects that fire during/after axe scan.
             // Listen for navigation, then give a brief window for pending redirects to complete.
             try {
@@ -401,6 +437,32 @@ const crawlSitemap = async ({
               results.actualUrl = actualUrl;
 
               await dataset.pushData(results);
+
+              // Discover <a> links from this page for the intelligent sitemap flow.
+              // This eliminates the need for a separate crawlDomain supplement phase
+              // that would re-visit all these pages just to extract links.
+              if (fromCrawlIntelligentSitemap) {
+                try {
+                  await enqueueLinks({
+                    selector: `a:not(${disallowedSelectorPatterns})`,
+                    strategy,
+                    requestQueue,
+                    transformRequestFunction: (req) => {
+                      try {
+                        req.url = req.url.replace(/(?<=&|\?)utm_.*?(&|$)/gim, '');
+                      } catch {}
+                      if (isDisallowedInRobotsTxt(req.url)) return null;
+                      if (isUrlPdf(req.url)) {
+                        req.skipNavigation = true;
+                      }
+                      req.label = req.url;
+                      return req;
+                    },
+                  });
+                } catch {
+                  // Best-effort link discovery; don't fail the scan
+                }
+              }
             }
           } else {
             guiInfoLog(guiInfoStatusTypes.SKIPPED, {
@@ -409,20 +471,37 @@ const crawlSitemap = async ({
             });
 
             if (isScanHtml) {
-              // carry through the HTTP status metadata
-              const status = response?.status();
-              const metadata =
-                typeof status === 'number'
-                  ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
-                  : STATUS_CODE_METADATA[2];
-
-              urlsCrawled.invalid.push({
-                actualUrl,
-                url: request.url,
-                pageTitle: request.url,
-                metadata,
-                httpStatusCode: typeof status === 'number' ? status : 0,
-              });
+              // Non-HTML content types (images, PDFs, binary files) and URLs
+              // that redirect to non-HTML resources should be classified as
+              // unsupported documents, not generic page errors.
+              const isNonHtmlContent =
+                contentType &&
+                !contentType.startsWith('text/html') &&
+                !contentType.includes('html');
+
+              if (isNonHtmlContent && status !== 0) {
+                urlsCrawled.userExcluded.push({
+                  actualUrl,
+                  url: request.url,
+                  pageTitle: request.url,
+                  metadata: STATUS_CODE_METADATA[1],
+                  httpStatusCode: 1,
+                });
+              } else {
+                const httpStatus = response?.status();
+                const metadata =
+                  typeof httpStatus === 'number'
+                    ? STATUS_CODE_METADATA[httpStatus] || STATUS_CODE_METADATA[599]
+                    : STATUS_CODE_METADATA[2];
+
+                urlsCrawled.invalid.push({
+                  actualUrl,
+                  url: request.url,
+                  pageTitle: request.url,
+                  metadata,
+                  httpStatusCode: typeof httpStatus === 'number' ? httpStatus : 0,
+                });
+              }
             }
           }
         } catch (e) {
@@ -437,7 +516,55 @@ const crawlSitemap = async ({
           return;
         }
 
+        // Handle download-triggered navigation errors: Playwright throws
+        // "Download is starting" when page.goto() hits a file download URL.
+        const isDownloadError = request.errorMessages?.some(
+          (msg: string) => msg.includes('Download is starting'),
+        );
+        if (isDownloadError) {
+          if (isScanPdfs) {
+            // Re-enqueue with skipNavigation so the requestHandler's PDF download path handles it
+            try {
+              await requestQueue.addRequest({
+                url: request.url,
+                skipNavigation: true,
+                label: request.url,
+                uniqueKey: `download_${request.url}`,
+              });
+            } catch {}
+          } else {
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
+            urlsCrawled.userExcluded.push({
+              url: request.url,
+              pageTitle: request.url,
+              actualUrl: request.url,
+              metadata: STATUS_CODE_METADATA[1],
+              httpStatusCode: 1,
+            });
+          }
+          return;
+        }
+
         const status = response?.status();
+
+        // Re-enqueue rate-limited (403) URLs once for a retry after concurrency recovers.
+        // Call onFailure to reduce concurrency immediately on rate-limit detection.
+        if (status === 403 && !request.userData?.rateLimitRetried) {
+          rateController.onFailure(status, crawler.autoscaledPool);
+          try {
+            await requestQueue.addRequest({
+              url: request.url,
+              label: request.url,
+              uniqueKey: `ratelimit_${request.url}`,
+              userData: { rateLimitRetried: true },
+            });
+          } catch {}
+          return;
+        }
+
         if (rateController.onFailure(status, crawler.autoscaledPool)) {
           consoleLogger.info(
             `Aborting crawl: consecutive HTTP failures threshold reached (site may be rate-limiting). Successfully scanned ${urlsCrawled.scanned.length} pages.`,

package/src/crawlers/custom/utils.ts

@@ -4,6 +4,7 @@
 import path from 'path';
 import { getDomain } from 'tldts';
 import { runAxeScript } from '../commonCrawlerFunc.js';
+import { capturePageData } from '../pageCapture.js';
 import { consoleLogger, guiInfoLog, silentLogger } from '../../logs.js';
 import { guiInfoStatusTypes } from '../../constants/constants.js';
 import { isSkippedUrl, validateCustomFlowLabel } from '../../constants/common.js';
@@ -159,6 +160,8 @@ export const runAxeScan = async (
 ) => {
   const result = await runAxeScript({ includeScreenshots, page, randomToken, customFlowDetails });
 
+  await capturePageData(page, page.url(), randomToken);
+
   await dataset.pushData(result);
 
   const rawTitle = result.pageTitle ?? '';

package/src/crawlers/pageCapture.ts

@@ -0,0 +1,172 @@
+import fs from 'fs-extra';
+import path from 'path';
+import crypto from 'crypto';
+import { Page, devices } from 'playwright';
+import { getStoragePath } from '../utils.js';
+
+const MOBILE_VIEWPORT_WIDTH = devices['iPhone 11'].viewport.width;
+const MOBILE_VIEWPORT_HEIGHT = devices['iPhone 11'].viewport.height;
+
+export interface PageCaptureEntry {
+  url: string;
+  hash: string;
+  domFile?: string;
+  desktopScreenshot?: string;
+  mobileScreenshot?: string;
+  errors: string[];
+}
+
+const captureEntries: Map<string, PageCaptureEntry> = new Map();
+
+export function getUrlHash(url: string): string {
+  return crypto.createHash('sha256').update(url).digest('hex').slice(0, 7);
+}
+
+function getTruncatedPath(url: string): string {
+  try {
+    const parsed = new URL(url);
+    let pathStr = parsed.pathname + (parsed.search || '');
+    pathStr = pathStr.replace(/^\//, '').replace(/\//g, '_').replace(/[^a-zA-Z0-9\-_.]/g, '_');
+    if (pathStr.length > 80) {
+      pathStr = pathStr.slice(0, 80);
+    }
+    return pathStr || 'index';
+  } catch {
+    return 'unknown';
+  }
+}
+
+function getPageDomsDir(randomToken: string): string {
+  const storagePath = getStoragePath(randomToken);
+  return path.join(storagePath, 'pageDOMs');
+}
+
+async function getUniqueFilePath(dir: string, baseName: string, ext: string): Promise<string> {
+  let candidate = path.join(dir, `${baseName}${ext}`);
+  if (!await fs.pathExists(candidate)) return candidate;
+
+  let counter = 2;
+  while (await fs.pathExists(candidate)) {
+    candidate = path.join(dir, `${baseName}-${counter}${ext}`);
+    counter++;
+  }
+  return candidate;
+}
+
+function getRelativeName(filePath: string, baseDir: string): string {
+  return path.relative(baseDir, filePath).replace(/\\/g, '/');
+}
+
+export function isSaveDomEnabled(): boolean {
+  return process.env.OOBEE_SAVE_DOM === '1' || process.env.OOBEE_SAVE_DOM === 'true';
+}
+
+export function isSavePageScreenshotEnabled(): boolean {
+  return (
+    process.env.OOBEE_SAVE_PAGE_SCREENSHOT === '1' ||
+    process.env.OOBEE_SAVE_PAGE_SCREENSHOT === 'true'
+  );
+}
+
+export function isPageCaptureEnabled(): boolean {
+  return isSaveDomEnabled() || isSavePageScreenshotEnabled();
+}
+
+export async function capturePageData(
+  page: Page,
+  url: string,
+  randomToken: string,
+): Promise<void> {
+  if (!isPageCaptureEnabled()) return;
+
+  const hash = getUrlHash(url);
+  const truncatedPath = getTruncatedPath(url);
+  const fileName = `${hash}-${truncatedPath}`;
+  const pageDomsDir = getPageDomsDir(randomToken);
+
+  const entry: PageCaptureEntry = {
+    url,
+    hash,
+    errors: [],
+  };
+
+  if (isSaveDomEnabled()) {
+    try {
+      await fs.ensureDir(pageDomsDir);
+      const domContent = await page.content();
+      const domFilePath = await getUniqueFilePath(pageDomsDir, fileName, '.html');
+      await fs.writeFile(domFilePath, domContent, 'utf-8');
+      entry.domFile = `pageDOMs/${getRelativeName(domFilePath, pageDomsDir)}`;
+    } catch (err) {
+      entry.errors.push(`DOM save failed: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+
+  if (isSavePageScreenshotEnabled()) {
+    const desktopDir = path.join(pageDomsDir, 'desktopPageScreenshots');
+    const mobileDir = path.join(pageDomsDir, 'mobilePageScreenshots');
+
+    try {
+      await fs.ensureDir(desktopDir);
+      const desktopPath = await getUniqueFilePath(desktopDir, fileName, '.png');
+      await page.screenshot({ path: desktopPath, fullPage: true });
+      entry.desktopScreenshot = `pageDOMs/desktopPageScreenshots/${getRelativeName(desktopPath, desktopDir)}`;
+    } catch (err) {
+      entry.errors.push(
+        `Desktop screenshot failed: ${err instanceof Error ? err.message : String(err)}`,
+      );
+    }
+
+    try {
+      await fs.ensureDir(mobileDir);
+      const currentViewport = page.viewportSize();
+
+      await page.setViewportSize({
+        width: MOBILE_VIEWPORT_WIDTH,
+        height: MOBILE_VIEWPORT_HEIGHT,
+      });
+      await page.waitForTimeout(500);
+
+      const mobilePath = await getUniqueFilePath(mobileDir, fileName, '.png');
+      await page.screenshot({ path: mobilePath, fullPage: true });
+      entry.mobileScreenshot = `pageDOMs/mobilePageScreenshots/${getRelativeName(mobilePath, mobileDir)}`;
+
+      if (currentViewport) {
+        await page.setViewportSize(currentViewport);
+      }
+    } catch (err) {
+      entry.errors.push(
+        `Mobile screenshot failed: ${err instanceof Error ? err.message : String(err)}`,
+      );
+    }
+  }
+
+  captureEntries.set(url, entry);
+}
+
+export async function writeManifest(randomToken: string): Promise<void> {
+  if (!isPageCaptureEnabled()) return;
+  if (captureEntries.size === 0) return;
+
+  const pageDomsDir = getPageDomsDir(randomToken);
+  await fs.ensureDir(pageDomsDir);
+
+  const manifest = {
+    generatedAt: new Date().toISOString(),
+    pages: Array.from(captureEntries.values()).map(entry => ({
+      url: entry.url,
+      hash: entry.hash,
+      ...(entry.domFile && { domFile: entry.domFile }),
+      ...(entry.desktopScreenshot && { desktopScreenshot: entry.desktopScreenshot }),
+      ...(entry.mobileScreenshot && { mobileScreenshot: entry.mobileScreenshot }),
+      errors: entry.errors,
+    })),
+  };
+
+  const manifestPath = path.join(pageDomsDir, 'domManifest.json');
+  await fs.writeFile(manifestPath, JSON.stringify(manifest, null, 2), 'utf-8');
+}
+
+export function resetCaptureEntries(): void {
+  captureEntries.clear();
+}

package/src/mergeAxeResults.ts

@@ -1110,7 +1110,7 @@ const generateArtifacts = async (
   }
 
   try {
-    const requestQueue = await RequestQueue.open(crawleeDir);
+    const requestQueue = await RequestQueue.open(`${crawleeDir}_rq`);
     await requestQueue.drop();
   } catch (error) {
     consoleLogger.info(`RequestQueue drop: ${error.message}`);
@@ -1120,6 +1120,7 @@ const generateArtifacts = async (
   const crawleePath = path.join(storagePath, 'crawlee');
   try {
     await fs.promises.rm(crawleePath, { recursive: true, force: true });
+    await fs.promises.rm(`${crawleePath}_rq`, { recursive: true, force: true });
   } catch {
     // Best-effort; storage was already dropped via API
   }

package/src/utils.ts

@@ -390,14 +390,31 @@ export const cleanUp = async (randomToken?: string, isError: boolean = false): P
       consoleLogger.warn(`Unable to force remove userDataDirectory: ${error.message}`);
     }
 
-    // Also remove _pool* sibling directories created by browser pool re-launches
-    try {
-      const poolDirs = globSync(`${constants.userDataDirectory}_pool*`);
+    // Also remove _pool* sibling directories created by browser pool re-launches.
+    // On Windows, Chrome writes files (optimization_guide_model_store,
+    // segmentation_platform, first_party_sets.db, etc.) during its shutdown
+    // sequence and may still hold file locks. Retry up to 10 times at 5s intervals.
+    const removePoolDirs = (): number => {
+      // On Windows, glob interprets backslashes as escape characters, not path
+      // separators. Normalize to forward slashes so the pattern actually matches.
+      const globPattern = `${constants.userDataDirectory}_pool*`.replace(/\\/g, '/');
+      const poolDirs = globSync(globPattern);
       for (const dir of poolDirs) {
         fs.rmSync(dir, { recursive: true, force: true });
       }
-    } catch (error) {
-      consoleLogger.warn(`Unable to remove pool directories: ${error.message}`);
+      return poolDirs.length;
+    };
+    for (let attempt = 0; attempt < 10; attempt++) {
+      try {
+        if (removePoolDirs() === 0) break; // nothing left to clean
+        break; // rmSync succeeded for all dirs
+      } catch {
+        if (attempt < 9) {
+          await new Promise(r => setTimeout(r, 5000));
+        } else {
+          consoleLogger.warn('Unable to remove pool directories after 10 attempts');
+        }
+      }
     }
   }
 
@@ -418,7 +435,7 @@ export const cleanUp = async (randomToken?: string, isError: boolean = false): P
       const crawleeDir = path.join(storagePath, 'crawlee');
       const dataset = await Dataset.open(crawleeDir);
       await dataset.drop();
-      const requestQueue = await RequestQueue.open(crawleeDir);
+      const requestQueue = await RequestQueue.open(`${crawleeDir}_rq`);
       await requestQueue.drop();
     } catch (error) {
       consoleLogger.info(`Crawlee storage drop in cleanUp: ${error.message}`);
@@ -428,6 +445,11 @@ export const cleanUp = async (randomToken?: string, isError: boolean = false): P
     } catch (error) {
       consoleLogger.warn(`Unable to force remove crawlee folder: ${error.message}`);
     }
+    try {
+      fs.rmSync(path.join(storagePath, 'crawlee_rq'), { recursive: true, force: true });
+    } catch (error) {
+      consoleLogger.warn(`Unable to force remove crawlee_rq folder: ${error.message}`);
+    }
 
     try {
       fs.rmSync(path.join(storagePath, 'pdfs'), { recursive: true, force: true });
@@ -1022,9 +1044,10 @@ export const zipResults = async (zipName: string, resultsPath: string): Promise<
   if (!stats.isDirectory()) {
     throw new Error(`resultsPath is not a directory: ${resultsPath}`);
   }
-  async function addFolderToZip(folderPath: string, zipFolder: JSZip): Promise<void> {
+  async function addFolderToZip(folderPath: string, zipFolder: JSZip, isRoot = false): Promise<void> {
     const items = await fs.readdir(folderPath);
     for (const item of items) {
+      if (isRoot && item.startsWith('crawlee')) continue;
       const fullPath = path.join(folderPath, item);
       const stats = await fs.stat(fullPath);
       if (stats.isDirectory()) {
@@ -1038,7 +1061,7 @@ export const zipResults = async (zipName: string, resultsPath: string): Promise<
   }
 
   const zip = new JSZip();
-  await addFolderToZip(resultsPath, zip);
+  await addFolderToZip(resultsPath, zip, true);
 
   const zipStream = zip.generateNodeStream({
     type: 'nodebuffer',