@govtechsg/oobee 0.10.97 → 0.11.0

package/src/crawlers/crawlIntelligentSitemap.ts

@@ -2,7 +2,7 @@ import fs from 'fs';
 import { chromium, Page } from 'playwright';
 import { EnqueueStrategy } from 'crawlee';
 import { createCrawleeSubFolders, splitAuthHeaders, addAuthRouteHandler } from './commonCrawlerFunc.js';
-import constants, { FileTypes, guiInfoStatusTypes, sitemapPaths } from '../constants/constants.js';
+import constants, { FileTypes, guiInfoStatusTypes, RuleFlags, sitemapPaths } from '../constants/constants.js';
 import { consoleLogger, guiInfoLog } from '../logs.js';
 import crawlDomain from './crawlDomain.js';
 import crawlSitemap from './crawlSitemap.js';
@@ -27,6 +27,7 @@ const crawlIntelligentSitemap = async (
   extraHTTPHeaders: Record<string, string>,
   safeMode: boolean,
   scanDuration: number,
+  ruleset: RuleFlags[] = [],
 ) => {
   const startTime = Date.now(); // Track start time
 
@@ -191,6 +192,7 @@ const crawlIntelligentSitemap = async (
       urlsCrawledFromIntelligent: urlsCrawled,
       crawledFromLocalFile: false,
       scanDuration: scanDuration > 0 ? remainingDuration : 0,
+      ruleset,
     });
   }
 
@@ -222,6 +224,7 @@ const crawlIntelligentSitemap = async (
       datasetFromIntelligent: dataset,
       urlsCrawledFromIntelligent: urlsCrawled,
       scanDuration: remainingScanDuration,
+      ruleset,
     });
   } else if (!hasDurationRemaining) {
     consoleLogger.info(

package/src/crawlers/crawlLocalFile.ts

@@ -127,6 +127,7 @@ export const crawlLocalFile = async ({
       datasetFromIntelligent,
       urlsCrawledFromIntelligent,
       crawledFromLocalFile: true,
+      ruleset,
     });
 
     urlsCrawled = { ...urlsCrawled, ...updatedUrlsCrawled };

package/src/crawlers/crawlRateController.ts

@@ -36,14 +36,10 @@ export class CrawlRateController {
   }
 
   onFailure(httpStatus: number | undefined, pool?: { maxConcurrency: number }): boolean {
-    if (typeof httpStatus !== 'number' || httpStatus < 400) {
-      return false;
-    }
-
     this.consecutiveSuccesses = 0;
     this.consecutiveFailures++;
 
-    if (pool && pool.maxConcurrency > 1) {
+    if (typeof httpStatus === 'number' && httpStatus >= 400 && pool && pool.maxConcurrency > 1) {
       pool.maxConcurrency = Math.max(1, Math.floor(pool.maxConcurrency / 2));
       consoleLogger.info(
         `Rate limited (HTTP ${httpStatus}) — reducing concurrency to ${pool.maxConcurrency}`,

package/src/crawlers/crawlSitemap.ts

@@ -4,6 +4,7 @@ import fs from 'fs';
 import {
   createCrawleeSubFolders,
   getPreLaunchHook,
+  getPostPageCloseHook,
   preNavigationHooks,
   runAxeScript,
   isUrlPdf,
@@ -14,12 +15,16 @@ import constants, {
   STATUS_CODE_METADATA,
   guiInfoStatusTypes,
   UrlsCrawled,
+  blackListedFileExtensions,
   disallowedListOfPatterns,
+  disallowedSelectorPatterns,
   FileTypes,
+  RuleFlags,
 } from '../constants/constants.js';
 import {
   getLinksFromSitemap,
   getPlaywrightLaunchOptions,
+  isDisallowedInRobotsTxt,
   isSkippedUrl,
   waitForPageLoaded,
   isFilePath,
@@ -55,6 +60,7 @@ const crawlSitemap = async ({
   datasetFromIntelligent = null,
   urlsCrawledFromIntelligent = null,
   crawledFromLocalFile = false,
+  ruleset = [],
 }: {
   sitemapUrl: string;
   randomToken: string;
@@ -76,14 +82,18 @@ const crawlSitemap = async ({
   datasetFromIntelligent?: Dataset;
   urlsCrawledFromIntelligent?: UrlsCrawled;
   crawledFromLocalFile?: boolean;
+  ruleset?: RuleFlags[];
 }) => {
   const crawlStartTime = Date.now();
   let dataset: crawlee.Dataset;
   let urlsCrawled: UrlsCrawled;
   let durationExceeded = false;
   let isAbortingScan = false;
+  const remainingBudget = fromCrawlIntelligentSitemap
+    ? Math.max(0, maxRequestsPerCrawl - urlsCrawledFromIntelligent.scanned.length)
+    : maxRequestsPerCrawl;
   const rateController = new CrawlRateController(
-    maxRequestsPerCrawl,
+    remainingBudget,
     specifiedMaxConcurrency || constants.maxConcurrency,
   );
   const initialNoSuccessFailureAbortThreshold = Math.max(5, Math.min(maxRequestsPerCrawl, 25));
@@ -127,6 +137,12 @@ const crawlSitemap = async ({
     sources: linksFromSitemap,
   });
 
+  // Always create a request queue alongside the request list. An empty queue
+  // has zero impact on crawl behavior (Crawlee processes RequestList first).
+  // Having it available enables: download re-enqueue for PDF scanning,
+  // 403 rate-limit retry, and enqueueLinks for intelligent sitemap discovery.
+  const { requestQueue } = await createCrawleeSubFolders(randomToken);
+
   const crawler = register(
     new crawlee.PlaywrightCrawler({
       launchContext: {
@@ -136,6 +152,8 @@ const crawlSitemap = async ({
       retryOnBlocked: true,
       browserPoolOptions: {
         useFingerprints: false,
+        retireBrowserAfterPageCount: 500,
+        closeInactiveBrowserAfterSecs: 30,
         preLaunchHooks: [
           getPreLaunchHook(userDataDirectory),
           async (_pageId, launchContext) => {
@@ -150,8 +168,10 @@ const crawlSitemap = async ({
             };
           },
         ],
+        postPageCloseHooks: [getPostPageCloseHook(userDataDirectory)],
       },
       requestList,
+      requestQueue,
       maxRequestRetries: 3,
       maxSessionRotations: 1,
       postNavigationHooks: [
@@ -191,6 +211,8 @@ const crawlSitemap = async ({
                 const root = document.documentElement || document.body || document;
                 if (!root || typeof observer.observe !== 'function') {
                   resolve('No root node to observe.');
+                } else {
+                  observer.observe(root, { childList: true, subtree: true });
                 }
               });
             });
@@ -215,16 +237,22 @@ const crawlSitemap = async ({
           if (isNotSupportedDocument) {
             request.skipNavigation = true;
             request.userData.isNotSupportedDocument = true;
-
-            // Log for verification (optional, but not required for correctness)
-            // console.log(`[SKIP] Not supported: ${request.url}`);
-
             return;
           }
+
+          try {
+            const pathname = new URL(url).pathname;
+            const ext = pathname.split('.').pop();
+            if (ext && blackListedFileExtensions.includes(ext)) {
+              request.skipNavigation = true;
+              request.userData.isNotSupportedDocument = true;
+              return;
+            }
+          } catch {}
         },
       ],
       requestHandlerTimeoutSecs: 90,
-      requestHandler: async ({ page, request, response, sendRequest }) => {
+      requestHandler: async ({ page, request, response, sendRequest, enqueueLinks }) => {
         // Log documents that are not supported
         if (request.userData?.isNotSupportedDocument) {
           guiInfoLog(guiInfoStatusTypes.SKIPPED, {
@@ -334,7 +362,7 @@ const crawlSitemap = async ({
               return;
             }
 
-            const results = await runAxeScript({ includeScreenshots, page, randomToken });
+            const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
 
             // Detect JS redirects that fire during/after axe scan.
             // Listen for navigation, then give a brief window for pending redirects to complete.
@@ -391,6 +419,32 @@ const crawlSitemap = async ({
               results.actualUrl = actualUrl;
 
               await dataset.pushData(results);
+
+              // Discover <a> links from this page for the intelligent sitemap flow.
+              // This eliminates the need for a separate crawlDomain supplement phase
+              // that would re-visit all these pages just to extract links.
+              if (fromCrawlIntelligentSitemap) {
+                try {
+                  await enqueueLinks({
+                    selector: `a:not(${disallowedSelectorPatterns})`,
+                    strategy,
+                    requestQueue,
+                    transformRequestFunction: (req) => {
+                      try {
+                        req.url = req.url.replace(/(?<=&|\?)utm_.*?(&|$)/gim, '');
+                      } catch {}
+                      if (isDisallowedInRobotsTxt(req.url)) return null;
+                      if (isUrlPdf(req.url)) {
+                        req.skipNavigation = true;
+                      }
+                      req.label = req.url;
+                      return req;
+                    },
+                  });
+                } catch {
+                  // Best-effort link discovery; don't fail the scan
+                }
+              }
             }
           } else {
             guiInfoLog(guiInfoStatusTypes.SKIPPED, {
@@ -399,20 +453,37 @@ const crawlSitemap = async ({
             });
 
             if (isScanHtml) {
-              // carry through the HTTP status metadata
-              const status = response?.status();
-              const metadata =
-                typeof status === 'number'
-                  ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
-                  : STATUS_CODE_METADATA[2];
-
-              urlsCrawled.invalid.push({
-                actualUrl,
-                url: request.url,
-                pageTitle: request.url,
-                metadata,
-                httpStatusCode: typeof status === 'number' ? status : 0,
-              });
+              // Non-HTML content types (images, PDFs, binary files) and URLs
+              // that redirect to non-HTML resources should be classified as
+              // unsupported documents, not generic page errors.
+              const isNonHtmlContent =
+                contentType &&
+                !contentType.startsWith('text/html') &&
+                !contentType.includes('html');
+
+              if (isNonHtmlContent && status !== 0) {
+                urlsCrawled.userExcluded.push({
+                  actualUrl,
+                  url: request.url,
+                  pageTitle: request.url,
+                  metadata: STATUS_CODE_METADATA[1],
+                  httpStatusCode: 1,
+                });
+              } else {
+                const httpStatus = response?.status();
+                const metadata =
+                  typeof httpStatus === 'number'
+                    ? STATUS_CODE_METADATA[httpStatus] || STATUS_CODE_METADATA[599]
+                    : STATUS_CODE_METADATA[2];
+
+                urlsCrawled.invalid.push({
+                  actualUrl,
+                  url: request.url,
+                  pageTitle: request.url,
+                  metadata,
+                  httpStatusCode: typeof httpStatus === 'number' ? httpStatus : 0,
+                });
+              }
             }
           }
         } catch (e) {
@@ -427,7 +498,55 @@ const crawlSitemap = async ({
           return;
         }
 
+        // Handle download-triggered navigation errors: Playwright throws
+        // "Download is starting" when page.goto() hits a file download URL.
+        const isDownloadError = request.errorMessages?.some(
+          (msg: string) => msg.includes('Download is starting'),
+        );
+        if (isDownloadError) {
+          if (isScanPdfs) {
+            // Re-enqueue with skipNavigation so the requestHandler's PDF download path handles it
+            try {
+              await requestQueue.addRequest({
+                url: request.url,
+                skipNavigation: true,
+                label: request.url,
+                uniqueKey: `download_${request.url}`,
+              });
+            } catch {}
+          } else {
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
+            urlsCrawled.userExcluded.push({
+              url: request.url,
+              pageTitle: request.url,
+              actualUrl: request.url,
+              metadata: STATUS_CODE_METADATA[1],
+              httpStatusCode: 1,
+            });
+          }
+          return;
+        }
+
         const status = response?.status();
+
+        // Re-enqueue rate-limited (403) URLs once for a retry after concurrency recovers.
+        // Don't call onFailure here — the re-enqueued request gets a fresh attempt.
+        // If it fails again (rateLimitRetried=true), it falls through to the normal path.
+        if (status === 403 && !request.userData?.rateLimitRetried) {
+          try {
+            await requestQueue.addRequest({
+              url: request.url,
+              label: request.url,
+              uniqueKey: `ratelimit_${request.url}`,
+              userData: { rateLimitRetried: true },
+            });
+          } catch {}
+          return;
+        }
+
         if (rateController.onFailure(status, crawler.autoscaledPool)) {
           consoleLogger.info(
             `Aborting crawl: consecutive HTTP failures threshold reached (site may be rate-limiting). Successfully scanned ${urlsCrawled.scanned.length} pages.`,

package/src/crawlers/custom/flagUnlabelledClickableElements.ts

@@ -457,8 +457,13 @@ function hasPointerCursor(node: Node): boolean {
   function isElementTooSmall(element: Element) {
     // Get the bounding rectangle of the element
     const rect = element.getBoundingClientRect();
-    
-    // Check if the element has a valid width or height
+
+    // If either dimension is 0, the element is non-perceivable
+    if (rect.width === 0 || rect.height === 0) {
+        return true;
+    }
+
+    // Check if the element has a valid width and height
     if (rect.width > 0 || rect.height > 0) {
         return false; // Element is not too small
     }

package/src/mergeAxeResults.ts

@@ -1110,7 +1110,7 @@ const generateArtifacts = async (
   }
 
   try {
-    const requestQueue = await RequestQueue.open(crawleeDir);
+    const requestQueue = await RequestQueue.open(`${crawleeDir}_rq`);
     await requestQueue.drop();
   } catch (error) {
     consoleLogger.info(`RequestQueue drop: ${error.message}`);
@@ -1120,6 +1120,7 @@ const generateArtifacts = async (
   const crawleePath = path.join(storagePath, 'crawlee');
   try {
     await fs.promises.rm(crawleePath, { recursive: true, force: true });
+    await fs.promises.rm(`${crawleePath}_rq`, { recursive: true, force: true });
   } catch {
     // Best-effort; storage was already dropped via API
   }

package/src/utils.ts

@@ -390,14 +390,31 @@ export const cleanUp = async (randomToken?: string, isError: boolean = false): P
       consoleLogger.warn(`Unable to force remove userDataDirectory: ${error.message}`);
     }
 
-    // Also remove _pool* sibling directories created by browser pool re-launches
-    try {
-      const poolDirs = globSync(`${constants.userDataDirectory}_pool*`);
+    // Also remove _pool* sibling directories created by browser pool re-launches.
+    // On Windows, Chrome writes files (optimization_guide_model_store,
+    // segmentation_platform, first_party_sets.db, etc.) during its shutdown
+    // sequence and may still hold file locks. Retry up to 10 times at 5s intervals.
+    const removePoolDirs = (): number => {
+      // On Windows, glob interprets backslashes as escape characters, not path
+      // separators. Normalize to forward slashes so the pattern actually matches.
+      const globPattern = `${constants.userDataDirectory}_pool*`.replace(/\\/g, '/');
+      const poolDirs = globSync(globPattern);
       for (const dir of poolDirs) {
         fs.rmSync(dir, { recursive: true, force: true });
       }
-    } catch (error) {
-      consoleLogger.warn(`Unable to remove pool directories: ${error.message}`);
+      return poolDirs.length;
+    };
+    for (let attempt = 0; attempt < 10; attempt++) {
+      try {
+        if (removePoolDirs() === 0) break; // nothing left to clean
+        break; // rmSync succeeded for all dirs
+      } catch {
+        if (attempt < 9) {
+          await new Promise(r => setTimeout(r, 5000));
+        } else {
+          consoleLogger.warn('Unable to remove pool directories after 10 attempts');
+        }
+      }
     }
   }
 
@@ -418,7 +435,7 @@ export const cleanUp = async (randomToken?: string, isError: boolean = false): P
       const crawleeDir = path.join(storagePath, 'crawlee');
       const dataset = await Dataset.open(crawleeDir);
       await dataset.drop();
-      const requestQueue = await RequestQueue.open(crawleeDir);
+      const requestQueue = await RequestQueue.open(`${crawleeDir}_rq`);
       await requestQueue.drop();
     } catch (error) {
       consoleLogger.info(`Crawlee storage drop in cleanUp: ${error.message}`);
@@ -428,6 +445,11 @@ export const cleanUp = async (randomToken?: string, isError: boolean = false): P
     } catch (error) {
       consoleLogger.warn(`Unable to force remove crawlee folder: ${error.message}`);
     }
+    try {
+      fs.rmSync(path.join(storagePath, 'crawlee_rq'), { recursive: true, force: true });
+    } catch (error) {
+      consoleLogger.warn(`Unable to force remove crawlee_rq folder: ${error.message}`);
+    }
 
     try {
       fs.rmSync(path.join(storagePath, 'pdfs'), { recursive: true, force: true });
@@ -1022,9 +1044,10 @@ export const zipResults = async (zipName: string, resultsPath: string): Promise<
   if (!stats.isDirectory()) {
     throw new Error(`resultsPath is not a directory: ${resultsPath}`);
   }
-  async function addFolderToZip(folderPath: string, zipFolder: JSZip): Promise<void> {
+  async function addFolderToZip(folderPath: string, zipFolder: JSZip, isRoot = false): Promise<void> {
     const items = await fs.readdir(folderPath);
     for (const item of items) {
+      if (isRoot && item.startsWith('crawlee')) continue;
       const fullPath = path.join(folderPath, item);
       const stats = await fs.stat(fullPath);
       if (stats.isDirectory()) {
@@ -1038,7 +1061,7 @@ export const zipResults = async (zipName: string, resultsPath: string): Promise<
   }
 
   const zip = new JSZip();
-  await addFolderToZip(resultsPath, zip);
+  await addFolderToZip(resultsPath, zip, true);
 
   const zipStream = zip.generateNodeStream({
     type: 'nodebuffer',