@govtechsg/oobee 0.10.98 → 0.11.0

package/src/crawlers/commonCrawlerFunc.ts

@@ -1135,10 +1135,9 @@ export const createCrawleeSubFolders = async (
   randomToken: string,
 ): Promise<{ dataset: Dataset; requestQueue: RequestQueue }> => {
 
-  const crawleeDir = path.join(getStoragePath(randomToken),"crawlee");
-
-  const dataset = await Dataset.open(crawleeDir);
-  const requestQueue = await RequestQueue.open(crawleeDir);
+  const baseDir = path.join(getStoragePath(randomToken), "crawlee");
+  const dataset = await Dataset.open(baseDir);
+  const requestQueue = await RequestQueue.open(`${baseDir}_rq`);
   return { dataset, requestQueue };
 };
 
@@ -1215,77 +1214,90 @@ export const postNavigationHooks = [
 
 export const getPreLaunchHook = (userDataDirectory: string) => {
   let launchCount = 0;
+  let previousPoolDir: string | null = null;
 
   return async (_pageId: string, launchContext: any) => {
     const fsp = await import('fs/promises').then(m => m.default);
     launchCount += 1;
 
-    // First launch uses the base directory; subsequent launches get a unique
-    // directory so that lingering file handles from a retired browser don't
-    // cause Chrome exit code 21 on Windows.
-    const effectiveDir =
-      launchCount === 1
-        ? userDataDirectory
-        : `${userDataDirectory}_pool${launchCount}`;
+    // Clean up the previous pool directory. When preLaunchHooks fires, the
+    // previous browser has been retired and is finishing its last pages.
+    // Schedule cleanup with enough delay for Chrome to fully exit —
+    // closeInactiveBrowserAfterSecs (30s) + Windows file-lock grace.
+    if (previousPoolDir) {
+      const dirToClean = previousPoolDir;
+      const isWin = process.platform === 'win32';
+      const delay = isWin ? 40000 : 35000;
+      setTimeout(async () => {
+        for (let attempt = 0; attempt < 3; attempt++) {
+          try {
+            await fsp.rm(dirToClean, { recursive: true, force: true });
+            return;
+          } catch {
+            await new Promise(r => setTimeout(r, 5000));
+          }
+        }
+      }, delay);
+    }
+
+    // Every browser gets its own directory. The base userDataDirectory is
+    // treated as a read-only cookie source (the pristine clone of the user's
+    // profile) and is never used directly by a running browser.
+    const effectiveDir = `${userDataDirectory}_pool${launchCount}`;
 
     await fsp.mkdir(effectiveDir, { recursive: true });
 
-    // For pool re-launches, copy only auth-relevant files from the base
-    // directory so authenticated sessions are preserved without cloning the
-    // entire profile (which grows with caches, IndexedDB, etc. during the crawl).
-    if (launchCount > 1) {
-      try {
-        // Copy top-level Local State (cookie encryption keys on Windows)
-        const localStateSrc = path.join(userDataDirectory, 'Local State');
-        if (await fsp.stat(localStateSrc).catch(() => null)) {
-          await fsp.copyFile(localStateSrc, path.join(effectiveDir, 'Local State')).catch(() => {});
-        }
+    // Copy auth-relevant files from the pristine base directory so
+    // authenticated sessions are preserved across pool rotations.
+    try {
+      const localStateSrc = path.join(userDataDirectory, 'Local State');
+      if (await fsp.stat(localStateSrc).catch(() => null)) {
+        await fsp.copyFile(localStateSrc, path.join(effectiveDir, 'Local State')).catch(() => {});
+      }
 
-        // Find profile directories (Default, Profile 1, Profile 2, etc.)
-        const entries = await fsp.readdir(userDataDirectory, { withFileTypes: true }).catch(() => []);
-        const profileDirs = (entries as any[]).filter(
-          (e: any) => e.isDirectory() && /^(Default|Profile \d+)$/i.test(e.name),
-        );
-
-        for (const profile of profileDirs) {
-          const srcProfile = path.join(userDataDirectory, profile.name);
-          const destProfile = path.join(effectiveDir, profile.name);
-          await fsp.mkdir(destProfile, { recursive: true }).catch(() => {});
-
-          // Cookies (macOS layout: <Profile>/Cookies)
-          const cookiesSrc = path.join(srcProfile, 'Cookies');
-          if (await fsp.stat(cookiesSrc).catch(() => null)) {
-            await fsp.copyFile(cookiesSrc, path.join(destProfile, 'Cookies')).catch(() => {});
-          }
+      const entries = await fsp.readdir(userDataDirectory, { withFileTypes: true }).catch(() => []);
+      const profileDirs = (entries as any[]).filter(
+        (e: any) => e.isDirectory() && /^(Default|Profile \d+)$/i.test(e.name),
+      );
 
-          // Cookies (Windows layout: <Profile>/Network/Cookies)
-          const networkCookiesSrc = path.join(srcProfile, 'Network', 'Cookies');
-          if (await fsp.stat(networkCookiesSrc).catch(() => null)) {
-            const destNetwork = path.join(destProfile, 'Network');
-            await fsp.mkdir(destNetwork, { recursive: true }).catch(() => {});
-            await fsp.copyFile(networkCookiesSrc, path.join(destNetwork, 'Cookies')).catch(() => {});
-          }
+      for (const profile of profileDirs) {
+        const srcProfile = path.join(userDataDirectory, profile.name);
+        const destProfile = path.join(effectiveDir, profile.name);
+        await fsp.mkdir(destProfile, { recursive: true }).catch(() => {});
 
-          // Local Storage (auth tokens, session data)
-          const localStorageSrc = path.join(srcProfile, 'Local Storage');
-          const localStorageStat = await fsp.stat(localStorageSrc).catch(() => null);
-          if (localStorageStat && localStorageStat.isDirectory()) {
-            const destLocalStorage = path.join(destProfile, 'Local Storage');
-            await fsp.mkdir(destLocalStorage, { recursive: true }).catch(() => {});
-            const lsFiles = await fsp.readdir(localStorageSrc).catch(() => []);
-            await Promise.all(
-              lsFiles.map(f =>
-                fsp.copyFile(
-                  path.join(localStorageSrc, f),
-                  path.join(destLocalStorage, f),
-                ).catch(() => {}),
-              ),
-            );
-          }
+        // Cookies (macOS layout: <Profile>/Cookies)
+        const cookiesSrc = path.join(srcProfile, 'Cookies');
+        if (await fsp.stat(cookiesSrc).catch(() => null)) {
+          await fsp.copyFile(cookiesSrc, path.join(destProfile, 'Cookies')).catch(() => {});
+        }
+
+        // Cookies (Windows layout: <Profile>/Network/Cookies)
+        const networkCookiesSrc = path.join(srcProfile, 'Network', 'Cookies');
+        if (await fsp.stat(networkCookiesSrc).catch(() => null)) {
+          const destNetwork = path.join(destProfile, 'Network');
+          await fsp.mkdir(destNetwork, { recursive: true }).catch(() => {});
+          await fsp.copyFile(networkCookiesSrc, path.join(destNetwork, 'Cookies')).catch(() => {});
+        }
+
+        // Local Storage (auth tokens, session data)
+        const localStorageSrc = path.join(srcProfile, 'Local Storage');
+        const localStorageStat = await fsp.stat(localStorageSrc).catch(() => null);
+        if (localStorageStat && localStorageStat.isDirectory()) {
+          const destLocalStorage = path.join(destProfile, 'Local Storage');
+          await fsp.mkdir(destLocalStorage, { recursive: true }).catch(() => {});
+          const lsFiles = await fsp.readdir(localStorageSrc).catch(() => []);
+          await Promise.all(
+            lsFiles.map(f =>
+              fsp.copyFile(
+                path.join(localStorageSrc, f),
+                path.join(destLocalStorage, f),
+              ).catch(() => {}),
+            ),
+          );
         }
-      } catch {
-        // Silent fallback: use empty profile if clone fails
       }
+    } catch {
+      // Silent fallback: use empty profile if clone fails
     }
 
     // Clean any stale lock files that may block browser launches on Windows
@@ -1299,11 +1311,38 @@ export const getPreLaunchHook = (userDataDirectory: string) => {
     ];
     await Promise.all(lockFiles.map(f => fsp.rm(f, { force: true }).catch(() => {})));
 
+    previousPoolDir = effectiveDir;
     // eslint-disable-next-line no-param-reassign
     launchContext.userDataDir = effectiveDir;
   };
 };
 
+export const getPostPageCloseHook = (userDataDirectory: string) => {
+  return async (_pageId: string, browserController: any) => {
+    if (browserController.activePages === 0) {
+      const dir = browserController.launchContext?.userDataDir;
+      if (dir && dir !== userDataDirectory && dir.includes('_pool')) {
+        const fsp = await import('fs/promises').then(m => m.default);
+        // Chrome on Windows writes optimization/segmentation data during shutdown;
+        // wait longer (5s) for the process to fully release file handles.
+        const isWin = process.platform === 'win32';
+        const initialDelay = isWin ? 5000 : 2000;
+        setTimeout(async () => {
+          for (let attempt = 0; attempt < 3; attempt++) {
+            try {
+              await fsp.rm(dir, { recursive: true, force: true });
+              return;
+            } catch {
+              // Retry after a short back-off (Windows file locks)
+              await new Promise(r => setTimeout(r, 2000));
+            }
+          }
+        }, initialDelay);
+      }
+    }
+  };
+};
+
 export const failedRequestHandler = async ({ request }: { request: Request }) => {
   guiInfoLog(guiInfoStatusTypes.ERROR, { numScanned: 0, urlScanned: request.url });
   log.error(`Failed Request - ${request.url}: ${request.errorMessages}`);

package/src/crawlers/crawlDomain.ts

@@ -5,6 +5,7 @@ import type { PlaywrightCrawlingContext, RequestOptions } from 'crawlee';
 import {
   createCrawleeSubFolders,
   getPreLaunchHook,
+  getPostPageCloseHook,
   preNavigationHooks,
   runAxeScript,
   isUrlPdf,
@@ -418,12 +419,23 @@ const crawlDomain = async ({
             };
           },
         ],
+        postPageCloseHooks: [getPostPageCloseHook(userDataDirectory)],
       },
       requestQueue,
       maxRequestRetries: 3,
       maxSessionRotations: 1,
       preNavigationHooks: [
         ...preNavigationHooks(extraHTTPHeaders),
+        async ({ request }) => {
+          const url = request.url.toLowerCase();
+          try {
+            const pathname = new URL(url).pathname;
+            const ext = pathname.split('.').pop();
+            if (ext && blackListedFileExtensions.includes(ext)) {
+              request.skipNavigation = true;
+            }
+          } catch {}
+        },
       ],
       postNavigationHooks: [
         async crawlingContext => {
@@ -555,21 +567,17 @@ const crawlDomain = async ({
             (request.skipNavigation && actualUrl === 'about:blank')
           ) {
             if (!isScanPdfs) {
-              // Don't inform the user it is skipped since web crawler is best-effort.
-              /*
-            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-              numScanned: urlsCrawled.scanned.length,
-              urlScanned: request.url,
-            });
-            urlsCrawled.userExcluded.push({
-              url: request.url,
-              pageTitle: request.url,
-              actualUrl: request.url, // because about:blank is not useful
-              metadata: STATUS_CODE_METADATA[1],
-              httpStatusCode: 0,
-            });
-            */
-
+              guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                numScanned: urlsCrawled.scanned.length,
+                urlScanned: request.url,
+              });
+              urlsCrawled.userExcluded.push({
+                url: request.url,
+                pageTitle: request.url,
+                actualUrl: request.url,
+                metadata: STATUS_CODE_METADATA[1],
+                httpStatusCode: 1,
+              });
               return;
             }
             const { pdfFileName, url: downloadedPdfUrl } = handlePdfDownload(
@@ -585,20 +593,17 @@ const crawlDomain = async ({
           }
 
           if (isBlacklistedFileExtensions(actualUrl, blackListedFileExtensions)) {
-            // Don't inform the user it is skipped since web crawler is best-effort.
-            /*
-          guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-            numScanned: urlsCrawled.scanned.length,
-            urlScanned: request.url,
-          });
-          urlsCrawled.userExcluded.push({
-            url: request.url,
-            pageTitle: request.url,
-            actualUrl, // because about:blank is not useful
-            metadata: STATUS_CODE_METADATA[1],
-            httpStatusCode: 0,
-          });
-          */
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
+            urlsCrawled.userExcluded.push({
+              url: request.url,
+              pageTitle: request.url,
+              actualUrl,
+              metadata: STATUS_CODE_METADATA[1],
+              httpStatusCode: 1,
+            });
             return;
           }
 
@@ -800,7 +805,59 @@ const crawlDomain = async ({
           return;
         }
 
+        // Handle download-triggered navigation errors: Playwright throws
+        // "Download is starting" when page.goto() hits a file download URL.
+        // Crawlee retries 3 times (all fail) then lands here.
+        const isDownloadError = request.errorMessages?.some(
+          (msg: string) => msg.includes('Download is starting'),
+        );
+        if (isDownloadError) {
+          if (isScanPdfs) {
+            // Re-enqueue with skipNavigation so the requestHandler's PDF download path handles it
+            try {
+              await requestQueue.addRequest({
+                url: request.url,
+                skipNavigation: true,
+                label: request.url,
+                uniqueKey: `download_${request.url}`,
+              });
+            } catch {}
+          } else {
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
+            urlsCrawled.userExcluded.push({
+              url: request.url,
+              pageTitle: request.url,
+              actualUrl: request.url,
+              metadata: STATUS_CODE_METADATA[1],
+              httpStatusCode: 1,
+            });
+          }
+          return;
+        }
+
         const status = response?.status();
+
+        // Re-enqueue rate-limited (403) URLs once for a retry after concurrency recovers.
+        // Without this, URLs that fail during a rate-limit burst are permanently lost
+        // even though the site is accessible at lower concurrency.
+        // Don't call onFailure here — the re-enqueued request will get a fresh attempt.
+        // If it fails again (rateLimitRetried=true), it falls through to the normal
+        // onFailure + circuit breaker path below.
+        if (status === 403 && !request.userData?.rateLimitRetried) {
+          try {
+            await requestQueue.addRequest({
+              url: request.url,
+              label: request.url,
+              uniqueKey: `ratelimit_${request.url}`,
+              userData: { rateLimitRetried: true },
+            });
+          } catch {}
+          return;
+        }
+
         if (rateController.onFailure(status, crawler.autoscaledPool)) {
           consoleLogger.info(
             `Aborting crawl: consecutive HTTP failures threshold reached (site may be rate-limiting). Successfully scanned ${urlsCrawled.scanned.length} pages.`,
@@ -844,7 +901,9 @@ const crawlDomain = async ({
 
   // Additional passes: keep re-visiting scanned seed-hostname pages for
   // click-discovery until no new pages are found or limits are reached.
-  if (!safeMode && !isAbortingScanNow && !durationExceeded) {
+  // Skip when called from intelligent sitemap — the domain phase is only meant
+  // to discover new pages via <a> links, not re-click 3000+ already-scanned pages.
+  if (!safeMode && !isAbortingScanNow && !durationExceeded && !fromCrawlIntelligentSitemap) {
     const seedHostname = new URL(url).hostname;
     const clickPassVisited = new Set<string>();
     let prevScannedCount: number;

package/src/crawlers/crawlSitemap.ts

@@ -4,6 +4,7 @@ import fs from 'fs';
 import {
   createCrawleeSubFolders,
   getPreLaunchHook,
+  getPostPageCloseHook,
   preNavigationHooks,
   runAxeScript,
   isUrlPdf,
@@ -14,13 +15,16 @@ import constants, {
   STATUS_CODE_METADATA,
   guiInfoStatusTypes,
   UrlsCrawled,
+  blackListedFileExtensions,
   disallowedListOfPatterns,
+  disallowedSelectorPatterns,
   FileTypes,
   RuleFlags,
 } from '../constants/constants.js';
 import {
   getLinksFromSitemap,
   getPlaywrightLaunchOptions,
+  isDisallowedInRobotsTxt,
   isSkippedUrl,
   waitForPageLoaded,
   isFilePath,
@@ -133,6 +137,12 @@ const crawlSitemap = async ({
     sources: linksFromSitemap,
   });
 
+  // Always create a request queue alongside the request list. An empty queue
+  // has zero impact on crawl behavior (Crawlee processes RequestList first).
+  // Having it available enables: download re-enqueue for PDF scanning,
+  // 403 rate-limit retry, and enqueueLinks for intelligent sitemap discovery.
+  const { requestQueue } = await createCrawleeSubFolders(randomToken);
+
   const crawler = register(
     new crawlee.PlaywrightCrawler({
       launchContext: {
@@ -158,8 +168,10 @@ const crawlSitemap = async ({
             };
           },
         ],
+        postPageCloseHooks: [getPostPageCloseHook(userDataDirectory)],
       },
       requestList,
+      requestQueue,
       maxRequestRetries: 3,
       maxSessionRotations: 1,
       postNavigationHooks: [
@@ -225,16 +237,22 @@ const crawlSitemap = async ({
           if (isNotSupportedDocument) {
             request.skipNavigation = true;
             request.userData.isNotSupportedDocument = true;
-
-            // Log for verification (optional, but not required for correctness)
-            // console.log(`[SKIP] Not supported: ${request.url}`);
-
             return;
           }
+
+          try {
+            const pathname = new URL(url).pathname;
+            const ext = pathname.split('.').pop();
+            if (ext && blackListedFileExtensions.includes(ext)) {
+              request.skipNavigation = true;
+              request.userData.isNotSupportedDocument = true;
+              return;
+            }
+          } catch {}
         },
       ],
       requestHandlerTimeoutSecs: 90,
-      requestHandler: async ({ page, request, response, sendRequest }) => {
+      requestHandler: async ({ page, request, response, sendRequest, enqueueLinks }) => {
         // Log documents that are not supported
         if (request.userData?.isNotSupportedDocument) {
           guiInfoLog(guiInfoStatusTypes.SKIPPED, {
@@ -401,6 +419,32 @@ const crawlSitemap = async ({
               results.actualUrl = actualUrl;
 
               await dataset.pushData(results);
+
+              // Discover <a> links from this page for the intelligent sitemap flow.
+              // This eliminates the need for a separate crawlDomain supplement phase
+              // that would re-visit all these pages just to extract links.
+              if (fromCrawlIntelligentSitemap) {
+                try {
+                  await enqueueLinks({
+                    selector: `a:not(${disallowedSelectorPatterns})`,
+                    strategy,
+                    requestQueue,
+                    transformRequestFunction: (req) => {
+                      try {
+                        req.url = req.url.replace(/(?<=&|\?)utm_.*?(&|$)/gim, '');
+                      } catch {}
+                      if (isDisallowedInRobotsTxt(req.url)) return null;
+                      if (isUrlPdf(req.url)) {
+                        req.skipNavigation = true;
+                      }
+                      req.label = req.url;
+                      return req;
+                    },
+                  });
+                } catch {
+                  // Best-effort link discovery; don't fail the scan
+                }
+              }
             }
           } else {
             guiInfoLog(guiInfoStatusTypes.SKIPPED, {
@@ -409,20 +453,37 @@ const crawlSitemap = async ({
             });
 
             if (isScanHtml) {
-              // carry through the HTTP status metadata
-              const status = response?.status();
-              const metadata =
-                typeof status === 'number'
-                  ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
-                  : STATUS_CODE_METADATA[2];
-
-              urlsCrawled.invalid.push({
-                actualUrl,
-                url: request.url,
-                pageTitle: request.url,
-                metadata,
-                httpStatusCode: typeof status === 'number' ? status : 0,
-              });
+              // Non-HTML content types (images, PDFs, binary files) and URLs
+              // that redirect to non-HTML resources should be classified as
+              // unsupported documents, not generic page errors.
+              const isNonHtmlContent =
+                contentType &&
+                !contentType.startsWith('text/html') &&
+                !contentType.includes('html');
+
+              if (isNonHtmlContent && status !== 0) {
+                urlsCrawled.userExcluded.push({
+                  actualUrl,
+                  url: request.url,
+                  pageTitle: request.url,
+                  metadata: STATUS_CODE_METADATA[1],
+                  httpStatusCode: 1,
+                });
+              } else {
+                const httpStatus = response?.status();
+                const metadata =
+                  typeof httpStatus === 'number'
+                    ? STATUS_CODE_METADATA[httpStatus] || STATUS_CODE_METADATA[599]
+                    : STATUS_CODE_METADATA[2];
+
+                urlsCrawled.invalid.push({
+                  actualUrl,
+                  url: request.url,
+                  pageTitle: request.url,
+                  metadata,
+                  httpStatusCode: typeof httpStatus === 'number' ? httpStatus : 0,
+                });
+              }
             }
           }
         } catch (e) {
@@ -437,7 +498,55 @@ const crawlSitemap = async ({
           return;
         }
 
+        // Handle download-triggered navigation errors: Playwright throws
+        // "Download is starting" when page.goto() hits a file download URL.
+        const isDownloadError = request.errorMessages?.some(
+          (msg: string) => msg.includes('Download is starting'),
+        );
+        if (isDownloadError) {
+          if (isScanPdfs) {
+            // Re-enqueue with skipNavigation so the requestHandler's PDF download path handles it
+            try {
+              await requestQueue.addRequest({
+                url: request.url,
+                skipNavigation: true,
+                label: request.url,
+                uniqueKey: `download_${request.url}`,
+              });
+            } catch {}
+          } else {
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
+            urlsCrawled.userExcluded.push({
+              url: request.url,
+              pageTitle: request.url,
+              actualUrl: request.url,
+              metadata: STATUS_CODE_METADATA[1],
+              httpStatusCode: 1,
+            });
+          }
+          return;
+        }
+
         const status = response?.status();
+
+        // Re-enqueue rate-limited (403) URLs once for a retry after concurrency recovers.
+        // Don't call onFailure here — the re-enqueued request gets a fresh attempt.
+        // If it fails again (rateLimitRetried=true), it falls through to the normal path.
+        if (status === 403 && !request.userData?.rateLimitRetried) {
+          try {
+            await requestQueue.addRequest({
+              url: request.url,
+              label: request.url,
+              uniqueKey: `ratelimit_${request.url}`,
+              userData: { rateLimitRetried: true },
+            });
+          } catch {}
+          return;
+        }
+
         if (rateController.onFailure(status, crawler.autoscaledPool)) {
           consoleLogger.info(
             `Aborting crawl: consecutive HTTP failures threshold reached (site may be rate-limiting). Successfully scanned ${urlsCrawled.scanned.length} pages.`,

package/src/mergeAxeResults.ts

@@ -1110,7 +1110,7 @@ const generateArtifacts = async (
   }
 
   try {
-    const requestQueue = await RequestQueue.open(crawleeDir);
+    const requestQueue = await RequestQueue.open(`${crawleeDir}_rq`);
     await requestQueue.drop();
   } catch (error) {
     consoleLogger.info(`RequestQueue drop: ${error.message}`);
@@ -1120,6 +1120,7 @@ const generateArtifacts = async (
   const crawleePath = path.join(storagePath, 'crawlee');
   try {
     await fs.promises.rm(crawleePath, { recursive: true, force: true });
+    await fs.promises.rm(`${crawleePath}_rq`, { recursive: true, force: true });
   } catch {
     // Best-effort; storage was already dropped via API
   }