@govtechsg/oobee 0.10.91 → 0.10.92

package/src/constants/common.ts

@@ -888,6 +888,7 @@ const getRobotsTxtViaPlaywright = async (
       browserContext = await constants.launcher.launchPersistentContext(robotsDataDir, {
         ...getPlaywrightLaunchOptions(browser),
         ...(extraHTTPHeaders && { extraHTTPHeaders }),
+        ...(process.env.OOBEE_USER_AGENT && { userAgent: process.env.OOBEE_USER_AGENT }),
       });
       register(browserContext);
     } else {
@@ -895,9 +896,10 @@ const getRobotsTxtViaPlaywright = async (
       const launchOptions = getPlaywrightLaunchOptions(browser);
       browserInstance = await constants.launcher.launch(launchOptions);
       register(browserInstance as unknown as { close: () => Promise<void> });
-      
+
       browserContext = await browserInstance.newContext({
         ...(extraHTTPHeaders && { extraHTTPHeaders }),
+        ...(process.env.OOBEE_USER_AGENT && { userAgent: process.env.OOBEE_USER_AGENT }),
       });
     }
 
@@ -975,7 +977,7 @@ export const isDisallowedInRobotsTxt = (url: string): boolean => {
 
 export const getLinksFromSitemap = async (
   sitemapUrl: string,
-  maxLinksCount: number,
+  _maxLinksCount: number,
   browser: string,
   userDataDirectory: string,
   userUrlInput: string,
@@ -985,9 +987,8 @@ export const getLinksFromSitemap = async (
   userUrl: string = userUrlInput,
 ) => {
   const scannedSitemaps = new Set<string>();
-  const urls: Record<string, Request> = {}; // dictionary of requests to urls to be scanned
-
-  const isLimitReached = () => Object.keys(urls).length >= maxLinksCount;
+  const sitemapLinkCounts: Record<string, number> = {};
+  const allUrls = new Set<string>(); // all discovered URLs (lightweight strings)
 
   const addToUrlList = (url: string) => {
     if (!url) return;
@@ -995,17 +996,7 @@ export const getLinksFromSitemap = async (
     if (!isFilePath(userUrl) && !isFollowStrategy(url, userUrl, strategy)) return;
 
     url = convertPathToLocalFile(url);
-
-    let request;
-    try {
-      request = new Request({ url });
-    } catch (e) {
-      console.log('Error creating request', e);
-    }
-    if (isUrlPdf(url)) {
-      request.skipNavigation = true;
-    }
-    urls[url] = request;
+    allUrls.add(url);
   };
 
   const calculateCloseness = (sitemapUrl: string) => {
@@ -1058,16 +1049,15 @@ export const getLinksFromSitemap = async (
       });
     }
 
-    // Add the sorted URLs to the main URL list
-    for (const { url } of urlList.slice(0, maxLinksCount)) {
+    // Add all URLs to the discovered list (limit applied later at return time)
+    for (const { url } of urlList) {
       addToUrlList(url);
     }
   };
 
   const processNonStandardSitemap = (data: string) => {
     const urlsFromData = crawlee
-      .extractUrls({ string: data, urlRegExp: new RegExp('^(http|https):/{2}.+$', 'gmi') })
-      .slice(0, maxLinksCount);
+      .extractUrls({ string: data, urlRegExp: new RegExp('^(http|https):/{2}.+$', 'gmi') });
     urlsFromData.forEach(url => {
       addToUrlList(url);
     });
@@ -1118,6 +1108,7 @@ export const getLinksFromSitemap = async (
               // Bug in Chrome which causes browser pool crash when userDataDirectory is set in non-headless mode
               ...(process.env.CRAWLEE_HEADLESS === '1' && { userDataDir: userDataDirectory }),
               ...(extraHTTPHeaders && { extraHTTPHeaders }),
+              ...(process.env.OOBEE_USER_AGENT && { userAgent: process.env.OOBEE_USER_AGENT }),
             },
           );
 
@@ -1127,9 +1118,10 @@ export const getLinksFromSitemap = async (
           const launchOptions = getPlaywrightLaunchOptions(browser);
           browserInstance = await constants.launcher.launch(launchOptions);
           register(browserInstance as unknown as { close: () => Promise<void> });
-          
+
           browserContext = await browserInstance.newContext({
             ...(extraHTTPHeaders && { extraHTTPHeaders }),
+            ...(process.env.OOBEE_USER_AGENT && { userAgent: process.env.OOBEE_USER_AGENT }),
           });
         }
 
@@ -1202,14 +1194,13 @@ export const getLinksFromSitemap = async (
       sitemapType = constants.xmlSitemapTypes.unknown;
     }
 
+    const countBefore = allUrls.size;
+
     switch (sitemapType) {
       case constants.xmlSitemapTypes.xmlIndex:
         consoleLogger.info(`This is a XML format sitemap index.`);
         for (const childSitemapUrl of $('loc')) {
           const childSitemapUrlText = $(childSitemapUrl).text();
-          if (isLimitReached()) {
-            break;
-          }
           if (childSitemapUrlText.endsWith('.xml') || childSitemapUrlText.endsWith('.txt')) {
             await fetchUrls(childSitemapUrlText, extraHTTPHeaders); // Recursive call for nested sitemaps
           } else {
@@ -1233,6 +1224,11 @@ export const getLinksFromSitemap = async (
         consoleLogger.info(`This is an unrecognised XML sitemap format.`);
         processNonStandardSitemap(data);
     }
+
+    const linksFromThisSitemap = allUrls.size - countBefore;
+    if (linksFromThisSitemap > 0) {
+      sitemapLinkCounts[url] = (sitemapLinkCounts[url] || 0) + linksFromThisSitemap;
+    }
   };
 
   try {
@@ -1241,7 +1237,41 @@ export const getLinksFromSitemap = async (
     consoleLogger.error(e);
   }
 
-  const requestList = Object.values(urls);
+  // Build Request objects for all discovered URLs; the crawler itself enforces
+  // maxRequestsPerCrawl by counting only successfully scanned pages.
+  const requestList: Request[] = [];
+  for (const url of allUrls) {
+    try {
+      const request = new Request({ url });
+      if (isUrlPdf(url)) {
+        request.skipNavigation = true;
+      }
+      requestList.push(request);
+    } catch (e) {
+      consoleLogger.info(`Error creating request for ${url}: ${e}`);
+    }
+  }
+
+  const totalLinksDiscovered = allUrls.size;
+  const fetchedSitemaps = Object.entries(sitemapLinkCounts).map(([url, fetchedLinks]) => ({
+    url,
+    fetchedLinks,
+  }));
+
+  const prev = constants.sitemapFetchedLinks;
+  constants.sitemapFetchedLinks = {
+    totalLinksFetchedFromSitemaps: (prev?.totalLinksFetchedFromSitemaps ?? 0) + totalLinksDiscovered,
+    fetchedSitemaps: [...(prev?.fetchedSitemaps ?? []), ...fetchedSitemaps],
+  };
+
+  if (totalLinksDiscovered > 0) {
+    const breakdown = fetchedSitemaps
+      .map(({ url, fetchedLinks }) => `${url} (${fetchedLinks})`)
+      .join(', ');
+    consoleLogger.info(
+      `There are a total of ${totalLinksDiscovered} links found across ${breakdown}.`,
+    );
+  }
 
   return requestList;
 };
@@ -1406,6 +1436,36 @@ export const getEdgeData = (randomToken: string) => {
  * @param {*} destDir destination directory
  * @returns boolean indicating whether the operation was successful
  */
+// Helper to copy a file with retry logic for transient EBUSY errors
+const copyFileWithRetry = (src: string, dest: string, maxRetries: number = 3): boolean => {
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    try {
+      fs.copyFileSync(src, dest);
+      if (attempt > 1) {
+        consoleLogger.info(`File copy succeeded on attempt ${attempt}: ${dest}`);
+      }
+      return true;
+    } catch (err: any) {
+      if (err.code === 'EBUSY' && attempt < maxRetries) {
+        // Transient lock — wait and retry
+        const delayMs = Math.min(100 * Math.pow(2, attempt - 1), 1000); // Exponential backoff: 100ms, 200ms, 400ms, capped at 1s
+        consoleLogger.warn(
+          `File copy attempt ${attempt}/${maxRetries} failed with EBUSY. Retrying after ${delayMs}ms: ${dest}`,
+        );
+        // Synchronous sleep via busy-wait (not ideal but avoids promise complications in sync context)
+        const endTime = Date.now() + delayMs;
+        while (Date.now() < endTime) {
+          // Busy wait
+        }
+        continue; // Retry
+      }
+      // Non-transient error or max retries reached
+      return false;
+    }
+  }
+  return false;
+};
+
 const cloneChromeProfileCookieFiles = (options: GlobOptionsWithFileTypesFalse, destDir: string) => {
   let profileCookiesDir;
   // Cookies file per profile is located in .../User Data/<profile name>/Network/Cookies for windows
@@ -1445,23 +1505,9 @@ const cloneChromeProfileCookieFiles = (options: GlobOptionsWithFileTypesFalse, d
 
         // Prevents duplicate cookies file if the cookies already exist
         if (!fs.existsSync(path.join(destProfileDir, 'Cookies'))) {
-          try {
-            fs.copyFileSync(dir, path.join(destProfileDir, 'Cookies'));
-          } catch (err) {
-            consoleLogger.error(err);
-            if (err.code === 'EBUSY') {
-              console.log(
-                `Unable to copy the file for ${profileName} because it is currently in use.`,
-              );
-              console.log(
-                'Please close any applications that might be using this file and try again.',
-              );
-            } else {
-              console.log(
-                `An unexpected error occurred for ${profileName} while copying the file: ${err.message}`,
-              );
-            }
-            // printMessage([err], messageOptions);
+          const destCookiesPath = path.join(destProfileDir, 'Cookies');
+          if (!copyFileWithRetry(dir, destCookiesPath)) {
+            consoleLogger.error(`Failed to copy Chrome profile cookies for ${profileName} after retries.`);
             success = false;
           }
         }
@@ -1475,12 +1521,6 @@ const cloneChromeProfileCookieFiles = (options: GlobOptionsWithFileTypesFalse, d
   return false;
 };
 
-/**
- * Clone the Chrome profile cookie files to the destination directory
- * @param {*} options glob options object
- * @param {*} destDir destination directory
- * @returns boolean indicating whether the operation was successful
- */
 const cloneEdgeProfileCookieFiles = (options: GlobOptionsWithFileTypesFalse, destDir: string) => {
   let profileCookiesDir;
   // Cookies file per profile is located in .../User Data/<profile name>/Network/Cookies for windows
@@ -1521,21 +1561,9 @@ const cloneEdgeProfileCookieFiles = (options: GlobOptionsWithFileTypesFalse, des
 
         // Prevents duplicate cookies file if the cookies already exist
         if (!fs.existsSync(path.join(destProfileDir, 'Cookies'))) {
-          try {
-            fs.copyFileSync(dir, path.join(destProfileDir, 'Cookies'));
-          } catch (err) {
-            consoleLogger.error(err);
-            if (err.code === 'EBUSY') {
-              console.log(
-                `Unable to copy the file for ${profileName} because it is currently in use.`,
-              );
-              console.log(
-                'Please close any applications that might be using this file and try again.',
-              );
-            } else {
-              console.log(`An unexpected error occurred while copying the file: ${err.message}`);
-            }
-            // printMessage([err], messageOptions);
+          const destCookiesPath = path.join(destProfileDir, 'Cookies');
+          if (!copyFileWithRetry(dir, destCookiesPath)) {
+            consoleLogger.error(`Failed to copy Edge profile cookies for ${profileName} after retries.`);
             success = false;
           }
         }
@@ -1566,19 +1594,9 @@ const cloneLocalStateFile = (options: GlobOptionsWithFileTypesFalse, destDir: st
 
     localState.forEach(dir => {
       const profileName = dir.match(profileNamesRegex)[1];
-      try {
-        fs.copyFileSync(dir, path.join(destDir, 'Local State'));
-      } catch (err) {
-        consoleLogger.error(err);
-        if (err.code === 'EBUSY') {
-          console.log(`Unable to copy the file because it is currently in use.`);
-          console.log('Please close any applications that might be using this file and try again.');
-        } else {
-          console.log(
-            `An unexpected error occurred for ${profileName} while copying the file: ${err.message}`,
-          );
-        }
-        printMessage([err], messageOptions);
+      const destPath = path.join(destDir, 'Local State');
+      if (!copyFileWithRetry(dir, destPath)) {
+        consoleLogger.error(`Failed to copy Local State file for ${profileName} after retries.`);
         success = false;
       }
     });
@@ -1629,6 +1647,17 @@ export const cloneChromeProfiles = (randomToken: string): string => {
     }
 
     consoleLogger.error('Failed to clone Chrome profiles. You may be logged out of your accounts.');
+
+    // Fall back to a clean profile directory to avoid launch failures from partial clones.
+    try {
+      fs.rmSync(destDir, { recursive: true, force: true });
+      fs.mkdirSync(destDir, { recursive: true });
+      consoleLogger.warn('Using an empty cloned Chrome profile directory due to clone failure.');
+    } catch (cleanupError) {
+      consoleLogger.error(
+        `Unable to reset cloned Chrome profile directory ${destDir}: ${cleanupError}`,
+      );
+    }
   }
   // For future reference, return a null instead to halt the scan
   return destDir;
@@ -1697,6 +1726,15 @@ export const cloneEdgeProfiles = (randomToken: string): string => {
     }
 
     consoleLogger.error('Failed to clone Edge profiles. You may be logged out of your accounts.');
+
+    // Fall back to a clean profile directory to avoid launch failures from partial clones.
+    try {
+      fs.rmSync(destDir, { recursive: true, force: true });
+      fs.mkdirSync(destDir, { recursive: true });
+      consoleLogger.warn('Using an empty cloned Edge profile directory due to clone failure.');
+    } catch (cleanupError) {
+      consoleLogger.error(`Unable to reset cloned Edge profile directory ${destDir}: ${cleanupError}`);
+    }
   }
 
   // For future reference, return a null instead to halt the scan
@@ -1725,7 +1763,14 @@ export const deleteClonedChromeProfiles = (randomToken?: string): void => {
   }
   let destDir: string[];
   if (randomToken) {
-    destDir = [`${baseDir}/oobee-${randomToken}`];
+    // Also match _pool* directories created by browser pool re-launches
+    destDir = globSync(`oobee-${randomToken}*`, {
+      cwd: baseDir,
+      absolute: true,
+    });
+    if (destDir.length === 0) {
+      destDir = [`${baseDir}/oobee-${randomToken}`];
+    }
   } else {
     // Find all the oobee directories in the Chrome data directory
     destDir = globSync('**/oobee*', {
@@ -1766,9 +1811,16 @@ export const deleteClonedEdgeProfiles = (randomToken?: string): void => {
   }
   let destDir: string[];
   if (randomToken) {
-    destDir = [`${baseDir}/oobee-${randomToken}`];
+    // Also match _pool* directories created by browser pool re-launches
+    destDir = globSync(`oobee-${randomToken}*`, {
+      cwd: baseDir,
+      absolute: true,
+    });
+    if (destDir.length === 0) {
+      destDir = [`${baseDir}/oobee-${randomToken}`];
+    }
   } else {
-    // Find all the oobee directories in the Chrome data directory
+    // Find all the oobee directories in the Edge data directory
     destDir = globSync('**/oobee*', {
       cwd: baseDir,
       absolute: true,

package/src/constants/constants.ts

@@ -946,6 +946,7 @@ export default {
   a11yRuleShortDescriptionMap,
   disabilityBadgesMap,
   robotsTxtUrls: null,
+  sitemapFetchedLinks: null as { totalLinksFetchedFromSitemaps: number; fetchedSitemaps: { url: string; fetchedLinks: number }[] } | null,
   userDataDirectory: null, // This will be set later in the code
   randomToken: null, // This will be set later in the code
   // Track all active Crawlee / Playwright resources for cleanup

package/src/crawlers/commonCrawlerFunc.ts

@@ -874,6 +874,13 @@ export const runAxeScript = async ({
   const browserContext: BrowserContext = page.context();
   const requestUrl = page.url();
 
+  let pageTitle: string | null = null;
+  try {
+    pageTitle = await page.evaluate(() => document.title);
+  } catch {
+    // Page may already be in a bad state; title will remain null
+  }
+
   try {
     // Checking for DOM mutations before proceeding to scan
     await page.evaluate(() => {
@@ -1012,7 +1019,42 @@ export const runAxeScript = async ({
           .run(selectors, {
             resultTypes: defaultResultTypes,
           })
-          .then(results => {
+          .then(async results => {
+            // Re-verify aria-hidden-focus violations against the live DOM to
+            // handle race conditions with JS that sets tabindex="-1" after
+            // aria-hidden (common in carousel/slider libraries like slick)
+            const ariaHiddenViolation = results.violations.find(
+              v => v.id === 'aria-hidden-focus',
+            );
+            if (ariaHiddenViolation) {
+              await new Promise(resolve => setTimeout(resolve, 0));
+              ariaHiddenViolation.nodes = ariaHiddenViolation.nodes.filter(node => {
+                const selector = node.target && node.target[0];
+                if (typeof selector !== 'string') return true;
+                try {
+                  const el = document.querySelector(selector);
+                  if (!el) return true;
+                  const focusables = el.querySelectorAll(
+                    'a[href], area[href], button:not([disabled]), input:not([disabled]):not([type="hidden"]), select:not([disabled]), textarea:not([disabled]), [tabindex]',
+                  );
+                  if (focusables.length === 0) return false;
+                  return Array.from(focusables).some(child => {
+                    const tabindex = child.getAttribute('tabindex');
+                    if (tabindex === null) return true;
+                    const parsed = parseInt(tabindex, 10);
+                    return isNaN(parsed) || parsed >= 0;
+                  });
+                } catch {
+                  return true;
+                }
+              });
+              if (ariaHiddenViolation.nodes.length === 0) {
+                results.violations = results.violations.filter(
+                  v => v.id !== 'aria-hidden-focus',
+                );
+              }
+            }
+
             if (disableOobee) {
               return results;
             }
@@ -1086,19 +1128,6 @@ export const runAxeScript = async ({
     results.incomplete = await takeScreenshotForHTMLElements(results.incomplete, page, randomToken);
   }
 
-  let pageTitle = null;
-  try {
-    pageTitle = await page.evaluate(() => document.title);
-  } catch (e) {
-    consoleLogger.info(`Error while getting page title: ${e}`);
-    if (page.isClosed()) {
-      consoleLogger.info(`Page was closed for ${requestUrl}, creating new page`);
-      page = await browserContext.newPage();
-      await page.goto(requestUrl, { waitUntil: 'domcontentloaded' });
-      pageTitle = await page.evaluate(() => document.title);
-    }
-  }
-
   return filterAxeResults(results, pageTitle, customFlowDetails);
 };
 
@@ -1130,6 +1159,66 @@ export const postNavigationHooks = [
   },
 ];
 
+export const getPreLaunchHook = (userDataDirectory: string) => {
+  let launchCount = 0;
+
+  return async (_pageId: string, launchContext: any) => {
+    const fsp = await import('fs/promises').then(m => m.default);
+    launchCount += 1;
+
+    // First launch uses the base directory; subsequent launches get a unique
+    // directory so that lingering file handles from a retired browser don't
+    // cause Chrome exit code 21 on Windows.
+    const effectiveDir =
+      launchCount === 1
+        ? userDataDirectory
+        : `${userDataDirectory}_pool${launchCount}`;
+
+    await fsp.mkdir(effectiveDir, { recursive: true });
+
+    // For pool re-launches, best-effort clone profile data from base directory
+    // so authenticated sessions are preserved across browser pool retirements.
+    if (launchCount > 1) {
+      try {
+        const copyRecursive = async (src: string, dest: string) => {
+          const stat = await fsp.stat(src).catch(() => null);
+          if (!stat) return;
+          if (stat.isDirectory()) {
+            await fsp.mkdir(dest, { recursive: true }).catch(() => {});
+            const entries = await fsp.readdir(src).catch(() => []);
+            await Promise.all(
+              entries
+                .filter(entry => !entry.startsWith('Singleton') && entry !== 'lockfile' && entry !== 'LOCK')
+                .map(entry =>
+                  copyRecursive(path.join(src, entry), path.join(dest, entry)).catch(() => {}),
+                ),
+            );
+          } else {
+            await fsp.copyFile(src, dest).catch(() => {});
+          }
+        };
+        await copyRecursive(userDataDirectory, effectiveDir).catch(() => {});
+      } catch {
+        // Silent fallback: use empty profile if clone fails
+      }
+    }
+
+    // Clean any stale lock files that may block browser launches on Windows
+    const lockFiles = [
+      path.join(effectiveDir, 'SingletonLock'),
+      path.join(effectiveDir, 'SingletonSocket'),
+      path.join(effectiveDir, 'SingletonCookie'),
+      path.join(effectiveDir, 'lockfile'),
+      path.join(effectiveDir, 'Default', 'LOCK'),
+      path.join(effectiveDir, 'Default', 'Network', 'LOCK'),
+    ];
+    await Promise.all(lockFiles.map(f => fsp.rm(f, { force: true }).catch(() => {})));
+
+    // eslint-disable-next-line no-param-reassign
+    launchContext.userDataDir = effectiveDir;
+  };
+};
+
 export const failedRequestHandler = async ({ request }: { request: Request }) => {
   guiInfoLog(guiInfoStatusTypes.ERROR, { numScanned: 0, urlScanned: request.url });
   log.error(`Failed Request - ${request.url}: ${request.errorMessages}`);