@govtechsg/oobee 0.10.91 → 0.10.92

package/dist/constants/common.js

@@ -713,6 +713,7 @@ const getRobotsTxtViaPlaywright = async (robotsUrl, browser, userDataDirectory,
             browserContext = await constants.launcher.launchPersistentContext(robotsDataDir, {
                 ...getPlaywrightLaunchOptions(browser),
                 ...(extraHTTPHeaders && { extraHTTPHeaders }),
+                ...(process.env.OOBEE_USER_AGENT && { userAgent: process.env.OOBEE_USER_AGENT }),
             });
             register(browserContext);
         }
@@ -723,6 +724,7 @@ const getRobotsTxtViaPlaywright = async (robotsUrl, browser, userDataDirectory,
             register(browserInstance);
             browserContext = await browserInstance.newContext({
                 ...(extraHTTPHeaders && { extraHTTPHeaders }),
+                ...(process.env.OOBEE_USER_AGENT && { userAgent: process.env.OOBEE_USER_AGENT }),
             });
         }
         const page = await browserContext.newPage();
@@ -784,10 +786,10 @@ export const isDisallowedInRobotsTxt = (url) => {
     }
     return false;
 };
-export const getLinksFromSitemap = async (sitemapUrl, maxLinksCount, browser, userDataDirectory, userUrlInput, isIntelligent, extraHTTPHeaders, strategy = EnqueueStrategy.All, userUrl = userUrlInput) => {
+export const getLinksFromSitemap = async (sitemapUrl, _maxLinksCount, browser, userDataDirectory, userUrlInput, isIntelligent, extraHTTPHeaders, strategy = EnqueueStrategy.All, userUrl = userUrlInput) => {
     const scannedSitemaps = new Set();
-    const urls = {}; // dictionary of requests to urls to be scanned
-    const isLimitReached = () => Object.keys(urls).length >= maxLinksCount;
+    const sitemapLinkCounts = {};
+    const allUrls = new Set(); // all discovered URLs (lightweight strings)
     const addToUrlList = (url) => {
         if (!url)
             return;
@@ -796,17 +798,7 @@ export const getLinksFromSitemap = async (sitemapUrl, maxLinksCount, browser, us
         if (!isFilePath(userUrl) && !isFollowStrategy(url, userUrl, strategy))
             return;
         url = convertPathToLocalFile(url);
-        let request;
-        try {
-            request = new Request({ url });
-        }
-        catch (e) {
-            console.log('Error creating request', e);
-        }
-        if (isUrlPdf(url)) {
-            request.skipNavigation = true;
-        }
-        urls[url] = request;
+        allUrls.add(url);
     };
     const calculateCloseness = (sitemapUrl) => {
         // Remove 'http://', 'https://', and 'www.' prefixes from the URLs
@@ -849,15 +841,14 @@ export const getLinksFromSitemap = async (sitemapUrl, maxLinksCount, browser, us
                 return (b.lastModifiedDate?.getTime() || 0) - (a.lastModifiedDate?.getTime() || 0);
             });
         }
-        // Add the sorted URLs to the main URL list
-        for (const { url } of urlList.slice(0, maxLinksCount)) {
+        // Add all URLs to the discovered list (limit applied later at return time)
+        for (const { url } of urlList) {
             addToUrlList(url);
         }
     };
     const processNonStandardSitemap = (data) => {
         const urlsFromData = crawlee
-            .extractUrls({ string: data, urlRegExp: new RegExp('^(http|https):/{2}.+$', 'gmi') })
-            .slice(0, maxLinksCount);
+            .extractUrls({ string: data, urlRegExp: new RegExp('^(http|https):/{2}.+$', 'gmi') });
         urlsFromData.forEach(url => {
             addToUrlList(url);
         });
@@ -900,6 +891,7 @@ export const getLinksFromSitemap = async (sitemapUrl, maxLinksCount, browser, us
                         // Bug in Chrome which causes browser pool crash when userDataDirectory is set in non-headless mode
                         ...(process.env.CRAWLEE_HEADLESS === '1' && { userDataDir: userDataDirectory }),
                         ...(extraHTTPHeaders && { extraHTTPHeaders }),
+                        ...(process.env.OOBEE_USER_AGENT && { userAgent: process.env.OOBEE_USER_AGENT }),
                     });
                     register(browserContext);
                 }
@@ -910,6 +902,7 @@ export const getLinksFromSitemap = async (sitemapUrl, maxLinksCount, browser, us
                     register(browserInstance);
                     browserContext = await browserInstance.newContext({
                         ...(extraHTTPHeaders && { extraHTTPHeaders }),
+                        ...(process.env.OOBEE_USER_AGENT && { userAgent: process.env.OOBEE_USER_AGENT }),
                     });
                 }
                 const page = await browserContext.newPage();
@@ -980,14 +973,12 @@ export const getLinksFromSitemap = async (sitemapUrl, maxLinksCount, browser, us
         else {
             sitemapType = constants.xmlSitemapTypes.unknown;
         }
+        const countBefore = allUrls.size;
         switch (sitemapType) {
             case constants.xmlSitemapTypes.xmlIndex:
                 consoleLogger.info(`This is a XML format sitemap index.`);
                 for (const childSitemapUrl of $('loc')) {
                     const childSitemapUrlText = $(childSitemapUrl).text();
-                    if (isLimitReached()) {
-                        break;
-                    }
                     if (childSitemapUrlText.endsWith('.xml') || childSitemapUrlText.endsWith('.txt')) {
                         await fetchUrls(childSitemapUrlText, extraHTTPHeaders); // Recursive call for nested sitemaps
                     }
@@ -1012,6 +1003,10 @@ export const getLinksFromSitemap = async (sitemapUrl, maxLinksCount, browser, us
                 consoleLogger.info(`This is an unrecognised XML sitemap format.`);
                 processNonStandardSitemap(data);
         }
+        const linksFromThisSitemap = allUrls.size - countBefore;
+        if (linksFromThisSitemap > 0) {
+            sitemapLinkCounts[url] = (sitemapLinkCounts[url] || 0) + linksFromThisSitemap;
+        }
     };
     try {
         await fetchUrls(sitemapUrl, extraHTTPHeaders);
@@ -1019,7 +1014,37 @@ export const getLinksFromSitemap = async (sitemapUrl, maxLinksCount, browser, us
     catch (e) {
         consoleLogger.error(e);
     }
-    const requestList = Object.values(urls);
+    // Build Request objects for all discovered URLs; the crawler itself enforces
+    // maxRequestsPerCrawl by counting only successfully scanned pages.
+    const requestList = [];
+    for (const url of allUrls) {
+        try {
+            const request = new Request({ url });
+            if (isUrlPdf(url)) {
+                request.skipNavigation = true;
+            }
+            requestList.push(request);
+        }
+        catch (e) {
+            consoleLogger.info(`Error creating request for ${url}: ${e}`);
+        }
+    }
+    const totalLinksDiscovered = allUrls.size;
+    const fetchedSitemaps = Object.entries(sitemapLinkCounts).map(([url, fetchedLinks]) => ({
+        url,
+        fetchedLinks,
+    }));
+    const prev = constants.sitemapFetchedLinks;
+    constants.sitemapFetchedLinks = {
+        totalLinksFetchedFromSitemaps: (prev?.totalLinksFetchedFromSitemaps ?? 0) + totalLinksDiscovered,
+        fetchedSitemaps: [...(prev?.fetchedSitemaps ?? []), ...fetchedSitemaps],
+    };
+    if (totalLinksDiscovered > 0) {
+        const breakdown = fetchedSitemaps
+            .map(({ url, fetchedLinks }) => `${url} (${fetchedLinks})`)
+            .join(', ');
+        consoleLogger.info(`There are a total of ${totalLinksDiscovered} links found across ${breakdown}.`);
+    }
     return requestList;
 };
 export const validEmail = (email) => {
@@ -1158,6 +1183,34 @@ export const getEdgeData = (randomToken) => {
  * @param {*} destDir destination directory
  * @returns boolean indicating whether the operation was successful
  */
+// Helper to copy a file with retry logic for transient EBUSY errors
+const copyFileWithRetry = (src, dest, maxRetries = 3) => {
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+        try {
+            fs.copyFileSync(src, dest);
+            if (attempt > 1) {
+                consoleLogger.info(`File copy succeeded on attempt ${attempt}: ${dest}`);
+            }
+            return true;
+        }
+        catch (err) {
+            if (err.code === 'EBUSY' && attempt < maxRetries) {
+                // Transient lock — wait and retry
+                const delayMs = Math.min(100 * Math.pow(2, attempt - 1), 1000); // Exponential backoff: 100ms, 200ms, 400ms, capped at 1s
+                consoleLogger.warn(`File copy attempt ${attempt}/${maxRetries} failed with EBUSY. Retrying after ${delayMs}ms: ${dest}`);
+                // Synchronous sleep via busy-wait (not ideal but avoids promise complications in sync context)
+                const endTime = Date.now() + delayMs;
+                while (Date.now() < endTime) {
+                    // Busy wait
+                }
+                continue; // Retry
+            }
+            // Non-transient error or max retries reached
+            return false;
+        }
+    }
+    return false;
+};
 const cloneChromeProfileCookieFiles = (options, destDir) => {
     let profileCookiesDir;
     // Cookies file per profile is located in .../User Data/<profile name>/Network/Cookies for windows
@@ -1196,19 +1249,9 @@ const cloneChromeProfileCookieFiles = (options, destDir) => {
                 }
                 // Prevents duplicate cookies file if the cookies already exist
                 if (!fs.existsSync(path.join(destProfileDir, 'Cookies'))) {
-                    try {
-                        fs.copyFileSync(dir, path.join(destProfileDir, 'Cookies'));
-                    }
-                    catch (err) {
-                        consoleLogger.error(err);
-                        if (err.code === 'EBUSY') {
-                            console.log(`Unable to copy the file for ${profileName} because it is currently in use.`);
-                            console.log('Please close any applications that might be using this file and try again.');
-                        }
-                        else {
-                            console.log(`An unexpected error occurred for ${profileName} while copying the file: ${err.message}`);
-                        }
-                        // printMessage([err], messageOptions);
+                    const destCookiesPath = path.join(destProfileDir, 'Cookies');
+                    if (!copyFileWithRetry(dir, destCookiesPath)) {
+                        consoleLogger.error(`Failed to copy Chrome profile cookies for ${profileName} after retries.`);
                         success = false;
                     }
                 }
@@ -1220,12 +1263,6 @@ const cloneChromeProfileCookieFiles = (options, destDir) => {
     printMessage(['Unable to find Chrome profile cookies file in the system.'], messageOptions);
     return false;
 };
-/**
- * Clone the Chrome profile cookie files to the destination directory
- * @param {*} options glob options object
- * @param {*} destDir destination directory
- * @returns boolean indicating whether the operation was successful
- */
 const cloneEdgeProfileCookieFiles = (options, destDir) => {
     let profileCookiesDir;
     // Cookies file per profile is located in .../User Data/<profile name>/Network/Cookies for windows
@@ -1265,19 +1302,9 @@ const cloneEdgeProfileCookieFiles = (options, destDir) => {
                 }
                 // Prevents duplicate cookies file if the cookies already exist
                 if (!fs.existsSync(path.join(destProfileDir, 'Cookies'))) {
-                    try {
-                        fs.copyFileSync(dir, path.join(destProfileDir, 'Cookies'));
-                    }
-                    catch (err) {
-                        consoleLogger.error(err);
-                        if (err.code === 'EBUSY') {
-                            console.log(`Unable to copy the file for ${profileName} because it is currently in use.`);
-                            console.log('Please close any applications that might be using this file and try again.');
-                        }
-                        else {
-                            console.log(`An unexpected error occurred while copying the file: ${err.message}`);
-                        }
-                        // printMessage([err], messageOptions);
+                    const destCookiesPath = path.join(destProfileDir, 'Cookies');
+                    if (!copyFileWithRetry(dir, destCookiesPath)) {
+                        consoleLogger.error(`Failed to copy Edge profile cookies for ${profileName} after retries.`);
                         success = false;
                     }
                 }
@@ -1305,19 +1332,9 @@ const cloneLocalStateFile = (options, destDir) => {
         let success = true;
         localState.forEach(dir => {
             const profileName = dir.match(profileNamesRegex)[1];
-            try {
-                fs.copyFileSync(dir, path.join(destDir, 'Local State'));
-            }
-            catch (err) {
-                consoleLogger.error(err);
-                if (err.code === 'EBUSY') {
-                    console.log(`Unable to copy the file because it is currently in use.`);
-                    console.log('Please close any applications that might be using this file and try again.');
-                }
-                else {
-                    console.log(`An unexpected error occurred for ${profileName} while copying the file: ${err.message}`);
-                }
-                printMessage([err], messageOptions);
+            const destPath = path.join(destDir, 'Local State');
+            if (!copyFileWithRetry(dir, destPath)) {
+                consoleLogger.error(`Failed to copy Local State file for ${profileName} after retries.`);
                 success = false;
             }
         });
@@ -1362,6 +1379,15 @@ export const cloneChromeProfiles = (randomToken) => {
             return destDir;
         }
         consoleLogger.error('Failed to clone Chrome profiles. You may be logged out of your accounts.');
+        // Fall back to a clean profile directory to avoid launch failures from partial clones.
+        try {
+            fs.rmSync(destDir, { recursive: true, force: true });
+            fs.mkdirSync(destDir, { recursive: true });
+            consoleLogger.warn('Using an empty cloned Chrome profile directory due to clone failure.');
+        }
+        catch (cleanupError) {
+            consoleLogger.error(`Unable to reset cloned Chrome profile directory ${destDir}: ${cleanupError}`);
+        }
     }
     // For future reference, return a null instead to halt the scan
     return destDir;
@@ -1418,6 +1444,15 @@ export const cloneEdgeProfiles = (randomToken) => {
             return destDir;
         }
         consoleLogger.error('Failed to clone Edge profiles. You may be logged out of your accounts.');
+        // Fall back to a clean profile directory to avoid launch failures from partial clones.
+        try {
+            fs.rmSync(destDir, { recursive: true, force: true });
+            fs.mkdirSync(destDir, { recursive: true });
+            consoleLogger.warn('Using an empty cloned Edge profile directory due to clone failure.');
+        }
+        catch (cleanupError) {
+            consoleLogger.error(`Unable to reset cloned Edge profile directory ${destDir}: ${cleanupError}`);
+        }
     }
     // For future reference, return a null instead to halt the scan
     return destDir;
@@ -1444,7 +1479,14 @@ export const deleteClonedChromeProfiles = (randomToken) => {
     }
     let destDir;
     if (randomToken) {
-        destDir = [`${baseDir}/oobee-${randomToken}`];
+        // Also match _pool* directories created by browser pool re-launches
+        destDir = globSync(`oobee-${randomToken}*`, {
+            cwd: baseDir,
+            absolute: true,
+        });
+        if (destDir.length === 0) {
+            destDir = [`${baseDir}/oobee-${randomToken}`];
+        }
     }
     else {
         // Find all the oobee directories in the Chrome data directory
@@ -1481,10 +1523,17 @@ export const deleteClonedEdgeProfiles = (randomToken) => {
     }
     let destDir;
     if (randomToken) {
-        destDir = [`${baseDir}/oobee-${randomToken}`];
+        // Also match _pool* directories created by browser pool re-launches
+        destDir = globSync(`oobee-${randomToken}*`, {
+            cwd: baseDir,
+            absolute: true,
+        });
+        if (destDir.length === 0) {
+            destDir = [`${baseDir}/oobee-${randomToken}`];
+        }
     }
     else {
-        // Find all the oobee directories in the Chrome data directory
+        // Find all the oobee directories in the Edge data directory
         destDir = globSync('**/oobee*', {
             cwd: baseDir,
             absolute: true,

package/dist/constants/constants.js

@@ -771,6 +771,7 @@ export default {
     a11yRuleShortDescriptionMap,
     disabilityBadgesMap,
     robotsTxtUrls: null,
+    sitemapFetchedLinks: null,
     userDataDirectory: null, // This will be set later in the code
     randomToken: null, // This will be set later in the code
     // Track all active Crawlee / Playwright resources for cleanup

package/dist/crawlers/commonCrawlerFunc.js

@@ -1,7 +1,7 @@
 import { Dataset, RequestQueue, log, playwrightUtils } from 'crawlee';
 import axe from 'axe-core';
 import { axeScript, disallowedListOfPatterns, guiInfoStatusTypes, RuleFlags, saflyIconSelector, } from '../constants/constants.js';
-import { consoleLogger, guiInfoLog } from '../logs.js';
+import { guiInfoLog } from '../logs.js';
 import { enrichColorContrastDOMContext, takeScreenshotForHTMLElements } from '../screenshotFunc/htmlScreenshotFunc.js';
 import { isFilePath } from '../constants/common.js';
 import { extractAndGradeText } from './custom/extractAndGradeText.js';
@@ -674,6 +674,13 @@ export const filterAxeResults = (results, pageTitle, customFlowDetails) => {
 export const runAxeScript = async ({ includeScreenshots, page, randomToken, customFlowDetails = null, selectors = [], ruleset = [], }) => {
     const browserContext = page.context();
     const requestUrl = page.url();
+    let pageTitle = null;
+    try {
+        pageTitle = await page.evaluate(() => document.title);
+    }
+    catch {
+        // Page may already be in a bad state; title will remain null
+    }
     try {
         // Checking for DOM mutations before proceeding to scan
         await page.evaluate(() => {
@@ -781,7 +788,40 @@ export const runAxeScript = async ({ includeScreenshots, page, randomToken, cust
                 .run(selectors, {
                 resultTypes: defaultResultTypes,
             })
-                .then(results => {
+                .then(async (results) => {
+                // Re-verify aria-hidden-focus violations against the live DOM to
+                // handle race conditions with JS that sets tabindex="-1" after
+                // aria-hidden (common in carousel/slider libraries like slick)
+                const ariaHiddenViolation = results.violations.find(v => v.id === 'aria-hidden-focus');
+                if (ariaHiddenViolation) {
+                    await new Promise(resolve => setTimeout(resolve, 0));
+                    ariaHiddenViolation.nodes = ariaHiddenViolation.nodes.filter(node => {
+                        const selector = node.target && node.target[0];
+                        if (typeof selector !== 'string')
+                            return true;
+                        try {
+                            const el = document.querySelector(selector);
+                            if (!el)
+                                return true;
+                            const focusables = el.querySelectorAll('a[href], area[href], button:not([disabled]), input:not([disabled]):not([type="hidden"]), select:not([disabled]), textarea:not([disabled]), [tabindex]');
+                            if (focusables.length === 0)
+                                return false;
+                            return Array.from(focusables).some(child => {
+                                const tabindex = child.getAttribute('tabindex');
+                                if (tabindex === null)
+                                    return true;
+                                const parsed = parseInt(tabindex, 10);
+                                return isNaN(parsed) || parsed >= 0;
+                            });
+                        }
+                        catch {
+                            return true;
+                        }
+                    });
+                    if (ariaHiddenViolation.nodes.length === 0) {
+                        results.violations = results.violations.filter(v => v.id !== 'aria-hidden-focus');
+                    }
+                }
                 if (disableOobee) {
                     return results;
                 }
@@ -847,19 +887,6 @@ export const runAxeScript = async ({ includeScreenshots, page, randomToken, cust
         results.violations = await takeScreenshotForHTMLElements(results.violations, page, randomToken);
         results.incomplete = await takeScreenshotForHTMLElements(results.incomplete, page, randomToken);
     }
-    let pageTitle = null;
-    try {
-        pageTitle = await page.evaluate(() => document.title);
-    }
-    catch (e) {
-        consoleLogger.info(`Error while getting page title: ${e}`);
-        if (page.isClosed()) {
-            consoleLogger.info(`Page was closed for ${requestUrl}, creating new page`);
-            page = await browserContext.newPage();
-            await page.goto(requestUrl, { waitUntil: 'domcontentloaded' });
-            pageTitle = await page.evaluate(() => document.title);
-        }
-    }
     return filterAxeResults(results, pageTitle, customFlowDetails);
 };
 export const createCrawleeSubFolders = async (randomToken) => {
@@ -883,6 +910,57 @@ export const postNavigationHooks = [
         guiInfoLog(guiInfoStatusTypes.COMPLETED, {});
     },
 ];
+export const getPreLaunchHook = (userDataDirectory) => {
+    let launchCount = 0;
+    return async (_pageId, launchContext) => {
+        const fsp = await import('fs/promises').then(m => m.default);
+        launchCount += 1;
+        // First launch uses the base directory; subsequent launches get a unique
+        // directory so that lingering file handles from a retired browser don't
+        // cause Chrome exit code 21 on Windows.
+        const effectiveDir = launchCount === 1
+            ? userDataDirectory
+            : `${userDataDirectory}_pool${launchCount}`;
+        await fsp.mkdir(effectiveDir, { recursive: true });
+        // For pool re-launches, best-effort clone profile data from base directory
+        // so authenticated sessions are preserved across browser pool retirements.
+        if (launchCount > 1) {
+            try {
+                const copyRecursive = async (src, dest) => {
+                    const stat = await fsp.stat(src).catch(() => null);
+                    if (!stat)
+                        return;
+                    if (stat.isDirectory()) {
+                        await fsp.mkdir(dest, { recursive: true }).catch(() => { });
+                        const entries = await fsp.readdir(src).catch(() => []);
+                        await Promise.all(entries
+                            .filter(entry => !entry.startsWith('Singleton') && entry !== 'lockfile' && entry !== 'LOCK')
+                            .map(entry => copyRecursive(path.join(src, entry), path.join(dest, entry)).catch(() => { })));
+                    }
+                    else {
+                        await fsp.copyFile(src, dest).catch(() => { });
+                    }
+                };
+                await copyRecursive(userDataDirectory, effectiveDir).catch(() => { });
+            }
+            catch {
+                // Silent fallback: use empty profile if clone fails
+            }
+        }
+        // Clean any stale lock files that may block browser launches on Windows
+        const lockFiles = [
+            path.join(effectiveDir, 'SingletonLock'),
+            path.join(effectiveDir, 'SingletonSocket'),
+            path.join(effectiveDir, 'SingletonCookie'),
+            path.join(effectiveDir, 'lockfile'),
+            path.join(effectiveDir, 'Default', 'LOCK'),
+            path.join(effectiveDir, 'Default', 'Network', 'LOCK'),
+        ];
+        await Promise.all(lockFiles.map(f => fsp.rm(f, { force: true }).catch(() => { })));
+        // eslint-disable-next-line no-param-reassign
+        launchContext.userDataDir = effectiveDir;
+    };
+};
 export const failedRequestHandler = async ({ request }) => {
     guiInfoLog(guiInfoStatusTypes.ERROR, { numScanned: 0, urlScanned: request.url });
     log.error(`Failed Request - ${request.url}: ${request.errorMessages}`);