@govtechsg/oobee 0.10.90 → 0.10.92

package/dist/mergeAxeResults.js

@@ -5,6 +5,7 @@ import printMessage from 'print-message';
 import path from 'path';
 import ejs from 'ejs';
 import { fileURLToPath } from 'url';
+import { Dataset, RequestQueue, Configuration } from 'crawlee';
 import constants, { BrowserTypes, ScannerTypes, WCAGclauses, a11yRuleShortDescriptionMap, disabilityBadgesMap, a11yRuleLongDescriptionMap, a11yRuleStepByStepGuide, } from './constants/constants.js';
 import { getBrowserToRun, getPlaywrightLaunchOptions } from './constants/common.js';
 import { createScreenshotsFolder, getStoragePath, getVersion, getWcagPassPercentage, getProgressPercentage, retryFunction, zipResults, getIssuesPercentage, register, } from './utils.js';
@@ -255,54 +256,84 @@ const cleanUpJsonFiles = async (filesToDelete) => {
     });
 };
 const writeSummaryPdf = async (storagePath, pagesScanned, filename = 'summary', browser, _userDataDirectory) => {
-    let browserInstance;
-    let context;
-    let page;
-    try {
-        const htmlFilePath = path.join(storagePath, `${filename}.html`);
-        const fileDestinationPath = path.join(storagePath, `${filename}.pdf`);
-        const htmlFileUrl = `file://${htmlFilePath}`;
-        const launchOptions = getPlaywrightLaunchOptions(browser);
-        browserInstance = await constants.launcher.launch({
-            ...launchOptions,
-            headless: true,
-        });
-        register(browserInstance);
-        context = await browserInstance.newContext();
-        page = await context.newPage();
-        await page.goto(htmlFileUrl, {
-            waitUntil: 'domcontentloaded',
-            timeout: 120000,
-        });
-        await page.emulateMedia({ media: 'print' });
-        await page.pdf({
-            margin: { bottom: '32px' },
-            path: fileDestinationPath,
-            format: 'A4',
-            displayHeaderFooter: true,
-            footerTemplate: `
+    const renderPdfWithBrowser = async (browserToUse) => {
+        let browserInstance;
+        let context;
+        let page;
+        try {
+            const htmlFilePath = path.join(storagePath, `${filename}.html`);
+            const fileDestinationPath = path.join(storagePath, `${filename}.pdf`);
+            const htmlFileUrl = `file://${htmlFilePath}`;
+            const launchOptions = getPlaywrightLaunchOptions(browserToUse);
+            browserInstance = await constants.launcher.launch({
+                ...launchOptions,
+                headless: true,
+            });
+            register(browserInstance);
+            context = await browserInstance.newContext();
+            page = await context.newPage();
+            await page.goto(htmlFileUrl, {
+                waitUntil: 'domcontentloaded',
+                timeout: 120000,
+            });
+            await page.emulateMedia({ media: 'print' });
+            await page.pdf({
+                margin: { bottom: '32px' },
+                path: fileDestinationPath,
+                format: 'A4',
+                displayHeaderFooter: true,
+                footerTemplate: `
     <div style="margin-top:50px;color:#26241b;font-family:Open Sans;text-align: center;width: 100%;font-weight:400">
       <span style="color:#26241b;font-size: 14px;font-weight:400">Page <span class="pageNumber"></span> of <span class="totalPages"></span></span>
     </div>
   `,
-        });
-        if (pagesScanned < 2000) {
+            });
             fs.unlinkSync(htmlFilePath);
         }
+        finally {
+            try {
+                await page?.close();
+            }
+            catch (err) {
+                consoleLogger.info(`Error at page close writeSummaryPDF ${err}`);
+            }
+            try {
+                await context?.close();
+            }
+            catch (err) {
+                consoleLogger.info(`Error at context close writeSummaryPDF ${err}`);
+            }
+            try {
+                await browserInstance?.close();
+            }
+            catch (err) {
+                consoleLogger.info(`Error at browserInstance close writeSummaryPDF ${err}`);
+            }
+        }
+    };
+    const browserAttempts = [browser];
+    // Runtime fallback: if Chrome launch fails on Windows, try Edge once for PDF generation.
+    if (process.platform === 'win32' && browser === BrowserTypes.CHROME) {
+        browserAttempts.push(BrowserTypes.EDGE);
     }
-    catch (err) {
-        consoleLogger.info(`Error at writeSummaryPDF ${err instanceof Error ? err.stack : err}`);
-    }
-    finally {
-        await page?.close().catch(err => {
-            consoleLogger.info(`Error at page close writeSummaryPDF ${err}`);
-        });
-        await context?.close().catch(err => {
-            consoleLogger.info(`Error at context close writeSummaryPDF ${err}`);
-        });
-        await browserInstance?.close().catch(err => {
-            consoleLogger.info(`Error at browserInstance close writeSummaryPDF ${err}`);
-        });
+    for (let i = 0; i < browserAttempts.length; i++) {
+        const currentBrowser = browserAttempts[i];
+        try {
+            await renderPdfWithBrowser(currentBrowser);
+            if (i > 0) {
+                consoleLogger.warn(`writeSummaryPDF succeeded with fallback browser '${currentBrowser}' after '${browser}' failed.`);
+            }
+            return;
+        }
+        catch (err) {
+            const isLastAttempt = i === browserAttempts.length - 1;
+            consoleLogger.info(`Error at writeSummaryPDF using browser '${currentBrowser}': ${err instanceof Error ? err.stack : err}`);
+            if (isLastAttempt) {
+                return;
+            }
+            const nextBrowser = browserAttempts[i + 1];
+            consoleLogger.warn(`writeSummaryPDF failed using browser '${currentBrowser}', retrying with '${nextBrowser}'.`);
+        }
     }
 };
 // Tracking WCAG occurrences
@@ -489,6 +520,7 @@ const extractRuleAiData = (ruleId, totalItems, items, callback) => {
 export const createRuleIdJson = async (allIssues, itemsStore) => {
     const compiledRuleJson = {};
     for (const category of ['mustFix', 'goodToFix', 'needsReview']) {
+        compiledRuleJson[category] = {};
         for (const rule of allIssues.items[category].rules) {
             let allItems = [];
             if (itemsStore) {
@@ -499,7 +531,7 @@ export const createRuleIdJson = async (allIssues, itemsStore) => {
             else {
                 allItems = rule.pagesAffected.flatMap(page => page.items || []);
             }
-            compiledRuleJson[rule.rule] = extractRuleAiData(rule.rule, rule.totalItems, allItems);
+            compiledRuleJson[category][rule.rule] = extractRuleAiData(rule.rule, rule.totalItems, allItems);
         }
     }
     return compiledRuleJson;
@@ -508,9 +540,10 @@ export const createRuleIdJson = async (allIssues, itemsStore) => {
 export const createBasicFormHTMLSnippet = filteredResults => {
     const compiledRuleJson = {};
     ['mustFix', 'goodToFix', 'needsReview'].forEach(category => {
+        compiledRuleJson[category] = {};
         if (filteredResults[category] && filteredResults[category].rules) {
             Object.entries(filteredResults[category].rules).forEach(([ruleId, ruleVal]) => {
-                compiledRuleJson[ruleId] = extractRuleAiData(ruleId, ruleVal.totalItems, ruleVal.items);
+                compiledRuleJson[category][ruleId] = extractRuleAiData(ruleId, ruleVal.totalItems, ruleVal.items);
             });
         }
     });
@@ -547,7 +580,7 @@ const formatAboutStartTime = (dateString) => {
     return htmlFormattedStartTime;
 };
 const generateArtifacts = async (randomToken, urlScanned, scanType, viewport, pagesScanned, pagesNotScanned, customFlowLabel, cypressScanAboutMetadata, scanDetails, zip = undefined, // optional
-generateJsonFiles = false) => {
+generateJsonFiles = false, preferredBrowser) => {
     consoleLogger.info('Generating report artifacts');
     const storagePath = getStoragePath(randomToken);
     const intermediateDatasetsPath = `${storagePath}/crawlee`;
@@ -564,6 +597,8 @@ generateJsonFiles = false) => {
         endTime: scanDetails.endTime ? scanDetails.endTime : new Date(),
         urlScanned,
         scanType,
+        totalLinksFetchedFromSitemaps: constants.sitemapFetchedLinks?.totalLinksFetchedFromSitemaps ?? 0,
+        fetchedSitemaps: constants.sitemapFetchedLinks?.fetchedSitemaps ?? [],
         deviceChosen: scanDetails.deviceChosen || 'Desktop',
         formatAboutStartTime,
         isCustomFlow,
@@ -752,16 +787,37 @@ generateJsonFiles = false) => {
             scanPagesSummaryBase64FilePath,
         ]);
     }
-    const browserChannel = getBrowserToRun(randomToken, BrowserTypes.CHROME, false).browserToRun;
+    const browserChannel = getBrowserToRun(randomToken, preferredBrowser || BrowserTypes.CHROME, false).browserToRun;
     // Should consider refactor constants.userDataDirectory to be a parameter in future
     await retryFunction(() => writeSummaryPdf(storagePath, pagesScanned.length, 'summary', browserChannel, constants.userDataDirectory), 1);
-    // Brief delay to allow lingering async crawlee storage operations to flush
-    await new Promise(resolve => setTimeout(resolve, 3000));
+    // Flush pending background storage operations (metadata writes, lock-file ops)
+    const storageClient = Configuration.getStorageClient();
+    if (storageClient.teardown) {
+        await storageClient.teardown();
+    }
+    // Gracefully drop Dataset and RequestQueue — releases locks and removes files
+    const crawleeDir = path.join(storagePath, 'crawlee');
     try {
-        await fs.promises.rm(path.join(storagePath, 'crawlee'), { recursive: true, force: true });
+        const dataset = await Dataset.open(crawleeDir);
+        await dataset.drop();
     }
     catch (error) {
-        // Silently ignore — folder may already be gone or still locked
+        consoleLogger.info(`Dataset drop: ${error.message}`);
+    }
+    try {
+        const requestQueue = await RequestQueue.open(crawleeDir);
+        await requestQueue.drop();
+    }
+    catch (error) {
+        consoleLogger.info(`RequestQueue drop: ${error.message}`);
+    }
+    // Fallback rm for any leftover files not managed by Crawlee's storage API
+    const crawleePath = path.join(storagePath, 'crawlee');
+    try {
+        await fs.promises.rm(crawleePath, { recursive: true, force: true });
+    }
+    catch {
+        // Best-effort; storage was already dropped via API
     }
     try {
         await fs.promises.rm(path.join(storagePath, 'pdfs'), { recursive: true, force: true });
@@ -769,6 +825,22 @@ generateJsonFiles = false) => {
     catch (error) {
         consoleLogger.warn(`Unable to force remove pdfs folder: ${error.message}`);
     }
+    // Generate scrubbed HTML Code Snippets
+    const ruleIdJson = await createRuleIdJson(allIssues, itemsStore);
+    // Clean up intermediate items files before zipping
+    await itemsStore.cleanup();
+    try {
+        await sendWcagBreakdownToSentry(oobeeAppVersion, wcagOccurrencesMap, ruleIdJson, {
+            entryUrl: urlScanned,
+            scanType,
+            browser: scanDetails.deviceChosen,
+            email: scanDetails.nameEmail?.email,
+            name: scanDetails.nameEmail?.name,
+        }, allIssues, pagesScanned.length);
+    }
+    catch (error) {
+        console.error('Error sending WCAG data to Sentry:', error);
+    }
     // Take option if set
     if (typeof zip === 'string') {
         constants.cliZipFileName = zip;
@@ -804,25 +876,6 @@ generateJsonFiles = false) => {
     catch (error) {
         printMessage([`Error in zipping results: ${error}`]);
     }
-    // Generate scrubbed HTML Code Snippets
-    const ruleIdJson = await createRuleIdJson(allIssues, itemsStore);
-    // Clean up intermediate items files
-    await itemsStore.cleanup();
-    // At the end of the function where results are generated, add:
-    try {
-        // Always send WCAG breakdown to Sentry, even if no violations were found
-        // This ensures that all criteria are reported, including those with 0 occurrences
-        await sendWcagBreakdownToSentry(oobeeAppVersion, wcagOccurrencesMap, ruleIdJson, {
-            entryUrl: urlScanned,
-            scanType,
-            browser: scanDetails.deviceChosen,
-            email: scanDetails.nameEmail?.email,
-            name: scanDetails.nameEmail?.name,
-        }, allIssues, pagesScanned.length);
-    }
-    catch (error) {
-        console.error('Error sending WCAG data to Sentry:', error);
-    }
     if (process.env.RUNNING_FROM_PH_GUI || process.env.OOBEE_VERBOSE)
         console.log('Report generated successfully');
     return ruleIdJson;

package/dist/npmIndex.js

@@ -290,6 +290,7 @@ thresholds = { mustFix: undefined, goodToFix: undefined }, scanAboutMetadata = u
     // max numbers of mustFix/goodToFix occurrences before test returns a fail
     const { mustFix: mustFixThreshold, goodToFix: goodToFixThreshold } = thresholds;
     process.env.CRAWLEE_STORAGE_DIR = randomToken;
+    constants.sitemapFetchedLinks = null;
     const scanDetails = {
         startTime: new Date(),
         endTime: new Date(),

package/dist/utils.js

@@ -5,6 +5,7 @@ import axe from 'axe-core';
 import { v4 as uuidv4 } from 'uuid';
 import { getDomain } from 'tldts';
 import { normalizeUrl } from '@apify/utilities';
+import { Dataset, RequestQueue, Configuration } from 'crawlee';
 import constants, { destinationPath, getIntermediateScreenshotsPath, } from './constants/constants.js';
 import { consoleLogger, errorsTxtPath } from './logs.js';
 import { getAxeConfiguration } from './crawlers/custom/getAxeConfiguration.js';
@@ -346,6 +347,20 @@ export const cleanUp = async (randomToken, isError = false) => {
         }
     if (randomToken !== undefined) {
         const storagePath = getStoragePath(randomToken);
+        try {
+            const storageClient = Configuration.getStorageClient();
+            if (storageClient.teardown) {
+                await storageClient.teardown();
+            }
+            const crawleeDir = path.join(storagePath, 'crawlee');
+            const dataset = await Dataset.open(crawleeDir);
+            await dataset.drop();
+            const requestQueue = await RequestQueue.open(crawleeDir);
+            await requestQueue.drop();
+        }
+        catch (error) {
+            consoleLogger.info(`Crawlee storage drop in cleanUp: ${error.message}`);
+        }
         try {
             fs.rmSync(path.join(storagePath, 'crawlee'), { recursive: true, force: true });
         }
@@ -358,32 +373,8 @@ export const cleanUp = async (randomToken, isError = false) => {
         catch (error) {
             consoleLogger.warn(`Unable to force remove pdfs folder: ${error.message}`);
         }
-        let deleteErrorLogFile = true;
-        if (isError) {
-            let logsPath = storagePath;
-            if (process.env.OOBEE_LOGS_PATH) {
-                logsPath = process.env.OOBEE_LOGS_PATH;
-            }
-            if (fs.existsSync(errorsTxtPath)) {
-                try {
-                    const logFilePath = path.join(logsPath, `logs-${randomToken}.txt`);
-                    fs.copyFileSync(errorsTxtPath, logFilePath);
-                    console.log(`An error occured. Log file is located at: ${logFilePath}`);
-                }
-                catch (copyError) {
-                    consoleLogger.error(`Error copying errors file during cleanup: ${copyError.message}`);
-                    console.log(`An error occured. Log file is located at: ${errorsTxtPath}`);
-                    deleteErrorLogFile = false; // Do not delete the log file if copy failed
-                }
-                if (deleteErrorLogFile && fs.existsSync(errorsTxtPath)) {
-                    try {
-                        fs.unlinkSync(errorsTxtPath);
-                    }
-                    catch (error) {
-                        consoleLogger.warn(`Unable to delete log file ${errorsTxtPath}: ${error.message}`);
-                    }
-                }
-            }
+        if (isError && fs.existsSync(errorsTxtPath)) {
+            console.log(`An error occured. Log file is located at: ${errorsTxtPath}`);
         }
         if (fs.existsSync(storagePath) && fs.readdirSync(storagePath).length === 0) {
             try {
@@ -854,6 +845,8 @@ export const randomThreeDigitNumberString = () => {
     return String(threeDigitNumber);
 };
 export const normUrl = (u) => (u ? normalizeUrl(u) || u : '');
+export const stripWwwPrefix = (hostname) => hostname.replace(/^www\./, '');
+export const isSameHostname = (hostname1, hostname2) => stripWwwPrefix(hostname1) === stripWwwPrefix(hostname2);
 export const isFollowStrategy = (link1, link2, rule) => {
     if (rule === 'all')
         return true;
@@ -861,7 +854,9 @@ export const isFollowStrategy = (link1, link2, rule) => {
         const parsedLink1 = new URL(link1);
         const parsedLink2 = new URL(link2);
         if (rule === 'same-origin') {
-            return parsedLink1.origin === parsedLink2.origin;
+            return parsedLink1.protocol === parsedLink2.protocol &&
+                isSameHostname(parsedLink1.hostname, parsedLink2.hostname) &&
+                parsedLink1.port === parsedLink2.port;
         }
         if (rule === 'same-domain') {
             const link1Domain = getDomain(parsedLink1.hostname, { allowPrivateDomains: true }) || parsedLink1.hostname;
@@ -869,7 +864,7 @@ export const isFollowStrategy = (link1, link2, rule) => {
             return link1Domain.toLowerCase() === link2Domain.toLowerCase();
         }
         // default: same-hostname
-        return parsedLink1.hostname === parsedLink2.hostname;
+        return isSameHostname(parsedLink1.hostname, parsedLink2.hostname);
     }
     catch {
         return false;

package/oobee-client-scanner.js

@@ -3,7 +3,7 @@
  * DO NOT EDIT MANUALLY. Re-generate with: node dist/generateOobeeClientScanner.js
  *
  * Embedded at generation time:
- *   App version : 0.10.90
+ *   App version : 0.10.92
  *   Sentry DSN  : (from OOBEE_SENTRY_DSN env var or constants.ts default)
  *   Sentry SDK  : @sentry/browser 9.47.1 (loaded from CDN at runtime)
  *
@@ -34883,7 +34883,7 @@
   // ── Sentry browser telemetry (Sentry JS SDK, loaded from CDN) ────────────
   
   var _oobeeSentryDsn          = "https://3b8c7ee46b06f33815a1301b6713ebc3@o4509047624761344.ingest.us.sentry.io/4509327783559168";
-  var _oobeeAppVersion         = "0.10.90";
+  var _oobeeAppVersion         = "0.10.92";
   var _oobeeSentryVersion      = "9.47.1";
   var _oobeeSentryInitialized  = false;
   var _oobeeSentryLoadPromise  = null;
@@ -35091,6 +35091,37 @@
       // Run axe-core + oobee custom checks
       var scanResult = await window.runA11yScan(elementsToScan, '');
 
+      // Re-verify aria-hidden-focus violations against the live DOM to handle
+      // race conditions with JS that sets tabindex="-1" after aria-hidden
+      var axeViolations = scanResult.axeScanResults.violations || [];
+      var ariaHiddenViolation = axeViolations.find(function(v) { return v.id === 'aria-hidden-focus'; });
+      if (ariaHiddenViolation) {
+        await new Promise(function(resolve) { setTimeout(resolve, 0); });
+        ariaHiddenViolation.nodes = ariaHiddenViolation.nodes.filter(function(node) {
+          var selector = node.target && node.target[0];
+          if (typeof selector !== 'string') return true;
+          try {
+            var el = document.querySelector(selector);
+            if (!el) return true;
+            var focusables = el.querySelectorAll(
+              'a[href], area[href], button:not([disabled]), input:not([disabled]):not([type="hidden"]), select:not([disabled]), textarea:not([disabled]), [tabindex]'
+            );
+            if (focusables.length === 0) return false;
+            return Array.from(focusables).some(function(child) {
+              var tabindex = child.getAttribute('tabindex');
+              if (tabindex === null) return true;
+              var parsed = parseInt(tabindex, 10);
+              return isNaN(parsed) || parsed >= 0;
+            });
+          } catch (e) { return true; }
+        });
+        if (ariaHiddenViolation.nodes.length === 0) {
+          scanResult.axeScanResults.violations = axeViolations.filter(function(v) {
+            return v.id !== 'aria-hidden-focus';
+          });
+        }
+      }
+
       // Convert raw axe results into oobee category structure
       var filtered = _oobeeFilterAxeResults(scanResult.axeScanResults, scanResult.pageTitle);
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.90",
+  "version": "0.10.92",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "bin": {
@@ -26,7 +26,7 @@
     "inquirer": "^9.2.12",
     "jsdom": "^29.0.0",
     "jszip": "^3.10.1",
-    "lodash": "^4.17.21",
+    "lodash": "^4.18.1",
     "mime": "^4.0.7",
     "mime-types": "^2.1.35",
     "minimatch": "^10.2.4",

package/src/cli.ts

@@ -228,6 +228,10 @@ if (!options.strategy) {
   options.strategy = options.scanner === ScannerTypes.SITEMAP ? 'ignore' : 'same-domain';
 }
 
+if (options.websiteTag) {
+  process.env.OOBEE_TAGGED_WEBSITE = options.websiteTag;
+}
+
 const scanInit = async (argvs: Answers): Promise<string> => {
   const updatedArgvs = { ...argvs };
 

package/src/combine.ts

@@ -6,7 +6,7 @@ import crawlLocalFile from './crawlers/crawlLocalFile.js';
 import crawlIntelligentSitemap from './crawlers/crawlIntelligentSitemap.js';
 import generateArtifacts from './mergeAxeResults.js';
 import { getHost, createAndUpdateResultsFolders, cleanUpAndExit, getStoragePath } from './utils.js';
-import { ScannerTypes, UrlsCrawled } from './constants/constants.js';
+import constants, { ScannerTypes, UrlsCrawled } from './constants/constants.js';
 import { getBlackListedPatterns, submitForm } from './constants/common.js';
 import { consoleLogger, silentLogger } from './logs.js';
 import runCustom from './crawlers/runCustom.js';
@@ -72,6 +72,7 @@ const combineRun = async (details: Data, deviceToScan: string) => {
 
   process.env.CRAWLEE_LOG_LEVEL = 'ERROR';
   process.env.CRAWLEE_STORAGE_DIR = randomToken;
+  constants.sitemapFetchedLinks = null;
 
   if (process.env.CRAWLEE_SYSTEM_INFO_V2 === undefined) {
     // Set the environment variable to enable system info v2
@@ -79,6 +80,18 @@ const combineRun = async (details: Data, deviceToScan: string) => {
     process.env.CRAWLEE_SYSTEM_INFO_V2 = '1';
   }
 
+  // Suppress non-fatal Crawlee ps-tree errors on Windows with non-English locales.
+  // The system info module tries to parse process listing headers and crashes when
+  // headers are in a different language (e.g. "Wo" instead of "PID").
+  const psTreeHandler = (err: Error) => {
+    if (err.message?.includes('Unknown process listing header')) {
+      consoleLogger.info(`Suppressed Crawlee ps-tree locale error: ${err.message}`);
+      return;
+    }
+    throw err;
+  };
+  process.on('uncaughtException', psTreeHandler);
+
   const host = type === ScannerTypes.SITEMAP || type === ScannerTypes.LOCALFILE ? '' : getHost(url);
 
   let blacklistedPatterns: string[] | null = null;
@@ -274,6 +287,7 @@ const combineRun = async (details: Data, deviceToScan: string) => {
         scanDetails,
         zip,
         generateJsonFiles,
+        browser,
       );
       const [name, email] = nameEmail.split(':');
 

package/src/constants/cliFunctions.ts

@@ -341,5 +341,12 @@ To obtain the JSON files, you need to base64-decode the file followed by gunzip.
     demandOption: false,
     coerce: val => Number(val),
   },
+  z: {
+    alias: 'websiteTag',
+    describe: 'Tag to identify the website in telemetry. Overrides OOBEE_TAGGED_WEBSITE env var.',
+    type: 'string',
+    requiresArg: true,
+    demandOption: false,
+  },
 };