@govtechsg/oobee 0.10.78-alpha1 → 0.10.84

package/Dockerfile

@@ -1,6 +1,6 @@
 # Use Microsoft Playwright image as base image
 # Node version is v22
-FROM mcr.microsoft.com/playwright:v1.55.0-noble
+FROM mcr.microsoft.com/playwright:v1.58.2-noble
 
 # Installation of packages for oobee and runner (locked versions from build log)
 RUN apt-get update && apt-get install -y \
@@ -23,7 +23,8 @@ ENV NODE_ENV=production
 ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD="true"
 
 # Install oobee dependencies
-RUN npm ci --omit=dev
+# TODO: Move back to npm ci --omit=dev once module package-lock issue vs MacOS is resolved
+RUN npm install --omit=dev
 
 # Compile TypeScript for oobee
 RUN npm run build || true # true exits with code 0 - workaround for TS errors

package/INTEGRATION.md

@@ -468,4 +468,4 @@ Scans raw HTML string(s). Note that this runs in a JSDOM environment (NodeJS) us
 
 **Returns:**
 
-- Scan results object containing categorized violations and pass counts.
+- Scan results object containing categorized violations (mustFix, goodToFux, needsReview).

package/README.md

@@ -90,6 +90,11 @@ verapdf --version
 | WARN_LEVEL | Only used in tests. |  |
 | OOBEE_DISABLE_BROWSER_DOWNLOAD | Experimental flag to disable file downloads on Chrome/Chromium/Edge.  Does not affect Local File scan | |
 | OOBEE_SLOWMO | Experimental flag to slow down web browser behaviour by specified duration (in miliseconds) | |
+| HTTP_PROXY | URL of the proxy server to be used for HTTP requests (e.g. `http://proxy.example.com:8080`). | |
+| HTTPS_PROXY | URL of the proxy server to be used for HTTPS requests (e.g. `https://proxy.example.com:8080`). | |
+| ALL_PROXY | URL of the proxy server to be used for all requests, typically used for SOCKS5 proxies (e.g. `socks5://proxy.example.com:1080`. Note: IPv6 direct connections may still continue even though socks5 proxy is specified due to a known issue with Chrome/Chromium. (Recommended workaround is to turn off IPv6 at host-level). | |
+| NO_PROXY | Comma-separated list of domains that should bypass the proxy (e.g. `localhost,127.0.0.1,.example.com`). | |
+| INCLUDE_PROXY | Comma-separated list of domains that should specifically be routed through the proxy. | |
 
 #### Environment variables used internally (Do not set)
 Do not set these environment variables or behaviour might change unexpectedly.
@@ -677,4 +682,4 @@ It uses the existing report *.json files for the embedded HTML dataset.
 
 ```
 npx tsx dev/runGenerateJustHtmlReport.ts results/<report directory>
-```
+```

package/dist/constants/common.js

@@ -288,16 +288,45 @@ const checkUrlConnectivityWithBrowser = async (url, browserToRun, clonedDataDir,
     extraHTTPHeaders.Accept ||= 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';
     await initModifiedUserAgent(browserToRun, playwrightDeviceDetailsObject, clonedDataDir);
     let browserContext;
+    let browserInstance;
+    const rawDevice = (playwrightDeviceDetailsObject || {});
+    const { viewport, isMobile, hasTouch, userAgent: deviceUserAgent, ...restDevice } = rawDevice;
+    const launchOptions = getPlaywrightLaunchOptions(browserToRun);
+    const contextOptions = {
+        ...restDevice,
+        ...(extraHTTPHeaders && { extraHTTPHeaders }),
+        ignoreHTTPSErrors: true,
+        ...(process.env.OOBEE_DISABLE_BROWSER_DOWNLOAD && { acceptDownloads: false }),
+    };
+    // Keep UA emulation explicitly.
+    contextOptions.userAgent = process.env.OOBEE_USER_AGENT || deviceUserAgent;
     try {
-        browserContext = await constants.launcher.launchPersistentContext(clonedDataDir, {
-            ...(extraHTTPHeaders && { extraHTTPHeaders }),
-            ignoreHTTPSErrors: true,
-            headless: true,
-            ...getPlaywrightLaunchOptions(browserToRun),
-            ...playwrightDeviceDetailsObject,
-            ...(process.env.OOBEE_DISABLE_BROWSER_DOWNLOAD && { acceptDownloads: false }),
-        });
-        register(browserContext);
+        const launchPersistent = async () => {
+            browserContext = await constants.launcher.launchPersistentContext(clonedDataDir, {
+                ...launchOptions,
+                ...contextOptions,
+            });
+            register(browserContext);
+        };
+        const launchEphemeral = async () => {
+            browserInstance = await constants.launcher.launch(launchOptions);
+            register(browserInstance);
+            browserContext = await browserInstance.newContext(contextOptions);
+        };
+        if (process.env.CRAWLEE_HEADLESS === '1') {
+            try {
+                await launchPersistent();
+            }
+            catch (error) {
+                // Fallback to ephemeral context if persistent context fails (e.g. protocol errors)
+                // More prone to falling back here when running localFile scans
+                consoleLogger.warn(`Persistent context launch failed, retrying with ephemeral context: ${error.message}`);
+                await launchEphemeral();
+            }
+        }
+        else {
+            await launchEphemeral();
+        }
     }
     catch (err) {
         printMessage([`Unable to launch browser\n${err}`], messageOptions);
@@ -335,7 +364,19 @@ const checkUrlConnectivityWithBrowser = async (url, browserToRun, clonedDataDir,
         });
         if (!response)
             throw new Error('No response from navigation');
-        // We use the response headers from the navigation we just performed.
+        // Wait briefly for JS/meta-refresh redirects to settle before reading the final URL.
+        // Server-side redirects are already reflected after goto(), but client-side redirects
+        // (e.g. domain.tld -> www.domain.tld via JS or meta-refresh) need extra time.
+        try {
+            await Promise.race([
+                page.waitForURL(currentUrl => currentUrl !== url, { timeout: 5000 }),
+                new Promise(resolve => setTimeout(resolve, 1000)), // minimum settle time
+            ]);
+        }
+        catch {
+            // No redirect happened within the window — that's fine, continue with current URL
+        }
+        // Re-read page.url() AFTER potential client-side redirects have resolved
         const finalUrl = page.url();
         const finalStatus = response.status();
         const headers = response.headers();
@@ -399,7 +440,10 @@ const checkUrlConnectivityWithBrowser = async (url, browserToRun, clonedDataDir,
         }
     }
     finally {
-        await browserContext.close();
+        await browserContext?.close();
+        if (browserInstance) {
+            await browserInstance.close();
+        }
     }
     return res;
 };
@@ -630,24 +674,49 @@ export const getUrlsFromRobotsTxt = async (url, browserToRun, userDataDirectory,
     constants.robotsTxtUrls[domain] = { disallowedUrls, allowedUrls };
 };
 const getRobotsTxtViaPlaywright = async (robotsUrl, browser, userDataDirectory, extraHTTPHeaders) => {
-    const robotsDataDir = '';
+    let robotsDataDir = '';
+    let browserContext;
+    let browserInstance;
     // Bug in Chrome which causes browser pool crash when userDataDirectory is set in non-headless mode
     if (process.env.CRAWLEE_HEADLESS === '1') {
         // Create robots own user data directory else SingletonLock: File exists (17) with crawlDomain or crawlSitemap's own browser
-        const robotsDataDir = path.join(userDataDirectory, 'robots');
+        robotsDataDir = path.join(userDataDirectory, 'robots');
         if (!fs.existsSync(robotsDataDir)) {
             fs.mkdirSync(robotsDataDir, { recursive: true });
         }
     }
-    const browserContext = await constants.launcher.launchPersistentContext(robotsDataDir, {
-        ...getPlaywrightLaunchOptions(browser),
-        ...(extraHTTPHeaders && { extraHTTPHeaders }),
-    });
-    register(browserContext);
-    const page = await browserContext.newPage();
-    await page.goto(robotsUrl, { waitUntil: 'networkidle', timeout: 30000 });
-    const robotsTxt = await page.evaluate(() => document.body.textContent);
-    return robotsTxt;
+    try {
+        if (process.env.CRAWLEE_HEADLESS === '1') {
+            browserContext = await constants.launcher.launchPersistentContext(robotsDataDir, {
+                ...getPlaywrightLaunchOptions(browser),
+                ...(extraHTTPHeaders && { extraHTTPHeaders }),
+            });
+            register(browserContext);
+        }
+        else {
+            // In headful mode, avoid launchPersistentContext with custom user data dir to prevent "Browser window not found"
+            const launchOptions = getPlaywrightLaunchOptions(browser);
+            browserInstance = await constants.launcher.launch(launchOptions);
+            register(browserInstance);
+            browserContext = await browserInstance.newContext({
+                ...(extraHTTPHeaders && { extraHTTPHeaders }),
+            });
+        }
+        const page = await browserContext.newPage();
+        await page.goto(robotsUrl, { waitUntil: 'networkidle', timeout: 30000 });
+        const robotsTxt = await page.evaluate(() => document.body.textContent);
+        return robotsTxt;
+    }
+    catch (e) {
+        consoleLogger.error(`Error fetching robots.txt: ${e.message}`);
+        throw e;
+    }
+    finally {
+        await browserContext?.close();
+        if (browserInstance) {
+            await browserInstance.close();
+        }
+    }
 };
 export const isDisallowedInRobotsTxt = (url) => {
     if (!constants.robotsTxtUrls)
@@ -771,39 +840,59 @@ export const getLinksFromSitemap = async (sitemapUrl, maxLinksCount, browser, us
             return;
         }
         const getDataUsingPlaywright = async () => {
-            const browserContext = await constants.launcher.launchPersistentContext(finalUserDataDirectory, {
-                ...getPlaywrightLaunchOptions(browser),
-                // Not necessary to parse http_credentials as I am parsing it directly in URL
-                // Bug in Chrome which causes browser pool crash when userDataDirectory is set in non-headless mode
-                ...(process.env.CRAWLEE_HEADLESS === '1' && { userDataDir: userDataDirectory }),
-                ...(extraHTTPHeaders && { extraHTTPHeaders }),
-            });
-            register(browserContext);
-            const page = await browserContext.newPage();
-            await page.goto(url, { waitUntil: 'networkidle', timeout: 60000 });
-            if ((await page.locator('body').count()) > 0) {
-                data = await page.locator('body').innerText();
-            }
-            else {
-                const urlSet = page.locator('urlset');
-                const sitemapIndex = page.locator('sitemapindex');
-                const rss = page.locator('rss');
-                const feed = page.locator('feed');
-                const isRoot = async (locator) => (await locator.count()) > 0;
-                if (await isRoot(urlSet)) {
-                    data = await urlSet.evaluate(elem => elem.outerHTML);
+            let browserContext;
+            let browserInstance;
+            try {
+                if (process.env.CRAWLEE_HEADLESS === '1') {
+                    browserContext = await constants.launcher.launchPersistentContext(finalUserDataDirectory, {
+                        ...getPlaywrightLaunchOptions(browser),
+                        // Not necessary to parse http_credentials as I am parsing it directly in URL
+                        // Bug in Chrome which causes browser pool crash when userDataDirectory is set in non-headless mode
+                        ...(process.env.CRAWLEE_HEADLESS === '1' && { userDataDir: userDataDirectory }),
+                        ...(extraHTTPHeaders && { extraHTTPHeaders }),
+                    });
+                    register(browserContext);
                 }
-                else if (await isRoot(sitemapIndex)) {
-                    data = await sitemapIndex.evaluate(elem => elem.outerHTML);
+                else {
+                    // In headful mode, avoid launchPersistentContext with custom user data dir to prevent "Browser window not found"
+                    const launchOptions = getPlaywrightLaunchOptions(browser);
+                    browserInstance = await constants.launcher.launch(launchOptions);
+                    register(browserInstance);
+                    browserContext = await browserInstance.newContext({
+                        ...(extraHTTPHeaders && { extraHTTPHeaders }),
+                    });
+                }
+                const page = await browserContext.newPage();
+                await page.goto(url, { waitUntil: 'networkidle', timeout: 60000 });
+                if ((await page.locator('body').count()) > 0) {
+                    data = await page.locator('body').innerText();
                 }
-                else if (await isRoot(rss)) {
-                    data = await rss.evaluate(elem => elem.outerHTML);
+                else {
+                    const urlSet = page.locator('urlset');
+                    const sitemapIndex = page.locator('sitemapindex');
+                    const rss = page.locator('rss');
+                    const feed = page.locator('feed');
+                    const isRoot = async (locator) => (await locator.count()) > 0;
+                    if (await isRoot(urlSet)) {
+                        data = await urlSet.evaluate(elem => elem.outerHTML);
+                    }
+                    else if (await isRoot(sitemapIndex)) {
+                        data = await sitemapIndex.evaluate(elem => elem.outerHTML);
+                    }
+                    else if (await isRoot(rss)) {
+                        data = await rss.evaluate(elem => elem.outerHTML);
+                    }
+                    else if (await isRoot(feed)) {
+                        data = await feed.evaluate(elem => elem.outerHTML);
+                    }
                 }
-                else if (await isRoot(feed)) {
-                    data = await feed.evaluate(elem => elem.outerHTML);
+            }
+            finally {
+                await browserContext?.close();
+                if (browserInstance) {
+                    await browserInstance.close();
                 }
             }
-            await browserContext.close();
         };
         if (validator.isURL(url, urlOptions)) {
             if (isUrlPdf(url)) {
@@ -1486,36 +1575,25 @@ export const submitForm = async (browserToRun, userDataDirectory, scannedUrl, en
     }
 };
 // Legacy code end - Google Sheets submission
-export async function initModifiedUserAgent(browser, playwrightDeviceDetailsObject, userDataDirectory) {
-    const isHeadless = process.env.CRAWLEE_HEADLESS === '1';
-    // If headless mode is enabled, ensure the headless flag is set.
-    if (isHeadless && !constants.launchOptionsArgs.includes('--headless=new')) {
-        constants.launchOptionsArgs.push('--headless=new');
-    }
-    // Build the launch options using your production settings.
-    // headless is forced to false as in your persistent context, and we merge in getPlaywrightLaunchOptions and device details.
-    const launchOptions = {
-        headless: false,
-        ...getPlaywrightLaunchOptions(browser),
-        ...playwrightDeviceDetailsObject,
-    };
-    // Launch a temporary persistent context with an empty userDataDir to mimic your production browser setup.
-    const effectiveUserDataDirectory = process.env.CRAWLEE_HEADLESS === '1' ? userDataDirectory : '';
-    const browserContext = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, launchOptions);
-    register(browserContext);
-    const page = await browserContext.newPage();
-    // Retrieve the default user agent.
-    const defaultUA = await page.evaluate(() => navigator.userAgent);
-    await browserContext.close();
-    // Modify the UA:
-    // Replace "HeadlessChrome" with "Chrome" if present.
-    const modifiedUA = defaultUA.includes('HeadlessChrome')
-        ? defaultUA.replace('HeadlessChrome', 'Chrome')
-        : defaultUA;
-    // Push the modified UA flag into your global launch options.
-    constants.launchOptionsArgs.push(`--user-agent=${modifiedUA}`);
-    // Optionally log the modified UA.
-    // console.log('Modified User Agent:', modifiedUA);
+export async function initModifiedUserAgent(browser, _playwrightDeviceDetailsObject, _userDataDirectory) {
+    // UA bootstrap must not use persistent context / user-data-dir.
+    const launchOptions = getPlaywrightLaunchOptions(browser);
+    const browserInstance = await constants.launcher.launch(launchOptions);
+    register(browserInstance);
+    try {
+        const context = await browserInstance.newContext();
+        const page = await context.newPage();
+        const defaultUA = await page.evaluate(() => navigator.userAgent);
+        await context.close();
+        const modifiedUA = defaultUA.includes('HeadlessChrome')
+            ? defaultUA.replace('HeadlessChrome', 'Chrome')
+            : defaultUA;
+        // Do not mutate global CLI args with --user-agent=
+        process.env.OOBEE_USER_AGENT = modifiedUA;
+    }
+    finally {
+        await browserInstance.close();
+    }
 }
 const cacheProxyInfo = getProxyInfo();
 /**
@@ -1525,12 +1603,13 @@ const cacheProxyInfo = getProxyInfo();
 export const getPlaywrightLaunchOptions = (browser) => {
     const channel = browser || undefined;
     const resolution = proxyInfoToResolution(cacheProxyInfo);
-    // Start with your base args
-    const finalArgs = [...constants.launchOptionsArgs];
+    // Start with your base args and sanitise
+    const finalArgs = [...constants.launchOptionsArgs].filter(arg => !arg.startsWith('--headless') &&
+        !arg.startsWith('--user-agent=') &&
+        arg !== '--mute-audio' &&
+        !(browser === BrowserTypes.CHROME && arg === '--edge-skip-compat-layer-relaunch'));
     // Headless flags (unchanged)
     if (process.env.CRAWLEE_HEADLESS === '1') {
-        if (!finalArgs.includes('--headless=new'))
-            finalArgs.push('--headless=new');
         if (!finalArgs.includes('--mute-audio'))
             finalArgs.push('--mute-audio');
     }
@@ -1551,21 +1630,17 @@ export const getPlaywrightLaunchOptions = (browser) => {
             break;
     }
     const options = {
-        ignoreDefaultArgs: ['--use-mock-keychain', '--headless'],
+        ignoreDefaultArgs: ['--use-mock-keychain'],
         args: finalArgs,
-        headless: false,
+        headless: process.env.CRAWLEE_HEADLESS === '1',
         ...(channel && { channel }),
         ...(proxyOpt ? { proxy: proxyOpt } : {}),
     };
-    // SlowMo (unchanged)
+    // SlowMo for debugging, can be set via env variable OOBEE_SLOWMO to avoid adding it as a CLI argument and causing confusion for users who don't need it
     if (!options.slowMo && process.env.OOBEE_SLOWMO && Number(process.env.OOBEE_SLOWMO) >= 1) {
         options.slowMo = Number(process.env.OOBEE_SLOWMO);
         consoleLogger.info(`Enabled browser slowMo with value: ${process.env.OOBEE_SLOWMO}ms`);
     }
-    // Edge on Windows should not be headless (unchanged)
-    if (browser === BrowserTypes.EDGE && os.platform() === 'win32') {
-        options.headless = false;
-    }
     return options;
 };
 export const waitForPageLoaded = async (page, timeout = 10000) => {

package/dist/constants/constants.js

@@ -725,6 +725,7 @@ export const disabilityBadgesMap = {
     'skip-link': ['Motor', 'Learning', 'Visual'],
     tabindex: ['Motor'],
     'meta-viewport': ['Visual'],
+    'summary-name': ['Visual'],
 };
 export default {
     cliZipFileName: 'oobee-scan-results.zip',