@govtechsg/oobee 0.10.86 → 0.10.88

package/.github/workflows/docker-push-ghcr.yml

@@ -0,0 +1,49 @@
+name: Build and Push Docker Image to GHCR
+
+on:
+  workflow_dispatch:
+    inputs:
+      release_tag:
+        description: 'Release tag for the image (e.g. 0.10.87)'
+        required: true
+        type: string
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Lowercase repository name
+        id: repo
+        run: echo "name=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT"
+
+      - name: Build and push multi-arch image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ghcr.io/${{ steps.repo.outputs.name }}:${{ inputs.release_tag }}
+            ghcr.io/${{ steps.repo.outputs.name }}:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

package/.github/workflows/image.yml

@@ -146,18 +146,17 @@ jobs:
           chmod -R u+w "$GITHUB_WORKSPACE/oobee"
 
           # Sign all Mach-O (exec bits OR dylib OR node native addons)
-          # Search $GITHUB_WORKSPACE (not just oobee/) to cover scripts copied to the parent dir
           while IFS= read -r f; do
             echo "Signing $f"
             codesign --force --options runtime --timestamp --sign "${CERTIFICATE_NAME}" "$f"
           done < <(
-            find "$GITHUB_WORKSPACE" -type f \
+            find "$GITHUB_WORKSPACE/oobee" -type f \
               \( -perm -111 -o -name "*.dylib" -o -name "*.node" \) \
               ! -path "*/.git/*"
           )
 
           echo "Verifying signatures of Mach-O files..."
-          find "$GITHUB_WORKSPACE" -type f \( -perm -111 -o -name "*.dylib" -o -name "*.node" \) \
+          find "$GITHUB_WORKSPACE/oobee" -type f \( -perm -111 -o -name "*.dylib" -o -name "*.node" \) \
             -exec codesign --verify --strict --verbose=2 {} \; || true
           
       - name: Cleanup keychain

package/DETAILS_OUTPUT_EXAMPLES.md

@@ -0,0 +1,178 @@
+# Enriched Details Output Examples
+
+These are real outputs captured from `scanPage` against intentionally non-compliant test pages.
+The **Details** panel in the HTML report renders the `message` field shown below.
+
+---
+
+## 1. `color-contrast` (WCAG AA — mustFix)
+
+**Element:**
+```html
+<p style="color: #999999; font-size: 14px;">This light gray text on white background fails AA contrast</p>
+```
+
+**Details message:**
+```
+Multiple text elements in this component fail WCAG 1.4.3 Color Contrast Minimum.
+Audit all visible text in the snippet and update every failing foreground color so
+normal text achieves at least 4.5:1 contrast against its actual background, with a
+safety margin above the minimum where possible. Known failing combinations in this
+snippet include foreground #999999 on #ffffff at 14px normal text (current contrast
+2.84, expected 4.5:1). Fix all failing text colors in the component, not just the
+first reported element. Recommendation: To meet the required contrast ratio, for
+foreground #999999 on background #ffffff (target 4.5:1), adjust foreground to #737373
+(rgb(115, 115, 115)) or background to #2e2e2e (rgb(46, 46, 46)).
+```
+
+**Element:**
+```html
+<button style="background-color: #55aa99; color: #e8ffe8; font-size: 12px;">Low contrast button</button>
+```
+
+**Details message:**
+```
+Multiple text elements in this component fail WCAG 1.4.3 Color Contrast Minimum.
+Audit all visible text in the snippet and update every failing foreground color so
+normal text achieves at least 4.5:1 contrast against its actual background, with a
+safety margin above the minimum where possible. Known failing combinations in this
+snippet include foreground #e8ffe8 on #55aa99 at 12px normal text (current contrast
+2.61, expected 4.5:1). Fix all failing text colors in the component, not just the
+first reported element. Recommendation: To meet the required contrast ratio, for
+foreground #e8ffe8 on background #55aa99 (target 4.5:1), adjust foreground to #003a00
+(rgb(0, 58, 0)) or background to #3d7a6e (rgb(61, 122, 110)).
+```
+
+---
+
+## 2. `color-contrast-enhanced` (WCAG AAA — goodToFix)
+
+**Element:**
+```html
+<p style="color: #757575; font-size: 14px;">This text passes AA but fails AAA needs 7 to 1</p>
+```
+
+**Details message:**
+```
+Multiple text elements in this component fail WCAG 1.4.3 Color Contrast Minimum.
+Audit all visible text in the snippet and update every failing foreground color so
+normal text achieves at least 7:1 contrast against its actual background, with a
+safety margin above the minimum where possible. Known failing combinations in this
+snippet include foreground #757575 on #ffffff at 14px normal text (current contrast
+4.6, expected 7:1). Fix all failing text colors in the component, not just the first
+reported element. Recommendation: To meet the required contrast ratio, for foreground
+#757575 on background #ffffff (target 7:1), adjust foreground to #555555
+(rgb(85, 85, 85)).
+```
+
+**Element:**
+```html
+<p style="color: #6b6b6b; font-size: 12px;">Small text needs 7 to 1 for AAA</p>
+```
+
+**Details message:**
+```
+Multiple text elements in this component fail WCAG 1.4.3 Color Contrast Minimum.
+Audit all visible text in the snippet and update every failing foreground color so
+normal text achieves at least 7:1 contrast against its actual background, with a
+safety margin above the minimum where possible. Known failing combinations in this
+snippet include foreground #6b6b6b on #ffffff at 12px normal text (current contrast
+5.32, expected 7:1). Fix all failing text colors in the component, not just the first
+reported element. Recommendation: To meet the required contrast ratio, for foreground
+#6b6b6b on background #ffffff (target 7:1), adjust foreground to #555555
+(rgb(85, 85, 85)).
+```
+
+---
+
+## 3. `target-size` (WCAG 2.5.8 — mustFix)
+
+### Example A: `content-box` elements (no WARNING)
+
+**Element:**
+```html
+<a href="/a" class="icon-link" style="width: 16px; height: 16px;">A</a>
+```
+
+**Details message:**
+```
+Fix any of the following:
+  Target has insufficient size (16px by 16px, should be at least 24px by 24px)
+  Target has insufficient space to its closest neighbors. Safe clickable space has a
+  diameter of 17px instead of at least 24px.
+  Computed hit area: 16px × 16px (box-sizing: content-box).
+```
+
+### Example B: `border-box` element with explicit inline width/height (WARNING appended)
+
+**Element:**
+```html
+<button style="width: 20px; height: 20px; padding: 0; box-sizing: border-box;">X</button>
+```
+
+**Details message:**
+```
+Fix any of the following:
+  Target has insufficient size (20px by 20px, should be at least 24px by 24px)
+  Target has insufficient space to its closest neighbors. Safe clickable space has a
+  diameter of 21px instead of at least 24px.
+  Computed hit area: 20px × 20px (box-sizing: border-box).
+  Tip: inline style sets width: 20px, height: 20px with box-sizing: border-box —
+  padding is included within those dimensions and will not increase the hit area. Fix:
+  remove the explicit width/height and use min-width: 24px; min-height: 24px instead,
+  or place the visual content in a child <span> element.
+```
+
+---
+
+## 4. `valid-lang` (mustFix)
+
+**Element:**
+```html
+<div lang="x-klingon">This section also has an invalid private-use lang tag with some sample text content for context.</div>
+```
+
+**Details message:**
+```
+Fix all of the following:
+  Value of lang attribute not included in the list of valid languages
+  Note: "x-klingon" uses a private-use "x-" prefix. axe-core's valid-lang rule also
+  rejects private-use subtags — you must use a registered IANA language code.
+  Original text: "This section also has an invalid private-use lang tag with some sample
+  text content for context.". Identify the actual language of this text and use its
+  registered BCP 47 code (e.g., lang="it" Italian, "es" Spanish, "fr" French,
+  "de" German, "zh" Chinese, "ja" Japanese, "ko" Korean, "pt" Portuguese, "ar" Arabic).
+```
+
+---
+
+## 5. `oobee-grading-text-contents` (WCAG AAA — needsReview)
+
+**Element:**
+```html
+<html lang="en">...</html>
+```
+
+**Details message:**
+```
+The text content is potentially difficult to read, with a Flesch-Kincaid Reading Ease
+score of 33.92. Difficult — college level or above.
+```
+
+### Score interpretation table (appended inline after the numeric score)
+
+Only scores in the range 1–50 trigger a violation (scores > 50 pass, scores ≤ 0 are filtered out).
+
+| Score Range | Interpretation appended to message |
+|---|---|
+| 31–50 | Difficult — college level or above. |
+| 1–30 | Very difficult — best understood by university graduates. |
+
+---
+
+## Notes
+
+- **color-contrast** and **color-contrast-enhanced** messages include computed color recommendations using WCAG relative luminance math with a binary search on HSL lightness.
+- **target-size** appends `Computed hit area` and, when `box-sizing: border-box` with explicit inline dimensions is detected, a `Tip` explaining why padding won't help.
+- **valid-lang** appends a `NOTE` when the lang value uses a private-use `x-*` prefix, plus the element's text content (up to 120 chars) to help identify the correct language code.
+- **oobee-grading-text-contents** now appends a plain-language interpretation of the Flesch-Kincaid score immediately after the numeric value.

package/Dockerfile

@@ -2,13 +2,12 @@
 # Node version is v22
 FROM mcr.microsoft.com/playwright:v1.58.2-noble
 
-# Installation of packages for oobee and runner (locked versions from build log)
-RUN apt-get update && apt-get install -y \
-    git=1:2.43.0-1ubuntu7.3 \
-    git-man=1:2.43.0-1ubuntu7.3 \
-    unzip=6.0-28ubuntu4.1 \
-    zip=3.0-13ubuntu0.2 \
- && rm -rf /var/lib/apt/lists/*
+# Installation of packages for oobee
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    unzip \
+    zip && \
+    rm -rf /var/lib/apt/lists/*
  
 WORKDIR /app/oobee
 

package/dist/cli.js

@@ -147,8 +147,11 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`)
 })
     .check(argvs => {
     const scanner = String(argvs.scanner ?? '');
-    if (argvs.strategy && scanner !== ScannerTypes.WEBSITE && scanner !== ScannerTypes.CUSTOM) {
-        throw new Error('-s or --strategy is only available in website and custom flow scans.');
+    if (argvs.strategy && scanner !== ScannerTypes.WEBSITE && scanner !== ScannerTypes.CUSTOM && scanner !== ScannerTypes.INTELLIGENT && scanner !== ScannerTypes.SITEMAP) {
+        throw new Error('-s or --strategy is only available in website, custom flow, intelligent, and sitemap scans.');
+    }
+    if (argvs.strategy === 'ignore' && scanner !== ScannerTypes.SITEMAP) {
+        throw new Error('-s ignore is only available for sitemap scans.');
     }
     return true;
 })
@@ -161,13 +164,19 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`)
     return duration;
 })
     .check(argvs => {
-    if (argvs.scanner !== ScannerTypes.WEBSITE && argvs.strategy) {
-        throw new Error('-s or --strategy is only available in website scans.');
+    if (argvs.scanner !== ScannerTypes.WEBSITE && argvs.scanner !== ScannerTypes.CUSTOM && argvs.scanner !== ScannerTypes.INTELLIGENT && argvs.scanner !== ScannerTypes.SITEMAP && argvs.strategy) {
+        throw new Error('-s or --strategy is only available in website, custom flow, intelligent, and sitemap scans.');
+    }
+    if (argvs.strategy === 'ignore' && argvs.scanner !== ScannerTypes.SITEMAP) {
+        throw new Error('-s ignore is only available for sitemap scans.');
     }
     return true;
 })
     .conflicts('d', 'w')
     .parse();
+if (!options.strategy) {
+    options.strategy = options.scanner === ScannerTypes.SITEMAP ? 'ignore' : 'same-domain';
+}
 const scanInit = async (argvs) => {
     const updatedArgvs = { ...argvs };
     // Cannot use data.browser and data.isHeadless as the connectivity check comes first before prepareData
@@ -187,7 +196,11 @@ const scanInit = async (argvs) => {
     if (res.httpStatus)
         consoleLogger.info(`Connectivity Check HTTP Response Code: ${res.httpStatus}`);
     if (res.status === statuses.success.code) {
-        data.url = res.url;
+        // Custom flow should continue from the user-provided entry URL so auth redirects
+        // do not replace the original domain used for overlay gating and navigation.
+        if (data.type !== ScannerTypes.CUSTOM) {
+            data.url = res.url;
+        }
         if (process.env.OOBEE_VALIDATE_URL) {
             consoleLogger.info('Url is valid');
             cleanUpAndExit(0, data.randomToken);

package/dist/combine.js

@@ -95,6 +95,8 @@ const combineRun = async (details, deviceToScan) => {
                 blacklistedPatterns,
                 includeScreenshots,
                 extraHTTPHeaders,
+                strategy,
+                userUrl: url,
                 scanDuration,
             });
             urlsCrawledObj = sitemapResult.urlsCrawled;
@@ -115,6 +117,7 @@ const combineRun = async (details, deviceToScan) => {
                 includeScreenshots,
                 extraHTTPHeaders,
                 scanDuration,
+                ruleset,
             });
             if (localFileResult) {
                 if ('urlsCrawled' in localFileResult) {

package/dist/constants/cliFunctions.js

@@ -147,8 +147,8 @@ export const cliOptions = {
     },
     s: {
         alias: 'strategy',
-        describe: 'Crawls up to general (same parent) domains, or only specific hostname. Defaults to "same-domain".',
-        choices: ['same-domain', 'same-hostname'],
+        describe: 'Crawls up to general (same parent) domains, or only specific hostname. Use "ignore" to disable URL filtering (default for sitemap scans). Defaults to "same-domain".',
+        choices: ['same-domain', 'same-hostname', 'ignore'],
         requiresArg: true,
         demandOption: false,
     },

package/dist/constants/common.js

@@ -26,7 +26,7 @@ formDataFields,
 ScannerTypes, BrowserTypes, FileTypes, getEnumKey, } from './constants.js';
 import { consoleLogger } from '../logs.js';
 import { isUrlPdf } from '../crawlers/commonCrawlerFunc.js';
-import { cleanUpAndExit, randomThreeDigitNumberString, register } from '../utils.js';
+import { cleanUpAndExit, isFollowStrategy, randomThreeDigitNumberString, register } from '../utils.js';
 import { getProxyInfo, proxyInfoToResolution } from '../proxyService.js';
 // validateDirPath validates a provided directory path
 // returns null if no error
@@ -592,7 +592,9 @@ export const prepareData = async (argv) => {
         viewportWidth,
         playwrightDeviceDetailsObject,
         maxRequestsPerCrawl: maxpages || constants.maxRequestsPerCrawl,
-        strategy: strategy === 'same-hostname' ? EnqueueStrategy.SameHostname : EnqueueStrategy.SameDomain,
+        strategy: strategy === 'same-hostname' ? EnqueueStrategy.SameHostname
+            : strategy === 'ignore' ? EnqueueStrategy.All
+                : EnqueueStrategy.SameDomain,
         isLocalFileScan,
         browser: browserToRun,
         nameEmail,
@@ -637,6 +639,10 @@ export const getUrlsFromRobotsTxt = async (url, browserToRun, userDataDirectory,
     let shouldCapture = false;
     const disallowedUrls = [];
     const allowedUrls = [];
+    // Returns 1–2 minimatch glob patterns for a single robots.txt path pattern.
+    // Two patterns are returned for bare paths (no trailing wildcard) so that
+    // both the exact URL and all child paths are blocked, matching robots.txt
+    // prefix semantics.
     const sanitisePattern = (pattern) => {
         const directoryRegex = /^\/(?:[^?#/]+\/)*[^?#]*$/;
         const subdirWildcardRegex = /\/\*\//g;
@@ -644,18 +650,29 @@ export const getUrlsFromRobotsTxt = async (url, browserToRun, userDataDirectory,
         if (subdirWildcardRegex.test(pattern)) {
             pattern = pattern.replace(subdirWildcardRegex, '/**/');
         }
+        // Query-string patterns (e.g. /faq?faqItem= or /faq/?faq&faqItem=):
+        // '?' is the query separator in robots.txt but a single-char wildcard in
+        // minimatch. Escape it to a literal match and append '*' so any query
+        // value after the stated prefix is also blocked.
+        if (pattern.includes('?')) {
+            return [domain + pattern.replace('?', '\\?') + '*'];
+        }
         if (pattern.match(directoryRegex) && !pattern.match(filePathRegex)) {
             if (pattern.endsWith('*')) {
-                pattern = pattern.concat('*');
+                // e.g. /ebook/* → /ebook/** (already covers all children)
+                return [domain + pattern.concat('*')];
             }
             else {
-                if (!pattern.endsWith('/'))
-                    pattern = pattern.concat('/');
-                pattern = pattern.concat('**');
+                // Bare path (e.g. /subscription/unsubscribe): robots.txt blocks the
+                // exact URL *and* every descendant. minimatch's '/**' glob does not
+                // match the bare path itself (no trailing slash), so we emit both the
+                // exact-path pattern and a children glob.
+                const base = domain + pattern;
+                const children = domain + (pattern.endsWith('/') ? pattern : pattern + '/') + '**';
+                return [base, children];
             }
         }
-        const final = domain.concat(pattern);
-        return final;
+        return [domain + pattern];
     };
     for (const line of lines) {
         if (line.toLowerCase().startsWith('user-agent: *')) {
@@ -667,15 +684,13 @@ export const getUrlsFromRobotsTxt = async (url, browserToRun, userDataDirectory,
         else if (shouldCapture && line.toLowerCase().startsWith('disallow:')) {
             let disallowed = line.substring('disallow: '.length).trim();
             if (disallowed) {
-                disallowed = sanitisePattern(disallowed);
-                disallowedUrls.push(disallowed);
+                disallowedUrls.push(...sanitisePattern(disallowed));
             }
         }
         else if (shouldCapture && line.toLowerCase().startsWith('allow:')) {
             let allowed = line.substring('allow: '.length).trim();
             if (allowed) {
-                allowed = sanitisePattern(allowed);
-                allowedUrls.push(allowed);
+                allowedUrls.push(...sanitisePattern(allowed));
             }
         }
     }
@@ -726,6 +741,31 @@ const getRobotsTxtViaPlaywright = async (robotsUrl, browser, userDataDirectory,
         }
     }
 };
+export const getSitemapsFromRobotsTxt = async (url, browser, userDataDirectory, extraHTTPHeaders) => {
+    const domain = new URL(url).origin;
+    const robotsUrl = domain.concat('/robots.txt');
+    let robotsTxt;
+    try {
+        robotsTxt = await getRobotsTxtViaPlaywright(robotsUrl, browser, userDataDirectory, extraHTTPHeaders);
+    }
+    catch (e) {
+        consoleLogger.info(`Unable to fetch robots.txt from ${robotsUrl} for sitemap discovery`);
+        return [];
+    }
+    if (!robotsTxt)
+        return [];
+    const sitemaps = [];
+    const lines = robotsTxt.split(/\r?\n/);
+    for (const line of lines) {
+        if (line.toLowerCase().startsWith('sitemap:')) {
+            const sitemapUrl = line.substring('sitemap:'.length).trim();
+            if (sitemapUrl) {
+                sitemaps.push(sitemapUrl);
+            }
+        }
+    }
+    return sitemaps;
+};
 export const isDisallowedInRobotsTxt = (url) => {
     if (!constants.robotsTxtUrls)
         return;
@@ -744,7 +784,7 @@ export const isDisallowedInRobotsTxt = (url) => {
     }
     return false;
 };
-export const getLinksFromSitemap = async (sitemapUrl, maxLinksCount, browser, userDataDirectory, userUrlInput, isIntelligent, extraHTTPHeaders) => {
+export const getLinksFromSitemap = async (sitemapUrl, maxLinksCount, browser, userDataDirectory, userUrlInput, isIntelligent, extraHTTPHeaders, strategy = EnqueueStrategy.All, userUrl = userUrlInput) => {
     const scannedSitemaps = new Set();
     const urls = {}; // dictionary of requests to urls to be scanned
     const isLimitReached = () => Object.keys(urls).length >= maxLinksCount;
@@ -753,6 +793,8 @@ export const getLinksFromSitemap = async (sitemapUrl, maxLinksCount, browser, us
             return;
         if (isDisallowedInRobotsTxt(url))
             return;
+        if (!isFilePath(userUrl) && !isFollowStrategy(url, userUrl, strategy))
+            return;
         url = convertPathToLocalFile(url);
         let request;
         try {