@govtechsg/oobee 0.10.83 → 0.10.84

package/dist/proxyService.js

@@ -281,7 +281,7 @@ function parseWindowsRegistry() {
     if (enabledManual && v.ProxyServer) {
         const s = v.ProxyServer.trim();
         if (s.includes('=')) {
-            for (const part of s.split(';')) {
+            for (const part of s.split(/[,;]/)) {
                 const [proto, addr] = part.split('=');
                 if (!proto || !addr)
                     continue;
@@ -315,15 +315,100 @@ function parseWindowsRegistry() {
 /* ============================ Public API ============================ */
 export function getProxyInfo() {
     const plat = os.platform();
+    let info;
     if (plat === 'win32')
-        return parseEnvProxyCommon() || parseWindowsRegistry();
-    if (plat === 'darwin')
-        return parseEnvProxyCommon() || parseMacScutil();
-    return parseEnvProxyCommon(); // Linux/others
+        info = parseEnvProxyCommon() || parseWindowsRegistry();
+    else if (plat === 'darwin')
+        info = parseEnvProxyCommon() || parseMacScutil();
+    else
+        info = parseEnvProxyCommon(); // Linux/others
+    // Apply INCLUDE_PROXY env: semicolon-separated domain globs that SHOULD use the proxy
+    const includeProxy = process.env.INCLUDE_PROXY;
+    if (includeProxy && info) {
+        const patterns = includeProxy
+            .split(/[,;]/)
+            .map(s => s.trim())
+            .filter(Boolean);
+        if (patterns.length > 0) {
+            // INCLUDE_PROXY and NO_PROXY are mutually exclusive; INCLUDE_PROXY takes precedence
+            const noProxy = process.env.NO_PROXY || process.env.no_proxy;
+            if (noProxy || info.bypassList) {
+                console.warn('INCLUDE_PROXY is set — ignoring NO_PROXY / bypass list. ' +
+                    'These two settings cannot be mixed; INCLUDE_PROXY takes precedence.');
+                info.bypassList = undefined;
+            }
+            info.includeList = patterns;
+        }
+    }
+    return info;
+}
+/**
+ * Convert a glob-style domain pattern (e.g. *.example.com) to a regex source string.
+ */
+function domainPatternToRegex(pattern) {
+    // Escape dots, replace * with [^.]* or .* depending on position
+    // *.example.com → match any subdomain of example.com
+    let escaped = pattern
+        .replace(/[.+^${}()|[\]\\]/g, '\\$&') // escape regex specials except *
+        .replace(/\*/g, '.*'); // convert glob * to regex .*
+    return `^${escaped}$`;
+}
+/**
+ * Build a PAC script that routes only includeList domains through the given proxy,
+ * and returns DIRECT for everything else.
+ */
+function buildIncludeOnlyPac(proxyServer, includeList) {
+    // proxyServer is like "http://host:port" or "socks5://host:port"
+    let pacProxy;
+    if (proxyServer.startsWith('socks5://')) {
+        pacProxy = `SOCKS5 ${proxyServer.replace('socks5://', '')}`;
+    }
+    else if (proxyServer.startsWith('socks://') || proxyServer.startsWith('socks4://')) {
+        pacProxy = `SOCKS ${proxyServer.replace(/^socks[4]?:\/\//, '')}`;
+    }
+    else {
+        // http:// or https:// → PROXY host:port
+        pacProxy = `PROXY ${proxyServer.replace(/^https?:\/\//, '')}`;
+    }
+    // Build JS conditions using shExpMatch (built-in PAC function that supports glob patterns)
+    const conditions = includeList
+        .map(pattern => `shExpMatch(host, ${JSON.stringify(pattern)})`)
+        .join(' || ');
+    const pac = [
+        'function FindProxyForURL(url, host) {',
+        `  if (${conditions}) {`,
+        `    return ${JSON.stringify(pacProxy)};`,
+        '  }',
+        '  return "DIRECT";',
+        '}',
+    ].join('\n');
+    return pac;
 }
 export function proxyInfoToResolution(info) {
     if (!info)
         return { kind: 'none' };
+    // If INCLUDE_PROXY is set, generate a PAC that only proxies the listed domains
+    if (info.includeList && info.includeList.length > 0) {
+        // Determine the proxy server string
+        let proxyServer;
+        if (info.http)
+            proxyServer = `http://${info.http}`;
+        else if (info.https)
+            proxyServer = `http://${info.https}`;
+        else if (info.socks)
+            proxyServer = `socks5://${info.socks}`;
+        if (proxyServer) {
+            // If credentials exist, embed them for the manual proxy auth
+            // PAC scripts themselves don't carry auth, but Playwright's proxy option can
+            // We use PAC for routing, and if creds are needed, Playwright will prompt or
+            // we set them via the proxy option. Unfortunately Playwright doesn't allow
+            // proxy.username with a PAC, so we embed in the launch arg and rely on
+            // Chromium's built-in auth handler for proxy auth challenges.
+            const pac = buildIncludeOnlyPac(proxyServer, info.includeList);
+            const pacDataUrl = `data:application/x-ns-proxy-autoconfig;base64,${Buffer.from(pac).toString('base64')}`;
+            return { kind: 'pac', pacUrl: pacDataUrl, bypass: info.bypassList };
+        }
+    }
     // Prefer manual proxies first (these work with Playwright's proxy option)
     if (info.http) {
         return { kind: 'manual', settings: {

package/dist/utils.js

@@ -3,6 +3,7 @@ import os from 'os';
 import fs from 'fs-extra';
 import axe from 'axe-core';
 import { v4 as uuidv4 } from 'uuid';
+import { getDomain } from 'tldts';
 import constants, { destinationPath, getIntermediateScreenshotsPath, } from './constants/constants.js';
 import { consoleLogger, errorsTxtPath } from './logs.js';
 import { getAxeConfiguration } from './crawlers/custom/getAxeConfiguration.js';
@@ -852,14 +853,26 @@ export const randomThreeDigitNumberString = () => {
     return String(threeDigitNumber);
 };
 export const isFollowStrategy = (link1, link2, rule) => {
-    const parsedLink1 = new URL(link1);
-    const parsedLink2 = new URL(link2);
-    if (rule === 'same-domain') {
-        const link1Domain = parsedLink1.hostname.split('.').slice(-2).join('.');
-        const link2Domain = parsedLink2.hostname.split('.').slice(-2).join('.');
-        return link1Domain === link2Domain;
+    try {
+        const parsedLink1 = new URL(link1);
+        const parsedLink2 = new URL(link2);
+        if (rule === 'all') {
+            return true;
+        }
+        if (rule === 'same-origin') {
+            return parsedLink1.origin === parsedLink2.origin;
+        }
+        if (rule === 'same-domain') {
+            const link1Domain = getDomain(parsedLink1.hostname, { allowPrivateDomains: true }) || parsedLink1.hostname;
+            const link2Domain = getDomain(parsedLink2.hostname, { allowPrivateDomains: true }) || parsedLink2.hostname;
+            return link1Domain.toLowerCase() === link2Domain.toLowerCase();
+        }
+        // default: same-hostname
+        return parsedLink1.hostname === parsedLink2.hostname;
+    }
+    catch {
+        return false;
     }
-    return parsedLink1.hostname === parsedLink2.hostname;
 };
 export const retryFunction = async (func, maxAttempt) => {
     let attemptCount = 0;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.83",
+  "version": "0.10.84",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "bin": {
@@ -19,12 +19,12 @@
     "cheerio": "^1.0.0-rc.12",
     "crawlee": "^3.13.10",
     "ejs": "^3.1.9",
-    "file-type": "^19.5.0",
+    "file-type": "^21.3.3",
     "fs-extra": "^11.2.0",
     "glob": "^13.0.6",
     "https": "^1.0.0",
     "inquirer": "^9.2.12",
-    "jsdom": "^21.1.2",
+    "jsdom": "^29.0.0",
     "jszip": "^3.10.1",
     "lodash": "^4.17.21",
     "mime": "^4.0.7",
@@ -36,6 +36,7 @@
     "print-message": "^3.0.1",
     "safe-regex": "^2.1.1",
     "text-readability": "^1.1.0",
+    "tldts": "^7.0.26",
     "typescript": "^5.4.5",
     "url": "^0.11.3",
     "uuid": "^11.0.3",
@@ -87,9 +88,8 @@
     "minimatch": "^10.2.4",
     "brace-expansion": "^5.0.4",
     "glob": "^13.0.6",
-    "jsdom": {
-      "@tootallnate/once": "^3.0.1"
-    }
+    "flatted": "^3.4.1",
+    "file-type": "^21.3.3"
   },
   "optionalDependencies": {
     "@napi-rs/canvas-darwin-arm64": "^0.1.53",

package/src/constants/common.ts

@@ -446,7 +446,19 @@ const checkUrlConnectivityWithBrowser = async (
 
     if (!response) throw new Error('No response from navigation');
 
-    // We use the response headers from the navigation we just performed.
+    // Wait briefly for JS/meta-refresh redirects to settle before reading the final URL.
+    // Server-side redirects are already reflected after goto(), but client-side redirects
+    // (e.g. domain.tld -> www.domain.tld via JS or meta-refresh) need extra time.
+    try {
+      await Promise.race([
+        page.waitForURL(currentUrl => currentUrl !== url, { timeout: 5000 }),
+        new Promise(resolve => setTimeout(resolve, 1000)), // minimum settle time
+      ]);
+    } catch {
+      // No redirect happened within the window — that's fine, continue with current URL
+    }
+
+    // Re-read page.url() AFTER potential client-side redirects have resolved
     const finalUrl = page.url();
     const finalStatus = response.status();
     const headers = response.headers();