@govtechsg/oobee 0.10.69 → 0.10.70

package/README.md

@@ -571,7 +571,7 @@ The following URL and file validation error codes are provided to troubleshoot t
 | 11   | invalidUrl           | Invalid URL. Please check and try again.                                | • Ensure the URL starts with `http://` or `https://`.<br>• Check for typos in the URL. |
 | 12   | cannotBeResolved     | URL cannot be accessed. Please verify whether the website exists.     | • Confirm the domain name is correct.<br>• Check DNS resolution with `ping` or `nslookup`.<br>• Ensure the site is publicly accessible (not behind VPN/firewall). |
 | 14   | systemError          | Something went wrong when verifying the URL. Please try again in a few minutes. If this issue persists, please contact the Oobee team.          | • Retry after a few minutes.<br>• Check internet connection.<br>• If persistent, report as a system issue. |
-| 15   | notASitemap          | Invalid sitemap URL format. Please enter a valid sitemap URL ending with .XML e.g. https://www.example.com/sitemap.xml.                                                | • Ensure the URL points to a valid XML sitemap.<br>• View [Examples of sitemaps sitemaps.org - Protocol](https://www.sitemaps.org/protocol.html)<br>• Test the URL in a browser to confirm it returns XML. |
+| 15   | notASitemap          | Invalid sitemap URL format. Please enter a valid sitemap URL ending with .XML or .TXT e.g. https://www.example.com/sitemap.xml.                                                | • Ensure the URL points to a valid XML sitemap.<br>• View [Examples of sitemaps sitemaps.org - Protocol](https://www.sitemaps.org/protocol.html)<br>• Test the URL in a browser to confirm it returns XML. |
 | 16   | unauthorised         | Login required. Please enter your credentials and try again.                                       | • Check if the site requires username/password.<br>• Provide credentials in Oobee if supported. |
 | 17   | browserError         | Incompatible browser. Please ensure you are using Chrome or Edge browser. | • Install the latest version of Chrome or Edge.|
 | 18   | sslProtocolError     | SSL certificate  error. Please check the SSL configuration of your website and try again. | • Verify SSL certificate validity (not expired, issued by trusted CA).<br>• Check for mismatched TLS versions or cipher issues.<br>• Use an SSL checker tool (e.g., Qualys SSL Labs). |

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.69",
+  "version": "0.10.70",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "dependencies": {

package/src/constants/constants.ts

@@ -397,7 +397,7 @@ const urlCheckStatuses = {
     message: 'Provided URL cannot be accessed. Server responded with code ', // append it with the response code received,
   },
   systemError: { code: 14, message: 'Something went wrong when verifying the URL. Please try again in a few minutes. If this issue persists, please contact the Oobee team.'},
-  notASitemap: { code: 15, message: 'Invalid sitemap URL format. Please enter a valid sitemap URL ending with .XML e.g. https://www.example.com/sitemap.xml.' },
+  notASitemap: { code: 15, message: 'Invalid sitemap URL format. Please enter a valid sitemap URL ending with .XML or .TXT e.g. https://www.example.com/sitemap.xml.' },
   unauthorised: { code: 16, message: 'Login required. Please enter your credentials and try again.' },
   // browserError means engine could not find a browser to run the scan
   browserError: {

package/src/crawlers/crawlLocalFile.ts

@@ -198,6 +198,10 @@ export const crawlLocalFile = async ({
     results.actualUrl = actualUrl;
 
     await dataset.pushData(results);
+    
+    // Ensure proper cleanup of browser context before PDF generation
+    await browserContext.close().catch(() => {});
+    
   } else {
 
     const pdfFileName = path.basename(url);

package/src/crawlers/guards/urlGuard.ts

@@ -0,0 +1,77 @@
+const ALLOWED_PROTOCOLS = new Set(['http:', 'https:']);
+
+export function addUrlGuardScript(context, opts = {}) {
+  const { fallbackUrl }: any = opts;
+
+  const lastAllowedUrlByPage = new WeakMap();
+
+  const attachGuardsToPage = (page) => {
+    if (!lastAllowedUrlByPage.has(page) && fallbackUrl) {
+      lastAllowedUrlByPage.set(page, String(fallbackUrl));
+    }
+
+    page.addInitScript(() => {
+      const isAllowedProtocol = (value) => {
+        try {
+          const s = value instanceof URL ? value.toString() : String(value);
+          const protocol = new URL(s, window.location.href).protocol;
+          return protocol === 'http:' || protocol === 'https:';
+        } catch {
+          return false;
+        }
+      };
+
+      const win = window;
+
+      const openOriginal = win.open;
+      win.open = function (targetUrl, ...args) {
+        if (!isAllowedProtocol(targetUrl)) return null;
+        return openOriginal.call(this, targetUrl, ...args);
+      };
+
+      const assignOriginal  = win.location.assign.bind(win.location);
+      const replaceOriginal = win.location.replace.bind(win.location);
+
+      win.location.assign  = (nextUrl) => { if (isAllowedProtocol(nextUrl)) assignOriginal(nextUrl); };
+      win.location.replace = (nextUrl) => { if (isAllowedProtocol(nextUrl)) replaceOriginal(nextUrl); };
+
+      Object.defineProperty(win.location, 'href', {
+        get() { return String(win.location.toString()); },
+        set(nextUrl) { if (isAllowedProtocol(nextUrl)) assignOriginal(nextUrl); },
+      });
+    });
+
+    const restoreToSafeUrl = async (page, attemptedUrl) => {
+      try {
+        const safeUrl = lastAllowedUrlByPage.get(page) || fallbackUrl || 'about:blank';
+        await page.goto(safeUrl, { waitUntil: 'domcontentloaded' });
+      } catch {
+        // page might be closing; ignore
+      }
+    };
+
+    page.on('framenavigated', async (frame) => {
+      if (frame !== page.mainFrame()) return;
+
+      const urlStr = frame.url();
+      let urlObj;
+      try {
+      	urlObj = new URL(urlStr);
+      } catch {
+      	return restoreToSafeUrl(page, urlStr);
+      }
+
+      if (ALLOWED_PROTOCOLS.has(urlObj.protocol)) {
+        lastAllowedUrlByPage.set(page, urlObj.toString());
+        return;
+      }
+      await restoreToSafeUrl(page, urlStr);
+    });
+  };
+
+  // Guard existing and future pages
+  for (const page of context.pages()) attachGuardsToPage(page);
+  context.on('page', attachGuardsToPage);
+}
+
+export default addUrlGuardScript;

package/src/crawlers/runCustom.ts

@@ -10,6 +10,7 @@ import constants, {
 import { DEBUG, initNewPage, log } from './custom/utils.js';
 import { guiInfoLog } from '../logs.js';
 import { ViewportSettingsClass } from '../combine.js';
+import { addUrlGuardScript } from './guards/urlGuard.js';
 
 // Export of classes
 
@@ -84,6 +85,8 @@ const runCustom = async (
 
     register(context);
 
+    addUrlGuardScript(context, { fallbackUrl: url });
+
     // Detection of new page
     context.on('page', async newPage => {
       await initNewPage(newPage, pageClosePromises, processPageParams, pagesDict);