@govtechsg/oobee 0.10.84 → 0.10.86

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.84",
+  "version": "0.10.86",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "bin": {
@@ -36,7 +36,7 @@
     "print-message": "^3.0.1",
     "safe-regex": "^2.1.1",
     "text-readability": "^1.1.0",
-    "tldts": "^7.0.26",
+    "tldts": "^7.0.27",
     "typescript": "^5.4.5",
     "url": "^0.11.3",
     "uuid": "^11.0.3",
@@ -86,7 +86,7 @@
     "fast-xml-parser": ">=5.3.8",
     "js-yaml": "^4.1.1",
     "minimatch": "^10.2.4",
-    "brace-expansion": "^5.0.4",
+    "brace-expansion": "^5.0.5",
     "glob": "^13.0.6",
     "flatted": "^3.4.1",
     "file-type": "^21.3.3"

package/src/cli.ts

@@ -5,7 +5,13 @@ import printMessage from 'print-message';
 import { devices } from 'playwright';
 import { fileURLToPath } from 'url';
 import path from 'path';
-import { setHeadlessMode, getVersion, getStoragePath, listenForCleanUp, cleanUpAndExit } from './utils.js';
+import {
+  setHeadlessMode,
+  getVersion,
+  getStoragePath,
+  listenForCleanUp,
+  cleanUpAndExit,
+} from './utils.js';
 import {
   checkUrl,
   prepareData,
@@ -185,12 +191,14 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
     return true;
   })
   .check(argvs => {
-    if (argvs.scanner !== ScannerTypes.WEBSITE && argvs.strategy) {
-      throw new Error('-s or --strategy is only available in website scans.');
+    const scanner = String(argvs.scanner ?? '');
+
+    if (argvs.strategy && scanner !== ScannerTypes.WEBSITE && scanner !== ScannerTypes.CUSTOM) {
+      throw new Error('-s or --strategy is only available in website and custom flow scans.');
     }
     return true;
   })
-  .coerce('l', (option) => {
+  .coerce('l', option => {
     const duration = Number(option);
     if (isNaN(duration) || duration < 0) {
       printMessage(
@@ -202,8 +210,8 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
     return duration;
   })
   .check(argvs => {
-    if (argvs.scanner === ScannerTypes.CUSTOM && typeof argvs.scanDuration === 'number' && argvs.scanDuration > 0) {
-      throw new Error('-l or --scanDuration is not allowed for custom flow scans.');
+    if (argvs.scanner !== ScannerTypes.WEBSITE && argvs.strategy) {
+      throw new Error('-s or --strategy is only available in website scans.');
     }
     return true;
   })
@@ -235,10 +243,11 @@ const scanInit = async (argvs: Answers): Promise<string> => {
     data.userDataDirectory,
     data.playwrightDeviceDetailsObject,
     data.extraHTTPHeaders,
-    data.fileTypes
+    data.fileTypes,
   );
 
-  if (res.httpStatus) consoleLogger.info(`Connectivity Check HTTP Response Code: ${res.httpStatus}`);
+  if (res.httpStatus)
+    consoleLogger.info(`Connectivity Check HTTP Response Code: ${res.httpStatus}`);
 
   if (res.status === statuses.success.code) {
     data.url = res.url;
@@ -267,15 +276,11 @@ const scanInit = async (argvs: Answers): Promise<string> => {
     }
   }
 
-  const screenToScan = getScreenToScan(
-    data.deviceChosen,
-    data.customDevice,
-    data.viewportWidth,
-  );
+  const screenToScan = getScreenToScan(data.deviceChosen, data.customDevice, data.viewportWidth);
 
   printMessage([`Oobee version: ${appVersion}`, 'Starting scan...'], messageOptions);
-  consoleLogger.info(`Oobee version: ${appVersion}`); 
-  
+  consoleLogger.info(`Oobee version: ${appVersion}`);
+
   await combineRun(data, screenToScan);
 
   return getStoragePath(data.randomToken);

package/src/combine.ts

@@ -135,6 +135,8 @@ const combineRun = async (details: Data, deviceToScan: string) => {
       const res = await runCustom(
         url,
         randomToken,
+        browser,
+        userDataDirectory,
         viewportSettings,
         blacklistedPatterns,
         includeScreenshots,
@@ -328,4 +330,4 @@ const combineRun = async (details: Data, deviceToScan: string) => {
   }
 };
 
-export default combineRun;
+export default combineRun;

package/src/constants/common.ts

@@ -213,6 +213,15 @@ export const validateXML = (content: string): { isValid: boolean; parsedContent:
   return { isValid, parsedContent };
 };
 
+export const validateTXT = (content: string): { isValid: boolean } => {
+  // Strip HTML tags first — browsers wrap .txt files in HTML when fetched via Playwright
+  const plainText = content.replace(/<[^>]+>/g, '\n');
+  const lines = plainText.split(/\r?\n/).map(l => l.trim()).filter(l => l.length > 0);
+  // Allow http, https and relative paths (starting with /) for txt sitemaps, as some sitemaps use relative paths and some txt sitemaps are fetched as HTML by Playwright
+  const urlPattern = /^(https?:\/\/|\/)[^\s]+$/i;
+  return { isValid: lines.some(line => urlPattern.test(line)) };
+};
+
 export const isSkippedUrl = (pageUrl: string, whitelistedDomains: string[]) => {
   const matched =
     whitelistedDomains.filter(p => {
@@ -541,14 +550,13 @@ export const isSitemapContent = (content: string) => {
 
   const regexForHtml = new RegExp('<(?:!doctype html|html|head|body)+?>', 'gmi');
   const regexForXmlSitemap = new RegExp('<(?:urlset|feed|rss)+?.*>', 'gmi');
-  const regexForUrl = new RegExp('^.*(http|https):/{2}.*$', 'gmi');
-
   if (content.match(regexForHtml) && content.match(regexForXmlSitemap)) {
     // is an XML sitemap wrapped in a HTML document
     return true;
   }
-  if (!content.match(regexForHtml) && content.match(regexForUrl)) {
-    // treat this as a txt sitemap where all URLs will be extracted for crawling
+  const { isValid: isTxtSitemap } = validateTXT(content);
+  if (isTxtSitemap) {
+    // treat this as a txt sitemap (plain text or browser-wrapped with HTML)
     return true;
   }
   // is HTML webpage
@@ -1924,14 +1932,16 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => {
   const channel = browser || undefined;
 
   const resolution = proxyInfoToResolution(cacheProxyInfo);
+  const shouldIgnoreMuteAudio =
+    process.env.OOBEE_PLAYWRIGHT_IGNORE_DEFAULT_ARGS === '--mute-audio';
 
   // Start with your base args and sanitise
   const finalArgs = [...constants.launchOptionsArgs].filter(
-  arg =>
-    !arg.startsWith('--headless') &&
-    !arg.startsWith('--user-agent=') &&
-    arg !== '--mute-audio' &&
-    !(browser === BrowserTypes.CHROME && arg === '--edge-skip-compat-layer-relaunch'),
+    arg =>
+      !arg.startsWith('--headless') &&
+      !arg.startsWith('--user-agent=') &&
+      arg !== '--mute-audio' &&
+      !(browser === BrowserTypes.CHROME && arg === '--edge-skip-compat-layer-relaunch'),
   );
 
   // Headless flags (unchanged)
@@ -1956,7 +1966,9 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => {
   }
 
   const options: LaunchOptions = {
-    ignoreDefaultArgs: ['--use-mock-keychain'],
+    ignoreDefaultArgs: shouldIgnoreMuteAudio
+      ? ['--use-mock-keychain', '--mute-audio']
+      : ['--use-mock-keychain'],
     args: finalArgs,
     headless: process.env.CRAWLEE_HEADLESS === '1',
     ...(channel && { channel }),