@govtechsg/oobee 0.10.85 → 0.10.86

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.85",
+  "version": "0.10.86",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "bin": {
@@ -86,7 +86,7 @@
     "fast-xml-parser": ">=5.3.8",
     "js-yaml": "^4.1.1",
     "minimatch": "^10.2.4",
-    "brace-expansion": "^5.0.4",
+    "brace-expansion": "^5.0.5",
     "glob": "^13.0.6",
     "flatted": "^3.4.1",
     "file-type": "^21.3.3"

package/src/combine.ts

@@ -135,6 +135,8 @@ const combineRun = async (details: Data, deviceToScan: string) => {
       const res = await runCustom(
         url,
         randomToken,
+        browser,
+        userDataDirectory,
         viewportSettings,
         blacklistedPatterns,
         includeScreenshots,
@@ -328,4 +330,4 @@ const combineRun = async (details: Data, deviceToScan: string) => {
   }
 };
 
-export default combineRun;
+export default combineRun;

package/src/constants/common.ts

@@ -213,6 +213,15 @@ export const validateXML = (content: string): { isValid: boolean; parsedContent:
   return { isValid, parsedContent };
 };
 
+export const validateTXT = (content: string): { isValid: boolean } => {
+  // Strip HTML tags first — browsers wrap .txt files in HTML when fetched via Playwright
+  const plainText = content.replace(/<[^>]+>/g, '\n');
+  const lines = plainText.split(/\r?\n/).map(l => l.trim()).filter(l => l.length > 0);
+  // Allow http, https and relative paths (starting with /) for txt sitemaps, as some sitemaps use relative paths and some txt sitemaps are fetched as HTML by Playwright
+  const urlPattern = /^(https?:\/\/|\/)[^\s]+$/i;
+  return { isValid: lines.some(line => urlPattern.test(line)) };
+};
+
 export const isSkippedUrl = (pageUrl: string, whitelistedDomains: string[]) => {
   const matched =
     whitelistedDomains.filter(p => {
@@ -541,14 +550,13 @@ export const isSitemapContent = (content: string) => {
 
   const regexForHtml = new RegExp('<(?:!doctype html|html|head|body)+?>', 'gmi');
   const regexForXmlSitemap = new RegExp('<(?:urlset|feed|rss)+?.*>', 'gmi');
-  const regexForUrl = new RegExp('^.*(http|https):/{2}.*$', 'gmi');
-
   if (content.match(regexForHtml) && content.match(regexForXmlSitemap)) {
     // is an XML sitemap wrapped in a HTML document
     return true;
   }
-  if (!content.match(regexForHtml) && content.match(regexForUrl)) {
-    // treat this as a txt sitemap where all URLs will be extracted for crawling
+  const { isValid: isTxtSitemap } = validateTXT(content);
+  if (isTxtSitemap) {
+    // treat this as a txt sitemap (plain text or browser-wrapped with HTML)
     return true;
   }
   // is HTML webpage
@@ -1924,14 +1932,16 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => {
   const channel = browser || undefined;
 
   const resolution = proxyInfoToResolution(cacheProxyInfo);
+  const shouldIgnoreMuteAudio =
+    process.env.OOBEE_PLAYWRIGHT_IGNORE_DEFAULT_ARGS === '--mute-audio';
 
   // Start with your base args and sanitise
   const finalArgs = [...constants.launchOptionsArgs].filter(
-  arg =>
-    !arg.startsWith('--headless') &&
-    !arg.startsWith('--user-agent=') &&
-    arg !== '--mute-audio' &&
-    !(browser === BrowserTypes.CHROME && arg === '--edge-skip-compat-layer-relaunch'),
+    arg =>
+      !arg.startsWith('--headless') &&
+      !arg.startsWith('--user-agent=') &&
+      arg !== '--mute-audio' &&
+      !(browser === BrowserTypes.CHROME && arg === '--edge-skip-compat-layer-relaunch'),
   );
 
   // Headless flags (unchanged)
@@ -1956,7 +1966,9 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => {
   }
 
   const options: LaunchOptions = {
-    ignoreDefaultArgs: ['--use-mock-keychain'],
+    ignoreDefaultArgs: shouldIgnoreMuteAudio
+      ? ['--use-mock-keychain', '--mute-audio']
+      : ['--use-mock-keychain'],
     args: finalArgs,
     headless: process.env.CRAWLEE_HEADLESS === '1',
     ...(channel && { channel }),