@govtechsg/oobee 0.10.70 → 0.10.74

package/src/constants/common.ts

@@ -57,7 +57,7 @@ export const validateDirPath = (dirPath: string): string => {
   }
 };
 
- export class RES {
+export class RES {
   status: number;
   httpStatus?: number;
   url: string;
@@ -289,16 +289,15 @@ export const sanitizeUrlInput = (url: string): { isValid: boolean; url: string }
 const isAllowedContentType = (ct: string): boolean => {
   const c = (ct || '').toLowerCase();
   return (
-    c.startsWith('text/html') ||              // html
-    c.startsWith('application/xhtml+xml') ||  // xhtml
-    c.startsWith('text/plain') ||             // txt
-    c.startsWith('application/xml') ||        // xml
-    c.startsWith('text/xml')        ||        // xml (alt)
-    c.startsWith('application/pdf')           // pdf
+    c.startsWith('text/html') || // html
+    c.startsWith('application/xhtml+xml') || // xhtml
+    c.startsWith('text/plain') || // txt
+    c.startsWith('application/xml') || // xml
+    c.startsWith('text/xml') || // xml (alt)
+    c.startsWith('application/pdf') // pdf
   );
 };
 
-
 const checkUrlConnectivityWithBrowser = async (
   url: string,
   browserToRun: string,
@@ -314,9 +313,8 @@ const checkUrlConnectivityWithBrowser = async (
     return res;
   }
 
-    // STEP 1: For local file scans
+  // STEP 1: For local file scans
   let contentType = '';
-
   const protocol = new URL(url).protocol;
 
   if (protocol !== 'http:' && protocol !== 'https:') {
@@ -353,7 +351,7 @@ const checkUrlConnectivityWithBrowser = async (
   }
 
   // Ensure Accept header for non-html content fallback
-  extraHTTPHeaders['Accept'] ||= 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';
+  extraHTTPHeaders.Accept ||= 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';
 
   await initModifiedUserAgent(browserToRun, playwrightDeviceDetailsObject, clonedDataDir);
 
@@ -386,45 +384,40 @@ const checkUrlConnectivityWithBrowser = async (
       consoleLogger.info(`Unable to set download deny: ${(e as Error).message}`);
     }
 
+    // OPTIMIZATION: Block heavy visual resources (Images/Fonts/CSS)
+    // This allows the "Connectivity Check" to pass as soon as HTML is ready
+    await page.route('**/*', (route) => {
+      const type = route.request().resourceType();
+      if (['image', 'media', 'font', 'stylesheet'].includes(type)) {
+        return route.abort(); 
+      }
+      return route.continue();
+    });
+
     // STEP 2: Navigate (follows server-side redirects)
-    page.once('download', () => { 
+    page.once('download', () => {
       res.status = constants.urlCheckStatuses.notASupportedDocument.code;
       return res;
     });
     
+    // OPTIMIZATION: Wait for 'domcontentloaded' only
     const response = await page.goto(url, {
       timeout: 15000,
       waitUntil: 'domcontentloaded', // enough to get status + allow potential client redirects to kick in
     });
 
-    // Give client-side redirects (meta refresh / JS location.*) a moment
-    try {
-      await page.waitForLoadState('networkidle', { timeout: 8000 });
-    } catch {
-      consoleLogger.info('networkidle not reached; proceeding with verification GET');
-    }
+    if (!response) throw new Error('No response from navigation');
 
-    // STEP 3: Verify final URL with a GET (follows redirects)
+    // We use the response headers from the navigation we just performed.
     const finalUrl = page.url();
-    let verifyResp = response;
-    try {
-      verifyResp = await page.request.fetch(finalUrl, {
-        method: 'GET',
-        headers: extraHTTPHeaders,
-      });
-    } catch (e) {
-      consoleLogger.info(`Verification GET failed, falling back to navigation response: ${e.message}`);
-    }
-
-    // Prefer verification GET; fall back to nav response
-    const finalStatus = verifyResp?.status?.() ?? response?.status?.() ?? 0;
-    const headers = (verifyResp?.headers?.() ?? response?.headers?.()) || {};
+    const finalStatus = response.status();
+    const headers = response.headers();
     contentType = headers['content-type'] || '';
 
     if (!isAllowedContentType(contentType)) {
       res.status = constants.urlCheckStatuses.notASupportedDocument.code;
       return res;
-    } 
+    }
 
     res.httpStatus = finalStatus;
     res.url = finalUrl;
@@ -437,7 +430,9 @@ const checkUrlConnectivityWithBrowser = async (
       // Some origins 405/501 but the browser-rendered page is still reachable after client redirects.
       // As a last resort, consider DOM presence as success if we actually have a document.
       const hasDOM = await page.evaluate(() => !!document && !!document.documentElement);
-      res.status = hasDOM ? constants.urlCheckStatuses.success.code : constants.urlCheckStatuses.systemError.code;
+      res.status = hasDOM
+        ? constants.urlCheckStatuses.success.code
+        : constants.urlCheckStatuses.systemError.code;
     } else {
       res.status = constants.urlCheckStatuses.systemError.code;
     }
@@ -448,11 +443,11 @@ const checkUrlConnectivityWithBrowser = async (
     } else {
       try {
         // Try to get a stable DOM; don't fail the check if it times out
+        // Note: Since we used 'domcontentloaded' in goto, this is fast, but kept for safety/stability
         await page.waitForLoadState('domcontentloaded', { timeout: 5000 });
       } catch {}
       res.content = await page.content();
     }
-
   } catch (error) {
     if (error.message.includes('net::ERR_INVALID_AUTH_CREDENTIALS')) {
       res.status = constants.urlCheckStatuses.unauthorised.code;
@@ -514,32 +509,30 @@ export const checkUrl = async (
   clonedDataDir: string,
   playwrightDeviceDetailsObject: DeviceDescriptor,
   extraHTTPHeaders: Record<string, string>,
-  fileTypes: FileTypes
+  fileTypes: FileTypes,
 ) => {
-
   const res = await checkUrlConnectivityWithBrowser(
-      url,
-      browser,
-      clonedDataDir,
-      playwrightDeviceDetailsObject,
-      extraHTTPHeaders,
+    url,
+    browser,
+    clonedDataDir,
+    playwrightDeviceDetailsObject,
+    extraHTTPHeaders,
   );
 
   // If response is 200 (meaning no other code was set earlier)
   if (res.status === constants.urlCheckStatuses.success.code) {
-
     // Check if document is pdf type
     const isPdf = isPdfContent(res.content);
 
     // Check if only HTML document is allowed to be scanned
     if (fileTypes === FileTypes.HtmlOnly && isPdf) {
       res.status = constants.urlCheckStatuses.notASupportedDocument.code;
-    
-    // Check if only PDF document is allowed to be scanned
+
+      // Check if only PDF document is allowed to be scanned
     } else if (fileTypes === FileTypes.PdfOnly && !isPdf) {
       res.status = constants.urlCheckStatuses.notAPdf.code;
 
-    // Check if sitemap is expected
+      // Check if sitemap is expected
     } else if (scanner === ScannerTypes.SITEMAP) {
       const isSitemap = isSitemapContent(res.content);
 
@@ -550,7 +543,7 @@ export const checkUrl = async (
 
     // else proceed as normal
   }
-  
+
   return res;
 };
 
@@ -606,7 +599,7 @@ export const prepareData = async (argv: Answers): Promise<Data> => {
     zip,
     ruleset,
     generateJsonFiles,
-    scanDuration
+    scanDuration,
   } = argv;
 
   const extraHTTPHeaders = parseHeaders(header);
@@ -632,8 +625,8 @@ export const prepareData = async (argv: Answers): Promise<Data> => {
     password = temp.password;
 
     if (username !== '' || password !== '') {
-      extraHTTPHeaders['Authorization'] = `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`;
-    } 
+      extraHTTPHeaders.Authorization = `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`;
+    }
 
     temp.username = '';
     temp.password = '';
@@ -653,17 +646,22 @@ export const prepareData = async (argv: Answers): Promise<Data> => {
   if (exportDirectory) {
     constants.exportDirectory = path.join(exportDirectory, resultFilename);
   }
-  
+
   // Creating the playwrightDeviceDetailObject
-  deviceChosen = customDevice === 'Desktop' || customDevice === 'Mobile' ? customDevice : deviceChosen;
-  
+  deviceChosen =
+    customDevice === 'Desktop' || customDevice === 'Mobile' ? customDevice : deviceChosen;
+
   const playwrightDeviceDetailsObject = getPlaywrightDeviceDetailsObject(
     deviceChosen,
     customDevice,
     viewportWidth,
   );
 
-  const { browserToRun: resolvedBrowser, clonedBrowserDataDir } = getBrowserToRun(resultFilename, browserToRun, true);
+  const { browserToRun: resolvedBrowser, clonedBrowserDataDir } = getBrowserToRun(
+    resultFilename,
+    browserToRun,
+    true,
+  );
   browserToRun = resolvedBrowser;
 
   const resolvedUserDataDirectory = getClonedProfilesWithRandomToken(browserToRun, resultFilename);
@@ -678,7 +676,7 @@ export const prepareData = async (argv: Answers): Promise<Data> => {
 
   return {
     type: scanner,
-    url: url,
+    url,
     entryUrl: url,
     isHeadless: headless,
     deviceChosen,
@@ -699,7 +697,7 @@ export const prepareData = async (argv: Answers): Promise<Data> => {
     includeScreenshots: !(additional === 'none'),
     metadata,
     followRobots,
-    extraHTTPHeaders: extraHTTPHeaders,
+    extraHTTPHeaders,
     safeMode,
     userDataDirectory: resolvedUserDataDirectory,
     zip,
@@ -709,7 +707,12 @@ export const prepareData = async (argv: Answers): Promise<Data> => {
   };
 };
 
-export const getUrlsFromRobotsTxt = async (url: string, browserToRun: string, userDataDirectory: string, extraHTTPHeaders: Record<string, string>): Promise<void> => {
+export const getUrlsFromRobotsTxt = async (
+  url: string,
+  browserToRun: string,
+  userDataDirectory: string,
+  extraHTTPHeaders: Record<string, string>,
+): Promise<void> => {
   if (!constants.robotsTxtUrls) return;
 
   const domain = new URL(url).origin;
@@ -718,7 +721,12 @@ export const getUrlsFromRobotsTxt = async (url: string, browserToRun: string, us
 
   let robotsTxt: string;
   try {
-    robotsTxt = await getRobotsTxtViaPlaywright(robotsUrl, browserToRun, userDataDirectory, extraHTTPHeaders);
+    robotsTxt = await getRobotsTxtViaPlaywright(
+      robotsUrl,
+      browserToRun,
+      userDataDirectory,
+      extraHTTPHeaders,
+    );
     consoleLogger.info(`Fetched robots.txt from ${robotsUrl}`);
   } catch (e) {
     // if robots.txt is not found, do nothing
@@ -729,7 +737,7 @@ export const getUrlsFromRobotsTxt = async (url: string, browserToRun: string, us
     constants.robotsTxtUrls[domain] = {};
     return;
   }
-  
+
   const lines = robotsTxt.split(/\r?\n/);
   let shouldCapture = false;
   const disallowedUrls = [];
@@ -777,9 +785,13 @@ export const getUrlsFromRobotsTxt = async (url: string, browserToRun: string, us
   constants.robotsTxtUrls[domain] = { disallowedUrls, allowedUrls };
 };
 
-const getRobotsTxtViaPlaywright = async (robotsUrl: string, browser: string, userDataDirectory: string, extraHTTPHeaders: Record<string, string>): Promise<string> => {
-
-  let robotsDataDir = '';
+const getRobotsTxtViaPlaywright = async (
+  robotsUrl: string,
+  browser: string,
+  userDataDirectory: string,
+  extraHTTPHeaders: Record<string, string>,
+): Promise<string> => {
+  const robotsDataDir = '';
   // Bug in Chrome which causes browser pool crash when userDataDirectory is set in non-headless mode
   if (process.env.CRAWLEE_HEADLESS === '1') {
     // Create robots own user data directory else SingletonLock: File exists (17) with crawlDomain or crawlSitemap's own browser
@@ -932,7 +944,7 @@ export const getLinksFromSitemap = async (
   const fetchUrls = async (url: string, extraHTTPHeaders: Record<string, string>) => {
     let data;
     let sitemapType;
-   
+
     if (scannedSitemaps.has(url)) {
       // Skip processing if the sitemap has already been scanned
       return;
@@ -948,7 +960,6 @@ export const getLinksFromSitemap = async (
       if (!fs.existsSync(url)) {
         return;
       }
-
     } else if (isValidHttpUrl(url)) {
       // Do nothing, url is valid
     } else {
@@ -973,7 +984,7 @@ export const getLinksFromSitemap = async (
 
       await page.goto(url, { waitUntil: 'networkidle', timeout: 60000 });
 
-      if (await page.locator('body').count() > 0) {
+      if ((await page.locator('body').count()) > 0) {
         data = await page.locator('body').innerText();
       } else {
         const urlSet = page.locator('urlset');
@@ -1003,7 +1014,6 @@ export const getLinksFromSitemap = async (
       }
 
       await getDataUsingPlaywright();
-
     } else {
       url = convertLocalFileToPath(url);
       data = fs.readFileSync(url, 'utf8');
@@ -1122,7 +1132,6 @@ export const getBrowserToRun = (
   preferredBrowser?: BrowserTypes,
   isCli = false,
 ): { browserToRun: BrowserTypes; clonedBrowserDataDir: string } => {
-  
   const platform = os.platform();
 
   // Prioritise Chrome on Windows and Mac platforms if user does not specify a browser
@@ -1248,14 +1257,14 @@ const cloneChromeProfileCookieFiles = (options: GlobOptionsWithFileTypesFalse, d
   if (os.platform() === 'win32') {
     profileCookiesDir = globSync('**/Network/Cookies', {
       ...options,
-      ignore: ['oobee/**'],
+      ignore: ['oobee*/**'],
     });
     profileNamesRegex = /User Data\\(.*?)\\Network/;
   } else if (os.platform() === 'darwin') {
     // maxDepth 2 to avoid copying cookies from the oobee directory if it exists
     profileCookiesDir = globSync('**/Cookies', {
       ...options,
-      ignore: 'oobee/**',
+      ignore: 'oobee*/**',
     });
     profileNamesRegex = /Chrome\/(.*?)\/Cookies/;
   }
@@ -1443,7 +1452,7 @@ export const cloneChromeProfiles = (randomToken: string): string => {
   destDir = path.join(baseDir, `oobee-${randomToken}`);
 
   if (fs.existsSync(destDir)) {
-    // Don't delete since it will be handled at the end of the scan  
+    // Don't delete since it will be handled at the end of the scan
     // deleteClonedChromeProfiles(randomToken);
     // Assume it cloned and don't re-clone
   } else {
@@ -1463,7 +1472,6 @@ export const cloneChromeProfiles = (randomToken: string): string => {
     }
 
     consoleLogger.error('Failed to clone Chrome profiles. You may be logged out of your accounts.');
-
   }
   // For future reference, return a null instead to halt the scan
   return destDir;
@@ -1481,8 +1489,7 @@ export const cloneChromiumProfiles = (randomToken: string): string => {
   destDir = path.join(baseDir, `oobee-${randomToken}`);
 
   if (fs.existsSync(destDir)) {
-     
-    // Don't delete since it will be handled at the end of the scan  
+    // Don't delete since it will be handled at the end of the scan
     // deleteClonedChromiumProfiles(randomToken);
     // Assume it cloned and don't re-clone
   } else {
@@ -1512,11 +1519,9 @@ export const cloneEdgeProfiles = (randomToken: string): string => {
   destDir = path.join(baseDir, `oobee-${randomToken}`);
 
   if (fs.existsSync(destDir)) {
-
-    // Don't delete since it will be handled at the end of the scan  
+    // Don't delete since it will be handled at the end of the scan
     // deleteClonedEdgeProfiles(randomToken);
     // Assume it cloned and don't re-clone
-
   } else {
     if (!fs.existsSync(destDir)) {
       fs.mkdirSync(destDir, { recursive: true });
@@ -1535,7 +1540,6 @@ export const cloneEdgeProfiles = (randomToken: string): string => {
     }
 
     consoleLogger.error('Failed to clone Edge profiles. You may be logged out of your accounts.');
-
   }
 
   // For future reference, return a null instead to halt the scan
@@ -1597,7 +1601,6 @@ export const deleteClonedChromeProfiles = (randomToken?: string): void => {
  * @returns null
  */
 export const deleteClonedEdgeProfiles = (randomToken?: string): void => {
-
   const baseDir = getDefaultEdgeDataDir();
 
   if (!baseDir) {
@@ -1714,12 +1717,9 @@ export const submitFormViaPlaywright = async (
   userDataDirectory: string,
   finalUrl: string,
 ) => {
-  const browserContext = await constants.launcher.launchPersistentContext(
-    userDataDirectory,
-    {
-      ...getPlaywrightLaunchOptions(browserToRun),
-    },
-  );
+  const browserContext = await constants.launcher.launchPersistentContext(userDataDirectory, {
+    ...getPlaywrightLaunchOptions(browserToRun),
+  });
 
   register(browserContext);
 
@@ -1778,7 +1778,6 @@ export const submitForm = async (
     finalUrl += `&${formDataFields.redirectUrlField}=${scannedUrl}`;
   }
 
-
   try {
     await axios.get(finalUrl, { timeout: 2000 });
   } catch (error) {
@@ -1788,7 +1787,6 @@ export const submitForm = async (
       }
     }
   }
-
 };
 // Legacy code end - Google Sheets submission
 
@@ -1797,7 +1795,6 @@ export async function initModifiedUserAgent(
   playwrightDeviceDetailsObject?: object,
   userDataDirectory?: string,
 ) {
-
   const isHeadless = process.env.CRAWLEE_HEADLESS === '1';
 
   // If headless mode is enabled, ensure the headless flag is set.
@@ -1814,13 +1811,14 @@ export async function initModifiedUserAgent(
   };
 
   // Launch a temporary persistent context with an empty userDataDir to mimic your production browser setup.
-  const effectiveUserDataDirectory = process.env.CRAWLEE_HEADLESS === '1'
-  ? userDataDirectory
-  : '';
+  const effectiveUserDataDirectory = process.env.CRAWLEE_HEADLESS === '1' ? userDataDirectory : '';
 
-  const browserContext = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, launchOptions);
+  const browserContext = await constants.launcher.launchPersistentContext(
+    effectiveUserDataDirectory,
+    launchOptions,
+  );
   register(browserContext);
-  
+
   const page = await browserContext.newPage();
 
   // Retrieve the default user agent.
@@ -1962,7 +1960,7 @@ export const waitForPageLoaded = async (page: Page, timeout = 10000) => {
         // Only observe if root is a Node
         observer.observe(root, {
           childList: true,
-          subtree:   true,
+          subtree: true,
           attributes: true,
         });
       });