@govtechsg/oobee 0.10.84 → 0.10.86

package/src/crawlers/guards/urlGuard.ts

@@ -5,41 +5,34 @@ export function addUrlGuardScript(context, opts = {}) {
 
   const lastAllowedUrlByPage = new WeakMap();
 
-  const attachGuardsToPage = (page) => {
+  const attachGuardsToPage = page => {
     if (!lastAllowedUrlByPage.has(page) && fallbackUrl) {
       lastAllowedUrlByPage.set(page, String(fallbackUrl));
     }
 
-    page.addInitScript(() => {
-      const isAllowedProtocol = (value) => {
-        try {
-          const s = value instanceof URL ? value.toString() : String(value);
-          const protocol = new URL(s, window.location.href).protocol;
-          return protocol === 'http:' || protocol === 'https:';
-        } catch {
-          return false;
-        }
-      };
+    page
+      .addInitScript(() => {
+        const isAllowedProtocol = value => {
+          try {
+            const s = value instanceof URL ? value.toString() : String(value);
+            const { protocol } = new URL(s, window.location.href);
+            return protocol === 'http:' || protocol === 'https:';
+          } catch {
+            return false;
+          }
+        };
 
-      const win = window;
+        const win = window;
 
-      const openOriginal = win.open;
-      win.open = function (targetUrl, ...args) {
-        if (!isAllowedProtocol(targetUrl)) return null;
-        return openOriginal.call(this, targetUrl, ...args);
-      };
-
-      const assignOriginal  = win.location.assign.bind(win.location);
-      const replaceOriginal = win.location.replace.bind(win.location);
-
-      win.location.assign  = (nextUrl) => { if (isAllowedProtocol(nextUrl)) assignOriginal(nextUrl); };
-      win.location.replace = (nextUrl) => { if (isAllowedProtocol(nextUrl)) replaceOriginal(nextUrl); };
-
-      Object.defineProperty(win.location, 'href', {
-        get() { return String(win.location.toString()); },
-        set(nextUrl) { if (isAllowedProtocol(nextUrl)) assignOriginal(nextUrl); },
+        const openOriginal = win.open;
+        win.open = function (targetUrl, ...args) {
+          if (!isAllowedProtocol(targetUrl)) return null;
+          return openOriginal.call(this, targetUrl, ...args);
+        };
+      })
+      .catch(() => {
+        // page may have closed before addInitScript completed; safe to ignore
       });
-    });
 
     const restoreToSafeUrl = async (page, attemptedUrl) => {
       try {
@@ -50,15 +43,15 @@ export function addUrlGuardScript(context, opts = {}) {
       }
     };
 
-    page.on('framenavigated', async (frame) => {
+    page.on('framenavigated', async frame => {
       if (frame !== page.mainFrame()) return;
 
       const urlStr = frame.url();
       let urlObj;
       try {
-      	urlObj = new URL(urlStr);
+        urlObj = new URL(urlStr);
       } catch {
-      	return restoreToSafeUrl(page, urlStr);
+        return restoreToSafeUrl(page, urlStr);
       }
 
       if (ALLOWED_PROTOCOLS.has(urlObj.protocol)) {

package/src/crawlers/runCustom.ts

@@ -1,5 +1,4 @@
 /* eslint-env browser */
-import { chromium } from 'playwright';
 import { createCrawleeSubFolders } from './commonCrawlerFunc.js';
 import { cleanUpAndExit, register, registerSoftClose } from '../utils.js';
 import constants, {
@@ -11,7 +10,12 @@ import { DEBUG, initNewPage, log } from './custom/utils.js';
 import { guiInfoLog } from '../logs.js';
 import { ViewportSettingsClass } from '../combine.js';
 import { addUrlGuardScript } from './guards/urlGuard.js';
-import { getPlaywrightLaunchOptions } from '../constants/common.js';
+import {
+  getBrowserToRun,
+  getPlaywrightLaunchOptions,
+  initModifiedUserAgent,
+} from '../constants/common.js';
+import { BrowserTypes } from '../constants/constants.js';
 
 // Export of classes
 
@@ -25,6 +29,8 @@ export class ProcessPageParams {
   randomToken: string;
   customFlowLabel?: string;
   stopAll?: () => Promise<void>;
+  entryUrl!: string;
+  strategy: string;
 
   constructor(
     scannedIdx: number,
@@ -48,6 +54,8 @@ export class ProcessPageParams {
 const runCustom = async (
   url: string,
   randomToken: string,
+  browserToRun: string,
+  userDataDirectory: string,
   viewportSettings: ViewportSettingsClass,
   blacklistedPatterns: string[] | null,
   includeScreenshots: boolean,
@@ -69,6 +77,8 @@ const runCustom = async (
     randomToken,
   );
 
+  processPageParams.entryUrl = url;
+
   if (initialCustomFlowLabel && initialCustomFlowLabel.trim()) {
     processPageParams.customFlowLabel = initialCustomFlowLabel.trim();
   }
@@ -77,28 +87,37 @@ const runCustom = async (
   const pageClosePromises = [];
 
   try {
+    const { browserToRun: resolvedBrowserToRun } = getBrowserToRun(
+      randomToken,
+      browserToRun as BrowserTypes,
+      false,
+    );
     const deviceConfig = viewportSettings.playwrightDeviceDetailsObject;
     const hasCustomViewport = !!deviceConfig;
+    const rawDevice = (deviceConfig || {}) as Record<string, unknown>;
+    const { userAgent: deviceUserAgent, ...contextDeviceOptions } = rawDevice;
+
+    await initModifiedUserAgent(resolvedBrowserToRun, viewportSettings.playwrightDeviceDetailsObject);
 
-    const baseLaunchOptions = getPlaywrightLaunchOptions('chrome');
+    const baseLaunchOptions = getPlaywrightLaunchOptions(resolvedBrowserToRun);
 
     // Merge base args with custom flow specific args
     const baseArgs = baseLaunchOptions.args || [];
     const customArgs = hasCustomViewport ? ['--window-size=1920,1040'] : ['--start-maximized'];
-    const mergedArgs = [...baseArgs.filter(a => !a.startsWith('--window-size') && a !== '--start-maximized'), ...customArgs];
+    const mergedArgs = [
+      ...baseArgs.filter(a => !a.startsWith('--window-size') && a !== '--start-maximized'),
+      ...customArgs,
+    ];
 
-    const browser = await chromium.launch({
+    const context = await constants.launcher.launchPersistentContext(userDataDirectory, {
       ...baseLaunchOptions,
       args: mergedArgs,
       headless: false,
-      channel: 'chrome',
-    });
-
-    const context = await browser.newContext({
       ignoreHTTPSErrors: true,
       serviceWorkers: 'block',
       viewport: null,
-      ...(hasCustomViewport ? deviceConfig : {}),
+      ...(hasCustomViewport ? contextDeviceOptions : {}),
+      userAgent: process.env.OOBEE_USER_AGENT || (deviceUserAgent as string | undefined),
     });
 
     register(context);
@@ -106,9 +125,7 @@ const runCustom = async (
     processPageParams.stopAll = async () => {
       try {
         await context.close().catch(() => {});
-        await browser.close().catch(() => {});
-      } catch {
-      }
+      } catch {}
     };
 
     // For handling closing playwright browser and continue generate artifacts etc
@@ -116,12 +133,18 @@ const runCustom = async (
 
     addUrlGuardScript(context, { fallbackUrl: url });
 
+    const page = context.pages().find(existingPage => !existingPage.isClosed()) || (await context.newPage());
+    await initNewPage(page, pageClosePromises, processPageParams, pagesDict);
+
     // Detection of new page
     context.on('page', async newPage => {
-      await initNewPage(newPage, pageClosePromises, processPageParams, pagesDict);
+      try {
+        await initNewPage(newPage, pageClosePromises, processPageParams, pagesDict);
+      } catch (e) {
+        log(`Error initializing new page: ${e}`);
+      }
     });
 
-    const page = await context.newPage();
     await page.goto(url, { timeout: 0 });
 
     // to execute and wait for all pages to close