@govtechsg/oobee 0.10.76 → 0.10.77

package/dist/crawlers/custom/xPathToCss.js

@@ -0,0 +1,147 @@
+export default function xPathToCss(expr) {
+    const isValidXPath = (expr) => typeof expr !== 'undefined' &&
+        expr.replace(/[\s-_=]/g, '') !== '' &&
+        expr.length ===
+            expr.replace(/[-_\w:.]+\(\)\s*=|=\s*[-_\w:.]+\(\)|\sor\s|\sand\s|\[(?:[^\/\]]+[\/\[]\/?.+)+\]|starts-with\(|\[.*last\(\)\s*[-\+<>=].+\]|number\(\)|not\(|count\(|text\(|first\(|normalize-space|[^\/]following-sibling|concat\(|descendant::|parent::|self::|child::|/gi, '').length;
+    const getValidationRegex = () => {
+        let regex = '(?P<node>' +
+            '(' +
+            '^id\\(["\\\']?(?P<idvalue>%(value)s)["\\\']?\\)' + // special case! `id(idValue)`
+            '|' +
+            '(?P<nav>//?(?:following-sibling::)?)(?P<tag>%(tag)s)' + //  `//div`
+            '(\\[(' +
+            '(?P<matched>(?P<mattr>@?%(attribute)s=["\\\'](?P<mvalue>%(value)s))["\\\']' + // `[@id="well"]` supported and `[text()="yes"]` is not
+            '|' +
+            '(?P<contained>contains\\((?P<cattr>@?%(attribute)s,\\s*["\\\'](?P<cvalue>%(value)s)["\\\']\\))' + // `[contains(@id, "bleh")]` supported and `[contains(text(), "some")]` is not
+            ')\\])?' +
+            '(\\[\\s*(?P<nth>\\d+|last\\(\\s*\\))\\s*\\])?' +
+            ')' +
+            ')';
+        const subRegexes = {
+            tag: '([a-zA-Z][a-zA-Z0-9:-]*|\\*)',
+            attribute: '[.a-zA-Z_:][-\\w:.]*(\\(\\))?)',
+            value: '\\s*[\\w/:][-/\\w\\s,:;.]*',
+        };
+        Object.keys(subRegexes).forEach((key) => {
+            regex = regex.replace(new RegExp(`%\\(${key}\\)s`, 'gi'), subRegexes[key]);
+        });
+        regex = regex.replace(/\?P<node>|\?P<idvalue>|\?P<nav>|\?P<tag>|\?P<matched>|\?P<mattr>|\?P<mvalue>|\?P<contained>|\?P<cattr>|\?P<cvalue>|\?P<nth>/gi, '');
+        return new RegExp(regex, 'gi');
+    };
+    const preParseXpath = (expr) => expr.replace(/contains\s*\(\s*concat\(["']\s+["']\s*,\s*@class\s*,\s*["']\s+["']\)\s*,\s*["']\s+([a-zA-Z0-9-_]+)\s+["']\)/gi, '@class="$1"');
+    function escapeCssIdSelectors(cssSelector) {
+        return cssSelector.replace(/#([^ >]+)/g, (_match, id) => {
+            // Escape special characters in the id part
+            return `#${id.replace(/[!"#$%&'()*+,./:;<=>?@[\\\]^`{|}~]/g, '\\$&')}`;
+        });
+    }
+    if (!expr) {
+        throw new Error('Missing XPath expression');
+    }
+    expr = preParseXpath(expr);
+    if (!isValidXPath(expr)) {
+        console.error(`Invalid or unsupported XPath: ${expr}`);
+        // do not throw error so that this function proceeds to convert xpath that it does not support
+        // for example, //*[@id="google_ads_iframe_/4654/dweb/imu1/homepage/landingpage/na_0"]/html/body/div[1]/a
+        // becomes #google_ads_iframe_/4654/dweb/imu1/homepage/landingpage/na_0 > html > body > div:first-of-type > div > a
+        // which is invalid because the slashes in the id selector are not escaped
+        // throw new Error('Invalid or unsupported XPath: ' + expr);
+    }
+    const xPathArr = expr.split('|');
+    const prog = getValidationRegex();
+    const cssSelectors = [];
+    let xindex = 0;
+    while (xPathArr[xindex]) {
+        const css = [];
+        let position = 0;
+        let nodes;
+        while ((nodes = prog.exec(xPathArr[xindex]))) {
+            let attr;
+            if (!nodes && position === 0) {
+                throw new Error(`Invalid or unsupported XPath: ${expr}`);
+            }
+            const match = {
+                node: nodes[5],
+                idvalue: nodes[12] || nodes[3],
+                nav: nodes[4],
+                tag: nodes[5],
+                matched: nodes[7],
+                mattr: nodes[10] || nodes[14],
+                mvalue: nodes[12] || nodes[16],
+                contained: nodes[13],
+                cattr: nodes[14],
+                cvalue: nodes[16],
+                nth: nodes[18],
+            };
+            let nav = '';
+            if (position != 0 && match.nav) {
+                if (~match.nav.indexOf('following-sibling::')) {
+                    nav = ' + ';
+                }
+                else {
+                    nav = match.nav == '//' ? ' ' : ' > ';
+                }
+            }
+            const tag = match.tag === '*' ? '' : match.tag || '';
+            if (match.contained) {
+                if (match.cattr.indexOf('@') === 0) {
+                    attr = `[${match.cattr.replace(/^@/, '')}*="${match.cvalue}"]`;
+                }
+                else {
+                    throw new Error(`Invalid or unsupported XPath attribute: ${match.cattr}`);
+                }
+            }
+            else if (match.matched) {
+                switch (match.mattr) {
+                    case '@id':
+                        attr = `#${match.mvalue.replace(/^\s+|\s+$/, '').replace(/\s/g, '#')}`;
+                        break;
+                    case '@class':
+                        attr = `.${match.mvalue.replace(/^\s+|\s+$/, '').replace(/\s/g, '.')}`;
+                        break;
+                    case 'text()':
+                    case '.':
+                        throw new Error(`Invalid or unsupported XPath attribute: ${match.mattr}`);
+                    default:
+                        if (match.mattr.indexOf('@') !== 0) {
+                            throw new Error(`Invalid or unsupported XPath attribute: ${match.mattr}`);
+                        }
+                        if (match.mvalue.indexOf(' ') !== -1) {
+                            match.mvalue = `\"${match.mvalue.replace(/^\s+|\s+$/, '')}\"`;
+                        }
+                        attr = `[${match.mattr.replace('@', '')}="${match.mvalue}"]`;
+                        break;
+                }
+            }
+            else if (match.idvalue) {
+                attr = `#${match.idvalue.replace(/\s/, '#')}`;
+            }
+            else {
+                attr = '';
+            }
+            let nth = '';
+            if (match.nth) {
+                if (match.nth.indexOf('last') === -1) {
+                    if (isNaN(parseInt(match.nth, 10))) {
+                        throw new Error(`Invalid or unsupported XPath attribute: ${match.nth}`);
+                    }
+                    nth = parseInt(match.nth, 10) !== 1 ? `:nth-of-type(${match.nth})` : ':first-of-type';
+                }
+                else {
+                    nth = ':last-of-type';
+                }
+            }
+            css.push(nav + tag + attr + nth);
+            position++;
+        }
+        const result = css.join('');
+        if (result === '') {
+            throw new Error('Invalid or unsupported XPath');
+        }
+        cssSelectors.push(result);
+        xindex++;
+    }
+    // return cssSelectors.join(', ');
+    const originalResult = cssSelectors.join(', ');
+    return escapeCssIdSelectors(originalResult);
+}

package/dist/crawlers/guards/urlGuard.js

@@ -0,0 +1,71 @@
+const ALLOWED_PROTOCOLS = new Set(['http:', 'https:']);
+export function addUrlGuardScript(context, opts = {}) {
+    const { fallbackUrl } = opts;
+    const lastAllowedUrlByPage = new WeakMap();
+    const attachGuardsToPage = (page) => {
+        if (!lastAllowedUrlByPage.has(page) && fallbackUrl) {
+            lastAllowedUrlByPage.set(page, String(fallbackUrl));
+        }
+        page.addInitScript(() => {
+            const isAllowedProtocol = (value) => {
+                try {
+                    const s = value instanceof URL ? value.toString() : String(value);
+                    const protocol = new URL(s, window.location.href).protocol;
+                    return protocol === 'http:' || protocol === 'https:';
+                }
+                catch {
+                    return false;
+                }
+            };
+            const win = window;
+            const openOriginal = win.open;
+            win.open = function (targetUrl, ...args) {
+                if (!isAllowedProtocol(targetUrl))
+                    return null;
+                return openOriginal.call(this, targetUrl, ...args);
+            };
+            const assignOriginal = win.location.assign.bind(win.location);
+            const replaceOriginal = win.location.replace.bind(win.location);
+            win.location.assign = (nextUrl) => { if (isAllowedProtocol(nextUrl))
+                assignOriginal(nextUrl); };
+            win.location.replace = (nextUrl) => { if (isAllowedProtocol(nextUrl))
+                replaceOriginal(nextUrl); };
+            Object.defineProperty(win.location, 'href', {
+                get() { return String(win.location.toString()); },
+                set(nextUrl) { if (isAllowedProtocol(nextUrl))
+                    assignOriginal(nextUrl); },
+            });
+        });
+        const restoreToSafeUrl = async (page, attemptedUrl) => {
+            try {
+                const safeUrl = lastAllowedUrlByPage.get(page) || fallbackUrl || 'about:blank';
+                await page.goto(safeUrl, { waitUntil: 'domcontentloaded' });
+            }
+            catch {
+                // page might be closing; ignore
+            }
+        };
+        page.on('framenavigated', async (frame) => {
+            if (frame !== page.mainFrame())
+                return;
+            const urlStr = frame.url();
+            let urlObj;
+            try {
+                urlObj = new URL(urlStr);
+            }
+            catch {
+                return restoreToSafeUrl(page, urlStr);
+            }
+            if (ALLOWED_PROTOCOLS.has(urlObj.protocol)) {
+                lastAllowedUrlByPage.set(page, urlObj.toString());
+                return;
+            }
+            await restoreToSafeUrl(page, urlStr);
+        });
+    };
+    // Guard existing and future pages
+    for (const page of context.pages())
+        attachGuardsToPage(page);
+    context.on('page', attachGuardsToPage);
+}
+export default addUrlGuardScript;

package/dist/crawlers/pdfScanFunc.js

@@ -0,0 +1,276 @@
+import { spawnSync } from 'child_process';
+import fs from 'fs';
+import { randomUUID } from 'crypto';
+import { createRequire } from 'module';
+import os from 'os';
+import path from 'path';
+import { ensureDirSync } from 'fs-extra';
+import { getPageFromContext, getPdfScreenshots } from '../screenshotFunc/pdfScreenshotFunc.js';
+import { consoleLogger, guiInfoLog } from '../logs.js';
+import constants, { getExecutablePath, guiInfoStatusTypes, STATUS_CODE_METADATA, } from '../constants/constants.js';
+import { cleanUpAndExit, getPdfStoragePath, getStoragePath } from '../utils.js';
+const require = createRequire(import.meta.url);
+// Classes
+class TranslatedObject {
+    constructor() {
+        this.url = '';
+        this.pageTitle = '';
+        this.filePath = '';
+        this.totalItems = 0;
+        this.goodToFix = {
+            rules: {},
+            totalItems: 0,
+        };
+        this.mustFix = {
+            rules: {},
+            totalItems: 0,
+        };
+        this.needsReview = {
+            rules: {},
+            totalItems: 0,
+        };
+    }
+}
+export class TransformedRuleObject {
+    constructor() {
+        this.description = '';
+        this.totalItems = 0;
+        this.conformance = [];
+        this.items = [];
+    }
+}
+// AAA: 1.4.8, 2.4.9
+// AA: 1.3.4, 1.4.3, 1.4.4, 1.4.10
+// A: 1.3.1, 4.1.1, 4.1.2
+const LEVEL_AAA = ['2.4.9', '1.4.8'];
+const LEVEL_AA = ['1.3.4', '1.4.3', '1.4.4', '1.4.10'];
+const LEVEL_A = ['1.3.1', '4.1.1', '4.1.2'];
+const clauseToLevel = {
+    // mapping of clause to its A/AA/AAA level
+    ...LEVEL_AAA.reduce((prev, curr) => {
+        prev[curr] = 'wcag2aaa';
+        return prev;
+    }, {}),
+    ...LEVEL_AA.reduce((prev, curr) => {
+        prev[curr] = 'wcag2aa';
+        return prev;
+    }, {}),
+    ...LEVEL_A.reduce((prev, curr) => {
+        prev[curr] = 'wcag2a';
+        return prev;
+    }, {}),
+};
+const metaToCategoryMap = {
+    critical: 'mustFix',
+    error: 'goodToFix',
+    serious: 'goodToFix',
+    warning: 'goodToFix',
+    ignore: 'goodToFix',
+};
+const EXCLUDED_RULES = {
+    '1.3.4': { 1: true }, // test for page orientation deemed a false positive, so its excluded
+};
+const isRuleExcluded = (rule) => {
+    const isExcluded = EXCLUDED_RULES[rule.clause]
+        ? EXCLUDED_RULES[rule.clause][rule.testNumber]
+        : false;
+    return isExcluded || LEVEL_AAA.includes(rule.clause);
+};
+const getVeraExecutable = () => {
+    let veraPdfExe;
+    if (os.platform() === 'win32') {
+        veraPdfExe = getExecutablePath('**/verapdf', 'verapdf.bat');
+    }
+    else {
+        veraPdfExe = getExecutablePath('**/verapdf', 'verapdf');
+    }
+    if (!veraPdfExe) {
+        const veraPdfExeNotFoundError = 'Could not find veraPDF executable.  Please ensure veraPDF is installed at current directory.';
+        consoleLogger.error(veraPdfExeNotFoundError);
+        consoleLogger.error(veraPdfExeNotFoundError);
+    }
+    return veraPdfExe;
+};
+const isPDF = (buffer) => {
+    return (Buffer.isBuffer(buffer) && buffer.lastIndexOf('%PDF-') === 0 && buffer.lastIndexOf('%%EOF') > -1);
+};
+export const handlePdfDownload = (randomToken, pdfDownloads, request, sendRequest, urlsCrawled) => {
+    const pdfFileName = randomUUID();
+    const { url } = request;
+    const pageTitle = decodeURI(request.url).split('/').pop();
+    pdfDownloads.push(new Promise(async (resolve) => {
+        let buf;
+        // Download from remote URL
+        const response = await sendRequest({ responseType: 'buffer' });
+        if (response.statusCode !== 200) {
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                numScanned: urlsCrawled.scanned.length,
+                urlScanned: request.url,
+            });
+            urlsCrawled.userExcluded.push({
+                url: request.url,
+                pageTitle: request.url,
+                actualUrl: request.url, // because about:blank is not useful
+                metadata: STATUS_CODE_METADATA[response.statusCode] || STATUS_CODE_METADATA[1],
+                httpStatusCode: 0,
+            });
+            resolve();
+            return;
+        }
+        buf = Buffer.isBuffer(response) ? response : response.body;
+        const downloadFile = fs.createWriteStream(`${getPdfStoragePath(randomToken)}/${pdfFileName}.pdf`, {
+            flags: 'w',
+        });
+        downloadFile.write(buf, 'binary');
+        downloadFile.end();
+        downloadFile.on('finish', () => {
+            if (isPDF(buf)) {
+                guiInfoLog(guiInfoStatusTypes.SCANNED, {
+                    numScanned: urlsCrawled.scanned.length,
+                    urlScanned: request.url,
+                });
+                urlsCrawled.scanned.push({
+                    url: request.url,
+                    pageTitle,
+                    actualUrl: url,
+                });
+            }
+            else {
+                guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                    numScanned: urlsCrawled.scanned.length,
+                    urlScanned: request.url,
+                });
+                urlsCrawled.invalid.push({
+                    url: request.url,
+                    pageTitle: url,
+                    actualUrl: url,
+                    metadata: STATUS_CODE_METADATA[1],
+                });
+            }
+            resolve();
+        });
+    }));
+    return { pdfFileName, url };
+};
+export const runPdfScan = async (randomToken) => {
+    const execFile = getVeraExecutable();
+    const veraPdfExe = `"${execFile}"`;
+    // const veraPdfProfile = getVeraProfile();
+    const veraPdfProfile = `"${path.join(path.dirname(execFile), 'profiles/veraPDF-validation-profiles-rel-1.26/PDF_UA/WCAG-2-2.xml')}"`;
+    if (!veraPdfExe || !veraPdfProfile) {
+        cleanUpAndExit(1);
+    }
+    const intermediateFolder = getPdfStoragePath(randomToken);
+    // store in a intermediate folder as we transfer final results later
+    const intermediateResultPath = `${intermediateFolder}/${constants.pdfScanResultFileName}`;
+    const veraPdfCmdArgs = [
+        '-p',
+        veraPdfProfile,
+        '--format',
+        'json',
+        '-r', // recurse through directory
+        `"${intermediateFolder}"`,
+    ];
+    const ls = spawnSync(veraPdfExe, veraPdfCmdArgs, { shell: true });
+    if (ls.stderr && ls.stderr.length > 0)
+        consoleLogger.error(ls.stderr.toString());
+    fs.writeFileSync(intermediateResultPath, ls.stdout, { encoding: 'utf-8' });
+};
+// transform results from veraPDF to desired format for report
+export const mapPdfScanResults = async (randomToken, uuidToUrlMapping) => {
+    const intermediateFolder = getPdfStoragePath(randomToken);
+    const intermediateResultPath = `${intermediateFolder}/${constants.pdfScanResultFileName}`;
+    const rawdata = fs.readFileSync(intermediateResultPath, 'utf-8');
+    let parsedJsonData;
+    try {
+        parsedJsonData = JSON.parse(rawdata);
+    }
+    catch (err) {
+        consoleLogger.error(err);
+    }
+    const errorMeta = require('../constants/errorMeta.json');
+    const resultsList = [];
+    if (parsedJsonData) {
+        // jobs: files that are scanned
+        const { report: { jobs }, } = parsedJsonData;
+        // loop through all jobs
+        for (let jobIdx = 0; jobIdx < jobs.length; jobIdx++) {
+            const translated = new TranslatedObject();
+            const { itemDetails, validationResult } = jobs[jobIdx];
+            const { name: fileName } = itemDetails;
+            const rawFileName = fileName.split(os.platform() === 'win32' ? '\\' : '/').pop();
+            const fileNameWithoutExt = rawFileName.replace(/\.pdf$/i, '');
+            const url = uuidToUrlMapping[rawFileName] || // exact match like 'Some-filename.pdf'
+                uuidToUrlMapping[fileNameWithoutExt] || // uuid-based key like 'a9f7ebbd-5a90...'
+                `file://${fileName}`; // fallback
+            const filePath = path.join(getPdfStoragePath(randomToken), rawFileName);
+            const pageTitle = decodeURI(url).split('/').pop();
+            translated.url = url;
+            translated.pageTitle = pageTitle;
+            translated.url = url;
+            translated.pageTitle = pageTitle;
+            translated.filePath = filePath;
+            if (!validationResult) {
+                // check for error in scan
+                consoleLogger.info(`Unable to scan ${pageTitle}, skipping`);
+                continue; // skip this job
+            }
+            // destructure validation result
+            const { passedChecks, failedChecks, ruleSummaries } = validationResult.details;
+            const totalChecks = passedChecks + failedChecks;
+            translated.totalItems = totalChecks;
+            // loop through all failed rules
+            for (let ruleIdx = 0; ruleIdx < ruleSummaries.length; ruleIdx++) {
+                const rule = ruleSummaries[ruleIdx];
+                const { specification, testNumber, clause } = rule;
+                if (isRuleExcluded(rule))
+                    continue;
+                const [ruleId, transformedRule] = await transformRule(rule, filePath);
+                // ignore if violation is not in the meta file
+                const meta = errorMeta[specification][clause][testNumber]?.STATUS ?? 'ignore';
+                const category = translated[metaToCategoryMap[meta]];
+                category.rules[ruleId] = transformedRule;
+                category.totalItems += transformedRule.totalItems;
+            }
+            resultsList.push(translated);
+        }
+    }
+    return resultsList;
+};
+const transformRule = async (rule, filePath) => {
+    // get specific rule
+    const transformed = new TransformedRuleObject();
+    const { specification, description, clause, testNumber, checks } = rule;
+    transformed.description = description;
+    transformed.totalItems = checks.length;
+    if (specification === 'WCAG2.1') {
+        transformed.conformance = [clauseToLevel[clause], `wcag${clause.split('.').join('')}`];
+    }
+    else {
+        transformed.conformance = ['best-practice'];
+    }
+    transformed.items = [];
+    for (let checkIdx = 0; checkIdx < checks.length; checkIdx++) {
+        const { errorMessage, context } = checks[checkIdx];
+        const page = await getPageFromContext(context, filePath);
+        transformed.items.push({ message: errorMessage, page, context });
+    }
+    const ruleId = `pdf-${specification}-${clause}-${testNumber}`.replaceAll(' ', '_');
+    return [ruleId, transformed];
+};
+export const doPdfScreenshots = async (randomToken, result) => {
+    const { filePath, pageTitle } = result;
+    const formattedPageTitle = pageTitle.replaceAll(' ', '_').split('.')[0];
+    const screenshotsDir = path.join(getStoragePath(randomToken), 'elemScreenshots', 'pdf');
+    ensureDirSync(screenshotsDir);
+    for (const category of ['mustFix', 'goodToFix']) {
+        const ruleItems = Object.entries(result[category].rules);
+        for (const [ruleId, ruleInfo] of ruleItems) {
+            const { items } = ruleInfo;
+            const filename = `${formattedPageTitle}-${category}-${ruleId}`;
+            const screenshotPath = path.join(screenshotsDir, filename);
+            const newItems = await getPdfScreenshots(filePath, items, screenshotPath);
+            ruleInfo.items = newItems;
+        }
+    }
+};

package/dist/crawlers/runCustom.js

@@ -0,0 +1,89 @@
+/* eslint-env browser */
+import { chromium } from 'playwright';
+import { createCrawleeSubFolders } from './commonCrawlerFunc.js';
+import { cleanUpAndExit, register, registerSoftClose } from '../utils.js';
+import constants, { getIntermediateScreenshotsPath, guiInfoStatusTypes, } from '../constants/constants.js';
+import { initNewPage, log } from './custom/utils.js';
+import { guiInfoLog } from '../logs.js';
+import { addUrlGuardScript } from './guards/urlGuard.js';
+// Export of classes
+export class ProcessPageParams {
+    constructor(scannedIdx, blacklistedPatterns, includeScreenshots, dataset, intermediateScreenshotsPath, urlsCrawled, randomToken) {
+        this.scannedIdx = scannedIdx;
+        this.blacklistedPatterns = blacklistedPatterns;
+        this.includeScreenshots = includeScreenshots;
+        this.dataset = dataset;
+        this.intermediateScreenshotsPath = intermediateScreenshotsPath;
+        this.urlsCrawled = urlsCrawled;
+        this.randomToken = randomToken;
+    }
+}
+const runCustom = async (url, randomToken, viewportSettings, blacklistedPatterns, includeScreenshots, initialCustomFlowLabel) => {
+    // checks and delete datasets path if it already exists
+    process.env.CRAWLEE_STORAGE_DIR = randomToken;
+    const urlsCrawled = { ...constants.urlsCrawledObj };
+    const { dataset } = await createCrawleeSubFolders(randomToken);
+    const intermediateScreenshotsPath = getIntermediateScreenshotsPath(randomToken);
+    const processPageParams = new ProcessPageParams(0, // scannedIdx
+    blacklistedPatterns, includeScreenshots, dataset, intermediateScreenshotsPath, urlsCrawled, randomToken);
+    if (initialCustomFlowLabel && initialCustomFlowLabel.trim()) {
+        processPageParams.customFlowLabel = initialCustomFlowLabel.trim();
+    }
+    const pagesDict = {};
+    const pageClosePromises = [];
+    try {
+        const deviceConfig = viewportSettings.playwrightDeviceDetailsObject;
+        const hasCustomViewport = !!deviceConfig;
+        const browser = await chromium.launch({
+            args: hasCustomViewport ? ['--window-size=1920,1040'] : ['--start-maximized'],
+            headless: false,
+            channel: 'chrome',
+            // bypassCSP: true,
+        });
+        const context = await browser.newContext({
+            ignoreHTTPSErrors: true,
+            serviceWorkers: 'block',
+            viewport: null,
+            ...(hasCustomViewport ? deviceConfig : {}),
+        });
+        register(context);
+        processPageParams.stopAll = async () => {
+            try {
+                await context.close().catch(() => { });
+                await browser.close().catch(() => { });
+            }
+            catch {
+            }
+        };
+        // For handling closing playwright browser and continue generate artifacts etc
+        registerSoftClose(processPageParams.stopAll);
+        addUrlGuardScript(context, { fallbackUrl: url });
+        // Detection of new page
+        context.on('page', async (newPage) => {
+            await initNewPage(newPage, pageClosePromises, processPageParams, pagesDict);
+        });
+        const page = await context.newPage();
+        await page.goto(url, { timeout: 0 });
+        // to execute and wait for all pages to close
+        // idea is for promise to be pending until page.on('close') detected
+        const allPagesClosedPromise = async (promises) => Promise.all(promises)
+            // necessary to recheck as during time of execution, more pages added
+            .then(() => {
+            if (Object.keys(pagesDict).length > 0) {
+                return allPagesClosedPromise(promises);
+            }
+            return Promise.resolve(true);
+        });
+        await allPagesClosedPromise(pageClosePromises);
+    }
+    catch (error) {
+        log(`PLAYWRIGHT EXECUTION ERROR ${error}`);
+        cleanUpAndExit(1, randomToken, true);
+    }
+    guiInfoLog(guiInfoStatusTypes.COMPLETED, {});
+    return {
+        urlsCrawled,
+        customFlowLabel: processPageParams.customFlowLabel,
+    };
+};
+export default runCustom;

package/dist/exclusions.txt

@@ -0,0 +1,7 @@
+\.*login.singpass.gov.sg\.*
+\.*login.id.singpass.gov.sg\.*
+\.*auth.singpass.gov.sg\.*
+\.*form.gov.sg\.*
+\.*login.microsoftonline.com\.*
+\.*id.atlassian.com\.*
+\.*cloudflareaccess.com\.*