@govtechsg/oobee 0.10.76 → 0.10.78-alpha1

package/dist/crawlers/crawlIntelligentSitemap.js

@@ -0,0 +1,135 @@
+import { createCrawleeSubFolders } from './commonCrawlerFunc.js';
+import constants, { guiInfoStatusTypes, sitemapPaths } from '../constants/constants.js';
+import { consoleLogger, guiInfoLog } from '../logs.js';
+import crawlDomain from './crawlDomain.js';
+import crawlSitemap from './crawlSitemap.js';
+import { getPlaywrightLaunchOptions } from '../constants/common.js';
+import { register } from '../utils.js';
+const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, strategy, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, followRobots, extraHTTPHeaders, safeMode, scanDuration) => {
+    const startTime = Date.now(); // Track start time
+    let urlsCrawledFinal;
+    const urlsCrawled = { ...constants.urlsCrawledObj };
+    let dataset;
+    let sitemapExist = false;
+    const fromCrawlIntelligentSitemap = true;
+    let sitemapUrl;
+    let durationExceeded = false;
+    ({ dataset } = await createCrawleeSubFolders(randomToken));
+    function getHomeUrl(parsedUrl) {
+        const urlObject = new URL(parsedUrl);
+        return `${urlObject.protocol}//${urlObject.hostname}${urlObject.port ? `:${urlObject.port}` : ''}`;
+    }
+    async function findSitemap(link, userDataDirectory, extraHTTPHeaders) {
+        const homeUrl = getHomeUrl(link);
+        let sitemapLink = '';
+        const effectiveUserDataDirectory = process.env.CRAWLEE_HEADLESS === '1' ? userDataDirectory : '';
+        const context = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, {
+            headless: process.env.CRAWLEE_HEADLESS === '1',
+            ...getPlaywrightLaunchOptions(browser),
+            ...(extraHTTPHeaders && { extraHTTPHeaders }),
+        });
+        register(context);
+        const page = await context.newPage();
+        for (const path of sitemapPaths) {
+            sitemapLink = homeUrl + path;
+            if (await checkUrlExists(page, sitemapLink)) {
+                sitemapExist = true;
+                break;
+            }
+        }
+        await page.close();
+        await context.close().catch(() => { });
+        return sitemapExist ? sitemapLink : '';
+    }
+    const checkUrlExists = async (page, parsedUrl) => {
+        try {
+            const response = await page.goto(parsedUrl);
+            return response.ok();
+        }
+        catch (e) {
+            consoleLogger.error(e);
+            return false;
+        }
+    };
+    try {
+        sitemapUrl = await findSitemap(url, userDataDirectory, extraHTTPHeaders);
+    }
+    catch (error) {
+        consoleLogger.error(error);
+    }
+    if (!sitemapExist) {
+        console.log('Unable to find sitemap. Commencing website crawl instead.');
+        return await crawlDomain({
+            url,
+            randomToken,
+            host,
+            viewportSettings,
+            maxRequestsPerCrawl,
+            browser,
+            userDataDirectory,
+            strategy,
+            specifiedMaxConcurrency,
+            fileTypes,
+            blacklistedPatterns,
+            includeScreenshots,
+            followRobots,
+            extraHTTPHeaders,
+            safeMode,
+            scanDuration, // Use full duration since no sitemap
+        });
+    }
+    console.log(`Sitemap found at ${sitemapUrl}`);
+    urlsCrawledFinal = await crawlSitemap({
+        sitemapUrl,
+        randomToken,
+        host,
+        viewportSettings,
+        maxRequestsPerCrawl,
+        browser,
+        userDataDirectory,
+        specifiedMaxConcurrency,
+        fileTypes,
+        blacklistedPatterns,
+        includeScreenshots,
+        extraHTTPHeaders,
+        fromCrawlIntelligentSitemap,
+        userUrlInputFromIntelligent: url,
+        datasetFromIntelligent: dataset,
+        urlsCrawledFromIntelligent: urlsCrawled,
+        crawledFromLocalFile: false,
+        scanDuration,
+    });
+    const elapsed = Date.now() - startTime;
+    const remainingScanDuration = Math.max(scanDuration - elapsed / 1000, 0); // in seconds
+    if (urlsCrawledFinal.scanned.length < maxRequestsPerCrawl && remainingScanDuration > 0) {
+        console.log(`Continuing crawl from root website. Remaining scan time: ${remainingScanDuration.toFixed(1)}s`);
+        urlsCrawledFinal = await crawlDomain({
+            url,
+            randomToken,
+            host,
+            viewportSettings,
+            maxRequestsPerCrawl,
+            browser,
+            userDataDirectory,
+            strategy,
+            specifiedMaxConcurrency,
+            fileTypes,
+            blacklistedPatterns,
+            includeScreenshots,
+            followRobots,
+            extraHTTPHeaders,
+            safeMode,
+            fromCrawlIntelligentSitemap,
+            datasetFromIntelligent: dataset,
+            urlsCrawledFromIntelligent: urlsCrawledFinal,
+            scanDuration: remainingScanDuration,
+        });
+    }
+    else if (remainingScanDuration <= 0) {
+        console.log(`Crawl duration exceeded before more pages could be found (limit: ${scanDuration}s).`);
+        durationExceeded = true;
+    }
+    guiInfoLog(guiInfoStatusTypes.COMPLETED, {});
+    return { urlsCrawled: urlsCrawledFinal, durationExceeded };
+};
+export default crawlIntelligentSitemap;

package/dist/crawlers/crawlLocalFile.js

@@ -0,0 +1,151 @@
+import { Request, RequestList } from 'crawlee';
+import fs from 'fs';
+import path from 'path';
+import { createCrawleeSubFolders, runAxeScript, isUrlPdf } from './commonCrawlerFunc.js';
+import constants, { guiInfoStatusTypes, } from '../constants/constants.js';
+import { getPlaywrightLaunchOptions, isFilePath, convertLocalFileToPath, convertPathToLocalFile, } from '../constants/common.js';
+import { runPdfScan, mapPdfScanResults, doPdfScreenshots } from './pdfScanFunc.js';
+import { guiInfoLog } from '../logs.js';
+import crawlSitemap from './crawlSitemap.js';
+import { getPdfStoragePath, register } from '../utils.js';
+export const crawlLocalFile = async ({ url, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, extraHTTPHeaders, scanDuration = 0, fromCrawlIntelligentSitemap = false, userUrlInputFromIntelligent = null, datasetFromIntelligent = null, urlsCrawledFromIntelligent = null, }) => {
+    let dataset;
+    let urlsCrawled;
+    let linksFromSitemap = [];
+    let sitemapUrl;
+    let durationExceeded = false;
+    // Boolean to omit axe scan for basic auth URL
+    let isBasicAuth;
+    const basicAuthPage = 0;
+    let finalLinks = [];
+    const { playwrightDeviceDetailsObject } = viewportSettings;
+    if (fromCrawlIntelligentSitemap) {
+        dataset = datasetFromIntelligent;
+        urlsCrawled = urlsCrawledFromIntelligent;
+    }
+    else {
+        ({ dataset } = await createCrawleeSubFolders(randomToken));
+        urlsCrawled = { ...constants.urlsCrawledObj };
+    }
+    // Checks if its in the right file format, and change it before placing into linksFromSitemap
+    url = convertLocalFileToPath(url);
+    // Check if the sitemapUrl is a local file and if it exists
+    if (!fs.existsSync(url) && !isFilePath(url)) {
+        // Convert to an absolute path
+        let normalizedPath = path.resolve(url);
+        // Normalize the path to handle different path separators
+        normalizedPath = path.normalize(normalizedPath);
+        // Check if the normalized path exists
+        if (!fs.existsSync(normalizedPath)) {
+            return;
+        }
+        // At this point, normalizedPath is a valid and existing file path
+        url = normalizedPath;
+    }
+    // XML Files
+    if (!(url.match(/\.xml$/i) || url.match(/\.txt$/i))) {
+        linksFromSitemap = [new Request({ url })];
+        // Non XML file
+    }
+    else {
+        sitemapUrl = url;
+        // Put it to crawlSitemap function to handle xml files
+        const updatedUrlsCrawled = await crawlSitemap({
+            sitemapUrl,
+            randomToken,
+            host,
+            viewportSettings,
+            maxRequestsPerCrawl,
+            browser,
+            userDataDirectory,
+            specifiedMaxConcurrency,
+            fileTypes,
+            blacklistedPatterns,
+            includeScreenshots,
+            extraHTTPHeaders,
+            scanDuration,
+            fromCrawlIntelligentSitemap,
+            userUrlInputFromIntelligent,
+            datasetFromIntelligent,
+            urlsCrawledFromIntelligent,
+            crawledFromLocalFile: true,
+        });
+        urlsCrawled = { ...urlsCrawled, ...updatedUrlsCrawled };
+        return urlsCrawled;
+    }
+    const uuidToPdfMapping = {}; // key and value of string type
+    finalLinks = [...finalLinks, ...linksFromSitemap];
+    await RequestList.open({
+        sources: finalLinks,
+    });
+    const request = linksFromSitemap[0];
+    let shouldAbort = false;
+    if (!isUrlPdf(url)) {
+        const effectiveUserDataDirectory = process.env.CRAWLEE_HEADLESS === '1' ? userDataDirectory : '';
+        const browserContext = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, {
+            headless: process.env.CRAWLEE_HEADLESS === '1',
+            ...getPlaywrightLaunchOptions(browser),
+            ...playwrightDeviceDetailsObject,
+            ...(process.env.OOBEE_DISABLE_BROWSER_DOWNLOAD && { acceptDownloads: false }),
+        });
+        register(browserContext);
+        const timeoutId = scanDuration > 0
+            ? setTimeout(() => {
+                console.log(`Crawl duration of ${scanDuration}s exceeded. Aborting local file scan.`);
+                durationExceeded = true;
+                shouldAbort = true;
+            }, scanDuration * 1000)
+            : null;
+        const page = await browserContext.newPage();
+        url = convertPathToLocalFile(url);
+        await page.goto(url);
+        if (shouldAbort) {
+            console.warn('Scan aborted due to timeout before page scan.');
+            await dataset.pushData({ scanned: [], scannedRedirects: [] });
+            await browserContext.close().catch(() => { });
+            return urlsCrawled;
+        }
+        const results = await runAxeScript({ includeScreenshots, page, randomToken });
+        const actualUrl = page.url() || request.loadedUrl || url;
+        guiInfoLog(guiInfoStatusTypes.SCANNED, {
+            numScanned: urlsCrawled.scanned.length,
+            urlScanned: url,
+        });
+        urlsCrawled.scanned.push({
+            url,
+            pageTitle: results.pageTitle,
+            actualUrl, // i.e. actualUrl
+        });
+        urlsCrawled.scannedRedirects.push({
+            fromUrl: url,
+            toUrl: actualUrl, // i.e. actualUrl
+        });
+        results.url = url;
+        results.actualUrl = actualUrl;
+        await dataset.pushData(results);
+        // Ensure proper cleanup of browser context before PDF generation
+        await browserContext.close().catch(() => { });
+    }
+    else {
+        const pdfFileName = path.basename(url);
+        const destinationFilePath = path.join(getPdfStoragePath(randomToken), pdfFileName);
+        fs.copyFileSync(url, destinationFilePath);
+        uuidToPdfMapping[pdfFileName] = url;
+        urlsCrawled.scanned.push({
+            url,
+            pageTitle: pdfFileName,
+            actualUrl: url,
+        });
+        await runPdfScan(randomToken);
+        // transform result format
+        const pdfResults = await mapPdfScanResults(randomToken, uuidToPdfMapping);
+        // get screenshots from pdf docs
+        if (includeScreenshots) {
+            await Promise.all(pdfResults.map(result => doPdfScreenshots(randomToken, result)));
+        }
+        // push results for each pdf document to key value store
+        await Promise.all(pdfResults.map(result => dataset.pushData(result)));
+    }
+    return { urlsCrawled, durationExceeded };
+};
+export default crawlLocalFile;

package/dist/crawlers/crawlSitemap.js

@@ -0,0 +1,303 @@
+import crawlee, { RequestList } from 'crawlee';
+import * as path from 'path';
+import fsp from 'fs/promises';
+import { createCrawleeSubFolders, preNavigationHooks, runAxeScript, } from './commonCrawlerFunc.js';
+import constants, { STATUS_CODE_METADATA, guiInfoStatusTypes, disallowedListOfPatterns, FileTypes, } from '../constants/constants.js';
+import { getLinksFromSitemap, getPlaywrightLaunchOptions, isSkippedUrl, waitForPageLoaded, isFilePath, } from '../constants/common.js';
+import { areLinksEqual, isWhitelistedContentType, register } from '../utils.js';
+import { handlePdfDownload, runPdfScan, mapPdfScanResults, doPdfScreenshots, } from './pdfScanFunc.js';
+import { guiInfoLog } from '../logs.js';
+const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, extraHTTPHeaders, scanDuration = 0, fromCrawlIntelligentSitemap = false, userUrlInputFromIntelligent = null, datasetFromIntelligent = null, urlsCrawledFromIntelligent = null, crawledFromLocalFile = false, }) => {
+    const crawlStartTime = Date.now();
+    let dataset;
+    let urlsCrawled;
+    let durationExceeded = false;
+    if (fromCrawlIntelligentSitemap) {
+        dataset = datasetFromIntelligent;
+        urlsCrawled = urlsCrawledFromIntelligent;
+    }
+    else {
+        ({ dataset } = await createCrawleeSubFolders(randomToken));
+        urlsCrawled = { ...constants.urlsCrawledObj };
+    }
+    if (!crawledFromLocalFile && isFilePath(sitemapUrl)) {
+        console.log('Local file crawling not supported for sitemap. Please provide a valid URL.');
+        return;
+    }
+    const linksFromSitemap = await getLinksFromSitemap(sitemapUrl, maxRequestsPerCrawl, browser, userDataDirectory, userUrlInputFromIntelligent, fromCrawlIntelligentSitemap, extraHTTPHeaders);
+    sitemapUrl = encodeURI(sitemapUrl);
+    const pdfDownloads = [];
+    const uuidToPdfMapping = {};
+    const isScanHtml = [FileTypes.All, FileTypes.HtmlOnly].includes(fileTypes);
+    const isScanPdfs = [FileTypes.All, FileTypes.PdfOnly].includes(fileTypes);
+    const { playwrightDeviceDetailsObject } = viewportSettings;
+    const { maxConcurrency } = constants;
+    const requestList = await RequestList.open({
+        sources: linksFromSitemap,
+    });
+    const crawler = register(new crawlee.PlaywrightCrawler({
+        launchContext: {
+            launcher: constants.launcher,
+            launchOptions: getPlaywrightLaunchOptions(browser),
+            // Bug in Chrome which causes browser pool crash when userDataDirectory is set in non-headless mode
+            ...(process.env.CRAWLEE_HEADLESS === '1' && { userDataDir: userDataDirectory }),
+        },
+        retryOnBlocked: true,
+        browserPoolOptions: {
+            useFingerprints: false,
+            preLaunchHooks: [
+                async (_pageId, launchContext) => {
+                    const baseDir = userDataDirectory; // e.g., /Users/young/.../Chrome/oobee-...
+                    // Ensure base exists
+                    await fsp.mkdir(baseDir, { recursive: true });
+                    // Create a unique subdir per browser
+                    const subProfileDir = path.join(baseDir, `profile-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
+                    await fsp.mkdir(subProfileDir, { recursive: true });
+                    // Assign to Crawlee's launcher
+                    launchContext.userDataDir = subProfileDir;
+                    // Safely extend launchOptions
+                    launchContext.launchOptions = {
+                        ...launchContext.launchOptions,
+                        ignoreHTTPSErrors: true,
+                        ...playwrightDeviceDetailsObject,
+                        ...(process.env.OOBEE_DISABLE_BROWSER_DOWNLOAD && { acceptDownloads: false }),
+                    };
+                    // Optionally log for debugging
+                    // console.log(`[HOOK] Using userDataDir: ${subProfileDir}`);
+                },
+            ],
+        },
+        requestList,
+        postNavigationHooks: [
+            async ({ page }) => {
+                try {
+                    // Wait for a quiet period in the DOM, but with safeguards
+                    await page.evaluate(() => {
+                        return new Promise(resolve => {
+                            let timeout;
+                            let mutationCount = 0;
+                            const MAX_MUTATIONS = 500; // stop if things never quiet down
+                            const OBSERVER_TIMEOUT = 5000; // hard cap on total wait
+                            const observer = new MutationObserver(() => {
+                                clearTimeout(timeout);
+                                mutationCount++;
+                                if (mutationCount > MAX_MUTATIONS) {
+                                    observer.disconnect();
+                                    resolve('Too many mutations, exiting.');
+                                    return;
+                                }
+                                // restart quiet‑period timer
+                                timeout = setTimeout(() => {
+                                    observer.disconnect();
+                                    resolve('DOM stabilized.');
+                                }, 1000);
+                            });
+                            // overall timeout in case the page never settles
+                            timeout = setTimeout(() => {
+                                observer.disconnect();
+                                resolve('Observer timeout reached.');
+                            }, OBSERVER_TIMEOUT);
+                            const root = document.documentElement || document.body || document;
+                            if (!root || typeof observer.observe !== 'function') {
+                                resolve('No root node to observe.');
+                            }
+                        });
+                    });
+                }
+                catch (err) {
+                    // Handle page navigation errors gracefully
+                    if (err.message.includes('was destroyed')) {
+                        return; // Page navigated or closed, no need to handle
+                    }
+                    throw err; // Rethrow unknown errors
+                }
+            },
+        ],
+        preNavigationHooks: [
+            async ({ request, page }, gotoOptions) => {
+                const url = request.url.toLowerCase();
+                const isNotSupportedDocument = disallowedListOfPatterns.some(pattern => url.startsWith(pattern));
+                if (isNotSupportedDocument) {
+                    request.skipNavigation = true;
+                    request.userData.isNotSupportedDocument = true;
+                    // Log for verification (optional, but not required for correctness)
+                    // console.log(`[SKIP] Not supported: ${request.url}`);
+                    return;
+                }
+                preNavigationHooks(extraHTTPHeaders);
+            },
+        ],
+        requestHandlerTimeoutSecs: 90,
+        requestHandler: async ({ page, request, response, sendRequest }) => {
+            // Log documents that are not supported
+            if (request.userData?.isNotSupportedDocument) {
+                guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                    numScanned: urlsCrawled.scanned.length,
+                    urlScanned: request.url,
+                });
+                urlsCrawled.userExcluded.push({
+                    url: request.url,
+                    pageTitle: request.url,
+                    actualUrl: request.url, // because about:blank is not useful
+                    metadata: STATUS_CODE_METADATA[1],
+                    httpStatusCode: 1,
+                });
+                return;
+            }
+            await waitForPageLoaded(page, 10000);
+            const actualUrl = page.url() || request.loadedUrl || request.url;
+            const hasExceededDuration = scanDuration > 0 && Date.now() - crawlStartTime > scanDuration * 1000;
+            if (urlsCrawled.scanned.length >= maxRequestsPerCrawl || hasExceededDuration) {
+                if (hasExceededDuration) {
+                    console.log(`Crawl duration of ${scanDuration}s exceeded. Aborting sitemap crawl.`);
+                    durationExceeded = true;
+                }
+                crawler.autoscaledPool.abort(); // stops new requests
+                return;
+            }
+            if (request.skipNavigation && actualUrl === 'about:blank') {
+                if (isScanPdfs) {
+                    // pushes download promise into pdfDownloads
+                    const { pdfFileName, url } = handlePdfDownload(randomToken, pdfDownloads, request, sendRequest, urlsCrawled);
+                    uuidToPdfMapping[pdfFileName] = url;
+                    return;
+                }
+                guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                    numScanned: urlsCrawled.scanned.length,
+                    urlScanned: request.url,
+                });
+                urlsCrawled.userExcluded.push({
+                    url: request.url,
+                    pageTitle: request.url,
+                    actualUrl: request.url, // because about:blank is not useful
+                    metadata: STATUS_CODE_METADATA[1],
+                    httpStatusCode: 1,
+                });
+                return;
+            }
+            const contentType = response?.headers?.()['content-type'] || '';
+            const status = response ? response.status() : 0;
+            if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
+                const isRedirected = !areLinksEqual(page.url(), request.url);
+                const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(item => (item.actualUrl || item.url) === page.url());
+                if (isRedirected && isLoadedUrlInCrawledUrls) {
+                    urlsCrawled.notScannedRedirects.push({
+                        fromUrl: request.url,
+                        toUrl: actualUrl, // i.e. actualUrl
+                    });
+                    return;
+                }
+                // This logic is different from crawlDomain, as it also checks if the pae is redirected before checking if it is excluded using exclusions.txt
+                if (isRedirected && blacklistedPatterns && isSkippedUrl(actualUrl, blacklistedPatterns)) {
+                    urlsCrawled.userExcluded.push({
+                        url: request.url,
+                        pageTitle: request.url,
+                        actualUrl,
+                        metadata: STATUS_CODE_METADATA[0],
+                        httpStatusCode: 0,
+                    });
+                    guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
+                    });
+                    return;
+                }
+                const results = await runAxeScript({ includeScreenshots, page, randomToken });
+                guiInfoLog(guiInfoStatusTypes.SCANNED, {
+                    numScanned: urlsCrawled.scanned.length,
+                    urlScanned: request.url,
+                });
+                urlsCrawled.scanned.push({
+                    url: request.url,
+                    pageTitle: results.pageTitle,
+                    actualUrl, // i.e. actualUrl
+                });
+                urlsCrawled.scannedRedirects.push({
+                    fromUrl: request.url,
+                    toUrl: actualUrl,
+                });
+                results.url = request.url;
+                results.actualUrl = actualUrl;
+                await dataset.pushData(results);
+            }
+            else {
+                guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                    numScanned: urlsCrawled.scanned.length,
+                    urlScanned: request.url,
+                });
+                if (isScanHtml) {
+                    // carry through the HTTP status metadata
+                    const status = response?.status();
+                    const metadata = typeof status === 'number'
+                        ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
+                        : STATUS_CODE_METADATA[2];
+                    urlsCrawled.invalid.push({
+                        actualUrl,
+                        url: request.url,
+                        pageTitle: request.url,
+                        metadata,
+                        httpStatusCode: typeof status === 'number' ? status : 0,
+                    });
+                }
+            }
+        },
+        failedRequestHandler: async ({ request, response, error }) => {
+            // check if scanned pages have reached limit due to multi-instances of handler running
+            if (urlsCrawled.scanned.length >= maxRequestsPerCrawl) {
+                return;
+            }
+            guiInfoLog(guiInfoStatusTypes.ERROR, {
+                numScanned: urlsCrawled.scanned.length,
+                urlScanned: request.url,
+            });
+            const status = response?.status();
+            const metadata = typeof status === 'number'
+                ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
+                : STATUS_CODE_METADATA[2];
+            urlsCrawled.error.push({
+                url: request.url,
+                pageTitle: request.url,
+                actualUrl: request.url,
+                metadata,
+                httpStatusCode: typeof status === 'number' ? status : 0,
+            });
+            crawlee.log.error(`Failed Request - ${request.url}: ${request.errorMessages}`);
+        },
+        maxRequestsPerCrawl: Infinity,
+        maxConcurrency: specifiedMaxConcurrency || maxConcurrency,
+        ...(process.env.OOBEE_FAST_CRAWLER && {
+            autoscaledPoolOptions: {
+                minConcurrency: specifiedMaxConcurrency ? Math.min(specifiedMaxConcurrency, 10) : 10,
+                maxConcurrency: specifiedMaxConcurrency || maxConcurrency,
+                desiredConcurrencyRatio: 0.98, // Increase threshold for scaling up
+                scaleUpStepRatio: 0.99, // Scale up faster
+                scaleDownStepRatio: 0.1, // Scale down slower
+            },
+        }),
+    }));
+    await crawler.run();
+    await requestList.isFinished();
+    if (pdfDownloads.length > 0) {
+        // wait for pdf downloads to complete
+        await Promise.all(pdfDownloads);
+        // scan and process pdf documents
+        await runPdfScan(randomToken);
+        // transform result format
+        const pdfResults = await mapPdfScanResults(randomToken, uuidToPdfMapping);
+        // get screenshots from pdf docs
+        if (includeScreenshots) {
+            await Promise.all(pdfResults.map(async (result) => await doPdfScreenshots(randomToken, result)));
+        }
+        // push results for each pdf document to key value store
+        await Promise.all(pdfResults.map(result => dataset.pushData(result)));
+    }
+    if (!fromCrawlIntelligentSitemap) {
+        guiInfoLog(guiInfoStatusTypes.COMPLETED, {});
+    }
+    if (scanDuration > 0) {
+        const elapsed = Math.round((Date.now() - crawlStartTime) / 1000);
+        console.log(`Crawl ended after ${elapsed}s (limit: ${scanDuration}s).`);
+    }
+    return { urlsCrawled, durationExceeded };
+};
+export default crawlSitemap;

package/dist/crawlers/custom/escapeCssSelector.js

@@ -0,0 +1,10 @@
+// for css id selectors starting with a digit, escape it with the unicode character e.g. #123 -> #\31 23
+export function escapeCssSelector(selector) {
+    try {
+        return selector.replace(/([#\.])(\d)/g, (_match, prefix, digit) => `${prefix}\\3${digit} `);
+    }
+    catch (e) {
+        console.error(`error escaping css selector: ${selector}`, e);
+        return selector;
+    }
+}

package/dist/crawlers/custom/evaluateAltText.js

@@ -0,0 +1,11 @@
+export function evaluateAltText(node) {
+    const altText = node.getAttribute('alt');
+    const confusingTexts = ['img', 'image', 'picture', 'photo', 'graphic'];
+    if (altText) {
+        const trimmedAltText = altText.trim().toLowerCase();
+        if (confusingTexts.includes(trimmedAltText)) {
+            return false;
+        }
+    }
+    return true;
+}

package/dist/crawlers/custom/extractAndGradeText.js

@@ -0,0 +1,44 @@
+import textReadability from 'text-readability';
+export async function extractAndGradeText(page) {
+    try {
+        // Extract text content from all specified elements (e.g., paragraphs)
+        const sentences = await page.evaluate(() => {
+            const elements = document.querySelectorAll('p'); // Adjust selector as needed
+            const extractedSentences = [];
+            elements.forEach(element => {
+                const text = element.innerText.trim();
+                // Split the text into individual sentences
+                const sentencePattern = /[^.!?]*[.!?]+/g; // Match sentences ending with ., !, or ?
+                const matches = text.match(sentencePattern);
+                if (matches) {
+                    // Add only sentences that end with punctuation
+                    matches.forEach(sentence => {
+                        const trimmedSentence = sentence.trim(); // Trim whitespace from each sentence
+                        if (trimmedSentence.length > 0) {
+                            extractedSentences.push(trimmedSentence);
+                        }
+                    });
+                }
+            });
+            return extractedSentences;
+        });
+        // Check if any valid sentences were extracted
+        if (sentences.length === 0) {
+            return ''; // Return an empty string if no valid sentences are found
+        }
+        // Join the valid sentences into a single string
+        const filteredText = sentences.join(' ').trim();
+        // Count the total number of words in the filtered text
+        const wordCount = filteredText.split(/\s+/).length;
+        // Grade the text content only if there are 20 words or more
+        const readabilityScore = wordCount >= 20 ? textReadability.fleschReadingEase(filteredText) : 0;
+        // Log details for debugging
+        // Determine the return value
+        const result = readabilityScore === 0 || readabilityScore > 50 ? '' : readabilityScore.toString(); // Convert readabilityScore to string
+        return result;
+    }
+    catch (error) {
+        console.error('Error extracting and grading text:', error);
+        return ''; // Return an empty string in case of an error
+    }
+}