@govtechsg/oobee 0.10.98 → 0.11.0

package/dist/crawlers/crawlDomain.js

@@ -1,6 +1,6 @@
 import crawlee from 'crawlee';
 import { CrawlRateController } from './crawlRateController.js';
-import { createCrawleeSubFolders, getPreLaunchHook, preNavigationHooks, runAxeScript, isUrlPdf, shouldSkipClickDueToDisallowedHref, shouldSkipDueToUnsupportedContent, splitAuthHeaders, } from './commonCrawlerFunc.js';
+import { createCrawleeSubFolders, getPreLaunchHook, getPostPageCloseHook, preNavigationHooks, runAxeScript, isUrlPdf, shouldSkipClickDueToDisallowedHref, shouldSkipDueToUnsupportedContent, splitAuthHeaders, } from './commonCrawlerFunc.js';
 import constants, { blackListedFileExtensions, guiInfoStatusTypes, cssQuerySelectors, STATUS_CODE_METADATA, disallowedListOfPatterns, disallowedSelectorPatterns, FileTypes, } from '../constants/constants.js';
 import { getPlaywrightLaunchOptions, isBlacklistedFileExtensions, isSkippedUrl, isDisallowedInRobotsTxt, getUrlsFromRobotsTxt, waitForPageLoaded, } from '../constants/common.js';
 import { areLinksEqual, isFollowStrategy, isSameHostname, normUrl, register } from '../utils.js';
@@ -304,12 +304,24 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                     };
                 },
             ],
+            postPageCloseHooks: [getPostPageCloseHook(userDataDirectory)],
         },
         requestQueue,
         maxRequestRetries: 3,
         maxSessionRotations: 1,
         preNavigationHooks: [
             ...preNavigationHooks(extraHTTPHeaders),
+            async ({ request }) => {
+                const url = request.url.toLowerCase();
+                try {
+                    const pathname = new URL(url).pathname;
+                    const ext = pathname.split('.').pop();
+                    if (ext && blackListedFileExtensions.includes(ext)) {
+                        request.skipNavigation = true;
+                    }
+                }
+                catch { }
+            },
         ],
         postNavigationHooks: [
             async (crawlingContext) => {
@@ -415,20 +427,17 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                 if (shouldSkipDueToUnsupportedContent(response, request.url) ||
                     (request.skipNavigation && actualUrl === 'about:blank')) {
                     if (!isScanPdfs) {
-                        // Don't inform the user it is skipped since web crawler is best-effort.
-                        /*
-                      guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-                        numScanned: urlsCrawled.scanned.length,
-                        urlScanned: request.url,
-                      });
-                      urlsCrawled.userExcluded.push({
-                        url: request.url,
-                        pageTitle: request.url,
-                        actualUrl: request.url, // because about:blank is not useful
-                        metadata: STATUS_CODE_METADATA[1],
-                        httpStatusCode: 0,
-                      });
-                      */
+                        guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                            numScanned: urlsCrawled.scanned.length,
+                            urlScanned: request.url,
+                        });
+                        urlsCrawled.userExcluded.push({
+                            url: request.url,
+                            pageTitle: request.url,
+                            actualUrl: request.url,
+                            metadata: STATUS_CODE_METADATA[1],
+                            httpStatusCode: 1,
+                        });
                         return;
                     }
                     const { pdfFileName, url: downloadedPdfUrl } = handlePdfDownload(randomToken, pdfDownloads, request, sendRequest, urlsCrawled);
@@ -436,20 +445,17 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                     return;
                 }
                 if (isBlacklistedFileExtensions(actualUrl, blackListedFileExtensions)) {
-                    // Don't inform the user it is skipped since web crawler is best-effort.
-                    /*
-                  guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-                    numScanned: urlsCrawled.scanned.length,
-                    urlScanned: request.url,
-                  });
-                  urlsCrawled.userExcluded.push({
-                    url: request.url,
-                    pageTitle: request.url,
-                    actualUrl, // because about:blank is not useful
-                    metadata: STATUS_CODE_METADATA[1],
-                    httpStatusCode: 0,
-                  });
-                  */
+                    guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
+                    });
+                    urlsCrawled.userExcluded.push({
+                        url: request.url,
+                        pageTitle: request.url,
+                        actualUrl,
+                        metadata: STATUS_CODE_METADATA[1],
+                        httpStatusCode: 1,
+                    });
                     return;
                 }
                 if (!isFollowStrategy(url, actualUrl, strategy) &&
@@ -634,7 +640,57 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
             if (isAbortingScanNow) {
                 return;
             }
+            // Handle download-triggered navigation errors: Playwright throws
+            // "Download is starting" when page.goto() hits a file download URL.
+            // Crawlee retries 3 times (all fail) then lands here.
+            const isDownloadError = request.errorMessages?.some((msg) => msg.includes('Download is starting'));
+            if (isDownloadError) {
+                if (isScanPdfs) {
+                    // Re-enqueue with skipNavigation so the requestHandler's PDF download path handles it
+                    try {
+                        await requestQueue.addRequest({
+                            url: request.url,
+                            skipNavigation: true,
+                            label: request.url,
+                            uniqueKey: `download_${request.url}`,
+                        });
+                    }
+                    catch { }
+                }
+                else {
+                    guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
+                    });
+                    urlsCrawled.userExcluded.push({
+                        url: request.url,
+                        pageTitle: request.url,
+                        actualUrl: request.url,
+                        metadata: STATUS_CODE_METADATA[1],
+                        httpStatusCode: 1,
+                    });
+                }
+                return;
+            }
             const status = response?.status();
+            // Re-enqueue rate-limited (403) URLs once for a retry after concurrency recovers.
+            // Without this, URLs that fail during a rate-limit burst are permanently lost
+            // even though the site is accessible at lower concurrency.
+            // Don't call onFailure here — the re-enqueued request will get a fresh attempt.
+            // If it fails again (rateLimitRetried=true), it falls through to the normal
+            // onFailure + circuit breaker path below.
+            if (status === 403 && !request.userData?.rateLimitRetried) {
+                try {
+                    await requestQueue.addRequest({
+                        url: request.url,
+                        label: request.url,
+                        uniqueKey: `ratelimit_${request.url}`,
+                        userData: { rateLimitRetried: true },
+                    });
+                }
+                catch { }
+                return;
+            }
             if (rateController.onFailure(status, crawler.autoscaledPool)) {
                 consoleLogger.info(`Aborting crawl: consecutive HTTP failures threshold reached (site may be rate-limiting). Successfully scanned ${urlsCrawled.scanned.length} pages.`);
                 isAbortingScanNow = true;
@@ -669,7 +725,9 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
     await crawler.run();
     // Additional passes: keep re-visiting scanned seed-hostname pages for
     // click-discovery until no new pages are found or limits are reached.
-    if (!safeMode && !isAbortingScanNow && !durationExceeded) {
+    // Skip when called from intelligent sitemap — the domain phase is only meant
+    // to discover new pages via <a> links, not re-click 3000+ already-scanned pages.
+    if (!safeMode && !isAbortingScanNow && !durationExceeded && !fromCrawlIntelligentSitemap) {
         const seedHostname = new URL(url).hostname;
         const clickPassVisited = new Set();
         let prevScannedCount;

package/dist/crawlers/crawlSitemap.js

@@ -1,8 +1,8 @@
 import crawlee, { EnqueueStrategy, RequestList } from 'crawlee';
 import { CrawlRateController } from './crawlRateController.js';
-import { createCrawleeSubFolders, getPreLaunchHook, preNavigationHooks, runAxeScript, splitAuthHeaders, } from './commonCrawlerFunc.js';
-import constants, { STATUS_CODE_METADATA, guiInfoStatusTypes, disallowedListOfPatterns, FileTypes, } from '../constants/constants.js';
-import { getLinksFromSitemap, getPlaywrightLaunchOptions, isSkippedUrl, waitForPageLoaded, isFilePath, } from '../constants/common.js';
+import { createCrawleeSubFolders, getPreLaunchHook, getPostPageCloseHook, preNavigationHooks, runAxeScript, isUrlPdf, splitAuthHeaders, } from './commonCrawlerFunc.js';
+import constants, { STATUS_CODE_METADATA, guiInfoStatusTypes, blackListedFileExtensions, disallowedListOfPatterns, disallowedSelectorPatterns, FileTypes, } from '../constants/constants.js';
+import { getLinksFromSitemap, getPlaywrightLaunchOptions, isDisallowedInRobotsTxt, isSkippedUrl, waitForPageLoaded, isFilePath, } from '../constants/common.js';
 import { areLinksEqual, isFollowStrategy, isWhitelistedContentType, normUrl, register } from '../utils.js';
 import { handlePdfDownload, runPdfScan, mapPdfScanResults, doPdfScreenshots, } from './pdfScanFunc.js';
 import { consoleLogger, guiInfoLog } from '../logs.js';
@@ -41,6 +41,11 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
     const requestList = await RequestList.open({
         sources: linksFromSitemap,
     });
+    // Always create a request queue alongside the request list. An empty queue
+    // has zero impact on crawl behavior (Crawlee processes RequestList first).
+    // Having it available enables: download re-enqueue for PDF scanning,
+    // 403 rate-limit retry, and enqueueLinks for intelligent sitemap discovery.
+    const { requestQueue } = await createCrawleeSubFolders(randomToken);
     const crawler = register(new crawlee.PlaywrightCrawler({
         launchContext: {
             launcher: constants.launcher,
@@ -65,8 +70,10 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                     };
                 },
             ],
+            postPageCloseHooks: [getPostPageCloseHook(userDataDirectory)],
         },
         requestList,
+        requestQueue,
         maxRequestRetries: 3,
         maxSessionRotations: 1,
         postNavigationHooks: [
@@ -125,14 +132,22 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                 if (isNotSupportedDocument) {
                     request.skipNavigation = true;
                     request.userData.isNotSupportedDocument = true;
-                    // Log for verification (optional, but not required for correctness)
-                    // console.log(`[SKIP] Not supported: ${request.url}`);
                     return;
                 }
+                try {
+                    const pathname = new URL(url).pathname;
+                    const ext = pathname.split('.').pop();
+                    if (ext && blackListedFileExtensions.includes(ext)) {
+                        request.skipNavigation = true;
+                        request.userData.isNotSupportedDocument = true;
+                        return;
+                    }
+                }
+                catch { }
             },
         ],
         requestHandlerTimeoutSecs: 90,
-        requestHandler: async ({ page, request, response, sendRequest }) => {
+        requestHandler: async ({ page, request, response, sendRequest, enqueueLinks }) => {
             // Log documents that are not supported
             if (request.userData?.isNotSupportedDocument) {
                 guiInfoLog(guiInfoStatusTypes.SKIPPED, {
@@ -268,6 +283,34 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                         results.url = request.url;
                         results.actualUrl = actualUrl;
                         await dataset.pushData(results);
+                        // Discover <a> links from this page for the intelligent sitemap flow.
+                        // This eliminates the need for a separate crawlDomain supplement phase
+                        // that would re-visit all these pages just to extract links.
+                        if (fromCrawlIntelligentSitemap) {
+                            try {
+                                await enqueueLinks({
+                                    selector: `a:not(${disallowedSelectorPatterns})`,
+                                    strategy,
+                                    requestQueue,
+                                    transformRequestFunction: (req) => {
+                                        try {
+                                            req.url = req.url.replace(/(?<=&|\?)utm_.*?(&|$)/gim, '');
+                                        }
+                                        catch { }
+                                        if (isDisallowedInRobotsTxt(req.url))
+                                            return null;
+                                        if (isUrlPdf(req.url)) {
+                                            req.skipNavigation = true;
+                                        }
+                                        req.label = req.url;
+                                        return req;
+                                    },
+                                });
+                            }
+                            catch {
+                                // Best-effort link discovery; don't fail the scan
+                            }
+                        }
                     }
                 }
                 else {
@@ -276,18 +319,34 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                         urlScanned: request.url,
                     });
                     if (isScanHtml) {
-                        // carry through the HTTP status metadata
-                        const status = response?.status();
-                        const metadata = typeof status === 'number'
-                            ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
-                            : STATUS_CODE_METADATA[2];
-                        urlsCrawled.invalid.push({
-                            actualUrl,
-                            url: request.url,
-                            pageTitle: request.url,
-                            metadata,
-                            httpStatusCode: typeof status === 'number' ? status : 0,
-                        });
+                        // Non-HTML content types (images, PDFs, binary files) and URLs
+                        // that redirect to non-HTML resources should be classified as
+                        // unsupported documents, not generic page errors.
+                        const isNonHtmlContent = contentType &&
+                            !contentType.startsWith('text/html') &&
+                            !contentType.includes('html');
+                        if (isNonHtmlContent && status !== 0) {
+                            urlsCrawled.userExcluded.push({
+                                actualUrl,
+                                url: request.url,
+                                pageTitle: request.url,
+                                metadata: STATUS_CODE_METADATA[1],
+                                httpStatusCode: 1,
+                            });
+                        }
+                        else {
+                            const httpStatus = response?.status();
+                            const metadata = typeof httpStatus === 'number'
+                                ? STATUS_CODE_METADATA[httpStatus] || STATUS_CODE_METADATA[599]
+                                : STATUS_CODE_METADATA[2];
+                            urlsCrawled.invalid.push({
+                                actualUrl,
+                                url: request.url,
+                                pageTitle: request.url,
+                                metadata,
+                                httpStatusCode: typeof httpStatus === 'number' ? httpStatus : 0,
+                            });
+                        }
                     }
                 }
             }
@@ -302,7 +361,53 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
             if (isAbortingScan) {
                 return;
             }
+            // Handle download-triggered navigation errors: Playwright throws
+            // "Download is starting" when page.goto() hits a file download URL.
+            const isDownloadError = request.errorMessages?.some((msg) => msg.includes('Download is starting'));
+            if (isDownloadError) {
+                if (isScanPdfs) {
+                    // Re-enqueue with skipNavigation so the requestHandler's PDF download path handles it
+                    try {
+                        await requestQueue.addRequest({
+                            url: request.url,
+                            skipNavigation: true,
+                            label: request.url,
+                            uniqueKey: `download_${request.url}`,
+                        });
+                    }
+                    catch { }
+                }
+                else {
+                    guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
+                    });
+                    urlsCrawled.userExcluded.push({
+                        url: request.url,
+                        pageTitle: request.url,
+                        actualUrl: request.url,
+                        metadata: STATUS_CODE_METADATA[1],
+                        httpStatusCode: 1,
+                    });
+                }
+                return;
+            }
             const status = response?.status();
+            // Re-enqueue rate-limited (403) URLs once for a retry after concurrency recovers.
+            // Don't call onFailure here — the re-enqueued request gets a fresh attempt.
+            // If it fails again (rateLimitRetried=true), it falls through to the normal path.
+            if (status === 403 && !request.userData?.rateLimitRetried) {
+                try {
+                    await requestQueue.addRequest({
+                        url: request.url,
+                        label: request.url,
+                        uniqueKey: `ratelimit_${request.url}`,
+                        userData: { rateLimitRetried: true },
+                    });
+                }
+                catch { }
+                return;
+            }
             if (rateController.onFailure(status, crawler.autoscaledPool)) {
                 consoleLogger.info(`Aborting crawl: consecutive HTTP failures threshold reached (site may be rate-limiting). Successfully scanned ${urlsCrawled.scanned.length} pages.`);
                 isAbortingScan = true;

package/dist/mergeAxeResults.js

@@ -831,7 +831,7 @@ generateJsonFiles = false, preferredBrowser) => {
         consoleLogger.info(`Dataset drop: ${error.message}`);
     }
     try {
-        const requestQueue = await RequestQueue.open(crawleeDir);
+        const requestQueue = await RequestQueue.open(`${crawleeDir}_rq`);
         await requestQueue.drop();
     }
     catch (error) {
@@ -841,6 +841,7 @@ generateJsonFiles = false, preferredBrowser) => {
     const crawleePath = path.join(storagePath, 'crawlee');
     try {
         await fs.promises.rm(crawleePath, { recursive: true, force: true });
+        await fs.promises.rm(`${crawleePath}_rq`, { recursive: true, force: true });
     }
     catch {
         // Best-effort; storage was already dropped via API

package/dist/utils.js

@@ -346,15 +346,34 @@ export const cleanUp = async (randomToken, isError = false) => {
         catch (error) {
             consoleLogger.warn(`Unable to force remove userDataDirectory: ${error.message}`);
         }
-        // Also remove _pool* sibling directories created by browser pool re-launches
-        try {
-            const poolDirs = globSync(`${constants.userDataDirectory}_pool*`);
+        // Also remove _pool* sibling directories created by browser pool re-launches.
+        // On Windows, Chrome writes files (optimization_guide_model_store,
+        // segmentation_platform, first_party_sets.db, etc.) during its shutdown
+        // sequence and may still hold file locks. Retry up to 10 times at 5s intervals.
+        const removePoolDirs = () => {
+            // On Windows, glob interprets backslashes as escape characters, not path
+            // separators. Normalize to forward slashes so the pattern actually matches.
+            const globPattern = `${constants.userDataDirectory}_pool*`.replace(/\\/g, '/');
+            const poolDirs = globSync(globPattern);
             for (const dir of poolDirs) {
                 fs.rmSync(dir, { recursive: true, force: true });
             }
-        }
-        catch (error) {
-            consoleLogger.warn(`Unable to remove pool directories: ${error.message}`);
+            return poolDirs.length;
+        };
+        for (let attempt = 0; attempt < 10; attempt++) {
+            try {
+                if (removePoolDirs() === 0)
+                    break; // nothing left to clean
+                break; // rmSync succeeded for all dirs
+            }
+            catch {
+                if (attempt < 9) {
+                    await new Promise(r => setTimeout(r, 5000));
+                }
+                else {
+                    consoleLogger.warn('Unable to remove pool directories after 10 attempts');
+                }
+            }
         }
     }
     if (process.env.TMPDIR && fs.existsSync('/.dockerenv'))
@@ -374,7 +393,7 @@ export const cleanUp = async (randomToken, isError = false) => {
             const crawleeDir = path.join(storagePath, 'crawlee');
             const dataset = await Dataset.open(crawleeDir);
             await dataset.drop();
-            const requestQueue = await RequestQueue.open(crawleeDir);
+            const requestQueue = await RequestQueue.open(`${crawleeDir}_rq`);
             await requestQueue.drop();
         }
         catch (error) {
@@ -386,6 +405,12 @@ export const cleanUp = async (randomToken, isError = false) => {
         catch (error) {
             consoleLogger.warn(`Unable to force remove crawlee folder: ${error.message}`);
         }
+        try {
+            fs.rmSync(path.join(storagePath, 'crawlee_rq'), { recursive: true, force: true });
+        }
+        catch (error) {
+            consoleLogger.warn(`Unable to force remove crawlee_rq folder: ${error.message}`);
+        }
         try {
             fs.rmSync(path.join(storagePath, 'pdfs'), { recursive: true, force: true });
         }
@@ -808,9 +833,11 @@ export const zipResults = async (zipName, resultsPath) => {
     if (!stats.isDirectory()) {
         throw new Error(`resultsPath is not a directory: ${resultsPath}`);
     }
-    async function addFolderToZip(folderPath, zipFolder) {
+    async function addFolderToZip(folderPath, zipFolder, isRoot = false) {
         const items = await fs.readdir(folderPath);
         for (const item of items) {
+            if (isRoot && item.startsWith('crawlee'))
+                continue;
             const fullPath = path.join(folderPath, item);
             const stats = await fs.stat(fullPath);
             if (stats.isDirectory()) {
@@ -824,7 +851,7 @@ export const zipResults = async (zipName, resultsPath) => {
         }
     }
     const zip = new JSZip();
-    await addFolderToZip(resultsPath, zip);
+    await addFolderToZip(resultsPath, zip, true);
     const zipStream = zip.generateNodeStream({
         type: 'nodebuffer',
         streamFiles: true,

package/oobee-client-scanner.js

@@ -3,7 +3,7 @@
  * DO NOT EDIT MANUALLY. Re-generate with: node dist/generateOobeeClientScanner.js
  *
  * Embedded at generation time:
- *   App version : 0.10.98
+ *   App version : 0.11.0
  *   Sentry DSN  : (from OOBEE_SENTRY_DSN env var or constants.ts default)
  *   Sentry SDK  : @sentry/browser 10.58.0 (loaded from CDN at runtime)
  *
@@ -34887,7 +34887,7 @@
   // ── Sentry browser telemetry (Sentry JS SDK, loaded from CDN) ────────────
   
   var _oobeeSentryDsn          = "https://3b8c7ee46b06f33815a1301b6713ebc3@o4509047624761344.ingest.us.sentry.io/4509327783559168";
-  var _oobeeAppVersion         = "0.10.98";
+  var _oobeeAppVersion         = "0.11.0";
   var _oobeeSentryVersion      = "10.58.0";
   var _oobeeSentryInitialized  = false;
   var _oobeeSentryLoadPromise  = null;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.98",
+  "version": "0.11.0",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "bin": {

package/src/combine.ts

@@ -96,6 +96,13 @@ const combineRun = async (details: Data, deviceToScan: string) => {
       consoleLogger.info(`Suppressed Playwright post-close connection error: ${err.message}`);
       return;
     }
+    // Suppress EPERM errors from Crawlee's async lock-file operations that fire
+    // after a crawl phase finishes. On Windows, the sitemap phase's async lock
+    // cleanup can race with the domain phase starting in the same directory.
+    if (err.message?.includes('EPERM') && err.message?.includes('.json.lock')) {
+      consoleLogger.info(`Suppressed Crawlee lock-file EPERM (stale async cleanup): ${err.message}`);
+      return;
+    }
     throw err;
   };
   process.on('uncaughtException', psTreeHandler);

package/src/constants/common.ts

@@ -2157,6 +2157,15 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => {
       !(browser === BrowserTypes.CHROME && arg === '--edge-skip-compat-layer-relaunch'),
   );
 
+  // Cap browser disk cache to 10MB per instance to prevent storage bloat
+  // during long crawls with multiple pool rotations
+  finalArgs.push('--disk-cache-size=10485760');
+
+  // Prevent Windows from throttling background Chromium processes
+  if (os.platform() === 'win32') {
+    finalArgs.push('--disable-features=UseEcoQoSForBackgroundProcess');
+  }
+
   // Headless flags (unchanged)
   if (process.env.CRAWLEE_HEADLESS === '1') {
     if (!finalArgs.includes('--mute-audio')) finalArgs.push('--mute-audio');

package/src/constants/constants.ts

@@ -30,10 +30,27 @@ export const blackListedFileExtensions = [
   'svg',
   'gif',
   'woff',
+  'woff2',
   'zip',
   'webp',
   'json',
   'xml',
+  'ico',
+  'bmp',
+  'tiff',
+  'tif',
+  'avi',
+  'mov',
+  'wmv',
+  'flv',
+  'ogg',
+  'wav',
+  'doc',
+  'docx',
+  'xls',
+  'xlsx',
+  'ppt',
+  'pptx',
 ];
 
 export const getIntermediateScreenshotsPath = (datasetsPath: string): string =>