@govtechsg/oobee 0.10.42 → 0.10.43

package/REPORTS.md

@@ -242,7 +242,9 @@ This file contains a summary of pages affected by accessibility issues.
     {
       "url": "<string>",
       "pageTitle": "<string>",
-      "actualUrl": "about:blank"
+      "actualUrl": "<string>",
+      "metadata": "<string>",
+      "httpStatusCode": number
     },
   ],
   "pagesNotScannedCount": <number>
@@ -340,7 +342,9 @@ This file contains a summary of accessibility issues found in a scan, categorize
     {
       "url": "<string>",
       "pageTitle": "<string>",
-      "actualUrl": "about:blank"
+      "actualUrl": "<string>",
+      "metadata": "<string>",
+      "httpStatusCode": number
     },
   ],
   "pagesNotScannedCount": <number>
@@ -360,3 +364,68 @@ To deflate the .json.gz.b64, use the following with `pako` library installed:
   // Parse and return the JSON object
   return JSON.parse(jsonString);
 ```
+
+## HTTP Status Codes Returned for Skipped Pages
+In scanPagesSummary.json and scanPagesDetail,json, within each `pagesNotScanned`, the following HTTP and Metadata is stored to provide a reason why the apge could not be scanned.
+
+| httpStatusCode | metadata                                    |
+|------|------------------------------------------------|
+| 0    | Page Excluded                                  |
+| 1    | Not A Supported Document                       |
+| 2    | Web Crawler Errored                            |
+| 100  | 100 – Continue                                 |
+| 101  | 101 – Switching Protocols                      |
+| 102  | 102 – Processing                               |
+| 103  | 103 – Early Hints                              |
+| 200  | 200 – However Page Could Not Be Scanned        |
+| 204  | 204 – No Content                               |
+| 205  | 205 – Reset Content                            |
+| 300  | 300 – Multiple Choices                         |
+| 301  | 301 – Moved Permanently                        |
+| 302  | 302 – Found                                    |
+| 303  | 303 – See Other                                |
+| 304  | 304 – Not Modified                             |
+| 305  | 305 – Use Proxy                                |
+| 307  | 307 – Temporary Redirect                       |
+| 308  | 308 – Permanent Redirect                       |
+| 400  | 400 – Bad Request                              |
+| 401  | 401 – Unauthorized                             |
+| 402  | 402 – Payment Required                         |
+| 403  | 403 – Forbidden                                |
+| 404  | 404 – Not Found                                |
+| 405  | 405 – Method Not Allowed                       |
+| 406  | 406 – Not Acceptable                           |
+| 407  | 407 – Proxy Authentication Required            |
+| 408  | 408 – Request Timeout                          |
+| 409  | 409 – Conflict                                 |
+| 410  | 410 – Gone                                     |
+| 411  | 411 – Length Required                          |
+| 412  | 412 – Precondition Failed                      |
+| 413  | 413 – Payload Too Large                        |
+| 414  | 414 – URI Too Long                             |
+| 415  | 415 – Unsupported Media Type                   |
+| 416  | 416 – Range Not Satisfiable                    |
+| 417  | 417 – Expectation Failed                       |
+| 418  | 418 – I’m a teapot                             |
+| 421  | 421 – Misdirected Request                      |
+| 422  | 422 – Unprocessable Content                    |
+| 423  | 423 – Locked                                   |
+| 424  | 424 – Failed Dependency                        |
+| 425  | 425 – Too Early                                |
+| 426  | 426 – Upgrade Required                         |
+| 428  | 428 – Precondition Required                    |
+| 429  | 429 – Too Many Requests                        |
+| 431  | 431 – Request Header Fields Too Large          |
+| 451  | 451 – Unavailable For Legal Reasons            |
+| 500  | 500 – Internal Server Error                    |
+| 501  | 501 – Not Implemented                          |
+| 502  | 502 – Bad Gateway                              |
+| 503  | 503 – Service Unavailable                      |
+| 504  | 504 – Gateway Timeout                          |
+| 505  | 505 – HTTP Version Not Supported               |
+| 506  | 506 – Variant Also Negotiates                  |
+| 507  | 507 – Insufficient Storage                     |
+| 508  | 508 – Loop Detected                            |
+| 510  | 510 – Not Extended                             |
+| 511  | 511 – Network Authentication Required          |
+| 599  | Uncommon Response Code Received                |

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.42",
+  "version": "0.10.43",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "dependencies": {
@@ -46,6 +46,7 @@
     "@types/fs-extra": "^11.0.4",
     "@types/inquirer": "^9.0.7",
     "@types/lodash": "^4.17.7",
+    "@types/mime-types": "^2.1.4",
     "@types/safe-regex": "^1.1.6",
     "@types/validator": "^13.11.10",
     "@types/which": "^3.0.4",
@@ -97,4 +98,4 @@
     "url": "https://github.com/GovTechSG/oobee/issues"
   },
   "homepage": "https://github.com/GovTechSG/oobee#readme"
-}
+}

package/src/cli.ts

@@ -137,9 +137,6 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
       printMessage([`Invalid blacklistedPatternsFilename file path. ${err}`], messageOptions);
       process.exit(1);
     }
-
-    // eslint-disable-next-line no-unreachable
-    return null;
   })
   .coerce('i', option => {
     const { choices } = cliOptions.i;
@@ -241,7 +238,7 @@ const scanInit = async (argvs: Answers): Promise<string> => {
     clonedDataDir,
     updatedArgvs.playwrightDeviceDetailsObject,
     isCustomFlow,
-    updatedArgvs.header,
+    parseHeaders(updatedArgvs.header),
   );
   switch (res.status) {
     case statuses.success.code: {
@@ -255,17 +252,14 @@ const scanInit = async (argvs: Answers): Promise<string> => {
     case statuses.unauthorised.code: {
       printMessage([statuses.unauthorised.message], messageOptions);
       process.exit(res.status);
-      break;
     }
     case statuses.cannotBeResolved.code: {
       printMessage([statuses.cannotBeResolved.message], messageOptions);
       process.exit(res.status);
-      break;
     }
     case statuses.systemError.code: {
       printMessage([statuses.systemError.message], messageOptions);
       process.exit(res.status);
-      break;
     }
     case statuses.invalidUrl.code: {
       if (
@@ -296,17 +290,14 @@ const scanInit = async (argvs: Answers): Promise<string> => {
     case statuses.notASitemap.code: {
       printMessage([statuses.notASitemap.message], messageOptions);
       process.exit(res.status);
-      break;
     }
     case statuses.notALocalFile.code: {
       printMessage([statuses.notALocalFile.message], messageOptions);
       process.exit(res.status);
-      break;
     }
     case statuses.browserError.code: {
       printMessage([statuses.browserError.message], messageOptions);
       process.exit(res.status);
-      break;
     }
     default:
       break;
@@ -362,7 +353,7 @@ const scanInit = async (argvs: Answers): Promise<string> => {
   }
 
   // Delete dataset and request queues
-  await cleanUp(data.randomToken);
+  cleanUp(data.randomToken);
 
   return getStoragePath(data.randomToken);
 };

package/src/constants/common.ts

@@ -15,8 +15,8 @@ import safe from 'safe-regex';
 import * as https from 'https';
 import os from 'os';
 import { minimatch } from 'minimatch';
-import { globSync } from 'glob';
-import { LaunchOptions, devices, webkit } from 'playwright';
+import { globSync, GlobOptionsWithFileTypesFalse } from 'glob';
+import { LaunchOptions, Locator, Page, devices, webkit } from 'playwright';
 import printMessage from 'print-message';
 import constants, {
   getDefaultChromeDataDir,
@@ -31,6 +31,7 @@ import { silentLogger } from '../logs.js';
 import { isUrlPdf } from '../crawlers/commonCrawlerFunc.js';
 import { randomThreeDigitNumberString } from '../utils.js';
 import { Answers, Data } from '../index.js';
+import { DeviceDescriptor } from '../types/types.js';
 
 // validateDirPath validates a provided directory path
 // returns null if no error
@@ -252,7 +253,7 @@ export const getUrlMessage = (scanner: ScannerTypes): string => {
   }
 };
 
-export const isInputValid = inputString => {
+export const isInputValid = (inputString: string): boolean => {
   if (!validator.isEmpty(inputString)) {
     const removeBlackListCharacters = validator.escape(inputString);
 
@@ -373,12 +374,12 @@ const requestToUrl = async (
 };
 
 const checkUrlConnectivityWithBrowser = async (
-  url,
-  browserToRun,
-  clonedDataDir,
-  playwrightDeviceDetailsObject,
-  isCustomFlow,
-  extraHTTPHeaders,
+  url: string,
+  browserToRun: string,
+  clonedDataDir: string,
+  playwrightDeviceDetailsObject: DeviceDescriptor,
+  isCustomFlow: boolean,
+  extraHTTPHeaders: Record<string, string>,
 ) => {
   const res = new RES();
 
@@ -468,7 +469,6 @@ const checkUrlConnectivityWithBrowser = async (
         res.content = responseFromUrl.content;
       }
     } catch (error) {
-
       // But this does work with the headless=new flag
       if (error.message.includes('net::ERR_INVALID_AUTH_CREDENTIALS')) {
         res.status = constants.urlCheckStatuses.unauthorised.code;
@@ -510,13 +510,13 @@ export const isSitemapContent = (content: string) => {
 };
 
 export const checkUrl = async (
-  scanner,
-  url,
-  browser,
-  clonedDataDir,
-  playwrightDeviceDetailsObject,
-  isCustomFlow,
-  extraHTTPHeaders,
+  scanner: ScannerTypes,
+  url: string,
+  browser: string,
+  clonedDataDir: string,
+  playwrightDeviceDetailsObject: DeviceDescriptor,
+  isCustomFlow: boolean,
+  extraHTTPHeaders: Record<string, string>,
 ) => {
   const res = await checkUrlConnectivityWithBrowser(
     url,
@@ -548,7 +548,7 @@ export const parseHeaders = (header?: string): Record<string, string> => {
   // parse HTTP headers from string
   if (!header) return {};
   const headerValues = header.split(', ');
-  const allHeaders = {};
+  const allHeaders: Record<string, string> = {};
   headerValues.map((headerValue: string) => {
     const headerValuePair = headerValue.split(/ (.*)/s);
     if (headerValuePair.length < 2) {
@@ -776,11 +776,11 @@ export const getLinksFromSitemap = async (
   password: string,
 ) => {
   const scannedSitemaps = new Set<string>();
-  const urls = {}; // dictionary of requests to urls to be scanned
+  const urls: Record<string, Request> = {}; // dictionary of requests to urls to be scanned
 
   const isLimitReached = () => Object.keys(urls).length >= maxLinksCount;
 
-  const addToUrlList = url => {
+  const addToUrlList = (url: string) => {
     if (!url) return;
     if (isDisallowedInRobotsTxt(url)) return;
 
@@ -803,14 +803,14 @@ export const getLinksFromSitemap = async (
     urls[url] = request;
   };
 
-  const addBasicAuthCredentials = (url, username, password) => {
+  const addBasicAuthCredentials = (url: string, username: string, password: string) => {
     const urlObject = new URL(url);
     urlObject.username = username;
     urlObject.password = password;
     return urlObject.toString();
   };
 
-  const calculateCloseness = sitemapUrl => {
+  const calculateCloseness = (sitemapUrl: string) => {
     // Remove 'http://', 'https://', and 'www.' prefixes from the URLs
     const normalizedSitemapUrl = sitemapUrl.replace(/^(https?:\/\/)?(www\.)?/, '');
     const normalizedUserUrlInput = userUrlInput
@@ -825,10 +825,16 @@ export const getLinksFromSitemap = async (
     }
     return 0;
   };
-  const processXmlSitemap = async ($, sitemapType, linkSelector, dateSelector, sectionSelector) => {
-    const urlList = [];
+  const processXmlSitemap = async (
+    $: cheerio.CheerioAPI,
+    sitemapType: number,
+    linkSelector: string,
+    dateSelector: string,
+    sectionSelector: string,
+  ) => {
+    const urlList: { url: string; lastModifiedDate: Date }[] = [];
     // Iterate through each URL element in the sitemap, collect url and modified date
-    $(sectionSelector).each((index, urlElement) => {
+    $(sectionSelector).each((_index, urlElement) => {
       let url;
       if (sitemapType === constants.xmlSitemapTypes.atom) {
         url = $(urlElement).find(linkSelector).prop('href');
@@ -850,8 +856,7 @@ export const getLinksFromSitemap = async (
         }
 
         // If closeness is the same, sort by last modified date in descending order
-        const dateDifference = (b.lastModifiedDate || 0) - (a.lastModifiedDate || 0);
-        return dateDifference !== 0 ? dateDifference : 0; // Maintain original order for equal dates
+        return (b.lastModifiedDate?.getTime() || 0) - (a.lastModifiedDate?.getTime() || 0);
       });
     }
 
@@ -861,7 +866,7 @@ export const getLinksFromSitemap = async (
     }
   };
 
-  const processNonStandardSitemap = data => {
+  const processNonStandardSitemap = (data: string) => {
     const urlsFromData = crawlee
       .extractUrls({ string: data, urlRegExp: new RegExp('^(http|https):/{2}.+$', 'gmi') })
       .slice(0, maxLinksCount);
@@ -934,7 +939,7 @@ export const getLinksFromSitemap = async (
         const sitemapIndex = page.locator('sitemapindex');
         const rss = page.locator('rss');
         const feed = page.locator('feed');
-        const isRoot = async locator => (await locator.count()) > 0;
+        const isRoot = async (locator: Locator) => (await locator.count()) > 0;
 
         if (await isRoot(urlSet)) {
           data = await urlSet.evaluate(elem => elem.outerHTML);
@@ -1054,14 +1059,14 @@ export const getLinksFromSitemap = async (
   return requestList;
 };
 
-export const validEmail = email => {
+export const validEmail = (email: string) => {
   const emailRegex = /^.+@.+\..+$/u;
 
   return emailRegex.test(email);
 };
 
 // For new user flow.
-export const validName = name => {
+export const validName = (name: string) => {
   // Allow only printable characters from any language
   const regex = /^[\p{L}\p{N}\s'".,()\[\]{}!?:؛،؟…]+$/u;
 
@@ -1213,11 +1218,11 @@ export const getEdgeData = () => {
  * @param {*} destDir destination directory
  * @returns boolean indicating whether the operation was successful
  */
-const cloneChromeProfileCookieFiles = (options, destDir) => {
+const cloneChromeProfileCookieFiles = (options: GlobOptionsWithFileTypesFalse, destDir: string) => {
   let profileCookiesDir;
   // Cookies file per profile is located in .../User Data/<profile name>/Network/Cookies for windows
   // and ../Chrome/<profile name>/Cookies for mac
-  let profileNamesRegex;
+  let profileNamesRegex: RegExp;
   if (os.platform() === 'win32') {
     profileCookiesDir = globSync('**/Network/Cookies', {
       ...options,
@@ -1288,11 +1293,11 @@ const cloneChromeProfileCookieFiles = (options, destDir) => {
  * @param {*} destDir destination directory
  * @returns boolean indicating whether the operation was successful
  */
-const cloneEdgeProfileCookieFiles = (options, destDir) => {
+const cloneEdgeProfileCookieFiles = (options: GlobOptionsWithFileTypesFalse, destDir: string) => {
   let profileCookiesDir;
   // Cookies file per profile is located in .../User Data/<profile name>/Network/Cookies for windows
   // and ../Chrome/<profile name>/Cookies for mac
-  let profileNamesRegex;
+  let profileNamesRegex: RegExp;
   // Ignores the cloned oobee directory if exists
   if (os.platform() === 'win32') {
     profileCookiesDir = globSync('**/Network/Cookies', {
@@ -1361,7 +1366,7 @@ const cloneEdgeProfileCookieFiles = (options, destDir) => {
  * @param {string} destDir - destination directory
  * @returns boolean indicating whether the operation was successful
  */
-const cloneLocalStateFile = (options, destDir) => {
+const cloneLocalStateFile = (options: GlobOptionsWithFileTypesFalse, destDir: string) => {
   const localState = globSync('**/*Local State', {
     ...options,
     maxDepth: 1,
@@ -1647,8 +1652,9 @@ export const getPlaywrightDeviceDetailsObject = (
   deviceChosen: string,
   customDevice: string,
   viewportWidth: number,
-) => {
-  let playwrightDeviceDetailsObject = {};
+): DeviceDescriptor => {
+  let playwrightDeviceDetailsObject = devices['Desktop Chrome']; // default to Desktop Chrome
+
   if (deviceChosen === 'Mobile' || customDevice === 'iPhone 11') {
     playwrightDeviceDetailsObject = devices['iPhone 11'];
   } else if (customDevice === 'Samsung Galaxy S9+') {
@@ -1656,6 +1662,11 @@ export const getPlaywrightDeviceDetailsObject = (
   } else if (viewportWidth) {
     playwrightDeviceDetailsObject = {
       viewport: { width: viewportWidth, height: 720 },
+      isMobile: false,
+      hasTouch: false,
+      userAgent: devices['Desktop Chrome'].userAgent,
+      deviceScaleFactor: 1,
+      defaultBrowserType: 'chromium',
     };
   } else if (customDevice) {
     playwrightDeviceDetailsObject = devices[customDevice.replace(/_/g, ' ')];
@@ -1777,14 +1788,17 @@ export const submitForm = async (
   }
 };
 
-export async function initModifiedUserAgent(browser?: string, playwrightDeviceDetailsObject?: object) {
+export async function initModifiedUserAgent(
+  browser?: string,
+  playwrightDeviceDetailsObject?: object,
+) {
   const isHeadless = process.env.CRAWLEE_HEADLESS === '1';
-  
+
   // If headless mode is enabled, ensure the headless flag is set.
   if (isHeadless && !constants.launchOptionsArgs.includes('--headless=new')) {
     constants.launchOptionsArgs.push('--headless=new');
   }
-  
+
   // Build the launch options using your production settings.
   // headless is forced to false as in your persistent context, and we merge in getPlaywrightLaunchOptions and device details.
   const launchOptions = {
@@ -1803,17 +1817,16 @@ export async function initModifiedUserAgent(browser?: string, playwrightDeviceDe
 
   // Modify the UA:
   // Replace "HeadlessChrome" with "Chrome" if present.
-  let modifiedUA = defaultUA.includes('HeadlessChrome')
+  const modifiedUA = defaultUA.includes('HeadlessChrome')
     ? defaultUA.replace('HeadlessChrome', 'Chrome')
     : defaultUA;
-    
+
   // Push the modified UA flag into your global launch options.
   constants.launchOptionsArgs.push(`--user-agent=${modifiedUA}`);
   // Optionally log the modified UA.
   // console.log('Modified User Agent:', modifiedUA);
 }
 
-
 /**
  * @param {string} browser browser name ("chrome" or "edge", null for chromium, the default Playwright browser)
  * @returns playwright launch options object. For more details: https://playwright.dev/docs/api/class-browsertype#browser-type-launch
@@ -1856,25 +1869,25 @@ export const urlWithoutAuth = (url: string): string => {
   return parsedUrl.toString();
 };
 
-export const waitForPageLoaded = async (page, timeout = 10000) => {
+export const waitForPageLoaded = async (page: Page, timeout = 10000) => {
   const OBSERVER_TIMEOUT = timeout; // Ensure observer timeout does not exceed the main timeout
 
   return Promise.race([
     page.waitForLoadState('load'), // Ensure page load completes
     page.waitForLoadState('networkidle'), // Wait for network requests to settle
     new Promise(resolve => setTimeout(resolve, timeout)), // Hard timeout as a fallback
-    page.evaluate((OBSERVER_TIMEOUT) => {
-      return new Promise((resolve) => {
+    page.evaluate(OBSERVER_TIMEOUT => {
+      return new Promise(resolve => {
         // Skip mutation check for PDFs
         if (document.contentType === 'application/pdf') {
           resolve('Skipping DOM mutation check for PDF.');
           return;
         }
 
-        let timeout;
+        let timeout: NodeJS.Timeout;
         let mutationCount = 0;
         const MAX_MUTATIONS = 250; // Limit max mutations
-        const mutationHash = {};
+        const mutationHash: Record<string, number> = {};
 
         const observer = new MutationObserver(mutationsList => {
           clearTimeout(timeout);
@@ -1916,14 +1929,17 @@ export const waitForPageLoaded = async (page, timeout = 10000) => {
           resolve('Observer timeout reached, exiting.');
         }, OBSERVER_TIMEOUT);
 
-        observer.observe(document.documentElement, { childList: true, subtree: true, attributes: true });
+        observer.observe(document.documentElement, {
+          childList: true,
+          subtree: true,
+          attributes: true,
+        });
       });
     }, OBSERVER_TIMEOUT), // Pass OBSERVER_TIMEOUT dynamically to the browser context
   ]);
 };
 
-
-function isValidHttpUrl(urlString) {
+function isValidHttpUrl(urlString: string) {
   const pattern = /^(http|https):\/\/[^ "]+$/;
   return pattern.test(urlString);
 }

package/src/constants/constants.ts

@@ -29,6 +29,7 @@ export const blackListedFileExtensions = [
   'zip',
   'webp',
   'json',
+  'xml'
 ];
 
 export const getIntermediateScreenshotsPath = (datasetsPath: string): string =>
@@ -217,7 +218,7 @@ export const guiInfoStatusTypes = {
   DUPLICATE: 'duplicate',
 };
 
-let launchOptionsArgs = [];
+let launchOptionsArgs: string[] = [];
 
 // Check if running in docker container
 if (fs.existsSync('/.dockerenv')) {
@@ -444,3 +445,82 @@ export enum RuleFlags {
   DISABLE_OOBEE = 'disable-oobee',
   ENABLE_WCAG_AAA = 'enable-wcag-aaa',
 }
+
+// Note: Not all status codes will appear as Crawler will handle it as best effort first. E.g. try to handle redirect
+export const STATUS_CODE_METADATA: Record<number,string> = {
+  // Custom Codes for Oobee's use
+  0: 'Page Excluded',
+  1: 'Not A Supported Document',
+  2: 'Web Crawler Errored',
+
+  // 599 is set because Crawlee returns response status 100, 102, 103 as 599
+  599: 'Uncommon Response Status Code Received',
+
+  // This is Status OK but thrown when the crawler cannot scan the page
+  200: '200 - However Page Could Not Be Scanned', 
+  
+  // 1xx - Informational
+  100: '100 - Continue',
+  101: '101 - Switching Protocols',
+  102: '102 - Processing',
+  103: '103 - Early Hints',
+
+  // 2xx - Browser Doesn't Support
+  204: '204 - No Content',
+  205: '205 - Reset Content',
+
+  // 3xx - Redirection
+  300: '300 - Multiple Choices',
+  301: '301 - Moved Permanently',
+  302: '302 - Found',
+  303: '303 - See Other',
+  304: '304 - Not Modified',
+  305: '305 - Use Proxy',
+  307: '307 - Temporary Redirect',
+  308: '308 - Permanent Redirect',
+  
+  // 4xx - Client Error
+  400: '400 - Bad Request',
+  401: '401 - Unauthorized',
+  402: '402 - Payment Required',
+  403: '403 - Forbidden',
+  404: '404 - Not Found',
+  405: '405 - Method Not Allowed',
+  406: '406 - Not Acceptable',
+  407: '407 - Proxy Authentication Required',
+  408: '408 - Request Timeout',
+  409: '409 - Conflict',
+  410: '410 - Gone',
+  411: '411 - Length Required',
+  412: '412 - Precondition Failed',
+  413: '413 - Payload Too Large',
+  414: '414 - URI Too Long',
+  415: '415 - Unsupported Media Type',
+  416: '416 - Range Not Satisfiable',
+  417: '417 - Expectation Failed',
+  418: "418 - I'm a teapot",
+  421: '421 - Misdirected Request',
+  422: '422 - Unprocessable Content',
+  423: '423 - Locked',
+  424: '424 - Failed Dependency',
+  425: '425 - Too Early',
+  426: '426 - Upgrade Required',
+  428: '428 - Precondition Required',
+  429: '429 - Too Many Requests',
+  431: '431 - Request Header Fields Too Large',
+  451: '451 - Unavailable For Legal Reasons',
+
+  // 5xx - Server Error
+  500: '500 - Internal Server Error',
+  501: '501 - Not Implemented',
+  502: '502 - Bad Gateway',
+  503: '503 - Service Unavailable',
+  504: '504 - Gateway Timeout',
+  505: '505 - HTTP Version Not Supported',
+  506: '506 - Variant Also Negotiates',
+  507: '507 - Insufficient Storage',
+  508: '508 - Loop Detected',
+  510: '510 - Not Extended',
+  511: '511 - Network Authentication Required',
+
+};

package/src/constants/oobeeAi.ts

@@ -24,7 +24,7 @@ export const oobeeAiRules = [
   'autocomplete-valid',
 ];
 
-export const oobeeAiHtmlETL = htmlSnippet => {
+export const oobeeAiHtmlETL = (htmlSnippet: string) => {
   // Whitelisted attributes (to not drop)
   // i.e. any other attribute will be dropped
   const whitelistedAttributes = [
@@ -60,12 +60,12 @@ export const oobeeAiHtmlETL = htmlSnippet => {
     `aria-labelledby`,
   ];
 
-  const sortAlphaAttributes = html => {
+  const sortAlphaAttributes = (html: string) => {
     let entireHtml = '';
     const htmlOpeningTagRegex = /<[^>]+/g;
     const htmlTagmatches = html.match(htmlOpeningTagRegex);
 
-    let sortedHtmlTag;
+    let sortedHtmlTag: string = '';
 
     htmlTagmatches.forEach(htmlTag => {
       const closingTag = htmlTag.trim().slice(-1) === '/' ? '/>' : '>';
@@ -112,7 +112,7 @@ export const oobeeAiHtmlETL = htmlSnippet => {
 
   // For all attributes within mutedAttributeValues array
   // replace their values with "something" while maintaining the attribute
-  const muteAttributeValues = html => {
+  const muteAttributeValues = (html: string) => {
     const regex = /(\s+)([\w-]+)(\s*=\s*")([^"]*)(")/g;
 
     // p1 is the whitespace before the attribute
@@ -120,7 +120,7 @@ export const oobeeAiHtmlETL = htmlSnippet => {
     // p3 is the attribute value before the replacement
     // p4 is the attribute value (replaced with "...")
     // p5 is the closing quote of the attribute value
-    return html.replace(regex, (match, p1, p2, p3, p4, p5) => {
+    return html.replace(regex, (match, p1, p2, p3, _p4, p5) => {
       if (mutedAttributeValues.includes(p2)) {
         return `${p1}${p2}${p3}...${p5}`;
       }
@@ -129,7 +129,7 @@ export const oobeeAiHtmlETL = htmlSnippet => {
   };
 
   // Drop all attributes from the HTML snippet except whitelisted
-  const dropAllExceptWhitelisted = html => {
+  const dropAllExceptWhitelisted = (html: string) => {
     const regex = new RegExp(
       `(\\s+)(?!${whitelistedAttributes.join(`|`)})([\\w-]+)(\\s*=\\s*"[^"]*")`,
       `g`,

package/src/constants/questions.ts

@@ -12,12 +12,13 @@ import {
   validEmail,
   validName,
   validateCustomFlowLabel,
+  parseHeaders,
 } from './common.js';
 import constants, { BrowserTypes, ScannerTypes } from './constants.js';
 
 const userData = getUserDataTxt();
 
-const questions = [];
+const questions: Question[] = [];
 
 const startScanQuestions = [
   {
@@ -95,7 +96,7 @@ const startScanQuestions = [
         clonedBrowserDataDir,
         playwrightDeviceDetailsObject,
         answers.scanner === ScannerTypes.CUSTOM,
-        answers.header,
+        parseHeaders(answers.header),
       );
 
       deleteClonedProfiles(browserToRun);