noibu-react-native 0.2.2 → 0.2.3

package/dist/monitors/httpDataBundler.js

@@ -1,725 +1,537 @@
-import { HUMAN_READABLE_CONTENT_TYPE_REGEX, DEFAULT_WEBSITE_SUBDOMAIN_PATTERN, SEVERITY, HTTP_BODY_NULL_STRING, HTTP_DATA_REQ_HEADERS_ATT_NAME, HTTP_DATA_PAYLOAD_ATT_NAME, HTTP_DATA_RESP_HEADERS_ATT_NAME, HTTP_DATA_RESP_PAYLOAD_ATT_NAME, HTTP_BODY_DROPPED_LENGTH_MSG, HTTP_BODY_DROPPED_TYPE_MSG, MAX_HTTP_DATA_PAYLOAD_LENGTH, CONTENT_TYPE, CONTENT_LENGTH, BLOCKED_HTTP_HEADER_KEYS, PII_REDACTION_REPLACEMENT_STRING, HTTP_PII_BLOCKING_PATTERNS } from '../constants.js';
+import { __awaiter } from 'tslib';
+import { HUMAN_READABLE_CONTENT_TYPE_REGEX, DEFAULT_WEBSITE_SUBDOMAIN_PATTERN, SEVERITY, HTTP_BODY_NULL_STRING, HTTP_DATA_REQ_HEADERS_ATT_NAME, HTTP_DATA_PAYLOAD_ATT_NAME, HTTP_DATA_RESP_HEADERS_ATT_NAME, HTTP_DATA_RESP_PAYLOAD_ATT_NAME, HTTP_BODY_DROPPED_LENGTH_MSG, HTTP_BODY_DROPPED_TYPE_MSG, CONTENT_TYPE, CONTENT_LENGTH, BLOCKED_HTTP_HEADER_KEYS, PII_REDACTION_REPLACEMENT_STRING, MAX_HTTP_DATA_PAYLOAD_LENGTH, MAX_SUCCESS_HTTP_DATA_PAYLOAD_LENGTH } from '../constants.js';
 import ClientConfig from '../api/clientConfig.js';
 import StoredMetrics from '../api/storedMetrics.js';
-import { iterateObjectRecursively } from '../utils/object.js';
-import { noibuLog } from '../utils/log.js';
-
-/** @module HTTPDataBundler */
+import { safeFromEntries, safeEntries } from '../utils/object.js';
+import { safeTrim, stringifyJSON, isString } from '../utils/function.js';
+import { removePII } from '../utils/piiRedactor.js';
 
 /** Bundles HTTP payloads and headers */
 class HTTPDataBundler {
-  /**
-   * Creates an instance of the ClickMonitor instance
-   */
-  constructor() {
-    // compile regex only once
-    this.contentTypeReadableRegex = new RegExp(
-      HUMAN_READABLE_CONTENT_TYPE_REGEX,
-      'i',
-    );
-
-    // pull out the domain hostname
-    const initialURL = ClientConfig.getInstance().globalUrl;
-    this.initialURLPartsReversed = [];
-    if (initialURL && initialURL.length > 0) {
-      try {
-        // Store URL parts to class variable
-        const initialHostname = new URL(initialURL).hostname;
-        this.initialURLPartsReversed = initialHostname.split('.');
-
-        // Remove www. etc.
-        // eslint-disable-next-line no-unused-expressions
-        DEFAULT_WEBSITE_SUBDOMAIN_PATTERN.test(
-          this.initialURLPartsReversed[0],
-        ) && this.initialURLPartsReversed.shift();
-
-        // Reverse array
-        this.initialURLPartsReversed.reverse();
-      } catch (e) {
-        ClientConfig.getInstance().postNoibuErrorAndOptionallyDisableClient(
-          `Unable to determine hostname for initial URL: ${e}`,
-          false,
-          SEVERITY.warn,
-        );
-      }
-    }
-
-    this.httpDataCollectionEnabled =
-      !!ClientConfig.getInstance().enableHttpDataCollection;
-
-    // compile the relative and full HTTP URL regexes
-    const allowedURLs = HTTPDataBundler.getHttpPayloadAllowedURLs();
-    this.httpDataAllowedAbsoluteRegex = HTTPDataBundler.buildAllowedRegex(
-      allowedURLs,
-      'absolute',
-    );
-    this.httpDataAllowedRelativeRegex = HTTPDataBundler.buildAllowedRegex(
-      allowedURLs,
-      'relative',
-    );
-    // track unique requests and only capture each once
-    // TODO: disabled for beta. NOI-4253
-    // this.uniqueRequests = new Set();
-
-    this.fuzzyFieldsToRedact = [
-      'password',
-      'address',
-      'credit',
-      'postal',
-      'token',
-      'phone',
-      'mobile',
-    ];
-
-    this.exactFieldsToRedact = [
-      'firstname',
-      'lastname',
-      'street',
-      'fullname',
-      'creditcard',
-      'postcode',
-      'zipcode',
-      'city',
-      'town',
-      'county',
-      'cc',
-    ];
-  }
-
-  /** gets http data payload allowed URLs */
-  static getHttpPayloadAllowedURLs() {
-    const noibuConfig = ClientConfig.getInstance();
-    // return the allowed list or an empty list
-    if (
-      noibuConfig.listOfUrlsToCollectHttpDataFrom &&
-      Array.isArray(noibuConfig.listOfUrlsToCollectHttpDataFrom)
-    ) {
-      return noibuConfig.listOfUrlsToCollectHttpDataFrom;
-    }
-    return [];
-  }
-
-  /**
-   * gets the singleton instance
-   * @returns {HTTPDataBundler}
-   */
-  static getInstance() {
-    if (!this.instance) {
-      this.instance = new HTTPDataBundler();
-    }
-
-    return this.instance;
-  }
-
-  /**
-   * Builds the HTTP payload allowed regexes for full and relative URLs
-   * @param allowedURLs A list of allowed URLs
-   * @param {'absolute' | 'relative'} strategy Use only absolute URLs if true, use only relative URL if false
-   * @returns a regex of allowed URLs
-   */
-  static buildAllowedRegex(allowedURLs, strategy) {
-    if (!allowedURLs) return null;
-    const allowedURLsFiltered = allowedURLs.filter(url => {
-      const isAbsolute = HTTPDataBundler.isAbsoluteURL(url);
-      return strategy === 'absolute' ? isAbsolute : !isAbsolute;
-    });
-    if (allowedURLsFiltered.length > 0) {
-      const lowerCasedURLs = allowedURLsFiltered.map(url =>
-        url.trim().toLowerCase(),
-      );
-      return new RegExp(lowerCasedURLs.join('|'));
-    }
-    return null;
-  }
-
-  /**
-   * Takes an iterator and returns a map of strings representing headers.
-   * @param {object} headersIterable any iterable object
-   * @returns a map of strings (as expected by metroplex) representing HTTP
-   * request or response headers
-   */
-  static headersMapFromIterable(headersIterable) {
-    const headerMap = new Map();
-    // eslint-disable-next-line no-restricted-syntax
-    for (const header of headersIterable) {
-      // Ensure we're working with strings
-      // This will also automatically convert null to "null"
-      if (typeof header[0] !== 'string') header[0] = String(header[0]);
-      if (typeof header[1] !== 'string') header[1] = String(header[1]);
-
-      headerMap.set(header[0].toLowerCase(), header[1]);
-    }
-    return headerMap;
-  }
-
-  /**
-   * Takes a string of headers with 'name: value' and returns
-   * a map of strings representing headers.
-   * @param {string} headersString is all the headers in one string
-   * @returns a map of strings (as expected by metroplex) representing HTTP
-   * request or response headers
-   */
-  static headersMapFromString(headersString) {
-    const headerMap = new Map();
-    if (!headersString || typeof headersString !== 'string') return headerMap;
-    const headerStringArray = headersString.split('\r\n').filter(Boolean);
-    headerStringArray.forEach(headerString => {
-      const split = headerString.split(': ');
-      if (split.length === 2 && split[0].length > 0 && split[1].length > 0) {
-        headerMap.set(split[0].toLowerCase(), split[1]);
-      }
-    });
-    return headerMap;
-  }
-
-  /**
-   * For an XHR object, checks the responseType property and handles the response or
-   * responseText property accordingly to return a string representation of the response.
-   * @param {object} xhr an XMLHTTPRequest object
-   * @returns a string representation of the response, or null if this fails.
-   */
-  static responseStringFromXHRResponseType(xhr) {
-    // This should not happen, as this function is called from a method on xhr.
-    if (xhr === undefined || xhr === null) return null;
-
-    // If the XHR object exists but the response is null, return null as a string.
-    // noinspection PointlessBooleanExpressionJS
-    if (xhr.response && xhr.response === null) return HTTP_BODY_NULL_STRING;
-
-    if (xhr.responseType === '' || xhr.responseType === 'text') {
-      return xhr.responseText;
-    }
-    if (
-      xhr.responseType === 'document' &&
-      !!xhr.response &&
-      !!xhr.response.documentElement &&
-      !!xhr.response.documentElement.innerHTML
-    ) {
-      return xhr.response.documentElement.innerHTML;
-    }
-    if (xhr.responseType === 'json') {
-      try {
-        return JSON.stringify(xhr.response);
-      } catch (e) {
-        ClientConfig.getInstance().postNoibuErrorAndOptionallyDisableClient(
-          `Unable to stringify JSON response: ${e}`,
-          false,
-          SEVERITY.warn,
-        );
+    /**
+     * Creates an instance of the ClickMonitor instance
+     */
+    constructor() {
+        // compile regex only once
+        this.contentTypeReadableRegex = new RegExp(HUMAN_READABLE_CONTENT_TYPE_REGEX, 'i');
+        // pull out the domain hostname
+        const initialURL = ClientConfig.getInstance().globalUrl;
+        this.initialURLPartsReversed = [];
+        this.hostname = '';
+        if (initialURL && initialURL.length > 0) {
+            try {
+                // Store URL parts to class variable
+                this.hostname = new URL(initialURL).hostname;
+                this.initialURLPartsReversed = this.hostname.split('.');
+                // Remove www. etc.
+                // eslint-disable-next-line no-unused-expressions
+                DEFAULT_WEBSITE_SUBDOMAIN_PATTERN.test(this.initialURLPartsReversed[0]) && this.initialURLPartsReversed.shift();
+                // Reverse array
+                this.initialURLPartsReversed.reverse();
+            }
+            catch (e) {
+                ClientConfig.getInstance().postNoibuErrorAndOptionallyDisableClient(`Unable to determine hostname for initial URL: ${e}`, false, SEVERITY.warn);
+            }
+        }
+        this.httpDataCollectionEnabled =
+            !!ClientConfig.getInstance().enableHttpDataCollection;
+        // compile the relative and full HTTP URL regexes
+        const allowedURLs = ClientConfig.getInstance().listOfUrlsToCollectHttpDataFrom;
+        this.httpDataAllowedAbsoluteRegex = HTTPDataBundler.buildAllowedRegex(allowedURLs, true);
+        this.httpDataAllowedRelativeRegex = HTTPDataBundler.buildAllowedRegex(allowedURLs, false);
+        // track unique requests and only capture each once
+        // TODO: disabled for beta. NOI-4253
+        // this.uniqueRequests = new Set();
+    }
+    /** gets the singleton instance
+     * @returns {HTTPDataBundler}
+     * */
+    static getInstance() {
+        if (!this.instance) {
+            this.instance = new HTTPDataBundler();
+        }
+        return this.instance;
+    }
+    /**
+     * Builds the HTTP payload allowed regexes for full and relative URLs
+     * @param allowedURLs A list of allowed URLs
+     * @param absolute Use only absolute URLs if true, use only relative URL if false
+     * @returns a regex of allowed URLs
+     */
+    static buildAllowedRegex(allowedURLs, absolute) {
+        if (!allowedURLs || !Array.isArray(allowedURLs))
+            return null;
+        const allowedURLsFiltered = allowedURLs
+            .map(url => safeTrim(url).toLowerCase())
+            .filter(url => {
+            const isAbsolute = HTTPDataBundler.isAbsoluteURL(url);
+            if (absolute) {
+                return url && isAbsolute;
+            }
+            return url && !isAbsolute;
+        });
+        if (allowedURLsFiltered.length > 0) {
+            return new RegExp(allowedURLsFiltered.join('|'));
+        }
         return null;
-      }
-    }
-    return null;
-  }
-
-  /**
-   * Takes a URL and returns true if it is determined to be on the same domain as the URL
-   * the script is running on. Ignores protocol and path, and allows the URL to be a subdomain.
-   * @param {string} requestURL the URL of a request to compare to the script website's URL
-   */
-  isURLSameDomain(requestURL) {
-    if (
-      typeof requestURL !== 'string' ||
-      !this.initialURLPartsReversed ||
-      this.initialURLPartsReversed.length < 1
-    ) {
-      return false;
-    }
-
-    let requestHostname; // has to be declared outside `try`
-
-    // Use URL constructor to extract hostname
-    try {
-      requestHostname = new URL(requestURL).hostname;
-    } catch (e) {
-      // Return false if URL() is unable to parse the request URL
-      ClientConfig.getInstance().postNoibuErrorAndOptionallyDisableClient(
-        `Unable to determine hostname for request URL: ${e}`,
-        false,
-        SEVERITY.warn,
-      );
-
-      return false;
-    }
-
-    const requestParts = requestHostname.split('.');
-
-    // Remove first subdomain from either URL if it is www., www1., etc.
-
-    if (requestParts.length < 1) return false;
-
-    /* eslint-disable no-unused-expressions */
-    DEFAULT_WEBSITE_SUBDOMAIN_PATTERN.test(requestParts[0]) &&
-      requestParts.shift();
-    /* eslint-enable no-unused-expressions */
-
-    if (requestParts.length < this.initialURLPartsReversed.length) return false;
-
-    // Reverse order array
-    requestParts.reverse();
-
-    // Return true if for every part in the website part, the same part exists
-    // in the request part.
-    return this.initialURLPartsReversed.every(
-      (part, i) => part === requestParts[i],
-    );
-  }
-
-  /**
-   * Builds an HTTP Data bundle
-   * @param  url
-   * @param  requestHeaders is a map of request headers
-   * @param  rawRequestPayload
-   * @param  responseHeaders is a map of response headers
-   * @param  responsePayload
-   * @param  method
-   * @param shouldCollectPayload
-   */
-  bundleHTTPData(
-    url,
-    requestHeaders,
-    rawRequestPayload,
-    responseHeaders,
-    responsePayload,
-    method,
-    shouldCollectPayload,
-  ) {
-    noibuLog('bundleHTTPData started for', method, url);
-    if (!this.isValidRequest(url, method)) {
-      noibuLog('quitting');
-      return null;
-    }
-
-    // stringify payload if correct type and not too large
-    let requestPayload = '';
-    if (shouldCollectPayload) {
-      noibuLog('bundleHTTPData collecting payload');
-      requestPayload =
-        this.dropPayloadIfNecessaryFromHeaders(requestHeaders) ||
-        this.stringFromRequestBody(rawRequestPayload);
-    } else {
-      noibuLog('bundleHTTPData not collecting payload');
-    }
-
-    // don't bundle if there is no data
-    const safeRequestHeaders = requestHeaders || new Map();
-    const safeRequestPayload = requestPayload || '';
-    const safeResponseHeaders = responseHeaders || new Map();
-    const safeResponsePayload = responsePayload || '';
-    if (
-      safeRequestHeaders.size === 0 &&
-      safeRequestPayload.length === 0 &&
-      safeResponseHeaders.size === 0 &&
-      safeResponsePayload.length === 0
-    ) {
-      noibuLog('bundleHTTPData payload is empty, quitting');
-      return null;
-    }
-
-    // Ensure payloads do not exceed the maximum size and redact PII
-    const requestPayloadUpdated = this.restrictPayload(requestPayload, url);
-    // Ensure payloads do not exceed the maximum size and redact PII
-    const responsePayloadUpdated = this.restrictPayload(responsePayload, url);
-
-    // Redact PII.
-    const cleanRequestHeaders = this.removePIIHeaders(requestHeaders);
-    const cleanResponseHeaders = this.removePIIHeaders(responseHeaders);
-
-    // TODO: disabled for beta. NOI-4253
-    // we are capturing this request, so add it to the unique requests set
-    // this.uniqueRequests.add(requestSignature);
-    // build the http bundle
-    return {
-      [HTTP_DATA_REQ_HEADERS_ATT_NAME]: cleanRequestHeaders
-        ? Object.fromEntries(cleanRequestHeaders)
-        : {},
-      [HTTP_DATA_PAYLOAD_ATT_NAME]: requestPayloadUpdated,
-      [HTTP_DATA_RESP_HEADERS_ATT_NAME]: cleanResponseHeaders
-        ? Object.fromEntries(cleanResponseHeaders)
-        : {},
-      [HTTP_DATA_RESP_PAYLOAD_ATT_NAME]: responsePayloadUpdated,
-    };
-  }
-
-  /**
-   * Validates a request based on the URL and method. When enabled, will handle
-   * de-duping the requests
-   * @param {string} url
-   * @param {string} method
-   * @returns boolean indicating whether the validation passed
-   */
-  isValidRequest(url, method) {
-    noibuLog('isValidRequest');
-    if (!this.httpDataCollectionEnabled) {
-      noibuLog('isValidRequest httpDataCollectionEnabled is off, quitting');
-      return false;
-    }
-    if (!method || typeof method !== 'string') {
-      noibuLog('isValidRequest method is invalid, quitting');
-      return false;
-    }
-    // TODO: Check below disabled for beta. NOI-4253
-    // only bundle and send once per request/method/response code combination
-    // const urlWithoutParams = url.split('?')[0];
-    // const requestSignature = `${urlWithoutParams}-${method}-${responseCode}`;
-    // if (this.uniqueRequests.has(requestSignature)) {
-    //   return false;
-    // }
-
-    return true;
-  }
-
-  /**
-   * Checks two things: that the URL is either on the same domain (or an address relative to the
-   * current domain), and also checks that the config http_data_collection flag is enabled.
-   * @param {string} url
-   * @returns boolean indicating whether the URL passed is either relative (in which case it is
-   * inherently on the current domain) or matches the current domain.
-   */
-  shouldContinueForURL(url) {
-    noibuLog('shouldContinueForURL started for', url);
-    if (!this.httpDataCollectionEnabled) {
-      noibuLog('shouldContinueForURL httpDataCollection turned off, quitting');
-      return false;
-    }
-    if (!url || typeof url !== 'string' || !this.initialURLPartsReversed) {
-      noibuLog('shouldContinueForURL url is not valid, quitting', {
-        initialUrlPartsReversed: this.initialURLPartsReversed,
-      });
-      return false;
-    }
-    // URL is absolute; either "http://example.com" or "//example.com"
-    if (HTTPDataBundler.isAbsoluteURL(url)) {
-      // Only capture requests on the same domain or in the allowed list
-      if (!this.isURLSameDomain(url) && !this.shouldCollectPayloadForURL(url)) {
-        noibuLog(
-          'shouldContinueForURL url is not same domain or is not in allow list, quitting',
-          {
-            shouldCollectPayloadForURL: this.shouldCollectPayloadForURL(url),
-          },
-        );
-        return false;
-      }
-    } // end `if` (if URL is relative, it is on the same domain.)
-
-    noibuLog('shouldContinueForURL - yes');
-    return true;
-  }
-
-  /**
-   * Determins if the URL is absolute or relative
-   * @param {string} url
-   * @returns boolean indicating whether the URL passed is either absolute or relative
-   */
-  static isAbsoluteURL(url) {
-    if (!url || typeof url !== 'string') {
-      return false;
-    }
-    return url.indexOf('://') > 0 || url.indexOf('//') === 0;
-  }
-
-  /**
-   * Checks whether HTTP payloads can be collected on this URL
-   * @param {string} url
-   * @returns boolean indicating whether HTTP payloads can be collected on this URL
-   */
-  shouldCollectPayloadForURL(url) {
-    noibuLog('shouldCollectPayloadForURL', url);
-    if (!url || typeof url !== 'string') {
-      noibuLog('shouldCollectPayloadForURL url is invalid, quitting');
-      return false;
-    }
-    // check if in the full URL allowed list
-    if (
-      this.httpDataAllowedAbsoluteRegex &&
-      this.httpDataAllowedAbsoluteRegex.test(url.toLowerCase())
-    ) {
-      noibuLog('shouldCollectPayloadForURL yes, absolute');
-      return true;
-    }
-    // check if in the relative URL allowed list, if on domain
-    if (
-      this.httpDataAllowedRelativeRegex &&
-      // if this is a relative URL (isURLSameDomain will fail) OR a full URL on domain
-      (!HTTPDataBundler.isAbsoluteURL(url) || this.isURLSameDomain(url)) &&
-      this.httpDataAllowedRelativeRegex.test(url.toLowerCase())
-    ) {
-      noibuLog('shouldCollectPayloadForURL yes, relative');
-      return true;
-    }
-
-    noibuLog('shouldCollectPayloadForURL answer is no, reason', {
-      httpDataAllowedAbsoluteRegexTest:
-        this.httpDataAllowedAbsoluteRegex &&
-        this.httpDataAllowedAbsoluteRegex.test(url.toLowerCase()),
-      httpDataAllowedRelativeRegexTest:
-        this.httpDataAllowedRelativeRegex &&
-        this.httpDataAllowedRelativeRegex.test(url.toLowerCase()),
-      isAbsoluteURL: HTTPDataBundler.isAbsoluteURL(url),
-      isURLSameDomain: this.isURLSameDomain(url),
-    });
-    return false;
-  }
-
-  /**
-   * Double checks content length if we couldn't read the headers, and redacts PII
-   * @param  {string} payload
-   * @param  {string} url
-   * @returns the restricted payload
-   */
-  restrictPayload(payload, url) {
-    // if no payload, or too large, payload already dropped,
-    // or payloads not allowed for this URL: nothing to do
-    if (!payload || !this.shouldCollectPayloadForURL(url)) {
-      return HTTP_BODY_NULL_STRING;
-    }
-    if (
-      payload === HTTP_BODY_NULL_STRING ||
-      payload.startsWith(HTTP_BODY_DROPPED_LENGTH_MSG) ||
-      payload.startsWith(HTTP_BODY_DROPPED_TYPE_MSG)
-    ) {
-      return payload;
     }
-    if (payload.length > MAX_HTTP_DATA_PAYLOAD_LENGTH) {
-      StoredMetrics.getInstance().addHttpDataDropByLength();
-      return `${HTTP_BODY_DROPPED_LENGTH_MSG} Payload length: ${payload.length}`;
-    }
-
-    // payload is good!
-    StoredMetrics.getInstance().addHttpDataPayloadCount();
-
-    return this.removePIIBody(payload);
-  }
-
-  /**
-   * Returns true if the content length is acceptable to collect
-   * If the headers are not found, check actual content for length
-   * @param  {Headers} headers
-   * @returns boolean true if acceptable to collect
-   */
-  contentLengthAcceptable(headers) {
-    // check content length
-    const contentLength = this.contentLength(headers);
-    if (contentLength < 0) {
-      // no content length or can't read, keep going
-      return true;
-    }
-    if (contentLength > MAX_HTTP_DATA_PAYLOAD_LENGTH) {
-      StoredMetrics.getInstance().addHttpDataDropByLength();
-      return false;
-    }
-    return true;
-  }
-
-  /**
-   * Returns true if the content type according to the headers is valid for collection.
-   * Also returns assumed true if content type is not stated in headers.
-   * @param {Map} headersMap
-   * @returns boolean true if acceptable to collect
-   */
-  contentTypeAcceptable(headersMap) {
-    if (!headersMap || !headersMap.get) return true;
-
-    // check content type
-    const contentType = headersMap.get(CONTENT_TYPE);
-    if (
-      contentType &&
-      !this.contentTypeReadableRegex.test(contentType.toLowerCase())
-    ) {
-      // not a readable request, drop the content
-      StoredMetrics.getInstance().addHttpDataDropByType();
-      return false;
-    }
-    return true;
-  }
-
-  /**
-   * Returns a descriptive string if we have to drop payload based on the length
-   * or type listed in the headers passed. Returns an empty string otherwise.
-   * @param {Headers} headers a Headers object
-   * @returns A string, which is empty if the payload doesn't need to be dropped, or is a
-   * descriptive string explaining the circumstances of the drop otherwise.
-   */
-  dropPayloadIfNecessaryFromHeaders(headers) {
-    // Set descriptive body if content is wrong type or too long.
-    let text = '';
-    if (!HTTPDataBundler.getInstance().contentTypeAcceptable(headers)) {
-      text = HTTP_BODY_DROPPED_TYPE_MSG;
-      if (headers && headers.get) {
-        text += ` Payload type: ${headers.get(CONTENT_TYPE)}`;
-      }
-    } else if (
-      !HTTPDataBundler.getInstance().contentLengthAcceptable(headers)
-    ) {
-      text = `${HTTP_BODY_DROPPED_LENGTH_MSG} Payload length: \
-${HTTPDataBundler.getInstance().contentLength(headers)}`;
-    }
-    return text;
-  }
-
-  /**
-   * Returns content length from the headers, if available
-   * If the headers are not found or not a number, return -1
-   * @param  {} headersObject
-   * @returns content length
-   */
-  contentLength(headersObject) {
-    if (!headersObject || !headersObject.get) return 0;
-
-    // get content length
-    let contentLength = 0;
-    const contentValueString = headersObject.get(CONTENT_LENGTH);
-    // no content length header found
-    if (!contentValueString) {
-      return -1;
-    }
-    try {
-      contentLength = parseInt(contentValueString, 10);
-      if (Number.isNaN(contentLength)) {
-        // bad content length heade
-        return -1;
-      }
-    } catch (err) {
-      // if we can't convert the value to a number, confirm after reading payload
-      return -1;
-    }
-    return contentLength;
-  }
-
-  /**
-   * Accepts a value that could be any type used as a request payload, and returns a string
-   * representation. First attemtps to find a .toString() method, then tries to handle it
-   * like an XML or HTML element, then finally falls back to JSON.stringify(). If none of
-   * these are successful, returns null.
-   * @param {*} value request paylad body, of any type
-   * @returns string representation of the value passed, or null if this fails.
-   */
-  stringFromRequestBody(value) {
-    // Nullish check to ensure we don't send 'null' or 'undefined' as a string
-    if (value == null) return null;
-
-    try {
-      // Case where value has a .toString() method
-      const returnVal = value.toString();
-      // Catch case where .toString() is on an object, we don't want that --
-      // should be stringified below.
-      if (!returnVal.includes('[object')) return returnVal;
-    } catch (error) {
-      // Nothing--try next try block
-    }
-    try {
-      // Case where value is a Document type (HTML/XML)
-      return value.documentElement.innerHTML;
-    } catch (error) {
-      // Nothing--try next try block
-    }
-    try {
-      // If the payload is a FormData object, turn it into an object.
-      if (value instanceof FormData) {
-        // eslint-disable-next-line no-param-reassign
-        value = Array.from(value.entries()).reduce((obj, [key, val]) => {
-          // eslint-disable-next-line no-param-reassign
-          obj[key] = typeof val === 'string' ? val : 'non-string value.';
-          return obj;
-        }, {});
-      }
-      // Case where value is any other object type
-      return JSON.stringify(value);
-    } catch (e) {
-      ClientConfig.getInstance().postNoibuErrorAndOptionallyDisableClient(
-        `Unable to stringify request body: ${e}`,
-        false,
-        SEVERITY.warn,
-      );
-    }
-    return null;
-  }
-
-  /**
-   * Removes possible PII from headers.
-   * @param {Map} dirtyHeaders a Map of HTTP response or request headers
-   * @returns a map of headers with PII redacted
-   */
-  removePIIHeaders(dirtyHeaders) {
-    // In order to be fail-safe, let's return null if we won't be able to remove PII headers.
-    if (!(dirtyHeaders instanceof Map)) return null;
-    if (dirtyHeaders.size < 1) return dirtyHeaders;
-
-    const cleanHeaders = new Map(dirtyHeaders);
-
-    cleanHeaders.forEach((headerVal, headerKey, map) => {
-      // If the key is known PII, redact
-      if (BLOCKED_HTTP_HEADER_KEYS.includes(headerKey.toLowerCase())) {
-        map.set(headerKey, PII_REDACTION_REPLACEMENT_STRING);
-        // If not, run the header's value through the string redaction function
-      } else {
-        map.set(headerKey, this.removePIIBody(headerVal));
-      }
-    });
-
-    return cleanHeaders;
-  }
-
-  /**
-   * Takes a body payload string and redacts any PII we're able to detect
-   *
-   * @param {string} dirtyBody the string from which we want to redact PII
-   * @returns the string with any PII we're able to detect redacted.
-   */
-  removePIIBody(dirtyBody) {
-    // In order to be fail-safe, let's return null if we won't be able to remove PII.
-    if (typeof dirtyBody !== 'string') return null;
-    if (dirtyBody.length < 1) return dirtyBody;
-
-    let cleanString = this.tryParseObjectAndRemovePII(dirtyBody);
-    // Go through each of the PII matching patterns
-    HTTP_PII_BLOCKING_PATTERNS.forEach(pattern => {
-      // Replace all matches with the appropriate number of asterisks
-      cleanString = cleanString.replace(
-        pattern,
-        PII_REDACTION_REPLACEMENT_STRING,
-      );
-    });
-
-    return cleanString;
-  }
-
-  /**
-   * Try to parse content as a JSON object and
-   * iterate it recursively removing PII.
-   * Returns original content if nothing was changes or error is thrown.
-   * @param {String} content
-   */
-  tryParseObjectAndRemovePII(content) {
-    const first = content[0];
-    const isParsable = first === '{' || first === '[';
-    if (!isParsable) {
-      return content;
+    /**
+     * Takes an iterator and returns a map of strings representing headers.
+     * @param {object} headersIterable any iterable object
+     * @returns a map of strings (as expected by metroplex) representing HTTP
+     * request or response headers
+     */
+    static headersMapFromIterable(headersIterable) {
+        const headerMap = new Map();
+        // eslint-disable-next-line no-restricted-syntax
+        for (const header of headersIterable) {
+            // Ensure we're working with strings
+            // This will also automatically convert null to "null"
+            if (typeof header[0] !== 'string')
+                header[0] = String(header[0]);
+            if (typeof header[1] !== 'string')
+                header[1] = String(header[1]);
+            headerMap.set(header[0].toLowerCase(), header[1]);
+        }
+        return headerMap;
+    }
+    /**
+     * Takes a string of headers with 'name: value' and returns
+     * a map of strings representing headers.
+     * headersString is all the headers in one string
+     * returns a map of strings (as expected by metroplex) representing HTTP
+     * request or response headers
+     */
+    static headersMapFromString(headersString) {
+        const headerMap = new Map();
+        if (!headersString || typeof headersString !== 'string')
+            return headerMap;
+        const headerStringArray = headersString.split('\r\n').filter(Boolean);
+        headerStringArray.forEach(function (headerString) {
+            const split = headerString.split(': ');
+            if (split.length === 2 && split[0].length > 0 && split[1].length > 0) {
+                headerMap.set(split[0].toLowerCase(), split[1]);
+            }
+        });
+        return headerMap;
+    }
+    /**
+     * For an XHR object, checks the responseType property and handles the response or
+     * responseText property accordingly to return a string representation of the response.
+     * @returns a string representation of the response, or null if this fails.
+     */
+    static getResponseStringFromXHR(xhr) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            // This should not happen, as this function is called from a method on xhr.
+            if (!xhr)
+                return null;
+            if (xhr.responseType === '' || xhr.responseType === 'text') {
+                return xhr.responseText;
+            }
+            // If the XHR object exists but the response is null, return null as a string.
+            if (!xhr.response)
+                return HTTP_BODY_NULL_STRING;
+            if ((_a = xhr.response.documentElement) === null || _a === void 0 ? void 0 : _a.innerHTML) {
+                return xhr.response.documentElement.innerHTML;
+            }
+            if (typeof xhr.response.text === 'function') {
+                return yield xhr.response.text();
+            }
+            if (xhr.responseType === 'json') {
+                try {
+                    const text = stringifyJSON(xhr.response);
+                    if (text === '{}')
+                        return null; // stringifyJSON returns {} of it fails to serialize a property
+                    return text;
+                }
+                catch (e) {
+                    ClientConfig.getInstance().postNoibuErrorAndOptionallyDisableClient(`Unable to stringify JSON response: ${e}`, false, SEVERITY.warn);
+                    return null;
+                }
+            }
+            return null;
+        });
+    }
+    /**
+     * Takes a URL and returns true if it is determined to be on the same domain as the URL
+     * the script is running on. Ignores protocol and path, and allows the URL to be a subdomain.
+     * @param {string} requestURL the URL of a request to compare to the script website's URL
+     * @param {bool} isHostnameCheck the URL is a domain hostname, could be a super or sub domain
+     */
+    isURLSameDomain(requestURL, isHostnameCheck = false) {
+        if (typeof requestURL !== 'string' ||
+            !this.initialURLPartsReversed ||
+            this.initialURLPartsReversed.length < 1) {
+            return false;
+        }
+        let requestHostname = isHostnameCheck ? requestURL : ''; // has to be declared outside of `try`
+        // Use URL constructor to extract hostname
+        if (!isHostnameCheck) {
+            try {
+                let requestURLWithProtocol = requestURL;
+                if (requestURL.startsWith('//')) {
+                    // eslint-disable-next-line prefer-template
+                    requestURLWithProtocol = 'https:' + requestURL;
+                }
+                requestHostname = new URL(requestURLWithProtocol).hostname;
+            }
+            catch (e) {
+                // Return false if URL() is unable to parse the request URL
+                ClientConfig.getInstance().postNoibuErrorAndOptionallyDisableClient(`Unable to determine hostname for request URL: ${e}`, false, SEVERITY.warn);
+                return false;
+            }
+        }
+        const requestParts = requestHostname.split('.');
+        // Remove first subdomain from either URL if it is www., www1., etc.
+        if (requestParts.length < 1)
+            return false;
+        /* eslint-disable no-unused-expressions */
+        DEFAULT_WEBSITE_SUBDOMAIN_PATTERN.test(requestParts[0]) &&
+            requestParts.shift();
+        /* eslint-enable no-unused-expressions */
+        // Reverse order array
+        requestParts.reverse();
+        if (!isHostnameCheck &&
+            requestParts.length < this.initialURLPartsReversed.length)
+            return false;
+        // Return true if for every part in the website part, the same part exists
+        // in the request part, or if alsoCheckSuperDomain is true.
+        if (isHostnameCheck) {
+            const isSubDomain = requestParts.length >= this.initialURLPartsReversed.length &&
+                this.initialURLPartsReversed.every((part, i) => part === requestParts[i]);
+            const isSuperDomain = requestParts.length <= this.initialURLPartsReversed.length &&
+                requestParts.every((part, i) => part === this.initialURLPartsReversed[i]);
+            return isSubDomain || isSuperDomain;
+        }
+        // Default return false if alsoCheckSuperDomain is not true
+        return this.initialURLPartsReversed.every((part, i) => part === requestParts[i]);
+    }
+    /**
+     * Builds an HTTP Data bundle
+     */
+    bundleHTTPData(url, requestHeaders, rawRequestPayload, responseHeaders, rawResponsePayload, method, isError) {
+        if (!this.isValidRequest(url, method)) {
+            return null;
+        }
+        // stringify payload if correct type and not too large
+        let requestPayload = '';
+        let responsePayload = '';
+        if (this.shouldCollectPayloadForURL(url)) {
+            requestPayload =
+                this.getReasonPayloadIsDropped(requestHeaders, isError) ||
+                    this.stringFromRequestBody(rawRequestPayload, requestHeaders);
+            responsePayload =
+                this.getReasonPayloadIsDropped(responseHeaders, isError) ||
+                    this.stringFromRequestBody(rawResponsePayload, responseHeaders);
+        }
+        // don't bundle if there is no data
+        const safeRequestHeaders = requestHeaders || new Map();
+        const safeRequestPayload = requestPayload || '';
+        const safeResponseHeaders = responseHeaders || new Map();
+        const safeResponsePayload = responsePayload || '';
+        if (safeRequestHeaders.size === 0 &&
+            !safeRequestPayload &&
+            safeResponseHeaders.size === 0 &&
+            !safeResponsePayload) {
+            return null;
+        }
+        // Ensure payloads do not exceed the maximum size and redact PII
+        const requestPayloadUpdated = this.restrictPayload(safeRequestPayload, url, isError);
+        // Ensure payloads do not exceed the maximum size and redact PII
+        const responsePayloadUpdated = this.restrictPayload(safeResponsePayload, url, isError);
+        // Redact PII.
+        const cleanRequestHeaders = safeFromEntries(this.removePIIHeaders(requestHeaders));
+        const cleanResponseHeaders = safeFromEntries(this.removePIIHeaders(responseHeaders));
+        return {
+            [HTTP_DATA_REQ_HEADERS_ATT_NAME]: cleanRequestHeaders,
+            [HTTP_DATA_PAYLOAD_ATT_NAME]: requestPayloadUpdated,
+            [HTTP_DATA_RESP_HEADERS_ATT_NAME]: cleanResponseHeaders,
+            [HTTP_DATA_RESP_PAYLOAD_ATT_NAME]: responsePayloadUpdated,
+        };
+    }
+    /**
+     * Validates a request based on the URL and method. When enabled, will handle
+     * de-duping the requests
+     * @param {string} url
+     * @param {string} method
+     * @returns boolean indicating whether the validation passed
+     */
+    isValidRequest(url, method) {
+        if (!this.httpDataCollectionEnabled)
+            return false;
+        if (!method || typeof method !== 'string') {
+            return false;
+        }
+        // TODO: Check below disabled for beta. NOI-4253
+        // only bundle and send once per request/method/response code combination
+        // const urlWithoutParams = url.split('?')[0];
+        // const requestSignature = `${urlWithoutParams}-${method}-${responseCode}`;
+        // if (this.uniqueRequests.has(requestSignature)) {
+        //   return false;
+        // }
+        return true;
+    }
+    /**
+     * Checks two things: that the URL is either on the same domain (or an address relative to the
+     * current domain), and also checks that the config http_data_collection flag is enabled.
+     * @param {string} url
+     * @returns boolean indicating whether the URL passed is either relative (in which case it is
+     * inherently on the current domain) or matches the current domain.
+     */
+    shouldContinueForURL(url) {
+        if (!this.httpDataCollectionEnabled)
+            return false;
+        if (!url || typeof url !== 'string' || !this.initialURLPartsReversed) {
+            return false;
+        }
+        // URL is absolute; either "http://example.com" or "//example.com"
+        if (HTTPDataBundler.isAbsoluteURL(url)) {
+            // Only capture requests on the same domain or in the allowed list
+            if (!this.isURLSameDomain(url) && !this.shouldCollectPayloadForURL(url))
+                return false;
+        } // end `if` (if URL is relative, it is on the same domain.)
+        return true;
+    }
+    /**
+     * Determins if the URL is absolute or relative
+     * @param {string} url
+     * @returns boolean indicating whether the URL passed is either absolute or relative
+     */
+    static isAbsoluteURL(url) {
+        if (!url || typeof url !== 'string') {
+            return false;
+        }
+        return url.indexOf('://') > 0 || url.indexOf('//') === 0;
+    }
+    /**
+     * Checks whether HTTP payloads can be collected on this URL
+     * @returns boolean indicating whether HTTP payloads can be collected on this URL
+     */
+    shouldCollectPayloadForURL(url) {
+        if (!url || typeof url !== 'string') {
+            return false;
+        }
+        // check if in the full URL allowed list
+        const isAllowedAsAbsolute = !!this.httpDataAllowedAbsoluteRegex &&
+            this.httpDataAllowedAbsoluteRegex.test(url.toLowerCase());
+        const isRelative = !HTTPDataBundler.isAbsoluteURL(url) || this.isURLSameDomain(url);
+        // check if in the relative URL allowed list or if it is on the same domain
+        const isAllowedAsRelative = 
+        // if this is a relative URL (isURLSameDomain will fail) OR a full URL on domain
+        isRelative &&
+            !!this.httpDataAllowedRelativeRegex &&
+            this.httpDataAllowedRelativeRegex.test(url.toLowerCase());
+        return isAllowedAsAbsolute || isAllowedAsRelative;
+    }
+    /**
+     * Double checks content length if we couldn't read the headers, and redacts PII
+     * returns the restricted payload
+     */
+    restrictPayload(payload, url, isError) {
+        // if no payload, too large, payload already dropped,
+        // or payloads not allowed for this URL: nothing to do
+        if (!payload || !this.shouldCollectPayloadForURL(url)) {
+            return HTTP_BODY_NULL_STRING;
+        }
+        if (typeof payload !== 'string') {
+            ClientConfig.getInstance().postNoibuErrorAndOptionallyDisableClient({
+                msg: `restrictPayload received non string payload`,
+                payloadType: typeof payload,
+            }, false, SEVERITY.error);
+            return HTTP_BODY_NULL_STRING;
+        }
+        if (payload === HTTP_BODY_NULL_STRING ||
+            (!!payload.startsWith &&
+                (payload.startsWith(HTTP_BODY_DROPPED_LENGTH_MSG) ||
+                    payload.startsWith(HTTP_BODY_DROPPED_TYPE_MSG))) ||
+            (!!payload.indexOf &&
+                (payload.indexOf(HTTP_BODY_DROPPED_LENGTH_MSG) === 0 ||
+                    payload.indexOf(HTTP_BODY_DROPPED_TYPE_MSG) === 0))) {
+            return payload;
+        }
+        let restrictSize = isError
+            ? MAX_HTTP_DATA_PAYLOAD_LENGTH
+            : MAX_SUCCESS_HTTP_DATA_PAYLOAD_LENGTH;
+        // TODO CHARLIE: Temporary hack for www.holtrenfrew.com to limit them to 1.5MB to help solve a bug
+        if (this.hostname === 'www.holtrenfrew.com') {
+            restrictSize = 1.5 * 1024 * 1024;
+        }
+        if (payload.length > restrictSize) {
+            StoredMetrics.getInstance().addHttpDataDropByLength();
+            return `${HTTP_BODY_DROPPED_LENGTH_MSG} Payload length: ${payload.length}`;
+        }
+        // payload is good!
+        StoredMetrics.getInstance().addHttpDataPayloadCount();
+        return removePII(payload);
+    }
+    /**
+     * Returns true if the content-length header is of acceptable size.
+     * Too big gets rejected
+     * If the headers are not found, check actual content for length
+     * @param  {Headers} headers
+     * @returns boolean true if acceptable to collect
+     */
+    contentLengthAcceptable(headers, isError) {
+        // TODO This entire check should move into the parent's parent so there is a single place
+        // that restricts paylods and unterstands these 2 values
+        let restrictSize = isError
+            ? MAX_HTTP_DATA_PAYLOAD_LENGTH
+            : MAX_SUCCESS_HTTP_DATA_PAYLOAD_LENGTH;
+        // TODO CHARLIE: Temporary hack for www.holtrenfrew.com to limit them to 1.5MB to help solve a bug
+        if (this.hostname === 'www.holtrenfrew.com') {
+            restrictSize = 1.5 * 1024 * 1024;
+        }
+        return this.contentLength(headers) <= restrictSize;
+    }
+    /**
+     * Returns true if the content type according to the headers is valid for collection.
+     * Also returns assumed true if content type is not stated in headers.
+     * @param {Map} headersMap
+     * @returns boolean true if acceptable to collect
+     */
+    contentTypeAcceptable(headersMap) {
+        // check content type
+        const contentType = headersMap.get(CONTENT_TYPE);
+        return !(contentType &&
+            !this.contentTypeReadableRegex.test(contentType.toLowerCase()));
+    }
+    /**
+     * Returns a descriptive string if we have to drop payload based on the length
+     * or type listed in the headers passed. Returns an empty string otherwise.
+     */
+    getReasonPayloadIsDropped(headers, isError) {
+        if (!(headers === null || headers === void 0 ? void 0 : headers.get)) {
+            return '';
+        }
+        const bundler = HTTPDataBundler.getInstance();
+        if (!bundler.contentTypeAcceptable(headers)) {
+            // not a readable request, drop the content
+            StoredMetrics.getInstance().addHttpDataDropByType();
+            return `${HTTP_BODY_DROPPED_TYPE_MSG} Payload type: ${headers.get(CONTENT_TYPE)}`;
+        }
+        if (!bundler.contentLengthAcceptable(headers, isError)) {
+            StoredMetrics.getInstance().addHttpDataDropByLength();
+            return `${HTTP_BODY_DROPPED_LENGTH_MSG} Payload length: ${bundler.contentLength(headers)}`;
+        }
+        return '';
+    }
+    /**
+     * Returns content length from the headers, if available.
+     * If headers are not found or length is not a number, return -1
+     */
+    contentLength(headersObject) {
+        if (!(headersObject === null || headersObject === void 0 ? void 0 : headersObject.get)) {
+            return 0;
+        }
+        // get content length
+        let contentLength = 0;
+        const contentValueString = headersObject.get(CONTENT_LENGTH);
+        if (!contentValueString) {
+            // no content length header found
+            return -1;
+        }
+        try {
+            contentLength = parseInt(contentValueString, 10);
+            if (Number.isNaN(contentLength)) {
+                // bad content length header
+                return -1;
+            }
+        }
+        catch (err) {
+            // if we can't convert the value to a number, confirm after reading payload
+            return -1;
+        }
+        return contentLength;
+    }
+    /**
+     * Accepts a value that could be any type used as a request payload,
+     * and returns a string representation, or null if this fails.
+     */
+    stringFromRequestBody(value, requestHeaders) {
+        /* eslint-disable no-param-reassign */
+        // Nullish check to ensure we don't send 'null' or 'undefined' as a string
+        if (value == null)
+            return null;
+        try {
+            if (isString(value) && requestHeaders instanceof Map) {
+                const contentType = requestHeaders.get(CONTENT_TYPE);
+                if (contentType &&
+                    contentType
+                        .toLowerCase()
+                        .includes('application/x-www-form-urlencoded')) {
+                    value = new URLSearchParams(value.toString());
+                }
+            }
+        }
+        catch (error) {
+            // Nothing--try next try block
+        }
+        try {
+            // If the payload is a FormData object, turn it into an object.
+            if (value instanceof FormData || value instanceof URLSearchParams) {
+                value = safeEntries(value).reduce((obj, [key, val]) => {
+                    obj[key] = this.stringFromRequestBody(val);
+                    return obj;
+                }, {});
+            }
+        }
+        catch (error) {
+            // Nothing--try next try block
+        }
+        try {
+            // Case where value has a .toString() method
+            const returnVal = value.toString();
+            // Catch case where .toString() is on an object, we don't want that --
+            // should be stringified below.
+            if (!returnVal.includes('[object'))
+                return returnVal;
+        }
+        catch (error) {
+            // Nothing--try next try block
+        }
+        try {
+            // Case where value is a Document type (HTML/XML)
+            return value.documentElement.innerHTML;
+        }
+        catch (error) {
+            // Nothing--try next try block
+        }
+        try {
+            // Case where value is any other object type
+            return stringifyJSON(value);
+        }
+        catch (e) {
+            ClientConfig.getInstance().postNoibuErrorAndOptionallyDisableClient(`Unable to stringify request body: ${e}`, false, SEVERITY.warn);
+        }
+        return null;
     }
-
-    let redacted = false;
-    try {
-      const instance = JSON.parse(content);
-      iterateObjectRecursively(instance, (current, property) => {
-        const propertyLowerCase = property.toLowerCase();
-        // check for exact matches
-        if (this.exactFieldsToRedact.some(v => propertyLowerCase === v)) {
-          redacted = true;
-          return PII_REDACTION_REPLACEMENT_STRING;
-        }
-        // check for fuzzy matches
-        if (this.fuzzyFieldsToRedact.some(v => propertyLowerCase.includes(v))) {
-          redacted = true;
-          return PII_REDACTION_REPLACEMENT_STRING;
-        }
-        return undefined;
-      });
-
-      return redacted ? JSON.stringify(instance) : content;
-    } catch (e) {
-      return content;
+    /**
+     * Removes possible PII from headers.
+     * @param {Map} dirtyHeaders a Map of HTTP response or request headers
+     * @returns {Map|null} a map of headers with PII redacted
+     */
+    removePIIHeaders(dirtyHeaders) {
+        // In order to be fail-safe, let's return null if we won't be able to remove PII headers.
+        if (!(dirtyHeaders instanceof Map)) {
+            return null;
+        }
+        if (!dirtyHeaders.size) {
+            return dirtyHeaders;
+        }
+        const cleanHeaders = new Map(dirtyHeaders);
+        cleanHeaders.forEach((headerVal, headerKey, map) => {
+            // If the key is known PII, redact
+            if (BLOCKED_HTTP_HEADER_KEYS.includes(headerKey.toLowerCase())) {
+                map.set(headerKey, PII_REDACTION_REPLACEMENT_STRING);
+                // If not, run the header's value through the string redaction function
+            }
+            else {
+                map.set(headerKey, removePII(headerVal));
+            }
+        });
+        return cleanHeaders;
     }
-  }
 }
 
 export { HTTPDataBundler };