noibu-react-native 0.0.1

package/dist/monitors/httpDataBundler.js

@@ -0,0 +1,665 @@
+import { HUMAN_READABLE_CONTENT_TYPE_REGEX, DEFAULT_WEBSITE_SUBDOMAIN_PATTERN, SEVERITY_WARN, HTTP_BODY_NULL_STRING, HTTP_DATA_REQ_HEADERS_ATT_NAME, HTTP_DATA_PAYLOAD_ATT_NAME, HTTP_DATA_RESP_HEADERS_ATT_NAME, HTTP_DATA_RESP_PAYLOAD_ATT_NAME, HTTP_BODY_DROPPED_LENGTH_MSG, HTTP_BODY_DROPPED_TYPE_MSG, MAX_HTTP_DATA_PAYLOAD_LENGTH, CONTENT_TYPE, CONTENT_LENGTH, BLOCKED_HTTP_HEADER_KEYS, PII_REDACTION_REPLACEMENT_STRING, HTTP_PII_BLOCKING_PATTERNS } from '../constants.js';
+import { getProperGlobalUrl, checkHttpDataCollectionEnabled, getHttpPayloadAllowedURLs } from '../utils/function.js';
+import ClientConfig from '../api/clientConfig.js';
+import StoredMetrics from '../api/storedMetrics.js';
+import { safeFromEntries, iterateObjectRecursively } from '../utils/object.js';
+
+/** @module HTTPDataBundler */
+
+/** Bundles HTTP payloads and headers */
+class HTTPDataBundler {
+  /**
+   * Creates an instance of the ClickMonitor instance
+   */
+  constructor() {
+    // compile regex only once
+    this.contentTypeReadableRegex = new RegExp(
+      HUMAN_READABLE_CONTENT_TYPE_REGEX,
+      'i',
+    );
+
+    // pull out the domain hostname
+    const initialURL = getProperGlobalUrl();
+    this.initialURLPartsReversed = [];
+    if (initialURL && initialURL.length > 0) {
+      try {
+        // Store URL parts to class variable
+        const initialHostname = new URL(initialURL).hostname;
+        this.initialURLPartsReversed = initialHostname.split('.');
+
+        // Remove www. etc.
+        // eslint-disable-next-line no-unused-expressions
+        DEFAULT_WEBSITE_SUBDOMAIN_PATTERN.test(
+          this.initialURLPartsReversed[0],
+        ) && this.initialURLPartsReversed.shift();
+
+        // Reverse array
+        this.initialURLPartsReversed.reverse();
+      } catch (e) {
+        ClientConfig.getInstance().postNoibuErrorAndOptionallyDisableClient(
+          `Unable to determine hostname for initial URL: ${e}`,
+          false,
+          SEVERITY_WARN,
+        );
+      }
+    }
+
+    this.httpDataCollectionEnabled = checkHttpDataCollectionEnabled();
+
+    // compile the relative and full HTTP URL regexes
+    const allowedURLs = getHttpPayloadAllowedURLs();
+    this.httpDataAllowedAbsoluteRegex = HTTPDataBundler.buildAllowedRegex(
+      allowedURLs,
+      true,
+    );
+    this.httpDataAllowedRelativeRegex = HTTPDataBundler.buildAllowedRegex(
+      allowedURLs,
+      false,
+    );
+    // track unique requests and only capture each once
+    // TODO: disabled for beta. NOI-4253
+    // this.uniqueRequests = new Set();
+
+    this.fuzzyFieldsToRedact = [
+      'password',
+      'address',
+      'credit',
+      'postal',
+      'token',
+      'phone',
+      'mobile',
+    ];
+
+    this.exactFieldsToRedact = [
+      'firstname',
+      'lastname',
+      'street',
+      'fullname',
+      'creditcard',
+      'postcode',
+      'zipcode',
+      'city',
+      'town',
+      'county',
+      'cc',
+    ];
+  }
+
+  /** gets the singleton instance */
+  static getInstance() {
+    if (!this.instance) {
+      this.instance = new HTTPDataBundler();
+    }
+
+    return this.instance;
+  }
+
+  /**
+   * Builds the HTTP payload allowed regexes for full and relative URLs
+   * @param allowedURLs A list of allowed URLs
+   * @param absolute Use only absolute URLs if true, use only relative URL if false
+   * @returns a regex of allowed URLs
+   */
+  static buildAllowedRegex(allowedURLs, absolute) {
+    if (!allowedURLs) return null;
+    const allowedURLsFiltered = allowedURLs.filter(url => {
+      const isAbsolute = HTTPDataBundler.isAbsoluteURL(url);
+      return absolute ? isAbsolute : !isAbsolute;
+    });
+    if (allowedURLsFiltered.length > 0) {
+      const lowerCasedURLs = allowedURLsFiltered.map(url =>
+        url.trim().toLowerCase(),
+      );
+      return new RegExp(lowerCasedURLs.join('|'));
+    }
+    return null;
+  }
+
+  /**
+   * Takes an iterator and returns a map of strings representing headers.
+   * @param {object} headersIterable any iterable object
+   * @returns a map of strings (as expected by metroplex) representing HTTP
+   * request or response headers
+   */
+  static headersMapFromIterable(headersIterable) {
+    const headerMap = new Map();
+    // eslint-disable-next-line no-restricted-syntax
+    for (const header of headersIterable) {
+      // Ensure we're working with strings
+      // This will also automatically convert null to "null"
+      if (typeof header[0] !== 'string') header[0] = String(header[0]);
+      if (typeof header[1] !== 'string') header[1] = String(header[1]);
+
+      headerMap.set(header[0].toLowerCase(), header[1]);
+    }
+    return headerMap;
+  }
+
+  /**
+   * Takes a string of headers with 'name: value' and returns
+   * a map of strings representing headers.
+   * @param {string} headersString is all the headers in one string
+   * @returns a map of strings (as expected by metroplex) representing HTTP
+   * request or response headers
+   */
+  static headersMapFromString(headersString) {
+    const headerMap = new Map();
+    if (!headersString || typeof headersString !== 'string') return headerMap;
+    const headerStringArray = headersString.split('\r\n').filter(Boolean);
+    headerStringArray.forEach(headerString => {
+      const split = headerString.split(': ');
+      if (split.length === 2 && split[0].length > 0 && split[1].length > 0) {
+        headerMap.set(split[0].toLowerCase(), split[1]);
+      }
+    });
+    return headerMap;
+  }
+
+  /**
+   * For an XHR object, checks the responseType property and handles the response or
+   * responseText property accordingly to return a string representation of the response.
+   * @param {object} xhr an XMLHTTPRequest object
+   * @returns a string representation of the response, or null if this fails.
+   */
+  static responseStringFromXHRResponseType(xhr) {
+    // This should not happen, as this function is called from a method on xhr.
+    if (xhr === undefined || xhr === null) return null;
+
+    // If the XHR object exists but the response is null, return null as a string.
+    if (xhr.response && xhr.response === null) return HTTP_BODY_NULL_STRING;
+
+    if (xhr.responseType === '' || xhr.responseType === 'text') {
+      return xhr.responseText;
+    }
+    if (
+      xhr.responseType === 'document' &&
+      !!xhr.response &&
+      !!xhr.response.documentElement &&
+      !!xhr.response.documentElement.innerHTML
+    ) {
+      return xhr.response.documentElement.innerHTML;
+    }
+    if (xhr.responseType === 'json') {
+      try {
+        return JSON.stringify(xhr.response);
+      } catch (e) {
+        ClientConfig.getInstance().postNoibuErrorAndOptionallyDisableClient(
+          `Unable to stringify JSON response: ${e}`,
+          false,
+          SEVERITY_WARN,
+        );
+        return null;
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Takes a URL and returns true if it is determined to be on the same domain as the URL
+   * the script is running on. Ignores protocol and path, and allows the URL to be a subdomain.
+   * @param {string} requestURL the URL of a request to compare to the script website's URL
+   */
+  isURLSameDomain(requestURL) {
+    if (
+      typeof requestURL !== 'string' ||
+      !this.initialURLPartsReversed ||
+      this.initialURLPartsReversed.length < 1
+    ) {
+      return false;
+    }
+
+    let requestHostname; // has to be declared outside of `try`
+
+    // Use URL constructor to extract hostname
+    try {
+      requestHostname = new URL(requestURL).hostname;
+    } catch (e) {
+      // Return false if URL() is unable to parse the request URL
+      ClientConfig.getInstance().postNoibuErrorAndOptionallyDisableClient(
+        `Unable to determine hostname for request URL: ${e}`,
+        false,
+        SEVERITY_WARN,
+      );
+
+      return false;
+    }
+
+    const requestParts = requestHostname.split('.');
+
+    // Remove first subdomain from either URL if it is www., www1., etc.
+
+    if (requestParts.length < 1) return false;
+
+    /* eslint-disable no-unused-expressions */
+    DEFAULT_WEBSITE_SUBDOMAIN_PATTERN.test(requestParts[0]) &&
+      requestParts.shift();
+    /* eslint-enable no-unused-expressions */
+
+    if (requestParts.length < this.initialURLPartsReversed.length) return false;
+
+    // Reverse order array
+    requestParts.reverse();
+
+    // Return true if for every part in the website part, the same part exists
+    // in the request part.
+    return this.initialURLPartsReversed.every(
+      (part, i) => part === requestParts[i],
+    );
+  }
+
+  /**
+   * Builds an HTTP Data bundle
+   * @param  {} url
+   * @param  {} requestHeaders is a map of request headers
+   * @param  {} requestPayload
+   * @param  {} responseHeaders is a map of response headers
+   * @param  {} responsePayload
+   * @param  {} method
+   * @param  {} isError Response code is an error
+   */
+  bundleHTTPData(
+    url,
+    requestHeaders,
+    rawRequestPayload,
+    responseHeaders,
+    responsePayload,
+    method,
+    isError,
+  ) {
+    if (!this.isValidRequest(url, method, isError)) return null;
+
+    // stringify payload if correct type and not too large
+    let requestPayload = '';
+    if (this.shouldCollectPayloadForURL(url)) {
+      requestPayload =
+        this.dropPayloadIfNecessaryFromHeaders(requestHeaders) ||
+        this.stringFromRequestBody(rawRequestPayload);
+    }
+
+    // don't bundle if there is no data
+    const safeRequestHeaders = requestHeaders || new Map();
+    const safeRequestPayload = requestPayload || '';
+    const safeResponseHeaders = responseHeaders || new Map();
+    const safeResponsePayload = responsePayload || '';
+    if (
+      safeRequestHeaders.size === 0 &&
+      safeRequestPayload.length === 0 &&
+      safeResponseHeaders.size === 0 &&
+      safeResponsePayload.length === 0
+    ) {
+      return null;
+    }
+
+    // Ensure payloads do not exceed the maximum size and redact PII
+    const requestPayloadUpdated = this.restrictPayload(requestPayload, url);
+    // Ensure payloads do not exceed the maximum size and redact PII
+    const responsePayloadUpdated = this.restrictPayload(responsePayload, url);
+
+    // Redact PII.
+    const cleanRequestHeaders = this.removePIIHeaders(requestHeaders);
+    const cleanResponseHeaders = this.removePIIHeaders(responseHeaders);
+
+    // TODO: disabled for beta. NOI-4253
+    // we are capturing this request, so add it to the unique requests set
+    // this.uniqueRequests.add(requestSignature);
+    // build the http bundle
+    return {
+      [HTTP_DATA_REQ_HEADERS_ATT_NAME]: cleanRequestHeaders
+        ? safeFromEntries(cleanRequestHeaders)
+        : {},
+      [HTTP_DATA_PAYLOAD_ATT_NAME]: requestPayloadUpdated,
+      [HTTP_DATA_RESP_HEADERS_ATT_NAME]: cleanResponseHeaders
+        ? safeFromEntries(cleanResponseHeaders)
+        : {},
+      [HTTP_DATA_RESP_PAYLOAD_ATT_NAME]: responsePayloadUpdated,
+    };
+  }
+
+  /**
+   * Validates a request based on the URL and method. When enabled, will handle
+   * de-duping the requests
+   * @param {string} url
+   * @param {string} method
+   * @param  {} isError Response code is an error
+   * @returns boolean indicating whether the validation passed
+   */
+  isValidRequest(url, method, isError) {
+    if (!this.httpDataCollectionEnabled) return false;
+    if (!method || typeof method !== 'string') {
+      return false;
+    }
+    if (!isError) return false;
+    // TODO: Check below disabled for beta. NOI-4253
+    // only bundle and send once per request/method/response code combination
+    // const urlWithoutParams = url.split('?')[0];
+    // const requestSignature = `${urlWithoutParams}-${method}-${responseCode}`;
+    // if (this.uniqueRequests.has(requestSignature)) {
+    //   return false;
+    // }
+
+    return true;
+  }
+
+  /**
+   * Checks two things: that the URL is either on the same domain (or an address relative to the
+   * current domain), and also checks that the config http_data_collection flag is enabled.
+   * @param {string} url
+   * @returns boolean indicating whether the URL passed is either relative (in which case it is
+   * inherently on the current domain) or matches the current domain.
+   */
+  shouldContinueForURL(url) {
+    if (!this.httpDataCollectionEnabled) return false;
+    if (!url || typeof url !== 'string' || !this.initialURLPartsReversed) {
+      return false;
+    }
+    // URL is absolute; either "http://example.com" or "//example.com"
+    if (HTTPDataBundler.isAbsoluteURL(url)) {
+      // Only capture requests on the same domain or in the allowed list
+      if (!this.isURLSameDomain(url) && !this.shouldCollectPayloadForURL(url))
+        return false;
+    } // end `if` (if URL is relative, it is on the same domain.)
+    return true;
+  }
+
+  /**
+   * Determins if the URL is absolute or relative
+   * @param {string} url
+   * @returns boolean indicating whether the URL passed is either absolute or relative
+   */
+  static isAbsoluteURL(url) {
+    if (!url || typeof url !== 'string') {
+      return false;
+    }
+    return url.indexOf('://') > 0 || url.indexOf('//') === 0;
+  }
+
+  /**
+   * Checks whether HTTP payloads can be collected on this URL
+   * @param {string} url
+   * @returns boolean indicating whether HTTP payloads can be collected on this URL
+   */
+  shouldCollectPayloadForURL(url) {
+    if (!url || typeof url !== 'string') {
+      return false;
+    }
+    // check if in the full URL allowed list
+    if (
+      this.httpDataAllowedAbsoluteRegex &&
+      this.httpDataAllowedAbsoluteRegex.test(url.toLowerCase())
+    ) {
+      return true;
+    }
+    // check if in the relative URL allowed list, if on domain
+    if (
+      this.httpDataAllowedRelativeRegex &&
+      // if this is a relative URL (isURLSameDomain will fail) OR a full URL on domain
+      (!HTTPDataBundler.isAbsoluteURL(url) || this.isURLSameDomain(url)) &&
+      this.httpDataAllowedRelativeRegex.test(url.toLowerCase())
+    ) {
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * Double checks content length if we couldn't read the headers, and redacts PII
+   * @param  {string} payload
+   * @param  {string} url
+   * @returns the restricted payload
+   */
+  restrictPayload(payload, url) {
+    // if no payload, too large, payload already dropped,
+    // or payloads not allowed for this URL: nothing to do
+    if (!payload || !this.shouldCollectPayloadForURL(url)) {
+      return HTTP_BODY_NULL_STRING;
+    }
+    if (
+      payload === HTTP_BODY_NULL_STRING ||
+      payload.startsWith(HTTP_BODY_DROPPED_LENGTH_MSG) ||
+      payload.startsWith(HTTP_BODY_DROPPED_TYPE_MSG)
+    ) {
+      return payload;
+    }
+    if (payload.length > MAX_HTTP_DATA_PAYLOAD_LENGTH) {
+      StoredMetrics.getInstance().addHttpDataDropByLength();
+      return `${HTTP_BODY_DROPPED_LENGTH_MSG} Payload length: ${payload.length}`;
+    }
+
+    // payload is good!
+    StoredMetrics.getInstance().addHttpDataPayloadCount();
+
+    return this.removePIIBody(payload);
+  }
+
+  /**
+   * Returns true if the content length is acceptable to collect
+   * If the headers are not found, check actual content for length
+   * @param  {Headers} headers
+   * @returns boolean true if acceptable to collect
+   */
+  contentLengthAcceptable(headers) {
+    // check content length
+    const contentLength = this.contentLength(headers);
+    if (contentLength < 0) {
+      // no content length or can't read, keep going
+      return true;
+    }
+    if (contentLength > MAX_HTTP_DATA_PAYLOAD_LENGTH) {
+      StoredMetrics.getInstance().addHttpDataDropByLength();
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Returns true if the content type according to the headers is valid for collection.
+   * Also returns assumed true if content type is not stated in headers.
+   * @param {Map} headersMap
+   * @returns boolean true if acceptable to collect
+   */
+  contentTypeAcceptable(headersMap) {
+    if (!headersMap || !headersMap.get) return true;
+
+    // check content type
+    const contentType = headersMap.get(CONTENT_TYPE);
+    if (
+      contentType &&
+      !this.contentTypeReadableRegex.test(contentType.toLowerCase())
+    ) {
+      // not a readable request, drop the content
+      StoredMetrics.getInstance().addHttpDataDropByType();
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Returns a descriptive string if we have to drop payload based on the length
+   * or type listed in the headers passed. Returns an empty string otherwise.
+   * @param {Headers} headers a Headers object
+   * @returns A string, which is empty if the payload doesn't need to be dropped, or is a
+   * descriptive string explaining the circumstances of the drop otherwise.
+   */
+  dropPayloadIfNecessaryFromHeaders(headers) {
+    // Set descriptive body if content is wrong type or too long.
+    let text = '';
+    if (!HTTPDataBundler.getInstance().contentTypeAcceptable(headers)) {
+      text = HTTP_BODY_DROPPED_TYPE_MSG;
+      if (headers && headers.get) {
+        text += ` Payload type: ${headers.get(CONTENT_TYPE)}`;
+      }
+    } else if (
+      !HTTPDataBundler.getInstance().contentLengthAcceptable(headers)
+    ) {
+      text = `${HTTP_BODY_DROPPED_LENGTH_MSG} Payload length: \
+${HTTPDataBundler.getInstance().contentLength(headers)}`;
+    }
+    return text;
+  }
+
+  /**
+   * Returns content length from the headers, if available
+   * If the headers are not found or not a number, return -1
+   * @param  {} headersObject
+   * @returns content length
+   */
+  contentLength(headersObject) {
+    if (!headersObject || !headersObject.get) return 0;
+
+    // get content length
+    let contentLength = 0;
+    const contentValueString = headersObject.get(CONTENT_LENGTH);
+    // no content length header found
+    if (!contentValueString) {
+      return -1;
+    }
+    try {
+      contentLength = parseInt(contentValueString, 10);
+      if (Number.isNaN(contentLength)) {
+        // bad content length heade
+        return -1;
+      }
+    } catch (err) {
+      // if we can't convert the value to a number, confirm after reading payload
+      return -1;
+    }
+    return contentLength;
+  }
+
+  /**
+   * Accepts a value that could be any type used as a request payload, and returns a string
+   * representation. First attemtps to find a .toString() method, then tries to handle it
+   * like an XML or HTML element, then finally falls back to JSON.stringify(). If none of
+   * these are successful, returns null.
+   * @param {*} value request paylad body, of any type
+   * @returns string representation of the value passed, or null if this fails.
+   */
+  stringFromRequestBody(value) {
+    // Nullish check to ensure we don't send 'null' or 'undefined' as a string
+    if (value == null) return null;
+
+    try {
+      // Case where value has a .toString() method
+      const returnVal = value.toString();
+      // Catch case where .toString() is on an object, we don't want that --
+      // should be stringified below.
+      if (!returnVal.includes('[object')) return returnVal;
+    } catch (error) {
+      // Nothing--try next try block
+    }
+    try {
+      // Case where value is a Document type (HTML/XML)
+      return value.documentElement.innerHTML;
+    } catch (error) {
+      // Nothing--try next try block
+    }
+    try {
+      // If the payload is a FormData object, turn it into an object.
+      if (value instanceof FormData) {
+        // eslint-disable-next-line no-param-reassign
+        value = Array.from(value.entries()).reduce((obj, [key, val]) => {
+          // eslint-disable-next-line no-param-reassign
+          obj[key] = typeof val === 'string' ? val : 'non-string value.';
+          return obj;
+        }, {});
+      }
+      // Case where value is any other object type
+      return JSON.stringify(value);
+    } catch (e) {
+      ClientConfig.getInstance().postNoibuErrorAndOptionallyDisableClient(
+        `Unable to stringify request body: ${e}`,
+        false,
+        SEVERITY_WARN,
+      );
+    }
+    return null;
+  }
+
+  /**
+   * Removes possible PII from headers.
+   * @param {Map} dirtyHeaders a Map of HTTP response or request headers
+   * @returns a map of headers with PII redacted
+   */
+  removePIIHeaders(dirtyHeaders) {
+    // In order to be fail-safe, let's return null if we won't be able to remove PII headers.
+    if (!(dirtyHeaders instanceof Map)) return null;
+    if (dirtyHeaders.size < 1) return dirtyHeaders;
+
+    const cleanHeaders = new Map(dirtyHeaders);
+
+    cleanHeaders.forEach((headerVal, headerKey, map) => {
+      // If the key is known PII, redact
+      if (BLOCKED_HTTP_HEADER_KEYS.includes(headerKey.toLowerCase())) {
+        map.set(headerKey, PII_REDACTION_REPLACEMENT_STRING);
+        // If not, run the header's value through the string redaction function
+      } else {
+        map.set(headerKey, this.removePIIBody(headerVal));
+      }
+    });
+
+    return cleanHeaders;
+  }
+
+  /**
+   * Takes a body payload string and redacts any PII we're able to detect
+   *
+   * @param {string} dirtyBody the string from which we want to redact PII
+   * @returns the string with any PII we're able to detect redacted.
+   */
+  removePIIBody(dirtyBody) {
+    // In order to be fail-safe, let's return null if we won't be able to remove PII.
+    if (typeof dirtyBody !== 'string') return null;
+    if (dirtyBody.length < 1) return dirtyBody;
+
+    let cleanString = this.tryPaseObjectAndRemovePII(dirtyBody);
+    // Go through each of the PII matching patterns
+    HTTP_PII_BLOCKING_PATTERNS.forEach(pattern => {
+      // Replace all matches with the appropriate number of asterisks
+      cleanString = cleanString.replace(
+        pattern,
+        PII_REDACTION_REPLACEMENT_STRING,
+      );
+    });
+
+    return cleanString;
+  }
+
+  /**
+   * Try to parse content as a JSON object and
+   * iterate it recursively removing PII.
+   * Returns original content if nothing was changes or error is thrown.
+   * @param {String} content
+   */
+  tryPaseObjectAndRemovePII(content) {
+    const first = content[0];
+    const isParsable = first === '{' || first === '[';
+    if (!isParsable) {
+      return content;
+    }
+
+    let redacted = false;
+    try {
+      const instance = JSON.parse(content);
+      iterateObjectRecursively(instance, (current, property) => {
+        const propertyLowerCase = property.toLowerCase();
+        // check for exact matches
+        if (this.exactFieldsToRedact.some(v => propertyLowerCase === v)) {
+          redacted = true;
+          return PII_REDACTION_REPLACEMENT_STRING;
+        }
+        // check for fuzzy matches
+        if (this.fuzzyFieldsToRedact.some(v => propertyLowerCase.includes(v))) {
+          redacted = true;
+          return PII_REDACTION_REPLACEMENT_STRING;
+        }
+        return undefined;
+      });
+
+      return redacted ? JSON.stringify(instance) : content;
+    } catch (e) {
+      return content;
+    }
+  }
+}
+
+export { HTTPDataBundler };