newrelic 5.8.0 → 5.11.0

package/lib/config/env.js

@@ -36,7 +36,10 @@ const ENV_MAPPING = {
       enabled: 'NEW_RELIC_ERROR_COLLECTOR_ATTRIBUTES_ENABLED'
     },
     enabled: 'NEW_RELIC_ERROR_COLLECTOR_ENABLED',
-    ignore_status_codes: 'NEW_RELIC_ERROR_COLLECTOR_IGNORE_ERROR_CODES'
+    ignore_status_codes: 'NEW_RELIC_ERROR_COLLECTOR_IGNORE_ERROR_CODES',
+    ignore_classes: 'NEW_RELIC_ERROR_COLLECTOR_IGNORE_ERRORS',
+    expected_status_codes: 'NEW_RELIC_ERROR_COLLECTOR_EXPECTED_ERROR_CODES',
+    expected_classes: 'NEW_RELIC_ERROR_COLLECTOR_EXPECTED_ERRORS',
   },
   strip_exception_messages: {
     enabled: 'NEW_RELIC_STRIP_EXCEPTION_MESSAGES_ENABLED'
@@ -147,6 +150,9 @@ const LIST_VARS = new Set([
   'NEW_RELIC_ATTRIBUTES_EXCLUDE',
   'NEW_RELIC_ATTRIBUTES_INCLUDE',
   'NEW_RELIC_ERROR_COLLECTOR_IGNORE_ERROR_CODES',
+  'NEW_RELIC_ERROR_COLLECTOR_IGNORE_ERRORS',
+  'NEW_RELIC_ERROR_COLLECTOR_EXPECTED_ERROR_CODES',
+  'NEW_RELIC_ERROR_COLLECTOR_EXPECTED_ERRORS',
   'NEW_RELIC_ERROR_COLLECTOR_ATTRIBUTES_EXCLUDE',
   'NEW_RELIC_ERROR_COLLECTOR_ATTRIBUTES_INCLUDE',
   'NEW_RELIC_TRANSACTION_TRACER_ATTRIBUTES_EXCLUDE',

package/lib/config/index.js

@@ -2,6 +2,7 @@
 
 const AttributeFilter = require('./attribute-filter')
 const CollectorResponse = require('../collector/response')
+const MergeServerConfig = require('./merge-server-config')
 const copy = require('../util/copy')
 const defaultConfig = require('./default').config
 const EventEmitter = require('events').EventEmitter
@@ -31,9 +32,11 @@ const CONFIG_FILE_LOCATIONS = [
   path.join(__dirname, '../../../..') // above node_modules
 ]
 
-const HAS_ARBITRARY_KEYS = [
+const HAS_ARBITRARY_KEYS = new Set([
+  'ignore_messages',
+  'expected_messages',
   'labels'
-]
+])
 
 const LASP_MAP = require('./lasp').LASP_MAP
 const ENV = require('./env')
@@ -72,7 +75,7 @@ function fromObjectList(setting) {
   try {
     return JSON.parse('[' + setting + ']')
   } catch (error) {
-    logger.error("New Relic configurator could not deserialize object list:")
+    logger.error('New Relic configurator could not deserialize object list:')
     logger.error(error.stack)
   }
 }
@@ -96,6 +99,8 @@ function _findConfigFile() {
 function Config(config) {
   EventEmitter.call(this)
 
+  // helper object for merging server side values
+  this.mergeServerConfig = new MergeServerConfig(this)
   // 1. start by cloning the defaults
   try {
     Object.assign(this, defaultConfig())
@@ -151,7 +156,7 @@ function Config(config) {
   this.browser_monitoring.loader_version = ''
 
   // Settings to play nice with DLPs (see NODE-1044).
-  this.compressed_content_encoding = "deflate"  // Deflate or gzip
+  this.compressed_content_encoding = 'deflate'  // Deflate or gzip
   this.simple_compression = false               // Disables subcomponent compression
   this.put_for_data_send = false                // Changes http verb for harvest
 
@@ -178,10 +183,10 @@ function Config(config) {
   if (this.high_security) {
     if (this.security_policies_token) {
       throw new Error(
-        "Security Policies and High Security Mode cannot both be present " +
-        "in the agent configuration. If Security Policies have been set " +
-        "for your account, please ensure the security_policies_token is " +
-        "set but high_security is disabled (default)."
+        'Security Policies and High Security Mode cannot both be present ' +
+        'in the agent configuration. If Security Policies have been set ' +
+        'for your account, please ensure the security_policies_token is ' +
+        'set but high_security is disabled (default).'
       )
     }
     this._applyHighSecurity()
@@ -398,7 +403,7 @@ Config.prototype._fromServer = function _fromServer(params, key) {
       )
       break
     case 'error_collector.ignore_status_codes':
-      this._updateNestedIfChanged(
+      this._validateThenUpdateStatusCodes(
         params,
         this.error_collector,
         'error_collector.ignore_status_codes',
@@ -406,6 +411,47 @@ Config.prototype._fromServer = function _fromServer(params, key) {
       )
       this._canonicalize()
       break
+    case 'error_collector.expected_status_codes':
+      this._validateThenUpdateStatusCodes(
+        params,
+        this.error_collector,
+        'error_collector.expected_status_codes',
+        'expected_status_codes'
+      )
+      this._canonicalize()
+      break
+    case 'error_collector.ignore_classes':
+      this._validateThenUpdateErrorClasses(
+        params,
+        this.error_collector,
+        'error_collector.ignore_classes',
+        'ignore_classes'
+      )
+      break
+    case 'error_collector.expected_classes':
+      this._validateThenUpdateErrorClasses(
+        params,
+        this.error_collector,
+        'error_collector.expected_classes',
+        'expected_classes'
+      )
+      break
+    case 'error_collector.ignore_messages':
+      this._validateThenUpdateErrorMessages(
+        params,
+        this.error_collector,
+        'error_collector.ignore_messages',
+        'ignore_messages'
+      )
+      break
+    case 'error_collector.expected_messages':
+      this._validateThenUpdateErrorMessages(
+        params,
+        this.error_collector,
+        'error_collector.expected_messages',
+        'expected_messages'
+      )
+      break
     case 'error_collector.capture_events':
       this._updateNestedIfChanged(
         params,
@@ -550,6 +596,153 @@ Config.prototype._updateIfChanged = function _updateIfChanged(json, key) {
   this._updateNestedIfChanged(json, this, key, key)
 }
 
+/**
+ * Expected and Ignored status code configuration values should look like this
+ *
+ *     [500,'501','503-507']
+ *
+ * If the server side config is not in this format, it might put the agent
+ * in a world of hurt.  So, before we pass everything on to
+ * _updateNestedIfChanged, we'll do some validation.
+ *
+ * @param {object} remote    JSON sent from New Relic.
+ * @param {object} local     A portion of this configuration object.
+ * @param {string} remoteKey The name sent by New Relic.
+ * @param {string} localKey  The local name.
+ */
+Config.prototype._validateThenUpdateStatusCodes = _validateThenUpdateStatusCodes
+function _validateThenUpdateStatusCodes(remote, local, remoteKey, localKey) {
+  let valueToTest = remote[remoteKey]
+  if (!Array.isArray(valueToTest)) {
+    logger.warn(
+      'Saw SSC (ignore|expect)_status_codes that is not an array, will not merge: %s',
+      valueToTest
+    )
+    return
+  }
+
+  let valid = true
+  valueToTest.forEach(function validateArray(thingToTest) {
+    if (!('string' === (typeof thingToTest) || 'number' === (typeof thingToTest))) {
+      logger.warn(
+        'Saw SSC (ignore|expect)_status_code that is not a number or string,' +
+        'will not merge: %s', thingToTest
+      )
+      valid = false
+    }
+  })
+  if (!valid) {
+    return
+  }
+
+  return this._updateNestedIfChanged(remote, local, remoteKey, localKey)
+}
+
+/**
+ * Expected and Ignored classes configuration values should look like this
+ *
+ *     ['Error','Again']
+ *
+ * If the server side config is not in this format, it might put the agent
+ * in a world of hurt.  So, before we pass everything on to
+ * _updateNestedIfChanged, we'll do some validation.
+ *
+ * @param {object} remote    JSON sent from New Relic.
+ * @param {object} local     A portion of this configuration object.
+ * @param {string} remoteKey The name sent by New Relic.
+ * @param {string} localKey  The local name.
+ */
+Config.prototype._validateThenUpdateErrorClasses = _validateThenUpdateErrorClasses
+
+function _validateThenUpdateErrorClasses(remote, local, remoteKey, localKey) {
+  let valueToTest = remote[remoteKey]
+  if (!Array.isArray(valueToTest)) {
+    logger.warn(
+      'Saw SSC (ignore|expect)_classes that is not an array, will not merge: %s',
+      valueToTest
+    )
+    return
+  }
+
+  let valid = true
+  Object.keys(valueToTest).forEach(function validateArray(key) {
+    let thingToTest = valueToTest[key]
+    if ('string' !== (typeof thingToTest)) {
+      logger.warn(
+        'Saw SSC (ignore|expect)_class that is not a string, will not merge: %s',
+        thingToTest
+      )
+      valid = false
+    }
+  })
+  if (!valid) {
+    return
+  }
+
+  return this._updateNestedIfChanged(remote, local, remoteKey, localKey)
+}
+
+/**
+ * Expected and Ignore messages configuration values should look like this
+ *
+ *     {'ErrorType':['Error Message']}
+ *
+ * If the server side config is not in this format, it might put the agent
+ * in a world of hurt.  So, before we pass everything on to
+ * _updateNestedIfChanged, we'll do some validation.
+ *
+ * @param {object} remote    JSON sent from New Relic.
+ * @param {object} local     A portion of this configuration object.
+ * @param {string} remoteKey The name sent by New Relic.
+ * @param {string} localKey  The local name.
+ */
+Config.prototype._validateThenUpdateErrorMessages = _validateThenUpdateErrorMessages
+
+function _validateThenUpdateErrorMessages(remote, local, remoteKey, localKey) {
+  let valueToTest = remote[remoteKey]
+  if (Array.isArray(valueToTest)) {
+    logger.warn(
+      'Saw SSC (ignore|expect)_message that is an Array, will not merge: %s',
+      valueToTest
+    )
+    return
+  }
+
+  if (!valueToTest) {
+    logger.warn('SSC ignore|expect_message is null or undefined, will not merge')
+    return
+  }
+
+  if ('object' !== typeof valueToTest) {
+    logger.warn(
+      'Saw SSC (ignore|expect)_message that is primitive/scaler, will not merge: %s',
+      valueToTest
+    )
+    return
+  }
+
+  if (!valueToTest) {
+    logger.warn('SSC ignore|expect_message is null or undefined, will not merge')
+    return
+  }
+
+  let valid = true
+  Object.keys(valueToTest).forEach(function validateArray(key) {
+    let arrayToTest = valueToTest[key]
+    if (!Array.isArray(arrayToTest)) {
+      logger.warn(
+        'Saw SSC message array that is not an array, will not merge: %s',
+        arrayToTest
+      )
+      valid = false
+    }
+  })
+  if (!valid) {
+    return
+  }
+
+  return this._updateNestedIfChanged(remote, local, remoteKey, localKey)
+}
 /**
  * Some parameter values are nested, need a simple way to change them as well.
  * Will merge local and remote if and only if both are arrays.
@@ -572,18 +765,13 @@ function _updateNestedIfChanged(remote, local, remoteKey, localKey) {
 Config.prototype._updateNestedIfChangedRaw = _updateNestedIfChangedRaw
 
 function _updateNestedIfChangedRaw(remote, local, remoteKey, localKey) {
-  var value = remote[remoteKey]
-  if (value != null && local[localKey] !== value) {
-    if (Array.isArray(value) && Array.isArray(local[localKey])) {
-      value.forEach(function pushIfNew(element) {
-        if (local[localKey].indexOf(element) === -1) local[localKey].push(element)
-      })
-    } else {
-      local[localKey] = value
-    }
-    this.emit(remoteKey, value)
-    logger.debug('Configuration of %s was changed to %s by New Relic.', remoteKey, value)
-  }
+  return this.mergeServerConfig.updateNestedIfChanged(
+    remote,
+    local,
+    remoteKey,
+    localKey,
+    logger
+  )
 }
 
 /**
@@ -645,7 +833,7 @@ Config.prototype.logUnsupported = function logUnsupported(json, key) {
 Config.prototype.logUnknown = function logUnknown(json, key) {
   var value = json[key]
   logger.debug(
-    "New Relic sent unknown configuration parameter %s with value %s.",
+    'New Relic sent unknown configuration parameter %s with value %s.',
     key,
     value
   )
@@ -813,8 +1001,8 @@ Config.prototype._fromPassed = function _fromPassed(external, internal, arbitrar
 
     if (typeof node === 'object' && !Array.isArray(node)) {
       // is top level and can have arbitrary keys
-      var isTop = internal === this && HAS_ARBITRARY_KEYS.indexOf(key) !== -1
-      this._fromPassed(node, internal[key], isTop)
+      var allowArbitrary = internal === this || HAS_ARBITRARY_KEYS.has(key)
+      this._fromPassed(node, internal[key], allowArbitrary)
     } else {
       internal[key] = node
     }
@@ -994,6 +1182,11 @@ Config.prototype._canonicalize = function _canonicalize() {
     this.error_collector.ignore_status_codes = _parseCodes(statusCodes)
   }
 
+  const expectedCodes = this.error_collector && this.error_collector.expected_status_codes
+  if (statusCodes) {
+    this.error_collector.expected_status_codes = _parseCodes(expectedCodes)
+  }
+
   var logAliases = {
     verbose: 'trace',
     debugging: 'debug',
@@ -1327,7 +1520,7 @@ function initialize(config) {
 
   config = new Config(userConf)
   config.config_file_path = filepath
-  logger.debug("Using configuration file %s.", filepath)
+  logger.debug('Using configuration file %s.', filepath)
 
   config.validateFlags()
 

package/lib/config/merge-server-config.js

@@ -0,0 +1,56 @@
+'use strict'
+class MergeServerConfig {
+  constructor(config) {
+    this.config = config
+  }
+
+  updateNestedIfChanged(remote, local, remoteKey, localKey, logger) {
+    var value = remote[remoteKey]
+
+    // if the value hasn't changed, skip this work
+    if (value === null || local[localKey] === value) {
+      return
+    }
+
+    // we need different update/merge logic if the server
+    // value is an array, a simple object, or anything else
+    if (Array.isArray(value) && Array.isArray(local[localKey])) {
+      this.updateArray(value, local, localKey)
+    } else if (this.isSimpleObject(value) && this.isSimpleObject(local[localKey])) {
+      this.updateObject(value, local, localKey)
+    } else {
+      local[localKey] = value
+    }
+    this.config.emit(remoteKey, value)
+    logger.debug('Configuration of %s was changed to %s by New Relic.', remoteKey, value)
+  }
+
+  updateArray(value, local, localKey) {
+    value.forEach( (element) => {
+      if (local[localKey].indexOf(element) === -1) local[localKey].push(element)
+    })
+  }
+
+  updateObject(value, local, localKey) {
+    // go through each key of the object and update it
+    Object.keys(value).forEach( (element) => {
+      if (Array.isArray(local[localKey][element]) && Array.isArray(value[element])) {
+        // if both key-values are arrays, push the remote value onto the local array
+        value[element].forEach( (elementValue) => {
+          if (-1 === local[localKey][element].indexOf(elementValue)) {
+            local[localKey][element].push(elementValue)
+          }
+        })
+      } else {
+        // otherwise, replace the local value with the server value
+        local[localKey][element] = value[element]
+      }
+    })
+  }
+
+  isSimpleObject(thing) {
+    return 'object' === (typeof thing) && this !== null
+  }
+}
+
+module.exports = MergeServerConfig

package/lib/errors/aggregator.js

@@ -3,9 +3,8 @@
 const errorsModule = require('./index')
 const EventAggregator = require('../event-aggregator')
 const logger = require('../logger').child({component: 'error_tracer'})
-const NAMES = require('../metrics/names')
 const urltils = require('../util/urltils')
-
+const errorHelper = require('../errors/helper')
 const createError = errorsModule.createError
 const createEvent = errorsModule.createEvent
 
@@ -37,6 +36,11 @@ class ErrorAggregator extends EventAggregator {
     this.errorCount = 0
     this.webTransactionErrorCount = 0
     this.otherTransactionErrorCount = 0
+
+    this.expectedErrorCount = 0
+    this.expectedWebTransactionErrorCount = 0
+    this.expectedOtherTransactionErrorCount = 0
+
     this.errors = []
     this.seenObjectsByTransaction = Object.create(null)
     this.seenStringsByTransaction = Object.create(null)
@@ -47,17 +51,21 @@ class ErrorAggregator extends EventAggregator {
    * possible.
    *
    * @param {Transaction} transaction
-   * @param {Metrics} metrics
+   *
+   * @return {number} The number of unexpected errors
    */
-  onTransactionFinished(transaction, metrics) {
+  onTransactionFinished(transaction) {
     if (!transaction) throw new Error('Error collector got a blank transaction.')
-    if (!metrics) throw new Error('Error collector requires metrics to count errors.')
     if (transaction.ignore) {
       return
     }
 
     // collect user errors even if status code is ignored
     let collectedErrors = 0
+    let expectedErrors = 0
+
+    // errors from noticeError are currently exempt from
+    // ignore and exclude rules
     if (transaction.userErrors.length) {
       for (let i = 0; i < transaction.userErrors.length; i++) {
         const exception = transaction.userErrors[i]
@@ -68,28 +76,38 @@ class ErrorAggregator extends EventAggregator {
     }
 
     const isErroredTransaction = urltils.isError(this.config, transaction.statusCode)
-    const isIgnoredErrorStatusCode = urltils.isIgnoredError(
+    const isExpectedErrorStatusCode = urltils.isExpectedError(
       this.config,
       transaction.statusCode
     )
 
     // collect other exceptions only if status code is not ignored
-    if (transaction.exceptions.length && !isIgnoredErrorStatusCode) {
+    if (transaction.exceptions.length) {
       for (let i = 0; i < transaction.exceptions.length; i++) {
         const exception = transaction.exceptions[i]
-        if (this._collect(transaction, exception[0], exception[1], exception[2])) {
+        if (this.collect(transaction, exception[0], exception[1], exception[2])) {
           ++collectedErrors
+          // if we could collect it, then check if expected
+          if (isExpectedErrorStatusCode ||
+            errorHelper.isExpectedException(
+              transaction,
+              exception[0],
+              this.config,
+              urltils
+            )
+          ) {
+            ++expectedErrors
+          }
         }
       }
-    } else if (isErroredTransaction && this._collect(transaction)) {
+    } else if (isErroredTransaction && this.collect(transaction)) {
       ++collectedErrors
+      if (isExpectedErrorStatusCode) {
+        ++expectedErrors
+      }
     }
 
-    // the metric should be incremented only if the error was actually collected
-    if (collectedErrors > 0) {
-      metrics.getOrCreateMetric(NAMES.ERRORS.PREFIX + transaction.getFullName())
-        .incrementCallCount(collectedErrors)
-    }
+    return collectedErrors - expectedErrors
   }
 
   /**
@@ -115,7 +133,7 @@ class ErrorAggregator extends EventAggregator {
     if (transaction) {
       transaction.addException(exception, customAttributes, timestamp)
     } else {
-      this._collect(transaction, exception, customAttributes, timestamp)
+      this.collect(transaction, exception, customAttributes, timestamp)
     }
   }
 
@@ -192,6 +210,34 @@ class ErrorAggregator extends EventAggregator {
     return false
   }
 
+  /**
+   * Wrapper for _collect, include logic for whether an error should
+   * be ignored or not.  Exists to allow userErrors to bypass ignore
+   * logic.
+   *
+   * NOTE: this interface is unofficial and may change in future.
+   *
+   * @param {?Transaction} transaction
+   *  Transaction associated with the error.
+   *
+   * @param {?Error} exception
+   *  The error to be traced.
+   *
+   * @param {?object} customAttributes
+   *  Any custom attributes associated with the request.
+   *
+   * @param {number} timestamp
+   *
+   * @return {bool} True if the error was collected.
+   */
+  collect(transaction, exception, customAttributes, timestamp) {
+    if (errorHelper.shouldIgnoreError(transaction, exception, this.config)) {
+      logger.trace("Ignoring error")
+      return
+    }
+    return this._collect(transaction, exception, customAttributes, timestamp)
+  }
+
   /**
    * Collects the error and also creates the error event.
    *
@@ -235,16 +281,6 @@ class ErrorAggregator extends EventAggregator {
       return
     }
 
-    this.errorCount++
-
-    if (transaction) {
-      if (transaction.isWeb()) {
-        this.webTransactionErrorCount++
-      } else {
-        this.otherTransactionErrorCount++
-      }
-    }
-
     // allow enabling & disabling the error tracer at runtime
     // TODO: it would be better to check config in the public add() to prevents collecting
     // errors on the transaction unnecessarily
@@ -260,8 +296,29 @@ class ErrorAggregator extends EventAggregator {
       logger.trace(exception, 'Got exception to trace:')
     }
 
+    this.errorCount++
+
+    if (transaction) {
+      if (transaction.isWeb()) {
+        this.webTransactionErrorCount++
+      } else {
+        this.otherTransactionErrorCount++
+      }
+    }
+
     var error = createError(transaction, exception, customAttributes, this.config)
 
+    if (true === error[4].intrinsics['error.expected'] ) {
+      this.expectedErrorCount++
+      if (transaction) {
+        if (transaction.isWeb()) {
+          this.expectedWebTransactionErrorCount++
+        } else {
+          this.expectedOtherTransactionErrorCount++
+        }
+      }
+    }
+
     if (this.errors.length < MAX_ERRORS) {
       logger.debug(error, 'Error to be sent to collector.')
       this.errors.push(error)
@@ -288,24 +345,32 @@ class ErrorAggregator extends EventAggregator {
   }
 
   /**
-   * Returns total number of collected errors.
+   * Returns total number of errors that matched `expected_*` configuration.
+   * Expected errors do not impact standard error counts.
+   */
+  getTotalExpectedErrorCount() {
+    return this.expectedErrorCount
+  }
+
+  /**
+   * Returns total number of unexpected errors.
    */
-  getTotalErrorCount() {
-    return this.errorCount
+  getTotalUnexpectedErrorCount() {
+    return this.errorCount - this.expectedErrorCount
   }
 
   /**
-   * Returns total number of errors collected during web transactions.
+   * Returns total number of unexpected errors collected during web transactions.
    */
-  getWebTransactionsErrorCount() {
-    return this.webTransactionErrorCount
+  getUnexpectedWebTransactionsErrorCount() {
+    return this.webTransactionErrorCount - this.expectedWebTransactionErrorCount
   }
 
   /**
-   * Returns total number of errors collected during background transactions.
+   * Returns total number of unexpected errors collected during background transactions.
    */
-  getOtherTransactionsErrorCount() {
-    return this.otherTransactionErrorCount
+  getUnexpectedOtherTransactionsErrorCount() {
+    return this.otherTransactionErrorCount - this.expectedOtherTransactionErrorCount
   }
 
   /**