newrelic 5.3.0 → 5.6.2

package/CONTRIBUTING.md

@@ -37,8 +37,7 @@ and not ready for general-purpose use.
 
 ### Setup
 
-To run the tests you need a GNU-compatible make, the openssl command-line
-binary, and some services:
+To run the tests you need an openssl command-line binary, and some services:
 
 * Cassandra
 * Memcached
@@ -50,7 +49,7 @@ binary, and some services:
 If you have these all running locally on the standard ports, then you are good
 to go. However, the suggested path is to use [Docker](http://www.docker.com).
 If you use OS X or Windows, use Docker Machine, which can be installed as a part of
-[Docker Toolbox](https://www.docker.com/docker-toolbox).  Then, run `make services`
+[Docker Toolbox](https://www.docker.com/docker-toolbox).  Then, run `npm run services`
 to start docker containers for each of the above services.
 
 If you have these services available on non-standard ports or elsewhere on your
@@ -75,7 +74,7 @@ tests if all of the unit tests pass.
 If you don't feel like dealing with the hassle of setting up the servers, just
 the unit tests can be run with:
 
-    make unit
+    npm run unit
 
 ### Writing tests
 

package/NEWS.md

@@ -1,3 +1,94 @@
+### 5.6.2 (2019-03-25):
+
+* Agent now respects attribute type restrictions on trace/segment attributes, as
+  well as error event/trace attributes.
+
+* Fixes potential for `RangeError: Maximum call stack size exceeded` error on
+  Transaction/Trace end.
+
+* Custom events no longer accept attributes with invalid types.
+
+  The only attribute types accepted by the backend are `boolean`, `string`, and
+  `number`; any attribute assigned to a custom event outside these types would be
+  dropped on ingest. The agent now filters these attributes out, and logs out a
+  helpful message detailing the issue.
+
+### 5.6.1 (2019-03-11):
+
+* Updated log message for not adding attributes and change the log level to debug.
+
+* Fixed an issue where exclusive time would be improperly calculated in some cases.
+
+### 5.6.0 (2019-03-04):
+
+* Added `product` attribute to existing datastore instrumentations.
+
+* Added `db.collection` to datastore span event attributes.
+
+* `trusted_account_key`, `account_id`, and `primary_application_id` may now be
+  configured via a configuration file while in serverless mode.
+
+* Fixed a bug where data belonging to distributed traces starting in the Node.js
+  agent would be prioritized over data produced from traces starting in other
+  language agents.
+
+  Previously, the agent would use the same random number for both the transaction
+  priority (used for data sampling) and the Distributed Tracing trace sampling
+  decision (whether to create DT data for a given transaction). This random number
+  reuse resulted in a bias that caused data from distributed traces started in the
+  Node.js agent to be prioritized above data that belongs to distributed traces
+  started in other language agents. The agent now makes individual rolls for each
+  of these quantities (i.e. the transaction priority and trace sampling decision),
+  eliminating the bias.
+
+* Optimized exclusive time duration calculator.
+
+  Previously, the agent would spend a lot of time sorting redundant arrays while
+  calculating the exclusive time for the segments of a trace. This has been
+  refactored into a single postorder traversal over the tree which will calculate
+  the exclusive time for all segments in the subtree rooted at a given segment.
+
+* Prevent a split on undefined location under certain conditions in Memcached.
+
+ Special thanks to Ben Wolfe (@bwolfe) for this fix!
+
+### 5.4.0 (2019-02-19):
+
+* Fixed issue where `shim.createSegment()` could result in modifying the parent
+  when opaque.
+
+* Fixed issue where `http-outbound` would modify parent segments when parent is
+  opaque.
+
+* Moved processing of exclusive time attribute out of `toJSON` and into `finalize`
+  to only be calculated once.
+
+  Previously, serializing a segment would result in calculating and caching exclusive
+  time which could result in issues if serialized prior to ending.
+
+* Added `SNS` to message shim library names.
+
+* Added check for `collect_span_events` in config sent from the server on connect.
+
+  Collection of span events can be disabled from the server configuration, but not
+  enabled.
+
+* Refactored `Segment#toJSON` to be more readable.
+
+* Added a `try/catch` to config initialization to safely handle invalid setting
+  combinations.
+
+  When an error is caught the agent is marked as disabled, which ultimately returns
+  a stub API and keeps the process running.
+
+* String truncation is now done using a binary search over the byte length of the
+  string.
+
+  Previously this truncation was done using a linear search for the proper byte
+  length.
+
+* Optimized segment and span attribute filtering.
+
 ### 5.3.0 (2019-02-12):
 
 * Added `span_events` and `transaction_segments` attribute destinations.

package/api.js

@@ -9,6 +9,7 @@ const hashes = require('./lib/util/hashes')
 const properties = require('./lib/util/properties')
 const stringify = require('json-stringify-safe')
 const shimmer = require('./lib/shimmer')
+const isValidType = require('./lib/util/attribute-types')
 const TransactionShim = require('./lib/shim/transaction-shim')
 const TransactionHandle = require('./lib/transaction/handle')
 const AwsLambda = require('./lib/serverless/aws-lambda')
@@ -16,7 +17,6 @@ const AwsLambda = require('./lib/serverless/aws-lambda')
 const ATTR_DEST = require('./lib/config/attribute-filter').DESTINATIONS
 const MODULE_TYPE = require('./lib/shim/constants').MODULE_TYPE
 const NAMES = require('./lib/metrics/names')
-
 /*
  *
  * CONSTANTS
@@ -365,7 +365,13 @@ API.prototype.noticeError = function noticeError(error, customAttributes) {
   }
   const transaction = this.agent.tracer.getTransaction()
 
-  this.agent.errors.addUserError(transaction, error, customAttributes)
+  // Filter all object type valued attributes out
+  let filteredAttributes = customAttributes
+  if (customAttributes) {
+    filteredAttributes = _filterAttributes(customAttributes, 'noticeError')
+  }
+
+  this.agent.errors.addUserError(transaction, error, filteredAttributes)
 }
 
 /**
@@ -1067,6 +1073,9 @@ API.prototype.recordCustomEvent = function recordCustomEvent(eventType, attribut
     return
   }
 
+  // Filter all object type valued attributes out
+  const filteredAttributes = _filterAttributes(attributes, `${eventType} custom event`)
+
   var instrinics = {
     type: eventType,
     timestamp: Date.now()
@@ -1074,7 +1083,7 @@ API.prototype.recordCustomEvent = function recordCustomEvent(eventType, attribut
 
   var tx = this.agent.getTransaction()
   var priority = tx && tx.priority || Math.random()
-  this.agent.customEvents.add([instrinics, attributes], priority)
+  this.agent.customEvents.add([instrinics, filteredAttributes], priority)
 }
 
 /**
@@ -1390,4 +1399,19 @@ API.prototype.setLambdaHandler = function setLambdaHandler(handler) {
   return this.awsLambda.patchLambdaHandler(handler)
 }
 
+function _filterAttributes(attributes, name) {
+  const filteredAttributes = Object.create(null)
+  Object.keys(attributes).forEach((attributeKey) => {
+    if (!isValidType(attributes[attributeKey])) {
+      logger.info(
+        `Omitting attribute ${attributeKey} from ${name} call, type must ` +
+        'be boolean, number, or string'
+      )
+      return
+    }
+    filteredAttributes[attributeKey] = attributes[attributeKey]
+  })
+  return filteredAttributes
+}
+
 module.exports = API

package/index.js

@@ -34,8 +34,8 @@ if (require.cache.__NR_cache) {
 
 function initialize() {
   logger.debug('Loading agent from %s', __dirname)
-  var agent = null
-  var message = null
+  let agent = null
+  let message = null
 
   try {
     logger.debug(
@@ -45,8 +45,8 @@ function initialize() {
 
     // TODO: Update this check when Node v6 is deprecated.
     if (psemver.satisfies('<6.0.0')) {
-      message = "New Relic for Node.js requires a version of Node equal to or\n" +
-                "greater than 6.0.0. Not starting!"
+      message = 'New Relic for Node.js requires a version of Node equal to or\n' +
+                'greater than 6.0.0. Not starting!'
 
       logger.error(message)
       throw new Error(message)
@@ -59,24 +59,24 @@ function initialize() {
       )
     }
 
-    logger.debug("Current working directory at module load is %s.", process.cwd())
-    logger.debug("Process title is %s.", process.title)
-    logger.debug("Application was invoked as %s.", process.argv.join(' '))
+    logger.debug('Current working directory at module load is %s.', process.cwd())
+    logger.debug('Process title is %s.', process.title)
+    logger.debug('Application was invoked as %s.', process.argv.join(' '))
 
-    var config = require('./lib/config').getOrCreateInstance()
+    const config = require('./lib/config').getOrCreateInstance()
 
     // Get the initialized logger as we likely have a bootstrap logger which
     // just pipes to stdout.
     logger = require('./lib/logger')
 
     if (!config || !config.agent_enabled) {
-      logger.info("Module not enabled in configuration; not starting.")
+      logger.info('Module not enabled in configuration; not starting.')
     } else {
       agent = createAgent(config)
       addStartupSupportabilities(agent)
     }
   } catch (error) {
-    message = "New Relic for Node.js was unable to bootstrap itself due to an error:"
+    message = 'New Relic for Node.js was unable to bootstrap itself due to an error:'
     logger.error(error, message)
 
     /* eslint-disable no-console */
@@ -85,7 +85,7 @@ function initialize() {
     /* eslint-enable no-console */
   }
 
-  var API = null
+  let API = null
   if (agent) {
     API = require('./api')
   } else {
@@ -97,7 +97,7 @@ function initialize() {
   // If we loaded an agent, record a startup time for the agent.
   // NOTE: Metrics are recorded in seconds, so divide the value by 1000.
   if (agent) {
-    var initDuration = (Date.now() - agentStart) / 1000
+    const initDuration = (Date.now() - agentStart) / 1000
     agent.recordSupportability('Nodejs/Application/Opening/Duration', preAgentTime)
     agent.recordSupportability('Nodejs/Application/Initialization/Duration', initDuration)
     agent.once('started', function timeAgentStart() {
@@ -117,9 +117,9 @@ function createAgent(config) {
    * multiple times, with undefined results. New Relic's instrumentation
    * can't be enabled or disabled without an application restart.
    */
-  var Agent = require('./lib/agent')
-  var agent = new Agent(config)
-  var appNames = agent.config.applications()
+  const Agent = require('./lib/agent')
+  const agent = new Agent(config)
+  const appNames = agent.config.applications()
 
   if (config.logging.diagnostics) {
     logger.warn(
@@ -128,7 +128,7 @@ function createAgent(config) {
   }
 
   if (appNames.length < 1) {
-    var message =
+    const message =
       'New Relic requires that you name this application!\n' +
       'Set app_name in your newrelic.js file or set environment variable\n' +
       'NEW_RELIC_APP_NAME. Not starting!'
@@ -136,17 +136,17 @@ function createAgent(config) {
     throw new Error(message)
   }
 
-  var shimmer = require('./lib/shimmer')
+  const shimmer = require('./lib/shimmer')
   shimmer.patchModule(agent)
   shimmer.bootstrapInstrumentation(agent)
 
   // Check for already loaded modules and warn about them.
-  var uninstrumented = require('./lib/uninstrumented')
+  const uninstrumented = require('./lib/uninstrumented')
   uninstrumented.check(shimmer.registeredInstrumentations)
 
   agent.start(function afterStart(error) {
     if (error) {
-      var errorMessage = 'New Relic for Node.js halted startup due to an error:'
+      const errorMessage = 'New Relic for Node.js halted startup due to an error:'
       logger.error(error, errorMessage)
 
       /* eslint-disable no-console */
@@ -165,15 +165,15 @@ function createAgent(config) {
 
 function addStartupSupportabilities(agent) {
   // TODO: As new versions come out, make sure to update Angler metrics.
-  var nodeMajor = /^v?(\d+)/.exec(process.version)
+  const nodeMajor = /^v?(\d+)/.exec(process.version)
   agent.recordSupportability(
     'Nodejs/Version/' + ((nodeMajor && nodeMajor[1]) || 'unknown')
   )
 
-  var configFlags = Object.keys(agent.config.feature_flag)
-  for (var i = 0; i < configFlags.length; ++i) {
-    var flag = configFlags[i]
-    var enabled = agent.config.feature_flag[flag]
+  const configFlags = Object.keys(agent.config.feature_flag)
+  for (let i = 0; i < configFlags.length; ++i) {
+    const flag = configFlags[i]
+    const enabled = agent.config.feature_flag[flag]
 
     if (enabled !== featureFlags[flag]) {
       agent.recordSupportability(

package/lib/adaptive-sampler.js

@@ -8,7 +8,7 @@ class AdaptiveSampler {
     this._samplingPeriod = 0
     this._samplingTarget = opts.target
     this._maxSamples = 2 * opts.target
-    this._minSampledPriority = 0
+    this._samplingThreshold = 0
     this._resetCount = 0
     this._resetInterval = null
 
@@ -24,8 +24,8 @@ class AdaptiveSampler {
     return this._sampled
   }
 
-  get minimumPriority() {
-    return this._minSampledPriority
+  get samplingThreshold() {
+    return this._samplingThreshold
   }
 
   get samplingTarget() {
@@ -73,14 +73,13 @@ class AdaptiveSampler {
    * Determines if an object should be sampled based on the object's priority and
    * the number of objects sampled in this window.
    *
-   * @param {object} obj          - The object to check for sampleability.
-   * @param {number} obj.priority - The priority of this object.
+   * @param {number} roll - The number to compare against the threshold
    *
    * @return {bool} True if the object should be sampled.
    */
-  shouldSample(obj) {
+  shouldSample(roll) {
     ++this._seen
-    if (obj.priority >= this._minSampledPriority) {
+    if (roll >= this._samplingThreshold) {
       this._incrementSampled()
       return true
     }
@@ -100,7 +99,7 @@ class AdaptiveSampler {
   }
 
   /**
-   * Increments the sampled counter and adjusted the minimum priority to maintain
+   * Increments the sampled counter and adjusted the sampling threshold to maintain
    * a steady sample rate.
    */
   _incrementSampled() {
@@ -126,7 +125,7 @@ class AdaptiveSampler {
   _adjustStats(target) {
     if (this._seen) {
       const ratio = Math.min(target / this._seen, 1)
-      this._minSampledPriority = 1 - ratio
+      this._samplingThreshold = 1 - ratio
     }
   }
 }

package/lib/attributes.js

@@ -1,6 +1,8 @@
 'use strict'
 
+const Config = require('./config')
 const logger = require('./logger').child({component: 'attributes'})
+const isValidType = require('./util/attribute-types')
 const byteUtils = require('./util/byte-limit')
 const properties = require('./util/properties')
 
@@ -9,9 +11,17 @@ const properties = require('./util/properties')
  * @private
  */
 class Attributes {
-  constructor(opts = Object.create(null)) {
-    this.filter = opts.filter
-    this.limit = opts.limit || Infinity
+  /**
+   * @param {string} scope
+   *  The scope of the attributes this will collect. Must be `transaction` or
+   *  `segment`.
+   *
+   * @param {number} [limit=Infinity]
+   *  The maximum number of attributes to retrieve for each destination.
+   */
+  constructor(scope, limit = Infinity) {
+    this.filter = makeFilter(scope)
+    this.limit = limit
     this.attributes = Object.create(null)
   }
 
@@ -21,7 +31,7 @@ class Attributes {
    * @param {string} str - Object key name or value
    */
   isValidLength(str) {
-    return byteUtils.isValidLength(str, 256)
+    return typeof str === 'number' || byteUtils.isValidLength(str, 255)
   }
 
   /**
@@ -87,15 +97,20 @@ class Attributes {
    * Adds given key-value pair to destination's agent attributes,
    * if it passes filtering rules.
    *
-   * @param {string}        scope         - Attribute scope ('transaction' or 'segment')
    * @param {DESTINATIONS}  destinations  - The default destinations for this key.
    * @param {string}        key           - The attribute name.
    * @param {string}        value         - The attribute value.
-   * @param {boolean}  [truncateExempt=false] - Flag marking value exempt from truncation
+   * @param {boolean} [truncateExempt=false] - Flag marking value exempt from truncation
    */
-  addAttribute(scope, destinations, key, value, truncateExempt = false) {
+  addAttribute(destinations, key, value, truncateExempt = false) {
     if (!isValidType(value)) {
-      return logger.warn('Not adding attribute %s with %s value type', key, typeof value)
+      return logger.debug(
+        'Not adding attribute %s with %s value type. This is expected for undefined' +
+        'attributes and only an issue if an attribute is not expected to be undefined' +
+        'or not of the type expected.',
+        key,
+        typeof value
+      )
     }
 
     if (!this.isValidLength(key)) {
@@ -106,38 +121,42 @@ class Attributes {
     }
 
     // Only set the attribute if at least one destination passed
-    destinations = this.filter.filter(scope, destinations, key)
-    if (destinations) {
-      this._set(destinations, key, value, truncateExempt)
+    const validDestinations = this.filter(destinations, key)
+    if (validDestinations) {
+      this._set(validDestinations, key, value, truncateExempt)
     }
   }
 
   /**
    * Passthrough method for adding multiple unknown attributes at once.
    *
-   * @param {string}        scope         - 'transaction' or 'segment'
    * @param {DESTINATIONS}  destinations  - The default destinations for these attributes.
    * @param {object}        attrs         - The attributes to add.
    */
-  addAttributes(scope, destinations, attrs) {
+  addAttributes(destinations, attrs) {
     for (let key in attrs) {
       if (properties.hasOwn(attrs, key)) {
-        this.addAttribute(scope, destinations, key, attrs[key])
+        this.addAttribute(destinations, key, attrs[key])
       }
     }
   }
 }
 
 /**
- * Checks incoming attribute value against valid types:
- * string, number, & boolean.
+ * Creates a filter function for the given scope.
  *
- * @param {*} val
+ * @param {string} scope - The scope of the filter to make.
  *
- * @return {boolean}
+ * @return {function} A function that performs attribute filtering for the given
+ *  scope.
  */
-function isValidType(val) {
-  return typeof val !== 'object' && val !== undefined
+function makeFilter(scope) {
+  const {attributeFilter} = Config.getInstance()
+  if (scope === 'transaction') {
+    return (d, k) => attributeFilter.filterTransaction(d, k)
+  } else if (scope === 'segment') {
+    return (d, k) => attributeFilter.filterSegment(d, k)
+  }
 }
 
 module.exports = Attributes

package/lib/collector/facts.js

@@ -88,8 +88,13 @@ function getAllIPAddresses() {
       const interfaceAddresses = interfaces[key].map(function getAddress(inter) {
         return inter.address
       })
-      Array.prototype.push.apply(addresses, interfaceAddresses)
+
+      for (let index = 0; index < interfaceAddresses.length; index++) {
+        const address = interfaceAddresses[index]
+        addresses.push(address)
+      }
     }
+
     return addresses
   }, [])
 }

package/lib/collector/remote-method.js

@@ -169,7 +169,7 @@ RemoteMethod.prototype._safeRequest = function _safeRequest(options) {
   const maxPayloadSize = this._config.max_payload_size_in_bytes
   var level = 'trace'
 
-  if (!isValidLength(options.body, maxPayloadSize + 1)) {
+  if (!isValidLength(options.body, maxPayloadSize)) {
     logger.warn(
       'The payload size %d being sent to method %s exceeded the maximum size of %d',
       Buffer.byteLength(options.body, 'utf8'),

package/lib/config/attribute-filter.js

@@ -40,11 +40,6 @@ const SEGMENT_SCOPE_DETAILS = [
 
 const DESTINATION_DETAILS = [...TRANS_SCOPE_DETAILS, ...SEGMENT_SCOPE_DETAILS]
 
-const SCOPES = {
-  segment: SEGMENT_SCOPE_DETAILS,
-  transaction: TRANS_SCOPE_DETAILS
-}
-
 module.exports = exports = AttributeFilter
 exports.DESTINATIONS = DESTINATIONS
 
@@ -84,16 +79,53 @@ function AttributeFilter(config) {
   this.update()
 }
 
+/**
+ * Tests a given key against the global and destination transaction filters.
+ *
+ * @param {DESTINATIONS}  destinations  - The locations the attribute wants to be put.
+ * @param {string}        key           - The name of the attribute to test.
+ *
+ * @return {DESTINATIONS} The destinations the attribute should be put.
+ */
+AttributeFilter.prototype.filterTransaction = filterTransaction
+function filterTransaction(destinations, key) {
+  return this._filter(TRANS_SCOPE_DETAILS, destinations, key)
+}
+
+/**
+ * Tests a given key against the global and destination segment filters.
+ *
+ * @param {DESTINATIONS}  destinations  - The locations the attribute wants to be put.
+ * @param {string}        key           - The name of the attribute to test.
+ *
+ * @return {DESTINATIONS} The destinations the attribute should be put.
+ */
+AttributeFilter.prototype.filterSegment = function filterSegment(destinations, key) {
+  return this._filter(SEGMENT_SCOPE_DETAILS, destinations, key)
+}
+
+/**
+ * Tests a given key against all global and destination filters.
+ *
+ * @param {DESTINATIONS}  destinations  - The locations the attribute wants to be put.
+ * @param {string}        key           - The name of the attribute to test.
+ *
+ * @return {DESTINATIONS} The destinations the attribute should be put.
+ */
+AttributeFilter.prototype.filterAll = function filterSegment(destinations, key) {
+  return this._filter(DESTINATION_DETAILS, destinations, key)
+}
+
 /**
  * Tests a given key against the global and destination filters.
  *
- * @param {array} scope               - 'transaction' or 'segment'.
- * @param {DESTINATIONS} destinations - The locations the attribute wants to be put.
- * @param {string} key                - The name of the attribute to test.
+ * @param {array}         scope         - The destination details for filtering.
+ * @param {DESTINATIONS}  destinations  - The locations the attribute wants to be put.
+ * @param {string}        key           - The name of the attribute to test.
  *
  * @return {DESTINATIONS} The destinations the attribute should be put.
  */
-AttributeFilter.prototype.filter = function test(scope, destinations, key) {
+AttributeFilter.prototype._filter = function _filter(scope, destinations, key) {
   // This method could be easily memoized since for a given destination and key
   // the result will always be the same until a configuration update happens. A
   // given application will also have a controllable set of destinations and
@@ -104,15 +136,13 @@ AttributeFilter.prototype.filter = function test(scope, destinations, key) {
     return DESTINATIONS.NONE
   }
 
-  const scopeDetails = SCOPES[scope] || DESTINATION_DETAILS
-
   // These are lazy computed to avoid calculating them for cached results.
   var globalInclude = null
   var globalExclude = null
 
   // Iterate over each desination and see if the rules apply.
-  for (var i = 0; i < scopeDetails.length; ++i) {
-    var dest = scopeDetails[i]
+  for (var i = 0; i < scope.length; ++i) {
+    var dest = scope[i]
     var destId = dest.id
     var destName = dest.name
     if (!(this._enabledDestinations & destId)) {

package/lib/config/default.js

@@ -88,6 +88,10 @@ exports.config = () => ({
    * @env NEW_RELIC_PROXY_PASS
    */
   proxy_pass: '',
+  // Serverless DT config defaults
+  trusted_account_key: null,
+  primary_application_id: null,
+  account_id: null,
   /**
    * Custom SSL certificates
    *
@@ -881,5 +885,5 @@ exports.config = () => ({
      * loop data.
      */
     native_metrics: {enabled: true}
-  }
+  },
 })

package/lib/config/env.js

@@ -137,7 +137,7 @@ const ENV_MAPPING = {
     enabled: 'NEW_RELIC_SERVERLESS_MODE_ENABLED'
   },
   trusted_account_key: 'NEW_RELIC_TRUSTED_ACCOUNT_KEY',
-  application_id: 'NEW_RELIC_APPLICATION_ID',
+  primary_application_id: 'NEW_RELIC_PRIMARY_APPLICATION_ID',
   account_id: 'NEW_RELIC_ACCOUNT_ID'
 }