newrelic 4.4.0 → 4.7.0

package/NEWS.md

@@ -1,3 +1,83 @@
+### 4.7.0 (2018-07-31):
+
+* Added support for distributed tracing.
+
+  Distributed tracing lets you see the path that a request takes as it travels
+  through your distributed system. By showing the distributed activity through a
+  unified view, you can troubleshoot and understand a complex system better than
+  ever before.
+
+  Distributed tracing is available with an APM Pro or equivalent subscription.
+  To see a complete distributed trace, you need to enable the feature on a set of
+  neighboring services. Enabling distributed tracing changes the behavior of some
+  New Relic features, so carefully consult the [transition guide](https://docs.newrelic.com/docs/transition-guide-distributed-tracing) before
+  you enable this feature.
+
+  To enable distributed tracing, set `distributed_tracing.enabled` to `true` in
+  your `newrelic.js` file, or set `NEW_RELIC_DISTRIBUTED_TRACING_ENABLED` in your
+  environment.
+
+* Added a warning for too-new versions of Node.js during agent startup.
+
+* Appropriately obfuscated SQL statements will now be included in all transaction
+  traces.
+
+  Previously, the agent would only include the SQL statements if the corresponding
+  query was sufficiently slow.
+
+* Added ability to execute instrumentation functions in the context of the segment
+  the segment descriptor is describing.
+
+  All `record*` methods supplied by all instrumentation shim classes now allow for
+  a function to be executed under the context of the segment the record call will
+  produce. This may be done by supplying a function in the `inContext` key for the
+  segment descriptor passed to the record method.
+
+* Reservoirs will now respect setting their size to 0.
+
+### 4.6.0 (2018-07-24):
+
+* Added full support for Node v10.
+
+* Added instrumentation for `crypto.scrypt`.
+
+* Added instrumentation for `fs.realpath.native`.
+
+* Added instrumentation for `process.setUncaughtExceptionCaptureCallback`.
+
+* Updated tests to use `asyncResource.runInAsyncScope` instead of `emitBefore` and
+  `emitAfter`
+
+* Pulled `distributed_tracing` config value from behind `feature_flag`.
+
+### 4.5.1 (2018-07-18):
+
+- The agent will now properly remerge event data on collection failure.
+
+  Previously, the agent wouldn't observe the correct format for remerging, causing
+  undefined events to be pushed into the reservoir.
+
+### 4.5.0 (2018-07-16):
+
+* Feature flags may now be set from environment variables.
+
+  Using the naming convention `NEW_RELIC_FEATURE_FLAG_<feature flag name in upper
+  case>`.
+
+* Transaction events may be harvested in two payloads now.
+
+  This change reduces the occurrence of harvests being rejected due to large
+  payloads. Payloads will only be split when they are large (greater than 1/3 the
+  maximum).
+
+* Updated Hapi v17 instrumentation to wrap `server` export, in addition to `Server`.
+
+* `ROOT` segment no longer turns into a span event.
+
+* Fixed span collection when transactions are `sampled=false`.
+
+* Removed `grandparentId` from spans.
+
 ### 4.4.0 (2018-07-12):
 
 * Added config `utilization` env vars to the `BOOLEAN_VARS` set.
@@ -9,6 +89,29 @@
 
 * Added node v10 to the test matrix.
 
+* Converted distributed trace `x-newrelic-trace` header name to `newrelic`.
+
+* Added support for different transport types in distributed tracing.
+
+* Added more tests around priority/sampled attributes on traces and events.
+
+* Lazily calculate transaction priority only when needed.
+
+* Transaction priority is now truncated to 6 decimal places on generation.
+
+* Adaptive sampling now uses the `sampling_target` and
+  `sampling_target_period_in_seconds` configuration values.
+
+  With these configurations, the adaptive sampling window is separated from the
+  harvest window.
+
+* Removed `nr.tripId` attribute from distributed trace intrinsics.
+
+* Default span events to enabled.
+
+  These are still protected behind `feature_flag.distributed_tracing` which defaults
+  to `false`.
+
 ### 4.3.0 (2018-07-09):
 
 * Added `nonce` option for `newrelic.getBrowserTimingHeader()`
@@ -30,7 +133,7 @@
 
 * Fixed issue with tracking external requests to default ports.
 
- Special thanks to Ryan King for pinpointing the cause of this issue.
+  Special thanks to Ryan King for pinpointing the cause of this issue.
 
 * Added extra check for handling arrays of functions when wrapping middleware
   mounters.
@@ -68,6 +171,25 @@
 
 * No longer download gcc on test suites that do not require it.
 
+* Added `url` parameter to `http` external segments.
+
+* Renamed request parameters on external segments.
+
+  Previously these were named just the parameter name (e.g. `/foo?bar=baz` would
+  become the parameter `"bar": "baz"`). Now they are prefixed with
+  `request.parameter`. (e.g. `"request.parameter.bar": "baz"`).
+
+* Added `EventAggregator` base class.
+
+  The `ErrorAggregator` class was refactored and most generic event aggregation
+  logic was moved to the new `EventAggregator` class.
+
+* Added `SpanEvent` and `SpanAggregator` classes.
+
+* Added Span event generation to the trace `end` method.
+
+* Added Span events to harvest cycle steps.
+
 ### 4.1.5 (2018-06-11):
 
 * Make `require()` statements explicitly reference `package.json` as a `.json` file.
@@ -110,6 +232,11 @@
   result in a false `uninstrumented` status, because the agent would interpret
   `redis.js` as the module itself.
 
+* Moved `computeSampled` call to `Transaction` constructor.
+
+  Previously it was only called in `createDistributedTracePayload`, but this
+  gives all transactions a `sampled` value, and potentially a boosted priority.
+
 ### 4.1.4 (2018-06-04):
 
 * Transaction stubs are now created properly in `api#getTransaction`
@@ -133,6 +260,25 @@
 
 * Modularlized configuration constants to improve readability.
 
+* Added `distributed_tracing` feature flag.
+
+* Added `acceptDistributedTracePayload` method to `Transaction`.
+
+* Added `createDistributedTracePayload` method to `Transaction`.
+
+* Updated `Agent#recordSupportability` to not include `Nodejs/` in the default metric name.
+
+* Added distributed tracing methods to `TransactionHandle`.
+
+* Added distributed tracing cases for `http` and `other` metric recorders.
+
+* Implemented `_addDistributedTraceInstrinsics` on `Transaction`.
+
+  If the `distributed_tracing` feature flag is enabled, the agent will ignore old
+  CAT attributes in favor of distributed trace–related ones.
+
+* Added integration tests around better CAT functionality.
+
 ### 4.1.2 (2018-05-22):
 
 * Fixed access to properties on promisified methods.

package/api.js

@@ -702,6 +702,13 @@ function createTracer(name, callback) {
   return arity.fixArity(callback, tracer.bindFunction(callback, segment, true))
 }
 
+/**
+ * @callback startSegmentCallback
+ * @param {function} cb
+ *   The function to time with the created segment.
+ * @return {Promise=} Returns a promise if cb returns a promise.
+ */
+
 /**
  * Wraps the given handler in a segment which may optionally be turned into a
  * metric.
@@ -720,7 +727,7 @@ function createTracer(name, callback) {
  *  up on the transaction breakdown table and server breakdown graph. Segments
  *  just show up in transaction traces.
  *
- * @param {function(cb) -> ?Promise} handler
+ * @param {startSegmentCallback} handler
  *  The function to track as a segment.
  *
  * @param {function} [callback]

package/index.js

@@ -1,21 +1,22 @@
 'use strict'
 
 // Record opening times before loading any other files.
-var preAgentTime = process.uptime()
-var agentStart = Date.now()
+const preAgentTime = process.uptime()
+const agentStart = Date.now()
 
 // Load unwrapped core now to ensure it gets the freshest properties.
 require('./lib/util/unwrapped-core')
 
-var featureFlags = require('./lib/feature_flags').prerelease
-var logger = require('./lib/logger')
-var psemver = require('./lib/util/process-version')
+const featureFlags = require('./lib/feature_flags').prerelease
+const psemver = require('./lib/util/process-version')
+let logger = require('./lib/logger') // Gets re-loaded after initialization.
 
 
-var agentVersion = require('./package.json').version
+const pkgJSON = require('./package.json')
 logger.info(
-  "Using New Relic for Node.js. Agent version: %s; Node version: %s.",
-  agentVersion, process.version
+  'Using New Relic for Node.js. Agent version: %s; Node version: %s.',
+  pkgJSON.version,
+  process.version
 )
 
 if (require.cache.__NR_cache) {
@@ -49,6 +50,13 @@ function initialize() {
 
       logger.error(message)
       throw new Error(message)
+    } else if (!psemver.satisfies(pkgJSON.engines.node)) {
+      logger.warn(
+        'New Relic for Node.js %s has not been tested on Node.js %s. Please ' +
+        'update the agent or downgrade your version of Node.js',
+        pkgJSON.version,
+        process.version
+      )
     }
 
     logger.debug("Current working directory at module load is %s.", process.cwd())

package/lib/agent.js

@@ -516,7 +516,7 @@ function _addIntrinsicAttrsFromTransaction(transaction) {
 
   // If CAT is enabled, extract the data gleened from the CAT headers.
   if (this.config.cross_application_tracer.enabled) {
-    if (this.config.feature_flag.distributed_tracing) {
+    if (this.config.distributed_tracing.enabled) {
       transaction.addDistributedTraceIntrinsics(intrinsicAttributes)
       if (transaction.parentSpanId) {
         intrinsicAttributes.parentSpanId = transaction.parentSpanId
@@ -677,6 +677,12 @@ Agent.prototype._listenForConfigChanges = function _listenForConfigChanges() {
       self.transactionSampler.samplingPeriod = period * 1000
     }
   )
+  this.config.on(
+    'transaction_events.max_samples_per_minute',
+    function updateEventSampleLimit(maxSamples) {
+      self.events.setLimit(maxSamples)
+    }
+  )
 }
 
 module.exports = Agent

package/lib/config/default.js

@@ -791,6 +791,23 @@ exports.config = {
     enabled: true
   },
 
+  /**
+   * Controls the method of cross agent tracing in the agent.
+   * Distributed tracing lets you see the path that a request takes through your
+   * distributed system. Enabling distributed tracing changes the behavior of some
+   * New Relic features, so carefully consult the transition guide before you enable
+   * this feature: https://docs.newrelic.com/docs/transition-guide-distributed-tracing
+   * Default is false.
+   */
+  distributed_tracing: {
+    /**
+     * Enables/disables distributed tracing.
+     *
+     * @env NEW_RELIC_DISTRIBUTED_TRACING_ENABLED
+     */
+    enabled: false
+  },
+
   /**
    * Controls the use of cross-application tracing.
    *

package/lib/config/env.js

@@ -114,6 +114,9 @@ const ENV_MAPPING = {
   span_events: {
     enabled: 'NEW_RELIC_SPAN_EVENTS_ENABLED'
   },
+  distributed_tracing: {
+    enabled: 'NEW_RELIC_DISTRIBUTED_TRACING_ENABLED'
+  },
   message_tracer: {
     segment_parameters: {
       enabled: 'NEW_RELIC_MESSAGE_TRACER_SEGMENT_PARAMETERS_ENABLED'
@@ -167,6 +170,7 @@ const BOOLEAN_VARS = new Set([
   'NEW_RELIC_HIGH_SECURITY',
   'NEW_RELIC_SLOW_SQL_ENABLED',
   'NEW_RELIC_SPAN_EVENTS_ENABLED',
+  'NEW_RELIC_DISTRIBUTED_TRACING_ENABLED',
   'NEW_RELIC_LOG_ENABLED',
   'NEW_RELIC_AUDIT_LOG_ENABLED',
   'NEW_RELIC_DATASTORE_DATABASE_NAME_REPORTING_ENABLED',

package/lib/config/index.js

@@ -21,11 +21,11 @@ var logger = null // Lazy-loaded in `initialize`.
 /**
  * CONSTANTS -- we gotta lotta 'em
  */
-var DEFAULT_CONFIG_PATH = require.resolve('./default')
-var DEFAULT_CONFIG = require('./default').config
-var DEFAULT_FILENAME = 'newrelic.js'
-var AZURE_APP_NAME = 'APP_POOL_ID'
-var CONFIG_FILE_LOCATIONS = [
+const DEFAULT_CONFIG_PATH = require.resolve('./default')
+const DEFAULT_CONFIG = require('./default').config
+const DEFAULT_FILENAME = 'newrelic.js'
+const AZURE_APP_NAME = 'APP_POOL_ID'
+const CONFIG_FILE_LOCATIONS = [
   process.env.NEW_RELIC_HOME,
   process.cwd(),
   process.env.HOME,
@@ -159,6 +159,7 @@ function Config(config) {
   this._fromSpecial()
 
   // 4. override config with environment variables
+  this._featureFlagsFromEnv()
   this._fromEnvironment()
 
   // 5. clean up anything that requires postprocessing
@@ -847,6 +848,21 @@ Config.prototype._fromSpecial = function _fromSpecial() {
   }
 }
 
+/**
+ * Iterate over all feature flags and check for the corresponding environment variable
+ * (of the form NEW_RELIC_FEATURE_FLAG_<feature flag name in upper case>).
+ */
+Config.prototype._featureFlagsFromEnv = function _featureFlagsFromEnv() {
+  const flags = Object.keys(feature_flag.prerelease).concat(feature_flag.released)
+  const config = this
+  flags.forEach(function checkFlag(flag) {
+    const envVal = process.env['NEW_RELIC_FEATURE_FLAG_' + flag.toUpperCase()]
+    if (envVal) {
+      config.feature_flag[flag] = isTruthular(envVal)
+    }
+  })
+}
+
 /**
  * Recursively visit the nodes of the constant containing the mapping between
  * environment variable names, overriding any configuration values that are

package/lib/db/tracer.js

@@ -46,27 +46,36 @@ QueryTracer.prototype.merge = function merge(tracer) {
 
 QueryTracer.prototype.addQuery = function addQuery(segment, type, query, trace) {
   const ttConfig = this.config.transaction_tracer
-  if (segment.getDurationInMillis() < ttConfig.explain_threshold) {
-    return
-  }
-  const slowQuery = new SlowQuery(segment, type, query, trace)
 
+  // If DT is enabled and the segment is part of a sampled transaction
+  // (i.e. we are creating a span event for this segment), then we need
+  // to collect the sql trace.
+  var slowQuery
   switch (ttConfig.record_sql) {
     case 'raw':
+      slowQuery = new SlowQuery(segment, type, query, trace)
       logger.trace('recording raw sql')
       segment.parameters.sql = slowQuery.query
       break
     case 'obfuscated':
+      slowQuery = new SlowQuery(segment, type, query, trace)
       logger.trace('recording obfuscated sql')
       segment.parameters.sql_obfuscated = slowQuery.obfuscated
       break
     default:
       logger.trace(
-        'not collecting slow-query because transaction_tracer.record_sql was set to %s',
+        'not recording sql statement, transaction_tracer.record_sql was set to %s',
         ttConfig.record_sql
       )
       return
   }
+
+  if (segment.getDurationInMillis() < ttConfig.explain_threshold) {
+    return
+  }
+
+  slowQuery = slowQuery || new SlowQuery(segment, type, query, trace)
+
   segment.parameters.backtrace = slowQuery.trace
 
   if (!this.config.slow_sql.enabled) {
@@ -158,7 +167,7 @@ QuerySample.prototype.getParams = function getParams() {
     params.database_name = segmentParams.database_name
   }
 
-  if (this.tracer.config.feature_flag.distributed_tracing) {
+  if (this.tracer.config.distributed_tracing.enabled) {
     this.trace.segment.transaction.addDistributedTraceIntrinsics(params)
   }
 

package/lib/environment.js

@@ -305,7 +305,7 @@ function getGlobalPackages(cb) {
  * package appears at most once, with all the versions joined into a
  * comma-delimited list.
  *
- * @return {Array.<String>[]} Sorted list of [name, version] pairs.
+ * @return {Array.<string[]>} Sorted list of [name, version] pairs.
  */
 function flattenVersions(packages) {
   var info = Object.create(null)

package/lib/errors/index.js

@@ -170,7 +170,7 @@ function _getErrorEventIntrinsicAttrs(transaction, errorClass, message, timestam
       attributes["nr.syntheticsMonitorId"] = transaction.syntheticsData.monitorId
     }
 
-    if (transaction.agent.config.feature_flag.distributed_tracing) {
+    if (transaction.agent.config.distributed_tracing.enabled) {
       transaction.addDistributedTraceIntrinsics(attributes)
     } else {
       attributes['nr.referringTransactionGuid'] = transaction.referringTransactionGuid

package/lib/feature_flags.js

@@ -5,7 +5,6 @@ exports.prerelease = {
   custom_instrumentation: true,
   custom_metrics: true,
   express5: false,
-  distributed_tracing: false,
   await_support: true,
   synthetics: true,
   native_metrics: true,

package/lib/harvest.js

@@ -17,7 +17,7 @@ class Harvest {
     this._errorEventsQueue = agent.errors.getQueue()
     this._errors = agent.errors.getErrors()
     this._traces = [].concat(agent.traces.syntheticsTraces)
-    this._events = agent.events
+    this._events = this._splitPayload(agent.events)
     this._queries = agent.queries
     this._spanEventsQueue = agent.spans.getQueue()
 
@@ -74,7 +74,13 @@ class Harvest {
       }
     }
     if (this._events) {
-      this.agent.events.merge(this._events)
+      for (let i = 0; i < this._events.length; ++i) {
+        if (this._events[i]) {
+          this.agent.events.merge(
+            this._events[i].toMerge
+          )
+        }
+      }
     }
     if (this._customEvents) {
       this.agent.customEvents.merge(this._customEvents)
@@ -203,22 +209,24 @@ class Harvest {
   }
 
   _prepareEvents() {
+    const eventAggr = this.agent.events
+
     // Create the metrics so they are at least set to 0
     const discarded = this.agent.metrics.getOrCreateMetric(NAMES.EVENTS.DISCARDED)
     const seen = this.agent.metrics.getOrCreateMetric(NAMES.EVENTS.SEEN)
     const sent = this.agent.metrics.getOrCreateMetric(NAMES.EVENTS.SENT)
 
-    seen.incrementCallCount(this._events.seen)
-    sent.incrementCallCount(this._events.sent)
+    seen.incrementCallCount(eventAggr.seen)
+    sent.incrementCallCount(eventAggr.sent)
 
-    // if we had to limit events and sample them, emit a warning
-    const diff = this._events.overflow()
+    // If we had to limit events and sample them, emit a warning
+    const diff = eventAggr.overflow()
     discarded.incrementCallCount(diff)
     if (diff > 0) {
       logger.warn(
         'analytics event overflow, dropped %d events; try increasing your limit above %d',
         diff,
-        this._events.limit
+        eventAggr.limit
       )
     }
   }
@@ -351,24 +359,20 @@ class Harvest {
       return setImmediate(callback)
     }
 
-    const metrics = {
-      reservoir_size: this._events.limit,
-      events_seen: this._events.seen
-    }
-    const payload = [
-      this.agent.config.run_id,
-      metrics,
-      this._events.toArray()
-    ]
-
-    // Send data to collector
     const self = this
     const agent = this.agent
-    agent.collector.analyticsEvents(payload, function onAnalyticsEvents(err) {
-      if (self._shouldClearData(err, 'events')) {
+    a.eachOfSeries(this._events, function sendEachPayload(obj, i, cb) {
+      const payload = obj.payload
+      agent.collector.analyticsEvents(payload, function onAnalyticsEvents(err) {
+        if (self._shouldClearData(err, 'events')) {
+          self._events[i] = null
+        }
+        cb(err)
+      })
+    }, function afterAllSent(err) {
+      if (!err) {
         self._events = null
       }
-
       callback(err)
     })
   }
@@ -468,7 +472,7 @@ class Harvest {
 
   _sendSpanEvents(callback) {
     const config = this.agent.config
-    if (!config.span_events.enabled || !config.feature_flag.distributed_tracing) {
+    if (!config.span_events.enabled || !config.distributed_tracing.enabled) {
       logger.debug('Span events collection disabled.')
       this._spanEventsQueues = null
       return setImmediate(callback)
@@ -511,6 +515,51 @@ class Harvest {
     logger.warn('Failed to send %j; re-sampling.', name)
     return false
   }
+
+  _splitPayload(queue) {
+    // If we're less than 1/3 full, don't bother splitting the payload.
+    if (queue.length === 0) {
+      return []
+    }
+    if (queue.length < queue.limit / 3) {
+      return [{
+        toMerge: queue,
+        payload: [
+          this.agent.config.run_id,
+          {reservoir_size: queue.limit, events_seen: queue.seen},
+          queue.toArray()
+        ]
+      }]
+    }
+
+    // Our payload is large, so split it in half.
+    // TODO: update this to pull the priority off the event when DT is released
+    const events = queue.getRawEvents()
+    const size = Math.floor(queue.length / 2)
+    const limit = Math.floor(queue.limit / 2)
+    const seen = Math.floor(queue.seen / 2)
+    const firstHalf = events.splice(0, size)
+
+    return [{
+      toMerge: firstHalf,
+      payload: [
+        this.agent.config.run_id,
+        {reservoir_size: limit, events_seen: seen},
+        firstHalf.map(rawEventsToValues)
+      ]
+    }, {
+      toMerge: events,
+      payload: [
+        this.agent.config.run_id,
+        {reservoir_size: queue.limit - limit, events_seen: queue.seen - seen},
+        events.map(rawEventsToValues)
+      ]
+    }]
+
+    function rawEventsToValues(ev) {
+      return ev.value
+    }
+  }
 }
 
 module.exports = Harvest

package/lib/instrumentation/core/crypto.js

@@ -12,7 +12,8 @@ function initialize(agent, crypto) {
       'pbkdf2',
       'randomBytes',
       'pseudoRandomBytes',
-      'randomFill'
+      'randomFill',
+      'scrypt'
     ],
     wrapCryptoMethod
   )

package/lib/instrumentation/core/fs.js

@@ -1,7 +1,7 @@
 'use strict'
 
-var record = require('../../metrics/recorders/generic.js')
-var NAMES = require('../../metrics/names.js')
+var record = require('../../metrics/recorders/generic')
+var NAMES = require('../../metrics/names')
 var wrap = require('../../shimmer').wrapMethod
 
 module.exports = initialize
@@ -46,6 +46,7 @@ function initialize(agent, fs) {
   ]
 
   wrap(fs, 'fs', methods, segment)
+  wrap(fs.realpath, 'fs.realpath', 'native', wrapRealPathNative)
   wrap(fs, 'fs', uninstrumented, agent.tracer.wrapFunctionNoSegment.bind(agent.tracer))
   wrap(fs, 'fs', ['watch'], wrapWatch)
   wrap(fs, 'fs', ['watchFile'], wrapWatchFile)
@@ -54,6 +55,10 @@ function initialize(agent, fs) {
     return agent.tracer.wrapFunctionLast(NAMES.FS.PREFIX + method, record, fn)
   }
 
+  function wrapRealPathNative(fn) {
+    return segment(fn, 'realpath.native')
+  }
+
   function wrapWatch(fn) {
     return function wrappedWatch() {
       var args = agent.tracer.slice(arguments)

package/lib/instrumentation/core/globals.js

@@ -8,6 +8,8 @@ module.exports = initialize
 
 
 function initialize(agent) {
+  let exceptionCallbackRegistered = false
+
   // `_fatalException` is an undocumented feature of domains, introduced in
   // Node.js v0.8. We use `_fatalException` because wrapping it will not
   // potentially change the behavior of the server unlike listening for
@@ -16,7 +18,7 @@ function initialize(agent) {
     return function wrappedFatalException(error) {
       // Only record the error if we are not currently within an instrumented
       // domain.
-      if (!process.domain) {
+      if (!process.domain && !exceptionCallbackRegistered) {
         agent.errors.add(null, error)
         agent.tracer.segment = null
       }
@@ -45,6 +47,20 @@ function initialize(agent) {
     }
   }
 
+  shimmer.wrapMethod(
+    process,
+    'process',
+    'setUncaughtExceptionCaptureCallback',
+    wrapUncaughtExceptionCallback
+  )
+
+  function wrapUncaughtExceptionCallback(original) {
+    return function wrapped(fn) {
+      exceptionCallbackRegistered = fn !== null
+      return original.apply(this, arguments)
+    }
+  }
+
   // This will initialize the most optimal native-promise instrumentation that
   // we have available.
   asyncHooks(agent)