newrelic 6.4.0 → 6.6.0

package/lib/errors/index.js

@@ -6,8 +6,18 @@ var props = require('../util/properties')
 var urltils = require('../util/urltils')
 const errorHelper = require('../errors/helper')
 
-module.exports.createError = createError
-module.exports.createEvent = createEvent
+class Exception {
+  constructor({error, timestamp, customAttributes, agentAttributes}) {
+    this.error = error
+    this.timestamp = timestamp || 0
+    this.customAttributes = customAttributes || {}
+    this.agentAttributes = agentAttributes || {}
+  }
+
+  getErrorDetails(config) {
+    return errorHelper.extractErrorInformation(null, this.error, config)
+  }
+}
 
 /**
  * Given either or both of a transaction and an exception, generate an error
@@ -17,21 +27,20 @@ module.exports.createEvent = createEvent
  * handler, which traps actual instances of Error, try to set sensible
  * defaults for everything.
  *
- * @param {Transaction} transaction      The agent transaction, presumably
- *                                       coming out of the instrumentation.
- * @param {Error}       exception        Something trapped by an error listener.
- * @param {object}      customAttributes Any custom attributes associated with
- *                                       the request (optional).
+ * @param {Transaction} transaction  The agent transaction, coming from the instrumentatation
+ * @param {Exception}   exception    An custom Exception object with the error and other information
+ * @param {object}      config       The configuration to use when creating the object
  */
-function createError(transaction, exception, customAttributes, config) {
+function createError(transaction, exception, config) {
+  const error = exception.error
   let {name, message, type} = errorHelper.extractErrorInformation(
     transaction,
-    exception,
+    error,
     config,
     urltils
   )
 
-  var params = {
+  let params = {
     userAttributes: Object.create(null),
     agentAttributes: Object.create(null),
     intrinsics: Object.create(null)
@@ -39,20 +48,26 @@ function createError(transaction, exception, customAttributes, config) {
 
   if (transaction) {
     // Copy all of the parameters off of the transaction.
-    params.agentAttributes = transaction.trace.attributes.get(DESTINATIONS.ERROR_EVENT)
     params.intrinsics = transaction.getIntrinsicAttributes()
+    let transactionAgentAttributes = transaction.trace.attributes.get(DESTINATIONS.ERROR_EVENT)
+    transactionAgentAttributes = transactionAgentAttributes || {}
+
+    // Merge the agent attributes specific to this error event with the transaction attributes
+    const agentAttributes = Object.assign(exception.agentAttributes, transactionAgentAttributes)
+    params.agentAttributes = agentAttributes
 
     // There should be no attributes to copy in HSM, but check status anyway
     if (!config.high_security) {
-      var custom = transaction.trace.custom.get(DESTINATIONS.ERROR_EVENT)
+      const custom = transaction.trace.custom.get(DESTINATIONS.ERROR_EVENT)
       urltils.overwriteParameters(custom, params.userAttributes)
     }
   }
 
+  const customAttributes = exception.customAttributes
   if (!config.high_security && config.api.custom_attributes_enabled && customAttributes) {
-    for (var key in customAttributes) {
+    for (let key in customAttributes) {
       if (props.hasOwn(customAttributes, key)) {
-        var dest = config.attributeFilter.filterTransaction(
+        const dest = config.attributeFilter.filterTransaction(
           DESTINATIONS.ERROR_EVENT,
           key
         )
@@ -63,12 +78,11 @@ function createError(transaction, exception, customAttributes, config) {
     }
   }
 
-
-  var stack = exception && exception.stack
+  const stack = exception.error && exception.error.stack
   if (stack) {
     params.stack_trace = ('' + stack).split(/[\n\r]/g)
     if (config.high_security || config.strip_exception_messages.enabled) {
-      params.stack_trace[0] = exception.name + ': <redacted>'
+      params.stack_trace[0] = exception.error.name + ': <redacted>'
     }
   }
 
@@ -77,7 +91,7 @@ function createError(transaction, exception, customAttributes, config) {
     params.intrinsics['error.expected'] = true
   }
 
-  var res = [0, name, message, type, params]
+  let res = [0, name, message, type, params]
   if (transaction) {
     res.transaction = transaction.id
   }
@@ -180,3 +194,7 @@ function _getErrorEventIntrinsicAttrs(
 
   return attributes
 }
+
+module.exports.createError = createError
+module.exports.createEvent = createEvent
+module.exports.Exception = Exception

package/lib/grpc/connection/states.js

@@ -0,0 +1,9 @@
+'use strict'
+module.exports = {
+  disconnected:0,
+  connecting:1,
+  connected:2,
+  0:'disconnected',
+  1:'connecting',
+  2:'connected'
+}

package/lib/grpc/connection.js

@@ -0,0 +1,344 @@
+'use strict'
+
+const protoLoader = require('@grpc/proto-loader')
+const grpc = require('../proxy/grpc')
+const logger = require('../logger').child({component: 'grpc_connection'})
+const EventEmitter = require('events')
+const NAMES = require('../metrics/names')
+const util = require('util')
+
+const connectionStates = require('./connection/states')
+
+const protoOptions = {keepCase: true,
+  longs: String,
+  enums: String,
+  defaults: true,
+  oneofs: true
+}
+
+const pathProtoDefinition = __dirname +
+  '../../../lib/grpc/endpoints/infinite-tracing/v1.proto'
+
+const defaultBackoffOpts = {
+  initialSeconds: 0,
+  seconds:15
+}
+
+/**
+ * Class for managing the GRPC connection
+ *
+ * Both @grpc/grpc-js and grpc will manage the http2 connections
+ * for us -- this class manages the _stream_ connection logic.
+ *
+ * Will emit events based on the connectionStates (see above
+ */
+class GrpcConnection extends EventEmitter {
+  /**
+   * GrpcConnection constructor
+   *
+   * Standard property setting/initilization, and sets an initial
+   * connection state of disconnected
+   *
+   * @param {Object} traceObserverConfig config item config.infinite_tracing.trace_observer
+   * @param {MetricAggregator} metrics metric aggregator, for supportability metrics
+   * @param {Object} backoffOpts allows injecting specific backoff times
+   */
+  constructor(traceObserverConfig, metrics, backoffOpts = defaultBackoffOpts) {
+    super()
+
+    // only set opts if the object is valid
+    if ( (backoffOpts.initialSeconds || backoffOpts.initialSeconds === 0) &&
+         (backoffOpts.seconds || backoffOpts.seconds === 0)) {
+      this._backoffOpts = backoffOpts
+    } else {
+      this._backoffOpts = defaultBackoffOpts
+      logger.trace(
+        'invalid backoff information passed to constructor, using default values'
+      )
+    }
+
+    // initial "stream connection" has a 0 second backoff, unless
+    // different values are injected via _backoffOpts
+    this._streamBackoffSeconds = this._backoffOpts.initialSeconds
+    this._metrics = metrics
+    this._setState(connectionStates.disconnected)
+
+    this._endpoint = this.getTraceObserverEndpoint(traceObserverConfig)
+    this._licensekey = null
+    this._runId = null
+
+    this.stream = null
+  }
+
+  /**
+   * Sets connection details
+   *
+   * Allows setting of connection details _after_ object constructions
+   * but before the actual connection.
+   *
+   * @param {string} endpoint the GRPC server's endpoint
+   * @param {string} license_key the agent license key
+   * @param {string} run_id the current agent run id (also called agent run token)
+   * @param {string} [rootCerts] string of root (ca) certificates to attach to the connection.
+   */
+  setConnectionDetails(license_key, run_id, rootCerts) {
+    this._licenseKey = license_key
+    this._runId = run_id
+    this._rootCerts = rootCerts
+
+    return this
+  }
+
+  getTraceObserverEndpoint(traceObserverConfig) {
+    return `${traceObserverConfig.host}:${traceObserverConfig.port}`
+  }
+
+  /**
+   * Sets the connection state
+   *
+   * Used to indicate a transition from one connection state to
+   * the next.  Also responsible for emitting the connect state event
+   *
+   * @param {int} state The connection state (See connectionStates above)
+   * @param {ClientDuplexStreamImpl} state The GRPC stream, when defined
+   */
+  _setState(state, stream = null) {
+    this._state = state
+    this.emit(connectionStates[state], stream)
+  }
+
+  /**
+   * Start the Connection
+   *
+   * Public Entry point -- initiates a connection
+   */
+  connectSpans() {
+    if (this._state !== connectionStates.disconnected) {
+      return
+    }
+
+    this._setState(connectionStates.connecting)
+    logger.trace('connecting to grpc endpoint in [%s] seconds', this._getBackoffSeconds())
+
+    setTimeout(() => {
+      try {
+        this.stream = this._connectSpans(this._endpoint, this._licenseKey, this._runId)
+
+        // May not actually be "connected" at this point but we can write to the stream
+        // immediately.
+        this._setState(connectionStates.connected, this.stream)
+
+        // any future stream connect/disconnect after initial connection
+        // should have a 15 second backoff
+        this._setStreamBackoffAfterInitialStreamSetup()
+      } catch (err) {
+        logger.trace(
+          err,
+          'Unexpected error establishing GRPC stream, will not attempt reconnect.'
+        )
+        this._disconnectWithoutReconnect()
+      }
+    }, this._getBackoffSeconds() * 1000)
+  }
+
+  _setStreamBackoffAfterInitialStreamSetup() {
+    this._streamBackoffSeconds = this._backoffOpts.seconds
+  }
+
+  _setStreamBackoffToInitialValue() {
+    this._streamBackoffSeconds = this._backoffOpts.initialSeconds
+  }
+
+  /**
+   * End the current stream and set state to disconnected.
+   *
+   * No more data can be sent until connected again.
+   */
+  disconnect() {
+    if (this._state === connectionStates.disconnected) {
+      return
+    }
+
+    this._disconnectWithoutReconnect()
+  }
+
+  /**
+   * Calculates backoff seconds
+   */
+  _getBackoffSeconds() {
+    return this._streamBackoffSeconds
+  }
+
+  /**
+   * Method returns GRPC metadata for initial connection
+   *
+   * @param {string} license_key
+   * @param {string} run_id
+   */
+  _getMetadata(license_key, run_id, env) {
+    const metadata = new grpc.Metadata()
+    metadata.add('license_key', license_key)
+    metadata.add('agent_run_token', run_id)
+
+    // check environment variables for testing parameters and
+    // pass to server via meta-data.
+    const flaky = parseInt(env.NEWRELIC_GRPCCONNECTION_METADATA_FLAKY, 10)
+    const delay = parseInt(env.NEWRELIC_GRPCCONNECTION_METADATA_DELAY, 10)
+    if (flaky) {
+      logger.trace('Adding flaky metadata: %s', flaky)
+      metadata.add('flaky', flaky)
+    }
+    if (delay) {
+      logger.trace('Adding delay metadata: %s', delay)
+      metadata.add('delay',delay)
+    }
+
+    return metadata
+  }
+
+  /**
+   * Sets internal object state to disconnected initiates a new connection
+   *
+   * Called when we receive a stream status that indicates a potential
+   * problem with the stream.  We set the state (which will emit an event),
+   * increment tries to honor the backoff, and then reconnect.
+   */
+  _reconnect() {
+    this._disconnect()
+    this.connectSpans()
+  }
+
+  _disconnect() {
+    if (this.stream) {
+      this.stream.removeAllListeners()
+
+      // Indicates to server we are done.
+      // Server officially closes the stream.
+      this.stream.end()
+      this.stream = null
+    }
+
+    this._setState(connectionStates.disconnected)
+  }
+
+  /**
+   * Disconnects Without Reconnect
+   *
+   * Certain GRPC statuses require us to disconnect and _not_
+   * attempt a stream reconnect.
+   */
+  _disconnectWithoutReconnect() {
+    this._disconnect()
+    this._setStreamBackoffToInitialValue()
+  }
+
+  /**
+   * Central location to setup stream observers
+   *
+   * Events from the GRPC stream (a ClientDuplexStreamImpl) are the main way
+   * we communicate with the GRPC server.
+   *
+   * @param {ClientDuplexStreamImpl} stream
+   */
+  _setupSpanStreamObservers(stream) {
+    // listen for responses from server and log
+    if (logger.traceEnabled()) {
+      stream.on('data', function data(response) {
+        logger.trace("grpc span response stream: %s", JSON.stringify(response))
+      })
+    }
+
+    // listen for status that indicate stream has ended,
+    // and we need to disconnect
+    stream.on('status', (grpcStatus) => {
+      logger.trace('GRPC Status Received [%s]: %s', grpcStatus.code, grpcStatus.details)
+      const grpcStatusName =
+        grpc.status[grpcStatus.code] ? grpc.status[grpcStatus.code] : 'UNKNOWN'
+
+      if (grpc.status[grpc.status.UNIMPLEMENTED] === grpcStatusName) {
+        this._metrics.getOrCreateMetric(
+          NAMES.INFINITE_TRACING.SPAN_RESPONSE_GRPC_UNIMPLEMENTED
+        ).incrementCallCount()
+
+        // per the spec, An UNIMPLEMENTED status code from gRPC indicates
+        // that the versioned Trace Observer is no longer available. Agents
+        // MUST NOT attempt to reconnect in this case
+        this._disconnectWithoutReconnect()
+      } else {
+        // all statuses are treated as "bad stuff" happened, and
+        // as a signal we need to reconnect (except UNIMPLEMENTED).
+        // Even an "OK status" indicates the call completed successfully,
+        // which indicates the stream is over.
+        if (grpc.status[grpc.status.OK] !== grpcStatusName) {
+          this._metrics.getOrCreateMetric(
+            util.format(NAMES.INFINITE_TRACING.SPAN_RESPONSE_GRPC_STATUS, grpcStatusName)
+          ).incrementCallCount()
+        }
+
+        this._reconnect()
+      }
+    })
+
+    // if we don't listen for the errors they'll bubble
+    // up and crash the application
+    stream.on('error', (err) => {
+      this._metrics.getOrCreateMetric(NAMES.INFINITE_TRACING.SPAN_RESPONSE_ERROR)
+        .incrementCallCount()
+
+      logger.trace('span stream error. Code: [%s]: %s',err.code, err.details)
+    })
+  }
+
+  /**
+   * Creates the GRPC credentials needed
+   */
+  _generateCredentials(grpcApi) {
+    let certBuffer = null
+
+    // Current settable value for testing. If allowed to be overriden via
+    // configuration, this should be removed in place of setting
+    // this._rootCerts from config via normal configuration precedence.
+    const envTestCerts = process.env.NEWRELIC_GRPCCONNECTION_CA
+    const rootCerts = this._rootCerts || envTestCerts
+    if (rootCerts) {
+      logger.debug('Infinite tracing root certificates found to attach to requests.')
+      try {
+        certBuffer = Buffer.from(rootCerts, 'utf-8')
+      } catch (err) {
+        logger.warn('Failed to create buffer from rootCerts, proceeding without.', err)
+      }
+    }
+
+    // null/undefined ca treated same as calling createSsl()
+    return grpcApi.credentials.createSsl(certBuffer)
+  }
+
+  /**
+   * Internal/private method for connection
+   *
+   * Contains the actual logic that connects to the GRPC service.
+   * "Connection" can be a somewhat misleading term here.  This method
+   * invokes the "recordSpan" remote proceduce call. Behind the scenes
+   * this makes an http2 request with the metadata, and then returns
+   * a stream for further writing.
+   */
+  _connectSpans(endpoint, license_key, run_id) {
+    const packageDefinition = protoLoader.loadSync(pathProtoDefinition, protoOptions)
+
+    const serviceDefinition = grpc.loadPackageDefinition(
+      packageDefinition
+    ).com.newrelic.trace.v1
+
+    const credentials = this._generateCredentials(grpc)
+    const client = new serviceDefinition.IngestService(
+      endpoint,
+      credentials
+    )
+    const metadata = this._getMetadata(license_key, run_id, process.env)
+    const stream = client.recordSpan(metadata)
+    this._setupSpanStreamObservers(stream)
+
+    return stream
+  }
+}
+module.exports = GrpcConnection

package/lib/grpc/endpoints/infinite-tracing/v1.proto

@@ -0,0 +1,29 @@
+syntax = "proto3";
+
+package com.newrelic.trace.v1;
+
+service IngestService {
+  // Accepts a stream of Span messages, and returns an irregular stream of
+  // RecordStatus messages.
+  rpc RecordSpan(stream Span) returns (stream RecordStatus) {}
+}
+
+message Span {
+  string trace_id = 1;
+  map<string, AttributeValue> intrinsics = 2;
+  map<string, AttributeValue> user_attributes = 3;
+  map<string, AttributeValue> agent_attributes = 4;
+}
+
+message AttributeValue {
+  oneof value {
+    string string_value = 1;
+    bool bool_value = 2;
+    int64 int_value = 3;
+    double double_value = 4;
+  }
+}
+
+message RecordStatus {
+  uint64 messages_seen = 1;
+}

package/lib/header-processing.js

@@ -0,0 +1,75 @@
+'use strict'
+
+const REQUEST_START_HEADER = 'x-request-start'
+const QUEUE_HEADER = 'x-queue-start'
+const CONTENT_LENGTH_REGEX = /^Content-Length$/i
+
+/**
+ * Extracts queue time from the incoming request headers.
+ *
+ * Queue time is provided by certain providers by stamping the request
+ * header with the time the request arrived at the router.
+ * @param {*} logger
+ * @param {*} requestHeaders
+ */
+function getQueueTime(logger, requestHeaders) {
+  const headerValue = requestHeaders[REQUEST_START_HEADER] || requestHeaders[QUEUE_HEADER]
+  if (!headerValue) {
+    return null
+  }
+
+  const split = headerValue.split('=')
+  const rawQueueTime = split.length > 1 ? split[1] : headerValue
+
+  const parsedQueueTime = parseFloat(rawQueueTime)
+  if (isNaN(parsedQueueTime)) {
+    logger.warn('Queue time header parsed as NaN. See trace level log for value.')
+
+    // This header can hold up to 4096 bytes which could quickly fill up logs.
+    // Do not log a level higher than debug.
+    logger.trace('Queue time: %s', rawQueueTime)
+
+    return null
+  }
+
+  const convertedQueueTime = convertUnit(parsedQueueTime)
+  return convertedQueueTime
+}
+
+function convertUnit(time) {
+  let convertedTime = time
+  if (convertedTime > 1e18) {
+    // nano seconds
+    convertedTime = convertedTime / 1e6
+  } else if (convertedTime > 1e15) {
+    // micro seconds
+    convertedTime = convertedTime / 1e3
+  } else if (convertedTime < 1e12) {
+    // seconds
+    convertedTime = convertedTime * 1e3
+  }
+
+  return convertedTime
+}
+
+/**
+ * Returns the value of the Content-Length header
+ *
+ * If no header is found, returns -1
+ *
+ * @param {*} headers
+ */
+function getContentLengthFromHeaders(headers) {
+  let contentLength = -1
+  for (const [headerName, headerValue] of Object.entries(headers)) {
+    if (CONTENT_LENGTH_REGEX.test(headerName)) {
+      return headerValue
+    }
+  }
+  return contentLength
+}
+
+module.exports = {
+  getQueueTime,
+  getContentLengthFromHeaders
+}

package/lib/instrumentation/core/child_process.js

@@ -33,8 +33,20 @@ function initialize(agent, childProcess, moduleName, shim) {
           const args = shim.argsToArray.apply(shim, arguments)
           const cbIndex = args.length - 1
 
+          const originalListener = args[cbIndex]
+          if (!shim.isFunction(originalListener)) {
+            return fn.apply(this, arguments)
+          }
+
           shim.bindSegment(args, cbIndex)
 
+          // Leverage events.removeListener() mechanism that checks listener
+          // property to allow our wrapped listeners to match and remove appropriately.
+          // Avoids having to instrument removeListener() and potentially doubling
+          // lookup. Since our wrapping will only be referenced by the events
+          // collection, we should not need to unwrap.
+          args[cbIndex].listener = originalListener
+
           return fn.apply(this, args)
         }
       }

package/lib/instrumentation/core/http-outbound.js

@@ -195,6 +195,10 @@ function handleError(segment, req, error) {
  * @param {http.IncomingMessage} res
  */
 function handleResponse(segment, hostname, req, res) {
+  // Add response attributes for spans
+  segment.addAttribute('http.statusCode', res.statusCode)
+  segment.addAttribute('http.statusText', res.statusMessage)
+  
   // If CAT is enabled, grab those headers!
   const agent = segment.transaction.agent
   if (