newrelic 6.4.0 → 6.6.0

package/lib/spans/span-streamer.js

@@ -0,0 +1,109 @@
+'use strict'
+
+const logger = require('../logger').child({component: 'span-streamer'})
+
+const NAMES = require('../metrics/names')
+
+const NO_STREAM_WARNING =
+  'Attempting to stream spans before connection created. ' +
+  'This warning will not appear again this agent run.'
+
+const BACK_PRESSURE_WARNING =
+  'Back pressure detected in SpanStreamer! Spans will be dropped until the current batch ' +
+  'has fully sent. Will not warn again for %s seconds.'
+const BACK_PRESSURE_WARNING_INTERVAL = 60 // in seconds
+
+const BACK_PRESSURE_STOP = 'Back pressure has ended, continuing to stream'
+
+class SpanStreamer {
+  constructor(license_key, connection, metrics) {
+    this.stream = null
+    this.license_key = license_key
+    this.connection = connection
+    this._metrics = metrics
+    this._writable = false
+
+    // 'connected' indicates a safely writeable stream.
+    // May still be mid-connect to gRPC server.
+    this.connection.on('connected', (stream) =>{
+      this.stream = stream
+      this._writable = true
+    })
+
+    this.connection.on('disconnected', () =>{
+      this.stream = null
+      this._writable = false
+    })
+  }
+
+  write(span) {
+    if (!this.stream) {
+      logger.warnOnce(NO_STREAM_WARNING)
+      return false
+    }
+
+    if (!this._writable) {
+      return false
+    }
+
+    const formattedSpan = span.toStreamingFormat()
+
+    try {
+      // false indicates the stream has reached the highWaterMark
+      // and future writes should be avoided until drained. written items,
+      // including the one that returned false, will still be buffered.
+      this._writable = this.stream.write(formattedSpan)
+
+      if (!this._writable) {
+        logger.infoOncePer(
+          'BACK_PRESSURE_START',
+          BACK_PRESSURE_WARNING_INTERVAL * 1000,
+          BACK_PRESSURE_WARNING,
+          BACK_PRESSURE_WARNING_INTERVAL
+        )
+
+        const waitDrainStart = Date.now()
+
+        this.stream.once('drain', () => {
+          const drainCompleted = Date.now()
+          const drainDurationMs = drainCompleted - waitDrainStart
+
+          // Metric can be used to see how frequently completing drains
+          // as well as average time to drain from when we first notice.
+          this._metrics.getOrCreateMetric(NAMES.INFINITE_TRACING.DRAIN_DURATION)
+            .recordValue(drainDurationMs / 1000)
+
+          logger.trace(BACK_PRESSURE_STOP)
+          this._writable = true
+        })
+      }
+
+      // span was added to internal node stream buffer to be sent while draining
+      // so we return true even when we should not write anymore afterwards
+      return true
+    } catch (err) {
+      logger.trace('Could not stream span.', err)
+      // TODO: something has gone horribly wrong.
+      // We may want to log and turn off this aggregator
+      // to prevent sending further spans. Maybe even "disable" their creation?
+      // or is there a situation where we can recover?
+
+      return false
+    }
+  }
+
+  connect(agent_run_id) {
+    this.connection.setConnectionDetails(
+      this.license_key,
+      agent_run_id
+    )
+
+    this.connection.connectSpans()
+  }
+
+  disconnect() {
+    this.connection.disconnect()
+  }
+}
+
+module.exports = SpanStreamer

package/lib/spans/streaming-span-attributes.js

@@ -0,0 +1,59 @@
+'use strict'
+
+const mapToStreamingType = require('./map-to-streaming-type')
+
+/**
+ * Specialized attribute collection class for use with infinite streaming.
+ * Currently designed to be sent over grpc via the v1.proto definition.
+ *
+ * @private
+ * @class
+ */
+class StreamingSpanAttributes {
+  constructor(attributes) {
+    if (attributes) {
+      this.addAttributes(attributes)
+    }
+  }
+
+  /**
+   * Add a key/value pair to the attribute collection.
+   * null/undefined values will be dropped.
+   *
+   * Does not apply filtering/truncation.
+   *
+   * @param {string} key Name of the attribute to be stored.
+   * @param {string|boolean|number} value Value of the attribute to be stored.
+   */
+  addAttribute(key, value) {
+    const streamingValue = mapToStreamingType(value)
+    if (streamingValue) {
+      this[key] = streamingValue
+      return true
+    }
+
+    return false
+  }
+
+  /**
+   * Adds all attributes in an object to the attribute collection.
+   * null/undefined values will be dropped.
+   *
+   * Does not apply filtering/truncation.
+   *
+   * @param {object} [attributes]
+   * @param {string} [attributes.key] Name of the attribute to be stored.
+   * @param {string|boolean|number} [attributes.value] Value of the attribute to be stored.
+   */
+  addAttributes(attributes) {
+    if (!attributes) {
+      return
+    }
+
+    for (let [key, value] of Object.entries(attributes)) {
+      this.addAttribute(key, value)
+    }
+  }
+}
+
+module.exports = StreamingSpanAttributes

package/lib/spans/streaming-span-event-aggregator.js

@@ -0,0 +1,98 @@
+'use strict'
+
+const Aggregator = require('../aggregators/base-aggregator')
+const StreamingSpanEvent = require('./streaming-span-event')
+const NAMES = require('../metrics/names')
+const logger = require('../logger').child({component: 'streaming-span-event-aggregator'})
+
+const SEND_WARNING =
+  'send() is not currently supported on streaming span event aggregator. ' +
+  'This warning will not appear again this agent run.'
+
+// TODO: this doesn't "aggregate". Perhaps we need a different terminology
+// for the base-class and then this implementation can avoid the misleading language.
+class StreamingSpanEventAggregator extends Aggregator {
+  constructor(opts, collector, metrics) {
+    opts = opts || {}
+    opts.periodMs = opts.periodMs ? opts.periodMs : 1000
+    opts.limit = opts.limit ? opts.limit : 10000
+    opts.method = opts.method || 'span_event_data'
+
+    super(opts, collector)
+
+    this.metricNames = opts.metricNames || NAMES.INFINITE_TRACING
+    this.stream = opts.span_streamer
+    this.metrics = metrics
+    this.started = false
+    this.isStream = true
+  }
+
+  start() {
+    if (this.started) {
+      return
+    }
+
+    logger.trace('StreamingSpanEventAggregator starting up')
+    this.stream.connect(this.runId)
+    this.started = true
+  }
+
+  stop() {
+    if (!this.started) {
+      return
+    }
+
+    logger.trace('StreamingSpanEventAggregator stopping')
+    this.stream.disconnect()
+    this.started = false
+  }
+
+  send() {
+    // Only log once started. This will get invoked on initial harvest
+    // prior to start which we'll just ignore.
+    if (this.started) {
+      logger.warnOnce(SEND_WARNING)
+    }
+
+    this.emit(`finished ${this.method} data send.`)
+
+    return
+  }
+
+  /**
+   * Not a payload based aggregator
+   *
+   * This is here to implement the implicit interface
+   */
+  _toPayloadSync() {
+    return
+  }
+
+  /**
+   * Attempts to add the given segment to the collection.
+   *
+   * @param {TraceSegment}  segment         - The segment to add.
+   * @param {string}        [parentId=null] - The GUID of the parent span.
+   *
+   * @return {bool} True if the segment was added, or false if it was discarded.
+   */
+  addSegment(segment, parentId, isRoot) {
+    if (!this.started) {
+      logger.trace('Aggregator has not yet started, dropping span (%s).', segment.name)
+      return
+    }
+
+    // SEEN/SENT are to understand where we've had to drop spans due back-pressure, errors,
+    // reconnects, etc. so moving after start check to avoid logging for a
+    // currently unsupported case.
+    this.metrics.getOrCreateMetric(this.metricNames.SEEN).incrementCallCount()
+
+    const span = StreamingSpanEvent.fromSegment(segment, parentId, isRoot)
+
+    if (this.stream.write(span)) {
+      this.metrics.getOrCreateMetric(this.metricNames.SENT).incrementCallCount()
+    }
+  }
+}
+
+module.exports = StreamingSpanEventAggregator

package/lib/spans/streaming-span-event.js

@@ -0,0 +1,261 @@
+'use strict'
+
+const StreamingSpanAttributes = require('./streaming-span-attributes')
+const {truncate} = require('../util/byte-limit')
+const Config = require('../config')
+
+const {DESTINATIONS} = require('../config/attribute-filter')
+const NAMES = require('../metrics/names')
+const HTTP_LIBRARY = 'http'
+const CLIENT_KIND = 'client'
+const CATEGORIES = {
+  HTTP: 'http',
+  DATASTORE: 'datastore',
+  GENERIC: 'generic'
+}
+
+/**
+ * Specialized span event class for use with infinite streaming.
+ * Currently designed to be sent over grpc via the v1.proto definition.
+ *
+ * @private
+ * @class
+ */
+class StreamingSpanEvent {
+  /**
+   * @param {*} traceId TraceId for the Span.
+   * @param {object} agentAttributes Initial set of agent attributes.
+   * Must be pre-filtered and truncated.
+   * @param {object} customAttributes Initial set of custom attributes.
+   * Must be pre-filtered and truncated.
+   */
+  constructor(traceId, agentAttributes, customAttributes) {
+    this._traceId = traceId
+
+    this._intrinsicAttributes = new StreamingSpanAttributes()
+    this._intrinsicAttributes.addAttribute('traceId', traceId)
+    this._intrinsicAttributes.addAttribute('type', 'Span')
+    this._intrinsicAttributes.addAttribute('category', CATEGORIES.GENERIC)
+
+    this._customAttributes = new StreamingSpanAttributes(customAttributes)
+    this._agentAttributes = new StreamingSpanAttributes(agentAttributes)
+  }
+
+  /**
+   * Add a key/value pair to the Span's instrinisics collection.
+   *
+   * @param {string} key Name of the attribute to be stored.
+   * @param {string|boolean|number} value Value of the attribute to be stored.
+   */
+  addIntrinsicAttribute(key, value) {
+    this._intrinsicAttributes.addAttribute(key, value)
+  }
+
+  /**
+   * Add a key/value pair to the Span's custom/user attributes collection.
+   * @param {string} key Name of the attribute to be stored.
+   * @param {string|boolean|number} value Value of the attribute to be stored.
+   * @param {boolean} [truncateExempt=false] Set to true if attribute should not be truncated.
+   */
+  addCustomAttribute(key, value, truncateExempt = false) {
+    const shouldKeep = this._checkFilter(key)
+    if (shouldKeep) {
+      const processedValue = truncateExempt ? value : _truncate(value)
+      this._customAttributes.addAttribute(key, processedValue)
+    }
+  }
+
+  /**
+   * Add a key/value pair to the Span's agent attributes collection.
+   * @param {string} key Name of the attribute to be stored.
+   * @param {string|boolean|number} value Value of the attribute to be stored.
+   * @param {boolean} [truncateExempt=false] Set to true if attribute should not be truncated.
+   */
+  addAgentAttribute(key, value, truncateExempt = false) {
+    const shouldKeep = this._checkFilter(key)
+    if (shouldKeep) {
+      const processedValue = truncateExempt ? value : _truncate(value)
+      this._agentAttributes.addAttribute(key, processedValue)
+    }
+  }
+
+  _checkFilter(key) {
+    const {attributeFilter} = Config.getInstance()
+    const dest = attributeFilter.filterSegment(DESTINATIONS.SPAN_EVENT, key)
+    return dest & DESTINATIONS.SPAN_EVENT
+  }
+
+  toStreamingFormat() {
+    // Attributes are pre-formatted.
+    const formatted = {
+      trace_id: this._traceId,
+      intrinsics: this._intrinsicAttributes,
+      user_attributes: this._customAttributes,
+      agent_attributes: this._agentAttributes
+    }
+    return formatted
+  }
+
+  static fromSegment(segment, parentId = null, isRoot = false) {
+    const agentAttributes = segment.attributes.get(DESTINATIONS.SPAN_EVENT)
+    const customAttributes = segment.customAttributes.get(DESTINATIONS.SPAN_EVENT)
+
+    const transaction = segment.transaction
+    const traceId = transaction.traceId
+
+    let span = null
+    if (StreamingHttpSpanEvent.isHttpSegment(segment)) {
+      span = new StreamingHttpSpanEvent(traceId, agentAttributes, customAttributes)
+    }  else if (StreamingDatastoreSpanEvent.isDatastoreSegment(segment)) {
+      span = new StreamingDatastoreSpanEvent(traceId, agentAttributes, customAttributes)
+    } else {
+      span = new StreamingSpanEvent(traceId, agentAttributes, customAttributes)
+    }
+
+    span.addIntrinsicAttribute('guid', segment.id)
+    span.addIntrinsicAttribute('parentId', parentId)
+    span.addIntrinsicAttribute('transactionId', transaction.id)
+    span.addIntrinsicAttribute('sampled', transaction.sampled)
+    span.addIntrinsicAttribute('priority', transaction.priority)
+    span.addIntrinsicAttribute('name', segment.name)
+
+    if (isRoot) {
+      span.addIntrinsicAttribute('trustedParentId', transaction.traceContext.trustedParentId)
+      if (transaction.traceContext.tracingVendors) {
+        span.addIntrinsicAttribute('tracingVendors', transaction.traceContext.tracingVendors)
+      }
+    }
+
+    // Only set this if it will be `true`. Must be `null` otherwise.
+    if (transaction.baseSegment === segment) {
+      span.addIntrinsicAttribute('nr.entryPoint', true)
+    }
+
+    // Timestamp in milliseconds, duration in seconds. Yay consistency!
+    span.addIntrinsicAttribute('timestamp', segment.timer.start)
+    span.addIntrinsicAttribute('duration', segment.timer.getDurationInMillis() / 1000)
+
+    return span
+  }
+}
+
+/**
+ * Specialized span event class for external requests for use with infinite streaming.
+ * Currently designed to be sent over grpc via the v1.proto definition.
+ *
+ * @private
+ * @class
+ */
+class StreamingHttpSpanEvent extends StreamingSpanEvent {
+  /**
+   * @param {*} traceId TraceId for the Span.
+   * @param {object} agentAttributes Initial set of agent attributes.
+   * Must be pre-filtered and truncated.
+   * @param {object} customAttributes Initial set of custom attributes.
+   * Must be pre-filtered and truncated.
+   */
+  constructor(traceId, agentAttributes, customAttributes) {
+    super(traceId, agentAttributes, customAttributes)
+
+    this.addIntrinsicAttribute('category', CATEGORIES.HTTP)
+    this.addIntrinsicAttribute('component', agentAttributes.library || HTTP_LIBRARY)
+    this.addIntrinsicAttribute('span.kind', CLIENT_KIND)
+
+    if (agentAttributes.library) {
+      agentAttributes.library = null
+    }
+
+    if (agentAttributes.url) {
+      this.addAgentAttribute('http.url', agentAttributes.url)
+      agentAttributes.url = null
+    }
+
+    if (agentAttributes.procedure) {
+      this.addAgentAttribute('http.method', agentAttributes.procedure)
+      agentAttributes.procedure = null
+    }
+  }
+
+  static isHttpSegment(segment) {
+    return segment.name.startsWith(NAMES.EXTERNAL.PREFIX)
+  }
+}
+
+/**
+ * Specialized span event class for datastore operations and queries for use with
+ * infinite streaming.
+ * Currently designed to be sent over grpc via the v1.proto definition.
+ *
+ * @private
+ * @class.
+ */
+class StreamingDatastoreSpanEvent extends StreamingSpanEvent {
+  /**
+   * @param {*} traceId TraceId for the Span.
+   * @param {object} agentAttributes Initial set of agent attributes.
+   * Must be pre-filtered and truncated.
+   * @param {object} customAttributes Initial set of custom attributes.
+   * Must be pre-filtered and truncated.
+   */
+  constructor(traceId, agentAttributes, customAttributes) {
+    super(traceId, agentAttributes, customAttributes)
+
+    this.addIntrinsicAttribute('category', CATEGORIES.DATASTORE)
+    this.addIntrinsicAttribute('span.kind', CLIENT_KIND)
+
+    if (agentAttributes.product) {
+      this.addIntrinsicAttribute('component', agentAttributes.product)
+      agentAttributes.product = null
+    }
+
+    if (agentAttributes.collection) {
+      this.addAgentAttribute('db.collection', agentAttributes.collection)
+      agentAttributes.collection = null
+    }
+
+    if (agentAttributes.sql || agentAttributes.sql_obfuscated) {
+      let sql = null
+      if (agentAttributes.sql_obfuscated) {
+        sql = _truncate(agentAttributes.sql_obfuscated)
+        agentAttributes.sql_obfuscated = null
+      } else if (agentAttributes.sql) {
+        sql = _truncate(agentAttributes.sql)
+        agentAttributes.sql = null
+      }
+
+      // Flag as exempt from normal attribute truncation
+      this.addAgentAttribute('db.statement', sql, true)
+    }
+
+    if (agentAttributes.database_name) {
+      this.addAgentAttribute('db.instance', agentAttributes.database_name)
+      agentAttributes.database_name = null
+    }
+
+    if (agentAttributes.host) {
+      this.addAgentAttribute('peer.hostname', agentAttributes.host)
+
+      if (agentAttributes.port_path_or_id) {
+        const address = `${agentAttributes.host}:${agentAttributes.port_path_or_id}`
+        this.addAgentAttribute('peer.address', address)
+        agentAttributes.port_path_or_id = null
+      }
+
+      agentAttributes.host = null
+    }
+  }
+
+  static isDatastoreSegment(segment) {
+    return segment.name.startsWith(NAMES.DB.PREFIX)
+  }
+}
+
+function _truncate(val) {
+  let truncated = truncate(val, 1997)
+  if (truncated !== val) {
+    truncated += '...'
+  }
+  return truncated
+}
+
+module.exports = StreamingSpanEvent

package/lib/transaction/index.js

@@ -678,19 +678,40 @@ Transaction.prototype.alternatePathHashes = function alternatePathHashes() {
   return altHashes.length === 0 ? null : altHashes.sort().join(',')
 }
 
+/**
+ * Add the error information to the current segment and add the segment ID as
+ * an attribute onto the exception.
+ *
+ * @param {Exception}   exception  The exception object to be collected.
+ */
+Transaction.prototype._linkExceptionToSegment = _linkExceptionToSegment
+
+function _linkExceptionToSegment(exception) {
+  const segment = this.agent.tracer.getSegment()
+  if (!segment) {
+    return
+  }
+
+  // Add error attributes to the span
+  const details = exception.getErrorDetails(this.agent.config)
+  segment.addAttribute('error.message', details.message)
+  segment.addAttribute('error.class', details.type)
+
+  // Add the span/segment ID to the exception as agent attributes
+  exception.agentAttributes.spanId = segment.id
+}
+
 /**
  * Associate an exception with the transaction.  When the transaction ends,
  * the exception will be collected along with the transaction details.
  *
- * @param {Error}   exception         The exception to be collected.
- * @param {object}  customAttributes  Any custom attributes associated with
- *                                    the request (optional).
- * @param {number}  timestamp         The timestamp for when the exception occurred.
+ * @param {Exception}   exception  The exception object to be collected.
  */
 Transaction.prototype.addException = _addException
 
-function _addException(exception, customAttributes, timestamp) {
-  this.exceptions.push([exception, customAttributes, timestamp])
+function _addException(exception) {
+  this._linkExceptionToSegment(exception)
+  this.exceptions.push(exception)
 }
 
 /**
@@ -698,20 +719,18 @@ function _addException(exception, customAttributes, timestamp) {
  * When the transaction ends, the exception will be collected along with the transaction
  * details.
  *
- * @param {Error}   exception         The exception to be collected.
- * @param {object}  customAttributes  Any custom attributes associated with
- *                                    the request (optional).
- * @param {number}  timestamp         The timestamp for when the exception occurred.
+ * @param {Exception}   exception  The exception object to be collected.
  */
 Transaction.prototype.addUserError = _addUserError
 
-function _addUserError(exception, customAttributes, timestamp) {
-  this.userErrors.push([exception, customAttributes, timestamp])
+function _addUserError(exception) {
+  this._linkExceptionToSegment(exception)
+  this.userErrors.push(exception)
 }
 
 /**
- * Returns true if an error happened during the transaction or if the transaction itself
- * is considered to be an error.
+ * Returns true if an error happened during the transaction or if the transaction itself is
+ * considered to be an error.
  */
 Transaction.prototype.hasErrors = function _hasErrors() {
   var isErroredTransaction = urltils.isError(this.agent.config, this.statusCode)
@@ -720,10 +739,7 @@ Transaction.prototype.hasErrors = function _hasErrors() {
   return (transactionHasExceptions || transactionHasuserErrors || isErroredTransaction)
 }
 
-/**
- * Returns true if all the errors/exceptions collected so far
- * are expected errors.
- */
+// Returns true if all the errors/exceptions collected so far are expected errors
 Transaction.prototype.hasOnlyExpectedErrors = function hasOnlyExpectedErrors() {
   if (0 === this.exceptions.length) {
     return false
@@ -735,13 +751,13 @@ Transaction.prototype.hasOnlyExpectedErrors = function hasOnlyExpectedErrors() {
     const isUnexpected = !(
       errorHelper.isExpectedException(
         this,
-        exception[0],
+        exception.error,
         this.agent.config,
         urltils
       ) ||
       errorHelper.shouldIgnoreError(
         this,
-        exception[0],
+        exception.error,
         this.agent.config
       )
     )

package/lib/transaction/trace/index.js

@@ -2,7 +2,7 @@
 
 var codec = require('../../util/codec')
 var Segment = require('./segment')
-var Attributes = require('../../attributes')
+var {Attributes, MAXIMUM_CUSTOM_ATTRIBUTES} = require('../../attributes')
 var logger = require('../../logger').child({component: 'trace'})
 
 
@@ -29,7 +29,7 @@ function Trace(transaction) {
   this.segmentsSeen = 0
   this.totalTimeCache = null
 
-  this.custom = new Attributes(ATTRIBUTE_SCOPE, 64)
+  this.custom = new Attributes(ATTRIBUTE_SCOPE, MAXIMUM_CUSTOM_ATTRIBUTES)
   this.attributes = new Attributes(ATTRIBUTE_SCOPE)
 
   // sending displayName if set by user
@@ -67,28 +67,31 @@ Trace.prototype.end = function end() {
  * Iterates over the trace tree and generates a span event for each segment.
  */
 Trace.prototype.generateSpanEvents = function generateSpanEvents() {
-  var config = this.transaction.agent.config
+  const config = this.transaction.agent.config
+
+  const infiniteTracingConfigured = Boolean(config.infinite_tracing.trace_observer.host)
 
   if (
-    this.transaction.sampled &&
+    (infiniteTracingConfigured || this.transaction.sampled) &&
     config.span_events.enabled &&
     config.distributed_tracing.enabled
   ) {
-    var toProcess = []
+    let toProcess = []
+
     // Root segment does not become a span, so we need to process it separately.
     const spanAggregator = this.transaction.agent.spanEventAggregator
+
     const children = this.root.getChildren()
     for (let i = 0; i < children.length; ++i) {
       toProcess.push(new DTTraceNode(children[i], this.transaction.parentSpanId, true))
     }
 
     while (toProcess.length) {
-      var segmentInfo = toProcess.pop()
-      var segment = segmentInfo.segment
+      let segmentInfo = toProcess.pop()
+      let segment = segmentInfo.segment
 
-      // Even though at some point we might want to stop adding events
-      // because all the priorities should be the same, we need to count
-      // the spans as seen.
+      // Even though at some point we might want to stop adding events because all the priorities
+      // should be the same, we need to count the spans as seen.
       spanAggregator.addSegment(segment, segmentInfo.parentId, segmentInfo.isRoot)
 
       const nodes = segment.getChildren()