dd-trace 3.3.1 → 3.12.1

package/packages/dd-trace/src/dogstatsd.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const lookup = require('dns').lookup // cache to avoid instrumentation
+const request = require('./exporters/common/request')
 const dgram = require('dgram')
 const isIP = require('net').isIP
 const log = require('./log')
@@ -11,6 +12,13 @@ class Client {
   constructor (options) {
     options = options || {}
 
+    if (options.metricsProxyUrl) {
+      this._httpOptions = {
+        url: options.metricsProxyUrl.toString(),
+        path: '/dogstatsd/v2/proxy'
+      }
+    }
+
     this._host = options.host || 'localhost'
     this._family = isIP(this._host)
     this._port = options.port || 8125
@@ -38,26 +46,50 @@ class Client {
 
     this._queue = []
 
+    if (this._httpOptions) {
+      this._sendHttp(queue)
+    } else {
+      this._sendUdp(queue)
+    }
+  }
+
+  _sendHttp (queue) {
+    const buffer = Buffer.concat(queue)
+    request(buffer, this._httpOptions, (err) => {
+      if (err) {
+        log.error('HTTP error from agent: ' + err.stack)
+        if (err.status) {
+          // Inside this if-block, we have connectivity to the agent, but
+          // we're not getting a 200 from the proxy endpoint. If it's a 404,
+          // then we know we'll never have the endpoint, so just clear out the
+          // options. Either way, we can give UDP a try.
+          if (err.status === 404) {
+            this._httpOptions = null
+          }
+          this._sendUdp(queue)
+        }
+      }
+    })
+  }
+
+  _sendUdp (queue) {
     if (this._family !== 0) {
-      this._sendAll(queue, this._host, this._family)
+      this._sendUdpFromQueue(queue, this._host, this._family)
     } else {
       lookup(this._host, (err, address, family) => {
         if (err) return log.error(err)
-        this._sendAll(queue, address, family)
+        this._sendUdpFromQueue(queue, address, family)
       })
     }
   }
 
-  _send (address, family, buffer) {
+  _sendUdpFromQueue (queue, address, family) {
     const socket = family === 6 ? this._udp6 : this._udp4
 
-    log.debug(`Sending to DogStatsD: ${buffer}`)
-
-    socket.send(buffer, 0, buffer.length, this._port, address)
-  }
-
-  _sendAll (queue, address, family) {
-    queue.forEach((buffer) => this._send(address, family, buffer))
+    queue.forEach((buffer) => {
+      log.debug(`Sending to DogStatsD: ${buffer}`)
+      socket.send(buffer, 0, buffer.length, this._port, address)
+    })
   }
 
   _add (stat, value, type, tags) {

package/packages/dd-trace/src/encode/agentless-ci-visibility.js

@@ -197,8 +197,16 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
   }
 
   _encode (bytes, trace) {
-    this._eventCount += trace.length
-    const events = trace.map(formatSpan)
+    const rawEvents = trace.map(formatSpan)
+
+    const testSessionEvents = rawEvents.filter(
+      event => event.type === 'test_session_end' || event.type === 'test_suite_end'
+    )
+
+    const isTestSessionTrace = !!testSessionEvents.length
+    const events = isTestSessionTrace ? testSessionEvents : rawEvents
+
+    this._eventCount += events.length
 
     for (const event of events) {
       this._encodeEvent(bytes, event)

package/packages/dd-trace/src/encode/coverage-ci-visibility.js

@@ -12,7 +12,6 @@ const MAXIMUM_NUM_COVERAGE_FILES = 100
 class CoverageCIVisibilityEncoder extends AgentEncoder {
   constructor () {
     super(...arguments)
-    this.codeCoverageBuffers = []
     this._coverageBytes = new Chunk()
     this.form = new FormData()
     this.fileIndex = 1

package/packages/dd-trace/src/exporter.js

@@ -3,6 +3,7 @@
 const AgentExporter = require('./exporters/agent')
 const LogExporter = require('./exporters/log')
 const AgentlessCiVisibilityExporter = require('./ci-visibility/exporters/agentless')
+const AgentProxyCiVisibilityExporter = require('./ci-visibility/exporters/agent-proxy')
 const exporters = require('../../../ext/exporters')
 const fs = require('fs')
 const constants = require('./constants')
@@ -18,6 +19,8 @@ module.exports = name => {
       return AgentExporter
     case exporters.DATADOG:
       return AgentlessCiVisibilityExporter
+    case exporters.AGENT_PROXY:
+      return AgentProxyCiVisibilityExporter
     default:
       return inAWSLambda && !usingLambdaExtension ? LogExporter : AgentExporter
   }

package/packages/dd-trace/src/exporters/agent/index.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const URL = require('url').URL
+const { URL, format } = require('url')
 const log = require('../../log')
 const Writer = require('./writer')
 
@@ -8,7 +8,11 @@ class AgentExporter {
   constructor (config, prioritySampler) {
     this._config = config
     const { url, hostname, port, lookup, protocolVersion, stats = {} } = config
-    this._url = url || new URL(`http://${hostname || 'localhost'}:${port}`)
+    this._url = url || new URL(format({
+      protocol: 'http:',
+      hostname: hostname || 'localhost',
+      port
+    }))
 
     const headers = {}
     if (stats.enabled) {
@@ -51,6 +55,10 @@ class AgentExporter {
       }, flushInterval).unref()
     }
   }
+
+  flush (done = () => {}) {
+    this._writer.flush(done)
+  }
 }
 
 module.exports = AgentExporter

package/packages/dd-trace/src/exporters/agent/writer.js

@@ -85,21 +85,14 @@ function makeRequest (version, data, count, url, headers, lookup, needsStartupLo
       'Datadog-Meta-Tracer-Version': tracerVersion,
       'X-Datadog-Trace-Count': String(count)
     },
-    lookup
+    lookup,
+    url
   }
 
   setHeader(options.headers, 'Datadog-Meta-Lang', 'nodejs')
   setHeader(options.headers, 'Datadog-Meta-Lang-Version', process.version)
   setHeader(options.headers, 'Datadog-Meta-Lang-Interpreter', process.jsEngine || 'v8')
 
-  if (url.protocol === 'unix:') {
-    options.socketPath = url.pathname
-  } else {
-    options.protocol = url.protocol
-    options.hostname = url.hostname
-    options.port = url.port
-  }
-
   log.debug(() => `Request to the agent: ${JSON.stringify(options)}`)
 
   request(data, options, (err, res, status) => {

package/packages/dd-trace/src/exporters/common/agent-info-exporter.js

@@ -0,0 +1,82 @@
+const { URL, format } = require('url')
+
+const request = require('./request')
+
+function fetchAgentInfo (url, callback) {
+  request('', {
+    path: '/info',
+    url
+  }, (err, res) => {
+    if (err) {
+      return callback(err)
+    }
+    try {
+      const response = JSON.parse(res)
+      return callback(null, response)
+    } catch (e) {
+      return callback(e)
+    }
+  })
+}
+
+/**
+ * Exporter that exposes a way to query /info endpoint from the agent and gives you the response.
+ * While this._writer is not initialized, exported traces are stored as is.
+ */
+class AgentInfoExporter {
+  constructor (tracerConfig) {
+    this._config = tracerConfig
+    const { url, hostname, port } = this._config
+    this._url = url || new URL(format({
+      protocol: 'http:',
+      hostname: hostname || 'localhost',
+      port
+    }))
+    this._traceBuffer = []
+    this._isInitialized = false
+  }
+
+  getAgentInfo (onReceivedInfo) {
+    fetchAgentInfo(this._url, onReceivedInfo)
+  }
+
+  export (trace) {
+    if (!this._isInitialized) {
+      this._traceBuffer.push(trace)
+      return
+    }
+    this._export(trace)
+  }
+
+  _export (payload, writer = this._writer, timerKey = '_timer') {
+    writer.append(payload)
+
+    const { flushInterval } = this._config
+
+    if (flushInterval === 0) {
+      writer.flush()
+    } else if (flushInterval > 0 && !this[timerKey]) {
+      this[timerKey] = setTimeout(() => {
+        writer.flush()
+        this[timerKey] = clearTimeout(this[timerKey])
+      }, flushInterval).unref()
+    }
+  }
+
+  getUncodedTraces () {
+    return this._traceBuffer
+  }
+
+  exportUncodedTraces () {
+    this.getUncodedTraces().forEach(uncodedTrace => {
+      this.export(uncodedTrace)
+    })
+    this.resetUncodedTraces()
+  }
+
+  resetUncodedTraces () {
+    this._traceBuffer = []
+  }
+}
+
+module.exports = AgentInfoExporter

package/packages/dd-trace/src/exporters/common/request.js

@@ -6,6 +6,7 @@
 const { Readable } = require('stream')
 const http = require('http')
 const https = require('https')
+const { parse: urlParse } = require('url')
 const docker = require('./docker')
 const { storage } = require('../../../../datadog-core')
 const log = require('../../log')
@@ -19,11 +20,54 @@ const containerId = docker.id()
 
 let activeRequests = 0
 
+// TODO: Replace with `url.urlToHttpOptions` when supported by all versions
+function urlToOptions (url) {
+  const agent = url.agent || http.globalAgent
+  const options = {
+    protocol: url.protocol || agent.protocol,
+    hostname: typeof url.hostname === 'string' && url.hostname.startsWith('[')
+      ? url.hostname.slice(1, -1)
+      : url.hostname ||
+      url.host ||
+      'localhost',
+    hash: url.hash,
+    search: url.search,
+    pathname: url.pathname,
+    path: `${url.pathname || ''}${url.search || ''}`,
+    href: url.href
+  }
+  if (url.port !== '') {
+    options.port = Number(url.port)
+  }
+  if (url.username || url.password) {
+    options.auth = `${url.username}:${url.password}`
+  }
+  return options
+}
+
+function fromUrlString (url) {
+  return typeof urlToHttpOptions === 'function'
+    ? urlToOptions(new URL(url))
+    : urlParse(url)
+}
+
 function request (data, options, callback) {
   if (!options.headers) {
     options.headers = {}
   }
 
+  if (options.url) {
+    const url = typeof options.url === 'object' ? options.url : fromUrlString(options.url)
+    if (url.protocol === 'unix:') {
+      options.socketPath = url.pathname
+    } else {
+      if (!options.path) options.path = url.path
+      options.protocol = url.protocol
+      options.hostname = url.hostname
+      options.port = url.port
+    }
+  }
+
   const isReadable = data instanceof Readable
 
   // The timeout should be kept low to avoid excessive queueing.
@@ -54,7 +98,13 @@ function request (data, options, callback) {
       if (res.statusCode >= 200 && res.statusCode <= 299) {
         callback(null, responseData, res.statusCode)
       } else {
-        const error = new Error(`Error from the endpoint: ${res.statusCode} ${http.STATUS_CODES[res.statusCode]}`)
+        const fullUrl = `${options.url || options.hostname || `localhost:${options.port}`}${options.path}`
+        // eslint-disable-next-line
+        let errorMessage = `Error from ${fullUrl}: ${res.statusCode} ${http.STATUS_CODES[res.statusCode]}.`
+        if (responseData) {
+          errorMessage += ` Response from the endpoint: "${responseData}"`
+        }
+        const error = new Error(errorMessage)
         error.status = res.statusCode
 
         callback(error, null, res.statusCode)

package/packages/dd-trace/src/exporters/span-stats/index.js

@@ -1,11 +1,15 @@
-const { URL } = require('url')
+const { URL, format } = require('url')
 
 const { Writer } = require('./writer')
 
 class SpanStatsExporter {
   constructor (config) {
     const { hostname = '127.0.0.1', port = 8126, tags, url } = config
-    this._url = url || new URL(`http://${hostname || 'localhost'}:${port}`)
+    this._url = url || new URL(format({
+      protocol: 'http:',
+      hostname: hostname || 'localhost',
+      port
+    }))
     this._writer = new Writer({ url: this._url, tags })
   }
 

package/packages/dd-trace/src/format.js

@@ -9,10 +9,18 @@ const SAMPLING_PRIORITY_KEY = constants.SAMPLING_PRIORITY_KEY
 const SAMPLING_RULE_DECISION = constants.SAMPLING_RULE_DECISION
 const SAMPLING_LIMIT_DECISION = constants.SAMPLING_LIMIT_DECISION
 const SAMPLING_AGENT_DECISION = constants.SAMPLING_AGENT_DECISION
+const SPAN_SAMPLING_MECHANISM = constants.SPAN_SAMPLING_MECHANISM
+const SPAN_SAMPLING_RULE_RATE = constants.SPAN_SAMPLING_RULE_RATE
+const SPAN_SAMPLING_MAX_PER_SECOND = constants.SPAN_SAMPLING_MAX_PER_SECOND
+const SAMPLING_MECHANISM_SPAN = constants.SAMPLING_MECHANISM_SPAN
 const MEASURED = tags.MEASURED
 const ORIGIN_KEY = constants.ORIGIN_KEY
 const HOSTNAME_KEY = constants.HOSTNAME_KEY
 const TOP_LEVEL_KEY = constants.TOP_LEVEL_KEY
+const PROCESS_ID = constants.PROCESS_ID
+const ERROR_MESSAGE = constants.ERROR_MESSAGE
+const ERROR_STACK = constants.ERROR_STACK
+const ERROR_TYPE = constants.ERROR_TYPE
 
 const map = {
   'service.name': 'service',
@@ -47,6 +55,13 @@ function formatSpan (span) {
   }
 }
 
+function setSingleSpanIngestionTags (span, options) {
+  if (!options) return
+  addTag({}, span.metrics, SPAN_SAMPLING_MECHANISM, SAMPLING_MECHANISM_SPAN)
+  addTag({}, span.metrics, SPAN_SAMPLING_RULE_RATE, options.sampleRate)
+  addTag({}, span.metrics, SPAN_SAMPLING_MAX_PER_SECOND, options.maxPerSecond)
+}
+
 function extractTags (trace, span) {
   const context = span.context()
   const origin = context._trace.origin
@@ -78,9 +93,9 @@ function extractTags (trace, span) {
           extractError(trace, tags[tag])
         }
         break
-      case 'error.type':
-      case 'error.msg':
-      case 'error.stack':
+      case ERROR_TYPE:
+      case ERROR_MESSAGE:
+      case ERROR_STACK:
         // HACK: remove when implemented in the backend
         if (context._name !== 'fs.operation') {
           trace.error = 1
@@ -92,10 +107,10 @@ function extractTags (trace, span) {
     }
   }
 
-  if (span.tracer()._service === tags['service.name']) {
-    addTag(trace.meta, trace.metrics, 'language', 'javascript')
-  }
+  setSingleSpanIngestionTags(trace, context._sampling.spanSampling)
 
+  addTag(trace.meta, trace.metrics, 'language', 'javascript')
+  addTag(trace.meta, trace.metrics, PROCESS_ID, process.pid)
   addTag(trace.meta, trace.metrics, SAMPLING_PRIORITY_KEY, priority)
   addTag(trace.meta, trace.metrics, ORIGIN_KEY, origin)
   addTag(trace.meta, trace.metrics, HOSTNAME_KEY, hostname)
@@ -131,9 +146,9 @@ function extractError (trace, error) {
   trace.error = 1
 
   if (isError(error)) {
-    addTag(trace.meta, trace.metrics, 'error.msg', error.message)
-    addTag(trace.meta, trace.metrics, 'error.type', error.name)
-    addTag(trace.meta, trace.metrics, 'error.stack', error.stack)
+    addTag(trace.meta, trace.metrics, ERROR_MESSAGE, error.message)
+    addTag(trace.meta, trace.metrics, ERROR_TYPE, error.name)
+    addTag(trace.meta, trace.metrics, ERROR_STACK, error.stack)
   }
 }
 
@@ -160,7 +175,7 @@ function addTag (meta, metrics, key, value, nested) {
         metrics[key] = value.toString()
       } else if (!Array.isArray(value) && !nested) {
         for (const prop in value) {
-          if (!value.hasOwnProperty(prop)) continue
+          if (!hasOwn(value, prop)) continue
 
           addTag(meta, metrics, `${key}.${prop}`, value[prop], true)
         }
@@ -170,6 +185,10 @@ function addTag (meta, metrics, key, value, nested) {
   }
 }
 
+function hasOwn (object, prop) {
+  return Object.prototype.hasOwnProperty.call(object, prop)
+}
+
 function isNodeBuffer (obj) {
   return obj.constructor && obj.constructor.name === 'Buffer' &&
     typeof obj.readInt8 === 'function' &&

package/packages/dd-trace/src/lambda/handler.js

@@ -0,0 +1,72 @@
+'use strict'
+
+const { channel } = require('../../../datadog-instrumentations/src/helpers/instrument')
+const { ERROR_MESSAGE, ERROR_TYPE } = require('../constants')
+const { ImpendingTimeout } = require('./runtime/errors')
+
+const globalTracer = global._ddtrace
+const tracer = globalTracer._tracer
+const timeoutChannel = channel('apm:aws:lambda:timeout')
+// Always crash the flushes when a message is received
+// from this channel.
+timeoutChannel.subscribe(_ => {
+  crashFlush()
+})
+
+let __lambdaTimeout
+
+/**
+ * Publishes to the `apm:aws:lambda:timeout` channel when
+ * the AWS Lambda run time is about to end.
+ *
+ * @param {*} context AWS Lambda context object.
+ */
+function checkTimeout (context) {
+  let remainingTimeInMillis = context.getRemainingTimeInMillis()
+  const apmFlushDeadline = parseInt(process.env.DD_APM_FLUSH_DEADLINE)
+  if (apmFlushDeadline && apmFlushDeadline <= remainingTimeInMillis) {
+    remainingTimeInMillis = apmFlushDeadline
+  }
+
+  __lambdaTimeout = setTimeout(() => {
+    timeoutChannel.publish(undefined)
+  }, remainingTimeInMillis - 50)
+}
+
+/**
+ * Grabs the current span, adds an error for an impending timeout.
+ *
+ * After that, it calls `killAll` on the tracer processor
+ * in order to kill remaining unfinished spans.
+ *
+ * Once that is done, it finishes the last span.
+ */
+function crashFlush () {
+  const activeSpan = tracer.scope().active()
+  const error = new ImpendingTimeout('Datadog detected an impending timeout')
+  activeSpan.addTags({
+    [ERROR_MESSAGE]: error.message,
+    [ERROR_TYPE]: error.name
+  })
+  tracer._processor.killAll()
+  activeSpan.finish()
+}
+
+/**
+ * Patches your AWS Lambda handler function to add some tracing support.
+ *
+ * @param {*} lambdaHandler a Lambda handler function.
+ */
+exports.datadog = function datadog (lambdaHandler) {
+  return (...args) => {
+    const context = args[1]
+    const patched = lambdaHandler.apply(this, args)
+    checkTimeout(context)
+
+    if (patched) {
+      // clear the timeout as soon as a result is returned
+      patched.then(_ => clearTimeout(__lambdaTimeout))
+    }
+    return patched
+  }
+}

package/packages/dd-trace/src/lambda/index.js

@@ -0,0 +1,5 @@
+'use strict'
+
+const { registerLambdaHook } = require('./runtime/ritm')
+
+registerLambdaHook()

package/packages/dd-trace/src/lambda/runtime/errors.js

@@ -0,0 +1,20 @@
+/**
+ * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Defines custom error types throwable by the runtime.
+ */
+'use strict'
+
+class ExtendedError extends Error {
+  constructor (reason) {
+    super(reason)
+    Object.setPrototypeOf(this, new.target.prototype)
+  }
+}
+
+class ImpendingTimeout extends ExtendedError {}
+ImpendingTimeout.prototype.name = 'Impending Timeout'
+
+module.exports = {
+  ImpendingTimeout
+}

package/packages/dd-trace/src/lambda/runtime/patch.js

@@ -0,0 +1,74 @@
+'use strict'
+
+const path = require('path')
+
+const { _extractModuleNameAndHandlerPath, _extractModuleRootAndHandler, _getLambdaFilePath } = require('./ritm')
+const { datadog } = require('../handler')
+const { addHook } = require('../../../../datadog-instrumentations/src/helpers/instrument')
+const shimmer = require('../../../../datadog-shimmer')
+
+/**
+ * Patches a Datadog Lambda module by calling `patchDatadogLambdaHandler`
+ * with the handler name `datadog`.
+ *
+ * @param {*} datadogLambdaModule node module to be patched.
+ * @returns a Datadog Lambda module with the `datadog` function from
+ * `datadog-lambda-js` patched.
+ */
+const patchDatadogLambdaModule = (datadogLambdaModule) => {
+  shimmer.wrap(datadogLambdaModule, 'datadog', patchDatadogLambdaHandler)
+
+  return datadogLambdaModule
+}
+
+/**
+ * Patches a Datadog Lambda handler in order to do
+ * Datadog instrumentation by getting the Lambda handler from its
+ * arguments.
+ *
+ * @param {*} datadogHandler the Datadog Lambda handler to destructure.
+ * @returns the datadogHandler with its arguments patched.
+ */
+function patchDatadogLambdaHandler (datadogHandler) {
+  return (userHandler) => {
+    return datadogHandler(datadog(userHandler))
+  }
+}
+
+/**
+ * Patches a Lambda module on the given handler path.
+ *
+ * @param {string} handlerPath path of the handler to be patched.
+ * @returns a module with the given handler path patched.
+ */
+const patchLambdaModule = (handlerPath) => (lambdaModule) => {
+  shimmer.wrap(lambdaModule, handlerPath, patchLambdaHandler)
+
+  return lambdaModule
+}
+
+/**
+ * Patches a Lambda handler in order to do Datadog instrumentation.
+ *
+ * @param {*} lambdaHandler the Lambda handler to be patched.
+ * @returns a function which patches the given Lambda handler.
+ */
+function patchLambdaHandler (lambdaHandler) {
+  return datadog(lambdaHandler)
+}
+
+const lambdaTaskRoot = process.env.LAMBDA_TASK_ROOT
+const originalLambdaHandler = process.env.DD_LAMBDA_HANDLER
+
+if (originalLambdaHandler !== undefined) {
+  const [moduleRoot, moduleAndHandler] = _extractModuleRootAndHandler(originalLambdaHandler)
+  const [_module, handlerPath] = _extractModuleNameAndHandlerPath(moduleAndHandler)
+
+  const lambdaStylePath = path.resolve(lambdaTaskRoot, moduleRoot, _module)
+  const lambdaFilePath = _getLambdaFilePath(lambdaStylePath)
+
+  addHook({ name: lambdaFilePath }, patchLambdaModule(handlerPath))
+} else {
+  // Instrumentation is done manually.
+  addHook({ name: 'datadog-lambda-js' }, patchDatadogLambdaModule)
+}