dd-trace 3.3.1 → 3.12.1

package/packages/dd-trace/src/profiling/config.js

@@ -2,7 +2,7 @@
 
 const coalesce = require('koalas')
 const os = require('os')
-const { URL } = require('url')
+const { URL, format } = require('url')
 const { AgentExporter } = require('./exporters/agent')
 const { FileExporter } = require('./exporters/file')
 const { ConsoleLogger } = require('./loggers/console')
@@ -33,6 +33,7 @@ class Config {
     const service = options.service || DD_SERVICE || 'node'
     const host = os.hostname()
     const version = coalesce(options.version, DD_VERSION)
+    const functionname = process.env.AWS_LAMBDA_FUNCTION_NAME
     // Must be longer than one minute so pad with five seconds
     const flushInterval = coalesce(options.interval, 65 * 1000)
     const uploadTimeout = coalesce(options.uploadTimeout,
@@ -46,12 +47,13 @@ class Config {
     this.service = service
     this.env = env
     this.host = host
+    this.functionname = functionname
 
     this.version = version
     this.tags = Object.assign(
       tagger.parse(DD_TAGS),
       tagger.parse(options.tags),
-      tagger.parse({ env, host, service, version })
+      tagger.parse({ env, host, service, version, functionname })
     )
     this.logger = ensureLogger(options.logger)
     this.flushInterval = flushInterval
@@ -59,10 +61,13 @@ class Config {
     this.sourceMap = sourceMap
     this.endpointCollection = endpointCollection
 
-    const hostname = coalesce(options.hostname, DD_AGENT_HOST, 'localhost')
-    const port = coalesce(options.port, DD_TRACE_AGENT_PORT, 8126)
-    this.url = new URL(coalesce(options.url, DD_TRACE_AGENT_URL,
-      `http://${hostname || 'localhost'}:${port || 8126}`))
+    const hostname = coalesce(options.hostname, DD_AGENT_HOST) || 'localhost'
+    const port = coalesce(options.port, DD_TRACE_AGENT_PORT) || 8126
+    this.url = new URL(coalesce(options.url, DD_TRACE_AGENT_URL, format({
+      protocol: 'http:',
+      hostname,
+      port
+    })))
 
     this.exporters = ensureExporters(options.exporters || [
       new AgentExporter(this)

package/packages/dd-trace/src/profiling/exporters/agent.js

@@ -6,11 +6,14 @@ const { request } = require('http')
 // TODO: avoid using dd-trace internals. Make this a separate module?
 const docker = require('../../exporters/common/docker')
 const FormData = require('../../exporters/common/form-data')
+const { storage } = require('../../../../datadog-core')
 const version = require('../../../../../package.json').version
 
 const containerId = docker.id()
 
 function sendRequest (options, form, callback) {
+  const store = storage.getStore()
+  storage.enterWith({ noop: true })
   const req = request(options, res => {
     if (res.statusCode >= 400) {
       const error = new Error(`HTTP Error ${res.statusCode}`)
@@ -22,6 +25,7 @@ function sendRequest (options, form, callback) {
   })
   req.on('error', callback)
   if (form) form.pipe(req)
+  storage.enterWith(store)
 }
 
 function getBody (stream, callback) {

package/packages/dd-trace/src/profiling/index.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { Profiler } = require('./profiler')
+const { Profiler, ServerlessProfiler } = require('./profiler')
 const CpuProfiler = require('./profilers/cpu')
 const WallProfiler = require('./profilers/wall')
 const SpaceProfiler = require('./profilers/space')
@@ -8,7 +8,7 @@ const { AgentExporter } = require('./exporters/agent')
 const { FileExporter } = require('./exporters/file')
 const { ConsoleLogger } = require('./loggers/console')
 
-const profiler = new Profiler()
+const profiler = process.env.AWS_LAMBDA_FUNCTION_NAME ? new ServerlessProfiler() : new Profiler()
 
 module.exports = {
   profiler,

package/packages/dd-trace/src/profiling/profiler.js

@@ -5,7 +5,6 @@ const { Config } = require('./config')
 
 function maybeSourceMap (sourceMap) {
   if (!sourceMap) return
-
   const { SourceMapper } = require('@datadog/pprof')
   return SourceMapper.create([
     process.cwd()
@@ -20,6 +19,7 @@ class Profiler extends EventEmitter {
     this._config = undefined
     this._timer = undefined
     this._lastStart = undefined
+    this._timeoutInterval = undefined
   }
 
   start (options) {
@@ -35,23 +35,35 @@ class Profiler extends EventEmitter {
 
     this._logger = config.logger
     this._enabled = true
+    this._setInterval()
 
+    // Log errors if the source map finder fails, but don't prevent the rest
+    // of the profiler from running without source maps.
+    let mapper
     try {
-      const mapper = await maybeSourceMap(config.sourceMap)
+      mapper = await maybeSourceMap(config.sourceMap)
+    } catch (err) {
+      this._logger.error(err)
+    }
 
+    try {
       for (const profiler of config.profilers) {
         // TODO: move this out of Profiler when restoring sourcemap support
         profiler.start({ mapper })
         this._logger.debug(`Started ${profiler.type} profiler`)
       }
 
-      this._capture(config.flushInterval)
+      this._capture(this._timeoutInterval)
     } catch (e) {
       this._logger.error(e)
       this.stop()
     }
   }
 
+  _setInterval () {
+    this._timeoutInterval = this._config.flushInterval
+  }
+
   stop () {
     if (!this._enabled) return
 
@@ -71,8 +83,7 @@ class Profiler extends EventEmitter {
   _capture (timeout) {
     if (!this._enabled) return
     this._lastStart = new Date()
-
-    if (!this._timer || timeout !== this._config.flushInterval) {
+    if (!this._timer || timeout !== this._timeoutInterval) {
       this._timer = setTimeout(() => this._collect(), timeout)
       this._timer.unref()
     } else {
@@ -99,7 +110,7 @@ class Profiler extends EventEmitter {
         })
       }
 
-      this._capture(this._config.flushInterval)
+      this._capture(this._timeoutInterval)
       await this._submit(profiles, start, end)
       this._logger.debug('Submitted profiles')
     } catch (err) {
@@ -126,4 +137,29 @@ class Profiler extends EventEmitter {
   }
 }
 
-module.exports = { Profiler }
+class ServerlessProfiler extends Profiler {
+  constructor () {
+    super()
+    this._profiledIntervals = 0
+    this._interval = 1
+    this._flushAfterIntervals = undefined
+  }
+
+  _setInterval () {
+    this._timeoutInterval = this._interval * 1000
+    this._flushAfterIntervals = this._config.flushInterval / 1000
+  }
+
+  async _collect () {
+    if (this._profiledIntervals >= this._flushAfterIntervals) {
+      this._profiledIntervals = 0
+      await super._collect()
+    } else {
+      this._profiledIntervals += 1
+      this._capture(this._timeoutInterval)
+      // Don't submit profile until 65 (flushAfterIntervals) intervals have elapsed
+    }
+  }
+}
+
+module.exports = { Profiler, ServerlessProfiler }

package/packages/dd-trace/src/proxy.js

@@ -1,5 +1,4 @@
 'use strict'
-
 const NoopProxy = require('./noop/proxy')
 const DatadogTracer = require('./tracer')
 const Config = require('./config')
@@ -8,7 +7,7 @@ const log = require('./log')
 const { setStartupLogPluginManager } = require('./startup-log')
 const telemetry = require('./telemetry')
 const PluginManager = require('./plugin_manager')
-const { sendGitMetadata } = require('./ci-visibility/exporters/git/git_metadata')
+const remoteConfig = require('./appsec/remote_config')
 
 class Tracer extends NoopProxy {
   constructor () {
@@ -26,8 +25,9 @@ class Tracer extends NoopProxy {
     try {
       const config = new Config(options) // TODO: support dynamic config
 
-      log.use(config.logger)
-      log.toggle(config.debug, config.logLevel, this)
+      if (config.remoteConfig.enabled && !config.isCiVisibility) {
+        remoteConfig.enable(config)
+      }
 
       if (config.profiling.enabled) {
         // do not stop tracer initialization if the profiler fails to be imported
@@ -48,25 +48,17 @@ class Tracer extends NoopProxy {
         if (config.appsec.enabled) {
           require('./appsec').enable(config)
         }
+
+        this._tracer = new DatadogTracer(config)
+
         if (config.iast.enabled) {
-          require('./appsec/iast').enable(config)
+          require('./appsec/iast').enable(config, this._tracer)
         }
 
-        this._tracer = new DatadogTracer(config)
         this._pluginManager.configure(config)
         setStartupLogPluginManager(this._pluginManager)
         telemetry.start(config, this._pluginManager)
       }
-
-      if (config.isGitUploadEnabled) {
-        sendGitMetadata(config.site, (err) => {
-          if (err) {
-            log.error(`Error uploading git metadata: ${err}`)
-          } else {
-            log.debug('Successfully uploaded git metadata')
-          }
-        })
-      }
     } catch (e) {
       log.error(e)
     }

package/packages/dd-trace/src/ritm.js

@@ -16,6 +16,8 @@ let cache = Object.create(null)
 let patching = Object.create(null)
 let patchedRequire = null
 const moduleLoadStartChannel = dc.channel('dd-trace:moduleLoadStart')
+const moduleLoadEndChannel = dc.channel('dd-trace:moduleLoadEnd')
+
 function Hook (modules, options, onrequire) {
   if (!(this instanceof Hook)) return new Hook(modules, options, onrequire)
   if (typeof modules === 'function') {
@@ -52,7 +54,6 @@ function Hook (modules, options, onrequire) {
     const filename = Module._resolveFilename(request, this)
     const core = filename.indexOf(path.sep) === -1
     let name, basedir, hooks
-
     // return known patched modules immediately
     if (cache[filename]) {
       // require.cache was potentially altered externally
@@ -66,33 +67,43 @@ function Hook (modules, options, onrequire) {
     // Check if this module has a patcher in-progress already.
     // Otherwise, mark this module as patching in-progress.
     const patched = patching[filename]
-    if (!patched) {
+    if (patched) {
+      // If it's already patched, just return it as-is.
+      return origRequire.apply(this, arguments)
+    } else {
       patching[filename] = true
     }
 
-    const exports = origRequire.apply(this, arguments)
+    const payload = {
+      filename,
+      request
+    }
 
-    // If it's already patched, just return it as-is.
-    if (patched) return exports
+    if (moduleLoadStartChannel.hasSubscribers) {
+      moduleLoadStartChannel.publish(payload)
+    }
+    const exports = origRequire.apply(this, arguments)
+    payload.module = exports
+    if (moduleLoadEndChannel.hasSubscribers) {
+      moduleLoadEndChannel.publish(payload)
+    }
 
     // The module has already been loaded,
     // so the patching mark can be cleaned up.
     delete patching[filename]
 
-    if (moduleLoadStartChannel.hasSubscribers) {
-      moduleLoadStartChannel.publish({
-        filename,
-        module: exports,
-        request
-      })
-    }
-
     if (core) {
       hooks = moduleHooks[filename]
       if (!hooks) return exports // abort if module name isn't on whitelist
       name = filename
     } else {
-      const stat = parse(filename)
+      const inAWSLambda = process.env.AWS_LAMBDA_FUNCTION_NAME !== undefined
+      const hasLambdaHandler = process.env.DD_LAMBDA_HANDLER !== undefined
+      const segments = filename.split(path.sep)
+      const filenameFromNodeModule = segments.lastIndexOf('node_modules') !== -1
+      // decide how to assign the stat
+      // first case will only happen when patching an AWS Lambda Handler
+      const stat = inAWSLambda && hasLambdaHandler && !filenameFromNodeModule ? { name: filename } : parse(filename)
       if (!stat) return exports // abort if filename could not be parsed
       name = stat.name
       basedir = stat.basedir

package/packages/dd-trace/src/span_processor.js

@@ -2,6 +2,7 @@
 
 const log = require('./log')
 const format = require('./format')
+const SpanSampler = require('./span_sampler')
 
 const { SpanStatsProcessor } = require('./span_stats')
 
@@ -13,8 +14,10 @@ class SpanProcessor {
     this._exporter = exporter
     this._prioritySampler = prioritySampler
     this._config = config
+    this._killAll = false
 
     this._stats = new SpanStatsProcessor(config)
+    this._spanSampler = new SpanSampler(config)
   }
 
   process (span) {
@@ -25,8 +28,10 @@ class SpanProcessor {
     const { flushMinSpans } = this._config
     const { started, finished } = trace
 
+    if (trace.record === false) return
     if (started.length === finished.length || finished.length >= flushMinSpans) {
       this._prioritySampler.sample(spanContext)
+      this._spanSampler.sample(spanContext)
 
       for (const span of started) {
         if (span._duration !== undefined) {
@@ -44,6 +49,18 @@ class SpanProcessor {
 
       this._erase(trace, active)
     }
+
+    if (this._killAll) {
+      started.map(startedSpan => {
+        if (!startedSpan._finished) {
+          startedSpan.finish()
+        }
+      })
+    }
+  }
+
+  killAll () {
+    this._killAll = true
   }
 
   _erase (trace, active) {

package/packages/dd-trace/src/span_sampler.js

@@ -0,0 +1,77 @@
+'use strict'
+const { globMatch } = require('../src/util')
+const { USER_KEEP, AUTO_KEEP } = require('../../../ext').priority
+const RateLimiter = require('./rate_limiter')
+
+class SpanSampler {
+  constructor ({ spanSamplingRules = [] }) {
+    this._rules = spanSamplingRules
+    this._limiters = {}
+  }
+
+  sample (spanContext) {
+    const decision = spanContext._sampling.priority
+    if (decision === USER_KEEP || decision === AUTO_KEEP) return
+
+    const { started } = spanContext._trace
+    for (const span of started) {
+      const service = span.tracer()._service
+      const name = span._name
+      const rule = findRule(this._rules, service, name)
+      if (!rule) continue
+
+      const sampleRate = getSampleRate(rule.sampleRate)
+      const maxPerSecond = getMaxPerSecond(rule.maxPerSecond)
+      const sampled = sample(sampleRate)
+      if (!sampled) continue
+
+      const key = `${service}:${name}`
+      const limiter = getLimiter(this._limiters, key, maxPerSecond)
+      if (limiter.isAllowed()) {
+        span.context()._sampling.spanSampling = {
+          sampleRate,
+          maxPerSecond
+        }
+      }
+    }
+  }
+}
+
+function findRule (rules, service, name) {
+  for (const rule of rules) {
+    const servicePattern = getService(rule.service)
+    const namePattern = getName(rule.name)
+    if (globMatch(servicePattern, service) && globMatch(namePattern, name)) {
+      return rule
+    }
+  }
+}
+
+function getLimiter (list, key, maxPerSecond) {
+  if (typeof list[key] === 'undefined') {
+    list[key] = new RateLimiter(maxPerSecond)
+  }
+  return list[key]
+}
+
+function sample (sampleRate) {
+  return Math.random() < sampleRate
+}
+
+function getService (service) {
+  return service || '*'
+}
+
+function getName (name) {
+  return name || '*'
+}
+
+function getSampleRate (sampleRate) {
+  return sampleRate || 1.0
+}
+
+function getMaxPerSecond (maxPerSecond) {
+  return maxPerSecond || Infinity
+}
+
+module.exports = SpanSampler

package/packages/dd-trace/src/span_stats.js

@@ -65,8 +65,8 @@ class SpanAggStats {
       TopLevelHits: this.topLevelHits,
       Errors: this.errors,
       Duration: this.duration,
-      OkSummary: this.okDistribution.toProto(),
-      ErrorSummary: this.errorDistribution.toProto()
+      OkSummary: this.okDistribution.toProto(), // TODO: custom proto encoding
+      ErrorSummary: this.errorDistribution.toProto() // TODO: custom proto encoding
     }
   }
 }

package/packages/dd-trace/src/telemetry/dependencies.js

@@ -7,7 +7,7 @@ const { sendData } = require('./send-data')
 const dc = require('diagnostics_channel')
 const { fileURLToPath } = require('url')
 
-const savedDependencies = []
+const savedDependencies = new Set()
 const detectedDependencyNames = new Set()
 const FILE_URI_START = `file://`
 const moduleLoadStartChannel = dc.channel('dd-trace:moduleLoadStart')
@@ -18,10 +18,14 @@ function waitAndSend (config, application, host) {
   if (!immediate) {
     immediate = setImmediate(() => {
       immediate = null
-      if (savedDependencies.length > 0) {
-        const dependencies = savedDependencies.splice(0, 1000)
+      if (savedDependencies.size > 0) {
+        const dependencies = Array.from(savedDependencies.values()).splice(0, 1000).map(pair => {
+          savedDependencies.delete(pair)
+          const [name, version] = pair.split(' ')
+          return { name, version }
+        })
         sendData(config, application, host, 'app-dependencies-loaded', { dependencies })
-        if (savedDependencies.length > 0) {
+        if (savedDependencies.size > 0) {
           waitAndSend(config, application, host)
         }
       }
@@ -49,7 +53,7 @@ function onModuleLoad (data) {
         if (basedir) {
           try {
             const { version } = requirePackageJson(basedir, module)
-            savedDependencies.push({ name, version })
+            savedDependencies.add(`${name} ${version}`)
             waitAndSend(config, application, host)
           } catch (e) {
             // can not read the package.json, do nothing
@@ -67,7 +71,17 @@ function start (_config, _application, _host) {
 }
 
 function isDependency (filename, request) {
-  return request.indexOf(`.${path.sep}`) !== 0 && request.indexOf(path.sep) !== 0
+  const isDependencyWithSlash = isDependencyWithSeparator(filename, request, '/')
+  if (isDependencyWithSlash && process.platform === 'win32') {
+    return isDependencyWithSeparator(filename, request, path.sep)
+  }
+  return isDependencyWithSlash
+}
+function isDependencyWithSeparator (filename, request, sep) {
+  return request.indexOf(`..${sep}`) !== 0 &&
+    request.indexOf(`.${sep}`) !== 0 &&
+    request.indexOf(sep) !== 0 &&
+    request.indexOf(`:${sep}`) !== 1
 }
 
 function stop () {
@@ -75,7 +89,7 @@ function stop () {
   application = null
   host = null
   detectedDependencyNames.clear()
-  savedDependencies.splice(0, savedDependencies.length)
+  savedDependencies.clear()
   if (moduleLoadStartChannel.hasSubscribers) {
     moduleLoadStartChannel.unsubscribe(onModuleLoad)
   }

package/packages/dd-trace/src/telemetry/index.js

@@ -6,6 +6,10 @@ const os = require('os')
 const dependencies = require('./dependencies')
 const { sendData } = require('./send-data')
 
+const HEARTBEAT_INTERVAL = process.env.DD_TELEMETRY_HEARTBEAT_INTERVAL
+  ? Number(process.env.DD_TELEMETRY_HEARTBEAT_INTERVAL) * 1000
+  : 60000
+
 let config
 let pluginManager
 
@@ -88,7 +92,9 @@ function start (aConfig, thePluginManager) {
   host = createHostObject()
   dependencies.start(config, application, host)
   sendData(config, application, host, 'app-started', appStarted())
-  interval = setInterval(() => sendData(config, application, host, 'app-heartbeat'), 60000)
+  interval = setInterval(() => {
+    sendData(config, application, host, 'app-heartbeat')
+  }, HEARTBEAT_INTERVAL)
   interval.unref()
   process.on('beforeExit', onBeforeExit)
 }

package/packages/dd-trace/src/telemetry/send-data.js

@@ -3,9 +3,11 @@ let seqId = 0
 function sendData (config, application, host, reqType, payload = {}) {
   const {
     hostname,
-    port
+    port,
+    url
   } = config
   const options = {
+    url,
     hostname,
     port,
     method: 'POST',

package/packages/dd-trace/src/tracer.js

@@ -6,6 +6,7 @@ const Scope = require('./scope')
 const { storage } = require('../../datadog-core')
 const { isError } = require('./util')
 const { setStartupLogConfig } = require('./startup-log')
+const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants')
 
 const SPAN_TYPE = tags.SPAN_TYPE
 const RESOURCE_NAME = tags.RESOURCE_NAME
@@ -44,11 +45,15 @@ class DatadogTracer extends Tracer {
       const result = this.scope().activate(span, () => fn(span))
 
       if (result && typeof result.then === 'function') {
-        result.then(
-          () => span.finish(),
+        return result.then(
+          value => {
+            span.finish()
+            return value
+          },
           err => {
             addError(span, err)
             span.finish()
+            throw err
           }
         )
       } else {
@@ -139,9 +144,9 @@ class DatadogTracer extends Tracer {
 function addError (span, error) {
   if (isError(error)) {
     span.addTags({
-      'error.type': error.name,
-      'error.msg': error.message,
-      'error.stack': error.stack
+      [ERROR_TYPE]: error.name,
+      [ERROR_MESSAGE]: error.message,
+      [ERROR_STACK]: error.stack
     })
   }
 }

package/packages/dd-trace/src/util.js

@@ -20,8 +20,50 @@ function isError (value) {
   return false
 }
 
+// Matches a glob pattern to a given subject string
+function globMatch (pattern, subject) {
+  let px = 0 // [p]attern inde[x]
+  let sx = 0 // [s]ubject inde[x]
+  let nextPx = 0
+  let nextSx = 0
+  while (px < pattern.length || sx < subject.length) {
+    if (px < pattern.length) {
+      const c = pattern[px]
+      switch (c) {
+        default: // ordinary character
+          if (sx < subject.length && subject[sx] === c) {
+            px++
+            sx++
+            continue
+          }
+          break
+        case '?':
+          if (sx < subject.length) {
+            px++
+            sx++
+            continue
+          }
+          break
+        case '*':
+          nextPx = px
+          nextSx = sx + 1
+          px++
+          continue
+      }
+    }
+    if (nextSx > 0 && nextSx <= subject.length) {
+      px = nextPx
+      sx = nextSx
+      continue
+    }
+    return false
+  }
+  return true
+}
+
 module.exports = {
   isTrue,
   isFalse,
-  isError
+  isError,
+  globMatch
 }