dd-trace 1.5.0 → 1.7.0

package/packages/dd-trace/src/plugins/util/user-provided-git.js

@@ -0,0 +1,72 @@
+const {
+  GIT_COMMIT_SHA,
+  GIT_BRANCH,
+  GIT_REPOSITORY_URL,
+  GIT_TAG,
+  GIT_COMMIT_MESSAGE,
+  GIT_COMMIT_COMMITTER_DATE,
+  GIT_COMMIT_COMMITTER_EMAIL,
+  GIT_COMMIT_COMMITTER_NAME,
+  GIT_COMMIT_AUTHOR_DATE,
+  GIT_COMMIT_AUTHOR_EMAIL,
+  GIT_COMMIT_AUTHOR_NAME
+} = require('./tags')
+
+const { normalizeRef } = require('./ci')
+
+function removeEmptyValues (tags) {
+  return Object.keys(tags).reduce((filteredTags, tag) => {
+    if (!tags[tag]) {
+      return filteredTags
+    }
+    return {
+      ...filteredTags,
+      [tag]: tags[tag]
+    }
+  }, {})
+}
+
+function getUserProviderGitMetadata () {
+  const {
+    DD_GIT_COMMIT_SHA,
+    DD_GIT_BRANCH,
+    DD_GIT_REPOSITORY_URL,
+    DD_GIT_TAG,
+    DD_GIT_COMMIT_MESSAGE,
+    DD_GIT_COMMIT_COMMITTER_NAME,
+    DD_GIT_COMMIT_COMMITTER_EMAIL,
+    DD_GIT_COMMIT_COMMITTER_DATE,
+    DD_GIT_COMMIT_AUTHOR_NAME,
+    DD_GIT_COMMIT_AUTHOR_EMAIL,
+    DD_GIT_COMMIT_AUTHOR_DATE
+  } = process.env
+
+  let branch = normalizeRef(DD_GIT_BRANCH)
+  let tag = normalizeRef(DD_GIT_TAG)
+
+  if (DD_GIT_TAG) {
+    branch = undefined
+  }
+
+  // if DD_GIT_BRANCH is a tag, we associate its value to TAG instead of BRANCH
+  if ((DD_GIT_BRANCH || '').includes('origin/tags') || (DD_GIT_BRANCH || '').includes('refs/heads/tags')) {
+    branch = undefined
+    tag = normalizeRef(DD_GIT_BRANCH)
+  }
+
+  return removeEmptyValues({
+    [GIT_COMMIT_SHA]: DD_GIT_COMMIT_SHA,
+    [GIT_BRANCH]: branch,
+    [GIT_REPOSITORY_URL]: DD_GIT_REPOSITORY_URL,
+    [GIT_TAG]: tag,
+    [GIT_COMMIT_MESSAGE]: DD_GIT_COMMIT_MESSAGE,
+    [GIT_COMMIT_COMMITTER_NAME]: DD_GIT_COMMIT_COMMITTER_NAME,
+    [GIT_COMMIT_COMMITTER_DATE]: DD_GIT_COMMIT_COMMITTER_DATE,
+    [GIT_COMMIT_COMMITTER_EMAIL]: DD_GIT_COMMIT_COMMITTER_EMAIL,
+    [GIT_COMMIT_AUTHOR_NAME]: DD_GIT_COMMIT_AUTHOR_NAME,
+    [GIT_COMMIT_AUTHOR_EMAIL]: DD_GIT_COMMIT_AUTHOR_EMAIL,
+    [GIT_COMMIT_AUTHOR_DATE]: DD_GIT_COMMIT_AUTHOR_DATE
+  })
+}
+
+module.exports = { getUserProviderGitMetadata }

package/packages/dd-trace/src/plugins/util/web.js

@@ -181,7 +181,7 @@ const web = {
 
   // Extract the parent span from the headers and start a new span as its child
   startChildSpan (tracer, name, headers) {
-    const childOf = tracer.extract(FORMAT_HTTP_HEADERS, headers)
+    const childOf = tracer.scope().active() || tracer.extract(FORMAT_HTTP_HEADERS, headers)
     const span = tracer.startSpan(name, { childOf })
 
     return span

package/packages/dd-trace/src/priority_sampler.js

@@ -1,14 +1,20 @@
 'use strict'
 
+const coalesce = require('koalas')
 const RateLimiter = require('./rate_limiter')
 const Sampler = require('./sampler')
 const ext = require('../../../ext')
 const { setSamplingRules } = require('./startup-log')
 
 const {
+  SAMPLING_MECHANISM_DEFAULT,
+  SAMPLING_MECHANISM_AGENT,
+  SAMPLING_MECHANISM_RULE,
+  SAMPLING_MECHANISM_MANUAL,
   SAMPLING_RULE_DECISION,
   SAMPLING_LIMIT_DECISION,
-  SAMPLING_AGENT_DECISION
+  SAMPLING_AGENT_DECISION,
+  UPSTREAM_SERVICES_KEY
 } = require('./constants')
 
 const SERVICE_NAME = ext.tags.SERVICE_NAME
@@ -21,6 +27,9 @@ const AUTO_KEEP = ext.priority.AUTO_KEEP
 const USER_KEEP = ext.priority.USER_KEEP
 const DEFAULT_KEY = 'service:,env:'
 
+const defaultSampler = new Sampler(AUTO_KEEP)
+const serviceNames = new Map()
+
 class PrioritySampler {
   constructor (env, { sampleRate, rateLimit = 100, rules = [] } = {}) {
     this._env = env
@@ -33,12 +42,8 @@ class PrioritySampler {
   }
 
   isSampled (span) {
-    const context = this._getContext(span)
-    const rule = this._findRule(context)
-
-    return rule
-      ? this._isSampledByRule(context, rule) && this._isSampledByRateLimit(context)
-      : this._isSampledByAgent(context)
+    const priority = this._getPriorityFromAuto(span)
+    return priority === USER_KEEP || priority === AUTO_KEEP
   }
 
   sample (span, auto = true) {
@@ -50,16 +55,18 @@ class PrioritySampler {
     if (context._sampling.priority !== undefined) return
     if (!root) return // noop span
 
-    const tag = this._getPriority(context._tags)
+    const tag = this._getPriorityFromTags(context._tags)
 
     if (this.validate(tag)) {
       context._sampling.priority = tag
+      context._sampling.mechanism = SAMPLING_MECHANISM_MANUAL
+    } else if (auto) {
+      context._sampling.priority = this._getPriorityFromAuto(root)
+    } else {
       return
     }
 
-    if (auto) {
-      context._sampling.priority = this.isSampled(root) ? AUTO_KEEP : AUTO_REJECT
-    }
+    this._addUpstreamService(root)
   }
 
   update (rates) {
@@ -72,7 +79,7 @@ class PrioritySampler {
       samplers[key] = sampler
     }
 
-    samplers[DEFAULT_KEY] = samplers[DEFAULT_KEY] || new Sampler(AUTO_KEEP)
+    samplers[DEFAULT_KEY] = samplers[DEFAULT_KEY] || defaultSampler
 
     this._samplers = samplers
   }
@@ -93,7 +100,16 @@ class PrioritySampler {
     return typeof span.context === 'function' ? span.context() : span
   }
 
-  _getPriority (tags) {
+  _getPriorityFromAuto (span) {
+    const context = this._getContext(span)
+    const rule = this._findRule(context)
+
+    return rule
+      ? this._getPriorityByRule(context, rule)
+      : this._getPriorityByAgent(context)
+  }
+
+  _getPriorityFromTags (tags) {
     if (tags.hasOwnProperty(MANUAL_KEEP) && tags[MANUAL_KEEP] !== false) {
       return USER_KEEP
     } else if (tags.hasOwnProperty(MANUAL_DROP) && tags[MANUAL_DROP] !== false) {
@@ -109,10 +125,11 @@ class PrioritySampler {
     }
   }
 
-  _isSampledByRule (context, rule) {
+  _getPriorityByRule (context, rule) {
     context._trace[SAMPLING_RULE_DECISION] = rule.sampleRate
+    context._sampling.mechanism = SAMPLING_MECHANISM_RULE
 
-    return rule.sampler.isSampled(context)
+    return rule.sampler.isSampled(context) && this._isSampledByRateLimit(context) ? USER_KEEP : USER_REJECT
   }
 
   _isSampledByRateLimit (context) {
@@ -123,13 +140,48 @@ class PrioritySampler {
     return allowed
   }
 
-  _isSampledByAgent (context) {
+  _getPriorityByAgent (context) {
     const key = `service:${context._tags[SERVICE_NAME]},env:${this._env}`
     const sampler = this._samplers[key] || this._samplers[DEFAULT_KEY]
 
     context._trace[SAMPLING_AGENT_DECISION] = sampler.rate()
 
-    return sampler.isSampled(context)
+    if (sampler === defaultSampler) {
+      context._sampling.mechanism = SAMPLING_MECHANISM_DEFAULT
+    } else {
+      context._sampling.mechanism = SAMPLING_MECHANISM_AGENT
+    }
+
+    return sampler.isSampled(context) ? AUTO_KEEP : AUTO_REJECT
+  }
+
+  _addUpstreamService (span) {
+    const context = span.context()
+    const trace = context._trace
+    const service = this._toBase64(context._tags['service.name'])
+    const priority = context._sampling.priority
+    const mechanism = context._sampling.mechanism
+    const rate = Math.ceil(coalesce(
+      context._trace[SAMPLING_RULE_DECISION],
+      context._trace[SAMPLING_AGENT_DECISION]
+    ) * 10000) / 10000
+    const group = `${service}|${priority}|${mechanism}|${rate}`
+    const groups = trace.tags[UPSTREAM_SERVICES_KEY]
+      ? `${trace.tags[UPSTREAM_SERVICES_KEY]};${group}`
+      : group
+
+    trace.tags[UPSTREAM_SERVICES_KEY] = groups
+  }
+
+  _toBase64 (serviceName) {
+    let encoded = serviceNames.get(serviceName)
+
+    if (!encoded) {
+      encoded = Buffer.from(serviceName).toString('base64')
+      serviceNames.set(serviceName, encoded)
+    }
+
+    return encoded
   }
 
   _normalizeRules (rules, sampleRate) {
@@ -151,9 +203,9 @@ class PrioritySampler {
     const service = context._tags['service.name']
 
     if (rule.name instanceof RegExp && !rule.name.test(name)) return false
-    if (rule.name && rule.name !== name) return false
+    if (typeof rule.name === 'string' && rule.name !== name) return false
     if (rule.service instanceof RegExp && !rule.service.test(service)) return false
-    if (rule.service && rule.service !== service) return false
+    if (typeof rule.service === 'string' && rule.service !== service) return false
 
     return true
   }

package/packages/dd-trace/src/profiling/exporters/agent.js

@@ -4,9 +4,13 @@ const retry = require('retry')
 const { request } = require('http')
 const FormData = require('form-data')
 
+// TODO: avoid using dd-trace internals. Make this a separate module?
+const docker = require('../../exporters/agent/docker')
 const version = require('../../../lib/version')
 
-function sendRequest (options, body, callback) {
+const containerId = docker.id()
+
+function sendRequest (options, form, callback) {
   const req = request(options, res => {
     if (res.statusCode >= 400) {
       const error = new Error(`HTTP Error ${res.statusCode}`)
@@ -17,7 +21,7 @@ function sendRequest (options, body, callback) {
     }
   })
   req.on('error', callback)
-  if (body) req.write(body)
+  if (form) form.pipe(req)
   req.end()
 }
 
@@ -36,7 +40,7 @@ function computeRetries (uploadTimeout) {
     tries++
     uploadTimeout /= 2
   }
-  return [tries, uploadTimeout]
+  return [tries, Math.floor(uploadTimeout)]
 }
 
 class AgentExporter {
@@ -51,7 +55,6 @@ class AgentExporter {
   }
 
   export ({ profiles, start, end, tags }) {
-    const form = new FormData()
     const types = Object.keys(profiles)
 
     const fields = [
@@ -71,10 +74,6 @@ class AgentExporter {
       ...Object.entries(tags).map(([key, value]) => ['tags[]', `${key}:${value}`])
     ]
 
-    for (const [key, value] of fields) {
-      form.append(key, value)
-    }
-
     this._logger.debug(() => {
       const body = fields.map(([key, value]) => `  ${key}: ${value}`).join('\n')
       return `Building agent export report: ${'\n' + body}`
@@ -89,33 +88,14 @@ class AgentExporter {
         return `Adding ${type} profile to agent export: ` + bytes
       })
 
-      form.append(`types[${index}]`, type)
-      form.append(`data[${index}]`, buffer, {
+      fields.push([`types[${index}]`, type])
+      fields.push([`data[${index}]`, buffer, {
         filename: `${type}.pb.gz`,
         contentType: 'application/octet-stream',
         knownLength: buffer.length
-      })
-    }
-
-    const body = form.getBuffer()
-    const options = {
-      method: 'POST',
-      path: '/profiling/v1/input',
-      headers: form.getHeaders()
-    }
-
-    if (this._url.protocol === 'unix:') {
-      options.socketPath = this._url.pathname
-    } else {
-      options.protocol = this._url.protocol
-      options.hostname = this._url.hostname
-      options.port = this._url.port
+      }])
     }
 
-    this._logger.debug(() => {
-      return `Submitting agent report to: ${JSON.stringify(options)}`
-    })
-
     return new Promise((resolve, reject) => {
       const operation = retry.operation({
         randomize: true,
@@ -124,8 +104,36 @@ class AgentExporter {
       })
 
       operation.attempt((attempt) => {
-        const timeout = Math.pow(this._backoffTime, attempt)
-        sendRequest({ ...options, timeout }, body, (err, response) => {
+        const form = new FormData()
+
+        for (const [key, value, options] of fields) {
+          form.append(key, value, options)
+        }
+
+        const options = {
+          method: 'POST',
+          path: '/profiling/v1/input',
+          headers: form.getHeaders(),
+          timeout: this._backoffTime * Math.pow(2, attempt)
+        }
+
+        if (containerId) {
+          options.headers['Datadog-Container-ID'] = containerId
+        }
+
+        if (this._url.protocol === 'unix:') {
+          options.socketPath = this._url.pathname
+        } else {
+          options.protocol = this._url.protocol
+          options.hostname = this._url.hostname
+          options.port = this._url.port
+        }
+
+        this._logger.debug(() => {
+          return `Submitting profiler agent report attempt #${attempt} to: ${JSON.stringify(options)}`
+        })
+
+        sendRequest(options, form, (err, response) => {
           if (operation.retry(err)) {
             this._logger.error(`Error from the agent: ${err.message}`)
             return
@@ -152,4 +160,4 @@ class AgentExporter {
   }
 }
 
-module.exports = { AgentExporter }
+module.exports = { AgentExporter, computeRetries }

package/packages/dd-trace/src/profiling/profilers/cpu.js

@@ -25,9 +25,7 @@ class NativeCpuProfiler {
 
   profile () {
     if (!this._stop) return
-    const profile = this._stop()
-    this._record()
-    return profile
+    return this._stop(true)
   }
 
   encode (profile) {

package/packages/dd-trace/src/proxy.js

@@ -46,7 +46,7 @@ class Tracer extends BaseTracer {
             metrics.start(config)
           }
 
-          // dirty require for now so zero appsec code is executed unless explicitely enabled
+          // dirty require for now so zero appsec code is executed unless explicitly enabled
           if (config.appsec.enabled) {
             require('./appsec').enable(config)
           }

package/packages/dd-trace/src/plugins/util/ci-app-spec.json

@@ -1,35 +0,0 @@
-[
-  "test.type",
-  "test.status",
-  "test.framework",
-  "test.suite",
-  "test.name",
-  "test.skip_reason",
-  "ci.pipeline.id",
-  "ci.pipeline.name",
-  "ci.pipeline.number",
-  "ci.pipeline.url",
-  "ci.provider.name",
-  "ci.workspace_path",
-  "git.repository_url",
-  "ci.job.url",
-  "ci.job.name",
-  "ci.stage.name",
-  "git.commit.sha",
-  "git.branch",
-  "git.tag",
-  "git.commit.message",
-  "git.commit.committer.date",
-  "git.commit.committer.email",
-  "git.commit.committer.name",
-  "git.commit.author.date",
-  "git.commit.author.email",
-  "git.commit.author.name",
-  "os.platform",
-  "os.version",
-  "os.architecture",
-  "runtime.name",
-  "runtime.version",
-  "test.parameters",
-  "_dd.origin"
-]