dd-trace 2.12.1 → 3.2.0

package/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js

@@ -2,30 +2,75 @@
 
 const URL = require('url').URL
 const Writer = require('./writer')
-const Scheduler = require('../../../exporters/scheduler')
+const CoverageWriter = require('./coverage-writer')
+
+const log = require('../../../log')
 
 class AgentlessCiVisibilityExporter {
   constructor (config) {
-    const { flushInterval, tags, site, url } = config
+    this._config = config
+    const { tags, site, url, isIntelligentTestRunnerEnabled } = config
+    this._isIntelligentTestRunnerEnabled = isIntelligentTestRunnerEnabled
     this._url = url || new URL(`https://citestcycle-intake.${site}`)
     this._writer = new Writer({ url: this._url, tags })
+    this._timer = undefined
+    this._coverageTimer = undefined
+
+    this._coverageUrl = url || new URL(`https://event-platform-intake.${site}`)
+    this._coverageWriter = new CoverageWriter({ url: this._coverageUrl })
+
+    process.once('beforeExit', () => {
+      this._writer.flush()
+      this._coverageWriter.flush()
+    })
+  }
 
-    if (flushInterval > 0) {
-      this._scheduler = new Scheduler(() => this._writer.flush(), flushInterval)
+  exportCoverage ({ testSpan, coverageFiles }) {
+    const formattedCoverage = {
+      traceId: testSpan.context()._traceId,
+      spanId: testSpan.context()._spanId,
+      files: coverageFiles
+    }
+    this._coverageWriter.append(formattedCoverage)
+
+    const { flushInterval } = this._config
+
+    if (flushInterval === 0) {
+      this._coverageWriter.flush()
+    } else if (flushInterval > 0 && !this._coverageTimer) {
+      this._coverageTimer = setTimeout(() => {
+        this._coverageWriter.flush()
+        this._coverageTimer = clearTimeout(this._coverageTimer)
+      }, flushInterval).unref()
     }
-    this._scheduler && this._scheduler.start()
   }
 
   export (trace) {
     this._writer.append(trace)
 
-    if (!this._scheduler) {
+    const { flushInterval } = this._config
+
+    if (flushInterval === 0) {
       this._writer.flush()
+    } else if (flushInterval > 0 && !this._timer) {
+      this._timer = setTimeout(() => {
+        this._writer.flush()
+        this._timer = clearTimeout(this._timer)
+      }, flushInterval).unref()
     }
   }
 
-  flush () {
-    this._writer.flush()
+  setUrl (url, coverageUrl = url) {
+    try {
+      url = new URL(url)
+      coverageUrl = new URL(coverageUrl)
+      this._url = url
+      this._coverageUrl = coverageUrl
+      this._writer.setUrl(url)
+      this._coverageWriter.setUrl(coverageUrl)
+    } catch (e) {
+      log.error(e)
+    }
   }
 }
 

package/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js

@@ -5,16 +5,37 @@ const log = require('../../../log')
 const { AgentlessCiVisibilityEncoder } = require('../../../encode/agentless-ci-visibility')
 const BaseWriter = require('../../../exporters/common/writer')
 
+function safeJSONStringify (value) {
+  return JSON.stringify(value, (key, value) =>
+    key !== 'dd-api-key' ? value : undefined
+  )
+}
+
 class Writer extends BaseWriter {
   constructor ({ url, tags }) {
     super(...arguments)
     const { 'runtime-id': runtimeId, env, service } = tags
     this._url = url
-    this._encoder = new AgentlessCiVisibilityEncoder({ runtimeId, env, service })
+    this._encoder = new AgentlessCiVisibilityEncoder(this, { runtimeId, env, service })
   }
 
   _sendPayload (data, _, done) {
-    makeRequest(data, this._url, (err, res) => {
+    const options = {
+      path: '/api/v2/citestcycle',
+      method: 'POST',
+      headers: {
+        'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY,
+        'Content-Type': 'application/msgpack'
+      },
+      timeout: 15000
+    }
+
+    options.protocol = this._url.protocol
+    options.hostname = this._url.hostname
+    options.port = this._url.port
+
+    log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
+    request(data, options, (err, res) => {
       if (err) {
         log.error(err)
         done()
@@ -26,26 +47,4 @@ class Writer extends BaseWriter {
   }
 }
 
-function makeRequest (data, url, cb) {
-  const options = {
-    path: '/api/v2/citestcycle',
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/msgpack',
-      'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY
-    },
-    timeout: 15000
-  }
-
-  options.protocol = url.protocol
-  options.hostname = url.hostname
-  options.port = url.port
-
-  log.debug(() => `Request to the intake: ${JSON.stringify(options)}`)
-
-  request(data, options, false, (err, res) => {
-    cb(err, res)
-  })
-}
-
 module.exports = Writer

package/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js

@@ -0,0 +1,220 @@
+
+const fs = require('fs')
+const https = require('https')
+const path = require('path')
+
+const FormData = require('../../../exporters/common/form-data')
+
+const log = require('../../../log')
+const {
+  getLatestCommits,
+  getRepositoryUrl,
+  generatePackFilesForCommits,
+  getCommitsToUpload
+} = require('../../../plugins/util/git')
+
+const isValidSha = (sha) => /[0-9a-f]{40}/.test(sha)
+
+function sanitizeCommits (commits) {
+  return commits.map(({ id: commitSha, type }) => {
+    if (type !== 'commit') {
+      throw new Error('Invalid commit response')
+    }
+    const sanitizedCommit = commitSha.replace(/[^0-9a-f]+/g, '')
+    if (sanitizedCommit !== commitSha || !isValidSha(sanitizedCommit)) {
+      throw new Error('Invalid commit format')
+    }
+    return sanitizedCommit
+  })
+}
+
+function getCommonRequestOptions (url) {
+  return {
+    method: 'POST',
+    headers: {
+      'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY
+    },
+    timeout: 15000,
+    protocol: url.protocol,
+    hostname: url.hostname,
+    port: url.port
+  }
+}
+
+/**
+ * This function posts the SHAs of the commits of the last month
+ * The response are the commits for which the backend already has information
+ * This response is used to know which commits can be ignored from there on
+ */
+function getCommitsToExclude ({ url, repositoryUrl }, callback) {
+  const latestCommits = getLatestCommits()
+  const [headCommit] = latestCommits
+
+  const commonOptions = getCommonRequestOptions(url)
+
+  const options = {
+    ...commonOptions,
+    headers: {
+      ...commonOptions.headers,
+      'Content-Type': 'application/json'
+    },
+    path: '/api/v2/git/repository/search_commits'
+  }
+
+  const localCommitData = JSON.stringify({
+    meta: {
+      repository_url: repositoryUrl
+    },
+    data: latestCommits.map(commit => ({
+      id: commit,
+      type: 'commit'
+    }))
+  })
+
+  const request = https.request(options, (res) => {
+    let responseData = ''
+
+    res.on('data', chunk => { responseData += chunk })
+    res.on('end', () => {
+      if (res.statusCode === 200) {
+        let commitsToExclude
+        try {
+          commitsToExclude = sanitizeCommits(JSON.parse(responseData).data)
+        } catch (e) {
+          callback(new Error(`Can't parse response: ${e.message}`))
+          return
+        }
+        callback(null, commitsToExclude, headCommit)
+      } else {
+        const error = new Error(`Error getting commits: ${res.statusCode} ${res.statusMessage}`)
+        callback(error)
+      }
+    })
+  })
+
+  request.write(localCommitData)
+  request.on('error', callback)
+
+  request.end()
+
+  return request
+}
+
+/**
+ * This function uploads a git packfile
+ */
+function uploadPackFile ({ url, packFileToUpload, repositoryUrl, headCommit }, callback) {
+  const form = new FormData()
+
+  const pushedSha = JSON.stringify({
+    data: {
+      id: headCommit,
+      type: 'commit'
+    },
+    meta: {
+      repository_url: repositoryUrl
+    }
+  })
+
+  form.append('pushedSha', pushedSha, { contentType: 'application/json' })
+
+  try {
+    const packFileContent = fs.readFileSync(packFileToUpload)
+    // The original filename includes a random prefix, so we remove it here
+    const [, filename] = path.basename(packFileToUpload).split('-')
+    form.append('packfile', packFileContent, {
+      filename,
+      contentType: 'application/octet-stream'
+    })
+  } catch (e) {
+    callback(new Error(`Error reading packfile: ${packFileToUpload}`))
+    return
+  }
+
+  const commonOptions = getCommonRequestOptions(url)
+
+  const options = {
+    ...commonOptions,
+    path: '/api/v2/git/repository/packfile',
+    headers: {
+      ...commonOptions.headers,
+      ...form.getHeaders()
+    }
+  }
+
+  const req = https.request(options, res => {
+    res.on('data', () => {})
+    res.on('end', () => {
+      if (res.statusCode === 204) {
+        callback(null)
+      } else {
+        const error = new Error(`Error uploading packfiles: ${res.statusCode} ${res.statusMessage}`)
+        error.status = res.statusCode
+
+        callback(error)
+      }
+    })
+  })
+
+  req.on('error', err => {
+    callback(err)
+  })
+  form.pipe(req)
+}
+
+/**
+ * This function uploads git metadata to CI Visibility's backend.
+*/
+function sendGitMetadata (site, callback) {
+  const url = new URL(`https://api.${site}`)
+
+  const repositoryUrl = getRepositoryUrl()
+
+  getCommitsToExclude({ url, repositoryUrl }, (err, commitsToExclude, headCommit) => {
+    if (err) {
+      callback(err)
+      return
+    }
+    const commitsToUpload = getCommitsToUpload(commitsToExclude)
+
+    if (!commitsToUpload.length) {
+      log.debug('No commits to upload')
+      callback(null)
+      return
+    }
+
+    const packFilesToUpload = generatePackFilesForCommits(commitsToUpload)
+
+    let packFileIndex = 0
+    // This uploads packfiles sequentially
+    const uploadPackFileCallback = (err) => {
+      if (err || packFileIndex === packFilesToUpload.length) {
+        callback(err)
+        return
+      }
+      return uploadPackFile(
+        {
+          packFileToUpload: packFilesToUpload[packFileIndex++],
+          url,
+          repositoryUrl,
+          headCommit
+        },
+        uploadPackFileCallback
+      )
+    }
+
+    uploadPackFile(
+      {
+        url,
+        packFileToUpload: packFilesToUpload[packFileIndex++],
+        repositoryUrl,
+        headCommit
+      },
+      uploadPackFileCallback
+    )
+  })
+}
+
+module.exports = {
+  sendGitMetadata
+}

package/packages/dd-trace/src/config.js

@@ -13,6 +13,17 @@ const uuid = require('crypto-randomuuid')
 const fromEntries = Object.fromEntries || (entries =>
   entries.reduce((obj, [k, v]) => Object.assign(obj, { [k]: v }), {}))
 
+// eslint-disable-next-line max-len
+const qsRegex = '(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:(?:\\s|%20)*(?:=|%3D)[^&]+|(?:"|%22)(?:\\s|%20)*(?::|%3A)(?:\\s|%20)*(?:"|%22)(?:%2[^2]|%[^2]|[^"%])+(?:"|%22))|bearer(?:\\s|%20)+[a-z0-9\\._\\-]|token(?::|%3A)[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L](?:[\\w=-]|%3D)+\\.ey[I-L](?:[\\w=-]|%3D)+(?:\\.(?:[\\w.+\\/=-]|%3D|%2F|%2B)+)?|[\\-]{5}BEGIN(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY[\\-]{5}[^\\-]+[\\-]{5}END(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY|ssh-rsa(?:\\s|%20)*(?:[a-z0-9\\/\\.+]|%2F|%5C|%2B){100,}'
+
+function safeJsonParse (input) {
+  try {
+    return JSON.parse(input)
+  } catch (err) {
+    return undefined
+  }
+}
+
 class Config {
   constructor (options) {
     options = options || {}
@@ -67,6 +78,12 @@ class Config {
       null
     )
     const DD_CIVISIBILITY_AGENTLESS_URL = process.env.DD_CIVISIBILITY_AGENTLESS_URL
+
+    const DD_CIVISIBILITY_ITR_ENABLED = coalesce(
+      process.env.DD_CIVISIBILITY_ITR_ENABLED,
+      false
+    )
+
     const DD_SERVICE = options.service ||
       process.env.DD_SERVICE ||
       process.env.DD_SERVICE_NAME ||
@@ -109,6 +126,18 @@ class Config {
       parseInt(process.env.DD_TRACE_PARTIAL_FLUSH_MIN_SPANS),
       1000
     )
+    const DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP = coalesce(
+      process.env.DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP,
+      qsRegex
+    )
+    const DD_TRACE_CLIENT_IP_HEADER_DISABLED = coalesce(
+      process.env.DD_TRACE_CLIENT_IP_HEADER_DISABLED,
+      false
+    )
+    const DD_TRACE_CLIENT_IP_HEADER = coalesce(
+      process.env.DD_TRACE_CLIENT_IP_HEADER,
+      null
+    )
     const DD_TRACE_B3_ENABLED = coalesce(
       options.experimental && options.experimental.b3,
       process.env.DD_TRACE_EXPERIMENTAL_B3_ENABLED,
@@ -134,6 +163,17 @@ class Config {
       false
     )
 
+    const DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH = coalesce(
+      process.env.DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH,
+      '512'
+    )
+
+    const DD_TRACE_STATS_COMPUTATION_ENABLED = coalesce(
+      options.stats,
+      process.env.DD_TRACE_STATS_COMPUTATION_ENABLED,
+      false
+    )
+
     let appsec = options.appsec || (options.experimental && options.experimental.appsec)
 
     const DD_APPSEC_ENABLED = coalesce(
@@ -174,19 +214,51 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
 |[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}`
     )
 
-    const sampler = (options.experimental && options.experimental.sampler) || {}
+    const iastOptions = options.experimental && options.experimental.iast
+    const DD_IAST_ENABLED = coalesce(
+      iastOptions &&
+      (iastOptions === true || iastOptions.enabled === true),
+      process.env.DD_IAST_ENABLED,
+      false
+    )
+
+    const defaultIastRequestSampling = 30
+    const iastRequestSampling = coalesce(
+      parseInt(iastOptions && iastOptions.requestSampling),
+      parseInt(process.env.DD_IAST_REQUEST_SAMPLING),
+      defaultIastRequestSampling
+    )
+    const DD_IAST_REQUEST_SAMPLING = iastRequestSampling < 0 ||
+      iastRequestSampling > 100 ? defaultIastRequestSampling : iastRequestSampling
+
+    const DD_IAST_MAX_CONCURRENT_REQUESTS = coalesce(
+      parseInt(iastOptions && iastOptions.maxConcurrentRequests),
+      parseInt(process.env.DD_IAST_MAX_CONCURRENT_REQUESTS),
+      2
+    )
+
+    const DD_IAST_MAX_CONTEXT_OPERATIONS = coalesce(
+      parseInt(iastOptions && iastOptions.maxContextOperations),
+      parseInt(process.env.DD_IAST_MAX_CONTEXT_OPERATIONS),
+      2
+    )
+
+    const DD_CIVISIBILITY_GIT_UPLOAD_ENABLED = coalesce(
+      process.env.DD_CIVISIBILITY_GIT_UPLOAD_ENABLED,
+      false
+    )
+
     const ingestion = options.ingestion || {}
     const dogstatsd = coalesce(options.dogstatsd, {})
-
-    Object.assign(sampler, {
+    const sampler = {
       sampleRate: coalesce(
         options.sampleRate,
-        ingestion.sampleRate,
-        sampler.sampleRate,
-        process.env.DD_TRACE_SAMPLE_RATE
+        process.env.DD_TRACE_SAMPLE_RATE,
+        ingestion.sampleRate
       ),
-      rateLimit: coalesce(ingestion.rateLimit, sampler.rateLimit, process.env.DD_TRACE_RATE_LIMIT)
-    })
+      rateLimit: coalesce(options.rateLimit, process.env.DD_TRACE_RATE_LIMIT, ingestion.rateLimit),
+      rules: coalesce(options.samplingRules, safeJsonParse(process.env.DD_TRACE_SAMPLING_RULES), [])
+    }
 
     const inAWSLambda = process.env.AWS_LAMBDA_FUNCTION_NAME !== undefined
     const defaultFlushInterval = inAWSLambda ? 0 : 2000
@@ -203,6 +275,9 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
     this.flushInterval = coalesce(parseInt(options.flushInterval, 10), defaultFlushInterval)
     this.flushMinSpans = DD_TRACE_PARTIAL_FLUSH_MIN_SPANS
     this.sampleRate = coalesce(Math.min(Math.max(sampler.sampleRate, 0), 1), 1)
+    this.queryStringObfuscation = DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP
+    this.clientIpHeaderDisabled = isTrue(DD_TRACE_CLIENT_IP_HEADER_DISABLED)
+    this.clientIpHeader = DD_TRACE_CLIENT_IP_HEADER
     this.logger = options.logger
     this.plugins = !!coalesce(options.plugins, true)
     this.service = DD_SERVICE
@@ -220,9 +295,9 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       traceparent: isTrue(DD_TRACE_TRACEPARENT_ENABLED),
       runtimeId: isTrue(DD_TRACE_RUNTIME_ID_ENABLED),
       exporter: DD_TRACE_EXPORTER,
-      enableGetRumData: isTrue(DD_TRACE_GET_RUM_DATA_ENABLED),
-      sampler
+      enableGetRumData: isTrue(DD_TRACE_GET_RUM_DATA_ENABLED)
     }
+    this.sampler = sampler
     this.reportHostname = isTrue(coalesce(options.reportHostname, process.env.DD_TRACE_REPORT_HOSTNAME, false))
     this.scope = process.env.DD_TRACE_SCOPE
     this.logLevel = coalesce(
@@ -240,6 +315,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
     // Disabled for CI Visibility's agentless
     this.telemetryEnabled = DD_TRACE_EXPORTER !== 'datadog' && isTrue(DD_TRACE_TELEMETRY_ENABLED)
     this.protocolVersion = DD_TRACE_AGENT_PROTOCOL_VERSION
+    this.tagsHeaderMaxLength = parseInt(DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH)
     this.appsec = {
       enabled: isTrue(DD_APPSEC_ENABLED),
       rules: DD_APPSEC_RULES,
@@ -248,6 +324,17 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       obfuscatorKeyRegex: DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP,
       obfuscatorValueRegex: DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP
     }
+    this.iast = {
+      enabled: isTrue(DD_IAST_ENABLED),
+      requestSampling: DD_IAST_REQUEST_SAMPLING,
+      maxConcurrentRequests: DD_IAST_MAX_CONCURRENT_REQUESTS,
+      maxContextOperations: DD_IAST_MAX_CONTEXT_OPERATIONS
+    }
+    this.isGitUploadEnabled = isTrue(DD_CIVISIBILITY_GIT_UPLOAD_ENABLED)
+    this.isIntelligentTestRunnerEnabled = isTrue(DD_CIVISIBILITY_ITR_ENABLED)
+    this.stats = {
+      enabled: isTrue(DD_TRACE_STATS_COMPUTATION_ENABLED)
+    }
 
     tagger.add(this.tags, {
       service: this.service,

package/packages/dd-trace/src/constants.js

@@ -5,8 +5,16 @@ module.exports = {
   ANALYTICS_KEY: '_dd1.sr.eausr',
   ORIGIN_KEY: '_dd.origin',
   HOSTNAME_KEY: '_dd.hostname',
+  TOP_LEVEL_KEY: '_dd.top_level',
   SAMPLING_RULE_DECISION: '_dd.rule_psr',
   SAMPLING_LIMIT_DECISION: '_dd.limit_psr',
   SAMPLING_AGENT_DECISION: '_dd.agent_psr',
-  DATADOG_LAMBDA_EXTENSION_PATH: '/opt/extensions/datadog-agent'
+  SAMPLING_MECHANISM_DEFAULT: 0,
+  SAMPLING_MECHANISM_AGENT: 1,
+  SAMPLING_MECHANISM_RULE: 3,
+  SAMPLING_MECHANISM_MANUAL: 4,
+  SAMPLING_MECHANISM_APPSEC: 5,
+  SAMPLING_MECHANISM_SPAN: 8,
+  DATADOG_LAMBDA_EXTENSION_PATH: '/opt/extensions/datadog-agent',
+  DECISION_MAKER_KEY: '_dd.p.dm'
 }