dd-trace 2.12.2 → 2.15.0

package/packages/dd-trace/src/appsec/iast/overhead-controller.js

@@ -0,0 +1,94 @@
+'use strict'
+
+const OVERHEAD_CONTROLLER_CONTEXT_KEY = 'oce'
+const REPORT_VULNERABILITY = 'REPORT_VULNERABILITY'
+
+const GLOBAL_OCE_CONTEXT = {}
+let config = {}
+let availableRequest = 0
+const OPERATIONS = {
+  REPORT_VULNERABILITY: {
+    hasQuota: (context) => {
+      const reserved = context && context.tokens && context.tokens[REPORT_VULNERABILITY] > 0
+      if (reserved) {
+        context.tokens[REPORT_VULNERABILITY]--
+      }
+      return reserved
+    },
+    name: REPORT_VULNERABILITY,
+    initialTokenBucketSize () {
+      return typeof config.maxContextOperations === 'number' ? config.maxContextOperations : 2
+    },
+    initContext: function (context) {
+      context.tokens[REPORT_VULNERABILITY] = this.initialTokenBucketSize()
+    }
+  }
+}
+
+function _getNewContext () {
+  const oceContext = {
+    tokens: {}
+  }
+
+  for (const operation in OPERATIONS) {
+    OPERATIONS[operation].initContext(oceContext)
+  }
+
+  return oceContext
+}
+
+function _getContext (iastContext) {
+  if (iastContext && iastContext[OVERHEAD_CONTROLLER_CONTEXT_KEY]) {
+    return iastContext[OVERHEAD_CONTROLLER_CONTEXT_KEY]
+  }
+  return GLOBAL_OCE_CONTEXT
+}
+
+function _resetGlobalContext () {
+  Object.assign(GLOBAL_OCE_CONTEXT, _getNewContext())
+}
+
+function acquireRequest (rootSpan) {
+  if (availableRequest > 0) {
+    const sampling = config && typeof config.requestSampling === 'number'
+      ? config.requestSampling : 30
+    if (rootSpan.context().toSpanId().slice(-2) <= sampling) {
+      availableRequest--
+      return true
+    }
+  }
+  return false
+}
+
+function releaseRequest () {
+  if (availableRequest < config.maxConcurrentRequests) {
+    availableRequest++
+  }
+}
+
+function hasQuota (operation, iastContext) {
+  const oceContext = _getContext(iastContext)
+  return operation.hasQuota(oceContext)
+}
+
+function initializeRequestContext (iastContext) {
+  if (iastContext) iastContext[OVERHEAD_CONTROLLER_CONTEXT_KEY] = _getNewContext()
+}
+
+function configure (cfg) {
+  config = cfg
+  availableRequest = config.maxConcurrentRequests
+}
+
+_resetGlobalContext()
+
+module.exports = {
+  OVERHEAD_CONTROLLER_CONTEXT_KEY,
+  OPERATIONS,
+  _resetGlobalContext,
+  initializeRequestContext,
+  hasQuota,
+  acquireRequest,
+  releaseRequest,
+  configure
+}

package/packages/dd-trace/src/appsec/iast/path-line.js

@@ -0,0 +1,70 @@
+const path = require('path')
+const pathLine = {
+  getFirstNonDDPathAndLine,
+  getFirstNonDDPathAndLineFromCallsites, // Exported only for test purposes
+  calculateDDBasePath, // Exported only for test purposes
+  ddBasePath: calculateDDBasePath(__dirname) // Only for test purposes
+}
+
+const EXCLUDED_PATHS = [
+  '/node_modules/diagnostics_channel'
+]
+const EXCLUDED_PATH_PREFIXES = [
+  'node:diagnostics_channel',
+  'diagnostics_channel'
+]
+
+function calculateDDBasePath (dirname) {
+  const dirSteps = dirname.split(path.sep)
+  const packagesIndex = dirSteps.indexOf('packages')
+  return dirSteps.slice(0, packagesIndex).join(path.sep) + path.sep
+}
+
+function getCallSiteInfo () {
+  const previousPrepareStackTrace = Error.prepareStackTrace
+  let callsiteList
+  Error.prepareStackTrace = function (_, callsites) {
+    callsiteList = callsites
+  }
+  const e = new Error()
+  e.stack
+  Error.prepareStackTrace = previousPrepareStackTrace
+  return callsiteList
+}
+
+function getFirstNonDDPathAndLineFromCallsites (callsites) {
+  if (callsites) {
+    for (let i = 0; i < callsites.length; i++) {
+      const callsite = callsites[i]
+      const path = callsite.getFileName()
+      if (!isExcluded(callsite) && path.indexOf(pathLine.ddBasePath) === -1) {
+        return {
+          path,
+          line: callsite.getLineNumber()
+        }
+      }
+    }
+  }
+  return null
+}
+
+function isExcluded (callsite) {
+  if (callsite.isNative()) return true
+  const filename = callsite.getFileName()
+  for (let i = 0; i < EXCLUDED_PATHS.length; i++) {
+    if (filename.indexOf(EXCLUDED_PATHS[i]) > -1) {
+      return true
+    }
+  }
+  for (let i = 0; i < EXCLUDED_PATH_PREFIXES.length; i++) {
+    if (filename.indexOf(EXCLUDED_PATH_PREFIXES[i]) === 0) {
+      return true
+    }
+  }
+  return false
+}
+
+function getFirstNonDDPathAndLine () {
+  return getFirstNonDDPathAndLineFromCallsites(getCallSiteInfo())
+}
+module.exports = pathLine

package/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js

@@ -0,0 +1,113 @@
+const { MANUAL_KEEP } = require('../../../../../ext/tags')
+const VULNERABILITIES_KEY = 'vulnerabilities'
+const IAST_JSON_TAG_KEY = '_dd.iast.json'
+
+function createVulnerability (type, evidence, spanId, location) {
+  if (type && evidence && spanId) {
+    return {
+      type,
+      evidence,
+      location: {
+        spanId,
+        ...location
+      },
+      hash: createHash(type, location)
+    }
+  }
+  return null
+}
+
+function createHash (type, location) {
+  let hashSource
+  if (location) {
+    hashSource = `${type}:${location.path}:${location.line}`
+  } else {
+    hashSource = type
+  }
+  let hash = 0
+  let offset = 0
+  const size = hashSource.length
+  for (let i = 0; i < size; i++) {
+    hash = ((hash << 5) - hash) + hashSource.charCodeAt(offset++)
+  }
+  return hash
+}
+
+function addVulnerability (iastContext, vulnerability) {
+  if (iastContext && vulnerability && vulnerability.evidence && vulnerability.type &&
+    vulnerability.location && vulnerability.location.spanId) {
+    iastContext[VULNERABILITIES_KEY] = iastContext[VULNERABILITIES_KEY] || []
+    iastContext[VULNERABILITIES_KEY].push(vulnerability)
+  }
+}
+
+function isValidVulnerability (vulnerability) {
+  return vulnerability && vulnerability.type &&
+    vulnerability.evidence && vulnerability.evidence.value &&
+    vulnerability.location && vulnerability.location.spanId
+}
+
+function jsonVulnerabilityFromVulnerability (vulnerability) {
+  const jsonVulnerability = {
+    type: vulnerability.type,
+    hash: vulnerability.hash,
+    evidence: {
+      value: vulnerability.evidence.value
+    },
+    location: {
+      spanId: vulnerability.location.spanId
+    }
+  }
+  if (vulnerability.location.path) {
+    jsonVulnerability.location.path = vulnerability.location.path
+  }
+  if (vulnerability.location.line) {
+    jsonVulnerability.location.line = vulnerability.location.line
+  }
+  return jsonVulnerability
+}
+
+function sendVulnerabilities (iastContext) {
+  if (iastContext && iastContext.rootSpan && iastContext[VULNERABILITIES_KEY] &&
+    iastContext[VULNERABILITIES_KEY].length && iastContext.rootSpan.addTags) {
+    const span = iastContext.rootSpan
+    const allVulnerabilities = iastContext[VULNERABILITIES_KEY]
+    // TODO support sources and ranges
+    const jsonToSend = {
+      vulnerabilities: []
+    }
+
+    deduplicateVulnerabilities(allVulnerabilities).forEach((vulnerability) => {
+      if (isValidVulnerability(vulnerability)) {
+        jsonToSend.vulnerabilities.push(jsonVulnerabilityFromVulnerability(vulnerability))
+      }
+    })
+
+    if (jsonToSend.vulnerabilities.length > 0) {
+      const tags = {}
+      // TODO: Store this outside of the span and set the tag in the exporter.
+      tags[IAST_JSON_TAG_KEY] = JSON.stringify(jsonToSend)
+      tags[MANUAL_KEEP] = 'true'
+      span.addTags(tags)
+    }
+  }
+  return IAST_JSON_TAG_KEY
+}
+
+function deduplicateVulnerabilities (vulnerabilities) {
+  const uniqueVulnerabilities = new Set()
+  return vulnerabilities.filter((vulnerability) => {
+    const key = `${vulnerability.type}${vulnerability.hash}`
+    if (!uniqueVulnerabilities.has(key)) {
+      uniqueVulnerabilities.add(key)
+      return true
+    }
+    return false
+  })
+}
+
+module.exports = {
+  createVulnerability,
+  addVulnerability,
+  sendVulnerabilities
+}

package/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js

@@ -0,0 +1,50 @@
+'use strict'
+const request = require('../../../exporters/common/request')
+const log = require('../../../log')
+
+const { CoverageCIVisibilityEncoder } = require('../../../encode/coverage-ci-visibility')
+const BaseWriter = require('../../../exporters/common/writer')
+
+function safeJSONStringify (value) {
+  return JSON.stringify(value, (key, value) =>
+    key !== 'dd-api-key' ? value : undefined
+  )
+}
+
+class Writer extends BaseWriter {
+  constructor ({ url }) {
+    super(...arguments)
+    this._url = url
+    this._encoder = new CoverageCIVisibilityEncoder()
+  }
+
+  _sendPayload (form, _, done) {
+    const options = {
+      path: '/api/v2/citestcov',
+      method: 'POST',
+      headers: {
+        'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY,
+        ...form.getHeaders()
+      },
+      timeout: 15000
+    }
+
+    options.protocol = this._url.protocol
+    options.hostname = this._url.hostname
+    options.port = this._url.port
+
+    log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
+
+    request(form, options, (err, res) => {
+      if (err) {
+        log.error(err)
+        done()
+        return
+      }
+      log.debug(`Response from the intake: ${res}`)
+      done()
+    })
+  }
+}
+
+module.exports = Writer

package/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js

@@ -2,30 +2,75 @@
 
 const URL = require('url').URL
 const Writer = require('./writer')
-const Scheduler = require('../../../exporters/scheduler')
+const CoverageWriter = require('./coverage-writer')
+
+const log = require('../../../log')
 
 class AgentlessCiVisibilityExporter {
   constructor (config) {
-    const { flushInterval, tags, site, url } = config
+    this._config = config
+    const { tags, site, url, isIntelligentTestRunnerEnabled } = config
+    this._isIntelligentTestRunnerEnabled = isIntelligentTestRunnerEnabled
     this._url = url || new URL(`https://citestcycle-intake.${site}`)
     this._writer = new Writer({ url: this._url, tags })
+    this._timer = undefined
+    this._coverageTimer = undefined
+
+    this._coverageUrl = url || new URL(`https://event-platform-intake.${site}`)
+    this._coverageWriter = new CoverageWriter({ url: this._coverageUrl })
+
+    process.once('beforeExit', () => {
+      this._writer.flush()
+      this._coverageWriter.flush()
+    })
+  }
 
-    if (flushInterval > 0) {
-      this._scheduler = new Scheduler(() => this._writer.flush(), flushInterval)
+  exportCoverage ({ testSpan, coverageFiles }) {
+    const formattedCoverage = {
+      traceId: testSpan.context()._traceId,
+      spanId: testSpan.context()._spanId,
+      files: coverageFiles
+    }
+    this._coverageWriter.append(formattedCoverage)
+
+    const { flushInterval } = this._config
+
+    if (flushInterval === 0) {
+      this._coverageWriter.flush()
+    } else if (flushInterval > 0 && !this._coverageTimer) {
+      this._coverageTimer = setTimeout(() => {
+        this._coverageWriter.flush()
+        this._coverageTimer = clearTimeout(this._coverageTimer)
+      }, flushInterval).unref()
     }
-    this._scheduler && this._scheduler.start()
   }
 
   export (trace) {
     this._writer.append(trace)
 
-    if (!this._scheduler) {
+    const { flushInterval } = this._config
+
+    if (flushInterval === 0) {
       this._writer.flush()
+    } else if (flushInterval > 0 && !this._timer) {
+      this._timer = setTimeout(() => {
+        this._writer.flush()
+        this._timer = clearTimeout(this._timer)
+      }, flushInterval).unref()
     }
   }
 
-  flush () {
-    this._writer.flush()
+  setUrl (url, coverageUrl = url) {
+    try {
+      url = new URL(url)
+      coverageUrl = new URL(coverageUrl)
+      this._url = url
+      this._coverageUrl = coverageUrl
+      this._writer.setUrl(url)
+      this._coverageWriter.setUrl(coverageUrl)
+    } catch (e) {
+      log.error(e)
+    }
   }
 }
 

package/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js

@@ -5,16 +5,37 @@ const log = require('../../../log')
 const { AgentlessCiVisibilityEncoder } = require('../../../encode/agentless-ci-visibility')
 const BaseWriter = require('../../../exporters/common/writer')
 
+function safeJSONStringify (value) {
+  return JSON.stringify(value, (key, value) =>
+    key !== 'dd-api-key' ? value : undefined
+  )
+}
+
 class Writer extends BaseWriter {
   constructor ({ url, tags }) {
     super(...arguments)
     const { 'runtime-id': runtimeId, env, service } = tags
     this._url = url
-    this._encoder = new AgentlessCiVisibilityEncoder({ runtimeId, env, service })
+    this._encoder = new AgentlessCiVisibilityEncoder(this, { runtimeId, env, service })
   }
 
   _sendPayload (data, _, done) {
-    makeRequest(data, this._url, (err, res) => {
+    const options = {
+      path: '/api/v2/citestcycle',
+      method: 'POST',
+      headers: {
+        'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY,
+        'Content-Type': 'application/msgpack'
+      },
+      timeout: 15000
+    }
+
+    options.protocol = this._url.protocol
+    options.hostname = this._url.hostname
+    options.port = this._url.port
+
+    log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
+    request(data, options, (err, res) => {
       if (err) {
         log.error(err)
         done()
@@ -26,26 +47,4 @@ class Writer extends BaseWriter {
   }
 }
 
-function makeRequest (data, url, cb) {
-  const options = {
-    path: '/api/v2/citestcycle',
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/msgpack',
-      'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY
-    },
-    timeout: 15000
-  }
-
-  options.protocol = url.protocol
-  options.hostname = url.hostname
-  options.port = url.port
-
-  log.debug(() => `Request to the intake: ${JSON.stringify(options)}`)
-
-  request(data, options, false, (err, res) => {
-    cb(err, res)
-  })
-}
-
 module.exports = Writer

package/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js

@@ -0,0 +1,220 @@
+
+const fs = require('fs')
+const https = require('https')
+const path = require('path')
+
+const FormData = require('../../../exporters/common/form-data')
+
+const log = require('../../../log')
+const {
+  getLatestCommits,
+  getRepositoryUrl,
+  generatePackFilesForCommits,
+  getCommitsToUpload
+} = require('../../../plugins/util/git')
+
+const isValidSha = (sha) => /[0-9a-f]{40}/.test(sha)
+
+function sanitizeCommits (commits) {
+  return commits.map(({ id: commitSha, type }) => {
+    if (type !== 'commit') {
+      throw new Error('Invalid commit response')
+    }
+    const sanitizedCommit = commitSha.replace(/[^0-9a-f]+/g, '')
+    if (sanitizedCommit !== commitSha || !isValidSha(sanitizedCommit)) {
+      throw new Error('Invalid commit format')
+    }
+    return sanitizedCommit
+  })
+}
+
+function getCommonRequestOptions (url) {
+  return {
+    method: 'POST',
+    headers: {
+      'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY
+    },
+    timeout: 15000,
+    protocol: url.protocol,
+    hostname: url.hostname,
+    port: url.port
+  }
+}
+
+/**
+ * This function posts the SHAs of the commits of the last month
+ * The response are the commits for which the backend already has information
+ * This response is used to know which commits can be ignored from there on
+ */
+function getCommitsToExclude ({ url, repositoryUrl }, callback) {
+  const latestCommits = getLatestCommits()
+  const [headCommit] = latestCommits
+
+  const commonOptions = getCommonRequestOptions(url)
+
+  const options = {
+    ...commonOptions,
+    headers: {
+      ...commonOptions.headers,
+      'Content-Type': 'application/json'
+    },
+    path: '/api/v2/git/repository/search_commits'
+  }
+
+  const localCommitData = JSON.stringify({
+    meta: {
+      repository_url: repositoryUrl
+    },
+    data: latestCommits.map(commit => ({
+      id: commit,
+      type: 'commit'
+    }))
+  })
+
+  const request = https.request(options, (res) => {
+    let responseData = ''
+
+    res.on('data', chunk => { responseData += chunk })
+    res.on('end', () => {
+      if (res.statusCode === 200) {
+        let commitsToExclude
+        try {
+          commitsToExclude = sanitizeCommits(JSON.parse(responseData).data)
+        } catch (e) {
+          callback(new Error(`Can't parse response: ${e.message}`))
+          return
+        }
+        callback(null, commitsToExclude, headCommit)
+      } else {
+        const error = new Error(`Error getting commits: ${res.statusCode} ${res.statusMessage}`)
+        callback(error)
+      }
+    })
+  })
+
+  request.write(localCommitData)
+  request.on('error', callback)
+
+  request.end()
+
+  return request
+}
+
+/**
+ * This function uploads a git packfile
+ */
+function uploadPackFile ({ url, packFileToUpload, repositoryUrl, headCommit }, callback) {
+  const form = new FormData()
+
+  const pushedSha = JSON.stringify({
+    data: {
+      id: headCommit,
+      type: 'commit'
+    },
+    meta: {
+      repository_url: repositoryUrl
+    }
+  })
+
+  form.append('pushedSha', pushedSha, { contentType: 'application/json' })
+
+  try {
+    const packFileContent = fs.readFileSync(packFileToUpload)
+    // The original filename includes a random prefix, so we remove it here
+    const [, filename] = path.basename(packFileToUpload).split('-')
+    form.append('packfile', packFileContent, {
+      filename,
+      contentType: 'application/octet-stream'
+    })
+  } catch (e) {
+    callback(new Error(`Error reading packfile: ${packFileToUpload}`))
+    return
+  }
+
+  const commonOptions = getCommonRequestOptions(url)
+
+  const options = {
+    ...commonOptions,
+    path: '/api/v2/git/repository/packfile',
+    headers: {
+      ...commonOptions.headers,
+      ...form.getHeaders()
+    }
+  }
+
+  const req = https.request(options, res => {
+    res.on('data', () => {})
+    res.on('end', () => {
+      if (res.statusCode === 204) {
+        callback(null)
+      } else {
+        const error = new Error(`Error uploading packfiles: ${res.statusCode} ${res.statusMessage}`)
+        error.status = res.statusCode
+
+        callback(error)
+      }
+    })
+  })
+
+  req.on('error', err => {
+    callback(err)
+  })
+  form.pipe(req)
+}
+
+/**
+ * This function uploads git metadata to CI Visibility's backend.
+*/
+function sendGitMetadata (site, callback) {
+  const url = new URL(`https://api.${site}`)
+
+  const repositoryUrl = getRepositoryUrl()
+
+  getCommitsToExclude({ url, repositoryUrl }, (err, commitsToExclude, headCommit) => {
+    if (err) {
+      callback(err)
+      return
+    }
+    const commitsToUpload = getCommitsToUpload(commitsToExclude)
+
+    if (!commitsToUpload.length) {
+      log.debug('No commits to upload')
+      callback(null)
+      return
+    }
+
+    const packFilesToUpload = generatePackFilesForCommits(commitsToUpload)
+
+    let packFileIndex = 0
+    // This uploads packfiles sequentially
+    const uploadPackFileCallback = (err) => {
+      if (err || packFileIndex === packFilesToUpload.length) {
+        callback(err)
+        return
+      }
+      return uploadPackFile(
+        {
+          packFileToUpload: packFilesToUpload[packFileIndex++],
+          url,
+          repositoryUrl,
+          headCommit
+        },
+        uploadPackFileCallback
+      )
+    }
+
+    uploadPackFile(
+      {
+        url,
+        packFileToUpload: packFilesToUpload[packFileIndex++],
+        repositoryUrl,
+        headCommit
+      },
+      uploadPackFileCallback
+    )
+  })
+}
+
+module.exports = {
+  sendGitMetadata
+}