dd-trace 2.12.1 → 2.14.0

package/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js

@@ -0,0 +1,220 @@
+
+const fs = require('fs')
+const https = require('https')
+const path = require('path')
+
+const FormData = require('../../../exporters/common/form-data')
+
+const log = require('../../../log')
+const {
+  getLatestCommits,
+  getRepositoryUrl,
+  generatePackFilesForCommits,
+  getCommitsToUpload
+} = require('../../../plugins/util/git')
+
+const isValidSha = (sha) => /[0-9a-f]{40}/.test(sha)
+
+function sanitizeCommits (commits) {
+  return commits.map(({ id: commitSha, type }) => {
+    if (type !== 'commit') {
+      throw new Error('Invalid commit response')
+    }
+    const sanitizedCommit = commitSha.replace(/[^0-9a-f]+/g, '')
+    if (sanitizedCommit !== commitSha || !isValidSha(sanitizedCommit)) {
+      throw new Error('Invalid commit format')
+    }
+    return sanitizedCommit
+  })
+}
+
+function getCommonRequestOptions (url) {
+  return {
+    method: 'POST',
+    headers: {
+      'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY
+    },
+    timeout: 15000,
+    protocol: url.protocol,
+    hostname: url.hostname,
+    port: url.port
+  }
+}
+
+/**
+ * This function posts the SHAs of the commits of the last month
+ * The response are the commits for which the backend already has information
+ * This response is used to know which commits can be ignored from there on
+ */
+function getCommitsToExclude ({ url, repositoryUrl }, callback) {
+  const latestCommits = getLatestCommits()
+  const [headCommit] = latestCommits
+
+  const commonOptions = getCommonRequestOptions(url)
+
+  const options = {
+    ...commonOptions,
+    headers: {
+      ...commonOptions.headers,
+      'Content-Type': 'application/json'
+    },
+    path: '/api/v2/git/repository/search_commits'
+  }
+
+  const localCommitData = JSON.stringify({
+    meta: {
+      repository_url: repositoryUrl
+    },
+    data: latestCommits.map(commit => ({
+      id: commit,
+      type: 'commit'
+    }))
+  })
+
+  const request = https.request(options, (res) => {
+    let responseData = ''
+
+    res.on('data', chunk => { responseData += chunk })
+    res.on('end', () => {
+      if (res.statusCode === 200) {
+        let commitsToExclude
+        try {
+          commitsToExclude = sanitizeCommits(JSON.parse(responseData).data)
+        } catch (e) {
+          callback(new Error(`Can't parse response: ${e.message}`))
+          return
+        }
+        callback(null, commitsToExclude, headCommit)
+      } else {
+        const error = new Error(`Error getting commits: ${res.statusCode} ${res.statusMessage}`)
+        callback(error)
+      }
+    })
+  })
+
+  request.write(localCommitData)
+  request.on('error', callback)
+
+  request.end()
+
+  return request
+}
+
+/**
+ * This function uploads a git packfile
+ */
+function uploadPackFile ({ url, packFileToUpload, repositoryUrl, headCommit }, callback) {
+  const form = new FormData()
+
+  const pushedSha = JSON.stringify({
+    data: {
+      id: headCommit,
+      type: 'commit'
+    },
+    meta: {
+      repository_url: repositoryUrl
+    }
+  })
+
+  form.append('pushedSha', pushedSha, { contentType: 'application/json' })
+
+  try {
+    const packFileContent = fs.readFileSync(packFileToUpload)
+    // The original filename includes a random prefix, so we remove it here
+    const [, filename] = path.basename(packFileToUpload).split('-')
+    form.append('packfile', packFileContent, {
+      filename,
+      contentType: 'application/octet-stream'
+    })
+  } catch (e) {
+    callback(new Error(`Error reading packfile: ${packFileToUpload}`))
+    return
+  }
+
+  const commonOptions = getCommonRequestOptions(url)
+
+  const options = {
+    ...commonOptions,
+    path: '/api/v2/git/repository/packfile',
+    headers: {
+      ...commonOptions.headers,
+      ...form.getHeaders()
+    }
+  }
+
+  const req = https.request(options, res => {
+    res.on('data', () => {})
+    res.on('end', () => {
+      if (res.statusCode === 204) {
+        callback(null)
+      } else {
+        const error = new Error(`Error uploading packfiles: ${res.statusCode} ${res.statusMessage}`)
+        error.status = res.statusCode
+
+        callback(error)
+      }
+    })
+  })
+
+  req.on('error', err => {
+    callback(err)
+  })
+  form.pipe(req)
+}
+
+/**
+ * This function uploads git metadata to CI Visibility's backend.
+*/
+function sendGitMetadata (site, callback) {
+  const url = new URL(`https://api.${site}`)
+
+  const repositoryUrl = getRepositoryUrl()
+
+  getCommitsToExclude({ url, repositoryUrl }, (err, commitsToExclude, headCommit) => {
+    if (err) {
+      callback(err)
+      return
+    }
+    const commitsToUpload = getCommitsToUpload(commitsToExclude)
+
+    if (!commitsToUpload.length) {
+      log.debug('No commits to upload')
+      callback(null)
+      return
+    }
+
+    const packFilesToUpload = generatePackFilesForCommits(commitsToUpload)
+
+    let packFileIndex = 0
+    // This uploads packfiles sequentially
+    const uploadPackFileCallback = (err) => {
+      if (err || packFileIndex === packFilesToUpload.length) {
+        callback(err)
+        return
+      }
+      return uploadPackFile(
+        {
+          packFileToUpload: packFilesToUpload[packFileIndex++],
+          url,
+          repositoryUrl,
+          headCommit
+        },
+        uploadPackFileCallback
+      )
+    }
+
+    uploadPackFile(
+      {
+        url,
+        packFileToUpload: packFilesToUpload[packFileIndex++],
+        repositoryUrl,
+        headCommit
+      },
+      uploadPackFileCallback
+    )
+  })
+}
+
+module.exports = {
+  sendGitMetadata
+}

package/packages/dd-trace/src/config.js

@@ -67,6 +67,12 @@ class Config {
       null
     )
     const DD_CIVISIBILITY_AGENTLESS_URL = process.env.DD_CIVISIBILITY_AGENTLESS_URL
+
+    const DD_CIVISIBILITY_ITR_ENABLED = coalesce(
+      process.env.DD_CIVISIBILITY_ITR_ENABLED,
+      false
+    )
+
     const DD_SERVICE = options.service ||
       process.env.DD_SERVICE ||
       process.env.DD_SERVICE_NAME ||
@@ -174,6 +180,11 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
 |[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}`
     )
 
+    const DD_CIVISIBILITY_GIT_UPLOAD_ENABLED = coalesce(
+      process.env.DD_CIVISIBILITY_GIT_UPLOAD_ENABLED,
+      false
+    )
+
     const sampler = (options.experimental && options.experimental.sampler) || {}
     const ingestion = options.ingestion || {}
     const dogstatsd = coalesce(options.dogstatsd, {})
@@ -248,6 +259,8 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       obfuscatorKeyRegex: DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP,
       obfuscatorValueRegex: DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP
     }
+    this.isGitUploadEnabled = isTrue(DD_CIVISIBILITY_GIT_UPLOAD_ENABLED)
+    this.isIntelligentTestRunnerEnabled = isTrue(DD_CIVISIBILITY_ITR_ENABLED)
 
     tagger.add(this.tags, {
       service: this.service,

package/packages/dd-trace/src/encode/0.4.js

@@ -13,7 +13,8 @@ float64Array[0] = -1
 const bigEndian = uInt8Float64Array[7] === 0
 
 class AgentEncoder {
-  constructor (writer) {
+  constructor (writer, limit = SOFT_LIMIT) {
+    this._limit = limit
     this._traceBytes = new Chunk()
     this._stringBytes = new Chunk()
     this._writer = writer
@@ -41,7 +42,8 @@ class AgentEncoder {
     })
 
     // we can go over the soft limit since the agent has a 50MB hard limit
-    if (this._traceBytes.length > SOFT_LIMIT || this._stringBytes.length > SOFT_LIMIT) {
+    if (this._traceBytes.length > this._limit || this._stringBytes.length > this._limit) {
+      log.debug('Buffer went over soft limit, flushing')
       this._writer.flush()
     }
   }
@@ -57,6 +59,10 @@ class AgentEncoder {
     return buffer
   }
 
+  reset () {
+    this._reset()
+  }
+
   _encode (bytes, trace) {
     this._encodeArrayPrefix(bytes, trace)
 
@@ -109,29 +115,37 @@ class AgentEncoder {
 
   _encodeArrayPrefix (bytes, value) {
     const length = value.length
-    const buffer = bytes.buffer
     const offset = bytes.length
 
     bytes.reserve(5)
     bytes.length += 5
 
-    buffer[offset] = 0xdd
-    buffer[offset + 1] = length >> 24
-    buffer[offset + 2] = length >> 16
-    buffer[offset + 3] = length >> 8
-    buffer[offset + 4] = length
+    bytes.buffer[offset] = 0xdd
+    bytes.buffer[offset + 1] = length >> 24
+    bytes.buffer[offset + 2] = length >> 16
+    bytes.buffer[offset + 3] = length >> 8
+    bytes.buffer[offset + 4] = length
   }
 
-  _encodeByte (bytes, value) {
-    const buffer = bytes.buffer
+  _encodeMapPrefix (bytes, keysLength) {
+    const offset = bytes.length
+
+    bytes.reserve(5)
+    bytes.length += 5
+    bytes.buffer[offset] = 0xdf
+    bytes.buffer[offset + 1] = keysLength >> 24
+    bytes.buffer[offset + 2] = keysLength >> 16
+    bytes.buffer[offset + 3] = keysLength >> 8
+    bytes.buffer[offset + 4] = keysLength
+  }
 
+  _encodeByte (bytes, value) {
     bytes.reserve(1)
 
-    buffer[bytes.length++] = value
+    bytes.buffer[bytes.length++] = value
   }
 
   _encodeId (bytes, id) {
-    const buffer = bytes.buffer
     const offset = bytes.length
 
     bytes.reserve(9)
@@ -139,33 +153,31 @@ class AgentEncoder {
 
     id = id.toArray()
 
-    buffer[offset] = 0xcf
-    buffer[offset + 1] = id[0]
-    buffer[offset + 2] = id[1]
-    buffer[offset + 3] = id[2]
-    buffer[offset + 4] = id[3]
-    buffer[offset + 5] = id[4]
-    buffer[offset + 6] = id[5]
-    buffer[offset + 7] = id[6]
-    buffer[offset + 8] = id[7]
+    bytes.buffer[offset] = 0xcf
+    bytes.buffer[offset + 1] = id[0]
+    bytes.buffer[offset + 2] = id[1]
+    bytes.buffer[offset + 3] = id[2]
+    bytes.buffer[offset + 4] = id[3]
+    bytes.buffer[offset + 5] = id[4]
+    bytes.buffer[offset + 6] = id[5]
+    bytes.buffer[offset + 7] = id[6]
+    bytes.buffer[offset + 8] = id[7]
   }
 
   _encodeInteger (bytes, value) {
-    const buffer = bytes.buffer
     const offset = bytes.length
 
     bytes.reserve(5)
     bytes.length += 5
 
-    buffer[offset] = 0xce
-    buffer[offset + 1] = value >> 24
-    buffer[offset + 2] = value >> 16
-    buffer[offset + 3] = value >> 8
-    buffer[offset + 4] = value
+    bytes.buffer[offset] = 0xce
+    bytes.buffer[offset + 1] = value >> 24
+    bytes.buffer[offset + 2] = value >> 16
+    bytes.buffer[offset + 3] = value >> 8
+    bytes.buffer[offset + 4] = value
   }
 
   _encodeLong (bytes, value) {
-    const buffer = bytes.buffer
     const offset = bytes.length
     const hi = (value / Math.pow(2, 32)) >> 0
     const lo = value >>> 0
@@ -173,40 +185,27 @@ class AgentEncoder {
     bytes.reserve(9)
     bytes.length += 9
 
-    buffer[offset] = 0xcf
-    buffer[offset + 1] = hi >> 24
-    buffer[offset + 2] = hi >> 16
-    buffer[offset + 3] = hi >> 8
-    buffer[offset + 4] = hi
-    buffer[offset + 5] = lo >> 24
-    buffer[offset + 6] = lo >> 16
-    buffer[offset + 7] = lo >> 8
-    buffer[offset + 8] = lo
+    bytes.buffer[offset] = 0xcf
+    bytes.buffer[offset + 1] = hi >> 24
+    bytes.buffer[offset + 2] = hi >> 16
+    bytes.buffer[offset + 3] = hi >> 8
+    bytes.buffer[offset + 4] = hi
+    bytes.buffer[offset + 5] = lo >> 24
+    bytes.buffer[offset + 6] = lo >> 16
+    bytes.buffer[offset + 7] = lo >> 8
+    bytes.buffer[offset + 8] = lo
   }
 
   _encodeMap (bytes, value) {
     const keys = Object.keys(value)
-    const buffer = bytes.buffer
-    const offset = bytes.length
-
-    bytes.reserve(5)
-    bytes.length += 5
-
-    let length = 0
+    const validKeys = keys.filter(key => typeof value[key] === 'string' || typeof value[key] === 'number')
 
-    for (const key of keys) {
-      if (typeof value[key] !== 'string' && typeof value[key] !== 'number') return
-      length++
+    this._encodeMapPrefix(bytes, validKeys.length)
 
+    for (const key of validKeys) {
       this._encodeString(bytes, key)
       this._encodeValue(bytes, value[key])
     }
-
-    buffer[offset] = 0xdf
-    buffer[offset + 1] = length >> 24
-    buffer[offset + 2] = length >> 16
-    buffer[offset + 3] = length >> 8
-    buffer[offset + 4] = length
   }
 
   _encodeValue (bytes, value) {
@@ -233,21 +232,19 @@ class AgentEncoder {
   _encodeFloat (bytes, value) {
     float64Array[0] = value
 
-    const buffer = bytes.buffer
     const offset = bytes.length
-
     bytes.reserve(9)
     bytes.length += 9
 
-    buffer[offset] = 0xcb
+    bytes.buffer[offset] = 0xcb
 
     if (bigEndian) {
       for (let i = 0; i <= 7; i++) {
-        buffer[offset + i + 1] = uInt8Float64Array[i]
+        bytes.buffer[offset + i + 1] = uInt8Float64Array[i]
       }
     } else {
       for (let i = 7; i >= 0; i--) {
-        buffer[bytes.length - i - 1] = uInt8Float64Array[i]
+        bytes.buffer[bytes.length - i - 1] = uInt8Float64Array[i]
       }
     }
   }

package/packages/dd-trace/src/encode/agentless-ci-visibility.js

@@ -1,10 +1,8 @@
 'use strict'
 const { truncateSpan, normalizeSpan } = require('./tags-processors')
-const Chunk = require('./chunk')
 const { AgentEncoder } = require('./0.4')
 const { version: ddTraceVersion } = require('../../../../package.json')
 const id = require('../../../dd-trace/src/id')
-
 const ENCODING_VERSION = 1
 
 const ALLOWED_CONTENT_TYPES = ['test_session_end', 'test_suite_end', 'test']
@@ -12,7 +10,7 @@ const ALLOWED_CONTENT_TYPES = ['test_session_end', 'test_suite_end', 'test']
 const TEST_SUITE_KEYS_LENGTH = 11
 const TEST_SESSION_KEYS_LENGTH = 10
 
-const CHUNK_SIZE = 4 * 1024 * 1024 // 4MB
+const INTAKE_SOFT_LIMIT = 2 * 1024 * 1024 // 2MB
 
 function formatSpan (span) {
   let encodingVersion = ENCODING_VERSION
@@ -27,16 +25,11 @@ function formatSpan (span) {
 }
 
 class AgentlessCiVisibilityEncoder extends AgentEncoder {
-  constructor ({ runtimeId, service, env }) {
-    super(...arguments)
-    this._events = []
+  constructor (writer, { runtimeId, service, env }) {
+    super(writer, INTAKE_SOFT_LIMIT)
     this.runtimeId = runtimeId
     this.service = service
     this.env = env
-    this._traceBytes = new Chunk(CHUNK_SIZE)
-    this._stringBytes = new Chunk(CHUNK_SIZE)
-    this._stringCount = 0
-    this._stringMap = {}
 
     // Used to keep track of the number of encoded events to update the
     // length of `payload.events` when calling `makePayload`
@@ -186,35 +179,21 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     const lo = value >>> 0
     const flag = isPositive ? 0xcf : 0xd3
 
-    const buffer = bytes.buffer
     const offset = bytes.length
 
     // int 64
     bytes.reserve(9)
     bytes.length += 9
 
-    buffer[offset] = flag
-    buffer[offset + 1] = hi >> 24
-    buffer[offset + 2] = hi >> 16
-    buffer[offset + 3] = hi >> 8
-    buffer[offset + 4] = hi
-    buffer[offset + 5] = lo >> 24
-    buffer[offset + 6] = lo >> 16
-    buffer[offset + 7] = lo >> 8
-    buffer[offset + 8] = lo
-  }
-
-  _encodeMapPrefix (bytes, keysLength) {
-    const buffer = bytes.buffer
-    const offset = bytes.length
-
-    bytes.reserve(5)
-    bytes.length += 5
-    buffer[offset] = 0xdf
-    buffer[offset + 1] = keysLength >> 24
-    buffer[offset + 2] = keysLength >> 16
-    buffer[offset + 3] = keysLength >> 8
-    buffer[offset + 4] = keysLength
+    bytes.buffer[offset] = flag
+    bytes.buffer[offset + 1] = hi >> 24
+    bytes.buffer[offset + 2] = hi >> 16
+    bytes.buffer[offset + 3] = hi >> 8
+    bytes.buffer[offset + 4] = hi
+    bytes.buffer[offset + 5] = lo >> 24
+    bytes.buffer[offset + 6] = lo >> 16
+    bytes.buffer[offset + 7] = lo >> 8
+    bytes.buffer[offset + 8] = lo
   }
 
   _encode (bytes, trace) {
@@ -240,7 +219,7 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     const traceSize = bytes.length
     const buffer = Buffer.allocUnsafe(traceSize)
 
-    bytes.buffer.copy(buffer, 0, 0, bytes.length)
+    bytes.buffer.copy(buffer, 0, 0, traceSize)
 
     this.reset()
 

package/packages/dd-trace/src/encode/coverage-ci-visibility.js

@@ -0,0 +1,84 @@
+'use strict'
+const { AgentEncoder } = require('./0.4')
+const Chunk = require('./chunk')
+
+const FormData = require('../exporters/common/form-data')
+
+const COVERAGE_PAYLOAD_VERSION = 1
+const COVERAGE_KEYS_LENGTH = 4
+const MAXIMUM_NUM_COVERAGE_FILES = 100
+
+class CoverageCIVisibilityEncoder extends AgentEncoder {
+  constructor () {
+    super(...arguments)
+    this.codeCoverageBuffers = []
+    this._coverageBytes = new Chunk()
+    this.reset()
+  }
+
+  count () {
+    return this.codeCoverageBuffers.length
+  }
+
+  encode (coverage) {
+    const bytes = this._coverageBytes
+    const coverageBuffer = this.encodeCodeCoverage(bytes, coverage)
+    this.codeCoverageBuffers.push(coverageBuffer)
+    this.reset()
+  }
+
+  encodeCodeCoverage (bytes, coverage) {
+    this._encodeMapPrefix(bytes, COVERAGE_KEYS_LENGTH)
+    this._encodeString(bytes, 'version')
+    this._encodeInteger(bytes, COVERAGE_PAYLOAD_VERSION)
+    this._encodeString(bytes, 'trace_id')
+    this._encodeId(bytes, coverage.traceId)
+    this._encodeString(bytes, 'span_id')
+    this._encodeId(bytes, coverage.spanId)
+    this._encodeString(bytes, 'files')
+    this._encodeArrayPrefix(bytes, coverage.files)
+    for (const filename of coverage.files) {
+      this._encodeMapPrefix(bytes, 1)
+      this._encodeString(bytes, 'filename')
+      this._encodeString(bytes, filename)
+    }
+    const traceSize = bytes.length
+    const buffer = Buffer.allocUnsafe(traceSize)
+
+    bytes.buffer.copy(buffer, 0, 0, bytes.length)
+
+    return buffer
+  }
+
+  reset () {
+    this._reset()
+    if (this._coverageBytes) {
+      this._coverageBytes.length = 0
+    }
+  }
+
+  makePayload () {
+    const form = new FormData()
+
+    let coverageFileIndex = 1
+
+    for (const coverageBuffer of this.codeCoverageBuffers.slice(0, MAXIMUM_NUM_COVERAGE_FILES)) {
+      const coverageFilename = `coverage${coverageFileIndex++}`
+      form.append(
+        coverageFilename,
+        coverageBuffer,
+        {
+          filename: `${coverageFilename}.msgpack`,
+          contentType: 'application/msgpack'
+        }
+      )
+    }
+    // 'event' is a backend requirement
+    form.append('event', JSON.stringify({}), { filename: 'event.json', contentType: 'application/json' })
+    this.codeCoverageBuffers = this.codeCoverageBuffers.slice(MAXIMUM_NUM_COVERAGE_FILES)
+
+    return form
+  }
+}
+
+module.exports = { CoverageCIVisibilityEncoder }

package/packages/dd-trace/src/exporters/agent/index.js

@@ -3,17 +3,16 @@
 const URL = require('url').URL
 const log = require('../../log')
 const Writer = require('./writer')
-const Scheduler = require('../scheduler')
 
 class AgentExporter {
-  constructor ({ url, hostname, port, flushInterval, lookup, protocolVersion }, prioritySampler) {
+  constructor (config, prioritySampler) {
+    this._config = config
+    const { url, hostname, port, lookup, protocolVersion } = config
     this._url = url || new URL(`http://${hostname || 'localhost'}:${port}`)
     this._writer = new Writer({ url: this._url, prioritySampler, lookup, protocolVersion })
 
-    if (flushInterval > 0) {
-      this._scheduler = new Scheduler(() => this._writer.flush(), flushInterval)
-    }
-    this._scheduler && this._scheduler.start()
+    this._timer = undefined
+    process.once('beforeExit', () => this._writer.flush())
   }
 
   setUrl (url) {
@@ -29,8 +28,15 @@ class AgentExporter {
   export (spans) {
     this._writer.append(spans)
 
-    if (!this._scheduler) {
+    const { flushInterval } = this._config
+
+    if (flushInterval === 0) {
       this._writer.flush()
+    } else if (flushInterval > 0 && !this._timer) {
+      this._timer = setTimeout(() => {
+        this._writer.flush()
+        this._timer = clearTimeout(this._timer)
+      }, flushInterval).unref()
     }
   }
 }

package/packages/dd-trace/src/exporters/agent/writer.js

@@ -99,7 +99,7 @@ function makeRequest (version, data, count, url, lookup, needsStartupLog, cb) {
 
   log.debug(() => `Request to the agent: ${JSON.stringify(options)}`)
 
-  request(data, options, true, (err, res, status) => {
+  request(data, options, (err, res, status) => {
     if (needsStartupLog) {
       // Note that logging will only happen once, regardless of how many times this is called.
       startupLog({