dd-trace 2.12.2 → 2.13.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "2.12.2",
+  "version": "2.13.0",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",
@@ -61,7 +61,7 @@
     "@datadog/native-appsec": "^1.2.1",
     "@datadog/native-metrics": "^1.4.2",
     "@datadog/pprof": "^1.0.2",
-    "@datadog/sketches-js": "^1.0.5",
+    "@datadog/sketches-js": "^2.0.0",
     "@types/node": ">=12",
     "crypto-randomuuid": "^1.0.0",
     "diagnostics_channel": "^1.1.0",

package/packages/datadog-instrumentations/index.js

@@ -1,54 +1,3 @@
 'use strict'
 
-require('./src/amqplib')
-require('./src/amqp10')
-require('./src/aws-sdk')
-require('./src/bluebird')
-require('./src/bunyan')
-require('./src/cassandra-driver')
-require('./src/connect')
-require('./src/couchbase')
-require('./src/cucumber')
-require('./src/dns')
-require('./src/elasticsearch')
-require('./src/express')
-require('./src/fastify')
-require('./src/find-my-way')
-require('./src/generic-pool')
-require('./src/google-cloud-pubsub')
-require('./src/graphql')
-require('./src/grpc')
-require('./src/hapi')
-require('./src/http')
-require('./src/http2')
-require('./src/ioredis')
-require('./src/jest')
-require('./src/kafkajs')
-require('./src/knex')
-require('./src/koa')
-require('./src/memcached')
-require('./src/microgateway-core')
-require('./src/moleculer')
-require('./src/mongodb-core')
-require('./src/mongoose')
-require('./src/mysql')
-require('./src/mysql2')
-require('./src/mocha')
-require('./src/net')
-require('./src/next')
-require('./src/oracledb')
-require('./src/paperplane')
-require('./src/pino')
-require('./src/pg')
-require('./src/promise')
-require('./src/promise-js')
-require('./src/q')
-require('./src/redis')
-require('./src/restify')
-require('./src/router')
-require('./src/rhea')
-require('./src/sharedb')
-require('./src/tedious')
-require('./src/when')
-require('./src/winston')
-require('./src/limitd-client')
+require('./src/helpers/register')

package/packages/datadog-instrumentations/src/helpers/hooks.js

@@ -0,0 +1,68 @@
+'use strict'
+
+module.exports = {
+  '@cucumber/cucumber': () => require('../cucumber'),
+  '@elastic/elasticsearch': () => require('../elasticsearch'),
+  '@elastic/transport': () => require('../elasticsearch'),
+  '@google-cloud/pubsub': () => require('../google-cloud-pubsub'),
+  '@grpc/grpc-js': () => require('../grpc'),
+  '@hapi/hapi': () => require('../hapi'),
+  '@koa/router': () => require('../koa'),
+  '@node-redis/client': () => require('../redis'),
+  'amqp10': () => require('../amqp10'),
+  'amqplib': () => require('../amqplib'),
+  'aws-sdk': () => require('../aws-sdk'),
+  'bluebird': () => require('../bluebird'),
+  'bunyan': () => require('../bunyan'),
+  'cassandra-driver': () => require('../cassandra-driver'),
+  'connect': () => require('../connect'),
+  'couchbase': () => require('../couchbase'),
+  'cypress': () => require('../cypress'),
+  'dns': () => require('../dns'),
+  'elasticsearch': () => require('../elasticsearch'),
+  'express': () => require('../express'),
+  'fastify': () => require('../fastify'),
+  'find-my-way': () => require('../find-my-way'),
+  'graphql': () => require('../graphql'),
+  'grpc': () => require('../grpc'),
+  'hapi': () => require('../hapi'),
+  'http': () => require('../http'),
+  'http2': () => require('../http2'),
+  'https': () => require('../http'),
+  'ioredis': () => require('../ioredis'),
+  'jest-environment-node': () => require('../jest'),
+  'jest-environment-jsdom': () => require('../jest'),
+  'jest-jasmine2': () => require('../jest'),
+  'koa': () => require('../koa'),
+  'koa-router': () => require('../koa'),
+  'kafkajs': () => require('../kafkajs'),
+  'limitd-client': () => require('../limitd-client'),
+  'memcached': () => require('../memcached'),
+  'microgateway-core': () => require('../microgateway-core'),
+  'mocha': () => require('../mocha'),
+  'mocha-each': () => require('../mocha'),
+  'moleculer': () => require('../moleculer'),
+  'mongodb': () => require('../mongodb-core'),
+  'mongodb-core': () => require('../mongodb-core'),
+  'mongoose': () => require('../mongoose'),
+  'mysql': () => require('../mysql'),
+  'mysql2': () => require('../mysql2'),
+  'net': () => require('../net'),
+  'next': () => require('../next'),
+  'oracledb': () => require('../oracledb'),
+  'paperplane': () => require('../paperplane'),
+  'pg': () => require('../pg'),
+  'pino': () => require('../pino'),
+  'pino-pretty': () => require('../pino'),
+  'promise-js': () => require('../promise-js'),
+  'promise': () => require('../promise'),
+  'q': () => require('../q'),
+  'redis': () => require('../redis'),
+  'restify': () => require('../restify'),
+  'rhea': () => require('../rhea'),
+  'router': () => require('../router'),
+  'sharedb': () => require('../sharedb'),
+  'tedious': () => require('../tedious'),
+  'when': () => require('../when'),
+  'winston': () => require('../winston')
+}

package/packages/datadog-instrumentations/src/helpers/instrument.js

@@ -1,16 +1,12 @@
 'use strict'
 
 const dc = require('diagnostics_channel')
-const path = require('path')
 const semver = require('semver')
-const Hook = require('./hook')
-const requirePackageJson = require('../../../dd-trace/src/require-package-json')
+const instrumentations = require('./instrumentations')
 const { AsyncResource } = require('async_hooks')
-const log = require('../../../dd-trace/src/log')
 
-const pathSepExpr = new RegExp(`\\${path.sep}`, 'g')
 const channelMap = {}
-exports.channel = function channel (name) {
+exports.channel = function (name) {
   const maybe = channelMap[name]
   if (maybe) return maybe
   const ch = dc.channel(name)
@@ -19,36 +15,11 @@ exports.channel = function channel (name) {
 }
 
 exports.addHook = function addHook ({ name, versions, file }, hook) {
-  const fullFilename = filename(name, file)
-
-  Hook([name], (moduleExports, moduleName, moduleBaseDir) => {
-    moduleName = moduleName.replace(pathSepExpr, '/')
-
-    if (moduleName !== fullFilename || !matchVersion(getVersion(moduleBaseDir), versions)) {
-      return moduleExports
-    }
-
-    try {
-      return hook(moduleExports)
-    } catch (e) {
-      log.error(e)
-      return moduleExports
-    }
-  })
-}
-
-function matchVersion (version, ranges) {
-  return !version || (ranges && ranges.some(range => semver.satisfies(semver.coerce(version), range)))
-}
-
-function getVersion (moduleBaseDir) {
-  if (moduleBaseDir) {
-    return requirePackageJson(moduleBaseDir, module).version
+  if (!instrumentations[name]) {
+    instrumentations[name] = []
   }
-}
 
-function filename (name, file) {
-  return [name, file].filter(val => val).join('/')
+  instrumentations[name].push({ name, versions, file, hook })
 }
 
 // AsyncResource.bind exists and binds `this` properly only from 17.8.0 and up.

package/packages/datadog-instrumentations/src/helpers/instrumentations.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = {}

package/packages/datadog-instrumentations/src/helpers/register.js

@@ -0,0 +1,59 @@
+'use strict'
+
+const { channel } = require('diagnostics_channel')
+const path = require('path')
+const semver = require('semver')
+const Hook = require('./hook')
+const requirePackageJson = require('../../../dd-trace/src/require-package-json')
+const log = require('../../../dd-trace/src/log')
+
+const hooks = require('./hooks')
+const instrumentations = require('./instrumentations')
+const names = Object.keys(hooks)
+const pathSepExpr = new RegExp(`\\${path.sep}`, 'g')
+
+const loadChannel = channel('dd-trace:instrumentation:load')
+
+// TODO: make this more efficient
+
+for (const packageName of names) {
+  Hook([packageName], (moduleExports, moduleName, moduleBaseDir) => {
+    moduleName = moduleName.replace(pathSepExpr, '/')
+
+    hooks[packageName]()
+
+    for (const { name, file, versions, hook } of instrumentations[packageName]) {
+      const fullFilename = filename(name, file)
+
+      if (moduleName === fullFilename) {
+        const version = getVersion(moduleBaseDir)
+
+        if (matchVersion(version, versions)) {
+          try {
+            loadChannel.publish({ name, version, file })
+
+            moduleExports = hook(moduleExports)
+          } catch (e) {
+            log.error(e)
+          }
+        }
+      }
+    }
+
+    return moduleExports
+  })
+}
+
+function matchVersion (version, ranges) {
+  return !version || (ranges && ranges.some(range => semver.satisfies(semver.coerce(version), range)))
+}
+
+function getVersion (moduleBaseDir) {
+  if (moduleBaseDir) {
+    return requirePackageJson(moduleBaseDir, module).version
+  }
+}
+
+function filename (name, file) {
+  return [name, file].filter(val => val).join('/')
+}

package/packages/datadog-plugin-mocha/src/index.js

@@ -161,12 +161,12 @@ class MochaPlugin extends Plugin {
     const testSuiteSpan = this._testSuites.get(test.parent)
 
     if (testSuiteSpan) {
-      const testSuiteId = testSuiteSpan.context()._spanId.toString('hex')
+      const testSuiteId = testSuiteSpan.context()._spanId.toString(16)
       testSuiteTags[TEST_SUITE_ID] = testSuiteId
     }
 
     if (this.testSessionSpan) {
-      const testSessionId = this.testSessionSpan.context()._traceId.toString('hex')
+      const testSessionId = this.testSessionSpan.context()._traceId.toString(16)
       testSuiteTags[TEST_SESSION_ID] = testSessionId
       testSuiteTags[TEST_COMMAND] = this.command
     }

package/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js

@@ -0,0 +1,220 @@
+
+const fs = require('fs')
+const https = require('https')
+const path = require('path')
+
+const FormData = require('../../../exporters/common/form-data')
+
+const log = require('../../../log')
+const {
+  getLatestCommits,
+  getRepositoryUrl,
+  generatePackFilesForCommits,
+  getCommitsToUpload
+} = require('../../../plugins/util/git')
+
+const isValidSha = (sha) => /[0-9a-f]{40}/.test(sha)
+
+function sanitizeCommits (commits) {
+  return commits.map(({ id: commitSha, type }) => {
+    if (type !== 'commit') {
+      throw new Error('Invalid commit response')
+    }
+    const sanitizedCommit = commitSha.replace(/[^0-9a-f]+/g, '')
+    if (sanitizedCommit !== commitSha || !isValidSha(sanitizedCommit)) {
+      throw new Error('Invalid commit format')
+    }
+    return sanitizedCommit
+  })
+}
+
+function getCommonRequestOptions (url) {
+  return {
+    method: 'POST',
+    headers: {
+      'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY
+    },
+    timeout: 15000,
+    protocol: url.protocol,
+    hostname: url.hostname,
+    port: url.port
+  }
+}
+
+/**
+ * This function posts the SHAs of the commits of the last month
+ * The response are the commits for which the backend already has information
+ * This response is used to know which commits can be ignored from there on
+ */
+function getCommitsToExclude ({ url, repositoryUrl }, callback) {
+  const latestCommits = getLatestCommits()
+  const [headCommit] = latestCommits
+
+  const commonOptions = getCommonRequestOptions(url)
+
+  const options = {
+    ...commonOptions,
+    headers: {
+      ...commonOptions.headers,
+      'Content-Type': 'application/json'
+    },
+    path: '/api/v2/git/repository/search_commits'
+  }
+
+  const localCommitData = JSON.stringify({
+    meta: {
+      repository_url: repositoryUrl
+    },
+    data: latestCommits.map(commit => ({
+      id: commit,
+      type: 'commit'
+    }))
+  })
+
+  const request = https.request(options, (res) => {
+    let responseData = ''
+
+    res.on('data', chunk => { responseData += chunk })
+    res.on('end', () => {
+      if (res.statusCode === 200) {
+        let commitsToExclude
+        try {
+          commitsToExclude = sanitizeCommits(JSON.parse(responseData).data)
+        } catch (e) {
+          callback(new Error(`Can't parse response: ${e.message}`))
+          return
+        }
+        callback(null, commitsToExclude, headCommit)
+      } else {
+        const error = new Error(`Error getting commits: ${res.statusCode} ${res.statusMessage}`)
+        callback(error)
+      }
+    })
+  })
+
+  request.write(localCommitData)
+  request.on('error', callback)
+
+  request.end()
+
+  return request
+}
+
+/**
+ * This function uploads a git packfile
+ */
+function uploadPackFile ({ url, packFileToUpload, repositoryUrl, headCommit }, callback) {
+  const form = new FormData()
+
+  const pushedSha = JSON.stringify({
+    data: {
+      id: headCommit,
+      type: 'commit'
+    },
+    meta: {
+      repository_url: repositoryUrl
+    }
+  })
+
+  form.append('pushedSha', pushedSha, { contentType: 'application/json' })
+
+  try {
+    const packFileContent = fs.readFileSync(packFileToUpload)
+    // The original filename includes a random prefix, so we remove it here
+    const [, filename] = path.basename(packFileToUpload).split('-')
+    form.append('packfile', packFileContent, {
+      filename,
+      contentType: 'application/octet-stream'
+    })
+  } catch (e) {
+    callback(new Error(`Error reading packfile: ${packFileToUpload}`))
+    return
+  }
+
+  const commonOptions = getCommonRequestOptions(url)
+
+  const options = {
+    ...commonOptions,
+    path: '/api/v2/git/repository/packfile',
+    headers: {
+      ...commonOptions.headers,
+      ...form.getHeaders()
+    }
+  }
+
+  const req = https.request(options, res => {
+    res.on('data', () => {})
+    res.on('end', () => {
+      if (res.statusCode === 204) {
+        callback(null)
+      } else {
+        const error = new Error(`Error uploading packfiles: ${res.statusCode} ${res.statusMessage}`)
+        error.status = res.statusCode
+
+        callback(error)
+      }
+    })
+  })
+
+  req.on('error', err => {
+    callback(err)
+  })
+  form.pipe(req)
+}
+
+/**
+ * This function uploads git metadata to CI Visibility's backend.
+*/
+function sendGitMetadata (site, callback) {
+  const url = new URL(`https://api.${site}`)
+
+  const repositoryUrl = getRepositoryUrl()
+
+  getCommitsToExclude({ url, repositoryUrl }, (err, commitsToExclude, headCommit) => {
+    if (err) {
+      callback(err)
+      return
+    }
+    const commitsToUpload = getCommitsToUpload(commitsToExclude)
+
+    if (!commitsToUpload.length) {
+      log.debug('No commits to upload')
+      callback(null)
+      return
+    }
+
+    const packFilesToUpload = generatePackFilesForCommits(commitsToUpload)
+
+    let packFileIndex = 0
+    // This uploads packfiles sequentially
+    const uploadPackFileCallback = (err) => {
+      if (err || packFileIndex === packFilesToUpload.length) {
+        callback(err)
+        return
+      }
+      return uploadPackFile(
+        {
+          packFileToUpload: packFilesToUpload[packFileIndex++],
+          url,
+          repositoryUrl,
+          headCommit
+        },
+        uploadPackFileCallback
+      )
+    }
+
+    uploadPackFile(
+      {
+        url,
+        packFileToUpload: packFilesToUpload[packFileIndex++],
+        repositoryUrl,
+        headCommit
+      },
+      uploadPackFileCallback
+    )
+  })
+}
+
+module.exports = {
+  sendGitMetadata
+}

package/packages/dd-trace/src/config.js

@@ -174,6 +174,11 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
 |[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}`
     )
 
+    const DD_CIVISIBILITY_GIT_UPLOAD_ENABLED = coalesce(
+      process.env.DD_CIVISIBILITY_GIT_UPLOAD_ENABLED,
+      false
+    )
+
     const sampler = (options.experimental && options.experimental.sampler) || {}
     const ingestion = options.ingestion || {}
     const dogstatsd = coalesce(options.dogstatsd, {})
@@ -248,6 +253,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       obfuscatorKeyRegex: DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP,
       obfuscatorValueRegex: DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP
     }
+    this.isGitUploadEnabled = isTrue(DD_CIVISIBILITY_GIT_UPLOAD_ENABLED)
 
     tagger.add(this.tags, {
       service: this.service,

package/packages/dd-trace/src/id.js

@@ -14,17 +14,17 @@ let batch = 0
 
 // Internal representation of a trace or span ID.
 class Identifier {
-  constructor (value, radix) {
+  constructor (value, radix = 16) {
     this._isUint64BE = true // msgpack-lite compatibility
-    this._buffer = typeof radix === 'number'
-      ? fromString(value, radix)
-      : createBuffer(value)
+    this._buffer = radix === 16
+      ? createBuffer(value)
+      : fromString(value, radix)
   }
 
-  toString (radix) {
-    return typeof radix === 'number'
-      ? toNumberString(this._buffer, radix)
-      : toHexString(this._buffer)
+  toString (radix = 16) {
+    return radix === 16
+      ? toHexString(this._buffer)
+      : toNumberString(this._buffer, radix)
   }
 
   toBuffer () {
@@ -49,10 +49,13 @@ function createBuffer (value) {
   if (value === '0') return zeroId
   if (!value) return pseudoRandom()
 
-  const size = Math.ceil(value.length / 2)
-  const buffer = new Array(size)
+  const size = Math.ceil(value.length / 16) * 16
+  const bytes = size / 2
+  const buffer = new Array(bytes)
 
-  for (let i = 0; i < size; i++) {
+  value = value.padStart(size, '0')
+
+  for (let i = 0; i < bytes; i++) {
     buffer[i] = parseInt(value.substring(i * 2, i * 2 + 2), 16)
   }
 
@@ -100,8 +103,8 @@ function fromString (str, raddix) {
 
 // Convert a buffer to a numerical string.
 function toNumberString (buffer, radix) {
-  let high = readInt32(buffer, 0)
-  let low = readInt32(buffer, 4)
+  let high = readInt32(buffer, buffer.length - 8)
+  let low = readInt32(buffer, buffer.length - 4)
   let str = ''
 
   radix = radix || 10

package/packages/dd-trace/src/opentracing/propagation/text_map.js

@@ -82,8 +82,8 @@ class TextMapPropagator {
   _injectB3 (spanContext, carrier) {
     if (!this._config.experimental.b3) return
 
-    carrier[b3TraceKey] = spanContext._traceId.toString('hex')
-    carrier[b3SpanKey] = spanContext._spanId.toString('hex')
+    carrier[b3TraceKey] = spanContext._traceId.toString(16)
+    carrier[b3SpanKey] = spanContext._spanId.toString(16)
     carrier[b3SampledKey] = spanContext._sampling.priority >= AUTO_KEEP ? '1' : '0'
 
     if (spanContext._sampling.priority > AUTO_KEEP) {
@@ -91,7 +91,7 @@ class TextMapPropagator {
     }
 
     if (spanContext._parentId) {
-      carrier[b3ParentKey] = spanContext._parentId.toString('hex')
+      carrier[b3ParentKey] = spanContext._parentId.toString(16)
     }
   }
 
@@ -99,8 +99,8 @@ class TextMapPropagator {
     if (!this._config.experimental.traceparent) return
 
     const sampling = spanContext._sampling.priority >= AUTO_KEEP ? '01' : '00'
-    const traceId = spanContext._traceId.toString('hex').padStart(32, '0')
-    const spanId = spanContext._spanId.toString('hex').padStart(16, '0')
+    const traceId = spanContext._traceId.toString(16).padStart(32, '0')
+    const spanId = spanContext._spanId.toString(16).padStart(16, '0')
     carrier[traceparentKey] = `01-${traceId}-${spanId}-${sampling}`
   }
 
@@ -129,7 +129,7 @@ class TextMapPropagator {
     const b3 = this._extractB3Headers(carrier)
     const debug = b3[b3FlagsKey] === '1'
     const priority = this._getPriority(b3[b3SampledKey], debug)
-    const spanContext = this._extractGenericContext(b3, b3TraceKey, b3SpanKey)
+    const spanContext = this._extractGenericContext(b3, b3TraceKey, b3SpanKey, 16)
 
     if (priority !== undefined) {
       if (!spanContext) {