dd-trace 2.4.2 → 2.5.0

package/packages/dd-trace/src/encode/agentless-ci-visibility.js

@@ -0,0 +1,193 @@
+'use strict'
+const { truncateSpan, normalizeSpan } = require('./tags-processors')
+const Chunk = require('./chunk')
+const { AgentEncoder } = require('./0.4')
+
+const ENCODING_VERSION = 1
+
+function formatSpan (span) {
+  return {
+    type: span.type === 'test' ? 'test' : 'span',
+    version: ENCODING_VERSION,
+    content: normalizeSpan(truncateSpan(span))
+  }
+}
+
+class AgentlessCiVisibilityEncoder extends AgentEncoder {
+  constructor ({ runtimeId, service, env }) {
+    super(...arguments)
+    this._events = []
+    this.runtimeId = runtimeId
+    this.service = service
+    this.env = env
+    this._traceBytes = new Chunk()
+    this._stringBytes = new Chunk()
+    this._stringCount = 0
+    this._stringMap = {}
+
+    // Used to keep track of the number of encoded events to update the
+    // length of `payload.events` when calling `makePayload`
+    this._eventCount = 0
+
+    this.reset()
+  }
+
+  _encodeEventContent (bytes, content) {
+    this._encodeMapPrefix(bytes, content)
+    if (content.type) {
+      this._encodeString(bytes, 'type')
+      this._encodeString(bytes, content.type)
+    }
+    this._encodeString(bytes, 'trace_id')
+    this._encodeId(bytes, content.trace_id)
+    this._encodeString(bytes, 'span_id')
+    this._encodeId(bytes, content.span_id)
+    this._encodeString(bytes, 'parent_id')
+    this._encodeId(bytes, content.parent_id)
+    this._encodeString(bytes, 'name')
+    this._encodeString(bytes, content.name)
+    this._encodeString(bytes, 'resource')
+    this._encodeString(bytes, content.resource)
+    this._encodeString(bytes, 'service')
+    this._encodeString(bytes, content.service)
+    this._encodeString(bytes, 'error')
+    this._encodeNumber(bytes, content.error)
+    this._encodeString(bytes, 'start')
+    this._encodeNumber(bytes, content.start)
+    this._encodeString(bytes, 'duration')
+    this._encodeNumber(bytes, content.duration)
+    this._encodeString(bytes, 'meta')
+    this._encodeMap(bytes, content.meta)
+    this._encodeString(bytes, 'metrics')
+    this._encodeMap(bytes, content.metrics)
+  }
+
+  _encodeEvent (bytes, event) {
+    this._encodeMapPrefix(bytes, event)
+    this._encodeString(bytes, 'type')
+    this._encodeString(bytes, event.type)
+
+    this._encodeString(bytes, 'version')
+    this._encodeNumber(bytes, event.version)
+
+    this._encodeString(bytes, 'content')
+    this._encodeEventContent(bytes, event.content)
+  }
+
+  _encodeNumber (bytes, value) {
+    if (Math.floor(value) !== value) { // float 64
+      return this._encodeFloat(bytes, value)
+    }
+    return this._encodeLong(bytes, value)
+  }
+
+  _encodeLong (bytes, value) {
+    const isPositive = value >= 0
+
+    const hi = isPositive ? (value / Math.pow(2, 32)) >> 0 : Math.floor(value / Math.pow(2, 32))
+    const lo = value >>> 0
+    const flag = isPositive ? 0xcf : 0xd3
+
+    const buffer = bytes.buffer
+    const offset = bytes.length
+
+    // int 64
+    bytes.reserve(9)
+    bytes.length += 9
+
+    buffer[offset] = flag
+    buffer[offset + 1] = hi >> 24
+    buffer[offset + 2] = hi >> 16
+    buffer[offset + 3] = hi >> 8
+    buffer[offset + 4] = hi
+    buffer[offset + 5] = lo >> 24
+    buffer[offset + 6] = lo >> 16
+    buffer[offset + 7] = lo >> 8
+    buffer[offset + 8] = lo
+  }
+
+  _encodeMapPrefix (bytes, map) {
+    const keys = Object.keys(map)
+    const buffer = bytes.buffer
+    const offset = bytes.length
+
+    bytes.reserve(5)
+    bytes.length += 5
+    buffer[offset] = 0xdf
+    buffer[offset + 1] = keys.length >> 24
+    buffer[offset + 2] = keys.length >> 16
+    buffer[offset + 3] = keys.length >> 8
+    buffer[offset + 4] = keys.length
+  }
+
+  _encode (bytes, trace) {
+    this._eventCount += trace.length
+    const events = trace.map(formatSpan)
+
+    for (const event of events) {
+      this._encodeEvent(bytes, event)
+    }
+  }
+
+  makePayload () {
+    const bytes = this._traceBytes
+    const eventsOffset = this._eventsOffset
+    const eventsCount = this._eventCount
+
+    bytes.buffer[eventsOffset] = 0xdd
+    bytes.buffer[eventsOffset + 1] = eventsCount >> 24
+    bytes.buffer[eventsOffset + 2] = eventsCount >> 16
+    bytes.buffer[eventsOffset + 3] = eventsCount >> 8
+    bytes.buffer[eventsOffset + 4] = eventsCount
+
+    const traceSize = bytes.length
+    const buffer = Buffer.allocUnsafe(traceSize)
+
+    bytes.buffer.copy(buffer, 0, 0, bytes.length)
+
+    this.reset()
+
+    return buffer
+  }
+
+  _encodePayloadStart (bytes) {
+    // encodes the payload up to `events`. `events` will be encoded via _encode
+    const payload = {
+      version: ENCODING_VERSION,
+      metadata: {
+        '*': {
+          'language': 'javascript'
+        }
+      },
+      events: []
+    }
+
+    if (this.env) {
+      payload.metadata['*'].env = this.env
+    }
+    if (this.runtimeId) {
+      payload.metadata['*']['runtime-id'] = this.runtimeId
+    }
+
+    this._encodeMapPrefix(bytes, payload)
+    this._encodeString(bytes, 'version')
+    this._encodeNumber(bytes, payload.version)
+    this._encodeString(bytes, 'metadata')
+    this._encodeMapPrefix(bytes, payload.metadata)
+    this._encodeString(bytes, '*')
+    this._encodeMap(bytes, payload.metadata['*'])
+    this._encodeString(bytes, 'events')
+    // Get offset of the events list to update the length of the array when calling `makePayload`
+    this._eventsOffset = bytes.length
+    bytes.reserve(5)
+    bytes.length += 5
+  }
+
+  reset () {
+    this._reset()
+    this._eventCount = 0
+    this._encodePayloadStart(this._traceBytes)
+  }
+}
+
+module.exports = { AgentlessCiVisibilityEncoder }

package/packages/dd-trace/src/encode/tags-processors.js

@@ -0,0 +1,116 @@
+// From agent truncators: https://github.com/DataDog/datadog-agent/blob/main/pkg/trace/agent/truncator.go
+
+// Values from: https://github.com/DataDog/datadog-agent/blob/main/pkg/trace/traceutil/truncate.go#L22-L27
+// MAX_RESOURCE_NAME_LENGTH the maximum length a span resource can have
+const MAX_RESOURCE_NAME_LENGTH = 5000
+// MAX_META_KEY_LENGTH the maximum length of metadata key
+const MAX_META_KEY_LENGTH = 200
+// MAX_META_VALUE_LENGTH the maximum length of metadata value
+const MAX_META_VALUE_LENGTH = 25000
+// MAX_METRIC_KEY_LENGTH the maximum length of a metric name key
+const MAX_METRIC_KEY_LENGTH = MAX_META_KEY_LENGTH
+// MAX_METRIC_VALUE_LENGTH the maximum length of a metric name value
+const MAX_METRIC_VALUE_LENGTH = MAX_META_VALUE_LENGTH
+
+// From agent normalizer:
+// https://github.com/DataDog/datadog-agent/blob/main/pkg/trace/traceutil/normalize.go
+// DEFAULT_SPAN_NAME is the default name we assign a span if it's missing and we have no reasonable fallback
+const DEFAULT_SPAN_NAME = 'unnamed_operation'
+// DEFAULT_SERVICE_NAME is the default name we assign a service if it's missing and we have no reasonable fallback
+const DEFAULT_SERVICE_NAME = 'unnamed-service'
+// MAX_NAME_LENGTH the maximum length a name can have
+const MAX_NAME_LENGTH = 100
+// MAX_SERVICE_LENGTH the maximum length a service can have
+const MAX_SERVICE_LENGTH = 100
+// MAX_TYPE_LENGTH the maximum length a span type can have
+const MAX_TYPE_LENGTH = 100
+
+const fromEntries = Object.fromEntries || (entries =>
+  entries.reduce((obj, [k, v]) => Object.assign(obj, { [k]: v }), {}))
+
+function truncateToLength (value, maxLength) {
+  if (!value) {
+    return value
+  }
+  if (value.length > maxLength) {
+    return `${value.slice(0, maxLength)}...`
+  }
+  return value
+}
+
+function truncateSpan (span) {
+  return fromEntries(Object.entries(span).map(([key, value]) => {
+    switch (key) {
+      case 'resource':
+        return ['resource', truncateToLength(value, MAX_RESOURCE_NAME_LENGTH)]
+      case 'meta':
+        return ['meta', fromEntries(Object.entries(value).map(([metaKey, metaValue]) =>
+          [truncateToLength(metaKey, MAX_META_KEY_LENGTH), truncateToLength(metaValue, MAX_META_VALUE_LENGTH)]
+        ))]
+      case 'metrics':
+        return ['metrics', fromEntries(Object.entries(value).map(([metricsKey, metricsValue]) =>
+          [truncateToLength(metricsKey, MAX_METRIC_KEY_LENGTH), truncateToLength(metricsValue, MAX_METRIC_VALUE_LENGTH)]
+        ))]
+      default:
+        return [key, value]
+    }
+  }))
+}
+
+function normalizeSpan (span) {
+  const normalizedSpan = fromEntries(Object.entries(span).map(([key, value]) => {
+    switch (key) {
+      case 'service':
+        if (!value) {
+          return [key, DEFAULT_SERVICE_NAME]
+        }
+        if (value.length > MAX_SERVICE_LENGTH) {
+          return [key, value.slice(0, MAX_SERVICE_LENGTH)]
+        }
+        break
+      case 'name':
+        if (!value) {
+          return [key, DEFAULT_SPAN_NAME]
+        }
+        if (value.length > MAX_NAME_LENGTH) {
+          return [key, value.slice(0, MAX_NAME_LENGTH)]
+        }
+        break
+      case 'resource':
+        if (!value) {
+          return [key, span.name || DEFAULT_SPAN_NAME]
+        }
+        break
+      case 'type':
+        if (!value) {
+          return [key, value]
+        }
+        if (value.length > MAX_TYPE_LENGTH) {
+          return [key, value.slice(0, MAX_TYPE_LENGTH)]
+        }
+    }
+    return [key, value]
+  }))
+  if (!normalizedSpan.service) {
+    normalizedSpan.service = DEFAULT_SERVICE_NAME
+  }
+  if (!normalizedSpan.name) {
+    normalizedSpan.name = DEFAULT_SPAN_NAME
+  }
+  return normalizedSpan
+}
+
+module.exports = {
+  truncateSpan,
+  normalizeSpan,
+  MAX_META_KEY_LENGTH,
+  MAX_META_VALUE_LENGTH,
+  MAX_METRIC_KEY_LENGTH,
+  MAX_METRIC_VALUE_LENGTH,
+  MAX_NAME_LENGTH,
+  MAX_SERVICE_LENGTH,
+  MAX_TYPE_LENGTH,
+  MAX_RESOURCE_NAME_LENGTH,
+  DEFAULT_SPAN_NAME,
+  DEFAULT_SERVICE_NAME
+}

package/packages/dd-trace/src/exporter.js

@@ -2,6 +2,7 @@
 
 const AgentExporter = require('./exporters/agent')
 const LogExporter = require('./exporters/log')
+const AgentlessCiVisibilityExporter = require('./ci-visibility/exporters/agentless')
 const exporters = require('../../../ext/exporters')
 const fs = require('fs')
 const constants = require('./constants')
@@ -15,6 +16,8 @@ module.exports = name => {
       return LogExporter
     case exporters.AGENT:
       return AgentExporter
+    case exporters.DATADOG:
+      return AgentlessCiVisibilityExporter
     default:
       return inAWSLambda && !usingLambdaExtension ? LogExporter : AgentExporter
   }

package/packages/dd-trace/src/exporters/agent/index.js

@@ -3,7 +3,7 @@
 const URL = require('url').URL
 const log = require('../../log')
 const Writer = require('./writer')
-const Scheduler = require('./scheduler')
+const Scheduler = require('../scheduler')
 
 class AgentExporter {
   constructor ({ url, hostname, port, flushInterval, lookup, protocolVersion }, prioritySampler) {

package/packages/dd-trace/src/exporters/agent/writer.js

@@ -1,28 +1,23 @@
 'use strict'
 
-const request = require('./request')
+const request = require('../common/request')
 const { startupLog } = require('../../startup-log')
 const metrics = require('../../metrics')
 const log = require('../../log')
 const tracerVersion = require('../../../lib/version')
+const BaseWriter = require('../common/writer')
 
 const METRIC_PREFIX = 'datadog.tracer.node.exporter.agent'
 
-class Writer {
-  constructor ({ url, prioritySampler, lookup, protocolVersion }) {
+class Writer extends BaseWriter {
+  constructor ({ prioritySampler, lookup, protocolVersion }) {
+    super(...arguments)
     const AgentEncoder = getEncoder(protocolVersion)
 
-    this._url = url
     this._prioritySampler = prioritySampler
     this._lookup = lookup
     this._protocolVersion = protocolVersion
-    this._encoderForVersion = new AgentEncoder(this)
-  }
-
-  append (spans) {
-    log.debug(() => `Encoding trace: ${JSON.stringify(spans)}`)
-
-    this._encode(spans)
+    this._encoder = new AgentEncoder(this)
   }
 
   _sendPayload (data, count, done) {
@@ -62,26 +57,6 @@ class Writer {
       done()
     })
   }
-
-  setUrl (url) {
-    this._url = url
-  }
-
-  _encode (trace) {
-    this._encoderForVersion.encode(trace)
-  }
-
-  flush (done = () => {}) {
-    const count = this._encoderForVersion.count()
-
-    if (count > 0) {
-      const payload = this._encoderForVersion.makePayload()
-
-      this._sendPayload(payload, count, done)
-    } else {
-      done()
-    }
-  }
 }
 
 function setHeader (headers, key, value) {
@@ -124,7 +99,7 @@ function makeRequest (version, data, count, url, lookup, needsStartupLog, cb) {
 
   log.debug(() => `Request to the agent: ${JSON.stringify(options)}`)
 
-  request(Object.assign({ data }, options), (err, res, status) => {
+  request(data, options, true, (err, res, status) => {
     if (needsStartupLog) {
       // Note that logging will only happen once, regardless of how many times this is called.
       startupLog({

package/packages/dd-trace/src/exporters/common/request.js

@@ -0,0 +1,83 @@
+'use strict'
+
+const http = require('http')
+const https = require('https')
+const log = require('../../log')
+const docker = require('./docker')
+const { storage } = require('../../../../datadog-core')
+
+const httpAgent = new http.Agent({ keepAlive: true })
+const httpsAgent = new https.Agent({ keepAlive: true })
+const containerId = docker.id()
+
+function request (data, options, keepAlive, callback) {
+  if (!options.headers) {
+    options.headers = {}
+  }
+  const isSecure = options.protocol === 'https:'
+  const client = isSecure ? https : http
+  const dataArray = [].concat(data)
+  options.headers['Content-Length'] = byteLength(dataArray)
+
+  if (containerId) {
+    options.headers['Datadog-Container-ID'] = containerId
+  }
+
+  if (keepAlive) {
+    options.agent = isSecure ? httpsAgent : httpAgent
+  }
+
+  const firstRequest = retriableRequest(options, client, callback)
+  dataArray.forEach(buffer => firstRequest.write(buffer))
+
+  // The first request will be retried
+  const firstRequestErrorHandler = () => {
+    log.debug('Retrying request to the intake')
+    const retriedReq = retriableRequest(options, client, callback)
+    dataArray.forEach(buffer => retriedReq.write(buffer))
+    // The retried request will fail normally
+    retriedReq.on('error', e => callback(new Error(`Network error trying to reach the intake: ${e.message}`)))
+    retriedReq.end()
+  }
+
+  firstRequest.on('error', firstRequestErrorHandler)
+  firstRequest.end()
+
+  return firstRequest
+}
+
+function retriableRequest (options, client, callback) {
+  const store = storage.getStore()
+
+  storage.enterWith({ noop: true })
+
+  const timeout = options.timeout || 15000
+
+  const request = client.request(options, res => {
+    let responseData = ''
+
+    res.setTimeout(timeout)
+
+    res.on('data', chunk => { responseData += chunk })
+    res.on('end', () => {
+      if (res.statusCode >= 200 && res.statusCode <= 299) {
+        callback(null, responseData, res.statusCode)
+      } else {
+        const error = new Error(`Error from the endpoint: ${res.statusCode} ${http.STATUS_CODES[res.statusCode]}`)
+        error.status = res.statusCode
+
+        callback(error, null, res.statusCode)
+      }
+    })
+  })
+  request.setTimeout(timeout, request.abort)
+  storage.enterWith(store)
+
+  return request
+}
+
+function byteLength (data) {
+  return data.length > 0 ? data.reduce((prev, next) => prev + next.length, 0) : 0
+}
+
+module.exports = request

package/packages/dd-trace/src/exporters/common/writer.js

@@ -0,0 +1,36 @@
+'use strict'
+const log = require('../../log')
+
+class Writer {
+  constructor ({ url }) {
+    this._url = url
+  }
+
+  flush (done = () => {}) {
+    const count = this._encoder.count()
+
+    if (count > 0) {
+      const payload = this._encoder.makePayload()
+
+      this._sendPayload(payload, count, done)
+    } else {
+      done()
+    }
+  }
+
+  append (spans) {
+    log.debug(() => `Encoding trace: ${JSON.stringify(spans)}`)
+
+    this._encode(spans)
+  }
+
+  _encode (trace) {
+    this._encoder.encode(trace)
+  }
+
+  setUrl (url) {
+    this._url = url
+  }
+}
+
+module.exports = Writer

package/packages/dd-trace/src/instrumenter.js

@@ -7,6 +7,7 @@ const Loader = require('./loader')
 const { isTrue } = require('./util')
 const plugins = require('./plugins')
 const Plugin = require('./plugins/plugin')
+const telemetry = require('./telemetry')
 
 const disabledPlugins = process.env.DD_TRACE_DISABLED_PLUGINS
 
@@ -54,6 +55,7 @@ class Instrumenter {
 
     try {
       this._set(plugin, { name, config })
+      telemetry.updateIntegrations()
     } catch (e) {
       log.debug(`Could not find a plugin named "${name}".`)
     }
@@ -154,6 +156,7 @@ class Instrumenter {
 
     if (!instrumented) {
       this._instrumented.set(instrumentation, instrumented = new Set())
+      telemetry.updateIntegrations()
     }
 
     if (!instrumented.has(this._defaultExport(moduleExports))) {

package/packages/dd-trace/src/pkg.js

@@ -11,7 +11,14 @@ function findRoot () {
 
 function findPkg () {
   const cwd = findRoot()
-  const filePath = findUp('package.json', cwd)
+  const directory = path.resolve(cwd)
+  const res = path.parse(directory)
+
+  if (!res) return {}
+
+  const { root } = res
+
+  const filePath = findUp('package.json', root, directory)
 
   try {
     return JSON.parse(fs.readFileSync(filePath, 'utf8'))
@@ -20,18 +27,16 @@ function findPkg () {
   }
 }
 
-function findUp (name, cwd) {
-  let directory = path.resolve(cwd)
-  const { root } = path.parse(directory)
-
+function findUp (name, root, directory) {
   while (true) {
     const current = path.resolve(directory, name)
 
     if (fs.existsSync(current)) return current
+
     if (directory === root) return
 
     directory = path.dirname(directory)
   }
 }
 
-module.exports = findPkg()
+module.exports = Object.assign(findPkg(), { findRoot, findUp })

package/packages/dd-trace/src/plugins/util/test.js

@@ -1,4 +1,7 @@
 const path = require('path')
+const fs = require('fs')
+
+const ignore = require('ignore')
 
 const { getGitMetadata } = require('./git')
 const { getUserProviderGitMetadata } = require('./user-provided-git')
@@ -25,6 +28,7 @@ const TEST_STATUS = 'test.status'
 const TEST_PARAMETERS = 'test.parameters'
 const TEST_SKIP_REASON = 'test.skip_reason'
 const TEST_IS_RUM_ACTIVE = 'test.is_rum_active'
+const TEST_CODE_OWNERS = 'test.codeowners'
 
 const ERROR_TYPE = 'error.type'
 const ERROR_MESSAGE = 'error.msg'
@@ -35,6 +39,7 @@ const CI_APP_ORIGIN = 'ciapp-test'
 const JEST_TEST_RUNNER = 'test.jest.test_runner'
 
 module.exports = {
+  TEST_CODE_OWNERS,
   TEST_FRAMEWORK,
   TEST_FRAMEWORK_VERSION,
   JEST_TEST_RUNNER,
@@ -53,7 +58,9 @@ module.exports = {
   getTestParametersString,
   finishAllTraceSpans,
   getTestParentSpan,
-  getTestSuitePath
+  getTestSuitePath,
+  getCodeOwnersFileEntries,
+  getCodeOwnersForFilename
 }
 
 function getTestEnvironmentMetadata (testFramework, config) {
@@ -140,3 +147,55 @@ function getTestSuitePath (testSuiteAbsolutePath, sourceRoot) {
 
   return testSuitePath.replace(path.sep, '/')
 }
+
+const POSSIBLE_CODEOWNERS_LOCATIONS = [
+  'CODEOWNERS',
+  '.github/CODEOWNERS',
+  'docs/CODEOWNERS',
+  '.gitlab/CODEOWNERS'
+]
+
+function getCodeOwnersFileEntries (rootDir = process.cwd()) {
+  let codeOwnersContent
+
+  POSSIBLE_CODEOWNERS_LOCATIONS.forEach(location => {
+    try {
+      codeOwnersContent = fs.readFileSync(`${rootDir}/${location}`).toString()
+    } catch (e) {
+      // retry with next path
+    }
+  })
+  if (!codeOwnersContent) {
+    return null
+  }
+
+  const entries = []
+  const lines = codeOwnersContent.split('\n')
+
+  for (const line of lines) {
+    const [content] = line.split('#')
+    const trimmed = content.trim()
+    if (trimmed === '') continue
+    const [pattern, ...owners] = trimmed.split(/\s+/)
+    entries.push({ pattern, owners })
+  }
+  // Reverse because rules defined last take precedence
+  return entries.reverse()
+}
+
+function getCodeOwnersForFilename (filename, entries) {
+  if (!entries) {
+    return null
+  }
+  for (const entry of entries) {
+    try {
+      const isResponsible = ignore().add(entry.pattern).ignores(filename)
+      if (isResponsible) {
+        return JSON.stringify(entry.owners)
+      }
+    } catch (e) {
+      return null
+    }
+  }
+  return null
+}

package/packages/dd-trace/src/profiling/exporters/agent.js

@@ -5,7 +5,7 @@ const { request } = require('http')
 const FormData = require('form-data')
 
 // TODO: avoid using dd-trace internals. Make this a separate module?
-const docker = require('../../exporters/agent/docker')
+const docker = require('../../exporters/common/docker')
 const version = require('../../../lib/version')
 
 const containerId = docker.id()