dd-trace 2.11.0 → 2.12.2

package/packages/dd-trace/src/encode/agentless-ci-visibility.js

@@ -3,13 +3,25 @@ const { truncateSpan, normalizeSpan } = require('./tags-processors')
 const Chunk = require('./chunk')
 const { AgentEncoder } = require('./0.4')
 const { version: ddTraceVersion } = require('../../../../package.json')
+const id = require('../../../dd-trace/src/id')
 
 const ENCODING_VERSION = 1
 
+const ALLOWED_CONTENT_TYPES = ['test_session_end', 'test_suite_end', 'test']
+
+const TEST_SUITE_KEYS_LENGTH = 11
+const TEST_SESSION_KEYS_LENGTH = 10
+
+const CHUNK_SIZE = 4 * 1024 * 1024 // 4MB
+
 function formatSpan (span) {
+  let encodingVersion = ENCODING_VERSION
+  if (span.type === 'test' && span.meta && span.meta.test_session_id) {
+    encodingVersion = 2
+  }
   return {
-    type: span.type === 'test' ? 'test' : 'span',
-    version: ENCODING_VERSION,
+    type: ALLOWED_CONTENT_TYPES.includes(span.type) ? span.type : 'span',
+    version: encodingVersion,
     content: normalizeSpan(truncateSpan(span))
   }
 }
@@ -21,8 +33,8 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     this.runtimeId = runtimeId
     this.service = service
     this.env = env
-    this._traceBytes = new Chunk()
-    this._stringBytes = new Chunk()
+    this._traceBytes = new Chunk(CHUNK_SIZE)
+    this._stringBytes = new Chunk(CHUNK_SIZE)
     this._stringCount = 0
     this._stringMap = {}
 
@@ -33,8 +45,69 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     this.reset()
   }
 
+  _encodeTestSuite (bytes, content) {
+    this._encodeMapPrefix(bytes, TEST_SUITE_KEYS_LENGTH)
+    this._encodeString(bytes, 'type')
+    this._encodeString(bytes, content.type)
+
+    this._encodeString(bytes, 'test_session_id')
+    this._encodeId(bytes, content.trace_id)
+
+    this._encodeString(bytes, 'test_suite_id')
+    this._encodeId(bytes, content.span_id)
+
+    this._encodeString(bytes, 'error')
+    this._encodeNumber(bytes, content.error)
+    this._encodeString(bytes, 'name')
+    this._encodeString(bytes, content.name)
+    this._encodeString(bytes, 'service')
+    this._encodeString(bytes, content.service)
+    this._encodeString(bytes, 'resource')
+    this._encodeString(bytes, content.resource)
+    this._encodeString(bytes, 'start')
+    this._encodeNumber(bytes, content.start)
+    this._encodeString(bytes, 'duration')
+    this._encodeNumber(bytes, content.duration)
+    this._encodeString(bytes, 'meta')
+    this._encodeMap(bytes, content.meta)
+    this._encodeString(bytes, 'metrics')
+    this._encodeMap(bytes, content.metrics)
+  }
+
+  _encodeTestSession (bytes, content) {
+    this._encodeMapPrefix(bytes, TEST_SESSION_KEYS_LENGTH)
+    this._encodeString(bytes, 'type')
+    this._encodeString(bytes, content.type)
+
+    this._encodeString(bytes, 'test_session_id')
+    this._encodeId(bytes, content.trace_id)
+
+    this._encodeString(bytes, 'error')
+    this._encodeNumber(bytes, content.error)
+    this._encodeString(bytes, 'name')
+    this._encodeString(bytes, content.name)
+    this._encodeString(bytes, 'service')
+    this._encodeString(bytes, content.service)
+    this._encodeString(bytes, 'resource')
+    this._encodeString(bytes, content.resource)
+    this._encodeString(bytes, 'start')
+    this._encodeNumber(bytes, content.start)
+    this._encodeString(bytes, 'duration')
+    this._encodeNumber(bytes, content.duration)
+    this._encodeString(bytes, 'meta')
+    this._encodeMap(bytes, content.meta)
+    this._encodeString(bytes, 'metrics')
+    this._encodeMap(bytes, content.metrics)
+  }
+
   _encodeEventContent (bytes, content) {
-    this._encodeMapPrefix(bytes, content)
+    const keysLength = Object.keys(content).length
+    if (content.meta.test_session_id) {
+      this._encodeMapPrefix(bytes, keysLength + 2)
+    } else {
+      this._encodeMapPrefix(bytes, keysLength)
+    }
+
     if (content.type) {
       this._encodeString(bytes, 'type')
       this._encodeString(bytes, content.type)
@@ -57,6 +130,24 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     this._encodeNumber(bytes, content.start)
     this._encodeString(bytes, 'duration')
     this._encodeNumber(bytes, content.duration)
+    /**
+     * We include `test_session_id` and `test_suite_id`
+     * in the root of the event by passing them via the `meta` dict.
+     * This is to avoid changing the span format in packages/dd-trace/src/format.js,
+     * which can have undesired side effects in other products.
+     * But `test_session_id` and `test_suite_id` are *not* supposed to be in `meta`,
+     * so we delete them before enconding the dictionary.
+     * TODO: find a better way to do this.
+     */
+    if (content.meta.test_session_id) {
+      this._encodeString(bytes, 'test_session_id')
+      this._encodeId(bytes, id(content.meta.test_session_id))
+      delete content.meta.test_session_id
+
+      this._encodeString(bytes, 'test_suite_id')
+      this._encodeId(bytes, id(content.meta.test_suite_id))
+      delete content.meta.test_suite_id
+    }
     this._encodeString(bytes, 'meta')
     this._encodeMap(bytes, content.meta)
     this._encodeString(bytes, 'metrics')
@@ -64,7 +155,7 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
   }
 
   _encodeEvent (bytes, event) {
-    this._encodeMapPrefix(bytes, event)
+    this._encodeMapPrefix(bytes, Object.keys(event).length)
     this._encodeString(bytes, 'type')
     this._encodeString(bytes, event.type)
 
@@ -72,7 +163,13 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     this._encodeNumber(bytes, event.version)
 
     this._encodeString(bytes, 'content')
-    this._encodeEventContent(bytes, event.content)
+    if (event.type === 'span' || event.type === 'test') {
+      this._encodeEventContent(bytes, event.content)
+    } else if (event.type === 'test_suite_end') {
+      this._encodeTestSuite(bytes, event.content)
+    } else if (event.type === 'test_session_end') {
+      this._encodeTestSession(bytes, event.content)
+    }
   }
 
   _encodeNumber (bytes, value) {
@@ -107,18 +204,17 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     buffer[offset + 8] = lo
   }
 
-  _encodeMapPrefix (bytes, map) {
-    const keys = Object.keys(map)
+  _encodeMapPrefix (bytes, keysLength) {
     const buffer = bytes.buffer
     const offset = bytes.length
 
     bytes.reserve(5)
     bytes.length += 5
     buffer[offset] = 0xdf
-    buffer[offset + 1] = keys.length >> 24
-    buffer[offset + 2] = keys.length >> 16
-    buffer[offset + 3] = keys.length >> 8
-    buffer[offset + 4] = keys.length
+    buffer[offset + 1] = keysLength >> 24
+    buffer[offset + 2] = keysLength >> 16
+    buffer[offset + 3] = keysLength >> 8
+    buffer[offset + 4] = keysLength
   }
 
   _encode (bytes, trace) {
@@ -171,11 +267,11 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
       payload.metadata['*']['runtime-id'] = this.runtimeId
     }
 
-    this._encodeMapPrefix(bytes, payload)
+    this._encodeMapPrefix(bytes, Object.keys(payload).length)
     this._encodeString(bytes, 'version')
     this._encodeNumber(bytes, payload.version)
     this._encodeString(bytes, 'metadata')
-    this._encodeMapPrefix(bytes, payload.metadata)
+    this._encodeMapPrefix(bytes, Object.keys(payload.metadata).length)
     this._encodeString(bytes, '*')
     this._encodeMap(bytes, payload.metadata['*'])
     this._encodeString(bytes, 'events')

package/packages/dd-trace/src/exporters/common/request.js

@@ -1,22 +1,33 @@
 'use strict'
 
+// TODO: Add test with slow or unresponsive agent.
+// TODO: Add telemetry for things like dropped requests, errors, etc.
+
 const http = require('http')
 const https = require('https')
-const log = require('../../log')
 const docker = require('./docker')
 const { storage } = require('../../../../datadog-core')
 
-const httpAgent = new http.Agent({ keepAlive: true })
-const httpsAgent = new https.Agent({ keepAlive: true })
+const keepAlive = true
+const maxTotalSockets = 1
+const maxActiveRequests = 8
+const httpAgent = new http.Agent({ keepAlive, maxTotalSockets })
+const httpsAgent = new https.Agent({ keepAlive, maxTotalSockets })
 const containerId = docker.id()
 
+let activeRequests = 0
+
 function request (data, options, keepAlive, callback) {
   if (!options.headers) {
     options.headers = {}
   }
+
+  // The timeout should be kept low to avoid excessive queueing.
+  const timeout = options.timeout || 2000
   const isSecure = options.protocol === 'https:'
   const client = isSecure ? https : http
   const dataArray = [].concat(data)
+
   options.headers['Content-Length'] = byteLength(dataArray)
 
   if (containerId) {
@@ -27,39 +38,15 @@ function request (data, options, keepAlive, callback) {
     options.agent = isSecure ? httpsAgent : httpAgent
   }
 
-  const firstRequest = retriableRequest(options, client, callback)
-  dataArray.forEach(buffer => firstRequest.write(buffer))
-
-  // The first request will be retried
-  const firstRequestErrorHandler = () => {
-    log.debug('Retrying request to the intake')
-    const retriedReq = retriableRequest(options, client, callback)
-    dataArray.forEach(buffer => retriedReq.write(buffer))
-    // The retried request will fail normally
-    retriedReq.on('error', e => callback(new Error(`Network error trying to reach the intake: ${e.message}`)))
-    retriedReq.end()
-  }
-
-  firstRequest.on('error', firstRequestErrorHandler)
-  firstRequest.end()
-
-  return firstRequest
-}
-
-function retriableRequest (options, client, callback) {
-  const store = storage.getStore()
-
-  storage.enterWith({ noop: true })
-
-  const timeout = options.timeout || 15000
-
-  const request = client.request(options, res => {
+  const onResponse = res => {
     let responseData = ''
 
     res.setTimeout(timeout)
 
     res.on('data', chunk => { responseData += chunk })
     res.on('end', () => {
+      activeRequests--
+
       if (res.statusCode >= 200 && res.statusCode <= 299) {
         callback(null, responseData, res.statusCode)
       } else {
@@ -69,15 +56,43 @@ function retriableRequest (options, client, callback) {
         callback(error, null, res.statusCode)
       }
     })
-  })
-  request.setTimeout(timeout, request.abort)
-  storage.enterWith(store)
+  }
 
-  return request
+  const makeRequest = onError => {
+    if (!request.writable) return callback(null)
+
+    activeRequests++
+
+    const store = storage.getStore()
+
+    storage.enterWith({ noop: true })
+
+    const req = client.request(options, onResponse)
+
+    req.once('error', err => {
+      activeRequests--
+      onError(err)
+    })
+
+    dataArray.forEach(buffer => req.write(buffer))
+
+    req.setTimeout(timeout, req.abort)
+    req.end()
+
+    storage.enterWith(store)
+  }
+
+  makeRequest(() => makeRequest(callback))
 }
 
 function byteLength (data) {
   return data.length > 0 ? data.reduce((prev, next) => prev + next.length, 0) : 0
 }
 
+Object.defineProperty(request, 'writable', {
+  get () {
+    return activeRequests < maxActiveRequests
+  }
+})
+
 module.exports = request

package/packages/dd-trace/src/exporters/common/writer.js

@@ -1,4 +1,6 @@
 'use strict'
+
+const request = require('./request')
 const log = require('../../log')
 
 class Writer {
@@ -9,7 +11,10 @@ class Writer {
   flush (done = () => {}) {
     const count = this._encoder.count()
 
-    if (count > 0) {
+    if (!request.writable) {
+      this._encoder.reset()
+      done()
+    } else if (count > 0) {
       const payload = this._encoder.makePayload()
 
       this._sendPayload(payload, count, done)
@@ -19,6 +24,8 @@ class Writer {
   }
 
   append (spans) {
+    if (!request.writable) return
+
     log.debug(() => `Encoding trace: ${JSON.stringify(spans)}`)
 
     this._encode(spans)

package/packages/dd-trace/src/noop/span.js

@@ -1,26 +1,26 @@
 'use strict'
 
-const Span = require('opentracing').Span
-const NoopSpanContext = require('../noop/span_context')
+const NoopSpanContext = require('./span_context')
 const id = require('../id')
 const { storage } = require('../../../datadog-core') // TODO: noop storage?
 
-class NoopSpan extends Span {
+class NoopSpan {
   constructor (tracer, parent) {
-    super()
-
     this._store = storage.getStore()
     this._noopTracer = tracer
     this._noopContext = this._createContext(parent)
   }
 
-  _context () {
-    return this._noopContext
-  }
-
-  _tracer () {
-    return this._noopTracer
-  }
+  context () { return this._noopContext }
+  tracer () { return this._noopTracer }
+  setOperationName (name) { return this }
+  setBaggageItem (key, value) { return this }
+  getBaggageItem (key) {}
+  setTag (key, value) { return this }
+  addTags (keyValueMap) { return this }
+  log () { return this }
+  logEvent () {}
+  finish (finishTime) {}
 
   _createContext (parent) {
     const spanId = id()

package/packages/dd-trace/src/noop/tracer.js

@@ -1,13 +1,10 @@
 'use strict'
 
-const Tracer = require('opentracing').Tracer
 const Scope = require('../noop/scope')
 const Span = require('./span')
 
-class NoopTracer extends Tracer {
+class NoopTracer {
   constructor (config) {
-    super(config)
-
     this._scope = new Scope()
     this._span = new Span(this)
   }
@@ -35,10 +32,16 @@ class NoopTracer extends Tracer {
   setUrl () {
   }
 
-  _startSpan (name, options) {
+  startSpan (name, options) {
     return this._span
   }
 
+  inject (spanContext, format, carrier) {}
+
+  extract (format, carrier) {
+    return this._span.context()
+  }
+
   setUser () {
     return this
   }

package/packages/dd-trace/src/opentracing/span.js

@@ -1,11 +1,9 @@
 'use strict'
 
 // TODO (new internal tracer): use DC events for lifecycle metrics and test them
-
-const opentracing = require('opentracing')
-const now = require('performance-now')
+const now = require('perf_hooks').performance.now
+const dateNow = Date.now
 const semver = require('semver')
-const Span = opentracing.Span
 const SpanContext = require('./span_context')
 const id = require('../id')
 const tagger = require('../tagger')
@@ -21,10 +19,8 @@ const {
 const unfinishedRegistry = createRegistry('unfinished')
 const finishedRegistry = createRegistry('finished')
 
-class DatadogSpan extends Span {
+class DatadogSpan {
   constructor (tracer, processor, prioritySampler, fields, debug) {
-    super()
-
     const operationName = fields.operationName
     const parent = fields.parent || null
     const tags = Object.assign({}, fields.tags)
@@ -70,66 +66,45 @@ class DatadogSpan extends Span {
     return `Span${json}`
   }
 
-  _createContext (parent) {
-    let spanContext
-
-    if (parent) {
-      spanContext = new SpanContext({
-        traceId: parent._traceId,
-        spanId: id(),
-        parentId: parent._spanId,
-        sampling: parent._sampling,
-        baggageItems: Object.assign({}, parent._baggageItems),
-        trace: parent._trace
-      })
-    } else {
-      const spanId = id()
-      spanContext = new SpanContext({
-        traceId: spanId,
-        spanId
-      })
-    }
-
-    spanContext._trace.started.push(this)
-    spanContext._trace.startTime = spanContext._trace.startTime || Date.now()
-    spanContext._trace.ticks = spanContext._trace.ticks || now()
-
-    return spanContext
-  }
-
-  _getTime () {
-    const { startTime, ticks } = this._spanContext._trace
-
-    return startTime + now() - ticks
-  }
-
-  _context () {
+  context () {
     return this._spanContext
   }
 
-  _tracer () {
+  tracer () {
     return this._parentTracer
   }
 
-  _setOperationName (name) {
+  setOperationName (name) {
     this._spanContext._name = name
+    return this
   }
 
-  _setBaggageItem (key, value) {
+  setBaggageItem (key, value) {
     this._spanContext._baggageItems[key] = value
+    return this
   }
 
-  _getBaggageItem (key) {
+  getBaggageItem (key) {
     return this._spanContext._baggageItems[key]
   }
 
-  _addTags (keyValuePairs) {
-    tagger.add(this._spanContext._tags, keyValuePairs)
+  setTag (key, value) {
+    this._addTags({ [key]: value })
+    return this
+  }
 
-    this._prioritySampler.sample(this, false)
+  addTags (keyValueMap) {
+    this._addTags(keyValueMap)
+    return this
+  }
+
+  log () {
+    return this
   }
 
-  _finish (finishTime) {
+  logEvent () {}
+
+  finish (finishTime) {
     if (this._duration !== undefined) {
       return
     }
@@ -157,6 +132,45 @@ class DatadogSpan extends Span {
     this._spanContext._isFinished = true
     this._processor.process(this)
   }
+
+  _createContext (parent) {
+    let spanContext
+
+    if (parent) {
+      spanContext = new SpanContext({
+        traceId: parent._traceId,
+        spanId: id(),
+        parentId: parent._spanId,
+        sampling: parent._sampling,
+        baggageItems: Object.assign({}, parent._baggageItems),
+        trace: parent._trace
+      })
+    } else {
+      const spanId = id()
+      spanContext = new SpanContext({
+        traceId: spanId,
+        spanId
+      })
+    }
+
+    spanContext._trace.started.push(this)
+    spanContext._trace.startTime = spanContext._trace.startTime || dateNow()
+    spanContext._trace.ticks = spanContext._trace.ticks || now()
+
+    return spanContext
+  }
+
+  _getTime () {
+    const { startTime, ticks } = this._spanContext._trace
+
+    return startTime + now() - ticks
+  }
+
+  _addTags (keyValuePairs) {
+    tagger.add(this._spanContext._tags, keyValuePairs)
+
+    this._prioritySampler.sample(this, false)
+  }
 }
 
 function createRegistry (type) {

package/packages/dd-trace/src/opentracing/span_context.js

@@ -1,11 +1,7 @@
 'use strict'
 
-const SpanContext = require('opentracing').SpanContext
-
-class DatadogSpanContext extends SpanContext {
+class DatadogSpanContext {
   constructor (props) {
-    super()
-
     props = props || {}
 
     this._traceId = props.traceId