dd-trace 5.30.0 → 5.31.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "5.30.0",
+  "version": "5.31.0",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",
@@ -82,7 +82,7 @@
   },
   "dependencies": {
     "@datadog/libdatadog": "^0.3.0",
-    "@datadog/native-appsec": "8.3.0",
+    "@datadog/native-appsec": "8.4.0",
     "@datadog/native-iast-rewriter": "2.6.1",
     "@datadog/native-iast-taint-tracking": "3.2.0",
     "@datadog/native-metrics": "^3.1.0",
@@ -145,7 +145,7 @@
     "jszip": "^3.5.0",
     "knex": "^2.4.2",
     "mkdirp": "^3.0.1",
-    "mocha": "^9",
+    "mocha": "^10",
     "msgpack-lite": "^0.1.26",
     "multer": "^1.4.5-lts.1",
     "nock": "^11.3.3",

package/packages/datadog-core/src/storage.js

@@ -21,8 +21,16 @@ class DatadogStorage {
     this._storage.exit(callback, ...args)
   }
 
-  getStore () {
-    const handle = this._storage.getStore()
+  // TODO: Refactor the Scope class to use a span-only store and remove this.
+  getHandle () {
+    return this._storage.getStore()
+  }
+
+  getStore (handle) {
+    if (!handle) {
+      handle = this._storage.getStore()
+    }
+
     return stores.get(handle)
   }
 
@@ -50,6 +58,7 @@ const storage = function (namespace) {
 storage.disable = legacyStorage.disable.bind(legacyStorage)
 storage.enterWith = legacyStorage.enterWith.bind(legacyStorage)
 storage.exit = legacyStorage.exit.bind(legacyStorage)
+storage.getHandle = legacyStorage.getHandle.bind(legacyStorage)
 storage.getStore = legacyStorage.getStore.bind(legacyStorage)
 storage.run = legacyStorage.run.bind(legacyStorage)
 

package/packages/datadog-instrumentations/src/aerospike.js

@@ -40,7 +40,7 @@ function wrapProcess (process) {
 addHook({
   name: 'aerospike',
   file: 'lib/commands/command.js',
-  versions: ['4', '5']
+  versions: ['4', '5', '6']
 },
 commandFactory => {
   return shimmer.wrapFunction(commandFactory, f => wrapCreateCommand(f))

package/packages/dd-trace/src/appsec/remote_config/manager.js

@@ -9,6 +9,7 @@ const log = require('../../log')
 const { getExtraServices } = require('../../service-naming/extra-services')
 const { UNACKNOWLEDGED, ACKNOWLEDGED, ERROR } = require('./apply_states')
 const Scheduler = require('./scheduler')
+const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('../../plugins/util/tags')
 
 const clientId = uuid()
 
@@ -33,6 +34,14 @@ class RemoteConfigManager extends EventEmitter {
       port: config.port
     }))
 
+    const tags = config.repositoryUrl
+      ? {
+          ...config.tags,
+          [GIT_REPOSITORY_URL]: config.repositoryUrl,
+          [GIT_COMMIT_SHA]: config.commitSHA
+        }
+      : config.tags
+
     this._handlers = new Map()
     const appliedConfigs = this.appliedConfigs = new Map()
 
@@ -67,7 +76,8 @@ class RemoteConfigManager extends EventEmitter {
           service: config.service,
           env: config.env,
           app_version: config.version,
-          extra_services: []
+          extra_services: [],
+          tags: Object.entries(tags).map((pair) => pair.join(':'))
         },
         capabilities: DEFAULT_CAPABILITY // updated by `updateCapabilities()`
       },

package/packages/dd-trace/src/debugger/devtools_client/config.js

@@ -9,7 +9,8 @@ const config = module.exports = {
   service: parentConfig.service,
   commitSHA: parentConfig.commitSHA,
   repositoryUrl: parentConfig.repositoryUrl,
-  parentThreadId
+  parentThreadId,
+  maxTotalPayloadSize: 5 * 1024 * 1024 // 5MB
 }
 
 updateUrl(parentConfig)

package/packages/dd-trace/src/debugger/devtools_client/index.js

@@ -129,9 +129,8 @@ session.on('Debugger.paused', async ({ params }) => {
     }
 
     // TODO: Process template (DEBUG-2628)
-    send(probe.template, logger, dd, snapshot, (err) => {
-      if (err) log.error('Debugger error', err)
-      else ackEmitting(probe)
+    send(probe.template, logger, dd, snapshot, () => {
+      ackEmitting(probe)
     })
   }
 })
@@ -141,6 +140,8 @@ function highestOrUndefined (num, max) {
 }
 
 async function getDD (callFrameId) {
+  // TODO: Consider if an `objectGroup` should be used, so it can be explicitly released using
+  // `Runtime.releaseObjectGroup`
   const { result } = await session.post('Debugger.evaluateOnCallFrame', {
     callFrameId,
     expression,

package/packages/dd-trace/src/debugger/devtools_client/json-buffer.js

@@ -0,0 +1,36 @@
+'use strict'
+
+class JSONBuffer {
+  constructor ({ size, timeout, onFlush }) {
+    this._maxSize = size
+    this._timeout = timeout
+    this._onFlush = onFlush
+    this._reset()
+  }
+
+  _reset () {
+    clearTimeout(this._timer)
+    this._timer = null
+    this._partialJson = null
+  }
+
+  _flush () {
+    const json = `${this._partialJson}]`
+    this._reset()
+    this._onFlush(json)
+  }
+
+  write (str, size = Buffer.byteLength(str)) {
+    if (this._timer === null) {
+      this._partialJson = `[${str}`
+      this._timer = setTimeout(() => this._flush(), this._timeout)
+    } else if (Buffer.byteLength(this._partialJson) + size + 2 > this._maxSize) {
+      this._flush()
+      this.write(str, size)
+    } else {
+      this._partialJson += `,${str}`
+    }
+  }
+}
+
+module.exports = JSONBuffer

package/packages/dd-trace/src/debugger/devtools_client/send.js

@@ -4,32 +4,35 @@ const { hostname: getHostname } = require('os')
 const { stringify } = require('querystring')
 
 const config = require('./config')
+const JSONBuffer = require('./json-buffer')
 const request = require('../../exporters/common/request')
 const { GIT_COMMIT_SHA, GIT_REPOSITORY_URL } = require('../../plugins/util/tags')
+const log = require('../../log')
+const { version } = require('../../../../../package.json')
 
 module.exports = send
 
-const MAX_PAYLOAD_SIZE = 1024 * 1024 // 1MB
+const MAX_LOG_PAYLOAD_SIZE = 1024 * 1024 // 1MB
 
 const ddsource = 'dd_debugger'
 const hostname = getHostname()
 const service = config.service
 
 const ddtags = [
+  ['env', process.env.DD_ENV],
+  ['version', process.env.DD_VERSION],
+  ['debugger_version', version],
+  ['host_name', hostname],
   [GIT_COMMIT_SHA, config.commitSHA],
   [GIT_REPOSITORY_URL, config.repositoryUrl]
 ].map((pair) => pair.join(':')).join(',')
 
 const path = `/debugger/v1/input?${stringify({ ddtags })}`
 
-function send (message, logger, dd, snapshot, cb) {
-  const opts = {
-    method: 'POST',
-    url: config.url,
-    path,
-    headers: { 'Content-Type': 'application/json; charset=utf-8' }
-  }
+let callbacks = []
+const jsonBuffer = new JSONBuffer({ size: config.maxTotalPayloadSize, timeout: 1000, onFlush })
 
+function send (message, logger, dd, snapshot, cb) {
   const payload = {
     ddsource,
     hostname,
@@ -41,8 +44,9 @@ function send (message, logger, dd, snapshot, cb) {
   }
 
   let json = JSON.stringify(payload)
+  let size = Buffer.byteLength(json)
 
-  if (Buffer.byteLength(json) > MAX_PAYLOAD_SIZE) {
+  if (size > MAX_LOG_PAYLOAD_SIZE) {
     // TODO: This is a very crude way to handle large payloads. Proper pruning will be implemented later (DEBUG-2624)
     const line = Object.values(payload['debugger.snapshot'].captures.lines)[0]
     line.locals = {
@@ -50,7 +54,26 @@ function send (message, logger, dd, snapshot, cb) {
       size: Object.keys(line.locals).length
     }
     json = JSON.stringify(payload)
+    size = Buffer.byteLength(json)
+  }
+
+  jsonBuffer.write(json, size)
+  callbacks.push(cb)
+}
+
+function onFlush (payload) {
+  const opts = {
+    method: 'POST',
+    url: config.url,
+    path,
+    headers: { 'Content-Type': 'application/json; charset=utf-8' }
   }
 
-  request(json, opts, cb)
+  const _callbacks = callbacks
+  callbacks = []
+
+  request(payload, opts, (err) => {
+    if (err) log.error('Could not send debugger payload', err)
+    else _callbacks.forEach(cb => cb())
+  })
 }

package/packages/dd-trace/src/debugger/devtools_client/status.js

@@ -2,6 +2,7 @@
 
 const LRUCache = require('lru-cache')
 const config = require('./config')
+const JSONBuffer = require('./json-buffer')
 const request = require('../../exporters/common/request')
 const FormData = require('../../exporters/common/form-data')
 const log = require('../../log')
@@ -25,6 +26,8 @@ const cache = new LRUCache({
   ttlAutopurge: true
 })
 
+const jsonBuffer = new JSONBuffer({ size: config.maxTotalPayloadSize, timeout: 1000, onFlush })
+
 const STATUSES = {
   RECEIVED: 'RECEIVED',
   INSTALLED: 'INSTALLED',
@@ -71,11 +74,15 @@ function ackError (err, { id: probeId, version }) {
 }
 
 function send (payload) {
+  jsonBuffer.write(JSON.stringify(payload))
+}
+
+function onFlush (payload) {
   const form = new FormData()
 
   form.append(
     'event',
-    JSON.stringify(payload),
+    payload,
     { filename: 'event.json', contentType: 'application/json; charset=utf-8' }
   )
 
@@ -87,7 +94,7 @@ function send (payload) {
   }
 
   request(form, options, (err) => {
-    if (err) log.error('[debugger:devtools_client] Error sending debugger payload', err)
+    if (err) log.error('[debugger:devtools_client] Error sending probe payload', err)
   })
 }
 

package/packages/dd-trace/src/llmobs/sdk.js

@@ -1,12 +1,12 @@
 'use strict'
 
-const { SPAN_KIND, OUTPUT_VALUE } = require('./constants/tags')
+const { SPAN_KIND, OUTPUT_VALUE, INPUT_VALUE } = require('./constants/tags')
 
 const {
   getFunctionArguments,
   validateKind
 } = require('./util')
-const { isTrue } = require('../util')
+const { isTrue, isError } = require('../util')
 
 const { storage } = require('./storage')
 
@@ -134,29 +134,63 @@ class LLMObs extends NoopLLMObs {
 
     function wrapped () {
       const span = llmobs._tracer.scope().active()
-
-      const result = llmobs._activate(span, { kind, options: llmobsOptions }, () => {
-        if (!['llm', 'embedding'].includes(kind)) {
-          llmobs.annotate(span, { inputData: getFunctionArguments(fn, arguments) })
+      const fnArgs = arguments
+
+      const lastArgId = fnArgs.length - 1
+      const cb = fnArgs[lastArgId]
+      const hasCallback = typeof cb === 'function'
+
+      if (hasCallback) {
+        const scopeBoundCb = llmobs._bind(cb)
+        fnArgs[lastArgId] = function () {
+          // it is standard practice to follow the callback signature (err, result)
+          // however, we try to parse the arguments to determine if the first argument is an error
+          // if it is not, and is not undefined, we will use that for the output value
+          const maybeError = arguments[0]
+          const maybeResult = arguments[1]
+
+          llmobs._autoAnnotate(
+            span,
+            kind,
+            getFunctionArguments(fn, fnArgs),
+            isError(maybeError) || maybeError == null ? maybeResult : maybeError
+          )
+
+          return scopeBoundCb.apply(this, arguments)
         }
+      }
 
-        return fn.apply(this, arguments)
-      })
+      try {
+        const result = llmobs._activate(span, { kind, options: llmobsOptions }, () => fn.apply(this, fnArgs))
+
+        if (result && typeof result.then === 'function') {
+          return result.then(
+            value => {
+              if (!hasCallback) {
+                llmobs._autoAnnotate(span, kind, getFunctionArguments(fn, fnArgs), value)
+              }
+              return value
+            },
+            err => {
+              llmobs._autoAnnotate(span, kind, getFunctionArguments(fn, fnArgs))
+              throw err
+            }
+          )
+        }
 
-      if (result && typeof result.then === 'function') {
-        return result.then(value => {
-          if (value && !['llm', 'retrieval'].includes(kind) && !LLMObsTagger.tagMap.get(span)?.[OUTPUT_VALUE]) {
-            llmobs.annotate(span, { outputData: value })
-          }
-          return value
-        })
-      }
+        // it is possible to return a value and have a callback
+        // however, since the span finishes when the callback is called, it is possible that
+        // the callback is called before the function returns (although unlikely)
+        // we do not want to throw for "annotating a finished span" in this case
+        if (!hasCallback) {
+          llmobs._autoAnnotate(span, kind, getFunctionArguments(fn, fnArgs), result)
+        }
 
-      if (result && !['llm', 'retrieval'].includes(kind) && !LLMObsTagger.tagMap.get(span)?.[OUTPUT_VALUE]) {
-        llmobs.annotate(span, { outputData: result })
+        return result
+      } catch (e) {
+        llmobs._autoAnnotate(span, kind, getFunctionArguments(fn, fnArgs))
+        throw e
       }
-
-      return result
     }
 
     return this._tracer.wrap(name, spanOptions, wrapped)
@@ -333,20 +367,34 @@ class LLMObs extends NoopLLMObs {
     flushCh.publish()
   }
 
+  _autoAnnotate (span, kind, input, output) {
+    const annotations = {}
+    if (input && !['llm', 'embedding'].includes(kind) && !LLMObsTagger.tagMap.get(span)?.[INPUT_VALUE]) {
+      annotations.inputData = input
+    }
+
+    if (output && !['llm', 'retrieval'].includes(kind) && !LLMObsTagger.tagMap.get(span)?.[OUTPUT_VALUE]) {
+      annotations.outputData = output
+    }
+
+    this.annotate(span, annotations)
+  }
+
   _active () {
     const store = storage.getStore()
     return store?.span
   }
 
-  _activate (span, { kind, options } = {}, fn) {
+  _activate (span, options, fn) {
     const parent = this._active()
     if (this.enabled) storage.enterWith({ span })
 
-    this._tagger.registerLLMObsSpan(span, {
-      ...options,
-      parent,
-      kind
-    })
+    if (options) {
+      this._tagger.registerLLMObsSpan(span, {
+        ...options,
+        parent
+      })
+    }
 
     try {
       return fn()
@@ -355,6 +403,22 @@ class LLMObs extends NoopLLMObs {
     }
   }
 
+  // bind function to active LLMObs span
+  _bind (fn) {
+    if (typeof fn !== 'function') return fn
+
+    const llmobs = this
+    const activeSpan = llmobs._active()
+
+    const bound = function () {
+      return llmobs._activate(activeSpan, null, () => {
+        return fn.apply(this, arguments)
+      })
+    }
+
+    return bound
+  }
+
   _extractOptions (options) {
     const {
       modelName,

package/packages/dd-trace/src/log/index.js

@@ -63,15 +63,14 @@ const log = {
 
       Error.captureStackTrace(logRecord, this.trace)
 
-      const fn = logRecord.stack.split('\n')[1].replace(/^\s+at ([^\s]+) .+/, '$1')
-      const params = args.map(a => {
-        return a && a.hasOwnProperty('toString') && typeof a.toString === 'function'
-          ? a.toString()
-          : inspect(a, { depth: 3, breakLength: Infinity, compact: true })
-      }).join(', ')
-      const formatted = logRecord.stack.replace('Error: ', `Trace: ${fn}(${params})`)
-
-      traceChannel.publish(Log.parse(formatted))
+      const stack = logRecord.stack.split('\n')
+      const fn = stack[1].replace(/^\s+at ([^\s]+) .+/, '$1')
+      const options = { depth: 2, breakLength: Infinity, compact: true, maxArrayLength: Infinity }
+      const params = args.map(a => inspect(a, options)).join(', ')
+
+      stack[0] = `Trace: ${fn}(${params})`
+
+      traceChannel.publish(Log.parse(stack.join('\n')))
     }
     return this
   },

package/packages/dd-trace/src/noop/proxy.js

@@ -10,7 +10,7 @@ const noopAppsec = new NoopAppsecSdk()
 const noopDogStatsDClient = new NoopDogStatsDClient()
 const noopLLMObs = new NoopLLMObsSDK(noop)
 
-class Tracer {
+class NoopProxy {
   constructor () {
     this._tracer = noop
     this.appsec = noopAppsec
@@ -91,4 +91,4 @@ class Tracer {
   }
 }
 
-module.exports = Tracer
+module.exports = NoopProxy

package/packages/dd-trace/src/noop/span.js

@@ -6,7 +6,7 @@ const { storage } = require('../../../datadog-core') // TODO: noop storage?
 
 class NoopSpan {
   constructor (tracer, parent) {
-    this._store = storage.getStore()
+    this._store = storage.getHandle()
     this._noopTracer = tracer
     this._noopContext = this._createContext(parent)
   }

package/packages/dd-trace/src/opentracing/span.js

@@ -14,6 +14,7 @@ const { storage } = require('../../../datadog-core')
 const telemetryMetrics = require('../telemetry/metrics')
 const { channel } = require('dc-polyfill')
 const spanleak = require('../spanleak')
+const util = require('util')
 
 const tracerMetrics = telemetryMetrics.manager.namespace('tracers')
 
@@ -64,7 +65,7 @@ class DatadogSpan {
     this._debug = debug
     this._processor = processor
     this._prioritySampler = prioritySampler
-    this._store = storage.getStore()
+    this._store = storage.getHandle()
     this._duration = undefined
 
     this._events = []
@@ -105,6 +106,15 @@ class DatadogSpan {
     }
   }
 
+  [util.inspect.custom] () {
+    return {
+      ...this,
+      _parentTracer: `[${this._parentTracer.constructor.name}]`,
+      _prioritySampler: `[${this._prioritySampler.constructor.name}]`,
+      _processor: `[${this._processor.constructor.name}]`
+    }
+  }
+
   toString () {
     const spanContext = this.context()
     const resourceName = spanContext._tags['resource.name'] || ''

package/packages/dd-trace/src/opentracing/span_context.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const util = require('util')
 const { AUTO_KEEP } = require('../../../../ext/priority')
 
 // the lowercase, hex encoded upper 64 bits of a 128-bit trace id, if present
@@ -31,6 +32,17 @@ class DatadogSpanContext {
     this._otelSpanContext = undefined
   }
 
+  [util.inspect.custom] () {
+    return {
+      ...this,
+      _trace: {
+        ...this._trace,
+        started: '[Array]',
+        finished: '[Array]'
+      }
+    }
+  }
+
   toTraceId (get128bitId = false) {
     if (get128bitId) {
       return this._traceId.toBuffer().length <= 8 && this._trace.tags[TRACE_ID_128]

package/packages/dd-trace/src/priority_sampler.js

@@ -120,13 +120,15 @@ class PrioritySampler {
     if (!span || !this.validate(samplingPriority)) return
 
     const context = this._getContext(span)
+    const root = context._trace.started[0]
+
+    if (!root) return // noop span
 
     context._sampling.priority = samplingPriority
     context._sampling.mechanism = mechanism
 
-    const root = context._trace.started[0]
-
     log.trace(span, samplingPriority, mechanism)
+
     this._addDecisionMaker(root)
   }
 

package/packages/dd-trace/src/scope.js

@@ -17,7 +17,7 @@ class Scope {
     if (typeof callback !== 'function') return callback
 
     const oldStore = storage.getStore()
-    const newStore = span ? span._store : oldStore
+    const newStore = span ? storage.getStore(span._store) : oldStore
 
     storage.enterWith({ ...newStore, span })
 

package/packages/dd-trace/src/telemetry/index.js

@@ -307,6 +307,8 @@ function updateConfig (changes, config) {
   if (!config.telemetry.enabled) return
   if (changes.length === 0) return
 
+  logger.trace(changes)
+
   const application = createAppObject(config)
   const host = createHostObject()