dd-trace 5.32.0 → 5.33.1

package/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js

@@ -6,6 +6,8 @@ const { IAST_ENABLED_TAG_KEY, IAST_JSON_TAG_KEY } = require('./tags')
 const standalone = require('../standalone')
 const { SAMPLING_MECHANISM_APPSEC } = require('../../constants')
 const { keepTrace } = require('../../priority_sampler')
+const { reportStackTrace, getCallsiteFrames, canReportStackTrace, STACK_TRACE_NAMESPACES } = require('../stack_trace')
+const { getOriginalPathAndLineFromSourceMap } = require('./taint-tracking/rewriter')
 
 const VULNERABILITIES_KEY = 'vulnerabilities'
 const VULNERABILITY_HASHES_MAX_SIZE = 1000
@@ -15,39 +17,60 @@ const RESET_VULNERABILITY_CACHE_INTERVAL = 60 * 60 * 1000 // 1 hour
 let tracer
 let resetVulnerabilityCacheTimer
 let deduplicationEnabled = true
+let stackTraceEnabled = true
+let stackTraceMaxDepth
+let maxStackTraces
 
-function addVulnerability (iastContext, vulnerability) {
-  if (vulnerability?.evidence && vulnerability?.type && vulnerability?.location) {
-    if (deduplicationEnabled && isDuplicatedVulnerability(vulnerability)) return
+function canAddVulnerability (vulnerability) {
+  const hasRequiredFields = vulnerability?.evidence && vulnerability?.type && vulnerability?.location
+  if (!hasRequiredFields) return false
 
-    VULNERABILITY_HASHES.set(`${vulnerability.type}${vulnerability.hash}`, true)
+  const isDuplicated = deduplicationEnabled && isDuplicatedVulnerability(vulnerability)
 
-    let span = iastContext?.rootSpan
+  return !isDuplicated
+}
 
-    if (!span && tracer) {
-      span = tracer.startSpan('vulnerability', {
-        type: 'vulnerability'
-      })
+function addVulnerability (iastContext, vulnerability, callSiteFrames) {
+  if (!canAddVulnerability(vulnerability)) return
 
-      vulnerability.location.spanId = span.context().toSpanId()
+  VULNERABILITY_HASHES.set(`${vulnerability.type}${vulnerability.hash}`, true)
 
-      span.addTags({
-        [IAST_ENABLED_TAG_KEY]: 1
-      })
-    }
+  let span = iastContext?.rootSpan
 
-    if (!span) return
+  if (!span && tracer) {
+    span = tracer.startSpan('vulnerability', {
+      type: 'vulnerability'
+    })
 
-    keepTrace(span, SAMPLING_MECHANISM_APPSEC)
-    standalone.sample(span)
+    vulnerability.location.spanId = span.context().toSpanId()
 
-    if (iastContext?.rootSpan) {
-      iastContext[VULNERABILITIES_KEY] = iastContext[VULNERABILITIES_KEY] || []
-      iastContext[VULNERABILITIES_KEY].push(vulnerability)
-    } else {
-      sendVulnerabilities([vulnerability], span)
-      span.finish()
-    }
+    span.addTags({
+      [IAST_ENABLED_TAG_KEY]: 1
+    })
+  }
+
+  if (!span) return
+
+  keepTrace(span, SAMPLING_MECHANISM_APPSEC)
+  standalone.sample(span)
+
+  if (stackTraceEnabled && canReportStackTrace(span, maxStackTraces, STACK_TRACE_NAMESPACES.IAST)) {
+    const originalCallSiteList = callSiteFrames.map(callsite => replaceCallSiteFromSourceMap(callsite))
+
+    reportStackTrace(
+      span,
+      vulnerability.stackId,
+      originalCallSiteList,
+      STACK_TRACE_NAMESPACES.IAST
+    )
+  }
+
+  if (iastContext?.rootSpan) {
+    iastContext[VULNERABILITIES_KEY] = iastContext[VULNERABILITIES_KEY] || []
+    iastContext[VULNERABILITIES_KEY].push(vulnerability)
+  } else {
+    sendVulnerabilities([vulnerability], span)
+    span.finish()
   }
 }
 
@@ -94,8 +117,34 @@ function isDuplicatedVulnerability (vulnerability) {
   return VULNERABILITY_HASHES.get(`${vulnerability.type}${vulnerability.hash}`)
 }
 
+function getVulnerabilityCallSiteFrames () {
+  return getCallsiteFrames(stackTraceMaxDepth)
+}
+
+function replaceCallSiteFromSourceMap (callsite) {
+  if (callsite) {
+    const { path, line, column } = getOriginalPathAndLineFromSourceMap(callsite)
+    if (path) {
+      callsite.file = path
+      callsite.path = path
+    }
+    if (line) {
+      callsite.line = line
+    }
+    if (column) {
+      callsite.column = column
+    }
+  }
+
+  return callsite
+}
+
 function start (config, _tracer) {
   deduplicationEnabled = config.iast.deduplicationEnabled
+  stackTraceEnabled = config.iast.stackTrace.enabled
+  stackTraceMaxDepth = config.appsec.stackTrace.maxDepth
+  maxStackTraces = config.appsec.stackTrace.maxStackTraces
+
   vulnerabilitiesFormatter.setRedactVulnerabilities(
     config.iast.redactionEnabled,
     config.iast.redactionNamePattern,
@@ -114,6 +163,8 @@ function stop () {
 module.exports = {
   addVulnerability,
   sendVulnerabilities,
+  getVulnerabilityCallSiteFrames,
+  replaceCallSiteFromSourceMap,
   clearCache,
   start,
   stop

package/packages/dd-trace/src/appsec/rasp/utils.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const web = require('../../plugins/util/web')
-const { reportStackTrace } = require('../stack_trace')
+const { getCallsiteFrames, reportStackTrace, canReportStackTrace } = require('../stack_trace')
 const { getBlockingAction } = require('../blocking')
 const log = require('../../log')
 
@@ -30,13 +30,18 @@ class DatadogRaspAbortError extends Error {
 
 function handleResult (actions, req, res, abortController, config) {
   const generateStackTraceAction = actions?.generate_stack
-  if (generateStackTraceAction && config.appsec.stackTrace.enabled) {
-    const rootSpan = web.root(req)
+
+  const { enabled, maxDepth, maxStackTraces } = config.appsec.stackTrace
+
+  const rootSpan = web.root(req)
+
+  if (generateStackTraceAction && enabled && canReportStackTrace(rootSpan, maxStackTraces)) {
+    const frames = getCallsiteFrames(maxDepth)
+
     reportStackTrace(
       rootSpan,
       generateStackTraceAction.stack_id,
-      config.appsec.stackTrace.maxDepth,
-      config.appsec.stackTrace.maxStackTraces
+      frames
     )
   }
 

package/packages/dd-trace/src/appsec/stack_trace.js

@@ -6,11 +6,18 @@ const ddBasePath = calculateDDBasePath(__dirname)
 
 const LIBRARY_FRAMES_BUFFER = 20
 
+const STACK_TRACE_NAMESPACES = {
+  RASP: 'exploit',
+  IAST: 'vulnerability'
+}
+
 function getCallSiteList (maxDepth = 100) {
   const previousPrepareStackTrace = Error.prepareStackTrace
   const previousStackTraceLimit = Error.stackTraceLimit
   let callsiteList
-  Error.stackTraceLimit = maxDepth
+  // Since some frames will be discarded because they come from tracer codebase, a buffer is added
+  // to the limit in order to get as close as `maxDepth` number of frames.
+  Error.stackTraceLimit = maxDepth + LIBRARY_FRAMES_BUFFER
 
   try {
     Error.prepareStackTrace = function (_, callsites) {
@@ -30,7 +37,10 @@ function filterOutFramesFromLibrary (callSiteList) {
   return callSiteList.filter(callSite => !callSite.getFileName()?.startsWith(ddBasePath))
 }
 
-function getFramesForMetaStruct (callSiteList, maxDepth = 32) {
+function getCallsiteFrames (maxDepth = 32, callSiteListGetter = getCallSiteList) {
+  if (maxDepth < 1) maxDepth = Infinity
+
+  const callSiteList = callSiteListGetter(maxDepth)
   const filteredFrames = filterOutFramesFromLibrary(callSiteList)
 
   const half = filteredFrames.length > maxDepth ? Math.round(maxDepth / 2) : Infinity
@@ -45,46 +55,46 @@ function getFramesForMetaStruct (callSiteList, maxDepth = 32) {
       line: callSite.getLineNumber(),
       column: callSite.getColumnNumber(),
       function: callSite.getFunctionName(),
-      class_name: callSite.getTypeName()
+      class_name: callSite.getTypeName(),
+      isNative: callSite.isNative()
     })
   }
 
   return indexedFrames
 }
 
-function reportStackTrace (rootSpan, stackId, maxDepth, maxStackTraces, callSiteListGetter = getCallSiteList) {
+function reportStackTrace (rootSpan, stackId, frames, namespace = STACK_TRACE_NAMESPACES.RASP) {
   if (!rootSpan) return
+  if (!Array.isArray(frames)) return
 
-  if (maxStackTraces < 1 || (rootSpan.meta_struct?.['_dd.stack']?.exploit?.length ?? 0) < maxStackTraces) {
-    // Since some frames will be discarded because they come from tracer codebase, a buffer is added
-    // to the limit in order to get as close as `maxDepth` number of frames.
-    if (maxDepth < 1) maxDepth = Infinity
-    const callSiteList = callSiteListGetter(maxDepth + LIBRARY_FRAMES_BUFFER)
-    if (!Array.isArray(callSiteList)) return
+  if (!rootSpan.meta_struct) {
+    rootSpan.meta_struct = {}
+  }
 
-    if (!rootSpan.meta_struct) {
-      rootSpan.meta_struct = {}
-    }
+  if (!rootSpan.meta_struct['_dd.stack']) {
+    rootSpan.meta_struct['_dd.stack'] = {}
+  }
 
-    if (!rootSpan.meta_struct['_dd.stack']) {
-      rootSpan.meta_struct['_dd.stack'] = {}
-    }
+  if (!rootSpan.meta_struct['_dd.stack'][namespace]) {
+    rootSpan.meta_struct['_dd.stack'][namespace] = []
+  }
 
-    if (!rootSpan.meta_struct['_dd.stack'].exploit) {
-      rootSpan.meta_struct['_dd.stack'].exploit = []
-    }
+  rootSpan.meta_struct['_dd.stack'][namespace].push({
+    id: stackId,
+    language: 'nodejs',
+    frames
+  })
+}
 
-    const frames = getFramesForMetaStruct(callSiteList, maxDepth)
+function canReportStackTrace (rootSpan, maxStackTraces, namespace = STACK_TRACE_NAMESPACES.RASP) {
+  if (!rootSpan) return false
 
-    rootSpan.meta_struct['_dd.stack'].exploit.push({
-      id: stackId,
-      language: 'nodejs',
-      frames
-    })
-  }
+  return maxStackTraces < 1 || (rootSpan.meta_struct?.['_dd.stack']?.[namespace]?.length ?? 0) < maxStackTraces
 }
 
 module.exports = {
-  getCallSiteList,
-  reportStackTrace
+  getCallsiteFrames,
+  reportStackTrace,
+  canReportStackTrace,
+  STACK_TRACE_NAMESPACES
 }

package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -87,9 +87,8 @@ class CiVisibilityExporter extends AgentInfoExporter {
 
   shouldRequestKnownTests () {
     return !!(
-      this._config.isEarlyFlakeDetectionEnabled &&
       this._canUseCiVisProtocol &&
-      this._libraryConfig?.isEarlyFlakeDetectionEnabled
+      this._libraryConfig?.isKnownTestsEnabled
     )
   }
 
@@ -197,7 +196,8 @@ class CiVisibilityExporter extends AgentInfoExporter {
       earlyFlakeDetectionNumRetries,
       earlyFlakeDetectionFaultyThreshold,
       isFlakyTestRetriesEnabled,
-      isDiEnabled
+      isDiEnabled,
+      isKnownTestsEnabled
     } = remoteConfiguration
     return {
       isCodeCoverageEnabled,
@@ -209,7 +209,8 @@ class CiVisibilityExporter extends AgentInfoExporter {
       earlyFlakeDetectionFaultyThreshold,
       isFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled && this._config.isFlakyTestRetriesEnabled,
       flakyTestRetriesCount: this._config.flakyTestRetriesCount,
-      isDiEnabled: isDiEnabled && this._config.isTestDynamicInstrumentationEnabled
+      isDiEnabled: isDiEnabled && this._config.isTestDynamicInstrumentationEnabled,
+      isKnownTestsEnabled
     }
   }
 

package/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js

@@ -93,7 +93,8 @@ function getLibraryConfiguration ({
               require_git: requireGit,
               early_flake_detection: earlyFlakeDetectionConfig,
               flaky_test_retries_enabled: isFlakyTestRetriesEnabled,
-              di_enabled: isDiEnabled
+              di_enabled: isDiEnabled,
+              known_tests_enabled: isKnownTestsEnabled
             }
           }
         } = JSON.parse(res)
@@ -103,13 +104,14 @@ function getLibraryConfiguration ({
           isSuitesSkippingEnabled,
           isItrEnabled,
           requireGit,
-          isEarlyFlakeDetectionEnabled: earlyFlakeDetectionConfig?.enabled ?? false,
+          isEarlyFlakeDetectionEnabled: isKnownTestsEnabled && (earlyFlakeDetectionConfig?.enabled ?? false),
           earlyFlakeDetectionNumRetries:
             earlyFlakeDetectionConfig?.slow_test_retries?.['5s'] || DEFAULT_EARLY_FLAKE_DETECTION_NUM_RETRIES,
           earlyFlakeDetectionFaultyThreshold:
             earlyFlakeDetectionConfig?.faulty_session_threshold ?? DEFAULT_EARLY_FLAKE_DETECTION_ERROR_THRESHOLD,
           isFlakyTestRetriesEnabled,
-          isDiEnabled: isDiEnabled && isFlakyTestRetriesEnabled
+          isDiEnabled: isDiEnabled && isFlakyTestRetriesEnabled,
+          isKnownTestsEnabled
         }
 
         log.debug(() => `Remote settings: ${JSON.stringify(settings)}`)

package/packages/dd-trace/src/config.js

@@ -497,6 +497,7 @@ class Config {
     this._setValue(defaults, 'iast.redactionValuePattern', null)
     this._setValue(defaults, 'iast.requestSampling', 30)
     this._setValue(defaults, 'iast.telemetryVerbosity', 'INFORMATION')
+    this._setValue(defaults, 'iast.stackTrace.enabled', true)
     this._setValue(defaults, 'injectionEnabled', [])
     this._setValue(defaults, 'isAzureFunction', false)
     this._setValue(defaults, 'isCiVisibility', false)
@@ -521,7 +522,7 @@ class Config {
     this._setValue(defaults, 'inferredProxyServicesEnabled', false)
     this._setValue(defaults, 'memcachedCommandEnabled', false)
     this._setValue(defaults, 'openAiLogsEnabled', false)
-    this._setValue(defaults, 'openaiSpanCharLimit', 128)
+    this._setValue(defaults, 'openai.spanCharLimit', 128)
     this._setValue(defaults, 'peerServiceMapping', {})
     this._setValue(defaults, 'plugins', true)
     this._setValue(defaults, 'port', '8126')
@@ -622,6 +623,7 @@ class Config {
       DD_IAST_REDACTION_VALUE_PATTERN,
       DD_IAST_REQUEST_SAMPLING,
       DD_IAST_TELEMETRY_VERBOSITY,
+      DD_IAST_STACK_TRACE_ENABLED,
       DD_INJECTION_ENABLED,
       DD_INSTRUMENTATION_TELEMETRY_ENABLED,
       DD_INSTRUMENTATION_CONFIG_ID,
@@ -787,6 +789,7 @@ class Config {
     }
     this._envUnprocessed['iast.requestSampling'] = DD_IAST_REQUEST_SAMPLING
     this._setString(env, 'iast.telemetryVerbosity', DD_IAST_TELEMETRY_VERBOSITY)
+    this._setBoolean(env, 'iast.stackTrace.enabled', DD_IAST_STACK_TRACE_ENABLED)
     this._setArray(env, 'injectionEnabled', DD_INJECTION_ENABLED)
     this._setBoolean(env, 'isAzureFunction', getIsAzureFunction())
     this._setBoolean(env, 'isGCPFunction', getIsGCPFunction())
@@ -802,7 +805,7 @@ class Config {
     // Requires an accompanying DD_APM_OBFUSCATION_MEMCACHED_KEEP_COMMAND=true in the agent
     this._setBoolean(env, 'memcachedCommandEnabled', DD_TRACE_MEMCACHED_COMMAND_ENABLED)
     this._setBoolean(env, 'openAiLogsEnabled', DD_OPENAI_LOGS_ENABLED)
-    this._setValue(env, 'openaiSpanCharLimit', maybeInt(DD_OPENAI_SPAN_CHAR_LIMIT))
+    this._setValue(env, 'openai.spanCharLimit', maybeInt(DD_OPENAI_SPAN_CHAR_LIMIT))
     this._envUnprocessed.openaiSpanCharLimit = DD_OPENAI_SPAN_CHAR_LIMIT
     if (DD_TRACE_PEER_SERVICE_MAPPING) {
       this._setValue(env, 'peerServiceMapping', fromEntries(
@@ -976,6 +979,7 @@ class Config {
       this._optsUnprocessed['iast.requestSampling'] = options.iast?.requestSampling
     }
     this._setString(opts, 'iast.telemetryVerbosity', options.iast && options.iast.telemetryVerbosity)
+    this._setBoolean(opts, 'iast.stackTrace.enabled', options.iast?.stackTrace?.enabled)
     this._setBoolean(opts, 'isCiVisibility', options.isCiVisibility)
     this._setBoolean(opts, 'legacyBaggageEnabled', options.legacyBaggageEnabled)
     this._setBoolean(opts, 'llmobs.agentlessEnabled', options.llmobs?.agentlessEnabled)

package/packages/dd-trace/src/llmobs/plugins/bedrockruntime.js

@@ -0,0 +1,59 @@
+const BaseLLMObsPlugin = require('./base')
+const { storage } = require('../../../../datadog-core')
+const llmobsStore = storage('llmobs')
+
+const {
+  extractRequestParams,
+  extractTextAndResponseReason,
+  parseModelId
+} = require('../../../../datadog-plugin-aws-sdk/src/services/bedrockruntime/utils')
+
+const enabledOperations = ['invokeModel']
+
+class BedrockRuntimeLLMObsPlugin extends BaseLLMObsPlugin {
+  constructor () {
+    super(...arguments)
+
+    this.addSub('apm:aws:request:complete:bedrockruntime', ({ response }) => {
+      const request = response.request
+      const operation = request.operation
+      // avoids instrumenting other non supported runtime operations
+      if (!enabledOperations.includes(operation)) {
+        return
+      }
+      const { modelProvider, modelName } = parseModelId(request.params.modelId)
+
+      // avoids instrumenting non llm type
+      if (modelName.includes('embed')) {
+        return
+      }
+      const span = storage.getStore()?.span
+      this.setLLMObsTags({ request, span, response, modelProvider, modelName })
+    })
+  }
+
+  setLLMObsTags ({ request, span, response, modelProvider, modelName }) {
+    const parent = llmobsStore.getStore()?.span
+    this._tagger.registerLLMObsSpan(span, {
+      parent,
+      modelName: modelName.toLowerCase(),
+      modelProvider: modelProvider.toLowerCase(),
+      kind: 'llm',
+      name: 'bedrock-runtime.command'
+    })
+
+    const requestParams = extractRequestParams(request.params, modelProvider)
+    const textAndResponseReason = extractTextAndResponseReason(response, modelProvider, modelName)
+
+    // add metadata tags
+    this._tagger.tagMetadata(span, {
+      temperature: parseFloat(requestParams.temperature) || 0.0,
+      max_tokens: parseInt(requestParams.maxTokens) || 0
+    })
+
+    // add I/O tags
+    this._tagger.tagLLMIO(span, requestParams.prompt, textAndResponseReason.message)
+  }
+}
+
+module.exports = BedrockRuntimeLLMObsPlugin

package/packages/dd-trace/src/plugins/ci_plugin.js

@@ -158,6 +158,7 @@ module.exports = class CiPlugin extends Plugin {
         if (err) {
           log.error('Known tests could not be fetched. %s', err.message)
           this.libraryConfig.isEarlyFlakeDetectionEnabled = false
+          this.libraryConfig.isKnownTestsEnabled = false
         }
         onDone({ err, knownTests })
       })

package/packages/dd-trace/src/plugins/util/inferred_proxy.js

@@ -2,7 +2,6 @@ const log = require('../../log')
 const tags = require('../../../../../ext/tags')
 
 const RESOURCE_NAME = tags.RESOURCE_NAME
-const HTTP_ROUTE = tags.HTTP_ROUTE
 const SPAN_KIND = tags.SPAN_KIND
 const SPAN_TYPE = tags.SPAN_TYPE
 const HTTP_URL = tags.HTTP_URL
@@ -54,7 +53,6 @@ function createInferredProxySpan (headers, childOf, tracer, context) {
         [SPAN_TYPE]: 'web',
         [HTTP_METHOD]: proxyContext.method,
         [HTTP_URL]: proxyContext.domainName + proxyContext.path,
-        [HTTP_ROUTE]: proxyContext.path,
         stage: proxyContext.stage
       }
     }

package/packages/dd-trace/src/plugins/util/llm.js

@@ -0,0 +1,35 @@
+const Sampler = require('../../sampler')
+
+const RE_NEWLINE = /\n/g
+const RE_TAB = /\t/g
+
+function normalize (text, limit = 128) {
+  if (!text) return
+  if (typeof text !== 'string' || !text || (typeof text === 'string' && text.length === 0)) return
+
+  text = text
+    .replace(RE_NEWLINE, '\\n')
+    .replace(RE_TAB, '\\t')
+
+  if (text.length > limit) {
+    return text.substring(0, limit) + '...'
+  }
+
+  return text
+}
+
+function isPromptCompletionSampled (sampler) {
+  return sampler.isSampled()
+}
+
+module.exports = function (integrationName, tracerConfig) {
+  const integrationConfig = tracerConfig[integrationName] || {}
+  const { spanCharLimit, spanPromptCompletionSampleRate } = integrationConfig
+
+  const sampler = new Sampler(spanPromptCompletionSampleRate ?? 1.0)
+
+  return {
+    normalize: str => normalize(str, spanCharLimit),
+    isPromptCompletionSampled: () => isPromptCompletionSampled(sampler)
+  }
+}

package/packages/dd-trace/src/plugins/util/test.js

@@ -59,6 +59,7 @@ const TEST_IS_NEW = 'test.is_new'
 const TEST_IS_RETRY = 'test.is_retry'
 const TEST_EARLY_FLAKE_ENABLED = 'test.early_flake.enabled'
 const TEST_EARLY_FLAKE_ABORT_REASON = 'test.early_flake.abort_reason'
+const TEST_RETRY_REASON = 'test.retry_reason'
 
 const CI_APP_ORIGIN = 'ciapp-test'
 
@@ -145,6 +146,7 @@ module.exports = {
   TEST_IS_RETRY,
   TEST_EARLY_FLAKE_ENABLED,
   TEST_EARLY_FLAKE_ABORT_REASON,
+  TEST_RETRY_REASON,
   getTestEnvironmentMetadata,
   getTestParametersString,
   finishAllTraceSpans,