dd-trace 5.32.0 → 5.33.1

package/packages/datadog-plugin-http/src/client.js

@@ -59,6 +59,11 @@ class HttpClientPlugin extends ClientPlugin {
     }
 
     if (this.shouldInjectTraceHeaders(options, uri)) {
+      // Clone the headers object in case an upstream lib has a reference to the original headers
+      // Implemented due to aws-sdk issue where request signing is broken if we mutate the headers
+      // Explained further in:
+      // https://github.com/open-telemetry/opentelemetry-js-contrib/issues/1609#issuecomment-1826167348
+      options.headers = Object.assign({}, options.headers)
       this.tracer.inject(span, HTTP_HEADERS, options.headers)
     }
 
@@ -72,10 +77,6 @@ class HttpClientPlugin extends ClientPlugin {
   }
 
   shouldInjectTraceHeaders (options, uri) {
-    if (hasAmazonSignature(options) && !this.config.enablePropagationWithAmazonHeaders) {
-      return false
-    }
-
     if (!this.config.propagationFilter(uri)) {
       return false
     }
@@ -212,31 +213,6 @@ function getHooks (config) {
   return { request }
 }
 
-function hasAmazonSignature (options) {
-  if (!options) {
-    return false
-  }
-
-  if (options.headers) {
-    const headers = Object.keys(options.headers)
-      .reduce((prev, next) => Object.assign(prev, {
-        [next.toLowerCase()]: options.headers[next]
-      }), {})
-
-    if (headers['x-amz-signature']) {
-      return true
-    }
-
-    if ([].concat(headers.authorization).some(startsWith('AWS4-HMAC-SHA256'))) {
-      return true
-    }
-  }
-
-  const search = options.search || options.path
-
-  return search && search.toLowerCase().indexOf('x-amz-signature=') !== -1
-}
-
 function extractSessionDetails (options) {
   if (typeof options === 'string') {
     return new URL(options).host
@@ -248,8 +224,4 @@ function extractSessionDetails (options) {
   return { host, port }
 }
 
-function startsWith (searchString) {
-  return value => String(value).startsWith(searchString)
-}
-
 module.exports = HttpClientPlugin

package/packages/datadog-plugin-jest/src/index.js

@@ -23,7 +23,8 @@ const {
   JEST_DISPLAY_NAME,
   TEST_IS_RUM_ACTIVE,
   TEST_BROWSER_DRIVER,
-  getFormattedError
+  getFormattedError,
+  TEST_RETRY_REASON
 } = require('../../dd-trace/src/plugins/util/test')
 const { COMPONENT } = require('../../dd-trace/src/constants')
 const id = require('../../dd-trace/src/id')
@@ -167,6 +168,7 @@ class JestPlugin extends CiPlugin {
         config._ddIsFlakyTestRetriesEnabled = this.libraryConfig?.isFlakyTestRetriesEnabled ?? false
         config._ddFlakyTestRetriesCount = this.libraryConfig?.flakyTestRetriesCount
         config._ddIsDiEnabled = this.libraryConfig?.isDiEnabled ?? false
+        config._ddIsKnownTestsEnabled = this.libraryConfig?.isKnownTestsEnabled ?? false
       })
     })
 
@@ -410,6 +412,7 @@ class JestPlugin extends CiPlugin {
       extraTags[TEST_IS_NEW] = 'true'
       if (isEfdRetry) {
         extraTags[TEST_IS_RETRY] = 'true'
+        extraTags[TEST_RETRY_REASON] = 'efd'
       }
     }
 

package/packages/datadog-plugin-langchain/src/handlers/default.js

@@ -1,16 +1,13 @@
 'use strict'
 
-const Sampler = require('../../../dd-trace/src/sampler')
+const makeUtilities = require('../../../dd-trace/src/plugins/util/llm')
 
-const RE_NEWLINE = /\n/g
-const RE_TAB = /\t/g
-
-// TODO: should probably refactor the OpenAI integration to use a shared LLMTracingPlugin base class
-// This logic isn't particular to LangChain
 class LangChainHandler {
-  constructor (config) {
-    this.config = config
-    this.sampler = new Sampler(config.spanPromptCompletionSampleRate)
+  constructor (tracerConfig) {
+    const utilities = makeUtilities('langchain', tracerConfig)
+
+    this.normalize = utilities.normalize
+    this.isPromptCompletionSampled = utilities.isPromptCompletionSampled
   }
 
   // no-op for default handler
@@ -27,27 +24,6 @@ class LangChainHandler {
 
   // no-op for default handler
   extractModel (instance) {}
-
-  normalize (text) {
-    if (!text) return
-    if (typeof text !== 'string' || !text || (typeof text === 'string' && text.length === 0)) return
-
-    const max = this.config.spanCharLimit
-
-    text = text
-      .replace(RE_NEWLINE, '\\n')
-      .replace(RE_TAB, '\\t')
-
-    if (text.length > max) {
-      return text.substring(0, max) + '...'
-    }
-
-    return text
-  }
-
-  isPromptCompletionSampled () {
-    return this.sampler.isSampled()
-  }
 }
 
 module.exports = LangChainHandler

package/packages/datadog-plugin-langchain/src/tracing.js

@@ -26,13 +26,12 @@ class LangChainTracingPlugin extends TracingPlugin {
   constructor () {
     super(...arguments)
 
-    const langchainConfig = this._tracerConfig.langchain || {}
     this.handlers = {
-      chain: new LangChainChainHandler(langchainConfig),
-      chat_model: new LangChainChatModelHandler(langchainConfig),
-      llm: new LangChainLLMHandler(langchainConfig),
-      embedding: new LangChainEmbeddingHandler(langchainConfig),
-      default: new LangChainHandler(langchainConfig)
+      chain: new LangChainChainHandler(this._tracerConfig),
+      chat_model: new LangChainChatModelHandler(this._tracerConfig),
+      llm: new LangChainLLMHandler(this._tracerConfig),
+      embedding: new LangChainEmbeddingHandler(this._tracerConfig),
+      default: new LangChainHandler(this._tracerConfig)
     }
   }
 

package/packages/datadog-plugin-mocha/src/index.js

@@ -30,7 +30,8 @@ const {
   TEST_SUITE,
   MOCHA_IS_PARALLEL,
   TEST_IS_RUM_ACTIVE,
-  TEST_BROWSER_DRIVER
+  TEST_BROWSER_DRIVER,
+  TEST_RETRY_REASON
 } = require('../../dd-trace/src/plugins/util/test')
 const { COMPONENT } = require('../../dd-trace/src/constants')
 const {
@@ -421,6 +422,7 @@ class MochaPlugin extends CiPlugin {
       extraTags[TEST_IS_NEW] = 'true'
       if (isEfdRetry) {
         extraTags[TEST_IS_RETRY] = 'true'
+        extraTags[TEST_RETRY_REASON] = 'efd'
       }
     }
 

package/packages/datadog-plugin-openai/src/tracing.js

@@ -9,12 +9,9 @@ const Sampler = require('../../dd-trace/src/sampler')
 const { MEASURED } = require('../../../ext/tags')
 const { estimateTokens } = require('./token-estimator')
 
-// String#replaceAll unavailable on Node.js@v14 (dd-trace@<=v3)
-const RE_NEWLINE = /\n/g
-const RE_TAB = /\t/g
+const makeUtilities = require('../../dd-trace/src/plugins/util/llm')
 
-// TODO: In the future we should refactor config.js to make it requirable
-let MAX_TEXT_LEN = 128
+let normalize
 
 function safeRequire (path) {
   try {
@@ -44,9 +41,11 @@ class OpenAiTracingPlugin extends TracingPlugin {
 
     this.sampler = new Sampler(0.1) // default 10% log sampling
 
-    // hoist the max length env var to avoid making all of these functions a class method
+    // hoist the normalize function to avoid making all of these functions a class method
     if (this._tracerConfig) {
-      MAX_TEXT_LEN = this._tracerConfig.openaiSpanCharLimit
+      const utilities = makeUtilities('openai', this._tracerConfig)
+
+      normalize = utilities.normalize
     }
   }
 
@@ -116,7 +115,7 @@ class OpenAiTracingPlugin extends TracingPlugin {
     // createEdit, createEmbedding, createModeration
     if (payload.input) {
       const normalized = normalizeStringOrTokenArray(payload.input, false)
-      tags['openai.request.input'] = truncateText(normalized)
+      tags['openai.request.input'] = normalize(normalized)
       openaiStore.input = normalized
     }
 
@@ -594,7 +593,7 @@ function commonImageResponseExtraction (tags, body) {
   for (let i = 0; i < body.data.length; i++) {
     const image = body.data[i]
     // exactly one of these two options is provided
-    tags[`openai.response.images.${i}.url`] = truncateText(image.url)
+    tags[`openai.response.images.${i}.url`] = normalize(image.url)
     tags[`openai.response.images.${i}.b64_json`] = image.b64_json && 'returned'
   }
 }
@@ -731,14 +730,14 @@ function commonCreateResponseExtraction (tags, body, openaiStore, methodName) {
 
     tags[`openai.response.choices.${choiceIdx}.finish_reason`] = choice.finish_reason
     tags[`openai.response.choices.${choiceIdx}.logprobs`] = specifiesLogProb ? 'returned' : undefined
-    tags[`openai.response.choices.${choiceIdx}.text`] = truncateText(choice.text)
+    tags[`openai.response.choices.${choiceIdx}.text`] = normalize(choice.text)
 
     // createChatCompletion only
     const message = choice.message || choice.delta // delta for streamed responses
     if (message) {
       tags[`openai.response.choices.${choiceIdx}.message.role`] = message.role
-      tags[`openai.response.choices.${choiceIdx}.message.content`] = truncateText(message.content)
-      tags[`openai.response.choices.${choiceIdx}.message.name`] = truncateText(message.name)
+      tags[`openai.response.choices.${choiceIdx}.message.content`] = normalize(message.content)
+      tags[`openai.response.choices.${choiceIdx}.message.name`] = normalize(message.name)
       if (message.tool_calls) {
         const toolCalls = message.tool_calls
         for (let toolIdx = 0; toolIdx < toolCalls.length; toolIdx++) {
@@ -795,24 +794,6 @@ function truncateApiKey (apiKey) {
   return apiKey && `sk-...${apiKey.substr(apiKey.length - 4)}`
 }
 
-/**
- * for cleaning up prompt and response
- */
-function truncateText (text) {
-  if (!text) return
-  if (typeof text !== 'string' || !text || (typeof text === 'string' && text.length === 0)) return
-
-  text = text
-    .replace(RE_NEWLINE, '\\n')
-    .replace(RE_TAB, '\\t')
-
-  if (text.length > MAX_TEXT_LEN) {
-    return text.substring(0, MAX_TEXT_LEN) + '...'
-  }
-
-  return text
-}
-
 function tagChatCompletionRequestContent (contents, messageIdx, tags) {
   if (typeof contents === 'string') {
     tags[`openai.request.messages.${messageIdx}.content`] = contents
@@ -824,10 +805,10 @@ function tagChatCompletionRequestContent (contents, messageIdx, tags) {
       const type = content.type
       tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.type`] = content.type
       if (type === 'text') {
-        tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.text`] = truncateText(content.text)
+        tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.text`] = normalize(content.text)
       } else if (type === 'image_url') {
         tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.image_url.url`] =
-          truncateText(content.image_url.url)
+          normalize(content.image_url.url)
       }
       // unsupported type otherwise, won't be tagged
     }
@@ -1004,7 +985,7 @@ function normalizeStringOrTokenArray (input, truncate) {
   const normalized = Array.isArray(input)
     ? `[${input.join(', ')}]` // "[1, 2, 999]"
     : input // "foo"
-  return truncate ? truncateText(normalized) : normalized
+  return truncate ? normalize(normalized) : normalized
 }
 
 function defensiveArrayLength (maybeArray) {

package/packages/datadog-plugin-playwright/src/index.js

@@ -15,7 +15,8 @@ const {
   TEST_IS_NEW,
   TEST_IS_RETRY,
   TEST_EARLY_FLAKE_ENABLED,
-  TELEMETRY_TEST_SESSION
+  TELEMETRY_TEST_SESSION,
+  TEST_RETRY_REASON
 } = require('../../dd-trace/src/plugins/util/test')
 const { RESOURCE_NAME } = require('../../../ext/tags')
 const { COMPONENT } = require('../../dd-trace/src/constants')
@@ -144,6 +145,7 @@ class PlaywrightPlugin extends CiPlugin {
         span.setTag(TEST_IS_NEW, 'true')
         if (isEfdRetry) {
           span.setTag(TEST_IS_RETRY, 'true')
+          span.setTag(TEST_RETRY_REASON, 'efd')
         }
       }
       if (isRetry) {

package/packages/datadog-plugin-vitest/src/index.js

@@ -17,7 +17,8 @@ const {
   TEST_SOURCE_START,
   TEST_IS_NEW,
   TEST_EARLY_FLAKE_ENABLED,
-  TEST_EARLY_FLAKE_ABORT_REASON
+  TEST_EARLY_FLAKE_ABORT_REASON,
+  TEST_RETRY_REASON
 } = require('../../dd-trace/src/plugins/util/test')
 const { COMPONENT } = require('../../dd-trace/src/constants')
 const {
@@ -60,7 +61,14 @@ class VitestPlugin extends CiPlugin {
       onDone(isFaulty)
     })
 
-    this.addSub('ci:vitest:test:start', ({ testName, testSuiteAbsolutePath, isRetry, isNew, mightHitProbe }) => {
+    this.addSub('ci:vitest:test:start', ({
+      testName,
+      testSuiteAbsolutePath,
+      isRetry,
+      isNew,
+      mightHitProbe,
+      isRetryReasonEfd
+    }) => {
       const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot)
       const store = storage.getStore()
 
@@ -73,6 +81,9 @@ class VitestPlugin extends CiPlugin {
       if (isNew) {
         extraTags[TEST_IS_NEW] = 'true'
       }
+      if (isRetryReasonEfd) {
+        extraTags[TEST_RETRY_REASON] = 'efd'
+      }
 
       const span = this.startTestSpan(
         testName,
@@ -147,7 +158,7 @@ class VitestPlugin extends CiPlugin {
       }
     })
 
-    this.addSub('ci:vitest:test:skip', ({ testName, testSuiteAbsolutePath }) => {
+    this.addSub('ci:vitest:test:skip', ({ testName, testSuiteAbsolutePath, isNew }) => {
       const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot)
       const testSpan = this.startTestSpan(
         testName,
@@ -156,7 +167,8 @@ class VitestPlugin extends CiPlugin {
         {
           [TEST_SOURCE_FILE]: testSuite,
           [TEST_SOURCE_START]: 1, // we can't get the proper start line in vitest
-          [TEST_STATUS]: 'skip'
+          [TEST_STATUS]: 'skip',
+          ...(isNew ? { [TEST_IS_NEW]: 'true' } : {})
         }
       )
       this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'test', {

package/packages/dd-trace/src/appsec/iast/analyzers/cookie-analyzer.js

@@ -54,15 +54,15 @@ class CookieAnalyzer extends Analyzer {
     return super._checkOCE(context, value)
   }
 
-  _getLocation (value) {
+  _getLocation (value, callSiteFrames) {
     if (!value) {
-      return super._getLocation()
+      return super._getLocation(value, callSiteFrames)
     }
 
     if (value.location) {
       return value.location
     }
-    const location = super._getLocation(value)
+    const location = super._getLocation(value, callSiteFrames)
     value.location = location
     return location
   }

package/packages/dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer.js

@@ -1,12 +1,15 @@
 'use strict'
 
 const { storage } = require('../../../../../datadog-core')
-const { getFirstNonDDPathAndLine } = require('../path-line')
-const { addVulnerability } = require('../vulnerability-reporter')
-const { getIastContext } = require('../iast-context')
+const { getNonDDCallSiteFrames } = require('../path-line')
+const { getIastContext, getIastStackTraceId } = require('../iast-context')
 const overheadController = require('../overhead-controller')
 const { SinkIastPlugin } = require('../iast-plugin')
-const { getOriginalPathAndLineFromSourceMap } = require('../taint-tracking/rewriter')
+const {
+  addVulnerability,
+  getVulnerabilityCallSiteFrames,
+  replaceCallSiteFromSourceMap
+} = require('../vulnerability-reporter')
 
 class Analyzer extends SinkIastPlugin {
   constructor (type) {
@@ -28,12 +31,24 @@ class Analyzer extends SinkIastPlugin {
   }
 
   _reportEvidence (value, context, evidence) {
-    const location = this._getLocation(value)
+    const callSiteFrames = getVulnerabilityCallSiteFrames()
+    const nonDDCallSiteFrames = getNonDDCallSiteFrames(callSiteFrames, this._getExcludedPaths())
+
+    const location = this._getLocation(value, nonDDCallSiteFrames)
+
     if (!this._isExcluded(location)) {
-      const locationSourceMap = this._replaceLocationFromSourceMap(location)
+      const originalLocation = this._getOriginalLocation(location)
       const spanId = context && context.rootSpan && context.rootSpan.context().toSpanId()
-      const vulnerability = this._createVulnerability(this._type, evidence, spanId, locationSourceMap)
-      addVulnerability(context, vulnerability)
+      const stackId = getIastStackTraceId(context)
+      const vulnerability = this._createVulnerability(
+        this._type,
+        evidence,
+        spanId,
+        originalLocation,
+        stackId
+      )
+
+      addVulnerability(context, vulnerability, nonDDCallSiteFrames)
     }
   }
 
@@ -49,24 +64,25 @@ class Analyzer extends SinkIastPlugin {
     return { value }
   }
 
-  _getLocation () {
-    return getFirstNonDDPathAndLine(this._getExcludedPaths())
+  _getLocation (value, callSiteFrames) {
+    return callSiteFrames[0]
   }
 
-  _replaceLocationFromSourceMap (location) {
-    if (location) {
-      const { path, line, column } = getOriginalPathAndLineFromSourceMap(location)
-      if (path) {
-        location.path = path
-      }
-      if (line) {
-        location.line = line
-      }
-      if (column) {
-        location.column = column
-      }
+  _getOriginalLocation (location) {
+    const locationFromSourceMap = replaceCallSiteFromSourceMap(location)
+    const originalLocation = {}
+
+    if (locationFromSourceMap?.path) {
+      originalLocation.path = locationFromSourceMap.path
+    }
+    if (locationFromSourceMap?.line) {
+      originalLocation.line = locationFromSourceMap.line
     }
-    return location
+    if (locationFromSourceMap?.column) {
+      originalLocation.column = locationFromSourceMap.column
+    }
+
+    return originalLocation
   }
 
   _getExcludedPaths () {}
@@ -102,12 +118,13 @@ class Analyzer extends SinkIastPlugin {
     return overheadController.hasQuota(overheadController.OPERATIONS.REPORT_VULNERABILITY, context)
   }
 
-  _createVulnerability (type, evidence, spanId, location) {
+  _createVulnerability (type, evidence, spanId, location, stackId) {
     if (type && evidence) {
       const _spanId = spanId || 0
       return {
         type,
         evidence,
+        stackId,
         location: {
           spanId: _spanId,
           ...location

package/packages/dd-trace/src/appsec/iast/iast-context.js

@@ -9,6 +9,17 @@ function getIastContext (store, topContext) {
   return iastContext
 }
 
+function getIastStackTraceId (iastContext) {
+  if (!iastContext) return 0
+
+  if (!iastContext.stackTraceId) {
+    iastContext.stackTraceId = 0
+  }
+
+  iastContext.stackTraceId += 1
+  return iastContext.stackTraceId
+}
+
 /* TODO Fix storage problem when the close event is called without
         finish event to remove `topContext` references
   We have to save the context in two places, because
@@ -51,6 +62,7 @@ module.exports = {
   getIastContext,
   saveIastContext,
   cleanIastContext,
+  getIastStackTraceId,
   IAST_CONTEXT_KEY,
   IAST_TRANSACTION_ID
 }

package/packages/dd-trace/src/appsec/iast/path-line.js

@@ -3,12 +3,10 @@
 const path = require('path')
 const process = require('process')
 const { calculateDDBasePath } = require('../../util')
-const { getCallSiteList } = require('../stack_trace')
 const pathLine = {
-  getFirstNonDDPathAndLine,
   getNodeModulesPaths,
   getRelativePath,
-  getFirstNonDDPathAndLineFromCallsites, // Exported only for test purposes
+  getNonDDCallSiteFrames,
   calculateDDBasePath, // Exported only for test purposes
   ddBasePath: calculateDDBasePath(__dirname) // Only for test purposes
 }
@@ -25,22 +23,24 @@ const EXCLUDED_PATH_PREFIXES = [
   'async_hooks'
 ]
 
-function getFirstNonDDPathAndLineFromCallsites (callsites, externallyExcludedPaths) {
-  if (callsites) {
-    for (let i = 0; i < callsites.length; i++) {
-      const callsite = callsites[i]
-      const filepath = callsite.getFileName()
-      if (!isExcluded(callsite, externallyExcludedPaths) && filepath.indexOf(pathLine.ddBasePath) === -1) {
-        return {
-          path: getRelativePath(filepath),
-          line: callsite.getLineNumber(),
-          column: callsite.getColumnNumber(),
-          isInternal: !path.isAbsolute(filepath)
-        }
-      }
+function getNonDDCallSiteFrames (callSiteFrames, externallyExcludedPaths) {
+  if (!callSiteFrames) {
+    return []
+  }
+
+  const result = []
+
+  for (const callsite of callSiteFrames) {
+    const filepath = callsite.file
+    if (!isExcluded(callsite, externallyExcludedPaths) && filepath.indexOf(pathLine.ddBasePath) === -1) {
+      callsite.path = getRelativePath(filepath)
+      callsite.isInternal = !path.isAbsolute(filepath)
+
+      result.push(callsite)
     }
   }
-  return null
+
+  return result
 }
 
 function getRelativePath (filepath) {
@@ -48,8 +48,8 @@ function getRelativePath (filepath) {
 }
 
 function isExcluded (callsite, externallyExcludedPaths) {
-  if (callsite.isNative()) return true
-  const filename = callsite.getFileName()
+  if (callsite.isNative) return true
+  const filename = callsite.file
   if (!filename) {
     return true
   }
@@ -73,10 +73,6 @@ function isExcluded (callsite, externallyExcludedPaths) {
   return false
 }
 
-function getFirstNonDDPathAndLine (externallyExcludedPaths) {
-  return getFirstNonDDPathAndLineFromCallsites(getCallSiteList(), externallyExcludedPaths)
-}
-
 function getNodeModulesPaths (...paths) {
   const nodeModulesPaths = []
 

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/index.js

@@ -84,6 +84,7 @@ class VulnerabilityFormatter {
     const formattedVulnerability = {
       type: vulnerability.type,
       hash: vulnerability.hash,
+      stackId: vulnerability.stackId,
       evidence: this.formatEvidence(vulnerability.type, vulnerability.evidence, sourcesIndexes, sources),
       location: {
         spanId: vulnerability.location.spanId