dd-trace 5.36.0 → 5.37.0

package/packages/dd-trace/src/appsec/iast/security-controls/index.js

@@ -0,0 +1,187 @@
+'use strict'
+
+const path = require('path')
+const dc = require('dc-polyfill')
+const { storage } = require('../../../../../datadog-core')
+const shimmer = require('../../../../../datadog-shimmer')
+const log = require('../../../log')
+const { parse, SANITIZER_TYPE } = require('./parser')
+const TaintTrackingOperations = require('../taint-tracking/operations')
+const { getIastContext } = require('../iast-context')
+const { iterateObjectStrings } = require('../utils')
+
+// esm
+const moduleLoadStartChannel = dc.channel('dd-trace:moduleLoadStart')
+
+// cjs
+const moduleLoadEndChannel = dc.channel('dd-trace:moduleLoadEnd')
+
+let controls
+let controlsKeys
+let hooks
+
+function configure (iastConfig) {
+  if (!iastConfig?.securityControlsConfiguration) return
+
+  try {
+    controls = parse(iastConfig.securityControlsConfiguration)
+    if (controls?.size > 0) {
+      hooks = new WeakSet()
+      controlsKeys = [...controls.keys()]
+
+      moduleLoadStartChannel.subscribe(onModuleLoaded)
+      moduleLoadEndChannel.subscribe(onModuleLoaded)
+    }
+  } catch (e) {
+    log.error('[ASM] Error configuring IAST Security Controls', e)
+  }
+}
+
+function onModuleLoaded (payload) {
+  if (!payload?.module || hooks?.has(payload.module)) return
+
+  const { filename, module } = payload
+
+  const controlsByFile = getControls(filename)
+  if (controlsByFile) {
+    const hook = hookModule(filename, module, controlsByFile)
+    payload.module = hook
+    hooks.add(hook)
+  }
+}
+
+function getControls (filename) {
+  if (filename.startsWith('file://')) {
+    filename = filename.substring(7)
+  }
+
+  let key = path.isAbsolute(filename) ? path.relative(process.cwd(), filename) : filename
+  key = key.replaceAll(path.sep, path.posix.sep)
+
+  if (key.includes('node_modules')) {
+    key = controlsKeys.find(file => key.endsWith(file))
+  }
+
+  return controls.get(key)
+}
+
+function hookModule (filename, module, controlsByFile) {
+  try {
+    controlsByFile.forEach(({ type, method, parameters, secureMarks }) => {
+      const { target, parent, methodName } = resolve(method, module)
+      if (!target) {
+        log.error('[ASM] Unable to resolve IAST security control %s:%s', filename, method)
+        return
+      }
+
+      let wrapper
+      if (type === SANITIZER_TYPE) {
+        wrapper = wrapSanitizer(target, secureMarks)
+      } else {
+        wrapper = wrapInputValidator(target, parameters, secureMarks)
+      }
+
+      if (methodName) {
+        parent[methodName] = wrapper
+      } else {
+        module = wrapper
+      }
+    })
+  } catch (e) {
+    log.error('[ASM] Error initializing IAST security control for %', filename, e)
+  }
+
+  return module
+}
+
+function resolve (path, obj, separator = '.') {
+  if (!path) {
+    // esm module with default export
+    if (obj?.default) {
+      return { target: obj.default, parent: obj, methodName: 'default' }
+    } else {
+      return { target: obj, parent: obj }
+    }
+  }
+
+  const properties = path.split(separator)
+
+  let parent
+  let methodName
+  const target = properties.reduce((prev, curr) => {
+    parent = prev
+    methodName = curr
+    return prev?.[curr]
+  }, obj)
+
+  return { target, parent, methodName }
+}
+
+function wrapSanitizer (target, secureMarks) {
+  return shimmer.wrapFunction(target, orig => function () {
+    const result = orig.apply(this, arguments)
+
+    try {
+      return addSecureMarks(result, secureMarks)
+    } catch (e) {
+      log.error('[ASM] Error adding Secure mark for sanitizer', e)
+    }
+
+    return result
+  })
+}
+
+function wrapInputValidator (target, parameters, secureMarks) {
+  const allParameters = !parameters?.length
+
+  return shimmer.wrapFunction(target, orig => function () {
+    try {
+      [...arguments].forEach((arg, index) => {
+        if (allParameters || parameters.includes(index)) {
+          addSecureMarks(arg, secureMarks, false)
+        }
+      })
+    } catch (e) {
+      log.error('[ASM] Error adding Secure mark for input validator', e)
+    }
+
+    return orig.apply(this, arguments)
+  })
+}
+
+function addSecureMarks (value, secureMarks, createNewTainted = true) {
+  if (!value) return
+
+  const store = storage('legacy').getStore()
+  const iastContext = getIastContext(store)
+
+  if (typeof value === 'string') {
+    return TaintTrackingOperations.addSecureMark(iastContext, value, secureMarks, createNewTainted)
+  } else {
+    iterateObjectStrings(value, (value, levelKeys, parent, lastKey) => {
+      try {
+        const securedTainted = TaintTrackingOperations.addSecureMark(iastContext, value, secureMarks, createNewTainted)
+        if (createNewTainted) {
+          parent[lastKey] = securedTainted
+        }
+      } catch (e) {
+        // if it is a readonly property, do nothing
+      }
+    })
+    return value
+  }
+}
+
+function disable () {
+  if (moduleLoadStartChannel.hasSubscribers) moduleLoadStartChannel.unsubscribe(onModuleLoaded)
+  if (moduleLoadEndChannel.hasSubscribers) moduleLoadEndChannel.unsubscribe(onModuleLoaded)
+
+  controls = undefined
+  controlsKeys = undefined
+  hooks = undefined
+}
+
+module.exports = {
+  configure,
+  disable
+}

package/packages/dd-trace/src/appsec/iast/security-controls/parser.js

@@ -0,0 +1,96 @@
+'use strict'
+
+const log = require('../../../log')
+const { getMarkFromVulnerabilityType, CUSTOM_SECURE_MARK } = require('../taint-tracking/secure-marks')
+
+const SECURITY_CONTROL_DELIMITER = ';'
+const SECURITY_CONTROL_FIELD_DELIMITER = ':'
+const SECURITY_CONTROL_ELEMENT_DELIMITER = ','
+
+const INPUT_VALIDATOR_TYPE = 'INPUT_VALIDATOR'
+const SANITIZER_TYPE = 'SANITIZER'
+
+const validTypes = [INPUT_VALIDATOR_TYPE, SANITIZER_TYPE]
+
+function parse (securityControlsConfiguration) {
+  const controls = new Map()
+
+  securityControlsConfiguration?.replace(/[\r\n\t\v\f]*/g, '')
+    .split(SECURITY_CONTROL_DELIMITER)
+    .map(parseControl)
+    .filter(control => !!control)
+    .forEach(control => {
+      if (!controls.has(control.file)) {
+        controls.set(control.file, [])
+      }
+      controls.get(control.file).push(control)
+    })
+
+  return controls
+}
+
+function parseControl (control) {
+  if (!control) return
+
+  const fields = control.split(SECURITY_CONTROL_FIELD_DELIMITER)
+
+  if (fields.length < 3 || fields.length > 5) {
+    log.warn('[ASM] Security control configuration is invalid: %s', control)
+    return
+  }
+
+  let [type, marks, file, method, parameters] = fields
+
+  type = type.trim().toUpperCase()
+  if (!validTypes.includes(type)) {
+    log.warn('[ASM] Invalid security control type: %s', type)
+    return
+  }
+
+  let secureMarks = CUSTOM_SECURE_MARK
+  getSecureMarks(marks).forEach(mark => { secureMarks |= mark })
+  if (secureMarks === CUSTOM_SECURE_MARK) {
+    log.warn('[ASM] Invalid security control mark: %s', marks)
+    return
+  }
+
+  file = file?.trim()
+
+  method = method?.trim()
+
+  try {
+    parameters = getParameters(parameters)
+  } catch (e) {
+    log.warn('[ASM] Invalid non-numeric security control parameter %s', parameters)
+    return
+  }
+
+  return { type, secureMarks, file, method, parameters }
+}
+
+function getSecureMarks (marks) {
+  return marks?.split(SECURITY_CONTROL_ELEMENT_DELIMITER)
+    .map(getMarkFromVulnerabilityType)
+    .filter(mark => !!mark)
+}
+
+function getParameters (parameters) {
+  return parameters?.split(SECURITY_CONTROL_ELEMENT_DELIMITER)
+    .map(param => {
+      const parsedParam = parseInt(param, 10)
+
+      // discard the securityControl if there is an incorrect parameter
+      if (isNaN(parsedParam)) {
+        throw new Error('Invalid non-numeric security control parameter')
+      }
+
+      return parsedParam
+    })
+}
+
+module.exports = {
+  parse,
+
+  INPUT_VALIDATOR_TYPE,
+  SANITIZER_TYPE
+}

package/packages/dd-trace/src/appsec/iast/taint-tracking/operations.js

@@ -84,10 +84,10 @@ function getRanges (iastContext, string) {
   return result
 }
 
-function addSecureMark (iastContext, string, mark) {
+function addSecureMark (iastContext, string, mark, createNewTainted = true) {
   const transactionId = iastContext?.[IAST_TRANSACTION_ID]
   if (transactionId) {
-    return TaintedUtils.addSecureMarksToTaintedString(transactionId, string, mark)
+    return TaintedUtils.addSecureMarksToTaintedString(transactionId, string, mark, createNewTainted)
   }
 
   return string

package/packages/dd-trace/src/appsec/iast/taint-tracking/secure-marks-generator.js

@@ -3,7 +3,7 @@
 let next = 0
 
 function getNextSecureMark () {
-  return 1 << next++
+  return (1 << next++) >>> 0
 }
 
 function reset () {

package/packages/dd-trace/src/appsec/iast/taint-tracking/secure-marks.js

@@ -0,0 +1,28 @@
+'use strict'
+
+const vulnerabilities = require('../vulnerabilities')
+const { getNextSecureMark } = require('./secure-marks-generator')
+
+const marks = {}
+Object.keys(vulnerabilities).forEach(vulnerability => {
+  marks[vulnerability + '_MARK'] = getNextSecureMark()
+})
+
+let asterisk = 0x0
+Object.values(marks).forEach(mark => { asterisk |= mark })
+
+marks.ASTERISK_MARK = asterisk
+marks.CUSTOM_SECURE_MARK = getNextSecureMark()
+
+function getMarkFromVulnerabilityType (vulnerabilityType) {
+  vulnerabilityType = vulnerabilityType?.trim()
+  const mark = vulnerabilityType === '*' ? 'ASTERISK_MARK' : vulnerabilityType + '_MARK'
+  return marks[mark]
+}
+
+module.exports = {
+  ...marks,
+  getMarkFromVulnerabilityType,
+
+  ALL: marks
+}

package/packages/dd-trace/src/appsec/iast/telemetry/iast-metric.js

@@ -83,6 +83,9 @@ const REQUEST_TAINTED = new NoTaggedIastMetric('request.tainted', Scope.REQUEST)
 const EXECUTED_PROPAGATION = new NoTaggedIastMetric('executed.propagation', Scope.REQUEST)
 const EXECUTED_TAINTED = new NoTaggedIastMetric('executed.tainted', Scope.REQUEST)
 
+const SUPPRESSED_VULNERABILITIES = new IastMetric('suppressed.vulnerabilities', Scope.REQUEST,
+  TagKey.VULNERABILITY_TYPE)
+
 module.exports = {
   INSTRUMENTED_PROPAGATION,
   INSTRUMENTED_SOURCE,
@@ -95,6 +98,8 @@ module.exports = {
 
   REQUEST_TAINTED,
 
+  SUPPRESSED_VULNERABILITIES,
+
   PropagationType,
   TagKey,
 

package/packages/dd-trace/src/appsec/iast/utils.js

@@ -0,0 +1,24 @@
+'use strict'
+
+function iterateObjectStrings (target, fn, levelKeys = [], depth = 20, visited = new Set()) {
+  if (target !== null && typeof target === 'object') {
+    if (visited.has(target)) return
+
+    visited.add(target)
+
+    Object.keys(target).forEach((key) => {
+      const nextLevelKeys = [...levelKeys, key]
+      const val = target[key]
+
+      if (typeof val === 'string') {
+        fn(val, nextLevelKeys, target, key)
+      } else if (depth > 0) {
+        iterateObjectStrings(val, fn, nextLevelKeys, depth - 1, visited)
+      }
+    })
+  }
+}
+
+module.exports = {
+  iterateObjectStrings
+}

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/index.js

@@ -81,21 +81,16 @@ class VulnerabilityFormatter {
   }
 
   formatVulnerability (vulnerability, sourcesIndexes, sources) {
+    const { type, hash, stackId, evidence, location } = vulnerability
+
     const formattedVulnerability = {
-      type: vulnerability.type,
-      hash: vulnerability.hash,
-      stackId: vulnerability.stackId,
-      evidence: this.formatEvidence(vulnerability.type, vulnerability.evidence, sourcesIndexes, sources),
-      location: {
-        spanId: vulnerability.location.spanId
-      }
-    }
-    if (vulnerability.location.path) {
-      formattedVulnerability.location.path = vulnerability.location.path
-    }
-    if (vulnerability.location.line) {
-      formattedVulnerability.location.line = vulnerability.location.line
+      type,
+      hash,
+      stackId,
+      evidence: this.formatEvidence(type, evidence, sourcesIndexes, sources),
+      location
     }
+
     return formattedVulnerability
   }
 

package/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js

@@ -131,6 +131,7 @@ function replaceCallSiteFromSourceMap (callsite) {
     if (line) {
       callsite.line = line
     }
+    // We send the column in the stack trace but not in the vulnerability location
     if (column) {
       callsite.column = column
     }

package/packages/dd-trace/src/appsec/rasp/command_injection.js

@@ -31,20 +31,20 @@ function analyzeCommandInjection ({ file, fileArgs, shell, abortController }) {
   const req = store?.req
   if (!req) return
 
-  const persistent = {}
+  const ephemeral = {}
   const raspRule = { type: RULE_TYPES.COMMAND_INJECTION }
   const params = fileArgs ? [file, ...fileArgs] : file
 
   if (shell) {
-    persistent[addresses.SHELL_COMMAND] = params
+    ephemeral[addresses.SHELL_COMMAND] = params
     raspRule.variant = 'shell'
   } else {
     const commandParams = Array.isArray(params) ? params : [params]
-    persistent[addresses.EXEC_COMMAND] = commandParams
+    ephemeral[addresses.EXEC_COMMAND] = commandParams
     raspRule.variant = 'exec'
   }
 
-  const result = waf.run({ persistent }, req, raspRule)
+  const result = waf.run({ ephemeral }, req, raspRule)
 
   const res = store?.res
   handleResult(result, req, res, abortController, config)

package/packages/dd-trace/src/appsec/rasp/lfi.js

@@ -54,13 +54,13 @@ function analyzeLfi (ctx) {
   if (!req || !fs) return
 
   getPaths(ctx, fs).forEach(path => {
-    const persistent = {
+    const ephemeral = {
       [FS_OPERATION_PATH]: path
     }
 
     const raspRule = { type: RULE_TYPES.LFI }
 
-    const result = waf.run({ persistent }, req, raspRule)
+    const result = waf.run({ ephemeral }, req, raspRule)
     handleResult(result, req, res, ctx.abortController, config)
   })
 }

package/packages/dd-trace/src/appsec/rasp/sql_injection.js

@@ -67,14 +67,14 @@ function analyzeSqlInjection (query, dbSystem, abortController) {
   }
   executedQueries.add(query)
 
-  const persistent = {
+  const ephemeral = {
     [addresses.DB_STATEMENT]: query,
     [addresses.DB_SYSTEM]: dbSystem
   }
 
   const raspRule = { type: RULE_TYPES.SQL_INJECTION }
 
-  const result = waf.run({ persistent }, req, raspRule)
+  const result = waf.run({ ephemeral }, req, raspRule)
 
   handleResult(result, req, res, abortController, config)
 }

package/packages/dd-trace/src/appsec/rasp/ssrf.js

@@ -25,13 +25,13 @@ function analyzeSsrf (ctx) {
 
   if (!req || !outgoingUrl) return
 
-  const persistent = {
+  const ephemeral = {
     [addresses.HTTP_OUTGOING_URL]: outgoingUrl
   }
 
   const raspRule = { type: RULE_TYPES.SSRF }
 
-  const result = waf.run({ persistent }, req, raspRule)
+  const result = waf.run({ ephemeral }, req, raspRule)
 
   const res = store?.res
   handleResult(result, req, res, ctx.abortController, config)

package/packages/dd-trace/src/ci-visibility/dynamic-instrumentation/worker/index.js

@@ -1,5 +1,5 @@
 'use strict'
-const path = require('path')
+
 const {
   workerData: {
     breakpointSetChannel,
@@ -8,10 +8,11 @@ const {
   }
 } = require('worker_threads')
 const { randomUUID } = require('crypto')
-const sourceMap = require('source-map')
 
 // TODO: move debugger/devtools_client/session to common place
 const session = require('../../../debugger/devtools_client/session')
+// TODO: move debugger/devtools_client/source-maps to common place
+const { getGeneratedPosition } = require('../../../debugger/devtools_client/source-maps')
 // TODO: move debugger/devtools_client/snapshot to common place
 const { getLocalStateForCallFrame } = require('../../../debugger/devtools_client/snapshot')
 // TODO: move debugger/devtools_client/state to common place
@@ -98,17 +99,23 @@ async function addBreakpoint (probe) {
     throw new Error(`No loaded script found for ${file}`)
   }
 
-  const [path, scriptId, sourceMapURL] = script
+  const { url, scriptId, sourceMapURL, source } = script
 
-  log.warn(`Adding breakpoint at ${path}:${line}`)
+  log.warn(`Adding breakpoint at ${url}:${line}`)
 
   let lineNumber = line
+  let columnNumber = 0
 
-  if (sourceMapURL && sourceMapURL.startsWith('data:')) {
+  if (sourceMapURL) {
     try {
-      lineNumber = await processScriptWithInlineSourceMap({ file, line, sourceMapURL })
+      ({ line: lineNumber, column: columnNumber } = await getGeneratedPosition(url, source, line, sourceMapURL))
     } catch (err) {
-      log.error('Error processing script with inline source map', err)
+      log.error('Error processing script with source map', err)
+    }
+    if (lineNumber === null) {
+      log.error('Could not find generated position for %s:%s', url, line)
+      lineNumber = line
+      columnNumber = 0
     }
   }
 
@@ -116,14 +123,15 @@ async function addBreakpoint (probe) {
     const { breakpointId } = await session.post('Debugger.setBreakpoint', {
       location: {
         scriptId,
-        lineNumber: lineNumber - 1
+        lineNumber: lineNumber - 1,
+        columnNumber
       }
     })
 
     breakpointIdToProbe.set(breakpointId, probe)
     probeIdToBreakpointId.set(probe.id, breakpointId)
   } catch (e) {
-    log.error(`Error setting breakpoint at ${path}:${line}:`, e)
+    log.error('Error setting breakpoint at %s:%s', url, line, e)
   }
 }
 
@@ -131,43 +139,3 @@ function start () {
   sessionStarted = true
   return session.post('Debugger.enable') // return instead of await to reduce number of promises created
 }
-
-async function processScriptWithInlineSourceMap (params) {
-  const { file, line, sourceMapURL } = params
-
-  // Extract the base64-encoded source map
-  const base64SourceMap = sourceMapURL.split('base64,')[1]
-
-  // Decode the base64 source map
-  const decodedSourceMap = Buffer.from(base64SourceMap, 'base64').toString('utf8')
-
-  // Parse the source map
-  const consumer = await new sourceMap.SourceMapConsumer(decodedSourceMap)
-
-  let generatedPosition
-
-  // Map to the generated position. We'll attempt with the full file path first, then with the basename.
-  // TODO: figure out why sometimes the full path doesn't work
-  generatedPosition = consumer.generatedPositionFor({
-    source: file,
-    line,
-    column: 0
-  })
-  if (generatedPosition.line === null) {
-    generatedPosition = consumer.generatedPositionFor({
-      source: path.basename(file),
-      line,
-      column: 0
-    })
-  }
-
-  consumer.destroy()
-
-  // If we can't find the line, just return the original line
-  if (generatedPosition.line === null) {
-    log.error(`Could not find generated position for ${file}:${line}`)
-    return line
-  }
-
-  return generatedPosition.line
-}

package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -6,6 +6,7 @@ const { sendGitMetadata: sendGitMetadataRequest } = require('./git/git_metadata'
 const { getLibraryConfiguration: getLibraryConfigurationRequest } = require('../requests/get-library-configuration')
 const { getSkippableSuites: getSkippableSuitesRequest } = require('../intelligent-test-runner/get-skippable-suites')
 const { getKnownTests: getKnownTestsRequest } = require('../early-flake-detection/get-known-tests')
+const { getQuarantinedTests: getQuarantinedTestsRequest } = require('../quarantined-tests/get-quarantined-tests')
 const log = require('../../log')
 const AgentInfoExporter = require('../../exporters/common/agent-info-exporter')
 const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('../../plugins/util/tags')
@@ -92,6 +93,14 @@ class CiVisibilityExporter extends AgentInfoExporter {
     )
   }
 
+  shouldRequestQuarantinedTests () {
+    return !!(
+      this._canUseCiVisProtocol &&
+      this._config.isTestManagementEnabled &&
+      this._libraryConfig?.isQuarantinedTestsEnabled
+    )
+  }
+
   shouldRequestLibraryConfiguration () {
     return this._config.isIntelligentTestRunnerEnabled
   }
@@ -138,6 +147,13 @@ class CiVisibilityExporter extends AgentInfoExporter {
     getKnownTestsRequest(this.getRequestConfiguration(testConfiguration), callback)
   }
 
+  getQuarantinedTests (testConfiguration, callback) {
+    if (!this.shouldRequestQuarantinedTests()) {
+      return callback(null)
+    }
+    getQuarantinedTestsRequest(this.getRequestConfiguration(testConfiguration), callback)
+  }
+
   /**
    * We can't request library configuration until we know whether we can use the
    * CI Visibility Protocol, hence the this._canUseCiVisProtocol promise.
@@ -197,7 +213,8 @@ class CiVisibilityExporter extends AgentInfoExporter {
       earlyFlakeDetectionFaultyThreshold,
       isFlakyTestRetriesEnabled,
       isDiEnabled,
-      isKnownTestsEnabled
+      isKnownTestsEnabled,
+      isQuarantinedTestsEnabled
     } = remoteConfiguration
     return {
       isCodeCoverageEnabled,
@@ -210,7 +227,8 @@ class CiVisibilityExporter extends AgentInfoExporter {
       isFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled && this._config.isFlakyTestRetriesEnabled,
       flakyTestRetriesCount: this._config.flakyTestRetriesCount,
       isDiEnabled: isDiEnabled && this._config.isTestDynamicInstrumentationEnabled,
-      isKnownTestsEnabled
+      isKnownTestsEnabled,
+      isQuarantinedTestsEnabled: isQuarantinedTestsEnabled && this._config.isTestManagementEnabled
     }
   }