dd-trace 4.51.1 → 4.53.0

package/packages/dd-trace/src/appsec/sdk/user_blocking.js

@@ -15,7 +15,7 @@ function isUserBlocked (user) {
 
 function checkUserAndSetUser (tracer, user) {
   if (!user || !user.id) {
-    log.warn('Invalid user provided to isUserBlocked')
+    log.warn('[ASM] Invalid user provided to isUserBlocked')
     return false
   }
 
@@ -25,7 +25,7 @@ function checkUserAndSetUser (tracer, user) {
       setUserTags(user, rootSpan)
     }
   } else {
-    log.warn('Root span not available in isUserBlocked')
+    log.warn('[ASM] Root span not available in isUserBlocked')
   }
 
   return isUserBlocked(user)
@@ -41,13 +41,13 @@ function blockRequest (tracer, req, res) {
   }
 
   if (!req || !res) {
-    log.warn('Requests or response object not available in blockRequest')
+    log.warn('[ASM] Requests or response object not available in blockRequest')
     return false
   }
 
   const rootSpan = getRootSpan(tracer)
   if (!rootSpan) {
-    log.warn('Root span not available in blockRequest')
+    log.warn('[ASM] Root span not available in blockRequest')
     return false
   }
 

package/packages/dd-trace/src/appsec/telemetry.js

@@ -172,6 +172,15 @@ function addRaspRequestMetrics (store, { duration, durationExt }) {
   store[DD_TELEMETRY_REQUEST_METRICS].raspEvalCount++
 }
 
+function incrementMissingUserLoginMetric (framework, eventType) {
+  if (!enabled) return
+
+  appsecMetrics.count('instrum.user_auth.missing_user_login', {
+    framework,
+    event_type: eventType
+  }).inc()
+}
+
 function getRequestMetrics (req) {
   if (req) {
     const store = getStore(req)
@@ -188,6 +197,7 @@ module.exports = {
   incrementWafInitMetric,
   incrementWafUpdatesMetric,
   incrementWafRequestsMetric,
+  incrementMissingUserLoginMetric,
 
   getRequestMetrics
 }

package/packages/dd-trace/src/appsec/user_tracking.js

@@ -0,0 +1,168 @@
+'use strict'
+
+const crypto = require('crypto')
+const log = require('../log')
+const telemetry = require('./telemetry')
+const addresses = require('./addresses')
+const { keepTrace } = require('../priority_sampler')
+const { SAMPLING_MECHANISM_APPSEC } = require('../constants')
+const standalone = require('./standalone')
+const waf = require('./waf')
+
+// the RFC doesn't include '_id', but it's common in MongoDB
+const USER_ID_FIELDS = ['id', '_id', 'email', 'username', 'login', 'user']
+
+let collectionMode
+
+function setCollectionMode (mode, overwrite = true) {
+  // don't overwrite if already set, only used in appsec/index.js to not overwrite RC values
+  if (!overwrite && collectionMode) return
+
+  /* eslint-disable no-fallthrough */
+  switch (mode) {
+    case 'safe':
+      log.warn('[ASM] Using deprecated value "safe" in config.appsec.eventTracking.mode')
+    case 'anon':
+    case 'anonymization':
+      collectionMode = 'anonymization'
+      break
+
+    case 'extended':
+      log.warn('[ASM] Using deprecated value "extended" in config.appsec.eventTracking.mode')
+    case 'ident':
+    case 'identification':
+      collectionMode = 'identification'
+      break
+
+    default:
+      collectionMode = 'disabled'
+  }
+  /* eslint-enable no-fallthrough */
+}
+
+function obfuscateIfNeeded (str) {
+  if (collectionMode === 'anonymization') {
+    // get first 16 bytes of sha256 hash in lowercase hex
+    return 'anon_' + crypto.createHash('sha256').update(str).digest().toString('hex', 0, 16).toLowerCase()
+  } else {
+    return str
+  }
+}
+
+// TODO: should we find other ways to get the user ID ?
+function getUserId (user) {
+  if (!user) return
+
+  for (const field of USER_ID_FIELDS) {
+    let id = user[field]
+
+    // try to find a field that can be stringified
+    if (id && typeof id.toString === 'function') {
+      id = id.toString()
+
+      if (typeof id !== 'string' || id.startsWith('[object ')) {
+        // probably not a usable ID ?
+        continue
+      }
+
+      return obfuscateIfNeeded(id)
+    }
+  }
+}
+
+function trackLogin (framework, login, user, success, rootSpan) {
+  if (!collectionMode || collectionMode === 'disabled') return
+
+  if (!rootSpan) {
+    log.error('[ASM] No rootSpan found in AppSec trackLogin')
+    return
+  }
+
+  if (typeof login !== 'string') {
+    log.error('[ASM] Invalid login provided to AppSec trackLogin')
+
+    telemetry.incrementMissingUserLoginMetric(framework, success ? 'login_success' : 'login_failure')
+    // note:
+    //  if we start supporting using userId if login is missing, we need to only give up if both are missing, and
+    //  implement 'appsec.instrum.user_auth.missing_user_id' telemetry too
+    return
+  }
+
+  login = obfuscateIfNeeded(login)
+  const userId = getUserId(user)
+
+  let newTags
+
+  const persistent = {
+    [addresses.USER_LOGIN]: login
+  }
+
+  const currentTags = rootSpan.context()._tags
+  const isSdkCalled = currentTags[`_dd.appsec.events.users.login.${success ? 'success' : 'failure'}.sdk`] === 'true'
+
+  // used to not overwrite tags set by SDK
+  function shouldSetTag (tag) {
+    return !(isSdkCalled && currentTags[tag])
+  }
+
+  if (success) {
+    newTags = {
+      'appsec.events.users.login.success.track': 'true',
+      '_dd.appsec.events.users.login.success.auto.mode': collectionMode,
+      '_dd.appsec.usr.login': login
+    }
+
+    if (shouldSetTag('appsec.events.users.login.success.usr.login')) {
+      newTags['appsec.events.users.login.success.usr.login'] = login
+    }
+
+    if (userId) {
+      newTags['_dd.appsec.usr.id'] = userId
+
+      if (shouldSetTag('usr.id')) {
+        newTags['usr.id'] = userId
+        persistent[addresses.USER_ID] = userId
+      }
+    }
+
+    persistent[addresses.LOGIN_SUCCESS] = null
+  } else {
+    newTags = {
+      'appsec.events.users.login.failure.track': 'true',
+      '_dd.appsec.events.users.login.failure.auto.mode': collectionMode,
+      '_dd.appsec.usr.login': login
+    }
+
+    if (shouldSetTag('appsec.events.users.login.failure.usr.login')) {
+      newTags['appsec.events.users.login.failure.usr.login'] = login
+    }
+
+    if (userId) {
+      newTags['_dd.appsec.usr.id'] = userId
+
+      if (shouldSetTag('appsec.events.users.login.failure.usr.id')) {
+        newTags['appsec.events.users.login.failure.usr.id'] = userId
+      }
+    }
+
+    /* TODO: if one day we have this info
+    if (exists != null && shouldSetTag('appsec.events.users.login.failure.usr.exists')) {
+      newTags['appsec.events.users.login.failure.usr.exists'] = exists
+    }
+    */
+
+    persistent[addresses.LOGIN_FAILURE] = null
+  }
+
+  keepTrace(rootSpan, SAMPLING_MECHANISM_APPSEC)
+  standalone.sample(rootSpan)
+
+  rootSpan.addTags(newTags)
+
+  return waf.run({ persistent })
+}
+
+module.exports = {
+  setCollectionMode,
+  trackLogin
+}

package/packages/dd-trace/src/appsec/waf/index.js

@@ -41,7 +41,7 @@ function update (newRules) {
   try {
     waf.wafManager.update(newRules)
   } catch (err) {
-    log.error('Could not apply rules from remote config')
+    log.error('[ASM] Could not apply rules from remote config')
     throw err
   }
 }
@@ -50,7 +50,7 @@ function run (data, req, raspRuleType) {
   if (!req) {
     const store = storage.getStore()
     if (!store || !store.req) {
-      log.warn('Request object not available in waf.run')
+      log.warn('[ASM] Request object not available in waf.run')
       return
     }
 

package/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js

@@ -23,7 +23,7 @@ class WAFContextWrapper {
 
   run ({ persistent, ephemeral }, raspRuleType) {
     if (this.ddwafContext.disposed) {
-      log.warn('Calling run on a disposed context')
+      log.warn('[ASM] Calling run on a disposed context')
       return
     }
 
@@ -101,8 +101,7 @@ class WAFContextWrapper {
 
       return result.actions
     } catch (err) {
-      log.error('Error while running the AppSec WAF')
-      log.error(err)
+      log.error('[ASM] Error while running the AppSec WAF', err)
     }
   }
 

package/packages/dd-trace/src/appsec/waf/waf_manager.js

@@ -25,7 +25,7 @@ class WAFManager {
       const { obfuscatorKeyRegex, obfuscatorValueRegex } = this.config
       return new DDWAF(rules, { obfuscatorKeyRegex, obfuscatorValueRegex })
     } catch (err) {
-      log.error('AppSec could not load native package. In-app WAF features will not be available.')
+      log.error('[ASM] AppSec could not load native package. In-app WAF features will not be available.')
 
       throw err
     }

package/packages/dd-trace/src/azure_metadata.js

@@ -1,6 +1,6 @@
 'use strict'
 
-// eslint-disable-next-line max-len
+// eslint-disable-next-line @stylistic/js/max-len
 // Modeled after https://github.com/DataDog/libdatadog/blob/f3994857a59bb5679a65967138c5a3aec418a65f/ddcommon/src/azure_app_services.rs
 
 const os = require('os')
@@ -79,7 +79,7 @@ function buildMetadata () {
 function getAzureAppMetadata () {
   // DD_AZURE_APP_SERVICES is an environment variable introduced by the .NET APM team and is set automatically for
   // anyone using the Datadog APM Extensions (.NET, Java, or Node) for Windows Azure App Services
-  // eslint-disable-next-line max-len
+  // eslint-disable-next-line @stylistic/js/max-len
   // See: https://github.com/DataDog/datadog-aas-extension/blob/01f94b5c28b7fa7a9ab264ca28bd4e03be603900/node/src/applicationHost.xdt#L20-L21
   return process.env.DD_AZURE_APP_SERVICES !== undefined ? buildMetadata() : undefined
 }
@@ -88,9 +88,9 @@ function getAzureFunctionMetadata () {
   return getIsAzureFunction() ? buildMetadata() : undefined
 }
 
-// eslint-disable-next-line max-len
+// eslint-disable-next-line @stylistic/js/max-len
 // Modeled after https://github.com/DataDog/libdatadog/blob/92272e90a7919f07178f3246ef8f82295513cfed/profiling/src/exporter/mod.rs#L187
-// eslint-disable-next-line max-len
+// eslint-disable-next-line @stylistic/js/max-len
 // and https://github.com/DataDog/libdatadog/blob/f3994857a59bb5679a65967138c5a3aec418a65f/trace-utils/src/trace_utils.rs#L533
 function getAzureTagsFromMetadata (metadata) {
   if (metadata === undefined) {

package/packages/dd-trace/src/ci-visibility/dynamic-instrumentation/index.js

@@ -73,8 +73,7 @@ class TestVisDynamicInstrumentation {
     // Allow the parent to exit even if the worker is still running
     this.worker.unref()
 
-    this.breakpointSetChannel.port2.on('message', (message) => {
-      const { probeId } = message
+    this.breakpointSetChannel.port2.on('message', ({ probeId }) => {
       const resolve = probeIdToResolveBreakpointSet.get(probeId)
       if (resolve) {
         resolve()
@@ -82,8 +81,7 @@ class TestVisDynamicInstrumentation {
       }
     }).unref()
 
-    this.breakpointHitChannel.port2.on('message', (message) => {
-      const { snapshot } = message
+    this.breakpointHitChannel.port2.on('message', ({ snapshot }) => {
       const { probe: { id: probeId } } = snapshot
       const resolve = probeIdToResolveBreakpointHit.get(probeId)
       if (resolve) {
@@ -91,6 +89,9 @@ class TestVisDynamicInstrumentation {
         probeIdToResolveBreakpointHit.delete(probeId)
       }
     }).unref()
+
+    this.worker.on('error', (err) => log.error('ci-visibility DI worker error', err))
+    this.worker.on('messageerror', (err) => log.error('ci-visibility DI worker messageerror', err))
   }
 }
 

package/packages/dd-trace/src/ci-visibility/dynamic-instrumentation/worker/index.js

@@ -1,6 +1,8 @@
 'use strict'
-
+const sourceMap = require('source-map')
+const path = require('path')
 const { workerData: { breakpointSetChannel, breakpointHitChannel } } = require('worker_threads')
+
 // TODO: move debugger/devtools_client/session to common place
 const session = require('../../../debugger/devtools_client/session')
 // TODO: move debugger/devtools_client/snapshot to common place
@@ -69,14 +71,24 @@ async function addBreakpoint (snapshotId, probe) {
   const script = findScriptFromPartialPath(file)
   if (!script) throw new Error(`No loaded script found for ${file}`)
 
-  const [path, scriptId] = script
+  const [path, scriptId, sourceMapURL] = script
 
   log.debug(`Adding breakpoint at ${path}:${line}`)
 
+  let lineNumber = line
+
+  if (sourceMapURL && sourceMapURL.startsWith('data:')) {
+    try {
+      lineNumber = await processScriptWithInlineSourceMap({ file, line, sourceMapURL })
+    } catch (err) {
+      log.error(err)
+    }
+  }
+
   const { breakpointId } = await session.post('Debugger.setBreakpoint', {
     location: {
       scriptId,
-      lineNumber: line - 1
+      lineNumber: lineNumber - 1
     }
   })
 
@@ -88,3 +100,27 @@ function start () {
   sessionStarted = true
   return session.post('Debugger.enable') // return instead of await to reduce number of promises created
 }
+
+async function processScriptWithInlineSourceMap (params) {
+  const { file, line, sourceMapURL } = params
+
+  // Extract the base64-encoded source map
+  const base64SourceMap = sourceMapURL.split('base64,')[1]
+
+  // Decode the base64 source map
+  const decodedSourceMap = Buffer.from(base64SourceMap, 'base64').toString('utf8')
+
+  // Parse the source map
+  const consumer = await new sourceMap.SourceMapConsumer(decodedSourceMap)
+
+  // Map to the generated position
+  const generatedPosition = consumer.generatedPositionFor({
+    source: path.basename(file), // this needs to be the file, not the filepath
+    line,
+    column: 0
+  })
+
+  consumer.destroy()
+
+  return generatedPosition.line
+}

package/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js

@@ -63,7 +63,7 @@ class Writer extends BaseWriter {
           TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,
           { endpoint: 'code_coverage' }
         )
-        log.error(err)
+        log.error('Error sending CI coverage payload', err)
         done()
         return
       }

package/packages/dd-trace/src/ci-visibility/exporters/agentless/di-logs-writer.js

@@ -40,7 +40,7 @@ class DynamicInstrumentationLogsWriter extends BaseWriter {
 
     request(data, options, (err, res) => {
       if (err) {
-        log.error(err)
+        log.error('Error sending DI logs payload', err)
         done()
         return
       }

package/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js

@@ -38,7 +38,7 @@ class AgentlessCiVisibilityExporter extends CiVisibilityExporter {
       apiUrl = new URL(apiUrl)
       this._apiUrl = apiUrl
     } catch (e) {
-      log.error(e)
+      log.error('Error setting CI exporter api url', e)
     }
   }
 

package/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js

@@ -64,7 +64,7 @@ class Writer extends BaseWriter {
           TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,
           { endpoint: 'test_cycle' }
         )
-        log.error(err)
+        log.error('Error sending CI agentless payload', err)
         done()
         return
       }

package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -73,6 +73,9 @@ class CiVisibilityExporter extends AgentInfoExporter {
       if (this._coverageWriter) {
         this._coverageWriter.flush()
       }
+      if (this._logsWriter) {
+        this._logsWriter.flush()
+      }
     })
   }
 
@@ -193,7 +196,8 @@ class CiVisibilityExporter extends AgentInfoExporter {
       isEarlyFlakeDetectionEnabled,
       earlyFlakeDetectionNumRetries,
       earlyFlakeDetectionFaultyThreshold,
-      isFlakyTestRetriesEnabled
+      isFlakyTestRetriesEnabled,
+      isDiEnabled
     } = remoteConfiguration
     return {
       isCodeCoverageEnabled,
@@ -204,7 +208,8 @@ class CiVisibilityExporter extends AgentInfoExporter {
       earlyFlakeDetectionNumRetries,
       earlyFlakeDetectionFaultyThreshold,
       isFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled && this._config.isFlakyTestRetriesEnabled,
-      flakyTestRetriesCount: this._config.flakyTestRetriesCount
+      flakyTestRetriesCount: this._config.flakyTestRetriesCount,
+      isDiEnabled: isDiEnabled && this._config.isTestDynamicInstrumentationEnabled
     }
   }
 
@@ -222,7 +227,7 @@ class CiVisibilityExporter extends AgentInfoExporter {
         repositoryUrl,
         (err) => {
           if (err) {
-            log.error(`Error uploading git metadata: ${err.message}`)
+            log.error('Error uploading git metadata: %s', err.message)
           } else {
             log.debug('Successfully uploaded git metadata')
           }
@@ -302,13 +307,28 @@ class CiVisibilityExporter extends AgentInfoExporter {
     if (!this._isInitialized) {
       return done()
     }
-    this._writer.flush(() => {
-      if (this._coverageWriter) {
-        this._coverageWriter.flush(done)
-      } else {
+
+    // TODO: safe to do them at once? Or do we want to do them one by one?
+    const writers = [
+      this._writer,
+      this._coverageWriter,
+      this._logsWriter
+    ].filter(writer => writer)
+
+    let remaining = writers.length
+
+    if (remaining === 0) {
+      return done()
+    }
+
+    const onFlushComplete = () => {
+      remaining -= 1
+      if (remaining === 0) {
         done()
       }
-    })
+    }
+
+    writers.forEach(writer => writer.flush(onFlushComplete))
   }
 
   exportUncodedCoverages () {
@@ -327,7 +347,7 @@ class CiVisibilityExporter extends AgentInfoExporter {
       this._writer.setUrl(url)
       this._coverageWriter.setUrl(coverageUrl)
     } catch (e) {
-      log.error(e)
+      log.error('Error setting CI exporter url', e)
     }
   }
 

package/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js

@@ -92,7 +92,8 @@ function getLibraryConfiguration ({
               itr_enabled: isItrEnabled,
               require_git: requireGit,
               early_flake_detection: earlyFlakeDetectionConfig,
-              flaky_test_retries_enabled: isFlakyTestRetriesEnabled
+              flaky_test_retries_enabled: isFlakyTestRetriesEnabled,
+              di_enabled: isDiEnabled
             }
           }
         } = JSON.parse(res)
@@ -107,7 +108,8 @@ function getLibraryConfiguration ({
             earlyFlakeDetectionConfig?.slow_test_retries?.['5s'] || DEFAULT_EARLY_FLAKE_DETECTION_NUM_RETRIES,
           earlyFlakeDetectionFaultyThreshold:
             earlyFlakeDetectionConfig?.faulty_session_threshold ?? DEFAULT_EARLY_FLAKE_DETECTION_ERROR_THRESHOLD,
-          isFlakyTestRetriesEnabled
+          isFlakyTestRetriesEnabled,
+          isDiEnabled: isDiEnabled && isFlakyTestRetriesEnabled
         }
 
         log.debug(() => `Remote settings: ${JSON.stringify(settings)}`)