dd-trace 3.46.0 → 3.48.0

package/packages/datadog-plugin-aws-sdk/src/services/kinesis.js

@@ -1,10 +1,69 @@
 'use strict'
+const {
+  CONTEXT_PROPAGATION_KEY,
+  getSizeOrZero
+} = require('../../../dd-trace/src/datastreams/processor')
+const { encodePathwayContext } = require('../../../dd-trace/src/datastreams/pathway')
 const log = require('../../../dd-trace/src/log')
 const BaseAwsSdkPlugin = require('../base')
+const { storage } = require('../../../datadog-core')
+
 class Kinesis extends BaseAwsSdkPlugin {
   static get id () { return 'kinesis' }
   static get peerServicePrecursors () { return ['streamname'] }
 
+  constructor (...args) {
+    super(...args)
+
+    // TODO(bengl) Find a way to create the response span tags without this WeakMap being populated
+    // in the base class
+    this.requestTags = new WeakMap()
+
+    this.addSub('apm:aws:response:start:kinesis', obj => {
+      const { request, response } = obj
+      const store = storage.getStore()
+      const plugin = this
+
+      // if we have either of these operations, we want to store the streamName param
+      // since it is not typically available during get/put records requests
+      if (request.operation === 'getShardIterator' || request.operation === 'listShards') {
+        this.storeStreamName(request.params, request.operation, store)
+        return
+      }
+
+      if (request.operation === 'getRecords') {
+        let span
+        const responseExtraction = this.responseExtract(request.params, request.operation, response)
+        if (responseExtraction && responseExtraction.maybeChildOf) {
+          obj.needsFinish = true
+          const options = {
+            childOf: responseExtraction.maybeChildOf,
+            tags: Object.assign(
+              {},
+              this.requestTags.get(request) || {},
+              { 'span.kind': 'server' }
+            )
+          }
+          span = plugin.tracer.startSpan('aws.response', options)
+          this.enter(span, store)
+        }
+
+        // get the stream name that should have been stored previously
+        const { streamName } = storage.getStore()
+
+        // extract DSM context after as we might not have a parent-child but may have a DSM context
+        this.responseExtractDSMContext(
+          request.operation, response, span ?? null, streamName
+        )
+      }
+    })
+
+    this.addSub('apm:aws:response:finish:kinesis', err => {
+      const { span } = storage.getStore()
+      this.finish(span, null, err)
+    })
+  }
+
   generateTags (params, operation, response) {
     if (!params || !params.StreamName) return {}
 
@@ -15,6 +74,58 @@ class Kinesis extends BaseAwsSdkPlugin {
     }
   }
 
+  storeStreamName (params, operation, store) {
+    if (!operation || (operation !== 'getShardIterator' && operation !== 'listShards')) return
+    if (!params || !params.StreamName) return
+
+    const streamName = params.StreamName
+    storage.enterWith({ ...store, streamName })
+  }
+
+  responseExtract (params, operation, response) {
+    if (operation !== 'getRecords') return
+    if (params.Limit && params.Limit !== 1) return
+    if (!response || !response.Records || !response.Records[0]) return
+
+    const record = response.Records[0]
+
+    try {
+      const decodedData = JSON.parse(Buffer.from(record.Data).toString())
+
+      return {
+        maybeChildOf: this.tracer.extract('text_map', decodedData._datadog),
+        parsedAttributes: decodedData._datadog
+      }
+    } catch (e) {
+      log.error(e)
+    }
+  }
+
+  responseExtractDSMContext (operation, response, span, streamName) {
+    if (!this.config.dsmEnabled) return
+    if (operation !== 'getRecords') return
+    if (!response || !response.Records || !response.Records[0]) return
+
+    // we only want to set the payloadSize on the span if we have one message, not repeatedly
+    span = response.Records.length > 1 ? null : span
+
+    response.Records.forEach(record => {
+      const parsedAttributes = JSON.parse(Buffer.from(record.Data).toString())
+
+      if (
+        parsedAttributes &&
+        parsedAttributes._datadog &&
+        parsedAttributes._datadog[CONTEXT_PROPAGATION_KEY] &&
+        streamName
+      ) {
+        const payloadSize = getSizeOrZero(record.Data)
+        this.tracer.decodeDataStreamsContext(Buffer.from(parsedAttributes._datadog[CONTEXT_PROPAGATION_KEY]))
+        this.tracer
+          .setCheckpoint(['direction:in', `topic:${streamName}`, 'type:kinesis'], span, payloadSize)
+      }
+    })
+  }
+
   // AWS-SDK will b64 kinesis payloads
   // or will accept an already b64 encoded payload
   // This method handles both
@@ -37,8 +148,9 @@ class Kinesis extends BaseAwsSdkPlugin {
       if (!request.params) {
         return
       }
-
       const traceData = {}
+
+      // inject data with DD context
       this.tracer.inject(span, 'text_map', traceData)
       let injectPath
       if (request.params.Records && request.params.Records.length > 0) {
@@ -49,9 +161,30 @@ class Kinesis extends BaseAwsSdkPlugin {
         log.error('No valid payload passed, unable to pass trace context')
         return
       }
+
       const parsedData = this._tryParse(injectPath.Data)
       if (parsedData) {
         parsedData._datadog = traceData
+
+        // set DSM hash if enabled
+        if (this.config.dsmEnabled) {
+          // get payload size of request data
+          const payloadSize = Buffer.from(JSON.stringify(parsedData)).byteLength
+          let stream
+          // users can optionally use either stream name or stream arn
+          if (request.params && request.params.StreamArn) {
+            stream = request.params.StreamArn
+          } else if (request.params && request.params.StreamName) {
+            stream = request.params.StreamName
+          }
+          const dataStreamsContext = this.tracer
+            .setCheckpoint(['direction:out', `topic:${stream}`, 'type:kinesis'], span, payloadSize)
+          if (dataStreamsContext) {
+            const pathwayCtx = encodePathwayContext(dataStreamsContext)
+            parsedData._datadog[CONTEXT_PROPAGATION_KEY] = pathwayCtx.toJSON()
+          }
+        }
+
         const finalData = Buffer.from(JSON.stringify(parsedData))
         const byteSize = finalData.length
         // Kinesis max payload size is 1MB

package/packages/datadog-plugin-aws-sdk/src/services/sns.js

@@ -1,4 +1,6 @@
 'use strict'
+const { CONTEXT_PROPAGATION_KEY, getHeadersSize } = require('../../../dd-trace/src/datastreams/processor')
+const { encodePathwayContext } = require('../../../dd-trace/src/datastreams/pathway')
 const log = require('../../../dd-trace/src/log')
 const BaseAwsSdkPlugin = require('../base')
 
@@ -11,6 +13,7 @@ class Sns extends BaseAwsSdkPlugin {
 
     if (!params.TopicArn && !(response.data && response.data.TopicArn)) return {}
     const TopicArn = params.TopicArn || response.data.TopicArn
+
     // Split the ARN into its parts
     // ex.'arn:aws:sns:us-east-1:123456789012:my-topic'
     const arnParts = TopicArn.split(':')
@@ -72,10 +75,25 @@ class Sns extends BaseAwsSdkPlugin {
     }
     const ddInfo = {}
     this.tracer.inject(span, 'text_map', ddInfo)
+    // add ddInfo before checking DSM so we can include DD attributes in payload size
     params.MessageAttributes._datadog = {
       DataType: 'Binary',
-      BinaryValue: Buffer.from(JSON.stringify(ddInfo)) // BINARY types are automatically base64 encoded
+      BinaryValue: ddInfo
+    }
+    if (this.config.dsmEnabled) {
+      const payloadSize = getHeadersSize({
+        Message: params.Message,
+        MessageAttributes: params.MessageAttributes
+      })
+      const dataStreamsContext = this.tracer
+        .setCheckpoint(['direction:out', `topic:${params.TopicArn}`, 'type:sns'], span, payloadSize)
+      if (dataStreamsContext) {
+        const pathwayCtx = encodePathwayContext(dataStreamsContext)
+        ddInfo[CONTEXT_PROPAGATION_KEY] = pathwayCtx.toJSON()
+      }
     }
+    // BINARY types are automatically base64 encoded
+    params.MessageAttributes._datadog.BinaryValue = Buffer.from(JSON.stringify(ddInfo))
   }
 }
 

package/packages/datadog-plugin-aws-sdk/src/services/sqs.js

@@ -3,6 +3,8 @@
 const log = require('../../../dd-trace/src/log')
 const BaseAwsSdkPlugin = require('../base')
 const { storage } = require('../../../datadog-core')
+const { CONTEXT_PROPAGATION_KEY, getHeadersSize } = require('../../../dd-trace/src/datastreams/processor')
+const { encodePathwayContext } = require('../../../dd-trace/src/datastreams/pathway')
 
 class Sqs extends BaseAwsSdkPlugin {
   static get id () { return 'sqs' }
@@ -19,20 +21,27 @@ class Sqs extends BaseAwsSdkPlugin {
       const { request, response } = obj
       const store = storage.getStore()
       const plugin = this
-      const maybeChildOf = this.responseExtract(request.params, request.operation, response)
-      if (maybeChildOf) {
+      const contextExtraction = this.responseExtract(request.params, request.operation, response)
+      let span
+      let parsedMessageAttributes
+      if (contextExtraction && contextExtraction.datadogContext) {
         obj.needsFinish = true
         const options = {
-          childOf: maybeChildOf,
+          childOf: contextExtraction.datadogContext,
           tags: Object.assign(
             {},
             this.requestTags.get(request) || {},
             { 'span.kind': 'server' }
           )
         }
-        const span = plugin.tracer.startSpan('aws.response', options)
+        parsedMessageAttributes = contextExtraction.parsedAttributes
+        span = plugin.tracer.startSpan('aws.response', options)
         this.enter(span, store)
       }
+      // extract DSM context after as we might not have a parent-child but may have a DSM context
+      this.responseExtractDSMContext(
+        request.operation, request.params, response, span ?? null, parsedMessageAttributes ?? null
+      )
     })
 
     this.addSub('apm:aws:response:finish:sqs', err => {
@@ -133,19 +142,69 @@ class Sqs extends BaseAwsSdkPlugin {
 
     const datadogAttribute = message.MessageAttributes._datadog
 
+    const parsedAttributes = this.parseDatadogAttributes(datadogAttribute)
+    if (parsedAttributes) {
+      return {
+        datadogContext: this.tracer.extract('text_map', parsedAttributes),
+        parsedAttributes: parsedAttributes
+      }
+    }
+  }
+
+  parseDatadogAttributes (attributes) {
     try {
-      if (datadogAttribute.StringValue) {
-        const textMap = datadogAttribute.StringValue
-        return this.tracer.extract('text_map', JSON.parse(textMap))
-      } else if (datadogAttribute.Type === 'Binary') {
-        const buffer = Buffer.from(datadogAttribute.Value, 'base64')
-        return this.tracer.extract('text_map', JSON.parse(buffer))
+      if (attributes.StringValue) {
+        const textMap = attributes.StringValue
+        return JSON.parse(textMap)
+      } else if (attributes.Type === 'Binary') {
+        const buffer = Buffer.from(attributes.Value, 'base64')
+        return JSON.parse(buffer)
       }
     } catch (e) {
       log.error(e)
     }
   }
 
+  responseExtractDSMContext (operation, params, response, span, parsedAttributes) {
+    if (!this.config.dsmEnabled) return
+    if (operation !== 'receiveMessage') return
+    if (!response || !response.Messages || !response.Messages[0]) return
+
+    // we only want to set the payloadSize on the span if we have one message
+    span = response.Messages.length > 1 ? null : span
+
+    response.Messages.forEach(message => {
+      // we may have already parsed the message attributes when extracting trace context
+      if (!parsedAttributes) {
+        if (message.Body) {
+          try {
+            const body = JSON.parse(message.Body)
+
+            // SNS to SQS
+            if (body.Type === 'Notification') {
+              message = body
+            }
+          } catch (e) {
+            // SQS to SQS
+          }
+        }
+        if (message.MessageAttributes && message.MessageAttributes._datadog) {
+          parsedAttributes = this.parseDatadogAttributes(message.MessageAttributes._datadog)
+        }
+      }
+      if (parsedAttributes && parsedAttributes[CONTEXT_PROPAGATION_KEY]) {
+        const payloadSize = getHeadersSize({
+          Body: message.Body,
+          MessageAttributes: message.MessageAttributes
+        })
+        const queue = params.QueueUrl.split('/').pop()
+        this.tracer.decodeDataStreamsContext(Buffer.from(parsedAttributes[CONTEXT_PROPAGATION_KEY]))
+        this.tracer
+          .setCheckpoint(['direction:in', `topic:${queue}`, 'type:sqs'], span, payloadSize)
+      }
+    })
+  }
+
   requestInject (span, request) {
     const operation = request.operation
     if (operation === 'sendMessage') {
@@ -164,6 +223,20 @@ class Sqs extends BaseAwsSdkPlugin {
         DataType: 'String',
         StringValue: JSON.stringify(ddInfo)
       }
+      if (this.config.dsmEnabled) {
+        const payloadSize = getHeadersSize({
+          Body: request.params.MessageBody,
+          MessageAttributes: request.params.MessageAttributes
+        })
+        const queue = request.params.QueueUrl.split('/').pop()
+        const dataStreamsContext = this.tracer
+          .setCheckpoint(['direction:out', `topic:${queue}`, 'type:sqs'], span, payloadSize)
+        if (dataStreamsContext) {
+          const pathwayCtx = encodePathwayContext(dataStreamsContext)
+          ddInfo[CONTEXT_PROPAGATION_KEY] = pathwayCtx.toJSON()
+        }
+      }
+      request.params.MessageAttributes._datadog.StringValue = JSON.stringify(ddInfo)
     }
   }
 }

package/packages/datadog-plugin-child_process/src/index.js

@@ -0,0 +1,91 @@
+'use strict'
+
+const TracingPlugin = require('../../dd-trace/src/plugins/tracing')
+const scrubChildProcessCmd = require('./scrub-cmd-params')
+
+const MAX_ARG_SIZE = 4096 // 4kB
+
+function truncateCommand (cmdFields) {
+  let size = cmdFields[0].length
+  let truncated = false
+  for (let i = 1; i < cmdFields.length; i++) {
+    if (size >= MAX_ARG_SIZE) {
+      truncated = true
+      cmdFields[i] = ''
+      continue
+    }
+
+    const argLen = cmdFields[i].length
+    if (size < MAX_ARG_SIZE && size + argLen > MAX_ARG_SIZE) {
+      cmdFields[i] = cmdFields[i].substring(0, 2)
+      truncated = true
+    }
+
+    size += argLen
+  }
+
+  return truncated
+}
+
+class ChildProcessPlugin extends TracingPlugin {
+  static get id () { return 'child_process' }
+  static get prefix () { return 'tracing:datadog:child_process:execution' }
+
+  get tracer () {
+    return this._tracer
+  }
+
+  start ({ command, shell }) {
+    if (typeof command !== 'string') {
+      return
+    }
+
+    const cmdFields = scrubChildProcessCmd(command)
+    const truncated = truncateCommand(cmdFields)
+    const property = (shell === true) ? 'cmd.shell' : 'cmd.exec'
+
+    const meta = {
+      'component': 'subprocess',
+      [property]: (shell === true) ? cmdFields.join(' ') : JSON.stringify(cmdFields)
+    }
+
+    if (truncated) {
+      meta['cmd.truncated'] = `${truncated}`
+    }
+
+    this.startSpan('command_execution', {
+      service: this.config.service,
+      resource: (shell === true) ? 'sh' : cmdFields[0],
+      type: 'system',
+      meta
+    })
+  }
+
+  end ({ result, error }) {
+    let exitCode
+
+    if (result !== undefined) {
+      exitCode = result?.status || 0
+    } else if (error !== undefined) {
+      exitCode = error?.status || error?.code || 0
+    } else {
+      // TracingChannels call start, end synchronously. Later when the promise is resolved then asyncStart asyncEnd.
+      // Therefore in the case of calling end with neither result nor error means that they will come in the asyncEnd.
+      return
+    }
+
+    this.activeSpan?.setTag('cmd.exit_code', `${exitCode}`)
+    this.activeSpan?.finish()
+  }
+
+  error (error) {
+    this.addError(error)
+  }
+
+  asyncEnd ({ result }) {
+    this.activeSpan?.setTag('cmd.exit_code', `${result}`)
+    this.activeSpan?.finish()
+  }
+}
+
+module.exports = ChildProcessPlugin

package/packages/datadog-plugin-child_process/src/scrub-cmd-params.js

@@ -0,0 +1,125 @@
+'use strict'
+
+const shellParser = require('shell-quote/parse')
+
+const ALLOWED_ENV_VARIABLES = ['LD_PRELOAD', 'LD_LIBRARY_PATH', 'PATH']
+const PROCESS_DENYLIST = ['md5']
+
+const VARNAMES_REGEX = /\$([\w\d_]*)(?:[^\w\d_]|$)/gmi
+// eslint-disable-next-line max-len
+const PARAM_PATTERN = '^-{0,2}(?:p(?:ass(?:w(?:or)?d)?)?|api_?key|secret|a(?:ccess|uth)_token|mysql_pwd|credentials|(?:stripe)?token)$'
+const regexParam = new RegExp(PARAM_PATTERN, 'i')
+const ENV_PATTERN = '^(\\w+=\\w+;)*\\w+=\\w+;?$'
+const envvarRegex = new RegExp(ENV_PATTERN)
+const REDACTED = '?'
+
+function extractVarNames (expression) {
+  const varNames = new Set()
+  let match
+
+  while ((match = VARNAMES_REGEX.exec(expression))) {
+    varNames.add(match[1])
+  }
+
+  const varNamesObject = {}
+  for (const varName of varNames.keys()) {
+    varNamesObject[varName] = `$${varName}`
+  }
+  return varNamesObject
+}
+
+function getTokensByExpression (expressionTokens) {
+  const expressionListTokens = []
+  let wipExpressionTokens = []
+  let isNewExpression = true
+
+  expressionTokens.forEach(token => {
+    if (isNewExpression) {
+      expressionListTokens.push(wipExpressionTokens)
+      isNewExpression = false
+    }
+
+    wipExpressionTokens.push(token)
+
+    if (token.op) {
+      wipExpressionTokens = []
+      isNewExpression = true
+    }
+  })
+  return expressionListTokens
+}
+
+function scrubChildProcessCmd (expression) {
+  const varNames = extractVarNames(expression)
+  const expressionTokens = shellParser(expression, varNames)
+
+  const expressionListTokens = getTokensByExpression(expressionTokens)
+
+  const result = []
+  expressionListTokens.forEach((expressionTokens) => {
+    let foundBinary = false
+    for (let index = 0; index < expressionTokens.length; index++) {
+      const token = expressionTokens[index]
+
+      if (typeof token === 'object') {
+        if (token.pattern) {
+          result.push(token.pattern)
+        } else if (token.op) {
+          result.push(token.op)
+        } else if (token.comment) {
+          result.push(`#${token.comment}`)
+        }
+      } else if (!foundBinary) {
+        if (envvarRegex.test(token)) {
+          const envSplit = token.split('=')
+
+          if (!ALLOWED_ENV_VARIABLES.includes(envSplit[0])) {
+            envSplit[1] = REDACTED
+
+            const newToken = envSplit.join('=')
+            expressionTokens[index] = newToken
+
+            result.push(newToken)
+          } else {
+            result.push(token)
+          }
+        } else {
+          foundBinary = true
+          result.push(token)
+
+          if (PROCESS_DENYLIST.includes(token)) {
+            for (index++; index < expressionTokens.length; index++) {
+              const token = expressionTokens[index]
+
+              if (token.op) {
+                result.push(token.op)
+              } else {
+                expressionTokens[index] = REDACTED
+                result.push(REDACTED)
+              }
+            }
+            break
+          }
+        }
+      } else {
+        const paramKeyValue = token.split('=')
+        const paramKey = paramKeyValue[0]
+
+        if (regexParam.test(paramKey)) {
+          if (paramKeyValue.length === 1) {
+            expressionTokens[index + 1] = REDACTED
+            result.push(token)
+          } else {
+            result.push(`${paramKey}=${REDACTED}`)
+          }
+        } else {
+          result.push(token)
+        }
+      }
+    }
+  })
+
+  return result
+}
+
+module.exports = scrubChildProcessCmd

package/packages/datadog-plugin-cucumber/src/index.js

@@ -14,7 +14,8 @@ const {
   TEST_ITR_UNSKIPPABLE,
   TEST_ITR_FORCED_RUN,
   TEST_CODE_OWNERS,
-  ITR_CORRELATION_ID
+  ITR_CORRELATION_ID,
+  TEST_SOURCE_FILE
 } = require('../../dd-trace/src/plugins/util/test')
 const { RESOURCE_NAME } = require('../../../ext/tags')
 const { COMPONENT, ERROR_MESSAGE } = require('../../dd-trace/src/constants')
@@ -47,7 +48,7 @@ class CucumberPlugin extends CiPlugin {
       hasUnskippableSuites,
       hasForcedToRunSuites
     }) => {
-      const { isSuitesSkippingEnabled, isCodeCoverageEnabled } = this.itrConfig || {}
+      const { isSuitesSkippingEnabled, isCodeCoverageEnabled } = this.libraryConfig || {}
       addIntelligentTestRunnerSpanTags(
         this.testSessionSpan,
         this.testModuleSpan,
@@ -71,7 +72,7 @@ class CucumberPlugin extends CiPlugin {
       this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'session')
       finishAllTraceSpans(this.testSessionSpan)
 
-      this.itrConfig = null
+      this.libraryConfig = null
       this.tracer._exporter.flush()
     })
 
@@ -102,7 +103,7 @@ class CucumberPlugin extends CiPlugin {
         }
       })
       this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite')
-      if (this.itrConfig?.isCodeCoverageEnabled) {
+      if (this.libraryConfig?.isCodeCoverageEnabled) {
         this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_STARTED, 'suite', { library: 'istanbul' })
       }
     })
@@ -114,7 +115,7 @@ class CucumberPlugin extends CiPlugin {
     })
 
     this.addSub('ci:cucumber:test-suite:code-coverage', ({ coverageFiles, suiteFile }) => {
-      if (!this.itrConfig?.isCodeCoverageEnabled) {
+      if (!this.libraryConfig?.isCodeCoverageEnabled) {
         return
       }
       if (!coverageFiles.length) {
@@ -122,7 +123,7 @@ class CucumberPlugin extends CiPlugin {
       }
 
       const relativeCoverageFiles = [...coverageFiles, suiteFile]
-        .map(filename => getTestSuitePath(filename, this.sourceRoot))
+        .map(filename => getTestSuitePath(filename, this.repositoryRoot))
 
       this.telemetry.distribution(TELEMETRY_CODE_COVERAGE_NUM_FILES, {}, relativeCoverageFiles.length)
 
@@ -136,10 +137,11 @@ class CucumberPlugin extends CiPlugin {
       this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_FINISHED, 'suite', { library: 'istanbul' })
     })
 
-    this.addSub('ci:cucumber:test:start', ({ testName, fullTestSuite, testSourceLine }) => {
+    this.addSub('ci:cucumber:test:start', ({ testName, testFileAbsolutePath, testSourceLine }) => {
       const store = storage.getStore()
-      const testSuite = getTestSuitePath(fullTestSuite, this.sourceRoot)
-      const testSpan = this.startTestSpan(testName, testSuite, testSourceLine)
+      const testSuite = getTestSuitePath(testFileAbsolutePath, this.sourceRoot)
+      const testSourceFile = getTestSuitePath(testFileAbsolutePath, this.repositoryRoot)
+      const testSpan = this.startTestSpan(testName, testSuite, testSourceFile, testSourceLine)
 
       this.enter(testSpan, store)
     })
@@ -191,12 +193,15 @@ class CucumberPlugin extends CiPlugin {
     })
   }
 
-  startTestSpan (testName, testSuite, testSourceLine) {
+  startTestSpan (testName, testSuite, testSourceFile, testSourceLine) {
     return super.startTestSpan(
       testName,
       testSuite,
       this.testSuiteSpan,
-      { [TEST_SOURCE_START]: testSourceLine }
+      {
+        [TEST_SOURCE_START]: testSourceLine,
+        [TEST_SOURCE_FILE]: testSourceFile
+      }
     )
   }
 }