dd-trace 5.2.0 → 5.4.0

package/packages/datadog-instrumentations/src/playwright.js

@@ -73,14 +73,41 @@ function getRootDir (playwrightRunner) {
   if (playwrightRunner._configDir) {
     return playwrightRunner._configDir
   }
-  if (playwrightRunner._config && playwrightRunner._config.config) {
-    return playwrightRunner._config.config.rootDir
+  if (playwrightRunner._config) {
+    return playwrightRunner._config.config?.rootDir || process.cwd()
   }
   return process.cwd()
 }
 
-function testBeginHandler (test) {
-  const { _requireFile: testSuiteAbsolutePath, title: testName, _type, location: { line: testSourceLine } } = test
+function getProjectsFromRunner (runner) {
+  const config = getPlaywrightConfig(runner)
+  return config.projects?.map(({ project }) => project)
+}
+
+function getProjectsFromDispatcher (dispatcher) {
+  const newConfig = dispatcher._config?.config?.projects
+  if (newConfig) {
+    return newConfig
+  }
+  // old
+  return dispatcher._loader?.fullConfig()?.projects
+}
+
+function getBrowserNameFromProjects (projects, projectId) {
+  if (!projects) {
+    return null
+  }
+  return projects.find(project =>
+    project.__projectId === projectId || project._id === projectId
+  )?.name
+}
+
+function testBeginHandler (test, browserName) {
+  const {
+    _requireFile: testSuiteAbsolutePath,
+    title: testName, _type,
+    location: { line: testSourceLine }
+  } = test
 
   if (_type === 'beforeAll' || _type === 'afterAll') {
     return
@@ -100,7 +127,7 @@ function testBeginHandler (test) {
   const testAsyncResource = new AsyncResource('bound-anonymous-fn')
   testToAr.set(test, testAsyncResource)
   testAsyncResource.runInAsyncScope(() => {
-    testStartCh.publish({ testName, testSuiteAbsolutePath, testSourceLine })
+    testStartCh.publish({ testName, testSuiteAbsolutePath, testSourceLine, browserName })
   })
 }
 
@@ -166,11 +193,12 @@ function dispatcherHook (dispatcherExport) {
   shimmer.wrap(dispatcherExport.Dispatcher.prototype, '_createWorker', createWorker => function () {
     const dispatcher = this
     const worker = createWorker.apply(this, arguments)
-
     worker.process.on('message', ({ method, params }) => {
       if (method === 'testBegin') {
         const { test } = dispatcher._testById.get(params.testId)
-        testBeginHandler(test)
+        const projects = getProjectsFromDispatcher(dispatcher)
+        const browser = getBrowserNameFromProjects(projects, test._projectId)
+        testBeginHandler(test, browser)
       } else if (method === 'testEnd') {
         const { test } = dispatcher._testById.get(params.testId)
 
@@ -203,7 +231,9 @@ function dispatcherHookNew (dispatcherExport, runWrapper) {
 
     worker.on('testBegin', ({ testId }) => {
       const test = getTestByTestId(dispatcher, testId)
-      testBeginHandler(test)
+      const projects = getProjectsFromDispatcher(dispatcher)
+      const browser = getBrowserNameFromProjects(projects, test._projectId)
+      testBeginHandler(test, browser)
     })
     worker.on('testEnd', ({ testId, status, errors, annotations }) => {
       const test = getTestByTestId(dispatcher, testId)
@@ -226,6 +256,7 @@ function runnerHook (runnerExport, playwrightVersion) {
     testSessionAsyncResource.runInAsyncScope(() => {
       testSessionStartCh.publish({ command, frameworkVersion: playwrightVersion, rootDir })
     })
+    const projects = getProjectsFromRunner(this)
 
     const runAllTestsReturn = await runAllTests.apply(this, arguments)
 
@@ -234,7 +265,8 @@ function runnerHook (runnerExport, playwrightVersion) {
       // there were tests that did not go through `testBegin` or `testEnd`,
       // because they were skipped
       tests.forEach(test => {
-        testBeginHandler(test)
+        const browser = getBrowserNameFromProjects(projects, test._projectId)
+        testBeginHandler(test, browser)
         testEndHandler(test, [], 'skip')
       })
     })

package/packages/datadog-plugin-amqplib/src/consumer.js

@@ -2,6 +2,7 @@
 
 const { TEXT_MAP } = require('../../../ext/formats')
 const ConsumerPlugin = require('../../dd-trace/src/plugins/consumer')
+const { getAmqpMessageSize, CONTEXT_PROPAGATION_KEY } = require('../../dd-trace/src/datastreams/processor')
 const { getResourceName } = require('./util')
 
 class AmqplibConsumerPlugin extends ConsumerPlugin {
@@ -13,7 +14,7 @@ class AmqplibConsumerPlugin extends ConsumerPlugin {
 
     const childOf = extract(this.tracer, message)
 
-    this.startSpan({
+    const span = this.startSpan({
       childOf,
       resource: getResourceName(method, fields),
       type: 'worker',
@@ -26,6 +27,14 @@ class AmqplibConsumerPlugin extends ConsumerPlugin {
         'amqp.destination': fields.destination
       }
     })
+
+    if (this.config.dsmEnabled && message) {
+      const payloadSize = getAmqpMessageSize({ headers: message.properties.headers, content: message.content })
+      const queue = fields.queue ?? fields.routingKey
+      this.tracer.decodeDataStreamsContext(message.properties.headers[CONTEXT_PROPAGATION_KEY])
+      this.tracer
+        .setCheckpoint(['direction:in', `topic:${queue}`, 'type:rabbitmq'], span, payloadSize)
+    }
   }
 }
 

package/packages/datadog-plugin-amqplib/src/producer.js

@@ -3,13 +3,15 @@
 const { TEXT_MAP } = require('../../../ext/formats')
 const { CLIENT_PORT_KEY } = require('../../dd-trace/src/constants')
 const ProducerPlugin = require('../../dd-trace/src/plugins/producer')
+const { encodePathwayContext } = require('../../dd-trace/src/datastreams/pathway')
+const { getAmqpMessageSize, CONTEXT_PROPAGATION_KEY } = require('../../dd-trace/src/datastreams/processor')
 const { getResourceName } = require('./util')
 
 class AmqplibProducerPlugin extends ProducerPlugin {
   static get id () { return 'amqplib' }
   static get operation () { return 'command' }
 
-  start ({ channel = {}, method, fields }) {
+  start ({ channel = {}, method, fields, message }) {
     if (method !== 'basic.publish') return
 
     const stream = (channel.connection && channel.connection.stream) || {}
@@ -30,6 +32,17 @@ class AmqplibProducerPlugin extends ProducerPlugin {
     fields.headers = fields.headers || {}
 
     this.tracer.inject(span, TEXT_MAP, fields.headers)
+
+    if (this.config.dsmEnabled) {
+      const hasRoutingKey = fields.routingKey != null
+      const payloadSize = getAmqpMessageSize({ content: message, headers: fields.headers })
+      const dataStreamsContext = this.tracer
+        .setCheckpoint(
+          ['direction:out', `exchange:${fields.exchange}`, `has_routing_key:${hasRoutingKey}`, 'type:rabbitmq']
+          , span, payloadSize)
+      const pathwayCtx = encodePathwayContext(dataStreamsContext)
+      fields.headers[CONTEXT_PROPAGATION_KEY] = pathwayCtx
+    }
   }
 }
 

package/packages/datadog-plugin-aws-sdk/src/services/kinesis.js

@@ -1,15 +1,69 @@
 'use strict'
 const {
-  CONTEXT_PROPAGATION_KEY
+  CONTEXT_PROPAGATION_KEY,
+  getSizeOrZero
 } = require('../../../dd-trace/src/datastreams/processor')
 const { encodePathwayContext } = require('../../../dd-trace/src/datastreams/pathway')
 const log = require('../../../dd-trace/src/log')
 const BaseAwsSdkPlugin = require('../base')
+const { storage } = require('../../../datadog-core')
 
 class Kinesis extends BaseAwsSdkPlugin {
   static get id () { return 'kinesis' }
   static get peerServicePrecursors () { return ['streamname'] }
 
+  constructor (...args) {
+    super(...args)
+
+    // TODO(bengl) Find a way to create the response span tags without this WeakMap being populated
+    // in the base class
+    this.requestTags = new WeakMap()
+
+    this.addSub('apm:aws:response:start:kinesis', obj => {
+      const { request, response } = obj
+      const store = storage.getStore()
+      const plugin = this
+
+      // if we have either of these operations, we want to store the streamName param
+      // since it is not typically available during get/put records requests
+      if (request.operation === 'getShardIterator' || request.operation === 'listShards') {
+        this.storeStreamName(request.params, request.operation, store)
+        return
+      }
+
+      if (request.operation === 'getRecords') {
+        let span
+        const responseExtraction = this.responseExtract(request.params, request.operation, response)
+        if (responseExtraction && responseExtraction.maybeChildOf) {
+          obj.needsFinish = true
+          const options = {
+            childOf: responseExtraction.maybeChildOf,
+            tags: Object.assign(
+              {},
+              this.requestTags.get(request) || {},
+              { 'span.kind': 'server' }
+            )
+          }
+          span = plugin.tracer.startSpan('aws.response', options)
+          this.enter(span, store)
+        }
+
+        // get the stream name that should have been stored previously
+        const { streamName } = storage.getStore()
+
+        // extract DSM context after as we might not have a parent-child but may have a DSM context
+        this.responseExtractDSMContext(
+          request.operation, response, span ?? null, streamName
+        )
+      }
+    })
+
+    this.addSub('apm:aws:response:finish:kinesis', err => {
+      const { span } = storage.getStore()
+      this.finish(span, null, err)
+    })
+  }
+
   generateTags (params, operation, response) {
     if (!params || !params.StreamName) return {}
 
@@ -20,6 +74,58 @@ class Kinesis extends BaseAwsSdkPlugin {
     }
   }
 
+  storeStreamName (params, operation, store) {
+    if (!operation || (operation !== 'getShardIterator' && operation !== 'listShards')) return
+    if (!params || !params.StreamName) return
+
+    const streamName = params.StreamName
+    storage.enterWith({ ...store, streamName })
+  }
+
+  responseExtract (params, operation, response) {
+    if (operation !== 'getRecords') return
+    if (params.Limit && params.Limit !== 1) return
+    if (!response || !response.Records || !response.Records[0]) return
+
+    const record = response.Records[0]
+
+    try {
+      const decodedData = JSON.parse(Buffer.from(record.Data).toString())
+
+      return {
+        maybeChildOf: this.tracer.extract('text_map', decodedData._datadog),
+        parsedAttributes: decodedData._datadog
+      }
+    } catch (e) {
+      log.error(e)
+    }
+  }
+
+  responseExtractDSMContext (operation, response, span, streamName) {
+    if (!this.config.dsmEnabled) return
+    if (operation !== 'getRecords') return
+    if (!response || !response.Records || !response.Records[0]) return
+
+    // we only want to set the payloadSize on the span if we have one message, not repeatedly
+    span = response.Records.length > 1 ? null : span
+
+    response.Records.forEach(record => {
+      const parsedAttributes = JSON.parse(Buffer.from(record.Data).toString())
+
+      if (
+        parsedAttributes &&
+        parsedAttributes._datadog &&
+        parsedAttributes._datadog[CONTEXT_PROPAGATION_KEY] &&
+        streamName
+      ) {
+        const payloadSize = getSizeOrZero(record.Data)
+        this.tracer.decodeDataStreamsContext(Buffer.from(parsedAttributes._datadog[CONTEXT_PROPAGATION_KEY]))
+        this.tracer
+          .setCheckpoint(['direction:in', `topic:${streamName}`, 'type:kinesis'], span, payloadSize)
+      }
+    })
+  }
+
   // AWS-SDK will b64 kinesis payloads
   // or will accept an already b64 encoded payload
   // This method handles both

package/packages/datadog-plugin-child_process/src/index.js

@@ -0,0 +1,91 @@
+'use strict'
+
+const TracingPlugin = require('../../dd-trace/src/plugins/tracing')
+const scrubChildProcessCmd = require('./scrub-cmd-params')
+
+const MAX_ARG_SIZE = 4096 // 4kB
+
+function truncateCommand (cmdFields) {
+  let size = cmdFields[0].length
+  let truncated = false
+  for (let i = 1; i < cmdFields.length; i++) {
+    if (size >= MAX_ARG_SIZE) {
+      truncated = true
+      cmdFields[i] = ''
+      continue
+    }
+
+    const argLen = cmdFields[i].length
+    if (size < MAX_ARG_SIZE && size + argLen > MAX_ARG_SIZE) {
+      cmdFields[i] = cmdFields[i].substring(0, 2)
+      truncated = true
+    }
+
+    size += argLen
+  }
+
+  return truncated
+}
+
+class ChildProcessPlugin extends TracingPlugin {
+  static get id () { return 'child_process' }
+  static get prefix () { return 'tracing:datadog:child_process:execution' }
+
+  get tracer () {
+    return this._tracer
+  }
+
+  start ({ command, shell }) {
+    if (typeof command !== 'string') {
+      return
+    }
+
+    const cmdFields = scrubChildProcessCmd(command)
+    const truncated = truncateCommand(cmdFields)
+    const property = (shell === true) ? 'cmd.shell' : 'cmd.exec'
+
+    const meta = {
+      'component': 'subprocess',
+      [property]: (shell === true) ? cmdFields.join(' ') : JSON.stringify(cmdFields)
+    }
+
+    if (truncated) {
+      meta['cmd.truncated'] = `${truncated}`
+    }
+
+    this.startSpan('command_execution', {
+      service: this.config.service,
+      resource: (shell === true) ? 'sh' : cmdFields[0],
+      type: 'system',
+      meta
+    })
+  }
+
+  end ({ result, error }) {
+    let exitCode
+
+    if (result !== undefined) {
+      exitCode = result?.status || 0
+    } else if (error !== undefined) {
+      exitCode = error?.status || error?.code || 0
+    } else {
+      // TracingChannels call start, end synchronously. Later when the promise is resolved then asyncStart asyncEnd.
+      // Therefore in the case of calling end with neither result nor error means that they will come in the asyncEnd.
+      return
+    }
+
+    this.activeSpan?.setTag('cmd.exit_code', `${exitCode}`)
+    this.activeSpan?.finish()
+  }
+
+  error (error) {
+    this.addError(error)
+  }
+
+  asyncEnd ({ result }) {
+    this.activeSpan?.setTag('cmd.exit_code', `${result}`)
+    this.activeSpan?.finish()
+  }
+}
+
+module.exports = ChildProcessPlugin

package/packages/datadog-plugin-child_process/src/scrub-cmd-params.js

@@ -0,0 +1,125 @@
+'use strict'
+
+const shellParser = require('shell-quote/parse')
+
+const ALLOWED_ENV_VARIABLES = ['LD_PRELOAD', 'LD_LIBRARY_PATH', 'PATH']
+const PROCESS_DENYLIST = ['md5']
+
+const VARNAMES_REGEX = /\$([\w\d_]*)(?:[^\w\d_]|$)/gmi
+// eslint-disable-next-line max-len
+const PARAM_PATTERN = '^-{0,2}(?:p(?:ass(?:w(?:or)?d)?)?|api_?key|secret|a(?:ccess|uth)_token|mysql_pwd|credentials|(?:stripe)?token)$'
+const regexParam = new RegExp(PARAM_PATTERN, 'i')
+const ENV_PATTERN = '^(\\w+=\\w+;)*\\w+=\\w+;?$'
+const envvarRegex = new RegExp(ENV_PATTERN)
+const REDACTED = '?'
+
+function extractVarNames (expression) {
+  const varNames = new Set()
+  let match
+
+  while ((match = VARNAMES_REGEX.exec(expression))) {
+    varNames.add(match[1])
+  }
+
+  const varNamesObject = {}
+  for (const varName of varNames.keys()) {
+    varNamesObject[varName] = `$${varName}`
+  }
+  return varNamesObject
+}
+
+function getTokensByExpression (expressionTokens) {
+  const expressionListTokens = []
+  let wipExpressionTokens = []
+  let isNewExpression = true
+
+  expressionTokens.forEach(token => {
+    if (isNewExpression) {
+      expressionListTokens.push(wipExpressionTokens)
+      isNewExpression = false
+    }
+
+    wipExpressionTokens.push(token)
+
+    if (token.op) {
+      wipExpressionTokens = []
+      isNewExpression = true
+    }
+  })
+  return expressionListTokens
+}
+
+function scrubChildProcessCmd (expression) {
+  const varNames = extractVarNames(expression)
+  const expressionTokens = shellParser(expression, varNames)
+
+  const expressionListTokens = getTokensByExpression(expressionTokens)
+
+  const result = []
+  expressionListTokens.forEach((expressionTokens) => {
+    let foundBinary = false
+    for (let index = 0; index < expressionTokens.length; index++) {
+      const token = expressionTokens[index]
+
+      if (typeof token === 'object') {
+        if (token.pattern) {
+          result.push(token.pattern)
+        } else if (token.op) {
+          result.push(token.op)
+        } else if (token.comment) {
+          result.push(`#${token.comment}`)
+        }
+      } else if (!foundBinary) {
+        if (envvarRegex.test(token)) {
+          const envSplit = token.split('=')
+
+          if (!ALLOWED_ENV_VARIABLES.includes(envSplit[0])) {
+            envSplit[1] = REDACTED
+
+            const newToken = envSplit.join('=')
+            expressionTokens[index] = newToken
+
+            result.push(newToken)
+          } else {
+            result.push(token)
+          }
+        } else {
+          foundBinary = true
+          result.push(token)
+
+          if (PROCESS_DENYLIST.includes(token)) {
+            for (index++; index < expressionTokens.length; index++) {
+              const token = expressionTokens[index]
+
+              if (token.op) {
+                result.push(token.op)
+              } else {
+                expressionTokens[index] = REDACTED
+                result.push(REDACTED)
+              }
+            }
+            break
+          }
+        }
+      } else {
+        const paramKeyValue = token.split('=')
+        const paramKey = paramKeyValue[0]
+
+        if (regexParam.test(paramKey)) {
+          if (paramKeyValue.length === 1) {
+            expressionTokens[index + 1] = REDACTED
+            result.push(token)
+          } else {
+            result.push(`${paramKey}=${REDACTED}`)
+          }
+        } else {
+          result.push(token)
+        }
+      }
+    }
+  })
+
+  return result
+}
+
+module.exports = scrubChildProcessCmd

package/packages/datadog-plugin-cucumber/src/index.js

@@ -14,7 +14,8 @@ const {
   TEST_ITR_UNSKIPPABLE,
   TEST_ITR_FORCED_RUN,
   TEST_CODE_OWNERS,
-  ITR_CORRELATION_ID
+  ITR_CORRELATION_ID,
+  TEST_SOURCE_FILE
 } = require('../../dd-trace/src/plugins/util/test')
 const { RESOURCE_NAME } = require('../../../ext/tags')
 const { COMPONENT, ERROR_MESSAGE } = require('../../dd-trace/src/constants')
@@ -47,7 +48,7 @@ class CucumberPlugin extends CiPlugin {
       hasUnskippableSuites,
       hasForcedToRunSuites
     }) => {
-      const { isSuitesSkippingEnabled, isCodeCoverageEnabled } = this.itrConfig || {}
+      const { isSuitesSkippingEnabled, isCodeCoverageEnabled } = this.libraryConfig || {}
       addIntelligentTestRunnerSpanTags(
         this.testSessionSpan,
         this.testModuleSpan,
@@ -71,7 +72,7 @@ class CucumberPlugin extends CiPlugin {
       this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'session')
       finishAllTraceSpans(this.testSessionSpan)
 
-      this.itrConfig = null
+      this.libraryConfig = null
       this.tracer._exporter.flush()
     })
 
@@ -102,7 +103,7 @@ class CucumberPlugin extends CiPlugin {
         }
       })
       this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite')
-      if (this.itrConfig?.isCodeCoverageEnabled) {
+      if (this.libraryConfig?.isCodeCoverageEnabled) {
         this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_STARTED, 'suite', { library: 'istanbul' })
       }
     })
@@ -114,7 +115,7 @@ class CucumberPlugin extends CiPlugin {
     })
 
     this.addSub('ci:cucumber:test-suite:code-coverage', ({ coverageFiles, suiteFile }) => {
-      if (!this.itrConfig?.isCodeCoverageEnabled) {
+      if (!this.libraryConfig?.isCodeCoverageEnabled) {
         return
       }
       if (!coverageFiles.length) {
@@ -122,7 +123,7 @@ class CucumberPlugin extends CiPlugin {
       }
 
       const relativeCoverageFiles = [...coverageFiles, suiteFile]
-        .map(filename => getTestSuitePath(filename, this.sourceRoot))
+        .map(filename => getTestSuitePath(filename, this.repositoryRoot))
 
       this.telemetry.distribution(TELEMETRY_CODE_COVERAGE_NUM_FILES, {}, relativeCoverageFiles.length)
 
@@ -136,10 +137,11 @@ class CucumberPlugin extends CiPlugin {
       this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_FINISHED, 'suite', { library: 'istanbul' })
     })
 
-    this.addSub('ci:cucumber:test:start', ({ testName, fullTestSuite, testSourceLine }) => {
+    this.addSub('ci:cucumber:test:start', ({ testName, testFileAbsolutePath, testSourceLine }) => {
       const store = storage.getStore()
-      const testSuite = getTestSuitePath(fullTestSuite, this.sourceRoot)
-      const testSpan = this.startTestSpan(testName, testSuite, testSourceLine)
+      const testSuite = getTestSuitePath(testFileAbsolutePath, this.sourceRoot)
+      const testSourceFile = getTestSuitePath(testFileAbsolutePath, this.repositoryRoot)
+      const testSpan = this.startTestSpan(testName, testSuite, testSourceFile, testSourceLine)
 
       this.enter(testSpan, store)
     })
@@ -191,12 +193,15 @@ class CucumberPlugin extends CiPlugin {
     })
   }
 
-  startTestSpan (testName, testSuite, testSourceLine) {
+  startTestSpan (testName, testSuite, testSourceFile, testSourceLine) {
     return super.startTestSpan(
       testName,
       testSuite,
       this.testSuiteSpan,
-      { [TEST_SOURCE_START]: testSourceLine }
+      {
+        [TEST_SOURCE_START]: testSourceLine,
+        [TEST_SOURCE_FILE]: testSourceFile
+      }
     )
   }
 }