dd-trace 4.18.0 → 5.6.0

package/packages/datadog-plugin-mocha/src/index.js

@@ -13,9 +13,26 @@ const {
   addIntelligentTestRunnerSpanTags,
   TEST_SOURCE_START,
   TEST_ITR_UNSKIPPABLE,
-  TEST_ITR_FORCED_RUN
+  TEST_ITR_FORCED_RUN,
+  TEST_CODE_OWNERS,
+  ITR_CORRELATION_ID,
+  TEST_SOURCE_FILE,
+  removeEfdStringFromTestName,
+  TEST_IS_NEW,
+  TEST_EARLY_FLAKE_IS_RETRY,
+  TEST_EARLY_FLAKE_IS_ENABLED
 } = require('../../dd-trace/src/plugins/util/test')
 const { COMPONENT } = require('../../dd-trace/src/constants')
+const {
+  TELEMETRY_EVENT_CREATED,
+  TELEMETRY_EVENT_FINISHED,
+  TELEMETRY_CODE_COVERAGE_STARTED,
+  TELEMETRY_CODE_COVERAGE_FINISHED,
+  TELEMETRY_ITR_FORCED_TO_RUN,
+  TELEMETRY_CODE_COVERAGE_EMPTY,
+  TELEMETRY_ITR_UNSKIPPABLE,
+  TELEMETRY_CODE_COVERAGE_NUM_FILES
+} = require('../../dd-trace/src/ci-visibility/telemetry')
 
 class MochaPlugin extends CiPlugin {
   static get id () {
@@ -26,15 +43,19 @@ class MochaPlugin extends CiPlugin {
     super(...args)
 
     this._testSuites = new Map()
-    this._testNameToParams = {}
+    this._testTitleToParams = {}
     this.sourceRoot = process.cwd()
 
     this.addSub('ci:mocha:test-suite:code-coverage', ({ coverageFiles, suiteFile }) => {
-      if (!this.itrConfig || !this.itrConfig.isCodeCoverageEnabled) {
+      if (!this.libraryConfig?.isCodeCoverageEnabled) {
         return
       }
       const testSuiteSpan = this._testSuites.get(suiteFile)
 
+      if (!coverageFiles.length) {
+        this.telemetry.count(TELEMETRY_CODE_COVERAGE_EMPTY)
+      }
+
       const relativeCoverageFiles = [...coverageFiles, suiteFile]
         .map(filename => getTestSuitePath(filename, this.sourceRoot))
 
@@ -47,9 +68,16 @@ class MochaPlugin extends CiPlugin {
       }
 
       this.tracer._exporter.exportCoverage(formattedCoverage)
+      this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_FINISHED, 'suite', { library: 'istanbul' })
+      this.telemetry.distribution(TELEMETRY_CODE_COVERAGE_NUM_FILES, {}, relativeCoverageFiles.length)
     })
 
-    this.addSub('ci:mocha:test-suite:start', ({ testSuite, isUnskippable, isForcedToRun }) => {
+    this.addSub('ci:mocha:test-suite:start', ({
+      testSuite,
+      isUnskippable,
+      isForcedToRun,
+      itrCorrelationId
+    }) => {
       const store = storage.getStore()
       const testSuiteMetadata = getTestSuiteCommonTags(
         this.command,
@@ -59,9 +87,11 @@ class MochaPlugin extends CiPlugin {
       )
       if (isUnskippable) {
         testSuiteMetadata[TEST_ITR_UNSKIPPABLE] = 'true'
+        this.telemetry.count(TELEMETRY_ITR_UNSKIPPABLE, { testLevel: 'suite' })
       }
       if (isForcedToRun) {
         testSuiteMetadata[TEST_ITR_FORCED_RUN] = 'true'
+        this.telemetry.count(TELEMETRY_ITR_FORCED_TO_RUN, { testLevel: 'suite' })
       }
 
       const testSuiteSpan = this.tracer.startSpan('mocha.test_suite', {
@@ -72,6 +102,13 @@ class MochaPlugin extends CiPlugin {
           ...testSuiteMetadata
         }
       })
+      this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite')
+      if (this.libraryConfig?.isCodeCoverageEnabled) {
+        this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_STARTED, 'suite', { library: 'istanbul' })
+      }
+      if (itrCorrelationId) {
+        testSuiteSpan.setTag(ITR_CORRELATION_ID, itrCorrelationId)
+      }
       this.enter(testSuiteSpan, store)
       this._testSuites.set(testSuite, testSuiteSpan)
     })
@@ -85,6 +122,7 @@ class MochaPlugin extends CiPlugin {
           span.setTag(TEST_STATUS, status)
         }
         span.finish()
+        this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite')
       }
     })
 
@@ -97,9 +135,9 @@ class MochaPlugin extends CiPlugin {
       }
     })
 
-    this.addSub('ci:mocha:test:start', ({ test, testStartLine }) => {
+    this.addSub('ci:mocha:test:start', (testInfo) => {
       const store = storage.getStore()
-      const span = this.startTestSpan(test, testStartLine)
+      const span = this.startTestSpan(testInfo)
 
       this.enter(span, store)
     })
@@ -113,16 +151,21 @@ class MochaPlugin extends CiPlugin {
         span.setTag(TEST_STATUS, status)
 
         span.finish()
+        this.telemetry.ciVisEvent(
+          TELEMETRY_EVENT_FINISHED,
+          'test',
+          { hasCodeOwners: !!span.context()._tags[TEST_CODE_OWNERS] }
+        )
         finishAllTraceSpans(span)
       }
     })
 
-    this.addSub('ci:mocha:test:skip', (test) => {
+    this.addSub('ci:mocha:test:skip', (testInfo) => {
       const store = storage.getStore()
       // skipped through it.skip, so the span is not created yet
       // for this test
       if (!store) {
-        const testSpan = this.startTestSpan(test)
+        const testSpan = this.startTestSpan(testInfo)
         this.enter(testSpan, store)
       }
     })
@@ -140,8 +183,8 @@ class MochaPlugin extends CiPlugin {
       }
     })
 
-    this.addSub('ci:mocha:test:parameterize', ({ name, params }) => {
-      this._testNameToParams[name] = params
+    this.addSub('ci:mocha:test:parameterize', ({ title, params }) => {
+      this._testTitleToParams[title] = params
     })
 
     this.addSub('ci:mocha:session:finish', ({
@@ -151,10 +194,11 @@ class MochaPlugin extends CiPlugin {
       numSkippedSuites,
       hasForcedToRunSuites,
       hasUnskippableSuites,
-      error
+      error,
+      isEarlyFlakeDetectionEnabled
     }) => {
       if (this.testSessionSpan) {
-        const { isSuitesSkippingEnabled, isCodeCoverageEnabled } = this.itrConfig || {}
+        const { isSuitesSkippingEnabled, isCodeCoverageEnabled } = this.libraryConfig || {}
         this.testSessionSpan.setTag(TEST_STATUS, status)
         this.testModuleSpan.setTag(TEST_STATUS, status)
 
@@ -178,21 +222,34 @@ class MochaPlugin extends CiPlugin {
           }
         )
 
+        if (isEarlyFlakeDetectionEnabled) {
+          this.testSessionSpan.setTag(TEST_EARLY_FLAKE_IS_ENABLED, 'true')
+        }
+
         this.testModuleSpan.finish()
+        this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module')
         this.testSessionSpan.finish()
+        this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'session')
         finishAllTraceSpans(this.testSessionSpan)
       }
-      this.itrConfig = null
+      this.libraryConfig = null
       this.tracer._exporter.flush()
     })
   }
 
-  startTestSpan (test, testStartLine) {
-    const testName = test.fullTitle()
-    const { file: testSuiteAbsolutePath, title } = test
+  startTestSpan (testInfo) {
+    const {
+      testSuiteAbsolutePath,
+      title,
+      isNew,
+      isEfdRetry,
+      testStartLine
+    } = testInfo
+
+    const testName = removeEfdStringFromTestName(testInfo.testName)
 
     const extraTags = {}
-    const testParametersString = getTestParametersString(this._testNameToParams, title)
+    const testParametersString = getTestParametersString(this._testTitleToParams, title)
     if (testParametersString) {
       extraTags[TEST_PARAMETERS] = testParametersString
     }
@@ -204,6 +261,19 @@ class MochaPlugin extends CiPlugin {
     const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.sourceRoot)
     const testSuiteSpan = this._testSuites.get(testSuiteAbsolutePath)
 
+    if (this.repositoryRoot !== this.sourceRoot && !!this.repositoryRoot) {
+      extraTags[TEST_SOURCE_FILE] = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot)
+    } else {
+      extraTags[TEST_SOURCE_FILE] = testSuite
+    }
+
+    if (isNew) {
+      extraTags[TEST_IS_NEW] = 'true'
+      if (isEfdRetry) {
+        extraTags[TEST_EARLY_FLAKE_IS_RETRY] = 'true'
+      }
+    }
+
     return super.startTestSpan(testName, testSuite, testSuiteSpan, extraTags)
   }
 }

package/packages/datadog-plugin-next/src/index.js

@@ -6,6 +6,8 @@ const analyticsSampler = require('../../dd-trace/src/analytics_sampler')
 const { COMPONENT } = require('../../dd-trace/src/constants')
 const web = require('../../dd-trace/src/plugins/util/web')
 
+const errorPages = ['/404', '/500', '/_error', '/_not-found']
+
 class NextPlugin extends ServerPlugin {
   static get id () {
     return 'next'
@@ -40,6 +42,13 @@ class NextPlugin extends ServerPlugin {
   }
 
   error ({ span, error }) {
+    if (!span) {
+      const store = storage.getStore()
+      if (!store) return
+
+      span = store.span
+    }
+
     this.addError(error, span)
   }
 
@@ -50,10 +59,20 @@ class NextPlugin extends ServerPlugin {
 
     const span = store.span
     const error = span.context()._tags['error']
-
-    if (!this.config.validateStatus(res.statusCode) && !error) {
-      span.setTag('error', req.error || nextRequest.error || true)
-      web.addError(req, req.error || nextRequest.error || true)
+    const requestError = req.error || nextRequest.error
+
+    if (requestError) {
+      // prioritize user-set errors from API routes
+      span.setTag('error', requestError)
+      web.addError(req, requestError)
+    } else if (error) {
+      // general error handling
+      span.setTag('error', error)
+      web.addError(req, requestError || error)
+    } else if (!this.config.validateStatus(res.statusCode)) {
+      // where there's no error, we still need to validate status
+      span.setTag('error', true)
+      web.addError(req, true)
     }
 
     span.addTags({
@@ -65,7 +84,7 @@ class NextPlugin extends ServerPlugin {
     span.finish()
   }
 
-  pageLoad ({ page, isAppPath = false }) {
+  pageLoad ({ page, isAppPath = false, isStatic = false }) {
     const store = storage.getStore()
 
     if (!store) return
@@ -73,21 +92,28 @@ class NextPlugin extends ServerPlugin {
     const span = store.span
     const req = this._requests.get(span)
 
+    // safeguard against missing req in complicated timeout scenarios
+    if (!req) return
+
     // Only use error page names if there's not already a name
     const current = span.context()._tags['next.page']
-    if (current && ['/404', '/500', '/_error', '/_not-found'].includes(page)) {
+    const isErrorPage = errorPages.includes(page)
+
+    if (current && isErrorPage) {
       return
     }
 
     // remove ending /route or /page for appDir projects
-    if (isAppPath) page = page.substring(0, page.lastIndexOf('/'))
-
-    // This is for static files whose 'page' includes the whole file path
-    // For normal page matches, like /api/hello/[name] and a req.url like /api/hello/world,
-    // nothing should happen
-    // For page matches like /User/something/public/text.txt and req.url like /text.txt,
-    // it should disregard the extra absolute path Next.js sometimes sets
-    if (page.includes(req.url)) page = req.url
+    // need to check if not an error page too, as those are marked as app directory
+    // in newer versions
+    if (isAppPath && !isErrorPage) page = page.substring(0, page.lastIndexOf('/'))
+
+    // handle static resource
+    if (isStatic) {
+      page = req.url.includes('_next/static')
+        ? '/_next/static/*'
+        : '/public/*'
+    }
 
     span.addTags({
       [COMPONENT]: this.constructor.id,

package/packages/datadog-plugin-playwright/src/index.js

@@ -8,10 +8,18 @@ const {
   finishAllTraceSpans,
   getTestSuitePath,
   getTestSuiteCommonTags,
-  TEST_SOURCE_START
+  TEST_SOURCE_START,
+  TEST_CODE_OWNERS,
+  TEST_SOURCE_FILE,
+  TEST_CONFIGURATION_BROWSER_NAME
 } = require('../../dd-trace/src/plugins/util/test')
 const { RESOURCE_NAME } = require('../../../ext/tags')
 const { COMPONENT } = require('../../dd-trace/src/constants')
+const {
+  TELEMETRY_EVENT_CREATED,
+  TELEMETRY_EVENT_FINISHED
+} = require('../../dd-trace/src/ci-visibility/telemetry')
+const { appClosing: appClosingTelemetry } = require('../../dd-trace/src/telemetry')
 
 class PlaywrightPlugin extends CiPlugin {
   static get id () {
@@ -22,15 +30,31 @@ class PlaywrightPlugin extends CiPlugin {
     super(...args)
 
     this._testSuites = new Map()
+    this.numFailedTests = 0
+    this.numFailedSuites = 0
 
     this.addSub('ci:playwright:session:finish', ({ status, onDone }) => {
       this.testModuleSpan.setTag(TEST_STATUS, status)
       this.testSessionSpan.setTag(TEST_STATUS, status)
 
+      if (this.numFailedSuites > 0) {
+        let errorMessage = `Test suites failed: ${this.numFailedSuites}.`
+        if (this.numFailedTests > 0) {
+          errorMessage += ` Tests failed: ${this.numFailedTests}`
+        }
+        const error = new Error(errorMessage)
+        this.testModuleSpan.setTag('error', error)
+        this.testSessionSpan.setTag('error', error)
+      }
+
       this.testModuleSpan.finish()
+      this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module')
       this.testSessionSpan.finish()
+      this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'session')
       finishAllTraceSpans(this.testSessionSpan)
+      appClosingTelemetry()
       this.tracer._exporter.flush(onDone)
+      this.numFailedTests = 0
     })
 
     this.addSub('ci:playwright:test-suite:start', (testSuiteAbsolutePath) => {
@@ -52,23 +76,36 @@ class PlaywrightPlugin extends CiPlugin {
           ...testSuiteMetadata
         }
       })
+      this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite')
       this.enter(testSuiteSpan, store)
 
       this._testSuites.set(testSuite, testSuiteSpan)
     })
 
-    this.addSub('ci:playwright:test-suite:finish', (status) => {
+    this.addSub('ci:playwright:test-suite:finish', ({ status, error }) => {
       const store = storage.getStore()
       const span = store && store.span
       if (!span) return
-      span.setTag(TEST_STATUS, status)
+      if (error) {
+        span.setTag('error', error)
+        span.setTag(TEST_STATUS, 'fail')
+      } else {
+        span.setTag(TEST_STATUS, status)
+      }
+
+      if (status === 'fail' || error) {
+        this.numFailedSuites++
+      }
+
       span.finish()
+      this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite')
     })
 
-    this.addSub('ci:playwright:test:start', ({ testName, testSuiteAbsolutePath, testSourceLine }) => {
+    this.addSub('ci:playwright:test:start', ({ testName, testSuiteAbsolutePath, testSourceLine, browserName }) => {
       const store = storage.getStore()
       const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.rootDir)
-      const span = this.startTestSpan(testName, testSuite, testSourceLine)
+      const testSourceFile = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot)
+      const span = this.startTestSpan(testName, testSuite, testSourceFile, testSourceLine, browserName)
 
       this.enter(span, store)
     })
@@ -100,17 +137,43 @@ class PlaywrightPlugin extends CiPlugin {
         if (step.error) {
           stepSpan.setTag('error', step.error)
         }
-        stepSpan.finish(stepStartTime + step.duration)
+        let stepDuration = step.duration
+        if (stepDuration <= 0 || isNaN(stepDuration)) {
+          stepDuration = 0
+        }
+        stepSpan.finish(stepStartTime + stepDuration)
       })
 
       span.finish()
+
+      if (testStatus === 'fail') {
+        this.numFailedTests++
+      }
+
+      this.telemetry.ciVisEvent(
+        TELEMETRY_EVENT_FINISHED,
+        'test',
+        { hasCodeOwners: !!span.context()._tags[TEST_CODE_OWNERS] }
+      )
+
       finishAllTraceSpans(span)
     })
   }
 
-  startTestSpan (testName, testSuite, testSourceLine) {
+  startTestSpan (testName, testSuite, testSourceFile, testSourceLine, browserName) {
     const testSuiteSpan = this._testSuites.get(testSuite)
-    return super.startTestSpan(testName, testSuite, testSuiteSpan, { [TEST_SOURCE_START]: testSourceLine })
+
+    const extraTags = {
+      [TEST_SOURCE_START]: testSourceLine
+    }
+    if (testSourceFile) {
+      extraTags[TEST_SOURCE_FILE] = testSourceFile || testSuite
+    }
+    if (browserName) {
+      extraTags[TEST_CONFIGURATION_BROWSER_NAME] = browserName
+    }
+
+    return super.startTestSpan(testName, testSuite, testSuiteSpan, extraTags)
   }
 }
 

package/packages/datadog-plugin-rhea/src/consumer.js

@@ -2,6 +2,8 @@
 
 const ConsumerPlugin = require('../../dd-trace/src/plugins/consumer')
 const { storage } = require('../../datadog-core')
+const { getAmqpMessageSize } = require('../../dd-trace/src/datastreams/processor')
+const { DsmPathwayCodec } = require('../../dd-trace/src/datastreams/pathway')
 
 class RheaConsumerPlugin extends ConsumerPlugin {
   static get id () { return 'rhea' }
@@ -19,7 +21,7 @@ class RheaConsumerPlugin extends ConsumerPlugin {
     const name = getResourceNameFromMessage(msgObj)
     const childOf = extractTextMap(msgObj, this.tracer)
 
-    this.startSpan({
+    const span = this.startSpan({
       childOf,
       resource: name,
       type: 'worker',
@@ -29,6 +31,19 @@ class RheaConsumerPlugin extends ConsumerPlugin {
         'amqp.link.role': 'receiver'
       }
     })
+
+    if (
+      this.config.dsmEnabled &&
+      msgObj?.message?.delivery_annotations &&
+      DsmPathwayCodec.contextExists(msgObj.message.delivery_annotations)
+    ) {
+      const payloadSize = getAmqpMessageSize(
+        { headers: msgObj.message.delivery_annotations, content: msgObj.message.body }
+      )
+      this.tracer.decodeDataStreamsContext(msgObj.message.delivery_annotations)
+      this.tracer
+        .setCheckpoint(['direction:in', `topic:${name}`, 'type:rabbitmq'], span, payloadSize)
+    }
   }
 }
 

package/packages/datadog-plugin-rhea/src/producer.js

@@ -2,6 +2,8 @@
 
 const { CLIENT_PORT_KEY } = require('../../dd-trace/src/constants')
 const ProducerPlugin = require('../../dd-trace/src/plugins/producer')
+const { DsmPathwayCodec } = require('../../dd-trace/src/datastreams/pathway')
+const { getAmqpMessageSize } = require('../../dd-trace/src/datastreams/processor')
 
 class RheaProducerPlugin extends ProducerPlugin {
   static get id () { return 'rhea' }
@@ -36,6 +38,14 @@ function addDeliveryAnnotations (msg, tracer, span) {
     msg.delivery_annotations = msg.delivery_annotations || {}
 
     tracer.inject(span, 'text_map', msg.delivery_annotations)
+
+    if (tracer._config.dsmEnabled) {
+      const targetName = span.context()._tags['amqp.link.target.address']
+      const payloadSize = getAmqpMessageSize({ content: msg.body, headers: msg.delivery_annotations })
+      const dataStreamsContext = tracer
+        .setCheckpoint(['direction:out', `exchange:${targetName}`, 'type:rabbitmq'], span, payloadSize)
+      DsmPathwayCodec.encode(dataStreamsContext, msg.delivery_annotations)
+    }
   }
 }
 

package/packages/dd-trace/src/appsec/activation.js

@@ -0,0 +1,29 @@
+'use strict'
+
+const Activation = {
+  ONECLICK: 'OneClick',
+  ENABLED: 'Enabled',
+  DISABLED: 'Disabled',
+
+  fromConfig (config) {
+    switch (config.appsec.enabled) {
+      // ASM is activated by an env var DD_APPSEC_ENABLED=true
+      case true:
+        return Activation.ENABLED
+
+      // ASM is disabled by an env var DD_APPSEC_ENABLED=false
+      case false:
+        return Activation.DISABLED
+
+      // ASM is activated by one click remote config
+      case undefined:
+        return Activation.ONECLICK
+
+      // Any other value should never occur
+      default:
+        return Activation.DISABLED
+    }
+  }
+}
+
+module.exports = Activation

package/packages/dd-trace/src/appsec/addresses.js

@@ -13,8 +13,12 @@ module.exports = {
   HTTP_INCOMING_RESPONSE_HEADERS: 'server.response.headers.no_cookies',
   // TODO: 'server.response.trailers',
   HTTP_INCOMING_GRAPHQL_RESOLVERS: 'graphql.server.all_resolvers',
+  HTTP_INCOMING_GRAPHQL_RESOLVER: 'graphql.server.resolver',
+
+  HTTP_OUTGOING_BODY: 'server.response.body',
 
   HTTP_CLIENT_IP: 'http.client_ip',
 
-  USER_ID: 'usr.id'
+  USER_ID: 'usr.id',
+  WAF_CONTEXT_PROCESSOR: 'waf.context.processor'
 }

package/packages/dd-trace/src/appsec/api_security_sampler.js

@@ -0,0 +1,61 @@
+'use strict'
+
+const log = require('../log')
+
+let enabled
+let requestSampling
+
+const sampledRequests = new WeakSet()
+
+function configure ({ apiSecurity }) {
+  enabled = apiSecurity.enabled
+  setRequestSampling(apiSecurity.requestSampling)
+}
+
+function disable () {
+  enabled = false
+}
+
+function setRequestSampling (sampling) {
+  requestSampling = parseRequestSampling(sampling)
+}
+
+function parseRequestSampling (requestSampling) {
+  let parsed = parseFloat(requestSampling)
+
+  if (isNaN(parsed)) {
+    log.warn(`Incorrect API Security request sampling value: ${requestSampling}`)
+
+    parsed = 0
+  } else {
+    parsed = Math.min(1, Math.max(0, parsed))
+  }
+
+  return parsed
+}
+
+function sampleRequest (req) {
+  if (!enabled || !requestSampling) {
+    return false
+  }
+
+  const shouldSample = Math.random() <= requestSampling
+
+  if (shouldSample) {
+    sampledRequests.add(req)
+  }
+
+  return shouldSample
+}
+
+function isSampled (req) {
+  return sampledRequests.has(req)
+}
+
+module.exports = {
+  configure,
+  disable,
+  setRequestSampling,
+  sampleRequest,
+  isSampled
+}

package/packages/dd-trace/src/appsec/blocked_templates.js

@@ -5,7 +5,10 @@ const html = `<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta n
 
 const json = `{"errors":[{"title":"You've been blocked","detail":"Sorry, you cannot access this page. Please contact the customer service team. Security provided by Datadog."}]}`
 
+const graphqlJson = `{"errors":[{"message":"You've been blocked","extensions":{"detail":"Sorry, you cannot perform this operation. Please contact the customer service team. Security provided by Datadog."}}]}`
+
 module.exports = {
   html,
-  json
+  json,
+  graphqlJson
 }