dd-trace 4.18.0 → 5.6.0

package/packages/datadog-plugin-grpc/src/util.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const pick = require('lodash.pick')
+const pick = require('../../datadog-core/src/utils/src/pick')
 const log = require('../../dd-trace/src/log')
 
 module.exports = {

package/packages/datadog-plugin-http/src/client.js

@@ -58,7 +58,7 @@ class HttpClientPlugin extends ClientPlugin {
       span._spanContext._trace.record = false
     }
 
-    if (!(hasAmazonSignature(options) || !this.config.propagationFilter(uri))) {
+    if (this.shouldInjectTraceHeaders(options, uri)) {
       this.tracer.inject(span, HTTP_HEADERS, options.headers)
     }
 
@@ -71,6 +71,18 @@ class HttpClientPlugin extends ClientPlugin {
     return message.currentStore
   }
 
+  shouldInjectTraceHeaders (options, uri) {
+    if (hasAmazonSignature(options) && !this.config.enablePropagationWithAmazonHeaders) {
+      return false
+    }
+
+    if (!this.config.propagationFilter(uri)) {
+      return false
+    }
+
+    return true
+  }
+
   bindAsyncStart ({ parentStore }) {
     return parentStore
   }
@@ -98,7 +110,7 @@ class HttpClientPlugin extends ClientPlugin {
     span.finish()
   }
 
-  error ({ span, error }) {
+  error ({ span, error, args, customRequestTimeout }) {
     if (!span) return
     if (error) {
       span.addTags({
@@ -107,6 +119,11 @@ class HttpClientPlugin extends ClientPlugin {
         [ERROR_STACK]: error.stack
       })
     } else {
+      // conditions for no error:
+      // 1. not using a custom agent instance with custom timeout specified
+      // 2. no invocation of `req.setTimeout`
+      if (!args.options.agent?.options?.timeout && !customRequestTimeout) return
+
       span.setTag('error', 1)
     }
   }

package/packages/datadog-plugin-jest/src/index.js

@@ -12,10 +12,27 @@ const {
   TEST_FRAMEWORK_VERSION,
   TEST_SOURCE_START,
   TEST_ITR_UNSKIPPABLE,
-  TEST_ITR_FORCED_RUN
+  TEST_ITR_FORCED_RUN,
+  TEST_CODE_OWNERS,
+  ITR_CORRELATION_ID,
+  TEST_SOURCE_FILE,
+  TEST_IS_NEW,
+  TEST_EARLY_FLAKE_IS_RETRY,
+  TEST_EARLY_FLAKE_IS_ENABLED,
+  JEST_DISPLAY_NAME
 } = require('../../dd-trace/src/plugins/util/test')
 const { COMPONENT } = require('../../dd-trace/src/constants')
 const id = require('../../dd-trace/src/id')
+const {
+  TELEMETRY_EVENT_CREATED,
+  TELEMETRY_EVENT_FINISHED,
+  TELEMETRY_CODE_COVERAGE_STARTED,
+  TELEMETRY_CODE_COVERAGE_FINISHED,
+  TELEMETRY_ITR_FORCED_TO_RUN,
+  TELEMETRY_CODE_COVERAGE_EMPTY,
+  TELEMETRY_ITR_UNSKIPPABLE,
+  TELEMETRY_CODE_COVERAGE_NUM_FILES
+} = require('../../dd-trace/src/ci-visibility/telemetry')
 
 const isJestWorker = !!process.env.JEST_WORKER_ID
 
@@ -27,6 +44,20 @@ class JestPlugin extends CiPlugin {
     return 'jest'
   }
 
+  // The lists are the same for every test suite, so we can cache them
+  getUnskippableSuites (unskippableSuitesList) {
+    if (!this.unskippableSuites) {
+      this.unskippableSuites = JSON.parse(unskippableSuitesList)
+    }
+    return this.unskippableSuites
+  }
+  getForcedToRunSuites (forcedToRunSuitesList) {
+    if (!this.forcedToRunSuites) {
+      this.forcedToRunSuites = JSON.parse(forcedToRunSuitesList)
+    }
+    return this.forcedToRunSuites
+  }
+
   constructor (...args) {
     super(...args)
 
@@ -55,7 +86,9 @@ class JestPlugin extends CiPlugin {
       numSkippedSuites,
       hasUnskippableSuites,
       hasForcedToRunSuites,
-      error
+      error,
+      isEarlyFlakeDetectionEnabled,
+      onDone
     }) => {
       this.testSessionSpan.setTag(TEST_STATUS, status)
       this.testModuleSpan.setTag(TEST_STATUS, status)
@@ -80,30 +113,47 @@ class JestPlugin extends CiPlugin {
         }
       )
 
+      if (isEarlyFlakeDetectionEnabled) {
+        this.testSessionSpan.setTag(TEST_EARLY_FLAKE_IS_ENABLED, 'true')
+      }
+
       this.testModuleSpan.finish()
+      this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module')
       this.testSessionSpan.finish()
+      this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'session')
       finishAllTraceSpans(this.testSessionSpan)
-      this.tracer._exporter.flush()
+
+      this.tracer._exporter.flush(() => {
+        if (onDone) {
+          onDone()
+        }
+      })
     })
 
     // Test suites can be run in a different process from jest's main one.
     // This subscriber changes the configuration objects from jest to inject the trace id
-    // of the test session to the processes that run the test suites.
+    // of the test session to the processes that run the test suites, and other data.
     this.addSub('ci:jest:session:configuration', configs => {
       configs.forEach(config => {
         config._ddTestSessionId = this.testSessionSpan.context().toTraceId()
         config._ddTestModuleId = this.testModuleSpan.context().toSpanId()
         config._ddTestCommand = this.testSessionSpan.context()._tags[TEST_COMMAND]
+        config._ddItrCorrelationId = this.itrCorrelationId
+        config._ddIsEarlyFlakeDetectionEnabled = !!this.libraryConfig?.isEarlyFlakeDetectionEnabled
+        config._ddEarlyFlakeDetectionNumRetries = this.libraryConfig?.earlyFlakeDetectionNumRetries ?? 0
+        config._ddRepositoryRoot = this.repositoryRoot
       })
     })
 
-    this.addSub('ci:jest:test-suite:start', ({ testSuite, testEnvironmentOptions, frameworkVersion }) => {
+    this.addSub('ci:jest:test-suite:start', ({ testSuite, testEnvironmentOptions, frameworkVersion, displayName }) => {
       const {
         _ddTestSessionId: testSessionId,
         _ddTestCommand: testCommand,
         _ddTestModuleId: testModuleId,
+        _ddItrCorrelationId: itrCorrelationId,
         _ddForcedToRun,
-        _ddUnskippable
+        _ddUnskippable,
+        _ddTestCodeCoverageEnabled
       } = testEnvironmentOptions
 
       const testSessionSpanContext = this.tracer.extract('text_map', {
@@ -114,11 +164,25 @@ class JestPlugin extends CiPlugin {
       const testSuiteMetadata = getTestSuiteCommonTags(testCommand, frameworkVersion, testSuite, 'jest')
 
       if (_ddUnskippable) {
-        testSuiteMetadata[TEST_ITR_UNSKIPPABLE] = 'true'
+        const unskippableSuites = this.getUnskippableSuites(_ddUnskippable)
+        if (unskippableSuites[testSuite]) {
+          this.telemetry.count(TELEMETRY_ITR_UNSKIPPABLE, { testLevel: 'suite' })
+          testSuiteMetadata[TEST_ITR_UNSKIPPABLE] = 'true'
+        }
         if (_ddForcedToRun) {
-          testSuiteMetadata[TEST_ITR_FORCED_RUN] = 'true'
+          const forcedToRunSuites = this.getForcedToRunSuites(_ddForcedToRun)
+          if (forcedToRunSuites[testSuite]) {
+            this.telemetry.count(TELEMETRY_ITR_FORCED_TO_RUN, { testLevel: 'suite' })
+            testSuiteMetadata[TEST_ITR_FORCED_RUN] = 'true'
+          }
         }
       }
+      if (itrCorrelationId) {
+        testSuiteMetadata[ITR_CORRELATION_ID] = itrCorrelationId
+      }
+      if (displayName) {
+        testSuiteMetadata[JEST_DISPLAY_NAME] = displayName
+      }
 
       this.testSuiteSpan = this.tracer.startSpan('jest.test_suite', {
         childOf: testSessionSpanContext,
@@ -128,6 +192,10 @@ class JestPlugin extends CiPlugin {
           ...testSuiteMetadata
         }
       })
+      this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite')
+      if (_ddTestCodeCoverageEnabled) {
+        this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_STARTED, 'suite', { library: 'istanbul' })
+      }
     })
 
     this.addSub('ci:jest:worker-report:trace', traces => {
@@ -164,6 +232,7 @@ class JestPlugin extends CiPlugin {
         this.testSuiteSpan.setTag('error', new Error(errorMessage))
       }
       this.testSuiteSpan.finish()
+      this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite')
       // Suites potentially run in a different process than the session,
       // so calling finishAllTraceSpans on the session span is not enough
       finishAllTraceSpans(this.testSuiteSpan)
@@ -176,18 +245,26 @@ class JestPlugin extends CiPlugin {
     })
 
     /**
-     * This can't use `this.itrConfig` like `ci:mocha:test-suite:code-coverage`
+     * This can't use `this.libraryConfig` like `ci:mocha:test-suite:code-coverage`
      * because this subscription happens in a different process from the one
      * fetching the ITR config.
      */
-    this.addSub('ci:jest:test-suite:code-coverage', (coverageFiles) => {
+    this.addSub('ci:jest:test-suite:code-coverage', ({ coverageFiles, testSuite }) => {
+      if (!coverageFiles.length) {
+        this.telemetry.count(TELEMETRY_CODE_COVERAGE_EMPTY)
+      }
+      const files = [...coverageFiles, testSuite]
+
       const { _traceId, _spanId } = this.testSuiteSpan.context()
       const formattedCoverage = {
         sessionId: _traceId,
         suiteId: _spanId,
-        files: coverageFiles
+        files
       }
+
       this.tracer._exporter.exportCoverage(formattedCoverage)
+      this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_FINISHED, 'suite', { library: 'istanbul' })
+      this.telemetry.distribution(TELEMETRY_CODE_COVERAGE_NUM_FILES, {}, files.length)
     })
 
     this.addSub('ci:jest:test:start', (test) => {
@@ -204,6 +281,11 @@ class JestPlugin extends CiPlugin {
         span.setTag(TEST_SOURCE_START, testStartLine)
       }
       span.finish()
+      this.telemetry.ciVisEvent(
+        TELEMETRY_EVENT_FINISHED,
+        'test',
+        { hasCodeOwners: !!span.context()._tags[TEST_CODE_OWNERS] }
+      )
       finishAllTraceSpans(span)
     })
 
@@ -226,7 +308,18 @@ class JestPlugin extends CiPlugin {
   }
 
   startTestSpan (test) {
-    const { suite, name, runner, testParameters, frameworkVersion, testStartLine } = test
+    const {
+      suite,
+      name,
+      runner,
+      displayName,
+      testParameters,
+      frameworkVersion,
+      testStartLine,
+      testSourceFile,
+      isNew,
+      isEfdRetry
+    } = test
 
     const extraTags = {
       [JEST_TEST_RUNNER]: runner,
@@ -236,6 +329,19 @@ class JestPlugin extends CiPlugin {
     if (testStartLine) {
       extraTags[TEST_SOURCE_START] = testStartLine
     }
+    // If for whatever we don't have the source file, we'll fall back to the suite name
+    extraTags[TEST_SOURCE_FILE] = testSourceFile || suite
+
+    if (displayName) {
+      extraTags[JEST_DISPLAY_NAME] = displayName
+    }
+
+    if (isNew) {
+      extraTags[TEST_IS_NEW] = 'true'
+      if (isEfdRetry) {
+        extraTags[TEST_EARLY_FLAKE_IS_RETRY] = 'true'
+      }
+    }
 
     return super.startTestSpan(name, suite, this.testSuiteSpan, extraTags)
   }

package/packages/datadog-plugin-jest/src/util.js

@@ -77,30 +77,52 @@ function isMarkedAsUnskippable (test) {
 }
 
 function getJestSuitesToRun (skippableSuites, originalTests, rootDir) {
-  return originalTests.reduce((acc, test) => {
+  const unskippableSuites = {}
+  const forcedToRunSuites = {}
+
+  const skippedSuites = []
+  const suitesToRun = []
+
+  for (const test of originalTests) {
     const relativePath = getTestSuitePath(test.path, rootDir)
     const shouldBeSkipped = skippableSuites.includes(relativePath)
-
     if (isMarkedAsUnskippable(test)) {
-      acc.suitesToRun.push(test)
-      if (test?.context?.config?.testEnvironmentOptions) {
-        test.context.config.testEnvironmentOptions['_ddUnskippable'] = true
-        acc.hasUnskippableSuites = true
-        if (shouldBeSkipped) {
-          test.context.config.testEnvironmentOptions['_ddForcedToRun'] = true
-          acc.hasForcedToRunSuites = true
-        }
+      suitesToRun.push(test)
+      unskippableSuites[relativePath] = true
+      if (shouldBeSkipped) {
+        forcedToRunSuites[relativePath] = true
       }
-      return acc
+      continue
     }
-
     if (shouldBeSkipped) {
-      acc.skippedSuites.push(relativePath)
+      skippedSuites.push(relativePath)
     } else {
-      acc.suitesToRun.push(test)
+      suitesToRun.push(test)
     }
-    return acc
-  }, { skippedSuites: [], suitesToRun: [], hasUnskippableSuites: false, hasForcedToRunSuites: false })
+  }
+
+  const hasUnskippableSuites = Object.keys(unskippableSuites).length > 0
+  const hasForcedToRunSuites = Object.keys(forcedToRunSuites).length > 0
+
+  if (originalTests.length) {
+    // The config object is shared by all tests, so we can just take the first one
+    const [test] = originalTests
+    if (test?.context?.config?.testEnvironmentOptions) {
+      if (hasUnskippableSuites) {
+        test.context.config.testEnvironmentOptions._ddUnskippable = JSON.stringify(unskippableSuites)
+      }
+      if (hasForcedToRunSuites) {
+        test.context.config.testEnvironmentOptions._ddForcedToRun = JSON.stringify(forcedToRunSuites)
+      }
+    }
+  }
+
+  return {
+    skippedSuites,
+    suitesToRun,
+    hasUnskippableSuites,
+    hasForcedToRunSuites
+  }
 }
 
 module.exports = { getFormattedJestTestParameters, getJestTestName, getJestSuitesToRun, isMarkedAsUnskippable }

package/packages/datadog-plugin-kafkajs/src/consumer.js

@@ -1,19 +1,71 @@
 'use strict'
 
+const dc = require('dc-polyfill')
+const { getMessageSize } = require('../../dd-trace/src/datastreams/processor')
+const { DsmPathwayCodec } = require('../../dd-trace/src/datastreams/pathway')
 const ConsumerPlugin = require('../../dd-trace/src/plugins/consumer')
 
+const afterStartCh = dc.channel('dd-trace:kafkajs:consumer:afterStart')
+const beforeFinishCh = dc.channel('dd-trace:kafkajs:consumer:beforeFinish')
+
 class KafkajsConsumerPlugin extends ConsumerPlugin {
   static get id () { return 'kafkajs' }
   static get operation () { return 'consume' }
 
-  start ({ topic, partition, message, groupId }) {
-    if (this.config.dsmEnabled) {
-      this.tracer.decodeDataStreamsContext(message.headers['dd-pathway-ctx'])
-      this.tracer
-        .setCheckpoint(['direction:in', `group:${groupId}`, `topic:${topic}`, 'type:kafka'])
+  constructor () {
+    super(...arguments)
+    this.addSub('apm:kafkajs:consume:commit', message => this.commit(message))
+  }
+
+  /**
+   * Transform individual commit details sent by kafkajs' event reporter
+   * into actionable backlog items for DSM
+   *
+   * @typedef {object} ConsumerBacklog
+   * @property {number} type
+   * @property {string} consumer_group
+   * @property {string} topic
+   * @property {number} partition
+   * @property {number} offset
+   *
+   * @typedef {object} CommitEventItem
+   * @property {string} groupId
+   * @property {string} topic
+   * @property {number} partition
+   * @property {import('kafkajs/utils/long').Long} offset
+   *
+   * @param {CommitEventItem} commit
+   * @returns {ConsumerBacklog}
+   */
+  transformCommit (commit) {
+    const { groupId, partition, offset, topic } = commit
+    return {
+      partition,
+      topic,
+      type: 'kafka_commit',
+      offset: Number(offset),
+      consumer_group: groupId
     }
+  }
+
+  commit (commitList) {
+    if (!this.config.dsmEnabled) return
+    const keys = [
+      'consumer_group',
+      'type',
+      'partition',
+      'offset',
+      'topic'
+    ]
+    for (const commit of commitList.map(this.transformCommit)) {
+      if (keys.some(key => !commit.hasOwnProperty(key))) continue
+      this.tracer.setOffset(commit)
+    }
+  }
+
+  start ({ topic, partition, message, groupId }) {
     const childOf = extract(this.tracer, message.headers)
-    this.startSpan({
+    const span = this.startSpan({
       childOf,
       resource: topic,
       type: 'worker',
@@ -26,6 +78,24 @@ class KafkajsConsumerPlugin extends ConsumerPlugin {
         'kafka.partition': partition
       }
     })
+    if (this.config.dsmEnabled && message?.headers && DsmPathwayCodec.contextExists(message.headers)) {
+      const payloadSize = getMessageSize(message)
+      this.tracer.decodeDataStreamsContext(message.headers)
+      this.tracer
+        .setCheckpoint(['direction:in', `group:${groupId}`, `topic:${topic}`, 'type:kafka'], span, payloadSize)
+    }
+
+    if (afterStartCh.hasSubscribers) {
+      afterStartCh.publish({ topic, partition, message, groupId })
+    }
+  }
+
+  finish () {
+    if (beforeFinishCh.hasSubscribers) {
+      beforeFinishCh.publish()
+    }
+
+    super.finish()
   }
 }
 

package/packages/datadog-plugin-kafkajs/src/producer.js

@@ -1,7 +1,9 @@
 'use strict'
 
 const ProducerPlugin = require('../../dd-trace/src/plugins/producer')
-const { encodePathwayContext } = require('../../dd-trace/src/datastreams/pathway')
+const { DsmPathwayCodec } = require('../../dd-trace/src/datastreams/pathway')
+const { getMessageSize } = require('../../dd-trace/src/datastreams/processor')
+
 const BOOTSTRAP_SERVERS_KEY = 'messaging.kafka.bootstrap.servers'
 
 class KafkajsProducerPlugin extends ProducerPlugin {
@@ -9,13 +11,62 @@ class KafkajsProducerPlugin extends ProducerPlugin {
   static get operation () { return 'produce' }
   static get peerServicePrecursors () { return [BOOTSTRAP_SERVERS_KEY] }
 
-  start ({ topic, messages, bootstrapServers }) {
-    let pathwayCtx
-    if (this.config.dsmEnabled) {
-      const dataStreamsContext = this.tracer
-        .setCheckpoint(['direction:out', `topic:${topic}`, 'type:kafka'])
-      pathwayCtx = encodePathwayContext(dataStreamsContext)
+  constructor () {
+    super(...arguments)
+    this.addSub('apm:kafkajs:produce:commit', message => this.commit(message))
+  }
+
+  /**
+   * Transform individual commit details sent by kafkajs' event reporter
+   * into actionable backlog items for DSM
+   *
+   * @typedef {object} ProducerBacklog
+   * @property {number} type
+   * @property {string} topic
+   * @property {number} partition
+   * @property {number} offset
+   *
+   * @typedef {object} ProducerResponseItem
+   * @property {string} topic
+   * @property {number} partition
+   * @property {import('kafkajs/utils/long').Long} [offset]
+   * @property {import('kafkajs/utils/long').Long} [baseOffset]
+   *
+   * @param {ProducerResponseItem} response
+   * @returns {ProducerBacklog}
+   */
+  transformProduceResponse (response) {
+    // In produce protocol >=v3, the offset key changes from `offset` to `baseOffset`
+    const { topicName: topic, partition, offset, baseOffset } = response
+    const offsetAsLong = offset || baseOffset
+    return {
+      type: 'kafka_produce',
+      partition,
+      offset: offsetAsLong ? Number(offsetAsLong) : undefined,
+      topic
+    }
+  }
+
+  /**
+   *
+   * @param {ProducerResponseItem[]} commitList
+   * @returns {void}
+   */
+  commit (commitList) {
+    if (!this.config.dsmEnabled) return
+    const keys = [
+      'type',
+      'partition',
+      'offset',
+      'topic'
+    ]
+    for (const commit of commitList.map(this.transformProduceResponse)) {
+      if (keys.some(key => !commit.hasOwnProperty(key))) continue
+      this.tracer.setOffset(commit)
     }
+  }
+
+  start ({ topic, messages, bootstrapServers }) {
     const span = this.startSpan({
       resource: topic,
       meta: {
@@ -31,8 +82,13 @@ class KafkajsProducerPlugin extends ProducerPlugin {
     }
     for (const message of messages) {
       if (typeof message === 'object') {
-        if (this.config.dsmEnabled) message.headers['dd-pathway-ctx'] = pathwayCtx
         this.tracer.inject(span, 'text_map', message.headers)
+        if (this.config.dsmEnabled) {
+          const payloadSize = getMessageSize(message)
+          const dataStreamsContext = this.tracer
+            .setCheckpoint(['direction:out', `topic:${topic}`, 'type:kafka'], span, payloadSize)
+          DsmPathwayCodec.encode(dataStreamsContext, message.headers)
+        }
       }
     }
   }