dd-trace 5.9.0 → 5.10.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "5.9.0",
+  "version": "5.10.0",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",
@@ -69,7 +69,7 @@
     "node": ">=18"
   },
   "dependencies": {
-    "@datadog/native-appsec": "7.1.0",
+    "@datadog/native-appsec": "7.1.1",
     "@datadog/native-iast-rewriter": "2.3.0",
     "@datadog/native-iast-taint-tracking": "1.7.0",
     "@datadog/native-metrics": "^2.0.0",

package/packages/datadog-instrumentations/src/jest.js

@@ -44,11 +44,14 @@ const knownTestsCh = channel('ci:jest:known-tests')
 
 const itrSkippedSuitesCh = channel('ci:jest:itr:skipped-suites')
 
+// Message sent by jest's main process to workers to run a test suite (=test file)
+// https://github.com/jestjs/jest/blob/1d682f21c7a35da4d3ab3a1436a357b980ebd0fa/packages/jest-worker/src/types.ts#L37
+const CHILD_MESSAGE_CALL = 1
 // Maximum time we'll wait for the tracer to flush
 const FLUSH_TIMEOUT = 10000
 
 let skippableSuites = []
-let knownTests = []
+let knownTests = {}
 let isCodeCoverageEnabled = false
 let isSuitesSkippingEnabled = false
 let isUserCodeCoverageEnabled = false
@@ -73,6 +76,7 @@ const specStatusToTestStatus = {
 const asyncResources = new WeakMap()
 const originalTestFns = new WeakMap()
 const retriedTestsToNumAttempts = new Map()
+const newTestsTestStatuses = new Map()
 
 // based on https://github.com/facebook/jest/blob/main/packages/jest-circus/src/formatNodeAssertErrors.ts#L41
 function formatJestError (errors) {
@@ -101,6 +105,13 @@ function getTestEnvironmentOptions (config) {
   return {}
 }
 
+function getEfdStats (testStatuses) {
+  return testStatuses.reduce((acc, testStatus) => {
+    acc[testStatus]++
+    return acc
+  }, { pass: 0, fail: 0 })
+}
+
 function getWrappedEnvironment (BaseEnvironment, jestVersion) {
   return class DatadogEnvironment extends BaseEnvironment {
     constructor (config, context) {
@@ -123,9 +134,12 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
       this.isEarlyFlakeDetectionEnabled = this.testEnvironmentOptions._ddIsEarlyFlakeDetectionEnabled
 
       if (this.isEarlyFlakeDetectionEnabled) {
+        const hasKnownTests = !!knownTests.jest
         earlyFlakeDetectionNumRetries = this.testEnvironmentOptions._ddEarlyFlakeDetectionNumRetries
         try {
-          this.knownTestsForThisSuite = this.getKnownTestsForSuite(this.testEnvironmentOptions._ddKnownTests)
+          this.knownTestsForThisSuite = hasKnownTests
+            ? (knownTests.jest[this.testSuite] || [])
+            : this.getKnownTestsForSuite(this.testEnvironmentOptions._ddKnownTests)
         } catch (e) {
           // If there has been an error parsing the tests, we'll disable Early Flake Deteciton
           this.isEarlyFlakeDetectionEnabled = false
@@ -145,7 +159,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
       if (typeof knownTestsForSuite === 'string') {
         knownTestsForSuite = JSON.parse(knownTestsForSuite)
       }
-      return knownTestsForSuite.jest?.[this.testSuite] || []
+      return knownTestsForSuite
     }
 
     // Add the `add_test` event we don't have the test object yet, so
@@ -242,6 +256,19 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
           })
           // restore in case it is retried
           event.test.fn = originalTestFns.get(event.test)
+          // We'll store the test statuses of the retries
+          if (this.isEarlyFlakeDetectionEnabled) {
+            const testName = getJestTestName(event.test)
+            const originalTestName = removeEfdStringFromTestName(testName)
+            const isNewTest = retriedTestsToNumAttempts.has(originalTestName)
+            if (isNewTest) {
+              if (newTestsTestStatuses.has(originalTestName)) {
+                newTestsTestStatuses.get(originalTestName).push(status)
+              } else {
+                newTestsTestStatuses.set(originalTestName, [status])
+              }
+            }
+          }
         })
       }
       if (event.name === 'test_skip' || event.name === 'test_todo') {
@@ -508,6 +535,28 @@ function cliWrapper (cli, jestVersion) {
 
     numSkippedSuites = 0
 
+    /**
+     * If Early Flake Detection (EFD) is enabled the logic is as follows:
+     * - If all attempts for a test are failing, the test has failed and we will let the test process fail.
+     * - If just a single attempt passes, we will prevent the test process from failing.
+     * The rationale behind is the following: you may still be able to block your CI pipeline by gating
+     * on flakiness (the test will be considered flaky), but you may choose to unblock the pipeline too.
+     */
+
+    if (isEarlyFlakeDetectionEnabled) {
+      let numFailedTestsToIgnore = 0
+      for (const testStatuses of newTestsTestStatuses.values()) {
+        const { pass, fail } = getEfdStats(testStatuses)
+        if (pass > 0) { // as long as one passes, we'll consider the test passed
+          numFailedTestsToIgnore += fail
+        }
+      }
+      // If every test that failed was an EFD retry, we'll consider the suite passed
+      if (numFailedTestsToIgnore !== 0 && result.results.numFailedTests === numFailedTestsToIgnore) {
+        result.results.success = true
+      }
+    }
+
     return result
   })
 
@@ -619,7 +668,6 @@ function configureTestEnvironment (readConfigsResult) {
   // because `jestAdapterWrapper` runs in a different process. We have to go through `testEnvironmentOptions`
   configs.forEach(config => {
     config.testEnvironmentOptions._ddTestCodeCoverageEnabled = isCodeCoverageEnabled
-    config.testEnvironmentOptions._ddKnownTests = knownTests
   })
 
   isUserCodeCoverageEnabled = !!readConfigsResult.globalConfig.collectCoverage
@@ -795,6 +843,38 @@ addHook({
   file: 'build/workers/ChildProcessWorker.js'
 }, (childProcessWorker) => {
   const ChildProcessWorker = childProcessWorker.default
+  shimmer.wrap(ChildProcessWorker.prototype, 'send', send => function (request) {
+    if (!isEarlyFlakeDetectionEnabled) {
+      return send.apply(this, arguments)
+    }
+    const [type] = request
+    // eslint-disable-next-line
+    // https://github.com/jestjs/jest/blob/1d682f21c7a35da4d3ab3a1436a357b980ebd0fa/packages/jest-worker/src/workers/ChildProcessWorker.ts#L424
+    if (type === CHILD_MESSAGE_CALL) {
+      // This is the message that the main process sends to the worker to run a test suite (=test file).
+      // In here we modify the config.testEnvironmentOptions to include the known tests for the suite.
+      // This way the suite only knows about the tests that are part of it.
+      const args = request[request.length - 1]
+      if (args.length > 1) {
+        return send.apply(this, arguments)
+      }
+      if (!args[0]?.config) {
+        return send.apply(this, arguments)
+      }
+      const [{ globalConfig, config, path: testSuiteAbsolutePath }] = args
+      const testSuite = getTestSuitePath(testSuiteAbsolutePath, globalConfig.rootDir || process.cwd())
+      const suiteKnownTests = knownTests.jest?.[testSuite] || []
+      args[0].config = {
+        ...config,
+        testEnvironmentOptions: {
+          ...config.testEnvironmentOptions,
+          _ddKnownTests: suiteKnownTests
+        }
+      }
+    }
+
+    return send.apply(this, arguments)
+  })
   shimmer.wrap(ChildProcessWorker.prototype, '_onMessage', _onMessage => function () {
     const [code, data] = arguments[0]
     if (code === JEST_WORKER_TRACE_PAYLOAD_CODE) { // datadog trace payload

package/packages/datadog-instrumentations/src/kafkajs.js

@@ -68,7 +68,10 @@ addHook({ name: 'kafkajs', file: 'src/index.js', versions: ['>=1.4'] }, (BaseKaf
           const result = send.apply(this, arguments)
 
           result.then(
-            innerAsyncResource.bind(() => producerFinishCh.publish(undefined)),
+            innerAsyncResource.bind(res => {
+              producerFinishCh.publish(undefined)
+              producerCommitCh.publish(res)
+            }),
             innerAsyncResource.bind(err => {
               if (err) {
                 producerErrorCh.publish(err)
@@ -77,12 +80,6 @@ addHook({ name: 'kafkajs', file: 'src/index.js', versions: ['>=1.4'] }, (BaseKaf
             })
           )
 
-          result.then(res => {
-            if (producerCommitCh.hasSubscribers) {
-              producerCommitCh.publish(res)
-            }
-          })
-
           return result
         } catch (e) {
           producerErrorCh.publish(e)

package/packages/datadog-instrumentations/src/oracledb.js

@@ -21,7 +21,7 @@ function finish (err) {
   finishChannel.publish(undefined)
 }
 
-addHook({ name: 'oracledb', versions: ['5'] }, oracledb => {
+addHook({ name: 'oracledb', versions: ['>=5'] }, oracledb => {
   shimmer.wrap(oracledb.Connection.prototype, 'execute', execute => {
     return function wrappedExecute (dbQuery, ...args) {
       if (!startChannel.hasSubscribers) {

package/packages/dd-trace/src/appsec/iast/analyzers/weak-hash-analyzer.js

@@ -20,11 +20,15 @@ const EXCLUDED_LOCATIONS = getNodeModulesPaths(
   'pusher/lib/utils.js',
   'redlock/dist/cjs',
   'sqreen/lib/package-reader/index.js',
-  'ws/lib/websocket-server.js'
+  'ws/lib/websocket-server.js',
+  'google-gax/build/src/grpc.js',
+  'cookie-signature/index.js'
 )
 
 const EXCLUDED_PATHS_FROM_STACK = [
-  path.join('node_modules', 'object-hash', path.sep)
+  path.join('node_modules', 'object-hash', path.sep),
+  path.join('node_modules', 'aws-sdk', 'lib', 'util.js'),
+  path.join('node_modules', 'keygrip', path.sep)
 ]
 class WeakHashAnalyzer extends Analyzer {
   constructor () {

package/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js

@@ -114,6 +114,10 @@ function getCommitsToUpload ({ url, repositoryUrl, latestCommits, isEvpProxy, ev
 
     const commitsToUpload = getCommitsRevList(alreadySeenCommits, commitsToInclude)
 
+    if (commitsToUpload === null) {
+      return callback(new Error('git rev-list failed'))
+    }
+
     callback(null, commitsToUpload)
   })
 }
@@ -252,9 +256,8 @@ function sendGitMetadata (url, { isEvpProxy, evpProxyPrefix }, configRepositoryU
     return callback(new Error('Repository URL is empty'))
   }
 
-  const latestCommits = getLatestCommits()
+  let latestCommits = getLatestCommits()
   log.debug(`There were ${latestCommits.length} commits since last month.`)
-  const [headCommit] = latestCommits
 
   const getOnFinishGetCommitsToUpload = (hasCheckedShallow) => (err, commitsToUpload) => {
     if (err) {
@@ -268,6 +271,7 @@ function sendGitMetadata (url, { isEvpProxy, evpProxyPrefix }, configRepositoryU
 
     // If it has already unshallowed or the clone is not shallow, we move on
     if (hasCheckedShallow || !isShallowRepository()) {
+      const [headCommit] = latestCommits
       return generateAndUploadPackFiles({
         url,
         isEvpProxy,
@@ -280,6 +284,9 @@ function sendGitMetadata (url, { isEvpProxy, evpProxyPrefix }, configRepositoryU
     // Otherwise we unshallow and get commits to upload again
     log.debug('It is shallow clone, unshallowing...')
     unshallowRepository()
+
+    // The latest commits change after unshallowing
+    latestCommits = getLatestCommits()
     getCommitsToUpload({
       url,
       repositoryUrl,

package/packages/dd-trace/src/config.js

@@ -27,6 +27,7 @@ const qsRegex = '(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private
 const defaultWafObfuscatorKeyRegex = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization'
 // eslint-disable-next-line max-len
 const defaultWafObfuscatorValueRegex = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\\s*=[^;]|"\\s*:\\s*"[^"]+")|bearer\\s+[a-z0-9\\._\\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=-]+\\.ey[I-L][\\w=-]+(?:\\.[\\w.+\\/=-]+)?|[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}'
+const runtimeId = uuid()
 
 function maybeFile (filepath) {
   if (!filepath) return
@@ -291,7 +292,7 @@ class Config {
       service: this.service,
       env: this.env,
       version: this.version,
-      'runtime-id': uuid()
+      'runtime-id': runtimeId
     })
 
     if (this.isCiVisibility) {
@@ -823,6 +824,7 @@ class Config {
       : undefined
 
     tagger.add(tags, options.tracing_tags)
+    if (Object.keys(tags).length) tags['runtime-id'] = runtimeId
 
     this._setUnit(opts, 'sampleRate', options.tracing_sampling_rate)
     this._setBoolean(opts, 'logInjection', options.log_injection_enabled)

package/packages/dd-trace/src/plugins/util/git.js

@@ -28,7 +28,7 @@ const {
 const { filterSensitiveInfoFromRepository } = require('./url')
 const { storage } = require('../../../../datadog-core')
 
-const GIT_REV_LIST_MAX_BUFFER = 8 * 1024 * 1024 // 8MB
+const GIT_REV_LIST_MAX_BUFFER = 12 * 1024 * 1024 // 12MB
 
 function sanitizedExec (
   cmd,
@@ -53,11 +53,15 @@ function sanitizedExec (
       distributionMetric(durationMetric.name, durationMetric.tags, Date.now() - startTime)
     }
     return result
-  } catch (e) {
+  } catch (err) {
     if (errorMetric) {
-      incrementCountMetric(errorMetric.name, { ...errorMetric.tags, exitCode: e.status })
+      incrementCountMetric(errorMetric.name, {
+        ...errorMetric.tags,
+        errorType: err.code,
+        exitCode: err.status || err.errno
+      })
     }
-    log.error(e)
+    log.error(err)
     return ''
   } finally {
     storage.enterWith(store)
@@ -129,7 +133,10 @@ function unshallowRepository () {
   } catch (err) {
     // If the local HEAD is a commit that has not been pushed to the remote, the above command will fail.
     log.error(err)
-    incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'unshallow', exitCode: err.status })
+    incrementCountMetric(
+      TELEMETRY_GIT_COMMAND_ERRORS,
+      { command: 'unshallow', errorType: err.code, exitCode: err.status || err.errno }
+    )
     const upstreamRemote = sanitizedExec('git', ['rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{upstream}'])
     try {
       cp.execFileSync('git', [
@@ -139,7 +146,10 @@ function unshallowRepository () {
     } catch (err) {
       // If the CI is working on a detached HEAD or branch tracking hasn’t been set up, the above command will fail.
       log.error(err)
-      incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'unshallow', exitCode: err.status })
+      incrementCountMetric(
+        TELEMETRY_GIT_COMMAND_ERRORS,
+        { command: 'unshallow', errorType: err.code, exitCode: err.status || err.errno }
+      )
       // We use sanitizedExec here because if this last option fails, we'll give up.
       sanitizedExec(
         'git',
@@ -175,13 +185,16 @@ function getLatestCommits () {
     return result
   } catch (err) {
     log.error(`Get latest commits failed: ${err.message}`)
-    incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'get_local_commits', errorType: err.status })
+    incrementCountMetric(
+      TELEMETRY_GIT_COMMAND_ERRORS,
+      { command: 'get_local_commits', errorType: err.status }
+    )
     return []
   }
 }
 
 function getCommitsRevList (commitsToExclude, commitsToInclude) {
-  let result = []
+  let result = null
 
   const commitsToExcludeString = commitsToExclude.map(commit => `^${commit}`)
 
@@ -205,7 +218,10 @@ function getCommitsRevList (commitsToExclude, commitsToInclude) {
       .filter(commit => commit)
   } catch (err) {
     log.error(`Get commits to upload failed: ${err.message}`)
-    incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'get_objects', errorType: err.status })
+    incrementCountMetric(
+      TELEMETRY_GIT_COMMAND_ERRORS,
+      { command: 'get_objects', errorType: err.code, exitCode: err.status || err.errno } // err.status might be null
+    )
   }
   distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'get_objects' }, Date.now() - startTime)
   return result
@@ -245,7 +261,10 @@ function generatePackFilesForCommits (commitsToUpload) {
     result = execGitPackObjects(temporaryPath)
   } catch (err) {
     log.error(err)
-    incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'pack_objects', errorType: err.status })
+    incrementCountMetric(
+      TELEMETRY_GIT_COMMAND_ERRORS,
+      { command: 'pack_objects', exitCode: err.status || err.errno, errorType: err.code }
+    )
     /**
      * The generation of pack files in the temporary folder (from `os.tmpdir()`)
      * sometimes fails in certain CI setups with the error message
@@ -262,7 +281,10 @@ function generatePackFilesForCommits (commitsToUpload) {
       result = execGitPackObjects(cwdPath)
     } catch (err) {
       log.error(err)
-      incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'pack_objects', errorType: err.status })
+      incrementCountMetric(
+        TELEMETRY_GIT_COMMAND_ERRORS,
+        { command: 'pack_objects', exitCode: err.status || err.errno, errorType: err.code }
+      )
     }
   }
   distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'pack_objects' }, Date.now() - startTime)