dd-trace 4.18.0 → 5.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (209) hide show
  1. package/CONTRIBUTING.md +98 -0
  2. package/LICENSE-3rdparty.csv +4 -5
  3. package/MIGRATING.md +15 -0
  4. package/README.md +20 -140
  5. package/ci/cypress/after-run.js +1 -0
  6. package/ci/cypress/after-spec.js +1 -0
  7. package/ci/init.js +1 -4
  8. package/ext/kinds.d.ts +1 -0
  9. package/ext/kinds.js +2 -1
  10. package/ext/tags.d.ts +2 -1
  11. package/ext/tags.js +6 -1
  12. package/index.d.ts +1523 -1460
  13. package/package.json +19 -19
  14. package/packages/datadog-core/src/storage/async_resource.js +1 -1
  15. package/packages/datadog-core/src/utils/src/get.js +11 -0
  16. package/packages/datadog-core/src/utils/src/has.js +14 -0
  17. package/packages/datadog-core/src/utils/src/kebabcase.js +16 -0
  18. package/packages/datadog-core/src/utils/src/pick.js +11 -0
  19. package/packages/datadog-core/src/utils/src/set.js +16 -0
  20. package/packages/datadog-core/src/utils/src/uniq.js +5 -0
  21. package/packages/datadog-esbuild/index.js +1 -20
  22. package/packages/datadog-instrumentations/src/aerospike.js +47 -0
  23. package/packages/datadog-instrumentations/src/amqplib.js +2 -2
  24. package/packages/datadog-instrumentations/src/apollo-server-core.js +41 -0
  25. package/packages/datadog-instrumentations/src/apollo-server.js +83 -0
  26. package/packages/datadog-instrumentations/src/child_process.js +150 -0
  27. package/packages/datadog-instrumentations/src/couchbase.js +5 -4
  28. package/packages/datadog-instrumentations/src/crypto.js +2 -1
  29. package/packages/datadog-instrumentations/src/cucumber.js +163 -46
  30. package/packages/datadog-instrumentations/src/dns.js +2 -1
  31. package/packages/datadog-instrumentations/src/express.js +20 -0
  32. package/packages/datadog-instrumentations/src/graphql.js +18 -4
  33. package/packages/datadog-instrumentations/src/grpc/client.js +56 -36
  34. package/packages/datadog-instrumentations/src/grpc/server.js +3 -1
  35. package/packages/datadog-instrumentations/src/helpers/bundler-register.js +1 -2
  36. package/packages/datadog-instrumentations/src/helpers/hooks.js +12 -3
  37. package/packages/datadog-instrumentations/src/helpers/instrument.js +9 -4
  38. package/packages/datadog-instrumentations/src/helpers/register.js +19 -3
  39. package/packages/datadog-instrumentations/src/http/client.js +12 -2
  40. package/packages/datadog-instrumentations/src/http/server.js +7 -4
  41. package/packages/datadog-instrumentations/src/http2/client.js +3 -1
  42. package/packages/datadog-instrumentations/src/http2/server.js +3 -1
  43. package/packages/datadog-instrumentations/src/jest.js +239 -52
  44. package/packages/datadog-instrumentations/src/kafkajs.js +27 -0
  45. package/packages/datadog-instrumentations/src/mocha.js +154 -18
  46. package/packages/datadog-instrumentations/src/mongodb-core.js +34 -3
  47. package/packages/datadog-instrumentations/src/mongoose.js +23 -10
  48. package/packages/datadog-instrumentations/src/mquery.js +65 -0
  49. package/packages/datadog-instrumentations/src/net.js +10 -2
  50. package/packages/datadog-instrumentations/src/next.js +35 -9
  51. package/packages/datadog-instrumentations/src/playwright.js +110 -16
  52. package/packages/datadog-instrumentations/src/restify.js +14 -1
  53. package/packages/datadog-instrumentations/src/rhea.js +15 -9
  54. package/packages/datadog-plugin-aerospike/src/index.js +113 -0
  55. package/packages/datadog-plugin-amqplib/src/consumer.js +14 -1
  56. package/packages/datadog-plugin-amqplib/src/producer.js +13 -1
  57. package/packages/datadog-plugin-aws-sdk/src/base.js +3 -2
  58. package/packages/datadog-plugin-aws-sdk/src/services/kinesis.js +163 -27
  59. package/packages/datadog-plugin-aws-sdk/src/services/sns.js +46 -8
  60. package/packages/datadog-plugin-aws-sdk/src/services/sqs.js +129 -22
  61. package/packages/datadog-plugin-child_process/src/index.js +91 -0
  62. package/packages/datadog-plugin-child_process/src/scrub-cmd-params.js +125 -0
  63. package/packages/datadog-plugin-cucumber/src/index.js +70 -13
  64. package/packages/datadog-plugin-cypress/src/after-run.js +3 -0
  65. package/packages/datadog-plugin-cypress/src/after-spec.js +3 -0
  66. package/packages/datadog-plugin-cypress/src/cypress-plugin.js +625 -0
  67. package/packages/datadog-plugin-cypress/src/plugin.js +6 -454
  68. package/packages/datadog-plugin-cypress/src/support.js +50 -3
  69. package/packages/datadog-plugin-google-cloud-pubsub/src/consumer.js +2 -0
  70. package/packages/datadog-plugin-graphql/src/index.js +1 -6
  71. package/packages/datadog-plugin-graphql/src/resolve.js +28 -18
  72. package/packages/datadog-plugin-grpc/src/client.js +16 -2
  73. package/packages/datadog-plugin-grpc/src/util.js +1 -1
  74. package/packages/datadog-plugin-http/src/client.js +19 -2
  75. package/packages/datadog-plugin-jest/src/index.js +118 -12
  76. package/packages/datadog-plugin-jest/src/util.js +38 -16
  77. package/packages/datadog-plugin-kafkajs/src/consumer.js +76 -6
  78. package/packages/datadog-plugin-kafkajs/src/producer.js +64 -8
  79. package/packages/datadog-plugin-mocha/src/index.js +87 -17
  80. package/packages/datadog-plugin-next/src/index.js +40 -14
  81. package/packages/datadog-plugin-playwright/src/index.js +71 -8
  82. package/packages/datadog-plugin-rhea/src/consumer.js +16 -1
  83. package/packages/datadog-plugin-rhea/src/producer.js +10 -0
  84. package/packages/dd-trace/src/appsec/activation.js +29 -0
  85. package/packages/dd-trace/src/appsec/addresses.js +5 -1
  86. package/packages/dd-trace/src/appsec/api_security_sampler.js +61 -0
  87. package/packages/dd-trace/src/appsec/blocked_templates.js +4 -1
  88. package/packages/dd-trace/src/appsec/blocking.js +95 -43
  89. package/packages/dd-trace/src/appsec/channels.js +7 -3
  90. package/packages/dd-trace/src/appsec/graphql.js +146 -0
  91. package/packages/dd-trace/src/appsec/iast/analyzers/analyzers.js +2 -0
  92. package/packages/dd-trace/src/appsec/iast/analyzers/command-injection-analyzer.js +1 -1
  93. package/packages/dd-trace/src/appsec/iast/analyzers/header-injection-analyzer.js +105 -0
  94. package/packages/dd-trace/src/appsec/iast/analyzers/nosql-injection-mongodb-analyzer.js +22 -17
  95. package/packages/dd-trace/src/appsec/iast/analyzers/sql-injection-analyzer.js +7 -28
  96. package/packages/dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer.js +10 -6
  97. package/packages/dd-trace/src/appsec/iast/analyzers/weak-randomness-analyzer.js +19 -0
  98. package/packages/dd-trace/src/appsec/iast/context/context-plugin.js +90 -0
  99. package/packages/dd-trace/src/appsec/iast/context/kafka-ctx-plugin.js +14 -0
  100. package/packages/dd-trace/src/appsec/iast/iast-log.js +1 -1
  101. package/packages/dd-trace/src/appsec/iast/iast-plugin.js +13 -2
  102. package/packages/dd-trace/src/appsec/iast/index.js +15 -5
  103. package/packages/dd-trace/src/appsec/iast/overhead-controller.js +1 -1
  104. package/packages/dd-trace/src/appsec/iast/path-line.js +1 -1
  105. package/packages/dd-trace/src/appsec/iast/taint-tracking/csi-methods.js +2 -0
  106. package/packages/dd-trace/src/appsec/iast/taint-tracking/index.js +10 -0
  107. package/packages/dd-trace/src/appsec/iast/taint-tracking/operations-taint-object.js +53 -0
  108. package/packages/dd-trace/src/appsec/iast/taint-tracking/operations.js +10 -46
  109. package/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js +13 -9
  110. package/packages/dd-trace/src/appsec/iast/taint-tracking/plugins/kafka.js +47 -0
  111. package/packages/dd-trace/src/appsec/iast/taint-tracking/rewriter.js +19 -6
  112. package/packages/dd-trace/src/appsec/iast/taint-tracking/source-types.js +3 -1
  113. package/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js +41 -3
  114. package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/constants.js +7 -0
  115. package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/command-sensitive-analyzer.js +12 -19
  116. package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/header-sensitive-analyzer.js +20 -0
  117. package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/json-sensitive-analyzer.js +6 -10
  118. package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/ldap-sensitive-analyzer.js +18 -25
  119. package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/sql-sensitive-analyzer.js +79 -85
  120. package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/url-sensitive-analyzer.js +27 -36
  121. package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-handler.js +14 -11
  122. package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/utils.js +1 -1
  123. package/packages/dd-trace/src/appsec/iast/vulnerabilities.js +2 -0
  124. package/packages/dd-trace/src/appsec/index.js +49 -33
  125. package/packages/dd-trace/src/appsec/recommended.json +1763 -106
  126. package/packages/dd-trace/src/appsec/remote_config/capabilities.js +7 -1
  127. package/packages/dd-trace/src/appsec/remote_config/index.js +42 -16
  128. package/packages/dd-trace/src/appsec/remote_config/manager.js +9 -8
  129. package/packages/dd-trace/src/appsec/reporter.js +51 -34
  130. package/packages/dd-trace/src/appsec/rule_manager.js +11 -8
  131. package/packages/dd-trace/src/appsec/sdk/user_blocking.js +1 -1
  132. package/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js +28 -13
  133. package/packages/dd-trace/src/appsec/waf/waf_manager.js +0 -1
  134. package/packages/dd-trace/src/ci-visibility/{intelligent-test-runner/get-itr-configuration.js → early-flake-detection/get-known-tests.js} +17 -22
  135. package/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js +25 -6
  136. package/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js +30 -1
  137. package/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js +2 -0
  138. package/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js +30 -1
  139. package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +95 -37
  140. package/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js +134 -61
  141. package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js +37 -4
  142. package/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js +131 -0
  143. package/packages/dd-trace/src/ci-visibility/telemetry.js +130 -0
  144. package/packages/dd-trace/src/config.js +561 -470
  145. package/packages/dd-trace/src/data_streams_context.js +1 -1
  146. package/packages/dd-trace/src/datastreams/pathway.js +58 -1
  147. package/packages/dd-trace/src/datastreams/processor.js +196 -27
  148. package/packages/dd-trace/src/datastreams/writer.js +11 -5
  149. package/packages/dd-trace/src/dogstatsd.js +3 -5
  150. package/packages/dd-trace/src/encode/agentless-ci-visibility.js +44 -6
  151. package/packages/dd-trace/src/encode/coverage-ci-visibility.js +14 -0
  152. package/packages/dd-trace/src/exporters/common/agent-info-exporter.js +4 -0
  153. package/packages/dd-trace/src/exporters/common/form-data.js +4 -0
  154. package/packages/dd-trace/src/exporters/common/request.js +21 -3
  155. package/packages/dd-trace/src/format.js +30 -2
  156. package/packages/dd-trace/src/id.js +12 -0
  157. package/packages/dd-trace/src/iitm.js +1 -1
  158. package/packages/dd-trace/src/log/channels.js +1 -1
  159. package/packages/dd-trace/src/noop/proxy.js +4 -0
  160. package/packages/dd-trace/src/noop/span.js +1 -0
  161. package/packages/dd-trace/src/opentelemetry/span.js +104 -4
  162. package/packages/dd-trace/src/opentelemetry/tracer.js +9 -10
  163. package/packages/dd-trace/src/opentracing/propagation/text_map.js +16 -7
  164. package/packages/dd-trace/src/opentracing/span.js +48 -4
  165. package/packages/dd-trace/src/opentracing/span_context.js +15 -6
  166. package/packages/dd-trace/src/opentracing/tracer.js +4 -3
  167. package/packages/dd-trace/src/plugin_manager.js +1 -1
  168. package/packages/dd-trace/src/plugins/ci_plugin.js +78 -19
  169. package/packages/dd-trace/src/plugins/database.js +1 -1
  170. package/packages/dd-trace/src/plugins/index.js +7 -0
  171. package/packages/dd-trace/src/plugins/plugin.js +1 -1
  172. package/packages/dd-trace/src/plugins/util/ci.js +6 -19
  173. package/packages/dd-trace/src/plugins/util/git.js +104 -22
  174. package/packages/dd-trace/src/plugins/util/ip_extractor.js +7 -6
  175. package/packages/dd-trace/src/plugins/util/test.js +60 -10
  176. package/packages/dd-trace/src/plugins/util/url.js +26 -0
  177. package/packages/dd-trace/src/plugins/util/user-provided-git.js +4 -16
  178. package/packages/dd-trace/src/plugins/util/web.js +1 -1
  179. package/packages/dd-trace/src/priority_sampler.js +30 -38
  180. package/packages/dd-trace/src/profiler.js +5 -3
  181. package/packages/dd-trace/src/profiling/config.js +77 -24
  182. package/packages/dd-trace/src/profiling/exporters/agent.js +77 -31
  183. package/packages/dd-trace/src/profiling/exporters/file.js +2 -1
  184. package/packages/dd-trace/src/profiling/profiler.js +33 -22
  185. package/packages/dd-trace/src/profiling/profilers/events.js +270 -0
  186. package/packages/dd-trace/src/profiling/profilers/shared.js +45 -0
  187. package/packages/dd-trace/src/profiling/profilers/space.js +18 -2
  188. package/packages/dd-trace/src/profiling/profilers/wall.js +146 -70
  189. package/packages/dd-trace/src/proxy.js +56 -24
  190. package/packages/dd-trace/src/ritm.js +1 -1
  191. package/packages/dd-trace/src/sampling_rule.js +130 -0
  192. package/packages/dd-trace/src/service-naming/schemas/v0/storage.js +5 -0
  193. package/packages/dd-trace/src/service-naming/schemas/v1/storage.js +4 -0
  194. package/packages/dd-trace/src/span_processor.js +9 -1
  195. package/packages/dd-trace/src/span_sampler.js +6 -64
  196. package/packages/dd-trace/src/spanleak.js +98 -0
  197. package/packages/dd-trace/src/startup-log.js +7 -1
  198. package/packages/dd-trace/src/telemetry/dependencies.js +56 -10
  199. package/packages/dd-trace/src/telemetry/index.js +182 -53
  200. package/packages/dd-trace/src/telemetry/logs/index.js +2 -2
  201. package/packages/dd-trace/src/telemetry/send-data.js +65 -7
  202. package/packages/dd-trace/src/tracer.js +12 -5
  203. package/register.js +4 -0
  204. package/scripts/install_plugin_modules.js +11 -3
  205. package/scripts/st.js +105 -0
  206. package/packages/datadog-instrumentations/src/child-process.js +0 -30
  207. package/packages/dd-trace/src/plugins/util/exec.js +0 -13
  208. package/packages/diagnostics_channel/index.js +0 -3
  209. package/packages/diagnostics_channel/src/index.js +0 -121
@@ -1,6 +1,6 @@
1
1
  'use strict'
2
2
 
3
- const dc = require('../../../diagnostics_channel')
3
+ const dc = require('dc-polyfill')
4
4
  const semver = require('semver')
5
5
  const instrumentations = require('./instrumentations')
6
6
  const { AsyncResource } = require('async_hooks')
@@ -21,11 +21,16 @@ exports.channel = function (name) {
21
21
  * @param Function hook
22
22
  */
23
23
  exports.addHook = function addHook ({ name, versions, file }, hook) {
24
- if (!instrumentations[name]) {
25
- instrumentations[name] = []
24
+ if (typeof name === 'string') {
25
+ name = [name]
26
26
  }
27
27
 
28
- instrumentations[name].push({ name, versions, file, hook })
28
+ for (const val of name) {
29
+ if (!instrumentations[val]) {
30
+ instrumentations[val] = []
31
+ }
32
+ instrumentations[val].push({ name: val, versions, file, hook })
33
+ }
29
34
  }
30
35
 
31
36
  // AsyncResource.bind exists and binds `this` properly only from 17.8.0 and up.
@@ -1,6 +1,6 @@
1
1
  'use strict'
2
2
 
3
- const { channel } = require('../../../diagnostics_channel')
3
+ const { channel } = require('dc-polyfill')
4
4
  const path = require('path')
5
5
  const semver = require('semver')
6
6
  const Hook = require('./hook')
@@ -24,6 +24,7 @@ if (!disabledInstrumentations.has('fetch')) {
24
24
  require('../fetch')
25
25
  }
26
26
 
27
+ const HOOK_SYMBOL = Symbol('hookExportsMap')
27
28
  // TODO: make this more efficient
28
29
 
29
30
  for (const packageName of names) {
@@ -42,14 +43,29 @@ for (const packageName of names) {
42
43
  for (const { name, file, versions, hook } of instrumentations[packageName]) {
43
44
  const fullFilename = filename(name, file)
44
45
 
46
+ // Create a WeakMap associated with the hook function so that patches on the same moduleExport only happens once
47
+ // for example by instrumenting both dns and node:dns double the spans would be created
48
+ // since they both patch the same moduleExport, this WeakMap is used to mitigate that
49
+ if (!hook[HOOK_SYMBOL]) {
50
+ hook[HOOK_SYMBOL] = new WeakMap()
51
+ }
52
+
45
53
  if (moduleName === fullFilename) {
46
54
  const version = moduleVersion || getVersion(moduleBaseDir)
47
55
 
48
56
  if (matchVersion(version, versions)) {
57
+ // Check if the hook already has a set moduleExport
58
+ if (hook[HOOK_SYMBOL].has(moduleExports)) {
59
+ return moduleExports
60
+ }
61
+
49
62
  try {
50
63
  loadChannel.publish({ name, version, file })
51
-
52
- moduleExports = hook(moduleExports, version)
64
+ // Send the name and version of the module back to the callback because now addHook
65
+ // takes in an array of names so by passing the name the callback will know which module name is being used
66
+ moduleExports = hook(moduleExports, version, name)
67
+ // Set the moduleExports in the hooks weakmap
68
+ hook[HOOK_SYMBOL].set(moduleExports, name)
53
69
  } catch (e) {
54
70
  log.error(e)
55
71
  }
@@ -14,9 +14,9 @@ const endChannel = channel('apm:http:client:request:end')
14
14
  const asyncStartChannel = channel('apm:http:client:request:asyncStart')
15
15
  const errorChannel = channel('apm:http:client:request:error')
16
16
 
17
- addHook({ name: 'https' }, hookFn)
17
+ const names = ['http', 'https', 'node:http', 'node:https']
18
18
 
19
- addHook({ name: 'http' }, hookFn)
19
+ addHook({ name: names }, hookFn)
20
20
 
21
21
  function hookFn (http) {
22
22
  patch(http, 'request')
@@ -58,6 +58,7 @@ function patch (http, methodName) {
58
58
  }
59
59
 
60
60
  const options = args.options
61
+
61
62
  const finish = () => {
62
63
  if (!finished) {
63
64
  finished = true
@@ -68,9 +69,17 @@ function patch (http, methodName) {
68
69
  try {
69
70
  const req = request.call(this, options, callback)
70
71
  const emit = req.emit
72
+ const setTimeout = req.setTimeout
71
73
 
72
74
  ctx.req = req
73
75
 
76
+ // tracked to accurately discern custom request socket timeout
77
+ let customRequestTimeout = false
78
+ req.setTimeout = function () {
79
+ customRequestTimeout = true
80
+ return setTimeout.apply(this, arguments)
81
+ }
82
+
74
83
  req.emit = function (eventName, arg) {
75
84
  switch (eventName) {
76
85
  case 'response': {
@@ -88,6 +97,7 @@ function patch (http, methodName) {
88
97
  case 'error':
89
98
  case 'timeout':
90
99
  ctx.error = arg
100
+ ctx.customRequestTimeout = customRequestTimeout
91
101
  errorChannel.publish(ctx)
92
102
  case 'abort': // deprecated and replaced by `close` in node 17
93
103
  case 'close':
@@ -15,14 +15,17 @@ const finishSetHeaderCh = channel('datadog:http:server:response:set-header:finis
15
15
 
16
16
  const requestFinishedSet = new WeakSet()
17
17
 
18
- addHook({ name: 'https' }, http => {
19
- // http.ServerResponse not present on https
18
+ const httpNames = ['http', 'node:http']
19
+ const httpsNames = ['https', 'node:https']
20
+
21
+ addHook({ name: httpNames }, http => {
22
+ shimmer.wrap(http.ServerResponse.prototype, 'emit', wrapResponseEmit)
20
23
  shimmer.wrap(http.Server.prototype, 'emit', wrapEmit)
21
24
  return http
22
25
  })
23
26
 
24
- addHook({ name: 'http' }, http => {
25
- shimmer.wrap(http.ServerResponse.prototype, 'emit', wrapResponseEmit)
27
+ addHook({ name: httpsNames }, http => {
28
+ // http.ServerResponse not present on https
26
29
  shimmer.wrap(http.Server.prototype, 'emit', wrapEmit)
27
30
  return http
28
31
  })
@@ -10,6 +10,8 @@ const asyncStartChannel = channel('apm:http2:client:request:asyncStart')
10
10
  const asyncEndChannel = channel('apm:http2:client:request:asyncEnd')
11
11
  const errorChannel = channel('apm:http2:client:request:error')
12
12
 
13
+ const names = ['http2', 'node:http2']
14
+
13
15
  function createWrapEmit (ctx) {
14
16
  return function wrapEmit (emit) {
15
17
  return function (event, arg1) {
@@ -66,7 +68,7 @@ function wrapConnect (connect) {
66
68
  }
67
69
  }
68
70
 
69
- addHook({ name: 'http2' }, http2 => {
71
+ addHook({ name: names }, http2 => {
70
72
  shimmer.wrap(http2, 'connect', wrapConnect)
71
73
 
72
74
  return http2
@@ -14,7 +14,9 @@ const startServerCh = channel('apm:http2:server:request:start')
14
14
  const errorServerCh = channel('apm:http2:server:request:error')
15
15
  const finishServerCh = channel('apm:http2:server:request:finish')
16
16
 
17
- addHook({ name: 'http2' }, http2 => {
17
+ const names = ['http2', 'node:http2']
18
+
19
+ addHook({ name: names }, http2 => {
18
20
  shimmer.wrap(http2, 'createSecureServer', wrapCreateServer)
19
21
  shimmer.wrap(http2, 'createServer', wrapCreateServer)
20
22
  return http2
@@ -9,7 +9,9 @@ const {
9
9
  JEST_WORKER_COVERAGE_PAYLOAD_CODE,
10
10
  getTestLineStart,
11
11
  getTestSuitePath,
12
- getTestParametersString
12
+ getTestParametersString,
13
+ EFD_STRING,
14
+ removeEfdStringFromTestName
13
15
  } = require('../../dd-trace/src/plugins/util/test')
14
16
  const {
15
17
  getFormattedJestTestParameters,
@@ -37,17 +39,26 @@ const testRunFinishCh = channel('ci:jest:test:finish')
37
39
  const testErrCh = channel('ci:jest:test:err')
38
40
 
39
41
  const skippableSuitesCh = channel('ci:jest:test-suite:skippable')
40
- const jestItrConfigurationCh = channel('ci:jest:itr-configuration')
42
+ const libraryConfigurationCh = channel('ci:jest:library-configuration')
43
+ const knownTestsCh = channel('ci:jest:known-tests')
41
44
 
42
45
  const itrSkippedSuitesCh = channel('ci:jest:itr:skipped-suites')
43
46
 
47
+ // Maximum time we'll wait for the tracer to flush
48
+ const FLUSH_TIMEOUT = 10000
49
+
44
50
  let skippableSuites = []
51
+ let knownTests = []
45
52
  let isCodeCoverageEnabled = false
46
53
  let isSuitesSkippingEnabled = false
54
+ let isUserCodeCoverageEnabled = false
47
55
  let isSuitesSkipped = false
48
56
  let numSkippedSuites = 0
49
57
  let hasUnskippableSuites = false
50
58
  let hasForcedToRunSuites = false
59
+ let isEarlyFlakeDetectionEnabled = false
60
+ let earlyFlakeDetectionNumRetries = 0
61
+ let hasFilteredSkippableSuites = false
51
62
 
52
63
  const sessionAsyncResource = new AsyncResource('bound-anonymous-fn')
53
64
 
@@ -61,6 +72,7 @@ const specStatusToTestStatus = {
61
72
 
62
73
  const asyncResources = new WeakMap()
63
74
  const originalTestFns = new WeakMap()
75
+ const retriedTestsToNumAttempts = new Map()
64
76
 
65
77
  // based on https://github.com/facebook/jest/blob/main/packages/jest-circus/src/formatNodeAssertErrors.ts#L41
66
78
  function formatJestError (errors) {
@@ -89,6 +101,10 @@ function getTestEnvironmentOptions (config) {
89
101
  return {}
90
102
  }
91
103
 
104
+ function getEfdTestName (testName, numAttempt) {
105
+ return `${EFD_STRING} (#${numAttempt}): ${testName}`
106
+ }
107
+
92
108
  function getWrappedEnvironment (BaseEnvironment, jestVersion) {
93
109
  return class DatadogEnvironment extends BaseEnvironment {
94
110
  constructor (config, context) {
@@ -99,7 +115,48 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
99
115
  this.nameToParams = {}
100
116
  this.global._ddtrace = global._ddtrace
101
117
 
118
+ this.displayName = config.projectConfig?.displayName?.name
102
119
  this.testEnvironmentOptions = getTestEnvironmentOptions(config)
120
+
121
+ const repositoryRoot = this.testEnvironmentOptions._ddRepositoryRoot
122
+
123
+ if (repositoryRoot) {
124
+ this.testSourceFile = getTestSuitePath(context.testPath, repositoryRoot)
125
+ }
126
+
127
+ this.isEarlyFlakeDetectionEnabled = this.testEnvironmentOptions._ddIsEarlyFlakeDetectionEnabled
128
+
129
+ if (this.isEarlyFlakeDetectionEnabled) {
130
+ earlyFlakeDetectionNumRetries = this.testEnvironmentOptions._ddEarlyFlakeDetectionNumRetries
131
+ try {
132
+ this.knownTestsForThisSuite = this.getKnownTestsForSuite(this.testEnvironmentOptions._ddKnownTests)
133
+ } catch (e) {
134
+ // If there has been an error parsing the tests, we'll disable Early Flake Deteciton
135
+ this.isEarlyFlakeDetectionEnabled = false
136
+ }
137
+ }
138
+ }
139
+
140
+ // Function that receives a list of known tests for a test service and
141
+ // returns the ones that belong to the current suite
142
+ getKnownTestsForSuite (knownTests) {
143
+ if (this.knownTestsForThisSuite) {
144
+ return this.knownTestsForThisSuite
145
+ }
146
+ let knownTestsForSuite = knownTests
147
+ // If jest is using workers, known tests are serialized to json.
148
+ // If jest runs in band, they are not.
149
+ if (typeof knownTestsForSuite === 'string') {
150
+ knownTestsForSuite = JSON.parse(knownTestsForSuite)
151
+ }
152
+ return knownTestsForSuite.jest?.[this.testSuite] || []
153
+ }
154
+
155
+ // Add the `add_test` event we don't have the test object yet, so
156
+ // we use its describe block to get the full name
157
+ getTestNameFromAddTestEvent (event, state) {
158
+ const describeSuffix = getJestTestName(state.currentDescribeBlock)
159
+ return removeEfdStringFromTestName(`${describeSuffix} ${event.testName}`).trim()
103
160
  }
104
161
 
105
162
  async handleTestEvent (event, state) {
@@ -123,23 +180,57 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
123
180
  }
124
181
  }
125
182
  if (event.name === 'test_start') {
183
+ let isNewTest = false
184
+ let numEfdRetry = null
126
185
  const testParameters = getTestParametersString(this.nameToParams, event.test.name)
127
186
  // Async resource for this test is created here
128
187
  // It is used later on by the test_done handler
129
188
  const asyncResource = new AsyncResource('bound-anonymous-fn')
130
189
  asyncResources.set(event.test, asyncResource)
190
+ const testName = getJestTestName(event.test)
191
+
192
+ if (this.isEarlyFlakeDetectionEnabled) {
193
+ const originalTestName = removeEfdStringFromTestName(testName)
194
+ isNewTest = retriedTestsToNumAttempts.has(originalTestName)
195
+ if (isNewTest) {
196
+ numEfdRetry = retriedTestsToNumAttempts.get(originalTestName)
197
+ retriedTestsToNumAttempts.set(originalTestName, numEfdRetry + 1)
198
+ }
199
+ }
200
+
131
201
  asyncResource.runInAsyncScope(() => {
132
202
  testStartCh.publish({
133
- name: getJestTestName(event.test),
203
+ name: removeEfdStringFromTestName(testName),
134
204
  suite: this.testSuite,
205
+ testSourceFile: this.testSourceFile,
135
206
  runner: 'jest-circus',
207
+ displayName: this.displayName,
136
208
  testParameters,
137
- frameworkVersion: jestVersion
209
+ frameworkVersion: jestVersion,
210
+ isNew: isNewTest,
211
+ isEfdRetry: numEfdRetry > 0
138
212
  })
139
213
  originalTestFns.set(event.test, event.test.fn)
140
214
  event.test.fn = asyncResource.bind(event.test.fn)
141
215
  })
142
216
  }
217
+ if (event.name === 'add_test') {
218
+ if (this.isEarlyFlakeDetectionEnabled) {
219
+ const testName = this.getTestNameFromAddTestEvent(event, state)
220
+ const isNew = !this.knownTestsForThisSuite?.includes(testName)
221
+ const isSkipped = event.mode === 'todo' || event.mode === 'skip'
222
+ if (isNew && !isSkipped && !retriedTestsToNumAttempts.has(testName)) {
223
+ retriedTestsToNumAttempts.set(testName, 0)
224
+ for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) {
225
+ if (this.global.test) {
226
+ this.global.test(getEfdTestName(event.testName, retryIndex), event.fn, event.timeout)
227
+ } else {
228
+ log.error('Early flake detection could not retry test because global.test is undefined')
229
+ }
230
+ }
231
+ }
232
+ }
233
+ }
143
234
  if (event.name === 'test_done') {
144
235
  const asyncResource = asyncResources.get(event.test)
145
236
  asyncResource.runInAsyncScope(() => {
@@ -163,7 +254,9 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
163
254
  testSkippedCh.publish({
164
255
  name: getJestTestName(event.test),
165
256
  suite: this.testSuite,
257
+ testSourceFile: this.testSourceFile,
166
258
  runner: 'jest-circus',
259
+ displayName: this.displayName,
167
260
  frameworkVersion: jestVersion,
168
261
  testStartLine: getTestLineStart(event.test.asyncError, this.testSuite)
169
262
  })
@@ -183,6 +276,23 @@ function getTestEnvironment (pkg, jestVersion) {
183
276
  return getWrappedEnvironment(pkg, jestVersion)
184
277
  }
185
278
 
279
+ function applySuiteSkipping (originalTests, rootDir, frameworkVersion) {
280
+ const jestSuitesToRun = getJestSuitesToRun(skippableSuites, originalTests, rootDir || process.cwd())
281
+ hasFilteredSkippableSuites = true
282
+ log.debug(
283
+ () => `${jestSuitesToRun.suitesToRun.length} out of ${originalTests.length} suites are going to run.`
284
+ )
285
+ hasUnskippableSuites = jestSuitesToRun.hasUnskippableSuites
286
+ hasForcedToRunSuites = jestSuitesToRun.hasForcedToRunSuites
287
+
288
+ isSuitesSkipped = jestSuitesToRun.suitesToRun.length !== originalTests.length
289
+ numSkippedSuites = jestSuitesToRun.skippedSuites.length
290
+
291
+ itrSkippedSuitesCh.publish({ skippedSuites: jestSuitesToRun.skippedSuites, frameworkVersion })
292
+ skippableSuites = []
293
+ return jestSuitesToRun.suitesToRun
294
+ }
295
+
186
296
  addHook({
187
297
  name: 'jest-environment-node',
188
298
  versions: ['>=24.8.0']
@@ -193,6 +303,51 @@ addHook({
193
303
  versions: ['>=24.8.0']
194
304
  }, getTestEnvironment)
195
305
 
306
+ function getWrappedScheduleTests (scheduleTests, frameworkVersion) {
307
+ return async function (tests) {
308
+ if (!isSuitesSkippingEnabled || hasFilteredSkippableSuites) {
309
+ return scheduleTests.apply(this, arguments)
310
+ }
311
+ const [test] = tests
312
+ const rootDir = test?.context?.config?.rootDir
313
+
314
+ arguments[0] = applySuiteSkipping(tests, rootDir, frameworkVersion)
315
+
316
+ return scheduleTests.apply(this, arguments)
317
+ }
318
+ }
319
+
320
+ addHook({
321
+ name: '@jest/core',
322
+ file: 'build/TestScheduler.js',
323
+ versions: ['>=27.0.0']
324
+ }, (testSchedulerPackage, frameworkVersion) => {
325
+ const oldCreateTestScheduler = testSchedulerPackage.createTestScheduler
326
+ const newCreateTestScheduler = async function () {
327
+ if (!isSuitesSkippingEnabled || hasFilteredSkippableSuites) {
328
+ return oldCreateTestScheduler.apply(this, arguments)
329
+ }
330
+ // If suite skipping is enabled and has not filtered skippable suites yet, we'll attempt to do it
331
+ const scheduler = await oldCreateTestScheduler.apply(this, arguments)
332
+ shimmer.wrap(scheduler, 'scheduleTests', scheduleTests => getWrappedScheduleTests(scheduleTests, frameworkVersion))
333
+ return scheduler
334
+ }
335
+ testSchedulerPackage.createTestScheduler = newCreateTestScheduler
336
+ return testSchedulerPackage
337
+ })
338
+
339
+ addHook({
340
+ name: '@jest/core',
341
+ file: 'build/TestScheduler.js',
342
+ versions: ['>=24.8.0 <27.0.0']
343
+ }, (testSchedulerPackage, frameworkVersion) => {
344
+ shimmer.wrap(
345
+ testSchedulerPackage.default.prototype,
346
+ 'scheduleTests', scheduleTests => getWrappedScheduleTests(scheduleTests, frameworkVersion)
347
+ )
348
+ return testSchedulerPackage
349
+ })
350
+
196
351
  addHook({
197
352
  name: '@jest/test-sequencer',
198
353
  versions: ['>=24.8.0']
@@ -200,29 +355,13 @@ addHook({
200
355
  shimmer.wrap(sequencerPackage.default.prototype, 'shard', shard => function () {
201
356
  const shardedTests = shard.apply(this, arguments)
202
357
 
203
- if (!shardedTests.length) {
358
+ if (!shardedTests.length || !isSuitesSkippingEnabled || !skippableSuites.length) {
204
359
  return shardedTests
205
360
  }
206
- // TODO: could we get the rootDir from each test?
207
361
  const [test] = shardedTests
208
- const rootDir = test && test.context && test.context.config && test.context.config.rootDir
209
-
210
- const jestSuitesToRun = getJestSuitesToRun(skippableSuites, shardedTests, rootDir || process.cwd())
211
-
212
- log.debug(
213
- () => `${jestSuitesToRun.suitesToRun.length} out of ${shardedTests.length} suites are going to run.`
214
- )
215
-
216
- hasUnskippableSuites = jestSuitesToRun.hasUnskippableSuites
217
- hasForcedToRunSuites = jestSuitesToRun.hasForcedToRunSuites
218
-
219
- isSuitesSkipped = jestSuitesToRun.suitesToRun.length !== shardedTests.length
220
- numSkippedSuites = jestSuitesToRun.skippedSuites.length
362
+ const rootDir = test?.context?.config?.rootDir
221
363
 
222
- itrSkippedSuitesCh.publish({ skippedSuites: jestSuitesToRun.skippedSuites, frameworkVersion })
223
-
224
- skippableSuites = []
225
- return jestSuitesToRun.suitesToRun
364
+ return applySuiteSkipping(shardedTests, rootDir, frameworkVersion)
226
365
  })
227
366
  return sequencerPackage
228
367
  })
@@ -233,24 +372,45 @@ function cliWrapper (cli, jestVersion) {
233
372
  const configurationPromise = new Promise((resolve) => {
234
373
  onDone = resolve
235
374
  })
236
- if (!jestItrConfigurationCh.hasSubscribers) {
375
+ if (!libraryConfigurationCh.hasSubscribers) {
237
376
  return runCLI.apply(this, arguments)
238
377
  }
239
378
 
240
379
  sessionAsyncResource.runInAsyncScope(() => {
241
- jestItrConfigurationCh.publish({ onDone })
380
+ libraryConfigurationCh.publish({ onDone })
242
381
  })
243
382
 
244
383
  try {
245
- const { err, itrConfig } = await configurationPromise
384
+ const { err, libraryConfig } = await configurationPromise
246
385
  if (!err) {
247
- isCodeCoverageEnabled = itrConfig.isCodeCoverageEnabled
248
- isSuitesSkippingEnabled = itrConfig.isSuitesSkippingEnabled
386
+ isCodeCoverageEnabled = libraryConfig.isCodeCoverageEnabled
387
+ isSuitesSkippingEnabled = libraryConfig.isSuitesSkippingEnabled
388
+ isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled
389
+ earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries
249
390
  }
250
391
  } catch (err) {
251
392
  log.error(err)
252
393
  }
253
394
 
395
+ if (isEarlyFlakeDetectionEnabled) {
396
+ const knownTestsPromise = new Promise((resolve) => {
397
+ onDone = resolve
398
+ })
399
+
400
+ sessionAsyncResource.runInAsyncScope(() => {
401
+ knownTestsCh.publish({ onDone })
402
+ })
403
+
404
+ try {
405
+ const { err, knownTests: receivedKnownTests } = await knownTestsPromise
406
+ if (!err) {
407
+ knownTests = receivedKnownTests
408
+ }
409
+ } catch (err) {
410
+ log.error(err)
411
+ }
412
+ }
413
+
254
414
  if (isSuitesSkippingEnabled) {
255
415
  const skippableSuitesPromise = new Promise((resolve) => {
256
416
  onDone = resolve
@@ -289,11 +449,14 @@ function cliWrapper (cli, jestVersion) {
289
449
  } = result
290
450
 
291
451
  let testCodeCoverageLinesTotal
292
- try {
293
- const { pct, total } = coverageMap.getCoverageSummary().lines
294
- testCodeCoverageLinesTotal = total !== 0 ? pct : 0
295
- } catch (e) {
296
- // ignore errors
452
+
453
+ if (isUserCodeCoverageEnabled) {
454
+ try {
455
+ const { pct, total } = coverageMap.getCoverageSummary().lines
456
+ testCodeCoverageLinesTotal = total !== 0 ? pct : 0
457
+ } catch (e) {
458
+ // ignore errors
459
+ }
297
460
  }
298
461
  let status, error
299
462
 
@@ -307,6 +470,21 @@ function cliWrapper (cli, jestVersion) {
307
470
  status = 'fail'
308
471
  error = new Error(`Failed test suites: ${numFailedTestSuites}. Failed tests: ${numFailedTests}`)
309
472
  }
473
+ let timeoutId
474
+
475
+ // Pass the resolve callback to defer it to DC listener
476
+ const flushPromise = new Promise((resolve) => {
477
+ onDone = () => {
478
+ clearTimeout(timeoutId)
479
+ resolve()
480
+ }
481
+ })
482
+
483
+ const timeoutPromise = new Promise((resolve) => {
484
+ timeoutId = setTimeout(() => {
485
+ resolve('timeout')
486
+ }, FLUSH_TIMEOUT).unref()
487
+ })
310
488
 
311
489
  sessionAsyncResource.runInAsyncScope(() => {
312
490
  testSessionFinishCh.publish({
@@ -318,9 +496,16 @@ function cliWrapper (cli, jestVersion) {
318
496
  numSkippedSuites,
319
497
  hasUnskippableSuites,
320
498
  hasForcedToRunSuites,
321
- error
499
+ error,
500
+ isEarlyFlakeDetectionEnabled,
501
+ onDone
322
502
  })
323
503
  })
504
+ const waitingResult = await Promise.race([flushPromise, timeoutPromise])
505
+
506
+ if (waitingResult === 'timeout') {
507
+ log.error('Timeout waiting for the tracer to flush')
508
+ }
324
509
 
325
510
  numSkippedSuites = 0
326
511
 
@@ -379,6 +564,7 @@ function jestAdapterWrapper (jestAdapter, jestVersion) {
379
564
  testSuiteStartCh.publish({
380
565
  testSuite: environment.testSuite,
381
566
  testEnvironmentOptions: environment.testEnvironmentOptions,
567
+ displayName: environment.displayName,
382
568
  frameworkVersion: jestVersion
383
569
  })
384
570
  return adapter.apply(this, arguments).then(suiteResults => {
@@ -399,7 +585,7 @@ function jestAdapterWrapper (jestAdapter, jestVersion) {
399
585
  const coverageFiles = getCoveredFilenamesFromCoverage(environment.global.__coverage__)
400
586
  .map(filename => getTestSuitePath(filename, environment.rootDir))
401
587
  asyncResource.runInAsyncScope(() => {
402
- testSuiteCodeCoverageCh.publish([...coverageFiles, environment.testSuite])
588
+ testSuiteCodeCoverageCh.publish({ coverageFiles, testSuite: environment.testSuite })
403
589
  })
404
590
  }
405
591
  testSuiteFinishCh.publish({ status, errorMessage })
@@ -434,8 +620,15 @@ function configureTestEnvironment (readConfigsResult) {
434
620
  // because `jestAdapterWrapper` runs in a different process. We have to go through `testEnvironmentOptions`
435
621
  configs.forEach(config => {
436
622
  config.testEnvironmentOptions._ddTestCodeCoverageEnabled = isCodeCoverageEnabled
623
+ config.testEnvironmentOptions._ddKnownTests = knownTests
437
624
  })
438
625
 
626
+ isUserCodeCoverageEnabled = !!readConfigsResult.globalConfig.collectCoverage
627
+
628
+ if (readConfigsResult.globalConfig.forceExit) {
629
+ log.warn("Jest's '--forceExit' flag has been passed. This may cause loss of data.")
630
+ }
631
+
439
632
  if (isCodeCoverageEnabled) {
440
633
  const globalConfig = {
441
634
  ...readConfigsResult.globalConfig,
@@ -489,6 +682,13 @@ addHook({
489
682
  _ddTestModuleId,
490
683
  _ddTestSessionId,
491
684
  _ddTestCommand,
685
+ _ddForcedToRun,
686
+ _ddUnskippable,
687
+ _ddItrCorrelationId,
688
+ _ddKnownTests,
689
+ _ddIsEarlyFlakeDetectionEnabled,
690
+ _ddEarlyFlakeDetectionNumRetries,
691
+ _ddRepositoryRoot,
492
692
  ...restOfTestEnvironmentOptions
493
693
  } = testEnvironmentOptions
494
694
 
@@ -513,13 +713,13 @@ addHook({
513
713
  const SearchSource = searchSourcePackage.default ? searchSourcePackage.default : searchSourcePackage
514
714
 
515
715
  shimmer.wrap(SearchSource.prototype, 'getTestPaths', getTestPaths => async function () {
516
- if (!skippableSuites.length) {
716
+ if (!isSuitesSkippingEnabled || !skippableSuites.length) {
517
717
  return getTestPaths.apply(this, arguments)
518
718
  }
519
719
 
520
720
  const [{ rootDir, shard }] = arguments
521
721
 
522
- if (shard && shard.shardIndex) {
722
+ if (shard?.shardCount > 1) {
523
723
  // If the user is using jest sharding, we want to apply the filtering of tests in the shard process.
524
724
  // The reason for this is the following:
525
725
  // The tests for different shards are likely being run in different CI jobs so
@@ -533,21 +733,8 @@ addHook({
533
733
  const testPaths = await getTestPaths.apply(this, arguments)
534
734
  const { tests } = testPaths
535
735
 
536
- const jestSuitesToRun = getJestSuitesToRun(skippableSuites, tests, rootDir)
537
-
538
- log.debug(() => `${jestSuitesToRun.suitesToRun.length} out of ${tests.length} suites are going to run.`)
539
-
540
- hasUnskippableSuites = jestSuitesToRun.hasUnskippableSuites
541
- hasForcedToRunSuites = jestSuitesToRun.hasForcedToRunSuites
542
-
543
- isSuitesSkipped = jestSuitesToRun.suitesToRun.length !== tests.length
544
- numSkippedSuites = jestSuitesToRun.skippedSuites.length
545
-
546
- itrSkippedSuitesCh.publish({ skippedSuites: jestSuitesToRun.skippedSuites, frameworkVersion })
547
-
548
- skippableSuites = []
549
-
550
- return { ...testPaths, tests: jestSuitesToRun.suitesToRun }
736
+ const suitesToRun = applySuiteSkipping(tests, rootDir, frameworkVersion)
737
+ return { ...testPaths, tests: suitesToRun }
551
738
  })
552
739
 
553
740
  return searchSourcePackage