dd-trace 6.11.0 → 6.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/LICENSE-3rdparty.csv +5 -0
  2. package/ci/test-optimization-validation/framework-adapters/cypress.js +9 -11
  3. package/ci/test-optimization-validation/scenarios/ci-wiring.js +5 -4
  4. package/index.d.ts +143 -25
  5. package/package.json +26 -25
  6. package/packages/datadog-instrumentations/src/bunyan.js +4 -2
  7. package/packages/datadog-instrumentations/src/cucumber.js +4 -4
  8. package/packages/datadog-instrumentations/src/helpers/bunyan.js +12 -3
  9. package/packages/datadog-instrumentations/src/helpers/pool-acquire.js +1 -0
  10. package/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/webdriverio.js +28 -0
  11. package/packages/datadog-instrumentations/src/helpers/rewriter/targets.json +1 -0
  12. package/packages/datadog-instrumentations/src/jest.js +140 -18
  13. package/packages/datadog-instrumentations/src/mariadb-bundle.js +1597 -0
  14. package/packages/datadog-instrumentations/src/mariadb.js +7 -0
  15. package/packages/datadog-instrumentations/src/mocha/utils.js +5 -2
  16. package/packages/datadog-instrumentations/src/pino.js +15 -4
  17. package/packages/datadog-instrumentations/src/playwright-reporter.js +149 -1
  18. package/packages/datadog-instrumentations/src/playwright.js +67 -53
  19. package/packages/datadog-instrumentations/src/vitest-main-no-worker-init.js +13 -12
  20. package/packages/datadog-instrumentations/src/vitest-worker.js +11 -0
  21. package/packages/datadog-instrumentations/src/webdriverio.js +26 -0
  22. package/packages/datadog-instrumentations/src/winston.js +6 -3
  23. package/packages/datadog-plugin-cypress/src/support.js +1 -0
  24. package/packages/datadog-plugin-http/src/client.js +2 -15
  25. package/packages/datadog-plugin-http2/src/client.js +2 -15
  26. package/packages/datadog-plugin-openai-agents/src/integration.js +15 -11
  27. package/packages/datadog-plugin-playwright/src/index.js +13 -1
  28. package/packages/datadog-plugin-undici/src/index.js +2 -15
  29. package/packages/dd-trace/src/aiguard/evaluation.js +94 -20
  30. package/packages/dd-trace/src/aiguard/integrations/vercel-ai.js +2 -3
  31. package/packages/dd-trace/src/aiguard/messages/anthropic.js +21 -9
  32. package/packages/dd-trace/src/aiguard/noop.js +9 -1
  33. package/packages/dd-trace/src/aiguard/redaction.js +183 -0
  34. package/packages/dd-trace/src/aiguard/sdk.js +7 -2
  35. package/packages/dd-trace/src/aiguard/tags.js +2 -0
  36. package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/utils.js +10 -9
  37. package/packages/dd-trace/src/appsec/reporter.js +2 -0
  38. package/packages/dd-trace/src/appsec/rule_manager.js +2 -0
  39. package/packages/dd-trace/src/appsec/telemetry/index.js +13 -0
  40. package/packages/dd-trace/src/appsec/telemetry/rasp.js +16 -0
  41. package/packages/dd-trace/src/appsec/telemetry/waf.js +24 -0
  42. package/packages/dd-trace/src/carrier.js +107 -48
  43. package/packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js +5 -5
  44. package/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js +2 -0
  45. package/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js +5 -3
  46. package/packages/dd-trace/src/ci-visibility/exporters/agentless/request-tracker.js +51 -26
  47. package/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js +4 -2
  48. package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +5 -11
  49. package/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js +6 -4
  50. package/packages/dd-trace/src/ci-visibility/final-flush.js +20 -0
  51. package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js +5 -3
  52. package/packages/dd-trace/src/ci-visibility/log-submission/log-submission-plugin.js +196 -29
  53. package/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js +4 -2
  54. package/packages/dd-trace/src/ci-visibility/requests/upload-coverage-report.js +4 -2
  55. package/packages/dd-trace/src/ci-visibility/requests/upload-test-screenshot.js +4 -2
  56. package/packages/dd-trace/src/ci-visibility/test-management/get-test-management-tests.js +5 -5
  57. package/packages/dd-trace/src/config/generated-config-types.d.ts +6 -0
  58. package/packages/dd-trace/src/config/index.js +2 -1
  59. package/packages/dd-trace/src/config/supported-configurations.json +27 -0
  60. package/packages/dd-trace/src/debugger/constants.js +1 -0
  61. package/packages/dd-trace/src/debugger/devtools_client/condition.js +1 -1
  62. package/packages/dd-trace/src/debugger/devtools_client/index.js +3 -10
  63. package/packages/dd-trace/src/debugger/devtools_client/send.js +4 -3
  64. package/packages/dd-trace/src/debugger/index.js +4 -2
  65. package/packages/dd-trace/src/debugger/inspect-segment.js +154 -0
  66. package/packages/dd-trace/src/dogstatsd.js +45 -18
  67. package/packages/dd-trace/src/encode/agentless-json.js +5 -4
  68. package/packages/dd-trace/src/evp_proxy/constants.js +8 -0
  69. package/packages/dd-trace/src/evp_proxy/direct.js +64 -0
  70. package/packages/dd-trace/src/evp_proxy/discovery.js +133 -0
  71. package/packages/dd-trace/src/evp_proxy/path.js +22 -0
  72. package/packages/dd-trace/src/exporters/agent/index.js +22 -2
  73. package/packages/dd-trace/src/exporters/agent/writer.js +4 -4
  74. package/packages/dd-trace/src/exporters/common/request.js +5 -2
  75. package/packages/dd-trace/src/exporters/common/writer.js +24 -2
  76. package/packages/dd-trace/src/exporters/span-stats/index.js +23 -4
  77. package/packages/dd-trace/src/external-logger/src/index.js +4 -3
  78. package/packages/dd-trace/src/flush.js +102 -0
  79. package/packages/dd-trace/src/llmobs/experiments/client.js +10 -2
  80. package/packages/dd-trace/src/llmobs/experiments/dataset.js +301 -14
  81. package/packages/dd-trace/src/llmobs/experiments/experiment.js +440 -112
  82. package/packages/dd-trace/src/llmobs/experiments/index.js +68 -44
  83. package/packages/dd-trace/src/llmobs/experiments/noop.js +103 -12
  84. package/packages/dd-trace/src/llmobs/experiments/result.js +1 -0
  85. package/packages/dd-trace/src/llmobs/experiments/util.js +81 -4
  86. package/packages/dd-trace/src/llmobs/index.js +80 -17
  87. package/packages/dd-trace/src/llmobs/noop.js +2 -1
  88. package/packages/dd-trace/src/llmobs/plugins/anthropic/util.js +8 -4
  89. package/packages/dd-trace/src/llmobs/plugins/claude-agent-sdk/index.js +6 -3
  90. package/packages/dd-trace/src/llmobs/plugins/openai/utils.js +13 -6
  91. package/packages/dd-trace/src/llmobs/plugins/openai-agents/utils.js +5 -7
  92. package/packages/dd-trace/src/llmobs/writers/base.js +60 -24
  93. package/packages/dd-trace/src/noop/span.js +5 -0
  94. package/packages/dd-trace/src/openfeature/constants/constants.js +0 -18
  95. package/packages/dd-trace/src/openfeature/index.js +7 -4
  96. package/packages/dd-trace/src/openfeature/writers/base.js +153 -11
  97. package/packages/dd-trace/src/openfeature/writers/exposures.js +70 -15
  98. package/packages/dd-trace/src/openfeature/writers/util.js +97 -15
  99. package/packages/dd-trace/src/opentelemetry/logs/batch_log_processor.js +45 -3
  100. package/packages/dd-trace/src/opentelemetry/logs/index.js +4 -1
  101. package/packages/dd-trace/src/opentelemetry/logs/logger_provider.js +7 -4
  102. package/packages/dd-trace/src/opentelemetry/metrics/index.js +3 -1
  103. package/packages/dd-trace/src/opentelemetry/metrics/meter_provider.js +8 -0
  104. package/packages/dd-trace/src/opentelemetry/metrics/otlp_http_metric_exporter.js +4 -2
  105. package/packages/dd-trace/src/opentelemetry/metrics/otlp_span_stats_exporter.js +4 -2
  106. package/packages/dd-trace/src/opentelemetry/metrics/periodic_metric_reader.js +25 -5
  107. package/packages/dd-trace/src/opentelemetry/otlp/otlp_http_exporter_base.js +61 -25
  108. package/packages/dd-trace/src/opentelemetry/tracer.js +26 -2
  109. package/packages/dd-trace/src/opentracing/propagation/text_map.js +429 -352
  110. package/packages/dd-trace/src/opentracing/span.js +28 -0
  111. package/packages/dd-trace/src/otel-thread-ctx.js +23 -5
  112. package/packages/dd-trace/src/plugin_manager.js +2 -0
  113. package/packages/dd-trace/src/plugins/util/status-validator.js +98 -0
  114. package/packages/dd-trace/src/plugins/util/test.js +74 -21
  115. package/packages/dd-trace/src/plugins/util/web.js +2 -65
  116. package/packages/dd-trace/src/profiling/oom.js +4 -3
  117. package/packages/dd-trace/src/profiling/profiler.js +7 -0
  118. package/packages/dd-trace/src/profiling/profilers/events.js +3 -3
  119. package/packages/dd-trace/src/profiling/profilers/wall.js +14 -14
  120. package/packages/dd-trace/src/proxy.js +36 -8
  121. package/packages/dd-trace/src/runtime_metrics/index.js +5 -0
  122. package/packages/dd-trace/src/runtime_metrics/otlp_runtime_metrics.js +5 -0
  123. package/packages/dd-trace/src/runtime_metrics/runtime_metrics.js +17 -6
  124. package/packages/dd-trace/src/serverless/telemetry-delivery-tracker.js +55 -0
  125. package/packages/dd-trace/src/serverless/vercel.js +122 -0
  126. package/packages/dd-trace/src/serverless.js +37 -20
  127. package/packages/dd-trace/src/span_stats.js +40 -3
  128. package/packages/dd-trace/src/telemetry/metrics.js +62 -8
  129. package/packages/dd-trace/src/telemetry/send-data.js +1 -0
  130. package/packages/dd-trace/src/tracer.js +22 -0
  131. package/packages/dd-trace/src/web-tags-cache.js +67 -13
  132. package/vendor/dist/@datadog/openfeature-node-server/index.js +1 -1
  133. package/vendor/dist/@datadog/sketches-js/index.js +1 -1
  134. package/vendor/dist/https-proxy-agent/LICENSE +22 -0
  135. package/vendor/dist/https-proxy-agent/index.js +4 -0
  136. package/vendor/dist/protobufjs/index.js +1 -1
  137. package/vendor/dist/protobufjs/minimal/index.js +1 -1
  138. package/vendor/dist/proxy-from-env/LICENSE +20 -0
  139. package/vendor/dist/proxy-from-env/index.js +1 -0
@@ -1,10 +1,10 @@
1
1
  'use strict'
2
2
 
3
3
  const getConfig = require('../../config')
4
- const request = require('../requests/request')
4
+ const { EVP_SUBDOMAIN_HEADER_NAME } = require('../../evp_proxy/constants')
5
+ const { joinEVPProxyPath } = require('../../evp_proxy/path')
5
6
  const id = require('../../id')
6
7
  const log = require('../../log')
7
-
8
8
  const {
9
9
  incrementCountMetric,
10
10
  distributionMetric,
@@ -14,8 +14,8 @@ const {
14
14
  TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS,
15
15
  TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES,
16
16
  } = require('../../ci-visibility/telemetry')
17
-
18
17
  const { getNumFromKnownTests } = require('../../plugins/util/test')
18
+ const request = require('../requests/request')
19
19
  const { buildCacheKey, writeToCache, withCache } = require('../requests/fs-cache')
20
20
  const { validateKnownTestsResponse } = require('../test-optimization-http-cache-schema')
21
21
 
@@ -162,8 +162,8 @@ function fetchFromApi ({
162
162
  }
163
163
 
164
164
  if (isEvpProxy) {
165
- options.path = `${evpProxyPrefix}/api/v2/ci/libraries/tests`
166
- options.headers['X-Datadog-EVP-Subdomain'] = 'api'
165
+ options.path = joinEVPProxyPath(evpProxyPrefix, '/api/v2/ci/libraries/tests')
166
+ options.headers[EVP_SUBDOMAIN_HEADER_NAME] = 'api'
167
167
  } else {
168
168
  const { DD_API_KEY } = getConfig()
169
169
  if (!DD_API_KEY) {
@@ -8,6 +8,8 @@ const request = require('../request')
8
8
  const { fetchAgentInfo } = require('../../../agent/info')
9
9
  const { DEBUGGER_INPUT_V1 } = require('../../../debugger/constants')
10
10
 
11
+ // Product-specific discovery: newest advertised version, skip v3 (citestcycle), gzip if >= v4.
12
+ // Shared `evp_proxy` discovery is an explicit path allowlist and does not cover this contract.
11
13
  const AGENT_EVP_PROXY_PATH_PREFIX = '/evp_proxy/v'
12
14
  const AGENT_EVP_PROXY_PATH_REGEX = /\/evp_proxy\/v(\d+)\/?/
13
15
 
@@ -1,7 +1,9 @@
1
1
  'use strict'
2
2
  const getConfig = require('../../../config')
3
- const log = require('../../../log')
3
+ const { EVP_SUBDOMAIN_HEADER_NAME } = require('../../../evp_proxy/constants')
4
+ const { joinEVPProxyPath } = require('../../../evp_proxy/path')
4
5
  const { safeJSONStringify } = require('../../../exporters/common/util')
6
+ const log = require('../../../log')
5
7
 
6
8
  const {
7
9
  incrementCountMetric,
@@ -53,9 +55,9 @@ class Writer extends BaseWriter {
53
55
  }
54
56
 
55
57
  if (this._evpProxyPrefix) {
56
- options.path = `${this._evpProxyPrefix}/api/v2/citestcov`
58
+ options.path = joinEVPProxyPath(this._evpProxyPrefix, '/api/v2/citestcov')
57
59
  delete options.headers['dd-api-key']
58
- options.headers['X-Datadog-EVP-Subdomain'] = 'citestcov-intake'
60
+ options.headers[EVP_SUBDOMAIN_HEADER_NAME] = 'citestcov-intake'
59
61
  }
60
62
 
61
63
  // eslint-disable-next-line eslint-rules/eslint-log-printf-style
@@ -1,22 +1,21 @@
1
1
  'use strict'
2
2
 
3
3
  const BaseWriter = require('../../../exporters/common/writer')
4
+ const { createFinalFlushTimeoutError } = require('../../final-flush')
4
5
 
5
- const FINAL_FLUSH_TIMEOUT_CODE = 'ERR_DD_TEST_OPTIMIZATION_FLUSH_TIMEOUT'
6
-
7
- class TestOptimizationRequestTracker {
8
- #writer
6
+ class FinalFlushRequestTracker {
7
+ #flush
9
8
  #pendingRequests = new Set()
10
9
  #finalFlushes = new Set()
11
10
  #activeFinalFlush
12
11
 
13
12
  /**
14
- * Creates request tracking for a Test Optimization writer.
13
+ * Creates request tracking for a Test Optimization exporter.
15
14
  *
16
- * @param {BaseWriter} writer
15
+ * @param {(done?: (error?: Error) => void, options?: { deadline?: number }) => void} flush
17
16
  */
18
- constructor (writer) {
19
- this.#writer = writer
17
+ constructor (flush) {
18
+ this.#flush = flush
20
19
  }
21
20
 
22
21
  /**
@@ -30,7 +29,7 @@ class TestOptimizationRequestTracker {
30
29
  */
31
30
  flush (done, options) {
32
31
  if (options?.deadline === undefined) {
33
- BaseWriter.prototype.flush.call(this.#writer, done, options)
32
+ this.#flush(done, options)
34
33
  return
35
34
  }
36
35
 
@@ -50,8 +49,7 @@ class TestOptimizationRequestTracker {
50
49
 
51
50
  const remaining = Math.max(0, options.deadline - Date.now())
52
51
  finalFlush.timeoutId = setTimeout(() => {
53
- const error = new Error('Timed out flushing Test Optimization data')
54
- error.code = FINAL_FLUSH_TIMEOUT_CODE
52
+ const error = createFinalFlushTimeoutError()
55
53
 
56
54
  finalFlush.error ||= error
57
55
  finalFlush.writerDone = true
@@ -72,7 +70,7 @@ class TestOptimizationRequestTracker {
72
70
  const previousFinalFlush = this.#activeFinalFlush
73
71
  this.#activeFinalFlush = finalFlush
74
72
  try {
75
- BaseWriter.prototype.flush.call(this.#writer, (error) => {
73
+ this.#flush((error) => {
76
74
  finalFlush.error ||= error
77
75
  finalFlush.writerDone = true
78
76
  this.#finishFinalFlush(finalFlush)
@@ -100,22 +98,37 @@ class TestOptimizationRequestTracker {
100
98
  this.#pendingRequests.add(pendingRequest)
101
99
  if (this.#activeFinalFlush) this.#attachRequest(this.#activeFinalFlush, pendingRequest)
102
100
 
103
- request(data, requestOptions, (error, result, statusCode, headers) => {
104
- if (error) {
105
- for (const finalFlush of pendingRequest.finalFlushes) finalFlush.error ||= error
106
- }
101
+ try {
102
+ request(data, requestOptions, (error, result, statusCode, headers) => {
103
+ if (error) {
104
+ for (const finalFlush of pendingRequest.finalFlushes) finalFlush.error ||= error
105
+ }
107
106
 
108
- try {
109
- callback(error, result, statusCode, headers)
110
- } finally {
111
- this.#pendingRequests.delete(pendingRequest)
112
- for (const finalFlush of pendingRequest.finalFlushes) {
113
- finalFlush.requests.delete(pendingRequest)
114
- this.#finishFinalFlush(finalFlush)
107
+ try {
108
+ callback(error, result, statusCode, headers)
109
+ } finally {
110
+ this.#dropRequest(pendingRequest)
115
111
  }
116
- pendingRequest.finalFlushes.clear()
117
- }
118
- })
112
+ })
113
+ } catch (error) {
114
+ this.#dropRequest(pendingRequest)
115
+ throw error
116
+ }
117
+ }
118
+
119
+ /**
120
+ * Stops tracking a settled request and releases final flushes waiting for it.
121
+ *
122
+ * @param {object} pendingRequest
123
+ * @returns {void}
124
+ */
125
+ #dropRequest (pendingRequest) {
126
+ this.#pendingRequests.delete(pendingRequest)
127
+ for (const finalFlush of pendingRequest.finalFlushes) {
128
+ finalFlush.requests.delete(pendingRequest)
129
+ this.#finishFinalFlush(finalFlush)
130
+ }
131
+ pendingRequest.finalFlushes.clear()
119
132
  }
120
133
 
121
134
  /**
@@ -161,4 +174,16 @@ class TestOptimizationRequestTracker {
161
174
  }
162
175
  }
163
176
 
177
+ class TestOptimizationRequestTracker extends FinalFlushRequestTracker {
178
+ /**
179
+ * Creates request tracking for a Test Optimization writer.
180
+ *
181
+ * @param {BaseWriter} writer
182
+ */
183
+ constructor (writer) {
184
+ super((done, options) => BaseWriter.prototype.flushDirect.call(writer, done, options))
185
+ }
186
+ }
187
+
164
188
  module.exports = TestOptimizationRequestTracker
189
+ module.exports.FinalFlushRequestTracker = FinalFlushRequestTracker
@@ -1,5 +1,7 @@
1
1
  'use strict'
2
2
  const getConfig = require('../../../config')
3
+ const { EVP_SUBDOMAIN_HEADER_NAME } = require('../../../evp_proxy/constants')
4
+ const { joinEVPProxyPath } = require('../../../evp_proxy/path')
3
5
  const { safeJSONStringify } = require('../../../exporters/common/util')
4
6
  const log = require('../../../log')
5
7
 
@@ -54,9 +56,9 @@ class Writer extends BaseWriter {
54
56
  }
55
57
 
56
58
  if (this._evpProxyPrefix) {
57
- options.path = `${this._evpProxyPrefix}/api/v2/citestcycle`
59
+ options.path = joinEVPProxyPath(this._evpProxyPrefix, '/api/v2/citestcycle')
58
60
  delete options.headers['dd-api-key']
59
- options.headers['X-Datadog-EVP-Subdomain'] = 'citestcycle-intake'
61
+ options.headers[EVP_SUBDOMAIN_HEADER_NAME] = 'citestcycle-intake'
60
62
  }
61
63
 
62
64
  // eslint-disable-next-line eslint-rules/eslint-log-printf-style
@@ -22,6 +22,11 @@ const { getSegment } = require('../../util')
22
22
  const BufferingExporter = require('../../exporters/common/buffering-exporter')
23
23
  const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('../../plugins/util/tags')
24
24
  const { TEST_STATUS } = require('../../plugins/util/test')
25
+ const {
26
+ createFinalFlushTimeoutError,
27
+ FINAL_FLUSH_FALLBACK_DELAY,
28
+ FINAL_FLUSH_TIMEOUT,
29
+ } = require('../final-flush')
25
30
  const { sendGitMetadata: sendGitMetadataRequest } = require('./git/git_metadata')
26
31
 
27
32
  const hostname = getHostname()
@@ -60,17 +65,6 @@ function getIsTestSessionTrace (trace) {
60
65
  const GIT_UPLOAD_TIMEOUT = 60_000 // 60 seconds
61
66
  const CAN_USE_CI_VIS_PROTOCOL_TIMEOUT = GIT_UPLOAD_TIMEOUT
62
67
  const MAX_COVERAGE_REPORT_FLAGS = 32
63
- const FINAL_FLUSH_TIMEOUT = 10_000
64
- const FINAL_FLUSH_FALLBACK_DELAY = 100
65
-
66
- /**
67
- * @returns {Error}
68
- */
69
- function createFinalFlushTimeoutError () {
70
- const error = new Error('Timed out waiting for Test Optimization to flush')
71
- error.code = 'ERR_DD_TEST_OPTIMIZATION_FLUSH_TIMEOUT'
72
- return error
73
- }
74
68
 
75
69
  function appendLogTag (tags, key, value) {
76
70
  if (value !== undefined) {
@@ -4,6 +4,8 @@ const fs = require('fs')
4
4
  const path = require('path')
5
5
 
6
6
  const getConfig = require('../../../config')
7
+ const { EVP_SUBDOMAIN_HEADER_NAME } = require('../../../evp_proxy/constants')
8
+ const { joinEVPProxyPath } = require('../../../evp_proxy/path')
7
9
  const FormData = require('../../../exporters/common/form-data')
8
10
  const request = require('../../../exporters/common/request')
9
11
 
@@ -83,8 +85,8 @@ function getCommitsToUpload ({ url, repositoryUrl, latestCommits, isEvpProxy, ev
83
85
  }
84
86
 
85
87
  if (isEvpProxy) {
86
- options.path = `${evpProxyPrefix}/api/v2/git/repository/search_commits`
87
- options.headers['X-Datadog-EVP-Subdomain'] = 'api'
88
+ options.path = joinEVPProxyPath(evpProxyPrefix, '/api/v2/git/repository/search_commits')
89
+ options.headers[EVP_SUBDOMAIN_HEADER_NAME] = 'api'
88
90
  delete options.headers['dd-api-key']
89
91
  }
90
92
 
@@ -179,8 +181,8 @@ function uploadPackFile ({ url, isEvpProxy, evpProxyPrefix, packFileToUpload, re
179
181
  }
180
182
 
181
183
  if (isEvpProxy) {
182
- options.path = `${evpProxyPrefix}/api/v2/git/repository/packfile`
183
- options.headers['X-Datadog-EVP-Subdomain'] = 'api'
184
+ options.path = joinEVPProxyPath(evpProxyPrefix, '/api/v2/git/repository/packfile')
185
+ options.headers[EVP_SUBDOMAIN_HEADER_NAME] = 'api'
184
186
  delete options.headers['dd-api-key']
185
187
  }
186
188
 
@@ -0,0 +1,20 @@
1
+ 'use strict'
2
+
3
+ const FINAL_FLUSH_TIMEOUT = 10_000
4
+ const FINAL_FLUSH_FALLBACK_DELAY = 100
5
+ const FINAL_FLUSH_TIMEOUT_CODE = 'ERR_DD_TEST_OPTIMIZATION_FLUSH_TIMEOUT'
6
+
7
+ /**
8
+ * @returns {Error & { code: string }}
9
+ */
10
+ function createFinalFlushTimeoutError () {
11
+ const error = new Error('Timed out waiting for Test Optimization to flush')
12
+ error.code = FINAL_FLUSH_TIMEOUT_CODE
13
+ return error
14
+ }
15
+
16
+ module.exports = {
17
+ createFinalFlushTimeoutError,
18
+ FINAL_FLUSH_FALLBACK_DELAY,
19
+ FINAL_FLUSH_TIMEOUT,
20
+ }
@@ -1,7 +1,8 @@
1
1
  'use strict'
2
2
 
3
3
  const getConfig = require('../../config')
4
- const request = require('../requests/request')
4
+ const { EVP_SUBDOMAIN_HEADER_NAME } = require('../../evp_proxy/constants')
5
+ const { joinEVPProxyPath } = require('../../evp_proxy/path')
5
6
  const log = require('../../log')
6
7
  const {
7
8
  incrementCountMetric,
@@ -13,6 +14,7 @@ const {
13
14
  TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS,
14
15
  TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES,
15
16
  } = require('../../ci-visibility/telemetry')
17
+ const request = require('../requests/request')
16
18
  const { buildCacheKey, writeToCache, withCache } = require('../requests/fs-cache')
17
19
  const { validateSkippableTestsResponse } = require('../test-optimization-http-cache-schema')
18
20
 
@@ -227,8 +229,8 @@ function fetchFromApi ({
227
229
  }
228
230
 
229
231
  if (isEvpProxy) {
230
- options.path = `${evpProxyPrefix}/api/v2/ci/tests/skippable`
231
- options.headers['X-Datadog-EVP-Subdomain'] = 'api'
232
+ options.path = joinEVPProxyPath(evpProxyPrefix, '/api/v2/ci/tests/skippable')
233
+ options.headers[EVP_SUBDOMAIN_HEADER_NAME] = 'api'
232
234
  } else {
233
235
  const { DD_API_KEY } = getConfig()
234
236
  if (!DD_API_KEY) {
@@ -1,53 +1,220 @@
1
1
  'use strict'
2
2
 
3
- const Plugin = require('../../plugins/plugin')
3
+ const { Writable } = require('node:stream')
4
+
5
+ const request = require('../../exporters/common/request')
4
6
  const log = require('../../log')
7
+ const Plugin = require('../../plugins/plugin')
8
+ const { FinalFlushRequestTracker } = require('../exporters/agentless/request-tracker')
9
+ const { FINAL_FLUSH_TIMEOUT } = require('../final-flush')
5
10
 
6
- function getWinstonLogSubmissionParameters (config) {
7
- const { site, service, DD_API_KEY, DD_AGENTLESS_LOG_SUBMISSION_URL } = config
11
+ const MAX_BATCH_BYTES = 5 * 1024 * 1024
12
+ const MAX_BATCH_LOGS = 1000
13
+ const BATCH_FLUSH_INTERVAL = 1000
8
14
 
9
- const defaultParameters = {
10
- host: `http-intake.logs.${site}`,
11
- path: `/api/v2/logs?ddsource=winston&service=${service}`,
12
- ssl: true,
13
- headers: {
14
- 'DD-API-KEY': DD_API_KEY,
15
- },
16
- }
15
+ /**
16
+ * @param {import('../../config/config-base')} config
17
+ * @returns {URL | undefined}
18
+ */
19
+ function getLogSubmissionUrl (config) {
20
+ if (config.DD_AGENTLESS_LOG_SUBMISSION_URL) {
21
+ try {
22
+ const url = new URL(config.DD_AGENTLESS_LOG_SUBMISSION_URL)
23
+ if (url.protocol === 'http:' || url.protocol === 'https:') return url
17
24
 
18
- if (!DD_AGENTLESS_LOG_SUBMISSION_URL) {
19
- return defaultParameters
25
+ log.error('Unsupported automatic log submission URL protocol: %s', url.protocol)
26
+ } catch {
27
+ log.error('Could not parse DD_AGENTLESS_LOG_SUBMISSION_URL')
28
+ }
29
+ return
20
30
  }
21
31
 
32
+ const hostname = `http-intake.logs.${config.site}`.toLowerCase()
22
33
  try {
23
- const url = new URL(DD_AGENTLESS_LOG_SUBMISSION_URL)
24
- return {
25
- host: url.hostname,
26
- port: url.port,
27
- ssl: url.protocol === 'https:',
28
- path: defaultParameters.path,
29
- headers: defaultParameters.headers,
30
- }
31
- } catch {
32
- log.error('Could not parse DD_AGENTLESS_LOG_SUBMISSION_URL')
33
- return defaultParameters
34
- }
34
+ const url = new URL(`https://${hostname}`)
35
+ if (url.hostname === hostname) return url
36
+ } catch {}
37
+
38
+ log.error('Could not parse automatic log submission site: %s', config.site)
39
+ }
40
+
41
+ /**
42
+ * @param {import('../../config/config-base')} config
43
+ * @param {string} source
44
+ * @returns {string}
45
+ */
46
+ function getLogSubmissionPath (config, source) {
47
+ return `/api/v2/logs?${new URLSearchParams({ ddsource: source, service: config.service })}`
35
48
  }
36
49
 
37
50
  class LogSubmissionPlugin extends Plugin {
38
51
  static id = 'log-submission'
39
52
 
53
+ /** @type {string[]} */
54
+ #batch = []
55
+ #batchBytes = 2
56
+ #batchSource
57
+ #logSubmissionUrl
58
+ #requestTracker
59
+ #timer
60
+ #beforeExitHandler = () => this.#flushLogs()
61
+ #createWinstonJsonFormat
62
+ #winstonStreamClass
63
+ // Winston formats records inside its transports, not at logger.write time, so (unlike Bunyan/Pino)
64
+ // the instrumentation can't publish a post-format line. A Stream transport pipes format.json()
65
+ // output through this Writable into the shared sender, reusing Winston's own cycle-safe serializer.
66
+ #winstonOutput = new Writable({
67
+ decodeStrings: false,
68
+ write: (message, encoding, callback) => {
69
+ this.#enqueueLog({ source: 'winston', message })
70
+ callback()
71
+ },
72
+ })
73
+
40
74
  constructor (...args) {
41
75
  super(...args)
76
+ this.#requestTracker = new FinalFlushRequestTracker((done) => {
77
+ this.#flushLogs()
78
+ done?.()
79
+ })
80
+
81
+ // The main-module hook (configure) and the logger.js hook (add-transport) can fire in either
82
+ // order depending on how Winston is required, so buffer loggers that arrive before configure.
83
+ const pendingWinstonLoggers = new Set()
84
+ const addWinstonTransport = (logger) => {
85
+ if (!this.#winstonStreamClass || !this.#createWinstonJsonFormat) {
86
+ pendingWinstonLoggers.add(logger)
87
+ return
88
+ }
89
+
90
+ logger.add(new this.#winstonStreamClass({
91
+ format: this.#createWinstonJsonFormat(),
92
+ stream: this.#winstonOutput,
93
+ }))
94
+ }
95
+
96
+ this.addSub('ci:log-submission:winston:configure', ({ StreamTransport, createJsonFormat }) => {
97
+ this.#winstonStreamClass = StreamTransport
98
+ this.#createWinstonJsonFormat = createJsonFormat
42
99
 
43
- this.addSub('ci:log-submission:winston:configure', (httpClass) => {
44
- this.HttpClass = httpClass
100
+ for (const logger of pendingWinstonLoggers) {
101
+ addWinstonTransport(logger)
102
+ }
103
+ pendingWinstonLoggers.clear()
45
104
  })
46
105
 
47
- this.addSub('ci:log-submission:winston:add-transport', (logger) => {
48
- logger.add(new this.HttpClass(getWinstonLogSubmissionParameters(this.config)))
106
+ this.addSub('ci:log-submission:winston:add-transport', addWinstonTransport)
107
+
108
+ this.addSub('ci:log-submission:log', (payload) => {
109
+ this.#enqueueLog(payload)
110
+ })
111
+ this.addSub('ci:log-submission:flush', ({ onDone } = {}) => {
112
+ if (!onDone) {
113
+ this.#flushLogs()
114
+ return
115
+ }
116
+
117
+ this.#requestTracker.flush(onDone, {
118
+ deadline: Date.now() + FINAL_FLUSH_TIMEOUT,
119
+ })
49
120
  })
50
121
  }
122
+
123
+ /**
124
+ * @param {boolean | (Record<string, unknown> & { enabled: boolean })} config
125
+ * @returns {void}
126
+ */
127
+ configure (config) {
128
+ if (this._enabled) this.#flushLogs()
129
+
130
+ const isEnabled = typeof config === 'boolean' ? config : config.enabled
131
+ this.#logSubmissionUrl = isEnabled && typeof config !== 'boolean'
132
+ ? getLogSubmissionUrl(config)
133
+ : undefined
134
+ super.configure(config)
135
+
136
+ const beforeExitHandlers = globalThis[Symbol.for('dd-trace')].beforeExitHandlers
137
+ if (this._enabled) {
138
+ beforeExitHandlers.add(this.#beforeExitHandler)
139
+ } else {
140
+ beforeExitHandlers.delete(this.#beforeExitHandler)
141
+ }
142
+ }
143
+
144
+ /**
145
+ * @param {{ source: string, message: string | Record<string, unknown> }} payload
146
+ * @returns {void}
147
+ */
148
+ #enqueueLog ({ source, message }) {
149
+ if (!this.#logSubmissionUrl) return
150
+
151
+ let serializedMessage
152
+ try {
153
+ serializedMessage = typeof message === 'string' ? message : JSON.stringify(message)
154
+ } catch (error) {
155
+ log.error('Could not serialize %s log for automatic submission', source, error)
156
+ return
157
+ }
158
+ if (serializedMessage === undefined) return
159
+
160
+ const messageBytes = Buffer.byteLength(serializedMessage)
161
+ if (messageBytes + 2 > MAX_BATCH_BYTES) {
162
+ log.error('Could not submit %s log because it exceeds the %d byte payload limit', source, MAX_BATCH_BYTES)
163
+ return
164
+ }
165
+
166
+ if (this.#batch.length > 0 &&
167
+ (this.#batchSource !== source || this.#batchBytes + messageBytes + 1 > MAX_BATCH_BYTES)) {
168
+ this.#flushLogs()
169
+ if (!this.#logSubmissionUrl) return
170
+ }
171
+
172
+ this.#batchSource = source
173
+ if (this.#batch.length > 0) this.#batchBytes++
174
+ this.#batch.push(serializedMessage)
175
+ this.#batchBytes += messageBytes
176
+
177
+ if (this.#batch.length === MAX_BATCH_LOGS || this.#batchBytes === MAX_BATCH_BYTES) {
178
+ this.#flushLogs()
179
+ } else if (this.#timer === undefined) {
180
+ this.#timer = setTimeout(() => this.#flushLogs(), BATCH_FLUSH_INTERVAL)
181
+ this.#timer.unref?.()
182
+ }
183
+ }
184
+
185
+ /**
186
+ * @returns {void}
187
+ */
188
+ #flushLogs () {
189
+ clearTimeout(this.#timer)
190
+ this.#timer = undefined
191
+
192
+ if (this.#batch.length === 0 || !this.#logSubmissionUrl) return
193
+
194
+ const source = this.#batchSource
195
+ const data = `[${this.#batch.join(',')}]`
196
+ this.#batch = []
197
+ this.#batchBytes = 2
198
+ this.#batchSource = undefined
199
+ const options = {
200
+ path: getLogSubmissionPath(this.config, source),
201
+ method: 'POST',
202
+ headers: {
203
+ 'DD-API-KEY': this.config.DD_API_KEY,
204
+ 'Content-Type': 'application/json',
205
+ },
206
+ url: this.#logSubmissionUrl,
207
+ }
208
+
209
+ try {
210
+ this.#requestTracker.send(request, data, options, error => {
211
+ if (error) log.error('Error submitting %s logs', source, error)
212
+ })
213
+ } catch (error) {
214
+ this.#logSubmissionUrl = undefined
215
+ log.error('Error submitting %s logs', source, error)
216
+ }
217
+ }
51
218
  }
52
219
 
53
220
  module.exports = LogSubmissionPlugin
@@ -1,6 +1,8 @@
1
1
  'use strict'
2
2
 
3
3
  const getConfig = require('../../config')
4
+ const { EVP_SUBDOMAIN_HEADER_NAME } = require('../../evp_proxy/constants')
5
+ const { joinEVPProxyPath } = require('../../evp_proxy/path')
4
6
  const id = require('../../id')
5
7
  const log = require('../../log')
6
8
  const { EARLY_FLAKE_DETECTION_RETRY_BUCKETS, createEfdRetryPolicy } = require('../efd-retry-policy')
@@ -238,8 +240,8 @@ function getLibraryConfiguration ({
238
240
  }
239
241
 
240
242
  if (isEvpProxy) {
241
- options.path = `${evpProxyPrefix}/api/v2/libraries/tests/services/setting`
242
- options.headers['X-Datadog-EVP-Subdomain'] = 'api'
243
+ options.path = joinEVPProxyPath(evpProxyPrefix, '/api/v2/libraries/tests/services/setting')
244
+ options.headers[EVP_SUBDOMAIN_HEADER_NAME] = 'api'
243
245
  } else {
244
246
  if (!config.DD_API_KEY) {
245
247
  return done(new Error('Request to settings endpoint was not done because Datadog API key is not defined.'))
@@ -4,6 +4,8 @@ const { closeSync, constants, fstatSync, openSync, readFileSync } = require('nod
4
4
  const { gzipSync } = require('node:zlib')
5
5
 
6
6
  const getConfig = require('../../config')
7
+ const { EVP_SUBDOMAIN_HEADER_NAME } = require('../../evp_proxy/constants')
8
+ const { joinEVPProxyPath } = require('../../evp_proxy/path')
7
9
  const FormData = require('../../exporters/common/form-data')
8
10
  const request = require('../../exporters/common/request')
9
11
  const log = require('../../log')
@@ -98,8 +100,8 @@ function uploadCoverageReport (
98
100
  }
99
101
 
100
102
  if (isEvpProxy) {
101
- options.path = `${evpProxyPrefix}/api/v2/cicovreprt`
102
- options.headers['X-Datadog-EVP-Subdomain'] = 'ci-intake'
103
+ options.path = joinEVPProxyPath(evpProxyPrefix, '/api/v2/cicovreprt')
104
+ options.headers[EVP_SUBDOMAIN_HEADER_NAME] = 'ci-intake'
103
105
  } else {
104
106
  options.path = '/api/v2/cicovreprt'
105
107
  options.headers['dd-api-key'] = DD_API_KEY
@@ -4,6 +4,8 @@ const { readFileSync } = require('node:fs')
4
4
  const { extname } = require('node:path')
5
5
 
6
6
  const getConfig = require('../../config')
7
+ const { EVP_SUBDOMAIN_HEADER_NAME } = require('../../evp_proxy/constants')
8
+ const { joinEVPProxyPath } = require('../../evp_proxy/path')
7
9
  const log = require('../../log')
8
10
  const request = require('../exporters/request')
9
11
 
@@ -134,8 +136,8 @@ function uploadTestScreenshot (
134
136
  if (isEvpProxy) {
135
137
  // Agent mode: prefix the evp_proxy path, tell the proxy which subdomain to forward to, and
136
138
  // drop the API key — the Agent injects it. The query params survive the proxy.
137
- options.path = `${evpProxyPrefix}${basePath}?${query}`
138
- options.headers['X-Datadog-EVP-Subdomain'] = 'api'
139
+ options.path = `${joinEVPProxyPath(evpProxyPrefix, basePath)}?${query}`
140
+ options.headers[EVP_SUBDOMAIN_HEADER_NAME] = 'api'
139
141
  } else {
140
142
  options.headers['DD-API-KEY'] = DD_API_KEY
141
143
  }
@@ -1,10 +1,10 @@
1
1
  'use strict'
2
2
 
3
3
  const getConfig = require('../../config')
4
- const request = require('../requests/request')
4
+ const { EVP_SUBDOMAIN_HEADER_NAME } = require('../../evp_proxy/constants')
5
+ const { joinEVPProxyPath } = require('../../evp_proxy/path')
5
6
  const id = require('../../id')
6
7
  const log = require('../../log')
7
-
8
8
  const {
9
9
  incrementCountMetric,
10
10
  distributionMetric,
@@ -14,7 +14,7 @@ const {
14
14
  TELEMETRY_TEST_MANAGEMENT_TESTS_RESPONSE_TESTS,
15
15
  TELEMETRY_TEST_MANAGEMENT_TESTS_RESPONSE_BYTES,
16
16
  } = require('../telemetry')
17
-
17
+ const request = require('../requests/request')
18
18
  const { buildCacheKey, writeToCache, withCache } = require('../requests/fs-cache')
19
19
  const { validateTestManagementTestsResponse } = require('../test-optimization-http-cache-schema')
20
20
 
@@ -132,8 +132,8 @@ function fetchFromApi ({
132
132
  }
133
133
 
134
134
  if (isEvpProxy) {
135
- options.path = `${evpProxyPrefix}/api/v2/test/libraries/test-management/tests`
136
- options.headers['X-Datadog-EVP-Subdomain'] = 'api'
135
+ options.path = joinEVPProxyPath(evpProxyPrefix, '/api/v2/test/libraries/test-management/tests')
136
+ options.headers[EVP_SUBDOMAIN_HEADER_NAME] = 'api'
137
137
  } else {
138
138
  const { DD_API_KEY } = getConfig()
139
139
  if (!DD_API_KEY) {