dd-trace 5.24.0 → 5.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/LICENSE-3rdparty.csv +3 -0
  2. package/index.d.ts +345 -8
  3. package/init.js +60 -47
  4. package/package.json +16 -7
  5. package/packages/datadog-code-origin/index.js +4 -4
  6. package/packages/datadog-core/index.js +1 -3
  7. package/packages/datadog-core/src/storage.js +21 -0
  8. package/packages/datadog-core/src/utils/src/parse-tags.js +33 -0
  9. package/packages/datadog-esbuild/index.js +4 -2
  10. package/packages/datadog-instrumentations/src/amqplib.js +65 -5
  11. package/packages/datadog-instrumentations/src/child_process.js +135 -27
  12. package/packages/datadog-instrumentations/src/express.js +1 -1
  13. package/packages/datadog-instrumentations/src/handlebars.js +40 -0
  14. package/packages/datadog-instrumentations/src/helpers/hooks.js +5 -0
  15. package/packages/datadog-instrumentations/src/helpers/register.js +9 -0
  16. package/packages/datadog-instrumentations/src/jest.js +6 -2
  17. package/packages/datadog-instrumentations/src/kafkajs.js +123 -63
  18. package/packages/datadog-instrumentations/src/mocha/utils.js +2 -2
  19. package/packages/datadog-instrumentations/src/multer.js +37 -0
  20. package/packages/datadog-instrumentations/src/openai.js +2 -2
  21. package/packages/datadog-instrumentations/src/pug.js +23 -0
  22. package/packages/datadog-instrumentations/src/router.js +2 -3
  23. package/packages/datadog-instrumentations/src/url.js +84 -0
  24. package/packages/datadog-instrumentations/src/utils/src/extract-package-and-module-path.js +7 -4
  25. package/packages/datadog-plugin-amqplib/src/consumer.js +6 -5
  26. package/packages/datadog-plugin-aws-sdk/src/base.js +5 -0
  27. package/packages/datadog-plugin-aws-sdk/src/services/eventbridge.js +1 -0
  28. package/packages/datadog-plugin-aws-sdk/src/services/kinesis.js +10 -7
  29. package/packages/datadog-plugin-aws-sdk/src/services/s3.js +35 -0
  30. package/packages/datadog-plugin-aws-sdk/src/services/sqs.js +11 -9
  31. package/packages/datadog-plugin-cypress/src/cypress-plugin.js +59 -45
  32. package/packages/datadog-plugin-cypress/src/support.js +1 -0
  33. package/packages/datadog-plugin-fastify/src/code_origin.js +2 -2
  34. package/packages/datadog-plugin-google-cloud-pubsub/src/consumer.js +10 -2
  35. package/packages/datadog-plugin-google-cloud-pubsub/src/producer.js +8 -0
  36. package/packages/datadog-plugin-grpc/src/client.js +3 -0
  37. package/packages/datadog-plugin-grpc/src/server.js +5 -1
  38. package/packages/datadog-plugin-http/src/client.js +42 -1
  39. package/packages/datadog-plugin-http2/src/client.js +26 -1
  40. package/packages/datadog-plugin-jest/src/index.js +2 -1
  41. package/packages/datadog-plugin-kafkajs/src/batch-consumer.js +6 -3
  42. package/packages/datadog-plugin-kafkajs/src/consumer.js +10 -5
  43. package/packages/datadog-plugin-kafkajs/src/producer.js +10 -4
  44. package/packages/datadog-plugin-mocha/src/index.js +5 -2
  45. package/packages/datadog-plugin-moleculer/src/server.js +2 -2
  46. package/packages/datadog-plugin-openai/src/index.js +9 -1015
  47. package/packages/datadog-plugin-openai/src/tracing.js +1023 -0
  48. package/packages/datadog-plugin-rhea/src/consumer.js +2 -1
  49. package/packages/datadog-plugin-vitest/src/index.js +2 -1
  50. package/packages/dd-trace/src/appsec/addresses.js +2 -0
  51. package/packages/dd-trace/src/appsec/api_security_sampler.js +50 -27
  52. package/packages/dd-trace/src/appsec/channels.js +3 -1
  53. package/packages/dd-trace/src/appsec/iast/analyzers/analyzers.js +1 -0
  54. package/packages/dd-trace/src/appsec/iast/analyzers/header-injection-analyzer.js +33 -16
  55. package/packages/dd-trace/src/appsec/iast/analyzers/template-injection-analyzer.js +18 -0
  56. package/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js +55 -7
  57. package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-handler.js +3 -2
  58. package/packages/dd-trace/src/appsec/iast/vulnerabilities.js +1 -0
  59. package/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js +4 -2
  60. package/packages/dd-trace/src/appsec/index.js +9 -6
  61. package/packages/dd-trace/src/appsec/rasp/command_injection.js +49 -0
  62. package/packages/dd-trace/src/appsec/rasp/index.js +3 -0
  63. package/packages/dd-trace/src/appsec/rasp/ssrf.js +4 -3
  64. package/packages/dd-trace/src/appsec/rasp/utils.js +3 -2
  65. package/packages/dd-trace/src/appsec/recommended.json +354 -158
  66. package/packages/dd-trace/src/appsec/remote_config/capabilities.js +2 -1
  67. package/packages/dd-trace/src/appsec/remote_config/index.js +2 -7
  68. package/packages/dd-trace/src/appsec/reporter.js +6 -4
  69. package/packages/dd-trace/src/appsec/sdk/track_event.js +5 -3
  70. package/packages/dd-trace/src/appsec/waf/waf_manager.js +4 -0
  71. package/packages/dd-trace/src/azure_metadata.js +120 -0
  72. package/packages/dd-trace/src/ci-visibility/dynamic-instrumentation/index.js +97 -0
  73. package/packages/dd-trace/src/ci-visibility/dynamic-instrumentation/worker/index.js +90 -0
  74. package/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js +19 -1
  75. package/packages/dd-trace/src/ci-visibility/exporters/agentless/di-logs-writer.js +53 -0
  76. package/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js +8 -1
  77. package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +43 -0
  78. package/packages/dd-trace/src/config.js +88 -10
  79. package/packages/dd-trace/src/constants.js +8 -1
  80. package/packages/dd-trace/src/crashtracking/crashtracker.js +98 -0
  81. package/packages/dd-trace/src/crashtracking/index.js +15 -0
  82. package/packages/dd-trace/src/crashtracking/noop.js +8 -0
  83. package/packages/dd-trace/src/datastreams/pathway.js +1 -0
  84. package/packages/dd-trace/src/debugger/devtools_client/index.js +9 -13
  85. package/packages/dd-trace/src/debugger/devtools_client/send.js +15 -1
  86. package/packages/dd-trace/src/debugger/devtools_client/snapshot/collector.js +57 -23
  87. package/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js +12 -2
  88. package/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js +31 -20
  89. package/packages/dd-trace/src/debugger/devtools_client/snapshot/symbols.js +6 -0
  90. package/packages/dd-trace/src/debugger/devtools_client/state.js +11 -2
  91. package/packages/dd-trace/src/debugger/index.js +10 -3
  92. package/packages/dd-trace/src/llmobs/constants/tags.js +34 -0
  93. package/packages/dd-trace/src/llmobs/constants/text.js +6 -0
  94. package/packages/dd-trace/src/llmobs/constants/writers.js +13 -0
  95. package/packages/dd-trace/src/llmobs/index.js +103 -0
  96. package/packages/dd-trace/src/llmobs/noop.js +82 -0
  97. package/packages/dd-trace/src/llmobs/plugins/base.js +65 -0
  98. package/packages/dd-trace/src/llmobs/plugins/openai.js +205 -0
  99. package/packages/dd-trace/src/llmobs/sdk.js +377 -0
  100. package/packages/dd-trace/src/llmobs/span_processor.js +195 -0
  101. package/packages/dd-trace/src/llmobs/storage.js +7 -0
  102. package/packages/dd-trace/src/llmobs/tagger.js +322 -0
  103. package/packages/dd-trace/src/llmobs/util.js +176 -0
  104. package/packages/dd-trace/src/llmobs/writers/base.js +111 -0
  105. package/packages/dd-trace/src/llmobs/writers/evaluations.js +29 -0
  106. package/packages/dd-trace/src/llmobs/writers/spans/agentProxy.js +23 -0
  107. package/packages/dd-trace/src/llmobs/writers/spans/agentless.js +17 -0
  108. package/packages/dd-trace/src/llmobs/writers/spans/base.js +52 -0
  109. package/packages/dd-trace/src/log/index.js +10 -13
  110. package/packages/dd-trace/src/log/log.js +52 -0
  111. package/packages/dd-trace/src/log/writer.js +50 -19
  112. package/packages/dd-trace/src/noop/proxy.js +3 -0
  113. package/packages/dd-trace/src/noop/span.js +4 -0
  114. package/packages/dd-trace/src/opentelemetry/span.js +16 -1
  115. package/packages/dd-trace/src/opentelemetry/tracer.js +1 -0
  116. package/packages/dd-trace/src/opentracing/propagation/text_map.js +106 -32
  117. package/packages/dd-trace/src/opentracing/span.js +26 -0
  118. package/packages/dd-trace/src/opentracing/span_context.js +1 -0
  119. package/packages/dd-trace/src/opentracing/tracer.js +8 -1
  120. package/packages/dd-trace/src/payload-tagging/config/aws.json +71 -3
  121. package/packages/dd-trace/src/plugins/outbound.js +9 -0
  122. package/packages/dd-trace/src/plugins/tracing.js +3 -3
  123. package/packages/dd-trace/src/plugins/util/inferred_proxy.js +121 -0
  124. package/packages/dd-trace/src/plugins/util/ip_extractor.js +0 -1
  125. package/packages/dd-trace/src/plugins/util/web.js +39 -11
  126. package/packages/dd-trace/src/priority_sampler.js +16 -0
  127. package/packages/dd-trace/src/profiling/config.js +3 -1
  128. package/packages/dd-trace/src/profiling/exporters/agent.js +7 -5
  129. package/packages/dd-trace/src/profiling/profilers/wall.js +2 -1
  130. package/packages/dd-trace/src/proxy.js +13 -1
  131. package/packages/dd-trace/src/span_processor.js +5 -0
  132. package/packages/dd-trace/src/telemetry/index.js +11 -1
  133. package/packages/dd-trace/src/telemetry/logs/index.js +16 -11
  134. package/packages/dd-trace/src/telemetry/logs/log-collector.js +3 -8
  135. package/packages/dd-trace/src/telemetry/metrics.js +6 -1
  136. package/packages/dd-trace/src/util.js +16 -1
  137. package/version.js +4 -2
  138. /package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/{code-injection-sensitive-analyzer.js → tainted-range-based-sensitive-analyzer.js} +0 -0
@@ -11,7 +11,7 @@ module.exports = {
11
11
  ASM_CUSTOM_RULES: 1n << 8n,
12
12
  ASM_CUSTOM_BLOCKING_RESPONSE: 1n << 9n,
13
13
  ASM_TRUSTED_IPS: 1n << 10n,
14
- ASM_API_SECURITY_SAMPLE_RATE: 1n << 11n,
14
+ ASM_API_SECURITY_SAMPLE_RATE: 1n << 11n, // deprecated
15
15
  APM_TRACING_SAMPLE_RATE: 1n << 12n,
16
16
  APM_TRACING_LOGS_INJECTION: 1n << 13n,
17
17
  APM_TRACING_HTTP_HEADER_TAGS: 1n << 14n,
@@ -20,6 +20,7 @@ module.exports = {
20
20
  ASM_RASP_SQLI: 1n << 21n,
21
21
  ASM_RASP_LFI: 1n << 22n,
22
22
  ASM_RASP_SSRF: 1n << 23n,
23
+ ASM_RASP_SHI: 1n << 24n,
23
24
  APM_TRACING_SAMPLE_RULES: 1n << 29n,
24
25
  ASM_ENDPOINT_FINGERPRINT: 1n << 32n,
25
26
  ASM_NETWORK_FINGERPRINT: 1n << 34n,
@@ -4,7 +4,6 @@ const Activation = require('../activation')
4
4
 
5
5
  const RemoteConfigManager = require('./manager')
6
6
  const RemoteConfigCapabilities = require('./capabilities')
7
- const apiSecuritySampler = require('../api_security_sampler')
8
7
 
9
8
  let rc
10
9
 
@@ -24,18 +23,12 @@ function enable (config, appsec) {
24
23
  rc.updateCapabilities(RemoteConfigCapabilities.ASM_ACTIVATION, true)
25
24
  }
26
25
 
27
- if (config.appsec.apiSecurity?.enabled) {
28
- rc.updateCapabilities(RemoteConfigCapabilities.ASM_API_SECURITY_SAMPLE_RATE, true)
29
- }
30
-
31
26
  rc.setProductHandler('ASM_FEATURES', (action, rcConfig) => {
32
27
  if (!rcConfig) return
33
28
 
34
29
  if (activation === Activation.ONECLICK) {
35
30
  enableOrDisableAppsec(action, rcConfig, config, appsec)
36
31
  }
37
-
38
- apiSecuritySampler.setRequestSampling(rcConfig.api_security?.request_sample_rate)
39
32
  })
40
33
  }
41
34
 
@@ -83,6 +76,7 @@ function enableWafUpdate (appsecConfig) {
83
76
  rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SQLI, true)
84
77
  rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SSRF, true)
85
78
  rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_LFI, true)
79
+ rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SHI, true)
86
80
  }
87
81
 
88
82
  // TODO: delete noop handlers and kPreUpdate and replace with batched handlers
@@ -114,6 +108,7 @@ function disableWafUpdate () {
114
108
  rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SQLI, false)
115
109
  rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SSRF, false)
116
110
  rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_LFI, false)
111
+ rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SHI, false)
117
112
 
118
113
  rc.removeProductHandler('ASM_DATA')
119
114
  rc.removeProductHandler('ASM_DD')
@@ -13,8 +13,9 @@ const {
13
13
  getRequestMetrics
14
14
  } = require('./telemetry')
15
15
  const zlib = require('zlib')
16
- const { MANUAL_KEEP } = require('../../../../ext/tags')
17
16
  const standalone = require('./standalone')
17
+ const { SAMPLING_MECHANISM_APPSEC } = require('../constants')
18
+ const { keepTrace } = require('../priority_sampler')
18
19
 
19
20
  // default limiter, configurable with setRateLimit()
20
21
  let limiter = new Limiter(100)
@@ -31,6 +32,7 @@ const contentHeaderList = [
31
32
 
32
33
  const EVENT_HEADERS_MAP = mapHeaderAndTags([
33
34
  ...ipHeaderList,
35
+ 'x-forwarded',
34
36
  'forwarded',
35
37
  'via',
36
38
  ...contentHeaderList,
@@ -96,8 +98,6 @@ function reportWafInit (wafVersion, rulesVersion, diagnosticsRules = {}) {
96
98
  metricsQueue.set('_dd.appsec.event_rules.errors', JSON.stringify(diagnosticsRules.errors))
97
99
  }
98
100
 
99
- metricsQueue.set(MANUAL_KEEP, 'true')
100
-
101
101
  incrementWafInitMetric(wafVersion, rulesVersion)
102
102
  }
103
103
 
@@ -129,7 +129,7 @@ function reportAttack (attackData) {
129
129
  }
130
130
 
131
131
  if (limiter.isAllowed()) {
132
- newTags[MANUAL_KEEP] = 'true'
132
+ keepTrace(rootSpan, SAMPLING_MECHANISM_APPSEC)
133
133
 
134
134
  standalone.sample(rootSpan)
135
135
  }
@@ -184,6 +184,8 @@ function finishRequest (req, res) {
184
184
  if (metricsQueue.size) {
185
185
  rootSpan.addTags(Object.fromEntries(metricsQueue))
186
186
 
187
+ keepTrace(rootSpan, SAMPLING_MECHANISM_APPSEC)
188
+
187
189
  standalone.sample(rootSpan)
188
190
 
189
191
  metricsQueue.clear()
@@ -2,10 +2,11 @@
2
2
 
3
3
  const log = require('../../log')
4
4
  const { getRootSpan } = require('./utils')
5
- const { MANUAL_KEEP } = require('../../../../../ext/tags')
6
5
  const { setUserTags } = require('./set_user')
7
6
  const standalone = require('../standalone')
8
7
  const waf = require('../waf')
8
+ const { SAMPLING_MECHANISM_APPSEC } = require('../../constants')
9
+ const { keepTrace } = require('../../priority_sampler')
9
10
 
10
11
  function trackUserLoginSuccessEvent (tracer, user, metadata) {
11
12
  // TODO: better user check here and in _setUser() ?
@@ -55,9 +56,10 @@ function trackEvent (eventName, fields, sdkMethodName, rootSpan, mode) {
55
56
  return
56
57
  }
57
58
 
59
+ keepTrace(rootSpan, SAMPLING_MECHANISM_APPSEC)
60
+
58
61
  const tags = {
59
- [`appsec.events.${eventName}.track`]: 'true',
60
- [MANUAL_KEEP]: 'true'
62
+ [`appsec.events.${eventName}.track`]: 'true'
61
63
  }
62
64
 
63
65
  if (mode === 'sdk') {
@@ -51,6 +51,10 @@ class WAFManager {
51
51
  update (newRules) {
52
52
  this.ddwaf.update(newRules)
53
53
 
54
+ if (this.ddwaf.diagnostics.ruleset_version) {
55
+ this.rulesVersion = this.ddwaf.diagnostics.ruleset_version
56
+ }
57
+
54
58
  Reporter.reportWafUpdate(this.ddwafVersion, this.rulesVersion)
55
59
  }
56
60
 
@@ -0,0 +1,120 @@
1
+ 'use strict'
2
+
3
+ // eslint-disable-next-line max-len
4
+ // Modeled after https://github.com/DataDog/libdatadog/blob/f3994857a59bb5679a65967138c5a3aec418a65f/ddcommon/src/azure_app_services.rs
5
+
6
+ const os = require('os')
7
+ const { getIsAzureFunction } = require('./serverless')
8
+
9
+ function extractSubscriptionID (ownerName) {
10
+ if (ownerName !== undefined) {
11
+ const subId = ownerName.split('+')[0].trim()
12
+ if (subId.length > 0) {
13
+ return subId
14
+ }
15
+ }
16
+ return undefined
17
+ }
18
+
19
+ function extractResourceGroup (ownerName) {
20
+ return /.+\+(.+)-.+webspace(-Linux)?/.exec(ownerName)?.[1]
21
+ }
22
+
23
+ function buildResourceID (subscriptionID, siteName, resourceGroup) {
24
+ if (subscriptionID === undefined || siteName === undefined || resourceGroup === undefined) {
25
+ return undefined
26
+ }
27
+ return `/subscriptions/${subscriptionID}/resourcegroups/${resourceGroup}/providers/microsoft.web/sites/${siteName}`
28
+ .toLowerCase()
29
+ }
30
+
31
+ function trimObject (obj) {
32
+ Object.entries(obj)
33
+ .filter(([_, value]) => value === undefined)
34
+ .forEach(([key, _]) => { delete obj[key] })
35
+ return obj
36
+ }
37
+
38
+ function buildMetadata () {
39
+ const {
40
+ COMPUTERNAME,
41
+ DD_AAS_DOTNET_EXTENSION_VERSION,
42
+ FUNCTIONS_EXTENSION_VERSION,
43
+ FUNCTIONS_WORKER_RUNTIME,
44
+ FUNCTIONS_WORKER_RUNTIME_VERSION,
45
+ WEBSITE_INSTANCE_ID,
46
+ WEBSITE_OWNER_NAME,
47
+ WEBSITE_OS,
48
+ WEBSITE_RESOURCE_GROUP,
49
+ WEBSITE_SITE_NAME
50
+ } = process.env
51
+
52
+ const subscriptionID = extractSubscriptionID(WEBSITE_OWNER_NAME)
53
+
54
+ const siteName = WEBSITE_SITE_NAME
55
+
56
+ const [siteKind, siteType] = getIsAzureFunction()
57
+ ? ['functionapp', 'function']
58
+ : ['app', 'app']
59
+
60
+ const resourceGroup = WEBSITE_RESOURCE_GROUP ?? extractResourceGroup(WEBSITE_OWNER_NAME)
61
+
62
+ return trimObject({
63
+ extensionVersion: DD_AAS_DOTNET_EXTENSION_VERSION,
64
+ functionRuntimeVersion: FUNCTIONS_EXTENSION_VERSION,
65
+ instanceID: WEBSITE_INSTANCE_ID,
66
+ instanceName: COMPUTERNAME,
67
+ operatingSystem: WEBSITE_OS ?? os.platform(),
68
+ resourceGroup,
69
+ resourceID: buildResourceID(subscriptionID, siteName, resourceGroup),
70
+ runtime: FUNCTIONS_WORKER_RUNTIME,
71
+ runtimeVersion: FUNCTIONS_WORKER_RUNTIME_VERSION,
72
+ siteKind,
73
+ siteName,
74
+ siteType,
75
+ subscriptionID
76
+ })
77
+ }
78
+
79
+ function getAzureAppMetadata () {
80
+ // DD_AZURE_APP_SERVICES is an environment variable introduced by the .NET APM team and is set automatically for
81
+ // anyone using the Datadog APM Extensions (.NET, Java, or Node) for Windows Azure App Services
82
+ // eslint-disable-next-line max-len
83
+ // See: https://github.com/DataDog/datadog-aas-extension/blob/01f94b5c28b7fa7a9ab264ca28bd4e03be603900/node/src/applicationHost.xdt#L20-L21
84
+ return process.env.DD_AZURE_APP_SERVICES !== undefined ? buildMetadata() : undefined
85
+ }
86
+
87
+ function getAzureFunctionMetadata () {
88
+ return getIsAzureFunction() ? buildMetadata() : undefined
89
+ }
90
+
91
+ // eslint-disable-next-line max-len
92
+ // Modeled after https://github.com/DataDog/libdatadog/blob/92272e90a7919f07178f3246ef8f82295513cfed/profiling/src/exporter/mod.rs#L187
93
+ // eslint-disable-next-line max-len
94
+ // and https://github.com/DataDog/libdatadog/blob/f3994857a59bb5679a65967138c5a3aec418a65f/trace-utils/src/trace_utils.rs#L533
95
+ function getAzureTagsFromMetadata (metadata) {
96
+ if (metadata === undefined) {
97
+ return {}
98
+ }
99
+ return trimObject({
100
+ 'aas.environment.extension_version': metadata.extensionVersion,
101
+ 'aas.environment.function_runtime': metadata.functionRuntimeVersion,
102
+ 'aas.environment.instance_id': metadata.instanceID,
103
+ 'aas.environment.instance_name': metadata.instanceName,
104
+ 'aas.environment.os': metadata.operatingSystem,
105
+ 'aas.environment.runtime': metadata.runtime,
106
+ 'aas.environment.runtime_version': metadata.runtimeVersion,
107
+ 'aas.resource.group': metadata.resourceGroup,
108
+ 'aas.resource.id': metadata.resourceID,
109
+ 'aas.site.kind': metadata.siteKind,
110
+ 'aas.site.name': metadata.siteName,
111
+ 'aas.site.type': metadata.siteType,
112
+ 'aas.subscription.id': metadata.subscriptionID
113
+ })
114
+ }
115
+
116
+ module.exports = {
117
+ getAzureAppMetadata,
118
+ getAzureFunctionMetadata,
119
+ getAzureTagsFromMetadata
120
+ }
@@ -0,0 +1,97 @@
1
+ 'use strict'
2
+
3
+ const { join } = require('path')
4
+ const { Worker } = require('worker_threads')
5
+ const { randomUUID } = require('crypto')
6
+ const log = require('../../log')
7
+
8
+ const probeIdToResolveBreakpointSet = new Map()
9
+ const probeIdToResolveBreakpointHit = new Map()
10
+
11
+ class TestVisDynamicInstrumentation {
12
+ constructor () {
13
+ this.worker = null
14
+ this._readyPromise = new Promise(resolve => {
15
+ this._onReady = resolve
16
+ })
17
+ this.breakpointSetChannel = new MessageChannel()
18
+ this.breakpointHitChannel = new MessageChannel()
19
+ }
20
+
21
+ // Return 3 elements:
22
+ // 1. Snapshot ID
23
+ // 2. Promise that's resolved when the breakpoint is set
24
+ // 3. Promise that's resolved when the breakpoint is hit
25
+ addLineProbe ({ file, line }) {
26
+ const snapshotId = randomUUID()
27
+ const probeId = randomUUID()
28
+
29
+ this.breakpointSetChannel.port2.postMessage({
30
+ snapshotId,
31
+ probe: { id: probeId, file, line }
32
+ })
33
+
34
+ return [
35
+ snapshotId,
36
+ new Promise(resolve => {
37
+ probeIdToResolveBreakpointSet.set(probeId, resolve)
38
+ }),
39
+ new Promise(resolve => {
40
+ probeIdToResolveBreakpointHit.set(probeId, resolve)
41
+ })
42
+ ]
43
+ }
44
+
45
+ isReady () {
46
+ return this._readyPromise
47
+ }
48
+
49
+ start () {
50
+ if (this.worker) return
51
+
52
+ const { NODE_OPTIONS, ...envWithoutNodeOptions } = process.env
53
+
54
+ log.debug('Starting Test Visibility - Dynamic Instrumentation client...')
55
+
56
+ this.worker = new Worker(
57
+ join(__dirname, 'worker', 'index.js'),
58
+ {
59
+ execArgv: [],
60
+ env: envWithoutNodeOptions,
61
+ workerData: {
62
+ breakpointSetChannel: this.breakpointSetChannel.port1,
63
+ breakpointHitChannel: this.breakpointHitChannel.port1
64
+ },
65
+ transferList: [this.breakpointSetChannel.port1, this.breakpointHitChannel.port1]
66
+ }
67
+ )
68
+ this.worker.on('online', () => {
69
+ log.debug('Test Visibility - Dynamic Instrumentation client is ready')
70
+ this._onReady()
71
+ })
72
+
73
+ // Allow the parent to exit even if the worker is still running
74
+ this.worker.unref()
75
+
76
+ this.breakpointSetChannel.port2.on('message', (message) => {
77
+ const { probeId } = message
78
+ const resolve = probeIdToResolveBreakpointSet.get(probeId)
79
+ if (resolve) {
80
+ resolve()
81
+ probeIdToResolveBreakpointSet.delete(probeId)
82
+ }
83
+ }).unref()
84
+
85
+ this.breakpointHitChannel.port2.on('message', (message) => {
86
+ const { snapshot } = message
87
+ const { probe: { id: probeId } } = snapshot
88
+ const resolve = probeIdToResolveBreakpointHit.get(probeId)
89
+ if (resolve) {
90
+ resolve({ snapshot })
91
+ probeIdToResolveBreakpointHit.delete(probeId)
92
+ }
93
+ }).unref()
94
+ }
95
+ }
96
+
97
+ module.exports = new TestVisDynamicInstrumentation()
@@ -0,0 +1,90 @@
1
+ 'use strict'
2
+
3
+ const { workerData: { breakpointSetChannel, breakpointHitChannel } } = require('worker_threads')
4
+ // TODO: move debugger/devtools_client/session to common place
5
+ const session = require('../../../debugger/devtools_client/session')
6
+ // TODO: move debugger/devtools_client/snapshot to common place
7
+ const { getLocalStateForCallFrame } = require('../../../debugger/devtools_client/snapshot')
8
+ // TODO: move debugger/devtools_client/state to common place
9
+ const {
10
+ findScriptFromPartialPath,
11
+ getStackFromCallFrames
12
+ } = require('../../../debugger/devtools_client/state')
13
+ const log = require('../../../log')
14
+
15
+ let sessionStarted = false
16
+
17
+ const breakpointIdToSnapshotId = new Map()
18
+ const breakpointIdToProbe = new Map()
19
+
20
+ session.on('Debugger.paused', async ({ params: { hitBreakpoints: [hitBreakpoint], callFrames } }) => {
21
+ const probe = breakpointIdToProbe.get(hitBreakpoint)
22
+ if (!probe) {
23
+ log.warn(`No probe found for breakpoint ${hitBreakpoint}`)
24
+ return session.post('Debugger.resume')
25
+ }
26
+
27
+ const stack = getStackFromCallFrames(callFrames)
28
+
29
+ const getLocalState = await getLocalStateForCallFrame(callFrames[0])
30
+
31
+ await session.post('Debugger.resume')
32
+
33
+ const snapshotId = breakpointIdToSnapshotId.get(hitBreakpoint)
34
+
35
+ const snapshot = {
36
+ id: snapshotId,
37
+ timestamp: Date.now(),
38
+ probe: {
39
+ id: probe.probeId,
40
+ version: '0',
41
+ location: probe.location
42
+ },
43
+ stack,
44
+ language: 'javascript'
45
+ }
46
+
47
+ const state = getLocalState()
48
+ if (state) {
49
+ snapshot.captures = {
50
+ lines: { [probe.location.lines[0]]: { locals: state } }
51
+ }
52
+ }
53
+
54
+ breakpointHitChannel.postMessage({ snapshot })
55
+ })
56
+
57
+ // TODO: add option to remove breakpoint
58
+ breakpointSetChannel.on('message', async ({ snapshotId, probe: { id: probeId, file, line } }) => {
59
+ await addBreakpoint(snapshotId, { probeId, file, line })
60
+ breakpointSetChannel.postMessage({ probeId })
61
+ })
62
+
63
+ async function addBreakpoint (snapshotId, probe) {
64
+ if (!sessionStarted) await start()
65
+ const { file, line } = probe
66
+
67
+ probe.location = { file, lines: [String(line)] }
68
+
69
+ const script = findScriptFromPartialPath(file)
70
+ if (!script) throw new Error(`No loaded script found for ${file}`)
71
+
72
+ const [path, scriptId] = script
73
+
74
+ log.debug(`Adding breakpoint at ${path}:${line}`)
75
+
76
+ const { breakpointId } = await session.post('Debugger.setBreakpoint', {
77
+ location: {
78
+ scriptId,
79
+ lineNumber: line - 1
80
+ }
81
+ })
82
+
83
+ breakpointIdToProbe.set(breakpointId, probe)
84
+ breakpointIdToSnapshotId.set(breakpointId, snapshotId)
85
+ }
86
+
87
+ function start () {
88
+ sessionStarted = true
89
+ return session.post('Debugger.enable') // return instead of await to reduce number of promises created
90
+ }
@@ -7,6 +7,7 @@ const CiVisibilityExporter = require('../ci-visibility-exporter')
7
7
 
8
8
  const AGENT_EVP_PROXY_PATH_PREFIX = '/evp_proxy/v'
9
9
  const AGENT_EVP_PROXY_PATH_REGEX = /\/evp_proxy\/v(\d+)\/?/
10
+ const AGENT_DEBUGGER_INPUT = '/debugger/v1/input'
10
11
 
11
12
  function getLatestEvpProxyVersion (err, agentInfo) {
12
13
  if (err) {
@@ -24,6 +25,10 @@ function getLatestEvpProxyVersion (err, agentInfo) {
24
25
  }, 0)
25
26
  }
26
27
 
28
+ function getCanForwardDebuggerLogs (err, agentInfo) {
29
+ return !err && agentInfo.endpoints.some(endpoint => endpoint === AGENT_DEBUGGER_INPUT)
30
+ }
31
+
27
32
  class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
28
33
  constructor (config) {
29
34
  super(config)
@@ -33,7 +38,8 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
33
38
  prioritySampler,
34
39
  lookup,
35
40
  protocolVersion,
36
- headers
41
+ headers,
42
+ isTestDynamicInstrumentationEnabled
37
43
  } = config
38
44
 
39
45
  this.getAgentInfo((err, agentInfo) => {
@@ -60,6 +66,18 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
60
66
  url: this._url,
61
67
  evpProxyPrefix
62
68
  })
69
+ if (isTestDynamicInstrumentationEnabled) {
70
+ const canFowardLogs = getCanForwardDebuggerLogs(err, agentInfo)
71
+ if (canFowardLogs) {
72
+ const DynamicInstrumentationLogsWriter = require('../agentless/di-logs-writer')
73
+ this._logsWriter = new DynamicInstrumentationLogsWriter({
74
+ url: this._url,
75
+ tags,
76
+ isAgentProxy: true
77
+ })
78
+ this._canForwardLogs = true
79
+ }
80
+ }
63
81
  } else {
64
82
  this._writer = new AgentWriter({
65
83
  url: this._url,
@@ -0,0 +1,53 @@
1
+ 'use strict'
2
+ const request = require('../../../exporters/common/request')
3
+ const log = require('../../../log')
4
+ const { safeJSONStringify } = require('../../../exporters/common/util')
5
+ const { JSONEncoder } = require('../../encode/json-encoder')
6
+
7
+ const BaseWriter = require('../../../exporters/common/writer')
8
+
9
+ // Writer used by the integration between Dynamic Instrumentation and Test Visibility
10
+ // It is used to encode and send logs to both the logs intake directly and the
11
+ // `/debugger/v1/input` endpoint in the agent, which is a proxy to the logs intake.
12
+ class DynamicInstrumentationLogsWriter extends BaseWriter {
13
+ constructor ({ url, timeout, isAgentProxy = false }) {
14
+ super(...arguments)
15
+ this._url = url
16
+ this._encoder = new JSONEncoder()
17
+ this._isAgentProxy = isAgentProxy
18
+ this.timeout = timeout
19
+ }
20
+
21
+ _sendPayload (data, _, done) {
22
+ const options = {
23
+ path: '/api/v2/logs',
24
+ method: 'POST',
25
+ headers: {
26
+ 'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY,
27
+ 'Content-Type': 'application/json'
28
+ },
29
+ // TODO: what's a good value for timeout for the logs intake?
30
+ timeout: this.timeout || 15000,
31
+ url: this._url
32
+ }
33
+
34
+ if (this._isAgentProxy) {
35
+ delete options.headers['dd-api-key']
36
+ options.path = '/debugger/v1/input'
37
+ }
38
+
39
+ log.debug(() => `Request to the logs intake: ${safeJSONStringify(options)}`)
40
+
41
+ request(data, options, (err, res) => {
42
+ if (err) {
43
+ log.error(err)
44
+ done()
45
+ return
46
+ }
47
+ log.debug(`Response from the logs intake: ${res}`)
48
+ done()
49
+ })
50
+ }
51
+ }
52
+
53
+ module.exports = DynamicInstrumentationLogsWriter
@@ -9,10 +9,11 @@ const log = require('../../../log')
9
9
  class AgentlessCiVisibilityExporter extends CiVisibilityExporter {
10
10
  constructor (config) {
11
11
  super(config)
12
- const { tags, site, url } = config
12
+ const { tags, site, url, isTestDynamicInstrumentationEnabled } = config
13
13
  // we don't need to request /info because we are using agentless by configuration
14
14
  this._isInitialized = true
15
15
  this._resolveCanUseCiVisProtocol(true)
16
+ this._canForwardLogs = true
16
17
 
17
18
  this._url = url || new URL(`https://citestcycle-intake.${site}`)
18
19
  this._writer = new Writer({ url: this._url, tags })
@@ -20,6 +21,12 @@ class AgentlessCiVisibilityExporter extends CiVisibilityExporter {
20
21
  this._coverageUrl = url || new URL(`https://citestcov-intake.${site}`)
21
22
  this._coverageWriter = new CoverageWriter({ url: this._coverageUrl })
22
23
 
24
+ if (isTestDynamicInstrumentationEnabled) {
25
+ const DynamicInstrumentationLogsWriter = require('./di-logs-writer')
26
+ this._logsUrl = url || new URL(`https://http-intake.logs.${site}`)
27
+ this._logsWriter = new DynamicInstrumentationLogsWriter({ url: this._logsUrl, tags })
28
+ }
29
+
23
30
  this._apiUrl = url || new URL(`https://api.${site}`)
24
31
  // Agentless is always gzip compatible
25
32
  this._isGzipCompatible = true
@@ -8,6 +8,7 @@ const { getSkippableSuites: getSkippableSuitesRequest } = require('../intelligen
8
8
  const { getKnownTests: getKnownTestsRequest } = require('../early-flake-detection/get-known-tests')
9
9
  const log = require('../../log')
10
10
  const AgentInfoExporter = require('../../exporters/common/agent-info-exporter')
11
+ const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('../../plugins/util/tags')
11
12
 
12
13
  function getTestConfigurationTags (tags) {
13
14
  if (!tags) {
@@ -36,6 +37,7 @@ class CiVisibilityExporter extends AgentInfoExporter {
36
37
  super(config)
37
38
  this._timer = undefined
38
39
  this._coverageTimer = undefined
40
+ this._logsTimer = undefined
39
41
  this._coverageBuffer = []
40
42
  // The library can use new features like ITR and test suite level visibility
41
43
  // AKA CI Vis Protocol
@@ -255,6 +257,47 @@ class CiVisibilityExporter extends AgentInfoExporter {
255
257
  this._export(formattedCoverage, this._coverageWriter, '_coverageTimer')
256
258
  }
257
259
 
260
+ formatLogMessage (testConfiguration, logMessage) {
261
+ const {
262
+ [GIT_REPOSITORY_URL]: gitRepositoryUrl,
263
+ [GIT_COMMIT_SHA]: gitCommitSha
264
+ } = testConfiguration
265
+
266
+ const { service, env, version } = this._config
267
+
268
+ return {
269
+ ddtags: [
270
+ ...(logMessage.ddtags || []),
271
+ `${GIT_REPOSITORY_URL}:${gitRepositoryUrl}`,
272
+ `${GIT_COMMIT_SHA}:${gitCommitSha}`
273
+ ].join(','),
274
+ level: 'error',
275
+ service,
276
+ dd: {
277
+ ...(logMessage.dd || []),
278
+ service,
279
+ env,
280
+ version
281
+ },
282
+ ddsource: 'dd_debugger',
283
+ ...logMessage
284
+ }
285
+ }
286
+
287
+ // DI logs
288
+ exportDiLogs (testConfiguration, logMessage) {
289
+ // TODO: could we lose logs if it's not initialized?
290
+ if (!this._config.isTestDynamicInstrumentationEnabled || !this._isInitialized || !this._canForwardLogs) {
291
+ return
292
+ }
293
+
294
+ this._export(
295
+ this.formatLogMessage(testConfiguration, logMessage),
296
+ this._logsWriter,
297
+ '_logsTimer'
298
+ )
299
+ }
300
+
258
301
  flush (done = () => {}) {
259
302
  if (!this._isInitialized) {
260
303
  return done()