dd-trace 5.2.0 → 5.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE-3rdparty.csv +1 -0
- package/README.md +1 -32
- package/ci/init.js +1 -4
- package/index.d.ts +21 -0
- package/package.json +7 -6
- package/packages/datadog-instrumentations/src/amqplib.js +1 -1
- package/packages/datadog-instrumentations/src/child_process.js +150 -0
- package/packages/datadog-instrumentations/src/cucumber.js +12 -12
- package/packages/datadog-instrumentations/src/express.js +20 -0
- package/packages/datadog-instrumentations/src/grpc/client.js +56 -36
- package/packages/datadog-instrumentations/src/helpers/hooks.js +2 -2
- package/packages/datadog-instrumentations/src/jest.js +149 -11
- package/packages/datadog-instrumentations/src/mocha.js +142 -16
- package/packages/datadog-instrumentations/src/mongoose.js +23 -10
- package/packages/datadog-instrumentations/src/next.js +17 -3
- package/packages/datadog-instrumentations/src/playwright.js +41 -9
- package/packages/datadog-plugin-amqplib/src/consumer.js +10 -1
- package/packages/datadog-plugin-amqplib/src/producer.js +14 -1
- package/packages/datadog-plugin-aws-sdk/src/services/kinesis.js +107 -1
- package/packages/datadog-plugin-child_process/src/index.js +91 -0
- package/packages/datadog-plugin-child_process/src/scrub-cmd-params.js +125 -0
- package/packages/datadog-plugin-cucumber/src/index.js +16 -11
- package/packages/datadog-plugin-cypress/src/plugin.js +52 -23
- package/packages/datadog-plugin-grpc/src/client.js +16 -2
- package/packages/datadog-plugin-http/src/client.js +1 -1
- package/packages/datadog-plugin-jest/src/index.js +43 -6
- package/packages/datadog-plugin-kafkajs/src/consumer.js +16 -0
- package/packages/datadog-plugin-mocha/src/index.js +47 -17
- package/packages/datadog-plugin-playwright/src/index.js +19 -5
- package/packages/datadog-plugin-rhea/src/consumer.js +11 -1
- package/packages/datadog-plugin-rhea/src/producer.js +11 -0
- package/packages/dd-trace/src/appsec/addresses.js +2 -0
- package/packages/dd-trace/src/appsec/api_security_sampler.js +16 -3
- package/packages/dd-trace/src/appsec/channels.js +2 -1
- package/packages/dd-trace/src/appsec/iast/analyzers/command-injection-analyzer.js +1 -1
- package/packages/dd-trace/src/appsec/iast/analyzers/sql-injection-analyzer.js +7 -28
- package/packages/dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer.js +10 -6
- package/packages/dd-trace/src/appsec/iast/context/context-plugin.js +90 -0
- package/packages/dd-trace/src/appsec/iast/context/kafka-ctx-plugin.js +14 -0
- package/packages/dd-trace/src/appsec/iast/iast-plugin.js +12 -1
- package/packages/dd-trace/src/appsec/iast/index.js +4 -4
- package/packages/dd-trace/src/appsec/iast/overhead-controller.js +1 -1
- package/packages/dd-trace/src/appsec/iast/taint-tracking/csi-methods.js +1 -0
- package/packages/dd-trace/src/appsec/iast/taint-tracking/index.js +10 -0
- package/packages/dd-trace/src/appsec/iast/taint-tracking/operations-taint-object.js +53 -0
- package/packages/dd-trace/src/appsec/iast/taint-tracking/operations.js +10 -46
- package/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js +13 -9
- package/packages/dd-trace/src/appsec/iast/taint-tracking/plugins/kafka.js +47 -0
- package/packages/dd-trace/src/appsec/iast/taint-tracking/source-types.js +3 -1
- package/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js +29 -2
- package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/utils.js +1 -1
- package/packages/dd-trace/src/appsec/index.js +17 -2
- package/packages/dd-trace/src/appsec/remote_config/capabilities.js +2 -1
- package/packages/dd-trace/src/appsec/remote_config/index.js +1 -0
- package/packages/dd-trace/src/appsec/rule_manager.js +2 -2
- package/packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js +83 -0
- package/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js +25 -6
- package/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js +2 -0
- package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +83 -41
- package/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js +30 -8
- package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js +7 -1
- package/packages/dd-trace/src/ci-visibility/{intelligent-test-runner/get-itr-configuration.js → requests/get-library-configuration.js} +18 -6
- package/packages/dd-trace/src/config.js +22 -9
- package/packages/dd-trace/src/datastreams/processor.js +6 -0
- package/packages/dd-trace/src/datastreams/writer.js +2 -5
- package/packages/dd-trace/src/dogstatsd.js +3 -5
- package/packages/dd-trace/src/encode/agentless-ci-visibility.js +5 -3
- package/packages/dd-trace/src/exporters/common/request.js +21 -3
- package/packages/dd-trace/src/format.js +25 -1
- package/packages/dd-trace/src/noop/span.js +1 -0
- package/packages/dd-trace/src/opentelemetry/span.js +9 -2
- package/packages/dd-trace/src/opentracing/span.js +38 -0
- package/packages/dd-trace/src/opentracing/span_context.js +12 -6
- package/packages/dd-trace/src/opentracing/tracer.js +2 -1
- package/packages/dd-trace/src/plugins/ci_plugin.js +25 -9
- package/packages/dd-trace/src/plugins/index.js +1 -0
- package/packages/dd-trace/src/plugins/util/git.js +6 -0
- package/packages/dd-trace/src/plugins/util/test.js +53 -8
- package/packages/dd-trace/src/profiling/config.js +22 -22
- package/packages/dd-trace/src/proxy.js +31 -23
- package/packages/dd-trace/src/span_processor.js +5 -1
- package/packages/dd-trace/src/telemetry/index.js +6 -0
- package/packages/dd-trace/src/telemetry/logs/index.js +2 -2
- package/packages/dd-trace/src/telemetry/send-data.js +0 -3
- package/packages/datadog-instrumentations/src/child-process.js +0 -29
- package/packages/dd-trace/src/plugins/util/exec.js +0 -34
|
@@ -11,7 +11,8 @@ const {
|
|
|
11
11
|
passportVerify,
|
|
12
12
|
queryParser,
|
|
13
13
|
nextBodyParsed,
|
|
14
|
-
nextQueryParsed
|
|
14
|
+
nextQueryParsed,
|
|
15
|
+
responseBody
|
|
15
16
|
} = require('./channels')
|
|
16
17
|
const waf = require('./waf')
|
|
17
18
|
const addresses = require('./addresses')
|
|
@@ -53,6 +54,7 @@ function enable (_config) {
|
|
|
53
54
|
nextQueryParsed.subscribe(onRequestQueryParsed)
|
|
54
55
|
queryParser.subscribe(onRequestQueryParsed)
|
|
55
56
|
cookieParser.subscribe(onRequestCookieParser)
|
|
57
|
+
responseBody.subscribe(onResponseBody)
|
|
56
58
|
|
|
57
59
|
if (_config.appsec.eventTracking.enabled) {
|
|
58
60
|
passportVerify.subscribe(onPassportVerify)
|
|
@@ -93,7 +95,7 @@ function incomingHttpStartTranslator ({ req, res, abortController }) {
|
|
|
93
95
|
persistent[addresses.HTTP_CLIENT_IP] = clientIp
|
|
94
96
|
}
|
|
95
97
|
|
|
96
|
-
if (apiSecuritySampler.sampleRequest()) {
|
|
98
|
+
if (apiSecuritySampler.sampleRequest(req)) {
|
|
97
99
|
persistent[addresses.WAF_CONTEXT_PROCESSOR] = { 'extract-schema': true }
|
|
98
100
|
}
|
|
99
101
|
|
|
@@ -194,6 +196,18 @@ function onRequestCookieParser ({ req, res, abortController, cookies }) {
|
|
|
194
196
|
handleResults(results, req, res, rootSpan, abortController)
|
|
195
197
|
}
|
|
196
198
|
|
|
199
|
+
function onResponseBody ({ req, body }) {
|
|
200
|
+
if (!body || typeof body !== 'object') return
|
|
201
|
+
if (!apiSecuritySampler.isSampled(req)) return
|
|
202
|
+
|
|
203
|
+
// we don't support blocking at this point, so no results needed
|
|
204
|
+
waf.run({
|
|
205
|
+
persistent: {
|
|
206
|
+
[addresses.HTTP_OUTGOING_BODY]: body
|
|
207
|
+
}
|
|
208
|
+
}, req)
|
|
209
|
+
}
|
|
210
|
+
|
|
197
211
|
function onPassportVerify ({ credentials, user }) {
|
|
198
212
|
const store = storage.getStore()
|
|
199
213
|
const rootSpan = store?.req && web.root(store.req)
|
|
@@ -233,6 +247,7 @@ function disable () {
|
|
|
233
247
|
if (incomingHttpRequestEnd.hasSubscribers) incomingHttpRequestEnd.unsubscribe(incomingHttpEndTranslator)
|
|
234
248
|
if (queryParser.hasSubscribers) queryParser.unsubscribe(onRequestQueryParsed)
|
|
235
249
|
if (cookieParser.hasSubscribers) cookieParser.unsubscribe(onRequestCookieParser)
|
|
250
|
+
if (responseBody.hasSubscribers) responseBody.unsubscribe(onResponseBody)
|
|
236
251
|
if (passportVerify.hasSubscribers) passportVerify.unsubscribe(onPassportVerify)
|
|
237
252
|
}
|
|
238
253
|
|
|
@@ -14,6 +14,7 @@ function enable (config) {
|
|
|
14
14
|
rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_HTTP_HEADER_TAGS, true)
|
|
15
15
|
rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_LOGS_INJECTION, true)
|
|
16
16
|
rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_SAMPLE_RATE, true)
|
|
17
|
+
rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_ENABLED, true)
|
|
17
18
|
|
|
18
19
|
const activation = Activation.fromConfig(config)
|
|
19
20
|
|
|
@@ -69,9 +69,9 @@ function updateWafFromRC ({ toUnapply, toApply, toModify }) {
|
|
|
69
69
|
item.apply_error = 'Multiple ruleset received in ASM_DD'
|
|
70
70
|
} else {
|
|
71
71
|
if (file && file.rules && file.rules.length) {
|
|
72
|
-
const { version, metadata, rules } = file
|
|
72
|
+
const { version, metadata, rules, processors, scanners } = file
|
|
73
73
|
|
|
74
|
-
newRuleset = { version, metadata, rules }
|
|
74
|
+
newRuleset = { version, metadata, rules, processors, scanners }
|
|
75
75
|
newRulesetId = id
|
|
76
76
|
}
|
|
77
77
|
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
const request = require('../../exporters/common/request')
|
|
2
|
+
const id = require('../../id')
|
|
3
|
+
const log = require('../../log')
|
|
4
|
+
|
|
5
|
+
function getKnownTests ({
|
|
6
|
+
url,
|
|
7
|
+
isEvpProxy,
|
|
8
|
+
evpProxyPrefix,
|
|
9
|
+
isGzipCompatible,
|
|
10
|
+
env,
|
|
11
|
+
service,
|
|
12
|
+
repositoryUrl,
|
|
13
|
+
sha,
|
|
14
|
+
osVersion,
|
|
15
|
+
osPlatform,
|
|
16
|
+
osArchitecture,
|
|
17
|
+
runtimeName,
|
|
18
|
+
runtimeVersion,
|
|
19
|
+
custom
|
|
20
|
+
}, done) {
|
|
21
|
+
const options = {
|
|
22
|
+
path: '/api/v2/ci/libraries/tests',
|
|
23
|
+
method: 'POST',
|
|
24
|
+
headers: {
|
|
25
|
+
'Content-Type': 'application/json'
|
|
26
|
+
},
|
|
27
|
+
timeout: 20000,
|
|
28
|
+
url
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
if (isGzipCompatible) {
|
|
32
|
+
options.headers['accept-encoding'] = 'gzip'
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
if (isEvpProxy) {
|
|
36
|
+
options.path = `${evpProxyPrefix}/api/v2/ci/libraries/tests`
|
|
37
|
+
options.headers['X-Datadog-EVP-Subdomain'] = 'api'
|
|
38
|
+
} else {
|
|
39
|
+
const apiKey = process.env.DATADOG_API_KEY || process.env.DD_API_KEY
|
|
40
|
+
if (!apiKey) {
|
|
41
|
+
return done(new Error('Known tests were not fetched because Datadog API key is not defined.'))
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
options.headers['dd-api-key'] = apiKey
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const data = JSON.stringify({
|
|
48
|
+
data: {
|
|
49
|
+
id: id().toString(10),
|
|
50
|
+
type: 'ci_app_libraries_tests_request',
|
|
51
|
+
attributes: {
|
|
52
|
+
configurations: {
|
|
53
|
+
'os.platform': osPlatform,
|
|
54
|
+
'os.version': osVersion,
|
|
55
|
+
'os.architecture': osArchitecture,
|
|
56
|
+
'runtime.name': runtimeName,
|
|
57
|
+
'runtime.version': runtimeVersion,
|
|
58
|
+
custom
|
|
59
|
+
},
|
|
60
|
+
service,
|
|
61
|
+
env,
|
|
62
|
+
repository_url: repositoryUrl,
|
|
63
|
+
sha
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
})
|
|
67
|
+
|
|
68
|
+
request(data, options, (err, res) => {
|
|
69
|
+
if (err) {
|
|
70
|
+
done(err)
|
|
71
|
+
} else {
|
|
72
|
+
try {
|
|
73
|
+
const { data: { attributes: { test_full_names: knownTests } } } = JSON.parse(res)
|
|
74
|
+
log.debug(() => `Number of received known tests: ${Object.keys(knownTests).length}`)
|
|
75
|
+
done(null, knownTests)
|
|
76
|
+
} catch (err) {
|
|
77
|
+
done(err)
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
})
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
module.exports = { getKnownTests }
|
|
@@ -5,10 +5,23 @@ const AgentlessWriter = require('../agentless/writer')
|
|
|
5
5
|
const CoverageWriter = require('../agentless/coverage-writer')
|
|
6
6
|
const CiVisibilityExporter = require('../ci-visibility-exporter')
|
|
7
7
|
|
|
8
|
-
const
|
|
8
|
+
const AGENT_EVP_PROXY_PATH_PREFIX = '/evp_proxy/v'
|
|
9
|
+
const AGENT_EVP_PROXY_PATH_REGEX = /\/evp_proxy\/v(\d+)\/?/
|
|
9
10
|
|
|
10
|
-
function
|
|
11
|
-
|
|
11
|
+
function getLatestEvpProxyVersion (err, agentInfo) {
|
|
12
|
+
if (err) {
|
|
13
|
+
return 0
|
|
14
|
+
}
|
|
15
|
+
return agentInfo.endpoints.reduce((acc, endpoint) => {
|
|
16
|
+
if (endpoint.includes(AGENT_EVP_PROXY_PATH_PREFIX)) {
|
|
17
|
+
const version = Number(endpoint.replace(AGENT_EVP_PROXY_PATH_REGEX, '$1'))
|
|
18
|
+
if (isNaN(version)) {
|
|
19
|
+
return acc
|
|
20
|
+
}
|
|
21
|
+
return version > acc ? version : acc
|
|
22
|
+
}
|
|
23
|
+
return acc
|
|
24
|
+
}, 0)
|
|
12
25
|
}
|
|
13
26
|
|
|
14
27
|
class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
|
|
@@ -25,17 +38,22 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
|
|
|
25
38
|
|
|
26
39
|
this.getAgentInfo((err, agentInfo) => {
|
|
27
40
|
this._isInitialized = true
|
|
28
|
-
const
|
|
41
|
+
const latestEvpProxyVersion = getLatestEvpProxyVersion(err, agentInfo)
|
|
42
|
+
const isEvpCompatible = latestEvpProxyVersion >= 2
|
|
43
|
+
const isGzipCompatible = latestEvpProxyVersion >= 4
|
|
44
|
+
|
|
45
|
+
const evpProxyPrefix = `${AGENT_EVP_PROXY_PATH_PREFIX}${latestEvpProxyVersion}`
|
|
29
46
|
if (isEvpCompatible) {
|
|
30
47
|
this._isUsingEvpProxy = true
|
|
48
|
+
this.evpProxyPrefix = evpProxyPrefix
|
|
31
49
|
this._writer = new AgentlessWriter({
|
|
32
50
|
url: this._url,
|
|
33
51
|
tags,
|
|
34
|
-
evpProxyPrefix
|
|
52
|
+
evpProxyPrefix
|
|
35
53
|
})
|
|
36
54
|
this._coverageWriter = new CoverageWriter({
|
|
37
55
|
url: this._url,
|
|
38
|
-
evpProxyPrefix
|
|
56
|
+
evpProxyPrefix
|
|
39
57
|
})
|
|
40
58
|
} else {
|
|
41
59
|
this._writer = new AgentWriter({
|
|
@@ -51,6 +69,7 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
|
|
|
51
69
|
this._resolveCanUseCiVisProtocol(isEvpCompatible)
|
|
52
70
|
this.exportUncodedTraces()
|
|
53
71
|
this.exportUncodedCoverages()
|
|
72
|
+
this._isGzipCompatible = isGzipCompatible
|
|
54
73
|
})
|
|
55
74
|
}
|
|
56
75
|
|
|
@@ -21,6 +21,8 @@ class AgentlessCiVisibilityExporter extends CiVisibilityExporter {
|
|
|
21
21
|
this._coverageWriter = new CoverageWriter({ url: this._coverageUrl })
|
|
22
22
|
|
|
23
23
|
this._apiUrl = url || new URL(`https://api.${site}`)
|
|
24
|
+
// Agentless is always gzip compatible
|
|
25
|
+
this._isGzipCompatible = true
|
|
24
26
|
}
|
|
25
27
|
|
|
26
28
|
setUrl (url, coverageUrl = url, apiUrl = url) {
|
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
const URL = require('url').URL
|
|
4
4
|
|
|
5
5
|
const { sendGitMetadata: sendGitMetadataRequest } = require('./git/git_metadata')
|
|
6
|
-
const {
|
|
6
|
+
const { getLibraryConfiguration: getLibraryConfigurationRequest } = require('../requests/get-library-configuration')
|
|
7
7
|
const { getSkippableSuites: getSkippableSuitesRequest } = require('../intelligent-test-runner/get-skippable-suites')
|
|
8
|
+
const { getKnownTests: getKnownTestsRequest } = require('../early-flake-detection/get-known-tests')
|
|
8
9
|
const log = require('../../log')
|
|
9
10
|
const AgentInfoExporter = require('../../exporters/common/agent-info-exporter')
|
|
10
11
|
|
|
@@ -76,11 +77,18 @@ class CiVisibilityExporter extends AgentInfoExporter {
|
|
|
76
77
|
shouldRequestSkippableSuites () {
|
|
77
78
|
return !!(this._config.isIntelligentTestRunnerEnabled &&
|
|
78
79
|
this._canUseCiVisProtocol &&
|
|
79
|
-
this.
|
|
80
|
-
this._itrConfig.isSuitesSkippingEnabled)
|
|
80
|
+
this._libraryConfig?.isSuitesSkippingEnabled)
|
|
81
81
|
}
|
|
82
82
|
|
|
83
|
-
|
|
83
|
+
shouldRequestKnownTests () {
|
|
84
|
+
return !!(
|
|
85
|
+
this._config.isEarlyFlakeDetectionEnabled &&
|
|
86
|
+
this._canUseCiVisProtocol &&
|
|
87
|
+
this._libraryConfig?.isEarlyFlakeDetectionEnabled
|
|
88
|
+
)
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
shouldRequestLibraryConfiguration () {
|
|
84
92
|
return this._config.isIntelligentTestRunnerEnabled
|
|
85
93
|
}
|
|
86
94
|
|
|
@@ -92,6 +100,19 @@ class CiVisibilityExporter extends AgentInfoExporter {
|
|
|
92
100
|
return this._canUseCiVisProtocol
|
|
93
101
|
}
|
|
94
102
|
|
|
103
|
+
getRequestConfiguration (testConfiguration) {
|
|
104
|
+
return {
|
|
105
|
+
url: this._getApiUrl(),
|
|
106
|
+
env: this._config.env,
|
|
107
|
+
service: this._config.service,
|
|
108
|
+
isEvpProxy: !!this._isUsingEvpProxy,
|
|
109
|
+
isGzipCompatible: this._isGzipCompatible,
|
|
110
|
+
evpProxyPrefix: this.evpProxyPrefix,
|
|
111
|
+
custom: getTestConfigurationTags(this._config.tags),
|
|
112
|
+
...testConfiguration
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
95
116
|
// We can't call the skippable endpoint until git upload has finished,
|
|
96
117
|
// hence the this._gitUploadPromise.then
|
|
97
118
|
getSkippableSuites (testConfiguration, callback) {
|
|
@@ -102,68 +123,84 @@ class CiVisibilityExporter extends AgentInfoExporter {
|
|
|
102
123
|
if (gitUploadError) {
|
|
103
124
|
return callback(gitUploadError, [])
|
|
104
125
|
}
|
|
105
|
-
|
|
106
|
-
url: this._getApiUrl(),
|
|
107
|
-
site: this._config.site,
|
|
108
|
-
env: this._config.env,
|
|
109
|
-
service: this._config.service,
|
|
110
|
-
isEvpProxy: !!this._isUsingEvpProxy,
|
|
111
|
-
custom: getTestConfigurationTags(this._config.tags),
|
|
112
|
-
...testConfiguration
|
|
113
|
-
}
|
|
114
|
-
getSkippableSuitesRequest(configuration, callback)
|
|
126
|
+
getSkippableSuitesRequest(this.getRequestConfiguration(testConfiguration), callback)
|
|
115
127
|
})
|
|
116
128
|
}
|
|
117
129
|
|
|
130
|
+
getKnownTests (testConfiguration, callback) {
|
|
131
|
+
if (!this.shouldRequestKnownTests()) {
|
|
132
|
+
return callback(null)
|
|
133
|
+
}
|
|
134
|
+
getKnownTestsRequest(this.getRequestConfiguration(testConfiguration), callback)
|
|
135
|
+
}
|
|
136
|
+
|
|
118
137
|
/**
|
|
119
|
-
* We can't request
|
|
138
|
+
* We can't request library configuration until we know whether we can use the
|
|
120
139
|
* CI Visibility Protocol, hence the this._canUseCiVisProtocol promise.
|
|
121
140
|
*/
|
|
122
|
-
|
|
141
|
+
getLibraryConfiguration (testConfiguration, callback) {
|
|
123
142
|
const { repositoryUrl } = testConfiguration
|
|
124
143
|
this.sendGitMetadata(repositoryUrl)
|
|
125
|
-
if (!this.
|
|
144
|
+
if (!this.shouldRequestLibraryConfiguration()) {
|
|
126
145
|
return callback(null, {})
|
|
127
146
|
}
|
|
128
147
|
this._canUseCiVisProtocolPromise.then((canUseCiVisProtocol) => {
|
|
129
148
|
if (!canUseCiVisProtocol) {
|
|
130
149
|
return callback(null, {})
|
|
131
150
|
}
|
|
132
|
-
const configuration =
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
service: this._config.service,
|
|
136
|
-
isEvpProxy: !!this._isUsingEvpProxy,
|
|
137
|
-
custom: getTestConfigurationTags(this._config.tags),
|
|
138
|
-
...testConfiguration
|
|
139
|
-
}
|
|
140
|
-
getItrConfigurationRequest(configuration, (err, itrConfig) => {
|
|
151
|
+
const configuration = this.getRequestConfiguration(testConfiguration)
|
|
152
|
+
|
|
153
|
+
getLibraryConfigurationRequest(configuration, (err, libraryConfig) => {
|
|
141
154
|
/**
|
|
142
|
-
* **Important**: this.
|
|
143
|
-
* where the tests run in a subprocess,
|
|
155
|
+
* **Important**: this._libraryConfig remains empty in testing frameworks
|
|
156
|
+
* where the tests run in a subprocess, like Jest,
|
|
157
|
+
* because `getLibraryConfiguration` is called only once in the main process.
|
|
144
158
|
*/
|
|
145
|
-
this.
|
|
159
|
+
this._libraryConfig = this.filterConfiguration(libraryConfig)
|
|
146
160
|
|
|
147
161
|
if (err) {
|
|
148
162
|
callback(err, {})
|
|
149
|
-
} else if (
|
|
163
|
+
} else if (libraryConfig?.requireGit) {
|
|
150
164
|
// If the backend requires git, we'll wait for the upload to finish and request settings again
|
|
151
165
|
this._gitUploadPromise.then(gitUploadError => {
|
|
152
166
|
if (gitUploadError) {
|
|
153
167
|
return callback(gitUploadError, {})
|
|
154
168
|
}
|
|
155
|
-
|
|
156
|
-
this.
|
|
157
|
-
callback(err,
|
|
169
|
+
getLibraryConfigurationRequest(configuration, (err, finalLibraryConfig) => {
|
|
170
|
+
this._libraryConfig = this.filterConfiguration(finalLibraryConfig)
|
|
171
|
+
callback(err, this._libraryConfig)
|
|
158
172
|
})
|
|
159
173
|
})
|
|
160
174
|
} else {
|
|
161
|
-
callback(null,
|
|
175
|
+
callback(null, this._libraryConfig)
|
|
162
176
|
}
|
|
163
177
|
})
|
|
164
178
|
})
|
|
165
179
|
}
|
|
166
180
|
|
|
181
|
+
// Takes into account potential kill switches
|
|
182
|
+
filterConfiguration (remoteConfiguration) {
|
|
183
|
+
if (!remoteConfiguration) {
|
|
184
|
+
return {}
|
|
185
|
+
}
|
|
186
|
+
const {
|
|
187
|
+
isCodeCoverageEnabled,
|
|
188
|
+
isSuitesSkippingEnabled,
|
|
189
|
+
isItrEnabled,
|
|
190
|
+
requireGit,
|
|
191
|
+
isEarlyFlakeDetectionEnabled,
|
|
192
|
+
earlyFlakeDetectionNumRetries
|
|
193
|
+
} = remoteConfiguration
|
|
194
|
+
return {
|
|
195
|
+
isCodeCoverageEnabled,
|
|
196
|
+
isSuitesSkippingEnabled,
|
|
197
|
+
isItrEnabled,
|
|
198
|
+
requireGit,
|
|
199
|
+
isEarlyFlakeDetectionEnabled: isEarlyFlakeDetectionEnabled && this._config.isEarlyFlakeDetectionEnabled,
|
|
200
|
+
earlyFlakeDetectionNumRetries
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
167
204
|
sendGitMetadata (repositoryUrl) {
|
|
168
205
|
if (!this._config.isGitUploadEnabled) {
|
|
169
206
|
return
|
|
@@ -172,14 +209,19 @@ class CiVisibilityExporter extends AgentInfoExporter {
|
|
|
172
209
|
if (!canUseCiVisProtocol) {
|
|
173
210
|
return
|
|
174
211
|
}
|
|
175
|
-
sendGitMetadataRequest(
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
212
|
+
sendGitMetadataRequest(
|
|
213
|
+
this._getApiUrl(),
|
|
214
|
+
{ isEvpProxy: !!this._isUsingEvpProxy, evpProxyPrefix: this.evpProxyPrefix },
|
|
215
|
+
repositoryUrl,
|
|
216
|
+
(err) => {
|
|
217
|
+
if (err) {
|
|
218
|
+
log.error(`Error uploading git metadata: ${err.message}`)
|
|
219
|
+
} else {
|
|
220
|
+
log.debug('Successfully uploaded git metadata')
|
|
221
|
+
}
|
|
222
|
+
this._resolveGit(err)
|
|
180
223
|
}
|
|
181
|
-
|
|
182
|
-
})
|
|
224
|
+
)
|
|
183
225
|
})
|
|
184
226
|
}
|
|
185
227
|
|
|
@@ -60,7 +60,7 @@ function getCommonRequestOptions (url) {
|
|
|
60
60
|
* The response are the commits for which the backend already has information
|
|
61
61
|
* This response is used to know which commits can be ignored from there on
|
|
62
62
|
*/
|
|
63
|
-
function getCommitsToUpload ({ url, repositoryUrl, latestCommits, isEvpProxy }, callback) {
|
|
63
|
+
function getCommitsToUpload ({ url, repositoryUrl, latestCommits, isEvpProxy, evpProxyPrefix }, callback) {
|
|
64
64
|
const commonOptions = getCommonRequestOptions(url)
|
|
65
65
|
|
|
66
66
|
const options = {
|
|
@@ -73,7 +73,7 @@ function getCommitsToUpload ({ url, repositoryUrl, latestCommits, isEvpProxy },
|
|
|
73
73
|
}
|
|
74
74
|
|
|
75
75
|
if (isEvpProxy) {
|
|
76
|
-
options.path =
|
|
76
|
+
options.path = `${evpProxyPrefix}/api/v2/git/repository/search_commits`
|
|
77
77
|
options.headers['X-Datadog-EVP-Subdomain'] = 'api'
|
|
78
78
|
delete options.headers['dd-api-key']
|
|
79
79
|
}
|
|
@@ -122,7 +122,7 @@ function getCommitsToUpload ({ url, repositoryUrl, latestCommits, isEvpProxy },
|
|
|
122
122
|
/**
|
|
123
123
|
* This function uploads a git packfile
|
|
124
124
|
*/
|
|
125
|
-
function uploadPackFile ({ url, isEvpProxy, packFileToUpload, repositoryUrl, headCommit }, callback) {
|
|
125
|
+
function uploadPackFile ({ url, isEvpProxy, evpProxyPrefix, packFileToUpload, repositoryUrl, headCommit }, callback) {
|
|
126
126
|
const form = new FormData()
|
|
127
127
|
|
|
128
128
|
const pushedSha = JSON.stringify({
|
|
@@ -162,7 +162,7 @@ function uploadPackFile ({ url, isEvpProxy, packFileToUpload, repositoryUrl, hea
|
|
|
162
162
|
}
|
|
163
163
|
|
|
164
164
|
if (isEvpProxy) {
|
|
165
|
-
options.path =
|
|
165
|
+
options.path = `${evpProxyPrefix}/api/v2/git/repository/packfile`
|
|
166
166
|
options.headers['X-Datadog-EVP-Subdomain'] = 'api'
|
|
167
167
|
delete options.headers['dd-api-key']
|
|
168
168
|
}
|
|
@@ -187,6 +187,7 @@ function uploadPackFile ({ url, isEvpProxy, packFileToUpload, repositoryUrl, hea
|
|
|
187
187
|
function generateAndUploadPackFiles ({
|
|
188
188
|
url,
|
|
189
189
|
isEvpProxy,
|
|
190
|
+
evpProxyPrefix,
|
|
190
191
|
commitsToUpload,
|
|
191
192
|
repositoryUrl,
|
|
192
193
|
headCommit
|
|
@@ -216,6 +217,7 @@ function generateAndUploadPackFiles ({
|
|
|
216
217
|
packFileToUpload: packFilesToUpload[packFileIndex++],
|
|
217
218
|
url,
|
|
218
219
|
isEvpProxy,
|
|
220
|
+
evpProxyPrefix,
|
|
219
221
|
repositoryUrl,
|
|
220
222
|
headCommit
|
|
221
223
|
},
|
|
@@ -228,6 +230,7 @@ function generateAndUploadPackFiles ({
|
|
|
228
230
|
packFileToUpload: packFilesToUpload[packFileIndex++],
|
|
229
231
|
url,
|
|
230
232
|
isEvpProxy,
|
|
233
|
+
evpProxyPrefix,
|
|
231
234
|
repositoryUrl,
|
|
232
235
|
headCommit
|
|
233
236
|
},
|
|
@@ -238,7 +241,7 @@ function generateAndUploadPackFiles ({
|
|
|
238
241
|
/**
|
|
239
242
|
* This function uploads git metadata to CI Visibility's backend.
|
|
240
243
|
*/
|
|
241
|
-
function sendGitMetadata (url, isEvpProxy, configRepositoryUrl, callback) {
|
|
244
|
+
function sendGitMetadata (url, { isEvpProxy, evpProxyPrefix }, configRepositoryUrl, callback) {
|
|
242
245
|
let repositoryUrl = configRepositoryUrl
|
|
243
246
|
if (!repositoryUrl) {
|
|
244
247
|
repositoryUrl = getRepositoryUrl()
|
|
@@ -266,15 +269,34 @@ function sendGitMetadata (url, isEvpProxy, configRepositoryUrl, callback) {
|
|
|
266
269
|
|
|
267
270
|
// If it has already unshallowed or the clone is not shallow, we move on
|
|
268
271
|
if (hasCheckedShallow || !isShallowRepository()) {
|
|
269
|
-
return generateAndUploadPackFiles({
|
|
272
|
+
return generateAndUploadPackFiles({
|
|
273
|
+
url,
|
|
274
|
+
isEvpProxy,
|
|
275
|
+
evpProxyPrefix,
|
|
276
|
+
commitsToUpload,
|
|
277
|
+
repositoryUrl,
|
|
278
|
+
headCommit
|
|
279
|
+
}, callback)
|
|
270
280
|
}
|
|
271
281
|
// Otherwise we unshallow and get commits to upload again
|
|
272
282
|
log.debug('It is shallow clone, unshallowing...')
|
|
273
283
|
unshallowRepository()
|
|
274
|
-
getCommitsToUpload({
|
|
284
|
+
getCommitsToUpload({
|
|
285
|
+
url,
|
|
286
|
+
repositoryUrl,
|
|
287
|
+
latestCommits,
|
|
288
|
+
isEvpProxy,
|
|
289
|
+
evpProxyPrefix
|
|
290
|
+
}, getOnFinishGetCommitsToUpload(true))
|
|
275
291
|
}
|
|
276
292
|
|
|
277
|
-
getCommitsToUpload({
|
|
293
|
+
getCommitsToUpload({
|
|
294
|
+
url,
|
|
295
|
+
repositoryUrl,
|
|
296
|
+
latestCommits,
|
|
297
|
+
isEvpProxy,
|
|
298
|
+
evpProxyPrefix
|
|
299
|
+
}, getOnFinishGetCommitsToUpload(false))
|
|
278
300
|
}
|
|
279
301
|
|
|
280
302
|
module.exports = {
|
|
@@ -15,6 +15,8 @@ const {
|
|
|
15
15
|
function getSkippableSuites ({
|
|
16
16
|
url,
|
|
17
17
|
isEvpProxy,
|
|
18
|
+
evpProxyPrefix,
|
|
19
|
+
isGzipCompatible,
|
|
18
20
|
env,
|
|
19
21
|
service,
|
|
20
22
|
repositoryUrl,
|
|
@@ -37,8 +39,12 @@ function getSkippableSuites ({
|
|
|
37
39
|
url
|
|
38
40
|
}
|
|
39
41
|
|
|
42
|
+
if (isGzipCompatible) {
|
|
43
|
+
options.headers['accept-encoding'] = 'gzip'
|
|
44
|
+
}
|
|
45
|
+
|
|
40
46
|
if (isEvpProxy) {
|
|
41
|
-
options.path =
|
|
47
|
+
options.path = `${evpProxyPrefix}/api/v2/ci/tests/skippable`
|
|
42
48
|
options.headers['X-Datadog-EVP-Subdomain'] = 'api'
|
|
43
49
|
} else {
|
|
44
50
|
const apiKey = process.env.DATADOG_API_KEY || process.env.DD_API_KEY
|
|
@@ -9,11 +9,14 @@ const {
|
|
|
9
9
|
TELEMETRY_GIT_REQUESTS_SETTINGS_ERRORS,
|
|
10
10
|
TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE,
|
|
11
11
|
getErrorTypeFromStatusCode
|
|
12
|
-
} = require('
|
|
12
|
+
} = require('../telemetry')
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
const DEFAULT_EARLY_FLAKE_DETECTION_NUM_RETRIES = 2
|
|
15
|
+
|
|
16
|
+
function getLibraryConfiguration ({
|
|
15
17
|
url,
|
|
16
18
|
isEvpProxy,
|
|
19
|
+
evpProxyPrefix,
|
|
17
20
|
env,
|
|
18
21
|
service,
|
|
19
22
|
repositoryUrl,
|
|
@@ -38,7 +41,7 @@ function getItrConfiguration ({
|
|
|
38
41
|
}
|
|
39
42
|
|
|
40
43
|
if (isEvpProxy) {
|
|
41
|
-
options.path =
|
|
44
|
+
options.path = `${evpProxyPrefix}/api/v2/libraries/tests/services/setting`
|
|
42
45
|
options.headers['X-Datadog-EVP-Subdomain'] = 'api'
|
|
43
46
|
} else {
|
|
44
47
|
const apiKey = process.env.DATADOG_API_KEY || process.env.DD_API_KEY
|
|
@@ -88,12 +91,21 @@ function getItrConfiguration ({
|
|
|
88
91
|
code_coverage: isCodeCoverageEnabled,
|
|
89
92
|
tests_skipping: isSuitesSkippingEnabled,
|
|
90
93
|
itr_enabled: isItrEnabled,
|
|
91
|
-
require_git: requireGit
|
|
94
|
+
require_git: requireGit,
|
|
95
|
+
early_flake_detection: earlyFlakeDetectionConfig
|
|
92
96
|
}
|
|
93
97
|
}
|
|
94
98
|
} = JSON.parse(res)
|
|
95
99
|
|
|
96
|
-
const settings = {
|
|
100
|
+
const settings = {
|
|
101
|
+
isCodeCoverageEnabled,
|
|
102
|
+
isSuitesSkippingEnabled,
|
|
103
|
+
isItrEnabled,
|
|
104
|
+
requireGit,
|
|
105
|
+
isEarlyFlakeDetectionEnabled: earlyFlakeDetectionConfig?.enabled ?? false,
|
|
106
|
+
earlyFlakeDetectionNumRetries:
|
|
107
|
+
earlyFlakeDetectionConfig?.slow_test_retries?.['5s'] || DEFAULT_EARLY_FLAKE_DETECTION_NUM_RETRIES
|
|
108
|
+
}
|
|
97
109
|
|
|
98
110
|
log.debug(() => `Remote settings: ${JSON.stringify(settings)}`)
|
|
99
111
|
|
|
@@ -116,4 +128,4 @@ function getItrConfiguration ({
|
|
|
116
128
|
})
|
|
117
129
|
}
|
|
118
130
|
|
|
119
|
-
module.exports = {
|
|
131
|
+
module.exports = { getLibraryConfiguration }
|