dd-trace 3.47.0 → 3.48.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE-3rdparty.csv +1 -0
- package/README.md +1 -32
- package/ci/init.js +1 -4
- package/index.d.ts +21 -0
- package/package.json +6 -5
- package/packages/datadog-instrumentations/src/amqplib.js +1 -1
- package/packages/datadog-instrumentations/src/child_process.js +150 -0
- package/packages/datadog-instrumentations/src/cucumber.js +12 -12
- package/packages/datadog-instrumentations/src/express.js +20 -0
- package/packages/datadog-instrumentations/src/grpc/client.js +56 -36
- package/packages/datadog-instrumentations/src/helpers/hooks.js +2 -2
- package/packages/datadog-instrumentations/src/jest.js +147 -10
- package/packages/datadog-instrumentations/src/mocha.js +3 -3
- package/packages/datadog-instrumentations/src/mongoose.js +23 -10
- package/packages/datadog-instrumentations/src/next.js +17 -3
- package/packages/datadog-instrumentations/src/playwright.js +41 -9
- package/packages/datadog-plugin-amqplib/src/consumer.js +10 -1
- package/packages/datadog-plugin-amqplib/src/producer.js +14 -1
- package/packages/datadog-plugin-aws-sdk/src/services/kinesis.js +107 -1
- package/packages/datadog-plugin-child_process/src/index.js +91 -0
- package/packages/datadog-plugin-child_process/src/scrub-cmd-params.js +125 -0
- package/packages/datadog-plugin-cucumber/src/index.js +16 -11
- package/packages/datadog-plugin-cypress/src/plugin.js +25 -12
- package/packages/datadog-plugin-grpc/src/client.js +16 -2
- package/packages/datadog-plugin-http/src/client.js +1 -1
- package/packages/datadog-plugin-jest/src/index.js +47 -6
- package/packages/datadog-plugin-mocha/src/index.js +14 -5
- package/packages/datadog-plugin-playwright/src/index.js +19 -5
- package/packages/datadog-plugin-rhea/src/consumer.js +11 -1
- package/packages/datadog-plugin-rhea/src/producer.js +11 -0
- package/packages/dd-trace/src/appsec/addresses.js +2 -0
- package/packages/dd-trace/src/appsec/api_security_sampler.js +16 -3
- package/packages/dd-trace/src/appsec/channels.js +2 -1
- package/packages/dd-trace/src/appsec/iast/analyzers/command-injection-analyzer.js +1 -1
- package/packages/dd-trace/src/appsec/iast/analyzers/sql-injection-analyzer.js +7 -28
- package/packages/dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer.js +10 -6
- package/packages/dd-trace/src/appsec/iast/iast-plugin.js +4 -1
- package/packages/dd-trace/src/appsec/index.js +17 -2
- package/packages/dd-trace/src/appsec/rule_manager.js +2 -2
- package/packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js +83 -0
- package/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js +25 -6
- package/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js +2 -0
- package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +83 -41
- package/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js +30 -8
- package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js +7 -1
- package/packages/dd-trace/src/ci-visibility/{intelligent-test-runner/get-itr-configuration.js → requests/get-library-configuration.js} +18 -6
- package/packages/dd-trace/src/config.js +22 -9
- package/packages/dd-trace/src/datastreams/processor.js +6 -0
- package/packages/dd-trace/src/datastreams/writer.js +2 -5
- package/packages/dd-trace/src/dogstatsd.js +3 -5
- package/packages/dd-trace/src/encode/agentless-ci-visibility.js +5 -3
- package/packages/dd-trace/src/exporters/common/request.js +21 -3
- package/packages/dd-trace/src/format.js +25 -1
- package/packages/dd-trace/src/noop/span.js +1 -0
- package/packages/dd-trace/src/opentelemetry/span.js +9 -2
- package/packages/dd-trace/src/opentracing/span.js +38 -0
- package/packages/dd-trace/src/opentracing/span_context.js +12 -6
- package/packages/dd-trace/src/opentracing/tracer.js +2 -1
- package/packages/dd-trace/src/plugins/ci_plugin.js +24 -8
- package/packages/dd-trace/src/plugins/index.js +1 -0
- package/packages/dd-trace/src/plugins/util/git.js +6 -0
- package/packages/dd-trace/src/plugins/util/test.js +36 -7
- package/packages/dd-trace/src/profiling/config.js +22 -22
- package/packages/dd-trace/src/proxy.js +31 -23
- package/packages/dd-trace/src/span_processor.js +5 -1
- package/packages/dd-trace/src/telemetry/index.js +3 -0
- package/packages/datadog-instrumentations/src/child-process.js +0 -29
- package/packages/dd-trace/src/plugins/util/exec.js +0 -34
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
const ConsumerPlugin = require('../../dd-trace/src/plugins/consumer')
|
|
4
4
|
const { storage } = require('../../datadog-core')
|
|
5
|
+
const { getAmqpMessageSize, CONTEXT_PROPAGATION_KEY } = require('../../dd-trace/src/datastreams/processor')
|
|
5
6
|
|
|
6
7
|
class RheaConsumerPlugin extends ConsumerPlugin {
|
|
7
8
|
static get id () { return 'rhea' }
|
|
@@ -19,7 +20,7 @@ class RheaConsumerPlugin extends ConsumerPlugin {
|
|
|
19
20
|
const name = getResourceNameFromMessage(msgObj)
|
|
20
21
|
const childOf = extractTextMap(msgObj, this.tracer)
|
|
21
22
|
|
|
22
|
-
this.startSpan({
|
|
23
|
+
const span = this.startSpan({
|
|
23
24
|
childOf,
|
|
24
25
|
resource: name,
|
|
25
26
|
type: 'worker',
|
|
@@ -29,6 +30,15 @@ class RheaConsumerPlugin extends ConsumerPlugin {
|
|
|
29
30
|
'amqp.link.role': 'receiver'
|
|
30
31
|
}
|
|
31
32
|
})
|
|
33
|
+
|
|
34
|
+
if (this.config.dsmEnabled && msgObj.message) {
|
|
35
|
+
const payloadSize = getAmqpMessageSize(
|
|
36
|
+
{ headers: msgObj.message.delivery_annotations, content: msgObj.message.body }
|
|
37
|
+
)
|
|
38
|
+
this.tracer.decodeDataStreamsContext(msgObj.message.delivery_annotations[CONTEXT_PROPAGATION_KEY])
|
|
39
|
+
this.tracer
|
|
40
|
+
.setCheckpoint(['direction:in', `topic:${name}`, 'type:rabbitmq'], span, payloadSize)
|
|
41
|
+
}
|
|
32
42
|
}
|
|
33
43
|
}
|
|
34
44
|
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
const { CLIENT_PORT_KEY } = require('../../dd-trace/src/constants')
|
|
4
4
|
const ProducerPlugin = require('../../dd-trace/src/plugins/producer')
|
|
5
|
+
const { encodePathwayContext } = require('../../dd-trace/src/datastreams/pathway')
|
|
6
|
+
const { getAmqpMessageSize, CONTEXT_PROPAGATION_KEY } = require('../../dd-trace/src/datastreams/processor')
|
|
5
7
|
|
|
6
8
|
class RheaProducerPlugin extends ProducerPlugin {
|
|
7
9
|
static get id () { return 'rhea' }
|
|
@@ -36,6 +38,15 @@ function addDeliveryAnnotations (msg, tracer, span) {
|
|
|
36
38
|
msg.delivery_annotations = msg.delivery_annotations || {}
|
|
37
39
|
|
|
38
40
|
tracer.inject(span, 'text_map', msg.delivery_annotations)
|
|
41
|
+
|
|
42
|
+
if (tracer._config.dsmEnabled) {
|
|
43
|
+
const targetName = span.context()._tags['amqp.link.target.address']
|
|
44
|
+
const payloadSize = getAmqpMessageSize({ content: msg.body, headers: msg.delivery_annotations })
|
|
45
|
+
const dataStreamsContext = tracer
|
|
46
|
+
.setCheckpoint(['direction:out', `exchange:${targetName}`, 'type:rabbitmq'], span, payloadSize)
|
|
47
|
+
const pathwayCtx = encodePathwayContext(dataStreamsContext)
|
|
48
|
+
msg.delivery_annotations[CONTEXT_PROPAGATION_KEY] = pathwayCtx
|
|
49
|
+
}
|
|
39
50
|
}
|
|
40
51
|
}
|
|
41
52
|
|
|
@@ -15,6 +15,8 @@ module.exports = {
|
|
|
15
15
|
HTTP_INCOMING_GRAPHQL_RESOLVERS: 'graphql.server.all_resolvers',
|
|
16
16
|
HTTP_INCOMING_GRAPHQL_RESOLVER: 'graphql.server.resolver',
|
|
17
17
|
|
|
18
|
+
HTTP_OUTGOING_BODY: 'server.response.body',
|
|
19
|
+
|
|
18
20
|
HTTP_CLIENT_IP: 'http.client_ip',
|
|
19
21
|
|
|
20
22
|
USER_ID: 'usr.id',
|
|
@@ -5,6 +5,8 @@ const log = require('../log')
|
|
|
5
5
|
let enabled
|
|
6
6
|
let requestSampling
|
|
7
7
|
|
|
8
|
+
const sampledRequests = new WeakSet()
|
|
9
|
+
|
|
8
10
|
function configure ({ apiSecurity }) {
|
|
9
11
|
enabled = apiSecurity.enabled
|
|
10
12
|
setRequestSampling(apiSecurity.requestSampling)
|
|
@@ -32,17 +34,28 @@ function parseRequestSampling (requestSampling) {
|
|
|
32
34
|
return parsed
|
|
33
35
|
}
|
|
34
36
|
|
|
35
|
-
function sampleRequest () {
|
|
37
|
+
function sampleRequest (req) {
|
|
36
38
|
if (!enabled || !requestSampling) {
|
|
37
39
|
return false
|
|
38
40
|
}
|
|
39
41
|
|
|
40
|
-
|
|
42
|
+
const shouldSample = Math.random() <= requestSampling
|
|
43
|
+
|
|
44
|
+
if (shouldSample) {
|
|
45
|
+
sampledRequests.add(req)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
return shouldSample
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function isSampled (req) {
|
|
52
|
+
return sampledRequests.has(req)
|
|
41
53
|
}
|
|
42
54
|
|
|
43
55
|
module.exports = {
|
|
44
56
|
configure,
|
|
45
57
|
disable,
|
|
46
58
|
setRequestSampling,
|
|
47
|
-
sampleRequest
|
|
59
|
+
sampleRequest,
|
|
60
|
+
isSampled
|
|
48
61
|
}
|
|
@@ -16,5 +16,6 @@ module.exports = {
|
|
|
16
16
|
queryParser: dc.channel('datadog:query:read:finish'),
|
|
17
17
|
setCookieChannel: dc.channel('datadog:iast:set-cookie'),
|
|
18
18
|
nextBodyParsed: dc.channel('apm:next:body-parsed'),
|
|
19
|
-
nextQueryParsed: dc.channel('apm:next:query-parsed')
|
|
19
|
+
nextQueryParsed: dc.channel('apm:next:query-parsed'),
|
|
20
|
+
responseBody: dc.channel('datadog:express:response:json:start')
|
|
20
21
|
}
|
|
@@ -8,7 +8,7 @@ class CommandInjectionAnalyzer extends InjectionAnalyzer {
|
|
|
8
8
|
}
|
|
9
9
|
|
|
10
10
|
onConfigure () {
|
|
11
|
-
this.addSub('datadog:child_process:execution:start', ({ command }) => this.analyze(command))
|
|
11
|
+
this.addSub('tracing:datadog:child_process:execution:start', ({ command }) => this.analyze(command))
|
|
12
12
|
}
|
|
13
13
|
}
|
|
14
14
|
|
|
@@ -4,8 +4,6 @@ const InjectionAnalyzer = require('./injection-analyzer')
|
|
|
4
4
|
const { SQL_INJECTION } = require('../vulnerabilities')
|
|
5
5
|
const { getRanges } = require('../taint-tracking/operations')
|
|
6
6
|
const { storage } = require('../../../../../datadog-core')
|
|
7
|
-
const { getIastContext } = require('../iast-context')
|
|
8
|
-
const { addVulnerability } = require('../vulnerability-reporter')
|
|
9
7
|
const { getNodeModulesPaths } = require('../path-line')
|
|
10
8
|
|
|
11
9
|
const EXCLUDED_PATHS = getNodeModulesPaths('mysql', 'mysql2', 'sequelize', 'pg-pool', 'knex')
|
|
@@ -16,9 +14,9 @@ class SqlInjectionAnalyzer extends InjectionAnalyzer {
|
|
|
16
14
|
}
|
|
17
15
|
|
|
18
16
|
onConfigure () {
|
|
19
|
-
this.addSub('apm:mysql:query:start', ({ sql }) => this.analyze(sql, 'MYSQL'))
|
|
20
|
-
this.addSub('apm:mysql2:query:start', ({ sql }) => this.analyze(sql, 'MYSQL'))
|
|
21
|
-
this.addSub('apm:pg:query:start', ({ query }) => this.analyze(query.text, 'POSTGRES'))
|
|
17
|
+
this.addSub('apm:mysql:query:start', ({ sql }) => this.analyze(sql, undefined, 'MYSQL'))
|
|
18
|
+
this.addSub('apm:mysql2:query:start', ({ sql }) => this.analyze(sql, undefined, 'MYSQL'))
|
|
19
|
+
this.addSub('apm:pg:query:start', ({ query }) => this.analyze(query.text, undefined, 'POSTGRES'))
|
|
22
20
|
|
|
23
21
|
this.addSub(
|
|
24
22
|
'datadog:sequelize:query:start',
|
|
@@ -42,7 +40,7 @@ class SqlInjectionAnalyzer extends InjectionAnalyzer {
|
|
|
42
40
|
getStoreAndAnalyze (query, dialect) {
|
|
43
41
|
const parentStore = storage.getStore()
|
|
44
42
|
if (parentStore) {
|
|
45
|
-
this.analyze(query,
|
|
43
|
+
this.analyze(query, parentStore, dialect)
|
|
46
44
|
|
|
47
45
|
storage.enterWith({ ...parentStore, sqlAnalyzed: true, sqlParentStore: parentStore })
|
|
48
46
|
}
|
|
@@ -60,29 +58,10 @@ class SqlInjectionAnalyzer extends InjectionAnalyzer {
|
|
|
60
58
|
return { value, ranges, dialect }
|
|
61
59
|
}
|
|
62
60
|
|
|
63
|
-
analyze (value,
|
|
61
|
+
analyze (value, store, dialect) {
|
|
62
|
+
store = store || storage.getStore()
|
|
64
63
|
if (!(store && store.sqlAnalyzed)) {
|
|
65
|
-
|
|
66
|
-
if (this._isInvalidContext(store, iastContext)) return
|
|
67
|
-
this._reportIfVulnerable(value, iastContext, dialect)
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
_reportIfVulnerable (value, context, dialect) {
|
|
72
|
-
if (this._isVulnerable(value, context) && this._checkOCE(context)) {
|
|
73
|
-
this._report(value, context, dialect)
|
|
74
|
-
return true
|
|
75
|
-
}
|
|
76
|
-
return false
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
_report (value, context, dialect) {
|
|
80
|
-
const evidence = this._getEvidence(value, context, dialect)
|
|
81
|
-
const location = this._getLocation()
|
|
82
|
-
if (!this._isExcluded(location)) {
|
|
83
|
-
const spanId = context && context.rootSpan && context.rootSpan.context().toSpanId()
|
|
84
|
-
const vulnerability = this._createVulnerability(this._type, evidence, spanId, location)
|
|
85
|
-
addVulnerability(context, vulnerability)
|
|
64
|
+
super.analyze(value, store, dialect)
|
|
86
65
|
}
|
|
87
66
|
}
|
|
88
67
|
|
|
@@ -22,8 +22,12 @@ class Analyzer extends SinkIastPlugin {
|
|
|
22
22
|
return false
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
-
_report (value, context) {
|
|
26
|
-
const evidence = this._getEvidence(value, context)
|
|
25
|
+
_report (value, context, meta) {
|
|
26
|
+
const evidence = this._getEvidence(value, context, meta)
|
|
27
|
+
this._reportEvidence(value, context, evidence)
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
_reportEvidence (value, context, evidence) {
|
|
27
31
|
const location = this._getLocation(value)
|
|
28
32
|
if (!this._isExcluded(location)) {
|
|
29
33
|
const locationSourceMap = this._replaceLocationFromSourceMap(location)
|
|
@@ -33,9 +37,9 @@ class Analyzer extends SinkIastPlugin {
|
|
|
33
37
|
}
|
|
34
38
|
}
|
|
35
39
|
|
|
36
|
-
_reportIfVulnerable (value, context) {
|
|
40
|
+
_reportIfVulnerable (value, context, meta) {
|
|
37
41
|
if (this._isVulnerable(value, context) && this._checkOCE(context, value)) {
|
|
38
|
-
this._report(value, context)
|
|
42
|
+
this._report(value, context, meta)
|
|
39
43
|
return true
|
|
40
44
|
}
|
|
41
45
|
return false
|
|
@@ -71,11 +75,11 @@ class Analyzer extends SinkIastPlugin {
|
|
|
71
75
|
return store && !iastContext
|
|
72
76
|
}
|
|
73
77
|
|
|
74
|
-
analyze (value, store = storage.getStore()) {
|
|
78
|
+
analyze (value, store = storage.getStore(), meta) {
|
|
75
79
|
const iastContext = getIastContext(store)
|
|
76
80
|
if (this._isInvalidContext(store, iastContext)) return
|
|
77
81
|
|
|
78
|
-
this._reportIfVulnerable(value, iastContext)
|
|
82
|
+
this._reportIfVulnerable(value, iastContext, meta)
|
|
79
83
|
}
|
|
80
84
|
|
|
81
85
|
analyzeAll (...values) {
|
|
@@ -127,10 +127,13 @@ class IastPlugin extends Plugin {
|
|
|
127
127
|
if (!channelName && !moduleName) return
|
|
128
128
|
|
|
129
129
|
if (!moduleName) {
|
|
130
|
-
|
|
130
|
+
let firstSep = channelName.indexOf(':')
|
|
131
131
|
if (firstSep === -1) {
|
|
132
132
|
moduleName = channelName
|
|
133
133
|
} else {
|
|
134
|
+
if (channelName.startsWith('tracing:')) {
|
|
135
|
+
firstSep = channelName.indexOf(':', 'tracing:'.length + 1)
|
|
136
|
+
}
|
|
134
137
|
const lastSep = channelName.indexOf(':', firstSep + 1)
|
|
135
138
|
moduleName = channelName.substring(firstSep + 1, lastSep !== -1 ? lastSep : channelName.length)
|
|
136
139
|
}
|
|
@@ -11,7 +11,8 @@ const {
|
|
|
11
11
|
passportVerify,
|
|
12
12
|
queryParser,
|
|
13
13
|
nextBodyParsed,
|
|
14
|
-
nextQueryParsed
|
|
14
|
+
nextQueryParsed,
|
|
15
|
+
responseBody
|
|
15
16
|
} = require('./channels')
|
|
16
17
|
const waf = require('./waf')
|
|
17
18
|
const addresses = require('./addresses')
|
|
@@ -53,6 +54,7 @@ function enable (_config) {
|
|
|
53
54
|
nextQueryParsed.subscribe(onRequestQueryParsed)
|
|
54
55
|
queryParser.subscribe(onRequestQueryParsed)
|
|
55
56
|
cookieParser.subscribe(onRequestCookieParser)
|
|
57
|
+
responseBody.subscribe(onResponseBody)
|
|
56
58
|
|
|
57
59
|
if (_config.appsec.eventTracking.enabled) {
|
|
58
60
|
passportVerify.subscribe(onPassportVerify)
|
|
@@ -93,7 +95,7 @@ function incomingHttpStartTranslator ({ req, res, abortController }) {
|
|
|
93
95
|
persistent[addresses.HTTP_CLIENT_IP] = clientIp
|
|
94
96
|
}
|
|
95
97
|
|
|
96
|
-
if (apiSecuritySampler.sampleRequest()) {
|
|
98
|
+
if (apiSecuritySampler.sampleRequest(req)) {
|
|
97
99
|
persistent[addresses.WAF_CONTEXT_PROCESSOR] = { 'extract-schema': true }
|
|
98
100
|
}
|
|
99
101
|
|
|
@@ -194,6 +196,18 @@ function onRequestCookieParser ({ req, res, abortController, cookies }) {
|
|
|
194
196
|
handleResults(results, req, res, rootSpan, abortController)
|
|
195
197
|
}
|
|
196
198
|
|
|
199
|
+
function onResponseBody ({ req, body }) {
|
|
200
|
+
if (!body || typeof body !== 'object') return
|
|
201
|
+
if (!apiSecuritySampler.isSampled(req)) return
|
|
202
|
+
|
|
203
|
+
// we don't support blocking at this point, so no results needed
|
|
204
|
+
waf.run({
|
|
205
|
+
persistent: {
|
|
206
|
+
[addresses.HTTP_OUTGOING_BODY]: body
|
|
207
|
+
}
|
|
208
|
+
}, req)
|
|
209
|
+
}
|
|
210
|
+
|
|
197
211
|
function onPassportVerify ({ credentials, user }) {
|
|
198
212
|
const store = storage.getStore()
|
|
199
213
|
const rootSpan = store?.req && web.root(store.req)
|
|
@@ -233,6 +247,7 @@ function disable () {
|
|
|
233
247
|
if (incomingHttpRequestEnd.hasSubscribers) incomingHttpRequestEnd.unsubscribe(incomingHttpEndTranslator)
|
|
234
248
|
if (queryParser.hasSubscribers) queryParser.unsubscribe(onRequestQueryParsed)
|
|
235
249
|
if (cookieParser.hasSubscribers) cookieParser.unsubscribe(onRequestCookieParser)
|
|
250
|
+
if (responseBody.hasSubscribers) responseBody.unsubscribe(onResponseBody)
|
|
236
251
|
if (passportVerify.hasSubscribers) passportVerify.unsubscribe(onPassportVerify)
|
|
237
252
|
}
|
|
238
253
|
|
|
@@ -69,9 +69,9 @@ function updateWafFromRC ({ toUnapply, toApply, toModify }) {
|
|
|
69
69
|
item.apply_error = 'Multiple ruleset received in ASM_DD'
|
|
70
70
|
} else {
|
|
71
71
|
if (file && file.rules && file.rules.length) {
|
|
72
|
-
const { version, metadata, rules } = file
|
|
72
|
+
const { version, metadata, rules, processors, scanners } = file
|
|
73
73
|
|
|
74
|
-
newRuleset = { version, metadata, rules }
|
|
74
|
+
newRuleset = { version, metadata, rules, processors, scanners }
|
|
75
75
|
newRulesetId = id
|
|
76
76
|
}
|
|
77
77
|
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
const request = require('../../exporters/common/request')
|
|
2
|
+
const id = require('../../id')
|
|
3
|
+
const log = require('../../log')
|
|
4
|
+
|
|
5
|
+
function getKnownTests ({
|
|
6
|
+
url,
|
|
7
|
+
isEvpProxy,
|
|
8
|
+
evpProxyPrefix,
|
|
9
|
+
isGzipCompatible,
|
|
10
|
+
env,
|
|
11
|
+
service,
|
|
12
|
+
repositoryUrl,
|
|
13
|
+
sha,
|
|
14
|
+
osVersion,
|
|
15
|
+
osPlatform,
|
|
16
|
+
osArchitecture,
|
|
17
|
+
runtimeName,
|
|
18
|
+
runtimeVersion,
|
|
19
|
+
custom
|
|
20
|
+
}, done) {
|
|
21
|
+
const options = {
|
|
22
|
+
path: '/api/v2/ci/libraries/tests',
|
|
23
|
+
method: 'POST',
|
|
24
|
+
headers: {
|
|
25
|
+
'Content-Type': 'application/json'
|
|
26
|
+
},
|
|
27
|
+
timeout: 20000,
|
|
28
|
+
url
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
if (isGzipCompatible) {
|
|
32
|
+
options.headers['accept-encoding'] = 'gzip'
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
if (isEvpProxy) {
|
|
36
|
+
options.path = `${evpProxyPrefix}/api/v2/ci/libraries/tests`
|
|
37
|
+
options.headers['X-Datadog-EVP-Subdomain'] = 'api'
|
|
38
|
+
} else {
|
|
39
|
+
const apiKey = process.env.DATADOG_API_KEY || process.env.DD_API_KEY
|
|
40
|
+
if (!apiKey) {
|
|
41
|
+
return done(new Error('Known tests were not fetched because Datadog API key is not defined.'))
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
options.headers['dd-api-key'] = apiKey
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const data = JSON.stringify({
|
|
48
|
+
data: {
|
|
49
|
+
id: id().toString(10),
|
|
50
|
+
type: 'ci_app_libraries_tests_request',
|
|
51
|
+
attributes: {
|
|
52
|
+
configurations: {
|
|
53
|
+
'os.platform': osPlatform,
|
|
54
|
+
'os.version': osVersion,
|
|
55
|
+
'os.architecture': osArchitecture,
|
|
56
|
+
'runtime.name': runtimeName,
|
|
57
|
+
'runtime.version': runtimeVersion,
|
|
58
|
+
custom
|
|
59
|
+
},
|
|
60
|
+
service,
|
|
61
|
+
env,
|
|
62
|
+
repository_url: repositoryUrl,
|
|
63
|
+
sha
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
})
|
|
67
|
+
|
|
68
|
+
request(data, options, (err, res) => {
|
|
69
|
+
if (err) {
|
|
70
|
+
done(err)
|
|
71
|
+
} else {
|
|
72
|
+
try {
|
|
73
|
+
const { data: { attributes: { test_full_names: knownTests } } } = JSON.parse(res)
|
|
74
|
+
log.debug(() => `Number of received known tests: ${Object.keys(knownTests).length}`)
|
|
75
|
+
done(null, knownTests)
|
|
76
|
+
} catch (err) {
|
|
77
|
+
done(err)
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
})
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
module.exports = { getKnownTests }
|
|
@@ -5,10 +5,23 @@ const AgentlessWriter = require('../agentless/writer')
|
|
|
5
5
|
const CoverageWriter = require('../agentless/coverage-writer')
|
|
6
6
|
const CiVisibilityExporter = require('../ci-visibility-exporter')
|
|
7
7
|
|
|
8
|
-
const
|
|
8
|
+
const AGENT_EVP_PROXY_PATH_PREFIX = '/evp_proxy/v'
|
|
9
|
+
const AGENT_EVP_PROXY_PATH_REGEX = /\/evp_proxy\/v(\d+)\/?/
|
|
9
10
|
|
|
10
|
-
function
|
|
11
|
-
|
|
11
|
+
function getLatestEvpProxyVersion (err, agentInfo) {
|
|
12
|
+
if (err) {
|
|
13
|
+
return 0
|
|
14
|
+
}
|
|
15
|
+
return agentInfo.endpoints.reduce((acc, endpoint) => {
|
|
16
|
+
if (endpoint.includes(AGENT_EVP_PROXY_PATH_PREFIX)) {
|
|
17
|
+
const version = Number(endpoint.replace(AGENT_EVP_PROXY_PATH_REGEX, '$1'))
|
|
18
|
+
if (isNaN(version)) {
|
|
19
|
+
return acc
|
|
20
|
+
}
|
|
21
|
+
return version > acc ? version : acc
|
|
22
|
+
}
|
|
23
|
+
return acc
|
|
24
|
+
}, 0)
|
|
12
25
|
}
|
|
13
26
|
|
|
14
27
|
class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
|
|
@@ -25,17 +38,22 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
|
|
|
25
38
|
|
|
26
39
|
this.getAgentInfo((err, agentInfo) => {
|
|
27
40
|
this._isInitialized = true
|
|
28
|
-
const
|
|
41
|
+
const latestEvpProxyVersion = getLatestEvpProxyVersion(err, agentInfo)
|
|
42
|
+
const isEvpCompatible = latestEvpProxyVersion >= 2
|
|
43
|
+
const isGzipCompatible = latestEvpProxyVersion >= 4
|
|
44
|
+
|
|
45
|
+
const evpProxyPrefix = `${AGENT_EVP_PROXY_PATH_PREFIX}${latestEvpProxyVersion}`
|
|
29
46
|
if (isEvpCompatible) {
|
|
30
47
|
this._isUsingEvpProxy = true
|
|
48
|
+
this.evpProxyPrefix = evpProxyPrefix
|
|
31
49
|
this._writer = new AgentlessWriter({
|
|
32
50
|
url: this._url,
|
|
33
51
|
tags,
|
|
34
|
-
evpProxyPrefix
|
|
52
|
+
evpProxyPrefix
|
|
35
53
|
})
|
|
36
54
|
this._coverageWriter = new CoverageWriter({
|
|
37
55
|
url: this._url,
|
|
38
|
-
evpProxyPrefix
|
|
56
|
+
evpProxyPrefix
|
|
39
57
|
})
|
|
40
58
|
} else {
|
|
41
59
|
this._writer = new AgentWriter({
|
|
@@ -51,6 +69,7 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
|
|
|
51
69
|
this._resolveCanUseCiVisProtocol(isEvpCompatible)
|
|
52
70
|
this.exportUncodedTraces()
|
|
53
71
|
this.exportUncodedCoverages()
|
|
72
|
+
this._isGzipCompatible = isGzipCompatible
|
|
54
73
|
})
|
|
55
74
|
}
|
|
56
75
|
|
|
@@ -21,6 +21,8 @@ class AgentlessCiVisibilityExporter extends CiVisibilityExporter {
|
|
|
21
21
|
this._coverageWriter = new CoverageWriter({ url: this._coverageUrl })
|
|
22
22
|
|
|
23
23
|
this._apiUrl = url || new URL(`https://api.${site}`)
|
|
24
|
+
// Agentless is always gzip compatible
|
|
25
|
+
this._isGzipCompatible = true
|
|
24
26
|
}
|
|
25
27
|
|
|
26
28
|
setUrl (url, coverageUrl = url, apiUrl = url) {
|
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
const URL = require('url').URL
|
|
4
4
|
|
|
5
5
|
const { sendGitMetadata: sendGitMetadataRequest } = require('./git/git_metadata')
|
|
6
|
-
const {
|
|
6
|
+
const { getLibraryConfiguration: getLibraryConfigurationRequest } = require('../requests/get-library-configuration')
|
|
7
7
|
const { getSkippableSuites: getSkippableSuitesRequest } = require('../intelligent-test-runner/get-skippable-suites')
|
|
8
|
+
const { getKnownTests: getKnownTestsRequest } = require('../early-flake-detection/get-known-tests')
|
|
8
9
|
const log = require('../../log')
|
|
9
10
|
const AgentInfoExporter = require('../../exporters/common/agent-info-exporter')
|
|
10
11
|
|
|
@@ -76,11 +77,18 @@ class CiVisibilityExporter extends AgentInfoExporter {
|
|
|
76
77
|
shouldRequestSkippableSuites () {
|
|
77
78
|
return !!(this._config.isIntelligentTestRunnerEnabled &&
|
|
78
79
|
this._canUseCiVisProtocol &&
|
|
79
|
-
this.
|
|
80
|
-
this._itrConfig.isSuitesSkippingEnabled)
|
|
80
|
+
this._libraryConfig?.isSuitesSkippingEnabled)
|
|
81
81
|
}
|
|
82
82
|
|
|
83
|
-
|
|
83
|
+
shouldRequestKnownTests () {
|
|
84
|
+
return !!(
|
|
85
|
+
this._config.isEarlyFlakeDetectionEnabled &&
|
|
86
|
+
this._canUseCiVisProtocol &&
|
|
87
|
+
this._libraryConfig?.isEarlyFlakeDetectionEnabled
|
|
88
|
+
)
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
shouldRequestLibraryConfiguration () {
|
|
84
92
|
return this._config.isIntelligentTestRunnerEnabled
|
|
85
93
|
}
|
|
86
94
|
|
|
@@ -92,6 +100,19 @@ class CiVisibilityExporter extends AgentInfoExporter {
|
|
|
92
100
|
return this._canUseCiVisProtocol
|
|
93
101
|
}
|
|
94
102
|
|
|
103
|
+
getRequestConfiguration (testConfiguration) {
|
|
104
|
+
return {
|
|
105
|
+
url: this._getApiUrl(),
|
|
106
|
+
env: this._config.env,
|
|
107
|
+
service: this._config.service,
|
|
108
|
+
isEvpProxy: !!this._isUsingEvpProxy,
|
|
109
|
+
isGzipCompatible: this._isGzipCompatible,
|
|
110
|
+
evpProxyPrefix: this.evpProxyPrefix,
|
|
111
|
+
custom: getTestConfigurationTags(this._config.tags),
|
|
112
|
+
...testConfiguration
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
95
116
|
// We can't call the skippable endpoint until git upload has finished,
|
|
96
117
|
// hence the this._gitUploadPromise.then
|
|
97
118
|
getSkippableSuites (testConfiguration, callback) {
|
|
@@ -102,68 +123,84 @@ class CiVisibilityExporter extends AgentInfoExporter {
|
|
|
102
123
|
if (gitUploadError) {
|
|
103
124
|
return callback(gitUploadError, [])
|
|
104
125
|
}
|
|
105
|
-
|
|
106
|
-
url: this._getApiUrl(),
|
|
107
|
-
site: this._config.site,
|
|
108
|
-
env: this._config.env,
|
|
109
|
-
service: this._config.service,
|
|
110
|
-
isEvpProxy: !!this._isUsingEvpProxy,
|
|
111
|
-
custom: getTestConfigurationTags(this._config.tags),
|
|
112
|
-
...testConfiguration
|
|
113
|
-
}
|
|
114
|
-
getSkippableSuitesRequest(configuration, callback)
|
|
126
|
+
getSkippableSuitesRequest(this.getRequestConfiguration(testConfiguration), callback)
|
|
115
127
|
})
|
|
116
128
|
}
|
|
117
129
|
|
|
130
|
+
getKnownTests (testConfiguration, callback) {
|
|
131
|
+
if (!this.shouldRequestKnownTests()) {
|
|
132
|
+
return callback(null)
|
|
133
|
+
}
|
|
134
|
+
getKnownTestsRequest(this.getRequestConfiguration(testConfiguration), callback)
|
|
135
|
+
}
|
|
136
|
+
|
|
118
137
|
/**
|
|
119
|
-
* We can't request
|
|
138
|
+
* We can't request library configuration until we know whether we can use the
|
|
120
139
|
* CI Visibility Protocol, hence the this._canUseCiVisProtocol promise.
|
|
121
140
|
*/
|
|
122
|
-
|
|
141
|
+
getLibraryConfiguration (testConfiguration, callback) {
|
|
123
142
|
const { repositoryUrl } = testConfiguration
|
|
124
143
|
this.sendGitMetadata(repositoryUrl)
|
|
125
|
-
if (!this.
|
|
144
|
+
if (!this.shouldRequestLibraryConfiguration()) {
|
|
126
145
|
return callback(null, {})
|
|
127
146
|
}
|
|
128
147
|
this._canUseCiVisProtocolPromise.then((canUseCiVisProtocol) => {
|
|
129
148
|
if (!canUseCiVisProtocol) {
|
|
130
149
|
return callback(null, {})
|
|
131
150
|
}
|
|
132
|
-
const configuration =
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
service: this._config.service,
|
|
136
|
-
isEvpProxy: !!this._isUsingEvpProxy,
|
|
137
|
-
custom: getTestConfigurationTags(this._config.tags),
|
|
138
|
-
...testConfiguration
|
|
139
|
-
}
|
|
140
|
-
getItrConfigurationRequest(configuration, (err, itrConfig) => {
|
|
151
|
+
const configuration = this.getRequestConfiguration(testConfiguration)
|
|
152
|
+
|
|
153
|
+
getLibraryConfigurationRequest(configuration, (err, libraryConfig) => {
|
|
141
154
|
/**
|
|
142
|
-
* **Important**: this.
|
|
143
|
-
* where the tests run in a subprocess,
|
|
155
|
+
* **Important**: this._libraryConfig remains empty in testing frameworks
|
|
156
|
+
* where the tests run in a subprocess, like Jest,
|
|
157
|
+
* because `getLibraryConfiguration` is called only once in the main process.
|
|
144
158
|
*/
|
|
145
|
-
this.
|
|
159
|
+
this._libraryConfig = this.filterConfiguration(libraryConfig)
|
|
146
160
|
|
|
147
161
|
if (err) {
|
|
148
162
|
callback(err, {})
|
|
149
|
-
} else if (
|
|
163
|
+
} else if (libraryConfig?.requireGit) {
|
|
150
164
|
// If the backend requires git, we'll wait for the upload to finish and request settings again
|
|
151
165
|
this._gitUploadPromise.then(gitUploadError => {
|
|
152
166
|
if (gitUploadError) {
|
|
153
167
|
return callback(gitUploadError, {})
|
|
154
168
|
}
|
|
155
|
-
|
|
156
|
-
this.
|
|
157
|
-
callback(err,
|
|
169
|
+
getLibraryConfigurationRequest(configuration, (err, finalLibraryConfig) => {
|
|
170
|
+
this._libraryConfig = this.filterConfiguration(finalLibraryConfig)
|
|
171
|
+
callback(err, this._libraryConfig)
|
|
158
172
|
})
|
|
159
173
|
})
|
|
160
174
|
} else {
|
|
161
|
-
callback(null,
|
|
175
|
+
callback(null, this._libraryConfig)
|
|
162
176
|
}
|
|
163
177
|
})
|
|
164
178
|
})
|
|
165
179
|
}
|
|
166
180
|
|
|
181
|
+
// Takes into account potential kill switches
|
|
182
|
+
filterConfiguration (remoteConfiguration) {
|
|
183
|
+
if (!remoteConfiguration) {
|
|
184
|
+
return {}
|
|
185
|
+
}
|
|
186
|
+
const {
|
|
187
|
+
isCodeCoverageEnabled,
|
|
188
|
+
isSuitesSkippingEnabled,
|
|
189
|
+
isItrEnabled,
|
|
190
|
+
requireGit,
|
|
191
|
+
isEarlyFlakeDetectionEnabled,
|
|
192
|
+
earlyFlakeDetectionNumRetries
|
|
193
|
+
} = remoteConfiguration
|
|
194
|
+
return {
|
|
195
|
+
isCodeCoverageEnabled,
|
|
196
|
+
isSuitesSkippingEnabled,
|
|
197
|
+
isItrEnabled,
|
|
198
|
+
requireGit,
|
|
199
|
+
isEarlyFlakeDetectionEnabled: isEarlyFlakeDetectionEnabled && this._config.isEarlyFlakeDetectionEnabled,
|
|
200
|
+
earlyFlakeDetectionNumRetries
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
167
204
|
sendGitMetadata (repositoryUrl) {
|
|
168
205
|
if (!this._config.isGitUploadEnabled) {
|
|
169
206
|
return
|
|
@@ -172,14 +209,19 @@ class CiVisibilityExporter extends AgentInfoExporter {
|
|
|
172
209
|
if (!canUseCiVisProtocol) {
|
|
173
210
|
return
|
|
174
211
|
}
|
|
175
|
-
sendGitMetadataRequest(
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
212
|
+
sendGitMetadataRequest(
|
|
213
|
+
this._getApiUrl(),
|
|
214
|
+
{ isEvpProxy: !!this._isUsingEvpProxy, evpProxyPrefix: this.evpProxyPrefix },
|
|
215
|
+
repositoryUrl,
|
|
216
|
+
(err) => {
|
|
217
|
+
if (err) {
|
|
218
|
+
log.error(`Error uploading git metadata: ${err.message}`)
|
|
219
|
+
} else {
|
|
220
|
+
log.debug('Successfully uploaded git metadata')
|
|
221
|
+
}
|
|
222
|
+
this._resolveGit(err)
|
|
180
223
|
}
|
|
181
|
-
|
|
182
|
-
})
|
|
224
|
+
)
|
|
183
225
|
})
|
|
184
226
|
}
|
|
185
227
|
|